@skyramp/mcp 0.3.3 → 0.3.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/build/playwright/registerPlaywrightTools.js +42 -1
  2. package/build/prompts/enhance-assertions/sharedAssertionRules.js +19 -0
  3. package/build/prompts/test-maintenance/actionsInstructions.js +2 -2
  4. package/build/prompts/test-recommendation/analysisOutputPrompt.js +1 -4
  5. package/build/prompts/test-recommendation/recommendationSections.d.ts +1 -1
  6. package/build/prompts/test-recommendation/recommendationSections.js +5 -5
  7. package/build/prompts/test-recommendation/test-recommendation-prompt.js +13 -7
  8. package/build/prompts/testbot/testbot-prompts.js +16 -14
  9. package/build/recommendation/discriminators.d.ts +7 -1
  10. package/build/recommendation/discriminators.js +16 -3
  11. package/build/resources/testbotResource.js +0 -1
  12. package/build/services/ScenarioGenerationService.js +5 -2
  13. package/build/services/TestExecutionService.js +25 -1
  14. package/build/services/TestGenerationService.js +24 -9
  15. package/build/services/containerEnv.d.ts +12 -1
  16. package/build/services/containerEnv.js +94 -1
  17. package/build/tools/executeSkyrampTestTool.d.ts +9 -0
  18. package/build/tools/executeSkyrampTestTool.js +20 -6
  19. package/build/tools/execution-video-state.d.ts +21 -0
  20. package/build/tools/execution-video-state.js +51 -0
  21. package/build/tools/generate-tests/generateBatchScenarioRestTool.js +31 -11
  22. package/build/tools/generate-tests/planGuard.d.ts +5 -5
  23. package/build/tools/generate-tests/planGuard.js +5 -17
  24. package/build/tools/submitReportTool.d.ts +83 -10
  25. package/build/tools/submitReportTool.js +257 -30
  26. package/build/tools/test-management/actionsTool.js +40 -39
  27. package/build/tools/test-management/analyzeChangesTool.d.ts +11 -0
  28. package/build/tools/test-management/analyzeChangesTool.js +41 -35
  29. package/build/tools/test-management/analyzeTestHealthTool.js +3 -3
  30. package/build/tools/test-management/registerTestPlanTool.js +191 -41
  31. package/build/types/TestExecution.d.ts +14 -0
  32. package/build/types/TestTypes.js +3 -2
  33. package/build/types/TestbotPromptOptions.d.ts +0 -1
  34. package/build/types/TestbotReport.d.ts +24 -1
  35. package/build/types/TestbotReport.js +10 -1
  36. package/build/types/index.d.ts +1 -0
  37. package/build/types/index.js +1 -0
  38. package/build/utils/AnalysisStateManager.d.ts +48 -2
  39. package/build/utils/AnalysisStateManager.js +49 -13
  40. package/build/utils/reportLanguage.d.ts +43 -0
  41. package/build/utils/reportLanguage.js +125 -0
  42. package/build/utils/reportVerification.d.ts +10 -0
  43. package/build/utils/reportVerification.js +31 -0
  44. package/build/utils/scenarioDrafting.js +7 -1
  45. package/build/utils/skyrampMdContent.d.ts +1 -1
  46. package/build/utils/skyrampMdContent.js +1 -1
  47. package/build/utils/urlPath.d.ts +37 -0
  48. package/build/utils/urlPath.js +55 -0
  49. package/build/utils/utils.d.ts +45 -0
  50. package/build/utils/utils.js +50 -0
  51. package/build/utils/versions.d.ts +3 -3
  52. package/build/utils/versions.js +1 -1
  53. package/build/utils/workspaceAuth.d.ts +15 -15
  54. package/build/utils/workspaceAuth.js +32 -17
  55. package/build/workspace/queryParamResolution.d.ts +93 -0
  56. package/build/workspace/queryParamResolution.js +201 -0
  57. package/build/workspace/workspace.d.ts +104 -0
  58. package/build/workspace/workspace.js +24 -0
  59. package/package.json +3 -2
@@ -1,3 +1,4 @@
1
+ import { logger } from "../utils/logger.js";
1
2
  /**
2
3
  * Rewrite localhost / 127.0.0.1 URLs to host.docker.internal so the Docker
3
4
  * executor container can reach services running on the host machine.
@@ -9,10 +10,67 @@
9
10
  export function rewriteLocalhostForDocker(url) {
10
11
  return url.replace(/^(https?:\/\/)(localhost|127\.0\.0\.1)([:/?#]|$)/, (_, scheme, _host, rest) => `${scheme}host.docker.internal${rest}`);
11
12
  }
13
+ /** Names a workspace declaration may never forward. Matching is
14
+ * case-insensitive: `process.env` is case-insensitive on Windows, so a
15
+ * lower-case declaration there returns the real value and would otherwise
16
+ * pass the checks below.
17
+ *
18
+ * Two distinct concerns share one list. The first is CI identity: testbot puts
19
+ * the workflow's GitHub token in the MCP process env so the agent's `gh` calls
20
+ * authenticate, and a declaration naming it would push it into the container
21
+ * and from there into a query string — where the SUT's own access log records
22
+ * it. The second is internal signals the executor sets for itself, which a
23
+ * declaration must not be able to spoof or override.
24
+ *
25
+ * Deliberately NOT a blanket `*_TOKEN`: a token for the customer's own SUT is
26
+ * exactly the kind of value an RBAC query param carries, and refusing it would
27
+ * break the feature this list exists to protect. */
28
+ // Names the executor sets for itself. API_KEY carries the Skyramp LLM key and
29
+ // is forwarded unconditionally by the built-in block below, so a declaration
30
+ // adds nothing. UI Smart LLM features keep working either way.
31
+ const PASSTHROUGH_RESERVED_EXACT = new Set([
32
+ "API_KEY",
33
+ "PYTEST_ADDOPTS",
34
+ "PLAYWRIGHT_SAVE_STORAGE_PATH",
35
+ ]);
36
+ const PASSTHROUGH_RESERVED_PREFIXES = ["SKYRAMP_"];
37
+ // CI identity and secret-shaped names. Testbot puts the workflow GitHub token
38
+ // in the MCP process env, and a declaration would send it to the container and
39
+ // into a query string, where the SUT access log records it.
40
+ const PASSTHROUGH_CREDENTIAL_EXACT = new Set(["GITHUB_TOKEN", "GH_TOKEN"]);
41
+ const PASSTHROUGH_CREDENTIAL_PREFIXES = ["ACTIONS_", "AWS_"];
42
+ const PASSTHROUGH_CREDENTIAL_SUBSTRINGS = ["_SECRET", "_PRIVATE_KEY"];
43
+ /** Why a name is refused, or null when it is allowed. The two reasons need
44
+ * different messages: a reserved name is already provided by the executor, so
45
+ * the declaration is only redundant, while a credential name is refused. */
46
+ export function passthroughDenyReason(name) {
47
+ const upper = name.toUpperCase();
48
+ if (PASSTHROUGH_CREDENTIAL_EXACT.has(upper) ||
49
+ PASSTHROUGH_CREDENTIAL_PREFIXES.some((p) => upper.startsWith(p)) ||
50
+ PASSTHROUGH_CREDENTIAL_SUBSTRINGS.some((s) => upper.includes(s))) {
51
+ return "credential";
52
+ }
53
+ if (PASSTHROUGH_RESERVED_EXACT.has(upper) ||
54
+ PASSTHROUGH_RESERVED_PREFIXES.some((p) => upper.startsWith(p))) {
55
+ return "reserved";
56
+ }
57
+ return null;
58
+ }
59
+ /** True when a name is refused regardless of who declared it. */
60
+ export function isDeniedPassthroughName(name) {
61
+ return passthroughDenyReason(name) !== null;
62
+ }
63
+ /** A name is interpolated directly into a Docker `Env` entry, which is a plain
64
+ * `NAME=value` string. A name holding `=`, whitespace or a newline produces a
65
+ * malformed entry and can inject a second variable. workspace.yml supplies
66
+ * these names, so `env.FOO=BAR` would yield the name `FOO=BAR`. */
67
+ export function isValidEnvVarName(name) {
68
+ return /^[A-Za-z_][A-Za-z0-9_]*$/.test(name);
69
+ }
12
70
  /**
13
71
  * Build the environment variable array for the Docker executor container.
14
72
  */
15
- export function buildContainerEnv(options, saveStoragePath, hostEnv = process.env) {
73
+ export function buildContainerEnv(options, saveStoragePath, hostEnv = process.env, passthroughNames = []) {
16
74
  const env = [
17
75
  // Omit entirely when empty so os.getenv() returns None in the test —
18
76
  // unauthenticated endpoints won't send an empty auth header (E7).
@@ -68,6 +126,41 @@ export function buildContainerEnv(options, saveStoragePath, hostEnv = process.en
68
126
  env.push(`${key}=${val}`);
69
127
  }
70
128
  }
129
+ // Environment variables the workspace's query params declare via `env.VAR`
130
+ // (SKYR-4127). These references resolve where the test RUNS — inside this
131
+ // container — so a name that does not reach it resolves to nothing, and an
132
+ // unresolved query param is dropped from the request silently, yielding a
133
+ // passing test that exercised the wrong identity.
134
+ //
135
+ // The list comes from what workspace.yml declared, never from enumerating the
136
+ // ambient environment: nothing is forwarded that the workspace did not name.
137
+ // Forwarded on presence, not truthiness, for the same reason as the
138
+ // credentials above — an empty value is a legitimate declaration.
139
+ const seen = new Set();
140
+ for (const name of passthroughNames) {
141
+ if (seen.has(name))
142
+ continue;
143
+ seen.add(name);
144
+ if (!isValidEnvVarName(name)) {
145
+ logger.warning(`Ignoring ${JSON.stringify(name)} declared in workspace.yml: it is not a valid environment variable name.`);
146
+ continue;
147
+ }
148
+ const denyReason = passthroughDenyReason(name);
149
+ if (denyReason === "reserved") {
150
+ logger.warning(`Ignoring the workspace.yml declaration of ${name}: the executor sets this variable itself. ` +
151
+ `Its value still reaches the test.`);
152
+ continue;
153
+ }
154
+ if (denyReason === "credential") {
155
+ logger.warning(`Refusing to forward ${name} to the test executor: the name is credential-shaped. ` +
156
+ `Declare a variable that carries only the value the request needs.`);
157
+ continue;
158
+ }
159
+ const val = hostEnv[name];
160
+ if (val !== undefined) {
161
+ env.push(`${name}=${val}`);
162
+ }
163
+ }
71
164
  if (hostEnv.SKYRAMP_DEBUG) {
72
165
  env.push(`SKYRAMP_DEBUG=${hostEnv.SKYRAMP_DEBUG}`);
73
166
  }
@@ -10,4 +10,13 @@ export type ContractExecutionMode = (typeof CONTRACT_EXECUTION_MODES)[number];
10
10
  */
11
11
  export declare function resolveEffectiveToken(unauthenticated?: boolean, paramToken?: string, envToken?: string): string;
12
12
  export declare function shouldInjectSkyrampBaseUrl(testType: TestType, contractMode?: ContractExecutionMode): boolean;
13
+ /**
14
+ * Append the recorded video path to execution output.
15
+ *
16
+ * Applied to the failure return as well as the success one (SKYR-4156): a video is
17
+ * most useful for a test that just failed, and surfacing it only on success meant a
18
+ * failing UI test reported no recording at all unless Playwright happened to print
19
+ * its own attachment line.
20
+ */
21
+ export declare function withVideoInfo(output: string, videoPath?: string): string;
13
22
  export declare function registerExecuteSkyrampTestTool(server: McpServer): void;
@@ -11,6 +11,7 @@ import { StateManager, getTestsRepoDir } from "../utils/AnalysisStateManager.js"
11
11
  import { TestSource } from "../types/TestAnalysis.js";
12
12
  import { logger } from "../utils/logger.js";
13
13
  import { toolError } from "../utils/utils.js";
14
+ import { recordExecutionVideo } from "./execution-video-state.js";
14
15
  const TOOL_NAME = "skyramp_execute_test";
15
16
  export const CONTRACT_EXECUTION_MODES = ["provider", "consumer"];
16
17
  /**
@@ -27,6 +28,17 @@ export function resolveEffectiveToken(unauthenticated, paramToken, envToken) {
27
28
  export function shouldInjectSkyrampBaseUrl(testType, contractMode) {
28
29
  return testType !== TestType.CONTRACT || contractMode !== "consumer";
29
30
  }
31
+ /**
32
+ * Append the recorded video path to execution output.
33
+ *
34
+ * Applied to the failure return as well as the success one (SKYR-4156): a video is
35
+ * most useful for a test that just failed, and surfacing it only on success meant a
36
+ * failing UI test reported no recording at all unless Playwright happened to print
37
+ * its own attachment line.
38
+ */
39
+ export function withVideoInfo(output, videoPath) {
40
+ return videoPath ? `${output}\n\nVideo recording: ${videoPath}` : output;
41
+ }
30
42
  export function registerExecuteSkyrampTestTool(server) {
31
43
  server.registerTool(TOOL_NAME, {
32
44
  description: `Execute a Skyramp-generated test in isolated containerized environments for reliable, deterministic testing. Call this once a test file exists on disk (from a skyramp_*_test_generation tool). First-time execution may take longer while Docker images download — this is expected, not a failure.`,
@@ -224,23 +236,25 @@ export function registerExecuteSkyrampTestTool(server) {
224
236
  // Don't fail the tool call if stateFile update fails
225
237
  }
226
238
  }
239
+ // Record the recording for the report before returning, so it happens on the
240
+ // failure path too (SKYR-4156). skyramp_submit_report reads these records to
241
+ // populate testResults[].videoPath — testbot uploads only the video
242
+ // directories the report references, so an unrecorded video is never seen.
243
+ await recordExecutionVideo(result, params.stateFile);
227
244
  // Progress is already reported by TestExecutionService
228
245
  // Only report final status if not already at 100%
229
246
  if (result.status !== TestExecutionStatus.Pass) {
230
- errorResult = toolError(`Test execution failed: ${stripVTControlCharacters(result.output || "")}
247
+ errorResult = toolError(withVideoInfo(`Test execution failed: ${stripVTControlCharacters(result.output || "")}
231
248
 
232
- If this failed with a 401 status code, ask the user for the authentication token (or where to read it from) and retry — leave the generated test file unchanged.`);
249
+ If this failed with a 401 status code, ask the user for the authentication token (or where to read it from) and retry — leave the generated test file unchanged.`, result.videoPath));
233
250
  return errorResult;
234
251
  }
235
252
  // Success - progress already reported by TestExecutionService
236
- const videoInfo = result.videoPath
237
- ? `\n\nVideo recording: ${result.videoPath}`
238
- : "";
239
253
  return {
240
254
  content: [
241
255
  {
242
256
  type: "text",
243
- text: `Test execution result: ${stripVTControlCharacters(result.output || "")}${videoInfo}`,
257
+ text: withVideoInfo(`Test execution result: ${stripVTControlCharacters(result.output || "")}`, result.videoPath),
244
258
  },
245
259
  ],
246
260
  };
@@ -0,0 +1,21 @@
1
+ import type { TestExecutionResult } from "../types/TestExecution.js";
2
+ /**
3
+ * Persistence for the server-derived video recording of a browser test execution,
4
+ * keyed by test-file basename (what `newTestsCreated[].fileName` carries, so
5
+ * `skyramp_submit_report` can match without path normalization).
6
+ *
7
+ * Why this exists (SKYR-4156): testbot uploads only the `.skyramp/videos`
8
+ * directories the submitted report actually references, so a recording nobody
9
+ * references is a recording nobody ever sees. The path is known only inside
10
+ * `skyramp_execute_test`, and asking the agent to carry it across tool calls left
11
+ * the signal prose-mediated — it decayed to zero referenced videos, and every
12
+ * recording of the run was discarded. Recording it here makes the reference
13
+ * server-derived, the same line already drawn for POM reuse and maintenance
14
+ * before/after status.
15
+ *
16
+ * Every write is best-effort. Video observability must never fail an execution
17
+ * that otherwise succeeded, so a missing/unreadable/unwritable state file degrades
18
+ * to "no video link in the report" — the same outcome as running outside a Testbot
19
+ * run, where there is no run state to record into.
20
+ */
21
+ export declare function recordExecutionVideo(result: TestExecutionResult, explicitStateFile?: string): Promise<void>;
@@ -0,0 +1,51 @@
1
+ import * as path from "path";
2
+ import { logger } from "../utils/logger.js";
3
+ import { resolveRunStatePath, StateManager, } from "../utils/AnalysisStateManager.js";
4
+ /**
5
+ * Persistence for the server-derived video recording of a browser test execution,
6
+ * keyed by test-file basename (what `newTestsCreated[].fileName` carries, so
7
+ * `skyramp_submit_report` can match without path normalization).
8
+ *
9
+ * Why this exists (SKYR-4156): testbot uploads only the `.skyramp/videos`
10
+ * directories the submitted report actually references, so a recording nobody
11
+ * references is a recording nobody ever sees. The path is known only inside
12
+ * `skyramp_execute_test`, and asking the agent to carry it across tool calls left
13
+ * the signal prose-mediated — it decayed to zero referenced videos, and every
14
+ * recording of the run was discarded. Recording it here makes the reference
15
+ * server-derived, the same line already drawn for POM reuse and maintenance
16
+ * before/after status.
17
+ *
18
+ * Every write is best-effort. Video observability must never fail an execution
19
+ * that otherwise succeeded, so a missing/unreadable/unwritable state file degrades
20
+ * to "no video link in the report" — the same outcome as running outside a Testbot
21
+ * run, where there is no run state to record into.
22
+ */
23
+ export async function recordExecutionVideo(result, explicitStateFile) {
24
+ if (!result.videoPath)
25
+ return;
26
+ const stateFile = resolveRunStatePath(explicitStateFile);
27
+ if (!stateFile)
28
+ return;
29
+ try {
30
+ const manager = StateManager.fromStatePath(stateFile);
31
+ const data = await manager.readData();
32
+ if (!data)
33
+ return;
34
+ // Last write wins: a retried execution leaves a video directory per attempt,
35
+ // and the report must reference the attempt whose status it reports.
36
+ await manager.writeData({
37
+ ...data,
38
+ executionVideos: {
39
+ ...data.executionVideos,
40
+ [path.basename(result.testFile)]: {
41
+ videoPath: result.videoPath,
42
+ status: result.status,
43
+ executedAt: result.executedAt,
44
+ },
45
+ },
46
+ });
47
+ }
48
+ catch (err) {
49
+ logger.warning("Could not record the test video — the report will omit its video link", { testFile: result.testFile, error: String(err) });
50
+ }
51
+ }
@@ -5,13 +5,16 @@ import { ScenarioGenerationService } from "../../services/ScenarioGenerationServ
5
5
  import fs from "fs";
6
6
  import { baseSchema, AUTH_PLACEHOLDER_TOKEN, HttpMethod } from "../../types/TestTypes.js";
7
7
  import { AnalyticsService } from "../../services/AnalyticsService.js";
8
- import { resolveAuthFromWorkspace, getWorkspaceDefaultQueryParams, mergeQueryParamsJson } from "../../utils/workspaceAuth.js";
8
+ import { resolveAuthFromWorkspace, getWorkspaceScopedQueryParams, mergeQueryParamsJson } from "../../utils/workspaceAuth.js";
9
+ import { resolveQueryParamsForPath, ensureLeadingSlash, contributesNothing } from "../../workspace/queryParamResolution.js";
10
+ import { deriveBasePath } from "../../utils/urlPath.js";
9
11
  import yaml from "js-yaml";
10
12
  import { logger } from "../../utils/logger.js";
11
13
  import { guardScenarioApproved } from "./planGuard.js";
12
14
  import { lintScenarioChaining } from "./scenarioLint.js";
13
15
  import { scenarioFileNameFor } from "./scenarioFileIdentity.js";
14
16
  import { toolError } from "../../utils/utils.js";
17
+ import { isRecord } from "ts-is-record";
15
18
  function isJsonValue(v) {
16
19
  if (v === undefined || v === null)
17
20
  return true;
@@ -40,8 +43,7 @@ function isJsonObject(v) {
40
43
  if (v === undefined || v === null)
41
44
  return true;
42
45
  try {
43
- const p = JSON.parse(v);
44
- return typeof p === "object" && !Array.isArray(p) && p !== null;
46
+ return isRecord(JSON.parse(v));
45
47
  }
46
48
  catch {
47
49
  return false;
@@ -83,7 +85,7 @@ Use this for GET request filters, search terms, pagination, sorting — any para
83
85
 
84
86
  CRITICAL: For search/filter/list endpoints (e.g., GET /products/search?q=bear&limit=10), parameters MUST go here, NOT in requestBody. GET request bodies are non-standard and may be ignored or rejected by servers and frameworks.
85
87
 
86
- Workspace-configured api.defaultQueryParams (if set) are merged into every step automatically — no need to repeat them here. An explicit value for the same key in a step overrides the workspace default for that step only.`),
88
+ Workspace-configured api.defaultQueryParams (if set) are merged into every step automatically, and any api.queryParamOverrides entry whose pathPattern matches that step's path is layered on top — no need to repeat them here. An explicit value for the same key in a step overrides both, for that step only.`),
87
89
  responseBody: z
88
90
  .string()
89
91
  .optional()
@@ -253,13 +255,31 @@ Call \`skyramp_integration_test_generation\` with the returned \`scenarioFile\`
253
255
  logger.warning("Could not resolve auth from workspace config");
254
256
  }
255
257
  }
256
- let defaultQueryParams;
258
+ // Workspace-declared query params (SKYR-4050, path-scoped in SKYR-4127).
259
+ // Loaded once here; resolved per step below, since each step has its own
260
+ // path and may match a different override.
261
+ let scopedQueryParams;
262
+ // The request ScenarioGenerationService actually emits is basePath +
263
+ // step.path, so pathPattern matching has to see that same composed path —
264
+ // otherwise a prefix-inclusive pattern (as the workspace.yml schema docs
265
+ // instruct authors to write) never matches a scenario step (SKYR-4127
266
+ // final review). deriveBasePath is the one definition of that prefix,
267
+ // shared with the service itself so the two cannot drift.
268
+ let basePrefix = "";
257
269
  try {
258
270
  const repoPath = params.outputDir || process.cwd();
259
- defaultQueryParams = await getWorkspaceDefaultQueryParams(repoPath);
260
- if (defaultQueryParams) {
261
- logger.info("Merged workspace defaultQueryParams into queryParams", {
262
- keys: Object.keys(defaultQueryParams),
271
+ scopedQueryParams = await getWorkspaceScopedQueryParams(repoPath);
272
+ if (scopedQueryParams && contributesNothing(scopedQueryParams)) {
273
+ // Skip the merge entirely rather than setting queryParams to "{}"
274
+ // where the old code left it untouched (Finding 4).
275
+ scopedQueryParams = undefined;
276
+ }
277
+ if (scopedQueryParams) {
278
+ basePrefix = deriveBasePath(params.baseURL);
279
+ logger.info("Loaded workspace query params for scenario steps", {
280
+ baseKeys: Object.keys(scopedQueryParams.base),
281
+ overrideCount: scopedQueryParams.overrides.length,
282
+ basePrefix,
263
283
  });
264
284
  }
265
285
  }
@@ -410,8 +430,8 @@ Call \`skyramp_integration_test_generation\` with the returned \`scenarioFile\`
410
430
  method: step.method,
411
431
  path: step.path,
412
432
  requestBody: step.requestBody,
413
- queryParams: defaultQueryParams
414
- ? mergeQueryParamsJson(step.queryParams, defaultQueryParams)
433
+ queryParams: scopedQueryParams
434
+ ? mergeQueryParamsJson(step.queryParams, resolveQueryParamsForPath(scopedQueryParams, basePrefix + ensureLeadingSlash(step.path)))
415
435
  : step.queryParams,
416
436
  responseBody: step.responseBody,
417
437
  statusCode: step.statusCode,
@@ -1,10 +1,10 @@
1
1
  import { PlanMatchQuery } from "../../utils/planMatchKeys.js";
2
2
  export type PlanGuardQuery = PlanMatchQuery;
3
- /** Extract the URL path from a generation tool's `endpointURL` param — used
4
- * for contract/direct-mode integration guard queries. Falls back to treating
5
- * the value as a bare path when it isn't a parseable absolute URL, and to
6
- * undefined (no path-based match attempted) when it's neither. */
7
- export declare function pathFromEndpointURL(endpointURL: string | undefined): string | undefined;
3
+ /** Re-exported for the generation tools that already import it from here.
4
+ * The implementation moved to utils/urlPath.ts (SKYR-4127) once workspace
5
+ * query-param resolution needed the same derivation see that module for why
6
+ * it must not be copied. */
7
+ export { pathFromEndpointURL } from "../../utils/urlPath.js";
8
8
  /**
9
9
  * Returns null when the generation call is allowed, or a human-readable error
10
10
  * message (for `toolError`) when it must be rejected because it doesn't match
@@ -7,23 +7,11 @@
7
7
  import { resolveRunStatePath, StateManager } from "../../utils/AnalysisStateManager.js";
8
8
  import { matchesApprovedPlan } from "../../utils/planMatchKeys.js";
9
9
  import { TOOL_REGISTER_TEST_PLAN } from "../../toolNames.js";
10
- /** Extract the URL path from a generation tool's `endpointURL` param — used
11
- * for contract/direct-mode integration guard queries. Falls back to treating
12
- * the value as a bare path when it isn't a parseable absolute URL, and to
13
- * undefined (no path-based match attempted) when it's neither. */
14
- export function pathFromEndpointURL(endpointURL) {
15
- if (!endpointURL)
16
- return undefined;
17
- try {
18
- // new URL() percent-encodes {param} placeholders ({id} → %7Bid%7D), while
19
- // register-time matchKeys are computed from raw scenario paths with braces
20
- // intact — restore them or parameterized endpoints never match the plan.
21
- return new URL(endpointURL).pathname.replace(/%7B/gi, "{").replace(/%7D/gi, "}");
22
- }
23
- catch {
24
- return endpointURL.startsWith("/") ? endpointURL : undefined;
25
- }
26
- }
10
+ /** Re-exported for the generation tools that already import it from here.
11
+ * The implementation moved to utils/urlPath.ts (SKYR-4127) once workspace
12
+ * query-param resolution needed the same derivation see that module for why
13
+ * it must not be copied. */
14
+ export { pathFromEndpointURL } from "../../utils/urlPath.js";
27
15
  /**
28
16
  * Returns null when the generation call is allowed, or a human-readable error
29
17
  * message (for `toolError`) when it must be rejected because it doesn't match
@@ -1,6 +1,20 @@
1
1
  import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
2
2
  import { z } from "zod";
3
3
  import { TestType, HttpMethod } from "../types/TestTypes.js";
4
+ /**
5
+ * Filename of the report, written beside the state file. SKYR-4147: the report path is
6
+ * derived here rather than accepted as a parameter. The caller builds the state file and
7
+ * the report path from a single temp dir (testbot's `skyrampTempDir()`), so the state
8
+ * file's directory IS the directory the caller reads the report back from. Taking the
9
+ * path from the agent instead meant the agent had to retype it out of a 600-character
10
+ * encoded prompt URI; one wrong character wrote the report somewhere nobody reads, and
11
+ * the tool still reported success, throwing away the whole run without a word.
12
+ *
13
+ * Must stay in sync with testbot, which hardcodes the same string in main.ts, post.ts and
14
+ * eval-entry.ts — the same duplication across the two repos that already exists for
15
+ * `RUN_STATE_FILE_NAME` / testbot's `STATE_FILE_NAME`.
16
+ */
17
+ export declare const REPORT_FILE_NAME = "testbot-result.txt";
4
18
  export declare const targetElementSchema: z.ZodObject<{
5
19
  role: z.ZodString;
6
20
  accessibleName: z.ZodString;
@@ -36,7 +50,7 @@ export declare const pageContextSchema: z.ZodObject<{
36
50
  url: string;
37
51
  pageHash?: string | undefined;
38
52
  }>;
39
- export declare const newTestSchema: z.ZodEffects<z.ZodObject<{
53
+ export declare const newTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
40
54
  testId: z.ZodString;
41
55
  testType: z.ZodNativeEnum<typeof TestType>;
42
56
  category: z.ZodEffects<z.ZodEnum<["business_rule", "security_boundary", "data_integrity", "breaking_change", "auth", "error_handling", "workflow", "data_validation", "crud"]>, "business_rule" | "security_boundary" | "data_integrity" | "breaking_change" | "auth" | "error_handling" | "workflow" | "data_validation" | "crud", unknown>;
@@ -94,7 +108,6 @@ export declare const newTestSchema: z.ZodEffects<z.ZodObject<{
94
108
  repository?: string | undefined;
95
109
  traceFile?: string | undefined;
96
110
  scenarioFile?: string | undefined;
97
- frontendTrace?: string | undefined;
98
111
  targetElements?: {
99
112
  role: string;
100
113
  accessibleName: string;
@@ -108,6 +121,7 @@ export declare const newTestSchema: z.ZodEffects<z.ZodObject<{
108
121
  url: string;
109
122
  pageHash?: string | undefined;
110
123
  } | undefined;
124
+ frontendTrace?: string | undefined;
111
125
  }, {
112
126
  description: string;
113
127
  testType: TestType;
@@ -119,7 +133,6 @@ export declare const newTestSchema: z.ZodEffects<z.ZodObject<{
119
133
  category?: unknown;
120
134
  traceFile?: string | undefined;
121
135
  scenarioFile?: string | undefined;
122
- frontendTrace?: string | undefined;
123
136
  targetElements?: {
124
137
  role: string;
125
138
  accessibleName: string;
@@ -133,6 +146,7 @@ export declare const newTestSchema: z.ZodEffects<z.ZodObject<{
133
146
  url: string;
134
147
  pageHash?: string | undefined;
135
148
  } | undefined;
149
+ frontendTrace?: string | undefined;
136
150
  }>, {
137
151
  description: string;
138
152
  testType: TestType;
@@ -144,7 +158,6 @@ export declare const newTestSchema: z.ZodEffects<z.ZodObject<{
144
158
  repository?: string | undefined;
145
159
  traceFile?: string | undefined;
146
160
  scenarioFile?: string | undefined;
147
- frontendTrace?: string | undefined;
148
161
  targetElements?: {
149
162
  role: string;
150
163
  accessibleName: string;
@@ -158,6 +171,7 @@ export declare const newTestSchema: z.ZodEffects<z.ZodObject<{
158
171
  url: string;
159
172
  pageHash?: string | undefined;
160
173
  } | undefined;
174
+ frontendTrace?: string | undefined;
161
175
  }, {
162
176
  description: string;
163
177
  testType: TestType;
@@ -169,7 +183,31 @@ export declare const newTestSchema: z.ZodEffects<z.ZodObject<{
169
183
  category?: unknown;
170
184
  traceFile?: string | undefined;
171
185
  scenarioFile?: string | undefined;
186
+ targetElements?: {
187
+ role: string;
188
+ accessibleName: string;
189
+ testId: string | null;
190
+ stableId: string | null;
191
+ contextText: string[] | null;
192
+ mutability?: "unknown" | "mutable" | "immutable" | undefined;
193
+ widgetType?: "unknown" | "custom" | "native" | undefined;
194
+ }[] | null | undefined;
195
+ pageContext?: {
196
+ url: string;
197
+ pageHash?: string | undefined;
198
+ } | undefined;
172
199
  frontendTrace?: string | undefined;
200
+ }>, {
201
+ description: string;
202
+ testType: TestType;
203
+ category: "business_rule" | "security_boundary" | "data_integrity" | "breaking_change" | "auth" | "error_handling" | "workflow" | "data_validation" | "crud";
204
+ fileName: string;
205
+ endpoint: string;
206
+ testId: string;
207
+ reasoning: string;
208
+ repository?: string | undefined;
209
+ traceFile?: string | undefined;
210
+ scenarioFile?: string | undefined;
173
211
  targetElements?: {
174
212
  role: string;
175
213
  accessibleName: string;
@@ -183,8 +221,9 @@ export declare const newTestSchema: z.ZodEffects<z.ZodObject<{
183
221
  url: string;
184
222
  pageHash?: string | undefined;
185
223
  } | undefined;
186
- }>;
187
- export declare const additionalRecommendationSchema: z.ZodEffects<z.ZodObject<{
224
+ frontendTrace?: string | undefined;
225
+ }, unknown>;
226
+ export declare const additionalRecommendationSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
188
227
  testId: z.ZodString;
189
228
  testType: z.ZodNativeEnum<typeof TestType>;
190
229
  category: z.ZodEffects<z.ZodEnum<["business_rule", "security_boundary", "data_integrity", "breaking_change", "auth", "error_handling", "workflow", "data_validation", "crud"]>, "business_rule" | "security_boundary" | "data_integrity" | "breaking_change" | "auth" | "error_handling" | "workflow" | "data_validation" | "crud", unknown>;
@@ -271,7 +310,6 @@ export declare const additionalRecommendationSchema: z.ZodEffects<z.ZodObject<{
271
310
  reasoning: string;
272
311
  repository?: string | undefined;
273
312
  scenarioName?: string | undefined;
274
- frontendTrace?: string | undefined;
275
313
  targetElements?: {
276
314
  role: string;
277
315
  accessibleName: string;
@@ -285,6 +323,7 @@ export declare const additionalRecommendationSchema: z.ZodEffects<z.ZodObject<{
285
323
  url: string;
286
324
  pageHash?: string | undefined;
287
325
  } | undefined;
326
+ frontendTrace?: string | undefined;
288
327
  primaryEndpoint?: string | undefined;
289
328
  openApiSpec?: string | undefined;
290
329
  backendTrace?: string | undefined;
@@ -305,7 +344,6 @@ export declare const additionalRecommendationSchema: z.ZodEffects<z.ZodObject<{
305
344
  repository?: string | undefined;
306
345
  scenarioName?: string | undefined;
307
346
  category?: unknown;
308
- frontendTrace?: string | undefined;
309
347
  targetElements?: {
310
348
  role: string;
311
349
  accessibleName: string;
@@ -319,6 +357,7 @@ export declare const additionalRecommendationSchema: z.ZodEffects<z.ZodObject<{
319
357
  url: string;
320
358
  pageHash?: string | undefined;
321
359
  } | undefined;
360
+ frontendTrace?: string | undefined;
322
361
  primaryEndpoint?: string | undefined;
323
362
  openApiSpec?: string | undefined;
324
363
  backendTrace?: string | undefined;
@@ -339,7 +378,6 @@ export declare const additionalRecommendationSchema: z.ZodEffects<z.ZodObject<{
339
378
  reasoning: string;
340
379
  repository?: string | undefined;
341
380
  scenarioName?: string | undefined;
342
- frontendTrace?: string | undefined;
343
381
  targetElements?: {
344
382
  role: string;
345
383
  accessibleName: string;
@@ -353,6 +391,7 @@ export declare const additionalRecommendationSchema: z.ZodEffects<z.ZodObject<{
353
391
  url: string;
354
392
  pageHash?: string | undefined;
355
393
  } | undefined;
394
+ frontendTrace?: string | undefined;
356
395
  primaryEndpoint?: string | undefined;
357
396
  openApiSpec?: string | undefined;
358
397
  backendTrace?: string | undefined;
@@ -373,7 +412,40 @@ export declare const additionalRecommendationSchema: z.ZodEffects<z.ZodObject<{
373
412
  repository?: string | undefined;
374
413
  scenarioName?: string | undefined;
375
414
  category?: unknown;
415
+ targetElements?: {
416
+ role: string;
417
+ accessibleName: string;
418
+ testId: string | null;
419
+ stableId: string | null;
420
+ contextText: string[] | null;
421
+ mutability?: "unknown" | "mutable" | "immutable" | undefined;
422
+ widgetType?: "unknown" | "custom" | "native" | undefined;
423
+ }[] | null | undefined;
424
+ pageContext?: {
425
+ url: string;
426
+ pageHash?: string | undefined;
427
+ } | undefined;
376
428
  frontendTrace?: string | undefined;
429
+ primaryEndpoint?: string | undefined;
430
+ openApiSpec?: string | undefined;
431
+ backendTrace?: string | undefined;
432
+ }>, {
433
+ description: string;
434
+ priority: "high" | "medium" | "low";
435
+ testType: TestType;
436
+ category: "business_rule" | "security_boundary" | "data_integrity" | "breaking_change" | "auth" | "error_handling" | "workflow" | "data_validation" | "crud";
437
+ steps: {
438
+ description: string;
439
+ path?: string | undefined;
440
+ method?: HttpMethod | undefined;
441
+ requestBody?: Record<string, any> | undefined;
442
+ responseBody?: Record<string, any> | undefined;
443
+ expectedStatusCode?: number | undefined;
444
+ }[];
445
+ testId: string;
446
+ reasoning: string;
447
+ repository?: string | undefined;
448
+ scenarioName?: string | undefined;
377
449
  targetElements?: {
378
450
  role: string;
379
451
  accessibleName: string;
@@ -387,8 +459,9 @@ export declare const additionalRecommendationSchema: z.ZodEffects<z.ZodObject<{
387
459
  url: string;
388
460
  pageHash?: string | undefined;
389
461
  } | undefined;
462
+ frontendTrace?: string | undefined;
390
463
  primaryEndpoint?: string | undefined;
391
464
  openApiSpec?: string | undefined;
392
465
  backendTrace?: string | undefined;
393
- }>;
466
+ }, unknown>;
394
467
  export declare function registerSubmitReportTool(server: McpServer): void;