@skyramp/mcp 0.2.7 → 0.2.150-rc.mntnc

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/build/index.js +5 -2
  2. package/build/prompts/enhance-assertions/uiAssertionsPrompt.js +24 -5
  3. package/build/prompts/sut-setup/modes/adaptWorkflowPrompt.d.ts +2 -0
  4. package/build/prompts/sut-setup/modes/adaptWorkflowPrompt.js +114 -0
  5. package/build/prompts/sut-setup/modes/dockerComposePrompt.d.ts +2 -0
  6. package/build/prompts/sut-setup/modes/dockerComposePrompt.js +292 -0
  7. package/build/prompts/sut-setup/shared.d.ts +39 -0
  8. package/build/prompts/sut-setup/shared.js +132 -0
  9. package/build/prompts/test-maintenance/driftAnalysisSections.js +9 -3
  10. package/build/prompts/test-recommendation/analysisOutputPrompt.js +1 -1
  11. package/build/prompts/testbot/testbot-prompts.d.ts +1 -1
  12. package/build/prompts/testbot/testbot-prompts.js +23 -16
  13. package/build/prompts/testbot/testbot-prompts.test.js +0 -44
  14. package/build/resources/sutSetupResource.d.ts +2 -0
  15. package/build/resources/sutSetupResource.js +45 -0
  16. package/build/resources/testbotResource.js +1 -1
  17. package/build/services/TestDiscoveryService.d.ts +2 -2
  18. package/build/services/TestDiscoveryService.js +22 -12
  19. package/build/services/TestExecutionService.d.ts +9 -0
  20. package/build/services/TestExecutionService.js +158 -26
  21. package/build/services/TestExecutionService.test.js +306 -0
  22. package/build/tools/executeSkyrampTestTool.js +17 -4
  23. package/build/tools/generate-tests/generateBatchScenarioRestTool.js +1 -1
  24. package/build/tools/submitReportTool.js +81 -35
  25. package/build/tools/submitReportTool.test.js +264 -50
  26. package/build/tools/test-management/actionsTool.js +10 -0
  27. package/build/tools/test-management/analyzeChangesTool.d.ts +0 -1
  28. package/build/tools/test-management/analyzeChangesTool.js +84 -41
  29. package/build/tools/test-management/analyzeChangesTool.test.js +4 -34
  30. package/build/tools/test-management/analyzeTestHealthTool.js +6 -3
  31. package/build/types/TestAnalysis.d.ts +2 -1
  32. package/build/types/TestExecution.d.ts +12 -2
  33. package/build/types/TestExecution.js +11 -1
  34. package/build/types/TestbotReport.d.ts +68 -0
  35. package/build/types/TestbotReport.js +1 -0
  36. package/build/types/index.d.ts +4 -0
  37. package/build/types/index.js +3 -0
  38. package/build/utils/AnalysisStateManager.js +10 -1
  39. package/build/utils/routeParsers.d.ts +0 -10
  40. package/build/utils/routeParsers.js +31 -38
  41. package/build/utils/scenarioDrafting.d.ts +1 -1
  42. package/build/utils/scenarioDrafting.js +23 -1
  43. package/build/utils/utils.d.ts +7 -1
  44. package/build/utils/utils.js +10 -0
  45. package/node_modules/playwright/LICENSE +202 -0
  46. package/node_modules/playwright/NOTICE +5 -0
  47. package/node_modules/playwright/README.md +168 -0
  48. package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +13 -9
  49. package/node_modules/playwright/lib/mcp/test/skyRampExport.js +4 -3
  50. package/node_modules/playwright/node_modules/playwright-core/LICENSE +202 -0
  51. package/node_modules/playwright/node_modules/playwright-core/NOTICE +5 -0
  52. package/node_modules/playwright/node_modules/playwright-core/ThirdPartyNotices.txt +23 -126
  53. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{codeMirrorModule-B0JOjboO.js → codeMirrorModule-B5kqh2EV.js} +1 -1
  54. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{defaultSettingsView-1anWeyDf.js → defaultSettingsView-CZ9npQ3N.js} +79 -79
  55. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{index.Cc5029a3.js → index.BCnxj-_b.js} +1 -1
  56. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.html +2 -2
  57. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{uiMode.Wo5yvvVh.js → uiMode.1Ym0Ivn8.js} +1 -1
  58. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.html +2 -2
  59. package/node_modules/playwright/node_modules/playwright-core/package.json +1 -1
  60. package/node_modules/playwright/package.json +1 -1
  61. package/package.json +9 -2
  62. package/node_modules/playwright/node_modules/playwright-core/.DS_Store +0 -0
  63. package/node_modules/playwright/node_modules/playwright-core/bundles/mcp/node_modules/.bin/node-which +0 -52
  64. package/node_modules/playwright/node_modules/playwright-core/bundles/utils/node_modules/.bin/is-docker +0 -5
  65. package/node_modules/playwright/node_modules/playwright-core/bundles/utils/node_modules/.bin/mime +0 -46
  66. package/node_modules/playwright/node_modules/playwright-core/bundles/utils/node_modules/.bin/yaml +0 -11
  67. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.CP3Y8Zsb.js +0 -2
@@ -24,6 +24,32 @@ import { buildAnalysisOutputText } from "../../prompts/test-recommendation/analy
24
24
  import { parseTraceFile, discoverTraceFiles, discoverPlaywrightZips, } from "../../utils/trace-parser.js";
25
25
  import { TestSource } from "../../types/TestAnalysis.js";
26
26
  import { parsePRComments } from "../../utils/pr-comment-parser.js";
27
+ // Generic module names that appear as imports in virtually every router file.
28
+ // Matching against these would extract ALL prefix= tokens from mount files,
29
+ // flooding changedResources with unrelated resources.
30
+ const GENERIC_MODULE_NAMES = new Set([
31
+ "models", "utils", "helpers", "schemas", "types", "constants",
32
+ "config", "settings", "exceptions", "errors", "base", "common",
33
+ "app", "main", "index", "server", "init", "deps", "dependencies",
34
+ ]);
35
+ function resolveMountPrefixTokens(mountFileContents, changedFiles) {
36
+ const tokens = changedFiles.flatMap((file) => {
37
+ const mod = path.basename(file).replace(/\.[^.]+$/, "").toLowerCase();
38
+ if (GENERIC_MODULE_NAMES.has(mod))
39
+ return [];
40
+ const modRe = new RegExp(`\\b${mod.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\b`);
41
+ return mountFileContents.flatMap((content) => {
42
+ if (!modRe.test(content.toLowerCase()))
43
+ return [];
44
+ const matches = [...content.matchAll(/prefix\s*=\s*["'](\/?[^"']+)/g)];
45
+ return matches.flatMap((m) => {
46
+ const token = extractResourceFromPath(m[1]);
47
+ return token !== "unknown" ? [token] : [];
48
+ });
49
+ });
50
+ });
51
+ return [...new Set(tokens)];
52
+ }
27
53
  /** Exported for testing: maps a parsed trace result to a TraceFile. */
28
54
  export function buildTraceFileEntry(tracePath, result) {
29
55
  return {
@@ -339,11 +365,6 @@ export const analyzeChangesInputSchema = {
339
365
  .number()
340
366
  .optional()
341
367
  .describe("GitHub PR number. When provided, fetches previous TestBot comments on this PR and skips re-recommending tests already suggested in earlier commits — reduces duplicate recommendations across multiple pushes to the same PR."),
342
- stateOutputFile: z
343
- .string()
344
- .refine((v) => path.isAbsolute(v), { message: "stateOutputFile must be an absolute path" })
345
- .optional()
346
- .describe("Absolute path where the state file should be written. When provided, overrides the default auto-generated temp path so the caller can locate it without log parsing."),
347
368
  testsRepoDir: z
348
369
  .string()
349
370
  .refine((v) => path.isAbsolute(v), { message: "testsRepoDir must be an absolute path" })
@@ -568,6 +589,30 @@ export function registerAnalyzeChangesTool(server) {
568
589
  // testsRepoDir is a cross-repo path override — honour it when set.
569
590
  // Otherwise always scan the full repo root.
570
591
  const testDir = params.testsRepoDir ?? undefined;
592
+ const routerMountContext = grepRouterMountingContext(params.repositoryPath);
593
+ // Read contents once — used for token extraction and LLM prompt injection.
594
+ const ROUTER_INLINE_LIMIT = 4096;
595
+ const routerMountContextContents = routerMountContext.map((f) => {
596
+ try {
597
+ const abs = path.isAbsolute(f) ? f : path.join(params.repositoryPath, f);
598
+ if (fs.statSync(abs).size > ROUTER_INLINE_LIMIT * 10)
599
+ return ""; // skip very large files
600
+ return fs.readFileSync(abs, "utf-8");
601
+ }
602
+ catch {
603
+ return "";
604
+ }
605
+ });
606
+ // Size-capped subset for LLM prompt injection.
607
+ const ROUTER_INLINE_MAX_FILES = 3;
608
+ const routerFileContents = routerMountContext
609
+ .slice(0, ROUTER_INLINE_MAX_FILES)
610
+ .flatMap((f, i) => {
611
+ const content = routerMountContextContents[i] ?? "";
612
+ if (!content || content.length > ROUTER_INLINE_LIMIT)
613
+ return [];
614
+ return [{ file: f, content: content.trimEnd() }];
615
+ });
571
616
  // Compute changedResources from classified endpoints for test discovery filtering.
572
617
  // undefined → full-repo mode (no diff context)
573
618
  // [] → PR mode, no endpoints found → skip external tests
@@ -596,14 +641,22 @@ export function registerAnalyzeChangesTool(server) {
596
641
  }
597
642
  else {
598
643
  // All endpoints resolved to "unknown" (e.g. NestJS relative paths) —
599
- // extract resource names from changed file paths instead.
600
- const candidateFiles = classifiedEndpoints.unmatchedFiles.length > 0
601
- ? classifiedEndpoints.unmatchedFiles
602
- : classifiedEndpoints.changedFiles ?? [];
603
- const fromFiles = candidateFiles
604
- .map((f) => extractResourceFromPath(f.replace(/\.[^./]+$/, "").replace(FRAMEWORK_SUFFIX_RE, "")))
605
- .filter((r, i, arr) => r !== "unknown" && arr.indexOf(r) === i);
606
- changedResources = fromFiles.length > 0 ? fromFiles : ["unknown"];
644
+ // first try to extract resource names from router mount prefixes.
645
+ const changedSourceFiles = classifiedEndpoints.changedFiles ?? [];
646
+ const mountedResources = resolveMountPrefixTokens(routerMountContextContents, changedSourceFiles);
647
+ if (mountedResources.length > 0) {
648
+ changedResources = mountedResources;
649
+ }
650
+ else {
651
+ // Fall back to extracting resource names from changed file paths.
652
+ const candidateFiles = classifiedEndpoints.unmatchedFiles.length > 0
653
+ ? classifiedEndpoints.unmatchedFiles
654
+ : classifiedEndpoints.changedFiles ?? [];
655
+ const fromFiles = candidateFiles
656
+ .map((f) => extractResourceFromPath(f.replace(/\.[^./]+$/, "").replace(FRAMEWORK_SUFFIX_RE, "")))
657
+ .filter((r, i, arr) => r !== "unknown" && arr.indexOf(r) === i);
658
+ changedResources = fromFiles.length > 0 ? fromFiles : ["unknown"];
659
+ }
607
660
  }
608
661
  }
609
662
  else if (classifiedEndpoints.unmatchedFiles.length > 0) {
@@ -622,7 +675,15 @@ export function registerAnalyzeChangesTool(server) {
622
675
  changedResources = fromFiles.length > 0 ? fromFiles : ["unknown"];
623
676
  }
624
677
  else {
625
- changedResources = [];
678
+ // No endpoints and no unmatched files — the diff touches only non-route
679
+ // files (e.g. schema, model, DTO). Extract resource tokens from the
680
+ // changed file paths so tests that import the changed symbol are still
681
+ // surfaced for drift assessment. Fall back to [] only if extraction
682
+ // yields nothing (e.g. a file with a non-resource name like utils.py).
683
+ const fromChangedFiles = (classifiedEndpoints.changedFiles ?? [])
684
+ .map((f) => extractResourceFromPath(f.replace(/\.[^./]+$/, "").replace(FRAMEWORK_SUFFIX_RE, "")))
685
+ .filter((r, i, arr) => r !== "unknown" && arr.indexOf(r) === i);
686
+ changedResources = fromChangedFiles.length > 0 ? fromChangedFiles : [];
626
687
  }
627
688
  }
628
689
  let existingTests = [];
@@ -903,10 +964,15 @@ export function registerAnalyzeChangesTool(server) {
903
964
  sourceFile: m.sourceFile ?? "",
904
965
  })))
905
966
  : [];
967
+ const removedEndpointsForDrafting = classifiedEndpoints?.removedEndpoints.flatMap((ep) => ep.methods.map((m) => ({
968
+ method: m,
969
+ path: ep.path,
970
+ sourceFile: ep.sourceFile,
971
+ }))) ?? [];
906
972
  const codeInferredScenarios = draftScenariosFromEndpoints(skeletonEndpoints, scenarioDraftSeed, wsAuthType, {
907
973
  changedEndpoints: changedEndpointsForSecurityExpansion,
908
974
  securityRelevantDiff,
909
- });
975
+ }, removedEndpointsForDrafting);
910
976
  let allDraftedScenarios = codeInferredScenarios;
911
977
  if (traceResult && traceResult.userFlows.length > 0) {
912
978
  const traceScenarios = traceResult.userFlows
@@ -1109,7 +1175,6 @@ export function registerAnalyzeChangesTool(server) {
1109
1175
  // can be persisted to the state file for downstream tools (health, drift).
1110
1176
  // Without them, analyzeTestHealth would work only off the static catalog
1111
1177
  // which has wrong paths for nested resources and unsupported frameworks.
1112
- const routerMountContext = grepRouterMountingContext(params.repositoryPath);
1113
1178
  const routeLikeUnmatchedFiles = [];
1114
1179
  for (const file of classifiedEndpoints?.unmatchedFiles ?? []) {
1115
1180
  const routeLike = SOURCE_EXTS.test(file) &&
@@ -1138,25 +1203,6 @@ export function registerAnalyzeChangesTool(server) {
1138
1203
  diffFilePath = path.join(os.tmpdir(), `skyramp-diff-${sessionId}.diff`);
1139
1204
  await fs.promises.writeFile(diffFilePath, diffData.diffContent, { encoding: "utf-8", mode: 0o600 });
1140
1205
  }
1141
- // Read router mount files server-side (size-capped) so the LLM has them
1142
- // inline and doesn't need an extra read step when no spec is available.
1143
- const ROUTER_INLINE_LIMIT = 4096; // bytes — skip files larger than ~4 KB
1144
- const ROUTER_INLINE_MAX_FILES = 3;
1145
- const routerFileContents = routerMountContext
1146
- .slice(0, ROUTER_INLINE_MAX_FILES)
1147
- .flatMap((f) => {
1148
- try {
1149
- const absPath = path.isAbsolute(f) ? f : path.join(params.repositoryPath, f);
1150
- const stat = fs.statSync(absPath);
1151
- if (stat.size > ROUTER_INLINE_LIMIT)
1152
- return [];
1153
- const content = fs.readFileSync(absPath, "utf-8").trimEnd();
1154
- return [{ file: f, content }];
1155
- }
1156
- catch {
1157
- return [];
1158
- }
1159
- });
1160
1206
  // Compute UI context from the diff's changed files using the shared
1161
1207
  // `isFrontendFile` classifier. Persisting this in the stateFile lets
1162
1208
  // skyramp_analyze_test_health and the recommendation prompt consume the
@@ -1252,17 +1298,14 @@ export function registerAnalyzeChangesTool(server) {
1252
1298
  relevantExternalTestPaths,
1253
1299
  },
1254
1300
  };
1255
- // Clean up old state files (>24 hours) before creating new one
1256
- const stateDir = params.stateOutputFile
1257
- ? path.dirname(path.resolve(params.stateOutputFile))
1258
- : undefined;
1301
+ // Clean up old state files (>24 hours) before creating new one.
1259
1302
  try {
1260
- await StateManager.cleanupOldFiles(24, stateDir);
1303
+ await StateManager.cleanupOldFiles(24);
1261
1304
  }
1262
1305
  catch (error) {
1263
1306
  logger.warning(`Failed to cleanup old state files: ${error.message}`);
1264
1307
  }
1265
- const stateManager = new StateManager("analysis", sessionId, undefined, params.stateOutputFile);
1308
+ const stateManager = new StateManager("analysis", sessionId);
1266
1309
  await stateManager.writeData(unifiedState, {
1267
1310
  repositoryPath: params.repositoryPath,
1268
1311
  step: "analyze_changes",
@@ -99,28 +99,6 @@ describe("isSecurityRelevantDiff", () => {
99
99
  expect(isSecurityRelevantDiff("move compression middleware to server setup")).toBe(false);
100
100
  });
101
101
  });
102
- describe("analyzeChangesInputSchema — stateOutputFile validation", () => {
103
- it("accepts a valid absolute path", () => {
104
- const result = schema.safeParse({
105
- repositoryPath: "/repo",
106
- stateOutputFile: "/tmp/analyze-changes-state.json",
107
- });
108
- expect(result.success).toBe(true);
109
- });
110
- it("rejects a relative path for stateOutputFile", () => {
111
- // stateOutputFile must be absolute so the caller can guarantee the file location.
112
- // Relative paths are silently ambiguous and should be rejected.
113
- const result = schema.safeParse({
114
- repositoryPath: "/repo",
115
- stateOutputFile: "relative/path/state.json",
116
- });
117
- expect(result.success).toBe(false);
118
- });
119
- it("accepts absence of stateOutputFile (optional field)", () => {
120
- const result = schema.safeParse({ repositoryPath: "/repo" });
121
- expect(result.success).toBe(true);
122
- });
123
- });
124
102
  describe("automatic old files cleanup", () => {
125
103
  let cleanupOldFilesSpy;
126
104
  beforeEach(() => {
@@ -128,31 +106,23 @@ describe("automatic old files cleanup", () => {
128
106
  cleanupOldFilesSpy = jest.fn().mockResolvedValue(0);
129
107
  StateManager.cleanupOldFiles = cleanupOldFilesSpy;
130
108
  });
131
- it("calls cleanupOldFiles with default temp dir when no stateOutputFile provided", async () => {
132
- // This test verifies the cleanup call is made without stateDir when using default location
109
+ it("calls cleanupOldFiles with default temp dir", async () => {
133
110
  expect(StateManager.cleanupOldFiles).toBeDefined();
134
111
  await StateManager.cleanupOldFiles(24, undefined);
135
112
  expect(cleanupOldFilesSpy).toHaveBeenCalledWith(24, undefined);
136
113
  });
137
- it("calls cleanupOldFiles with custom dir when stateOutputFile is provided", async () => {
138
- // This test verifies the cleanup call is made with the directory of stateOutputFile
114
+ it("calls cleanupOldFiles with custom dir", async () => {
139
115
  const customPath = "/custom/dir";
140
116
  await StateManager.cleanupOldFiles(24, customPath);
141
117
  expect(cleanupOldFilesSpy).toHaveBeenCalledWith(24, customPath);
142
118
  });
143
- it("calls cleanupOldFiles with empty stateTypes to restrict to diff files only", async () => {
144
- // analyzeTestHealthTool passes [] so state files (skyramp-analysis-*, skyramp-recommendation-*)
145
- // are never deleted — the caller still needs args.stateFile for skyramp_actions.
119
+ it("calls cleanupOldFiles with empty stateTypes", async () => {
146
120
  await StateManager.cleanupOldFiles(24, undefined, []);
147
121
  expect(cleanupOldFilesSpy).toHaveBeenCalledWith(24, undefined, []);
148
122
  });
149
123
  it("continues execution if cleanup fails", async () => {
150
- // Cleanup failures should not crash the analyze flow
151
124
  cleanupOldFilesSpy.mockRejectedValue(new Error("permission denied"));
152
- await expect(StateManager.cleanupOldFiles(24).catch(() => {
153
- // In the real code, this is caught and logged as a warning
154
- return Promise.resolve();
155
- })).resolves.toBeUndefined();
125
+ await expect(StateManager.cleanupOldFiles(24).catch(() => Promise.resolve())).resolves.toBeUndefined();
156
126
  });
157
127
  });
158
128
  // ─────────────────────────────────────────────────────────────────────────────
@@ -15,11 +15,11 @@ export function registerAnalyzeTestHealthTool(server) {
15
15
  idempotentHint: true,
16
16
  openWorldHint: false,
17
17
  },
18
- description: `Generate drift assessment instructions for existing tests — second step of the unified Test Health Analysis Flow. Optionally execute tests with skyramp_execute_test before calling skyramp_actions to capture beforeStatus.`,
18
+ description: `Generate drift assessment instructions for existing tests — second step of the unified Test Health Analysis Flow.`,
19
19
  inputSchema: {
20
20
  stateFile: z
21
21
  .string()
22
- .describe("Path to state file from skyramp_analyze_changes. Assessment results exist only in the LLM's reasoning context this tool does not write back to the stateFile."),
22
+ .describe("Path to the analyze-changes-state.json file returned by skyramp_analyze_changes (the stateFile field in its output). Do NOT pass summaryOutputFile or testbot-result.txt here."),
23
23
  },
24
24
  outputSchema: {
25
25
  prompt: z.string().describe("LLM drift analysis prompt and assessment instructions."),
@@ -37,7 +37,10 @@ export function registerAnalyzeTestHealthTool(server) {
37
37
  return toolError(`State file is empty or invalid: ${args.stateFile}. Call skyramp_analyze_changes first to generate a valid state file.`);
38
38
  }
39
39
  if (!repositoryPath || typeof repositoryPath !== "string") {
40
- return toolError(`repositoryPath not found in state file metadata. The state file was likely created by an older versionre-run skyramp_analyze_changes to regenerate it.`);
40
+ return toolError(`repositoryPath not found in state file metadata. The stateFile parameter must be the analyze-changes-state.json path returned by skyramp_analyze_changes not summaryOutputFile or testbot-result.txt. Re-run skyramp_analyze_changes to regenerate it.`);
41
+ }
42
+ if (!Array.isArray(stateData.existingTests)) {
43
+ return toolError(`stateFile does not contain test discovery data (existingTests missing). The stateFile parameter must be the analyze-changes-state.json path returned by skyramp_analyze_changes — not summaryOutputFile or testbot-result.txt.`);
41
44
  }
42
45
  // Skyramp tests: full drift analysis + all actions permitted.
43
46
  // Relevant external tests (user-written, relevant to this PR's endpoints): drift analysis
@@ -92,5 +92,6 @@ export interface TestAnalysisResult {
92
92
  changes: DriftChange[];
93
93
  affectedFiles: AffectedFiles;
94
94
  };
95
- execution?: TestExecutionResult;
95
+ executionBefore?: TestExecutionResult;
96
+ executionAfter?: TestExecutionResult;
96
97
  }
@@ -3,18 +3,28 @@
3
3
  *
4
4
  * These types define the structure for test execution results
5
5
  */
6
+ /** Execution outcome for a test run recorded in the maintenance report. */
7
+ export declare enum TestExecutionStatus {
8
+ Pass = "Pass",
9
+ Fail = "Fail",
10
+ Error = "Error",
11
+ /** Deliberately not executed — VERIFY/IGNORE actions or DELETE (file removed). */
12
+ Skipped = "Skipped",
13
+ /** Execution was expected (UPDATE/REGENERATE) but no result was recorded. */
14
+ Unknown = "Unknown"
15
+ }
6
16
  export interface TestExecutionData {
7
- passed: boolean;
17
+ status: TestExecutionStatus;
8
18
  executedAt: string;
9
19
  duration: number;
10
20
  errors: string[];
11
21
  warnings: string[];
12
- crashed: boolean;
13
22
  output?: string;
14
23
  exitCode?: number;
15
24
  }
16
25
  export interface TestExecutionResult extends TestExecutionData {
17
26
  testFile: string;
27
+ videoPath?: string;
18
28
  }
19
29
  export interface BatchExecutionResult {
20
30
  totalTests: number;
@@ -3,4 +3,14 @@
3
3
  *
4
4
  * These types define the structure for test execution results
5
5
  */
6
- export {};
6
+ /** Execution outcome for a test run recorded in the maintenance report. */
7
+ export var TestExecutionStatus;
8
+ (function (TestExecutionStatus) {
9
+ TestExecutionStatus["Pass"] = "Pass";
10
+ TestExecutionStatus["Fail"] = "Fail";
11
+ TestExecutionStatus["Error"] = "Error";
12
+ /** Deliberately not executed — VERIFY/IGNORE actions or DELETE (file removed). */
13
+ TestExecutionStatus["Skipped"] = "Skipped";
14
+ /** Execution was expected (UPDATE/REGENERATE) but no result was recorded. */
15
+ TestExecutionStatus["Unknown"] = "Unknown";
16
+ })(TestExecutionStatus || (TestExecutionStatus = {}));
@@ -0,0 +1,68 @@
1
+ import type { DriftAction } from "./TestAnalysis.js";
2
+ import type { TestExecutionStatus } from "./TestExecution.js";
3
+ import type { HttpMethod, TestType } from "./TestTypes.js";
4
+ /**
5
+ * Shape of the JSON report written by skyramp_submit_report and read by testbot
6
+ * for rendering as Markdown. All fields mirror the corresponding Zod schemas in
7
+ * submitReportTool.ts — keep the two in sync.
8
+ */
9
+ export interface TestbotReport {
10
+ businessCaseAnalysis: string;
11
+ newTestsCreated: {
12
+ testId: string;
13
+ testType: TestType;
14
+ endpoint: string;
15
+ fileName: string;
16
+ reasoning: string;
17
+ description?: string;
18
+ scenarioFile?: string;
19
+ traceFile?: string;
20
+ frontendTrace?: string;
21
+ }[];
22
+ /** Omitted when no existing tests were found (agent omits the field). */
23
+ testMaintenance?: {
24
+ testType: TestType;
25
+ endpoint: string;
26
+ fileName: string;
27
+ description: string;
28
+ action: DriftAction;
29
+ beforeStatus: TestExecutionStatus;
30
+ beforeDetails: string;
31
+ afterStatus: TestExecutionStatus;
32
+ afterDetails: string;
33
+ }[];
34
+ testResults: {
35
+ testType: TestType;
36
+ endpoint: string;
37
+ status: "Pass" | "Fail" | "Skipped";
38
+ details: string;
39
+ videoPath?: string;
40
+ }[];
41
+ additionalRecommendations?: {
42
+ testId: string;
43
+ testType: TestType;
44
+ category?: string;
45
+ primaryEndpoint?: string;
46
+ scenarioName?: string;
47
+ steps: {
48
+ method?: HttpMethod;
49
+ path?: string;
50
+ description: string;
51
+ expectedStatusCode?: number;
52
+ requestBody?: Record<string, unknown>;
53
+ responseBody?: Record<string, unknown>;
54
+ }[];
55
+ description: string;
56
+ priority: "high" | "medium" | "low";
57
+ reasoning: string;
58
+ openApiSpec?: string;
59
+ backendTrace?: string;
60
+ frontendTrace?: string;
61
+ }[];
62
+ issuesFound: {
63
+ description: string;
64
+ severity?: "critical" | "high" | "medium" | "low";
65
+ }[];
66
+ nextSteps: string[];
67
+ commitMessage: string;
68
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,4 @@
1
+ export { TestExecutionStatus } from "./TestExecution.js";
2
+ export { DriftAction } from "./TestAnalysis.js";
3
+ export { TestType, HttpMethod } from "./TestTypes.js";
4
+ export type { TestbotReport } from "./TestbotReport.js";
@@ -0,0 +1,3 @@
1
+ export { TestExecutionStatus } from "./TestExecution.js";
2
+ export { DriftAction } from "./TestAnalysis.js";
3
+ export { TestType, HttpMethod } from "./TestTypes.js";
@@ -119,6 +119,15 @@ export class StateManager {
119
119
  if (stateFilePath) {
120
120
  this.stateFile = stateFilePath;
121
121
  }
122
+ else if (stateType === "analysis" && process.env.RUNNER_TEMP) {
123
+ // In CI (testbot action), RUNNER_TEMP is a job-isolated dir that both the
124
+ // action and this tool can independently derive — no LLM input needed.
125
+ // TODO: interim step — remove this branch when stateFile and summaryOutputFile are merged
126
+ // (see getTestbotPrompt TODO). RUNNER_TEMP is GitHub Actions-specific; other CI systems
127
+ // (Jenkins, GitLab, Buildkite) must set it explicitly. Once merged, path flows through
128
+ // summaryOutputFile which is already testbot-controlled and CI-agnostic.
129
+ this.stateFile = path.join(process.env.RUNNER_TEMP, "skyramp", "analyze-changes-state.json");
130
+ }
122
131
  else {
123
132
  const baseDir = stateDir || os.tmpdir();
124
133
  const prefix = STATE_FILE_PREFIXES[stateType];
@@ -260,7 +269,7 @@ export class StateManager {
260
269
  * @returns Number of files deleted
261
270
  */
262
271
  static async cleanupOldFiles(maxAgeHours = 24, stateDir, stateTypes) {
263
- const baseDir = stateDir || os.tmpdir();
272
+ const baseDir = stateDir || (process.env.RUNNER_TEMP ? path.join(process.env.RUNNER_TEMP, "skyramp") : os.tmpdir());
264
273
  const files = await fs.promises.readdir(baseDir).catch(() => []);
265
274
  const statePrefixes = stateTypes
266
275
  ? stateTypes.map((t) => STATE_FILE_PREFIXES[t])
@@ -50,14 +50,4 @@ export interface ClassifiedEndpoints {
50
50
  baseBranch: string;
51
51
  affectedServices: string[];
52
52
  }
53
- /**
54
- * Classify endpoints by cross-referencing `changedFiles` against `scannedEndpoints[].sourceFile`.
55
- *
56
- * This replaces `parseEndpointsFromDiff` — instead of regex-parsing diff hunks for route
57
- * annotations (fragile for 15+ frameworks), it uses the already-scanned endpoint catalog
58
- * which has full, resolved paths and concrete HTTP methods.
59
- *
60
- * For deleted files, the caller must supply endpoints recovered from the base branch
61
- * (via `parseFileEndpoints` on `git show base:<file>`) in `deletedFileEndpoints`.
62
- */
63
53
  export declare function classifyEndpointsByChangedFiles(diffData: BranchDiffData, scannedEndpoints: ScannedEndpoint[], deletedFileEndpoints?: ScannedEndpoint[]): ClassifiedEndpoints;
@@ -406,21 +406,36 @@ const GENERIC_ROUTE_BASENAMES = new Set(["api", "app", "handler", "handlers", "i
406
406
  function normalizeSourcePath(filePath) {
407
407
  return filePath.replace(/\\/g, "/");
408
408
  }
409
- /**
410
- * Classify endpoints by cross-referencing `changedFiles` against `scannedEndpoints[].sourceFile`.
411
- *
412
- * This replaces `parseEndpointsFromDiff` — instead of regex-parsing diff hunks for route
413
- * annotations (fragile for 15+ frameworks), it uses the already-scanned endpoint catalog
414
- * which has full, resolved paths and concrete HTTP methods.
415
- *
416
- * For deleted files, the caller must supply endpoints recovered from the base branch
417
- * (via `parseFileEndpoints` on `git show base:<file>`) in `deletedFileEndpoints`.
418
- */
409
+ function staticPathSegments(urlPath) {
410
+ return urlPath.split("/").filter(s => s && !/^[:{[]/.test(s)).map(s => s.toLowerCase());
411
+ }
412
+ function endpointSourceResource(ep) {
413
+ const sourceSegments = ep.sourceFile.split(/[\\/]/).filter(Boolean);
414
+ const filename = sourceSegments.at(-1)?.replace(/\.[^.]+$/, "") ?? "";
415
+ const normalizedFilename = filename.replace(/^(?:old|new)[-_]/i, "").toLowerCase();
416
+ const isDynamic = /^\[.*\]$/.test(normalizedFilename) || /^\{.*\}$/.test(normalizedFilename);
417
+ if (!isDynamic && !GENERIC_ROUTE_BASENAMES.has(normalizedFilename) && normalizedFilename) {
418
+ return normalizedFilename;
419
+ }
420
+ else if (GENERIC_ROUTE_BASENAMES.has(normalizedFilename)) {
421
+ // Generic filename: walk up the directory tree for a more specific ancestor name.
422
+ const parentSegments = sourceSegments.slice(0, -1).reverse();
423
+ const contextSegment = parentSegments
424
+ .map(s => s.replace(/\.[^.]+$/, "").replace(/^(?:old|new)[-_]/i, "").toLowerCase())
425
+ .find(s => s && !GENERIC_ROUTE_BASENAMES.has(s) && !/^\[.*\]$/.test(s) && !/^\{.*\}$/.test(s));
426
+ if (contextSegment)
427
+ return `${contextSegment}/${normalizedFilename}`;
428
+ }
429
+ // Fall back to the last static segment of the endpoint's URL path.
430
+ const segments = staticPathSegments(ep.path);
431
+ return segments.at(-1) ?? normalizedFilename;
432
+ }
433
+ function endpointMovedKey(path, resource, method) {
434
+ return method ? `${path}::${resource}::${method}` : `${path}::${resource}`;
435
+ }
419
436
  export function classifyEndpointsByChangedFiles(diffData, scannedEndpoints, deletedFileEndpoints) {
420
437
  const newFileSet = new Set(diffData.newFiles.map(normalizeSourcePath));
421
438
  const deletedFileSet = new Set(diffData.deletedFiles.map(normalizeSourcePath));
422
- // Build sourceFile -> ScannedEndpoint[] map from the post-change catalog.
423
- // Git diffs use POSIX separators; scanner output can reflect OS-native paths.
424
439
  const bySourceFile = new Map();
425
440
  for (const ep of scannedEndpoints) {
426
441
  const sourceKey = normalizeSourcePath(ep.sourceFile);
@@ -435,7 +450,6 @@ export function classifyEndpointsByChangedFiles(diffData, scannedEndpoints, dele
435
450
  const unmatchedFiles = [];
436
451
  for (const file of diffData.changedFiles) {
437
452
  const fileKey = normalizeSourcePath(file);
438
- // Deleted files won't appear in scanned catalog — handled separately below
439
453
  if (deletedFileSet.has(fileKey))
440
454
  continue;
441
455
  const eps = bySourceFile.get(fileKey);
@@ -454,47 +468,26 @@ export function classifyEndpointsByChangedFiles(diffData, scannedEndpoints, dele
454
468
  // Filter out endpoints that still exist in the current catalog as moved/refactored.
455
469
  // Same router-relative paths can exist in sibling files, so only treat them as moved
456
470
  // when the source resource also matches.
457
- function endpointSourceResource(ep) {
458
- const sourceSegments = ep.sourceFile.split(/[\\/]/).filter(Boolean);
459
- const base = sourceSegments.at(-1)?.replace(/\.[^.]+$/, "") ?? "";
460
- const normalizedBase = base.replace(/^(?:old|new)[-_]/i, "").toLowerCase();
461
- const isDynamicBase = /^\[.*\]$/.test(normalizedBase) || /^\{.*\}$/.test(normalizedBase);
462
- if (normalizedBase && !isDynamicBase && !GENERIC_ROUTE_BASENAMES.has(normalizedBase)) {
463
- return normalizedBase;
464
- }
465
- if (normalizedBase && GENERIC_ROUTE_BASENAMES.has(normalizedBase)) {
466
- const contextSegment = [...sourceSegments]
467
- .slice(0, -1)
468
- .reverse()
469
- .map((segment) => segment.replace(/\.[^.]+$/, "").replace(/^(?:old|new)[-_]/i, "").toLowerCase())
470
- .find((segment) => segment && !GENERIC_ROUTE_BASENAMES.has(segment) && !/^\[.*\]$/.test(segment) && !/^\{.*\}$/.test(segment));
471
- if (contextSegment)
472
- return `${contextSegment}/${normalizedBase}`;
473
- }
474
- const staticSegments = ep.path.split("/").filter(Boolean).filter((segment) => !/^[:{[]/.test(segment));
475
- return staticSegments[staticSegments.length - 1]?.toLowerCase() ?? normalizedBase;
476
- }
477
471
  const currentMethodKeys = new Set();
478
472
  const currentPathResourceKeys = new Set();
479
473
  const currentMultiKeys = new Set();
480
- const movedKey = (path, resource, method) => method ? `${path}::${resource}::${method}` : `${path}::${resource}`;
481
474
  for (const current of scannedEndpoints) {
482
475
  const currentResource = endpointSourceResource(current);
483
- const pathResourceKey = movedKey(current.path, currentResource);
476
+ const pathResourceKey = endpointMovedKey(current.path, currentResource);
484
477
  currentPathResourceKeys.add(pathResourceKey);
485
478
  for (const method of current.methods) {
486
479
  if (method === "MULTI")
487
480
  currentMultiKeys.add(pathResourceKey);
488
- currentMethodKeys.add(movedKey(current.path, currentResource, method));
481
+ currentMethodKeys.add(endpointMovedKey(current.path, currentResource, method));
489
482
  }
490
483
  }
491
484
  function isMovedEndpoint(deleted, method) {
492
485
  const deletedResource = endpointSourceResource(deleted);
493
- const pathResourceKey = movedKey(deleted.path, deletedResource);
486
+ const pathResourceKey = endpointMovedKey(deleted.path, deletedResource);
494
487
  if (method === "MULTI")
495
488
  return currentPathResourceKeys.has(pathResourceKey);
496
489
  return currentMultiKeys.has(pathResourceKey) ||
497
- currentMethodKeys.has(movedKey(deleted.path, deletedResource, method));
490
+ currentMethodKeys.has(endpointMovedKey(deleted.path, deletedResource, method));
498
491
  }
499
492
  const removedEndpoints = (deletedFileEndpoints ?? [])
500
493
  .map((ep) => ({
@@ -41,7 +41,7 @@ export type ResourceRelationships = Map<string, Set<string>>;
41
41
  * the caller to fall back to heuristic pairing.
42
42
  */
43
43
  export declare function inferResourceRelationships(endpoints: Array<EndpointInput>): ResourceRelationships;
44
- export declare function draftScenariosFromEndpoints(endpoints: Array<EndpointInput>, newEndpoints?: Array<EndpointRef>, wsAuthType?: WorkspaceAuthType, options?: ScenarioDraftOptions): DraftedScenario[];
44
+ export declare function draftScenariosFromEndpoints(endpoints: Array<EndpointInput>, newEndpoints?: Array<EndpointRef>, wsAuthType?: WorkspaceAuthType, options?: ScenarioDraftOptions, removedEndpoints?: Array<EndpointRef>): DraftedScenario[];
45
45
  /**
46
46
  * Draft scenarios that directly test each new endpoint in the branch diff.
47
47
  *
@@ -142,7 +142,7 @@ export function inferResourceRelationships(endpoints) {
142
142
  }
143
143
  return relationships;
144
144
  }
145
- export function draftScenariosFromEndpoints(endpoints, newEndpoints = [], wsAuthType, options = {}) {
145
+ export function draftScenariosFromEndpoints(endpoints, newEndpoints = [], wsAuthType, options = {}, removedEndpoints = []) {
146
146
  const scenarios = [];
147
147
  const resourceGroups = new Map();
148
148
  for (const ep of endpoints) {
@@ -191,6 +191,28 @@ export function draftScenariosFromEndpoints(endpoints, newEndpoints = [], wsAuth
191
191
  }
192
192
  scenarios.push(...draftDiffDirectScenarios(newEndpoints, resourceGroups, wsAuthType));
193
193
  scenarios.push(...draftAttackSurfaceExpansionScenarios(endpoints, options.changedEndpoints ?? [], wsAuthType, options.securityRelevantDiff ?? false));
194
+ // 404-guard scenarios for removed endpoints — absence assertions confirming
195
+ // the endpoint is gone. Single-step contract tests, no chaining needed.
196
+ for (const ep of removedEndpoints) {
197
+ scenarios.push({
198
+ scenarioName: `verify-removed-${ep.method.toLowerCase()}-${ep.path.replace(/[^a-z0-9]+/gi, "-").replace(/^-|-$/g, "")}`,
199
+ description: `Verify ${ep.method} ${ep.path} returns 404 — endpoint was removed in this PR`,
200
+ category: "breaking_change",
201
+ priority: "high",
202
+ steps: [{
203
+ order: 1,
204
+ method: ep.method,
205
+ path: ep.path,
206
+ description: `${ep.method} ${ep.path} → 404 (endpoint removed)`,
207
+ interactionType: "error",
208
+ expectedStatusCode: 404,
209
+ }],
210
+ chainingKeys: [],
211
+ requiresAuth: false,
212
+ estimatedComplexity: "simple",
213
+ source: ScenarioSource.CodeInferred,
214
+ });
215
+ }
194
216
  return capScenarios(scenarios);
195
217
  }
196
218
  const MAX_TOTAL_SCENARIOS = 30;