@skyramp/mcp 0.3.5 → 0.3.6-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/build/playwright/registerPlaywrightTools.js +92 -30
  2. package/build/playwright/traceRecordingPrompt.d.ts +6 -0
  3. package/build/playwright/traceRecordingPrompt.js +6 -2
  4. package/build/prompts/code-reuse.d.ts +1 -2
  5. package/build/prompts/code-reuse.js +182 -77
  6. package/build/prompts/modularization/integration-test-modularization.d.ts +2 -0
  7. package/build/prompts/modularization/integration-test-modularization.js +83 -41
  8. package/build/prompts/modularization/render.d.ts +18 -0
  9. package/build/prompts/modularization/render.js +12 -0
  10. package/build/prompts/modularization/ui-test-modularization.d.ts +3 -1
  11. package/build/prompts/modularization/ui-test-modularization.js +89 -47
  12. package/build/prompts/pom-aware-code-reuse.js +3 -1
  13. package/build/prompts/shared-helper-policy.d.ts +57 -0
  14. package/build/prompts/shared-helper-policy.js +135 -0
  15. package/build/prompts/test-recommendation/diffExecutionPlan.js +62 -56
  16. package/build/prompts/test-recommendation/fullRepoCatalog.js +19 -8
  17. package/build/prompts/test-recommendation/recommendationShared.d.ts +28 -6
  18. package/build/prompts/test-recommendation/recommendationShared.js +90 -16
  19. package/build/prompts/test-recommendation/registerRecommendTestsPrompt.js +22 -0
  20. package/build/prompts/test-recommendation/test-recommendation-prompt.d.ts +2 -2
  21. package/build/prompts/test-recommendation/test-recommendation-prompt.js +3 -3
  22. package/build/prompts/testbot/testbot-prompts.js +80 -34
  23. package/build/recommendation/budgeters/shared.js +105 -27
  24. package/build/recommendation/discriminators.js +13 -2
  25. package/build/recommendation/planRanker.d.ts +6 -6
  26. package/build/recommendation/planRanker.js +6 -61
  27. package/build/services/AnalyticsService.d.ts +7 -0
  28. package/build/services/AnalyticsService.js +7 -1
  29. package/build/services/ModularizationService.js +1 -3
  30. package/build/services/TestDiscoveryService.d.ts +0 -2
  31. package/build/services/TestDiscoveryService.js +2 -37
  32. package/build/services/TestGenerationService.d.ts +16 -0
  33. package/build/services/TestGenerationService.js +86 -10
  34. package/build/services/containerEnv.js +13 -12
  35. package/build/tools/code-refactor/codeReuseTool.js +279 -93
  36. package/build/tools/code-refactor/enhance-state.d.ts +49 -0
  37. package/build/tools/code-refactor/enhance-state.js +109 -0
  38. package/build/tools/code-refactor/enhanceAssertionsTool.js +34 -1
  39. package/build/tools/code-refactor/modularizationTool.js +9 -2
  40. package/build/tools/code-refactor/reuse-outcome.d.ts +23 -1
  41. package/build/tools/code-refactor/reuse-outcome.js +14 -4
  42. package/build/tools/code-refactor/reuse-state.d.ts +127 -5
  43. package/build/tools/code-refactor/reuse-state.js +628 -16
  44. package/build/tools/code-refactor/utils-verify-gates.d.ts +26 -0
  45. package/build/tools/code-refactor/utils-verify-gates.js +100 -0
  46. package/build/tools/code-refactor/verify-gates.d.ts +2 -1
  47. package/build/tools/code-refactor/verify-gates.js +90 -25
  48. package/build/tools/executeSkyrampTestTool.d.ts +19 -0
  49. package/build/tools/executeSkyrampTestTool.js +158 -8
  50. package/build/tools/generate-tests/generateBatchScenarioRestTool.js +2 -2
  51. package/build/tools/generate-tests/generateE2ERestTool.js +16 -0
  52. package/build/tools/generate-tests/generateUIRestTool.d.ts +1 -0
  53. package/build/tools/generate-tests/generateUIRestTool.js +22 -0
  54. package/build/tools/generate-tests/scenarioLint.d.ts +2 -0
  55. package/build/tools/generate-tests/scenarioLint.js +127 -19
  56. package/build/tools/generate-tests/trace-reuse-guard.d.ts +20 -0
  57. package/build/tools/generate-tests/trace-reuse-guard.js +93 -0
  58. package/build/tools/submitReportTool.d.ts +38 -38
  59. package/build/tools/submitReportTool.js +411 -114
  60. package/build/tools/test-management/analyzeChangesTool.d.ts +24 -1
  61. package/build/tools/test-management/analyzeChangesTool.js +71 -10
  62. package/build/tools/test-management/analyzeTestHealthTool.js +7 -7
  63. package/build/tools/test-management/registerTestPlanTool.d.ts +203 -0
  64. package/build/tools/test-management/registerTestPlanTool.js +70 -12
  65. package/build/types/Recommendation.d.ts +34 -5
  66. package/build/types/RepositoryAnalysis.d.ts +133 -114
  67. package/build/types/RepositoryAnalysis.js +1 -1
  68. package/build/types/ReuseOutcome.d.ts +102 -6
  69. package/build/types/ReuseOutcome.js +16 -2
  70. package/build/types/TestRecommendation.js +21 -3
  71. package/build/types/TestTypes.js +14 -8
  72. package/build/types/TestbotReport.d.ts +10 -1
  73. package/build/types/index.d.ts +2 -2
  74. package/build/types/index.js +1 -1
  75. package/build/utils/AnalysisStateManager.d.ts +57 -1
  76. package/build/utils/AnalysisStateManager.js +54 -5
  77. package/build/utils/branchDiff.d.ts +10 -0
  78. package/build/utils/branchDiff.js +28 -0
  79. package/build/utils/changedRoutes.d.ts +29 -0
  80. package/build/utils/changedRoutes.js +87 -0
  81. package/build/utils/featureFlags.d.ts +21 -0
  82. package/build/utils/featureFlags.js +23 -0
  83. package/build/utils/frontendIntegration.js +34 -4
  84. package/build/utils/importerHop.d.ts +2 -8
  85. package/build/utils/importerHop.js +15 -53
  86. package/build/utils/pathMatching.d.ts +38 -0
  87. package/build/utils/pathMatching.js +71 -0
  88. package/build/utils/pathSignatures.d.ts +22 -0
  89. package/build/utils/pathSignatures.js +57 -0
  90. package/build/utils/planMatchKeys.d.ts +16 -3
  91. package/build/utils/planMatchKeys.js +26 -10
  92. package/build/utils/pluralization.d.ts +10 -0
  93. package/build/utils/pluralization.js +18 -0
  94. package/build/utils/pom-catalog-parse.d.ts +52 -0
  95. package/build/utils/pom-catalog-parse.js +141 -0
  96. package/build/utils/pom-scope/selector-extractor.d.ts +12 -0
  97. package/build/utils/pom-scope/selector-extractor.js +34 -8
  98. package/build/utils/pom-verify/verify.d.ts +6 -5
  99. package/build/utils/pom-verify/verify.js +8 -6
  100. package/build/utils/reportVerification.d.ts +64 -4
  101. package/build/utils/reportVerification.js +228 -3
  102. package/build/utils/reuseRouting.d.ts +3 -0
  103. package/build/utils/reuseRouting.js +50 -0
  104. package/build/utils/routeParsers.d.ts +2 -0
  105. package/build/utils/routeParsers.js +65 -8
  106. package/build/utils/scenarioDrafting.d.ts +1 -1
  107. package/build/utils/scenarioDrafting.js +57 -45
  108. package/build/utils/subjectEndpoints.d.ts +19 -0
  109. package/build/utils/subjectEndpoints.js +98 -0
  110. package/build/utils/testFileClassification.d.ts +11 -0
  111. package/build/utils/testFileClassification.js +47 -0
  112. package/build/utils/uiPageEnumerator.d.ts +45 -19
  113. package/build/utils/uiPageEnumerator.js +95 -51
  114. package/build/utils/utils-verify/allow.d.ts +16 -0
  115. package/build/utils/utils-verify/allow.js +68 -0
  116. package/build/utils/utils-verify/call-sites.d.ts +34 -0
  117. package/build/utils/utils-verify/call-sites.js +154 -0
  118. package/build/utils/utils-verify/index.d.ts +7 -0
  119. package/build/utils/utils-verify/index.js +7 -0
  120. package/build/utils/utils-verify/language-spec.d.ts +91 -0
  121. package/build/utils/utils-verify/language-spec.js +210 -0
  122. package/build/utils/utils-verify/locate.d.ts +39 -0
  123. package/build/utils/utils-verify/locate.js +199 -0
  124. package/build/utils/utils-verify/parse.d.ts +34 -0
  125. package/build/utils/utils-verify/parse.js +177 -0
  126. package/build/utils/utils-verify/stage.d.ts +24 -0
  127. package/build/utils/utils-verify/stage.js +107 -0
  128. package/build/utils/utils-verify/verify.d.ts +63 -0
  129. package/build/utils/utils-verify/verify.js +168 -0
  130. package/build/utils/utils.d.ts +3 -1
  131. package/build/utils/utils.js +3 -1
  132. package/build/workspace/workspace.d.ts +32 -32
  133. package/node_modules/playwright/lib/mcp/skyramp/assertTool.js +9 -5
  134. package/node_modules/playwright/lib/mcp/skyramp/loadTraceTool.js +16 -0
  135. package/node_modules/playwright/lib/mcp/skyramp/skyRampImport.js +2 -0
  136. package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +115 -14
  137. package/node_modules/playwright/lib/mcp/test/skyRampExport.js +13 -1
  138. package/node_modules/playwright/node_modules/playwright-core/.DS_Store +0 -0
  139. package/node_modules/playwright/node_modules/playwright-core/lib/server/codegen/skyramp/jsonlReader.js +2 -0
  140. package/node_modules/playwright/node_modules/playwright-core/lib/vite/htmlReport/index.html +27 -253
  141. package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/assets/{codeMirrorModule-DtudTj_v.js → codeMirrorModule-DJMC4zNo.js} +1 -1
  142. package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/assets/index-BW82eAUI.js +196 -0
  143. package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/index.html +1 -1
  144. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{codeMirrorModule-FNMuBzX1.js → codeMirrorModule-CZfp96qZ.js} +1 -1
  145. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/defaultSettingsView-gpLo02E0.js +809 -0
  146. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.Bq1r1URj.js +2 -0
  147. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.html +2 -2
  148. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.VEfqi1qN.js +5 -0
  149. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.html +2 -2
  150. package/node_modules/playwright/node_modules/playwright-core/package.json +1 -1
  151. package/node_modules/playwright/node_modules/playwright-core/src/server/codegen/skyramp/jsonlReader.ts +1 -1
  152. package/node_modules/playwright/package.json +1 -1
  153. package/package.json +2 -2
  154. package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/assets/index-BpDwp16L.js +0 -422
  155. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/defaultSettingsView-Co9upU5h.js +0 -1035
  156. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.DXNIQ_dx.js +0 -2
  157. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.CIKB3XSv.js +0 -5
@@ -0,0 +1,199 @@
1
+ import { execFile } from "child_process";
2
+ import { promisify } from "util";
3
+ import * as fs from "fs";
4
+ import * as path from "path";
5
+ import { detectGitRoot } from "../gitStaging.js";
6
+ import { logger } from "../logger.js";
7
+ import { SKYRAMP_UTILS_HEADER } from "../utils.js";
8
+ import { utilsSpecFor, utilsSpecForFile, } from "./language-spec.js";
9
+ const execFileAsync = promisify(execFile);
10
+ /** The header `generateSkyrampHeader` writes into a utils file: `Generated by Skyramp on
11
+ * <timestamp>`. Deliberately NOT the bare `SKYRAMP_UTILS_HEADER`, which every generated
12
+ * TEST also carries as its codegen marker (`Generated by Skyramp v<version>`). */
13
+ export const UTILS_FILE_HEADER_RE = new RegExp(`${SKYRAMP_UTILS_HEADER.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\s+on\\b`);
14
+ const HEADER_SCAN_LINES = 5;
15
+ /** Enough bytes for five header lines; a utils file is never read whole to find them. */
16
+ const HEADER_SCAN_BYTES = 2048;
17
+ /**
18
+ * Every shared utils file relevant to `testFile`, by CONTENT, never by name.
19
+ *
20
+ * The reuse prompt names a conventional file (`SkyrampUtils.py`) but explicitly allows
21
+ * "a new file with a different name" once that one is large, so any scheme deriving the
22
+ * path from the test path or a fixed filename fails silently on exactly that branch. A
23
+ * utils file is instead recognised by the header the prompt makes the agent write.
24
+ *
25
+ * Two sources, unioned, both scoped to the spec:
26
+ * 1. files this run touched (untracked, modified, or staged) IN THE SPEC'S OWN
27
+ * DIRECTORY, in the language's extensions — the file the agent just created or
28
+ * extended beside the test, which is what staging needs. Not repo-wide: a monorepo
29
+ * has one utils file per service directory, and one service's broken helper must
30
+ * not fail — or stage — another service's verify (review finding on #792).
31
+ * 2. modules the delivered spec imports, resolved beside it and required to stay
32
+ * inside the repository — a pre-existing utils file the spec merely imports from
33
+ * (SKYR-4196 run 8: byte-identical, so absent from source 1).
34
+ *
35
+ * Never throws: outside a git repo source 1 is empty and source 2 still runs; any
36
+ * other failure logs and contributes nothing.
37
+ */
38
+ export async function locateUtilsFiles(params) {
39
+ const spec = utilsSpecFor(params.language) ?? utilsSpecForFile(params.testFile);
40
+ if (!spec)
41
+ return [];
42
+ const found = new Set();
43
+ const root = params.cwd ?? detectGitRoot(params.testFile);
44
+ // Git reports paths under the REAL root while a tool-supplied testFile may go
45
+ // through a symlink (macOS: /tmp → /private/tmp), so the two sources would name one
46
+ // file twice. Everything is compared and returned in canonical form.
47
+ const testFile = await realpath(params.testFile);
48
+ const specDir = path.dirname(testFile);
49
+ const realRoot = root ? await realpath(root) : undefined;
50
+ if (realRoot) {
51
+ for (const rel of await changedFiles(realRoot)) {
52
+ if (!spec.extensions.some((e) => rel.toLowerCase().endsWith(e)))
53
+ continue;
54
+ const abs = await realpath(path.resolve(realRoot, rel));
55
+ if (abs === testFile || path.dirname(abs) !== specDir)
56
+ continue;
57
+ if (await hasUtilsHeader(abs))
58
+ found.add(abs);
59
+ }
60
+ }
61
+ for (const abs of await importedModules(testFile, spec)) {
62
+ const real = await realpath(abs);
63
+ // A relative import can climb out of the repository; a file there is not this
64
+ // repo's artifact and must never be verified or staged from here.
65
+ if (realRoot && !isInside(real, realRoot))
66
+ continue;
67
+ if (await hasUtilsHeader(real))
68
+ found.add(real);
69
+ }
70
+ return [...found].sort();
71
+ }
72
+ /** Canonical path, or the input unchanged when it does not exist (yet). */
73
+ export async function realpath(p) {
74
+ try {
75
+ return await fs.promises.realpath(p);
76
+ }
77
+ catch {
78
+ return path.resolve(p);
79
+ }
80
+ }
81
+ function isInside(file, dir) {
82
+ const rel = path.relative(dir, file);
83
+ return rel !== "" && !rel.startsWith("..") && !path.isAbsolute(rel);
84
+ }
85
+ async function changedFiles(root) {
86
+ const out = new Set();
87
+ // A caller-supplied root need not be a repository (the eval fixtures are plain
88
+ // directories); source 1 then contributes nothing, silently.
89
+ try {
90
+ await execFileAsync("git", ["rev-parse", "--is-inside-work-tree"], {
91
+ cwd: root,
92
+ encoding: "utf8",
93
+ });
94
+ }
95
+ catch {
96
+ return [];
97
+ }
98
+ // `-z`: NUL-separated, unquoted. Without it git C-quotes any non-ASCII path
99
+ // (`"tests/caf\303\251_helpers.py"`), which no later step could resolve.
100
+ const cmds = [
101
+ ["ls-files", "-z", "--others", "--exclude-standard"],
102
+ ["diff", "-z", "--name-only"],
103
+ ["diff", "-z", "--cached", "--name-only"],
104
+ ];
105
+ for (const args of cmds) {
106
+ try {
107
+ const { stdout } = await execFileAsync("git", args, {
108
+ cwd: root,
109
+ encoding: "utf8",
110
+ });
111
+ for (const entry of stdout.split("\0"))
112
+ if (entry)
113
+ out.add(entry);
114
+ }
115
+ catch (err) {
116
+ logger.warning("Could not list changed files while locating utils files", { root, args, error: String(err) });
117
+ }
118
+ }
119
+ return [...out];
120
+ }
121
+ /** True when the first lines of `file` carry the utils header. Reads only the head of
122
+ * the file — a utils file is small, but nothing here should depend on that. */
123
+ export async function hasUtilsHeader(file) {
124
+ let fh;
125
+ try {
126
+ fh = await fs.promises.open(file, "r");
127
+ const buf = Buffer.alloc(HEADER_SCAN_BYTES);
128
+ const { bytesRead } = await fh.read(buf, 0, HEADER_SCAN_BYTES, 0);
129
+ const head = buf
130
+ .toString("utf8", 0, bytesRead)
131
+ .split("\n")
132
+ .slice(0, HEADER_SCAN_LINES)
133
+ .join("\n");
134
+ return UTILS_FILE_HEADER_RE.test(head);
135
+ }
136
+ catch (err) {
137
+ // A missing file is the common, silent case. Anything else means a file exists
138
+ // that cannot be read — it must not pass as "not a utils file" without a trace.
139
+ if (err?.code !== "ENOENT") {
140
+ logger.warning("Could not read a candidate utils file — treating it as not a utils file", {
141
+ file,
142
+ error: String(err),
143
+ });
144
+ }
145
+ return false;
146
+ }
147
+ finally {
148
+ await fh?.close();
149
+ }
150
+ }
151
+ /** Absolute paths of sibling modules `testFile` imports, for the module forms the reuse
152
+ * prompt produces. Only paths that exist are returned. */
153
+ async function importedModules(testFile, spec) {
154
+ let content;
155
+ try {
156
+ content = await fs.promises.readFile(testFile, "utf8");
157
+ }
158
+ catch (err) {
159
+ if (err?.code !== "ENOENT") {
160
+ logger.warning("Could not read the test file to resolve its imports", {
161
+ testFile,
162
+ error: String(err),
163
+ });
164
+ }
165
+ return [];
166
+ }
167
+ const dir = path.dirname(testFile);
168
+ const candidates = new Set();
169
+ if (spec.language === "python") {
170
+ for (const m of content.matchAll(/^\s*(?:from\s+([\w.]+)\s+import\b|import\s+([\w.]+)(?:\s+as\s+\w+)?\s*$)/gm)) {
171
+ const mod = (m[1] ?? m[2]).replace(/^\.+/, "");
172
+ if (!mod)
173
+ continue;
174
+ candidates.add(path.join(dir, ...mod.split(".")) + ".py");
175
+ }
176
+ }
177
+ else {
178
+ for (const m of content.matchAll(/(?:from\s*|require\(\s*|import\(\s*)["'](\.{1,2}\/[^"']+)["']/g)) {
179
+ const base = path.resolve(dir, m[1]);
180
+ const stripped = base.replace(/\.[cm]?[jt]sx?$/, "");
181
+ for (const ext of spec.extensions)
182
+ candidates.add(stripped + ext);
183
+ // A `.js` specifier in an ESM TypeScript spec resolves to the `.ts` source.
184
+ candidates.add(stripped + ".ts");
185
+ candidates.add(stripped + ".js");
186
+ }
187
+ }
188
+ const existing = [];
189
+ for (const c of candidates) {
190
+ try {
191
+ if ((await fs.promises.stat(c)).isFile())
192
+ existing.push(c);
193
+ }
194
+ catch {
195
+ /* not present */
196
+ }
197
+ }
198
+ return existing;
199
+ }
@@ -0,0 +1,34 @@
1
+ import { type UtilsLanguageSpec } from "./language-spec.js";
2
+ export interface HelperAssertion {
3
+ line: number;
4
+ text: string;
5
+ /** True for the one permitted shape — a status-code comparison. */
6
+ allowed: boolean;
7
+ }
8
+ export interface UtilsHelper {
9
+ name: string;
10
+ /** 1-based line of the definition. */
11
+ line: number;
12
+ /** 1-based line of the last body line (inclusive). */
13
+ endLine: number;
14
+ method?: string;
15
+ path?: string;
16
+ /** `path` with every path parameter collapsed to `{}`; the duplicate key with `method`. */
17
+ normalizedPath?: string;
18
+ assertions: HelperAssertion[];
19
+ }
20
+ /**
21
+ * Structured extraction of the module-level helpers in a utils file.
22
+ *
23
+ * Not a general parser. These files are highly regular by construction — one SDK
24
+ * request call plus assertions per helper, authored by a prompt that forbids
25
+ * classes and nested structure — so line-oriented extraction is sufficient and
26
+ * far more robust than a per-language AST dependency. Python bodies end at the
27
+ * next non-blank column-0 line; TS/JS bodies end when the brace depth opened by
28
+ * the definition returns to zero.
29
+ */
30
+ export declare function parseUtilsFile(content: string, spec: UtilsLanguageSpec, allowedAssertion?: RegExp | ((expr: string) => boolean)): UtilsHelper[];
31
+ /** Comments and string CONTENTS blanked (quotes kept, line count preserved). Python
32
+ * comments are `#…`; TS/JS go through the shared stripper. */
33
+ /** String CONTENTS blanked (quotes kept, comments kept, line count preserved). */
34
+ export declare function blankStrings(src: string, spec: UtilsLanguageSpec): string;
@@ -0,0 +1,177 @@
1
+ import { normalizePath } from "./language-spec.js";
2
+ import { stripCommentsAndStrings, stripStrings } from "../pom-scope/strip.js";
3
+ /**
4
+ * Structured extraction of the module-level helpers in a utils file.
5
+ *
6
+ * Not a general parser. These files are highly regular by construction — one SDK
7
+ * request call plus assertions per helper, authored by a prompt that forbids
8
+ * classes and nested structure — so line-oriented extraction is sufficient and
9
+ * far more robust than a per-language AST dependency. Python bodies end at the
10
+ * next non-blank column-0 line; TS/JS bodies end when the brace depth opened by
11
+ * the definition returns to zero.
12
+ */
13
+ export function parseUtilsFile(content, spec, allowedAssertion = spec.allowedAssertion) {
14
+ const isAllowed = typeof allowedAssertion === "function"
15
+ ? allowedAssertion
16
+ : (expr) => allowedAssertion.test(expr);
17
+ const lines = content.split("\n");
18
+ const helpers = [];
19
+ let i = 0;
20
+ while (i < lines.length) {
21
+ const m = spec.helperStart.exec(lines[i]);
22
+ if (!m) {
23
+ i++;
24
+ continue;
25
+ }
26
+ const name = m.slice(1).find((g) => g !== undefined) ?? "";
27
+ const start = i;
28
+ const end = spec.language === "python"
29
+ ? pythonBodyEnd(lines, start)
30
+ : braceBodyEnd(lines, start);
31
+ const body = lines.slice(start, end);
32
+ // Scan a copy with comments and string contents blanked, so a commented-out
33
+ // `path=` or an `expect(` inside a string never keys or flags anything; the
34
+ // original line is what the agent is shown.
35
+ const scan = blankCommentsAndStrings(body.join("\n"), spec).split("\n");
36
+ const helper = {
37
+ name,
38
+ line: start + 1,
39
+ endLine: Math.max(end, start + 1),
40
+ assertions: [],
41
+ };
42
+ let requests = 0;
43
+ for (let k = 0; k < body.length; k++) {
44
+ const text = body[k];
45
+ const code = scan[k] ?? "";
46
+ if (/\b(?:send_request|sendRequest)\s*\(/.test(code))
47
+ requests++;
48
+ const method = spec.methodRe.exec(text);
49
+ if (method && /\bmethod\b/.test(code) && !helper.method)
50
+ helper.method = method[1].toUpperCase();
51
+ const p = spec.pathRe.exec(text);
52
+ // Key only a path given as ONE literal (followed by `,` or `)`); a concatenation
53
+ // (`"/api/orders/" + str(id) + "/items"`) has no route this can name.
54
+ if (p &&
55
+ /\bpath\b/.test(code) &&
56
+ !helper.path &&
57
+ isSingleLiteral(text, p)) {
58
+ helper.path = p[1];
59
+ helper.normalizedPath = normalizePath(p[1]);
60
+ }
61
+ // The definition line itself is never an assertion, and comment lines are prose.
62
+ // A formatted assertion may span lines (`await expect(\n locator\n).toBeVisible()`):
63
+ // collect it until its parentheses balance, classify the whole expression.
64
+ if (k > 0 && spec.assertionRe.test(code)) {
65
+ let expr = code;
66
+ let last = k;
67
+ while (depthOf(expr) > 0 && last + 1 < body.length && last - k < 8) {
68
+ last++;
69
+ expr += " " + (scan[last] ?? "").trim();
70
+ }
71
+ // Classified on the ORIGINAL text: the allowed patterns look inside string
72
+ // literals (a scenario literal in a locator disqualifies a structural wait).
73
+ const original = body
74
+ .slice(k, last + 1)
75
+ .map((l) => l.trim())
76
+ .join(" ");
77
+ helper.assertions.push({
78
+ line: start + k + 1,
79
+ text: original,
80
+ allowed: isAllowed(original),
81
+ });
82
+ k = last;
83
+ }
84
+ }
85
+ // A helper wrapping two requests is a sequence, not "one SDK call": it has no single
86
+ // route to be a duplicate of, and the shared-helper rules do not apply to it.
87
+ if (requests > 1) {
88
+ helper.method = undefined;
89
+ helper.path = undefined;
90
+ helper.normalizedPath = undefined;
91
+ }
92
+ helpers.push(helper);
93
+ i = Math.max(end, start + 1);
94
+ }
95
+ return helpers;
96
+ }
97
+ function depthOf(code) {
98
+ let d = 0;
99
+ for (const ch of code)
100
+ if (ch === "(")
101
+ d++;
102
+ else if (ch === ")")
103
+ d--;
104
+ return d;
105
+ }
106
+ /** True when the `path` value is a single string literal ending the argument. */
107
+ function isSingleLiteral(line, m) {
108
+ const after = line.slice(m.index + m[0].length).replace(/^\s+/, "");
109
+ return (after === "" ||
110
+ after.startsWith(",") ||
111
+ after.startsWith(")") ||
112
+ after.startsWith("}"));
113
+ }
114
+ /** Comments and string CONTENTS blanked (quotes kept, line count preserved). Python
115
+ * comments are `#…`; TS/JS go through the shared stripper. */
116
+ /** String CONTENTS blanked (quotes kept, comments kept, line count preserved). */
117
+ export function blankStrings(src, spec) {
118
+ if (spec.language !== "python")
119
+ return stripStrings(src);
120
+ return src
121
+ .split("\n")
122
+ .map((line) => line.replace(/(["'])(?:\\.|(?!\1).)*\1/g, (q) => q[0] + " ".repeat(Math.max(0, q.length - 2)) + q[0]))
123
+ .join("\n");
124
+ }
125
+ function blankCommentsAndStrings(src, spec) {
126
+ if (spec.language !== "python")
127
+ return stripCommentsAndStrings(src);
128
+ return src
129
+ .split("\n")
130
+ .map((line) => line
131
+ .replace(/(["'])(?:\\.|(?!\1).)*\1/g, (q) => q[0] + " ".repeat(Math.max(0, q.length - 2)) + q[0])
132
+ .replace(/#.*$/, ""))
133
+ .join("\n");
134
+ }
135
+ /** Exclusive end index of a Python def starting at `start`: the next non-blank line at
136
+ * column 0 (a following def, import, or module statement). */
137
+ function pythonBodyEnd(lines, start) {
138
+ let i = start + 1;
139
+ while (i < lines.length) {
140
+ const l = lines[i];
141
+ if (l.trim() !== "" && !/^\s/.test(l))
142
+ break;
143
+ i++;
144
+ }
145
+ return i;
146
+ }
147
+ /** Exclusive end index of a brace-delimited definition starting at `start`. Strings and
148
+ * comments are not tracked — helper bodies contain request paths, not braces in
149
+ * strings — and an unbalanced body runs to end of file rather than throwing. */
150
+ function braceBodyEnd(lines, start) {
151
+ let depth = 0;
152
+ let opened = false;
153
+ for (let i = start; i < lines.length; i++) {
154
+ for (const ch of lines[i]) {
155
+ if (ch === "{") {
156
+ depth++;
157
+ opened = true;
158
+ }
159
+ else if (ch === "}") {
160
+ depth--;
161
+ }
162
+ }
163
+ if (opened && depth <= 0)
164
+ return i + 1;
165
+ // No brace yet: a wrapped signature (Prettier splits any signature over 80 columns
166
+ // across lines) continues until its `{`. An arrow helper with an expression body
167
+ // ends at the first line that closes its `=>` expression without a brace.
168
+ if (!opened &&
169
+ i > start &&
170
+ /=>\s*[^{\s]/.test(lines[start]) &&
171
+ /;\s*$/.test(lines[i - 1]))
172
+ return i;
173
+ if (!opened && i - start > 12)
174
+ return i;
175
+ }
176
+ return lines.length;
177
+ }
@@ -0,0 +1,24 @@
1
+ /**
2
+ * Stage every shared utils file relevant to `testFile`, located by content.
3
+ *
4
+ * The reuse tool stages only the test file, and does so BEFORE the utils file exists,
5
+ * so a utils file the agent then creates reaches the output commit only through a
6
+ * prompt-level `git add` or the action's testDirectory sweep — neither of which the
7
+ * eval harness has, and neither of which is deterministic (SKYR-4196 Part B run 2
8
+ * shipped a test importing a module the PR did not contain).
9
+ *
10
+ * Called from the utils verify pass AND from the tools that follow reuse on the same
11
+ * spec (`skyramp_enhance_assertions`, `skyramp_execute_test`), so staging never
12
+ * depends on the agent choosing to verify. Best-effort: a staging failure is logged,
13
+ * never surfaced — it must not fail a tool that otherwise succeeded. Returns the paths
14
+ * it staged. No-op outside a testbot run (see stageGeneratedPaths).
15
+ */
16
+ export interface StagingResult {
17
+ /** Files that reached the index. */
18
+ staged: string[];
19
+ /** Utils files that `git add` refused (gitignored, outside the repo, git failure) —
20
+ * the tests importing them will fail in the output commit, so the verify pass
21
+ * treats a non-empty list as a failed verification. */
22
+ failed: string[];
23
+ }
24
+ export declare function stageUtilsArtifacts(testFile: string, language?: string): Promise<StagingResult>;
@@ -0,0 +1,107 @@
1
+ import { execFile } from "child_process";
2
+ import { promisify } from "util";
3
+ import * as fs from "fs";
4
+ import * as path from "path";
5
+ import { detectGitRoot, stageGeneratedPaths } from "../gitStaging.js";
6
+ import { logger } from "../logger.js";
7
+ import { isTestbotEnabled } from "../featureFlags.js";
8
+ import { locateUtilsFiles, realpath } from "./locate.js";
9
+ import { CODEGEN_MARKER_RE } from "./call-sites.js";
10
+ import { utilsSpecFor, utilsSpecForFile } from "./language-spec.js";
11
+ const execFileAsync = promisify(execFile);
12
+ export async function stageUtilsArtifacts(testFile, language) {
13
+ const staged = [];
14
+ const failed = [];
15
+ const result = { staged, failed };
16
+ // Outside a testbot run stageGeneratedPaths is a no-op; report nothing as staged
17
+ // rather than a list of files nothing touched.
18
+ if (!isTestbotEnabled())
19
+ return result;
20
+ let files = [];
21
+ try {
22
+ files = await locateUtilsFiles({ testFile, language });
23
+ }
24
+ catch (err) {
25
+ logger.warning("Could not locate shared utils files to stage", {
26
+ testFile,
27
+ error: String(err),
28
+ });
29
+ return result;
30
+ }
31
+ for (const file of files) {
32
+ try {
33
+ await stageGeneratedPaths(file);
34
+ staged.push(file);
35
+ }
36
+ catch (err) {
37
+ // A gitignored utils file lands here (`git add` refuses it): the test would
38
+ // import a module the output commit cannot contain.
39
+ logger.warning("Could not stage shared utils file — if it is gitignored, the tests importing it will fail in the output commit", { file, error: String(err) });
40
+ failed.push(file);
41
+ }
42
+ }
43
+ // Sibling generated tests this run EDITED to import from the utils file (STEP 5c, or
44
+ // a maintenance update): tracked, modified, codegen-marked, importing a located utils
45
+ // stem. Discovery staged only the current test and the loop above only the utils
46
+ // file, so without this the sibling's edit drops out of the output commit while the
47
+ // utils change lands — the same class of hole as the unstaged utils file.
48
+ if (files.length > 0) {
49
+ for (const sibling of await modifiedSiblingsImporting(testFile, files, language)) {
50
+ try {
51
+ await stageGeneratedPaths(sibling);
52
+ staged.push(sibling);
53
+ }
54
+ catch (err) {
55
+ logger.warning("Could not stage an edited sibling test", {
56
+ file: sibling,
57
+ error: String(err),
58
+ });
59
+ }
60
+ }
61
+ }
62
+ return result;
63
+ }
64
+ async function modifiedSiblingsImporting(testFile, utilsFiles, language) {
65
+ const spec = utilsSpecFor(language) ?? utilsSpecForFile(testFile);
66
+ const root = detectGitRoot(testFile);
67
+ if (!spec || !root)
68
+ return [];
69
+ let modified;
70
+ try {
71
+ // `-z`: unquoted, NUL-separated — git C-quotes non-ASCII paths otherwise.
72
+ const { stdout } = await execFileAsync("git", ["diff", "-z", "--name-only"], {
73
+ cwd: root,
74
+ encoding: "utf8",
75
+ });
76
+ modified = stdout.split("\0").filter(Boolean);
77
+ }
78
+ catch (err) {
79
+ logger.warning("Could not list modified files — edited sibling tests will not be staged", {
80
+ root,
81
+ error: String(err),
82
+ });
83
+ return [];
84
+ }
85
+ const stems = utilsFiles.map((f) => path.basename(f).replace(/\.[^.]+$/, ""));
86
+ const self = await realpath(testFile);
87
+ const out = [];
88
+ for (const rel of modified) {
89
+ const abs = await realpath(path.resolve(root, rel));
90
+ if (abs === self || utilsFiles.includes(abs))
91
+ continue;
92
+ if (!spec.extensions.some((e) => abs.toLowerCase().endsWith(e)))
93
+ continue;
94
+ let content;
95
+ try {
96
+ content = await fs.promises.readFile(abs, "utf8");
97
+ }
98
+ catch {
99
+ continue;
100
+ }
101
+ if (!CODEGEN_MARKER_RE.test(content.split("\n").slice(0, 5).join("\n")))
102
+ continue;
103
+ if (stems.some((stem) => spec.importedNames(content, stem).length > 0))
104
+ out.push(abs);
105
+ }
106
+ return out;
107
+ }
@@ -0,0 +1,63 @@
1
+ import { type InlineCallSite } from "./call-sites.js";
2
+ import { type UtilsViolationKind } from "./allow.js";
3
+ import { type HelperFamilySpec, type UtilsLanguageSpec } from "./language-spec.js";
4
+ export interface UtilsViolation {
5
+ kind: UtilsViolationKind;
6
+ /** Absolute path of the utils file. */
7
+ file: string;
8
+ helper: string;
9
+ line: number;
10
+ detail: string;
11
+ }
12
+ export interface UtilsVerifyResult {
13
+ /** No unallowed violations and no malformed allow markers. */
14
+ ok: boolean;
15
+ /** Utils files located for this spec (absolute). Empty = nothing was written or imported. */
16
+ utilsFiles: string[];
17
+ /** Helpers defined across the utils files. */
18
+ helpers: number;
19
+ /** Helper names the delivered spec imports from those files. */
20
+ imported: number;
21
+ violations: UtilsViolation[];
22
+ /** Violations covered by a `reuse-verify: allow` marker — a documented decline. */
23
+ allowed: UtilsViolation[];
24
+ /** Findings that inform but never fail the pass. `scenario-name` lives here: a naming
25
+ * heuristic is a quality signal, not a broken shared helper, and blocking a
26
+ * customer's run on a false positive of it is the worse outcome. */
27
+ advisories: UtilsViolation[];
28
+ /** Allow-marker lines the grammar cannot read, per file. */
29
+ malformedAllows: {
30
+ file: string;
31
+ line: string;
32
+ }[];
33
+ spec?: UtilsLanguageSpec;
34
+ /** Which invariant set applied — decided by the test type, see helperFamilyFor. */
35
+ family: HelperFamilySpec;
36
+ /** Advisory (does not affect `ok`): inline request calls in sibling generated tests
37
+ * that a utils helper already wraps — see findSiblingInlineCallSites. API family only. */
38
+ inlineCallSites: InlineCallSite[];
39
+ }
40
+ export interface VerifyUtilsParams {
41
+ testFile: string;
42
+ language?: string;
43
+ /** Selects the helper family (api vs browser). Absent → api. */
44
+ testType?: string;
45
+ cwd?: string;
46
+ }
47
+ /**
48
+ * Deterministic post-reuse check of the shared utils file(s) a spec depends on.
49
+ *
50
+ * Three invariants, each a class of drift measured in the SKYR-4196 Part B evals:
51
+ * - `duplicate-helper`: two helpers for one method+path (run 4 — `create_product`
52
+ * beside `create_list_reflect_product`); the shared file must hold one canonical
53
+ * helper per step, extended with defaulted parameters rather than duplicated.
54
+ * - `body-assertion`: any assertion other than a status-code comparison inside a
55
+ * helper (runs 5 and 11 — `assert total_amount == 16.0` in `update_order`); a
56
+ * body assertion carries one scenario's values into a function every scenario calls.
57
+ * - `scenario-name` (ADVISORY only): a helper named for its scenario rather than its step.
58
+ *
59
+ * Analyses the utils files' INTERNAL invariants — unlike the POM verifier, which
60
+ * checks a spec's cross-file references — so it reads the utils files, not the spec,
61
+ * except to count what the spec imports.
62
+ */
63
+ export declare function verifyUtils(params: VerifyUtilsParams): Promise<UtilsVerifyResult>;