@skyramp/mcp 0.3.4 → 0.3.6-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. package/build/playwright/registerPlaywrightTools.js +92 -30
  2. package/build/playwright/traceRecordingPrompt.d.ts +6 -0
  3. package/build/playwright/traceRecordingPrompt.js +6 -2
  4. package/build/prompts/code-reuse.d.ts +1 -2
  5. package/build/prompts/code-reuse.js +182 -77
  6. package/build/prompts/modularization/integration-test-modularization.d.ts +2 -0
  7. package/build/prompts/modularization/integration-test-modularization.js +83 -41
  8. package/build/prompts/modularization/render.d.ts +18 -0
  9. package/build/prompts/modularization/render.js +12 -0
  10. package/build/prompts/modularization/ui-test-modularization.d.ts +3 -1
  11. package/build/prompts/modularization/ui-test-modularization.js +89 -47
  12. package/build/prompts/pom-aware-code-reuse.js +3 -1
  13. package/build/prompts/shared-helper-policy.d.ts +57 -0
  14. package/build/prompts/shared-helper-policy.js +135 -0
  15. package/build/prompts/test-recommendation/diffExecutionPlan.js +62 -56
  16. package/build/prompts/test-recommendation/fullRepoCatalog.js +19 -8
  17. package/build/prompts/test-recommendation/recommendationShared.d.ts +28 -6
  18. package/build/prompts/test-recommendation/recommendationShared.js +90 -16
  19. package/build/prompts/test-recommendation/registerRecommendTestsPrompt.js +22 -0
  20. package/build/prompts/test-recommendation/test-recommendation-prompt.d.ts +2 -2
  21. package/build/prompts/test-recommendation/test-recommendation-prompt.js +3 -3
  22. package/build/prompts/testbot/testbot-prompts.js +88 -33
  23. package/build/recommendation/budgeters/shared.js +105 -27
  24. package/build/recommendation/discriminators.js +13 -2
  25. package/build/recommendation/planRanker.d.ts +6 -6
  26. package/build/recommendation/planRanker.js +6 -61
  27. package/build/services/AnalyticsService.d.ts +7 -0
  28. package/build/services/AnalyticsService.js +7 -1
  29. package/build/services/ModularizationService.js +1 -3
  30. package/build/services/TestDiscoveryService.d.ts +0 -2
  31. package/build/services/TestDiscoveryService.js +2 -37
  32. package/build/services/TestGenerationService.d.ts +16 -0
  33. package/build/services/TestGenerationService.js +86 -10
  34. package/build/services/containerEnv.js +13 -12
  35. package/build/tools/code-refactor/codeReuseTool.js +279 -93
  36. package/build/tools/code-refactor/enhance-state.d.ts +49 -0
  37. package/build/tools/code-refactor/enhance-state.js +109 -0
  38. package/build/tools/code-refactor/enhanceAssertionsTool.js +34 -1
  39. package/build/tools/code-refactor/modularizationTool.js +9 -2
  40. package/build/tools/code-refactor/reuse-outcome.d.ts +23 -1
  41. package/build/tools/code-refactor/reuse-outcome.js +14 -4
  42. package/build/tools/code-refactor/reuse-state.d.ts +127 -5
  43. package/build/tools/code-refactor/reuse-state.js +628 -16
  44. package/build/tools/code-refactor/utils-verify-gates.d.ts +26 -0
  45. package/build/tools/code-refactor/utils-verify-gates.js +100 -0
  46. package/build/tools/code-refactor/verify-gates.d.ts +2 -1
  47. package/build/tools/code-refactor/verify-gates.js +90 -25
  48. package/build/tools/executeSkyrampTestTool.d.ts +19 -0
  49. package/build/tools/executeSkyrampTestTool.js +158 -8
  50. package/build/tools/generate-tests/generateBatchScenarioRestTool.js +2 -2
  51. package/build/tools/generate-tests/generateE2ERestTool.js +16 -0
  52. package/build/tools/generate-tests/generateUIRestTool.d.ts +1 -0
  53. package/build/tools/generate-tests/generateUIRestTool.js +22 -0
  54. package/build/tools/generate-tests/scenarioLint.d.ts +2 -0
  55. package/build/tools/generate-tests/scenarioLint.js +127 -19
  56. package/build/tools/generate-tests/trace-reuse-guard.d.ts +20 -0
  57. package/build/tools/generate-tests/trace-reuse-guard.js +93 -0
  58. package/build/tools/submitReportTool.d.ts +38 -38
  59. package/build/tools/submitReportTool.js +487 -104
  60. package/build/tools/test-management/analyzeChangesTool.d.ts +24 -1
  61. package/build/tools/test-management/analyzeChangesTool.js +75 -12
  62. package/build/tools/test-management/analyzeTestHealthTool.js +7 -7
  63. package/build/tools/test-management/registerTestPlanTool.d.ts +203 -0
  64. package/build/tools/test-management/registerTestPlanTool.js +149 -23
  65. package/build/types/Recommendation.d.ts +34 -5
  66. package/build/types/RepositoryAnalysis.d.ts +133 -114
  67. package/build/types/RepositoryAnalysis.js +1 -1
  68. package/build/types/ReuseOutcome.d.ts +102 -6
  69. package/build/types/ReuseOutcome.js +16 -2
  70. package/build/types/TestRecommendation.js +21 -3
  71. package/build/types/TestTypes.js +14 -8
  72. package/build/types/TestbotReport.d.ts +25 -3
  73. package/build/types/index.d.ts +2 -2
  74. package/build/types/index.js +1 -1
  75. package/build/utils/AnalysisStateManager.d.ts +69 -1
  76. package/build/utils/AnalysisStateManager.js +69 -5
  77. package/build/utils/branchDiff.d.ts +10 -0
  78. package/build/utils/branchDiff.js +28 -0
  79. package/build/utils/changedRoutes.d.ts +29 -0
  80. package/build/utils/changedRoutes.js +87 -0
  81. package/build/utils/featureFlags.d.ts +21 -0
  82. package/build/utils/featureFlags.js +23 -0
  83. package/build/utils/frontendIntegration.js +34 -4
  84. package/build/utils/importerHop.d.ts +2 -8
  85. package/build/utils/importerHop.js +15 -53
  86. package/build/utils/pathMatching.d.ts +38 -0
  87. package/build/utils/pathMatching.js +71 -0
  88. package/build/utils/pathSignatures.d.ts +22 -0
  89. package/build/utils/pathSignatures.js +57 -0
  90. package/build/utils/planMatchKeys.d.ts +16 -3
  91. package/build/utils/planMatchKeys.js +26 -10
  92. package/build/utils/pluralization.d.ts +10 -0
  93. package/build/utils/pluralization.js +18 -0
  94. package/build/utils/pom-catalog-parse.d.ts +52 -0
  95. package/build/utils/pom-catalog-parse.js +141 -0
  96. package/build/utils/pom-scope/selector-extractor.d.ts +12 -0
  97. package/build/utils/pom-scope/selector-extractor.js +34 -8
  98. package/build/utils/pom-verify/verify.d.ts +6 -5
  99. package/build/utils/pom-verify/verify.js +8 -6
  100. package/build/utils/reportLanguage.d.ts +43 -0
  101. package/build/utils/reportLanguage.js +125 -0
  102. package/build/utils/reportVerification.d.ts +74 -4
  103. package/build/utils/reportVerification.js +259 -3
  104. package/build/utils/reuseRouting.d.ts +3 -0
  105. package/build/utils/reuseRouting.js +50 -0
  106. package/build/utils/routeParsers.d.ts +2 -0
  107. package/build/utils/routeParsers.js +65 -8
  108. package/build/utils/scenarioDrafting.d.ts +1 -1
  109. package/build/utils/scenarioDrafting.js +57 -45
  110. package/build/utils/subjectEndpoints.d.ts +19 -0
  111. package/build/utils/subjectEndpoints.js +98 -0
  112. package/build/utils/testFileClassification.d.ts +11 -0
  113. package/build/utils/testFileClassification.js +47 -0
  114. package/build/utils/uiPageEnumerator.d.ts +45 -19
  115. package/build/utils/uiPageEnumerator.js +95 -51
  116. package/build/utils/utils-verify/allow.d.ts +16 -0
  117. package/build/utils/utils-verify/allow.js +68 -0
  118. package/build/utils/utils-verify/call-sites.d.ts +34 -0
  119. package/build/utils/utils-verify/call-sites.js +154 -0
  120. package/build/utils/utils-verify/index.d.ts +7 -0
  121. package/build/utils/utils-verify/index.js +7 -0
  122. package/build/utils/utils-verify/language-spec.d.ts +91 -0
  123. package/build/utils/utils-verify/language-spec.js +210 -0
  124. package/build/utils/utils-verify/locate.d.ts +39 -0
  125. package/build/utils/utils-verify/locate.js +199 -0
  126. package/build/utils/utils-verify/parse.d.ts +34 -0
  127. package/build/utils/utils-verify/parse.js +177 -0
  128. package/build/utils/utils-verify/stage.d.ts +24 -0
  129. package/build/utils/utils-verify/stage.js +107 -0
  130. package/build/utils/utils-verify/verify.d.ts +63 -0
  131. package/build/utils/utils-verify/verify.js +168 -0
  132. package/build/utils/utils.d.ts +3 -1
  133. package/build/utils/utils.js +3 -1
  134. package/build/workspace/workspace.d.ts +32 -32
  135. package/node_modules/playwright/lib/mcp/skyramp/assertTool.js +9 -5
  136. package/node_modules/playwright/lib/mcp/skyramp/loadTraceTool.js +16 -0
  137. package/node_modules/playwright/lib/mcp/skyramp/skyRampImport.js +2 -0
  138. package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +115 -14
  139. package/node_modules/playwright/lib/mcp/test/skyRampExport.js +13 -1
  140. package/node_modules/playwright/node_modules/playwright-core/.DS_Store +0 -0
  141. package/node_modules/playwright/node_modules/playwright-core/lib/server/codegen/skyramp/jsonlReader.js +2 -0
  142. package/node_modules/playwright/node_modules/playwright-core/lib/vite/htmlReport/index.html +27 -253
  143. package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/assets/{codeMirrorModule-DtudTj_v.js → codeMirrorModule-DJMC4zNo.js} +1 -1
  144. package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/assets/index-BW82eAUI.js +196 -0
  145. package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/index.html +1 -1
  146. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{codeMirrorModule-FNMuBzX1.js → codeMirrorModule-CZfp96qZ.js} +1 -1
  147. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/defaultSettingsView-gpLo02E0.js +809 -0
  148. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.Bq1r1URj.js +2 -0
  149. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.html +2 -2
  150. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.VEfqi1qN.js +5 -0
  151. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.html +2 -2
  152. package/node_modules/playwright/node_modules/playwright-core/package.json +1 -1
  153. package/node_modules/playwright/node_modules/playwright-core/src/server/codegen/skyramp/jsonlReader.ts +1 -1
  154. package/node_modules/playwright/package.json +1 -1
  155. package/package.json +2 -2
  156. package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/assets/index-BpDwp16L.js +0 -422
  157. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/defaultSettingsView-Co9upU5h.js +0 -1035
  158. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.DXNIQ_dx.js +0 -2
  159. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.CIKB3XSv.js +0 -5
@@ -1,9 +1,17 @@
1
1
  import * as path from "path";
2
- import { readFile } from "fs/promises";
2
+ import { readFile, stat, access, constants as fsConstants } from "fs/promises";
3
+ import { realpathSync } from "fs";
4
+ import { execFile } from "child_process";
5
+ import { promisify } from "util";
3
6
  import { logger } from "../../utils/logger.js";
4
7
  import { resolveRunStatePath, StateManager, } from "../../utils/AnalysisStateManager.js";
5
- import { verifyReuse } from "../../utils/pom-verify/index.js";
6
- import { buildSkipped, findUnparsableKeptInline, flaggedFrom, mergeFlagged, verificationFrom, ReuseVerificationOutcome, } from "./reuse-outcome.js";
8
+ import { verifyReuse, } from "../../utils/pom-verify/index.js";
9
+ import { buildSkipped, findUnparsableKeptInline, flaggedFrom, mergeFlagged, verificationFrom, ReuseVerificationOutcome, HelperVerificationOutcome, } from "./reuse-outcome.js";
10
+ import { verifyUtils, realpath, utilsSpecFor, } from "../../utils/utils-verify/index.js";
11
+ import { detectGitRoot } from "../../utils/gitStaging.js";
12
+ const execFileAsync = promisify(execFile);
13
+ import { resolvePomCatalogPath } from "../../utils/pom-catalog.js";
14
+ import { parsePomCatalog, findMissedReuse, } from "../../utils/pom-catalog-parse.js";
7
15
  /**
8
16
  * Persistence for the server-derived POM-reuse outcome, keyed by test-file
9
17
  * basename (what `newTestsCreated[].fileName` carries, so
@@ -25,29 +33,55 @@ import { buildSkipped, findUnparsableKeptInline, flaggedFrom, mergeFlagged, veri
25
33
  async function updateReuseOutcome(testFile, patch, explicitStateFile) {
26
34
  const stateFile = resolveRunStatePath(explicitStateFile);
27
35
  if (!stateFile)
28
- return;
36
+ return false;
29
37
  try {
30
38
  const manager = StateManager.fromStatePath(stateFile);
31
39
  const data = await manager.readData();
32
40
  if (!data)
33
- return;
41
+ return false;
34
42
  const key = path.basename(testFile);
35
43
  const existing = data.reuseOutcomes?.[key] ?? {};
36
44
  await manager.writeData({
37
45
  ...data,
38
- reuseOutcomes: { ...data.reuseOutcomes, [key]: { ...existing, ...patch(existing) } },
46
+ reuseOutcomes: {
47
+ ...data.reuseOutcomes,
48
+ [key]: { ...existing, ...patch(existing) },
49
+ },
39
50
  });
51
+ return true;
40
52
  }
41
53
  catch (err) {
42
54
  logger.warning("Could not record POM-reuse outcome — report will omit the reuse summary", {
43
55
  testFile,
44
56
  error: String(err),
45
57
  });
58
+ return false;
46
59
  }
47
60
  }
48
- /** Record the tier-1 candidate count STEP 1 detected. */
49
- export async function recordCandidates(testFile, candidatesDetected, explicitStateFile) {
50
- await updateReuseOutcome(testFile, () => ({ candidatesDetected }), explicitStateFile);
61
+ /** Read one spec's record, or `undefined` when there is no run state, no record, or the
62
+ * state file cannot be read. Shares `updateReuseOutcome`'s resolve-read-basename plumbing
63
+ * so the two cannot disagree about which key a spec maps to. */
64
+ async function readReuseRecord(testFile, explicitStateFile) {
65
+ const stateFile = resolveRunStatePath(explicitStateFile);
66
+ if (!stateFile)
67
+ return undefined;
68
+ const data = await StateManager.fromStatePath(stateFile).readData();
69
+ return data?.reuseOutcomes?.[path.basename(testFile)];
70
+ }
71
+ /**
72
+ * Record the tier-1 candidate count STEP 1 detected, plus the identity needed to
73
+ * re-derive the counts from the delivered spec later.
74
+ *
75
+ * The identity is written here and not only by {@link recordVerifyOutcome} because
76
+ * the verify pass is a call the agent decides whether to make. When it skips it,
77
+ * a record holding the candidate count alone makes `rederiveReuseOutcome` bail out,
78
+ * and the report states "N candidate POM files detected" with no reuse numbers —
79
+ * indistinguishable from a run where reuse was measured and simply not summarized.
80
+ * Recording it unconditionally is what makes the zero-reuse case reportable without
81
+ * the agent's cooperation.
82
+ */
83
+ export async function recordCandidates(testFile, candidatesDetected, language, explicitStateFile) {
84
+ await updateReuseOutcome(testFile, () => ({ candidatesDetected, testFilePath: testFile, language }), explicitStateFile);
51
85
  }
52
86
  /** The prompt-generation call found no reusable POM layer at all. No candidate
53
87
  * count is recorded: "0 candidates" adds nothing to "no POM layer was found",
@@ -79,6 +113,549 @@ export async function recordVerifyOutcome(testFile, r, gateFired, language, expl
79
113
  };
80
114
  }, explicitStateFile);
81
115
  }
116
+ /**
117
+ * Has this spec been handed POM mapping instructions it never came back to verify?
118
+ *
119
+ * The enforcement half of SKYR-4115, and deliberately NOT a second copy of the
120
+ * zero-reuse policy. `zeroReuseGateFailure` is already the right check in the right
121
+ * place — inside `skyramp_reuse_code`'s verify branch. Its only defect was being
122
+ * reachable solely through a call the agent chooses to make. So this enforces the
123
+ * CALL, not the outcome: the gate then runs where it was designed to, with its own
124
+ * mapping instructions and its own documented-decline escape hatch.
125
+ *
126
+ * The debt needs no field of its own — it is exactly "detection recorded candidates
127
+ * and no verify pass has recorded a verdict". `recordCandidates` writes the former,
128
+ * `recordVerifyOutcome` always writes the latter, so ONE `verify: true` call clears
129
+ * this unconditionally. That is what makes a hard block safe without a retry budget:
130
+ * the agent cannot be stuck, because the action that clears it always succeeds.
131
+ *
132
+ * A FAILED verdict does NOT clear it. `formatVerifyReport` already tells the agent
133
+ * "do not finish until it reports PASSED" — enforcing that in prose is the exact
134
+ * failure this ticket exists to replace, and this is the one place positioned to
135
+ * enforce it in code.
136
+ *
137
+ * Blocking on a failure needs no retry budget either, because these checkpoints are
138
+ * NOT terminal. A PASS is always reachable in one edit plus one call (substitute the
139
+ * members, or document each candidate with a `// kept inline:` marker, which the gate
140
+ * accepts), and an agent that resolves neither can still walk past these tools to
141
+ * `skyramp_submit_report`, which does not block and reports the honest zero. So the
142
+ * worst case degrades to the reporting half rather than to a run that produces
143
+ * nothing — which is what a bound would have been protecting against.
144
+ */
145
+ export async function pendingReuseVerification(testFile, explicitStateFile) {
146
+ try {
147
+ const record = await readReuseRecord(testFile, explicitStateFile);
148
+ // candidatesDetected is written only for a POM-aware target whose candidates came
149
+ // from the conventional globs, so its presence is what limits this to specs that
150
+ // actually had a page-object layer to reuse.
151
+ if (!record?.candidatesDetected)
152
+ return undefined;
153
+ // `reuseOutcomes` is keyed by BASENAME across the whole run, so in a multi-repo run
154
+ // two specs can share one record. Blocking a tool on another spec's debt is worse
155
+ // than missing this one, so a mismatch fails open. `recordCandidates` always writes
156
+ // the path, which is what makes the owner checkable at all.
157
+ if (record.testFilePath && !samePath(record.testFilePath, testFile))
158
+ return undefined;
159
+ const cleared = record.verification === ReuseVerificationOutcome.Passed ||
160
+ record.verification === ReuseVerificationOutcome.SkippedNoPom;
161
+ if (cleared)
162
+ return undefined;
163
+ const spec = path.basename(testFile);
164
+ const state = record.verification === ReuseVerificationOutcome.Failed
165
+ ? `the last verification of ${spec} did not pass`
166
+ : `the refactor of ${spec} was never verified`;
167
+ return (`${record.candidatesDetected} reusable page-object candidate${record.candidatesDetected === 1 ? "" : "s"} ` +
168
+ `were detected and ${state}. Call skyramp_reuse_code with verify: true on this file and resolve what it ` +
169
+ `reports — it names the POM members the spec should use, and accepts a documented decline ` +
170
+ `(\`// kept inline: <POM file basename> — <reason>\`) when a candidate genuinely cannot cover the test.`);
171
+ }
172
+ catch (err) {
173
+ // Fail open: an unreadable state file is not evidence the agent skipped anything.
174
+ logger.warning("Could not check for a pending POM-reuse verification — proceeding", {
175
+ testFile,
176
+ error: String(err),
177
+ });
178
+ return undefined;
179
+ }
180
+ }
181
+ // ---------------------------------------------------------------------------
182
+ // SkyrampUtils (shared-helper) path — SKYR-4220
183
+ // ---------------------------------------------------------------------------
184
+ /**
185
+ * The utils path handed out its reuse instructions for this spec. This is the debt:
186
+ * from here until a `verify: true` pass records PASSED or NoUtilsFile, the tools that
187
+ * follow reuse on the same spec refuse.
188
+ *
189
+ * Same shape as {@link recordCandidates} for the POM path, and for the same reason:
190
+ * the verify call is the agent's to make, and the utils file it stages and checks is
191
+ * the artifact a skipped call would ship broken (SKYR-4196 Part B run 2). Recorded
192
+ * unconditionally on the utils path — even a spec that ends up moving nothing clears
193
+ * cheaply (NoUtilsFile), and the cost of a skipped debt is a test importing a module
194
+ * the PR does not contain.
195
+ */
196
+ export async function recordUtilsReuseIssued(testFile, language, framework, testType, explicitStateFile) {
197
+ await updateReuseOutcome(testFile, (existing) => ({
198
+ testFilePath: testFile,
199
+ language,
200
+ // A re-issued prompt (the agent calling discovery twice) must not erase a
201
+ // verdict already earned; only the identity is refreshed. But a record OWNED
202
+ // BY ANOTHER SPEC (same basename, another directory) is not this spec's history:
203
+ // inheriting its verdict would let this spec skip verify entirely — a debt born
204
+ // cleared — so ownership is checked on the write, not only on the read.
205
+ // Ownership change resets the WHOLE record: a foreign spec's POM facts must not
206
+ // make this spec owe the foreign POM debt.
207
+ ...(owns(existing, testFile) ? {} : freshRecord()),
208
+ utils: {
209
+ ...(owns(existing, testFile) ? existing.utils : {}),
210
+ issued: true,
211
+ framework,
212
+ testType,
213
+ },
214
+ }), explicitStateFile);
215
+ }
216
+ /** Every field cleared — the patch spread onto a record owned by another spec. */
217
+ function freshRecord() {
218
+ return {
219
+ candidatesDetected: undefined,
220
+ verification: undefined,
221
+ flagged: undefined,
222
+ language: undefined,
223
+ utils: undefined,
224
+ modularized: undefined,
225
+ };
226
+ }
227
+ /** A verify pass that threw. Recorded so the debt FAILS OPEN — a verifier that cannot
228
+ * run must not leave a debt nothing can pay — and cleared by the next successful pass. */
229
+ export async function recordUtilsVerifyError(testFile, explicitStateFile) {
230
+ await updateReuseOutcome(testFile, (existing) => owns(existing, testFile) && existing.utils
231
+ ? { utils: { ...existing.utils, verifyError: true } }
232
+ : {}, explicitStateFile);
233
+ }
234
+ /** Whether `record` describes `testFile` (or nobody yet). Basename-keyed run state
235
+ * means two specs can share one record; only the owner's history carries over. */
236
+ function owns(record, testFile) {
237
+ return !record.testFilePath || samePath(record.testFilePath, testFile);
238
+ }
239
+ /** Canonical comparison: a record written through a symlinked path (macOS /var →
240
+ * /private/var) must still be recognised as this spec's when checked by realpath. */
241
+ function samePath(a, b) {
242
+ return canon(a) === canon(b);
243
+ }
244
+ function canon(p) {
245
+ try {
246
+ return realpathSync(p);
247
+ }
248
+ catch {
249
+ return path.resolve(p);
250
+ }
251
+ }
252
+ /** Verdict of one utils verify pass. The verdict alone is stored — counts and the
253
+ * file list are re-derived from the delivered files at report time. */
254
+ export async function recordUtilsVerifyOutcome(testFile, r, language, framework, testType, explicitStateFile) {
255
+ const verification = r.utilsFiles.length === 0
256
+ ? HelperVerificationOutcome.NoUtilsFile
257
+ : r.ok
258
+ ? HelperVerificationOutcome.Passed
259
+ : HelperVerificationOutcome.Failed;
260
+ return updateReuseOutcome(testFile, (existing) => {
261
+ const mine = owns(existing, testFile);
262
+ const prior = mine ? existing.utils : undefined;
263
+ return {
264
+ ...(mine ? {} : freshRecord()),
265
+ testFilePath: testFile,
266
+ language,
267
+ // A verify without a prior discovery records the verdict but NOT a debt:
268
+ // `issued` stays false, so a spec that was never handed reuse instructions is
269
+ // never blocked by this record. The test type is persisted from the verify
270
+ // call itself, so the report re-derives with the same helper family the gate
271
+ // used (a browser helper's toBeVisible must not become a body assertion at
272
+ // report time).
273
+ utils: {
274
+ issued: prior?.issued ?? false,
275
+ framework: prior?.framework || framework,
276
+ testType: testType ?? prior?.testType,
277
+ verification,
278
+ verifyError: false,
279
+ },
280
+ };
281
+ }, explicitStateFile);
282
+ }
283
+ /** Whether the run state file can be written — the precondition for any debt this
284
+ * module holds, since only a write clears it. */
285
+ async function runStateWritable(explicitStateFile) {
286
+ const stateFile = resolveRunStatePath(explicitStateFile);
287
+ if (!stateFile)
288
+ return false;
289
+ try {
290
+ await access(stateFile, fsConstants.W_OK);
291
+ return true;
292
+ }
293
+ catch {
294
+ return false;
295
+ }
296
+ }
297
+ /** `skyramp_modularization` was called for this spec. Ownership on the write, like
298
+ * the utils debt: a same-basename spec elsewhere does not inherit the mark. */
299
+ export async function recordModularized(testFile, explicitStateFile) {
300
+ await updateReuseOutcome(testFile, (existing) => owns(existing, testFile)
301
+ ? { testFilePath: existing.testFilePath ?? testFile, modularized: true }
302
+ : {
303
+ testFilePath: testFile,
304
+ modularized: true,
305
+ utils: undefined,
306
+ verification: undefined,
307
+ candidatesDetected: undefined,
308
+ flagged: undefined,
309
+ }, explicitStateFile);
310
+ }
311
+ /**
312
+ * Does this spec owe `skyramp_modularization` before reuse may be issued? Only a spec
313
+ * generation handed off (modularize-first flows: the hand-off names both calls, in
314
+ * order) and only until the mark exists. Measured: eval run u4219f — the agent skipped
315
+ * modularization for both new tests and went straight to reuse; the reuse half was
316
+ * enforced, the modularization half was prose. Fails open without a hand-off, on a
317
+ * record owned by another spec, on an unwritable or unreadable state file.
318
+ */
319
+ export async function pendingModularization(testFile, explicitStateFile) {
320
+ try {
321
+ const stateFile = resolveRunStatePath(explicitStateFile);
322
+ if (!stateFile)
323
+ return undefined;
324
+ const data = await StateManager.fromStatePath(stateFile).readData();
325
+ const handOff = data?.reuseHandOffs?.[await realpath(testFile)];
326
+ if (!handOff)
327
+ return undefined;
328
+ const record = data?.reuseOutcomes?.[path.basename(testFile)];
329
+ if (record && !owns(record, testFile))
330
+ return undefined;
331
+ if (record?.modularized)
332
+ return undefined;
333
+ if (!(await runStateWritable(explicitStateFile)))
334
+ return undefined;
335
+ return (`${path.basename(testFile)} was generated with code reuse enabled, and its flow modularizes BEFORE reuse: ` +
336
+ `modularization is what creates the named helpers reuse consolidates. Call skyramp_modularization with ` +
337
+ `{ testFile: "${testFile}", testType: "${handOff.testType}", isTraceBased: true }, apply the returned instructions to the file, ` +
338
+ `then call skyramp_reuse_code again with the same arguments.`);
339
+ }
340
+ catch (err) {
341
+ logger.warning("Could not check for a pending modularization — proceeding", {
342
+ testFile,
343
+ error: String(err),
344
+ });
345
+ return undefined;
346
+ }
347
+ }
348
+ /** The exact `skyramp_reuse_code` verify call for a spec — handed to the agent verbatim
349
+ * so the gate never depends on the agent interpolating a placeholder correctly. */
350
+ function utilsVerifyCall(record, testFile) {
351
+ const args = [
352
+ `testFile: "${testFile}"`,
353
+ `language: "${record.language ?? ""}"`,
354
+ `framework: "${record.utils?.framework ?? ""}"`,
355
+ record.utils?.testType ? `testType: "${record.utils.testType}"` : undefined,
356
+ "verify: true",
357
+ ].filter(Boolean);
358
+ return `skyramp_reuse_code with { ${args.join(", ")} }`;
359
+ }
360
+ /**
361
+ * Has this spec been handed SkyrampUtils reuse instructions it never came back to
362
+ * verify? The utils-path twin of {@link pendingReuseVerification}, needed because that
363
+ * predicate keys on `candidatesDetected`, which only a POM-aware target writes — a
364
+ * utils debt riding it would fail open on every repo without a page-object layer,
365
+ * precisely the population this exists for.
366
+ *
367
+ * Failure mode, stated: an unresolved utils debt BLOCKS execution. On the POM path
368
+ * a missed verify degrades a report; here it ships a test importing a file the PR
369
+ * does not contain, which cannot run at all — so losing the execution attempt is
370
+ * the cheaper failure. Same no-retry-budget reasoning as the POM twin: PASSED is one
371
+ * edit plus one call away (fix the invariant, or document it with the
372
+ * `reuse-verify: allow` line the verify text hands over), NoUtilsFile clears it for
373
+ * free, and `skyramp_submit_report` never reads it, so nothing hangs.
374
+ */
375
+ export async function pendingUtilsVerification(testFile, explicitStateFile) {
376
+ try {
377
+ const record = await readReuseRecord(testFile, explicitStateFile);
378
+ if (record?.utils?.issued !== true)
379
+ return undefined;
380
+ // The last verify pass threw: the agent was told not to loop on it, so the debt
381
+ // must not hold either.
382
+ if (record.utils.verifyError)
383
+ return undefined;
384
+ // Basename-keyed across the run — fail open on another spec's record (see the POM twin).
385
+ if (record.testFilePath && !samePath(record.testFilePath, testFile))
386
+ return undefined;
387
+ // A debt only a state write can clear must not outlive the ability to write. If the
388
+ // state file is not writable, the verify pass has already said PASSED and could not
389
+ // record it — refusing here would leave no action that clears the block.
390
+ if (!(await runStateWritable(explicitStateFile)))
391
+ return undefined;
392
+ const v = record.utils.verification;
393
+ if (v === HelperVerificationOutcome.Passed ||
394
+ v === HelperVerificationOutcome.NoUtilsFile)
395
+ return undefined;
396
+ const spec = path.basename(testFile);
397
+ const state = v === HelperVerificationOutcome.Failed
398
+ ? `its last shared-helper verification did not pass`
399
+ : `its shared-helper reuse was never verified`;
400
+ return (`${spec} was handed code-reuse instructions and ${state}. Call ${utilsVerifyCall(record, testFile)} ` +
401
+ `and resolve what it reports — it stages the shared utils file for the output commit and checks its invariants ` +
402
+ `(API helpers: one helper per method+path, status-code-only assertions, method+resource names; browser helpers: actions and structural waits only). It passes trivially when no ` +
403
+ `utils file was written, and accepts a documented decline per violation (the exact \`reuse-verify: allow\` line is in its output).`);
404
+ }
405
+ catch (err) {
406
+ logger.warning("Could not check for a pending shared-helper verification — proceeding", {
407
+ testFile,
408
+ error: String(err),
409
+ });
410
+ return undefined;
411
+ }
412
+ }
413
+ /**
414
+ * Generation emitted a modularize→reuse hand-off for the tests it wrote into
415
+ * `outputDir`. Recorded at the SAME site and under the SAME condition as the
416
+ * hand-off text, so the debt can never be owed for a test that was never told to
417
+ * pay it (a gate that cannot be satisfied is the SKYR-4115 failure inverted).
418
+ */
419
+ export async function recordReuseHandOff(files, testType, language, framework, explicitStateFile) {
420
+ if (files.length === 0)
421
+ return;
422
+ // A language the utils verifier does not cover (Java) has no payable reuse half —
423
+ // nothing on the reuse side records for it — so no hand-off is owed either.
424
+ if (!utilsSpecFor(language))
425
+ return;
426
+ const stateFile = resolveRunStatePath(explicitStateFile);
427
+ if (!stateFile)
428
+ return;
429
+ try {
430
+ const manager = StateManager.fromStatePath(stateFile);
431
+ const data = await manager.readData();
432
+ if (!data)
433
+ return;
434
+ const entries = {};
435
+ for (const f of files)
436
+ entries[await realpath(f)] = { testType, language, framework };
437
+ await manager.writeData({
438
+ ...data,
439
+ reuseHandOffs: { ...data.reuseHandOffs, ...entries },
440
+ });
441
+ }
442
+ catch (err) {
443
+ logger.warning("Could not record the reuse hand-off — chain compliance will not be checked", {
444
+ files,
445
+ error: String(err),
446
+ });
447
+ }
448
+ }
449
+ /** True when `file` is not in HEAD — i.e. this run created it. The chain debt applies
450
+ * only to tests generation wrote; pre-existing tests in the same directory are
451
+ * executed by the same tool and must never be held to it. Fails open (false) when
452
+ * git cannot answer. */
453
+ async function isNewInRepo(file) {
454
+ const root = detectGitRoot(file);
455
+ if (!root)
456
+ return false;
457
+ // Both sides canonical: git reports the REAL root, and a tool-supplied path through
458
+ // a symlink (macOS /tmp → /private/tmp) would otherwise produce a `../../` relative
459
+ // path that `cat-file` cannot resolve — reading every pre-existing test as new
460
+ // (measured: eval run 2026-08-26 blocked two maintained tests on the chain debt).
461
+ const rel = path
462
+ .relative(await realpath(root), await realpath(file))
463
+ .split(path.sep)
464
+ .join("/");
465
+ if (rel.startsWith(".."))
466
+ return false;
467
+ try {
468
+ await execFileAsync("git", ["cat-file", "-e", `HEAD:${rel}`], {
469
+ cwd: root,
470
+ });
471
+ return false; // present at HEAD → pre-existing
472
+ }
473
+ catch (err) {
474
+ // Only "absent from HEAD" (or no HEAD yet) means new. Anything else — git missing,
475
+ // a corrupt object store, a permission error — is git not answering, and the
476
+ // documented answer to that is false: never block, never accuse, on a malfunction.
477
+ const e = err;
478
+ const text = `${e.stderr ?? ""} ${String(err)}`;
479
+ if (/does not exist|Not a valid object name|bad revision|exists on disk, but not in/i.test(text))
480
+ return true;
481
+ logger.warning("Could not determine whether a file is new in the repository — treating it as pre-existing", {
482
+ file,
483
+ error: String(err),
484
+ });
485
+ return false;
486
+ }
487
+ }
488
+ /**
489
+ * Does this spec owe the modularize→reuse pair its generation handed off? The
490
+ * chain-compliance half of SKYR-4220: 2 of 8 SKYR-4196 Part B runs wrote a second
491
+ * integration test's imports by hand and skipped the pair. A hand-off of the spec's
492
+ * test type exists for its directory (integration, or ui since SKYR-4224), the spec
493
+ * is new in this run, and no reuse pass was ever recorded for it → refuse, naming
494
+ * the two calls. Fails open on everything else.
495
+ */
496
+ export async function pendingReuseChain(testFile, explicitStateFile, testType) {
497
+ try {
498
+ const stateFile = resolveRunStatePath(explicitStateFile);
499
+ if (!stateFile)
500
+ return undefined;
501
+ const data = await StateManager.fromStatePath(stateFile).readData();
502
+ // Keyed by the GENERATED FILE: generation snapshots its outputDir before and after
503
+ // the codegen call and records exactly the files it wrote, so a test generated
504
+ // without codeReuse into the same directory owes nothing, and neither does a
505
+ // pre-existing test beside it.
506
+ const handOff = data?.reuseHandOffs?.[await realpath(testFile)];
507
+ if (!handOff)
508
+ return undefined;
509
+ // A caller that knows the file's type and disagrees with the hand-off is exempt;
510
+ // an unknown type still owes (the hand-off is authoritative for this file).
511
+ if (testType && testType !== handOff.testType)
512
+ return undefined;
513
+ const record = data?.reuseOutcomes?.[path.basename(testFile)];
514
+ // A FOREIGN record fails open (refusing this spec over another spec's history is
515
+ // the worse error). The OWNER's record clears only if reuse actually ran for it —
516
+ // a modularization mark or a bare verify verdict (`issued: false`) is not reuse,
517
+ // and treating it as such would let an agent modularize, skip skyramp_reuse_code,
518
+ // and execute.
519
+ if (record && !owns(record, testFile))
520
+ return undefined;
521
+ if (record && reuseRan(record))
522
+ return undefined;
523
+ if (!(await isNewInRepo(testFile)))
524
+ return undefined;
525
+ return (`${path.basename(testFile)} was generated with code reuse enabled, but its modularize→reuse steps were never run. ` +
526
+ `First call skyramp_modularization with { testFile: "${testFile}", testType: "${handOff.testType}", isTraceBased: true } and apply the result; ` +
527
+ `then, as the very next skyramp call, skyramp_reuse_code with { testFile: "${testFile}", language: "${handOff.language}", framework: "${handOff.framework}", testType: "${handOff.testType}" } ` +
528
+ `and follow its steps (including its final verify: true call). A "nothing to reuse" result is normal — do not retry.`);
529
+ }
530
+ catch (err) {
531
+ logger.warning("Could not check for a pending modularize→reuse chain — proceeding", {
532
+ testFile,
533
+ error: String(err),
534
+ });
535
+ return undefined;
536
+ }
537
+ }
538
+ /** Every reuse-phase debt a downstream tool must refuse on, first one wins. One call
539
+ * site for both checkpoints so they cannot drift apart. */
540
+ export async function pendingReuseDebt(testFile, explicitStateFile, testType, opts = {}) {
541
+ return ((await pendingReuseVerification(testFile, explicitStateFile)) ??
542
+ (await pendingUtilsVerification(testFile, explicitStateFile)) ??
543
+ // The chain debt belongs to the LAST checkpoint only: the UI utils flow runs
544
+ // skyramp_enhance_assertions BEFORE modularize→reuse by design, so checking the
545
+ // chain there would refuse the very first call the generator prescribes.
546
+ (opts.chain === false
547
+ ? undefined
548
+ : await pendingReuseChain(testFile, explicitStateFile, testType)));
549
+ }
550
+ /**
551
+ * Report-time chain verdict for one report row. `fileName` is a basename, so it is
552
+ * matched against the basenames of the handed-off files of the same test type. True
553
+ * only when such a file exists, is new in this run, and no reuse record was ever
554
+ * written for that basename — never `false`, so a consumer omits rather than renders a
555
+ * negative.
556
+ */
557
+ export async function reuseChainSkipped(fileName, testType, record, handOffs) {
558
+ // Same rule as the gate: a foreign record fails open; the owner's record suppresses
559
+ // the signal only when reuse was actually issued for it.
560
+ if (!handOffs)
561
+ return undefined;
562
+ const base = path.basename(fileName);
563
+ for (const [file, h] of Object.entries(handOffs)) {
564
+ if (h.testType !== testType || path.basename(file) !== base)
565
+ continue;
566
+ if (record) {
567
+ if (!owns(record, file))
568
+ continue; // foreign basename — fail open for this file
569
+ if (reuseRan(record))
570
+ return undefined;
571
+ }
572
+ try {
573
+ if (!(await stat(file)).isFile())
574
+ continue;
575
+ }
576
+ catch {
577
+ continue;
578
+ }
579
+ // The hand-off names files generation wrote, so newness is implied — but the
580
+ // guard is kept for state written by an older recorder that keyed directories.
581
+ if (await isNewInRepo(file))
582
+ return true;
583
+ }
584
+ return undefined;
585
+ }
586
+ /** Whether a record shows the reuse tool ran for its spec: the utils path issued its
587
+ * instructions, or the POM path recorded detection/verification. A modularization mark
588
+ * alone, or a verify verdict recorded without a prior discovery, is NOT reuse. */
589
+ function reuseRan(record) {
590
+ return (record.utils?.issued === true ||
591
+ record.candidatesDetected !== undefined ||
592
+ record.verification !== undefined);
593
+ }
594
+ /**
595
+ * Re-derive the shared-helper outcome from the delivered files. Everything is
596
+ * measured now: the utils file is located by header, its invariants re-checked, the
597
+ * imports re-counted — the recorded verdict is not consulted, so a utils file
598
+ * rewritten after the verify pass is described as it stands. Returns `undefined`
599
+ * (omit `helpers`) when no utils file exists for the spec.
600
+ */
601
+ async function rederiveHelperOutcome(record) {
602
+ const { testFilePath, language } = record;
603
+ if (!testFilePath || !language)
604
+ return undefined;
605
+ const r = await verifyUtils({
606
+ testFile: testFilePath,
607
+ language,
608
+ testType: record.utils?.testType,
609
+ });
610
+ if (r.utilsFiles.length === 0)
611
+ return {};
612
+ return {
613
+ helpers: {
614
+ utilsFile: r.utilsFiles.map((f) => path.basename(f)).join(", "),
615
+ helpersImported: r.imported,
616
+ ...(r.inlineCallSites.length > 0
617
+ ? { siblingInlineCallSites: r.inlineCallSites.length }
618
+ : {}),
619
+ verification: r.ok
620
+ ? HelperVerificationOutcome.Passed
621
+ : HelperVerificationOutcome.Failed,
622
+ },
623
+ };
624
+ }
625
+ /**
626
+ * Raw-locator coverage for the delivered spec, or `undefined` when it cannot be
627
+ * established.
628
+ *
629
+ * Fails open at every step. No catalog is the common case — a POM-less repo, or a
630
+ * run where the agent never reached STEP 2 — and that case stays silent: it is
631
+ * expected, not a defect. An unreadable or unparseable catalog is different — the
632
+ * file exists but the agent wrote something this cannot use — and that case is
633
+ * logged, per the design's failure-mode contract, so the gap is diagnosable
634
+ * without a customer-facing symptom. Either way, no number reaches the report that
635
+ * is not backed by the spec on disk.
636
+ */
637
+ async function coverageFrom(testFilePath, specContent) {
638
+ const catalogPath = resolvePomCatalogPath(testFilePath);
639
+ try {
640
+ const markdown = await readFile(catalogPath, "utf8");
641
+ const members = parsePomCatalog(markdown);
642
+ if (members.length === 0)
643
+ return undefined;
644
+ return findMissedReuse(specContent, members);
645
+ }
646
+ catch (err) {
647
+ // ENOENT (no catalog at all) is the common, expected case — silent. Anything
648
+ // else (permission error, unreadable file) means a catalog exists but could
649
+ // not be used, which the design's failure-modes table promises a warning for.
650
+ if (err?.code !== "ENOENT") {
651
+ logger.warning("Could not read the POM catalog — omitting reuse coverage from the report", {
652
+ catalogPath,
653
+ error: String(err),
654
+ });
655
+ }
656
+ return undefined;
657
+ }
658
+ }
82
659
  /**
83
660
  * Re-derive a recorded outcome from the spec as it stands NOW, so the report
84
661
  * describes the delivered artifact rather than the state at verify time.
@@ -99,14 +676,28 @@ export async function recordVerifyOutcome(testFile, r, gateFired, language, expl
99
676
  * reads as though the empty result had been blessed.
100
677
  *
101
678
  * Returns the recorded outcome minus internals when re-derivation is impossible
102
- * (no path recorded — e.g. only the candidate count was ever written), and
103
- * `undefined` when it fails outright. Failing closed matters: falling back to the
104
- * recorded counts is exactly the false claim this exists to prevent.
679
+ * (no path recorded — now only the no-POM-layer path, which has no spec to measure
680
+ * and no candidates to contrast against), and `undefined` when it fails outright.
681
+ * Failing closed matters: falling back to the recorded counts is exactly the false
682
+ * claim this exists to prevent.
105
683
  */
106
684
  export async function rederiveReuseOutcome(record) {
107
685
  const { candidatesDetected, verification, flagged, testFilePath, language } = record;
108
- // No verify pass ran for this spec, so there are no counts to derive — only the
109
- // detection-time facts, which no spec change can invalidate.
686
+ // A SkyrampUtils-path record has nothing for the POM verifier to measure; its
687
+ // outcome is re-derived from the utils file instead.
688
+ if (record.utils) {
689
+ try {
690
+ return await rederiveHelperOutcome(record);
691
+ }
692
+ catch (err) {
693
+ logger.warning("Could not re-derive the shared-helper outcome from the delivered files — omitting it rather than reporting a stale verdict", { testFile: testFilePath, error: String(err) });
694
+ return undefined;
695
+ }
696
+ }
697
+ // No spec identity was ever recorded, so there is nothing to measure — only the
698
+ // detection-time facts, which no spec change can invalidate. Reached by the
699
+ // no-POM-layer path; a run that detected candidates always records the identity
700
+ // (see recordCandidates), precisely so the zero-reuse case does not land here.
110
701
  if (!testFilePath || !language)
111
702
  return { candidatesDetected, verification };
112
703
  try {
@@ -122,14 +713,35 @@ export async function rederiveReuseOutcome(record) {
122
713
  "Use `// kept inline: <file basename> — <reason>` (em dash, or a hyphen with spaces around it).", { testFile: testFilePath, markers: unparsable });
123
714
  }
124
715
  const skipped = buildSkipped(specContent, flagged);
716
+ const coverage = await coverageFrom(testFilePath, specContent);
717
+ const reuseHappened = r.reusedCalls > 0;
125
718
  return {
126
719
  candidatesDetected,
127
720
  callsReused: r.reusedCalls,
128
- callsVerified: r.checkedCalls,
721
+ // Only a POM call can be checked, so `checkedCalls` is a subset of
722
+ // `reusedCalls` and at zero reuse both it and the unverifiable remainder
723
+ // are structurally implied. Emitting them anyway is worse than redundant:
724
+ // two equal counts render as "0 POM calls reused and verified", asserting
725
+ // that a verification blessed the empty result. The pair is dropped
726
+ // together so `callsVerified + callsUnverifiable === callsReused` stays
727
+ // statable. Same reasoning as dropping `verification` just below.
728
+ ...(reuseHappened
729
+ ? {
730
+ callsVerified: r.checkedCalls,
731
+ // From the counts, not from the list: these are the numbers a renderer
732
+ // subtracts with, and they must close.
733
+ callsUnverifiable: r.reusedCalls - r.checkedCalls,
734
+ }
735
+ : {}),
736
+ unverifiableCalls: r.unverifiableCalls.length > 0 ? r.unverifiableCalls : undefined,
737
+ // Raw-locator coverage describes the delivered spec, not the reuse count, so
738
+ // it is reported at zero reuse too — that is where it says the most.
739
+ rawLocatorsInline: coverage?.rawLocatorsInline,
740
+ missedReuse: coverage && coverage.missed.length > 0 ? coverage.missed : undefined,
129
741
  // A PASSED verdict describes a spec that no longer exists once reuse is gone,
130
742
  // and pairing it with a zero count reads as though the empty result had been
131
743
  // blessed.
132
- verification: r.reusedCalls > 0 ? verification : undefined,
744
+ verification: reuseHappened ? verification : undefined,
133
745
  skipped: skipped.length > 0 ? skipped : undefined,
134
746
  };
135
747
  }