@skyramp/mcp 0.4.0 → 0.4.1-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/build/prompts/enhance-assertions/contractProviderAssertionsPrompt.js +2 -1
  2. package/build/prompts/enhance-assertions/integrationAssertionsPrompt.js +2 -1
  3. package/build/prompts/enhance-assertions/sharedAssertionRules.d.ts +1 -1
  4. package/build/prompts/enhance-assertions/sharedAssertionRules.js +57 -22
  5. package/build/prompts/enhance-assertions/uiAssertionsPrompt.js +17 -9
  6. package/build/prompts/test-recommendation/diffExecutionPlan.js +0 -2
  7. package/build/prompts/test-recommendation/test-recommendation-prompt.js +11 -6
  8. package/build/prompts/testbot/testbot-prompts.js +19 -12
  9. package/build/recommendation/answers.d.ts +11 -7
  10. package/build/recommendation/answers.js +14 -10
  11. package/build/recommendation/pullRequestText.d.ts +18 -0
  12. package/build/recommendation/pullRequestText.js +31 -0
  13. package/build/recommendation/registerPlan.d.ts +9 -1
  14. package/build/recommendation/registerPlan.js +8 -1
  15. package/build/recommendation/runVerifiers.js +6 -0
  16. package/build/recommendation/types.d.ts +58 -0
  17. package/build/recommendation/verifierContracts.d.ts +108 -13
  18. package/build/recommendation/verifierContracts.js +147 -29
  19. package/build/recommendation/verifiers/coverage.d.ts +10 -0
  20. package/build/recommendation/verifiers/coverage.js +144 -22
  21. package/build/recommendation/verifiers/defects.d.ts +9 -0
  22. package/build/recommendation/verifiers/defects.js +117 -0
  23. package/build/recommendation/verifiers/deliveredMatchesPlan.d.ts +22 -0
  24. package/build/recommendation/verifiers/deliveredMatchesPlan.js +43 -0
  25. package/build/recommendation/verifiers/existingCoverage.js +53 -0
  26. package/build/recommendation/verifiers/expectedValueSourced.d.ts +14 -0
  27. package/build/recommendation/verifiers/expectedValueSourced.js +246 -0
  28. package/build/recommendation/verifiers/issueTraceability.d.ts +52 -0
  29. package/build/recommendation/verifiers/issueTraceability.js +197 -0
  30. package/build/recommendation/verifiers/requirementSourced.d.ts +2 -0
  31. package/build/recommendation/verifiers/requirementSourced.js +168 -0
  32. package/build/services/TestGenerationService.js +3 -1
  33. package/build/tools/code-refactor/codeReuseTool.js +1 -1
  34. package/build/tools/code-refactor/reuse-outcome.d.ts +1 -1
  35. package/build/tools/code-refactor/reuse-state.d.ts +85 -7
  36. package/build/tools/code-refactor/reuse-state.js +239 -34
  37. package/build/tools/code-refactor/utils-verify-gates.d.ts +5 -0
  38. package/build/tools/code-refactor/utils-verify-gates.js +103 -11
  39. package/build/tools/generate-tests/generateBatchScenarioRestTool.js +2 -1
  40. package/build/tools/submitReportTool.js +276 -39
  41. package/build/tools/test-management/actionsTool.js +5 -0
  42. package/build/tools/test-management/analyzeChangesTool.d.ts +53 -0
  43. package/build/tools/test-management/analyzeChangesTool.js +55 -2
  44. package/build/tools/test-management/registerTestPlanTool.d.ts +32 -17
  45. package/build/tools/test-management/registerTestPlanTool.js +105 -16
  46. package/build/types/ReuseOutcome.d.ts +73 -7
  47. package/build/types/TestAnalysis.d.ts +6 -0
  48. package/build/types/TestbotReport.d.ts +22 -4
  49. package/build/utils/AnalysisStateManager.d.ts +7 -1
  50. package/build/utils/AnalysisStateManager.js +5 -1
  51. package/build/utils/assertion-verify/api-shared-lints.js +118 -11
  52. package/build/utils/assertion-verify/format.js +2 -2
  53. package/build/utils/assertion-verify/helper-imports.d.ts +7 -0
  54. package/build/utils/assertion-verify/helper-imports.js +119 -27
  55. package/build/utils/assertion-verify/lint-types.d.ts +31 -2
  56. package/build/utils/assertion-verify/lint-types.js +66 -0
  57. package/build/utils/assertion-verify/metrics.d.ts +13 -0
  58. package/build/utils/assertion-verify/metrics.js +16 -0
  59. package/build/utils/assertion-verify/verify.d.ts +11 -6
  60. package/build/utils/assertion-verify/verify.js +56 -15
  61. package/build/utils/canonicalJson.d.ts +11 -0
  62. package/build/utils/canonicalJson.js +17 -0
  63. package/build/utils/utils-verify/action-key.d.ts +27 -0
  64. package/build/utils/utils-verify/action-key.js +292 -0
  65. package/build/utils/utils-verify/allow.d.ts +8 -1
  66. package/build/utils/utils-verify/allow.js +14 -1
  67. package/build/utils/utils-verify/call-sites.d.ts +76 -8
  68. package/build/utils/utils-verify/call-sites.js +256 -70
  69. package/build/utils/utils-verify/language-spec.d.ts +3 -2
  70. package/build/utils/utils-verify/parse.d.ts +22 -3
  71. package/build/utils/utils-verify/parse.js +123 -52
  72. package/build/utils/utils-verify/verify.d.ts +33 -3
  73. package/build/utils/utils-verify/verify.js +126 -12
  74. package/build/utils/workspaceAuth.d.ts +59 -19
  75. package/build/utils/workspaceAuth.js +228 -31
  76. package/package.json +1 -1
  77. package/plugin/prompts/generate-tests/execution-plan.md +3 -3
  78. package/plugin/prompts/generate-tests/generation.md +1 -0
  79. package/plugin/prompts/plan-tests.md +41 -15
  80. package/plugin/prompts/testbot-task1.md +3 -9
  81. package/build/prompts/testbot/planDeclarations.d.ts +0 -6
  82. package/build/prompts/testbot/planDeclarations.js +0 -9
  83. package/plugin/prompts/declaring-a-plan.md +0 -20
@@ -1,4 +1,5 @@
1
1
  import type { TestExecutionStatus } from "./TestExecution.js";
2
+ import type { UtilsViolationKind } from "../utils/utils-verify/allow.js";
2
3
  /**
3
4
  * The code-reuse observability shape carried in a TestbotReport — the wire contract
4
5
  * between this server and the consumers that render it (test-bot.git). It covers
@@ -86,6 +87,26 @@ export declare enum HelperVerificationOutcome {
86
87
  /** The test neither wrote nor imports a shared utils file. */
87
88
  NoUtilsFile = "no-utils-file"
88
89
  }
90
+ /** Every blocking kind a report-time reuse verification can find. Shared-helper
91
+ * path: the `UtilsViolationKind`s, plus `malformed-allow-marker` for an allow
92
+ * marker the grammar cannot read (which fails a verify on its own). Page-object
93
+ * path: a POM member the spec calls that its source does not define, a spec that
94
+ * does not parse, a spec whose Skyramp runtime imports were stripped, and a spec
95
+ * with no page-object reuse whose selectors match detected page objects and carry
96
+ * no `// kept inline:` decline. A union, so a misspelled kind is a compile error. */
97
+ export type ReuseVerificationFailureKind = UtilsViolationKind | "malformed-allow-marker" | "pom-member-missing" | "parse-error" | "skyramp-infra-missing" | "zero-reuse";
98
+ /** One blocking kind a failed reuse verification found in the delivered files,
99
+ * aggregated: the verifier reports a finding per helper, import line or POM call,
100
+ * and the row states the kind once with its count. `detail` is the findings' own
101
+ * detail text (joined with `; ` when there are several) — for an `assertion-loss`
102
+ * that is the sentence `verify.ts` builds with the counts behind the verdict
103
+ * (baseline, delivered, in the test, in the helpers); it is NOT the gate's
104
+ * PASSED/FAILED prose. */
105
+ export interface ReuseVerificationFailure {
106
+ kind: ReuseVerificationFailureKind;
107
+ count: number;
108
+ detail: string;
109
+ }
89
110
  /** A pre-existing Skyramp-generated test the reuse pass edited to import from the
90
111
  * shared utils file. `execution` is the recorded post-edit run; a retrofit without
91
112
  * one cannot be reported (skyramp_submit_report refuses). */
@@ -99,13 +120,24 @@ export interface RetrofitOutcome {
99
120
  * Absent altogether when no utils file exists for the test (the same omission rule
100
121
  * as the POM fields at zero candidates: a zero says nothing "no file" does not). */
101
122
  export interface HelperReuseOutcome {
102
- /** BASENAME of the shared utils file the test imports from or this run wrote —
103
- * found by its header, never by a conventional name (the reuse prompt allows "a
104
- * new file with a different name" once the conventional one is large). Several
105
- * files are joined with `, `. */
123
+ /** REPOSITORY-RELATIVE PATH (posix) of the shared utils file the test imports
124
+ * from or this run wrote — found by its header, never by a conventional name (the
125
+ * reuse prompt allows "a new file with a different name" once the conventional
126
+ * one is large). A path, not a basename, so two same-named modules in one
127
+ * delivery (an API and a UI `skyrampUtils.ts`) stay distinguishable; never
128
+ * absolute, which would leak the runner's temp directory. Unique within ONE
129
+ * repository only: in a multi-repo report the same path can occur in two
130
+ * checkouts, so a renderer must key on the row's `repository` plus this field.
131
+ * Several files are joined with `, `. */
106
132
  utilsFile: string;
107
133
  /** Helpers the delivered test imports from that file that the file defines. */
108
134
  helpersImported?: number;
135
+ /** Those helpers by name, sorted; empty when nothing is imported. Always equals
136
+ * `helpersImported` in length — both come from the same set.
137
+ *
138
+ * TODO: remove `helpersImported` once the testbot renderer reads this field. The
139
+ * two carry one fact; the count stays only because testbot#349 reads it today. */
140
+ helperNames: string[];
109
141
  /** Inline request calls in OTHER Skyramp-generated tests beside this one that a
110
142
  * helper in `utilsFile` already wraps — reuse that was available and not taken.
111
143
  * Omitted at zero. Informational, like {@link ReuseOutcome.missedReuse}. */
@@ -114,13 +146,35 @@ export interface HelperReuseOutcome {
114
146
  * per method+path, status-code-only assertions, method+resource names — with
115
147
  * documented declines (`reuse-verify: allow …`) counted as holding. */
116
148
  verification?: HelperVerificationOutcome;
149
+ /** Why `verification` is `failed`: one entry per blocking kind found, with how
150
+ * many findings of it and the finding's own detail text — see
151
+ * {@link ReuseVerificationFailure}. Present only on a `failed` row. A bare
152
+ * `failed` cannot be acted on and its cause cannot be reconstructed from the
153
+ * artifacts; this is the statement of what failed.
154
+ *
155
+ * How it reaches a report: skyramp_submit_report refuses a report carrying a
156
+ * blocking verdict and names these entries to the agent, up to a bound per file
157
+ * (`REUSE_SUBMIT_MAX_REFUSALS`); past the bound the report is accepted and the
158
+ * row ships with this field set, so the fault is disclosed rather than delivered
159
+ * in silence — or not delivered at all. */
160
+ verificationFailures?: ReuseVerificationFailure[];
117
161
  /** Pre-existing generated tests this spec's reuse pass rewired onto the utils
118
162
  * file, each with its recorded execution. Omitted when none. */
119
163
  retrofits?: RetrofitOutcome[];
120
- /** Whether the delivered test plus the helpers it imports still hold every
121
- * assertion (and page-error guard) present when modularization was handed out.
122
- * Present only when that baseline exists. */
164
+ /** Whether the delivered test plus the helpers it imports and calls still hold at
165
+ * least as many assertions as when modularization was handed out. A measured
166
+ * count: a documented decline (`reuse-verify: allow assertion-loss …`) clears
167
+ * `verification` and leaves this `false`. Present only when that baseline exists.
168
+ * Says nothing about the page-error guard — that is
169
+ * {@link pageErrorGuardPreserved}; folding the two into one flag once sent an
170
+ * investigation after a lost assertion when a guard had been deleted. */
123
171
  assertionsPreserved?: boolean;
172
+ /** Whether the `page.on('pageerror', …)` guard present at the modularization
173
+ * hand-out is still registered, in the test or in a shared helper it imports and
174
+ * calls. `true` when the hand-out had no guard to lose. Present whenever the
175
+ * baseline exists, absent otherwise — like every optional member here, absent
176
+ * means "not measured", never "false". */
177
+ pageErrorGuardPreserved?: boolean;
124
178
  }
125
179
  /**
126
180
  * What lands in a report's `reuse` field for a generated test. Every member is
@@ -161,7 +215,19 @@ export interface ReuseOutcome {
161
215
  /** The subset of those raw locators that a catalogued POM member covers.
162
216
  * Informational; see {@link ReuseMissedEntry}. */
163
217
  missedReuse?: ReuseMissedEntry[];
218
+ /** Copies of operations written in two or more Skyramp-generated tests beside (and
219
+ * including) this one that NO shared helper wraps — extraction that was due and
220
+ * not performed, the counterpart of `helpers.siblingInlineCallSites` (use not
221
+ * taken). Top level, not on the helpers row: it describes the spec's directory,
222
+ * not the module, and is most telling when no module exists at all — the case
223
+ * where `helpers` is rightly absent. Omitted at zero. Informational. */
224
+ unextractedDuplicateSites?: number;
164
225
  verification?: ReuseVerificationOutcome;
226
+ /** Why `verification` is `failed` on the page-object path, on the same terms as
227
+ * {@link HelperReuseOutcome.verificationFailures}: present only when the
228
+ * delivered spec fails the live POM check at report time and the report was
229
+ * accepted past the refusal bound. */
230
+ verificationFailures?: ReuseVerificationFailure[];
165
231
  skipped?: ReuseSkippedEntry[];
166
232
  /** Shared-helper (SkyrampUtils) reuse. See {@link HelperReuseOutcome}. */
167
233
  helpers?: HelperReuseOutcome;
@@ -53,6 +53,12 @@ export interface MaintenanceActionCore {
53
53
  * the rewritten PNG — a listed baseline must never exempt the selector edit the same
54
54
  * verdict claimed (SKYR-3883 stays in force). */
55
55
  rebaselineOnly?: boolean;
56
+ /** True when the action is a RECOMMENDATION the developer applies by hand and this run
57
+ * touched nothing: a REGENERATE or DELETE on an external (user-authored) test. Part of
58
+ * the core shape because the action stays REAL rather than flattening to VERIFY, so
59
+ * without this a downstream stage cannot tell a regeneration that rewrote a file from
60
+ * one that only advised it — and maintenance coverage would credit the advice. */
61
+ reportOnly?: boolean;
56
62
  }
57
63
  /** Normalized internal recommendation built from LLM-supplied args.recommendations. */
58
64
  export interface DriftRecommendation extends MaintenanceActionCore {
@@ -72,6 +72,10 @@ export interface TestbotReport {
72
72
  editedFileName?: string;
73
73
  description: string;
74
74
  action: DriftAction;
75
+ /** `owner/repo` the maintained test lives in. Always set, unlike the other
76
+ * sections: `fileName` is a basename, so in a multi-repo run this is the
77
+ * only thing separating two same-named test files (SKYR-4233). */
78
+ repository: string;
75
79
  beforeStatus: TestExecutionStatus;
76
80
  beforeDetails: string;
77
81
  afterStatus: TestExecutionStatus;
@@ -119,10 +123,14 @@ export interface TestbotReport {
119
123
  }[];
120
124
  issuesFound: {
121
125
  description: string;
122
- /** The planned test this issue is about, when it is about one — set on the
123
- * issue that writes up a `requirement_conflict` so the conflict names the
124
- * test that asserts it. Absent on an issue about no particular test. */
126
+ /** The test that proves this issue: one that expects to fail until the issue
127
+ * is fixed, or the `requirement_conflict` test that asserts the requirement.
128
+ * Absent on an issue no test proves; a `bug` entry without one draws an
129
+ * `issueTraceability:<n>` objection. */
125
130
  plannedTestId?: string;
131
+ /** The plan defect this issue reports, by the id the plan's `defects` list
132
+ * gave it. Absent on an issue the plan did not declare. */
133
+ defectId?: string;
126
134
  severity?: "critical" | "high" | "medium" | "low";
127
135
  /** Required by the submit_report schema since 0.3.4; absent in reports
128
136
  * written by older MCP versions. Readers treat absence as Bug. */
@@ -150,8 +158,18 @@ export interface TestbotReport {
150
158
  text: string;
151
159
  source: string;
152
160
  /** `plannedTestId` of each delivered test citing this change. Empty where the
153
- * agent answered instead of planning one. */
161
+ * agent answered instead of planning one, and where the change is covered by
162
+ * an existing test the run maintained — `maintainedBy` carries those. */
154
163
  testedBy: string[];
164
+ /** Path of each existing test the run edited that cites this change: a
165
+ * `maintains` entry from the plan, kept only where a `testMaintenance` row
166
+ * names the same file. Checked at name level, which is the level v2 allows
167
+ * for a file, so this says the run reported editing it and not that the edit
168
+ * is known to be right. A claim no maintenance row corroborates is left out
169
+ * and draws `deliveredMatchesPlan:maintains:` instead. A change covered this
170
+ * way has no planned test and needs no answer, so without this field the
171
+ * change table would read as untested. */
172
+ maintainedBy?: string[];
155
173
  /** The agent's answer to `coverage:change:<id>`, where it gave one. */
156
174
  answer?: string;
157
175
  }[];
@@ -133,7 +133,7 @@ export interface UiAnalysisContext {
133
133
  * PATH. Recorded in-process where staging finds the edit; skyramp_submit_report
134
134
  * refuses a report that leaves one undisclosed or unexecuted. */
135
135
  export interface ReuseRetrofit {
136
- /** BASENAME(s) of the utils file(s) it now imports from. */
136
+ /** Repository-relative path(s) of the utils file(s) it now imports from. */
137
137
  utilsFile: string;
138
138
  /** Absolute path of the generated test whose reuse pass caused the edit. */
139
139
  testFile: string;
@@ -184,6 +184,12 @@ export interface UnifiedAnalysisState {
184
184
  reuseHandOffs?: Record<string, ReuseHandOff>;
185
185
  /** See {@link ReuseRetrofit}. Never supplied by the LLM. */
186
186
  reuseRetrofits?: Record<string, ReuseRetrofit>;
187
+ /** How many times skyramp_submit_report has refused the report over a blocking
188
+ * reuse verdict on each spec, keyed by the spec's ABSOLUTE PATH (the verdict's
189
+ * `file`). The refusal is bounded (`REUSE_SUBMIT_MAX_REFUSALS`): past the bound
190
+ * the report is accepted with the fault recorded in the row. Written by
191
+ * skyramp_submit_report only; never supplied by the LLM. */
192
+ reuseRefusals?: Record<string, number>;
187
193
  /**
188
194
  * SKYR-4156. Recorded video per executed browser test, keyed by test-file
189
195
  * BASENAME (the same key `reuseOutcomes` uses, so matching needs no path
@@ -295,7 +295,11 @@ async function writeFileAtomically(target, contents) {
295
295
  throw error;
296
296
  }
297
297
  }
298
- const ROOT_FIELDS_OWNED_ELSEWHERE = ["plan", "registrationCount", "reportObjections"];
298
+ const ROOT_FIELDS_OWNED_ELSEWHERE = [
299
+ "plan",
300
+ "registrationCount",
301
+ "reportObjections",
302
+ ];
299
303
  function carryForwardRootFields(data, existing) {
300
304
  if (!existing)
301
305
  return {};
@@ -1,6 +1,6 @@
1
1
  import { escapeRegExp } from "../regex.js";
2
2
  import { strippedSources } from "./strip-dispatch.js";
3
- import { balancedCloseIndex, identifierRe, inScope, interpolationRe, lineOfOffset, } from "./lint-types.js";
3
+ import { balancedCloseIndex, identifierRe, inScope, interpolationRe, lineOfOffset, testBlockIndex, testStartOffsets, } from "./lint-types.js";
4
4
  /**
5
5
  * Deterministic checks shared by integration and provider-contract tests — the
6
6
  * machine-checkable slice of the D5 rubric's echo_computed and array_depth
@@ -12,6 +12,12 @@ const RESPONSE_BINDING_RES = {
12
12
  python: /^[ \t]*([A-Za-z_]\w*)\s*=\s*[\w.]*send_request\s*\(/gm,
13
13
  };
14
14
  const STATUS_USE_RE = /^\s*\.\s*(?:statusCode|status_code|status)\b/;
15
+ /** The SDK body-read call for a binding, spelled for the language. */
16
+ function helperCall(language, varName) {
17
+ return language === "python"
18
+ ? `skyramp.get_response_value(${varName}, "json.path")`
19
+ : `getValue(${varName}, "json.path")`;
20
+ }
15
21
  /** All send_request/sendRequest bindings in the stripped source. */
16
22
  function responseBindings(stripped, language) {
17
23
  const bindings = [];
@@ -148,11 +154,17 @@ function lintStatusOnlyResponses(stripped, commentless, language, bindings) {
148
154
  message: statusUses > 0
149
155
  ? severity === "hard"
150
156
  ? `Error response \`${varName}\` is only checked for its status code — the error body (code/message/detail) must be asserted.`
151
- : `Response \`${varName}\` is only checked for its status code — if this endpoint returns a body, assert its fields (skip only for genuinely empty-body responses).`
157
+ : `Response \`${varName}\` is only checked for its status code.`
152
158
  : `Response \`${varName}\` is captured but never asserted.`,
153
- remediation: language === "python"
154
- ? `Assert body fields via \`skyramp.get_response_value(${varName}, "json.path")\` after the status assertion.`
155
- : `Assert body fields via \`getValue(${varName}, "json.path")\` after the status assertion.`,
159
+ // The advisory case is phrased as a question, not an instruction. A
160
+ // response no behaviour in the file depends on is CORRECT with its
161
+ // status code and nothing more (SKYR-4310) telling the agent to
162
+ // assert the body regardless is how a 136-assertion test gets written
163
+ // to prove a default page size. The hard case is unchanged: an error
164
+ // body carries the evidence, so it is never optional.
165
+ remediation: severity === "warn"
166
+ ? `If a behaviour this file tests depends on this response's body, assert the fields it depends on via \`${helperCall(language, varName)}\`. If no behaviour depends on it, the status code alone is the right answer — leave it.`
167
+ : `Assert the error body's fields via \`${helperCall(language, varName)}\` after the status assertion.`,
156
168
  });
157
169
  }
158
170
  return findings;
@@ -294,6 +306,87 @@ function lintArrayDepth(commentless) {
294
306
  }
295
307
  return findings;
296
308
  }
309
+ // An error-indicator field asserted absent on a response binding:
310
+ // expect(getValue(r, "error")).toBeUndefined() / .toBeNull()
311
+ // assert skyramp.get_response_value(r, "error") is None
312
+ // Group 1 is the binding (JS/TS identifiers may carry `$`), group 2 the field.
313
+ const ERROR_ABSENT_RE = /(?:getValue|getResponseValue|get_response_value)\s*\(\s*([\w$.]+)\s*,\s*['"](error|errors|message)['"]\s*\)\s*(?:(?:,\s*['"][^'"]*['"])?\s*\)\s*\.\s*toBe(?:Undefined|Null)\s*\(|is\s+None\b)/g;
314
+ // A populated-collection claim on a binding:
315
+ // expect(getValue(r, "items").length).toBeGreaterThan(0)
316
+ // assert len(skyramp.get_response_value(r, "items")) > 0
317
+ const NON_EMPTY_RE = /(?:getValue|getResponseValue)\s*\(\s*([\w$.]+)\s*,\s*['"][^'"]+['"]\s*\)\s*\.\s*length\s*(?:,\s*['"][^'"]*['"]\s*)?\)\s*\.\s*toBeGreaterThan\s*\(\s*0\s*\)|len\s*\(\s*[\w.]*get_response_value\s*\(\s*([\w$.]+)\s*,\s*['"][^'"]+['"]\s*\)\s*\)\s*>\s*0/g;
318
+ /** A success shape asserted together with the absence of an error field on the
319
+ * same response binding, inside the same test body. Both are legitimate on a
320
+ * real success body, so this is advisory only: the pattern is exactly what a
321
+ * test looks like when its expectations were written for the outcome the
322
+ * author wanted rather than the body the endpoint returns (a no-match or
323
+ * validation error carried in `error`). The verifier cannot see the response,
324
+ * so it asks the author to confirm the classification rather than blocking. */
325
+ function lintSuccessShapeWithErrorAbsent(commentless) {
326
+ const starts = testStartOffsets(commentless);
327
+ let m;
328
+ // key: `<testBlock>:<binding>` → { offset, field }
329
+ const errorAbsent = new Map();
330
+ ERROR_ABSENT_RE.lastIndex = 0;
331
+ while ((m = ERROR_ABSENT_RE.exec(commentless)) !== null) {
332
+ const key = `${testBlockIndex(starts, m.index)}:${m[1]}`;
333
+ if (!errorAbsent.has(key))
334
+ errorAbsent.set(key, { offset: m.index, field: m[2] });
335
+ }
336
+ if (errorAbsent.size === 0)
337
+ return [];
338
+ const findings = [];
339
+ const reported = new Set();
340
+ NON_EMPTY_RE.lastIndex = 0;
341
+ while ((m = NON_EMPTY_RE.exec(commentless)) !== null) {
342
+ const varName = m[1] ?? m[2];
343
+ const key = `${testBlockIndex(starts, m.index)}:${varName}`;
344
+ const absent = errorAbsent.get(key);
345
+ if (!absent || reported.has(key))
346
+ continue;
347
+ reported.add(key);
348
+ findings.push({
349
+ rule: "success-shape-with-error-absent",
350
+ severity: "warn",
351
+ line: lineOfOffset(commentless, absent.offset),
352
+ message: `\`${varName}\` is asserted to have no \`${absent.field}\` field and a non-empty collection — a success shape.`,
353
+ remediation: 'Confirm the recorded response for this exact request is a success body. If the endpoint answers this request with an error or no-match body (a validation 4xx, or a 2xx carrying `{"error": ...}`), assert that body\'s exact fields instead and report the gap in issuesFound. Keep the assertion as written only when the plan declared this test `expected.outcome: fail` (a bug_caught or requirement_conflict test that deliberately asserts the intended behaviour).',
354
+ });
355
+ }
356
+ return findings;
357
+ }
358
+ // The existence idiom on an SDK-helper read: `not.toBeNull()` / `is not None`.
359
+ // Weight 1 in the strength model, and the one shape MAINTENANCE_SCOPE_NOTE
360
+ // names by example.
361
+ const WEAK_EXISTENCE_RE = /(?:getValue|getResponseValue)\s*\(\s*[\w$.]+\s*,\s*['"]([^'"]+)['"][^)]*\)\s*(?:,\s*['"][^'"]*['"]\s*)?\)\s*\.\s*not\s*\.\s*toBeNull\s*\(|get_response_value\s*\(\s*[\w.]+\s*,\s*['"]([^'"]+)['"]\s*\)\s*is\s+not\s+None\b/g;
362
+ /** Maintenance only: an existence-tier assertion left standing inside a test
363
+ * function the agent TOUCHED. `MAINTENANCE_SCOPE_NOTE` holds such a function
364
+ * to the same standard as a new test — "an existing `is not None` on a field
365
+ * the diff gives a knowable value becomes that value".
366
+ *
367
+ * Warn, never hard: the verifier has no diff, so it cannot tell which fields
368
+ * the change made knowable. It reports the candidates and lets the agent
369
+ * decide, which is the honest split — the alternative is blocking a spec on a
370
+ * field whose value genuinely still varies. */
371
+ function lintWeakAssertionInTouchedTest(commentless, opts) {
372
+ const findings = [];
373
+ WEAK_EXISTENCE_RE.lastIndex = 0;
374
+ let m;
375
+ while ((m = WEAK_EXISTENCE_RE.exec(commentless)) !== null) {
376
+ const line = lineOfOffset(commentless, m.index);
377
+ if (!inScope(line, opts))
378
+ continue;
379
+ const field = m[1] ?? m[2];
380
+ findings.push({
381
+ rule: "weak-assertion-in-touched-test",
382
+ severity: "warn",
383
+ line,
384
+ message: `\`${field}\` is asserted only to exist, in a test function this change touches.`,
385
+ remediation: `A test function you touch is held to the same standard as a new one. If the diff gives \`${field}\` a knowable value, assert that value exactly. Leave the existence check only when the value genuinely varies across runs.`,
386
+ });
387
+ }
388
+ return findings;
389
+ }
297
390
  /** Checks shared by integration and provider-contract tests. Java is never
298
391
  * linted (count+hash gates only). */
299
392
  export function lintApiShared(raw, language, opts) {
@@ -302,14 +395,28 @@ export function lintApiShared(raw, language, opts) {
302
395
  const maintenance = opts?.scopeLines !== undefined;
303
396
  const { commentless, stripped } = strippedSources(raw, language);
304
397
  const findings = [];
398
+ // Bindings discovered once, shared by both binding-scoped lints.
399
+ const bindings = responseBindings(stripped, language);
400
+ findings.push(...lintStatusOnlyResponses(stripped, commentless, language, bindings));
401
+ findings.push(...lintSdkHelperBypass(stripped, bindings, language));
402
+ findings.push(...lintShapeOnlyChecks(commentless));
403
+ findings.push(...lintSuccessShapeWithErrorAbsent(commentless));
305
404
  if (!maintenance) {
306
- // Bindings discovered once, shared by both binding-scoped lints.
307
- const bindings = responseBindings(stripped, language);
308
- findings.push(...lintStatusOnlyResponses(stripped, commentless, language, bindings));
309
- findings.push(...lintSdkHelperBypass(stripped, bindings, language));
310
- findings.push(...lintShapeOnlyChecks(commentless));
405
+ // Array-depth findings anchor to a path, not a line, so there is nothing
406
+ // to scope them by — running them in maintenance would report untouched
407
+ // customer arrays with no way to filter them out.
311
408
  findings.push(...lintArrayDepth(commentless));
312
409
  }
410
+ else {
411
+ findings.push(...lintWeakAssertionInTouchedTest(commentless, opts));
412
+ }
313
413
  findings.push(...lintPermissiveStatusMatchers(commentless, opts));
314
- return findings;
414
+ // Maintenance used to skip every lint above except the status-matcher one,
415
+ // because line-scoping to the ADDED lines would have flagged pre-existing
416
+ // customer code. The scope is now the touched test FUNCTION (SKYR-4310), so
417
+ // these can run: each finding is kept only when it lands inside a function
418
+ // the agent edited. A finding with no line cannot be placed and is dropped.
419
+ return maintenance
420
+ ? findings.filter((f) => f.line !== undefined && inScope(f.line, opts))
421
+ : findings;
315
422
  }
@@ -51,8 +51,8 @@ export function formatAssertionVerifyReport(r, testFile) {
51
51
  if (r.strengthGateFailed) {
52
52
  lines.push("- Assertion strength did not increase over the generated baseline: no assertion was added and no weak matcher was strengthened (e.g. `toBeVisible` → `toHaveText`). Apply the enhancement rules. A documented decline (`assertions complete` comment) is accepted only AFTER at least one assertion was added or replaced — a marker-only edit does not pass this gate.");
53
53
  }
54
- if (r.weakAdditionsOnly) {
55
- lines.push("- Every assertion added since the baseline is existence/visibility-tier (`toBeVisible`, `not.toBeNull`, `is not None`, ranges). When the exact text/value/count is knowable from the trace, source, or request, assert it exactly (`toHaveText`/`toHaveCount`/`toBe`/`==`). If only existence is genuinely knowable, document the decline:", ` ${token} assertions complete: ${spec} — <reason exact values are not knowable>`);
54
+ if (r.weakAdditionsDominant) {
55
+ lines.push(`- The assertions added since the baseline are mostly existence/visibility-tier (\`toBeVisible\`, \`not.toBeNull\`, \`is not None\`, ranges)${r.meanAddedWeight !== undefined ? ` — mean strength ${r.meanAddedWeight.toFixed(1)} against a bar of 2.0` : ""}. When the exact text/value/count is knowable from the trace, source, or request, assert it exactly (\`toHaveText\`/\`toHaveCount\`/\`toBe\`/\`==\`). Adding MORE weak assertions lowers this figure — the fix is to strengthen the ones the behaviour under test depends on, and to drop the ones it does not. If only existence is genuinely knowable, document the decline:`, ` ${token} assertions complete: ${spec} — <reason exact values are not knowable>`);
56
56
  }
57
57
  lines.push(...warningsBlock(r.warnings, finding));
58
58
  lines.push("Re-run this verification (verify: true) after every fix pass — skyramp_execute_test runs the same checks automatically and will not execute this test until they pass.");
@@ -4,3 +4,10 @@
4
4
  * uncalled helper is skipped, so its subjects don't excuse anything.
5
5
  */
6
6
  export declare function importedHelperSubjects(testFile: string, content: string): Promise<Set<string>>;
7
+ /**
8
+ * The same subjects with their CAPACITY — how many removals each may excuse:
9
+ * one per call site of the helper function that asserts it (an assertion the
10
+ * parser cannot place in a function gets one per call through the import).
11
+ * The removal gate consumes this budget per removed baseline subject.
12
+ */
13
+ export declare function importedHelperSubjectCapacities(testFile: string, content: string): Promise<Map<string, number>>;