@skyramp/mcp 0.4.0 → 0.4.1-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/prompts/enhance-assertions/contractProviderAssertionsPrompt.js +2 -1
- package/build/prompts/enhance-assertions/integrationAssertionsPrompt.js +2 -1
- package/build/prompts/enhance-assertions/sharedAssertionRules.d.ts +1 -1
- package/build/prompts/enhance-assertions/sharedAssertionRules.js +57 -22
- package/build/prompts/enhance-assertions/uiAssertionsPrompt.js +17 -9
- package/build/prompts/test-recommendation/diffExecutionPlan.js +0 -2
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +11 -6
- package/build/prompts/testbot/testbot-prompts.js +19 -12
- package/build/recommendation/answers.d.ts +11 -7
- package/build/recommendation/answers.js +14 -10
- package/build/recommendation/pullRequestText.d.ts +18 -0
- package/build/recommendation/pullRequestText.js +31 -0
- package/build/recommendation/registerPlan.d.ts +9 -1
- package/build/recommendation/registerPlan.js +8 -1
- package/build/recommendation/runVerifiers.js +6 -0
- package/build/recommendation/types.d.ts +58 -0
- package/build/recommendation/verifierContracts.d.ts +108 -13
- package/build/recommendation/verifierContracts.js +147 -29
- package/build/recommendation/verifiers/coverage.d.ts +10 -0
- package/build/recommendation/verifiers/coverage.js +144 -22
- package/build/recommendation/verifiers/defects.d.ts +9 -0
- package/build/recommendation/verifiers/defects.js +117 -0
- package/build/recommendation/verifiers/deliveredMatchesPlan.d.ts +22 -0
- package/build/recommendation/verifiers/deliveredMatchesPlan.js +43 -0
- package/build/recommendation/verifiers/existingCoverage.js +53 -0
- package/build/recommendation/verifiers/expectedValueSourced.d.ts +14 -0
- package/build/recommendation/verifiers/expectedValueSourced.js +246 -0
- package/build/recommendation/verifiers/issueTraceability.d.ts +52 -0
- package/build/recommendation/verifiers/issueTraceability.js +197 -0
- package/build/recommendation/verifiers/requirementSourced.d.ts +2 -0
- package/build/recommendation/verifiers/requirementSourced.js +168 -0
- package/build/services/TestGenerationService.js +3 -1
- package/build/tools/code-refactor/codeReuseTool.js +1 -1
- package/build/tools/code-refactor/reuse-outcome.d.ts +1 -1
- package/build/tools/code-refactor/reuse-state.d.ts +85 -7
- package/build/tools/code-refactor/reuse-state.js +239 -34
- package/build/tools/code-refactor/utils-verify-gates.d.ts +5 -0
- package/build/tools/code-refactor/utils-verify-gates.js +103 -11
- package/build/tools/generate-tests/generateBatchScenarioRestTool.js +2 -1
- package/build/tools/submitReportTool.js +276 -39
- package/build/tools/test-management/actionsTool.js +5 -0
- package/build/tools/test-management/analyzeChangesTool.d.ts +53 -0
- package/build/tools/test-management/analyzeChangesTool.js +55 -2
- package/build/tools/test-management/registerTestPlanTool.d.ts +32 -17
- package/build/tools/test-management/registerTestPlanTool.js +105 -16
- package/build/types/ReuseOutcome.d.ts +73 -7
- package/build/types/TestAnalysis.d.ts +6 -0
- package/build/types/TestbotReport.d.ts +22 -4
- package/build/utils/AnalysisStateManager.d.ts +7 -1
- package/build/utils/AnalysisStateManager.js +5 -1
- package/build/utils/assertion-verify/api-shared-lints.js +118 -11
- package/build/utils/assertion-verify/format.js +2 -2
- package/build/utils/assertion-verify/helper-imports.d.ts +7 -0
- package/build/utils/assertion-verify/helper-imports.js +119 -27
- package/build/utils/assertion-verify/lint-types.d.ts +31 -2
- package/build/utils/assertion-verify/lint-types.js +66 -0
- package/build/utils/assertion-verify/metrics.d.ts +13 -0
- package/build/utils/assertion-verify/metrics.js +16 -0
- package/build/utils/assertion-verify/verify.d.ts +11 -6
- package/build/utils/assertion-verify/verify.js +56 -15
- package/build/utils/canonicalJson.d.ts +11 -0
- package/build/utils/canonicalJson.js +17 -0
- package/build/utils/utils-verify/action-key.d.ts +27 -0
- package/build/utils/utils-verify/action-key.js +292 -0
- package/build/utils/utils-verify/allow.d.ts +8 -1
- package/build/utils/utils-verify/allow.js +14 -1
- package/build/utils/utils-verify/call-sites.d.ts +76 -8
- package/build/utils/utils-verify/call-sites.js +256 -70
- package/build/utils/utils-verify/language-spec.d.ts +3 -2
- package/build/utils/utils-verify/parse.d.ts +22 -3
- package/build/utils/utils-verify/parse.js +123 -52
- package/build/utils/utils-verify/verify.d.ts +33 -3
- package/build/utils/utils-verify/verify.js +126 -12
- package/build/utils/workspaceAuth.d.ts +59 -19
- package/build/utils/workspaceAuth.js +228 -31
- package/package.json +1 -1
- package/plugin/prompts/generate-tests/execution-plan.md +3 -3
- package/plugin/prompts/generate-tests/generation.md +1 -0
- package/plugin/prompts/plan-tests.md +41 -15
- package/plugin/prompts/testbot-task1.md +3 -9
- package/build/prompts/testbot/planDeclarations.d.ts +0 -6
- package/build/prompts/testbot/planDeclarations.js +0 -9
- package/plugin/prompts/declaring-a-plan.md +0 -20
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { TestExecutionStatus } from "./TestExecution.js";
|
|
2
|
+
import type { UtilsViolationKind } from "../utils/utils-verify/allow.js";
|
|
2
3
|
/**
|
|
3
4
|
* The code-reuse observability shape carried in a TestbotReport — the wire contract
|
|
4
5
|
* between this server and the consumers that render it (test-bot.git). It covers
|
|
@@ -86,6 +87,26 @@ export declare enum HelperVerificationOutcome {
|
|
|
86
87
|
/** The test neither wrote nor imports a shared utils file. */
|
|
87
88
|
NoUtilsFile = "no-utils-file"
|
|
88
89
|
}
|
|
90
|
+
/** Every blocking kind a report-time reuse verification can find. Shared-helper
|
|
91
|
+
* path: the `UtilsViolationKind`s, plus `malformed-allow-marker` for an allow
|
|
92
|
+
* marker the grammar cannot read (which fails a verify on its own). Page-object
|
|
93
|
+
* path: a POM member the spec calls that its source does not define, a spec that
|
|
94
|
+
* does not parse, a spec whose Skyramp runtime imports were stripped, and a spec
|
|
95
|
+
* with no page-object reuse whose selectors match detected page objects and carry
|
|
96
|
+
* no `// kept inline:` decline. A union, so a misspelled kind is a compile error. */
|
|
97
|
+
export type ReuseVerificationFailureKind = UtilsViolationKind | "malformed-allow-marker" | "pom-member-missing" | "parse-error" | "skyramp-infra-missing" | "zero-reuse";
|
|
98
|
+
/** One blocking kind a failed reuse verification found in the delivered files,
|
|
99
|
+
* aggregated: the verifier reports a finding per helper, import line or POM call,
|
|
100
|
+
* and the row states the kind once with its count. `detail` is the findings' own
|
|
101
|
+
* detail text (joined with `; ` when there are several) — for an `assertion-loss`
|
|
102
|
+
* that is the sentence `verify.ts` builds with the counts behind the verdict
|
|
103
|
+
* (baseline, delivered, in the test, in the helpers); it is NOT the gate's
|
|
104
|
+
* PASSED/FAILED prose. */
|
|
105
|
+
export interface ReuseVerificationFailure {
|
|
106
|
+
kind: ReuseVerificationFailureKind;
|
|
107
|
+
count: number;
|
|
108
|
+
detail: string;
|
|
109
|
+
}
|
|
89
110
|
/** A pre-existing Skyramp-generated test the reuse pass edited to import from the
|
|
90
111
|
* shared utils file. `execution` is the recorded post-edit run; a retrofit without
|
|
91
112
|
* one cannot be reported (skyramp_submit_report refuses). */
|
|
@@ -99,13 +120,24 @@ export interface RetrofitOutcome {
|
|
|
99
120
|
* Absent altogether when no utils file exists for the test (the same omission rule
|
|
100
121
|
* as the POM fields at zero candidates: a zero says nothing "no file" does not). */
|
|
101
122
|
export interface HelperReuseOutcome {
|
|
102
|
-
/**
|
|
103
|
-
* found by its header, never by a conventional name (the
|
|
104
|
-
* new file with a different name" once the conventional
|
|
105
|
-
*
|
|
123
|
+
/** REPOSITORY-RELATIVE PATH (posix) of the shared utils file the test imports
|
|
124
|
+
* from or this run wrote — found by its header, never by a conventional name (the
|
|
125
|
+
* reuse prompt allows "a new file with a different name" once the conventional
|
|
126
|
+
* one is large). A path, not a basename, so two same-named modules in one
|
|
127
|
+
* delivery (an API and a UI `skyrampUtils.ts`) stay distinguishable; never
|
|
128
|
+
* absolute, which would leak the runner's temp directory. Unique within ONE
|
|
129
|
+
* repository only: in a multi-repo report the same path can occur in two
|
|
130
|
+
* checkouts, so a renderer must key on the row's `repository` plus this field.
|
|
131
|
+
* Several files are joined with `, `. */
|
|
106
132
|
utilsFile: string;
|
|
107
133
|
/** Helpers the delivered test imports from that file that the file defines. */
|
|
108
134
|
helpersImported?: number;
|
|
135
|
+
/** Those helpers by name, sorted; empty when nothing is imported. Always equals
|
|
136
|
+
* `helpersImported` in length — both come from the same set.
|
|
137
|
+
*
|
|
138
|
+
* TODO: remove `helpersImported` once the testbot renderer reads this field. The
|
|
139
|
+
* two carry one fact; the count stays only because testbot#349 reads it today. */
|
|
140
|
+
helperNames: string[];
|
|
109
141
|
/** Inline request calls in OTHER Skyramp-generated tests beside this one that a
|
|
110
142
|
* helper in `utilsFile` already wraps — reuse that was available and not taken.
|
|
111
143
|
* Omitted at zero. Informational, like {@link ReuseOutcome.missedReuse}. */
|
|
@@ -114,13 +146,35 @@ export interface HelperReuseOutcome {
|
|
|
114
146
|
* per method+path, status-code-only assertions, method+resource names — with
|
|
115
147
|
* documented declines (`reuse-verify: allow …`) counted as holding. */
|
|
116
148
|
verification?: HelperVerificationOutcome;
|
|
149
|
+
/** Why `verification` is `failed`: one entry per blocking kind found, with how
|
|
150
|
+
* many findings of it and the finding's own detail text — see
|
|
151
|
+
* {@link ReuseVerificationFailure}. Present only on a `failed` row. A bare
|
|
152
|
+
* `failed` cannot be acted on and its cause cannot be reconstructed from the
|
|
153
|
+
* artifacts; this is the statement of what failed.
|
|
154
|
+
*
|
|
155
|
+
* How it reaches a report: skyramp_submit_report refuses a report carrying a
|
|
156
|
+
* blocking verdict and names these entries to the agent, up to a bound per file
|
|
157
|
+
* (`REUSE_SUBMIT_MAX_REFUSALS`); past the bound the report is accepted and the
|
|
158
|
+
* row ships with this field set, so the fault is disclosed rather than delivered
|
|
159
|
+
* in silence — or not delivered at all. */
|
|
160
|
+
verificationFailures?: ReuseVerificationFailure[];
|
|
117
161
|
/** Pre-existing generated tests this spec's reuse pass rewired onto the utils
|
|
118
162
|
* file, each with its recorded execution. Omitted when none. */
|
|
119
163
|
retrofits?: RetrofitOutcome[];
|
|
120
|
-
/** Whether the delivered test plus the helpers it imports still hold
|
|
121
|
-
*
|
|
122
|
-
*
|
|
164
|
+
/** Whether the delivered test plus the helpers it imports and calls still hold at
|
|
165
|
+
* least as many assertions as when modularization was handed out. A measured
|
|
166
|
+
* count: a documented decline (`reuse-verify: allow assertion-loss …`) clears
|
|
167
|
+
* `verification` and leaves this `false`. Present only when that baseline exists.
|
|
168
|
+
* Says nothing about the page-error guard — that is
|
|
169
|
+
* {@link pageErrorGuardPreserved}; folding the two into one flag once sent an
|
|
170
|
+
* investigation after a lost assertion when a guard had been deleted. */
|
|
123
171
|
assertionsPreserved?: boolean;
|
|
172
|
+
/** Whether the `page.on('pageerror', …)` guard present at the modularization
|
|
173
|
+
* hand-out is still registered, in the test or in a shared helper it imports and
|
|
174
|
+
* calls. `true` when the hand-out had no guard to lose. Present whenever the
|
|
175
|
+
* baseline exists, absent otherwise — like every optional member here, absent
|
|
176
|
+
* means "not measured", never "false". */
|
|
177
|
+
pageErrorGuardPreserved?: boolean;
|
|
124
178
|
}
|
|
125
179
|
/**
|
|
126
180
|
* What lands in a report's `reuse` field for a generated test. Every member is
|
|
@@ -161,7 +215,19 @@ export interface ReuseOutcome {
|
|
|
161
215
|
/** The subset of those raw locators that a catalogued POM member covers.
|
|
162
216
|
* Informational; see {@link ReuseMissedEntry}. */
|
|
163
217
|
missedReuse?: ReuseMissedEntry[];
|
|
218
|
+
/** Copies of operations written in two or more Skyramp-generated tests beside (and
|
|
219
|
+
* including) this one that NO shared helper wraps — extraction that was due and
|
|
220
|
+
* not performed, the counterpart of `helpers.siblingInlineCallSites` (use not
|
|
221
|
+
* taken). Top level, not on the helpers row: it describes the spec's directory,
|
|
222
|
+
* not the module, and is most telling when no module exists at all — the case
|
|
223
|
+
* where `helpers` is rightly absent. Omitted at zero. Informational. */
|
|
224
|
+
unextractedDuplicateSites?: number;
|
|
164
225
|
verification?: ReuseVerificationOutcome;
|
|
226
|
+
/** Why `verification` is `failed` on the page-object path, on the same terms as
|
|
227
|
+
* {@link HelperReuseOutcome.verificationFailures}: present only when the
|
|
228
|
+
* delivered spec fails the live POM check at report time and the report was
|
|
229
|
+
* accepted past the refusal bound. */
|
|
230
|
+
verificationFailures?: ReuseVerificationFailure[];
|
|
165
231
|
skipped?: ReuseSkippedEntry[];
|
|
166
232
|
/** Shared-helper (SkyrampUtils) reuse. See {@link HelperReuseOutcome}. */
|
|
167
233
|
helpers?: HelperReuseOutcome;
|
|
@@ -53,6 +53,12 @@ export interface MaintenanceActionCore {
|
|
|
53
53
|
* the rewritten PNG — a listed baseline must never exempt the selector edit the same
|
|
54
54
|
* verdict claimed (SKYR-3883 stays in force). */
|
|
55
55
|
rebaselineOnly?: boolean;
|
|
56
|
+
/** True when the action is a RECOMMENDATION the developer applies by hand and this run
|
|
57
|
+
* touched nothing: a REGENERATE or DELETE on an external (user-authored) test. Part of
|
|
58
|
+
* the core shape because the action stays REAL rather than flattening to VERIFY, so
|
|
59
|
+
* without this a downstream stage cannot tell a regeneration that rewrote a file from
|
|
60
|
+
* one that only advised it — and maintenance coverage would credit the advice. */
|
|
61
|
+
reportOnly?: boolean;
|
|
56
62
|
}
|
|
57
63
|
/** Normalized internal recommendation built from LLM-supplied args.recommendations. */
|
|
58
64
|
export interface DriftRecommendation extends MaintenanceActionCore {
|
|
@@ -72,6 +72,10 @@ export interface TestbotReport {
|
|
|
72
72
|
editedFileName?: string;
|
|
73
73
|
description: string;
|
|
74
74
|
action: DriftAction;
|
|
75
|
+
/** `owner/repo` the maintained test lives in. Always set, unlike the other
|
|
76
|
+
* sections: `fileName` is a basename, so in a multi-repo run this is the
|
|
77
|
+
* only thing separating two same-named test files (SKYR-4233). */
|
|
78
|
+
repository: string;
|
|
75
79
|
beforeStatus: TestExecutionStatus;
|
|
76
80
|
beforeDetails: string;
|
|
77
81
|
afterStatus: TestExecutionStatus;
|
|
@@ -119,10 +123,14 @@ export interface TestbotReport {
|
|
|
119
123
|
}[];
|
|
120
124
|
issuesFound: {
|
|
121
125
|
description: string;
|
|
122
|
-
/** The
|
|
123
|
-
*
|
|
124
|
-
*
|
|
126
|
+
/** The test that proves this issue: one that expects to fail until the issue
|
|
127
|
+
* is fixed, or the `requirement_conflict` test that asserts the requirement.
|
|
128
|
+
* Absent on an issue no test proves; a `bug` entry without one draws an
|
|
129
|
+
* `issueTraceability:<n>` objection. */
|
|
125
130
|
plannedTestId?: string;
|
|
131
|
+
/** The plan defect this issue reports, by the id the plan's `defects` list
|
|
132
|
+
* gave it. Absent on an issue the plan did not declare. */
|
|
133
|
+
defectId?: string;
|
|
126
134
|
severity?: "critical" | "high" | "medium" | "low";
|
|
127
135
|
/** Required by the submit_report schema since 0.3.4; absent in reports
|
|
128
136
|
* written by older MCP versions. Readers treat absence as Bug. */
|
|
@@ -150,8 +158,18 @@ export interface TestbotReport {
|
|
|
150
158
|
text: string;
|
|
151
159
|
source: string;
|
|
152
160
|
/** `plannedTestId` of each delivered test citing this change. Empty where the
|
|
153
|
-
* agent answered instead of planning one
|
|
161
|
+
* agent answered instead of planning one, and where the change is covered by
|
|
162
|
+
* an existing test the run maintained — `maintainedBy` carries those. */
|
|
154
163
|
testedBy: string[];
|
|
164
|
+
/** Path of each existing test the run edited that cites this change: a
|
|
165
|
+
* `maintains` entry from the plan, kept only where a `testMaintenance` row
|
|
166
|
+
* names the same file. Checked at name level, which is the level v2 allows
|
|
167
|
+
* for a file, so this says the run reported editing it and not that the edit
|
|
168
|
+
* is known to be right. A claim no maintenance row corroborates is left out
|
|
169
|
+
* and draws `deliveredMatchesPlan:maintains:` instead. A change covered this
|
|
170
|
+
* way has no planned test and needs no answer, so without this field the
|
|
171
|
+
* change table would read as untested. */
|
|
172
|
+
maintainedBy?: string[];
|
|
155
173
|
/** The agent's answer to `coverage:change:<id>`, where it gave one. */
|
|
156
174
|
answer?: string;
|
|
157
175
|
}[];
|
|
@@ -133,7 +133,7 @@ export interface UiAnalysisContext {
|
|
|
133
133
|
* PATH. Recorded in-process where staging finds the edit; skyramp_submit_report
|
|
134
134
|
* refuses a report that leaves one undisclosed or unexecuted. */
|
|
135
135
|
export interface ReuseRetrofit {
|
|
136
|
-
/**
|
|
136
|
+
/** Repository-relative path(s) of the utils file(s) it now imports from. */
|
|
137
137
|
utilsFile: string;
|
|
138
138
|
/** Absolute path of the generated test whose reuse pass caused the edit. */
|
|
139
139
|
testFile: string;
|
|
@@ -184,6 +184,12 @@ export interface UnifiedAnalysisState {
|
|
|
184
184
|
reuseHandOffs?: Record<string, ReuseHandOff>;
|
|
185
185
|
/** See {@link ReuseRetrofit}. Never supplied by the LLM. */
|
|
186
186
|
reuseRetrofits?: Record<string, ReuseRetrofit>;
|
|
187
|
+
/** How many times skyramp_submit_report has refused the report over a blocking
|
|
188
|
+
* reuse verdict on each spec, keyed by the spec's ABSOLUTE PATH (the verdict's
|
|
189
|
+
* `file`). The refusal is bounded (`REUSE_SUBMIT_MAX_REFUSALS`): past the bound
|
|
190
|
+
* the report is accepted with the fault recorded in the row. Written by
|
|
191
|
+
* skyramp_submit_report only; never supplied by the LLM. */
|
|
192
|
+
reuseRefusals?: Record<string, number>;
|
|
187
193
|
/**
|
|
188
194
|
* SKYR-4156. Recorded video per executed browser test, keyed by test-file
|
|
189
195
|
* BASENAME (the same key `reuseOutcomes` uses, so matching needs no path
|
|
@@ -295,7 +295,11 @@ async function writeFileAtomically(target, contents) {
|
|
|
295
295
|
throw error;
|
|
296
296
|
}
|
|
297
297
|
}
|
|
298
|
-
const ROOT_FIELDS_OWNED_ELSEWHERE = [
|
|
298
|
+
const ROOT_FIELDS_OWNED_ELSEWHERE = [
|
|
299
|
+
"plan",
|
|
300
|
+
"registrationCount",
|
|
301
|
+
"reportObjections",
|
|
302
|
+
];
|
|
299
303
|
function carryForwardRootFields(data, existing) {
|
|
300
304
|
if (!existing)
|
|
301
305
|
return {};
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { escapeRegExp } from "../regex.js";
|
|
2
2
|
import { strippedSources } from "./strip-dispatch.js";
|
|
3
|
-
import { balancedCloseIndex, identifierRe, inScope, interpolationRe, lineOfOffset, } from "./lint-types.js";
|
|
3
|
+
import { balancedCloseIndex, identifierRe, inScope, interpolationRe, lineOfOffset, testBlockIndex, testStartOffsets, } from "./lint-types.js";
|
|
4
4
|
/**
|
|
5
5
|
* Deterministic checks shared by integration and provider-contract tests — the
|
|
6
6
|
* machine-checkable slice of the D5 rubric's echo_computed and array_depth
|
|
@@ -12,6 +12,12 @@ const RESPONSE_BINDING_RES = {
|
|
|
12
12
|
python: /^[ \t]*([A-Za-z_]\w*)\s*=\s*[\w.]*send_request\s*\(/gm,
|
|
13
13
|
};
|
|
14
14
|
const STATUS_USE_RE = /^\s*\.\s*(?:statusCode|status_code|status)\b/;
|
|
15
|
+
/** The SDK body-read call for a binding, spelled for the language. */
|
|
16
|
+
function helperCall(language, varName) {
|
|
17
|
+
return language === "python"
|
|
18
|
+
? `skyramp.get_response_value(${varName}, "json.path")`
|
|
19
|
+
: `getValue(${varName}, "json.path")`;
|
|
20
|
+
}
|
|
15
21
|
/** All send_request/sendRequest bindings in the stripped source. */
|
|
16
22
|
function responseBindings(stripped, language) {
|
|
17
23
|
const bindings = [];
|
|
@@ -148,11 +154,17 @@ function lintStatusOnlyResponses(stripped, commentless, language, bindings) {
|
|
|
148
154
|
message: statusUses > 0
|
|
149
155
|
? severity === "hard"
|
|
150
156
|
? `Error response \`${varName}\` is only checked for its status code — the error body (code/message/detail) must be asserted.`
|
|
151
|
-
: `Response \`${varName}\` is only checked for its status code
|
|
157
|
+
: `Response \`${varName}\` is only checked for its status code.`
|
|
152
158
|
: `Response \`${varName}\` is captured but never asserted.`,
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
159
|
+
// The advisory case is phrased as a question, not an instruction. A
|
|
160
|
+
// response no behaviour in the file depends on is CORRECT with its
|
|
161
|
+
// status code and nothing more (SKYR-4310) — telling the agent to
|
|
162
|
+
// assert the body regardless is how a 136-assertion test gets written
|
|
163
|
+
// to prove a default page size. The hard case is unchanged: an error
|
|
164
|
+
// body carries the evidence, so it is never optional.
|
|
165
|
+
remediation: severity === "warn"
|
|
166
|
+
? `If a behaviour this file tests depends on this response's body, assert the fields it depends on via \`${helperCall(language, varName)}\`. If no behaviour depends on it, the status code alone is the right answer — leave it.`
|
|
167
|
+
: `Assert the error body's fields via \`${helperCall(language, varName)}\` after the status assertion.`,
|
|
156
168
|
});
|
|
157
169
|
}
|
|
158
170
|
return findings;
|
|
@@ -294,6 +306,87 @@ function lintArrayDepth(commentless) {
|
|
|
294
306
|
}
|
|
295
307
|
return findings;
|
|
296
308
|
}
|
|
309
|
+
// An error-indicator field asserted absent on a response binding:
|
|
310
|
+
// expect(getValue(r, "error")).toBeUndefined() / .toBeNull()
|
|
311
|
+
// assert skyramp.get_response_value(r, "error") is None
|
|
312
|
+
// Group 1 is the binding (JS/TS identifiers may carry `$`), group 2 the field.
|
|
313
|
+
const ERROR_ABSENT_RE = /(?:getValue|getResponseValue|get_response_value)\s*\(\s*([\w$.]+)\s*,\s*['"](error|errors|message)['"]\s*\)\s*(?:(?:,\s*['"][^'"]*['"])?\s*\)\s*\.\s*toBe(?:Undefined|Null)\s*\(|is\s+None\b)/g;
|
|
314
|
+
// A populated-collection claim on a binding:
|
|
315
|
+
// expect(getValue(r, "items").length).toBeGreaterThan(0)
|
|
316
|
+
// assert len(skyramp.get_response_value(r, "items")) > 0
|
|
317
|
+
const NON_EMPTY_RE = /(?:getValue|getResponseValue)\s*\(\s*([\w$.]+)\s*,\s*['"][^'"]+['"]\s*\)\s*\.\s*length\s*(?:,\s*['"][^'"]*['"]\s*)?\)\s*\.\s*toBeGreaterThan\s*\(\s*0\s*\)|len\s*\(\s*[\w.]*get_response_value\s*\(\s*([\w$.]+)\s*,\s*['"][^'"]+['"]\s*\)\s*\)\s*>\s*0/g;
|
|
318
|
+
/** A success shape asserted together with the absence of an error field on the
|
|
319
|
+
* same response binding, inside the same test body. Both are legitimate on a
|
|
320
|
+
* real success body, so this is advisory only: the pattern is exactly what a
|
|
321
|
+
* test looks like when its expectations were written for the outcome the
|
|
322
|
+
* author wanted rather than the body the endpoint returns (a no-match or
|
|
323
|
+
* validation error carried in `error`). The verifier cannot see the response,
|
|
324
|
+
* so it asks the author to confirm the classification rather than blocking. */
|
|
325
|
+
function lintSuccessShapeWithErrorAbsent(commentless) {
|
|
326
|
+
const starts = testStartOffsets(commentless);
|
|
327
|
+
let m;
|
|
328
|
+
// key: `<testBlock>:<binding>` → { offset, field }
|
|
329
|
+
const errorAbsent = new Map();
|
|
330
|
+
ERROR_ABSENT_RE.lastIndex = 0;
|
|
331
|
+
while ((m = ERROR_ABSENT_RE.exec(commentless)) !== null) {
|
|
332
|
+
const key = `${testBlockIndex(starts, m.index)}:${m[1]}`;
|
|
333
|
+
if (!errorAbsent.has(key))
|
|
334
|
+
errorAbsent.set(key, { offset: m.index, field: m[2] });
|
|
335
|
+
}
|
|
336
|
+
if (errorAbsent.size === 0)
|
|
337
|
+
return [];
|
|
338
|
+
const findings = [];
|
|
339
|
+
const reported = new Set();
|
|
340
|
+
NON_EMPTY_RE.lastIndex = 0;
|
|
341
|
+
while ((m = NON_EMPTY_RE.exec(commentless)) !== null) {
|
|
342
|
+
const varName = m[1] ?? m[2];
|
|
343
|
+
const key = `${testBlockIndex(starts, m.index)}:${varName}`;
|
|
344
|
+
const absent = errorAbsent.get(key);
|
|
345
|
+
if (!absent || reported.has(key))
|
|
346
|
+
continue;
|
|
347
|
+
reported.add(key);
|
|
348
|
+
findings.push({
|
|
349
|
+
rule: "success-shape-with-error-absent",
|
|
350
|
+
severity: "warn",
|
|
351
|
+
line: lineOfOffset(commentless, absent.offset),
|
|
352
|
+
message: `\`${varName}\` is asserted to have no \`${absent.field}\` field and a non-empty collection — a success shape.`,
|
|
353
|
+
remediation: 'Confirm the recorded response for this exact request is a success body. If the endpoint answers this request with an error or no-match body (a validation 4xx, or a 2xx carrying `{"error": ...}`), assert that body\'s exact fields instead and report the gap in issuesFound. Keep the assertion as written only when the plan declared this test `expected.outcome: fail` (a bug_caught or requirement_conflict test that deliberately asserts the intended behaviour).',
|
|
354
|
+
});
|
|
355
|
+
}
|
|
356
|
+
return findings;
|
|
357
|
+
}
|
|
358
|
+
// The existence idiom on an SDK-helper read: `not.toBeNull()` / `is not None`.
|
|
359
|
+
// Weight 1 in the strength model, and the one shape MAINTENANCE_SCOPE_NOTE
|
|
360
|
+
// names by example.
|
|
361
|
+
const WEAK_EXISTENCE_RE = /(?:getValue|getResponseValue)\s*\(\s*[\w$.]+\s*,\s*['"]([^'"]+)['"][^)]*\)\s*(?:,\s*['"][^'"]*['"]\s*)?\)\s*\.\s*not\s*\.\s*toBeNull\s*\(|get_response_value\s*\(\s*[\w.]+\s*,\s*['"]([^'"]+)['"]\s*\)\s*is\s+not\s+None\b/g;
|
|
362
|
+
/** Maintenance only: an existence-tier assertion left standing inside a test
|
|
363
|
+
* function the agent TOUCHED. `MAINTENANCE_SCOPE_NOTE` holds such a function
|
|
364
|
+
* to the same standard as a new test — "an existing `is not None` on a field
|
|
365
|
+
* the diff gives a knowable value becomes that value".
|
|
366
|
+
*
|
|
367
|
+
* Warn, never hard: the verifier has no diff, so it cannot tell which fields
|
|
368
|
+
* the change made knowable. It reports the candidates and lets the agent
|
|
369
|
+
* decide, which is the honest split — the alternative is blocking a spec on a
|
|
370
|
+
* field whose value genuinely still varies. */
|
|
371
|
+
function lintWeakAssertionInTouchedTest(commentless, opts) {
|
|
372
|
+
const findings = [];
|
|
373
|
+
WEAK_EXISTENCE_RE.lastIndex = 0;
|
|
374
|
+
let m;
|
|
375
|
+
while ((m = WEAK_EXISTENCE_RE.exec(commentless)) !== null) {
|
|
376
|
+
const line = lineOfOffset(commentless, m.index);
|
|
377
|
+
if (!inScope(line, opts))
|
|
378
|
+
continue;
|
|
379
|
+
const field = m[1] ?? m[2];
|
|
380
|
+
findings.push({
|
|
381
|
+
rule: "weak-assertion-in-touched-test",
|
|
382
|
+
severity: "warn",
|
|
383
|
+
line,
|
|
384
|
+
message: `\`${field}\` is asserted only to exist, in a test function this change touches.`,
|
|
385
|
+
remediation: `A test function you touch is held to the same standard as a new one. If the diff gives \`${field}\` a knowable value, assert that value exactly. Leave the existence check only when the value genuinely varies across runs.`,
|
|
386
|
+
});
|
|
387
|
+
}
|
|
388
|
+
return findings;
|
|
389
|
+
}
|
|
297
390
|
/** Checks shared by integration and provider-contract tests. Java is never
|
|
298
391
|
* linted (count+hash gates only). */
|
|
299
392
|
export function lintApiShared(raw, language, opts) {
|
|
@@ -302,14 +395,28 @@ export function lintApiShared(raw, language, opts) {
|
|
|
302
395
|
const maintenance = opts?.scopeLines !== undefined;
|
|
303
396
|
const { commentless, stripped } = strippedSources(raw, language);
|
|
304
397
|
const findings = [];
|
|
398
|
+
// Bindings discovered once, shared by both binding-scoped lints.
|
|
399
|
+
const bindings = responseBindings(stripped, language);
|
|
400
|
+
findings.push(...lintStatusOnlyResponses(stripped, commentless, language, bindings));
|
|
401
|
+
findings.push(...lintSdkHelperBypass(stripped, bindings, language));
|
|
402
|
+
findings.push(...lintShapeOnlyChecks(commentless));
|
|
403
|
+
findings.push(...lintSuccessShapeWithErrorAbsent(commentless));
|
|
305
404
|
if (!maintenance) {
|
|
306
|
-
//
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
findings.push(...lintSdkHelperBypass(stripped, bindings, language));
|
|
310
|
-
findings.push(...lintShapeOnlyChecks(commentless));
|
|
405
|
+
// Array-depth findings anchor to a path, not a line, so there is nothing
|
|
406
|
+
// to scope them by — running them in maintenance would report untouched
|
|
407
|
+
// customer arrays with no way to filter them out.
|
|
311
408
|
findings.push(...lintArrayDepth(commentless));
|
|
312
409
|
}
|
|
410
|
+
else {
|
|
411
|
+
findings.push(...lintWeakAssertionInTouchedTest(commentless, opts));
|
|
412
|
+
}
|
|
313
413
|
findings.push(...lintPermissiveStatusMatchers(commentless, opts));
|
|
314
|
-
|
|
414
|
+
// Maintenance used to skip every lint above except the status-matcher one,
|
|
415
|
+
// because line-scoping to the ADDED lines would have flagged pre-existing
|
|
416
|
+
// customer code. The scope is now the touched test FUNCTION (SKYR-4310), so
|
|
417
|
+
// these can run: each finding is kept only when it lands inside a function
|
|
418
|
+
// the agent edited. A finding with no line cannot be placed and is dropped.
|
|
419
|
+
return maintenance
|
|
420
|
+
? findings.filter((f) => f.line !== undefined && inScope(f.line, opts))
|
|
421
|
+
: findings;
|
|
315
422
|
}
|
|
@@ -51,8 +51,8 @@ export function formatAssertionVerifyReport(r, testFile) {
|
|
|
51
51
|
if (r.strengthGateFailed) {
|
|
52
52
|
lines.push("- Assertion strength did not increase over the generated baseline: no assertion was added and no weak matcher was strengthened (e.g. `toBeVisible` → `toHaveText`). Apply the enhancement rules. A documented decline (`assertions complete` comment) is accepted only AFTER at least one assertion was added or replaced — a marker-only edit does not pass this gate.");
|
|
53
53
|
}
|
|
54
|
-
if (r.
|
|
55
|
-
lines.push(
|
|
54
|
+
if (r.weakAdditionsDominant) {
|
|
55
|
+
lines.push(`- The assertions added since the baseline are mostly existence/visibility-tier (\`toBeVisible\`, \`not.toBeNull\`, \`is not None\`, ranges)${r.meanAddedWeight !== undefined ? ` — mean strength ${r.meanAddedWeight.toFixed(1)} against a bar of 2.0` : ""}. When the exact text/value/count is knowable from the trace, source, or request, assert it exactly (\`toHaveText\`/\`toHaveCount\`/\`toBe\`/\`==\`). Adding MORE weak assertions lowers this figure — the fix is to strengthen the ones the behaviour under test depends on, and to drop the ones it does not. If only existence is genuinely knowable, document the decline:`, ` ${token} assertions complete: ${spec} — <reason exact values are not knowable>`);
|
|
56
56
|
}
|
|
57
57
|
lines.push(...warningsBlock(r.warnings, finding));
|
|
58
58
|
lines.push("Re-run this verification (verify: true) after every fix pass — skyramp_execute_test runs the same checks automatically and will not execute this test until they pass.");
|
|
@@ -4,3 +4,10 @@
|
|
|
4
4
|
* uncalled helper is skipped, so its subjects don't excuse anything.
|
|
5
5
|
*/
|
|
6
6
|
export declare function importedHelperSubjects(testFile: string, content: string): Promise<Set<string>>;
|
|
7
|
+
/**
|
|
8
|
+
* The same subjects with their CAPACITY — how many removals each may excuse:
|
|
9
|
+
* one per call site of the helper function that asserts it (an assertion the
|
|
10
|
+
* parser cannot place in a function gets one per call through the import).
|
|
11
|
+
* The removal gate consumes this budget per removed baseline subject.
|
|
12
|
+
*/
|
|
13
|
+
export declare function importedHelperSubjectCapacities(testFile: string, content: string): Promise<Map<string, number>>;
|