@skyramp/mcp 0.4.0 → 0.4.1-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/prompts/enhance-assertions/contractProviderAssertionsPrompt.js +2 -1
- package/build/prompts/enhance-assertions/integrationAssertionsPrompt.js +2 -1
- package/build/prompts/enhance-assertions/sharedAssertionRules.d.ts +1 -1
- package/build/prompts/enhance-assertions/sharedAssertionRules.js +57 -22
- package/build/prompts/enhance-assertions/uiAssertionsPrompt.js +17 -9
- package/build/prompts/test-recommendation/diffExecutionPlan.js +0 -2
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +11 -6
- package/build/prompts/testbot/testbot-prompts.js +19 -12
- package/build/recommendation/answers.d.ts +11 -7
- package/build/recommendation/answers.js +14 -10
- package/build/recommendation/pullRequestText.d.ts +18 -0
- package/build/recommendation/pullRequestText.js +31 -0
- package/build/recommendation/registerPlan.d.ts +9 -1
- package/build/recommendation/registerPlan.js +8 -1
- package/build/recommendation/runVerifiers.js +6 -0
- package/build/recommendation/types.d.ts +58 -0
- package/build/recommendation/verifierContracts.d.ts +108 -13
- package/build/recommendation/verifierContracts.js +147 -29
- package/build/recommendation/verifiers/coverage.d.ts +10 -0
- package/build/recommendation/verifiers/coverage.js +144 -22
- package/build/recommendation/verifiers/defects.d.ts +9 -0
- package/build/recommendation/verifiers/defects.js +117 -0
- package/build/recommendation/verifiers/deliveredMatchesPlan.d.ts +22 -0
- package/build/recommendation/verifiers/deliveredMatchesPlan.js +43 -0
- package/build/recommendation/verifiers/existingCoverage.js +53 -0
- package/build/recommendation/verifiers/expectedValueSourced.d.ts +14 -0
- package/build/recommendation/verifiers/expectedValueSourced.js +246 -0
- package/build/recommendation/verifiers/issueTraceability.d.ts +52 -0
- package/build/recommendation/verifiers/issueTraceability.js +197 -0
- package/build/recommendation/verifiers/requirementSourced.d.ts +2 -0
- package/build/recommendation/verifiers/requirementSourced.js +168 -0
- package/build/services/TestGenerationService.js +3 -1
- package/build/tools/code-refactor/codeReuseTool.js +1 -1
- package/build/tools/code-refactor/reuse-outcome.d.ts +1 -1
- package/build/tools/code-refactor/reuse-state.d.ts +85 -7
- package/build/tools/code-refactor/reuse-state.js +239 -34
- package/build/tools/code-refactor/utils-verify-gates.d.ts +5 -0
- package/build/tools/code-refactor/utils-verify-gates.js +103 -11
- package/build/tools/generate-tests/generateBatchScenarioRestTool.js +2 -1
- package/build/tools/submitReportTool.js +276 -39
- package/build/tools/test-management/actionsTool.js +5 -0
- package/build/tools/test-management/analyzeChangesTool.d.ts +53 -0
- package/build/tools/test-management/analyzeChangesTool.js +55 -2
- package/build/tools/test-management/registerTestPlanTool.d.ts +32 -17
- package/build/tools/test-management/registerTestPlanTool.js +105 -16
- package/build/types/ReuseOutcome.d.ts +73 -7
- package/build/types/TestAnalysis.d.ts +6 -0
- package/build/types/TestbotReport.d.ts +22 -4
- package/build/utils/AnalysisStateManager.d.ts +7 -1
- package/build/utils/AnalysisStateManager.js +5 -1
- package/build/utils/assertion-verify/api-shared-lints.js +118 -11
- package/build/utils/assertion-verify/format.js +2 -2
- package/build/utils/assertion-verify/helper-imports.d.ts +7 -0
- package/build/utils/assertion-verify/helper-imports.js +119 -27
- package/build/utils/assertion-verify/lint-types.d.ts +31 -2
- package/build/utils/assertion-verify/lint-types.js +66 -0
- package/build/utils/assertion-verify/metrics.d.ts +13 -0
- package/build/utils/assertion-verify/metrics.js +16 -0
- package/build/utils/assertion-verify/verify.d.ts +11 -6
- package/build/utils/assertion-verify/verify.js +56 -15
- package/build/utils/canonicalJson.d.ts +11 -0
- package/build/utils/canonicalJson.js +17 -0
- package/build/utils/utils-verify/action-key.d.ts +27 -0
- package/build/utils/utils-verify/action-key.js +292 -0
- package/build/utils/utils-verify/allow.d.ts +8 -1
- package/build/utils/utils-verify/allow.js +14 -1
- package/build/utils/utils-verify/call-sites.d.ts +76 -8
- package/build/utils/utils-verify/call-sites.js +256 -70
- package/build/utils/utils-verify/language-spec.d.ts +3 -2
- package/build/utils/utils-verify/parse.d.ts +22 -3
- package/build/utils/utils-verify/parse.js +123 -52
- package/build/utils/utils-verify/verify.d.ts +33 -3
- package/build/utils/utils-verify/verify.js +126 -12
- package/build/utils/workspaceAuth.d.ts +59 -19
- package/build/utils/workspaceAuth.js +228 -31
- package/package.json +1 -1
- package/plugin/prompts/generate-tests/execution-plan.md +3 -3
- package/plugin/prompts/generate-tests/generation.md +1 -0
- package/plugin/prompts/plan-tests.md +41 -15
- package/plugin/prompts/testbot-task1.md +3 -9
- package/build/prompts/testbot/planDeclarations.d.ts +0 -6
- package/build/prompts/testbot/planDeclarations.js +0 -9
- package/plugin/prompts/declaring-a-plan.md +0 -20
|
@@ -1,46 +1,82 @@
|
|
|
1
1
|
import { normalizePath } from "./language-spec.js";
|
|
2
|
+
import { actionKeyOf } from "./action-key.js";
|
|
2
3
|
import { stripComments, stripCommentsAndStrings, stripStrings, } from "../pom-scope/strip.js";
|
|
3
|
-
/**
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
|
|
4
|
+
/** Whether the `<` at `i` opens a generic (`Record<string, …>`, `Array<{…}>`) rather
|
|
5
|
+
* than comparing (`i < 3`): a generic follows an identifier or `>` and is followed
|
|
6
|
+
* by a type start; a comparison is followed by a number, a space and a value, or
|
|
7
|
+
* an operator. */
|
|
8
|
+
function opensGeneric(sig, i) {
|
|
9
|
+
let p = i - 1;
|
|
10
|
+
while (p >= 0 && /\s/.test(sig[p]))
|
|
11
|
+
p--;
|
|
12
|
+
if (p < 0 || !/[\w$>\]]/.test(sig[p]))
|
|
13
|
+
return false;
|
|
14
|
+
let n = i + 1;
|
|
15
|
+
while (n < sig.length && /\s/.test(sig[n]))
|
|
16
|
+
n++;
|
|
17
|
+
return /[A-Za-z_${[(]/.test(sig[n] ?? "");
|
|
18
|
+
}
|
|
19
|
+
/** Bracket depth tracker for a signature: `(`/`[`/`{` always nest; `<` nests only as
|
|
20
|
+
* a generic and `>` closes only an open generic — never the `>` of `=>`, and never
|
|
21
|
+
* a comparison in a default value (`qty = max > 0 ? max : 1`). */
|
|
22
|
+
function bracketStep(sig, i, d) {
|
|
23
|
+
const ch = sig[i];
|
|
24
|
+
if ("([{".includes(ch))
|
|
25
|
+
d.depth++;
|
|
26
|
+
else if (")]}".includes(ch))
|
|
27
|
+
d.depth--;
|
|
28
|
+
else if (ch === "<" && opensGeneric(sig, i)) {
|
|
29
|
+
d.depth++;
|
|
30
|
+
d.angle++;
|
|
31
|
+
}
|
|
32
|
+
else if (ch === ">" && d.angle > 0 && sig[i - 1] !== "=") {
|
|
33
|
+
d.depth--;
|
|
34
|
+
d.angle--;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
/** The declared parameters of a signature, raw text trimmed, empty entries dropped
|
|
38
|
+
* (a trailing comma). Reads the signature from the definition line to its body
|
|
39
|
+
* opener (Prettier wraps long signatures), takes the first balanced `(...)`, and
|
|
40
|
+
* splits it on top-level commas — so a comma inside a default value, a generic, or
|
|
41
|
+
* a binding pattern never splits. `undefined` when the text holds no readable
|
|
42
|
+
* parameter list. Language-neutral: both a TS `{` and a Python `:` terminator lie
|
|
43
|
+
* outside the parentheses this reads. */
|
|
44
|
+
export function signatureParams(sigLines) {
|
|
7
45
|
const sig = sigLines.join(" ");
|
|
8
|
-
|
|
46
|
+
// The parameter list's `(` is the first one outside a generic: a constraint may
|
|
47
|
+
// hold parentheses of its own (`<T extends (a: string) => void>(page, cb)`).
|
|
48
|
+
let open = -1;
|
|
49
|
+
const d = { depth: 0, angle: 0 };
|
|
50
|
+
for (let i = 0; i < sig.length; i++) {
|
|
51
|
+
if (sig[i] === "(" && d.angle === 0) {
|
|
52
|
+
open = i;
|
|
53
|
+
break;
|
|
54
|
+
}
|
|
55
|
+
bracketStep(sig, i, d);
|
|
56
|
+
}
|
|
9
57
|
if (open === -1) {
|
|
10
|
-
// Bare-arrow form: `export const open = async page => …` — one parameter
|
|
11
|
-
// untyped by construction (a typed single parameter requires parentheses).
|
|
58
|
+
// Bare-arrow form: `export const open = async page => …` — one parameter.
|
|
12
59
|
const bare = /(?:^|=)\s*(?:async\s+)?([A-Za-z_$][\w$]*)\s*=>/.exec(sig);
|
|
13
|
-
return bare ? [bare[1]] :
|
|
60
|
+
return bare ? [bare[1]] : undefined;
|
|
14
61
|
}
|
|
15
|
-
// `>` closes a nesting level only when it is not the arrow token's tail:
|
|
16
|
-
// `(callback: (x: string) => void, page)` must not close at `=>`.
|
|
17
|
-
const closes = (ch, prev) => ch === ")" || ch === "]" || ch === "}" || (ch === ">" && prev !== "=");
|
|
18
|
-
let depth = 0;
|
|
19
62
|
let close = -1;
|
|
63
|
+
const c = { depth: 0, angle: 0 };
|
|
20
64
|
for (let i = open; i < sig.length; i++) {
|
|
21
|
-
|
|
22
|
-
if (
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
depth--;
|
|
26
|
-
if (depth === 0) {
|
|
27
|
-
close = i;
|
|
28
|
-
break;
|
|
29
|
-
}
|
|
65
|
+
bracketStep(sig, i, c);
|
|
66
|
+
if (c.depth === 0) {
|
|
67
|
+
close = i;
|
|
68
|
+
break;
|
|
30
69
|
}
|
|
31
70
|
}
|
|
32
71
|
if (close === -1)
|
|
33
|
-
return
|
|
72
|
+
return undefined;
|
|
34
73
|
const params = [];
|
|
35
74
|
let cur = "";
|
|
36
|
-
|
|
75
|
+
const e = { depth: 0, angle: 0 };
|
|
37
76
|
for (let i = open + 1; i < close; i++) {
|
|
38
77
|
const ch = sig[i];
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
else if (closes(ch, sig[i - 1] ?? ""))
|
|
42
|
-
depth--;
|
|
43
|
-
if (ch === "," && depth === 0) {
|
|
78
|
+
bracketStep(sig, i, e);
|
|
79
|
+
if (ch === "," && e.depth === 0) {
|
|
44
80
|
params.push(cur);
|
|
45
81
|
cur = "";
|
|
46
82
|
}
|
|
@@ -48,11 +84,17 @@ export function untypedParamsOf(sigLines) {
|
|
|
48
84
|
cur += ch;
|
|
49
85
|
}
|
|
50
86
|
params.push(cur);
|
|
87
|
+
// Python's bare `*` and `/` are keyword-only / positional-only MARKERS, not
|
|
88
|
+
// parameters.
|
|
89
|
+
return params
|
|
90
|
+
.map((p) => p.trim())
|
|
91
|
+
.filter((p) => p !== "" && p !== "*" && p !== "/");
|
|
92
|
+
}
|
|
93
|
+
/** The parameter names in a TS signature that carry no `: type`. */
|
|
94
|
+
export function untypedParamsOf(sigLines) {
|
|
51
95
|
const out = [];
|
|
52
|
-
for (const raw of
|
|
53
|
-
const param = raw.
|
|
54
|
-
if (!param)
|
|
55
|
-
continue;
|
|
96
|
+
for (const raw of signatureParams(sigLines) ?? []) {
|
|
97
|
+
const param = raw.replace(/^\.\.\./, "");
|
|
56
98
|
// A default value means TypeScript infers the type — not a finding, even
|
|
57
99
|
// unannotated (`expectedStatus = 200`). `=>` inside a function-type
|
|
58
100
|
// annotation is not a default.
|
|
@@ -62,17 +104,13 @@ export function untypedParamsOf(sigLines) {
|
|
|
62
104
|
// `{ page: browserPage }` is implicitly any unless a `:` follows the
|
|
63
105
|
// complete pattern.
|
|
64
106
|
if (param.startsWith("{") || param.startsWith("[")) {
|
|
65
|
-
|
|
107
|
+
const bd = { depth: 0, angle: 0 };
|
|
66
108
|
let after = -1;
|
|
67
109
|
for (let i = 0; i < param.length; i++) {
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
if (d === 0) {
|
|
73
|
-
after = i + 1;
|
|
74
|
-
break;
|
|
75
|
-
}
|
|
110
|
+
bracketStep(param, i, bd);
|
|
111
|
+
if (bd.depth === 0) {
|
|
112
|
+
after = i + 1;
|
|
113
|
+
break;
|
|
76
114
|
}
|
|
77
115
|
}
|
|
78
116
|
if (after === -1 || !param.slice(after).trimStart().startsWith(":"))
|
|
@@ -103,6 +141,11 @@ export function untypedParamsOf(sigLines) {
|
|
|
103
141
|
*/
|
|
104
142
|
export function parseUtilsFile(content, spec) {
|
|
105
143
|
const lines = content.split("\n");
|
|
144
|
+
// Structure — where a body ends — is read on a copy with comments and string
|
|
145
|
+
// contents blanked (same line count and lengths): a `{` typed into a field
|
|
146
|
+
// (`fill("{")`) or a `#` in a Python string must not move a helper's end and
|
|
147
|
+
// swallow every helper after it.
|
|
148
|
+
const structure = blankCommentsAndStrings(content, spec).split("\n");
|
|
106
149
|
const helpers = [];
|
|
107
150
|
let i = 0;
|
|
108
151
|
while (i < lines.length) {
|
|
@@ -114,13 +157,23 @@ export function parseUtilsFile(content, spec) {
|
|
|
114
157
|
const name = m.slice(1).find((g) => g !== undefined) ?? "";
|
|
115
158
|
const start = i;
|
|
116
159
|
const end = spec.language === "python"
|
|
117
|
-
? pythonBodyEnd(
|
|
118
|
-
: braceBodyEnd(
|
|
160
|
+
? pythonBodyEnd(structure, start)
|
|
161
|
+
: braceBodyEnd(structure, start);
|
|
162
|
+
// A `typing.overload` stub declares a signature of the ONE function that follows
|
|
163
|
+
// it; it is not a second helper, and reading it as one would report the pair as
|
|
164
|
+
// a name collision.
|
|
165
|
+
if (spec.language === "python" && isOverloadStub(lines, start)) {
|
|
166
|
+
i = Math.max(end, start + 1);
|
|
167
|
+
continue;
|
|
168
|
+
}
|
|
119
169
|
const body = lines.slice(start, end);
|
|
120
170
|
// Scan a copy with comments and string contents blanked, so a commented-out
|
|
121
171
|
// `path=` or an `expect(` inside a string never keys or flags anything; the
|
|
122
172
|
// original line is what the agent is shown.
|
|
123
173
|
const scan = blankCommentsAndStrings(body.join("\n"), spec).split("\n");
|
|
174
|
+
// The blanked copy: a comment inside a wrapped signature must not reach the
|
|
175
|
+
// agent as a "parameter", and a comma inside a default string must not split one.
|
|
176
|
+
const signature = signatureLines(scan);
|
|
124
177
|
const helper = {
|
|
125
178
|
name,
|
|
126
179
|
line: start + 1,
|
|
@@ -128,11 +181,11 @@ export function parseUtilsFile(content, spec) {
|
|
|
128
181
|
wrapsRequest: false,
|
|
129
182
|
hasGuard: hasPageErrorGuard(body.join("\n"), spec),
|
|
130
183
|
assertions: [],
|
|
131
|
-
untypedParams: spec.language === "typescript"
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
184
|
+
untypedParams: spec.language === "typescript" ? untypedParamsOf(signature) : [],
|
|
185
|
+
arity: signatureParams(signature)?.length,
|
|
186
|
+
// Comments blanked, strings kept: a commented-out click is not a step, and
|
|
187
|
+
// the selector literals are the key.
|
|
188
|
+
actionKey: actionKeyOf(blankComments(body.join("\n"), spec), spec.language),
|
|
136
189
|
};
|
|
137
190
|
let requests = 0;
|
|
138
191
|
for (let k = 0; k < body.length; k++) {
|
|
@@ -198,8 +251,9 @@ function depthOf(code) {
|
|
|
198
251
|
d--;
|
|
199
252
|
return d;
|
|
200
253
|
}
|
|
201
|
-
/** True when the `path` value is a single string literal ending the argument
|
|
202
|
-
|
|
254
|
+
/** True when the `path` value is a single string literal ending the argument — a
|
|
255
|
+
* concatenation (`"/api/orders/" + str(id) + "/items"`) has no route this can name. */
|
|
256
|
+
export function isSingleLiteral(line, m) {
|
|
203
257
|
const after = line.slice(m.index + m[0].length).replace(/^\s+/, "");
|
|
204
258
|
return (after === "" ||
|
|
205
259
|
after.startsWith(",") ||
|
|
@@ -299,13 +353,30 @@ function signatureLines(body) {
|
|
|
299
353
|
}
|
|
300
354
|
return out;
|
|
301
355
|
}
|
|
356
|
+
/** Whether the def at `start` carries `@overload` / `@typing.overload` anywhere in
|
|
357
|
+
* its decorator stack (comments and other decorators may sit between). */
|
|
358
|
+
function isOverloadStub(lines, start) {
|
|
359
|
+
// The decorator stack sits above the def with comments and blank lines allowed
|
|
360
|
+
// between; any `@overload` in that stack marks the def a stub.
|
|
361
|
+
for (let i = start - 1; i >= 0; i--) {
|
|
362
|
+
const l = lines[i].trim();
|
|
363
|
+
if (l === "" || l.startsWith("#"))
|
|
364
|
+
continue;
|
|
365
|
+
if (!l.startsWith("@"))
|
|
366
|
+
return false;
|
|
367
|
+
if (/^@(?:typing\.)?overload\b/.test(l))
|
|
368
|
+
return true;
|
|
369
|
+
}
|
|
370
|
+
return false;
|
|
371
|
+
}
|
|
302
372
|
/** Exclusive end index of a Python def starting at `start`: the next non-blank line at
|
|
303
|
-
* column 0 (a following def, import, or module statement).
|
|
373
|
+
* column 0 (a following def, import, or module statement). A column-0 `):` is the
|
|
374
|
+
* close of a Black-wrapped signature, not a statement — it stays in the helper. */
|
|
304
375
|
function pythonBodyEnd(lines, start) {
|
|
305
376
|
let i = start + 1;
|
|
306
377
|
while (i < lines.length) {
|
|
307
378
|
const l = lines[i];
|
|
308
|
-
if (l.trim() !== "" &&
|
|
379
|
+
if (l.trim() !== "" && !/^[\s)]/.test(l))
|
|
309
380
|
break;
|
|
310
381
|
i++;
|
|
311
382
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type InlineCallSite } from "./call-sites.js";
|
|
1
|
+
import { type InlineCallSite, type UnextractedDuplicate } from "./call-sites.js";
|
|
2
2
|
import { type UtilsAdvisoryKind, type UtilsViolationKind } from "./allow.js";
|
|
3
3
|
import { type SingleImporterHelper } from "./importers.js";
|
|
4
4
|
import { type HelperFamilySpec, type UtilsLanguageSpec } from "./language-spec.js";
|
|
@@ -27,6 +27,8 @@ export interface UtilsVerifyResult {
|
|
|
27
27
|
helpers: number;
|
|
28
28
|
/** Helper names the delivered spec imports from those files. */
|
|
29
29
|
imported: number;
|
|
30
|
+
/** Those names, sorted — what the report row shows beside the count. */
|
|
31
|
+
importedNames: string[];
|
|
30
32
|
violations: UtilsViolation[];
|
|
31
33
|
/** Violations covered by a `reuse-verify: allow` marker — a documented decline. */
|
|
32
34
|
allowed: UtilsViolation[];
|
|
@@ -47,9 +49,23 @@ export interface UtilsVerifyResult {
|
|
|
47
49
|
/** Advisory (does not affect `ok`): inline request calls in sibling generated tests
|
|
48
50
|
* that a utils helper already wraps — see findSiblingInlineCallSites. API family only. */
|
|
49
51
|
inlineCallSites: InlineCallSite[];
|
|
52
|
+
/** Advisory (does not affect `ok`): operations written in two or more sibling
|
|
53
|
+
* generated tests (this one included) that NO utils helper wraps — the extraction
|
|
54
|
+
* not performed, beside `inlineCallSites`' helper not used. See
|
|
55
|
+
* findUnextractedDuplicates. API family only. */
|
|
56
|
+
unextractedDuplicates: UnextractedDuplicate[];
|
|
50
57
|
/** Helpers imported by 0 or 1 delivered test (SKYR-4276 A6). Advisory: a first run
|
|
51
58
|
* seeds every helper single-use by design; the count is the trend to watch. */
|
|
52
59
|
singleImporters: SingleImporterHelper[];
|
|
60
|
+
/** The numbers the baseline assertion check compared — present only when a
|
|
61
|
+
* baseline was supplied and the spec was readable. The check is a COUNT: it
|
|
62
|
+
* cannot see one assertion swapped for another. The caller's text says so. */
|
|
63
|
+
assertionCount?: {
|
|
64
|
+
baseline: number;
|
|
65
|
+
delivered: number;
|
|
66
|
+
inSpec: number;
|
|
67
|
+
inHelpers: number;
|
|
68
|
+
};
|
|
53
69
|
}
|
|
54
70
|
export interface VerifyUtilsParams {
|
|
55
71
|
testFile: string;
|
|
@@ -70,8 +86,14 @@ export interface VerifyUtilsParams {
|
|
|
70
86
|
*
|
|
71
87
|
* Three invariants, each a class of drift measured in the SKYR-4196 Part B evals:
|
|
72
88
|
* - `duplicate-helper`: two helpers for one method+path (run 4 — `create_product`
|
|
73
|
-
* beside `create_list_reflect_product`)
|
|
74
|
-
*
|
|
89
|
+
* beside `create_list_reflect_product`), or one NAME defined with two parameter
|
|
90
|
+
* lists in any family (`addProductToCart(page, quantity)` beside
|
|
91
|
+
* `addProductToCart(page)`); the shared file must hold one canonical helper per
|
|
92
|
+
* step, extended with defaulted parameters rather than duplicated. Browser helpers
|
|
93
|
+
* have no route; two of them performing one action sequence (see actionKeyOf)
|
|
94
|
+
* are the `duplicate-action` ADVISORY.
|
|
95
|
+
* - `helper-redefined`: one name defined twice in ONE file — the module does not
|
|
96
|
+
* load (TypeScript) or the second def replaces the first (Python). Never markerable.
|
|
75
97
|
* - `body-assertion`: any assertion other than a status-code comparison inside a
|
|
76
98
|
* helper (runs 5 and 11 — `assert total_amount == 16.0` in `update_order`); a
|
|
77
99
|
* body assertion carries one scenario's values into a function every scenario calls.
|
|
@@ -82,6 +104,14 @@ export interface VerifyUtilsParams {
|
|
|
82
104
|
* except to count what the spec imports.
|
|
83
105
|
*/
|
|
84
106
|
export declare function verifyUtils(params: VerifyUtilsParams): Promise<UtilsVerifyResult>;
|
|
107
|
+
/** The compared counts as one phrase, shared by the assertion-loss detail and the
|
|
108
|
+
* PASSED line's note so the two cannot drift. */
|
|
109
|
+
export declare function describeAssertionCount(c: {
|
|
110
|
+
baseline: number;
|
|
111
|
+
delivered: number;
|
|
112
|
+
inSpec: number;
|
|
113
|
+
inHelpers: number;
|
|
114
|
+
}): string;
|
|
85
115
|
/**
|
|
86
116
|
* Python only. The import form that works here is not the one that works in the
|
|
87
117
|
* customer's repo, and the difference is invisible in this run.
|
|
@@ -4,7 +4,7 @@ import { logger } from "../logger.js";
|
|
|
4
4
|
import { locateUtilsFiles } from "./locate.js";
|
|
5
5
|
import { blankCommentsAndStrings, countAssertions, hasPageErrorGuard, parseUtilsFile, } from "./parse.js";
|
|
6
6
|
import { escapeRegExp } from "../regex.js";
|
|
7
|
-
import { findSiblingInlineCallSites, generatedSiblings, } from "./call-sites.js";
|
|
7
|
+
import { findSiblingInlineCallSites, findUnextractedDuplicates, generatedSiblings, } from "./call-sites.js";
|
|
8
8
|
import { parseUtilsAllows, } from "./allow.js";
|
|
9
9
|
import { singleImporterHelpers, } from "./importers.js";
|
|
10
10
|
import { helperFamilyFor, helperFamilyOf, scenarioNameViolation, utilsSpecFor, utilsSpecForFile, } from "./language-spec.js";
|
|
@@ -13,8 +13,14 @@ import { helperFamilyFor, helperFamilyOf, scenarioNameViolation, utilsSpecFor, u
|
|
|
13
13
|
*
|
|
14
14
|
* Three invariants, each a class of drift measured in the SKYR-4196 Part B evals:
|
|
15
15
|
* - `duplicate-helper`: two helpers for one method+path (run 4 — `create_product`
|
|
16
|
-
* beside `create_list_reflect_product`)
|
|
17
|
-
*
|
|
16
|
+
* beside `create_list_reflect_product`), or one NAME defined with two parameter
|
|
17
|
+
* lists in any family (`addProductToCart(page, quantity)` beside
|
|
18
|
+
* `addProductToCart(page)`); the shared file must hold one canonical helper per
|
|
19
|
+
* step, extended with defaulted parameters rather than duplicated. Browser helpers
|
|
20
|
+
* have no route; two of them performing one action sequence (see actionKeyOf)
|
|
21
|
+
* are the `duplicate-action` ADVISORY.
|
|
22
|
+
* - `helper-redefined`: one name defined twice in ONE file — the module does not
|
|
23
|
+
* load (TypeScript) or the second def replaces the first (Python). Never markerable.
|
|
18
24
|
* - `body-assertion`: any assertion other than a status-code comparison inside a
|
|
19
25
|
* helper (runs 5 and 11 — `assert total_amount == 16.0` in `update_order`); a
|
|
20
26
|
* body assertion carries one scenario's values into a function every scenario calls.
|
|
@@ -39,6 +45,7 @@ export async function verifyUtils(params) {
|
|
|
39
45
|
utilsFiles,
|
|
40
46
|
helpers: 0,
|
|
41
47
|
imported: 0,
|
|
48
|
+
importedNames: [],
|
|
42
49
|
violations: [],
|
|
43
50
|
allowed: [],
|
|
44
51
|
advisories: [],
|
|
@@ -46,6 +53,7 @@ export async function verifyUtils(params) {
|
|
|
46
53
|
spec,
|
|
47
54
|
fallbackFamily: family,
|
|
48
55
|
inlineCallSites: [],
|
|
56
|
+
unextractedDuplicates: [],
|
|
49
57
|
singleImporters: [],
|
|
50
58
|
};
|
|
51
59
|
if (!spec)
|
|
@@ -65,9 +73,12 @@ export async function verifyUtils(params) {
|
|
|
65
73
|
const importedNames = new Set();
|
|
66
74
|
const importedByFile = new Map();
|
|
67
75
|
const helpersOfFile = new Map();
|
|
68
|
-
// Route
|
|
69
|
-
// across files, and the same method+path
|
|
76
|
+
// Route and name maps span every utils file: the reuse prompt allows the helpers
|
|
77
|
+
// to split across files, and the same method+path — or the same name — in two of
|
|
78
|
+
// them is still one duplicate.
|
|
70
79
|
const byRoute = new Map();
|
|
80
|
+
const byName = new Map();
|
|
81
|
+
const byAction = new Map();
|
|
71
82
|
for (const file of utilsFiles) {
|
|
72
83
|
let content;
|
|
73
84
|
try {
|
|
@@ -120,7 +131,7 @@ export async function verifyUtils(params) {
|
|
|
120
131
|
const k = `${a.kind}\0${a.helper}`;
|
|
121
132
|
budget.set(k, (budget.get(k) ?? 0) + 1);
|
|
122
133
|
}
|
|
123
|
-
for (const v of violationsIn(helpers, file, family, spec, byRoute)) {
|
|
134
|
+
for (const v of violationsIn(helpers, file, family, spec, { byRoute, byName, byAction }, result.advisories)) {
|
|
124
135
|
const k = `${v.kind}\0${v.helper}`;
|
|
125
136
|
const left = budget.get(k) ?? 0;
|
|
126
137
|
if (left > 0) {
|
|
@@ -136,7 +147,8 @@ export async function verifyUtils(params) {
|
|
|
136
147
|
}
|
|
137
148
|
}
|
|
138
149
|
result.imported = importedNames.size;
|
|
139
|
-
|
|
150
|
+
result.importedNames = [...importedNames].sort();
|
|
151
|
+
// One directory walk feeds every sibling advisory: each sibling is read exactly once.
|
|
140
152
|
let siblings = [];
|
|
141
153
|
try {
|
|
142
154
|
siblings = await generatedSiblings(params.testFile, spec, {
|
|
@@ -170,7 +182,14 @@ export async function verifyUtils(params) {
|
|
|
170
182
|
guardInHelpers = true;
|
|
171
183
|
}
|
|
172
184
|
}
|
|
173
|
-
|
|
185
|
+
const inSpec = countAssertions(specContent, spec);
|
|
186
|
+
result.assertionCount = {
|
|
187
|
+
baseline: params.baseline.assertions,
|
|
188
|
+
delivered: inSpec + inHelpers,
|
|
189
|
+
inSpec,
|
|
190
|
+
inHelpers,
|
|
191
|
+
};
|
|
192
|
+
for (const v of baselineViolations(params.baseline, params.testFile, spec, specContent, inSpec, inHelpers, guardInHelpers)) {
|
|
174
193
|
// A marker in any utils file documents an assertion-loss (never a removed
|
|
175
194
|
// guard). At most one loss finding exists per verify, so a matching marker's
|
|
176
195
|
// presence is the whole budget — nothing to consume.
|
|
@@ -194,15 +213,25 @@ export async function verifyUtils(params) {
|
|
|
194
213
|
catch {
|
|
195
214
|
/* advisory only — never fails a verify */
|
|
196
215
|
}
|
|
216
|
+
try {
|
|
217
|
+
result.unextractedDuplicates = await findUnextractedDuplicates(params.testFile, utilsFiles, spec, siblings);
|
|
218
|
+
}
|
|
219
|
+
catch {
|
|
220
|
+
/* advisory only — never fails a verify */
|
|
221
|
+
}
|
|
197
222
|
}
|
|
198
223
|
return result;
|
|
199
224
|
}
|
|
225
|
+
/** The compared counts as one phrase, shared by the assertion-loss detail and the
|
|
226
|
+
* PASSED line's note so the two cannot drift. */
|
|
227
|
+
export function describeAssertionCount(c) {
|
|
228
|
+
return `${c.baseline} assertions at the modularization hand-out, ${c.delivered} delivered (${c.inSpec} in the test, ${c.inHelpers} in the helpers it imports and calls)`;
|
|
229
|
+
}
|
|
200
230
|
/** The baseline (A7/A8) findings for a delivered spec — pure; marker handling is the
|
|
201
231
|
* caller's. Runs whether or not a utils file was located: deleting the utils file
|
|
202
232
|
* (with the moved assertions and guard in it) must not read as "nothing to verify". */
|
|
203
|
-
function baselineViolations(baseline, testFile, spec, specContent, inHelpers, guardInHelpers) {
|
|
233
|
+
function baselineViolations(baseline, testFile, spec, specContent, inSpec, inHelpers, guardInHelpers) {
|
|
204
234
|
const found = [];
|
|
205
|
-
const inSpec = countAssertions(specContent, spec);
|
|
206
235
|
const delivered = inSpec + inHelpers;
|
|
207
236
|
// Marker grammar takes an identifier: `orders.spec.ts` → `orders_spec_ts`.
|
|
208
237
|
const stem = path.basename(testFile).replace(/\W/g, "_");
|
|
@@ -212,7 +241,7 @@ function baselineViolations(baseline, testFile, spec, specContent, inHelpers, gu
|
|
|
212
241
|
file: testFile,
|
|
213
242
|
helper: stem,
|
|
214
243
|
line: 1,
|
|
215
|
-
detail: `${baseline.assertions
|
|
244
|
+
detail: `${describeAssertionCount({ baseline: baseline.assertions, delivered, inSpec, inHelpers })} — restore each removed check in the test body after the helper call it verifies, or document a deliberate consolidation with the allow marker`,
|
|
216
245
|
});
|
|
217
246
|
if (baseline.pageErrorGuard &&
|
|
218
247
|
!guardInHelpers &&
|
|
@@ -334,14 +363,19 @@ export function importFormViolations(testFile, specContent, utilsFiles) {
|
|
|
334
363
|
function blankTripleQuoted(src) {
|
|
335
364
|
return src.replace(/("""|''')[\s\S]*?\1/g, (m) => m.replace(/[^\n]/g, " "));
|
|
336
365
|
}
|
|
337
|
-
function violationsIn(helpers, file, fallback, spec,
|
|
366
|
+
function violationsIn(helpers, file, fallback, spec, seen, advisories) {
|
|
367
|
+
const { byRoute, byName, byAction } = seen;
|
|
368
|
+
// Names seen in THIS file: a second definition here is a redefinition, not drift.
|
|
369
|
+
const inThisFile = new Map();
|
|
338
370
|
const out = [];
|
|
339
371
|
for (const h of helpers) {
|
|
340
372
|
const family = helperFamilyOf(h, fallback);
|
|
373
|
+
let routeDuplicateOf;
|
|
341
374
|
if (family.routeDuplicates && h.method && h.normalizedPath) {
|
|
342
375
|
const key = `${h.method} ${h.normalizedPath}`;
|
|
343
376
|
const first = byRoute.get(key);
|
|
344
377
|
if (first) {
|
|
378
|
+
routeDuplicateOf = first;
|
|
345
379
|
out.push({
|
|
346
380
|
kind: "duplicate-helper",
|
|
347
381
|
file,
|
|
@@ -354,6 +388,75 @@ function violationsIn(helpers, file, fallback, spec, byRoute) {
|
|
|
354
388
|
byRoute.set(key, { ...h, file });
|
|
355
389
|
}
|
|
356
390
|
}
|
|
391
|
+
// One name defined twice. In ONE file that is a redefinition, whatever the
|
|
392
|
+
// parameter lists: a TypeScript module with two top-level functions of one name
|
|
393
|
+
// does not load, and in Python the second def silently replaces the first — so it
|
|
394
|
+
// blocks and takes no marker. Across two files each caller imports the module it
|
|
395
|
+
// uses; different parameter lists there are the drift the ticket measured
|
|
396
|
+
// (`addProductToCart(page, quantity)` in one run, `addProductToCart(page)` in
|
|
397
|
+
// the next), reported as `duplicate-helper`, documentable, once per pair — a pair
|
|
398
|
+
// the route check already reported carries its repair. Same name, same arity,
|
|
399
|
+
// two files: nothing here; the action key speaks if the bodies agree.
|
|
400
|
+
let nameReported = false;
|
|
401
|
+
const inFile = inThisFile.get(h.name);
|
|
402
|
+
const sameName = byName.get(h.name);
|
|
403
|
+
if (inFile) {
|
|
404
|
+
nameReported = true;
|
|
405
|
+
out.push({
|
|
406
|
+
kind: "helper-redefined",
|
|
407
|
+
file,
|
|
408
|
+
helper: h.name,
|
|
409
|
+
line: h.line,
|
|
410
|
+
detail: spec.language === "python"
|
|
411
|
+
? `${h.name} is defined twice in this file (also at ${path.basename(inFile.file)}:${inFile.line}); this def silently replaces the first, and every caller written against the first signature breaks. Keep ONE definition, give every parameter the other had a default, and update the call sites`
|
|
412
|
+
: `${h.name} is defined twice in this file (also at ${path.basename(inFile.file)}:${inFile.line}); a module with two top-level functions of one name does not load. Keep ONE definition, give every parameter the other had a default, and update the call sites`,
|
|
413
|
+
});
|
|
414
|
+
}
|
|
415
|
+
else if (sameName &&
|
|
416
|
+
h.arity !== undefined &&
|
|
417
|
+
sameName.arity !== undefined &&
|
|
418
|
+
h.arity !== sameName.arity &&
|
|
419
|
+
!(routeDuplicateOf?.file === sameName.file &&
|
|
420
|
+
routeDuplicateOf?.line === sameName.line)) {
|
|
421
|
+
nameReported = true;
|
|
422
|
+
out.push({
|
|
423
|
+
kind: "duplicate-helper",
|
|
424
|
+
file,
|
|
425
|
+
helper: h.name,
|
|
426
|
+
line: h.line,
|
|
427
|
+
detail: `${h.name} is defined in two modules with different parameter lists — ${h.arity} parameter${h.arity === 1 ? "" : "s"} here, ${sameName.arity} at ${path.basename(sameName.file)}:${sameName.line}; the two modules disagree on one operation's signature, and the next test to import it gets whichever it finds first. Keep ONE definition, give every parameter the other lacked a default, and point every importer at it`,
|
|
428
|
+
});
|
|
429
|
+
}
|
|
430
|
+
if (!inFile)
|
|
431
|
+
inThisFile.set(h.name, { ...h, file });
|
|
432
|
+
if (!sameName)
|
|
433
|
+
byName.set(h.name, { ...h, file });
|
|
434
|
+
// The browser analogue of the route key: two request-free helpers performing one
|
|
435
|
+
// action sequence are one operation under two names. Advisory — the key is read
|
|
436
|
+
// from selector literals, and its false positive must not block a run. A pair the
|
|
437
|
+
// name checks just reported is one finding, not two; a same-name pair they were
|
|
438
|
+
// silent on (two files, equal arity) is this check's — exactly one detector
|
|
439
|
+
// speaks for any pair.
|
|
440
|
+
if (family.family === "browser" && h.actionKey) {
|
|
441
|
+
const first = byAction.get(h.actionKey);
|
|
442
|
+
if (first && !(nameReported && first.name === h.name)) {
|
|
443
|
+
advisories.push({
|
|
444
|
+
kind: "duplicate-action",
|
|
445
|
+
file,
|
|
446
|
+
helper: h.name,
|
|
447
|
+
line: h.line,
|
|
448
|
+
// Only the parameter COUNT is compared, and only when both were read.
|
|
449
|
+
detail: first.name === h.name
|
|
450
|
+
? `${h.name} is defined twice (also at ${path.basename(first.file)}:${first.line}) with the same ${h.arity !== undefined && h.arity === first.arity
|
|
451
|
+
? "parameter count and the same "
|
|
452
|
+
: ""}action sequence: ${describeActionKey(h.actionKey)}; keep one definition and point every importer at it`
|
|
453
|
+
: `${h.name} and ${first.name} (${path.basename(first.file)}:${first.line}) perform the same action sequence: ${describeActionKey(h.actionKey)}`,
|
|
454
|
+
});
|
|
455
|
+
}
|
|
456
|
+
else if (!first) {
|
|
457
|
+
byAction.set(h.actionKey, { ...h, file });
|
|
458
|
+
}
|
|
459
|
+
}
|
|
357
460
|
for (const a of h.assertions) {
|
|
358
461
|
if (!family.isAllowed(spec, a.text)) {
|
|
359
462
|
out.push({
|
|
@@ -378,3 +481,14 @@ function violationsIn(helpers, file, fallback, spec, byRoute) {
|
|
|
378
481
|
}
|
|
379
482
|
return out;
|
|
380
483
|
}
|
|
484
|
+
/** `click empty-cart-btn; fill qty` → `click \`empty-cart-btn\`; fill \`qty\`` — the
|
|
485
|
+
* key as the agent reads it, selectors quoted so a `#id` never reads as prose. */
|
|
486
|
+
function describeActionKey(key) {
|
|
487
|
+
return key
|
|
488
|
+
.split("; ")
|
|
489
|
+
.map((step) => {
|
|
490
|
+
const at = step.indexOf(" ");
|
|
491
|
+
return `${step.slice(0, at)} \`${step.slice(at + 1)}\``;
|
|
492
|
+
})
|
|
493
|
+
.join("; ");
|
|
494
|
+
}
|