@skyramp/mcp 0.4.0 → 0.4.1-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/prompts/enhance-assertions/contractProviderAssertionsPrompt.js +2 -1
- package/build/prompts/enhance-assertions/integrationAssertionsPrompt.js +2 -1
- package/build/prompts/enhance-assertions/sharedAssertionRules.d.ts +1 -1
- package/build/prompts/enhance-assertions/sharedAssertionRules.js +57 -22
- package/build/prompts/enhance-assertions/uiAssertionsPrompt.js +17 -9
- package/build/prompts/test-recommendation/diffExecutionPlan.js +0 -2
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +11 -6
- package/build/prompts/testbot/testbot-prompts.js +19 -12
- package/build/recommendation/answers.d.ts +11 -7
- package/build/recommendation/answers.js +14 -10
- package/build/recommendation/pullRequestText.d.ts +18 -0
- package/build/recommendation/pullRequestText.js +31 -0
- package/build/recommendation/registerPlan.d.ts +9 -1
- package/build/recommendation/registerPlan.js +8 -1
- package/build/recommendation/runVerifiers.js +6 -0
- package/build/recommendation/types.d.ts +58 -0
- package/build/recommendation/verifierContracts.d.ts +108 -13
- package/build/recommendation/verifierContracts.js +147 -29
- package/build/recommendation/verifiers/coverage.d.ts +10 -0
- package/build/recommendation/verifiers/coverage.js +144 -22
- package/build/recommendation/verifiers/defects.d.ts +9 -0
- package/build/recommendation/verifiers/defects.js +117 -0
- package/build/recommendation/verifiers/deliveredMatchesPlan.d.ts +22 -0
- package/build/recommendation/verifiers/deliveredMatchesPlan.js +43 -0
- package/build/recommendation/verifiers/existingCoverage.js +53 -0
- package/build/recommendation/verifiers/expectedValueSourced.d.ts +14 -0
- package/build/recommendation/verifiers/expectedValueSourced.js +246 -0
- package/build/recommendation/verifiers/issueTraceability.d.ts +52 -0
- package/build/recommendation/verifiers/issueTraceability.js +197 -0
- package/build/recommendation/verifiers/requirementSourced.d.ts +2 -0
- package/build/recommendation/verifiers/requirementSourced.js +168 -0
- package/build/services/TestGenerationService.js +3 -1
- package/build/tools/code-refactor/codeReuseTool.js +1 -1
- package/build/tools/code-refactor/reuse-outcome.d.ts +1 -1
- package/build/tools/code-refactor/reuse-state.d.ts +85 -7
- package/build/tools/code-refactor/reuse-state.js +239 -34
- package/build/tools/code-refactor/utils-verify-gates.d.ts +5 -0
- package/build/tools/code-refactor/utils-verify-gates.js +103 -11
- package/build/tools/generate-tests/generateBatchScenarioRestTool.js +2 -1
- package/build/tools/submitReportTool.js +276 -39
- package/build/tools/test-management/actionsTool.js +5 -0
- package/build/tools/test-management/analyzeChangesTool.d.ts +53 -0
- package/build/tools/test-management/analyzeChangesTool.js +55 -2
- package/build/tools/test-management/registerTestPlanTool.d.ts +32 -17
- package/build/tools/test-management/registerTestPlanTool.js +105 -16
- package/build/types/ReuseOutcome.d.ts +73 -7
- package/build/types/TestAnalysis.d.ts +6 -0
- package/build/types/TestbotReport.d.ts +22 -4
- package/build/utils/AnalysisStateManager.d.ts +7 -1
- package/build/utils/AnalysisStateManager.js +5 -1
- package/build/utils/assertion-verify/api-shared-lints.js +118 -11
- package/build/utils/assertion-verify/format.js +2 -2
- package/build/utils/assertion-verify/helper-imports.d.ts +7 -0
- package/build/utils/assertion-verify/helper-imports.js +119 -27
- package/build/utils/assertion-verify/lint-types.d.ts +31 -2
- package/build/utils/assertion-verify/lint-types.js +66 -0
- package/build/utils/assertion-verify/metrics.d.ts +13 -0
- package/build/utils/assertion-verify/metrics.js +16 -0
- package/build/utils/assertion-verify/verify.d.ts +11 -6
- package/build/utils/assertion-verify/verify.js +56 -15
- package/build/utils/canonicalJson.d.ts +11 -0
- package/build/utils/canonicalJson.js +17 -0
- package/build/utils/utils-verify/action-key.d.ts +27 -0
- package/build/utils/utils-verify/action-key.js +292 -0
- package/build/utils/utils-verify/allow.d.ts +8 -1
- package/build/utils/utils-verify/allow.js +14 -1
- package/build/utils/utils-verify/call-sites.d.ts +76 -8
- package/build/utils/utils-verify/call-sites.js +256 -70
- package/build/utils/utils-verify/language-spec.d.ts +3 -2
- package/build/utils/utils-verify/parse.d.ts +22 -3
- package/build/utils/utils-verify/parse.js +123 -52
- package/build/utils/utils-verify/verify.d.ts +33 -3
- package/build/utils/utils-verify/verify.js +126 -12
- package/build/utils/workspaceAuth.d.ts +59 -19
- package/build/utils/workspaceAuth.js +228 -31
- package/package.json +1 -1
- package/plugin/prompts/generate-tests/execution-plan.md +3 -3
- package/plugin/prompts/generate-tests/generation.md +1 -0
- package/plugin/prompts/plan-tests.md +41 -15
- package/plugin/prompts/testbot-task1.md +3 -9
- package/build/prompts/testbot/planDeclarations.d.ts +0 -6
- package/build/prompts/testbot/planDeclarations.js +0 -9
- package/plugin/prompts/declaring-a-plan.md +0 -20
|
@@ -3,6 +3,8 @@ import { readFile, stat } from "fs/promises";
|
|
|
3
3
|
import { logger } from "../logger.js";
|
|
4
4
|
import { strippedSources } from "./strip-dispatch.js";
|
|
5
5
|
import { identifierRe } from "./lint-types.js";
|
|
6
|
+
import { blankCommentsAndStrings, parseUtilsFile, } from "../utils-verify/parse.js";
|
|
7
|
+
import { utilsSpecFor } from "../utils-verify/language-spec.js";
|
|
6
8
|
import { computeAssertionMetrics, detectAssertionLanguage, subjectOf, } from "./metrics.js";
|
|
7
9
|
/**
|
|
8
10
|
* Read-only scan of the spec's locally imported AND CALLED modules (the
|
|
@@ -14,12 +16,21 @@ import { computeAssertionMetrics, detectAssertionLanguage, subjectOf, } from "./
|
|
|
14
16
|
* - One-directional — helper assertions can only EXCUSE removals, never add
|
|
15
17
|
* to the spec's count or strength score: a shared utils file is imported by
|
|
16
18
|
* many specs, and crediting its assertions to each one would inflate every
|
|
17
|
-
* baseline comparison.
|
|
19
|
+
* baseline comparison. And each excuse is BOUNDED: a helper assertion
|
|
20
|
+
* excuses one removal per call site of that helper, so a single
|
|
21
|
+
* `verifyStatus()` call cannot clear three deleted status checks.
|
|
18
22
|
* - CALLED, not merely imported — an import statement plus a dead helper file
|
|
19
23
|
* would otherwise turn a correctly-blocked gutting into a PASS (same
|
|
20
24
|
* call-position requirement as the reuse gate's call-site matching). Only
|
|
21
|
-
* helpers whose imported bindings appear at a call position in the spec
|
|
22
|
-
*
|
|
25
|
+
* helpers whose imported bindings appear at a call position in the spec are
|
|
26
|
+
* read, and within such a file only the FUNCTIONS the spec calls through
|
|
27
|
+
* that import's bindings contribute subjects: an uncalled sibling in a
|
|
28
|
+
* shared utils file asserts nothing on this spec's behalf. A call through an
|
|
29
|
+
* import alias (`{ x as y }`, `{ x: y }`, python `x as y`) IS a call of the
|
|
30
|
+
* declared helper — the binding is resolved, not dropped. An assertion the
|
|
31
|
+
* utils parser cannot place in a function (module level, `export default
|
|
32
|
+
* function`, `exports.x = function`) falls back to the file-level rule: it
|
|
33
|
+
* counts when any binding of the import is called, bounded by those calls.
|
|
23
34
|
*
|
|
24
35
|
* Helpers are never modified.
|
|
25
36
|
*/
|
|
@@ -36,8 +47,13 @@ const JS_IMPORT_RES = [
|
|
|
36
47
|
/(?:const|let|var)\s+([^=]+?)\s*=\s*require\s*\(\s*['"](\.{1,2}\/[^'"]+)['"]\s*\)/g,
|
|
37
48
|
];
|
|
38
49
|
const PY_IMPORT_RES = [
|
|
39
|
-
// from .x import a, b as c | from x import a
|
|
40
|
-
|
|
50
|
+
// from .x import a, b as c | from x import (\n a,\n b as c,\n) |
|
|
51
|
+
// from x import a, \<newline> b. The bodies are separate alternatives: a body
|
|
52
|
+
// that stops at the newline bound nothing on a wrapped import (the `(` fell to
|
|
53
|
+
// the name filter), so the helper was never read and every moved assertion
|
|
54
|
+
// became a false removal. A backslash continuation is consumed with its
|
|
55
|
+
// newline for the same reason. Runs on comment-stripped source.
|
|
56
|
+
/^[ \t]*from\s+(\.*[\w.]+)\s+import\s+(?:\(([^)]*)\)|((?:[^\n#\\]|\\\r?\n)+))/gm,
|
|
41
57
|
// import x (module itself becomes the callable namespace)
|
|
42
58
|
/^[ \t]*import\s+([\w.]+)(?:\s+as\s+([\w]+))?/gm,
|
|
43
59
|
];
|
|
@@ -58,18 +74,20 @@ function pyCandidates(specDir, specifier) {
|
|
|
58
74
|
const moduleName = specifier.replace(/^\.+/, "").split(".")[0];
|
|
59
75
|
return moduleName ? [path.resolve(specDir, `${moduleName}.py`)] : [];
|
|
60
76
|
}
|
|
61
|
-
/**
|
|
62
|
-
function
|
|
63
|
-
const
|
|
77
|
+
/** Bindings of a JS import/require clause ("d, { a, b as c }" / "{ a: b }"). */
|
|
78
|
+
function jsBindings(clause) {
|
|
79
|
+
const bindings = [];
|
|
64
80
|
const star = /\*\s*as\s+([\w$]+)/.exec(clause);
|
|
65
81
|
if (star)
|
|
66
|
-
|
|
82
|
+
bindings.push({ local: star[1] });
|
|
67
83
|
const braces = /\{([^}]*)\}/.exec(clause);
|
|
68
84
|
if (braces) {
|
|
69
85
|
for (const part of braces[1].split(",")) {
|
|
70
|
-
const
|
|
71
|
-
|
|
72
|
-
|
|
86
|
+
const [declared, local = declared] = part
|
|
87
|
+
.split(/\s+as\s+|\s*:\s*/)
|
|
88
|
+
.map((n) => n.trim());
|
|
89
|
+
if (/^[\w$]+$/.test(declared) && /^[\w$]+$/.test(local))
|
|
90
|
+
bindings.push({ declared, local });
|
|
73
91
|
}
|
|
74
92
|
}
|
|
75
93
|
const bare = clause
|
|
@@ -78,8 +96,8 @@ function jsBoundNames(clause) {
|
|
|
78
96
|
.replace(/,/g, " ")
|
|
79
97
|
.trim();
|
|
80
98
|
if (/^[\w$]+$/.test(bare))
|
|
81
|
-
|
|
82
|
-
return
|
|
99
|
+
bindings.push({ local: bare });
|
|
100
|
+
return bindings;
|
|
83
101
|
}
|
|
84
102
|
/** Relative/local imports with the names they bind. */
|
|
85
103
|
function localImports(content, language) {
|
|
@@ -90,15 +108,20 @@ function localImports(content, language) {
|
|
|
90
108
|
fromRe.lastIndex = 0;
|
|
91
109
|
let m;
|
|
92
110
|
while ((m = fromRe.exec(commentless)) !== null) {
|
|
93
|
-
const
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
111
|
+
const bindings = [];
|
|
112
|
+
// A continuation's backslash-newline is whitespace between names.
|
|
113
|
+
for (const part of (m[2] ?? m[3]).replace(/\\\r?\n/g, " ").split(",")) {
|
|
114
|
+
const [declared, local = declared] = part
|
|
115
|
+
.split(/\s+as\s+/)
|
|
116
|
+
.map((n) => n.trim());
|
|
117
|
+
if (/^\w+$/.test(declared) && /^\w+$/.test(local))
|
|
118
|
+
bindings.push({ declared, local });
|
|
119
|
+
}
|
|
120
|
+
imports.push({ specifier: m[1], bindings });
|
|
98
121
|
}
|
|
99
122
|
importRe.lastIndex = 0;
|
|
100
123
|
while ((m = importRe.exec(commentless)) !== null) {
|
|
101
|
-
imports.push({ specifier: m[1],
|
|
124
|
+
imports.push({ specifier: m[1], bindings: [{ local: m[2] ?? m[1] }] });
|
|
102
125
|
}
|
|
103
126
|
return imports;
|
|
104
127
|
}
|
|
@@ -106,7 +129,7 @@ function localImports(content, language) {
|
|
|
106
129
|
re.lastIndex = 0;
|
|
107
130
|
let m;
|
|
108
131
|
while ((m = re.exec(commentless)) !== null) {
|
|
109
|
-
imports.push({ specifier: m[2],
|
|
132
|
+
imports.push({ specifier: m[2], bindings: jsBindings(m[1]) });
|
|
110
133
|
}
|
|
111
134
|
}
|
|
112
135
|
return imports;
|
|
@@ -119,13 +142,60 @@ function anyNameCalled(strippedSpec, names) {
|
|
|
119
142
|
// the suffix requires a CALL position — `name(...)` or `name.member(...)`.
|
|
120
143
|
return names.some((name) => new RegExp(`(?<!\\.)${identifierRe(name).source}\\s*(?:\\(|\\.\\s*[\\w$]+\\s*\\()`).test(strippedSpec));
|
|
121
144
|
}
|
|
145
|
+
/** Number of call sites of the helper declared as `fnName` through this
|
|
146
|
+
* import's bindings: bare calls of the LOCAL name bound to it (the declared
|
|
147
|
+
* name itself, or its alias) plus member calls on a namespace binding
|
|
148
|
+
* (`ns.fnName(`). A namespace-style binding may itself BE a helper — a
|
|
149
|
+
* default import or whole-module require of `export default checkStatus` /
|
|
150
|
+
* `module.exports = checkStatus` — so its bare calls count as calls of
|
|
151
|
+
* `defaultName`. With `fnName` undefined, counts every call through any of
|
|
152
|
+
* the bindings (the file-level fallback). The `(?<!\.)` guards keep a
|
|
153
|
+
* same-named member of some other object (`other.ns.fn(`) from counting.
|
|
154
|
+
* Comment/string-stripped source. */
|
|
155
|
+
function callSitesThroughImport(strippedSpec, bindings, fnName, defaultName) {
|
|
156
|
+
const fn = fnName === undefined ? "[\\w$]+" : identifierRe(fnName).source;
|
|
157
|
+
let count = 0;
|
|
158
|
+
for (const b of bindings) {
|
|
159
|
+
const local = identifierRe(b.local).source;
|
|
160
|
+
const bare = new RegExp(`(?<!\\.)${local}\\s*\\(`, "g");
|
|
161
|
+
if (b.declared === undefined) {
|
|
162
|
+
count +=
|
|
163
|
+
strippedSpec.match(new RegExp(`(?<!\\.)${local}\\s*\\.\\s*${fn}\\s*\\(`, "g"))?.length ?? 0;
|
|
164
|
+
if (fnName === undefined || fnName === defaultName)
|
|
165
|
+
count += strippedSpec.match(bare)?.length ?? 0;
|
|
166
|
+
}
|
|
167
|
+
else if (fnName === undefined || b.declared === fnName) {
|
|
168
|
+
count += strippedSpec.match(bare)?.length ?? 0;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
return count;
|
|
172
|
+
}
|
|
173
|
+
/** The name a helper module exports as its default / whole export
|
|
174
|
+
* (`export default checkStatus`, `module.exports = checkStatus`), on
|
|
175
|
+
* comment/string-stripped source; undefined when there is none or it is not
|
|
176
|
+
* a bare identifier. */
|
|
177
|
+
function defaultExportName(strippedHelper) {
|
|
178
|
+
const m = /(?:^|\n)\s*export\s+default\s+([A-Za-z_$][\w$]*)\s*;?\s*(?:\n|$)/.exec(strippedHelper) ??
|
|
179
|
+
/(?:^|\n)\s*module\.exports\s*=\s*([A-Za-z_$][\w$]*)\s*;?\s*(?:\n|$)/.exec(strippedHelper);
|
|
180
|
+
return m?.[1];
|
|
181
|
+
}
|
|
122
182
|
/**
|
|
123
183
|
* Assertion SUBJECTS found in the spec's locally imported, actually-called
|
|
124
184
|
* helper files. Strict-by-default failure direction: an unreadable/oversized/
|
|
125
185
|
* uncalled helper is skipped, so its subjects don't excuse anything.
|
|
126
186
|
*/
|
|
127
187
|
export async function importedHelperSubjects(testFile, content) {
|
|
128
|
-
|
|
188
|
+
return new Set((await importedHelperSubjectCapacities(testFile, content)).keys());
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* The same subjects with their CAPACITY — how many removals each may excuse:
|
|
192
|
+
* one per call site of the helper function that asserts it (an assertion the
|
|
193
|
+
* parser cannot place in a function gets one per call through the import).
|
|
194
|
+
* The removal gate consumes this budget per removed baseline subject.
|
|
195
|
+
*/
|
|
196
|
+
export async function importedHelperSubjectCapacities(testFile, content) {
|
|
197
|
+
const subjects = new Map();
|
|
198
|
+
const add = (subject, n) => subjects.set(subject, (subjects.get(subject) ?? 0) + n);
|
|
129
199
|
const language = detectAssertionLanguage(testFile);
|
|
130
200
|
// Not a no-op: an empty subject set means NO removal gets excused by a
|
|
131
201
|
// helper — the strict direction. Unknown languages have no import scanner;
|
|
@@ -136,7 +206,8 @@ export async function importedHelperSubjects(testFile, content) {
|
|
|
136
206
|
const strippedSpec = strippedSources(content, language).stripped;
|
|
137
207
|
const specDir = path.dirname(testFile);
|
|
138
208
|
let scanned = 0;
|
|
139
|
-
for (const { specifier,
|
|
209
|
+
for (const { specifier, bindings } of localImports(content, language)) {
|
|
210
|
+
const localNames = bindings.map((b) => b.local);
|
|
140
211
|
if (scanned >= MAX_HELPER_FILES) {
|
|
141
212
|
logger.warning("Assertion verification scanned the maximum number of imported helper files — remaining imports ignored", { testFile, max: MAX_HELPER_FILES });
|
|
142
213
|
break;
|
|
@@ -160,11 +231,32 @@ export async function importedHelperSubjects(testFile, content) {
|
|
|
160
231
|
if (helperLanguage === undefined)
|
|
161
232
|
continue;
|
|
162
233
|
const helperContent = await readFile(candidate, "utf8");
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
234
|
+
// Attribute each assertion to the helper function whose body holds it
|
|
235
|
+
// (utils-verify's line-range parser, on a string-blanked copy so a
|
|
236
|
+
// brace inside a literal cannot end a body early; line count is
|
|
237
|
+
// preserved). A function's sites count once per call of that function
|
|
238
|
+
// THROUGH THIS IMPORT — a bare call of a binding it declares, or a
|
|
239
|
+
// member call on one of its bindings (`ns.fn(`); a same-named function
|
|
240
|
+
// called via another module's binding does not count. A site in no
|
|
241
|
+
// parsed function falls back to the import's total call count.
|
|
242
|
+
const utilsSpec = utilsSpecFor(helperLanguage);
|
|
243
|
+
const helpers = utilsSpec
|
|
244
|
+
? parseUtilsFile(blankCommentsAndStrings(helperContent, utilsSpec), utilsSpec)
|
|
245
|
+
: [];
|
|
246
|
+
const defaultName = defaultExportName(strippedSources(helperContent, helperLanguage).stripped);
|
|
247
|
+
const callsOf = new Map(helpers.map((h) => [
|
|
248
|
+
h,
|
|
249
|
+
callSitesThroughImport(strippedSpec, bindings, h.name, defaultName),
|
|
250
|
+
]));
|
|
251
|
+
const fallbackCalls = callSitesThroughImport(strippedSpec, bindings);
|
|
252
|
+
for (const site of computeAssertionMetrics(helperContent, helperLanguage).sites) {
|
|
253
|
+
const owner = helpers.find((h) => site.line >= h.line && site.line <= h.endLine);
|
|
254
|
+
const calls = owner === undefined ? fallbackCalls : (callsOf.get(owner) ?? 0);
|
|
255
|
+
if (calls === 0)
|
|
256
|
+
continue;
|
|
257
|
+
const subject = subjectOf(site.fingerprint);
|
|
166
258
|
if (subject !== undefined)
|
|
167
|
-
|
|
259
|
+
add(subject, calls);
|
|
168
260
|
}
|
|
169
261
|
scanned++;
|
|
170
262
|
break;
|
|
@@ -14,12 +14,41 @@ export interface LintFinding {
|
|
|
14
14
|
remediation: string;
|
|
15
15
|
}
|
|
16
16
|
export interface LintOptions {
|
|
17
|
-
/** Maintenance scope:
|
|
17
|
+
/** Maintenance scope: the lines of every test function the agent TOUCHED —
|
|
18
|
+
* each function containing an assertion added since the baseline, whole.
|
|
18
19
|
* When set, site-scoped lints only report findings on these lines and
|
|
19
20
|
* structural (whole-file) lints — including the import lint — are skipped:
|
|
20
|
-
* they would flag pre-existing customer code the agent was told not to touch.
|
|
21
|
+
* they would flag pre-existing customer code the agent was told not to touch.
|
|
22
|
+
*
|
|
23
|
+
* Scoped to the touched FUNCTION, not to the added lines alone, because
|
|
24
|
+
* MAINTENANCE_SCOPE_NOTE holds a touched test to the same standard as a new
|
|
25
|
+
* one. Under line-scoping, the pre-existing `is not None` the note tells the
|
|
26
|
+
* agent to upgrade sat outside the scope of every lint, so nothing could
|
|
27
|
+
* ever report it (SKYR-4310). Functions the agent left alone stay out —
|
|
28
|
+
* that is what the scope is still for. */
|
|
21
29
|
scopeLines?: Set<number>;
|
|
22
30
|
}
|
|
31
|
+
/** Starts of test bodies: JS/TS `it(` / `test(` (with `.only`/`.skip`/`.each`
|
|
32
|
+
* modifiers) and Python `def test_`. Two assertions belong to the same test
|
|
33
|
+
* only when no such start lies between them — a `const r = …` re-bound in the
|
|
34
|
+
* next `it` block is a different response, not the same one. */
|
|
35
|
+
export declare const TEST_START_RE: RegExp;
|
|
36
|
+
/** Offsets of every test-body start in a comment-stripped source, ascending. */
|
|
37
|
+
export declare function testStartOffsets(commentless: string): number[];
|
|
38
|
+
/** Index of the test block an offset falls in — 0 for anything before the
|
|
39
|
+
* first start (file preamble), otherwise the 1-based ordinal of the enclosing
|
|
40
|
+
* start. Two offsets share a block exactly when this returns the same value. */
|
|
41
|
+
export declare function testBlockIndex(starts: number[], offset: number): number;
|
|
42
|
+
/** Widen a set of individual lines to every line of each test function one of
|
|
43
|
+
* them falls in.
|
|
44
|
+
*
|
|
45
|
+
* Two cases deliberately do NOT widen, and both keep their original line:
|
|
46
|
+
* a file with no recognizable test start at all (widening would put the whole
|
|
47
|
+
* file in scope — the pre-existing customer code the scope exists to keep
|
|
48
|
+
* out), and a line before the first test start (module preamble or shared
|
|
49
|
+
* setup, which is not a test function and is not what
|
|
50
|
+
* `MAINTENANCE_SCOPE_NOTE` holds to the new-test standard). */
|
|
51
|
+
export declare function expandToTestBlocks(commentless: string, lines: Set<number>): Set<number>;
|
|
23
52
|
export declare function lineOfOffset(src: string, offset: number): number;
|
|
24
53
|
/** `\b` breaks on `$`-names, so identifier boundaries use lookarounds. Single
|
|
25
54
|
* home for the pattern — it is subtle enough that hand-inlined copies drifted
|
|
@@ -1,4 +1,70 @@
|
|
|
1
1
|
import { escapeRegExp } from "../regex.js";
|
|
2
|
+
/** Starts of test bodies: JS/TS `it(` / `test(` (with `.only`/`.skip`/`.each`
|
|
3
|
+
* modifiers) and Python `def test_`. Two assertions belong to the same test
|
|
4
|
+
* only when no such start lies between them — a `const r = …` re-bound in the
|
|
5
|
+
* next `it` block is a different response, not the same one. */
|
|
6
|
+
export const TEST_START_RE = /(?<![\w$.])(?:it|test)(?:\s*\.\s*\w+)*\s*\(|^[ \t]*(?:async\s+)?def\s+test_/gm;
|
|
7
|
+
/** Offsets of every test-body start in a comment-stripped source, ascending. */
|
|
8
|
+
export function testStartOffsets(commentless) {
|
|
9
|
+
const starts = [];
|
|
10
|
+
TEST_START_RE.lastIndex = 0;
|
|
11
|
+
let m;
|
|
12
|
+
while ((m = TEST_START_RE.exec(commentless)) !== null)
|
|
13
|
+
starts.push(m.index);
|
|
14
|
+
return starts;
|
|
15
|
+
}
|
|
16
|
+
/** Index of the test block an offset falls in — 0 for anything before the
|
|
17
|
+
* first start (file preamble), otherwise the 1-based ordinal of the enclosing
|
|
18
|
+
* start. Two offsets share a block exactly when this returns the same value. */
|
|
19
|
+
export function testBlockIndex(starts, offset) {
|
|
20
|
+
let lo = 0;
|
|
21
|
+
let hi = starts.length;
|
|
22
|
+
while (lo < hi) {
|
|
23
|
+
const mid = (lo + hi) >> 1;
|
|
24
|
+
if (starts[mid] <= offset)
|
|
25
|
+
lo = mid + 1;
|
|
26
|
+
else
|
|
27
|
+
hi = mid;
|
|
28
|
+
}
|
|
29
|
+
return lo;
|
|
30
|
+
}
|
|
31
|
+
/** Widen a set of individual lines to every line of each test function one of
|
|
32
|
+
* them falls in.
|
|
33
|
+
*
|
|
34
|
+
* Two cases deliberately do NOT widen, and both keep their original line:
|
|
35
|
+
* a file with no recognizable test start at all (widening would put the whole
|
|
36
|
+
* file in scope — the pre-existing customer code the scope exists to keep
|
|
37
|
+
* out), and a line before the first test start (module preamble or shared
|
|
38
|
+
* setup, which is not a test function and is not what
|
|
39
|
+
* `MAINTENANCE_SCOPE_NOTE` holds to the new-test standard). */
|
|
40
|
+
export function expandToTestBlocks(commentless, lines) {
|
|
41
|
+
if (lines.size === 0)
|
|
42
|
+
return lines;
|
|
43
|
+
const startLines = testStartOffsets(commentless).map((o) => lineOfOffset(commentless, o));
|
|
44
|
+
if (startLines.length === 0)
|
|
45
|
+
return lines;
|
|
46
|
+
const totalLines = commentless.split("\n").length;
|
|
47
|
+
const expanded = new Set();
|
|
48
|
+
const seen = new Set();
|
|
49
|
+
for (const line of lines) {
|
|
50
|
+
// startLines is ascending, so the enclosing function is the last start at
|
|
51
|
+
// or before this line; 0 means the line precedes every test.
|
|
52
|
+
let block = 0;
|
|
53
|
+
while (block < startLines.length && startLines[block] <= line)
|
|
54
|
+
block++;
|
|
55
|
+
if (block === 0) {
|
|
56
|
+
expanded.add(line);
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
if (seen.has(block))
|
|
60
|
+
continue;
|
|
61
|
+
seen.add(block);
|
|
62
|
+
const last = block < startLines.length ? startLines[block] - 1 : totalLines;
|
|
63
|
+
for (let l = startLines[block - 1]; l <= last; l++)
|
|
64
|
+
expanded.add(l);
|
|
65
|
+
}
|
|
66
|
+
return expanded;
|
|
67
|
+
}
|
|
2
68
|
// One-entry memo of a source's newline offsets: every scanner asks for line
|
|
3
69
|
// numbers of many offsets in the SAME string (assertion sites, lint findings),
|
|
4
70
|
// so counting newlines from index 0 per call was O(n × sites). The memo is
|
|
@@ -27,4 +27,17 @@ export declare function detectAssertionLanguage(testFile: string): AssertionLang
|
|
|
27
27
|
* truncated fingerprint) — callers treat that conservatively.
|
|
28
28
|
*/
|
|
29
29
|
export declare function subjectOf(fingerprint: string): string | undefined;
|
|
30
|
+
/**
|
|
31
|
+
* The comparison key for matching a baseline subject against an imported
|
|
32
|
+
* helper's subject: the property path with the leading receiver identifier
|
|
33
|
+
* removed (`recordGetResponse.statusCode` and `response.statusCode` both
|
|
34
|
+
* become `.statusCode`; so does `response?.statusCode`, the optional-chaining
|
|
35
|
+
* spelling). A helper renames the receiver by construction — the spec names
|
|
36
|
+
* the response after the call it came from, the helper names it by its own
|
|
37
|
+
* parameter — so exact subject equality never matched a moved assertion. A
|
|
38
|
+
* bare identifier, or a subject that does not start with one, is returned
|
|
39
|
+
* unchanged. Used ONLY for the helper excuse in the removal gate; the in-spec
|
|
40
|
+
* replaced/removed correlation keeps exact subjects.
|
|
41
|
+
*/
|
|
42
|
+
export declare function helperSubjectKey(subject: string): string;
|
|
30
43
|
export declare function computeAssertionMetrics(content: string, language: AssertionLanguage): AssertionMetrics;
|
|
@@ -361,6 +361,22 @@ export function subjectOf(fingerprint) {
|
|
|
361
361
|
}
|
|
362
362
|
return undefined;
|
|
363
363
|
}
|
|
364
|
+
/**
|
|
365
|
+
* The comparison key for matching a baseline subject against an imported
|
|
366
|
+
* helper's subject: the property path with the leading receiver identifier
|
|
367
|
+
* removed (`recordGetResponse.statusCode` and `response.statusCode` both
|
|
368
|
+
* become `.statusCode`; so does `response?.statusCode`, the optional-chaining
|
|
369
|
+
* spelling). A helper renames the receiver by construction — the spec names
|
|
370
|
+
* the response after the call it came from, the helper names it by its own
|
|
371
|
+
* parameter — so exact subject equality never matched a moved assertion. A
|
|
372
|
+
* bare identifier, or a subject that does not start with one, is returned
|
|
373
|
+
* unchanged. Used ONLY for the helper excuse in the removal gate; the in-spec
|
|
374
|
+
* replaced/removed correlation keeps exact subjects.
|
|
375
|
+
*/
|
|
376
|
+
export function helperSubjectKey(subject) {
|
|
377
|
+
// `?.` before `[` is consumed whole so `res?.["x"]` keys like `res["x"]`.
|
|
378
|
+
return subject.replace(/^[A-Za-z_$][\w$]*(?:\?\.(?=\[)|\?)?(?=[.[])/, "");
|
|
379
|
+
}
|
|
364
380
|
export function computeAssertionMetrics(content, language) {
|
|
365
381
|
let sites;
|
|
366
382
|
const { stripped } = strippedSources(content, language);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type AssertionMetrics } from "./metrics.js";
|
|
2
|
-
import type
|
|
2
|
+
import { type LintFinding } from "./lint-types.js";
|
|
3
3
|
export type AssertionEnhanceType = "generation" | "maintenance";
|
|
4
4
|
/** Snapshot taken when the enhancement instructions were handed out. */
|
|
5
5
|
export interface AssertionBaseline {
|
|
@@ -36,11 +36,16 @@ export interface AssertionVerifyResult {
|
|
|
36
36
|
/** Current − baseline strength score; undefined without a baseline. */
|
|
37
37
|
strengthDelta?: number;
|
|
38
38
|
strengthGateFailed: boolean;
|
|
39
|
-
/** Assertions were added, but
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
|
|
39
|
+
/** Assertions were added, but they are PREDOMINANTLY existence/visibility-tier
|
|
40
|
+
* (mean weight below `MIN_MEAN_ADDED_WEIGHT`) — the computed-values rubric
|
|
41
|
+
* wants exact matchers. Runs in generation AND maintenance (it requires
|
|
42
|
+
* additions to exist, so value-only maintenance fixes cannot trip it).
|
|
43
|
+
* Marker-clearable, like the strength gate. */
|
|
44
|
+
weakAdditionsDominant: boolean;
|
|
45
|
+
/** Mean weight of the assertions added since the baseline; undefined when
|
|
46
|
+
* there is no baseline diff or nothing was added. Reported so the agent can
|
|
47
|
+
* see how far a failing file is from the bar. */
|
|
48
|
+
meanAddedWeight?: number;
|
|
44
49
|
hardFindings: LintFinding[];
|
|
45
50
|
warnings: LintFinding[];
|
|
46
51
|
markerReason?: string;
|
|
@@ -1,11 +1,20 @@
|
|
|
1
1
|
import { createHash } from "crypto";
|
|
2
2
|
import { readFile } from "fs/promises";
|
|
3
|
-
import { computeAssertionMetrics, detectAssertionLanguage, subjectOf, } from "./metrics.js";
|
|
3
|
+
import { computeAssertionMetrics, detectAssertionLanguage, helperSubjectKey, subjectOf, } from "./metrics.js";
|
|
4
|
+
import { expandToTestBlocks } from "./lint-types.js";
|
|
5
|
+
import { strippedSources } from "./strip-dispatch.js";
|
|
4
6
|
import { lintUiSpec } from "./ui-lints.js";
|
|
5
7
|
import { lintIntegrationTest } from "./integration-lints.js";
|
|
6
8
|
import { lintContractTest } from "./contract-lints.js";
|
|
7
9
|
import { findAssertionsCompleteMarker, findUnparsableAssertionsComplete, markerCommentToken, } from "./marker.js";
|
|
8
|
-
import {
|
|
10
|
+
import { importedHelperSubjectCapacities } from "./helper-imports.js";
|
|
11
|
+
/** The mean weight the assertions added since the baseline must reach.
|
|
12
|
+
* Weights are 3 exact / 2 partial-or-format / 1 existence-tier, so 2 means
|
|
13
|
+
* "on average at least a format check". A lean enhancement of exact values
|
|
14
|
+
* plus a couple of format checks sits near 2.7; a file padded with
|
|
15
|
+
* existence-tier assertions falls below 2 however many exact ones it also
|
|
16
|
+
* carries. */
|
|
17
|
+
const MIN_MEAN_ADDED_WEIGHT = 2;
|
|
9
18
|
export function sha256Of(content) {
|
|
10
19
|
return createHash("sha256").update(content).digest("hex");
|
|
11
20
|
}
|
|
@@ -45,7 +54,7 @@ export async function verifyAssertionEnhancement(params) {
|
|
|
45
54
|
baselinePresent: baseline !== undefined,
|
|
46
55
|
hashUnchanged,
|
|
47
56
|
strengthGateFailed: false,
|
|
48
|
-
|
|
57
|
+
weakAdditionsDominant: false,
|
|
49
58
|
hardFindings: [],
|
|
50
59
|
warnings: [],
|
|
51
60
|
};
|
|
@@ -88,11 +97,28 @@ export async function verifyAssertionEnhancement(params) {
|
|
|
88
97
|
});
|
|
89
98
|
// Only pay the helper-file scan when something actually looks removed.
|
|
90
99
|
if (removed.length > 0) {
|
|
91
|
-
const helperSubjects = await
|
|
100
|
+
const helperSubjects = await importedHelperSubjectCapacities(testFile, content);
|
|
92
101
|
if (helperSubjects.size > 0) {
|
|
102
|
+
// Both sides go through the same key so a helper's renamed receiver
|
|
103
|
+
// (`response.statusCode` for the spec's `recordGetResponse.statusCode`)
|
|
104
|
+
// still matches — the exact-subject check above stays exact. Each key
|
|
105
|
+
// carries a budget (one per call site of the helper asserting it) so
|
|
106
|
+
// one helper call cannot excuse every removed subject on its path.
|
|
107
|
+
const budget = new Map();
|
|
108
|
+
for (const [subject, n] of helperSubjects) {
|
|
109
|
+
const key = helperSubjectKey(subject);
|
|
110
|
+
budget.set(key, (budget.get(key) ?? 0) + n);
|
|
111
|
+
}
|
|
93
112
|
const stillRemoved = removed.filter((fp) => {
|
|
94
113
|
const subject = subjectOf(fp);
|
|
95
|
-
|
|
114
|
+
if (subject === undefined)
|
|
115
|
+
return true;
|
|
116
|
+
const key = helperSubjectKey(subject);
|
|
117
|
+
const left = budget.get(key) ?? 0;
|
|
118
|
+
if (left === 0)
|
|
119
|
+
return true;
|
|
120
|
+
budget.set(key, left - 1);
|
|
121
|
+
return false;
|
|
96
122
|
});
|
|
97
123
|
movedToHelperCount = removed.length - stillRemoved.length;
|
|
98
124
|
removed = stillRemoved;
|
|
@@ -110,10 +136,13 @@ export async function verifyAssertionEnhancement(params) {
|
|
|
110
136
|
const added = baseline?.fingerprints !== undefined
|
|
111
137
|
? addedSites(metrics, baseline.fingerprints)
|
|
112
138
|
: undefined;
|
|
113
|
-
// Maintenance scopes site lints to
|
|
114
|
-
//
|
|
139
|
+
// Maintenance scopes site lints to the test functions the agent TOUCHED —
|
|
140
|
+
// widened from the added lines to the whole enclosing function, because
|
|
141
|
+
// MAINTENANCE_SCOPE_NOTE holds a touched test to the same standard as a new
|
|
142
|
+
// one (SKYR-4310). The baseline fingerprints are what make that diff
|
|
143
|
+
// computable; functions with no added assertion stay out of scope.
|
|
115
144
|
const scopeLines = enhanceType === "maintenance" && added !== undefined
|
|
116
|
-
? new Set(added.map((s) => s.line))
|
|
145
|
+
? expandToTestBlocks(strippedSources(content, language).commentless, new Set(added.map((s) => s.line)))
|
|
117
146
|
: undefined;
|
|
118
147
|
const lintOpts = scopeLines !== undefined ? { scopeLines } : undefined;
|
|
119
148
|
// Per-test-type lints; Java is never linted or strength-gated — count+hash
|
|
@@ -164,16 +193,27 @@ export async function verifyAssertionEnhancement(params) {
|
|
|
164
193
|
strengthDelta !== undefined &&
|
|
165
194
|
strengthDelta <= 0 &&
|
|
166
195
|
!markerAccepted;
|
|
167
|
-
// Computed-values rubric: additions must not be
|
|
196
|
+
// Computed-values rubric: additions must not be predominantly existence/
|
|
168
197
|
// visibility-tier. A replacement's new side counts as an added site, so a
|
|
169
198
|
// weak→strong upgrade satisfies this. Marker-clearable (strength family).
|
|
170
199
|
// Unlike the strength-increase gate this runs in MAINTENANCE too: it is
|
|
171
200
|
// conditional on additions existing, so a value-only drift fix (no added
|
|
172
201
|
// sites) can never trip it — only genuinely weak new assertions do.
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
202
|
+
//
|
|
203
|
+
// The bar is the MEAN added weight, not "every addition is weak". The
|
|
204
|
+
// all-weak form let one exact assertion license unlimited padding: 5 exact
|
|
205
|
+
// + 40 `not.toBeNull()` passed, and scored HIGHER than the 5 alone, because
|
|
206
|
+
// strengthScore is a sum. That is a quota, and it points the opposite way
|
|
207
|
+
// from the rules the agent is handed, which ask it to assert the behaviour
|
|
208
|
+
// under test and leave the rest of the body alone (SKYR-4310). Averaging
|
|
209
|
+
// removes the payoff for padding without capping assertion count — a budget
|
|
210
|
+
// is not knowable from the file.
|
|
211
|
+
const meanAddedWeight = added !== undefined && added.length > 0
|
|
212
|
+
? added.reduce((sum, s) => sum + s.weight, 0) / added.length
|
|
213
|
+
: undefined;
|
|
214
|
+
const weakAdditionsDominant = language !== "java" &&
|
|
215
|
+
meanAddedWeight !== undefined &&
|
|
216
|
+
meanAddedWeight < MIN_MEAN_ADDED_WEIGHT &&
|
|
177
217
|
!markerAccepted;
|
|
178
218
|
// The hash gate ("byte-identical → nothing applied") is generation-only: in
|
|
179
219
|
// maintenance the file edits precede the enhance call, and "no assertion
|
|
@@ -194,7 +234,7 @@ export async function verifyAssertionEnhancement(params) {
|
|
|
194
234
|
!removalGateFailed &&
|
|
195
235
|
hardFindings.length === 0 &&
|
|
196
236
|
!strengthGateFailed &&
|
|
197
|
-
!
|
|
237
|
+
!weakAdditionsDominant,
|
|
198
238
|
maintenanceNoAssertionChanges,
|
|
199
239
|
enhanceType,
|
|
200
240
|
baselinePresent: baseline !== undefined,
|
|
@@ -206,7 +246,8 @@ export async function verifyAssertionEnhancement(params) {
|
|
|
206
246
|
baselineFilePath: baseline?.baselineFilePath,
|
|
207
247
|
strengthDelta,
|
|
208
248
|
strengthGateFailed,
|
|
209
|
-
|
|
249
|
+
weakAdditionsDominant,
|
|
250
|
+
meanAddedWeight,
|
|
210
251
|
hardFindings,
|
|
211
252
|
warnings: findings.filter((f) => f.severity === "warn"),
|
|
212
253
|
markerReason: markerAccepted ? marker?.reason : undefined,
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/** Whether a value is a plain object — not null, not an array — so its keys can
|
|
2
|
+
* be read and sorted. */
|
|
3
|
+
export declare function isPlainObject(value: unknown): value is Record<string, unknown>;
|
|
4
|
+
/** One value as text with object keys sorted, so the same value written twice by
|
|
5
|
+
* hand is not a difference merely because its keys are ordered differently. List
|
|
6
|
+
* order still counts: two orders are two different lists.
|
|
7
|
+
*
|
|
8
|
+
* UNDEFINED for a value `JSON.stringify` renders as nothing — `undefined` itself,
|
|
9
|
+
* a function, a symbol. Callers that compare two results must know a pair of
|
|
10
|
+
* those reads as equal. */
|
|
11
|
+
export declare function canonicalJson(value: unknown): string | undefined;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/** Whether a value is a plain object — not null, not an array — so its keys can
|
|
2
|
+
* be read and sorted. */
|
|
3
|
+
export function isPlainObject(value) {
|
|
4
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
5
|
+
}
|
|
6
|
+
/** One value as text with object keys sorted, so the same value written twice by
|
|
7
|
+
* hand is not a difference merely because its keys are ordered differently. List
|
|
8
|
+
* order still counts: two orders are two different lists.
|
|
9
|
+
*
|
|
10
|
+
* UNDEFINED for a value `JSON.stringify` renders as nothing — `undefined` itself,
|
|
11
|
+
* a function, a symbol. Callers that compare two results must know a pair of
|
|
12
|
+
* those reads as equal. */
|
|
13
|
+
export function canonicalJson(value) {
|
|
14
|
+
return JSON.stringify(value, (_key, entry) => isPlainObject(entry)
|
|
15
|
+
? Object.fromEntries(Object.keys(entry).sort().map((key) => [key, entry[key]]))
|
|
16
|
+
: entry);
|
|
17
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { UtilsLanguage } from "./language-spec.js";
|
|
2
|
+
/**
|
|
3
|
+
* The comparison key of a browser helper: the analogue of `method + path` for a
|
|
4
|
+
* helper that sends no request. One step per action, in body order: the verb plus
|
|
5
|
+
* the LOCATOR CHAIN the action is called on, as written — the prompt's own
|
|
6
|
+
* definition of a browser duplicate is "same primitives in the same order,
|
|
7
|
+
* selectors identical", and that is a textual identity, not a set of tokens. So
|
|
8
|
+
* `getByRole("heading", { level: 1 })` and `{ level: 2 }` differ, `getByTestId("x")`
|
|
9
|
+
* and `getByText("x")` differ, `a.getByText("b")` and `b.getByText("a")` differ, and
|
|
10
|
+
* `.first()` / `.nth(2)` count. Formatting does not: whitespace and braces outside
|
|
11
|
+
* strings are dropped and quotes normalised, so a Prettier-wrapped chain and its
|
|
12
|
+
* one-line twin key the same. Argument VALUES are not part of the key, except a
|
|
13
|
+
* `goto` target and a `dragTo` destination, which are the identity of those steps.
|
|
14
|
+
* The legacy page-level form (`page.click("#save")`) is keyed on its selector
|
|
15
|
+
* argument.
|
|
16
|
+
*
|
|
17
|
+
* `undefined` when the helper performs no action, or when any action's locator is
|
|
18
|
+
* not built from plain string literals (a template literal, a variable, a regex, an
|
|
19
|
+
* f-string): a sequence with an unreadable step is not comparable. A `goto` target
|
|
20
|
+
* may be a template literal, keyed on its text with the interpolation kept: the
|
|
21
|
+
* generated `${baseUrl}/…` prefix names one module constant.
|
|
22
|
+
*
|
|
23
|
+
* `body` is the helper source with comments blanked and strings kept. Playwright for
|
|
24
|
+
* Python is read through the JavaScript spelling (see toPlaywrightJs), so the same
|
|
25
|
+
* helper in either language gets the same key.
|
|
26
|
+
*/
|
|
27
|
+
export declare function actionKeyOf(body: string, language?: UtilsLanguage): string | undefined;
|