@skyramp/mcp 0.4.1-rc.1 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/prompts/enhance-assertions/contractProviderAssertionsPrompt.js +2 -1
- package/build/prompts/enhance-assertions/integrationAssertionsPrompt.js +2 -1
- package/build/prompts/enhance-assertions/sharedAssertionRules.d.ts +1 -1
- package/build/prompts/enhance-assertions/sharedAssertionRules.js +41 -22
- package/build/prompts/enhance-assertions/uiAssertionsPrompt.js +17 -9
- package/build/prompts/test-recommendation/diffExecutionPlan.js +0 -2
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +8 -2
- package/build/prompts/testbot/testbot-prompts.js +12 -7
- package/build/recommendation/registerPlan.d.ts +5 -1
- package/build/recommendation/registerPlan.js +5 -0
- package/build/recommendation/types.d.ts +25 -2
- package/build/recommendation/verifierContracts.d.ts +16 -4
- package/build/recommendation/verifierContracts.js +20 -4
- package/build/recommendation/verifiers/coverage.d.ts +10 -0
- package/build/recommendation/verifiers/coverage.js +144 -22
- package/build/recommendation/verifiers/deliveredMatchesPlan.d.ts +22 -0
- package/build/recommendation/verifiers/deliveredMatchesPlan.js +43 -0
- package/build/recommendation/verifiers/existingCoverage.js +53 -0
- package/build/recommendation/verifiers/expectedValueSourced.js +111 -14
- package/build/services/TestGenerationService.js +3 -1
- package/build/tools/code-refactor/codeReuseTool.js +1 -1
- package/build/tools/code-refactor/reuse-outcome.d.ts +1 -1
- package/build/tools/code-refactor/reuse-state.d.ts +85 -7
- package/build/tools/code-refactor/reuse-state.js +239 -34
- package/build/tools/code-refactor/utils-verify-gates.d.ts +5 -0
- package/build/tools/code-refactor/utils-verify-gates.js +103 -11
- package/build/tools/generate-tests/generateBatchScenarioRestTool.js +2 -1
- package/build/tools/submitReportTool.js +259 -38
- package/build/tools/test-management/actionsTool.js +5 -0
- package/build/tools/test-management/analyzeChangesTool.d.ts +53 -0
- package/build/tools/test-management/analyzeChangesTool.js +55 -2
- package/build/tools/test-management/registerTestPlanTool.d.ts +2 -1
- package/build/tools/test-management/registerTestPlanTool.js +29 -14
- package/build/types/ReuseOutcome.d.ts +73 -7
- package/build/types/TestAnalysis.d.ts +6 -0
- package/build/types/TestbotReport.d.ts +15 -1
- package/build/utils/AnalysisStateManager.d.ts +7 -1
- package/build/utils/AnalysisStateManager.js +5 -1
- package/build/utils/assertion-verify/api-shared-lints.js +71 -34
- package/build/utils/assertion-verify/format.js +2 -2
- package/build/utils/assertion-verify/helper-imports.d.ts +7 -0
- package/build/utils/assertion-verify/helper-imports.js +119 -27
- package/build/utils/assertion-verify/lint-types.d.ts +31 -2
- package/build/utils/assertion-verify/lint-types.js +66 -0
- package/build/utils/assertion-verify/metrics.d.ts +13 -0
- package/build/utils/assertion-verify/metrics.js +16 -0
- package/build/utils/assertion-verify/verify.d.ts +11 -6
- package/build/utils/assertion-verify/verify.js +56 -15
- package/build/utils/canonicalJson.d.ts +11 -0
- package/build/utils/canonicalJson.js +17 -0
- package/build/utils/utils-verify/action-key.d.ts +27 -0
- package/build/utils/utils-verify/action-key.js +292 -0
- package/build/utils/utils-verify/allow.d.ts +8 -1
- package/build/utils/utils-verify/allow.js +14 -1
- package/build/utils/utils-verify/call-sites.d.ts +76 -8
- package/build/utils/utils-verify/call-sites.js +256 -70
- package/build/utils/utils-verify/language-spec.d.ts +3 -2
- package/build/utils/utils-verify/parse.d.ts +22 -3
- package/build/utils/utils-verify/parse.js +123 -52
- package/build/utils/utils-verify/verify.d.ts +33 -3
- package/build/utils/utils-verify/verify.js +126 -12
- package/build/utils/workspaceAuth.d.ts +59 -19
- package/build/utils/workspaceAuth.js +228 -31
- package/package.json +1 -1
- package/plugin/prompts/generate-tests/execution-plan.md +1 -1
- package/plugin/prompts/generate-tests/generation.md +1 -0
- package/plugin/prompts/plan-tests.md +33 -16
|
@@ -88,8 +88,16 @@ export const EXISTING_COVERAGE_CONTRACT = {
|
|
|
88
88
|
message: "This planned test names an existing test file that is not in the checkout.",
|
|
89
89
|
suggestion: "Name the file repository-relative, spelled as the analysis lists it. Drop the entry if it names a test you did not read — naming none is an answer too.",
|
|
90
90
|
},
|
|
91
|
+
maintainedTestMissing: {
|
|
92
|
+
message: "`maintains` names an existing test file that is not in the checkout.",
|
|
93
|
+
suggestion: "Name the file repository-relative, spelled as the analysis lists it. A maintenance entry is what makes its changes count as covered, so it has to name a file this run can edit.",
|
|
94
|
+
},
|
|
95
|
+
maintainedUnknownChange: {
|
|
96
|
+
message: "A `maintains` entry cites a change id that this plan does not declare.",
|
|
97
|
+
suggestion: "Spell the change's `id` exactly as your own `changes` list gives it — the match is exact apart from surrounding space. Declare the change if it is missing.",
|
|
98
|
+
},
|
|
91
99
|
},
|
|
92
|
-
suggestion: "Which existing tests did you read on this change,
|
|
100
|
+
suggestion: "Which existing tests did you read on this change, how does this planned test differ from them, and which changes does the maintenance you are doing cover?",
|
|
93
101
|
};
|
|
94
102
|
export const EXPECTED_OUTCOME_CONTRACT = {
|
|
95
103
|
id: "expectedOutcome",
|
|
@@ -141,7 +149,7 @@ export const COVERAGE_CONTRACT = {
|
|
|
141
149
|
objections: {
|
|
142
150
|
change: {
|
|
143
151
|
message: "This declared change has no test in the plan.",
|
|
144
|
-
suggestion: "Plan a test that exercises it. If this run cannot write one, answer it and set `blocker` to what stopped the run — a service that is not running, a paired branch that no longer exists, the one credential the run holds. An answer with no blocker leaves this open: a change nothing prevented a test from reaching takes the test. Coverage per file is the basic minimum; coverage per change is what the plan is judged by, so a file covered by a shape test still leaves the change it makes untested.",
|
|
152
|
+
suggestion: "Plan a test that exercises it. If an existing test already covers it and this run edits that test rather than writing a new one, declare it in `maintains` with the file, what your edit changes, and this change's id — maintenance is coverage, and a duplicate spec beside the one you edited is the mistake. If this run cannot write one, answer it and set `blocker` to what stopped the run — a service that is not running, a paired branch that no longer exists, the one credential the run holds. An answer with no blocker leaves this open: a change nothing prevented a test from reaching takes the test. Coverage per file is the basic minimum; coverage per change is what the plan is judged by, so a file covered by a shape test still leaves the change it makes untested.",
|
|
145
153
|
},
|
|
146
154
|
noChanges: {
|
|
147
155
|
message: "The plan declares no changes, so nothing says what the pull request must make different.",
|
|
@@ -253,14 +261,22 @@ export const EXPECTED_VALUE_SOURCED_CONTRACT = {
|
|
|
253
261
|
// 34414643135 objected to five cases whose values that rule decides.
|
|
254
262
|
quoteMissing: {
|
|
255
263
|
message: "This case reads its value from the pull request, and its change does not quote the sentence that states the rule.",
|
|
256
|
-
suggestion: "Put the sentence from the title or description on the change as `quote`, as written there.
|
|
264
|
+
suggestion: "Put the sentence from the title or description on the change as `quote`, as written there. If that sentence states the rule and not the number, put the rule in `derived` and leave `expectedValue` out.",
|
|
265
|
+
},
|
|
266
|
+
// The other half of the same reading. Run 34434835959 declared
|
|
267
|
+
// `expectedValue: 159.998` from `pr-description` on a change whose quote states
|
|
268
|
+
// how a percentage discount works and writes no number; the agent computed 10%
|
|
269
|
+
// of a subtotal it had seen and attributed the result to the description.
|
|
270
|
+
valueNotQuoted: {
|
|
271
|
+
message: "This case reads its value from the pull request, and the pull request does not state this value.",
|
|
272
|
+
suggestion: "Quote the sentence that states it, or, if the pull request states a rule, put the rule in `derived` and leave `expectedValue` out; the test computes the value from what it sent.",
|
|
257
273
|
},
|
|
258
274
|
// A test that expects to fail asserts what the code does not return today, so
|
|
259
275
|
// some source outside the code states that value. Run 34406282133 declared such
|
|
260
276
|
// tests and put `expectedValue` on no case at all.
|
|
261
277
|
failWithoutValue: {
|
|
262
278
|
message: "This test expects to fail, and no case on the changes it cites states the value it must assert.",
|
|
263
|
-
suggestion: "Put the value the code does not return today on the case this test sends, as `expectedValue`, and name where you read it. If
|
|
279
|
+
suggestion: "Put the value the code does not return today on the case this test sends, as `expectedValue`, and name where you read it. If the source states a rule rather than a number, put the rule in `derived` instead. If no source states either, answer with what the test asserts instead and where that comes from.",
|
|
264
280
|
},
|
|
265
281
|
},
|
|
266
282
|
suggestion: "Where does the correct value for this case come from?",
|
|
@@ -1,4 +1,14 @@
|
|
|
1
1
|
import { Verifier } from "../types.js";
|
|
2
|
+
/** One declared case as the objection spells it.
|
|
3
|
+
*
|
|
4
|
+
* Exported: `expectedValueSourced` keys its objections on the same rendering, so
|
|
5
|
+
* two cases on one parameter are two objections there too. */
|
|
6
|
+
export declare function renderCase(declared: {
|
|
7
|
+
param: string;
|
|
8
|
+
value?: unknown;
|
|
9
|
+
absent?: true;
|
|
10
|
+
expect: string;
|
|
11
|
+
}): string;
|
|
2
12
|
/** The completeness check, over three surfaces: every declared behaviour, every
|
|
3
13
|
* mutated route, and every changed file needs a test in the plan. More is
|
|
4
14
|
* welcome, fewer is the objection. ONE RULE for the file half, no classification
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { allChangedFiles } from "../types.js";
|
|
2
|
-
import {
|
|
2
|
+
import { canonicalJson } from "../../utils/canonicalJson.js";
|
|
3
|
+
import { escapesRepo, LOCKFILE, NOT_A_ROUTE_FILE, NOT_SOURCE_DIR } from "./endpointGrounded.js";
|
|
3
4
|
import { isTestPath } from "../../utils/testFileClassification.js";
|
|
4
5
|
import { normalizeCitedPath } from "./citedPath.js";
|
|
5
6
|
import { COVERAGE_CONTRACT, fillPlaceholders } from "../verifierContracts.js";
|
|
@@ -37,9 +38,31 @@ function citedFiles(plannedTests) {
|
|
|
37
38
|
}
|
|
38
39
|
return cited;
|
|
39
40
|
}
|
|
40
|
-
/** The behaviour ids
|
|
41
|
-
|
|
41
|
+
/** The behaviour ids the plan cites, trimmed — from its planned tests AND from the
|
|
42
|
+
* existing tests it maintains. A change covered by a spec this run edits is tested;
|
|
43
|
+
* counting only planned tests made `coverage:change:` unanswerable for a correct
|
|
44
|
+
* maintenance decision, since that objection closes with a `blocker` alone and
|
|
45
|
+
* nothing stopped the run.
|
|
46
|
+
*
|
|
47
|
+
* A maintenance entry credits its ids only when the file it names RESOLVES in the
|
|
48
|
+
* checkout. `existingCoverage` objects about an unresolvable one, but that objection
|
|
49
|
+
* takes prose, so crediting first would have handed `coverage:change:` — the one
|
|
50
|
+
* class a sentence cannot close, and the reason `BLOCKER_ONLY_PREFIX` exists — a
|
|
51
|
+
* route out through a file name nobody can open. */
|
|
52
|
+
function citedBehaviors(plannedTests, maintains = [], fileResolves = () => false) {
|
|
42
53
|
const cited = new Set();
|
|
54
|
+
for (const entry of maintains) {
|
|
55
|
+
const ids = entry?.changes;
|
|
56
|
+
if (!Array.isArray(ids))
|
|
57
|
+
continue;
|
|
58
|
+
if (!fileResolves(String(entry?.file ?? "").trim()))
|
|
59
|
+
continue;
|
|
60
|
+
for (const id of ids) {
|
|
61
|
+
const trimmed = String(id ?? "").trim();
|
|
62
|
+
if (trimmed)
|
|
63
|
+
cited.add(trimmed);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
43
66
|
for (const plannedTest of plannedTests) {
|
|
44
67
|
const declared = plannedTest?.declarations?.changes;
|
|
45
68
|
if (!Array.isArray(declared))
|
|
@@ -58,7 +81,7 @@ function citedBehaviors(plannedTests) {
|
|
|
58
81
|
*
|
|
59
82
|
* Checked whether or not the run has a diff: the agent declared these itself, so
|
|
60
83
|
* there is always something to hold it to. */
|
|
61
|
-
function claimObjections(registration) {
|
|
84
|
+
function claimObjections(registration, ctx) {
|
|
62
85
|
const changes = Array.isArray(registration.changes) ? registration.changes : [];
|
|
63
86
|
const plannedTests = registration.plannedTests ?? [];
|
|
64
87
|
const objections = [];
|
|
@@ -79,7 +102,8 @@ function claimObjections(registration) {
|
|
|
79
102
|
if (id)
|
|
80
103
|
declared.add(id);
|
|
81
104
|
}
|
|
82
|
-
const
|
|
105
|
+
const maintains = Array.isArray(registration.maintains) ? registration.maintains : [];
|
|
106
|
+
const cited = citedBehaviors(plannedTests, maintains, (file) => !!file && !escapesRepo(file) && ctx.citedFileExists(file));
|
|
83
107
|
const seen = new Set();
|
|
84
108
|
for (const change of changes) {
|
|
85
109
|
const id = String(change?.id ?? "").trim();
|
|
@@ -207,28 +231,100 @@ function sameValue(sent, declared) {
|
|
|
207
231
|
return false;
|
|
208
232
|
if (declared === null)
|
|
209
233
|
return sent === null;
|
|
234
|
+
// A list or an object compares WHOLE, and only against another one. `String` made
|
|
235
|
+
// every structure a scalar: `String(["alpha","beta"])` is `"alpha,beta"`, so a
|
|
236
|
+
// case declaring that string was answered by a list no entry of which sends it,
|
|
237
|
+
// and `String([])` is `""`, so an empty list answered `tags=""`. An entry of the
|
|
238
|
+
// list is still a scalar of its own and still answers a scalar case.
|
|
239
|
+
const sentIsStructure = sent !== null && typeof sent === "object";
|
|
240
|
+
const declaredIsStructure = typeof declared === "object";
|
|
241
|
+
if (sentIsStructure || declaredIsStructure) {
|
|
242
|
+
// Both sides are non-null objects by the guard above, so neither call can be
|
|
243
|
+
// the `undefined` the helper returns for a value JSON renders as nothing.
|
|
244
|
+
return sentIsStructure && declaredIsStructure && canonicalJson(sent) === canonicalJson(declared);
|
|
245
|
+
}
|
|
210
246
|
const sentNumber = asNumber(sent);
|
|
211
247
|
const declaredNumber = asNumber(declared);
|
|
212
248
|
if (sentNumber !== undefined && declaredNumber !== undefined)
|
|
213
249
|
return sentNumber === declaredNumber;
|
|
214
250
|
return String(sent) === String(declared);
|
|
215
251
|
}
|
|
216
|
-
/**
|
|
217
|
-
*
|
|
218
|
-
|
|
219
|
-
|
|
252
|
+
/** The path a case's `param` names, as segments. Numeric segments drop: a plan
|
|
253
|
+
* writes `items.0` for "the line items", not for the first one only. */
|
|
254
|
+
function fieldPath(param) {
|
|
255
|
+
const segments = param
|
|
256
|
+
.split(/[.[\]]/)
|
|
257
|
+
.map((segment) => segment.trim())
|
|
258
|
+
.filter((segment) => segment.length > 0 && !/^\d+$/.test(segment));
|
|
259
|
+
return segments.length > 0 ? segments : [param];
|
|
260
|
+
}
|
|
261
|
+
/** Every value at `rest`, below a key that already matched the path's first
|
|
262
|
+
* segment. Only arrays are transparent between segments, so `billing.postcode` is
|
|
263
|
+
* not sent by `billing.shipping.postcode`. A matched array yields the array AND
|
|
264
|
+
* its entries, because `String(["beta"]) === "beta"` matched a one-item list only.
|
|
265
|
+
*
|
|
266
|
+
* `open` holds the objects on the way down to this one, so a payload that points
|
|
267
|
+
* back at itself ends rather than recurses. It is the way down and not every
|
|
268
|
+
* object seen, because one object reached twice by two different routes is two
|
|
269
|
+
* values the payload really sends. */
|
|
270
|
+
function valuesAt(value, rest, open) {
|
|
271
|
+
if (rest.length === 0)
|
|
272
|
+
return [value, ...(Array.isArray(value) ? value : [])];
|
|
273
|
+
if (value === null || typeof value !== "object" || open.has(value))
|
|
274
|
+
return [];
|
|
275
|
+
open.add(value);
|
|
276
|
+
const found = Array.isArray(value)
|
|
277
|
+
? value.flatMap((entry) => valuesAt(entry, rest, open))
|
|
278
|
+
: (() => {
|
|
279
|
+
const [head, ...next] = rest;
|
|
280
|
+
const fields = value;
|
|
281
|
+
return head in fields ? valuesAt(fields[head], next, open) : [];
|
|
282
|
+
})();
|
|
283
|
+
open.delete(value);
|
|
284
|
+
return found;
|
|
285
|
+
}
|
|
286
|
+
/** Every value carried at `path`, anywhere inside one payload. Only the FIRST
|
|
287
|
+
* segment is found at any depth, which is what makes `quantity` find
|
|
288
|
+
* `items.0.quantity`; the rest of the path is matched by `valuesAt`. */
|
|
289
|
+
function valuesUnder(payload, path, open = new Set()) {
|
|
290
|
+
// A body that points back at itself ends the walk; how deeply a real body nests
|
|
291
|
+
// is the payload's business, and a depth cap here read a valid one as sending
|
|
292
|
+
// nothing.
|
|
293
|
+
if (payload === null || typeof payload !== "object" || open.has(payload))
|
|
294
|
+
return [];
|
|
295
|
+
open.add(payload);
|
|
296
|
+
const found = [];
|
|
297
|
+
if (Array.isArray(payload)) {
|
|
298
|
+
for (const entry of payload)
|
|
299
|
+
found.push(...valuesUnder(entry, path, open));
|
|
300
|
+
}
|
|
301
|
+
else {
|
|
302
|
+
for (const [key, value] of Object.entries(payload)) {
|
|
303
|
+
if (key === path[0])
|
|
304
|
+
found.push(...valuesAt(value, path.slice(1), open));
|
|
305
|
+
// The same head, deeper: this is the any-depth half of the search.
|
|
306
|
+
found.push(...valuesUnder(value, path, open));
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
open.delete(payload);
|
|
310
|
+
return found;
|
|
311
|
+
}
|
|
312
|
+
/** Every value of one parameter a step sends: the query string, the `queryParams`
|
|
313
|
+
* object, or anywhere inside the request body. */
|
|
220
314
|
function valuesSent(step, param) {
|
|
221
315
|
const found = [];
|
|
316
|
+
// A query parameter is flat and its own name can hold a dot, so the query answers
|
|
317
|
+
// the parameter as written. Only the body is read as a path.
|
|
222
318
|
const query = String(step?.path ?? "").split("?").slice(1).join("?");
|
|
223
319
|
if (query)
|
|
224
320
|
found.push(...new URLSearchParams(query).getAll(param));
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
}
|
|
321
|
+
const queryParams = step?.queryParams;
|
|
322
|
+
if (queryParams && typeof queryParams === "object" && !Array.isArray(queryParams)) {
|
|
323
|
+
const fields = queryParams;
|
|
324
|
+
if (param in fields)
|
|
325
|
+
found.push(fields[param]);
|
|
231
326
|
}
|
|
327
|
+
found.push(...valuesUnder(step?.requestBody, fieldPath(param)));
|
|
232
328
|
return found;
|
|
233
329
|
}
|
|
234
330
|
/** The parameter name that names the request's verb rather than one of its inputs. */
|
|
@@ -316,6 +412,15 @@ function sendsInPath(step, slot, value) {
|
|
|
316
412
|
}
|
|
317
413
|
return sameValue(segments[slot.index], value);
|
|
318
414
|
}
|
|
415
|
+
/** Whether a step writes the declared value as one of its own path segments. The
|
|
416
|
+
* fallback for a plan that spells every route concretely, so no slot exists. Only
|
|
417
|
+
* used when the parameter has no slot at all: where the plan does say where the
|
|
418
|
+
* parameter sits, the slot rule stands. */
|
|
419
|
+
function sendsAsPathSegment(step, value) {
|
|
420
|
+
if (value === undefined || value === null || String(value).trim().length === 0)
|
|
421
|
+
return false;
|
|
422
|
+
return pathSegments(step).some((segment) => sameValue(segment.trim(), value));
|
|
423
|
+
}
|
|
319
424
|
/** Whether a step's verb is the declared value. A rejected method is the input. */
|
|
320
425
|
function sendsAsMethod(step, value) {
|
|
321
426
|
const method = String(step?.method ?? "").trim();
|
|
@@ -332,8 +437,11 @@ function omittedBy(step, param) {
|
|
|
332
437
|
return false;
|
|
333
438
|
return omits.some((named) => String(named ?? "").trim() === param);
|
|
334
439
|
}
|
|
335
|
-
/** One declared case as the objection spells it.
|
|
336
|
-
|
|
440
|
+
/** One declared case as the objection spells it.
|
|
441
|
+
*
|
|
442
|
+
* Exported: `expectedValueSourced` keys its objections on the same rendering, so
|
|
443
|
+
* two cases on one parameter are two objections there too. */
|
|
444
|
+
export function renderCase(declared) {
|
|
337
445
|
const param = String(declared.param).trim();
|
|
338
446
|
const sent = declared.absent === true ? "absent" : `=${JSON.stringify(declared.value)}`;
|
|
339
447
|
return `${param}${declared.absent === true ? " " : ""}${sent} (${declared.expect})`;
|
|
@@ -381,6 +489,7 @@ function caseObjections(registration) {
|
|
|
381
489
|
const steps = plannedTest?.scenario?.steps;
|
|
382
490
|
return Array.isArray(steps) ? steps : [];
|
|
383
491
|
});
|
|
492
|
+
const writesAnyPlaceholder = citingSteps.some((step) => pathSegments(step).some((segment) => placeholderName(segment) !== undefined));
|
|
384
493
|
const missing = cases.filter((declared) => {
|
|
385
494
|
const param = String(declared?.param ?? "").trim();
|
|
386
495
|
if (!param)
|
|
@@ -389,9 +498,14 @@ function caseObjections(registration) {
|
|
|
389
498
|
return !citingSteps.some((step) => omittedBy(step, param));
|
|
390
499
|
const slots = pathSlotsFor(citingSteps, cases, param);
|
|
391
500
|
const isMethod = param.toLowerCase() === METHOD_PARAM;
|
|
501
|
+
// Only for a plan that wrote NO template at all, and only for a parameter that
|
|
502
|
+
// names a record. Wider than that, the URL answered everything: a `quantity=5`
|
|
503
|
+
// case passed on `GET /api/v1/orders/5`.
|
|
504
|
+
const byValueInPath = slots.length === 0 && !writesAnyPlaceholder && ID_PARAM.test(param) && !isMethod;
|
|
392
505
|
return !citingSteps.some((step) => valuesSent(step, param).some((sent) => sameValue(sent, declared.value)) ||
|
|
393
506
|
(isMethod && sendsAsMethod(step, declared.value)) ||
|
|
394
|
-
slots.some((slot) => sendsInPath(step, slot, declared.value))
|
|
507
|
+
slots.some((slot) => sendsInPath(step, slot, declared.value)) ||
|
|
508
|
+
(byValueInPath && sendsAsPathSegment(step, declared.value)));
|
|
395
509
|
});
|
|
396
510
|
// One objection per case. A single objection listing three cases took one
|
|
397
511
|
// answer that addressed two of them and closed: run 34167252679 answered
|
|
@@ -405,7 +519,13 @@ function caseObjections(registration) {
|
|
|
405
519
|
objectionId: `coverage:cases:${id}:${rendered}`,
|
|
406
520
|
verifier: "coverage",
|
|
407
521
|
message: fillPlaceholders(COVERAGE_CONTRACT.objections.cases.message, { missing: rendered }),
|
|
408
|
-
evidence: `${citing.map((plannedTest) => plannedTest.plannedTestId).join(", ")} cite ${id}; no step of theirs sends ${rendered}
|
|
522
|
+
evidence: `${citing.map((plannedTest) => plannedTest.plannedTestId).join(", ")} cite ${id}; no step of theirs sends ${rendered}.` +
|
|
523
|
+
// Which spelling WOULD have matched, or the agent re-sends the same plan.
|
|
524
|
+
// Id parameters only: it asked for `{quantity}` in a route otherwise.
|
|
525
|
+
(ID_PARAM.test(String(declared?.param ?? "").trim()) &&
|
|
526
|
+
citingSteps.some((step) => sendsAsPathSegment(step, declared.value))
|
|
527
|
+
? ` A step's path carries that value, in a segment this plan's own placeholders do not name \`${String(declared?.param ?? "").trim()}\`; write the route as a template, with \`{${String(declared?.param ?? "").trim()}}\` where it sits.`
|
|
528
|
+
: ""),
|
|
409
529
|
suggestion: COVERAGE_CONTRACT.objections.cases.suggestion,
|
|
410
530
|
});
|
|
411
531
|
}
|
|
@@ -416,8 +536,10 @@ function caseObjections(registration) {
|
|
|
416
536
|
* sends the same spelling, so `{memberUserId}` in a case is that record and a
|
|
417
537
|
* literal is a value the plan chose. */
|
|
418
538
|
const PLACEHOLDER_VALUE = /^\{[A-Za-z][A-Za-z0-9]*\}$/;
|
|
419
|
-
/** A parameter that names a record by id, by its own name
|
|
420
|
-
|
|
539
|
+
/** A parameter that names a record by id, by its own name: `id` itself, `id` after
|
|
540
|
+
* a separator, or `Id`/`ID` starting a word. `/id$/i` also read `paid`, `valid` and
|
|
541
|
+
* `fluid` as record ids, which let `valid=5` take its value from `/orders/5`. */
|
|
542
|
+
const ID_PARAM = /(?:^|[_\-.])[Ii][Dd]$|[a-z0-9](?:Id|ID)$/;
|
|
421
543
|
function isPlaceholderValue(value) {
|
|
422
544
|
return typeof value === "string" && PLACEHOLDER_VALUE.test(value.trim());
|
|
423
545
|
}
|
|
@@ -578,7 +700,7 @@ export const coverage = {
|
|
|
578
700
|
name: "coverage",
|
|
579
701
|
run(registration, ctx) {
|
|
580
702
|
const objections = [
|
|
581
|
-
...claimObjections(registration),
|
|
703
|
+
...claimObjections(registration, ctx),
|
|
582
704
|
...stateTestObjections(registration),
|
|
583
705
|
...caseObjections(registration),
|
|
584
706
|
...rejectObjections(registration),
|
|
@@ -4,8 +4,30 @@ import { Plan } from "../registerPlan.js";
|
|
|
4
4
|
export interface DeliveredTest {
|
|
5
5
|
plannedTestId: string;
|
|
6
6
|
}
|
|
7
|
+
/** One maintenance row that LEFT COVERAGE BEHIND, as the report publishes it. The
|
|
8
|
+
* caller filters by action: a row this check is handed is one whose edit a later
|
|
9
|
+
* run can still rely on, so a no-op assessment and a deleted test never reach it.
|
|
10
|
+
* Both fields are absolute paths. */
|
|
11
|
+
export interface MaintainedTest {
|
|
12
|
+
testFilePath?: string;
|
|
13
|
+
/** Where the edit landed when that is not the spec — a page object it delegates
|
|
14
|
+
* its selectors to. Counted, so a spec maintained through its POM still joins. */
|
|
15
|
+
pomFile?: string;
|
|
16
|
+
}
|
|
7
17
|
/** Verifier 5, post-execution half. Both directions on one pass: a report entry
|
|
8
18
|
* that was in no plan, and a planned planned test that never shipped. The two lists
|
|
9
19
|
* join on `plannedTestId` exactly — never a name, an endpoint or a similarity
|
|
10
20
|
* score. Objections go to the report: the tests already exist. */
|
|
11
21
|
export declare function checkDeliveredMatchesPlan(plan: Plan, delivered: DeliveredTest[]): Objection[];
|
|
22
|
+
/** The maintenance mirror of "planned but not delivered". A `maintains` entry is
|
|
23
|
+
* what makes its changes count as covered, and until this ran nothing downstream
|
|
24
|
+
* checked the file was edited at all: the plan claimed it, `coverage` credited it,
|
|
25
|
+
* and the report published it, three records that never met.
|
|
26
|
+
*
|
|
27
|
+
* Matched with `testFileMatches`, which keeps the directories: the plan names a file
|
|
28
|
+
* repository-relative and a row names it absolutely, so the row's path has to END at
|
|
29
|
+
* the claimed one on a segment boundary. Basename alone would let an edit to
|
|
30
|
+
* `tests/admin/navbar.spec.ts` answer for `tests/store/navbar.spec.ts`. Answerable,
|
|
31
|
+
* unlike `coverage:change:` — an edit that landed somewhere the rows do not name is
|
|
32
|
+
* a real case, and the answer says where. */
|
|
33
|
+
export declare function checkMaintenanceDelivered(plan: Plan, maintained: MaintainedTest[]): Objection[];
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { testFileMatches } from "../../utils/utils.js";
|
|
1
2
|
/** Verifier 5, post-execution half. Both directions on one pass: a report entry
|
|
2
3
|
* that was in no plan, and a planned planned test that never shipped. The two lists
|
|
3
4
|
* join on `plannedTestId` exactly — never a name, an endpoint or a similarity
|
|
@@ -31,3 +32,45 @@ export function checkDeliveredMatchesPlan(plan, delivered) {
|
|
|
31
32
|
}
|
|
32
33
|
return objections;
|
|
33
34
|
}
|
|
35
|
+
/** The maintenance mirror of "planned but not delivered". A `maintains` entry is
|
|
36
|
+
* what makes its changes count as covered, and until this ran nothing downstream
|
|
37
|
+
* checked the file was edited at all: the plan claimed it, `coverage` credited it,
|
|
38
|
+
* and the report published it, three records that never met.
|
|
39
|
+
*
|
|
40
|
+
* Matched with `testFileMatches`, which keeps the directories: the plan names a file
|
|
41
|
+
* repository-relative and a row names it absolutely, so the row's path has to END at
|
|
42
|
+
* the claimed one on a segment boundary. Basename alone would let an edit to
|
|
43
|
+
* `tests/admin/navbar.spec.ts` answer for `tests/store/navbar.spec.ts`. Answerable,
|
|
44
|
+
* unlike `coverage:change:` — an edit that landed somewhere the rows do not name is
|
|
45
|
+
* a real case, and the answer says where. */
|
|
46
|
+
export function checkMaintenanceDelivered(plan, maintained) {
|
|
47
|
+
const claimed = Array.isArray(plan?.maintains) ? plan.maintains : [];
|
|
48
|
+
if (claimed.length === 0)
|
|
49
|
+
return [];
|
|
50
|
+
const text = (value) => (typeof value === "string" ? value.trim() : "");
|
|
51
|
+
const editedPaths = [];
|
|
52
|
+
for (const row of Array.isArray(maintained) ? maintained : []) {
|
|
53
|
+
for (const full of [text(row?.testFilePath), text(row?.pomFile)])
|
|
54
|
+
if (full)
|
|
55
|
+
editedPaths.push(full);
|
|
56
|
+
}
|
|
57
|
+
const objections = [];
|
|
58
|
+
const seen = new Set();
|
|
59
|
+
for (const entry of claimed) {
|
|
60
|
+
const file = text(entry?.file);
|
|
61
|
+
if (!file || editedPaths.some((full) => testFileMatches(full, file)))
|
|
62
|
+
continue;
|
|
63
|
+
const objectionId = `deliveredMatchesPlan:maintains:${file}`;
|
|
64
|
+
if (seen.has(objectionId))
|
|
65
|
+
continue;
|
|
66
|
+
seen.add(objectionId);
|
|
67
|
+
objections.push({
|
|
68
|
+
objectionId,
|
|
69
|
+
verifier: "deliveredMatchesPlan",
|
|
70
|
+
message: "The plan says this existing test covers a change, and no maintenance row says it was edited.",
|
|
71
|
+
evidence: `plan maintains ${file}; the maintenance rows that changed a file name ${editedPaths.length > 0 ? editedPaths.join(", ") : "no file"}`,
|
|
72
|
+
suggestion: "The maintenance rows come from the triage, not from this call, so this is not a field to fill in here: make the edit on the file you claimed and let the maintenance step record it, or drop the entry from `maintains` and register the plan again — coverage counted the change because the plan said you were editing that test. If the test already covered the change and needed no edit, or the edit landed somewhere the rows do not name, say which as the answer.",
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
return objections;
|
|
76
|
+
}
|
|
@@ -42,6 +42,59 @@ export const existingCoverage = {
|
|
|
42
42
|
suggestion: EXISTING_COVERAGE_CONTRACT.objections.citedTestMissing.suggestion,
|
|
43
43
|
});
|
|
44
44
|
}
|
|
45
|
+
// `maintains` is what makes a change covered by an edited test count in
|
|
46
|
+
// `coverage`, so the same file check applies to it, plus the change ids it
|
|
47
|
+
// cites: an id no `changes` entry declares would credit coverage for nothing.
|
|
48
|
+
const maintains = Array.isArray(registration?.maintains) ? registration.maintains : [];
|
|
49
|
+
const declaredChanges = new Set((Array.isArray(registration?.changes) ? registration.changes : [])
|
|
50
|
+
.map((change) => String(change?.id ?? "").trim())
|
|
51
|
+
.filter(Boolean));
|
|
52
|
+
// Gathered per file BEFORE any objection is pushed: the schema allows two
|
|
53
|
+
// entries naming the same file, the objection id is keyed on the file so the
|
|
54
|
+
// agent answers a file once, and keying alone dropped the second entry's ids.
|
|
55
|
+
const missingByFile = new Map();
|
|
56
|
+
const unknownByFile = new Map();
|
|
57
|
+
for (const entry of maintains) {
|
|
58
|
+
const file = String(entry?.file ?? "").trim();
|
|
59
|
+
const key = file || "unnamed";
|
|
60
|
+
const missing = !file
|
|
61
|
+
? "an entry names no file"
|
|
62
|
+
: escapesRepo(file)
|
|
63
|
+
? `${file} (points outside the repository)`
|
|
64
|
+
: !ctx.citedFileExists(file)
|
|
65
|
+
? `${file} (not found)`
|
|
66
|
+
: "";
|
|
67
|
+
if (missing && !missingByFile.has(key))
|
|
68
|
+
missingByFile.set(key, missing);
|
|
69
|
+
const ids = Array.isArray(entry?.changes) ? entry.changes : [];
|
|
70
|
+
for (const raw of ids) {
|
|
71
|
+
const id = String(raw ?? "").trim();
|
|
72
|
+
if (!id || declaredChanges.has(id))
|
|
73
|
+
continue;
|
|
74
|
+
const unknown = unknownByFile.get(key) ?? [];
|
|
75
|
+
if (!unknown.includes(id))
|
|
76
|
+
unknown.push(id);
|
|
77
|
+
unknownByFile.set(key, unknown);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
for (const [key, missing] of missingByFile) {
|
|
81
|
+
objections.push({
|
|
82
|
+
objectionId: `existingCoverage:maintains:${key}`,
|
|
83
|
+
verifier: "existingCoverage",
|
|
84
|
+
message: EXISTING_COVERAGE_CONTRACT.objections.maintainedTestMissing.message,
|
|
85
|
+
evidence: `maintained test not in the checkout: ${missing}`,
|
|
86
|
+
suggestion: EXISTING_COVERAGE_CONTRACT.objections.maintainedTestMissing.suggestion,
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
for (const [key, unknown] of unknownByFile) {
|
|
90
|
+
objections.push({
|
|
91
|
+
objectionId: `existingCoverage:maintains:unknownChange:${key}`,
|
|
92
|
+
verifier: "existingCoverage",
|
|
93
|
+
message: EXISTING_COVERAGE_CONTRACT.objections.maintainedUnknownChange.message,
|
|
94
|
+
evidence: `${key === "unnamed" ? "an unnamed entry" : key} cites ${unknown.map((id) => `"${id}"`).join(", ")}; this plan declares ${[...declaredChanges].map((id) => `"${id}"`).join(", ") || "no changes"}`,
|
|
95
|
+
suggestion: EXISTING_COVERAGE_CONTRACT.objections.maintainedUnknownChange.suggestion,
|
|
96
|
+
});
|
|
97
|
+
}
|
|
45
98
|
return objections;
|
|
46
99
|
},
|
|
47
100
|
};
|