@skyramp/mcp 0.4.0 → 0.4.1-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/prompts/enhance-assertions/contractProviderAssertionsPrompt.js +2 -1
- package/build/prompts/enhance-assertions/integrationAssertionsPrompt.js +2 -1
- package/build/prompts/enhance-assertions/sharedAssertionRules.d.ts +1 -1
- package/build/prompts/enhance-assertions/sharedAssertionRules.js +57 -22
- package/build/prompts/enhance-assertions/uiAssertionsPrompt.js +17 -9
- package/build/prompts/test-recommendation/diffExecutionPlan.js +0 -2
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +11 -6
- package/build/prompts/testbot/testbot-prompts.js +19 -12
- package/build/recommendation/answers.d.ts +11 -7
- package/build/recommendation/answers.js +14 -10
- package/build/recommendation/pullRequestText.d.ts +18 -0
- package/build/recommendation/pullRequestText.js +31 -0
- package/build/recommendation/registerPlan.d.ts +9 -1
- package/build/recommendation/registerPlan.js +8 -1
- package/build/recommendation/runVerifiers.js +6 -0
- package/build/recommendation/types.d.ts +58 -0
- package/build/recommendation/verifierContracts.d.ts +108 -13
- package/build/recommendation/verifierContracts.js +147 -29
- package/build/recommendation/verifiers/coverage.d.ts +10 -0
- package/build/recommendation/verifiers/coverage.js +144 -22
- package/build/recommendation/verifiers/defects.d.ts +9 -0
- package/build/recommendation/verifiers/defects.js +117 -0
- package/build/recommendation/verifiers/deliveredMatchesPlan.d.ts +22 -0
- package/build/recommendation/verifiers/deliveredMatchesPlan.js +43 -0
- package/build/recommendation/verifiers/existingCoverage.js +53 -0
- package/build/recommendation/verifiers/expectedValueSourced.d.ts +14 -0
- package/build/recommendation/verifiers/expectedValueSourced.js +246 -0
- package/build/recommendation/verifiers/issueTraceability.d.ts +52 -0
- package/build/recommendation/verifiers/issueTraceability.js +197 -0
- package/build/recommendation/verifiers/requirementSourced.d.ts +2 -0
- package/build/recommendation/verifiers/requirementSourced.js +168 -0
- package/build/services/TestGenerationService.js +3 -1
- package/build/tools/code-refactor/codeReuseTool.js +1 -1
- package/build/tools/code-refactor/reuse-outcome.d.ts +1 -1
- package/build/tools/code-refactor/reuse-state.d.ts +85 -7
- package/build/tools/code-refactor/reuse-state.js +239 -34
- package/build/tools/code-refactor/utils-verify-gates.d.ts +5 -0
- package/build/tools/code-refactor/utils-verify-gates.js +103 -11
- package/build/tools/generate-tests/generateBatchScenarioRestTool.js +2 -1
- package/build/tools/submitReportTool.js +276 -39
- package/build/tools/test-management/actionsTool.js +5 -0
- package/build/tools/test-management/analyzeChangesTool.d.ts +53 -0
- package/build/tools/test-management/analyzeChangesTool.js +55 -2
- package/build/tools/test-management/registerTestPlanTool.d.ts +32 -17
- package/build/tools/test-management/registerTestPlanTool.js +105 -16
- package/build/types/ReuseOutcome.d.ts +73 -7
- package/build/types/TestAnalysis.d.ts +6 -0
- package/build/types/TestbotReport.d.ts +22 -4
- package/build/utils/AnalysisStateManager.d.ts +7 -1
- package/build/utils/AnalysisStateManager.js +5 -1
- package/build/utils/assertion-verify/api-shared-lints.js +118 -11
- package/build/utils/assertion-verify/format.js +2 -2
- package/build/utils/assertion-verify/helper-imports.d.ts +7 -0
- package/build/utils/assertion-verify/helper-imports.js +119 -27
- package/build/utils/assertion-verify/lint-types.d.ts +31 -2
- package/build/utils/assertion-verify/lint-types.js +66 -0
- package/build/utils/assertion-verify/metrics.d.ts +13 -0
- package/build/utils/assertion-verify/metrics.js +16 -0
- package/build/utils/assertion-verify/verify.d.ts +11 -6
- package/build/utils/assertion-verify/verify.js +56 -15
- package/build/utils/canonicalJson.d.ts +11 -0
- package/build/utils/canonicalJson.js +17 -0
- package/build/utils/utils-verify/action-key.d.ts +27 -0
- package/build/utils/utils-verify/action-key.js +292 -0
- package/build/utils/utils-verify/allow.d.ts +8 -1
- package/build/utils/utils-verify/allow.js +14 -1
- package/build/utils/utils-verify/call-sites.d.ts +76 -8
- package/build/utils/utils-verify/call-sites.js +256 -70
- package/build/utils/utils-verify/language-spec.d.ts +3 -2
- package/build/utils/utils-verify/parse.d.ts +22 -3
- package/build/utils/utils-verify/parse.js +123 -52
- package/build/utils/utils-verify/verify.d.ts +33 -3
- package/build/utils/utils-verify/verify.js +126 -12
- package/build/utils/workspaceAuth.d.ts +59 -19
- package/build/utils/workspaceAuth.js +228 -31
- package/package.json +1 -1
- package/plugin/prompts/generate-tests/execution-plan.md +3 -3
- package/plugin/prompts/generate-tests/generation.md +1 -0
- package/plugin/prompts/plan-tests.md +41 -15
- package/plugin/prompts/testbot-task1.md +3 -9
- package/build/prompts/testbot/planDeclarations.d.ts +0 -6
- package/build/prompts/testbot/planDeclarations.js +0 -9
- package/plugin/prompts/declaring-a-plan.md +0 -20
|
@@ -4,8 +4,30 @@ import { Plan } from "../registerPlan.js";
|
|
|
4
4
|
export interface DeliveredTest {
|
|
5
5
|
plannedTestId: string;
|
|
6
6
|
}
|
|
7
|
+
/** One maintenance row that LEFT COVERAGE BEHIND, as the report publishes it. The
|
|
8
|
+
* caller filters by action: a row this check is handed is one whose edit a later
|
|
9
|
+
* run can still rely on, so a no-op assessment and a deleted test never reach it.
|
|
10
|
+
* Both fields are absolute paths. */
|
|
11
|
+
export interface MaintainedTest {
|
|
12
|
+
testFilePath?: string;
|
|
13
|
+
/** Where the edit landed when that is not the spec — a page object it delegates
|
|
14
|
+
* its selectors to. Counted, so a spec maintained through its POM still joins. */
|
|
15
|
+
pomFile?: string;
|
|
16
|
+
}
|
|
7
17
|
/** Verifier 5, post-execution half. Both directions on one pass: a report entry
|
|
8
18
|
* that was in no plan, and a planned planned test that never shipped. The two lists
|
|
9
19
|
* join on `plannedTestId` exactly — never a name, an endpoint or a similarity
|
|
10
20
|
* score. Objections go to the report: the tests already exist. */
|
|
11
21
|
export declare function checkDeliveredMatchesPlan(plan: Plan, delivered: DeliveredTest[]): Objection[];
|
|
22
|
+
/** The maintenance mirror of "planned but not delivered". A `maintains` entry is
|
|
23
|
+
* what makes its changes count as covered, and until this ran nothing downstream
|
|
24
|
+
* checked the file was edited at all: the plan claimed it, `coverage` credited it,
|
|
25
|
+
* and the report published it, three records that never met.
|
|
26
|
+
*
|
|
27
|
+
* Matched with `testFileMatches`, which keeps the directories: the plan names a file
|
|
28
|
+
* repository-relative and a row names it absolutely, so the row's path has to END at
|
|
29
|
+
* the claimed one on a segment boundary. Basename alone would let an edit to
|
|
30
|
+
* `tests/admin/navbar.spec.ts` answer for `tests/store/navbar.spec.ts`. Answerable,
|
|
31
|
+
* unlike `coverage:change:` — an edit that landed somewhere the rows do not name is
|
|
32
|
+
* a real case, and the answer says where. */
|
|
33
|
+
export declare function checkMaintenanceDelivered(plan: Plan, maintained: MaintainedTest[]): Objection[];
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { testFileMatches } from "../../utils/utils.js";
|
|
1
2
|
/** Verifier 5, post-execution half. Both directions on one pass: a report entry
|
|
2
3
|
* that was in no plan, and a planned planned test that never shipped. The two lists
|
|
3
4
|
* join on `plannedTestId` exactly — never a name, an endpoint or a similarity
|
|
@@ -31,3 +32,45 @@ export function checkDeliveredMatchesPlan(plan, delivered) {
|
|
|
31
32
|
}
|
|
32
33
|
return objections;
|
|
33
34
|
}
|
|
35
|
+
/** The maintenance mirror of "planned but not delivered". A `maintains` entry is
|
|
36
|
+
* what makes its changes count as covered, and until this ran nothing downstream
|
|
37
|
+
* checked the file was edited at all: the plan claimed it, `coverage` credited it,
|
|
38
|
+
* and the report published it, three records that never met.
|
|
39
|
+
*
|
|
40
|
+
* Matched with `testFileMatches`, which keeps the directories: the plan names a file
|
|
41
|
+
* repository-relative and a row names it absolutely, so the row's path has to END at
|
|
42
|
+
* the claimed one on a segment boundary. Basename alone would let an edit to
|
|
43
|
+
* `tests/admin/navbar.spec.ts` answer for `tests/store/navbar.spec.ts`. Answerable,
|
|
44
|
+
* unlike `coverage:change:` — an edit that landed somewhere the rows do not name is
|
|
45
|
+
* a real case, and the answer says where. */
|
|
46
|
+
export function checkMaintenanceDelivered(plan, maintained) {
|
|
47
|
+
const claimed = Array.isArray(plan?.maintains) ? plan.maintains : [];
|
|
48
|
+
if (claimed.length === 0)
|
|
49
|
+
return [];
|
|
50
|
+
const text = (value) => (typeof value === "string" ? value.trim() : "");
|
|
51
|
+
const editedPaths = [];
|
|
52
|
+
for (const row of Array.isArray(maintained) ? maintained : []) {
|
|
53
|
+
for (const full of [text(row?.testFilePath), text(row?.pomFile)])
|
|
54
|
+
if (full)
|
|
55
|
+
editedPaths.push(full);
|
|
56
|
+
}
|
|
57
|
+
const objections = [];
|
|
58
|
+
const seen = new Set();
|
|
59
|
+
for (const entry of claimed) {
|
|
60
|
+
const file = text(entry?.file);
|
|
61
|
+
if (!file || editedPaths.some((full) => testFileMatches(full, file)))
|
|
62
|
+
continue;
|
|
63
|
+
const objectionId = `deliveredMatchesPlan:maintains:${file}`;
|
|
64
|
+
if (seen.has(objectionId))
|
|
65
|
+
continue;
|
|
66
|
+
seen.add(objectionId);
|
|
67
|
+
objections.push({
|
|
68
|
+
objectionId,
|
|
69
|
+
verifier: "deliveredMatchesPlan",
|
|
70
|
+
message: "The plan says this existing test covers a change, and no maintenance row says it was edited.",
|
|
71
|
+
evidence: `plan maintains ${file}; the maintenance rows that changed a file name ${editedPaths.length > 0 ? editedPaths.join(", ") : "no file"}`,
|
|
72
|
+
suggestion: "The maintenance rows come from the triage, not from this call, so this is not a field to fill in here: make the edit on the file you claimed and let the maintenance step record it, or drop the entry from `maintains` and register the plan again — coverage counted the change because the plan said you were editing that test. If the test already covered the change and needed no edit, or the edit landed somewhere the rows do not name, say which as the answer.",
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
return objections;
|
|
76
|
+
}
|
|
@@ -42,6 +42,59 @@ export const existingCoverage = {
|
|
|
42
42
|
suggestion: EXISTING_COVERAGE_CONTRACT.objections.citedTestMissing.suggestion,
|
|
43
43
|
});
|
|
44
44
|
}
|
|
45
|
+
// `maintains` is what makes a change covered by an edited test count in
|
|
46
|
+
// `coverage`, so the same file check applies to it, plus the change ids it
|
|
47
|
+
// cites: an id no `changes` entry declares would credit coverage for nothing.
|
|
48
|
+
const maintains = Array.isArray(registration?.maintains) ? registration.maintains : [];
|
|
49
|
+
const declaredChanges = new Set((Array.isArray(registration?.changes) ? registration.changes : [])
|
|
50
|
+
.map((change) => String(change?.id ?? "").trim())
|
|
51
|
+
.filter(Boolean));
|
|
52
|
+
// Gathered per file BEFORE any objection is pushed: the schema allows two
|
|
53
|
+
// entries naming the same file, the objection id is keyed on the file so the
|
|
54
|
+
// agent answers a file once, and keying alone dropped the second entry's ids.
|
|
55
|
+
const missingByFile = new Map();
|
|
56
|
+
const unknownByFile = new Map();
|
|
57
|
+
for (const entry of maintains) {
|
|
58
|
+
const file = String(entry?.file ?? "").trim();
|
|
59
|
+
const key = file || "unnamed";
|
|
60
|
+
const missing = !file
|
|
61
|
+
? "an entry names no file"
|
|
62
|
+
: escapesRepo(file)
|
|
63
|
+
? `${file} (points outside the repository)`
|
|
64
|
+
: !ctx.citedFileExists(file)
|
|
65
|
+
? `${file} (not found)`
|
|
66
|
+
: "";
|
|
67
|
+
if (missing && !missingByFile.has(key))
|
|
68
|
+
missingByFile.set(key, missing);
|
|
69
|
+
const ids = Array.isArray(entry?.changes) ? entry.changes : [];
|
|
70
|
+
for (const raw of ids) {
|
|
71
|
+
const id = String(raw ?? "").trim();
|
|
72
|
+
if (!id || declaredChanges.has(id))
|
|
73
|
+
continue;
|
|
74
|
+
const unknown = unknownByFile.get(key) ?? [];
|
|
75
|
+
if (!unknown.includes(id))
|
|
76
|
+
unknown.push(id);
|
|
77
|
+
unknownByFile.set(key, unknown);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
for (const [key, missing] of missingByFile) {
|
|
81
|
+
objections.push({
|
|
82
|
+
objectionId: `existingCoverage:maintains:${key}`,
|
|
83
|
+
verifier: "existingCoverage",
|
|
84
|
+
message: EXISTING_COVERAGE_CONTRACT.objections.maintainedTestMissing.message,
|
|
85
|
+
evidence: `maintained test not in the checkout: ${missing}`,
|
|
86
|
+
suggestion: EXISTING_COVERAGE_CONTRACT.objections.maintainedTestMissing.suggestion,
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
for (const [key, unknown] of unknownByFile) {
|
|
90
|
+
objections.push({
|
|
91
|
+
objectionId: `existingCoverage:maintains:unknownChange:${key}`,
|
|
92
|
+
verifier: "existingCoverage",
|
|
93
|
+
message: EXISTING_COVERAGE_CONTRACT.objections.maintainedUnknownChange.message,
|
|
94
|
+
evidence: `${key === "unnamed" ? "an unnamed entry" : key} cites ${unknown.map((id) => `"${id}"`).join(", ")}; this plan declares ${[...declaredChanges].map((id) => `"${id}"`).join(", ") || "no changes"}`,
|
|
95
|
+
suggestion: EXISTING_COVERAGE_CONTRACT.objections.maintainedUnknownChange.suggestion,
|
|
96
|
+
});
|
|
97
|
+
}
|
|
45
98
|
return objections;
|
|
46
99
|
},
|
|
47
100
|
};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { Verifier } from "../types.js";
|
|
2
|
+
/** Verifier 10. A case that states what the response must carry says where that
|
|
3
|
+
* value came from, and the value did not come from the code under test.
|
|
4
|
+
*
|
|
5
|
+
* This is the check for the failure that costs the most and looks like success: a
|
|
6
|
+
* test whose expected value was copied out of the running app passes on the broken
|
|
7
|
+
* code and fails the moment the code is fixed, so it defends the defect instead of
|
|
8
|
+
* catching it. The plan is the only place the value can be decided, because by
|
|
9
|
+
* generation time the response is the only thing to hand.
|
|
10
|
+
*
|
|
11
|
+
* Silent on a case with no `expectedValue`: asserting the shape of a value nobody
|
|
12
|
+
* states is a legitimate position, and the objection would fire on every
|
|
13
|
+
* rejection case in every plan. */
|
|
14
|
+
export declare const expectedValueSourced: Verifier;
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
import { allChangedFiles } from "../types.js";
|
|
2
|
+
import { changedPathSet, normalizeCitedPath } from "./citedPath.js";
|
|
3
|
+
import { EXPECTED_VALUE_SOURCED_CONTRACT as CONTRACT } from "../verifierContracts.js";
|
|
4
|
+
import { appearsIn, searchable } from "../pullRequestText.js";
|
|
5
|
+
import { escapeRegExp } from "../../utils/regex.js";
|
|
6
|
+
import { renderCase } from "./coverage.js";
|
|
7
|
+
function readSource(raw) {
|
|
8
|
+
const source = typeof raw === "string" ? raw.trim() : "";
|
|
9
|
+
if (source.length === 0)
|
|
10
|
+
return { kind: "none", file: "" };
|
|
11
|
+
if (/^code$/i.test(source))
|
|
12
|
+
return { kind: "code", file: "" };
|
|
13
|
+
if (/^(pr-title|pr-description)$/i.test(source))
|
|
14
|
+
return { kind: "pullRequest", file: "" };
|
|
15
|
+
const named = /^(spec|convention):(.*)$/i.exec(source);
|
|
16
|
+
if (!named)
|
|
17
|
+
return { kind: "none", file: "" };
|
|
18
|
+
const kind = named[1].toLowerCase();
|
|
19
|
+
const cited = named[2].split(/[#\u00a7]/)[0].replace(/(?::\d+)+(?:-\d+)?$/, "").trim();
|
|
20
|
+
// `spec:` with nothing after it names no file, so there is nothing to check and
|
|
21
|
+
// nothing to trust either. A path that IS written and still normalises away is a
|
|
22
|
+
// named source, and it is named wrongly.
|
|
23
|
+
if (cited.length === 0)
|
|
24
|
+
return { kind: "none", file: "" };
|
|
25
|
+
const file = normalizeCitedPath(cited);
|
|
26
|
+
return file ? { kind, file } : { kind: "unresolvable", file: "" };
|
|
27
|
+
}
|
|
28
|
+
/** One case's address, for the evidence line. A case has no id of its own, so it
|
|
29
|
+
* is named the way the agent wrote it. */
|
|
30
|
+
function caseLabel(change, entry) {
|
|
31
|
+
return `${String(change?.id ?? "?")}:${String(entry?.param ?? "?")}`;
|
|
32
|
+
}
|
|
33
|
+
/** Every number the pull request writes, as a number. One value has many
|
|
34
|
+
* spellings — `1e21`, `1e+21`, `1E21`, the digits in full — and matching the
|
|
35
|
+
* value's OWN spelling refused every other one, so a pull request stating the
|
|
36
|
+
* limit as `1e21` did not state `1e21`.
|
|
37
|
+
*
|
|
38
|
+
* A token glued to a word or to another number is not a number the text states:
|
|
39
|
+
* that is what keeps `3600` from stating 60, `159.9989` from stating 159.998, and
|
|
40
|
+
* the `60` of `1e60` from being a number at all. The sign belongs to the token, so
|
|
41
|
+
* `-60` states -60 and not 60, while `+60` states 60. `Number` reads the rest:
|
|
42
|
+
* `1599.80` is 1599.8 and `10.0` is 10. */
|
|
43
|
+
function statedNumbers(prText) {
|
|
44
|
+
const stated = [];
|
|
45
|
+
for (const match of prText.matchAll(/[+-]?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?/g)) {
|
|
46
|
+
const token = match[0];
|
|
47
|
+
const start = match.index ?? 0;
|
|
48
|
+
const end = start + token.length;
|
|
49
|
+
const before = prText[start - 1] ?? "";
|
|
50
|
+
const after = prText[end] ?? "";
|
|
51
|
+
if (/[\dA-Za-z.]/.test(before))
|
|
52
|
+
continue;
|
|
53
|
+
if (/[\dA-Za-z]/.test(after))
|
|
54
|
+
continue;
|
|
55
|
+
// A dot that carries more digits makes this the first part of something
|
|
56
|
+
// longer — a version, a list — rather than a number of its own.
|
|
57
|
+
if (after === "." && /\d/.test(prText[end + 1] ?? ""))
|
|
58
|
+
continue;
|
|
59
|
+
const parsed = Number(token);
|
|
60
|
+
if (Number.isFinite(parsed))
|
|
61
|
+
stated.push(parsed);
|
|
62
|
+
}
|
|
63
|
+
return stated;
|
|
64
|
+
}
|
|
65
|
+
/** Whether the pull request writes this word or phrase, on its own rather than
|
|
66
|
+
* inside a longer word. A plain substring let `active` read itself out of
|
|
67
|
+
* `inactive`. The boundary is a letter or a digit, so a phrase that ends in
|
|
68
|
+
* punctuation still matches the sentence that carries it. */
|
|
69
|
+
function statesTheString(prText, value) {
|
|
70
|
+
const folded = value.replace(/\s+/g, " ").trim().toLowerCase();
|
|
71
|
+
if (!folded)
|
|
72
|
+
return false;
|
|
73
|
+
return new RegExp(`(?<![\\p{L}\\p{N}])${escapeRegExp(folded)}(?![\\p{L}\\p{N}])`, "u").test(prText);
|
|
74
|
+
}
|
|
75
|
+
/** Whether the pull request writes this value, anywhere in the title or the
|
|
76
|
+
* description. A string or a literal — `true`, `false`, `null` — has to appear as
|
|
77
|
+
* a word of its own; a number is compared against the numbers the text writes,
|
|
78
|
+
* whatever spelling either side uses.
|
|
79
|
+
*
|
|
80
|
+
* TRUE for a value that is not there to state: a case whose expectation is only a
|
|
81
|
+
* `derived` rule declares no value, and the rule is checked through the change's
|
|
82
|
+
* quote instead. */
|
|
83
|
+
function statesTheValue(prText, value) {
|
|
84
|
+
if (typeof value === "string")
|
|
85
|
+
return statesTheString(prText, value);
|
|
86
|
+
// Three values with no spelling of their own. An empty spelling list used to read
|
|
87
|
+
// as "stated", so a case could claim `true` from a pull request that writes the
|
|
88
|
+
// opposite.
|
|
89
|
+
if (value === true || value === false || value === null)
|
|
90
|
+
return statesTheString(prText, String(value));
|
|
91
|
+
// Nothing to state: `undefined` is the derived-only case, and a non-finite
|
|
92
|
+
// number has no spelling a pull request could carry.
|
|
93
|
+
if (typeof value !== "number" || !Number.isFinite(value))
|
|
94
|
+
return true;
|
|
95
|
+
return statedNumbers(prText).includes(value);
|
|
96
|
+
}
|
|
97
|
+
const text = (value) => (typeof value === "string" ? value.trim() : "");
|
|
98
|
+
/** Verifier 10. A case that states what the response must carry says where that
|
|
99
|
+
* value came from, and the value did not come from the code under test.
|
|
100
|
+
*
|
|
101
|
+
* This is the check for the failure that costs the most and looks like success: a
|
|
102
|
+
* test whose expected value was copied out of the running app passes on the broken
|
|
103
|
+
* code and fails the moment the code is fixed, so it defends the defect instead of
|
|
104
|
+
* catching it. The plan is the only place the value can be decided, because by
|
|
105
|
+
* generation time the response is the only thing to hand.
|
|
106
|
+
*
|
|
107
|
+
* Silent on a case with no `expectedValue`: asserting the shape of a value nobody
|
|
108
|
+
* states is a legitimate position, and the objection would fire on every
|
|
109
|
+
* rejection case in every plan. */
|
|
110
|
+
export const expectedValueSourced = {
|
|
111
|
+
name: "expectedValueSourced",
|
|
112
|
+
run(registration, ctx) {
|
|
113
|
+
const objections = [];
|
|
114
|
+
// The plan comes off disk, so both arrays hold their declared type only on the
|
|
115
|
+
// validated path. A throw here reaches the agent as `:crashed` and hides
|
|
116
|
+
// whatever real objection this pass would have raised.
|
|
117
|
+
// A `convention:` is trusted BECAUSE the pull request did not change it. Comparing
|
|
118
|
+
// against the run's own changed files is the only way to hold it to that.
|
|
119
|
+
const changed = changedPathSet(allChangedFiles(ctx), true);
|
|
120
|
+
const readable = typeof ctx?.citedFileExists === "function" ? ctx.citedFileExists : undefined;
|
|
121
|
+
const prText = searchable(ctx?.pullRequest ?? { title: "", description: "" });
|
|
122
|
+
const raised = new Set();
|
|
123
|
+
const changes = Array.isArray(registration.changes) ? registration.changes : [];
|
|
124
|
+
for (const change of changes) {
|
|
125
|
+
const changeId = String(change?.id ?? "?");
|
|
126
|
+
const cases = Array.isArray(change?.cases) ? change.cases : [];
|
|
127
|
+
cases.forEach((entry, index) => {
|
|
128
|
+
// `undefined` is "this case states no expected value"; `null` is a stated
|
|
129
|
+
// expectation that the field comes back null, which needs a source like
|
|
130
|
+
// any other value.
|
|
131
|
+
// A case states an expectation either as a value or as the rule behind one,
|
|
132
|
+
// and BOTH are read from somewhere: crediting `derived` without reading its
|
|
133
|
+
// `expectedFrom` let a rule "read" off the running app satisfy the check
|
|
134
|
+
// this verifier exists to be.
|
|
135
|
+
if (!entry || (entry.expectedValue === undefined && text(entry.derived).length === 0))
|
|
136
|
+
return;
|
|
137
|
+
const label = caseLabel(change, entry);
|
|
138
|
+
const where = `${label} (source "${String(entry.expectedFrom ?? "")}")`;
|
|
139
|
+
const source = readSource(entry.expectedFrom);
|
|
140
|
+
// One objection per CASE. A single objection listing three cases took one
|
|
141
|
+
// answer that addressed two of them and closed, and the id an answer is
|
|
142
|
+
// carried forward by must therefore name the case as well as the change.
|
|
143
|
+
const param = String(entry?.param ?? "").trim() || `case-${index + 1}`;
|
|
144
|
+
// The CASE, not its parameter. Two bound cases on one parameter each invent
|
|
145
|
+
// their own value, and keying on the parameter published one of them and
|
|
146
|
+
// carried its answer to the other. The rendering is the case's content, so
|
|
147
|
+
// it is the same string whatever order the cases arrive in — the identity
|
|
148
|
+
// `coverage:cases` already uses.
|
|
149
|
+
const caseId = renderCase({ param, value: entry.value, absent: entry.absent, expect: String(entry.expect ?? "") });
|
|
150
|
+
const raise = (kind, objection, evidence) => {
|
|
151
|
+
if (raised.has(`${kind}:${changeId}:${caseId}`))
|
|
152
|
+
return;
|
|
153
|
+
raised.add(`${kind}:${changeId}:${caseId}`);
|
|
154
|
+
objections.push({
|
|
155
|
+
objectionId: `expectedValueSourced:${kind}:${changeId}:${caseId}`,
|
|
156
|
+
verifier: "expectedValueSourced",
|
|
157
|
+
message: objection.message,
|
|
158
|
+
evidence,
|
|
159
|
+
suggestion: objection.suggestion,
|
|
160
|
+
});
|
|
161
|
+
};
|
|
162
|
+
if (source.kind === "none") {
|
|
163
|
+
raise("unsourced", CONTRACT.objections.missingSource, `expected value with no usable \`expectedFrom\`: ${where}`);
|
|
164
|
+
}
|
|
165
|
+
else if (source.kind === "code") {
|
|
166
|
+
raise("code", CONTRACT.objections.sourcedFromCode,
|
|
167
|
+
// A derived-only case pins no value, so quoting one read `expected
|
|
168
|
+
// undefined` — a sentence naming nothing the agent can act on.
|
|
169
|
+
`expected value read from the code under test: ${label} (${entry.expectedValue === undefined
|
|
170
|
+
? `derived as "${text(entry.derived)}"`
|
|
171
|
+
: `expected ${JSON.stringify(entry.expectedValue)}`})`);
|
|
172
|
+
}
|
|
173
|
+
else if (source.kind === "unresolvable") {
|
|
174
|
+
// A path no repository file can answer, so the citation opens nothing.
|
|
175
|
+
raise("unreadable", CONTRACT.objections.unreadableSource, `source file not in the checkout: ${where}`);
|
|
176
|
+
}
|
|
177
|
+
else if (source.kind === "pullRequest") {
|
|
178
|
+
// Two readings, and the quote settles which one applies. A rule the pull
|
|
179
|
+
// request states decides values it never writes — "over 60 is rejected"
|
|
180
|
+
// states no 61, and demanding the value itself objected to five correct
|
|
181
|
+
// cases in run 34414643135 — so a case whose source states a rule puts
|
|
182
|
+
// that rule in `derived` and declares no number.
|
|
183
|
+
const quote = String(change?.quote ?? "").trim();
|
|
184
|
+
if (quote.length === 0) {
|
|
185
|
+
raise("noQuote", CONTRACT.objections.quoteMissing, `${label} reads its value from the pull request; change ${changeId} states no \`quote\``);
|
|
186
|
+
}
|
|
187
|
+
else if (!appearsIn(prText, quote)) {
|
|
188
|
+
raise("noQuote", CONTRACT.objections.quoteMissing, `${label} reads its value from the pull request; change ${changeId} quotes \`${quote}\`, which is in neither the title nor the description`);
|
|
189
|
+
}
|
|
190
|
+
else if (!statesTheValue(prText, entry.expectedValue)) {
|
|
191
|
+
// `derived` is the case saying it pins NO number, so a case that still
|
|
192
|
+
// carries one is not excused by it: the number is what has to be stated.
|
|
193
|
+
raise("valueNotQuoted", CONTRACT.objections.valueNotQuoted, `${label} reads ${JSON.stringify(entry.expectedValue)} from the pull request, which writes no such value${text(entry.derived) ? " — and `derived` does not stand in for a value the case still declares" : ""}`);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
else if (!source.file) {
|
|
197
|
+
// A kind that names no file and is not the pull request has nothing to check.
|
|
198
|
+
}
|
|
199
|
+
else if (source.kind === "convention" && changed.has(source.file)) {
|
|
200
|
+
// Asked BEFORE readability: a convention cited in a file the pull request
|
|
201
|
+
// DELETED is not a spelling problem, it is the rule this check exists for.
|
|
202
|
+
raise("changedConvention", CONTRACT.objections.changedConvention, `convention cited in a file this pull request changed: ${where}`);
|
|
203
|
+
}
|
|
204
|
+
else if (readable && !readable(source.file)) {
|
|
205
|
+
// A named source nothing opens is a source in name only.
|
|
206
|
+
raise("unreadable", CONTRACT.objections.unreadableSource, `source file not in the checkout: ${where}`);
|
|
207
|
+
}
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
// A test that expects to fail asserts what the code does not return today, so a
|
|
211
|
+
// source outside the code has to state that value, and a case has to carry it.
|
|
212
|
+
const plannedTests = Array.isArray(registration?.plannedTests) ? registration.plannedTests : [];
|
|
213
|
+
for (const plannedTest of plannedTests) {
|
|
214
|
+
if (plannedTest?.declarations?.expected?.outcome !== "fail")
|
|
215
|
+
continue;
|
|
216
|
+
// A test that calls nothing sends no case, so it has no case to state a value
|
|
217
|
+
// on. That is the plan-only lane, whose ui tests declare a page and no steps.
|
|
218
|
+
if (!Array.isArray(plannedTest?.scenario?.steps) || plannedTest.scenario.steps.length === 0)
|
|
219
|
+
continue;
|
|
220
|
+
const citedIds = (Array.isArray(plannedTest?.declarations?.changes) ? plannedTest.declarations.changes : [])
|
|
221
|
+
.map((changeId) => (typeof changeId === "string" ? changeId.trim() : ""))
|
|
222
|
+
.filter((changeId) => changeId.length > 0);
|
|
223
|
+
// An id no change carries is the coverage check's finding, not this one.
|
|
224
|
+
const cited = changes.filter((change) => citedIds.includes(String(change?.id ?? "").trim()));
|
|
225
|
+
if (cited.length === 0)
|
|
226
|
+
continue;
|
|
227
|
+
// `derived` counts: it states the rule the value follows from, which is a
|
|
228
|
+
// stated value in every sense except that no number is written down.
|
|
229
|
+
const statesAValue = cited.some((change) => (Array.isArray(change?.cases) ? change.cases : []).some((entry) => entry && (entry.expectedValue !== undefined || text(entry.derived).length > 0)));
|
|
230
|
+
if (statesAValue)
|
|
231
|
+
continue;
|
|
232
|
+
const plannedTestId = String(plannedTest?.plannedTestId ?? "").trim();
|
|
233
|
+
objections.push({
|
|
234
|
+
objectionId: `expectedValueSourced:noValue:${plannedTestId || "plan"}`,
|
|
235
|
+
verifier: "expectedValueSourced",
|
|
236
|
+
...(plannedTestId ? { plannedTestId } : {}),
|
|
237
|
+
message: CONTRACT.objections.failWithoutValue.message,
|
|
238
|
+
evidence: `planned test ${plannedTestId || "(unnamed)"} expects to fail and cites ${cited
|
|
239
|
+
.map((change) => String(change?.id ?? "?"))
|
|
240
|
+
.join(", ")}; no case on those changes states an \`expectedValue\` or a \`derived\` rule`,
|
|
241
|
+
suggestion: CONTRACT.objections.failWithoutValue.suggestion,
|
|
242
|
+
});
|
|
243
|
+
}
|
|
244
|
+
return objections;
|
|
245
|
+
},
|
|
246
|
+
};
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { Objection } from "../types.js";
|
|
2
|
+
import { Plan } from "../registerPlan.js";
|
|
3
|
+
/** One `issuesFound` entry, as much of it as these checks read. Every field is
|
|
4
|
+
* optional: the entries come off a submitted report, and only the schema-validated
|
|
5
|
+
* path guarantees a shape. */
|
|
6
|
+
export interface ReportedIssueForTraceability {
|
|
7
|
+
description?: string;
|
|
8
|
+
category?: string;
|
|
9
|
+
sourceFile?: string;
|
|
10
|
+
plannedTestId?: string;
|
|
11
|
+
defectId?: string;
|
|
12
|
+
}
|
|
13
|
+
/** One delivered test, as much of it as these checks read. */
|
|
14
|
+
export interface DeliveredForTraceability {
|
|
15
|
+
plannedTestId?: string;
|
|
16
|
+
}
|
|
17
|
+
/** Which lane the report comes from. Passed by the caller, not read off the
|
|
18
|
+
* process, so the check stays a pure function of its inputs. */
|
|
19
|
+
export interface TraceabilityLane {
|
|
20
|
+
/** The plan-only lane delivers nothing: `newTestsCreated` DECLARES the plan.
|
|
21
|
+
* A planned test is as far as a trace can go there. */
|
|
22
|
+
planOnly: boolean;
|
|
23
|
+
}
|
|
24
|
+
/** Report-time. Every `category: bug` issue names the test that proves it, by
|
|
25
|
+
* `plannedTestId` or through a `defectId`, or it draws an objection the agent
|
|
26
|
+
* answers. Run 34283539284 reported six bugs with no test behind any of them and
|
|
27
|
+
* nothing said so; the one test near the headline bug asserted a value that
|
|
28
|
+
* passes against the broken output.
|
|
29
|
+
*
|
|
30
|
+
* A test proves a bug when the run DELIVERED it and the plan declares it as one
|
|
31
|
+
* that expects to fail. Both halves matter. A planned test the run never wrote is
|
|
32
|
+
* a promise, not proof, and the plan-time objections already hold the agent to
|
|
33
|
+
* those — except in the plan-only lane, which delivers nothing by design, so
|
|
34
|
+
* there a planned test is as far as a trace can go. A test the plan declares
|
|
35
|
+
* green is the run-34283539284 case itself: it passes against the broken output,
|
|
36
|
+
* so it says nothing about the bug. The plan is the only place a delivered test's
|
|
37
|
+
* expected outcome is written down, so a delivered test that joins to no planned
|
|
38
|
+
* test carries no such declaration and does not count either.
|
|
39
|
+
*
|
|
40
|
+
* A `defectId` counts the same way, and only for an id the plan's own `defects`
|
|
41
|
+
* list declares: `declarations.defects` is free text until it is joined to that
|
|
42
|
+
* list, so an unjoined id would let a report invent a defect and trace every bug
|
|
43
|
+
* through it.
|
|
44
|
+
*
|
|
45
|
+
* A declared `defectId` also counts when the agent answered `defects:untested:<id>`
|
|
46
|
+
* at plan time (or closed it with a blocker). The plan-time verifier asked why no
|
|
47
|
+
* test proves that defect and the agent said; asking again here is the same
|
|
48
|
+
* question twice, and the answer is already published with the plan. */
|
|
49
|
+
export declare function checkIssueTraceability(plan: Plan, delivered: DeliveredForTraceability[], issues: ReportedIssueForTraceability[], lane?: TraceabilityLane): Objection[];
|
|
50
|
+
/** Report-time. Every defect the plan declared is an `issuesFound` entry that
|
|
51
|
+
* names it by `defectId`. The review found it; the report must show it. */
|
|
52
|
+
export declare function checkDefectsReported(plan: Plan, issues: ReportedIssueForTraceability[]): Objection[];
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
const NORMAL_LANE = { planOnly: false };
|
|
2
|
+
/** An id as the plan spells it: trimmed, and exact otherwise. */
|
|
3
|
+
function idKey(raw) {
|
|
4
|
+
return typeof raw === "string" ? raw.trim() : "";
|
|
5
|
+
}
|
|
6
|
+
/** The plan's defects that carry an id. Read defensively: the plan comes off disk,
|
|
7
|
+
* and a plan stored before this field existed has none. */
|
|
8
|
+
function declaredDefects(plan) {
|
|
9
|
+
const raw = plan?.defects;
|
|
10
|
+
if (!Array.isArray(raw))
|
|
11
|
+
return [];
|
|
12
|
+
return raw.filter((defect) => idKey(defect?.id).length > 0);
|
|
13
|
+
}
|
|
14
|
+
/** The defect ids one planned test cites. */
|
|
15
|
+
function citedDefects(plannedTest) {
|
|
16
|
+
const declared = plannedTest?.declarations?.defects;
|
|
17
|
+
return Array.isArray(declared) ? declared.map(idKey).filter(Boolean) : [];
|
|
18
|
+
}
|
|
19
|
+
/** Whether the plan declares this test as one that expects to fail. Undeclared
|
|
20
|
+
* reads as not failing, exactly as the plan-time `defects` verifier reads it: a
|
|
21
|
+
* test that says nothing about its outcome proves nothing about a bug. */
|
|
22
|
+
function expectsToFail(plannedTest) {
|
|
23
|
+
return plannedTest?.declarations?.expected?.outcome === "fail";
|
|
24
|
+
}
|
|
25
|
+
/** How many characters of an issue's description the objection quotes. */
|
|
26
|
+
const QUOTE_LENGTH = 80;
|
|
27
|
+
function quote(description) {
|
|
28
|
+
const text = String(description ?? "").trim().replace(/\s+/g, " ");
|
|
29
|
+
return text.length > QUOTE_LENGTH ? `${text.slice(0, QUOTE_LENGTH)}…` : text;
|
|
30
|
+
}
|
|
31
|
+
/** The plan-time `defects:untested:<id>` objection, as the plan-time verifier ids it. */
|
|
32
|
+
const UNTESTED_PREFIX = "defects:untested:";
|
|
33
|
+
/** Defect ids whose `defects:untested:<id>` objection the agent closed when it
|
|
34
|
+
* registered the plan, by an answer or by a blocker. Both buckets: a blocker
|
|
35
|
+
* close is stored apart from an answer and is a close all the same. Read
|
|
36
|
+
* defensively: the plan comes off disk. */
|
|
37
|
+
function answeredUntested(plan) {
|
|
38
|
+
const closed = [
|
|
39
|
+
...(Array.isArray(plan?.answeredObjections) ? plan.answeredObjections : []),
|
|
40
|
+
...(Array.isArray(plan?.unverifiedCloses) ? plan.unverifiedCloses : []),
|
|
41
|
+
];
|
|
42
|
+
const ids = new Set();
|
|
43
|
+
for (const entry of closed) {
|
|
44
|
+
const objectionId = idKey(entry?.objection?.objectionId);
|
|
45
|
+
if (!objectionId.startsWith(UNTESTED_PREFIX))
|
|
46
|
+
continue;
|
|
47
|
+
if (idKey(entry?.answer).length === 0)
|
|
48
|
+
continue;
|
|
49
|
+
const id = idKey(objectionId.slice(UNTESTED_PREFIX.length));
|
|
50
|
+
if (id)
|
|
51
|
+
ids.add(id);
|
|
52
|
+
}
|
|
53
|
+
return ids;
|
|
54
|
+
}
|
|
55
|
+
/** Where the defect sits, as the objection quotes it. */
|
|
56
|
+
function locate(defect) {
|
|
57
|
+
const file = String(defect?.file ?? "").trim() || "no file";
|
|
58
|
+
return typeof defect?.line === "number" ? `${file}:${defect.line}` : file;
|
|
59
|
+
}
|
|
60
|
+
/** Report-time. Every `category: bug` issue names the test that proves it, by
|
|
61
|
+
* `plannedTestId` or through a `defectId`, or it draws an objection the agent
|
|
62
|
+
* answers. Run 34283539284 reported six bugs with no test behind any of them and
|
|
63
|
+
* nothing said so; the one test near the headline bug asserted a value that
|
|
64
|
+
* passes against the broken output.
|
|
65
|
+
*
|
|
66
|
+
* A test proves a bug when the run DELIVERED it and the plan declares it as one
|
|
67
|
+
* that expects to fail. Both halves matter. A planned test the run never wrote is
|
|
68
|
+
* a promise, not proof, and the plan-time objections already hold the agent to
|
|
69
|
+
* those — except in the plan-only lane, which delivers nothing by design, so
|
|
70
|
+
* there a planned test is as far as a trace can go. A test the plan declares
|
|
71
|
+
* green is the run-34283539284 case itself: it passes against the broken output,
|
|
72
|
+
* so it says nothing about the bug. The plan is the only place a delivered test's
|
|
73
|
+
* expected outcome is written down, so a delivered test that joins to no planned
|
|
74
|
+
* test carries no such declaration and does not count either.
|
|
75
|
+
*
|
|
76
|
+
* A `defectId` counts the same way, and only for an id the plan's own `defects`
|
|
77
|
+
* list declares: `declarations.defects` is free text until it is joined to that
|
|
78
|
+
* list, so an unjoined id would let a report invent a defect and trace every bug
|
|
79
|
+
* through it.
|
|
80
|
+
*
|
|
81
|
+
* A declared `defectId` also counts when the agent answered `defects:untested:<id>`
|
|
82
|
+
* at plan time (or closed it with a blocker). The plan-time verifier asked why no
|
|
83
|
+
* test proves that defect and the agent said; asking again here is the same
|
|
84
|
+
* question twice, and the answer is already published with the plan. */
|
|
85
|
+
export function checkIssueTraceability(plan, delivered, issues, lane = NORMAL_LANE) {
|
|
86
|
+
const shipped = new Set(delivered.map((test) => idKey(test?.plannedTestId)).filter(Boolean));
|
|
87
|
+
const plannedTests = Array.isArray(plan?.plannedTests) ? plan.plannedTests : [];
|
|
88
|
+
// The join that gives a delivered test its expected outcome. First entry wins:
|
|
89
|
+
// a repeated id is the plan's own duplicate, and the checks that own it are the
|
|
90
|
+
// plan-time ones.
|
|
91
|
+
const plannedById = new Map();
|
|
92
|
+
for (const plannedTest of plannedTests) {
|
|
93
|
+
const id = idKey(plannedTest?.plannedTestId);
|
|
94
|
+
if (id && !plannedById.has(id))
|
|
95
|
+
plannedById.set(id, plannedTest);
|
|
96
|
+
}
|
|
97
|
+
// In the plan-only lane a planned test stands where a delivered one would.
|
|
98
|
+
const inHand = (plannedTestId) => shipped.has(plannedTestId) || (lane.planOnly && plannedById.has(plannedTestId));
|
|
99
|
+
const proves = (plannedTestId) => inHand(plannedTestId) && expectsToFail(plannedById.get(plannedTestId));
|
|
100
|
+
/** Why this id proves nothing, in the words the objection uses. */
|
|
101
|
+
const gap = (plannedTestId) => {
|
|
102
|
+
if (!inHand(plannedTestId)) {
|
|
103
|
+
return plannedById.has(plannedTestId)
|
|
104
|
+
? `plannedTestId "${plannedTestId}" names a planned test the run did not deliver`
|
|
105
|
+
: `plannedTestId "${plannedTestId}" names no delivered or planned test`;
|
|
106
|
+
}
|
|
107
|
+
const word = lane.planOnly ? "planned" : "delivered";
|
|
108
|
+
return plannedById.has(plannedTestId)
|
|
109
|
+
? `plannedTestId "${plannedTestId}" names a ${word} test the plan declares as expecting to pass, and a test that passes today proves nothing about a bug`
|
|
110
|
+
: `plannedTestId "${plannedTestId}" names a ${word} test that no planned test declares, so nothing says it expects to fail`;
|
|
111
|
+
};
|
|
112
|
+
const known = new Set(declaredDefects(plan).map((defect) => idKey(defect.id)));
|
|
113
|
+
// Declared defect id -> the tests that prove it: in hand, and red by declaration.
|
|
114
|
+
const provenBy = new Map();
|
|
115
|
+
// Declared defect id -> the tests that cite it and prove nothing. Evidence only.
|
|
116
|
+
const citedWeakly = new Map();
|
|
117
|
+
for (const plannedTest of plannedTests) {
|
|
118
|
+
const id = idKey(plannedTest?.plannedTestId);
|
|
119
|
+
if (!id || !inHand(id))
|
|
120
|
+
continue;
|
|
121
|
+
const bucket = proves(id) ? provenBy : citedWeakly;
|
|
122
|
+
for (const defectId of citedDefects(plannedTest)) {
|
|
123
|
+
if (!known.has(defectId))
|
|
124
|
+
continue;
|
|
125
|
+
bucket.set(defectId, [...(bucket.get(defectId) ?? []), id]);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
const answered = answeredUntested(plan);
|
|
129
|
+
const objections = [];
|
|
130
|
+
issues.forEach((issue, index) => {
|
|
131
|
+
if (String(issue?.category ?? "").trim().toLowerCase() !== "bug")
|
|
132
|
+
return;
|
|
133
|
+
const plannedTestId = idKey(issue?.plannedTestId);
|
|
134
|
+
if (plannedTestId && proves(plannedTestId))
|
|
135
|
+
return;
|
|
136
|
+
const defectId = idKey(issue?.defectId);
|
|
137
|
+
const declaredDefect = defectId.length > 0 && known.has(defectId);
|
|
138
|
+
if (declaredDefect && ((provenBy.get(defectId) ?? []).length > 0 || answered.has(defectId)))
|
|
139
|
+
return;
|
|
140
|
+
const named = [];
|
|
141
|
+
if (plannedTestId)
|
|
142
|
+
named.push(gap(plannedTestId));
|
|
143
|
+
if (defectId) {
|
|
144
|
+
const word = lane.planOnly ? "planned" : "delivered";
|
|
145
|
+
if (!declaredDefect) {
|
|
146
|
+
named.push(`defectId "${defectId}" names no defect this plan declares`);
|
|
147
|
+
}
|
|
148
|
+
else if ((citedWeakly.get(defectId) ?? []).length > 0) {
|
|
149
|
+
named.push(`defectId "${defectId}" is cited by no ${word} test that expects to fail: ${(citedWeakly.get(defectId) ?? []).join(", ")} cite it and the plan declares each as expecting to pass`);
|
|
150
|
+
}
|
|
151
|
+
else {
|
|
152
|
+
named.push(`defectId "${defectId}" is cited by no ${word} test that expects to fail and no plan-time answer closed ${UNTESTED_PREFIX}${defectId}`);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
const where = String(issue?.sourceFile ?? "").trim() || "no sourceFile";
|
|
156
|
+
objections.push({
|
|
157
|
+
// Keyed on the entry's position: an issue has no id of its own. The
|
|
158
|
+
// message NAMES the issue because the report-stage carry matches on the id
|
|
159
|
+
// AND the message: with a fixed message, an answer given for the bug at
|
|
160
|
+
// index 0 would follow index 0 to whatever bug the next call put there.
|
|
161
|
+
objectionId: `issueTraceability:${index}`,
|
|
162
|
+
verifier: "issueTraceability",
|
|
163
|
+
message: `No test proves this bug: "${quote(issue?.description)}" (${where}).`,
|
|
164
|
+
evidence: `issuesFound[${index}]: ${named.length > 0 ? named.join("; ") : "no plannedTestId and no defectId"}.`,
|
|
165
|
+
suggestion: "Set `plannedTestId` to the delivered test that proves this bug — one the plan declares with `expected.outcome: fail`, so it stays red until the bug is fixed — or set `defectId` to a declared plan defect such a test cites. If no test proves it, answer in one line why.",
|
|
166
|
+
});
|
|
167
|
+
});
|
|
168
|
+
return objections;
|
|
169
|
+
}
|
|
170
|
+
/** Report-time. Every defect the plan declared is an `issuesFound` entry that
|
|
171
|
+
* names it by `defectId`. The review found it; the report must show it. */
|
|
172
|
+
export function checkDefectsReported(plan, issues) {
|
|
173
|
+
// Only a `category: bug` entry reports a defect. The schema takes a `defectId`
|
|
174
|
+
// on a lint, type or config entry too, and the report renders those in its own
|
|
175
|
+
// Configuration Errors section — a defect parked there is one the reader never
|
|
176
|
+
// meets under Issues Found, and it would close this check all the same.
|
|
177
|
+
const reported = new Set(issues
|
|
178
|
+
.filter((issue) => String(issue?.category ?? "").trim().toLowerCase() === "bug")
|
|
179
|
+
.map((issue) => idKey(issue?.defectId))
|
|
180
|
+
.filter(Boolean));
|
|
181
|
+
const objections = [];
|
|
182
|
+
const seen = new Set();
|
|
183
|
+
for (const defect of declaredDefects(plan)) {
|
|
184
|
+
const id = idKey(defect.id);
|
|
185
|
+
if (seen.has(id) || reported.has(id))
|
|
186
|
+
continue;
|
|
187
|
+
seen.add(id);
|
|
188
|
+
objections.push({
|
|
189
|
+
objectionId: `defects:unreported:${id}`,
|
|
190
|
+
verifier: "defects",
|
|
191
|
+
message: "The plan declares this defect and no issue in the report names it.",
|
|
192
|
+
evidence: `defect ${id} (${locate(defect)}): "${String(defect.description ?? "").trim()}"; no issuesFound entry carries defectId "${id}".`,
|
|
193
|
+
suggestion: "Add an `issuesFound` entry for it with `category: bug`, `defectId` set to this id, and the `plannedTestId` of the test that proves it. If the review was wrong and there is no defect, answer in one line what you found.",
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
return objections;
|
|
197
|
+
}
|