@skyramp/mcp 0.4.0 → 0.4.1-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/build/prompts/enhance-assertions/contractProviderAssertionsPrompt.js +2 -1
  2. package/build/prompts/enhance-assertions/integrationAssertionsPrompt.js +2 -1
  3. package/build/prompts/enhance-assertions/sharedAssertionRules.d.ts +1 -1
  4. package/build/prompts/enhance-assertions/sharedAssertionRules.js +57 -22
  5. package/build/prompts/enhance-assertions/uiAssertionsPrompt.js +17 -9
  6. package/build/prompts/test-recommendation/diffExecutionPlan.js +0 -2
  7. package/build/prompts/test-recommendation/test-recommendation-prompt.js +11 -6
  8. package/build/prompts/testbot/testbot-prompts.js +19 -12
  9. package/build/recommendation/answers.d.ts +11 -7
  10. package/build/recommendation/answers.js +14 -10
  11. package/build/recommendation/pullRequestText.d.ts +18 -0
  12. package/build/recommendation/pullRequestText.js +31 -0
  13. package/build/recommendation/registerPlan.d.ts +9 -1
  14. package/build/recommendation/registerPlan.js +8 -1
  15. package/build/recommendation/runVerifiers.js +6 -0
  16. package/build/recommendation/types.d.ts +58 -0
  17. package/build/recommendation/verifierContracts.d.ts +108 -13
  18. package/build/recommendation/verifierContracts.js +147 -29
  19. package/build/recommendation/verifiers/coverage.d.ts +10 -0
  20. package/build/recommendation/verifiers/coverage.js +144 -22
  21. package/build/recommendation/verifiers/defects.d.ts +9 -0
  22. package/build/recommendation/verifiers/defects.js +117 -0
  23. package/build/recommendation/verifiers/deliveredMatchesPlan.d.ts +22 -0
  24. package/build/recommendation/verifiers/deliveredMatchesPlan.js +43 -0
  25. package/build/recommendation/verifiers/existingCoverage.js +53 -0
  26. package/build/recommendation/verifiers/expectedValueSourced.d.ts +14 -0
  27. package/build/recommendation/verifiers/expectedValueSourced.js +246 -0
  28. package/build/recommendation/verifiers/issueTraceability.d.ts +52 -0
  29. package/build/recommendation/verifiers/issueTraceability.js +197 -0
  30. package/build/recommendation/verifiers/requirementSourced.d.ts +2 -0
  31. package/build/recommendation/verifiers/requirementSourced.js +168 -0
  32. package/build/services/TestGenerationService.js +3 -1
  33. package/build/tools/code-refactor/codeReuseTool.js +1 -1
  34. package/build/tools/code-refactor/reuse-outcome.d.ts +1 -1
  35. package/build/tools/code-refactor/reuse-state.d.ts +85 -7
  36. package/build/tools/code-refactor/reuse-state.js +239 -34
  37. package/build/tools/code-refactor/utils-verify-gates.d.ts +5 -0
  38. package/build/tools/code-refactor/utils-verify-gates.js +103 -11
  39. package/build/tools/generate-tests/generateBatchScenarioRestTool.js +2 -1
  40. package/build/tools/submitReportTool.js +276 -39
  41. package/build/tools/test-management/actionsTool.js +5 -0
  42. package/build/tools/test-management/analyzeChangesTool.d.ts +53 -0
  43. package/build/tools/test-management/analyzeChangesTool.js +55 -2
  44. package/build/tools/test-management/registerTestPlanTool.d.ts +32 -17
  45. package/build/tools/test-management/registerTestPlanTool.js +105 -16
  46. package/build/types/ReuseOutcome.d.ts +73 -7
  47. package/build/types/TestAnalysis.d.ts +6 -0
  48. package/build/types/TestbotReport.d.ts +22 -4
  49. package/build/utils/AnalysisStateManager.d.ts +7 -1
  50. package/build/utils/AnalysisStateManager.js +5 -1
  51. package/build/utils/assertion-verify/api-shared-lints.js +118 -11
  52. package/build/utils/assertion-verify/format.js +2 -2
  53. package/build/utils/assertion-verify/helper-imports.d.ts +7 -0
  54. package/build/utils/assertion-verify/helper-imports.js +119 -27
  55. package/build/utils/assertion-verify/lint-types.d.ts +31 -2
  56. package/build/utils/assertion-verify/lint-types.js +66 -0
  57. package/build/utils/assertion-verify/metrics.d.ts +13 -0
  58. package/build/utils/assertion-verify/metrics.js +16 -0
  59. package/build/utils/assertion-verify/verify.d.ts +11 -6
  60. package/build/utils/assertion-verify/verify.js +56 -15
  61. package/build/utils/canonicalJson.d.ts +11 -0
  62. package/build/utils/canonicalJson.js +17 -0
  63. package/build/utils/utils-verify/action-key.d.ts +27 -0
  64. package/build/utils/utils-verify/action-key.js +292 -0
  65. package/build/utils/utils-verify/allow.d.ts +8 -1
  66. package/build/utils/utils-verify/allow.js +14 -1
  67. package/build/utils/utils-verify/call-sites.d.ts +76 -8
  68. package/build/utils/utils-verify/call-sites.js +256 -70
  69. package/build/utils/utils-verify/language-spec.d.ts +3 -2
  70. package/build/utils/utils-verify/parse.d.ts +22 -3
  71. package/build/utils/utils-verify/parse.js +123 -52
  72. package/build/utils/utils-verify/verify.d.ts +33 -3
  73. package/build/utils/utils-verify/verify.js +126 -12
  74. package/build/utils/workspaceAuth.d.ts +59 -19
  75. package/build/utils/workspaceAuth.js +228 -31
  76. package/package.json +1 -1
  77. package/plugin/prompts/generate-tests/execution-plan.md +3 -3
  78. package/plugin/prompts/generate-tests/generation.md +1 -0
  79. package/plugin/prompts/plan-tests.md +41 -15
  80. package/plugin/prompts/testbot-task1.md +3 -9
  81. package/build/prompts/testbot/planDeclarations.d.ts +0 -6
  82. package/build/prompts/testbot/planDeclarations.js +0 -9
  83. package/plugin/prompts/declaring-a-plan.md +0 -20
@@ -4,8 +4,30 @@ import { Plan } from "../registerPlan.js";
4
4
  export interface DeliveredTest {
5
5
  plannedTestId: string;
6
6
  }
7
+ /** One maintenance row that LEFT COVERAGE BEHIND, as the report publishes it. The
8
+ * caller filters by action: a row this check is handed is one whose edit a later
9
+ * run can still rely on, so a no-op assessment and a deleted test never reach it.
10
+ * Both fields are absolute paths. */
11
+ export interface MaintainedTest {
12
+ testFilePath?: string;
13
+ /** Where the edit landed when that is not the spec — a page object it delegates
14
+ * its selectors to. Counted, so a spec maintained through its POM still joins. */
15
+ pomFile?: string;
16
+ }
7
17
  /** Verifier 5, post-execution half. Both directions on one pass: a report entry
8
18
  * that was in no plan, and a planned planned test that never shipped. The two lists
9
19
  * join on `plannedTestId` exactly — never a name, an endpoint or a similarity
10
20
  * score. Objections go to the report: the tests already exist. */
11
21
  export declare function checkDeliveredMatchesPlan(plan: Plan, delivered: DeliveredTest[]): Objection[];
22
+ /** The maintenance mirror of "planned but not delivered". A `maintains` entry is
23
+ * what makes its changes count as covered, and until this ran nothing downstream
24
+ * checked the file was edited at all: the plan claimed it, `coverage` credited it,
25
+ * and the report published it, three records that never met.
26
+ *
27
+ * Matched with `testFileMatches`, which keeps the directories: the plan names a file
28
+ * repository-relative and a row names it absolutely, so the row's path has to END at
29
+ * the claimed one on a segment boundary. Basename alone would let an edit to
30
+ * `tests/admin/navbar.spec.ts` answer for `tests/store/navbar.spec.ts`. Answerable,
31
+ * unlike `coverage:change:` — an edit that landed somewhere the rows do not name is
32
+ * a real case, and the answer says where. */
33
+ export declare function checkMaintenanceDelivered(plan: Plan, maintained: MaintainedTest[]): Objection[];
@@ -1,3 +1,4 @@
1
+ import { testFileMatches } from "../../utils/utils.js";
1
2
  /** Verifier 5, post-execution half. Both directions on one pass: a report entry
2
3
  * that was in no plan, and a planned planned test that never shipped. The two lists
3
4
  * join on `plannedTestId` exactly — never a name, an endpoint or a similarity
@@ -31,3 +32,45 @@ export function checkDeliveredMatchesPlan(plan, delivered) {
31
32
  }
32
33
  return objections;
33
34
  }
35
+ /** The maintenance mirror of "planned but not delivered". A `maintains` entry is
36
+ * what makes its changes count as covered, and until this ran nothing downstream
37
+ * checked the file was edited at all: the plan claimed it, `coverage` credited it,
38
+ * and the report published it, three records that never met.
39
+ *
40
+ * Matched with `testFileMatches`, which keeps the directories: the plan names a file
41
+ * repository-relative and a row names it absolutely, so the row's path has to END at
42
+ * the claimed one on a segment boundary. Basename alone would let an edit to
43
+ * `tests/admin/navbar.spec.ts` answer for `tests/store/navbar.spec.ts`. Answerable,
44
+ * unlike `coverage:change:` — an edit that landed somewhere the rows do not name is
45
+ * a real case, and the answer says where. */
46
+ export function checkMaintenanceDelivered(plan, maintained) {
47
+ const claimed = Array.isArray(plan?.maintains) ? plan.maintains : [];
48
+ if (claimed.length === 0)
49
+ return [];
50
+ const text = (value) => (typeof value === "string" ? value.trim() : "");
51
+ const editedPaths = [];
52
+ for (const row of Array.isArray(maintained) ? maintained : []) {
53
+ for (const full of [text(row?.testFilePath), text(row?.pomFile)])
54
+ if (full)
55
+ editedPaths.push(full);
56
+ }
57
+ const objections = [];
58
+ const seen = new Set();
59
+ for (const entry of claimed) {
60
+ const file = text(entry?.file);
61
+ if (!file || editedPaths.some((full) => testFileMatches(full, file)))
62
+ continue;
63
+ const objectionId = `deliveredMatchesPlan:maintains:${file}`;
64
+ if (seen.has(objectionId))
65
+ continue;
66
+ seen.add(objectionId);
67
+ objections.push({
68
+ objectionId,
69
+ verifier: "deliveredMatchesPlan",
70
+ message: "The plan says this existing test covers a change, and no maintenance row says it was edited.",
71
+ evidence: `plan maintains ${file}; the maintenance rows that changed a file name ${editedPaths.length > 0 ? editedPaths.join(", ") : "no file"}`,
72
+ suggestion: "The maintenance rows come from the triage, not from this call, so this is not a field to fill in here: make the edit on the file you claimed and let the maintenance step record it, or drop the entry from `maintains` and register the plan again — coverage counted the change because the plan said you were editing that test. If the test already covered the change and needed no edit, or the edit landed somewhere the rows do not name, say which as the answer.",
73
+ });
74
+ }
75
+ return objections;
76
+ }
@@ -42,6 +42,59 @@ export const existingCoverage = {
42
42
  suggestion: EXISTING_COVERAGE_CONTRACT.objections.citedTestMissing.suggestion,
43
43
  });
44
44
  }
45
+ // `maintains` is what makes a change covered by an edited test count in
46
+ // `coverage`, so the same file check applies to it, plus the change ids it
47
+ // cites: an id no `changes` entry declares would credit coverage for nothing.
48
+ const maintains = Array.isArray(registration?.maintains) ? registration.maintains : [];
49
+ const declaredChanges = new Set((Array.isArray(registration?.changes) ? registration.changes : [])
50
+ .map((change) => String(change?.id ?? "").trim())
51
+ .filter(Boolean));
52
+ // Gathered per file BEFORE any objection is pushed: the schema allows two
53
+ // entries naming the same file, the objection id is keyed on the file so the
54
+ // agent answers a file once, and keying alone dropped the second entry's ids.
55
+ const missingByFile = new Map();
56
+ const unknownByFile = new Map();
57
+ for (const entry of maintains) {
58
+ const file = String(entry?.file ?? "").trim();
59
+ const key = file || "unnamed";
60
+ const missing = !file
61
+ ? "an entry names no file"
62
+ : escapesRepo(file)
63
+ ? `${file} (points outside the repository)`
64
+ : !ctx.citedFileExists(file)
65
+ ? `${file} (not found)`
66
+ : "";
67
+ if (missing && !missingByFile.has(key))
68
+ missingByFile.set(key, missing);
69
+ const ids = Array.isArray(entry?.changes) ? entry.changes : [];
70
+ for (const raw of ids) {
71
+ const id = String(raw ?? "").trim();
72
+ if (!id || declaredChanges.has(id))
73
+ continue;
74
+ const unknown = unknownByFile.get(key) ?? [];
75
+ if (!unknown.includes(id))
76
+ unknown.push(id);
77
+ unknownByFile.set(key, unknown);
78
+ }
79
+ }
80
+ for (const [key, missing] of missingByFile) {
81
+ objections.push({
82
+ objectionId: `existingCoverage:maintains:${key}`,
83
+ verifier: "existingCoverage",
84
+ message: EXISTING_COVERAGE_CONTRACT.objections.maintainedTestMissing.message,
85
+ evidence: `maintained test not in the checkout: ${missing}`,
86
+ suggestion: EXISTING_COVERAGE_CONTRACT.objections.maintainedTestMissing.suggestion,
87
+ });
88
+ }
89
+ for (const [key, unknown] of unknownByFile) {
90
+ objections.push({
91
+ objectionId: `existingCoverage:maintains:unknownChange:${key}`,
92
+ verifier: "existingCoverage",
93
+ message: EXISTING_COVERAGE_CONTRACT.objections.maintainedUnknownChange.message,
94
+ evidence: `${key === "unnamed" ? "an unnamed entry" : key} cites ${unknown.map((id) => `"${id}"`).join(", ")}; this plan declares ${[...declaredChanges].map((id) => `"${id}"`).join(", ") || "no changes"}`,
95
+ suggestion: EXISTING_COVERAGE_CONTRACT.objections.maintainedUnknownChange.suggestion,
96
+ });
97
+ }
45
98
  return objections;
46
99
  },
47
100
  };
@@ -0,0 +1,14 @@
1
+ import { Verifier } from "../types.js";
2
+ /** Verifier 10. A case that states what the response must carry says where that
3
+ * value came from, and the value did not come from the code under test.
4
+ *
5
+ * This is the check for the failure that costs the most and looks like success: a
6
+ * test whose expected value was copied out of the running app passes on the broken
7
+ * code and fails the moment the code is fixed, so it defends the defect instead of
8
+ * catching it. The plan is the only place the value can be decided, because by
9
+ * generation time the response is the only thing to hand.
10
+ *
11
+ * Silent on a case with no `expectedValue`: asserting the shape of a value nobody
12
+ * states is a legitimate position, and the objection would fire on every
13
+ * rejection case in every plan. */
14
+ export declare const expectedValueSourced: Verifier;
@@ -0,0 +1,246 @@
1
+ import { allChangedFiles } from "../types.js";
2
+ import { changedPathSet, normalizeCitedPath } from "./citedPath.js";
3
+ import { EXPECTED_VALUE_SOURCED_CONTRACT as CONTRACT } from "../verifierContracts.js";
4
+ import { appearsIn, searchable } from "../pullRequestText.js";
5
+ import { escapeRegExp } from "../../utils/regex.js";
6
+ import { renderCase } from "./coverage.js";
7
+ function readSource(raw) {
8
+ const source = typeof raw === "string" ? raw.trim() : "";
9
+ if (source.length === 0)
10
+ return { kind: "none", file: "" };
11
+ if (/^code$/i.test(source))
12
+ return { kind: "code", file: "" };
13
+ if (/^(pr-title|pr-description)$/i.test(source))
14
+ return { kind: "pullRequest", file: "" };
15
+ const named = /^(spec|convention):(.*)$/i.exec(source);
16
+ if (!named)
17
+ return { kind: "none", file: "" };
18
+ const kind = named[1].toLowerCase();
19
+ const cited = named[2].split(/[#\u00a7]/)[0].replace(/(?::\d+)+(?:-\d+)?$/, "").trim();
20
+ // `spec:` with nothing after it names no file, so there is nothing to check and
21
+ // nothing to trust either. A path that IS written and still normalises away is a
22
+ // named source, and it is named wrongly.
23
+ if (cited.length === 0)
24
+ return { kind: "none", file: "" };
25
+ const file = normalizeCitedPath(cited);
26
+ return file ? { kind, file } : { kind: "unresolvable", file: "" };
27
+ }
28
+ /** One case's address, for the evidence line. A case has no id of its own, so it
29
+ * is named the way the agent wrote it. */
30
+ function caseLabel(change, entry) {
31
+ return `${String(change?.id ?? "?")}:${String(entry?.param ?? "?")}`;
32
+ }
33
+ /** Every number the pull request writes, as a number. One value has many
34
+ * spellings — `1e21`, `1e+21`, `1E21`, the digits in full — and matching the
35
+ * value's OWN spelling refused every other one, so a pull request stating the
36
+ * limit as `1e21` did not state `1e21`.
37
+ *
38
+ * A token glued to a word or to another number is not a number the text states:
39
+ * that is what keeps `3600` from stating 60, `159.9989` from stating 159.998, and
40
+ * the `60` of `1e60` from being a number at all. The sign belongs to the token, so
41
+ * `-60` states -60 and not 60, while `+60` states 60. `Number` reads the rest:
42
+ * `1599.80` is 1599.8 and `10.0` is 10. */
43
+ function statedNumbers(prText) {
44
+ const stated = [];
45
+ for (const match of prText.matchAll(/[+-]?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?/g)) {
46
+ const token = match[0];
47
+ const start = match.index ?? 0;
48
+ const end = start + token.length;
49
+ const before = prText[start - 1] ?? "";
50
+ const after = prText[end] ?? "";
51
+ if (/[\dA-Za-z.]/.test(before))
52
+ continue;
53
+ if (/[\dA-Za-z]/.test(after))
54
+ continue;
55
+ // A dot that carries more digits makes this the first part of something
56
+ // longer — a version, a list — rather than a number of its own.
57
+ if (after === "." && /\d/.test(prText[end + 1] ?? ""))
58
+ continue;
59
+ const parsed = Number(token);
60
+ if (Number.isFinite(parsed))
61
+ stated.push(parsed);
62
+ }
63
+ return stated;
64
+ }
65
+ /** Whether the pull request writes this word or phrase, on its own rather than
66
+ * inside a longer word. A plain substring let `active` read itself out of
67
+ * `inactive`. The boundary is a letter or a digit, so a phrase that ends in
68
+ * punctuation still matches the sentence that carries it. */
69
+ function statesTheString(prText, value) {
70
+ const folded = value.replace(/\s+/g, " ").trim().toLowerCase();
71
+ if (!folded)
72
+ return false;
73
+ return new RegExp(`(?<![\\p{L}\\p{N}])${escapeRegExp(folded)}(?![\\p{L}\\p{N}])`, "u").test(prText);
74
+ }
75
+ /** Whether the pull request writes this value, anywhere in the title or the
76
+ * description. A string or a literal — `true`, `false`, `null` — has to appear as
77
+ * a word of its own; a number is compared against the numbers the text writes,
78
+ * whatever spelling either side uses.
79
+ *
80
+ * TRUE for a value that is not there to state: a case whose expectation is only a
81
+ * `derived` rule declares no value, and the rule is checked through the change's
82
+ * quote instead. */
83
+ function statesTheValue(prText, value) {
84
+ if (typeof value === "string")
85
+ return statesTheString(prText, value);
86
+ // Three values with no spelling of their own. An empty spelling list used to read
87
+ // as "stated", so a case could claim `true` from a pull request that writes the
88
+ // opposite.
89
+ if (value === true || value === false || value === null)
90
+ return statesTheString(prText, String(value));
91
+ // Nothing to state: `undefined` is the derived-only case, and a non-finite
92
+ // number has no spelling a pull request could carry.
93
+ if (typeof value !== "number" || !Number.isFinite(value))
94
+ return true;
95
+ return statedNumbers(prText).includes(value);
96
+ }
97
+ const text = (value) => (typeof value === "string" ? value.trim() : "");
98
+ /** Verifier 10. A case that states what the response must carry says where that
99
+ * value came from, and the value did not come from the code under test.
100
+ *
101
+ * This is the check for the failure that costs the most and looks like success: a
102
+ * test whose expected value was copied out of the running app passes on the broken
103
+ * code and fails the moment the code is fixed, so it defends the defect instead of
104
+ * catching it. The plan is the only place the value can be decided, because by
105
+ * generation time the response is the only thing to hand.
106
+ *
107
+ * Silent on a case with no `expectedValue`: asserting the shape of a value nobody
108
+ * states is a legitimate position, and the objection would fire on every
109
+ * rejection case in every plan. */
110
+ export const expectedValueSourced = {
111
+ name: "expectedValueSourced",
112
+ run(registration, ctx) {
113
+ const objections = [];
114
+ // The plan comes off disk, so both arrays hold their declared type only on the
115
+ // validated path. A throw here reaches the agent as `:crashed` and hides
116
+ // whatever real objection this pass would have raised.
117
+ // A `convention:` is trusted BECAUSE the pull request did not change it. Comparing
118
+ // against the run's own changed files is the only way to hold it to that.
119
+ const changed = changedPathSet(allChangedFiles(ctx), true);
120
+ const readable = typeof ctx?.citedFileExists === "function" ? ctx.citedFileExists : undefined;
121
+ const prText = searchable(ctx?.pullRequest ?? { title: "", description: "" });
122
+ const raised = new Set();
123
+ const changes = Array.isArray(registration.changes) ? registration.changes : [];
124
+ for (const change of changes) {
125
+ const changeId = String(change?.id ?? "?");
126
+ const cases = Array.isArray(change?.cases) ? change.cases : [];
127
+ cases.forEach((entry, index) => {
128
+ // `undefined` is "this case states no expected value"; `null` is a stated
129
+ // expectation that the field comes back null, which needs a source like
130
+ // any other value.
131
+ // A case states an expectation either as a value or as the rule behind one,
132
+ // and BOTH are read from somewhere: crediting `derived` without reading its
133
+ // `expectedFrom` let a rule "read" off the running app satisfy the check
134
+ // this verifier exists to be.
135
+ if (!entry || (entry.expectedValue === undefined && text(entry.derived).length === 0))
136
+ return;
137
+ const label = caseLabel(change, entry);
138
+ const where = `${label} (source "${String(entry.expectedFrom ?? "")}")`;
139
+ const source = readSource(entry.expectedFrom);
140
+ // One objection per CASE. A single objection listing three cases took one
141
+ // answer that addressed two of them and closed, and the id an answer is
142
+ // carried forward by must therefore name the case as well as the change.
143
+ const param = String(entry?.param ?? "").trim() || `case-${index + 1}`;
144
+ // The CASE, not its parameter. Two bound cases on one parameter each invent
145
+ // their own value, and keying on the parameter published one of them and
146
+ // carried its answer to the other. The rendering is the case's content, so
147
+ // it is the same string whatever order the cases arrive in — the identity
148
+ // `coverage:cases` already uses.
149
+ const caseId = renderCase({ param, value: entry.value, absent: entry.absent, expect: String(entry.expect ?? "") });
150
+ const raise = (kind, objection, evidence) => {
151
+ if (raised.has(`${kind}:${changeId}:${caseId}`))
152
+ return;
153
+ raised.add(`${kind}:${changeId}:${caseId}`);
154
+ objections.push({
155
+ objectionId: `expectedValueSourced:${kind}:${changeId}:${caseId}`,
156
+ verifier: "expectedValueSourced",
157
+ message: objection.message,
158
+ evidence,
159
+ suggestion: objection.suggestion,
160
+ });
161
+ };
162
+ if (source.kind === "none") {
163
+ raise("unsourced", CONTRACT.objections.missingSource, `expected value with no usable \`expectedFrom\`: ${where}`);
164
+ }
165
+ else if (source.kind === "code") {
166
+ raise("code", CONTRACT.objections.sourcedFromCode,
167
+ // A derived-only case pins no value, so quoting one read `expected
168
+ // undefined` — a sentence naming nothing the agent can act on.
169
+ `expected value read from the code under test: ${label} (${entry.expectedValue === undefined
170
+ ? `derived as "${text(entry.derived)}"`
171
+ : `expected ${JSON.stringify(entry.expectedValue)}`})`);
172
+ }
173
+ else if (source.kind === "unresolvable") {
174
+ // A path no repository file can answer, so the citation opens nothing.
175
+ raise("unreadable", CONTRACT.objections.unreadableSource, `source file not in the checkout: ${where}`);
176
+ }
177
+ else if (source.kind === "pullRequest") {
178
+ // Two readings, and the quote settles which one applies. A rule the pull
179
+ // request states decides values it never writes — "over 60 is rejected"
180
+ // states no 61, and demanding the value itself objected to five correct
181
+ // cases in run 34414643135 — so a case whose source states a rule puts
182
+ // that rule in `derived` and declares no number.
183
+ const quote = String(change?.quote ?? "").trim();
184
+ if (quote.length === 0) {
185
+ raise("noQuote", CONTRACT.objections.quoteMissing, `${label} reads its value from the pull request; change ${changeId} states no \`quote\``);
186
+ }
187
+ else if (!appearsIn(prText, quote)) {
188
+ raise("noQuote", CONTRACT.objections.quoteMissing, `${label} reads its value from the pull request; change ${changeId} quotes \`${quote}\`, which is in neither the title nor the description`);
189
+ }
190
+ else if (!statesTheValue(prText, entry.expectedValue)) {
191
+ // `derived` is the case saying it pins NO number, so a case that still
192
+ // carries one is not excused by it: the number is what has to be stated.
193
+ raise("valueNotQuoted", CONTRACT.objections.valueNotQuoted, `${label} reads ${JSON.stringify(entry.expectedValue)} from the pull request, which writes no such value${text(entry.derived) ? " — and `derived` does not stand in for a value the case still declares" : ""}`);
194
+ }
195
+ }
196
+ else if (!source.file) {
197
+ // A kind that names no file and is not the pull request has nothing to check.
198
+ }
199
+ else if (source.kind === "convention" && changed.has(source.file)) {
200
+ // Asked BEFORE readability: a convention cited in a file the pull request
201
+ // DELETED is not a spelling problem, it is the rule this check exists for.
202
+ raise("changedConvention", CONTRACT.objections.changedConvention, `convention cited in a file this pull request changed: ${where}`);
203
+ }
204
+ else if (readable && !readable(source.file)) {
205
+ // A named source nothing opens is a source in name only.
206
+ raise("unreadable", CONTRACT.objections.unreadableSource, `source file not in the checkout: ${where}`);
207
+ }
208
+ });
209
+ }
210
+ // A test that expects to fail asserts what the code does not return today, so a
211
+ // source outside the code has to state that value, and a case has to carry it.
212
+ const plannedTests = Array.isArray(registration?.plannedTests) ? registration.plannedTests : [];
213
+ for (const plannedTest of plannedTests) {
214
+ if (plannedTest?.declarations?.expected?.outcome !== "fail")
215
+ continue;
216
+ // A test that calls nothing sends no case, so it has no case to state a value
217
+ // on. That is the plan-only lane, whose ui tests declare a page and no steps.
218
+ if (!Array.isArray(plannedTest?.scenario?.steps) || plannedTest.scenario.steps.length === 0)
219
+ continue;
220
+ const citedIds = (Array.isArray(plannedTest?.declarations?.changes) ? plannedTest.declarations.changes : [])
221
+ .map((changeId) => (typeof changeId === "string" ? changeId.trim() : ""))
222
+ .filter((changeId) => changeId.length > 0);
223
+ // An id no change carries is the coverage check's finding, not this one.
224
+ const cited = changes.filter((change) => citedIds.includes(String(change?.id ?? "").trim()));
225
+ if (cited.length === 0)
226
+ continue;
227
+ // `derived` counts: it states the rule the value follows from, which is a
228
+ // stated value in every sense except that no number is written down.
229
+ const statesAValue = cited.some((change) => (Array.isArray(change?.cases) ? change.cases : []).some((entry) => entry && (entry.expectedValue !== undefined || text(entry.derived).length > 0)));
230
+ if (statesAValue)
231
+ continue;
232
+ const plannedTestId = String(plannedTest?.plannedTestId ?? "").trim();
233
+ objections.push({
234
+ objectionId: `expectedValueSourced:noValue:${plannedTestId || "plan"}`,
235
+ verifier: "expectedValueSourced",
236
+ ...(plannedTestId ? { plannedTestId } : {}),
237
+ message: CONTRACT.objections.failWithoutValue.message,
238
+ evidence: `planned test ${plannedTestId || "(unnamed)"} expects to fail and cites ${cited
239
+ .map((change) => String(change?.id ?? "?"))
240
+ .join(", ")}; no case on those changes states an \`expectedValue\` or a \`derived\` rule`,
241
+ suggestion: CONTRACT.objections.failWithoutValue.suggestion,
242
+ });
243
+ }
244
+ return objections;
245
+ },
246
+ };
@@ -0,0 +1,52 @@
1
+ import { Objection } from "../types.js";
2
+ import { Plan } from "../registerPlan.js";
3
+ /** One `issuesFound` entry, as much of it as these checks read. Every field is
4
+ * optional: the entries come off a submitted report, and only the schema-validated
5
+ * path guarantees a shape. */
6
+ export interface ReportedIssueForTraceability {
7
+ description?: string;
8
+ category?: string;
9
+ sourceFile?: string;
10
+ plannedTestId?: string;
11
+ defectId?: string;
12
+ }
13
+ /** One delivered test, as much of it as these checks read. */
14
+ export interface DeliveredForTraceability {
15
+ plannedTestId?: string;
16
+ }
17
+ /** Which lane the report comes from. Passed by the caller, not read off the
18
+ * process, so the check stays a pure function of its inputs. */
19
+ export interface TraceabilityLane {
20
+ /** The plan-only lane delivers nothing: `newTestsCreated` DECLARES the plan.
21
+ * A planned test is as far as a trace can go there. */
22
+ planOnly: boolean;
23
+ }
24
+ /** Report-time. Every `category: bug` issue names the test that proves it, by
25
+ * `plannedTestId` or through a `defectId`, or it draws an objection the agent
26
+ * answers. Run 34283539284 reported six bugs with no test behind any of them and
27
+ * nothing said so; the one test near the headline bug asserted a value that
28
+ * passes against the broken output.
29
+ *
30
+ * A test proves a bug when the run DELIVERED it and the plan declares it as one
31
+ * that expects to fail. Both halves matter. A planned test the run never wrote is
32
+ * a promise, not proof, and the plan-time objections already hold the agent to
33
+ * those — except in the plan-only lane, which delivers nothing by design, so
34
+ * there a planned test is as far as a trace can go. A test the plan declares
35
+ * green is the run-34283539284 case itself: it passes against the broken output,
36
+ * so it says nothing about the bug. The plan is the only place a delivered test's
37
+ * expected outcome is written down, so a delivered test that joins to no planned
38
+ * test carries no such declaration and does not count either.
39
+ *
40
+ * A `defectId` counts the same way, and only for an id the plan's own `defects`
41
+ * list declares: `declarations.defects` is free text until it is joined to that
42
+ * list, so an unjoined id would let a report invent a defect and trace every bug
43
+ * through it.
44
+ *
45
+ * A declared `defectId` also counts when the agent answered `defects:untested:<id>`
46
+ * at plan time (or closed it with a blocker). The plan-time verifier asked why no
47
+ * test proves that defect and the agent said; asking again here is the same
48
+ * question twice, and the answer is already published with the plan. */
49
+ export declare function checkIssueTraceability(plan: Plan, delivered: DeliveredForTraceability[], issues: ReportedIssueForTraceability[], lane?: TraceabilityLane): Objection[];
50
+ /** Report-time. Every defect the plan declared is an `issuesFound` entry that
51
+ * names it by `defectId`. The review found it; the report must show it. */
52
+ export declare function checkDefectsReported(plan: Plan, issues: ReportedIssueForTraceability[]): Objection[];
@@ -0,0 +1,197 @@
1
+ const NORMAL_LANE = { planOnly: false };
2
+ /** An id as the plan spells it: trimmed, and exact otherwise. */
3
+ function idKey(raw) {
4
+ return typeof raw === "string" ? raw.trim() : "";
5
+ }
6
+ /** The plan's defects that carry an id. Read defensively: the plan comes off disk,
7
+ * and a plan stored before this field existed has none. */
8
+ function declaredDefects(plan) {
9
+ const raw = plan?.defects;
10
+ if (!Array.isArray(raw))
11
+ return [];
12
+ return raw.filter((defect) => idKey(defect?.id).length > 0);
13
+ }
14
+ /** The defect ids one planned test cites. */
15
+ function citedDefects(plannedTest) {
16
+ const declared = plannedTest?.declarations?.defects;
17
+ return Array.isArray(declared) ? declared.map(idKey).filter(Boolean) : [];
18
+ }
19
+ /** Whether the plan declares this test as one that expects to fail. Undeclared
20
+ * reads as not failing, exactly as the plan-time `defects` verifier reads it: a
21
+ * test that says nothing about its outcome proves nothing about a bug. */
22
+ function expectsToFail(plannedTest) {
23
+ return plannedTest?.declarations?.expected?.outcome === "fail";
24
+ }
25
+ /** How many characters of an issue's description the objection quotes. */
26
+ const QUOTE_LENGTH = 80;
27
+ function quote(description) {
28
+ const text = String(description ?? "").trim().replace(/\s+/g, " ");
29
+ return text.length > QUOTE_LENGTH ? `${text.slice(0, QUOTE_LENGTH)}…` : text;
30
+ }
31
+ /** The plan-time `defects:untested:<id>` objection, as the plan-time verifier ids it. */
32
+ const UNTESTED_PREFIX = "defects:untested:";
33
+ /** Defect ids whose `defects:untested:<id>` objection the agent closed when it
34
+ * registered the plan, by an answer or by a blocker. Both buckets: a blocker
35
+ * close is stored apart from an answer and is a close all the same. Read
36
+ * defensively: the plan comes off disk. */
37
+ function answeredUntested(plan) {
38
+ const closed = [
39
+ ...(Array.isArray(plan?.answeredObjections) ? plan.answeredObjections : []),
40
+ ...(Array.isArray(plan?.unverifiedCloses) ? plan.unverifiedCloses : []),
41
+ ];
42
+ const ids = new Set();
43
+ for (const entry of closed) {
44
+ const objectionId = idKey(entry?.objection?.objectionId);
45
+ if (!objectionId.startsWith(UNTESTED_PREFIX))
46
+ continue;
47
+ if (idKey(entry?.answer).length === 0)
48
+ continue;
49
+ const id = idKey(objectionId.slice(UNTESTED_PREFIX.length));
50
+ if (id)
51
+ ids.add(id);
52
+ }
53
+ return ids;
54
+ }
55
+ /** Where the defect sits, as the objection quotes it. */
56
+ function locate(defect) {
57
+ const file = String(defect?.file ?? "").trim() || "no file";
58
+ return typeof defect?.line === "number" ? `${file}:${defect.line}` : file;
59
+ }
60
+ /** Report-time. Every `category: bug` issue names the test that proves it, by
61
+ * `plannedTestId` or through a `defectId`, or it draws an objection the agent
62
+ * answers. Run 34283539284 reported six bugs with no test behind any of them and
63
+ * nothing said so; the one test near the headline bug asserted a value that
64
+ * passes against the broken output.
65
+ *
66
+ * A test proves a bug when the run DELIVERED it and the plan declares it as one
67
+ * that expects to fail. Both halves matter. A planned test the run never wrote is
68
+ * a promise, not proof, and the plan-time objections already hold the agent to
69
+ * those — except in the plan-only lane, which delivers nothing by design, so
70
+ * there a planned test is as far as a trace can go. A test the plan declares
71
+ * green is the run-34283539284 case itself: it passes against the broken output,
72
+ * so it says nothing about the bug. The plan is the only place a delivered test's
73
+ * expected outcome is written down, so a delivered test that joins to no planned
74
+ * test carries no such declaration and does not count either.
75
+ *
76
+ * A `defectId` counts the same way, and only for an id the plan's own `defects`
77
+ * list declares: `declarations.defects` is free text until it is joined to that
78
+ * list, so an unjoined id would let a report invent a defect and trace every bug
79
+ * through it.
80
+ *
81
+ * A declared `defectId` also counts when the agent answered `defects:untested:<id>`
82
+ * at plan time (or closed it with a blocker). The plan-time verifier asked why no
83
+ * test proves that defect and the agent said; asking again here is the same
84
+ * question twice, and the answer is already published with the plan. */
85
+ export function checkIssueTraceability(plan, delivered, issues, lane = NORMAL_LANE) {
86
+ const shipped = new Set(delivered.map((test) => idKey(test?.plannedTestId)).filter(Boolean));
87
+ const plannedTests = Array.isArray(plan?.plannedTests) ? plan.plannedTests : [];
88
+ // The join that gives a delivered test its expected outcome. First entry wins:
89
+ // a repeated id is the plan's own duplicate, and the checks that own it are the
90
+ // plan-time ones.
91
+ const plannedById = new Map();
92
+ for (const plannedTest of plannedTests) {
93
+ const id = idKey(plannedTest?.plannedTestId);
94
+ if (id && !plannedById.has(id))
95
+ plannedById.set(id, plannedTest);
96
+ }
97
+ // In the plan-only lane a planned test stands where a delivered one would.
98
+ const inHand = (plannedTestId) => shipped.has(plannedTestId) || (lane.planOnly && plannedById.has(plannedTestId));
99
+ const proves = (plannedTestId) => inHand(plannedTestId) && expectsToFail(plannedById.get(plannedTestId));
100
+ /** Why this id proves nothing, in the words the objection uses. */
101
+ const gap = (plannedTestId) => {
102
+ if (!inHand(plannedTestId)) {
103
+ return plannedById.has(plannedTestId)
104
+ ? `plannedTestId "${plannedTestId}" names a planned test the run did not deliver`
105
+ : `plannedTestId "${plannedTestId}" names no delivered or planned test`;
106
+ }
107
+ const word = lane.planOnly ? "planned" : "delivered";
108
+ return plannedById.has(plannedTestId)
109
+ ? `plannedTestId "${plannedTestId}" names a ${word} test the plan declares as expecting to pass, and a test that passes today proves nothing about a bug`
110
+ : `plannedTestId "${plannedTestId}" names a ${word} test that no planned test declares, so nothing says it expects to fail`;
111
+ };
112
+ const known = new Set(declaredDefects(plan).map((defect) => idKey(defect.id)));
113
+ // Declared defect id -> the tests that prove it: in hand, and red by declaration.
114
+ const provenBy = new Map();
115
+ // Declared defect id -> the tests that cite it and prove nothing. Evidence only.
116
+ const citedWeakly = new Map();
117
+ for (const plannedTest of plannedTests) {
118
+ const id = idKey(plannedTest?.plannedTestId);
119
+ if (!id || !inHand(id))
120
+ continue;
121
+ const bucket = proves(id) ? provenBy : citedWeakly;
122
+ for (const defectId of citedDefects(plannedTest)) {
123
+ if (!known.has(defectId))
124
+ continue;
125
+ bucket.set(defectId, [...(bucket.get(defectId) ?? []), id]);
126
+ }
127
+ }
128
+ const answered = answeredUntested(plan);
129
+ const objections = [];
130
+ issues.forEach((issue, index) => {
131
+ if (String(issue?.category ?? "").trim().toLowerCase() !== "bug")
132
+ return;
133
+ const plannedTestId = idKey(issue?.plannedTestId);
134
+ if (plannedTestId && proves(plannedTestId))
135
+ return;
136
+ const defectId = idKey(issue?.defectId);
137
+ const declaredDefect = defectId.length > 0 && known.has(defectId);
138
+ if (declaredDefect && ((provenBy.get(defectId) ?? []).length > 0 || answered.has(defectId)))
139
+ return;
140
+ const named = [];
141
+ if (plannedTestId)
142
+ named.push(gap(plannedTestId));
143
+ if (defectId) {
144
+ const word = lane.planOnly ? "planned" : "delivered";
145
+ if (!declaredDefect) {
146
+ named.push(`defectId "${defectId}" names no defect this plan declares`);
147
+ }
148
+ else if ((citedWeakly.get(defectId) ?? []).length > 0) {
149
+ named.push(`defectId "${defectId}" is cited by no ${word} test that expects to fail: ${(citedWeakly.get(defectId) ?? []).join(", ")} cite it and the plan declares each as expecting to pass`);
150
+ }
151
+ else {
152
+ named.push(`defectId "${defectId}" is cited by no ${word} test that expects to fail and no plan-time answer closed ${UNTESTED_PREFIX}${defectId}`);
153
+ }
154
+ }
155
+ const where = String(issue?.sourceFile ?? "").trim() || "no sourceFile";
156
+ objections.push({
157
+ // Keyed on the entry's position: an issue has no id of its own. The
158
+ // message NAMES the issue because the report-stage carry matches on the id
159
+ // AND the message: with a fixed message, an answer given for the bug at
160
+ // index 0 would follow index 0 to whatever bug the next call put there.
161
+ objectionId: `issueTraceability:${index}`,
162
+ verifier: "issueTraceability",
163
+ message: `No test proves this bug: "${quote(issue?.description)}" (${where}).`,
164
+ evidence: `issuesFound[${index}]: ${named.length > 0 ? named.join("; ") : "no plannedTestId and no defectId"}.`,
165
+ suggestion: "Set `plannedTestId` to the delivered test that proves this bug — one the plan declares with `expected.outcome: fail`, so it stays red until the bug is fixed — or set `defectId` to a declared plan defect such a test cites. If no test proves it, answer in one line why.",
166
+ });
167
+ });
168
+ return objections;
169
+ }
170
+ /** Report-time. Every defect the plan declared is an `issuesFound` entry that
171
+ * names it by `defectId`. The review found it; the report must show it. */
172
+ export function checkDefectsReported(plan, issues) {
173
+ // Only a `category: bug` entry reports a defect. The schema takes a `defectId`
174
+ // on a lint, type or config entry too, and the report renders those in its own
175
+ // Configuration Errors section — a defect parked there is one the reader never
176
+ // meets under Issues Found, and it would close this check all the same.
177
+ const reported = new Set(issues
178
+ .filter((issue) => String(issue?.category ?? "").trim().toLowerCase() === "bug")
179
+ .map((issue) => idKey(issue?.defectId))
180
+ .filter(Boolean));
181
+ const objections = [];
182
+ const seen = new Set();
183
+ for (const defect of declaredDefects(plan)) {
184
+ const id = idKey(defect.id);
185
+ if (seen.has(id) || reported.has(id))
186
+ continue;
187
+ seen.add(id);
188
+ objections.push({
189
+ objectionId: `defects:unreported:${id}`,
190
+ verifier: "defects",
191
+ message: "The plan declares this defect and no issue in the report names it.",
192
+ evidence: `defect ${id} (${locate(defect)}): "${String(defect.description ?? "").trim()}"; no issuesFound entry carries defectId "${id}".`,
193
+ suggestion: "Add an `issuesFound` entry for it with `category: bug`, `defectId` set to this id, and the `plannedTestId` of the test that proves it. If the review was wrong and there is no defect, answer in one line what you found.",
194
+ });
195
+ }
196
+ return objections;
197
+ }