jev-agent-tools 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. package/CHANGELOG.md +37 -1
  2. package/CONTRIBUTING.md +3 -0
  3. package/README.md +22 -14
  4. package/SECURITY.md +17 -1
  5. package/dist/adapters/ask-files.js +11 -2
  6. package/dist/adapters/ask-proof.js +63 -7
  7. package/dist/adapters/command.js +82 -29
  8. package/dist/adapters/docs.js +30 -10
  9. package/dist/adapters/evidence-context.js +119 -0
  10. package/dist/adapters/files.js +141 -16
  11. package/dist/adapters/find.js +34 -6
  12. package/dist/adapters/git-base.js +7 -1
  13. package/dist/adapters/git.js +51 -7
  14. package/dist/adapters/locate-file.js +47 -9
  15. package/dist/adapters/private-storage.js +14 -6
  16. package/dist/adapters/risk-callers.js +3 -0
  17. package/dist/adapters/shell.js +23 -7
  18. package/dist/adapters/test-inventory.js +10 -2
  19. package/dist/configuration.js +17 -7
  20. package/dist/constants.js +26 -5
  21. package/dist/core/ask-references.js +193 -109
  22. package/dist/core/asks.js +78 -7
  23. package/dist/core/locate.js +8 -8
  24. package/dist/core/output.js +17 -0
  25. package/dist/core/result-report.js +302 -0
  26. package/dist/core/secret-path.js +34 -0
  27. package/dist/core/state.js +8 -1
  28. package/dist/core/units.js +1 -1
  29. package/dist/jev/client.js +34 -12
  30. package/dist/mcp/protocol.js +50 -27
  31. package/dist/mcp/tools.js +20 -7
  32. package/dist/render.js +72 -0
  33. package/dist/report-schema.js +1356 -0
  34. package/dist/result-types.js +1 -0
  35. package/dist/texts/ask-files.js +3 -1
  36. package/dist/texts/ask.js +3 -1
  37. package/dist/texts/check-diff.js +7 -4
  38. package/dist/texts/find.js +7 -2
  39. package/dist/texts/guide.js +3 -16
  40. package/dist/texts/instructions.js +72 -0
  41. package/dist/texts/locate.js +7 -2
  42. package/dist/texts/select-tests.js +3 -1
  43. package/dist/tools/ask-files.js +248 -15
  44. package/dist/tools/ask.js +523 -62
  45. package/dist/tools/check-diff.js +222 -30
  46. package/dist/tools/docs-check.js +122 -13
  47. package/dist/tools/find.js +320 -27
  48. package/dist/tools/locate.js +317 -18
  49. package/dist/tools/review-report.js +230 -0
  50. package/dist/tools/select-tests.js +273 -19
  51. package/dist/tools/spec-check.js +119 -22
  52. package/docs/adr/0001-strict-typescript-pure-core-offline-tests.md +3 -3
  53. package/docs/agent-instructions.md +59 -30
  54. package/docs/design.md +13 -1
  55. package/docs/mcp.md +8 -6
  56. package/docs/tools/jev_ask.md +8 -5
  57. package/docs/tools/jev_ask_files.md +2 -1
  58. package/docs/tools/jev_check_diff.md +4 -1
  59. package/docs/tools/jev_find_files.md +2 -1
  60. package/docs/tools/jev_locate_in_file.md +5 -0
  61. package/docs/tools/jev_select_tests.md +4 -1
  62. package/package.json +1 -1
  63. package/rules/jev-ask.md +22 -1
  64. package/server.json +2 -2
  65. package/src/adapters/ask-files.ts +11 -3
  66. package/src/adapters/ask-proof.ts +69 -11
  67. package/src/adapters/command.ts +96 -33
  68. package/src/adapters/docs.ts +33 -14
  69. package/src/adapters/evidence-context.ts +169 -0
  70. package/src/adapters/files.ts +146 -16
  71. package/src/adapters/find.ts +37 -7
  72. package/src/adapters/git-base.ts +7 -1
  73. package/src/adapters/git.ts +61 -8
  74. package/src/adapters/locate-file.ts +51 -9
  75. package/src/adapters/private-storage.ts +17 -5
  76. package/src/adapters/risk-callers.ts +3 -0
  77. package/src/adapters/shell.ts +23 -7
  78. package/src/adapters/test-inventory.ts +12 -4
  79. package/src/configuration.ts +16 -2
  80. package/src/constants.ts +26 -5
  81. package/src/core/ask-references.ts +262 -146
  82. package/src/core/asks.ts +79 -7
  83. package/src/core/import-boundaries.ts +8 -3
  84. package/src/core/locate.ts +8 -5
  85. package/src/core/output.ts +34 -0
  86. package/src/core/result-report.ts +410 -0
  87. package/src/core/secret-path.ts +37 -0
  88. package/src/core/state.ts +8 -1
  89. package/src/core/units.ts +3 -2
  90. package/src/index.ts +3 -0
  91. package/src/jev/client.ts +54 -16
  92. package/src/jev/types.ts +18 -3
  93. package/src/mcp/protocol.ts +91 -41
  94. package/src/mcp/tools.ts +26 -13
  95. package/src/render.ts +109 -0
  96. package/src/report-schema.ts +1380 -0
  97. package/src/result-types.ts +234 -0
  98. package/src/result.ts +4 -1
  99. package/src/runtime.ts +6 -0
  100. package/src/texts/ask-files.ts +4 -1
  101. package/src/texts/ask.ts +8 -1
  102. package/src/texts/check-diff.ts +7 -4
  103. package/src/texts/find.ts +8 -2
  104. package/src/texts/guide.ts +8 -16
  105. package/src/texts/instructions.ts +98 -0
  106. package/src/texts/locate.ts +8 -2
  107. package/src/texts/run-end.ts +2 -2
  108. package/src/texts/select-tests.ts +4 -1
  109. package/src/tools/ask-files.ts +309 -14
  110. package/src/tools/ask.ts +700 -77
  111. package/src/tools/check-diff.ts +331 -28
  112. package/src/tools/docs-check.ts +241 -39
  113. package/src/tools/find.ts +386 -29
  114. package/src/tools/locate.ts +384 -19
  115. package/src/tools/review-report.ts +308 -0
  116. package/src/tools/select-tests.ts +479 -21
  117. package/src/tools/spec-check.ts +193 -19
@@ -1,8 +1,12 @@
1
- import { matchesGlob, relative, resolve } from "node:path";
1
+ import { isAbsolute, matchesGlob, relative, resolve } from "node:path";
2
2
  import type { ToolDefinition } from "@earendil-works/pi-coding-agent";
3
3
  import { type Static, Type } from "@sinclair/typebox";
4
4
  import { createAnalysisContext } from "../adapters/analysis-context.ts";
5
- import { canonicalPath } from "../adapters/canonical-path.ts";
5
+ import {
6
+ type EvidenceContext,
7
+ resolveEvidenceContext,
8
+ withEvidenceContext,
9
+ } from "../adapters/evidence-context.ts";
6
10
  import { collectUnits } from "../adapters/git.ts";
7
11
  import { resolveBase } from "../adapters/git-base.ts";
8
12
  import { shareGitInventory } from "../adapters/git-inventory.ts";
@@ -26,6 +30,11 @@ import type {
26
30
  Limitation,
27
31
  } from "../core/output.ts";
28
32
  import { buildEnvelope } from "../core/output.ts";
33
+ import {
34
+ type Cause,
35
+ type ResultReportV1,
36
+ known as reportKnown,
37
+ } from "../core/result-report.ts";
29
38
  import { buildRunnerCommands } from "../core/test-commands.ts";
30
39
  import { prepareCoverageWitnesses } from "../core/test-coverage.ts";
31
40
  import type { TestEntry } from "../core/test-discovery.ts";
@@ -42,12 +51,14 @@ import {
42
51
  buildCoverageWitnessUnits,
43
52
  evaluateBatchWitnessHealth,
44
53
  } from "../presets/witnesses.ts";
45
- import { renderEnvelope } from "../render.ts";
54
+ import { renderResultReport } from "../render.ts";
46
55
  import type { ToolDependencies } from "../runtime.ts";
47
56
  import { SELECT_TESTS_DESCRIPTION } from "../texts/select-tests.ts";
57
+ import { controlsFor, ReviewReport, reportMetrics } from "./review-report.ts";
48
58
 
49
59
  export const selectTestsParameters = Type.Object(
50
60
  {
61
+ root: Type.Optional(Type.String()),
51
62
  base: Type.Optional(Type.String({ minLength: 1 })),
52
63
  paths: Type.Optional(Type.Array(Type.String({ minLength: 1 }))),
53
64
  witnesses: Type.Optional(
@@ -126,7 +137,13 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
126
137
  ctx: { cwd: string } & GuideContext,
127
138
  ) {
128
139
  const client = dependencies.client;
129
- const cwd = await canonicalPath(ctx.cwd);
140
+ const evidence = await resolveEvidenceContext(ctx.cwd, args.root, {
141
+ exec: execute,
142
+ signal,
143
+ origin: dependencies.evidenceOrigin,
144
+ });
145
+ const evidenceContext = evidence.context;
146
+ const cwd = evidence.ok ? evidence.cwd : evidenceContext.authority.path;
130
147
  const started = performance.now();
131
148
  const exec = shareGitInventory(execute);
132
149
  const totals = {
@@ -136,10 +153,17 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
136
153
  cacheRequests: 0,
137
154
  usage: { inputTokens: 0, costUsd: 0 },
138
155
  };
156
+ const selectionEvidence = {
157
+ criteria: [] as { criterion: string; matches: string[] }[],
158
+ wideningTriggers: [] as string[],
159
+ inventory: [] as string[],
160
+ };
161
+ const report = new ReviewReport();
139
162
  let costKnown = false;
163
+ let unknownCost = false;
140
164
  let sent = 0;
141
165
  let budget: BudgetRefusal | undefined;
142
- const finish = (input: Omit<EnvelopeInput, "yield">) => {
166
+ const finish = (input: Omit<EnvelopeInput, "yield">, cause?: Cause) => {
143
167
  const limitKeys = new Set<string>();
144
168
  const uniqueLimits = input.limitations?.filter((limit) => {
145
169
  const key = JSON.stringify([limit.fact, limit.next]);
@@ -152,26 +176,78 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
152
176
  limitations: uniqueLimits,
153
177
  yield: {
154
178
  ...totals,
155
- costUsd: costKnown ? totals.usage.costUsd : undefined,
179
+ costUsd:
180
+ costKnown && !unknownCost ? totals.usage.costUsd : undefined,
156
181
  elapsedMs: performance.now() - started,
157
182
  },
158
183
  });
184
+ if (input.refusal) {
185
+ report.refusal = true;
186
+ report.diagnose(cause ?? "internal_error", input.refusal);
187
+ }
188
+ if (!report.items.size && !input.refusal)
189
+ report.diagnose(
190
+ "collection_empty",
191
+ "No test decision candidates in the discovered inventory; this is not proof of no affected tests.",
192
+ undefined,
193
+ [],
194
+ false,
195
+ );
196
+ const result = report.build(
197
+ "jev_select_tests",
198
+ evidenceContext,
199
+ reportMetrics(envelope),
200
+ );
159
201
  runtime.session.record(envelope);
160
202
  runtime.guide.deliver(ctx);
161
- const details: Judgment & { limitations?: readonly Limitation[] } = {
203
+ const details: Judgment & {
204
+ result: ResultReportV1;
205
+ limitations?: readonly Limitation[];
206
+ evidenceContext: EvidenceContext;
207
+ selectionEvidence: typeof selectionEvidence;
208
+ } = {
162
209
  ok: true,
163
210
  answers: {},
164
211
  ...totals,
212
+ evidenceContext,
213
+ selectionEvidence,
214
+ result,
165
215
  limitations: uniqueLimits,
166
216
  };
167
217
  return {
168
- content: [{ type: "text" as const, text: renderEnvelope(envelope) }],
218
+ content: [
219
+ {
220
+ type: "text" as const,
221
+ text: renderResultReport(result, { details: envelope }),
222
+ },
223
+ ],
169
224
  details,
170
225
  ...hostUsage(host.isOmp, costKnown ? totals.usage : undefined),
171
226
  };
172
227
  };
228
+ if (!evidence.ok)
229
+ return finish(
230
+ { refusal: evidence.error },
231
+ evidence.cause ?? "invalid_root",
232
+ );
233
+ evidenceContext.requestedBase = args.base ?? "HEAD";
234
+ for (const path of args.paths ?? [])
235
+ if (
236
+ isAbsolute(path) ||
237
+ path.split(/[\\/]/).includes("..") ||
238
+ path.split(/[\\/]/).includes(".git")
239
+ )
240
+ return finish(
241
+ { refusal: `Path not permitted: ${path}` },
242
+ "forbidden_path",
243
+ );
173
244
  const comparison = await resolveBase(exec, cwd, args.base, signal);
174
- if (!comparison.ok) return finish({ refusal: comparison.error });
245
+ if (!comparison.ok)
246
+ return finish(
247
+ { refusal: comparison.error },
248
+ comparison.cause ?? "invalid_base",
249
+ );
250
+ evidenceContext.resolvedBase = comparison.base;
175
251
  const analysis = await createAnalysisContext();
176
252
  const [inventory, diff] = await Promise.all([
177
253
  collectTestInventory(exec, cwd, signal),
@@ -181,8 +257,16 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
181
257
  analysis.parser,
182
258
  ),
183
259
  ]);
184
- if (!inventory.ok) return finish({ refusal: inventory.error });
185
- if (!diff.ok) return finish({ refusal: diff.error });
260
+ if (!inventory.ok)
261
+ return finish(
262
+ { refusal: inventory.error },
263
+ inventory.cause ?? "file_unavailable",
264
+ );
265
+ if (!diff.ok)
266
+ return finish({ refusal: diff.error }, diff.cause ?? "git_failure");
267
+ if (evidenceContext.effectiveRoot)
268
+ evidenceContext.effectiveRoot.path = inventory.cwd;
269
+ selectionEvidence.inventory = [...inventory.paths];
186
270
  const known = new Set(inventory.paths);
187
271
  const sourceFiles = new Map<string, ImportSource>();
188
272
  const read = async (path: string): Promise<ImportSource | undefined> => {
@@ -277,6 +361,23 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
277
361
  !graph.edges.has(file.path) ||
278
362
  !/\.[cm]?[jt]sx?$|\.py$/.test(file.path),
279
363
  );
364
+ selectionEvidence.wideningTriggers = diff.files
365
+ .filter(
366
+ (file) =>
367
+ !graph.edges.has(file.path) ||
368
+ !/\.[cm]?[jt]sx?$|\.py$/.test(file.path),
369
+ )
370
+ .map((file) => file.path);
371
+ selectionEvidence.criteria = (args.paths ?? []).map((criterion) => ({
372
+ criterion,
373
+ matches: versionedEntries
374
+ .filter(
375
+ (entry) =>
376
+ matchesGlob(entry.path, criterion) ||
377
+ entry.path.startsWith(`${criterion.replace(/\/$/, "")}/`),
378
+ )
379
+ .map((entry) => entry.path),
380
+ }));
280
381
  const candidates = versionedEntries
281
382
  .filter(
282
383
  (entry) =>
@@ -308,6 +409,77 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
308
409
  outside,
309
410
  );
310
411
  });
412
+ report.inventories.push({
413
+ id: "test-candidates",
414
+ kind: "tests",
415
+ rules: [
416
+ "Tracked supported test declarations and literal project runner configuration",
417
+ "Import closure selection; conservative fallback preserves unjudged decisions",
418
+ ],
419
+ restrictions: args.paths ?? [],
420
+ discovered: reportKnown(versionedEntries.length),
421
+ considered: reportKnown(candidates.length),
422
+ scopeRestricted: Boolean(args.paths?.length),
423
+ criteria: selectionEvidence.criteria.map((criterion) => ({
424
+ criterion: criterion.criterion,
425
+ matches: reportKnown(criterion.matches.length),
426
+ outcome: criterion.matches.length ? "matched" : "no_match",
427
+ diagnosticIds: [],
428
+ })),
429
+ });
430
+ for (const candidate of candidates) {
431
+ const scenarios = candidate.entry.scenarios;
432
+ if (!scenarios.length)
433
+ report.expect(
434
+ `test:${candidate.entry.path}:unknown`,
435
+ candidate.entry.path,
436
+ "scenario",
437
+ );
438
+ for (const scenario of scenarios)
439
+ report.expect(
440
+ `test:${candidate.entry.path}:${scenario.id}`,
441
+ `${candidate.entry.path}: ${scenario.name}`,
442
+ "scenario",
443
+ `test:${candidate.entry.path}`,
444
+ );
445
+ }
446
+ for (const criterion of selectionEvidence.criteria)
447
+ if (!criterion.matches.length) {
448
+ report.diagnose(
449
+ "criteria_no_match",
450
+ `No discovered tests match criterion ${criterion.criterion}`,
451
+ criterion.criterion,
452
+ );
453
+ const excluded = inventory.limits.filter(
454
+ (limit) =>
455
+ matchesGlob(limit.path, criterion.criterion) ||
456
+ limit.path.startsWith(
457
+ `${criterion.criterion.replace(/\/$/, "")}/`,
458
+ ),
459
+ );
460
+ if (excluded.length) {
461
+ report.diagnose(
462
+ "outside_inventory",
463
+ "Requested criterion names entries excluded from the admitted inventory",
464
+ criterion.criterion,
465
+ [],
466
+ false,
467
+ excluded.map((limit) => limit.path),
468
+ );
469
+ const entry = report.inventories[0]?.criteria.find(
470
+ (entry) => entry.criterion === criterion.criterion,
471
+ );
472
+ if (entry) entry.outcome = "outside_inventory";
473
+ }
474
+ }
475
+ if (selectionEvidence.wideningTriggers.length)
476
+ report.diagnose(
477
+ "conservative_widening",
478
+ `Changed files outside supported dependency graph: ${selectionEvidence.wideningTriggers.join(", ")}`,
479
+ undefined,
480
+ [],
481
+ false,
482
+ );
311
483
  const selected: {
312
484
  entry: TestEntry;
313
485
  scenarioIds: readonly string[] | null;
@@ -373,9 +545,11 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
373
545
  questions: Record<string, Question>,
374
546
  witnessIds?: readonly string[],
375
547
  ) => {
548
+ state = withEvidenceContext(state, evidenceContext);
376
549
  if (JSON.stringify(state).length > STATE_MAX_CHARS)
377
550
  return {
378
551
  ok: false as const,
552
+ cause: "evidence_too_large" as const,
379
553
  error: `required evidence exceeds STATE_MAX_CHARS=${STATE_MAX_CHARS}`,
380
554
  };
381
555
  if (args.max_calls !== undefined && sent >= args.max_calls) {
@@ -390,6 +564,8 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
390
564
  signal,
391
565
  witnesses: witnessIds,
392
566
  ...runtime.session.requestGate(),
567
+ admissionCause: () =>
568
+ budget?.kind === "session" ? "session_budget" : "call_budget",
393
569
  beforeRequest: (questionCount) => {
394
570
  if (args.max_calls !== undefined && sent >= args.max_calls) {
395
571
  budget = {
@@ -412,6 +588,7 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
412
588
  totals.questions += result.questions ?? 0;
413
589
  totals.cacheHits += result.cacheHits ?? 0;
414
590
  totals.cacheRequests += result.cacheRequests ?? 0;
591
+ unknownCost ||= result.usage === undefined;
415
592
  if (result.usage) {
416
593
  costKnown = true;
417
594
  totals.usage.inputTokens += result.usage.inputTokens;
@@ -422,6 +599,27 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
422
599
  const pointerResults = await Promise.all(
423
600
  candidates.map(async (candidate) => {
424
601
  const { entry } = candidate;
602
+ const reportIds = [...report.items.values()]
603
+ .filter(
604
+ (item) =>
605
+ item.groupId === `test:${entry.path}` ||
606
+ item.id === `test:${entry.path}:unknown`,
607
+ )
608
+ .map((item) => item.id);
609
+ if (candidate.touched)
610
+ for (const id of reportIds) report.static(id, "touched", true);
611
+ if (
612
+ !candidate.units.length &&
613
+ !candidate.uncertain &&
614
+ !outside &&
615
+ !candidate.touched
616
+ )
617
+ for (const id of reportIds)
618
+ report.static(
619
+ id,
620
+ "static import closure cannot reach changed units",
621
+ false,
622
+ );
425
623
  if (candidate.touched)
426
624
  return {
427
625
  candidate,
@@ -440,6 +638,12 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
440
638
  if (
441
639
  units.some((unit) => unit.before === null && unit.after === null)
442
640
  ) {
641
+ report.diagnose(
642
+ "binary_or_non_utf8",
643
+ "Changed source unavailable",
644
+ entry.path,
645
+ reportIds,
646
+ );
443
647
  for (const unit of units) incompleteUnits.add(unit.id);
444
648
  unjudged.push({
445
649
  label: entry.path,
@@ -449,6 +653,13 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
449
653
  return { candidate, selected: null, touched: false };
450
654
  }
451
655
  if (!entry.scenarios.length) {
656
+ report.totalUnknown = true;
657
+ report.diagnose(
658
+ "unsupported_syntax",
659
+ "Scenario names or count unresolved",
660
+ entry.path,
661
+ reportIds,
662
+ );
452
663
  unjudged.push({
453
664
  label: entry.path,
454
665
  reason: "scenario names or count unresolved",
@@ -461,6 +672,9 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
461
672
  (item) => item.scenario.id,
462
673
  );
463
674
  for (const item of prepared.unjudged) {
675
+ report.diagnose("evidence_too_large", item.reason, entry.path, [
676
+ `test:${entry.path}:${item.scenario.id}`,
677
+ ]);
464
678
  for (const unit of units) incompleteUnits.add(unit.id);
465
679
  unjudged.push({
466
680
  label: `${entry.path}: ${item.scenario.name}`,
@@ -494,6 +708,41 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
494
708
  ]),
495
709
  );
496
710
  const result = await judge(batch.state, questions);
711
+ const batchIds = batch.scenarios.map(
712
+ (scenario) => `test:${entry.path}:${scenario.id}`,
713
+ );
714
+ if (!result.ok)
715
+ report.failure(
716
+ result,
717
+ batchIds,
718
+ budget
719
+ ? budget.kind === "session"
720
+ ? "session_budget"
721
+ : "call_budget"
722
+ : !client
723
+ ? "not_configured"
724
+ : undefined,
725
+ );
726
+ else
727
+ for (const scenario of batch.scenarios) {
728
+ const answer = result.answers[scenario.id];
729
+ const id = `test:${entry.path}:${scenario.id}`;
730
+ report.answer(id, answer, {
731
+ band: prepared.unjudged.length ? "unsure" : "verdict",
732
+ });
733
+ const item = report.items.get(id);
734
+ if (!item)
735
+ throw new Error(`Unregistered selection result ${id}`);
736
+ item.selection = {
737
+ selected:
738
+ answer?.type !== "choice" ||
739
+ 1 - (answer.probabilities.none ?? 0) >= SELECT_MIN,
740
+ reason:
741
+ answer?.type === "choice"
742
+ ? "changed-unit pointer selection threshold"
743
+ : "conservative_fallback",
744
+ };
745
+ }
497
746
  if (!result.ok) {
498
747
  fallback ||= !budget;
499
748
  for (const unit of units) incompleteUnits.add(unit.id);
@@ -587,6 +836,24 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
587
836
  preliminary.state,
588
837
  candidate.entry.scenarios,
589
838
  );
839
+ for (const unit of units)
840
+ for (const scenario of candidate.entry.scenarios)
841
+ report.expect(
842
+ `coverage:${candidate.entry.path}:${scenario.id}:${unit.id}`,
843
+ `${candidate.entry.path}: ${scenario.name} executes ${unit.name}`,
844
+ "scenario",
845
+ `coverage:${candidate.entry.path}:${scenario.id}`,
846
+ );
847
+ for (const omitted of prepared.unjudged)
848
+ for (const unit of units)
849
+ report.diagnose(
850
+ "evidence_too_large",
851
+ omitted.reason,
852
+ candidate.entry.path,
853
+ [
854
+ `coverage:${candidate.entry.path}:${omitted.scenario.id}:${unit.id}`,
855
+ ],
856
+ );
590
857
  if (prepared.unjudged.length || !candidate.entry.scenarios.length)
591
858
  for (const unit of units) incompleteUnits.add(unit.id);
592
859
  await Promise.all(
@@ -620,6 +887,72 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
620
887
  result,
621
888
  );
622
889
  limitations.push(...health.failures);
890
+ for (const failure of health.failures)
891
+ report.diagnose(
892
+ "control_failure",
893
+ failure.fact,
894
+ candidate.entry.path,
895
+ units.flatMap((unit) =>
896
+ batch.scenarios
897
+ .filter((scenario) =>
898
+ health.unhealthyQuestionIds.has(
899
+ `${scenario.id}_${unit.id}`,
900
+ ),
901
+ )
902
+ .map(
903
+ (scenario) =>
904
+ `coverage:${candidate.entry.path}:${scenario.id}:${unit.id}`,
905
+ ),
906
+ ),
907
+ false,
908
+ );
909
+ report.countControls(
910
+ result,
911
+ witnessQuestions.witnesses.map((witness) => witness.id),
912
+ );
913
+ for (const unit of units)
914
+ for (const scenario of batch.scenarios) {
915
+ const id = `coverage:${candidate.entry.path}:${scenario.id}:${unit.id}`;
916
+ report.expect(
917
+ id,
918
+ `${candidate.entry.path}: ${scenario.name} executes ${unit.name}`,
919
+ "scenario",
920
+ `coverage:${candidate.entry.path}:${scenario.id}`,
921
+ );
922
+ if (!result.ok)
923
+ report.failure(
924
+ result,
925
+ [id],
926
+ budget
927
+ ? budget.kind === "session"
928
+ ? "session_budget"
929
+ : "call_budget"
930
+ : !client
931
+ ? "not_configured"
932
+ : undefined,
933
+ );
934
+ else
935
+ report.answer(
936
+ id,
937
+ result.answers[`${scenario.id}_${unit.id}`],
938
+ {
939
+ band: health.unhealthyQuestionIds.has(
940
+ `${scenario.id}_${unit.id}`,
941
+ )
942
+ ? "unsure"
943
+ : "verdict",
944
+ reason: health.unhealthyQuestionIds.get(
945
+ `${scenario.id}_${unit.id}`,
946
+ ),
947
+ },
948
+ controlsFor(
949
+ `${scenario.id}_${unit.id}`,
950
+ result,
951
+ witnessQuestions.witnesses.map((witness) => witness.id),
952
+ ),
953
+ false,
954
+ );
955
+ }
623
956
  for (const unit of units) {
624
957
  if (!result.ok) {
625
958
  fallback ||= !budget;
@@ -685,16 +1018,34 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
685
1018
  evidenceLimits.length > 0 ||
686
1019
  Boolean(args.paths?.length);
687
1020
  for (const unit of residual)
688
- if ((coverage.get(unit.id) ?? 0) < SELECT_MIN)
689
- answers.push({
690
- label: `changed, run by no discovered test: ${unit.name} (${unit.file})`,
691
- value: {
692
- head: "within discovered inventory only",
693
- p: coverage.get(unit.id) ?? 0,
694
- },
695
- band:
696
- incomplete || incompleteUnits.has(unit.id) ? "unsure" : "verdict",
697
- });
1021
+ if ((coverage.get(unit.id) ?? 0) < SELECT_MIN) {
1022
+ const supportingIds = [...report.items.keys()].filter(
1023
+ (id) => id.startsWith("coverage:") && id.endsWith(`:${unit.id}`),
1024
+ );
1025
+ report.diagnose(
1026
+ incomplete || incompleteUnits.has(unit.id)
1027
+ ? "collection_omitted"
1028
+ : "conservative_widening",
1029
+ `Derived from retained coverage decisions: no discovered test established execution of ${unit.name} (${unit.file}) within the considered inventory only.${incomplete || incompleteUnits.has(unit.id) ? " Absence remains unsure because evidence, scope, or controls are incomplete." : " This is not a global coverage claim."}`,
1030
+ unit.file,
1031
+ supportingIds,
1032
+ false,
1033
+ );
1034
+ if (!supportingIds.length) {
1035
+ const diagnostic = report.diagnostics.at(-1);
1036
+ if (diagnostic)
1037
+ diagnostic.scope = {
1038
+ kind: "inventory",
1039
+ inventoryIds: ["test-candidates"],
1040
+ };
1041
+ const action = report.actions.at(-1);
1042
+ if (action)
1043
+ action.scope = {
1044
+ kind: "inventory",
1045
+ inventoryIds: ["test-candidates"],
1046
+ };
1047
+ }
1048
+ }
698
1049
  if (fallback) {
699
1050
  selected.length = 0;
700
1051
  selected.push(
@@ -703,6 +1054,13 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
703
1054
  scenarioIds: null,
704
1055
  })),
705
1056
  );
1057
+ report.diagnose(
1058
+ "conservative_widening",
1059
+ "Unavailable judgment conservatively selects all considered test candidates; static reachability exclusions do not narrow this fallback.",
1060
+ undefined,
1061
+ [],
1062
+ false,
1063
+ );
706
1064
  limitations.push({
707
1065
  fact: "fallback: all",
708
1066
  next: "run all discovered tests with their project runner",
@@ -719,11 +1077,111 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
719
1077
  })),
720
1078
  ),
721
1079
  );
1080
+ for (const limit of commands.limits)
1081
+ for (const path of limit.files)
1082
+ report.diagnose("unresolved_runner", limit.reason, path, [], false);
1083
+ for (const limit of discovery.limits)
1084
+ report.diagnose(
1085
+ "unresolved_runner",
1086
+ limit.kind,
1087
+ limit.path,
1088
+ [],
1089
+ limit.kind !== "local_runner_unproven" &&
1090
+ limit.kind !== "interactive_script_skipped",
1091
+ );
1092
+ for (const limit of inventory.limits)
1093
+ report.diagnose(
1094
+ limit.kind === "secret_pattern"
1095
+ ? "secret_pattern"
1096
+ : "collection_omitted",
1097
+ limit.kind,
1098
+ limit.path,
1099
+ );
1100
+ for (const limit of diff.limits)
1101
+ report.diagnose(
1102
+ limit.kind === "secret_pattern"
1103
+ ? "secret_pattern"
1104
+ : "collection_omitted",
1105
+ limit.kind,
1106
+ limit.file,
1107
+ );
1108
+ for (const limit of graph.limits)
1109
+ report.diagnose(
1110
+ "dynamic_dependency",
1111
+ `${limit.kind}${limit.specifier ? ` (${limit.specifier})` : ""}`,
1112
+ limit.path,
1113
+ [],
1114
+ false,
1115
+ );
1116
+ for (const item of report.items.values()) {
1117
+ const candidate = candidates.find(
1118
+ (candidate) =>
1119
+ item.id.startsWith(`test:${candidate.entry.path}:`) ||
1120
+ item.id.startsWith(`coverage:${candidate.entry.path}:`),
1121
+ );
1122
+ if (!candidate) continue;
1123
+ const plan = selected.find((plan) => plan.entry === candidate.entry);
1124
+ const scenario = candidate.entry.scenarios.find(
1125
+ (scenario) =>
1126
+ item.id === `test:${candidate.entry.path}:${scenario.id}` ||
1127
+ item.id.startsWith(
1128
+ `coverage:${candidate.entry.path}:${scenario.id}:`,
1129
+ ),
1130
+ );
1131
+ const isSelected = Boolean(
1132
+ plan &&
1133
+ (plan.scenarioIds === null ||
1134
+ (scenario && plan.scenarioIds.includes(scenario.id))),
1135
+ );
1136
+ item.selection = {
1137
+ selected: isSelected,
1138
+ reason: fallback
1139
+ ? "conservative_fallback"
1140
+ : item.treatment === "static"
1141
+ ? item.staticReason
1142
+ : item.treatment === "not_judged"
1143
+ ? "conservative_fallback"
1144
+ : "unchanged selection policy and whole-file widening",
1145
+ };
1146
+ }
1147
+ for (const criterion of report.inventories[0]?.criteria ?? [])
1148
+ criterion.diagnosticIds = report.diagnostics
1149
+ .filter(
1150
+ (diagnostic) =>
1151
+ diagnostic.cause === "criteria_no_match" &&
1152
+ diagnostic.target.status === "known" &&
1153
+ diagnostic.target.value === criterion.criterion,
1154
+ )
1155
+ .map((diagnostic) => diagnostic.id);
1156
+ report.actions.push({
1157
+ id: "execute-selection-plan",
1158
+ code: "execute_plan",
1159
+ target: reportKnown(inventory.cwd),
1160
+ scope: { kind: "call" },
1161
+ condition:
1162
+ "When verification is authorized and the listed runner/configuration is available.",
1163
+ instruction:
1164
+ "Execute the retained runner commands separately; this tool has not executed any test.",
1165
+ repeatUnchanged: false,
1166
+ });
722
1167
  if (skipped)
723
1168
  limitations.push({
724
1169
  fact: `${skipped} tests skipped because their imports cannot reach the diff`,
725
1170
  next: "an alias or dynamic import would have kept a test in",
726
1171
  });
1172
+ for (const criterion of selectionEvidence.criteria)
1173
+ if (!criterion.matches.length)
1174
+ limitations.push({
1175
+ path: criterion.criterion,
1176
+ cause: "selection criterion has no discovered test match",
1177
+ fact: `selection criterion without match: ${criterion.criterion}`,
1178
+ next: "Change the criterion or supply the missing test/configuration evidence; an empty scope does not prove absence of impact.",
1179
+ });
1180
+ if (selectionEvidence.wideningTriggers.length)
1181
+ limitations.push({
1182
+ fact: `selection widened conservatively: ${selectionEvidence.wideningTriggers.join(", ")}`,
1183
+ next: "Run the widened static plans; dependency reachability is not established for these changed files.",
1184
+ });
727
1185
  return finish({
728
1186
  answers,
729
1187
  unjudged,