jev-agent-tools 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -1
- package/CONTRIBUTING.md +3 -0
- package/README.md +22 -14
- package/SECURITY.md +17 -1
- package/dist/adapters/ask-files.js +11 -2
- package/dist/adapters/ask-proof.js +63 -7
- package/dist/adapters/command.js +82 -29
- package/dist/adapters/docs.js +30 -10
- package/dist/adapters/evidence-context.js +119 -0
- package/dist/adapters/files.js +141 -16
- package/dist/adapters/find.js +34 -6
- package/dist/adapters/git-base.js +7 -1
- package/dist/adapters/git.js +51 -7
- package/dist/adapters/locate-file.js +47 -9
- package/dist/adapters/private-storage.js +14 -6
- package/dist/adapters/risk-callers.js +3 -0
- package/dist/adapters/shell.js +23 -7
- package/dist/adapters/test-inventory.js +10 -2
- package/dist/configuration.js +17 -7
- package/dist/constants.js +26 -5
- package/dist/core/ask-references.js +193 -109
- package/dist/core/asks.js +78 -7
- package/dist/core/locate.js +8 -8
- package/dist/core/output.js +17 -0
- package/dist/core/result-report.js +302 -0
- package/dist/core/secret-path.js +34 -0
- package/dist/core/state.js +8 -1
- package/dist/core/units.js +1 -1
- package/dist/jev/client.js +34 -12
- package/dist/mcp/protocol.js +50 -27
- package/dist/mcp/tools.js +20 -7
- package/dist/render.js +72 -0
- package/dist/report-schema.js +1356 -0
- package/dist/result-types.js +1 -0
- package/dist/texts/ask-files.js +3 -1
- package/dist/texts/ask.js +3 -1
- package/dist/texts/check-diff.js +7 -4
- package/dist/texts/find.js +7 -2
- package/dist/texts/guide.js +3 -16
- package/dist/texts/instructions.js +72 -0
- package/dist/texts/locate.js +7 -2
- package/dist/texts/select-tests.js +3 -1
- package/dist/tools/ask-files.js +248 -15
- package/dist/tools/ask.js +523 -62
- package/dist/tools/check-diff.js +222 -30
- package/dist/tools/docs-check.js +122 -13
- package/dist/tools/find.js +320 -27
- package/dist/tools/locate.js +317 -18
- package/dist/tools/review-report.js +230 -0
- package/dist/tools/select-tests.js +273 -19
- package/dist/tools/spec-check.js +119 -22
- package/docs/adr/0001-strict-typescript-pure-core-offline-tests.md +3 -3
- package/docs/agent-instructions.md +59 -30
- package/docs/design.md +13 -1
- package/docs/mcp.md +8 -6
- package/docs/tools/jev_ask.md +8 -5
- package/docs/tools/jev_ask_files.md +2 -1
- package/docs/tools/jev_check_diff.md +4 -1
- package/docs/tools/jev_find_files.md +2 -1
- package/docs/tools/jev_locate_in_file.md +5 -0
- package/docs/tools/jev_select_tests.md +4 -1
- package/package.json +1 -1
- package/rules/jev-ask.md +22 -1
- package/server.json +2 -2
- package/src/adapters/ask-files.ts +11 -3
- package/src/adapters/ask-proof.ts +69 -11
- package/src/adapters/command.ts +96 -33
- package/src/adapters/docs.ts +33 -14
- package/src/adapters/evidence-context.ts +169 -0
- package/src/adapters/files.ts +146 -16
- package/src/adapters/find.ts +37 -7
- package/src/adapters/git-base.ts +7 -1
- package/src/adapters/git.ts +61 -8
- package/src/adapters/locate-file.ts +51 -9
- package/src/adapters/private-storage.ts +17 -5
- package/src/adapters/risk-callers.ts +3 -0
- package/src/adapters/shell.ts +23 -7
- package/src/adapters/test-inventory.ts +12 -4
- package/src/configuration.ts +16 -2
- package/src/constants.ts +26 -5
- package/src/core/ask-references.ts +262 -146
- package/src/core/asks.ts +79 -7
- package/src/core/import-boundaries.ts +8 -3
- package/src/core/locate.ts +8 -5
- package/src/core/output.ts +34 -0
- package/src/core/result-report.ts +410 -0
- package/src/core/secret-path.ts +37 -0
- package/src/core/state.ts +8 -1
- package/src/core/units.ts +3 -2
- package/src/index.ts +3 -0
- package/src/jev/client.ts +54 -16
- package/src/jev/types.ts +18 -3
- package/src/mcp/protocol.ts +91 -41
- package/src/mcp/tools.ts +26 -13
- package/src/render.ts +109 -0
- package/src/report-schema.ts +1380 -0
- package/src/result-types.ts +234 -0
- package/src/result.ts +4 -1
- package/src/runtime.ts +6 -0
- package/src/texts/ask-files.ts +4 -1
- package/src/texts/ask.ts +8 -1
- package/src/texts/check-diff.ts +7 -4
- package/src/texts/find.ts +8 -2
- package/src/texts/guide.ts +8 -16
- package/src/texts/instructions.ts +98 -0
- package/src/texts/locate.ts +8 -2
- package/src/texts/run-end.ts +2 -2
- package/src/texts/select-tests.ts +4 -1
- package/src/tools/ask-files.ts +309 -14
- package/src/tools/ask.ts +700 -77
- package/src/tools/check-diff.ts +331 -28
- package/src/tools/docs-check.ts +241 -39
- package/src/tools/find.ts +386 -29
- package/src/tools/locate.ts +384 -19
- package/src/tools/review-report.ts +308 -0
- package/src/tools/select-tests.ts +479 -21
- package/src/tools/spec-check.ts +193 -19
|
@@ -1,8 +1,12 @@
|
|
|
1
|
-
import { matchesGlob, relative, resolve } from "node:path";
|
|
1
|
+
import { isAbsolute, matchesGlob, relative, resolve } from "node:path";
|
|
2
2
|
import type { ToolDefinition } from "@earendil-works/pi-coding-agent";
|
|
3
3
|
import { type Static, Type } from "@sinclair/typebox";
|
|
4
4
|
import { createAnalysisContext } from "../adapters/analysis-context.ts";
|
|
5
|
-
import {
|
|
5
|
+
import {
|
|
6
|
+
type EvidenceContext,
|
|
7
|
+
resolveEvidenceContext,
|
|
8
|
+
withEvidenceContext,
|
|
9
|
+
} from "../adapters/evidence-context.ts";
|
|
6
10
|
import { collectUnits } from "../adapters/git.ts";
|
|
7
11
|
import { resolveBase } from "../adapters/git-base.ts";
|
|
8
12
|
import { shareGitInventory } from "../adapters/git-inventory.ts";
|
|
@@ -26,6 +30,11 @@ import type {
|
|
|
26
30
|
Limitation,
|
|
27
31
|
} from "../core/output.ts";
|
|
28
32
|
import { buildEnvelope } from "../core/output.ts";
|
|
33
|
+
import {
|
|
34
|
+
type Cause,
|
|
35
|
+
type ResultReportV1,
|
|
36
|
+
known as reportKnown,
|
|
37
|
+
} from "../core/result-report.ts";
|
|
29
38
|
import { buildRunnerCommands } from "../core/test-commands.ts";
|
|
30
39
|
import { prepareCoverageWitnesses } from "../core/test-coverage.ts";
|
|
31
40
|
import type { TestEntry } from "../core/test-discovery.ts";
|
|
@@ -42,12 +51,14 @@ import {
|
|
|
42
51
|
buildCoverageWitnessUnits,
|
|
43
52
|
evaluateBatchWitnessHealth,
|
|
44
53
|
} from "../presets/witnesses.ts";
|
|
45
|
-
import {
|
|
54
|
+
import { renderResultReport } from "../render.ts";
|
|
46
55
|
import type { ToolDependencies } from "../runtime.ts";
|
|
47
56
|
import { SELECT_TESTS_DESCRIPTION } from "../texts/select-tests.ts";
|
|
57
|
+
import { controlsFor, ReviewReport, reportMetrics } from "./review-report.ts";
|
|
48
58
|
|
|
49
59
|
export const selectTestsParameters = Type.Object(
|
|
50
60
|
{
|
|
61
|
+
root: Type.Optional(Type.String()),
|
|
51
62
|
base: Type.Optional(Type.String({ minLength: 1 })),
|
|
52
63
|
paths: Type.Optional(Type.Array(Type.String({ minLength: 1 }))),
|
|
53
64
|
witnesses: Type.Optional(
|
|
@@ -126,7 +137,13 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
|
|
|
126
137
|
ctx: { cwd: string } & GuideContext,
|
|
127
138
|
) {
|
|
128
139
|
const client = dependencies.client;
|
|
129
|
-
const
|
|
140
|
+
const evidence = await resolveEvidenceContext(ctx.cwd, args.root, {
|
|
141
|
+
exec: execute,
|
|
142
|
+
signal,
|
|
143
|
+
origin: dependencies.evidenceOrigin,
|
|
144
|
+
});
|
|
145
|
+
const evidenceContext = evidence.context;
|
|
146
|
+
const cwd = evidence.ok ? evidence.cwd : evidenceContext.authority.path;
|
|
130
147
|
const started = performance.now();
|
|
131
148
|
const exec = shareGitInventory(execute);
|
|
132
149
|
const totals = {
|
|
@@ -136,10 +153,17 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
|
|
|
136
153
|
cacheRequests: 0,
|
|
137
154
|
usage: { inputTokens: 0, costUsd: 0 },
|
|
138
155
|
};
|
|
156
|
+
const selectionEvidence = {
|
|
157
|
+
criteria: [] as { criterion: string; matches: string[] }[],
|
|
158
|
+
wideningTriggers: [] as string[],
|
|
159
|
+
inventory: [] as string[],
|
|
160
|
+
};
|
|
161
|
+
const report = new ReviewReport();
|
|
139
162
|
let costKnown = false;
|
|
163
|
+
let unknownCost = false;
|
|
140
164
|
let sent = 0;
|
|
141
165
|
let budget: BudgetRefusal | undefined;
|
|
142
|
-
const finish = (input: Omit<EnvelopeInput, "yield"
|
|
166
|
+
const finish = (input: Omit<EnvelopeInput, "yield">, cause?: Cause) => {
|
|
143
167
|
const limitKeys = new Set<string>();
|
|
144
168
|
const uniqueLimits = input.limitations?.filter((limit) => {
|
|
145
169
|
const key = JSON.stringify([limit.fact, limit.next]);
|
|
@@ -152,26 +176,78 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
|
|
|
152
176
|
limitations: uniqueLimits,
|
|
153
177
|
yield: {
|
|
154
178
|
...totals,
|
|
155
|
-
costUsd:
|
|
179
|
+
costUsd:
|
|
180
|
+
costKnown && !unknownCost ? totals.usage.costUsd : undefined,
|
|
156
181
|
elapsedMs: performance.now() - started,
|
|
157
182
|
},
|
|
158
183
|
});
|
|
184
|
+
if (input.refusal) {
|
|
185
|
+
report.refusal = true;
|
|
186
|
+
report.diagnose(cause ?? "internal_error", input.refusal);
|
|
187
|
+
}
|
|
188
|
+
if (!report.items.size && !input.refusal)
|
|
189
|
+
report.diagnose(
|
|
190
|
+
"collection_empty",
|
|
191
|
+
"No test decision candidates in the discovered inventory; this is not proof of no affected tests.",
|
|
192
|
+
undefined,
|
|
193
|
+
[],
|
|
194
|
+
false,
|
|
195
|
+
);
|
|
196
|
+
const result = report.build(
|
|
197
|
+
"jev_select_tests",
|
|
198
|
+
evidenceContext,
|
|
199
|
+
reportMetrics(envelope),
|
|
200
|
+
);
|
|
159
201
|
runtime.session.record(envelope);
|
|
160
202
|
runtime.guide.deliver(ctx);
|
|
161
|
-
const details: Judgment & {
|
|
203
|
+
const details: Judgment & {
|
|
204
|
+
result: ResultReportV1;
|
|
205
|
+
limitations?: readonly Limitation[];
|
|
206
|
+
evidenceContext: EvidenceContext;
|
|
207
|
+
selectionEvidence: typeof selectionEvidence;
|
|
208
|
+
} = {
|
|
162
209
|
ok: true,
|
|
163
210
|
answers: {},
|
|
164
211
|
...totals,
|
|
212
|
+
evidenceContext,
|
|
213
|
+
selectionEvidence,
|
|
214
|
+
result,
|
|
165
215
|
limitations: uniqueLimits,
|
|
166
216
|
};
|
|
167
217
|
return {
|
|
168
|
-
content: [
|
|
218
|
+
content: [
|
|
219
|
+
{
|
|
220
|
+
type: "text" as const,
|
|
221
|
+
text: renderResultReport(result, { details: envelope }),
|
|
222
|
+
},
|
|
223
|
+
],
|
|
169
224
|
details,
|
|
170
225
|
...hostUsage(host.isOmp, costKnown ? totals.usage : undefined),
|
|
171
226
|
};
|
|
172
227
|
};
|
|
228
|
+
if (!evidence.ok)
|
|
229
|
+
return finish(
|
|
230
|
+
{ refusal: evidence.error },
|
|
231
|
+
evidence.cause ?? "invalid_root",
|
|
232
|
+
);
|
|
233
|
+
evidenceContext.requestedBase = args.base ?? "HEAD";
|
|
234
|
+
for (const path of args.paths ?? [])
|
|
235
|
+
if (
|
|
236
|
+
isAbsolute(path) ||
|
|
237
|
+
path.split(/[\\/]/).includes("..") ||
|
|
238
|
+
path.split(/[\\/]/).includes(".git")
|
|
239
|
+
)
|
|
240
|
+
return finish(
|
|
241
|
+
{ refusal: `Path not permitted: ${path}` },
|
|
242
|
+
"forbidden_path",
|
|
243
|
+
);
|
|
173
244
|
const comparison = await resolveBase(exec, cwd, args.base, signal);
|
|
174
|
-
if (!comparison.ok)
|
|
245
|
+
if (!comparison.ok)
|
|
246
|
+
return finish(
|
|
247
|
+
{ refusal: comparison.error },
|
|
248
|
+
comparison.cause ?? "invalid_base",
|
|
249
|
+
);
|
|
250
|
+
evidenceContext.resolvedBase = comparison.base;
|
|
175
251
|
const analysis = await createAnalysisContext();
|
|
176
252
|
const [inventory, diff] = await Promise.all([
|
|
177
253
|
collectTestInventory(exec, cwd, signal),
|
|
@@ -181,8 +257,16 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
|
|
|
181
257
|
analysis.parser,
|
|
182
258
|
),
|
|
183
259
|
]);
|
|
184
|
-
if (!inventory.ok)
|
|
185
|
-
|
|
260
|
+
if (!inventory.ok)
|
|
261
|
+
return finish(
|
|
262
|
+
{ refusal: inventory.error },
|
|
263
|
+
inventory.cause ?? "file_unavailable",
|
|
264
|
+
);
|
|
265
|
+
if (!diff.ok)
|
|
266
|
+
return finish({ refusal: diff.error }, diff.cause ?? "git_failure");
|
|
267
|
+
if (evidenceContext.effectiveRoot)
|
|
268
|
+
evidenceContext.effectiveRoot.path = inventory.cwd;
|
|
269
|
+
selectionEvidence.inventory = [...inventory.paths];
|
|
186
270
|
const known = new Set(inventory.paths);
|
|
187
271
|
const sourceFiles = new Map<string, ImportSource>();
|
|
188
272
|
const read = async (path: string): Promise<ImportSource | undefined> => {
|
|
@@ -277,6 +361,23 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
|
|
|
277
361
|
!graph.edges.has(file.path) ||
|
|
278
362
|
!/\.[cm]?[jt]sx?$|\.py$/.test(file.path),
|
|
279
363
|
);
|
|
364
|
+
selectionEvidence.wideningTriggers = diff.files
|
|
365
|
+
.filter(
|
|
366
|
+
(file) =>
|
|
367
|
+
!graph.edges.has(file.path) ||
|
|
368
|
+
!/\.[cm]?[jt]sx?$|\.py$/.test(file.path),
|
|
369
|
+
)
|
|
370
|
+
.map((file) => file.path);
|
|
371
|
+
selectionEvidence.criteria = (args.paths ?? []).map((criterion) => ({
|
|
372
|
+
criterion,
|
|
373
|
+
matches: versionedEntries
|
|
374
|
+
.filter(
|
|
375
|
+
(entry) =>
|
|
376
|
+
matchesGlob(entry.path, criterion) ||
|
|
377
|
+
entry.path.startsWith(`${criterion.replace(/\/$/, "")}/`),
|
|
378
|
+
)
|
|
379
|
+
.map((entry) => entry.path),
|
|
380
|
+
}));
|
|
280
381
|
const candidates = versionedEntries
|
|
281
382
|
.filter(
|
|
282
383
|
(entry) =>
|
|
@@ -308,6 +409,77 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
|
|
|
308
409
|
outside,
|
|
309
410
|
);
|
|
310
411
|
});
|
|
412
|
+
report.inventories.push({
|
|
413
|
+
id: "test-candidates",
|
|
414
|
+
kind: "tests",
|
|
415
|
+
rules: [
|
|
416
|
+
"Tracked supported test declarations and literal project runner configuration",
|
|
417
|
+
"Import closure selection; conservative fallback preserves unjudged decisions",
|
|
418
|
+
],
|
|
419
|
+
restrictions: args.paths ?? [],
|
|
420
|
+
discovered: reportKnown(versionedEntries.length),
|
|
421
|
+
considered: reportKnown(candidates.length),
|
|
422
|
+
scopeRestricted: Boolean(args.paths?.length),
|
|
423
|
+
criteria: selectionEvidence.criteria.map((criterion) => ({
|
|
424
|
+
criterion: criterion.criterion,
|
|
425
|
+
matches: reportKnown(criterion.matches.length),
|
|
426
|
+
outcome: criterion.matches.length ? "matched" : "no_match",
|
|
427
|
+
diagnosticIds: [],
|
|
428
|
+
})),
|
|
429
|
+
});
|
|
430
|
+
for (const candidate of candidates) {
|
|
431
|
+
const scenarios = candidate.entry.scenarios;
|
|
432
|
+
if (!scenarios.length)
|
|
433
|
+
report.expect(
|
|
434
|
+
`test:${candidate.entry.path}:unknown`,
|
|
435
|
+
candidate.entry.path,
|
|
436
|
+
"scenario",
|
|
437
|
+
);
|
|
438
|
+
for (const scenario of scenarios)
|
|
439
|
+
report.expect(
|
|
440
|
+
`test:${candidate.entry.path}:${scenario.id}`,
|
|
441
|
+
`${candidate.entry.path}: ${scenario.name}`,
|
|
442
|
+
"scenario",
|
|
443
|
+
`test:${candidate.entry.path}`,
|
|
444
|
+
);
|
|
445
|
+
}
|
|
446
|
+
for (const criterion of selectionEvidence.criteria)
|
|
447
|
+
if (!criterion.matches.length) {
|
|
448
|
+
report.diagnose(
|
|
449
|
+
"criteria_no_match",
|
|
450
|
+
`No discovered tests match criterion ${criterion.criterion}`,
|
|
451
|
+
criterion.criterion,
|
|
452
|
+
);
|
|
453
|
+
const excluded = inventory.limits.filter(
|
|
454
|
+
(limit) =>
|
|
455
|
+
matchesGlob(limit.path, criterion.criterion) ||
|
|
456
|
+
limit.path.startsWith(
|
|
457
|
+
`${criterion.criterion.replace(/\/$/, "")}/`,
|
|
458
|
+
),
|
|
459
|
+
);
|
|
460
|
+
if (excluded.length) {
|
|
461
|
+
report.diagnose(
|
|
462
|
+
"outside_inventory",
|
|
463
|
+
"Requested criterion names entries excluded from the admitted inventory",
|
|
464
|
+
criterion.criterion,
|
|
465
|
+
[],
|
|
466
|
+
false,
|
|
467
|
+
excluded.map((limit) => limit.path),
|
|
468
|
+
);
|
|
469
|
+
const entry = report.inventories[0]?.criteria.find(
|
|
470
|
+
(entry) => entry.criterion === criterion.criterion,
|
|
471
|
+
);
|
|
472
|
+
if (entry) entry.outcome = "outside_inventory";
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
if (selectionEvidence.wideningTriggers.length)
|
|
476
|
+
report.diagnose(
|
|
477
|
+
"conservative_widening",
|
|
478
|
+
`Changed files outside supported dependency graph: ${selectionEvidence.wideningTriggers.join(", ")}`,
|
|
479
|
+
undefined,
|
|
480
|
+
[],
|
|
481
|
+
false,
|
|
482
|
+
);
|
|
311
483
|
const selected: {
|
|
312
484
|
entry: TestEntry;
|
|
313
485
|
scenarioIds: readonly string[] | null;
|
|
@@ -373,9 +545,11 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
|
|
|
373
545
|
questions: Record<string, Question>,
|
|
374
546
|
witnessIds?: readonly string[],
|
|
375
547
|
) => {
|
|
548
|
+
state = withEvidenceContext(state, evidenceContext);
|
|
376
549
|
if (JSON.stringify(state).length > STATE_MAX_CHARS)
|
|
377
550
|
return {
|
|
378
551
|
ok: false as const,
|
|
552
|
+
cause: "evidence_too_large" as const,
|
|
379
553
|
error: `required evidence exceeds STATE_MAX_CHARS=${STATE_MAX_CHARS}`,
|
|
380
554
|
};
|
|
381
555
|
if (args.max_calls !== undefined && sent >= args.max_calls) {
|
|
@@ -390,6 +564,8 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
|
|
|
390
564
|
signal,
|
|
391
565
|
witnesses: witnessIds,
|
|
392
566
|
...runtime.session.requestGate(),
|
|
567
|
+
admissionCause: () =>
|
|
568
|
+
budget?.kind === "session" ? "session_budget" : "call_budget",
|
|
393
569
|
beforeRequest: (questionCount) => {
|
|
394
570
|
if (args.max_calls !== undefined && sent >= args.max_calls) {
|
|
395
571
|
budget = {
|
|
@@ -412,6 +588,7 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
|
|
|
412
588
|
totals.questions += result.questions ?? 0;
|
|
413
589
|
totals.cacheHits += result.cacheHits ?? 0;
|
|
414
590
|
totals.cacheRequests += result.cacheRequests ?? 0;
|
|
591
|
+
unknownCost ||= result.usage === undefined;
|
|
415
592
|
if (result.usage) {
|
|
416
593
|
costKnown = true;
|
|
417
594
|
totals.usage.inputTokens += result.usage.inputTokens;
|
|
@@ -422,6 +599,27 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
|
|
|
422
599
|
const pointerResults = await Promise.all(
|
|
423
600
|
candidates.map(async (candidate) => {
|
|
424
601
|
const { entry } = candidate;
|
|
602
|
+
const reportIds = [...report.items.values()]
|
|
603
|
+
.filter(
|
|
604
|
+
(item) =>
|
|
605
|
+
item.groupId === `test:${entry.path}` ||
|
|
606
|
+
item.id === `test:${entry.path}:unknown`,
|
|
607
|
+
)
|
|
608
|
+
.map((item) => item.id);
|
|
609
|
+
if (candidate.touched)
|
|
610
|
+
for (const id of reportIds) report.static(id, "touched", true);
|
|
611
|
+
if (
|
|
612
|
+
!candidate.units.length &&
|
|
613
|
+
!candidate.uncertain &&
|
|
614
|
+
!outside &&
|
|
615
|
+
!candidate.touched
|
|
616
|
+
)
|
|
617
|
+
for (const id of reportIds)
|
|
618
|
+
report.static(
|
|
619
|
+
id,
|
|
620
|
+
"static import closure cannot reach changed units",
|
|
621
|
+
false,
|
|
622
|
+
);
|
|
425
623
|
if (candidate.touched)
|
|
426
624
|
return {
|
|
427
625
|
candidate,
|
|
@@ -440,6 +638,12 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
|
|
|
440
638
|
if (
|
|
441
639
|
units.some((unit) => unit.before === null && unit.after === null)
|
|
442
640
|
) {
|
|
641
|
+
report.diagnose(
|
|
642
|
+
"binary_or_non_utf8",
|
|
643
|
+
"Changed source unavailable",
|
|
644
|
+
entry.path,
|
|
645
|
+
reportIds,
|
|
646
|
+
);
|
|
443
647
|
for (const unit of units) incompleteUnits.add(unit.id);
|
|
444
648
|
unjudged.push({
|
|
445
649
|
label: entry.path,
|
|
@@ -449,6 +653,13 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
|
|
|
449
653
|
return { candidate, selected: null, touched: false };
|
|
450
654
|
}
|
|
451
655
|
if (!entry.scenarios.length) {
|
|
656
|
+
report.totalUnknown = true;
|
|
657
|
+
report.diagnose(
|
|
658
|
+
"unsupported_syntax",
|
|
659
|
+
"Scenario names or count unresolved",
|
|
660
|
+
entry.path,
|
|
661
|
+
reportIds,
|
|
662
|
+
);
|
|
452
663
|
unjudged.push({
|
|
453
664
|
label: entry.path,
|
|
454
665
|
reason: "scenario names or count unresolved",
|
|
@@ -461,6 +672,9 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
|
|
|
461
672
|
(item) => item.scenario.id,
|
|
462
673
|
);
|
|
463
674
|
for (const item of prepared.unjudged) {
|
|
675
|
+
report.diagnose("evidence_too_large", item.reason, entry.path, [
|
|
676
|
+
`test:${entry.path}:${item.scenario.id}`,
|
|
677
|
+
]);
|
|
464
678
|
for (const unit of units) incompleteUnits.add(unit.id);
|
|
465
679
|
unjudged.push({
|
|
466
680
|
label: `${entry.path}: ${item.scenario.name}`,
|
|
@@ -494,6 +708,41 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
|
|
|
494
708
|
]),
|
|
495
709
|
);
|
|
496
710
|
const result = await judge(batch.state, questions);
|
|
711
|
+
const batchIds = batch.scenarios.map(
|
|
712
|
+
(scenario) => `test:${entry.path}:${scenario.id}`,
|
|
713
|
+
);
|
|
714
|
+
if (!result.ok)
|
|
715
|
+
report.failure(
|
|
716
|
+
result,
|
|
717
|
+
batchIds,
|
|
718
|
+
budget
|
|
719
|
+
? budget.kind === "session"
|
|
720
|
+
? "session_budget"
|
|
721
|
+
: "call_budget"
|
|
722
|
+
: !client
|
|
723
|
+
? "not_configured"
|
|
724
|
+
: undefined,
|
|
725
|
+
);
|
|
726
|
+
else
|
|
727
|
+
for (const scenario of batch.scenarios) {
|
|
728
|
+
const answer = result.answers[scenario.id];
|
|
729
|
+
const id = `test:${entry.path}:${scenario.id}`;
|
|
730
|
+
report.answer(id, answer, {
|
|
731
|
+
band: prepared.unjudged.length ? "unsure" : "verdict",
|
|
732
|
+
});
|
|
733
|
+
const item = report.items.get(id);
|
|
734
|
+
if (!item)
|
|
735
|
+
throw new Error(`Unregistered selection result ${id}`);
|
|
736
|
+
item.selection = {
|
|
737
|
+
selected:
|
|
738
|
+
answer?.type !== "choice" ||
|
|
739
|
+
1 - (answer.probabilities.none ?? 0) >= SELECT_MIN,
|
|
740
|
+
reason:
|
|
741
|
+
answer?.type === "choice"
|
|
742
|
+
? "changed-unit pointer selection threshold"
|
|
743
|
+
: "conservative_fallback",
|
|
744
|
+
};
|
|
745
|
+
}
|
|
497
746
|
if (!result.ok) {
|
|
498
747
|
fallback ||= !budget;
|
|
499
748
|
for (const unit of units) incompleteUnits.add(unit.id);
|
|
@@ -587,6 +836,24 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
|
|
|
587
836
|
preliminary.state,
|
|
588
837
|
candidate.entry.scenarios,
|
|
589
838
|
);
|
|
839
|
+
for (const unit of units)
|
|
840
|
+
for (const scenario of candidate.entry.scenarios)
|
|
841
|
+
report.expect(
|
|
842
|
+
`coverage:${candidate.entry.path}:${scenario.id}:${unit.id}`,
|
|
843
|
+
`${candidate.entry.path}: ${scenario.name} executes ${unit.name}`,
|
|
844
|
+
"scenario",
|
|
845
|
+
`coverage:${candidate.entry.path}:${scenario.id}`,
|
|
846
|
+
);
|
|
847
|
+
for (const omitted of prepared.unjudged)
|
|
848
|
+
for (const unit of units)
|
|
849
|
+
report.diagnose(
|
|
850
|
+
"evidence_too_large",
|
|
851
|
+
omitted.reason,
|
|
852
|
+
candidate.entry.path,
|
|
853
|
+
[
|
|
854
|
+
`coverage:${candidate.entry.path}:${omitted.scenario.id}:${unit.id}`,
|
|
855
|
+
],
|
|
856
|
+
);
|
|
590
857
|
if (prepared.unjudged.length || !candidate.entry.scenarios.length)
|
|
591
858
|
for (const unit of units) incompleteUnits.add(unit.id);
|
|
592
859
|
await Promise.all(
|
|
@@ -620,6 +887,72 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
|
|
|
620
887
|
result,
|
|
621
888
|
);
|
|
622
889
|
limitations.push(...health.failures);
|
|
890
|
+
for (const failure of health.failures)
|
|
891
|
+
report.diagnose(
|
|
892
|
+
"control_failure",
|
|
893
|
+
failure.fact,
|
|
894
|
+
candidate.entry.path,
|
|
895
|
+
units.flatMap((unit) =>
|
|
896
|
+
batch.scenarios
|
|
897
|
+
.filter((scenario) =>
|
|
898
|
+
health.unhealthyQuestionIds.has(
|
|
899
|
+
`${scenario.id}_${unit.id}`,
|
|
900
|
+
),
|
|
901
|
+
)
|
|
902
|
+
.map(
|
|
903
|
+
(scenario) =>
|
|
904
|
+
`coverage:${candidate.entry.path}:${scenario.id}:${unit.id}`,
|
|
905
|
+
),
|
|
906
|
+
),
|
|
907
|
+
false,
|
|
908
|
+
);
|
|
909
|
+
report.countControls(
|
|
910
|
+
result,
|
|
911
|
+
witnessQuestions.witnesses.map((witness) => witness.id),
|
|
912
|
+
);
|
|
913
|
+
for (const unit of units)
|
|
914
|
+
for (const scenario of batch.scenarios) {
|
|
915
|
+
const id = `coverage:${candidate.entry.path}:${scenario.id}:${unit.id}`;
|
|
916
|
+
report.expect(
|
|
917
|
+
id,
|
|
918
|
+
`${candidate.entry.path}: ${scenario.name} executes ${unit.name}`,
|
|
919
|
+
"scenario",
|
|
920
|
+
`coverage:${candidate.entry.path}:${scenario.id}`,
|
|
921
|
+
);
|
|
922
|
+
if (!result.ok)
|
|
923
|
+
report.failure(
|
|
924
|
+
result,
|
|
925
|
+
[id],
|
|
926
|
+
budget
|
|
927
|
+
? budget.kind === "session"
|
|
928
|
+
? "session_budget"
|
|
929
|
+
: "call_budget"
|
|
930
|
+
: !client
|
|
931
|
+
? "not_configured"
|
|
932
|
+
: undefined,
|
|
933
|
+
);
|
|
934
|
+
else
|
|
935
|
+
report.answer(
|
|
936
|
+
id,
|
|
937
|
+
result.answers[`${scenario.id}_${unit.id}`],
|
|
938
|
+
{
|
|
939
|
+
band: health.unhealthyQuestionIds.has(
|
|
940
|
+
`${scenario.id}_${unit.id}`,
|
|
941
|
+
)
|
|
942
|
+
? "unsure"
|
|
943
|
+
: "verdict",
|
|
944
|
+
reason: health.unhealthyQuestionIds.get(
|
|
945
|
+
`${scenario.id}_${unit.id}`,
|
|
946
|
+
),
|
|
947
|
+
},
|
|
948
|
+
controlsFor(
|
|
949
|
+
`${scenario.id}_${unit.id}`,
|
|
950
|
+
result,
|
|
951
|
+
witnessQuestions.witnesses.map((witness) => witness.id),
|
|
952
|
+
),
|
|
953
|
+
false,
|
|
954
|
+
);
|
|
955
|
+
}
|
|
623
956
|
for (const unit of units) {
|
|
624
957
|
if (!result.ok) {
|
|
625
958
|
fallback ||= !budget;
|
|
@@ -685,16 +1018,34 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
|
|
|
685
1018
|
evidenceLimits.length > 0 ||
|
|
686
1019
|
Boolean(args.paths?.length);
|
|
687
1020
|
for (const unit of residual)
|
|
688
|
-
if ((coverage.get(unit.id) ?? 0) < SELECT_MIN)
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
1021
|
+
if ((coverage.get(unit.id) ?? 0) < SELECT_MIN) {
|
|
1022
|
+
const supportingIds = [...report.items.keys()].filter(
|
|
1023
|
+
(id) => id.startsWith("coverage:") && id.endsWith(`:${unit.id}`),
|
|
1024
|
+
);
|
|
1025
|
+
report.diagnose(
|
|
1026
|
+
incomplete || incompleteUnits.has(unit.id)
|
|
1027
|
+
? "collection_omitted"
|
|
1028
|
+
: "conservative_widening",
|
|
1029
|
+
`Derived from retained coverage decisions: no discovered test established execution of ${unit.name} (${unit.file}) within the considered inventory only.${incomplete || incompleteUnits.has(unit.id) ? " Absence remains unsure because evidence, scope, or controls are incomplete." : " This is not a global coverage claim."}`,
|
|
1030
|
+
unit.file,
|
|
1031
|
+
supportingIds,
|
|
1032
|
+
false,
|
|
1033
|
+
);
|
|
1034
|
+
if (!supportingIds.length) {
|
|
1035
|
+
const diagnostic = report.diagnostics.at(-1);
|
|
1036
|
+
if (diagnostic)
|
|
1037
|
+
diagnostic.scope = {
|
|
1038
|
+
kind: "inventory",
|
|
1039
|
+
inventoryIds: ["test-candidates"],
|
|
1040
|
+
};
|
|
1041
|
+
const action = report.actions.at(-1);
|
|
1042
|
+
if (action)
|
|
1043
|
+
action.scope = {
|
|
1044
|
+
kind: "inventory",
|
|
1045
|
+
inventoryIds: ["test-candidates"],
|
|
1046
|
+
};
|
|
1047
|
+
}
|
|
1048
|
+
}
|
|
698
1049
|
if (fallback) {
|
|
699
1050
|
selected.length = 0;
|
|
700
1051
|
selected.push(
|
|
@@ -703,6 +1054,13 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
|
|
|
703
1054
|
scenarioIds: null,
|
|
704
1055
|
})),
|
|
705
1056
|
);
|
|
1057
|
+
report.diagnose(
|
|
1058
|
+
"conservative_widening",
|
|
1059
|
+
"Unavailable judgment conservatively selects all considered test candidates; static reachability exclusions do not narrow this fallback.",
|
|
1060
|
+
undefined,
|
|
1061
|
+
[],
|
|
1062
|
+
false,
|
|
1063
|
+
);
|
|
706
1064
|
limitations.push({
|
|
707
1065
|
fact: "fallback: all",
|
|
708
1066
|
next: "run all discovered tests with their project runner",
|
|
@@ -719,11 +1077,111 @@ export function createSelectTestsTool(dependencies: ToolDependencies) {
|
|
|
719
1077
|
})),
|
|
720
1078
|
),
|
|
721
1079
|
);
|
|
1080
|
+
for (const limit of commands.limits)
|
|
1081
|
+
for (const path of limit.files)
|
|
1082
|
+
report.diagnose("unresolved_runner", limit.reason, path, [], false);
|
|
1083
|
+
for (const limit of discovery.limits)
|
|
1084
|
+
report.diagnose(
|
|
1085
|
+
"unresolved_runner",
|
|
1086
|
+
limit.kind,
|
|
1087
|
+
limit.path,
|
|
1088
|
+
[],
|
|
1089
|
+
limit.kind !== "local_runner_unproven" &&
|
|
1090
|
+
limit.kind !== "interactive_script_skipped",
|
|
1091
|
+
);
|
|
1092
|
+
for (const limit of inventory.limits)
|
|
1093
|
+
report.diagnose(
|
|
1094
|
+
limit.kind === "secret_pattern"
|
|
1095
|
+
? "secret_pattern"
|
|
1096
|
+
: "collection_omitted",
|
|
1097
|
+
limit.kind,
|
|
1098
|
+
limit.path,
|
|
1099
|
+
);
|
|
1100
|
+
for (const limit of diff.limits)
|
|
1101
|
+
report.diagnose(
|
|
1102
|
+
limit.kind === "secret_pattern"
|
|
1103
|
+
? "secret_pattern"
|
|
1104
|
+
: "collection_omitted",
|
|
1105
|
+
limit.kind,
|
|
1106
|
+
limit.file,
|
|
1107
|
+
);
|
|
1108
|
+
for (const limit of graph.limits)
|
|
1109
|
+
report.diagnose(
|
|
1110
|
+
"dynamic_dependency",
|
|
1111
|
+
`${limit.kind}${limit.specifier ? ` (${limit.specifier})` : ""}`,
|
|
1112
|
+
limit.path,
|
|
1113
|
+
[],
|
|
1114
|
+
false,
|
|
1115
|
+
);
|
|
1116
|
+
for (const item of report.items.values()) {
|
|
1117
|
+
const candidate = candidates.find(
|
|
1118
|
+
(candidate) =>
|
|
1119
|
+
item.id.startsWith(`test:${candidate.entry.path}:`) ||
|
|
1120
|
+
item.id.startsWith(`coverage:${candidate.entry.path}:`),
|
|
1121
|
+
);
|
|
1122
|
+
if (!candidate) continue;
|
|
1123
|
+
const plan = selected.find((plan) => plan.entry === candidate.entry);
|
|
1124
|
+
const scenario = candidate.entry.scenarios.find(
|
|
1125
|
+
(scenario) =>
|
|
1126
|
+
item.id === `test:${candidate.entry.path}:${scenario.id}` ||
|
|
1127
|
+
item.id.startsWith(
|
|
1128
|
+
`coverage:${candidate.entry.path}:${scenario.id}:`,
|
|
1129
|
+
),
|
|
1130
|
+
);
|
|
1131
|
+
const isSelected = Boolean(
|
|
1132
|
+
plan &&
|
|
1133
|
+
(plan.scenarioIds === null ||
|
|
1134
|
+
(scenario && plan.scenarioIds.includes(scenario.id))),
|
|
1135
|
+
);
|
|
1136
|
+
item.selection = {
|
|
1137
|
+
selected: isSelected,
|
|
1138
|
+
reason: fallback
|
|
1139
|
+
? "conservative_fallback"
|
|
1140
|
+
: item.treatment === "static"
|
|
1141
|
+
? item.staticReason
|
|
1142
|
+
: item.treatment === "not_judged"
|
|
1143
|
+
? "conservative_fallback"
|
|
1144
|
+
: "unchanged selection policy and whole-file widening",
|
|
1145
|
+
};
|
|
1146
|
+
}
|
|
1147
|
+
for (const criterion of report.inventories[0]?.criteria ?? [])
|
|
1148
|
+
criterion.diagnosticIds = report.diagnostics
|
|
1149
|
+
.filter(
|
|
1150
|
+
(diagnostic) =>
|
|
1151
|
+
diagnostic.cause === "criteria_no_match" &&
|
|
1152
|
+
diagnostic.target.status === "known" &&
|
|
1153
|
+
diagnostic.target.value === criterion.criterion,
|
|
1154
|
+
)
|
|
1155
|
+
.map((diagnostic) => diagnostic.id);
|
|
1156
|
+
report.actions.push({
|
|
1157
|
+
id: "execute-selection-plan",
|
|
1158
|
+
code: "execute_plan",
|
|
1159
|
+
target: reportKnown(inventory.cwd),
|
|
1160
|
+
scope: { kind: "call" },
|
|
1161
|
+
condition:
|
|
1162
|
+
"When verification is authorized and the listed runner/configuration is available.",
|
|
1163
|
+
instruction:
|
|
1164
|
+
"Execute the retained runner commands separately; this tool has not executed any test.",
|
|
1165
|
+
repeatUnchanged: false,
|
|
1166
|
+
});
|
|
722
1167
|
if (skipped)
|
|
723
1168
|
limitations.push({
|
|
724
1169
|
fact: `${skipped} tests skipped because their imports cannot reach the diff`,
|
|
725
1170
|
next: "an alias or dynamic import would have kept a test in",
|
|
726
1171
|
});
|
|
1172
|
+
for (const criterion of selectionEvidence.criteria)
|
|
1173
|
+
if (!criterion.matches.length)
|
|
1174
|
+
limitations.push({
|
|
1175
|
+
path: criterion.criterion,
|
|
1176
|
+
cause: "selection criterion has no discovered test match",
|
|
1177
|
+
fact: `selection criterion without match: ${criterion.criterion}`,
|
|
1178
|
+
next: "Change the criterion or supply the missing test/configuration evidence; an empty scope does not prove absence of impact.",
|
|
1179
|
+
});
|
|
1180
|
+
if (selectionEvidence.wideningTriggers.length)
|
|
1181
|
+
limitations.push({
|
|
1182
|
+
fact: `selection widened conservatively: ${selectionEvidence.wideningTriggers.join(", ")}`,
|
|
1183
|
+
next: "Run the widened static plans; dependency reachability is not established for these changed files.",
|
|
1184
|
+
});
|
|
727
1185
|
return finish({
|
|
728
1186
|
answers,
|
|
729
1187
|
unjudged,
|