auditor-lambda 0.11.1 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/confirmIntentStep.d.ts +13 -0
- package/dist/cli/confirmIntentStep.js +68 -0
- package/dist/cli/dispatch.d.ts +1 -0
- package/dist/cli/dispatch.js +6 -2
- package/dist/cli/nextStepCommand.js +32 -0
- package/dist/cli/steps.d.ts +1 -1
- package/dist/orchestrator/advance.js +2 -2
- package/dist/orchestrator/executors.js +2 -2
- package/dist/orchestrator/intentCheckpointExecutor.d.ts +31 -1
- package/dist/orchestrator/intentCheckpointExecutor.js +44 -13
- package/dist/orchestrator/nextStep.js +1 -0
- package/dist/orchestrator/planningExecutors.js +7 -1
- package/dist/orchestrator/scope.d.ts +12 -0
- package/dist/orchestrator/scope.js +33 -0
- package/dist/orchestrator/state.js +1 -0
- package/dist/orchestrator/synthesisExecutors.js +9 -3
- package/dist/prompts/renderWorkerPrompt.js +1 -0
- package/dist/reporting/agentReflections.d.ts +38 -0
- package/dist/reporting/agentReflections.js +162 -0
- package/dist/reporting/synthesis.d.ts +13 -0
- package/dist/reporting/synthesis.js +11 -0
- package/dist/validation/artifacts.js +9 -0
- package/dist/validation/sessionConfig.js +6 -3
- package/package.json +1 -1
- package/schemas/agent_reflection.schema.json +44 -0
- package/schemas/intent_checkpoint.schema.json +77 -0
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { ScopePreDigest } from "../orchestrator/intentCheckpointExecutor.js";
|
|
2
|
+
/**
|
|
3
|
+
* Render the host-facing prompt for the `confirm_intent` step. Shows the
|
|
4
|
+
* deterministically-computed scope picture and asks the host to write (or
|
|
5
|
+
* refine) `intent_checkpoint.json` — confirming scope/intent and optionally
|
|
6
|
+
* adding exclusions the disposition pass missed (the scope-pollution case),
|
|
7
|
+
* must-not-touch globs, and free-form audit intent that is threaded into
|
|
8
|
+
* worker prompts.
|
|
9
|
+
*/
|
|
10
|
+
export declare function renderConfirmIntentPrompt(preDigest: ScopePreDigest, opts: {
|
|
11
|
+
intentCheckpointPath: string;
|
|
12
|
+
continueCommand: string;
|
|
13
|
+
}): string;
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Render the host-facing prompt for the `confirm_intent` step. Shows the
|
|
3
|
+
* deterministically-computed scope picture and asks the host to write (or
|
|
4
|
+
* refine) `intent_checkpoint.json` — confirming scope/intent and optionally
|
|
5
|
+
* adding exclusions the disposition pass missed (the scope-pollution case),
|
|
6
|
+
* must-not-touch globs, and free-form audit intent that is threaded into
|
|
7
|
+
* worker prompts.
|
|
8
|
+
*/
|
|
9
|
+
export function renderConfirmIntentPrompt(preDigest, opts) {
|
|
10
|
+
const dirLines = preDigest.scope_dirs
|
|
11
|
+
.slice(0, 20)
|
|
12
|
+
.map((d) => `- \`${d.dir}\` — ${d.files} file(s)`)
|
|
13
|
+
.join("\n") || "_(none)_";
|
|
14
|
+
const excludedLines = preDigest.auto_excluded.length > 0
|
|
15
|
+
? preDigest.auto_excluded
|
|
16
|
+
.map((e) => `- \`${e.path}\` (${e.status})`)
|
|
17
|
+
.join("\n")
|
|
18
|
+
: "_(none)_";
|
|
19
|
+
return [
|
|
20
|
+
"# Confirm Audit Scope and Intent",
|
|
21
|
+
"",
|
|
22
|
+
"Before planning, confirm what this audit should cover. The scope below was",
|
|
23
|
+
"discovered deterministically from intake. Your job is to **confirm it** and,",
|
|
24
|
+
"if needed, **prune scope pollution** the automatic disposition missed (build",
|
|
25
|
+
"output, vendored code, fixtures, generated files, scratch directories).",
|
|
26
|
+
"",
|
|
27
|
+
`**Mode:** ${preDigest.mode}${preDigest.since ? ` (since ${preDigest.since})` : ""}`,
|
|
28
|
+
`**Files in scope:** ${preDigest.files_in_scope}`,
|
|
29
|
+
"",
|
|
30
|
+
"## In-scope top-level directories",
|
|
31
|
+
"",
|
|
32
|
+
dirLines,
|
|
33
|
+
"",
|
|
34
|
+
"## Already excluded (deterministic disposition)",
|
|
35
|
+
"",
|
|
36
|
+
excludedLines,
|
|
37
|
+
"",
|
|
38
|
+
"## What to do",
|
|
39
|
+
"",
|
|
40
|
+
"Write `intent_checkpoint.json` to:",
|
|
41
|
+
"",
|
|
42
|
+
` ${opts.intentCheckpointPath}`,
|
|
43
|
+
"",
|
|
44
|
+
"Use this shape (only `scope_summary` and `intent_summary` are required; add",
|
|
45
|
+
"the optional fields to constrain the run):",
|
|
46
|
+
"",
|
|
47
|
+
"```json",
|
|
48
|
+
"{",
|
|
49
|
+
' "schema_version": "intent-checkpoint/v1",',
|
|
50
|
+
' "confirmed_at": "<ISO-8601 timestamp>",',
|
|
51
|
+
' "confirmed_by": "host",',
|
|
52
|
+
' "scope_summary": "<what is in scope>",',
|
|
53
|
+
' "intent_summary": "<the goal, e.g. full-audit / security-focused>",',
|
|
54
|
+
' "free_form_intent": "<optional: what to focus on; threaded into worker prompts>",',
|
|
55
|
+
' "excluded_scope": [{ "path": "<path or prefix>", "reason": "<why>" }],',
|
|
56
|
+
' "must_not_touch": ["<glob>"]',
|
|
57
|
+
"}",
|
|
58
|
+
"```",
|
|
59
|
+
"",
|
|
60
|
+
"- `excluded_scope` entries are pruned from planning so excluded files never",
|
|
61
|
+
" become audit tasks, and they are listed in the final report under",
|
|
62
|
+
' "Excluded / Out-of-Scope".',
|
|
63
|
+
"- Leave the optional fields out to audit the full discovered scope.",
|
|
64
|
+
"",
|
|
65
|
+
`Then run: ${opts.continueCommand}`,
|
|
66
|
+
"",
|
|
67
|
+
].join("\n");
|
|
68
|
+
}
|
package/dist/cli/dispatch.d.ts
CHANGED
package/dist/cli/dispatch.js
CHANGED
|
@@ -303,13 +303,17 @@ export function buildTaskSections(packetTasks, lensDefs, lineIndex) {
|
|
|
303
303
|
* Wraps the 75-line array-join block and returns the assembled prompt string.
|
|
304
304
|
*/
|
|
305
305
|
export function buildPacketPrompt(params) {
|
|
306
|
-
const { packet, fileList, largeFileSection, taskSections, submitCommand, repoRoot } = params;
|
|
306
|
+
const { packet, fileList, largeFileSection, taskSections, submitCommand, repoRoot, freeFormIntent } = params;
|
|
307
307
|
const largeFileMode = isIsolatedLargeFilePacket(packet);
|
|
308
|
+
const intentSection = freeFormIntent?.trim()
|
|
309
|
+
? ["## Audit intent", freeFormIntent.trim(), ""]
|
|
310
|
+
: [];
|
|
308
311
|
return [
|
|
309
312
|
"You are a code auditor. Review this packet once, then submit exactly one result per listed task.",
|
|
310
313
|
repoRoot ? `Repository root: ${repoRoot}` : "Repository root: use the root from the step contract.",
|
|
311
314
|
"Set the shell/tool workdir to the repository root when running backend commands.",
|
|
312
315
|
"",
|
|
316
|
+
...intentSection,
|
|
313
317
|
"## Packet",
|
|
314
318
|
`packet_id: ${packet.packet_id}`,
|
|
315
319
|
`task_count: ${packet.task_ids.length}`,
|
|
@@ -642,7 +646,7 @@ export async function prepareDispatchArtifacts(params) {
|
|
|
642
646
|
result_path: resultPathByTaskId.get(task.task_id),
|
|
643
647
|
});
|
|
644
648
|
}
|
|
645
|
-
const prompt = buildPacketPrompt({ packet, packetTasks, fileList, largeFileSection, taskSections, submitCommand, repoRoot: reviewRoot });
|
|
649
|
+
const prompt = buildPacketPrompt({ packet, packetTasks, fileList, largeFileSection, taskSections, submitCommand, repoRoot: reviewRoot, freeFormIntent: bundle.intent_checkpoint?.free_form_intent });
|
|
646
650
|
await writeFile(promptPath, prompt, "utf8");
|
|
647
651
|
const packetWritePaths = packetTasks
|
|
648
652
|
.map((task) => resultPathByTaskId.get(task.task_id))
|
|
@@ -10,6 +10,7 @@ import { deriveAuditState } from "../orchestrator/state.js";
|
|
|
10
10
|
import { checkFileIntegrity } from "../orchestrator/fileIntegrity.js";
|
|
11
11
|
import { buildEdgeReasoningPrompt, collectLowConfidenceEdges, edgeReasoningContentHash, } from "../orchestrator/edgeReasoning.js";
|
|
12
12
|
import { renderDesignReviewPrompt } from "../orchestrator/designReviewPrompt.js";
|
|
13
|
+
import { computeScopePreDigest } from "../orchestrator/intentCheckpointExecutor.js";
|
|
13
14
|
import { renderSynthesisNarrativePrompt } from "../reporting/synthesisNarrativePrompt.js";
|
|
14
15
|
import { buildPathLookup } from "../extractors/graph.js";
|
|
15
16
|
import { buildDispositionMap } from "../extractors/disposition.js";
|
|
@@ -21,6 +22,7 @@ import { runAuditStep } from "./auditStep.js";
|
|
|
21
22
|
import { writeHandoffOnly, ensureSemanticReviewRun, persistConfigErrorHandoff, } from "./reviewRun.js";
|
|
22
23
|
import { buildPendingAuditTasks } from "./dispatch.js";
|
|
23
24
|
import { renderSemanticReviewStep } from "./semanticReviewStep.js";
|
|
25
|
+
import { renderConfirmIntentPrompt } from "./confirmIntentStep.js";
|
|
24
26
|
import { writeCurrentStep } from "./steps.js";
|
|
25
27
|
import { nextStepCommand, renderAnalyzerInstallPrompt, renderBlockedStepPrompt, renderEdgeReasoningDispatchPrompt, renderEdgeReasoningStepPrompt, renderPresentReportPrompt, } from "./prompts.js";
|
|
26
28
|
import { getArtifactsDir, getFlag, getHostMaxActiveSubagents, getMaxRuns, getOptionalBooleanFlag, getRootDir, getTimeoutMs, resolveHostDispatchCapability, warnIfNotGitRepo, } from "./args.js";
|
|
@@ -344,6 +346,13 @@ async function runDeterministicForNextStep(params) {
|
|
|
344
346
|
continue;
|
|
345
347
|
return branch.result;
|
|
346
348
|
}
|
|
349
|
+
// Confirm-intent host step: when the checkpoint is missing, hand control to
|
|
350
|
+
// the host to confirm scope/intent. The host writes intent_checkpoint.json
|
|
351
|
+
// (detected by deriveAuditState on re-invocation), so there is no incoming
|
|
352
|
+
// artifact to consume — emit the step directly.
|
|
353
|
+
if (decision.selected_executor === "intent_checkpoint_executor") {
|
|
354
|
+
return { kind: "confirm_intent", state, bundle };
|
|
355
|
+
}
|
|
347
356
|
if (isHostDelegationExecutor(decision.selected_executor ?? "")) {
|
|
348
357
|
return {
|
|
349
358
|
kind: "semantic_review",
|
|
@@ -532,6 +541,29 @@ export async function cmdNextStep(argv) {
|
|
|
532
541
|
console.log(JSON.stringify(step, null, 2));
|
|
533
542
|
return;
|
|
534
543
|
}
|
|
544
|
+
if (result.kind === "confirm_intent") {
|
|
545
|
+
const intentCheckpointPath = join(artifactsDir, "intent_checkpoint.json");
|
|
546
|
+
const continueCommand = nextStepCommand(root, artifactsDir);
|
|
547
|
+
const preDigest = computeScopePreDigest(result.bundle, root, getFlag(argv, "--since"));
|
|
548
|
+
const step = await writeCurrentStep({
|
|
549
|
+
artifactsDir,
|
|
550
|
+
stepKind: "confirm_intent",
|
|
551
|
+
status: "ready",
|
|
552
|
+
runId: null,
|
|
553
|
+
allowedCommands: [continueCommand],
|
|
554
|
+
stopCondition: "Write intent_checkpoint.json with the confirmed scope and intent, then run next-step.",
|
|
555
|
+
repoRoot: root,
|
|
556
|
+
artifactPaths: {
|
|
557
|
+
intent_checkpoint: intentCheckpointPath,
|
|
558
|
+
},
|
|
559
|
+
prompt: renderConfirmIntentPrompt(preDigest, {
|
|
560
|
+
intentCheckpointPath,
|
|
561
|
+
continueCommand,
|
|
562
|
+
}),
|
|
563
|
+
});
|
|
564
|
+
console.log(JSON.stringify(step, null, 2));
|
|
565
|
+
return;
|
|
566
|
+
}
|
|
535
567
|
if (result.kind === "analyzer_install") {
|
|
536
568
|
const decisionsPath = join(artifactsDir, "incoming", "analyzer-decisions.json");
|
|
537
569
|
await mkdir(join(artifactsDir, "incoming"), { recursive: true });
|
package/dist/cli/steps.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { StepStatus } from "@audit-tools/shared";
|
|
2
2
|
import type { AccessDeclaration } from "../types/workerSession.js";
|
|
3
3
|
export declare const STEP_CONTRACT_VERSION = "audit-code-step/v1alpha1";
|
|
4
|
-
export type StepKind = "dispatch_review" | "single_task_fallback" | "design_review" | "analyzer_install" | "edge_reasoning" | "edge_reasoning_dispatch" | "synthesis_narrative" | "present_report" | "blocked";
|
|
4
|
+
export type StepKind = "dispatch_review" | "single_task_fallback" | "design_review" | "confirm_intent" | "analyzer_install" | "edge_reasoning" | "edge_reasoning_dispatch" | "synthesis_narrative" | "present_report" | "blocked";
|
|
5
5
|
/**
|
|
6
6
|
* Lightweight run-level orientation surfaced in the step contract so a host
|
|
7
7
|
* resuming an in-flight audit knows where it stands without reading artifacts.
|
|
@@ -3,7 +3,7 @@ import { decideNextStep, findObligation } from "./nextStep.js";
|
|
|
3
3
|
import { deriveAuditState } from "./state.js";
|
|
4
4
|
import { computeArtifactMetadata } from "./artifactMetadata.js";
|
|
5
5
|
import { runIntakeExecutor } from "./intakeExecutors.js";
|
|
6
|
-
import {
|
|
6
|
+
import { runIntentCheckpointAutoComplete } from "./intentCheckpointExecutor.js";
|
|
7
7
|
import { runStructureExecutor, runDesignAssessmentExecutor, runDesignReviewAutoComplete, } from "./structureExecutors.js";
|
|
8
8
|
import { runPlanningExecutor } from "./planningExecutors.js";
|
|
9
9
|
import { runResultIngestionExecutor, runRuntimeValidationExecutor, runRuntimeValidationUpdateExecutor, runExternalAnalyzerImportExecutor, } from "./ingestionExecutors.js";
|
|
@@ -94,7 +94,7 @@ export async function advanceAudit(bundle, options = {}) {
|
|
|
94
94
|
}
|
|
95
95
|
case "intent_checkpoint_executor": {
|
|
96
96
|
const root = requireRoot(options.root, "intent_checkpoint_executor");
|
|
97
|
-
run =
|
|
97
|
+
run = runIntentCheckpointAutoComplete(bundle, root, options.since);
|
|
98
98
|
break;
|
|
99
99
|
}
|
|
100
100
|
case "structure_executor":
|
|
@@ -16,9 +16,9 @@ export const EXECUTOR_REGISTRY = [
|
|
|
16
16
|
},
|
|
17
17
|
{
|
|
18
18
|
id: "intent_checkpoint_executor",
|
|
19
|
-
kind: "
|
|
19
|
+
kind: "host_delegation",
|
|
20
20
|
obligation_ids: ["intent_checkpoint_current"],
|
|
21
|
-
description: "
|
|
21
|
+
description: "Pause for the host to confirm scope and intent (the confirm_intent step writes intent_checkpoint.json); deterministic auto-complete writes a default full-scope checkpoint when run headless.",
|
|
22
22
|
},
|
|
23
23
|
{
|
|
24
24
|
id: "structure_executor",
|
|
@@ -1,3 +1,33 @@
|
|
|
1
1
|
import type { ArtifactBundle } from "../io/artifacts.js";
|
|
2
2
|
import type { ExecutorRunResult } from "./executorResult.js";
|
|
3
|
-
|
|
3
|
+
/**
|
|
4
|
+
* Deterministic pre-digest of the audit scope, shown to the host in the
|
|
5
|
+
* `confirm_intent` step and used to seed the headless auto-complete checkpoint.
|
|
6
|
+
* Everything here is computed deterministically from the intake artifacts; the
|
|
7
|
+
* host uses it to confirm the discovered scope and add any exclusions the
|
|
8
|
+
* disposition pass missed (the scope-pollution case).
|
|
9
|
+
*/
|
|
10
|
+
export interface ScopePreDigest {
|
|
11
|
+
mode: "full" | "delta";
|
|
12
|
+
since: string | null;
|
|
13
|
+
files_in_scope: number;
|
|
14
|
+
/** Top-level directories of in-scope files, with file counts (desc). */
|
|
15
|
+
scope_dirs: Array<{
|
|
16
|
+
dir: string;
|
|
17
|
+
files: number;
|
|
18
|
+
}>;
|
|
19
|
+
/** A sample of files already excluded by the deterministic disposition pass. */
|
|
20
|
+
auto_excluded: Array<{
|
|
21
|
+
path: string;
|
|
22
|
+
status: string;
|
|
23
|
+
}>;
|
|
24
|
+
}
|
|
25
|
+
export declare function computeScopePreDigest(bundle: ArtifactBundle, root: string, since?: string): ScopePreDigest;
|
|
26
|
+
/**
|
|
27
|
+
* Headless deterministic fallback for the intent checkpoint — the analog of
|
|
28
|
+
* `runDesignReviewAutoComplete`. The conversation-first flow instead emits a
|
|
29
|
+
* `confirm_intent` host step (see `cli/confirmIntentStep.ts`); this runs only
|
|
30
|
+
* when `advanceAudit` is driven headlessly with no host to confirm scope,
|
|
31
|
+
* writing a default full-scope checkpoint so the pipeline can proceed.
|
|
32
|
+
*/
|
|
33
|
+
export declare function runIntentCheckpointAutoComplete(bundle: ArtifactBundle, root: string, since?: string): ExecutorRunResult;
|
|
@@ -1,29 +1,60 @@
|
|
|
1
1
|
import { resolveAuditScope } from "./scope.js";
|
|
2
2
|
import { isAuditExcludedStatus } from "../extractors/disposition.js";
|
|
3
|
-
|
|
3
|
+
const AUTO_EXCLUDED_SAMPLE_LIMIT = 25;
|
|
4
|
+
export function computeScopePreDigest(bundle, root, since) {
|
|
4
5
|
const scope = resolveAuditScope({ root, since, bundle });
|
|
5
|
-
|
|
6
|
+
const dispositionFiles = bundle.file_disposition?.files ?? [];
|
|
7
|
+
const auditable = dispositionFiles.filter((file) => !isAuditExcludedStatus(file.status));
|
|
8
|
+
const excluded = dispositionFiles.filter((file) => isAuditExcludedStatus(file.status));
|
|
9
|
+
let inScopePaths;
|
|
6
10
|
if (scope.mode === "delta") {
|
|
7
|
-
|
|
11
|
+
inScopePaths = [...scope.seed_files, ...scope.expanded_files];
|
|
12
|
+
}
|
|
13
|
+
else if (auditable.length > 0) {
|
|
14
|
+
inScopePaths = auditable.map((file) => file.path);
|
|
8
15
|
}
|
|
9
16
|
else {
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
17
|
+
inScopePaths = bundle.repo_manifest?.files.map((file) => file.path) ?? [];
|
|
18
|
+
}
|
|
19
|
+
const dirCounts = new Map();
|
|
20
|
+
for (const path of inScopePaths) {
|
|
21
|
+
const top = path.split(/[\\/]/)[0] || ".";
|
|
22
|
+
dirCounts.set(top, (dirCounts.get(top) ?? 0) + 1);
|
|
13
23
|
}
|
|
24
|
+
const scope_dirs = [...dirCounts.entries()]
|
|
25
|
+
.map(([dir, files]) => ({ dir, files }))
|
|
26
|
+
.sort((a, b) => b.files - a.files);
|
|
27
|
+
return {
|
|
28
|
+
mode: scope.mode === "delta" ? "delta" : "full",
|
|
29
|
+
since: scope.since ?? null,
|
|
30
|
+
files_in_scope: inScopePaths.length,
|
|
31
|
+
scope_dirs,
|
|
32
|
+
auto_excluded: excluded
|
|
33
|
+
.slice(0, AUTO_EXCLUDED_SAMPLE_LIMIT)
|
|
34
|
+
.map((file) => ({ path: file.path, status: file.status })),
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Headless deterministic fallback for the intent checkpoint — the analog of
|
|
39
|
+
* `runDesignReviewAutoComplete`. The conversation-first flow instead emits a
|
|
40
|
+
* `confirm_intent` host step (see `cli/confirmIntentStep.ts`); this runs only
|
|
41
|
+
* when `advanceAudit` is driven headlessly with no host to confirm scope,
|
|
42
|
+
* writing a default full-scope checkpoint so the pipeline can proceed.
|
|
43
|
+
*/
|
|
44
|
+
export function runIntentCheckpointAutoComplete(bundle, root, since) {
|
|
45
|
+
const preDigest = computeScopePreDigest(bundle, root, since);
|
|
14
46
|
const intent = {
|
|
15
47
|
schema_version: "intent-checkpoint/v1",
|
|
16
48
|
confirmed_at: new Date().toISOString(),
|
|
17
|
-
scope_summary: `Root: ${root}${scope.since ? ` (since ${scope.since})` : ""}, files in scope: ${filesInScope}`,
|
|
18
|
-
intent_summary: scope.mode === "delta" ? `delta-audit since ${scope.since}` : "full-audit",
|
|
19
49
|
confirmed_by: "host",
|
|
50
|
+
scope_summary: `Root: ${root}${preDigest.since ? ` (since ${preDigest.since})` : ""}, files in scope: ${preDigest.files_in_scope}`,
|
|
51
|
+
intent_summary: preDigest.mode === "delta"
|
|
52
|
+
? `delta-audit since ${preDigest.since}`
|
|
53
|
+
: "full-audit",
|
|
20
54
|
};
|
|
21
55
|
return {
|
|
22
|
-
updated: {
|
|
23
|
-
...bundle,
|
|
24
|
-
intent_checkpoint: intent,
|
|
25
|
-
},
|
|
56
|
+
updated: { ...bundle, intent_checkpoint: intent },
|
|
26
57
|
artifacts_written: ["intent_checkpoint.json"],
|
|
27
|
-
progress_summary: `
|
|
58
|
+
progress_summary: `Auto-completed scope/intent checkpoint (headless): ${intent.scope_summary} (${intent.intent_summary}).`,
|
|
28
59
|
};
|
|
29
60
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { initializeCoverageFromPlan } from "./planning.js";
|
|
2
|
-
import { applyScopeToCoverage, fullAuditScope } from "./scope.js";
|
|
2
|
+
import { applyIntentExclusionsToCoverage, applyScopeToCoverage, fullAuditScope, } from "./scope.js";
|
|
3
3
|
import { buildFlowCoverage } from "./flowCoverage.js";
|
|
4
4
|
import { buildRequeuePayload } from "./requeueCommand.js";
|
|
5
5
|
import { buildRuntimeValidationTasks, discoverRuntimeValidationCommand, mergeRuntimeValidationReport, } from "./runtimeValidation.js";
|
|
@@ -25,6 +25,9 @@ export async function runPlanningExecutor(bundle, root, lineIndex = {}, sizeInde
|
|
|
25
25
|
// Delta scope: only seed + expanded files stay pending; the rest inherit prior
|
|
26
26
|
// completion or are excluded from this run. Full scope is a no-op.
|
|
27
27
|
applyScopeToCoverage(coverage, resolvedScope, bundle.coverage_matrix);
|
|
28
|
+
// Layer the host-confirmed intent exclusions on top of disposition + scope so
|
|
29
|
+
// user-pruned scope pollution never becomes an audit task.
|
|
30
|
+
const intentExcludedPaths = applyIntentExclusionsToCoverage(coverage, bundle.intent_checkpoint?.excluded_scope);
|
|
28
31
|
const flowCoverage = buildFlowCoverage(bundle.critical_flows, coverage);
|
|
29
32
|
const runtimeCommand = await discoverRuntimeValidationCommand(root);
|
|
30
33
|
const runtimeValidationTasks = buildRuntimeValidationTasks({
|
|
@@ -103,6 +106,9 @@ export async function runPlanningExecutor(bundle, root, lineIndex = {}, sizeInde
|
|
|
103
106
|
(skippedTrivialPaths.length > 0
|
|
104
107
|
? ` Skipped ${skippedTrivialPaths.length} trivial path${skippedTrivialPaths.length === 1 ? "" : "s"} from semantic review.`
|
|
105
108
|
: "") +
|
|
109
|
+
(intentExcludedPaths.length > 0
|
|
110
|
+
? ` Excluded ${intentExcludedPaths.length} path${intentExcludedPaths.length === 1 ? "" : "s"} per the intent checkpoint.`
|
|
111
|
+
: "") +
|
|
106
112
|
(runtimeCommand
|
|
107
113
|
? ` Runtime validation will use: ${runtimeCommand.join(" ")}.`
|
|
108
114
|
: " No deterministic runtime validation command was discovered."),
|
|
@@ -60,3 +60,15 @@ export declare function resolveAuditScope(input: ResolveAuditScopeInput): AuditS
|
|
|
60
60
|
* exclusions (non-auditable/trivial) are left untouched. A full scope is a no-op.
|
|
61
61
|
*/
|
|
62
62
|
export declare function applyScopeToCoverage(coverage: CoverageMatrix, scope: AuditScopeManifest, priorCoverage?: CoverageMatrix): CoverageMatrix;
|
|
63
|
+
/**
|
|
64
|
+
* Apply the intent checkpoint's `excluded_scope` to a coverage matrix: any file
|
|
65
|
+
* whose path matches an exclusion (exact or directory-prefix) is marked excluded
|
|
66
|
+
* so it never becomes an audit task. The user's exclusions layer on top of the
|
|
67
|
+
* deterministic disposition — they catch scope pollution the automatic pass
|
|
68
|
+
* missed. Returns the newly-excluded paths (for the run summary / report); a
|
|
69
|
+
* checkpoint with no exclusions is a no-op.
|
|
70
|
+
*/
|
|
71
|
+
export declare function applyIntentExclusionsToCoverage(coverage: CoverageMatrix, excludedScope: Array<{
|
|
72
|
+
path: string;
|
|
73
|
+
reason: string;
|
|
74
|
+
}> | undefined): string[];
|
|
@@ -225,3 +225,36 @@ export function applyScopeToCoverage(coverage, scope, priorCoverage) {
|
|
|
225
225
|
}
|
|
226
226
|
return coverage;
|
|
227
227
|
}
|
|
228
|
+
function pathMatchesExclusion(filePath, entryPath) {
|
|
229
|
+
const f = filePath.replace(/\\/g, "/");
|
|
230
|
+
const p = entryPath.replace(/\\/g, "/").replace(/\/+$/, "");
|
|
231
|
+
if (!p)
|
|
232
|
+
return false;
|
|
233
|
+
return f === p || f.startsWith(`${p}/`);
|
|
234
|
+
}
|
|
235
|
+
/**
|
|
236
|
+
* Apply the intent checkpoint's `excluded_scope` to a coverage matrix: any file
|
|
237
|
+
* whose path matches an exclusion (exact or directory-prefix) is marked excluded
|
|
238
|
+
* so it never becomes an audit task. The user's exclusions layer on top of the
|
|
239
|
+
* deterministic disposition — they catch scope pollution the automatic pass
|
|
240
|
+
* missed. Returns the newly-excluded paths (for the run summary / report); a
|
|
241
|
+
* checkpoint with no exclusions is a no-op.
|
|
242
|
+
*/
|
|
243
|
+
export function applyIntentExclusionsToCoverage(coverage, excludedScope) {
|
|
244
|
+
if (!excludedScope || excludedScope.length === 0)
|
|
245
|
+
return [];
|
|
246
|
+
const excluded = [];
|
|
247
|
+
for (const file of coverage.files) {
|
|
248
|
+
if (file.audit_status === "excluded")
|
|
249
|
+
continue;
|
|
250
|
+
if (excludedScope.some((entry) => pathMatchesExclusion(file.path, entry.path))) {
|
|
251
|
+
file.required_lenses = [];
|
|
252
|
+
file.completed_lenses = [];
|
|
253
|
+
file.unit_ids = [];
|
|
254
|
+
file.audit_status = "excluded";
|
|
255
|
+
file.classification_status = "out_of_scope_intent";
|
|
256
|
+
excluded.push(file.path);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
return excluded;
|
|
260
|
+
}
|
|
@@ -32,6 +32,7 @@ export function deriveAuditState(bundle) {
|
|
|
32
32
|
obligations.push(obligation("graph_enrichment_current", staleOrSatisfied(staleArtifacts, ["analyzer_capability.json"], has(bundle.analyzer_capability))));
|
|
33
33
|
obligations.push(obligation("design_assessment_current", staleOrSatisfied(staleArtifacts, ["design_assessment.json"], has(bundle.design_assessment))));
|
|
34
34
|
obligations.push(obligation("design_review_completed", bundle.design_assessment?.reviewed ? "satisfied" : "missing"));
|
|
35
|
+
obligations.push(obligation("intent_checkpoint_current", staleOrSatisfied(staleArtifacts, ["intent_checkpoint.json"], has(bundle.intent_checkpoint))));
|
|
35
36
|
const planningReady = has(bundle.coverage_matrix) &&
|
|
36
37
|
has(bundle.flow_coverage) &&
|
|
37
38
|
has(bundle.runtime_validation_tasks) &&
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { applyNarrative, buildAuditFindingsReport, buildAuditReportModel, renderAuditReportMarkdown, } from "../reporting/synthesis.js";
|
|
2
2
|
function buildBaseFindingsReport(bundle, results) {
|
|
3
|
-
|
|
3
|
+
const report = buildAuditFindingsReport(buildAuditReportModel({
|
|
4
4
|
results,
|
|
5
5
|
unitManifest: bundle.unit_manifest,
|
|
6
6
|
graphBundle: bundle.graph_bundle,
|
|
@@ -11,6 +11,12 @@ function buildBaseFindingsReport(bundle, results) {
|
|
|
11
11
|
externalAnalyzerResults: bundle.external_analyzer_results,
|
|
12
12
|
designAssessment: bundle.design_assessment,
|
|
13
13
|
}));
|
|
14
|
+
// Record the host-confirmed exclusions in the machine contract so omissions
|
|
15
|
+
// are explicit and machine-readable, not just rendered in the markdown.
|
|
16
|
+
const excludedScope = bundle.intent_checkpoint?.excluded_scope;
|
|
17
|
+
return excludedScope && excludedScope.length > 0
|
|
18
|
+
? { ...report, excluded_scope: excludedScope }
|
|
19
|
+
: report;
|
|
14
20
|
}
|
|
15
21
|
export function runSynthesisExecutor(bundle, results) {
|
|
16
22
|
const finalResults = results ?? bundle.audit_results ?? [];
|
|
@@ -28,7 +34,7 @@ export function runSynthesisExecutor(bundle, results) {
|
|
|
28
34
|
updated: {
|
|
29
35
|
...bundle,
|
|
30
36
|
audit_findings: findings,
|
|
31
|
-
audit_report: renderAuditReportMarkdown(findings, { scope: bundle.scope }),
|
|
37
|
+
audit_report: renderAuditReportMarkdown(findings, { scope: bundle.scope, intent_checkpoint: bundle.intent_checkpoint }),
|
|
32
38
|
},
|
|
33
39
|
artifacts_written: ["audit-findings.json", "audit-report.md"],
|
|
34
40
|
progress_summary: `Rendered deterministic audit report and canonical findings for ${finalResults.length} audit result entries.`,
|
|
@@ -78,7 +84,7 @@ export function runSynthesisNarrativeExecutor(bundle, narrative) {
|
|
|
78
84
|
updated: {
|
|
79
85
|
...bundle,
|
|
80
86
|
audit_findings: enriched,
|
|
81
|
-
audit_report: renderAuditReportMarkdown(enriched, { scope: bundle.scope }),
|
|
87
|
+
audit_report: renderAuditReportMarkdown(enriched, { scope: bundle.scope, intent_checkpoint: bundle.intent_checkpoint }),
|
|
82
88
|
synthesis_narrative: record,
|
|
83
89
|
},
|
|
84
90
|
artifacts_written: [
|
|
@@ -49,6 +49,7 @@ export function renderWorkerPrompt(task) {
|
|
|
49
49
|
"'[' + (ConvertTo-Json $obj -Depth 12) + ']', or build the array with Write-Output -NoEnumerate.",
|
|
50
50
|
`Write only the JSON array of AuditResult objects to: ${task.audit_results_path}`,
|
|
51
51
|
];
|
|
52
|
+
lines.push("Optional — never let this delay or replace the audit result: if you hit task", "ambiguity, tool friction, or unclear instructions, you MAY append one JSON", `reflection line to ${task.artifacts_dir}/agent-feedback.jsonl with shape:`, " {task_id, lens, instruction_clarity (clear|mostly_clear|ambiguous|unclear),", " ambiguities: [string], tool_friction: [string], suggestions: [string],", " severity (info|low|medium|high)}. One object per line; never overwrite existing lines.");
|
|
52
53
|
if (usesDeferredWorkerCommand(task)) {
|
|
53
54
|
lines.push("Deferred mode: write results, do not execute worker_command.");
|
|
54
55
|
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
export type ReflectionClarity = "clear" | "mostly_clear" | "ambiguous" | "unclear";
|
|
2
|
+
export type ReflectionSeverity = "info" | "low" | "medium" | "high";
|
|
3
|
+
export interface AgentReflection {
|
|
4
|
+
task_id: string;
|
|
5
|
+
lens?: string;
|
|
6
|
+
instruction_clarity: ReflectionClarity;
|
|
7
|
+
ambiguities?: string[];
|
|
8
|
+
tool_friction?: string[];
|
|
9
|
+
suggestions?: string[];
|
|
10
|
+
severity: ReflectionSeverity;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Parse NDJSON reflection text, keeping only schema-valid objects. Blank lines,
|
|
14
|
+
* non-JSON lines, and objects missing the required `task_id`/`instruction_clarity`/
|
|
15
|
+
* `severity` (or with out-of-enum values) are skipped silently — the channel is
|
|
16
|
+
* opt-in and best-effort, so a bad reflection must never break synthesis.
|
|
17
|
+
*/
|
|
18
|
+
export declare function parseReflectionsNdjson(text: string): AgentReflection[];
|
|
19
|
+
export interface ReflectionAggregate {
|
|
20
|
+
total: number;
|
|
21
|
+
clarity_breakdown: Record<ReflectionClarity, number>;
|
|
22
|
+
severity_breakdown: Record<ReflectionSeverity, number>;
|
|
23
|
+
/** Deduped notes, highest reported impact first (ties broken alphabetically). */
|
|
24
|
+
friction: string[];
|
|
25
|
+
ambiguities: string[];
|
|
26
|
+
suggestions: string[];
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Tally clarity/severity and dedupe the free-text notes across reflections,
|
|
30
|
+
* ranking each distinct note by the highest severity it was reported under so the
|
|
31
|
+
* most impactful friction surfaces first.
|
|
32
|
+
*/
|
|
33
|
+
export declare function aggregateReflections(reflections: AgentReflection[]): ReflectionAggregate;
|
|
34
|
+
/**
|
|
35
|
+
* Render the "## Process Feedback" section. Returns `[]` when there are no
|
|
36
|
+
* reflections so the report omits the section entirely.
|
|
37
|
+
*/
|
|
38
|
+
export declare function renderProcessFeedbackSection(reflections: AgentReflection[]): string[];
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
// Agent meta-audit reflections: a canonical, opt-in feedback channel. Workers may
|
|
2
|
+
// append one reflection per task (NDJSON) to `agent-feedback.jsonl` — schema
|
|
3
|
+
// `schemas/agent_reflection.schema.json`. Synthesis aggregates them into a
|
|
4
|
+
// "Process Feedback" report section so recurring operational friction is visible
|
|
5
|
+
// without hand-reading the JSONL. The channel is best-effort: a malformed line is
|
|
6
|
+
// skipped, never fatal, and never competes with the actual audit obligation.
|
|
7
|
+
const CLARITY_VALUES = new Set([
|
|
8
|
+
"clear",
|
|
9
|
+
"mostly_clear",
|
|
10
|
+
"ambiguous",
|
|
11
|
+
"unclear",
|
|
12
|
+
]);
|
|
13
|
+
const SEVERITY_VALUES = new Set([
|
|
14
|
+
"info",
|
|
15
|
+
"low",
|
|
16
|
+
"medium",
|
|
17
|
+
"high",
|
|
18
|
+
]);
|
|
19
|
+
const SEVERITY_RANK = {
|
|
20
|
+
high: 3,
|
|
21
|
+
medium: 2,
|
|
22
|
+
low: 1,
|
|
23
|
+
info: 0,
|
|
24
|
+
};
|
|
25
|
+
function isStringArray(value) {
|
|
26
|
+
return Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Parse NDJSON reflection text, keeping only schema-valid objects. Blank lines,
|
|
30
|
+
* non-JSON lines, and objects missing the required `task_id`/`instruction_clarity`/
|
|
31
|
+
* `severity` (or with out-of-enum values) are skipped silently — the channel is
|
|
32
|
+
* opt-in and best-effort, so a bad reflection must never break synthesis.
|
|
33
|
+
*/
|
|
34
|
+
export function parseReflectionsNdjson(text) {
|
|
35
|
+
const reflections = [];
|
|
36
|
+
for (const rawLine of text.split(/\r?\n/)) {
|
|
37
|
+
const line = rawLine.trim();
|
|
38
|
+
if (line.length === 0)
|
|
39
|
+
continue;
|
|
40
|
+
let parsed;
|
|
41
|
+
try {
|
|
42
|
+
parsed = JSON.parse(line);
|
|
43
|
+
}
|
|
44
|
+
catch {
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
|
|
48
|
+
continue;
|
|
49
|
+
const record = parsed;
|
|
50
|
+
if (typeof record.task_id !== "string" || record.task_id.length === 0) {
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
if (typeof record.instruction_clarity !== "string" ||
|
|
54
|
+
!CLARITY_VALUES.has(record.instruction_clarity)) {
|
|
55
|
+
continue;
|
|
56
|
+
}
|
|
57
|
+
if (typeof record.severity !== "string" ||
|
|
58
|
+
!SEVERITY_VALUES.has(record.severity)) {
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
const reflection = {
|
|
62
|
+
task_id: record.task_id,
|
|
63
|
+
instruction_clarity: record.instruction_clarity,
|
|
64
|
+
severity: record.severity,
|
|
65
|
+
};
|
|
66
|
+
if (typeof record.lens === "string")
|
|
67
|
+
reflection.lens = record.lens;
|
|
68
|
+
if (isStringArray(record.ambiguities))
|
|
69
|
+
reflection.ambiguities = record.ambiguities;
|
|
70
|
+
if (isStringArray(record.tool_friction))
|
|
71
|
+
reflection.tool_friction = record.tool_friction;
|
|
72
|
+
if (isStringArray(record.suggestions))
|
|
73
|
+
reflection.suggestions = record.suggestions;
|
|
74
|
+
reflections.push(reflection);
|
|
75
|
+
}
|
|
76
|
+
return reflections;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Tally clarity/severity and dedupe the free-text notes across reflections,
|
|
80
|
+
* ranking each distinct note by the highest severity it was reported under so the
|
|
81
|
+
* most impactful friction surfaces first.
|
|
82
|
+
*/
|
|
83
|
+
export function aggregateReflections(reflections) {
|
|
84
|
+
const clarity_breakdown = {
|
|
85
|
+
clear: 0,
|
|
86
|
+
mostly_clear: 0,
|
|
87
|
+
ambiguous: 0,
|
|
88
|
+
unclear: 0,
|
|
89
|
+
};
|
|
90
|
+
const severity_breakdown = {
|
|
91
|
+
info: 0,
|
|
92
|
+
low: 0,
|
|
93
|
+
medium: 0,
|
|
94
|
+
high: 0,
|
|
95
|
+
};
|
|
96
|
+
const friction = new Map();
|
|
97
|
+
const ambiguities = new Map();
|
|
98
|
+
const suggestions = new Map();
|
|
99
|
+
const collect = (target, items, severity) => {
|
|
100
|
+
for (const item of items ?? []) {
|
|
101
|
+
const key = item.trim();
|
|
102
|
+
if (key.length === 0)
|
|
103
|
+
continue;
|
|
104
|
+
target.set(key, Math.max(target.get(key) ?? 0, SEVERITY_RANK[severity]));
|
|
105
|
+
}
|
|
106
|
+
};
|
|
107
|
+
for (const reflection of reflections) {
|
|
108
|
+
clarity_breakdown[reflection.instruction_clarity] += 1;
|
|
109
|
+
severity_breakdown[reflection.severity] += 1;
|
|
110
|
+
collect(friction, reflection.tool_friction, reflection.severity);
|
|
111
|
+
collect(ambiguities, reflection.ambiguities, reflection.severity);
|
|
112
|
+
collect(suggestions, reflection.suggestions, reflection.severity);
|
|
113
|
+
}
|
|
114
|
+
const rankedKeys = (target) => [...target.entries()]
|
|
115
|
+
.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
|
|
116
|
+
.map(([key]) => key);
|
|
117
|
+
return {
|
|
118
|
+
total: reflections.length,
|
|
119
|
+
clarity_breakdown,
|
|
120
|
+
severity_breakdown,
|
|
121
|
+
friction: rankedKeys(friction),
|
|
122
|
+
ambiguities: rankedKeys(ambiguities),
|
|
123
|
+
suggestions: rankedKeys(suggestions),
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
function formatCounts(counts) {
|
|
127
|
+
const parts = Object.entries(counts)
|
|
128
|
+
.filter(([, count]) => count > 0)
|
|
129
|
+
.map(([key, count]) => `${key}: ${count}`);
|
|
130
|
+
return parts.length > 0 ? parts.join(", ") : "none";
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Render the "## Process Feedback" section. Returns `[]` when there are no
|
|
134
|
+
* reflections so the report omits the section entirely.
|
|
135
|
+
*/
|
|
136
|
+
export function renderProcessFeedbackSection(reflections) {
|
|
137
|
+
if (reflections.length === 0)
|
|
138
|
+
return [];
|
|
139
|
+
const aggregate = aggregateReflections(reflections);
|
|
140
|
+
const lines = [
|
|
141
|
+
"## Process Feedback",
|
|
142
|
+
"",
|
|
143
|
+
`Aggregated from ${aggregate.total} agent reflection(s) appended during the run ` +
|
|
144
|
+
`(opt-in; schema: agent_reflection.schema.json).`,
|
|
145
|
+
"",
|
|
146
|
+
`- Instruction clarity: ${formatCounts(aggregate.clarity_breakdown)}`,
|
|
147
|
+
`- Reported impact: ${formatCounts(aggregate.severity_breakdown)}`,
|
|
148
|
+
"",
|
|
149
|
+
];
|
|
150
|
+
const block = (title, items) => {
|
|
151
|
+
if (items.length === 0)
|
|
152
|
+
return;
|
|
153
|
+
lines.push(`### ${title}`, "");
|
|
154
|
+
for (const item of items)
|
|
155
|
+
lines.push(`- ${item}`);
|
|
156
|
+
lines.push("");
|
|
157
|
+
};
|
|
158
|
+
block("Tool & instruction friction", aggregate.friction);
|
|
159
|
+
block("Ambiguities", aggregate.ambiguities);
|
|
160
|
+
block("Suggestions", aggregate.suggestions);
|
|
161
|
+
return lines;
|
|
162
|
+
}
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import type { AuditResult, CoverageMatrix, Finding, UnitManifest } from "../types.js";
|
|
2
2
|
import type { AuditScopeManifest } from "../types/auditScope.js";
|
|
3
|
+
import type { IntentCheckpoint } from "@audit-tools/shared";
|
|
3
4
|
import type { DesignAssessment } from "../types/designAssessment.js";
|
|
4
5
|
import type { ExternalAnalyzerResults } from "../types/externalAnalyzer.js";
|
|
5
6
|
import type { AuditFindingsReport, CriticalFlowManifest, Finding as SharedFinding, FindingTheme, GraphBundle, SynthesisNarrative } from "@audit-tools/shared";
|
|
6
7
|
import type { RuntimeValidationReport, RuntimeValidationTaskManifest } from "../types/runtimeValidation.js";
|
|
7
8
|
import { type WorkBlock } from "./workBlocks.js";
|
|
9
|
+
import { type AgentReflection } from "./agentReflections.js";
|
|
8
10
|
/** Contract version stamped onto the canonical `audit-findings.json`. */
|
|
9
11
|
export declare const AUDIT_FINDINGS_CONTRACT_VERSION = "audit-tools/audit-findings/v1";
|
|
10
12
|
/**
|
|
@@ -71,6 +73,17 @@ export declare function applyNarrative(report: AuditFindingsReport, narrative: S
|
|
|
71
73
|
export interface RenderAuditReportOptions {
|
|
72
74
|
/** Scope manifest for the run; when delta, the report header reports it honestly. */
|
|
73
75
|
scope?: AuditScopeManifest;
|
|
76
|
+
/**
|
|
77
|
+
* Opt-in agent meta-audit reflections to surface in a "Process Feedback"
|
|
78
|
+
* section. Omitted/empty renders nothing. The synthesis disk-load that
|
|
79
|
+
* populates this from `agent-feedback.jsonl` is wired separately.
|
|
80
|
+
*/
|
|
81
|
+
reflections?: AgentReflection[];
|
|
82
|
+
/**
|
|
83
|
+
* The accepted intent checkpoint; its `excluded_scope` is surfaced in an
|
|
84
|
+
* "Excluded / Out-of-Scope" section so omissions are explicit in the report.
|
|
85
|
+
*/
|
|
86
|
+
intent_checkpoint?: IntentCheckpoint;
|
|
74
87
|
}
|
|
75
88
|
export declare function renderAuditReportMarkdown(report: RenderableAuditReport, options?: RenderAuditReportOptions): string;
|
|
76
89
|
/**
|
|
@@ -2,6 +2,7 @@ import { AUDITOR_REPORT_MARKER } from "@audit-tools/shared";
|
|
|
2
2
|
import { buildWorkBlocks } from "./workBlocks.js";
|
|
3
3
|
import { mergeFindings } from "./mergeFindings.js";
|
|
4
4
|
import { assignStableFindingIds } from "./findingIdentity.js";
|
|
5
|
+
import { renderProcessFeedbackSection, } from "./agentReflections.js";
|
|
5
6
|
/** Contract version stamped onto the canonical `audit-findings.json`. */
|
|
6
7
|
export const AUDIT_FINDINGS_CONTRACT_VERSION = "audit-tools/audit-findings/v1";
|
|
7
8
|
function countBy(items, selectKey) {
|
|
@@ -215,6 +216,16 @@ export function renderAuditReportMarkdown(report, options = {}) {
|
|
|
215
216
|
lines.push("");
|
|
216
217
|
}
|
|
217
218
|
}
|
|
219
|
+
lines.push(...renderProcessFeedbackSection(options.reflections ?? []));
|
|
220
|
+
const excludedScope = options.intent_checkpoint?.excluded_scope ?? [];
|
|
221
|
+
if (excludedScope.length > 0) {
|
|
222
|
+
lines.push("## Excluded / Out-of-Scope", "");
|
|
223
|
+
lines.push(`${excludedScope.length} path(s) were excluded from this audit per the intent checkpoint:`, "");
|
|
224
|
+
for (const entry of excludedScope) {
|
|
225
|
+
lines.push(`- \`${entry.path}\` — ${entry.reason}`);
|
|
226
|
+
}
|
|
227
|
+
lines.push("");
|
|
228
|
+
}
|
|
218
229
|
lines.push("## Scope and Coverage", "");
|
|
219
230
|
const scope = options.scope;
|
|
220
231
|
if (scope && scope.mode === "delta") {
|
|
@@ -28,6 +28,15 @@ export function validateArtifactBundle(bundle) {
|
|
|
28
28
|
"budget",
|
|
29
29
|
]));
|
|
30
30
|
}
|
|
31
|
+
if (bundle.intent_checkpoint) {
|
|
32
|
+
issues.push(...requireKeys(bundle.intent_checkpoint, "intent_checkpoint", [
|
|
33
|
+
"schema_version",
|
|
34
|
+
"confirmed_at",
|
|
35
|
+
"confirmed_by",
|
|
36
|
+
"scope_summary",
|
|
37
|
+
"intent_summary",
|
|
38
|
+
]));
|
|
39
|
+
}
|
|
31
40
|
if (bundle.graph_bundle) {
|
|
32
41
|
issues.push(...requireKeys(bundle.graph_bundle, "graph_bundle", ["graphs"]));
|
|
33
42
|
}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
2
2
|
import { accessSync, constants } from "node:fs";
|
|
3
3
|
import { promisify } from "node:util";
|
|
4
4
|
import { ANALYZER_SETTINGS, PROVIDER_NAMES, SESSION_UI_MODES, isRecord, pushValidationIssue, } from "@audit-tools/shared";
|
|
5
|
-
const
|
|
5
|
+
const execFileAsync = promisify(execFile);
|
|
6
6
|
const VALID_PROVIDERS = new Set(PROVIDER_NAMES);
|
|
7
7
|
const VALID_UI_MODES = new Set(SESSION_UI_MODES);
|
|
8
8
|
const VALID_ANALYZER_SETTINGS = new Set(ANALYZER_SETTINGS);
|
|
@@ -85,7 +85,10 @@ function validateAgentProviderSection(value, path, issues) {
|
|
|
85
85
|
async function commandExists(command) {
|
|
86
86
|
const lookupCommand = process.platform === "win32" ? "where" : "which";
|
|
87
87
|
try {
|
|
88
|
-
|
|
88
|
+
// execFile (no shell) passes `command` as a literal argv entry, so shell
|
|
89
|
+
// metacharacters in a config-supplied command cannot be interpreted or
|
|
90
|
+
// executed during environment validation.
|
|
91
|
+
await execFileAsync(lookupCommand, [command]);
|
|
89
92
|
return true;
|
|
90
93
|
}
|
|
91
94
|
catch {
|
package/package.json
CHANGED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "agent_reflection.schema.json",
|
|
4
|
+
"title": "Agent Reflection",
|
|
5
|
+
"description": "One opt-in meta-audit reflection appended (NDJSON, one object per line) to agent-feedback.jsonl by a worker, describing how the tool felt to operate. Best-effort and never a substitute for the actual audit/remediation obligation.",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"required": ["task_id", "instruction_clarity", "severity"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"task_id": {
|
|
10
|
+
"type": "string",
|
|
11
|
+
"description": "The audit task or remediation item this reflection is about."
|
|
12
|
+
},
|
|
13
|
+
"lens": {
|
|
14
|
+
"type": "string",
|
|
15
|
+
"description": "Audit lens (or remediation phase) the worker operated under, when applicable."
|
|
16
|
+
},
|
|
17
|
+
"instruction_clarity": {
|
|
18
|
+
"type": "string",
|
|
19
|
+
"enum": ["clear", "mostly_clear", "ambiguous", "unclear"],
|
|
20
|
+
"description": "How clear the task instructions and scope were."
|
|
21
|
+
},
|
|
22
|
+
"ambiguities": {
|
|
23
|
+
"type": "array",
|
|
24
|
+
"items": { "type": "string" },
|
|
25
|
+
"description": "Specific things that were unclear about the task, scope, or contracts."
|
|
26
|
+
},
|
|
27
|
+
"tool_friction": {
|
|
28
|
+
"type": "array",
|
|
29
|
+
"items": { "type": "string" },
|
|
30
|
+
"description": "Friction encountered operating the tool (commands, prompts, artifacts, environment)."
|
|
31
|
+
},
|
|
32
|
+
"suggestions": {
|
|
33
|
+
"type": "array",
|
|
34
|
+
"items": { "type": "string" },
|
|
35
|
+
"description": "Concrete suggestions to reduce the friction or ambiguity."
|
|
36
|
+
},
|
|
37
|
+
"severity": {
|
|
38
|
+
"type": "string",
|
|
39
|
+
"enum": ["info", "low", "medium", "high"],
|
|
40
|
+
"description": "How much the reported friction/ambiguity impeded the work."
|
|
41
|
+
}
|
|
42
|
+
},
|
|
43
|
+
"additionalProperties": false
|
|
44
|
+
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "intent_checkpoint.schema.json",
|
|
4
|
+
"title": "Intent Checkpoint",
|
|
5
|
+
"description": "intent_checkpoint.json — the accepted scope and intent for a run, confirmed by the host (or auto-completed headlessly) before planning. Single-sourced across audit-code and remediate-code: audit-code consumes `excluded_scope`/`must_not_touch`/`free_form_intent` to prune planning and thread intent into worker prompts; remediate-code additionally uses `filters` to narrow findings. Sits upstream of the planning artifacts in the staleness DAG.",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"required": [
|
|
8
|
+
"schema_version",
|
|
9
|
+
"confirmed_at",
|
|
10
|
+
"confirmed_by",
|
|
11
|
+
"scope_summary",
|
|
12
|
+
"intent_summary"
|
|
13
|
+
],
|
|
14
|
+
"properties": {
|
|
15
|
+
"schema_version": {
|
|
16
|
+
"const": "intent-checkpoint/v1",
|
|
17
|
+
"description": "Contract version marker."
|
|
18
|
+
},
|
|
19
|
+
"confirmed_at": {
|
|
20
|
+
"type": "string",
|
|
21
|
+
"format": "date-time",
|
|
22
|
+
"description": "When the scope/intent was confirmed."
|
|
23
|
+
},
|
|
24
|
+
"confirmed_by": {
|
|
25
|
+
"const": "host",
|
|
26
|
+
"description": "Who confirmed the checkpoint (the host, or the headless harness acting as host)."
|
|
27
|
+
},
|
|
28
|
+
"scope_summary": {
|
|
29
|
+
"type": "string",
|
|
30
|
+
"description": "Human-readable description of the confirmed scope."
|
|
31
|
+
},
|
|
32
|
+
"intent_summary": {
|
|
33
|
+
"type": "string",
|
|
34
|
+
"description": "Human-readable description of the goal (e.g. full-audit / delta / security-focused)."
|
|
35
|
+
},
|
|
36
|
+
"free_form_intent": {
|
|
37
|
+
"type": "string",
|
|
38
|
+
"description": "Free-form intent threaded into worker/dispatch prompts."
|
|
39
|
+
},
|
|
40
|
+
"excluded_scope": {
|
|
41
|
+
"type": "array",
|
|
42
|
+
"description": "Paths intentionally excluded from the run; pruned from planning and listed in the report.",
|
|
43
|
+
"items": {
|
|
44
|
+
"type": "object",
|
|
45
|
+
"required": ["path", "reason"],
|
|
46
|
+
"properties": {
|
|
47
|
+
"path": {
|
|
48
|
+
"type": "string",
|
|
49
|
+
"description": "Path or path prefix to exclude."
|
|
50
|
+
},
|
|
51
|
+
"reason": {
|
|
52
|
+
"type": "string",
|
|
53
|
+
"description": "Why the path is excluded."
|
|
54
|
+
}
|
|
55
|
+
},
|
|
56
|
+
"additionalProperties": false
|
|
57
|
+
}
|
|
58
|
+
},
|
|
59
|
+
"must_not_touch": {
|
|
60
|
+
"type": "array",
|
|
61
|
+
"items": { "type": "string" },
|
|
62
|
+
"description": "Path globs that must never be written to."
|
|
63
|
+
},
|
|
64
|
+
"filters": {
|
|
65
|
+
"type": "object",
|
|
66
|
+
"description": "Remediate-only finding filters; audit-code ignores these.",
|
|
67
|
+
"properties": {
|
|
68
|
+
"severity": { "type": "array", "items": { "type": "string" } },
|
|
69
|
+
"lenses": { "type": "array", "items": { "type": "string" } },
|
|
70
|
+
"packages": { "type": "array", "items": { "type": "string" } },
|
|
71
|
+
"themes": { "type": "array", "items": { "type": "string" } }
|
|
72
|
+
},
|
|
73
|
+
"additionalProperties": false
|
|
74
|
+
}
|
|
75
|
+
},
|
|
76
|
+
"additionalProperties": false
|
|
77
|
+
}
|