auditor-lambda 0.2.5 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -7
- package/audit-code-wrapper-lib.mjs +1612 -331
- package/dist/cli.js +397 -38
- package/dist/coverage.d.ts +2 -2
- package/dist/coverage.js +5 -5
- package/dist/extractors/disposition.js +10 -1
- package/dist/extractors/flows.js +7 -1
- package/dist/extractors/pathPatterns.d.ts +3 -0
- package/dist/extractors/pathPatterns.js +15 -0
- package/dist/extractors/risk.js +7 -1
- package/dist/io/artifacts.d.ts +6 -6
- package/dist/io/artifacts.js +14 -17
- package/dist/io/json.d.ts +2 -0
- package/dist/io/json.js +15 -0
- package/dist/io/runArtifacts.d.ts +3 -1
- package/dist/io/runArtifacts.js +20 -5
- package/dist/mcp/server.d.ts +1 -0
- package/dist/mcp/server.js +579 -0
- package/dist/orchestrator/advance.js +9 -2
- package/dist/orchestrator/dependencyMap.js +9 -13
- package/dist/orchestrator/executors.js +7 -2
- package/dist/orchestrator/flowRequeue.d.ts +2 -2
- package/dist/orchestrator/flowRequeue.js +16 -3
- package/dist/orchestrator/internalExecutors.d.ts +2 -1
- package/dist/orchestrator/internalExecutors.js +129 -48
- package/dist/orchestrator/requeue.js +10 -4
- package/dist/orchestrator/requeueCommand.js +15 -2
- package/dist/orchestrator/resultIngestion.d.ts +2 -1
- package/dist/orchestrator/resultIngestion.js +26 -6
- package/dist/orchestrator/runtimeValidation.d.ts +7 -2
- package/dist/orchestrator/runtimeValidation.js +61 -49
- package/dist/orchestrator/runtimeValidationUpdate.js +2 -4
- package/dist/orchestrator/state.js +28 -14
- package/dist/orchestrator/taskBuilder.js +4 -2
- package/dist/orchestrator/trivialAudit.d.ts +4 -0
- package/dist/orchestrator/trivialAudit.js +49 -0
- package/dist/prompts/renderWorkerPrompt.js +6 -2
- package/dist/providers/spawnLoggedCommand.js +17 -0
- package/dist/reporting/mergeFindings.js +3 -11
- package/dist/reporting/rootCause.js +92 -9
- package/dist/reporting/synthesis.d.ts +25 -22
- package/dist/reporting/synthesis.js +92 -59
- package/dist/reporting/workBlocks.d.ts +12 -3
- package/dist/reporting/workBlocks.js +124 -70
- package/dist/supervisor/sessionConfig.js +4 -2
- package/dist/types/flows.d.ts +2 -0
- package/dist/types/runtimeValidation.d.ts +2 -1
- package/dist/types.d.ts +8 -6
- package/dist/validation/auditResults.d.ts +5 -2
- package/dist/validation/auditResults.js +335 -43
- package/docs/agent-integrations.md +38 -29
- package/docs/artifacts.md +18 -51
- package/docs/bootstrap-install.md +60 -30
- package/docs/contract.md +25 -117
- package/docs/field-trial-bug-report.md +237 -0
- package/docs/next-steps.md +59 -44
- package/docs/packaging.md +13 -3
- package/docs/production-launch-bar.md +2 -2
- package/docs/production-readiness.md +9 -5
- package/docs/releasing.md +81 -0
- package/docs/session-config.md +20 -1
- package/docs/usage.md +22 -0
- package/package.json +4 -1
- package/schemas/audit_result.schema.json +4 -5
- package/schemas/audit_task.schema.json +10 -0
- package/schemas/runtime_validation_report.schema.json +1 -1
- package/skills/audit-code/SKILL.md +11 -2
- package/skills/audit-code/audit-code.prompt.md +11 -10
- package/schemas/merged_findings.schema.json +0 -19
- package/schemas/root_cause_clusters.schema.json +0 -28
- package/schemas/synthesis_report.schema.json +0 -61
|
@@ -20,14 +20,19 @@ export const EXECUTOR_REGISTRY = [
|
|
|
20
20
|
description: "Ingest available audit result artifacts and refresh dependent coverage artifacts.",
|
|
21
21
|
},
|
|
22
22
|
{
|
|
23
|
-
id: "
|
|
23
|
+
id: "runtime_validation_executor",
|
|
24
24
|
obligation_ids: ["runtime_validation_current"],
|
|
25
25
|
description: "Merge runtime validation evidence updates when provided.",
|
|
26
26
|
},
|
|
27
|
+
{
|
|
28
|
+
id: "runtime_validation_update_executor",
|
|
29
|
+
obligation_ids: [],
|
|
30
|
+
description: "Merge imported runtime validation evidence updates.",
|
|
31
|
+
},
|
|
27
32
|
{
|
|
28
33
|
id: "synthesis_executor",
|
|
29
34
|
obligation_ids: ["synthesis_current"],
|
|
30
|
-
description: "
|
|
35
|
+
description: "Render the final deterministic Markdown audit report.",
|
|
31
36
|
},
|
|
32
37
|
{
|
|
33
38
|
id: "external_analyzer_import_executor",
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ExternalAnalyzerResults } from "../types/externalAnalyzer.js";
|
|
2
|
-
import type { AuditTask } from "../types.js";
|
|
2
|
+
import type { AuditTask, CoverageMatrix } from "../types.js";
|
|
3
3
|
import type { FlowCoverageManifest } from "../types/flowCoverage.js";
|
|
4
4
|
import type { CriticalFlowManifest } from "../types/flows.js";
|
|
5
|
-
export declare function buildFlowRequeueTasks(criticalFlows: CriticalFlowManifest, flowCoverage: FlowCoverageManifest, externalAnalyzerResults?: ExternalAnalyzerResults): AuditTask[];
|
|
5
|
+
export declare function buildFlowRequeueTasks(criticalFlows: CriticalFlowManifest, flowCoverage: FlowCoverageManifest, coverageMatrix: CoverageMatrix, externalAnalyzerResults?: ExternalAnalyzerResults): AuditTask[];
|
|
@@ -19,7 +19,14 @@ function taskPriority(hasExternalSignal, lens) {
|
|
|
19
19
|
}
|
|
20
20
|
return hasExternalSignal ? "medium" : "low";
|
|
21
21
|
}
|
|
22
|
-
|
|
22
|
+
function fileStillNeedsLens(coverageMatrix, path, lens) {
|
|
23
|
+
const record = coverageMatrix.files.find((file) => file.path === path);
|
|
24
|
+
if (!record || record.audit_status === "excluded") {
|
|
25
|
+
return false;
|
|
26
|
+
}
|
|
27
|
+
return !record.completed_lenses.includes(lens);
|
|
28
|
+
}
|
|
29
|
+
export function buildFlowRequeueTasks(criticalFlows, flowCoverage, coverageMatrix, externalAnalyzerResults) {
|
|
23
30
|
const flowMap = new Map(criticalFlows.flows.map((flow) => [flow.id, flow]));
|
|
24
31
|
const tasks = [];
|
|
25
32
|
const seen = new Set();
|
|
@@ -35,6 +42,9 @@ export function buildFlowRequeueTasks(criticalFlows, flowCoverage, externalAnaly
|
|
|
35
42
|
continue;
|
|
36
43
|
}
|
|
37
44
|
for (const path of flow.paths) {
|
|
45
|
+
if (!fileStillNeedsLens(coverageMatrix, path, lensName)) {
|
|
46
|
+
continue;
|
|
47
|
+
}
|
|
38
48
|
const signature = `${flow.id}|${lensName}|${path}`;
|
|
39
49
|
if (seen.has(signature)) {
|
|
40
50
|
continue;
|
|
@@ -47,9 +57,12 @@ export function buildFlowRequeueTasks(criticalFlows, flowCoverage, externalAnaly
|
|
|
47
57
|
pass_id: `flow-requeue:${lensName}`,
|
|
48
58
|
lens: lensName,
|
|
49
59
|
file_paths: [path],
|
|
50
|
-
rationale: `
|
|
60
|
+
rationale: `Mandatory audit coverage is still missing for critical flow ${flow.id} at ${path} under the ${lensName} lens.`,
|
|
51
61
|
priority: taskPriority(hasExternalSignal, lensName),
|
|
52
|
-
tags: hasExternalSignal
|
|
62
|
+
tags: hasExternalSignal
|
|
63
|
+
? ["critical_flow_followup", "external_analyzer_signal"]
|
|
64
|
+
: ["critical_flow_followup"],
|
|
65
|
+
status: "pending",
|
|
53
66
|
});
|
|
54
67
|
}
|
|
55
68
|
}
|
|
@@ -9,8 +9,9 @@ export interface ExecutorRunResult {
|
|
|
9
9
|
}
|
|
10
10
|
export declare function runIntakeExecutor(bundle: ArtifactBundle, root: string): Promise<ExecutorRunResult>;
|
|
11
11
|
export declare function runStructureExecutor(bundle: ArtifactBundle): ExecutorRunResult;
|
|
12
|
-
export declare function runPlanningExecutor(bundle: ArtifactBundle, lineIndex?: Record<string, number>): ExecutorRunResult
|
|
12
|
+
export declare function runPlanningExecutor(bundle: ArtifactBundle, root: string, lineIndex?: Record<string, number>): Promise<ExecutorRunResult>;
|
|
13
13
|
export declare function runResultIngestionExecutor(bundle: ArtifactBundle, results: AuditResult[]): ExecutorRunResult;
|
|
14
|
+
export declare function runRuntimeValidationExecutor(bundle: ArtifactBundle, root: string): Promise<ExecutorRunResult>;
|
|
14
15
|
export declare function runRuntimeValidationUpdateExecutor(bundle: ArtifactBundle, updates: RuntimeValidationReport): ExecutorRunResult;
|
|
15
16
|
export declare function runSynthesisExecutor(bundle: ArtifactBundle, results?: AuditResult[]): ExecutorRunResult;
|
|
16
17
|
export declare function runExternalAnalyzerImportExecutor(bundle: ArtifactBundle, externalResults: ExternalAnalyzerResults): ExecutorRunResult;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
1
2
|
import { buildFileDisposition } from "../extractors/disposition.js";
|
|
2
3
|
import { buildGraphBundle } from "../extractors/graph.js";
|
|
3
4
|
import { buildCriticalFlowManifest } from "../extractors/flows.js";
|
|
@@ -7,18 +8,48 @@ import { initializeCoverageFromPlan } from "./planning.js";
|
|
|
7
8
|
import { buildFlowCoverage } from "./flowCoverage.js";
|
|
8
9
|
import { buildFlowAwareTaskAugmentations } from "./flowPlanning.js";
|
|
9
10
|
import { buildRequeuePayload } from "./requeueCommand.js";
|
|
10
|
-
import { buildRuntimeValidationTasks,
|
|
11
|
-
import {
|
|
11
|
+
import { buildRuntimeValidationTasks, discoverRuntimeValidationCommand, mergeRuntimeValidationReport, } from "./runtimeValidation.js";
|
|
12
|
+
import { buildAuditReportModel, renderAuditReportMarkdown, } from "../reporting/synthesis.js";
|
|
12
13
|
import { buildChunkedAuditTasks, buildExternalSignalTasks, } from "./taskBuilder.js";
|
|
13
14
|
import { buildUnitManifest } from "./unitBuilder.js";
|
|
14
15
|
import { buildRepoManifestFromFs } from "../extractors/fsIntake.js";
|
|
15
16
|
import { loadIgnoreFile } from "../extractors/ignore.js";
|
|
16
|
-
import { ingestAuditResults } from "./resultIngestion.js";
|
|
17
|
+
import { ingestAuditResults, updateAuditTaskStatuses, } from "./resultIngestion.js";
|
|
17
18
|
import { updateRuntimeValidationReport } from "./runtimeValidationUpdate.js";
|
|
18
|
-
function
|
|
19
|
-
return
|
|
20
|
-
|
|
21
|
-
|
|
19
|
+
async function runCommand(command, cwd) {
|
|
20
|
+
return await new Promise((resolve) => {
|
|
21
|
+
const child = spawn(command[0], command.slice(1), {
|
|
22
|
+
cwd,
|
|
23
|
+
env: process.env,
|
|
24
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
25
|
+
});
|
|
26
|
+
let stdout = "";
|
|
27
|
+
let stderr = "";
|
|
28
|
+
child.stdout.on("data", (chunk) => {
|
|
29
|
+
stdout += String(chunk);
|
|
30
|
+
});
|
|
31
|
+
child.stderr.on("data", (chunk) => {
|
|
32
|
+
stderr += String(chunk);
|
|
33
|
+
});
|
|
34
|
+
child.on("error", (error) => {
|
|
35
|
+
resolve({
|
|
36
|
+
status: "inconclusive",
|
|
37
|
+
summary: `Failed to execute ${command.join(" ")}: ${error.message}`,
|
|
38
|
+
evidence: [],
|
|
39
|
+
});
|
|
40
|
+
});
|
|
41
|
+
child.on("exit", (code) => {
|
|
42
|
+
const output = `${stdout}\n${stderr}`.trim();
|
|
43
|
+
const evidence = output.length > 0 ? output.split(/\r?\n/).slice(-10) : [];
|
|
44
|
+
resolve({
|
|
45
|
+
status: code === 0 ? "confirmed" : "not_confirmed",
|
|
46
|
+
summary: code === 0
|
|
47
|
+
? `Deterministic runtime command succeeded: ${command.join(" ")}`
|
|
48
|
+
: `Deterministic runtime command failed with exit code ${code}: ${command.join(" ")}`,
|
|
49
|
+
evidence,
|
|
50
|
+
});
|
|
51
|
+
});
|
|
52
|
+
});
|
|
22
53
|
}
|
|
23
54
|
export async function runIntakeExecutor(bundle, root) {
|
|
24
55
|
const ignore = await loadIgnoreFile(root);
|
|
@@ -46,8 +77,8 @@ export function runStructureExecutor(bundle) {
|
|
|
46
77
|
const disposition = bundle.file_disposition ?? buildFileDisposition(bundle.repo_manifest);
|
|
47
78
|
const unitManifest = buildUnitManifest(bundle.repo_manifest, disposition);
|
|
48
79
|
const surfaceManifest = buildSurfaceManifest(bundle.repo_manifest, disposition);
|
|
49
|
-
const criticalFlows = buildCriticalFlowManifest(bundle.repo_manifest, surfaceManifest, disposition);
|
|
50
80
|
const graphBundle = buildGraphBundle(bundle.repo_manifest, disposition);
|
|
81
|
+
const criticalFlows = buildCriticalFlowManifest(bundle.repo_manifest, surfaceManifest, disposition);
|
|
51
82
|
const riskRegister = buildRiskRegister(unitManifest, criticalFlows, externalAnalyzerResults);
|
|
52
83
|
return {
|
|
53
84
|
updated: {
|
|
@@ -67,10 +98,13 @@ export function runStructureExecutor(bundle) {
|
|
|
67
98
|
"critical_flows.json",
|
|
68
99
|
"risk_register.json",
|
|
69
100
|
],
|
|
70
|
-
progress_summary: `Built structure artifacts for ${unitManifest.units.length} units and ${criticalFlows.flows.length} critical flows
|
|
101
|
+
progress_summary: `Built structure artifacts for ${unitManifest.units.length} units and ${criticalFlows.flows.length} critical flows.` +
|
|
102
|
+
(criticalFlows.fallback_required
|
|
103
|
+
? " Deterministic flow inference did not fully meet the confidence bar."
|
|
104
|
+
: ""),
|
|
71
105
|
};
|
|
72
106
|
}
|
|
73
|
-
export function runPlanningExecutor(bundle, lineIndex = {}) {
|
|
107
|
+
export async function runPlanningExecutor(bundle, root, lineIndex = {}) {
|
|
74
108
|
if (!bundle.repo_manifest) {
|
|
75
109
|
throw new Error("Cannot run planning executor without repo_manifest");
|
|
76
110
|
}
|
|
@@ -84,14 +118,25 @@ export function runPlanningExecutor(bundle, lineIndex = {}) {
|
|
|
84
118
|
const externalAnalyzerResults = bundle.external_analyzer_results;
|
|
85
119
|
const coverage = initializeCoverageFromPlan(bundle.repo_manifest, bundle.unit_manifest, bundle.file_disposition, externalAnalyzerResults);
|
|
86
120
|
const flowCoverage = buildFlowCoverage(bundle.critical_flows, coverage);
|
|
87
|
-
const
|
|
88
|
-
const
|
|
121
|
+
const runtimeCommand = await discoverRuntimeValidationCommand(root);
|
|
122
|
+
const runtimeValidationTasks = buildRuntimeValidationTasks({
|
|
123
|
+
unitManifest: bundle.unit_manifest,
|
|
124
|
+
criticalFlows: bundle.critical_flows,
|
|
125
|
+
flowCoverage,
|
|
126
|
+
command: runtimeCommand,
|
|
127
|
+
});
|
|
128
|
+
const runtimeValidationReport = runtimeValidationTasks.tasks.length > 0
|
|
129
|
+
? mergeRuntimeValidationReport(runtimeValidationTasks, bundle.runtime_validation_report)
|
|
130
|
+
: undefined;
|
|
89
131
|
const baseTasks = buildChunkedAuditTasks(bundle.unit_manifest, lineIndex, {
|
|
90
132
|
external_analyzer_results: externalAnalyzerResults,
|
|
91
133
|
});
|
|
92
134
|
const analyzerTasks = buildExternalSignalTasks(coverage, lineIndex, externalAnalyzerResults);
|
|
93
135
|
const flowTasks = buildFlowAwareTaskAugmentations([...baseTasks, ...analyzerTasks], bundle.critical_flows, lineIndex);
|
|
94
|
-
const auditTasks = [...baseTasks, ...analyzerTasks, ...flowTasks]
|
|
136
|
+
const auditTasks = [...baseTasks, ...analyzerTasks, ...flowTasks].map((task) => ({
|
|
137
|
+
...task,
|
|
138
|
+
status: task.status ?? "pending",
|
|
139
|
+
}));
|
|
95
140
|
const requeuePayload = buildRequeuePayload(coverage, bundle.critical_flows, flowCoverage, externalAnalyzerResults);
|
|
96
141
|
return {
|
|
97
142
|
updated: {
|
|
@@ -102,16 +147,20 @@ export function runPlanningExecutor(bundle, lineIndex = {}) {
|
|
|
102
147
|
runtime_validation_report: runtimeValidationReport,
|
|
103
148
|
audit_tasks: auditTasks,
|
|
104
149
|
requeue_tasks: requeuePayload.tasks,
|
|
150
|
+
audit_report: undefined,
|
|
105
151
|
},
|
|
106
152
|
artifacts_written: [
|
|
107
153
|
"coverage_matrix.json",
|
|
108
154
|
"flow_coverage.json",
|
|
109
155
|
"runtime_validation_tasks.json",
|
|
110
|
-
"runtime_validation_report.json",
|
|
156
|
+
...(runtimeValidationReport ? ["runtime_validation_report.json"] : []),
|
|
111
157
|
"audit_tasks.json",
|
|
112
158
|
"requeue_tasks.json",
|
|
113
159
|
],
|
|
114
|
-
progress_summary: `Built planning artifacts; generated ${auditTasks.length} tasks and ${requeuePayload.task_count} requeue tasks
|
|
160
|
+
progress_summary: `Built planning artifacts; generated ${auditTasks.length} tasks and ${requeuePayload.task_count} requeue tasks.` +
|
|
161
|
+
(runtimeCommand
|
|
162
|
+
? ` Runtime validation will use: ${runtimeCommand.join(" ")}.`
|
|
163
|
+
: " No deterministic runtime validation command was discovered."),
|
|
115
164
|
};
|
|
116
165
|
}
|
|
117
166
|
export function runResultIngestionExecutor(bundle, results) {
|
|
@@ -122,78 +171,109 @@ export function runResultIngestionExecutor(bundle, results) {
|
|
|
122
171
|
const flowCoverage = bundle.critical_flows
|
|
123
172
|
? buildFlowCoverage(bundle.critical_flows, updatedCoverageMatrix)
|
|
124
173
|
: bundle.flow_coverage;
|
|
125
|
-
const runtimeValidationTasks = bundle.unit_manifest
|
|
126
|
-
? buildRuntimeValidationTasks(bundle.unit_manifest, bundle.critical_flows, flowCoverage)
|
|
127
|
-
: bundle.runtime_validation_tasks;
|
|
128
|
-
const runtimeValidationReport = runtimeValidationTasks
|
|
129
|
-
? preserveOrPlaceholder(runtimeValidationTasks, bundle.runtime_validation_report)
|
|
130
|
-
: bundle.runtime_validation_report;
|
|
131
174
|
const requeuePayload = buildRequeuePayload(updatedCoverageMatrix, bundle.critical_flows, flowCoverage, bundle.external_analyzer_results);
|
|
132
175
|
const mergedResults = [...(bundle.audit_results ?? []), ...results];
|
|
133
|
-
const
|
|
176
|
+
const updatedAuditTasks = updateAuditTaskStatuses(bundle.audit_tasks, mergedResults);
|
|
134
177
|
return {
|
|
135
178
|
updated: {
|
|
136
179
|
...bundle,
|
|
137
180
|
coverage_matrix: updatedCoverageMatrix,
|
|
138
181
|
flow_coverage: flowCoverage,
|
|
139
|
-
runtime_validation_tasks: runtimeValidationTasks,
|
|
140
|
-
runtime_validation_report: runtimeValidationReport,
|
|
141
182
|
audit_results: mergedResults,
|
|
183
|
+
audit_tasks: updatedAuditTasks,
|
|
142
184
|
requeue_tasks: requeuePayload.tasks,
|
|
143
|
-
|
|
185
|
+
audit_report: undefined,
|
|
144
186
|
},
|
|
145
187
|
artifacts_written: [
|
|
146
188
|
"coverage_matrix.json",
|
|
147
189
|
"flow_coverage.json",
|
|
148
|
-
"runtime_validation_tasks.json",
|
|
149
|
-
"runtime_validation_report.json",
|
|
150
190
|
"audit_results.jsonl",
|
|
191
|
+
"audit_tasks.json",
|
|
151
192
|
"requeue_tasks.json",
|
|
152
|
-
"synthesis_report.json",
|
|
153
193
|
],
|
|
154
194
|
progress_summary: `Ingested ${results.length} audit result entries and refreshed dependent artifacts.`,
|
|
155
195
|
};
|
|
156
196
|
}
|
|
197
|
+
export async function runRuntimeValidationExecutor(bundle, root) {
|
|
198
|
+
if (!bundle.runtime_validation_tasks) {
|
|
199
|
+
throw new Error("Cannot execute runtime validation without runtime_validation_tasks");
|
|
200
|
+
}
|
|
201
|
+
const existing = bundle.runtime_validation_report ?? { results: [] };
|
|
202
|
+
const byTaskId = new Map(existing.results.map((result) => [result.task_id, result]));
|
|
203
|
+
const byCommand = new Map();
|
|
204
|
+
for (const task of bundle.runtime_validation_tasks.tasks) {
|
|
205
|
+
const prior = byTaskId.get(task.id);
|
|
206
|
+
if (prior &&
|
|
207
|
+
["confirmed", "not_confirmed", "inconclusive", "not_required"].includes(prior.status)) {
|
|
208
|
+
continue;
|
|
209
|
+
}
|
|
210
|
+
if (!task.command || task.command.length === 0) {
|
|
211
|
+
byTaskId.set(task.id, {
|
|
212
|
+
task_id: task.id,
|
|
213
|
+
status: "not_required",
|
|
214
|
+
summary: `No deterministic runtime command was available for ${task.id}.`,
|
|
215
|
+
evidence: [],
|
|
216
|
+
notes: ["Runtime validation was not planned for this task."],
|
|
217
|
+
});
|
|
218
|
+
continue;
|
|
219
|
+
}
|
|
220
|
+
const signature = task.command.join("\0");
|
|
221
|
+
const outcome = byCommand.get(signature) ?? (await runCommand(task.command, root));
|
|
222
|
+
byCommand.set(signature, outcome);
|
|
223
|
+
byTaskId.set(task.id, {
|
|
224
|
+
task_id: task.id,
|
|
225
|
+
status: outcome.status,
|
|
226
|
+
summary: outcome.summary,
|
|
227
|
+
evidence: outcome.evidence,
|
|
228
|
+
notes: [`Target paths: ${task.target_paths.join(", ")}`],
|
|
229
|
+
});
|
|
230
|
+
}
|
|
231
|
+
return {
|
|
232
|
+
updated: {
|
|
233
|
+
...bundle,
|
|
234
|
+
runtime_validation_report: {
|
|
235
|
+
results: [...byTaskId.values()].sort((a, b) => a.task_id.localeCompare(b.task_id)),
|
|
236
|
+
},
|
|
237
|
+
audit_report: undefined,
|
|
238
|
+
},
|
|
239
|
+
artifacts_written: ["runtime_validation_report.json"],
|
|
240
|
+
progress_summary: `Executed deterministic runtime validation for ${bundle.runtime_validation_tasks.tasks.length} task(s).`,
|
|
241
|
+
};
|
|
242
|
+
}
|
|
157
243
|
export function runRuntimeValidationUpdateExecutor(bundle, updates) {
|
|
158
244
|
if (!bundle.runtime_validation_tasks) {
|
|
159
245
|
throw new Error("Cannot update runtime validation without runtime_validation_tasks");
|
|
160
246
|
}
|
|
161
|
-
const existingReport = bundle.runtime_validation_report ??
|
|
162
|
-
buildPlaceholderRuntimeValidationReport(bundle.runtime_validation_tasks);
|
|
247
|
+
const existingReport = bundle.runtime_validation_report ?? { results: [] };
|
|
163
248
|
const mergedReport = updateRuntimeValidationReport(bundle.runtime_validation_tasks, existingReport, updates);
|
|
164
|
-
const synthesisReport = buildSynthesisReport(bundle.audit_results ?? [], mergedReport, bundle.external_analyzer_results);
|
|
165
249
|
return {
|
|
166
250
|
updated: {
|
|
167
251
|
...bundle,
|
|
168
252
|
runtime_validation_report: mergedReport,
|
|
169
|
-
|
|
253
|
+
audit_report: undefined,
|
|
170
254
|
},
|
|
171
|
-
artifacts_written: [
|
|
172
|
-
"runtime_validation_report.json",
|
|
173
|
-
"synthesis_report.json",
|
|
174
|
-
],
|
|
255
|
+
artifacts_written: ["runtime_validation_report.json"],
|
|
175
256
|
progress_summary: `Merged ${updates.results.length} runtime validation updates.`,
|
|
176
257
|
};
|
|
177
258
|
}
|
|
178
259
|
export function runSynthesisExecutor(bundle, results) {
|
|
179
260
|
const finalResults = results ?? bundle.audit_results ?? [];
|
|
180
|
-
const
|
|
181
|
-
|
|
182
|
-
|
|
261
|
+
const model = buildAuditReportModel({
|
|
262
|
+
results: finalResults,
|
|
263
|
+
unitManifest: bundle.unit_manifest,
|
|
264
|
+
graphBundle: bundle.graph_bundle,
|
|
265
|
+
criticalFlows: bundle.critical_flows,
|
|
266
|
+
coverageMatrix: bundle.coverage_matrix,
|
|
267
|
+
runtimeValidationReport: bundle.runtime_validation_report,
|
|
268
|
+
});
|
|
183
269
|
return {
|
|
184
270
|
updated: {
|
|
185
271
|
...bundle,
|
|
186
272
|
audit_results: finalResults,
|
|
187
|
-
|
|
188
|
-
root_cause_clusters: rootCauseClusters,
|
|
189
|
-
synthesis_report: synthesisReport,
|
|
273
|
+
audit_report: renderAuditReportMarkdown(model),
|
|
190
274
|
},
|
|
191
|
-
artifacts_written: [
|
|
192
|
-
|
|
193
|
-
"root_cause_clusters.json",
|
|
194
|
-
"synthesis_report.json",
|
|
195
|
-
],
|
|
196
|
-
progress_summary: `Refreshed synthesis for ${finalResults.length} audit result entries.`,
|
|
275
|
+
artifacts_written: ["audit-report.md"],
|
|
276
|
+
progress_summary: `Rendered deterministic audit report for ${finalResults.length} audit result entries.`,
|
|
197
277
|
};
|
|
198
278
|
}
|
|
199
279
|
export function runExternalAnalyzerImportExecutor(bundle, externalResults) {
|
|
@@ -202,6 +282,7 @@ export function runExternalAnalyzerImportExecutor(bundle, externalResults) {
|
|
|
202
282
|
updated: {
|
|
203
283
|
...bundle,
|
|
204
284
|
external_analyzer_results: externalResults,
|
|
285
|
+
audit_report: undefined,
|
|
205
286
|
},
|
|
206
287
|
artifacts_written: ["external_analyzer_results.json"],
|
|
207
288
|
progress_summary: summary,
|
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
import { buildRequeueTargets } from "../coverage.js";
|
|
2
|
-
function taskPriority(hasExternalSignal) {
|
|
3
|
-
|
|
2
|
+
function taskPriority(hasExternalSignal, lens) {
|
|
3
|
+
if (hasExternalSignal)
|
|
4
|
+
return "high";
|
|
5
|
+
if (lens === "security" || lens === "data_integrity" || lens === "reliability") {
|
|
6
|
+
return "medium";
|
|
7
|
+
}
|
|
8
|
+
return "low";
|
|
4
9
|
}
|
|
5
10
|
export function buildRequeueTasks(matrix, externalAnalyzerResults) {
|
|
6
11
|
const targets = buildRequeueTargets(matrix);
|
|
@@ -15,9 +20,10 @@ export function buildRequeueTasks(matrix, externalAnalyzerResults) {
|
|
|
15
20
|
pass_id: `requeue:${lens}`,
|
|
16
21
|
lens,
|
|
17
22
|
file_paths: [target.path],
|
|
18
|
-
rationale: `
|
|
19
|
-
priority: taskPriority(hasExternalSignal),
|
|
23
|
+
rationale: `Mandatory audit coverage is still missing for ${target.path} under the ${lens} lens.`,
|
|
24
|
+
priority: taskPriority(hasExternalSignal, lens),
|
|
20
25
|
tags: hasExternalSignal ? ["external_analyzer_signal"] : [],
|
|
26
|
+
status: "pending",
|
|
21
27
|
});
|
|
22
28
|
}
|
|
23
29
|
}
|
|
@@ -12,12 +12,25 @@ function dedupeTasks(tasks) {
|
|
|
12
12
|
}
|
|
13
13
|
return deduped;
|
|
14
14
|
}
|
|
15
|
+
function dedupeByScope(tasks) {
|
|
16
|
+
const seen = new Set();
|
|
17
|
+
const deduped = [];
|
|
18
|
+
for (const task of tasks) {
|
|
19
|
+
const signature = `${task.lens}:${[...task.file_paths].sort().join(",")}`;
|
|
20
|
+
if (seen.has(signature)) {
|
|
21
|
+
continue;
|
|
22
|
+
}
|
|
23
|
+
seen.add(signature);
|
|
24
|
+
deduped.push(task);
|
|
25
|
+
}
|
|
26
|
+
return deduped;
|
|
27
|
+
}
|
|
15
28
|
export function buildRequeuePayload(matrix, criticalFlows, flowCoverage, externalAnalyzerResults) {
|
|
16
29
|
const fileTasks = dedupeTasks(buildRequeueTasks(matrix, externalAnalyzerResults));
|
|
17
30
|
const flowTasks = criticalFlows && flowCoverage
|
|
18
|
-
? dedupeTasks(buildFlowRequeueTasks(criticalFlows, flowCoverage, externalAnalyzerResults))
|
|
31
|
+
? dedupeTasks(buildFlowRequeueTasks(criticalFlows, flowCoverage, matrix, externalAnalyzerResults))
|
|
19
32
|
: [];
|
|
20
|
-
const tasks = dedupeTasks([...fileTasks, ...flowTasks]);
|
|
33
|
+
const tasks = dedupeByScope(dedupeTasks([...fileTasks, ...flowTasks]));
|
|
21
34
|
return {
|
|
22
35
|
task_count: tasks.length,
|
|
23
36
|
file_task_count: fileTasks.length,
|
|
@@ -1,2 +1,3 @@
|
|
|
1
|
-
import type { AuditResult, CoverageMatrix } from "../types.js";
|
|
1
|
+
import type { AuditResult, AuditTask, CoverageMatrix } from "../types.js";
|
|
2
2
|
export declare function ingestAuditResults(coverageMatrix: CoverageMatrix, results: AuditResult[]): CoverageMatrix;
|
|
3
|
+
export declare function updateAuditTaskStatuses(tasks: AuditTask[] | undefined, results: AuditResult[]): AuditTask[] | undefined;
|
|
@@ -1,14 +1,34 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { applyFileCoverage } from "../coverage.js";
|
|
2
2
|
export function ingestAuditResults(coverageMatrix, results) {
|
|
3
3
|
const matrix = JSON.parse(JSON.stringify(coverageMatrix));
|
|
4
|
-
const
|
|
5
|
-
path:
|
|
6
|
-
|
|
7
|
-
end: range.end,
|
|
4
|
+
const fileCoverage = results.flatMap((result) => result.file_coverage.map((coverage) => ({
|
|
5
|
+
path: coverage.path,
|
|
6
|
+
total_lines: coverage.total_lines,
|
|
8
7
|
pass_id: result.pass_id,
|
|
9
8
|
lens: result.lens,
|
|
10
9
|
agent_role: result.agent_role,
|
|
11
10
|
})));
|
|
12
|
-
|
|
11
|
+
applyFileCoverage(matrix, fileCoverage);
|
|
13
12
|
return matrix;
|
|
14
13
|
}
|
|
14
|
+
export function updateAuditTaskStatuses(tasks, results) {
|
|
15
|
+
if (!tasks) {
|
|
16
|
+
return undefined;
|
|
17
|
+
}
|
|
18
|
+
const completedTaskIds = new Set(results.map((result) => result.task_id));
|
|
19
|
+
const completedAt = new Date().toISOString();
|
|
20
|
+
return tasks.map((task) => {
|
|
21
|
+
if (completedTaskIds.has(task.task_id)) {
|
|
22
|
+
return {
|
|
23
|
+
...task,
|
|
24
|
+
status: "complete",
|
|
25
|
+
completed_at: task.completed_at ?? completedAt,
|
|
26
|
+
completion_reason: task.completion_reason ?? "result_ingested",
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
return {
|
|
30
|
+
...task,
|
|
31
|
+
status: task.status ?? "pending",
|
|
32
|
+
};
|
|
33
|
+
});
|
|
34
|
+
}
|
|
@@ -2,6 +2,11 @@ import type { UnitManifest } from "../types.js";
|
|
|
2
2
|
import type { FlowCoverageManifest } from "../types/flowCoverage.js";
|
|
3
3
|
import type { CriticalFlowManifest } from "../types/flows.js";
|
|
4
4
|
import type { RuntimeValidationReport, RuntimeValidationTaskManifest } from "../types/runtimeValidation.js";
|
|
5
|
-
export declare function
|
|
6
|
-
export declare function
|
|
5
|
+
export declare function discoverRuntimeValidationCommand(root: string): Promise<string[] | undefined>;
|
|
6
|
+
export declare function buildRuntimeValidationTasks(params: {
|
|
7
|
+
unitManifest: UnitManifest;
|
|
8
|
+
criticalFlows?: CriticalFlowManifest;
|
|
9
|
+
flowCoverage?: FlowCoverageManifest;
|
|
10
|
+
command?: string[];
|
|
11
|
+
}): RuntimeValidationTaskManifest;
|
|
7
12
|
export declare function mergeRuntimeValidationReport(tasks: RuntimeValidationTaskManifest, existing?: RuntimeValidationReport): RuntimeValidationReport;
|
|
@@ -1,23 +1,59 @@
|
|
|
1
|
+
import { access, readFile } from "node:fs/promises";
|
|
1
2
|
function checksForFlow(requiredLenses) {
|
|
2
3
|
const checks = [];
|
|
3
4
|
if (requiredLenses.includes("security")) {
|
|
4
5
|
checks.push("Exercise malformed or unauthorized inputs against the flow.");
|
|
5
6
|
}
|
|
6
7
|
if (requiredLenses.includes("reliability")) {
|
|
7
|
-
checks.push("Exercise retries
|
|
8
|
+
checks.push("Exercise retries or repeated submissions.");
|
|
8
9
|
}
|
|
9
10
|
if (requiredLenses.includes("correctness")) {
|
|
10
|
-
checks.push("Exercise representative success and edge-case
|
|
11
|
+
checks.push("Exercise representative success and edge-case behavior.");
|
|
11
12
|
}
|
|
12
13
|
if (requiredLenses.includes("data_integrity")) {
|
|
13
|
-
checks.push("Verify state transitions and persistence invariants
|
|
14
|
+
checks.push("Verify state transitions and persistence invariants.");
|
|
14
15
|
}
|
|
15
16
|
return checks;
|
|
16
17
|
}
|
|
17
|
-
|
|
18
|
+
async function exists(path) {
|
|
19
|
+
try {
|
|
20
|
+
await access(path);
|
|
21
|
+
return true;
|
|
22
|
+
}
|
|
23
|
+
catch {
|
|
24
|
+
return false;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
export async function discoverRuntimeValidationCommand(root) {
|
|
28
|
+
const packageJsonPath = `${root}/package.json`;
|
|
29
|
+
if (await exists(packageJsonPath)) {
|
|
30
|
+
try {
|
|
31
|
+
const packageJson = JSON.parse(await readFile(packageJsonPath, "utf8"));
|
|
32
|
+
const testScript = packageJson.scripts?.test?.trim();
|
|
33
|
+
if (testScript &&
|
|
34
|
+
!/no test specified/i.test(testScript)) {
|
|
35
|
+
return ["npm", "test"];
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
// ignore unreadable package.json for runtime discovery
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
if (await exists(`${root}/go.mod`)) {
|
|
43
|
+
return ["go", "test", "./..."];
|
|
44
|
+
}
|
|
45
|
+
if (await exists(`${root}/pyproject.toml`) || await exists(`${root}/pytest.ini`)) {
|
|
46
|
+
return ["python", "-m", "pytest"];
|
|
47
|
+
}
|
|
48
|
+
return undefined;
|
|
49
|
+
}
|
|
50
|
+
export function buildRuntimeValidationTasks(params) {
|
|
51
|
+
if (!params.command) {
|
|
52
|
+
return { tasks: [] };
|
|
53
|
+
}
|
|
18
54
|
const tasks = [];
|
|
19
55
|
const seen = new Set();
|
|
20
|
-
const highRiskUnits = unitManifest.units.filter((unit) => (unit.risk_score ?? 0) >= 5 ||
|
|
56
|
+
const highRiskUnits = params.unitManifest.units.filter((unit) => (unit.risk_score ?? 0) >= 5 ||
|
|
21
57
|
unit.required_lenses.includes("security") ||
|
|
22
58
|
unit.required_lenses.includes("data_integrity"));
|
|
23
59
|
for (const unit of highRiskUnits) {
|
|
@@ -31,16 +67,17 @@ export function buildRuntimeValidationTasks(unitManifest, criticalFlows, flowCov
|
|
|
31
67
|
target_paths: unit.files,
|
|
32
68
|
reason: `Unit ${unit.unit_id} is high risk or touches sensitive concerns.`,
|
|
33
69
|
priority: (unit.risk_score ?? 0) >= 7 ? "high" : "medium",
|
|
70
|
+
command: params.command,
|
|
34
71
|
suggested_checks: [
|
|
35
|
-
"Run
|
|
36
|
-
"
|
|
72
|
+
"Run the deterministic runtime command for the repository.",
|
|
73
|
+
"Confirm the affected unit does not regress under the command output.",
|
|
37
74
|
],
|
|
38
75
|
source_artifacts: ["unit_manifest.json", "risk_register.json"],
|
|
39
76
|
});
|
|
40
77
|
}
|
|
41
|
-
if (criticalFlows && flowCoverage) {
|
|
42
|
-
const flowMap = new Map(criticalFlows.flows.map((flow) => [flow.id, flow]));
|
|
43
|
-
for (const record of flowCoverage.flows) {
|
|
78
|
+
if (params.criticalFlows && params.flowCoverage) {
|
|
79
|
+
const flowMap = new Map(params.criticalFlows.flows.map((flow) => [flow.id, flow]));
|
|
80
|
+
for (const record of params.flowCoverage.flows) {
|
|
44
81
|
if (record.status === "complete") {
|
|
45
82
|
continue;
|
|
46
83
|
}
|
|
@@ -56,53 +93,28 @@ export function buildRuntimeValidationTasks(unitManifest, criticalFlows, flowCov
|
|
|
56
93
|
id,
|
|
57
94
|
kind: "critical-flow-check",
|
|
58
95
|
target_paths: flow.paths,
|
|
59
|
-
reason: `Critical flow ${record.flow_id} is still ${record.status} and
|
|
96
|
+
reason: `Critical flow ${record.flow_id} is still ${record.status} and needs deterministic runtime validation.`,
|
|
60
97
|
priority: record.status === "pending" ? "high" : "medium",
|
|
98
|
+
command: params.command,
|
|
61
99
|
suggested_checks: checksForFlow(record.required_lenses),
|
|
62
100
|
source_artifacts: ["critical_flows.json", "flow_coverage.json"],
|
|
63
101
|
});
|
|
64
102
|
}
|
|
65
103
|
}
|
|
66
|
-
const RUNTIME_TASK_CAP = 100;
|
|
67
|
-
if (tasks.length > RUNTIME_TASK_CAP) {
|
|
68
|
-
process.stderr.write(`[runtime-validation] generated ${tasks.length} tasks which exceeds the cap of ${RUNTIME_TASK_CAP}; truncating to avoid runaway expansion\n`);
|
|
69
|
-
tasks.splice(RUNTIME_TASK_CAP);
|
|
70
|
-
}
|
|
71
104
|
return { tasks };
|
|
72
105
|
}
|
|
73
|
-
export function buildPlaceholderRuntimeValidationReport(tasks) {
|
|
74
|
-
return {
|
|
75
|
-
results: tasks.tasks.map((task) => ({
|
|
76
|
-
task_id: task.id,
|
|
77
|
-
status: "pending",
|
|
78
|
-
summary: `No runtime evidence recorded yet for ${task.id}.`,
|
|
79
|
-
evidence: [],
|
|
80
|
-
notes: ["Placeholder entry generated from runtime validation task list."],
|
|
81
|
-
})),
|
|
82
|
-
};
|
|
83
|
-
}
|
|
84
106
|
export function mergeRuntimeValidationReport(tasks, existing) {
|
|
85
107
|
const existingMap = new Map((existing?.results ?? []).map((result) => [result.task_id, result]));
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
return {
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
}
|
|
99
|
-
return {
|
|
100
|
-
task_id: task.id,
|
|
101
|
-
status: "pending",
|
|
102
|
-
summary: `No runtime evidence recorded yet for ${task.id}.`,
|
|
103
|
-
evidence: [],
|
|
104
|
-
notes: ["Placeholder entry generated from runtime validation task list."],
|
|
105
|
-
};
|
|
106
|
-
});
|
|
107
|
-
return { results: mergedResults };
|
|
108
|
+
return {
|
|
109
|
+
results: tasks.tasks.map((task) => {
|
|
110
|
+
const prior = existingMap.get(task.id);
|
|
111
|
+
return (prior ?? {
|
|
112
|
+
task_id: task.id,
|
|
113
|
+
status: "pending",
|
|
114
|
+
summary: `Deterministic runtime validation has not executed yet for ${task.id}.`,
|
|
115
|
+
evidence: [],
|
|
116
|
+
notes: [],
|
|
117
|
+
});
|
|
118
|
+
}),
|
|
119
|
+
};
|
|
108
120
|
}
|