auditor-lambda 0.2.6 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -7
- package/audit-code-wrapper-lib.mjs +1605 -330
- package/dist/cli.js +78 -16
- package/dist/coverage.d.ts +2 -2
- package/dist/coverage.js +5 -5
- package/dist/extractors/disposition.js +10 -1
- package/dist/extractors/flows.js +7 -1
- package/dist/extractors/pathPatterns.d.ts +3 -0
- package/dist/extractors/pathPatterns.js +15 -0
- package/dist/extractors/risk.js +7 -1
- package/dist/io/artifacts.d.ts +6 -6
- package/dist/io/artifacts.js +14 -17
- package/dist/io/json.d.ts +2 -0
- package/dist/io/json.js +15 -0
- package/dist/io/runArtifacts.d.ts +1 -0
- package/dist/io/runArtifacts.js +18 -4
- package/dist/mcp/server.d.ts +1 -0
- package/dist/mcp/server.js +579 -0
- package/dist/orchestrator/advance.js +9 -2
- package/dist/orchestrator/dependencyMap.js +9 -13
- package/dist/orchestrator/executors.js +7 -2
- package/dist/orchestrator/flowRequeue.js +1 -1
- package/dist/orchestrator/internalExecutors.d.ts +2 -1
- package/dist/orchestrator/internalExecutors.js +120 -63
- package/dist/orchestrator/requeue.js +9 -4
- package/dist/orchestrator/resultIngestion.js +5 -6
- package/dist/orchestrator/runtimeValidation.d.ts +7 -2
- package/dist/orchestrator/runtimeValidation.js +61 -49
- package/dist/orchestrator/runtimeValidationUpdate.js +2 -4
- package/dist/orchestrator/state.js +18 -13
- package/dist/orchestrator/trivialAudit.js +8 -5
- package/dist/prompts/renderWorkerPrompt.js +3 -2
- package/dist/reporting/mergeFindings.js +0 -11
- package/dist/reporting/synthesis.d.ts +25 -22
- package/dist/reporting/synthesis.js +92 -59
- package/dist/reporting/workBlocks.d.ts +12 -3
- package/dist/reporting/workBlocks.js +124 -70
- package/dist/types/flows.d.ts +2 -0
- package/dist/types/runtimeValidation.d.ts +2 -1
- package/dist/types.d.ts +4 -7
- package/dist/validation/auditResults.js +64 -99
- package/docs/agent-integrations.md +38 -29
- package/docs/artifacts.md +16 -56
- package/docs/bootstrap-install.md +60 -30
- package/docs/contract.md +22 -205
- package/docs/next-steps.md +59 -44
- package/docs/packaging.md +13 -3
- package/docs/production-launch-bar.md +2 -2
- package/docs/production-readiness.md +9 -5
- package/docs/releasing.md +81 -0
- package/package.json +4 -1
- package/schemas/audit_result.schema.json +4 -6
- package/schemas/runtime_validation_report.schema.json +1 -1
- package/skills/audit-code/SKILL.md +11 -2
- package/skills/audit-code/audit-code.prompt.md +5 -8
- package/schemas/merged_findings.schema.json +0 -19
- package/schemas/root_cause_clusters.schema.json +0 -28
- package/schemas/synthesis_report.schema.json +0 -61
|
@@ -20,14 +20,19 @@ export const EXECUTOR_REGISTRY = [
|
|
|
20
20
|
description: "Ingest available audit result artifacts and refresh dependent coverage artifacts.",
|
|
21
21
|
},
|
|
22
22
|
{
|
|
23
|
-
id: "
|
|
23
|
+
id: "runtime_validation_executor",
|
|
24
24
|
obligation_ids: ["runtime_validation_current"],
|
|
25
25
|
description: "Merge runtime validation evidence updates when provided.",
|
|
26
26
|
},
|
|
27
|
+
{
|
|
28
|
+
id: "runtime_validation_update_executor",
|
|
29
|
+
obligation_ids: [],
|
|
30
|
+
description: "Merge imported runtime validation evidence updates.",
|
|
31
|
+
},
|
|
27
32
|
{
|
|
28
33
|
id: "synthesis_executor",
|
|
29
34
|
obligation_ids: ["synthesis_current"],
|
|
30
|
-
description: "
|
|
35
|
+
description: "Render the final deterministic Markdown audit report.",
|
|
31
36
|
},
|
|
32
37
|
{
|
|
33
38
|
id: "external_analyzer_import_executor",
|
|
@@ -57,7 +57,7 @@ export function buildFlowRequeueTasks(criticalFlows, flowCoverage, coverageMatri
|
|
|
57
57
|
pass_id: `flow-requeue:${lensName}`,
|
|
58
58
|
lens: lensName,
|
|
59
59
|
file_paths: [path],
|
|
60
|
-
rationale: `
|
|
60
|
+
rationale: `Mandatory audit coverage is still missing for critical flow ${flow.id} at ${path} under the ${lensName} lens.`,
|
|
61
61
|
priority: taskPriority(hasExternalSignal, lensName),
|
|
62
62
|
tags: hasExternalSignal
|
|
63
63
|
? ["critical_flow_followup", "external_analyzer_signal"]
|
|
@@ -9,8 +9,9 @@ export interface ExecutorRunResult {
|
|
|
9
9
|
}
|
|
10
10
|
export declare function runIntakeExecutor(bundle: ArtifactBundle, root: string): Promise<ExecutorRunResult>;
|
|
11
11
|
export declare function runStructureExecutor(bundle: ArtifactBundle): ExecutorRunResult;
|
|
12
|
-
export declare function runPlanningExecutor(bundle: ArtifactBundle, lineIndex?: Record<string, number>): ExecutorRunResult
|
|
12
|
+
export declare function runPlanningExecutor(bundle: ArtifactBundle, root: string, lineIndex?: Record<string, number>): Promise<ExecutorRunResult>;
|
|
13
13
|
export declare function runResultIngestionExecutor(bundle: ArtifactBundle, results: AuditResult[]): ExecutorRunResult;
|
|
14
|
+
export declare function runRuntimeValidationExecutor(bundle: ArtifactBundle, root: string): Promise<ExecutorRunResult>;
|
|
14
15
|
export declare function runRuntimeValidationUpdateExecutor(bundle: ArtifactBundle, updates: RuntimeValidationReport): ExecutorRunResult;
|
|
15
16
|
export declare function runSynthesisExecutor(bundle: ArtifactBundle, results?: AuditResult[]): ExecutorRunResult;
|
|
16
17
|
export declare function runExternalAnalyzerImportExecutor(bundle: ArtifactBundle, externalResults: ExternalAnalyzerResults): ExecutorRunResult;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
1
2
|
import { buildFileDisposition } from "../extractors/disposition.js";
|
|
2
3
|
import { buildGraphBundle } from "../extractors/graph.js";
|
|
3
4
|
import { buildCriticalFlowManifest } from "../extractors/flows.js";
|
|
@@ -7,26 +8,48 @@ import { initializeCoverageFromPlan } from "./planning.js";
|
|
|
7
8
|
import { buildFlowCoverage } from "./flowCoverage.js";
|
|
8
9
|
import { buildFlowAwareTaskAugmentations } from "./flowPlanning.js";
|
|
9
10
|
import { buildRequeuePayload } from "./requeueCommand.js";
|
|
10
|
-
import { buildRuntimeValidationTasks,
|
|
11
|
-
import {
|
|
11
|
+
import { buildRuntimeValidationTasks, discoverRuntimeValidationCommand, mergeRuntimeValidationReport, } from "./runtimeValidation.js";
|
|
12
|
+
import { buildAuditReportModel, renderAuditReportMarkdown, } from "../reporting/synthesis.js";
|
|
12
13
|
import { buildChunkedAuditTasks, buildExternalSignalTasks, } from "./taskBuilder.js";
|
|
13
14
|
import { buildUnitManifest } from "./unitBuilder.js";
|
|
14
15
|
import { buildRepoManifestFromFs } from "../extractors/fsIntake.js";
|
|
15
16
|
import { loadIgnoreFile } from "../extractors/ignore.js";
|
|
16
17
|
import { ingestAuditResults, updateAuditTaskStatuses, } from "./resultIngestion.js";
|
|
17
18
|
import { updateRuntimeValidationReport } from "./runtimeValidationUpdate.js";
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
19
|
+
async function runCommand(command, cwd) {
|
|
20
|
+
return await new Promise((resolve) => {
|
|
21
|
+
const child = spawn(command[0], command.slice(1), {
|
|
22
|
+
cwd,
|
|
23
|
+
env: process.env,
|
|
24
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
25
|
+
});
|
|
26
|
+
let stdout = "";
|
|
27
|
+
let stderr = "";
|
|
28
|
+
child.stdout.on("data", (chunk) => {
|
|
29
|
+
stdout += String(chunk);
|
|
30
|
+
});
|
|
31
|
+
child.stderr.on("data", (chunk) => {
|
|
32
|
+
stderr += String(chunk);
|
|
33
|
+
});
|
|
34
|
+
child.on("error", (error) => {
|
|
35
|
+
resolve({
|
|
36
|
+
status: "inconclusive",
|
|
37
|
+
summary: `Failed to execute ${command.join(" ")}: ${error.message}`,
|
|
38
|
+
evidence: [],
|
|
39
|
+
});
|
|
40
|
+
});
|
|
41
|
+
child.on("exit", (code) => {
|
|
42
|
+
const output = `${stdout}\n${stderr}`.trim();
|
|
43
|
+
const evidence = output.length > 0 ? output.split(/\r?\n/).slice(-10) : [];
|
|
44
|
+
resolve({
|
|
45
|
+
status: code === 0 ? "confirmed" : "not_confirmed",
|
|
46
|
+
summary: code === 0
|
|
47
|
+
? `Deterministic runtime command succeeded: ${command.join(" ")}`
|
|
48
|
+
: `Deterministic runtime command failed with exit code ${code}: ${command.join(" ")}`,
|
|
49
|
+
evidence,
|
|
50
|
+
});
|
|
51
|
+
});
|
|
52
|
+
});
|
|
30
53
|
}
|
|
31
54
|
export async function runIntakeExecutor(bundle, root) {
|
|
32
55
|
const ignore = await loadIgnoreFile(root);
|
|
@@ -54,8 +77,8 @@ export function runStructureExecutor(bundle) {
|
|
|
54
77
|
const disposition = bundle.file_disposition ?? buildFileDisposition(bundle.repo_manifest);
|
|
55
78
|
const unitManifest = buildUnitManifest(bundle.repo_manifest, disposition);
|
|
56
79
|
const surfaceManifest = buildSurfaceManifest(bundle.repo_manifest, disposition);
|
|
57
|
-
const criticalFlows = buildCriticalFlowManifest(bundle.repo_manifest, surfaceManifest, disposition);
|
|
58
80
|
const graphBundle = buildGraphBundle(bundle.repo_manifest, disposition);
|
|
81
|
+
const criticalFlows = buildCriticalFlowManifest(bundle.repo_manifest, surfaceManifest, disposition);
|
|
59
82
|
const riskRegister = buildRiskRegister(unitManifest, criticalFlows, externalAnalyzerResults);
|
|
60
83
|
return {
|
|
61
84
|
updated: {
|
|
@@ -75,10 +98,13 @@ export function runStructureExecutor(bundle) {
|
|
|
75
98
|
"critical_flows.json",
|
|
76
99
|
"risk_register.json",
|
|
77
100
|
],
|
|
78
|
-
progress_summary: `Built structure artifacts for ${unitManifest.units.length} units and ${criticalFlows.flows.length} critical flows
|
|
101
|
+
progress_summary: `Built structure artifacts for ${unitManifest.units.length} units and ${criticalFlows.flows.length} critical flows.` +
|
|
102
|
+
(criticalFlows.fallback_required
|
|
103
|
+
? " Deterministic flow inference did not fully meet the confidence bar."
|
|
104
|
+
: ""),
|
|
79
105
|
};
|
|
80
106
|
}
|
|
81
|
-
export function runPlanningExecutor(bundle, lineIndex = {}) {
|
|
107
|
+
export async function runPlanningExecutor(bundle, root, lineIndex = {}) {
|
|
82
108
|
if (!bundle.repo_manifest) {
|
|
83
109
|
throw new Error("Cannot run planning executor without repo_manifest");
|
|
84
110
|
}
|
|
@@ -91,10 +117,17 @@ export function runPlanningExecutor(bundle, lineIndex = {}) {
|
|
|
91
117
|
}
|
|
92
118
|
const externalAnalyzerResults = bundle.external_analyzer_results;
|
|
93
119
|
const coverage = initializeCoverageFromPlan(bundle.repo_manifest, bundle.unit_manifest, bundle.file_disposition, externalAnalyzerResults);
|
|
94
|
-
const autoSkippedTrivialFiles = autoCompleteTrivialCoverage(coverage, lineIndex, externalAnalyzerResults);
|
|
95
120
|
const flowCoverage = buildFlowCoverage(bundle.critical_flows, coverage);
|
|
96
|
-
const
|
|
97
|
-
const
|
|
121
|
+
const runtimeCommand = await discoverRuntimeValidationCommand(root);
|
|
122
|
+
const runtimeValidationTasks = buildRuntimeValidationTasks({
|
|
123
|
+
unitManifest: bundle.unit_manifest,
|
|
124
|
+
criticalFlows: bundle.critical_flows,
|
|
125
|
+
flowCoverage,
|
|
126
|
+
command: runtimeCommand,
|
|
127
|
+
});
|
|
128
|
+
const runtimeValidationReport = runtimeValidationTasks.tasks.length > 0
|
|
129
|
+
? mergeRuntimeValidationReport(runtimeValidationTasks, bundle.runtime_validation_report)
|
|
130
|
+
: undefined;
|
|
98
131
|
const baseTasks = buildChunkedAuditTasks(bundle.unit_manifest, lineIndex, {
|
|
99
132
|
external_analyzer_results: externalAnalyzerResults,
|
|
100
133
|
});
|
|
@@ -114,22 +147,20 @@ export function runPlanningExecutor(bundle, lineIndex = {}) {
|
|
|
114
147
|
runtime_validation_report: runtimeValidationReport,
|
|
115
148
|
audit_tasks: auditTasks,
|
|
116
149
|
requeue_tasks: requeuePayload.tasks,
|
|
150
|
+
audit_report: undefined,
|
|
117
151
|
},
|
|
118
152
|
artifacts_written: [
|
|
119
153
|
"coverage_matrix.json",
|
|
120
154
|
"flow_coverage.json",
|
|
121
155
|
"runtime_validation_tasks.json",
|
|
122
|
-
"runtime_validation_report.json",
|
|
156
|
+
...(runtimeValidationReport ? ["runtime_validation_report.json"] : []),
|
|
123
157
|
"audit_tasks.json",
|
|
124
158
|
"requeue_tasks.json",
|
|
125
159
|
],
|
|
126
160
|
progress_summary: `Built planning artifacts; generated ${auditTasks.length} tasks and ${requeuePayload.task_count} requeue tasks.` +
|
|
127
|
-
(
|
|
128
|
-
? `
|
|
129
|
-
: "")
|
|
130
|
-
(externalAnalyzerResults?.results.length
|
|
131
|
-
? ` External analyzer signals influenced lenses and produced ${analyzerTasks.length} dedicated follow-up task(s).`
|
|
132
|
-
: ""),
|
|
161
|
+
(runtimeCommand
|
|
162
|
+
? ` Runtime validation will use: ${runtimeCommand.join(" ")}.`
|
|
163
|
+
: " No deterministic runtime validation command was discovered."),
|
|
133
164
|
};
|
|
134
165
|
}
|
|
135
166
|
export function runResultIngestionExecutor(bundle, results) {
|
|
@@ -140,84 +171,109 @@ export function runResultIngestionExecutor(bundle, results) {
|
|
|
140
171
|
const flowCoverage = bundle.critical_flows
|
|
141
172
|
? buildFlowCoverage(bundle.critical_flows, updatedCoverageMatrix)
|
|
142
173
|
: bundle.flow_coverage;
|
|
143
|
-
const runtimeValidationTasks = bundle.unit_manifest
|
|
144
|
-
? buildRuntimeValidationTasks(bundle.unit_manifest, bundle.critical_flows, flowCoverage)
|
|
145
|
-
: bundle.runtime_validation_tasks;
|
|
146
|
-
const runtimeValidationReport = runtimeValidationTasks
|
|
147
|
-
? preserveOrPlaceholder(runtimeValidationTasks, bundle.runtime_validation_report)
|
|
148
|
-
: bundle.runtime_validation_report;
|
|
149
174
|
const requeuePayload = buildRequeuePayload(updatedCoverageMatrix, bundle.critical_flows, flowCoverage, bundle.external_analyzer_results);
|
|
150
175
|
const mergedResults = [...(bundle.audit_results ?? []), ...results];
|
|
151
176
|
const updatedAuditTasks = updateAuditTaskStatuses(bundle.audit_tasks, mergedResults);
|
|
152
|
-
const synthesisReport = buildSynthesisReport(mergedResults, runtimeValidationReport, bundle.external_analyzer_results);
|
|
153
|
-
const synthesisArtifacts = buildSynthesisArtifacts(synthesisReport);
|
|
154
177
|
return {
|
|
155
178
|
updated: {
|
|
156
179
|
...bundle,
|
|
157
180
|
coverage_matrix: updatedCoverageMatrix,
|
|
158
181
|
flow_coverage: flowCoverage,
|
|
159
|
-
runtime_validation_tasks: runtimeValidationTasks,
|
|
160
|
-
runtime_validation_report: runtimeValidationReport,
|
|
161
182
|
audit_results: mergedResults,
|
|
162
183
|
audit_tasks: updatedAuditTasks,
|
|
163
184
|
requeue_tasks: requeuePayload.tasks,
|
|
164
|
-
|
|
185
|
+
audit_report: undefined,
|
|
165
186
|
},
|
|
166
187
|
artifacts_written: [
|
|
167
188
|
"coverage_matrix.json",
|
|
168
189
|
"flow_coverage.json",
|
|
169
|
-
"runtime_validation_tasks.json",
|
|
170
|
-
"runtime_validation_report.json",
|
|
171
190
|
"audit_results.jsonl",
|
|
172
191
|
"audit_tasks.json",
|
|
173
192
|
"requeue_tasks.json",
|
|
174
|
-
"merged_findings.json",
|
|
175
|
-
"root_cause_clusters.json",
|
|
176
|
-
"synthesis_report.json",
|
|
177
193
|
],
|
|
178
194
|
progress_summary: `Ingested ${results.length} audit result entries and refreshed dependent artifacts.`,
|
|
179
195
|
};
|
|
180
196
|
}
|
|
197
|
+
export async function runRuntimeValidationExecutor(bundle, root) {
|
|
198
|
+
if (!bundle.runtime_validation_tasks) {
|
|
199
|
+
throw new Error("Cannot execute runtime validation without runtime_validation_tasks");
|
|
200
|
+
}
|
|
201
|
+
const existing = bundle.runtime_validation_report ?? { results: [] };
|
|
202
|
+
const byTaskId = new Map(existing.results.map((result) => [result.task_id, result]));
|
|
203
|
+
const byCommand = new Map();
|
|
204
|
+
for (const task of bundle.runtime_validation_tasks.tasks) {
|
|
205
|
+
const prior = byTaskId.get(task.id);
|
|
206
|
+
if (prior &&
|
|
207
|
+
["confirmed", "not_confirmed", "inconclusive", "not_required"].includes(prior.status)) {
|
|
208
|
+
continue;
|
|
209
|
+
}
|
|
210
|
+
if (!task.command || task.command.length === 0) {
|
|
211
|
+
byTaskId.set(task.id, {
|
|
212
|
+
task_id: task.id,
|
|
213
|
+
status: "not_required",
|
|
214
|
+
summary: `No deterministic runtime command was available for ${task.id}.`,
|
|
215
|
+
evidence: [],
|
|
216
|
+
notes: ["Runtime validation was not planned for this task."],
|
|
217
|
+
});
|
|
218
|
+
continue;
|
|
219
|
+
}
|
|
220
|
+
const signature = task.command.join("\0");
|
|
221
|
+
const outcome = byCommand.get(signature) ?? (await runCommand(task.command, root));
|
|
222
|
+
byCommand.set(signature, outcome);
|
|
223
|
+
byTaskId.set(task.id, {
|
|
224
|
+
task_id: task.id,
|
|
225
|
+
status: outcome.status,
|
|
226
|
+
summary: outcome.summary,
|
|
227
|
+
evidence: outcome.evidence,
|
|
228
|
+
notes: [`Target paths: ${task.target_paths.join(", ")}`],
|
|
229
|
+
});
|
|
230
|
+
}
|
|
231
|
+
return {
|
|
232
|
+
updated: {
|
|
233
|
+
...bundle,
|
|
234
|
+
runtime_validation_report: {
|
|
235
|
+
results: [...byTaskId.values()].sort((a, b) => a.task_id.localeCompare(b.task_id)),
|
|
236
|
+
},
|
|
237
|
+
audit_report: undefined,
|
|
238
|
+
},
|
|
239
|
+
artifacts_written: ["runtime_validation_report.json"],
|
|
240
|
+
progress_summary: `Executed deterministic runtime validation for ${bundle.runtime_validation_tasks.tasks.length} task(s).`,
|
|
241
|
+
};
|
|
242
|
+
}
|
|
181
243
|
export function runRuntimeValidationUpdateExecutor(bundle, updates) {
|
|
182
244
|
if (!bundle.runtime_validation_tasks) {
|
|
183
245
|
throw new Error("Cannot update runtime validation without runtime_validation_tasks");
|
|
184
246
|
}
|
|
185
|
-
const existingReport = bundle.runtime_validation_report ??
|
|
186
|
-
buildPlaceholderRuntimeValidationReport(bundle.runtime_validation_tasks);
|
|
247
|
+
const existingReport = bundle.runtime_validation_report ?? { results: [] };
|
|
187
248
|
const mergedReport = updateRuntimeValidationReport(bundle.runtime_validation_tasks, existingReport, updates);
|
|
188
|
-
const synthesisReport = buildSynthesisReport(bundle.audit_results ?? [], mergedReport, bundle.external_analyzer_results);
|
|
189
|
-
const synthesisArtifacts = buildSynthesisArtifacts(synthesisReport);
|
|
190
249
|
return {
|
|
191
250
|
updated: {
|
|
192
251
|
...bundle,
|
|
193
252
|
runtime_validation_report: mergedReport,
|
|
194
|
-
|
|
253
|
+
audit_report: undefined,
|
|
195
254
|
},
|
|
196
|
-
artifacts_written: [
|
|
197
|
-
"runtime_validation_report.json",
|
|
198
|
-
"merged_findings.json",
|
|
199
|
-
"root_cause_clusters.json",
|
|
200
|
-
"synthesis_report.json",
|
|
201
|
-
],
|
|
255
|
+
artifacts_written: ["runtime_validation_report.json"],
|
|
202
256
|
progress_summary: `Merged ${updates.results.length} runtime validation updates.`,
|
|
203
257
|
};
|
|
204
258
|
}
|
|
205
259
|
export function runSynthesisExecutor(bundle, results) {
|
|
206
260
|
const finalResults = results ?? bundle.audit_results ?? [];
|
|
207
|
-
const
|
|
208
|
-
|
|
261
|
+
const model = buildAuditReportModel({
|
|
262
|
+
results: finalResults,
|
|
263
|
+
unitManifest: bundle.unit_manifest,
|
|
264
|
+
graphBundle: bundle.graph_bundle,
|
|
265
|
+
criticalFlows: bundle.critical_flows,
|
|
266
|
+
coverageMatrix: bundle.coverage_matrix,
|
|
267
|
+
runtimeValidationReport: bundle.runtime_validation_report,
|
|
268
|
+
});
|
|
209
269
|
return {
|
|
210
270
|
updated: {
|
|
211
271
|
...bundle,
|
|
212
272
|
audit_results: finalResults,
|
|
213
|
-
|
|
273
|
+
audit_report: renderAuditReportMarkdown(model),
|
|
214
274
|
},
|
|
215
|
-
artifacts_written: [
|
|
216
|
-
|
|
217
|
-
"root_cause_clusters.json",
|
|
218
|
-
"synthesis_report.json",
|
|
219
|
-
],
|
|
220
|
-
progress_summary: `Refreshed synthesis for ${finalResults.length} audit result entries.`,
|
|
275
|
+
artifacts_written: ["audit-report.md"],
|
|
276
|
+
progress_summary: `Rendered deterministic audit report for ${finalResults.length} audit result entries.`,
|
|
221
277
|
};
|
|
222
278
|
}
|
|
223
279
|
export function runExternalAnalyzerImportExecutor(bundle, externalResults) {
|
|
@@ -226,6 +282,7 @@ export function runExternalAnalyzerImportExecutor(bundle, externalResults) {
|
|
|
226
282
|
updated: {
|
|
227
283
|
...bundle,
|
|
228
284
|
external_analyzer_results: externalResults,
|
|
285
|
+
audit_report: undefined,
|
|
229
286
|
},
|
|
230
287
|
artifacts_written: ["external_analyzer_results.json"],
|
|
231
288
|
progress_summary: summary,
|
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
import { buildRequeueTargets } from "../coverage.js";
|
|
2
|
-
function taskPriority(hasExternalSignal) {
|
|
3
|
-
|
|
2
|
+
function taskPriority(hasExternalSignal, lens) {
|
|
3
|
+
if (hasExternalSignal)
|
|
4
|
+
return "high";
|
|
5
|
+
if (lens === "security" || lens === "data_integrity" || lens === "reliability") {
|
|
6
|
+
return "medium";
|
|
7
|
+
}
|
|
8
|
+
return "low";
|
|
4
9
|
}
|
|
5
10
|
export function buildRequeueTasks(matrix, externalAnalyzerResults) {
|
|
6
11
|
const targets = buildRequeueTargets(matrix);
|
|
@@ -15,8 +20,8 @@ export function buildRequeueTasks(matrix, externalAnalyzerResults) {
|
|
|
15
20
|
pass_id: `requeue:${lens}`,
|
|
16
21
|
lens,
|
|
17
22
|
file_paths: [target.path],
|
|
18
|
-
rationale: `
|
|
19
|
-
priority: taskPriority(hasExternalSignal),
|
|
23
|
+
rationale: `Mandatory audit coverage is still missing for ${target.path} under the ${lens} lens.`,
|
|
24
|
+
priority: taskPriority(hasExternalSignal, lens),
|
|
20
25
|
tags: hasExternalSignal ? ["external_analyzer_signal"] : [],
|
|
21
26
|
status: "pending",
|
|
22
27
|
});
|
|
@@ -1,15 +1,14 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { applyFileCoverage } from "../coverage.js";
|
|
2
2
|
export function ingestAuditResults(coverageMatrix, results) {
|
|
3
3
|
const matrix = JSON.parse(JSON.stringify(coverageMatrix));
|
|
4
|
-
const
|
|
5
|
-
path:
|
|
6
|
-
|
|
7
|
-
end: range.end,
|
|
4
|
+
const fileCoverage = results.flatMap((result) => result.file_coverage.map((coverage) => ({
|
|
5
|
+
path: coverage.path,
|
|
6
|
+
total_lines: coverage.total_lines,
|
|
8
7
|
pass_id: result.pass_id,
|
|
9
8
|
lens: result.lens,
|
|
10
9
|
agent_role: result.agent_role,
|
|
11
10
|
})));
|
|
12
|
-
|
|
11
|
+
applyFileCoverage(matrix, fileCoverage);
|
|
13
12
|
return matrix;
|
|
14
13
|
}
|
|
15
14
|
export function updateAuditTaskStatuses(tasks, results) {
|
|
@@ -2,6 +2,11 @@ import type { UnitManifest } from "../types.js";
|
|
|
2
2
|
import type { FlowCoverageManifest } from "../types/flowCoverage.js";
|
|
3
3
|
import type { CriticalFlowManifest } from "../types/flows.js";
|
|
4
4
|
import type { RuntimeValidationReport, RuntimeValidationTaskManifest } from "../types/runtimeValidation.js";
|
|
5
|
-
export declare function
|
|
6
|
-
export declare function
|
|
5
|
+
export declare function discoverRuntimeValidationCommand(root: string): Promise<string[] | undefined>;
|
|
6
|
+
export declare function buildRuntimeValidationTasks(params: {
|
|
7
|
+
unitManifest: UnitManifest;
|
|
8
|
+
criticalFlows?: CriticalFlowManifest;
|
|
9
|
+
flowCoverage?: FlowCoverageManifest;
|
|
10
|
+
command?: string[];
|
|
11
|
+
}): RuntimeValidationTaskManifest;
|
|
7
12
|
export declare function mergeRuntimeValidationReport(tasks: RuntimeValidationTaskManifest, existing?: RuntimeValidationReport): RuntimeValidationReport;
|
|
@@ -1,23 +1,59 @@
|
|
|
1
|
+
import { access, readFile } from "node:fs/promises";
|
|
1
2
|
function checksForFlow(requiredLenses) {
|
|
2
3
|
const checks = [];
|
|
3
4
|
if (requiredLenses.includes("security")) {
|
|
4
5
|
checks.push("Exercise malformed or unauthorized inputs against the flow.");
|
|
5
6
|
}
|
|
6
7
|
if (requiredLenses.includes("reliability")) {
|
|
7
|
-
checks.push("Exercise retries
|
|
8
|
+
checks.push("Exercise retries or repeated submissions.");
|
|
8
9
|
}
|
|
9
10
|
if (requiredLenses.includes("correctness")) {
|
|
10
|
-
checks.push("Exercise representative success and edge-case
|
|
11
|
+
checks.push("Exercise representative success and edge-case behavior.");
|
|
11
12
|
}
|
|
12
13
|
if (requiredLenses.includes("data_integrity")) {
|
|
13
|
-
checks.push("Verify state transitions and persistence invariants
|
|
14
|
+
checks.push("Verify state transitions and persistence invariants.");
|
|
14
15
|
}
|
|
15
16
|
return checks;
|
|
16
17
|
}
|
|
17
|
-
|
|
18
|
+
async function exists(path) {
|
|
19
|
+
try {
|
|
20
|
+
await access(path);
|
|
21
|
+
return true;
|
|
22
|
+
}
|
|
23
|
+
catch {
|
|
24
|
+
return false;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
export async function discoverRuntimeValidationCommand(root) {
|
|
28
|
+
const packageJsonPath = `${root}/package.json`;
|
|
29
|
+
if (await exists(packageJsonPath)) {
|
|
30
|
+
try {
|
|
31
|
+
const packageJson = JSON.parse(await readFile(packageJsonPath, "utf8"));
|
|
32
|
+
const testScript = packageJson.scripts?.test?.trim();
|
|
33
|
+
if (testScript &&
|
|
34
|
+
!/no test specified/i.test(testScript)) {
|
|
35
|
+
return ["npm", "test"];
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
// ignore unreadable package.json for runtime discovery
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
if (await exists(`${root}/go.mod`)) {
|
|
43
|
+
return ["go", "test", "./..."];
|
|
44
|
+
}
|
|
45
|
+
if (await exists(`${root}/pyproject.toml`) || await exists(`${root}/pytest.ini`)) {
|
|
46
|
+
return ["python", "-m", "pytest"];
|
|
47
|
+
}
|
|
48
|
+
return undefined;
|
|
49
|
+
}
|
|
50
|
+
export function buildRuntimeValidationTasks(params) {
|
|
51
|
+
if (!params.command) {
|
|
52
|
+
return { tasks: [] };
|
|
53
|
+
}
|
|
18
54
|
const tasks = [];
|
|
19
55
|
const seen = new Set();
|
|
20
|
-
const highRiskUnits = unitManifest.units.filter((unit) => (unit.risk_score ?? 0) >= 5 ||
|
|
56
|
+
const highRiskUnits = params.unitManifest.units.filter((unit) => (unit.risk_score ?? 0) >= 5 ||
|
|
21
57
|
unit.required_lenses.includes("security") ||
|
|
22
58
|
unit.required_lenses.includes("data_integrity"));
|
|
23
59
|
for (const unit of highRiskUnits) {
|
|
@@ -31,16 +67,17 @@ export function buildRuntimeValidationTasks(unitManifest, criticalFlows, flowCov
|
|
|
31
67
|
target_paths: unit.files,
|
|
32
68
|
reason: `Unit ${unit.unit_id} is high risk or touches sensitive concerns.`,
|
|
33
69
|
priority: (unit.risk_score ?? 0) >= 7 ? "high" : "medium",
|
|
70
|
+
command: params.command,
|
|
34
71
|
suggested_checks: [
|
|
35
|
-
"Run
|
|
36
|
-
"
|
|
72
|
+
"Run the deterministic runtime command for the repository.",
|
|
73
|
+
"Confirm the affected unit does not regress under the command output.",
|
|
37
74
|
],
|
|
38
75
|
source_artifacts: ["unit_manifest.json", "risk_register.json"],
|
|
39
76
|
});
|
|
40
77
|
}
|
|
41
|
-
if (criticalFlows && flowCoverage) {
|
|
42
|
-
const flowMap = new Map(criticalFlows.flows.map((flow) => [flow.id, flow]));
|
|
43
|
-
for (const record of flowCoverage.flows) {
|
|
78
|
+
if (params.criticalFlows && params.flowCoverage) {
|
|
79
|
+
const flowMap = new Map(params.criticalFlows.flows.map((flow) => [flow.id, flow]));
|
|
80
|
+
for (const record of params.flowCoverage.flows) {
|
|
44
81
|
if (record.status === "complete") {
|
|
45
82
|
continue;
|
|
46
83
|
}
|
|
@@ -56,53 +93,28 @@ export function buildRuntimeValidationTasks(unitManifest, criticalFlows, flowCov
|
|
|
56
93
|
id,
|
|
57
94
|
kind: "critical-flow-check",
|
|
58
95
|
target_paths: flow.paths,
|
|
59
|
-
reason: `Critical flow ${record.flow_id} is still ${record.status} and
|
|
96
|
+
reason: `Critical flow ${record.flow_id} is still ${record.status} and needs deterministic runtime validation.`,
|
|
60
97
|
priority: record.status === "pending" ? "high" : "medium",
|
|
98
|
+
command: params.command,
|
|
61
99
|
suggested_checks: checksForFlow(record.required_lenses),
|
|
62
100
|
source_artifacts: ["critical_flows.json", "flow_coverage.json"],
|
|
63
101
|
});
|
|
64
102
|
}
|
|
65
103
|
}
|
|
66
|
-
const RUNTIME_TASK_CAP = 100;
|
|
67
|
-
if (tasks.length > RUNTIME_TASK_CAP) {
|
|
68
|
-
process.stderr.write(`[runtime-validation] generated ${tasks.length} tasks which exceeds the cap of ${RUNTIME_TASK_CAP}; truncating to avoid runaway expansion\n`);
|
|
69
|
-
tasks.splice(RUNTIME_TASK_CAP);
|
|
70
|
-
}
|
|
71
104
|
return { tasks };
|
|
72
105
|
}
|
|
73
|
-
export function buildPlaceholderRuntimeValidationReport(tasks) {
|
|
74
|
-
return {
|
|
75
|
-
results: tasks.tasks.map((task) => ({
|
|
76
|
-
task_id: task.id,
|
|
77
|
-
status: "pending",
|
|
78
|
-
summary: `No runtime evidence recorded yet for ${task.id}.`,
|
|
79
|
-
evidence: [],
|
|
80
|
-
notes: ["Placeholder entry generated from runtime validation task list."],
|
|
81
|
-
})),
|
|
82
|
-
};
|
|
83
|
-
}
|
|
84
106
|
export function mergeRuntimeValidationReport(tasks, existing) {
|
|
85
107
|
const existingMap = new Map((existing?.results ?? []).map((result) => [result.task_id, result]));
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
return {
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
}
|
|
99
|
-
return {
|
|
100
|
-
task_id: task.id,
|
|
101
|
-
status: "pending",
|
|
102
|
-
summary: `No runtime evidence recorded yet for ${task.id}.`,
|
|
103
|
-
evidence: [],
|
|
104
|
-
notes: ["Placeholder entry generated from runtime validation task list."],
|
|
105
|
-
};
|
|
106
|
-
});
|
|
107
|
-
return { results: mergedResults };
|
|
108
|
+
return {
|
|
109
|
+
results: tasks.tasks.map((task) => {
|
|
110
|
+
const prior = existingMap.get(task.id);
|
|
111
|
+
return (prior ?? {
|
|
112
|
+
task_id: task.id,
|
|
113
|
+
status: "pending",
|
|
114
|
+
summary: `Deterministic runtime validation has not executed yet for ${task.id}.`,
|
|
115
|
+
evidence: [],
|
|
116
|
+
notes: [],
|
|
117
|
+
});
|
|
118
|
+
}),
|
|
119
|
+
};
|
|
108
120
|
}
|
|
@@ -38,11 +38,9 @@ export function updateRuntimeValidationReport(tasks, existing, updates) {
|
|
|
38
38
|
merged.set(task.id, {
|
|
39
39
|
task_id: task.id,
|
|
40
40
|
status: "pending",
|
|
41
|
-
summary: `
|
|
41
|
+
summary: `Deterministic runtime validation has not executed yet for ${task.id}.`,
|
|
42
42
|
evidence: [],
|
|
43
|
-
notes: [
|
|
44
|
-
"Placeholder entry generated from runtime validation task list.",
|
|
45
|
-
],
|
|
43
|
+
notes: [],
|
|
46
44
|
});
|
|
47
45
|
}
|
|
48
46
|
}
|
|
@@ -60,15 +60,23 @@ export function deriveAuditState(bundle) {
|
|
|
60
60
|
has(bundle.audit_tasks)) {
|
|
61
61
|
obligations.push(obligation("audit_tasks_completed", "satisfied"));
|
|
62
62
|
}
|
|
63
|
-
obligations.push(obligation("audit_results_ingested",
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
63
|
+
obligations.push(obligation("audit_results_ingested", (bundle.audit_tasks?.length ?? 0) === 0 || has(bundle.audit_results)
|
|
64
|
+
? "present"
|
|
65
|
+
: "missing"));
|
|
66
|
+
const runtimeTasks = bundle.runtime_validation_tasks?.tasks ?? [];
|
|
67
|
+
const runtimeResults = bundle.runtime_validation_report?.results ?? [];
|
|
68
|
+
const runtimeReady = runtimeTasks.length === 0 ||
|
|
69
|
+
(runtimeTasks.length > 0 &&
|
|
70
|
+
runtimeTasks.every((task) => runtimeResults.some((result) => result.task_id === task.id &&
|
|
71
|
+
result.status !== "pending")));
|
|
72
|
+
obligations.push(obligation("runtime_validation_current", runtimeReady
|
|
73
|
+
? "satisfied"
|
|
74
|
+
: has(bundle.runtime_validation_report)
|
|
75
|
+
? "missing"
|
|
76
|
+
: "missing", runtimeTasks.length === 0
|
|
77
|
+
? "No deterministic runtime validation tasks were planned."
|
|
78
|
+
: undefined));
|
|
79
|
+
obligations.push(obligation("synthesis_current", staleOrSatisfied(staleArtifacts, ["audit-report.md"], has(bundle.audit_report))));
|
|
72
80
|
let status = "not_started";
|
|
73
81
|
if (!has(bundle.repo_manifest)) {
|
|
74
82
|
status = "not_started";
|
|
@@ -80,10 +88,7 @@ export function deriveAuditState(bundle) {
|
|
|
80
88
|
status = "active";
|
|
81
89
|
}
|
|
82
90
|
const incomplete = obligations.some((o) => o.state === "missing" || o.state === "stale");
|
|
83
|
-
if (!incomplete &&
|
|
84
|
-
has(bundle.synthesis_report) &&
|
|
85
|
-
has(bundle.merged_findings) &&
|
|
86
|
-
has(bundle.root_cause_clusters)) {
|
|
91
|
+
if (!incomplete && has(bundle.audit_report)) {
|
|
87
92
|
status = "complete";
|
|
88
93
|
}
|
|
89
94
|
return {
|
|
@@ -20,6 +20,9 @@ export function isTrivialAuditPath(path, lineCount, hasExternalSignal = false) {
|
|
|
20
20
|
if (name === "__init__.py" && lineCount <= 3) {
|
|
21
21
|
return true;
|
|
22
22
|
}
|
|
23
|
+
if (lineCount <= 1) {
|
|
24
|
+
return true;
|
|
25
|
+
}
|
|
23
26
|
return false;
|
|
24
27
|
}
|
|
25
28
|
export function autoCompleteTrivialCoverage(coverage, lineIndex, externalAnalyzerResults) {
|
|
@@ -35,11 +38,11 @@ export function autoCompleteTrivialCoverage(coverage, lineIndex, externalAnalyze
|
|
|
35
38
|
if (file.required_lenses.length === 0) {
|
|
36
39
|
continue;
|
|
37
40
|
}
|
|
38
|
-
file.completed_lenses = [
|
|
39
|
-
file.
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
41
|
+
file.completed_lenses = [];
|
|
42
|
+
file.required_lenses = [];
|
|
43
|
+
file.audit_status = "excluded";
|
|
44
|
+
file.classification_status = "excluded_trivial";
|
|
45
|
+
file.unit_ids = [];
|
|
43
46
|
skipped.push(file.path);
|
|
44
47
|
}
|
|
45
48
|
return skipped.sort();
|