auditor-lambda 0.6.10 → 0.6.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/cleanup.d.ts +1 -0
- package/dist/cli/cleanup.js +23 -0
- package/dist/cli/nextStepCommand.d.ts +1 -0
- package/dist/cli/nextStepCommand.js +553 -0
- package/dist/cli/paths.d.ts +1 -0
- package/dist/cli/paths.js +7 -0
- package/dist/cli/reviewRun.d.ts +5 -0
- package/dist/cli/reviewRun.js +24 -1
- package/dist/cli/runToCompletion.d.ts +1 -0
- package/dist/cli/runToCompletion.js +878 -0
- package/dist/cli/semanticReviewStep.d.ts +11 -0
- package/dist/cli/semanticReviewStep.js +103 -0
- package/dist/cli.d.ts +2 -2
- package/dist/cli.js +16 -1543
- package/dist/extractors/graphManifestEdges.d.ts +2 -4
- package/dist/extractors/graphManifestEdges.js +5 -28
- package/dist/extractors/graphPathUtils.d.ts +16 -0
- package/dist/extractors/graphPathUtils.js +40 -0
- package/dist/orchestrator/reviewPackets.d.ts +2 -1
- package/dist/orchestrator/reviewPackets.js +4 -21
- package/package.json +1 -1
|
@@ -0,0 +1,878 @@
|
|
|
1
|
+
import { mkdir, readFile } from "node:fs/promises";
|
|
2
|
+
import { join, resolve } from "node:path";
|
|
3
|
+
import { readJsonFile, writeJsonFile } from "@audit-tools/shared";
|
|
4
|
+
import { buildQuotaSource } from "@audit-tools/shared/quota/compositeQuotaSource";
|
|
5
|
+
import { loadArtifactBundle, writeCoreArtifacts, promoteFinalAuditReport, } from "../io/artifacts.js";
|
|
6
|
+
import { decideNextStep } from "../orchestrator/nextStep.js";
|
|
7
|
+
import { deriveAuditState } from "../orchestrator/state.js";
|
|
8
|
+
import { estimateTaskGroupTokens, sizeIndexFromManifest, } from "../orchestrator/reviewPackets.js";
|
|
9
|
+
import { createFreshSessionProvider, resolveFreshSessionProviderName, } from "../providers/index.js";
|
|
10
|
+
import { LOCAL_SUBPROCESS_PROVIDER_NAME } from "../providers/constants.js";
|
|
11
|
+
import { loadSessionConfig } from "../supervisor/sessionConfig.js";
|
|
12
|
+
import { appendRunLedgerEntry } from "../supervisor/runLedger.js";
|
|
13
|
+
import { buildRunId, clearDispatchFiles, ensureSupervisorDirs, getRunPaths, writeDispatchBatchFiles, writeWorkerTaskFiles, } from "../io/runArtifacts.js";
|
|
14
|
+
import { renderWorkerPrompt } from "../prompts/renderWorkerPrompt.js";
|
|
15
|
+
import { validateAuditResults, formatAuditResultIssues, } from "../validation/auditResults.js";
|
|
16
|
+
import { scheduleWave, buildProviderModelKey, readQuotaState, recordWaveOutcome, resolveHostActiveSubagentLimit, detectRateLimitError, computeCooldownUntil, runSlidingWindow, lookupDiscoveredLimits, updateDiscoveredLimits, mergeDiscoveredLimits, getHeaderExtractorForProvider, } from "../quota/index.js";
|
|
17
|
+
import { runAuditStep } from "./auditStep.js";
|
|
18
|
+
import { persistConfigErrorHandoff } from "./reviewRun.js";
|
|
19
|
+
import { buildBlockedAuditState, buildManualReviewBlocker, emitEnvelope, shouldRunInlineExecutor, } from "./envelope.js";
|
|
20
|
+
import { renderSemanticReviewStep } from "./semanticReviewStep.js";
|
|
21
|
+
import { buildPendingAuditTasks } from "./dispatch.js";
|
|
22
|
+
import { addFileLineCountHints, buildLineIndexForPaths, } from "./lineIndex.js";
|
|
23
|
+
import { WORKER_RESULT_CONTRACT_VERSION, buildWorkerResult, persistWorkerRunArtifacts, isWorkerResult, buildWorkerFailureBlocker, } from "./workerResult.js";
|
|
24
|
+
import { readWaveManifest, writeWaveManifest, removeWaveManifest, buildWaveSlotEntry, } from "./waveManifest.js";
|
|
25
|
+
import { cleanupStaleArtifactsDir } from "./cleanup.js";
|
|
26
|
+
import { getRootDir, warnIfNotGitRepo, getArtifactsDir, getUiMode, getMaxRuns, getAgentBatchSize, getParallelWorkers, getTimeoutMs, getHostModel, getBatchResultsDir, getFlag, listBatchResultFiles, getExplicitProvider, chunkArray, resolveHostDispatchCapability, getOptionalBooleanFlag, getHostMaxActiveSubagents, summarizeLaunchExit, } from "./args.js";
|
|
27
|
+
export async function cmdRunToCompletion(argv) {
|
|
28
|
+
const root = getRootDir(argv);
|
|
29
|
+
warnIfNotGitRepo(root);
|
|
30
|
+
const artifactsDir = getArtifactsDir(argv);
|
|
31
|
+
await cleanupStaleArtifactsDir(artifactsDir);
|
|
32
|
+
await mkdir(artifactsDir, { recursive: true });
|
|
33
|
+
await ensureSupervisorDirs(artifactsDir);
|
|
34
|
+
let sessionConfig;
|
|
35
|
+
try {
|
|
36
|
+
sessionConfig = await loadSessionConfig(artifactsDir);
|
|
37
|
+
}
|
|
38
|
+
catch (error) {
|
|
39
|
+
await persistConfigErrorHandoff({
|
|
40
|
+
root,
|
|
41
|
+
artifactsDir,
|
|
42
|
+
progressSummary: error instanceof Error ? error.message : String(error),
|
|
43
|
+
});
|
|
44
|
+
throw error;
|
|
45
|
+
}
|
|
46
|
+
const explicitProvider = getExplicitProvider(argv);
|
|
47
|
+
const provider = createFreshSessionProvider(explicitProvider, sessionConfig);
|
|
48
|
+
const uiMode = getUiMode(argv, sessionConfig.ui_mode ?? "headless");
|
|
49
|
+
const maxRuns = getMaxRuns(argv);
|
|
50
|
+
const agentBatchSize = getAgentBatchSize(argv, sessionConfig);
|
|
51
|
+
const parallelWorkers = getParallelWorkers(argv, sessionConfig);
|
|
52
|
+
const timeoutMs = getTimeoutMs(argv, sessionConfig);
|
|
53
|
+
const hostModel = getHostModel(argv);
|
|
54
|
+
const selfCliPath = resolve(argv[1] ?? process.argv[1] ?? "");
|
|
55
|
+
const batchResultsDir = getBatchResultsDir(argv);
|
|
56
|
+
if (batchResultsDir && getFlag(argv, "--results")) {
|
|
57
|
+
throw new Error("Use either --results <file> or --batch-results <dir>, not both.");
|
|
58
|
+
}
|
|
59
|
+
let pendingBatchAuditResults = batchResultsDir
|
|
60
|
+
? await listBatchResultFiles(batchResultsDir)
|
|
61
|
+
: [];
|
|
62
|
+
let pendingAuditResultsPath = getFlag(argv, "--results");
|
|
63
|
+
let pendingRuntimeUpdatesPath = getFlag(argv, "--updates");
|
|
64
|
+
let pendingExternalAnalyzerPath = getFlag(argv, "--external-analyzer-results");
|
|
65
|
+
let runCount = 0;
|
|
66
|
+
let deepeningCycles = 0;
|
|
67
|
+
const MAX_DEEPENING_CYCLES = 3;
|
|
68
|
+
let anyProgress = false;
|
|
69
|
+
let lastResult = null;
|
|
70
|
+
const artifactsWritten = new Set();
|
|
71
|
+
while (runCount < maxRuns) {
|
|
72
|
+
const bundle = await loadArtifactBundle(artifactsDir);
|
|
73
|
+
const decision = decideNextStep(bundle);
|
|
74
|
+
// Resume interrupted parallel wave: ingest any results that workers
|
|
75
|
+
// wrote before the previous process exited.
|
|
76
|
+
const priorWave = await readWaveManifest(artifactsDir);
|
|
77
|
+
if (priorWave) {
|
|
78
|
+
process.stderr.write(`[audit-code] Recovering interrupted wave (${priorWave.slots.length} slot(s), obligation ${priorWave.obligation_id}).\n`);
|
|
79
|
+
let recoveredProgress = false;
|
|
80
|
+
for (const entry of priorWave.slots) {
|
|
81
|
+
try {
|
|
82
|
+
const results = await readJsonFile(entry.audit_results_path);
|
|
83
|
+
if (!results || results.length === 0)
|
|
84
|
+
continue;
|
|
85
|
+
const stepResult = await runAuditStep({
|
|
86
|
+
root,
|
|
87
|
+
artifactsDir,
|
|
88
|
+
preferredExecutor: "result_ingestion_executor",
|
|
89
|
+
auditResultsPath: entry.audit_results_path,
|
|
90
|
+
});
|
|
91
|
+
if (stepResult.progress_made) {
|
|
92
|
+
recoveredProgress = true;
|
|
93
|
+
anyProgress = true;
|
|
94
|
+
for (const a of stepResult.artifacts_written)
|
|
95
|
+
artifactsWritten.add(a);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
process.stderr.write(`[audit-code] Skipping unreadable results for ${entry.run_id}.\n`);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
await removeWaveManifest(artifactsDir);
|
|
103
|
+
if (recoveredProgress)
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
if (decision.selected_executor === "agent" &&
|
|
107
|
+
bundle.audit_tasks?.some((t) => t.tags?.includes("selective_deepening") &&
|
|
108
|
+
t.status !== "complete") &&
|
|
109
|
+
!bundle.audit_tasks?.some((t) => !t.tags?.includes("selective_deepening") &&
|
|
110
|
+
t.status !== "complete")) {
|
|
111
|
+
deepeningCycles++;
|
|
112
|
+
if (deepeningCycles > MAX_DEEPENING_CYCLES) {
|
|
113
|
+
process.stderr.write(`[audit-code] Reached max deepening cycles (${MAX_DEEPENING_CYCLES}). Stopping to prevent churn.\n`);
|
|
114
|
+
break;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
let preferredExecutor = decision.selected_executor;
|
|
118
|
+
let obligationId = decision.selected_obligation;
|
|
119
|
+
let auditResultsPath;
|
|
120
|
+
let runtimeUpdatesPath;
|
|
121
|
+
let externalAnalyzerPath;
|
|
122
|
+
if (pendingExternalAnalyzerPath) {
|
|
123
|
+
preferredExecutor = "external_analyzer_import_executor";
|
|
124
|
+
obligationId = "external_analyzer_import";
|
|
125
|
+
externalAnalyzerPath = pendingExternalAnalyzerPath;
|
|
126
|
+
}
|
|
127
|
+
else if (pendingBatchAuditResults.length > 0 && bundle.coverage_matrix) {
|
|
128
|
+
preferredExecutor = "result_ingestion_executor";
|
|
129
|
+
obligationId = "audit_results_ingested";
|
|
130
|
+
auditResultsPath = pendingBatchAuditResults[0];
|
|
131
|
+
}
|
|
132
|
+
else if (pendingAuditResultsPath && bundle.coverage_matrix) {
|
|
133
|
+
preferredExecutor = "result_ingestion_executor";
|
|
134
|
+
obligationId = "audit_results_ingested";
|
|
135
|
+
auditResultsPath = pendingAuditResultsPath;
|
|
136
|
+
}
|
|
137
|
+
else if (pendingRuntimeUpdatesPath && bundle.runtime_validation_tasks) {
|
|
138
|
+
preferredExecutor = "runtime_validation_update_executor";
|
|
139
|
+
obligationId = "runtime_validation_current";
|
|
140
|
+
runtimeUpdatesPath = pendingRuntimeUpdatesPath;
|
|
141
|
+
}
|
|
142
|
+
if (preferredExecutor === "agent" && provider.name === LOCAL_SUBPROCESS_PROVIDER_NAME) {
|
|
143
|
+
const blocker = buildManualReviewBlocker(provider.name);
|
|
144
|
+
const blockedState = buildBlockedAuditState({
|
|
145
|
+
state: decision.state,
|
|
146
|
+
obligationId,
|
|
147
|
+
executor: preferredExecutor,
|
|
148
|
+
blocker,
|
|
149
|
+
});
|
|
150
|
+
await writeCoreArtifacts(artifactsDir, {
|
|
151
|
+
...bundle,
|
|
152
|
+
audit_state: blockedState,
|
|
153
|
+
});
|
|
154
|
+
const blockRunId = buildRunId(obligationId, runCount + 1);
|
|
155
|
+
const blockPaths = getRunPaths(artifactsDir, blockRunId);
|
|
156
|
+
const blockPendingTasks = await addFileLineCountHints(root, buildPendingAuditTasks(bundle));
|
|
157
|
+
const blockPendingTasksPath = join(blockPaths.runDir, "pending-audit-tasks.json");
|
|
158
|
+
const blockAuditResultsPath = join(blockPaths.runDir, "audit-results.json");
|
|
159
|
+
const blockReadPaths = new Set();
|
|
160
|
+
for (const pt of blockPendingTasks) {
|
|
161
|
+
for (const fp of pt.file_paths)
|
|
162
|
+
blockReadPaths.add(fp);
|
|
163
|
+
}
|
|
164
|
+
const blockTask = {
|
|
165
|
+
contract_version: "audit-code-worker/v1alpha1",
|
|
166
|
+
run_id: blockRunId,
|
|
167
|
+
repo_root: root,
|
|
168
|
+
artifacts_dir: artifactsDir,
|
|
169
|
+
obligation_id: obligationId,
|
|
170
|
+
preferred_executor: preferredExecutor,
|
|
171
|
+
result_path: blockPaths.resultPath,
|
|
172
|
+
worker_command: [
|
|
173
|
+
process.execPath,
|
|
174
|
+
selfCliPath,
|
|
175
|
+
"worker-run",
|
|
176
|
+
"--task",
|
|
177
|
+
blockPaths.taskPath,
|
|
178
|
+
],
|
|
179
|
+
audit_results_path: blockAuditResultsPath,
|
|
180
|
+
pending_audit_tasks_path: blockPendingTasksPath,
|
|
181
|
+
timeout_ms: timeoutMs,
|
|
182
|
+
max_retries: 0,
|
|
183
|
+
access: {
|
|
184
|
+
read_paths: [...blockReadPaths],
|
|
185
|
+
write_paths: [blockAuditResultsPath, blockPaths.resultPath],
|
|
186
|
+
},
|
|
187
|
+
};
|
|
188
|
+
const blockPrompt = renderWorkerPrompt(blockTask);
|
|
189
|
+
await writeWorkerTaskFiles(blockTask, blockPrompt, blockPaths, artifactsDir, blockPendingTasks);
|
|
190
|
+
await writeJsonFile(blockPendingTasksPath, blockPendingTasks);
|
|
191
|
+
const reviewRun = {
|
|
192
|
+
run_id: blockRunId,
|
|
193
|
+
task_path: blockPaths.taskPath,
|
|
194
|
+
prompt_path: blockPaths.promptPath,
|
|
195
|
+
pending_audit_tasks_path: blockPendingTasksPath,
|
|
196
|
+
audit_results_path: blockAuditResultsPath,
|
|
197
|
+
worker_command: blockTask.worker_command,
|
|
198
|
+
};
|
|
199
|
+
// Render the actionable dispatch / single-task step here instead of
|
|
200
|
+
// leaving the host to issue next-step as a second command. Capability is
|
|
201
|
+
// resolved from flags/config/env with a sane default, so nothing is
|
|
202
|
+
// required from the host to make progress. If rendering fails we still
|
|
203
|
+
// emit the hand-off below — run-to-completion is never worse than before,
|
|
204
|
+
// and next-step will re-render and surface the error loudly.
|
|
205
|
+
try {
|
|
206
|
+
await renderSemanticReviewStep({
|
|
207
|
+
root,
|
|
208
|
+
artifactsDir,
|
|
209
|
+
activeReviewRun: reviewRun,
|
|
210
|
+
hostCanDispatch: resolveHostDispatchCapability({
|
|
211
|
+
explicit: getOptionalBooleanFlag(argv, "--host-can-dispatch-subagents"),
|
|
212
|
+
sessionConfig,
|
|
213
|
+
}),
|
|
214
|
+
hostMaxActiveSubagents: getHostMaxActiveSubagents(argv),
|
|
215
|
+
hostCanRestrictSubagentTools: getOptionalBooleanFlag(argv, "--host-can-restrict-subagent-tools") ?? false,
|
|
216
|
+
hostCanSelectSubagentModel: getOptionalBooleanFlag(argv, "--host-can-select-subagent-model") ?? false,
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
catch (stepError) {
|
|
220
|
+
process.stderr.write(`[audit-code] Could not pre-render the review step; the operator hand-off points to next-step instead. ${stepError instanceof Error ? stepError.message : String(stepError)}\n`);
|
|
221
|
+
}
|
|
222
|
+
await emitEnvelope({
|
|
223
|
+
root,
|
|
224
|
+
artifactsDir,
|
|
225
|
+
bundle: {
|
|
226
|
+
...bundle,
|
|
227
|
+
audit_state: blockedState,
|
|
228
|
+
},
|
|
229
|
+
audit_state: blockedState,
|
|
230
|
+
selected_obligation: obligationId,
|
|
231
|
+
selected_executor: preferredExecutor,
|
|
232
|
+
progress_made: anyProgress,
|
|
233
|
+
artifacts_written: Array.from(new Set([...artifactsWritten, "audit_state.json"])),
|
|
234
|
+
progress_summary: blocker,
|
|
235
|
+
next_likely_step: null,
|
|
236
|
+
providerName: provider.name,
|
|
237
|
+
activeReviewRun: reviewRun,
|
|
238
|
+
});
|
|
239
|
+
return;
|
|
240
|
+
}
|
|
241
|
+
if (!preferredExecutor) {
|
|
242
|
+
const state = decision.state;
|
|
243
|
+
await clearDispatchFiles(artifactsDir);
|
|
244
|
+
await emitEnvelope({
|
|
245
|
+
root,
|
|
246
|
+
artifactsDir,
|
|
247
|
+
bundle,
|
|
248
|
+
audit_state: state,
|
|
249
|
+
selected_obligation: anyProgress
|
|
250
|
+
? (lastResult?.obligation_id ?? null)
|
|
251
|
+
: null,
|
|
252
|
+
selected_executor: anyProgress
|
|
253
|
+
? (lastResult?.selected_executor ?? null)
|
|
254
|
+
: null,
|
|
255
|
+
progress_made: anyProgress,
|
|
256
|
+
artifacts_written: Array.from(artifactsWritten),
|
|
257
|
+
progress_summary: anyProgress && state.status === "complete"
|
|
258
|
+
? `Completed audit in ${runCount} fresh worker runs.`
|
|
259
|
+
: decision.reason,
|
|
260
|
+
next_likely_step: state.status === "complete" ? null : decision.selected_obligation,
|
|
261
|
+
providerName: provider.name,
|
|
262
|
+
});
|
|
263
|
+
if (state.status === "complete") {
|
|
264
|
+
await promoteFinalAuditReport({ artifactsDir, repoRoot: root });
|
|
265
|
+
}
|
|
266
|
+
return;
|
|
267
|
+
}
|
|
268
|
+
if (preferredExecutor === "agent" && parallelWorkers > 1) {
|
|
269
|
+
const quotaState = await readQuotaState();
|
|
270
|
+
const providerModelKey = buildProviderModelKey(provider.name, hostModel);
|
|
271
|
+
const quotaStateEntry = quotaState.entries[providerModelKey] ?? null;
|
|
272
|
+
const allCandidateTasks = buildPendingAuditTasks(bundle);
|
|
273
|
+
const candidateGroups = chunkArray(allCandidateTasks.slice(0, parallelWorkers * agentBatchSize), agentBatchSize);
|
|
274
|
+
const candidateSizeIndex = sizeIndexFromManifest(bundle.repo_manifest);
|
|
275
|
+
const slotTokenEstimates = candidateGroups.map((g) => estimateTaskGroupTokens(g, candidateSizeIndex));
|
|
276
|
+
const providerLimits = await provider.queryLimits?.(hostModel)
|
|
277
|
+
.then((r) => r ? { ...r, source: "provider_query" } : null)
|
|
278
|
+
.catch(() => null)
|
|
279
|
+
?? null;
|
|
280
|
+
const cachedLimits = await lookupDiscoveredLimits(providerModelKey).catch(() => null);
|
|
281
|
+
const discoveredLimits = mergeDiscoveredLimits(providerLimits, cachedLimits);
|
|
282
|
+
const halfLifeHours = sessionConfig.quota?.empirical_half_life_hours ?? 24;
|
|
283
|
+
const quotaSource = buildQuotaSource({ halfLifeHours });
|
|
284
|
+
const quotaSourceSnapshot = await quotaSource.queryCurrentUsage(providerModelKey).catch(() => null);
|
|
285
|
+
const hostConcurrencyLimit = resolveHostActiveSubagentLimit({
|
|
286
|
+
sessionConfig,
|
|
287
|
+
});
|
|
288
|
+
const waveSchedule = scheduleWave({
|
|
289
|
+
providerName: resolveFreshSessionProviderName(getExplicitProvider(argv), sessionConfig),
|
|
290
|
+
sessionConfig,
|
|
291
|
+
hostModel,
|
|
292
|
+
requestedConcurrency: parallelWorkers,
|
|
293
|
+
estimatedSlotTokens: slotTokenEstimates,
|
|
294
|
+
quotaStateEntry,
|
|
295
|
+
hostConcurrencyLimit,
|
|
296
|
+
quotaSourceSnapshot,
|
|
297
|
+
discoveredLimits,
|
|
298
|
+
});
|
|
299
|
+
const waveSize = waveSchedule.wave_size;
|
|
300
|
+
if (waveSchedule.cooldown_until) {
|
|
301
|
+
const waitMs = new Date(waveSchedule.cooldown_until).getTime() - Date.now();
|
|
302
|
+
if (waitMs > 0) {
|
|
303
|
+
const cappedWait = Math.min(waitMs, 120_000);
|
|
304
|
+
process.stderr.write(`[quota] Cooldown active — waiting ${Math.ceil(cappedWait / 1000)}s before next wave.\n`);
|
|
305
|
+
await new Promise((r) => setTimeout(r, cappedWait));
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
const taskGroups = candidateGroups.slice(0, waveSize);
|
|
309
|
+
const workerSlots = [];
|
|
310
|
+
for (const rawGroup of taskGroups) {
|
|
311
|
+
const group = await addFileLineCountHints(root, rawGroup);
|
|
312
|
+
runCount += 1;
|
|
313
|
+
const slotRunId = buildRunId(obligationId, runCount);
|
|
314
|
+
const slotPaths = getRunPaths(artifactsDir, slotRunId);
|
|
315
|
+
const slotAuditResultsPath = join(slotPaths.runDir, "audit-results.json");
|
|
316
|
+
const slotPendingTasksPath = join(slotPaths.runDir, "pending-audit-tasks.json");
|
|
317
|
+
const slotReadPaths = new Set();
|
|
318
|
+
for (const t of group) {
|
|
319
|
+
for (const fp of t.file_paths)
|
|
320
|
+
slotReadPaths.add(fp);
|
|
321
|
+
}
|
|
322
|
+
const slotTask = {
|
|
323
|
+
contract_version: "audit-code-worker/v1alpha1",
|
|
324
|
+
run_id: slotRunId,
|
|
325
|
+
repo_root: root,
|
|
326
|
+
artifacts_dir: artifactsDir,
|
|
327
|
+
obligation_id: obligationId,
|
|
328
|
+
preferred_executor: "agent",
|
|
329
|
+
result_path: slotPaths.resultPath,
|
|
330
|
+
worker_command: [process.execPath, selfCliPath, "worker-run", "--task", slotPaths.taskPath],
|
|
331
|
+
audit_results_path: slotAuditResultsPath,
|
|
332
|
+
pending_audit_tasks_path: slotPendingTasksPath,
|
|
333
|
+
worker_command_mode: "deferred",
|
|
334
|
+
timeout_ms: timeoutMs,
|
|
335
|
+
max_retries: 0,
|
|
336
|
+
access: {
|
|
337
|
+
read_paths: [...slotReadPaths],
|
|
338
|
+
write_paths: [slotAuditResultsPath, slotPaths.resultPath],
|
|
339
|
+
},
|
|
340
|
+
};
|
|
341
|
+
const slotPrompt = renderWorkerPrompt(slotTask);
|
|
342
|
+
await writeWorkerTaskFiles(slotTask, slotPrompt, slotPaths, artifactsDir, group, { updateDispatch: false });
|
|
343
|
+
await writeJsonFile(slotPendingTasksPath, group);
|
|
344
|
+
workerSlots.push({ runId: slotRunId, paths: slotPaths, auditResultsPath: slotAuditResultsPath, pendingTasksPath: slotPendingTasksPath, group });
|
|
345
|
+
}
|
|
346
|
+
await writeDispatchBatchFiles(artifactsDir, workerSlots.map((slot) => ({
|
|
347
|
+
run_id: slot.runId,
|
|
348
|
+
task_path: slot.paths.taskPath,
|
|
349
|
+
prompt_path: slot.paths.promptPath,
|
|
350
|
+
result_path: slot.paths.resultPath,
|
|
351
|
+
status_path: slot.paths.statusPath,
|
|
352
|
+
audit_results_path: slot.auditResultsPath,
|
|
353
|
+
pending_audit_tasks_path: slot.pendingTasksPath,
|
|
354
|
+
})), workerSlots.flatMap((slot) => slot.group));
|
|
355
|
+
const parallelStartedAt = new Date().toISOString();
|
|
356
|
+
await writeWaveManifest(artifactsDir, {
|
|
357
|
+
obligation_id: obligationId ?? "unknown",
|
|
358
|
+
started_at: parallelStartedAt,
|
|
359
|
+
pid: process.pid,
|
|
360
|
+
slots: workerSlots.map(buildWaveSlotEntry),
|
|
361
|
+
});
|
|
362
|
+
const { results: launchResults } = await runSlidingWindow(workerSlots.map((slot) => () => provider.launch({
|
|
363
|
+
repoRoot: root,
|
|
364
|
+
runId: slot.runId,
|
|
365
|
+
obligationId,
|
|
366
|
+
promptPath: slot.paths.promptPath,
|
|
367
|
+
taskPath: slot.paths.taskPath,
|
|
368
|
+
resultPath: slot.paths.resultPath,
|
|
369
|
+
stdoutPath: slot.paths.stdoutPath,
|
|
370
|
+
stderrPath: slot.paths.stderrPath,
|
|
371
|
+
uiMode,
|
|
372
|
+
timeoutMs,
|
|
373
|
+
})), waveSize);
|
|
374
|
+
const launchErrorsByRunId = new Map();
|
|
375
|
+
for (let index = 0; index < launchResults.length; index++) {
|
|
376
|
+
const outcome = launchResults[index];
|
|
377
|
+
if (outcome?.status === "rejected") {
|
|
378
|
+
launchErrorsByRunId.set(workerSlots[index].runId, outcome.reason instanceof Error
|
|
379
|
+
? outcome.reason.message
|
|
380
|
+
: String(outcome.reason));
|
|
381
|
+
}
|
|
382
|
+
else if (outcome?.status === "fulfilled") {
|
|
383
|
+
const launchExitSummary = summarizeLaunchExit(outcome.value);
|
|
384
|
+
if (launchExitSummary) {
|
|
385
|
+
launchErrorsByRunId.set(workerSlots[index].runId, launchExitSummary);
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
// Result ingestion is intentionally sequential even though agent launch
|
|
390
|
+
// was parallel. Writing to coverage_matrix.json is not atomic, so
|
|
391
|
+
// concurrent ingest calls would race and corrupt coverage state.
|
|
392
|
+
let batchProgress = false;
|
|
393
|
+
const batchErrors = [];
|
|
394
|
+
for (const slot of workerSlots) {
|
|
395
|
+
const parallelEndedAt = new Date().toISOString();
|
|
396
|
+
let workerResult = buildWorkerResult({
|
|
397
|
+
runId: slot.runId,
|
|
398
|
+
obligationId,
|
|
399
|
+
status: "no_progress",
|
|
400
|
+
progressMade: false,
|
|
401
|
+
selectedExecutor: "agent",
|
|
402
|
+
artifactsWritten: [],
|
|
403
|
+
summary: "Parallel worker batch made no progress.",
|
|
404
|
+
nextLikelyStep: obligationId,
|
|
405
|
+
errors: [],
|
|
406
|
+
});
|
|
407
|
+
try {
|
|
408
|
+
const launchError = launchErrorsByRunId.get(slot.runId);
|
|
409
|
+
if (launchError) {
|
|
410
|
+
throw new Error(`Worker launch failed: ${launchError}`);
|
|
411
|
+
}
|
|
412
|
+
const auditResults = await readJsonFile(slot.auditResultsPath);
|
|
413
|
+
const pendingTaskIds = new Set(slot.group.map((t) => t.task_id));
|
|
414
|
+
const matchedCount = auditResults.filter((r) => pendingTaskIds.has(r.task_id)).length;
|
|
415
|
+
if (slot.group.length > 0 && matchedCount === 0) {
|
|
416
|
+
throw new Error("Worker did not emit any audit results for the assigned tasks.");
|
|
417
|
+
}
|
|
418
|
+
const issues = validateAuditResults(auditResults, slot.group, {
|
|
419
|
+
lineIndex: await buildLineIndexForPaths(root, slot.group.flatMap((task) => task.file_paths)),
|
|
420
|
+
});
|
|
421
|
+
const errors = issues.filter((issue) => issue.severity === "error");
|
|
422
|
+
const warnings = issues.filter((issue) => issue.severity === "warning");
|
|
423
|
+
if (warnings.length > 0) {
|
|
424
|
+
process.stderr.write(`audit-results validation: ${warnings.length} warning(s) for ${slot.runId}:\n` +
|
|
425
|
+
formatAuditResultIssues(warnings) + "\n");
|
|
426
|
+
}
|
|
427
|
+
if (errors.length > 0) {
|
|
428
|
+
throw new Error(`audit-results validation failed with ${errors.length} error(s):\n` +
|
|
429
|
+
formatAuditResultIssues(errors));
|
|
430
|
+
}
|
|
431
|
+
const stepResult = await runAuditStep({
|
|
432
|
+
root,
|
|
433
|
+
artifactsDir,
|
|
434
|
+
preferredExecutor: "result_ingestion_executor",
|
|
435
|
+
auditResultsPath: slot.auditResultsPath,
|
|
436
|
+
});
|
|
437
|
+
workerResult = buildWorkerResult({
|
|
438
|
+
runId: slot.runId,
|
|
439
|
+
obligationId,
|
|
440
|
+
status: stepResult.progress_made ? "completed" : "no_progress",
|
|
441
|
+
progressMade: stepResult.progress_made,
|
|
442
|
+
selectedExecutor: stepResult.selected_executor,
|
|
443
|
+
artifactsWritten: stepResult.artifacts_written,
|
|
444
|
+
summary: stepResult.progress_summary,
|
|
445
|
+
nextLikelyStep: stepResult.next_likely_step,
|
|
446
|
+
errors: [],
|
|
447
|
+
});
|
|
448
|
+
batchProgress ||= stepResult.progress_made;
|
|
449
|
+
if (stepResult.progress_made)
|
|
450
|
+
anyProgress = true;
|
|
451
|
+
for (const a of stepResult.artifacts_written)
|
|
452
|
+
artifactsWritten.add(a);
|
|
453
|
+
}
|
|
454
|
+
catch (error) {
|
|
455
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
456
|
+
batchErrors.push(`${slot.runId}: ${message}`);
|
|
457
|
+
workerResult = buildWorkerResult({
|
|
458
|
+
runId: slot.runId,
|
|
459
|
+
obligationId,
|
|
460
|
+
status: "failed",
|
|
461
|
+
progressMade: false,
|
|
462
|
+
selectedExecutor: "agent",
|
|
463
|
+
artifactsWritten: [],
|
|
464
|
+
summary: `Worker failed for executor agent: ${message}`,
|
|
465
|
+
nextLikelyStep: obligationId,
|
|
466
|
+
errors: [message],
|
|
467
|
+
});
|
|
468
|
+
process.stderr.write(`[agent-batch] ${slot.runId} failed: ${message}\n`);
|
|
469
|
+
}
|
|
470
|
+
await persistWorkerRunArtifacts(slot.paths, workerResult, "parallel-deferred-agent");
|
|
471
|
+
await appendRunLedgerEntry(artifactsDir, {
|
|
472
|
+
run_id: slot.runId,
|
|
473
|
+
provider: provider.name,
|
|
474
|
+
obligation_id: obligationId,
|
|
475
|
+
selected_executor: workerResult.selected_executor,
|
|
476
|
+
status: workerResult.status,
|
|
477
|
+
started_at: parallelStartedAt,
|
|
478
|
+
ended_at: parallelEndedAt,
|
|
479
|
+
result_path: slot.paths.resultPath,
|
|
480
|
+
});
|
|
481
|
+
artifactsWritten.add("run-ledger.json");
|
|
482
|
+
}
|
|
483
|
+
// Record outcome for adaptive learning (best-effort — never blocks dispatch)
|
|
484
|
+
{
|
|
485
|
+
const rateLimitResults = batchErrors.map((e) => detectRateLimitError(e));
|
|
486
|
+
const rateLimitHit = rateLimitResults.find((r) => r.isRateLimited);
|
|
487
|
+
const retryAfterMs = rateLimitHit?.retryAfterMs ?? null;
|
|
488
|
+
await recordWaveOutcome(providerModelKey, {
|
|
489
|
+
concurrency: workerSlots.length,
|
|
490
|
+
estimated_tokens: slotTokenEstimates.slice(0, workerSlots.length).reduce((a, b) => a + b, 0),
|
|
491
|
+
outcome: rateLimitHit ? "rate_limited" : batchErrors.length > 0 ? "timeout" : "success",
|
|
492
|
+
cooldown_until: rateLimitHit ? computeCooldownUntil(retryAfterMs) : null,
|
|
493
|
+
}, sessionConfig.quota?.empirical_half_life_hours ?? 24).catch(() => undefined);
|
|
494
|
+
}
|
|
495
|
+
// Extract rate-limit headers from worker stderr (best-effort)
|
|
496
|
+
{
|
|
497
|
+
const extractor = getHeaderExtractorForProvider(provider.name);
|
|
498
|
+
for (const slot of workerSlots) {
|
|
499
|
+
try {
|
|
500
|
+
const stderr = await readFile(slot.paths.stderrPath, "utf8");
|
|
501
|
+
const extracted = extractor.extract(stderr);
|
|
502
|
+
if (extracted && (extracted.requests_per_minute != null || extracted.input_tokens_per_minute != null)) {
|
|
503
|
+
await updateDiscoveredLimits(providerModelKey, {
|
|
504
|
+
requests_per_minute: extracted.requests_per_minute,
|
|
505
|
+
input_tokens_per_minute: extracted.input_tokens_per_minute,
|
|
506
|
+
source: "header_extraction",
|
|
507
|
+
});
|
|
508
|
+
break; // one successful extraction is enough
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
catch {
|
|
512
|
+
// stderr file missing or unreadable — skip
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
await removeWaveManifest(artifactsDir);
|
|
517
|
+
if (batchErrors.length > 0) {
|
|
518
|
+
const bundleAfter = await loadArtifactBundle(artifactsDir);
|
|
519
|
+
const blockedState = buildBlockedAuditState({
|
|
520
|
+
state: bundleAfter.audit_state ?? deriveAuditState(bundleAfter),
|
|
521
|
+
obligationId,
|
|
522
|
+
executor: "agent",
|
|
523
|
+
blocker: `Parallel worker batch failed for ${batchErrors.length} run(s). ` +
|
|
524
|
+
batchErrors.slice(0, 3).join(" | "),
|
|
525
|
+
});
|
|
526
|
+
await writeCoreArtifacts(artifactsDir, {
|
|
527
|
+
...bundleAfter,
|
|
528
|
+
audit_state: blockedState,
|
|
529
|
+
});
|
|
530
|
+
await emitEnvelope({
|
|
531
|
+
root,
|
|
532
|
+
artifactsDir,
|
|
533
|
+
bundle: { ...bundleAfter, audit_state: blockedState },
|
|
534
|
+
audit_state: blockedState,
|
|
535
|
+
selected_obligation: obligationId,
|
|
536
|
+
selected_executor: "agent",
|
|
537
|
+
progress_made: anyProgress,
|
|
538
|
+
artifacts_written: Array.from(new Set([...artifactsWritten, "audit_state.json"])),
|
|
539
|
+
progress_summary: `Parallel worker batch failed for ${batchErrors.length} run(s).\n` +
|
|
540
|
+
batchErrors.join("\n"),
|
|
541
|
+
next_likely_step: null,
|
|
542
|
+
providerName: provider.name,
|
|
543
|
+
});
|
|
544
|
+
return;
|
|
545
|
+
}
|
|
546
|
+
if (!batchProgress) {
|
|
547
|
+
const bundleAfter = await loadArtifactBundle(artifactsDir);
|
|
548
|
+
const state = bundleAfter.audit_state ?? deriveAuditState(bundleAfter);
|
|
549
|
+
await emitEnvelope({
|
|
550
|
+
root,
|
|
551
|
+
artifactsDir,
|
|
552
|
+
bundle: bundleAfter,
|
|
553
|
+
audit_state: state,
|
|
554
|
+
selected_obligation: obligationId,
|
|
555
|
+
selected_executor: "agent",
|
|
556
|
+
progress_made: anyProgress,
|
|
557
|
+
artifacts_written: Array.from(artifactsWritten),
|
|
558
|
+
progress_summary: "Parallel worker batch made no progress.",
|
|
559
|
+
next_likely_step: obligationId,
|
|
560
|
+
providerName: provider.name,
|
|
561
|
+
});
|
|
562
|
+
return;
|
|
563
|
+
}
|
|
564
|
+
continue;
|
|
565
|
+
}
|
|
566
|
+
runCount += 1;
|
|
567
|
+
const runId = buildRunId(obligationId, runCount);
|
|
568
|
+
const paths = getRunPaths(artifactsDir, runId);
|
|
569
|
+
if (shouldRunInlineExecutor(preferredExecutor)) {
|
|
570
|
+
await clearDispatchFiles(artifactsDir);
|
|
571
|
+
const startedAt = new Date().toISOString();
|
|
572
|
+
let workerResult;
|
|
573
|
+
try {
|
|
574
|
+
const result = await runAuditStep({
|
|
575
|
+
root,
|
|
576
|
+
artifactsDir,
|
|
577
|
+
preferredExecutor,
|
|
578
|
+
auditResultsPath,
|
|
579
|
+
runtimeUpdatesPath,
|
|
580
|
+
externalAnalyzerPath,
|
|
581
|
+
analyzers: sessionConfig.analyzers,
|
|
582
|
+
since: getFlag(argv, "--since"),
|
|
583
|
+
});
|
|
584
|
+
workerResult = {
|
|
585
|
+
contract_version: WORKER_RESULT_CONTRACT_VERSION,
|
|
586
|
+
run_id: runId,
|
|
587
|
+
obligation_id: obligationId,
|
|
588
|
+
status: result.progress_made ? "completed" : "no_progress",
|
|
589
|
+
progress_made: result.progress_made,
|
|
590
|
+
selected_executor: result.selected_executor,
|
|
591
|
+
artifacts_written: result.artifacts_written,
|
|
592
|
+
summary: result.progress_summary,
|
|
593
|
+
next_likely_step: result.next_likely_step,
|
|
594
|
+
errors: [],
|
|
595
|
+
};
|
|
596
|
+
}
|
|
597
|
+
catch (error) {
|
|
598
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
599
|
+
workerResult = {
|
|
600
|
+
contract_version: WORKER_RESULT_CONTRACT_VERSION,
|
|
601
|
+
run_id: runId,
|
|
602
|
+
obligation_id: obligationId,
|
|
603
|
+
status: "failed",
|
|
604
|
+
progress_made: false,
|
|
605
|
+
selected_executor: preferredExecutor,
|
|
606
|
+
artifacts_written: [],
|
|
607
|
+
summary: `Inline executor failed for ${preferredExecutor}: ${message}`,
|
|
608
|
+
next_likely_step: decision.selected_obligation,
|
|
609
|
+
errors: [message],
|
|
610
|
+
};
|
|
611
|
+
}
|
|
612
|
+
await persistWorkerRunArtifacts(paths, workerResult, "inline");
|
|
613
|
+
await appendRunLedgerEntry(artifactsDir, {
|
|
614
|
+
run_id: runId,
|
|
615
|
+
provider: provider.name,
|
|
616
|
+
obligation_id: obligationId,
|
|
617
|
+
selected_executor: workerResult.selected_executor,
|
|
618
|
+
status: workerResult.status,
|
|
619
|
+
started_at: startedAt,
|
|
620
|
+
ended_at: new Date().toISOString(),
|
|
621
|
+
result_path: paths.resultPath,
|
|
622
|
+
});
|
|
623
|
+
lastResult = workerResult;
|
|
624
|
+
if (workerResult.progress_made) {
|
|
625
|
+
anyProgress = true;
|
|
626
|
+
}
|
|
627
|
+
for (const artifact of workerResult.artifacts_written) {
|
|
628
|
+
artifactsWritten.add(artifact);
|
|
629
|
+
}
|
|
630
|
+
artifactsWritten.add("run-ledger.json");
|
|
631
|
+
if (externalAnalyzerPath)
|
|
632
|
+
pendingExternalAnalyzerPath = undefined;
|
|
633
|
+
if (auditResultsPath &&
|
|
634
|
+
pendingBatchAuditResults[0] === auditResultsPath &&
|
|
635
|
+
preferredExecutor === "result_ingestion_executor" &&
|
|
636
|
+
workerResult.status !== "failed" &&
|
|
637
|
+
workerResult.status !== "blocked") {
|
|
638
|
+
pendingBatchAuditResults.shift();
|
|
639
|
+
}
|
|
640
|
+
if (auditResultsPath)
|
|
641
|
+
pendingAuditResultsPath = undefined;
|
|
642
|
+
if (runtimeUpdatesPath)
|
|
643
|
+
pendingRuntimeUpdatesPath = undefined;
|
|
644
|
+
if (workerResult.status === "failed" ||
|
|
645
|
+
workerResult.status === "blocked" ||
|
|
646
|
+
workerResult.status === "no_progress") {
|
|
647
|
+
const bundleAfter = await loadArtifactBundle(artifactsDir);
|
|
648
|
+
const shouldBlock = workerResult.status === "failed" || workerResult.status === "blocked";
|
|
649
|
+
const state = shouldBlock
|
|
650
|
+
? buildBlockedAuditState({
|
|
651
|
+
state: bundleAfter.audit_state ?? deriveAuditState(bundleAfter),
|
|
652
|
+
obligationId: workerResult.obligation_id,
|
|
653
|
+
executor: workerResult.selected_executor,
|
|
654
|
+
blocker: buildWorkerFailureBlocker(workerResult),
|
|
655
|
+
})
|
|
656
|
+
: bundleAfter.audit_state ?? deriveAuditState(bundleAfter);
|
|
657
|
+
if (shouldBlock) {
|
|
658
|
+
await writeCoreArtifacts(artifactsDir, {
|
|
659
|
+
...bundleAfter,
|
|
660
|
+
audit_state: state,
|
|
661
|
+
});
|
|
662
|
+
}
|
|
663
|
+
await emitEnvelope({
|
|
664
|
+
root,
|
|
665
|
+
artifactsDir,
|
|
666
|
+
bundle: shouldBlock
|
|
667
|
+
? { ...bundleAfter, audit_state: state }
|
|
668
|
+
: bundleAfter,
|
|
669
|
+
audit_state: state,
|
|
670
|
+
selected_obligation: workerResult.obligation_id,
|
|
671
|
+
selected_executor: workerResult.selected_executor,
|
|
672
|
+
progress_made: anyProgress,
|
|
673
|
+
artifacts_written: Array.from(shouldBlock
|
|
674
|
+
? new Set([...artifactsWritten, "audit_state.json"])
|
|
675
|
+
: artifactsWritten),
|
|
676
|
+
progress_summary: buildWorkerFailureBlocker(workerResult),
|
|
677
|
+
next_likely_step: shouldBlock ? null : workerResult.next_likely_step,
|
|
678
|
+
providerName: provider.name,
|
|
679
|
+
});
|
|
680
|
+
return;
|
|
681
|
+
}
|
|
682
|
+
continue;
|
|
683
|
+
}
|
|
684
|
+
const pendingAuditTasks = preferredExecutor === "agent"
|
|
685
|
+
? await addFileLineCountHints(root, buildPendingAuditTasks(bundle))
|
|
686
|
+
: undefined;
|
|
687
|
+
const pendingAuditTasksPath = preferredExecutor === "agent"
|
|
688
|
+
? join(paths.runDir, "pending-audit-tasks.json")
|
|
689
|
+
: undefined;
|
|
690
|
+
const providerAuditResultsPath = preferredExecutor === "agent"
|
|
691
|
+
? join(paths.runDir, "audit-results.json")
|
|
692
|
+
: auditResultsPath;
|
|
693
|
+
const providerReadPaths = new Set();
|
|
694
|
+
if (pendingAuditTasks) {
|
|
695
|
+
for (const pt of pendingAuditTasks) {
|
|
696
|
+
for (const fp of pt.file_paths)
|
|
697
|
+
providerReadPaths.add(fp);
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
const task = {
|
|
701
|
+
contract_version: "audit-code-worker/v1alpha1",
|
|
702
|
+
run_id: runId,
|
|
703
|
+
repo_root: root,
|
|
704
|
+
artifacts_dir: artifactsDir,
|
|
705
|
+
obligation_id: obligationId,
|
|
706
|
+
preferred_executor: preferredExecutor,
|
|
707
|
+
result_path: paths.resultPath,
|
|
708
|
+
worker_command: [
|
|
709
|
+
process.execPath,
|
|
710
|
+
selfCliPath,
|
|
711
|
+
"worker-run",
|
|
712
|
+
"--task",
|
|
713
|
+
paths.taskPath,
|
|
714
|
+
],
|
|
715
|
+
audit_results_path: providerAuditResultsPath,
|
|
716
|
+
pending_audit_tasks_path: pendingAuditTasksPath,
|
|
717
|
+
runtime_updates_path: runtimeUpdatesPath,
|
|
718
|
+
external_analyzer_results_path: externalAnalyzerPath,
|
|
719
|
+
timeout_ms: timeoutMs,
|
|
720
|
+
max_retries: 0,
|
|
721
|
+
access: providerReadPaths.size > 0 ? {
|
|
722
|
+
read_paths: [...providerReadPaths],
|
|
723
|
+
write_paths: [providerAuditResultsPath ?? paths.resultPath, paths.resultPath],
|
|
724
|
+
} : undefined,
|
|
725
|
+
};
|
|
726
|
+
const prompt = renderWorkerPrompt(task);
|
|
727
|
+
await writeWorkerTaskFiles(task, prompt, paths, artifactsDir, pendingAuditTasks);
|
|
728
|
+
if (pendingAuditTasksPath && pendingAuditTasks) {
|
|
729
|
+
await writeJsonFile(pendingAuditTasksPath, pendingAuditTasks);
|
|
730
|
+
}
|
|
731
|
+
const startedAt = new Date().toISOString();
|
|
732
|
+
let workerResult;
|
|
733
|
+
let launchResult = null;
|
|
734
|
+
try {
|
|
735
|
+
launchResult = await provider.launch({
|
|
736
|
+
repoRoot: root,
|
|
737
|
+
runId,
|
|
738
|
+
obligationId,
|
|
739
|
+
promptPath: paths.promptPath,
|
|
740
|
+
taskPath: paths.taskPath,
|
|
741
|
+
resultPath: paths.resultPath,
|
|
742
|
+
stdoutPath: paths.stdoutPath,
|
|
743
|
+
stderrPath: paths.stderrPath,
|
|
744
|
+
uiMode,
|
|
745
|
+
timeoutMs,
|
|
746
|
+
});
|
|
747
|
+
const candidate = await readJsonFile(paths.resultPath);
|
|
748
|
+
if (isWorkerResult(candidate)) {
|
|
749
|
+
workerResult = candidate;
|
|
750
|
+
}
|
|
751
|
+
else {
|
|
752
|
+
const launchExitSummary = summarizeLaunchExit(launchResult);
|
|
753
|
+
workerResult = {
|
|
754
|
+
contract_version: WORKER_RESULT_CONTRACT_VERSION,
|
|
755
|
+
run_id: runId,
|
|
756
|
+
obligation_id: obligationId,
|
|
757
|
+
status: "failed",
|
|
758
|
+
progress_made: false,
|
|
759
|
+
selected_executor: preferredExecutor,
|
|
760
|
+
artifacts_written: [],
|
|
761
|
+
summary: launchExitSummary
|
|
762
|
+
? `Worker did not emit a valid worker result after provider exit: ${launchExitSummary}`
|
|
763
|
+
: "Worker did not emit a valid worker result.",
|
|
764
|
+
next_likely_step: decision.selected_obligation,
|
|
765
|
+
errors: ["Invalid worker result contract."],
|
|
766
|
+
};
|
|
767
|
+
}
|
|
768
|
+
}
|
|
769
|
+
catch (error) {
|
|
770
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
771
|
+
const launchExitSummary = launchResult && summarizeLaunchExit(launchResult);
|
|
772
|
+
workerResult = {
|
|
773
|
+
contract_version: WORKER_RESULT_CONTRACT_VERSION,
|
|
774
|
+
run_id: runId,
|
|
775
|
+
obligation_id: obligationId,
|
|
776
|
+
status: "failed",
|
|
777
|
+
progress_made: false,
|
|
778
|
+
selected_executor: preferredExecutor,
|
|
779
|
+
artifacts_written: [],
|
|
780
|
+
summary: `Worker launch failed for ${preferredExecutor}: ${launchExitSummary ?? message}`,
|
|
781
|
+
next_likely_step: decision.selected_obligation,
|
|
782
|
+
errors: launchExitSummary ? [message, launchExitSummary] : [message],
|
|
783
|
+
};
|
|
784
|
+
await persistWorkerRunArtifacts(paths, workerResult, "provider-launch");
|
|
785
|
+
}
|
|
786
|
+
await appendRunLedgerEntry(artifactsDir, {
|
|
787
|
+
run_id: runId,
|
|
788
|
+
provider: provider.name,
|
|
789
|
+
obligation_id: obligationId,
|
|
790
|
+
selected_executor: workerResult.selected_executor,
|
|
791
|
+
status: workerResult.status,
|
|
792
|
+
started_at: startedAt,
|
|
793
|
+
ended_at: new Date().toISOString(),
|
|
794
|
+
result_path: paths.resultPath,
|
|
795
|
+
});
|
|
796
|
+
lastResult = workerResult;
|
|
797
|
+
if (workerResult.progress_made) {
|
|
798
|
+
anyProgress = true;
|
|
799
|
+
}
|
|
800
|
+
for (const artifact of workerResult.artifacts_written) {
|
|
801
|
+
artifactsWritten.add(artifact);
|
|
802
|
+
}
|
|
803
|
+
artifactsWritten.add("run-ledger.json");
|
|
804
|
+
if (externalAnalyzerPath)
|
|
805
|
+
pendingExternalAnalyzerPath = undefined;
|
|
806
|
+
if (auditResultsPath &&
|
|
807
|
+
pendingBatchAuditResults[0] === auditResultsPath &&
|
|
808
|
+
preferredExecutor === "result_ingestion_executor" &&
|
|
809
|
+
workerResult.status !== "failed" &&
|
|
810
|
+
workerResult.status !== "blocked") {
|
|
811
|
+
pendingBatchAuditResults.shift();
|
|
812
|
+
}
|
|
813
|
+
if (providerAuditResultsPath)
|
|
814
|
+
pendingAuditResultsPath = undefined;
|
|
815
|
+
if (runtimeUpdatesPath)
|
|
816
|
+
pendingRuntimeUpdatesPath = undefined;
|
|
817
|
+
if (workerResult.status === "failed" ||
|
|
818
|
+
workerResult.status === "blocked" ||
|
|
819
|
+
workerResult.status === "no_progress") {
|
|
820
|
+
const bundleAfter = await loadArtifactBundle(artifactsDir);
|
|
821
|
+
const shouldBlock = workerResult.status === "failed" || workerResult.status === "blocked";
|
|
822
|
+
const state = shouldBlock
|
|
823
|
+
? buildBlockedAuditState({
|
|
824
|
+
state: deriveAuditState(bundleAfter),
|
|
825
|
+
obligationId: workerResult.obligation_id,
|
|
826
|
+
executor: workerResult.selected_executor,
|
|
827
|
+
blocker: buildWorkerFailureBlocker(workerResult),
|
|
828
|
+
})
|
|
829
|
+
: deriveAuditState(bundleAfter);
|
|
830
|
+
if (shouldBlock) {
|
|
831
|
+
await writeCoreArtifacts(artifactsDir, {
|
|
832
|
+
...bundleAfter,
|
|
833
|
+
audit_state: state,
|
|
834
|
+
});
|
|
835
|
+
}
|
|
836
|
+
await emitEnvelope({
|
|
837
|
+
root,
|
|
838
|
+
artifactsDir,
|
|
839
|
+
bundle: shouldBlock
|
|
840
|
+
? { ...bundleAfter, audit_state: state }
|
|
841
|
+
: bundleAfter,
|
|
842
|
+
audit_state: state,
|
|
843
|
+
selected_obligation: workerResult.obligation_id,
|
|
844
|
+
selected_executor: workerResult.selected_executor,
|
|
845
|
+
progress_made: anyProgress,
|
|
846
|
+
artifacts_written: Array.from(shouldBlock
|
|
847
|
+
? new Set([...artifactsWritten, "audit_state.json"])
|
|
848
|
+
: artifactsWritten),
|
|
849
|
+
progress_summary: buildWorkerFailureBlocker(workerResult),
|
|
850
|
+
next_likely_step: shouldBlock ? null : workerResult.next_likely_step,
|
|
851
|
+
providerName: provider.name,
|
|
852
|
+
});
|
|
853
|
+
return;
|
|
854
|
+
}
|
|
855
|
+
}
|
|
856
|
+
const bundle = await loadArtifactBundle(artifactsDir);
|
|
857
|
+
const decision = decideNextStep(bundle);
|
|
858
|
+
const state = decision.state;
|
|
859
|
+
if (state.status === "complete") {
|
|
860
|
+
await clearDispatchFiles(artifactsDir);
|
|
861
|
+
}
|
|
862
|
+
await emitEnvelope({
|
|
863
|
+
root,
|
|
864
|
+
artifactsDir,
|
|
865
|
+
bundle,
|
|
866
|
+
audit_state: state,
|
|
867
|
+
selected_obligation: lastResult?.obligation_id ?? decision.selected_obligation,
|
|
868
|
+
selected_executor: lastResult?.selected_executor ?? decision.selected_executor,
|
|
869
|
+
progress_made: anyProgress,
|
|
870
|
+
artifacts_written: Array.from(artifactsWritten),
|
|
871
|
+
progress_summary: `Reached max run limit (${maxRuns}) before terminal state.`,
|
|
872
|
+
next_likely_step: state.status === "complete" ? null : decision.selected_obligation,
|
|
873
|
+
providerName: provider.name,
|
|
874
|
+
});
|
|
875
|
+
if (state.status === "complete") {
|
|
876
|
+
await promoteFinalAuditReport({ artifactsDir, repoRoot: root });
|
|
877
|
+
}
|
|
878
|
+
}
|