auditor-lambda 0.7.0 → 0.9.0

package/dist/cli/mergeAndIngestCommand.js

@@ -0,0 +1,219 @@
+import { readFile, readdir } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { isFileMissingError, readJsonFile, writeJsonFile } from "@audit-tools/shared";
+import { validateAuditResults } from "../validation/auditResults.js";
+import { runAuditStep } from "./auditStep.js";
+import { DISPATCH_RESULT_MAP_FILENAME, ACTIVE_DISPATCH_FILENAME, loadDispatchResultMap, entriesByTaskId, buildPendingAuditTasks, } from "./dispatch.js";
+import { addFileLineCountHints } from "./lineIndex.js";
+import { isCanonicalResultFilename, getArtifactsDir, getFlag } from "./args.js";
+import { buildWorkerResult } from "./workerResult.js";
+export async function cmdMergeAndIngest(argv) {
+    const runId = getFlag(argv, "--run-id");
+    if (!runId)
+        throw new Error("merge-and-ingest requires --run-id <run_id>");
+    const artifactsDir = getArtifactsDir(argv);
+    const runDir = join(artifactsDir, "runs", runId);
+    const taskResultsDir = join(runDir, "task-results");
+    const auditResultsPath = join(runDir, "audit-results.json");
+    const taskPath = join(runDir, "task.json");
+    const tasksPath = join(runDir, "pending-audit-tasks.json");
+    const workerTask = await readJsonFile(taskPath);
+    const resultMap = await loadDispatchResultMap(runDir);
+    if (!resultMap) {
+        throw new Error(`No ${DISPATCH_RESULT_MAP_FILENAME} found for run ${runId}; run prepare-dispatch first.`);
+    }
+    let allTasks = [];
+    try {
+        allTasks = await readJsonFile(tasksPath);
+    }
+    catch { /* may not exist */ }
+    const entryByTaskId = entriesByTaskId(resultMap.entries);
+    if (entryByTaskId.size !== resultMap.entries.length) {
+        throw new Error(`Dispatch result map for run ${runId} contains duplicate task entries.`);
+    }
+    const expectedPaths = new Set(resultMap.entries.map((entry) => resolve(entry.result_path)));
+    let files;
+    try {
+        files = (await readdir(taskResultsDir)).filter(f => f.endsWith(".json")).sort();
+    }
+    catch {
+        files = [];
+    }
+    const passing = [];
+    const failing = [];
+    const seenTaskIds = new Set();
+    let spuriousFileCount = 0;
+    const fallbackByTaskId = new Map();
+    for (const filename of files) {
+        const filePath = resolve(join(taskResultsDir, filename));
+        if (expectedPaths.has(filePath))
+            continue;
+        // Not part of this round's plan. Still read it so a current task can be
+        // recovered by task_id (e.g. a subagent wrote a valid result under a
+        // non-assigned name).
+        try {
+            const raw = await readFile(filePath, "utf8");
+            const parsed = JSON.parse(raw);
+            if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+                const tid = typeof parsed.task_id === "string"
+                    ? String(parsed.task_id) : undefined;
+                if (tid && !fallbackByTaskId.has(tid)) {
+                    fallbackByTaskId.set(tid, parsed);
+                }
+            }
+        }
+        catch { /* not parseable — skip */ }
+        // Only genuinely stray files are "spurious". Canonical per-task result files
+        // (<stem>_<digest>.json) left by prior deepening rounds in the same
+        // task-results/ dir are legitimate and must not inflate the count or bury
+        // the real stray-file signal (3 -> 191 over a run before this fix).
+        if (!isCanonicalResultFilename(filename)) {
+            spuriousFileCount++;
+            process.stderr.write(`[merge-and-ingest] Warning: unexpected file in task-results/: ${filename}\n`);
+        }
+    }
+    for (const task of allTasks) {
+        const entry = entryByTaskId.get(task.task_id);
+        if (!entry) {
+            failing.push({
+                task_id: task.task_id,
+                errors: ["Missing dispatch result-map entry for assigned task."],
+            });
+            continue;
+        }
+        const filePath = entry.result_path;
+        let obj;
+        try {
+            obj = JSON.parse(await readFile(filePath, "utf8"));
+        }
+        catch (e) {
+            if (isFileMissingError(e)) {
+                const fallback = fallbackByTaskId.get(task.task_id);
+                if (fallback) {
+                    process.stderr.write(`[merge-and-ingest] Recovered result for '${task.task_id}' from unexpected file (matched by task_id)\n`);
+                    obj = fallback;
+                }
+                else {
+                    failing.push({
+                        task_id: task.task_id,
+                        errors: ["Missing audit result for assigned task."],
+                    });
+                    continue;
+                }
+            }
+            else {
+                failing.push({ task_id: task.task_id, errors: [`Invalid JSON: ${e.message}`] });
+                continue;
+            }
+        }
+        const record = obj && typeof obj === "object" && !Array.isArray(obj)
+            ? obj
+            : undefined;
+        const taskId = typeof record?.task_id === "string"
+            ? String(record.task_id) : undefined;
+        const resultErrors = [];
+        if (taskId) {
+            if (seenTaskIds.has(taskId)) {
+                resultErrors.push(`Duplicate audit result for assigned task '${taskId}'.`);
+            }
+            else {
+                seenTaskIds.add(taskId);
+            }
+            if (taskId !== task.task_id) {
+                resultErrors.push(`Result file is assigned to '${task.task_id}' but contains task_id '${taskId}'.`);
+            }
+        }
+        const issues = validateAuditResults([obj], [task], { lineIndex: task.file_line_counts ?? {} });
+        resultErrors.push(...issues
+            .filter(i => i.severity === "error")
+            .map(i => i.message));
+        if (resultErrors.length === 0) {
+            passing.push(obj);
+        }
+        else {
+            failing.push({ task_id: taskId ?? task.task_id, errors: resultErrors });
+        }
+    }
+    await writeJsonFile(auditResultsPath, passing);
+    const failedTasksPath = join(runDir, "failed-tasks.json");
+    if (failing.length > 0) {
+        await writeJsonFile(failedTasksPath, failing);
+    }
+    if (passing.length === 0 && failing.length > 0) {
+        throw new Error(`All ${failing.length} assigned task result(s) were missing or invalid; blocked before ingestion. See ${failedTasksPath}`);
+    }
+    const findingCount = passing.reduce((sum, result) => sum + result.findings.length, 0);
+    let result = null;
+    if (passing.length > 0) {
+        result = await runAuditStep({
+            root: workerTask.repo_root,
+            artifactsDir,
+            preferredExecutor: "result_ingestion_executor",
+            auditResultsPath,
+        });
+        const updatedPendingTasks = await addFileLineCountHints(workerTask.repo_root, buildPendingAuditTasks(result.updated_bundle));
+        await writeJsonFile(tasksPath, updatedPendingTasks);
+    }
+    const activeDispatchPath = join(artifactsDir, ACTIVE_DISPATCH_FILENAME);
+    try {
+        const dispatch = await readJsonFile(activeDispatchPath);
+        if (dispatch.run_id === runId) {
+            dispatch.status = failing.length > 0 ? "active" : "merged";
+            await writeJsonFile(activeDispatchPath, dispatch);
+        }
+    }
+    catch { /* no active dispatch file — skip */ }
+    let retryDispatchPath = null;
+    if (failing.length > 0) {
+        const failedTaskIds = new Set(failing.map((f) => f.task_id));
+        const failedPacketIds = [
+            ...new Set(resultMap.entries
+                .filter((e) => failedTaskIds.has(e.task_id))
+                .map((e) => e.packet_id)),
+        ];
+        const retryDispatch = {
+            run_id: runId,
+            retry_packet_ids: failedPacketIds,
+            failed_task_count: failing.length,
+            accepted_task_count: passing.length,
+        };
+        retryDispatchPath = join(runDir, "retry-dispatch.json");
+        await writeJsonFile(retryDispatchPath, retryDispatch);
+        process.stderr.write(`[merge-and-ingest] ${passing.length} accepted, ${failing.length} failed. ` +
+            `Retry packets: ${failedPacketIds.join(", ")}\n`);
+    }
+    const status = failing.length > 0
+        ? "partial"
+        : (result?.progress_made ? "completed" : "no_progress");
+    const workerResult = buildWorkerResult({
+        runId,
+        obligationId: workerTask.obligation_id,
+        status: failing.length > 0 ? "no_progress" : (result?.progress_made ? "completed" : "no_progress"),
+        progressMade: result?.progress_made ?? false,
+        selectedExecutor: result?.selected_executor ?? null,
+        artifactsWritten: result?.artifacts_written ?? [],
+        summary: result?.progress_summary ?? `${failing.length} task(s) failed`,
+        nextLikelyStep: result?.next_likely_step ?? null,
+        errors: [],
+    });
+    await writeJsonFile(workerTask.result_path, workerResult);
+    console.log(JSON.stringify({
+        run_id: runId,
+        status,
+        accepted_count: passing.length,
+        rejected_count: failing.length,
+        spurious_file_count: spuriousFileCount,
+        finding_count: findingCount,
+        audit_results_path: auditResultsPath,
+        ...(retryDispatchPath ? { retry_dispatch_path: retryDispatchPath } : {}),
+        ...(result ? {
+            selected_executor: workerResult.selected_executor,
+            progress_made: workerResult.progress_made,
+            progress_summary: workerResult.summary,
+            next_likely_step: workerResult.next_likely_step,
+        } : {}),
+    }, null, 2));
+    if (failing.length > 0) {
+        process.exitCode = 2;
+    }
+}

package/dist/cli/nextStepCommand.js

@@ -71,7 +71,11 @@ async function runDeterministicForNextStep(params) {
             if (taskFiles.size > 0) {
                 const integrity = await checkFileIntegrity(params.root, bundle.repo_manifest, [...taskFiles]);
                 if (!integrity.is_clean) {
-                    console.log(`File integrity check: ${integrity.changed_files.length} changed, ${integrity.missing_files.length} missing — re-running intake.`);
+                    // Route this diagnostic OFF stdout: cmdNextStep emits the step
+                    // contract as the sole stdout payload via console.log(JSON.stringify),
+                    // so a console.log here would corrupt the JSON-on-stdout contract.
+                    process.stderr.write(`[audit-code] nextStep: integrity check — ${integrity.changed_files.length} changed, ` +
+                        `${integrity.missing_files.length} missing, ${integrity.io_errors.length} io-error(s); re-running intake.\n`);
                     await advanceAudit(bundle, { root: params.root, preferredExecutor: "intake_executor", opentoken: params.opentoken });
                     continue;
                 }

package/dist/cli/runToCompletion.d.ts

@@ -1 +1,10 @@
+import type { AuditTask } from "../types.js";
+import { type RunPaths } from "../io/runArtifacts.js";
+export interface WorkerSlot {
+    runId: string;
+    paths: RunPaths;
+    auditResultsPath: string;
+    pendingTasksPath: string;
+    group: AuditTask[];
+}
 export declare function cmdRunToCompletion(argv: string[]): Promise<void>;