audit-tools 0.42.0 → 0.43.0

package/dist/remediate/steps/dispatch/hostHandoff.js

@@ -1,13 +1,11 @@
 import { spawnSync } from "node:child_process";
 import { mkdir } from "node:fs/promises";
 import { isAbsolute, join, resolve } from "node:path";
-import { headCommit, FindingSchema, SUBMISSION_ISSUE_CODES, assertSubmissionRunId, hashContent, isGitRepo, normalizeRepoPath, readSubmissionDocument, repoRelativePath, resolveContainedPath, spawnSyncHidden, stableStringify, submissionPathFor, writeJsonFile, } from "audit-tools/shared";
+import { headCommit, FindingSchema, SUBMISSION_ISSUE_CODES, SUBMISSION_LEDGER_EVENT_CONTRACT_VERSION, appendSubmissionEvent, assertSubmissionRunId, hashContent, isGitRepo, normalizeRepoPath, readSubmissionDocument, readSubmissionLedger, repoRelativePath, resolveContainedPath, spawnSyncHidden, stableStringify, submissionPathFor, writeJsonFile, } from "audit-tools/shared";
 import { RemediationHostHandoffRecordSchema, RemediationPlanSchema, isClarificationCategory, } from "../../state/types.js";
-import { ITEM_STATUSES, isVerifiedCompleteStatus } from "../../state/itemStatus.js";
+import { ITEM_STATUSES, isTerminalStatus, isVerifiedCompleteStatus, } from "../../state/itemStatus.js";
+import { REMEDIATION_HOST_DECISION_CONTRACT_VERSION as DECISION_CONTRACT_VERSION, REMEDIATION_HOST_RESULT_CONTRACT_VERSION as RESULT_CONTRACT_VERSION, REMEDIATION_HOST_WORKLOAD_CONTRACT_VERSION as WORKLOAD_CONTRACT_VERSION, } from "../types.js";
 const STATE_CONTRACT_VERSION = "remediate-code-state/v1alpha1";
-const WORKLOAD_CONTRACT_VERSION = "remediation-host-workload/v1alpha1";
-const RESULT_CONTRACT_VERSION = "remediation-host-result/v1alpha1";
-const DECISION_CONTRACT_VERSION = "remediation-host-decision/v1alpha1";
 const HANDOFF_RECORD_CONTRACT_VERSION = "remediation-host-handoff-record/v1alpha1";
 /**
  * Remediation's issue vocabulary: the SHARED submission codes plus this draw's
@@ -31,6 +29,47 @@ export const REMEDIATION_ISSUE_CODES = [
     "changed_files_mismatch",
     "run_start_dirty_overlap",
     "required_test_failed",
+    /**
+     * A required test exceeded its deadline. DISTINCT from `required_test_failed`
+     * by code alone: a hung suite and a genuine red are different facts about the
+     * work, and telling them apart must not require parsing a joined message.
+     */
+    "required_test_timed_out",
+    /**
+     * A required test produced more output than the capture buffer holds, so the
+     * runner killed it. NOT a verdict on the tests: the child was terminated by
+     * the capture cap, and whether the suite would have passed is unknown. It has
+     * its own code because it was previously indistinguishable from a hang — node
+     * kills an over-buffer child with a signal, which the old discriminator read
+     * as a deadline miss.
+     */
+    "required_test_output_overflow",
+    /**
+     * A plan block declares a dependency id that exists in NO block of the plan.
+     * The block is unschedulable — never level 0 — and the producer bug is named
+     * rather than absorbed.
+     */
+    "dependency_missing",
+    /**
+     * A block arrived outside the normalized write-scope / declared-command shape
+     * this boundary consumes (artifact:normalized-block-write-scope). Refused, not
+     * silently normalized: a silently sorted, deduped or re-rooted write scope
+     * hides the producer bug and widens what a host may touch.
+     */
+    "block_contract_invalid",
+    /**
+     * A recovery-mode acceptance could not be marked on the submission ledger, so
+     * it was refused. An acceptance that used the relaxation MUST stay
+     * distinguishable from a clean one; an unrecordable mark is a refusal, never
+     * a silent acceptance.
+     */
+    "recovery_unrecorded",
+    /**
+     * The repository HEAD moved between the recovery verb's unlocked test phase
+     * and its locked write phase, so the pre-computed test verdicts describe a
+     * tree that is no longer current. The whole recovery aborts.
+     */
+    "tree_moved_between_phases",
 ];
 const CURRENT_STATE_KEYS = new Set([
     "applied_edit_surface",
@@ -140,6 +179,207 @@ function normalizeDeclaredPath(root, candidate, label) {
     }
     return repoRelativePath(root, candidate, label);
 }
+/**
+ * Does a command leave the declared single-invocation shape?
+ *
+ * A `targeted_command` is executed VERBATIM through `shell: true` in the
+ * repository root, so anything that chains, redirects, substitutes or subshells
+ * turns one declared test into arbitrary execution. A flat regex cannot decide
+ * this: `node -e "process.exit(0)"` is an ordinary test invocation whose parens
+ * are inside quotes, and refusing it would refuse the normal case.
+ *
+ * So the scan is QUOTE-AWARE — a tiny, fully-owned grammar, not a shell parser.
+ * It is deliberately narrower than EITHER shell's grammar, because `shell: true`
+ * is TWO grammars and no per-grammar state machine is sound for both: `/bin/sh
+ * -c` on posix, `cmd.exe /d /s /c` on win32. Where they disagree, tracking state
+ * under one of them MIS-CLASSIFIES the other:
+ *
+ *   - `'` quotes on sh and is an ORDINARY CHARACTER on cmd.exe, so crediting it
+ *     reads `echo '& evil.exe'` as fully quoted while cmd.exe reads `&` as a
+ *     command separator and starts a second process.
+ *   - `\` escapes a quote on sh, so `echo \" & evil \"` de-syncs any
+ *     double-quote tracking: the scan believes `&` is quoted, sh sees an escaped
+ *     literal quote and a live separator.
+ *   - `%VAR%` is expanded by cmd.exe BEFORE the line is split into commands, and
+ *     expands inside double quotes too, so a `%…%` reference can introduce
+ *     separators that were never in the scanned string.
+ *   - `^` is cmd.exe's escape character and de-syncs quote state the same way.
+ *   - CR and LF are not inert inside cmd.exe's double quotes: LF truncates the
+ *     command there and CR is deleted outright, so what runs is not what was
+ *     scanned. Every control character is refused for that reason.
+ *
+ * What remains is the one construct both shells agree on: a double quote makes
+ * the enclosed metacharacters literal. So `' \ ^ % $` and backtick — plus every
+ * control character — are refused in EVERY position, quoted or not; the
+ * chaining/redirection/grouping set is refused outside double quotes; and an
+ * unterminated double quote is itself a refusal, because the rest of the string
+ * cannot be classified.
+ *
+ * The fail direction is REFUSAL. Over-refusing a legitimate command costs a
+ * producer-side split, and the producer-side obligation still owns the declared
+ * shape (artifact:normalized-block-write-scope); under-admitting one hands a
+ * shell an extra process.
+ */
+function leavesDeclaredCommandShape(command) {
+    let inDoubleQuotes = false;
+    for (const character of command) {
+        const code = character.codePointAt(0) ?? 0;
+        if (code < 0x20 || code === 0x7f)
+            return true;
+        if ("'\\^%$`".includes(character))
+            return true;
+        if (character === '"') {
+            inDoubleQuotes = !inDoubleQuotes;
+            continue;
+        }
+        if (inDoubleQuotes)
+            continue;
+        if ("&|;<>()".includes(character))
+            return true;
+    }
+    return inDoubleQuotes;
+}
+/**
+ * A block that arrived outside the shape this boundary CONSUMES.
+ *
+ * The producer half of the write-scope contract is owned upstream
+ * (artifact:normalized-block-write-scope). This is the consumer half, and it
+ * exists because absorbing a malformed block is worse than refusing it: an
+ * absolute or escaping `touched_files` entry silently widens what a host may
+ * write, and a shell-chained `targeted_command` is executed verbatim. Both are
+ * producer bugs, and a boundary that normalizes them away means neither ever
+ * surfaces. The refusal is CLASSIFIED (`block_contract_invalid`) and names the
+ * block, so the bug is attributable to the module that wrote it.
+ */
+class BlockContractError extends Error {
+    blockId;
+    detail;
+    constructor(blockId, detail) {
+        super(`block '${blockId}' is outside the normalized write-scope contract: ${detail}`);
+        this.blockId = blockId;
+        this.detail = detail;
+        this.name = "BlockContractError";
+    }
+}
+/**
+ * Refuse a block whose declared write scope or commands leave the consumed
+ * shape. Throws {@link BlockContractError}; callers turn it into a classified
+ * issue. Runs BEFORE anything is built from the block, so a refused block never
+ * becomes a work item and its commands never run.
+ *
+ * COVERS THE HANDOFF BOUNDARY ONLY — state the uncovered half rather than let
+ * the covered half read as a close. `reverifyBlockedItemAgainstTree` in
+ * `src/remediate/phases/triage.ts` spawns the SAME `block.targeted_commands`
+ * through `shell: true` with no gate in front of it, so a command this boundary
+ * would refuse still reaches a shell on the triage path. Routing that spawn
+ * through this gate is tracked as backlog work, not covered here.
+ */
+function assertBlockContract(root, block) {
+    for (const raw of block.touched_files) {
+        if (typeof raw !== "string" || raw.trim().length === 0) {
+            throw new BlockContractError(block.block_id, "touched_files carries an empty entry");
+        }
+        if (isAbsolute(raw)) {
+            throw new BlockContractError(block.block_id, `touched_files entry ${JSON.stringify(raw)} is absolute, not repository-relative`);
+        }
+        let normalized;
+        try {
+            normalized = repoRelativePath(root, raw, `${block.block_id}.touched_files[]`);
+        }
+        catch {
+            throw new BlockContractError(block.block_id, `touched_files entry ${JSON.stringify(raw)} does not resolve beneath the repository root`);
+        }
+        if (normalized !== raw) {
+            throw new BlockContractError(block.block_id, `touched_files entry ${JSON.stringify(raw)} is not in normalized repo-relative form ` +
+                `(${JSON.stringify(normalized)})`);
+        }
+    }
+    for (const command of block.targeted_commands ?? []) {
+        if (typeof command !== "string" || command.trim().length === 0) {
+            throw new BlockContractError(block.block_id, "targeted_commands carries an empty command");
+        }
+        if (leavesDeclaredCommandShape(command)) {
+            throw new BlockContractError(block.block_id, `targeted_command ${JSON.stringify(command)} leaves the declared shape — it chains, ` +
+                "redirects or substitutes, and this boundary executes commands verbatim through a shell");
+        }
+    }
+}
+/**
+ * Every block of the plan that cannot be scheduled, with the reason, as
+ * classified ingest issues. Two producer bugs live here: a dependency id that
+ * resolves to no block (unschedulable forever — see `hostDependencyLevels`),
+ * and a block outside the consumed write-scope/command shape.
+ *
+ * The scanned set is BOUND ∪ UNSETTLED, and it is that union because those are
+ * exactly the blocks something else re-derives:
+ *   - UNSETTLED (any item not terminal) — the blocks still to be scheduled. A
+ *     settled block's historical shape is not this ingest's business, and
+ *     reporting it would turn every later ingest into a repeat of the same noise.
+ *   - BOUND (`block_id` in `host_handoff.work_item_ids`, WHATEVER its items'
+ *     statuses) — because `parseWorkItem` re-derives every bound item through
+ *     `buildWorkItem` regardless of status. A status filter alone therefore
+ *     scanned a DIFFERENT set than the one that can throw: a bound block whose
+ *     items had all reached terminal still failed the workload parse when its
+ *     contract was malformed, and surfaced as a bare `workload_invalid` naming no
+ *     block. Scanning the union is what makes "the block that broke the parse is
+ *     always named" true rather than usually true.
+ */
+function planBlockIssues(root, state) {
+    const blockIds = new Set(state.plan.blocks.map((block) => block.block_id));
+    const boundIds = new Set(state.host_handoff?.work_item_ids ?? []);
+    const issues = [];
+    for (const block of state.plan.blocks) {
+        const unsettled = block.items.some((findingId) => {
+            const status = state.items[findingId]?.status;
+            return status !== undefined && !isTerminalStatus(status);
+        });
+        if (!unsettled && !boundIds.has(block.block_id))
+            continue;
+        const missing = (block.dependencies ?? []).filter((dependencyId) => !blockIds.has(dependencyId));
+        if (missing.length > 0) {
+            issues.push({
+                code: "dependency_missing",
+                work_item_id: block.block_id,
+                message: `block '${block.block_id}' declares ${missing.length === 1 ? "a dependency" : "dependencies"} ` +
+                    `${missing.map((id) => `'${id}'`).join(", ")} present in no block of the plan, so it can ` +
+                    "never be dependency-verified and is never scheduled",
+            });
+            continue;
+        }
+        try {
+            assertBlockContract(root, block);
+        }
+        catch (error) {
+            if (!(error instanceof BlockContractError))
+                throw error;
+            issues.push({
+                code: "block_contract_invalid",
+                work_item_id: block.block_id,
+                message: error.message,
+            });
+        }
+    }
+    return issues;
+}
+/**
+ * The classified "cannot prepare" message for a producer defect that reached the
+ * build path as a throw. Aggregates the whole plan scan so an operator sees
+ * EVERY malformed block, not just the first one the builder tripped on.
+ *
+ * The THROWER is attributed by name, not by whether the scan happened to find
+ * anything. Falling back only on an EMPTY scan silently dropped the raised error
+ * whenever the scan named some OTHER block — a plan with one ghost dependency
+ * elsewhere was enough to make the message describe a block that did not throw
+ * and omit the one that did.
+ */
+function cannotPrepareMessage(root, state, raised) {
+    const scanned = planBlockIssues(root, state);
+    const messages = scanned.map((issue) => issue.message);
+    if (!scanned.some((issue) => issue.work_item_id === raised.blockId)) {
+        messages.push(raised.message);
+    }
+    return `Cannot prepare a remediation host workload: ${messages.join("; ")}`;
+}
 function resolveBoundaryPaths(params) {
     assertSubmissionRunId(params.runId, "remediation host run id");
     const root = resolve(params.root);
@@ -224,7 +464,14 @@ export function hostDependencyLevels(state) {
     const permanentlyIneligible = (block) => {
         for (const dependencyId of block.dependencies ?? []) {
             const dependency = blockById.get(dependencyId);
-            if (dependency && !isVerifiedNow(dependency) && !isPending(dependency)) {
+            // An id that resolves to NO block is not a harmless declaration — it is a
+            // prerequisite that can never be verified, so the block can never become
+            // eligible. Guarding on `dependency &&` skipped exactly this case, which
+            // is the second half of the same hole as the readiness predicate below:
+            // closing only one leaves the block reaching the host anyway.
+            if (dependency === undefined)
+                return true;
+            if (!isVerifiedNow(dependency) && !isPending(dependency)) {
                 return true;
             }
         }
@@ -237,7 +484,13 @@ export function hostDependencyLevels(state) {
         const ready = remaining.filter((block) => phaseBarrierClear(phaseOf(block)) &&
             (block.dependencies ?? []).every((dependencyId) => {
                 const dependency = blockById.get(dependencyId);
-                if (!dependency || isVerifiedNow(dependency))
+                // DEPENDENCY READINESS REQUIRES EXISTENCE. `!dependency` used to read
+                // as "satisfied", so a plan naming a block that does not exist had its
+                // dependent placed at level 0 and dispatched with the prerequisite
+                // never verified — silently, because no other check looks at it.
+                if (dependency === undefined)
+                    return false;
+                if (isVerifiedNow(dependency))
                     return true;
                 return dependency.items.every((findingId) => isVerifiedCompleteStatus(items[findingId]?.status) ||
                     (items[findingId]?.status === "pending" &&
@@ -292,6 +545,10 @@ function buildFindingAssignments(state, block) {
     });
 }
 function buildWorkItem(paths, block, baselineCommit, state) {
+    // The consumed-shape gate runs FIRST: a block outside the write-scope /
+    // command contract must never become a work item, so nothing downstream can
+    // dispatch it or execute its commands.
+    assertBlockContract(paths.root, block);
     const allowedFiles = [...new Set(block.touched_files)].map((path) => normalizeDeclaredPath(paths.root, path, `${block.block_id}.touched_files[]`)).sort(compareCodeUnits);
     const resultPath = resultPathFor(paths, block.block_id);
     const requiredTests = [...(block.targeted_commands ?? [])];
@@ -557,6 +814,31 @@ function gitCommitIsAncestor(root, ancestor, descendant) {
     const result = spawnSyncHidden("git", ["merge-base", "--is-ancestor", ancestor, descendant], { cwd: root, encoding: "utf8", shell: false });
     return !result.error && result.status === 0;
 }
+/**
+ * Is this commit ORPHANED — unreachable from anything the repository still
+ * keeps?
+ *
+ * "Not an ancestor of HEAD" is NOT orphanhood. A baseline sitting on an
+ * unmerged `feature` branch while the work landed on trunk fails the ancestry
+ * test exactly like a rewritten-away commit does, and treating that as orphaned
+ * would hand the relaxation to the ordinary cross-branch case — precisely the
+ * stale-worker situation the ancestry check exists to catch.
+ *
+ * So orphanhood is the CONJUNCTION of two probes: `git for-each-ref --contains`
+ * lists every branch/tag/remote ref whose history contains the commit (empty
+ * output = no live ref keeps it), and the HEAD ancestry check rides alongside
+ * it because a detached HEAD is not a ref `for-each-ref` enumerates and would
+ * otherwise scan clean. A failed scan is not evidence of orphanhood — it fails
+ * closed, so a git that cannot answer never unlocks the relaxation.
+ */
+function gitCommitIsOrphaned(root, commit) {
+    if (gitCommitIsAncestor(root, commit, "HEAD"))
+        return false;
+    const result = spawnSyncHidden("git", ["for-each-ref", "--contains", commit, "--format=%(refname)"], { cwd: root, encoding: "utf8", shell: false });
+    if (result.error || result.status !== 0)
+        return false;
+    return (result.stdout ?? "").trim().length === 0;
+}
 function gitChangedFilesOfCommit(root, commit) {
     const result = spawnSyncHidden("git", [
         "diff-tree",
@@ -571,26 +853,246 @@ function gitChangedFilesOfCommit(root, commit) {
         return null;
     return [...new Set((result.stdout ?? "").split("\0").filter(Boolean))].sort(compareCodeUnits);
 }
-function rerunRequiredTests(root, commands) {
+/**
+ * Per-command deadline. A required test is host-authored and may legitimately be
+ * a full suite, so the bound is generous; what changed is that hitting it is now
+ * a NAMED outcome instead of an unlabelled failure string.
+ */
+const REQUIRED_TEST_TIMEOUT_MS = 10 * 60 * 1_000;
+/**
+ * Captured output is bounded and TAIL-biased: a failing suite's verdict is at
+ * the end, and an unbounded capture would put a whole test log into state and
+ * into every rendered issue.
+ */
+const CAPTURED_OUTPUT_LIMIT = 4_000;
+/**
+ * The spawn's raw capture buffer. Exceeding it does not truncate — node KILLS
+ * the child — so the cap is a named constant the `output_overflow` message can
+ * quote, rather than a literal buried in the spawn options.
+ */
+const REQUIRED_TEST_MAX_BUFFER_BYTES = 8 * 1_024 * 1_024;
+function tail(value) {
+    const text = value ?? "";
+    return text.length <= CAPTURED_OUTPUT_LIMIT
+        ? text
+        : `…${text.slice(text.length - CAPTURED_OUTPUT_LIMIT)}`;
+}
+/**
+ * Render one classified failure for a host-facing issue message.
+ *
+ * `output_overflow` says the verdict is UNKNOWN, not that the tests were fine: a
+ * child killed at the buffer cap may have been heading for exit 0 or exit 3, and
+ * the runner cannot tell which. Either way the item is refused — the honest
+ * report is "we could not find out", and it fails closed.
+ *
+ * A signal-killed child renders the SIGNAL, not `exit null`: `exit_code` is null
+ * for every non-exit outcome, so printing it there described nothing.
+ */
+function describeRequiredTestFailure(failure) {
+    const head = failure.outcome === "timed_out"
+        ? `${failure.command} (timed out)`
+        : failure.outcome === "output_overflow"
+            ? `${failure.command} (killed after exceeding the ${String(REQUIRED_TEST_MAX_BUFFER_BYTES)}-byte ` +
+                "output buffer — the run ended at the capture cap, so whether the tests pass is UNKNOWN)"
+            : failure.outcome === "spawn_error"
+                ? `${failure.command} (could not be started)`
+                : failure.exit_code === null
+                    ? `${failure.command} (terminated by ${failure.signal ?? "an unreported signal"})`
+                    : `${failure.command} (exit ${String(failure.exit_code)})`;
+    const captured = [failure.stdout, failure.stderr]
+        .filter((stream) => stream.trim().length > 0)
+        .join("\n");
+    return captured.length > 0 ? `${head}: ${captured}` : head;
+}
+/**
+ * Length-prefixed so the root/command boundary is unambiguous for any path, and
+ * printable so the source stays text (a raw separator byte would make the file
+ * binary to git and invisible to grep). The root is part of the key because a
+ * verdict is a fact about one command in one working tree, and nothing
+ * guarantees a single process only ever ingests for one root.
+ */
+function requiredTestVerdictKey(root, command) {
+    return `${String(root.length)}:${root}:${command}`;
+}
+/**
+ * The ONE place a required-test command is spawned.
+ *
+ * `timeoutMs` is a parameter so the deadline is exercisable: a hang is a
+ * first-class outcome of this function, and an outcome that can only be reached
+ * by waiting ten real minutes is an outcome nothing ever tests.
+ */
+export function runRequiredTest(root, command, timeoutMs = REQUIRED_TEST_TIMEOUT_MS) {
+    const result = spawnSync(command, {
+        cwd: root,
+        shell: true,
+        // Captured, not discarded: without it a red ingest reports that something
+        // failed and nothing about why.
+        encoding: "utf8",
+        maxBuffer: REQUIRED_TEST_MAX_BUFFER_BYTES,
+        timeout: timeoutMs,
+        windowsHide: true,
+    });
+    const stdout = tail(result.stdout);
+    const stderr = tail(result.stderr);
+    // The ERROR CODE discriminates, never `signal`. node kills an over-deadline
+    // child AND an over-`maxBuffer` child, and an external `kill` sets `signal`
+    // too — so `signal !== null` was true for three unrelated facts and reported
+    // all of them as a hang, including a command killed purely for printing more
+    // than the buffer holds.
+    //
+    // ASSUMPTION, stated: a deadline miss reports `ETIMEDOUT`. Verified on win32;
+    // it is node's documented contract, not a platform quirk this code confirmed
+    // everywhere. On a platform that killed a child at the deadline WITHOUT that
+    // code, the case degrades to `spawn_error` — a less specific refusal, still a
+    // refusal, so the fail direction holds and only the label is lost.
+    const code = result.error?.code;
+    if (code === "ETIMEDOUT") {
+        return { command, outcome: "timed_out", exit_code: null, stdout, stderr };
+    }
+    if (code === "ENOBUFS") {
+        return {
+            command,
+            outcome: "output_overflow",
+            exit_code: null,
+            stdout,
+            stderr,
+        };
+    }
+    if (result.error) {
+        return {
+            command,
+            outcome: "spawn_error",
+            exit_code: null,
+            stdout,
+            stderr: stderr.length > 0 ? stderr : result.error.message,
+        };
+    }
+    // Killed by something outside this runner (an operator `kill`, an OOM reaper).
+    // Reported as FAILED with the signal named: the command did not complete, and
+    // calling it a deadline miss would attribute it to a bound this runner set.
+    //
+    // POSIX-ONLY IN PRACTICE, and UNTESTED for that reason: Windows has no signal
+    // delivery to report here — a killed child surfaces as an ordinary non-zero
+    // `status` with `signal` null — so this branch is unreachable on the platform
+    // this repo runs its suites on, and no test exercises it. It is kept because
+    // the runner is OS-agnostic by contract, not because it has been observed.
+    if (result.signal !== null) {
+        return {
+            command,
+            outcome: "failed",
+            exit_code: null,
+            stdout,
+            stderr,
+            signal: result.signal,
+        };
+    }
+    if (result.status !== 0) {
+        return {
+            command,
+            outcome: "failed",
+            exit_code: result.status,
+            stdout,
+            stderr,
+        };
+    }
+    return null;
+}
+function rerunRequiredTests(root, commands, 
+/** `null` on the normal lane — see {@link RemediationRequiredTestVerdicts}. */
+verdicts) {
     const failures = [];
     for (const command of commands) {
-        const result = spawnSync(command, {
-            cwd: root,
-            shell: true,
-            stdio: "ignore",
-            timeout: 10 * 60 * 1_000,
-            windowsHide: true,
-        });
-        if (result.error || result.status !== 0) {
-            failures.push(`${command} (${result.error?.message ?? `exit ${String(result.status)}`})`);
+        if (verdicts) {
+            const verdict = verdicts.get(requiredTestVerdictKey(root, command));
+            if (verdict === undefined) {
+                failures.push({
+                    command,
+                    outcome: "spawn_error",
+                    exit_code: null,
+                    stdout: "",
+                    stderr: "no pre-computed verdict — refusing to spawn a test while the state lock is held",
+                });
+            }
+            else if (verdict !== null) {
+                failures.push(verdict);
+            }
+            continue;
         }
+        const failure = runRequiredTest(root, command);
+        if (failure !== null)
+            failures.push(failure);
     }
     return failures;
 }
+/**
+ * The classified issue for a set of required-test failures. An ENVIRONMENT fact
+ * anywhere in the set wins over a red sibling, timeout first: a hung or
+ * buffer-killed suite is the fact that explains the ingest, and burying it under
+ * a sibling's exit code is exactly the conflation the code split exists to end.
+ * Only a set where every failure is a genuine non-zero exit reads as
+ * `required_test_failed`.
+ */
+function requiredTestIssue(workItem, failures) {
+    return {
+        code: failures.some((failure) => failure.outcome === "timed_out")
+            ? "required_test_timed_out"
+            : failures.some((failure) => failure.outcome === "output_overflow")
+                ? "required_test_output_overflow"
+                : "required_test_failed",
+        work_item_id: workItem.id,
+        result_path: workItem.result_path,
+        message: `mechanical required-test rerun failed: ${failures
+            .map(describeRequiredTestFailure)
+            .join("; ")}`,
+    };
+}
+/**
+ * Run every required-test command a recovery ingest could need, ONCE each, and
+ * return the finished verdict table. Call this OUTSIDE the state lock — that is
+ * the entire point (see {@link RemediationRequiredTestVerdicts}).
+ *
+ * Candidates are the work items with at least one still-pending finding whose
+ * result file is present and parses as JSON; an item with no result file is
+ * refused before its tests would ever run, so spawning for it is pure cost. The
+ * filter is deliberately generous otherwise — over-inclusion costs one spawn,
+ * while under-inclusion becomes a fail-closed refusal of a good result.
+ */
+export async function precomputeRecoveryTestVerdicts(params) {
+    const state = parseCurrentState(params.state);
+    if (!state)
+        return "unsupported_retired_state";
+    const paths = resolveBoundaryPaths(params);
+    const verdicts = new Map();
+    const workloadRead = await readSubmissionDocument(paths.workloadPath);
+    if (workloadRead.kind !== "value")
+        return verdicts;
+    const workload = parseWorkload(workloadRead.value, paths, params.runId, state);
+    if (!workload)
+        return verdicts;
+    const commands = [];
+    for (const workItem of workload.work_items) {
+        const hasPending = workItem.finding_ids.some((findingId) => state.items[findingId]?.status === "pending");
+        if (!hasPending)
+            continue;
+        const absoluteResultPath = resolveContainedPath(paths.root, workItem.result_path, `result path for ${workItem.id}`);
+        const resultRead = await readSubmissionDocument(absoluteResultPath);
+        if (resultRead.kind !== "value")
+            continue;
+        for (const command of workItem.required_tests) {
+            if (!commands.includes(command))
+                commands.push(command);
+        }
+    }
+    for (const command of commands) {
+        verdicts.set(requiredTestVerdictKey(paths.root, command), runRequiredTest(paths.root, command));
+    }
+    return verdicts;
+}
 function corroborateHostResult(params) {
-    const { root, state, workItem, result } = params;
+    const { root, state, workItem, result, verdicts } = params;
     const baseline = workItem.baseline_commit;
     const landed = result.commit_evidence.after;
+    let usedRecovery = false;
     if (!gitCommitExists(root, baseline) || !gitCommitExists(root, landed)) {
         return {
             ok: false,
@@ -599,11 +1101,32 @@ function corroborateHostResult(params) {
         };
     }
     if (!gitCommitIsAncestor(root, baseline, landed)) {
-        return {
-            ok: false,
-            code: "baseline_not_ancestor",
-            message: "the trusted workload baseline is not an ancestor of the claimed landed commit",
-        };
+        if (!params.recovery) {
+            return {
+                ok: false,
+                code: "baseline_not_ancestor",
+                message: "the trusted workload baseline is not an ancestor of the claimed landed commit",
+            };
+        }
+        // The relaxation is precondition-bound: it applies ONLY when the trusted
+        // baseline is genuinely ORPHANED — contained by no ref AND unreachable from
+        // HEAD (see gitCommitIsOrphaned). That is the one state in which no landed
+        // commit could ever descend from it, so the item is unacceptable under
+        // every preparable binding. A baseline the repository still keeps — on an
+        // unmerged branch, a tag, a remote ref, or HEAD itself — is a HEALTHY
+        // binding, and a landed commit that does not descend from it is exactly the
+        // stale-worker case the ancestry check exists to catch; recovery refuses it
+        // identically to the normal lane.
+        if (!gitCommitIsOrphaned(root, baseline)) {
+            return {
+                ok: false,
+                code: "baseline_not_ancestor",
+                message: "the trusted workload baseline is not an ancestor of the claimed landed commit, " +
+                    "and the baseline is NOT orphaned (a ref still contains it, or it is reachable " +
+                    "from HEAD), so the stale-worker protection stands and recovery cannot waive it",
+            };
+        }
+        usedRecovery = true;
     }
     if (!gitCommitIsAncestor(root, landed, "HEAD")) {
         return {
@@ -636,15 +1159,12 @@ function corroborateHostResult(params) {
             message: `landed files overlap pre-existing run-start dirt: ${dirtyOverlap.join(", ")}`,
         };
     }
-    const failedTests = rerunRequiredTests(root, workItem.required_tests);
+    const failedTests = rerunRequiredTests(root, workItem.required_tests, verdicts);
     if (failedTests.length > 0) {
-        return {
-            ok: false,
-            code: "required_test_failed",
-            message: `mechanical required-test rerun failed: ${failedTests.join("; ")}`,
-        };
+        const issue = requiredTestIssue(workItem, failedTests);
+        return { ok: false, code: issue.code, message: issue.message };
     }
-    return { ok: true, changedFiles: actualFiles };
+    return { ok: true, changedFiles: actualFiles, usedRecovery };
 }
 export async function prepareRemediationHostHandoff(params) {
     const state = parseCurrentState(params.state);
@@ -665,17 +1185,39 @@ export async function prepareRemediationHostHandoff(params) {
         }
     }
     const baselineCommit = existingRecord?.baseline_commit ?? params.baselineCommit;
-    const workload = buildCanonicalWorkload({
-        paths,
-        state,
-        runId: params.runId,
-        baselineCommit,
-        ...(existingRecord
-            ? { workItemIds: existingRecord.work_item_ids }
-            : {}),
-    });
+    let workload;
+    try {
+        workload = buildCanonicalWorkload({
+            paths,
+            state,
+            runId: params.runId,
+            baselineCommit,
+            ...(existingRecord
+                ? { workItemIds: existingRecord.work_item_ids }
+                : {}),
+        });
+    }
+    catch (error) {
+        // A malformed block ON the frontier reaches this as a raw BlockContractError
+        // — an uncaught throw whose stack says nothing about which producer wrote the
+        // bad block, and which every retry reproduces. Re-raised in the SAME
+        // classified aggregate form the empty-workload branch below uses, so both
+        // producer-defect exits read alike.
+        if (!(error instanceof BlockContractError))
+            throw error;
+        throw new Error(cannotPrepareMessage(paths.root, state, error));
+    }
     if (workload.work_items.length === 0) {
-        throw new Error("Cannot prepare an empty remediation host workload");
+        // Name the producer defect when it is the cause. An empty level 0 that is
+        // really "every candidate block declares a prerequisite that does not
+        // exist" used to surface as a bare "empty workload", sending the operator
+        // to look at scheduling rather than at the plan.
+        const blocked = planBlockIssues(paths.root, state);
+        throw new Error(blocked.length === 0
+            ? "Cannot prepare an empty remediation host workload"
+            : `Cannot prepare a remediation host workload: ${blocked
+                .map((issue) => issue.message)
+                .join("; ")}`);
     }
     const workloadDigest = hostWorkloadSha256(workload);
     if (existingRecord &&
@@ -697,6 +1239,48 @@ export async function prepareRemediationHostHandoff(params) {
         handoff_record: handoffRecord,
     };
 }
+/**
+ * Consume the host's landed results for the trusted workload.
+ *
+ * ## The `recovery` option, and what it actually buys
+ *
+ * A trusted binding can be stranded: a post-prepare `git commit --amend` (or
+ * any history rewrite) re-mints the baseline the workload was bound to, leaving
+ * it ORPHANED — contained by no ref and unreachable from HEAD. Every commit the
+ * host then lands sits on the re-minted line, so `baseline → landed` ancestry is
+ * false for all of them, and re-preparing does not help: a fresh binding must be
+ * minted at HEAD, and HEAD is a DESCENDANT of the landed work. The items are
+ * unacceptable under every preparable binding, with real, reachable,
+ * correctly-scoped commits on disk.
+ *
+ * `recovery` waives ONE check — baseline→landed ancestry — and only when
+ * the baseline is genuinely orphaned by BOTH probes in `gitCommitIsOrphaned`: no
+ * branch/tag/remote ref contains it, and it is not reachable from HEAD. A
+ * baseline the repository still keeps (an unmerged feature branch, a tag, a
+ * remote ref) also fails the ancestry test when work lands elsewhere, and that
+ * is the ordinary stale-worker case — recovery refuses it. Every other
+ * corroboration check runs unchanged (the landed commit exists and is reachable
+ * from HEAD; its mechanically derived changed files exactly equal
+ * `changed_files` and lie within the prompt-bound `allowed_files`; no overlap
+ * with run-start dirt; the required tests rerun green), `parseResult` stays
+ * fully strict, and dependency/phase eligibility is enforced exactly as on the
+ * normal lane.
+ *
+ * RESIDUAL RISK, stated plainly: under an orphaned baseline the evidence bar
+ * drops to "the claimed commit is reachable from the current green HEAD and
+ * matches this item's scope exactly". That CANNOT prove the work was built on
+ * the trusted baseline — a commit landed from a stale or unrelated starting
+ * tree satisfies it as long as its own file set stays in scope. Ancestry is the
+ * check that would have caught that, and it is the one being waived. Which is
+ * precisely why the relaxation costs an explicit operator verb, is gated on the
+ * orphan precondition, and is marked `accepted_via_recovery` on the submission
+ * ledger before the item lands — and why the normal lane keeps the full check.
+ *
+ * In recovery mode this function performs NO required-test spawn: the verdicts
+ * arrive pre-computed on the `recovery` option, and a command missing from that
+ * table is treated as failed. Its caller runs the tests first, unlocked — see
+ * `recoverIngestHostResults`.
+ */
 export async function ingestRemediationHostResults(params) {
     const state = parseCurrentState(params.state);
     if (!state)
@@ -725,33 +1309,48 @@ export async function ingestRemediationHostResults(params) {
             state: nextState,
         };
     }
+    // Producer-side plan defects are reported BEFORE the workload is parsed: a
+    // block with an unresolvable dependency or an unnormalized write scope makes
+    // the whole workload fail to re-derive, and `workload_invalid` alone would
+    // name the symptom while hiding which block caused it.
+    //
+    // REPORTED, never fatal. A defect in a NON-frontier block says nothing about a
+    // frontier item's landed result, and refusing the whole ingest over one made
+    // the run unadvanceable: every ingest returned zero acceptances, `next-step`
+    // read `state_changed: false` and re-emitted the same items against the same
+    // malformed plan, forever. The frontier's OWN defect is enforced elsewhere and
+    // does not rely on this: `parseWorkItem` re-derives each bound work item
+    // through `buildWorkItem`, whose `assertBlockContract` throws, so a malformed
+    // bound block fails the workload parse and its commands never run.
+    issues.push(...planBlockIssues(paths.root, state));
     if (isGitRepo(paths.root) && !state.host_handoff) {
+        issues.push({
+            code: "trusted_binding_missing",
+            message: "a git-backed remediation workload requires the tool-owned host_handoff state binding",
+        });
         return {
             accepted_count: 0,
             completed_work_item_ids: [],
             pending_work_item_ids: [],
-            issues: [
-                {
-                    code: "trusted_binding_missing",
-                    message: "a git-backed remediation workload requires the tool-owned host_handoff state binding",
-                },
-            ],
+            issues,
             state_changed: false,
             state: nextState,
         };
     }
     const workload = parseWorkload(workloadRead.value, paths, params.runId, state);
     if (!workload) {
+        // Accumulated, not replaced: when a block-contract defect is WHY the
+        // canonical re-derivation failed, the block-attributed issue is the only
+        // thing that names the cause.
+        issues.push({
+            code: "workload_invalid",
+            message: "the workload does not match its canonical state shape and persisted digest binding",
+        });
         return {
             accepted_count: 0,
             completed_work_item_ids: [],
             pending_work_item_ids: state.host_handoff?.work_item_ids ?? [],
-            issues: [
-                {
-                    code: "workload_invalid",
-                    message: "the workload does not match its canonical state shape and persisted digest binding",
-                },
-            ],
+            issues,
             state_changed: false,
             state: nextState,
         };
@@ -759,6 +1358,15 @@ export async function ingestRemediationHostResults(params) {
     const eligibleIds = new Set((hostDependencyLevels(state)[0] ?? []).map((block) => block.block_id));
     const resultIds = new Set();
     const completed = [];
+    // Recovery-only answer table; the normal lane gets null and spawns exactly as
+    // it always has. See RemediationRequiredTestVerdicts.
+    const requiredTestVerdicts = params.recovery?.requiredTestVerdicts ?? null;
+    // Lazily loaded on the first recovery-marked acceptance: the recovery marks
+    // the ledger ALREADY carries. A crash between the append and the state write
+    // leaves a mark whose item is still pending, and the natural response is to
+    // re-run the verb — which must converge, not accumulate a second record of
+    // the same acceptance.
+    let recordedRecoveryMarks = null;
     const landedFiles = new Set(nextState.applied_edit_surface ?? []);
     const requireRepositoryCorroboration = state.host_handoff !== undefined || isGitRepo(paths.root);
     for (const workItem of workload.work_items) {
@@ -820,14 +1428,9 @@ export async function ingestRemediationHostResults(params) {
             const result = parsed.result;
             const outcome = result.outcome;
             if (outcome.status === "resolved_no_change") {
-                const failedTests = rerunRequiredTests(paths.root, workItem.required_tests);
+                const failedTests = rerunRequiredTests(paths.root, workItem.required_tests, requiredTestVerdicts);
                 if (failedTests.length > 0) {
-                    issues.push({
-                        code: "required_test_failed",
-                        work_item_id: workItem.id,
-                        result_path: workItem.result_path,
-                        message: `mechanical required-test rerun failed: ${failedTests.join("; ")}`,
-                    });
+                    issues.push(requiredTestIssue(workItem, failedTests));
                     continue;
                 }
             }
@@ -874,6 +1477,8 @@ export async function ingestRemediationHostResults(params) {
                 state,
                 workItem,
                 result,
+                verdicts: requiredTestVerdicts,
+                recovery: params.recovery !== undefined,
             });
             if (!corroborated.ok) {
                 issues.push({
@@ -884,6 +1489,55 @@ export async function ingestRemediationHostResults(params) {
                 });
                 continue;
             }
+            if (corroborated.usedRecovery) {
+                // No acceptance without a record. The mark goes down BEFORE the item is
+                // marked resolved, and an append that throws refuses this item rather
+                // than landing an acceptance the ledger cannot account for — the run
+                // must never read as one that never drifted. The refusal is per item:
+                // an unwritable ledger is not a reason to discard the whole ingest.
+                try {
+                    recordedRecoveryMarks ??= (await readSubmissionLedger(paths.artifactsDir)).filter((event) => event.kind === "accepted_via_recovery");
+                    // The mark's identity is (run, item, LANDED COMMIT), not just
+                    // (run, item): an item re-opened and later re-accepted from a
+                    // DIFFERENT landing is a different relaxed acceptance and earns its
+                    // own record. Only a retry of the SAME landing is a duplicate. The
+                    // landed sha is matched inside the message because the shared event
+                    // contract carries no commit field, and a 40-hex sha this writer
+                    // itself emitted is an unambiguous token to match on.
+                    const landedCommit = result.commit_evidence.after;
+                    const alreadyMarked = recordedRecoveryMarks.some((event) => event.run_id === params.runId &&
+                        event.submission_id === workItem.id &&
+                        (event.message ?? "").includes(landedCommit));
+                    if (!alreadyMarked) {
+                        const event = {
+                            contract_version: SUBMISSION_LEDGER_EVENT_CONTRACT_VERSION,
+                            run_id: params.runId,
+                            submission_id: workItem.id,
+                            lane: workItem.id,
+                            kind: "accepted_via_recovery",
+                            // Derived from what was actually probed, never asserted: the
+                            // baseline was found in no ref and unreachable from HEAD.
+                            message: `accepted under recovery: the trusted baseline ${workItem.baseline_commit} is ` +
+                                "contained by no ref and unreachable from HEAD, so landed commit " +
+                                `${landedCommit} was corroborated against HEAD and this ` +
+                                "item's bound scope instead of against baseline ancestry",
+                            recorded_at: new Date().toISOString(),
+                        };
+                        await appendSubmissionEvent(paths.artifactsDir, event);
+                        recordedRecoveryMarks.push(event);
+                    }
+                }
+                catch (error) {
+                    issues.push({
+                        code: "recovery_unrecorded",
+                        work_item_id: workItem.id,
+                        result_path: workItem.result_path,
+                        message: "the recovery acceptance could not be recorded on the submission ledger, so it " +
+                            `was refused: ${error instanceof Error ? error.message : String(error)}`,
+                    });
+                    continue;
+                }
+            }
             acceptedFiles = corroborated.changedFiles;
         }
         const completedAt = new Date().toISOString();