@openrig/cli 0.1.11 → 0.2.0

package/daemon/dist/domain/restore-orchestrator.js

@@ -1,7 +1,22 @@
 import { writeFileSync } from "node:fs";
 import { join } from "node:path";
+import { assessNativeResumeProbe } from "./native-resume-probe.js";
 // Only these edge kinds constrain launch order
 const LAUNCH_DEPENDENCY_KINDS = new Set(["delegates_to", "spawned_by"]);
+export function rollupRestoreRigResult(nodes) {
+    if (nodes.length === 0)
+        return "failed";
+    // L3: `attention_required` is non-terminal failure (alive but blocked on
+    // operator action). It rolls up as `partially_restored`. `operator_recovered`
+    // is a clean post-reconciliation outcome and rolls up like `resumed`.
+    const allFailed = nodes.every((node) => node.status === "failed");
+    if (allFailed)
+        return "failed";
+    if (nodes.some((node) => node.status === "fresh" || node.status === "failed" || node.status === "attention_required")) {
+        return "partially_restored";
+    }
+    return "fully_restored";
+}
 export class RestoreOrchestrator {
     db;
     activeRestores = new Set();
@@ -62,8 +77,13 @@ export class RestoreOrchestrator {
         if (!rig) {
             return { ok: false, code: "rig_not_found", message: `Rig ${rigId} not found` };
         }
-        if (this.hasRunningSessions(rigId)) {
-            return { ok: false, code: "rig_not_stopped", message: `Rig ${rigId} must be stopped before restore` };
+        // Classify DB-running sessions against tmux reality WITHOUT mutating DB.
+        // This determines whether the rig is safe to restore before any state
+        // changes occur — critical for pre_restore snapshot ordering (snapshot
+        // must capture original DB state, not post-reconciliation state).
+        const classification = await this.classifyRunningSessions(rigId);
+        if (classification.live.length > 0 || classification.unknown.length > 0) {
+            return { ok: false, code: "rig_not_stopped", message: `Rig ${rigId} has live sessions. Stop the rig with 'rig down' before restoring, or use the latest auto-pre-down snapshot.` };
         }
         // Per-rig concurrency lock
         if (this.activeRestores.has(rigId)) {
@@ -71,17 +91,52 @@ export class RestoreOrchestrator {
         }
         this.activeRestores.add(rigId);
         try {
-            // 2. Capture pre-restore snapshot BEFORE any mutations
+            const validation = this.validatePreRestore(snapshot.data, {
+                fsOps: opts?.fsOps,
+                servicesRecord: this.rigRepo.getServicesRecord(rigId),
+            });
+            if (validation.blockers.length > 0) {
+                const result = {
+                    snapshotId,
+                    preRestoreSnapshotId: null,
+                    rigResult: "not_attempted",
+                    nodes: [],
+                    warnings: validation.warnings,
+                    blockers: validation.blockers,
+                };
+                return {
+                    ok: false,
+                    code: "pre_restore_validation_failed",
+                    message: "Restore pre-validation failed; no restore mutation was attempted.",
+                    result,
+                };
+            }
+            // 2. Capture pre-restore snapshot BEFORE any DB mutations —
+            // DB still reflects original session state (running for stale sessions)
             const preRestoreSnapshot = this.snapshotCapture.captureSnapshot(rigId, "pre_restore");
-            // 3. Emit restore.started
-            this.eventBus.emit({ type: "restore.started", rigId, snapshotId });
+            // 2b. NOW mark stale sessions as detached (safe: we've captured the
+            // pre-restore snapshot and confirmed no live/unknown sessions remain)
+            for (const sessionId of classification.stale) {
+                this.sessionRegistry.markDetached(sessionId);
+            }
+            // 3. Emit restore.started — the persisted event seq IS the attempt id
+            //    (Decision 1: no separate restore_attempts table).
+            const restoreStartedEvent = this.eventBus.emit({ type: "restore.started", rigId, snapshotId });
+            const attemptId = restoreStartedEvent.seq;
+            try {
+                opts?.onAttemptStarted?.(attemptId);
+            }
+            catch {
+                // onAttemptStarted is fire-and-forget; never let a route's response
+                // logic crash the restore pipeline.
+            }
             // 3b. Service gate: boot services before agent restore if this rig has services
             if (this.serviceOrchestrator) {
                 const svcRecord = this.rigRepo.getServicesRecord(rigId);
                 if (svcRecord) {
                     const bootResult = await this.serviceOrchestrator.boot(rigId);
                     if (!bootResult.ok) {
-                        this.eventBus.emit({ type: "restore.completed", rigId, snapshotId, result: { snapshotId, preRestoreSnapshotId: preRestoreSnapshot.id, nodes: [], warnings: [`Service boot failed: ${bootResult.error}`] } });
+                        this.eventBus.emit({ type: "restore.completed", rigId, snapshotId, result: { snapshotId, preRestoreSnapshotId: preRestoreSnapshot.id, rigResult: "failed", nodes: [], warnings: [`Service boot failed: ${bootResult.error}`] } });
                         return { ok: false, code: "service_boot_failed", message: `Service boot failed before agent restore: ${bootResult.error}` };
                     }
                 }
@@ -90,7 +145,7 @@ export class RestoreOrchestrator {
             const plan = this.computeRestorePlan(snapshot.data);
             // 5. Execute restore with compensating pattern per node
             const nodeResults = [];
-            const restoreWarnings = [];
+            const restoreWarnings = [...validation.warnings];
             for (const entry of plan) {
                 const result = await this.restoreNodeWithCompensation(entry, rigId, snapshotId, snapshot.data, opts, restoreWarnings);
                 nodeResults.push(result);
@@ -98,6 +153,7 @@ export class RestoreOrchestrator {
             const restoreResult = {
                 snapshotId,
                 preRestoreSnapshotId: preRestoreSnapshot.id,
+                rigResult: rollupRestoreRigResult(nodeResults),
                 nodes: nodeResults,
                 warnings: restoreWarnings,
             };
@@ -116,6 +172,169 @@ export class RestoreOrchestrator {
             this.activeRestores.delete(rigId);
         }
     }
+    validatePreRestore(data, opts) {
+        const blockers = [];
+        const warnings = [];
+        const exists = opts.fsOps?.exists ?? (() => true);
+        const add = (blocker) => blockers.push(blocker);
+        const nodes = Array.isArray(data.nodes) ? data.nodes : null;
+        const sessions = Array.isArray(data.sessions) ? data.sessions : null;
+        const edges = Array.isArray(data.edges) ? data.edges : null;
+        const checkpoints = data.checkpoints && typeof data.checkpoints === "object" ? data.checkpoints : null;
+        if (!data.rig || typeof data.rig.id !== "string") {
+            add({
+                code: "invalid_snapshot_data",
+                severity: "critical",
+                target: "snapshot.rig",
+                message: "Snapshot is missing the rig record needed for restore.",
+                remediation: "Capture a new snapshot or restore from a structurally valid snapshot.",
+            });
+        }
+        if (!nodes) {
+            add({
+                code: "invalid_snapshot_data",
+                severity: "critical",
+                target: "snapshot.nodes",
+                message: "Snapshot is missing the node list needed for restore.",
+                remediation: "Capture a new snapshot or restore from a structurally valid snapshot.",
+            });
+        }
+        if (!sessions) {
+            add({
+                code: "invalid_snapshot_data",
+                severity: "critical",
+                target: "snapshot.sessions",
+                message: "Snapshot is missing session records needed for restore.",
+                remediation: "Capture a new snapshot or restore from a structurally valid snapshot.",
+            });
+        }
+        if (!edges) {
+            add({
+                code: "invalid_snapshot_data",
+                severity: "critical",
+                target: "snapshot.edges",
+                message: "Snapshot is missing topology edges needed for restore planning.",
+                remediation: "Capture a new snapshot or restore from a structurally valid snapshot.",
+            });
+        }
+        if (!checkpoints) {
+            add({
+                code: "invalid_snapshot_data",
+                severity: "critical",
+                target: "snapshot.checkpoints",
+                message: "Snapshot is missing the checkpoint map needed for restore.",
+                remediation: "Capture a new snapshot or restore from a structurally valid snapshot.",
+            });
+        }
+        if (!nodes || !checkpoints) {
+            return { blockers, warnings };
+        }
+        for (const node of nodes) {
+            const checkpoint = checkpoints[node.id] ?? null;
+            if (checkpoint && !node.cwd) {
+                add({
+                    code: "checkpoint_missing_node_cwd",
+                    severity: "critical",
+                    nodeId: node.id,
+                    logicalId: node.logicalId,
+                    target: "checkpoint",
+                    message: `Checkpoint exists for ${node.logicalId}, but the node has no cwd to receive it.`,
+                    remediation: "Update the rig spec to include a cwd for this node, then capture a new snapshot or restore manually.",
+                });
+            }
+            const startupCtx = data.nodeStartupContext?.[node.id] ?? null;
+            if (!startupCtx)
+                continue;
+            for (const file of startupCtx.resolvedStartupFiles ?? []) {
+                if (!file.required) {
+                    if (this.pathLike(file.absolutePath) && !exists(file.absolutePath)) {
+                        warnings.push(`Restore pre-validation: optional startup file missing for ${node.logicalId}: ${file.absolutePath}`);
+                    }
+                    continue;
+                }
+                if (this.pathLike(file.ownerRoot) && !exists(file.ownerRoot)) {
+                    add({
+                        code: "startup_owner_root_missing",
+                        severity: "critical",
+                        nodeId: node.id,
+                        logicalId: node.logicalId,
+                        target: file.path,
+                        path: file.ownerRoot,
+                        message: `Required startup file owner root is missing for ${node.logicalId}: ${file.ownerRoot}`,
+                        remediation: "Restore the agent/source root or capture a new snapshot with reachable startup context.",
+                    });
+                }
+                if (this.pathLike(file.absolutePath) && !exists(file.absolutePath)) {
+                    add({
+                        code: "required_startup_file_missing",
+                        severity: "critical",
+                        nodeId: node.id,
+                        logicalId: node.logicalId,
+                        target: file.path,
+                        path: file.absolutePath,
+                        message: `Required startup file is missing for ${node.logicalId}: ${file.absolutePath}`,
+                        remediation: "Restore the missing startup file or capture a new snapshot before retrying restore.",
+                    });
+                }
+            }
+            for (const entry of startupCtx.projectionEntries ?? []) {
+                if (this.pathLike(entry.sourcePath) && !exists(entry.sourcePath)) {
+                    add({
+                        code: "projection_source_missing",
+                        severity: "critical",
+                        nodeId: node.id,
+                        logicalId: node.logicalId,
+                        target: entry.effectiveId,
+                        path: entry.sourcePath,
+                        message: `Projection source root is missing for ${node.logicalId}: ${entry.sourcePath}`,
+                        remediation: "Restore the agent/source root that owns this projected resource or capture a new snapshot.",
+                    });
+                }
+                if (this.pathLike(entry.absolutePath) && !exists(entry.absolutePath)) {
+                    add({
+                        code: "projection_entry_missing",
+                        severity: "critical",
+                        nodeId: node.id,
+                        logicalId: node.logicalId,
+                        target: entry.effectiveId,
+                        path: entry.absolutePath,
+                        message: `Projection entry is missing for ${node.logicalId}: ${entry.absolutePath}`,
+                        remediation: "Restore the projected source artifact or capture a new snapshot before retrying restore.",
+                    });
+                }
+            }
+        }
+        const servicesRecord = opts.servicesRecord ?? null;
+        if (servicesRecord) {
+            if (this.pathLike(servicesRecord.rigRoot) && !exists(servicesRecord.rigRoot)) {
+                add({
+                    code: "service_rig_root_missing",
+                    severity: "critical",
+                    target: "services.rigRoot",
+                    path: servicesRecord.rigRoot,
+                    message: `Service rig root is missing: ${servicesRecord.rigRoot}`,
+                    remediation: "Restore the service rig root or update the services record before retrying restore.",
+                });
+            }
+            if (this.pathLike(servicesRecord.composeFile) && !exists(servicesRecord.composeFile)) {
+                add({
+                    code: "service_compose_file_missing",
+                    severity: "critical",
+                    target: "services.composeFile",
+                    path: servicesRecord.composeFile,
+                    message: `Service compose file is missing: ${servicesRecord.composeFile}`,
+                    remediation: "Restore the compose file or update the services record before retrying restore.",
+                });
+            }
+        }
+        return { blockers, warnings };
+    }
+    pathLike(value) {
+        return typeof value === "string" && value.trim().length > 0 && (value.startsWith("/")
+            || value.startsWith("./")
+            || value.startsWith("../")
+            || value.startsWith("~"));
+    }
     captureNodeState(nodeId, rigId) {
         const binding = this.sessionRegistry.getBindingForNode(nodeId);
         const sessions = this.sessionRegistry.getSessionsForRig(rigId)
@@ -123,12 +342,35 @@ export class RestoreOrchestrator {
             .map((s) => ({ id: s.id, status: s.status }));
         return { binding, sessions };
     }
-    hasRunningSessions(rigId) {
-        const latestByNode = new Map();
+    /**
+     * Classify ALL DB-running sessions against tmux reality without mutating DB.
+     * Scans every session (not just latest-per-node) to catch older live sessions
+     * behind newer detached rows. Returns structured classification for the caller
+     * to act on: live sessions block restore, stale sessions get marked detached
+     * AFTER the pre_restore snapshot is captured, unknown sessions fail closed.
+     */
+    async classifyRunningSessions(rigId) {
+        const live = [];
+        const stale = [];
+        const unknown = [];
         for (const session of this.sessionRegistry.getSessionsForRig(rigId)) {
-            latestByNode.set(session.nodeId, session);
+            if (session.status !== "running")
+                continue;
+            try {
+                const alive = await this.tmuxAdapter.hasSession(session.sessionName);
+                if (alive) {
+                    live.push(session.id);
+                }
+                else {
+                    stale.push(session.id);
+                }
+            }
+            catch {
+                // tmux check failed — fail closed: classify as unknown so restore blocks
+                unknown.push(session.id);
+            }
         }
-        return Array.from(latestByNode.values()).some((session) => session.status === "running");
+        return { live, stale, unknown };
     }
     clearStaleState(nodeId, rigId) {
         this.sessionRegistry.clearBinding(nodeId);
@@ -315,14 +557,28 @@ export class RestoreOrchestrator {
         if (resumeRequested && !isPodAware) {
             // Legacy resume path
             if (!resumeToken) {
-                return { nodeId: node.id, logicalId: node.logicalId, status: "failed", error: `Resume requested but no token available. Restore the node manually or launch fresh with: rig up` };
-            }
-            const resumed = await this.attemptResume(sessionName, resumeType, resumeToken, node.cwd ?? "/");
-            if (resumed) {
-                baseStatus = "resumed";
+                return { nodeId: node.id, logicalId: node.logicalId, status: "failed", error: `Resume requested but no token available. Restore the node manually or launch fresh explicitly.` };
             }
             else {
-                return { nodeId: node.id, logicalId: node.logicalId, status: "failed", error: `Resume attempted but failed. Check the harness state manually or launch fresh with: rig up` };
+                const resumeOutcome = await this.attemptResume(sessionName, resumeType, resumeToken, node.cwd ?? "/");
+                if (resumeOutcome.kind === "resumed") {
+                    baseStatus = "resumed";
+                }
+                else if (resumeOutcome.kind === "attention_required") {
+                    // L3 Decision 2: Claude resume-selection prompt -> attention_required.
+                    // Do NOT auto-answer. Reconcile later via reconcileNodeRuntimeTruth
+                    // when the operator reaches a usable pane state.
+                    return {
+                        nodeId: node.id,
+                        logicalId: node.logicalId,
+                        status: "attention_required",
+                        error: resumeOutcome.message,
+                        attentionEvidence: resumeOutcome.evidence ?? null,
+                    };
+                }
+                else {
+                    return { nodeId: node.id, logicalId: node.logicalId, status: "failed", error: `Resume attempted but failed. Check the harness state manually or launch fresh explicitly.` };
+                }
             }
         }
         else if (resumeRequested && isPodAware) {
@@ -334,7 +590,7 @@ export class RestoreOrchestrator {
                     nodeId: node.id,
                     logicalId: node.logicalId,
                     status: "failed",
-                    error: "Resume requested but no token available. Restore the node manually or launch fresh with: rig up",
+                    error: "Resume requested but no token available. Restore the node manually or launch fresh explicitly.",
                 };
             }
         }
@@ -400,11 +656,13 @@ export class RestoreOrchestrator {
                     const binding = {
                         ...launchResult.binding,
                         cwd: node.cwd ?? ".",
+                        codexConfigProfile: node.codexConfigProfile ?? undefined,
                     };
                     try {
                         const { StartupOrchestrator } = await import("./startup-orchestrator.js");
                         const startupOrch = new StartupOrchestrator({ db: this.db, sessionRegistry: this.sessionRegistry, eventBus: this.eventBus, tmuxAdapter: this.tmuxAdapter });
                         const replayAsRestore = baseStatus !== "fresh";
+                        const shouldLaunchHarness = isPodAware;
                         const startupResult = await startupOrch.startNode({
                             rigId,
                             nodeId: node.id,
@@ -415,15 +673,46 @@ export class RestoreOrchestrator {
                             resolvedStartupFiles: filteredFiles,
                             startupActions: startupCtx.startupActions,
                             isRestore: replayAsRestore,
-                            skipHarnessLaunch: !isPodAware, // Pod-aware: use launchHarness with resumeToken. Legacy: old helpers already handled.
+                            skipHarnessLaunch: !shouldLaunchHarness,
                             resumeToken: (isPodAware && resumeRequested) ? resumeToken ?? undefined : undefined,
                             sessionName: sessionName,
+                            allowFreshFallback: !(isPodAware && resumeRequested),
                         });
                         if (startupResult.ok) {
-                            // Pod-aware nodes with resume token: startup used launchHarness with the token → resumed
-                            const finalStatus = (isPodAware && resumeRequested) ? "resumed" : baseStatus;
+                            const nativeContinuityProved = isPodAware
+                                && resumeRequested
+                                && this.launchedSessionMatchesSnapshotResume(launchResult.session.id, resumeType, resumeToken);
+                            if (isPodAware && resumeRequested && startupResult.continuityOutcome === "fresh" && !nativeContinuityProved) {
+                                return {
+                                    nodeId: node.id,
+                                    logicalId: node.logicalId,
+                                    status: "failed",
+                                    error: "Resume attempted but runtime reported fresh continuity. Launch fresh explicitly if that degradation is acceptable.",
+                                };
+                            }
+                            const finalStatus = (isPodAware && resumeRequested)
+                                ? ((startupResult.continuityOutcome === "resumed" || nativeContinuityProved) ? "resumed" : baseStatus)
+                                : baseStatus;
                             return { nodeId: node.id, logicalId: node.logicalId, status: finalStatus };
                         }
+                        // Pod-aware attention_required: hoisted above both the
+                        // resume-requested and non-resume-requested failed branches so
+                        // that pod-aware Codex auth-refusal (verifyResumeLaunch →
+                        // recovery: "attention_required") surfaces honestly regardless
+                        // of whether resume was requested. Mirrors the runtime-agnostic
+                        // legacy mapping at lines 725-735. This MUST come before the
+                        // line 859 / 867 failed branches; otherwise the production
+                        // pod-aware-resume path (most common) returns `status: "failed"`
+                        // and the slice's "attention_required end-to-end" claim breaks.
+                        if (isPodAware && startupResult.startupStatus === "attention_required") {
+                            return {
+                                nodeId: node.id,
+                                logicalId: node.logicalId,
+                                status: "attention_required",
+                                error: `Restore startup requires attention: ${startupResult.errors.join("; ")}`,
+                                attentionEvidence: startupResult.evidence ?? null,
+                            };
+                        }
                         if (isPodAware && resumeRequested) {
                             return {
                                 nodeId: node.id,
@@ -432,6 +721,17 @@ export class RestoreOrchestrator {
                                 error: startupResult.errors.join("; "),
                             };
                         }
+                        if (isPodAware) {
+                            const prefix = startupResult.startupStatus === "attention_required"
+                                ? "Restore startup requires attention"
+                                : "Restore startup failed";
+                            return {
+                                nodeId: node.id,
+                                logicalId: node.logicalId,
+                                status: "failed",
+                                error: `${prefix}: ${startupResult.errors.join("; ")}`,
+                            };
+                        }
                         warnings?.push(`Restore startup failed for ${node.logicalId}: ${startupResult.errors.join("; ")}`);
                     }
                     catch (err) {
@@ -443,6 +743,14 @@ export class RestoreOrchestrator {
                                 error: `Restore startup error: ${err.message}`,
                             };
                         }
+                        if (isPodAware) {
+                            return {
+                                nodeId: node.id,
+                                logicalId: node.logicalId,
+                                status: "failed",
+                                error: `Restore startup error: ${err.message}`,
+                            };
+                        }
                         warnings?.push(`Restore startup error for ${node.logicalId}: ${err.message}`);
                     }
                 }
@@ -450,16 +758,160 @@ export class RestoreOrchestrator {
         }
         return { nodeId: node.id, logicalId: node.logicalId, status: baseStatus };
     }
+    launchedSessionMatchesSnapshotResume(sessionId, resumeType, resumeToken) {
+        if (!resumeType || !resumeToken)
+            return false;
+        const row = this.db.prepare("SELECT resume_type, resume_token FROM sessions WHERE id = ?").get(sessionId);
+        if (!row?.resume_type || !row.resume_token)
+            return false;
+        return row.resume_type === resumeType && row.resume_token === resumeToken;
+    }
     async attemptResume(sessionName, resumeType, resumeToken, cwd) {
         if (this.claudeResume.canResume(resumeType, resumeToken)) {
             const result = await this.claudeResume.resume(sessionName, resumeType, resumeToken, cwd);
-            return result.ok;
+            if (result.ok)
+                return { kind: "resumed" };
+            if (result.code === "retry_fresh")
+                return { kind: "retry_fresh" };
+            // L3: surface attention_required from the Claude probe (resume-selection prompt).
+            if (result.code === "attention_required") {
+                return {
+                    kind: "attention_required",
+                    message: result.message,
+                    evidence: result.evidence,
+                };
+            }
+            return { kind: "failed", message: result.message };
         }
         if (this.codexResume.canResume(resumeType, resumeToken)) {
             const result = await this.codexResume.resume(sessionName, resumeType, resumeToken, cwd);
-            return result.ok;
+            if (result.ok)
+                return { kind: "resumed" };
+            if (result.code === "retry_fresh")
+                return { kind: "retry_fresh" };
+            // Codex auth-refusal: stored OAuth token can no longer be refreshed.
+            // Recoverable — operator runs `codex login` and the seat continues.
+            // Per-node mapping at lines 725-735 emits `status: "attention_required"`
+            // with `attentionEvidence` for both runtimes; no further wiring needed.
+            if (result.code === "attention_required") {
+                return {
+                    kind: "attention_required",
+                    message: result.message,
+                    evidence: result.evidence,
+                };
+            }
+            return { kind: "failed", message: result.message };
         }
-        return false;
+        return { kind: "failed", message: "No resume adapter available for this runtime/token combination." };
+    }
+    /**
+     * L3 Decision 3: runtime-truth reconciliation. Given a node whose original
+     * `restoreOutcome` was `failed` or `attention_required`, examine current
+     * runtime state. If ALL four visible-evidence preconditions hold, append
+     * `restore.outcome_reconciled` so the node's effective post-reconciliation
+     * outcome becomes `operator_recovered`. Never mutates or deletes the
+     * original failure event; never produces `ready`.
+     *
+     * Returns `{ ok: true, attemptId, from, to, evidence }` on upgrade, or
+     * `{ ok: false, code, detail }` describing exactly which precondition
+     * failed (or "no_attempt" / "outcome_not_upgradable" when there is nothing
+     * to reconcile).
+     */
+    async reconcileNodeRuntimeTruth(rigId, nodeId) {
+        // Locate the latest restore attempt for this rig.
+        const startedRow = this.db.prepare("SELECT seq, payload FROM events WHERE rig_id = ? AND type = 'restore.started' ORDER BY seq DESC LIMIT 1").get(rigId);
+        if (!startedRow) {
+            return { ok: false, code: "no_attempt", detail: "No restore.started event recorded for this rig." };
+        }
+        const attemptId = startedRow.seq;
+        // Find the node's most recent post-attempt outcome from the most-recent
+        // restore.completed event for this rig. If the latest outcome is not
+        // failed or attention_required, the reconciler has nothing to upgrade.
+        const completedRow = this.db.prepare("SELECT payload FROM events WHERE rig_id = ? AND type = 'restore.completed' AND seq > ? ORDER BY seq DESC LIMIT 1").get(rigId, attemptId);
+        let nodeStatus = null;
+        let nodeLogicalId = null;
+        if (completedRow) {
+            try {
+                const parsed = JSON.parse(completedRow.payload);
+                const found = parsed.result?.nodes?.find((n) => n.nodeId === nodeId);
+                if (found) {
+                    nodeStatus = found.status;
+                    nodeLogicalId = found.logicalId;
+                }
+            }
+            catch {
+                // payload corruption — treat as no node record found
+            }
+        }
+        if (!nodeStatus) {
+            return { ok: false, code: "node_not_found", detail: `Node ${nodeId} has no record in the latest restore.completed event for rig ${rigId}.` };
+        }
+        if (nodeStatus !== "failed" && nodeStatus !== "attention_required") {
+            return { ok: false, code: "outcome_not_upgradable", detail: `Reconciliation only upgrades failed or attention_required; current outcome is ${nodeStatus}.` };
+        }
+        const fromStatus = nodeStatus;
+        // Resolve canonical session name for this node so we can probe tmux/pane.
+        const bindingRow = this.db.prepare("SELECT tmux_session FROM bindings WHERE node_id = ?").get(nodeId);
+        const sessionName = bindingRow?.tmux_session ?? null;
+        if (!sessionName) {
+            return { ok: false, code: "tmux_session_missing", detail: "No tmux session bound for this node." };
+        }
+        // Precondition #1: tmux session exists.
+        let alive = false;
+        try {
+            alive = await this.tmuxAdapter.hasSession(sessionName);
+        }
+        catch {
+            // L1 fail-closed: ambiguous probe failure stays as not-alive for the
+            // reconciler. Original failure event remains untouched.
+            alive = false;
+        }
+        if (!alive) {
+            return { ok: false, code: "tmux_session_missing", detail: `Tmux session ${sessionName} is not currently alive.` };
+        }
+        // Resolve runtime + capture pane state for preconditions #2 and #4.
+        const nodeRow = this.db.prepare("SELECT runtime FROM nodes WHERE id = ?").get(nodeId);
+        const runtime = nodeRow?.runtime ?? null;
+        const paneCommand = await this.tmuxAdapter.getPaneCommand(sessionName);
+        const paneContent = (await this.tmuxAdapter.capturePaneContent(sessionName, 40)) ?? "";
+        // Precondition #2: foreground process is the runtime (claude or codex).
+        const probe = assessNativeResumeProbe({ runtime, paneCommand, paneContent });
+        const fgProcess = paneCommand && (paneCommand === "claude" || paneCommand.startsWith("codex"))
+            ? (paneCommand === "claude" ? "claude" : "codex")
+            : null;
+        if (!fgProcess) {
+            return { ok: false, code: "fg_process_not_runtime", detail: `Foreground process is "${paneCommand ?? "(unknown)"}", not claude/codex.` };
+        }
+        // Precondition #3: the resume token was actually used at launch.
+        // Stored on the latest session row; null/empty means resume was not
+        // exercised so a "recovered" claim has no basis.
+        const sessRow = this.db.prepare("SELECT resume_token FROM sessions WHERE node_id = ? ORDER BY id DESC LIMIT 1").get(nodeId);
+        const resumeTokenUsed = typeof sessRow?.resume_token === "string" && sessRow.resume_token.length > 0;
+        if (!resumeTokenUsed) {
+            return { ok: false, code: "resume_token_not_used", detail: "No resume token recorded on the latest session row." };
+        }
+        // Precondition #4: pane is at a usable/idle state — explicitly NOT a
+        // resume-selection prompt and not the "returned to shell" failure mode.
+        if (probe.status !== "resumed") {
+            return { ok: false, code: "pane_not_usable", detail: `Pane state is ${probe.status} (${probe.code}); reconciliation requires resumed.` };
+        }
+        // All four preconditions hold. Append (never mutate) the audit event.
+        this.eventBus.emit({
+            type: "restore.outcome_reconciled",
+            rigId,
+            nodeId,
+            attemptId,
+            from: fromStatus,
+            to: "operator_recovered",
+            evidence: { tmux: true, fgProcess, resumeTokenUsed: true, paneState: "usable" },
+        });
+        return {
+            ok: true,
+            attemptId,
+            from: fromStatus,
+            to: "operator_recovered",
+            evidence: { tmux: true, fgProcess, resumeTokenUsed: true, paneState: "usable" },
+        };
     }
     writeCheckpointFile(cwd, checkpoint) {
         try {