@openrig/cli 0.1.12 → 0.3.0

package/daemon/dist/domain/restore-orchestrator.js

@@ -1,7 +1,22 @@
 import { writeFileSync } from "node:fs";
 import { join } from "node:path";
+import { assessNativeResumeProbe } from "./native-resume-probe.js";
 // Only these edge kinds constrain launch order
 const LAUNCH_DEPENDENCY_KINDS = new Set(["delegates_to", "spawned_by"]);
+export function rollupRestoreRigResult(nodes) {
+    if (nodes.length === 0)
+        return "failed";
+    // L3: `attention_required` is non-terminal failure (alive but blocked on
+    // operator action). It rolls up as `partially_restored`. `operator_recovered`
+    // is a clean post-reconciliation outcome and rolls up like `resumed`.
+    const allFailed = nodes.every((node) => node.status === "failed");
+    if (allFailed)
+        return "failed";
+    if (nodes.some((node) => node.status === "fresh" || node.status === "failed" || node.status === "attention_required")) {
+        return "partially_restored";
+    }
+    return "fully_restored";
+}
 export class RestoreOrchestrator {
     db;
     activeRestores = new Set();
@@ -62,8 +77,13 @@ export class RestoreOrchestrator {
         if (!rig) {
             return { ok: false, code: "rig_not_found", message: `Rig ${rigId} not found` };
         }
-        if (this.hasRunningSessions(rigId)) {
-            return { ok: false, code: "rig_not_stopped", message: `Rig ${rigId} must be stopped before restore` };
+        // Classify DB-running sessions against tmux reality WITHOUT mutating DB.
+        // This determines whether the rig is safe to restore before any state
+        // changes occur — critical for pre_restore snapshot ordering (snapshot
+        // must capture original DB state, not post-reconciliation state).
+        const classification = await this.classifyRunningSessions(rigId);
+        if (classification.live.length > 0 || classification.unknown.length > 0) {
+            return { ok: false, code: "rig_not_stopped", message: `Rig ${rigId} has live sessions. Stop the rig with 'rig down' before restoring, or use the latest auto-pre-down snapshot.` };
         }
         // Per-rig concurrency lock
         if (this.activeRestores.has(rigId)) {
@@ -71,17 +91,52 @@ export class RestoreOrchestrator {
         }
         this.activeRestores.add(rigId);
         try {
-            // 2. Capture pre-restore snapshot BEFORE any mutations
+            const validation = this.validatePreRestore(snapshot.data, {
+                fsOps: opts?.fsOps,
+                servicesRecord: this.rigRepo.getServicesRecord(rigId),
+            });
+            if (validation.blockers.length > 0) {
+                const result = {
+                    snapshotId,
+                    preRestoreSnapshotId: null,
+                    rigResult: "not_attempted",
+                    nodes: [],
+                    warnings: validation.warnings,
+                    blockers: validation.blockers,
+                };
+                return {
+                    ok: false,
+                    code: "pre_restore_validation_failed",
+                    message: "Restore pre-validation failed; no restore mutation was attempted.",
+                    result,
+                };
+            }
+            // 2. Capture pre-restore snapshot BEFORE any DB mutations —
+            // DB still reflects original session state (running for stale sessions)
             const preRestoreSnapshot = this.snapshotCapture.captureSnapshot(rigId, "pre_restore");
-            // 3. Emit restore.started
-            this.eventBus.emit({ type: "restore.started", rigId, snapshotId });
+            // 2b. NOW mark stale sessions as detached (safe: we've captured the
+            // pre-restore snapshot and confirmed no live/unknown sessions remain)
+            for (const sessionId of classification.stale) {
+                this.sessionRegistry.markDetached(sessionId);
+            }
+            // 3. Emit restore.started — the persisted event seq IS the attempt id
+            //    (Decision 1: no separate restore_attempts table).
+            const restoreStartedEvent = this.eventBus.emit({ type: "restore.started", rigId, snapshotId });
+            const attemptId = restoreStartedEvent.seq;
+            try {
+                opts?.onAttemptStarted?.(attemptId);
+            }
+            catch {
+                // onAttemptStarted is fire-and-forget; never let a route's response
+                // logic crash the restore pipeline.
+            }
             // 3b. Service gate: boot services before agent restore if this rig has services
             if (this.serviceOrchestrator) {
                 const svcRecord = this.rigRepo.getServicesRecord(rigId);
                 if (svcRecord) {
                     const bootResult = await this.serviceOrchestrator.boot(rigId);
                     if (!bootResult.ok) {
-                        this.eventBus.emit({ type: "restore.completed", rigId, snapshotId, result: { snapshotId, preRestoreSnapshotId: preRestoreSnapshot.id, nodes: [], warnings: [`Service boot failed: ${bootResult.error}`] } });
+                        this.eventBus.emit({ type: "restore.completed", rigId, snapshotId, result: { snapshotId, preRestoreSnapshotId: preRestoreSnapshot.id, rigResult: "failed", nodes: [], warnings: [`Service boot failed: ${bootResult.error}`] } });
                         return { ok: false, code: "service_boot_failed", message: `Service boot failed before agent restore: ${bootResult.error}` };
                     }
                 }
@@ -90,7 +145,7 @@ export class RestoreOrchestrator {
             const plan = this.computeRestorePlan(snapshot.data);
             // 5. Execute restore with compensating pattern per node
             const nodeResults = [];
-            const restoreWarnings = [];
+            const restoreWarnings = [...validation.warnings];
             for (const entry of plan) {
                 const result = await this.restoreNodeWithCompensation(entry, rigId, snapshotId, snapshot.data, opts, restoreWarnings);
                 nodeResults.push(result);
@@ -98,6 +153,7 @@ export class RestoreOrchestrator {
             const restoreResult = {
                 snapshotId,
                 preRestoreSnapshotId: preRestoreSnapshot.id,
+                rigResult: rollupRestoreRigResult(nodeResults),
                 nodes: nodeResults,
                 warnings: restoreWarnings,
             };
@@ -116,6 +172,169 @@ export class RestoreOrchestrator {
             this.activeRestores.delete(rigId);
         }
     }
+    validatePreRestore(data, opts) {
+        const blockers = [];
+        const warnings = [];
+        const exists = opts.fsOps?.exists ?? (() => true);
+        const add = (blocker) => blockers.push(blocker);
+        const nodes = Array.isArray(data.nodes) ? data.nodes : null;
+        const sessions = Array.isArray(data.sessions) ? data.sessions : null;
+        const edges = Array.isArray(data.edges) ? data.edges : null;
+        const checkpoints = data.checkpoints && typeof data.checkpoints === "object" ? data.checkpoints : null;
+        if (!data.rig || typeof data.rig.id !== "string") {
+            add({
+                code: "invalid_snapshot_data",
+                severity: "critical",
+                target: "snapshot.rig",
+                message: "Snapshot is missing the rig record needed for restore.",
+                remediation: "Capture a new snapshot or restore from a structurally valid snapshot.",
+            });
+        }
+        if (!nodes) {
+            add({
+                code: "invalid_snapshot_data",
+                severity: "critical",
+                target: "snapshot.nodes",
+                message: "Snapshot is missing the node list needed for restore.",
+                remediation: "Capture a new snapshot or restore from a structurally valid snapshot.",
+            });
+        }
+        if (!sessions) {
+            add({
+                code: "invalid_snapshot_data",
+                severity: "critical",
+                target: "snapshot.sessions",
+                message: "Snapshot is missing session records needed for restore.",
+                remediation: "Capture a new snapshot or restore from a structurally valid snapshot.",
+            });
+        }
+        if (!edges) {
+            add({
+                code: "invalid_snapshot_data",
+                severity: "critical",
+                target: "snapshot.edges",
+                message: "Snapshot is missing topology edges needed for restore planning.",
+                remediation: "Capture a new snapshot or restore from a structurally valid snapshot.",
+            });
+        }
+        if (!checkpoints) {
+            add({
+                code: "invalid_snapshot_data",
+                severity: "critical",
+                target: "snapshot.checkpoints",
+                message: "Snapshot is missing the checkpoint map needed for restore.",
+                remediation: "Capture a new snapshot or restore from a structurally valid snapshot.",
+            });
+        }
+        if (!nodes || !checkpoints) {
+            return { blockers, warnings };
+        }
+        for (const node of nodes) {
+            const checkpoint = checkpoints[node.id] ?? null;
+            if (checkpoint && !node.cwd) {
+                add({
+                    code: "checkpoint_missing_node_cwd",
+                    severity: "critical",
+                    nodeId: node.id,
+                    logicalId: node.logicalId,
+                    target: "checkpoint",
+                    message: `Checkpoint exists for ${node.logicalId}, but the node has no cwd to receive it.`,
+                    remediation: "Update the rig spec to include a cwd for this node, then capture a new snapshot or restore manually.",
+                });
+            }
+            const startupCtx = data.nodeStartupContext?.[node.id] ?? null;
+            if (!startupCtx)
+                continue;
+            for (const file of startupCtx.resolvedStartupFiles ?? []) {
+                if (!file.required) {
+                    if (this.pathLike(file.absolutePath) && !exists(file.absolutePath)) {
+                        warnings.push(`Restore pre-validation: optional startup file missing for ${node.logicalId}: ${file.absolutePath}`);
+                    }
+                    continue;
+                }
+                if (this.pathLike(file.ownerRoot) && !exists(file.ownerRoot)) {
+                    add({
+                        code: "startup_owner_root_missing",
+                        severity: "critical",
+                        nodeId: node.id,
+                        logicalId: node.logicalId,
+                        target: file.path,
+                        path: file.ownerRoot,
+                        message: `Required startup file owner root is missing for ${node.logicalId}: ${file.ownerRoot}`,
+                        remediation: "Restore the agent/source root or capture a new snapshot with reachable startup context.",
+                    });
+                }
+                if (this.pathLike(file.absolutePath) && !exists(file.absolutePath)) {
+                    add({
+                        code: "required_startup_file_missing",
+                        severity: "critical",
+                        nodeId: node.id,
+                        logicalId: node.logicalId,
+                        target: file.path,
+                        path: file.absolutePath,
+                        message: `Required startup file is missing for ${node.logicalId}: ${file.absolutePath}`,
+                        remediation: "Restore the missing startup file or capture a new snapshot before retrying restore.",
+                    });
+                }
+            }
+            for (const entry of startupCtx.projectionEntries ?? []) {
+                if (this.pathLike(entry.sourcePath) && !exists(entry.sourcePath)) {
+                    add({
+                        code: "projection_source_missing",
+                        severity: "critical",
+                        nodeId: node.id,
+                        logicalId: node.logicalId,
+                        target: entry.effectiveId,
+                        path: entry.sourcePath,
+                        message: `Projection source root is missing for ${node.logicalId}: ${entry.sourcePath}`,
+                        remediation: "Restore the agent/source root that owns this projected resource or capture a new snapshot.",
+                    });
+                }
+                if (this.pathLike(entry.absolutePath) && !exists(entry.absolutePath)) {
+                    add({
+                        code: "projection_entry_missing",
+                        severity: "critical",
+                        nodeId: node.id,
+                        logicalId: node.logicalId,
+                        target: entry.effectiveId,
+                        path: entry.absolutePath,
+                        message: `Projection entry is missing for ${node.logicalId}: ${entry.absolutePath}`,
+                        remediation: "Restore the projected source artifact or capture a new snapshot before retrying restore.",
+                    });
+                }
+            }
+        }
+        const servicesRecord = opts.servicesRecord ?? null;
+        if (servicesRecord) {
+            if (this.pathLike(servicesRecord.rigRoot) && !exists(servicesRecord.rigRoot)) {
+                add({
+                    code: "service_rig_root_missing",
+                    severity: "critical",
+                    target: "services.rigRoot",
+                    path: servicesRecord.rigRoot,
+                    message: `Service rig root is missing: ${servicesRecord.rigRoot}`,
+                    remediation: "Restore the service rig root or update the services record before retrying restore.",
+                });
+            }
+            if (this.pathLike(servicesRecord.composeFile) && !exists(servicesRecord.composeFile)) {
+                add({
+                    code: "service_compose_file_missing",
+                    severity: "critical",
+                    target: "services.composeFile",
+                    path: servicesRecord.composeFile,
+                    message: `Service compose file is missing: ${servicesRecord.composeFile}`,
+                    remediation: "Restore the compose file or update the services record before retrying restore.",
+                });
+            }
+        }
+        return { blockers, warnings };
+    }
+    pathLike(value) {
+        return typeof value === "string" && value.trim().length > 0 && (value.startsWith("/")
+            || value.startsWith("./")
+            || value.startsWith("../")
+            || value.startsWith("~"));
+    }
     captureNodeState(nodeId, rigId) {
         const binding = this.sessionRegistry.getBindingForNode(nodeId);
         const sessions = this.sessionRegistry.getSessionsForRig(rigId)
@@ -123,12 +342,35 @@ export class RestoreOrchestrator {
             .map((s) => ({ id: s.id, status: s.status }));
         return { binding, sessions };
     }
-    hasRunningSessions(rigId) {
-        const latestByNode = new Map();
+    /**
+     * Classify ALL DB-running sessions against tmux reality without mutating DB.
+     * Scans every session (not just latest-per-node) to catch older live sessions
+     * behind newer detached rows. Returns structured classification for the caller
+     * to act on: live sessions block restore, stale sessions get marked detached
+     * AFTER the pre_restore snapshot is captured, unknown sessions fail closed.
+     */
+    async classifyRunningSessions(rigId) {
+        const live = [];
+        const stale = [];
+        const unknown = [];
         for (const session of this.sessionRegistry.getSessionsForRig(rigId)) {
-            latestByNode.set(session.nodeId, session);
+            if (session.status !== "running")
+                continue;
+            try {
+                const alive = await this.tmuxAdapter.hasSession(session.sessionName);
+                if (alive) {
+                    live.push(session.id);
+                }
+                else {
+                    stale.push(session.id);
+                }
+            }
+            catch {
+                // tmux check failed — fail closed: classify as unknown so restore blocks
+                unknown.push(session.id);
+            }
         }
-        return Array.from(latestByNode.values()).some((session) => session.status === "running");
+        return { live, stale, unknown };
     }
     clearStaleState(nodeId, rigId) {
         this.sessionRegistry.clearBinding(nodeId);
@@ -249,7 +491,9 @@ export class RestoreOrchestrator {
         this.clearStaleState(nodeId, rigId);
         // Derive canonical session name for pod-aware nodes
         const rig = this.rigRepo.getRig(rigId);
-        let launchOpts;
+        let launchOpts = node.cwd
+            ? { cwd: node.cwd }
+            : undefined;
         let expectedSessionName;
         if (node.podId && rig) {
             // Pod-aware: derive {pod}-{member}@{rigName} from node identity
@@ -259,7 +503,7 @@ export class RestoreOrchestrator {
                 const memberPart = parts.slice(1).join(".");
                 const { deriveCanonicalSessionName, deriveSessionName } = await import("./session-name.js");
                 expectedSessionName = deriveCanonicalSessionName(podPart, memberPart, rig.rig.name);
-                launchOpts = { sessionName: expectedSessionName };
+                launchOpts = { ...launchOpts, sessionName: expectedSessionName };
             }
         }
         if (!expectedSessionName && rig) {
@@ -309,34 +553,33 @@ export class RestoreOrchestrator {
         const resumeToken = session?.resumeToken ?? null;
         const resumeRequested = restorePolicy === "resume_if_possible" && !!resumeType && resumeType !== "none";
         let baseStatus = "fresh";
-        let needsFreshLaunchFallback = false;
         // Pod-aware nodes: resume via launchHarness (handled in startup orchestrator with skipHarnessLaunch: false)
         // Legacy nodes: resume via old claude-resume/codex-resume helpers
         const isPodAware = !!node.podId;
         if (resumeRequested && !isPodAware) {
             // Legacy resume path
             if (!resumeToken) {
-                if (this.shouldFallbackFreshWithoutResume(node.runtime, resumeType)) {
-                    warnings?.push(`Node ${node.logicalId}: resume requested but no token was available; launched fresh instead.`);
-                    baseStatus = "fresh";
-                    needsFreshLaunchFallback = true;
-                }
-                else {
-                    return { nodeId: node.id, logicalId: node.logicalId, status: "failed", error: `Resume requested but no token available. Restore the node manually or launch fresh with: rig up` };
-                }
+                return { nodeId: node.id, logicalId: node.logicalId, status: "failed", error: `Resume requested but no token available. Restore the node manually or launch fresh explicitly.` };
             }
             else {
                 const resumeOutcome = await this.attemptResume(sessionName, resumeType, resumeToken, node.cwd ?? "/");
-                if (resumeOutcome === "resumed") {
+                if (resumeOutcome.kind === "resumed") {
                     baseStatus = "resumed";
                 }
-                else if (resumeOutcome === "retry_fresh") {
-                    warnings?.push(`Node ${node.logicalId}: resume was unavailable; launched fresh instead.`);
-                    baseStatus = "fresh";
-                    needsFreshLaunchFallback = true;
+                else if (resumeOutcome.kind === "attention_required") {
+                    // L3 Decision 2: Claude resume-selection prompt -> attention_required.
+                    // Do NOT auto-answer. Reconcile later via reconcileNodeRuntimeTruth
+                    // when the operator reaches a usable pane state.
+                    return {
+                        nodeId: node.id,
+                        logicalId: node.logicalId,
+                        status: "attention_required",
+                        error: resumeOutcome.message,
+                        attentionEvidence: resumeOutcome.evidence ?? null,
+                    };
                 }
                 else {
-                    return { nodeId: node.id, logicalId: node.logicalId, status: "failed", error: `Resume attempted but failed. Check the harness state manually or launch fresh with: rig up` };
+                    return { nodeId: node.id, logicalId: node.logicalId, status: "failed", error: `Resume attempted but failed. Check the harness state manually or launch fresh explicitly.` };
                 }
             }
         }
@@ -345,18 +588,12 @@ export class RestoreOrchestrator {
             // if resume was requested but continuity state is unavailable, fail loudly
             // instead of silently downgrading to a fresh launch with amnesia.
             if (!resumeToken) {
-                if (this.shouldFallbackFreshWithoutResume(node.runtime, resumeType)) {
-                    warnings?.push(`Node ${node.logicalId}: resume requested but no token was available; launched fresh instead.`);
-                    needsFreshLaunchFallback = true;
-                }
-                else {
-                    return {
-                        nodeId: node.id,
-                        logicalId: node.logicalId,
-                        status: "failed",
-                        error: "Resume requested but no token available. Restore the node manually or launch fresh with: rig up",
-                    };
-                }
+                return {
+                    nodeId: node.id,
+                    logicalId: node.logicalId,
+                    status: "failed",
+                    error: "Resume requested but no token available. Restore the node manually or launch fresh explicitly.",
+                };
             }
         }
         // Checkpoint delivery (if not already resumed)
@@ -421,12 +658,13 @@ export class RestoreOrchestrator {
                     const binding = {
                         ...launchResult.binding,
                         cwd: node.cwd ?? ".",
+                        codexConfigProfile: node.codexConfigProfile ?? undefined,
                     };
                     try {
                         const { StartupOrchestrator } = await import("./startup-orchestrator.js");
                         const startupOrch = new StartupOrchestrator({ db: this.db, sessionRegistry: this.sessionRegistry, eventBus: this.eventBus, tmuxAdapter: this.tmuxAdapter });
                         const replayAsRestore = baseStatus !== "fresh";
-                        const shouldLaunchHarness = isPodAware || needsFreshLaunchFallback;
+                        const shouldLaunchHarness = isPodAware;
                         const startupResult = await startupOrch.startNode({
                             rigId,
                             nodeId: node.id,
@@ -438,24 +676,46 @@ export class RestoreOrchestrator {
                             startupActions: startupCtx.startupActions,
                             isRestore: replayAsRestore,
                             skipHarnessLaunch: !shouldLaunchHarness,
-                            resumeToken: (isPodAware && resumeRequested && !needsFreshLaunchFallback) ? resumeToken ?? undefined : undefined,
+                            resumeToken: (isPodAware && resumeRequested) ? resumeToken ?? undefined : undefined,
                             sessionName: sessionName,
+                            allowFreshFallback: !(isPodAware && resumeRequested),
                         });
                         if (startupResult.ok) {
                             const nativeContinuityProved = isPodAware
                                 && resumeRequested
                                 && this.launchedSessionMatchesSnapshotResume(launchResult.session.id, resumeType, resumeToken);
-                            // Pod-aware nodes with resume token may either truly resume or
-                            // prove that the saved session is gone and fall back to fresh.
                             if (isPodAware && resumeRequested && startupResult.continuityOutcome === "fresh" && !nativeContinuityProved) {
-                                warnings?.push(`Node ${node.logicalId}: resume was unavailable; launched fresh instead.`);
+                                return {
+                                    nodeId: node.id,
+                                    logicalId: node.logicalId,
+                                    status: "failed",
+                                    error: "Resume attempted but runtime reported fresh continuity. Launch fresh explicitly if that degradation is acceptable.",
+                                };
                             }
                             const finalStatus = (isPodAware && resumeRequested)
                                 ? ((startupResult.continuityOutcome === "resumed" || nativeContinuityProved) ? "resumed" : baseStatus)
                                 : baseStatus;
                             return { nodeId: node.id, logicalId: node.logicalId, status: finalStatus };
                         }
-                        if ((isPodAware && resumeRequested) || needsFreshLaunchFallback) {
+                        // Pod-aware attention_required: hoisted above both the
+                        // resume-requested and non-resume-requested failed branches so
+                        // that pod-aware Codex auth-refusal (verifyResumeLaunch →
+                        // recovery: "attention_required") surfaces honestly regardless
+                        // of whether resume was requested. Mirrors the runtime-agnostic
+                        // legacy mapping at lines 725-735. This MUST come before the
+                        // line 859 / 867 failed branches; otherwise the production
+                        // pod-aware-resume path (most common) returns `status: "failed"`
+                        // and the slice's "attention_required end-to-end" claim breaks.
+                        if (isPodAware && startupResult.startupStatus === "attention_required") {
+                            return {
+                                nodeId: node.id,
+                                logicalId: node.logicalId,
+                                status: "attention_required",
+                                error: `Restore startup requires attention: ${startupResult.errors.join("; ")}`,
+                                attentionEvidence: startupResult.evidence ?? null,
+                            };
+                        }
+                        if (isPodAware && resumeRequested) {
                             return {
                                 nodeId: node.id,
                                 logicalId: node.logicalId,
@@ -463,10 +723,29 @@ export class RestoreOrchestrator {
                                 error: startupResult.errors.join("; "),
                             };
                         }
+                        if (isPodAware) {
+                            const prefix = startupResult.startupStatus === "attention_required"
+                                ? "Restore startup requires attention"
+                                : "Restore startup failed";
+                            return {
+                                nodeId: node.id,
+                                logicalId: node.logicalId,
+                                status: "failed",
+                                error: `${prefix}: ${startupResult.errors.join("; ")}`,
+                            };
+                        }
                         warnings?.push(`Restore startup failed for ${node.logicalId}: ${startupResult.errors.join("; ")}`);
                     }
                     catch (err) {
-                        if ((isPodAware && resumeRequested) || needsFreshLaunchFallback) {
+                        if (isPodAware && resumeRequested) {
+                            return {
+                                nodeId: node.id,
+                                logicalId: node.logicalId,
+                                status: "failed",
+                                error: `Restore startup error: ${err.message}`,
+                            };
+                        }
+                        if (isPodAware) {
                             return {
                                 nodeId: node.id,
                                 logicalId: node.logicalId,
@@ -479,21 +758,8 @@ export class RestoreOrchestrator {
                 }
             }
         }
-        if (needsFreshLaunchFallback) {
-            return {
-                nodeId: node.id,
-                logicalId: node.logicalId,
-                status: "failed",
-                error: "Resume fallback required a fresh launch, but no startup context was available to relaunch the runtime",
-            };
-        }
         return { nodeId: node.id, logicalId: node.logicalId, status: baseStatus };
     }
-    shouldFallbackFreshWithoutResume(runtime, resumeType) {
-        return !!resumeType
-            && ((runtime === "codex" && resumeType.startsWith("codex"))
-                || (runtime === "claude-code" && resumeType.startsWith("claude")));
-    }
     launchedSessionMatchesSnapshotResume(sessionId, resumeType, resumeToken) {
         if (!resumeType || !resumeToken)
             return false;
@@ -506,14 +772,148 @@ export class RestoreOrchestrator {
         if (this.claudeResume.canResume(resumeType, resumeToken)) {
             const result = await this.claudeResume.resume(sessionName, resumeType, resumeToken, cwd);
             if (result.ok)
-                return "resumed";
-            return result.code === "retry_fresh" ? "retry_fresh" : "failed";
+                return { kind: "resumed" };
+            if (result.code === "retry_fresh")
+                return { kind: "retry_fresh" };
+            // L3: surface attention_required from the Claude probe (resume-selection prompt).
+            if (result.code === "attention_required") {
+                return {
+                    kind: "attention_required",
+                    message: result.message,
+                    evidence: result.evidence,
+                };
+            }
+            return { kind: "failed", message: result.message };
         }
         if (this.codexResume.canResume(resumeType, resumeToken)) {
             const result = await this.codexResume.resume(sessionName, resumeType, resumeToken, cwd);
-            return result.ok ? "resumed" : "failed";
+            if (result.ok)
+                return { kind: "resumed" };
+            if (result.code === "retry_fresh")
+                return { kind: "retry_fresh" };
+            // Codex auth-refusal: stored OAuth token can no longer be refreshed.
+            // Recoverable — operator runs `codex login` and the seat continues.
+            // Per-node mapping at lines 725-735 emits `status: "attention_required"`
+            // with `attentionEvidence` for both runtimes; no further wiring needed.
+            if (result.code === "attention_required") {
+                return {
+                    kind: "attention_required",
+                    message: result.message,
+                    evidence: result.evidence,
+                };
+            }
+            return { kind: "failed", message: result.message };
         }
-        return "failed";
+        return { kind: "failed", message: "No resume adapter available for this runtime/token combination." };
+    }
+    /**
+     * L3 Decision 3: runtime-truth reconciliation. Given a node whose original
+     * `restoreOutcome` was `failed` or `attention_required`, examine current
+     * runtime state. If ALL four visible-evidence preconditions hold, append
+     * `restore.outcome_reconciled` so the node's effective post-reconciliation
+     * outcome becomes `operator_recovered`. Never mutates or deletes the
+     * original failure event; never produces `ready`.
+     *
+     * Returns `{ ok: true, attemptId, from, to, evidence }` on upgrade, or
+     * `{ ok: false, code, detail }` describing exactly which precondition
+     * failed (or "no_attempt" / "outcome_not_upgradable" when there is nothing
+     * to reconcile).
+     */
+    async reconcileNodeRuntimeTruth(rigId, nodeId) {
+        // Locate the latest restore attempt for this rig.
+        const startedRow = this.db.prepare("SELECT seq, payload FROM events WHERE rig_id = ? AND type = 'restore.started' ORDER BY seq DESC LIMIT 1").get(rigId);
+        if (!startedRow) {
+            return { ok: false, code: "no_attempt", detail: "No restore.started event recorded for this rig." };
+        }
+        const attemptId = startedRow.seq;
+        // Find the node's most recent post-attempt outcome from the most-recent
+        // restore.completed event for this rig. If the latest outcome is not
+        // failed or attention_required, the reconciler has nothing to upgrade.
+        const completedRow = this.db.prepare("SELECT payload FROM events WHERE rig_id = ? AND type = 'restore.completed' AND seq > ? ORDER BY seq DESC LIMIT 1").get(rigId, attemptId);
+        let nodeStatus = null;
+        let nodeLogicalId = null;
+        if (completedRow) {
+            try {
+                const parsed = JSON.parse(completedRow.payload);
+                const found = parsed.result?.nodes?.find((n) => n.nodeId === nodeId);
+                if (found) {
+                    nodeStatus = found.status;
+                    nodeLogicalId = found.logicalId;
+                }
+            }
+            catch {
+                // payload corruption — treat as no node record found
+            }
+        }
+        if (!nodeStatus) {
+            return { ok: false, code: "node_not_found", detail: `Node ${nodeId} has no record in the latest restore.completed event for rig ${rigId}.` };
+        }
+        if (nodeStatus !== "failed" && nodeStatus !== "attention_required") {
+            return { ok: false, code: "outcome_not_upgradable", detail: `Reconciliation only upgrades failed or attention_required; current outcome is ${nodeStatus}.` };
+        }
+        const fromStatus = nodeStatus;
+        // Resolve canonical session name for this node so we can probe tmux/pane.
+        const bindingRow = this.db.prepare("SELECT tmux_session FROM bindings WHERE node_id = ?").get(nodeId);
+        const sessionName = bindingRow?.tmux_session ?? null;
+        if (!sessionName) {
+            return { ok: false, code: "tmux_session_missing", detail: "No tmux session bound for this node." };
+        }
+        // Precondition #1: tmux session exists.
+        let alive = false;
+        try {
+            alive = await this.tmuxAdapter.hasSession(sessionName);
+        }
+        catch {
+            // L1 fail-closed: ambiguous probe failure stays as not-alive for the
+            // reconciler. Original failure event remains untouched.
+            alive = false;
+        }
+        if (!alive) {
+            return { ok: false, code: "tmux_session_missing", detail: `Tmux session ${sessionName} is not currently alive.` };
+        }
+        // Resolve runtime + capture pane state for preconditions #2 and #4.
+        const nodeRow = this.db.prepare("SELECT runtime FROM nodes WHERE id = ?").get(nodeId);
+        const runtime = nodeRow?.runtime ?? null;
+        const paneCommand = await this.tmuxAdapter.getPaneCommand(sessionName);
+        const paneContent = (await this.tmuxAdapter.capturePaneContent(sessionName, 40)) ?? "";
+        // Precondition #2: foreground process is the runtime (claude or codex).
+        const probe = assessNativeResumeProbe({ runtime, paneCommand, paneContent });
+        const fgProcess = paneCommand && (paneCommand === "claude" || paneCommand.startsWith("codex"))
+            ? (paneCommand === "claude" ? "claude" : "codex")
+            : null;
+        if (!fgProcess) {
+            return { ok: false, code: "fg_process_not_runtime", detail: `Foreground process is "${paneCommand ?? "(unknown)"}", not claude/codex.` };
+        }
+        // Precondition #3: the resume token was actually used at launch.
+        // Stored on the latest session row; null/empty means resume was not
+        // exercised so a "recovered" claim has no basis.
+        const sessRow = this.db.prepare("SELECT resume_token FROM sessions WHERE node_id = ? ORDER BY id DESC LIMIT 1").get(nodeId);
+        const resumeTokenUsed = typeof sessRow?.resume_token === "string" && sessRow.resume_token.length > 0;
+        if (!resumeTokenUsed) {
+            return { ok: false, code: "resume_token_not_used", detail: "No resume token recorded on the latest session row." };
+        }
+        // Precondition #4: pane is at a usable/idle state — explicitly NOT a
+        // resume-selection prompt and not the "returned to shell" failure mode.
+        if (probe.status !== "resumed") {
+            return { ok: false, code: "pane_not_usable", detail: `Pane state is ${probe.status} (${probe.code}); reconciliation requires resumed.` };
+        }
+        // All four preconditions hold. Append (never mutate) the audit event.
+        this.eventBus.emit({
+            type: "restore.outcome_reconciled",
+            rigId,
+            nodeId,
+            attemptId,
+            from: fromStatus,
+            to: "operator_recovered",
+            evidence: { tmux: true, fgProcess, resumeTokenUsed: true, paneState: "usable" },
+        });
+        return {
+            ok: true,
+            attemptId,
+            from: fromStatus,
+            to: "operator_recovered",
+            evidence: { tmux: true, fgProcess, resumeTokenUsed: true, paneState: "usable" },
+        };
     }
     writeCheckpointFile(cwd, checkpoint) {
         try {