@openrig/cli 0.6.9 → 0.6.11

package/daemon/dist/domain/restore-orchestrator.js

@@ -1,11 +1,13 @@
 import { nonInterruptiveNotice, nonInterruptiveSummary } from "../adapters/non-interruptive.js";
-import { readFileSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { NativePermissionStore } from "./native-permission-store.js";
 import { resolvePermissionPolicyAttachment } from "./permission-policy/policy-ref.js";
+import { claudeResumeRotation } from "./session-registry.js";
 import { assessNativeResumeProbe } from "./native-resume-probe.js";
-import { verifyClaudePaneProcess } from "./native-process-lineage.js";
+import { observeClaudeDelivery, observeClaudeResumeLaunch, verifyClaudePaneProcess } from "./native-process-lineage.js";
 import { reanchorBuiltinStartupFile, reanchorShippedProjectionEntry } from "./builtin-startup-files.js";
+import { seatGuidance, restoreMissingGuidance, missingGuidanceBlocks } from "./restore-guidance.js";
 import { AppliedLaunchObservationStore } from "./applied-launch-observation-store.js";
 import { classifyPaneRuntimeMatch } from "./seat-identity-reconciler.js";
 import { isShellForeground } from "./shell-classifier.js";
@@ -19,7 +21,7 @@ const LAUNCH_DEPENDENCY_KINDS = new Set(["delegates_to", "spawned_by"]);
 // in the pure leaf module active-occupant.ts, shared with preview, snapshot
 // usability, and lifecycle projection. Imported and re-exported here so the
 // existing export surface and the execution call sites below are unchanged.
-import { resolveActiveSnapshotSession, activeOccupantAmbiguityError } from "./active-occupant.js";
+import { resolveActiveSnapshotSession, activeOccupantAmbiguityError, absentSeatResumeHistory, absentSeatResumeHistoryError } from "./active-occupant.js";
 export { resolveActiveSnapshotSession } from "./active-occupant.js";
 export function rollupRestoreRigResult(nodes) {
     if (nodes.length === 0)
@@ -30,6 +32,9 @@ export function rollupRestoreRigResult(nodes) {
     const allFailed = nodes.every((node) => node.status === "failed");
     if (allFailed)
         return "failed";
+    // A seat running without part of its guidance keeps its own status; the rig isn't fully restored.
+    if (nodes.some((node) => (node.guidanceGaps?.length ?? 0) > 0))
+        return "partially_restored";
     if (nodes.some((node) => node.status === "fresh" || node.status === "fresh-primed" || node.status === "awaiting-decision" || node.status === "failed" || node.status === "attention_required")) {
         return "partially_restored";
     }
@@ -48,6 +53,30 @@ export const NON_RUNNING_LAUNCH_STATUSES = new Set([
 export function launchStatusIsRunning(status) {
     return !NON_RUNNING_LAUNCH_STATUSES.has(status);
 }
+/** What `rig launch` does for this node from this snapshot without --fresh, read as the restore path reads it: the
+ *  occupant, then the pre-launch stop-and-ask check, then the resume decision (restoreNodeWithCompensation and the
+ *  resume step after it). An exited agent's message offers stop-then-launch as a resume only when this says so. */
+function snapshotLaunchOutcome(data, nodeId) {
+    const occupant = resolveActiveSnapshotSession(data, nodeId);
+    if (occupant.kind === "ambiguous")
+        return { kind: "unrecoverable", why: "can't tell which saved session is this seat's" };
+    const session = occupant.kind === "resolved" ? occupant.session : null;
+    if (!session) {
+        // The restore's own rule for a seat stopped at capture whose earlier occupant left a token.
+        const prior = occupant.kind === "none" ? absentSeatResumeHistory(data.sessions, nodeId) : null;
+        return prior
+            ? { kind: "decision", why: `had no running occupant for this seat when it was taken, but an earlier occupant left a '${prior.resumeType}' resume token` }
+            : { kind: "fresh", why: "saves no session for this seat" };
+    }
+    const policy = session.restorePolicy ?? "resume_if_possible";
+    if (policy !== "resume_if_possible")
+        return { kind: "fresh", why: `saves this seat with restore policy '${policy}'` };
+    if (!session.resumeToken)
+        return { kind: "decision", why: "has no resume token for this seat" };
+    if (!session.resumeType || session.resumeType === "none")
+        return { kind: "fresh", why: "records no resume source for this seat" };
+    return { kind: "resume" };
+}
 export class RestoreOrchestrator {
     db;
     activeRestores = new Set();
@@ -334,11 +363,28 @@ export class RestoreOrchestrator {
                 .filter((s) => s.nodeId === node.id && s.status === "running");
             let isLive = false;
             let isUnknown = false;
+            let idleSession;
             for (const session of sessions) {
                 try {
                     const alive = await this.tmuxAdapter.hasSession(session.sessionName);
                     if (alive) {
+                        // A preserved shell is not a running agent. Reuse the process-lineage
+                        // check's positive idle-shell evidence; opaque wrappers stay untouched.
                         isLive = true;
+                        const binding = this.sessionRegistry.getBindingForNode(node.id);
+                        const pane = binding?.tmuxSession === session.sessionName ? binding.tmuxPane : null;
+                        const runtime = allNodes.find(current => current.id === node.id)?.runtime ?? node.runtime;
+                        // Launch needs consistent positive evidence: both samples idle with the same fingerprint, and the
+                        // bound pane still in this session (a recycled pane id could name another session's shell).
+                        if (pane && ["claude-code", "codex", "pi", "omp"].includes(runtime ?? "")
+                            && isShellForeground(await this.tmuxAdapter.getPaneCommand(pane) ?? "")
+                            && (await this.tmuxAdapter.listPanes(session.sessionName)).some(candidate => candidate.id === pane)
+                            && (await observeClaudeDelivery({ target: pane, tmux: this.tmuxAdapter, listProcesses: this.listProcesses }, { unknownKeepsIdle: false })).state === "idle_shell"
+                            && isShellForeground(await this.tmuxAdapter.getPaneCommand(pane) ?? "")) {
+                            idleSession = session.sessionName;
+                            isLive = false;
+                            continue;
+                        }
                         break;
                     }
                 }
@@ -350,6 +396,26 @@ export class RestoreOrchestrator {
                 alreadyRunning.push({ nodeId: node.id, logicalId: node.logicalId });
                 continue;
             }
+            if (idleSession) {
+                // Keep the pane/history and the existing per-target failure contract.
+                // Choosing a fresh versus resumed conversation belongs to the person. Stopping closes the pane, so
+                // stop-then-launch is offered as a resume only when this snapshot can resume the seat; otherwise the
+                // message says what that launch would do instead. The command pins the snapshot that was checked.
+                const outcome = snapshotLaunchOutcome(snapshot.data, node.id);
+                const launchCommand = `rig launch ${rigId} ${node.logicalId} --snapshot-id ${snapshot.id}`;
+                const relaunch = outcome.kind === "resume"
+                    ? `To resume its conversation: rig seat stop ${idleSession} --reason "<why>", then ${launchCommand} (this closes the pane, and launch resumes from that snapshot's resume token). `
+                    : `Stopping it and launching from snapshot ${snapshot.id} won't resume this conversation: that snapshot ${outcome.why}, so launch would `
+                        + `${outcome.kind === "fresh" ? "start a fresh conversation" : outcome.kind === "decision" ? "start nothing and ask for a decision" : "fail"}. `;
+                launched.push({
+                    nodeId: node.id, logicalId: node.logicalId, status: "attention_required",
+                    error: `Session alive, agent not running for '${node.logicalId}': tmux session '${idleSession}' holds an idle shell, and its pane and history are preserved. `
+                        + `The person chooses how to bring the agent back. ${relaunch}`
+                        + `To start a blank occupant: rig seat launch ${idleSession} --fresh --stop --reason "<why>". `
+                        + `To keep the pane: attach to tmux session '${idleSession}' and restart the agent there by hand.`,
+                });
+                continue;
+            }
             if (isUnknown) {
                 // OPR.0.4.3.28 correction — INVERT the fail-closed-on-unknown launch default. A failed tmux
                 // liveness probe is NOT positive evidence of a live seat (only isLive, above, is). Deny-by-
@@ -574,6 +640,10 @@ export class RestoreOrchestrator {
             node: nodeById.get(id),
         }));
     }
+    /**
+     * Restore one planned seat, checking continuity before launching its occupant.
+     * Preserve prior state so a failed launch or resume can be compensated.
+     */
     async restoreNodeWithCompensation(entry, rigId, snapshotId, data, opts, warnings) {
         const node = entry.node;
         const nodeId = node.id;
@@ -582,7 +652,7 @@ export class RestoreOrchestrator {
         // a pod node in 'restoring' otherwise returns a silent status "fresh"
         // (classified running downstream) while the occupant truth is ambiguous,
         // bypassing A1's loud-failure semantics. Resolved/none/legacy states
-        // fall through and keep the restoring/degraded behavior unchanged.
+        // fall through to absent-history classification and the continuity check.
         const occupantResolution = resolveActiveSnapshotSession(data, nodeId);
         if (occupantResolution.kind === "ambiguous") {
             return {
@@ -592,6 +662,15 @@ export class RestoreOrchestrator {
                 error: activeOccupantAmbiguityError(occupantResolution.candidateIds, occupantResolution.detail),
             };
         }
+        // Classify absent-seat history before the continuity short-circuit: a
+        // restoring seat must not report fresh while its forecast awaits a decision.
+        const freshRequested = opts?.freshLogicalIds?.includes(node.logicalId) ?? false;
+        const priorWithToken = occupantResolution.kind === "none" && !freshRequested
+            ? absentSeatResumeHistory(data.sessions, nodeId)
+            : null;
+        if (priorWithToken) {
+            return { nodeId, logicalId: node.logicalId, status: "awaiting-decision", error: absentSeatResumeHistoryError(node.logicalId, priorWithToken.resumeType, this.rigRepo.getRig(rigId)?.rig.name ?? data.rig.name) };
+        }
         // Consult live continuity state BEFORE clearing stale state
         if (node.podId) {
             const continuityRow = this.db.prepare("SELECT status FROM continuity_state WHERE pod_id = ? AND node_id = ?").get(node.podId, nodeId);
@@ -623,7 +702,6 @@ export class RestoreOrchestrator {
             // returned there, so only resolved/none reach here.
             const snapSession = occupantResolution.kind === "resolved" ? occupantResolution.session : null;
             const policy = snapSession?.restorePolicy ?? "resume_if_possible";
-            const freshRequested = opts?.freshLogicalIds?.includes(node.logicalId) ?? false;
             const resumeSourceRecorded = !!snapSession?.resumeType && snapSession.resumeType !== "none";
             if (policy === "resume_if_possible" && snapSession && !snapSession.resumeToken && !freshRequested) {
                 const sourceNote = resumeSourceRecorded
@@ -724,7 +802,39 @@ export class RestoreOrchestrator {
         if (launchResult.warnings?.length) {
             warnings?.push(...launchResult.warnings);
         }
-        return this.postLaunchRestore(entry, rigId, data, launchResult.sessionName, launchResult, opts, warnings, priorState);
+        const restored = await this.postLaunchRestore(entry, rigId, data, launchResult.sessionName, launchResult, opts, warnings, priorState);
+        return this.checkRestoredGuidance(restored, entry.node, rigId, data, warnings);
+    }
+    /** A seat that came back running should also have its role, culture and SOP guidance. If its
+     *  guidance file still lacks a managed block it should hold, the seat keeps its true status (a
+     *  live resumed session is still running) and the gap is disclosed on the node and in the
+     *  warnings, so the rig is reported partially_restored, not fully_restored. */
+    checkRestoredGuidance(result, node, rigId, data, warnings) {
+        // Every status with a live session; attention_required keeps its status and gains the disclosure.
+        if (!["resumed", "fresh-primed", "fresh", "rebuilt", "attention_required"].includes(result.status))
+            return result;
+        let guidance;
+        try {
+            guidance = seatGuidance(data.nodeStartupContext?.[node.id] ?? null, node.cwd, this.rigRepo.getRigClaudeManagedBlockFile(rigId), existsSync, (path) => readFileSync(path, "utf-8"));
+        }
+        catch (err) {
+            warnings?.push(`Restore: ${node.logicalId}: could not check its guidance: ${err.message}`);
+            return { ...result, guidanceGaps: ["(guidance check failed)"] };
+        }
+        if (!guidance || guidance.items.length === 0)
+            return result;
+        let missing;
+        try {
+            missing = missingGuidanceBlocks(guidance, { exists: existsSync, readFile: (path) => readFileSync(path, "utf-8") });
+        }
+        catch (err) {
+            missing = [`(could not read ${guidance.targetPath}: ${err.message})`];
+        }
+        if (missing.length === 0)
+            return result;
+        const gap = `${node.logicalId} is running without part of its guidance: ${guidance.targetPath} lacks the OpenRig blocks ${missing.join(", ")}.`;
+        warnings?.push(`Restore: ${gap}`);
+        return { ...result, guidanceGaps: missing };
     }
     /** OPR.0.3.4.2 (B) — roll a just-launched session back to ZERO sessions for
      *  the awaiting-decision outcome: kill the live blank session, mark its row
@@ -804,6 +914,34 @@ export class RestoreOrchestrator {
                     warnings?.push(`Restore activity hooks: ${error.message}`);
                 }
         }
+        // A narrow exception to the replay containment below: before either native resume path
+        // starts the harness, put back only the MISSING managed guidance blocks from the saved
+        // startup selection. Ordinary `rig down` strips them; without this the resumed
+        // seat came back with no role, culture or SOP. Existing blocks are never refreshed; nothing
+        // reaches the conversation; no action, skill or plugin runs. Never blocks the resume.
+        if (resumeRequested && resumeToken && launchResult) {
+            let guidance = null;
+            try {
+                guidance = seatGuidance(startupCtx, node.cwd, this.rigRepo.getRigClaudeManagedBlockFile(rigId), existsSync, (path) => readFileSync(path, "utf-8"));
+            }
+            catch (err) {
+                // Guidance never blocks the exact resume; the post-launch check discloses the gap.
+                warnings?.push(`Restore guidance: ${node.logicalId}: could not read its saved guidance selection: ${err.message}`);
+            }
+            if (guidance && guidance.items.length > 0) {
+                const repair = restoreMissingGuidance(guidance, {
+                    exists: existsSync,
+                    readFile: (path) => readFileSync(path, "utf-8"),
+                    writeFile: (path, content) => writeFileSync(path, content, "utf-8"),
+                    mkdirp: (path) => mkdirSync(path, { recursive: true }),
+                });
+                if (repair.restored.length > 0) {
+                    warnings?.push(`Restore guidance: ${node.logicalId}: put back missing blocks ${repair.restored.join(", ")} in ${guidance.targetPath} from their current source files.`);
+                }
+                for (const gap of repair.gaps)
+                    warnings?.push(`Restore guidance: ${node.logicalId}: ${gap}`);
+            }
+        }
         if (resumeRequested && !isPodAware) {
             // Legacy resume path
             if (!resumeToken) {
@@ -815,9 +953,28 @@ export class RestoreOrchestrator {
                 return { nodeId: node.id, logicalId: node.logicalId, status: "awaiting-decision", error: `Original session unresumable: resume requested but no token available. No session is running. Re-run with --fresh ${node.logicalId} for a deliberate fresh-primed seat, or restore the original session manually.` };
             }
             else {
+                const launchedSessionId = launchResult?.session.id;
+                if (launchedSessionId && this.claudeResume.canResume(resumeType, resumeToken)) {
+                    // Before the resume: Claude's SessionStart hook can land before attemptResume returns.
+                    try {
+                        this.sessionRegistry.recordResumeLaunch(launchedSessionId, resumeToken);
+                    }
+                    catch { /* best-effort: without it a rotation stays unproved, as before */ }
+                }
                 const resumeOutcome = await this.attemptResume(node.id, sessionName, resumeType, resumeToken, node.cwd ?? "/", node.codexConfigProfile, node.model, this.resolveRestorePosture(node.id, rigId), node.effort, warnings);
                 if (resumeOutcome.kind === "resumed") {
                     baseStatus = "resumed";
+                    if (launchedSessionId && this.claudeResume.canResume(resumeType, resumeToken)) {
+                        // The process this resume started, as the launch itself observes it (#1077): the one its
+                        // identity check proved, else one observed now.
+                        try {
+                            const launched = resumeOutcome.launchedProcess ?? await observeClaudeResumeLaunch({ target: sessionName, tmux: this.tmuxAdapter,
+                                ...(this.listProcesses ? { listProcesses: this.listProcesses } : {}), token: resumeToken });
+                            if (launched)
+                                this.sessionRegistry.recordResumeLaunchProcess(launchedSessionId, resumeToken, launched);
+                        }
+                        catch { /* best-effort: without it a rotation stays unproved, as before */ }
+                    }
                 }
                 else if (resumeOutcome.kind === "attention_required") {
                     // L3 Decision 2: Claude resume-selection prompt -> attention_required.
@@ -896,6 +1053,9 @@ export class RestoreOrchestrator {
         // explicit+versioned+durable+idempotent contract in the D4 operation-id
         // phase, where its durability primitives live. Deliberate fresh-primed
         // launches are new histories and keep their replay.
+        // One narrow exception: MISSING managed guidance blocks are put back before the harness
+        // starts (see restoreMissingGuidance above); existing blocks are still never rewritten, and
+        // nothing else is replayed.
         const replayContained = resumeRequested && !!resumeToken;
         const startupRuntime = startupCtx?.runtime ?? node.runtime ?? null;
         const startupAdapter = startupRuntime ? opts?.adapters?.[startupRuntime] : undefined;
@@ -980,7 +1140,8 @@ export class RestoreOrchestrator {
                     };
                     try {
                         const { StartupOrchestrator } = await import("./startup-orchestrator.js");
-                        const startupOrch = new StartupOrchestrator({ db: this.db, sessionRegistry: this.sessionRegistry, eventBus: this.eventBus, tmuxAdapter: this.tmuxAdapter });
+                        const startupOrch = new StartupOrchestrator({ db: this.db, sessionRegistry: this.sessionRegistry, eventBus: this.eventBus, tmuxAdapter: this.tmuxAdapter,
+                            ...(this.listProcesses ? { listProcesses: this.listProcesses } : {}) });
                         const replayAsRestore = baseStatus !== "fresh-primed";
                         const shouldLaunchHarness = isPodAware;
                         const startupResult = await startupOrch.startNode({
@@ -1048,6 +1209,18 @@ export class RestoreOrchestrator {
                             };
                         }
                         if (isPodAware && resumeRequested) {
+                            // Only the runtime's own report that the conversation doesn't exist rolls the seat back
+                            // as unresumable. A pane that went back to its shell for another reason stays failed,
+                            // with its pane and binding kept, so its output can still be read.
+                            if (startupResult.recovery === "retry_fresh" && startupResult.missingConversation) {
+                                await this.rollbackToZeroSession(node.id, sessionName, launchResult?.session.id, priorState);
+                                return {
+                                    nodeId: node.id,
+                                    logicalId: node.logicalId,
+                                    status: "awaiting-decision",
+                                    error: `Original session unresumable: ${startupResult.errors.join("; ")}. The blank session was rolled back; no session is running. Re-run with --fresh ${node.logicalId} to deliberately start a fresh-primed seat, or restore the original session manually.`,
+                                };
+                            }
                             return {
                                 nodeId: node.id,
                                 logicalId: node.logicalId,
@@ -1140,7 +1313,10 @@ export class RestoreOrchestrator {
             // join; a hook/operator may have changed it meanwhile. Never backfill here.
             const retained = sameSession && (managedClaudeResume
                 ? this.sessionRegistry.resumeTokenMatches(sessionId, "claude_id", resumeToken)
+                    // Claude continued the resumed conversation under a new id (its hook said so).
+                    || this.sessionRegistry.claudeResumeRotatedFrom(sessionId, resumeToken)
                 : current.resume_token === resumeToken
+                    || (node.runtime === "claude-code" && this.sessionRegistry.claudeResumeRotatedFrom(sessionId, resumeToken))
                     || (!current.resume_token && this.sessionRegistry.updateResumeToken(sessionId, node.runtime === "codex" ? "codex_id" : "claude_id", resumeToken, "scrape")));
             if (!retained) {
                 markManagedResumeAttention();
@@ -1272,7 +1448,7 @@ export class RestoreOrchestrator {
                     warnings?.push(`${sessionName}: ${notice}`);
                 if (result.appliedLaunch && launchGeneration)
                     this.appliedLaunchStore.recordGeneration(launchGeneration, result.appliedLaunch);
-                return { kind: "resumed" };
+                return { kind: "resumed", ...(result.launchedProcess ? { launchedProcess: result.launchedProcess } : {}) };
             }
             if (result.code === "retry_fresh")
                 return { kind: "retry_fresh" };
@@ -1412,11 +1588,15 @@ export class RestoreOrchestrator {
         if (!sessionName) {
             return { ok: false, code: "tmux_session_missing", detail: "No tmux session bound for this node." };
         }
-        const sessRow = this.db.prepare("SELECT session_name, resume_token FROM sessions WHERE node_id = ? ORDER BY created_at DESC, id DESC LIMIT 1").get(nodeId);
+        const sessRow = this.db.prepare("SELECT session_name, resume_type, resume_token, resume_provenance, resume_rotated_from, resume_rotated_process, resume_launch_process FROM sessions WHERE node_id = ? ORDER BY created_at DESC, id DESC LIMIT 1").get(nodeId);
         if (!sessRow || sessRow.session_name !== sessionName) {
             return { ok: false, code: "binding_mismatch", detail: `Canonical binding ${sessionName} does not match the latest session row.` };
         }
         const expectedResumeToken = sessRow.resume_token;
+        // The one alternative argv may carry: the token OpenRig launched this row to resume, when the
+        // first hook after that launch recorded Claude continuing it as the stored token (#1077), and
+        // only in the process that sent that hook.
+        const rotation = sessRow.resume_type === "claude_id" ? claudeResumeRotation(sessRow) : null;
         if (!expectedResumeToken) {
             return { ok: false, code: "resume_token_not_used", detail: "No resume token recorded on the latest session row." };
         }
@@ -1437,6 +1617,7 @@ export class RestoreOrchestrator {
         // basename alone is never sufficient for no-input recovery.
         const nodeRow = this.db.prepare("SELECT runtime FROM nodes WHERE id = ?").get(nodeId);
         const runtime = nodeRow?.runtime ?? null;
+        // One exact-lineage check per process, accepting the stored token or the recorded rotation.
         const identity = await rebindAndVerifyPaneIdentity({
             db: this.db,
             sessionRegistry: this.sessionRegistry,
@@ -1446,6 +1627,7 @@ export class RestoreOrchestrator {
             runtime,
             expectedResumeToken,
             requireExactResumeLineage: true,
+            resumeRotation: rotation,
             ...(this.listProcesses ? { listProcesses: this.listProcesses } : {}),
         });
         if (!identity.ok) {
@@ -1457,6 +1639,7 @@ export class RestoreOrchestrator {
             target: identity.pane,
             tmux: this.tmuxAdapter,
             expectedToken: expectedResumeToken,
+            rotation,
             requireResume: true,
             ...(this.listProcesses ? { listProcesses: this.listProcesses } : {}),
         });