omp-conductor 0.15.12 → 0.15.13

package/src/daemon.ts

@@ -8,7 +8,7 @@
  * here and enforced before anything is claimed.
  */
 import { createHash } from "node:crypto";
-import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
 import { dirname, join, relative } from "node:path";
 import {
   configPath,
@@ -39,6 +39,10 @@ import { runCommand, verifyPendingUpgrade, type UpgradeVerifyDeps } from "./upgr
 import { fleetLayers } from "./fleet.ts";
 import { startOrchestrator } from "./orchestrator.ts";
 import type { OrchestratorHandle } from "./orchestrator.ts";
+import {
+  formatOrchestratorDown,
+  reconcileOrchestratorDown,
+} from "./orchestrator-down.ts";
 import {
   createReportOutbox,
   enqueueAvailableHeldNotices,
@@ -78,6 +82,7 @@ import type {
   MergedPrInfo,
   OpenCloser,
   ReleaseShape,
+  OrchestratorIncident,
   PrState,
   ProjectConfig,
   ReadyIssue,
@@ -100,6 +105,7 @@ import {
   type WorkerPausePhase,
   type WorkerResult,
   type RunWorkerDeps,
+  ORPHAN_RESUME_PROMPT,
   renderBrief,
   runWorker,
 } from "./worker.ts";
@@ -227,6 +233,13 @@ interface Deps {
   workerDeps?: RunWorkerDeps;
   integrity: IntegrityGate;
   stall: StallGate;
+  /**
+   * The embedded orchestrator session handle when one started; absent when it
+   * failed to start or the project uses an external orchestrator. Feeds the
+   * orchestrator-down reconcile ({@link reconcileOrchestratorDown}) so a
+   * crashed session pages once per incident instead of degrading quietly.
+   */
+  orchestrator?: OrchestratorHandle;
   cleanup?: RetainedCleanupCursor;
   /**
    * Reads the connecting uid off a verb socket (#126). Resolved once at startup
@@ -531,6 +544,19 @@ export function pauseInstance(
   }
 }
 
+/**
+ * The pause sentinel's `source=` token, proven from a verb. The sentinel's
+ * source line is read back as a single `\S+` token (see {@link pauseInstance}
+ * and {@link pauseProvenance}), so a verb that contains a space (`setup host`)
+ * is unrepresentable verbatim and must be encoded before it reaches disk —
+ * otherwise the fence cannot prove its own pause and refuses forever (#552).
+ * Spaces become `-`; the human-readable verb is preserved in the sentinel's
+ * `reason=` instead.
+ */
+export function pauseSourceToken(verb: string): string {
+  return verb.trim().replace(/\s+/g, "-");
+}
+
 export function setPaused(
   v: boolean,
   why?: { source: string; reason?: string },
@@ -1607,6 +1633,50 @@ export async function inheritedPrForContinuation(
   return { prUrl: prior.prUrl, ...(prior.headSha === undefined ? {} : { headSha: prior.headSha }) };
 }
 
+/**
+ * #536: whether an orphan-clean requeue may resume the interrupted session
+ * instead of dispatching fresh.
+ *
+ * `reconcileOrphanedRuns` keeps the worktree (salvage commit included) and the
+ * transcript is file-backed, so a daemon restart can hand the next attempt
+ * back its own memory: the same worktree, the same session directory, and
+ * `resume: true` at the harness. The old dispatch built a fresh
+ * `run-<uuid>` session and re-read the repo from zero — the exact rediscovery
+ * orphan-clean spent turns on in #535.
+ *
+ * Only `orphan-clean` resumes. A cap-killed or otherwise failed worker was
+ * killed for cause, and orphan-dirty is held precisely because the worktree is
+ * the only copy. The checks here are what the daemon can prove cheaply before
+ * the claim (the transcript is present and non-empty, the worktree is
+ * present); the harness's own `continueRecent` is the backstop, and a corrupt
+ * transcript it silently falls back from is surfaced loudly by the
+ * `sessionFile` lineage compare at the dispatch site rather than left quiet.
+ */
+function orphanResumeVerdict(
+  prior: RunRecord | undefined,
+): { kind: "resume"; prior: RunRecord } | { kind: "fresh"; reason?: string } {
+  if (prior === undefined || prior.state !== "orphaned" || prior.failureClass !== "orphan-clean") {
+    return { kind: "fresh" };
+  }
+  if (prior.sessionFile === undefined) {
+    return { kind: "fresh", reason: "the orphaned attempt recorded no transcript" };
+  }
+  try {
+    if (!existsSync(prior.sessionFile)) {
+      return { kind: "fresh", reason: `transcript ${prior.sessionFile} is gone` };
+    }
+    if (statSync(prior.sessionFile).size === 0) {
+      return { kind: "fresh", reason: `transcript ${prior.sessionFile} is empty` };
+    }
+  } catch (err) {
+    return { kind: "fresh", reason: `transcript ${prior.sessionFile} is unreadable (${errText(err)})` };
+  }
+  if (prior.worktree === "" || !existsSync(prior.worktree)) {
+    return { kind: "fresh", reason: `worktree ${prior.worktree} is gone` };
+  }
+  return { kind: "resume", prior };
+}
+
 export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
   const { project, caps, tracker, store } = d;
   const issue = r.issue.number;
@@ -1770,6 +1840,49 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
     // fresh issue always starts on the primary (#286). Read before the claim
     // writes this attempt's row, which would otherwise break the streak.
     const chainFacts = providerFailureFacts(store.runsForIssue(project.name, issue));
+    // Resolved before the claim, not after provisioning: the model is also
+    // part of the #536 resume decision, which has to be made before the
+    // dispatch shape (fresh provision vs kept worktree) is chosen. With no
+    // `modelFallbacks` configured this is the primary model — or none, for an
+    // unconfigured project — and today's dispatch is byte for byte what it
+    // has always been.
+    const chainConfigured = (project.modelFallbacks?.length ?? 0) > 0;
+    const choice = resolveDispatchModel({
+      workerModel: project.workerModel,
+      modelFallbacks: project.modelFallbacks,
+      threshold: project.modelFallbackThreshold ?? DEFAULT_MODEL_FALLBACK_THRESHOLD,
+      streak: chainFacts.streak,
+    });
+    const clause = fallbackClause(choice, chainFacts, project.workerModel);
+
+    // #536: an orphan-clean requeue resumes the interrupted session instead of
+    // re-reading the repo from zero. Decided here, before this attempt's row
+    // exists, because `prior` is still the attempt whose work this one
+    // inherits and the whole dispatch shape follows the verdict.
+    let resuming: RunRecord | undefined;
+    const verdict = orphanResumeVerdict(prior);
+    if (verdict.kind === "resume") {
+      const resumePrior = verdict.prior;
+      // The continuation must stay on the model the interrupted session was
+      // using; a chain that now resolves differently dispatches fresh rather
+      // than quietly continuing on another model, which would smear one
+      // attempt's work across two models (#286 attribution).
+      if (resumePrior.model !== undefined && choice.model !== undefined && resumePrior.model !== choice.model) {
+        log(
+          `#${issue} attempt ${attempt} not resumed: attempt ${resumePrior.attempt} ran on ${resumePrior.model} but dispatch ` +
+            `now resolves ${choice.model} — fresh dispatch`,
+        );
+      } else {
+        resuming = resumePrior;
+        log(
+          `#${issue} attempt ${attempt} continuing session of attempt ${resuming.attempt} → ` +
+            `transcript ${resuming.sessionFile}, worktree kept`,
+        );
+      }
+    } else if (verdict.reason !== undefined) {
+      log(`#${issue} attempt ${attempt} not resumed: ${verdict.reason} — fresh dispatch`);
+    }
+
     run = store.createRun({
       project: project.name,
       issue,
@@ -1822,22 +1935,30 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
 
     // A run's tree is <workspaceRoot>/<issue> and addRunRepo refuses to reuse
     // an existing path, so a retry — or a tree kept from a failed attempt — has
-    // to be cleared first. Both helpers are pure path math and removeWorktree
-    // tolerates a mirror or tree that is not there yet, so this is safe on a
-    // first attempt. addRunRepo does its own ensureMirror; calling it here too
-    // would cost a second network fetch per attempt.
-    await removeWorktree(mirrorPath, worktreePathFor(project.workspaceRoot, issue));
-    if (await settleStopBeforeSession()) return;
-    if (await settleDrainBeforeSession()) return;
-    const provisioned = await addRunRepo(
-      r.repo,
-      project.mirrorRoot,
-      project.workspaceRoot,
-      issue,
-      branch,
-    );
-    worktreePath = provisioned.path;
-    runRepo = { repo: r.repo, runRepoPath: worktreePath, branch };
+    // to be cleared first. Both helpers are pure path math, and removeWorktree
+    // tolerates a mirror or tree that is not there yet. An attempted resume
+    // skips the whole dance: the orphaned run's tree is the work to continue
+    // (its salvage commit is already on the branch), and re-cloning it from
+    // the mirror would be exactly the rediscovery this feature exists to skip.
+    let provisioned: Awaited<ReturnType<typeof addRunRepo>> | undefined;
+    if (resuming !== undefined) {
+      worktreePath = resuming.worktree;
+      runRepo = { repo: r.repo, runRepoPath: worktreePath, branch };
+    } else {
+      await removeWorktree(mirrorPath, worktreePathFor(project.workspaceRoot, issue));
+      if (await settleStopBeforeSession()) return;
+      if (await settleDrainBeforeSession()) return;
+      const provisionedTree = await addRunRepo(
+        r.repo,
+        project.mirrorRoot,
+        project.workspaceRoot,
+        issue,
+        branch,
+      );
+      provisioned = provisionedTree;
+      worktreePath = provisionedTree.path;
+      runRepo = { repo: r.repo, runRepoPath: worktreePath, branch };
+    }
     if (await settleStopBeforeSession()) return;
     if (await settleDrainBeforeSession()) return;
 
@@ -1846,9 +1967,15 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
     // here would put a file that never gets written into an escalation.
     //
     // Per run rather than one shared directory, so one run's transcript cannot
-    // be truncated or replaced by the next.
+    // be truncated or replaced by the next — except for a resumed attempt,
+    // which deliberately reuses the interrupted session's directory so the
+    // SDK's `continueRecent(cwd, dir)` picks up that transcript and keeps
+    // writing it.
     const runTreeRoot = stateDir();
-    const sessionDir = join(runTreeRoot, "sessions", `run-${String(runId)}`);
+    const sessionDir =
+      resuming === undefined
+        ? join(runTreeRoot, "sessions", `run-${String(runId)}`)
+        : dirname(resuming.sessionFile!);
     mkdirSync(sessionDir, { recursive: true });
 
     // ---- the run's mutation channel (#126) -------------------------------
@@ -1884,50 +2011,51 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
     store.updateRun(runId, { worktree: worktreePath, state: "running" });
 
     // Where this attempt goes, and (when the failover fired) the clause that
-    // makes it attributable: with no `modelFallbacks` configured this is the
-    // primary model — or none, for an unconfigured project — and today's
-    // dispatch is byte for byte what it has always been.
-    const chainConfigured = (project.modelFallbacks?.length ?? 0) > 0;
-    const choice = resolveDispatchModel({
-      workerModel: project.workerModel,
-      modelFallbacks: project.modelFallbacks,
-      threshold: project.modelFallbackThreshold ?? DEFAULT_MODEL_FALLBACK_THRESHOLD,
-      streak: chainFacts.streak,
-    });
+    // makes it attributable. `chainConfigured`/`choice`/`clause` were resolved
+    // before the claim — the resume verdict had to be made before the dispatch
+    // shape was chosen (#536) — so only the record write lives here.
     // Recorded before the launch, so even a run killed mid-flight leaves the
     // model it chose on its row. Only a chain-configured project writes the
     // column: absent `modelFallbacks` must preserve today's rows byte for byte.
     if (chainConfigured && choice.model !== undefined) {
       store.updateRun(runId, { model: choice.model });
     }
-    const clause = fallbackClause(choice, chainFacts, project.workerModel);
 
     log(
       `#${issue} attempt ${attempt}${clause === undefined ? "" : ` ${clause}`} → ${r.repo.name} ${branch}` +
-        (provisioned.reattached ? " (continuation: reattached existing branch)" : ""),
+        (provisioned?.reattached ? " (continuation: reattached existing branch)" : ""),
     );
 
-    // The discussion is rendered at dispatch so a worker never depends on a
-    // runtime `gh` read to see the orchestrator's grooming (#517). The read is
-    // best-effort, but its failure is not silent: an unreadable tracker names
-    // itself in the brief's Discussion section instead of reading as "no
-    // comments" — the exact confusion this fix removes.
-    let comments: IssueComment[] | "unread";
-    try {
-      comments = await tracker.listComments(issue);
-    } catch (err) {
-      log(`#${issue} issue comments unreadable at dispatch; the brief will say so: ${errText(err)}`);
-      comments = "unread";
-    }
+    // The continuation notice replaces the brief for a resumed attempt (#536).
+    // The original brief is already in the resumed transcript; re-sending it is
+    // how a resumed worker ends up re-doing the work it just did. Everything
+    // below the brief is fresh-dispatch-only, exactly as today.
+    let brief: string;
+    if (resuming !== undefined) {
+      brief = ORPHAN_RESUME_PROMPT;
+    } else {
+      // The discussion is rendered at dispatch so a worker never depends on a
+      // runtime `gh` read to see the orchestrator's grooming (#517). The read is
+      // best-effort, but its failure is not silent: an unreadable tracker names
+      // itself in the brief's Discussion section instead of reading as "no
+      // comments" — the exact confusion this fix removes.
+      let comments: IssueComment[] | "unread";
+      try {
+        comments = await tracker.listComments(issue);
+      } catch (err) {
+        log(`#${issue} issue comments unreadable at dispatch; the brief will say so: ${errText(err)}`);
+        comments = "unread";
+      }
 
-    const brief = await buildBrief(project, r, branch, worktreePath, {
-      continuation: provisioned.reattached,
-      defaultBranch: r.repo.defaultBranch,
-      ...(provisioned.reattached && priorSalvage !== undefined
-        ? { salvagedSha: priorSalvage }
-        : {}),
-      comments,
-    });
+      brief = await buildBrief(project, r, branch, worktreePath, {
+        continuation: provisioned?.reattached === true,
+        defaultBranch: r.repo.defaultBranch,
+        ...(provisioned?.reattached === true && priorSalvage !== undefined
+          ? { salvagedSha: priorSalvage }
+          : {}),
+        comments,
+      });
+    }
     if (await settleStopBeforeSession()) return;
     if (await settleDrainBeforeSession()) return;
 
@@ -1946,6 +2074,7 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
           workerControl?.install(control);
         },
         sessionDir,
+        ...(resuming === undefined ? {} : { resume: true }),
         // The session's control socket, under the daemon's own state directory —
         // a child process of the daemon reaches it directly.
         socketPath: join(sessionDir, "ipc.sock"),
@@ -1974,7 +2103,21 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
         // ends: `omp-conductor tail` resolves an issue to a file through this row,
         // and a path written at completion is a path nobody can follow live. The
         // completion-time update below writes the same value again, harmlessly.
-        onSessionFile: (f) => store.updateRun(runId, { sessionFile: f }),
+        onSessionFile: (f) => {
+          store.updateRun(runId, { sessionFile: f });
+          // #536: the harness's own `continueRecent` can still fall back to a
+          // blank session (corrupt transcript, nothing to continue) — and a
+          // blank session in the "resumed" worktree is indistinguishable from
+          // today's dispatch unless the downgrade is named. Same-file lineage
+          // is the proof the resume happened: the resumed run must keep
+          // writing the orphaned attempt's transcript.
+          if (resuming !== undefined && f !== resuming.sessionFile) {
+            log(
+              `#${issue} attempt ${attempt} resume fell back to a fresh session: opened ${f} ` +
+                `instead of the orphaned attempt's ${resuming.sessionFile}`,
+            );
+          }
+        },
         // The last fence (#374): every pre-launch settle check above has
         // passed, but the stop can still land while the session socket is
         // binding inside `createSession`. This gate is re-checked there,
@@ -3418,7 +3561,13 @@ export async function admitCandidates(
     if (parent !== undefined) {
       const occupied = occupiedParents.get(parent);
       const blocker = occupied?.get(r.repo.name) ?? occupied?.get("");
-      if (blocker !== undefined) {
+      // The gate serializes siblings under one epic: a held candidate must not
+      // proceed while a *different* child of the parent is occupied. But a
+      // candidate's own worker-free pushed-green row is exactly the work it is
+      // continuing, not a rival — the unblocked continuation of that same
+      // issue must not be rejected by its own occupancy, or the retained
+      // continuation deadlocks forever with the PR open.
+      if (blocker !== undefined && blocker !== issue) {
         hold(issue, "sibling-active");
         log(`#${issue} skipped: sibling #${blocker} in flight under epic #${parent} in ${r.repo.name}`);
         continue;
@@ -3723,6 +3872,18 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
   // failure happened. A pause silences claiming, not the operator's right to
   // know their supervising session stopped reading its queue.
   await watchOrchestrator(d);
+  // The down incident is reconciled the same place and for the same reason: a
+  // session that has actually died is as much the operator's concern as one
+  // that is wedged, and restarting it is the daemon's restart either way. This
+  // is what turns a crashed orchestrator into one page ("down since <t>") plus
+  // a diverting count, instead of a warning only in daemon.log.
+  await reconcileOrchestratorDown({
+    project: d.project,
+    store: d.store,
+    orchestrator: d.orchestrator,
+    escalate: (event) => d.escalate(event),
+    log,
+  });
 
   // Settlement is maintenance, not dispatch. Run it before every gate that can
   // stop claiming — pause, integrity, spend, and capacity — so status converges
@@ -4450,6 +4611,13 @@ export interface StatusSnapshot {
    * see rather than a silent gap.
    */
   labelOps?: { pending: number; oldestAgeMs: number };
+  /**
+   * The orchestrator-down incident, when the embedded orchestrator is down:
+   * mode, since-moment and the tier-1 escalations diverted to issue comments
+   * so far. Absent when the orchestrator is healthy (or external), so recovery
+   * drops the degrade row from `status` (#288).
+   */
+  orchestratorDown?: OrchestratorIncident;
 }
 
 /** Builds a status reading from an already-open store. Long-lived operator
@@ -4469,6 +4637,8 @@ export function statusSnapshotFromStore(
   const dispatch = store.latestDispatch(p.name);
   const labelOpsPending = store.countPendingLabelOps(p.name);
   const oldestLabelOpAt = store.oldestPendingLabelOpAt(p.name);
+  // Read once: `status` renders the degrade row off this while it is down.
+  const orchestratorDown = store.orchestratorIncident(p.name);
   // Read once: the provenance read touches the filesystem, and the renderer
   // should never pay for it twice per status.
   const reason = pauseProvenance(p.name)?.reason;
@@ -4501,6 +4671,7 @@ export function statusSnapshotFromStore(
       ? {}
       : { labelOps: { pending: labelOpsPending, oldestAgeMs: now - oldestLabelOpAt } }),
     baseHealth: store.baseHealth(p.name),
+    ...(orchestratorDown === undefined ? {} : { orchestratorDown }),
   };
 }
 
@@ -4619,6 +4790,7 @@ export function formatStatus(s: StatusSnapshot): string {
     `config    ${s.configPath}`,
     `state     ${s.stateDir}`,
     "",
+    ...(s.orchestratorDown === undefined ? [] : formatOrchestratorDown(s.orchestratorDown)),
     "caps",
     `  workers            ${s.liveWorkers} / ${s.caps.maxConcurrentWorkers}`,
     `  issues today       ${s.runsToday}`,
@@ -5597,6 +5769,8 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
     const { brief, releaseGrants } = orchestratorStandingOrders(project);
     let orchestrator: OrchestratorHandle | undefined;
     let orchestratorVerbs: VerbListener | undefined;
+    /** First start-failure cause, surfaced by the orchestrator-down incident (#288). */
+    let orchestratorStartError: string | undefined;
     if (project.escalation.orchestrator === "external") {
       projectLog(
         "orchestrator: external — tier-1 escalations post as issue comments for the external session's drain duty",
@@ -5636,9 +5810,10 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
         const transcript = orchestrator.sessionFile();
         projectLog(`orchestrator session ready${transcript === undefined ? "" : ` · ${transcript}`}`);
       } catch (err) {
+        orchestratorStartError = errText(err);
         projectLog(
           "WARNING: orchestrator session failed to start; tier-1 escalations will fall back to issue " +
-            `comments: ${errText(err)}`,
+            `comments: ${orchestratorStartError}`,
         );
         await orchestratorVerbs?.close();
         orchestratorVerbs = undefined;
@@ -5655,6 +5830,17 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
       orchestrator,
       Date.now,
       deliveryPolicyValid,
+      (e) => {
+        // Every tier-1 escalation that lands on the issue-comment fallback was
+        // diverted from the orchestrator. Count it durably on the open incident
+        // (a no-op when none is open), so the page and status name how much
+        // the outage diverted (#288).
+        store.bumpOrchestratorDiverted(project.name, 1);
+        projectLog(
+          `orchestrator: tier-1 escalation on ${escalationIssueRef(e.issue)} diverted to issue comments ` +
+            `while the orchestrator was down`,
+        );
+      },
     );
     const outbox = createReportOutbox({
       project: currentProject,
@@ -5685,6 +5871,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
       workerControls,
       integrity,
       stall: { paged: false },
+      ...(orchestrator === undefined ? {} : { orchestrator }),
       cleanup: { next: 0 },
       probeCriticalBase: (repo, markers, branch) =>
         probeCriticalBase(project, repo, branch, markers),
@@ -5692,6 +5879,18 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
       verbActions,
     };
     runtimeDeps = d;
+    // Startup reconciliation: close an incident carried over from a previous
+    // process when the orchestrator is up (one recovery notice), or open one
+    // when it failed to start (one down page). A daemon restarted while still
+    // down rediscovers the open incident and does not re-page it.
+    await reconcileOrchestratorDown({
+      project,
+      store,
+      orchestrator,
+      escalate: (event) => d.escalate(event),
+      ...(orchestratorStartError === undefined ? {} : { startCause: orchestratorStartError }),
+      log: projectLog,
+    });
     runtimes.push({
       d,
       outbox,