omp-conductor 0.19.7 → 0.20.0

package/src/daemon/panes.ts

@@ -0,0 +1,180 @@
+/**
+ * Worker panes: reconciling the terminal representation of a run against the
+ * run rows, and answering who owns a workspace.
+ *
+ * A pane is a *representation* of a run, not the run itself — the session lives
+ * or dies whether or not a pane ever opened. That is exactly why this is its own
+ * module: pane reconciliation is a convergence loop with its own bounded
+ * reattempts, and the dispatch path must be able to launch, settle and hand back
+ * an issue without ever depending on it having converged.
+ *
+ * `workspaceOwnership` sits here because a pane and a run agree on a workspace
+ * or they do not, and all three callers — `handleIssue`, this reconcile, and
+ * `runDaemon`'s startup sweep — must ask the question exactly once, the same way.
+ */
+import { PANE_REATTEMPT_MAX, reconcileWorkerPanes, type WorkspaceOwnership } from "../fleet.ts";
+import { log } from "../log.ts";
+import type { RunRecord } from "../types.ts";
+import type { Deps } from "./deps.ts";
+
+/**
+ * Pane re-establishment attempts spent per run, for this daemon process only
+ * (#998).
+ *
+ * Deliberately in memory rather than on the run row: a restart is exactly the
+ * event that makes another attempt worth making (Herdr came back), so a fresh
+ * process legitimately gets a fresh budget, and giving up never marks a run
+ * permanently unrepresentable. Pruned against the live set on every pass, so it
+ * cannot outgrow the fleet.
+ */
+export const paneAttemptsSpent = new Map<string, number>();
+
+/** The store-backed {@link WorkspaceOwnership} adapter (#1035 review): the
+ *  conductor store is the durable leg of workspace discovery, surviving both
+ *  daemon and Herdr restarts. One shape for every caller so they cannot
+ *  disagree about what ownership reads. */
+export function workspaceOwnership(store: Deps["store"]): WorkspaceOwnership {
+  return {
+    recordedWorkspaces: (project) => store.workerWorkspaceIds(project),
+    rememberWorkspace: (project, workspaceId) => store.rememberWorkerWorkspace(project, workspaceId),
+    forgetWorkspace: (project, workspaceId) => store.forgetWorkerWorkspace(project, workspaceId),
+  };
+}
+
+/** Exported for the lifecycle tests (#1035): the pass is pure over the store
+ *  and the injected Herdr runner, so a test drives it exactly as the tick does. */
+/**
+ * Make the worker workspaces agree with the live run set, once per pass
+ * (#841, reworked by #1035).
+ *
+ * Runs on every dispatch pass rather than only at startup, because the thing it
+ * repairs — a Herdr restart — is not a conductor event and announces itself
+ * nowhere. It is idempotent by construction: a reconciled fleet reports `intact`
+ * for every live worker and finds nothing left to close, so repeated passes
+ * converge instead of accumulating panes or churning them. This pass owns this
+ * project's worker workspace even when no run is live — that is how a settled-out
+ * project's stale shells converge to zero and its empty workspace is removed.
+ *
+ * Nothing here can stop a worker: panes hold followers only; the authoritative
+ * child lives in this daemon's process tree and is never signalled.
+ */
+export function reconcilePanes(
+  d: Pick<Deps, "store" | "herdrRun">,
+  project: string,
+  log: (message: string) => void,
+): void {
+  const live = d.store.liveRuns(project);
+  const liveKeys = new Set(live.map((run) => `${project}\u0000${run.id}`));
+  for (const key of [...paneAttemptsSpent.keys()]) {
+    if (key.startsWith(`${project}\u0000`) && !liveKeys.has(key)) paneAttemptsSpent.delete(key);
+  }
+  const attempts = new Map(
+    live.map((run) => [run.id, paneAttemptsSpent.get(`${project}\u0000${run.id}`) ?? 0]),
+  );
+  const result = reconcileWorkerPanes(
+    live.map((run) => ({
+      runId: run.id,
+      issue: run.issue,
+      attempt: run.attempt,
+      project,
+      ...(run.workerPid === undefined ? {} : { pid: run.workerPid }),
+      ...(run.paneId === undefined ? {} : { paneId: run.paneId }),
+      ...(run.paneLabel === undefined ? {} : { paneLabel: run.paneLabel }),
+      ...(run.sessionFile === undefined ? {} : { sessionFile: run.sessionFile }),
+    })),
+    { ...(d.herdrRun === undefined ? {} : { run: d.herdrRun }), ownership: workspaceOwnership(d.store) },
+    attempts,
+    PANE_REATTEMPT_MAX,
+    [project],
+  );
+  if (!result.ok) {
+    // An unreadable workspace is not evidence that anything is stale, so nothing
+    // is released and no run is relabelled — but a fleet that cannot see its own
+    // panes says so, on every pass, rather than going quiet.
+    log(`worker panes not reconciled: ${result.reason}`);
+    return;
+  }
+  for (const outcome of result.outcomes) {
+    switch (outcome.kind) {
+      case "intact":
+        break;
+      case "visual-unreadable":
+        // The pane keeps standing: an unreadable follower table is no evidence
+        // of death, and a live worker's representation is never destroyed on a
+        // guess (#1035 review). Said once per pass, not acted on.
+        log(
+          `#${runIssue(live, outcome.runId)} herdr pane ${outcome.paneId} visual unreadable, left standing: ${outcome.reason}`,
+        );
+        break;
+      case "reassociated":
+        d.store.updateRun(outcome.runId, {
+          paneId: outcome.paneId,
+          paneLabel: outcome.label,
+          paneUnavailable: null,
+        });
+        log(`#${runIssue(live, outcome.runId)} herdr pane re-associated as ${outcome.paneId} after a restart`);
+        break;
+      case "untracked": {
+        d.store.updateRun(outcome.runId, { paneId: null, paneLabel: null, paneUnavailable: outcome.reason });
+        if (outcome.attempted !== true) {
+          // Cost no attempt (no recorded pid): say so every pass, as before.
+          log(`#${runIssue(live, outcome.runId)} has no herdr pane: ${outcome.reason}`);
+          break;
+        }
+        // #998: a terminal refusing to split will refuse the next one too, and
+        // each retry made the pile worse — spend the whole budget at once. The
+        // phase is structured now (#1035 review): the old substring sniff died
+        // when the reason strings changed and never fired again.
+        const key = `${project}\u0000${outcome.runId}`;
+        const spent = outcome.phase === "split" ? PANE_REATTEMPT_MAX : (paneAttemptsSpent.get(key) ?? 0) + 1;
+        paneAttemptsSpent.set(key, spent);
+        // Once per run, at the moment the budget runs out — not once per pass.
+        // Four identical lines a minute for #986 is what made the real cause
+        // (a leaked pane per attempt) hard to see on 2026-08-23.
+        log(
+          spent >= PANE_REATTEMPT_MAX
+            ? `#${runIssue(live, outcome.runId)} has no herdr pane after ${spent} attempt(s), not retrying: ${outcome.reason}`
+            : `#${runIssue(live, outcome.runId)} has no herdr pane (attempt ${spent}/${PANE_REATTEMPT_MAX}): ${outcome.reason}`,
+        );
+        break;
+      }
+      case "attempts-exhausted":
+        // Already reported when the budget ran out; the run keeps working with
+        // no pane, and a later restart gets a fresh budget.
+        break;
+      case "stale-released": {
+        const what =
+          outcome.cause === "duplicate"
+            ? "closed duplicate herdr pane"
+            : outcome.cause === "unidentified"
+              ? "closed unidentified herdr pane"
+              : "released settled herdr pane";
+        log(`${what} ${outcome.paneId} (run ${outcome.runId || "unidentified"}${outcome.cause === "settled" ? " is not live" : ""})`);
+        break;
+      }
+      case "stale-release-failed": {
+        const what =
+          outcome.cause === "duplicate"
+            ? "duplicate herdr pane"
+            : outcome.cause === "unidentified"
+              ? "unidentified herdr pane"
+              : "settled herdr pane";
+        log(`${what} ${outcome.paneId} could not be closed: ${outcome.reason}`);
+        break;
+      }
+      case "workspace-removed":
+        log(`removed empty herdr worker workspace ${outcome.workspaceId} for ${outcome.project}`);
+        break;
+      case "workspace-remove-failed":
+        log(
+          `herdr worker workspace ${outcome.workspaceId} for ${outcome.project} could not be removed: ${outcome.reason}`,
+        );
+        break;
+    }
+  }
+}
+
+/** The issue a run id belongs to, for a log line a human reads. */
+export function runIssue(live: readonly RunRecord[], runId: string): string {
+  return String(live.find((run) => run.id === runId)?.issue ?? "?");
+}