catherd-cli 1.3.0 → 1.5.0

package/src/services/notifier.ts

@@ -1,34 +1,40 @@
 import { existsSync, readFileSync } from "node:fs";
 import { join } from "node:path";
-import { errorMessage } from "../domain/errors.ts";
+import { adapterFor } from "../adapters/registry.ts";
+import "../adapters/all.ts";
+import { CatherdError, errorMessage } from "../domain/errors.ts";
 import { envelope, formatNotices, type Notice, type NoticePriority, priorityOf } from "../domain/notice.ts";
+import { sessionKey, type HostSessionRef } from "../domain/host.ts";
+import { sendToCodexQueue, type QueueSendResult } from "../infra/codex-queue.ts";
+import {
+  deliveryState,
+  readDelivery,
+  ROLE_THREAD_REFUSAL,
+  writeDeliveryAttempt,
+  type DeliveryAttempt,
+} from "../infra/delivery.ts";
 import type { RunRecord } from "../domain/record.ts";
 import { awaitsCollect, dispatchPaths } from "../infra/dispatch-dir.ts";
 import { tryLock } from "../infra/filelock.ts";
 import { log } from "../infra/log.ts";
 import { type SendResult, sendToInbox } from "../infra/peer-inbox.ts";
-import { writeJsonAtomic } from "../infra/store.ts";
-import { type Settled, settledHooks, type Stalled, stallHooks } from "./dispatch-service.ts";
-import { type Dispatch, listDispatches, readFailover } from "./dispatches.ts";
+import { nonBlankLines, writeJsonAtomic } from "../infra/store.ts";
+import {
+  readStallQuietMs,
+  type Settled,
+  settledHooks,
+  type Stalled,
+  stallHooks,
+} from "./dispatch-service.ts";
+import { type Dispatch, listDispatches, readFailover, roleThreadOf } from "./dispatches.ts";
 import type { Deps } from "./ports.ts";
 import { listRuns, readRecords, type Run } from "./run-store.ts";
-import { currentSession, ownsRun } from "./sessions.ts";
-
-/**
- * Spec §3.4: runs only inside the MCP server, the session's child and so its only sender (§3.2). For every run this
- * session owns, a dispatch that is settled with its record still unread is announced to the session's peer inbox;
- * notices that arrive within the window of each other go as one message; a message that went out is written down
- * (`notified.json`), so a restart never sends it twice, and claimed while it goes, so two servers of one session never
- * both send it. Ownership is asked again under the claim and once more after the message went: a run another session
- * claimed meanwhile is left unmarked, so its new owner announces it too (a copy to the session that gave the run up is
- * acceptable, a notice lost to the owner is not); a notice whose claim another server holds is tried again after the
- * window until it is sent, read, marked or no longer this session's. Disk stays the truth: a notice that cannot be
- * sent is dropped (its claim freed), and the record waits, unread, for `result` or `peek`.
- */
+import { currentSession, ownsRun, readSessionRows, runOwner } from "./sessions.ts";
 
 export interface NotifierOptions {
   /** notices this close to each other go as one message (spec: 3 s); tests shorten it */
   coalesceMs?: number;
+  sendCodex?: (target: HostSessionRef, content: string) => Promise<QueueSendResult>;
   /** the sender; tests may replace it */
   send?: (
     target: { socketPath: string | null; token: string | null },
@@ -50,7 +56,40 @@ export interface Notifier {
   stop(): void;
 }
 
-const notified = (dir: string): boolean => existsSync(dispatchPaths(dir).notified);
+function legacyNotified(q: Queued, target: HostSessionRef): boolean {
+  if (target.host !== "claude-code") return false;
+  try {
+    const marker = JSON.parse(readFileSync(q.mark, "utf8")) as {
+      schema?: unknown;
+      at?: unknown;
+      msgId?: unknown;
+    };
+    if (marker.schema !== 1 || typeof marker.at !== "string" || typeof marker.msgId !== "string")
+      return false;
+    const at = Date.parse(marker.at);
+    if (!Number.isFinite(at)) return false;
+    const historical = readSessionRows(q.run)
+      .filter((row) => Date.parse(row.at) <= at)
+      .at(-1);
+    const owner = runOwner(q.run);
+    const attributed = historical ?? (owner && Date.parse(owner.since) <= at ? owner : null);
+    return attributed !== null && sessionKey(attributed) === sessionKey(target);
+  } catch {
+    return false;
+  }
+}
+
+/** A "submitting" claim left by a sender that died is ambiguous: retrying it may duplicate input. Under the notify lock. */
+export function recoverSubmission(dir: string, eventId: string): void {
+  for (const a of readDelivery(dir)) {
+    if (a.status === "submitting" && a.eventIds.includes(eventId))
+      writeDeliveryAttempt(dir, {
+        ...a,
+        status: "ambiguous",
+        reason: "Sender ended without a verified receipt; retry may duplicate input. Use peek/result.",
+      });
+  }
+}
 
 function replyOf(run: Run, r: RunRecord): string {
   try {
@@ -77,12 +116,14 @@ export function finishedNotice(run: Run, d: Dispatch, r: RunRecord): Notice {
   const urgent = r.status === "limit" || r.replyStatus === "blocked" || r.replyStatus === "refused";
   return {
     kind: "finished",
+    eventId: JSON.stringify([run.id, r.dispatchId, "finished"]),
     runId: run.id,
     runTitle: run.meta.title,
     dispatchId: r.dispatchId,
     name: r.name,
     role: r.role,
     lane: r.lane,
+    thread: r.thread,
     rung: r.rung,
     status,
     replyStatus: r.replyStatus,
@@ -93,16 +134,33 @@ export function finishedNotice(run: Run, d: Dispatch, r: RunRecord): Notice {
   };
 }
 
+/** A live fresh role's thread, as its CLI named it early in its stream; null before it did. */
+function streamThread(d: Dispatch): string | null {
+  const a = adapterFor(d.admit.backend);
+  if (!a) return null;
+  for (const line of nonBlankLines(dispatchPaths(d.dir).events)) {
+    try {
+      const t = a.parse(line).thread;
+      if (t) return t;
+    } catch {
+      // a line the adapter cannot read names no thread
+    }
+  }
+  return null;
+}
+
 /** Spec §3.6: a live role that went quiet, once, at `next`. */
 export function stalledNotice(run: Run, d: Dispatch, quietMs: number, now: number): Notice {
   return {
     kind: "stalled",
+    eventId: JSON.stringify([run.id, d.admit.dispatchId, "stalled"]),
     runId: run.id,
     runTitle: run.meta.title,
     dispatchId: d.admit.dispatchId,
     name: d.admit.name,
     role: d.admit.role,
     lane: d.admit.lane,
+    thread: d.admit.thread ?? streamThread(d),
     rung: d.admit.rung,
     status: `stalled: no output for ${Math.max(1, Math.round(quietMs / 60_000))} min`,
     replyStatus: null,
@@ -119,16 +177,32 @@ interface Queued {
   /** the run it is about: its owner may change before the message goes */
   run: Run;
   notice: Notice;
+  dir: string;
   /** the mark a message that went out leaves */
   mark: string;
   /** whether it is still news when the message goes */
   due: () => boolean;
+  /** how many passes could not write its delivery claim */
+  claimFailures?: number;
+  /** how many sends the transport provably did not submit */
+  sendFailures?: number;
 }
 
+/** Passes a notice whose delivery claim cannot be written is kept for before it is given up (and logged). */
+const MAX_CLAIM_FAILURES = 5;
+
+/** Sends a notice the transport provably did not submit is tried, a coalescing window apart, before peek/result. */
+const MAX_SEND_FAILURES = 5;
+
 /** Starts the notifier for this process's session and hooks it to every settled dispatch. */
 export function startNotifier(deps: Deps, o: NotifierOptions = {}): Notifier {
+  return notifierFor(deps, o);
+}
+
+function notifierFor(deps: Deps, o: NotifierOptions, retryEvent?: string): Notifier {
   const window = o.coalesceMs ?? 3_000;
   const send = o.send ?? sendToInbox;
+  const sendCodex = o.sendCodex ?? ((target, content) => sendToCodexQueue(target, content, process.env));
   const queue = new Map<string, Queued>();
   let timer: ReturnType<typeof setTimeout> | null = null;
   let firstAt = 0;
@@ -158,89 +232,252 @@ export function startNotifier(deps: Deps, o: NotifierOptions = {}): Notifier {
     timer = setTimeout(flush, Math.max(0, Math.min(window, firstAt + 5 * window - now)));
   };
 
+  const pending = (q: Queued, target: HostSessionRef): boolean => {
+    if (!q.due() || legacyNotified(q, target)) return false;
+    const state = deliveryState(q.dir, target, q.notice.eventId);
+    return state === "pending" || (q.notice.eventId === retryEvent && state === "ambiguous");
+  };
+
   async function deliver(batch: Queued[]): Promise<void> {
-    if (!deps.session) return;
-    // Each notice is claimed before it goes (an exclusive lock on its mark, freed if its holder dies): two servers
-    // of one session, a replacement starting while the old one still watches, never both send it. Whether it is
-    // still news is asked under the claim: the other may have announced it just before letting go.
+    const target = currentSession(deps);
+    if (stopped || !target || (target.host === "claude-code" && !deps.session)) return;
     const claims: (() => void)[] = [];
     const due: Queued[] = [];
     const busy: Queued[] = [];
     try {
       for (const q of batch) {
-        // read (or announced) meanwhile, or the run moved to another session (whose server tells it)
-        if (!q.due() || !owned(q.run)) continue;
-        const release = tryLock(q.mark);
-        // another server is announcing it: asked again after the window, since that server may leave it unmarked
-        // (the run moved to this session while its message was on its way)
-        if (!release) {
-          busy.push(q);
-          continue;
+        // one notice's damaged evidence drops that notice alone, never the rest of the batch
+        try {
+          if (!q.due() || !owned(q.run)) continue;
+          const release = tryLock(q.mark);
+          if (!release) {
+            busy.push(q);
+            continue;
+          }
+          claims.push(release);
+          const live = currentSession(deps);
+          if (stopped || !live || sessionKey(live) !== sessionKey(target) || !owned(q.run)) continue;
+          recoverSubmission(q.dir, q.notice.eventId);
+          if (pending(q, target)) due.push(q);
+        } catch (e) {
+          log("warn", "notify", { error: errorMessage(e), dispatch: q.notice.dispatchId });
         }
-        claims.push(release);
-        // asked again under the claim: the run may have moved, or the notice gone out, while it was taken
-        if (q.due() && owned(q.run)) due.push(q);
       }
-      if (due.length === 0) return;
-      const notices = due.map((q) => q.notice);
-      const r = await send(
-        { socketPath: deps.session.socketPath, token: deps.session.token },
-        envelope(formatNotices(notices)),
-        priorityOf(notices),
-      ).catch((e: unknown): SendResult => ({ outcome: "error", reason: errorMessage(e) }));
-      if (r.outcome !== "sent") {
-        log("warn", "notify", {
-          outcome: r.outcome,
-          reason: r.reason,
-          dispatches: notices.map((n) => n.dispatchId),
+      // spec 1.5 plan 21: a notice never goes to a role's own thread (a `codex exec` thread exits after its turn,
+      // so what is queued there is lost). It fails loudly, and `status` says so, instead of enqueue-accepted.
+      const roles = new Map(due.map((q) => [q, roleThreadOf(q.run, target.sessionId)]));
+      for (const [q, role] of roles) {
+        if (role === null) continue;
+        due.splice(due.indexOf(q), 1);
+        const reason = `${ROLE_THREAD_REFUSAL} (${role}, ${target.host} ${target.sessionId}), not the orchestrator's; peek(run) from the orchestrator's session takes the run back`;
+        log("error", "notify", {
+          run: q.run.id,
+          dispatch: q.notice.dispatchId,
+          target: target.sessionId,
+          reason,
         });
-        return;
+        try {
+          writeDeliveryAttempt(q.dir, {
+            attemptId: crypto.randomUUID(),
+            target: {
+              host: target.host,
+              sessionId: target.sessionId,
+              hostSessionId: target.hostSessionId,
+              name: target.name,
+            },
+            eventIds: [q.notice.eventId],
+            at: new Date(deps.now()).toISOString(),
+            status: "failed",
+            msgId: null,
+            reason,
+          });
+        } catch (e) {
+          log("warn", "notify", { error: errorMessage(e), dispatch: q.notice.dispatchId });
+        }
       }
-      const at = new Date(deps.now()).toISOString();
-      // a run that moved while the message was on its way is left unmarked: its new owner announces it too
-      const marked = due.filter((q) => owned(q.run));
-      for (const q of marked) writeJsonAtomic(q.mark, { schema: 1, msgId: r.msgId, at });
-      const unmarked = due.filter((q) => !marked.includes(q)).map((q) => q.notice.dispatchId);
-      log("info", "notify", { msgId: r.msgId, dispatches: notices.map((n) => n.dispatchId), unmarked });
+      // nothing is sent unless every notice's claim is on disk. A claim that cannot be written rolls the batch's
+      // written claims back to failed (provably not submitted, so still deliverable), keeps that notice for a
+      // later pass, and claims the rest again at once: one bad file never holds back the others
+      let attempt: DeliveryAttempt;
+      for (;;) {
+        if (!due.length) return;
+        attempt = {
+          attemptId: crypto.randomUUID(),
+          target: {
+            host: target.host,
+            sessionId: target.sessionId,
+            hostSessionId: target.hostSessionId,
+            name: target.name,
+          },
+          eventIds: due.map((q) => q.notice.eventId),
+          at: new Date(deps.now()).toISOString(),
+          status: "submitting",
+          msgId: null,
+          reason: null,
+        };
+        const claimed: Queued[] = [];
+        const broken = due.find((q) => {
+          try {
+            writeDeliveryAttempt(q.dir, attempt);
+            claimed.push(q);
+            return false;
+          } catch (e) {
+            log("warn", "notify", { error: errorMessage(e), dispatch: q.notice.dispatchId });
+            return true;
+          }
+        });
+        if (!broken) break;
+        const undone: DeliveryAttempt = {
+          ...attempt,
+          status: "failed",
+          reason: "A claim in this batch was not persisted; nothing was submitted.",
+        };
+        for (const c of claimed)
+          try {
+            writeDeliveryAttempt(c.dir, undone);
+          } catch (u) {
+            log("warn", "notify", { error: errorMessage(u), dispatch: c.notice.dispatchId });
+          }
+        // a claim that fails again and again is a damaged file, not a passing fault: it is given up then
+        broken.claimFailures = (broken.claimFailures ?? 0) + 1;
+        if (broken.claimFailures < MAX_CLAIM_FAILURES) busy.push(broken);
+        due.splice(due.indexOf(broken), 1);
+      }
+      const notices = due.map((q) => q.notice);
+      let outcome: QueueSendResult;
+      const content = envelope(formatNotices(notices));
+      try {
+        if (target.host === "codex") outcome = await sendCodex(target, content);
+        else {
+          const r = await send(
+            { socketPath: deps.session!.socketPath, token: deps.session!.token },
+            content,
+            priorityOf(notices),
+          );
+          outcome =
+            r.outcome === "sent" && r.msgId
+              ? { outcome: "accepted", msgId: r.msgId }
+              : {
+                  // an error before any frame was written (no msg_id) provably submitted nothing: retried later
+                  outcome:
+                    r.outcome === "no-session" ||
+                    r.outcome === "refused" ||
+                    (r.outcome === "error" && !r.msgId)
+                      ? "not-submitted"
+                      : "ambiguous",
+                  reason: "Claude inbox has no verified receipt; use peek/result.",
+                };
+        }
+      } catch {
+        outcome = {
+          outcome: "ambiguous",
+          reason: "Sender ended without a verified receipt; retry may duplicate input. Use peek/result.",
+        };
+      }
+      const receipt: DeliveryAttempt = {
+        ...attempt,
+        status:
+          outcome.outcome === "accepted"
+            ? "accepted"
+            : outcome.outcome === "not-submitted"
+              ? "failed"
+              : "ambiguous",
+        msgId: outcome.outcome === "accepted" ? outcome.msgId : null,
+        reason:
+          outcome.outcome === "accepted"
+            ? null
+            : outcome.outcome === "not-submitted"
+              ? "Transport did not submit input; check native transport and use peek/result."
+              : "No verified receipt; retry may duplicate input. Use peek/result.",
+      };
+      // the outcome is known for every notice: one damaged delivery file never discards it for the others
+      for (const q of due)
+        try {
+          writeDeliveryAttempt(q.dir, receipt);
+        } catch (e) {
+          log("warn", "notify", { error: errorMessage(e), dispatch: q.notice.dispatchId });
+        }
+      const live = currentSession(deps);
+      const stillHere = !stopped && live !== null && sessionKey(live) === sessionKey(target);
+      const marked = due.filter((q) => stillHere && owned(q.run));
+      // provably not submitted, so a retry cannot duplicate input: tried again a window later, a few times
+      if (receipt.status === "failed")
+        for (const q of marked) {
+          q.sendFailures = (q.sendFailures ?? 0) + 1;
+          if (q.sendFailures < MAX_SEND_FAILURES) busy.push(q);
+        }
+      if (receipt.status === "accepted" && target.host === "claude-code")
+        for (const q of marked)
+          try {
+            if (!existsSync(q.mark))
+              writeJsonAtomic(q.mark, { schema: 1, msgId: receipt.msgId, at: receipt.at });
+          } catch (e) {
+            log("warn", "notify", { error: errorMessage(e), dispatch: q.notice.dispatchId });
+          }
+      log(receipt.status === "accepted" ? "info" : "warn", "notify", {
+        outcome: receipt.status,
+        msgId: receipt.msgId,
+        reason: receipt.reason,
+        dispatches: notices.map((n) => n.dispatchId),
+        unmarked: due.filter((q) => !marked.includes(q)).map((q) => q.notice.dispatchId),
+      });
     } finally {
       for (const release of claims) release();
       requeue(busy);
     }
   }
 
-  /** Queues again the notices another server held the claim on; the next pass drops those no longer due or owned. */
+  /**
+   * Queues again the notices another server held the claim on, whose batch could not claim them all, or that
+   * the transport provably did not submit; the next pass drops those no longer due or owned.
+   */
   function requeue(busy: Queued[]): void {
     if (stopped || busy.length === 0) return;
     for (const q of busy) if (!queue.has(q.key)) queue.set(q.key, q);
     schedule();
   }
 
+  function queueNotice(q: Queued): void {
+    const target = currentSession(deps);
+    if (stopped || !target || queue.has(q.key) || !q.due() || !owned(q.run)) return;
+    let recovering = false;
+    try {
+      recovering = readDelivery(q.dir).some(
+        (a) => a.status === "submitting" && a.eventIds.includes(q.notice.eventId),
+      );
+    } catch (e) {
+      log("warn", "notify", { error: errorMessage(e), dispatch: q.notice.dispatchId });
+      return;
+    }
+    if (!recovering && !pending(q, target)) return;
+    queue.set(q.key, q);
+    schedule();
+  }
+
   const enqueue = (run: Run, d: Dispatch, record: RunRecord): void => {
-    if (queue.has(record.dispatchId) || notified(d.dir) || !awaitsCollect(d.dir) || !owned(run)) return;
-    queue.set(record.dispatchId, {
-      key: record.dispatchId,
+    const notice = finishedNotice(run, d, record);
+    queueNotice({
+      key: notice.eventId,
       run,
-      notice: finishedNotice(run, d, record),
+      dir: d.dir,
+      notice,
       mark: dispatchPaths(d.dir).notified,
-      due: () => awaitsCollect(d.dir) && !notified(d.dir),
+      due: () => awaitsCollect(d.dir),
     });
-    schedule();
   };
 
   const onStall = (s: Stalled): void => {
     try {
       const p = dispatchPaths(s.d.dir);
-      const key = `${s.d.admit.dispatchId} stall`;
-      if (queue.has(key) || existsSync(p.stallNotified) || existsSync(p.exit) || !owned(s.run)) return;
-      queue.set(key, {
-        key,
+      const notice = stalledNotice(s.run, s.d, s.quietMs, deps.now());
+      queueNotice({
+        key: notice.eventId,
         run: s.run,
-        notice: stalledNotice(s.run, s.d, s.quietMs, deps.now()),
+        dir: s.d.dir,
+        notice,
         mark: p.stallNotified,
-        // it finished meanwhile: its record is the news now
-        due: () => !existsSync(p.stallNotified) && !existsSync(p.exit),
+        due: () => !existsSync(p.exit),
       });
-      schedule();
     } catch (e) {
       log("warn", "notify", { run: s.run.id, name: s.d.admit.name, error: errorMessage(e) });
     }
@@ -264,9 +501,30 @@ export function startNotifier(deps: Deps, o: NotifierOptions = {}): Notifier {
           if (!owned(run)) continue;
           const records = new Map(readRecords(run).records.map((r) => [r.dispatchId, r]));
           for (const d of listDispatches(run)) {
-            const r = records.get(d.admit.dispatchId);
-            // a limit not yet failed over is reconcile's to settle first: its hook announces it then
-            if (r && !(r.status === "limit" && !readFailover(d.dir))) enqueue(run, d, r);
+            // one dispatch's damaged evidence stays ambiguous for it alone: the rest of the run is still scanned
+            try {
+              const p = dispatchPaths(d.dir);
+              for (const [kind, mark] of [
+                ["finished", p.notified],
+                ["stalled", p.stallNotified],
+              ] as const) {
+                const release = tryLock(mark);
+                if (!release) continue;
+                try {
+                  recoverSubmission(d.dir, JSON.stringify([run.id, d.admit.dispatchId, kind]));
+                } finally {
+                  release();
+                }
+              }
+              const r = records.get(d.admit.dispatchId);
+              // a limit not yet failed over is reconcile's to settle first: its hook announces it then
+              if (r && !(r.status === "limit" && !readFailover(d.dir))) enqueue(run, d, r);
+              if (existsSync(p.stall) && !existsSync(p.exit)) {
+                onStall({ run, d, quietMs: readStallQuietMs(d.dir) });
+              }
+            } catch (e) {
+              log("warn", "notify", { run: run.id, name: d.admit.name, error: errorMessage(e) });
+            }
           }
         } catch (e) {
           log("warn", "notify", { run: run.id, error: errorMessage(e) });
@@ -290,7 +548,54 @@ export function startNotifier(deps: Deps, o: NotifierOptions = {}): Notifier {
       queue.clear();
     },
   };
-  settledHooks.add(onSettled);
-  stallHooks.add(onStall);
+  if (!retryEvent) {
+    settledHooks.add(onSettled);
+    stallHooks.add(onStall);
+  }
   return n;
 }
+
+/** Retry only a stored event for the current owner, after an explicit duplicate-risk decision. */
+export async function retryDelivery(
+  deps: Deps,
+  dir: string,
+  eventId: string,
+  decision: { allowPossibleDuplicate: true },
+): Promise<void> {
+  if (decision?.allowPossibleDuplicate !== true)
+    throw new CatherdError("E_INPUT_INVALID", "Retry requires acknowledging possible duplicate input");
+  const target = currentSession(deps);
+  const located = listRuns().runs.flatMap((run) =>
+    listDispatches(run)
+      .filter((d) => d.dir === dir)
+      .map((d) => ({ run, d })),
+  )[0];
+  if (!target || !located || !ownsRun(deps, located.run))
+    throw new CatherdError(
+      "E_INPUT_INVALID",
+      "Retry requires the validated current owner and stored dispatch",
+    );
+  const { run, d } = located;
+  const record = readRecords(run).records.find((r) => r.dispatchId === d.admit.dispatchId);
+  const finished = JSON.stringify([run.id, d.admit.dispatchId, "finished"]);
+  const stalled = JSON.stringify([run.id, d.admit.dispatchId, "stalled"]);
+  if (eventId !== finished && eventId !== stalled)
+    throw new CatherdError("E_INPUT_INVALID", "Event does not belong to this dispatch");
+  if (eventId === finished && (!record || !awaitsCollect(dir)))
+    throw new CatherdError("E_INPUT_INVALID", "Finished result is absent or already collected");
+  if (eventId === stalled && (!existsSync(dispatchPaths(dir).stall) || existsSync(dispatchPaths(dir).exit)))
+    throw new CatherdError("E_INPUT_INVALID", "Stalled event is absent or already finished");
+  // Corrupt evidence cannot be safely replaced by an explicit retry.
+  readDelivery(dir);
+  const n = notifierFor(deps, { coalesceMs: 0 }, eventId);
+  try {
+    if (eventId === finished)
+      n.onSettled({ run, d, record: record!, hints: [], started: null, pause: null, stateHints: [] });
+    else {
+      n.onStall({ run, d, quietMs: readStallQuietMs(dir) });
+    }
+    await n.idle();
+  } finally {
+    n.stop();
+  }
+}

package/src/services/orchestrator-wait.ts

@@ -0,0 +1,82 @@
+import type { RunRecord } from "../domain/record.ts";
+import { liveSessionFile, readSessionFiles, type SessionFile } from "../infra/claude-session.ts";
+import { awaitsCollect } from "../infra/dispatch-dir.ts";
+import { listDispatches, liveDispatches } from "./dispatches.ts";
+import { readRecords, type Run } from "./run-store.ts";
+import { runOwner } from "./sessions.ts";
+
+export const ORCHESTRATOR_STALL_MS = 5 * 60_000;
+
+/**
+ * Plan 22 (#42 finding 3): the line shows only for a current owner (its Claude Code session live, or seen in the
+ * last day) and an unread record that ended in the last day, so an abandoned run never reads "stalled" forever.
+ */
+export const WAIT_WINDOW_MS = 24 * 3_600_000;
+
+export interface OrchestratorWait {
+  since: string;
+  seconds: number;
+  stalled: boolean;
+  unread: number;
+  /** when the line stops showing, unless the owner is seen again or another record ends */
+  until: string;
+}
+
+/**
+ * When the owner was last seen acting on the run: it took the run (`since`), or one of the run's dispatches was
+ * admitted from its session.
+ */
+function ownerSeen(run: Run, owner: { sessionId: string; since: string }): number {
+  let seen = Date.parse(owner.since) || 0;
+  for (const d of listDispatches(run))
+    if (d.admit.sessionId?.toLowerCase() === owner.sessionId.toLowerCase())
+      seen = Math.max(seen, Date.parse(d.admit.admittedAt) || 0);
+  return seen;
+}
+
+/**
+ * Pure diagnostic: queue acceptance cannot show that the orchestrator processed a completion. Null unless the
+ * run has a current owner, no live role, and an unread record that ended within WAIT_WINDOW_MS.
+ */
+export function orchestratorWait(
+  run: Run,
+  now: number,
+  records: RunRecord[] = readRecords(run).records,
+  hasLive = liveDispatches(run, now).length > 0,
+  files: () => SessionFile[] = readSessionFiles,
+): OrchestratorWait | null {
+  const owner = runOwner(run);
+  if (!owner || hasLive) return null;
+  const unread = new Set(
+    listDispatches(run)
+      .filter((d) => awaitsCollect(d.dir))
+      .map((d) => d.admit.dispatchId),
+  );
+  const recent = records.filter(
+    (r) => unread.has(r.dispatchId) && now - (Date.parse(r.endedAt) || 0) < WAIT_WINDOW_MS,
+  );
+  if (!recent.length) return null;
+  const ended = Math.max(...recent.map((r) => Date.parse(r.endedAt) || 0));
+  const live = owner.host === "claude-code" && liveSessionFile(owner.sessionId, files()) !== null;
+  const seen = live ? now : ownerSeen(run, owner);
+  if (now - seen >= WAIT_WINDOW_MS) return null;
+  const since = Math.max(ended, Date.parse(owner.since) || 0);
+  const elapsed = Math.max(0, now - since);
+  return {
+    since: new Date(since).toISOString(),
+    seconds: Math.floor(elapsed / 1000),
+    stalled: elapsed >= ORCHESTRATOR_STALL_MS,
+    unread: records.filter((r) => unread.has(r.dispatchId)).length,
+    until: new Date(Math.min(ended, seen) + WAIT_WINDOW_MS).toISOString(),
+  };
+}
+
+/**
+ * The one "waiting for orchestrator" line every view shows (#42 finding 7): seconds and `stalled` from `since` at
+ * `now`, so a memoised wait stays true between recomputes; null once it is past `until`.
+ */
+export function waitingLine(w: OrchestratorWait | null | undefined, now: number): string | null {
+  if (!w || now >= Date.parse(w.until)) return null;
+  const elapsed = Math.max(0, now - Date.parse(w.since));
+  return `${elapsed >= ORCHESTRATOR_STALL_MS ? "stalled · " : ""}waiting for orchestrator ${Math.floor(elapsed / 1000)}s`;
+}