omp-conductor 0.15.9 → 0.15.11

package/src/daemon.ts

@@ -20,16 +20,23 @@ import {
 } from "./config.ts";
 import { availabilityState, type AvailabilityState } from "./availability.ts";
 import {
+  UNREADABLE_TREE_FLAG,
   analyseSettlement,
+  deriveChangedLine,
   formatSettlementFlags,
   settlementFlagSummary,
+  withDerivedChangedLine,
 } from "./diff-flags.ts";
 import { digestScheduleState, type DigestScheduleState } from "./digest-schedule.ts";
 import { createEscalator, escalationIssueRef } from "./escalate.ts";
 import { pendingCodeGraph, probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
 import { graphHint } from "./graph.ts";
-import { livingDaemon } from "./lifecycle.ts";
+import { acquireOnceLease, healthCheck, livingDaemon } from "./lifecycle.ts";
 import { requestImmediateTick, resolveTickConfigCwd, STALL_MARKER_FILE } from "./orchestrator-tick.ts";
+import { runDoctor } from "./doctor.ts";
+import { appendJournal, launchTransientUnit, readUpgradeJournal } from "./upgrade-journal.ts";
+import { runCommand, verifyPendingUpgrade, type UpgradeVerifyDeps } from "./upgrade-verify.ts";
+import { fleetLayers } from "./fleet.ts";
 import { startOrchestrator } from "./orchestrator.ts";
 import type { OrchestratorHandle } from "./orchestrator.ts";
 import {
@@ -219,6 +226,13 @@ interface Deps {
    * the real one.
    */
   verbActions?: VerbActions;
+  /**
+   * The fleet-installs-itself verification pass (#486), run on the first tick
+   * after a detached upgrade. The default is the journal-gated
+   * {@link verifyPendingUpgradeTick}; a test injects its own to drive the tick
+   * without touching the host's packages, journal or outbox.
+   */
+  upgradeVerifier?: (d: Deps) => Promise<void>;
 }
 
 /**
@@ -981,6 +995,56 @@ export function hasContinuationBudget(stops: number, maxContinuations: number):
   return stops <= maxContinuations;
 }
 
+/** True while unspent failed-implementation attempts remain. This is the
+ *  dispatcher's admission gate: once every `maxAttemptsPerIssue` slot is
+ *  spent, the issue is held as `failed-attempts` forever, and the `unblock`
+ *  verb withholds the queue label on the same predicate (#348). */
+export function hasFailedAttemptBudget(failures: number, maxAttempts: number): boolean {
+  return failures < maxAttempts;
+}
+
+/** The failure classes `countContinuations` deliberately does not charge — the
+ *  inverted copy of its exclusions, kept beside the breakdown that consumes it
+ *  so the two can only drift together (#439). `orphan-clean` is absent on
+ *  purpose: daemon orphans consume the continuation budget, which is exactly
+ *  why the requeue side has to respect the ceiling instead of racing it. */
+const NON_CONTINUATION_CLASSES: Partial<Record<FailureClass, true>> = {
+  "admin-kill": true,
+  "settlement-stuck": true,
+  "env-start-failure": true,
+  "dispatch-infra": true,
+  "provider-credit": true,
+  "provider-transient": true,
+};
+
+/** How one issue spent its continuation budget, grouped by failure class —
+ *  the exact rows `continuationsFor` charges, so an exhaustion escalation
+ *  reports the same budget it says is spent. `unclassified` groups rows that
+ *  charged before the class was written (a pre-upgrade NULL). */
+function continuationBreakdown(runs: readonly RunRecord[]): Map<string, number> {
+  const perClass = new Map<string, number>();
+  for (const r of runs) {
+    const chargedAsKilledOrOrphaned =
+      (r.state === "killed" || r.state === "orphaned" || r.state === "blocked") &&
+      (r.failureClass === undefined || NON_CONTINUATION_CLASSES[r.failureClass] === undefined);
+    const chargedAsReturned = r.state === "failed" && r.failureClass === "returned-for-revision";
+    if (!chargedAsKilledOrOrphaned && !chargedAsReturned) continue;
+    const cls = r.failureClass ?? "unclassified";
+    perClass.set(cls, (perClass.get(cls) ?? 0) + 1);
+  }
+  return perClass;
+}
+
+/** The newest attempt's preserved work, if any, so an exhaustion escalation can
+ *  say whether continuing is worthwhile: `salvageSha`/`headSha`/`prUrl` are the
+ *  three artifacts a run can leave, and all null on a branch nothing reached. */
+function newestContinuableRun(runs: readonly RunRecord[]): RunRecord | undefined {
+  const newestFirst = [...runs].reverse();
+  return newestFirst.find(
+    (r) => r.salvageSha !== undefined || r.headSha !== undefined || r.prUrl !== undefined,
+  );
+}
+
 export type ExtendTurnLimitResult =
   | { kind: "extended"; runId: string; maxTurns: number }
   | { kind: "not-increase"; runId: string; maxTurns: number }
@@ -1184,6 +1248,19 @@ export async function verifyPushedGreenClaim(
   };
 }
 
+/** What the settlement audit of one green run produced: the advisory flags
+ *  (test weakening only — the file-list disclosure is derived, not flagged),
+ *  whether the diff was cut short, and the `changed:` line composed from the
+ *  PR's own diff. */
+export interface SettlementAuditResult {
+  flags: SettlementFlag[];
+  truncated: boolean;
+  /** The `changed:` file list derived from the PR's diff, present whenever the
+   *  diff could be read. Absent means the tree could not be read, and `flags`
+   *  then carries exactly {@link UNREADABLE_TREE_FLAG}. */
+  changedLine?: string;
+}
+
 /**
  * Audit a worker's own account of its work against the pull request it pushed.
  *
@@ -1192,27 +1269,21 @@ export async function verifyPushedGreenClaim(
  * shares none of its authority — that function decides a run's state, this one
  * cannot, by construction. It returns evidence and the caller appends it.
  *
- * Returns undefined when the audit could not run at all: no PR to read, or a
- * tracker that could not produce a diff. Undefined is *not* an empty flag list.
- * An empty list means "read the whole diff, found nothing"; undefined means
- * "did not look", and the report says so rather than presenting silence as a
- * clean bill.
+ * A diff that cannot be read is a finding (`changed-line-missing`), never an
+ * empty flag list: nothing was derived and nothing was checked, and that must
+ * not read as a clean bill.
  */
 export async function collectSettlementFlags(
   tracker: Pick<Tracker, "prDiff">,
-  claim: { prUrl?: string; report: string; priorReports?: readonly string[]; issueText: string },
-): Promise<{ flags: SettlementFlag[]; truncated: boolean } | undefined> {
-  if (claim.prUrl === undefined) return undefined;
+  claim: { prUrl?: string; issueText: string },
+): Promise<SettlementAuditResult> {
+  if (claim.prUrl === undefined) return { flags: [UNREADABLE_TREE_FLAG], truncated: false };
   const diff = await tracker.prDiff(claim.prUrl);
-  if (diff === undefined) return undefined;
+  if (diff === undefined) return { flags: [UNREADABLE_TREE_FLAG], truncated: false };
   return {
-    flags: analyseSettlement({
-      report: claim.report,
-      issueText: claim.issueText,
-      diff,
-      priorReports: claim.priorReports,
-    }),
+    flags: analyseSettlement({ issueText: claim.issueText, diff }),
     truncated: diff.truncated,
+    changedLine: deriveChangedLine(diff),
   };
 }
 
@@ -1273,6 +1344,48 @@ export function workerReleaseBlockRecorder(
     );
 }
 
+/**
+ * The exact PR a new claim inherits from its terminal predecessor (#434).
+ *
+ * `recentRuns` shows only the newest attempt per issue, and the mediated verbs
+ * resolve a PR's owner through the store — so the moment an orphan-clean
+ * requeue creates its continuation row, the predecessor's green PR (recorded
+ * on the older row) stops resolving and every mediated verb refuses it as "not
+ * a pull request any run opened". Seeding the continuation row with the same
+ * PR at claim time closes that window: ownership never depends on a worker
+ * rediscovering and re-reporting the PR.
+ *
+ * The checks are deliberately the exact counterpart of the store query's scope:
+ * the claim's project/issue/repo/branch must all match the predecessor, the
+ * predecessor must be terminal (a live row is still doing its own work, and a
+ * claim beside one is already a double dispatch), and the tracker must confirm
+ * the PR is still open — a closed or merged predecessor PR is settled by the
+ * sweep against the row that owns it, never carried onto a fresh attempt. An
+ * unreadable tracker fails closed: the new row simply starts without PR
+ * ownership, exactly as before #434, and adoption or the worker's own
+ * pr_create recovers the link.
+ */
+export async function inheritedPrForContinuation(
+  tracker: Tracker,
+  prior: RunRecord | undefined,
+  claim: { project: string; issue: number; repo: string; branch: string },
+): Promise<{ prUrl: string; headSha?: string } | undefined> {
+  if (prior === undefined || prior.prUrl === undefined) return undefined;
+  // Terminal only. A live predecessor owns its PR outright and is not done
+  // with it; inheriting would launder a double dispatch's row.
+  if (prior.state === "claimed" || prior.state === "running") return undefined;
+  if (prior.project !== claim.project || prior.issue !== claim.issue) return undefined;
+  if (prior.repo !== claim.repo || prior.branch !== claim.branch) return undefined;
+  let state: PrState | undefined;
+  try {
+    state = await tracker.prState(prior.prUrl);
+  } catch {
+    return undefined;
+  }
+  if (state !== "open") return undefined;
+  return { prUrl: prior.prUrl, ...(prior.headSha === undefined ? {} : { headSha: prior.headSha }) };
+}
+
 export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
   const { project, caps, tracker, store } = d;
   const issue = r.issue.number;
@@ -1429,7 +1542,8 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
     // (#201).
     // Read before this attempt's own row exists, so `latestRun` still means the
     // attempt whose work this one inherits.
-    const priorSalvage = store.latestRun(project.name, issue)?.salvageSha;
+    const prior = store.latestRun(project.name, issue);
+    const priorSalvage = prior?.salvageSha;
     run = store.createRun({
       project: project.name,
       issue,
@@ -1443,6 +1557,27 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
       maxTurns: caps.workerMaxTurns,
       startedAt: Date.now(),
     });
+    // #434: carry a terminal predecessor's open PR onto the continuation row.
+    // The claim itself stays synchronous — `handleIssue` claims at the top of
+    // its dispatch burst, the window the shutdown fence and its tests rely on
+    // (#374) — so the tuple-exact, tracker-verified inheritance lands here,
+    // still before any provisioning, network call or verb socket. The new row
+    // is `recentRuns`' newest attempt for the issue the moment it exists, so
+    // without this a requeue hides the green PR the orphaned attempt opened
+    // until a worker happens to rediscover it.
+    const inherited = await inheritedPrForContinuation(tracker, prior, {
+      project: project.name,
+      issue,
+      repo: r.repo.name,
+      branch,
+    });
+    if (inherited !== undefined && prior !== undefined) {
+      store.updateRun(run.id, {
+        prUrl: inherited.prUrl,
+        ...(inherited.headSha === undefined ? {} : { headSha: inherited.headSha }),
+      });
+      log(`#${issue} inherited PR ${inherited.prUrl} from attempt ${prior.attempt} (${prior.state})`);
+    }
     const maxTurns = run.maxTurns;
     const turnOverride = maxTurns > caps.workerMaxTurns ? maxTurns : undefined;
     const runId = run.id;
@@ -1618,39 +1753,37 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
     // The other half of not believing a worker about its own run (#128). The
     // claim being audited is `state: pushed-green`, so the audit runs on the
     // worker's claim rather than on what verification made of it: a claim that
-    // GitHub then contradicted is exactly the report whose `changed:` line is
-    // worth reading twice. Advisory throughout — nothing below reads `flags`
-    // when deciding `state`, and the ordering here makes that checkable.
+    // GitHub then contradicted is exactly the report whose details matter
+    // most. Advisory throughout — nothing below reads `flags` when deciding
+    // `state`, and the ordering here makes that checkable.
     const audit =
       result.state === "pushed-green"
         ? await collectSettlementFlags(tracker, {
             prUrl: result.prUrl,
-            report: result.report,
-            // Earlier attempts' reports pool as disclosure coverage (#199). The
-            // current row has not settled yet, so this is exactly the prior set;
-            // the strict `< attempt` guard keeps an already-persisted copy of
-            // this attempt out just in case. Pre-#199 rows have NULL reports and
-            // contribute nothing — today's behaviour.
-            priorReports: store
-              .attemptReports(project.name, issue)
-              .filter((r) => r.attempt < attempt)
-              .map((r) => r.report),
             issueText: `${r.issue.title}\n${r.issue.body}`,
           })
         : undefined;
-    if (result.state === "pushed-green") {
-      if (audit === undefined) log(`#${issue} settlement audit skipped: no readable PR diff`);
-      else if (audit.truncated) log(`#${issue} settlement audit read a truncated PR diff`);
+    if (result.state === "pushed-green" && audit?.truncated) {
+      log(`#${issue} settlement audit read a truncated PR diff`);
     }
     const auditLines =
       audit === undefined
         ? []
-        : formatSettlementFlags(audit.flags, { truncated: audit.truncated, attempts: attempt });
+        : formatSettlementFlags(audit.flags, { truncated: audit.truncated });
+    // The report's file list is derived from the PR's own diff, not retyped
+    // from the worker's memory of the session: the narrative stays the
+    // worker's, the disclosure becomes the diff's (#488). A diff that could
+    // not be read leaves the worker's text untouched and the audit's
+    // `changed-line-missing` flag says so.
+    const settlementReport =
+      audit?.changedLine === undefined
+        ? result.report
+        : withDerivedChangedLine(result.report, audit.changedLine);
 
     const finalReport = [
       ...(verified.reason === undefined ? [] : [verified.reason, ""]),
       ...(auditLines.length === 0 ? [] : [...auditLines, ""]),
-      result.report,
+      settlementReport,
     ].join("\n");
 
     // What becomes of the tree, decided once, before any label or page. A
@@ -1695,12 +1828,18 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
       endedAt: Date.now(),
       turns: result.turns,
       spendUsd: result.spendUsd,
-      prUrl: result.prUrl,
-      headSha: result.headSha,
+      // The worker only reports these when it actually established them; a kill
+      // or a settle whose report named no PR must not wipe what a verb recorded
+      // earlier in the same run (#468). The sink in `updateRun` skips undefined
+      // too, but a settle should not even build such a patch.
+      ...(result.prUrl === undefined ? {} : { prUrl: result.prUrl }),
+      ...(result.headSha === undefined ? {} : { headSha: result.headSha }),
       sessionFile: result.sessionFile,
-      // Every terminal state persists the worker's report, not just a green
-      // push: a stopped attempt's partial report is still part of its audit trail.
-      report: result.report,
+      // Every terminal state persists the worker's report — with the `changed:`
+      // file list derived from the PR's diff where one could be read — not
+      // just a green push: a stopped attempt's partial report is still part of
+      // its audit trail.
+      report: settlementReport,
       ...settlement?.patch,
       ...(audit === undefined || audit.flags.length === 0
         ? {}
@@ -2275,8 +2414,13 @@ export async function watchBaseHealth(
 
 export async function settlePushedGreen(
   d: Pick<Deps, "project" | "tracker" | "store">,
-): Promise<void> {
+): Promise<number> {
   const { project, tracker, store } = d;
+  // Runs the sweep resolved by terminalising the row. Every terminal write
+  // below increments it; the tick attributes it as `settled` on the pass's
+  // dispatch record, so a held pass is visible as work done, not just as a
+  // clock that moved (#497).
+  let settled = 0;
   // Filtered from the active set rather than asked for with a new query: active
   // is live workers plus these, so the list is bounded by the worker cap plus
   // the number of PRs awaiting a merge — a handful, by construction. A fleet
@@ -2343,6 +2487,7 @@ export async function settlePushedGreen(
         patch.recoveryAction = "none";
       }
       store.updateRun(run.id, patch);
+      settled += 1;
       log(`#${run.issue} settled: ${settlement.reason}`);
       continue;
     }
@@ -2357,7 +2502,7 @@ export async function settlePushedGreen(
     }
     if (verification === undefined) continue;
     if (verification.status === "green") {
-      store.updateRun(run.id, { state: "pushed-green", lastError: undefined });
+      store.updateRun(run.id, { state: "pushed-green", lastError: null });
       log(`#${run.issue} checks settled: ${verification.reason}`);
     } else if (verification.status === "failed") {
       // Equally terminal, so the release is enqueued before the row writes,
@@ -2366,11 +2511,13 @@ export async function settlePushedGreen(
       // and its live PR is exactly the work the label must keep guarding.
       releaseInProgress(d, run.issue, verification.reason);
       store.updateRun(run.id, { state: "failed", lastError: verification.reason });
+      settled += 1;
       log(`#${run.issue} checks failed: ${verification.reason}`);
     } else {
       store.updateRun(run.id, { lastError: verification.reason });
     }
   }
+  return settled;
 }
 
 const ADOPTABLE_PR_STATES: Partial<Record<RunState, true>> = {
@@ -2612,6 +2759,7 @@ export function summarizeDispatch(
   admitted: number,
   holds: readonly AdmissionHold[],
   completedAt = Date.now(),
+  settled = 0,
 ): DispatchSummary {
   const groups = new Map<AdmissionHoldReason, { count: number; issues: number[] }>();
   for (const hold of holds) {
@@ -2630,6 +2778,27 @@ export function summarizeDispatch(
     holds: [...groups]
       .sort(([a], [b]) => a.localeCompare(b))
       .map(([reason, group]) => ({ reason, ...group })),
+    settled,
+  };
+}
+
+/**
+ * The record of a held pass (#497): the loop ran — settling and reconciling
+ * above the pause gate — but routing never happened and nothing was admitted.
+ * `ready`/`routed`/`claimed` stay 0 because they were never computed; the
+ * `paused` flag is what keeps a reader from reading them as an empty queue.
+ */
+export function summarizeHeldPass(settled: number, completedAt = Date.now()): DispatchSummary {
+  return {
+    completedAt,
+    ready: 0,
+    routed: 0,
+    claimed: 0,
+    admitted: 0,
+    degraded: false,
+    holds: [],
+    settled,
+    paused: true,
   };
 }
 
@@ -2837,7 +3006,7 @@ export async function admitCandidates(
 
     const priorRuns = store.attemptsFor(project.name, issue);
     const failures = store.failuresFor(project.name, issue);
-    if (failures >= caps.maxAttemptsPerIssue) {
+    if (!hasFailedAttemptBudget(failures, caps.maxAttemptsPerIssue)) {
       hold(issue, "failed-attempts");
       await safeEscalate(d, {
         tier: 1,
@@ -3103,6 +3272,70 @@ export function wakeOrchestratorForMetConditions(
   }
 }
 
+
+/**
+ * The first-tick verification for a fleet-initiated upgrade (#486).
+ *
+ * The detached install unit journals its progress and stops, leaving dispatch
+ * paused; whichever daemon comes back after the restart runs this on its
+ * first tick — above every gate that a pause or an integrity mismatch could
+ * close, because the install's own pause is the thing under verification.
+ * A day with no journal costs one file read; only a pending request pays for
+ * the version/health/ticks/pane/doctor checks and the outbox rows.
+ */
+export function upgradeVerifyDepsFor(d: Pick<Deps, "project" | "store">): UpgradeVerifyDeps {
+  return {
+    projectName: d.project.name,
+    journal: (entry) => appendJournal(stateDir(), entry),
+    run: runCommand,
+    layers: (project) => fleetLayers(project),
+    health: healthCheck,
+    doctor: (name) => runDoctor(name),
+    enqueue: (draft) => {
+      try {
+        d.store.enqueueReport(draft);
+      } catch (err) {
+        log(`upgrade verification report could not be queued: ${errText(err)}`);
+      }
+    },
+    pause: pauseInstance,
+    resume: (project) => setPaused(false, undefined, project),
+    launchRollback: async (version) => {
+      const launched = await launchTransientUnit(
+        (argv) => runCommand(argv[0]!, argv.slice(1)),
+        process.env,
+        { kind: "rollback", version },
+      );
+      return launched.ok === true ? { ok: true, unit: launched.unit } : { ok: false, stderr: launched.stderr };
+    },
+    log,
+    now: () => Date.now(),
+  };
+}
+
+async function verifyPendingUpgradeTick(
+  d: Deps,
+  run = verifyPendingUpgrade,
+): Promise<void> {
+  // Cheap pre-test: no journal, no pending request, nothing to touch.
+  if (readUpgradeJournal(stateDir()).length === 0) return;
+  const verifyDeps = upgradeVerifyDepsFor(d);
+  try {
+    const outcome = await run(verifyDeps);
+    if (outcome.handled !== "none" && outcome.handled !== "already-closed") {
+      log(
+        `upgrade journal handled: ${outcome.handled}` +
+          `${outcome.version === undefined ? "" : ` for omp-conductor@${outcome.version}`}` +
+          `${outcome.detail === undefined ? "" : ` — ${outcome.detail}`}`,
+      );
+    }
+  } catch (err) {
+    // A torn journal must not take the tick down: the operator still reads it
+    // through `status`, and the next tick retries the same pass.
+    log(`upgrade verification pass failed: ${errText(err)}`);
+  }
+}
+
 export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
   // A config edit takes effect on the next tick, not the next daemon restart
   // (#170). Re-resolve the project and its caps at the tick boundary so a tick
@@ -3124,6 +3357,16 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
     d.deliveryPolicyValid = false;
   }
 
+  // The fleet-installs-itself transaction (#486): the detached install unit
+  // paused dispatch and stopped; the first tick after the restart verifies
+  // against its journal and closes or rolls back. Above the pause gate on
+  // purpose — the install's own pause is exactly the state being verified.
+  try {
+    await (d.upgradeVerifier ?? verifyPendingUpgradeTick)(d);
+  } catch (err) {
+    log(`upgrade journal pass failed: ${errText(err)}`);
+  }
+
   // Availability-held notices are already durable. Once the freshly reloaded
   // policy opens (or newly allows their category), atomically hand a bounded
   // batch to the report outbox. Pauses do not suppress delivery.
@@ -3150,8 +3393,10 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
   // Settlement is maintenance, not dispatch. Run it before every gate that can
   // stop claiming — pause, integrity, spend, and capacity — so status converges
   // while the fleet is parked or workers are still active. Resident workers run
-  // through the pool without blocking this five-minute tick.
-  await settlePushedGreen(d);
+  // through the pool without blocking this five-minute tick. The return value
+  // is what this pass settled; a held pass records it so the paused state reads
+  // as work done, not just a clock that moved (#497).
+  let settled = await settlePushedGreen(d);
   try {
     await watchMergedBase(d);
   } catch (err) {
@@ -3177,7 +3422,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
   // Each is guarded on its own: a tracker that fails mid-classification must not
   // stop the label reconcile, and neither may stop the tick.
   try {
-    await classifyAndRecover(d);
+    settled += await classifyAndRecover(d);
   } catch (err) {
     log(`classification sweep failed: ${errText(err)}`);
   }
@@ -3228,8 +3473,15 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
     });
 
   // A paused fleet claims nothing. Checked first so pausing takes effect on the
-  // next tick without signalling the process.
-  if (isPaused(d.project.name)) return;
+  // next tick without signalling the process. But the pass still ran, and the
+  // operator has to be able to see it: record it as a held pass — the work the
+  // maintenance sweeps settled above this gate, and nothing admitted below it —
+  // so `last dispatch` keeps moving while the fleet is deliberately parked. A
+  // frozen clock is then a stall report, not a hold (#497).
+  if (isPaused(d.project.name)) {
+    d.store.recordDispatch(d.project.name, summarizeHeldPass(settled));
+    return;
+  }
 
   const { project, caps, store } = d;
 
@@ -3319,7 +3571,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
   const recordDispatch = (admitted: number, holds: readonly AdmissionHold[]): void => {
     store.recordDispatch(
       project.name,
-      summarizeDispatch(ready.length, routed.length, claimed, admitted, holds),
+      summarizeDispatch(ready.length, routed.length, claimed, admitted, holds, Date.now(), settled),
     );
   };
 
@@ -3577,7 +3829,7 @@ export async function turnLimitResponse(
 export async function workerControlResponse(
   req: Request,
   project: string,
-  store: Pick<Store, "latestRun">,
+  store: Pick<Store, "latestRun" | "updateRun">,
   registry: WorkerControlRegistry,
 ): Promise<Response | undefined> {
   const url = new URL(req.url);
@@ -3671,6 +3923,28 @@ export async function workerControlResponse(
       state: latest.state,
     });
   }
+  // A live-state row whose controller is absent from this process is a ghost
+  // (#431): a session controller lives in the daemon that launched it, and once
+  // ownership is exclusive (the once lease makes a drill hold the same pidfile
+  // as the long dispatcher), a row that is live but has no controller in *this*
+  // owner is a row whose owner already died — not work another live daemon is
+  // still writing to. Refuse nothing: recover it as an orphan (worktree kept
+  // for triage, labels untouched, live slot freed) so the issue can be
+  // dispatched again and the row stops counting as a live worker.
+  if (action === "stop") {
+    store.updateRun(latest.id, {
+      state: "orphaned",
+      endedAt: Date.now(),
+      lastError: reason,
+    });
+    return Response.json({
+      outcome: "stopped",
+      runId: latest.id,
+      state: "orphaned",
+      reason: reason!,
+      worktree: latest.worktree,
+    });
+  }
   return Response.json(
     {
       error:
@@ -3683,7 +3957,7 @@ export async function workerControlResponse(
 
 export interface DaemonHttpProjectDeps {
   project: string;
-  store: Pick<Store, "latestRun" | "setTurnOverride">;
+  store: Pick<Store, "latestRun" | "setTurnOverride" | "updateRun">;
   caps: () => Pick<Caps, "workerMaxTurns" | "workerMaxTurnsCeiling">;
 }
 
@@ -3692,21 +3966,32 @@ export interface DaemonHttpDeps {
   turnLimits: TurnLimitRegistry;
   workerControls: WorkerControlRegistry;
   health: () => DaemonHealth;
+  /** Request an immediate dispatch pass (`resume`'s wake, #380). */
+  wake: () => void;
 }
 
 /**
  * The whole HTTP surface, in one named function so a test can pin what is *not*
  * on it.
  *
- * Three route families: the health read, turn-limit control, and live-worker
- * pause/resume/stop control. Everything else is 404. The controls mutate only
- * daemon-owned live sessions; tracker and repository mutations stay on the
- * authenticated per-run channel described at the `Bun.serve` call (#126).
+ * Four route families: the health read, the dispatch wake, turn-limit control,
+ * and live-worker pause/resume/stop control. Everything else is 404. The
+ * controls mutate only daemon-owned live sessions; tracker and repository
+ * mutations stay on the authenticated per-run channel described at the
+ * `Bun.serve` call (#126).
  */
 export async function daemonHttpResponse(req: Request, d: DaemonHttpDeps): Promise<Response> {
   const url = new URL(req.url);
   if (req.method === "GET" && url.pathname === "/healthz") return Response.json(d.health());
 
+  if (req.method === "POST" && url.pathname === "/wake") {
+    // A resume wake: daemon-wide and loopback-only, like /healthz. The pass it
+    // produces re-reads every project's pause file, so a project that is still
+    // held stays held — the wake shortens the wait, it never bypasses a gate.
+    d.wake();
+    return new Response(null, { status: 204 });
+  }
+
   let selected = d.projects[0];
   if (
     req.method === "PUT" &&
@@ -3903,21 +4188,31 @@ export function formatDispatchSummary(summary?: DispatchSummary): string {
   if (summary === undefined) return "last dispatch  (none recorded)";
   const lines = [
     `last dispatch  ${new Date(summary.completedAt).toISOString()}${summary.degraded ? "  DEGRADED" : ""}`,
-    `  candidates    ${summary.ready} ready / ${summary.claimed ?? 0} in flight / ${summary.routed} spare`,
-    `  admitted      ${summary.admitted}`,
   ];
-  if (summary.holds.length === 0) {
-    lines.push("  held          0");
+  if (summary.paused === true) {
+    // A held pass never routed the queue: printing zero candidates here would
+    // read as "the queue was empty". Say what the pass was and attribute what
+    // it did instead (#497).
+    lines.push(`  pass          held — nothing admitted`);
   } else {
-    lines.push("  held");
-    for (const hold of summary.holds) {
-      const sample =
-        hold.issues.length === 0
-          ? ""
-          : ` (#${hold.issues.join(", #")}${hold.count > hold.issues.length ? ", …" : ""})`;
-      lines.push(`    ${hold.reason}  ${hold.count}${sample}`);
+    lines.push(
+      `  candidates    ${summary.ready} ready / ${summary.claimed ?? 0} in flight / ${summary.routed} spare`,
+      `  admitted      ${summary.admitted}`,
+    );
+    if (summary.holds.length === 0) {
+      lines.push("  held          0");
+    } else {
+      lines.push("  held");
+      for (const hold of summary.holds) {
+        const sample =
+          hold.issues.length === 0
+            ? ""
+            : ` (#${hold.issues.join(", #")}${hold.count > hold.issues.length ? ", …" : ""})`;
+        lines.push(`    ${hold.reason}  ${hold.count}${sample}`);
+      }
     }
   }
+  if ((summary.settled ?? 0) > 0) lines.push(`  settled       ${summary.settled}`);
   return lines.join("\n");
 }
 
@@ -4237,8 +4532,12 @@ export function readSessionError(sessionFile: string | undefined): SessionError
  * it is healthy, and writing a class onto it would take it out of this sweep for
  * good.
  */
-export async function classifyAndRecover(d: Deps): Promise<void> {
-  const { project, tracker, store } = d;
+export async function classifyAndRecover(d: Deps): Promise<number> {
+  const { project, caps, tracker, store } = d;
+  // Settle recoveries, counted for the pass's dispatch record — a row whose PR
+  // merged is settled here when the settle sweep could not establish identity
+  // (#497). Every other exit returns 0.
+  let settled = 0;
   for (const run of store.runsNeedingClassification(project.name, CLASSIFY_BATCH)) {
     const facts: ClassifyFacts = {};
     let classifiedRun = run;
@@ -4280,7 +4579,7 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
       continue;
     }
 
-    const { cls, recovery, evidence } = classifyRun(classifiedRun, facts);
+    const { cls, recovery, evidence } = classifyRun(classifiedRun, facts, caps);
 
     // A healthy green PR is not a failure of any class. Leaving the row
     // unclassified is what keeps it eligible for the sweep on the tick where its
@@ -4294,8 +4593,10 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
         ? `#${run.issue} retrying ${recovery} for ${cls}: ${evidence}`
         : `#${run.issue} classified ${cls} → ${recovery}: ${evidence}`,
     );
+    if (recovery === "settle") settled += 1;
     await recoverRun(d, classifiedRun, cls, recovery, evidence);
   }
+  return settled;
 }
 
 /** Performs the one action a class names. Never chooses one of its own. */
@@ -4306,7 +4607,7 @@ async function recoverRun(
   recovery: RecoveryAction,
   evidence: string,
 ): Promise<void> {
-  const { project, tracker, store } = d;
+  const { project, caps, tracker, store } = d;
   const inProgress = project.stateLabels.inProgress;
 
   if (recovery === "settle") {
@@ -4337,6 +4638,50 @@ async function recoverRun(
       return;
     }
 
+    // `wall-clock-cap-progress`: the completion path only auto-continues
+    // turn-cap kills, so a wall-clock kill with work to show reaches this
+    // sweep still holding the failed label. Swap it for the queue — the branch
+    // is retained, so the next dispatch reattaches it and briefs a resume from
+    // the recorded work. Killed rows gathered no tracker facts, so the
+    // issue-open guard mirrors the requeue path's. The continuation gate is
+    // the same one the turns path applies at kill time
+    // (`shouldContinueAfterTurnsCap`): the row is already charged, and once
+    // the ceiling is spent, handing back the queue label would offer a
+    // candidate admission can never accept — only the failed label comes off,
+    // and the exhaustion reaches a human (#490, #348).
+    if (cls === "wall-clock-cap-progress") {
+      const state = await tracker.issueState(run.issue).catch(() => undefined);
+      if (state !== "open") {
+        log(`#${run.issue} not continued from ${cls}: issue is ${state ?? "unreadable"}`);
+        return;
+      }
+      const continuation = store.continuationsFor(project.name, run.issue);
+      if (hasContinuationBudget(continuation, caps.maxContinuationsPerIssue)) {
+        swapToQueue(d, run.issue, project.stateLabels.failed);
+        store.updateRun(run.id, { recoveredAt: Date.now() });
+        log(`#${run.issue} requeued for a wall-clock continuation: ${evidence}`);
+      } else {
+        store.enqueueLabelOps(project.name, [
+          { issue: run.issue, op: "remove", label: project.stateLabels.failed },
+        ]);
+        store.updateRun(run.id, { recoveredAt: Date.now() });
+        await safeEscalate(d, {
+          tier: 1,
+          project: project.name,
+          issue: run.issue,
+          summary: `#${run.issue} exhausted its continuation budget on wall-clock cap kills`,
+          detail: [
+            `Attempt ${run.attempt} hit the wall-clock cap with work to continue from, but ${continuation} continuation(s) are already charged against a cap of ${caps.maxContinuationsPerIssue}.`,
+            evidence,
+            `Work to continue: branch ${run.branch} at ${run.headSha ?? run.salvageSha}${run.prUrl === undefined ? "" : ` — ${run.prUrl}`}.`,
+            "The issue cannot finish inside the wall-clock cap, so another run would burn a worker slot for the same outcome. Re-scope it, raise maxContinuationsPerIssue for it, or finish the remaining work by hand.",
+          ].join("\n"),
+        });
+        log(`#${run.issue} not continued from ${cls}: continuation budget exhausted`);
+      }
+      return;
+    }
+
     // `merge-conflict`: the branch is retained and its PR is open, so #50's
     // continuation guard admits it and the next tick briefs a rebase.
     //
@@ -4413,6 +4758,68 @@ async function recoverRun(
       log(`#${run.issue} not requeued from ${cls}: issue is ${state ?? "unreadable"}`);
       return;
     }
+    // A clean orphan whose queue label is already absent is a deliberate
+    // withdrawal — the operator took the issue out of the queue (e.g. so a
+    // daemon restart could not re-dispatch work known to be unsafe) — and
+    // recovery must not recreate that intent (#423). Read the live label set;
+    // when the queue label is gone, release the dispatcher-owned in-progress
+    // label and stop, leaving the operator's withdrawal to survive recovery.
+    if (cls === "orphan-clean") {
+      const snapshot = await tracker.issueSnapshot(run.issue).catch(() => undefined);
+      if (snapshot === undefined) {
+        log(`#${run.issue} not requeued from orphan-clean: cannot confirm ${project.queueLabel} (unreadable, retrying)`);
+        return;
+      }
+      if (!snapshot.labels.includes(project.queueLabel)) {
+        store.enqueueLabelOps(project.name, [{ issue: run.issue, op: "remove", label: inProgress }]);
+        store.updateRun(run.id, { recoveredAt: Date.now() });
+        log(`#${run.issue} not requeued from orphan-clean: ${project.queueLabel} removed before recovery (operator withdrawal)`);
+        return;
+      }
+      // #439: an orphan-clean row charges the continuation budget, so once
+      // `hasContinuationBudget` is spent — exactly the predicate `admitCandidates`
+      // holds the issue on — requeueing re-adds a queue label for a candidate the
+      // dispatcher can never admit. Stop handing it back and hold it instead: the
+      // queue label comes off (so admission never re-holds on every dispatch),
+      // the in-progress label is released, and a single diagnosis escalates once.
+      // `orphan-clean` is deliberately NOT a global exclusion from the budget (a
+      // worker that genuinely keeps dying mid-work must still be bounded); this is
+      // the missing ceiling check this path never had (#348's invariant).
+      const continuations = store.continuationsFor(project.name, run.issue);
+      if (!hasContinuationBudget(continuations, caps.maxContinuationsPerIssue)) {
+        const runs = store.runsForIssue(project.name, run.issue);
+        const breakdown = continuationBreakdown(runs);
+        const artifact = newestContinuableRun(runs);
+        const onlyDaemonStops = breakdown.size === 1 && breakdown.get("orphan-clean") === continuations;
+        const classLine = Array.from(breakdown, ([cls, n]) => `${n} ${cls}`).join(", ");
+        const artifactLine =
+          artifact === undefined
+            ? "The attempts left no salvage commit, head SHA or PR — the branch is empty, so start clean from a re-scope rather than continuing from nothing."
+            : `Work to continue: branch ${artifact.branch} at ${artifact.headSha ?? artifact.salvageSha}${artifact.prUrl === undefined ? "" : ` — ${artifact.prUrl}`}.`;
+        store.enqueueLabelOps(project.name, [
+          { issue: run.issue, op: "remove", label: inProgress },
+          { issue: run.issue, op: "remove", label: project.queueLabel },
+        ]);
+        store.updateRun(run.id, { recoveredAt: Date.now() });
+        await safeEscalate(d, {
+          tier: 1,
+          project: project.name,
+          issue: run.issue,
+          summary: `#${run.issue} exhausted its ${caps.maxContinuationsPerIssue}-continuation budget on ${cls}`,
+          detail: [
+            `Attempt ${run.attempt} is not requeued: ${continuations} continuation(s) charged against a cap of ${caps.maxContinuationsPerIssue}.`,
+            `Continuations by failure class: ${classLine}.`,
+            onlyDaemonStops
+              ? "Every continuation was a daemon stop — the work never failed; the budget was spent by daemon deaths, not the issue."
+              : "Continuations span real work — inspect what each attempt left behind before continuing.",
+            artifactLine,
+            "What you can do: raise maxContinuationsPerIssue for this issue, re-scope it, or continue from the preserved work (or start clean if none).",
+          ].join("\n"),
+        });
+        log(`#${run.issue} not requeued from orphan-clean: continuation budget exhausted`);
+        return;
+      }
+    }
     const label = cls === "orphan-clean" ? inProgress : project.stateLabels.failed;
     swapToQueue(d, run.issue, label);
     store.updateRun(run.id, { recoveredAt: Date.now() });
@@ -4431,14 +4838,14 @@ async function recoverRun(
     // Back to pending rather than green: the existing settle sweep re-verifies
     // it against the recorded head on a later tick, so nothing here has to guess
     // whether the re-run passed.
-    store.updateRun(run.id, { state: "pushed-pending", lastError: undefined, recoveredAt: Date.now() });
+    store.updateRun(run.id, { state: "pushed-pending", lastError: null, recoveredAt: Date.now() });
     log(`#${run.issue} re-ran infrastructure checks: ${evidence}`);
     return;
   }
 
   if (recovery === "escalate") {
     const detail = [evidence];
-    if (cls === "turn-cap-spinning") {
+    if (cls === "turn-cap-spinning" || cls === "wall-clock-cap-spinning") {
       const calls = lastToolCalls(run.sessionFile);
       detail.push(
         calls.length === 0
@@ -4669,7 +5076,112 @@ function orchestratorStandingOrders(project: ProjectConfig): {
   };
 }
 
+/**
+ * Pacing for the resident dispatch loop, interruptible so an operator action —
+ * `resume` — does not wait out the full interval (#380).
+ *
+ * `requestWake` is sticky until a pass absorbs it: while the long sleep is in
+ * flight it is cut short, and when a pass is mid-flight the loop skips the
+ * *following* sleep. Either way the next pass starts without the interval
+ * wait. The single boolean coalesces any number of near-simultaneous wakes
+ * into one prompt pass, and the strictly sequential loop means a wake can
+ * never start a pass on top of one already running.
+ */
+export interface DispatchPace {
+  /** Ask for a prompt pass; cuts an in-flight sleep short. Coalesces. */
+  requestWake(): void;
+  /** Absorb a pending request — true when the pass that just finished should
+   *  skip its sleep. */
+  takeWake(): boolean;
+  /** The long interval sleep; resolves early when {@link requestWake} fires. */
+  sleep(ms: number): Promise<void>;
+}
+
+export function createDispatchPace(): DispatchPace {
+  let pendingWake = false;
+  let interruptSleep: (() => void) | undefined;
+  return {
+    requestWake() {
+      pendingWake = true;
+      interruptSleep?.();
+    },
+    takeWake() {
+      const pending = pendingWake;
+      pendingWake = false;
+      return pending;
+    },
+    async sleep(ms) {
+      // A wake that landed in the gap between the loop's takeWake check and
+      // this call is absorbed here: skip the interval, and consume the flag
+      // so the pass that follows it is the only prompt one.
+      if (pendingWake) {
+        pendingWake = false;
+        return;
+      }
+      await new Promise<void>((resolve) => {
+        const timer = setTimeout(resolve, ms);
+        interruptSleep = () => {
+          // The interruption *is* the absorption: the pass that follows the
+          // sleep is the prompt pass this wake asked for, so the loop sleeps
+          // normally after it instead of running a second immediate pass.
+          pendingWake = false;
+          clearTimeout(timer);
+          resolve();
+        };
+        // A wake can land between the loop's takeWake check and this
+        // constructor; resolve immediately instead of letting it wait out
+        // the full interval.
+        if (pendingWake) interruptSleep();
+      });
+      interruptSleep = undefined;
+    },
+  };
+}
+
+export interface DispatchLoopOptions {
+  /** One dispatch pass over every configured project runtime. */
+  tick(): Promise<void>;
+  /** True once a stop is requested; checked between passes. */
+  shouldStop(): boolean;
+  pace: DispatchPace;
+  /** The scheduled interval between passes (`TICK_INTERVAL_MS` in production). */
+  tickIntervalMs: number;
+}
+
+/**
+ * The resident daemon's dispatch loop: one pass immediately, then a
+ * `tickIntervalMs` sleep the {@link DispatchPace} can cut short. Exported so
+ * the resume-wake behaviour can be pinned deterministically without a live
+ * daemon (#380).
+ */
+export async function runDispatchLoop(o: DispatchLoopOptions): Promise<void> {
+  const { tick, shouldStop, pace, tickIntervalMs } = o;
+  while (!shouldStop()) {
+    await tick();
+    if (shouldStop()) break;
+    // A wake that landed while the pass was in flight skips the following
+    // sleep: that pass made its admission decisions against pre-resume state,
+    // and the resumed project must not wait a full interval for the next one.
+    if (pace.takeWake()) continue;
+    await pace.sleep(tickIntervalMs);
+  }
+}
+
 export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
+  // A `--once` tick is still a dispatcher: it settles rows, projects labels,
+  // admits and launches workers, so it must hold the same exclusive daemon/
+  // state ownership as the long-running dispatcher for its whole lifecycle
+  // (#431). This is the one choke point every once tick passes through (both
+  // `daemon --once` and the `setup host` smoke), so ownership is acquired here
+  // — before any settlement, label, admission, worktree or worker mutation —
+  // and refused (having done none of those) while another live daemon owns the
+  // project or another live `--once` holds the lease.
+  const onceLease = o.once ? acquireOnceLease({ port: o.port, project: o.project }) : undefined;
+  if (onceLease !== undefined && !onceLease.ok) {
+    throw new Error(
+      `another daemon is alive (pid ${onceLease.pid}); stop it first — a --once tick refuses to run beside it`,
+    );
+  }
   const cfg = loadConfig();
   const projects = o.project === undefined ? cfg.projects : [findProject(cfg, o.project)];
   const store = openStore(dbPath());
@@ -4843,6 +5355,10 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
       await workers.drain();
       for (const runtime of runtimes) await runtime.outbox.deliverDue();
     } finally {
+      // Drop exclusivity only while it is still ours; a manager that claimed
+      // the record mid-tick is left alone. A lease acquired but never released
+      // (config/lease failure before this branch) is stale recovery's job.
+      if (onceLease !== undefined && onceLease.ok) onceLease.lease.release();
       for (const runtime of runtimes.toReversed()) {
         await runtime.orchestrator?.dispose();
         await runtime.orchestratorVerbs?.close();
@@ -4891,7 +5407,10 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
   }, REPORT_DELIVERY_INTERVAL_MS);
 
   let stopping = false;
-  let wake: (() => void) | undefined;
+  // The wake surface (#380): `resume` POSTs /wake, `stop` interrupts the same
+  // sleep. A wake request is sticky and coalescing — several resumes in quick
+  // succession produce one prompt pass, never an overlapping one.
+  const pace = createDispatchPace();
   const stop = (): void => {
     if (stopping) return;
     stopping = true;
@@ -4901,7 +5420,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
     // wait for (#374).
     drain.draining = true;
     log("shutting down after active workers finish");
-    wake?.();
+    pace.requestWake();
   };
   process.on("SIGINT", stop);
   process.on("SIGTERM", stop);
@@ -4921,6 +5440,10 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
         })),
         turnLimits,
         workerControls,
+        wake: () => {
+          log("dispatch wake requested — an immediate pass is starting");
+          pace.requestWake();
+        },
         health: () =>
           daemonHealth(
             runtimes.map((runtime) =>
@@ -4942,27 +5465,23 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
   );
 
   try {
-    while (!stopping) {
-      for (const runtime of runtimes) {
-        try {
-          await tick(runtime.d, workers);
-        } catch (err) {
-          log(
-            `${projects.length === 1 ? "" : `[${runtime.d.project.name}] `}` +
-              `tick failed: ${errText(err)}`,
-          );
+    await runDispatchLoop({
+      tick: async () => {
+        for (const runtime of runtimes) {
+          try {
+            await tick(runtime.d, workers);
+          } catch (err) {
+            log(
+              `${projects.length === 1 ? "" : `[${runtime.d.project.name}] `}` +
+                `tick failed: ${errText(err)}`,
+            );
+          }
         }
-      }
-      if (stopping) break;
-      await new Promise<void>((resolve) => {
-        const timer = setTimeout(resolve, TICK_INTERVAL_MS);
-        wake = () => {
-          clearTimeout(timer);
-          resolve();
-        };
-      });
-      wake = undefined;
-    }
+      },
+      shouldStop: () => stopping,
+      pace,
+      tickIntervalMs: TICK_INTERVAL_MS,
+    });
   } finally {
     process.off("SIGINT", stop);
     process.off("SIGTERM", stop);