omp-conductor 0.20.1 → 0.20.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "omp-conductor",
-  "version": "0.20.1",
+  "version": "0.20.2",
   "type": "module",
   "license": "MIT",
   "description": "A 24/7 dispatcher that takes ready GitHub issues to green, mergeable PRs using omp coding sessions, with tiered escalation first to an orchestrator session and then to a human.",

package/schema/config.schema.json

@@ -92,6 +92,18 @@
           "minimum": 0,
           "description": "Wall-clock ceiling for one worker (default 5400000)"
         },
+        "workerStallSilenceMs": {
+          "anyOf": [
+            {
+              "type": "number",
+              "minimum": 0
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "description": "How long a live worker's transcript may stay unwritten before the daemon settles it as a progress stall; null derives a third of workerWallClockMs"
+        },
         "maxAttemptsPerIssue": {
           "type": "number",
           "minimum": 0,
@@ -357,6 +369,18 @@
                 "minimum": 0,
                 "description": "Wall-clock ceiling for one worker (default 5400000)"
               },
+              "workerStallSilenceMs": {
+                "anyOf": [
+                  {
+                    "type": "number",
+                    "minimum": 0
+                  },
+                  {
+                    "type": "null"
+                  }
+                ],
+                "description": "How long a live worker's transcript may stay unwritten before the daemon settles it as a progress stall; null derives a third of workerWallClockMs"
+              },
               "maxAttemptsPerIssue": {
                 "type": "number",
                 "minimum": 0,

package/src/commands/companion.ts

@@ -53,6 +53,52 @@ export function drawFrame(lines: readonly string[], previous: number): string {
   return lines.length < previous ? `${HOME}${body}\n${ERASE_BELOW}` : `${HOME}${body}\n`;
 }
 
+/**
+ * How the pane sleeps out the interval between renders. Production is
+ * {@link timerSleep}; tests inject a fake (#1085) so no real timer runs and
+ * the wait's scheduling is asserted directly.
+ */
+export type CompanionSleep = (ms: number, signal: AbortSignal) => Promise<void>;
+
+/** The production sleep: one ref'd timeout for the whole interval, ended
+ * early by an abort. Ref'd deliberately (#1085): the pending deadline is the
+ * event loop's reason to sleep out the minute. The old wait unref'd this
+ * timer and polled `stopped` every 250 ms instead — with nothing ref'd left
+ * to wait on, the loop never slept, and the pane burned ~99% of a core
+ * between renders (~10k epoll wakes a second, measured). The unrefs' real
+ * job — never outlive the pane — belongs to cleanup, not to the handle flag:
+ * every exit path here clears the timer and detaches its listener, SIGINT
+ * and SIGTERM abort within microseconds of delivery, and a pane closed by
+ * its terminal still kills the process by signal regardless of all this.
+ * Same shape as `sleepUntilAbort` in commands/tail.ts and board.ts's
+ * `waitForInput`, which never had the spin because its deadline stayed
+ * ref'd. */
+export const timerSleep: CompanionSleep = (ms, signal) => {
+  // An already-aborted signal can never fire its listener.
+  if (signal.aborted) return Promise.resolve();
+  return new Promise<void>((resolve) => {
+    const done = (): void => {
+      clearTimeout(timer);
+      signal.removeEventListener("abort", done);
+      resolve();
+    };
+    const timer = setTimeout(done, ms);
+    signal.addEventListener("abort", done, { once: true });
+  });
+};
+/** Wait out one refresh interval, ending early the moment `stopped` aborts.
+ * Exactly one sleep is scheduled per render — the poll this replaced
+ * scheduled four a second and spun the process hot between them (#1085).
+ * A stop that has already landed ends the wait before anything is scheduled.
+ * Tests drive the wait through an injected {@link CompanionSleep}. */
+export async function waitOutRefreshInterval(
+  stopped: AbortSignal,
+  sleep: CompanionSleep = timerSleep,
+): Promise<void> {
+  if (stopped.aborted) return;
+  await sleep(COMPANION_REFRESH_SECONDS * 1_000, stopped);
+}
+
 export async function companionCommand(ctx: CommandContext): Promise<void> {
   const sub = ctx.argv[1];
   if (sub !== "decisions") {
@@ -65,14 +111,14 @@ export async function companionCommand(ctx: CommandContext): Promise<void> {
   const once = ctx.argv.includes("--once");
 
   let previous = 0;
-  let stopped = false;
+  const stopped = new AbortController();
   const stop = (): void => {
-    stopped = true;
+    stopped.abort();
   };
   process.on("SIGINT", stop);
   process.on("SIGTERM", stop);
 
-  while (!stopped) {
+  while (!stopped.signal.aborted) {
     // Opened per render, closed immediately: a companion pane lives for days,
     // and a handle held that long across daemon restarts and db snapshots is a
     // handle to a file that may no longer be the store.
@@ -86,18 +132,8 @@ export async function companionCommand(ctx: CommandContext): Promise<void> {
     process.stdout.write(drawFrame(lines, previous));
     previous = lines.length;
     if (once) return;
-    await new Promise<void>((resolve) => {
-      const timer = setTimeout(resolve, COMPANION_REFRESH_SECONDS * 1_000);
-      // Never hold the process open past a signal: the pane is closed by
-      // closing the pane, and a lingering timer would outlive it.
-      timer.unref?.();
-      const poll = setInterval(() => {
-        if (!stopped) return;
-        clearInterval(poll);
-        clearTimeout(timer);
-        resolve();
-      }, 250);
-      poll.unref?.();
-    });
+    // One ref'd sleep for the whole minute, aborted early on SIGINT/SIGTERM
+    // — see timerSleep above for why it must stay ref'd (#1085).
+    await waitOutRefreshInterval(stopped.signal);
   }
 }

package/src/commands/drain.ts

@@ -24,13 +24,15 @@ usage:
   omp-conductor drain cancel [--project NAME]
 
 A drain is a durable, self-expiring admission fence: new claims pause while
-existing runs settle, and admission resumes automatically at the absolute
+existing runs settle, an emptied fleet OPENS the release window rather than
+ending the drain (#1078), and admission resumes automatically at the absolute
 deadline — even if the orchestrator crashes. start replaces any prior drain of
 the project; --until takes an ISO instant or a relative duration (90s, 45m,
 2h, 1d) that must be bounded and in the future. status reports the active
-drain's creation time, absolute expiry, reason, and remaining active runs.
-cancel removes the project's drain and is idempotent. A drain never touches
-the pause sentinel, the arm marker, or any queue label — it is the file record
+drain's creation time, absolute expiry, reason, remaining active runs, and
+whether the window is still draining or already drained and holding. cancel
+removes the project's drain and is idempotent. A drain never touches the
+pause sentinel, the arm marker, or any queue label — it is the file record
 that expires on its own.`;
 
 /** The flags each drain subcommand accepts, after the subcommand itself. */
@@ -143,7 +145,7 @@ export async function drainCommand(ctx: CommandContext): Promise<void> {
     );
     process.stdout.write(
       dim(
-        "the drain is a durable record: it survives crashes and resumes admission at the deadline on its own",
+        "the drain is a durable record: it survives crashes, holds the window open once the fleet empties, and resumes admission at the deadline on its own",
       ) + "\n",
     );
     return;
@@ -161,6 +163,11 @@ export async function drainCommand(ctx: CommandContext): Promise<void> {
       `  expires at  ${new Date(drain.expiresAt).toISOString()}`,
       ...(drain.reason === undefined ? [] : [`  reason      ${drain.reason}`]),
       `  remaining   ${drain.remainingRuns} active run${drain.remainingRuns === 1 ? "" : "s"}`,
+      ...(drain.remainingRuns === 0
+        ? [
+            `  state       drained and holding until ${new Date(drain.expiresAt).toISOString()} — admission resumes at the deadline`,
+          ]
+        : ["  state       draining — the window holds while runs settle"]),
     ];
     process.stdout.write(`${lines.join("\n")}\n`);
     return;

package/src/commands/worker.ts

@@ -33,6 +33,10 @@ if (sub === "stop" && (reason === undefined || reason === "" || reason.length >
   );
   process.exit(2);
 }
+// A bare presence flag, parsed like every other boolean flag in this CLI:
+// the daemon refuses a stop that would strand an open PR (#1101), and this is
+// the operator's explicit record of choosing to do so anyway.
+const allowOpenPr = sub === "stop" && ctx.argv.includes("--allow-open-pr");
 const project = findProject(loadConfig(), ctx.projectFlag);
 // The pidfile is authoritative when it names a live process; a missing or
 // stale record falls back to the systemd unit, proved through its own
@@ -51,6 +55,7 @@ const response = await fetch(
       project: project.name,
       source: "cli",
       ...(reason === undefined ? {} : { reason }),
+      ...(allowOpenPr ? { allowOpenPr: true } : {}),
     }),
   },
 );

package/src/config-schema.ts

@@ -137,6 +137,14 @@ const capsSchema = z
     workerMaxTurns: z.number().min(0).describe(`Turn ceiling for one worker (default ${DEFAULT_CAPS.workerMaxTurns})`),
     workerMaxTurnsCeiling: z.number().min(0).describe(`Maximum turn budget assignable to one issue's next attempt`),
     workerWallClockMs: z.number().min(0).describe(`Wall-clock ceiling for one worker (default ${DEFAULT_CAPS.workerWallClockMs})`),
+    workerStallSilenceMs: z
+      .number()
+      .min(0)
+      .nullable()
+      .describe(
+        "How long a live worker's transcript may stay unwritten before the daemon settles it as a " +
+          "progress stall; null derives a third of workerWallClockMs",
+      ),
     maxAttemptsPerIssue: z.number().min(0).describe(`Failed attempts allowed before escalation (default ${DEFAULT_CAPS.maxAttemptsPerIssue})`),
     maxContinuationsPerIssue: z.number().min(0).describe(`Operational continuations allowed before crash/resume escalation (default ${DEFAULT_CAPS.maxContinuationsPerIssue})`),
   })

package/src/config.ts

@@ -293,6 +293,11 @@ export function resolveCaps(p: ProjectConfig, defaults: Caps): Caps {
       o.workerMaxTurnsCeiling ??
       (o.workerMaxTurns === undefined ? defaults.workerMaxTurnsCeiling : workerMaxTurns * 2),
     workerWallClockMs: o.workerWallClockMs ?? defaults.workerWallClockMs,
+    // Same `!== undefined` reading as `maxRunSpendUsd`: an explicit `null`
+    // means "derive from workerWallClockMs", not "inherit a global override"
+    // (#1086).
+    workerStallSilenceMs:
+      o.workerStallSilenceMs !== undefined ? o.workerStallSilenceMs : defaults.workerStallSilenceMs,
     maxAttemptsPerIssue: o.maxAttemptsPerIssue ?? defaults.maxAttemptsPerIssue,
     maxContinuationsPerIssue:
       o.maxContinuationsPerIssue ?? defaults.maxContinuationsPerIssue,
@@ -1109,7 +1114,9 @@ function capProblem(key: string, found: string): string {
   }
   return key === "dailySpendUsd"
     ? `caps.dailySpendUsd must be a non-negative finite number or null (no cap), found ${found}`
-    : `caps.${key} must be a non-negative finite number, found ${found}`;
+    : key === "workerStallSilenceMs"
+      ? `caps.workerStallSilenceMs must be a non-negative finite number or null (derive it from workerWallClockMs), found ${found}`
+      : `caps.${key} must be a non-negative finite number, found ${found}`;
 }
 
 /** Quted fallback key lists for policy unknown-key errors. */
@@ -1729,10 +1736,18 @@ function reconcileCaps(
       out.dailySpendUsd = null;
       continue;
     }
+    if (key === "workerStallSilenceMs" && v === null) {
+      // Null is the derive-it value (#1086): a third of the wall-clock ceiling,
+      // computed where the window is read, so it passes through untouched.
+      out.workerStallSilenceMs = null;
+      continue;
+    }
     problems.push(
       key === "dailySpendUsd"
         ? `${label}.${key} must be a non-negative finite number or null (no cap), found ${JSON.stringify(v)}`
-        : `${label}.${key} must be a non-negative finite number, found ${JSON.stringify(v)}`,
+        : key === "workerStallSilenceMs"
+          ? `${label}.${key} must be a non-negative finite number or null (derive it from workerWallClockMs), found ${JSON.stringify(v)}`
+          : `${label}.${key} must be a non-negative finite number, found ${JSON.stringify(v)}`,
     );
   }
 

package/src/daemon/drain.ts

@@ -35,6 +35,10 @@ export interface DrainRecord {
   createdAt: string;
   /** Absolute ISO deadline: admission resumes automatically at or after it. */
   expiresAt: string;
+  /** ISO instant a dispatch pass first observed an empty active set under
+   *  this drain — the moment the release window opened (#1078). Written by
+   *  the tick's drain pass, never by the operator; absent until then. */
+  drainedAt?: string;
   /** Purpose recorded at creation (release window, maintenance…). */
   reason?: string;
 }
@@ -54,6 +58,7 @@ export type DrainProblem =
   | "invalid-project"
   | "invalid-created-at"
   | "invalid-expires-at"
+  | "invalid-drained-at"
   | "expiry-not-future"
   | "invalid-reason";
 
@@ -143,6 +148,13 @@ export function readDrain(project: string, now = Date.now()): DrainVerdict {
   if (reason !== undefined && typeof reason !== "string") {
     return { kind: "error", problem: "invalid-reason" };
   }
+  const drainedAt = rec["drainedAt"];
+  if (
+    drainedAt !== undefined &&
+    (typeof drainedAt !== "string" || Number.isNaN(Date.parse(drainedAt)))
+  ) {
+    return { kind: "error", problem: "invalid-drained-at" };
+  }
   if (rec["project"] !== project) {
     // A record persisted at this project's path but naming another project is
     // either a copy or a rename mishap; it fences nobody (that project's drain
@@ -157,6 +169,7 @@ export function readDrain(project: string, now = Date.now()): DrainVerdict {
     createdAt: rec["createdAt"],
     expiresAt: rec["expiresAt"],
     ...(reason === undefined ? {} : { reason }),
+    ...(drainedAt === undefined ? {} : { drainedAt }),
   };
   return { kind: "active", drain };
 }
@@ -179,6 +192,26 @@ export function consumeDrain(project: string, now = Date.now()): DrainVerdict {
   return verdict;
 }
 
+/**
+ * Marks an active drain with the instant its active set first reached zero —
+ * the moment the release window opened (#1078). A no-op for anything that is
+ * not a fresh, unmarked fence: absent, expired, malformed and already-marked
+ * records are left exactly as they were, so only the opening transition ever
+ * writes. The rewrite is atomic (tmp + rename, exactly like
+ * {@link createDrain}), so a crash mid-write can never leave a half-record.
+ * The annotation is what makes the window's opening durable and keeps the
+ * tick's log line a once-per-drain event instead of once-per-pass noise.
+ */
+export function markDrained(project: string, now = Date.now()): void {
+  const verdict = readDrain(project, now);
+  if (verdict.kind !== "active" || verdict.drain.drainedAt !== undefined) return;
+  const marked: DrainRecord = { ...verdict.drain, drainedAt: new Date(now).toISOString() };
+  const path = drainPath(project);
+  const tmp = `${path}.${process.pid.toString(36)}.${Date.now().toString(36)}.tmp`;
+  writeFileSync(tmp, `${JSON.stringify(marked, null, 2)}\n`);
+  renameSync(tmp, path);
+}
+
 /** Removes this project's drain, idempotently — a second cancel is a no-op. */
 export function cancelDrain(project: string): void {
   rmSync(drainPath(project), { force: true });

package/src/daemon/http.ts

@@ -176,6 +176,34 @@ export async function workerControlResponse(
 
 
   const issue = Number(match[1]);
+  // The #1101 front door: stopping a live run that has already pushed a PR
+  // strands that PR — the review verb could not return it and dispatch will
+  // not re-claim an issue with an open PR. Refuse and name the PR so the
+  // operator chooses knowingly; the explicit flag records that choice. A row
+  // that is not live settles as already-terminal below, where no strand is
+  // possible, and a run that never pushed has nothing to strand.
+  const rawAllowOpenPr = Reflect.get(body, "allowOpenPr");
+  if (rawAllowOpenPr !== undefined && typeof rawAllowOpenPr !== "boolean") {
+    return Response.json({ error: "allowOpenPr must be a boolean when present" }, { status: 400 });
+  }
+  const controlled = action === "stop" ? store.latestRun(project, issue) : undefined;
+  if (
+    action === "stop" &&
+    rawAllowOpenPr !== true &&
+    controlled !== undefined &&
+    controlled.prUrl !== undefined &&
+    LIVE_STATES.includes(controlled.state)
+  ) {
+    return Response.json(
+      {
+        error:
+          `#${issue} run ${controlled.id} already pushed ${controlled.prUrl} — stopping now would strand it: ` +
+          "a stopped run cannot be returned for review, and dispatch will not re-claim an issue with an open PR. " +
+          "Pass allowOpenPr (worker stop --allow-open-pr) to stop it anyway.",
+      },
+      { status: 409 },
+    );
+  }
   const outcome =
     action === "pause"
       ? await registry.pause(project, issue, source)

package/src/daemon/review.ts

@@ -62,15 +62,15 @@ export async function dispatchReviewRevisions(d: Deps, pool?: WorkerPool): Promi
       );
       break;
     }
-    // A capped/failed run has no pushed-green settle sweep keeping its row
-    // honest (#795 review round 1): nothing transitions a `failed` / `killed`
-    // row when its PR merges or closes, so a PR that changed after the verb
-    // recorded the round would otherwise be claimed and a worker resumed
-    // against a dead PR. The settled-green origin keeps its own interlock
-    // (the settle sweep flips the row and the claim below refuses it), so
-    // only terminal-origin rounds re-read the reviewed PR fact here: the
-    // round dispatches only while the PR is still open at the exact reviewed
-    // head.
+    // A capped/failed/stopped run has no pushed-green settle sweep keeping its
+    // row honest (#795 review round 1, #1101): nothing transitions a `failed`
+    // / `killed` / `stopped` row when its PR merges or closes, so a PR that
+    // changed after the verb recorded the round would otherwise be claimed and
+    // a worker resumed against a dead PR. The settled-green origin keeps its
+    // own interlock (the settle sweep flips the row and the claim below
+    // refuses it), so every other revisable origin re-reads the reviewed PR
+    // fact here: the round dispatches only while the PR is still open at the
+    // exact reviewed head.
     //
     // The skip decision is decisive-fact only (review round 2): a definitively
     // missing PR (`GhPrMissingError` — a corroborated 404, #779), a PR that
@@ -86,7 +86,7 @@ export async function dispatchReviewRevisions(d: Deps, pool?: WorkerPool): Promi
     // #1047's launch decision rests on are only readable here.
     const originRun = d.store.getRun(revision.runId);
     const origin = originRun?.state;
-    if (origin === "failed" || origin === "killed") {
+    if (origin === "failed" || origin === "killed" || origin === "stopped") {
       let prState: PrState | undefined;
       try {
         prState = await d.tracker.prState(revision.prUrl);
@@ -161,7 +161,7 @@ export async function dispatchReviewRevisions(d: Deps, pool?: WorkerPool): Promi
     if (!d.store.claimRunForReview(revision.runId)) {
       d.store.settleReviewRevision(revision.id, "skipped", Date.now());
       log(
-        `#${revision.issue} review round ${revision.round} skipped: run ${revision.runId} is no longer pushed-green`,
+        `#${revision.issue} review round ${revision.round} skipped: run ${revision.runId} is ${origin ?? "gone"} — no longer reclaimable for review`,
       );
       continue;
     }

package/src/daemon/supervision.ts

@@ -14,12 +14,15 @@
  * pass into the dispatch path's import graph, for two functions whose subject —
  * "make the orchestrator look at this now" — is already this module's.
  */
-import { existsSync, readFileSync } from "node:fs";
+import { existsSync, readFileSync, statSync } from "node:fs";
 import { join } from "node:path";
 import { stateDir } from "../config.ts";
+import { stallSilenceMs } from "../failure-class.ts";
 import { resolvePaneHaltPath, stopConductorPane } from "../fleet.ts";
 import { errText, log, safeEscalate } from "../log.ts";
 import { STALL_MARKER_FILE, readTickRequestReason, requestImmediateTick, resolveTickConfigCwd } from "../orchestrator-tick.ts";
+import { LIVE_STATES } from "../store.ts";
+import { wakeDispatch } from "../wake.ts";
 import { NO_ISSUE, type Deps, type StallGate } from "./deps.ts";
 import { markPaged } from "./integrity.ts";
 
@@ -436,3 +439,143 @@ export function wakeOrchestratorForBlockedRun(
     );
   }
 }
+
+
+// ----------------------------------------------------------------- worker progress
+
+/**
+ * Watching live workers for silence, and settling a run that stopped making
+ * progress (#1086).
+ *
+ * The only bounds a worker's own session enforces are the turn ceiling and the
+ * wall clock — neither fires until the whole budget is gone, so a session that
+ * hangs at minute 44 keeps its slot for another 46 minutes and then dies as a
+ * `wall-clock-cap-*` verdict that reads like an oversized slice about work that
+ * may already be merged. The signal that separates "thinking" from "gone" is
+ * already on disk: a live session writes its transcript continuously, so the
+ * file's own mtime is when the worker was last demonstrably alive.
+ *
+ * Deliberately transcript-only: the row's turn counter rides in the evidence,
+ * but it is never the trigger — one turn can legitimately run for minutes on a
+ * slow provider, and this fleet has measured 1.5 min/turn.
+ *
+ * A stall whose PR is already pushed and green settles `pushed-green` — the
+ * exact state any delivered run holds — so the settle sweep owns the merge and
+ * no continuation is charged for work that landed (#1086). Every other stall
+ * settles `killed` over the observed facts; the classifier turns those into
+ * `progress-stall`, whose recovery splits on artifacts exactly as the caps do.
+ */
+export async function watchWorkerProgress(
+  d: Pick<Deps, "project" | "caps" | "store" | "tracker" | "workerControls" | "escalate">,
+  now = Date.now(),
+): Promise<void> {
+  const thresholdMs = stallSilenceMs(d.caps);
+  // A paused worker banks its wall clock and goes quiet on purpose (#938):
+  // silence under pause is the pause working, never a hang.
+  const pausedIssues = new Set(
+    d.workerControls
+      .snapshot(d.project.name)
+      .filter((w) => w.phase === "pausing" || w.phase === "paused")
+      .map((w) => w.issue),
+  );
+  for (const run of d.store.liveRuns(d.project.name)) {
+    if (pausedIssues.has(run.issue)) continue;
+    if (run.sessionFile === undefined) continue;
+    let wroteAt: number;
+    try {
+      wroteAt = Math.trunc(statSync(run.sessionFile).mtimeMs);
+    } catch {
+      // No transcript yet (claimed, session opening) or none readable: nothing
+      // to observe, which is never evidence of a stall.
+      continue;
+    }
+    // Record before judging: `status` renders this instant as the run's
+    // silence interval whether or not the threshold fires (#1086).
+    if (run.lastProgressAt !== wroteAt) {
+      d.store.updateRun(run.id, { lastProgressAt: wroteAt });
+    }
+    const silentMs = now - wroteAt;
+    if (silentMs < thresholdMs) continue;
+
+    // Re-read before settling: the session can finish between the stat above
+    // and here, and a terminal row must never be re-settled over.
+    const current = d.store.getRun(run.id);
+    if (current === undefined || !LIVE_STATES.includes(current.state)) continue;
+    const minutes = Math.round(silentMs / 60_000);
+    const silent =
+      minutes < 60 ? `${minutes}m` : `${Math.floor(minutes / 60)}h${minutes % 60 === 0 ? "" : `${minutes % 60}m`}`;
+    const why = `transcript silent ${silent} at turn ${current.turns}/${current.maxTurns}`;
+    log(`#${current.issue} ${why} — settling (silence window ${Math.round(thresholdMs / 60_000)}m)`);
+
+    // Best-effort abort through the run-control surface, so a session that can
+    // still answer exits through its own settlement path. One that cannot is
+    // exactly the case this pass exists for: the row settles below regardless,
+    // and the re-read guard keeps whichever writer lands second out.
+    try {
+      void d.workerControls.stop(d.project.name, current.issue, `progress stall: ${why}`).catch(() => {});
+    } catch {
+      // No live control for this run — the settle below is the only writer.
+    }
+
+    // Work that already landed is delivered work — but only verified green
+    // counts: landing `pushed-green` puts the row exactly where any delivered
+    // run sits, under the settle sweep's merge watch, with nothing charged.
+    let deliveredGreen = false;
+    if (current.prUrl !== undefined && current.headSha !== undefined) {
+      try {
+        deliveredGreen = (await d.tracker.verifyPr(current.prUrl, current.headSha))?.status === "green";
+      } catch {
+        deliveredGreen = false;
+      }
+    }
+    if (deliveredGreen) {
+      d.store.updateRun(current.id, { state: "pushed-green", endedAt: now });
+      log(`#${current.issue} stalled but ${current.prUrl} was already green — settled as delivered`);
+      await safeEscalate(d, {
+        tier: 1,
+        project: d.project.name,
+        issue: current.issue,
+        runId: current.id,
+        summary: `#${current.issue} went silent (${why}) with its PR already green — settled as delivered`,
+        detail: [
+          current.prUrl ?? "(no PR URL)",
+          `Branch ${current.branch} at ${current.headSha}.`,
+          `Session: ${current.sessionFile ?? "(no transcript)"}`,
+          "",
+          "The worker never reported, so there is no settlement narrative — the PR's diff is the record.",
+          "The slot is released; the settle sweep watches the merge from here.",
+        ].join("\n"),
+      });
+      await wakeDispatchLogged(d.project.name);
+      continue;
+    }
+
+    d.store.updateRun(current.id, {
+      state: "killed",
+      endedAt: now,
+      lastError: `progress-stall: ${why}`,
+    });
+    await safeEscalate(d, {
+      tier: 1,
+      project: d.project.name,
+      issue: current.issue,
+      runId: current.id,
+      summary: `[progress-stall] #${current.issue} attempt ${current.attempt}: ${why}`,
+      detail: [
+        `${current.prUrl === undefined ? "No PR" : `PR ${current.prUrl} not verified green`} — the classifier decides continue vs escalate from what attempt ${current.attempt} left.`,
+        `Branch ${current.branch}; worktree ${current.worktree === "" ? "(none)" : current.worktree}.`,
+        `Session: ${current.sessionFile ?? "(no transcript)"}`,
+      ].join("\n"),
+    });
+    await wakeDispatchLogged(d.project.name);
+  }
+}
+
+/** Best-effort dispatch wake once a stall settle freed its slot (#1086). */
+async function wakeDispatchLogged(projectName: string): Promise<void> {
+  try {
+    log(`dispatch wake after stall settle: ${await wakeDispatch(projectName)}`);
+  } catch (err) {
+    log(`dispatch wake after stall settle failed: ${errText(err)}`);
+  }
+}

package/src/daemon/tick.ts

@@ -42,13 +42,13 @@ import { writeAdmissionAck } from "./ack.ts";
 import { dispatchAdmissions, summarizeDispatch, summarizeHeldPass, type WorkerPool } from "./admission-pass.ts";
 import { NO_ISSUE, PACKAGE_SRC_DIR, UNROUTABLE_TEXT, type Deps } from "./deps.ts";
 import { handleIssue } from "./dispatch.ts";
-import { cancelDrain, consumeDrain } from "./drain.ts";
+import { consumeDrain, markDrained } from "./drain.ts";
 import { dispatchToSpecGrooming } from "./groom-pass.ts";
 import { INTEGRITY_SAMPLE, checkIntegrity, markPaged, packageManifest } from "./integrity.ts";
 import { reconcilePanes } from "./panes.ts";
 import { applyAdjudicationDispositions, dispatchReviewAdjudications, dispatchReviewRevisions } from "./review.ts";
 import { cleanupRetainedRuns, watchBaseHealth, watchMergedBase } from "./settle-pass.ts";
-import { wakeOrchestratorForMetConditions, watchOrchestrator } from "./supervision.ts";
+import { wakeOrchestratorForMetConditions, watchOrchestrator, watchWorkerProgress } from "./supervision.ts";
 
 /**
  * Record what this host has installed, once per dispatch pass (#919).
@@ -388,6 +388,16 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
   // failure happened. A pause silences claiming, not the operator's right to
   // know their supervising session stopped reading its queue.
   await watchOrchestrator(d);
+  // The worker-side sibling, and above the pause gate with it: a stalled run
+  // holds its slot on a parked fleet exactly as hard as on a busy one, and the
+  // #1086 incident was an upgrade drain blocked for 26 minutes by one silent
+  // session. Paused workers are skipped inside the pass — their silence is
+  // the pause working (#938) — so this gate costs nothing there.
+  try {
+    await watchWorkerProgress(d);
+  } catch (err) {
+    log(`worker-progress sweep failed: ${errText(err)}`);
+  }
   // The down incident is reconciled the same place and for the same reason: a
   // session that has actually died is as much the operator's concern as one
   // that is wedged, and restarting it is the daemon's restart either way. This
@@ -584,11 +594,16 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
   // the same admission boundary — settlement above it, nothing claimed below —
   // but the intent is durable (it survives orchestrator loss) and bounded (the
   // record carries an absolute deadline, so a crash can never strand
-  // admission). Three shapes, three behaviours:
+  // admission). Reaching an empty active set SUCCEEDS the drain; it does not
+  // end it (#1078): the release window the drain exists to create opens at
+  // that moment and holds claims down until the deadline — exactly the quiet
+  // interval a tick-driven orchestrator needs to cut a release, which the old
+  // eight-second auto-clear destroyed before any tick could use it. Four
+  // shapes, four behaviours:
   //   - fresh drain with live runs → a held pass, exactly like a pause;
-  //   - fresh drain with nothing left to wait for → the drain is satisfied
-  //     and clears itself, so a completed drain never needs a second operator
-  //     action and this pass proceeds normally;
+  //   - fresh drain whose active set just reached zero → the opening is marked
+  //     on the record once and logged once, and this pass admits nothing;
+  //   - fresh drain already marked drained → the same held pass, silent;
   //   - malformed record → fails closed for THIS pass (it might be a fresh
   //     fence we cannot read), and the same consume removed it, so it can
   //     never become an unbounded permanent drain.
@@ -596,15 +611,17 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
   if (drain.kind === "active") {
     // Completion is the ACTIVE set, not the live-worker set: pushed-pending
     // and pushed-green PRs still make the `runs-settled` release gate fail, so
-    // a drain that cleared while one remained would admit work on top of a
-    // batch the releases still see as unfinished (#776 review #2).
-    if (d.store.activeRuns(d.project.name).length === 0) {
-      cancelDrain(d.project.name);
-      log("project drain completed: no active runs remain — drain cleared");
-    } else {
-      d.store.recordDispatch(d.project.name, summarizeHeldPass(settled));
-      return;
+    // a window declared open over a remaining run would admit work on top of
+    // a batch the releases still see as unfinished (#776 review #2).
+    if (d.store.activeRuns(d.project.name).length === 0 && drain.drain.drainedAt === undefined) {
+      markDrained(d.project.name);
+      log(
+        `project drain: fleet drained — release window open until ${drain.drain.expiresAt}` +
+          " (claims stay paused; admission resumes at the deadline or on drain cancel)",
+      );
     }
+    d.store.recordDispatch(d.project.name, summarizeHeldPass(settled));
+    return;
   } else if (drain.kind === "error") {
     log(`project drain record invalid (${drain.problem}) — removed; this pass admits nothing`);
     d.store.recordDispatch(d.project.name, summarizeHeldPass(settled));