omp-conductor 0.9.1 → 0.12.0

package/src/daemon.ts

@@ -35,7 +35,7 @@ import { recordReleaseBlock } from "./release-policy.ts";
 import { branchName, effectiveLabels, route } from "./routing.ts";
 import type { Routed, UnroutableReason } from "./routing.ts";
 import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
-import { classifyRun, type ClassifyFacts } from "./failure-class.ts";
+import { classifyRun, providerCreditRefusal, providerTransientFault, type ClassifyFacts } from "./failure-class.ts";
 import { projectLabels } from "./label-projection.ts";
 import { dbPath, openStore, utcDay } from "./store.ts";
 import { makeTracker, type RateLimitStatus } from "./tracker/github.ts";
@@ -61,7 +61,14 @@ import type {
   Tracker,
   VerbLedgerEntry,
 } from "./types.ts";
-import { type KilledBy, type WorkerResult, renderBrief, runWorker } from "./worker.ts";
+import {
+  type KilledBy,
+  type WorkerPauseControl,
+  type WorkerPausePhase,
+  type WorkerResult,
+  renderBrief,
+  runWorker,
+} from "./worker.ts";
 import {
   addRunRepo,
   cleanupRetainedWorktree,
@@ -86,7 +93,7 @@ import {
 } from "./verbs/socket.ts";
 import { homedir } from "node:os";
 
-import { pushRunBranch, type RunRepoRef } from "./gitops.ts";
+import { pushRunBranch, readBaseChain, type RunRepoRef } from "./gitops.ts";
 import {
   planUsageLine,
   readPlanUsage,
@@ -108,6 +115,11 @@ const REPORT_DELIVERY_INTERVAL_MS = 30_000;
  *  means the mirror itself is broken, not unlucky, and the sweep escalates
  *  instead of burning a turn-0 run per tick forever (#168, #177). */
 const DISPATCH_INFRA_MAX_STRIKES = 3;
+/** A provider-transient requeue (stream stalled mid-run) is retried, but only a
+ *  bounded number of times: three aborted streams for one issue means the
+ *  provider itself is degraded, not unlucky, and the sweep escalates to a
+ *  human instead of requeueing into a down provider forever (#220). */
+const PROVIDER_TRANSIENT_MAX_STRIKES = 3;
 const DEFAULT_PORT = 8787;
 const BRIEF_TEMPLATE_PATH = join(import.meta.dir, "briefs", "worker.md");
 
@@ -144,6 +156,7 @@ interface Deps {
   usage: UsageSource;
   escalate(e: Escalation): Promise<void>;
   turnLimits: TurnLimitRegistry;
+  workerControls: WorkerControlRegistry;
   integrity: IntegrityGate;
   stall: StallGate;
   cleanup?: RetainedCleanupCursor;
@@ -190,6 +203,7 @@ export function verbDeps(d: Pick<Deps, "project" | "store" | "tracker" | "verbAc
     pausedAt,
     log,
     now: () => Date.now(),
+    chain: { readBaseChain },
   };
 }
 
@@ -264,6 +278,7 @@ async function watchOrchestrator(d: Deps): Promise<void> {
   log(`ERROR: the orchestrator session is not draining its queue — ${verdict.since ?? "no timestamp"}`);
   const delivered = await safeEscalate(d, {
     tier: 2,
+    category: "confirmed-failure",
     project: d.project.name,
     issue: NO_ISSUE,
     // Keyed on the marker's own timestamp, not the date. The dedup ledger keys
@@ -823,6 +838,91 @@ export function createTurnLimitRegistry(
   };
 }
 
+export type WorkerControlResult =
+  | { kind: "ok"; runId: string; phase: WorkerPausePhase }
+  | { kind: "refused"; runId: string; error: string }
+  | { kind: "not-active" };
+
+export interface WorkerControlSlot {
+  install(control: WorkerPauseControl): void;
+  close(): void;
+}
+
+export interface WorkerControlRegistry {
+  open(project: string, issue: number, runId: string): WorkerControlSlot;
+  pause(project: string, issue: number): Promise<WorkerControlResult>;
+  resume(project: string, issue: number): WorkerControlResult;
+  /** Live runs whose phase is not `running` — what /healthz and the board show. */
+  snapshot(project: string): { issue: number; runId: string; phase: WorkerPausePhase }[];
+}
+
+/** Authoritative controls for sessions owned by this daemon process. */
+export function createWorkerControlRegistry(): WorkerControlRegistry {
+  const active = new Map<
+    string,
+    { project: string; issue: number; runId: string; control?: WorkerPauseControl }
+  >();
+  const key = (project: string, issue: number): string => `${project}\0${issue}`;
+  return {
+    open(project, issue, runId) {
+      const k = key(project, issue);
+      if (active.has(k)) throw new Error(`#${issue} already has a live worker controller`);
+      const entry = { project, issue, runId } as {
+        project: string;
+        issue: number;
+        runId: string;
+        control?: WorkerPauseControl;
+      };
+      active.set(k, entry);
+      return {
+        install: (control) => {
+          if (active.get(k) === entry) entry.control = control;
+        },
+        close: () => {
+          if (active.get(k) === entry) active.delete(k);
+        },
+      };
+    },
+    async pause(project, issue) {
+      const entry = active.get(key(project, issue));
+      if (entry?.control === undefined) return { kind: "not-active" };
+      try {
+        await entry.control.pause();
+        return { kind: "ok", runId: entry.runId, phase: entry.control.phase() };
+      } catch (err) {
+        return {
+          kind: "refused",
+          runId: entry.runId,
+          error: err instanceof Error ? err.message : String(err),
+        };
+      }
+    },
+    resume(project, issue) {
+      const entry = active.get(key(project, issue));
+      if (entry?.control === undefined) return { kind: "not-active" };
+      try {
+        entry.control.resume();
+        return { kind: "ok", runId: entry.runId, phase: entry.control.phase() };
+      } catch (err) {
+        return {
+          kind: "refused",
+          runId: entry.runId,
+          error: err instanceof Error ? err.message : String(err),
+        };
+      }
+    },
+    snapshot(project) {
+      const workers: { issue: number; runId: string; phase: WorkerPausePhase }[] = [];
+      for (const entry of active.values()) {
+        if (entry.project !== project || entry.control === undefined) continue;
+        const phase = entry.control.phase();
+        if (phase !== "running") workers.push({ issue: entry.issue, runId: entry.runId, phase });
+      }
+      return workers;
+    },
+  };
+}
+
 export async function verifyPushedGreenClaim(
   tracker: Pick<Tracker, "verifyPr">,
   claim: Pick<WorkerResult, "prUrl" | "headSha">,
@@ -926,6 +1026,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
   const mirrorPath = mirrorPathFor(r.repo, project.mirrorRoot);
   let worktreePath: string | undefined;
   let turnLimit: TurnLimitController | undefined;
+  let workerControl: WorkerControlSlot | undefined;
   // The run's own repository. Hoisted for the same reason `worktreePath` is —
   // the catch and finally paths have to publish the branch.
   let runRepo: RunRepoRef | undefined;
@@ -981,6 +1082,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
     store.enqueueLabelOps(project.name, [{ issue, op: "add", label: inProgress }]);
     claimed = true;
     turnLimit = d.turnLimits.open(project.name, issue, runId, caps.workerMaxTurns);
+    workerControl = d.workerControls.open(project.name, issue, runId);
 
     // A run's tree is <workspaceRoot>/<issue> and addRunRepo refuses to reuse
     // an existing path, so a retry — or a tree kept from a failed attempt — has
@@ -1057,6 +1159,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
         cwd: worktreePath,
         caps,
         maxTurns: () => turnLimit?.maxTurns() ?? caps.workerMaxTurns,
+        onPauseControl: (control) => workerControl?.install(control),
         sessionDir,
         // The session's control socket, under the daemon's own state directory —
         // a child process of the daemon reaches it directly.
@@ -1081,6 +1184,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
         onKilled: () => {
           turnLimit?.close();
           turnLimit = undefined;
+          workerControl?.close();
+          workerControl = undefined;
         },
         // Recorded the moment the session opens its transcript, not when the run
         // ends: `omp-conductor tail` resolves an issue to a file through this row,
@@ -1093,6 +1198,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
       // PR verification or terminal row writes can leave stale `running` state.
       turnLimit?.close();
       turnLimit = undefined;
+      workerControl?.close();
+      workerControl = undefined;
     }
 
     // A configured model the harness could not honour means this run was done by
@@ -1108,6 +1215,13 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
         : { state: result.state };
     const state = verified.state;
 
+    // Read before the row is written so `lastError` carries the provider's own
+    // message — it names the fix and the URL, which no classification can.
+    const sessionErr = state === "failed" || state === "killed" ? readSessionError(result.sessionFile) : undefined;
+    const providerCredit = sessionErr === undefined ? undefined : providerCreditRefusal(sessionErr);
+    const providerTransient =
+      providerCredit !== undefined || sessionErr === undefined ? undefined : providerTransientFault(sessionErr);
+
     // The other half of not believing a worker about its own run (#128). The
     // claim being audited is `state: pushed-green`, so the audit runs on the
     // worker's claim rather than on what verification made of it: a claim that
@@ -1192,7 +1306,11 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
       // push: the report of a killed attempt is exactly the one a later
       // continuation must pool its disclosures from (#199).
       report: result.report,
-      ...(verified.reason === undefined ? {} : { lastError: verified.reason }),
+      ...(providerCredit !== undefined || providerTransient !== undefined
+        ? { lastError: providerCredit ?? providerTransient }
+        : verified.reason === undefined
+          ? {}
+          : { lastError: verified.reason }),
       ...settlement?.patch,
       ...(audit === undefined || audit.flags.length === 0
         ? {}
@@ -1224,7 +1342,33 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
         maxContinuations: caps.maxContinuationsPerIssue,
       });
 
-      if (continueTurns) {
+      if (providerCredit !== undefined) {
+        // Pause here rather than at classification: the sweep runs on the tick,
+        // and three issues each burned an attempt in the fifteen minutes between
+        // the first 402 and a human noticing (#220).
+        setPaused(true, { source: "provider-credit", reason: providerCredit });
+        log(`#${issue} provider refused for credit — dispatch paused: ${providerCredit}`);
+        swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
+        // Fleet-scoped and run-independent on purpose. The notification ledger
+        // dedupes on `project:issue:tier:summary`, so `NO_ISSUE` plus a summary
+        // carrying no run or attempt is what makes this page once for the fleet
+        // instead of once per affected run.
+        await safeEscalate(d, {
+          tier: 2,
+          category: "fleet-stopped",
+          project: project.name,
+          issue: NO_ISSUE,
+          summary: `Model provider refused for credit — ${project.name} is paused`,
+          detail: [
+            providerCredit,
+            "",
+            "No implementation attempt was charged: this is a billing state, not a",
+            "failed implementation. Each affected issue keeps its queue label and",
+            "re-dispatches on `omp-conductor resume` once the provider has credit.",
+            `Session: ${result.sessionFile ?? "(no transcript)"}`,
+          ].join("\n"),
+        });
+      } else if (continueTurns) {
         // Requeue as one ordered pair: the in-progress removal before the
         // queue add, exactly the order the projector will apply them in (#201).
         store.enqueueLabelOps(project.name, [
@@ -1325,6 +1469,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
     // inner settlement guard exists. Latch it before any terminal write or await.
     turnLimit?.close();
     turnLimit = undefined;
+    workerControl?.close();
+    workerControl = undefined;
     const detail = errText(err);
     log(`#${issue} errored: ${detail}`);
     // A crash lands anywhere, including mid-edit in a tree holding the only
@@ -1370,6 +1516,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
     // a failure path, and whatever it still held is now a commit on the branch.
   } finally {
     turnLimit?.close();
+    workerControl?.close();
+    workerControl = undefined;
     // The run is over, so its channel is too. Closed here rather than beside
     // the session so the crash path closes it as well: a listener left bound
     // after its run settled is a socket whose `run-not-live` check is the only
@@ -1691,6 +1839,7 @@ const DEGRADED_HOLDS: ReadonlySet<AdmissionHoldReason> = new Set([
 export function summarizeDispatch(
   ready: number,
   routed: number,
+  claimed: number,
   admitted: number,
   holds: readonly AdmissionHold[],
   completedAt = Date.now(),
@@ -1706,6 +1855,7 @@ export function summarizeDispatch(
     completedAt,
     ready,
     routed,
+    claimed,
     admitted,
     degraded: holds.some((hold) => DEGRADED_HOLDS.has(hold.reason)),
     holds: [...groups]
@@ -1753,6 +1903,7 @@ function planUsageEscalation(project: string, plan: PlanUsageStatus): Escalation
     return {
       ...base,
       tier: 2,
+      category: "fleet-stopped",
       // Dated: a meter that breaks again next month is a new incident, not a
       // repeat of this one.
       summary: `Plan usage source unreadable — ${project} is not claiming new work (${new Date().toISOString().slice(0, 10)})`,
@@ -1771,6 +1922,7 @@ function planUsageEscalation(project: string, plan: PlanUsageStatus): Escalation
     return {
       ...base,
       tier: 2,
+      category: "fleet-stopped",
       summary: `Plan usage cap names an unusable window "${plan.cap?.windowId ?? "?"}" — ${project} is not claiming new work`,
       detail: [
         plan.detail,
@@ -2245,6 +2397,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
     if (integrity.page) {
       const delivered = await safeEscalate(d, {
         tier: 2,
+        category: "fleet-stopped",
         project: project.name,
         issue: NO_ISSUE,
         // Dated for the same reason the spend cap is: the dedup key is the
@@ -2288,6 +2441,10 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
     return pending.length === 0 ? issue : { ...issue, labels: effectiveLabels(issue.labels, pending) };
   });
   const { routed, unroutable } = route(effective, project);
+  // Ready issues that route() dropped without a reason carry a state label —
+  // claimed by a live or settling run — so the claimed count is subtraction
+  // rather than a routed-state signal (#228).
+  const claimed = effective.length - routed.length - unroutable.length;
   const routingHolds: AdmissionHold[] = unroutable.map((u) => ({
     issue: u.issue.number,
     reason: `unroutable:${u.reason}`,
@@ -2295,7 +2452,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
   const recordDispatch = (admitted: number, holds: readonly AdmissionHold[]): void => {
     store.recordDispatch(
       project.name,
-      summarizeDispatch(ready.length, routed.length, admitted, holds),
+      summarizeDispatch(ready.length, routed.length, claimed, admitted, holds),
     );
   };
 
@@ -2329,6 +2486,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
     setPaused(true, { source: "spend-cap", reason: `daily spend reached $${caps.dailySpendUsd}` });
     await safeEscalate(d, {
       tier: 2,
+      category: "fleet-stopped",
       project: project.name,
       issue: NO_ISSUE,
       // Dated so the same cap pages again tomorrow, but only once per day.
@@ -2394,6 +2552,8 @@ export interface DaemonHealthSnapshot {
   rssBytes: number;
   dispatch?: DispatchSummary;
   codeGraph?: CodeGraphHealth;
+  /** Live workers in a non-running pause phase; absent/empty = nothing paused. */
+  workers?: { issue: number; runId: string; phase: WorkerPausePhase }[];
 }
 
 export function daemonHealthSnapshot(
@@ -2402,6 +2562,7 @@ export function daemonHealthSnapshot(
   paused = isPaused(),
   rssBytes = process.memoryUsage().rss,
   codeGraph?: CodeGraphHealth,
+  workerControls?: WorkerControlRegistry,
 ): DaemonHealthSnapshot {
   const dispatch = store.latestDispatch(project);
   return {
@@ -2412,6 +2573,7 @@ export function daemonHealthSnapshot(
     rssBytes,
     ...(dispatch === undefined ? {} : { dispatch }),
     ...(codeGraph?.configured === true ? { codeGraph } : {}),
+    ...(workerControls === undefined ? {} : { workers: workerControls.snapshot(project) }),
   };
 }
 
@@ -2476,10 +2638,70 @@ export async function turnLimitResponse(
   );
 }
 
+export async function workerControlResponse(
+  req: Request,
+  project: string,
+  store: Pick<Store, "latestRun">,
+  registry: WorkerControlRegistry,
+): Promise<Response | undefined> {
+  const url = new URL(req.url);
+  const match = /^\/runs\/(\d+)\/(pause|resume)$/.exec(url.pathname);
+  if (req.method !== "PUT" || match === null) return undefined;
+  if (!req.headers.get("content-type")?.startsWith("application/json")) {
+    return Response.json({ error: "content-type must be application/json" }, { status: 415 });
+  }
+
+  let body: unknown;
+  try {
+    body = await req.json();
+  } catch {
+    return Response.json({ error: "request body must be valid JSON" }, { status: 400 });
+  }
+  if (body === null || typeof body !== "object") {
+    return Response.json({ error: "request body must be a JSON object" }, { status: 400 });
+  }
+  const requestedProject = Reflect.get(body, "project");
+  if (typeof requestedProject !== "string" || requestedProject.length === 0) {
+    return Response.json({ error: "project must be a non-empty string" }, { status: 400 });
+  }
+  if (requestedProject !== project) {
+    return Response.json(
+      { error: `daemon serves project "${project}", not requested project "${requestedProject}"` },
+      { status: 409 },
+    );
+  }
+
+  const issue = Number(match[1]);
+  const outcome =
+    match[2] === "pause"
+      ? await registry.pause(project, issue)
+      : registry.resume(project, issue);
+  if (outcome.kind === "ok") {
+    return Response.json({ runId: outcome.runId, phase: outcome.phase });
+  }
+  if (outcome.kind === "refused") {
+    return Response.json({ error: `#${issue}: ${outcome.error}` }, { status: 409 });
+  }
+
+  const latest = store.latestRun(project, issue);
+  if (latest === undefined) {
+    return Response.json({ error: `no run recorded for #${issue}` }, { status: 404 });
+  }
+  return Response.json(
+    {
+      error:
+        `#${issue} has no live worker controller; its session already settled ` +
+        `or belongs to another daemon (stored state: ${latest.state})`,
+    },
+    { status: 409 },
+  );
+}
+
 export interface DaemonHttpDeps {
   project: string;
   store: Pick<Store, "latestRun">;
   turnLimits: TurnLimitRegistry;
+  workerControls: WorkerControlRegistry;
   health: () => DaemonHealthSnapshot;
 }
 
@@ -2487,17 +2709,18 @@ export interface DaemonHttpDeps {
  * The whole HTTP surface, in one named function so a test can pin what is *not*
  * on it.
  *
- * Two routes: a health read, and the turn-limit control. Everything else is
- * 404, and that is the contract — see the prohibition at the `Bun.serve` call
- * for why no mutation may be added here. Extracted from the serve callback
- * precisely so "the daemon's HTTP port exposes no mutation route" is something
- * a test asserts rather than something a reviewer has to notice (#126).
+ * Three route families: the health read, turn-limit control, and worker
+ * pause/resume control. Everything else is 404. The controls mutate only
+ * daemon-owned live sessions; tracker and repository mutations stay on the
+ * authenticated per-run channel described at the `Bun.serve` call (#126).
  */
 export async function daemonHttpResponse(req: Request, d: DaemonHttpDeps): Promise<Response> {
   const url = new URL(req.url);
   if (req.method === "GET" && url.pathname === "/healthz") return Response.json(d.health());
-  const control = await turnLimitResponse(req, d.project, d.store, d.turnLimits);
-  return control ?? new Response("not found\n", { status: 404 });
+  const turnLimit = await turnLimitResponse(req, d.project, d.store, d.turnLimits);
+  if (turnLimit !== undefined) return turnLimit;
+  const workerControl = await workerControlResponse(req, d.project, d.store, d.workerControls);
+  return workerControl ?? new Response("not found\n", { status: 404 });
 }
 
 export interface StatusSnapshot {
@@ -2505,6 +2728,12 @@ export interface StatusSnapshot {
   configPath: string;
   stateDir: string;
   paused: boolean;
+  /**
+   * Why the fleet is paused, when whatever paused it said. Rendered beside
+   * `(PAUSED)` so a pause an operator did not issue names itself instead of
+   * reading like a mistake (#220).
+   */
+  pauseReason?: string;
   caps: Caps;
   /**
    * The effective per-shape release grants. On the snapshot rather than re-read
@@ -2579,11 +2808,15 @@ export function statusSnapshotFromStore(
   const dispatch = store.latestDispatch(p.name);
   const labelOpsPending = store.countPendingLabelOps(p.name);
   const oldestLabelOpAt = store.oldestPendingLabelOpAt(p.name);
+  // Read once: the provenance read touches the filesystem, and the renderer
+  // should never pay for it twice per status.
+  const reason = pauseProvenance()?.reason;
   return {
     project: p.name,
     configPath: configPath(),
     stateDir: stateDir(),
     paused: isPaused(),
+    ...(reason === undefined ? {} : { pauseReason: reason }),
     caps,
     releaseGrants: resolveReleaseGrants(p),
     activeRuns: store.activeRuns(p.name),
@@ -2623,7 +2856,7 @@ export function formatDispatchSummary(summary?: DispatchSummary): string {
   if (summary === undefined) return "last dispatch  (none recorded)";
   const lines = [
     `last dispatch  ${new Date(summary.completedAt).toISOString()}${summary.degraded ? "  DEGRADED" : ""}`,
-    `  candidates    ${summary.ready} ready / ${summary.routed} routed`,
+    `  candidates    ${summary.ready} ready / ${summary.claimed ?? 0} in flight / ${summary.routed} spare`,
     `  admitted      ${summary.admitted}`,
   ];
   if (summary.holds.length === 0) {
@@ -2857,6 +3090,57 @@ export function lastToolCalls(sessionFile: string | undefined, limit = SPIN_EVID
   return names.slice(-limit);
 }
 
+/** A provider refusal a session recorded before dying, or undefined for none. */
+export interface SessionError {
+  status?: number;
+  message: string;
+}
+
+/**
+ * The last error a transcript recorded, or undefined when it recorded none.
+ *
+ * The harness writes `{"stopReason":"error","errorStatus":402,"errorId":402,
+ * "errorMessage":"402 This request requires more credits, ..."}`. The daemon
+ * read none of it, so three runs died `unknown` with an empty `lastError` and
+ * charged an attempt each for a billing state (#220).
+ *
+ * Scanned newest-first: a session that recovered from an early error and then
+ * died of something else must report the something else, and a session that
+ * recovered from its only error and finished cleanly reports the error anyway
+ * because there is no terminal verdict to outrank it (#220).
+ */
+export function readSessionError(sessionFile: string | undefined): SessionError | undefined {
+  if (sessionFile === undefined) return undefined;
+  let text: string;
+  try {
+    text = readFileSync(sessionFile, "utf8");
+  } catch {
+    return undefined;
+  }
+  const lines = text.split("\n");
+  for (let i = lines.length - 1; i >= 0; i -= 1) {
+    const line = lines[i];
+    if (line === undefined || line.length === 0) continue;
+    let row: unknown;
+    try {
+      row = JSON.parse(line) as unknown;
+    } catch {
+      continue;
+    }
+    if (row === null || typeof row !== "object") continue;
+    const rec = row as { readonly [key: string]: unknown };
+    if (rec["stopReason"] !== "error") continue;
+    const message = rec["errorMessage"];
+    if (typeof message !== "string" || message.trim() === "") continue;
+    const status = rec["errorStatus"];
+    return {
+      ...(typeof status === "number" && Number.isFinite(status) ? { status } : {}),
+      message: message.trim(),
+    };
+  }
+  return undefined;
+}
+
 /**
  * Classify every unclassified terminal run, persist the verdict, and perform the
  * one recovery its class names (#132).
@@ -3004,6 +3288,28 @@ async function recoverRun(
       log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
       return;
     }
+    // Same bound for provider-transient: an issue whose stream keeps stalling
+    // mid-run is requeued free (no attempt, no continuation charged) — but a
+    // provider that aborts three times for one issue is down, and a human has
+    // to check its status before hand-requeueing (#220).
+    if (
+      cls === "provider-transient" &&
+      store.classCountFor(project.name, run.issue, "provider-transient") >= PROVIDER_TRANSIENT_MAX_STRIKES
+    ) {
+      await safeEscalate(d, {
+        tier: 1,
+        project: project.name,
+        issue: run.issue,
+        summary: `[provider-transient] #${run.issue}: the provider keeps aborting mid-stream — ${evidence}`,
+        detail: [
+          `The provider aborted the stream for #${run.issue} ${PROVIDER_TRANSIENT_MAX_STRIKES} times without the run ever producing a verdict (0 tokens billed each time).`,
+          "Check provider status before requeueing by hand.",
+        ].join("\n"),
+      });
+      store.updateRun(run.id, { recoveredAt: Date.now() });
+      log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
+      return;
+    }
     // Only when the tracker still shows this issue as ours to hand back. An
     // issue that is closed, or has no state label, was resolved by another route
     // and requeueing it would dispatch work nobody asked for.
@@ -3421,6 +3727,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
   const turnLimits = createTurnLimitRegistry((runId, maxTurns) => {
     store.updateRun(runId, { maxTurns });
   });
+  const workerControls = createWorkerControlRegistry();
   const d: Deps = {
     project,
     caps,
@@ -3431,6 +3738,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
     usage: sharedUsageSource(),
     escalate: (e) => escalator.escalate(e),
     turnLimits,
+    workerControls,
     integrity,
     // Fresh per daemon run, like the integrity gate: a restart is entitled to
     // page again about a stall that is still on disk.
@@ -3507,14 +3815,15 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
   process.on("SIGINT", stop);
   process.on("SIGTERM", stop);
 
-  // ── NO MUTATION ROUTE BELONGS ON THIS PORT ────────────────────────────────
+  // ── NO TRACKER OR REPOSITORY MUTATION BELONGS ON THIS PORT ───────────────
   //
   // This is unauthenticated loopback TCP. Every local user can reach it, it
   // carries no credential of any kind, and it cannot tell one caller from
-  // another: `127.0.0.1` is not an identity. `turnLimitResponse` already shows
-  // what that costs — it trusts a body-supplied `project`, which is exactly the
-  // shape "identity from the payload" takes when nobody is watching, and it is
-  // tolerable only because raising a turn ceiling is bounded and reversible.
+  // another: `127.0.0.1` is not an identity. The turn-limit and worker
+  // pause/resume controls trust a body-supplied `project`, which is exactly the
+  // shape "identity from the payload" takes when nobody is watching. Those
+  // controls are tolerable only because they are bounded, live-run-local, and
+  // reversible.
   //
   // A merge, a push, a release or a label is none of those things. Do not add
   // one here, and do not add "just a small one" behind a shared secret either:
@@ -3530,7 +3839,9 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
         project: project.name,
         store,
         turnLimits,
-        health: () => daemonHealthSnapshot(store, project.name, undefined, undefined, codeGraph),
+        workerControls,
+        health: () =>
+          daemonHealthSnapshot(store, project.name, undefined, undefined, codeGraph, workerControls),
       }),
   });
   log(`serving /healthz on :${server.port}, project ${project.name}`);