@tonoid/agent-loop 1.0.0 → 1.1.0

package/src/router/providers/claude.ts

@@ -1,7 +1,8 @@
 import type { AccountConfig, AccountUsage, UsageReader, Window } from "../../types"
 import { CLAUDE_WINDOW_MINUTES, checkWindows } from "../window"
 import { expandHome } from "../../paths"
-import { renameSync, writeFileSync, readFileSync, unlinkSync, statSync } from "node:fs"
+import { renameSync, writeFileSync, readFileSync, unlinkSync, statSync, mkdirSync } from "node:fs"
+import { dirname } from "node:path"
 
 export const USAGE_URL = "https://api.anthropic.com/api/oauth/usage"
 export const TOKEN_URL = "https://console.anthropic.com/v1/oauth/token"
@@ -11,6 +12,15 @@ export const DEFAULT_CLIENT_ID = "9d1c250a-e61b-44d9-88ed-5944d1962f5e"
 // Access tokens live 8 hours. Refresh inside the last 10 minutes so a tick never
 // starts a read against a token that expires mid-flight.
 export const REFRESH_MARGIN_MS = 10 * 60_000
+// The usage endpoint answers about 5 calls per token per minute, and that
+// budget is not ours alone: an interactive claude spends it on its status
+// line, `c` spends it on its account table, and two workspaces tick in the
+// same minute. So the answer is cached in the account's own config dir, in
+// the format `c` reads and writes, and whoever asks first pays for everyone.
+export const USAGE_TTL_MS = 60_000
+// A 429 is that burst limit, not a quota. Asking again a minute later just
+// draws from the same empty bucket, so hold off longer.
+export const USAGE_429_TTL_MS = 5 * 60_000
 
 export interface Creds {
   accessToken: string
@@ -22,7 +32,46 @@ export interface Creds {
 export interface ClaudeDeps {
   readCreds(configDir: string): Promise<Creds | null>
   refresh(c: Creds, clientId: string, configDir: string): Promise<Creds>
-  getUsage(token: string): Promise<{ status: number; body: any }>
+  // configDir only so the live implementation can share its answer through
+  // that account's cache. A reader that has its own source ignores it.
+  getUsage(token: string, configDir: string): Promise<{ status: number; body: any }>
+}
+
+export interface CachedUsage {
+  at: number
+  status: number
+  body: any
+}
+
+export const usageCachePath = (configDir: string): string =>
+  `${expandHome(configDir)}/cache/usage.json`
+
+export function readUsageCache(configDir: string, now: number): CachedUsage | null {
+  let c: CachedUsage
+  try {
+    c = JSON.parse(readFileSync(usageCachePath(configDir), "utf8"))
+  } catch {
+    // Absent, or half-written by a reader that lost the race. Both mean ask.
+    return null
+  }
+  if (!c?.at || typeof c.status !== "number") return null
+  return now - c.at < (c.status === 429 ? USAGE_429_TTL_MS : USAGE_TTL_MS) ? c : null
+}
+
+// Body verbatim, because the readers of this file want different parts of it:
+// `limits[]` here, `five_hour` in c. Neither has to know about the other.
+export function writeUsageCache(configDir: string, entry: CachedUsage): void {
+  const path = usageCachePath(configDir)
+  // Unique per writer, for the reason spelled out in writeCreds.
+  const tmp = `${path}.${process.pid}.tmp`
+  try {
+    mkdirSync(dirname(path), { recursive: true })
+    writeFileSync(tmp, JSON.stringify(entry), { mode: 0o600 })
+    renameSync(tmp, path)
+  } catch {
+    // A cache we cannot write costs a call, not a tick.
+    try { unlinkSync(tmp) } catch {}
+  }
 }
 
 // Atomic: a torn credentials file locks the account out until a human logs in
@@ -53,8 +102,21 @@ export function writeCreds(configDir: string, creds: Creds): void {
   }
 }
 
-const no = (reason: string, exhausted?: boolean): AccountUsage =>
-  exhausted ? { readable: false, reason, exhausted } : { readable: false, reason }
+const no = (reason: string): AccountUsage => ({ readable: false, reason })
+
+// The endpoint was busy or briefly broken and the account is fine, so an older
+// reading may still price it. Distinct from every other failure here: missing
+// credentials and a refused refresh are facts about the account, and a worker
+// started on one of those meets "Invalid API key, please run /login" instead of
+// its brief.
+const busy = (reason: string): AccountUsage => ({ readable: false, reason, transient: true })
+
+// A window that has never started has nothing to reset, so a brand new login
+// answers 200 with no window carrying a resets_at. That is an account with no
+// usage yet, not a broken one, and the router admits one worker on it rather
+// than skipping it forever: nothing else on the box will record the first
+// window for it.
+const fresh = (reason: string): AccountUsage => ({ readable: false, reason, fresh: true })
 
 function resetsAtOf(raw: unknown): Date | null {
   if (raw === null || raw === undefined) return null
@@ -80,20 +142,23 @@ export function makeClaudeReader(d: ClaudeDeps): UsageReader {
       }
     }
 
-    let res = await d.getUsage(creds.accessToken)
+    let res = await d.getUsage(creds.accessToken, a.configDir)
     if (res.status === 401) {
       try {
         creds = await d.refresh(creds, clientId, a.configDir)
       } catch (err) {
         return no(`refresh after 401 failed: ${err}`)
       }
-      res = await d.getUsage(creds.accessToken)
+      res = await d.getUsage(creds.accessToken, a.configDir)
       if (res.status === 401) return no("401 after refresh")
     }
-    // Exhausted is strictly more information than unknown, and unlike every
-    // other unreadable state allowWhenUnreadable must not resurrect it.
-    if (res.status === 429) return no("429 from the usage endpoint", true)
-    if (res.status !== 200) return no(`usage endpoint ${res.status}`)
+    // Unreadable, and nothing stronger. A 429 here is the metering endpoint's
+    // own burst budget, which anything else polling this account can spend on
+    // our behalf, and it says nothing about quota: an account measured at 3%
+    // of its session and 39% of its week answers 429 on the fifth call in a
+    // minute. Real exhaustion arrives as a 200 with the windows at 100%.
+    if (res.status === 429) return busy("429 from the usage endpoint (burst limit, not quota)")
+    if (res.status !== 200) return busy(`usage endpoint ${res.status}`)
 
     const windows: Window[] = []
     for (const l of (res.body?.limits ?? []) as any[]) {
@@ -119,7 +184,7 @@ export function makeClaudeReader(d: ClaudeDeps): UsageReader {
       })
     }
 
-    if (windows.length === 0) return no("no usable limit windows in the payload")
+    if (windows.length === 0) return fresh("no usage windows yet, nothing has run on this account")
     const bad = checkWindows(windows, now)
     return bad ? no(bad) : { readable: true, windows }
   }
@@ -153,25 +218,47 @@ export function liveClaudeDeps(live = false): ClaudeDeps {
       const j: any = await r.json()
       const next = {
         accessToken: String(j.access_token),
-        refreshToken: c.refreshToken,
+        // The response's own refresh token when it carries one, because the
+        // endpoint may rotate: it hands back a new one and invalidates the one
+        // that bought it. Keeping the old one and writing it back over the file
+        // locked an account out twice in two days, the next refresh answering
+        // `invalid_grant, Refresh token not found or invalid` with the stored
+        // expiry still 27 days away and only an interactive login able to
+        // recover it. Worse on an account a human also uses: their session
+        // refreshes, stores the new token, and this wrote the dead one back
+        // over it. A response with no refresh_token means the stored one is
+        // still live, so it has to survive rather than become undefined.
+        refreshToken: j.refresh_token ? String(j.refresh_token) : c.refreshToken,
         expiresAt: Date.now() + Number(j.expires_in ?? 0) * 1000,
         refreshTokenExpiresAt: c.refreshTokenExpiresAt,
       }
-      // A dry run keeps the refreshed token in memory: refresh tokens are not
-      // rotated, so the stored one keeps working and the file stays the live
-      // agent's to own. A live run writes it back, because an account nobody
-      // refreshes goes blind within a day.
+      // A dry run keeps the refreshed token in memory rather than writing it,
+      // so the file stays the live agent's to own. A live run writes it back,
+      // because an account nobody refreshes goes blind within a day, and
+      // because a rotated token that is never stored is a token thrown away.
       if (live) writeCreds(configDir, next)
       return next
     },
-    async getUsage(token) {
+    async getUsage(token, configDir) {
+      const hit = readUsageCache(configDir, Date.now())
+      if (hit) return { status: hit.status, body: hit.body }
+
       const r = await fetch(USAGE_URL, {
         headers: {
           authorization: `Bearer ${token}`,
           "anthropic-beta": "oauth-2025-04-20",
         },
       })
-      return { status: r.status, body: r.status === 200 ? await r.json() : null }
+      const entry: CachedUsage = {
+        at: Date.now(),
+        status: r.status,
+        body: r.status === 200 ? await r.json() : null,
+      }
+      // Never cache a 401: it is a fact about this token, not about the
+      // account, and the caller refreshes and retries immediately. A cached
+      // one would answer that retry with the answer it just refreshed away.
+      if (entry.status !== 401) writeUsageCache(configDir, entry)
+      return { status: entry.status, body: entry.body }
     },
   }
 }

package/src/router/providers/grok.ts

@@ -2,8 +2,7 @@ import type { UsageReader } from "../../types"
 
 // No usage signal exists: the billing record carries a period boundary and
 // zeroed on-demand credits, and the session logs record spend, not remaining.
-// Unreadable, and deliberately not exhausted, so allowWhenUnreadable can opt
-// the account back in.
+// Unreadable, so it runs only where allowWhenUnreadable opts it back in.
 export const grokReader: UsageReader = async () => ({
   readable: false,
   reason: "no usage signal exists for this provider",

package/src/router/route.ts

@@ -1,6 +1,7 @@
-import type { AccountConfig, AccountUsage, Ctx, Job, WorkItem } from "../types"
+import type { AccountConfig, AccountUsage, Ctx, Job, WorkItem, Window } from "../types"
 import { selects } from "../config"
 import { concurrencyFor } from "./budget"
+import { checkWindows } from "./window"
 import { rateOf, recordAndLearn } from "./rate"
 
 export type Route =
@@ -11,6 +12,51 @@ export type Route =
 
 export const BUILT_BY = /^built-by:\s*(\S+)\s*$/m
 
+// How old a recorded reading may be and still price an account the reader
+// cannot reach. The usage endpoint answers about five calls a minute per token
+// and shares that budget with every consumer of it: each interactive session's
+// status line, each worker the loop starts, and the tick itself. A busy account
+// draws a 429, and the reader holds a 429 for five minutes on purpose, so one
+// unlucky probe costs five minutes and unlucky probes chain. Measured on one
+// box over an evening, on a token held by four sessions and two workers: gaps
+// of 10, 14, 20, 20 and 27 minutes between successful readings, and the account
+// unreadable for 151 of 210 minutes. Ten minutes rode out a single hold and not
+// a run of them, which is what benched an account with 93 percent of its week
+// left. Half an hour covers the runs actually seen. Past it an account is not
+// bursting, it is unreachable, and a stale number would hide that.
+export const STALE_USAGE_MS = 30 * 60_000
+
+// The reading to rank on when the live one did not arrive. Nothing is invented
+// here: these are readings this loop took and recorded, replayed against a
+// later clock, and windowSane still refuses one older than its own window.
+// Empty for a fresh account, which has no recorded reading by definition, and
+// for an account whose last reading is older than the bound.
+//
+// Aged forward by what the account could have spent since, because half an hour
+// is long enough for the number to matter near a ceiling, and the ceiling is
+// what stops a worker starting into a window that will refuse it mid-task. At
+// least one consumer even with no worker of ours in flight: an account nothing
+// is using does not drain a metering bucket, so a reading we cannot refresh is
+// itself evidence that something is spending. The rate is the same one the
+// budget prices workers with, so this errs the way that model errs and adds no
+// second opinion of its own.
+function staleWindows(ctx: Ctx, a: AccountConfig, usage: AccountUsage, workers: number): Window[] {
+  // Only when the metering endpoint failed, never when the account did. An
+  // account whose credentials are missing or whose refresh was refused cannot
+  // authenticate a worker, and no usage number changes that: on 2026-09-01 half
+  // an hour of a missing credentials file spawned six builders that each met
+  // "Invalid API key, please run /login", failed, and labelled a sound issue
+  // agent-failed on the way out.
+  if (usage.readable || usage.fresh || !usage.transient) return []
+  const windows = ctx.global.lastWindows(a.id, ctx.now.getTime() - STALE_USAGE_MS)
+  if (windows.length === 0 || checkWindows(windows, ctx.now)) return []
+  return windows.map((w) => {
+    const minutes = (ctx.now.getTime() - w.observedAt.getTime()) / 60000
+    const rate = rateOf(ctx.global, a.provider, w.kind, ctx.config.workerRateSeed, w.windowMinutes)
+    return { ...w, percent: Math.min(100, w.percent + rate * Math.max(workers, 1) * minutes) }
+  })
+}
+
 function startOfUtcDay(now: Date): number {
   return Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())
 }
@@ -41,7 +87,16 @@ async function inFlightByAccount(ctx: Ctx): Promise<Map<string, number>> {
     const agents = await ctx.cache("engine:agents", () => ctx.herdr.agents())
     const out = new Map<string, number>()
     for (const a of agents) {
-      if (a.status === "missing") continue
+      // "missing" is no information, and "done" is an agent that has finished:
+      // neither is spending quota, and counting either holds an account's budget
+      // behind a worker that has gone home. On 2026-08-31 a maplista reviewer
+      // finished its round and left three polling shells behind, which kept
+      // herdr reporting "working" and starved the only eligible account with the
+      // merge one step away. The shells were the bug and a Stop hook now kills
+      // them, but the same starvation follows from any finished agent whose
+      // worktree is legitimately held open, which for a reviewer is every round
+      // until its pull request closes.
+      if (a.status === "missing" || a.status === "done") continue
       const account = attribute(ctx, a.cwd)
       if (account) out.set(account, (out.get(account) ?? 0) + 1)
     }
@@ -74,8 +129,13 @@ interface Ranked {
 export async function chooseAccount(ctx: Ctx, p: Job, item: WorkItem): Promise<Route> {
   const cfg = ctx.config
 
+  // A job the cap does not apply to still spends the account's quota and still
+  // waits for a worker slot, so usageMax, the reserves and maxConcurrent are
+  // what pace it. Only the box-wide day counter is lifted.
+  const cap = p.ignoresSpawnCap ? Infinity : cfg.maxSpawnsPerDay
+
   const spawned = ctx.global.spawnsSince(startOfUtcDay(ctx.now))
-  if (spawned >= cfg.maxSpawnsPerDay) {
+  if (spawned >= cap) {
     return { ok: false, global: true, reason: `CAP ${spawned}/${cfg.maxSpawnsPerDay} spawns today` }
   }
 
@@ -111,24 +171,43 @@ export async function chooseAccount(ctx: Ctx, p: Job, item: WorkItem): Promise<R
     // it gets turned into the same unreadable shape a reader would return on
     // purpose.
     const usage = await ctx.usage(a).catch(
-      (err): AccountUsage => ({ readable: false, reason: `read failed: ${err}` }),
+      // Transient: a reader that threw is a reader that could not complete its
+      // read, which says nothing about the account behind it.
+      (err): AccountUsage => ({ readable: false, reason: `read failed: ${err}`, transient: true }),
     )
     const max = a.maxConcurrent ?? cfg.maxConcurrentPerAccount
     const have = inFlight.get(a.id) ?? 0
+    const stale = staleWindows(ctx, a, usage, have)
     let concurrency: number
     let why: string
+    let paused = false
 
-    if (!usage.readable) {
+    if (!usage.readable && stale.length === 0) {
       // Unreadable is ineligible, not "usable but ranked last": the last-resort
       // reading sends work to the account most likely already exhausted,
-      // precisely when every other account is out. A 429 is never opted back in.
-      if (usage.exhausted || !a.allowWhenUnreadable) continue
-      concurrency = max
-      why = `unreadable but allowed (${usage.reason})`
+      // precisely when every other account is out. allowWhenUnreadable is the
+      // deliberate way back in, and it covers every unreadable reason: no
+      // reader can prove an account spent without a reading to prove it with.
+      if (!usage.fresh && !a.allowWhenUnreadable) continue
+      // A fresh account gets one worker and not its clamp. It has no readings,
+      // so there is no evidence of headroom to spend, and one worker is all it
+      // takes to record the first window: the tick after it can rank the
+      // account on measurements like any other. One is also what stops a
+      // payload that never gains windows from spawning every tick, because the
+      // worker already in flight fails the concurrency <= have test below.
+      concurrency = usage.fresh ? Math.min(1, max) : max
+      why = usage.fresh
+        ? `fresh, one worker to record the first window (${usage.reason})`
+        : `unreadable but allowed (${usage.reason})`
     } else {
-      recordAndLearn(ctx.global, a, usage.windows, have)
+      // A stale reading prices the account and teaches nothing. The rate EWMA
+      // is the delta between two live readings, and a reading replayed against
+      // a later clock is the same number at a different time: it would report
+      // an account that spent nothing while its workers ran.
+      const windows = usage.readable ? usage.windows : stale
+      if (usage.readable) recordAndLearn(ctx.global, a, usage.windows, have)
       const b = concurrencyFor({
-        windows: usage.windows,
+        windows,
         now: ctx.now,
         reserve: a.reserve,
         reservePerWeekday: a.reservePerWeekday,
@@ -136,12 +215,23 @@ export async function chooseAccount(ctx: Ctx, p: Job, item: WorkItem): Promise<R
         usageMax: cfg.usageMax,
         releaseBefore: cfg.releaseBefore,
         maxConcurrent: max,
+        workerRunMin: cfg.workerRunMin,
         rateFor: (w) => rateOf(ctx.global, a.provider, w.kind, cfg.workerRateSeed, w.windowMinutes),
       })
       concurrency = b.concurrency
-      why = `${b.limiting} ${b.detail} -> ${concurrency} workers, ${have} in flight`
+      // The clock is what refuses here, not the quota, and it clears within the
+      // hour. A job the rest of the queue waits on takes one worker through it.
+      if (b.paused && p.ignoresPace && have < 1) paused = true
+      const age = Math.round((ctx.now.getTime() - windows[0]!.observedAt.getTime()) / 60000)
+      // The reason line is what an operator reads after a STARVED run turns
+      // into a SPAWN, so a decision taken on an old number says so and says
+      // which refusal made it old.
+      const from = usage.readable ? "" : ` (stale by ${age}m, ${usage.reason})`
+      why = `${b.limiting} ${b.detail} -> ${concurrency} workers, ${have} in flight${from}`
+      if (paused) why = `${why}, one worker through the pause`
     }
 
+    if (paused) concurrency = 1
     if (concurrency <= have) continue
     const preferIdx = p.prefer?.findIndex((s) => selects(a, s)) ?? -1
     ranked.push({
@@ -153,6 +243,39 @@ export async function chooseAccount(ctx: Ctx, p: Job, item: WorkItem): Promise<R
     })
   }
 
+  // Last resort for a job that must run. Every account is out of headroom under
+  // usageMax and its own reserve, which is the pacing model saying "wait for the
+  // window to roll". A scheduled job cannot wait: its occurrence is a slot, and a
+  // slot missed is work that never happens. email-digest was starved on 2263
+  // ticks across five consecutive days in August, 19 mailings to real customers
+  // that simply never went out, while the loop ticked normally and said so once
+  // every two minutes. So a flagged job spends the reserve rather than skip: the
+  // account with the most room left of what is left, still bounded by that
+  // account's own worker ceiling, because two workers where one fits is not a
+  // quota decision but an out-of-memory one.
+  if (ranked.length === 0 && p.ignoresReserve) {
+    for (const a of pool) {
+      const max = a.maxConcurrent ?? cfg.maxConcurrentPerAccount
+      const have = inFlight.get(a.id) ?? 0
+      if (have >= max) continue
+      const usage = await ctx.usage(a).catch((): AccountUsage => ({ readable: false, reason: "read failed" }))
+      // The worst window is what would refuse the spawn, so it is what ranks the
+      // account here too. An unreadable account sorts last and is still eligible:
+      // this is the pass that runs when the alternative is not running at all.
+      const spent = usage.readable ? Math.max(...usage.windows.map((w) => w.percent), 0) : 100
+      ranked.push({
+        account: a,
+        demoted: builder !== null && builder === a.id,
+        prefer: p.prefer?.findIndex((sel) => selects(a, sel)) ?? -1,
+        headroom: 100 - spent,
+        why: `no account had headroom, spending the reserve for a job that must run (${a.id} at ${spent}% of its worst window, ${have} in flight)`,
+      })
+    }
+    // prefer: -1 means "not preferred" in the loop above, but the sort reads a
+    // lower number as better, so normalize before it decides.
+    for (const r of ranked) if (r.prefer === -1) r.prefer = Number.MAX_SAFE_INTEGER
+  }
+
   if (ranked.length === 0) return { ok: false, global: false, reason: "STARVED no eligible account" }
 
   ranked.sort(
@@ -170,7 +293,7 @@ export async function chooseAccount(ctx: Ctx, p: Job, item: WorkItem): Promise<R
     // Choose-and-reserve is one step: counting first and inserting afterwards
     // is the race the daily cap exists to survive.
     const key = await p.key(ctx, item)
-    if (!ctx.global.reserve(won.account.id, ctx.workspace.name, p.name, key, ctx.now, cfg.maxSpawnsPerDay, startOfUtcDay(ctx.now))) {
+    if (!ctx.global.reserve(won.account.id, ctx.workspace.name, p.name, key, ctx.now, cap, startOfUtcDay(ctx.now))) {
       return { ok: false, global: true, reason: `CAP ${cfg.maxSpawnsPerDay} spawns today, reservation refused` }
     }
   }

package/src/runtime/worker.ts

@@ -20,20 +20,67 @@ export interface WorkerSpec {
   brief: string
 }
 
+// Enough of the screen to carry a dialog's options and the cursor on one of them.
+export const DIALOG_LINES = 40
+
+// Both startup dialogs list their refusing answer first and highlight it, so a
+// bare Enter answers neither: on the folder-trust one it picks "No, exit", the
+// agent quits, herdr deregisters it, and every later read of that pane fails
+// with agent_not_found. Read which option the cursor is on instead and walk it
+// to the one that starts with "Yes", which is the answer these workers have
+// always meant to give. A screen with no such options keeps the old bare Enter,
+// because a dialog nobody has seen yet is still more likely to want confirming
+// than cancelling.
+export function dialogKeys(screen: string): string[] {
+  const options = screen
+    .split("\n")
+    .map((l) => ({ selected: /^\s*[\u276f>]/.test(l), text: l.replace(/^\s*[\u276f>]?\s*/, "") }))
+    .filter((o) => /^(Yes|No),/.test(o.text))
+  const at = options.findIndex((o) => o.selected)
+  const yes = options.findIndex((o) => o.text.startsWith("Yes"))
+  if (at < 0 || yes < 0) return ["Enter"]
+  const step = yes > at ? "Down" : "Up"
+  return [...Array(Math.abs(yes - at)).fill(step), "Enter"]
+}
+
+// An agent that comes up on a startup dialog is running, not broken: it is
+// waiting on a keypress, and every worktree is a directory the agent has never
+// seen, so the folder-trust and external-CLAUDE.md-import questions are the
+// normal case rather than the exception. Stacked dialogs get one answer per
+// start attempt, which is what the retry loop is for.
+async function answerStartupDialog(ctx: Ctx, pane: string): Promise<boolean> {
+  if ((await ctx.herdr.agentStatus(pane)) !== "blocked") return false
+  await ctx.herdr.agentSendKeys(pane, dialogKeys(await ctx.herdr.agentRead(pane, DIALOG_LINES)))
+  await ctx.sleep(RECOVER_WAIT_MS)
+  return (await ctx.herdr.agentStatus(pane)) !== "blocked"
+}
+
 export async function startWorker(ctx: Ctx, w: WorkerSpec): Promise<void> {
-  let lastErr: unknown = null
+  // The first error, not the last: once an attempt has registered the name,
+  // every retry after it fails with agent_name_taken, and reporting that hides
+  // the only error that says why the start failed in the first place.
+  let firstErr: unknown = null
+  let started = false
   for (let attempt = 1; attempt <= START_RETRIES; attempt++) {
     try {
       await ctx.herdr.agentStart({ pane: w.pane, kind: w.kind, name: w.name, args: w.args })
-      lastErr = null
+      started = true
       break
     } catch (err) {
-      lastErr = err
+      if (firstErr === null) firstErr = err
+      // Answering can fail in its own right: a dialog answered with the wrong
+      // key quits the agent, and the reads here then throw agent_not_found.
+      // Letting that escape replaces the start error with a symptom of the
+      // recovery and skips every attempt that was left.
+      if (await answerStartupDialog(ctx, w.pane).catch(() => false)) {
+        started = true
+        break
+      }
       if (attempt < START_RETRIES) await ctx.sleep(START_DELAY_MS)
     }
   }
-  if (lastErr) {
-    throw new Error(`agent start failed after ${START_RETRIES} attempts: ${lastErr}`)
+  if (!started) {
+    throw new Error(`agent start failed after ${START_RETRIES} attempts: ${firstErr}`)
   }
   await sendBrief(ctx, w.pane, w.brief)
 }

package/src/types.ts

@@ -66,6 +66,14 @@ export interface Config {
   releaseBefore: number
   maxSpawnsPerDay: number
   blockedTimeoutMin: number
+  // A hold this old has one notification sent about it. 0 turns it off.
+  holdTimeoutMin: number
+  // An agent sitting in a held worktree with no work left is closed after this
+  // many minutes, freeing the account slot it counts against. 0 turns it off.
+  staleAgentMin: number
+  // How long one worker runs, in minutes. Prices a burst while a window is
+  // behind its line: a worker only starts if the points left can pay for it.
+  workerRunMin: number
   // Percentage points per minute per worker, used for a provider/window pair
   // with no measured EWMA yet. Must be > 0.
   workerRateSeed: number
@@ -86,7 +94,11 @@ export interface WorkItem {
   createdAt?: string
 }
 
-export type AgentStatus = "working" | "blocked" | "idle" | "missing"
+// herdr's own enum is idle, working, blocked, done, unknown. "done" is herdr
+// saying the agent finished and is not coming back on its own, which is a fact
+// worth acting on; "unknown" and anything herdr adds later map to "missing",
+// which every consumer here treats as no information and holds on.
+export type AgentStatus = "working" | "blocked" | "idle" | "done" | "missing"
 
 export interface AgentView {
   cwd: string
@@ -157,6 +169,29 @@ export interface Job {
   // account by headroom: the same job must run the same model wherever it lands.
   model?: string
   sweepIgnoresWorking?: boolean
+  // Exempt from maxSpawnsPerDay. The cap is a runaway-loop breaker sized for
+  // whichever workspace on the box actually loops, and a scheduled job that
+  // spawns a handful of times a day is not what it is aimed at: on 2026-09-02 a
+  // builder re-picking one issue every tick spent all 60 by 03:52 and the
+  // 04:15 content run never started. Spawns still count, so an exempt job is
+  // visible in the day's total and shortens the capped jobs' budget.
+  ignoresSpawnCap?: boolean
+  // Spend the account reserve rather than skip. Only when no account has
+  // headroom at all, so the reserve still holds whenever the pacing model has
+  // any room to give: this is the difference between a job that waits for the
+  // window to roll and one whose slot is simply lost. For scheduled work with
+  // an outside consumer (a mailing, a daily run), where skipping is not a
+  // delay, it is a cancellation.
+  ignoresReserve?: boolean
+  // Run through a pacing pause, but never through a full account. Pacing
+  // spreads a window's ceiling evenly and waits whenever the account is ahead
+  // of that line, which is right for work that can be done later and wrong for
+  // the one job the rest of the queue is waiting on: a reviewer that cannot run
+  // is what makes a builder's review debt permanent, so a 0.4-point overshoot
+  // idled the whole maplista lane for 45 minutes on 2026-09-08. It takes one
+  // worker, not the account's clamp, and only while the points left still pay
+  // for a run, so usageMax and the reserve both still hold.
+  ignoresPace?: boolean
   deleteRemote?: boolean
   // Selectors: each matches an account by id or by provider.
   requires?: string[]
@@ -187,7 +222,7 @@ export interface Job {
 
 export type Decision =
   | { pass: "gc"; removed: number }
-  | { pass: "sweep"; job: string; worktree: string; branch: string; action: "clean" | "hold"; reason: string }
+  | { pass: "sweep"; job: string; worktree: string; branch: string; action: "clean" | "hold" | "overdue" | "reap"; reason: string }
   | { pass: "monitor"; job: string; key: string; action: MonitorAction; reason: string }
   | { pass: "spawn"; job: string; key: string; action: "spawn" | "skip"; account?: string; reason: string }
   // The workspace this tick pass covered, or "total" for the whole process:
@@ -204,7 +239,7 @@ export type Decision =
 
 export type MonitorAction =
   | "done" | "external" | "busy" | "blocked" | "escalate"
-  | "restart" | "nudge" | "fail" | "hold"
+  | "restart" | "nudge" | "fail" | "hold" | "overdue"
 
 export interface Window {
   kind: string          // 'session' | 'weekly_all' | 'w300' | ...
@@ -216,10 +251,21 @@ export interface Window {
   observedAt: Date
 }
 
-// `exhausted` is a 429: strictly more information than "unknown", and the one
-// unreadable state that allowWhenUnreadable must not resurrect.
+// Unreadable is one state, not two. There was once an `exhausted` variant for
+// a 429, on the theory that it proved the account was spent; it proves only
+// that the usage endpoint's own burst budget is spent, which anything else
+// polling the same account can do on our behalf. An account with no budget
+// left says so in a 200, with its windows at 100%. `fresh` is the other
+// direction: no window has started, so there is no usage to read rather than
+// too much of it, and the account needs one worker before it can be read.
 export type AccountUsage =
   | { readable: true; windows: Window[] }
-  | { readable: false; reason: string; exhausted?: boolean }
+  // `transient` marks a failure of the metering endpoint rather than of the
+  // account: the endpoint was busy or briefly broken, and the account itself is
+  // fine. Only those may be priced on an older reading, because a stale usage
+  // number cannot rescue an account a worker cannot authenticate to. Opt-in on
+  // purpose: a new failure mode that forgets to set it costs a skipped tick,
+  // and the other default costs a spawned worker that cannot work.
+  | { readable: false; reason: string; fresh?: boolean; transient?: boolean }
 
 export type UsageReader = (a: AccountConfig, now: Date) => Promise<AccountUsage>