@tonoid/agent-loop 1.0.0 → 1.1.0

package/src/discover.ts

@@ -17,7 +17,7 @@ const NAMING_KEYS = ["labels", "mergeMethod"]
 const LABEL_KEYS = ["claim", "failed", "park", "priority"]
 // `options` is opaque to the loader and validated by the kind; everything else
 // here is engine-level and validated for every kind (spec 3.3).
-const JOB_KEYS = ["kind", "repo", "slots", "order", "model", "requires", "prefer", "distinctFrom", "brief", "options"]
+const JOB_KEYS = ["kind", "repo", "slots", "order", "model", "requires", "prefer", "distinctFrom", "ignoresSpawnCap", "ignoresReserve", "ignoresPace", "brief", "options"]
 
 export interface LoadOpts {
   kinds: Record<string, Kind>
@@ -182,6 +182,15 @@ function loadJob(
   if (raw.distinctFrom !== undefined && typeof raw.distinctFrom !== "boolean") {
     errs.push(`${at}: distinctFrom must be true or false`)
   }
+  if (raw.ignoresSpawnCap !== undefined && typeof raw.ignoresSpawnCap !== "boolean") {
+    errs.push(`${at}: ignoresSpawnCap must be true or false`)
+  }
+  if (raw.ignoresPace !== undefined && typeof raw.ignoresPace !== "boolean") {
+    errs.push(`${at}: ignoresPace must be true or false`)
+  }
+  if (raw.ignoresReserve !== undefined && typeof raw.ignoresReserve !== "boolean") {
+    errs.push(`${at}: ignoresReserve must be true or false`)
+  }
   if (raw.model !== undefined && (typeof raw.model !== "string" || !raw.model)) {
     errs.push(`${at}: model must be the agent's own model name, like opus or sonnet`)
   }
@@ -217,6 +226,9 @@ function loadJob(
     requires: requires.length ? requires : undefined,
     prefer: prefer.length ? prefer : undefined,
     distinctFrom: raw.distinctFrom === true ? true : built.distinctFrom,
+    ignoresSpawnCap: raw.ignoresSpawnCap === true ? true : built.ignoresSpawnCap,
+    ignoresReserve: raw.ignoresReserve === true ? true : built.ignoresReserve,
+    ignoresPace: raw.ignoresPace === true ? true : built.ignoresPace,
   }
 }
 

package/src/effects/monitor.ts

@@ -3,6 +3,7 @@ import { worktreePath, matchesCwd } from "../engine/naming"
 import { itemKind, repoOf, trackerless } from "../engine/item"
 import { startWorker } from "../runtime/worker"
 import { preClean } from "./spawn"
+import { appendJournal } from "../journal"
 
 // The same write as the spawn rollback's. A finished item keeps its claim
 // label otherwise, and discoverClaimed reads --state all, so the label would
@@ -145,4 +146,17 @@ export async function applyFail(ctx: Ctx, p: Job, item: WorkItem, key: string):
     item.number,
     `The loop could not finish this item and has stopped working it.\n\nLast ${FAIL_TAIL_LINES} lines of the worker's transcript:\n\n\`\`\`\n${tail}\n\`\`\`\n\nRemove the \`${labels.failed}\` label to let the loop retry it.`,
   )
+  // Last, because the comment is the post-mortem a human actually answers and
+  // must not be lost to a journal that cannot be written. The label above
+  // already stops the item being re-picked, so there is nothing to race.
+  //
+  // Every other outcome writes a journal line and this one did not, so when six
+  // builders failed inside seventeen minutes on 2026-09-01 they left nothing
+  // behind but a label each, and reconstructing it took the tick log and six
+  // workers' transcripts. One line: the tail belongs on the item, where the
+  // human answering it needs it, and the journal is read as a story of the day.
+  appendJournal(
+    ctx,
+    `FAIL ${p.name} ${key}: gave up and labelled ${labels.failed} on ${repo}#${item.number}, transcript tail on the item`,
+  )
 }

package/src/effects/sweep.ts

@@ -3,16 +3,20 @@ import type { Worktree } from "../adapters/git"
 import { matchesCwd } from "../engine/naming"
 import { withRepoLock } from "../lock"
 
+// By tab id from pane list: a workspace id would close the whole loop
+// workspace and every other worker in it, and a worker whose agent already
+// exited still has a tab, so pane list rather than agent list.
+export async function applyReap(ctx: Ctx, wt: Worktree): Promise<void> {
+  const panes = await ctx.cache("engine:panes", () => ctx.herdr.panes())
+  const pane = panes.find((x) => matchesCwd(x.cwd, wt.path))
+  if (pane) await ctx.herdr.tabClose(pane.tabId)
+}
+
 export async function applySweep(ctx: Ctx, p: Job, wt: Worktree): Promise<void> {
   const repo = ctx.workspace.repos[p.repo ?? ""]
   if (!repo) return
 
-  // By tab id from pane list: a workspace id would close the whole loop
-  // workspace and every other worker in it, and a worker whose agent already
-  // exited still has a tab, so pane list rather than agent list.
-  const panes = await ctx.cache("engine:panes", () => ctx.herdr.panes())
-  const pane = panes.find((x) => matchesCwd(x.cwd, wt.path))
-  if (pane) await ctx.herdr.tabClose(pane.tabId)
+  await applyReap(ctx, wt)
 
   const git = ctx.git(repo)
   await withRepoLock(repo, ctx.lock, async () => {

package/src/engine/monitor.ts

@@ -1,5 +1,6 @@
 import type { Ctx, Job, Decision, MonitorAction } from "../types"
 import { worktreePath, matchesCwd } from "./naming"
+import { notifyOverdue } from "../overdue"
 import { applyDone, applyNudge, applyEscalate, applyRestart, applyFail, applyNotifyBlocked } from "../effects/monitor"
 
 export async function monitorJob(ctx: Ctx, p: Job): Promise<Decision[]> {
@@ -47,7 +48,11 @@ export async function monitorJob(ctx: Ctx, p: Job): Promise<Decision[]> {
     const agent = agents.find((a) => matchesCwd(a.cwd, wt))
 
     if (agent?.status === "working") {
-      out.push(mk("busy", "agent working"))
+      // Working is the one agent state with no way out of it: an agent that
+      // never stops reporting it keeps its claim and its job's slot for good.
+      // Say so once and go on holding, since the agent may be genuinely busy.
+      const overdue = await notifyOverdue(ctx, p.name, key, "still working")
+      out.push(overdue ? mk("overdue", overdue) : mk("busy", "agent working"))
       continue
     }
 

package/src/engine/sweep.ts

@@ -1,6 +1,8 @@
-import type { Ctx, Job, Decision, WorkItem } from "../types"
+import type { Ctx, Job, Decision, WorkItem, AgentStatus } from "../types"
+import type { Worktree } from "../adapters/git"
 import { owns, keyOf, matchesCwd } from "./naming"
-import { applySweep } from "../effects/sweep"
+import { applySweep, applyReap } from "../effects/sweep"
+import { notifyOverdue } from "../overdue"
 import { auditFiling } from "../filing"
 import { appendJournal } from "../journal"
 import { renderDecision } from "../render"
@@ -22,6 +24,41 @@ async function isFinished(ctx: Ctx, p: Job, rawKey: string): Promise<boolean> {
   return p.done(ctx, synthetic)
 }
 
+// An agent in one of these has nothing left to run: "idle" is a session
+// sitting at its prompt and "done" is one herdr has seen finish. "working" is
+// still going and "blocked" wants a human, so neither is ever cut off here.
+const REAPABLE: AgentStatus[] = ["idle", "done"]
+
+// A worker that finishes its round and never exits keeps counting against its
+// account in inFlightByAccount, for as long as its worktree waits on a
+// sweepOk somebody outside the loop owns, which for a reviewer is until the
+// pull request closes. The monitor cannot help: done() has already turned
+// true, so the item lost its claim and monitor stopped looking at it. On
+// 2026-09-07 two of them held the only account maplista may use, one for nine
+// hours and one for eight, and every tick in between read as a healthy
+// STARVED. Close the tab and leave the worktree, which is the half of a sweep
+// that is safe while sweepOk is still false.
+async function reapable(ctx: Ctx, p: Job, wt: Worktree, rawKey: string): Promise<string | null> {
+  const stale = ctx.config.staleAgentMin
+  if (!stale) return null
+  const agents = await ctx.cache("engine:agents", () => ctx.herdr.agents())
+  const agent = agents.find((a) => matchesCwd(a.cwd, wt.path) && REAPABLE.includes(a.status))
+  if (!agent) {
+    // It went away on its own, or went back to working. Either way the clock
+    // this run started should not carry into the next idle spell.
+    ctx.marks.clear(p.name, rawKey, "stale")
+    return null
+  }
+  const age = ctx.marks.age(p.name, rawKey, "stale")
+  if (age === null) {
+    ctx.marks.set(p.name, rawKey, "stale")
+    return null
+  }
+  if (age < stale) return null
+  ctx.marks.clear(p.name, rawKey, "stale")
+  return `agent ${agent.status} ${age}m >= ${stale}m, closing the tab`
+}
+
 export async function sweepJob(ctx: Ctx, p: Job): Promise<Decision[]> {
   const repo = ctx.workspace.repos[p.repo ?? ""]
   if (!repo) return []
@@ -36,18 +73,39 @@ export async function sweepJob(ctx: Ctx, p: Job): Promise<Decision[]> {
   for (const wt of worktrees) {
     if (!owns(p.name, base, wt)) continue
     const rawKey = keyOf(p.name, wt.branch)!
-    const mk = (action: "clean" | "hold", reason: string): Decision => ({
+    const mk = (action: "clean" | "hold" | "overdue" | "reap", reason: string): Decision => ({
       pass: "sweep", job: p.name, worktree: wt.path, branch: wt.branch!, action, reason,
     })
+    // Both holds below are unbounded by design, and this pass is the only one
+    // that sees the second of them: an item whose done() has turned true has
+    // already lost its claim, so monitor stops looking at it while its worktree
+    // waits here on a sweepOk that something outside the loop owns.
+    const held = async (reason: string, what: string) => {
+      const overdue = await notifyOverdue(ctx, p.name, rawKey, what)
+      out.push(overdue ? mk("overdue", overdue) : mk("hold", reason))
+    }
 
     const live = agents.some((a) => a.status === "working" && matchesCwd(a.cwd, wt.path))
     if (live && !p.sweepIgnoresWorking) {
-      out.push(mk("hold", "agent working"))
+      await held("agent working", "holding a worktree open for a working agent")
       continue
     }
     const predicate = p.sweepOk ? "sweepOk" : "done"
     if (!(await isFinished(ctx, p, rawKey))) {
-      out.push(mk("hold", `${predicate}(${rawKey}) false`))
+      const reason = await reapable(ctx, p, wt, rawKey)
+      if (reason) {
+        out.push(mk("reap", reason))
+        if (ctx.live) {
+          try {
+            await applyReap(ctx, wt)
+          } catch (err) {
+            // The worktree still holds below either way: a tab that will not
+            // close is a slot left occupied, not a reason to skip the hold.
+            out.push({ pass: "error", job: p.name, where: "sweep", reason: String(err) })
+          }
+        }
+      }
+      await held(`${predicate}(${rawKey}) false`, `waiting on ${predicate}(${rawKey})`)
       continue
     }
     out.push(mk("clean", `${predicate}(${rawKey})`))

package/src/globalstate.ts

@@ -10,6 +10,11 @@ export interface UsageSample {
 export interface GlobalStore {
   recordUsage(accountId: string, w: Window): void
   lastUsage(accountId: string, kind: string, beforeMs: number): UsageSample | null
+  // The newest row per kind, for an account the reader cannot reach right now.
+  // `group` and `scope` are not stored, so what comes back prices a window and
+  // does not describe one: group falls back to the kind, and a model scope is
+  // gone. Both are read before a window is recorded, never after.
+  lastWindows(accountId: string, notBeforeMs: number): Window[]
   rate(provider: string, kind: string): number | null
   observeRate(provider: string, kind: string, sample: number): number
   spawnAdd(accountId: string, workspace: string, job: string, key: string, at: Date): void
@@ -80,6 +85,16 @@ export function openGlobalState(path: string): GlobalStore {
      WHERE account = ? AND kind = ? AND at < ?
      ORDER BY at DESC LIMIT 1`,
   )
+  // Bare columns beside MAX(at): SQLite defines them as coming from the row
+  // the aggregate chose, which is the whole point here. One row per kind.
+  const recentUsage = db.query<
+    { kind: string; percent: number; resets_at: number; window_minutes: number; at: number },
+    [string, number]
+  >(
+    `SELECT kind, percent, resets_at, window_minutes, MAX(at) AS at FROM usage
+     WHERE account = ? AND at >= ?
+     GROUP BY kind`,
+  )
   const getRate = db.query<{ ewma: number }, [string, string]>(
     "SELECT ewma FROM rates WHERE provider = ? AND kind = ?",
   )
@@ -134,6 +149,16 @@ export function openGlobalState(path: string): GlobalStore {
     lastUsage(accountId, kind, beforeMs) {
       return prevUsage.get(accountId, kind, beforeMs)
     },
+    lastWindows(accountId, notBeforeMs) {
+      return recentUsage.all(accountId, notBeforeMs).map((r) => ({
+        kind: r.kind,
+        group: r.kind,
+        percent: r.percent,
+        resetsAt: new Date(r.resets_at),
+        windowMinutes: r.window_minutes,
+        observedAt: new Date(r.at),
+      }))
+    },
     rate(provider, kind) {
       return getRate.get(provider, kind)?.ewma ?? null
     },

package/src/kinds/builder.ts

@@ -1,8 +1,19 @@
 import type { Job, WorkItem, Ctx } from "../types"
 import type { Kind } from "./validate"
-import { issues, prs, unblocked, byPriority, newestByHead } from "./shared"
+import { issues, prs, unblocked, byPriority, newestByHead, humanOwned } from "./shared"
 import { renderBrief } from "../brief"
 import { branchName } from "../engine/naming"
+import { repoOf, itemKind } from "../engine/item"
+
+// The one reading of a closed pull request, shared by guard(), done() and
+// sweepOk() because the three have to agree about it: whichever of them reads
+// it differently is the one that makes the loop spawn, release and sweep the
+// same issue every tick. Human-owned is not a retry: `agent-failed` on a pull
+// request is the monitor's tombstone and `needs-human` is a question nobody
+// answered, and re-picking either is the loop overruling the human it asked.
+function retriable(ctx: Ctx, pr: WorkItem): boolean {
+  return pr.state === "CLOSED" && !humanOwned(ctx, pr)
+}
 
 interface Options {
   base: string
@@ -95,25 +106,75 @@ export const builder: Kind = {
       // issue stays open until the merge closes it, so without this the next
       // tick re-picks the issue and the spawn's pre-clean destroys the
       // worktree the reviewer's rounds are still working in.
+      //
+      // A human-owned closed pull request is the other answer, and until
+      // 2026-09-06 it was a silent one: the park or failed label sat on a
+      // closed pull request, which no backlog view shows, while the issue kept
+      // its plain labels and looked pickable. Six issues sat that way (#219,
+      // #292, #415, #447, #551, #555), skipped here on every tick with nothing
+      // in the log and nothing on the issue. Mirroring the label onto the issue
+      // makes the backlog say what the loop knows: unblocked() then drops it
+      // from discover(), sweepOk() lets its worktrees go, and a human sees it
+      // in the same list as every other parked item.
       guard: async (ctx, item) => {
         const pr = await prFor(ctx, keyFor(item))
-        // A closed, unmerged pull request is a legitimate retry.
-        return pr === null || pr.state === "CLOSED"
+        if (pr === null || retriable(ctx, pr)) return true
+        const l = ctx.workspace.naming.labels
+        const add = [l.park, l.failed].filter((x) => pr.labels.includes(x) && !item.labels.includes(x))
+        if (add.length && ctx.live) await ctx.gh.label(repoOf(item), itemKind(item), item.number, { add })
+        return false
       },
 
       // The work is over when the pull request exists: what happens to it after
       // that belongs to the reviewer, and holding the claim would count this
       // issue against the builder's slots through every review round.
+      //
+      // Except for the one pull request guard() re-picks. This read "a pull
+      // request exists" until 2026-09-02, when a closed one made the two
+      // disagree: the retry was declared finished before its worker had done
+      // anything, the claim came off, and the next tick picked the same issue
+      // again. 41 workers on one issue in 82 minutes, the box's whole daily
+      // spawn budget, and the scheduled routines sharing the box missed their
+      // slots behind a cap they had not spent.
       async done(ctx, item) {
-        return (await prFor(ctx, keyFor(item))) !== null
+        const pr = await prFor(ctx, keyFor(item))
+        return pr !== null && !retriable(ctx, pr)
       },
 
       // Not done: the worktree has to survive until the pull request is
       // finished, because the reviewer's rounds ask the builder's worker for
       // changes in it.
+      //
+      // With no pull request at all there is nothing to wait for, and a builder
+      // that fails opens none: it labels its issue for a human and exits. Read
+      // as "no pull request yet" that answered false forever, and a hold never
+      // kills, so six dead worktrees and their tabs accumulated on one box in a
+      // day while the log carried a HOLD line for each every two minutes. The
+      // park and failed labels are the two states a human owns, and a human
+      // owning the item is when the machine should let go. The pull request
+      // still decides whenever there is one: a failed label on an issue whose
+      // pull request is open is a review round that found something, and its
+      // worktree is where the next round works.
       async sweepOk(ctx, rawKey) {
         const pr = await prFor(ctx, rawKey)
-        return pr !== null && pr.state !== "OPEN"
+        // The label on the pull request itself, which is the monitor
+        // tombstoning it rather than a review round asking for changes. A
+        // failed pull request is out of the reviewer's discovery, so nothing
+        // will ever move it and both worktrees wait on a state change only a
+        // human can cause: two pairs sat 19 and 21 hours that way, holding
+        // 1.4GB of live worker processes between them.
+        if (pr !== null && !retriable(ctx, pr)) return pr.state !== "OPEN" || humanOwned(ctx, pr)
+        // No pull request, or the one guard() is going to try again: either way
+        // the worktree belongs to a run that is not finished. sweepIgnoresWorking
+        // is on for builders, so nothing else holds it, and a retry whose fresh
+        // worktree is deleted ninety seconds in works in a directory that no
+        // longer exists. The issue decides instead: a closed one is out of
+        // discover() and a human-owned one is a state only a human can change,
+        // so neither is ever picked up again and neither worktree is waiting
+        // for anything.
+        const number = Number(rawKey.replace(/^\D+/, ""))
+        const issue = (await issues(ctx, job, "all")).find((i) => i.number === number)
+        return issue !== undefined && (issue.state !== "OPEN" || humanOwned(ctx, issue))
       },
 
       brief: (ctx, item) =>

package/src/kinds/reviewer.ts

@@ -1,6 +1,6 @@
 import type { Ctx, FilingConfig, Job, WorkItem } from "../types"
 import { type Kind, oneOf, unknownKey } from "./validate"
-import { issues, prs, unblocked } from "./shared"
+import { issues, prs, unblocked, humanOwned } from "./shared"
 import { renderBrief } from "../brief"
 import { filingBudget } from "../filing"
 import { repoOf } from "../engine/item"
@@ -173,9 +173,23 @@ export const reviewer: Kind = {
       // tab forever.
       async sweepOk(ctx, rawKey) {
         const number = numberOf(rawKey)
+        // Before the identity lookups, because those answer on the issue and an
+        // issue stays open while its own pull request is tombstoned: the
+        // monitor labels the pull request on failure, a labelled one is out of
+        // discovery, and the worktree then waits for a verdict nobody will ever
+        // give. Keyed through job.key so it holds under every identity, and
+        // only for pull requests actually carrying the label, which is a short
+        // list on any healthy day and empty on most.
+        for (const p of inScope(await prs(ctx, job, "all"))) {
+          if (p.state !== "OPEN" || !humanOwned(ctx, p)) continue
+          if ((await job.key(ctx, p)) === rawKey) return true
+        }
         if (o.identity === "closing-issue") {
           const issue = (await issues(ctx, job, "all")).find((i) => i.number === number)
-          if (issue) return issue.state !== "OPEN"
+          // Human-owned is the builder's rule too: a parked issue is a state
+          // only a human can change, and r447 held its worktree 41 hours
+          // behind one on 2026-09-06 waiting for a close that never comes.
+          if (issue) return issue.state !== "OPEN" || humanOwned(ctx, issue)
         }
         if (o.identity === "head-ref-issue") {
           const all = await prs(ctx, job, "all")

package/src/kinds/shared.ts

@@ -47,3 +47,14 @@ export function newestByHead(items: WorkItem[], head: string): WorkItem | null {
   const mine = items.filter((i) => i.headRef === head)
   return mine.length ? mine.reduce((a, b) => (b.number > a.number ? b : a)) : null
 }
+
+// The two labels a human owns: the park is a question nobody answered, the
+// failed label is the monitor's tombstone. Either means no job will pick the
+// item up again, so nothing the loop does will ever change its state, and a
+// sweep predicate waiting on one waits forever. A human owning the item is
+// exactly when the machine should let go of the worktree.
+export function humanOwned(ctx: Ctx, item: WorkItem): boolean {
+  const l = ctx.workspace.naming.labels
+  const owned = [l.park, l.failed].filter(Boolean)
+  return item.labels.some((name) => owned.includes(name))
+}

package/src/overdue.ts

@@ -0,0 +1,45 @@
+import type { Ctx } from "./types"
+
+// Every hold in the loop is deliberate and fail-safe: it cannot kill a live
+// agent and cannot tombstone an item, so whatever it fails to resolve it simply
+// keeps. What it has no way to say is that it has been keeping it too long.
+// `blocked` is the one state carrying a clock (blockedTimeoutMin, then
+// escalate). A worker wedged in `working` and a worktree whose sweepOk never
+// turns true both hold with nothing counting the minutes, and both did: a
+// content run held its job's only slot for two hours behind a leftover shell,
+// and a review worktree waited eight for a pull request CI had quietly declined
+// to merge. Neither is visible in a tick log that says HOLD every two minutes
+// and has said HOLD every two minutes for a week.
+//
+// So: the same clock, and nothing else. No escalation and no kill, because a
+// hold is the right response to not knowing and a timer is a bad reason to
+// interrupt a run that might be mid-write. Silence was the defect, not holding.
+//
+// The age comes from the `spawned` mark, which covers the whole occurrence
+// rather than time since this pass first noticed. An occurrence with no such
+// mark has no age to read and is left alone, so an adopted or imported worktree
+// never pings on a number nobody set.
+export async function notifyOverdue(
+  ctx: Ctx,
+  job: string,
+  key: string,
+  what: string,
+): Promise<string | null> {
+  const timeout = ctx.config.holdTimeoutMin
+  if (!timeout) return null
+  const age = ctx.marks.age(job, key, "spawned")
+  if (age === null || age < timeout) return null
+  // Once per occurrence. The mark is what makes it once: a hold that lasts a
+  // day is one notification, not seven hundred.
+  if (ctx.marks.has(job, key, "overdue")) return null
+  ctx.marks.set(job, key, "overdue")
+  if (ctx.live) {
+    await ctx.herdr
+      .notify(
+        `${job} has been held ${age}m`,
+        `${job} ${key} is ${what}, and has been in flight ${age}m against a ${timeout}m timeout. Nothing is failing here, and nothing is finishing either.`,
+      )
+      .catch(() => {})
+  }
+  return `${what}, ${age}m >= ${timeout}m`
+}

package/src/render.ts

@@ -14,13 +14,17 @@ export function renderDecision(d: Decision, live = false): string {
     case "tick":
       return `TICK ${d.workspace} ${d.ms}ms`
     case "sweep":
-      return d.action === "clean"
-        ? `${would("sweep")} ${d.job} ${d.worktree} (${d.reason})`
-        : `HOLD ${d.job} ${d.worktree} (${d.reason})`
+      if (d.action === "clean") return `${would("sweep")} ${d.job} ${d.worktree} (${d.reason})`
+      // OVERDUE is a held worktree with a notification sent about it, so it
+      // reads as its own verb and appears once, not every tick like HOLD.
+      if (d.action === "overdue") return `OVERDUE ${d.job} ${d.worktree} (${d.reason})`
+      if (d.action === "reap") return `${would("reap")} ${d.job} ${d.worktree} (${d.reason})`
+      return `HOLD ${d.job} ${d.worktree} (${d.reason})`
     case "monitor":
       if (d.action === "busy") return `BUSY ${d.job} ${d.key} (${d.reason})`
       if (d.action === "hold") return `HOLD ${d.job} ${d.key} (${d.reason})`
       if (d.action === "blocked") return `BLOCKED ${d.job} ${d.key} (${d.reason})`
+      if (d.action === "overdue") return `OVERDUE ${d.job} ${d.key} (${d.reason})`
       return WOULD.has(d.action)
         ? `${would(d.action)} ${d.job} ${d.key} (${d.reason})`
         : `${d.action.toUpperCase()} ${d.job} ${d.key} (${d.reason})`

package/src/router/budget.ts

@@ -12,6 +12,10 @@ export interface BudgetIn {
   usageMax: number
   releaseBefore: number
   maxConcurrent: number
+  // How long one worker runs, in minutes. Used to price a burst while the
+  // window is behind its line: a worker is only started when the points left
+  // can pay for a run of this length.
+  workerRunMin: number
   // Percentage points per minute per worker for this window.
   rateFor(w: Window): number
 }
@@ -20,6 +24,11 @@ export interface BudgetOut {
   concurrency: number
   limiting: string
   detail: string
+  // A zero that is the clock talking rather than the quota: the account is
+  // ahead of its line, but the points left still pay for a run. It clears on
+  // its own within the hour, so a job that unblocks others can be let through
+  // it, while a zero without this flag means there is nothing left to spend.
+  paused: boolean
 }
 
 // How much working time a human still has inside this window, in weekday
@@ -63,19 +72,43 @@ export function concurrencyFor(i: BudgetIn): BudgetOut {
     const reserveNow =
       minutesToReset <= i.releaseBefore ? 0 : Math.min(100, Math.max(i.reserve, perWeekday))
     const ceiling = Math.min(i.usageMax, 100 - reserveNow)
-    const budgetRate = (ceiling - w.percent) / minutesToReset
     const rate = i.rateFor(w)
-    const workers = rate > 0 ? budgetRate / rate : 0
+    // What one run costs, which is the unit everything below is counted in: a
+    // worker is started or it is not, and half a run is not a thing to allow.
+    const runCost = rate * Math.max(1, i.workerRunMin)
+    // The budget spent evenly across the window, which is what the account has
+    // earned the right to spend by now. Being under it is credit, being over
+    // it is a debt the clock pays off.
+    const elapsed = Math.min(1, Math.max(0, 1 - minutesToReset / Math.max(1, w.windowMinutes)))
+    const line = ceiling * elapsed
+    const credit = line - w.percent
+    // Never start a run the remaining points cannot pay for: usageMax exists so
+    // a worker does not meet a 429 mid-task, and that holds however far behind
+    // the line the account is.
+    const affordable = runCost > 0 ? (ceiling - w.percent) / runCost : 0
+    // One worker at the line, more the further behind it the account is, none
+    // while it is ahead. The previous rule priced a worker as running without a
+    // break until the window resets, which no worker here does: a lane waits on
+    // continuous integration, on a review round, on a pull request closing. On
+    // a seven-day window that arithmetic asked for 171 points of headroom
+    // before it would allow a second worker against a ceiling of 75, so it
+    // could only ever allow one, and it dropped that one to zero at 19 percent
+    // spent. It also inverts near a reset, because the divisor shrinks: the
+    // maplista lane ran 102 and 100 spawns on the two days before its weekly
+    // reset and 22 on the day after, and each of those weeks still ended with
+    // 35 to 73 points expiring unspent.
+    const workers = w.percent > line ? 0 : Math.min(affordable, Math.max(1, credit / runCost))
     const concurrency = Math.max(0, Math.min(i.maxConcurrent, Math.round(workers)))
 
     if (!best || concurrency < best.concurrency) {
       best = {
         concurrency,
         limiting: w.kind,
-        detail: `${w.percent.toFixed(1)}% of ${ceiling.toFixed(1)} with ${Math.round(minutesToReset)}m left`,
+        detail: `${w.percent.toFixed(1)}% of ${ceiling.toFixed(1)}, line ${line.toFixed(1)}, with ${Math.round(minutesToReset)}m left`,
+        paused: concurrency === 0 && w.percent > line && affordable >= 1,
       }
     }
   }
 
-  return best ?? { concurrency: 0, limiting: "none", detail: "no windows" }
+  return best ?? { concurrency: 0, limiting: "none", detail: "no windows", paused: false }
 }