@tonoid/agent-loop 1.0.0

package/src/router/route.ts

@@ -0,0 +1,179 @@
+import type { AccountConfig, AccountUsage, Ctx, Job, WorkItem } from "../types"
+import { selects } from "../config"
+import { concurrencyFor } from "./budget"
+import { rateOf, recordAndLearn } from "./rate"
+
+export type Route =
+  | { ok: true; account: string; reason: string }
+  // `global` marks a refusal that describes the box or the day rather than this
+  // job, so the spawn walk stops instead of asking the next job.
+  | { ok: false; global: boolean; reason: string }
+
+export const BUILT_BY = /^built-by:\s*(\S+)\s*$/m
+
+function startOfUtcDay(now: Date): number {
+  return Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())
+}
+
+// Which account a live worker belongs to, from its cwd. Every workspace on the
+// box is walked, not just this one: concurrency is an account-scoped fact, and
+// counting only this workspace's workers lets a second workspace spawn against
+// an account whose slots are already full.
+function attribute(ctx: Ctx, cwd: string): string | null {
+  for (const ws of ctx.workspaces) {
+    for (const p of ws.jobs) {
+      const prefix = `${ws.worktreeBase}/wt-${p.name}-`
+      if (!cwd.startsWith(prefix)) continue
+      // The worktree directory may carry a title slug after the key, so only
+      // the first path segment under the base is used and the spawns table
+      // does the matching.
+      const dir = cwd.slice(prefix.length).split("/")[0]!
+      return ctx.global.accountFor(ws.name, p.name, dir)
+    }
+  }
+  return null
+}
+
+// Live workers, attributed to the account that spawned them through the spawns
+// table.
+async function inFlightByAccount(ctx: Ctx): Promise<Map<string, number>> {
+  return ctx.cache("engine:inflight-by-account", async () => {
+    const agents = await ctx.cache("engine:agents", () => ctx.herdr.agents())
+    const out = new Map<string, number>()
+    for (const a of agents) {
+      if (a.status === "missing") continue
+      const account = attribute(ctx, a.cwd)
+      if (account) out.set(account, (out.get(account) ?? 0) + 1)
+    }
+    return out
+  })
+}
+
+// The account that built an item is recorded in the PR body, so the constraint
+// derives from the outside world like every other claim and the database stays
+// a pure cache.
+async function builtBy(ctx: Ctx, item: WorkItem): Promise<string | null> {
+  // Anchored at the end so "<owner>/<name>/pull/<n>" is unambiguous, and host
+  // agnostic so a self-hosted forge resolves the same way.
+  const m = item.url?.match(/\/([^/]+\/[^/]+)\/pull\/(\d+)\/?$/)
+  if (!m) return null
+  const body = await ctx.cache(`engine:body:${item.id}`, () =>
+    ctx.gh.prView(m[1]!, m[2]!, ["body"]),
+  )
+  return String(body?.body ?? "").match(BUILT_BY)?.[1] ?? null
+}
+
+interface Ranked {
+  account: AccountConfig
+  demoted: boolean
+  prefer: number
+  headroom: number
+  why: string
+}
+
+export async function chooseAccount(ctx: Ctx, p: Job, item: WorkItem): Promise<Route> {
+  const cfg = ctx.config
+
+  const spawned = ctx.global.spawnsSince(startOfUtcDay(ctx.now))
+  if (spawned >= cfg.maxSpawnsPerDay) {
+    return { ok: false, global: true, reason: `CAP ${spawned}/${cfg.maxSpawnsPerDay} spawns today` }
+  }
+
+  const inFlight = await inFlightByAccount(ctx)
+  const busy = [...inFlight.values()].reduce((a, b) => a + b, 0)
+  if (busy > 0) {
+    // Quota headroom says nothing about RAM, and workers each run installs, dev
+    // servers and typecheckers. With nothing in flight the box is as free as it
+    // will ever be, so refusing then would deadlock the loop.
+    const free = await ctx.memAvailableMb()
+    if (free < cfg.minFreeMb) {
+      return { ok: false, global: true, reason: `LOWMEM ${Math.round(free)}MB < ${cfg.minFreeMb}MB` }
+    }
+  }
+
+  const pool = cfg.accounts.filter((a) => !p.requires || p.requires.some((s) => selects(a, s)))
+  if (pool.length === 0) {
+    return { ok: false, global: false, reason: `STARVED no account matches requires` }
+  }
+
+  const builder = p.distinctFrom ? await builtBy(ctx, item) : null
+  const unsatisfiable = p.distinctFrom && builder === null ? ", built-by missing so distinctFrom ignored" : ""
+
+  const ranked: Ranked[] = []
+  for (const a of pool) {
+    // A reader can throw (missing dir, unrecognized payload shape, a bare
+    // fetch, a database that won't open) and readers are meant to: the spec
+    // wants that loud. But ctx.usage is memoized per account by ctx.cache, so
+    // an unguarded throw here would reject the *cached* promise, and every
+    // later job that ranks this same account in this tick would re-throw
+    // it too, starving the whole pool over one account's bad day. Only this
+    // boundary knows the failure is scoped to one account, so this is where
+    // it gets turned into the same unreadable shape a reader would return on
+    // purpose.
+    const usage = await ctx.usage(a).catch(
+      (err): AccountUsage => ({ readable: false, reason: `read failed: ${err}` }),
+    )
+    const max = a.maxConcurrent ?? cfg.maxConcurrentPerAccount
+    const have = inFlight.get(a.id) ?? 0
+    let concurrency: number
+    let why: string
+
+    if (!usage.readable) {
+      // Unreadable is ineligible, not "usable but ranked last": the last-resort
+      // reading sends work to the account most likely already exhausted,
+      // precisely when every other account is out. A 429 is never opted back in.
+      if (usage.exhausted || !a.allowWhenUnreadable) continue
+      concurrency = max
+      why = `unreadable but allowed (${usage.reason})`
+    } else {
+      recordAndLearn(ctx.global, a, usage.windows, have)
+      const b = concurrencyFor({
+        windows: usage.windows,
+        now: ctx.now,
+        reserve: a.reserve,
+        reservePerWeekday: a.reservePerWeekday,
+        weekendWeight: a.weekendWeight,
+        usageMax: cfg.usageMax,
+        releaseBefore: cfg.releaseBefore,
+        maxConcurrent: max,
+        rateFor: (w) => rateOf(ctx.global, a.provider, w.kind, cfg.workerRateSeed, w.windowMinutes),
+      })
+      concurrency = b.concurrency
+      why = `${b.limiting} ${b.detail} -> ${concurrency} workers, ${have} in flight`
+    }
+
+    if (concurrency <= have) continue
+    const preferIdx = p.prefer?.findIndex((s) => selects(a, s)) ?? -1
+    ranked.push({
+      account: a,
+      demoted: builder !== null && builder === a.id,
+      prefer: preferIdx === -1 ? Number.MAX_SAFE_INTEGER : preferIdx,
+      headroom: concurrency - have,
+      why,
+    })
+  }
+
+  if (ranked.length === 0) return { ok: false, global: false, reason: "STARVED no eligible account" }
+
+  ranked.sort(
+    (x, y) =>
+      Number(x.demoted) - Number(y.demoted) ||
+      x.prefer - y.prefer ||
+      y.headroom - x.headroom ||
+      x.account.id.localeCompare(y.account.id),
+  )
+  const won = ranked[0]!
+  const demoted = won.demoted ? ", demoted by distinctFrom and last standing" : ""
+  const reason = `${won.why}${demoted}${unsatisfiable}`
+
+  if (ctx.live) {
+    // Choose-and-reserve is one step: counting first and inserting afterwards
+    // is the race the daily cap exists to survive.
+    const key = await p.key(ctx, item)
+    if (!ctx.global.reserve(won.account.id, ctx.workspace.name, p.name, key, ctx.now, cfg.maxSpawnsPerDay, startOfUtcDay(ctx.now))) {
+      return { ok: false, global: true, reason: `CAP ${cfg.maxSpawnsPerDay} spawns today, reservation refused` }
+    }
+  }
+
+  return { ok: true, account: won.account.id, reason }
+}

package/src/router/window.ts

@@ -0,0 +1,39 @@
+import type { Window } from "../types"
+
+// The payload does not carry a window length, so it is inferred from the kind.
+// Documented assumption: session is 5 hours, both weekly kinds are 7 days.
+export const CLAUDE_WINDOW_MINUTES: Record<string, number> = {
+  session: 300,
+  weekly_all: 10080,
+  weekly_scoped: 10080,
+}
+
+// A window only describes its own span. A reset time further than one span away
+// in either direction is a unit error or gross clock skew, and a snapshot taken
+// more than one span ago says nothing about the window it names. Never infer
+// "percent is 0" from a reset time in the past.
+export function windowSane(w: Window, now: Date): boolean {
+  const span = w.windowMinutes * 60000
+  const t = w.resetsAt.getTime()
+  if (Number.isNaN(t) || !(w.windowMinutes > 0)) return false
+  if (t < now.getTime() - span || t > now.getTime() + span) return false
+  // Both readers do Number(...) on a payload field that may be absent, so a
+  // missing percent flows through as NaN rather than throwing. Left
+  // unchecked, NaN <= have is false (so the account is never skipped as
+  // exhausted) and y.headroom - x.headroom is NaN in the ranking sort
+  // (falsy, so it decides nothing), letting a poisoned account outrank a
+  // healthy one on alphabetical id alone. Same NaN-poisoning class as the
+  // claude reader's throw-on-unknown-kind, guarded here in the opposite
+  // direction: reject the bad value instead of throwing on it.
+  if (!Number.isFinite(w.percent)) return false
+  return now.getTime() - w.observedAt.getTime() <= span
+}
+
+export function checkWindows(ws: Window[], now: Date): string | null {
+  for (const w of ws) {
+    if (!windowSane(w, now)) {
+      return `window "${w.kind}" is not sane: resetsAt ${w.resetsAt.toISOString()}, observedAt ${w.observedAt.toISOString()}, span ${w.windowMinutes}m`
+    }
+  }
+  return null
+}

package/src/runtime/worker.ts

@@ -0,0 +1,75 @@
+import type { AgentStatus, Ctx } from "../types"
+
+// The pane's shell is not ready the instant the tab exists, and this is the
+// failure that shows up most often in production start logs.
+export const START_RETRIES = 5
+export const START_DELAY_MS = 3000
+export const PROMPT_TIMEOUT_MS = 15000
+export const RECOVER_WAIT_MS = 5000
+// A session whose SessionStart hooks are still running reports idle and moves
+// no state, so herdr answers agent_prompt_stalled and the brief is dropped on
+// the floor. That is a slow start, not a refusal, and the only recovery is to
+// send it again once the session has settled.
+export const PROMPT_ATTEMPTS = 5
+
+export interface WorkerSpec {
+  pane: string
+  kind: string
+  name: string
+  args: string[]
+  brief: string
+}
+
+export async function startWorker(ctx: Ctx, w: WorkerSpec): Promise<void> {
+  let lastErr: unknown = null
+  for (let attempt = 1; attempt <= START_RETRIES; attempt++) {
+    try {
+      await ctx.herdr.agentStart({ pane: w.pane, kind: w.kind, name: w.name, args: w.args })
+      lastErr = null
+      break
+    } catch (err) {
+      lastErr = err
+      if (attempt < START_RETRIES) await ctx.sleep(START_DELAY_MS)
+    }
+  }
+  if (lastErr) {
+    throw new Error(`agent start failed after ${START_RETRIES} attempts: ${lastErr}`)
+  }
+  await sendBrief(ctx, w.pane, w.brief)
+}
+
+export async function sendBrief(ctx: Ctx, pane: string, brief: string): Promise<void> {
+  let status: AgentStatus = "idle"
+  for (let attempt = 1; attempt <= PROMPT_ATTEMPTS; attempt++) {
+    // From the second attempt on only: never re-prompt an agent that is
+    // already working, because the previous send may have landed a moment
+    // after its own status check and a second one would queue the brief twice.
+    // On the first pass there is nothing to double, and checking first would
+    // skip the send entirely for an agent herdr already calls working.
+    if (attempt > 1) {
+      status = await ctx.herdr.agentStatus(pane)
+      if (status === "working") return
+    }
+
+    try {
+      await ctx.herdr.agentPrompt(pane, brief, { until: "working", timeoutMs: PROMPT_TIMEOUT_MS })
+    } catch {
+      // A timeout waiting for "working", or the stall herdr reports when the
+      // session moved no state at all. Neither is fatal here; the assertions
+      // below are what decide, and the throw at the end carries the status
+      // actually seen.
+    }
+    status = await ctx.herdr.agentStatus(pane)
+    if (status === "working") return
+
+    // A stuck composer holds the brief as unsent text and presents as idle.
+    // One Enter, one more look, then round again.
+    await ctx.herdr.agentSendKeys(pane, ["Enter"])
+    await ctx.sleep(RECOVER_WAIT_MS)
+    status = await ctx.herdr.agentStatus(pane)
+    if (status === "working") return
+  }
+  throw new Error(
+    `agent did not start working after ${PROMPT_ATTEMPTS} briefs (status ${status})`,
+  )
+}

package/src/state.ts

@@ -0,0 +1,128 @@
+import { Database } from "bun:sqlite"
+import type { Marks } from "./types"
+
+export interface MarkRow { job: string; key: string; mark: string; at: number }
+
+export interface State extends Marks {
+  // Every mark, for `agent-loop adopt --list`. The loop itself never needs the
+  // whole table: it asks about one job, key and mark at a time.
+  all(): MarkRow[]
+  // Test seam: move a mark back in time so gc() can be tested without waiting.
+  backdate(job: string, key: string, mark: string, minutes: number): void
+  close(): void
+}
+
+export function openState(path: string): State {
+  const db = new Database(path)
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS marks (
+      job TEXT NOT NULL,
+      key    TEXT NOT NULL,
+      mark   TEXT NOT NULL,
+      at     INTEGER NOT NULL,
+      PRIMARY KEY (job, key, mark)
+    )
+  `)
+  // A database created before the rename has a `plugin` column, and CREATE
+  // TABLE IF NOT EXISTS is a no-op against it, so every statement below would
+  // reference a column that does not exist. Marks are a cache the spec says is
+  // recoverable, but a loop that throws on every tick is not recovery.
+  const columns = db.query<{ name: string }, []>("PRAGMA table_info(marks)").all()
+  if (columns.some((c) => c.name === "plugin")) {
+    db.exec("ALTER TABLE marks RENAME COLUMN plugin TO job")
+  }
+
+  const get = db.query<{ at: number }, [string, string, string]>(
+    "SELECT at FROM marks WHERE job = ? AND key = ? AND mark = ?",
+  )
+  const ins = db.query(
+    "INSERT OR IGNORE INTO marks (job, key, mark, at) VALUES (?, ?, ?, ?)",
+  )
+  const del = db.query("DELETE FROM marks WHERE job = ? AND key = ? AND mark = ?")
+  const move = db.query(
+    "UPDATE marks SET at = ? WHERE job = ? AND key = ? AND mark = ?",
+  )
+  const sweep = db.query("DELETE FROM marks WHERE at < ?")
+  const list = db.query<MarkRow, []>(
+    "SELECT job, key, mark, at FROM marks ORDER BY job, key, mark",
+  )
+
+  return {
+    has(job, key, mark) {
+      return get.get(job, key, mark) !== null
+    },
+    age(job, key, mark) {
+      const row = get.get(job, key, mark)
+      if (row === null) return null
+      return Math.max(0, Math.round((Date.now() - row.at) / 60000))
+    },
+    set(job, key, mark) {
+      ins.run(job, key, mark, Date.now())
+    },
+    clear(job, key, mark) {
+      del.run(job, key, mark)
+    },
+    backdate(job, key, mark, minutes) {
+      move.run(Date.now() - minutes * 60000, job, key, mark)
+    },
+    gc(olderThanDays) {
+      const cutoff = Date.now() - olderThanDays * 24 * 60 * 60000
+      return sweep.run(cutoff).changes
+    },
+    all() {
+      return list.all()
+    },
+    close() {
+      db.close()
+    },
+  }
+}
+
+// A mark records that the loop did something: nudged a worker, restarted one,
+// spawned an item. A tick without --live does none of those, so persisting the
+// mark writes a claim about the world that never happened, and the next tick
+// reads it back as history: the second dry tick fails an item the first only
+// intended to nudge. It also means the shadow week of the cutover (docs/cutover.md
+// phase 1) hands the first live tick a state directory full of stamps nothing
+// earned. Writes stay in this process instead, so one tick is still internally
+// consistent (the monitor's second visit to an item sees its own first) and
+// leaves nothing behind. Reads fall through, so a stamp an operator imported
+// still counts.
+export function dryMarks<T extends Marks>(inner: T): T {
+  const pending = new Map<string, number | null>()
+  const id = (job: string, key: string, mark: string) => `${job} ${key} ${mark}`
+  const has = (job: string, key: string, mark: string): boolean => {
+    const at = pending.get(id(job, key, mark))
+    return at === undefined ? inner.has(job, key, mark) : at !== null
+  }
+  return {
+    ...inner,
+    has,
+    age(job: string, key: string, mark: string) {
+      const at = pending.get(id(job, key, mark))
+      if (at === undefined) return inner.age(job, key, mark)
+      return at === null ? null : Math.max(0, Math.round((Date.now() - at) / 60000))
+    },
+    set(job: string, key: string, mark: string) {
+      // INSERT OR IGNORE in the database above, and the same rule here: a mark
+      // that is already there keeps its first timestamp, so the age the monitor
+      // reads against the blocked timeout never resets.
+      if (has(job, key, mark)) return
+      pending.set(id(job, key, mark), Date.now())
+    },
+    clear(job: string, key: string, mark: string) {
+      pending.set(id(job, key, mark), null)
+    },
+    // The test seam, and the one write here that still reaches the database,
+    // for a mark that is only there. Nothing in the loop calls it.
+    backdate(job: string, key: string, mark: string, minutes: number) {
+      const at = pending.get(id(job, key, mark))
+      if (at === undefined || at === null) (inner as Partial<State>).backdate?.(job, key, mark, minutes)
+      else pending.set(id(job, key, mark), Date.now() - minutes * 60000)
+    },
+    // tradeoff: a dry tick collects nothing, so the count is honestly zero.
+    // Collection is the one write here that would be harmless, and skipping it
+    // keeps the rule one sentence: without --live, this database is read-only.
+    gc: () => 0,
+  } as T
+}

package/src/status.ts

@@ -0,0 +1,41 @@
+import type { AccountConfig, AccountUsage } from "./types"
+
+export interface StatusInput {
+  now: Date
+  accounts: AccountConfig[]
+  usageFor: (a: AccountConfig) => Promise<AccountUsage>
+  // Null for a provider with no refresh token to expire.
+  refreshExpiryFor: (a: AccountConfig) => Promise<Date | null>
+  spawnsToday: number
+  workspaces: { name: string; paused: string[] }[]
+}
+
+export async function renderStatus(o: StatusInput): Promise<string[]> {
+  const lines: string[] = []
+  for (const a of o.accounts) {
+    const usage = await o.usageFor(a).catch((err): AccountUsage => ({ readable: false, reason: String(err) }))
+    if (!usage.readable) {
+      lines.push(`${a.id}: unreadable (${usage.reason})`)
+    } else {
+      const windows = usage.windows
+        .map((w) => {
+          const minutes = Math.max(0, Math.round((w.resetsAt.getTime() - o.now.getTime()) / 60000))
+          return `${w.kind} ${w.percent.toFixed(1)}% resets in ${minutes}m`
+        })
+        .join(", ")
+      lines.push(`${a.id}: ${windows}`)
+    }
+    const expiry = await o.refreshExpiryFor(a).catch(() => null)
+    if (expiry) {
+      // Refresh tokens are not rotated on refresh, so this ceiling is hard:
+      // past it only an interactive login recovers the account.
+      const days = Math.floor((expiry.getTime() - o.now.getTime()) / 86400_000)
+      lines.push(`${a.id}: refresh token expires in ${days}d`)
+    }
+  }
+  lines.push(`spawns today: ${o.spawnsToday}`)
+  for (const w of o.workspaces) {
+    lines.push(`${w.name}: paused ${w.paused.length ? w.paused.join(", ") : "nothing"}`)
+  }
+  return lines
+}