@tonoid/agent-loop 1.0.0

package/src/engine/tick.ts

@@ -0,0 +1,25 @@
+import type { Ctx, Job, Decision } from "../types"
+import { sweepAll } from "./sweep"
+import { monitorAll } from "./monitor"
+import { spawnOne } from "./spawn"
+
+export interface TickOpts { paused: boolean; pausedJobs?: string[]; gcDays?: number }
+
+export async function runTick(ctx: Ctx, jobs: Job[], opts: TickOpts): Promise<Decision[]> {
+  const started = Date.now()
+  const out: Decision[] = []
+  const emit = (ds: Decision[]) => {
+    for (const d of ds) {
+      out.push(d)
+      ctx.log(d)
+    }
+  }
+
+  emit([{ pass: "gc", removed: ctx.marks.gc(opts.gcDays ?? 14) }])
+  emit(await sweepAll(ctx, jobs))
+  emit(await monitorAll(ctx, jobs))
+  if (!opts.paused) emit(await spawnOne(ctx, jobs, opts.pausedJobs ?? []))
+
+  emit([{ pass: "tick", workspace: ctx.workspace.name, ms: Date.now() - started }])
+  return out
+}

package/src/filing.ts

@@ -0,0 +1,56 @@
+import type { Ctx, Job } from "./types"
+import { issues } from "./kinds/shared"
+
+export type { FilingConfig } from "./types"
+
+// openQueue is the consumer's own discover() result: unclaimed and unparked by
+// construction, because that is what a job offers the spawn walk. Cached under
+// the engine's key so the consumer's own spawn pass in this same tick pays for
+// the read once.
+export async function filingBudget(
+  ctx: Ctx,
+  job: Job,
+): Promise<{ openQueue: number; filingBudget: number }> {
+  const f = job.filing
+  if (!f) return { openQueue: 0, filingBudget: 0 }
+  const consumer = ctx.workspace.jobs.find((j) => j.name === f.queue)
+  if (!consumer) throw new Error(`job "${job.name}": filing.queue "${f.queue}" names no job in this workspace`)
+  const items = await ctx.cache(`engine:discover:${consumer.name}`, () => consumer.discover(ctx))
+  const openQueue = items.length
+  return {
+    openQueue,
+    filingBudget: openQueue >= f.maxOpen ? 0 : Math.min(f.perRound, f.maxOpen - openQueue),
+  }
+}
+
+// Enforcement stays out of the engine (spec 5.2): a job that overfiles has a
+// brief problem, and a loop that blocked `gh issue create` would break the one
+// write the fences deliberately allow. So this counts and reports.
+//
+// tradeoff: attribution is by time window, not by author, so two workers of the
+// same job filing in the same window share the count. Per-worker attribution
+// needs the forge to record who filed what, which is a query this does not have.
+export async function auditFiling(
+  ctx: Ctx,
+  job: Job,
+  key: string,
+): Promise<{ filed: number; budget: number } | null> {
+  const f = job.filing
+  if (!f) return null
+  const consumer = ctx.workspace.jobs.find((j) => j.name === f.queue)
+  if (!consumer) return null
+  // How long ago this worker started. No mark means the loop never spawned this
+  // key (a reboot, a hand-run), so there is no run to audit.
+  const age = ctx.marks.age(job.name, key, "spawned")
+  if (age === null) return null
+  const since = ctx.now.getTime() - age * 60000
+  // The shared helper, so this shares the consumer's own cached list rather
+  // than issuing the same --state all read again under a second key.
+  const items = await issues(ctx, consumer, "all")
+  const filed = items.filter((i) => i.createdAt && Date.parse(i.createdAt) >= since).length
+  // perRound rather than the budget the brief carried: the budget is recomputed
+  // from a queue depth that has moved since, and perRound is the half of it that
+  // does not move. A run that filed within perRound but above the room left is
+  // therefore not flagged.
+  return { filed, budget: f.perRound }
+}

package/src/globalstate.ts

@@ -0,0 +1,228 @@
+// src/globalstate.ts
+import { Database } from "bun:sqlite"
+import type { Window } from "./types"
+
+export interface UsageSample {
+  percent: number
+  at: number
+}
+
+export interface GlobalStore {
+  recordUsage(accountId: string, w: Window): void
+  lastUsage(accountId: string, kind: string, beforeMs: number): UsageSample | null
+  rate(provider: string, kind: string): number | null
+  observeRate(provider: string, kind: string, sample: number): number
+  spawnAdd(accountId: string, workspace: string, job: string, key: string, at: Date): void
+  reserve(accountId: string, workspace: string, job: string, key: string, at: Date, cap: number, sinceMs: number): boolean
+  confirm(job: string, key: string, at: Date): void
+  release(job: string, key: string, at: Date): void
+  // Scoped by workspace: job names and item-derived keys collide freely across
+  // repos, so two services each running a job called "build" keyed on issue
+  // numbers both hold a "b7". Unscoped, a restart in one looks up the other's
+  // account and starts the wrong agent kind against the wrong config dir.
+  accountFor(workspace: string, job: string, pathKey: string): string | null
+  spawnsSince(sinceMs: number): number
+  close(): void
+}
+
+// One sample moves the estimate a third of the way. High enough to follow a
+// model or plan change within a few ticks, low enough that a single noisy
+// interval does not swing the whole fleet's concurrency.
+export const EWMA_ALPHA = 0.3
+
+export function openGlobalState(path: string): GlobalStore {
+  const db = new Database(path)
+  migrate(db)
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS usage (
+      account        TEXT NOT NULL,
+      kind           TEXT NOT NULL,
+      percent        REAL NOT NULL,
+      resets_at      INTEGER NOT NULL,
+      window_minutes INTEGER NOT NULL,
+      at             INTEGER NOT NULL,
+      PRIMARY KEY (account, kind, at)
+    );
+    CREATE TABLE IF NOT EXISTS rates (
+      provider TEXT NOT NULL,
+      kind     TEXT NOT NULL,
+      ewma     REAL NOT NULL,
+      samples  INTEGER NOT NULL,
+      PRIMARY KEY (provider, kind)
+    );
+    CREATE TABLE IF NOT EXISTS spawns (
+      account   TEXT NOT NULL,
+      workspace TEXT NOT NULL DEFAULT '',
+      job       TEXT NOT NULL,
+      key       TEXT NOT NULL,
+      at        INTEGER NOT NULL,
+      pending   INTEGER NOT NULL DEFAULT 0,
+      PRIMARY KEY (job, key, at)
+    );
+  `)
+  // This database is shared across workspaces (spec section 7), and the
+  // router now writes a usage row and a rate EWMA update on every tick, so
+  // two workspaces ticking in the same minute will collide on a write lock.
+  // Without a busy timeout that is SQLITE_BUSY on a normal day; WAL lets
+  // reads and writes overlap and the timeout gives a blocked writer room to
+  // wait its turn instead of failing immediately.
+  db.exec(`
+    PRAGMA journal_mode = WAL;
+    PRAGMA busy_timeout = 5000;
+  `)
+
+  const insUsage = db.query(
+    `INSERT OR IGNORE INTO usage (account, kind, percent, resets_at, window_minutes, at)
+     VALUES (?, ?, ?, ?, ?, ?)`,
+  )
+  const prevUsage = db.query<UsageSample, [string, string, number]>(
+    `SELECT percent, at FROM usage
+     WHERE account = ? AND kind = ? AND at < ?
+     ORDER BY at DESC LIMIT 1`,
+  )
+  const getRate = db.query<{ ewma: number }, [string, string]>(
+    "SELECT ewma FROM rates WHERE provider = ? AND kind = ?",
+  )
+  const putRate = db.query(
+    `INSERT INTO rates (provider, kind, ewma, samples) VALUES (?, ?, ?, 1)
+     ON CONFLICT (provider, kind) DO UPDATE SET ewma = ?, samples = samples + 1`,
+  )
+  const insSpawn = db.query(
+    "INSERT OR IGNORE INTO spawns (account, workspace, job, key, at) VALUES (?, ?, ?, ?, ?)",
+  )
+  const insPending = db.query(
+    "INSERT OR IGNORE INTO spawns (account, workspace, job, key, at, pending) VALUES (?, ?, ?, ?, ?, 1)",
+  )
+  // Scoped by `at`, not just job/key: job names and item-derived keys collide
+  // freely across repos, so two workspaces can each hold a distinct pending
+  // row for the same (job, key) at different `at` values. Each workspace gets
+  // its own `now`, so the instant is what separates them; without the `at`
+  // predicate one workspace's confirm or release would match the other's live
+  // reservation.
+  const setConfirmed = db.query(
+    "UPDATE spawns SET pending = 0 WHERE job = ? AND key = ? AND at = ? AND pending = 1",
+  )
+  const delPending = db.query(
+    "DELETE FROM spawns WHERE job = ? AND key = ? AND at = ? AND pending = 1",
+  )
+  // The exact key, or a directory that carries a title slug after it.
+  // A legacy row (no workspace, from before the column existed) never
+  // matches here: it carries no workspace to disambiguate, so resolving it
+  // for any workspace's query is the cross-workspace collision this column
+  // exists to prevent. It still counts toward the daily cap (spawnsSince is
+  // workspace-agnostic); it just never hands back an account.
+  const findAccount = db.query<{ account: string }, [string, string, string, string]>(
+    `SELECT account FROM spawns
+     WHERE workspace = ? AND job = ? AND (key = ? OR ? LIKE key || '-%')
+     ORDER BY at DESC LIMIT 1`,
+  )
+  const countSince = db.query<{ n: number }, [number]>(
+    "SELECT COUNT(*) AS n FROM spawns WHERE at >= ?",
+  )
+
+  return {
+    recordUsage(accountId, w) {
+      insUsage.run(
+        accountId,
+        w.kind,
+        w.percent,
+        w.resetsAt.getTime(),
+        w.windowMinutes,
+        w.observedAt.getTime(),
+      )
+    },
+    lastUsage(accountId, kind, beforeMs) {
+      return prevUsage.get(accountId, kind, beforeMs)
+    },
+    rate(provider, kind) {
+      return getRate.get(provider, kind)?.ewma ?? null
+    },
+    observeRate(provider, kind, sample) {
+      const prev = getRate.get(provider, kind)?.ewma
+      const next = prev === undefined ? sample : EWMA_ALPHA * sample + (1 - EWMA_ALPHA) * prev
+      putRate.run(provider, kind, next, next)
+      return next
+    },
+    spawnAdd(accountId, workspace, job, key, at) {
+      insSpawn.run(accountId, workspace, job, key, at.getTime())
+    },
+    // tradeoff: nothing reaps an orphaned pending row (one left by a hard
+    // kill between reserve and confirm/release). It sits consuming a cap
+    // slot until spawnsSince's window rolls past UTC midnight, at which
+    // point it stops counting on its own. Add a reaper if that daily
+    // self-heal turns out not to be good enough in practice.
+    reserve(accountId, workspace, job, key, at, cap, sinceMs) {
+      // BEGIN IMMEDIATE takes the write lock before the count, so two
+      // workspaces ticking in the same minute cannot both read "one slot left"
+      // and both take it.
+      db.exec("BEGIN IMMEDIATE")
+      try {
+        const used = countSince.get(sinceMs)?.n ?? 0
+        if (used >= cap) {
+          db.exec("ROLLBACK")
+          return false
+        }
+        // OR IGNORE means the insert can silently do nothing (an exact
+        // account/job/key/at row already exists), and a caller must not be
+        // told it holds a reservation with no row backing it.
+        const { changes } = insPending.run(accountId, workspace, job, key, at.getTime())
+        db.exec("COMMIT")
+        return changes === 1
+      } catch (err) {
+        try {
+          db.exec("ROLLBACK")
+        } catch {
+          // SQLite may have already rolled back on its own; that must not
+          // replace the original error, which is the one that explains what
+          // actually went wrong.
+        }
+        throw err
+      }
+    },
+    confirm(job, key, at) {
+      setConfirmed.run(job, key, at.getTime())
+    },
+    release(job, key, at) {
+      // Only a pending row is deletable: a confirmed spawn stays on the record
+      // even if a later step decides to tidy up.
+      delPending.run(job, key, at.getTime())
+    },
+    accountFor(workspace, job, pathKey) {
+      return findAccount.get(workspace, job, pathKey, pathKey)?.account ?? null
+    },
+    spawnsSince(sinceMs) {
+      return countSince.get(sinceMs)?.n ?? 0
+    },
+    close() {
+      db.close()
+    },
+  }
+}
+
+// A database written before the ledger-to-spawns rename keeps a populated
+// `ledger` table, and CREATE TABLE IF NOT EXISTS happily creates an empty
+// `spawns` beside it: the daily cap resets to zero, every account reads zero
+// in flight and over-admits, and accountFor returns null for every worker
+// already running, which throws in applyRestart and tombstones the item on the
+// next tick. So the rename happens first, before any CREATE runs.
+function migrate(db: Database): void {
+  const has = (name: string) =>
+    db.query<{ n: number }, [string]>(
+      "SELECT COUNT(*) AS n FROM sqlite_master WHERE type = 'table' AND name = ?",
+    ).get(name)!.n > 0
+
+  if (has("ledger") && !has("spawns")) {
+    db.exec("ALTER TABLE ledger RENAME TO spawns")
+    db.exec("ALTER TABLE spawns RENAME COLUMN plugin TO job")
+  }
+  if (!has("spawns")) return
+  const columns = db.query<{ name: string }, []>("PRAGMA table_info(spawns)").all()
+  if (!columns.some((c) => c.name === "workspace")) {
+    // Pre-existing rows carry no workspace, and the empty string is the
+    // honest value for them: accountFor never matches it, so a legacy row
+    // still counts toward the daily cap but resolves no account. A worker
+    // that was in flight across the upgrade then fails once, restartable by
+    // hand, rather than risk resolving the wrong workspace's account.
+    db.exec("ALTER TABLE spawns ADD COLUMN workspace TEXT NOT NULL DEFAULT ''")
+  }
+}

package/src/journal.ts

@@ -0,0 +1,13 @@
+import { appendFileSync, mkdirSync } from "node:fs"
+import { dirname } from "node:path"
+import type { Ctx } from "./types"
+
+// The journal is markdown at ~/.agent-loop/<workspace>/journal.md, the one file
+// the fences let a worker write outside its worktree (spec 8). The engine
+// appends to the same file, so the operator reads one story.
+export function appendJournal(ctx: Ctx, line: string): void {
+  if (!ctx.live) return
+  const path = ctx.workspace.journalPath
+  mkdirSync(dirname(path), { recursive: true })
+  appendFileSync(path, `${ctx.now.toISOString()} ${line}\n`)
+}

package/src/kinds/builder.ts

@@ -0,0 +1,128 @@
+import type { Job, WorkItem, Ctx } from "../types"
+import type { Kind } from "./validate"
+import { issues, prs, unblocked, byPriority, newestByHead } from "./shared"
+import { renderBrief } from "../brief"
+import { branchName } from "../engine/naming"
+
+interface Options {
+  base: string
+  reviewDebt: number
+  debtIgnoreLabels: string[]
+  issueLabel: string
+  deleteRemote: boolean
+  sweepIgnoresWorking: boolean
+  copyIntoWorktree: string[]
+  journal: boolean
+  screenshots: boolean
+}
+
+export const builder: Kind = {
+  name: "builder",
+  workload: "builder",
+  fields: [
+    { name: "base", type: "string", default: "origin/main", doc: "the ref new work branches from" },
+    { name: "reviewDebt", type: "number", default: 0, doc: "stop taking issues at this many open pull requests, 0 to never stop" },
+    { name: "debtIgnoreLabels", type: "string[]", default: [], doc: "labels that mean a pull request is no longer the reviewer's debt, typically a reviewer's passLabel" },
+    { name: "issueLabel", type: "string", default: "", doc: "only take issues carrying this label, empty for all of them" },
+    { name: "deleteRemote", type: "boolean", default: true, doc: "delete the pushed branch when the work is swept" },
+    { name: "sweepIgnoresWorking", type: "boolean", default: true, doc: "sweep even while an agent works, because a merge is definitive here" },
+    { name: "copyIntoWorktree", type: "string[]", default: [], doc: "files copied from the repository into a fresh worktree" },
+    { name: "journal", type: "boolean", default: false, doc: "ask the worker to append one line to the journal" },
+    { name: "screenshots", type: "boolean", default: false, doc: "allow screenshots on an orphan asset branch" },
+  ],
+  build(spec) {
+    const o = spec.options as unknown as Options
+    const optional = [o.journal ? "journal" : "", o.screenshots ? "screenshots" : ""].filter(Boolean)
+
+    // One place the key format is written. It names the branch, the worktree
+    // and the pull request lookup, so a second spelling of it is a bug that
+    // only shows up as work rebuilt from scratch.
+    const keyFor = (item: WorkItem): string => `b${item.number}`
+
+    // A pull request whose head is this job's branch for that key, newest
+    // first: a retried issue's old closed pull request must not answer for the
+    // live one.
+    const prFor = async (ctx: Ctx, key: string): Promise<WorkItem | null> =>
+      newestByHead(await prs(ctx, job, "all"), branchName(job.name, key))
+
+    const job: Job = {
+      name: spec.name,
+      dir: spec.dir,
+      repo: spec.repo,
+      workload: "builder",
+      deleteRemote: o.deleteRemote,
+      sweepIgnoresWorking: o.sweepIgnoresWorking,
+      copyIntoWorktree: o.copyIntoWorktree,
+
+      // The throttle reads the reviewer's queue rather than this job's own
+      // item, which is why it is admit() and not guard(). Spec 5.1: the queue
+      // is every open pull request a reviewer still owes work on, claimed ones
+      // very much included, minus the two states a human owns. Excluding
+      // claimed items would count the queue as empty exactly when it is
+      // busiest, and two parked items would otherwise deadlock the pipeline
+      // forever, the builder throttled and the reviewer idle.
+      async admit(ctx) {
+        if (!o.reviewDebt) return null
+        const l = ctx.workspace.naming.labels
+        // debtIgnoreLabels alongside the two the spec names: a pass label is
+        // terminal for the reviewer, and under mergeMode none such a pull
+        // request stays open until continuous integration merges it, so
+        // counting it would throttle the builder on finished work.
+        const done = new Set([l.park, l.failed, ...o.debtIgnoreLabels].filter(Boolean))
+        const open = (await prs(ctx, job, "open")).filter(
+          (p) => !p.labels.some((name) => done.has(name)),
+        )
+        return open.length >= o.reviewDebt ? `review debt ${open.length}/${o.reviewDebt}` : null
+      },
+
+      async discover(ctx) {
+        const open = await issues(ctx, job, "open")
+        const mine = o.issueLabel ? open.filter((i) => i.labels.includes(o.issueLabel)) : open
+        return byPriority(ctx, unblocked(ctx, mine))
+      },
+
+      // --state all, per spec 4.5: a claim label left on a closed issue would
+      // otherwise never be seen, and its slot would be held forever.
+      async discoverClaimed(ctx) {
+        const claim = ctx.workspace.naming.labels.claim
+        return (await issues(ctx, job, "all")).filter((i) => i.labels.includes(claim))
+      },
+
+      key: async (_ctx, item) => keyFor(item),
+      base: async () => o.base,
+
+      // done() releases the claim as soon as the pull request exists, and the
+      // issue stays open until the merge closes it, so without this the next
+      // tick re-picks the issue and the spawn's pre-clean destroys the
+      // worktree the reviewer's rounds are still working in.
+      guard: async (ctx, item) => {
+        const pr = await prFor(ctx, keyFor(item))
+        // A closed, unmerged pull request is a legitimate retry.
+        return pr === null || pr.state === "CLOSED"
+      },
+
+      // The work is over when the pull request exists: what happens to it after
+      // that belongs to the reviewer, and holding the claim would count this
+      // issue against the builder's slots through every review round.
+      async done(ctx, item) {
+        return (await prFor(ctx, keyFor(item))) !== null
+      },
+
+      // Not done: the worktree has to survive until the pull request is
+      // finished, because the reviewer's rounds ask the builder's worker for
+      // changes in it.
+      async sweepOk(ctx, rawKey) {
+        const pr = await prFor(ctx, rawKey)
+        return pr !== null && pr.state !== "OPEN"
+      },
+
+      brief: (ctx, item) =>
+        renderBrief(ctx, job, item, keyFor(item), {
+          extends: spec.brief?.extends ?? "default/build",
+          optional,
+          append: spec.brief?.append,
+        }),
+    }
+    return job
+  },
+}

package/src/kinds/index.ts

@@ -0,0 +1,13 @@
+import type { Kind } from "./validate"
+import { builder } from "./builder"
+import { reviewer } from "./reviewer"
+import { routine } from "./routine"
+
+// The shipped kinds, and the whole registry. Nothing under a project folder is
+// imported, so this is a contract between the engine and these three rather
+// than an extension point (spec 5).
+export const KINDS: Record<string, Kind> = { builder, reviewer, routine }
+
+// The public surface of "./kinds" is unchanged: every existing importer, the
+// loader included, still reads the field table and the helpers from here.
+export * from "./validate"