@tonoid/agent-loop 1.0.0

package/src/effects/spawn.ts

@@ -0,0 +1,246 @@
+import { copyFileSync, existsSync, mkdirSync, statSync, chmodSync } from "node:fs"
+import { dirname } from "node:path"
+import type { Ctx, Job, WorkItem, AccountConfig } from "../types"
+import { owns, keyOf, matchesCwd, worktreePath, branchName } from "../engine/naming"
+import { itemKind, repoOf, trackerless } from "../engine/item"
+import { withRepoLock } from "../lock"
+import { expandHome } from "../paths"
+import { startWorker } from "../runtime/worker"
+
+function repoPath(ctx: Ctx, p: Job): string {
+  const repo = ctx.workspace.repos[p.repo ?? ""]
+  if (!repo) throw new Error(`job "${p.name}" has no resolvable repo`)
+  return repo
+}
+
+// Idempotent, and it runs on every path. Round two of a multi-round job
+// otherwise fails at `worktree add -b` against a path and branch that both
+// still exist, and a crash between the claim and the worker start leaves an
+// orphan no sweep predicate can match.
+export async function preClean(ctx: Ctx, p: Job, key: string): Promise<void> {
+  const repo = repoPath(ctx, p)
+  const base = ctx.workspace.worktreeBase
+  const git = ctx.git(repo)
+
+  // Read fresh rather than through the tick cache: the sweep pass earlier in
+  // this same tick may already have removed some of these.
+  const worktrees = await git.worktrees()
+  const mine = worktrees.filter((wt) => owns(p.name, base, wt) && keyOf(p.name, wt.branch) === key)
+
+  const panes = await ctx.herdr.panes()
+  for (const wt of mine) {
+    const pane = panes.find((x) => matchesCwd(x.cwd, wt.path))
+    if (pane) await ctx.herdr.tabClose(pane.tabId)
+  }
+
+  await withRepoLock(repo, ctx.lock, async () => {
+    for (const wt of mine) await git.worktreeRemove(wt.path)
+    // By computed name rather than per worktree, and last so no worktree still
+    // holds it: `mine` only ever contains this key's branch anyway, and a
+    // branch whose worktree removal succeeded while its delete failed is listed
+    // by nothing here or in the sweep, yet `worktree add -b` refuses forever
+    // while it exists. Failure is ignored: usually it simply is not there.
+    await git.branchDelete(branchName(p.name, key)).catch(() => {})
+  })
+}
+
+// The write is checked. Unchecked, a forge 5xx produces a live worker with no
+// claim, the item is re-picked next tick, and the second spawn's pre-clean
+// removes the first worker's worktree out from under a running agent.
+export async function claim(ctx: Ctx, p: Job, item: WorkItem): Promise<boolean> {
+  // Nothing to claim, so nothing can fail to stick: the spawn proceeds and the
+  // spawned mark is the claim.
+  if (trackerless(item)) return true
+  const label = ctx.workspace.naming.labels.claim
+  const repo = repoOf(item)
+  const kind = itemKind(item)
+  await ctx.gh.label(repo, kind, item.number, { add: [label] })
+  return (await ctx.gh.labelsOf(repo, kind, item.number)).includes(label)
+}
+
+export async function unclaim(ctx: Ctx, item: WorkItem): Promise<void> {
+  if (trackerless(item)) return
+  await ctx.gh.label(repoOf(item), itemKind(item), item.number, {
+    remove: [ctx.workspace.naming.labels.claim],
+  })
+}
+
+export async function createWorktree(
+  ctx: Ctx,
+  p: Job,
+  item: WorkItem,
+  key: string,
+): Promise<string> {
+  if (!p.base) throw new Error(`job "${p.name}" has no base(), so it cannot spawn`)
+  const repo = repoPath(ctx, p)
+  const path = worktreePath(ctx.workspace.worktreeBase, p.name, key)
+  const branch = branchName(p.name, key)
+  const from = await p.base(ctx, item)
+  const git = ctx.git(repo)
+
+  // Both under the lock: two workspaces ticking the same minute against one
+  // repo collide on .git/worktrees/* and index.lock, and a transient git lock
+  // failure here would unclaim the item and count a strike against it.
+  await withRepoLock(repo, ctx.lock, async () => {
+    await git.fetch()
+    await git.worktreeAdd(path, branch, from)
+  })
+  return path
+}
+
+export async function prepareWorktree(ctx: Ctx, p: Job, path: string): Promise<void> {
+  const repo = repoPath(ctx, p)
+  for (const rel of p.copyIntoWorktree ?? []) {
+    const src = `${repo}/${rel}`
+    // A local env file that does not exist on this box is not a reason to
+    // fail the spawn and take the pipeline down with it.
+    if (!existsSync(src)) continue
+    const dst = `${path}/${rel}`
+    mkdirSync(dirname(dst), { recursive: true })
+    copyFileSync(src, dst)
+    // Preserve the mode: these are usually secrets at 0600, and copyFileSync
+    // applies the process umask instead.
+    chmodSync(dst, statSync(src).mode & 0o777)
+  }
+  if (p.prepare) await p.prepare(ctx, path)
+}
+
+// tab create is the only verb that accepts --env, so this is the sole channel
+// by which the router's account choice reaches a worker.
+export const CONFIG_ENV_BY_PROVIDER: Record<string, string> = {
+  claude: "CLAUDE_CONFIG_DIR",
+  codex: "CODEX_HOME",
+  grok: "GROK_HOME", // unverified: set configEnv on the account before routing real work
+}
+
+export const PANE_RETRIES = 5
+export const PANE_DELAY_MS = 1000
+
+// The pane appears a moment after the tab, and its id is never cached: herdr
+// ids are not stable, and the same logical worker has been seen at four
+// different pane ids across four rounds.
+export async function paneAt(ctx: Ctx, cwd: string): Promise<string> {
+  for (let attempt = 1; attempt <= PANE_RETRIES; attempt++) {
+    const panes = await ctx.herdr.panes()
+    const pane = panes.find((x) => matchesCwd(x.cwd, cwd))
+    if (pane) return pane.paneId
+    if (attempt < PANE_RETRIES) await ctx.sleep(PANE_DELAY_MS)
+  }
+  throw new Error(`no pane appeared at ${cwd} after ${PANE_RETRIES} looks`)
+}
+
+export async function startTab(
+  ctx: Ctx,
+  p: Job,
+  item: WorkItem,
+  key: string,
+  account: AccountConfig,
+): Promise<void> {
+  const label = ctx.workspace.herdrWorkspace
+  // By label, every spawn. A cached id survives exactly until the herdr
+  // server restarts, and then points at somebody else's workspace.
+  const workspaces = await ctx.herdr.workspaces()
+  const ws = workspaces.find((w) => w.label === label)
+  if (!ws) throw new Error(`no herdr workspace labelled "${label}"`)
+
+  const cwd = worktreePath(ctx.workspace.worktreeBase, p.name, key)
+  const name = `${p.name}-${key}`
+  const envVar = account.configEnv ?? CONFIG_ENV_BY_PROVIDER[account.provider] ?? "AGENT_CONFIG_DIR"
+
+  await ctx.herdr.tabCreate({
+    workspaceId: ws.id,
+    cwd,
+    label: name,
+    env: { [envVar]: expandHome(account.configDir) },
+  })
+
+  await startWorker(ctx, {
+    pane: await paneAt(ctx, cwd),
+    kind: account.agentKind ?? account.provider,
+    name,
+    // The account's args say how a worker starts on this account; the job's
+    // model says what it should be thinking with. Appended last so a job that
+    // names one wins over an account that already passed --model.
+    args: [...(account.startArgs ?? []), ...(p.model ? ["--model", p.model] : [])],
+    brief: await p.brief(ctx, item),
+  })
+}
+
+// One bad minute is infrastructure, not a verdict on the work; two on the same
+// key is a pattern worth a human's attention.
+export const STRIKE_MARK = "spawn-fail"
+
+export async function rollback(ctx: Ctx, p: Job, item: WorkItem, key: string): Promise<void> {
+  // Order matters: unclaim first, so a rollback that itself fails half way
+  // still leaves the item free for the next tick rather than claimed forever.
+  await unclaim(ctx, item)
+  await preClean(ctx, p, key)
+}
+
+export async function applySpawn(
+  ctx: Ctx,
+  p: Job,
+  item: WorkItem,
+  key: string,
+  account: AccountConfig,
+): Promise<void> {
+  // The reservation was taken in the router, before applySpawn ever runs, so
+  // any throw from here on out - including a pre-clean that fails or a claim
+  // that never sticks - must release it. Otherwise the row stays pending
+  // forever and keeps counting against the daily cap with nothing to show
+  // for it. A failed claim still means nothing was created, so it stays a
+  // skip, not a strike: only the try block below marks strikes.
+  try {
+    await preClean(ctx, p, key)
+
+    // A failed claim means nothing was created, so this is a skip, not a strike.
+    if (!(await claim(ctx, p, item))) {
+      throw new Error(`claim did not stick for ${p.name} ${key}, skipping this tick`)
+    }
+  } catch (err) {
+    ctx.global.release(p.name, key, ctx.now)
+    throw err
+  }
+
+  try {
+    const path = await createWorktree(ctx, p, item, key)
+    await prepareWorktree(ctx, p, path)
+    await startTab(ctx, p, item, key, account)
+  } catch (err) {
+    await rollback(ctx, p, item, key).catch(() => {
+      // A rollback that fails must not replace the original error, which is
+      // the one that explains what actually went wrong.
+    })
+    try {
+      ctx.global.release(p.name, key, ctx.now)
+    } catch {
+      // A release that fails must not replace the original error either, or
+      // skip the strike bookkeeping below.
+    }
+    if (ctx.marks.has(p.name, key, STRIKE_MARK)) {
+      const labels = ctx.workspace.naming.labels
+      // A routine has no url and so no label to move: its second strike is the
+      // mark being cleared and the error being reported, nothing more. And
+      // like the rollback and the release above, a label that fails must not
+      // replace the original error, which is the one explaining what broke.
+      if (!trackerless(item)) {
+        await ctx.gh
+          .label(repoOf(item), itemKind(item), item.number, {
+            add: [labels.failed],
+            remove: [labels.claim],
+          })
+          .catch(() => {})
+      }
+      ctx.marks.clear(p.name, key, STRIKE_MARK)
+    } else {
+      ctx.marks.set(p.name, key, STRIKE_MARK)
+    }
+    throw err
+  }
+
+  // Last, and only now: confirming any earlier would attribute quota to a
+  // worker that never started and skew the in-flight count for the day.
+  ctx.global.confirm(p.name, key, ctx.now)
+  ctx.marks.set(p.name, key, "spawned")
+  ctx.marks.clear(p.name, key, STRIKE_MARK)
+}

package/src/effects/sweep.ts

@@ -0,0 +1,27 @@
+import type { Ctx, Job } from "../types"
+import type { Worktree } from "../adapters/git"
+import { matchesCwd } from "../engine/naming"
+import { withRepoLock } from "../lock"
+
+export async function applySweep(ctx: Ctx, p: Job, wt: Worktree): Promise<void> {
+  const repo = ctx.workspace.repos[p.repo ?? ""]
+  if (!repo) return
+
+  // By tab id from pane list: a workspace id would close the whole loop
+  // workspace and every other worker in it, and a worker whose agent already
+  // exited still has a tab, so pane list rather than agent list.
+  const panes = await ctx.cache("engine:panes", () => ctx.herdr.panes())
+  const pane = panes.find((x) => matchesCwd(x.cwd, wt.path))
+  if (pane) await ctx.herdr.tabClose(pane.tabId)
+
+  const git = ctx.git(repo)
+  await withRepoLock(repo, ctx.lock, async () => {
+    await git.worktreeRemove(wt.path)
+    if (wt.branch) {
+      await git.branchDelete(wt.branch)
+      // Review branches are never pushed, so only a job that pushes asks
+      // for the remote side to be deleted.
+      if (p.deleteRemote) await git.remoteDelete(wt.branch)
+    }
+  })
+}

package/src/engine/item.ts

@@ -0,0 +1,32 @@
+import type { WorkItem } from "../types"
+
+// The gh adapter stamps every item's id as "<kind>:<number>".
+export function itemKind(item: WorkItem): "issue" | "pr" {
+  return item.id.startsWith("pr:") ? "pr" : "issue"
+}
+
+// gh takes an "owner/name" slug while a job's repo names a local path, so
+// the slug is derived from the item's own url. Anchored at the end so the
+// path is unambiguous, and host agnostic so a self-hosted forge resolves the
+// same way. A missing or unusable url yields null rather than a guess.
+const REPO_FROM_URL = /\/([^/]+\/[^/]+)\/(?:pull|issues)\/\d+\/?$/
+
+export function ghRepo(item: WorkItem): string | null {
+  return item.url?.match(REPO_FROM_URL)?.[1] ?? null
+}
+
+// A caller that needs a repo to act (label, comment) has no fallback: an item
+// with no usable url is an error to report, not a silent no-op. Shared here
+// because spawn's executor needs the identical wrapper.
+export function repoOf(item: WorkItem): string {
+  const repo = ghRepo(item)
+  if (!repo) throw new Error(`item ${item.id} has no url to derive a repo from`)
+  return repo
+}
+
+// A routine occurrence has no issue and no pull request behind it: the loop's
+// own mark and the worktree are its whole record. The writes that label and
+// unlabel an item have nothing to address, and must not throw on the way past.
+export function trackerless(item: WorkItem): boolean {
+  return ghRepo(item) === null
+}

package/src/engine/monitor.ts

@@ -0,0 +1,117 @@
+import type { Ctx, Job, Decision, MonitorAction } from "../types"
+import { worktreePath, matchesCwd } from "./naming"
+import { applyDone, applyNudge, applyEscalate, applyRestart, applyFail, applyNotifyBlocked } from "../effects/monitor"
+
+export async function monitorJob(ctx: Ctx, p: Job): Promise<Decision[]> {
+  const claimed = await ctx.cache(`engine:claimed:${p.name}`, () => p.discoverClaimed(ctx))
+  const agents = await ctx.cache("engine:agents", () => ctx.herdr.agents())
+  const panes = await ctx.cache("engine:panes", () => ctx.herdr.panes())
+  const timeout = ctx.config.blockedTimeoutMin ?? 180
+  const out: Decision[] = []
+
+  for (const item of claimed) {
+    const key = await p.key(ctx, item)
+    const wt = worktreePath(ctx.workspace.worktreeBase, p.name, key)
+    const mk = (action: MonitorAction, reason: string): Decision => ({
+      pass: "monitor", job: p.name, key, action, reason,
+    })
+    const act = async (fn: () => Promise<void>) => {
+      if (!ctx.live) return
+      try {
+        await fn()
+      } catch (err) {
+        // One item's action failing must not stop the pass: the remaining
+        // claimed items are unrelated to this failure.
+        out.push({ pass: "error", job: p.name, where: "monitor", reason: String(err) })
+      }
+    }
+
+    if (item.state !== "OPEN") {
+      out.push(mk("done", `state ${item.state}`))
+      await act(() => applyDone(ctx, item))
+      continue
+    }
+    if (await p.done(ctx, item)) {
+      if (ctx.marks.has(p.name, key, "spawned")) {
+        out.push(mk("done", "done() true"))
+      } else {
+        ctx.marks.set(p.name, key, "spawned")
+        out.push(mk("external", "done() true with no spawned mark"))
+      }
+      // Either way the work is over, so the claim comes off: an item that keeps
+      // it counts against the job's slots for good.
+      await act(() => applyDone(ctx, item))
+      continue
+    }
+
+    const agent = agents.find((a) => matchesCwd(a.cwd, wt))
+
+    if (agent?.status === "working") {
+      out.push(mk("busy", "agent working"))
+      continue
+    }
+
+    if (agent?.status === "blocked") {
+      const age = ctx.marks.age(p.name, key, "blocked")
+      if (age === null) {
+        ctx.marks.set(p.name, key, "blocked")
+        out.push(mk("blocked", "first sighting, notify once"))
+        await act(() => applyNotifyBlocked(ctx, p, item, key))
+      } else if (age >= timeout) {
+        ctx.marks.clear(p.name, key, "blocked")
+        out.push(mk("escalate", `blocked ${age}m >= ${timeout}m`))
+        await act(() => applyEscalate(ctx, p, item, key))
+      } else {
+        out.push(mk("hold", `blocked ${age}m < ${timeout}m`))
+      }
+      continue
+    }
+
+    if (agent?.status === "missing") {
+      // An unrecognized herdr agent_status maps to "missing" (see adapters/herdr.ts).
+      // Hold, not nudge or fail: hold cannot kill a live agent and cannot
+      // tombstone an item, so it is fail-safe if herdr renames or adds a status.
+      out.push(mk("hold", "agent status missing, holding"))
+      continue
+    }
+
+    if (!agent) {
+      const paneAlive = panes.some((pane) => matchesCwd(pane.cwd, wt))
+      // Once, per spec 11. A restart takes no spawns row, so an agent that
+      // keeps dying would otherwise be restarted every tick forever, burning
+      // real quota outside maxSpawnsPerDay.
+      if (paneAlive && !ctx.marks.has(p.name, key, "restarted")) {
+        ctx.marks.set(p.name, key, "restarted")
+        out.push(mk("restart", "pane alive, agent gone"))
+        await act(() => applyRestart(ctx, p, item, key))
+        continue
+      }
+      out.push(mk("fail", paneAlive ? "agent gone again after a restart" : "no agent and no pane"))
+      await act(() => applyFail(ctx, p, item, key))
+      continue
+    }
+
+    if (!ctx.marks.has(p.name, key, "nudged")) {
+      ctx.marks.set(p.name, key, "nudged")
+      out.push(mk("nudge", agent ? `agent ${agent.status}` : "no agent"))
+      await act(() => applyNudge(ctx, p, item, key))
+      continue
+    }
+
+    out.push(mk("fail", "still not done after a nudge"))
+    await act(() => applyFail(ctx, p, item, key))
+  }
+  return out
+}
+
+export async function monitorAll(ctx: Ctx, jobs: Job[]): Promise<Decision[]> {
+  const out: Decision[] = []
+  for (const p of jobs) {
+    try {
+      out.push(...(await monitorJob(ctx, p)))
+    } catch (err) {
+      out.push({ pass: "error", job: p.name, where: "monitor", reason: String(err) })
+    }
+  }
+  return out
+}

package/src/engine/naming.ts

@@ -0,0 +1,45 @@
+import type { Worktree } from "../adapters/git"
+
+export function worktreePath(base: string, job: string, key: string): string {
+  return `${base}/wt-${job}-${key}`
+}
+
+export function branchName(job: string, key: string): string {
+  return `${job}/${key}`
+}
+
+export function keyOf(job: string, branch: string | null): string | null {
+  if (!branch) return null
+  const prefix = `${job}/`
+  if (!branch.startsWith(prefix)) return null
+  const key = branch.slice(prefix.length)
+  return key.length > 0 ? key : null
+}
+
+export function owns(job: string, base: string, wt: Worktree): boolean {
+  const pathPrefix = `${base}/wt-${job}-`
+  if (!wt.path.startsWith(pathPrefix)) return false
+  const key = keyOf(job, wt.branch)
+  if (key === null) return false
+  // The path and the branch must name the same key. A worktree whose checked-out
+  // branch has drifted to another key would otherwise be swept under the wrong
+  // identity. A trailing "-<slug>" is still the same key, because spawn pre-clean
+  // matches worktrees by the "<key>-*" glob and slugged directories are real.
+  const suffix = wt.path.slice(pathPrefix.length)
+  return suffix === key || suffix.startsWith(`${key}-`)
+}
+
+export function matchesCwd(cwd: string, worktree: string): boolean {
+  if (cwd === worktree) return true
+  if (!cwd.startsWith(worktree)) return false
+  const rest = cwd.slice(worktree.length)
+  if (rest.startsWith("/")) return true
+  // A trailing "-" belongs to a title slug. A digit right after it is treated
+  // as a longer key instead (so key "b4" cannot match worktree "b4-8", which
+  // belongs to key "b48" or similar). The cost: a worktree whose title slug
+  // happens to start with a digit, e.g. "wt-review-r80-2fa-login", fails this
+  // match and becomes invisible to monitor and sweep even while a healthy
+  // agent is working in it. No slug-vs-key disambiguation exists to fix this
+  // without also changing how keys and slugs are told apart elsewhere.
+  return rest.startsWith("-") && !/^-\d/.test(rest)
+}

package/src/engine/spawn.ts

@@ -0,0 +1,101 @@
+import type { Ctx, Job, Decision } from "../types"
+import { chooseAccount } from "../router/route"
+import { applySpawn } from "../effects/spawn"
+
+// Shared with render.ts, which uses it to tell an idle job (no candidates)
+// apart from a job skipped for any other reason.
+export const IDLE_REASON = "idle"
+
+export async function spawnOne(
+  ctx: Ctx,
+  jobs: Job[],
+  paused: string[] = [],
+): Promise<Decision[]> {
+  const out: Decision[] = []
+  const skip = (job: string, reason: string): Decision => ({
+    pass: "spawn", job, key: "", action: "skip", reason,
+  })
+
+  for (const p of jobs) {
+    try {
+      if (paused.includes(p.name)) {
+        out.push(skip(p.name, "paused"))
+        continue
+      }
+
+      if (p.admit) {
+        const reason = await p.admit(ctx)
+        if (reason) {
+          out.push(skip(p.name, reason))
+          continue
+        }
+      }
+
+      const slots = p.slots ?? 1
+      const inFlight = await ctx.cache(`engine:claimed:${p.name}`, () => p.discoverClaimed(ctx))
+      if (inFlight.length >= slots) {
+        out.push(skip(p.name, `slots ${inFlight.length}/${slots} in flight`))
+        continue
+      }
+
+      // Cached under the same key the filing budget reads, so a reviewer that
+      // asks how deep the builder's queue is and the builder's own spawn pass
+      // pay for one remote read between them.
+      const candidates = await ctx.cache(`engine:discover:${p.name}`, () => p.discover(ctx))
+      if (candidates.length === 0) {
+        out.push(skip(p.name, IDLE_REASON))
+        continue
+      }
+
+      let picked = null
+      for (const c of candidates) {
+        if (p.guard && !(await p.guard(ctx, c))) continue
+        picked = c
+        break
+      }
+      if (!picked) {
+        out.push(skip(p.name, `all ${candidates.length} candidates guarded out`))
+        continue
+      }
+
+      const key = await p.key(ctx, picked)
+      const route = await chooseAccount(ctx, p, picked)
+      if (!route.ok) {
+        out.push({ pass: "spawn", job: p.name, key, action: "skip", reason: route.reason })
+        // CAP and LOWMEM describe the day and the box, so no later job can
+        // do better. STARVED is this job's own requires/prefer, so the walk
+        // continues; stopping there would let one constrained job mute the
+        // whole workspace.
+        if (route.global) return out
+        continue
+      }
+
+      out.push({
+        pass: "spawn",
+        job: p.name,
+        key,
+        action: "spawn",
+        account: route.account,
+        reason: route.reason,
+      })
+      if (ctx.live) {
+        const account = ctx.config.accounts.find((a) => a.id === route.account)!
+        try {
+          await applySpawn(ctx, p, picked, key, account)
+        } catch (err) {
+          out.push({ pass: "error", job: p.name, where: "spawn", reason: String(err) })
+        }
+      }
+      // A live tick spawns one worker and stops, because the next tick re-reads
+      // the world with that worker in it. A dry tick spawns nothing and its
+      // marks are not persisted, so stopping here would hide every job after
+      // this one behind the first due item and the mask would never lift: a
+      // shadow week reports one job for seven days. Nothing downstream of the
+      // decision is reserved without ctx.live, so the walk is free to continue.
+      if (ctx.live) return out
+    } catch (err) {
+      out.push({ pass: "error", job: p.name, where: "spawn", reason: String(err) })
+    }
+  }
+  return out
+}

package/src/engine/sweep.ts

@@ -0,0 +1,91 @@
+import type { Ctx, Job, Decision, WorkItem } from "../types"
+import { owns, keyOf, matchesCwd } from "./naming"
+import { applySweep } from "../effects/sweep"
+import { auditFiling } from "../filing"
+import { appendJournal } from "../journal"
+import { renderDecision } from "../render"
+
+async function isFinished(ctx: Ctx, p: Job, rawKey: string): Promise<boolean> {
+  if (p.sweepOk) return p.sweepOk(ctx, rawKey)
+  const digits = rawKey.match(/\d+/g) ?? []
+  const synthetic: WorkItem = {
+    id: `key:${rawKey}`,
+    // A synthetic item has no real number. Use one only when the key holds a
+    // single digit run ("r80" -> 80). A multi-group key such as a date is not a
+    // number, and a fabricated one would mislead done(); those jobs define
+    // sweepOk instead.
+    number: digits.length === 1 ? Number.parseInt(digits[0]!, 10) : 0,
+    title: "",
+    state: "OPEN",
+    labels: [],
+  }
+  return p.done(ctx, synthetic)
+}
+
+export async function sweepJob(ctx: Ctx, p: Job): Promise<Decision[]> {
+  const repo = ctx.workspace.repos[p.repo ?? ""]
+  if (!repo) return []
+  const base = ctx.workspace.worktreeBase
+  // "engine:" prefixes every cache key the engine owns (here and in monitor.ts
+  // and spawn.ts), reserving that namespace so a job's own ctx.cache key can
+  // never collide with, and poison, the engine's snapshot for this tick.
+  const worktrees = await ctx.cache(`engine:worktrees:${repo}`, () => ctx.git(repo).worktrees())
+  const agents = await ctx.cache("engine:agents", () => ctx.herdr.agents())
+  const out: Decision[] = []
+
+  for (const wt of worktrees) {
+    if (!owns(p.name, base, wt)) continue
+    const rawKey = keyOf(p.name, wt.branch)!
+    const mk = (action: "clean" | "hold", reason: string): Decision => ({
+      pass: "sweep", job: p.name, worktree: wt.path, branch: wt.branch!, action, reason,
+    })
+
+    const live = agents.some((a) => a.status === "working" && matchesCwd(a.cwd, wt.path))
+    if (live && !p.sweepIgnoresWorking) {
+      out.push(mk("hold", "agent working"))
+      continue
+    }
+    const predicate = p.sweepOk ? "sweepOk" : "done"
+    if (!(await isFinished(ctx, p, rawKey))) {
+      out.push(mk("hold", `${predicate}(${rawKey}) false`))
+      continue
+    }
+    out.push(mk("clean", `${predicate}(${rawKey})`))
+    // The run is over, so this is the moment its output can be counted. Failure
+    // here is reported and never blocks the cleanup: an audit is bookkeeping.
+    if (p.filing) {
+      try {
+        const audit = await auditFiling(ctx, p, rawKey)
+        if (audit && audit.filed > audit.budget) {
+          const d: Decision = { pass: "audit", job: p.name, key: rawKey, filed: audit.filed, budget: audit.budget }
+          out.push(d)
+          appendJournal(ctx, renderDecision(d, ctx.live))
+        }
+      } catch (err) {
+        out.push({ pass: "error", job: p.name, where: "sweep", reason: String(err) })
+      }
+    }
+    if (ctx.live) {
+      try {
+        await applySweep(ctx, p, wt)
+      } catch (err) {
+        // One worktree that will not clean up must not strand the others: the
+        // next worktree in this job's list is unrelated to this failure.
+        out.push({ pass: "error", job: p.name, where: "sweep", reason: String(err) })
+      }
+    }
+  }
+  return out
+}
+
+export async function sweepAll(ctx: Ctx, jobs: Job[]): Promise<Decision[]> {
+  const out: Decision[] = []
+  for (const p of jobs) {
+    try {
+      out.push(...(await sweepJob(ctx, p)))
+    } catch (err) {
+      out.push({ pass: "error", job: p.name, where: "sweep", reason: String(err) })
+    }
+  }
+  return out
+}