@oakgreencc/cambium 0.0.28 → 0.0.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -39,7 +39,7 @@ Every command takes `--config <file>`; the default is `$CAMBIUM_CONFIG`, then `~
39
39
  Each step is bounded by `stepTimeoutSeconds`. A step that throws or times out is logged and recorded in the heartbeat, and the tick moves on. Nothing waits on a worker.
40
40
 
41
41
  1. **Read.** One GraphQL query per repo (`graft graphql`): open issues with any `q:` label, open and recently merged PRs on `agent/*` with their check rollup, merge-queue state, `mergeable`, `reviewDecision` and the issues they close. GitHub links closing references only for a PR into the default branch, so cambium also reads `Closes/Fixes/Resolves #n` from the PR body, fetching issues the first query did not read in one more. A worker's PR that names no issue (found by its branch) gets `Closes #n` added at submission.
42
- 2. **Reconcile.** For each tracked worker: alive and under its wall-clock cap → nothing. Over the cap → kill its process group and every descendant (Claude Code runs each Bash command in a group of its own, so a group kill alone left a test run loading the next attempt). Gone → exit check (a PR on its branch, or closing its issue, that is open or merged and not still on the commit CI already failed) → `submitted`; otherwise requeue with the stderr tail (a failed prepare step requeues as `spawn-failed`). Then the worktree is torn down if `graft status` says GitHub holds everything in it, else kept with a `warn` saying why.
42
+ 2. **Reconcile.** For each tracked worker: alive and under its wall-clock cap → nothing. Over the cap → terminate it: every process below `claude` (its Bash tools, a test run) is SIGKILLed at once, and `claude` gets SIGTERM, on which Claude Code ends its interaction span and flushes its exporter (a bare SIGKILL left `claude_code.*` spans orphaned under a root that was never exported). It usually exits within a second; still alive 30 s past the cap (and at least one tick later) → kill its process group and every descendant (Claude Code runs each Bash command in a group of its own, so a group kill alone left a test run loading the next attempt). Gone → exit check (a PR on its branch, or closing its issue, that is open or merged and not still on the commit CI already failed) → `submitted`; otherwise requeue with the stderr tail (a failed prepare step requeues as `spawn-failed`). Then the worktree is torn down if `graft status` says GitHub holds everything in it and no process has its cwd inside it (`lsof`; the worktree's own nx daemon excepted), else kept with a `warn` saying why. Teardown stops the worktree's nx daemon (nx keys each daemon to its workspace root, so no other checkout's is touched; left running, it served the next client from a deleted tree until `nx reset`), moves the tree to `<paths.home>/trash` with one rename, prunes git's record, and erases the trash in the background: a worktree with CocoaPods in it took `git worktree remove` past the step timeout.
43
43
  3. **Watch** (no agent). Graph engine: the [PR graph](#the-pr-graph), once per tick for each agent PR whose CI has settled and whose issue has no Task queued or running: merge, rework, close, or `q:human`. Legacy, for agent PRs whose issue is in no queue: merged → close the issue, delete the branch; red → requeue with the failing checks' summaries; left the merge queue without merging → requeue; not merged `mergeQueueMinutes` after submission → `q:human`.
44
44
  4. **Sweep.** An issue labelled `running` with this host (or no host) in its marker and no tracked worker → requeue. This is what makes losing the SQLite file cost one requeue.
45
45
  5. **Schedule.** Start Task workers for `q:task` issues without `running`, by priority label then age, up to `phases.task.cap` counted from live workers. A task requeued earlier in the same tick can start in that tick. Then, with `engine: "graph"` and `phases.dispatch.cap > 0`, start triage workers for `q:intake` issues the same way ([the dispatch graph](#the-dispatch-graph)).
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@oakgreencc/cambium",
3
- "version": "0.0.28",
3
+ "version": "0.0.29",
4
4
  "description": "The scheduler daemon: reads the q:* queues from GitHub every tick, runs headless claude workers in their own worktrees up to a cap, and makes every queue transition itself.",
5
5
  "license": "AGPL-3.0-only",
6
6
  "type": "module",
@@ -15,6 +15,7 @@ import { move, Q, WHY } from '../labels.mjs'
15
15
  import { newMarker, withHistory } from '../marker.mjs'
16
16
  import { readTriage, TriageResult, triageComment, triagePrompt } from '../phases/dispatch.mjs'
17
17
  import { traceWorker } from '../tracing.mjs'
18
+ import { stopOverdue } from '../workers.mjs'
18
19
  import { task } from './task.mjs'
19
20
 
20
21
  /**
@@ -242,14 +243,11 @@ const triaging = declare(
242
243
  return { ready: true, output: { exit: 'lost', ...lostLeaseOf(ctx) } }
243
244
  }
244
245
  const caps = config.phases.dispatch
245
- if (procs.alive(w.pid) && !reaped?.has(w.pid)) {
246
- if (!w.killedReason && ctx.tick.now - w.startedAt <= caps.wallClockMinutes * MIN) return { ready: false }
247
- const reason = w.killedReason ?? `killed: over the ${caps.wallClockMinutes} min wall-clock cap`
248
- procs.kill(w.pid)
249
- store.markKilled(w.repo, w.issue, reason)
250
- w.killedReason = reason
251
- ctx.log('warn', `${reason} (pid ${w.pid})`, { repo: w.repo, issue: w.issue })
252
- }
246
+ if (
247
+ !reaped?.has(w.pid) &&
248
+ stopOverdue({ procs, store, log: ctx.log }, w, caps.wallClockMinutes, ctx.tick.now) !== 'gone'
249
+ )
250
+ return { ready: false }
253
251
  if (!snap) return { ready: false }
254
252
  if (issue?.state !== 'OPEN' || !issue.labels.includes(Q.intake)) {
255
253
  ctx.log('info', 'triage worker gone; issue is closed or left q:intake', { repo: w.repo, issue: w.issue })
@@ -11,13 +11,14 @@
11
11
  import { z } from 'zod'
12
12
  import { defineGraph, defineNode, defineWait, END } from '../graph/index.mjs'
13
13
  import { branchName } from '../convention.mjs'
14
- import { iso, key, markerOf, MIN, setLabels, setMarker } from '../issue-state.mjs'
14
+ import { iso, key, markerOf, setLabels, setMarker } from '../issue-state.mjs'
15
15
  import { move, Q, WHY } from '../labels.mjs'
16
16
  import { withHistory } from '../marker.mjs'
17
17
  import { linkIssue, prFor, taskExit, taskPrompt } from '../phases/task.mjs'
18
18
  import { PREPARE_FAILED_LINE } from '../prepare.mjs'
19
19
  import { traceWorker } from '../tracing.mjs'
20
20
  import { baseRed, isRepair } from '../base-health.mjs'
21
+ import { stopOverdue } from '../workers.mjs'
21
22
 
22
23
  /**
23
24
  * @typedef {import('../github.mjs').Issue} Issue
@@ -266,14 +267,11 @@ const workerRunning = declare(
266
267
  return { ready: true, output: { exit: 'failed', ...lostLeaseOf(ctx) } }
267
268
  }
268
269
  const caps = config.phases[/** @type {'task'} */ (w.phase)] ?? config.phases.task
269
- if (procs.alive(w.pid) && !reaped?.has(w.pid)) {
270
- if (!w.killedReason && ctx.tick.now - w.startedAt <= caps.wallClockMinutes * MIN) return { ready: false }
271
- const reason = w.killedReason ?? `killed: over the ${caps.wallClockMinutes} min wall-clock cap`
272
- procs.kill(w.pid)
273
- store.markKilled(w.repo, w.issue, reason)
274
- w.killedReason = reason
275
- ctx.log('warn', `${reason} (pid ${w.pid})`, { repo: w.repo, issue: w.issue })
276
- }
270
+ if (
271
+ !reaped?.has(w.pid) &&
272
+ stopOverdue({ procs, store, log: ctx.log }, w, caps.wallClockMinutes, ctx.tick.now) !== 'gone'
273
+ )
274
+ return { ready: false }
277
275
  if (!snap) return { ready: false } // a dead worker waits for a readable snapshot
278
276
  if (issue?.state !== 'OPEN') {
279
277
  ctx.log('info', 'worker gone; issue is closed or no longer queued', { repo: w.repo, issue: w.issue })
package/src/tick.mjs CHANGED
@@ -17,6 +17,7 @@ import { releaseDue, releaseNotBuilt } from './phases/release.mjs'
17
17
  import { releaseGraphs } from './graphs/release.mjs'
18
18
  import { DISPATCH_NOT_BUILT, dispatchNotBuilt } from './phases/dispatch.mjs'
19
19
  import { isRepair } from './base-health.mjs'
20
+ import { stopOverdue } from './workers.mjs'
20
21
 
21
22
  /**
22
23
  * @typedef {import('./github.mjs').Issue} Issue
@@ -235,14 +236,7 @@ async function reconcile(deps, ctx) {
235
236
  store.workers(),
236
237
  async (w) => {
237
238
  const caps = config.phases[/** @type {'task'} */ (w.phase)] ?? config.phases.task
238
- if (procs.alive(w.pid)) {
239
- if (!w.killedReason && ctx.now - w.startedAt <= caps.wallClockMinutes * MIN) return
240
- const reason = w.killedReason ?? `killed: over the ${caps.wallClockMinutes} min wall-clock cap`
241
- procs.kill(w.pid)
242
- store.markKilled(w.repo, w.issue, reason)
243
- w.killedReason = reason
244
- deps.log('warn', `${reason} (pid ${w.pid})`, { repo: w.repo, issue: w.issue })
245
- }
239
+ if (stopOverdue({ procs, store, log: deps.log }, w, caps.wallClockMinutes, ctx.now) !== 'gone') return
246
240
  const snap = ctx.snaps.get(w.repo)
247
241
  if (snap) await finish(deps, ctx, w, snap)
248
242
  },
package/src/workers.mjs CHANGED
@@ -14,11 +14,13 @@ import {
14
14
  readdirSync,
15
15
  readFileSync,
16
16
  readSync,
17
+ realpathSync,
18
+ renameSync,
17
19
  rmSync,
18
20
  statSync,
19
21
  writeFileSync,
20
22
  } from 'node:fs'
21
- import { dirname, join } from 'node:path'
23
+ import { basename, dirname, join } from 'node:path'
22
24
  import { fileURLToPath } from 'node:url'
23
25
  import { readConvention, topicOf } from './convention.mjs'
24
26
  import { exec as defaultExec } from './github.mjs'
@@ -47,7 +49,15 @@ import { exec as defaultExec } from './github.mjs'
47
49
  * tail(path: string, lines?: number): string,
48
50
  * readLog(path: string): string,
49
51
  * }} Spawner
50
- * @typedef {{ alive(pid: number): boolean, kill(pid: number): void }} Procs
52
+ * @typedef {{ alive(pid: number): boolean, terminate(pid: number): void, kill(pid: number): void }} Procs
53
+ * `terminate`: the graceful first step (see {@link stopOverdue}); `kill`: SIGKILL, the whole tree.
54
+ * @typedef {{
55
+ * holders(worktree: string): number[],
56
+ * stopNxDaemon(worktree: string): number | null,
57
+ * erase(dir: string): void,
58
+ * }} WorktreeSys
59
+ * `holders`: pids with their cwd in the worktree, its nx daemon's tree excluded; `stopNxDaemon`: SIGTERM the
60
+ * worktree's own nx daemon, returning its pid; `erase`: delete a directory without waiting for it.
51
61
  */
52
62
 
53
63
  /** The workers' tools. Labels and the issue body are cambium's, so those graft verbs are denied. */
@@ -143,7 +153,7 @@ export function readLogFile(path) {
143
153
 
144
154
  /**
145
155
  * @param {{ config: import('./config.mjs').Config, exec?: import('./github.mjs').Exec,
146
- * spawnFn?: typeof spawn, stamp?: () => string }} opts
156
+ * spawnFn?: typeof spawn, stamp?: () => string, sys?: WorktreeSys }} opts
147
157
  * @returns {Spawner}
148
158
  */
149
159
  export function processSpawner({
@@ -151,9 +161,37 @@ export function processSpawner({
151
161
  exec = defaultExec,
152
162
  spawnFn = spawn,
153
163
  stamp = () => new Date().toISOString().replace(/[:.]/g, '-'),
164
+ sys = realWorktreeSys,
154
165
  }) {
155
166
  const git = (/** @type {string} */ repoPath, /** @type {string[]} */ args) => exec('git', ['-C', repoPath, ...args])
156
167
  const graftEnv = { ...process.env, ...(config.graft.app ? { GRAFT_APP: config.graft.app } : {}) }
168
+ const trash = join(config.paths.home, 'trash')
169
+ // A previous daemon's erase may have died with it (a reboot): finish the job.
170
+ if (existsSync(trash)) for (const d of readdirSync(trash)) sys.erase(join(trash, d))
171
+
172
+ /**
173
+ * Take a worktree away without waiting for its bytes to go: stop its nx daemon (left running, it answers the
174
+ * next client from a tree that no longer exists, until `nx reset`), move it into `<home>/trash` (one rename),
175
+ * prune git's record of it, and erase the trash in the background. A Pods-sized tree took `git worktree remove`
176
+ * past the 120 s step timeout. A rename that fails (another volume) falls back to removing it in place.
177
+ *
178
+ * @param {import('./config.mjs').Repo} repo @param {string} worktree
179
+ */
180
+ async function discard(repo, worktree) {
181
+ sys.stopNxDaemon(worktree)
182
+ const to = join(trash, `${basename(worktree)}-${stamp()}`)
183
+ try {
184
+ mkdirSync(trash, { recursive: true })
185
+ renameSync(worktree, to)
186
+ } catch {
187
+ if (existsSync(join(worktree, '.git'))) await git(repo.path, ['worktree', 'remove', '--force', worktree])
188
+ else rmSync(worktree, { recursive: true, force: true })
189
+ await git(repo.path, ['worktree', 'prune']).catch(() => {})
190
+ return
191
+ }
192
+ await git(repo.path, ['worktree', 'prune']).catch(() => {})
193
+ sys.erase(to)
194
+ }
157
195
 
158
196
  /**
159
197
  * Fast-forward the central checkout to the just-fetched base, so the convention
@@ -213,8 +251,12 @@ export function processSpawner({
213
251
  }
214
252
  const h = await held({ repo, worktree, branch })
215
253
  if (!h.clean) return { removed: false, reason: h.reason }
254
+ // The worker is gone, but a process it left (or a person) may still work in here; removing the tree from
255
+ // under it is worse than keeping it a while. GC retries later.
256
+ const busy = sys.holders(worktree)
257
+ if (busy.length) return { removed: false, reason: `in use by pid ${busy.join(', ')}` }
216
258
  const head = (await git(worktree, ['symbolic-ref', '--short', '-q', 'HEAD']).catch(() => '')).trim()
217
- await git(repo.path, ['worktree', 'remove', '--force', worktree])
259
+ await discard(repo, worktree)
218
260
  if (head === branch) await git(repo.path, ['branch', '-D', branch]).catch(() => {})
219
261
  return { removed: true, reason: null }
220
262
  }
@@ -224,11 +266,8 @@ export function processSpawner({
224
266
  * @param {import('./config.mjs').Repo} repo @param {string} worktree @returns {Promise<Teardown>}
225
267
  */
226
268
  async function dispose(repo, worktree) {
227
- if (existsSync(join(worktree, '.git'))) await git(repo.path, ['worktree', 'remove', '--force', worktree])
228
- else {
229
- rmSync(worktree, { recursive: true, force: true })
230
- await git(repo.path, ['worktree', 'prune']).catch(() => {})
231
- }
269
+ if (existsSync(worktree)) await discard(repo, worktree)
270
+ else await git(repo.path, ['worktree', 'prune']).catch(() => {})
232
271
  return { removed: true, reason: null }
233
272
  }
234
273
 
@@ -359,16 +398,64 @@ export function descendants(root, table) {
359
398
  return out
360
399
  }
361
400
 
362
- /** @param {number} target */
363
- function sigkill(target) {
401
+ /** @param {number} target @param {NodeJS.Signals} [signal] */
402
+ function sigkill(target, signal = 'SIGKILL') {
364
403
  try {
365
- process.kill(target, 'SIGKILL')
404
+ process.kill(target, signal)
366
405
  return true
367
406
  } catch {
368
407
  return false
369
408
  }
370
409
  }
371
410
 
411
+ const psTable = () => {
412
+ const ps = spawnSync('ps', ['-A', '-o', 'pid=,ppid='], { encoding: 'utf8' })
413
+ return ps.status === 0 ? ps.stdout : ''
414
+ }
415
+
416
+ /** The direct children of `pid` in a `pid ppid` table. @param {number} pid @param {string} table */
417
+ export const childrenOf = (pid, table) =>
418
+ table
419
+ .split('\n')
420
+ .map((l) => l.trim().split(/\s+/).map(Number))
421
+ .filter(([p, pp]) => pp === pid && p !== pid && Number.isInteger(p))
422
+ .map(([p]) => p)
423
+
424
+ /** How long a terminated worker gets to end its spans and exit before SIGKILL. */
425
+ export const KILL_GRACE_MS = 30 * 1000
426
+ const MIN_MS = 60 * 1000
427
+
428
+ /**
429
+ * The wall-clock cap, gracefully. Over the cap, {@link Procs.terminate}: the worker's tool processes (a test
430
+ * run) die at once, so the tool span ends, and Claude Code gets SIGTERM, on which it ends its interaction span
431
+ * and flushes its exporter (a bare SIGKILL left `claude_code.*` spans orphaned under a root that never came).
432
+ * Still alive {@link KILL_GRACE_MS} past the cap, and at least one call later: SIGKILL, the whole tree.
433
+ *
434
+ * @param {{ procs: Procs, store: { markKilled(repo: string, issue: number, reason: string): void },
435
+ * log: (level: any, msg: string, at?: any) => void }} deps
436
+ * @param {{ repo: string, issue: number, pid: number, startedAt: number, killedReason: string | null }} w
437
+ * @param {number} wallClockMinutes @param {number} now
438
+ * @returns {'running' | 'stopping' | 'gone'} `gone`: run the exit check now
439
+ */
440
+ export function stopOverdue({ procs, store, log }, w, wallClockMinutes, now) {
441
+ if (!procs.alive(w.pid)) return 'gone'
442
+ const at = { repo: w.repo, issue: w.issue }
443
+ const deadline = w.startedAt + wallClockMinutes * MIN_MS
444
+ if (!w.killedReason) {
445
+ if (now <= deadline) return 'running'
446
+ const reason = `killed: over the ${wallClockMinutes} min wall-clock cap`
447
+ procs.terminate(w.pid)
448
+ store.markKilled(w.repo, w.issue, reason)
449
+ w.killedReason = reason
450
+ log('warn', `${reason} (pid ${w.pid}; SIGTERM, SIGKILL in ${KILL_GRACE_MS / 1000} s)`, at)
451
+ return procs.alive(w.pid) ? 'stopping' : 'gone'
452
+ }
453
+ if (now <= deadline + KILL_GRACE_MS) return 'stopping'
454
+ procs.kill(w.pid)
455
+ log('warn', `pid ${w.pid} outlived SIGTERM; SIGKILL`, at)
456
+ return 'gone'
457
+ }
458
+
372
459
  /** @type {Procs} */
373
460
  export const realProcs = {
374
461
  alive(pid) {
@@ -383,10 +470,86 @@ export const realProcs = {
383
470
  // The process group is not enough: Claude Code runs each Bash command in a group of its own, so a test
384
471
  // run the worker started outlives the group kill and loads the next attempt. Snapshot the tree first
385
472
  // (killed children reparent to launchd and drop out of it), then kill it and the group.
386
- const ps = spawnSync('ps', ['-A', '-o', 'pid=,ppid='], { encoding: 'utf8' })
387
- const tree = ps.status === 0 ? descendants(pid, ps.stdout) : []
473
+ const tree = descendants(pid, psTable())
388
474
  // A detached worker leads its own process group: the negative pid takes its children with it.
389
475
  if (!sigkill(-pid)) sigkill(pid)
390
476
  for (const p of tree) sigkill(p)
391
477
  },
478
+ terminate(pid) {
479
+ // `pid` is the prepare step; its child is `claude`, which it exits with. Everything below `claude` (Bash
480
+ // tools, a test run) is SIGKILLed now, `claude` gets SIGTERM. Signalling `pid` itself would orphan `claude`.
481
+ const table = psTable()
482
+ const direct = childrenOf(pid, table)
483
+ const keep = new Set(direct)
484
+ for (const p of descendants(pid, table)) if (!keep.has(p)) sigkill(p)
485
+ if (direct.length) for (const p of direct) sigkill(p, 'SIGTERM')
486
+ else sigkill(pid, 'SIGTERM')
487
+ },
488
+ }
489
+
490
+ const NX_DAEMON_FILE = join('.nx', 'workspace-data', 'd', 'server-process.json')
491
+
492
+ /** A pid's command line, or '' when it has none (gone). @param {number} pid */
493
+ const commandOf = (pid) => {
494
+ const ps = spawnSync('ps', ['-o', 'command=', '-p', String(pid)], { encoding: 'utf8' })
495
+ return ps.status === 0 ? ps.stdout.trim() : ''
496
+ }
497
+
498
+ /**
499
+ * The nx daemon serving `worktree`: nx keys each daemon (pid file, socket) to its workspace root, so this one
500
+ * serves nobody else. Null unless its pid file names a live process whose command runs from this worktree (a
501
+ * reused pid is not ours).
502
+ *
503
+ * @param {string} worktree @param {{ read?: (p: string) => string, command?: (pid: number) => string }} [io]
504
+ */
505
+ export function nxDaemonOf(worktree, { read = (p) => readFileSync(p, 'utf8'), command = commandOf } = {}) {
506
+ let pid
507
+ try {
508
+ pid = JSON.parse(read(join(worktree, NX_DAEMON_FILE))).processId
509
+ } catch {
510
+ return null
511
+ }
512
+ if (!Number.isInteger(pid) || pid <= 0) return null
513
+ return command(pid).includes(`${worktree}/`) ? pid : null
514
+ }
515
+
516
+ /**
517
+ * Pids whose cwd is a root or inside it, from `lsof -F pn` output, less `except`.
518
+ * @param {string} out @param {string[]} roots the worktree, and its realpath if different @param {Set<number>} except
519
+ */
520
+ export function cwdHolders(out, roots, except) {
521
+ /** @type {number[]} */
522
+ const pids = []
523
+ let pid = 0
524
+ for (const line of out.split('\n')) {
525
+ if (line.startsWith('p')) pid = Number(line.slice(1))
526
+ else if (line.startsWith('n') && pid && !except.has(pid)) {
527
+ const cwd = line.slice(1)
528
+ if (roots.some((r) => cwd === r || cwd.startsWith(`${r}/`)) && !pids.includes(pid)) pids.push(pid)
529
+ }
530
+ }
531
+ return pids
532
+ }
533
+
534
+ /** @type {WorktreeSys} */
535
+ export const realWorktreeSys = {
536
+ holders(worktree) {
537
+ const lsof = spawnSync('lsof', ['-a', '-d', 'cwd', '-F', 'pn'], { encoding: 'utf8', timeout: 20_000 })
538
+ if (!lsof.stdout) return [] // no lsof: no evidence either way, as before this check existed
539
+ const daemon = nxDaemonOf(worktree)
540
+ const except = new Set([process.pid, ...(daemon ? [daemon, ...descendants(daemon, psTable())] : [])])
541
+ let real = worktree
542
+ try {
543
+ real = realpathSync(worktree)
544
+ } catch {}
545
+ return cwdHolders(lsof.stdout, [...new Set([worktree, real])], except)
546
+ },
547
+ stopNxDaemon(worktree) {
548
+ const pid = nxDaemonOf(worktree)
549
+ if (pid) sigkill(pid, 'SIGTERM') // nx stops its plugin workers and removes its socket on SIGTERM
550
+ return pid
551
+ },
552
+ erase(dir) {
553
+ spawn('rm', ['-rf', dir], { detached: true, stdio: 'ignore' }).unref()
554
+ },
392
555
  }