@oakgreencc/cambium 0.0.27 → 0.0.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +81 -60
- package/package.json +1 -1
- package/src/cli.mjs +1 -1
- package/src/config.mjs +5 -2
- package/src/github.mjs +8 -1
- package/src/graph-engine.mjs +95 -39
- package/src/graphs/dispatch.mjs +6 -8
- package/src/graphs/pr.mjs +420 -0
- package/src/graphs/task.mjs +67 -321
- package/src/marker.mjs +3 -1
- package/src/phases/task.mjs +4 -0
- package/src/tick.mjs +2 -8
- package/src/workers.mjs +177 -14
package/src/workers.mjs
CHANGED
|
@@ -14,11 +14,13 @@ import {
|
|
|
14
14
|
readdirSync,
|
|
15
15
|
readFileSync,
|
|
16
16
|
readSync,
|
|
17
|
+
realpathSync,
|
|
18
|
+
renameSync,
|
|
17
19
|
rmSync,
|
|
18
20
|
statSync,
|
|
19
21
|
writeFileSync,
|
|
20
22
|
} from 'node:fs'
|
|
21
|
-
import { dirname, join } from 'node:path'
|
|
23
|
+
import { basename, dirname, join } from 'node:path'
|
|
22
24
|
import { fileURLToPath } from 'node:url'
|
|
23
25
|
import { readConvention, topicOf } from './convention.mjs'
|
|
24
26
|
import { exec as defaultExec } from './github.mjs'
|
|
@@ -47,7 +49,15 @@ import { exec as defaultExec } from './github.mjs'
|
|
|
47
49
|
* tail(path: string, lines?: number): string,
|
|
48
50
|
* readLog(path: string): string,
|
|
49
51
|
* }} Spawner
|
|
50
|
-
* @typedef {{ alive(pid: number): boolean, kill(pid: number): void }} Procs
|
|
52
|
+
* @typedef {{ alive(pid: number): boolean, terminate(pid: number): void, kill(pid: number): void }} Procs
|
|
53
|
+
* `terminate`: the graceful first step (see {@link stopOverdue}); `kill`: SIGKILL, the whole tree.
|
|
54
|
+
* @typedef {{
|
|
55
|
+
* holders(worktree: string): number[],
|
|
56
|
+
* stopNxDaemon(worktree: string): number | null,
|
|
57
|
+
* erase(dir: string): void,
|
|
58
|
+
* }} WorktreeSys
|
|
59
|
+
* `holders`: pids with their cwd in the worktree, its nx daemon's tree excluded; `stopNxDaemon`: SIGTERM the
|
|
60
|
+
* worktree's own nx daemon, returning its pid; `erase`: delete a directory without waiting for it.
|
|
51
61
|
*/
|
|
52
62
|
|
|
53
63
|
/** The workers' tools. Labels and the issue body are cambium's, so those graft verbs are denied. */
|
|
@@ -143,7 +153,7 @@ export function readLogFile(path) {
|
|
|
143
153
|
|
|
144
154
|
/**
|
|
145
155
|
* @param {{ config: import('./config.mjs').Config, exec?: import('./github.mjs').Exec,
|
|
146
|
-
* spawnFn?: typeof spawn, stamp?: () => string }} opts
|
|
156
|
+
* spawnFn?: typeof spawn, stamp?: () => string, sys?: WorktreeSys }} opts
|
|
147
157
|
* @returns {Spawner}
|
|
148
158
|
*/
|
|
149
159
|
export function processSpawner({
|
|
@@ -151,9 +161,37 @@ export function processSpawner({
|
|
|
151
161
|
exec = defaultExec,
|
|
152
162
|
spawnFn = spawn,
|
|
153
163
|
stamp = () => new Date().toISOString().replace(/[:.]/g, '-'),
|
|
164
|
+
sys = realWorktreeSys,
|
|
154
165
|
}) {
|
|
155
166
|
const git = (/** @type {string} */ repoPath, /** @type {string[]} */ args) => exec('git', ['-C', repoPath, ...args])
|
|
156
167
|
const graftEnv = { ...process.env, ...(config.graft.app ? { GRAFT_APP: config.graft.app } : {}) }
|
|
168
|
+
const trash = join(config.paths.home, 'trash')
|
|
169
|
+
// A previous daemon's erase may have died with it (a reboot): finish the job.
|
|
170
|
+
if (existsSync(trash)) for (const d of readdirSync(trash)) sys.erase(join(trash, d))
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* Take a worktree away without waiting for its bytes to go: stop its nx daemon (left running, it answers the
|
|
174
|
+
* next client from a tree that no longer exists, until `nx reset`), move it into `<home>/trash` (one rename),
|
|
175
|
+
* prune git's record of it, and erase the trash in the background. A Pods-sized tree took `git worktree remove`
|
|
176
|
+
* past the 120 s step timeout. A rename that fails (another volume) falls back to removing it in place.
|
|
177
|
+
*
|
|
178
|
+
* @param {import('./config.mjs').Repo} repo @param {string} worktree
|
|
179
|
+
*/
|
|
180
|
+
async function discard(repo, worktree) {
|
|
181
|
+
sys.stopNxDaemon(worktree)
|
|
182
|
+
const to = join(trash, `${basename(worktree)}-${stamp()}`)
|
|
183
|
+
try {
|
|
184
|
+
mkdirSync(trash, { recursive: true })
|
|
185
|
+
renameSync(worktree, to)
|
|
186
|
+
} catch {
|
|
187
|
+
if (existsSync(join(worktree, '.git'))) await git(repo.path, ['worktree', 'remove', '--force', worktree])
|
|
188
|
+
else rmSync(worktree, { recursive: true, force: true })
|
|
189
|
+
await git(repo.path, ['worktree', 'prune']).catch(() => {})
|
|
190
|
+
return
|
|
191
|
+
}
|
|
192
|
+
await git(repo.path, ['worktree', 'prune']).catch(() => {})
|
|
193
|
+
sys.erase(to)
|
|
194
|
+
}
|
|
157
195
|
|
|
158
196
|
/**
|
|
159
197
|
* Fast-forward the central checkout to the just-fetched base, so the convention
|
|
@@ -213,8 +251,12 @@ export function processSpawner({
|
|
|
213
251
|
}
|
|
214
252
|
const h = await held({ repo, worktree, branch })
|
|
215
253
|
if (!h.clean) return { removed: false, reason: h.reason }
|
|
254
|
+
// The worker is gone, but a process it left (or a person) may still work in here; removing the tree from
|
|
255
|
+
// under it is worse than keeping it a while. GC retries later.
|
|
256
|
+
const busy = sys.holders(worktree)
|
|
257
|
+
if (busy.length) return { removed: false, reason: `in use by pid ${busy.join(', ')}` }
|
|
216
258
|
const head = (await git(worktree, ['symbolic-ref', '--short', '-q', 'HEAD']).catch(() => '')).trim()
|
|
217
|
-
await
|
|
259
|
+
await discard(repo, worktree)
|
|
218
260
|
if (head === branch) await git(repo.path, ['branch', '-D', branch]).catch(() => {})
|
|
219
261
|
return { removed: true, reason: null }
|
|
220
262
|
}
|
|
@@ -224,11 +266,8 @@ export function processSpawner({
|
|
|
224
266
|
* @param {import('./config.mjs').Repo} repo @param {string} worktree @returns {Promise<Teardown>}
|
|
225
267
|
*/
|
|
226
268
|
async function dispose(repo, worktree) {
|
|
227
|
-
if (existsSync(
|
|
228
|
-
else {
|
|
229
|
-
rmSync(worktree, { recursive: true, force: true })
|
|
230
|
-
await git(repo.path, ['worktree', 'prune']).catch(() => {})
|
|
231
|
-
}
|
|
269
|
+
if (existsSync(worktree)) await discard(repo, worktree)
|
|
270
|
+
else await git(repo.path, ['worktree', 'prune']).catch(() => {})
|
|
232
271
|
return { removed: true, reason: null }
|
|
233
272
|
}
|
|
234
273
|
|
|
@@ -359,16 +398,64 @@ export function descendants(root, table) {
|
|
|
359
398
|
return out
|
|
360
399
|
}
|
|
361
400
|
|
|
362
|
-
/** @param {number} target */
|
|
363
|
-
function sigkill(target) {
|
|
401
|
+
/** @param {number} target @param {NodeJS.Signals} [signal] */
|
|
402
|
+
function sigkill(target, signal = 'SIGKILL') {
|
|
364
403
|
try {
|
|
365
|
-
process.kill(target,
|
|
404
|
+
process.kill(target, signal)
|
|
366
405
|
return true
|
|
367
406
|
} catch {
|
|
368
407
|
return false
|
|
369
408
|
}
|
|
370
409
|
}
|
|
371
410
|
|
|
411
|
+
const psTable = () => {
|
|
412
|
+
const ps = spawnSync('ps', ['-A', '-o', 'pid=,ppid='], { encoding: 'utf8' })
|
|
413
|
+
return ps.status === 0 ? ps.stdout : ''
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
/** The direct children of `pid` in a `pid ppid` table. @param {number} pid @param {string} table */
|
|
417
|
+
export const childrenOf = (pid, table) =>
|
|
418
|
+
table
|
|
419
|
+
.split('\n')
|
|
420
|
+
.map((l) => l.trim().split(/\s+/).map(Number))
|
|
421
|
+
.filter(([p, pp]) => pp === pid && p !== pid && Number.isInteger(p))
|
|
422
|
+
.map(([p]) => p)
|
|
423
|
+
|
|
424
|
+
/** How long a terminated worker gets to end its spans and exit before SIGKILL. */
|
|
425
|
+
export const KILL_GRACE_MS = 30 * 1000
|
|
426
|
+
const MIN_MS = 60 * 1000
|
|
427
|
+
|
|
428
|
+
/**
|
|
429
|
+
* The wall-clock cap, gracefully. Over the cap, {@link Procs.terminate}: the worker's tool processes (a test
|
|
430
|
+
* run) die at once, so the tool span ends, and Claude Code gets SIGTERM, on which it ends its interaction span
|
|
431
|
+
* and flushes its exporter (a bare SIGKILL left `claude_code.*` spans orphaned under a root that never came).
|
|
432
|
+
* Still alive {@link KILL_GRACE_MS} past the cap, and at least one call later: SIGKILL, the whole tree.
|
|
433
|
+
*
|
|
434
|
+
* @param {{ procs: Procs, store: { markKilled(repo: string, issue: number, reason: string): void },
|
|
435
|
+
* log: (level: any, msg: string, at?: any) => void }} deps
|
|
436
|
+
* @param {{ repo: string, issue: number, pid: number, startedAt: number, killedReason: string | null }} w
|
|
437
|
+
* @param {number} wallClockMinutes @param {number} now
|
|
438
|
+
* @returns {'running' | 'stopping' | 'gone'} `gone`: run the exit check now
|
|
439
|
+
*/
|
|
440
|
+
export function stopOverdue({ procs, store, log }, w, wallClockMinutes, now) {
|
|
441
|
+
if (!procs.alive(w.pid)) return 'gone'
|
|
442
|
+
const at = { repo: w.repo, issue: w.issue }
|
|
443
|
+
const deadline = w.startedAt + wallClockMinutes * MIN_MS
|
|
444
|
+
if (!w.killedReason) {
|
|
445
|
+
if (now <= deadline) return 'running'
|
|
446
|
+
const reason = `killed: over the ${wallClockMinutes} min wall-clock cap`
|
|
447
|
+
procs.terminate(w.pid)
|
|
448
|
+
store.markKilled(w.repo, w.issue, reason)
|
|
449
|
+
w.killedReason = reason
|
|
450
|
+
log('warn', `${reason} (pid ${w.pid}; SIGTERM, SIGKILL in ${KILL_GRACE_MS / 1000} s)`, at)
|
|
451
|
+
return procs.alive(w.pid) ? 'stopping' : 'gone'
|
|
452
|
+
}
|
|
453
|
+
if (now <= deadline + KILL_GRACE_MS) return 'stopping'
|
|
454
|
+
procs.kill(w.pid)
|
|
455
|
+
log('warn', `pid ${w.pid} outlived SIGTERM; SIGKILL`, at)
|
|
456
|
+
return 'gone'
|
|
457
|
+
}
|
|
458
|
+
|
|
372
459
|
/** @type {Procs} */
|
|
373
460
|
export const realProcs = {
|
|
374
461
|
alive(pid) {
|
|
@@ -383,10 +470,86 @@ export const realProcs = {
|
|
|
383
470
|
// The process group is not enough: Claude Code runs each Bash command in a group of its own, so a test
|
|
384
471
|
// run the worker started outlives the group kill and loads the next attempt. Snapshot the tree first
|
|
385
472
|
// (killed children reparent to launchd and drop out of it), then kill it and the group.
|
|
386
|
-
const
|
|
387
|
-
const tree = ps.status === 0 ? descendants(pid, ps.stdout) : []
|
|
473
|
+
const tree = descendants(pid, psTable())
|
|
388
474
|
// A detached worker leads its own process group: the negative pid takes its children with it.
|
|
389
475
|
if (!sigkill(-pid)) sigkill(pid)
|
|
390
476
|
for (const p of tree) sigkill(p)
|
|
391
477
|
},
|
|
478
|
+
terminate(pid) {
|
|
479
|
+
// `pid` is the prepare step; its child is `claude`, which it exits with. Everything below `claude` (Bash
|
|
480
|
+
// tools, a test run) is SIGKILLed now, `claude` gets SIGTERM. Signalling `pid` itself would orphan `claude`.
|
|
481
|
+
const table = psTable()
|
|
482
|
+
const direct = childrenOf(pid, table)
|
|
483
|
+
const keep = new Set(direct)
|
|
484
|
+
for (const p of descendants(pid, table)) if (!keep.has(p)) sigkill(p)
|
|
485
|
+
if (direct.length) for (const p of direct) sigkill(p, 'SIGTERM')
|
|
486
|
+
else sigkill(pid, 'SIGTERM')
|
|
487
|
+
},
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
const NX_DAEMON_FILE = join('.nx', 'workspace-data', 'd', 'server-process.json')
|
|
491
|
+
|
|
492
|
+
/** A pid's command line, or '' when it has none (gone). @param {number} pid */
|
|
493
|
+
const commandOf = (pid) => {
|
|
494
|
+
const ps = spawnSync('ps', ['-o', 'command=', '-p', String(pid)], { encoding: 'utf8' })
|
|
495
|
+
return ps.status === 0 ? ps.stdout.trim() : ''
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
/**
|
|
499
|
+
* The nx daemon serving `worktree`: nx keys each daemon (pid file, socket) to its workspace root, so this one
|
|
500
|
+
* serves nobody else. Null unless its pid file names a live process whose command runs from this worktree (a
|
|
501
|
+
* reused pid is not ours).
|
|
502
|
+
*
|
|
503
|
+
* @param {string} worktree @param {{ read?: (p: string) => string, command?: (pid: number) => string }} [io]
|
|
504
|
+
*/
|
|
505
|
+
export function nxDaemonOf(worktree, { read = (p) => readFileSync(p, 'utf8'), command = commandOf } = {}) {
|
|
506
|
+
let pid
|
|
507
|
+
try {
|
|
508
|
+
pid = JSON.parse(read(join(worktree, NX_DAEMON_FILE))).processId
|
|
509
|
+
} catch {
|
|
510
|
+
return null
|
|
511
|
+
}
|
|
512
|
+
if (!Number.isInteger(pid) || pid <= 0) return null
|
|
513
|
+
return command(pid).includes(`${worktree}/`) ? pid : null
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
/**
|
|
517
|
+
* Pids whose cwd is a root or inside it, from `lsof -F pn` output, less `except`.
|
|
518
|
+
* @param {string} out @param {string[]} roots the worktree, and its realpath if different @param {Set<number>} except
|
|
519
|
+
*/
|
|
520
|
+
export function cwdHolders(out, roots, except) {
|
|
521
|
+
/** @type {number[]} */
|
|
522
|
+
const pids = []
|
|
523
|
+
let pid = 0
|
|
524
|
+
for (const line of out.split('\n')) {
|
|
525
|
+
if (line.startsWith('p')) pid = Number(line.slice(1))
|
|
526
|
+
else if (line.startsWith('n') && pid && !except.has(pid)) {
|
|
527
|
+
const cwd = line.slice(1)
|
|
528
|
+
if (roots.some((r) => cwd === r || cwd.startsWith(`${r}/`)) && !pids.includes(pid)) pids.push(pid)
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
return pids
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
/** @type {WorktreeSys} */
|
|
535
|
+
export const realWorktreeSys = {
|
|
536
|
+
holders(worktree) {
|
|
537
|
+
const lsof = spawnSync('lsof', ['-a', '-d', 'cwd', '-F', 'pn'], { encoding: 'utf8', timeout: 20_000 })
|
|
538
|
+
if (!lsof.stdout) return [] // no lsof: no evidence either way, as before this check existed
|
|
539
|
+
const daemon = nxDaemonOf(worktree)
|
|
540
|
+
const except = new Set([process.pid, ...(daemon ? [daemon, ...descendants(daemon, psTable())] : [])])
|
|
541
|
+
let real = worktree
|
|
542
|
+
try {
|
|
543
|
+
real = realpathSync(worktree)
|
|
544
|
+
} catch {}
|
|
545
|
+
return cwdHolders(lsof.stdout, [...new Set([worktree, real])], except)
|
|
546
|
+
},
|
|
547
|
+
stopNxDaemon(worktree) {
|
|
548
|
+
const pid = nxDaemonOf(worktree)
|
|
549
|
+
if (pid) sigkill(pid, 'SIGTERM') // nx stops its plugin workers and removes its socket on SIGTERM
|
|
550
|
+
return pid
|
|
551
|
+
},
|
|
552
|
+
erase(dir) {
|
|
553
|
+
spawn('rm', ['-rf', dir], { detached: true, stdio: 'ignore' }).unref()
|
|
554
|
+
},
|
|
392
555
|
}
|