@oakgreencc/cambium 0.0.27 → 0.0.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/workers.mjs CHANGED
@@ -14,11 +14,13 @@ import {
14
14
  readdirSync,
15
15
  readFileSync,
16
16
  readSync,
17
+ realpathSync,
18
+ renameSync,
17
19
  rmSync,
18
20
  statSync,
19
21
  writeFileSync,
20
22
  } from 'node:fs'
21
- import { dirname, join } from 'node:path'
23
+ import { basename, dirname, join } from 'node:path'
22
24
  import { fileURLToPath } from 'node:url'
23
25
  import { readConvention, topicOf } from './convention.mjs'
24
26
  import { exec as defaultExec } from './github.mjs'
@@ -47,7 +49,15 @@ import { exec as defaultExec } from './github.mjs'
47
49
  * tail(path: string, lines?: number): string,
48
50
  * readLog(path: string): string,
49
51
  * }} Spawner
50
- * @typedef {{ alive(pid: number): boolean, kill(pid: number): void }} Procs
52
+ * @typedef {{ alive(pid: number): boolean, terminate(pid: number): void, kill(pid: number): void }} Procs
53
+ * `terminate`: the graceful first step (see {@link stopOverdue}); `kill`: SIGKILL, the whole tree.
54
+ * @typedef {{
55
+ * holders(worktree: string): number[],
56
+ * stopNxDaemon(worktree: string): number | null,
57
+ * erase(dir: string): void,
58
+ * }} WorktreeSys
59
+ * `holders`: pids with their cwd in the worktree, its nx daemon's tree excluded; `stopNxDaemon`: SIGTERM the
60
+ * worktree's own nx daemon, returning its pid; `erase`: delete a directory without waiting for it.
51
61
  */
52
62
 
53
63
  /** The workers' tools. Labels and the issue body are cambium's, so those graft verbs are denied. */
@@ -143,7 +153,7 @@ export function readLogFile(path) {
143
153
 
144
154
  /**
145
155
  * @param {{ config: import('./config.mjs').Config, exec?: import('./github.mjs').Exec,
146
- * spawnFn?: typeof spawn, stamp?: () => string }} opts
156
+ * spawnFn?: typeof spawn, stamp?: () => string, sys?: WorktreeSys }} opts
147
157
  * @returns {Spawner}
148
158
  */
149
159
  export function processSpawner({
@@ -151,9 +161,37 @@ export function processSpawner({
151
161
  exec = defaultExec,
152
162
  spawnFn = spawn,
153
163
  stamp = () => new Date().toISOString().replace(/[:.]/g, '-'),
164
+ sys = realWorktreeSys,
154
165
  }) {
155
166
  const git = (/** @type {string} */ repoPath, /** @type {string[]} */ args) => exec('git', ['-C', repoPath, ...args])
156
167
  const graftEnv = { ...process.env, ...(config.graft.app ? { GRAFT_APP: config.graft.app } : {}) }
168
+ const trash = join(config.paths.home, 'trash')
169
+ // A previous daemon's erase may have died with it (a reboot): finish the job.
170
+ if (existsSync(trash)) for (const d of readdirSync(trash)) sys.erase(join(trash, d))
171
+
172
+ /**
173
+ * Take a worktree away without waiting for its bytes to go: stop its nx daemon (left running, it answers the
174
+ * next client from a tree that no longer exists, until `nx reset`), move it into `<home>/trash` (one rename),
175
+ * prune git's record of it, and erase the trash in the background. A Pods-sized tree took `git worktree remove`
176
+ * past the 120 s step timeout. A rename that fails (another volume) falls back to removing it in place.
177
+ *
178
+ * @param {import('./config.mjs').Repo} repo @param {string} worktree
179
+ */
180
+ async function discard(repo, worktree) {
181
+ sys.stopNxDaemon(worktree)
182
+ const to = join(trash, `${basename(worktree)}-${stamp()}`)
183
+ try {
184
+ mkdirSync(trash, { recursive: true })
185
+ renameSync(worktree, to)
186
+ } catch {
187
+ if (existsSync(join(worktree, '.git'))) await git(repo.path, ['worktree', 'remove', '--force', worktree])
188
+ else rmSync(worktree, { recursive: true, force: true })
189
+ await git(repo.path, ['worktree', 'prune']).catch(() => {})
190
+ return
191
+ }
192
+ await git(repo.path, ['worktree', 'prune']).catch(() => {})
193
+ sys.erase(to)
194
+ }
157
195
 
158
196
  /**
159
197
  * Fast-forward the central checkout to the just-fetched base, so the convention
@@ -213,8 +251,12 @@ export function processSpawner({
213
251
  }
214
252
  const h = await held({ repo, worktree, branch })
215
253
  if (!h.clean) return { removed: false, reason: h.reason }
254
+ // The worker is gone, but a process it left (or a person) may still work in here; removing the tree from
255
+ // under it is worse than keeping it a while. GC retries later.
256
+ const busy = sys.holders(worktree)
257
+ if (busy.length) return { removed: false, reason: `in use by pid ${busy.join(', ')}` }
216
258
  const head = (await git(worktree, ['symbolic-ref', '--short', '-q', 'HEAD']).catch(() => '')).trim()
217
- await git(repo.path, ['worktree', 'remove', '--force', worktree])
259
+ await discard(repo, worktree)
218
260
  if (head === branch) await git(repo.path, ['branch', '-D', branch]).catch(() => {})
219
261
  return { removed: true, reason: null }
220
262
  }
@@ -224,11 +266,8 @@ export function processSpawner({
224
266
  * @param {import('./config.mjs').Repo} repo @param {string} worktree @returns {Promise<Teardown>}
225
267
  */
226
268
  async function dispose(repo, worktree) {
227
- if (existsSync(join(worktree, '.git'))) await git(repo.path, ['worktree', 'remove', '--force', worktree])
228
- else {
229
- rmSync(worktree, { recursive: true, force: true })
230
- await git(repo.path, ['worktree', 'prune']).catch(() => {})
231
- }
269
+ if (existsSync(worktree)) await discard(repo, worktree)
270
+ else await git(repo.path, ['worktree', 'prune']).catch(() => {})
232
271
  return { removed: true, reason: null }
233
272
  }
234
273
 
@@ -359,16 +398,64 @@ export function descendants(root, table) {
359
398
  return out
360
399
  }
361
400
 
362
- /** @param {number} target */
363
- function sigkill(target) {
401
+ /** @param {number} target @param {NodeJS.Signals} [signal] */
402
+ function sigkill(target, signal = 'SIGKILL') {
364
403
  try {
365
- process.kill(target, 'SIGKILL')
404
+ process.kill(target, signal)
366
405
  return true
367
406
  } catch {
368
407
  return false
369
408
  }
370
409
  }
371
410
 
411
+ const psTable = () => {
412
+ const ps = spawnSync('ps', ['-A', '-o', 'pid=,ppid='], { encoding: 'utf8' })
413
+ return ps.status === 0 ? ps.stdout : ''
414
+ }
415
+
416
+ /** The direct children of `pid` in a `pid ppid` table. @param {number} pid @param {string} table */
417
+ export const childrenOf = (pid, table) =>
418
+ table
419
+ .split('\n')
420
+ .map((l) => l.trim().split(/\s+/).map(Number))
421
+ .filter(([p, pp]) => pp === pid && p !== pid && Number.isInteger(p))
422
+ .map(([p]) => p)
423
+
424
+ /** How long a terminated worker gets to end its spans and exit before SIGKILL. */
425
+ export const KILL_GRACE_MS = 30 * 1000
426
+ const MIN_MS = 60 * 1000
427
+
428
+ /**
429
+ * The wall-clock cap, gracefully. Over the cap, {@link Procs.terminate}: the worker's tool processes (a test
430
+ * run) die at once, so the tool span ends, and Claude Code gets SIGTERM, on which it ends its interaction span
431
+ * and flushes its exporter (a bare SIGKILL left `claude_code.*` spans orphaned under a root that never came).
432
+ * Still alive {@link KILL_GRACE_MS} past the cap, and at least one call later: SIGKILL, the whole tree.
433
+ *
434
+ * @param {{ procs: Procs, store: { markKilled(repo: string, issue: number, reason: string): void },
435
+ * log: (level: any, msg: string, at?: any) => void }} deps
436
+ * @param {{ repo: string, issue: number, pid: number, startedAt: number, killedReason: string | null }} w
437
+ * @param {number} wallClockMinutes @param {number} now
438
+ * @returns {'running' | 'stopping' | 'gone'} `gone`: run the exit check now
439
+ */
440
+ export function stopOverdue({ procs, store, log }, w, wallClockMinutes, now) {
441
+ if (!procs.alive(w.pid)) return 'gone'
442
+ const at = { repo: w.repo, issue: w.issue }
443
+ const deadline = w.startedAt + wallClockMinutes * MIN_MS
444
+ if (!w.killedReason) {
445
+ if (now <= deadline) return 'running'
446
+ const reason = `killed: over the ${wallClockMinutes} min wall-clock cap`
447
+ procs.terminate(w.pid)
448
+ store.markKilled(w.repo, w.issue, reason)
449
+ w.killedReason = reason
450
+ log('warn', `${reason} (pid ${w.pid}; SIGTERM, SIGKILL in ${KILL_GRACE_MS / 1000} s)`, at)
451
+ return procs.alive(w.pid) ? 'stopping' : 'gone'
452
+ }
453
+ if (now <= deadline + KILL_GRACE_MS) return 'stopping'
454
+ procs.kill(w.pid)
455
+ log('warn', `pid ${w.pid} outlived SIGTERM; SIGKILL`, at)
456
+ return 'gone'
457
+ }
458
+
372
459
  /** @type {Procs} */
373
460
  export const realProcs = {
374
461
  alive(pid) {
@@ -383,10 +470,86 @@ export const realProcs = {
383
470
  // The process group is not enough: Claude Code runs each Bash command in a group of its own, so a test
384
471
  // run the worker started outlives the group kill and loads the next attempt. Snapshot the tree first
385
472
  // (killed children reparent to launchd and drop out of it), then kill it and the group.
386
- const ps = spawnSync('ps', ['-A', '-o', 'pid=,ppid='], { encoding: 'utf8' })
387
- const tree = ps.status === 0 ? descendants(pid, ps.stdout) : []
473
+ const tree = descendants(pid, psTable())
388
474
  // A detached worker leads its own process group: the negative pid takes its children with it.
389
475
  if (!sigkill(-pid)) sigkill(pid)
390
476
  for (const p of tree) sigkill(p)
391
477
  },
478
+ terminate(pid) {
479
+ // `pid` is the prepare step; its child is `claude`, which it exits with. Everything below `claude` (Bash
480
+ // tools, a test run) is SIGKILLed now, `claude` gets SIGTERM. Signalling `pid` itself would orphan `claude`.
481
+ const table = psTable()
482
+ const direct = childrenOf(pid, table)
483
+ const keep = new Set(direct)
484
+ for (const p of descendants(pid, table)) if (!keep.has(p)) sigkill(p)
485
+ if (direct.length) for (const p of direct) sigkill(p, 'SIGTERM')
486
+ else sigkill(pid, 'SIGTERM')
487
+ },
488
+ }
489
+
490
+ const NX_DAEMON_FILE = join('.nx', 'workspace-data', 'd', 'server-process.json')
491
+
492
+ /** A pid's command line, or '' when it has none (gone). @param {number} pid */
493
+ const commandOf = (pid) => {
494
+ const ps = spawnSync('ps', ['-o', 'command=', '-p', String(pid)], { encoding: 'utf8' })
495
+ return ps.status === 0 ? ps.stdout.trim() : ''
496
+ }
497
+
498
+ /**
499
+ * The nx daemon serving `worktree`: nx keys each daemon (pid file, socket) to its workspace root, so this one
500
+ * serves nobody else. Null unless its pid file names a live process whose command runs from this worktree (a
501
+ * reused pid is not ours).
502
+ *
503
+ * @param {string} worktree @param {{ read?: (p: string) => string, command?: (pid: number) => string }} [io]
504
+ */
505
+ export function nxDaemonOf(worktree, { read = (p) => readFileSync(p, 'utf8'), command = commandOf } = {}) {
506
+ let pid
507
+ try {
508
+ pid = JSON.parse(read(join(worktree, NX_DAEMON_FILE))).processId
509
+ } catch {
510
+ return null
511
+ }
512
+ if (!Number.isInteger(pid) || pid <= 0) return null
513
+ return command(pid).includes(`${worktree}/`) ? pid : null
514
+ }
515
+
516
+ /**
517
+ * Pids whose cwd is a root or inside it, from `lsof -F pn` output, less `except`.
518
+ * @param {string} out @param {string[]} roots the worktree, and its realpath if different @param {Set<number>} except
519
+ */
520
+ export function cwdHolders(out, roots, except) {
521
+ /** @type {number[]} */
522
+ const pids = []
523
+ let pid = 0
524
+ for (const line of out.split('\n')) {
525
+ if (line.startsWith('p')) pid = Number(line.slice(1))
526
+ else if (line.startsWith('n') && pid && !except.has(pid)) {
527
+ const cwd = line.slice(1)
528
+ if (roots.some((r) => cwd === r || cwd.startsWith(`${r}/`)) && !pids.includes(pid)) pids.push(pid)
529
+ }
530
+ }
531
+ return pids
532
+ }
533
+
534
+ /** @type {WorktreeSys} */
535
+ export const realWorktreeSys = {
536
+ holders(worktree) {
537
+ const lsof = spawnSync('lsof', ['-a', '-d', 'cwd', '-F', 'pn'], { encoding: 'utf8', timeout: 20_000 })
538
+ if (!lsof.stdout) return [] // no lsof: no evidence either way, as before this check existed
539
+ const daemon = nxDaemonOf(worktree)
540
+ const except = new Set([process.pid, ...(daemon ? [daemon, ...descendants(daemon, psTable())] : [])])
541
+ let real = worktree
542
+ try {
543
+ real = realpathSync(worktree)
544
+ } catch {}
545
+ return cwdHolders(lsof.stdout, [...new Set([worktree, real])], except)
546
+ },
547
+ stopNxDaemon(worktree) {
548
+ const pid = nxDaemonOf(worktree)
549
+ if (pid) sigkill(pid, 'SIGTERM') // nx stops its plugin workers and removes its socket on SIGTERM
550
+ return pid
551
+ },
552
+ erase(dir) {
553
+ spawn('rm', ['-rf', dir], { detached: true, stdio: 'ignore' }).unref()
554
+ },
392
555
  }