claude-bulkhead 0.2.2 → 0.3.0

package/src/slot-cli.js

@@ -1,10 +1,16 @@
 'use strict'
 const fs = require('fs')
+const os = require('os')
 const path = require('path')
 const { spawn } = require('child_process')
 const { stateDir, log } = require('./hook')
 const { readMemory } = require('./memory')
-const { take, release, readSlots, staleReason, slotDir, admit, queueDir, enqueue } = require('./slots')
+const { option: sharedOption, numberOption } = require('./cli-util')
+const { take, release, readSlots, staleReason, slotDir, admit, queueDir, enqueue, readQueue, waitersBefore, signature } = require('./slots')
+
+const POLL_MS = 1000
+const RECHECK_MS = 5000
+const PROGRESS_MS = Number(process.env.BULKHEAD_SLOT_PROGRESS_MS) || 30000
 
 const USAGE = `usage:
   bulkhead-slot run [--gb=N] [--wait=MINUTES] [--label=TEXT] -- <command...>
@@ -12,16 +18,27 @@ const USAGE = `usage:
 
 run waits until the machine has room for a job of N GB (default 3), runs the
 command, and releases the slot when it exits, even on failure. The exit code is
-the command's. status shows who holds slots and whether a new job would start.`
+the command's. While it waits it prints one line to stderr every 30 seconds with its
+position in line. --wait is in minutes (default 30); set it below the Bash tool's timeout
+if the caller must not wait longer. status shows who holds slots, which session and
+project each job belongs to, who waits, and whether a new job would start.`
 
+// --name=value or --name value
 function option(args, name, fallback) {
-  const hit = args.find(a => a.startsWith(`--${name}=`))
-  return hit ? hit.slice(name.length + 3) : fallback
+  const v = sharedOption(args, name)
+  return typeof v === 'string' ? v : fallback
+}
+
+// The job size, or an error to print: --gb 4 and --gb=4 both work, nothing else is guessed.
+function gbOption(args, totalGb = os.totalmem() / 2 ** 30) {
+  return numberOption(args, 'gb', {
+    fallback: 3, max: totalGb, tooBig: n => `--gb ${n} is more than this machine's RAM (${Math.round(totalGb * 10) / 10} GB)`,
+  })
 }
 
-function gbOption(args) {
-  const n = Number(option(args, 'gb', 3))
-  return n > 0 && n <= 256 ? n : 3
+function bad(message) {
+  console.error(`bulkhead-slot: ${message}`)
+  process.exit(1)
 }
 
 function findGitBash(env = process.env, exists = fs.existsSync) {
@@ -59,20 +76,40 @@ function run(args) {
   }
   const opts = args.slice(0, sep)
   const command = args.slice(sep + 1)
-  const gb = gbOption(opts)
+  const g = gbOption(opts)
+  if (g.error) return bad(g.error)
+  const gb = g.value
+  const w = numberOption(opts, 'wait', { fallback: 30, inclusive: true })
+  if (w.error) return bad(w.error)
   const label = option(opts, 'label', command.join(' ').slice(0, 80))
-  const deadline = Date.now() + Number(option(opts, 'wait', 30)) * 60000
+  const deadline = Date.now() + w.value * 60000
   const dir = slotDir(stateDir())
-  let announced = false
+  const me = { pid: process.pid, at: Date.now() }
+  const qdir = queueDir(stateDir())
   let queued = null
+  let lastMem = null
+  let lastSig = null
+  let lastFull = 0
+  let lastLine = 0
+  let lastReason = ''
   const unqueue = () => {
     if (queued) try { fs.unlinkSync(queued) } catch {}
     queued = null
   }
   process.on('exit', unqueue)
+  const measureKept = withCommit => (lastMem = measure(withCommit))
+
+  // position, what is held, what is free: enough for the agent to see the job has not hung
+  const progress = (verb, reason) => {
+    const line = waitLine(dir, qdir, me, lastMem)
+    console.error(`[bulkhead-slot] ${verb} (${line}): ${reason}`)
+    lastLine = Date.now()
+  }
 
   const attempt = () => {
-    const got = take(dir, { label, gb, pid: process.pid }, measure, (slot, why) => log('slot', `reclaimed ${slot.label}: ${why}`))
+    const got = take(dir, { label, gb, pid: process.pid }, measureKept, (slot, why) => log('slot', `reclaimed ${slot.label}: ${why}`), waitersBefore(qdir, me))
+    lastFull = Date.now()
+    lastReason = got.reason
     if (got.file) {
       unqueue()
       return start(got.file)
@@ -81,15 +118,23 @@ function run(args) {
       console.error(`[bulkhead-slot] gave up waiting: ${got.reason}. Holders: ${holders(dir)}`)
       return process.exit(3)
     }
-    if (!announced) {
-      console.error(`[bulkhead-slot] waiting: ${got.reason}. Holders: ${holders(dir)}`)
-      announced = true
+    const first = !queued
+    queued = enqueue(qdir, { label, gb, pid: process.pid, at: me.at, reason: got.reason })
+    if (first) {
+      progress('waiting', `${got.reason}. Holders: ${holders(dir)}`)
       for (const sig of ['SIGINT', 'SIGTERM', 'SIGHUP']) process.on(sig, () => process.exit(130))
     }
-    queued = enqueue(queueDir(stateDir()), { label, gb, pid: process.pid, at: queuedAt, reason: got.reason })
-    setTimeout(attempt, 15000)
+    lastSig = signature(dir, qdir, me)
+    setTimeout(poll, POLL_MS)
+  }
+
+  // Once a second only the slot files and the waiters ahead are read; the memory check and the
+  // lock are taken when one of them changed, and every few seconds for the free-RAM wait.
+  const poll = () => {
+    if (Date.now() - lastFull >= RECHECK_MS || Date.now() > deadline || signature(dir, qdir, me) !== lastSig) return attempt()
+    if (Date.now() - lastLine >= PROGRESS_MS) progress('still waiting', lastReason)
+    setTimeout(poll, POLL_MS)
   }
-  const queuedAt = Date.now()
 
   const start = file => {
     for (const sig of ['SIGINT', 'SIGTERM', 'SIGHUP']) process.removeAllListeners(sig)
@@ -116,20 +161,46 @@ function run(args) {
   attempt()
 }
 
+// "#2 of 3 in line, 8 GB held by 2 jobs, 6.1 GB free"
+function waitLine(dir, qdir, me, mem) {
+  const live = readSlots(dir).filter(s => !staleReason(s))
+  const ahead = waitersBefore(qdir, me).length
+  const held = live.reduce((n, s) => n + (s.gb || 0), 0)
+  return `#${ahead + 1} in line, ${held} GB held by ${live.length} job${live.length === 1 ? '' : 's'}` +
+    (mem ? `, ${mem.freeGb.toFixed(1)} GB free` : '')
+}
+
 function holders(dir) {
   return readSlots(dir).map(s => `${s.label} (${s.gb} GB, pid ${s.pid})`).join(', ') || 'none'
 }
 
+// "session 812, ~/src/app": which Claude session a job runs under. Empty when the process scan fails.
+function whose(jobs) {
+  try {
+    const st = require('./status').collect({ slots: jobs.slots, queue: jobs.queue, mem: { freeGb: 0, totalGb: 0 } })
+    const text = j => (j.session ? `, session ${j.session}${j.project ? ` in ${j.project}` : ''}` : '')
+    return { slot: i => text(st.slots[i]), wait: i => text(st.queue[i]) }
+  } catch {
+    return { slot: () => '', wait: () => '' }
+  }
+}
+
 function status(args) {
   const dir = slotDir(stateDir())
-  const gb = gbOption(args)
+  const g = gbOption(args)
+  if (g.error) return bad(g.error)
+  const gb = g.value
   const held = readSlots(dir)
+  const waiting = readQueue(queueDir(stateDir()))
+  const owner = whose({ slots: held.filter(s => !staleReason(s)), queue: waiting })
+  const { duration } = require('./status')
   if (!held.length) console.log('no slots held')
+  let liveIndex = 0
   for (const s of held) {
     const why = staleReason(s)
-    const mins = Math.round((Date.now() - s.at) / 60000)
-    console.log(`${path.basename(s.file)}: ${s.label} (${s.gb} GB, pid ${s.pid}, ${mins} min)${why ? ` stale: ${why}` : ''}`)
+    console.log(`${path.basename(s.file)}: ${s.label} (${s.gb} GB, pid ${s.pid}, ${duration(Date.now() - s.at)}${why ? '' : owner.slot(liveIndex++)})${why ? ` stale: ${why}` : ''}`)
   }
+  waiting.forEach((q, i) => console.log(`waiting #${i + 1}: ${q.label} (${q.gb} GB, pid ${q.pid}, waited ${duration(Date.now() - q.at)}${owner.wait(i)})${q.reason ? ` - ${q.reason}` : ''}`))
   const live = held.filter(s => !staleReason(s))
   const mem = measure(live.length > 0)
   const reason = admit({ held: live, mem, gb })
@@ -147,4 +218,4 @@ function main(argv) {
 
 if (require.main === module) main(process.argv.slice(2))
 
-module.exports = { main, shellCommand, findGitBash }
+module.exports = { main, shellCommand, findGitBash, USAGE }

package/src/slots.js

@@ -13,10 +13,16 @@ const SMALL_MACHINE_GB = 24
 // So the defaults are 15% and 25% of total RAM, capped at those values.
 const round1 = n => Math.round(n * 10) / 10
 
+// What the two RAM limits are when the config leaves them on auto.
+function autoLimits(totalGb = os.totalmem() / 2 ** 30) {
+  return { minFreeGb: Math.min(3, round1(totalGb * 0.15)), floorGb: Math.min(5, round1(totalGb * 0.25)) }
+}
+
 function limits(totalGb = os.totalmem() / 2 ** 30) {
+  const auto = autoLimits(totalGb)
   return {
-    minFreeGb: setting('memory.minFreeGb', Math.min(3, round1(totalGb * 0.15))),
-    floorGb: setting('memory.slotFloorGb', Math.min(5, round1(totalGb * 0.25))),
+    minFreeGb: setting('memory.minFreeGb', auto.minFreeGb),
+    floorGb: setting('memory.slotFloorGb', auto.floorGb),
     maxCommitPct: setting('memory.maxCommitPct', 75),
   }
 }
@@ -87,6 +93,27 @@ function reclaim(dir, onReclaim = () => {}) {
   }
 }
 
+// A lock older than 30 s was left by a process that died. Renaming it away is atomic, so when two
+// waiters both see it only one rename succeeds; a plain rmdir let both remove it and both enter.
+// If what was moved turns out to be fresh (another waiter had already replaced the stale lock),
+// it is put back.
+const STALE_LOCK_MS = 30000
+function breakStale(lock, now = Date.now()) {
+  try {
+    if (now - fs.statSync(lock).mtimeMs <= STALE_LOCK_MS) return false
+    const moved = `${lock}.stale-${process.pid}-${Math.random().toString(36).slice(2)}`
+    fs.renameSync(lock, moved)
+    if (now - fs.statSync(moved).mtimeMs <= STALE_LOCK_MS) {
+      try { fs.renameSync(moved, lock) } catch { fs.rmdirSync(moved) }
+      return false
+    }
+    fs.rmdirSync(moved)
+    return true
+  } catch {
+    return false
+  }
+}
+
 // mkdir is atomic on every platform we support, so it doubles as a cross-process lock.
 function withLock(dir, fn, timeoutMs = 15000) {
   const lock = path.join(dir, '.lock')
@@ -96,9 +123,7 @@ function withLock(dir, fn, timeoutMs = 15000) {
       fs.mkdirSync(lock)
       break
     } catch {
-      try {
-        if (Date.now() - fs.statSync(lock).mtimeMs > 30000) fs.rmdirSync(lock)
-      } catch {}
+      breakStale(lock)
       if (Date.now() > until) return null
       Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 100)
     }
@@ -110,23 +135,33 @@ function withLock(dir, fn, timeoutMs = 15000) {
   }
 }
 
-// Returns { file } on success or { reason } when the job has to wait.
-function take(dir, { label, gb, pid }, measure, onReclaim) {
+// The oldest waiter that would fit right now goes first. A bigger job that does not fit yet does
+// not hold up smaller ones behind it.
+function waiterAhead(waiters, held, mem, now) {
+  return waiters.find(w => !admit({ held, mem, gb: w.gb, now }))
+}
+
+// Returns { file } on success or { reason } when the job has to wait. `waiters` are the queue
+// entries that were waiting before this job (see readQueue); `extra` is written into the slot file.
+function take(dir, { label, gb, pid, ...extra }, measure, onReclaim, waiters = []) {
   fs.mkdirSync(dir, { recursive: true })
   const result = withLock(dir, () => {
     reclaim(dir, onReclaim)
     const held = readSlots(dir)
-    const reason = admit({ held, mem: measure(held.length > 0), gb })
-    if (reason) return { reason }
-    const body = JSON.stringify({ label, gb, pid, at: Date.now() })
+    const mem = measure(held.length > 0)
+    const first = waiterAhead(waiters, held, mem, Date.now())
+    if (first) return { reason: `${first.label} has been waiting longer and goes first`, mem }
+    const reason = admit({ held, mem, gb })
+    if (reason) return { reason, mem }
+    const body = JSON.stringify({ label, gb, pid, at: Date.now(), ...extra })
     for (let i = 1; i <= MAX_SLOTS; i++) {
       const file = path.join(dir, `slot-${i}.json`)
       try {
         fs.writeFileSync(file, body + '\n', { flag: 'wx' })
-        return { file }
+        return { file, mem }
       } catch {}
     }
-    return { reason: `all ${MAX_SLOTS} slots are taken` }
+    return { reason: `all ${MAX_SLOTS} slots are taken`, mem }
   })
   return result || { reason: 'another session is holding the slot lock' }
 }
@@ -161,8 +196,21 @@ function readQueue(dir, isAlive = alive) {
   return out.sort((a, b) => a.at - b.at)
 }
 
+// Waiters older than this job, oldest first. Ties on the timestamp go to the lower PID.
+function waitersBefore(dir, me, isAlive = alive) {
+  return readQueue(dir, isAlive).filter(q => q.pid !== me.pid && (q.at < me.at || (q.at === me.at && q.pid < me.pid)))
+    .sort((a, b) => a.at - b.at || a.pid - b.pid)
+}
+
+// Cheap change detector for the 1 s poll: the live slot files and the waiters ahead. When it is
+// the same as last time, nothing that could let this job in has happened.
+function signature(slotsDir, queuesDir, me, isAlive = alive) {
+  const held = readSlots(slotsDir).filter(s => s.pid && isAlive(s.pid)).map(s => path.basename(s.file)).sort()
+  return held.join(',') + '|' + waitersBefore(queuesDir, me, isAlive).map(q => q.pid).join(',')
+}
+
 function release(file) {
   try { fs.unlinkSync(file) } catch {}
 }
 
-module.exports = { admit, take, release, reclaim, readSlots, staleReason, slotDir, limits, queueDir, enqueue, readQueue }
+module.exports = { autoLimits, breakStale, withLock, waitersBefore, waiterAhead, signature, admit, take, release, reclaim, readSlots, staleReason, slotDir, limits, queueDir, enqueue, readQueue }

package/src/stable-port.js

@@ -0,0 +1,61 @@
+'use strict'
+const crypto = require('crypto')
+const path = require('path')
+
+const DEFAULT_RANGE = { min: 3000, max: 3999 }
+// Ports browsers refuse to open; the same list port-cli.js uses.
+const BROWSER_BLOCKED = new Set([1719, 1720, 1723, 2049, 3659, 4045, 5060, 5061, 6000, 6566, 6665, 6666, 6667, 6668, 6669, 6697, 10080])
+
+// "3000-3999" as { min, max }, or null.
+function parseRange(text) {
+  const m = /^(\d+)-(\d+)$/.exec(String(text || '').trim())
+  if (!m) return null
+  const min = Number(m[1])
+  const max = Number(m[2])
+  return min >= 1024 && max <= 65535 && min <= max ? { min, max } : null
+}
+
+// The string a worktree root is hashed as: its resolved path, lowercased on Windows.
+function keyFor(root, platform = process.platform) {
+  const p = path.resolve(root)
+  return platform === 'win32' ? p.toLowerCase() : p
+}
+
+// The same key always starts at the same port: first 4 bytes of its sha1, as a uint32, modulo the range size.
+function startPort(key, range = DEFAULT_RANGE) {
+  const size = range.max - range.min + 1
+  return range.min + (crypto.createHash('sha1').update(String(key)).digest().readUInt32BE(0) % size)
+}
+
+// Offset of a root's start port inside a range of `size` ports.
+function startOffset(root, size, platform = process.platform) {
+  return startPort(keyFor(root, platform), { min: 0, max: size - 1 })
+}
+
+// The fixed order of candidates: the start, then upward, wrapping inside the range, browser-blocked ports left out.
+function sequence(key, range = DEFAULT_RANGE) {
+  const size = range.max - range.min + 1
+  const first = startPort(key, range) - range.min
+  const out = []
+  for (let i = 0; i < size; i++) {
+    const p = range.min + ((first + i) % size)
+    if (!BROWSER_BLOCKED.has(p)) out.push(p)
+  }
+  return out
+}
+
+// Synchronous variant for callers that already know the busy ports (port-guard).
+function pickFree(key, taken = new Set(), range = DEFAULT_RANGE) {
+  for (const p of sequence(key, range)) if (!taken.has(p)) return p
+  return null
+}
+
+// The stable port of a worktree root: its start port, or the next free one in the fixed order.
+// null when the whole range is busy. `bind(host, port)` resolves true when the bind works.
+async function stablePort(root, range = DEFAULT_RANGE, { bind, platform = process.platform } = {}) {
+  const { isFree, canBind } = require('./port-cli')
+  for (const p of sequence(keyFor(root, platform), range)) if (await isFree(p, bind || canBind)) return p
+  return null
+}
+
+module.exports = { stablePort, startOffset, pickFree, sequence, startPort, keyFor, parseRange, DEFAULT_RANGE, BROWSER_BLOCKED }

package/src/status.js

@@ -5,7 +5,8 @@ const { stateDir } = require('./hook')
 const { rejectArgs } = require('./cli-util')
 const { readMemory } = require('./memory')
 const { readProcesses } = require('./processes')
-const { readSessionFiles, findSessions } = require('./sessions')
+const { readSessionFiles, findSessions, assignOwners, isClaudeCli } = require('./sessions')
+const { enrich, fitPath, homeShort, stateLabel, branchLabel, mcpLine } = require('./session-info')
 const { readSlots, slotDir, staleReason, readQueue, queueDir } = require('./slots')
 
 const GB = 2 ** 30
@@ -21,16 +22,35 @@ function collect({
   queue = readQueue(queueDir(stateDir())),
   mem = readMemory({ withCommit: true }),
   now = Date.now(),
+  enrichOpts = {},
 } = {}) {
+  const found = snapshot ? enrich(findSessions(snapshot, files), snapshot, files, enrichOpts) : { sessions: [], mcp: [] }
+  const sessions = found.sessions
+  // the Claude session a job runs under: its process is somewhere below the session's process
+  const owner = snapshot ? assignOwners(snapshot.procs, new Set(snapshot.procs.filter(isClaudeCli).map(p => p.pid))) : new Map()
+  const whose = pid => {
+    const session = owner.get(pid) || null
+    const hit = sessions.find(x => x.pid === session)
+    return { session, project: (hit && hit.cwd) || null }
+  }
   return {
     at: now,
-    sessions: snapshot ? findSessions(snapshot, files) : [],
-    slots: slots.map(s => ({ slot: path.basename(s.file || '', '.json'), label: s.label, gb: s.gb, pid: s.pid, at: s.at })),
-    queue: queue.map(q => ({ label: q.label, gb: q.gb, pid: q.pid, at: q.at, reason: q.reason })),
+    sessions,
+    mcp: found.mcp,
+    slots: slots.map(s => ({ slot: path.basename(s.file || '', '.json'), label: s.label, gb: s.gb, pid: s.pid, at: s.at, ...whose(s.pid) })),
+    queue: queue.map(q => ({ label: q.label, gb: q.gb, pid: q.pid, at: q.at, reason: q.reason, ...whose(q.pid) })),
     memory: mem,
   }
 }
 
+// 45 s, 12 min, 2 h 5 min
+function duration(ms) {
+  const s = Math.max(0, Math.round(ms / 1000))
+  if (s < 60) return `${s} s`
+  const m = Math.floor(s / 60)
+  return m < 60 ? `${m} min` : `${Math.floor(m / 60)} h ${m % 60} min`
+}
+
 const pad2 = n => String(n).padStart(2, '0')
 
 function when(ms, now) {
@@ -44,8 +64,7 @@ function size(bytes) {
 }
 
 function shortPath(p, home) {
-  if (!p) return '?'
-  return home && p.toLowerCase().startsWith(home.toLowerCase()) ? '~' + p.slice(home.length).replace(/\\/g, '/') : p
+  return homeShort(p, home) || '(unknown)'
 }
 
 function table(rows) {
@@ -53,36 +72,54 @@ function table(rows) {
   return rows.map(r => r.map((c, i) => (i === r.length - 1 ? String(c) : String(c).padEnd(widths[i]))).join('  ').trimEnd())
 }
 
-function render(st, { color = false, home = os.homedir() } = {}) {
+function render(st, { color = false, home = os.homedir(), width = Infinity } = {}) {
   const head = s => (color ? `\x1b[1m${s}\x1b[0m` : s)
+  const paintState = (s, text) => (!color ? text : s.state === 'permission' ? `\x1b[31m${text}\x1b[0m` : s.state === 'waiting' ? `\x1b[33m${text}\x1b[0m` : text)
   const out = []
   if (st.sessions.length) {
-    const rows = [['SESSION', 'STARTED', 'RSS', 'PROJECT']]
+    // the AGENT column only appears once a session is not Claude Code, so a Claude-only list reads as before
+    const withAgent = st.sessions.some(s => s.agent && s.agent !== 'claude')
+    const titles = ['SESSION', 'STATE', 'STARTED', 'RSS', 'BRANCH', 'PROJECT']
+    if (withAgent) titles.splice(1, 0, 'AGENT')
+    const rows = [titles]
+    const stateCol = withAgent ? 2 : 1
+    const projectCol = titles.length - 1
     const portsAfter = new Map()
     for (const s of st.sessions) {
-      rows.push([s.pid, when(s.start, st.at), size(s.rss), shortPath(s.cwd, home)])
+      const cells = [s.pid, stateLabel(s, st.at), when(s.start, st.at), size(s.rss), branchLabel(s), shortPath(s.cwd, home)]
+      if (withAgent) cells.splice(1, 0, s.agent || 'claude')
+      rows.push(cells)
       portsAfter.set(rows.length - 1, s.ports)
     }
+    // the project column takes what the other columns leave, never less than 12 characters
+    const widths = rows[0].slice(0, -1).map((_, i) => Math.max(...rows.map(r => String(r[i]).length)))
+    const room = Math.max(12, width - widths.reduce((n, w) => n + w + 2, 0))
+    for (const r of rows.slice(1)) r[projectCol] = fitPath(r[projectCol], room)
     const lines = table(rows)
     out.push(head(lines[0]))
     for (let i = 1; i < lines.length; i++) {
-      out.push(lines[i])
+      const s = st.sessions[i - 1]
+      const stateText = rows[i][stateCol]
+      out.push(color && s.state ? lines[i].replace(stateText, paintState(s, stateText)) : lines[i])
       for (const p of portsAfter.get(i)) out.push(`  :${p.port}  ${p.name}  pid ${p.pid}`)
     }
+    for (const m of (st.mcp || []).filter(x => x.duplicate)) {
+      out.push(`duplicate MCP: ${mcpLine(m, size)} (sessions ${m.sessions.join(', ')})`)
+    }
   } else {
     out.push('no Claude sessions found')
   }
   out.push('')
   if (st.slots.length) {
-    const lines = table([['SLOT', 'GB', 'PID', 'AGE', 'COMMAND'],
-      ...st.slots.map(s => [s.slot, s.gb, s.pid, `${Math.round((st.at - s.at) / 60000)} min`, s.label])])
+    const lines = table([['SLOT', 'GB', 'PID', 'AGE', 'SESSION', 'PROJECT', 'COMMAND'],
+      ...st.slots.map(s => [s.slot, s.gb, s.pid, duration(st.at - s.at), s.session || '-', s.project ? shortPath(s.project, home) : '-', s.label])])
     out.push(head(lines[0]), ...lines.slice(1), '')
   } else {
     out.push('no heavy jobs in bulkhead-slot', '')
   }
   if (st.queue && st.queue.length) {
-    const lines = table([['WAITING', 'GB', 'PID', 'FOR', 'COMMAND'],
-      ...st.queue.map((q, i) => [`#${i + 1}`, q.gb, q.pid, `${Math.round((st.at - q.at) / 60000)} min`, q.label])])
+    const lines = table([['WAITING', 'GB', 'PID', 'FOR', 'SESSION', 'PROJECT', 'COMMAND'],
+      ...st.queue.map((q, i) => [`#${i + 1}`, q.gb, q.pid, duration(st.at - q.at), q.session || '-', q.project ? shortPath(q.project, home) : '-', q.label])])
     out.push(head(lines[0]), ...lines.slice(1), '')
   }
   const m = st.memory
@@ -93,7 +130,8 @@ function render(st, { color = false, home = os.homedir() } = {}) {
 function toJson(st) {
   return {
     at: new Date(st.at).toISOString(),
-    sessions: st.sessions.map(s => ({ ...s, start: new Date(s.start).toISOString() })),
+    sessions: st.sessions.map(({ stateSince, ...s }) => ({ ...s, start: new Date(s.start).toISOString(), stateSince: stateSince ? new Date(stateSince).toISOString() : null })),
+    mcp: st.mcp || [],
     slots: st.slots.map(s => ({ ...s, at: new Date(s.at).toISOString() })),
     queue: (st.queue || []).map(q => ({ ...q, at: new Date(q.at).toISOString() })),
     memory: st.memory,
@@ -105,7 +143,7 @@ function main(args, env = process.env, stdout = process.stdout) {
   const st = collect()
   if (args.includes('--json')) return stdout.write(JSON.stringify(toJson(st), null, 2) + '\n')
   const color = Boolean(stdout.isTTY) && !('NO_COLOR' in env) && env.TERM !== 'dumb'
-  stdout.write(render(st, { color }) + '\n')
+  stdout.write(render(st, { color, width: stdout.isTTY ? stdout.columns || Infinity : Infinity }) + '\n')
 }
 
-module.exports = { collect, render, toJson, main, size, when }
+module.exports = { collect, render, toJson, main, size, when, duration }

package/src/statusline.js

@@ -5,6 +5,7 @@ const { stateDir } = require('./hook')
 const { readMemory } = require('./memory')
 const { execFileSync } = require('child_process')
 const { readSessionFiles, isClaudeCli } = require('./sessions')
+const { readStates } = require('./session-state')
 const { readSlots, slotDir, staleReason } = require('./slots')
 
 const CACHE_MS = 5000
@@ -18,31 +19,56 @@ function alive(pid) {
   }
 }
 
-// Claude Code CLI processes running now, from one `ps` listing (about 20 ms), the same test
-// `bulkhead status` uses. A session file outlives its process and the OS reuses pids, so counting
-// files whose pid is merely alive overcounts. null when there is no cheap listing (Windows).
-function claudeCount({ platform = process.platform, run = execFileSync } = {}) {
+const PS_LINE = /^\s*(\d+)\s+(\w+ +\w+ +\d+ +[\d:]+ +\d+)\s+(.*)$/
+const START_SLACK_MS = 120 * 1000
+
+// Sessions running now: the session files, narrowed by one `ps` for just those pids (a few
+// milliseconds, not a listing of every process). A file outlives its process and the OS reuses
+// pids, so a pid counts only when it is a Claude Code CLI that started when the file says.
+// null when there is no cheap check (Windows), and the caller counts live pids instead.
+function claudeCount({ platform = process.platform, run = execFileSync, files = new Map(), isAlive = alive } = {}) {
   if (platform === 'win32') return null
+  const pids = [...files.keys()].filter(isAlive)
+  if (!pids.length) return 0
   let out
   try {
-    out = run('ps', ['-axww', '-o', 'pid=,args='], { encoding: 'utf8', timeout: 2000, maxBuffer: 64 * 1024 * 1024, env: { ...process.env, LC_ALL: 'C' }, stdio: ['ignore', 'pipe', 'ignore'] })
-  } catch { return null }
+    out = run('ps', ['-ww', '-o', 'pid=,lstart=,args=', '-p', pids.join(',')], { encoding: 'utf8', timeout: 2000, env: { ...process.env, LC_ALL: 'C' }, stdio: ['ignore', 'pipe', 'ignore'] })
+  } catch (e) {
+    out = e && e.stdout ? String(e.stdout) : null // ps exits 1 when one of the pids is gone
+    if (out == null) return null
+  }
   let n = 0
   for (const l of out.split('\n')) {
-    const m = /^\s*(\d+)\s+(.*)$/.exec(l)
+    const m = PS_LINE.exec(l)
     if (!m) continue
-    const cmd = m[2]
-    if (isClaudeCli({ name: (cmd.split(' ')[0] || '').split(/[\\/]/).pop(), cmd })) n++
+    const cmd = m[3]
+    if (!isClaudeCli({ name: (cmd.split(' ')[0] || '').split(/[\\/]/).pop(), cmd })) continue
+    const f = files.get(Number(m[1]))
+    const started = new Date(m[2].replace(/ +/g, ' ')).getTime()
+    if (f && f.startedAt && started && Math.abs(f.startedAt - started) >= START_SLACK_MS) continue
+    n++
+  }
+  return n
+}
+
+// Live sessions that are waiting for you, from the state files the session-state hook writes.
+function waitingCount(files, states = readStates(), isAlive = alive) {
+  let n = 0
+  for (const rec of states.values()) {
+    if (rec.state === 'working') continue
+    const hit = [...files].find(([pid, f]) => (f.sessionId && f.sessionId === rec.session) || pid === rec.ppid)
+    if (hit && isAlive(hit[0])) n++
   }
   return n
 }
 
-// Only cheap reads: session files, slot files and free RAM. No process scan, so it stays
-// well under the 300 ms the status line debounce allows.
-function line({ files = readSessionFiles(), slots = readSlots(slotDir(stateDir())), mem = readMemory(), isAlive = alive, count = null } = {}) {
-  const sessions = count != null ? count : (claudeCount() ?? [...files.keys()].filter(isAlive).length)
+// Only cheap reads: session files, slot files, state files and free RAM. One small `ps`, so
+// it stays well under the 100 ms the status line debounce allows.
+function line({ files = readSessionFiles(), slots = readSlots(slotDir(stateDir())), mem = readMemory(), isAlive = alive, count = null, waiting = null } = {}) {
+  const sessions = count != null ? count : (claudeCount({ files, isAlive }) ?? [...files.keys()].filter(isAlive).length)
   const heavy = slots.filter(s => !staleReason(s, Date.now(), isAlive)).length
-  return `${sessions} session${sessions === 1 ? '' : 's'}  ${heavy} heavy  ${mem.freeGb.toFixed(1)} GB free`
+  const wait = waiting != null ? waiting : waitingCount(files, undefined, isAlive)
+  return `${sessions} session${sessions === 1 ? '' : 's'}${wait ? `  ${wait} waiting` : ''}  ${heavy} heavy  ${mem.freeGb.toFixed(1)} GB free`
 }
 
 function cached(file, now = Date.now(), compute = line) {
@@ -53,7 +79,10 @@ function cached(file, now = Date.now(), compute = line) {
   const text = compute()
   try {
     fs.mkdirSync(path.dirname(file), { recursive: true })
-    fs.writeFileSync(file, JSON.stringify({ at: now, text }))
+    // tmp and rename: sessions share this file and must never read half of it
+    const tmp = `${file}.${process.pid}.tmp`
+    fs.writeFileSync(tmp, JSON.stringify({ at: now, text }))
+    fs.renameSync(tmp, file)
   } catch {}
   return text
 }
@@ -66,4 +95,4 @@ function main() {
   process.stdout.write(text + '\n')
 }
 
-module.exports = { line, cached, main, claudeCount }
+module.exports = { line, cached, main, claudeCount, waitingCount }

package/src/subagent-guard.js

@@ -0,0 +1,24 @@
+'use strict'
+const { runHook, blocked, setting, stateDir } = require('./hook')
+const pressure = require('./pressure')
+
+// With memoryGuard.subagents on, a new subagent is not started while memory pressure is critical:
+// every subagent is a whole Claude process plus the commands it runs. Runs on the Agent tool
+// (Task in older Claude Code versions).
+function check(input, read = () => pressure.current({ stateDir: stateDir() })) {
+  const p = read()
+  if (!p || !p.critical) return null
+  return `[bulkhead] subagent-guard: memory pressure is critical (${p.detail}). Do not start another subagent now.\n` +
+    'Finish the work you have, or wait for the running subagents to return, then try again in a minute. `bulkhead status` shows free RAM.'
+}
+
+if (require.main === module) {
+  runHook('memory-guard', input => {
+    if (setting('memoryGuard.subagents', false) !== true) return
+    const why = check(input)
+    if (!why) return
+    return blocked('subagent-guard', 'subagent-pressure', input, why, input.tool_input && input.tool_input.description)
+  })
+}
+
+module.exports = { check }