@vzn/vx 0.0.18 → 0.0.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/LICENSE +1 -1
  2. package/README.md +107 -70
  3. package/package.json +10 -20
  4. package/src/bin.ts +64 -5
  5. package/src/cache/archive.ts +279 -28
  6. package/src/cache/cache.ts +952 -1584
  7. package/src/cache/chained-cache.ts +14 -10
  8. package/src/cache/config-evals.ts +112 -0
  9. package/src/cache/file-hashes.ts +290 -0
  10. package/src/cache/git-inputs.ts +966 -0
  11. package/src/cache/index.ts +23 -7
  12. package/src/cache/inputs.ts +553 -739
  13. package/src/cache/key-fold.ts +232 -0
  14. package/src/cache/layer.ts +674 -0
  15. package/src/cache/layered-cache.ts +211 -49
  16. package/src/cache/output-index.ts +323 -0
  17. package/src/cache/policy.ts +84 -0
  18. package/src/cache/run-history.ts +152 -0
  19. package/src/cache/zstd.ts +183 -0
  20. package/src/cli/cache.ts +88 -30
  21. package/src/cli/completions.ts +104 -0
  22. package/src/cli/core-alias.ts +25 -0
  23. package/src/cli/format.ts +4 -15
  24. package/src/cli/help.ts +76 -38
  25. package/src/cli/index.ts +82 -17
  26. package/src/cli/info.ts +162 -91
  27. package/src/cli/init.ts +87 -0
  28. package/src/cli/last.ts +130 -23
  29. package/src/cli/lock.ts +36 -18
  30. package/src/cli/plan-format.ts +18 -7
  31. package/src/cli/plugin-commands.ts +34 -12
  32. package/src/cli/run.ts +138 -313
  33. package/src/cli/select.ts +342 -0
  34. package/src/cli/show.ts +151 -71
  35. package/src/cli/upgrade.ts +136 -15
  36. package/src/cli/watch.ts +1175 -116
  37. package/src/cli/why.ts +51 -29
  38. package/src/cli/workspace-config.ts +138 -0
  39. package/src/config.ts +183 -126
  40. package/src/exec/env.ts +11 -0
  41. package/src/exec/executor.ts +62 -31
  42. package/src/exec/index.ts +16 -2
  43. package/src/exec/kill-tree.ts +281 -0
  44. package/src/exec/local-executor.ts +41 -0
  45. package/src/exec/runner.ts +361 -77
  46. package/src/exec/sandbox-binds.ts +211 -0
  47. package/src/exec/sandbox-paths.ts +70 -0
  48. package/src/exec/sandbox-runtime.ts +1152 -415
  49. package/src/exec/sandbox-violations.ts +293 -0
  50. package/src/graph/dependency-spec.ts +14 -14
  51. package/src/graph/index.ts +6 -4
  52. package/src/graph/priorities.ts +130 -0
  53. package/src/graph/scheduler.ts +332 -329
  54. package/src/graph/task-graph.ts +552 -188
  55. package/src/index.ts +111 -102
  56. package/src/orchestrator/admission.ts +161 -0
  57. package/src/orchestrator/colors.ts +8 -3
  58. package/src/orchestrator/deferred-outputs.ts +25 -7
  59. package/src/orchestrator/doctor.ts +363 -0
  60. package/src/orchestrator/events.ts +111 -8
  61. package/src/orchestrator/excluded-keys.ts +155 -0
  62. package/src/orchestrator/execute-task.ts +450 -565
  63. package/src/orchestrator/failure-mode.ts +169 -28
  64. package/src/orchestrator/failure-recap.ts +121 -0
  65. package/src/orchestrator/fingerprint-watch.ts +89 -0
  66. package/src/orchestrator/framed-output.ts +121 -49
  67. package/src/orchestrator/history.ts +123 -104
  68. package/src/orchestrator/hit-restore.ts +285 -0
  69. package/src/orchestrator/index.ts +42 -5
  70. package/src/orchestrator/keyed-projects.ts +89 -0
  71. package/src/orchestrator/local-shortcircuit.ts +83 -23
  72. package/src/orchestrator/lockfile-claim.ts +401 -0
  73. package/src/orchestrator/logger.ts +128 -41
  74. package/src/orchestrator/metrics.ts +201 -72
  75. package/src/orchestrator/miss-save.ts +232 -0
  76. package/src/orchestrator/options.ts +63 -34
  77. package/src/orchestrator/persistent.ts +99 -0
  78. package/src/orchestrator/placement.ts +212 -0
  79. package/src/orchestrator/plan.ts +24 -11
  80. package/src/orchestrator/plugin-host.ts +200 -49
  81. package/src/orchestrator/plugin.ts +190 -14
  82. package/src/orchestrator/prepare.ts +168 -137
  83. package/src/orchestrator/projects.ts +270 -0
  84. package/src/orchestrator/remote-prefetch.ts +22 -2
  85. package/src/orchestrator/run-artifacts.ts +34 -3
  86. package/src/orchestrator/run-context.ts +20 -5
  87. package/src/orchestrator/run-lock.ts +349 -0
  88. package/src/orchestrator/run-records.ts +132 -0
  89. package/src/orchestrator/run-report.ts +6 -4
  90. package/src/orchestrator/run.ts +467 -611
  91. package/src/orchestrator/sandbox-request.ts +569 -0
  92. package/src/orchestrator/save-lane.ts +44 -0
  93. package/src/orchestrator/shell-verdict.ts +117 -0
  94. package/src/orchestrator/signals.ts +167 -0
  95. package/src/orchestrator/stable-keys.ts +259 -10
  96. package/src/orchestrator/status-line.ts +35 -2
  97. package/src/orchestrator/summary.ts +115 -25
  98. package/src/orchestrator/task-hash.ts +103 -21
  99. package/src/orchestrator/task-log-buffer.ts +15 -18
  100. package/src/orchestrator/telemetry-host.ts +27 -8
  101. package/src/orchestrator/telemetry.ts +71 -44
  102. package/src/orchestrator/upstream.ts +91 -45
  103. package/src/util/bun-version.ts +43 -0
  104. package/src/util/cgroup.ts +160 -0
  105. package/src/util/edit-distance.ts +49 -0
  106. package/src/util/errors.ts +86 -0
  107. package/src/util/hash.ts +10 -1
  108. package/src/util/index.ts +42 -5
  109. package/src/util/paths.ts +168 -5
  110. package/src/util/procfs.ts +20 -0
  111. package/src/util/settle.ts +23 -2
  112. package/src/util/size.ts +33 -0
  113. package/src/util/task-id.ts +20 -0
  114. package/src/util/timing.ts +11 -1
  115. package/src/util/verbs.ts +39 -0
  116. package/src/util/which.ts +48 -0
  117. package/src/workspace/affected.ts +368 -60
  118. package/src/workspace/config-cache.ts +271 -32
  119. package/src/workspace/config-eval.ts +60 -16
  120. package/src/workspace/config-imports.ts +131 -9
  121. package/src/workspace/config-schema.ts +1033 -0
  122. package/src/workspace/filter.ts +69 -17
  123. package/src/workspace/fingerprint.ts +59 -7
  124. package/src/workspace/index.ts +37 -3
  125. package/src/workspace/json-data.ts +90 -0
  126. package/src/workspace/load-reads.ts +32 -0
  127. package/src/workspace/lockfile.ts +41 -14
  128. package/src/workspace/migrate-scripts.ts +187 -0
  129. package/src/workspace/migration.ts +353 -0
  130. package/src/workspace/package-graph.ts +239 -31
  131. package/src/workspace/project-loader.ts +295 -815
  132. package/src/workspace/workspace.ts +266 -31
  133. package/plugins/local-cache/index.ts +0 -2
  134. package/plugins/local-executor/index.ts +0 -2
  135. package/plugins/schedule-history/index.ts +0 -2
  136. package/src/cache/cas-backend.ts +0 -115
  137. package/src/cache/digest.ts +0 -32
  138. package/src/cli/migrate-emit.ts +0 -21
  139. package/src/cli/migrate-nx.ts +0 -473
  140. package/src/cli/migrate-scripts.ts +0 -127
  141. package/src/cli/migrate-turbo.ts +0 -401
  142. package/src/cli/migrate.ts +0 -305
  143. package/src/cli/prune.ts +0 -291
  144. package/src/orchestrator/missing-plugin.ts +0 -9
  145. package/src/orchestrator/resources.ts +0 -45
  146. package/src/orchestrator/verify.ts +0 -174
  147. package/src/plugins/local-cache/index.ts +0 -11
  148. package/src/plugins/local-executor/index.ts +0 -45
  149. package/src/plugins/schedule-history/index.ts +0 -109
@@ -4,7 +4,19 @@
4
4
  // process exits. cpuMs / peakRssBytes are then surfaced on RunResult and
5
5
  // folded into the v11 `runs` table by the orchestrator.
6
6
 
7
+ import { existsSync, readFileSync } from 'node:fs'
7
8
  import { constants as osConstants } from 'node:os'
9
+ import { executablePath, isExecutableMissing, killGraceMs } from '../util/index.js'
10
+ import {
11
+ closeSignalChannel,
12
+ holdGroups,
13
+ killTree,
14
+ markGroupIfGone,
15
+ releaseGroup,
16
+ signalThrough,
17
+ spawnGuarded,
18
+ untilGroupsGone,
19
+ } from './kill-tree.js'
8
20
 
9
21
  export interface RunResult {
10
22
  exitCode: number
@@ -124,21 +136,30 @@ const SHELL_BUILTINS = new Set([
124
136
  /**
125
137
  * Prepend `exec ` to a command that is a single EXTERNAL program (no
126
138
  * shell control characters, not a builtin, no leading env-assignment).
127
- * `exec` REPLACES the wrapping `sh` with the program, so on a teardown
128
- * SIGTERM there is no intermediate shell whose death would orphan the
129
- * real process — the documented grandchild-orphan limitation, resolved
130
- * for the common single-command case (dev servers: `astro dev`, `vite`,
131
- * `next dev`; one-shot compilers). It also makes `resourceUsage` measure
132
- * the program itself rather than the shell. Compound commands, builtins,
133
- * and `FOO=bar cmd` forms keep the shell (compound-command grandchildren
134
- * still orphan on a hard programmatic kill — the residual limit every
135
- * non-cgroup runner shares).
139
+ * `exec` REPLACES the wrapping `sh` with the program: `resourceUsage`
140
+ * measures the program itself rather than the shell, a signal lands on
141
+ * the program directly, and there is one process fewer per task.
142
+ * (Until kill-tree.ts this was also what kept a single command's real
143
+ * process from orphaning on a teardown; the process group covers every
144
+ * shape now.) Compound commands, builtins, and `FOO=bar cmd` forms keep
145
+ * the shell.
136
146
  */
137
147
  export function execWrap(command: string): string {
138
- if (SHELL_CONTROL.test(command)) return command
148
+ const first = execWord(command)
149
+ return first === undefined ? command : `exec ${command}`
150
+ }
151
+
152
+ /**
153
+ * The one program a command runs when it is a plain `word args…` — the
154
+ * word `execWrap` execs, and the word the shell names in "not found" when
155
+ * it exits 127. Undefined for a pipeline, a builtin, an assignment: there
156
+ * the 127 could be any segment's.
157
+ */
158
+ export function execWord(command: string): string | undefined {
159
+ if (SHELL_CONTROL.test(command)) return undefined
139
160
  const first = command.trimStart().split(/\s+/)[0] ?? ''
140
- if (first === '' || first.includes('=') || SHELL_BUILTINS.has(first)) return command
141
- return `exec ${command}`
161
+ if (first === '' || first.includes('=') || SHELL_BUILTINS.has(first)) return undefined
162
+ return first
142
163
  }
143
164
 
144
165
  /**
@@ -153,43 +174,68 @@ export function signalExitCode(signal: string): number {
153
174
  }
154
175
 
155
176
  /**
156
- * Arm a SIGTERM timeout on a spawned child. Returns a handle whose
157
- * `timedOut()` reports whether the timer fired — so the caller can
158
- * classify the resulting SIGTERM as a real failure rather than a
159
- * Ctrl-C abort — and `clear()` cancels the timer once the child exits
160
- * on its own. A no-op (never fires, nothing to clear) when `timeoutMs`
161
- * is undefined.
177
+ * The reverse: the signal an exit above 128 stands for (137 → SIGKILL),
178
+ * by the platform's numbering; undefined for a plain exit. The shell
179
+ * reports 128 + n for a death by signal n, so the read is the shell's
180
+ * convention, not proof — a command may exit 137 on its own.
162
181
  */
182
+ export function exitSignal(code: number): string | undefined {
183
+ if (code <= 128 || code >= 128 + 65) return undefined
184
+ const signals = osConstants.signals as Partial<Record<string, number>>
185
+ const num = code - 128
186
+ return Object.keys(signals).find((k) => signals[k] === num && !SIGNAL_ALIASES.has(k))
187
+ }
188
+
189
+ // Second names for one number (SIGIOT = SIGABRT, SIGPOLL = SIGIO); the
190
+ // canonical one reads.
191
+ const SIGNAL_ALIASES = new Set(['SIGIOT', 'SIGPOLL', 'SIGCLD'])
192
+
163
193
  /**
164
194
  * Grace after a timeout SIGTERM before escalating to SIGKILL — a child that
165
195
  * ignores SIGTERM must still be bounded. Matches the persistent-shutdown grace.
166
196
  */
167
197
  const TIMEOUT_SIGKILL_GRACE_MS = 2000
168
198
 
199
+ /**
200
+ * Arm a SIGTERM timeout on a spawned child. Returns a handle whose
201
+ * `timedOut()` reports whether the timer fired — so the caller can
202
+ * classify the resulting SIGTERM as a real failure rather than a
203
+ * Ctrl-C abort — and `settle()`, awaited once the child has exited:
204
+ * it cancels the timers and, when the timeout fired, waits out the rest
205
+ * of the grace for the child's GROUP and SIGKILLs whoever is left. The
206
+ * shell dying on the SIGTERM is not the tree dying: a backgrounded
207
+ * process that ignores it lived on under init after vx exited (nx#11782's
208
+ * sibling, reproduced on vx 2026-09-24). A no-op (never fires, nothing
209
+ * to settle) when `timeoutMs` is undefined.
210
+ */
169
211
  export function armTimeout(
170
212
  proc: ReturnType<typeof Bun.spawn>,
171
213
  timeoutMs: number | undefined,
172
- ): { timedOut: () => boolean; clear: () => void } {
173
- if (timeoutMs === undefined) return { timedOut: () => false, clear: () => {} }
174
- let fired = false
214
+ ): { timedOut: () => boolean; settle: () => Promise<void> } {
215
+ if (timeoutMs === undefined) return { timedOut: () => false, settle: async () => {} }
216
+ let firedAt: number | undefined
175
217
  let killTimer: ReturnType<typeof setTimeout> | undefined
218
+ const graceMs = killGraceMs(TIMEOUT_SIGKILL_GRACE_MS)
176
219
  const timer = setTimeout(() => {
177
- fired = true
178
- proc.kill('SIGTERM')
220
+ firedAt = Date.now()
221
+ killTree(proc, 'SIGTERM')
179
222
  // Escalate to SIGKILL after a grace: a child that TRAPS+IGNORES SIGTERM
180
223
  // (`trap '' TERM`) would otherwise defeat the timeout entirely and hang
181
224
  // `await proc.exited` until its natural exit — there is no run-level
182
225
  // timeout, so a wedged child hangs the whole run forever. Mirrors the
183
226
  // end-of-run persistent-shutdown escalation. Unref'd so it never keeps
184
227
  // the CLI alive.
185
- killTimer = setTimeout(() => proc.kill('SIGKILL'), TIMEOUT_SIGKILL_GRACE_MS)
228
+ killTimer = setTimeout(() => killTree(proc, 'SIGKILL'), graceMs)
186
229
  killTimer.unref?.()
187
230
  }, timeoutMs)
188
231
  return {
189
- timedOut: () => fired,
190
- clear: () => {
232
+ timedOut: () => firedAt !== undefined,
233
+ settle: async () => {
191
234
  clearTimeout(timer)
192
235
  if (killTimer !== undefined) clearTimeout(killTimer)
236
+ if (firedAt === undefined) return
237
+ const left = await untilGroupsGone([proc], Math.max(0, firedAt + graceMs - Date.now()))
238
+ for (const child of left) killTree(child, 'SIGKILL')
193
239
  },
194
240
  }
195
241
  }
@@ -203,27 +249,44 @@ export function armTimeout(
203
249
  * at once, so the readers win this race immediately and normal tasks pay
204
250
  * nothing; only a stuck reader waits out the grace. Residual buffered output is
205
251
  * bounded by the OS pipe buffer (streamToString drains continuously DURING the
206
- * run), so this is far longer than any real drain needs.
252
+ * run), so this is far longer than any real drain needs. What the leftover
253
+ * process writes after the bound is lost, and `POST_EXIT_CUT_LINE` says so.
207
254
  */
208
255
  const POST_EXIT_DRAIN_MS = 250
209
256
 
257
+ /**
258
+ * The line a task's frame gets when the drain bound cut its pipes. The cut
259
+ * was silent: a backgrounded child's late output vanished from the frame and
260
+ * the cached replay with no word (nx#35302 reproduced on vx, 2026-09-24).
261
+ * The bound stays, since it is what keeps a leftover server from hanging the
262
+ * run; the line makes the loss visible.
263
+ */
264
+ export const POST_EXIT_CUT_LINE =
265
+ `\n[vx] output after the task's shell exited was cut: a process it left running ` +
266
+ `still held its stdout/stderr ${POST_EXIT_DRAIN_MS} ms later\n`
267
+
210
268
  /**
211
269
  * After the direct child has exited, wait for the stdout/stderr readers to
212
270
  * reach EOF — but bound it: an orphaned grandchild holding the pipe open would
213
271
  * hang the run forever (there is no default task timeout). If the grace expires
214
272
  * first, abort the reader signal so `streamToString` cancels its read and
215
- * returns whatever it captured. The timer is cleared AND unref'd so it can
216
- * never keep the CLI alive after the readers settle (the plugin-flush lesson).
273
+ * returns whatever it captured, and resolve true: the caller adds
274
+ * `POST_EXIT_CUT_LINE` to the task's stderr. The timer is unref'd so it can
275
+ * never keep the CLI alive after the readers settle (the plugin-flush
276
+ * lesson); it is not cleared, because a late resolve on a race already won
277
+ * by the drain changes nothing (item 636 deleted the clear and nothing
278
+ * reddened).
217
279
  */
218
- export async function drainOrAbort(streams: Promise<unknown>, ac: AbortController): Promise<void> {
219
- let timer: ReturnType<typeof setTimeout> | undefined
280
+ export async function drainOrAbort(
281
+ streams: Promise<unknown>,
282
+ ac: AbortController,
283
+ ): Promise<boolean> {
220
284
  const deadline = new Promise<'timeout'>((resolve) => {
221
- timer = setTimeout(() => resolve('timeout'), POST_EXIT_DRAIN_MS)
222
- timer.unref?.()
285
+ setTimeout(() => resolve('timeout'), POST_EXIT_DRAIN_MS).unref?.()
223
286
  })
224
287
  const winner = await Promise.race([streams.then(() => 'drained' as const), deadline])
225
- if (timer !== undefined) clearTimeout(timer)
226
288
  if (winner === 'timeout') ac.abort()
289
+ return winner === 'timeout'
227
290
  }
228
291
 
229
292
  /**
@@ -233,6 +296,22 @@ export async function drainOrAbort(streams: Promise<unknown>, ac: AbortControlle
233
296
  */
234
297
  const READY_MATCH_WINDOW_CHARS = 64 * 1024
235
298
 
299
+ /**
300
+ * Why a persistent task never became ready — the reason every label and
301
+ * record reads instead of a fabricated exit (item 270). `exitCode` is the
302
+ * child's own when it exited before the pattern matched.
303
+ */
304
+ export class PersistentReadyError extends Error {
305
+ readonly reason: 'timeout' | 'exited' | 'spawn'
306
+ readonly exitCode: number | undefined
307
+ constructor(message: string, reason: 'timeout' | 'exited' | 'spawn', exitCode?: number) {
308
+ super(message)
309
+ this.name = 'PersistentReadyError'
310
+ this.reason = reason
311
+ this.exitCode = exitCode
312
+ }
313
+ }
314
+
236
315
  export interface PersistentSpawn {
237
316
  /** Underlying Bun subprocess so the orchestrator can SIGTERM it later. */
238
317
  child: ReturnType<typeof Bun.spawn>
@@ -264,6 +343,12 @@ export interface PersistentOptions extends Omit<RunOptions, 'forwardArgs' | 'cap
264
343
  * resolves before the timer can fire).
265
344
  */
266
345
  timeoutMs?: number
346
+ /**
347
+ * Spawn it with fd 3 as its signal channel and route its polite signals
348
+ * there (`signalThrough`): a Linux sandboxed command, whose group signal
349
+ * would kill bwrap instead (`wrapSandboxedCommand`'s `forwardsSignals`).
350
+ */
351
+ signalChannel?: boolean
267
352
  }
268
353
 
269
354
  /**
@@ -283,18 +368,38 @@ export function runPersistent(opts: PersistentOptions): PersistentSpawn {
283
368
 
284
369
  let child: ReturnType<typeof Bun.spawn>
285
370
  try {
286
- child = Bun.spawn(['sh', '-c', execWrap(opts.command)], {
287
- cwd: opts.cwd,
288
- env: opts.env as Record<string, string>,
289
- stdin: 'ignore',
290
- stdout: 'pipe',
291
- stderr: 'pipe',
292
- })
371
+ child = spawnGuarded(() =>
372
+ Bun.spawn([executablePath('sh'), '-c', execWrap(opts.command)], {
373
+ argv0: 'sh',
374
+ cwd: opts.cwd,
375
+ env: opts.env as Record<string, string>,
376
+ // A pipe vx holds and never writes: stdin stays open while vx
377
+ // lives and ends when it does. A dev server that exits on stdin
378
+ // EOF (esbuild --watch, Vite's case in turborepo#8915) became ready
379
+ // and exited 0 under 'ignore'. Not the terminal: several servers
380
+ // would steal each other's keystrokes, and a CI's /dev/null stdin
381
+ // is the same EOF. The one-shot spawn below keeps 'ignore', so a
382
+ // task that reads stdin can never hang CI.
383
+ stdio:
384
+ opts.signalChannel === true ? ['pipe', 'pipe', 'pipe', 'pipe'] : ['pipe', 'pipe', 'pipe'],
385
+ // Its own session and process group, so a kill reaches what it
386
+ // forked (kill-tree.ts). stdin is a pipe, so a background group
387
+ // never stops on a terminal read.
388
+ detached: true,
389
+ }),
390
+ )
391
+ if (opts.signalChannel === true) {
392
+ signalThrough(child, child.stdio[3] as number)
393
+ const spawned = child
394
+ void spawned.exited.then(() => closeSignalChannel(spawned))
395
+ }
293
396
  } catch (err) {
294
397
  const message = err instanceof Error ? err.message : String(err)
295
398
  return {
296
399
  child: undefined as unknown as ReturnType<typeof Bun.spawn>,
297
- ready: Promise.reject(new Error(`failed to spawn persistent task: ${message}`)),
400
+ ready: Promise.reject(
401
+ new PersistentReadyError(`failed to spawn persistent task: ${message}`, 'spawn'),
402
+ ),
298
403
  readyMs: () => Date.now() - start,
299
404
  }
300
405
  }
@@ -389,12 +494,26 @@ export function runPersistent(opts: PersistentOptions): PersistentSpawn {
389
494
  readyTimer = setTimeout(() => {
390
495
  if (readyAt === undefined) {
391
496
  rejectReady(
392
- new Error(
497
+ new PersistentReadyError(
393
498
  `persistent task not ready within ${opts.timeoutMs}ms — ` +
394
499
  `readyWhen pattern never matched; child killed`,
500
+ 'timeout',
395
501
  ),
396
502
  )
397
- child.kill('SIGTERM')
503
+ // Listed on the group guard until the SIGKILL: the shell may die
504
+ // on the SIGTERM and let the group go while the server runs out
505
+ // the grace, and a vx that exits inside it (the timer is unref'd)
506
+ // leaves the server to the guard (kill-tree.ts, item 865).
507
+ const letGo = holdGroups([child])
508
+ killTree(child, 'SIGTERM')
509
+ // Same escalation as `armTimeout`: a server that traps TERM and
510
+ // never became ready is not in the persistent registry, so nothing
511
+ // else would ever kill it — it outlived the run under init.
512
+ const killTimer = setTimeout(() => {
513
+ killTree(child, 'SIGKILL')
514
+ letGo()
515
+ }, killGraceMs(TIMEOUT_SIGKILL_GRACE_MS))
516
+ killTimer.unref?.()
398
517
  }
399
518
  }, opts.timeoutMs)
400
519
  }
@@ -403,12 +522,16 @@ export function runPersistent(opts: PersistentOptions): PersistentSpawn {
403
522
  // — reject the ready promise so the caller can surface it.
404
523
  void child.exited.then((code) => {
405
524
  opts.liveChildren?.delete(child)
525
+ markGroupIfGone(child)
526
+ releaseGroup(child)
406
527
  if (readyTimer !== undefined) clearTimeout(readyTimer)
407
528
  if (readyAt === undefined) {
408
529
  rejectReady(
409
- new Error(
530
+ new PersistentReadyError(
410
531
  `persistent task exited before becoming ready (exit ${code ?? '?'})` +
411
532
  (readyRe ? ` — readyWhen pattern never matched` : ''),
533
+ 'exited',
534
+ code ?? undefined,
412
535
  ),
413
536
  )
414
537
  }
@@ -425,6 +548,22 @@ export function runPersistent(opts: PersistentOptions): PersistentSpawn {
425
548
  }
426
549
  }
427
550
 
551
+ /**
552
+ * What a task's frame says when `Bun.spawn` itself threw. The text goes
553
+ * through `onStderr` AND onto the result: the orchestrator retains no
554
+ * stderr (execute-task's capture), so a reason that only sat on the result
555
+ * reached nobody — a box without `sh` showed "failed (exit 127)" under a
556
+ * bare `$ <command>` and nothing else (2026-09-16). An ENOENT with the
557
+ * working directory in place is the shell; Bun's own text names the rest.
558
+ */
559
+ export function spawnFailureText(err: unknown, cwd: string, what = 'task'): string {
560
+ if (isExecutableMissing(err) && existsSync(cwd)) {
561
+ return `\n[vx] vx runs each task with sh -c: failed to spawn 'sh' (working dir: ${cwd}). Install a POSIX sh and re-run.\n`
562
+ }
563
+ const message = err instanceof Error ? err.message : String(err)
564
+ return `\n[vx] failed to spawn ${what}: ${message}\n`
565
+ }
566
+
428
567
  export async function runCommand(opts: RunOptions): Promise<RunResult> {
429
568
  const start = Date.now()
430
569
  const fullCommand =
@@ -434,21 +573,24 @@ export async function runCommand(opts: RunOptions): Promise<RunResult> {
434
573
 
435
574
  let proc: ReturnType<typeof Bun.spawn>
436
575
  try {
437
- proc = Bun.spawn(['sh', '-c', execWrap(fullCommand)], {
438
- cwd: opts.cwd,
439
- env: opts.env as Record<string, string>,
440
- stdin: 'ignore',
441
- stdout: 'pipe',
442
- stderr: 'pipe',
443
- })
576
+ proc = spawnGuarded(() =>
577
+ Bun.spawn([executablePath('sh'), '-c', execWrap(fullCommand)], {
578
+ argv0: 'sh',
579
+ cwd: opts.cwd,
580
+ env: opts.env as Record<string, string>,
581
+ stdin: 'ignore',
582
+ stdout: 'pipe',
583
+ stderr: 'pipe',
584
+ // Its own session and process group, so a kill reaches what it
585
+ // forked (kill-tree.ts). stdin is ignored, so a background group
586
+ // never stops on a terminal read.
587
+ detached: true,
588
+ }),
589
+ )
444
590
  } catch (err) {
445
- const message = err instanceof Error ? err.message : String(err)
446
- return {
447
- exitCode: 127,
448
- durationMs: Date.now() - start,
449
- stdout: '',
450
- stderr: `\n[vx] failed to spawn: ${message}\n`,
451
- }
591
+ const stderr = spawnFailureText(err, opts.cwd)
592
+ opts.onStderr?.(stderr)
593
+ return { exitCode: 127, durationMs: Date.now() - start, stdout: '', stderr }
452
594
  }
453
595
 
454
596
  opts.liveChildren?.add(proc)
@@ -465,11 +607,15 @@ export async function runCommand(opts: RunOptions): Promise<RunResult> {
465
607
  // lets a clean exit EOF immediately and only cuts off a stuck reader after a
466
608
  // brief grace — without this the run hangs forever.
467
609
  await proc.exited
468
- timeout.clear()
610
+ await timeout.settle()
611
+ let cut = false
469
612
  if (timeout.timedOut()) ac.abort()
470
- else await drainOrAbort(streams, ac)
471
- const [stdout, stderr] = await streams
613
+ else cut = await drainOrAbort(streams, ac)
614
+ const [stdout, streamed] = await streams
615
+ if (cut) opts.onStderr?.(POST_EXIT_CUT_LINE)
616
+ const stderr = cut ? streamed + POST_EXIT_CUT_LINE : streamed
472
617
  opts.liveChildren?.delete(proc)
618
+ releaseGroup(proc)
473
619
  const exitCode = proc.exitCode ?? (proc.signalCode ? signalExitCode(proc.signalCode) : 1)
474
620
  return {
475
621
  exitCode,
@@ -478,7 +624,100 @@ export async function runCommand(opts: RunOptions): Promise<RunResult> {
478
624
  stderr,
479
625
  ...(proc.signalCode ? { signal: proc.signalCode } : {}),
480
626
  ...(timeout.timedOut() ? { timedOut: true } : {}),
481
- ...resourceUsageToCpuRss(proc.resourceUsage()),
627
+ ...resourceUsageToCpuRss(proc.resourceUsage(), ownRssHighWater()),
628
+ }
629
+ }
630
+
631
+ /**
632
+ * This process's own RSS high-water mark, in bytes: the floor under which
633
+ * a child's `ru_maxrss` says nothing about the child. Linux folds the
634
+ * forking parent's high-water mark into the child's figure at exec (a
635
+ * forked child starts with its parent's pages; `exec_mmap` keeps the old
636
+ * mm's peak), so a task lighter than vx itself reads vx's footprint —
637
+ * `true` read 44 MB through vx while its shell's `VmHWM` was 1.9 MB, and
638
+ * 300 MB allocated in the parent made `true` read 328 MB (2026-09-12).
639
+ * Read after the child exits so it covers the whole task's span (the mark
640
+ * is monotonic). Linux reads `VmHWM`; elsewhere the current RSS is the
641
+ * bound in hand.
642
+ */
643
+ export function ownRssHighWater(): number {
644
+ if (process.platform === 'linux') {
645
+ try {
646
+ const m = /VmHWM:\s+(\d+) kB/.exec(readFileSync('/proc/self/status', 'utf8'))
647
+ if (m !== null) return Number(m[1]) * 1024
648
+ } catch {
649
+ // /proc unreadable: fall through to the current RSS.
650
+ }
651
+ }
652
+ return process.memoryUsage.rss()
653
+ }
654
+
655
+ /**
656
+ * What a task's captured output keeps: the first `CAPTURE_HEAD_CHARS` and
657
+ * the last `CAPTURE_TAIL_CHARS` characters, the dropped middle counted and
658
+ * named where it was. The live stream is never bounded — every byte still
659
+ * reaches the terminal as the task writes it — only the copy vx retains
660
+ * for the cache entry and its replay. Unbounded, a task printing 200 MB
661
+ * cost vx 620 MB of RSS on the miss AND on every hit, and its stdout sat
662
+ * whole in `cache.db` (measured 2026-09-16); a realistic chatty suite is
663
+ * 5–20 MB, so the bound is above what a replay is worth reading anyway.
664
+ */
665
+ export const CAPTURE_HEAD_CHARS = 8 * 1024 * 1024
666
+ export const CAPTURE_TAIL_CHARS = 8 * 1024 * 1024
667
+
668
+ const mib = (n: number): string => `${(n / (1024 * 1024)).toFixed(1)} MiB`
669
+
670
+ /** The line that stands where the dropped middle was. */
671
+ export function droppedOutputLine(dropped: number): string {
672
+ return (
673
+ `\n[vx] ${mib(dropped)} of output not kept — vx keeps the first ${mib(CAPTURE_HEAD_CHARS)} ` +
674
+ `and the last ${mib(CAPTURE_TAIL_CHARS)} of a task's output for its cache entry and replay\n`
675
+ )
676
+ }
677
+
678
+ /**
679
+ * A head-and-tail accumulator: the head fills once, the tail is a ring
680
+ * of chunks trimmed from the front, so memory is bounded by the two
681
+ * limits plus one chunk whatever the task prints.
682
+ */
683
+ class BoundedCapture {
684
+ private head = ''
685
+ private readonly tail: string[] = []
686
+ private tailLen = 0
687
+ private dropped = 0
688
+
689
+ push(chunk: string): void {
690
+ if (this.head.length < CAPTURE_HEAD_CHARS) {
691
+ const room = CAPTURE_HEAD_CHARS - this.head.length
692
+ if (chunk.length <= room) {
693
+ this.head += chunk
694
+ return
695
+ }
696
+ this.head += chunk.slice(0, room)
697
+ chunk = chunk.slice(room)
698
+ }
699
+ this.tail.push(chunk)
700
+ this.tailLen += chunk.length
701
+ while (this.tailLen > CAPTURE_TAIL_CHARS) {
702
+ const first = this.tail[0]!
703
+ const excess = this.tailLen - CAPTURE_TAIL_CHARS
704
+ if (first.length <= excess) {
705
+ this.tail.shift()
706
+ this.tailLen -= first.length
707
+ this.dropped += first.length
708
+ } else {
709
+ this.tail[0] = first.slice(excess)
710
+ this.tailLen -= excess
711
+ this.dropped += excess
712
+ }
713
+ }
714
+ }
715
+
716
+ text(): string {
717
+ const tail = this.tail.join('')
718
+ return this.dropped === 0
719
+ ? this.head + tail
720
+ : this.head + droppedOutputLine(this.dropped) + tail
482
721
  }
483
722
  }
484
723
 
@@ -501,7 +740,7 @@ export async function streamToString(
501
740
  // so the runtime value is always a ReadableStream; the `number` branch
502
741
  // is unreachable but typed.
503
742
  if (!stream || typeof stream === 'number') return ''
504
- let full = ''
743
+ const full = new BoundedCapture()
505
744
  const reader = stream.getReader()
506
745
  const decoder = new TextDecoder()
507
746
  // On abort, cancel the read so a pending `reader.read()` resolves
@@ -517,19 +756,63 @@ export async function streamToString(
517
756
  const { value, done } = await reader.read()
518
757
  if (done) break
519
758
  const chunk = decoder.decode(value, { stream: true })
520
- if (retain) full += chunk
759
+ if (retain) full.push(chunk)
521
760
  onChunk?.(chunk)
522
761
  }
523
762
  const tail = decoder.decode()
524
763
  if (tail.length > 0) {
525
- if (retain) full += tail
764
+ if (retain) full.push(tail)
526
765
  onChunk?.(tail)
527
766
  }
528
767
  } finally {
529
768
  signal?.removeEventListener('abort', onAbort)
530
769
  reader.releaseLock()
531
770
  }
532
- return full
771
+ return full.text()
772
+ }
773
+
774
+ /**
775
+ * How far above the parent's own mark a child's `ru_maxrss` must read to
776
+ * count as the child's. A light child inherits the parent's footprint at
777
+ * exec, so its reading sits ON the floor by construction, and the kernel's
778
+ * per-thread RSS counters lag by up to 64 pages between syncs — a `true`
779
+ * spawned from a 300 MB parent read 376 MB against a floor a few pages
780
+ * lower on one CI run in twelve (2026-09-16). Four MiB is above any
781
+ * accounting jitter and below what any reservation resolves (64 MB steps).
782
+ */
783
+ export const RSS_FLOOR_SLACK_BYTES = 4 * 1024 * 1024
784
+
785
+ const MIN_PLAUSIBLE_PEAK_BYTES = 1024 * 1024
786
+
787
+ /**
788
+ * A child's `maxRSS` in BYTES, whatever unit the runtime reported it in.
789
+ *
790
+ * Bun >= 1.4 — vx's declared floor — normalizes the kernel's `ru_maxrss`
791
+ * (kilobytes on Linux, bytes on macOS) before handing it over, and this file
792
+ * used to state that as "BYTES on every platform". It is not true of every
793
+ * Bun: 1.3.11 passes the raw kilobytes through, so every peak read 1024×
794
+ * SMALL, fell under the parent floor and recorded nothing. The opposite
795
+ * mistake is in this file's own history — an unconditional ×1024 on Linux
796
+ * made a 64 MB suite read as 64 GB and, once reservations were learned from
797
+ * that history, ran every task alone (2026-09-12). Both directions come from
798
+ * asserting a platform unit instead of deciding it from the number in hand.
799
+ *
800
+ * The number decides it. No process peaks under a megabyte — a bare `true`
801
+ * costs a couple — so a reading below that is not bytes, it is the kernel's
802
+ * kilobytes, and multiplying is right by the same margin that makes the test
803
+ * safe. A real byte figure is never near the threshold, and neither is a real
804
+ * kilobyte one: the two live 1024× apart.
805
+ *
806
+ * Calibrating against this process's OWN `process.resourceUsage()` was the
807
+ * first attempt and it was wrong: that is the node-compatible API, which
808
+ * reports kilobytes even where `Subprocess.resourceUsage()` reports bytes, so
809
+ * the ratio read 1024 on a runtime already handing over bytes and doubled the
810
+ * old 64 GB defect. It passed here, where both APIs use kilobytes, and turned
811
+ * CI red on both platforms (2026-09-20). Compare like with like, or do not
812
+ * compare at all.
813
+ */
814
+ export function peakRssBytes(maxRSS: number): number {
815
+ return maxRSS > 0 && maxRSS < MIN_PLAUSIBLE_PEAK_BYTES ? maxRSS * 1024 : maxRSS
533
816
  }
534
817
 
535
818
  /**
@@ -540,18 +823,19 @@ export async function streamToString(
540
823
  */
541
824
  export function resourceUsageToCpuRss(
542
825
  usage: ReturnType<ReturnType<typeof Bun.spawn>['resourceUsage']>,
543
- platform: NodeJS.Platform = process.platform,
826
+ /** The parent's own high-water mark (`ownRssHighWater`); a peak at or under it is inherited, not the child's, and is not reported. */
827
+ floorBytes = 0,
544
828
  ): { cpuMs?: number; peakRssBytes?: number } {
545
829
  if (!usage) return {}
546
830
  // cpuTime.total is microseconds as a bigint → ms.
547
831
  const cpuMs = Number(usage.cpuTime.total) / 1000
548
- // maxRSS is the raw kernel `ru_maxrss`, whose UNIT IS PLATFORM-SPECIFIC and
549
- // Bun does NOT normalize it:
550
- // • Linux → kilobytes (multiply by 1024 for bytes)
551
- // • macOS / BSD → bytes (already bytes; do NOT multiply)
552
- // • Windows → bytes (PeakWorkingSetSize)
553
- // Treating macOS's byte value as KB inflates peak RSS by 1024× — e.g. a
554
- // real 460 MB showed up as 460 GB. Convert per platform.
555
- const peakRssBytes = platform === 'linux' ? usage.maxRSS * 1024 : usage.maxRSS
556
- return { cpuMs, peakRssBytes }
832
+ // `maxRSS` in whatever unit this runtime reports, in bytes by the rule
833
+ // above. `tests/runner.test.ts` allocates a known number of bytes and
834
+ // reads the peak back within a bounded factor of it, which is the
835
+ // end-to-end check on both the unit and the floor.
836
+ // A reading at or within the slack of the parent's own mark is the
837
+ // parent's (see `ownRssHighWater`, `RSS_FLOOR_SLACK_BYTES`): the child's
838
+ // peak is unknown, bounded by it.
839
+ const peak = peakRssBytes(usage.maxRSS)
840
+ return peak > floorBytes + RSS_FLOOR_SLACK_BYTES ? { cpuMs, peakRssBytes: peak } : { cpuMs }
557
841
  }