@vzn/vx 0.0.21 → 0.0.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/README.md +21 -18
  2. package/package.json +6 -6
  3. package/src/bin.ts +52 -3
  4. package/src/cache/archive.ts +170 -21
  5. package/src/cache/cache.ts +667 -288
  6. package/src/cache/chained-cache.ts +14 -2
  7. package/src/cache/config-evals.ts +29 -6
  8. package/src/cache/file-hashes.ts +63 -35
  9. package/src/cache/git-inputs.ts +398 -69
  10. package/src/cache/index.ts +8 -1
  11. package/src/cache/inputs.ts +403 -102
  12. package/src/cache/key-fold.ts +232 -0
  13. package/src/cache/layer.ts +81 -11
  14. package/src/cache/layered-cache.ts +199 -32
  15. package/src/cache/output-index.ts +109 -10
  16. package/src/cache/run-history.ts +8 -3
  17. package/src/cache/zstd.ts +53 -30
  18. package/src/cli/cache.ts +51 -21
  19. package/src/cli/format.ts +4 -15
  20. package/src/cli/help.ts +13 -3
  21. package/src/cli/index.ts +35 -9
  22. package/src/cli/info.ts +49 -7
  23. package/src/cli/init.ts +17 -5
  24. package/src/cli/last.ts +96 -21
  25. package/src/cli/lock.ts +34 -17
  26. package/src/cli/plan-format.ts +4 -4
  27. package/src/cli/plugin-commands.ts +20 -5
  28. package/src/cli/run.ts +55 -12
  29. package/src/cli/select.ts +23 -13
  30. package/src/cli/show.ts +9 -4
  31. package/src/cli/upgrade.ts +134 -14
  32. package/src/cli/watch.ts +611 -102
  33. package/src/cli/why.ts +8 -16
  34. package/src/cli/workspace-config.ts +21 -9
  35. package/src/config.ts +46 -9
  36. package/src/exec/executor.ts +51 -5
  37. package/src/exec/index.ts +10 -0
  38. package/src/exec/kill-tree.ts +281 -0
  39. package/src/exec/local-executor.ts +1 -1
  40. package/src/exec/runner.ts +327 -82
  41. package/src/exec/sandbox-binds.ts +34 -5
  42. package/src/exec/sandbox-paths.ts +22 -2
  43. package/src/exec/sandbox-runtime.ts +627 -102
  44. package/src/exec/sandbox-violations.ts +36 -23
  45. package/src/graph/dependency-spec.ts +14 -14
  46. package/src/graph/index.ts +12 -2
  47. package/src/graph/priorities.ts +130 -0
  48. package/src/graph/scheduler.ts +182 -134
  49. package/src/graph/task-graph.ts +552 -188
  50. package/src/index.ts +16 -2
  51. package/src/orchestrator/admission.ts +41 -17
  52. package/src/orchestrator/colors.ts +8 -3
  53. package/src/orchestrator/deferred-outputs.ts +22 -7
  54. package/src/orchestrator/doctor.ts +122 -14
  55. package/src/orchestrator/events.ts +71 -2
  56. package/src/orchestrator/excluded-keys.ts +155 -0
  57. package/src/orchestrator/execute-task.ts +331 -37
  58. package/src/orchestrator/failure-recap.ts +121 -0
  59. package/src/orchestrator/fingerprint-watch.ts +89 -0
  60. package/src/orchestrator/framed-output.ts +81 -14
  61. package/src/orchestrator/history.ts +6 -7
  62. package/src/orchestrator/hit-restore.ts +82 -18
  63. package/src/orchestrator/index.ts +7 -1
  64. package/src/orchestrator/keyed-projects.ts +89 -0
  65. package/src/orchestrator/local-shortcircuit.ts +83 -23
  66. package/src/orchestrator/lockfile-claim.ts +152 -32
  67. package/src/orchestrator/logger.ts +116 -20
  68. package/src/orchestrator/metrics.ts +156 -60
  69. package/src/orchestrator/miss-save.ts +106 -43
  70. package/src/orchestrator/options.ts +30 -4
  71. package/src/orchestrator/persistent.ts +41 -21
  72. package/src/orchestrator/placement.ts +55 -25
  73. package/src/orchestrator/plan.ts +24 -11
  74. package/src/orchestrator/plugin-host.ts +17 -23
  75. package/src/orchestrator/plugin.ts +31 -8
  76. package/src/orchestrator/prepare.ts +99 -13
  77. package/src/orchestrator/projects.ts +19 -8
  78. package/src/orchestrator/remote-prefetch.ts +21 -1
  79. package/src/orchestrator/run-artifacts.ts +9 -0
  80. package/src/orchestrator/run-context.ts +19 -4
  81. package/src/orchestrator/run-lock.ts +349 -0
  82. package/src/orchestrator/run-records.ts +6 -23
  83. package/src/orchestrator/run-report.ts +4 -2
  84. package/src/orchestrator/run.ts +264 -57
  85. package/src/orchestrator/sandbox-request.ts +388 -62
  86. package/src/orchestrator/save-lane.ts +4 -7
  87. package/src/orchestrator/shell-verdict.ts +117 -0
  88. package/src/orchestrator/signals.ts +86 -30
  89. package/src/orchestrator/stable-keys.ts +202 -10
  90. package/src/orchestrator/status-line.ts +33 -1
  91. package/src/orchestrator/summary.ts +49 -2
  92. package/src/orchestrator/task-hash.ts +87 -7
  93. package/src/orchestrator/task-log-buffer.ts +7 -7
  94. package/src/orchestrator/telemetry-host.ts +16 -7
  95. package/src/orchestrator/telemetry.ts +69 -28
  96. package/src/orchestrator/upstream.ts +91 -45
  97. package/src/util/bun-version.ts +43 -0
  98. package/src/util/errors.ts +86 -0
  99. package/src/util/hash.ts +10 -1
  100. package/src/util/index.ts +32 -3
  101. package/src/util/paths.ts +147 -5
  102. package/src/util/procfs.ts +20 -0
  103. package/src/util/settle.ts +4 -2
  104. package/src/util/size.ts +33 -0
  105. package/src/util/task-id.ts +20 -0
  106. package/src/util/timing.ts +11 -1
  107. package/src/util/which.ts +48 -0
  108. package/src/workspace/affected.ts +272 -64
  109. package/src/workspace/config-cache.ts +262 -32
  110. package/src/workspace/config-eval.ts +49 -12
  111. package/src/workspace/config-imports.ts +126 -5
  112. package/src/workspace/config-schema.ts +237 -41
  113. package/src/workspace/filter.ts +33 -18
  114. package/src/workspace/fingerprint.ts +29 -11
  115. package/src/workspace/index.ts +7 -0
  116. package/src/workspace/json-data.ts +90 -0
  117. package/src/workspace/load-reads.ts +32 -0
  118. package/src/workspace/lockfile.ts +33 -11
  119. package/src/workspace/migrate-scripts.ts +60 -7
  120. package/src/workspace/migration.ts +40 -1
  121. package/src/workspace/package-graph.ts +154 -16
  122. package/src/workspace/project-loader.ts +227 -77
  123. package/src/workspace/workspace.ts +173 -23
@@ -3,6 +3,7 @@
3
3
  // so the layers can be swapped without touching the others.
4
4
 
5
5
  import type { ProjectEntry } from '../workspace/index.js'
6
+ import { loadWorkspace, unreachedHint, unreachedPackages } from '../workspace/index.js'
6
7
  import path from 'node:path'
7
8
  import { type CacheLayer, type CachePolicy, FULL_CACHE_POLICY } from '../cache/index.js'
8
9
  import { VERSION } from '../version.js'
@@ -18,36 +19,51 @@ import {
18
19
  type TaskOutcome,
19
20
  } from '../graph/index.js'
20
21
  import {
22
+ formatBytes,
21
23
  mark,
24
+ killGraceMs,
22
25
  MAX_TIMEOUT_MS,
26
+ parseDuration,
27
+ parseSize,
23
28
  printTimings,
24
29
  ulid,
25
30
  nearest,
26
31
  UserError,
27
32
  machineParallelism,
33
+ teardownTimeoutMs,
28
34
  } from '../util/index.js'
35
+ import { keyedProjects } from './keyed-projects.js'
29
36
  import { prepareSandbox } from './sandbox-request.js'
30
37
  import type { OutputDirSnapshot } from './miss-save.js'
31
38
  import { admitTasks, taintTracker } from './admission.js'
32
- import { busLogger, createEventBus, terminalSubscriber } from './events.js'
39
+ import { excludedTaint } from './excluded-keys.js'
40
+ import { keyUpstream } from './upstream.js'
41
+ import { busLogger, createEventBus, terminalSubscriber, type EventBus } from './events.js'
33
42
  import { installPlugins } from './plugin.js'
34
43
  import { buildAdmission, resolveExecutors, teardownPlugins } from './plugin-host.js'
35
44
  import { subscribeTelemetry, type TelemetryHandle } from './telemetry-host.js'
36
45
  import { assembleRunSummary, isPassStatus } from './telemetry.js'
37
46
  import type { RunContextRecord } from './telemetry.js'
38
- import { defaultLogger, resolveOutputView } from './logger.js'
39
- import { detectColors } from './colors.js'
47
+ import { defaultLogger, resolveOutputView, type Logger } from './logger.js'
48
+ import { detectColors, type ColorSupport } from './colors.js'
40
49
  import { formatPersistentList } from './framed-output.js'
41
50
  import { LocalHistoryProvider } from './history.js'
42
51
  import { plan, type RunPlan } from './plan.js'
43
- import { prepareRun } from './prepare.js'
44
- import { forwardSignals, terminateChildren } from './signals.js'
52
+ import { prepareRun, type PreparedRun } from './prepare.js'
53
+ import { acquireRunLock } from './run-lock.js'
54
+ import {
55
+ forwardedSignal,
56
+ forwardSignals,
57
+ SIGNAL_SHUTDOWN_GRACE_MS,
58
+ terminateChildren,
59
+ } from './signals.js'
45
60
  import {
46
61
  hasPooledExecutor,
47
62
  placeTasks,
48
63
  planExecutorOf,
49
64
  poolOfPlacement,
50
65
  UNPLACED_EXECUTOR,
66
+ locallyPlaced,
51
67
  } from './placement.js'
52
68
  import {
53
69
  captureDefaultBranch,
@@ -60,10 +76,15 @@ import { startRemotePrefetch } from './remote-prefetch.js'
60
76
  import { startLocalShortCircuit, type ShortCircuit } from './local-shortcircuit.js'
61
77
 
62
78
  import { assembleRunRecords } from './run-records.js'
63
- import { selectKeepAlive, shutdownPersistent } from './persistent.js'
79
+ import { hasEnded, selectKeepAlive, shutdownPersistent } from './persistent.js'
64
80
  import { writeRunProfile, writeRunSummary } from './run-artifacts.js'
65
81
  import { createSaveLane } from './save-lane.js'
66
- import { formatAbortedSection, formatFlakySection, formatRunSummary } from './summary.js'
82
+ import {
83
+ formatAbortedSection,
84
+ formatFlakySection,
85
+ formatRunSummary,
86
+ formatSkippedSection,
87
+ } from './summary.js'
67
88
  import { detectFlaky, type FlakyCandidate } from './failure-mode.js'
68
89
  import type { RunOptions, RunSummary } from './options.js'
69
90
 
@@ -167,11 +188,31 @@ export async function run(options: RunOptions): Promise<RunSummary> {
167
188
  // additional subscribers. The fan-out is synchronous and order-
168
189
  // preserving, so terminal output is byte-identical to a direct call.
169
190
  // See docs/design/event-stream-2026-06.md.
170
- const sink = options.log ?? defaultLogger(colors, resolveOutputView(options))
171
- // An injected bus (e.g. from `--ui`) already has surfaces subscribed;
172
- // we just add the terminal renderer. Otherwise a fresh internal bus.
191
+ const terminal =
192
+ options.log === undefined
193
+ ? defaultLogger(colors, resolveOutputView(options), process.stdout, { coalesce: true })
194
+ : null
195
+ const sink = options.log ?? terminal!
196
+ // An injected bus already has surfaces subscribed; we add the terminal
197
+ // renderer for this run and take it off again on every way out: the bus
198
+ // outlives the run, and a renderer left behind reported the next run on
199
+ // it twice (item 635). Otherwise a fresh internal bus.
173
200
  const bus = options.bus ?? createEventBus()
174
- bus.subscribe(terminalSubscriber(sink))
201
+ const unsubscribeTerminal = bus.subscribe(terminalSubscriber(sink))
202
+ try {
203
+ return await runOnBus(options, bus, colors, () => terminal?.failureRecap() ?? [])
204
+ } finally {
205
+ unsubscribeTerminal()
206
+ terminal?.settle()
207
+ }
208
+ }
209
+
210
+ async function runOnBus(
211
+ options: RunOptions,
212
+ bus: EventBus,
213
+ colors: ColorSupport,
214
+ failureRecap: () => string[],
215
+ ): Promise<RunSummary> {
175
216
  const log = busLogger(bus)
176
217
 
177
218
  const prepared = await prepareRun(options, log)
@@ -198,7 +239,7 @@ export async function run(options: RunOptions): Promise<RunSummary> {
198
239
  // case too; the message is identical, so that branch stays below.
199
240
  if (prepared.unresolvedTasks.length > 0) {
200
241
  log.status(
201
- `No projects declare task(s): ${prepared.unresolvedTasks.join(', ')}.${didYouMean(prepared.unresolvedTasks, prepared.projects)}${initHint(prepared)}`,
242
+ `No projects declare task(s): ${prepared.unresolvedTasks.join(', ')}.${didYouMean(prepared.unresolvedTasks, prepared.projects)}${await initHint(prepared)}`,
202
243
  )
203
244
  prepared.cache.close()
204
245
  return { ok: false, outcomes: [] }
@@ -210,7 +251,7 @@ export async function run(options: RunOptions): Promise<RunSummary> {
210
251
  // buildTaskGraph semantics but logged just in case.
211
252
  const msg =
212
253
  prepared.empty === 'no-tasks-declared'
213
- ? `No projects declare task(s): ${options.tasks.join(', ')}.${initHint(prepared)}`
254
+ ? `No projects declare task(s): ${options.tasks.join(', ')}.${await initHint(prepared)}`
214
255
  : 'No tasks to run.'
215
256
  log.status(msg)
216
257
  prepared.cache.close()
@@ -242,6 +283,7 @@ export async function run(options: RunOptions): Promise<RunSummary> {
242
283
  cache,
243
284
  nodes,
244
285
  workspaceFingerprint,
286
+ fingerprintWatch,
245
287
  nestedDirsByProject,
246
288
  gitFilesCache,
247
289
  hashCache,
@@ -268,7 +310,7 @@ export async function run(options: RunOptions): Promise<RunSummary> {
268
310
  // can admit a remote-pooled task against its pool instead of a local
269
311
  // worker slot. Group tasks run nothing; persistent tasks never reach an
270
312
  // executor (local by construction) — both stay off the map.
271
- const placements = placeTasks(nodes, executors, false)
313
+ const placements = placeTasks(nodes, executors)
272
314
  // A `remote: 'only'` task nobody takes succeeds WITHOUT running. That is
273
315
  // deliberate — on a machine with no remote pool the ambient state already
274
316
  // is what the task would have produced — but it must not be SILENT: a task
@@ -313,9 +355,7 @@ export async function run(options: RunOptions): Promise<RunSummary> {
313
355
  // `--download` (default `all`) decides ONCE per task, here, whether a
314
356
  // remote execution's outputs come home.
315
357
  const downloadPolicy = options.download ?? 'all'
316
- const localPlaced = new Set(
317
- [...nodes.keys()].filter((id) => placements.executors.get(id)?.remote !== true),
318
- )
358
+ const localPlaced = locallyPlaced(placements, nodes.keys())
319
359
  const download = resolveDownloadModes({
320
360
  nodes,
321
361
  policy: downloadPolicy,
@@ -339,25 +379,59 @@ export async function run(options: RunOptions): Promise<RunSummary> {
339
379
  // exit; ownership moves here so the orchestrator can SIGTERM
340
380
  // them once the rest of the graph finishes.
341
381
  //
342
- // A SIGINT/SIGTERM mid-run forwards SIGTERM to everything in both
382
+ // A SIGINT/SIGTERM mid-run forwards that signal to everything in both
343
383
  // (`signals.ts`); the handlers are removed in the finally below so
344
384
  // repeated run() calls (test suites) never stack listeners.
345
385
  const liveChildren = new Set<ReturnType<typeof Bun.spawn>>()
346
386
  const persistentRegistry = new Map<string, ReturnType<typeof Bun.spawn>>()
387
+ // One stop for the run: an embedder's `RunOptions.signal` and a process
388
+ // signal both abort it. The scheduler stops dispatching (it reads the
389
+ // signal), the children are torn down, and run() returns through its own
390
+ // end-of-run path. Detached in the finally below.
391
+ const stopRun = new AbortController()
392
+ const forwardAbort = (): void => stopRun.abort(options.signal?.reason)
393
+ if (options.signal?.aborted === true) forwardAbort()
394
+ else options.signal?.addEventListener('abort', forwardAbort, { once: true })
395
+ // A run a process signal stopped records no history: until item 849 the
396
+ // handler exited before any outcome landed, and a Ctrl-C'd run then read
397
+ // in `vx last` as FAILED with 0 tasks (item 854). An embedder's abort is
398
+ // its own call and keeps its record.
399
+ let stoppedBySignal = false
400
+ let leftRun = (): void => {}
401
+ const runLeft = new Promise<void>((resolve) => {
402
+ leftRun = resolve
403
+ })
347
404
  const signals = forwardSignals({
348
405
  enabled: options.handleSignals ?? true,
349
406
  log,
350
407
  cache,
351
408
  liveChildren,
352
409
  persistentRegistry,
410
+ stop: (signal) => {
411
+ stoppedBySignal = true
412
+ stopRun.abort(signal)
413
+ },
414
+ done: runLeft,
415
+ // The grace, then one flush and each teardown at their own bound, and
416
+ // slack for the summary and the cache close.
417
+ boundMs:
418
+ killGraceMs(SIGNAL_SHUTDOWN_GRACE_MS) +
419
+ teardownTimeoutMs() * (1 + prepared.plugins.filter((p) => p.teardown).length) +
420
+ 2_000,
353
421
  })
354
- // `RunOptions.signal`: the same teardown the process handler runs, minus
355
- // the exit — the scheduler stops dispatching (it reads the signal) and
356
- // run() returns to its caller. Detached in the finally below.
422
+ // Awaited before run() leaves: a task's shell can die of the signal at
423
+ // once while a child it backgrounded ignores it and dies only to the
424
+ // SIGKILL after the grace, and a run that returned first left that child
425
+ // running past vx's exit (turborepo#14043's class).
426
+ let aborting: Promise<void> | undefined
357
427
  const onAbort = (): void => {
358
- void terminateChildren(() => [...liveChildren, ...persistentRegistry.values()])
428
+ aborting = terminateChildren(
429
+ () => [...liveChildren, ...persistentRegistry.values()],
430
+ forwardedSignal(stopRun.signal.reason),
431
+ )
359
432
  }
360
- options.signal?.addEventListener('abort', onAbort, { once: true })
433
+ if (stopRun.signal.aborted) onAbort()
434
+ else stopRun.signal.addEventListener('abort', onAbort, { once: true })
361
435
  // The cache handle must be released on EVERY exit path, not just the
362
436
  // happy one: `close()` is also where the run's deferred `accessed_at`
363
437
  // bumps are flushed, so a throw between opening the cache and the
@@ -368,11 +442,25 @@ export async function run(options: RunOptions): Promise<RunSummary> {
368
442
  // holding the handle open for a dev server's whole lifetime would be
369
443
  // worse — and because close() re-runs its retention DELETEs.
370
444
  let cacheClosed = false
371
- const closeCache = (): void => {
445
+ const closeCache = async (): Promise<void> => {
372
446
  if (cacheClosed) return
373
447
  cacheClosed = true
374
448
  cache.close()
449
+ // Released with the handle: before a persistent task's wait, on every
450
+ // exit path (see run-lock.ts for why a run holds one at all). AWAITED:
451
+ // fired and forgotten, the release lost the race with the CLI's exit —
452
+ // `bin.ts` ends the process as soon as run() resolves — and every CLI
453
+ // run left its lock directory in the temp dir (658 of them after one
454
+ // day's gates, 2026-09-16); the next run reclaimed it, so nothing
455
+ // waited, but a directory per run is litter the box never sheds.
456
+ await releaseRunLock()
375
457
  }
458
+ // Taken after the early exits above (nothing they do touches a tree) and
459
+ // before the schedule: from here on tasks clean, restore and write.
460
+ const releaseRunLock = await acquireRunLock(prepared.workspaceRoot, {
461
+ log: (m) => log.status(m),
462
+ signal: stopRun.signal,
463
+ })
376
464
  try {
377
465
  // One run-id per `vx run` invocation. Every task in the resulting
378
466
  // graph carries it so analytics queries can group by invocation.
@@ -463,6 +551,7 @@ export async function run(options: RunOptions): Promise<RunSummary> {
463
551
  }
464
552
 
465
553
  const sandboxArmer = prepareSandbox(nodes.values())
554
+ const keyed = keyedProjects(nodes)
466
555
  const outputDirSnapshots: OutputDirSnapshot[] = []
467
556
  // Saves run off the execution slot, twice the cap at once (memory:
468
557
  // each pack holds an artifact's bytes); a failed save is a miss next
@@ -566,9 +655,18 @@ export async function run(options: RunOptions): Promise<RunSummary> {
566
655
  }
567
656
 
568
657
  // Whether this task runs behind a failure (`continueMode: 'always'`
569
- // only) — its save is withheld and the taint propagates; see
658
+ // only) or on the key of a dependency `--exclude-dependencies` kept
659
+ // from running — its save is withheld and the taint propagates; see
570
660
  // admission.ts.
571
- const isTainted = taintTracker(options.continueMode === 'always')
661
+ const excluded = excludedTaint(nodes)
662
+ if (excluded.unsaved > 0) {
663
+ log.status(
664
+ `[vx] --exclude-dependencies: ${excluded.unsaved} cached task(s) build on a skipped dependency; what they build is not saved`,
665
+ )
666
+ }
667
+ const isTainted = taintTracker(options.continueMode === 'always', excluded.seeds)
668
+ const dependedOn = new Set<string>()
669
+ for (const n of nodes.values()) for (const d of n.deps) dependedOn.add(d)
572
670
 
573
671
  const buildExecuteArgs = (node: TaskNode, upstream: TaskOutcome[], reuseProbe = true) => {
574
672
  const probe = reuseProbe ? shortCircuit.preProbed.get(node.id) : undefined
@@ -597,10 +695,14 @@ export async function run(options: RunOptions): Promise<RunSummary> {
597
695
  hashCache,
598
696
  ...(probe !== undefined ? { preProbed: probe } : {}),
599
697
  ...(taint ? { taintedUpstream: true } : {}),
698
+ ...(dependedOn.has(node.id) ? {} : { noDependants: true as const }),
699
+ fingerprintWatch,
600
700
  ...(sandboxArmer !== null ? { armSandbox: () => sandboxArmer.arm() } : {}),
701
+ keyedProjects: keyed,
601
702
  outputDirSnapshots,
602
703
  deferSave: saveLane.defer,
603
704
  deferredSaves,
705
+ stopSignal: stopRun.signal,
604
706
  }
605
707
  }
606
708
 
@@ -628,7 +730,7 @@ export async function run(options: RunOptions): Promise<RunSummary> {
628
730
  ...(hasPooledExecutor(executors) ? { poolOf: poolOfPlacement(placements) } : {}),
629
731
  ...(admit !== undefined ? { admit } : {}),
630
732
  ...(options.continueMode !== undefined ? { continueMode: options.continueMode } : {}),
631
- ...(options.signal !== undefined ? { signal: options.signal } : {}),
733
+ signal: stopRun.signal,
632
734
  onStart: (node) => {
633
735
  log.taskStart?.(node)
634
736
  },
@@ -636,7 +738,7 @@ export async function run(options: RunOptions): Promise<RunSummary> {
636
738
  log.taskComplete(o.node, o)
637
739
  narrowDemand(o.node.id)
638
740
  },
639
- execute: executeWithDedup,
741
+ execute: (node, upstream) => executeWithDedup(node, keyUpstream(node, upstream)),
640
742
  // A `schedule` plugin's weights; the scheduler keeps its structural
641
743
  // baseline as the tie-break. Empty map → baseline only.
642
744
  ...(prepared.priorities.size > 0 ? { priorities: prepared.priorities } : {}),
@@ -653,15 +755,17 @@ export async function run(options: RunOptions): Promise<RunSummary> {
653
755
  // signal loop) and embedders that manage lifecycle themselves — both
654
756
  // expect run() to return, not block on a server.
655
757
  const foreground = options.log === undefined && (options.handleSignals ?? true)
656
- const keepAlive = selectKeepAlive(persistentRegistry, nodes, foreground)
657
- await shutdownPersistent(persistentRegistry, keepAlive.children)
758
+ // An aborted run's children are already being torn down: nothing to hold.
759
+ const hold = options.holdPersistent === true && !stopRun.signal.aborted
760
+ const keepAlive = selectKeepAlive(persistentRegistry, nodes, foreground || hold)
761
+ const crashedPersistent = await shutdownPersistent(persistentRegistry, keepAlive.children)
658
762
 
659
763
  mark('run graph')
660
764
  // Clear the status line for good before the summary prints.
661
765
  log.runEnd?.()
662
766
 
663
767
  const list = [...outcomes.values()]
664
- const ok = list.every((o) => isPassStatus(o.status))
768
+ const ok = list.every((o) => isPassStatus(o.status)) && crashedPersistent.length === 0
665
769
 
666
770
  // The summary + artifact writers + recordRun pass all exclude group
667
771
  // tasks via the shared tallyOutcomes helper. We pass the full
@@ -670,13 +774,23 @@ export async function run(options: RunOptions): Promise<RunSummary> {
670
774
  const totalMs = Number(process.hrtime.bigint() - runStartHrTimeNs) / 1_000_000
671
775
  // Foreground dev mode: between the task frame and the footer, list
672
776
  // the persistent tasks still running (see the keep-alive block below).
673
- if (keepAlive.nodes.length > 0) {
674
- for (const line of formatPersistentList(keepAlive.nodes, colors)) log.status(line)
777
+ // Only the ones still up: a requested server that crashed while its
778
+ // dependants ran is dead, and the wait below says so.
779
+ const stillUp = keepAlive.nodes.filter((_, i) => !hasEnded(keepAlive.children[i]!))
780
+ if (stillUp.length > 0) {
781
+ for (const line of formatPersistentList(stillUp, colors)) log.status(line)
675
782
  }
676
783
  for (const line of formatRunSummary(list, totalMs, colors, runContext)) log.status(line)
677
784
  // A task killed by a shutdown signal is in no bucket above, yet it makes
678
785
  // `ok` false — name it, or the red exit is undiagnosable.
679
786
  for (const line of formatAbortedSection(list)) log.status(line)
787
+ // Likewise a dependency-only server that died before the end of the
788
+ // graph stopped it: its outcome says `success` (it became ready).
789
+ for (const c of crashedPersistent)
790
+ log.status(`vx: ${c.id} exited with code ${c.code} before the run stopped it`)
791
+ // The footer's "N skipped" names no task; this names each under the
792
+ // failure that blocked it.
793
+ for (const line of formatSkippedSection(list)) log.status(line)
680
794
  // Judged against the history BEFORE this run's rows land, so the query
681
795
  // is one scan over the executed tasks' keys and nothing at all on a run
682
796
  // that executed none (every hit, every skip).
@@ -692,6 +806,8 @@ export async function run(options: RunOptions): Promise<RunSummary> {
692
806
  ` Deferred: ${stillDeferred.length} task(s) left outputs remote (--download=none): ${stillDeferred.join(', ')}`,
693
807
  )
694
808
  }
809
+ // Last, where a long log's reader lands: each failure's own last lines.
810
+ for (const line of failureRecap()) log.status(line)
695
811
 
696
812
  // Optional artifacts. Errors are surfaced to the user but don't
697
813
  // change the run's exit code — the run already happened.
@@ -755,7 +871,16 @@ export async function run(options: RunOptions): Promise<RunSummary> {
755
871
  host: hostContext,
756
872
  withTelemetry: telemetry !== undefined,
757
873
  })
758
- cache.recordRunBundle(records)
874
+ try {
875
+ if (!stoppedBySignal) cache.recordRunBundle(records)
876
+ } catch (err) {
877
+ // History is observability: a full cache disk at the very end must
878
+ // not turn a finished run's verdict into a stack and exit 1 (seen as
879
+ // an unprivileged user on a 2 MiB disk, 2026-09-16). The run said
880
+ // what happened; `vx last` and the flaky list will not know this one.
881
+ const message = err instanceof Error ? err.message : String(err)
882
+ log.status(`[vx] run history not recorded: ${message} — the verdict above stands`)
883
+ }
759
884
  mark('record history')
760
885
  // Hand the per-run summary to the telemetry sinks + drain them. Only
761
886
  // when a sink is active (telemetry !== undefined) — otherwise this
@@ -768,6 +893,7 @@ export async function run(options: RunOptions): Promise<RunSummary> {
768
893
  endedAt: endedAtMs,
769
894
  totalDurationMs: Math.round(totalMs),
770
895
  exitOk: ok,
896
+ abortedCount: list.filter((o) => o.status === 'aborted' && !isGroupTask(o.node)).length,
771
897
  })
772
898
  telemetry.emitSummary(summary)
773
899
  await telemetry.flush()
@@ -792,16 +918,16 @@ export async function run(options: RunOptions): Promise<RunSummary> {
792
918
  // write with nothing but a warning. The seam's contract is that a
793
919
  // cache layer stays usable until the run's uploads have settled.
794
920
  await prefetchDone
795
- // Every deferred save settles first: the upload drain below carries
796
- // the entries the saves queued, and the snapshot loop after it reads
797
- // what the saves pushed.
798
- await saveLane.drain()
799
- mark('save lane')
921
+ // Every deferred save has settled by here: the scheduler finishes a
922
+ // task only once its `settledOf` promise (the lane's `landed`) has,
923
+ // so `runGraph` resolved with every save in the cache. A second
924
+ // drain of the lane stood here until item 634; deleting it changed
925
+ // nothing the suite could see, because this gate is the rule.
800
926
  await cache.drainUploads?.()
801
927
  // The miss path's output-directory snapshots, taken now that the
802
928
  // directories are old enough for the snapshot's racy window (see
803
929
  // miss-save.ts). A few at a time: each is an lstat + readdir per
804
- // prefix and one index transaction.
930
+ // prefix; the rows land together at close, in one transaction.
805
931
  for (let i = 0; i < outputDirSnapshots.length; i += 32) {
806
932
  await Promise.all(
807
933
  outputDirSnapshots
@@ -810,8 +936,11 @@ export async function run(options: RunOptions): Promise<RunSummary> {
810
936
  )
811
937
  }
812
938
  mark('output dir snapshots')
939
+ // Not on a stopped run: one stopped while it waited on another run's
940
+ // lock never held it, and its prune evicted under that run (item 858).
941
+ if (!stopRun.signal.aborted) await applyCacheRetention(prepared, log)
813
942
  await teardownPlugins(prepared.plugins, (m) => log.status(m))
814
- closeCache()
943
+ await closeCache()
815
944
  mark('close')
816
945
  printTimings()
817
946
 
@@ -841,29 +970,51 @@ export async function run(options: RunOptions): Promise<RunSummary> {
841
970
  // run that never returned. One status line names the server that
842
971
  // ended the session and its code — the summary above said `success`
843
972
  // for it, and an exit 1 with no word about why is a mystery in a log.
973
+ if (keepAlive.children.length > 0 && hold) {
974
+ const held = keepAlive.children
975
+ return {
976
+ ok,
977
+ outcomes: list,
978
+ persistent: {
979
+ ids: keepAlive.nodes.map((n) => n.id),
980
+ stop: (signal) => terminateChildren(() => held, signal),
981
+ },
982
+ }
983
+ }
844
984
  if (keepAlive.children.length > 0) {
845
985
  const first = await Promise.race(
846
986
  keepAlive.children.map((c, i) => c.exited.then((code) => ({ code, i }))),
847
987
  )
848
988
  const node = keepAlive.nodes[first.i]!
849
989
  const others = keepAlive.nodes.length - 1
850
- log.status(
851
- `vx: ${node.id} exited with code ${first.code}` +
852
- (others > 0
853
- ? `; stopping ${others} other persistent task${others === 1 ? '' : 's'}`
854
- : ''),
855
- )
990
+ // Not when the run was stopped: the server ended because the user
991
+ // stopped it, and "exited with code 130" read as a crash after every
992
+ // Ctrl-C once a stop let run() finish its own path (item 852).
993
+ if (!stopRun.signal.aborted) {
994
+ log.status(
995
+ `vx: ${node.id} exited with code ${first.code}` +
996
+ (others > 0
997
+ ? `; stopping ${others} other persistent task${others === 1 ? '' : 's'}`
998
+ : ''),
999
+ )
1000
+ }
856
1001
  await terminateChildren(() => keepAlive.children)
857
1002
  return { ok: ok && first.code === 0, outcomes: list }
858
1003
  }
859
1004
 
860
1005
  return { ok, outcomes: list }
861
1006
  } finally {
862
- // Idempotent; also reached on mid-run throws, so a crashed cycle
863
- // can't leave a live status-line ticker behind.
1007
+ // Idempotent. The status-line ticker starts in runStart, and every
1008
+ // call between it and the success path's runEnd is crash-isolated
1009
+ // today (admit, the observers, execute's rejection arm, the
1010
+ // short-circuit, the prefetch, killTree; the bus swallows a
1011
+ // subscriber's throw, so even a logger's own runStart cannot reach
1012
+ // here), so deleting this line reddens nothing (item 635). It is the
1013
+ // floor for a throw a later change adds, not a path a row can drive.
864
1014
  log.runEnd?.()
865
1015
  signals.remove()
866
- options.signal?.removeEventListener('abort', onAbort)
1016
+ options.signal?.removeEventListener('abort', forwardAbort)
1017
+ stopRun.signal.removeEventListener('abort', onAbort)
867
1018
  // Plugins installed at the top of run() get their bus subscriptions
868
1019
  // released here. Idempotent; safe even if installPlugins threw.
869
1020
  disposePlugins?.()
@@ -874,10 +1025,50 @@ export async function run(options: RunOptions): Promise<RunSummary> {
874
1025
  // NOT drained here: they hold no database state, and awaiting a
875
1026
  // wedged remote would turn a failing run into a hanging one.
876
1027
  try {
877
- closeCache()
1028
+ await closeCache()
878
1029
  } catch {
879
1030
  // teardown must not throw on the way out
880
1031
  }
1032
+ await aborting?.catch(() => {})
1033
+ // Last: a process signal waits for this before it exits.
1034
+ leftRun()
1035
+ }
1036
+ }
1037
+
1038
+ /**
1039
+ * The workspace's `cacheRetention`, after every save and upload of this run
1040
+ * has landed (so nothing this run wrote is mid-flight) and before the cache
1041
+ * closes. Housekeeping, not the run's work: a failure is one warning, never
1042
+ * a failed run. Declared nowhere → one property read.
1043
+ */
1044
+ async function applyCacheRetention(prepared: PreparedRun, log: Logger): Promise<void> {
1045
+ const retention = prepared.workspaceConfig?.cacheRetention
1046
+ if (retention === undefined) return
1047
+ // Validated at load (`validateRetention`), so both parse.
1048
+ const maxAgeMs =
1049
+ retention.olderThan === undefined ? undefined : parseDuration(retention.olderThan)!
1050
+ const maxBytes = retention.maxSize === undefined ? undefined : parseSize(retention.maxSize)!
1051
+ try {
1052
+ const result = await prepared.localCache.evictIfDue({
1053
+ ...(maxAgeMs !== undefined ? { maxAgeMs } : {}),
1054
+ ...(maxBytes !== undefined ? { maxBytes } : {}),
1055
+ })
1056
+ if (result !== null && (result.evicted > 0 || result.orphans > 0)) {
1057
+ const said: string[] = []
1058
+ if (result.evicted > 0) {
1059
+ said.push(
1060
+ `evicted ${result.evicted} entr${result.evicted === 1 ? 'y' : 'ies'} (${formatBytes(result.bytesFreed)})`,
1061
+ )
1062
+ }
1063
+ if (result.orphans > 0) {
1064
+ said.push(
1065
+ `reaped ${result.orphans} orphaned artifact${result.orphans === 1 ? '' : 's'} (${formatBytes(result.orphanBytes)})`,
1066
+ )
1067
+ }
1068
+ log.status(`vx: cache retention ${said.join(', ')}`)
1069
+ }
1070
+ } catch (err) {
1071
+ log.status(`vx: cache retention skipped: ${err instanceof Error ? err.message : String(err)}`)
881
1072
  }
882
1073
  }
883
1074
 
@@ -900,7 +1091,10 @@ export async function planRun(options: RunOptions): Promise<RunPlan> {
900
1091
  return { tasks: [], unresolvedTasks: prepared.unresolvedTasks }
901
1092
  }
902
1093
  if (prepared.empty !== null) return { tasks: [] }
903
- return await plan({
1094
+ // Its own mark: a dry run's plan (every task's hash, the cache lookups,
1095
+ // the history p50s) was booked under `close`, the next mark, and read
1096
+ // as 105 ms of closing a cache at 1,000 projects (item 601).
1097
+ const planned = await plan({
904
1098
  nodes: prepared.nodes,
905
1099
  workspaceRoot: prepared.workspaceRoot,
906
1100
  workspaceFingerprint: prepared.workspaceFingerprint,
@@ -926,6 +1120,8 @@ export async function planRun(options: RunOptions): Promise<RunPlan> {
926
1120
  // cache capability, so plan mode gains no new class of side effect.
927
1121
  ...(await planExecutorOf(prepared, log, options.download ?? 'all')),
928
1122
  })
1123
+ mark('plan')
1124
+ return planned
929
1125
  } finally {
930
1126
  prepared.cache.close()
931
1127
  // The same table a run prints: a dry run is how the prepare stages
@@ -939,12 +1135,23 @@ export async function planRun(options: RunOptions): Promise<RunPlan> {
939
1135
  /**
940
1136
  * The first-run case, told apart from a typo: no package in the workspace
941
1137
  * has a `vx.config.*` at all, so no name could have resolved. Reached only
942
- * on the error path.
1138
+ * on the error path. A workspace whose package globs match NOTHING is
1139
+ * told that instead — `vx init` would find no package to write for.
943
1140
  */
944
- function initHint(prepared: { anyProjectConfig: boolean }): string {
945
- return prepared.anyProjectConfig
946
- ? ''
947
- : ' No package declares a vx.config — run `vx init` to write one per package from its package.json scripts.'
1141
+ async function initHint(prepared: {
1142
+ anyProjectConfig: boolean
1143
+ workspaceProjectCount: number
1144
+ workspaceRoot: string
1145
+ }): Promise<string> {
1146
+ if (prepared.anyProjectConfig) return ''
1147
+ if (prepared.workspaceProjectCount === 0) {
1148
+ return " No package matched the workspace's package globs — check `workspaces` in package.json (or pnpm-workspace.yaml)."
1149
+ }
1150
+ // Single-project mode with packages the root's missing `workspaces`
1151
+ // never reaches: `vx init` would find no scripts either (item 248).
1152
+ const unreached = await unreachedPackages(await loadWorkspace(prepared.workspaceRoot))
1153
+ if (unreached.length > 0) return ` ${unreachedHint(unreached)}`
1154
+ return ' No package declares a vx.config — run `vx init` to write one per package from its package.json scripts.'
948
1155
  }
949
1156
 
950
1157
  /**