@vzn/vx 0.0.638 → 0.0.639

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/docs/caching.md +44 -15
  2. package/docs/cli.md +168 -345
  3. package/docs/execution.md +18 -43
  4. package/docs/features.md +7 -10
  5. package/docs/patterns.md +5 -7
  6. package/docs/schema.md +47 -49
  7. package/package.json +5 -5
  8. package/schemas/last.json +7 -4
  9. package/skills/vx/SKILL.md +6 -4
  10. package/src/cache/cache.ts +40 -10
  11. package/src/cache/git-inputs.ts +6 -0
  12. package/src/cache/index.ts +2 -2
  13. package/src/cache/layer.ts +25 -11
  14. package/src/cache/layered-cache.ts +108 -315
  15. package/src/cache/plugin-cache.ts +2 -10
  16. package/src/cache/remote-lane.ts +383 -0
  17. package/src/cache/run-history.ts +24 -8
  18. package/src/cache/run-store.ts +4 -4
  19. package/src/cache/schema.ts +6 -4
  20. package/src/cache/scoped-cache.ts +7 -0
  21. package/src/cli/completions.ts +2 -2
  22. package/src/cli/foreign-flags.ts +10 -22
  23. package/src/cli/help.ts +6 -31
  24. package/src/cli/info.ts +2 -1
  25. package/src/cli/last.ts +6 -3
  26. package/src/cli/plan-format.ts +5 -58
  27. package/src/cli/plugin-commands.ts +2 -2
  28. package/src/cli/run.ts +23 -185
  29. package/src/cli/show.ts +6 -0
  30. package/src/cli/upgrade.ts +123 -39
  31. package/src/cli/watch-cycle.ts +2 -1
  32. package/src/cli/watch.ts +29 -14
  33. package/src/cli/why.ts +4 -4
  34. package/src/config.ts +0 -6
  35. package/src/exec/executor.ts +1 -1
  36. package/src/exec/sandbox-runtime.ts +1 -1
  37. package/src/exec/sandbox-violations.ts +3 -1
  38. package/src/graph/index.ts +9 -1
  39. package/src/graph/priorities.ts +1 -1
  40. package/src/graph/scheduler.ts +4 -16
  41. package/src/graph/task-graph.ts +87 -17
  42. package/src/graph/task-status.ts +95 -0
  43. package/src/{orchestrator → history}/failure-mode.ts +1 -1
  44. package/src/{orchestrator → history}/history.ts +1 -1
  45. package/src/history/index.ts +48 -0
  46. package/src/{orchestrator → history}/metrics.ts +7 -20
  47. package/src/{orchestrator → history}/miss-reason.ts +1 -1
  48. package/src/index.ts +10 -28
  49. package/src/orchestrator/doctor.ts +1 -1
  50. package/src/orchestrator/execute-args.ts +80 -0
  51. package/src/orchestrator/execute-task.ts +21 -386
  52. package/src/orchestrator/fingerprint-watch.ts +2 -2
  53. package/src/orchestrator/index.ts +1 -73
  54. package/src/orchestrator/miss-save.ts +33 -3
  55. package/src/orchestrator/options.ts +1 -36
  56. package/src/orchestrator/output-host.ts +4 -3
  57. package/src/orchestrator/persistent-task.ts +321 -0
  58. package/src/orchestrator/persistent.ts +130 -3
  59. package/src/orchestrator/placement.ts +3 -3
  60. package/src/orchestrator/plan.ts +18 -5
  61. package/src/orchestrator/plugin-host.ts +67 -47
  62. package/src/orchestrator/plugin.ts +6 -6
  63. package/src/orchestrator/prepare.ts +28 -29
  64. package/src/orchestrator/remote-prefetch.ts +14 -1
  65. package/src/orchestrator/run-artifacts.ts +5 -177
  66. package/src/orchestrator/run-context.ts +55 -0
  67. package/src/orchestrator/run-failures.ts +15 -3
  68. package/src/orchestrator/run-records.ts +35 -3
  69. package/src/orchestrator/run-report.ts +81 -179
  70. package/src/orchestrator/run-settings.ts +1 -16
  71. package/src/orchestrator/run-stop.ts +153 -0
  72. package/src/orchestrator/run.ts +260 -601
  73. package/src/orchestrator/sandbox-fix.ts +0 -16
  74. package/src/orchestrator/signals.ts +14 -6
  75. package/src/orchestrator/stable-keys.ts +107 -3
  76. package/src/orchestrator/steps.ts +50 -0
  77. package/src/orchestrator/task-hash.ts +3 -16
  78. package/src/orchestrator/task-log-buffer.ts +1 -1
  79. package/src/orchestrator/task-process.ts +94 -0
  80. package/src/orchestrator/telemetry-host.ts +1 -1
  81. package/src/orchestrator/telemetry.ts +5 -82
  82. package/src/{orchestrator → report}/events.ts +41 -8
  83. package/src/{orchestrator → report}/forecast.ts +35 -2
  84. package/src/{orchestrator → report}/framed-output.ts +39 -33
  85. package/src/report/index.ts +36 -0
  86. package/src/{orchestrator → report}/logger.ts +158 -20
  87. package/src/{orchestrator → report}/status-line.ts +84 -7
  88. package/src/{orchestrator → report}/summary.ts +126 -35
  89. package/src/report/tally.ts +83 -0
  90. package/src/settings.ts +1 -22
  91. package/src/util/colors.ts +44 -4
  92. package/src/util/git-ignored.ts +143 -0
  93. package/src/util/index.ts +19 -4
  94. package/src/util/report.ts +13 -5
  95. package/src/util/settle.ts +14 -39
  96. package/src/util/tail.ts +1 -1
  97. package/src/util/ui.ts +17 -6
  98. package/schemas/summary.json +0 -133
  99. package/src/cache/chained-cache.ts +0 -280
  100. package/src/orchestrator/tally.ts +0 -112
  101. /package/src/{orchestrator → report}/path-links.ts +0 -0
  102. /package/src/{orchestrator → report}/plain-output.ts +0 -0
  103. /package/src/{orchestrator → report}/terminal-signals.ts +0 -0
@@ -12,10 +12,6 @@ import {
12
12
  type CachePolicy,
13
13
  cachesNothing,
14
14
  formatCachePolicy,
15
- packRunOutputs,
16
- resolveOutputs,
17
- resolveWorkspaceOutputs,
18
- runEntryName,
19
15
  stopRuntimeProbes,
20
16
  } from '../cache/index.js'
21
17
  import { VERSION } from '../version.js'
@@ -40,7 +36,6 @@ import {
40
36
  import {
41
37
  mark,
42
38
  stageTimes,
43
- killGraceMs,
44
39
  MAX_TIMEOUT_MS,
45
40
  printTimings,
46
41
  ulid,
@@ -49,60 +44,64 @@ import {
49
44
  nearest,
50
45
  UserError,
51
46
  machineParallelism,
52
- teardownTimeoutMs,
53
47
  secretMask,
54
48
  isFsRefusal,
55
49
  isUserError,
56
50
  detectColors,
57
- relPosix,
58
51
  type ColorSupport,
59
52
  } from '../util/index.js'
60
53
  import { keyedProjects } from './keyed-projects.js'
61
54
  import { isDefaultBuild } from './projects.js'
62
55
  import { commandWriteReach, prepareSandbox } from './sandbox-request.js'
63
- import type { OutputDirSnapshot, SaveFacts } from './miss-save.js'
56
+ import { recordOutputDirSnapshots, type OutputDirSnapshot, type SaveFacts } from './miss-save.js'
64
57
  import { admitTasks, taintTracker } from './admission.js'
65
58
  import { unkeyedGroupOutcome, type ExecuteArgs } from './execute-task.js'
59
+ import { executeArgsOf, type RunScope } from './execute-args.js'
66
60
  import { excludedTaint } from './excluded-keys.js'
67
61
  import { keyUpstream } from './upstream.js'
68
- import { busLogger, createEventBus, terminalSubscriber, type EventBus } from './events.js'
69
- import { installPlugins, PluginSetupError, type RunReport, type RunReportTask } from './plugin.js'
70
- import { decodeOutputLog } from './output-log.js'
71
- import { buildAdmission, executorLabel, resolveExecutors, teardownPlugins } from './plugin-host.js'
62
+ import {
63
+ busLogger,
64
+ createEventBus,
65
+ createForecast,
66
+ defaultLogger,
67
+ type DefaultLogger,
68
+ type EventBus,
69
+ formatOutcomeTable,
70
+ formatPersistentList,
71
+ formatRunSummary,
72
+ type Logger,
73
+ plainOutput,
74
+ resolveOutputView,
75
+ terminalSubscriber,
76
+ warnOf,
77
+ } from '../report/index.js'
78
+ import { installPlugins, PluginSetupError, type RunReport } from './plugin.js'
79
+ import { keepRunBytes } from './run-report.js'
80
+ import { buildAdmission, executorDemand, resolveExecutors, teardownPlugins } from './plugin-host.js'
81
+ import { createSteps } from './steps.js'
72
82
  import { subscribeTelemetry, type TelemetryHandle } from './telemetry-host.js'
73
83
  import { claimOutput, guardedOutput } from './output-host.js'
74
- import { assembleRunSummary, isPassStatus } from './telemetry.js'
84
+ import { assembleRunSummary } from './telemetry.js'
85
+ import { isPassStatus } from '../graph/index.js'
75
86
  import type { RunContextRecord } from './telemetry.js'
76
- import { defaultLogger, type DefaultLogger, resolveOutputView, type Logger } from './logger.js'
77
- import { plainOutput } from './plain-output.js'
78
- import { formatPersistentList } from './framed-output.js'
79
- import { createForecast } from './forecast.js'
80
- import { LocalHistoryProvider } from './history.js'
81
87
  import { plan, type RunPlan } from './plan.js'
82
88
  import type { AffectedReason } from './affected-tasks.js'
83
89
  import { prepareRun, type PreparedRun } from './prepare.js'
84
90
  import { acquireRunLock } from './run-lock.js'
85
- import {
86
- forwardedSignal,
87
- type StopSignal,
88
- forwardSignals,
89
- SIGNAL_SHUTDOWN_GRACE_MS,
90
- terminateChildren,
91
- } from './signals.js'
91
+ import { terminateChildren } from './signals.js'
92
+ import { runStop } from './run-stop.js'
92
93
  import {
93
94
  hasPooledExecutor,
94
95
  placeTasks,
95
96
  planExecutorOf,
96
97
  poolOfPlacement,
97
98
  terminalHolders,
98
- UNPLACED_EXECUTOR,
99
99
  locallyPlaced,
100
100
  } from './placement.js'
101
101
  import {
102
- captureDefaultBranch,
102
+ telemetryRunContext,
103
103
  captureGitContext,
104
104
  captureHostContext,
105
- captureWorkspaceIdentity,
106
105
  detectCi,
107
106
  } from './run-context.js'
108
107
  import { startRemotePrefetch } from './remote-prefetch.js'
@@ -114,26 +113,25 @@ import {
114
113
  import { deriveStableKeys, probesAfterWrites } from './stable-keys.js'
115
114
 
116
115
  import { writeRunFailures } from './run-failures.js'
117
- import { assembleRunRecords } from './run-records.js'
116
+ import { assembleRunRecords, recordLateRemoteErrors } from './run-records.js'
118
117
  import {
119
118
  hasEnded,
120
- selectKeepAlive,
121
- serverConfig,
122
- shutdownPersistent,
119
+ disposeServers,
120
+ failServer,
121
+ holdServers,
122
+ sessionEnd,
123
123
  takeHeldServers,
124
124
  } from './persistent.js'
125
- import {
126
- runSummaryJson,
127
- writeRunProfile,
128
- writeRunSummary,
129
- type RunSummaryJson,
130
- } from './run-artifacts.js'
125
+ import { writeRunProfile } from './run-artifacts.js'
131
126
  import { createSaveLane } from './save-lane.js'
132
127
  import { commandWindows } from './command-windows.js'
133
- import { formatOutcomeTable, formatRunSummary } from './summary.js'
134
- import { detectFlaky, type FlakyCandidate, type FlakyFinding } from './failure-mode.js'
135
- import { createMissExplainer } from './miss-reason.js'
136
- import type { HeldServer, RunOptions, RunSummary } from './options.js'
128
+ import {
129
+ createMissExplainer,
130
+ detectFlaky,
131
+ type FlakyCandidate,
132
+ LocalHistoryProvider,
133
+ } from '../history/index.js'
134
+ import type { RunOptions, RunSummary } from './options.js'
137
135
  import { withRunSettings } from './run-settings.js'
138
136
  import { resolveSetting } from '../settings.js'
139
137
 
@@ -146,21 +144,6 @@ const emptyShortCircuit = (): ShortCircuit => ({
146
144
  groupKeys: new Map(),
147
145
  })
148
146
 
149
- /** The files `node`'s declared outputs match on disk now, workspace-relative and sorted. */
150
- async function outputsLeft(
151
- node: TaskNode,
152
- workspaceRoot: string,
153
- nestedProjectDirs: string[],
154
- ): Promise<string[]> {
155
- const declared = node.config.cache?.outputs
156
- if (declared === undefined) return []
157
- const [own, ws] = await Promise.all([
158
- resolveOutputs({ projectDir: node.projectDir, outputs: declared.files, nestedProjectDirs }),
159
- resolveWorkspaceOutputs({ workspaceRoot, outputs: declared.workspaceFiles ?? [] }),
160
- ])
161
- return [...new Set([...own, ...ws].map((p) => relPosix(workspaceRoot, p)))].sort()
162
- }
163
-
164
147
  /**
165
148
  * The policy that will actually govern the run. The remote axes are inert
166
149
  * without a remote layer to serve them: a `remote:w` policy over a bare
@@ -210,17 +193,6 @@ function maskInvocation(command: string): string {
210
193
  return secretMask([process.env])?.mask(command) ?? command
211
194
  }
212
195
 
213
- /**
214
- * A run's `--tag`s, stored on its history row and handed to telemetry
215
- * (`vx.tag.<key>`): a tag that carries a secret is masked as the
216
- * invocation line that spells it is (L-38).
217
- */
218
- function maskTags(tags: Readonly<Record<string, string>>): Record<string, string> {
219
- const mask = secretMask([process.env])
220
- if (mask === null) return { ...tags }
221
- return Object.fromEntries(Object.entries(tags).map(([k, v]) => [mask.mask(k), mask.mask(v)]))
222
- }
223
-
224
196
  /**
225
197
  * The command line a run hands to its telemetry sinks, with what follows
226
198
  * `--` counted, not quoted. Those are the task's arguments,
@@ -295,15 +267,11 @@ function refuseRunOptions(options: RunOptions): void {
295
267
  refuse('concurrency', concurrency, 'a positive integer')
296
268
  if (retries !== undefined && !(Number.isInteger(retries) && retries >= 0))
297
269
  refuse('retries', retries, 'a non-negative integer')
298
- for (const [name, ms] of [
299
- ['timeout', timeout],
300
- ['killGrace', options.killGrace],
301
- ['teardownTimeout', options.teardownTimeout],
302
- ['readyNotice', options.readyNotice],
303
- ] as const) {
304
- if (ms !== undefined && !(Number.isInteger(ms) && ms > 0 && ms <= MAX_TIMEOUT_MS))
305
- refuse(name, ms, `a positive integer of ms, at most ${MAX_TIMEOUT_MS}`)
306
- }
270
+ if (
271
+ timeout !== undefined &&
272
+ !(Number.isInteger(timeout) && timeout > 0 && timeout <= MAX_TIMEOUT_MS)
273
+ )
274
+ refuse('timeout', timeout, `a positive integer of ms, at most ${MAX_TIMEOUT_MS}`)
307
275
  }
308
276
 
309
277
  export async function run(options: RunOptions): Promise<RunSummary> {
@@ -324,17 +292,12 @@ export async function run(options: RunOptions): Promise<RunSummary> {
324
292
  // See docs/design/event-stream-2026-06.md.
325
293
  const terminal =
326
294
  options.log === undefined
327
- ? defaultLogger(
328
- colors,
329
- resolveOutputView(options),
330
- options.json === true ? process.stderr : process.stdout,
331
- {
332
- coalesce: true,
333
- ...(options.tty === true ? { tty: true } : {}),
334
- ...(options.forwardArgs !== undefined ? { forwardArgs: options.forwardArgs } : {}),
335
- ...(options.footer === false ? { footer: false } : {}),
336
- },
337
- )
295
+ ? defaultLogger(colors, resolveOutputView(options), process.stdout, {
296
+ coalesce: true,
297
+ ...(options.tty === true ? { tty: true } : {}),
298
+ ...(options.forwardArgs !== undefined ? { forwardArgs: options.forwardArgs } : {}),
299
+ ...(options.footer === false ? { footer: false } : {}),
300
+ })
338
301
  : null
339
302
  const sink = options.log ?? terminal!
340
303
  // An injected bus already has surfaces subscribed; we add the terminal
@@ -364,7 +327,10 @@ export async function run(options: RunOptions): Promise<RunSummary> {
364
327
  }
365
328
  let held = false
366
329
  try {
367
- const summary = await runOnBus(options, bus, colors, terminal, takeOutput)
330
+ // vx's own steps are timed only where something renders them.
331
+ const showsSteps =
332
+ terminal !== null ? terminal.showsSteps : options.log?.stepStart !== undefined
333
+ const summary = await runOnBus(options, bus, colors, terminal, showsSteps, takeOutput)
368
334
  if (summary.persistent === undefined) return summary
369
335
  // Servers handed back still running still write: `vx watch dev` printed
370
336
  // none of its server's log while it idled, the renderer gone with this
@@ -388,9 +354,11 @@ async function runOnBus(
388
354
  bus: EventBus,
389
355
  colors: ColorSupport,
390
356
  terminal: DefaultLogger | null,
357
+ showsSteps: boolean,
391
358
  takeOutput?: (claimed: { logger: Logger; owner: string }) => void,
392
359
  ): Promise<RunSummary> {
393
360
  const log = busLogger(bus)
361
+ const { step, stepSync } = createSteps(log, showsSteps)
394
362
  // A server's output goes through a cell of its own: a later watch cycle
395
363
  // that keeps the server points it at that cycle's bus (`RunOptions.keep`).
396
364
  const serverOuts = new Map<string, { bus: EventBus }>()
@@ -404,7 +372,7 @@ async function runOnBus(
404
372
  }
405
373
  }
406
374
 
407
- const prepared = await prepareRun(options, log)
375
+ const prepared = await step('vx:load', () => prepareRun(options, log))
408
376
  options = withRunSettings(options, prepared.workspaceConfig)
409
377
  mark('plugin stages')
410
378
  // Every exit from here tears the plugins down: their capability
@@ -418,7 +386,7 @@ async function runOnBus(
418
386
  if (tornDown) return
419
387
  tornDown = true
420
388
  const plugins = prepared.plugins.filter((p) => p !== except)
421
- await teardownPlugins(plugins, (m) => log.status(m), report)
389
+ await teardownPlugins(plugins, warnOf(log), report, step)
422
390
  }
423
391
  // A task whose command re-enters `vx run` in the workspace running it
424
392
  // is refused. When the inner run reaches this task again it forks a run
@@ -489,10 +457,11 @@ async function runOnBus(
489
457
  bus,
490
458
  workspaceRoot: prepared.workspaceRoot,
491
459
  cacheDir: prepared.cacheDir,
492
- warn: (m) => log.status(m),
460
+ warn: warnOf(log),
493
461
  cache: prepared.cacheFor,
494
462
  ci: ciContext.ci,
495
463
  branch: prepared.branch(),
464
+ step,
496
465
  })
497
466
  } catch (err) {
498
467
  disposePlugins?.()
@@ -545,7 +514,7 @@ async function runOnBus(
545
514
  resolveExecutors(prepared.plugins, {
546
515
  workspaceRoot: prepared.workspaceRoot,
547
516
  cacheDir: prepared.cacheDir,
548
- warn: (m: string) => log.status(m),
517
+ warn: warnOf(log),
549
518
  concurrency,
550
519
  }),
551
520
  )
@@ -569,51 +538,11 @@ async function runOnBus(
569
538
  : nodes.get(id)?.config.cache === undefined
570
539
  ? 'no remote executor accepted it — the task declares no `cache` block, so there is no input set to ship'
571
540
  : 'no remote executor accepted it'
572
- log.status(`[vx] ${id}: exec.remote is 'only' and ${why} — nothing ran`)
573
- }
574
- }
575
- // Demand: what each executor still has left to run. An executor that
576
- // provisions per task (a container, an allocation, a pod) otherwise has to
577
- // guess from `capacity` alone and hold everything until teardown. Built
578
- // ONLY for executors that asked — with none declared there is no map, no
579
- // set, and no call in the completion path.
580
- const demandOf = new Map<TaskExecutor, Set<string>>()
581
- for (const [id, executor] of placements.executors) {
582
- if (executor.demand === undefined) continue
583
- let remaining = demandOf.get(executor)
584
- if (remaining === undefined) demandOf.set(executor, (remaining = new Set()))
585
- remaining.add(id)
586
- }
587
- // The SAME set is handed back every time, narrowed in place: a consumer
588
- // reading it later sees the truth rather than a stale snapshot. A hint,
589
- // as `admit` is: one that throws is named once and asked no more, where
590
- // it surfaced as a bare stack, or mid-run from the completion path
591
- // (item 1022).
592
- const tellDemand = (executor: TaskExecutor, remaining: ReadonlySet<string>): void => {
593
- const failed = (err: unknown): void => {
594
- if (!demandOf.delete(executor)) return
595
- const m = err instanceof Error ? err.message : String(err)
596
- log.status(`[vx] ${executorLabel(executor)} failed in demand: ${m}; not asked again this run`)
597
- }
598
- try {
599
- // An `async demand` that rejects did so where no one listened: an
600
- // unhandled rejection, a stack, the run killed. Its rejection is a
601
- // throw like any other.
602
- const ret: unknown = executor.demand!(remaining)
603
- if (ret instanceof Promise) ret.catch(failed)
604
- } catch (err) {
605
- failed(err)
541
+ warnOf(log)(`[vx] ${id}: exec.remote is 'only' and ${why} — nothing ran`)
606
542
  }
607
543
  }
608
- for (const [executor, remaining] of demandOf) tellDemand(executor, remaining)
609
- const narrowDemand =
610
- demandOf.size === 0
611
- ? (): void => {}
612
- : (id: string): void => {
613
- for (const [executor, remaining] of demandOf) {
614
- if (remaining.delete(id)) tellDemand(executor, remaining)
615
- }
616
- }
544
+ // Demand: what each executor still has left to run (`executorDemand`).
545
+ const narrowDemand = executorDemand(placements.executors, warnOf(log))
617
546
 
618
547
  // `cache.download` (default `all`) decides ONCE per task, here, whether a
619
548
  // remote execution's outputs come home.
@@ -640,88 +569,19 @@ async function runOnBus(
640
569
  | boolean
641
570
  | undefined
642
571
 
643
- // Run-scoped registries of live subprocesses:
644
- // - `liveChildren`: in-flight children. The runner adds/removes
645
- // each child around its spawn (persistent children stay until
646
- // they exit).
647
- // - `persistentRegistry`: ready persistent tasks (dev servers,
648
- // watchers). executeTask spawns them but does NOT await their
649
- // exit; ownership moves here so the orchestrator can SIGTERM
650
- // them once the rest of the graph finishes.
651
- //
652
- // A SIGINT/SIGTERM mid-run forwards that signal to everything in both
653
- // (`signals.ts`); the handlers are removed in the finally below so
654
- // repeated run() calls (test suites) never stack listeners.
655
- const liveChildren = new Set<ReturnType<typeof Bun.spawn>>()
656
- // A server that ends on its own while the graph still runs is said at
657
- // once: its dependants were failing against it, and the end of the run
658
- // was the first word of it (C-69). Not once the graph is done (the end
659
- // of the run and the keep-alive wait say it) nor under a stop.
660
- let graphDone = false
661
- const persistentRegistry = new (class extends Map<string, ReturnType<typeof Bun.spawn>> {
662
- override set(id: string, child: ReturnType<typeof Bun.spawn>): this {
663
- super.set(id, child)
664
- void child.exited.then((code) => {
665
- if (code !== 0 && !graphDone && !stopRun.signal.aborted)
666
- log.status(`vx: ${id} exited with code ${code} while the run went on`)
667
- })
668
- return this
669
- }
670
- })()
671
- // One stop for the run: an embedder's `RunOptions.signal` and a process
672
- // signal both abort it. The scheduler stops dispatching (it reads the
673
- // signal), the children are torn down, and run() returns through its own
674
- // end-of-run path. Detached in the finally below.
675
- const stopRun = new AbortController()
676
- const forwardAbort = (): void => stopRun.abort(options.signal?.reason)
677
- if (options.signal?.aborted === true) forwardAbort()
678
- else options.signal?.addEventListener('abort', forwardAbort, { once: true })
679
- // A run a process signal stopped records no history: until item 849 the
680
- // handler exited before any outcome landed, and a Ctrl-C'd run then read
681
- // in `vx last` as FAILED with 0 tasks (item 854). An embedder's abort is
682
- // its own call and keeps its record.
683
- let stoppedBy: StopSignal | undefined
684
- let leftRun = (): void => {}
685
- const runLeft = new Promise<void>((resolve) => {
686
- leftRun = resolve
687
- })
688
- const signals = forwardSignals({
689
- enabled: options.handleSignals ?? true,
572
+ const stop = runStop({
690
573
  log,
691
574
  cache,
692
- liveChildren,
693
- persistentRegistry,
694
- stop: (signal) => {
695
- stoppedBy = signal
696
- stopRun.abort(signal)
697
- },
698
- done: runLeft,
575
+ external: options.signal,
576
+ handleSignals: options.handleSignals ?? true,
699
577
  handsTerminal: holders.size > 0,
700
- // The grace, then one flush and each teardown at their own bound, and
701
- // slack for the summary and the cache close.
702
- boundMs:
703
- killGraceMs(SIGNAL_SHUTDOWN_GRACE_MS) +
704
- teardownTimeoutMs() * (1 + prepared.plugins.filter((p) => p.teardown).length) +
705
- 2_000,
578
+ teardowns: prepared.plugins.filter((p) => p.teardown).length,
579
+ onStop: () => {
580
+ stopRuntimeProbes(hashCache.runtime, hashCache.workspaceRuntime)
581
+ cache.stopUploads?.()
582
+ },
706
583
  })
707
- // Awaited before run() leaves: a task's shell can die of the signal at
708
- // once while a child it backgrounded ignores it and dies only to the
709
- // SIGKILL after the grace, and a run that returned first left that child
710
- // running past vx's exit (turborepo#14043's class).
711
- let aborting: Promise<void> | undefined
712
- // The servers that had ended when the stop landed: those crashed. One that
713
- // ends after it is the stop's own kill, and was named a crash (item 1061).
714
- let endedBeforeStop: ReadonlySet<ReturnType<typeof Bun.spawn>> | undefined
715
- const onAbort = (): void => {
716
- stopRuntimeProbes(hashCache.runtime, hashCache.workspaceRuntime)
717
- endedBeforeStop = new Set([...persistentRegistry.values()].filter(hasEnded))
718
- aborting = terminateChildren(
719
- () => [...liveChildren, ...persistentRegistry.values()],
720
- forwardedSignal(stopRun.signal.reason),
721
- )
722
- }
723
- if (stopRun.signal.aborted) onAbort()
724
- else stopRun.signal.addEventListener('abort', onAbort, { once: true })
584
+ const { liveChildren, persistentRegistry } = stop
725
585
  // The cache handle must be released on EVERY exit path, not just the
726
586
  // happy one: `close()` is also where the run's deferred `accessed_at`
727
587
  // bumps are flushed, so a throw between opening the cache and the
@@ -748,10 +608,12 @@ async function runOnBus(
748
608
  // Taken after the early exits above (nothing they do touches a tree) and
749
609
  // before the schedule: from here on tasks clean, restore and write.
750
610
  const releaseRunLock = await abandoning(() =>
751
- acquireRunLock(prepared.workspaceRoot, {
752
- log: (m) => log.status(m),
753
- signal: stopRun.signal,
754
- }),
611
+ step('vx:run-lock', () =>
612
+ acquireRunLock(prepared.workspaceRoot, {
613
+ log: warnOf(log),
614
+ signal: stop.signal,
615
+ }),
616
+ ),
755
617
  )
756
618
  try {
757
619
  const runStartHrTimeNs = process.hrtime.bigint()
@@ -764,7 +626,7 @@ async function runOnBus(
764
626
  // happened.
765
627
  const policy: CachePolicy = effectiveCachePolicy(prepared.cachePolicy, prepared.hasRemoteLayer)
766
628
  if (options.remoteRequested === true && !prepared.hasRemoteLayer) {
767
- log.status(
629
+ warnOf(log)(
768
630
  'vx: --cache-policy named the remote cache, but no cache plugin in .vx/workspace.* supplies one — the remote axes are off for this run',
769
631
  )
770
632
  }
@@ -807,51 +669,29 @@ async function runOnBus(
807
669
  // but declines still pays — its answer is only knowable by asking.
808
670
  const hasTelemetryPlugin = prepared.plugins.some((p) => p.telemetry !== undefined)
809
671
  if (hasTelemetryPlugin || options.telemetrySinks !== undefined) {
810
- // Workspace identity (telemetry v2), paid only when a telemetry
811
- // consumer can exist — a plain run never reaches here. The remote URL
812
- // is the enumeration's `git var -l`: a spawn here cost ~4 ms.
813
- const wsIdentity = captureWorkspaceIdentity(
814
- workspaceRoot,
815
- gitFilesCache.configValue('remote.origin.url'),
816
- )
817
- runContextRecord = {
672
+ runContextRecord = telemetryRunContext({
818
673
  runId,
819
- vxVersion: VERSION,
820
674
  // An embedder's `command` is redacted as the argv is: it passed a
821
675
  // token after `--` to every sink verbatim (C-67).
822
- // Masked as the stored line is: `--tag key=$DEPLOY_KEY` sits before
676
+ // Masked as the stored line is: a flag's secret value sits before
823
677
  // the `--` the count covers (L-44).
824
678
  command: maskInvocation(
825
679
  invocationCommand(options.command?.split(' ') ?? process.argv.slice(1)),
826
680
  ),
827
- requestedTasks: [...options.tasks],
681
+ requestedTasks: options.tasks,
828
682
  cachePolicy: formatCachePolicy(policy),
829
683
  concurrency,
830
684
  flow: options.flow ?? null,
831
- commitSha: gitContext.commitSha,
832
- branch: gitContext.branch,
833
- defaultBranch: captureDefaultBranch(process.env, workspaceRoot),
834
- dirty: gitContext.dirty,
835
- ci: ciContext.ci,
836
- ciProvider: ciContext.provider,
837
- ...(ciContext.runUrl !== undefined ? { ciRunUrl: ciContext.runUrl } : {}),
838
- ...(ciContext.change !== undefined ? { ciChange: ciContext.change } : {}),
839
- ...(ciContext.pipeline !== undefined ? { ciPipeline: ciContext.pipeline } : {}),
840
- ...(ciContext.job !== undefined ? { ciJob: ciContext.job } : {}),
841
- ...(ciContext.attempt !== undefined ? { ciAttempt: ciContext.attempt } : {}),
842
- host: hostContext.host,
843
- os: hostContext.os,
844
- arch: hostContext.arch,
845
- workspaceId: wsIdentity.id,
846
- workspaceName: wsIdentity.name,
847
- ...(wsIdentity.repository !== undefined ? { repository: wsIdentity.repository } : {}),
848
- ...(wsIdentity.path !== undefined ? { workspacePath: wsIdentity.path } : {}),
849
- tags: maskTags(options.tags ?? {}),
850
- }
685
+ git: gitContext,
686
+ ci: ciContext,
687
+ host: hostContext,
688
+ workspaceRoot,
689
+ originUrl: gitFilesCache.configValue('remote.origin.url'),
690
+ })
851
691
  telemetry = await subscribeTelemetry(
852
692
  prepared.plugins,
853
693
  bus,
854
- { workspaceRoot, cacheDir, warn: (m: string) => log.status(m) },
694
+ { workspaceRoot, cacheDir, warn: warnOf(log) },
855
695
  runContextRecord,
856
696
  options.telemetrySinks,
857
697
  nodes,
@@ -884,7 +724,7 @@ async function runOnBus(
884
724
  // each pack holds an artifact's bytes); a failed save is a miss next
885
725
  // time, said once — the task's work ran.
886
726
  const saveLane = createSaveLane(2 * concurrency, (err) =>
887
- log.status(`[vx] cache save failed: ${err instanceof Error ? err.message : String(err)}`),
727
+ warnOf(log)(`[vx] cache save failed: ${err instanceof Error ? err.message : String(err)}`),
888
728
  )
889
729
  const deferredSaves = new Map<string, Promise<SaveFacts>>()
890
730
  const windows = commandWindows()
@@ -900,9 +740,10 @@ async function runOnBus(
900
740
  // for display. Marks `node.surfaced`; never touches `requested`.
901
741
  markSurfacedDeps(nodes)
902
742
 
903
- // Header counts: unique projects covered by the graph (including
904
- // dependsOn-pulled deps, not just the user-requested set), and the
905
- // total number of real (non-group) task executions. Mirrors the
743
+ // Header counts: unique projects with a real (non-group) task in the
744
+ // graph (including dependsOn-pulled deps, not just the user-requested
745
+ // set; a `^build` placeholder in a project with no build is not one),
746
+ // and the total number of real task executions. Mirrors the
906
747
  // count the end-of-run summary reports under "total". The
907
748
  // requested count drives the focused logger's live-vs-buffered
908
749
  // decision, so surfaced nodes count toward it too — they display
@@ -911,8 +752,8 @@ async function runOnBus(
911
752
  let taskCount = 0
912
753
  let requestedCount = 0
913
754
  for (const node of nodes.values()) {
914
- packagesInScope.add(node.projectName)
915
755
  if (!isGroupTask(node)) {
756
+ packagesInScope.add(node.projectName)
916
757
  taskCount++
917
758
  if (node.requested || node.surfaced === true) requestedCount++
918
759
  }
@@ -920,7 +761,7 @@ async function runOnBus(
920
761
  // Admission over the worker count is a plugin's (`admit`): built once
921
762
  // into the predicate the scheduler asks at every local dispatch. No
922
763
  // plugin answers → undefined → the count-only path, byte for byte.
923
- const admit = buildAdmission(prepared.plugins, nodes, concurrency, (m) => log.status(m))
764
+ const admit = buildAdmission(prepared.plugins, nodes, concurrency, warnOf(log))
924
765
 
925
766
  // Run context for the footer. The top-of-run header is gone — the
926
767
  // banner now lives in the summary, where the eye lands at the end.
@@ -932,7 +773,7 @@ async function runOnBus(
932
773
  local: policy.localRead || policy.localWrite,
933
774
  localWrite: policy.localWrite,
934
775
  remote: remoteCacheEnabled && (policy.remoteRead || policy.remoteWrite),
935
- uploads: prepared.localCache.uploads,
776
+ traffic: prepared.localCache.traffic,
936
777
  notDownloaded: () => deferredOutputs.pending().length,
937
778
  },
938
779
  concurrency,
@@ -941,14 +782,12 @@ async function runOnBus(
941
782
 
942
783
  // Asked only when a plugin declares `output`: a plain run pays nothing.
943
784
  if (takeOutput !== undefined && prepared.plugins.some((p) => p.output !== undefined)) {
944
- const json = options.json === true
945
785
  const claimed = await claimOutput(prepared.plugins, {
946
786
  workspaceRoot,
947
787
  cacheDir,
948
- warn: (m: string) => log.status(m),
949
- isTTY: (json ? process.stderr : process.stdout).isTTY === true,
788
+ warn: warnOf(log),
789
+ isTTY: process.stdout.isTTY === true,
950
790
  ci: ciContext.ci,
951
- json,
952
791
  })
953
792
  if (claimed !== undefined) takeOutput(claimed)
954
793
  }
@@ -984,18 +823,21 @@ async function runOnBus(
984
823
  // unchanged (no upfront key pass, no local probing).
985
824
  let prefetchDone: Promise<void> = Promise.resolve()
986
825
  if (prepared.hasRemoteLayer) {
987
- prefetchDone = startRemotePrefetch({
988
- nodes,
989
- cache,
990
- workspaceRoot,
991
- claimedFiles: fingerprintWatch.names,
992
- forwardArgs: options.forwardArgs,
993
- nestedDirsByProject,
994
- gitFilesCache,
995
- hashCache,
996
- concurrency,
997
- remoteRead: policy.remoteRead,
998
- })
826
+ prefetchDone = step('vx:prefetch', () =>
827
+ startRemotePrefetch({
828
+ nodes,
829
+ cache,
830
+ workspaceRoot,
831
+ claimedFiles: fingerprintWatch.names,
832
+ forwardArgs: options.forwardArgs,
833
+ nestedDirsByProject,
834
+ gitFilesCache,
835
+ hashCache,
836
+ concurrency,
837
+ remoteRead: policy.remoteRead,
838
+ remoteWrite: policy.remoteWrite,
839
+ }),
840
+ )
999
841
  }
1000
842
 
1001
843
  // Local cache short-circuit (default-on). Up-front classify: derive
@@ -1016,17 +858,19 @@ async function runOnBus(
1016
858
  prepared.localCache.holdsNothing()
1017
859
  if (coldLocal) prewarm()
1018
860
  else if (shouldShortCircuit(nodes, policy, cache)) {
1019
- shortCircuit = await startLocalShortCircuit({
1020
- nodes,
1021
- cache,
1022
- workspaceRoot,
1023
- claimedFiles: fingerprintWatch.names,
1024
- forwardArgs: options.forwardArgs,
1025
- nestedDirsByProject,
1026
- gitFilesCache,
1027
- hashCache,
1028
- concurrency,
1029
- })
861
+ shortCircuit = await step('vx:keys', () =>
862
+ startLocalShortCircuit({
863
+ nodes,
864
+ cache,
865
+ workspaceRoot,
866
+ claimedFiles: fingerprintWatch.names,
867
+ forwardArgs: options.forwardArgs,
868
+ nestedDirsByProject,
869
+ gitFilesCache,
870
+ hashCache,
871
+ concurrency,
872
+ }),
873
+ )
1030
874
  // A confirmed miss runs too.
1031
875
  if (sandboxed.some((n) => shortCircuit.preProbed.get(n.id)?.hit === null)) prewarm()
1032
876
  }
@@ -1037,7 +881,7 @@ async function runOnBus(
1037
881
  // admission.ts.
1038
882
  const excluded = excludedTaint(nodes)
1039
883
  if (excluded.unsaved > 0) {
1040
- log.status(
884
+ warnOf(log)(
1041
885
  `[vx] --exclude-dependencies: ${excluded.unsaved} cached task(s) build on a skipped dependency; what they build is not saved`,
1042
886
  )
1043
887
  }
@@ -1066,19 +910,50 @@ async function runOnBus(
1066
910
  // The scheduler's fail-fast stop, as the retry loop of a task already
1067
911
  // in flight sees it.
1068
912
  const failFast = new AbortController()
1069
- // A literal of the fields every task carries, then each optional one
1070
- // assigned only when present: twelve conditional spreads per task cost
1071
- // ~18 ms of object copying on a 1,090-package warm run (X-149).
1072
- const track = telemetry?.track
1073
- const armSandbox = sandboxArmer !== null ? () => sandboxArmer.arm() : undefined
913
+ const scope: RunScope = {
914
+ workspaceRoot,
915
+ cache,
916
+ cachePolicy: policy,
917
+ forwardArgs: options.forwardArgs,
918
+ deferred: deferredOutputs,
919
+ cacheDir,
920
+ runStartHrTimeNs,
921
+ persistentRegistry,
922
+ liveChildren,
923
+ gitFilesCache,
924
+ hashCache,
925
+ probesAfterWrites: lateProbes,
926
+ fingerprintWatch,
927
+ keyedProjects: keyed,
928
+ outputDirSnapshots,
929
+ deferSave: saveLane.defer,
930
+ deferredSaves,
931
+ commandWindows: windows,
932
+ stopSignal: stop.signal,
933
+ failFast: failFast.signal,
934
+ inputStability,
935
+ }
936
+ if (options.retries !== undefined) scope.retries = options.retries
937
+ if (taskTimeoutDefault !== undefined) scope.timeout = taskTimeoutDefault
938
+ if (cleanOutputs !== undefined) scope.cleanOutputs = cleanOutputs
939
+ if (telemetry?.track !== undefined) scope.track = telemetry.track
940
+ if (explainMiss !== undefined) scope.explainMiss = explainMiss
941
+ if (keepRunLog) scope.keepRunLog = true
942
+ if (sandboxArmer !== null) scope.armSandbox = () => sandboxArmer.arm()
943
+ const argsOf = executeArgsOf(scope, {
944
+ log: (node) => (node.config.exec?.persistent !== undefined ? serverLog(node.id) : log),
945
+ placements,
946
+ nestedDirsByProject,
947
+ downloadModeOf: download.modeOf,
948
+ shortCircuit,
949
+ dependedOn,
950
+ holders,
951
+ })
1074
952
  const buildExecuteArgs = (
1075
953
  node: TaskNode,
1076
954
  upstream: TaskOutcome[],
1077
955
  reuseProbe = true,
1078
956
  ): ExecuteArgs => {
1079
- const probe = reuseProbe ? shortCircuit.preProbed.get(node.id) : undefined
1080
- const upfrontKey = shortCircuit.uncachedKeys.get(node.id)
1081
- const upfrontGroupKey = shortCircuit.groupKeys.get(node.id)
1082
957
  let deadServer: string | undefined
1083
958
  if (options.continueMode === 'always') {
1084
959
  for (const d of node.deps) {
@@ -1089,52 +964,7 @@ async function runOnBus(
1089
964
  }
1090
965
  const tainted = taint.judge(node, upstream)
1091
966
  if (tainted) taintedRan.set(node.id, deadServer ?? taint.cause(node, upstream))
1092
- const a: ExecuteArgs = {
1093
- node,
1094
- upstream,
1095
- workspaceRoot,
1096
- cache,
1097
- cachePolicy: policy,
1098
- forwardArgs: options.forwardArgs,
1099
- log: node.config.exec?.persistent !== undefined ? serverLog(node.id) : log,
1100
- executor: placements.executors.get(node.id) ?? UNPLACED_EXECUTOR,
1101
- deferred: deferredOutputs,
1102
- nestedProjectDirs: nestedDirsByProject.get(node.projectName) ?? [],
1103
- cacheDir,
1104
- runStartHrTimeNs,
1105
- persistentRegistry,
1106
- liveChildren,
1107
- gitFilesCache,
1108
- hashCache,
1109
- probesAfterWrites: lateProbes,
1110
- fingerprintWatch,
1111
- keyedProjects: keyed,
1112
- outputDirSnapshots,
1113
- deferSave: saveLane.defer,
1114
- deferredSaves,
1115
- commandWindows: windows,
1116
- stopSignal: stopRun.signal,
1117
- failFast: failFast.signal,
1118
- }
1119
- if (options.retries !== undefined) a.retries = options.retries
1120
- if (taskTimeoutDefault !== undefined) a.timeout = taskTimeoutDefault
1121
- if (options.readyNotice !== undefined) a.readyNotice = options.readyNotice
1122
- a.inputStability = inputStability
1123
- if (cleanOutputs !== undefined) a.cleanOutputs = cleanOutputs
1124
- if (download.modeOf.get(node.id) === 'deferred') a.download = 'deferred'
1125
- if (placements.remoteOnlyNoop.has(node.id)) a.remoteOnlyNoop = true
1126
- if (placements.remoteOnly.has(node.id)) a.remoteOnly = true
1127
- if (track !== undefined) a.track = track
1128
- if (explainMiss !== undefined) a.explainMiss = explainMiss
1129
- if (probe !== undefined) a.preProbed = probe
1130
- if (upfrontKey !== undefined) a.upfrontKey = upfrontKey
1131
- if (upfrontGroupKey !== undefined) a.upfrontGroupKey = upfrontGroupKey
1132
- if (tainted) a.taintedUpstream = true
1133
- if (keepRunLog) a.keepRunLog = true
1134
- if (!dependedOn.has(node.id)) a.noDependants = true
1135
- if (holders.has(node.id)) a.terminal = true
1136
- if (armSandbox !== undefined) a.armSandbox = armSandbox
1137
- return a
967
+ return argsOf(node, upstream, reuseProbe, tainted)
1138
968
  }
1139
969
 
1140
970
  const executeWithDedup = admitTasks({
@@ -1160,13 +990,12 @@ async function runOnBus(
1160
990
  (n) => n.config.exec?.command === undefined || shortCircuit.restoreTier.has(n.id),
1161
991
  )
1162
992
  ? new Map<string, number>()
1163
- : await prepared.priorities()
993
+ : await step('vx:schedule', () => prepared.priorities())
1164
994
  mark('schedule')
1165
995
  // Judged as each task finishes, against the history BEFORE this run's
1166
996
  // rows land, so the row that prints next carries the verdict: nothing
1167
997
  // prints below the footer. A hit or a skip asks nothing; a green miss
1168
998
  // is one probe of the failed-row index.
1169
- const flaky: FlakyFinding[] = []
1170
999
  const historyDb = prepared.localCache.dbHandle()
1171
1000
  const cacheOff = cachesNothing(policy)
1172
1001
  const judgeFlaky = (o: TaskOutcome): void => {
@@ -1175,7 +1004,6 @@ async function runOnBus(
1175
1004
  try {
1176
1005
  const found = detectFlaky(historyDb, candidates)[0]
1177
1006
  if (found === undefined) return
1178
- flaky.push(found)
1179
1007
  o.flaky = { passes: found.passes, failures: found.failures }
1180
1008
  } catch {
1181
1009
  // History is observability: an unreadable one judges nothing.
@@ -1211,7 +1039,7 @@ async function runOnBus(
1211
1039
  ...(options.continueMode !== undefined ? { continueMode: options.continueMode } : {}),
1212
1040
  serverDied,
1213
1041
  onFailFast: () => failFast.abort(),
1214
- signal: stopRun.signal,
1042
+ signal: stop.signal,
1215
1043
  onStart: (node) => {
1216
1044
  log.taskStart?.(node)
1217
1045
  },
@@ -1262,7 +1090,7 @@ async function runOnBus(
1262
1090
  // Empty when the short-circuit didn't fire → byte-identical.
1263
1091
  restoreTier: shortCircuit.restoreTier,
1264
1092
  })
1265
- graphDone = true
1093
+ stop.graphDone()
1266
1094
 
1267
1095
  // Which persistent children outlive the graph, and the bounded SIGTERM
1268
1096
  // of the rest, before the summary prints. Scoped to the real CLI
@@ -1272,49 +1100,19 @@ async function runOnBus(
1272
1100
  // expect run() to return, not block on a server.
1273
1101
  const foreground = options.log === undefined && (options.handleSignals ?? true)
1274
1102
  // An aborted run's children are already being torn down: nothing to hold.
1275
- const hold = options.holdPersistent === true && !stopRun.signal.aborted
1276
- // A run that failed elsewhere holds nothing: `vx run dev --all` with one
1277
- // server that never became ready sat on the others for good, so a
1278
- // script hung, and the Ctrl-C that ended it read 130 over the failure
1279
- // (C-60). A kept server's own crash ends the wait below as before, and
1280
- // `--continue=always` asked to keep going, and does. The watch loop
1281
- // (`holdPersistent`) still holds: a failing test in a cycle stopped
1282
- // the dev server the next change would restart anyway.
1283
- let keepAlive = selectKeepAlive(persistentRegistry, nodes, foreground || hold)
1284
- const failedElsewhere =
1285
- ![...outcomes.values()].every((o) => isPassStatus(o.status)) ||
1286
- [...persistentRegistry.values()].some(
1287
- (c) => hasEnded(c) && c.exitCode !== 0 && !keepAlive.children.includes(c),
1288
- )
1289
- if (failedElsewhere && !hold && options.continueMode !== 'always')
1290
- keepAlive = { nodes: [], children: [] }
1291
- const crashedPersistent = (
1292
- await shutdownPersistent(persistentRegistry, keepAlive.children)
1293
- ).filter((c) => endedBeforeStop?.has(persistentRegistry.get(c.id)!) ?? true)
1103
+ const hold = options.holdPersistent === true && !stop.signal.aborted
1104
+ const { keepAlive, crashed: crashedPersistent } = await disposeServers({
1105
+ registry: persistentRegistry,
1106
+ nodes,
1107
+ outcomes,
1108
+ keep: foreground || hold,
1109
+ hold,
1110
+ continueAlways: options.continueMode === 'always',
1111
+ endedBeforeStop: stop.endedBeforeStop,
1112
+ })
1294
1113
 
1295
1114
  mark('run graph')
1296
1115
 
1297
- // A server that died on its own before the stop failed, whatever its
1298
- // outcome said when it became ready: the footer counted it a success
1299
- // over the run's red exit (item 1071). A kept one that has already
1300
- // died is the same; the keep-alive wait below names it. Its code is its
1301
- // own, or the shell's report of the signal that took it.
1302
- const failServer = (id: string, code: number | string): void => {
1303
- const o = outcomes.get(id)
1304
- if (o === undefined) return
1305
- // In place: the renderer holds this outcome (events carry live refs)
1306
- // and closes the server's output block from it at runEnd, where a
1307
- // copy left it reading `running` under its exit line (WD-10).
1308
- o.status = 'failed'
1309
- o.exitCode = typeof code === 'number' ? code : signalExitCode(code)
1310
- }
1311
- for (const c of crashedPersistent) failServer(c.id, c.code)
1312
- keepAlive.nodes.forEach((n, i) => {
1313
- const child = keepAlive.children[i]!
1314
- if (!hasEnded(child) || child.exitCode === 0) return
1315
- if (endedBeforeStop !== undefined && !endedBeforeStop.has(child)) return
1316
- failServer(n.id, child.exitCode ?? child.signalCode ?? 'unknown')
1317
- })
1318
1116
  // Clear the status line for good before the summary prints.
1319
1117
  log.runEnd?.()
1320
1118
  const list = [...outcomes.values()]
@@ -1340,7 +1138,7 @@ async function runOnBus(
1340
1138
  // 1102).
1341
1139
  for (const c of crashedPersistent) {
1342
1140
  const code = typeof c.code === 'number' ? c.code : signalExitCode(c.code)
1343
- log.status(`vx: ${c.id} exited with code ${code} before the run stopped it`)
1141
+ warnOf(log)(`vx: ${c.id} exited with code ${code} before the run stopped it`)
1344
1142
  }
1345
1143
  // Outputs that never came home are not an error, but a silent `dist/`
1346
1144
  // that is empty-or-stale would be: name every task whose bytes are
@@ -1356,105 +1154,20 @@ async function runOnBus(
1356
1154
  // change the run's exit code — the run already happened.
1357
1155
  // Written again after the keep-alive wait: a kept server's crash or a
1358
1156
  // Ctrl-C there is the process's exit, and the first write said ok.
1359
- // The run's own bytes, kept in a remote that takes its writes (owner
1360
- // 2026-10-15): a failed task's log and the files its outputs left, an
1361
- // uncached task's log. Then the report every plugin's teardown reads.
1362
- const keepRunBytes = async (final: readonly TaskOutcome[]): Promise<RunReport> => {
1363
- const keep = prepared.keepRunEntry
1364
- const tasks: RunReportTask[] = []
1365
- const work: (() => Promise<void>)[] = []
1366
- for (const o of final) {
1367
- if (isGroupTask(o.node)) continue
1368
- const task: { -readonly [K in keyof RunReportTask]: RunReportTask[K] } = {
1369
- id: o.node.id,
1370
- status: o.status,
1371
- durationMs: Math.round(o.durationMs),
1372
- ...(o.hash !== undefined ? { key: o.hash } : {}),
1373
- }
1374
- tasks.push(task)
1375
- if (keep === undefined) continue
1376
- const runLog = o.runLog
1377
- if (runLog !== undefined) {
1378
- work.push(async () => {
1379
- const text = decodeOutputLog(runLog)
1380
- .map((c) => c.text)
1381
- .join('')
1382
- if (text.length === 0) return
1383
- const name = runEntryName(runId, o.node.id, 'log')
1384
- if (await keep(name, new TextEncoder().encode(text))) task.log = name
1385
- })
1386
- }
1387
- if (o.status === 'failed' && o.where === undefined) {
1388
- work.push(async () => {
1389
- const bytes = await packRunOutputs(
1390
- prepared.workspaceRoot,
1391
- await outputsLeft(
1392
- o.node,
1393
- prepared.workspaceRoot,
1394
- prepared.nestedDirsByProject.get(o.node.projectName) ?? [],
1395
- ),
1396
- )
1397
- if (bytes === null) return
1398
- const name = runEntryName(runId, o.node.id, 'outputs')
1399
- if (await keep(name, bytes)) task.outputs = name
1400
- })
1401
- }
1402
- }
1403
- for (let i = 0; i < work.length; i += 8) {
1404
- await Promise.all(work.slice(i, i + 8).map((w) => w()))
1405
- }
1406
- return { runId, commit: head.commitSha, ciRun: ciContext, ok, tasks }
1407
- }
1408
- let json: RunSummaryJson | undefined
1409
- const summarize = async (runOk: boolean, final = list): Promise<void> => {
1410
- if (options.json === true) {
1411
- json = runSummaryJson({
1412
- runId,
1413
- startedAtMs: endedAtMsAtStart,
1414
- endedAtMs,
1415
- totalMs,
1416
- ok: runOk,
1417
- ...(stoppedBy !== undefined && { exitCode: signalExitCode(stoppedBy) }),
1418
- outcomes: final,
1419
- flaky,
1420
- })
1421
- }
1422
- if (options.summarize === undefined) return
1423
- // The rewrite after the keep-alive wait says nothing: the footer is
1424
- // the run's last word.
1425
- const quiet = final !== list
1157
+ const profile = options.profile
1158
+ if (profile !== undefined) {
1426
1159
  try {
1427
- const wrote = await writeRunSummary({
1428
- target: options.summarize,
1429
- cacheDir,
1430
- cwd: options.cwd,
1431
- runId,
1432
- startedAtMs: endedAtMsAtStart,
1433
- endedAtMs,
1434
- totalMs,
1435
- ok: runOk,
1436
- ...(stoppedBy !== undefined && { exitCode: signalExitCode(stoppedBy) }),
1437
- outcomes: final,
1438
- flaky,
1439
- })
1440
- if (!quiet) log.status(`vx: summary written to ${wrote}`)
1441
- } catch (err) {
1442
- const msg = err instanceof Error ? err.message : String(err)
1443
- log.status(`vx: failed to write summary: ${msg}`)
1444
- }
1445
- }
1446
- await summarize(ok)
1447
- if (options.profile !== undefined) {
1448
- try {
1449
- const wrote = await writeRunProfile({
1450
- target: options.profile,
1451
- cwd: options.cwd,
1452
- outcomes: list,
1453
- })
1160
+ const wrote = await step('vx:profile', () =>
1161
+ writeRunProfile({
1162
+ target: profile,
1163
+ cwd: options.cwd,
1164
+ outcomes: list,
1165
+ }),
1166
+ )
1454
1167
  log.status(`vx: profile written to ${wrote}`)
1455
1168
  } catch (err) {
1456
1169
  const msg = err instanceof Error ? err.message : String(err)
1457
- log.status(`vx: failed to write profile: ${msg}`)
1170
+ warnOf(log)(`vx: failed to write profile: ${msg}`)
1458
1171
  }
1459
1172
  }
1460
1173
  // Record each task to the run history in a single SQLite transaction
@@ -1482,11 +1195,11 @@ async function runOnBus(
1482
1195
  concurrency,
1483
1196
  flow: options.flow ?? null,
1484
1197
  forwardArgs: options.forwardArgs,
1485
- tags: maskTags(options.tags ?? {}),
1486
1198
  git: gitContext,
1487
1199
  ci: ciContext,
1488
1200
  host: hostContext,
1489
1201
  withTelemetry: telemetry !== undefined,
1202
+ remoteTraffic: prepared.localCache.traffic.keys,
1490
1203
  })
1491
1204
  const records = recordsOf(list, ok)
1492
1205
  const recordHistory = (write: () => void): void => {
@@ -1498,27 +1211,21 @@ async function runOnBus(
1498
1211
  // an unprivileged user on a 2 MiB disk, 2026-09-16). The run said
1499
1212
  // what happened; `vx last` and the flaky list will not know this one.
1500
1213
  const message = err instanceof Error ? err.message : String(err)
1501
- log.status(`[vx] run history not recorded: ${message} — the verdict above stands`)
1214
+ warnOf(log)(`[vx] run history not recorded: ${message} — the verdict above stands`)
1502
1215
  }
1503
1216
  }
1504
1217
  // A server kept in the foreground can still fail the run after the
1505
1218
  // cache closes: its history is written once the wait has decided, or it
1506
1219
  // read `ok` over the exit 1 and fed the flaky list a pass (X-23).
1507
- const historyAfterWait = keepAlive.children.length > 0 && !hold && stoppedBy === undefined
1508
- // A failed task's output, for `vx last` and an agent: files beside the
1509
- // index, so a refused write (a full disk, a read-only cache dir) costs
1510
- // only that output.
1511
- const recordFailures = (final: readonly TaskOutcome[]): void => {
1512
- try {
1513
- writeRunFailures(prepared.cacheDir, runId, final)
1514
- } catch (err) {
1515
- if (!isFsRefusal(err)) throw err
1516
- log.status(`[vx] failed tasks' output not kept: ${err.message} — the verdict above stands`)
1517
- }
1518
- }
1519
- if (stoppedBy === undefined && !historyAfterWait) {
1520
- recordHistory(() => cache.recordRunBundle(records))
1521
- recordFailures(list)
1220
+ const historyAfterWait = keepAlive.children.length > 0 && !hold && stop.stoppedBy === undefined
1221
+ // A failed task's output, for `vx last` and an agent (run-failures.ts).
1222
+ const recordFailures = (final: readonly TaskOutcome[]): void =>
1223
+ writeRunFailures(prepared.cacheDir, runId, final, warnOf(log))
1224
+ if (stop.stoppedBy === undefined && !historyAfterWait) {
1225
+ stepSync('vx:history', () => {
1226
+ recordHistory(() => cache.recordRunBundle(records))
1227
+ recordFailures(list)
1228
+ })
1522
1229
  }
1523
1230
  mark('record history')
1524
1231
  // Drain any still-in-flight background prefetches before closing the
@@ -1536,14 +1243,29 @@ async function runOnBus(
1536
1243
  // channel shut from under the upload — silently losing every remote
1537
1244
  // write with nothing but a warning. The seam's contract is that a
1538
1245
  // cache layer stays usable until the run's uploads have settled.
1539
- await prefetchDone
1540
1246
  // Every deferred save has settled by here: the scheduler finishes a
1541
1247
  // task only once its `settledOf` promise (the lane's `landed`) has,
1542
1248
  // so `runGraph` resolved with every save in the cache. A second
1543
1249
  // drain of the lane stood here until item 634; deleting it changed
1544
1250
  // nothing the suite could see, because this gate is the rule.
1545
- await cache.drainUploads?.()
1546
- const report = await keepRunBytes(list)
1251
+ await step('vx:uploads', async () => {
1252
+ await prefetchDone
1253
+ await cache.drainUploads?.()
1254
+ })
1255
+ // An upload that failed after the rows went in: its row says so too.
1256
+ if (prepared.localCache.uploads.failed > 0 && stop.stoppedBy === undefined && !historyAfterWait)
1257
+ recordHistory(() => recordLateRemoteErrors(prepared.localCache, records.runs, runId))
1258
+ const report = await step('vx:keep-run', () =>
1259
+ keepRunBytes(list, {
1260
+ runId,
1261
+ commit: head.commitSha,
1262
+ ciRun: ciContext,
1263
+ ok,
1264
+ workspaceRoot: prepared.workspaceRoot,
1265
+ nestedDirsByProject: prepared.nestedDirsByProject,
1266
+ keep: prepared.keepRunEntry,
1267
+ }),
1268
+ )
1547
1269
  // Hand the per-run summary to the telemetry sinks + drain them. Only
1548
1270
  // when a sink is active (telemetry !== undefined) — otherwise this
1549
1271
  // whole block is skipped and the run is byte-identical to before.
@@ -1562,72 +1284,52 @@ async function runOnBus(
1562
1284
  uploads: prepared.localCache.uploads,
1563
1285
  })
1564
1286
  telemetry.emitSummary(summary)
1565
- await telemetry.flush()
1287
+ const sinks = telemetry
1288
+ await step('vx:telemetry', () => sinks.flush())
1566
1289
  }
1567
1290
  // End-of-run plugin lifecycle: each plugin's teardown(). Crash-isolated
1568
1291
  // + time-bounded inside teardownPlugins, so a faulty plugin can neither
1569
1292
  // fail nor hang the run. Here on the normal path, after the drains
1570
1293
  // above; every other exit takes it through `teardown` too.
1571
- // The miss path's output-directory snapshots, taken now that the
1572
- // directories are old enough for the snapshot's racy window (see
1573
- // miss-save.ts). A few at a time: each is an lstat + readdir per
1574
- // prefix; the rows land together at close, in one transaction.
1575
- // Each walk is held to its entry's rows (item 1087), read here in one
1576
- // batch: read per snapshot, each read also flushed the snapshots before
1577
- // it, one transaction apiece (+110 ms at 1,000).
1578
- const snapshotRows = new Map<string, ReadonlyArray<{ path: string }>>()
1579
- for (let i = 0; i < outputDirSnapshots.length; i += 900) {
1580
- const batch = cache.loadOutputFilesBatch(
1581
- outputDirSnapshots.slice(i, i + 900).map((s) => s.hash),
1582
- )
1583
- for (const [hash, rows] of batch) snapshotRows.set(hash, rows)
1584
- }
1585
- for (let i = 0; i < outputDirSnapshots.length; i += 32) {
1586
- await Promise.all(
1587
- outputDirSnapshots
1588
- .slice(i, i + 32)
1589
- .map((s) =>
1590
- cache.recordOutputDirs?.(s.hash, s.projectDir, s.prefixes, (files) =>
1591
- s.holds(files, snapshotRows.get(s.hash) ?? []),
1592
- ),
1593
- ),
1594
- )
1595
- }
1294
+ // The miss path's output-directory snapshots first (miss-save.ts).
1295
+ await step('vx:snapshots', () => recordOutputDirSnapshots(cache, outputDirSnapshots))
1596
1296
  mark('output dir snapshots')
1597
1297
  // Not on a stopped run: one stopped while it waited on another run's
1598
1298
  // lock never held it, and its prune evicted under that run (item 858).
1599
- if (!stopRun.signal.aborted) await applyCacheRetention(prepared)
1299
+ if (!stop.signal.aborted) await step('vx:retention', () => applyCacheRetention(prepared))
1600
1300
  // A plugin hears the run until its teardown and nothing after: released
1601
1301
  // only in the finally, its handlers heard a kept server through the
1602
1302
  // whole keep-alive wait below (C-66). Idempotent; the finally's stay.
1603
1303
  disposePlugins?.()
1604
1304
  telemetry?.dispose()
1605
1305
  await teardown(undefined, report)
1606
- await closeCache()
1306
+ await step('vx:close', closeCache)
1607
1307
  mark('close')
1608
1308
  printTimings()
1609
1309
 
1610
1310
  // Tear down SRT's network bridge + (on macOS) log monitor. No-op if
1611
1311
  // no task was sandboxed; otherwise SRT keeps proxy servers alive and
1612
1312
  // the next vx run would init on top of stale state.
1613
- await prewarming
1614
- if (sandboxArmer?.armed) {
1615
- try {
1616
- await resetSandbox()
1617
- } catch (err) {
1618
- const msg = err instanceof Error ? err.message : String(err)
1619
- log.status(`vx: sandbox cleanup failed: ${msg}`)
1313
+ await step('vx:sandbox', async () => {
1314
+ await prewarming
1315
+ if (sandboxArmer?.armed) {
1316
+ try {
1317
+ await resetSandbox()
1318
+ } catch (err) {
1319
+ const msg = err instanceof Error ? err.message : String(err)
1320
+ warnOf(log)(`vx: sandbox cleanup failed: ${msg}`)
1321
+ }
1620
1322
  }
1621
- }
1323
+ })
1622
1324
 
1623
1325
  // The footer is the run's last word (owner, 2026-10-06): it prints once
1624
1326
  // the run's own work is done, history, uploads, plugin teardown and the
1625
1327
  // sandbox reset included, so any warning they raise lands above it. A
1626
1328
  // task's own facts (flaky, blocked) ride its row. Only a kept server's
1627
1329
  // own output follows.
1330
+ // The footer replaces the live region in place.
1331
+ log.runDone?.()
1628
1332
  if (options.summaryTable === true) for (const line of formatOutcomeTable(list)) log.status(line)
1629
- const above = options.beforeFooter?.(list, ok)
1630
- if (above) log.status(above.replace(/\n$/, ''))
1631
1333
  if (options.footer !== false)
1632
1334
  for (const line of formatRunSummary(list, totalMs, colors, runContext)) log.status(line)
1633
1335
 
@@ -1646,57 +1348,18 @@ async function runOnBus(
1646
1348
  // ended the session and its code — the summary above said `success`
1647
1349
  // for it, and an exit 1 with no word about why is a mystery in a log.
1648
1350
  if (keepAlive.children.length > 0 && hold) {
1649
- const held = keepAlive.children
1650
- // A held server that dies on its own is said, as the foreground's
1651
- // keep-alive says it: `vx watch` sat on "watching" over a dead dev
1652
- // server. The holder's own stop() is not such a death.
1653
- let stopping = false
1654
- // Taken by a later run (`RunOptions.keep`): kept or stopped there.
1655
- const handed = new Set<ReturnType<typeof Bun.spawn>>()
1656
- const servers = new Map<string, HeldServer>()
1657
- keepAlive.nodes.forEach((n, i) => {
1658
- const child = held[i]!
1659
- servers.set(n.id, {
1660
- child,
1661
- config: serverConfig(n, options.forwardArgs),
1662
- out: serverOuts.get(n.id) ?? { bus },
1663
- handOver: () => void handed.add(child),
1664
- })
1665
- // One that died during the graph was said then.
1666
- if (hasEnded(child)) return
1667
- void child.exited.then((code) => {
1668
- if (!stopping && !handed.has(child) && code !== 0)
1669
- log.status(`vx: ${n.id} exited with code ${code}`)
1670
- })
1671
- })
1672
1351
  return {
1673
1352
  ok,
1674
1353
  outcomes: list,
1675
- persistent: {
1676
- ids: keepAlive.nodes.map((n) => n.id),
1677
- servers,
1678
- stop: (signal) => {
1679
- stopping = true
1680
- return terminateChildren(() => held.filter((c) => !handed.has(c)), signal)
1681
- },
1682
- },
1354
+ persistent: holdServers(keepAlive, {
1355
+ forwardArgs: options.forwardArgs,
1356
+ outOf: (id) => serverOuts.get(id) ?? { bus },
1357
+ warn: warnOf(log),
1358
+ }),
1683
1359
  }
1684
1360
  }
1685
1361
  if (keepAlive.children.length > 0) {
1686
- // A server kept only as a dependency that exits 0 is a daemon that
1687
- // forked and returned (schema.md), not the end of the session: a
1688
- // `db: 'docker compose up -d'` under `app#dev` stopped the dev server
1689
- // right after the summary (WD-3). It ends the wait only as the last.
1690
- const first = await new Promise<{ code: number; i: number }>((resolve) => {
1691
- let left = keepAlive.children.length
1692
- keepAlive.children.forEach((c, i) => {
1693
- void c.exited.then((code) => {
1694
- left--
1695
- const n = keepAlive.nodes[i]!
1696
- if (code !== 0 || n.requested || n.surfaced === true || left === 0) resolve({ code, i })
1697
- })
1698
- })
1699
- })
1362
+ const first = await sessionEnd(keepAlive)
1700
1363
  const node = keepAlive.nodes[first.i]!
1701
1364
  // Only the ones still up are stopped: a kept server already dead was
1702
1365
  // counted as one (WD-11).
@@ -1704,8 +1367,8 @@ async function runOnBus(
1704
1367
  // Not when the run was stopped: the server ended because the user
1705
1368
  // stopped it, and "exited with code 130" read as a crash after every
1706
1369
  // Ctrl-C once a stop let run() finish its own path (item 852).
1707
- if (!stopRun.signal.aborted) {
1708
- log.status(
1370
+ if (!stop.signal.aborted) {
1371
+ warnOf(log)(
1709
1372
  `vx: ${node.id} exited with code ${first.code}` +
1710
1373
  (others > 0
1711
1374
  ? `; stopping ${others} other persistent task${others === 1 ? '' : 's'}`
@@ -1715,14 +1378,11 @@ async function runOnBus(
1715
1378
  // Under a stop the abort is already taking them down with the stop's
1716
1379
  // signal; a second SIGTERM in its grace cut short the graceful
1717
1380
  // shutdown of a server that reads it as "quit now" (WD-16).
1718
- if (stopRun.signal.aborted) await aborting?.catch(() => {})
1381
+ if (stop.signal.aborted) await stop.aborting?.catch(() => {})
1719
1382
  else await terminateChildren(() => keepAlive.children)
1720
- // The server that ended the session on its own, not cleanly, failed:
1721
- // the rewritten summary said `ok: false` over every task `success`
1722
- // and `failed: 0`, and so did the outcomes `--report` renders (C-53).
1723
- if (!stopRun.signal.aborted && first.code !== 0) failServer(node.id, first.code)
1383
+ // The server that ended the session on its own, not cleanly, failed.
1384
+ if (!stop.signal.aborted && first.code !== 0) failServer(outcomes, node.id, first.code)
1724
1385
  const final = [...outcomes.values()]
1725
- await summarize(ok && first.code === 0, final)
1726
1386
  if (historyAfterWait) {
1727
1387
  recordHistory(() => {
1728
1388
  // A writer's open: another vx version may have reset the index
@@ -1736,10 +1396,10 @@ async function runOnBus(
1736
1396
  })
1737
1397
  recordFailures(final)
1738
1398
  }
1739
- return { ok: ok && first.code === 0, outcomes: final, ...(json && { json }) }
1399
+ return { ok: ok && first.code === 0, outcomes: final }
1740
1400
  }
1741
1401
 
1742
- return { ok, outcomes: list, ...(json && { json }) }
1402
+ return { ok, outcomes: list }
1743
1403
  } finally {
1744
1404
  // Idempotent. The status-line ticker starts in runStart, and every
1745
1405
  // call between it and the success path's runEnd is crash-isolated
@@ -1749,9 +1409,8 @@ async function runOnBus(
1749
1409
  // here), so deleting this line reddens nothing (item 635). It is the
1750
1410
  // floor for a throw a later change adds, not a path a row can drive.
1751
1411
  log.runEnd?.()
1752
- signals.remove()
1753
- options.signal?.removeEventListener('abort', forwardAbort)
1754
- stopRun.signal.removeEventListener('abort', onAbort)
1412
+ log.runDone?.()
1413
+ stop.detach()
1755
1414
  // Plugins installed at the top of run() get their bus subscriptions
1756
1415
  // released here. Idempotent; safe even if installPlugins threw.
1757
1416
  disposePlugins?.()
@@ -1768,9 +1427,9 @@ async function runOnBus(
1768
1427
  } catch {
1769
1428
  // teardown must not throw on the way out
1770
1429
  }
1771
- await aborting?.catch(() => {})
1430
+ await stop.aborting?.catch(() => {})
1772
1431
  // Last: a process signal waits for this before it exits.
1773
- leftRun()
1432
+ stop.left()
1774
1433
  }
1775
1434
  }
1776
1435
 
@@ -1815,7 +1474,7 @@ function noneAffected(prepared: PreparedRun, options: RunOptions): string {
1815
1474
  * Planning mode. Same setup as `run()` — workspace discovery, config
1816
1475
  * load, package graph, task graph — but stops short of execution.
1817
1476
  * Returns a `RunPlan` predicting the cache hit/miss outcome of every
1818
- * task. Used by `--dry-run` and `--graph`.
1477
+ * task. Used by `--dry`.
1819
1478
  *
1820
1479
  * Side-effects are limited to opening + closing the local Cache handle
1821
1480
  * (and running `cache.inputs.runtime` probe commands, which key
@@ -1890,7 +1549,7 @@ export async function planRun(options: RunOptions): Promise<RunPlan> {
1890
1549
  } finally {
1891
1550
  // The plan called the cache and executor factories; teardown releases
1892
1551
  // what they opened, as a run's end does (item 1021).
1893
- await teardownPlugins(prepared.plugins, (m) => log.status(m))
1552
+ await teardownPlugins(prepared.plugins, warnOf(log))
1894
1553
  prepared.cache.close()
1895
1554
  // The same table a run prints: a dry run is how the prepare stages
1896
1555
  // (discovery, config load, the git enumeration) get profiled on a