@vzn/vx 0.0.21 → 0.0.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/README.md +21 -18
  2. package/package.json +6 -6
  3. package/src/bin.ts +52 -3
  4. package/src/cache/archive.ts +170 -21
  5. package/src/cache/cache.ts +667 -288
  6. package/src/cache/chained-cache.ts +14 -2
  7. package/src/cache/config-evals.ts +29 -6
  8. package/src/cache/file-hashes.ts +63 -35
  9. package/src/cache/git-inputs.ts +398 -69
  10. package/src/cache/index.ts +8 -1
  11. package/src/cache/inputs.ts +403 -102
  12. package/src/cache/key-fold.ts +232 -0
  13. package/src/cache/layer.ts +81 -11
  14. package/src/cache/layered-cache.ts +199 -32
  15. package/src/cache/output-index.ts +109 -10
  16. package/src/cache/run-history.ts +8 -3
  17. package/src/cache/zstd.ts +53 -30
  18. package/src/cli/cache.ts +51 -21
  19. package/src/cli/format.ts +4 -15
  20. package/src/cli/help.ts +13 -3
  21. package/src/cli/index.ts +35 -9
  22. package/src/cli/info.ts +49 -7
  23. package/src/cli/init.ts +17 -5
  24. package/src/cli/last.ts +96 -21
  25. package/src/cli/lock.ts +34 -17
  26. package/src/cli/plan-format.ts +4 -4
  27. package/src/cli/plugin-commands.ts +20 -5
  28. package/src/cli/run.ts +55 -12
  29. package/src/cli/select.ts +23 -13
  30. package/src/cli/show.ts +9 -4
  31. package/src/cli/upgrade.ts +134 -14
  32. package/src/cli/watch.ts +611 -102
  33. package/src/cli/why.ts +8 -16
  34. package/src/cli/workspace-config.ts +21 -9
  35. package/src/config.ts +46 -9
  36. package/src/exec/executor.ts +51 -5
  37. package/src/exec/index.ts +10 -0
  38. package/src/exec/kill-tree.ts +281 -0
  39. package/src/exec/local-executor.ts +1 -1
  40. package/src/exec/runner.ts +327 -82
  41. package/src/exec/sandbox-binds.ts +34 -5
  42. package/src/exec/sandbox-paths.ts +22 -2
  43. package/src/exec/sandbox-runtime.ts +627 -102
  44. package/src/exec/sandbox-violations.ts +36 -23
  45. package/src/graph/dependency-spec.ts +14 -14
  46. package/src/graph/index.ts +12 -2
  47. package/src/graph/priorities.ts +130 -0
  48. package/src/graph/scheduler.ts +182 -134
  49. package/src/graph/task-graph.ts +552 -188
  50. package/src/index.ts +16 -2
  51. package/src/orchestrator/admission.ts +41 -17
  52. package/src/orchestrator/colors.ts +8 -3
  53. package/src/orchestrator/deferred-outputs.ts +22 -7
  54. package/src/orchestrator/doctor.ts +122 -14
  55. package/src/orchestrator/events.ts +71 -2
  56. package/src/orchestrator/excluded-keys.ts +155 -0
  57. package/src/orchestrator/execute-task.ts +331 -37
  58. package/src/orchestrator/failure-recap.ts +121 -0
  59. package/src/orchestrator/fingerprint-watch.ts +89 -0
  60. package/src/orchestrator/framed-output.ts +81 -14
  61. package/src/orchestrator/history.ts +6 -7
  62. package/src/orchestrator/hit-restore.ts +82 -18
  63. package/src/orchestrator/index.ts +7 -1
  64. package/src/orchestrator/keyed-projects.ts +89 -0
  65. package/src/orchestrator/local-shortcircuit.ts +83 -23
  66. package/src/orchestrator/lockfile-claim.ts +152 -32
  67. package/src/orchestrator/logger.ts +116 -20
  68. package/src/orchestrator/metrics.ts +156 -60
  69. package/src/orchestrator/miss-save.ts +106 -43
  70. package/src/orchestrator/options.ts +30 -4
  71. package/src/orchestrator/persistent.ts +41 -21
  72. package/src/orchestrator/placement.ts +55 -25
  73. package/src/orchestrator/plan.ts +24 -11
  74. package/src/orchestrator/plugin-host.ts +17 -23
  75. package/src/orchestrator/plugin.ts +31 -8
  76. package/src/orchestrator/prepare.ts +99 -13
  77. package/src/orchestrator/projects.ts +19 -8
  78. package/src/orchestrator/remote-prefetch.ts +21 -1
  79. package/src/orchestrator/run-artifacts.ts +9 -0
  80. package/src/orchestrator/run-context.ts +19 -4
  81. package/src/orchestrator/run-lock.ts +349 -0
  82. package/src/orchestrator/run-records.ts +6 -23
  83. package/src/orchestrator/run-report.ts +4 -2
  84. package/src/orchestrator/run.ts +264 -57
  85. package/src/orchestrator/sandbox-request.ts +388 -62
  86. package/src/orchestrator/save-lane.ts +4 -7
  87. package/src/orchestrator/shell-verdict.ts +117 -0
  88. package/src/orchestrator/signals.ts +86 -30
  89. package/src/orchestrator/stable-keys.ts +202 -10
  90. package/src/orchestrator/status-line.ts +33 -1
  91. package/src/orchestrator/summary.ts +49 -2
  92. package/src/orchestrator/task-hash.ts +87 -7
  93. package/src/orchestrator/task-log-buffer.ts +7 -7
  94. package/src/orchestrator/telemetry-host.ts +16 -7
  95. package/src/orchestrator/telemetry.ts +69 -28
  96. package/src/orchestrator/upstream.ts +91 -45
  97. package/src/util/bun-version.ts +43 -0
  98. package/src/util/errors.ts +86 -0
  99. package/src/util/hash.ts +10 -1
  100. package/src/util/index.ts +32 -3
  101. package/src/util/paths.ts +147 -5
  102. package/src/util/procfs.ts +20 -0
  103. package/src/util/settle.ts +4 -2
  104. package/src/util/size.ts +33 -0
  105. package/src/util/task-id.ts +20 -0
  106. package/src/util/timing.ts +11 -1
  107. package/src/util/which.ts +48 -0
  108. package/src/workspace/affected.ts +272 -64
  109. package/src/workspace/config-cache.ts +262 -32
  110. package/src/workspace/config-eval.ts +49 -12
  111. package/src/workspace/config-imports.ts +126 -5
  112. package/src/workspace/config-schema.ts +237 -41
  113. package/src/workspace/filter.ts +33 -18
  114. package/src/workspace/fingerprint.ts +29 -11
  115. package/src/workspace/index.ts +7 -0
  116. package/src/workspace/json-data.ts +90 -0
  117. package/src/workspace/load-reads.ts +32 -0
  118. package/src/workspace/lockfile.ts +33 -11
  119. package/src/workspace/migrate-scripts.ts +60 -7
  120. package/src/workspace/migration.ts +40 -1
  121. package/src/workspace/package-graph.ts +154 -16
  122. package/src/workspace/project-loader.ts +227 -77
  123. package/src/workspace/workspace.ts +173 -23
@@ -1,11 +1,15 @@
1
1
  import path from 'node:path'
2
2
  import type { ExecConfig, TaskConfig, CacheConfig } from '../config.js'
3
3
  import {
4
+ ArtifactVanishedError,
4
5
  type CacheEntry,
5
6
  type CacheLayer,
6
7
  type CachePolicy,
7
8
  cleanOutputs,
8
9
  cleanWorkspaceOutputs,
10
+ type OutputStamp,
11
+ ownOutputsSince,
12
+ stampOutputs,
9
13
  FULL_CACHE_POLICY,
10
14
  type GitFilesCache,
11
15
  } from '../cache/index.js'
@@ -20,16 +24,31 @@ import {
20
24
  signalExitCode,
21
25
  type CaptureConfig,
22
26
  type ExecuteRequest,
27
+ assertExecuteResult,
23
28
  type ExecuteResult,
24
29
  type SandboxViolation,
25
30
  type TaskExecutor,
26
31
  type TaskInputs,
32
+ PersistentReadyError,
27
33
  } from '../exec/index.js'
28
- import { isGroupTask, type TaskNode, type TaskOutcome } from '../graph/index.js'
29
- import { span } from '../util/index.js'
30
- import { sandboxRequestFor } from './sandbox-request.js'
31
- import { saveMiss, type OutputDirSnapshot } from './miss-save.js'
34
+ import { isGroupTask, RestoreDemoted, type TaskNode, type TaskOutcome } from '../graph/index.js'
35
+ import { relPosix, span } from '../util/index.js'
36
+ import {
37
+ mayWriteFingerprint,
38
+ type Placeholder,
39
+ placeholderSweeper,
40
+ reachedWithheld,
41
+ sandboxRequestFor,
42
+ sweepPlaceholders,
43
+ undeclaredWriteReach,
44
+ untouchedPlaceholderLine,
45
+ type WithheldLink,
46
+ withheldLinkLine,
47
+ } from './sandbox-request.js'
48
+ import { markUnsaved, saveMiss, type OutputDirSnapshot } from './miss-save.js'
49
+ import type { FingerprintWatch } from './fingerprint-watch.js'
32
50
  import { restoreHit } from './hit-restore.js'
51
+ import { shellVerdict } from './shell-verdict.js'
33
52
  // The hit path's entry stays importable from here (tests).
34
53
  export { restoreHit, type RestoreHitArgs } from './hit-restore.js'
35
54
  import type { DeferredOutputs } from './deferred-outputs.js'
@@ -39,8 +58,10 @@ import {
39
58
  computeTaskHash,
40
59
  describeTaskInputs,
41
60
  type HashCache,
61
+ movedInput,
42
62
  type TaskInputComponent,
43
63
  } from './task-hash.js'
64
+ import { getContext } from './remote-prefetch.js'
44
65
 
45
66
  export interface ExecuteArgs {
46
67
  node: TaskNode
@@ -81,6 +102,13 @@ export interface ExecuteArgs {
81
102
  nestedProjectDirs: string[]
82
103
  /** Anchor for hrtime spans across all tasks in this run. */
83
104
  runStartHrTimeNs: bigint
105
+ /**
106
+ * The run's stop (a process signal or the embedder's abort). An attempt
107
+ * that ends while it is aborted is `aborted`, whatever its exit: a task
108
+ * that traps the forwarded SIGINT and exits 0 did not finish on its own
109
+ * terms (item 962).
110
+ */
111
+ stopSignal?: AbortSignal
84
112
  /**
85
113
  * Registry the orchestrator owns. For each persistent task we
86
114
  * spawn, we stash the subprocess handle here so the orchestrator
@@ -129,6 +157,12 @@ export interface ExecuteArgs {
129
157
  * sandbox. A cache hit never calls it: a hit needs no sandbox.
130
158
  */
131
159
  armSandbox?: () => Promise<void>
160
+ /**
161
+ * The run's keyed-set lookup (keyed-projects.ts): the project directories
162
+ * a task's key answers for. A sandboxed task that declares `cache` is
163
+ * granted a linked workspace package only when it is in this set.
164
+ */
165
+ keyedProjects: (node: TaskNode) => ReadonlySet<string>
132
166
  /**
133
167
  * Run-scoped list the miss path appends its output-directory snapshot
134
168
  * request to, taken at run end (run.ts) instead of right after the save:
@@ -154,6 +188,18 @@ export interface ExecuteArgs {
154
188
  * clean run's stale hit.
155
189
  */
156
190
  taintedUpstream?: boolean
191
+ /**
192
+ * Whether a task has rewritten a file the workspace fingerprint folded
193
+ * since the run read it (fingerprint-watch.ts): past that, no key taken
194
+ * on the old digest is probed or saved.
195
+ */
196
+ fingerprintWatch?: FingerprintWatch
197
+ /**
198
+ * No task in the run depends on this one, so none is ordered after it to
199
+ * read what it wrote: a miss that saves nothing marks nothing (the output
200
+ * walk cost 1,000 read-only misses 1.62 → 1.78 s, item 750).
201
+ */
202
+ noDependants?: true
157
203
  }
158
204
 
159
205
  /**
@@ -231,16 +277,34 @@ async function executePersistentTask(args: ExecuteArgs): Promise<TaskOutcome> {
231
277
  // (Until 2026-09-09 the block was accepted and silently ignored.)
232
278
  let command = plainCommand
233
279
  let bridgeTag: string | undefined
280
+ let signalChannel = false
281
+ // The empty files the request pre-created for a literal write grant; a
282
+ // server that never writes one gets it taken back when it exits, and a
283
+ // server that dies before readiness is told the directory spelling — the
284
+ // same trap and the same answer as a one-shot task's (sandbox-request.ts).
285
+ let placeholders: Placeholder[] = []
234
286
  if (step.sandbox !== undefined) {
235
287
  await args.armSandbox?.()
236
- const sb = await sandboxRequestFor(node, step.sandbox, args.workspaceRoot)
288
+ // A persistent task never declares `cache` (the loader refuses it), so
289
+ // it keeps the whole linked grant: no key of its can be stale.
290
+ const sb = await sandboxRequestFor(
291
+ node,
292
+ step.sandbox,
293
+ args.workspaceRoot,
294
+ undefined,
295
+ args.nestedProjectDirs,
296
+ )
297
+ placeholders = sb.placeholders
237
298
  const wrapped = await wrapSandboxedCommand({
238
299
  command: plainCommand,
239
300
  cwd: node.projectDir,
240
- ...sb,
301
+ env,
302
+ ...sb.sandbox,
303
+ server: true,
241
304
  })
242
305
  command = wrapped.wrapped
243
306
  bridgeTag = wrapped.tag
307
+ signalChannel = wrapped.forwardsSignals
244
308
  }
245
309
  const persistentOpts: Parameters<typeof runPersistent>[0] = {
246
310
  command,
@@ -249,6 +313,7 @@ async function executePersistentTask(args: ExecuteArgs): Promise<TaskOutcome> {
249
313
  onStdout: (chunk) => log.taskStdout(node, chunk),
250
314
  onStderr: (chunk) => log.taskStderr(node, chunk),
251
315
  ...(args.liveChildren !== undefined ? { liveChildren: args.liveChildren } : {}),
316
+ ...(signalChannel ? { signalChannel } : {}),
252
317
  }
253
318
  if (step.persistent.readyWhen !== undefined) {
254
319
  persistentOpts.readyWhen = step.persistent.readyWhen
@@ -263,12 +328,28 @@ async function executePersistentTask(args: ExecuteArgs): Promise<TaskOutcome> {
263
328
  const spawn = runPersistent(persistentOpts)
264
329
  // The host side of a port bridge lives exactly as long as the server:
265
330
  // released on the child's exit, whether the run tore it down or it died.
266
- if (bridgeTag !== undefined) {
331
+ // A placeholder the server never wrote goes back the same way.
332
+ // ONE sweep, shared by the exit handler and the readiness failure below
333
+ // — whichever asks second must get the SAME list. A second sweep cannot
334
+ // produce it: the first one's `rm` already happened, so the file is gone
335
+ // and `sweepPlaceholders` skips it. Collecting both lists and unioning
336
+ // them does not close that, because the union only covers an exit
337
+ // handler that FINISHED; one still between its `rm` and its return has
338
+ // published nothing yet, and the readiness path then reports no
339
+ // placeholder at all. The failure then says only "File exists", which is
340
+ // the message the hint exists to explain (seen under the gate's parallel
341
+ // load, and reproduced deterministically by delaying each side, item 450).
342
+ const sweptUntouched = placeholderSweeper(placeholders)
343
+ if (bridgeTag !== undefined || placeholders.length > 0) {
267
344
  const tag = bridgeTag
268
- void spawn.child?.exited?.then(
269
- () => releaseBridges(tag),
270
- () => releaseBridges(tag),
271
- )
345
+ const onExit = async (): Promise<void> => {
346
+ if (tag !== undefined) releaseBridges(tag)
347
+ await sweptUntouched()
348
+ }
349
+ // A spawn that failed has no child to wait on: release now, or the
350
+ // tag would hold the sandbox's reset for the rest of the process.
351
+ if (spawn.child === undefined) void onExit()
352
+ else void spawn.child.exited.then(onExit, onExit)
272
353
  }
273
354
  try {
274
355
  await spawn.ready
@@ -278,17 +359,28 @@ async function executePersistentTask(args: ExecuteArgs): Promise<TaskOutcome> {
278
359
  // reader looks for why a task failed, and a run with a custom logger
279
360
  // (an embedder, the MCP server) never saw a bare stderr write at all.
280
361
  log.taskStderr(node, `\n[vx] ${node.id}: persistent task failed to become ready: ${message}\n`)
362
+ // The server is dead or being torn down, so the sweep on its exit may
363
+ // already be running: ask the shared one rather than starting a second.
364
+ for (const p of await sweptUntouched()) {
365
+ log.taskStderr(node, `${untouchedPlaceholderLine(node.projectDir, p)}\n`)
366
+ }
367
+ // The reason rides the outcome (every label reads it), and a child that
368
+ // exited before ready keeps its own exit code rather than a made-up 1.
369
+ const ready = err instanceof PersistentReadyError ? err : undefined
281
370
  return {
282
371
  node,
283
372
  status: 'failed',
284
- exitCode: 1,
373
+ exitCode: ready?.exitCode ?? 1,
285
374
  durationMs: spawn.readyMs(),
375
+ ...(ready !== undefined ? { notReady: ready.reason } : {}),
286
376
  wallclockStartNs,
287
377
  wallclockEndNs: process.hrtime.bigint() - args.runStartHrTimeNs,
288
378
  }
289
379
  }
290
380
 
291
381
  args.persistentRegistry?.set(node.id, spawn.child)
382
+ forgetUndeclaredWrites(args, undeclaredWriteReach(node, args.workspaceRoot))
383
+ if (mayWriteFingerprint(node, args.workspaceRoot)) args.fingerprintWatch?.wrote()
292
384
  return {
293
385
  node,
294
386
  status: 'success',
@@ -412,6 +504,14 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
412
504
  nestedProjectDirs: args.nestedProjectDirs,
413
505
  }
414
506
  const wsCleanArgs = { workspaceRoot: args.workspaceRoot, outputs: wsOutputs }
507
+ // An ADDITIVE task (its outputs overlap an upstream's, with the edge that
508
+ // orders them; item 588) does not clean by glob — the glob selects the
509
+ // upstream's files it adds beside. Its outputs are stamped once before
510
+ // the first attempt, and after a 0 exit its own set is what the run
511
+ // added or changed against that stamp. Stale files of its own from an
512
+ // earlier run are its command's to clean, as they are under Turbo.
513
+ const additive = (node.addsToOutputsOf?.length ?? 0) > 0
514
+ let stampedBefore: ReadonlyMap<string, OutputStamp> | undefined
415
515
 
416
516
  // Cache lookup. On hit, time the user-perceived restore op
417
517
  // (clean+restore+log-replay) — that's what the framed-block footer
@@ -421,17 +521,37 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
421
521
  // to the run path as a known stable miss. No second cache.get.
422
522
  if (willRead) {
423
523
  const cacheOpStart = performance.now()
524
+ // A hit whose artifact was removed after the probe (a prune, another
525
+ // workspace's retention on a shared cache directory) is a miss. The
526
+ // restore has wiped the declared outputs by then; the run path cleans
527
+ // them again and the task writes them.
528
+ const restoreOrMiss = async (hit: CacheEntry): Promise<TaskOutcome | null> => {
529
+ try {
530
+ return await restoreHit({ args, hash, hit, cacheOpStart, taskStartNs })
531
+ } catch (err) {
532
+ if (!(err instanceof ArtifactVanishedError)) throw err
533
+ log.status(`[vx] ${node.id}: ${err.message} — running it`)
534
+ return null
535
+ }
536
+ }
424
537
  if (preProbed !== undefined) {
425
538
  if (preProbed.hit !== null) {
426
- return restoreHit({ args, hash, hit: preProbed.hit, cacheOpStart, taskStartNs })
539
+ const restored = await restoreOrMiss(preProbed.hit)
540
+ if (restored !== null) return restored
541
+ // The up-front probe's hit may be restoring AHEAD of this task's
542
+ // deps, and a command run now would build from outputs they have
543
+ // not written yet and save that under the good key. The scheduler
544
+ // runs it again once they are done (admission drops the probe).
545
+ throw new RestoreDemoted(node.id)
427
546
  }
428
547
  // Confirmed stable miss — skip the probe, fall through to run.
429
- } else {
548
+ } else if (!fingerprintMoved()) {
430
549
  const endProbe = span('cache.get')
431
- const hit = await cache.get(hash, { taskId: node.id, command: step.command })
550
+ const hit = await cache.get(hash, getContext(node, step.command))
432
551
  endProbe()
433
552
  if (hit) {
434
- return restoreHit({ args, hash, hit, cacheOpStart, taskStartNs })
553
+ const restored = await restoreOrMiss(hit)
554
+ if (restored !== null) return restored
435
555
  }
436
556
  }
437
557
  }
@@ -482,6 +602,12 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
482
602
  // the single source of truth, and a violation fails the task (below).
483
603
  const userSandbox = cfg.exec?.sandbox !== undefined
484
604
  let violations: SandboxViolation[] = []
605
+ // The empty files the sandbox request created for a literal write grant;
606
+ // swept after the attempt (`sweepPlaceholders`).
607
+ let placeholders: Placeholder[] = []
608
+ // The linked workspace packages the request withheld: the key does not
609
+ // answer for them.
610
+ let withheld: WithheldLink[] = []
485
611
 
486
612
  // Cache miss path (or caching disabled), up to `1 + retries` attempts.
487
613
  // Explicit config wins over the run-level `--retry` default, including
@@ -506,7 +632,11 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
506
632
  // tree would replace a stale build with nothing at all. The eligibility
507
633
  // gate guarantees no key in this run can see what stays behind, and the
508
634
  // summary names every deferred task so `dist/` is not silently stale.
509
- if (willWrite && !deferralRequested && outputs.length > 0) {
635
+ if (additive && willWrite && !deferralRequested && outputs.length > 0) {
636
+ const endStamp = span('miss: stamp outputs')
637
+ stampedBefore ??= await stampOutputs(cleanArgs)
638
+ endStamp()
639
+ } else if (willWrite && !deferralRequested && outputs.length > 0) {
510
640
  // Mark the wiped paths, exactly as the workspace twin below does. The
511
641
  // git snapshot still lists them with their committed index OIDs, and
512
642
  // resolveFiles SKIPS its existence probe for any path carrying a
@@ -536,13 +666,38 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
536
666
  // no other trace. Rethrown unchanged — the scheduler still classifies it,
537
667
  // and still prints it plainly for a UserError.
538
668
  const endExec = span('miss: execute')
539
- const res = await args.executor.execute(req).catch((err: unknown) => {
540
- const message = err instanceof Error ? err.message : String(err)
541
- log.taskStderr(node, `${message}\n`)
542
- throw err
543
- })
669
+ const res = await args.executor
670
+ .execute(req)
671
+ .then((r: unknown) => {
672
+ assertExecuteResult(args.executor.name, node.id, r)
673
+ return r
674
+ })
675
+ .catch(async (err: unknown) => {
676
+ const message = err instanceof Error ? err.message : String(err)
677
+ log.taskStderr(node, `${message}\n`)
678
+ await sweepPlaceholders(placeholders)
679
+ throw err
680
+ })
544
681
  endExec()
545
682
  violations = [...res.violations]
683
+ // A denial under a dependency the key does not answer for says only
684
+ // ENOENT; the line beside it names the package and how to key it. Only
685
+ // with a denial there, so it never reddens a pass.
686
+ for (const w of reachedWithheld(withheld, violations)) {
687
+ violations.push({ timestamp: new Date(), line: withheldLinkLine(node.id, w) })
688
+ }
689
+ // A placeholder the task never wrote is not its output: take it back
690
+ // before the outputs are collected. On a failure it is also the one
691
+ // clue to a grant that meant a directory — say so beside the failure.
692
+ const untouched = await sweepPlaceholders(placeholders)
693
+ if (res.exitCode !== 0 && violations.length === 0) {
694
+ for (const p of untouched) {
695
+ violations.push({
696
+ timestamp: new Date(),
697
+ line: untouchedPlaceholderLine(node.projectDir, p),
698
+ })
699
+ }
700
+ }
546
701
  // Fail-on-violation, on BOTH platforms: macOS reads SRT's structured
547
702
  // violation store, Linux parses the strace log the sandboxed spawn
548
703
  // writes. (This comment used to claim Linux violations are "always 0"
@@ -572,6 +727,20 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
572
727
  // child already reports 143, so this only rewrites the trap-exit-0 case.
573
728
  if (code === 0) code = signalExitCode('SIGTERM')
574
729
  }
730
+ // The shell's 127 and 126 name the word and nothing about why — the
731
+ // PATH vx built, or a `#!` line the file itself carries (items 257,
732
+ // 258) — and an exit above 128 is a signal's number and nothing
733
+ // about what sent it (259). One frame line names the rule.
734
+ if (!res.timedOut) {
735
+ const verdict = shellVerdict({
736
+ code,
737
+ command: step.command,
738
+ cwd: node.projectDir,
739
+ bins: taskBinDirs(node, args.workspaceRoot),
740
+ signal: res.signal,
741
+ })
742
+ if (verdict !== undefined) log.taskStderr(node, `\n${verdict}\n`)
743
+ }
575
744
  return { result: res, exitCode: code }
576
745
  }
577
746
 
@@ -599,19 +768,33 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
599
768
  }
600
769
  }
601
770
 
771
+ // Pass or fail, the command may have written where its project's
772
+ // run-start facts describe; a remote executor wrote on its own disk.
773
+ const writeReach =
774
+ args.executor.remote === true ? 'none' : undeclaredWriteReach(node, args.workspaceRoot)
775
+ const writesFingerprint =
776
+ args.executor.remote !== true && mayWriteFingerprint(node, args.workspaceRoot)
602
777
  for (;;) {
603
778
  attempt++
604
779
  const a = await runAttempt()
780
+ forgetUndeclaredWrites(args, writeReach)
781
+ if (writesFingerprint) args.fingerprintWatch?.wrote()
605
782
  result = a.result
606
783
  effectiveExitCode = a.exitCode
607
784
 
608
- // A child killed by a shutdown signal (Ctrl-C / SIGTERM teardown)
609
- // never finished on its own terms — revert it to aborted so it's
610
- // neither cached, counted, shown, nor RETRIED (the run is tearing
611
- // down). SIGKILL (OOM, forced) stays a real failure. A timeout also
612
- // SIGTERMs, but `timedOut` marks it as our own deadline, not a
613
- // shutdown — so it stays a real (retryable) failure.
614
- if ((result.signal === 'SIGINT' || result.signal === 'SIGTERM') && !result.timedOut) {
785
+ // An attempt that ended in a shutdown never finished on its own terms —
786
+ // it is aborted, so it is neither cached, counted, shown, nor RETRIED.
787
+ // Two ways to know: the child died of SIGINT / SIGTERM (Ctrl-C, or a
788
+ // `kill` vx never saw; a timeout also SIGTERMs, but `timedOut` marks it
789
+ // as our own deadline, a real and retryable failure), or the RUN is
790
+ // stopping. The child alone was not enough: one that traps the
791
+ // forwarded SIGINT and exits 0 was cached as a success with its partial
792
+ // outputs, one that exits 1 was retried after Ctrl-C, and one SIGKILLed
793
+ // at the end of the grace read as an OOM failure (item 962).
794
+ const shutdown =
795
+ args.stopSignal?.aborted === true ||
796
+ ((result.signal === 'SIGINT' || result.signal === 'SIGTERM') && !result.timedOut)
797
+ if (shutdown) {
615
798
  return {
616
799
  node,
617
800
  status: 'aborted',
@@ -626,7 +809,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
626
809
  if (effectiveExitCode === 0 || attempt >= maxAttempts) break
627
810
  log.taskStderr(
628
811
  node,
629
- `vx: retrying ${node.id} (attempt ${attempt + 1}/${maxAttempts}) after exit ${effectiveExitCode}\n`,
812
+ `vx: retrying ${node.id} (attempt ${attempt + 1}/${maxAttempts}) after ${result.timedOut === true ? 'a timeout' : `exit ${effectiveExitCode}`}\n`,
630
813
  )
631
814
  }
632
815
 
@@ -655,7 +838,16 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
655
838
  }
656
839
  if (!userSandbox) return base
657
840
  await args.armSandbox?.()
658
- return { ...base, sandbox: await sandboxRequestFor(node, step.sandbox!, args.workspaceRoot) }
841
+ const sb = await sandboxRequestFor(
842
+ node,
843
+ step.sandbox!,
844
+ args.workspaceRoot,
845
+ cfgCacheable ? args.keyedProjects(node) : undefined,
846
+ args.nestedProjectDirs,
847
+ )
848
+ placeholders = sb.placeholders
849
+ withheld = sb.withheld
850
+ return { ...base, sandbox: sb.sandbox }
659
851
  }
660
852
 
661
853
  const wallclockEndNs = process.hrtime.bigint() - args.runStartHrTimeNs
@@ -681,8 +873,13 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
681
873
  },
682
874
  })
683
875
  }
684
- } else if (effectiveExitCode === 0 && willSave) {
876
+ } else if (effectiveExitCode === 0 && willSave && (await keyStillTrue())) {
877
+ const ownOutputFiles =
878
+ additive && stampedBefore !== undefined
879
+ ? await ownOutputsSince(cleanArgs, stampedBefore)
880
+ : undefined
685
881
  const { landed } = await saveMiss({
882
+ ...(ownOutputFiles !== undefined ? { ownOutputFiles } : {}),
686
883
  node,
687
884
  hash,
688
885
  cache,
@@ -702,6 +899,76 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
702
899
  deferSave: args.deferSave,
703
900
  })
704
901
  args.deferredSaves?.set(node.id, landed)
902
+ } else if (cfgCacheable && !remoteOnly && !deferralRequested && args.noDependants !== true) {
903
+ // Ran here and saves nothing — failed, a read-only policy, a tainted
904
+ // upstream, a key that no longer held — yet it wrote what a save would
905
+ // have marked, and may have rewritten an input the re-check before a
906
+ // save catches. A same-run reader keyed from the snapshot's OIDs for
907
+ // either replayed the bytes from before the command (item 750). A
908
+ // withheld save already dropped the project; this finds the same move
909
+ // again and drops it again.
910
+ const endCheck = span('miss: recheck inputs')
911
+ const moved = await movedSinceKey()
912
+ endCheck()
913
+ if (moved !== undefined) {
914
+ forgetUndeclaredWrites(args, wsOutputs.length > 0 ? 'workspace' : 'project')
915
+ } else {
916
+ await markUnsaved({
917
+ node,
918
+ workspaceRoot: args.workspaceRoot,
919
+ nestedProjectDirs: args.nestedProjectDirs,
920
+ gitFilesCache: args.gitFilesCache,
921
+ outputs,
922
+ wsOutputs,
923
+ })
924
+ }
925
+ }
926
+
927
+ /**
928
+ * The workspace fingerprint every key here folded has moved (a task
929
+ * rewrote the lockfile), said once per run.
930
+ */
931
+ function fingerprintMoved(): boolean {
932
+ if (args.fingerprintWatch?.moved() === undefined) return false
933
+ args.fingerprintWatch.say(log)
934
+ return true
935
+ }
936
+
937
+ /**
938
+ * The input that no longer holds what the key folded: `null` when the
939
+ * key re-derived just before the command already differed (the file
940
+ * unnamed), undefined when none moved.
941
+ */
942
+ async function movedSinceKey(): Promise<string | null | undefined> {
943
+ return described!.hash !== hash
944
+ ? null
945
+ : await movedInput(described!.facts, cache, described!.describedAt)
946
+ }
947
+
948
+ /**
949
+ * Does the key still describe the inputs the command ran over? It was
950
+ * taken before the command — up front, or before `describeTaskInputs` —
951
+ * and a save files the outputs under it. A user's edit mid-run, or a
952
+ * task rewriting its own input (a formatter), saved bytes built from one
953
+ * state under the key of another: restore the old state and the next run
954
+ * replayed them as up-to-date (turborepo#10111, #1146, item 743). The
955
+ * result stands; only the entry is withheld, and the facts about the
956
+ * project go, since something wrote there. A lockfile a task rewrote
957
+ * withholds it too: the key folded the old one.
958
+ */
959
+ async function keyStillTrue(): Promise<boolean> {
960
+ if (fingerprintMoved()) return false
961
+ const endCheck = span('miss: recheck inputs')
962
+ const moved = await movedSinceKey()
963
+ endCheck()
964
+ if (moved === undefined) return true
965
+ const what = moved === null ? 'its inputs' : `\`${relPosix(args.workspaceRoot, moved)}\``
966
+ log.status(
967
+ `[vx] ${node.id}: ${what} changed after its key was taken — the result stands, ` +
968
+ `but is not saved under a key that no longer describes it`,
969
+ )
970
+ forgetUndeclaredWrites(args, wsOutputs.length > 0 ? 'workspace' : 'project')
971
+ return false
705
972
  }
706
973
 
707
974
  const finalViolations = violations
@@ -713,6 +980,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
713
980
  durationMs: result.durationMs,
714
981
  hash,
715
982
  ...(attempt > 1 ? { attempts: attempt } : {}),
983
+ ...(result.timedOut === true && effectiveExitCode !== 0 ? { timedOut: true as const } : {}),
716
984
  ...(result.cpuMs !== undefined ? { cpuMs: result.cpuMs } : {}),
717
985
  ...(result.peakRssBytes !== undefined ? { peakRssBytes: result.peakRssBytes } : {}),
718
986
  ...(result.where !== undefined ? { where: result.where } : {}),
@@ -730,6 +998,36 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
730
998
  }
731
999
  }
732
1000
 
1001
+ /**
1002
+ * Drop the run's facts about files a task may have written without
1003
+ * declaring them (`undeclaredWriteReach`): the project's git snapshot and
1004
+ * its index OIDs, and its `package.json` digest. The next reader
1005
+ * re-enumerates the project (one `git ls-files`) and hashes its files by
1006
+ * content; the per-declaration file-list memos are keyed on the snapshot
1007
+ * array, so they miss with it (item 743).
1008
+ */
1009
+ function forgetUndeclaredWrites(args: ExecuteArgs, reach: 'none' | 'project' | 'workspace'): void {
1010
+ if (reach === 'none') return
1011
+ if (reach === 'workspace') {
1012
+ args.gitFilesCache?.clear()
1013
+ args.hashCache?.packageJson.clear()
1014
+ return
1015
+ }
1016
+ args.gitFilesCache?.delete(args.node.projectDir)
1017
+ args.gitFilesCache?.invalidateWorkspacePartition()
1018
+ args.hashCache?.packageJson.delete(args.node.projectDir)
1019
+ }
1020
+
1021
+ /** The two bin directories a task's PATH starts with: its own, then the root's (once when they coincide). */
1022
+ function taskBinDirs(node: TaskNode, workspaceRoot: string): string[] {
1023
+ const bins = [path.join(node.projectDir, 'node_modules', '.bin')]
1024
+ const rootBin = path.join(workspaceRoot, 'node_modules', '.bin')
1025
+ // Identical when the root is itself a project — dedupe rather than list it
1026
+ // twice, so PATH reads the same either way.
1027
+ if (rootBin !== bins[0]) bins.push(rootBin)
1028
+ return bins
1029
+ }
1030
+
733
1031
  /**
734
1032
  * Build the child-process env for one task. Same arguments at every call site
735
1033
  * (persistent + cached). Two `node_modules/.bin` directories are prepended to
@@ -747,11 +1045,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
747
1045
  * it. npm/pnpm/yarn all put the ancestor chain on PATH for the same reason.
748
1046
  */
749
1047
  function taskEnv(node: TaskNode, step: ExecConfig, workspaceRoot: string): NodeJS.ProcessEnv {
750
- const bins = [path.join(node.projectDir, 'node_modules', '.bin')]
751
- const rootBin = path.join(workspaceRoot, 'node_modules', '.bin')
752
- // Identical when the root is itself a project — dedupe rather than list it
753
- // twice, so PATH reads the same either way.
754
- if (rootBin !== bins[0]) bins.push(rootBin)
1048
+ const bins = taskBinDirs(node, workspaceRoot)
755
1049
  const env = buildIsolatedEnv({
756
1050
  passThrough: step.env?.passThrough ?? [],
757
1051
  define: step.env?.define ?? {},