@vzn/vx 0.0.21 → 0.0.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/README.md +21 -18
  2. package/package.json +6 -6
  3. package/src/bin.ts +52 -3
  4. package/src/cache/archive.ts +170 -21
  5. package/src/cache/cache.ts +667 -288
  6. package/src/cache/chained-cache.ts +14 -2
  7. package/src/cache/config-evals.ts +29 -6
  8. package/src/cache/file-hashes.ts +63 -35
  9. package/src/cache/git-inputs.ts +398 -69
  10. package/src/cache/index.ts +8 -1
  11. package/src/cache/inputs.ts +403 -102
  12. package/src/cache/key-fold.ts +232 -0
  13. package/src/cache/layer.ts +81 -11
  14. package/src/cache/layered-cache.ts +199 -32
  15. package/src/cache/output-index.ts +109 -10
  16. package/src/cache/run-history.ts +8 -3
  17. package/src/cache/zstd.ts +53 -30
  18. package/src/cli/cache.ts +51 -21
  19. package/src/cli/format.ts +4 -15
  20. package/src/cli/help.ts +13 -3
  21. package/src/cli/index.ts +35 -9
  22. package/src/cli/info.ts +49 -7
  23. package/src/cli/init.ts +17 -5
  24. package/src/cli/last.ts +96 -21
  25. package/src/cli/lock.ts +34 -17
  26. package/src/cli/plan-format.ts +4 -4
  27. package/src/cli/plugin-commands.ts +20 -5
  28. package/src/cli/run.ts +55 -12
  29. package/src/cli/select.ts +23 -13
  30. package/src/cli/show.ts +9 -4
  31. package/src/cli/upgrade.ts +134 -14
  32. package/src/cli/watch.ts +622 -102
  33. package/src/cli/why.ts +8 -16
  34. package/src/cli/workspace-config.ts +21 -9
  35. package/src/config.ts +46 -9
  36. package/src/exec/executor.ts +51 -5
  37. package/src/exec/index.ts +10 -0
  38. package/src/exec/kill-tree.ts +281 -0
  39. package/src/exec/local-executor.ts +1 -1
  40. package/src/exec/runner.ts +327 -82
  41. package/src/exec/sandbox-binds.ts +34 -5
  42. package/src/exec/sandbox-paths.ts +22 -2
  43. package/src/exec/sandbox-runtime.ts +627 -102
  44. package/src/exec/sandbox-violations.ts +36 -23
  45. package/src/graph/dependency-spec.ts +14 -14
  46. package/src/graph/index.ts +12 -2
  47. package/src/graph/priorities.ts +130 -0
  48. package/src/graph/scheduler.ts +182 -134
  49. package/src/graph/task-graph.ts +552 -188
  50. package/src/index.ts +16 -2
  51. package/src/orchestrator/admission.ts +41 -17
  52. package/src/orchestrator/colors.ts +8 -3
  53. package/src/orchestrator/deferred-outputs.ts +22 -7
  54. package/src/orchestrator/doctor.ts +122 -14
  55. package/src/orchestrator/events.ts +71 -2
  56. package/src/orchestrator/excluded-keys.ts +155 -0
  57. package/src/orchestrator/execute-task.ts +331 -37
  58. package/src/orchestrator/failure-recap.ts +121 -0
  59. package/src/orchestrator/fingerprint-watch.ts +89 -0
  60. package/src/orchestrator/framed-output.ts +81 -14
  61. package/src/orchestrator/history.ts +6 -7
  62. package/src/orchestrator/hit-restore.ts +82 -18
  63. package/src/orchestrator/index.ts +7 -1
  64. package/src/orchestrator/keyed-projects.ts +89 -0
  65. package/src/orchestrator/local-shortcircuit.ts +83 -23
  66. package/src/orchestrator/lockfile-claim.ts +152 -32
  67. package/src/orchestrator/logger.ts +116 -20
  68. package/src/orchestrator/metrics.ts +156 -60
  69. package/src/orchestrator/miss-save.ts +106 -43
  70. package/src/orchestrator/options.ts +30 -4
  71. package/src/orchestrator/persistent.ts +41 -21
  72. package/src/orchestrator/placement.ts +55 -25
  73. package/src/orchestrator/plan.ts +24 -11
  74. package/src/orchestrator/plugin-host.ts +17 -23
  75. package/src/orchestrator/plugin.ts +31 -8
  76. package/src/orchestrator/prepare.ts +99 -13
  77. package/src/orchestrator/projects.ts +19 -8
  78. package/src/orchestrator/remote-prefetch.ts +21 -1
  79. package/src/orchestrator/run-artifacts.ts +9 -0
  80. package/src/orchestrator/run-context.ts +19 -4
  81. package/src/orchestrator/run-lock.ts +349 -0
  82. package/src/orchestrator/run-records.ts +6 -23
  83. package/src/orchestrator/run-report.ts +4 -2
  84. package/src/orchestrator/run.ts +264 -57
  85. package/src/orchestrator/sandbox-request.ts +388 -62
  86. package/src/orchestrator/save-lane.ts +4 -7
  87. package/src/orchestrator/shell-verdict.ts +117 -0
  88. package/src/orchestrator/signals.ts +86 -30
  89. package/src/orchestrator/stable-keys.ts +202 -10
  90. package/src/orchestrator/status-line.ts +33 -1
  91. package/src/orchestrator/summary.ts +49 -2
  92. package/src/orchestrator/task-hash.ts +87 -7
  93. package/src/orchestrator/task-log-buffer.ts +7 -7
  94. package/src/orchestrator/telemetry-host.ts +16 -7
  95. package/src/orchestrator/telemetry.ts +69 -28
  96. package/src/orchestrator/upstream.ts +91 -45
  97. package/src/util/bun-version.ts +43 -0
  98. package/src/util/errors.ts +86 -0
  99. package/src/util/hash.ts +10 -1
  100. package/src/util/index.ts +32 -3
  101. package/src/util/paths.ts +147 -5
  102. package/src/util/procfs.ts +20 -0
  103. package/src/util/settle.ts +4 -2
  104. package/src/util/size.ts +33 -0
  105. package/src/util/task-id.ts +20 -0
  106. package/src/util/timing.ts +11 -1
  107. package/src/util/which.ts +48 -0
  108. package/src/workspace/affected.ts +272 -64
  109. package/src/workspace/config-cache.ts +262 -32
  110. package/src/workspace/config-eval.ts +49 -12
  111. package/src/workspace/config-imports.ts +126 -5
  112. package/src/workspace/config-schema.ts +237 -41
  113. package/src/workspace/filter.ts +33 -18
  114. package/src/workspace/fingerprint.ts +29 -11
  115. package/src/workspace/index.ts +7 -0
  116. package/src/workspace/json-data.ts +90 -0
  117. package/src/workspace/load-reads.ts +32 -0
  118. package/src/workspace/lockfile.ts +33 -11
  119. package/src/workspace/migrate-scripts.ts +60 -7
  120. package/src/workspace/migration.ts +40 -1
  121. package/src/workspace/package-graph.ts +154 -16
  122. package/src/workspace/project-loader.ts +227 -77
  123. package/src/workspace/workspace.ts +173 -23
package/README.md CHANGED
@@ -67,7 +67,7 @@ export default defineProject({
67
67
  vx init # scaffold vx.workspace.ts + a vx.config.ts per package from scripts
68
68
  vx run build # cwd project + its workspace deps
69
69
  vx run test --all # every project that declares `test`
70
- vx run ci --affected # only what changed since origin/HEAD
70
+ vx run ci --affected # changed since origin/HEAD, and what depends on it
71
71
  vx watch lint # re-run on file changes
72
72
  vx run build --dry # show the plan, don't execute
73
73
  ```
@@ -136,9 +136,11 @@ vx lock --check && vx run ci --frozen # CI: audit, then run EXACTLY that gra
136
136
  | `vx lock --check` | full graph | compares — catches env and import drift that byte hashes cannot |
137
137
 
138
138
  Env values read at lock time are frozen by design — cache keys become
139
- reproducible across machines. Bonus: `--frozen` runs skip config
140
- evaluation entirely (~120 ms back per 1,000 packages). No other
141
- runner has an equivalent.
139
+ reproducible across machines. Take it for that, not for speed: measured
140
+ on the 1,000-project bench, a plain warm run and a frozen one are a tie
141
+ (177 ms against 165, 2026-09-12), because the config-evaluation cache
142
+ already serves a pure config without evaluating it. No other runner has
143
+ an equivalent.
142
144
 
143
145
  ## Everything you need, nothing to configure twice
144
146
 
@@ -201,17 +203,17 @@ distributed ships in this repo; the seams are how you build it.
201
203
 
202
204
  ## How it compares
203
205
 
204
- | | vx | Turborepo | Nx |
205
- | ------------------------- | ---------------------------------------------------------- | ------------------------------ | ---------------- |
206
- | Fully cached, 100 pkgs¹ | **144 ms** | 279 ms | 583+ ms |
207
- | Config | TypeScript, evaluated into the cache key | JSON (static) | JSON (static) |
208
- | Output ownership | **Strict** — wiped before exec AND restore | Additive (stale files survive) | Additive |
209
- | Clean-tree hashing | **Zero reads** (git index OIDs) | git OIDs | re-hash / daemon |
210
- | Daemon required for speed | **No** | Optional | Yes |
211
- | Per-task sandbox | **Yes** — kernel-level, opt-in | No | No |
212
- | Plugin API | **Yes** — executor / cache / telemetry seams | No | Yes (TS-tied) |
213
- | OTel CI/CD spans | **Yes** — `otel()` plugin, zero OTel-SDK deps | No | Paid |
214
- | Install | **Single binary** — npm or 1 curl line, no Node/Bun needed | npm + Node | npm + Node |
206
+ | | vx | Turborepo | Nx |
207
+ | ------------------------- | ---------------------------------------------------------------- | ------------------------------ | ---------------- |
208
+ | Fully cached, 100 pkgs¹ | **144 ms** | 279 ms | 583+ ms |
209
+ | Config | TypeScript, evaluated into the cache key | JSON (static) | JSON (static) |
210
+ | Output ownership | **Strict** — wiped before exec AND restore | Additive (stale files survive) | Additive |
211
+ | Clean-tree hashing | **Zero reads** (git index OIDs) | git OIDs | re-hash / daemon |
212
+ | Daemon required for speed | **No** | Optional | Yes |
213
+ | Per-task sandbox | **Yes** — kernel-level, opt-in | No | No |
214
+ | Plugin API | **Yes** — executor / cache / telemetry seams | No | Yes (TS-tied) |
215
+ | OTel CI/CD spans | **Yes** — `otel()` plugin, zero OTel-SDK deps | No | Paid |
216
+ | Install | **Single binary** — npm, or one release file; no Node/Bun needed | npm + Node | npm + Node |
215
217
 
216
218
  ¹ Wall-clock, direct binaries, same machine and workspace — full
217
219
  methodology and more scenarios in
@@ -306,7 +308,8 @@ maintainers' handoff is [`packages/vx/docs/STATUS.md`](packages/vx/docs/STATUS.m
306
308
  ## Status
307
309
 
308
310
  **Pre-alpha.** The schema is settling; we bump `CACHE_VERSION` rather
309
- than maintain back-compat. **~2,500 core tests plus the package suites; CI green on every commit**;
311
+ than maintain back-compat. What 1.0 will promise is in
312
+ [`versioning-1.0.md`](packages/vx/docs/design/versioning-1.0.md). **~3,000 core tests plus the package suites; CI green on every commit**;
310
313
  the project dogfoods itself (`vx run ci`). Published on npm:
311
314
  [`@vzn/vx`](https://www.npmjs.com/package/@vzn/vx) (a prebuilt standalone
312
315
  binary).
@@ -318,8 +321,8 @@ build.
318
321
 
319
322
  | Surface | Maturity | Notes |
320
323
  | -------------------------------------------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- |
321
- | Core task runner + caching | **production-ready** | dogfooded continuously; ~2,500 core tests + the package suites, green |
322
- | Plugin pipeline (9 hooks, `commands` included) | **shippable** | crash-isolated, re-validated; the local executor + cache are the floor under every plugin |
324
+ | Core task runner + caching | **production-ready** | dogfooded continuously; ~3,000 core tests + the package suites, green |
325
+ | Plugin pipeline (13 hooks, `commands` included) | **shippable** | crash-isolated, re-validated; the local executor + cache are the floor under every plugin |
323
326
  | `vx init` / `@vzn/vx-migrate` (scripts; Turbo, Nx) | **shippable** | one config per package, TODOs where a source cannot say |
324
327
  | REAPI remote cache + execution (`@vzn/vx-reapi`) | **shippable** | Bazel AC + CAS + Execute; NativeLink / BuildBuddy / Buildbarn / bazel-remote |
325
328
  | OTel export (`@vzn/vx-otel`) | **shippable** | OTLP traces + metrics + logs, zero SDK deps |
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@vzn/vx",
3
- "version": "0.0.21",
3
+ "version": "0.0.23",
4
4
  "description": "An open, extensible monorepo task runner.",
5
5
  "type": "module",
6
6
  "exports": {
@@ -17,13 +17,13 @@
17
17
  "node": ">=18"
18
18
  },
19
19
  "optionalDependencies": {
20
- "@vzn/vx-linux-x64": "0.0.21",
21
- "@vzn/vx-linux-arm64": "0.0.21",
22
- "@vzn/vx-darwin-x64": "0.0.21",
23
- "@vzn/vx-darwin-arm64": "0.0.21"
20
+ "@vzn/vx-linux-x64": "0.0.23",
21
+ "@vzn/vx-linux-arm64": "0.0.23",
22
+ "@vzn/vx-darwin-x64": "0.0.23",
23
+ "@vzn/vx-darwin-arm64": "0.0.23"
24
24
  },
25
25
  "dependencies": {
26
- "@anthropic-ai/sandbox-runtime": "^0.0.75"
26
+ "@anthropic-ai/sandbox-runtime": "^0.0.76"
27
27
  },
28
28
  "files": [
29
29
  "index.ts",
package/src/bin.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  #!/usr/bin/env bun
2
2
  import { registerCoreAlias, run } from './cli/index.js'
3
- import { isUserError } from './util/index.js'
3
+ import { fsRefusalHint, isFsRefusal, isUserError } from './util/index.js'
4
4
 
5
5
  // Every `import … from '@vzn/vx'` this process evaluates — a plugin
6
6
  // package, a workspace or project config — resolves to THIS core, not to
@@ -10,9 +10,23 @@ registerCoreAlias(() => import('./index.js') as Promise<Record<string, unknown>>
10
10
 
11
11
  // Wrapped in an explicit async main so `bun build --compile` accepts
12
12
  // the file. The compile target doesn't allow top-level await.
13
+ let settled = false
14
+
13
15
  async function main(): Promise<void> {
14
16
  try {
15
- process.exit(await run(process.argv.slice(2)))
17
+ const code = await run(process.argv.slice(2))
18
+ // NOTHING calls `process.exit` here, and that is the fix rather than a
19
+ // simplification. Bun drops what a pipe has not yet taken when
20
+ // `process.exit` follows a large write: 300 KB written then exit
21
+ // delivered 64 KiB (128 KiB after a tick), and `vx history --format
22
+ // json` on a 300-project workspace was cut mid-string (2026-09-15).
23
+ // `stdout.end(cb)` fixed that on 1.4.2 — and on 1.3.11 the callback
24
+ // still fires early: 2 MiB written, 214 KB delivered (2026-09-20).
25
+ // Setting the code and letting the loop drain is the one form that
26
+ // does not depend on when a runtime decides a pipe is flushed; it
27
+ // costs a hang if a verb leaves a handle open, which the suite's
28
+ // several hundred spawns of this binary would show at once.
29
+ process.exitCode = code
16
30
  } catch (err) {
17
31
  // UserError (workspace not found, cycle, config invalid, ...) —
18
32
  // print the message only; the stack is noise the user can't act
@@ -24,12 +38,47 @@ async function main(): Promise<void> {
24
38
  // it produced `vx: vx why: …` (walkthrough, 2026-09-04).
25
39
  const m = err.message
26
40
  process.stderr.write(m.startsWith('vx ') ? `${m}\n` : `vx: ${m}\n`)
41
+ } else if (isFsRefusal(err)) {
42
+ // The file system's refusal names the path; the stack would name
43
+ // the verb's write, which the reader cannot act on either.
44
+ process.stderr.write(`vx: ${err.message} — ${fsRefusalHint(err)}\n`)
27
45
  } else {
28
46
  const message = err instanceof Error ? (err.stack ?? err.message) : String(err)
29
47
  process.stderr.write(`vx: ${message}\n`)
30
48
  }
31
- process.exit(1)
49
+ // Same reason as the success path above: a large stderr is truncated by
50
+ // `process.exit` too, and an error message cut in half is the one a
51
+ // reader most needs whole.
52
+ process.exitCode = 1
32
53
  }
54
+ settled = true
55
+ }
56
+
57
+ // The loop drained while the verb was still pending: something it awaits
58
+ // can never settle (a plugin hook whose promise is never resolved), and
59
+ // with no exit code set Bun exits 0 — a run whose task failed reported
60
+ // green, and its task had not even run (item 921). An exit that no
61
+ // verdict reached is a failure, and says so.
62
+ process.on('beforeExit', () => {
63
+ if (settled) return
64
+ settled = true
65
+ process.stderr.write(
66
+ 'vx: the run stopped before it finished: something it awaited can never settle (a plugin hook?)\n',
67
+ )
68
+ process.exitCode = 1
69
+ })
70
+
71
+ // A reader that leaves is not the run's failure. `vx run build | head -1`
72
+ // closes the pipe after one line; every later write gets EPIPE, which Bun
73
+ // raises as an `error` event on the stream, and an `error` nobody listens
74
+ // for is an uncaught exception: the run died with a stack after its task
75
+ // had succeeded, exit 1, its lock directory left behind (2026-09-16). With
76
+ // a listener the write returns false, the run finishes, saves, releases,
77
+ // and exits with its own verdict; what was going to the reader goes
78
+ // nowhere. Same for stderr (`2>&1 | head`). Here and not in the logger:
79
+ // an embedder's streams are the embedder's.
80
+ for (const stream of [process.stdout, process.stderr]) {
81
+ stream.on('error', () => {})
33
82
  }
34
83
 
35
84
  void main()
@@ -38,7 +38,7 @@
38
38
  // otherwise silently loses the claim rather than acting on it.
39
39
 
40
40
  import { chmodSync, renameSync, statSync, utimesSync } from 'node:fs'
41
- import { mkdir, realpath, rmdir, stat, unlink, writeFile } from 'node:fs/promises'
41
+ import { mkdir, readlink, realpath, rmdir, stat, unlink, writeFile } from 'node:fs/promises'
42
42
  import path from 'node:path'
43
43
  import { UserError } from '../util/index.js'
44
44
  import { TarFormatError, type TarInput, tarEntries, tarPack, tarSize } from './tar-stream.js'
@@ -47,7 +47,7 @@ import { TarFormatError, type TarInput, tarEntries, tarPack, tarSize } from './t
47
47
  const META_ENTRY = '.vx-meta.json'
48
48
 
49
49
  /** Archive entry name of the always-present stdout record. */
50
- export const STDOUT_ENTRY = 'stdout'
50
+ const STDOUT_ENTRY = 'stdout'
51
51
 
52
52
  /**
53
53
  * Sidecar shape. `files` maps an entry name to `[mode, mtimeMs]` — a
@@ -59,6 +59,14 @@ interface MetaFile {
59
59
  /** Bumped only if the sidecar's own shape changes; the artifact
60
60
  * container is versioned by CACHE_VERSION, which gates reads. */
61
61
  version: 1
62
+ /**
63
+ * The cache key the artifact was packed under (v35, item 943). Ingest
64
+ * refuses bytes whose key is not the one it asked for: nothing else ties
65
+ * an artifact to its key, so a remote layer that answered one key with
66
+ * another's bytes (a truncated or colliding mapping) replayed the other
67
+ * task's outputs under a green `cache-hit-remote`.
68
+ */
69
+ key?: string
62
70
  files: Record<string, [mode: number, mtimeMs: number]>
63
71
  /**
64
72
  * What the PRODUCING execution used (2026-09-12, additive: an artifact
@@ -102,6 +110,8 @@ export class ArchiveSecurityError extends Error {
102
110
 
103
111
  /** What `packArtifactStream` packs: stdout plus archive name → absolute source path. */
104
112
  export interface PackArgs {
113
+ /** The cache key, recorded in the sidecar (item 943); ingest refuses an artifact without it. */
114
+ key?: string
105
115
  stdout: string
106
116
  outputs: ReadonlyMap<string, string>
107
117
  /** The producing execution's usage, when the runner reported it. */
@@ -124,6 +134,7 @@ export interface ArtifactPlan {
124
134
  */
125
135
  export async function planArtifact(args: PackArgs): Promise<ArtifactPlan> {
126
136
  const meta: MetaFile = { version: 1, files: {} }
137
+ if (args.key !== undefined) meta.key = args.key
127
138
  const exec = usageOf(args.exec)
128
139
  if (exec !== undefined) meta.exec = exec
129
140
  const files = await Promise.all(
@@ -136,14 +147,14 @@ export async function planArtifact(args: PackArgs): Promise<ArtifactPlan> {
136
147
  const st = await stat(abs).catch((err: NodeJS.ErrnoException) => {
137
148
  if (err.code === 'ENOENT') {
138
149
  throw new UserError(
139
- `output ${shown} is a dangling symlink: vx stores regular files only — emit a file, or exclude it from cache.outputs`,
150
+ `output ${shown} is a dangling symlink: vx stores regular files only — emit a file there, or narrow cache.outputs.files to the files the task produces (output globs take no '!')`,
140
151
  )
141
152
  }
142
153
  throw err
143
154
  })
144
155
  if (!st.isFile()) {
145
156
  throw new UserError(
146
- `output ${shown} is not a regular file (a symlink to a directory?): vx stores regular files only — emit a file, or exclude it from cache.outputs`,
157
+ `output ${shown} is not a regular file (a symlink to a directory?): vx stores regular files only — emit a file there, or narrow cache.outputs.files to the files the task produces (output globs take no '!')`,
147
158
  )
148
159
  }
149
160
  meta.files[name] = [st.mode & 0o777, Math.floor(st.mtimeMs)]
@@ -224,12 +235,16 @@ function streamOf(gen: AsyncGenerator<Uint8Array>): ReadableStream<Uint8Array> {
224
235
  * name is unsafe — a partial reading of a poisoned artifact is the
225
236
  * outcome the traversal defense exists to prevent.
226
237
  */
227
- export async function scanArtifact(
228
- tar: ReadableStream<Uint8Array>,
229
- ): Promise<{ entries: ArchiveEntry[]; stdout: string | null; exec: ExecUsage | undefined }> {
238
+ export async function scanArtifact(tar: ReadableStream<Uint8Array>): Promise<{
239
+ entries: ArchiveEntry[]
240
+ stdout: string | null
241
+ exec: ExecUsage | undefined
242
+ key: string | undefined
243
+ }> {
230
244
  const seen: Array<{ name: string; size: number; mtimeMs: number }> = []
231
245
  let meta: MetaFile['files'] = {}
232
246
  let exec: ExecUsage | undefined
247
+ let key: string | undefined
233
248
  let stdout: string | null = null
234
249
  for await (const e of tarEntries(tar)) {
235
250
  if (e.type !== '0') continue
@@ -239,6 +254,7 @@ export async function scanArtifact(
239
254
  // The ingest side is the untrusted boundary: a foreign sidecar's
240
255
  // usage is taken only when it is a plain non-negative number.
241
256
  exec = usageOf(parsed.exec)
257
+ if (typeof parsed.key === 'string') key = parsed.key
242
258
  continue
243
259
  }
244
260
  assertSafeName(e.name)
@@ -255,6 +271,7 @@ export async function scanArtifact(
255
271
  }),
256
272
  stdout,
257
273
  exec,
274
+ key,
258
275
  }
259
276
  }
260
277
 
@@ -286,7 +303,10 @@ export async function extractArtifactStream(
286
303
  workspaceDest: string | undefined,
287
304
  verify?: (provided: ReadonlySet<string>) => void,
288
305
  ): Promise<Set<string>> {
289
- await mkdir(destDir, { recursive: true })
306
+ // No `mkdir -p` of the destination here: `stage` creates each entry's
307
+ // directory chain, the destination included, and the cache restores
308
+ // into a project directory that exists. The eager one cost every
309
+ // restore a thread-pool round trip for an EEXIST and a stat (item 627).
290
310
  const x = new Extractor(destDir, workspaceDest)
291
311
  const provided = new Set<string>()
292
312
  const headerMtime = new Map<string, number>()
@@ -305,7 +325,12 @@ export async function extractArtifactStream(
305
325
  await x.assertContained(dest.base, target, e.name)
306
326
  provided.add(e.name)
307
327
  headerMtime.set(e.name, e.mtimeMs)
308
- await x.stage(e.name, target, e.size <= SMALL_ENTRY ? await bytesOf(e.body) : e.body)
328
+ await x.stage(
329
+ e.name,
330
+ dest.base,
331
+ target,
332
+ e.size <= SMALL_ENTRY ? await bytesOf(e.body) : e.body,
333
+ )
309
334
  }
310
335
  verify?.(provided)
311
336
  await x.commit((name) => {
@@ -448,22 +473,21 @@ class Extractor {
448
473
  }
449
474
  const parentResolved = path.dirname(targetResolved)
450
475
  if (this.containedDirs.has(parentResolved)) return
451
- await this.assertParentContained(base, targetResolved, name)
476
+ await this.assertParentContained(base, targetResolved)
452
477
  this.containedDirs.add(parentResolved)
453
478
  }
454
479
 
455
- private async assertParentContained(
456
- base: string,
457
- targetResolved: string,
458
- name: string,
459
- ): Promise<void> {
480
+ private async assertParentContained(base: string, targetResolved: string): Promise<void> {
460
481
  const baseResolved = path.resolve(base)
461
- const realBase = await this.realBaseOf(base)
462
482
  // Only ancestors strictly BELOW the base are candidates — those are the
463
483
  // ones a poisoned entry could follow out of the tree. The walk must never
464
484
  // climb past the base: when the base itself does not exist yet (the
465
485
  // workspace-outputs anchor is created lazily), its parent legitimately
466
486
  // resolves outside and comparing against it would reject every entry.
487
+ // The base's own real path is resolved only once an ancestor EXISTS to
488
+ // compare it with: after a clean pruned the output directories (the
489
+ // common restore) nothing below the base is there, and the memoised
490
+ // resolve was a round trip spent on a comparison never made (item 627).
467
491
  let probe = path.dirname(targetResolved)
468
492
  while (probe.startsWith(baseResolved + path.sep)) {
469
493
  const real = await realpath(probe).then(
@@ -471,10 +495,9 @@ class Extractor {
471
495
  () => null,
472
496
  )
473
497
  if (real !== null) {
498
+ const realBase = await this.realBaseOf(base)
474
499
  if (real !== realBase && !real.startsWith(realBase + path.sep)) {
475
- throw new ArchiveSecurityError(
476
- `archive entry escapes destDir via a symlinked parent: ${name}`,
477
- )
500
+ throw await linkOutError(baseResolved, probe, realBase)
478
501
  }
479
502
  return
480
503
  }
@@ -484,6 +507,33 @@ class Extractor {
484
507
  // pre-existing symlink to follow — the chain will be created fresh.
485
508
  }
486
509
 
510
+ /**
511
+ * `mkdir -p` does not follow a DANGLING link: it fails EEXIST at the link,
512
+ * ENOENT below it, ELOOP in a cycle — and the containment walk passed it,
513
+ * finding nothing that resolves between it and the base. A live link that
514
+ * stays inside the base is written through, so a dangling one is too: the
515
+ * directory it names is created and the entry lands through the link,
516
+ * which stays the user's (never replaced, nx#37061). One that leads out of
517
+ * the base is the refusal a live link out gets. Only the failure pays for
518
+ * the walk; a clean tree never reaches it.
519
+ */
520
+ private async mkdirThroughLink(
521
+ base: string,
522
+ dir: string,
523
+ err: NodeJS.ErrnoException,
524
+ ): Promise<string | undefined> {
525
+ if (err.code !== 'EEXIST' && err.code !== 'ENOENT' && err.code !== 'ELOOP') throw err
526
+ const real = await resolveThrough(dir)
527
+ const realBase = await this.realBaseOf(base)
528
+ if (real !== realBase && !real.startsWith(realBase + path.sep)) {
529
+ throw await linkOutError(path.resolve(base), dir, realBase)
530
+ }
531
+ // What stands there without a link on the way (a file named like the
532
+ // directory) fails this mkdir as it failed the first: the caller names
533
+ // it as a stray.
534
+ return mkdir(real, { recursive: true })
535
+ }
536
+
487
537
  /**
488
538
  * Write the entry beside its target. A buffer is written without
489
539
  * waiting, bounded by `INFLIGHT_BYTES`; a chunk stream goes to a file
@@ -491,13 +541,18 @@ class Extractor {
491
541
  */
492
542
  async stage(
493
543
  name: string,
544
+ base: string,
494
545
  target: string,
495
546
  body: Uint8Array | AsyncIterable<Uint8Array>,
496
547
  ): Promise<void> {
497
548
  const dir = path.dirname(target)
498
549
  let created: string | undefined
499
550
  if (!this.ensuredDirs.has(dir)) {
500
- created = await mkdir(dir, { recursive: true })
551
+ try {
552
+ created = await mkdir(dir, { recursive: true })
553
+ } catch (err) {
554
+ created = await this.mkdirThroughLink(base, dir, err as NodeJS.ErrnoException)
555
+ }
501
556
  this.ensuredDirs.add(dir)
502
557
  }
503
558
  // Write beside the target and RENAME into place. rename(2) replaces
@@ -517,7 +572,13 @@ class Extractor {
517
572
  //
518
573
  // A directory at the target makes the rename fail — the same
519
574
  // fail-closed outcome the plain write had.
520
- const tmp = `${target}.vx-tmp-${process.pid.toString(36)}-${(tmpSeq++).toString(36)}`
575
+ // The temp is a sibling with its own short name, never `<target>.suffix`:
576
+ // a suffix pushed a legal name of 242–255 bytes past NAME_MAX, and the
577
+ // restore of a valid artifact failed with ENAMETOOLONG (item 666).
578
+ const tmp = path.join(
579
+ path.dirname(target),
580
+ `.vx-tmp-${process.pid.toString(36)}-${(tmpSeq++).toString(36)}`,
581
+ )
521
582
  this.staged.push({ name, tmp, target, created })
522
583
  if (body instanceof Uint8Array) {
523
584
  this.inflightBytes += body.byteLength
@@ -594,6 +655,9 @@ class Extractor {
594
655
  if (s.created === undefined) continue
595
656
  let dir = path.dirname(s.target)
596
657
  const top = path.resolve(s.created)
658
+ // Created at a dangling link's resolved target (`mkdirThroughLink`),
659
+ // so the chain is pruned from where the link leads.
660
+ if (!dir.startsWith(top)) dir = await realpath(dir).catch(() => dir)
597
661
  while (dir.startsWith(top)) {
598
662
  if (
599
663
  !(await rmdir(dir).then(
@@ -618,6 +682,65 @@ let tmpSeq = 0
618
682
 
619
683
  const WORKSPACE_PREFIX = 'workspace-outputs/'
620
684
 
685
+ /**
686
+ * The on-disk half of containment failed: `probe`, a directory under
687
+ * `base`, resolves outside it, so a directory between them is a link out.
688
+ * That is the tree, not the artifact — an output directory made a link
689
+ * after the entry was saved (upstream survey, nx#37061) — so it is the
690
+ * user's to fix, and the refusal names the link and where it leads. The
691
+ * restore never writes through it and never replaces it: the link is the
692
+ * user's, and replacing it is the bug nx#37061 reports.
693
+ */
694
+ async function linkOutError(base: string, probe: string, realBase: string): Promise<UserError> {
695
+ let dir = base
696
+ let real = realBase
697
+ for (const part of path.relative(base, probe).split(path.sep)) {
698
+ dir = path.join(dir, part)
699
+ real = await resolveThrough(dir)
700
+ if (real !== realBase && !real.startsWith(realBase + path.sep)) break
701
+ }
702
+ return new UserError(
703
+ `${dir} is a symbolic link to ${real}, outside ${base} — a cache restore never writes ` +
704
+ 'through a link that leaves its directory. Remove the link and re-run (the restore ' +
705
+ 'puts a real directory there), or stop declaring outputs under it.',
706
+ )
707
+ }
708
+
709
+ /**
710
+ * Where `p` leads, every link on the way followed, a dangling one included:
711
+ * `realpath` for the part that exists, and past the first missing name the
712
+ * rest as written — where `mkdir -p` would create it if it followed links.
713
+ * A cycle is the user's tree, not the artifact, so it is refused by name.
714
+ */
715
+ async function resolveThrough(p: string): Promise<string> {
716
+ const pending = path.resolve(p).split(path.sep).filter(Boolean).reverse()
717
+ let at: string = path.sep
718
+ let hops = 0
719
+ while (pending.length > 0) {
720
+ const next = path.join(at, pending.pop()!)
721
+ const link = await readlink(next).catch((err: NodeJS.ErrnoException) => {
722
+ if (err.code === 'EINVAL') return null
723
+ if (err.code === 'ENOENT' || err.code === 'ENOTDIR') return undefined
724
+ throw err
725
+ })
726
+ if (link === undefined) return path.join(next, ...pending.reverse())
727
+ if (link === null) {
728
+ at = next
729
+ continue
730
+ }
731
+ // Linux's own bound on links followed in one lookup (MAXSYMLINKS).
732
+ if (++hops > 40) {
733
+ throw new UserError(
734
+ `${p} goes through a cycle of symbolic links (at ${next}) — a cache restore cannot ` +
735
+ 'write through it. Remove the link and re-run.',
736
+ )
737
+ }
738
+ if (path.isAbsolute(link)) at = path.sep
739
+ pending.push(...link.split(path.sep).filter(Boolean).reverse())
740
+ }
741
+ return at
742
+ }
743
+
621
744
  /**
622
745
  * Path-traversal defense, applied to every entry name at read time.
623
746
  * Mirrors Turbo's `restore.rs:check_path` normalization: an artifact is
@@ -651,8 +774,34 @@ function assertSafeName(name: string): void {
651
774
  if (name.includes('\0')) {
652
775
  throw new ArchiveSecurityError(`archive entry name contains a null byte (unsafe): ${name}`)
653
776
  }
777
+ // Only a pax `path` record can carry a name this long. Left to the file
778
+ // system it is a raw ENAMETOOLONG, which the restore reports as an
779
+ // internal error rather than a refused artifact. A UTF-16 unit is at most
780
+ // three UTF-8 bytes, so the common short name never pays for the count.
781
+ if (name.length * 3 >= PATH_MAX && Buffer.byteLength(name) >= PATH_MAX) {
782
+ throw new ArchiveSecurityError(
783
+ `archive entry name is ${Buffer.byteLength(name)} bytes, past PATH_MAX (${PATH_MAX}) (unsafe): ${name.slice(0, 64)}…`,
784
+ )
785
+ }
786
+ // The same raw ENAMETOOLONG for one component past NAME_MAX, the same on
787
+ // Linux and macOS. Same cheap bound: a name under 86 units cannot hold one.
788
+ if (name.length * 3 > NAME_MAX) {
789
+ for (const part of name.split('/')) {
790
+ if (part.length * 3 > NAME_MAX && Buffer.byteLength(part) > NAME_MAX) {
791
+ throw new ArchiveSecurityError(
792
+ `archive entry has a ${Buffer.byteLength(part)}-byte component, past NAME_MAX (${NAME_MAX}) (unsafe): ${part.slice(0, 64)}…`,
793
+ )
794
+ }
795
+ }
796
+ }
654
797
  }
655
798
 
799
+ /** Bytes one path component may hold (limits.h), on Linux and macOS alike. */
800
+ const NAME_MAX = 255
801
+
802
+ /** Bytes a path may hold, its terminating NUL included (limits.h). */
803
+ const PATH_MAX = process.platform === 'darwin' ? 1024 : 4096
804
+
656
805
  function hasParentSegment(p: string): boolean {
657
806
  if (p === '..' || p.startsWith('../') || p.endsWith('/..')) return true
658
807
  return p.includes('/../')