@vzn/vx 0.0.641 → 0.0.643

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/package.json +5 -12
  2. package/skills/vx/SKILL.md +4 -4
  3. package/src/cache/archive.ts +525 -104
  4. package/src/cache/cache.ts +85 -11
  5. package/src/cache/fold-memo.ts +58 -0
  6. package/src/cache/git-inputs.ts +213 -31
  7. package/src/cache/index.ts +12 -3
  8. package/src/cache/inputs.ts +169 -79
  9. package/src/cache/key-fold.ts +90 -69
  10. package/src/cache/layer.ts +4 -6
  11. package/src/cache/layered-cache.ts +96 -10
  12. package/src/cache/output-index.ts +25 -4
  13. package/src/cache/plugin-cache.ts +23 -16
  14. package/src/cache/remote-lane.ts +25 -8
  15. package/src/cache/run-store.ts +4 -1
  16. package/src/cache/schema.ts +13 -3
  17. package/src/cache/scoped-cache.ts +74 -237
  18. package/src/cache/umask.ts +29 -0
  19. package/src/cli/docs.ts +71 -7
  20. package/src/cli/help.ts +6 -8
  21. package/src/cli/plan-format.ts +4 -1
  22. package/src/cli/plugin-commands.ts +11 -6
  23. package/src/cli/run.ts +14 -15
  24. package/src/cli/select.ts +9 -9
  25. package/src/cli/workspace-config.ts +8 -2
  26. package/src/config.ts +7 -16
  27. package/src/exec/runner.ts +5 -0
  28. package/src/exec/sandbox-runtime.ts +81 -1
  29. package/src/graph/priorities.ts +14 -1
  30. package/src/history/history.ts +43 -33
  31. package/src/index.ts +14 -0
  32. package/src/orchestrator/affected-tasks.ts +23 -2
  33. package/src/orchestrator/deferred-outputs.ts +3 -2
  34. package/src/orchestrator/execute-task.ts +20 -14
  35. package/src/orchestrator/hit-restore.ts +10 -2
  36. package/src/orchestrator/index.ts +5 -0
  37. package/src/orchestrator/lockfile-claim.ts +54 -8
  38. package/src/orchestrator/miss-save.ts +27 -17
  39. package/src/orchestrator/options.ts +9 -1
  40. package/src/orchestrator/output-host.ts +9 -1
  41. package/src/orchestrator/plan.ts +4 -0
  42. package/src/orchestrator/plugin-host.ts +38 -4
  43. package/src/orchestrator/plugin.ts +49 -5
  44. package/src/orchestrator/prepare.ts +141 -22
  45. package/src/orchestrator/projects.ts +25 -6
  46. package/src/orchestrator/run-context.ts +80 -0
  47. package/src/orchestrator/run.ts +38 -15
  48. package/src/orchestrator/sandbox-request.ts +1 -48
  49. package/src/orchestrator/save-lane.ts +17 -10
  50. package/src/orchestrator/signals.ts +7 -10
  51. package/src/orchestrator/stable-keys.ts +83 -25
  52. package/src/orchestrator/task-hash.ts +52 -12
  53. package/src/report/events.ts +12 -0
  54. package/src/report/framed-output.ts +2 -1
  55. package/src/report/logger.ts +53 -13
  56. package/src/report/status-line.ts +45 -4
  57. package/src/report/summary.ts +8 -2
  58. package/src/settings.ts +11 -11
  59. package/src/util/errors.ts +1 -1
  60. package/src/util/hidden-prompt.ts +74 -0
  61. package/src/util/index.ts +12 -2
  62. package/src/util/paths.ts +33 -14
  63. package/src/util/secrets.ts +150 -0
  64. package/src/workspace/affected.ts +182 -75
  65. package/src/workspace/config-schema.ts +81 -19
  66. package/src/workspace/index.ts +3 -0
  67. package/src/workspace/project-loader.ts +1 -0
  68. package/src/workspace/workspace.ts +17 -11
  69. package/docs/caching.md +0 -2376
  70. package/docs/cli.md +0 -3285
  71. package/docs/execution.md +0 -676
  72. package/docs/features.md +0 -178
  73. package/docs/patterns.md +0 -219
  74. package/docs/schema.md +0 -2468
  75. package/docs/security.md +0 -253
  76. package/src/cli/docs-corpus.ts +0 -21
  77. package/src/cli/docs-text.d.ts +0 -5
@@ -34,10 +34,10 @@
34
34
  // cache artifact is attacker-reachable (a poisoned remote, a tampered
35
35
  // local entry) and "zip slip" is the class.
36
36
  //
37
- // Entries that are not regular files (symlinks, hardlinks, devices,
37
+ // Tar entries that are not regular files (symlinks, hardlinks, devices,
38
38
  // FIFOs) are never materialised — the reader reports them only to be
39
- // skipped — vx's outputs are regular files, and an artifact that claims
40
- // otherwise silently loses the claim rather than acting on it.
39
+ // skipped. An output symlink rides the sidecar instead (`links`), its
40
+ // target held to the workspace on both sides.
41
41
 
42
42
  import {
43
43
  chmodSync,
@@ -51,11 +51,13 @@ import {
51
51
  lstatSync,
52
52
  openSync,
53
53
  readFileSync,
54
+ readlinkSync,
54
55
  realpathSync,
55
56
  statSync,
56
57
  utimesSync,
57
58
  writeFileSync,
58
59
  } from 'node:fs'
60
+ import type { Stats } from 'node:fs'
59
61
  import {
60
62
  copyFile,
61
63
  lstat,
@@ -63,8 +65,10 @@ import {
63
65
  readlink,
64
66
  realpath,
65
67
  rename,
68
+ rm,
66
69
  rmdir,
67
70
  stat,
71
+ symlink,
68
72
  unlink,
69
73
  writeFile,
70
74
  } from 'node:fs/promises'
@@ -81,6 +85,7 @@ import {
81
85
  tarSize,
82
86
  } from './tar-stream.js'
83
87
  import { ON_THREAD_MAX } from './zstd.js'
88
+ import { readUmask, restoredMode } from './umask.js'
84
89
 
85
90
  /** Archive entry name carrying the per-output mode/mtime sidecar. */
86
91
  const META_ENTRY = '.vx-meta.json'
@@ -195,6 +200,18 @@ interface MetaFile {
195
200
  */
196
201
  key?: string
197
202
  files: Record<string, [mode: number, mtimeMs: number]>
203
+ /**
204
+ * The empty directories under the outputs, as entry names
205
+ * (`outputs/dist/empty`): a tar of regular files cannot hold them, and
206
+ * a hit that dropped them left another tree than the miss (v44).
207
+ */
208
+ dirs?: string[]
209
+ /**
210
+ * The symlinks under the outputs, entry name → the link's target as
211
+ * written (relative, inside the workspace): a hit makes the link, as
212
+ * Turbo and Nx do, where it once packed a copy of the target (v45).
213
+ */
214
+ links?: Record<string, string>
198
215
  /**
199
216
  * What the PRODUCING execution used (2026-09-12, additive: an artifact
200
217
  * without it reads as before). Rides the artifact so a machine that
@@ -241,22 +258,12 @@ interface PackArgs {
241
258
  key?: string
242
259
  stdout: string
243
260
  outputs: ReadonlyMap<string, string>
261
+ /** Entry names of the empty directories under the outputs. */
262
+ dirs?: readonly string[] | undefined
244
263
  /** The producing execution's usage, when the runner reported it. */
245
264
  exec?: ExecUsage | undefined
246
- /**
247
- * The project directory a symlinked output must resolve inside. vx reads
248
- * outputs outside the task's sandbox, so a link a sandboxed task planted
249
- * to a file it could not read (another project's, the user's home) packed
250
- * that file into the artifact and the remote (L-23). A link to another
251
- * output of the same artifact is packed wherever it is.
252
- */
253
- within?: string
254
- /**
255
- * For a sandboxed task: whether it could read this canonical path. A
256
- * symlinked output inside `within` whose target it could not read is
257
- * refused like one outside it.
258
- */
259
- readable?: ((real: string) => boolean) | undefined
265
+ /** The workspace root an output symlink's target must stay inside; absent, every link is refused. */
266
+ root?: string
260
267
  }
261
268
 
262
269
  /** The largest second count ustar's 12-byte mtime field holds: 11 octal digits. */
@@ -276,8 +283,7 @@ export interface ArtifactPlan {
276
283
  * mtime) and lay out the tar — stdout first, the outputs, the sidecar
277
284
  * last — with its exact size, so the caller can choose the one-call pack
278
285
  * for a small artifact and the streamed one for a large one before a
279
- * byte is read. A symlinked output is stored as its target's content,
280
- * when that target is inside `within`.
286
+ * byte is read. A symlinked output is stored as its link (`links`).
281
287
  */
282
288
  export async function planArtifact(args: PackArgs): Promise<ArtifactPlan> {
283
289
  const meta: MetaFile = { version: 1, files: {} }
@@ -285,29 +291,13 @@ export async function planArtifact(args: PackArgs): Promise<ArtifactPlan> {
285
291
  // cost more than the stat. Many go to the pool together, as before.
286
292
  const few = args.outputs.size <= ON_THREAD_STATS
287
293
  const lstatOf = few ? async (p: string) => lstatSync(p) : lstat
288
- const statOf = few ? async (p: string) => statSync(p) : stat
289
- const realpathOf = (p: string) => realOf(p, few)
290
- let withinReal: Promise<string> | undefined
291
- // Where this artifact's own non-link outputs really are: a link to one of
292
- // them packs bytes the task wrote itself. A root-anchored output tree is
293
- // outside every project, so `gen/latest -> v2.txt` there was refused and
294
- // the task never cached (J's lead).
295
- let ownReal: Promise<Set<string>> | undefined
296
- const ownOutputs = (): Promise<Set<string>> =>
297
- (ownReal ??= Promise.all(
298
- [...args.outputs.values()].map(async (abs) =>
299
- (await lstatOf(abs)).isSymbolicLink() ? null : realpathOf(abs),
300
- ),
301
- ).then((all) => new Set(all.filter((p): p is string => p !== null))))
294
+ const readlinkOf = few ? async (p: string) => readlinkSync(p) : readlink
302
295
  if (args.key !== undefined) meta.key = args.key
296
+ if (args.dirs !== undefined && args.dirs.length > 0) meta.dirs = [...args.dirs]
303
297
  const exec = usageOf(args.exec)
304
298
  if (exec !== undefined) meta.exec = exec
305
299
  const files = await Promise.all(
306
300
  [...args.outputs].map(async ([name, abs]) => {
307
- // `stat` follows a symlink, so a link to a file is packed as that
308
- // file's bytes and comes back as a regular file. A link to a
309
- // directory, or a dangling one, has no bytes to pack: refuse loudly
310
- // rather than store an entry that restores to nothing.
311
301
  // The path as the config spells it: a root-anchored output's entry
312
302
  // name is vx's own `workspace-outputs/…` (M-66).
313
303
  const shown = name.startsWith('outputs/')
@@ -325,45 +315,33 @@ export async function planArtifact(args: PackArgs): Promise<ArtifactPlan> {
325
315
  `${shown} is under workspace-outputs/, a name vx's artifacts reserve for outputs.workspaceFiles — write the task's files to another directory, or take them back with a '!' entry`,
326
316
  )
327
317
  }
328
- const dangling = (err: NodeJS.ErrnoException): never => {
329
- if (err.code === 'ENOENT') {
318
+ const st = await lstatOf(abs)
319
+ if (st.isSymbolicLink()) {
320
+ // Kept as the link, never read through: no byte of its target
321
+ // enters the artifact, so a link to a file the task could not
322
+ // read packs nothing of it (L-23).
323
+ const to = await readlinkOf(abs)
324
+ if (path.isAbsolute(to)) {
330
325
  throw new UserError(
331
- `${shown} is a dangling symlink: vx stores regular files only — emit a file there, or narrow cache.outputs.files to the files the task produces, or take it back with a '!' entry`,
326
+ `${shown} is a symlink to an absolute path (${to}): vx restores a link as written, and an absolute one names this machine's checkout — make it relative, or take it back with a '!' entry`,
332
327
  )
333
328
  }
334
- throw err
335
- }
336
- // lstat, so a regular file costs no second call; only a link is
337
- // resolved, and its body is read from the path this check resolved.
338
- let src = abs
339
- let st = await lstatOf(abs)
340
- if (st.isSymbolicLink()) {
341
- src = await realpathOf(abs).catch(dangling)
342
- if (args.within !== undefined) {
343
- withinReal ??= realpathOf(args.within)
344
- const root = await withinReal
345
- if (!src.startsWith(root + path.sep) && !(await ownOutputs()).has(src)) {
346
- throw new UserError(
347
- `${shown} is a symlink to ${src}, outside the project: vx packs a symlinked output as its target's bytes, and a project's outputs come from its own directory — emit a file there, or take it back with a '!' entry`,
348
- )
349
- }
350
- }
351
- if (args.readable !== undefined && !args.readable(src) && !(await ownOutputs()).has(src)) {
329
+ if (args.root === undefined || !inside(path.resolve(path.dirname(abs), to), args.root)) {
352
330
  throw new UserError(
353
- `${shown} is a symlink to ${src}, which the task's sandbox does not let it read: vx packs a symlinked output as its target's bytes — emit a file there, grant the target in exec.sandbox.allow.read, or take it back with a '!' entry`,
331
+ `${shown} is a symlink to ${to}, outside the workspace: vx restores a link as written, and a project's outputs stay in its workspace — emit a file there, or take it back with a '!' entry`,
354
332
  )
355
333
  }
356
- st = await statOf(src).catch(dangling)
334
+ return { name, link: to }
357
335
  }
358
336
  if (!st.isFile()) {
359
337
  throw new UserError(
360
- `${shown} is not a regular file (a symlink to a directory?): vx stores regular files only — emit a file there, or narrow cache.outputs.files to the files the task produces, or take it back with a '!' entry`,
338
+ `${shown} is not a regular file: vx stores regular files and symlinks only — emit a file there, or narrow cache.outputs.files to the files the task produces, or take it back with a '!' entry`,
361
339
  )
362
340
  }
363
341
  const mode = st.mode & 0o777
364
342
  return {
365
343
  name,
366
- abs: src,
344
+ abs,
367
345
  size: st.size,
368
346
  mode,
369
347
  mtimeMs: Math.floor(st.mtimeMs),
@@ -375,12 +353,19 @@ export async function planArtifact(args: PackArgs): Promise<ArtifactPlan> {
375
353
  }),
376
354
  )
377
355
  // In the outputs' order, not as each stat settled: one tree, one artifact.
378
- for (const f of files) meta.files[f.name] = [f.mode, f.mtimeMs]
356
+ const regular: Array<Exclude<(typeof files)[number], { link: string }>> = []
357
+ for (const f of files) {
358
+ if ('link' in f) (meta.links ??= {})[f.name] = f.link
359
+ else {
360
+ meta.files[f.name] = [f.mode, f.mtimeMs]
361
+ regular.push(f)
362
+ }
363
+ }
379
364
  const stdout = new TextEncoder().encode(args.stdout)
380
365
  const metaBytes = new TextEncoder().encode(JSON.stringify(meta))
381
366
  const inputs: TarInput[] = [
382
367
  { name: STDOUT_ENTRY, size: stdout.byteLength, body: stdout },
383
- ...files.map((f) => ({
368
+ ...regular.map((f) => ({
384
369
  name: f.name,
385
370
  size: f.size,
386
371
  mode: f.mode,
@@ -489,6 +474,10 @@ function streamOf(gen: AsyncGenerator<Uint8Array>): ReadableStream<Uint8Array> {
489
474
  /** What a scan of an artifact reads out of it. */
490
475
  interface ArtifactScan {
491
476
  entries: ArchiveEntry[]
477
+ /** The sidecar's empty directories, each name proven safe. */
478
+ dirs: string[]
479
+ /** The sidecar's symlinks, `[name, target]`, each name and target proven safe. */
480
+ links: Array<[name: string, target: string]>
492
481
  stdout: string | null
493
482
  exec: ExecUsage | undefined
494
483
  key: string | undefined
@@ -498,6 +487,8 @@ interface ArtifactScan {
498
487
  class ScanFold {
499
488
  private readonly seen: Array<{ name: string; size: number; mtimeMs: number }> = []
500
489
  private files: MetaFile['files'] = {}
490
+ private dirs: string[] = []
491
+ private links: Array<[string, string]> = []
501
492
  private exec: ExecUsage | undefined
502
493
  private key: string | undefined
503
494
  stdout: string | null = null
@@ -505,6 +496,8 @@ class ScanFold {
505
496
  sidecar(text: string): void {
506
497
  const parsed = JSON.parse(text) as MetaFile
507
498
  this.files = parsed.files ?? {}
499
+ this.dirs = dirsOf(parsed)
500
+ this.links = linksOf(parsed)
508
501
  // The ingest side is the untrusted boundary: a foreign sidecar's
509
502
  // usage is taken only when it is a plain non-negative number.
510
503
  this.exec = usageOf(parsed.exec)
@@ -524,6 +517,8 @@ class ScanFold {
524
517
  // sidecar is metadata, never the authority on whether bytes exist.
525
518
  return { name: s.name, size: s.size, mode: m?.[0] ?? 0o644, mtimeMs: m?.[1] ?? s.mtimeMs }
526
519
  }),
520
+ dirs: this.dirs,
521
+ links: this.links,
527
522
  stdout: this.stdout,
528
523
  exec: this.exec,
529
524
  key: this.key,
@@ -589,6 +584,56 @@ export function scanTarBytes(tar: Uint8Array): ArtifactScan {
589
584
  return fold.result()
590
585
  }
591
586
 
587
+ /**
588
+ * The sidecar's directory names, each held to the rules a file entry's
589
+ * name is (the sidecar is as attacker-reachable as the tar), and only
590
+ * under an output namespace.
591
+ */
592
+ function dirsOf(meta: MetaFile): string[] {
593
+ const raw: unknown = meta.dirs
594
+ if (raw === undefined) return []
595
+ if (!Array.isArray(raw)) throw new ArchiveSecurityError('archive sidecar dirs is not a list')
596
+ return raw.map((name: unknown) => {
597
+ if (
598
+ typeof name !== 'string' ||
599
+ !(name.startsWith('outputs/') || name.startsWith(WORKSPACE_PREFIX))
600
+ ) {
601
+ throw new ArchiveSecurityError(`archive directory is not an output: ${String(name)}`)
602
+ }
603
+ assertSafeName(name)
604
+ return name
605
+ })
606
+ }
607
+
608
+ /**
609
+ * The sidecar's links, each name held to a file entry's rules and each
610
+ * target to a relative path with no NUL: where it may point is decided at
611
+ * the restore, which knows the workspace.
612
+ */
613
+ function linksOf(meta: MetaFile): Array<[string, string]> {
614
+ const raw: unknown = meta.links
615
+ if (raw === undefined) return []
616
+ if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) {
617
+ throw new ArchiveSecurityError('archive sidecar links is not a map')
618
+ }
619
+ return Object.entries(raw as Record<string, unknown>).map(([name, to]) => {
620
+ if (!(name.startsWith('outputs/') || name.startsWith(WORKSPACE_PREFIX))) {
621
+ throw new ArchiveSecurityError(`archive link is not an output: ${name}`)
622
+ }
623
+ assertSafeName(name)
624
+ if (typeof to !== 'string' || to.length === 0 || to.includes('\0') || path.isAbsolute(to)) {
625
+ throw new ArchiveSecurityError(`archive link ${name} has an unsafe target: ${String(to)}`)
626
+ }
627
+ return [name, to]
628
+ })
629
+ }
630
+
631
+ /** `p` is `root` or under it, lexically. */
632
+ const inside = (p: string, root: string): boolean => {
633
+ const r = path.resolve(root)
634
+ return p === r || p.startsWith(r + path.sep)
635
+ }
636
+
592
637
  /** The usage worth carrying: each axis a finite non-negative number, or absent; nothing → undefined. */
593
638
  function usageOf(raw: unknown): ExecUsage | undefined {
594
639
  if (typeof raw !== 'object' || raw === null) return undefined
@@ -624,10 +669,13 @@ export async function extractArtifactStream(
624
669
  // directory chain, the destination included, and the cache restores
625
670
  // into a project directory that exists. The eager one cost every
626
671
  // restore a thread-pool round trip for an EEXIST and a stat (item 627).
672
+ readUmask()
627
673
  const x = new Extractor(destDir, workspaceDest, stageRoot)
628
674
  const provided = new Set<string>()
629
675
  const headerMtime = new Map<string, number>()
630
676
  let meta: MetaFile['files'] = {}
677
+ let dirs: string[] = []
678
+ let links: Array<[string, string]> = []
631
679
  try {
632
680
  const check = new SumCheck()
633
681
  for await (const e of tarEntries(tar)) {
@@ -635,7 +683,10 @@ export async function extractArtifactStream(
635
683
  const body = await check.body(e.name, e.body)
636
684
  if (body === null) continue
637
685
  if (e.name === META_ENTRY) {
638
- meta = (JSON.parse(await textOf(body)) as MetaFile).files ?? {}
686
+ const parsed = JSON.parse(await textOf(body)) as MetaFile
687
+ meta = parsed.files ?? {}
688
+ dirs = dirsOf(parsed)
689
+ links = linksOf(parsed)
639
690
  continue
640
691
  }
641
692
  assertSafeName(e.name)
@@ -658,6 +709,38 @@ export async function extractArtifactStream(
658
709
  await x.stage(e.name, dest.base, target, e.size <= SMALL_ENTRY ? await bytesOf(body) : body)
659
710
  }
660
711
  check.done()
712
+ for (const name of dirs) {
713
+ if (accept !== undefined && !accept(name)) {
714
+ throw new ArchiveSecurityError(
715
+ `archive directory is not one of the task's declared outputs: ${name}`,
716
+ )
717
+ }
718
+ const dest = x.destFor(name)
719
+ if (dest === null || dest.rel.length === 0) continue
720
+ const target = path.join(dest.base, dest.rel)
721
+ await x.assertContained(dest.base, target, name)
722
+ await x.makeDir(dest.base, target)
723
+ // The index rows name a directory with a trailing slash.
724
+ provided.add(`${name}/`)
725
+ }
726
+ // A link may point only inside the workspace (the project, without one).
727
+ const bound = stageRoot ?? workspaceDest ?? destDir
728
+ for (const [name, to] of links) {
729
+ if (accept !== undefined && !accept(name)) {
730
+ throw new ArchiveSecurityError(
731
+ `archive link is not one of the task's declared outputs: ${name}`,
732
+ )
733
+ }
734
+ const dest = x.destFor(name)
735
+ if (dest === null || dest.rel.length === 0) continue
736
+ const target = path.join(dest.base, dest.rel)
737
+ await x.assertContained(dest.base, target, name)
738
+ if (!inside(path.resolve(path.dirname(target), to), bound)) {
739
+ throw new ArchiveSecurityError(`archive link ${name} points out of the workspace: ${to}`)
740
+ }
741
+ await x.stageLink(name, dest.base, target, to)
742
+ provided.add(name)
743
+ }
661
744
  verify?.(provided)
662
745
  // The sidecar's stat is the truth, a mode of 000 and an mtime at or
663
746
  // before the epoch included (`SOURCE_DATE_EPOCH=0`). Without one, a
@@ -720,7 +803,7 @@ function keep(s: Staged, ino: number, mode: number, mtimeMs: number | undefined)
720
803
  if (st.ino !== ino || st.nlink !== 1) {
721
804
  throw new Error(`${s.target} changed while it was restored`)
722
805
  }
723
- if ((st.mode & 0o7777) !== (mode & 0o777)) fchmodSync(fd, mode & 0o777)
806
+ if ((st.mode & 0o7777) !== restoredMode(mode)) fchmodSync(fd, restoredMode(mode))
724
807
  if (mtimeMs !== undefined && Math.floor(st.mtimeMs) !== mtimeMs) {
725
808
  const t = new Date(mtimeMs)
726
809
  futimesSync(fd, t, t)
@@ -745,6 +828,20 @@ interface Staged {
745
828
  base: string
746
829
  /** The target already holds the entry's bytes: nothing was written, and only its mode and mtime may change. */
747
830
  same?: number
831
+ /** `tmp` was copied beside `target`, which is on another filesystem than the staging (`crossOver`). */
832
+ beside?: true
833
+ /** A symlink to this target, made at `tmp`: no mode, no mtime. */
834
+ link?: string
835
+ /** `tmp` is inside the staged copy of this missing directory, which commit renames whole. */
836
+ tree?: string
837
+ }
838
+
839
+ /** A directory missing in the tree, built in the staging directory and renamed in whole. */
840
+ interface Tree {
841
+ at: string
842
+ base: string
843
+ /** The empty directories under it the entry holds. */
844
+ dirs: string[]
748
845
  }
749
846
 
750
847
  /**
@@ -754,6 +851,8 @@ interface Staged {
754
851
  */
755
852
  class Extractor {
756
853
  private readonly staged: Staged[] = []
854
+ /** Empty output directories made, with the top of each chain created. */
855
+ private readonly madeDirs: Array<{ dir: string; created: string }> = []
757
856
  private inflight: Promise<unknown>[] = []
758
857
  private inflightBytes = 0
759
858
  private readonly realBaseCache = new Map<string, string>()
@@ -767,6 +866,11 @@ class Extractor {
767
866
  // entry's own check and its write.
768
867
  private readonly containedDirs = new Set<string>()
769
868
  private readonly ensuredDirs = new Set<string>()
869
+ private readonly trees = new Map<string, Tree>()
870
+ // Directories the containment walk found missing, which `treeFor` would
871
+ // otherwise probe again. A directory made since fails the tree's rename,
872
+ // and the entries then go in one by one.
873
+ private readonly seenAbsent = new Set<string>()
770
874
 
771
875
  constructor(
772
876
  private readonly destDir: string,
@@ -860,6 +964,7 @@ class Extractor {
860
964
  // trip and error (X-163). A dangling link is there and falls through,
861
965
  // as does an lstat that fails otherwise: the realpath decides.
862
966
  if (absent(probe)) {
967
+ this.seenAbsent.add(probe)
863
968
  probe = path.dirname(probe)
864
969
  continue
865
970
  }
@@ -907,6 +1012,82 @@ class Extractor {
907
1012
  return mkdir(real, { recursive: true })
908
1013
  }
909
1014
 
1015
+ /**
1016
+ * Where an entry in `dir` is written when a directory on the way to it is
1017
+ * missing: inside a copy of the topmost missing one, built in the staging
1018
+ * directory and renamed into place whole on commit. One rename per
1019
+ * missing directory, not one per file: every file of a restore into a
1020
+ * cleaned tree renamed across directories, all out of one staging
1021
+ * directory, and the kernel time of a 5,000-artifact restore doubled
1022
+ * (2026-10-17). A reader sees the directory complete or not at all.
1023
+ * Undefined where the group or filesystem would differ from a write in
1024
+ * place: the entry is then staged and renamed alone.
1025
+ */
1026
+ private async treeFor(base: string, dir: string): Promise<string | undefined> {
1027
+ if (this.stageRoot === undefined) return undefined
1028
+ const abs = path.resolve(dir)
1029
+ for (const [top, t] of this.trees) {
1030
+ if (abs === top || abs.startsWith(top + path.sep)) return t.at + abs.slice(top.length)
1031
+ }
1032
+ const baseResolved = path.resolve(base)
1033
+ let top: string | undefined
1034
+ let probe = abs
1035
+ while (
1036
+ probe.startsWith(baseResolved + path.sep) &&
1037
+ (this.seenAbsent.has(probe) || absent(probe))
1038
+ ) {
1039
+ top = probe
1040
+ probe = path.dirname(probe)
1041
+ }
1042
+ if (top === undefined) return undefined
1043
+ const staging = await stagingDir(this.stageRoot)
1044
+ const st = statSync(staging)
1045
+ const parent = statSync(probe, { throwIfNoEntry: false })
1046
+ if (
1047
+ parent === undefined ||
1048
+ !parent.isDirectory() ||
1049
+ parent.dev !== st.dev ||
1050
+ ((parent.mode | st.mode) & 0o2000) !== 0 ||
1051
+ (INHERITS_GROUP && parent.gid !== st.gid)
1052
+ ) {
1053
+ return undefined
1054
+ }
1055
+ const at = path.join(staging, (tmpSeq++).toString(36))
1056
+ this.trees.set(top, { at, base, dirs: [] })
1057
+ return at + abs.slice(top.length)
1058
+ }
1059
+
1060
+ /** `dir` made, once per extraction. */
1061
+ private async ensure(dir: string): Promise<void> {
1062
+ if (this.ensuredDirs.has(dir)) return
1063
+ await mkdir(dir, { recursive: true })
1064
+ this.ensuredDirs.add(dir)
1065
+ }
1066
+
1067
+ /** The tree `target` is staged in. */
1068
+ private treeOf(target: string): string | undefined {
1069
+ for (const top of this.trees.keys()) if (target.startsWith(top + path.sep)) return top
1070
+ return undefined
1071
+ }
1072
+
1073
+ /** An empty output directory: made now, pruned by `abort` if it was not there. */
1074
+ async makeDir(base: string, dir: string): Promise<void> {
1075
+ const staged = await this.treeFor(base, dir)
1076
+ if (staged !== undefined) {
1077
+ await this.ensure(staged)
1078
+ this.trees.get(this.treeOf(dir + path.sep)!)!.dirs.push(dir)
1079
+ return
1080
+ }
1081
+ let created: string | undefined
1082
+ try {
1083
+ created = await mkdir(dir, { recursive: true })
1084
+ } catch (err) {
1085
+ created = await this.mkdirThroughLink(base, dir, err as NodeJS.ErrnoException)
1086
+ }
1087
+ this.ensuredDirs.add(dir)
1088
+ if (created !== undefined) this.madeDirs.push({ dir, created })
1089
+ }
1090
+
910
1091
  /**
911
1092
  * Write the entry beside its target. A buffer is written without
912
1093
  * waiting, bounded by `INFLIGHT_BYTES`; a chunk stream goes to a file
@@ -919,6 +1100,25 @@ class Extractor {
919
1100
  body: Uint8Array | AsyncIterable<Uint8Array>,
920
1101
  ): Promise<void> {
921
1102
  const dir = path.dirname(target)
1103
+ const inTree = await this.treeFor(base, dir)
1104
+ if (inTree !== undefined) {
1105
+ await this.ensure(inTree)
1106
+ const tmp = path.join(inTree, path.basename(target))
1107
+ const streamed = !(body instanceof Uint8Array)
1108
+ const tree = this.treeOf(target)!
1109
+ this.staged.push({
1110
+ staged: true,
1111
+ name,
1112
+ tmp,
1113
+ target,
1114
+ created: undefined,
1115
+ streamed,
1116
+ base,
1117
+ tree,
1118
+ })
1119
+ await this.write(tmp, body)
1120
+ return
1121
+ }
922
1122
  let created: string | undefined
923
1123
  if (!this.ensuredDirs.has(dir)) {
924
1124
  try {
@@ -980,6 +1180,10 @@ class Extractor {
980
1180
  : siblingTemp(target)
981
1181
  const streamed = !(body instanceof Uint8Array)
982
1182
  this.staged.push({ staged, name, tmp, target, created, streamed, base })
1183
+ await this.write(tmp, body)
1184
+ }
1185
+
1186
+ private async write(tmp: string, body: Uint8Array | AsyncIterable<Uint8Array>): Promise<void> {
983
1187
  if (body instanceof Uint8Array) {
984
1188
  this.inflightBytes += body.byteLength
985
1189
  // `writeFile`, not `Bun.write`: for a buffer this size `Bun.write`
@@ -1006,6 +1210,49 @@ class Extractor {
1006
1210
  }
1007
1211
  }
1008
1212
 
1213
+ /** A symlink: made apart from the target like a file, renamed over it on commit. */
1214
+ async stageLink(name: string, base: string, target: string, to: string): Promise<void> {
1215
+ const dir = path.dirname(target)
1216
+ const inTree = await this.treeFor(base, dir)
1217
+ if (inTree !== undefined) {
1218
+ await this.ensure(inTree)
1219
+ const tmp = path.join(inTree, path.basename(target))
1220
+ const tree = this.treeOf(target)!
1221
+ this.staged.push({
1222
+ staged: true,
1223
+ name,
1224
+ tmp,
1225
+ target,
1226
+ created: undefined,
1227
+ streamed: false,
1228
+ base,
1229
+ link: to,
1230
+ tree,
1231
+ })
1232
+ await symlink(to, tmp)
1233
+ return
1234
+ }
1235
+ let created: string | undefined
1236
+ if (!this.ensuredDirs.has(dir)) {
1237
+ try {
1238
+ created = await mkdir(dir, { recursive: true })
1239
+ } catch (err) {
1240
+ created = await this.mkdirThroughLink(base, dir, err as NodeJS.ErrnoException)
1241
+ }
1242
+ this.ensuredDirs.add(dir)
1243
+ }
1244
+ if (created === undefined) {
1245
+ const st = lstatSync(target, { throwIfNoEntry: false })
1246
+ if (st?.isSymbolicLink() === true && readlinkSync(target) === to) return
1247
+ }
1248
+ const staged = this.stageRoot !== undefined
1249
+ const tmp = staged
1250
+ ? path.join(await stagingDir(this.stageRoot!), (tmpSeq++).toString(36))
1251
+ : siblingTemp(target)
1252
+ this.staged.push({ staged, name, tmp, target, created, streamed: false, base, link: to })
1253
+ await symlink(to, tmp)
1254
+ }
1255
+
1009
1256
  private async drain(): Promise<void> {
1010
1257
  const pending = this.inflight
1011
1258
  this.inflight = []
@@ -1028,6 +1275,17 @@ class Extractor {
1028
1275
  metaFor: (name: string) => [mode: number, mtimeMs: number | undefined],
1029
1276
  ): Promise<void> {
1030
1277
  await this.drain()
1278
+ // One stat per directory, shared by the filesystem check and the group.
1279
+ const dirStats = new Map<string, Stats>()
1280
+ const statDir = (dir: string): Stats => {
1281
+ let st = dirStats.get(dir)
1282
+ if (st === undefined) {
1283
+ st = statSync(dir)
1284
+ dirStats.set(dir, st)
1285
+ }
1286
+ return st
1287
+ }
1288
+ await this.crossOver(statDir)
1031
1289
  const created = new Map<boolean, { mode: number; uid: number; gid: number }>()
1032
1290
  // The group a file written in place would take, by directory: a setgid
1033
1291
  // directory's on Linux, every directory's on BSD/macOS. A file staged
@@ -1036,44 +1294,147 @@ class Extractor {
1036
1294
  let n = 0
1037
1295
  let renames: Promise<void>[] = []
1038
1296
  const checked = new Set<string>()
1039
- for (const s of this.staged) {
1040
- await this.assertStillContained(s, checked)
1041
- const [mode, mtimeMs] = metaFor(s.name)
1042
- if (s.same !== undefined) {
1043
- keep(s, s.same, mode, mtimeMs)
1044
- continue
1045
- }
1046
- let made = created.get(s.streamed)
1047
- if (made === undefined) {
1048
- const st = statSync(s.tmp)
1049
- made = { mode: st.mode & 0o777, uid: st.uid, gid: st.gid }
1050
- created.set(s.streamed, made)
1051
- }
1052
- if ((mode & 0o777) !== made.mode) chmodSync(s.tmp, mode & 0o777)
1053
- // A Date, not seconds: Bun reads a negative number of seconds as
1054
- // "now", and a Date before 1970 as itself (Bun 1.4.2).
1055
- const t = mtimeMs === undefined ? undefined : new Date(mtimeMs)
1056
- if (t !== undefined) utimesSync(s.tmp, t, t)
1057
- if (s.staged) {
1058
- const dir = path.dirname(s.target)
1059
- let gid = setgid.get(dir)
1060
- if (gid === undefined) {
1061
- const st = statSync(dir)
1062
- gid = INHERITS_GROUP || (st.mode & 0o2000) !== 0 ? st.gid : null
1063
- setgid.set(dir, gid)
1297
+ try {
1298
+ for (const s of this.staged) {
1299
+ if (s.tree === undefined)
1300
+ await this.assertStillContained(s.base, path.dirname(s.target), checked)
1301
+ const [mode, mtimeMs] = metaFor(s.name)
1302
+ if (s.same !== undefined) {
1303
+ keep(s, s.same, mode, mtimeMs)
1304
+ continue
1305
+ }
1306
+ if (s.link !== undefined) {
1307
+ if (s.tree !== undefined) continue
1308
+ renames.push(handled(this.place(s, mode, undefined)))
1309
+ if (++n % RENAME_BATCH === 0) {
1310
+ await settle(renames)
1311
+ renames = []
1312
+ }
1313
+ continue
1314
+ }
1315
+ if (s.beside === true) {
1316
+ chmodSync(s.tmp, restoredMode(mode))
1317
+ if (mtimeMs !== undefined) utimesSync(s.tmp, new Date(mtimeMs), new Date(mtimeMs))
1318
+ renames.push(handled(this.place(s, mode, undefined)))
1319
+ if (++n % RENAME_BATCH === 0) {
1320
+ await settle(renames)
1321
+ renames = []
1322
+ }
1323
+ continue
1324
+ }
1325
+ let made = created.get(s.streamed)
1326
+ if (made === undefined) {
1327
+ const st = statSync(s.tmp)
1328
+ made = { mode: st.mode & 0o777, uid: st.uid, gid: st.gid }
1329
+ created.set(s.streamed, made)
1330
+ }
1331
+ if (restoredMode(mode) !== made.mode) chmodSync(s.tmp, restoredMode(mode))
1332
+ // A Date, not seconds: Bun reads a negative number of seconds as
1333
+ // "now", and a Date before 1970 as itself (Bun 1.4.2).
1334
+ const t = mtimeMs === undefined ? undefined : new Date(mtimeMs)
1335
+ if (t !== undefined) utimesSync(s.tmp, t, t)
1336
+ if (s.tree !== undefined) continue
1337
+ if (s.staged) {
1338
+ const dir = path.dirname(s.target)
1339
+ let gid = setgid.get(dir)
1340
+ if (gid === undefined) {
1341
+ const st = statDir(dir)
1342
+ gid = INHERITS_GROUP || (st.mode & 0o2000) !== 0 ? st.gid : null
1343
+ setgid.set(dir, gid)
1344
+ }
1345
+ if (gid !== null && gid !== made.gid) chownSync(s.tmp, made.uid, gid)
1346
+ }
1347
+ renames.push(handled(this.place(s, mode, t)))
1348
+ if (++n % RENAME_BATCH === 0) {
1349
+ await settle(renames)
1350
+ renames = []
1064
1351
  }
1065
- if (gid !== null && gid !== made.gid) chownSync(s.tmp, made.uid, gid)
1066
- }
1067
- renames.push(this.place(s, mode, t))
1068
- if (++n % RENAME_BATCH === 0) {
1069
- await settle(renames)
1070
- renames = []
1071
1352
  }
1353
+ } catch (err) {
1354
+ // Every rename in flight lands or fails before `abort` cleans up
1355
+ // (a fuzz run on macOS, 2026-10-17).
1356
+ await Promise.allSettled(renames)
1357
+ throw err
1072
1358
  }
1073
1359
  await settle(renames)
1360
+ for (const [top, t] of this.trees) await this.placeTree(top, t, checked)
1361
+ await this.assertLandedInside()
1362
+ this.trees.clear()
1074
1363
  this.staged.length = 0
1075
1364
  }
1076
1365
 
1366
+ /**
1367
+ * Rename a staged tree to the directory it stands for. Something made
1368
+ * there since staging (another restore of the same outputs) takes the
1369
+ * tree's entries one by one instead, each renamed over its target.
1370
+ */
1371
+ private async placeTree(top: string, t: Tree, checked: Set<string>): Promise<void> {
1372
+ await this.assertStillContained(t.base, path.dirname(top), checked)
1373
+ try {
1374
+ await rename(t.at, top)
1375
+ return
1376
+ } catch (err) {
1377
+ const code = (err as NodeJS.ErrnoException).code
1378
+ if (code !== 'EEXIST' && code !== 'ENOTEMPTY' && code !== 'ENOTDIR' && code !== 'EISDIR') {
1379
+ throw err
1380
+ }
1381
+ }
1382
+ for (const dir of t.dirs) await this.makeDirIn(t.base, dir)
1383
+ for (const s of this.staged) {
1384
+ if (s.tree !== top) continue
1385
+ const dir = path.dirname(s.target)
1386
+ await this.makeDirIn(t.base, dir)
1387
+ await this.assertStillContained(t.base, dir, checked)
1388
+ await rename(s.tmp, s.target)
1389
+ }
1390
+ await rm(t.at, { recursive: true, force: true })
1391
+ }
1392
+
1393
+ private async makeDirIn(base: string, dir: string): Promise<void> {
1394
+ try {
1395
+ await mkdir(dir, { recursive: true })
1396
+ } catch (err) {
1397
+ await this.mkdirThroughLink(base, dir, err as NodeJS.ErrnoException)
1398
+ }
1399
+ }
1400
+
1401
+ /**
1402
+ * Copy every staged entry whose target is on another filesystem than
1403
+ * `.vx/tmp/` to a temp beside its target, before any rename: a rename
1404
+ * there would fail EXDEV, and copying at the rename, one file at a time,
1405
+ * left the files renamed before a failed copy (ENOSPC) new and the rest
1406
+ * old (Growth's parity run, 2026-10-16). A failed copy throws with
1407
+ * nothing renamed, and `abort` removes the temps made so far.
1408
+ */
1409
+ private async crossOver(statDir: (dir: string) => Stats): Promise<void> {
1410
+ const dev = (dir: string): number => statDir(dir).dev
1411
+ const cross = this.staged.filter(
1412
+ (s) =>
1413
+ s.staged &&
1414
+ s.same === undefined &&
1415
+ s.tree === undefined &&
1416
+ dev(path.dirname(s.tmp)) !== dev(path.dirname(s.target)),
1417
+ )
1418
+ for (let i = 0; i < cross.length; i += RENAME_BATCH) {
1419
+ await settle(
1420
+ cross.slice(i, i + RENAME_BATCH).map(async (s) => {
1421
+ const side = siblingTemp(s.target)
1422
+ try {
1423
+ if (s.link !== undefined) await symlink(s.link, side)
1424
+ else await copyFile(s.tmp, side)
1425
+ } catch (err) {
1426
+ await unlink(side).catch(() => undefined)
1427
+ throw err
1428
+ }
1429
+ const staged = s.tmp
1430
+ s.tmp = side
1431
+ s.beside = true
1432
+ await unlink(staged).catch(() => undefined)
1433
+ }),
1434
+ )
1435
+ }
1436
+ }
1437
+
1077
1438
  /**
1078
1439
  * Rename `s` over its target. A target on another filesystem than
1079
1440
  * `.vx/tmp/` (EXDEV) is reached through a temp beside it, for that file
@@ -1086,9 +1447,12 @@ class Extractor {
1086
1447
  if (!s.staged || (err as NodeJS.ErrnoException).code !== 'EXDEV') throw err
1087
1448
  const side = siblingTemp(s.target)
1088
1449
  try {
1089
- await copyFile(s.tmp, side)
1090
- chmodSync(side, mode & 0o777)
1091
- if (t !== undefined) utimesSync(side, t, t)
1450
+ if (s.link !== undefined) await symlink(s.link, side)
1451
+ else {
1452
+ await copyFile(s.tmp, side)
1453
+ chmodSync(side, restoredMode(mode))
1454
+ if (t !== undefined) utimesSync(side, t, t)
1455
+ }
1092
1456
  await rename(side, s.target)
1093
1457
  } catch (e) {
1094
1458
  await unlink(side).catch(() => undefined)
@@ -1104,14 +1468,44 @@ class Extractor {
1104
1468
  * Bun has no `renameat` against a held directory fd, so this is by path,
1105
1469
  * once per directory per commit.
1106
1470
  */
1107
- private async assertStillContained(s: Staged, checked: Set<string>): Promise<void> {
1108
- const dir = path.dirname(s.target)
1471
+ private async assertStillContained(
1472
+ base: string,
1473
+ dir: string,
1474
+ checked: Set<string>,
1475
+ ): Promise<void> {
1109
1476
  if (checked.has(dir)) return
1110
1477
  checked.add(dir)
1111
- const realBase = await this.realBaseOf(s.base)
1478
+ const realBase = await this.realBaseOf(base)
1112
1479
  const real = await realOf(dir, true)
1113
1480
  if (real !== realBase && !real.startsWith(realBase + path.sep)) {
1114
- throw await linkOutError(path.resolve(s.base), dir, realBase)
1481
+ throw await linkOutError(path.resolve(base), dir, realBase)
1482
+ }
1483
+ }
1484
+
1485
+ /**
1486
+ * The check before each rename, once more after all landed: the rename
1487
+ * runs on the thread pool, and a directory swapped for a link out in that
1488
+ * gap carried the entry with it (restore-link-swap-rename.test.ts). An
1489
+ * entry that landed outside is removed from there and the restore fails.
1490
+ */
1491
+ private async assertLandedInside(): Promise<void> {
1492
+ const landed = new Map<string, { base: string; placed: string[] }>()
1493
+ const add = (base: string, dir: string, placed: string): void => {
1494
+ const at = landed.get(dir)
1495
+ if (at === undefined) landed.set(dir, { base, placed: [placed] })
1496
+ else at.placed.push(placed)
1497
+ }
1498
+ for (const s of this.staged) {
1499
+ if (s.same === undefined && s.tree === undefined)
1500
+ add(s.base, path.dirname(s.target), s.target)
1501
+ }
1502
+ for (const [top, t] of this.trees) add(t.base, path.dirname(top), top)
1503
+ for (const [dir, { base, placed }] of landed) {
1504
+ const realBase = await this.realBaseOf(base)
1505
+ const real = await realOf(dir, true).catch(() => null)
1506
+ if (real === null || real === realBase || real.startsWith(realBase + path.sep)) continue
1507
+ await Promise.allSettled(placed.map((p) => rm(p, { recursive: true, force: true })))
1508
+ throw await linkOutError(path.resolve(base), dir, realBase)
1115
1509
  }
1116
1510
  }
1117
1511
 
@@ -1129,6 +1523,8 @@ class Extractor {
1129
1523
  s.same === undefined ? unlink(s.tmp).catch(() => undefined) : undefined,
1130
1524
  ),
1131
1525
  )
1526
+ for (const t of this.trees.values()) await rm(t.at, { recursive: true, force: true })
1527
+ this.trees.clear()
1132
1528
  // Newest first: a later entry's chain can sit inside an earlier one's
1133
1529
  // (`dist/` then `dist/sub/`), never the reverse.
1134
1530
  for (const s of this.staged.toReversed()) {
@@ -1150,6 +1546,21 @@ class Extractor {
1150
1546
  dir = path.dirname(dir)
1151
1547
  }
1152
1548
  }
1549
+ for (const { dir: made, created } of this.madeDirs.toReversed()) {
1550
+ const top = path.resolve(created)
1551
+ let dir = made.startsWith(top) ? made : await realOf(made).catch(() => made)
1552
+ while (
1553
+ dir.startsWith(top) &&
1554
+ (await rmdir(dir).then(
1555
+ () => true,
1556
+ () => false,
1557
+ ))
1558
+ ) {
1559
+ if (dir === top) break
1560
+ dir = path.dirname(dir)
1561
+ }
1562
+ }
1563
+ this.madeDirs.length = 0
1153
1564
  this.staged.length = 0
1154
1565
  }
1155
1566
  }
@@ -1164,6 +1575,16 @@ class Extractor {
1164
1575
  */
1165
1576
  const RENAME_BATCH = 64
1166
1577
 
1578
+ /**
1579
+ * `p`, its failure marked handled at once: `settle` reads it later, and a
1580
+ * rename that fails while the commit awaits something else is not an
1581
+ * unhandled rejection that fails the process.
1582
+ */
1583
+ function handled(p: Promise<void>): Promise<void> {
1584
+ p.catch(() => undefined)
1585
+ return p
1586
+ }
1587
+
1167
1588
  /** Wait for every rename, then throw the first failure: none is left in flight for `abort`. */
1168
1589
  async function settle(renames: readonly Promise<void>[]): Promise<void> {
1169
1590
  for (const r of await Promise.allSettled(renames)) if (r.status === 'rejected') throw r.reason