@vzn/vx 0.0.18 → 0.0.22

package/src/cache/archive.ts

@@ -37,15 +37,17 @@
 // skipped — vx's outputs are regular files, and an artifact that claims
 // otherwise silently loses the claim rather than acting on it.
 
-import { mkdir, chmod, realpath, rename, rmdir, stat, unlink, utimes } from 'node:fs/promises'
+import { chmodSync, renameSync, statSync, utimesSync } from 'node:fs'
+import { mkdir, readlink, realpath, rmdir, stat, unlink, writeFile } from 'node:fs/promises'
 import path from 'node:path'
+import { UserError } from '../util/index.js'
 import { TarFormatError, type TarInput, tarEntries, tarPack, tarSize } from './tar-stream.js'
 
 /** Archive entry name carrying the per-output mode/mtime sidecar. */
 const META_ENTRY = '.vx-meta.json'
 
 /** Archive entry name of the always-present stdout record. */
-export const STDOUT_ENTRY = 'stdout'
+const STDOUT_ENTRY = 'stdout'
 
 /**
  * Sidecar shape. `files` maps an entry name to `[mode, mtimeMs]` — a
@@ -57,7 +59,29 @@ interface MetaFile {
   /** Bumped only if the sidecar's own shape changes; the artifact
    *  container is versioned by CACHE_VERSION, which gates reads. */
   version: 1
+  /**
+   * The cache key the artifact was packed under (v35, item 943). Ingest
+   * refuses bytes whose key is not the one it asked for: nothing else ties
+   * an artifact to its key, so a remote layer that answered one key with
+   * another's bytes (a truncated or colliding mapping) replayed the other
+   * task's outputs under a green `cache-hit-remote`.
+   */
+  key?: string
   files: Record<string, [mode: number, mtimeMs: number]>
+  /**
+   * What the PRODUCING execution used (2026-09-12, additive: an artifact
+   * without it reads as before). Rides the artifact so a machine that
+   * never ran the task — a fresh CI runner on a remote hit — still learns
+   * what it needs; every wire ships the bytes verbatim, so no seam
+   * changes.
+   */
+  exec?: ExecUsage
+}
+
+/** CPU time and peak RSS of the execution that produced an artifact. */
+export interface ExecUsage {
+  cpuMs?: number
+  peakRssBytes?: number
 }
 
 /** One regular file in an artifact, with its restore metadata resolved. */
@@ -86,8 +110,12 @@ export class ArchiveSecurityError extends Error {
 
 /** What `packArtifactStream` packs: stdout plus archive name → absolute source path. */
 export interface PackArgs {
+  /** The cache key, recorded in the sidecar (item 943); ingest refuses an artifact without it. */
+  key?: string
   stdout: string
   outputs: ReadonlyMap<string, string>
+  /** The producing execution's usage, when the runner reported it. */
+  exec?: ExecUsage | undefined
 }
 
 /** A packed artifact's plan: its entries with their stats, and the tar's exact size. */
@@ -106,9 +134,29 @@ export interface ArtifactPlan {
  */
 export async function planArtifact(args: PackArgs): Promise<ArtifactPlan> {
   const meta: MetaFile = { version: 1, files: {} }
+  if (args.key !== undefined) meta.key = args.key
+  const exec = usageOf(args.exec)
+  if (exec !== undefined) meta.exec = exec
   const files = await Promise.all(
     [...args.outputs].map(async ([name, abs]) => {
-      const st = await stat(abs)
+      // `stat` follows a symlink, so a link to a file is packed as that
+      // file's bytes and comes back as a regular file. A link to a
+      // directory, or a dangling one, has no bytes to pack: refuse loudly
+      // rather than store an entry that restores to nothing.
+      const shown = name.startsWith('outputs/') ? name.slice('outputs/'.length) : name
+      const st = await stat(abs).catch((err: NodeJS.ErrnoException) => {
+        if (err.code === 'ENOENT') {
+          throw new UserError(
+            `output ${shown} is a dangling symlink: vx stores regular files only — emit a file there, or narrow cache.outputs.files to the files the task produces (output globs take no '!')`,
+          )
+        }
+        throw err
+      })
+      if (!st.isFile()) {
+        throw new UserError(
+          `output ${shown} is not a regular file (a symlink to a directory?): vx stores regular files only — emit a file there, or narrow cache.outputs.files to the files the task produces (output globs take no '!')`,
+        )
+      }
       meta.files[name] = [st.mode & 0o777, Math.floor(st.mtimeMs)]
       return {
         name,
@@ -187,16 +235,26 @@ function streamOf(gen: AsyncGenerator<Uint8Array>): ReadableStream<Uint8Array> {
  * name is unsafe — a partial reading of a poisoned artifact is the
  * outcome the traversal defense exists to prevent.
  */
-export async function scanArtifact(
-  tar: ReadableStream<Uint8Array>,
-): Promise<{ entries: ArchiveEntry[]; stdout: string | null }> {
+export async function scanArtifact(tar: ReadableStream<Uint8Array>): Promise<{
+  entries: ArchiveEntry[]
+  stdout: string | null
+  exec: ExecUsage | undefined
+  key: string | undefined
+}> {
   const seen: Array<{ name: string; size: number; mtimeMs: number }> = []
   let meta: MetaFile['files'] = {}
+  let exec: ExecUsage | undefined
+  let key: string | undefined
   let stdout: string | null = null
   for await (const e of tarEntries(tar)) {
     if (e.type !== '0') continue
     if (e.name === META_ENTRY) {
-      meta = (JSON.parse(await textOf(e.body)) as MetaFile).files ?? {}
+      const parsed = JSON.parse(await textOf(e.body)) as MetaFile
+      meta = parsed.files ?? {}
+      // The ingest side is the untrusted boundary: a foreign sidecar's
+      // usage is taken only when it is a plain non-negative number.
+      exec = usageOf(parsed.exec)
+      if (typeof parsed.key === 'string') key = parsed.key
       continue
     }
     assertSafeName(e.name)
@@ -212,9 +270,22 @@ export async function scanArtifact(
       return { name: s.name, size: s.size, mode: m?.[0] ?? 0o644, mtimeMs: m?.[1] ?? s.mtimeMs }
     }),
     stdout,
+    exec,
+    key,
   }
 }
 
+/** The usage worth carrying: each axis a finite non-negative number, or absent; nothing → undefined. */
+function usageOf(raw: unknown): ExecUsage | undefined {
+  if (typeof raw !== 'object' || raw === null) return undefined
+  const { cpuMs, peakRssBytes } = raw as { cpuMs?: unknown; peakRssBytes?: unknown }
+  const out: { cpuMs?: number; peakRssBytes?: number } = {}
+  if (typeof cpuMs === 'number' && Number.isFinite(cpuMs) && cpuMs >= 0) out.cpuMs = cpuMs
+  if (typeof peakRssBytes === 'number' && Number.isFinite(peakRssBytes) && peakRssBytes >= 0)
+    out.peakRssBytes = peakRssBytes
+  return out.cpuMs === undefined && out.peakRssBytes === undefined ? undefined : out
+}
+
 /**
  * Restore from `tar`, the DECOMPRESSED archive as a byte stream:
  * `outputs/<rel>` lands under `destDir/<rel>` and — when `workspaceDest`
@@ -232,7 +303,10 @@ export async function extractArtifactStream(
   workspaceDest: string | undefined,
   verify?: (provided: ReadonlySet<string>) => void,
 ): Promise<Set<string>> {
-  await mkdir(destDir, { recursive: true })
+  // No `mkdir -p` of the destination here: `stage` creates each entry's
+  // directory chain, the destination included, and the cache restores
+  // into a project directory that exists. The eager one cost every
+  // restore a thread-pool round trip for an EEXIST and a stat (item 627).
   const x = new Extractor(destDir, workspaceDest)
   const provided = new Set<string>()
   const headerMtime = new Map<string, number>()
@@ -251,7 +325,12 @@ export async function extractArtifactStream(
       await x.assertContained(dest.base, target, e.name)
       provided.add(e.name)
       headerMtime.set(e.name, e.mtimeMs)
-      await x.stage(e.name, target, e.size <= SMALL_ENTRY ? await bytesOf(e.body) : e.body)
+      await x.stage(
+        e.name,
+        dest.base,
+        target,
+        e.size <= SMALL_ENTRY ? await bytesOf(e.body) : e.body,
+      )
     }
     verify?.(provided)
     await x.commit((name) => {
@@ -314,6 +393,16 @@ class Extractor {
   private inflight: Promise<unknown>[] = []
   private inflightBytes = 0
   private readonly realBaseCache = new Map<string, string>()
+  // Per-directory memos. An artifact holds a few files per directory
+  // (payload's ui: 4,069 files in 535 directories), and the containment
+  // walk and the `mkdir -p` are properties of the directory, not the
+  // entry: doing both once per directory took the restore of that
+  // artifact from 220–300 µs per file to the write itself. The window
+  // this opens — a symlink planted at a validated directory between two
+  // of its entries — is the same one that already lies between an
+  // entry's own check and its write.
+  private readonly containedDirs = new Set<string>()
+  private readonly ensuredDirs = new Set<string>()
 
   constructor(
     private readonly destDir: string,
@@ -382,12 +471,23 @@ class Extractor {
     if (targetResolved !== baseResolved && !targetResolved.startsWith(baseResolved + path.sep)) {
       throw new ArchiveSecurityError(`archive entry escapes destDir (unsafe): ${name}`)
     }
-    const realBase = await this.realBaseOf(base)
+    const parentResolved = path.dirname(targetResolved)
+    if (this.containedDirs.has(parentResolved)) return
+    await this.assertParentContained(base, targetResolved)
+    this.containedDirs.add(parentResolved)
+  }
+
+  private async assertParentContained(base: string, targetResolved: string): Promise<void> {
+    const baseResolved = path.resolve(base)
     // Only ancestors strictly BELOW the base are candidates — those are the
     // ones a poisoned entry could follow out of the tree. The walk must never
     // climb past the base: when the base itself does not exist yet (the
     // workspace-outputs anchor is created lazily), its parent legitimately
     // resolves outside and comparing against it would reject every entry.
+    // The base's own real path is resolved only once an ancestor EXISTS to
+    // compare it with: after a clean pruned the output directories (the
+    // common restore) nothing below the base is there, and the memoised
+    // resolve was a round trip spent on a comparison never made (item 627).
     let probe = path.dirname(targetResolved)
     while (probe.startsWith(baseResolved + path.sep)) {
       const real = await realpath(probe).then(
@@ -395,10 +495,9 @@ class Extractor {
         () => null,
       )
       if (real !== null) {
+        const realBase = await this.realBaseOf(base)
         if (real !== realBase && !real.startsWith(realBase + path.sep)) {
-          throw new ArchiveSecurityError(
-            `archive entry escapes destDir via a symlinked parent: ${name}`,
-          )
+          throw await linkOutError(baseResolved, probe, realBase)
         }
         return
       }
@@ -408,6 +507,33 @@ class Extractor {
     // pre-existing symlink to follow — the chain will be created fresh.
   }
 
+  /**
+   * `mkdir -p` does not follow a DANGLING link: it fails EEXIST at the link,
+   * ENOENT below it, ELOOP in a cycle — and the containment walk passed it,
+   * finding nothing that resolves between it and the base. A live link that
+   * stays inside the base is written through, so a dangling one is too: the
+   * directory it names is created and the entry lands through the link,
+   * which stays the user's (never replaced, nx#37061). One that leads out of
+   * the base is the refusal a live link out gets. Only the failure pays for
+   * the walk; a clean tree never reaches it.
+   */
+  private async mkdirThroughLink(
+    base: string,
+    dir: string,
+    err: NodeJS.ErrnoException,
+  ): Promise<string | undefined> {
+    if (err.code !== 'EEXIST' && err.code !== 'ENOENT' && err.code !== 'ELOOP') throw err
+    const real = await resolveThrough(dir)
+    const realBase = await this.realBaseOf(base)
+    if (real !== realBase && !real.startsWith(realBase + path.sep)) {
+      throw await linkOutError(path.resolve(base), dir, realBase)
+    }
+    // What stands there without a link on the way (a file named like the
+    // directory) fails this mkdir as it failed the first: the caller names
+    // it as a stray.
+    return mkdir(real, { recursive: true })
+  }
+
   /**
    * Write the entry beside its target. A buffer is written without
    * waiting, bounded by `INFLIGHT_BYTES`; a chunk stream goes to a file
@@ -415,10 +541,20 @@ class Extractor {
    */
   async stage(
     name: string,
+    base: string,
     target: string,
     body: Uint8Array | AsyncIterable<Uint8Array>,
   ): Promise<void> {
-    const created = await mkdir(path.dirname(target), { recursive: true })
+    const dir = path.dirname(target)
+    let created: string | undefined
+    if (!this.ensuredDirs.has(dir)) {
+      try {
+        created = await mkdir(dir, { recursive: true })
+      } catch (err) {
+        created = await this.mkdirThroughLink(base, dir, err as NodeJS.ErrnoException)
+      }
+      this.ensuredDirs.add(dir)
+    }
     // Write beside the target and RENAME into place. rename(2) replaces
     // the destination's directory ENTRY without following it, which is
     // what makes this both link-safe and concurrency-safe:
@@ -436,11 +572,24 @@ class Extractor {
     //
     // A directory at the target makes the rename fail — the same
     // fail-closed outcome the plain write had.
-    const tmp = `${target}.vx-tmp-${process.pid.toString(36)}-${(tmpSeq++).toString(36)}`
+    // The temp is a sibling with its own short name, never `<target>.suffix`:
+    // a suffix pushed a legal name of 242–255 bytes past NAME_MAX, and the
+    // restore of a valid artifact failed with ENAMETOOLONG (item 666).
+    const tmp = path.join(
+      path.dirname(target),
+      `.vx-tmp-${process.pid.toString(36)}-${(tmpSeq++).toString(36)}`,
+    )
     this.staged.push({ name, tmp, target, created })
     if (body instanceof Uint8Array) {
       this.inflightBytes += body.byteLength
-      const write = Bun.write(tmp, body)
+      // `writeFile`, not `Bun.write`: for a buffer this size `Bun.write`
+      // does the open, write and close on the calling thread and only
+      // its promise is asynchronous, so ten restores in flight wrote
+      // their 14,430 files one after another — a serial restore of
+      // payload took the same 6–7 s as one at concurrency 10, and a CPU
+      // profile put 6.7 of its 8.0 s inside `write` (2026-09-11). The
+      // thread-pool write lets the restores overlap their I/O.
+      const write = writeFile(tmp, body)
       // A write that fails before `commit` or `abort` awaits it must not
       // surface as an unhandled rejection; the awaiting `Promise.all` still
       // sees the error through the original promise.
@@ -464,20 +613,31 @@ class Extractor {
     await Promise.all(pending)
   }
 
-  /** Apply each entry's mode and mtime, then rename everything into place. */
+  /**
+   * Apply each entry's mode and mtime, then rename everything into place.
+   * Synchronous calls in batches: the three metadata calls per file are a
+   * few microseconds each on the calling thread and ~40 µs each as
+   * thread-pool round trips, and a large artifact has thousands of them
+   * (measured 2026-09-11 on payload's ui, 4,069 files: 892 ms → see the
+   * archive bench). A yield every `COMMIT_BATCH` files keeps the other
+   * restores' round trips flowing. The chmod is skipped when the mode is
+   * the one the temp file was created with — the common case.
+   */
   async commit(metaFor: (name: string) => [mode: number, mtimeMs: number]): Promise<void> {
     await this.drain()
-    await Promise.all(
-      this.staged.map(async (s) => {
-        const [mode, mtimeMs] = metaFor(s.name)
-        if (mode !== 0) await chmod(s.tmp, mode & 0o777)
-        if (mtimeMs > 0) {
-          const t = mtimeMs / 1000
-          await utimes(s.tmp, t, t)
-        }
-        await rename(s.tmp, s.target)
-      }),
-    )
+    const first = this.staged[0]
+    const createdMode = first === undefined ? -1 : statSync(first.tmp).mode & 0o777
+    let n = 0
+    for (const s of this.staged) {
+      const [mode, mtimeMs] = metaFor(s.name)
+      if (mode !== 0 && (mode & 0o777) !== createdMode) chmodSync(s.tmp, mode & 0o777)
+      if (mtimeMs > 0) {
+        const t = mtimeMs / 1000
+        utimesSync(s.tmp, t, t)
+      }
+      renameSync(s.tmp, s.target)
+      if (++n % COMMIT_BATCH === 0) await new Promise<void>((r) => setImmediate(r))
+    }
     this.staged.length = 0
   }
 
@@ -495,6 +655,9 @@ class Extractor {
       if (s.created === undefined) continue
       let dir = path.dirname(s.target)
       const top = path.resolve(s.created)
+      // Created at a dangling link's resolved target (`mkdirThroughLink`),
+      // so the chain is pruned from where the link leads.
+      if (!dir.startsWith(top)) dir = await realpath(dir).catch(() => dir)
       while (dir.startsWith(top)) {
         if (
           !(await rmdir(dir).then(
@@ -511,11 +674,73 @@ class Extractor {
   }
 }
 
+/** Files renamed into place between two yields of a commit. */
+const COMMIT_BATCH = 256
+
 /** Per-process counter for extract temp names; uniqueness only. */
 let tmpSeq = 0
 
 const WORKSPACE_PREFIX = 'workspace-outputs/'
 
+/**
+ * The on-disk half of containment failed: `probe`, a directory under
+ * `base`, resolves outside it, so a directory between them is a link out.
+ * That is the tree, not the artifact — an output directory made a link
+ * after the entry was saved (upstream survey, nx#37061) — so it is the
+ * user's to fix, and the refusal names the link and where it leads. The
+ * restore never writes through it and never replaces it: the link is the
+ * user's, and replacing it is the bug nx#37061 reports.
+ */
+async function linkOutError(base: string, probe: string, realBase: string): Promise<UserError> {
+  let dir = base
+  let real = realBase
+  for (const part of path.relative(base, probe).split(path.sep)) {
+    dir = path.join(dir, part)
+    real = await resolveThrough(dir)
+    if (real !== realBase && !real.startsWith(realBase + path.sep)) break
+  }
+  return new UserError(
+    `${dir} is a symbolic link to ${real}, outside ${base} — a cache restore never writes ` +
+      'through a link that leaves its directory. Remove the link and re-run (the restore ' +
+      'puts a real directory there), or stop declaring outputs under it.',
+  )
+}
+
+/**
+ * Where `p` leads, every link on the way followed, a dangling one included:
+ * `realpath` for the part that exists, and past the first missing name the
+ * rest as written — where `mkdir -p` would create it if it followed links.
+ * A cycle is the user's tree, not the artifact, so it is refused by name.
+ */
+async function resolveThrough(p: string): Promise<string> {
+  const pending = path.resolve(p).split(path.sep).filter(Boolean).reverse()
+  let at: string = path.sep
+  let hops = 0
+  while (pending.length > 0) {
+    const next = path.join(at, pending.pop()!)
+    const link = await readlink(next).catch((err: NodeJS.ErrnoException) => {
+      if (err.code === 'EINVAL') return null
+      if (err.code === 'ENOENT' || err.code === 'ENOTDIR') return undefined
+      throw err
+    })
+    if (link === undefined) return path.join(next, ...pending.reverse())
+    if (link === null) {
+      at = next
+      continue
+    }
+    // Linux's own bound on links followed in one lookup (MAXSYMLINKS).
+    if (++hops > 40) {
+      throw new UserError(
+        `${p} goes through a cycle of symbolic links (at ${next}) — a cache restore cannot ` +
+          'write through it. Remove the link and re-run.',
+      )
+    }
+    if (path.isAbsolute(link)) at = path.sep
+    pending.push(...link.split(path.sep).filter(Boolean).reverse())
+  }
+  return at
+}
+
 /**
  * Path-traversal defense, applied to every entry name at read time.
  * Mirrors Turbo's `restore.rs:check_path` normalization: an artifact is
@@ -549,8 +774,34 @@ function assertSafeName(name: string): void {
   if (name.includes('\0')) {
     throw new ArchiveSecurityError(`archive entry name contains a null byte (unsafe): ${name}`)
   }
+  // Only a pax `path` record can carry a name this long. Left to the file
+  // system it is a raw ENAMETOOLONG, which the restore reports as an
+  // internal error rather than a refused artifact. A UTF-16 unit is at most
+  // three UTF-8 bytes, so the common short name never pays for the count.
+  if (name.length * 3 >= PATH_MAX && Buffer.byteLength(name) >= PATH_MAX) {
+    throw new ArchiveSecurityError(
+      `archive entry name is ${Buffer.byteLength(name)} bytes, past PATH_MAX (${PATH_MAX}) (unsafe): ${name.slice(0, 64)}…`,
+    )
+  }
+  // The same raw ENAMETOOLONG for one component past NAME_MAX, the same on
+  // Linux and macOS. Same cheap bound: a name under 86 units cannot hold one.
+  if (name.length * 3 > NAME_MAX) {
+    for (const part of name.split('/')) {
+      if (part.length * 3 > NAME_MAX && Buffer.byteLength(part) > NAME_MAX) {
+        throw new ArchiveSecurityError(
+          `archive entry has a ${Buffer.byteLength(part)}-byte component, past NAME_MAX (${NAME_MAX}) (unsafe): ${part.slice(0, 64)}…`,
+        )
+      }
+    }
+  }
 }
 
+/** Bytes one path component may hold (limits.h), on Linux and macOS alike. */
+const NAME_MAX = 255
+
+/** Bytes a path may hold, its terminating NUL included (limits.h). */
+const PATH_MAX = process.platform === 'darwin' ? 1024 : 4096
+
 function hasParentSegment(p: string): boolean {
   if (p === '..' || p.startsWith('../') || p.endsWith('/..')) return true
   return p.includes('/../')