@vzn/vx 0.0.630 → 0.0.632

package/src/cache/archive.ts

@@ -50,7 +50,15 @@ import { lstat, mkdir, readlink, realpath, rmdir, stat, unlink, writeFile } from
 import path from 'node:path'
 import { UserError } from '../util/index.js'
 import { WORKSPACE_OUTPUT_PREFIX } from './layer.js'
-import { TarFormatError, type TarInput, tarEntries, tarPack, tarSize } from './tar-stream.js'
+import {
+  TarFormatError,
+  type TarInput,
+  tarEntries,
+  tarEntriesSync,
+  tarPack,
+  tarPackInto,
+  tarSize,
+} from './tar-stream.js'
 import { ON_THREAD_MAX } from './zstd.js'
 
 /** Archive entry name carrying the per-output mode/mtime sidecar. */
@@ -59,6 +67,10 @@ const META_ENTRY = '.vx-meta.json'
 /** Archive entry name of the always-present stdout record. */
 const STDOUT_ENTRY = 'stdout'
 
+// One decoder, its defaults (a leading BOM dropped), for both scanners.
+// `ignoreBOM`: a leading U+FEFF in a task's stdout is the task's.
+const utf8 = new TextDecoder('utf-8', { ignoreBOM: true })
+
 /**
  * The last entry: a CRC-32 over every entry before it, each its name, a
  * NUL, then its body, as 8 lowercase hex digits (v36, L-19). Nothing else
@@ -107,19 +119,30 @@ class SumCheck {
     name: string,
     body: AsyncIterable<Uint8Array>,
   ): Promise<AsyncIterable<Uint8Array> | null> {
+    if (!this.enter(name)) return this.sum.through(body)
+    this.verify(await textOf(body))
+    return null
+  }
+  /**
+   * The next regular entry: true when it is the sum entry, whose text the
+   * caller hands to `verify`; otherwise the caller hands its bytes to `add`.
+   */
+  enter(name: string): boolean {
     if (this.seen) throw new TarFormatError(`entry ${name} follows the artifact's checksum`)
-    if (name === SUM_ENTRY) {
-      const got = await textOf(body)
-      if (got !== this.sum.hex()) {
-        throw new TarFormatError(
-          `artifact content does not match its checksum (${got}, read ${this.sum.hex()})`,
-        )
-      }
-      this.seen = true
-      return null
-    }
+    if (name === SUM_ENTRY) return true
     this.sum.entry(name)
-    return this.sum.through(body)
+    return false
+  }
+  add(bytes: Uint8Array): void {
+    this.sum.add(bytes)
+  }
+  verify(got: string): void {
+    if (got !== this.sum.hex()) {
+      throw new TarFormatError(
+        `artifact content does not match its checksum (${got}, read ${this.sum.hex()})`,
+      )
+    }
+    this.seen = true
   }
   done(): void {
     if (!this.seen) throw new TarFormatError('artifact carries no checksum')
@@ -373,16 +396,12 @@ async function* summed(inputs: readonly TarInput[]): AsyncGenerator<TarInput> {
  * otherwise (measured 2026-09-03: 238 → 293 ms per 1 000 saves).
  */
 export async function packArtifactBytes(plan: ArtifactPlan): Promise<Uint8Array> {
-  // A small artifact's files are read on this thread (`ON_THREAD_MAX`).
-  const onThread = plan.size <= ON_THREAD_MAX
+  // A small artifact is read and packed on this thread (`ON_THREAD_MAX`):
+  // no promise or generator hop per file or block.
+  if (plan.size <= ON_THREAD_MAX) return packSmall(plan)
   const inputs = await Promise.all(
     plan.inputs.map(async (i) =>
-      i.body instanceof Blob
-        ? {
-            ...i,
-            body: onThread ? readFileSync((i.body as Bun.BunFile).name!) : await i.body.bytes(),
-          }
-        : i,
+      i.body instanceof Blob ? { ...i, body: await i.body.bytes() } : i,
     ),
   )
   const out = new Uint8Array(plan.size)
@@ -395,6 +414,28 @@ export async function packArtifactBytes(plan: ArtifactPlan): Promise<Uint8Array>
   return out
 }
 
+/** `summed` + `tarPack` for a plan whose bodies fit in memory, synchronously. */
+function packSmall(plan: ArtifactPlan): Uint8Array {
+  const sum = new EntrySum()
+  const inputs: (TarInput & { body: Uint8Array | string })[] = []
+  for (const i of plan.inputs) {
+    const body =
+      i.body instanceof Blob
+        ? readFileSync((i.body as Bun.BunFile).name!)
+        : typeof i.body === 'string'
+          ? new TextEncoder().encode(i.body)
+          : (i.body as Uint8Array)
+    sum.entry(i.name)
+    sum.add(body)
+    inputs.push({ ...i, body })
+  }
+  inputs.push({ name: SUM_ENTRY, size: SUM_SIZE, body: sum.hex() })
+  const out = new Uint8Array(plan.size)
+  const n = tarPackInto(inputs, out)
+  if (n !== plan.size) throw new TarFormatError(`packed ${n} bytes, planned ${plan.size}`)
+  return out
+}
+
 /** Plan and pack in one go, in memory. */
 export async function packArtifact(args: PackArgs): Promise<Uint8Array> {
   return await packArtifactBytes(await planArtifact(args))
@@ -413,6 +454,51 @@ function streamOf(gen: AsyncGenerator<Uint8Array>): ReadableStream<Uint8Array> {
   })
 }
 
+/** What a scan of an artifact reads out of it. */
+interface ArtifactScan {
+  entries: ArchiveEntry[]
+  stdout: string | null
+  exec: ExecUsage | undefined
+  key: string | undefined
+}
+
+/** A scan's result, built entry by entry in tar order: the one fold both scanners share. */
+class ScanFold {
+  private readonly seen: Array<{ name: string; size: number; mtimeMs: number }> = []
+  private files: MetaFile['files'] = {}
+  private exec: ExecUsage | undefined
+  private key: string | undefined
+  stdout: string | null = null
+
+  sidecar(text: string): void {
+    const parsed = JSON.parse(text) as MetaFile
+    this.files = parsed.files ?? {}
+    // The ingest side is the untrusted boundary: a foreign sidecar's
+    // usage is taken only when it is a plain non-negative number.
+    this.exec = usageOf(parsed.exec)
+    if (typeof parsed.key === 'string') this.key = parsed.key
+  }
+
+  entry(name: string, size: number, mtimeMs: number): void {
+    this.seen.push({ name, size, mtimeMs })
+  }
+
+  result(): ArtifactScan {
+    return {
+      entries: this.seen.map((s) => {
+        const m = this.files[s.name]
+        // A foreign artifact (or one whose sidecar lost an entry) restores
+        // readable-but-not-executable rather than failing the hit: the
+        // sidecar is metadata, never the authority on whether bytes exist.
+        return { name: s.name, size: s.size, mode: m?.[0] ?? 0o644, mtimeMs: m?.[1] ?? s.mtimeMs }
+      }),
+      stdout: this.stdout,
+      exec: this.exec,
+      key: this.key,
+    }
+  }
+}
+
 /**
  * List an artifact's regular entries with their restore metadata, and
  * its stdout text (`null` when the entry is absent — not a vx artifact),
@@ -420,49 +506,55 @@ function streamOf(gen: AsyncGenerator<Uint8Array>): ReadableStream<Uint8Array> {
  * name is unsafe — a partial reading of a poisoned artifact is the
  * outcome the traversal defense exists to prevent.
  */
-export async function scanArtifact(tar: ReadableStream<Uint8Array>): Promise<{
-  entries: ArchiveEntry[]
-  stdout: string | null
-  exec: ExecUsage | undefined
-  key: string | undefined
-}> {
-  const seen: Array<{ name: string; size: number; mtimeMs: number }> = []
-  let meta: MetaFile['files'] = {}
-  let exec: ExecUsage | undefined
-  let key: string | undefined
-  let stdout: string | null = null
+export async function scanArtifact(tar: ReadableStream<Uint8Array>): Promise<ArtifactScan> {
+  const fold = new ScanFold()
   const check = new SumCheck()
   for await (const e of tarEntries(tar)) {
     if (e.type !== '0') continue
     const body = await check.body(e.name, e.body)
     if (body === null) continue
     if (e.name === META_ENTRY) {
-      const parsed = JSON.parse(await textOf(body)) as MetaFile
-      meta = parsed.files ?? {}
-      // The ingest side is the untrusted boundary: a foreign sidecar's
-      // usage is taken only when it is a plain non-negative number.
-      exec = usageOf(parsed.exec)
-      if (typeof parsed.key === 'string') key = parsed.key
+      fold.sidecar(await textOf(body))
       continue
     }
     assertSafeName(e.name)
-    if (e.name === STDOUT_ENTRY) stdout = await textOf(body)
+    if (e.name === STDOUT_ENTRY) fold.stdout = await textOf(body)
     else await drain(body)
-    seen.push({ name: e.name, size: e.size, mtimeMs: e.mtimeMs })
+    fold.entry(e.name, e.size, e.mtimeMs)
   }
   check.done()
-  return {
-    entries: seen.map((s) => {
-      const m = meta[s.name]
-      // A foreign artifact (or one whose sidecar lost an entry) restores
-      // readable-but-not-executable rather than failing the hit: the
-      // sidecar is metadata, never the authority on whether bytes exist.
-      return { name: s.name, size: s.size, mode: m?.[0] ?? 0o644, mtimeMs: m?.[1] ?? s.mtimeMs }
-    }),
-    stdout,
-    exec,
-    key,
+  return fold.result()
+}
+
+/**
+ * `scanArtifact` over a tar in memory, synchronously: a save scans the
+ * tar it just packed. Same decoder, same checks in the same order, same
+ * result. The async generators, per-chunk promises and drain were most
+ * of a small save's scan: one 400 B output, 49 → 14 µs (X-191).
+ */
+export function scanTarBytes(tar: Uint8Array): ArtifactScan {
+  const fold = new ScanFold()
+  const check = new SumCheck()
+  for (const e of tarEntriesSync(tar)) {
+    if (e.type !== '0') continue
+    if (check.enter(e.name)) {
+      check.verify(utf8.decode(e.body()))
+      continue
+    }
+    if (e.name === META_ENTRY) {
+      const body = e.body()
+      check.add(body)
+      fold.sidecar(utf8.decode(body))
+      continue
+    }
+    assertSafeName(e.name)
+    const body = e.body()
+    check.add(body)
+    if (e.name === STDOUT_ENTRY) fold.stdout = utf8.decode(body)
+    fold.entry(e.name, e.size, e.mtimeMs)
   }
+  check.done()
+  return fold.result()
 }
 
 /** The usage worth carrying: each axis a finite non-negative number, or absent; nothing → undefined. */
@@ -569,7 +661,7 @@ async function bytesOf(body: AsyncIterable<Uint8Array>): Promise<Uint8Array> {
 }
 
 const textOf = async (body: AsyncIterable<Uint8Array>): Promise<string> =>
-  new TextDecoder().decode(await bytesOf(body))
+  utf8.decode(await bytesOf(body))
 
 /** One staged entry: written at `tmp`, renamed to `target` on commit. */
 interface Staged {
@@ -1012,6 +1104,12 @@ function assertSafeName(name: string): void {
   if (name.includes('//')) {
     throw new ArchiveSecurityError(`archive entry name has empty path component (unsafe): ${name}`)
   }
+  // vx never packs a `.` segment, and `outputs/.` names the anchor itself:
+  // ingest kept it, and every hit then failed renaming a file over the
+  // project directory, blamed on the user's tree.
+  if (hasDotSegment(name)) {
+    throw new ArchiveSecurityError(`archive entry name has a '.' segment (unsafe): ${name}`)
+  }
   // No backslash refusal: vx runs on Linux and macOS (Windows through WSL),
   // where it is a name character, and the save scans its own artifact with
   // these checks, so refusing one made `dist/back\slash` uncacheable.
@@ -1049,6 +1147,10 @@ const NAME_MAX = 255
 /** Bytes a path may hold, its terminating NUL included (limits.h). */
 const PATH_MAX = process.platform === 'darwin' ? 1024 : 4096
 
+function hasDotSegment(p: string): boolean {
+  return p === '.' || p.startsWith('./') || p.endsWith('/.') || p.includes('/./')
+}
+
 function hasParentSegment(p: string): boolean {
   if (p === '..' || p.startsWith('../') || p.endsWith('/..')) return true
   return p.includes('/../')

package/src/cache/cache.ts

@@ -70,6 +70,7 @@ import {
   type ExecUsage,
   extractArtifactStream,
   scanArtifact,
+  scanTarBytes,
   packArtifactBytes,
   packArtifactStream,
   planArtifact,
@@ -100,7 +101,6 @@ import {
   bytesOf,
   decodedTar,
   MAX_DECOMPRESSED_ARTIFACT_BYTES,
-  oneChunk,
   STREAM_DECODE_FROM,
   zstdCompress,
   zstdEncoder,
@@ -122,6 +122,16 @@ import { CACHE_VERSION, foldKey } from './key-fold.js'
  */
 const ORPHAN_GRACE_MS = 60 * 60 * 1000
 
+/**
+ * A hit renews an entry's `accessed_at` only once it is older than this:
+ * renewing every hit rewrote 2,180 rows at a warm 1,090-package run's
+ * close, ~30 ms of it. So `accessed_at` trails the last use by at most
+ * this much, and the retention policy reads it plus this as the last use:
+ * an entry used within the limit is never pruned, and one may stay up to
+ * this long past it (owner, 2026-10-08).
+ */
+const ACCESS_REFRESH_MS = 60 * 60 * 1000
+
 /**
  * The only names the orphan sweep may unlink: an artifact under a key
  * `foldKey` prints (16 lowercase hex) and the temp `tempPath` makes of
@@ -298,11 +308,12 @@ function pickVictims(
   const kept: Array<{ at: number; size: number; row?: string; file?: RowlessFile }> = []
   let remaining = 0
   for (const r of rows) {
-    if (olderThanMs !== undefined && r.accessed_at < olderThanMs) {
+    const at = r.accessed_at + ACCESS_REFRESH_MS
+    if (olderThanMs !== undefined && at < olderThanMs) {
       hashes.add(r.hash)
       bytesFreed += r.size_bytes
     } else {
-      kept.push({ at: r.accessed_at, size: r.size_bytes, row: r.hash })
+      kept.push({ at, size: r.size_bytes, row: r.hash })
       remaining += r.size_bytes
     }
   }
@@ -536,6 +547,7 @@ function ensurePrivateDir(dir: string): string | null {
 // enumeration on the bench workspace before its generator ignored `.vx`. An
 // ignored directory is skipped by the walk entirely. Only written when
 // absent, so a user's own file wins.
+const utf8 = new TextEncoder()
 const IGNORE_ALL = '*\n'
 
 /**
@@ -830,6 +842,7 @@ export class Cache implements CacheLayer {
       if (mode !== 'open') {
         throw new UserError(
           `the cache at ${cacheDir} holds index schema ${found} from another vx version; this vx reads ${SCHEMA_VERSION}, so nothing in it is readable here, and this verb leaves it untouched. The next \`vx run\` resets it`,
+          'VX_E_CACHE_VERSION',
         )
       }
     }
@@ -966,7 +979,7 @@ export class Cache implements CacheLayer {
     )
     this.upsertStdout = lazyStatement(
       this.db,
-      'INSERT INTO entry_stdout(hash, stdout) VALUES (?, ?) ON CONFLICT(hash) DO UPDATE SET stdout = excluded.stdout',
+      'INSERT INTO entry_stdout(hash, stdout) VALUES (?, CAST(? AS TEXT)) ON CONFLICT(hash) DO UPDATE SET stdout = excluded.stdout',
     )
     this.deleteStdout = lazyStatement(this.db, 'DELETE FROM entry_stdout WHERE hash = ?')
     this.selectEntry = lazyStatement(this.db, `${SELECT_ENTRY} WHERE e.hash = ?`)
@@ -1347,7 +1360,7 @@ export class Cache implements CacheLayer {
     // full-cache run. Hashes are collected and flushed as ONE batched
     // UPDATE by flushAccessed() (called from prune/stats/close), which
     // is when accessed_at is actually read.
-    this.touched.add(hash)
+    if (row.accessed_at < Date.now() - ACCESS_REFRESH_MS) this.touched.add(hash)
 
     // Pure SQL: outputFiles come from the output_files rows and
     // stdout from the entries row. The artifact is NOT touched here —
@@ -1392,6 +1405,7 @@ export class Cache implements CacheLayer {
     const rows = this.db.query(`${SELECT_ENTRY} WHERE e.hash ${test}`).all(...params) as EntryRow[]
     if (rows.length === 0) return out
     const graceStart = Date.now() - ORPHAN_GRACE_MS
+    const refreshStart = Date.now() - ACCESS_REFRESH_MS
     const live = rows.filter((r) => {
       const st = statSync(this.tarPath(r.hash), { throwIfNoEntry: false })
       if (st !== undefined && st.mtimeMs < graceStart) this.staleTimes.add(r.hash)
@@ -1402,7 +1416,7 @@ export class Cache implements CacheLayer {
     this.outputs.hold(fileRows)
     const dirRows = this.loadOutputDirsBatch(liveHashes)
     for (const row of live) {
-      this.touched.add(row.hash)
+      if (row.accessed_at < refreshStart) this.touched.add(row.hash)
       const entry = entryOf(row, fileRows.get(row.hash) ?? [])
       entry.outputDirRows = dirRows.get(row.hash) ?? []
       out.set(row.hash, entry)
@@ -1990,17 +2004,20 @@ export class Cache implements CacheLayer {
     }
     let scanned: Awaited<ReturnType<typeof scanArtifact>>
     try {
-      // ingest() is the UNTRUSTED boundary — its temp holds bytes just
-      // pulled from a remote. Refuse a bomb (declared or sizeless) before it
-      // can expand into memory.
-      const source =
-        compressed instanceof Uint8Array && compressed.byteLength <= STREAM_DECODE_FROM
-          ? compressed
-          : Bun.file(tmpPath)
       const endScan = span('save: scan')
-      scanned = await scanArtifact(
-        tar !== undefined ? oneChunk(tar) : await decodedTar(source, hash, this.artifactCeiling),
-      )
+      if (tar !== undefined) {
+        // Our own tar, in memory: read in place, on this thread.
+        scanned = scanTarBytes(tar)
+      } else {
+        // ingest() is the UNTRUSTED boundary — its temp holds bytes just
+        // pulled from a remote. Refuse a bomb (declared or sizeless) before it
+        // can expand into memory.
+        const source =
+          compressed instanceof Uint8Array && compressed.byteLength <= STREAM_DECODE_FROM
+            ? compressed
+            : Bun.file(tmpPath)
+        scanned = await scanArtifact(await decodedTar(source, hash, this.artifactCeiling))
+      }
       endScan()
       // v17 invariant: every artifact carries a `stdout` entry. Its
       // absence means the bytes decompressed but aren't a vx artifact.
@@ -2118,7 +2135,13 @@ export class Cache implements CacheLayer {
         scanned.exec?.peakRssBytes ?? null,
       )
       if (stdoutText === '') deleteStdout.run(hash)
-      else upsertStdout.run(hash, stdoutText)
+      // bun:sqlite drops a leading U+FEFF from a bound string; its UTF-8
+      // bytes, cast to TEXT, keep it.
+      else
+        upsertStdout.run(
+          hash,
+          stdoutText.startsWith('\uFEFF') ? utf8.encode(stdoutText) : stdoutText,
+        )
       outputs.replaceFileRows(hash, outputFileRows)
       // INSERT OR IGNORE: identical inputs derive this same hash, so a
       // re-save's rows are identical — keep the first set, skip the rest.
@@ -2143,7 +2166,8 @@ export class Cache implements CacheLayer {
 
   /** Apply the deferred accessed_at bumps in one statement. */
   private flushAccessed(): void {
-    if (this.touched.size === 0) return
+    // Apart: a hit whose row is fresh still renews an old file time.
+    if (this.touched.size === 0 && this.staleTimes.size === 0) return
     const hashes = [...this.touched]
     this.touched.clear()
     const now = Date.now()
@@ -2158,6 +2182,7 @@ export class Cache implements CacheLayer {
       }
     }
     this.staleTimes.clear()
+    if (hashes.length === 0) return
     // LRU bookkeeping: on a full disk the bumps are dropped, never the prune
     // or the stats that asked for them (A-14).
     try {
@@ -2260,7 +2285,8 @@ export class Cache implements CacheLayer {
         'SELECT MIN(accessed_at) AS oldest, COALESCE(SUM(size_bytes), 0) AS bytes FROM entries',
       )
       .get() as { oldest: number | null; bytes: number }
-    const ageDue = olderThanMs !== undefined && oldest !== null && oldest < olderThanMs
+    const ageDue =
+      olderThanMs !== undefined && oldest !== null && oldest + ACCESS_REFRESH_MS < olderThanMs
     const sizeDue = policy.maxBytes !== undefined && bytes > policy.maxBytes
     if (ageDue || sizeDue) {
       return this.prune({

package/src/cache/layer.ts

@@ -215,7 +215,8 @@ export interface RunRecord {
   /**
    * Why the task failed or was skipped, as the run's own footer said it
    * (v27 columns; NULL on older rows and where the reason does not apply):
-   * a skip's root blocker, vx's own timeout, the sandbox's violation count,
+   * a skip's root blocker (or the one a task that ran behind a failure ran
+   * behind), vx's own timeout, the sandbox's violation count,
    * and why a persistent task never became ready.
    */
   blockedBy?: string

package/src/cache/schema.ts

@@ -95,7 +95,8 @@ export function createTables(db: Database, store: 'main' | 'store' = 'main'): vo
       -- time by design; NULL on rows older than the column.
       cached              INTEGER,
       -- v27: why a task failed or was skipped, as the run's own footer
-      -- said it (items 267–270): a skip's root blocker (a task id), vx's
+      -- said it (items 267–270): a skip's root blocker (a task id), or
+      -- the one a task that ran behind a failure ran behind (X-183), vx's
       -- own timeout (1), the sandbox's violation count, and why a
       -- persistent task never became ready ('timeout' | 'exited' |
       -- 'spawn'). NULL where the reason does not apply.