@vzn/vx 0.0.16 → 0.0.21

package/src/cache/chained-cache.ts

@@ -0,0 +1,194 @@
+// Several declared cache layers, consulted in declaration order. Lookup
+// walks the layers until one answers; a save reaches every layer; the FIRST
+// layer owns the run index (history, stats, prune) so a run is recorded
+// once. Restore goes to the layer that produced the hit — remembered per
+// hash — because an entry's artifact lives wherever it was found.
+
+import type {
+  Cache,
+  CacheEntry,
+  CacheGetContext,
+  CacheKeyInput,
+  CacheLayer,
+  CacheStats,
+  CacheStatsOptions,
+  IngestMeta,
+  InvocationRecord,
+  OutputDirRow,
+  OutputFileRow,
+  PruneOptions,
+  PruneResult,
+  RunRecord,
+} from './cache.js'
+
+export class ChainedCache implements CacheLayer {
+  readonly hasRemote: boolean
+  private readonly hitLayer = new Map<string, CacheLayer>()
+
+  constructor(readonly layers: readonly CacheLayer[]) {
+    if (layers.length < 2) throw new Error('ChainedCache needs at least two layers')
+    this.hasRemote = layers.some((l) => l.hasRemote === true)
+  }
+
+  get local(): Cache | undefined {
+    return this.layers[0]!.local
+  }
+
+  private owner(hash: string): CacheLayer {
+    return this.hitLayer.get(hash) ?? this.layers[0]!
+  }
+
+  key(input: CacheKeyInput): Promise<string> {
+    return this.layers[0]!.key(input)
+  }
+
+  async get(hash: string, ctx?: CacheGetContext): Promise<CacheEntry | null> {
+    for (const layer of this.layers) {
+      const entry = await layer.get(hash, ctx)
+      if (entry !== null) {
+        this.hitLayer.set(hash, layer)
+        return entry
+      }
+    }
+    return null
+  }
+
+  async has(hash: string): Promise<'local' | 'remote' | null> {
+    for (const layer of this.layers) {
+      const where = await layer.has(hash)
+      if (where !== null) {
+        this.hitLayer.set(hash, layer)
+        return where
+      }
+    }
+    return null
+  }
+
+  async prefetch(hash: string, ctx?: CacheGetContext): Promise<boolean> {
+    for (const layer of this.layers) {
+      if (await layer.prefetch(hash, ctx)) {
+        this.hitLayer.set(hash, layer)
+        return true
+      }
+    }
+    return false
+  }
+
+  async remoteHasMany(hashes: readonly string[]): Promise<Set<string> | null> {
+    // The caller (remote-prefetch) treats a non-null answer as authoritative
+    // for the WHOLE chain: complement = absent, broadcast. That is sound only
+    // if EVERY remote layer answered — a partial union would poison a layer
+    // that cannot batch with another layer's negatives, and its later lazy
+    // get() would skip a real remote hit. So: each answering layer gets its
+    // OWN complement marked here (its own truth — this also spares it the
+    // per-hash GETs for hashes only a sibling holds), and the merged answer
+    // is returned only when no remote layer was left unanswered.
+    let out: Set<string> | null = null
+    let complete = true
+    for (const layer of this.layers) {
+      if (layer.hasRemote !== true) continue
+      const found = (await layer.remoteHasMany?.(hashes)) ?? null
+      if (found === null) {
+        complete = false
+        continue
+      }
+      layer.markRemoteAbsent?.(hashes.filter((h) => !found.has(h)))
+      out ??= new Set()
+      for (const h of found) out.add(h)
+    }
+    return complete ? out : null
+  }
+
+  markRemoteAbsent(hashes: Iterable<string>): void {
+    const list = [...hashes]
+    for (const layer of this.layers) layer.markRemoteAbsent?.(list)
+  }
+
+  async drainUploads(): Promise<void> {
+    await Promise.all(this.layers.map((l) => l.drainUploads?.()))
+  }
+
+  loadOutputFilesBatch(hashes: readonly string[]): Map<string, OutputFileRow[]> {
+    const out = new Map<string, OutputFileRow[]>()
+    for (const layer of this.layers) {
+      for (const [h, rows] of layer.loadOutputFilesBatch(hashes)) {
+        if (!out.has(h)) out.set(h, rows)
+      }
+    }
+    return out
+  }
+
+  isOutputsCurrent(projectDir: string, expected: readonly OutputFileRow[]): Promise<boolean> {
+    return this.layers[0]!.isOutputsCurrent(projectDir, expected)
+  }
+
+  // The directory short-circuit lives with the layer that owns the local
+  // rows — the first, like the file check.
+  recordOutputDirs(hash: string, projectDir: string, prefixes: readonly string[]): Promise<void> {
+    return this.layers[0]!.recordOutputDirs?.(hash, projectDir, prefixes) ?? Promise.resolve()
+  }
+
+  loadOutputDirsBatch(hashes: readonly string[]): Map<string, OutputDirRow[]> {
+    return this.layers[0]!.loadOutputDirsBatch?.(hashes) ?? new Map()
+  }
+
+  outputDirsCurrent(projectDir: string, rows: readonly OutputDirRow[]): Promise<boolean> {
+    return this.layers[0]!.outputDirsCurrent?.(projectDir, rows) ?? Promise.resolve(false)
+  }
+
+  restoreOutputs(hash: string, projectDir: string, workspaceRoot?: string): Promise<void> {
+    return this.owner(hash).restoreOutputs(hash, projectDir, workspaceRoot)
+  }
+
+  async save(args: Parameters<CacheLayer['save']>[0]): Promise<void> {
+    // Layers wrapping the SAME local handle (two remote plugins over
+    // ctx.localCache) would each pack + write the identical artifact; the
+    // first write is the only one that matters, so later layers get
+    // `skipLocalWrite` and go straight to their remote upload (which reads
+    // the artifact the first layer just wrote — same handle, same path).
+    const seenLocals = new Set<NonNullable<CacheLayer['local']>>()
+    for (const layer of this.layers) {
+      const local = layer.local
+      const skip = local !== undefined && seenLocals.has(local)
+      await layer.save(skip ? { ...args, skipLocalWrite: true } : args)
+      if (local !== undefined) seenLocals.add(local)
+    }
+  }
+
+  ingest(hash: string, compressed: Uint8Array, meta: IngestMeta): Promise<void> {
+    return this.layers[0]!.ingest(hash, compressed, meta)
+  }
+
+  recordRunBundle(bundle: { runs: readonly RunRecord[]; invocation: InvocationRecord }): void {
+    this.layers[0]!.recordRunBundle(bundle)
+  }
+
+  stats(opts?: CacheStatsOptions): CacheStats {
+    return this.layers[0]!.stats(opts)
+  }
+
+  hashFile(filePath: string): Promise<string> {
+    return this.layers[0]!.hashFile(filePath)
+  }
+
+  outputsPath(hash: string): string {
+    return this.owner(hash).outputsPath(hash)
+  }
+
+  prune(options: PruneOptions): Promise<PruneResult> {
+    return this.layers[0]!.prune(options)
+  }
+
+  close(): void {
+    // Every layer closes even if an earlier one throws; the first error wins.
+    let failure: unknown
+    for (const layer of this.layers) {
+      try {
+        layer.close()
+      } catch (err) {
+        failure ??= err
+      }
+    }
+    if (failure !== undefined) throw failure
+  }
+}

package/src/cache/config-evals.ts

@@ -0,0 +1,89 @@
+// The config-evaluation cache tables (`config_evals`, `config_closures`):
+// what `workspace/config-cache.ts` reads and writes through the
+// `ConfigEvalStore` contract, honouring the run's local read/write axes.
+// Owns its statements over the store's handle; `Cache` delegates.
+
+import type { Database, SQLQueryBindings } from 'bun:sqlite'
+
+export class ConfigEvalTable {
+  private readonly selectConfigEval: ReturnType<Database['prepare']>
+  private readonly insertConfigEval: ReturnType<Database['prepare']>
+  private readonly read: boolean
+  private readonly write: boolean
+
+  constructor(
+    private readonly db: Database,
+    policy: { read: boolean; write: boolean },
+  ) {
+    this.read = policy.read
+    this.write = policy.write
+    this.selectConfigEval = this.db.prepare('SELECT json FROM config_evals WHERE key = ?')
+    this.insertConfigEval = this.db.prepare(
+      'INSERT OR REPLACE INTO config_evals(key, json, created_at) VALUES (?, ?, ?)',
+    )
+  }
+
+  /** `ConfigEvalStore`: a cached config evaluation, honouring the local READ axis. */
+  getConfigEval(key: string): string | null {
+    if (!this.read) return null
+    const row = this.selectConfigEval.get(key) as { json: string } | null
+    return row?.json ?? null
+  }
+
+  /** `ConfigEvalStore`: each config's ordered closure, one `IN` query per 900 paths. */
+  getConfigClosures(configPaths: readonly string[]): Map<string, string[]> {
+    const out = new Map<string, string[]>()
+    if (!this.read || configPaths.length === 0) return out
+    for (let i = 0; i < configPaths.length; i += 900) {
+      const chunk = configPaths.slice(i, i + 900)
+      const rows = this.db
+        .query(
+          `SELECT config_path, files_json FROM config_closures WHERE config_path IN (${chunk.map(() => '?').join(',')})`,
+        )
+        .all(...(chunk as readonly SQLQueryBindings[])) as Array<{
+        config_path: string
+        files_json: string
+      }>
+      for (const r of rows) out.set(r.config_path, JSON.parse(r.files_json) as string[])
+    }
+    return out
+  }
+
+  /** `ConfigEvalStore`: remember a config's ordered closure, honouring the local WRITE axis. */
+  putConfigClosure(configPath: string, files: readonly string[]): void {
+    if (!this.write) return
+    this.db
+      .prepare(
+        'INSERT INTO config_closures(config_path, files_json, created_at) VALUES (?, ?, ?) ON CONFLICT(config_path) DO UPDATE SET files_json = excluded.files_json, created_at = excluded.created_at',
+      )
+      .run(configPath, JSON.stringify(files), Date.now())
+  }
+
+  /** `ConfigEvalStore`: the batched read — one `IN` query per 900 keys, honouring the local READ axis. */
+  getConfigEvals(keys: readonly string[]): Map<string, string> {
+    const out = new Map<string, string>()
+    if (!this.read || keys.length === 0) return out
+    for (let i = 0; i < keys.length; i += 900) {
+      const chunk = keys.slice(i, i + 900)
+      const rows = this.db
+        .query(
+          `SELECT key, json FROM config_evals WHERE key IN (${chunk.map(() => '?').join(',')})`,
+        )
+        .all(...(chunk as readonly SQLQueryBindings[])) as Array<{ key: string; json: string }>
+      for (const r of rows) out.set(r.key, r.json)
+    }
+    return out
+  }
+
+  /** `ConfigEvalStore`: remember a validated evaluation, honouring the local WRITE axis. */
+  putConfigEval(key: string, json: string): void {
+    if (!this.write) return
+    this.insertConfigEval.run(key, json, Date.now())
+  }
+
+  /** Retention: a config not loaded since `cutoff` was edited or its project left. */
+  pruneOlderThan(cutoff: number): void {
+    this.db.prepare('DELETE FROM config_evals WHERE created_at < ?').run(cutoff)
+    this.db.prepare('DELETE FROM config_closures WHERE created_at < ?').run(cutoff)
+  }
+}

package/src/cache/file-hashes.ts

@@ -0,0 +1,262 @@
+// The per-file content-hash memo. A file's cache-key contribution is its
+// git blob OID; a clean tracked file takes it from the index (inputs.ts),
+// everything else is hashed here from the worktree bytes — memoized on
+// (mtime, size, ctime, ino) in `file_hashes`, the same four fields git's
+// own index keys on — and a symlink hashes as the blob of its target
+// string, exactly as the index stores it. Owns its two statements over
+// the store's handle; `Cache` delegates.
+
+import type { Database } from 'bun:sqlite'
+import { lstatSync, readlinkSync, statSync } from 'node:fs'
+import path from 'node:path'
+import { FILE_HASH_RACY_MS } from './layer.js'
+
+export class FileHashStore {
+  private readonly selectFileHash: ReturnType<Database['prepare']>
+  private readonly upsertFileHash: ReturnType<Database['prepare']>
+  private objectFormat: 'sha1' | 'sha256' | null = null
+
+  constructor(
+    private readonly db: Database,
+    /** Where `git rev-parse --show-object-format` is asked, once, when a file misses the memo. */
+    private readonly cacheDir: string,
+  ) {
+    this.selectFileHash = this.db.prepare(
+      'SELECT mtime_ms, size_bytes, ctime_ms, ino, content_hash FROM file_hashes WHERE path = ?',
+    )
+    this.upsertFileHash = this.db.prepare(`
+      INSERT INTO file_hashes(path, mtime_ms, size_bytes, ctime_ms, ino, content_hash, seen_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?)
+      ON CONFLICT(path) DO UPDATE SET
+        mtime_ms     = excluded.mtime_ms,
+        size_bytes   = excluded.size_bytes,
+        ctime_ms     = excluded.ctime_ms,
+        ino          = excluded.ino,
+        content_hash = excluded.content_hash,
+        seen_at      = excluded.seen_at
+    `)
+  }
+
+  /**
+   * Content-hash a file (as a git blob OID, v20) with an mtime+size
+   * fast path. If the `file_hashes` table has a row for `path` whose
+   * `(mtime_ms, size_bytes, ctime_ms, ino)` all match the current
+   * stat, we reuse the stored content_hash (a memory + SQLite lookup,
+   * no disk read). Otherwise we read + hash + upsert. All four fields
+   * are load-bearing — see the comment at the comparison.
+   *
+   * The OID is byte-identical to what `git hash-object` (and the git
+   * index) computes for the same content — for a symlink, the index's
+   * mode-120000 blob of the link text — so this fallback and the
+   * `CacheKeyInput.fileHashes` index-OID fast path agree on any file
+   * git stores verbatim. They do NOT agree when a clean filter
+   * (`text`/`eol`/`ident`) is active: the index blob is the filtered
+   * form while this hashes the worktree bytes, so `inputs.ts` drops
+   * the index OID for those paths and routes them here.
+   */
+  async hashFile(filePath: string): Promise<string> {
+    // lstatSync intentional: a single stat is ~1.6µs (Bun 1.3); the
+    // async-stat equivalent adds ~75µs of Promise machinery per call.
+    // Promise.all over the batched callers (key derivation) gives no
+    // I/O parallelism benefit because the stat is faster than the
+    // threadpool dispatch overhead. lstat, not stat, so a symlink is
+    // seen as one.
+    let st
+    try {
+      st = lstatSync(filePath)
+    } catch {
+      // Caller is responsible for skipping files that don't exist;
+      // fall through to the content-hash path which will throw with
+      // a more useful error.
+      return await this.hashFileFromDisk(filePath)
+    }
+    // A symlink folds as git folds it: the blob of its TARGET STRING, which
+    // is its mode-120000 index OID. Not the bytes behind it — a link to a
+    // directory has none, a dangling one has none, and a link to a file
+    // outside the project would fold bytes `git diff` and `--affected`
+    // cannot see. A link to a file inside the project still tracks that
+    // file's content, because the file is an input in its own right. No
+    // memo: readlink is one syscall, and the row would be keyed on the
+    // link's own stat, not its target's.
+    if (st.isSymbolicLink()) return this.hashBlob(new TextEncoder().encode(readlinkSync(filePath)))
+    const mtimeMs = Math.floor(st.mtimeMs)
+    const size = st.size
+    // ctime + ino are what make this memo SAFE, not merely fast. mtime is
+    // caller-settable, so (mtime, size) alone hands back the previous run's
+    // digest for genuinely different bytes whenever a producer preserves
+    // mtime — `tar -x`, `unzip`, `cp -p`, `rsync --times`, any
+    // SOURCE_DATE_EPOCH generator — which is a stale cache hit. `utimes`
+    // cannot suppress ctime without root, and an atomic write-then-rename
+    // changes the inode; git's own index keys on ctime+ino+dev for exactly
+    // this reason. Both fields come from the stat we already took.
+    const ctimeMs = Math.floor(st.ctimeMs)
+    const ino = Number(st.ino)
+    const row = this.selectFileHash.get(filePath) as
+      | {
+          mtime_ms: number
+          size_bytes: number
+          ctime_ms: number
+          ino: number
+          content_hash: string
+        }
+      | undefined
+    if (
+      row &&
+      row.mtime_ms === mtimeMs &&
+      row.size_bytes === size &&
+      row.ctime_ms === ctimeMs &&
+      row.ino === ino
+    ) {
+      return row.content_hash
+    }
+    const ch = await this.hashFileFromDisk(filePath)
+    // git's racy-clean rule, applied to the memo: a stat taken in the same
+    // tick as the file's last change cannot be trusted next time, because a
+    // rewrite landing in that tick keeps ctime (and, with mtime restored,
+    // every other field) equal and would hand back THIS digest for other
+    // bytes — seen once on ubuntu CI, on a docs-only commit. So a file
+    // changed within the window is hashed again on its next call rather
+    // than memoised; the warm path never meets it (keys are derived long
+    // after the files were written).
+    if (Date.now() - ctimeMs >= FILE_HASH_RACY_MS) {
+      this.upsertFileHash.run(filePath, mtimeMs, size, ctimeMs, ino, ch, Date.now())
+    }
+    return ch
+  }
+
+  /**
+   * `hashFile` for many paths at once, with ONE memo query. The per-file
+   * form costs a SQLite point lookup each; a warm config load keys 1,000
+   * closures that way (7.7 ms of `get` in a 2026-09-09 profile), where one
+   * `IN` query over all of them costs well under a millisecond. Same stat
+   * fields, same racy-clean rule, same digest. A path that cannot be
+   * stat'ed is absent from the result (the caller decides what a missing
+   * identity means) rather than thrown, so one vanished preset does not
+   * fail the batch for every other config.
+   */
+  async hashFiles(paths: readonly string[]): Promise<Map<string, string>> {
+    const out = new Map<string, string>()
+    if (paths.length === 0) return out
+    interface Stat {
+      mtimeMs: number
+      size: number
+      ctimeMs: number
+      ino: number
+    }
+    const stats = new Map<string, Stat>()
+    for (const p of paths) {
+      if (stats.has(p)) continue
+      try {
+        const st = statSync(p)
+        stats.set(p, {
+          mtimeMs: Math.floor(st.mtimeMs),
+          size: st.size,
+          ctimeMs: Math.floor(st.ctimeMs),
+          ino: Number(st.ino),
+        })
+      } catch {
+        // absent from the result
+      }
+    }
+    const wanted = [...stats.keys()]
+    const misses: string[] = []
+    // SQLite's bound-variable ceiling is 32,766 (999 on old builds); 500
+    // keeps a closure list of any size inside either.
+    for (let i = 0; i < wanted.length; i += 500) {
+      const chunk = wanted.slice(i, i + 500)
+      const rows = this.db
+        .query(
+          `SELECT path, mtime_ms, size_bytes, ctime_ms, ino, content_hash FROM file_hashes WHERE path IN (${chunk.map(() => '?').join(',')})`,
+        )
+        .all(...chunk) as Array<{
+        path: string
+        mtime_ms: number
+        size_bytes: number
+        ctime_ms: number
+        ino: number
+        content_hash: string
+      }>
+      const byPath = new Map(rows.map((r) => [r.path, r]))
+      for (const p of chunk) {
+        const st = stats.get(p)!
+        const row = byPath.get(p)
+        if (
+          row &&
+          row.mtime_ms === st.mtimeMs &&
+          row.size_bytes === st.size &&
+          row.ctime_ms === st.ctimeMs &&
+          row.ino === st.ino
+        ) {
+          out.set(p, row.content_hash)
+        } else misses.push(p)
+      }
+    }
+    if (misses.length === 0) return out
+    const digests = await Promise.all(
+      misses.map((p) => this.hashFileFromDisk(p).catch(() => undefined)),
+    )
+    const now = Date.now()
+    this.db.transaction(() => {
+      for (let i = 0; i < misses.length; i++) {
+        const digest = digests[i]
+        if (digest === undefined) continue
+        const p = misses[i]!
+        const st = stats.get(p)!
+        out.set(p, digest)
+        // The same racy-clean rule as `hashFile`: a stat taken within the
+        // window of the file's last change is not memoised.
+        if (now - st.ctimeMs >= FILE_HASH_RACY_MS) {
+          this.upsertFileHash.run(p, st.mtimeMs, st.size, st.ctimeMs, st.ino, digest, now)
+        }
+      }
+    })()
+    return out
+  }
+
+  /**
+   * Git blob OID of the file's bytes:
+   * `hex(HASH("blob " + byteLength + "\0" + content))`, where HASH is
+   * the repo's object format. Same value `git hash-object` prints and
+   * the same value the index stores. Computed in-process — no git
+   * spawn per file.
+   */
+  private async hashFileFromDisk(filePath: string): Promise<string> {
+    return this.hashBlob(await Bun.file(filePath).bytes(), filePath)
+  }
+
+  /** `git hash-object` of `bytes`: the blob OID in the repo's object format. */
+  private hashBlob(bytes: Uint8Array, nearPath?: string): string {
+    const hasher = new Bun.CryptoHasher(
+      this.objectFormat ?? this.detectObjectFormat(nearPath ?? this.cacheDir),
+    )
+    hasher.update(`blob ${bytes.byteLength}\0`)
+    hasher.update(bytes)
+    return hasher.digest('hex')
+  }
+
+  /**
+   * Repo object format — sha1 unless the repo was created with
+   * `--object-format=sha256`. One `git rev-parse` spawn per Cache
+   * lifetime, and only when at least one file misses the mtime+size
+   * memo. Outside a repo (unit fixtures) we default to sha1, which is
+   * still a deterministic blob-OID domain.
+   */
+  private detectObjectFormat(nearPath: string): 'sha1' | 'sha256' {
+    let detected: 'sha1' | 'sha256' = 'sha1'
+    try {
+      const proc = Bun.spawnSync({
+        cmd: ['git', 'rev-parse', '--show-object-format'],
+        cwd: path.dirname(nearPath),
+        stdout: 'pipe',
+        stderr: 'pipe',
+      })
+      if (proc.exitCode === 0 && new TextDecoder().decode(proc.stdout).trim() === 'sha256') {
+        detected = 'sha256'
+      }
+    } catch {
+      // git unavailable → sha1 default keeps hashing deterministic.
+    }
+    this.objectFormat = detected
+    return detected
+  }
+}