@vzn/vx 0.0.641 → 0.0.642

package/src/cache/cache.ts

@@ -109,9 +109,10 @@ import {
   zstdEncoder,
 } from './zstd.js'
 import { FileHashStore } from './file-hashes.js'
-import { OutputIndex } from './output-index.js'
+import { LINK_MODE, OutputIndex } from './output-index.js'
 import { RunHistory } from './run-history.js'
 import { CACHE_VERSION, foldKey } from './key-fold.js'
+import { FoldMemo } from './fold-memo.js'
 import { behindLink } from './linked-inputs.js'
 
 /**
@@ -315,7 +316,11 @@ function outOfFdsAtOpen(dbFile: string, err: unknown): UserError | undefined {
 //        unchanged.
 //   v40: runs.remote_error — each remote read or write of a task's key that
 //        failed, for `vx last`. The cache KEY is unchanged.
-export const SCHEMA_VERSION = 'v40'
+//   v41: the artifacts table is a rowid table (layout a2). The cache KEY
+//        is unchanged.
+//   v42: fold_memo — each task's last key by its fold up to the files and
+//        a token of their git records. The cache KEY is unchanged.
+export const SCHEMA_VERSION = 'v42'
 
 /**
  * An entry's key components as its one `entry_inputs` row (v34): JSON
@@ -363,7 +368,7 @@ const ARTIFACT_TABLES = ['artifacts', 'artifacts_meta']
  * reset, a new index or store, a store moved), so a change to it bumps
  * `SCHEMA_VERSION` too: a warm open reads no statement for it.
  */
-const ARTIFACT_LAYOUT = 'a1'
+const ARTIFACT_LAYOUT = 'a2'
 
 /**
  * The largest compressed artifact kept in the index rather than in a file
@@ -807,6 +812,12 @@ export interface RemoteTraffic {
   /** Uploads queued or running across every remote. */
   inFlight: number
   readonly keys: Map<string, RemoteKeyTraffic>
+  /**
+   * A key a remote is asked under that is not the task's own (a scope's)
+   * → the task key it answers for: its traffic counts under the task key,
+   * which the footer and the run's rows look up.
+   */
+  readonly aliases: Map<string, string>
 }
 
 export class Cache implements CacheLayer {
@@ -819,7 +830,7 @@ export class Cache implements CacheLayer {
    */
   readonly uploads: UploadTally = { count: 0, bytes: 0, ms: 0, failed: 0 }
   /** The same remotes' traffic per key, filled by the same `LayeredCache`. */
-  readonly traffic: RemoteTraffic = { inFlight: 0, keys: new Map() }
+  readonly traffic: RemoteTraffic = { inFlight: 0, keys: new Map(), aliases: new Map() }
 
   private readonly db: Database
   private readonly insertEntry: ReturnType<Database['prepare']>
@@ -851,6 +862,7 @@ export class Cache implements CacheLayer {
   /** The per-file (mtime, size) → blob-OID memo behind `hashFile`. */
   private readonly files: FileHashStore
   private readonly outputs: OutputIndex
+  private readonly folds: FoldMemo
   private readonly history: RunHistory
 
   /**
@@ -866,6 +878,8 @@ export class Cache implements CacheLayer {
   private readonly inspecting: boolean
   /** Why this process cannot write into `cacheDir`, or `null`; decided at open. */
   private readonly writeBlocked: string | null
+  /** The index is in memory: nothing on disk to read. */
+  private readonly empty: boolean
   private readonly dbFile: string
   /** Where the artifacts live: the shared store, or `cacheDir` itself. */
   private readonly artifactDir: string
@@ -952,8 +966,15 @@ export class Cache implements CacheLayer {
       (!existsSync(dbFile) || (mode === 'preview' && indexOfAnotherSchema(dbFile)))
     this.writeBlocked = absent ? 'no index there yet' : openCacheDir(cacheDir, repoDir)
     this.write = localPolicy.write && this.writeBlocked === null
+    // A directory this user cannot write with no index in it yet: SQLite
+    // cannot make one, and its `unable to open database file` stack reached
+    // the user before `assertWritable` could name the directory (a
+    // read-only mount, Growth's parity run 2026-10-16). Nothing there to
+    // read, so an empty index in memory, as a reader's typo gets.
+    const memory = absent || (this.writeBlocked !== null && !existsSync(dbFile))
+    this.empty = memory
     try {
-      this.db = new Database(absent ? ':memory:' : dbFile, { create: true })
+      this.db = new Database(memory ? ':memory:' : dbFile, { create: true })
     } catch (err) {
       throw outOfFdsAtOpen(dbFile, err) ?? err
     }
@@ -1062,7 +1083,7 @@ export class Cache implements CacheLayer {
             // keeps it so).
             if (found !== undefined) {
               this.db.exec(
-                'DROP TABLE IF EXISTS main.entries; DROP TABLE IF EXISTS main.runs; DROP TABLE IF EXISTS main.file_hashes; DROP TABLE IF EXISTS main.blob_sizes; DROP TABLE IF EXISTS main.blob_verdicts; DROP TABLE IF EXISTS main.output_files; DROP TABLE IF EXISTS main.output_stamps; DROP TABLE IF EXISTS main.invocations; DROP TABLE IF EXISTS main.run_task_inputs; DROP TABLE IF EXISTS main.entry_inputs; DROP TABLE IF EXISTS main.config_evals; DROP TABLE IF EXISTS main.config_closures; DROP TABLE IF EXISTS main.output_dirs; DROP TABLE IF EXISTS main.entry_stdout; DROP TABLE IF EXISTS main.task_durations; DROP TABLE IF EXISTS main.plugin_blobs; DROP TABLE IF EXISTS main.task_outcomes; DROP TABLE IF EXISTS main.store_meta;',
+                'DROP TABLE IF EXISTS main.entries; DROP TABLE IF EXISTS main.runs; DROP TABLE IF EXISTS main.file_hashes; DROP TABLE IF EXISTS main.blob_sizes; DROP TABLE IF EXISTS main.blob_verdicts; DROP TABLE IF EXISTS main.output_files; DROP TABLE IF EXISTS main.output_stamps; DROP TABLE IF EXISTS main.invocations; DROP TABLE IF EXISTS main.run_task_inputs; DROP TABLE IF EXISTS main.entry_inputs; DROP TABLE IF EXISTS main.config_evals; DROP TABLE IF EXISTS main.config_closures; DROP TABLE IF EXISTS main.output_dirs; DROP TABLE IF EXISTS main.entry_stdout; DROP TABLE IF EXISTS main.task_durations; DROP TABLE IF EXISTS main.plugin_blobs; DROP TABLE IF EXISTS main.task_outcomes; DROP TABLE IF EXISTS main.store_meta; DROP TABLE IF EXISTS main.fold_memo;',
               )
             }
             // The tables land under the same write lock as the stamp: made
@@ -1224,6 +1245,7 @@ export class Cache implements CacheLayer {
     // the schema above is the one place every table is declared.
     this.files = new FileHashStore(this.db, cacheDir, this.write, repoDir)
     this.outputs = new OutputIndex(this.db)
+    this.folds = new FoldMemo(this.db, this.write && !this.inspecting)
     this.history = new RunHistory(this.db, (v) => this.digestValue(v))
 
     // A CACHE_VERSION bump keeps the index but moves every key, so the run
@@ -1493,6 +1515,11 @@ export class Cache implements CacheLayer {
     return rel
   }
 
+  /** Each task's last key by its probe (`computeTaskHash`). */
+  foldMemo(): FoldMemo {
+    return this.folds
+  }
+
   key(input: CacheKeyInput): Promise<string> {
     const hashFile = (f: string): Promise<string> => this.hashFile(f)
     return foldKey(
@@ -2091,17 +2118,31 @@ export class Cache implements CacheLayer {
     )
   }
 
+  /**
+   * A run over a cache directory it cannot write: one holding entries
+   * serves its hits and saves nothing (M, 2026-10-16), and the line this
+   * returns says so; an empty one has nothing to serve and refuses.
+   */
+  readOnlyNotice(): string | null {
+    if (this.writeBlocked === null) return null
+    if (this.empty) this.assertWritable()
+    return (
+      `cache directory ${this.cacheDir} is not writable (${this.writeBlocked}) — serving hits ` +
+      `from it; this run saves nothing and records no history`
+    )
+  }
+
   async save(args: {
     hash: string
     /** See `CacheLayer.save` — `exitCode` is not a caller's to supply. */
     entry: Omit<CacheEntry, 'hash' | 'storedAt' | 'outputFiles' | 'exitCode'>
     projectDir: string
     outputFiles: string[]
+    outputDirs?: string[]
     workspaceOutputFiles?: string[]
     workspaceRoot?: string
     inputComponents?: readonly TaskInputRow[]
     /** See `CacheLayer.save`. */
-    readable?: ((real: string) => boolean) | undefined
   }): Promise<void> {
     // Layout (v17, extended additively for workspaceFiles): one
     // `<hash>.tar.zst` per entry. Tar carries ONLY the things you'd
@@ -2267,6 +2308,7 @@ export class Cache implements CacheLayer {
   private outputsOf(args: {
     projectDir: string
     outputFiles: string[]
+    outputDirs?: string[]
     workspaceOutputFiles?: string[]
     workspaceRoot?: string
   }): Map<string, string> {
@@ -2299,9 +2341,9 @@ export class Cache implements CacheLayer {
     entry: Omit<CacheEntry, 'hash' | 'storedAt' | 'outputFiles' | 'exitCode'>
     projectDir: string
     outputFiles: string[]
+    outputDirs?: string[]
     workspaceOutputFiles?: string[]
     workspaceRoot?: string
-    readable?: ((real: string) => boolean) | undefined
   }): Promise<Uint8Array> {
     // stdout is ALWAYS present in the artifact, even if empty, so the
     // layout is predictable: a successful read finds `stdout` and
@@ -2323,9 +2365,9 @@ export class Cache implements CacheLayer {
       key: args.hash,
       stdout: args.entry.stdout ?? '',
       outputs: this.outputsOf(args),
+      dirs: args.outputDirs?.map((d) => `outputs/${relPosix(args.projectDir, d)}`),
       exec: usageOfEntry(args.entry),
-      within: args.projectDir,
-      readable: args.readable,
+      root: args.workspaceRoot ?? args.projectDir,
     })
     if (plan.size > this.artifactCeiling) {
       throw new Error(
@@ -2463,7 +2505,10 @@ export class Cache implements CacheLayer {
             : `artifact was stored under another key (${scanned.key})`,
         )
       }
-      assertArtifactNames(hash, scanned.entries, meta.outputs)
+      assertArtifactNames(hash, scanned.entries, meta.outputs, [
+        ...scanned.dirs,
+        ...scanned.links.map(([name]) => name),
+      ])
     } catch (err) {
       if (tmpPath !== undefined) await unlink(tmpPath).catch(() => undefined)
       if (err instanceof ArchiveSecurityError || err instanceof CorruptArtifactError) throw err
@@ -2496,6 +2541,28 @@ export class Cache implements CacheLayer {
       if (rowPath === null) continue
       outputFileRows.push([rowPath, e.size, e.mode & 0o777, Math.floor(e.mtimeMs)])
     }
+    // An empty directory's row is its rel with a trailing slash: the hit's
+    // check asks only that a directory stands there.
+    for (const d of scanned.dirs) {
+      outputFileRows.push([
+        `${d.startsWith('outputs/') ? d.slice('outputs/'.length) : d}/`,
+        0,
+        0,
+        0,
+      ])
+    }
+    // A link's row: its rel, the target's length and CRC-32 where a file
+    // has its size and mtime, and the link type in the mode, so the hit's
+    // check compares the target and never follows it.
+    for (const [name, to] of scanned.links) {
+      const bytes = new TextEncoder().encode(to)
+      outputFileRows.push([
+        name.startsWith('outputs/') ? name.slice('outputs/'.length) : name,
+        bytes.byteLength,
+        LINK_MODE,
+        Bun.hash.crc32(bytes),
+      ])
+    }
     const stdoutText = scanned.stdout
 
     const [project, task] = splitTaskId(meta.taskId)
@@ -2665,13 +2732,18 @@ export class Cache implements CacheLayer {
   }
 
   // --- run history: delegated to `RunHistory` (see run-history.ts) ---
+  // A run over a directory it cannot write records no history:
+  // `readOnlyNotice` said so once.
   recordRun(run: RunRecord): void {
+    if (this.writeBlocked !== null) return
     return this.guard(() => this.history.recordRun(run))
   }
   recordRuns(runs: readonly RunRecord[]): void {
+    if (this.writeBlocked !== null) return
     return this.guard(() => this.history.recordRuns(runs))
   }
   recordRunBundle(bundle: { runs: readonly RunRecord[]; invocation: InvocationRecord }): void {
+    if (this.writeBlocked !== null) return
     return this.guard(() => this.history.recordRunBundle(bundle))
   }
   recordRemoteErrors(runId: string, errors: readonly (readonly [string, string])[]): void {
@@ -3026,6 +3098,7 @@ export class Cache implements CacheLayer {
     try {
       this.flushAccessed()
       this.outputs.flushOutputDirs()
+      this.folds.flush(Date.now() - 30 * 24 * 60 * 60 * 1000)
     } catch {
       // Same contract as the retention prune above, which was already
       // guarded while this sibling was not: `accessed_at` is LRU
@@ -3083,8 +3156,15 @@ function assertArtifactNames(
   hash: string,
   entries: ReadonlyArray<{ name: string }>,
   declared: DeclaredOutputs | undefined,
+  dirs: readonly string[] = [],
 ): void {
   const names = new Set(entries.map((e) => e.name))
+  for (const d of dirs) {
+    if (names.has(d)) {
+      throw new CorruptArtifactError(hash, `artifact holds ${d} as a file and as a directory`)
+    }
+    names.add(d)
+  }
   for (const name of names) {
     for (let sep = name.indexOf('/'); sep !== -1; sep = name.indexOf('/', sep + 1)) {
       if (names.has(name.slice(0, sep))) {
@@ -3116,7 +3196,7 @@ function assertArtifactNames(
  */
 function declaredName(declared: DeclaredOutputs): (name: string) => boolean {
   const isFile = restorableOutput(declared.files)
-  const isWsFile = restorableOutput(declared.workspaceFiles)
+  const isWsFile = restorableOutput(declared.workspaceFiles, 'workspace')
   return (name) =>
     name.startsWith('outputs/')
       ? isFile(name.slice('outputs/'.length))

package/src/cache/fold-memo.ts

@@ -0,0 +1,58 @@
+// Each task's last key, by what its input files were: a `probe` that is the
+// fold up to the files (`foldHead`) seeded into a token of the files' git
+// records and the declaration that picks among them (`filesToken`). Equal
+// probes are equal folds, so a warm task's key is read back instead of
+// resolving and folding its files again: ~300 ms of a 5,000-project warm
+// run (2026-10-17). One row per task, rewritten when its probe moves; a
+// pure memo, so a lost or swept row is one fold more.
+
+import type { Database } from 'bun:sqlite'
+
+export interface FoldMemoRow {
+  probe: string
+  key: string
+}
+
+export class FoldMemo {
+  private rows: Map<string, FoldMemoRow> | undefined
+  private readonly dirty = new Map<string, FoldMemoRow>()
+
+  constructor(
+    private readonly db: Database,
+    /** The local WRITE axis: off, and a fold is answered but not remembered. */
+    private readonly write: boolean,
+  ) {}
+
+  /** The key remembered for `taskId` under `probe`, if that is the probe it was folded under. */
+  get(taskId: string, probe: string): string | undefined {
+    // One read of every row: a warm run asks for nearly all of them.
+    this.rows ??= new Map(
+      (
+        this.db.query('SELECT task_id, probe, key FROM fold_memo').all() as Array<{
+          task_id: string
+          probe: string
+          key: string
+        }>
+      ).map((r) => [r.task_id, { probe: r.probe, key: r.key }]),
+    )
+    const row = this.dirty.get(taskId) ?? this.rows.get(taskId)
+    return row?.probe === probe ? row.key : undefined
+  }
+
+  set(taskId: string, probe: string, key: string): void {
+    if (this.write) this.dirty.set(taskId, { probe, key })
+  }
+
+  /** Write the folds this run made; rows a run has not rewritten for `cutoff` go. */
+  flush(cutoff: number, now: number = Date.now()): void {
+    if (this.dirty.size === 0) return
+    const upsert = this.db.prepare(
+      'INSERT OR REPLACE INTO fold_memo(task_id, probe, key, seen_at) VALUES (?, ?, ?, ?)',
+    )
+    this.db.transaction(() => {
+      for (const [taskId, r] of this.dirty) upsert.run(taskId, r.probe, r.key, now)
+      this.db.prepare('DELETE FROM fold_memo WHERE seen_at < ?').run(cutoff)
+    })()
+    this.dirty.clear()
+  }
+}