@vzn/vx 0.0.650 → 0.0.652

package/src/cache/cache.ts

@@ -51,7 +51,13 @@ import {
 } from 'node:fs'
 import { chmod, lstat, mkdir, readdir, rm, stat, unlink, utimes, writeFile } from 'node:fs/promises'
 import path from 'node:path'
-import { createArtifactTables, createTables, inHashes, lazyStatement } from './schema.js'
+import {
+  createArtifactTables,
+  createTables,
+  inHashes,
+  lazyStatement,
+  SQLITE_FCNTL_PERSIST_WAL,
+} from './schema.js'
 import {
   UserError,
   DOT_VX_GITIGNORE,
@@ -130,6 +136,9 @@ import { behindLink } from './linked-inputs.js'
  * this, so the file time trails the last use by at most this much, and
  * the policy reads the file time plus this as the last use.
  */
+// About 1.3 ms of inserts per turn: a task's exit waits at most that.
+const HELD_SLICE = 250
+
 const ORPHAN_GRACE_MS = 60 * 60 * 1000
 
 /**
@@ -272,7 +281,7 @@ function cantOpenIndex(dbFile: string, err: unknown): UserError | undefined {
 //        the memo simply stops answering wrongly, so an affected task's
 //        key moves from a WRONG value to the right one: it misses once,
 //        re-runs, re-caches. Self-healing, never a wrong hit.
-//   v25: runs.cached — whether the task declared a `cache` block. An
+//   v25: runs.cached — whether the task declared `cache: true`. An
 //        uncached task's row looked like any executed miss, so `vx why`
 //        headlined "cache key changed (inputs differ)" for a task that
 //        runs every time by design, and `vx last` could not mark it the
@@ -355,7 +364,10 @@ function cantOpenIndex(dbFile: string, err: unknown): UserError | undefined {
 //   v50: steps, warnings, failures and lockfile_memos — what a run said
 //        and a lockfile claimant parsed, files under the cache dir until
 //        now (if it can be in the DB, it is). The cache KEY is unchanged.
-export const SCHEMA_VERSION = 'v50'
+//   v51: runs.queued_ms, output_bytes, blocks, flaky_passes and
+//        flaky_failures — the rest of what a results row shows, so `vx
+//        last --sql` reads it. The cache KEY is unchanged.
+export const SCHEMA_VERSION = 'v51'
 
 /**
  * An entry's key components as its one `entry_inputs` row (v34): JSON
@@ -942,10 +954,21 @@ export class Cache implements CacheLayer {
   private adoptStats = 0
   /** The artifact directory's names, listed once past `ADOPT_STATS`; this process's writes kept in it. */
   private artifactNames: Set<string> | undefined
+  private heldBlobSizes: Array<[string, number]> | undefined
+  private heldTimer: ReturnType<typeof setTimeout> | undefined
+  /** Saves' index writes waiting for this turn's one transaction (`indexInBatch`). */
+  private indexQueue: Array<{ write: () => void; settle: (err?: unknown) => void }> = []
+  private heldRest: () => Array<[string, number]> = () => []
   private readonly insertEntryInput: ReturnType<Database['prepare']>
   /** The per-file (mtime, size) → blob-OID memo behind `hashFile`. */
   private readonly files: FileHashStore
   private readonly outputs: OutputIndex
+  /**
+   * Each entry `save()` wrote: its files' bytes, summed as the rows went in,
+   * for the miss row's `outputs` (a read back was one statement a miss;
+   * principle 9). Taken once, by `outputBytes`.
+   */
+  private readonly savedBytes = new Map<string, number>()
   private readonly folds: IndexFoldMemo
   private readonly history: RunHistory
   /** What the remotes said they hold, across runs (remote-hits.ts). */
@@ -1235,12 +1258,13 @@ export class Cache implements CacheLayer {
         }
       }
     }
-    if (storeDir !== undefined && mode === 'open' && this.writeBlocked === null) {
-      // An index that held its entries itself would shadow the store's:
-      // SQLite resolves an unqualified name in `main` first. Its history
-      // and memos stay; its entries' artifact files are orphans `prune`
-      // reaps, and its inline artifacts go with their table.
-      const holds = readable(
+    // An index that held its entries itself (no store yet: a repository
+    // before its first commit) hands them to the store below.
+    const holds =
+      storeDir !== undefined &&
+      mode === 'open' &&
+      this.writeBlocked === null &&
+      readable(
         () =>
           this.db
             .prepare(
@@ -1248,16 +1272,6 @@ export class Cache implements CacheLayer {
             )
             .get() != null,
       )
-      if (holds) {
-        this.db
-          .transaction(() => {
-            for (const t of [...STORE_TABLES, ...ARTIFACT_TABLES]) {
-              this.db.exec(`DROP TABLE IF EXISTS main.${t}`)
-            }
-          })
-          .immediate()
-      }
-    }
     this.storeDir = storeDir
     this.artifactDir = storeDir ?? cacheDir
     if (storeDir !== undefined) {
@@ -1271,6 +1285,7 @@ export class Cache implements CacheLayer {
     if (mode === 'open' && this.writeBlocked === null && (foreign || storeDir !== recorded)) {
       readable(() => this.matchArtifactLayout(storeDir === undefined ? 'main' : 'store'))
     }
+    if (holds) this.carryEntries(cacheDir, storeDir!)
     if (mode === 'open' && this.writeBlocked === null && storeDir !== recorded) {
       if (storeDir === undefined)
         this.db.prepare("DELETE FROM schema_meta WHERE key = 'store_dir'").run()
@@ -1396,6 +1411,68 @@ export class Cache implements CacheLayer {
     }
   }
 
+  /**
+   * The entries an index held itself, moved into the store it now opens:
+   * the same keys over the same bytes, so the first run after a
+   * repository's first commit replays what the run before it saved. Then
+   * dropped from the index, where an unqualified name would shadow the
+   * store's. An artifact file that cannot be renamed there (another
+   * device) is left for `prune`, and its entry is a miss.
+   */
+  private carryEntries(cacheDir: string, storeDir: string): void {
+    const has = (schema: 'main' | 'store', table: string): boolean =>
+      this.db
+        .prepare(`SELECT 1 FROM ${schema}.sqlite_master WHERE type = 'table' AND name = ?`)
+        .get(table) != null
+    const layout = (schema: 'main' | 'store'): string | undefined =>
+      has(schema, 'artifacts_meta')
+        ? (
+            this.db
+              .prepare(`SELECT value FROM ${schema}.artifacts_meta WHERE key = 'layout'`)
+              .get() as { value: string } | null
+          )?.value
+        : undefined
+    if (storeDir !== cacheDir) {
+      let names: string[] = []
+      try {
+        names = readdirSync(cacheDir)
+      } catch {
+        // No directory: nothing to carry.
+      }
+      for (const name of names) {
+        if (!name.endsWith('.tar.zst') && !name.endsWith('.tree')) continue
+        const to = path.join(storeDir, name)
+        if (existsSync(to)) continue
+        try {
+          renameSync(path.join(cacheDir, name), to)
+        } catch {
+          // Another device: the artifact stays an orphan here.
+        }
+      }
+    }
+    this.db
+      .transaction(() => {
+        // `entries` first: the other tables' rows reference it.
+        for (const t of STORE_TABLES.toReversed()) {
+          if (t !== 'store_meta' && has('main', t) && has('store', t)) {
+            this.db.exec(`INSERT OR IGNORE INTO store.${t} SELECT * FROM main.${t}`)
+          }
+        }
+        if (
+          has('main', 'artifacts') &&
+          has('store', 'artifacts') &&
+          layout('main') === ARTIFACT_LAYOUT &&
+          layout('store') === ARTIFACT_LAYOUT
+        ) {
+          this.db.exec('INSERT OR IGNORE INTO store.artifacts SELECT * FROM main.artifacts')
+        }
+        for (const t of [...STORE_TABLES, ...ARTIFACT_TABLES]) {
+          this.db.exec(`DROP TABLE IF EXISTS main.${t}`)
+        }
+      })
+      .immediate()
+  }
+
   /**
    * Attach the shared store as `store`, with the pragmas the index takes:
    * WAL for both, so a workspace's run and another's share the store as
@@ -1580,7 +1657,32 @@ export class Cache implements CacheLayer {
   }
   /** `BlobSizeMemo`: remember blob sizes, honouring the local WRITE axis. */
   rememberBlobSizes(sizes: ReadonlyMap<string, number>): void {
-    this.guard(() => this.files.rememberBlobSizes(sizes))
+    if (this.heldBlobSizes !== undefined) for (const e of sizes) this.heldBlobSizes.push(e)
+    else this.guard(() => this.files.rememberBlobSizes(sizes))
+  }
+  /**
+   * Hold the blob sizes remembered from now on until `storeHeldBlobSizes`
+   * or `close`: a run learns them before its first task, where storing
+   * 40,030 cost a cold 9,603-task run 217 ms (2026-11-01).
+   */
+  holdBlobSizes(): void {
+    this.heldBlobSizes ??= []
+  }
+  /**
+   * Store the held blob sizes `HELD_SLICE` per event-loop turn from the
+   * next turn on, while the run's tasks run; `close` stores the rest.
+   */
+  storeHeldBlobSizes(): void {
+    const held = this.heldBlobSizes
+    this.heldBlobSizes = undefined
+    if (held === undefined || held.length === 0) return
+    let at = 0
+    const step = (): void => {
+      this.guard(() => this.files.rememberBlobSizes(new Map(held.slice(at, (at += HELD_SLICE)))))
+      this.heldTimer = at < held.length ? setTimeout(step, 0) : undefined
+    }
+    this.heldRest = () => held.slice(at)
+    this.heldTimer = setTimeout(step, 0)
   }
   /** `BlobSizeMemo`: the paths an index with this digest distrusts, when known. */
   blobVerdict(digest: string): string[] | undefined {
@@ -1755,7 +1857,7 @@ export class Cache implements CacheLayer {
   }
 
   /** An adoption's verdict: an artifact that fails the checks is a miss. */
-  private async adopted(indexing: Promise<void>): Promise<boolean> {
+  private async adopted(indexing: Promise<unknown>): Promise<boolean> {
     try {
       await indexing
       return true
@@ -1953,6 +2055,11 @@ export class Cache implements CacheLayer {
   outputsPath(hash: string): string {
     return this.tarPath(hash)
   }
+  outputBytes(hash: string): number | undefined {
+    const bytes = this.savedBytes.get(hash)
+    this.savedBytes.delete(hash)
+    return bytes
+  }
   artifactSize(hash: string): number | undefined {
     return this.guard(
       () =>
@@ -2046,7 +2153,15 @@ export class Cache implements CacheLayer {
    * uploaded under a fresh sum.
    */
   private treePlan(hash: string, tree: string, rows: readonly OutputFileRow[]): ArtifactPlan {
-    const plan = planOfTree(tree, this.stdoutText(hash))
+    let plan: ArtifactPlan
+    try {
+      plan = planOfTree(tree, this.stdoutText(hash))
+    } catch (err) {
+      // A sidecar cut short at rest, as a file is: named as the damage it is.
+      if (err instanceof SyntaxError)
+        throw new CorruptArtifactError(hash, 'its sidecar does not parse')
+      throw err
+    }
     const size = new Map(rows.map((r) => [treeName(r.path), r.size]))
     for (const i of plan.inputs) {
       const want = size.get(i.name)
@@ -2488,7 +2603,16 @@ export class Cache implements CacheLayer {
     if (!this.write) return
     // A cache that reads saves only on a miss. One that does not (`--force`)
     // re-executes a key it may hold: that entry is kept as it is.
-    if (!this.read && this.guard(() => this.storedHere(args.hash))) return
+    // The kept entry's bytes are this run's too (same key): the row's
+    // `outputs` counts them as a hit would.
+    if (!this.read && this.guard(() => this.storedHere(args.hash))) {
+      const rows = this.loadOutputFilesBatch([args.hash]).get(args.hash) ?? []
+      this.savedBytes.set(
+        args.hash,
+        rows.reduce((n, r) => n + r.size, 0),
+      )
+      return
+    }
     // The cache directory is the constructor's (`mkdirSync`, with a
     // refusal recorded for `assertWritable`); re-creating it here and
     // again in `writeArtifactAndIndex` cost every save two thread-pool
@@ -2505,7 +2629,7 @@ export class Cache implements CacheLayer {
       throw err
     })
     endPack()
-    await this.guard(() =>
+    const bytes = await this.guard(() =>
       this.writeArtifactAndIndex(
         args.hash,
         'tar' in packed ? packed.compressed : 'plan' in packed ? packed.saved : packed,
@@ -2518,6 +2642,7 @@ export class Cache implements CacheLayer {
         'tar' in packed ? packed.tar : undefined,
       ),
     )
+    this.savedBytes.set(args.hash, bytes)
   }
 
   /**
@@ -2732,7 +2857,7 @@ export class Cache implements CacheLayer {
     if (plan.size > this.artifactCeiling) {
       throw new Error(
         `${args.entry.taskId} is not cached: its outputs pack to ${formatBytes(plan.size)}, past the ` +
-          `${formatBytes(this.artifactCeiling)} artifact ceiling a restore enforces — narrow cache.outputs.files`,
+          `${formatBytes(this.artifactCeiling)} artifact ceiling a restore enforces — narrow outputs.files`,
       )
     }
     return plan
@@ -2827,7 +2952,7 @@ export class Cache implements CacheLayer {
      * scanned as is rather than decoded back out of the bytes just encoded.
      */
     tar?: Uint8Array,
-  ): Promise<void> {
+  ): Promise<number> {
     // Validate BEFORE anything touches the final path. `ingest()` feeds
     // us a temp of network bytes; a truncated/garbage body that went live first
     // would leave a corrupt `<hash>.tar.zst` behind (with no SQL row,
@@ -3049,6 +3174,7 @@ export class Cache implements CacheLayer {
     const upsertStdout = this.upsertStdout
     const deleteStdout = this.deleteStdout
     const outputs = this.outputs
+    let outputBytes = 0
     const insertEntryInput = this.insertEntryInput
     const inputComponents = meta.inputComponents
     // The artifact goes live INSIDE the write transaction, taken
@@ -3140,7 +3266,7 @@ export class Cache implements CacheLayer {
           hash,
           stdoutText.startsWith('\uFEFF') ? utf8.encode(stdoutText) : stdoutText,
         )
-      outputs.replaceFileRows(hash, outputFileRows)
+      outputBytes = outputs.replaceFileRows(hash, outputFileRows)
       // INSERT OR IGNORE: identical inputs derive this same hash, so a
       // re-save's row is identical — keep the first, skip the rest.
       if (inputComponents !== undefined && inputComponents.length > 0) {
@@ -3152,7 +3278,7 @@ export class Cache implements CacheLayer {
       // Built inside the try: on a database a stopping run closed while a
       // download streamed in, building it throws, and the temp or tree
       // the body went to stayed behind.
-      this.db.transaction(index).immediate()
+      await this.indexInBatch(index)
     } catch (err) {
       endTx()
       const left = renamed ?? tree ?? tmpPath
@@ -3162,6 +3288,57 @@ export class Cache implements CacheLayer {
     }
     endTx()
     for (const a of aside) await rm(a, { recursive: true, force: true })
+    return outputBytes
+  }
+
+  /**
+   * Run `write` in a write transaction shared with every other save's that
+   * arrives in this turn: settles once it has committed, or rejects with
+   * its own error (its savepoint rolled back) or the commit's. A cold run's
+   * saves land a few per turn, and a transaction each spent half its index
+   * time on BEGIN and COMMIT on the main thread, the run's busiest
+   * (2026-11-03: 81 of 164 ms over 500 saves).
+   */
+  private indexInBatch(write: () => void): Promise<void> {
+    return new Promise((resolve, reject) => {
+      this.indexQueue.push({
+        write,
+        settle: (err) => (err === undefined ? resolve() : reject(err)),
+      })
+      if (this.indexQueue.length === 1) setImmediate(() => this.flushIndex())
+    })
+  }
+
+  private flushIndex(): void {
+    const batch = this.indexQueue
+    if (batch.length === 0) return
+    this.indexQueue = []
+    const failed = new Map<number, unknown>()
+    try {
+      this.db
+        .transaction(() => {
+          for (let i = 0; i < batch.length; i++) {
+            // An error SQLite answers by rolling the whole transaction
+            // back (a full disk, a busy lock) leaves none to go on in: the
+            // rest are not run, and fail with it.
+            if (failed.size > 0 && !this.db.inTransaction) {
+              failed.set(i, failed.values().next().value)
+              continue
+            }
+            try {
+              // Nested: a savepoint, so one write's failure undoes only its own rows.
+              this.db.transaction(batch[i]!.write)()
+            } catch (err) {
+              failed.set(i, err)
+            }
+          }
+        })
+        .immediate()
+    } catch (err) {
+      for (let i = 0; i < batch.length; i++) batch[i]!.settle(failed.get(i) ?? err)
+      return
+    }
+    for (let i = 0; i < batch.length; i++) batch[i]!.settle(failed.get(i))
   }
 
   private tempTreePath(hash: string): string {
@@ -3515,6 +3692,12 @@ export class Cache implements CacheLayer {
 
   close(): void {
     try {
+      this.flushIndex()
+      clearTimeout(this.heldTimer)
+      const held = this.heldBlobSizes ?? (this.heldTimer !== undefined ? this.heldRest() : [])
+      this.heldBlobSizes = undefined
+      this.heldTimer = undefined
+      if (held.length > 0) this.files.rememberBlobSizes(new Map(held))
       this.flushAccessed()
       this.outputs.flushOutputDirs()
       this.folds.flush()
@@ -3550,8 +3733,16 @@ export class Cache implements CacheLayer {
   }
 }
 
-/** SQLite's file-control opcode (sqlite3.h). */
-export const SQLITE_FCNTL_PERSIST_WAL = 10
+/**
+ * The files a second connection writes a run's hits through
+ * (history-stream.ts); undefined when the store keeps no history on disk.
+ * A function, as `indexDb` is: no plugin writes the history.
+ */
+export function historyFiles(cache: Cache): { index: string; store?: string } | undefined {
+  if (cache['empty'] || cache['writeBlocked'] !== null || cache['inspecting']) return undefined
+  if (cache.storeDir === undefined) return { index: cache['dbFile'] }
+  return { index: cache['dbFile'], store: path.join(cache.storeDir, 'store.db') }
+}
 
 /**
  * The cache's `bun:sqlite` handle, for core's own readers (`vx last`, `vx

package/src/cache/file-hashes.ts

@@ -72,6 +72,9 @@ export class FileHashStore {
   /** Each of `oids` whose size a `rememberBlobSizes` stored: one `IN` query per 900. */
   knownBlobSizes(oids: readonly string[]): Map<string, number> {
     const out = new Map<string, number>()
+    // An empty memo (a cold cache) answers nothing: 40,030 lookups there
+    // cost a 9,603-task run 57 ms before its first task (2026-11-01).
+    if (this.db.query('SELECT 1 FROM blob_sizes LIMIT 1').get() === null) return out
     for (let i = 0; i < oids.length; i += 900) {
       const chunk = oids.slice(i, i + 900)
       const rows = this.db