@vzn/vx 0.0.641 → 0.0.642

package/src/cache/git-inputs.ts

@@ -5,10 +5,12 @@
 // 2026-09-10: this file talks to git; `inputs.ts` decides which files a
 // task declared. Nothing here reads a config or applies a boundary.
 
+import os from 'node:os'
 import path from 'node:path'
 import {
   existsSync,
   lstatSync,
+  mkdtempSync,
   readdirSync,
   readFileSync,
   realpathSync,
@@ -19,6 +21,7 @@ import {
   type Stats,
 } from 'node:fs'
 import {
+  checkIgnoreSync,
   executablePath,
   gitEnv,
   gitSpawnRefusal,
@@ -30,6 +33,9 @@ import {
 } from '../util/index.js'
 import { FILE_HASH_RACY_MS, racyWindowMs } from './layer.js'
 
+/** A name no ignore rule names, standing for a file a task may create. */
+const FRESH_NAME = '__vx_probe__'
+
 /** Three facts of the repository a directory is in: read off the disk, or one `git rev-parse`. */
 export interface RepoFacts {
   /** `--show-prefix`: the directory's path below the worktree root, `''` at the root. */
@@ -195,6 +201,47 @@ function configObjectFormat(config: string): 'sha1' | 'sha256' | undefined {
   return format
 }
 
+interface PartitionDigest {
+  snapshot: readonly string[]
+  oids: OidSlot | undefined
+  digest: string
+}
+
+/**
+ * A partition's trusted OIDs, built on first read: a warm run whose keys
+ * come back from the fold memo reads none, and building 125,000 abs-path
+ * entries up front was ~90 ms of a 5,000-project warm run.
+ */
+class OidSlot {
+  private built: Map<string, string> | undefined
+  /** Paths dropped before the map was built: taken out when it is. */
+  private dropped: string[] | undefined
+  constructor(private build: () => Map<string, string>) {}
+  get map(): Map<string, string> {
+    if (this.built === undefined) {
+      this.built = this.build()
+      for (const p of this.dropped ?? []) this.built.delete(p)
+      this.dropped = undefined
+    }
+    return this.built
+  }
+  /**
+   * Forget these paths' OIDs. An unbuilt map stays unbuilt: each restore
+   * marks its outputs changed, and building the map to drop them was ~28 ms
+   * of a 1,000-project restore whose keys no later read needed.
+   */
+  drop(paths: Iterable<string>): void {
+    if (this.built !== undefined) for (const p of paths) this.built.delete(p)
+    else (this.dropped ??= []).push(...paths)
+  }
+}
+
+interface IgnoredListing {
+  root: string
+  paths: readonly string[]
+  asked: Map<string, boolean>
+}
+
 export class GitFilesCache extends Map<string, readonly string[]> {
   private changed = new Map<string, string[]>()
   /**
@@ -210,7 +257,9 @@ export class GitFilesCache extends Map<string, readonly string[]> {
    * (`Cache.hashFile`) computes the identical blob OID from disk, so
    * dropping is a pure perf concession, never a correctness one.
    */
-  private oids = new Map<string, Map<string, string>>()
+  private oids = new Map<string, OidSlot>()
+  /** Per partition, `filesDigest`'s answer and the snapshot and OIDs it describes. */
+  private digests = new Map<string, PartitionDigest>()
   /**
    * Set when `populateGitFilesCache` stored a WORKSPACE-WIDE partition
    * (any loaded task declares `inputs.workspaceFiles`). Null when the
@@ -236,8 +285,11 @@ export class GitFilesCache extends Map<string, readonly string[]> {
    * What git tracks, by absolute path, within `trackedScopes`: a file there
    * no clean deletes (`committedIn`). Filled by the
    * enumeration, and per directory on demand for one it did not cover.
+   * Built on the first question: a run that cleans nothing asks none, and
+   * 125,000 paths joined and hashed were ~90 ms of a warm 5,000-project run.
    */
   private tracked = new Set<string>()
+  private trackedPending: (() => Iterable<string>)[] = []
   private trackedScopes: string[] = []
   /**
    * Under `core.ignorecase` (macOS's default volume) git's `Foo.txt` is the
@@ -256,10 +308,23 @@ export class GitFilesCache extends Map<string, readonly string[]> {
   }
 
   noteTracked(scopes: readonly string[], absPaths: readonly string[]): void {
-    for (const p of absPaths) this.tracked.add(this.trackedKey(p))
+    this.noteTrackedLater(scopes, () => absPaths)
+  }
+
+  /** `noteTracked`, the paths asked for only when a clean asks about one. */
+  noteTrackedLater(scopes: readonly string[], absPaths: () => Iterable<string>): void {
+    this.trackedPending.push(absPaths)
     this.trackedScopes.push(...scopes)
   }
 
+  private isTracked(abs: string): boolean {
+    if (this.trackedPending.length > 0) {
+      for (const paths of this.trackedPending.splice(0))
+        for (const p of paths()) this.tracked.add(this.trackedKey(p))
+    }
+    return this.tracked.has(this.trackedKey(abs))
+  }
+
   /**
    * Whether git tracks a file under `dir`: committed (or `git add`ed)
    * files are the user's, never cleaned (owner, 2026-10-14: "never
@@ -277,7 +342,7 @@ export class GitFilesCache extends Map<string, readonly string[]> {
         if (!covered) this.noteTracked([dir], listTracked(dir))
       }
       for (let p = abs; p.length > dir.length; p = path.dirname(p))
-        if (this.tracked.has(this.trackedKey(p))) return true
+        if (this.isTracked(p)) return true
       return false
     }
   }
@@ -296,6 +361,46 @@ export class GitFilesCache extends Map<string, readonly string[]> {
    */
   ignoredOutputs: ReadonlySet<string> = new Set()
 
+  /** What `git status --ignored=matching` named, root-relative and sorted, and the root. */
+  private ignored: IgnoredListing | null = null
+
+  setIgnored(root: string, paths: readonly string[]): void {
+    this.ignored = { root, paths, asked: new Map() }
+  }
+
+  /**
+   * Whether git ignores a new file directly in `dir` (absolute) while it
+   * reports an ignored path under it: `gen/*` beside a kept `gen/.gitkeep`,
+   * or `gen/` with a force-added file. Such a directory is ignored in all
+   * but the files git lists, so naming it is asking for its ignored files
+   * too. A pattern that ignores only some names (`*.generated.js` in a
+   * listed `src`) answers false. Asked once per directory per enumeration,
+   * and only where status named an ignored path under it.
+   */
+  newFilesIgnored(dir: string): boolean {
+    const ig = this.ignored
+    if (ig === null || ig.paths.length === 0) return false
+    const rel = relPosix(ig.root, dir)
+    if (rel === '' || rel.startsWith('../')) return false
+    let known = ig.asked.get(rel)
+    if (known !== undefined) return known
+    const under = `${rel}/`
+    let lo = 0
+    let hi = ig.paths.length
+    while (lo < hi) {
+      const mid = (lo + hi) >>> 1
+      if (ig.paths[mid]! < under) lo = mid + 1
+      else hi = mid
+    }
+    known = false
+    if (ig.paths[lo]?.startsWith(under) === true) {
+      const probe = `${under}${FRESH_NAME}`
+      known = checkIgnoreSync(ig.root, [probe])?.has(probe) === true
+    }
+    ig.asked.set(rel, known)
+    return known
+  }
+
   get undecodableNames(): ReadonlySet<string> {
     return this.undecodable
   }
@@ -396,19 +501,39 @@ export class GitFilesCache extends Map<string, readonly string[]> {
     else this.changed.set(partitionDir, [...relPaths])
     // These paths just changed on disk; their index OIDs (if any) no
     // longer describe the working-tree content.
-    const partitionOids = this.oids.get(partitionDir)
-    if (partitionOids) {
-      for (const rel of relPaths) partitionOids.delete(path.resolve(partitionDir, rel))
-    }
+    this.oids.get(partitionDir)?.drop(relPaths.map((rel) => path.resolve(partitionDir, rel)))
+  }
+
+  /**
+   * A digest of `snapshot`'s paths and index OIDs, when the enumeration
+   * trusted an OID for every one, none is a symlink and every name decodes:
+   * then the files a task resolves there, and what it folds for them, are
+   * a function of this digest and its declaration (`filesToken`). Undefined
+   * once anything moved the partition: a re-enumeration, a changed path.
+   */
+  filesDigest(projectDir: string, snapshot: readonly string[]): string | undefined {
+    const d = this.digests.get(projectDir)
+    if (d === undefined || d.snapshot !== snapshot || this.get(projectDir) !== snapshot) return
+    if (this.oids.get(projectDir) !== d.oids || this.changed.has(projectDir)) return
+    return d.digest
+  }
+
+  noteFilesDigest(projectDir: string, snapshot: readonly string[], digest: string): void {
+    this.digests.set(projectDir, { snapshot, oids: this.oids.get(projectDir), digest })
   }
 
   /** Trusted index OIDs for a project (abs path → oid), if any survive. */
   oidsFor(projectDir: string): ReadonlyMap<string, string> | undefined {
-    return this.oids.get(projectDir)
+    return this.oids.get(projectDir)?.map
   }
 
   setOids(projectDir: string, oids: Map<string, string>): void {
-    this.oids.set(projectDir, oids)
+    this.oids.set(projectDir, new OidSlot(() => oids))
+  }
+
+  /** What builds the project's OIDs when a read first asks. */
+  setLazyOids(projectDir: string, build: () => Map<string, string>): void {
+    this.oids.set(projectDir, new OidSlot(build))
   }
 
   /** Snapshot if still valid for these input globs; undefined → re-spawn. */
@@ -425,18 +550,21 @@ export class GitFilesCache extends Map<string, readonly string[]> {
   override set(key: string, value: readonly string[]): this {
     this.changed.delete(key)
     this.oids.delete(key)
+    this.digests.delete(key)
     return super.set(key, value)
   }
 
   override delete(key: string): boolean {
     this.changed.delete(key)
     this.oids.delete(key)
+    this.digests.delete(key)
     return super.delete(key)
   }
 
   override clear(): void {
     this.changed.clear()
     this.oids.clear()
+    this.digests.clear()
     super.clear()
   }
 }
@@ -1201,17 +1329,20 @@ function parseStatusOutput(
   untracked: string[]
   undecodable: Set<string>
   ignoredAttributes: boolean
+  ignored: string[]
 } {
   const tokens = out.split('\0')
   const dirty = new Set<string>()
   const untracked: string[] = []
   const undecodable = new Set<string>()
+  const ignored: string[] = []
   let ignoredAttributes = false
   for (const token of tokens) {
     if (token.length < 4) continue
     if (token[0] === '!') {
       if (token === '!! .gitattributes' || token.endsWith('/.gitattributes'))
         ignoredAttributes = true
+      ignored.push(token.slice(3))
       continue
     }
     if (undecodableRecords.size > 0 && undecodableRecords.has(token))
@@ -1224,7 +1355,7 @@ function parseStatusOutput(
     // rename's source is a record of its own (items 976, A-59).
     dirty.add(token.slice(3))
   }
-  return { dirty, untracked, undecodable, ignoredAttributes }
+  return { dirty, untracked, undecodable, ignoredAttributes, ignored }
 }
 
 /**
@@ -1309,6 +1440,8 @@ export interface GitEnumeration {
   untracked: readonly string[] | null
   /** Workspace-relative paths whose names are not UTF-8 (`decodeGitZ`). */
   undecodable: readonly string[]
+  /** What `git status` named ignored, workspace-relative and sorted (a directory with its `/`). */
+  ignored: readonly string[]
   /** `Date.now()` before the spawns: what `trusted` says is true as of no earlier. */
   startedAtMs: number
   /** The blob-size check's input (`dropResizedOids`). */
@@ -1446,8 +1579,8 @@ export async function startGitEnumeration(
     stdin?: string,
     env?: Record<string, string>,
   ): Promise<GitRun | null> => {
-    try {
-      const proc = Bun.spawn({
+    const spawn = <Out extends 'pipe' | Bun.BunFile>(stdout: Out) =>
+      Bun.spawn({
         // `status` refreshes the index when it can take `index.lock`, so
         // the user's own `git add` or `commit` failed on the lock while a
         // vx run held it (12 in 1,165 across 80 runs, item 880). Read-only, as
@@ -1455,19 +1588,45 @@ export async function startGitEnumeration(
         cmd: [executablePath('git'), '--no-optional-locks', ...args],
         cwd: workspaceRoot,
         stdin: stdin === undefined ? 'ignore' : new TextEncoder().encode(stdin),
-        stdout: 'pipe',
+        stdout,
         stderr: 'pipe',
         env: env !== undefined ? { ...gitEnv(), ...env } : gitEnv(),
       })
-      const [stdout, stderr, exitCode] = await Promise.all([
-        new Response(proc.stdout).bytes(),
-        new Response(proc.stderr).text(),
-        proc.exited,
-      ])
+    // To a file, not a pipe: the run's thread is busy discovering and
+    // loading configs while git lists, and a pipe held a 10 MB listing
+    // until the thread came back to drain it 64 KiB at a time (~90 ms of
+    // a warm 5,000-project run). In a directory of its own, so no link
+    // another user planted in a shared temp directory takes the bytes. A
+    // temp directory that takes none falls back to the pipe.
+    let dir: string | undefined
+    try {
+      dir = mkdtempSync(path.join(os.tmpdir(), 'vx-git-'))
+    } catch {
+      dir = undefined
+    }
+    try {
+      let stdout: Uint8Array
+      let stderr: string
+      let exitCode: number
+      if (dir !== undefined) {
+        const out = path.join(dir, 'out')
+        const proc = spawn(Bun.file(out))
+        ;[stderr, exitCode] = await Promise.all([new Response(proc.stderr).text(), proc.exited])
+        stdout = await Bun.file(out).bytes()
+      } else {
+        const piped = spawn('pipe')
+        ;[stdout, stderr, exitCode] = await Promise.all([
+          new Response(piped.stdout).bytes(),
+          new Response(piped.stderr).text(),
+          piped.exited,
+        ])
+      }
       const { text, undecodable } = decodeGitZ(stdout)
       return { exitCode, stdout: text, stderr, undecodable }
     } catch {
       return null
+    } finally {
+      if (dir !== undefined) rmSync(dir, { recursive: true, force: true })
     }
   }
   const startedAtMs = Date.now()
@@ -1692,6 +1851,10 @@ export async function startGitEnumeration(
     changed: dirty === null ? null : [...dirty, ...untracked],
     untracked: dirty === null ? null : untracked,
     undecodable,
+    ignored:
+      parsedStatus === null
+        ? []
+        : [...stripPrefixFromSet(new Set(parsedStatus.ignored), gitPrefix)].sort(),
     startedAtMs,
     blobCheck: {
       key: indexKey,
@@ -1758,11 +1921,12 @@ export async function applyGitEnumeration(
   const { all, trusted } = enumeration
   cache.setWorktreeDirty(enumeration.dirty)
   cache.setGitVars(enumeration.vars)
+  cache.setIgnored(workspaceRoot, enumeration.ignored)
   cache.enumeratedAtMs = enumeration.startedAtMs
   cache.markUndecodable(enumeration.undecodable.map((rel) => path.join(workspaceRoot, rel)))
-  cache.noteTracked(
+  cache.noteTrackedLater(
     enumeration.scopes.map((rel) => path.join(workspaceRoot, rel)),
-    enumeration.tracked.map((rel) => path.join(workspaceRoot, rel)),
+    () => enumeration.tracked.map((rel) => path.join(workspaceRoot, rel)),
   )
   // Sort once, then each project's files are a contiguous range found
   // by binary search on its `dir/` prefix — O((F+P) log F) instead of
@@ -1780,6 +1944,8 @@ export async function applyGitEnumeration(
     }
   }
   const sorted = inOrder ? all : [...all].sort()
+  const undecodable =
+    enumeration.undecodable.length === 0 ? undefined : new Set(enumeration.undecodable)
   // Git prints normalized relative paths, so under an absolute, normalized
   // root a join is a concatenation: `path.join` per tracked file and
   // `path.relative` per project were most of this pass (I-30's rule).
@@ -1803,6 +1969,11 @@ export async function applyGitEnumeration(
     }
     return lo
   }
+  const rootOids = (): Map<string, string> => {
+    const oids = new Map<string, string>()
+    for (const [rel, oid] of trusted) oids.set(abs(rel), oid)
+    return oids
+  }
   for (const projectDir of projectDirs) {
     const spelled =
       base !== undefined && projectDir.startsWith(base) && path.normalize(projectDir) === projectDir
@@ -1815,23 +1986,35 @@ export async function applyGitEnumeration(
     const relPrefix = spelled.normalize('NFC')
     if (relPrefix === '' || relPrefix === '.') {
       cache.set(projectDir, all)
-      const rootOids = new Map<string, string>()
-      for (const [rel, oid] of trusted) rootOids.set(abs(rel), oid)
-      cache.setOids(projectDir, rootOids)
+      cache.setLazyOids(projectDir, rootOids)
       continue
     }
     const prefix = `${relPrefix}/`
     const start = lowerBound(prefix)
     const end = lowerBound(`${relPrefix}0`)
     const matches: string[] = []
-    const projOids = new Map<string, string>()
     const projAbs =
       relPrefix === spelled ? abs : (rel: string) => path.join(projectDir, rel.slice(prefix.length))
+    const projOids = (): Map<string, string> => {
+      const oids = new Map<string, string>()
+      for (let i = start; i < end; i++) {
+        const rel = sorted[i]!
+        const oid = trusted.get(rel)
+        if (oid !== undefined) oids.set(projAbs(rel), oid)
+      }
+      return oids
+    }
+    // One hash of the records joined: one per record was ~25 ms of 125,000.
+    let records: string | undefined = ''
     for (let i = start; i < end; i++) {
       const rel = sorted[i]!
-      matches.push(rel.slice(prefix.length))
+      const own = rel.slice(prefix.length)
+      matches.push(own)
+      if (records === undefined) continue
       const oid = trusted.get(rel)
-      if (oid !== undefined) projOids.set(projAbs(rel), oid)
+      if (oid === undefined || oid.startsWith('120000:') || undecodable?.has(rel) === true) {
+        records = undefined
+      } else records += `${own}\0${oid}\n`
     }
     // An empty slice is a directory git did not see, not an empty project:
     // a project has at least its package.json, tracked or untracked. A
@@ -1847,15 +2030,14 @@ export async function applyGitEnumeration(
     // trusted from here) — one spawn per such project per run, nothing for
     // a workspace without one.
     if (matches.length === 0) continue
-    // setOids AFTER set — set() drops the project's OID slot.
+    // setLazyOids AFTER set — set() drops the project's OID slot.
     cache.set(projectDir, matches)
-    cache.setOids(projectDir, projOids)
+    cache.setLazyOids(projectDir, projOids)
+    if (records !== undefined) cache.noteFilesDigest(projectDir, matches, xxh3hex(records))
   }
   if (workspaceWide) {
-    const rootOids = new Map<string, string>()
-    for (const [rel, oid] of trusted) rootOids.set(abs(rel), oid)
     cache.set(workspaceRoot, all)
-    cache.setOids(workspaceRoot, rootOids)
+    cache.setLazyOids(workspaceRoot, rootOids)
     cache.setWorkspaceRoot(workspaceRoot)
   }
 }

package/src/cache/index.ts

@@ -34,7 +34,8 @@ export {
   SCHEMA_VERSION,
   vxCacheEntries,
 } from './cache.js'
-export { ABSENT_INPUT, absentOr, CACHE_VERSION } from './key-fold.js'
+export { ABSENT_INPUT, absentOr, CACHE_VERSION, foldHead } from './key-fold.js'
+export type { FoldMemo } from './fold-memo.js'
 export { measureFsClockLag, setFsClockLag } from './fs-clock.js'
 export { filesBehind } from './linked-inputs.js'
 export {
@@ -46,7 +47,10 @@ export {
   declaresInput,
   namedOutright,
   workspaceFilesReachInto,
+  filesToken,
   resolveInputs,
+  type ResolveInputsArgs,
+  resolveOutputTree,
   resolveOutputs,
   resolveWorkspaceOutputs,
   sweepRestoreTemps,
@@ -75,7 +79,7 @@ export {
   type GitEnumeration,
   type BlobSizeMemo,
 } from './git-inputs.js'
-export { LayeredCache, type RemoteCacheLayer } from './layered-cache.js'
+export { LayeredCache, type RemoteCacheLayer, type RemoteOpenContext } from './layered-cache.js'
 export type { ResolvedRemote } from './remote-lane.js'
 export { ScopedCache } from './scoped-cache.js'
 export {