@vzn/vx 0.0.636 → 0.0.638

package/src/cache/git-inputs.ts

@@ -6,9 +6,21 @@
 // task declared. Nothing here reads a config or applies a boundary.
 
 import path from 'node:path'
-import { existsSync, lstatSync, readdirSync, readFileSync, realpathSync } from 'node:fs'
+import {
+  existsSync,
+  lstatSync,
+  readdirSync,
+  readFileSync,
+  realpathSync,
+  renameSync,
+  rmSync,
+  utimesSync,
+  writeFileSync,
+  type Stats,
+} from 'node:fs'
 import {
   executablePath,
+  gitEnv,
   gitSpawnRefusal,
   isInstalledPath,
   notAWorkTree,
@@ -60,6 +72,7 @@ export function repoFacts(dir: string): RepoFacts | null {
         '--git-path',
         'index',
       ],
+      env: gitEnv(),
       cwd: dir,
       stdout: 'pipe',
       stderr: 'ignore',
@@ -99,6 +112,7 @@ export function gitRefusal(dir: string): UserError | undefined {
   try {
     proc = Bun.spawnSync({
       cmd: [executablePath('git'), 'rev-parse', '--is-inside-work-tree'],
+      env: gitEnv(),
       cwd: dir,
       stdout: 'ignore',
       stderr: 'pipe',
@@ -218,6 +232,56 @@ export class GitFilesCache extends Map<string, readonly string[]> {
    * be opened from a string, so `resolveFiles` refuses one it would fold.
    */
   private undecodable = new Set<string>()
+  /**
+   * What git tracks, by absolute path, within `trackedScopes`: a file there
+   * no clean deletes (`committedIn`). Filled by the
+   * enumeration, and per directory on demand for one it did not cover.
+   */
+  private tracked = new Set<string>()
+  private trackedScopes: string[] = []
+  /**
+   * Under `core.ignorecase` (macOS's default volume) git's `Foo.txt` is the
+   * disk's `foo.txt` after a rename git never saw, so paths are compared
+   * case-folded. Decided at the first note: the run's `git var -l`, else
+   * the platform.
+   */
+  private foldCase: boolean | undefined
+
+  private trackedKey(abs: string): string {
+    this.foldCase ??=
+      this.vars === null
+        ? process.platform === 'darwin'
+        : this.vars.get('core.ignorecase')?.trim().toLowerCase() === 'true'
+    return this.foldCase ? abs.toLowerCase() : abs
+  }
+
+  noteTracked(scopes: readonly string[], absPaths: readonly string[]): void {
+    for (const p of absPaths) this.tracked.add(this.trackedKey(p))
+    this.trackedScopes.push(...scopes)
+  }
+
+  /**
+   * Whether git tracks a file under `dir`: committed (or `git add`ed)
+   * files are the user's, never cleaned (owner, 2026-10-14: "never
+   * delete committed file"). A directory no enumeration covered (a
+   * nested repository, a cache no enumeration filled) is listed once.
+   * git lists a submodule as its gitlink path alone, so a file under a
+   * tracked path is committed too.
+   */
+  committedIn(dir: string): (abs: string) => boolean {
+    let listed = false
+    return (abs) => {
+      if (!listed) {
+        listed = true
+        const covered = this.trackedScopes.some((s) => dir === s || dir.startsWith(s + path.sep))
+        if (!covered) this.noteTracked([dir], listTracked(dir))
+      }
+      for (let p = abs; p.length > dir.length; p = path.dirname(p))
+        if (this.tracked.has(this.trackedKey(p))) return true
+      return false
+    }
+  }
+
   /**
    * When the enumeration that vouched for `oids` started (ms since the
    * epoch): an index OID says what a file held when `git status` looked,
@@ -332,30 +396,6 @@ export class GitFilesCache extends Map<string, readonly string[]> {
     }
   }
 
-  /** Tracked files a task's clean removed this run, and whose clean (A-48). */
-  private trackedCleans: Array<{ path: string; by: string }> = []
-
-  /**
-   * Note which of the files a task's clean just removed git tracks. Called
-   * before `markOutputsChanged`, which drops their OIDs. A committed output
-   * another task reads with no edge to its producer is gone for as long as
-   * the producer runs (vueuse's `metadata/index.json`, N's dogfood), and the
-   * reader failed naming only the missing file (A-48).
-   */
-  noteClean(by: string, dir: string, rels: readonly string[]): void {
-    const oids = this.oids.get(dir)
-    if (oids === undefined) return
-    for (const rel of rels) {
-      const abs = path.resolve(dir, rel)
-      if (oids.has(abs)) this.trackedCleans.push({ path: abs, by })
-    }
-  }
-
-  /** The tracked files other tasks' cleans removed that are still missing. */
-  trackedCleansMissing(except: string): Array<{ path: string; by: string }> {
-    return this.trackedCleans.filter((c) => c.by !== except && !existsSync(c.path))
-  }
-
   /** Trusted index OIDs for a project (abs path → oid), if any survive. */
   oidsFor(projectDir: string): ReadonlyMap<string, string> | undefined {
     return this.oids.get(projectDir)
@@ -476,6 +516,7 @@ export function runGitLsFiles(cwd: string): GitLsResult {
   try {
     proc = Bun.spawnSync({
       cmd: [executablePath('git'), 'ls-files', '-s', '--others', '--exclude-standard', '-z', '.'],
+      env: gitEnv(),
       cwd,
       stdout: 'pipe',
       stderr: 'pipe',
@@ -496,6 +537,32 @@ export function runGitLsFiles(cwd: string): GitLsResult {
   return parsed
 }
 
+/**
+ * The files git tracks under `dir`, absolute: one `ls-files` of the index,
+ * no worktree walk. Outside a work tree (an embedder's temp dir) nothing
+ * is tracked.
+ */
+function listTracked(dir: string): string[] {
+  let proc
+  try {
+    proc = Bun.spawnSync({
+      cmd: [executablePath('git'), 'ls-files', '-z', '--', '.'],
+      env: gitEnv(),
+      cwd: dir,
+      stdout: 'pipe',
+      stderr: 'pipe',
+    })
+  } catch {
+    return []
+  }
+  if (proc.exitCode !== 0) return []
+  const { text } = decodeGitZ(proc.stdout)
+  return text
+    .split('\0')
+    .filter((rel) => rel.length > 0)
+    .map((rel) => path.join(dir, rel))
+}
+
 /**
  * Replace each nested repository in a listing — a gitlink (a submodule) or
  * an untracked `dir/` (an embedded repository) — with the files its OWN git
@@ -853,7 +920,10 @@ async function dropFilteredOids(
   // index blob keyed it through an edit (item 977).
   if (!attributesPossible) {
     for (const rel of args.listed) {
-      if (rel === '.gitattributes' || rel.endsWith('/.gitattributes')) {
+      if (
+        (rel === '.gitattributes' || rel.endsWith('/.gitattributes')) &&
+        attributesCanConvert(path.join(args.workspaceRoot, rel))
+      ) {
         attributesPossible = true
         break
       }
@@ -915,11 +985,40 @@ function attributesInIndex(args: {
     return true
   }
   for (const rel of args.tracked) {
-    if (rel === '.gitattributes' || rel.endsWith('/.gitattributes')) return true
+    if (
+      (rel === '.gitattributes' || rel.endsWith('/.gitattributes')) &&
+      attributesCanConvert(path.join(args.workspaceRoot, rel))
+    )
+      return true
   }
   return attributesAbove(args)
 }
 
+/**
+ * Whether a `.gitattributes` can set an attribute that converts bytes. One
+ * that names none of them, nor `binary` (git's one built-in macro) nor a
+ * macro definition, cannot: `.vx/.gitattributes`' `linguist-generated`
+ * line asked `check-attr` on every run (21 ms) and changed nothing. A macro
+ * defined outside the tree already counts as an attributes source. A file
+ * that does not read counts as one that converts. Read once per enumeration.
+ */
+function attributesCanConvert(file: string): boolean {
+  let can = attributeReads.get(file)
+  if (can === undefined) {
+    try {
+      can = CONVERTING_ATTRIBUTE.test(readFileSync(file, 'utf8'))
+    } catch {
+      can = true
+    }
+    attributeReads.set(file, can)
+  }
+  return can
+}
+
+const CONVERTING_ATTRIBUTE =
+  /\[attr\]|\b(?:text|eol|ident|filter|working-tree-encoding|crlf|binary)\b/
+const attributeReads = new Map<string, boolean>()
+
 /**
  * The repository root, derived from the workspace root and git's own
  * `--show-prefix` (the repo→workspace path). Exact and free: the prefix
@@ -950,7 +1049,8 @@ function attributesAbove(args: {
     for (;;) {
       if (!seen.has(dir)) {
         seen.add(dir)
-        if (existsSync(path.join(dir, '.gitattributes'))) return true
+        const file = path.join(dir, '.gitattributes')
+        if (existsSync(file) && attributesCanConvert(file)) return true
       }
       const parent = path.dirname(dir)
       // Stop at the repo root, and never walk past the filesystem root on a
@@ -1180,6 +1280,10 @@ export async function populateGitFilesCache(
 export interface GitEnumeration {
   /** Every enumerated path, workspace-relative, tracked and untracked. */
   all: string[]
+  /** The paths the index lists (a submodule as its gitlink), workspace-relative: what git tracks. */
+  tracked: readonly string[]
+  /** The directories `tracked` covers, workspace-relative (`.` for the whole tree). */
+  scopes: readonly string[]
   /** Workspace-relative path → index OID, for files whose worktree state matched the index. */
   trusted: Map<string, string>
   /** Whether the worktree had uncommitted changes; null when `git status` failed. */
@@ -1230,6 +1334,91 @@ export function gitPathspecs(
   return scoped ? rels : ['.']
 }
 
+/**
+ * Whether the index holds a racily clean entry: a file stamped no earlier
+ * than the second the index was written. git cannot trust its stat and
+ * re-hashes the file on every `status` until something rewrites the index,
+ * and a fresh checkout is all such entries. A layout this does not read
+ * answers false.
+ */
+function hasRacyEntry(index: Uint8Array, mtimeSec: number, oidBytes: number): boolean {
+  const view = new DataView(index.buffer, index.byteOffset, index.byteLength)
+  if (index.length < 12 || view.getUint32(0) !== 0x44495243) return false
+  const version = view.getUint32(4)
+  if (version < 2 || version > 4) return false
+  const count = view.getUint32(8)
+  let at = 12
+  for (let i = 0; i < count; i++) {
+    const flagsAt = at + 40 + oidBytes
+    if (flagsAt + 2 > index.length) return false
+    if (view.getUint32(at + 8) >= mtimeSec) return true
+    let pathAt = flagsAt + 2
+    if (version >= 3 && (view.getUint16(flagsAt) & 0x4000) !== 0) pathAt += 2
+    // v4 opens the path with a varint (bytes of the previous path to drop).
+    if (version === 4) while ((index[pathAt++] ?? 0) & 0x80);
+    const end = index.indexOf(0, pathAt)
+    if (end < 0) return false
+    at = version === 4 ? end + 1 : at + ((end - at + 8) & ~7)
+  }
+  return false
+}
+
+/** A private copy a run older than this may still have handed to `status`. */
+const STALE_INDEX_MS = 60_000
+
+/**
+ * The index `status` reads when the real one holds a racily clean entry: a
+ * copy refreshed once and kept beside it, named by the real index's bytes.
+ * `status` reads the real index without writing it (item 880), so in a
+ * checkout nothing else had refreshed, every run re-hashed every racy file:
+ * 347 against 8 ms on 1,800 files. The copy keeps the real mtime until git
+ * refreshes it, so git's own racy rule holds for it; only stats move, the
+ * entries are the real index's. Undefined: read the real index.
+ */
+async function refreshedIndex(
+  indexFile: string,
+  read: { bytes: Uint8Array; hash: string; st: Stats },
+  objectFormat: 'sha1' | 'sha256',
+  spawnGit: (
+    args: string[],
+    stdin?: string,
+    env?: Record<string, string>,
+  ) => Promise<GitRun | null>,
+): Promise<string | undefined> {
+  if (GIT_LOCATION_ENV.some((name) => process.env[name] !== undefined)) return undefined
+  const { bytes, hash, st } = read
+  if (!hasRacyEntry(bytes, Math.floor(st.mtimeMs / 1000), objectFormat === 'sha256' ? 32 : 20)) {
+    return undefined
+  }
+  const dir = path.dirname(indexFile)
+  const name = `vx-index-${hash}`
+  const copy = path.join(dir, name)
+  if (existsSync(copy)) return copy
+  const tmp = `${copy}.${process.pid}`
+  try {
+    writeFileSync(tmp, bytes)
+    utimesSync(tmp, st.atime, st.mtime)
+    const run = await spawnGit(['update-index', '-q', '--refresh'], undefined, {
+      GIT_INDEX_FILE: tmp,
+    })
+    if (run === null || run.exitCode !== 0) throw new Error('refresh failed')
+    renameSync(tmp, copy)
+  } catch {
+    rmSync(tmp, { force: true })
+    return undefined
+  }
+  const stale = Date.now() - STALE_INDEX_MS
+  for (const other of readdirSync(dir)) {
+    if (!other.startsWith('vx-index-') || other === name) continue
+    try {
+      if (lstatSync(path.join(dir, other)).mtimeMs < stale) rmSync(path.join(dir, other))
+    } catch {
+      // another run swept it
+    }
+  }
+  return copy
+}
+
 /**
  * The spawn half of `populateGitFilesCache`, separated so an UNSCOPED run
  * can start it before the configs are evaluated — the enumeration needs
@@ -1241,6 +1430,7 @@ export async function startGitEnumeration(
   workspaceRoot: string,
   pathspecs: readonly string[],
 ): Promise<GitEnumeration> {
+  attributeReads.clear()
   // The spawns are independent — run them concurrently so the
   // bulk-populate costs max(ls-files, status) wall time, not the sum
   // (status alone is ~74 ms on a 1000-project tree; serial spawning
@@ -1261,7 +1451,7 @@ export async function startGitEnumeration(
         stdin: stdin === undefined ? 'ignore' : new TextEncoder().encode(stdin),
         stdout: 'pipe',
         stderr: 'pipe',
-        ...(env !== undefined ? { env: { ...process.env, ...env } } : {}),
+        env: env !== undefined ? { ...gitEnv(), ...env } : gitEnv(),
       })
       const [stdout, stderr, exitCode] = await Promise.all([
         new Response(proc.stdout).bytes(),
@@ -1284,33 +1474,15 @@ export async function startGitEnumeration(
   // `ls-files --others` for the untracked set walked the same tree a second
   // time (~50 ms of CPU, concurrent with status but contending with it).
   const listing = spawnGit(['ls-files', '-s', '-v', '-z', '--', ...pathspecs])
-  const running = Promise.all([
-    listing,
-    // `--ignored=matching` names an ignored path without descending into an
-    // ignored directory: git applies an ignored `.gitattributes` as it does
-    // any other, and the filter gate below must see it (A-19).
-    // `--no-renames`: a deletion paired as an unmerged path's rename source
-    // prints only as `UU <path>`, and the deleted file stayed trusted (A-59).
-    spawnGit([
-      'status',
-      '--porcelain',
-      '-z',
-      '-uall',
-      '--ignored=matching',
-      '--no-renames',
-      '--',
-      ...pathspecs,
-    ]),
-    // The gate for whether a clean filter can rewrite bytes between the
-    // index and the worktree: git's merged config (`core.autocrlf`) and,
-    // from git 2.42, the attributes files it reads outside the tree
-    // (`attributeFilesOutsideTree`). No tree scan. It replaced a
-    // `config --get-regexp` of three keys at the same cost (1.29 against
-    // 1.32 ms, min of 30), which could not name the default global file.
-    // Not foldable into the rev-parse below: rev-parse prints no config.
-    spawnGit(['var', '-l']),
-  ])
-  // Asked while the three above run, from the memo the file hasher reads
+  // The gate for whether a clean filter can rewrite bytes between the
+  // index and the worktree: git's merged config (`core.autocrlf`) and,
+  // from git 2.42, the attributes files it reads outside the tree
+  // (`attributeFilesOutsideTree`). No tree scan. It replaced a
+  // `config --get-regexp` of three keys at the same cost (1.29 against
+  // 1.32 ms, min of 30), which could not name the default global file.
+  // Not foldable into the rev-parse below: rev-parse prints no config.
+  const varsRun = spawnGit(['var', '-l'])
+  // Asked while the two above run, from the memo the file hasher reads
   // too (`repoFacts`). `prefix` is the repo→workspace path (empty when the
   // workspace root IS the git root): `ls-files` prints cwd(workspace)-
   // relative paths but `status` prints repo-root-relative ones, so when the
@@ -1336,23 +1508,59 @@ export async function startGitEnumeration(
     facts === null || facts.indexFile === ''
       ? undefined
       : path.resolve(workspaceRoot, facts.indexFile)
-  const readIndexKey = async (): Promise<
-    { key: string; stamp: string; ctimeMs: number } | undefined
+  const readIndex = async (): Promise<
+    { bytes: Uint8Array; hash: string; st: Stats; key: string; stamp: string } | undefined
   > => {
     if (indexFile === undefined) return undefined
     try {
+      // Stat before the read: a write between them leaves an mtime older
+      // than the bytes, which only makes more entries racy.
       const st = lstatSync(indexFile)
       const bytes = await Bun.file(indexFile).bytes()
+      const hash = xxh3hex(bytes)
       return {
-        key: `${xxh3hex(bytes)}:${pathspecKey}`,
+        bytes,
+        hash,
+        st,
+        key: `${hash}:${pathspecKey}`,
         stamp: `${st.ino}:${st.ctimeMs}:${st.size}`,
-        ctimeMs: st.ctimeMs,
       }
     } catch {
       return undefined
     }
   }
-  const indexKeyRead = readIndexKey()
+  const indexKeyRead = readIndex()
+  const statusIndex = indexKeyRead.then((read) =>
+    read === undefined || indexFile === undefined
+      ? undefined
+      : refreshedIndex(indexFile, read, facts!.objectFormat, spawnGit),
+  )
+  const running = Promise.all([
+    listing,
+    // `--ignored=matching` names an ignored path without descending into an
+    // ignored directory: git applies an ignored `.gitattributes` as it does
+    // any other, and the filter gate below must see it (A-19).
+    // `--no-renames`: a deletion paired as an unmerged path's rename source
+    // prints only as `UU <path>`, and the deleted file stayed trusted (A-59).
+    statusIndex.then((index) => {
+      const env = index === undefined ? undefined : { GIT_INDEX_FILE: index }
+      return spawnGit(
+        [
+          'status',
+          '--porcelain',
+          '-z',
+          '-uall',
+          '--ignored=matching',
+          '--no-renames',
+          '--',
+          ...pathspecs,
+        ],
+        undefined,
+        env,
+      )
+    }),
+    varsRun,
+  ])
   const gitPrefix = facts?.prefix ?? ''
   const gitDir = facts?.commonDir ?? ''
   const ls = await listing
@@ -1376,7 +1584,7 @@ export async function startGitEnumeration(
   let indexKey: string | undefined
   if (
     read !== undefined &&
-    read.ctimeMs < startedAtMs - racyWindowMs(read.ctimeMs, FILE_HASH_RACY_MS)
+    read.st.ctimeMs < startedAtMs - racyWindowMs(read.st.ctimeMs, FILE_HASH_RACY_MS)
   ) {
     try {
       const st = lstatSync(indexFile!)
@@ -1470,6 +1678,8 @@ export async function startGitEnumeration(
   }
   return {
     all,
+    tracked: gitlinks.size === 0 ? tracked : tracked.concat([...gitlinks]),
+    scopes: pathspecs,
     trusted,
     dirty: worktreeDirty,
     vars: vars !== null && vars.exitCode === 0 ? vars.stdout : null,
@@ -1493,7 +1703,7 @@ export async function startGitEnumeration(
         }
         return blobs
       },
-      rekey: async () => (await readIndexKey())?.key,
+      rekey: async () => (await readIndex())?.key,
     },
     // A missing blob is an answer ("missing"), never a fetch from a partial
     // clone's promisor remote.
@@ -1544,6 +1754,10 @@ export async function applyGitEnumeration(
   cache.setGitVars(enumeration.vars)
   cache.enumeratedAtMs = enumeration.startedAtMs
   cache.markUndecodable(enumeration.undecodable.map((rel) => path.join(workspaceRoot, rel)))
+  cache.noteTracked(
+    enumeration.scopes.map((rel) => path.join(workspaceRoot, rel)),
+    enumeration.tracked.map((rel) => path.join(workspaceRoot, rel)),
+  )
   // Sort once, then each project's files are a contiguous range found
   // by binary search on its `dir/` prefix — O((F+P) log F) instead of
   // the O(P·F) per-project startsWith scan (54 ms at 1090 projects ×

package/src/cache/index.ts

@@ -5,19 +5,21 @@
 // re-exported through cache.ts.
 
 export {
+  applyCacheFlags,
   ArtifactVanishedError,
   Cache,
+  type CacheFlags,
   type CacheEntry,
   type CacheKeyInput,
   type CacheLayer,
   type CachePolicy,
   cachesNothing,
   EXECUTED_RUNS_SQL,
+  formatCachePolicy,
   FULL_CACHE_POLICY,
   type InvocationRecord,
   KEYED_RUNS_SQL,
   parseCachePolicy,
-  scopeCachePolicy,
   type RunRecord,
   type TaskInputRow,
   WORKSPACE_OUTPUT_PREFIX,
@@ -28,12 +30,14 @@ export {
   racyWindowMs,
   type SchemaReset,
   SCHEMA_VERSION,
+  vxCacheEntries,
 } from './cache.js'
 export { ABSENT_INPUT, absentOr, CACHE_VERSION } from './key-fold.js'
 export {
   addedInput,
   type InputListing,
   cleanOutputs,
+  committedFiles,
   cleanWorkspaceOutputs,
   declaresInput,
   workspaceFilesReachInto,
@@ -65,4 +69,14 @@ export {
   type BlobSizeMemo,
 } from './git-inputs.js'
 export { LayeredCache, type RemoteCacheLayer } from './layered-cache.js'
+export { ScopedCache } from './scoped-cache.js'
+export {
+  absentPluginCache,
+  pluginCache,
+  PLUGIN_ENTRY_MAX_BYTES,
+  type PluginCache,
+  type PluginCacheRemote,
+} from './plugin-cache.js'
+export { packRunOutputs, putRunEntry, runEntryName, writesRunStore } from './run-store.js'
 export { ChainedCache } from './chained-cache.js'
+export { branchSource } from './run-history.js'