@vzn/vx 0.0.21 → 0.0.22

package/src/orchestrator/local-shortcircuit.ts

@@ -32,16 +32,19 @@
 //   - Only STABLE-key, cacheable, local-read tasks are classified. A
 //     stable miss stays in the normal (dep-gated) schedule; an unstable
 //     task is never probed here.
-//   - If ANY task in the graph declares `cache.outputs.workspaceFiles`,
-//     no task is restore-tiered (the boundary-ignoring escape hatch
-//     could let a task write where a restore touches — a blanket
-//     conservative exclusion). Those tasks still get an exec-tier probe
-//     reuse entry so there's no double work.
+//   - A task that declares `cache.outputs.workspaceFiles` (the
+//     boundary-ignoring escape hatch) can write where a restore touches,
+//     so every task whose project directory a workspace-output glob's
+//     static prefix REACHES stays out of the restore tier, and so does
+//     every transitive dependant of one (its up-front key folds a key
+//     that is preliminary). A glob with no literal prefix reaches every
+//     project, which is the old graph-wide rule. Excluded tasks still get
+//     an exec-tier probe reuse entry so there's no double work.
 
-import { span } from '../util/index.js'
+import { normalizeGlob, relPosix, span, staticPrefix } from '../util/index.js'
 import type { CacheEntry, CacheLayer, GitFilesCache } from '../cache/index.js'
 import type { TaskNode } from '../graph/index.js'
-import { deriveStableKeys } from './stable-keys.js'
+import { deriveStableKeys, workspaceInputsReach } from './stable-keys.js'
 import type { HashCache } from './task-hash.js'
 
 export interface ShortCircuitArgs {
@@ -104,22 +107,13 @@ export async function startLocalShortCircuit(args: ShortCircuitArgs): Promise<Sh
   } finally {
     endKeys()
   }
-  const candidates = stableKeys.filter(({ node }) => node.config.cache !== undefined)
+  // Every stable key is a cacheable task's: deriveStableKeys pushes only
+  // `cacheEnabled && !unstable` (item 640 deleted a second filter here and
+  // nothing reddened).
+  const candidates = stableKeys
   if (candidates.length === 0) return EMPTY
 
-  // Belt-and-suspenders for the boundary-ignoring escape hatch: a
-  // root-anchored output can land anywhere, so if ANY task declares
-  // workspace outputs a task could in principle write where a
-  // restore-tier task restores. Disable the RESTORE tier graph-wide —
-  // workspace outputs are rare bad-practice. (Probe reuse still applies,
-  // so there's no double work; those tasks just stay dep-gated.)
-  let anyWorkspaceOutputs = false
-  for (const node of args.nodes.values()) {
-    if ((node.config.cache?.outputs.workspaceFiles?.length ?? 0) > 0) {
-      anyWorkspaceOutputs = true
-      break
-    }
-  }
+  const keptOut = restoreTierExclusions(args.nodes, args.workspaceRoot)
 
   const preProbed = new Map<string, ProbedEntry>()
   const restoreTier = new Set<string>()
@@ -139,7 +133,7 @@ export async function startLocalShortCircuit(args: ShortCircuitArgs): Promise<Sh
       for (const { hash, node } of candidates) {
         const hit = hits.get(hash) ?? null
         preProbed.set(node.id, { hash, hit })
-        if (hit !== null && !anyWorkspaceOutputs) restoreTier.add(node.id)
+        if (hit !== null && !keptOut.has(node.id)) restoreTier.add(node.id)
       }
       return { preProbed, restoreTier }
     } catch {
@@ -156,7 +150,7 @@ export async function startLocalShortCircuit(args: ShortCircuitArgs): Promise<Sh
         const command = node.config.exec?.command ?? ''
         const hit = await args.cache.get(hash, { taskId: node.id, command })
         preProbed.set(node.id, { hash, hit })
-        if (hit !== null && !anyWorkspaceOutputs) restoreTier.add(node.id)
+        if (hit !== null && !keptOut.has(node.id)) restoreTier.add(node.id)
       } catch {
         // Leave this task out of preProbed → it probes lazily in
         // execute(), exactly as today.
@@ -167,3 +161,69 @@ export async function startLocalShortCircuit(args: ShortCircuitArgs): Promise<Sh
 
   return { preProbed, restoreTier }
 }
+
+/**
+ * The tasks a workspace-output declaration keeps out of the restore tier.
+ *
+ * A root-anchored output can land in any project's directory, edge or no
+ * edge (`tests/local-shortcircuit.test.ts` § "a workspace-output writer"
+ * rows, item 425), so the question is not who depends on the writer but
+ * WHERE it can write: each declared glob's static prefix is tested for
+ * reach against every project directory, and against every task's own
+ * `workspaceFiles` inputs, with the ancestor-or-equal relation
+ * `workspaceInputsReach` already uses. A prefix that is the root reaches
+ * everything — the graph-wide rule this replaced (item 584). Exclusion then
+ * follows the edges DOWN: a dependant's up-front key folds an excluded
+ * task's key, which is preliminary, so it cannot restore early either.
+ * Cost: one pass over the nodes, no filesystem.
+ */
+function restoreTierExclusions(nodes: Map<string, TaskNode>, workspaceRoot: string): Set<string> {
+  const prefixes: string[] = []
+  let everything = false
+  for (const node of nodes.values()) {
+    for (const raw of node.config.cache?.outputs.workspaceFiles ?? []) {
+      // No negation to skip: the schema refuses '!' in output globs
+      // (`validateWorkspaceGlobs`), so the `continue` that stood here
+      // guarded a shape no config can carry (item 640).
+      const glob = normalizeGlob(raw)
+      const prefix = staticPrefix(glob)
+      if (prefix === '.' || prefix === '' || prefix === '/') everything = true
+      else prefixes.push(prefix.replace(/^\.\//, ''))
+    }
+  }
+  const out = new Set<string>()
+  if (everything) {
+    for (const id of nodes.keys()) out.add(id)
+    return out
+  }
+  if (prefixes.length === 0) return out
+  const reaches = (dir: string): boolean =>
+    dir === '' ||
+    dir === '.' ||
+    prefixes.some((p) => p === dir || p.startsWith(`${dir}/`) || dir.startsWith(`${p}/`))
+  const dependants = new Map<string, string[]>()
+  for (const node of nodes.values()) {
+    const dir = relPosix(workspaceRoot, node.projectDir)
+    const wsInputs = node.config.cache?.inputs?.workspaceFiles ?? []
+    if (reaches(dir) || (wsInputs.length > 0 && workspaceInputsReach(wsInputs, prefixes))) {
+      out.add(node.id)
+    }
+    for (const dep of node.deps) {
+      const list = dependants.get(dep)
+      if (list === undefined) dependants.set(dep, [node.id])
+      else list.push(node.id)
+    }
+  }
+  // Up the dependant edges on an explicit stack: a chain is as deep as the
+  // graph, and a recursion per edge threw `RangeError` at the 50,000 the
+  // builder takes (item 737).
+  const stack = [...out]
+  while (stack.length > 0) {
+    for (const up of dependants.get(stack.pop()!) ?? []) {
+      if (out.has(up)) continue
+      out.add(up)
+      stack.push(up)
+    }
+  }
+  return out
+}

package/src/orchestrator/lockfile-claim.ts

@@ -15,7 +15,7 @@
 
 import path from 'node:path'
 import { rename, stat } from 'node:fs/promises'
-import { xxh3hex } from '../util/index.js'
+import { xxh3, xxh3hex } from '../util/index.js'
 import type { TaskNode } from '../graph/index.js'
 import type {
   FingerprintChange,
@@ -33,7 +33,15 @@ export interface LockfileClaimOptions {
    * distinct content; must be deterministic and throw on a file it
    * cannot read (the plugin's name heads the error).
    */
-  readonly digest: (text: string) => ReadonlyMap<string, string>
+  readonly digest: (text: string, files: ReadonlyMap<string, string>) => ReadonlyMap<string, string>
+  /**
+   * Root-relative files the lockfile names whose BYTES the install uses but
+   * the lockfile does not pin (bun.lock's `patchedDependencies` names a patch
+   * file and records no hash of it). Their content hashes ('' when absent)
+   * reach `digest` as `files` and join the memo's identity, so an edited
+   * patch moves the key (item 1014). Called only when the memo misses.
+   */
+  readonly extraFiles?: (text: string) => readonly string[]
   /** Folded into the memo's identity: bump when `digest` folds differently. */
   readonly version: number
   /**
@@ -64,9 +72,35 @@ interface Digests {
 interface Memo {
   version: number
   lock: string
+  /** `extraFiles` → content hash when the digests were computed. */
+  extras?: Record<string, string>
   importers: Record<string, string>
 }
 
+/** A root-relative extra file's content hash, '' when absent; a path leaving the root is never read. */
+async function hashExtra(workspaceRoot: string, rel: string): Promise<string> {
+  if (path.isAbsolute(rel) || rel.split(/[\\/]/).includes('..')) return 'outside-root'
+  try {
+    return xxh3hex(await Bun.file(path.join(workspaceRoot, rel)).bytes())
+  } catch {
+    return ''
+  }
+}
+
+async function statsOf(
+  workspaceRoot: string,
+  rels: readonly string[],
+): Promise<Array<{ size: number; mtimeMs: number }>> {
+  return Promise.all(
+    rels.map((rel) =>
+      stat(path.join(workspaceRoot, rel)).then(
+        (st) => ({ size: st.size, mtimeMs: st.mtimeMs }),
+        () => ({ size: -1, mtimeMs: -1 }),
+      ),
+    ),
+  )
+}
+
 export function lockfileClaim(options: LockfileClaimOptions): LockfileClaimHooks {
   const { file, digest, version } = options
   const scope = options.scope ?? 'project'
@@ -76,7 +110,16 @@ export function lockfileClaim(options: LockfileClaimOptions): LockfileClaimHooks
   }
   // Per workspace root: what the last read saw. `size` + `mtimeMs` gate a
   // re-read; the content hash decides whether the digests are current.
-  const seen = new Map<string, { size: number; mtimeMs: number; digests: Digests }>()
+  const seen = new Map<
+    string,
+    {
+      size: number
+      mtimeMs: number
+      extras: readonly string[]
+      extraStats: ReadonlyArray<{ size: number; mtimeMs: number }>
+      digests: Digests
+    }
+  >()
   // Per run: every `key` call of one run gets the same context object.
   const perRun = new WeakMap<object, Promise<Digests>>()
 
@@ -87,27 +130,74 @@ export function lockfileClaim(options: LockfileClaimOptions): LockfileClaimHooks
       st = await stat(full)
     } catch {
       const digests: Digests = { lock: '', importers: new Map() }
-      seen.set(workspaceRoot, { size: -1, mtimeMs: -1, digests })
+      seen.set(workspaceRoot, { size: -1, mtimeMs: -1, extras: [], extraStats: [], digests })
       return digests
     }
     const last = seen.get(workspaceRoot)
     if (last !== undefined && last.size === st.size && last.mtimeMs === st.mtimeMs) {
-      return last.digests
+      const now = await statsOf(workspaceRoot, last.extras)
+      if (
+        now.every(
+          (s, i) =>
+            s.size === last.extraStats[i]!.size && s.mtimeMs === last.extraStats[i]!.mtimeMs,
+        )
+      ) {
+        return last.digests
+      }
     }
     const bytes = await Bun.file(full).bytes()
-    const lock = xxh3hex(bytes)
-    if (last !== undefined && last.digests.lock === lock) {
-      seen.set(workspaceRoot, { size: st.size, mtimeMs: st.mtimeMs, digests: last.digests })
-      return last.digests
-    }
+    const lockHash = xxh3hex(bytes)
     const memoFile = path.join(cacheDir, 'lockfile-claims', `${file}.json`)
-    const importers =
-      scope === 'workspace'
-        ? new Map<string, string>()
-        : ((await readMemo(memoFile, version, lock)) ??
-          (await computeAndMemo(memoFile, version, lock, digest(decode(bytes)))))
+    // The extra files the last digest read, and their hashes now: a warm run
+    // re-hashes the files the memo names and never parses to learn them.
+    const memo = await readMemo(memoFile, version, lockHash)
+    let extras: Record<string, string> = {}
+    let importers: ReadonlyMap<string, string> | undefined
+    if (memo !== undefined) {
+      const now = Object.fromEntries(
+        await Promise.all(
+          Object.keys(memo.extras).map(
+            async (rel) => [rel, await hashExtra(workspaceRoot, rel)] as const,
+          ),
+        ),
+      )
+      if (Object.entries(memo.extras).every(([rel, h]) => now[rel] === h)) {
+        extras = memo.extras
+        importers = memo.importers
+      }
+    }
+    if (importers === undefined) {
+      const text = decode(bytes)
+      const rels = [...new Set(options.extraFiles?.(text) ?? [])].sort()
+      extras = Object.fromEntries(
+        await Promise.all(
+          rels.map(async (rel) => [rel, await hashExtra(workspaceRoot, rel)] as const),
+        ),
+      )
+      importers =
+        scope === 'workspace'
+          ? new Map<string, string>()
+          : await computeAndMemo(
+              memoFile,
+              version,
+              lockHash,
+              extras,
+              digest(text, new Map(Object.entries(extras))),
+            )
+    }
+    const rels = Object.keys(extras)
+    const lock =
+      rels.length === 0
+        ? lockHash
+        : xxh3hex([lockHash, ...rels.map((r) => `${r}\0${extras[r]}`)].join('\0'))
     const digests: Digests = { lock, importers }
-    seen.set(workspaceRoot, { size: st.size, mtimeMs: st.mtimeMs, digests })
+    seen.set(workspaceRoot, {
+      size: st.size,
+      mtimeMs: st.mtimeMs,
+      extras: rels,
+      extraStats: await statsOf(workspaceRoot, rels),
+      digests,
+    })
     return digests
   }
   const loadOnce = (ctx: KeyHookContext): Promise<Digests> => {
@@ -127,8 +217,19 @@ export function lockfileClaim(options: LockfileClaimOptions): LockfileClaimHooks
         // installed tree is in question, and core's answer (all) is right.
         if (change.before === null || change.after === null) return undefined
         if (scope === 'workspace') return undefined
-        const before = digest(decode(change.before))
-        const after = digest(decode(change.after))
+        // WHICH side failed is the whole difference between "your
+        // lockfile is broken" and "the base commit's is" — a migration
+        // commit hits the second, and the bare parser message named
+        // neither (2026-09-20).
+        const sideOf = (bytes: Uint8Array, side: string): ReadonlyMap<string, string> => {
+          try {
+            return digest(decode(bytes), new Map())
+          } catch (err) {
+            throw new Error(`${err instanceof Error ? err.message : String(err)} (${side})`)
+          }
+        }
+        const before = sideOf(change.before, `as of the base ref`)
+        const after = sideOf(change.after, `in the working tree`)
         const names: string[] = []
         for (const p of ctx.projects) {
           const importer = importerOf(ctx.workspaceRoot, p.dir)
@@ -149,13 +250,22 @@ export function lockfileClaim(options: LockfileClaimOptions): LockfileClaimHooks
 }
 
 /**
- * A project's digest: its own importer's, or — for a project the lockfile
- * has no importer for (outside the workspace's `packages`) — the root
- * importer's, the only installed tree it can resolve from. A file with
- * neither folds a constant.
+ * A project's digest: its own importer's AND the root importer's. The
+ * root package's dependencies reach every task — their bins through the
+ * root `node_modules/.bin` core puts on every PATH, their modules through
+ * Node's resolution walking up to the root `node_modules` (`@types/*`,
+ * a config's plugins) — so a root devDependency bump that moved only the
+ * root's digest replayed the old tool's output (nx#36415 class,
+ * 2026-09-24). A project the lockfile has no importer for (outside the
+ * workspace's `packages`) folds the root's alone; a file with neither
+ * folds a constant.
  */
 function digestFor(importers: ReadonlyMap<string, string>, importer: string): string {
-  return importers.get(importer) ?? importers.get('.') ?? 'no-importer'
+  const own = importers.get(importer)
+  const root = importers.get('.')
+  if (own === undefined) return root ?? 'no-importer'
+  if (root === undefined || importer === '.') return own
+  return xxh3hex(`${own}\0${root}`)
 }
 
 /** The lockfile's importer path for a project directory: `.` for the root, POSIX otherwise. */
@@ -168,11 +278,11 @@ async function readMemo(
   memoFile: string,
   version: number,
   lock: string,
-): Promise<Map<string, string> | undefined> {
+): Promise<{ extras: Record<string, string>; importers: Map<string, string> } | undefined> {
   try {
     const memo = (await Bun.file(memoFile).json()) as Memo
     if (memo.version !== version || memo.lock !== lock) return undefined
-    return new Map(Object.entries(memo.importers))
+    return { extras: memo.extras ?? {}, importers: new Map(Object.entries(memo.importers)) }
   } catch {
     return undefined
   }
@@ -182,9 +292,10 @@ async function computeAndMemo(
   memoFile: string,
   version: number,
   lock: string,
+  extras: Record<string, string>,
   importers: ReadonlyMap<string, string>,
 ): Promise<ReadonlyMap<string, string>> {
-  const memo: Memo = { version, lock, importers: Object.fromEntries(importers) }
+  const memo: Memo = { version, lock, extras, importers: Object.fromEntries(importers) }
   // Write-then-rename: a reader never sees a half-written memo, and two
   // concurrent runs each land a whole one.
   const tmp = `${memoFile}.tmp-${process.pid}-${Date.now()}`
@@ -216,8 +327,8 @@ export interface ReachGraph {
  * does not. Lockfiles carry dependency cycles, so the unit is the
  * strongly connected component: Tarjan's walk (iterative — a dependency
  * chain can be thousands deep) emits components children-first, and each
- * folds its members' material (sorted) and its child components' digests
- * (sorted). O(nodes + edges): 1000 importers over 3000 packages digest in
+ * folds its members (sorted), each as its material and the materials its
+ * edges land on, and its child components' digests (sorted). O(nodes + edges): 1000 importers over 3000 packages digest in
  * ~20 ms where one traversal per importer took 400.
  */
 export function reachDigests(g: ReachGraph): string[] {
@@ -270,10 +381,19 @@ export function reachDigests(g: ReachGraph): string[] {
       for (const m of members) {
         for (const w of g.edges[m]!) if (comp[w] !== id) children.add(compHash[comp[w]!]!)
       }
-      let h = Bun.hash.xxHash3(`members:${members.length}`)
-      for (const m of members.map((m) => g.material[m]!).sort()) h = Bun.hash.xxHash3(`${m}\n`, h)
-      h = Bun.hash.xxHash3(`children:${children.size}`, h)
-      for (const c of [...children].sort()) h = Bun.hash.xxHash3(`${c}\n`, h)
+      // A member is its material AND where its edges land: all members
+      // share one digest, so retargeting an edge between two of them (an
+      // importer's `y@1.0.0` → `y@1.1.0` inside one cycle) moved nothing
+      // and replayed a stale output (item 1013). A self-loop reaches
+      // nothing new and is left out.
+      const lines = members.map((m) => {
+        const targets = [...new Set(g.edges[m]!.filter((w) => w !== m).map((w) => g.material[w]!))]
+        return `${g.material[m]!}\0${targets.sort().join('\0')}`
+      })
+      let h = xxh3(`members:${members.length}`)
+      for (const line of lines.sort()) h = xxh3(`${line}\n`, h)
+      h = xxh3(`children:${children.size}`, h)
+      for (const c of [...children].sort()) h = xxh3(`${c}\n`, h)
       compHash.push(h.toString(16).padStart(16, '0'))
     }
   }