@vzn/vx-migrate 0.0.579 → 0.0.581

package/README.md

@@ -64,6 +64,7 @@ Rules:
 - Framework inference (Turbo hashes and passes Next's `NEXT_PUBLIC_*`, Vite's `VITE_*`, … for a package that depends on the framework): `turbo()` expands the prefix over the run's environment; the CLI writes the names the tracked source and `.env.example` of the package and its workspace dependencies spell (the bare prefix, as a `startsWith` test spells it, is none) (Next bundles theirs: cal.com's web, 23 alone, 58 with them), and a note asks for any only an installed dependency reads.
 
 - **Two tasks of one package on one output path** (strapi's `build`, `build:code` and `build:types`, all on `dist/**`): vx cleans a task's outputs before it runs and before a restore, so the loader refuses two cached tasks whose outputs provably overlap. Under vx's default `rules.exclusiveOutputs` an edge between the two changes nothing (twenty's `build:individual` writing into `build`'s `dist`). The mapping resolves it before the file is written — the task with a `^` edge keeps its cache (the first declared when none has one); the rest run uncached with a todo naming the keeper and the fix, their own output path. Same rule for Nx targets.
+- **A task whose inputs read another task's outputs** (a `test` keyed on `**/*` beside a `build` writing `dist/**`): vx's default `rules.upfrontKeys` refuses it, since such a key cannot be known before the producer ran. Each overlapping output is taken back from the reader's inputs with `!` (`['**/*', '!dist/**']`), in package and workspace inputs alike; Turbo and Nx hash what git tracks, and a build's outputs are not, so the key reads what the adopted tool read. A literal input another task writes cannot be taken back: that task runs uncached with a todo. A committed file under such an output is still a source: a reader the `!` would hide it from runs uncached with a todo. Same for Nx targets.
 - **Two packages' tasks on one workspace output** (cal.com's shared `post-install` writes `../../node_modules/@prisma/client/**` from every package with the script): core refuses two cached tasks on one path with no edge between them, so `turbo()` (and `nx()`) keeps the first (in package and task order) cached and runs the rest uncached, each with a todo naming the keeper. A package's own outputs count at their workspace path: typescript-eslint's root project caches `dist` and each package's typecheck `dist/packages/<name>`, one path twice. Edges across packages are not read, so an ordered pair loses its cache too.
 - `cache: false` or `persistent: true` → no `cache` block; a persistent task gets `exec.persistent: {}` and, when some task depends on it, the consumer's `persistentTodo`.
 - A task's `description` (Turbo 2.11.5's schema) is the vx task's `description`.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vzn/vx-migrate",
-  "version": "0.0.579",
+  "version": "0.0.581",
   "license": "MIT",
   "description": "Adopt @vzn/vx from Turborepo or Nx: run the repo unchanged, migrate it to vx.config.ts, and keep your Turbo or Nx remote cache",
   "keywords": [
@@ -39,7 +39,7 @@
     "access": "public"
   },
   "peerDependencies": {
-    "@vzn/vx": "^0.0.579"
+    "@vzn/vx": "^0.0.581"
   },
   "repository": {
     "type": "git",

package/src/nx/nx-map.ts

@@ -28,6 +28,8 @@ import { mapRunCommands, shellQuote } from '../nx-command.js'
 import { scriptCommand, yarnPnp } from '../script-command.js'
 import {
   ownFileTakeBacks,
+  excludeSiblingOutputs,
+  excludeWorkspaceOutputs,
   resolveSharedOutputs,
   resolveSharedWorkspaceOutputs,
   wildcardOutput,
@@ -557,11 +559,12 @@ export async function mapNxWorkspace(
       dir: meta.dir,
       importLines: [],
       ...(tags.length > 0 ? { tags } : {}),
-      tasks: resolveSharedOutputs(tasks),
+      tasks: excludeSiblingOutputs(resolveSharedOutputs(tasks)),
     })
   }
 
   resolveAtomizedWorkspaceOutputs(root, projects, split)
+  excludeWorkspaceOutputs(root, projects)
   // A migration leaves Nx behind, so `nx sync` is no advice there: what a
   // generator keeps (the TypeScript one's tsconfig `references`) is the
   // user's to keep. `nx()` runs with Nx installed.

package/src/shared-outputs.ts

@@ -230,6 +230,106 @@ export function resolveSharedOutputs(tasks: GeneratedTask[]): GeneratedTask[] {
   return tasks
 }
 
+/**
+ * Two globs can match one path only when one's literal prefix is a
+ * directory of the other's (`packages/*\/dist/**` and `packages/a/dist/**`),
+ * so an indexed glob is a candidate only along its prefix's chain. Every
+ * pair was compared: 1,000 packages' builds and tests took 9 s to map.
+ * `candidates` is a superset of the indexed entries a glob can overlap.
+ */
+function prefixIndex(): {
+  index: (i: number, globs: readonly string[]) => void
+  candidates: (globs: readonly string[]) => number[]
+} {
+  const at = new Map<string, number[]>()
+  const under = new Map<string, number[]>()
+  const chain = (g: string): string[] => {
+    const segs: string[] = []
+    for (const seg of g.split('/')) {
+      if (/[*?{}[\]()!]/.test(seg)) break
+      segs.push(seg)
+    }
+    return segs.map((_, i) => segs.slice(0, i + 1).join('/'))
+  }
+  return {
+    index(i, globs) {
+      for (const g of globs) {
+        const c = chain(g)
+        const push = (m: Map<string, number[]>, k: string) => {
+          const list = m.get(k)
+          if (list === undefined) m.set(k, [i])
+          else if (list.at(-1) !== i) list.push(i)
+        }
+        push(at, c.at(-1) ?? '')
+        for (const k of ['', ...c]) push(under, k)
+      }
+    },
+    candidates(globs) {
+      const found = new Set<number>()
+      for (const g of globs) {
+        const c = chain(g)
+        for (const k of ['', ...c]) for (const i of at.get(k) ?? []) found.add(i)
+        for (const i of under.get(c.at(-1) ?? '') ?? []) found.add(i)
+      }
+      return [...found].sort((a, b) => a - b)
+    },
+  }
+}
+
+/** A task's cached `inputs` block, when it has one with a `files` list. */
+function inputsOf(t: GeneratedTask): { files: string[]; workspaceFiles?: unknown } | undefined {
+  const inputs = (t.task?.['cache'] as { inputs?: { files?: unknown } } | undefined)?.inputs
+  return Array.isArray(inputs?.files)
+    ? (inputs as { files: string[]; workspaceFiles?: unknown })
+    : undefined
+}
+
+/**
+ * Core refuses a task whose inputs can match another task's declared
+ * outputs (`rules.upfrontKeys`, on by default, core X-54): such a key
+ * reads what a producer writes this run and cannot be known before it ran.
+ * Turbo and Nx default a task's inputs to every file of its package
+ * (`**\/*`), which matches a sibling build's `dist`. Each overlapping
+ * output is taken back with a `!` entry, in place: Turbo and Nx hash the
+ * files git tracks, and a build's outputs are not, so the adopted tool
+ * never keyed them either. A literal input another task writes cannot be
+ * taken back (core refuses a `!` over a literal input), so that task runs
+ * uncached with a todo. Committed files under an output are the tracked
+ * pass's (`spareTrackedOutputs`).
+ */
+export function excludeSiblingOutputs(tasks: GeneratedTask[]): GeneratedTask[] {
+  const writers = tasks.flatMap((t, index) => {
+    const c = cachedOutputs(t, index)
+    return c === null ? [] : [c]
+  })
+  if (writers.length === 0) return tasks
+  tasks.forEach((t, index) => {
+    const inputs = inputsOf(t)
+    if (inputs === undefined) return
+    const positive = inputs.files.filter((g) => typeof g === 'string' && !g.startsWith('!'))
+    for (const w of writers) {
+      if (w.index === index) continue
+      for (const out of w.files) {
+        if (inputs.files.includes(`!${out}`)) continue
+        const read = positive.filter((g) => outputsOverlap(g, out))
+        if (read.length === 0) continue
+        const literal = read.find((g) => isLiteralPattern(g))
+        if (literal !== undefined) {
+          delete t.task!['cache']
+          t.todos.push(
+            `reads ${JSON.stringify(literal)}, which ${JSON.stringify(w.name)} writes — vx keys a ` +
+              'task only on files no other task writes, so it runs uncached; read the source ' +
+              'instead in a vx.config to cache it',
+          )
+          return
+        }
+        inputs.files.push(`!${out}`)
+      }
+    }
+  })
+  return tasks
+}
+
 /**
  * The same across projects, for workspace outputs: cal.com's shared
  * `post-install` writes `../../node_modules/@prisma/client/**` from every
@@ -255,41 +355,7 @@ export function resolveSharedWorkspaceOutputs(
       ? v.filter((f): f is string => typeof f === 'string' && !f.startsWith('!'))
       : []
   const kept: { id: string; project: string; ws: string[]; own: string[] }[] = []
-  // Two globs can match one path only when one's literal prefix is a
-  // directory of the other's (`packages/*/dist/**` and `packages/a/dist/**`),
-  // so a kept glob is a candidate only along its prefix's chain. Every pair
-  // was compared: 1,000 packages' builds and tests took 9 s to map.
-  const at = new Map<string, number[]>()
-  const under = new Map<string, number[]>()
-  const chain = (g: string): string[] => {
-    const segs: string[] = []
-    for (const seg of g.split('/')) {
-      if (/[*?{}[\]()!]/.test(seg)) break
-      segs.push(seg)
-    }
-    return segs.map((_, i) => segs.slice(0, i + 1).join('/'))
-  }
-  const index = (i: number, globs: readonly string[]): void => {
-    for (const g of globs) {
-      const c = chain(g)
-      const push = (m: Map<string, number[]>, k: string) => {
-        const list = m.get(k)
-        if (list === undefined) m.set(k, [i])
-        else if (list.at(-1) !== i) list.push(i)
-      }
-      push(at, c.at(-1) ?? '')
-      for (const k of ['', ...c]) push(under, k)
-    }
-  }
-  const candidates = (globs: readonly string[]): number[] => {
-    const found = new Set<number>()
-    for (const g of globs) {
-      const c = chain(g)
-      for (const k of ['', ...c]) for (const i of at.get(k) ?? []) found.add(i)
-      for (const i of under.get(c.at(-1) ?? '') ?? []) found.add(i)
-    }
-    return [...found].sort((a, b) => a - b)
-  }
+  const { index, candidates } = prefixIndex()
   for (const p of projects) {
     const rel = relPosix(root, p.dir)
     for (const t of p.tasks) {
@@ -332,6 +398,79 @@ export function resolveSharedWorkspaceOutputs(
   }
 }
 
+/**
+ * `excludeSiblingOutputs` for root-anchored `cache.inputs.workspaceFiles`:
+ * every other task's `outputs.workspaceFiles`, and its `outputs.files` at
+ * their workspace path, is taken back from a reader's workspace inputs
+ * (Turbo's `globalDependencies` and `$TURBO_ROOT$` entries, Nx's
+ * `{workspaceRoot}` filesets). Run after `resolveSharedWorkspaceOutputs`,
+ * over what stays cached.
+ */
+export function excludeWorkspaceOutputs(
+  root: string,
+  projects: readonly {
+    readonly name: string
+    readonly dir: string
+    readonly tasks: GeneratedTask[]
+  }[],
+): void {
+  const positive = (v: unknown): string[] =>
+    Array.isArray(v)
+      ? v.filter((f): f is string => typeof f === 'string' && !f.startsWith('!'))
+      : []
+  const writers: { task: GeneratedTask; id: string; globs: string[] }[] = []
+  const { index, candidates } = prefixIndex()
+  for (const p of projects) {
+    const rel = relPosix(root, p.dir)
+    for (const t of p.tasks) {
+      const outputs = (
+        t.task?.['cache'] as { outputs?: { files?: unknown; workspaceFiles?: unknown } } | undefined
+      )?.outputs
+      const globs = [
+        ...positive(outputs?.workspaceFiles),
+        ...positive(outputs?.files).map((g) =>
+          rel === '' ? g.replace(/^(\.\/)+/, '') : `${rel}/${g.replace(/^(\.\/)+/, '')}`,
+        ),
+      ]
+      if (globs.length === 0) continue
+      index(writers.length, globs)
+      writers.push({ task: t, id: `${p.name}#${t.name}`, globs })
+    }
+  }
+  if (writers.length === 0) return
+  for (const p of projects) {
+    for (const t of p.tasks) {
+      const inputs = (t.task?.['cache'] as { inputs?: { workspaceFiles?: unknown } } | undefined)
+        ?.inputs
+      if (inputs === undefined || !Array.isArray(inputs.workspaceFiles)) continue
+      const files = inputs.workspaceFiles as string[]
+      const reads = positive(files)
+      if (reads.length === 0) continue
+      for (const i of candidates(reads)) {
+        const w = writers[i]!
+        if (w.task === t) continue
+        for (const out of w.globs) {
+          if (files.includes(`!${out}`)) continue
+          const read = reads.filter((g) => outputsOverlap(g, out))
+          if (read.length === 0) continue
+          const literal = read.find((g) => isLiteralPattern(g))
+          if (literal !== undefined) {
+            delete t.task!['cache']
+            t.todos.push(
+              `reads ${JSON.stringify(literal)}, which ${w.id} writes — vx keys a task only on ` +
+                'files no other task writes, so it runs uncached; read the source instead in a ' +
+                'vx.config to cache it',
+            )
+            break
+          }
+          files.push(`!${out}`)
+        }
+        if (t.task?.['cache'] === undefined) break
+      }
+    }
+  }
+}
+
 /** A `!` output with no positive beside it takes back nothing, and core refuses a list of them alone. */
 export function takingBack(globs: readonly string[]): string[] {
   return globs.some((g) => !g.startsWith('!')) ? [...globs] : []

package/src/tracked-outputs.ts

@@ -178,6 +178,18 @@ function coveredTracked(globs: readonly string[], files: readonly string[]): str
   return [...hit].sort()
 }
 
+/** The first of `files` a `!` entry of `globs` takes back though a positive one reads it. */
+function hidden(globs: unknown, files: readonly string[]): string | undefined {
+  if (!Array.isArray(globs) || files.length === 0) return undefined
+  const strings = globs.filter((g): g is string => typeof g === 'string')
+  const negative = strings.filter((g) => g.startsWith('!')).map((g) => g.slice(1))
+  if (negative.length === 0) return undefined
+  const positive = strings.filter((g) => !g.startsWith('!'))
+  return files.find(
+    (f) => negative.some((n) => outputsOverlap(n, f)) && positive.some((g) => outputsOverlap(g, f)),
+  )
+}
+
 /** Past this many, a task's take-backs cost its runs more than its cache saves. */
 export const MAX_SPARED = 16
 
@@ -223,6 +235,30 @@ export function spareTrackedOutputs(
       }
       const spared = hits.flatMap(([, , hit]) => hit)
       if (spared.length === 0) continue
+      // A committed file under these outputs is still a source, and the `!`
+      // entry `excludeSiblingOutputs` gave a reader over the whole output
+      // hides it from that reader's key: such a reader runs uncached (X-54).
+      const local = hits.flatMap(([key, , hit]) => (key === 'files' ? hit : []))
+      const rooted = hits.flatMap(([key, , hit]) =>
+        key === 'files' ? hit.map((f) => (rel === '' ? f : `${rel}/${f}`)) : hit,
+      )
+      for (const q of projects) {
+        for (const r of q.tasks) {
+          if (r === t) continue
+          const inputs = (r.task?.['cache'] as { inputs?: Outputs } | undefined)?.inputs
+          if (inputs === undefined) continue
+          const f =
+            (q === p ? hidden(inputs.files, local) : undefined) ??
+            hidden(inputs.workspaceFiles, rooted)
+          if (f === undefined) continue
+          delete r.task!['cache']
+          todos.push([
+            `${q.name}#${r.name}`,
+            `reads the committed ${f}, which its inputs take back with ${p.name}#${t.name}'s ` +
+              'outputs — task runs uncached; declare its inputs in a vx.config to cache it',
+          ])
+        }
+      }
       const shown = spared.slice(0, 3).join(', ') + (spared.length > 3 ? ', …' : '')
       // Each `!` is matched against every output path on each save, clean
       // and restore: 4,200 of them cost typescript-eslint's warm run 1.2 s.

package/src/turbo/turbo-map.ts

@@ -19,6 +19,8 @@ import { scriptCommand, yarnPnp } from '../script-command.js'
 import {
   ownFileOutput,
   ownFileTodo,
+  excludeSiblingOutputs,
+  excludeWorkspaceOutputs,
   resolveSharedOutputs,
   resolveSharedWorkspaceOutputs,
   takingBack,
@@ -1375,7 +1377,7 @@ export async function mapTurboWorkspace(
         npmScriptEnv(mapped.task, name, command === script, meta, opts)
       tasks.push(mapped)
     }
-    resolveSharedOutputs(tasks)
+    excludeSiblingOutputs(resolveSharedOutputs(tasks))
     projects.push({ name: meta.name, dir: meta.dir, tasks })
   }
 
@@ -1398,6 +1400,7 @@ export async function mapTurboWorkspace(
     )
   literalEnvGapsOnce(projects, notes)
   resolveSharedWorkspaceOutputs(root, projects)
+  excludeWorkspaceOutputs(root, projects)
   pruneUnreachedPersistentNotes(projects, metas, opts.persistentTodo)
   pruneOrphanPersistentNotes(projects, opts.persistentTodo)
   return { projects, notes, globals }