@vzn/vx 0.0.18 → 0.0.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/LICENSE +1 -1
  2. package/README.md +99 -65
  3. package/package.json +10 -20
  4. package/src/bin.ts +12 -2
  5. package/src/cache/archive.ts +120 -18
  6. package/src/cache/cache.ts +313 -1324
  7. package/src/cache/chained-cache.ts +0 -8
  8. package/src/cache/config-evals.ts +89 -0
  9. package/src/cache/file-hashes.ts +262 -0
  10. package/src/cache/git-inputs.ts +637 -0
  11. package/src/cache/index.ts +16 -7
  12. package/src/cache/inputs.ts +246 -733
  13. package/src/cache/layer.ts +604 -0
  14. package/src/cache/layered-cache.ts +12 -17
  15. package/src/cache/output-index.ts +224 -0
  16. package/src/cache/policy.ts +84 -0
  17. package/src/cache/run-history.ts +147 -0
  18. package/src/cache/zstd.ts +160 -0
  19. package/src/cli/cache.ts +39 -11
  20. package/src/cli/completions.ts +104 -0
  21. package/src/cli/core-alias.ts +25 -0
  22. package/src/cli/help.ts +64 -36
  23. package/src/cli/index.ts +54 -15
  24. package/src/cli/info.ts +120 -91
  25. package/src/cli/init.ts +75 -0
  26. package/src/cli/last.ts +38 -6
  27. package/src/cli/lock.ts +2 -1
  28. package/src/cli/plan-format.ts +14 -3
  29. package/src/cli/plugin-commands.ts +15 -8
  30. package/src/cli/run.ts +84 -302
  31. package/src/cli/select.ts +332 -0
  32. package/src/cli/show.ts +144 -69
  33. package/src/cli/upgrade.ts +2 -1
  34. package/src/cli/watch.ts +647 -97
  35. package/src/cli/why.ts +44 -14
  36. package/src/cli/workspace-config.ts +126 -0
  37. package/src/config.ts +156 -136
  38. package/src/exec/env.ts +11 -0
  39. package/src/exec/executor.ts +11 -26
  40. package/src/exec/index.ts +6 -2
  41. package/src/{plugins/local-executor/index.ts → exec/local-executor.ts} +12 -16
  42. package/src/exec/runner.ts +57 -18
  43. package/src/exec/sandbox-binds.ts +182 -0
  44. package/src/exec/sandbox-paths.ts +50 -0
  45. package/src/exec/sandbox-runtime.ts +582 -370
  46. package/src/exec/sandbox-violations.ts +280 -0
  47. package/src/graph/index.ts +1 -9
  48. package/src/graph/scheduler.ts +158 -203
  49. package/src/graph/task-graph.ts +2 -2
  50. package/src/index.ts +97 -102
  51. package/src/orchestrator/admission.ts +137 -0
  52. package/src/orchestrator/deferred-outputs.ts +3 -0
  53. package/src/orchestrator/doctor.ts +255 -0
  54. package/src/orchestrator/events.ts +42 -8
  55. package/src/orchestrator/execute-task.ts +145 -554
  56. package/src/orchestrator/failure-mode.ts +169 -28
  57. package/src/orchestrator/framed-output.ts +41 -36
  58. package/src/orchestrator/history.ts +118 -98
  59. package/src/orchestrator/hit-restore.ts +221 -0
  60. package/src/orchestrator/index.ts +35 -4
  61. package/src/orchestrator/lockfile-claim.ts +281 -0
  62. package/src/orchestrator/logger.ts +13 -22
  63. package/src/orchestrator/metrics.ts +53 -20
  64. package/src/orchestrator/miss-save.ts +169 -0
  65. package/src/orchestrator/options.ts +34 -31
  66. package/src/orchestrator/persistent.ts +79 -0
  67. package/src/orchestrator/placement.ts +182 -0
  68. package/src/orchestrator/plugin-host.ts +195 -38
  69. package/src/orchestrator/plugin.ts +162 -9
  70. package/src/orchestrator/prepare.ts +77 -132
  71. package/src/orchestrator/projects.ts +259 -0
  72. package/src/orchestrator/remote-prefetch.ts +1 -1
  73. package/src/orchestrator/run-artifacts.ts +25 -3
  74. package/src/orchestrator/run-context.ts +1 -1
  75. package/src/orchestrator/run-records.ts +149 -0
  76. package/src/orchestrator/run-report.ts +2 -2
  77. package/src/orchestrator/run.ts +246 -597
  78. package/src/orchestrator/sandbox-request.ts +243 -0
  79. package/src/orchestrator/save-lane.ts +47 -0
  80. package/src/orchestrator/signals.ts +111 -0
  81. package/src/orchestrator/stable-keys.ts +60 -3
  82. package/src/orchestrator/status-line.ts +2 -1
  83. package/src/orchestrator/summary.ts +68 -25
  84. package/src/orchestrator/task-hash.ts +20 -18
  85. package/src/orchestrator/task-log-buffer.ts +8 -11
  86. package/src/orchestrator/telemetry-host.ts +11 -1
  87. package/src/orchestrator/telemetry.ts +3 -17
  88. package/src/util/cgroup.ts +160 -0
  89. package/src/util/edit-distance.ts +49 -0
  90. package/src/util/index.ts +11 -3
  91. package/src/util/paths.ts +23 -2
  92. package/src/util/settle.ts +19 -0
  93. package/src/util/verbs.ts +39 -0
  94. package/src/workspace/affected.ts +111 -11
  95. package/src/workspace/config-cache.ts +11 -2
  96. package/src/workspace/config-eval.ts +11 -4
  97. package/src/workspace/config-imports.ts +5 -4
  98. package/src/workspace/config-schema.ts +837 -0
  99. package/src/workspace/filter.ts +43 -6
  100. package/src/workspace/fingerprint.ts +37 -3
  101. package/src/workspace/index.ts +30 -3
  102. package/src/workspace/lockfile.ts +12 -7
  103. package/src/{cli → workspace}/migrate-scripts.ts +26 -19
  104. package/src/{cli/migrate.ts → workspace/migration.ts} +133 -124
  105. package/src/workspace/package-graph.ts +98 -28
  106. package/src/workspace/project-loader.ts +76 -746
  107. package/src/workspace/workspace.ts +98 -13
  108. package/plugins/local-cache/index.ts +0 -2
  109. package/plugins/local-executor/index.ts +0 -2
  110. package/plugins/schedule-history/index.ts +0 -2
  111. package/src/cache/cas-backend.ts +0 -115
  112. package/src/cache/digest.ts +0 -32
  113. package/src/cli/migrate-emit.ts +0 -21
  114. package/src/cli/migrate-nx.ts +0 -473
  115. package/src/cli/migrate-turbo.ts +0 -401
  116. package/src/cli/prune.ts +0 -291
  117. package/src/orchestrator/missing-plugin.ts +0 -9
  118. package/src/orchestrator/resources.ts +0 -45
  119. package/src/orchestrator/verify.ts +0 -174
  120. package/src/plugins/local-cache/index.ts +0 -11
  121. package/src/plugins/schedule-history/index.ts +0 -109
@@ -6,23 +6,37 @@
6
6
  // `cache.inputs.env` is the cache-tracking axis for env vars; it's
7
7
  // independent of `exec.env`, which controls what reaches the child.
8
8
  //
9
- // File enumeration defers to git — same as Turbo and Nx. We ask git for
10
- // the file set via `git ls-files --cached --others --exclude-standard`,
11
- // which gives us:
12
- // - all tracked files,
13
- // - plus untracked-but-not-ignored files,
14
- // - with nested .gitignore + .git/info/exclude + global excludes
15
- // correctly applied (because git already does the cascade).
16
- // The user's `inputs.files` globs are then matched as a *filter* on
17
- // top of that file set. vx requires git to be installed and the
18
- // workspace to be a git work tree; non-git environments are not
19
- // supported.
9
+ // File enumeration defers to git — same as Turbo and Nx. git-inputs.ts
10
+ // asks for the tracked set with `git ls-files -s -v` (index OIDs and the
11
+ // skip-worktree flag in the same spawn) and for the dirty and untracked
12
+ // paths with `git status -uall`, with nested .gitignore, .git/info/exclude
13
+ // and the global excludes applied because git already does the cascade.
14
+ // The user's `inputs.files` globs are then matched as a *filter* on top
15
+ // of that file set. vx requires git to be installed and the workspace to
16
+ // be a git work tree; non-git environments are not supported.
20
17
 
21
18
  import path from 'node:path'
22
- import { existsSync } from 'node:fs'
23
- import { realpath, rm } from 'node:fs/promises'
19
+ import { lstatSync } from 'node:fs'
20
+ import { realpath, rm, rmdir } from 'node:fs/promises'
24
21
  import type { CacheInputs } from '../config.js'
25
22
  import { UserError } from '../util/index.js'
23
+ import { GitFilesCache, runGitLsFiles } from './git-inputs.js'
24
+ import { normalizeGlob } from '../util/index.js'
25
+
26
+ // The git side lives in git-inputs.ts; its whole public surface is
27
+ // re-exported here so a reader that reaches the resolver for it (the
28
+ // tests do, by deep import) keeps working.
29
+ export {
30
+ GitFilesCache,
31
+ populateGitFilesCache,
32
+ runGitLsFiles,
33
+ startGitEnumeration,
34
+ applyGitEnumeration,
35
+ gitPathspecs,
36
+ parseCheckAttrOutput,
37
+ autocrlfConverts,
38
+ type GitEnumeration,
39
+ } from './git-inputs.js'
26
40
 
27
41
  // vx-lock.json is committed (so git enumerates it) but it's vx's own
28
42
  // frozen-config metadata — never a task input. Excluded globally so a
@@ -75,7 +89,11 @@ export interface ResolveInputsArgs {
75
89
  * Run-scoped memo for `cache.inputs.runtime` command execution, keyed
76
90
  * by `projectDir + '\0' + command`. Shared across a run's tasks so a
77
91
  * project's command runs once even across build/test/lint and across
78
- * the hash + sandbox-baseline resolveInputs calls.
92
+ * the hash + sandbox-baseline resolveInputs calls. The key is sound
93
+ * ONLY because the probe runs in vx's own ambient environment, never a
94
+ * task's `exec.env` (`runRuntimeCommand`): two tasks with different
95
+ * `define`s share one value. Threading the task env in would have to
96
+ * widen this key too, or task A's probe becomes task B's key component.
79
97
  */
80
98
  runtimeCache?: Map<string, Promise<string>>
81
99
  /**
@@ -83,8 +101,21 @@ export interface ResolveInputsArgs {
83
101
  * only — global dedup so a root-level probe spawns once per run.
84
102
  */
85
103
  workspaceRuntimeCache?: Map<string, Promise<string>>
104
+ /**
105
+ * Run-scoped memo for `cache.inputs.workspaceFiles`, keyed by the
106
+ * declaration and valid for one enumeration snapshot. A Turbo-mapped
107
+ * workspace gives every task the same `globalDependencies`, and
108
+ * resolving one literal against medusa's 24k-file enumeration per task
109
+ * was 930 ms of a 3.0 s warm no-op (159 scans, 2026-09-11).
110
+ */
111
+ workspaceFilesCache?: WorkspaceFilesCache
86
112
  }
87
113
 
114
+ export type WorkspaceFilesCache = Map<
115
+ string,
116
+ { snapshot: readonly string[]; result: Promise<string[]> }
117
+ >
118
+
88
119
  export async function resolveInputs(args: ResolveInputsArgs): Promise<ResolvedInputs> {
89
120
  const projectFiles = await resolveFiles({
90
121
  projectDir: args.projectDir,
@@ -102,26 +133,29 @@ export async function resolveInputs(args: ResolveInputsArgs): Promise<ResolvedIn
102
133
  workspaceFiles: wsDecl,
103
134
  ownWorkspaceOutputs: args.ownWorkspaceOutputs ?? [],
104
135
  ...(args.gitFilesCache !== undefined ? { gitFilesCache: args.gitFilesCache } : {}),
136
+ ...(args.workspaceFilesCache !== undefined ? { memo: args.workspaceFilesCache } : {}),
105
137
  })
106
138
  // Dedupe: when the project dir IS the workspace root (or a glob
107
139
  // overlaps), the same absolute path can arrive via both lists —
108
140
  // it must contribute to the key exactly once.
109
141
  if (wsFiles.length > 0) files = [...new Set([...projectFiles, ...wsFiles])].sort()
110
142
  }
111
- const [runtimeValues, workspaceRuntimeValues] = await Promise.all([
112
- resolveRuntimeValues(
113
- args.inputs?.runtime ?? [],
114
- args.projectDir,
115
- args.runtimeCache,
116
- `${args.projectDir}\0`,
117
- ),
118
- resolveRuntimeValues(
119
- args.inputs?.workspaceRuntime ?? [],
120
- args.workspaceRoot,
121
- args.workspaceRuntimeCache,
122
- '',
123
- ),
124
- ])
143
+ // Two awaited empty resolutions per task were ~3 ms of a 1000-task warm
144
+ // run (profiled 2026-09-09); a task declaring neither skips the fan-out.
145
+ const runtimeDecl = args.inputs?.runtime ?? []
146
+ const wsRuntimeDecl = args.inputs?.workspaceRuntime ?? []
147
+ const [runtimeValues, workspaceRuntimeValues] =
148
+ runtimeDecl.length === 0 && wsRuntimeDecl.length === 0
149
+ ? [[], []]
150
+ : await Promise.all([
151
+ resolveRuntimeValues(
152
+ runtimeDecl,
153
+ args.projectDir,
154
+ args.runtimeCache,
155
+ `${args.projectDir}\0`,
156
+ ),
157
+ resolveRuntimeValues(wsRuntimeDecl, args.workspaceRoot, args.workspaceRuntimeCache, ''),
158
+ ])
125
159
  return {
126
160
  files,
127
161
  envValues: resolveEnvValues(args.inputs?.env ?? [], args.envSource),
@@ -146,6 +180,7 @@ async function resolveWorkspaceFiles(args: {
146
180
  workspaceFiles: readonly string[]
147
181
  ownWorkspaceOutputs: readonly string[]
148
182
  gitFilesCache?: GitFilesCache
183
+ memo?: WorkspaceFilesCache
149
184
  }): Promise<string[]> {
150
185
  const positive: string[] = []
151
186
  const negative: string[] = []
@@ -155,8 +190,12 @@ async function resolveWorkspaceFiles(args: {
155
190
  }
156
191
  if (positive.length === 0) return []
157
192
 
158
- const excludeGlobs = [...ALWAYS_IGNORE, ...args.ownWorkspaceOutputs, ...negative].map(globFor)
159
- const positiveGlobs = positive.map(globFor)
193
+ const excludeGlobs = [
194
+ ...ALWAYS_IGNORE,
195
+ ...asTrees(args.ownWorkspaceOutputs),
196
+ ...asTrees(negative),
197
+ ].map(globFor)
198
+ const positiveGlobs = asTrees(positive).map(globFor)
160
199
  // Workspace-wide partition, keyed by the workspace root. Populated
161
200
  // up-front by `populateGitFilesCache(..., workspaceWide: true)` when
162
201
  // any loaded task declares workspaceFiles; a missing/invalidated
@@ -166,14 +205,39 @@ async function resolveWorkspaceFiles(args: {
166
205
  gitFiles = runGitLsFiles(args.workspaceRoot).files
167
206
  args.gitFilesCache?.set(args.workspaceRoot, gitFiles)
168
207
  }
208
+ // The memo is valid for the snapshot it was computed over: a task that
209
+ // wrote workspace outputs mid-run replaces the partition, and the next
210
+ // caller sees a different array and scans again.
211
+ const memoKey =
212
+ args.memo === undefined
213
+ ? undefined
214
+ : JSON.stringify([positive, negative, args.ownWorkspaceOutputs])
215
+ if (memoKey !== undefined) {
216
+ const hit = args.memo!.get(memoKey)
217
+ if (hit !== undefined && hit.snapshot === gitFiles) return hit.result
218
+ }
219
+ const result = resolveWorkspaceFilesOver(args, gitFiles, positive, positiveGlobs, excludeGlobs)
220
+ if (memoKey !== undefined) args.memo!.set(memoKey, { snapshot: gitFiles, result })
221
+ return result
222
+ }
223
+
224
+ async function resolveWorkspaceFilesOver(
225
+ args: { workspaceRoot: string; gitFilesCache?: GitFilesCache },
226
+ gitFiles: readonly string[],
227
+ positive: readonly string[],
228
+ positiveGlobs: readonly Bun.Glob[],
229
+ excludeGlobs: readonly Bun.Glob[],
230
+ ): Promise<string[]> {
169
231
  // Second call site of the literal-input guard. `resolveWorkspaceFiles`
170
232
  // carries its own copy of the filter-over-git-set design, so the same
171
233
  // silently-folds-nothing hazard exists here — and a fix applied only to the
172
234
  // project half would pass that half's tests while leaving this one live.
173
- const unmatchedLiterals = new Set(positive.filter(isLiteralPath))
235
+ const unmatchedLiterals = new Set(
236
+ positive.map(normalizeGlob).filter(isLiteralPath).map(stripTrailingSlash),
237
+ )
174
238
  const candidates: string[] = []
175
239
  for (const rel of gitFiles) {
176
- if (unmatchedLiterals.size > 0) unmatchedLiterals.delete(rel)
240
+ if (unmatchedLiterals.size > 0) settleLiterals(unmatchedLiterals, rel)
177
241
  if (!positiveGlobs.some((g) => g.match(rel))) continue
178
242
  if (excludeGlobs.some((g) => g.match(rel))) continue
179
243
  candidates.push(path.resolve(args.workspaceRoot, rel))
@@ -184,14 +248,27 @@ async function resolveWorkspaceFiles(args: {
184
248
  // Same OID-trust shortcut as project files: a clean-per-status
185
249
  // tracked file necessarily exists on disk.
186
250
  const oids = args.gitFilesCache?.oidsFor(args.workspaceRoot)
187
- const exists = await Promise.all(
188
- candidates.map((abs) => oids?.has(abs) === true || Bun.file(abs).exists()),
189
- )
190
- const matches: string[] = []
191
- for (let i = 0; i < candidates.length; i++) {
192
- if (exists[i]) matches.push(candidates[i]!)
251
+ return candidates.filter((abs) => oids?.has(abs) === true || isInputOnDisk(abs)).sort()
252
+ }
253
+
254
+ /**
255
+ * What an enumerated path must be to count as an input: a regular file, or
256
+ * a symlink (to anything, or to nothing — its target STRING is what folds,
257
+ * as in git). A directory is not one: git lists a gitlink (a submodule) at
258
+ * its path and the glob `**\/*` matches it, and the hasher has nothing to
259
+ * read there. `Bun.file(p).exists()` answered false for every symlink to a
260
+ * directory and every dangling link too, which silently dropped a tracked
261
+ * link from the key — retargeting it was a stale hit. lstat, synchronously:
262
+ * the paths that reach this probe are the ones without a trusted index OID
263
+ * (untracked and dirty files), a handful on a warm run.
264
+ */
265
+ function isInputOnDisk(abs: string): boolean {
266
+ try {
267
+ const st = lstatSync(abs)
268
+ return st.isFile() || st.isSymbolicLink()
269
+ } catch {
270
+ return false
193
271
  }
194
- return matches.sort()
195
272
  }
196
273
 
197
274
  function resolveEnvValues(
@@ -205,6 +282,11 @@ function resolveEnvValues(
205
282
  * Run one runtime-input command via `sh -c` (so pipelines / redirects
206
283
  * work — "shell is the API"). Returns trimmed stdout+stderr. A non-zero
207
284
  * exit is a hard UserError naming the command (fail-loud, like git).
285
+ *
286
+ * The probe inherits vx's AMBIENT environment on purpose — not the task's
287
+ * `exec.env.define` / `passThrough`, which describe the command's
288
+ * environment, not the machine's. That is what lets the run-scoped memo
289
+ * key on (projectDir, command) alone; see `runtimeCache`.
208
290
  */
209
291
  async function runRuntimeCommand(command: string, cwd: string): Promise<string> {
210
292
  let proc
@@ -271,7 +353,9 @@ export async function resolveOutputs(args: {
271
353
  }): Promise<string[]> {
272
354
  if (args.outputs.length === 0) return []
273
355
  const excludeGlobs = boundaryIgnorePatterns(args.projectDir, args.nestedProjectDirs).map(globFor)
274
- const scanned = [...(await scanUnion(args.outputs, excludeGlobs, args.projectDir))]
356
+ const scanned = [
357
+ ...(await scanUnion(asTrees(args.outputs), excludeGlobs, args.projectDir, 'outputs')),
358
+ ]
275
359
  // Containment, enforced HERE and not only at the loader. `cleanOutputs`
276
360
  // DELETES whatever this returns, and `Bun.Glob.scan` happily walks `..` out
277
361
  // of its cwd — so the loader's `..`/absolute rejection alone was a single
@@ -355,12 +439,43 @@ export async function cleanOutputs(args: {
355
439
  const files = await resolveOutputs(args)
356
440
  // `force: true` makes rm tolerate ENOENT (e.g. when two output
357
441
  // globs overlap and a sibling already deleted a path mid-iteration).
442
+ // A symlink is unlinked, never followed.
358
443
  await Promise.all(files.map((f) => rm(f, { force: true })))
444
+ await pruneEmptiedDirs(args.projectDir, files)
359
445
  // Project-relative posix paths of what was removed — the caller
360
446
  // feeds these to GitFilesCache.markOutputsChanged after a restore.
361
447
  return files.map((f) => path.relative(args.projectDir, f).split(path.sep).join('/'))
362
448
  }
363
449
 
450
+ /**
451
+ * Remove the directories a clean emptied, bottom-up, never `root` itself.
452
+ * A restore writes files where the cached entry has them; a directory left
453
+ * standing where the entry holds a FILE of the same name (a task that once
454
+ * wrote `dist/out/…` and now writes `dist/out`) blocks the rename, and an
455
+ * empty directory is not an output anyone declared. A directory that still
456
+ * holds something — a stray the globs do not cover — stays, and the restore
457
+ * says so if it is in the way.
458
+ */
459
+ async function pruneEmptiedDirs(root: string, removed: readonly string[]): Promise<void> {
460
+ const rootResolved = path.resolve(root)
461
+ const tried = new Set<string>()
462
+ // Deepest first, so a parent is attempted after every child had its turn.
463
+ const dirs = [...new Set(removed.map((f) => path.dirname(f)))].sort(
464
+ (a, b) => b.split(path.sep).length - a.split(path.sep).length,
465
+ )
466
+ for (let dir of dirs) {
467
+ while (dir !== rootResolved && dir.startsWith(rootResolved + path.sep) && !tried.has(dir)) {
468
+ tried.add(dir)
469
+ const gone = await rmdir(dir).then(
470
+ () => true,
471
+ (err: NodeJS.ErrnoException) => err.code === 'ENOENT',
472
+ )
473
+ if (!gone) break
474
+ dir = path.dirname(dir)
475
+ }
476
+ }
477
+ }
478
+
364
479
  /**
365
480
  * Resolve declared `outputs.workspaceFiles` globs (workspace-root-
366
481
  * relative) to actual produced files. Live-FS glob like
@@ -373,7 +488,7 @@ export async function resolveWorkspaceOutputs(args: {
373
488
  outputs: string[]
374
489
  }): Promise<string[]> {
375
490
  if (args.outputs.length === 0) return []
376
- const scanned = [...(await scanUnion(args.outputs, [], args.workspaceRoot))]
491
+ const scanned = [...(await scanUnion(asTrees(args.outputs), [], args.workspaceRoot, 'outputs'))]
377
492
  // Same containment as the project twin, anchored one level out. These globs
378
493
  // deliberately ignore PROJECT boundaries — that is the escape hatch — but
379
494
  // escaping the WORKSPACE was never part of it, and `cleanWorkspaceOutputs`
@@ -395,181 +510,48 @@ export async function cleanWorkspaceOutputs(args: {
395
510
  }): Promise<string[]> {
396
511
  const files = await resolveWorkspaceOutputs(args)
397
512
  await Promise.all(files.map((f) => rm(f, { force: true })))
513
+ await pruneEmptiedDirs(args.workspaceRoot, files)
398
514
  return files.map((f) => path.relative(args.workspaceRoot, f).split(path.sep).join('/'))
399
515
  }
400
516
 
401
- /**
402
- * Per-run memo of each project's `git ls-files` output, plus the
403
- * staleness bookkeeping that lets the warm path avoid re-spawning git.
404
- *
405
- * After a cache-hit restore we know EXACTLY which paths changed on
406
- * disk: the declared outputs `cleanOutputs` wiped plus the artifact's
407
- * output files. `markOutputsChanged` records them; `snapshotFor`
408
- * hands back the existing snapshot when a resolving task's input
409
- * globs can't match any changed path — provably identical to what a
410
- * re-spawn would return, since glob matching ignores gitignore status
411
- * entirely when the path doesn't match. When globs DO overlap,
412
- * returning undefined forces the caller down the re-spawn path so
413
- * gitignore semantics stay byte-identical.
414
- *
415
- * The cache-miss save path still uses plain `delete` — an executed
416
- * task may write files outside its declared outputs, and only git can
417
- * see those.
418
- */
419
- export class GitFilesCache extends Map<string, readonly string[]> {
420
- private changed = new Map<string, string[]>()
421
- /**
422
- * Per-project trusted index OIDs: absolute path → git blob OID, for
423
- * tracked regular files whose working-tree state matched the index
424
- * at populate time (per one `git status --porcelain` snapshot).
425
- * These feed `Cache.key` via `CacheKeyInput.fileHashes` so a clean
426
- * tree derives input hashes with zero reads / stats / SQLite.
427
- *
428
- * Dropped wholesale on `set` / `delete`: a mid-run re-enumeration
429
- * proves the project's tree changed, and index OIDs can't be
430
- * re-trusted without a fresh status — the per-file fallback
431
- * (`Cache.hashFile`) computes the identical blob OID from disk, so
432
- * dropping is a pure perf concession, never a correctness one.
433
- */
434
- private oids = new Map<string, Map<string, string>>()
435
- /**
436
- * Set when `populateGitFilesCache` stored a WORKSPACE-WIDE partition
437
- * (any loaded task declares `inputs.workspaceFiles`). Null when the
438
- * feature is unused — every workspace-partition hook below is then a
439
- * no-op, so unused-feature behavior stays byte-identical.
440
- */
441
- private wsRoot: string | null = null
442
- /**
443
- * Whether the worktree had any uncommitted changes at populate time,
444
- * derived from the SAME `git status --porcelain` spawn that prunes
445
- * dirty paths from the trusted-OID set. Lets the Tier-3 invocation
446
- * record report `dirty` without a SECOND status spawn. `null` until
447
- * populate runs, or when the status spawn failed (non-repo).
448
- */
449
- private dirty: boolean | null = null
450
-
451
- setWorkspaceRoot(root: string): void {
452
- this.wsRoot = root
453
- }
454
-
455
- /** Aggregate worktree dirtiness from the populate-time status spawn. */
456
- get worktreeDirty(): boolean | null {
457
- return this.dirty
458
- }
459
-
460
- setWorktreeDirty(dirty: boolean | null): void {
461
- this.dirty = dirty
462
- }
463
-
464
- markOutputsChanged(projectDir: string, relPaths: readonly string[]): void {
465
- this.recordChanged(projectDir, relPaths)
466
- // The workspace-wide partition sees the same files under
467
- // root-relative names; forward so a downstream workspaceFiles
468
- // task can't reuse a snapshot its globs could now contradict.
469
- if (this.wsRoot !== null && this.wsRoot !== projectDir) {
470
- this.recordChanged(
471
- this.wsRoot,
472
- relPaths.map((rel) =>
473
- path.relative(this.wsRoot!, path.resolve(projectDir, rel)).split(path.sep).join('/'),
474
- ),
475
- )
476
- }
477
- }
478
-
479
- /**
480
- * Record root-anchored changed paths (cleaned/restored workspace
481
- * outputs) against EVERY partition that can see them: the workspace
482
- * partition under their root-relative names, and any project
483
- * partition whose dir contains them (workspace outputs may land
484
- * inside other projects' dirs — the no-boundary escape hatch).
485
- */
486
- markWorkspaceOutputsChanged(workspaceRoot: string, relPaths: readonly string[]): void {
487
- if (relPaths.length === 0) return
488
- for (const key of this.keys()) {
489
- if (key === workspaceRoot) {
490
- this.recordChanged(key, relPaths)
491
- continue
492
- }
493
- const under: string[] = []
494
- for (const rel of relPaths) {
495
- const abs = path.resolve(workspaceRoot, rel)
496
- if (abs.startsWith(key + path.sep)) {
497
- under.push(path.relative(key, abs).split(path.sep).join('/'))
498
- }
499
- }
500
- if (under.length > 0) this.recordChanged(key, under)
501
- }
502
- }
503
-
504
- /**
505
- * Drop the workspace-wide partition (if one exists). Called after a
506
- * cache-miss save — an executed task may have written undeclared
507
- * files anywhere in its project dir, and the workspace partition
508
- * spans that subtree; only a fresh enumeration can see them. No-op
509
- * when the feature is unused.
510
- */
511
- invalidateWorkspacePartition(): void {
512
- if (this.wsRoot !== null) this.delete(this.wsRoot)
513
- }
514
-
515
- private recordChanged(partitionDir: string, relPaths: readonly string[]): void {
516
- if (!this.has(partitionDir)) return
517
- const cur = this.changed.get(partitionDir)
518
- if (cur) cur.push(...relPaths)
519
- else this.changed.set(partitionDir, [...relPaths])
520
- // These paths just changed on disk; their index OIDs (if any) no
521
- // longer describe the working-tree content.
522
- const partitionOids = this.oids.get(partitionDir)
523
- if (partitionOids) {
524
- for (const rel of relPaths) partitionOids.delete(path.resolve(partitionDir, rel))
525
- }
526
- }
527
-
528
- /** Trusted index OIDs for a project (abs path → oid), if any survive. */
529
- oidsFor(projectDir: string): ReadonlyMap<string, string> | undefined {
530
- return this.oids.get(projectDir)
531
- }
517
+ function isLiteralPath(glob: string): boolean {
518
+ return !/[*?[\]{}]/.test(glob)
519
+ }
532
520
 
533
- setOids(projectDir: string, oids: Map<string, string>): void {
534
- this.oids.set(projectDir, oids)
535
- }
521
+ function stripTrailingSlash(p: string): string {
522
+ return p.replace(/\/+$/, '')
523
+ }
536
524
 
537
- /** Snapshot if still valid for these input globs; undefined → re-spawn. */
538
- snapshotFor(projectDir: string, inputGlobs: readonly Bun.Glob[]): readonly string[] | undefined {
539
- const snap = this.get(projectDir)
540
- if (snap === undefined) return undefined
541
- const pending = this.changed.get(projectDir)
542
- if (pending !== undefined && pending.some((p) => inputGlobs.some((g) => g.match(p)))) {
543
- return undefined
525
+ /**
526
+ * A literal entry names a file OR a directory tree: `src/` and `src` both
527
+ * mean everything under `src`, as they do in Turbo and every `.gitignore`.
528
+ * A glob matcher sees only the literal path, so `['src/']` folded ZERO
529
+ * files — a key that never moves with its source, and the most common
530
+ * turbo.json shape (`"outputs": ["dist"]`) captured nothing. Every
531
+ * literal therefore compiles to itself plus its subtree; a literal that
532
+ * names a file still matches exactly that file, since `x/**` matches
533
+ * nothing under a file.
534
+ */
535
+ export function asTrees(patterns: readonly string[]): string[] {
536
+ const out: string[] = []
537
+ for (const raw of patterns) {
538
+ const p = normalizeGlob(raw)
539
+ if (!isLiteralPath(p)) {
540
+ out.push(p)
541
+ continue
544
542
  }
545
- return snap
543
+ const lit = stripTrailingSlash(p)
544
+ if (lit.length === 0) continue
545
+ out.push(lit, `${lit}/**`)
546
546
  }
547
-
548
- override set(key: string, value: readonly string[]): this {
549
- this.changed.delete(key)
550
- this.oids.delete(key)
551
- return super.set(key, value)
552
- }
553
-
554
- override delete(key: string): boolean {
555
- this.changed.delete(key)
556
- this.oids.delete(key)
557
- return super.delete(key)
558
- }
559
- }
560
-
561
- interface ResolveFilesArgs {
562
- projectDir: string
563
- workspaceRoot: string
564
- files: string[] | undefined
565
- ownOutputs: string[]
566
- nestedProjectDirs: string[]
567
- gitFilesCache?: GitFilesCache
547
+ return out
568
548
  }
569
549
 
570
- /** No glob metacharacter — the entry names one exact path. */
571
- function isLiteralPath(glob: string): boolean {
572
- return !/[*?[\]{}]/.test(glob)
550
+ /** A literal is answered by the path itself or by anything under it. */
551
+ function settleLiterals(unmatched: Set<string>, rel: string): void {
552
+ for (const lit of unmatched) {
553
+ if (rel === lit || rel.startsWith(`${lit}/`)) unmatched.delete(lit)
554
+ }
573
555
  }
574
556
 
575
557
  /**
@@ -612,6 +594,15 @@ async function assertNoInvisibleLiteralInputs(
612
594
  }
613
595
  }
614
596
 
597
+ interface ResolveFilesArgs {
598
+ projectDir: string
599
+ workspaceRoot: string
600
+ files: string[] | undefined
601
+ ownOutputs: string[]
602
+ nestedProjectDirs: string[]
603
+ gitFilesCache?: GitFilesCache
604
+ }
605
+
615
606
  async function resolveFiles(args: ResolveFilesArgs): Promise<string[]> {
616
607
  const positive: string[] = []
617
608
  const negative: string[] = []
@@ -628,9 +619,12 @@ async function resolveFiles(args: ResolveFilesArgs): Promise<string[]> {
628
619
  if (positive.length === 0) return []
629
620
 
630
621
  const boundaryIgnores = boundaryIgnorePatterns(args.projectDir, args.nestedProjectDirs)
631
- const excludeGlobs = [...ALWAYS_IGNORE, ...boundaryIgnores, ...args.ownOutputs, ...negative].map(
632
- globFor,
633
- )
622
+ const excludeGlobs = [
623
+ ...ALWAYS_IGNORE,
624
+ ...boundaryIgnores,
625
+ ...asTrees(args.ownOutputs),
626
+ ...asTrees(negative),
627
+ ].map(globFor)
634
628
 
635
629
  // Defer to git for the file set (Turbo / Nx parity). Nested .gitignore
636
630
  // files, .git/info/exclude, and global excludes all participate
@@ -640,7 +634,7 @@ async function resolveFiles(args: ResolveFilesArgs): Promise<string[]> {
640
634
  // per task (build + test + lint + …). Spawning git N times for the
641
635
  // same project per run is wasteful; we cache the result for the
642
636
  // duration of one orchestrator run.
643
- const positiveGlobs = positive.map(globFor)
637
+ const positiveGlobs = asTrees(positive).map(globFor)
644
638
  let gitFiles = args.gitFilesCache?.snapshotFor(args.projectDir, positiveGlobs)
645
639
  if (gitFiles === undefined) {
646
640
  // Mid-run re-enumeration. The OIDs this spawn could yield are NOT
@@ -663,11 +657,13 @@ async function resolveFiles(args: ResolveFilesArgs): Promise<string[]> {
663
657
  // an artifact built from an older version of a file the config explicitly
664
658
  // claims as an input. See the refusal below for why this is not simply
665
659
  // honoured instead.
666
- const unmatchedLiterals = new Set(positive.filter(isLiteralPath))
660
+ const unmatchedLiterals = new Set(
661
+ positive.map(normalizeGlob).filter(isLiteralPath).map(stripTrailingSlash),
662
+ )
667
663
  // First pass: glob-filter to candidate absolute paths (no I/O).
668
664
  const candidates: string[] = []
669
665
  for (const rel of gitFiles) {
670
- if (unmatchedLiterals.size > 0) unmatchedLiterals.delete(rel)
666
+ if (unmatchedLiterals.size > 0) settleLiterals(unmatchedLiterals, rel)
671
667
  let matched = false
672
668
  for (const g of positiveGlobs) {
673
669
  if (g.match(rel)) {
@@ -682,542 +678,59 @@ async function resolveFiles(args: ResolveFilesArgs): Promise<string[]> {
682
678
  if (unmatchedLiterals.size > 0) {
683
679
  await assertNoInvisibleLiteralInputs(unmatchedLiterals, args.projectDir, 'files')
684
680
  }
685
- // Second pass: parallel existence check — but ONLY for paths
686
- // without a trusted index OID. A clean-per-status tracked file
687
- // necessarily exists on disk, so skipping its probe keeps the warm
688
- // path free of per-file syscalls. Paths without an OID keep the
689
- // probe: `git ls-files -s` can surface staged entries whose
690
- // working-tree file is gone; the hasher would otherwise throw
691
- // ENOENT.
681
+ // Second pass: existence check — but ONLY for paths without a
682
+ // trusted index OID. A clean-per-status tracked file necessarily
683
+ // exists on disk, so skipping its probe keeps the warm path free of
684
+ // per-file syscalls. Paths without an OID keep the probe:
685
+ // `git ls-files -s` can surface staged entries whose working-tree
686
+ // file is gone; the hasher would otherwise throw ENOENT.
692
687
  const oids = args.gitFilesCache?.oidsFor(args.projectDir)
693
- const exists = await Promise.all(
694
- candidates.map((abs) => oids?.has(abs) === true || Bun.file(abs).exists()),
695
- )
696
- const matches: string[] = []
697
- for (let i = 0; i < candidates.length; i++) {
698
- if (exists[i]) matches.push(candidates[i]!)
699
- }
700
- return matches.sort()
701
- }
702
-
703
- interface GitLsResult {
704
- /** cwd-relative paths — same visibility set as `--cached --others`. */
705
- files: string[]
706
- /**
707
- * cwd-relative path → index blob OID, for tracked REGULAR files
708
- * (mode 100644 / 100755) at stage 0 only. Symlinks are excluded
709
- * (their OID hashes the link-target string, not the dereferenced
710
- * content our fallback hasher would read); merge-conflict stages
711
- * and gitlinks are excluded too. NOT yet filtered by working-tree
712
- * dirtiness — callers intersect with `git status` before trusting.
713
- */
714
- oids: Map<string, string>
715
- /** Paths flagged skip-worktree / assume-unchanged (only when `-v` was passed). */
716
- flagged: Set<string>
717
- }
718
-
719
- // `<mode> <oid> <stage>\t<path>` — the staged-entry form of
720
- // `ls-files -s`. `--others` paths print bare; with `-z`,
721
- // core.quotePath quoting is off, so a bare path containing a literal
722
- // tab still can't match this fixed-form prefix.
723
- // `ls-files -s -v`: an optional cache-state flag (`H`, `S`, `h`, …) then the
724
- // stage record. Both answers come from ONE spawn.
725
- const LS_FILES_STAGE_RE = /^(?:([A-Za-z]) )?([0-7]{6}) ([0-9a-f]{40,64}) ([0-3])\t/
726
-
727
- /**
728
- * Run `git ls-files -s --others --exclude-standard -z .` in `cwd`.
729
- * One spawn yields BOTH the file list (identical visibility to the
730
- * pre-v20 `--cached --others` form, verified empirically: same set
731
- * including staged-but-deleted files and per-stage conflict
732
- * duplicates) AND each tracked file's index OID — the heart of the
733
- * Turbo-parity "hashes come from git's index" fast path. Throws a
734
- * `UserError` when git is unavailable or `cwd` isn't a git work
735
- * tree; vx requires git. `-z` survives filenames with newlines /
736
- * spaces.
737
- */
738
- function runGitLsFiles(cwd: string): GitLsResult {
739
- let proc
740
- try {
741
- proc = Bun.spawnSync({
742
- cmd: ['git', 'ls-files', '-s', '--others', '--exclude-standard', '-z', '.'],
743
- cwd,
744
- stdout: 'pipe',
745
- stderr: 'pipe',
746
- })
747
- } catch {
748
- throw new UserError(
749
- `vx requires git: failed to spawn 'git' (working dir: ${cwd}). Install git and re-run.`,
750
- )
751
- }
752
- if (proc.exitCode !== 0) {
753
- // Exit 128 = not a git work tree; other non-zero = git failure.
754
- // Either way we can't enumerate inputs reliably.
755
- const stderr = new TextDecoder().decode(proc.stderr).trim()
756
- throw new UserError(
757
- `vx requires git: ${cwd} is not inside a git work tree. ` +
758
- `Run 'git init' in your workspace root.${stderr ? ` (git: ${stderr})` : ''}`,
759
- )
760
- }
761
- return parseLsFilesOutput(new TextDecoder().decode(proc.stdout))
762
- }
763
-
764
- function parseLsFilesOutput(out: string): GitLsResult {
765
- const files: string[] = []
766
- const oids = new Map<string, string>()
767
- const flagged = new Set<string>()
768
- if (out.length === 0) return { files, oids, flagged }
769
- // NUL-separated; trailing NUL produces an empty segment we skip.
770
- for (const record of out.split('\0')) {
771
- if (record.length === 0) continue
772
- const m = LS_FILES_STAGE_RE.exec(record)
773
- if (m === null) {
774
- files.push(record) // --others entry: bare path
775
- continue
776
- }
777
- const filePath = record.slice(m[0].length)
778
- files.push(filePath)
779
- const mode = m[2]!
780
- const stage = m[4]!
781
- if ((mode === '100644' || mode === '100755') && stage === '0') {
782
- oids.set(filePath, m[3]!)
783
- }
784
- // A LOWERCASE letter means skip-worktree or assume-unchanged (`S` is the
785
- // older spelling of skip-worktree): git has been told to stop looking at
786
- // the worktree file, so its index OID says nothing about the disk.
787
- const flag = m[1]
788
- if (flag !== undefined && (flag === 'S' || (flag >= 'a' && flag <= 'z'))) flagged.add(filePath)
789
- }
790
- return { files, oids, flagged }
791
- }
792
-
793
- /** One completed `git` invocation. */
794
- interface GitRun {
795
- exitCode: number
796
- stdout: string
797
- stderr: string
798
- }
799
-
800
- /**
801
- * Parse `git check-attr -z text eol ident` output — a flat stream of
802
- * `<path>\0<attr>\0<value>\0` triples — into the set of paths where a clean
803
- * filter can rewrite bytes. `unspecified` means no rule matched and
804
- * `unset` (`-text`) explicitly disables conversion; both leave the index blob
805
- * byte-identical to the worktree file, so those OIDs stay trusted.
806
- */
807
- export function parseCheckAttrOutput(out: string): Set<string> {
808
- const affected = new Set<string>()
809
- const fields = out.split('\0')
810
- for (let i = 0; i + 2 < fields.length; i += 3) {
811
- const value = fields[i + 2]!
812
- if (value !== 'unspecified' && value !== 'unset') affected.add(fields[i]!)
813
- }
814
- return affected
815
- }
816
-
817
- /**
818
- * Remove from `trusted` every path whose index blob may differ from its
819
- * worktree bytes because a clean filter applies.
820
- *
821
- * Gated in two steps so the common case pays NOTHING. The precise answer —
822
- * `git ls-files --eol`, comparing `i/` to `w/` — was measured at 240 ms on a
823
- * 15k-file tree because it must READ every worktree file: 13x the entire
824
- * enumeration, on a run whose warm total is ~130 ms, and in the common Linux
825
- * case it finds nothing. So instead:
826
- *
827
- * 1. If `core.autocrlf` is true/input, conversion applies to every
828
- * auto-detected text file with no attribute needed — trust nothing.
829
- * 2. Else, if no attributes source exists anywhere (no in-tree
830
- * `.gitattributes`, no `$GIT_DIR/info/attributes`, no
831
- * `core.attributesFile`), no rule can name a filter — return untouched,
832
- * zero extra work. This is the default `git init` repo.
833
- * 3. Otherwise ask `git check-attr` (measured 21 ms; it resolves attributes
834
- * from the index WITHOUT reading worktree content) and drop only the
835
- * paths that actually carry `text`/`eol`/`ident`.
836
- *
837
- * Never throws: a probe that fails leaves the map as-is, which is exactly the
838
- * behaviour before this gate existed.
839
- */
840
- async function dropFilteredOids(
841
- trusted: Map<string, string>,
842
- args: {
843
- workspaceRoot: string
844
- gitDir: string
845
- coreConfig: string
846
- spawnGit: (a: string[], stdin?: string) => Promise<GitRun | null>
847
- },
848
- ): Promise<void> {
849
- if (trusted.size === 0) return
850
- if (autocrlfConverts(args.coreConfig)) {
851
- trusted.clear()
852
- return
853
- }
854
- // Cheap checks first, and nothing is materialized until one of them fires —
855
- // on a 15k-file repo with no attributes this whole function is a size check,
856
- // a substring scan of an empty string, one stat, and a key walk that exits
857
- // on the first `.gitattributes` it does not find.
858
- let attributesPossible =
859
- args.coreConfig.toLowerCase().includes('core.attributesfile') ||
860
- (args.gitDir !== '' &&
861
- existsSync(path.resolve(args.workspaceRoot, args.gitDir, 'info', 'attributes')))
862
- if (!attributesPossible) {
863
- for (const rel of trusted.keys()) {
864
- if (rel === '.gitattributes' || rel.endsWith('/.gitattributes')) {
865
- attributesPossible = true
866
- break
867
- }
868
- }
869
- }
870
- if (!attributesPossible) return
871
-
872
- const res = await args.spawnGit(
873
- ['check-attr', '--stdin', '-z', 'text', 'eol', 'ident'],
874
- [...trusted.keys()].join('\0'),
875
- )
876
- if (res === null || res.exitCode !== 0) return
877
- for (const rel of parseCheckAttrOutput(res.stdout)) trusted.delete(rel)
878
- }
879
-
880
- /** `true` when git may rewrite bytes for EVERY auto-detected text file. */
881
- export function autocrlfConverts(coreConfig: string): boolean {
882
- for (const line of coreConfig.split('\n')) {
883
- const [name, value] = [line.slice(0, line.indexOf(' ')), line.slice(line.indexOf(' ') + 1)]
884
- if (name.toLowerCase() === 'core.autocrlf') {
885
- const v = value.trim().toLowerCase()
886
- return v === 'true' || v === 'input'
887
- }
888
- }
889
- return false
890
- }
891
-
892
- /**
893
- * Paths `git ls-files -v -z` marks as not-watched: `S` is skip-worktree, and
894
- * ANY lowercase letter is assume-unchanged layered on that entry's state.
895
- * Either way git has been told to stop comparing the worktree file, so its
896
- * index OID says nothing about what is — or isn't — on disk. Each
897
- * NUL-terminated record is `<letter><space><path>`.
898
- */
899
- export function parseFlaggedOutput(out: string): Set<string> {
900
- const flagged = new Set<string>()
901
- for (const record of out.split('\0')) {
902
- if (record.length < 3 || record[1] !== ' ') continue
903
- const letter = record[0]!
904
- if (letter === 'S' || (letter >= 'a' && letter <= 'z')) flagged.add(record.slice(2))
905
- }
906
- return flagged
907
- }
908
-
909
- function parseStatusOutput(out: string): { dirty: Set<string>; untracked: string[] } {
910
- const tokens = out.split('\0')
911
- const dirty = new Set<string>()
912
- const untracked: string[] = []
913
- for (let i = 0; i < tokens.length; i++) {
914
- const token = tokens[i]!
915
- if (token.length < 4) continue
916
- if (token[0] === '?') {
917
- untracked.push(token.slice(3))
918
- continue
919
- }
920
- dirty.add(token.slice(3))
921
- // X = R or C → the next token is the rename/copy source path.
922
- const x = token[0]
923
- if ((x === 'R' || x === 'C') && i + 1 < tokens.length && tokens[i + 1]!.length > 0) {
924
- i++
925
- dirty.add(tokens[i]!)
926
- }
927
- }
928
- return { dirty, untracked }
929
- }
930
-
931
- /**
932
- * Strip a repo→workspace `--show-prefix` (e.g. `code/`) from each path, keeping
933
- * only paths inside the workspace — so a repo-root-relative set (from `git
934
- * status`/`diff`) is re-keyed workspace-relative to match `git ls-files`. Empty
935
- * prefix → returns the set unchanged (workspace root IS the git root).
936
- */
937
- function stripPrefixFromSet(paths: Set<string>, prefix: string): Set<string> {
938
- if (prefix === '') return paths
939
- const out = new Set<string>()
940
- for (const p of paths) if (p.startsWith(prefix)) out.add(p.slice(prefix.length))
941
- return out
942
- }
943
-
944
- /**
945
- * Run `git ls-files` ONCE at the workspace root, then partition the
946
- * result by project. Populates `cache` for every project in
947
- * `projectDirs` with project-relative path lists matching what a
948
- * per-project spawn would have produced. Throws `UserError` if the
949
- * workspace isn't a git work tree (vx requires git).
950
- *
951
- * Why bulk: each spawn costs ~5-10ms (fork+exec). On a 200-project
952
- * workspace that's 1-2s of pure overhead reclaimed.
953
- *
954
- * Files in nested-project subtrees stay in their parent's list — the
955
- * boundary-ignore globs in `resolveFiles` filter them out the same way
956
- * they did before. Cheaper to filter once-per-task than to subtract
957
- * here.
958
- *
959
- * v20: the same `ls-files -s` spawn also yields each tracked file's
960
- * index OID. A second spawn (`git status --porcelain`) prunes paths
961
- * whose working tree diverges from the index; what survives is
962
- * stored per project via `cache.setOids` and feeds `Cache.key`
963
- * directly — clean-tree input hashing costs zero reads/stats/SQLite.
964
- *
965
- * `workspaceWide`: set when any loaded task declares
966
- * `cache.inputs.workspaceFiles`. Disables pathspec scoping (those
967
- * globs must see every file from the root) and additionally stores a
968
- * workspace-wide partition keyed by `workspaceRoot` (files + trusted
969
- * OIDs), which `resolveWorkspaceFiles` consumes. When false, the
970
- * enumeration behavior — pathspecs, spawn count, stored partitions —
971
- * is byte-identical to the pre-workspaceFiles code.
972
- */
973
- export async function populateGitFilesCache(
974
- workspaceRoot: string,
975
- projectDirs: readonly string[],
976
- cache: GitFilesCache,
977
- workspaceWide = false,
978
- ): Promise<void> {
979
- const enumeration = await startGitEnumeration(
980
- workspaceRoot,
981
- gitPathspecs(workspaceRoot, projectDirs, workspaceWide),
982
- )
983
- applyGitEnumeration(enumeration, workspaceRoot, projectDirs, cache, workspaceWide)
984
- }
985
-
986
- /** What one workspace-wide enumeration learned, before it is partitioned per project. */
987
- export interface GitEnumeration {
988
- /** Every enumerated path, workspace-relative, tracked and untracked. */
989
- all: string[]
990
- /** Workspace-relative path → index OID, for files whose worktree state matched the index. */
991
- trusted: Map<string, string>
992
- /** Whether the worktree had uncommitted changes; null when `git status` failed. */
993
- dirty: boolean | null
994
- }
995
-
996
- /**
997
- * Pathspec scoping: when the run only needs a handful of projects
998
- * (scoped config loading), let git scan just those dirs — 75 ms →
999
- * 11 ms on an 11k-file repo. Above 64 dirs (or when a project IS
1000
- * the root) the whole-tree scan wins on arg/exec overhead anyway.
1001
- */
1002
- export function gitPathspecs(
1003
- workspaceRoot: string,
1004
- projectDirs: readonly string[],
1005
- workspaceWide: boolean,
1006
- ): string[] {
1007
- const rels = projectDirs.map((d) => path.relative(workspaceRoot, d).split(path.sep).join('/'))
1008
- const scoped =
1009
- !workspaceWide &&
1010
- rels.length > 0 &&
1011
- rels.length <= 64 &&
1012
- rels.every((r) => r !== '' && r !== '.')
1013
- return scoped ? rels : ['.']
1014
- }
1015
-
1016
- /**
1017
- * The spawn half of `populateGitFilesCache`, separated so an UNSCOPED run
1018
- * can start it before the configs are evaluated — the enumeration needs
1019
- * only the pathspecs, and `['.']` is right for every unscoped run — and
1020
- * overlap ~60 ms of git with ~80 ms of config evaluation on a
1021
- * 1000-project tree instead of paying them back to back.
1022
- */
1023
- export async function startGitEnumeration(
1024
- workspaceRoot: string,
1025
- pathspecs: readonly string[],
1026
- ): Promise<GitEnumeration> {
1027
- // The spawns are independent — run them concurrently so the
1028
- // bulk-populate costs max(ls-files, status) wall time, not the sum
1029
- // (status alone is ~74 ms on a 1000-project tree; serial spawning
1030
- // was a measurable warm-path regression vs the pre-OID code).
1031
- const spawnGit = async (args: string[], stdin?: string): Promise<GitRun | null> => {
1032
- try {
1033
- const proc = Bun.spawn({
1034
- cmd: ['git', ...args],
1035
- cwd: workspaceRoot,
1036
- stdin: stdin === undefined ? 'ignore' : new TextEncoder().encode(stdin),
1037
- stdout: 'pipe',
1038
- stderr: 'pipe',
1039
- })
1040
- const [stdout, stderr, exitCode] = await Promise.all([
1041
- new Response(proc.stdout).text(),
1042
- new Response(proc.stderr).text(),
1043
- proc.exited,
1044
- ])
1045
- return { exitCode, stdout, stderr }
1046
- } catch {
1047
- return null
1048
- }
1049
- }
1050
- // Four spawns, one worktree walk. `ls-files -s -v` reads the INDEX only
1051
- // (~9 ms on a 1000-project tree) and answers two questions at once: every
1052
- // tracked path's OID and its cache-state flag. `status -uall` is the one
1053
- // command that walks the worktree, and it answers two as well: which
1054
- // tracked paths are dirty AND which files are untracked. Asking
1055
- // `ls-files --others` for the untracked set walked the same tree a second
1056
- // time (~50 ms of CPU, concurrent with status but contending with it).
1057
- const [ls, status, prefixRes, coreCfg] = await Promise.all([
1058
- spawnGit(['ls-files', '-s', '-v', '-z', '--', ...pathspecs]),
1059
- spawnGit(['status', '--porcelain', '-z', '-uall', '--', ...pathspecs]),
1060
- // Two answers from one spawn. `--show-prefix` is the repo→workspace path
1061
- // (empty when the workspace root IS the git root): `ls-files` prints
1062
- // cwd(workspace)-relative paths but `status` prints repo-root-relative
1063
- // ones, so when the workspace root is a SUBDIR of the git repo the two
1064
- // disagree; this lets us key both the same way below. `--git-dir` locates
1065
- // `info/attributes` for the filter gate (it is not always `.git/` — a
1066
- // linked worktree's `.git` is a FILE pointing elsewhere).
1067
- spawnGit(['rev-parse', '--show-prefix', '--git-dir']),
1068
- // Reads three config keys, no tree scan — the gate for whether a clean
1069
- // filter can rewrite bytes between the index and the worktree. Exits 1
1070
- // when none are set, which is the common case and means "no gate".
1071
- spawnGit(['config', '--get-regexp', '^core\\.(autocrlf|eol|attributesfile)$']),
1072
- ])
1073
- if (ls === null) {
1074
- throw new UserError(
1075
- `vx requires git: failed to spawn 'git' (working dir: ${workspaceRoot}). Install git and re-run.`,
1076
- )
1077
- }
1078
- if (ls.exitCode !== 0) {
1079
- const stderr = ls.stderr.trim()
1080
- throw new UserError(
1081
- `vx requires git: ${workspaceRoot} is not inside a git work tree. ` +
1082
- `Run 'git init' in your workspace root.${stderr ? ` (git: ${stderr})` : ''}`,
1083
- )
1084
- }
1085
- const { files: tracked, oids, flagged } = parseLsFilesOutput(ls.stdout)
1086
- // Normalize `status`'s repo-root-relative paths to workspace-relative (strip
1087
- // the `--show-prefix`) so the dirty set is keyed identically to the trusted
1088
- // OID map. Without this, when the workspace root is a git subdir, a modified
1089
- // tracked file is never pruned from `trusted` and keeps its committed OID —
1090
- // a STALE cache hit serving old outputs. Empty prefix (workspace == git root,
1091
- // the common case) is a zero-cost no-op. Paths above the workspace can't be
1092
- // inputs, so they drop out of the set.
1093
- // One spawn, two lines: `--show-prefix` then `--git-dir`.
1094
- const revLines =
1095
- prefixRes !== null && prefixRes.exitCode === 0 ? prefixRes.stdout.split('\n') : []
1096
- const gitPrefix = (revLines[0] ?? '').trim()
1097
- const gitDir = (revLines[1] ?? '').trim()
1098
- const parsedStatus =
1099
- status !== null && status.exitCode === 0 ? parseStatusOutput(status.stdout) : null
1100
- const dirty = parsedStatus === null ? null : stripPrefixFromSet(parsedStatus.dirty, gitPrefix)
1101
- // Untracked files are inputs too (a new file is the commonest edit there
1102
- // is). They come from the status walk, repo-root-relative like the dirty
1103
- // set. Without a status answer the enumeration is the index alone.
1104
- const untracked =
1105
- parsedStatus === null ? [] : [...stripPrefixFromSet(new Set(parsedStatus.untracked), gitPrefix)]
1106
- const all = untracked.length === 0 ? tracked : tracked.concat(untracked)
1107
- // Aggregate dirtiness for the Tier-3 invocation record — derived from
1108
- // this same status spawn so `run()` needs no second `git status`.
1109
- // null when the status spawn failed (non-repo / git error).
1110
- const worktreeDirty = dirty === null ? null : dirty.size > 0
1111
- const trusted = dirty === null ? new Map<string, string>() : oids
1112
- if (dirty !== null) {
1113
- for (const rel of dirty) trusted.delete(rel)
1114
- }
1115
- // A skip-worktree / assume-unchanged path sits at stage 0 and `git status`
1116
- // reports nothing for it, so it would otherwise keep a trusted OID — and
1117
- // resolveFiles SKIPS its existence probe for OID-carrying paths. A sparse
1118
- // checkout would then count a file that is not on disk as an input, so
1119
- // materializing it later changes no key and the old artifact is replayed.
1120
- // Dropping the OID sends these back through the probe, where they correctly
1121
- // fall out of the input set while unmaterialized.
1122
- for (const rel of flagged) trusted.delete(rel)
1123
- // An index OID is only the file's content hash when git stores the worktree
1124
- // bytes VERBATIM. Under a clean filter (`text`/`eol`/`ident`) the blob is a
1125
- // DIFFERENT sequence of bytes — the LF-normalized form — while the task
1126
- // reads the CRLF worktree file. `git status` compares AFTER filtering, so
1127
- // such a file reports clean and keeps its OID: the CRLF and LF states then
1128
- // fold the SAME key and a real content change is invisible.
1129
- //
1130
- // Dropping an OID is not over-invalidation. It routes the path to
1131
- // `hashFile`, which hashes the worktree bytes — the source that was correct
1132
- // all along. The only cost is the read, so the gate below is about paying it
1133
- // ONLY where a filter can actually apply.
1134
- await dropFilteredOids(trusted, {
1135
- workspaceRoot,
1136
- gitDir,
1137
- coreConfig: coreCfg !== null && coreCfg.exitCode === 0 ? coreCfg.stdout : '',
1138
- spawnGit,
1139
- })
1140
- return { all, trusted, dirty: worktreeDirty }
1141
- }
1142
-
1143
- /** The partition half of `populateGitFilesCache`: store per-project slices of one enumeration. */
1144
- export function applyGitEnumeration(
1145
- enumeration: GitEnumeration,
1146
- workspaceRoot: string,
1147
- projectDirs: readonly string[],
1148
- cache: GitFilesCache,
1149
- workspaceWide = false,
1150
- ): void {
1151
- const { all, trusted } = enumeration
1152
- cache.setWorktreeDirty(enumeration.dirty)
1153
- // Sort once, then each project's files are a contiguous range found
1154
- // by binary search on its `dir/` prefix — O((F+P) log F) instead of
1155
- // the O(P·F) per-project startsWith scan (54 ms at 1090 projects ×
1156
- // ~9k files; ~5 ms this way). '/' sorts below most filename chars,
1157
- // so the range [prefix, prefix+'\xff…') is contiguous in the sorted
1158
- // array; lowerBound on `prefix` and on `prefix + '￿'` bracket it.
1159
- const sorted = [...all].sort()
1160
- const lowerBound = (key: string): number => {
1161
- let lo = 0
1162
- let hi = sorted.length
1163
- while (lo < hi) {
1164
- const mid = (lo + hi) >>> 1
1165
- if (sorted[mid]! < key) lo = mid + 1
1166
- else hi = mid
1167
- }
1168
- return lo
1169
- }
1170
- for (const projectDir of projectDirs) {
1171
- const relPrefix = path.relative(workspaceRoot, projectDir).split(path.sep).join('/')
1172
- if (relPrefix === '' || relPrefix === '.') {
1173
- cache.set(projectDir, all)
1174
- const rootOids = new Map<string, string>()
1175
- for (const [rel, oid] of trusted) rootOids.set(path.join(workspaceRoot, rel), oid)
1176
- cache.setOids(projectDir, rootOids)
1177
- continue
1178
- }
1179
- const prefix = `${relPrefix}/`
1180
- const start = lowerBound(prefix)
1181
- const end = lowerBound(`${prefix}￿`)
1182
- const matches: string[] = []
1183
- const projOids = new Map<string, string>()
1184
- for (let i = start; i < end; i++) {
1185
- const rel = sorted[i]!
1186
- matches.push(rel.slice(prefix.length))
1187
- const oid = trusted.get(rel)
1188
- if (oid !== undefined) projOids.set(path.join(workspaceRoot, rel), oid)
1189
- }
1190
- // setOids AFTER set — set() drops the project's OID slot.
1191
- cache.set(projectDir, matches)
1192
- cache.setOids(projectDir, projOids)
1193
- }
1194
- if (workspaceWide) {
1195
- const rootOids = new Map<string, string>()
1196
- for (const [rel, oid] of trusted) rootOids.set(path.join(workspaceRoot, rel), oid)
1197
- cache.set(workspaceRoot, all)
1198
- cache.setOids(workspaceRoot, rootOids)
1199
- cache.setWorkspaceRoot(workspaceRoot)
1200
- }
688
+ return candidates.filter((abs) => oids?.has(abs) === true || isInputOnDisk(abs)).sort()
1201
689
  }
1202
690
 
1203
691
  /**
1204
692
  * Union of files matching any positive pattern in `cwd`, minus files
1205
693
  * matching any exclude glob (tested by Bun.Glob.match on the relative
1206
694
  * path). Bun.Glob takes a single pattern per instance, so we iterate.
695
+ *
696
+ * `outputs` mode also yields SYMLINKS (never followed, never descended):
697
+ * a task that emits `dist/out -> ../src/x` has produced an output, so the
698
+ * save must capture it (as its target's bytes, `planArtifact`) and the
699
+ * clean must remove it — a link the clean leaves standing blocks the
700
+ * directory a later entry restores there. Bun's `onlyFiles` walk drops
701
+ * every symlink, so this mode lists everything and sorts by `lstat`.
1207
702
  */
1208
703
  async function scanUnion(
1209
704
  positive: readonly string[],
1210
705
  excludeGlobs: readonly Bun.Glob[],
1211
706
  cwd: string,
707
+ mode: 'files' | 'outputs' = 'files',
1212
708
  ): Promise<Set<string>> {
1213
709
  const matches = new Set<string>()
1214
710
  for (const pattern of positive) {
1215
711
  const glob = globFor(pattern)
1216
- // Async scan, deliberately: the sync walk is faster in isolation (37 µs
1217
- // vs 56 µs on a one-file `dist/**`) but SERIALISES on the main thread,
1218
- // and this runs under the scheduler's concurrency — measured 2026-09-02,
1219
- // switching it to scanSync made a 1000-hit warm run 40 ms SLOWER.
1220
- for await (const rel of glob.scan({ cwd, onlyFiles: true, dot: true })) {
712
+ if (mode === 'outputs') {
713
+ for (const rel of glob.scanSync({
714
+ cwd,
715
+ onlyFiles: false,
716
+ followSymlinks: false,
717
+ dot: true,
718
+ })) {
719
+ if (excludeGlobs.some((g) => g.match(rel))) continue
720
+ const abs = path.resolve(cwd, rel)
721
+ const st = lstatSync(abs, { throwIfNoEntry: false })
722
+ if (st !== undefined && (st.isFile() || st.isSymbolicLink())) matches.add(abs)
723
+ }
724
+ continue
725
+ }
726
+ // Sync scan. The async walk was chosen on 2026-09-02, when a warm HIT
727
+ // globbed its outputs and the thread pool overlapped a thousand of
728
+ // them; the directory short-circuit took the glob off the hit path, and
729
+ // what is left runs on the MISS path twice per task (clean, then
730
+ // resolve), where the walk is CPU-bound and each async chunk costs a
731
+ // main-thread round trip. Measured 2026-09-09 on a 1,000-task cold run
732
+ // (see STATUS).
733
+ for (const rel of glob.scanSync({ cwd, onlyFiles: true, dot: true })) {
1221
734
  if (excludeGlobs.some((g) => g.match(rel))) continue
1222
735
  matches.add(path.resolve(cwd, rel))
1223
736
  }