@vzn/vx 0.0.349 → 0.0.352

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@vzn/vx",
3
- "version": "0.0.349",
3
+ "version": "0.0.352",
4
4
  "description": "A fast, correct task runner and build cache for JavaScript monorepos.",
5
5
  "type": "module",
6
6
  "exports": {
@@ -21,10 +21,10 @@
21
21
  "node": ">=18"
22
22
  },
23
23
  "optionalDependencies": {
24
- "@vzn/vx-linux-x64": "0.0.349",
25
- "@vzn/vx-linux-arm64": "0.0.349",
26
- "@vzn/vx-darwin-x64": "0.0.349",
27
- "@vzn/vx-darwin-arm64": "0.0.349"
24
+ "@vzn/vx-linux-x64": "0.0.352",
25
+ "@vzn/vx-linux-arm64": "0.0.352",
26
+ "@vzn/vx-darwin-x64": "0.0.352",
27
+ "@vzn/vx-darwin-arm64": "0.0.352"
28
28
  },
29
29
  "dependencies": {
30
30
  "@anthropic-ai/sandbox-runtime": "^0.0.76"
@@ -605,6 +605,14 @@ export class Cache implements CacheLayer {
605
605
  // every open there failed `attempt to write a readonly database`
606
606
  // before the not-writable refusal could name the directory (O-10).
607
607
  if (!absent) this.db.fileControl(SQLITE_FCNTL_PERSIST_WAL, 1)
608
+ // Any non-negative limit makes the last clean close truncate a
609
+ // persistent `-wal` to zero bytes once it is checkpointed (SQLite's
610
+ // walClose). At the default (-1) it kept every frame, so each later
611
+ // connection recovered them at open and copied them into the
612
+ // database again at its close: 738 frames, 4.5 MB, on a 1,000-project
613
+ // cache. The file stays (O-10 above); 64 MiB is far above what a run
614
+ // writes, so the limit never trims a WAL mid-run.
615
+ this.db.exec('PRAGMA journal_size_limit = 67108864')
608
616
  this.db.exec('PRAGMA synchronous = NORMAL')
609
617
  this.db.exec('PRAGMA foreign_keys = ON')
610
618
  this.db.exec(`
@@ -666,7 +674,7 @@ export class Cache implements CacheLayer {
666
674
  }
667
675
  if (found === SCHEMA_VERSION) return null
668
676
  this.db.exec(
669
- 'DROP TABLE IF EXISTS entries; DROP TABLE IF EXISTS runs; DROP TABLE IF EXISTS file_hashes; DROP TABLE IF EXISTS output_files; DROP TABLE IF EXISTS invocations; DROP TABLE IF EXISTS run_task_inputs; DROP TABLE IF EXISTS entry_inputs; DROP TABLE IF EXISTS config_evals; DROP TABLE IF EXISTS config_closures; DROP TABLE IF EXISTS output_dirs;',
677
+ 'DROP TABLE IF EXISTS entries; DROP TABLE IF EXISTS runs; DROP TABLE IF EXISTS file_hashes; DROP TABLE IF EXISTS blob_sizes; DROP TABLE IF EXISTS output_files; DROP TABLE IF EXISTS invocations; DROP TABLE IF EXISTS run_task_inputs; DROP TABLE IF EXISTS entry_inputs; DROP TABLE IF EXISTS config_evals; DROP TABLE IF EXISTS config_closures; DROP TABLE IF EXISTS output_dirs;',
670
678
  )
671
679
  this.db
672
680
  .prepare("UPDATE schema_meta SET value = ? WHERE key = 'version'")
@@ -790,6 +798,14 @@ export class Cache implements CacheLayer {
790
798
  hashFiles(paths: readonly string[]): Promise<Map<string, string>> {
791
799
  return this.guard(() => this.files.hashFiles(paths))
792
800
  }
801
+ /** `BlobSizeMemo`: the sizes of these blobs this cache has learned. */
802
+ knownBlobSizes(oids: readonly string[]): Map<string, number> {
803
+ return this.guard(() => this.files.knownBlobSizes(oids))
804
+ }
805
+ /** `BlobSizeMemo`: remember blob sizes, honouring the local WRITE axis. */
806
+ rememberBlobSizes(sizes: ReadonlyMap<string, number>): void {
807
+ this.guard(() => this.files.rememberBlobSizes(sizes))
808
+ }
793
809
  /**
794
810
  * `relPosix` against the run's workspace root, memoized: the same three
795
811
  * thousand files are re-relativized for every task of the project, which
@@ -799,11 +815,25 @@ export class Cache implements CacheLayer {
799
815
  */
800
816
  private relMemo = new Map<string, string>()
801
817
  private relMemoRoot: string | undefined
818
+ /** `root` and a slash, when a file under it is relative by slicing; else undefined. */
819
+ private relPrefix: string | undefined
802
820
  private relFor(root: string, file: string): string {
803
821
  if (root !== this.relMemoRoot) {
804
822
  this.relMemoRoot = root
805
823
  this.relMemo.clear()
824
+ this.relPrefix =
825
+ path.sep === '/' && path.isAbsolute(root) && path.normalize(root) === root
826
+ ? root.endsWith('/')
827
+ ? root
828
+ : `${root}/`
829
+ : undefined
806
830
  }
831
+ // An input file is a normalized absolute path (`resolveInputs` makes
832
+ // every one), and under a normalized root that is the root, a slash and
833
+ // its relative path: a slice, where the memo hashed the whole path per
834
+ // file per task (~15 ms over 36,000 files, I-31).
835
+ const prefix = this.relPrefix
836
+ if (prefix !== undefined && file.startsWith(prefix)) return file.slice(prefix.length)
807
837
  let rel = this.relMemo.get(file)
808
838
  if (rel === undefined) this.relMemo.set(file, (rel = relPosix(root, file)))
809
839
  return rel
@@ -68,12 +68,42 @@ export class FileHashStore {
68
68
  if (last !== null && now - Number(last.value) < DAY_MS) return
69
69
  this.db.transaction(() => {
70
70
  this.db.prepare('DELETE FROM file_hashes WHERE seen_at < ?').run(cutoff)
71
+ this.db.prepare('DELETE FROM blob_sizes WHERE seen_at < ?').run(cutoff)
71
72
  this.db
72
73
  .prepare('INSERT OR REPLACE INTO schema_meta(key, value) VALUES (?, ?)')
73
74
  .run(FILE_HASHES_SWEPT_AT, String(now))
74
75
  })()
75
76
  }
76
77
 
78
+ /** Each of `oids` whose size a `rememberBlobSizes` stored: one `IN` query per 900. */
79
+ knownBlobSizes(oids: readonly string[]): Map<string, number> {
80
+ const out = new Map<string, number>()
81
+ for (let i = 0; i < oids.length; i += 900) {
82
+ const chunk = oids.slice(i, i + 900)
83
+ const rows = this.db
84
+ .query(`SELECT oid, size FROM blob_sizes WHERE oid IN (${chunk.map(() => '?').join(',')})`)
85
+ .all(...chunk) as Array<{ oid: string; size: number }>
86
+ for (const r of rows) out.set(r.oid, r.size)
87
+ }
88
+ return out
89
+ }
90
+
91
+ /** Store blob sizes in one transaction; a memo, so a full disk skips it. */
92
+ rememberBlobSizes(sizes: ReadonlyMap<string, number>): void {
93
+ if (!this.write || sizes.size === 0) return
94
+ const now = Date.now()
95
+ const insert = this.db.prepare(
96
+ 'INSERT OR REPLACE INTO blob_sizes(oid, size, seen_at) VALUES (?, ?, ?)',
97
+ )
98
+ try {
99
+ this.db.transaction(() => {
100
+ for (const [oid, size] of sizes) insert.run(oid, size, now)
101
+ })()
102
+ } catch (err) {
103
+ if (!isIndexFull(err)) throw err
104
+ }
105
+ }
106
+
77
107
  /**
78
108
  * Content-hash a file (as a git blob OID, v20) with an mtime+size
79
109
  * fast path. If the `file_hashes` table has a row for `path` whose
@@ -6,7 +6,7 @@
6
6
  // task declared. Nothing here reads a config or applies a boundary.
7
7
 
8
8
  import path from 'node:path'
9
- import { existsSync, lstatSync, realpathSync } from 'node:fs'
9
+ import { existsSync, lstatSync, readdirSync, realpathSync } from 'node:fs'
10
10
  import { UserError, executablePath, gitSpawnRefusal } from '../util/index.js'
11
11
 
12
12
  /** Three facts of the repository a directory is in, from one `git rev-parse`. */
@@ -394,7 +394,14 @@ function expandNestedRepos(
394
394
  }
395
395
  out ??= files.slice(0, i)
396
396
  const abs = path.join(cwd, nested)
397
- if (!existsSync(path.join(abs, '.git'))) continue
397
+ if (!existsSync(path.join(abs, '.git'))) {
398
+ // No repository answers for it: a submodule never initialised (empty)
399
+ // or one whose `.git` was removed to vendor its files, the gitlink
400
+ // left in the index and `git status` silent. Its files are what the
401
+ // task reads, so a walk lists them; they hash by content (A-61).
402
+ for (const f of walkFiles(abs)) out.push(`${nested}/${f}`)
403
+ continue
404
+ }
398
405
  const inner = runGitLsFiles(abs)
399
406
  for (const f of inner.files) out.push(`${nested}/${f}`)
400
407
  for (const f of inner.undecodable) undecodable.push(`${nested}/${f}`)
@@ -402,6 +409,24 @@ function expandNestedRepos(
402
409
  return { files: out ?? files, undecodable }
403
410
  }
404
411
 
412
+ /** Every file and symlink under `dir`, relative and `/`-separated; a `.git` is skipped, a link not followed. */
413
+ function walkFiles(dir: string, rel = ''): string[] {
414
+ let entries
415
+ try {
416
+ entries = readdirSync(path.join(dir, rel), { withFileTypes: true })
417
+ } catch {
418
+ return []
419
+ }
420
+ const out: string[] = []
421
+ for (const e of entries) {
422
+ if (e.name === '.git') continue
423
+ const r = rel === '' ? e.name : `${rel}/${e.name}`
424
+ if (e.isDirectory()) out.push(...walkFiles(dir, r))
425
+ else out.push(r)
426
+ }
427
+ return out
428
+ }
429
+
405
430
  /**
406
431
  * A file's identity as the key folds it: its blob OID, prefixed by its git
407
432
  * mode unless that is a plain file's (100644). A blob OID holds no mode, so
@@ -451,6 +476,108 @@ function parseLsFilesOutput(out: string, undecodableRecords: ReadonlySet<string>
451
476
  return { files, oids, flagged, gitlinks, undecodable }
452
477
  }
453
478
 
479
+ /**
480
+ * `ls-files -s -v -z --debug` as the plain `-s -v -z` stream, plus the size
481
+ * the index records for each regular stage-0 entry's worktree file and its
482
+ * raw OID. With `-z`, each record's NUL is followed by its five
483
+ * newline-ended debug lines (ctime, mtime, dev/ino, uid/gid, size/flags),
484
+ * then the next record.
485
+ */
486
+ function stripLsDebug(run: GitRun): {
487
+ plain: string
488
+ undecodable: Set<string>
489
+ indexed: Map<string, { oid: string; size: number }>
490
+ } {
491
+ const segs = run.stdout.split('\0')
492
+ const records: string[] = []
493
+ const undecodable = new Set<string>()
494
+ const indexed = new Map<string, { oid: string; size: number }>()
495
+ let record = segs[0]!
496
+ let recordRaw = record
497
+ for (let i = 1; i <= segs.length; i++) {
498
+ const seg = segs[i]
499
+ let rest = ''
500
+ let size = -1
501
+ if (seg !== undefined) {
502
+ let at = 0
503
+ for (let k = 0; k < 5 && at >= 0; k++) {
504
+ const nl = seg.indexOf('\n', at)
505
+ if (nl < 0) at = -1
506
+ else {
507
+ if (k === 4) size = Number(/size: (\d+)/.exec(seg.slice(at, nl))?.[1] ?? -1)
508
+ at = nl + 1
509
+ }
510
+ }
511
+ rest = at < 0 ? '' : seg.slice(at)
512
+ }
513
+ if (record.length > 0) {
514
+ records.push(record)
515
+ if (run.undecodable.has(recordRaw)) undecodable.add(record)
516
+ const m = LS_FILES_STAGE_RE.exec(record)
517
+ if (m !== null && m[4] === '0' && (m[2] === '100644' || m[2] === '100755')) {
518
+ indexed.set(record.slice(m[0].length), { oid: m[3]!, size })
519
+ }
520
+ }
521
+ record = rest
522
+ recordRaw = seg ?? ''
523
+ }
524
+ return { plain: records.join('\0'), undecodable, indexed }
525
+ }
526
+
527
+ /** Blob sizes learned once and kept: a blob's size is fixed for its OID. `Cache` is one. */
528
+ export interface BlobSizeMemo {
529
+ knownBlobSizes(oids: readonly string[]): Map<string, number>
530
+ rememberBlobSizes(sizes: ReadonlyMap<string, number>): void
531
+ }
532
+
533
+ /**
534
+ * Drop each trusted OID whose blob is not the size the index recorded for
535
+ * the worktree file (A-60). A clean filter's blob differs from the file it
536
+ * was added from, and git holds a stat-clean entry clean without re-reading
537
+ * it once the filter is gone (`core.autocrlf` turned off, a `.gitattributes`
538
+ * rule removed): `status` and the filter gate, which reads today's config,
539
+ * both let the blob stand for bytes it does not hold. The recorded sizes
540
+ * cost no read of the worktree; the blob sizes come from `memo`, and only
541
+ * the ones it lacks from one `cat-file` (65 ms over 3,000 loose objects,
542
+ * the whole of the cost, measured 2026-10-02). A smudged (racy) entry
543
+ * records 0 and is hashed from disk. An answer that cannot be read trusts
544
+ * nothing.
545
+ */
546
+ async function dropResizedOids(enumeration: GitEnumeration, memo?: BlobSizeMemo): Promise<void> {
547
+ const { trusted, indexed } = enumeration
548
+ const wanted = new Set<string>()
549
+ for (const rel of trusted.keys()) {
550
+ const entry = indexed.get(rel)
551
+ if (entry !== undefined) wanted.add(entry.oid)
552
+ }
553
+ if (wanted.size === 0) return
554
+ const sizes = memo?.knownBlobSizes([...wanted]) ?? new Map<string, number>()
555
+ const unknown = [...wanted].filter((oid) => !sizes.has(oid))
556
+ if (unknown.length > 0) {
557
+ const run = await enumeration.catFile(unknown.join('\n') + '\n')
558
+ if (run === null || run.exitCode !== 0) {
559
+ trusted.clear()
560
+ return
561
+ }
562
+ const learned = new Map<string, number>()
563
+ for (const line of run.stdout.split('\n')) {
564
+ const sp = line.indexOf(' ')
565
+ const n = Number(line.slice(sp + 1))
566
+ if (sp > 0 && line.slice(sp + 1) !== '' && Number.isInteger(n))
567
+ learned.set(line.slice(0, sp), n)
568
+ }
569
+ for (const [oid, n] of learned) sizes.set(oid, n)
570
+ memo?.rememberBlobSizes(learned)
571
+ }
572
+ for (const rel of trusted.keys()) {
573
+ const entry = indexed.get(rel)
574
+ if (entry === undefined) continue // a symlink: its blob is its target string
575
+ const size = sizes.get(entry.oid)
576
+ // The index keeps the low 32 bits of a size.
577
+ if (size === undefined || size % 2 ** 32 !== entry.size) trusted.delete(rel)
578
+ }
579
+ }
580
+
454
581
  /** One completed `git` invocation. */
455
582
  interface GitRun {
456
583
  exitCode: number
@@ -825,7 +952,7 @@ export async function populateGitFilesCache(
825
952
  workspaceRoot,
826
953
  gitPathspecs(workspaceRoot, projectDirs, workspaceWide),
827
954
  )
828
- applyGitEnumeration(enumeration, workspaceRoot, projectDirs, cache, workspaceWide)
955
+ await applyGitEnumeration(enumeration, workspaceRoot, projectDirs, cache, workspaceWide)
829
956
  }
830
957
 
831
958
  /** What one workspace-wide enumeration learned, before it is partitioned per project. */
@@ -851,8 +978,15 @@ export interface GitEnumeration {
851
978
  undecodable: readonly string[]
852
979
  /** `Date.now()` before the spawns: what `trusted` says is true as of no earlier. */
853
980
  startedAtMs: number
981
+ /** Each regular stage-0 entry's raw OID and the worktree size the index recorded. */
982
+ indexed: ReadonlyMap<string, { oid: string; size: number }>
983
+ /** `git cat-file --batch-check` over `stdin`'s OIDs; null when it could not spawn. */
984
+ catFile(stdin: string): Promise<{ exitCode: number; stdout: string } | null>
854
985
  }
855
986
 
987
+ /** Above this many project dirs the enumeration walks the whole tree. */
988
+ export const MAX_SCOPED_PATHSPECS = 64
989
+
856
990
  /**
857
991
  * Pathspec scoping: when the run only needs a handful of projects
858
992
  * (scoped config loading), let git scan just those dirs — 75 ms →
@@ -868,7 +1002,7 @@ export function gitPathspecs(
868
1002
  const scoped =
869
1003
  !workspaceWide &&
870
1004
  rels.length > 0 &&
871
- rels.length <= 64 &&
1005
+ rels.length <= MAX_SCOPED_PATHSPECS &&
872
1006
  rels.every((r) => r !== '' && r !== '.')
873
1007
  return scoped ? rels : ['.']
874
1008
  }
@@ -888,7 +1022,11 @@ export async function startGitEnumeration(
888
1022
  // bulk-populate costs max(ls-files, status) wall time, not the sum
889
1023
  // (status alone is ~74 ms on a 1000-project tree; serial spawning
890
1024
  // was a measurable warm-path regression vs the pre-OID code).
891
- const spawnGit = async (args: string[], stdin?: string): Promise<GitRun | null> => {
1025
+ const spawnGit = async (
1026
+ args: string[],
1027
+ stdin?: string,
1028
+ env?: Record<string, string>,
1029
+ ): Promise<GitRun | null> => {
892
1030
  try {
893
1031
  const proc = Bun.spawn({
894
1032
  // `status` refreshes the index when it can take `index.lock`, so
@@ -900,6 +1038,7 @@ export async function startGitEnumeration(
900
1038
  stdin: stdin === undefined ? 'ignore' : new TextEncoder().encode(stdin),
901
1039
  stdout: 'pipe',
902
1040
  stderr: 'pipe',
1041
+ ...(env !== undefined ? { env: { ...process.env, ...env } } : {}),
903
1042
  })
904
1043
  const [stdout, stderr, exitCode] = await Promise.all([
905
1044
  new Response(proc.stdout).bytes(),
@@ -921,8 +1060,14 @@ export async function startGitEnumeration(
921
1060
  // tracked paths are dirty AND which files are untracked. Asking
922
1061
  // `ls-files --others` for the untracked set walked the same tree a second
923
1062
  // time (~50 ms of CPU, concurrent with status but contending with it).
1063
+ // `--debug` adds each entry's recorded stat, for the blob-size check
1064
+ // (`dropResizedOids`, A-60).
1065
+ const listing = spawnGit(['ls-files', '-s', '-v', '-z', '--debug', '--', ...pathspecs]).then(
1066
+ (run) =>
1067
+ run === null || run.exitCode !== 0 ? { run, debug: null } : { run, debug: stripLsDebug(run) },
1068
+ )
924
1069
  const running = Promise.all([
925
- spawnGit(['ls-files', '-s', '-v', '-z', '--', ...pathspecs]),
1070
+ listing,
926
1071
  // `--ignored=matching` names an ignored path without descending into an
927
1072
  // ignored directory: git applies an ignored `.gitattributes` as it does
928
1073
  // any other, and the filter gate below must see it (A-19).
@@ -962,7 +1107,7 @@ export async function startGitEnumeration(
962
1107
  // at all. `--git-dir` named the per-worktree directory, so the gate
963
1108
  // looked where the rule can never be.
964
1109
  const facts = repoFacts(workspaceRoot)
965
- const [ls, status, vars] = await running
1110
+ const [{ run: ls, debug }, status, vars] = await running
966
1111
  if (ls === null) {
967
1112
  throw gitSpawnRefusal(workspaceRoot)
968
1113
  }
@@ -979,7 +1124,7 @@ export async function startGitEnumeration(
979
1124
  flagged,
980
1125
  gitlinks,
981
1126
  undecodable,
982
- } = parseLsFilesOutput(ls.stdout, ls.undecodable)
1127
+ } = parseLsFilesOutput(debug!.plain, debug!.undecodable)
983
1128
  // Normalize `status`'s repo-root-relative paths to workspace-relative (strip
984
1129
  // the `--show-prefix`) so the dirty set is keyed identically to the trusted
985
1130
  // OID map. Without this, when the workspace root is a git subdir, a modified
@@ -1057,6 +1202,13 @@ export async function startGitEnumeration(
1057
1202
  untracked: dirty === null ? null : untracked,
1058
1203
  undecodable,
1059
1204
  startedAtMs,
1205
+ indexed: debug!.indexed,
1206
+ // A missing blob is an answer ("missing"), never a fetch from a partial
1207
+ // clone's promisor remote.
1208
+ catFile: (stdin) =>
1209
+ spawnGit(['cat-file', '--batch-check=%(objectname) %(objectsize)'], stdin, {
1210
+ GIT_NO_LAZY_FETCH: '1',
1211
+ }),
1060
1212
  }
1061
1213
  }
1062
1214
 
@@ -1086,13 +1238,15 @@ export function lazyGitEnumeration(workspaceRoot: string): LazyGitEnumeration {
1086
1238
  }
1087
1239
 
1088
1240
  /** The partition half of `populateGitFilesCache`: store per-project slices of one enumeration. */
1089
- export function applyGitEnumeration(
1241
+ export async function applyGitEnumeration(
1090
1242
  enumeration: GitEnumeration,
1091
1243
  workspaceRoot: string,
1092
1244
  projectDirs: readonly string[],
1093
1245
  cache: GitFilesCache,
1094
1246
  workspaceWide = false,
1095
- ): void {
1247
+ memo?: BlobSizeMemo,
1248
+ ): Promise<void> {
1249
+ await dropResizedOids(enumeration, memo)
1096
1250
  const { all, trusted } = enumeration
1097
1251
  cache.setWorktreeDirty(enumeration.dirty)
1098
1252
  cache.enumeratedAtMs = enumeration.startedAtMs
@@ -43,12 +43,14 @@ export {
43
43
  ownWorkspaceOutputsSince,
44
44
  stampOutputs,
45
45
  stampWorkspaceOutputs,
46
+ stopRuntimeProbes,
46
47
  type OutputStamp,
47
48
  } from './inputs.js'
48
49
  export {
49
50
  GitFilesCache,
50
51
  applyGitEnumeration,
51
52
  gitPathspecs,
53
+ MAX_SCOPED_PATHSPECS,
52
54
  repoRootOf,
53
55
  startGitEnumeration,
54
56
  lazyGitEnumeration,
@@ -448,6 +448,22 @@ async function runRuntimeCommand(
448
448
  */
449
449
  const liveProbes = new Set<ReturnType<typeof Bun.spawn>>()
450
450
  let probeExitHooked = false
451
+
452
+ /**
453
+ * Kill every runtime probe still running, with its tree. A run's stop asks
454
+ * this: a Ctrl-C while a probe ran waited for the probe, or for the signal
455
+ * handler's bound, about 7 s, before vx exited (C-65). Its answer is no
456
+ * longer needed; its caller sees it fail and the run reads it aborted.
457
+ */
458
+ export function stopRuntimeProbes(): void {
459
+ for (const p of liveProbes) {
460
+ try {
461
+ process.kill(-p.pid, 'SIGKILL')
462
+ } catch {
463
+ // the group is gone
464
+ }
465
+ }
466
+ }
451
467
  function killProbesOnExit(): void {
452
468
  if (probeExitHooked) return
453
469
  probeExitHooked = true
@@ -1032,27 +1048,75 @@ function refuseOneAlternativeBrace(
1032
1048
  }
1033
1049
  }
1034
1050
 
1035
- async function resolveFiles(args: ResolveFilesArgs): Promise<string[]> {
1051
+ /** What a `files` declaration and its task's outputs compile to, whatever the project. */
1052
+ interface FilesPlan {
1053
+ positive: string[]
1054
+ negative: string[]
1055
+ excludeGlobs: Bun.Glob[]
1056
+ ownOutput: (rel: string) => boolean
1057
+ positiveGlobs: Bun.Glob[]
1058
+ /** The literal entries, each naming one path (see `unmatchedLiterals`). */
1059
+ literals: string[]
1060
+ /**
1061
+ * Whether a project-relative path is an input by the declaration alone:
1062
+ * a positive glob selects it, and no exclude and no own output takes it
1063
+ * back. Projects repeat their relative paths (`src/index.ts`,
1064
+ * `package.json`), so each is matched once per plan.
1065
+ */
1066
+ verdicts: Map<string, boolean>
1067
+ }
1068
+
1069
+ /**
1070
+ * Compiled once per declaration for the life of the process: a workspace
1071
+ * declares a handful of `files` lists over thousands of tasks, and
1072
+ * splitting, normalizing and compiling them per task was ~6 ms of a
1073
+ * 1,000-task warm run (I-29). A refused declaration is never stored, so it
1074
+ * throws for every task that carries it.
1075
+ */
1076
+ const filesPlans = new Map<string, FilesPlan | null>()
1077
+
1078
+ function filesPlan(
1079
+ files: readonly string[] | undefined,
1080
+ ownOutputs: readonly string[],
1081
+ ): FilesPlan | null {
1082
+ const key = JSON.stringify([files ?? null, ownOutputs])
1083
+ const memo = filesPlans.get(key)
1084
+ if (memo !== undefined) return memo
1036
1085
  const positive: string[] = []
1037
1086
  const negative: string[] = []
1038
-
1039
- if (args.files === undefined) {
1087
+ if (files === undefined) {
1040
1088
  positive.push(...DEFAULT_FILE_GLOBS)
1041
1089
  } else {
1042
- refuseOneAlternativeBrace(args.files, 'files')
1043
- for (const entry of args.files) {
1090
+ refuseOneAlternativeBrace(files, 'files')
1091
+ for (const entry of files) {
1044
1092
  if (entry.startsWith('!')) negative.push(entry.slice(1))
1045
1093
  else positive.push(entry)
1046
1094
  }
1047
1095
  }
1096
+ const plan =
1097
+ positive.length === 0
1098
+ ? null
1099
+ : {
1100
+ positive,
1101
+ negative,
1102
+ excludeGlobs: [...ALWAYS_IGNORE, ...asTrees(negative)].map(globFor),
1103
+ // A path the task's own outputs take back with `!` is no output, so it
1104
+ // stays an input: a tracked file under `dist` the build reads (A-44).
1105
+ ownOutput: outputMatcher(ownOutputs, globFor),
1106
+ positiveGlobs: asTrees(positive).map(globFor),
1107
+ literals: positive.map(normalizeGlob).filter(isLiteralPattern).map(stripTrailingSlash),
1108
+ verdicts: new Map<string, boolean>(),
1109
+ }
1110
+ filesPlans.set(key, plan)
1111
+ return plan
1112
+ }
1048
1113
 
1049
- if (positive.length === 0) return []
1114
+ async function resolveFiles(args: ResolveFilesArgs): Promise<string[]> {
1115
+ const plan = filesPlan(args.files, args.ownOutputs)
1116
+ if (plan === null) return []
1117
+ const { positive, negative, excludeGlobs, ownOutput, positiveGlobs } = plan
1050
1118
 
1051
1119
  const nested = inNestedProject(args.projectDir, args.nestedProjectDirs)
1052
- const excludeGlobs = [...ALWAYS_IGNORE, ...asTrees(negative)].map(globFor)
1053
- // A path the task's own outputs take back with `!` is no output, so it
1054
- // stays an input: a tracked file under `dist` the build reads (A-44).
1055
- const ownOutput = outputMatcher(args.ownOutputs, globFor)
1056
1120
 
1057
1121
  // Defer to git for the file set (Turbo / Nx parity). Nested .gitignore
1058
1122
  // files, .git/info/exclude, and global excludes all participate
@@ -1062,7 +1126,6 @@ async function resolveFiles(args: ResolveFilesArgs): Promise<string[]> {
1062
1126
  // per task (build + test + lint + …). Spawning git N times for the
1063
1127
  // same project per run is wasteful; we cache the result for the
1064
1128
  // duration of one orchestrator run.
1065
- const positiveGlobs = asTrees(positive).map(globFor)
1066
1129
  // Everything below the snapshot that decides the result: the project, what
1067
1130
  // it declares, what it excludes as its own outputs, and the boundaries.
1068
1131
  const memoKey = `${args.projectDir}\0${positive.join('\u0001')}\0${negative.join('\u0001')}\0${args.ownOutputs.join('\u0001')}\0${args.nestedProjectDirs.join('\u0001')}`
@@ -1102,22 +1165,21 @@ async function resolveFiles(args: ResolveFilesArgs): Promise<string[]> {
1102
1165
  // an artifact built from an older version of a file the config explicitly
1103
1166
  // claims as an input. See the refusal below for why this is not simply
1104
1167
  // honoured instead.
1105
- const unmatchedLiterals = new Set(
1106
- positive.map(normalizeGlob).filter(isLiteralPattern).map(stripTrailingSlash),
1107
- )
1168
+ const unmatchedLiterals = new Set(plan.literals)
1108
1169
  // First pass: glob-filter to candidate absolute paths (no I/O).
1109
1170
  const candidates: string[] = []
1171
+ const verdicts = plan.verdicts
1110
1172
  for (const rel of gitFiles) {
1111
1173
  if (unmatchedLiterals.size > 0) settleLiterals(unmatchedLiterals, rel)
1112
- let matched = false
1113
- for (const g of positiveGlobs) {
1114
- if (g.match(rel)) {
1115
- matched = true
1116
- break
1117
- }
1174
+ let input = verdicts.get(rel)
1175
+ if (input === undefined) {
1176
+ input =
1177
+ positiveGlobs.some((g) => g.match(rel)) &&
1178
+ !excludeGlobs.some((g) => g.match(rel)) &&
1179
+ !ownOutput(rel)
1180
+ verdicts.set(rel, input)
1118
1181
  }
1119
- if (!matched) continue
1120
- if (nested(rel) || excludeGlobs.some((g) => g.match(rel)) || ownOutput(rel)) continue
1182
+ if (!input || nested(rel)) continue
1121
1183
  candidates.push(path.resolve(args.projectDir, rel))
1122
1184
  }
1123
1185
  if (unmatchedLiterals.size > 0) {
@@ -99,7 +99,11 @@ import type { CacheKeyInput } from './layer.js'
99
99
  // v37: an entry saved while `git status` hid a deletion (paired as an
100
100
  // unmerged path's rename source) holds output built without a file its key
101
101
  // folds (A-59).
102
- export const CACHE_VERSION = 'vx-cache-v37'
102
+ // v38: an entry saved while the key folded a filtered LF blob for CRLF
103
+ // bytes on disk, the filter since removed, holds the CRLF build (A-60).
104
+ // v39: an entry saved while a gitlink without `.git` held files the key
105
+ // never listed sits under the key of that directory empty (A-61).
106
+ export const CACHE_VERSION = 'vx-cache-v39'
103
107
 
104
108
  /**
105
109
  * The digest a key folds for an input gone between its enumeration and its
@@ -115,6 +115,15 @@ export function createTables(db: Database): void {
115
115
  content_hash TEXT NOT NULL,
116
116
  seen_at INTEGER NOT NULL
117
117
  );
118
+ -- The size of each index blob the enumeration checked against the
119
+ -- worktree size git recorded (A-60). Fixed for its OID, so a warm
120
+ -- run asks git for none; swept with file_hashes by seen_at, the
121
+ -- time the row was written.
122
+ CREATE TABLE IF NOT EXISTS blob_sizes (
123
+ oid TEXT PRIMARY KEY,
124
+ size INTEGER NOT NULL,
125
+ seen_at INTEGER NOT NULL
126
+ );
118
127
  -- v16: per-output-file fingerprints, scoped by the cache entry
119
128
  -- that produced them. Lets loadOutputFilesBatch(hashes) answer
120
129
  -- "for entry X, what are its outputs supposed to look like?"
package/src/cli/show.ts CHANGED
@@ -339,6 +339,12 @@ function taskBlock(taskName: string, raw: TaskConfig): string {
339
339
  add('outputs.workspaceFiles', list(cache.outputs.workspaceFiles))
340
340
  }
341
341
  const labelW = Math.max(...rows.map(([label]) => label.length))
342
- const body = rows.map(([label, value]) => ` ${`${label}:`.padEnd(labelW + 1)} ${value}`)
342
+ // A multi-line value (a generated `vx_script` wrapper, a heredoc)
343
+ // continues under its first line, not at column 0 where it reads as
344
+ // the next task's header.
345
+ const indent = `\n${' '.repeat(labelW + 4)}`
346
+ const body = rows.map(
347
+ ([label, value]) => ` ${`${label}:`.padEnd(labelW + 1)} ${value.replaceAll('\n', indent)}`,
348
+ )
343
349
  return `${taskName}\n${body.join('\n')}\n`
344
350
  }
@@ -141,7 +141,7 @@ export async function loadCliProjects(
141
141
  projectMetas: metas,
142
142
  seeds: scope,
143
143
  closure: false,
144
- lock,
144
+ lock: lock === null ? null : async () => lock,
145
145
  evalCache:
146
146
  cache === null
147
147
  ? undefined
package/src/exec/index.ts CHANGED
@@ -29,7 +29,13 @@ export {
29
29
  thrownReason,
30
30
  } from './sandbox-runtime.js'
31
31
  export { type DeniedCall } from './sandbox-violations.js'
32
- export { atOrUnder, isMountableLiteral, MOUNT_WILDCARDS, toRealPath } from './sandbox-paths.js'
32
+ export {
33
+ atOrUnder,
34
+ isMountableLiteral,
35
+ MOUNT_WILDCARDS,
36
+ sandboxReads,
37
+ toRealPath,
38
+ } from './sandbox-paths.js'
33
39
  export { bindableWrites, punchWalls } from './sandbox-binds.js'
34
40
  export { isLocalExecutor, localExecutor } from './local-executor.js'
35
41
  export { holdGroups, killTree, untilGroupsGone } from './kill-tree.js'
@@ -76,3 +76,25 @@ export function unique(arr: readonly string[]): string[] {
76
76
  export function localBindingOn(c: { localBinding?: boolean | readonly number[] }): boolean {
77
77
  return c.localBinding === true || (Array.isArray(c.localBinding) && c.localBinding.length > 0)
78
78
  }
79
+
80
+ /**
81
+ * Whether a sandboxed task may read `file`: it lies under a read grant,
82
+ * a write grant or a baseline read, by its canonical path. A path no
83
+ * grant holds is not there inside the sandbox, which the shell reports
84
+ * as "not found" with no denial a trace can see (an `execve`).
85
+ */
86
+ export function sandboxReads(
87
+ sandbox: {
88
+ readonly baseAllowRead: readonly string[]
89
+ readonly config: {
90
+ readonly allowRead: readonly string[]
91
+ readonly allowWrite: readonly string[]
92
+ }
93
+ },
94
+ file: string,
95
+ ): boolean {
96
+ const real = toRealPath(file)
97
+ return [...sandbox.baseAllowRead, ...sandbox.config.allowRead, ...sandbox.config.allowWrite].some(
98
+ (g) => atOrUnder(real, toRealPath(g)),
99
+ )
100
+ }