@vzn/vx 0.0.594 → 0.0.596

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +1 -1
  2. package/package.json +5 -5
  3. package/src/bin.ts +26 -11
  4. package/src/cache/archive.ts +2 -2
  5. package/src/cache/cache.ts +235 -174
  6. package/src/cache/chained-cache.ts +25 -7
  7. package/src/cache/config-evals.ts +19 -4
  8. package/src/cache/git-inputs.ts +19 -8
  9. package/src/cache/index.ts +1 -0
  10. package/src/cache/inputs.ts +122 -49
  11. package/src/cache/layer.ts +7 -5
  12. package/src/cache/layered-cache.ts +25 -2
  13. package/src/cache/policy.ts +5 -0
  14. package/src/cache/tar-stream.ts +1 -1
  15. package/src/cache/zstd.ts +10 -4
  16. package/src/cli/cache.ts +11 -15
  17. package/src/cli/completions.ts +42 -6
  18. package/src/cli/foreign-flags.ts +26 -5
  19. package/src/cli/help.ts +5 -3
  20. package/src/cli/init.ts +1 -1
  21. package/src/cli/last.ts +4 -1
  22. package/src/cli/plan-format.ts +9 -0
  23. package/src/cli/plugin-templates.ts +2 -2
  24. package/src/cli/run.ts +60 -42
  25. package/src/cli/select.ts +30 -3
  26. package/src/cli/show.ts +42 -3
  27. package/src/cli/watch-filter.ts +77 -13
  28. package/src/cli/watch-fs.ts +15 -7
  29. package/src/cli/watch-judge.ts +16 -9
  30. package/src/cli/watch.ts +22 -15
  31. package/src/config.ts +26 -26
  32. package/src/exec/env.ts +5 -0
  33. package/src/exec/index.ts +1 -0
  34. package/src/exec/proc-sample.ts +6 -2
  35. package/src/exec/runner.ts +98 -40
  36. package/src/exec/sandbox-paths.ts +12 -6
  37. package/src/exec/sandbox-runtime.ts +136 -44
  38. package/src/exec/sandbox-violations.ts +58 -11
  39. package/src/graph/index.ts +1 -0
  40. package/src/graph/scheduler.ts +29 -13
  41. package/src/graph/task-graph.ts +134 -25
  42. package/src/index.ts +6 -0
  43. package/src/orchestrator/admission.ts +12 -3
  44. package/src/orchestrator/events.ts +10 -1
  45. package/src/orchestrator/execute-task.ts +65 -20
  46. package/src/orchestrator/failure-mode.ts +8 -2
  47. package/src/orchestrator/framed-output.ts +3 -3
  48. package/src/orchestrator/hit-restore.ts +6 -3
  49. package/src/orchestrator/local-shortcircuit.ts +4 -1
  50. package/src/orchestrator/lockfile-claim.ts +43 -46
  51. package/src/orchestrator/logger.ts +11 -6
  52. package/src/orchestrator/metrics.ts +19 -8
  53. package/src/orchestrator/miss-save.ts +2 -2
  54. package/src/orchestrator/options.ts +7 -0
  55. package/src/orchestrator/plan.ts +28 -4
  56. package/src/orchestrator/plugin-host.ts +25 -3
  57. package/src/orchestrator/plugin.ts +39 -17
  58. package/src/orchestrator/prepare.ts +13 -4
  59. package/src/orchestrator/run-artifacts.ts +2 -1
  60. package/src/orchestrator/run.ts +72 -12
  61. package/src/orchestrator/sandbox-request.ts +65 -9
  62. package/src/orchestrator/save-lane.ts +5 -5
  63. package/src/orchestrator/signals.ts +5 -2
  64. package/src/orchestrator/summary.ts +16 -5
  65. package/src/orchestrator/telemetry.ts +16 -7
  66. package/src/util/index.ts +7 -1
  67. package/src/util/paths.ts +23 -3
  68. package/src/util/secret-mask.ts +36 -0
  69. package/src/util/settle.ts +15 -0
  70. package/src/workspace/affected.ts +30 -8
  71. package/src/workspace/config-cache.ts +25 -4
  72. package/src/workspace/config-imports.ts +8 -6
  73. package/src/workspace/config-schema.ts +74 -9
  74. package/src/workspace/filter.ts +19 -0
  75. package/src/workspace/migrate-scripts.ts +5 -3
  76. package/src/workspace/migration.ts +1 -1
  77. package/src/workspace/package-graph.ts +15 -3
  78. package/src/workspace/project-loader.ts +76 -2
  79. package/src/workspace/repo-id.ts +42 -4
  80. package/src/workspace/workspace.ts +98 -20
@@ -44,6 +44,8 @@ import {
44
44
  openSync,
45
45
  readdirSync,
46
46
  renameSync,
47
+ statSync,
48
+ utimesSync,
47
49
  writeFileSync,
48
50
  } from 'node:fs'
49
51
  import { readdir, rm, stat, unlink, writeFile } from 'node:fs/promises'
@@ -110,12 +112,13 @@ import { RunHistory } from './run-history.js'
110
112
  import { CACHE_VERSION, foldKey } from './key-fold.js'
111
113
 
112
114
  /**
113
- * An artifact or temp file without an `entries` row is reaped by
114
- * `prune()` only once it is this old. A save renames the artifact into
115
- * place inside its `BEGIN IMMEDIATE` row transaction (A-3), so a live
116
- * artifact without a row is one whose commit failed and was taken back
117
- * out; a crashed save's temp is stale long before this. Generous on purpose — a false orphan costs a
118
- * re-run, a leaked temp costs disk.
115
+ * An artifact or temp file without an `entries` row is never reaped
116
+ * younger than this: a crashed save's temp is stale long before it. A
117
+ * row-less artifact may still be in use (another vx version's open dropped
118
+ * its row), so past this it is judged by the retention policy, its file
119
+ * time standing for `accessed_at`. A hit renews a file time older than
120
+ * this, so the file time trails the last use by at most this much, and
121
+ * the policy reads the file time plus this as the last use.
119
122
  */
120
123
  const ORPHAN_GRACE_MS = 60 * 60 * 1000
121
124
 
@@ -242,10 +245,90 @@ export const SCHEMA_VERSION = 'v32'
242
245
  /** The tables a store holds: dropped from a workspace index that held them itself. */
243
246
  const STORE_TABLES = ['entry_inputs', 'output_files', 'entry_stdout', 'store_meta', 'entries']
244
247
 
248
+ /**
249
+ * An index that records a schema other than this vx's: the one an opener
250
+ * that writes resets. Reads the recorded version alone and touches
251
+ * nothing; an unreadable file is left to the open to refuse.
252
+ */
253
+ function indexOfAnotherSchema(dbFile: string): boolean {
254
+ let found: string | undefined
255
+ try {
256
+ const db = new Database(dbFile, { readonly: true })
257
+ try {
258
+ found = (
259
+ db.query("SELECT value FROM schema_meta WHERE key = 'version'").get() as {
260
+ value: string
261
+ } | null
262
+ )?.value
263
+ } finally {
264
+ closeDb(db)
265
+ }
266
+ } catch {
267
+ return false
268
+ }
269
+ return found !== undefined && found !== SCHEMA_VERSION
270
+ }
271
+
245
272
  /** An entry row with its stdout, which lives apart (v29); none stored reads as ''. */
246
273
  const SELECT_ENTRY =
247
274
  "SELECT e.*, COALESCE(s.stdout, '') AS stdout FROM entries e LEFT JOIN entry_stdout s ON s.hash = e.hash"
248
275
 
276
+ /** A file the orphan sweep may judge: a row-less artifact, or a temp. */
277
+ interface RowlessFile {
278
+ file: string
279
+ size: number
280
+ mtimeMs: number
281
+ temp: boolean
282
+ }
283
+
284
+ /**
285
+ * What a prune takes under `policy`: index rows by `accessed_at` and
286
+ * row-less artifacts by file time, under one rule (a temp always). The
287
+ * rows are the non-phantom ones; the files are past the in-flight grace.
288
+ */
289
+ function pickVictims(
290
+ rows: ReadonlyArray<{ hash: string; size_bytes: number; accessed_at: number }>,
291
+ files: readonly RowlessFile[],
292
+ policy: { olderThanMs?: number; maxBytes?: number },
293
+ ): { hashes: Set<string>; bytesFreed: number; files: RowlessFile[] } {
294
+ const { olderThanMs, maxBytes } = policy
295
+ const hashes = new Set<string>()
296
+ let bytesFreed = 0
297
+ const taken: RowlessFile[] = []
298
+ const kept: Array<{ at: number; size: number; row?: string; file?: RowlessFile }> = []
299
+ let remaining = 0
300
+ for (const r of rows) {
301
+ if (olderThanMs !== undefined && r.accessed_at < olderThanMs) {
302
+ hashes.add(r.hash)
303
+ bytesFreed += r.size_bytes
304
+ } else {
305
+ kept.push({ at: r.accessed_at, size: r.size_bytes, row: r.hash })
306
+ remaining += r.size_bytes
307
+ }
308
+ }
309
+ for (const f of files) {
310
+ const at = f.mtimeMs + ORPHAN_GRACE_MS
311
+ if (f.temp || (olderThanMs !== undefined && at < olderThanMs)) taken.push(f)
312
+ else {
313
+ kept.push({ at, size: f.size, file: f })
314
+ remaining += f.size
315
+ }
316
+ }
317
+ if (maxBytes !== undefined && remaining > maxBytes) {
318
+ kept.sort((a, b) => a.at - b.at)
319
+ for (const k of kept) {
320
+ if (remaining <= maxBytes) break
321
+ remaining -= k.size
322
+ if (k.file !== undefined) taken.push(k.file)
323
+ else {
324
+ hashes.add(k.row!)
325
+ bytesFreed += k.size
326
+ }
327
+ }
328
+ }
329
+ return { hashes, bytesFreed, files: taken }
330
+ }
331
+
249
332
  /**
250
333
  * Artifacts `indexed` does not hold, and temps, past the in-flight grace
251
334
  * window: one readdir, one stat per candidate. `indexed` is asked only
@@ -256,7 +339,7 @@ const SELECT_ENTRY =
256
339
  async function scanOrphanFiles(
257
340
  cacheDir: string,
258
341
  indexedHashes: () => ReadonlySet<string>,
259
- ): Promise<Array<{ file: string; size: number }>> {
342
+ ): Promise<RowlessFile[]> {
260
343
  let names: string[]
261
344
  try {
262
345
  names = await readdir(cacheDir)
@@ -265,19 +348,22 @@ async function scanOrphanFiles(
265
348
  }
266
349
  const indexed = indexedHashes()
267
350
  const cutoff = Date.now() - ORPHAN_GRACE_MS
268
- const candidates: string[] = []
351
+ const candidates: Array<{ name: string; temp: boolean }> = []
269
352
  for (const name of names) {
270
353
  const m = VX_ARTIFACT_NAME.exec(name)
271
354
  if (m === null) continue
272
- if (m[2] !== undefined || !indexed.has(m[1]!)) candidates.push(name)
355
+ const temp = m[2] !== undefined
356
+ if (temp || !indexed.has(m[1]!)) candidates.push({ name, temp })
273
357
  }
274
- const found: Array<{ file: string; size: number }> = []
358
+ const found: RowlessFile[] = []
275
359
  await Promise.all(
276
- candidates.map(async (name) => {
360
+ candidates.map(async ({ name, temp }) => {
277
361
  const file = path.join(cacheDir, name)
278
362
  try {
279
363
  const st = await stat(file)
280
- if (st.isFile() && st.mtimeMs <= cutoff) found.push({ file, size: st.size })
364
+ if (st.isFile() && st.mtimeMs <= cutoff) {
365
+ found.push({ file, size: st.size, mtimeMs: st.mtimeMs, temp })
366
+ }
281
367
  } catch {
282
368
  // Gone between readdir and stat: not an orphan any more.
283
369
  }
@@ -545,6 +631,8 @@ export class Cache implements CacheLayer {
545
631
  private readonly deleteStdout: ReturnType<Database['prepare']>
546
632
  private readonly entryExists: ReturnType<Database['prepare']>
547
633
  private readonly touched = new Set<string>()
634
+ /** Hit artifacts whose file time is past the grace: renewed at the flush (`ORPHAN_GRACE_MS`). */
635
+ private readonly staleTimes = new Set<string>()
548
636
  private readonly insertEntryInput: ReturnType<Database['prepare']>
549
637
  /** The per-file (mtime, size) → blob-OID memo behind `hashFile`. */
550
638
  private readonly files: FileHashStore
@@ -608,41 +696,6 @@ export class Cache implements CacheLayer {
608
696
  return new Cache(cacheDir, undefined, undefined, undefined, 'inspect')
609
697
  }
610
698
 
611
- /**
612
- * What a real prune reaps from an index of an EARLIER schema, which it
613
- * resets first, leaving every aged artifact row-less: a dry run that
614
- * refused it could not preview the biggest prune there is, the one
615
- * after an upgrade (item 1083). Reads the recorded version alone and
616
- * touches nothing. Null for an absent, current, newer or unreadable
617
- * index; `Cache.inspect` answers those.
618
- */
619
- static async orphansBeforeReset(
620
- cacheDir: string,
621
- ): Promise<{ found: string; orphans: number; orphanBytes: number } | null> {
622
- const dbFile = path.join(cacheDir, 'cache.db')
623
- if (!existsSync(dbFile)) return null
624
- let found: string | undefined
625
- try {
626
- const db = new Database(dbFile, { readonly: true })
627
- try {
628
- found = (
629
- db.prepare("SELECT value FROM schema_meta WHERE key = 'version'").get() as {
630
- value: string
631
- } | null
632
- )?.value
633
- } finally {
634
- closeDb(db)
635
- }
636
- } catch {
637
- return null
638
- }
639
- if (found === undefined || found === SCHEMA_VERSION) return null
640
- const aged = await scanOrphanFiles(cacheDir, () => new Set())
641
- let orphanBytes = 0
642
- for (const o of aged) orphanBytes += o.size
643
- return { found, orphans: aged.length, orphanBytes }
644
- }
645
-
646
699
  constructor(
647
700
  private readonly cacheDir: string,
648
701
  localPolicy: { read: boolean; write: boolean } = { read: true, write: true },
@@ -655,23 +708,25 @@ export class Cache implements CacheLayer {
655
708
  */
656
709
  private readonly artifactCeiling: number = MAX_DECOMPRESSED_ARTIFACT_BYTES,
657
710
  /**
658
- * `'inspect'`: a reading verb (`why`, `last`, `info`, a dry prune). It
659
- * never resets the index: a schema it cannot read is refused, named,
660
- * and left as it was.
711
+ * `'inspect'`: a reading verb (`why`, `last`, `info`). It never resets
712
+ * the index: a schema it cannot read is refused, named, and left as it
713
+ * was. `'preview'`: a dry prune, which reads such an index as the reset
714
+ * the real prune does first leaves it, empty, with the store still
715
+ * reached, so it names what that prune takes (item 1083).
661
716
  */
662
- mode: 'open' | 'inspect' = 'open',
717
+ mode: 'open' | 'inspect' | 'preview' = 'open',
663
718
  /**
664
719
  * The shared store's directory, unversioned: every key is seeded with
665
720
  * `CACHE_VERSION`, so two vx versions never read each other's entries,
666
721
  * and the store's own schema is `store_meta.schema` (`matchStoreSchema`). Or
667
722
  * `null` for an index that holds its entries itself (a `cacheDir`).
668
723
  * Undefined follows the layout the index records: a reading verb, a
669
- * plugin's handle. An `'inspect'` open reads it only where the index
724
+ * plugin's handle. A reading open reads it only where the index
670
725
  * records none (deleted, or never written beside a run's store).
671
726
  */
672
727
  storeRoot?: string | null,
673
728
  ) {
674
- this.inspecting = mode === 'inspect'
729
+ this.inspecting = mode !== 'open'
675
730
  this.read = localPolicy.read
676
731
  // The directory exists before the DB opens — bun:sqlite won't create
677
732
  // parent dirs for us. A directory this user cannot write into is a
@@ -687,7 +742,9 @@ export class Cache implements CacheLayer {
687
742
  // said "no recorded runs yet" (item 900).
688
743
  const dbFile = path.join(cacheDir, 'cache.db')
689
744
  this.dbFile = dbFile
690
- const absent = mode === 'inspect' && !existsSync(dbFile)
745
+ const absent =
746
+ mode !== 'open' &&
747
+ (!existsSync(dbFile) || (mode === 'preview' && indexOfAnotherSchema(dbFile)))
691
748
  this.writeBlocked = absent ? 'no index there yet' : openCacheDir(cacheDir, repoDir)
692
749
  this.write = localPolicy.write && this.writeBlocked === null
693
750
  try {
@@ -779,7 +836,7 @@ export class Cache implements CacheLayer {
779
836
  // 2026-10-06; a newer one used to be refused, item 896). A reading verb
780
837
  // leaves it as it was.
781
838
  const refuseUnreadable = (found: string): void => {
782
- if (mode === 'inspect') {
839
+ if (mode !== 'open') {
783
840
  throw new UserError(
784
841
  `the cache at ${cacheDir} holds index schema ${found} from another vx version; this vx reads ${SCHEMA_VERSION}, so nothing in it is readable here, and this verb leaves it untouched. The next \`vx run\` resets it`,
785
842
  )
@@ -833,7 +890,7 @@ export class Cache implements CacheLayer {
833
890
  )?.value,
834
891
  )
835
892
  let storeDir =
836
- mode === 'inspect'
893
+ mode !== 'open'
837
894
  ? (recorded ?? (typeof storeRoot === 'string' ? storeRoot : undefined))
838
895
  : storeRoot === undefined
839
896
  ? recorded
@@ -877,17 +934,10 @@ export class Cache implements CacheLayer {
877
934
  this.storeDir = storeDir
878
935
  this.artifactDir = storeDir ?? cacheDir
879
936
  if (storeDir !== undefined) {
880
- this.attachStore(storeDir, mode === 'inspect')
937
+ this.attachStore(storeDir, mode !== 'open')
881
938
  this.storeReset = this.matchStoreSchema(mode === 'open' && this.writeBlocked === null)
882
- }
883
-
884
- createTables(this.db, storeDir === undefined ? 'main' : 'store')
885
- if (storeDir !== undefined && mode === 'open' && this.writeBlocked === null) {
886
- this.db
887
- .prepare(
888
- "INSERT INTO store.store_meta(key, value) VALUES ('schema', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value",
889
- )
890
- .run(SCHEMA_VERSION)
939
+ } else {
940
+ createTables(this.db, 'main')
891
941
  }
892
942
  if (mode === 'open' && this.writeBlocked === null && storeDir !== recorded) {
893
943
  if (storeDir === undefined)
@@ -1018,50 +1068,63 @@ export class Cache implements CacheLayer {
1018
1068
  * reads it: a store another `SCHEMA_VERSION` wrote has its tables dropped
1019
1069
  * (owner, 2026-10-06), the artifacts kept. Each is indexed again from its
1020
1070
  * own bytes on its next hit (`adopt`). A reading verb reads such a store
1021
- * as empty and changes nothing.
1071
+ * as empty and changes nothing. Creates the store's tables either way.
1022
1072
  */
1023
1073
  private matchStoreSchema(writable: boolean): SchemaReset | null {
1024
- const tables = new Set(
1025
- (
1026
- this.db
1027
- .prepare("SELECT name FROM store.sqlite_master WHERE type = 'table'")
1028
- .all() as Array<{
1029
- name: string
1030
- }>
1031
- ).map((r) => r.name),
1032
- )
1033
- if (!tables.has('entries')) return null
1034
- const found = tables.has('store_meta')
1035
- ? (
1036
- this.db.prepare("SELECT value FROM store.store_meta WHERE key = 'schema'").get() as {
1037
- value: string
1038
- } | null
1039
- )?.value
1040
- : undefined
1041
- if (found === SCHEMA_VERSION) return null
1074
+ const read = (): { has: boolean; found: string | undefined } => {
1075
+ const tables = new Set(
1076
+ (
1077
+ this.db
1078
+ .prepare("SELECT name FROM store.sqlite_master WHERE type = 'table'")
1079
+ .all() as Array<{
1080
+ name: string
1081
+ }>
1082
+ ).map((r) => r.name),
1083
+ )
1084
+ const found = tables.has('store_meta')
1085
+ ? (
1086
+ this.db.prepare("SELECT value FROM store.store_meta WHERE key = 'schema'").get() as {
1087
+ value: string
1088
+ } | null
1089
+ )?.value
1090
+ : undefined
1091
+ return { has: tables.has('entries'), found }
1092
+ }
1093
+ const seen = read()
1094
+ if (seen.has && seen.found === SCHEMA_VERSION) {
1095
+ createTables(this.db, 'store')
1096
+ return null
1097
+ }
1042
1098
  if (!writable) {
1043
- this.db.exec('DETACH DATABASE store')
1044
- this.db.prepare('ATTACH DATABASE ? AS store').run(':memory:')
1099
+ if (seen.has) {
1100
+ this.db.exec('DETACH DATABASE store')
1101
+ this.db.prepare('ATTACH DATABASE ? AS store').run(':memory:')
1102
+ }
1103
+ createTables(this.db, 'store')
1045
1104
  return null
1046
1105
  }
1047
- let reset: SchemaReset | null = null
1048
- this.db
1049
- .transaction(() => {
1050
- // Re-read under the write lock: another workspace's open may have
1051
- // reset it first.
1052
- const now = (
1053
- this.db.prepare("SELECT value FROM store.store_meta WHERE key = 'schema'").get() as {
1054
- value: string
1055
- } | null
1056
- )?.value
1057
- if (now === SCHEMA_VERSION) return
1058
- for (const t of STORE_TABLES) {
1059
- if (t !== 'store_meta') this.db.exec(`DROP TABLE IF EXISTS store.${t}`)
1106
+ // Re-read, drop, create and stamp under one write lock: as separate
1107
+ // transactions, another vx's open landing after the drop made its own
1108
+ // tables, and this one's `CREATE IF NOT EXISTS` kept them under its stamp.
1109
+ return this.db
1110
+ .transaction((): SchemaReset | null => {
1111
+ const now = read()
1112
+ let reset: SchemaReset | null = null
1113
+ if (now.has && now.found !== SCHEMA_VERSION) {
1114
+ for (const t of STORE_TABLES) {
1115
+ if (t !== 'store_meta') this.db.exec(`DROP TABLE IF EXISTS store.${t}`)
1116
+ }
1117
+ reset = { from: now.found ?? 'an earlier schema', to: SCHEMA_VERSION }
1060
1118
  }
1061
- reset = { from: found ?? 'an earlier schema', to: SCHEMA_VERSION }
1119
+ createTables(this.db, 'store')
1120
+ this.db
1121
+ .prepare(
1122
+ "INSERT INTO store.store_meta(key, value) VALUES ('schema', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value",
1123
+ )
1124
+ .run(SCHEMA_VERSION)
1125
+ return reset
1062
1126
  })
1063
1127
  .immediate()
1064
- return reset
1065
1128
  }
1066
1129
 
1067
1130
  // --- config evaluations: `ConfigEvalStore`, delegated to `ConfigEvalTable` ---
@@ -1222,6 +1285,12 @@ export class Cache implements CacheLayer {
1222
1285
  // A second name for the same bytes: the index step renames it over
1223
1286
  // the artifact inside its write transaction, as for a save.
1224
1287
  linkSync(finalPath, tmpPath)
1288
+ // The link shares the artifact's inode and so its mtime, which an
1289
+ // artifact that needs adopting has had for hours: another process's
1290
+ // orphan sweep read the temp as a crashed save's and took it, and the
1291
+ // artifact with it, while this adopt was scanning them.
1292
+ const now = new Date()
1293
+ utimesSync(tmpPath, now, now)
1225
1294
  } catch {
1226
1295
  return false
1227
1296
  }
@@ -1262,7 +1331,9 @@ export class Cache implements CacheLayer {
1262
1331
 
1263
1332
  // Verify the tar artifact actually exists. The DB and the
1264
1333
  // filesystem can drift if someone manually deletes the cache dir.
1265
- if (!existsSync(this.tarPath(hash))) return null
1334
+ const st = statSync(this.tarPath(hash), { throwIfNoEntry: false })
1335
+ if (st === undefined) return null
1336
+ if (st.mtimeMs < Date.now() - ORPHAN_GRACE_MS) this.staleTimes.add(hash)
1266
1337
 
1267
1338
  // Deferred: per-hit UPDATEs cost ~60 ms across 2000+ probes on a
1268
1339
  // full-cache run. Hashes are collected and flushed as ONE batched
@@ -1295,7 +1366,7 @@ export class Cache implements CacheLayer {
1295
1366
  ): Promise<Map<string, CacheEntry>> {
1296
1367
  return this.guard(async () => {
1297
1368
  const out = await this.getManyEntries(hashes)
1298
- if (ctx === undefined || !this.write || out.size === hashes.length) return out
1369
+ if (ctx === undefined || !this.read || !this.write || out.size === hashes.length) return out
1299
1370
  // An artifact the index does not know is still a hit (`adopt`).
1300
1371
  for (const hash of hashes) {
1301
1372
  if (out.has(hash) || !(await this.adopt(hash, ctx(hash)))) continue
@@ -1312,8 +1383,12 @@ export class Cache implements CacheLayer {
1312
1383
  const { test, params } = inHashes(hashes)
1313
1384
  const rows = this.db.query(`${SELECT_ENTRY} WHERE e.hash ${test}`).all(...params) as EntryRow[]
1314
1385
  if (rows.length === 0) return out
1315
- const present = rows.map((r) => existsSync(this.tarPath(r.hash)))
1316
- const live = rows.filter((_r, i) => present[i])
1386
+ const graceStart = Date.now() - ORPHAN_GRACE_MS
1387
+ const live = rows.filter((r) => {
1388
+ const st = statSync(this.tarPath(r.hash), { throwIfNoEntry: false })
1389
+ if (st !== undefined && st.mtimeMs < graceStart) this.staleTimes.add(r.hash)
1390
+ return st !== undefined
1391
+ })
1317
1392
  const liveHashes = live.map((r) => r.hash)
1318
1393
  const fileRows = this.loadOutputFilesBatch(liveHashes)
1319
1394
  const dirRows = this.loadOutputDirsBatch(liveHashes)
@@ -2041,6 +2116,17 @@ export class Cache implements CacheLayer {
2041
2116
  const hashes = [...this.touched]
2042
2117
  this.touched.clear()
2043
2118
  const now = Date.now()
2119
+ // The file time is the last use another vx version's sweep can see:
2120
+ // its open dropped this index's rows (`pickVictims`).
2121
+ const at = new Date(now)
2122
+ for (const hash of this.staleTimes) {
2123
+ try {
2124
+ utimesSync(this.tarPath(hash), at, at)
2125
+ } catch {
2126
+ // Gone, or another user's file: a sweep judges its old time.
2127
+ }
2128
+ }
2129
+ this.staleTimes.clear()
2044
2130
  // LRU bookkeeping: on a full disk the bumps are dropped, never the prune
2045
2131
  // or the stats that asked for them (A-14).
2046
2132
  try {
@@ -2155,21 +2241,26 @@ export class Cache implements CacheLayer {
2155
2241
  // above never sees them (upstream survey, nx#35483: 9 MiB of them sat
2156
2242
  // under a 1 MB limit). Listing the directory to find them costs 0.5 ms
2157
2243
  // per 1,000 entries (4.9 ms at 10,000), every run; the sweep's own
2158
- // clock in `schema_meta` costs one indexed read. An orphan is reapable
2159
- // only an hour after its last write anyway, so an hourly sweep reaps
2160
- // it within two.
2244
+ // clock in `schema_meta` costs one indexed read. The sweep is the
2245
+ // policy's own prune, with the row-less files counted.
2161
2246
  const swept = this.db.prepare('SELECT value FROM schema_meta WHERE key = ?').get(SWEPT_AT) as {
2162
2247
  value: string
2163
2248
  } | null
2164
2249
  if (swept !== null && now - Number(swept.value) < ORPHAN_GRACE_MS) return null
2165
- return { evicted: 0, bytesFreed: 0, ...(await this.reapOrphans(now)) }
2250
+ return this.pruneIndex(
2251
+ {
2252
+ ...(olderThanMs !== undefined ? { olderThanMs } : {}),
2253
+ ...(policy.maxBytes !== undefined ? { maxBytes: policy.maxBytes } : {}),
2254
+ },
2255
+ now,
2256
+ )
2166
2257
  }
2167
2258
 
2168
2259
  async prune(options: PruneOptions): Promise<PruneResult> {
2169
2260
  return this.guard(() => this.pruneIndex(options))
2170
2261
  }
2171
2262
 
2172
- private async pruneIndex(options: PruneOptions): Promise<PruneResult> {
2263
+ private async pruneIndex(options: PruneOptions, now: number = Date.now()): Promise<PruneResult> {
2173
2264
  this.flushAccessed()
2174
2265
  // Before any entry is deleted, so a kept entry's pending snapshot is on
2175
2266
  // disk whatever the eviction does next. A pruned hash's snapshot cannot
@@ -2181,50 +2272,24 @@ export class Cache implements CacheLayer {
2181
2272
  throw new Error('prune: pass at least one of `olderThanMs` or `maxBytes`')
2182
2273
  }
2183
2274
 
2184
- const victims = new Set<string>()
2185
- let bytesFreed = 0
2186
2275
  // Rows whose artifact is gone (deleted by hand, or by a disk cleaner):
2187
2276
  // never a hit, and their bytes are on no disk, but `--max-size` counted
2188
2277
  // them and evicted real entries to make room for them (item 975). None
2189
2278
  // is evicted or freed, whatever its age (item 1081); the ones past the
2190
2279
  // grace window are dropped.
2191
- const { phantoms, stale } = await this.phantomRows()
2192
-
2193
- if (olderThanMs !== undefined) {
2194
- const rows = this.db
2195
- .prepare('SELECT hash, size_bytes FROM entries WHERE accessed_at < ?')
2196
- .all(olderThanMs) as Array<{ hash: string; size_bytes: number }>
2197
- for (const r of rows) {
2198
- if (phantoms.has(r.hash)) continue
2199
- victims.add(r.hash)
2200
- bytesFreed += r.size_bytes
2201
- }
2202
- }
2203
-
2204
- if (maxBytes !== undefined) {
2205
- const totalRow = this.db
2206
- .prepare('SELECT COALESCE(SUM(size_bytes), 0) AS bytes FROM entries')
2207
- .get() as { bytes: number }
2208
- let phantomBytes = 0
2209
- for (const b of phantoms.values()) phantomBytes += b
2210
- let remaining = totalRow.bytes - phantomBytes - bytesFreed
2211
- if (remaining > maxBytes) {
2212
- // Exclude already-picked victims in JS, not via a SQL NOT-IN —
2213
- // an IN-list over tens of thousands of TTL victims would blow
2214
- // SQLite's bound-parameter ceiling (see flushAccessed's 900 cap).
2215
- const candidates = (
2216
- this.db
2217
- .prepare('SELECT hash, size_bytes FROM entries ORDER BY accessed_at ASC')
2218
- .all() as Array<{ hash: string; size_bytes: number }>
2219
- ).filter((row) => !victims.has(row.hash) && !phantoms.has(row.hash))
2220
- for (const row of candidates) {
2221
- if (remaining <= maxBytes) break
2222
- victims.add(row.hash)
2223
- bytesFreed += row.size_bytes
2224
- remaining -= row.size_bytes
2225
- }
2226
- }
2227
- }
2280
+ const { phantoms, stale, rows } = await this.phantomRows()
2281
+ const picked = pickVictims(
2282
+ rows.filter((r) => !phantoms.has(r.hash)),
2283
+ await this.scanOrphans(),
2284
+ {
2285
+ ...(olderThanMs !== undefined ? { olderThanMs } : {}),
2286
+ ...(maxBytes !== undefined ? { maxBytes } : {}),
2287
+ },
2288
+ )
2289
+ const victims = picked.hashes
2290
+ const { bytesFreed } = picked
2291
+ let orphanBytes = 0
2292
+ for (const o of picked.files) orphanBytes += o.size
2228
2293
 
2229
2294
  // Delete DB rows in a single transaction (one fsync; ON DELETE
2230
2295
  // CASCADE clears `output_files`) and unlink artifacts in parallel.
@@ -2233,13 +2298,7 @@ export class Cache implements CacheLayer {
2233
2298
  // flushAccessed so a huge eviction stays under any build's
2234
2299
  // bound-parameter ceiling.
2235
2300
  if (dryRun) {
2236
- const orphans = await this.orphanStats()
2237
- return {
2238
- evicted: victims.size,
2239
- bytesFreed,
2240
- orphans: orphans.orphans,
2241
- orphanBytes: orphans.orphanBytes,
2242
- }
2301
+ return { evicted: victims.size, bytesFreed, orphans: picked.files.length, orphanBytes }
2243
2302
  }
2244
2303
  if (victims.size > 0 || stale.length > 0) {
2245
2304
  const hashes = [...victims]
@@ -2269,23 +2328,21 @@ export class Cache implements CacheLayer {
2269
2328
  } catch (err) {
2270
2329
  if (!isIndexFull(err)) throw err
2271
2330
  }
2272
- const orphans = await this.reapOrphans()
2331
+ const orphans = await this.reapOrphans(picked.files, now)
2273
2332
  return { evicted: victims.size, bytesFreed, ...orphans }
2274
2333
  }
2275
2334
 
2276
2335
  /**
2277
- * Unlink artifacts the index does not know: a `<hash>.tar.zst` with no
2278
- * `entries` row (a `SCHEMA_VERSION` drop, a `cache.db` deleted by hand)
2279
- * and a `<hash>.tar.zst.tmp-*` a crashed save never renamed. Nothing
2280
- * else reaps them — a lookup starts at the row, and a save of the same
2281
- * key renames over the file, so a key that never recurs leaks its bytes
2282
- * forever. Files younger than the grace window are left alone: a save
2283
- * renames the artifact into place BEFORE its row commits, and its temp
2284
- * exists while the bytes are still being written, so a fresh file
2285
- * without a row is a save in flight, not an orphan.
2336
+ * Unlink the row-less files `pickVictims` took: a `<hash>.tar.zst` with
2337
+ * no `entries` row (another vx version's open dropped it, a `cache.db`
2338
+ * deleted by hand) past the policy, and a `<hash>.tar.zst.tmp-*` a
2339
+ * crashed save never renamed. Nothing else reaps them — a lookup starts
2340
+ * at the row, and a save of the same key renames over the file, so a key
2341
+ * that never recurs leaks its bytes forever.
2286
2342
  */
2287
2343
  private async reapOrphans(
2288
- now: number = Date.now(),
2344
+ taken: readonly RowlessFile[],
2345
+ now: number,
2289
2346
  ): Promise<{ orphans: number; orphanBytes: number }> {
2290
2347
  try {
2291
2348
  this.db
@@ -2298,7 +2355,7 @@ export class Cache implements CacheLayer {
2298
2355
  let orphans = 0
2299
2356
  let orphanBytes = 0
2300
2357
  await Promise.all(
2301
- (await this.scanOrphans()).map(async (o) => {
2358
+ taken.map(async (o) => {
2302
2359
  try {
2303
2360
  // unlink, not `rm({ force })`: force swallows ENOENT, and a file a
2304
2361
  // concurrent prune took first must not be counted as ours.
@@ -2322,7 +2379,11 @@ export class Cache implements CacheLayer {
2322
2379
  * dropped — a row is never deleted on one listing's word while it may be
2323
2380
  * in use. An unreadable directory judges nothing.
2324
2381
  */
2325
- private async phantomRows(): Promise<{ phantoms: Map<string, number>; stale: string[] }> {
2382
+ private async phantomRows(): Promise<{
2383
+ phantoms: Map<string, number>
2384
+ stale: string[]
2385
+ rows: Array<{ hash: string; size_bytes: number; accessed_at: number }>
2386
+ }> {
2326
2387
  const graceStart = Date.now() - ORPHAN_GRACE_MS
2327
2388
  const rows = this.db
2328
2389
  .prepare('SELECT hash, size_bytes, accessed_at FROM entries')
@@ -2335,7 +2396,7 @@ export class Cache implements CacheLayer {
2335
2396
  try {
2336
2397
  names = await readdir(this.artifactDir)
2337
2398
  } catch {
2338
- return { phantoms: new Map(), stale: [] }
2399
+ return { phantoms: new Map(), stale: [], rows }
2339
2400
  }
2340
2401
  const present = new Set(names)
2341
2402
  const phantoms = new Map<string, number>()
@@ -2345,10 +2406,10 @@ export class Cache implements CacheLayer {
2345
2406
  phantoms.set(r.hash, r.size_bytes)
2346
2407
  if (r.accessed_at < graceStart) stale.push(r.hash)
2347
2408
  }
2348
- return { phantoms, stale }
2409
+ return { phantoms, stale, rows }
2349
2410
  }
2350
2411
 
2351
- /** What `prune()` would reap right now, for `vx info` to say before anyone prunes. */
2412
+ /** Row-less files past the in-flight grace, for `vx info`: a prune reaps them by its policy. */
2352
2413
  async orphanStats(): Promise<{ orphans: number; orphanBytes: number }> {
2353
2414
  const found = await this.scanOrphans()
2354
2415
  let orphanBytes = 0
@@ -2357,7 +2418,7 @@ export class Cache implements CacheLayer {
2357
2418
  }
2358
2419
 
2359
2420
  /** Row-less artifacts and temps past the in-flight grace window: one readdir, one stat per candidate. */
2360
- private async scanOrphans(): Promise<Array<{ file: string; size: number }>> {
2421
+ private async scanOrphans(): Promise<RowlessFile[]> {
2361
2422
  const found = await scanOrphanFiles(
2362
2423
  this.artifactDir,
2363
2424
  () =>