@vzn/vx 0.0.50 → 0.0.52

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/README.md +38 -31
  2. package/package.json +5 -5
  3. package/src/bin.ts +1 -1
  4. package/src/cache/archive.ts +18 -6
  5. package/src/cache/cache.ts +150 -67
  6. package/src/cache/chained-cache.ts +9 -2
  7. package/src/cache/file-hashes.ts +32 -3
  8. package/src/cache/git-inputs.ts +96 -11
  9. package/src/cache/index.ts +1 -0
  10. package/src/cache/layer.ts +28 -3
  11. package/src/cache/layered-cache.ts +7 -2
  12. package/src/cache/output-index.ts +16 -7
  13. package/src/cache/run-history.ts +5 -1
  14. package/src/cache/zstd.ts +40 -8
  15. package/src/cli/cache.ts +35 -9
  16. package/src/cli/completions.ts +10 -7
  17. package/src/cli/help.ts +48 -4
  18. package/src/cli/index.ts +5 -3
  19. package/src/cli/info.ts +1 -1
  20. package/src/cli/plugin-commands.ts +10 -2
  21. package/src/cli/run.ts +4 -0
  22. package/src/cli/select.ts +32 -2
  23. package/src/cli/upgrade.ts +110 -22
  24. package/src/cli/watch.ts +38 -20
  25. package/src/config.ts +18 -5
  26. package/src/exec/executor.ts +3 -2
  27. package/src/exec/index.ts +1 -1
  28. package/src/exec/runner.ts +102 -24
  29. package/src/exec/sandbox-binds.ts +1 -1
  30. package/src/exec/sandbox-runtime.ts +149 -40
  31. package/src/exec/sandbox-violations.ts +42 -10
  32. package/src/graph/scheduler.ts +11 -5
  33. package/src/graph/task-graph.ts +96 -12
  34. package/src/index.ts +1 -8
  35. package/src/orchestrator/admission.ts +72 -17
  36. package/src/orchestrator/execute-task.ts +39 -25
  37. package/src/orchestrator/hit-restore.ts +7 -1
  38. package/src/orchestrator/miss-save.ts +29 -1
  39. package/src/orchestrator/options.ts +8 -1
  40. package/src/orchestrator/placement.ts +23 -3
  41. package/src/orchestrator/plugin-host.ts +1 -1
  42. package/src/orchestrator/plugin.ts +9 -1
  43. package/src/orchestrator/prepare.ts +28 -10
  44. package/src/orchestrator/run.ts +75 -12
  45. package/src/orchestrator/sandbox-request.ts +19 -10
  46. package/src/orchestrator/summary.ts +26 -15
  47. package/src/orchestrator/task-hash.ts +7 -2
  48. package/src/orchestrator/telemetry.ts +14 -4
  49. package/src/util/paths.ts +16 -9
  50. package/src/workspace/affected.ts +201 -38
  51. package/src/workspace/config-cache.ts +0 -2
  52. package/src/workspace/config-eval.ts +4 -1
  53. package/src/workspace/config-schema.ts +70 -10
  54. package/src/workspace/fingerprint.ts +6 -0
  55. package/src/workspace/migration.ts +1 -1
  56. package/src/workspace/project-loader.ts +53 -1
  57. package/src/workspace/workspace.ts +54 -14
package/README.md CHANGED
@@ -101,10 +101,12 @@ with the invariant that keeps it valid —
101
101
 
102
102
  ## Built for trust
103
103
 
104
- - **Corruption can't go live.** A remote artifact is verified against
105
- its content digest and validated before it enters the store
106
- (zstd-bomb and oversize downloads refused); bad bytes degrade to a
107
- cache miss, never a wrong hit and never a crash.
104
+ - **Corruption can't go live.** A remote artifact is validated before
105
+ it enters the store: a zstd bomb, bytes that are not a vx archive,
106
+ and an archive recording another key than the one it was fetched
107
+ under are refused, and `@vzn/vx-reapi` re-hashes every blob against
108
+ its content digest; bad bytes degrade to a cache miss, never a wrong
109
+ hit and never a crash.
108
110
  - **Clean exits.** SIGINT/SIGTERM reap every child process — no
109
111
  orphaned dev servers in CI.
110
112
  - **Readiness you can bound.** Persistent tasks gate downstream work
@@ -129,13 +131,15 @@ vx lock # evaluate everything once, write vx-lock.json
129
131
  vx lock --check && vx run ci --frozen # CI: audit, then run EXACTLY that graph
130
132
  ```
131
133
 
132
- | Command | Evaluates configs | Uses lock |
133
- | ----------------- | ----------------- | --------------------------------------------------------------- |
134
- | `vx run` | always, live | never — local truth has no asterisks |
135
- | `vx run --frozen` | never | yes; refuses if absent or a config file changed since locking |
136
- | `vx lock --check` | full graph | compares — catches env and import drift that byte hashes cannot |
134
+ | Command | Evaluates configs | Uses lock |
135
+ | ----------------- | ----------------- | ---------------------------------------------------------------- |
136
+ | `vx run` | always, live | never — local truth has no asterisks |
137
+ | `vx run --frozen` | never | yes, as written; refuses only a missing lock or project entry |
138
+ | `vx lock --check` | full graph | compares — a changed file, and env or import drift it cannot see |
137
139
 
138
- Env values read at lock time are frozen by design — cache keys become
140
+ `--frozen` checks nothing about staleness itself: an edited config runs
141
+ its locked form until `vx lock` re-writes it, and `vx lock --check` is
142
+ the guard that fails first. Env values read at lock time are frozen by design — cache keys become
139
143
  reproducible across machines. Take it for that, not for speed: measured
140
144
  on the 1,000-project bench, a plain warm run and a frozen one are a tie
141
145
  (177 ms against 165, 2026-09-12), because the config-evaluation cache
@@ -162,17 +166,20 @@ task graph, derive keys, schedule, execute, cache, observe. Plugins
162
166
  declared in `vx.workspace.ts` hook each stage, Vite-style, on one
163
167
  `VxPlugin` object:
164
168
 
165
- | Stage | Hook | A plugin can… |
166
- | --------- | -------------------------- | ----------------------------------------------------------- |
167
- | workspace | `config(ws, ctx)` | edit the workspace config before it is used |
168
- | project | `project(config, ctx)` | add, remove or rewrite a project's tasks (keyed like yours) |
169
- | graph | `graph(nodes, ctx)` | add or drop edges, mark tasks requested |
170
- | key | `key(task, ctx)` | fold extra material into the cache key (named in `vx why`) |
171
- | schedule | `schedule(nodes, ctx)` | decide which ready task runs first |
172
- | execute | `executor(ctx)` | decide WHERE a task's command runs (local, a REAPI worker) |
173
- | store | `cache(ctx)` | decide where artifacts live (local, a shared remote) |
174
- | observe | `telemetry(ctx)` / `setup` | receive every run record, or the raw event bus |
175
- | cli | `commands` | add verbs to `vx` |
169
+ | Stage | Hook | A plugin can… |
170
+ | --------- | -------------------------- | ------------------------------------------------------------ |
171
+ | workspace | `config(ws, ctx)` | edit the workspace config before it is used |
172
+ | project | `project(config, ctx)` | add, remove or rewrite a project's tasks (keyed like yours) |
173
+ | graph | `graph(nodes, ctx)` | add or drop edges, mark tasks requested |
174
+ | key | `key(task, ctx)` | fold extra material into the cache key (named in `vx why`) |
175
+ | key | `fingerprint` (a claim) | claim a root file (a lockfile) and key each task on its part |
176
+ | schedule | `schedule(nodes, ctx)` | decide which ready task runs first |
177
+ | schedule | `admit(task, ctx)` | hold a ready task until something running here finishes |
178
+ | execute | `executor(ctx)` | decide WHERE a task's command runs (local, a REAPI worker) |
179
+ | store | `cache(ctx)` | decide where artifacts live (local, a shared remote) |
180
+ | observe | `telemetry(ctx)` / `setup` | receive every run record, or the raw event bus |
181
+ | run | `setup` / `teardown` | acquire before the run, release after it |
182
+ | cli | `commands` | add verbs to `vx` |
176
183
 
177
184
  Core applies **no plugin by default** and names none. Running here and
178
185
  caching here are its floor — the tail of every executor list and cache
@@ -194,10 +201,9 @@ Context Protocol server for AI coding agents, no SDK),
194
201
  artifacts are vx's),
195
202
  [`@vzn/vx-lockfile`](packages/vx-lockfile) (`pnpm()`, `bun()`, `npm()`,
196
203
  `yarn()`: the lockfile keyed per project, so one install re-keys only the
197
- projects it reaches, and `--affected` follows), and [`@vzn/vx-schedule-history`](packages/vx-schedule-history)
198
- (order by learned critical path) and [`@vzn/vx-migrate`](packages/vx-migrate)
199
- (`bunx @vzn/vx-migrate`: turbo.json or an Nx graph → vx.config.ts, through
200
- core's migration seam). Core ships no plugin and reads no other runner's
204
+ projects it reaches, and `--affected` follows) and [`@vzn/vx-schedule-history`](packages/vx-schedule-history)
205
+ (order by learned critical path, and pack admission by what past runs
206
+ used). Core ships no plugin and reads no other runner's
201
207
  format; nothing
202
208
  distributed ships in this repo; the seams are how you build it.
203
209
 
@@ -205,7 +211,7 @@ distributed ships in this repo; the seams are how you build it.
205
211
 
206
212
  | | vx | Turborepo | Nx |
207
213
  | ------------------------- | ---------------------------------------------------------------- | ------------------------------ | ---------------- |
208
- | Fully cached, 100 pkgs¹ | **144 ms** | 279 ms | 583+ ms |
214
+ | Fully cached, 1,090 pkgs¹ | **510 ms** | 760 ms | 3.59 s |
209
215
  | Config | TypeScript, evaluated into the cache key | JSON (static) | JSON (static) |
210
216
  | Output ownership | **Strict** — wiped before exec AND restore | Additive (stale files survive) | Additive |
211
217
  | Clean-tree hashing | **Zero reads** (git index OIDs) | git OIDs | re-hash / daemon |
@@ -215,9 +221,10 @@ distributed ships in this repo; the seams are how you build it.
215
221
  | OTel CI/CD spans | **Yes** — `otel()` plugin, zero OTel-SDK deps | No | Paid |
216
222
  | Install | **Single binary** — npm, or one release file; no Node/Bun needed | npm + Node | npm + Node |
217
223
 
218
- ¹ Wall-clock, direct binaries, same machine and workspace — full
219
- methodology and more scenarios in
220
- [`packages/vx/docs/benchmarks.md`](packages/vx/docs/benchmarks.md).
224
+ ¹ Warm, nothing to rebuild, on the 3,270-task graph quoted at the top:
225
+ wall-clock, direct binaries, same machine and workspace — methodology
226
+ and more scenarios in
227
+ [`packages/vx/docs/benchmarks.md`](packages/vx/docs/benchmarks.md) § A real monorepo.
221
228
 
222
229
  ## Switching from another runner
223
230
 
@@ -309,7 +316,7 @@ maintainers' handoff is [`packages/vx/docs/STATUS.md`](packages/vx/docs/STATUS.m
309
316
 
310
317
  **Pre-alpha.** The schema is settling; we bump `CACHE_VERSION` rather
311
318
  than maintain back-compat. What 1.0 will promise is in
312
- [`versioning-1.0.md`](packages/vx/docs/design/versioning-1.0.md). **~3,000 core tests plus the package suites; CI green on every commit**;
319
+ [`versioning-1.0.md`](packages/vx/docs/design/versioning-1.0.md). **over 4,000 core tests plus the package suites; CI green on every commit**;
313
320
  the project dogfoods itself (`vx run ci`). Published on npm:
314
321
  [`@vzn/vx`](https://www.npmjs.com/package/@vzn/vx) (a prebuilt standalone
315
322
  binary).
@@ -321,7 +328,7 @@ build.
321
328
 
322
329
  | Surface | Maturity | Notes |
323
330
  | -------------------------------------------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- |
324
- | Core task runner + caching | **production-ready** | dogfooded continuously; ~3,000 core tests + the package suites, green |
331
+ | Core task runner + caching | **production-ready** | dogfooded continuously; over 4,000 core tests + the package suites, green |
325
332
  | Plugin pipeline (13 hooks, `commands` included) | **shippable** | crash-isolated, re-validated; the local executor + cache are the floor under every plugin |
326
333
  | `vx init` / `@vzn/vx-migrate` (scripts; Turbo, Nx) | **shippable** | one config per package, TODOs where a source cannot say |
327
334
  | REAPI remote cache + execution (`@vzn/vx-reapi`) | **shippable** | Bazel AC + CAS + Execute; NativeLink / BuildBuddy / Buildbarn / bazel-remote |
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@vzn/vx",
3
- "version": "0.0.50",
3
+ "version": "0.0.52",
4
4
  "description": "An open, extensible monorepo task runner.",
5
5
  "type": "module",
6
6
  "exports": {
@@ -17,10 +17,10 @@
17
17
  "node": ">=18"
18
18
  },
19
19
  "optionalDependencies": {
20
- "@vzn/vx-linux-x64": "0.0.50",
21
- "@vzn/vx-linux-arm64": "0.0.50",
22
- "@vzn/vx-darwin-x64": "0.0.50",
23
- "@vzn/vx-darwin-arm64": "0.0.50"
20
+ "@vzn/vx-linux-x64": "0.0.52",
21
+ "@vzn/vx-linux-arm64": "0.0.52",
22
+ "@vzn/vx-darwin-x64": "0.0.52",
23
+ "@vzn/vx-darwin-arm64": "0.0.52"
24
24
  },
25
25
  "dependencies": {
26
26
  "@anthropic-ai/sandbox-runtime": "^0.0.76"
package/src/bin.ts CHANGED
@@ -1,4 +1,4 @@
1
- #!/usr/bin/env bun
1
+ #!/usr/bin/env -S bun --no-env-file
2
2
  import { registerCoreAlias, run } from './cli/index.js'
3
3
  import { fsRefusalHint, isFsRefusal, isUserError } from './util/index.js'
4
4
 
@@ -163,7 +163,10 @@ export async function planArtifact(args: PackArgs): Promise<ArtifactPlan> {
163
163
  abs,
164
164
  size: st.size,
165
165
  mode: st.mode & 0o777,
166
- mtime: Math.floor(st.mtimeMs / 1000),
166
+ // ustar's octal field holds no sign: an mtime before 1970 made the
167
+ // header unreadable and every save of it a "corrupt artifact". The
168
+ // sidecar above carries the real value.
169
+ mtime: Math.max(0, Math.floor(st.mtimeMs / 1000)),
167
170
  }
168
171
  }),
169
172
  )
@@ -333,9 +336,14 @@ export async function extractArtifactStream(
333
336
  )
334
337
  }
335
338
  verify?.(provided)
339
+ // The sidecar's stat is the truth, a mode of 000 and an mtime at or
340
+ // before the epoch included (`SOURCE_DATE_EPOCH=0`). Without one, a
341
+ // header mtime of 0 is a producer that wrote none: unknown.
336
342
  await x.commit((name) => {
337
343
  const m = meta[name]
338
- return [m?.[0] ?? 0o644, m?.[1] ?? headerMtime.get(name) ?? 0]
344
+ if (m !== undefined) return [m[0], m[1]]
345
+ const header = headerMtime.get(name)
346
+ return [0o644, header !== undefined && header > 0 ? header : undefined]
339
347
  })
340
348
  } catch (err) {
341
349
  await x.abort()
@@ -623,16 +631,20 @@ class Extractor {
623
631
  * restores' round trips flowing. The chmod is skipped when the mode is
624
632
  * the one the temp file was created with — the common case.
625
633
  */
626
- async commit(metaFor: (name: string) => [mode: number, mtimeMs: number]): Promise<void> {
634
+ async commit(
635
+ metaFor: (name: string) => [mode: number, mtimeMs: number | undefined],
636
+ ): Promise<void> {
627
637
  await this.drain()
628
638
  const first = this.staged[0]
629
639
  const createdMode = first === undefined ? -1 : statSync(first.tmp).mode & 0o777
630
640
  let n = 0
631
641
  for (const s of this.staged) {
632
642
  const [mode, mtimeMs] = metaFor(s.name)
633
- if (mode !== 0 && (mode & 0o777) !== createdMode) chmodSync(s.tmp, mode & 0o777)
634
- if (mtimeMs > 0) {
635
- const t = mtimeMs / 1000
643
+ if ((mode & 0o777) !== createdMode) chmodSync(s.tmp, mode & 0o777)
644
+ if (mtimeMs !== undefined) {
645
+ // A Date, not seconds: Bun reads a negative number of seconds as
646
+ // "now", and a Date before 1970 as itself (Bun 1.4.2).
647
+ const t = new Date(mtimeMs)
636
648
  utimesSync(s.tmp, t, t)
637
649
  }
638
650
  renameSync(s.tmp, s.target)
@@ -26,8 +26,8 @@
26
26
  // layer speaks is `CacheLayer` in layer.ts; `plugin-host.ts` enforces it.
27
27
 
28
28
  import { Database, type SQLQueryBindings } from 'bun:sqlite'
29
- import { accessSync, constants, existsSync, mkdirSync, writeFileSync } from 'node:fs'
30
- import { readdir, rename, rm, stat, unlink, writeFile } from 'node:fs/promises'
29
+ import { accessSync, constants, existsSync, mkdirSync, renameSync, writeFileSync } from 'node:fs'
30
+ import { readdir, rm, stat, unlink, writeFile } from 'node:fs/promises'
31
31
  import path from 'node:path'
32
32
  import {
33
33
  UserError,
@@ -188,6 +188,46 @@ function schemaOrdinal(version: string): number {
188
188
  return m === null ? -1 : Number(m[1])
189
189
  }
190
190
 
191
+ /**
192
+ * Artifacts `indexed` does not hold, and temps, past the in-flight grace
193
+ * window: one readdir, one stat per candidate. `indexed` is asked only
194
+ * once the readdir succeeded: with the directory gone, macOS answers a
195
+ * read of the index SQLITE_IOERR_VNODE, and an unreadable directory
196
+ * reports nothing.
197
+ */
198
+ async function scanOrphanFiles(
199
+ cacheDir: string,
200
+ indexedHashes: () => ReadonlySet<string>,
201
+ ): Promise<Array<{ file: string; size: number }>> {
202
+ let names: string[]
203
+ try {
204
+ names = await readdir(cacheDir)
205
+ } catch {
206
+ return []
207
+ }
208
+ const indexed = indexedHashes()
209
+ const cutoff = Date.now() - ORPHAN_GRACE_MS
210
+ const candidates: string[] = []
211
+ for (const name of names) {
212
+ const m = VX_ARTIFACT_NAME.exec(name)
213
+ if (m === null) continue
214
+ if (m[2] !== undefined || !indexed.has(m[1]!)) candidates.push(name)
215
+ }
216
+ const found: Array<{ file: string; size: number }> = []
217
+ await Promise.all(
218
+ candidates.map(async (name) => {
219
+ const file = path.join(cacheDir, name)
220
+ try {
221
+ const st = await stat(file)
222
+ if (st.isFile() && st.mtimeMs <= cutoff) found.push({ file, size: st.size })
223
+ } catch {
224
+ // Gone between readdir and stat: not an orphan any more.
225
+ }
226
+ }),
227
+ )
228
+ return found
229
+ }
230
+
191
231
  /**
192
232
  * SQL predicate selecting `runs` rows that record an EXECUTION.
193
233
  *
@@ -400,6 +440,41 @@ export class Cache implements CacheLayer {
400
440
  return new Cache(cacheDir, undefined, undefined, undefined, 'inspect')
401
441
  }
402
442
 
443
+ /**
444
+ * What a real prune reaps from an index of an EARLIER schema, which it
445
+ * resets first, leaving every aged artifact row-less: a dry run that
446
+ * refused it could not preview the biggest prune there is, the one
447
+ * after an upgrade (item 1083). Reads the recorded version alone and
448
+ * touches nothing. Null for an absent, current, newer or unreadable
449
+ * index; `Cache.inspect` answers those.
450
+ */
451
+ static async orphansBeforeReset(
452
+ cacheDir: string,
453
+ ): Promise<{ found: string; orphans: number; orphanBytes: number } | null> {
454
+ const dbFile = path.join(cacheDir, 'cache.db')
455
+ if (!existsSync(dbFile)) return null
456
+ let found: string | undefined
457
+ try {
458
+ const db = new Database(dbFile, { readonly: true })
459
+ try {
460
+ found = (
461
+ db.prepare("SELECT value FROM schema_meta WHERE key = 'version'").get() as {
462
+ value: string
463
+ } | null
464
+ )?.value
465
+ } finally {
466
+ db.close()
467
+ }
468
+ } catch {
469
+ return null
470
+ }
471
+ if (found === undefined || schemaOrdinal(found) >= schemaOrdinal(SCHEMA_VERSION)) return null
472
+ const aged = await scanOrphanFiles(cacheDir, () => new Set())
473
+ let orphanBytes = 0
474
+ for (const o of aged) orphanBytes += o.size
475
+ return { found, orphans: aged.length, orphanBytes }
476
+ }
477
+
403
478
  constructor(
404
479
  private readonly cacheDir: string,
405
480
  localPolicy: { read: boolean; write: boolean } = { read: true, write: true },
@@ -785,7 +860,7 @@ export class Cache implements CacheLayer {
785
860
  const format = this.db
786
861
  .prepare("SELECT value FROM schema_meta WHERE key = 'cache_version'")
787
862
  .get() as { value: string } | null
788
- if (format?.value !== CACHE_VERSION && this.writeBlocked === null) {
863
+ if (format?.value !== CACHE_VERSION && this.writeBlocked === null && !this.inspecting) {
789
864
  const hasEntries = this.db.prepare('SELECT 1 FROM entries LIMIT 1').get() != null
790
865
  if (this.schemaReset === null && (format !== null || hasEntries)) {
791
866
  this.formatChange = { from: format?.value ?? 'an earlier format', to: CACHE_VERSION }
@@ -976,8 +1051,9 @@ export class Cache implements CacheLayer {
976
1051
  hash: string,
977
1052
  projectDir: string,
978
1053
  prefixes: readonly string[],
1054
+ holds?: (files: readonly string[]) => boolean,
979
1055
  ): Promise<void> {
980
- await this.outputs.recordOutputDirs(hash, projectDir, prefixes)
1056
+ await this.outputs.recordOutputDirs(hash, projectDir, prefixes, holds)
981
1057
  }
982
1058
  recordOutputStamps(hash: string, projectDir: string, workspaceRoot: string): void {
983
1059
  this.outputs.recordOutputStamps(hash, projectDir, workspaceRoot)
@@ -1070,8 +1146,11 @@ export class Cache implements CacheLayer {
1070
1146
  }
1071
1147
  // What is on disk, not what is in the archive: a directory standing
1072
1148
  // where the entry holds a file, or a file where it needs a directory.
1073
- // The clean removes everything the output globs cover, so this is a
1074
- // stray they do not — name it as such, not as a corrupt artifact. A
1149
+ // The clean removes the FILES the output globs select and the
1150
+ // directories that removal emptied, so this is a path it left: one
1151
+ // the globs do not cover, or an empty directory where a file goes,
1152
+ // which `dist/**` covers and the clean does not take (item 1094).
1153
+ // Named as such, not as a corrupt artifact. A
1075
1154
  // link on the way never lands here: the extractor writes through one
1076
1155
  // that stays in the project, dangling or not, and names one that
1077
1156
  // leaves it or loops (item 748).
@@ -1079,7 +1158,7 @@ export class Cache implements CacheLayer {
1079
1158
  if (code === 'EISDIR' || code === 'ENOTDIR' || code === 'EEXIST' || code === 'ENOTEMPTY') {
1080
1159
  throw new UserError(
1081
1160
  `restore of ${hash} into ${projectDir} was blocked by what is on disk (${code}: ${(err as Error).message}). ` +
1082
- `Declared outputs are wiped before a restore, so this is a path the output globs do not cover — remove it and re-run.`,
1161
+ `The clean before a restore removes the files the output globs select, not a directory standing where the entry holds a file, nor a path the globs do not cover — remove it and re-run.`,
1083
1162
  )
1084
1163
  }
1085
1164
  // A legal name under a destination deep enough that the two together
@@ -1405,18 +1484,8 @@ export class Cache implements CacheLayer {
1405
1484
  throw new CorruptArtifactError(hash, 'artifact is not a readable archive', err)
1406
1485
  }
1407
1486
  const { entries } = scanned
1408
- // POSIX rename atomically REPLACES the destination if it exists,
1409
- // so we don't need a pre-rm. The pre-rm was actively harmful —
1410
- // it opened a race window where writer B could delete writer A's
1411
- // just-renamed file BEFORE A's subsequent stat, producing a
1412
- // spurious ENOENT. The rename itself preserves the "either-or"
1413
- // semantics for concurrent readers.
1414
- const endRename = span('save: rename')
1415
- await rename(tmpPath, finalPath)
1416
- endRename()
1417
-
1418
1487
  const totalBytes =
1419
- compressed instanceof Uint8Array ? compressed.byteLength : Bun.file(finalPath).size
1488
+ compressed instanceof Uint8Array ? compressed.byteLength : Bun.file(tmpPath).size
1420
1489
  const outputFileRows: Array<[string, number, number, number]> = []
1421
1490
  // Per-output-file fingerprint rows feed the skip-restore check.
1422
1491
  // Row paths: project entries store the bare rel (`outputs/`
@@ -1455,7 +1524,23 @@ export class Cache implements CacheLayer {
1455
1524
  const outputs = this.outputs
1456
1525
  const insertEntryInput = this.insertEntryInput
1457
1526
  const inputComponents = meta.inputComponents
1527
+ // The artifact goes live INSIDE the write transaction, taken
1528
+ // IMMEDIATE: the rename happens only once this writer holds the
1529
+ // database's write lock, and its rows commit before any other writer
1530
+ // can rename. Renamed first and indexed after, a commit that failed
1531
+ // (`SQLITE_BUSY` past the busy timeout, `SQLITE_FULL`) left these bytes
1532
+ // beside the previous save's rows, and two writers of one key (two
1533
+ // workspaces on one `--cache-dir`) could pair one's bytes with the
1534
+ // other's rows: every later hit failed the task on an artifact
1535
+ // "missing" a recorded output (A-3). POSIX rename replaces the
1536
+ // destination atomically, so a reader holding the old file reads it
1537
+ // whole. A commit that fails after the rename takes the artifact back
1538
+ // out: the old rows then name no artifact, which a probe reads as a
1539
+ // miss.
1540
+ let renamed = false
1458
1541
  const tx = this.db.transaction(() => {
1542
+ renameSync(tmpPath, finalPath)
1543
+ renamed = true
1459
1544
  insertEntry.run(
1460
1545
  hash,
1461
1546
  project,
@@ -1487,8 +1572,14 @@ export class Cache implements CacheLayer {
1487
1572
  }
1488
1573
  })
1489
1574
  const endTx = span('save: index tx')
1490
- tx()
1491
- endTx()
1575
+ try {
1576
+ tx.immediate()
1577
+ } catch (err) {
1578
+ await unlink(renamed ? finalPath : tmpPath).catch(() => undefined)
1579
+ throw err
1580
+ } finally {
1581
+ endTx()
1582
+ }
1492
1583
  }
1493
1584
 
1494
1585
  /** Apply the deferred accessed_at bumps in one statement. */
@@ -1618,9 +1709,10 @@ export class Cache implements CacheLayer {
1618
1709
  let bytesFreed = 0
1619
1710
  // Rows whose artifact is gone (deleted by hand, or by a disk cleaner):
1620
1711
  // never a hit, and their bytes are on no disk, but `--max-size` counted
1621
- // them and evicted real entries to make room for them (item 975). They
1622
- // are dropped, neither evicted nor freed.
1623
- const phantoms = await this.phantomRows()
1712
+ // them and evicted real entries to make room for them (item 975). None
1713
+ // is evicted or freed, whatever its age (item 1081); the ones past the
1714
+ // grace window are dropped.
1715
+ const { phantoms, stale } = await this.phantomRows()
1624
1716
 
1625
1717
  if (olderThanMs !== undefined) {
1626
1718
  const rows = this.db
@@ -1673,9 +1765,9 @@ export class Cache implements CacheLayer {
1673
1765
  orphanBytes: orphans.orphanBytes,
1674
1766
  }
1675
1767
  }
1676
- if (victims.size > 0 || phantoms.size > 0) {
1768
+ if (victims.size > 0 || stale.length > 0) {
1677
1769
  const hashes = [...victims]
1678
- const rows = [...hashes, ...phantoms.keys()]
1770
+ const rows = [...hashes, ...stale]
1679
1771
  this.db.transaction(() => {
1680
1772
  for (let i = 0; i < rows.length; i += 900) {
1681
1773
  const chunk = rows.slice(i, i + 900)
@@ -1728,25 +1820,37 @@ export class Cache implements CacheLayer {
1728
1820
 
1729
1821
  /**
1730
1822
  * Index rows whose artifact is not in the directory, with their recorded
1731
- * sizes. Only rows last written or used before the grace window: a save
1732
- * renames its artifact in before its row commits, so a fresh row without
1733
- * one is a listing that raced a save, not a phantom. An unreadable
1734
- * directory judges nothing.
1823
+ * sizes, and the ones among them last used before the grace window. The
1824
+ * rows are read BEFORE the directory: a save renames its artifact in
1825
+ * before its row commits, so every row read has its artifact listed
1826
+ * unless it is gone, and the count is exact. Only `stale` rows are
1827
+ * dropped — a row is never deleted on one listing's word while it may be
1828
+ * in use. An unreadable directory judges nothing.
1735
1829
  */
1736
- private async phantomRows(): Promise<Map<string, number>> {
1830
+ private async phantomRows(): Promise<{ phantoms: Map<string, number>; stale: string[] }> {
1831
+ const graceStart = Date.now() - ORPHAN_GRACE_MS
1832
+ const rows = this.db
1833
+ .prepare('SELECT hash, size_bytes, accessed_at FROM entries')
1834
+ .all() as Array<{
1835
+ hash: string
1836
+ size_bytes: number
1837
+ accessed_at: number
1838
+ }>
1737
1839
  let names: string[]
1738
1840
  try {
1739
1841
  names = await readdir(this.cacheDir)
1740
1842
  } catch {
1741
- return new Map()
1843
+ return { phantoms: new Map(), stale: [] }
1742
1844
  }
1743
1845
  const present = new Set(names)
1744
- const rows = this.db
1745
- .prepare('SELECT hash, size_bytes FROM entries WHERE accessed_at < ?')
1746
- .all(Date.now() - ORPHAN_GRACE_MS) as Array<{ hash: string; size_bytes: number }>
1747
- const out = new Map<string, number>()
1748
- for (const r of rows) if (!present.has(`${r.hash}.tar.zst`)) out.set(r.hash, r.size_bytes)
1749
- return out
1846
+ const phantoms = new Map<string, number>()
1847
+ const stale: string[] = []
1848
+ for (const r of rows) {
1849
+ if (present.has(`${r.hash}.tar.zst`)) continue
1850
+ phantoms.set(r.hash, r.size_bytes)
1851
+ if (r.accessed_at < graceStart) stale.push(r.hash)
1852
+ }
1853
+ return { phantoms, stale }
1750
1854
  }
1751
1855
 
1752
1856
  /** What `prune()` would reap right now, for `vx info` to say before anyone prunes. */
@@ -1759,37 +1863,15 @@ export class Cache implements CacheLayer {
1759
1863
 
1760
1864
  /** Row-less artifacts and temps past the in-flight grace window: one readdir, one stat per candidate. */
1761
1865
  private async scanOrphans(): Promise<Array<{ file: string; size: number }>> {
1762
- let names: string[]
1763
- try {
1764
- names = await readdir(this.cacheDir)
1765
- } catch {
1766
- return []
1767
- }
1768
- const indexed = new Set(
1769
- (this.db.prepare('SELECT hash FROM entries').all() as Array<{ hash: string }>).map(
1770
- (r) => r.hash,
1771
- ),
1772
- )
1773
- const cutoff = Date.now() - ORPHAN_GRACE_MS
1774
- const candidates: string[] = []
1775
- for (const name of names) {
1776
- const m = VX_ARTIFACT_NAME.exec(name)
1777
- if (m === null) continue
1778
- if (m[2] !== undefined || !indexed.has(m[1]!)) candidates.push(name)
1779
- }
1780
- const found: Array<{ file: string; size: number }> = []
1781
- await Promise.all(
1782
- candidates.map(async (name) => {
1783
- const file = path.join(this.cacheDir, name)
1784
- try {
1785
- const st = await stat(file)
1786
- if (st.isFile() && st.mtimeMs <= cutoff) found.push({ file, size: st.size })
1787
- } catch {
1788
- // Gone between readdir and stat: not an orphan any more.
1789
- }
1790
- }),
1866
+ return scanOrphanFiles(
1867
+ this.cacheDir,
1868
+ () =>
1869
+ new Set(
1870
+ (this.db.prepare('SELECT hash FROM entries').all() as Array<{ hash: string }>).map(
1871
+ (r) => r.hash,
1872
+ ),
1873
+ ),
1791
1874
  )
1792
- return found
1793
1875
  }
1794
1876
 
1795
1877
  close(): void {
@@ -1810,6 +1892,7 @@ export class Cache implements CacheLayer {
1810
1892
  const cutoff = Date.now() - 30 * 24 * 60 * 60 * 1000
1811
1893
  this.history.pruneOlderThan(cutoff)
1812
1894
  this.configEvals.pruneOlderThan(cutoff)
1895
+ if (this.write) this.files.pruneOlderThan(cutoff)
1813
1896
  }
1814
1897
  } catch {
1815
1898
  // Retention is best-effort; never block closing the handle.
@@ -161,8 +161,15 @@ export class ChainedCache implements CacheLayer {
161
161
 
162
162
  // The directory short-circuit lives with the layer that owns the local
163
163
  // rows — the first, like the file check.
164
- recordOutputDirs(hash: string, projectDir: string, prefixes: readonly string[]): Promise<void> {
165
- return this.layers[0]!.recordOutputDirs?.(hash, projectDir, prefixes) ?? Promise.resolve()
164
+ recordOutputDirs(
165
+ hash: string,
166
+ projectDir: string,
167
+ prefixes: readonly string[],
168
+ holds?: (files: readonly string[]) => boolean,
169
+ ): Promise<void> {
170
+ return (
171
+ this.layers[0]!.recordOutputDirs?.(hash, projectDir, prefixes, holds) ?? Promise.resolve()
172
+ )
166
173
  }
167
174
  recordOutputStamps(hash: string, projectDir: string, workspaceRoot: string): void {
168
175
  this.layers[0]!.recordOutputStamps?.(hash, projectDir, workspaceRoot)
@@ -10,7 +10,10 @@ import type { Database } from 'bun:sqlite'
10
10
  import { lstatSync, readlinkSync } from 'node:fs'
11
11
  import path from 'node:path'
12
12
  import { fileIdentity, repoFacts } from './git-inputs.js'
13
- import { FILE_HASH_RACY_MS } from './layer.js'
13
+ import { FILE_HASH_RACY_MS, racyWindowMs } from './layer.js'
14
+
15
+ const FILE_HASHES_SWEPT_AT = 'file_hashes_swept_at'
16
+ const DAY_MS = 24 * 60 * 60 * 1000
14
17
 
15
18
  export class FileHashStore {
16
19
  private readonly selectFileHash: ReturnType<Database['prepare']>
@@ -47,6 +50,29 @@ export class FileHashStore {
47
50
  `)
48
51
  }
49
52
 
53
+ /**
54
+ * Drop the memo rows last written before `cutoff`: a path that is gone
55
+ * (a deleted generated input, a worktree removed, a CI checkout at a new
56
+ * path) keeps its row forever otherwise, and `cache.db` grows with every
57
+ * path ever hashed. A hit does not refresh `seen_at` (a write per file
58
+ * per run), so a file unchanged for the window loses its row too and is
59
+ * read once more; a dropped row is only a memo miss. The scan costs
60
+ * 8 ms at 100,000 rows, so it runs once a day, on its own clock in
61
+ * `schema_meta` (item 1082).
62
+ */
63
+ pruneOlderThan(cutoff: number, now: number = Date.now()): void {
64
+ const last = this.db
65
+ .prepare('SELECT value FROM schema_meta WHERE key = ?')
66
+ .get(FILE_HASHES_SWEPT_AT) as { value: string } | null
67
+ if (last !== null && now - Number(last.value) < DAY_MS) return
68
+ this.db.transaction(() => {
69
+ this.db.prepare('DELETE FROM file_hashes WHERE seen_at < ?').run(cutoff)
70
+ this.db
71
+ .prepare('INSERT OR REPLACE INTO schema_meta(key, value) VALUES (?, ?)')
72
+ .run(FILE_HASHES_SWEPT_AT, String(now))
73
+ })()
74
+ }
75
+
50
76
  /**
51
77
  * Content-hash a file (as a git blob OID, v20) with an mtime+size
52
78
  * fast path. If the `file_hashes` table has a row for `path` whose
@@ -136,7 +162,7 @@ export class FileHashStore {
136
162
  // changed within the window is hashed again on its next call rather
137
163
  // than memoised; the warm path never meets it (keys are derived long
138
164
  // after the files were written).
139
- if (this.write && Date.now() - ctimeMs >= FILE_HASH_RACY_MS) {
165
+ if (this.write && Date.now() - ctimeMs >= racyWindowMs(st.ctimeMs, FILE_HASH_RACY_MS)) {
140
166
  this.upsertFileHash.run(filePath, mtimeMs, size, ctimeMs, ino, ch, Date.now())
141
167
  }
142
168
  return fileIdentity(mode, ch)
@@ -161,6 +187,8 @@ export class FileHashStore {
161
187
  mtimeMs: number
162
188
  size: number
163
189
  ctimeMs: number
190
+ /** Unfloored, for the racy window: a whole second is a whole-second clock only here. */
191
+ ctimeRaw: number
164
192
  ino: number
165
193
  }
166
194
  const stats = new Map<string, Stat>()
@@ -188,6 +216,7 @@ export class FileHashStore {
188
216
  mtimeMs: Math.floor(st.mtimeMs),
189
217
  size: st.size,
190
218
  ctimeMs: Math.floor(st.ctimeMs),
219
+ ctimeRaw: st.ctimeMs,
191
220
  ino: Number(st.ino),
192
221
  })
193
222
  } catch {
@@ -241,7 +270,7 @@ export class FileHashStore {
241
270
  out.set(p, fileIdentity(st.mode, digest))
242
271
  // The same racy-clean rule as `hashFile`: a stat taken within the
243
272
  // window of the file's last change is not memoised.
244
- if (this.write && now - st.ctimeMs >= FILE_HASH_RACY_MS) {
273
+ if (this.write && now - st.ctimeMs >= racyWindowMs(st.ctimeRaw, FILE_HASH_RACY_MS)) {
245
274
  this.upsertFileHash.run(p, st.mtimeMs, st.size, st.ctimeMs, st.ino, digest, now)
246
275
  }
247
276
  }