@vzn/vx 0.0.18 → 0.0.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/LICENSE +1 -1
  2. package/README.md +107 -70
  3. package/package.json +10 -20
  4. package/src/bin.ts +64 -5
  5. package/src/cache/archive.ts +279 -28
  6. package/src/cache/cache.ts +952 -1584
  7. package/src/cache/chained-cache.ts +14 -10
  8. package/src/cache/config-evals.ts +112 -0
  9. package/src/cache/file-hashes.ts +290 -0
  10. package/src/cache/git-inputs.ts +966 -0
  11. package/src/cache/index.ts +23 -7
  12. package/src/cache/inputs.ts +553 -739
  13. package/src/cache/key-fold.ts +232 -0
  14. package/src/cache/layer.ts +674 -0
  15. package/src/cache/layered-cache.ts +211 -49
  16. package/src/cache/output-index.ts +323 -0
  17. package/src/cache/policy.ts +84 -0
  18. package/src/cache/run-history.ts +152 -0
  19. package/src/cache/zstd.ts +183 -0
  20. package/src/cli/cache.ts +88 -30
  21. package/src/cli/completions.ts +104 -0
  22. package/src/cli/core-alias.ts +25 -0
  23. package/src/cli/format.ts +4 -15
  24. package/src/cli/help.ts +76 -38
  25. package/src/cli/index.ts +82 -17
  26. package/src/cli/info.ts +162 -91
  27. package/src/cli/init.ts +87 -0
  28. package/src/cli/last.ts +130 -23
  29. package/src/cli/lock.ts +36 -18
  30. package/src/cli/plan-format.ts +18 -7
  31. package/src/cli/plugin-commands.ts +34 -12
  32. package/src/cli/run.ts +138 -313
  33. package/src/cli/select.ts +342 -0
  34. package/src/cli/show.ts +151 -71
  35. package/src/cli/upgrade.ts +136 -15
  36. package/src/cli/watch.ts +1175 -116
  37. package/src/cli/why.ts +51 -29
  38. package/src/cli/workspace-config.ts +138 -0
  39. package/src/config.ts +183 -126
  40. package/src/exec/env.ts +11 -0
  41. package/src/exec/executor.ts +62 -31
  42. package/src/exec/index.ts +16 -2
  43. package/src/exec/kill-tree.ts +281 -0
  44. package/src/exec/local-executor.ts +41 -0
  45. package/src/exec/runner.ts +361 -77
  46. package/src/exec/sandbox-binds.ts +211 -0
  47. package/src/exec/sandbox-paths.ts +70 -0
  48. package/src/exec/sandbox-runtime.ts +1152 -415
  49. package/src/exec/sandbox-violations.ts +293 -0
  50. package/src/graph/dependency-spec.ts +14 -14
  51. package/src/graph/index.ts +6 -4
  52. package/src/graph/priorities.ts +130 -0
  53. package/src/graph/scheduler.ts +332 -329
  54. package/src/graph/task-graph.ts +552 -188
  55. package/src/index.ts +111 -102
  56. package/src/orchestrator/admission.ts +161 -0
  57. package/src/orchestrator/colors.ts +8 -3
  58. package/src/orchestrator/deferred-outputs.ts +25 -7
  59. package/src/orchestrator/doctor.ts +363 -0
  60. package/src/orchestrator/events.ts +111 -8
  61. package/src/orchestrator/excluded-keys.ts +155 -0
  62. package/src/orchestrator/execute-task.ts +450 -565
  63. package/src/orchestrator/failure-mode.ts +169 -28
  64. package/src/orchestrator/failure-recap.ts +121 -0
  65. package/src/orchestrator/fingerprint-watch.ts +89 -0
  66. package/src/orchestrator/framed-output.ts +121 -49
  67. package/src/orchestrator/history.ts +123 -104
  68. package/src/orchestrator/hit-restore.ts +285 -0
  69. package/src/orchestrator/index.ts +42 -5
  70. package/src/orchestrator/keyed-projects.ts +89 -0
  71. package/src/orchestrator/local-shortcircuit.ts +83 -23
  72. package/src/orchestrator/lockfile-claim.ts +401 -0
  73. package/src/orchestrator/logger.ts +128 -41
  74. package/src/orchestrator/metrics.ts +201 -72
  75. package/src/orchestrator/miss-save.ts +232 -0
  76. package/src/orchestrator/options.ts +63 -34
  77. package/src/orchestrator/persistent.ts +99 -0
  78. package/src/orchestrator/placement.ts +212 -0
  79. package/src/orchestrator/plan.ts +24 -11
  80. package/src/orchestrator/plugin-host.ts +200 -49
  81. package/src/orchestrator/plugin.ts +190 -14
  82. package/src/orchestrator/prepare.ts +168 -137
  83. package/src/orchestrator/projects.ts +270 -0
  84. package/src/orchestrator/remote-prefetch.ts +22 -2
  85. package/src/orchestrator/run-artifacts.ts +34 -3
  86. package/src/orchestrator/run-context.ts +20 -5
  87. package/src/orchestrator/run-lock.ts +349 -0
  88. package/src/orchestrator/run-records.ts +132 -0
  89. package/src/orchestrator/run-report.ts +6 -4
  90. package/src/orchestrator/run.ts +467 -611
  91. package/src/orchestrator/sandbox-request.ts +569 -0
  92. package/src/orchestrator/save-lane.ts +44 -0
  93. package/src/orchestrator/shell-verdict.ts +117 -0
  94. package/src/orchestrator/signals.ts +167 -0
  95. package/src/orchestrator/stable-keys.ts +259 -10
  96. package/src/orchestrator/status-line.ts +35 -2
  97. package/src/orchestrator/summary.ts +115 -25
  98. package/src/orchestrator/task-hash.ts +103 -21
  99. package/src/orchestrator/task-log-buffer.ts +15 -18
  100. package/src/orchestrator/telemetry-host.ts +27 -8
  101. package/src/orchestrator/telemetry.ts +71 -44
  102. package/src/orchestrator/upstream.ts +91 -45
  103. package/src/util/bun-version.ts +43 -0
  104. package/src/util/cgroup.ts +160 -0
  105. package/src/util/edit-distance.ts +49 -0
  106. package/src/util/errors.ts +86 -0
  107. package/src/util/hash.ts +10 -1
  108. package/src/util/index.ts +42 -5
  109. package/src/util/paths.ts +168 -5
  110. package/src/util/procfs.ts +20 -0
  111. package/src/util/settle.ts +23 -2
  112. package/src/util/size.ts +33 -0
  113. package/src/util/task-id.ts +20 -0
  114. package/src/util/timing.ts +11 -1
  115. package/src/util/verbs.ts +39 -0
  116. package/src/util/which.ts +48 -0
  117. package/src/workspace/affected.ts +368 -60
  118. package/src/workspace/config-cache.ts +271 -32
  119. package/src/workspace/config-eval.ts +60 -16
  120. package/src/workspace/config-imports.ts +131 -9
  121. package/src/workspace/config-schema.ts +1033 -0
  122. package/src/workspace/filter.ts +69 -17
  123. package/src/workspace/fingerprint.ts +59 -7
  124. package/src/workspace/index.ts +37 -3
  125. package/src/workspace/json-data.ts +90 -0
  126. package/src/workspace/load-reads.ts +32 -0
  127. package/src/workspace/lockfile.ts +41 -14
  128. package/src/workspace/migrate-scripts.ts +187 -0
  129. package/src/workspace/migration.ts +353 -0
  130. package/src/workspace/package-graph.ts +239 -31
  131. package/src/workspace/project-loader.ts +295 -815
  132. package/src/workspace/workspace.ts +266 -31
  133. package/plugins/local-cache/index.ts +0 -2
  134. package/plugins/local-executor/index.ts +0 -2
  135. package/plugins/schedule-history/index.ts +0 -2
  136. package/src/cache/cas-backend.ts +0 -115
  137. package/src/cache/digest.ts +0 -32
  138. package/src/cli/migrate-emit.ts +0 -21
  139. package/src/cli/migrate-nx.ts +0 -473
  140. package/src/cli/migrate-scripts.ts +0 -127
  141. package/src/cli/migrate-turbo.ts +0 -401
  142. package/src/cli/migrate.ts +0 -305
  143. package/src/cli/prune.ts +0 -291
  144. package/src/orchestrator/missing-plugin.ts +0 -9
  145. package/src/orchestrator/resources.ts +0 -45
  146. package/src/orchestrator/verify.ts +0 -174
  147. package/src/plugins/local-cache/index.ts +0 -11
  148. package/src/plugins/local-executor/index.ts +0 -45
  149. package/src/plugins/schedule-history/index.ts +0 -109
@@ -12,7 +12,7 @@
12
12
  // metadata — taskId, command, exitCode, durationMs, storedAt — lives
13
13
  // in the SQLite `entries` row. The same tar.zst bytes ship to a remote
14
14
  // cache server unchanged; on remote-hit, the caller supplies metadata
15
- // via the `ingest(hash, bytes, meta)` API so the local SQL index gets
15
+ // via the `ingest(hash, body, meta)` API so the local SQL index gets
16
16
  // populated without sniffing the artifact.
17
17
  //
18
18
  // We never cache failed runs, so stderr is dropped from the cached
@@ -20,89 +20,117 @@
20
20
  // for the user to see — but on a cache hit there's nothing to replay
21
21
  // (the original run was successful and stderr typically empty).
22
22
  //
23
- // Replace this module to plug in remote storage. The contract is:
24
- // key() : derive a stable hash from a task's identity + inputs
25
- // get(hash, ctx?) : retrieve a previous run's metadata, or null
26
- // restoreOutputs : extract the artifact's outputs/ into the project dir
27
- // save : persist outputs + stdout under a hash
28
- // ingest : adopt an artifact produced elsewhere (remote-hit path)
29
- // recordRun : append a row to the run history table (for stats)
30
- // close : release the SQLite handle
23
+ // This is core's FLOOR, not a module to replace: remote storage is a
24
+ // `RemoteCacheLayer` (has / get / put) that `LayeredCache` wraps around
25
+ // this handle, declared by a plugin's `cache` hook. The contract every
26
+ // layer speaks is `CacheLayer` in layer.ts; `plugin-host.ts` enforces it.
31
27
 
32
28
  import { Database, type SQLQueryBindings } from 'bun:sqlite'
33
- import { existsSync, mkdirSync, statSync, writeFileSync } from 'node:fs'
34
- import { lstat, mkdir, readdir, rename, rm, stat, unlink, writeFile } from 'node:fs/promises'
29
+ import { accessSync, constants, existsSync, mkdirSync, writeFileSync } from 'node:fs'
30
+ import { readdir, rename, rm, stat, unlink, writeFile } from 'node:fs/promises'
35
31
  import path from 'node:path'
36
- import { relPosix, UserError, xxh3, xxh3hex, span } from '../util/index.js'
32
+ import {
33
+ UserError,
34
+ asTrees,
35
+ formatBytes,
36
+ isDiskFull,
37
+ isFsRefusal,
38
+ relPosix,
39
+ span,
40
+ splitTaskId,
41
+ taskGlob,
42
+ } from '../util/index.js'
37
43
  import {
38
44
  ArchiveSecurityError,
45
+ type ArtifactPlan,
46
+ type ExecUsage,
39
47
  extractArtifactStream,
40
48
  scanArtifact,
41
49
  packArtifactBytes,
42
50
  packArtifactStream,
43
51
  planArtifact,
44
52
  } from './archive.js'
45
- import { FsCASBackend } from './cas-backend.js'
46
-
47
- // v17: artifact carries only logs + outputs (stdout + outputs/<rel>).
48
- // Local and remote layers transport the SAME tar.zst bytes — no
49
- // separate stage/meta.json/tar.gz dance for remote, no
50
- // `cache-archive.ts`. stderr is no longer cached: we only cache
51
- // successful runs and stderr is rarely meaningful on success.
52
- // v19: '^task' dependsOn expansion switched from transitive-deps to
53
- // nearest-holder frontier — upstream-hash sets shrink, so keys change.
54
- // v20: input-file content hashes switched from xxh3 to git blob OIDs
55
- // (Turbo parity). Clean tracked files take their OID straight from
56
- // the index (harvested by the bulk `git ls-files -s`); dirty /
57
- // untracked files get the identical OID computed in-process. Every
58
- // file's hash bytes change → bump. SCHEMA_VERSION moves with it:
59
- // pre-v20 `file_hashes.content_hash` rows hold xxh3 digests that
60
- // must not leak into the OID domain via the mtime+size memo.
61
- // v22: reverted the v21 output-fold "early cutoff". Downstream keys
62
- // fold the upstream's INPUT key (its task hash) again — pure-input
63
- // transitive hashing (Turbo/Nx model). No output content participates
64
- // in any cache key. SCHEMA v21 drops the now-unused outputs_hash
65
- // column. Early cutoff removed (an upstream that re-emits identical
66
- // output still re-runs dependents) — rare, not worth the cascade.
67
- // v23: fold cache.inputs.runtime / workspaceRuntime command output into
68
- // the key (two namespaced sections after env-values). Command strings
69
- // live in the resolved config (frozen by `vx lock`); the OUTPUT is
70
- // resolved live every run, so it stays correct under --frozen. No
71
- // SCHEMA bump — only Cache.key derivation changed.
72
- // v24: vx-lock.json added to the always-ignored input set (cache/
73
- // inputs.ts ALWAYS_IGNORE). A task whose globs matched the root
74
- // lockfile (broad `**/*` on the root project) drops it from the hashed
75
- // file set, so those keys change; tasks that never matched it are
76
- // byte-identical. No SCHEMA bump — only the input file set changed.
77
- //
78
- // CACHE_VERSION stays v24 through the Tier-3 schema roll (SCHEMA v22).
79
- // Tier 3 persists the very components already fed to `Cache.key()` —
80
- // it adds nothing to, reorders nothing in, and reweights nothing
81
- // inside the key fold (the new `captureInto` sink is a pure
82
- // side-channel). A task's hash is byte-identical before and after, so
83
- // existing artifacts stay valid and there is no cache-version bump.
84
- // v25: the ARTIFACT BYTES are wrong in every existing entry, while the
85
- // key that addresses them is unchanged — the one situation a version
86
- // bump exists for. Two defects: (a) `packArtifact` staged outputs with
87
- // `Bun.write`, which does not carry the source mode, so every artifact
88
- // records 0644 and a cache hit strips the executable bit off any binary
89
- // or generated script; (b) it packed `--format=ustar`, which splits a
90
- // name over 100 bytes into prefix+name, and the reader read only `name`
91
- // — so those entries lost their `outputs/` prefix and were silently
92
- // dropped from both the restore and the output_files index. Neither is
93
- // self-healing: the stored bytes/headers are already wrong, so without a
94
- // new namespace the fixed code would keep replaying them forever.
95
- // v26: same situation, different producer. A task whose child was killed by
96
- // a shutdown signal reported `aborted`, but `aborted` did not propagate to
97
- // dependents — so a dependent ran against PARTIAL outputs, succeeded, and
98
- // cached what it built. The fold takes the upstream's INPUT key, which a
99
- // signal does not change, so that entry sits under exactly the key a healthy
100
- // run derives and replays forever as a green hit. Fixing the propagation
101
- // stops new poison but cannot reach entries already written, and a
102
- // `LayeredCache` uploads them, so the reach is a whole team's shared cache
103
- // rather than one developer's disk. Pre-alpha, so one cold rebuild is the
104
- // cheap side of that trade.
105
- const CACHE_VERSION = 'vx-cache-v27'
53
+ import {
54
+ ArtifactVanishedError,
55
+ type CacheEntry,
56
+ type CacheGetContext,
57
+ type CacheKeyInput,
58
+ type CacheLayer,
59
+ type CacheStats,
60
+ type CacheStatsOptions,
61
+ CorruptArtifactError,
62
+ type DeclaredOutputs,
63
+ type IngestMeta,
64
+ type InvocationRecord,
65
+ type OutputDirRow,
66
+ type OutputFileRow,
67
+ type PruneOptions,
68
+ type PruneResult,
69
+ type RunRecord,
70
+ type SaveArgs,
71
+ type TaskInputRow,
72
+ WORKSPACE_OUTPUT_PREFIX,
73
+ } from './layer.js'
74
+ import {
75
+ bytesOf,
76
+ decodedTar,
77
+ MAX_DECOMPRESSED_ARTIFACT_BYTES,
78
+ STREAM_DECODE_FROM,
79
+ zstdEncoder,
80
+ } from './zstd.js'
81
+ import { ConfigEvalTable } from './config-evals.js'
82
+ import { FileHashStore } from './file-hashes.js'
83
+ import { OutputIndex } from './output-index.js'
84
+ import { RunHistory } from './run-history.js'
85
+ import { CACHE_VERSION, foldKey } from './key-fold.js'
86
+
87
+ /**
88
+ * An artifact or temp file without an `entries` row is reaped by
89
+ * `prune()` only once it is this old. A save renames the artifact into
90
+ * place and commits its row in the same tick; a crashed save's temp is
91
+ * stale long before this. Generous on purpose — a false orphan costs a
92
+ * re-run, a leaked temp costs disk.
93
+ */
94
+ const ORPHAN_GRACE_MS = 60 * 60 * 1000
95
+
96
+ /**
97
+ * The only names the orphan sweep may unlink: an artifact under a key
98
+ * `foldKey` prints (16 lowercase hex) and the temp `tempPath` makes of
99
+ * one. `cacheDir` is the user's to point anywhere (`build/`, a shared
100
+ * directory), and the sweep once took any `*.tar.zst` an hour old there,
101
+ * the user's own release tarball included (item 968).
102
+ */
103
+ const VX_ARTIFACT_NAME = /^([0-9a-f]{16})\.tar\.zst(\.tmp-\d+-\d+-[0-9a-z]*)?$/
104
+
105
+ /** The `schema_meta` key holding when the orphan sweep last ran (ms epoch). */
106
+ const SWEPT_AT = 'orphans_swept_at'
107
+
108
+ export interface SchemaReset {
109
+ from: string
110
+ to: string
111
+ }
112
+
113
+ /** SQLite's answer for a file that is not a readable database: corrupt, or not one at all. */
114
+ function unreadableIndex(err: unknown): boolean {
115
+ const code = (err as { code?: unknown }).code
116
+ return typeof code === 'string' && (code.startsWith('SQLITE_CORRUPT') || code === 'SQLITE_NOTADB')
117
+ }
118
+
119
+ /** Say once, on the channel the opener has, that an upgrade emptied the index. */
120
+ export function noteSchemaReset(cache: Cache, warn: (message: string) => void): void {
121
+ if (cache.formatChange !== null) {
122
+ const { from, to } = cache.formatChange
123
+ warn(
124
+ `[vx] cache format changed: ${from} → ${to} (vx upgraded); every cached task misses once and re-saves, and the old entries, never read again, age out under \`vx cache prune --older-than\` or \`cacheRetention\``,
125
+ )
126
+ return
127
+ }
128
+ if (cache.schemaReset === null) return
129
+ const { from, to } = cache.schemaReset
130
+ warn(
131
+ `[vx] cache index reset: schema ${from} → ${to} (vx upgraded); every cached task misses once and re-saves, and \`vx cache prune\` reclaims the old artifacts`,
132
+ )
133
+ }
106
134
  // SCHEMA history (drop+recreate on mismatch; pre-alpha, no migrations):
107
135
  // v20: file_hashes.content_hash (git blob OIDs).
108
136
  // v21: dropped the unused outputs_hash column (pure-input hashing).
@@ -133,7 +161,32 @@ const CACHE_VERSION = 'vx-cache-v27'
133
161
  // the memo simply stops answering wrongly, so an affected task's
134
162
  // key moves from a WRONG value to the right one: it misses once,
135
163
  // re-runs, re-caches. Self-healing, never a wrong hit.
136
- const SCHEMA_VERSION = 'v24'
164
+ // v25: runs.cached — whether the task declared a `cache` block. An
165
+ // uncached task's row looked like any executed miss, so `vx why`
166
+ // headlined "cache key changed (inputs differ)" for a task that
167
+ // runs every time by design, and `vx last` could not mark it the
168
+ // way the terminal summary does (`no-cache`). Analytics-only —
169
+ // the cache KEY is unchanged.
170
+ // v26: entries.cpu_ms / peak_rss_bytes — what the producing execution
171
+ // used, read out of the artifact's sidecar at save and ingest
172
+ // alike, so a hit (a remote one on a fresh runner above all) can
173
+ // tell the history what the task needs. The cache KEY and the
174
+ // artifact container are unchanged (no CACHE_VERSION bump: an
175
+ // artifact without the field reads as before).
176
+ // v28: output_files.ino + .ctime_ms — the skip-restore check's guard
177
+ // against another entry's bytes. Two entries whose outputs carry
178
+ // one fixed mtime (`tar -x`, `cp -p`, SOURCE_DATE_EPOCH) and one
179
+ // size matched each other's (size, mode, mtime) rows, so a hit left
180
+ // the last entry's bytes on disk under a green run (item 886). A row
181
+ // is current only with the inode and ctime recorded after the save
182
+ // or restore that wrote it. The cache KEY is unchanged.
183
+ export const SCHEMA_VERSION = 'v28'
184
+
185
+ /** A schema version's number (`v28` → 28); one that is not `v<n>` is older than any. */
186
+ function schemaOrdinal(version: string): number {
187
+ const m = /^v(\d+)$/.exec(version)
188
+ return m === null ? -1 : Number(m[1])
189
+ }
137
190
 
138
191
  /**
139
192
  * SQL predicate selecting `runs` rows that record an EXECUTION.
@@ -160,840 +213,139 @@ export const EXECUTED_RUNS_SQL = "status <> 'skipped'"
160
213
  */
161
214
  export const KEYED_RUNS_SQL = "hash <> ''"
162
215
 
163
- /**
164
- * Artifact + `output_files` namespace prefix for workspace-root-
165
- * anchored outputs (`cache.outputs.workspaceFiles`). Project outputs
166
- * keep their bare project-relative `path` rows; workspace rows store
167
- * the full `workspace-outputs/<rel-to-root>` tar entry name as the
168
- * discriminator — least-invasive row format, no schema change. A
169
- * project output dir literally named `workspace-outputs/` would
170
- * collide with the namespace; the name is reserved.
171
- */
172
- export const WORKSPACE_OUTPUT_PREFIX = 'workspace-outputs/'
173
-
174
- /**
175
- * Independent read/write control over the two cache layers (local +
176
- * remote). Replaces the old single `noCache` boolean: each axis can be
177
- * toggled on its own so `--force` (re-execute but still refresh the
178
- * cache) is distinct from `--no-cache` (disable everything).
179
- *
180
- * Only the task-artifact get/save path is gated. `recordRun`, `stats`,
181
- * `prune`, key derivation, and prefetch-ingest are never affected — they
182
- * are bookkeeping/analytics that a run policy has no business disabling.
183
- */
184
- export interface CachePolicy {
185
- localRead: boolean
186
- localWrite: boolean
187
- remoteRead: boolean
188
- remoteWrite: boolean
189
- }
190
-
191
- /** All four axes on — the default when no cache flag is passed. */
192
- export const FULL_CACHE_POLICY: CachePolicy = {
193
- localRead: true,
194
- localWrite: true,
195
- remoteRead: true,
196
- remoteWrite: true,
197
- }
216
+ // The contract, the policy grammar and the zstd framing live beside this
217
+ // file; they are re-exported here so an importer of `./cache.js` — the
218
+ // module index, the sibling layers, the tests — keeps one path.
219
+ export * from './layer.js'
220
+ export * from './policy.js'
221
+ export { zstdContentSize } from './zstd.js'
198
222
 
199
- /**
200
- * Parse a `--cache=<spec>` value into a `CachePolicy`, starting from a
201
- * base (defaults to {@link FULL_CACHE_POLICY}). The spec is a
202
- * comma-separated list of `layer:flags` segments where `layer` is
203
- * `local` or `remote` and `flags` is any subset of `r` (read) and `w`
204
- * (write), order-independent and possibly empty.
205
- *
206
- * A mentioned layer is set EXACTLY to its flags (read = includes `r`,
207
- * write = includes `w`); an unmentioned layer keeps its base value.
208
- * So `local:rw,remote:r` = remote read-only, `remote:` = remote fully
209
- * off, `local:r` = local read-only with remote untouched.
210
- *
211
- * Throws a {@link UserError} on an unknown layer, an unknown flag, a
212
- * duplicated flag, a missing colon, or a repeated layer.
213
- */
214
- export function parseCachePolicy(spec: string, base: CachePolicy = FULL_CACHE_POLICY): CachePolicy {
215
- const out: CachePolicy = { ...base }
216
- const seen = new Set<string>()
217
- for (const rawSeg of spec.split(',')) {
218
- const seg = rawSeg.trim()
219
- if (seg.length === 0) continue
220
- const colon = seg.indexOf(':')
221
- if (colon < 0) {
222
- throw new UserError(`invalid --cache segment '${seg}': expected '<layer>:<flags>'`)
223
- }
224
- const layer = seg.slice(0, colon)
225
- const flags = seg.slice(colon + 1)
226
- if (layer !== 'local' && layer !== 'remote') {
227
- throw new UserError(`invalid --cache layer '${layer}': expected 'local' or 'remote'`)
228
- }
229
- if (seen.has(layer)) {
230
- throw new UserError(`--cache layer '${layer}' specified twice`)
231
- }
232
- seen.add(layer)
233
- const flagSet = new Set<string>()
234
- for (const ch of flags) {
235
- if (ch !== 'r' && ch !== 'w') {
236
- throw new UserError(`invalid --cache flag '${ch}' for '${layer}': expected 'r' and/or 'w'`)
237
- }
238
- if (flagSet.has(ch)) {
239
- throw new UserError(`--cache flag '${ch}' repeated for '${layer}'`)
240
- }
241
- flagSet.add(ch)
242
- }
243
- if (layer === 'local') {
244
- out.localRead = flagSet.has('r')
245
- out.localWrite = flagSet.has('w')
246
- } else {
247
- out.remoteRead = flagSet.has('r')
248
- out.remoteWrite = flagSet.has('w')
249
- }
250
- }
251
- return out
252
- }
253
-
254
- export interface CacheKeyInput {
255
- taskId: string
256
- /**
257
- * Hash of the resolved task config (post-evaluation). Folds in everything
258
- * the user wrote — command, env declarations (passThrough names + define
259
- * key/value pairs), dependsOn, cache.inputs declarations, outputs — including
260
- * values that arrived via `import` at config-load time.
261
- */
262
- taskConfigHash: string
263
- /**
264
- * Runtime values of declared cache-input env names (from parent at hash
265
- * time). Independent of `exec.env`; lives here for cache identity.
266
- */
267
- envValues: Array<[name: string, value: string]>
268
- /**
269
- * Resolved `cache.inputs.runtime` commands as [command, output] pairs
270
- * (output = trimmed stdout+stderr, resolved live at hash time). Folded
271
- * into the key in a namespace distinct from workspaceRuntimeValues.
272
- */
273
- runtimeValues?: Array<[command: string, output: string]>
274
- /** Resolved `cache.inputs.workspaceRuntime` pairs (root-cwd commands). */
275
- workspaceRuntimeValues?: Array<[command: string, output: string]>
276
- /** Absolute paths to input files. */
277
- inputFiles: string[]
278
- workspaceRoot: string
279
- /** Cache keys of upstream tasks this one depends on, sorted. */
280
- upstreamHashes: string[]
281
- /**
282
- * Upstream hash → upstream task id, for `captureInto` row NAMING only.
283
- * Never folded into the digest — the key already folds the sorted
284
- * hashes. Lets the persisted `entry_inputs` row name which upstream
285
- * task a hash came from (the diff reads better than a bare hash). When
286
- * absent, the captured `upstream` row falls back to `name = hash`.
287
- */
288
- upstreamIds?: ReadonlyMap<string, string>
289
- /**
290
- * The task's DEPENDENCY closure with GROUP tasks expanded into the real
291
- * tasks they stand for — what an input-shipping executor must place in the
292
- * input root. A different question from what the key folds, and derived
293
- * differently in both directions:
294
- *
295
- * - It expands groups, which contribute a synthetic roll-up hash and no
296
- * outputs of their own, so a dependent of one would otherwise describe
297
- * an empty closure.
298
- * - It ignores `cache.inputs.tasks`. That filter says which upstream KEYS
299
- * this task's key folds; what the task may READ is `dependsOn`, and
300
- * locally every dependency's outputs are on disk before the command runs
301
- * however the filter is written.
302
- *
303
- * NEVER folded into the digest — the key cascades through a group's own
304
- * roll-up hash already, and folding either difference in would move every
305
- * existing dependent's key without telling it anything new.
306
- */
307
- upstreamGraft?: ReadonlyArray<{
308
- readonly taskId: string
309
- readonly hash: string
310
- readonly projectDir: string
311
- }>
312
- /**
313
- * Workspace-level fingerprint — typically a hash of `pnpm-lock.yaml` +
314
- * `pnpm-workspace.yaml`. Folds resolved dep versions and workspace shape
315
- * into every task's key, so a lockfile bump invalidates everything.
316
- */
317
- workspaceFingerprint: string
318
- /**
319
- * CLI args forwarded to the task (after `--`). Folded into the key so that
320
- * the same command with different forwarded args is treated as a distinct
321
- * run, never a spurious cache hit.
322
- */
323
- forwardArgs?: readonly string[]
324
- /**
325
- * Hash of the project's `package.json` bytes. Folded into the key
326
- * implicitly (Turbo / Nx parity) so dep changes invalidate every
327
- * task in that project, even when `cache.inputs.files` doesn't
328
- * cover package.json. Empty string when the project has no
329
- * package.json (impossible in practice — workspace discovery
330
- * requires one — but we don't fail-loud here).
331
- */
332
- projectPackageJsonHash: string
333
- /**
334
- * Precomputed content hashes (git blob OIDs) keyed by absolute
335
- * path — typically the trusted-index OID map harvested by the
336
- * run's bulk `git ls-files -s`. Paths present here skip `hashFile`
337
- * entirely (no stat, no SQLite, no read); missing paths fall back
338
- * to `hashFile`, which computes the byte-identical blob OID from
339
- * disk. Pure fast path: the derived key never depends on whether a
340
- * hash arrived via the map or the fallback.
341
- */
342
- fileHashes?: ReadonlyMap<string, string>
343
- /**
344
- * Material a plugin's `key` stage contributed, as sorted `[name, value]`
345
- * pairs. Folded only when non-empty, so a workspace without a key plugin
346
- * derives exactly the key it always did.
347
- */
348
- pluginParts?: ReadonlyArray<readonly [name: string, value: string]>
349
- /**
350
- * When set, `key()` pushes each component (kind, name, hash) it folds
351
- * — at the same fold sites, in fold order. Pure SIDE-CHANNEL: it does
352
- * not change the returned digest in any way. On a cache MISS the
353
- * orchestrator allocates an array, passes it here, and persists the
354
- * rows to `entry_inputs` (inside the entry-save transaction) so a
355
- * later run can diff its inputs against this one (the Tier-3 "why did
356
- * this re-run?" moat). Capturing at the fold sites keeps the
357
- * recorded set in lockstep with the key by construction: a future
358
- * component that forgets to capture is a one-line miss, not silent
359
- * drift. The per-file OIDs are already awaited here, so file rows cost
360
- * zero extra I/O — just array pushes.
361
- */
362
- captureInto?: Array<{ kind: string; name: string; hash: string }>
363
- }
364
-
365
- export interface CacheEntry {
223
+ interface EntryRow {
366
224
  hash: string
367
- taskId: string
368
- command: string
369
- exitCode: number
370
- durationMs: number
371
- outputFiles: string[]
372
- /**
373
- * The `output_files` rows behind `outputFiles` (size / mode / mtime), when
374
- * the layer that produced this entry had them in hand. `restoreHit` reads
375
- * them for its tree-is-current check instead of re-querying — one SQL
376
- * round trip per cache hit, saved.
377
- */
378
- outputRows?: OutputFileRow[]
379
- /**
380
- * The `output_dirs` rows behind the directory short-circuit, when the
381
- * batched `getMany` loaded them — one query for the run instead of one
382
- * per hit (0.12 ms each across a warm 1000-project run before this).
383
- */
384
- outputDirRows?: OutputDirRow[]
385
- /** Captured stdout, always present (may be empty). stderr is not cached. */
386
- stdout: string
387
- storedAt: string
388
- /**
389
- * Where this hit was resolved from. `'local'` for a SQLite-backed
390
- * Cache; `'remote'` when LayeredCache pulled the artifact from the
391
- * remote layer this lookup (even though it's been materialized into
392
- * local for next time). Lets the orchestrator surface
393
- * `cache-hit-remote` so users see when remote caching actually saved
394
- * them work vs. a stale-local replay.
395
- */
396
- source?: 'local' | 'remote'
397
- }
398
-
399
- export interface RunRecord {
400
- /**
401
- * The task's cache key. ABSENT when the outcome never derived one — a
402
- * `skipped` task (its upstream failed, so it never probed) or a
403
- * `persistent` one (a dev server is never cached). Such a row is still a
404
- * task of the run and must be recorded, so the column keeps a `''`
405
- * sentinel rather than going nullable: `''` is impossible for a real key
406
- * (`Cache.key` returns 16 hex chars), the reads that must not treat it as
407
- * a key already guard it, and it matches what the cloud `task_runs.hash`
408
- * column has stored for the same concept since its first migration.
409
- */
410
- hash?: string
411
225
  project: string
412
226
  task: string
413
- status: 'success' | 'failed' | 'cache-hit' | 'cache-hit-remote' | 'skipped'
414
- exitCode: number
415
- durationMs: number
416
- forwardArgs?: readonly string[]
417
- startedAt: number // ms-epoch wall clock
418
- endedAt: number // ms-epoch wall clock
419
- /**
420
- * Optional analytics columns. Populated by the orchestrator/runner;
421
- * stored as NULL on rows from older runs. Surfaced via `vx stats`
422
- * and consumable from CI by reading cache.db directly.
423
- */
424
- runId?: string // ULID shared across every task in one `vx run` invocation
425
- cpuMs?: number // sum of user + system CPU time for the child process
426
- peakRssBytes?: number // peak resident set size of the child process
427
- wallclockStartNs?: bigint // hrtime span relative to run t=0
428
- wallclockEndNs?: bigint
429
- cacheHit?: boolean // convenience for flamegraph color; derivable from status
430
- attempts?: number // >1 when the task retried; the direct within-run flaky signal
431
- }
432
-
433
- /**
434
- * One header row per `vx run` invocation (the `invocations` table). All
435
- * fields mirror the columns; nullable VCS/host columns are `null` when
436
- * the probe failed (not a git repo, hostname unavailable). Recorded
437
- * once per run inside the same transaction as the per-task `runs` rows.
438
- */
439
- export interface InvocationRecord {
440
- runId: string
441
- command: string
442
- /** JSON-serialized `string[]` of the requested task names. */
443
- requestedTasks: string
444
- /** Compact policy flags, e.g. `'lR,lW,rR,rW'` (an axis omitted = off). */
445
- cachePolicy: string
446
- concurrency: number
447
- flow: 'focused' | 'broad' | null
448
- startedAt: number
449
- endedAt: number
450
- totalDurationMs: number
451
- taskCount: number
452
- failedCount: number
453
- hitCount: number
454
- hitLocalCount: number
455
- hitRemoteCount: number
456
- exitOk: boolean
457
- commitSha: string | null
458
- branch: string | null
459
- dirty: boolean | null
460
- ci: boolean
461
- ciProvider: string | null
462
- host: string | null
463
- os: string | null
464
- arch: string | null
465
- vxVersion: string
466
- /** JSON object `{k:v}` of `--tag` pairs. */
467
- tags: string
468
- }
469
-
470
- /**
471
- * One cache-key component row for the `entry_inputs` table — keyed by
472
- * the cache-entry HASH it belongs to, not a run. Written inside the
473
- * entry-save transaction (`writeArtifactAndIndex`) on a miss/save; a
474
- * cache HIT does not save, so it persists nothing (warm runs are free).
475
- * The diff (Phase B/B1) reads these by the run's task hash
476
- * (`runs.hash → entry_inputs[entry_hash]`).
477
- */
478
- export interface TaskInputRow {
479
- entryHash: string
480
- kind: string
481
- name: string
482
- hash: string
483
- }
484
-
485
- export interface CacheStats {
486
- entryCount: number
487
- totalBytes: number
488
- runCountLast24h: number
489
- hitCountLast24h: number
490
- }
491
-
492
- export interface CacheStatsOptions {
493
- /** Narrow every aggregate to one project. Absent = the whole workspace. */
494
- project?: string
495
- }
496
-
497
- export interface PruneOptions {
498
- /** Drop entries last accessed before this ms-epoch threshold. */
499
- olderThanMs?: number
500
- /**
501
- * After applying olderThanMs, if the cache still exceeds this size in
502
- * bytes, evict LRU (smallest `accessed_at` first) until under it.
503
- */
504
- maxBytes?: number
505
- }
506
-
507
- export interface PruneResult {
508
- evicted: number
509
- bytesFreed: number
510
- }
511
-
512
- /**
513
- * Per-output-file fingerprint, scoped by the cache entry that
514
- * produced it. Batch-loaded once at the top of a run via
515
- * `loadOutputFilesBatch(hashes)` so the orchestrator's "is this
516
- * tree already current?" probe becomes an in-memory Map lookup
517
- * plus N parallel stat calls.
518
- *
519
- * `path` is project-relative (e.g. `dist/index.js`), matching how
520
- * outputs are addressed under `<projectDir>/` — except workspace
521
- * outputs, which carry the full `workspace-outputs/<rel-to-root>`
522
- * tar entry name (see `WORKSPACE_OUTPUT_PREFIX`); callers split on
523
- * the prefix and anchor those at the workspace root.
524
- */
525
- export interface OutputFileRow {
526
- path: string
527
- size: number
528
- mode: number
529
- mtimeMs: number
530
- }
531
-
532
- /** One directory under a whole-subtree output glob, as it stood after the last save or restore on THIS machine. */
533
- export interface OutputDirRow {
534
- path: string
535
- mtimeMs: number
536
- }
537
-
538
- /** More directories than this under a task's output prefixes: record nothing, keep the walk. */
539
- export const OUTPUT_DIRS_CAP = 256
540
-
541
- /**
542
- * A directory whose mtime lies within this many ms of the snapshot is RACY
543
- * and the whole snapshot is dropped. File timestamps are coarse on Linux
544
- * (a kernel tick, up to 10 ms), so a write landing in the same tick as the
545
- * one that made the directory's recorded mtime leaves the mtime unchanged
546
- * and the change invisible — git's index distrusts stats this young for
547
- * the same reason (a stray survived a hit on the ubuntu job, 2026-09-03).
548
- * The next hit walks, and once the tree is older than the window it is
549
- * recorded for good.
550
- */
551
- export const OUTPUT_DIRS_RACY_MS = 50
552
-
553
- /**
554
- * The shape every cache implementation honors. `Cache` (the local v10
555
- * implementation) and `LayeredCache` both `implements` this so the
556
- * orchestrator's `executeTask` can take either without a discriminated
557
- * union and we get a compile-time guarantee the surfaces stay congruent.
558
- */
559
- /**
560
- * Context passed to `get()`. Optional, but required when the lookup
561
- * may resolve through the remote layer — the local SQL row inserted on
562
- * remote-hit needs `taskId` + `command` to be queryable later (the
563
- * artifact itself doesn't carry them). `Cache` (local) ignores this
564
- * field; `LayeredCache` forwards it to `Cache.ingest`.
565
- */
566
- export interface CacheGetContext {
567
- taskId: string
568
227
  command: string
228
+ exit_code: number
229
+ duration_ms: number
230
+ size_bytes: number
231
+ stdout: string
232
+ created_at: number
233
+ accessed_at: number
234
+ cpu_ms: number | null
235
+ peak_rss_bytes: number | null
569
236
  }
570
237
 
571
- /** Metadata supplied at ingest time — values the artifact does not carry. */
572
- export interface IngestMeta {
573
- taskId: string
574
- command: string
575
- /** Wall-clock time of the original task execution. */
576
- durationMs: number
577
- /**
578
- * Cache-key components (Tier-3 input fingerprint) to persist into
579
- * `entry_inputs` in the same transaction as the entry row. Omitted on
580
- * the remote-hit ingest path (the artifact doesn't carry them — the
581
- * fingerprint is the saving machine's local moat). Only computed on a
582
- * miss/save, so a cache hit never reaches here.
583
- */
584
- inputComponents?: readonly TaskInputRow[]
238
+ function entryOf(row: EntryRow, fileRows: OutputFileRow[]): CacheEntry {
239
+ return {
240
+ hash: row.hash,
241
+ taskId: `${row.project}#${row.task}`,
242
+ command: row.command,
243
+ exitCode: row.exit_code,
244
+ durationMs: row.duration_ms,
245
+ ...(row.cpu_ms !== null ? { cpuMs: row.cpu_ms } : {}),
246
+ ...(row.peak_rss_bytes !== null ? { peakRssBytes: row.peak_rss_bytes } : {}),
247
+ outputFiles: fileRows.map((r) => r.path),
248
+ outputRows: fileRows,
249
+ stdout: row.stdout,
250
+ storedAt: new Date(row.created_at).toISOString(),
251
+ source: 'local',
252
+ }
585
253
  }
586
254
 
587
- /**
588
- * The supplied artifact bytes don't decompress/parse as a vx artifact.
589
- * Thrown by `save`/`ingest` BEFORE anything reaches the final cache
590
- * path — a rejected artifact leaves no `<hash>.tar.zst` and no SQL row.
591
- * The LayeredCache treats this as a remote fault on the remote-hit
592
- * path (degrades to a cache miss).
593
- */
594
- export class CorruptArtifactError extends Error {
595
- constructor(
596
- public readonly hash: string,
597
- reason: string,
598
- public override readonly cause?: unknown,
599
- ) {
600
- super(`cache: corrupt artifact for ${hash}: ${reason}`)
601
- this.name = 'CorruptArtifactError'
255
+ /** The sidecar's `exec` from a save's entry: only the axes the runner reported. */
256
+ function usageOfEntry(entry: { cpuMs?: number; peakRssBytes?: number }): ExecUsage | undefined {
257
+ if (entry.cpuMs === undefined && entry.peakRssBytes === undefined) return undefined
258
+ return {
259
+ ...(entry.cpuMs !== undefined ? { cpuMs: entry.cpuMs } : {}),
260
+ ...(entry.peakRssBytes !== undefined ? { peakRssBytes: entry.peakRssBytes } : {}),
602
261
  }
603
262
  }
604
263
 
605
264
  /**
606
- * A decompressed artifact above this is refused as a zstd bomb rather than
607
- * expanded into memory. 2 GiB comfortably exceeds any real build output while
608
- * bounding a malicious/compromised remote's ability to OOM a victim who takes
609
- * a cache hit.
610
- */
611
- const MAX_DECOMPRESSED_ARTIFACT_BYTES = 2 * 1024 * 1024 * 1024
612
-
613
- /**
614
- * Read a zstd frame's declared Frame_Content_Size (RFC 8878 §3.1.1) WITHOUT
615
- * decompressing. Returns null when the frame omits it (streaming frames) or
616
- * the header is too short to parse. vx's own producer (single-shot
617
- * `Bun.zstdCompress` of a known buffer) always writes it, so an artifact that
618
- * declares an enormous size can be rejected before a byte is allocated.
265
+ * Make `cacheDir` exist and say why this process cannot write into it, or
266
+ * `null` when it can: the directory must take new files (the WAL, the
267
+ * artifacts) and `cache.db`, when it exists, must take pages. The file
268
+ * system's answer, not a trial write — under WAL a rolled-back write never
269
+ * reaches the disk, so `BEGIN IMMEDIATE … ROLLBACK` passes on a handle
270
+ * SQLite opened read-only (proven as an unprivileged user, 2026-09-16).
619
271
  *
620
- * Exported for tests: pins the per-`fcsFlag` byte layouts (incl. the 2-byte
621
- * `+256` adjustment and the `dictIdFlag` offset) that a bomb-refusal e2e can't
622
- * discriminate (their max declarable sizes sit below the ceiling).
272
+ * Every verb opens the cache, so each question here is one call whose
273
+ * failure is another question's answer: `access` on the directory is also
274
+ * "does it exist" (made only on ENOENT), `access` on the database also "is
275
+ * it there", and the ignore file is created exclusively instead of probed
276
+ * first. An existing cache costs three calls; it cost six.
623
277
  */
624
- export function zstdContentSize(b: Uint8Array): bigint | null {
625
- if (b.length < 5) return null
626
- // Magic_Number 0xFD2FB528, little-endian.
627
- if (b[0] !== 0x28 || b[1] !== 0xb5 || b[2] !== 0x2f || b[3] !== 0xfd) return null
628
- const desc = b[4]!
629
- const fcsFlag = desc >> 6
630
- const singleSegment = (desc >> 5) & 1
631
- const dictIdFlag = desc & 3
632
- let off = 5
633
- if (singleSegment === 0) off += 1 // Window_Descriptor byte
634
- off += dictIdFlag === 3 ? 4 : dictIdFlag // Dictionary_ID: 0,1,2,4 bytes
635
- let fcsSize: number
636
- if (fcsFlag === 0) fcsSize = singleSegment === 1 ? 1 : 0
637
- else if (fcsFlag === 1) fcsSize = 2
638
- else if (fcsFlag === 2) fcsSize = 4
639
- else fcsSize = 8
640
- if (fcsSize === 0 || b.length < off + fcsSize) return null
641
- let v = 0n
642
- for (let i = 0; i < fcsSize; i++) v |= BigInt(b[off + i]!) << BigInt(8 * i)
643
- if (fcsSize === 2) v += 256n // per spec, the 2-byte field stores value − 256
644
- return v
645
- }
646
-
647
- /**
648
- * Decompress a zstd artifact that DECLARES its content size, with a hard
649
- * output ceiling: refused before a byte is allocated when the declaration
650
- * is over the cap, and again on the actual length. A frame with no
651
- * declaration (a streamed producer's — vx's own, above 4 MiB) never comes
652
- * here: `decodedTar` decodes it as a stream under the running count, so
653
- * a sizeless bomb has nowhere to expand.
654
- */
655
- async function zstdDecompressBounded(compressed: Uint8Array, hash: string): Promise<Uint8Array> {
656
- assertDeclaredSize(compressed, hash)
657
- const out = await Bun.zstdDecompress(compressed)
658
- if (out.length > MAX_DECOMPRESSED_ARTIFACT_BYTES) {
659
- throw new CorruptArtifactError(
660
- hash,
661
- `decompressed to ${out.length} bytes (> ${MAX_DECOMPRESSED_ARTIFACT_BYTES} cap)`,
662
- )
278
+ function openCacheDir(cacheDir: string): string | null {
279
+ const ignore = path.join(cacheDir, '.gitignore')
280
+ try {
281
+ accessSync(cacheDir, constants.W_OK)
282
+ } catch (err) {
283
+ makeCacheDir(cacheDir)
284
+ if ((err as NodeJS.ErrnoException).code !== 'ENOENT') return errorText(err)
285
+ writeFileSync(ignore, IGNORE_ALL)
286
+ return null
663
287
  }
664
- return out
665
- }
666
-
667
- /** The pre-decompress half of the ceiling: the frame header's own claim, when it makes one. */
668
- function assertDeclaredSize(compressed: Uint8Array, hash: string): bigint | null {
669
- const declared = zstdContentSize(compressed)
670
- if (declared !== null && declared > BigInt(MAX_DECOMPRESSED_ARTIFACT_BYTES)) {
671
- throw new CorruptArtifactError(
672
- hash,
673
- `declares ${declared} decompressed bytes (> ${MAX_DECOMPRESSED_ARTIFACT_BYTES} cap)`,
674
- )
288
+ try {
289
+ accessSync(path.join(cacheDir, 'cache.db'), constants.W_OK)
290
+ } catch (err) {
291
+ const code = (err as NodeJS.ErrnoException).code
292
+ // A FILE at `cacheDir` passed the `access` above; mkdir names it.
293
+ if (code === 'ENOTDIR') makeCacheDir(cacheDir)
294
+ if (code !== 'ENOENT') return errorText(err)
295
+ refuseManifestDir(cacheDir)
296
+ }
297
+ try {
298
+ writeFileSync(ignore, IGNORE_ALL, { flag: 'wx' })
299
+ } catch (err) {
300
+ if ((err as NodeJS.ErrnoException).code !== 'EEXIST') throw err
675
301
  }
676
- return declared
302
+ return null
677
303
  }
678
304
 
679
- /** Compressed artifacts above this size are decoded as a stream, on restore and on ingest. */
680
- const STREAM_DECODE_FROM = 4 * 1024 * 1024
681
-
682
- /** Collect a byte stream; only ever used where the seam wants bytes. */
683
- async function bytesOf(stream: ReadableStream<Uint8Array>): Promise<Uint8Array> {
684
- return new Uint8Array(await new Response(stream).arrayBuffer())
685
- }
305
+ // Make the cache dir invisible to git, every time it is created: a `*`
306
+ // .gitignore inside it (the Cargo / Nx convention). Two reasons, both
307
+ // measured. A cache nobody ignored gets COMMITTED by the next `git add -A`;
308
+ // and vx's own `git status -uall` walks it — 1000 artifacts doubled the
309
+ // enumeration on the bench workspace before its generator ignored `.vx`. An
310
+ // ignored directory is skipped by the walk entirely. Only written when
311
+ // absent, so a user's own file wins.
312
+ const IGNORE_ALL = '*\n'
686
313
 
687
314
  /**
688
- * Bun's `CompressionStream` types do not satisfy `pipeThrough`'s pair
689
- * (its readable side is typed `NonSharedUint8Array`); the runtime object
690
- * is a plain byte transform.
315
+ * A first index in a directory that holds a manifest is a cache pointed at
316
+ * the workspace or a project (`cacheDir: ''`, `'.'`, `'packages/a'`): the
317
+ * `*` ignore file above hid every file in it from git, so its inputs
318
+ * matched nothing and a changed source replayed the old output, and the
319
+ * artifacts landed among the sources (item 997). Asked only when there is
320
+ * no index yet, so an open cache pays nothing for it.
691
321
  */
692
- const zstdEncoder = (): TransformStream<Uint8Array, Uint8Array> =>
693
- new CompressionStream('zstd') as unknown as TransformStream<Uint8Array, Uint8Array>
694
-
695
- const oneChunk = (bytes: Uint8Array): ReadableStream<Uint8Array> =>
696
- new ReadableStream({
697
- start(c) {
698
- c.enqueue(bytes)
699
- c.close()
700
- },
701
- })
702
-
703
- /**
704
- * The decoded tar as a stream: one call for a small artifact that
705
- * declares its size, a streamed decode otherwise — the stream setup is
706
- * ~35 µs, which matters at a thousand one-file artifacts and nowhere
707
- * else. Bytes in memory are only ever the small case (a large source
708
- * must be a FILE: a Blob copies its bytes and hands the decoder
709
- * everything at once, measured +519 MiB on 150 MiB against +448 for the
710
- * plain decode). The 2 GiB ceiling applies either way: the declaration
711
- * and the result length in one call, a running count on the stream.
712
- */
713
- async function decodedTar(
714
- source: Uint8Array | Bun.BunFile,
715
- hash: string,
716
- ): Promise<ReadableStream<Uint8Array>> {
717
- if (source instanceof Uint8Array) {
718
- if (assertDeclaredSize(source, hash) === null) return zstdDecodeStream(new Blob([source]), hash)
719
- return oneChunk(await zstdDecompressBounded(source, hash))
720
- }
721
- if (source.size <= STREAM_DECODE_FROM) {
722
- const bytes = await source.bytes()
723
- if (assertDeclaredSize(bytes, hash) === null) return zstdDecodeStream(source, hash)
724
- return oneChunk(await zstdDecompressBounded(bytes, hash))
322
+ function refuseManifestDir(cacheDir: string): void {
323
+ for (const manifest of ['package.json', 'pnpm-workspace.yaml']) {
324
+ if (!existsSync(path.join(cacheDir, manifest))) continue
325
+ throw new UserError(
326
+ `cache directory ${cacheDir} holds a ${manifest}: it is the workspace's or a project's own ` +
327
+ `directory, and vx would keep its index there under a \`*\` .gitignore that hides every file ` +
328
+ `in it from git and from the cache keys. Point \`cacheDir\` in vx.workspace.ts (or ` +
329
+ `--cache-dir) at a directory of its own, such as .vx/cache.`,
330
+ )
725
331
  }
726
- assertDeclaredSize(await source.slice(0, 32).bytes(), hash)
727
- return zstdDecodeStream(source, hash)
728
332
  }
729
333
 
730
- /**
731
- * The streaming twin of `zstdDecompressBounded`: the decoded bytes as a
732
- * stream, refused past the same ceiling. A malicious frame cannot expand
733
- * past the cap here either — the count runs as bytes are produced, before
734
- * any of them reach the reader's next entry.
735
- */
736
- function zstdDecodeStream(source: Blob, hash: string): ReadableStream<Uint8Array> {
737
- let total = 0
738
- return source
739
- .stream()
740
- .pipeThrough(new DecompressionStream('zstd'))
741
- .pipeThrough(
742
- new TransformStream<Uint8Array, Uint8Array>({
743
- transform(chunk, controller) {
744
- total += chunk.byteLength
745
- if (total > MAX_DECOMPRESSED_ARTIFACT_BYTES) {
746
- controller.error(
747
- new CorruptArtifactError(
748
- hash,
749
- `decompresses past ${MAX_DECOMPRESSED_ARTIFACT_BYTES} bytes (cap)`,
750
- ),
751
- )
752
- return
753
- }
754
- controller.enqueue(chunk)
755
- },
756
- }),
334
+ function makeCacheDir(cacheDir: string): void {
335
+ try {
336
+ mkdirSync(cacheDir, { recursive: true })
337
+ } catch (err) {
338
+ // A read-only checkout with no cache yet: every verb opens the cache,
339
+ // so this is the first thing any of them says there.
340
+ throw new UserError(
341
+ `cannot create cache directory ${cacheDir} (${errorText(err)}) — vx keeps its cache there; make ` +
342
+ `the workspace writable, set \`cacheDir\` in vx.workspace.ts, or pass --cache-dir <path>`,
757
343
  )
344
+ }
758
345
  }
759
346
 
760
- export interface CacheLayer {
761
- /**
762
- * The local handle this layer wraps, when it wraps one (`LayeredCache`).
763
- * `resolveCache` uses it to drop a bare local layer another declared layer
764
- * already contains, so `[cloud(), localCachePlugin()]` does not write the
765
- * local store twice.
766
- */
767
- readonly local?: Cache | undefined
768
- /**
769
- * `true` iff a REMOTE cache sits behind this layer. THE answer to "can
770
- * the remote axes of the cache policy do anything on this run?" — the
771
- * orchestrator clamps `remoteRead`/`remoteWrite` off when it is absent,
772
- * skips the up-front local classify when it is present (that classify's
773
- * `cache.get` would be a remote read-through awaited before scheduling),
774
- * and drives the prefetch pass off it.
775
- *
776
- * A layer must answer TRUTHFULLY: identity against the local cache
777
- * ("something other than the handle I passed in") is NOT the same
778
- * question — an ordinary pass-through decorator with no remote at all
779
- * answers yes to it. `LayeredCache` sets it, a bare `Cache` denies it,
780
- * and a third-party layer opts in when, and only when, it really has a
781
- * remote. Optional so a layer written before this contract keeps
782
- * type-checking; absent reads as "no remote", the safe answer.
783
- */
784
- readonly hasRemote?: boolean
785
- /**
786
- * Batch existence probe over the remote — the subset of `hashes` stored
787
- * remotely, in ONE round-trip, or `null` for "no batch info; use
788
- * per-hash". Only meaningful with `hasRemote`; a remote layer that can't
789
- * batch omits it and the prefetch pass falls back to per-hash GETs.
790
- */
791
- remoteHasMany?(hashes: readonly string[]): Promise<Set<string> | null>
792
- /**
793
- * Record that the remote has NO artifact for each of `hashes` (from a
794
- * batch probe), so a later `get`/`prefetch` short-circuits to a miss with
795
- * no round-trip. Only meaningful with `hasRemote`.
796
- */
797
- markRemoteAbsent?(hashes: Iterable<string>): void
798
- /**
799
- * Resolve once every background write-through upload settles. The
800
- * orchestrator awaits it before `close()` — an upload reading layer state
801
- * after close would race. Only meaningful with `hasRemote`; a layer that
802
- * uploads synchronously (or not at all) omits it.
803
- */
804
- drainUploads?(): Promise<void>
805
- key(input: CacheKeyInput): Promise<string>
806
- get(hash: string, ctx?: CacheGetContext): Promise<CacheEntry | null>
807
- /**
808
- * Optional batched `get` (same answers, fewer round trips). The
809
- * short-circuit probe uses it when a layer offers one; a layer without it
810
- * is probed hash by hash.
811
- */
812
- getMany?(hashes: readonly string[]): Promise<Map<string, CacheEntry>>
813
- /**
814
- * Lightweight existence probe. `'local'` / `'remote'` names the layer
815
- * that holds the artifact; `null` is a miss. NEVER moves bytes: no
816
- * artifact read, no remote download, no local ingest, no accessed_at
817
- * bump (the LayeredCache's remote side is an HTTP HEAD). Planning
818
- * (`--dry` / `--graph`) predicts hits with this instead of `get` so a
819
- * dry run can't pull N artifacts over the network. Remote errors
820
- * degrade to `null` — an existence probe never fails anything.
821
- */
822
- has(hash: string): Promise<'local' | 'remote' | null>
823
- /**
824
- * Best-effort warm of `hash` from a slower layer (the remote cache)
825
- * into this one, so a later `get(hash)` resolves locally without a
826
- * round-trip on the task's critical path. Returns `true` if the
827
- * artifact is now present locally, `false` on a miss / error
828
- * (degrades, never throws). The local `Cache` is a no-op (nothing
829
- * slower to warm from); `LayeredCache` owns the real implementation
830
- * plus the in-flight de-dup so prefetch + get share ONE remote GET.
831
- */
832
- prefetch(hash: string, ctx?: CacheGetContext): Promise<boolean>
833
- /**
834
- * Batched lookup of per-output-file fingerprints for many cache
835
- * entries in one SQL round-trip. Returns a Map keyed by entry hash.
836
- *
837
- * Orchestrator pattern: call this once at `prepareRun` for every
838
- * task whose hash is known up-front, then per-task `executeCachedTask`
839
- * does a Map.get (O(1)) + parallel stat checks to decide whether the
840
- * on-disk tree is already current.
841
- *
842
- * Hashes with no rows (cache misses) are absent from the result.
843
- */
844
- loadOutputFilesBatch(hashes: readonly string[]): Map<string, OutputFileRow[]>
845
- /**
846
- * Stat each `expected` row's target under `projectDir` and return
847
- * `true` iff every (size, mode, mtime) matches. Missing files,
848
- * stat errors, or any mismatch → `false`.
849
- *
850
- * Pure FS check — no DB access. Caller batches the expected rows
851
- * via `loadOutputFilesBatch`. Lets the orchestrator skip
852
- * `cleanOutputs + restoreOutputs` entirely when the cached
853
- * snapshot is already in place. Integrity-preserving: detects
854
- * out-of-band file edits or deletions and falls through to a real
855
- * restore.
856
- */
857
- isOutputsCurrent(projectDir: string, expected: readonly OutputFileRow[]): Promise<boolean>
858
- /**
859
- * The directory-mtime short-circuit behind a warm hit (optional — a layer
860
- * without it keeps the output walk). `recordOutputDirs` snapshots every
861
- * directory under each of `prefixes` (project-relative, whole-subtree
862
- * globs only — see `wholeSubtreePrefixes`) after a save or restore;
863
- * `loadOutputDirsBatch` reads them back; `outputDirsCurrent` is true iff
864
- * every recorded directory still carries its recorded mtime, which proves
865
- * no file was added or removed anywhere the glob could see. Machine-local
866
- * state, like the output rows: a remote ingest records none.
867
- */
868
- recordOutputDirs?(hash: string, projectDir: string, prefixes: readonly string[]): Promise<void>
869
- loadOutputDirsBatch?(hashes: readonly string[]): Map<string, OutputDirRow[]>
870
- outputDirsCurrent?(projectDir: string, rows: readonly OutputDirRow[]): Promise<boolean>
871
- /**
872
- * Extract the artifact's `outputs/` entries into `projectDir` and —
873
- * when `workspaceRoot` is given — its `workspace-outputs/` entries
874
- * into the workspace root. Callers restoring entries that may carry
875
- * workspace outputs must pass `workspaceRoot`.
876
- */
877
- restoreOutputs(hash: string, projectDir: string, workspaceRoot?: string): Promise<void>
878
- save(args: {
879
- hash: string
880
- /**
881
- * `exitCode` is deliberately NOT accepted: vx caches only successes, so
882
- * the stored value is pinned to 0 and there is nothing for a caller to
883
- * decide. Accepting a number and discarding it invited the one shape that
884
- * launders a failure into a success — cache a failing task's outputs, read
885
- * the entry back as exit 0, and the hit classifies `cache-hit` while the
886
- * broken build's files are restored over a good tree. Unrepresentable
887
- * beats guarded.
888
- */
889
- entry: Omit<CacheEntry, 'hash' | 'storedAt' | 'outputFiles' | 'exitCode'>
890
- projectDir: string
891
- outputFiles: string[]
892
- /**
893
- * Set only by ChainedCache when an EARLIER layer in the chain already
894
- * wrote this artifact to the same local handle: the local pack + write
895
- * is skipped and only the layer's remote side acts. A layer with no
896
- * remote side treats it as a full no-op.
897
- */
898
- skipLocalWrite?: boolean
899
- /**
900
- * Resolved `outputs.workspaceFiles` (absolute paths) + the root
901
- * they're relative to. Packed under `workspace-outputs/<rel>`.
902
- * Omitted → artifact bytes identical to the pre-workspaceFiles
903
- * format.
904
- */
905
- workspaceOutputFiles?: string[]
906
- workspaceRoot?: string
907
- /**
908
- * Cache-key components for this entry (the Tier-3 input
909
- * fingerprint). Persisted to `entry_inputs` inside the same
910
- * transaction as the entry row, via `INSERT OR IGNORE` (a re-save
911
- * of the same hash is a no-op). Omitted/empty → nothing written.
912
- * Only computed on the miss/save path — never on a hit.
913
- */
914
- inputComponents?: readonly TaskInputRow[]
915
- }): Promise<void>
916
- /**
917
- * Adopt an artifact produced elsewhere — the remote-hit path. Writes
918
- * the compressed bytes to `<cacheDir>/<hash>.tar.zst`, parses the
919
- * tar headers to populate the `output_files` rows, and inserts the
920
- * `entries` row using the caller-supplied `meta`. After this returns,
921
- * the next `get(hash)` resolves locally.
922
- */
923
- ingest(hash: string, compressed: Uint8Array, meta: IngestMeta): Promise<void>
924
- recordRun(run: RunRecord): void
925
- /**
926
- * Append every run in `runs` to the history in a single SQLite
927
- * transaction. ~10× faster than calling `recordRun` in a loop when
928
- * `runs.length > ~50` (one fsync vs. N).
929
- */
930
- recordRuns(runs: readonly RunRecord[]): void
931
- /**
932
- * Record a whole `vx run` atomically: the per-task `runs` rows and
933
- * the one `invocations` header row — in a SINGLE transaction (one
934
- * fsync). Replaces the bare `recordRuns` call in the orchestrator's
935
- * end-of-run path. Input-fingerprint rows are NOT written here — they
936
- * ride the entry-save transaction (`save`/`ingest`) so a warm
937
- * all-cache-hit run writes nothing.
938
- */
939
- recordRunBundle(bundle: { runs: readonly RunRecord[]; invocation: InvocationRecord }): void
940
- stats(opts?: CacheStatsOptions): CacheStats
941
- /**
942
- * Content-hash a file with an mtime+size fast path. If the
943
- * `(mtime_ms, size_bytes)` of `filePath` match a previously seen
944
- * row, return the stored xxh3 digest instead of re-reading the
945
- * bytes. Otherwise read + hash + upsert. The hash is byte-for-byte
946
- * identical to what a fresh content-hash would produce — pure
947
- * optimization, no cache-key change.
948
- */
949
- hashFile(filePath: string): Promise<string>
950
- /**
951
- * Absolute path to the on-disk outputs artifact for a hash —
952
- * `<cacheDir>/<hash>.tar.zst`. Returns the path whether or
953
- * not the artifact exists. Exposed for telemetry / dashboards;
954
- * `restoreOutputs` is the canonical way to materialize the bytes.
955
- */
956
- outputsPath(hash: string): string
957
- prune(options: PruneOptions): Promise<PruneResult>
958
- close(): void
959
- }
960
-
961
- /**
962
- * Convenience alias for the `save()` args. Used by `LayeredCache` to
963
- * forward call args without redeclaring the structural shape — NOT
964
- * part of the conceptual cache contract; consumers should call
965
- * `CacheLayer.save({ ... })` directly.
966
- *
967
- * @internal
968
- */
969
- export type SaveArgs = Parameters<CacheLayer['save']>[0]
970
-
971
- interface EntryRow {
972
- hash: string
973
- project: string
974
- task: string
975
- command: string
976
- exit_code: number
977
- duration_ms: number
978
- size_bytes: number
979
- stdout: string
980
- created_at: number
981
- accessed_at: number
982
- }
983
-
984
- function entryOf(row: EntryRow, fileRows: OutputFileRow[]): CacheEntry {
985
- return {
986
- hash: row.hash,
987
- taskId: `${row.project}#${row.task}`,
988
- command: row.command,
989
- exitCode: row.exit_code,
990
- durationMs: row.duration_ms,
991
- outputFiles: fileRows.map((r) => r.path),
992
- outputRows: fileRows,
993
- stdout: row.stdout,
994
- storedAt: new Date(row.created_at).toISOString(),
995
- source: 'local',
996
- }
347
+ function errorText(err: unknown): string {
348
+ return err instanceof Error ? err.message : String(err)
997
349
  }
998
350
 
999
351
  export class Cache implements CacheLayer {
@@ -1005,19 +357,12 @@ export class Cache implements CacheLayer {
1005
357
  private readonly selectEntry: ReturnType<Database['prepare']>
1006
358
  private readonly bumpAccessed: ReturnType<Database['prepare']>
1007
359
  private readonly touched = new Set<string>()
1008
- private readonly insertRun: ReturnType<Database['prepare']>
1009
- private readonly insertInvocation: ReturnType<Database['prepare']>
1010
360
  private readonly insertEntryInput: ReturnType<Database['prepare']>
1011
- private readonly selectFileHash: ReturnType<Database['prepare']>
1012
- private readonly upsertFileHash: ReturnType<Database['prepare']>
1013
- private readonly insertOutputFile: ReturnType<Database['prepare']>
1014
- private readonly insertOutputDir: ReturnType<Database['prepare']>
1015
- private readonly deleteOutputDirs: ReturnType<Database['prepare']>
1016
- private readonly deleteOutputFiles: ReturnType<Database['prepare']>
1017
- private readonly selectConfigEval: ReturnType<Database['prepare']>
1018
- private readonly insertConfigEval: ReturnType<Database['prepare']>
1019
- /** Memoized repo object format for blob-OID hashing (lazy-detected). */
1020
- private objectFormat: 'sha1' | 'sha256' | null = null
361
+ /** The per-file (mtime, size) → blob-OID memo behind `hashFile`. */
362
+ private readonly files: FileHashStore
363
+ private readonly configEvals: ConfigEvalTable
364
+ private readonly outputs: OutputIndex
365
+ private readonly history: RunHistory
1021
366
 
1022
367
  /**
1023
368
  * Local-layer read/write gates for the task ARTIFACT path only.
@@ -1028,61 +373,171 @@ export class Cache implements CacheLayer {
1028
373
  */
1029
374
  private readonly read: boolean
1030
375
  private readonly write: boolean
376
+ /** A reading verb's handle (`Cache.inspect`): it prunes no history. */
377
+ private readonly inspecting: boolean
378
+ /** Why this process cannot write into `cacheDir`, or `null`; decided at open. */
379
+ private readonly writeBlocked: string | null
380
+
381
+ /**
382
+ * Set when THIS open found an index written by another `SCHEMA_VERSION`
383
+ * and dropped every table. The next open sees the current version and
384
+ * reports null, so whoever opened first is the one that can say so —
385
+ * `noteSchemaReset` is that one line, at every opener that has a
386
+ * channel. Without it an upgrade empties the cache and the history in
387
+ * silence, and the all-miss run that follows looks like a bug.
388
+ */
389
+ readonly schemaReset: SchemaReset | null = null
390
+
391
+ /**
392
+ * Set when THIS open found entries written under another
393
+ * `CACHE_VERSION`: the index survives, but no old key is derived again,
394
+ * so every cached task misses once. Reported by `noteSchemaReset`.
395
+ */
396
+ readonly formatChange: SchemaReset | null = null
397
+
398
+ /** A reading verb's open: it never resets the index (the constructor's `mode`). */
399
+ static inspect(cacheDir: string): Cache {
400
+ return new Cache(cacheDir, undefined, undefined, undefined, 'inspect')
401
+ }
1031
402
 
1032
403
  constructor(
1033
404
  private readonly cacheDir: string,
1034
405
  localPolicy: { read: boolean; write: boolean } = { read: true, write: true },
406
+ /** The workspace root, where the file hasher asks git for the object format (file-hashes.ts). */
407
+ repoDir?: string,
408
+ /**
409
+ * The largest artifact, decoded, this cache saves, ingests or
410
+ * restores (zstd.ts). Lowered only by a test, through
411
+ * `RunOptions.artifactCeiling`: 2 GiB of output is out of a test's reach.
412
+ */
413
+ private readonly artifactCeiling: number = MAX_DECOMPRESSED_ARTIFACT_BYTES,
414
+ /**
415
+ * `'inspect'`: a reading verb (`why`, `last`, `info`, a dry prune). It
416
+ * never resets the index: a schema it cannot read is refused, named,
417
+ * and left as it was.
418
+ */
419
+ mode: 'open' | 'inspect' = 'open',
1035
420
  ) {
421
+ this.inspecting = mode === 'inspect'
1036
422
  this.read = localPolicy.read
1037
- this.write = localPolicy.write
1038
- // Ensure the directory exists before opening the DB — bun:sqlite
1039
- // won't create parent dirs for us. The constructor stays sync
1040
- // because callers use `new Cache(...)` directly; `mkdirSync` keeps
1041
- // that property without a subprocess fork.
1042
- mkdirSync(cacheDir, { recursive: true })
1043
- // Make the cache dir invisible to git, every time it is created: a
1044
- // `*` .gitignore inside it (the Cargo / Nx convention). Two reasons,
1045
- // both measured. A cache nobody ignored gets COMMITTED by the next
1046
- // `git add -A`; and vx's own `git status -uall` walks it — 1000
1047
- // artifacts doubled the enumeration on the bench workspace before
1048
- // its generator ignored `.vx`. An ignored directory is skipped by the
1049
- // walk entirely. Only written when absent, so a user's own file wins.
1050
- const ignore = path.join(cacheDir, '.gitignore')
1051
- if (!existsSync(ignore)) writeFileSync(ignore, '*\n')
1052
- this.db = new Database(path.join(cacheDir, 'cache.db'), { create: true })
1053
- this.db.exec('PRAGMA journal_mode = WAL')
1054
- this.db.exec('PRAGMA synchronous = NORMAL')
1055
- this.db.exec('PRAGMA foreign_keys = ON')
423
+ // The directory exists before the DB opens — bun:sqlite won't create
424
+ // parent dirs for us. A directory this user cannot write into is a
425
+ // read-only cache for every reader (`vx show`, `why`, `last`, the
426
+ // doctor, the watch sweep): the write axis goes off, so the
427
+ // config-evaluation store and the file-hash memo skip their upserts
428
+ // instead of dying in SQLite on the first miss (an unprivileged user on
429
+ // a root-owned `.vx`, 2026-09-16). A run wants more than a quiet
430
+ // read-only cache — `assertWritable()`.
431
+ // A reading verb over a directory with no index yet reads an empty one
432
+ // in memory and makes nothing on disk: `vx last --cache-dir .vx/cahce`
433
+ // created the typo's directory, a `.gitignore` and a database, then
434
+ // said "no recorded runs yet" (item 900).
435
+ const dbFile = path.join(cacheDir, 'cache.db')
436
+ const absent = mode === 'inspect' && !existsSync(dbFile)
437
+ this.writeBlocked = absent ? 'no index there yet' : openCacheDir(cacheDir)
438
+ this.write = localPolicy.write && this.writeBlocked === null
439
+ this.db = new Database(absent ? ':memory:' : dbFile, { create: true })
1056
440
  // busy_timeout makes concurrent writers wait for the lock instead of
1057
- // failing immediately with SQLITE_BUSY. Two parallel `vx run`
1058
- // invocations in CI is a normal pattern; without this the second one
1059
- // crashes in recordRun().
1060
- this.db.exec('PRAGMA busy_timeout = 5000')
1061
-
1062
- this.db.exec(`
1063
- CREATE TABLE IF NOT EXISTS schema_meta (
1064
- key TEXT PRIMARY KEY,
1065
- value TEXT NOT NULL
1066
- );
1067
- `)
441
+ // failing immediately with SQLITE_BUSY. Two parallel `vx run`
442
+ // invocations in CI is a normal pattern; without this the second one
443
+ // crashes in recordRun(). FIRST, before the journal-mode switch: that
444
+ // pragma takes a lock of its own, and set after it the timeout did not
445
+ // cover it — two CLI runs opening one cache at the same instant met
446
+ // `database is locked` right here, on macOS's slower disk first
447
+ // (2026-09-16, the run lock's own e2e; the lock is taken after the
448
+ // cache opens, so the open is the one moment two runs still overlap).
449
+ // A file SQLite cannot read as a database answers the first statement
450
+ // that reads it, and reached every verb as `SQLiteError: file is not a
451
+ // database` and a stack, `vx run` and the doctor included (item 1005).
452
+ const readable = <T>(read: () => T): T => {
453
+ try {
454
+ return read()
455
+ } catch (err) {
456
+ if (!unreadableIndex(err)) throw err
457
+ this.db.close()
458
+ throw new UserError(
459
+ `the cache index ${dbFile} is not a database SQLite can read (${(err as Error).message}) — ` +
460
+ `it holds nothing a run cannot rebuild: remove it with its -wal and -shm files and ` +
461
+ `re-run, and \`vx cache prune\` reclaims the artifacts it indexed`,
462
+ )
463
+ }
464
+ }
465
+ readable(() => {
466
+ this.db.exec('PRAGMA busy_timeout = 5000')
467
+ this.db.exec('PRAGMA journal_mode = WAL')
468
+ this.db.exec('PRAGMA synchronous = NORMAL')
469
+ this.db.exec('PRAGMA foreign_keys = ON')
470
+ this.db.exec(`
471
+ CREATE TABLE IF NOT EXISTS schema_meta (
472
+ key TEXT PRIMARY KEY,
473
+ value TEXT NOT NULL
474
+ );
475
+ `)
476
+ })
1068
477
 
1069
478
  // Schema-version gate runs BEFORE the rest of the schema lands so
1070
479
  // a column rename (e.g. v15's `sha256` → `content_hash`) actually
1071
480
  // takes effect on stale DBs. Pre-alpha: no migrations, just drop
1072
- // and recreate. Outputs on disk become orphans; they'll be ignored
1073
- // on next miss and reaped by `vx cache prune`.
1074
- const meta = this.db.prepare("SELECT value FROM schema_meta WHERE key = 'version'").get() as
1075
- | { value: string }
1076
- | undefined
1077
- if (meta && meta.value !== SCHEMA_VERSION) {
1078
- this.db.exec(
1079
- 'DROP TABLE IF EXISTS entries; DROP TABLE IF EXISTS runs; DROP TABLE IF EXISTS file_hashes; DROP TABLE IF EXISTS output_files; DROP TABLE IF EXISTS invocations; DROP TABLE IF EXISTS run_task_inputs; DROP TABLE IF EXISTS entry_inputs; DROP TABLE IF EXISTS config_evals;',
1080
- )
1081
- this.db.prepare("UPDATE schema_meta SET value = ? WHERE key = 'version'").run(SCHEMA_VERSION)
1082
- } else if (!meta) {
1083
- this.db
1084
- .prepare("INSERT INTO schema_meta(key, value) VALUES ('version', ?)")
1085
- .run(SCHEMA_VERSION)
481
+ // and recreate. Artifacts on disk become orphans: a lookup never
482
+ // sees them (rows first, then the file), the next save of the same
483
+ // key renames over them, and `prune()`'s orphan sweep unlinks the
484
+ // rest.
485
+ //
486
+ // The write half runs under the write lock and reads the row again: two
487
+ // processes opening one NEW cache both read no row, and the second
488
+ // insert died on the primary key (upstream survey, nx#28608). A warm
489
+ // open reads the current version and takes no lock.
490
+ const readVersion = (): string | undefined =>
491
+ (
492
+ this.db.prepare("SELECT value FROM schema_meta WHERE key = 'version'").get() as
493
+ | { value: string }
494
+ | undefined
495
+ )?.value
496
+ // Only an EARLIER schema is reset, and only by an opener that may write
497
+ // it. A newer one is another vx's index and history: an older binary
498
+ // (a global install beside a workspace's own) dropped every table of it
499
+ // and announced "vx upgraded", and so did a reading verb, a dry prune
500
+ // among them (item 896).
501
+ const refuseUnreadable = (found: string): void => {
502
+ if (schemaOrdinal(found) > schemaOrdinal(SCHEMA_VERSION)) {
503
+ throw new UserError(
504
+ `the cache at ${cacheDir} holds index schema ${found}, written by a newer vx; this vx reads ${SCHEMA_VERSION} and leaves it untouched. Run the newer vx, or give this one another --cache-dir`,
505
+ )
506
+ }
507
+ if (mode === 'inspect') {
508
+ throw new UserError(
509
+ `the cache at ${cacheDir} holds index schema ${found} from an earlier vx; this vx reads ${SCHEMA_VERSION}, so nothing in it is readable here. The next \`vx run\` resets it; a reading verb leaves it untouched`,
510
+ )
511
+ }
512
+ }
513
+ const current = readable(readVersion)
514
+ if (current !== SCHEMA_VERSION) {
515
+ try {
516
+ if (current !== undefined) refuseUnreadable(current)
517
+ this.schemaReset = this.db
518
+ .transaction((): SchemaReset | null => {
519
+ const found = readVersion()
520
+ if (found !== undefined && found !== SCHEMA_VERSION) refuseUnreadable(found)
521
+ if (found === undefined) {
522
+ this.db
523
+ .prepare("INSERT INTO schema_meta(key, value) VALUES ('version', ?)")
524
+ .run(SCHEMA_VERSION)
525
+ return null
526
+ }
527
+ if (found === SCHEMA_VERSION) return null
528
+ this.db.exec(
529
+ 'DROP TABLE IF EXISTS entries; DROP TABLE IF EXISTS runs; DROP TABLE IF EXISTS file_hashes; DROP TABLE IF EXISTS output_files; DROP TABLE IF EXISTS invocations; DROP TABLE IF EXISTS run_task_inputs; DROP TABLE IF EXISTS entry_inputs; DROP TABLE IF EXISTS config_evals;',
530
+ )
531
+ this.db
532
+ .prepare("UPDATE schema_meta SET value = ? WHERE key = 'version'")
533
+ .run(SCHEMA_VERSION)
534
+ return { from: found, to: SCHEMA_VERSION }
535
+ })
536
+ .immediate()
537
+ } catch (err) {
538
+ this.db.close()
539
+ throw err
540
+ }
1086
541
  }
1087
542
 
1088
543
  // Cached config evaluations (workspace/config-cache.ts): the validated
@@ -1111,7 +566,10 @@ export class Cache implements CacheLayer {
1111
566
  size_bytes INTEGER NOT NULL,
1112
567
  stdout TEXT NOT NULL DEFAULT '',
1113
568
  created_at INTEGER NOT NULL,
1114
- accessed_at INTEGER NOT NULL
569
+ accessed_at INTEGER NOT NULL,
570
+ -- v26: the producing execution's usage, from the artifact's sidecar.
571
+ cpu_ms INTEGER,
572
+ peak_rss_bytes INTEGER
1115
573
  );
1116
574
  CREATE TABLE IF NOT EXISTS runs (
1117
575
  id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -1137,18 +595,46 @@ export class Cache implements CacheLayer {
1137
595
  cache_hit INTEGER,
1138
596
  -- v23: attempts a retried task took (>1); NULL for a once-run task.
1139
597
  -- The direct within-run flaky signal.
1140
- attempts INTEGER
598
+ attempts INTEGER,
599
+ -- v25: 1 when the task declared a cache block, 0 when it runs every
600
+ -- time by design; NULL on rows older than the column.
601
+ cached INTEGER,
602
+ -- v27: why a task failed or was skipped, as the run's own footer
603
+ -- said it (items 267–270): a skip's root blocker (a task id), vx's
604
+ -- own timeout (1), the sandbox's violation count, and why a
605
+ -- persistent task never became ready ('timeout' | 'exited' |
606
+ -- 'spawn'). NULL where the reason does not apply.
607
+ blocked_by TEXT,
608
+ timed_out INTEGER,
609
+ sandbox_violations INTEGER,
610
+ not_ready TEXT
1141
611
  );
1142
- -- runs_hash had no reader: every consumer of runs.hash looks the row
1143
- -- up by run_id or (project, task) first. Dropped 2026-09-03 — it cost
1144
- -- 1,000 warm-run inserts 11.5 → 3.9 ms; the DROP sheds it from
1145
- -- existing databases (a schema-meta bump is for stored shapes, and
1146
- -- an index is not one).
612
+ -- Two whole-table indexes only, and both APPEND: every row of a run carries the
613
+ -- same run_id and a started_at newer than everything before it, so
614
+ -- 1,000 inserts touch a handful of leaf pages. The dropped ones did
615
+ -- not — the DROPs shed them from existing databases (a schema-meta
616
+ -- bump is for stored shapes, and an index is not one):
617
+ -- runs_hash (2026-09-03) had no reader; 11.5 → 3.9 ms per 1,000.
618
+ -- runs_project (project, task) (2026-09-09) scattered every run's
619
+ -- rows over one leaf per pair: the record stage of a 1,000-hit
620
+ -- warm run was 57–79 ms with it and 14–19 ms without, at 166k
621
+ -- rows, and it bought its readers nothing — the history reader
622
+ -- now scans a rowid slice (history.ts), vx why walks
623
+ -- started_at newest-first and stops at the first match.
624
+ -- runs_ended (2026-09-09) served only the retention DELETE, which
625
+ -- prunes on started_at now (a row ends after it starts, so the
626
+ -- 30-day window moves by at most one task's duration).
1147
627
  DROP INDEX IF EXISTS runs_hash;
628
+ DROP INDEX IF EXISTS runs_project;
629
+ DROP INDEX IF EXISTS runs_ended;
1148
630
  CREATE INDEX IF NOT EXISTS runs_started_at ON runs(started_at);
1149
- CREATE INDEX IF NOT EXISTS runs_project ON runs(project, task);
1150
- CREATE INDEX IF NOT EXISTS runs_ended ON runs(ended_at);
1151
631
  CREATE INDEX IF NOT EXISTS runs_run_id ON runs(run_id);
632
+ -- The one keyed index, PARTIAL over failed rows: a green run's
633
+ -- inserts only evaluate its predicate, so the append-only cost above
634
+ -- holds, and the flakiness probe after a miss (failure-mode.ts,
635
+ -- "did this key ever fail?") reads a handful of leaves instead of
636
+ -- scanning the table (2026-09-10: 10–95 ms at 170k rows without it).
637
+ CREATE INDEX IF NOT EXISTS runs_failed ON runs(hash) WHERE status = 'failed';
1152
638
  -- Per-file (mtime, size, content_hash) cache. Lets Cache.key()
1153
639
  -- skip the content-hash on inputs whose stat hasn't changed
1154
640
  -- since the last run. Pure performance optimization; the stored
@@ -1178,14 +664,13 @@ export class Cache implements CacheLayer {
1178
664
  size_bytes INTEGER NOT NULL,
1179
665
  mode INTEGER NOT NULL,
1180
666
  mtime_ms INTEGER NOT NULL,
667
+ -- v28: the inode and ctime this machine saw after the save or
668
+ -- restore that left the file equal to the entry; NULL until then.
669
+ ino INTEGER,
670
+ ctime_ms INTEGER,
1181
671
  PRIMARY KEY (entry_hash, path),
1182
672
  FOREIGN KEY (entry_hash) REFERENCES entries(hash) ON DELETE CASCADE
1183
673
  );
1184
- -- Every directory under a whole-subtree output glob, with its mtime
1185
- -- as of the last save/restore on THIS machine (2026-09-03). On a warm
1186
- -- hit, unchanged mtimes prove the output SET is unchanged, replacing
1187
- -- the glob walk that cost 0.36 ms per hit. Machine-local: a remote
1188
- -- ingest writes none, and the first hit after it walks and records.
1189
674
  -- Each config's ORDERED import closure (the config first), so a warm
1190
675
  -- load keys it by stat-hashing the list (the file_hashes memo) instead
1191
676
  -- of reading and scanning every file. Machine-local; pruned with
@@ -1195,6 +680,11 @@ export class Cache implements CacheLayer {
1195
680
  files_json TEXT NOT NULL,
1196
681
  created_at INTEGER NOT NULL
1197
682
  );
683
+ -- Every directory under a whole-subtree output glob, with its mtime
684
+ -- as of the last save/restore on THIS machine (2026-09-03). On a warm
685
+ -- hit, unchanged mtimes prove the output SET is unchanged, replacing
686
+ -- the glob walk that cost 0.36 ms per hit. Machine-local: a remote
687
+ -- ingest writes none, and the first hit after it walks and records.
1198
688
  CREATE TABLE IF NOT EXISTS output_dirs (
1199
689
  entry_hash TEXT NOT NULL,
1200
690
  path TEXT NOT NULL,
@@ -1258,41 +748,22 @@ export class Cache implements CacheLayer {
1258
748
  `)
1259
749
 
1260
750
  this.insertEntry = this.db.prepare(`
1261
- INSERT INTO entries(hash, project, task, command, exit_code, duration_ms, size_bytes, stdout, created_at, accessed_at)
1262
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
751
+ INSERT INTO entries(hash, project, task, command, exit_code, duration_ms, size_bytes, stdout, created_at, accessed_at, cpu_ms, peak_rss_bytes)
752
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
1263
753
  ON CONFLICT(hash) DO UPDATE SET
1264
- stdout = excluded.stdout,
1265
- project = excluded.project,
1266
- task = excluded.task,
1267
- command = excluded.command,
1268
- exit_code = excluded.exit_code,
1269
- duration_ms = excluded.duration_ms,
1270
- size_bytes = excluded.size_bytes,
1271
- accessed_at = excluded.accessed_at
754
+ stdout = excluded.stdout,
755
+ project = excluded.project,
756
+ task = excluded.task,
757
+ command = excluded.command,
758
+ exit_code = excluded.exit_code,
759
+ duration_ms = excluded.duration_ms,
760
+ size_bytes = excluded.size_bytes,
761
+ accessed_at = excluded.accessed_at,
762
+ cpu_ms = excluded.cpu_ms,
763
+ peak_rss_bytes = excluded.peak_rss_bytes
1272
764
  `)
1273
765
  this.selectEntry = this.db.prepare('SELECT * FROM entries WHERE hash = ?')
1274
766
  this.bumpAccessed = this.db.prepare('UPDATE entries SET accessed_at = ? WHERE hash = ?')
1275
- this.insertRun = this.db.prepare(`
1276
- INSERT INTO runs(
1277
- hash, project, task, status, exit_code, duration_ms, forward_args,
1278
- started_at, ended_at,
1279
- run_id, cpu_ms, peak_rss_bytes, wallclock_start_ns, wallclock_end_ns,
1280
- cache_hit, attempts
1281
- )
1282
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
1283
- `)
1284
- this.insertInvocation = this.db.prepare(`
1285
- INSERT INTO invocations(
1286
- run_id, command, requested_tasks, cache_policy, concurrency, flow,
1287
- started_at, ended_at, total_duration_ms,
1288
- task_count, failed_count, hit_count, hit_local_count, hit_remote_count,
1289
- exit_ok,
1290
- commit_sha, branch, dirty, ci, ci_provider,
1291
- host, os, arch, vx_version, tags
1292
- )
1293
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
1294
- ON CONFLICT(run_id) DO NOTHING
1295
- `)
1296
767
  // INSERT OR IGNORE: re-saving the same hash (idempotent ingest /
1297
768
  // overlapping concurrent saves) leaves the existing rows untouched —
1298
769
  // identical inputs derive the identical hash, so the rows are too.
@@ -1300,304 +771,90 @@ export class Cache implements CacheLayer {
1300
771
  INSERT OR IGNORE INTO entry_inputs(entry_hash, kind, name, hash)
1301
772
  VALUES (?, ?, ?, ?)
1302
773
  `)
1303
- this.selectFileHash = this.db.prepare(
1304
- 'SELECT mtime_ms, size_bytes, ctime_ms, ino, content_hash FROM file_hashes WHERE path = ?',
1305
- )
1306
- this.upsertFileHash = this.db.prepare(`
1307
- INSERT INTO file_hashes(path, mtime_ms, size_bytes, ctime_ms, ino, content_hash, seen_at)
1308
- VALUES (?, ?, ?, ?, ?, ?, ?)
1309
- ON CONFLICT(path) DO UPDATE SET
1310
- mtime_ms = excluded.mtime_ms,
1311
- size_bytes = excluded.size_bytes,
1312
- ctime_ms = excluded.ctime_ms,
1313
- ino = excluded.ino,
1314
- content_hash = excluded.content_hash,
1315
- seen_at = excluded.seen_at
1316
- `)
1317
- this.insertOutputFile = this.db.prepare(`
1318
- INSERT INTO output_files(entry_hash, path, size_bytes, mode, mtime_ms)
1319
- VALUES (?, ?, ?, ?, ?)
1320
- ON CONFLICT(entry_hash, path) DO UPDATE SET
1321
- size_bytes = excluded.size_bytes,
1322
- mode = excluded.mode,
1323
- mtime_ms = excluded.mtime_ms
1324
- `)
1325
- this.deleteOutputFiles = this.db.prepare('DELETE FROM output_files WHERE entry_hash = ?')
1326
- this.insertOutputDir = this.db.prepare(
1327
- 'INSERT INTO output_dirs(entry_hash, path, mtime_ms) VALUES (?, ?, ?)',
1328
- )
1329
- this.deleteOutputDirs = this.db.prepare('DELETE FROM output_dirs WHERE entry_hash = ?')
1330
- this.selectConfigEval = this.db.prepare('SELECT json FROM config_evals WHERE key = ?')
1331
- this.insertConfigEval = this.db.prepare(
1332
- 'INSERT OR REPLACE INTO config_evals(key, json, created_at) VALUES (?, ?, ?)',
1333
- )
774
+ // The slices: each owns its statements over this handle and its table(s);
775
+ // the schema above is the one place every table is declared.
776
+ this.files = new FileHashStore(this.db, cacheDir, this.write, repoDir)
777
+ this.configEvals = new ConfigEvalTable(this.db, { read: this.read, write: this.write })
778
+ this.outputs = new OutputIndex(this.db)
779
+ this.history = new RunHistory(this.db)
780
+
781
+ // A CACHE_VERSION bump keeps the index but moves every key, so the run
782
+ // after an upgrade misses everything. Roadmap 3.3: that is announced,
783
+ // never silent. A store with no recorded version and no entries is
784
+ // new; one with entries predates the record (item 671).
785
+ const format = this.db
786
+ .prepare("SELECT value FROM schema_meta WHERE key = 'cache_version'")
787
+ .get() as { value: string } | null
788
+ if (format?.value !== CACHE_VERSION && this.writeBlocked === null) {
789
+ const hasEntries = this.db.prepare('SELECT 1 FROM entries LIMIT 1').get() != null
790
+ if (this.schemaReset === null && (format !== null || hasEntries)) {
791
+ this.formatChange = { from: format?.value ?? 'an earlier format', to: CACHE_VERSION }
792
+ }
793
+ this.db
794
+ .prepare(
795
+ "INSERT INTO schema_meta(key, value) VALUES ('cache_version', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value",
796
+ )
797
+ .run(CACHE_VERSION)
798
+ }
1334
799
  }
1335
800
 
1336
- /** `ConfigEvalStore`: a cached config evaluation, honouring the local READ axis. */
801
+ // --- config evaluations: `ConfigEvalStore`, delegated to `ConfigEvalTable` ---
1337
802
  getConfigEval(key: string): string | null {
1338
- if (!this.read) return null
1339
- const row = this.selectConfigEval.get(key) as { json: string } | null
1340
- return row?.json ?? null
803
+ return this.configEvals.getConfigEval(key)
1341
804
  }
1342
-
1343
- /** `ConfigEvalStore`: each config's ordered closure, one `IN` query per 900 paths. */
1344
805
  getConfigClosures(configPaths: readonly string[]): Map<string, string[]> {
1345
- const out = new Map<string, string[]>()
1346
- if (!this.read || configPaths.length === 0) return out
1347
- for (let i = 0; i < configPaths.length; i += 900) {
1348
- const chunk = configPaths.slice(i, i + 900)
1349
- const rows = this.db
1350
- .query(
1351
- `SELECT config_path, files_json FROM config_closures WHERE config_path IN (${chunk.map(() => '?').join(',')})`,
1352
- )
1353
- .all(...(chunk as readonly SQLQueryBindings[])) as Array<{
1354
- config_path: string
1355
- files_json: string
1356
- }>
1357
- for (const r of rows) out.set(r.config_path, JSON.parse(r.files_json) as string[])
1358
- }
1359
- return out
806
+ return this.configEvals.getConfigClosures(configPaths)
1360
807
  }
1361
-
1362
- /** `ConfigEvalStore`: remember a config's ordered closure, honouring the local WRITE axis. */
1363
808
  putConfigClosure(configPath: string, files: readonly string[]): void {
1364
- if (!this.write) return
1365
- this.db
1366
- .prepare(
1367
- 'INSERT INTO config_closures(config_path, files_json, created_at) VALUES (?, ?, ?) ON CONFLICT(config_path) DO UPDATE SET files_json = excluded.files_json, created_at = excluded.created_at',
1368
- )
1369
- .run(configPath, JSON.stringify(files), Date.now())
809
+ this.configEvals.putConfigClosure(configPath, files)
1370
810
  }
1371
-
1372
- /** `ConfigEvalStore`: the batched read — one `IN` query per 900 keys, honouring the local READ axis. */
1373
811
  getConfigEvals(keys: readonly string[]): Map<string, string> {
1374
- const out = new Map<string, string>()
1375
- if (!this.read || keys.length === 0) return out
1376
- for (let i = 0; i < keys.length; i += 900) {
1377
- const chunk = keys.slice(i, i + 900)
1378
- const rows = this.db
1379
- .query(
1380
- `SELECT key, json FROM config_evals WHERE key IN (${chunk.map(() => '?').join(',')})`,
1381
- )
1382
- .all(...(chunk as readonly SQLQueryBindings[])) as Array<{ key: string; json: string }>
1383
- for (const r of rows) out.set(r.key, r.json)
1384
- }
1385
- return out
812
+ return this.configEvals.getConfigEvals(keys)
1386
813
  }
1387
-
1388
- /** `ConfigEvalStore`: remember a validated evaluation, honouring the local WRITE axis. */
1389
814
  putConfigEval(key: string, json: string): void {
1390
- if (!this.write) return
1391
- this.insertConfigEval.run(key, json, Date.now())
815
+ this.configEvals.putConfigEval(key, json)
1392
816
  }
1393
-
1394
- /**
1395
- * Content-hash a file (as a git blob OID, v20) with an mtime+size
1396
- * fast path. If the `file_hashes` table has a row for `path` whose
1397
- * `(mtime_ms, size_bytes, ctime_ms, ino)` all match the current
1398
- * stat, we reuse the stored content_hash (a memory + SQLite lookup,
1399
- * no disk read). Otherwise we read + hash + upsert. All four fields
1400
- * are load-bearing — see the comment at the comparison.
1401
- *
1402
- * The OID is byte-identical to what `git hash-object` (and the git
1403
- * index) computes for the same content, so this fallback and the
1404
- * `CacheKeyInput.fileHashes` index-OID fast path agree on any file
1405
- * git stores verbatim. They do NOT agree when a clean filter
1406
- * (`text`/`eol`/`ident`) is active: the index blob is the filtered
1407
- * form while this hashes the worktree bytes, so `inputs.ts` drops
1408
- * the index OID for those paths and routes them here.
1409
- */
1410
- async hashFile(filePath: string): Promise<string> {
1411
- // statSync intentional: a single stat is ~1.6µs (Bun 1.3); the
1412
- // async-stat equivalent adds ~75µs of Promise machinery per call.
1413
- // Promise.all over the batched callers (key derivation) gives no
1414
- // I/O parallelism benefit because the stat is faster than the
1415
- // threadpool dispatch overhead.
1416
- let st
1417
- try {
1418
- st = statSync(filePath)
1419
- } catch {
1420
- // Caller is responsible for skipping files that don't exist;
1421
- // fall through to the content-hash path which will throw with
1422
- // a more useful error.
1423
- return await this.hashFileFromDisk(filePath)
1424
- }
1425
- const mtimeMs = Math.floor(st.mtimeMs)
1426
- const size = st.size
1427
- // ctime + ino are what make this memo SAFE, not merely fast. mtime is
1428
- // caller-settable, so (mtime, size) alone hands back the previous run's
1429
- // digest for genuinely different bytes whenever a producer preserves
1430
- // mtime — `tar -x`, `unzip`, `cp -p`, `rsync --times`, any
1431
- // SOURCE_DATE_EPOCH generator — which is a stale cache hit. `utimes`
1432
- // cannot suppress ctime without root, and an atomic write-then-rename
1433
- // changes the inode; git's own index keys on ctime+ino+dev for exactly
1434
- // this reason. Both fields come from the stat we already took.
1435
- const ctimeMs = Math.floor(st.ctimeMs)
1436
- const ino = Number(st.ino)
1437
- const row = this.selectFileHash.get(filePath) as
1438
- | {
1439
- mtime_ms: number
1440
- size_bytes: number
1441
- ctime_ms: number
1442
- ino: number
1443
- content_hash: string
1444
- }
1445
- | undefined
1446
- if (
1447
- row &&
1448
- row.mtime_ms === mtimeMs &&
1449
- row.size_bytes === size &&
1450
- row.ctime_ms === ctimeMs &&
1451
- row.ino === ino
1452
- ) {
1453
- return row.content_hash
1454
- }
1455
- const ch = await this.hashFileFromDisk(filePath)
1456
- this.upsertFileHash.run(filePath, mtimeMs, size, ctimeMs, ino, ch, Date.now())
1457
- return ch
817
+ putConfigEvals(entries: ReadonlyArray<readonly [string, string]>): void {
818
+ this.configEvals.putConfigEvals(entries)
1458
819
  }
1459
-
1460
- /**
1461
- * Git blob OID of the file's bytes:
1462
- * `hex(HASH("blob " + byteLength + "\0" + content))`, where HASH is
1463
- * the repo's object format. Same value `git hash-object` prints and
1464
- * the same value the index stores. Computed in-process — no git
1465
- * spawn per file.
1466
- */
1467
- private async hashFileFromDisk(filePath: string): Promise<string> {
1468
- const bytes = await Bun.file(filePath).bytes()
1469
- const hasher = new Bun.CryptoHasher(this.objectFormat ?? this.detectObjectFormat(filePath))
1470
- hasher.update(`blob ${bytes.byteLength}\0`)
1471
- hasher.update(bytes)
1472
- return hasher.digest('hex')
820
+ putConfigClosures(entries: ReadonlyArray<readonly [string, readonly string[]]>): void {
821
+ this.configEvals.putConfigClosures(entries)
822
+ }
823
+ // --- input file hashes: delegated to `FileHashStore` (see file-hashes.ts) ---
824
+ hashFile(filePath: string): Promise<string> {
825
+ return this.files.hashFile(filePath)
826
+ }
827
+ hashBytes(bytes: Uint8Array, nearPath: string): string {
828
+ return this.files.hashBytes(bytes, nearPath)
829
+ }
830
+ hashFiles(paths: readonly string[]): Promise<Map<string, string>> {
831
+ return this.files.hashFiles(paths)
1473
832
  }
1474
-
1475
833
  /**
1476
- * Repo object format — sha1 unless the repo was created with
1477
- * `--object-format=sha256`. One `git rev-parse` spawn per Cache
1478
- * lifetime, and only when at least one file misses the mtime+size
1479
- * memo. Outside a repo (unit fixtures) we default to sha1, which is
1480
- * still a deterministic blob-OID domain.
834
+ * `relPosix` against the run's workspace root, memoized: the same three
835
+ * thousand files are re-relativized for every task of the project, which
836
+ * is 132,000 calls for 3,000 answers on this repo's own gate. Cleared
837
+ * when a caller arrives with a different root, so the memo can never
838
+ * answer for a workspace it did not measure.
1481
839
  */
1482
- private detectObjectFormat(nearPath: string): 'sha1' | 'sha256' {
1483
- let detected: 'sha1' | 'sha256' = 'sha1'
1484
- try {
1485
- const proc = Bun.spawnSync({
1486
- cmd: ['git', 'rev-parse', '--show-object-format'],
1487
- cwd: path.dirname(nearPath),
1488
- stdout: 'pipe',
1489
- stderr: 'pipe',
1490
- })
1491
- if (proc.exitCode === 0 && new TextDecoder().decode(proc.stdout).trim() === 'sha256') {
1492
- detected = 'sha256'
1493
- }
1494
- } catch {
1495
- // git unavailable → sha1 default keeps hashing deterministic.
840
+ private relMemo = new Map<string, string>()
841
+ private relMemoRoot: string | undefined
842
+ private relFor(root: string, file: string): string {
843
+ if (root !== this.relMemoRoot) {
844
+ this.relMemoRoot = root
845
+ this.relMemo.clear()
1496
846
  }
1497
- this.objectFormat = detected
1498
- return detected
847
+ let rel = this.relMemo.get(file)
848
+ if (rel === undefined) this.relMemo.set(file, (rel = relPosix(root, file)))
849
+ return rel
1499
850
  }
1500
851
 
1501
- async key(input: CacheKeyInput): Promise<string> {
1502
- // Seed-chained xxHash3: each step folds one field into the
1503
- // running digest via `xxh3(part, prevDigest)`. Equivalent to the
1504
- // old CryptoHasher.update() pattern, no intermediate buffer.
1505
- // Field-order matters; each line is prefixed with its label so
1506
- // adjacent fields can't collide via concat.
1507
- // Optional capture sink — see CacheKeyInput.captureInto. Pure
1508
- // side-channel; every push below mirrors the fold immediately
1509
- // above/beside it so the recorded set can't drift from the key.
1510
- const cap = input.captureInto
1511
- let h = xxh3(CACHE_VERSION)
1512
- h = xxh3(`task:${input.taskId}`, h)
1513
- h = xxh3(`workspace:${input.workspaceFingerprint}`, h)
1514
- h = xxh3(`pkg:${input.projectPackageJsonHash}`, h)
1515
- h = xxh3(`config:${input.taskConfigHash}`, h)
1516
- if (cap) {
1517
- cap.push({ kind: 'workspace', name: 'fingerprint', hash: input.workspaceFingerprint })
1518
- cap.push({ kind: 'package', name: 'package.json', hash: input.projectPackageJsonHash })
1519
- cap.push({ kind: 'config', name: 'config', hash: input.taskConfigHash })
1520
- }
1521
-
1522
- // NOTE on `cap` (entry_inputs capture): the `hash` field stores a
1523
- // DIGEST of each component, never the raw value. For value-bearing kinds
1524
- // (env, runtime, ws-runtime, forward) the payload is a secret or
1525
- // sensitive string (API keys via cache.inputs.env, runtime-command
1526
- // output, args after `--`), so we push `xxh3hex(v)` — the diff consumer
1527
- // only needs to know whether a component CHANGED, which a digest
1528
- // preserves losslessly. cache.db must never hold plaintext secrets at
1529
- // rest. The cache KEY (`h`) folds the plaintext separately below and is
1530
- // unaffected.
1531
- const forwarded = input.forwardArgs ?? []
1532
- h = xxh3(`forward-args:${forwarded.length}`, h)
1533
- for (const a of forwarded) h = xxh3(a, h)
1534
- if (cap && forwarded.length > 0) {
1535
- cap.push({ kind: 'forward', name: 'argv', hash: xxh3hex(JSON.stringify(forwarded)) })
1536
- }
1537
-
1538
- h = xxh3(`env-values:${input.envValues.length}`, h)
1539
- // \0 delimiter, not `=`: names and values may themselves contain
1540
- // `=`, and `A` + `B=C` must never fold the same bytes as `A=B` + `C`.
1541
- for (const [n, v] of input.envValues) h = xxh3(`${n}\0${v}`, h)
1542
- if (cap)
1543
- for (const [n, v] of input.envValues) cap.push({ kind: 'env', name: n, hash: xxh3hex(v) })
1544
-
1545
- const runtimeValues = input.runtimeValues ?? []
1546
- h = xxh3(`runtime-values:${runtimeValues.length}`, h)
1547
- for (const [c, o] of runtimeValues) h = xxh3(`${c}\0${o}`, h)
1548
- if (cap)
1549
- for (const [c, o] of runtimeValues) cap.push({ kind: 'runtime', name: c, hash: xxh3hex(o) })
1550
-
1551
- const wsRuntimeValues = input.workspaceRuntimeValues ?? []
1552
- h = xxh3(`ws-runtime-values:${wsRuntimeValues.length}`, h)
1553
- for (const [c, o] of wsRuntimeValues) h = xxh3(`${c}\0${o}`, h)
1554
- if (cap)
1555
- for (const [c, o] of wsRuntimeValues)
1556
- cap.push({ kind: 'ws-runtime', name: c, hash: xxh3hex(o) })
1557
-
1558
- const upstream = [...input.upstreamHashes].sort()
1559
- h = xxh3(`upstream:${upstream.length}`, h)
1560
- for (const u of upstream) h = xxh3(u, h)
1561
- if (cap) {
1562
- for (const u of upstream) {
1563
- cap.push({ kind: 'upstream', name: input.upstreamIds?.get(u) ?? u, hash: u })
1564
- }
1565
- }
1566
-
1567
- const pluginParts = input.pluginParts ?? []
1568
- if (pluginParts.length > 0) {
1569
- h = xxh3(`plugin:${pluginParts.length}`, h)
1570
- for (const [n, v] of pluginParts) h = xxh3(`${n}\0${v}`, h)
1571
- if (cap) {
1572
- for (const [n, v] of pluginParts) cap.push({ kind: 'plugin', name: n, hash: xxh3hex(v) })
1573
- }
1574
- }
1575
-
1576
- const sortedInputs = [...input.inputFiles].sort()
1577
- h = xxh3(`inputs:${sortedInputs.length}`, h)
1578
- // Per-file hash source, in preference order: the caller-supplied
1579
- // index-OID map (clean tracked files — zero I/O), then hashFile's
1580
- // stat memo (no read), then a full in-process blob-OID computation.
1581
- // All three describe the WORKTREE bytes, which is what the task
1582
- // reads — the index-OID map is only populated for paths whose
1583
- // worktree form matches their index form, because a clean filter
1584
- // (`text`/`eol`/`ident`) makes the index blob a DIFFERENT sequence
1585
- // of bytes and folding it would let two distinct worktree contents
1586
- // share a key. The fold order is locked to `sortedInputs` so
1587
- // results are stable across runs.
1588
- const fileHashes = await Promise.all(
1589
- sortedInputs.map((f) => input.fileHashes?.get(f) ?? this.hashFile(f)),
852
+ key(input: CacheKeyInput): Promise<string> {
853
+ return foldKey(
854
+ input,
855
+ (f) => this.hashFile(f),
856
+ (f) => this.relFor(input.workspaceRoot, f),
1590
857
  )
1591
- for (let i = 0; i < sortedInputs.length; i++) {
1592
- const file = sortedInputs[i]!
1593
- const rel = relPosix(input.workspaceRoot, file)
1594
- const oid = fileHashes[i]!
1595
- h = xxh3(`${rel}\0${oid}`, h)
1596
- // The OID is already awaited — file capture is zero extra I/O.
1597
- if (cap) cap.push({ kind: 'file', name: rel, hash: oid })
1598
- }
1599
-
1600
- return h.toString(16).padStart(16, '0')
1601
858
  }
1602
859
 
1603
860
  // ctx is accepted but ignored — the local layer reads metadata from
@@ -1607,7 +864,7 @@ export class Cache implements CacheLayer {
1607
864
  // Local reads disabled (e.g. `--force` / `--cache=local:w`): report a
1608
865
  // miss so the task re-executes. The artifact + index are untouched.
1609
866
  if (!this.read) return null
1610
- return await this.readEntry(hash)
867
+ return this.readEntry(hash)
1611
868
  }
1612
869
 
1613
870
  /**
@@ -1619,8 +876,8 @@ export class Cache implements CacheLayer {
1619
876
  * `ingest` had just written, so the task re-executed and re-uploaded on
1620
877
  * every single run.
1621
878
  */
1622
- async getIngested(hash: string): Promise<CacheEntry | null> {
1623
- return await this.readEntry(hash)
879
+ getIngested(hash: string): Promise<CacheEntry | null> {
880
+ return this.readEntry(hash)
1624
881
  }
1625
882
 
1626
883
  private async readEntry(hash: string): Promise<CacheEntry | null> {
@@ -1629,7 +886,7 @@ export class Cache implements CacheLayer {
1629
886
 
1630
887
  // Verify the tar artifact actually exists. The DB and the
1631
888
  // filesystem can drift if someone manually deletes the cache dir.
1632
- if (!(await Bun.file(this.tarPath(hash)).exists())) return null
889
+ if (!existsSync(this.tarPath(hash))) return null
1633
890
 
1634
891
  // Deferred: per-hit UPDATEs cost ~60 ms across 2000+ probes on a
1635
892
  // full-cache run. Hashes are collected and flushed as ONE batched
@@ -1670,7 +927,7 @@ export class Cache implements CacheLayer {
1670
927
  )
1671
928
  }
1672
929
  if (rows.length === 0) return out
1673
- const present = await Promise.all(rows.map((r) => Bun.file(this.tarPath(r.hash)).exists()))
930
+ const present = rows.map((r) => existsSync(this.tarPath(r.hash)))
1674
931
  const live = rows.filter((_r, i) => present[i])
1675
932
  const liveHashes = live.map((r) => r.hash)
1676
933
  const fileRows = this.loadOutputFilesBatch(liveHashes)
@@ -1690,7 +947,7 @@ export class Cache implements CacheLayer {
1690
947
  if (!this.read) return null
1691
948
  const row = this.selectEntry.get(hash) as EntryRow | undefined
1692
949
  if (!row) return null
1693
- return (await Bun.file(this.tarPath(hash)).exists()) ? 'local' : null
950
+ return existsSync(this.tarPath(hash)) ? 'local' : null
1694
951
  }
1695
952
 
1696
953
  // Local cache has no slower layer to warm from — prefetch is a no-op.
@@ -1700,169 +957,37 @@ export class Cache implements CacheLayer {
1700
957
  return false
1701
958
  }
1702
959
 
960
+ // --- output fingerprints: delegated to `OutputIndex` (see output-index.ts) ---
961
+ // The slices' proofs and hashes are returned as the slice's own promise —
962
+ // no `async` wrapper, no `return await`: each of those added a promise and
963
+ // a microtask hop per call, and a 1,000-hit warm run pays the proofs
964
+ // 2,000 times (measured 2026-09-10: +3 ms on `classify + probe`).
1703
965
  loadOutputFilesBatch(hashes: readonly string[]): Map<string, OutputFileRow[]> {
1704
- const out = new Map<string, OutputFileRow[]>()
1705
- if (hashes.length === 0) return out
1706
- // Inline placeholders for an IN-list — bun:sqlite doesn't ship
1707
- // rarray, but `IN (?, ?, …)` with N≤~999 is fast and avoids per-
1708
- // hash select.get() overhead. `db.query` (not `db.prepare`) caches the
1709
- // compiled statement keyed by the SQL text — so the dominant single-hash
1710
- // warm-hit path (called up to 3× per hit) reuses one statement instead of
1711
- // recompiling on every call.
1712
- const placeholders = hashes.map(() => '?').join(',')
1713
- const stmt = this.db.query(
1714
- `SELECT entry_hash, path, size_bytes, mode, mtime_ms FROM output_files WHERE entry_hash IN (${placeholders})`,
1715
- )
1716
- const rows = stmt.all(...(hashes as readonly SQLQueryBindings[])) as Array<{
1717
- entry_hash: string
1718
- path: string
1719
- size_bytes: number
1720
- mode: number
1721
- mtime_ms: number
1722
- }>
1723
- for (const r of rows) {
1724
- let list = out.get(r.entry_hash)
1725
- if (!list) {
1726
- list = []
1727
- out.set(r.entry_hash, list)
1728
- }
1729
- list.push({ path: r.path, size: r.size_bytes, mode: r.mode, mtimeMs: r.mtime_ms })
1730
- }
1731
- return out
966
+ return this.outputs.loadOutputFilesBatch(hashes)
1732
967
  }
1733
-
1734
- async isOutputsCurrent(projectDir: string, expected: readonly OutputFileRow[]): Promise<boolean> {
1735
- // Empty manifest case (a task produced no outputs) → trivially
1736
- // current; the on-disk tree under projectDir is whatever it was,
1737
- // and nothing was supposed to land there.
1738
- if (expected.length === 0) return true
1739
- // Promise-form stat, deliberately: statSync is ~2 µs against ~13 µs in
1740
- // isolation, but this runs under the scheduler's concurrency and the
1741
- // async form keeps the stats on the thread pool in parallel — the sync
1742
- // version measured slower on a 1000-hit warm run (2026-09-02).
1743
- const results = await Promise.all(
1744
- expected.map(async (e) => {
1745
- try {
1746
- const s = await stat(path.join(projectDir, e.path))
1747
- return (
1748
- s.size === e.size &&
1749
- (s.mode & 0o777) === (e.mode & 0o777) &&
1750
- // MILLISECOND comparison (sub-ms tolerance for the float
1751
- // round-trip through utimes). Save rows carry stat-ms and
1752
- // restoreOutputs re-syncs restored files to the row value,
1753
- // so equality holds exactly in steady state; legacy
1754
- // second-precision rows converge on their first restore.
1755
- // Residual blind spot: a same-size edit landing in the SAME
1756
- // millisecond as the recorded write, or a deliberately
1757
- // forged mtime (touch -r) — the trade every mtime-based
1758
- // skip check accepts.
1759
- Math.abs(s.mtimeMs - e.mtimeMs) < 1
1760
- )
1761
- } catch {
1762
- return false
1763
- }
1764
- }),
1765
- )
1766
- return results.every(Boolean)
968
+ isOutputsCurrent(projectDir: string, expected: readonly OutputFileRow[]): Promise<boolean> {
969
+ return this.outputs.isOutputsCurrent(projectDir, expected)
1767
970
  }
1768
-
1769
971
  outputsPath(hash: string): string {
1770
972
  return this.tarPath(hash)
1771
973
  }
1772
974
 
1773
- /**
1774
- * Snapshot every directory under each of `prefixes` for `hash`. Called
1775
- * after a save and after a restore, when the tree is known to equal the
1776
- * entry's set. Symlinked directories are not descended (the output walk
1777
- * refuses them too). Over `OUTPUT_DIRS_CAP` directories, or on any
1778
- * error, the rows are cleared and the next hit keeps the walk.
1779
- */
1780
975
  async recordOutputDirs(
1781
976
  hash: string,
1782
977
  projectDir: string,
1783
978
  prefixes: readonly string[],
1784
979
  ): Promise<void> {
1785
- const rows: Array<[string, number]> = []
1786
- const walk = async (rel: string): Promise<boolean> => {
1787
- const abs = path.join(projectDir, rel)
1788
- let st
1789
- try {
1790
- st = await lstat(abs)
1791
- } catch {
1792
- return false
1793
- }
1794
- if (!st.isDirectory()) return false
1795
- rows.push([rel, st.mtimeMs])
1796
- if (rows.length > OUTPUT_DIRS_CAP) return false
1797
- let entries
1798
- try {
1799
- entries = await readdir(abs, { withFileTypes: true })
1800
- } catch {
1801
- return false
1802
- }
1803
- for (const e of entries) {
1804
- if (e.isDirectory() && !e.isSymbolicLink()) {
1805
- if (!(await walk(`${rel}/${e.name}`))) return false
1806
- }
1807
- }
1808
- return true
1809
- }
1810
- let ok = true
1811
- for (const prefix of prefixes) {
1812
- if (!(await walk(prefix))) {
1813
- ok = false
1814
- break
1815
- }
1816
- }
1817
- // All or nothing: a racy directory dropped alone would leave its
1818
- // parent trusted while an addition inside it bumps only the dropped one.
1819
- const youngest = Date.now() - OUTPUT_DIRS_RACY_MS
1820
- if (rows.some(([, mtime]) => mtime > youngest)) ok = false
1821
- this.db.transaction(() => {
1822
- this.deleteOutputDirs.run(hash)
1823
- if (!ok) return
1824
- for (const [rel, mtime] of rows) this.insertOutputDir.run(hash, rel, mtime)
1825
- })()
980
+ await this.outputs.recordOutputDirs(hash, projectDir, prefixes)
981
+ }
982
+ recordOutputStamps(hash: string, projectDir: string, workspaceRoot: string): void {
983
+ this.outputs.recordOutputStamps(hash, projectDir, workspaceRoot)
1826
984
  }
1827
-
1828
985
  loadOutputDirsBatch(hashes: readonly string[]): Map<string, OutputDirRow[]> {
1829
- const out = new Map<string, OutputDirRow[]>()
1830
- if (hashes.length === 0) return out
1831
- const placeholders = hashes.map(() => '?').join(',')
1832
- const rows = this.db
1833
- .query(
1834
- `SELECT entry_hash, path, mtime_ms FROM output_dirs WHERE entry_hash IN (${placeholders})`,
1835
- )
1836
- .all(...(hashes as readonly SQLQueryBindings[])) as Array<{
1837
- entry_hash: string
1838
- path: string
1839
- mtime_ms: number
1840
- }>
1841
- for (const r of rows) {
1842
- const list = out.get(r.entry_hash)
1843
- const row = { path: r.path, mtimeMs: r.mtime_ms }
1844
- if (list) list.push(row)
1845
- else out.set(r.entry_hash, [row])
1846
- }
1847
- return out
986
+ return this.outputs.loadOutputDirsBatch(hashes)
1848
987
  }
1849
-
1850
- /** True iff every recorded directory exists with its recorded mtime (ms). Same forged-mtime trade as the file check. */
1851
- async outputDirsCurrent(projectDir: string, rows: readonly OutputDirRow[]): Promise<boolean> {
1852
- if (rows.length === 0) return false
1853
- const results = await Promise.all(
1854
- rows.map(async (r) => {
1855
- try {
1856
- const st = await stat(path.join(projectDir, r.path))
1857
- return st.isDirectory() && Math.abs(st.mtimeMs - r.mtimeMs) < 1
1858
- } catch {
1859
- return false
1860
- }
1861
- }),
1862
- )
1863
- return results.every(Boolean)
988
+ outputDirsCurrent(projectDir: string, rows: readonly OutputDirRow[]): Promise<boolean> {
989
+ return this.outputs.outputDirsCurrent(projectDir, rows)
1864
990
  }
1865
-
1866
991
  async restoreOutputs(hash: string, projectDir: string, workspaceRoot?: string): Promise<void> {
1867
992
  // In-process extraction — no fork+exec on the hot path
1868
993
  // (~5-10ms reclaimed per hit vs the prior subprocess `tar -xf`).
@@ -1877,13 +1002,23 @@ export class Cache implements CacheLayer {
1877
1002
  // replay through the logger.
1878
1003
  const src = this.tarPath(hash)
1879
1004
  // The caller already committed to this hit and WIPED the declared
1880
- // outputs, so returning quietly here would report a green cache hit over
1881
- // an emptied output tree. The artifact existed when `get()` probed it, so
1882
- // its absence now means something removed it underneath us (a concurrent
1883
- // `vx cache prune` is the documented way) — fail loud; the task re-runs.
1884
- if (!(await Bun.file(src).exists())) {
1885
- throw new CorruptArtifactError(hash, 'artifact file vanished before restore')
1886
- }
1005
+ // outputs, so a restore that did not fail here would report a green cache
1006
+ // hit over an emptied output tree. The artifact existed when `get()`
1007
+ // probed it, so its absence now means something removed it underneath us
1008
+ // (a `vx cache prune` in another shell, or another workspace's retention
1009
+ // on a shared `--cache-dir`) — `ArtifactVanishedError`, which the caller
1010
+ // turns into a miss: the task runs.
1011
+ //
1012
+ // The FAILING is held twice: the decode below reaches the missing file
1013
+ // and throws `CorruptArtifactError` from the extract catch either way.
1014
+ // What the `ENOENT` mapping in that catch carries alone is the CLASS,
1015
+ // and the two ask for opposite answers — a prune raced this run (run
1016
+ // the task) versus the cache holds bad bytes (fail, and say so). The
1017
+ // roundtrip row asserts the class (item 481); a row that only asserts
1018
+ // "the restore fails" passes with the mapping deleted. It was a
1019
+ // separate `exists()` probe until item 627: one thread-pool round trip
1020
+ // per restore, spent to learn what the read reports itself.
1021
+ const endRows = span('restore: rows')
1887
1022
  // The index says exactly which files this entry materializes. If the
1888
1023
  // archive cannot produce one of them, restoring "successfully" leaves a
1889
1024
  // hole that no later run detects: the skip-restore check compares the
@@ -1894,6 +1029,7 @@ export class Cache implements CacheLayer {
1894
1029
  const expected = rows
1895
1030
  .filter((r) => workspaceRoot !== undefined || !r.path.startsWith(WORKSPACE_OUTPUT_PREFIX))
1896
1031
  .map((r) => (r.path.startsWith(WORKSPACE_OUTPUT_PREFIX) ? r.path : `outputs/${r.path}`))
1032
+ endRows()
1897
1033
  const verify = (provided: ReadonlySet<string>): void => {
1898
1034
  const missing = expected.filter((name) => !provided.has(name))
1899
1035
  if (missing.length > 0) {
@@ -1913,15 +1049,98 @@ export class Cache implements CacheLayer {
1913
1049
  // when every artifact is a one-file `dist/` (measured: 390 vs 355 ms
1914
1050
  // per 1 000). The local artifact was validated at ingest, so a missing
1915
1051
  // declared size is allowed; the output ceiling applies to both.
1052
+ const endExtract = span('restore: extract')
1916
1053
  try {
1917
- const tar = await decodedTar(Bun.file(src), hash)
1054
+ // Every step here is an async filesystem call on purpose: a
1055
+ // synchronous restore of a small artifact is 2× faster ALONE (1.25 →
1056
+ // 0.62 ms sequential, 2026-09-10) and 30% slower in the run, where
1057
+ // four workers overlap their round trips and a blocking one stalls
1058
+ // the other three (1,000-project restore row 1.1–1.2 s → 1.5 s).
1059
+ const tar = await decodedTar(Bun.file(src), hash, this.artifactCeiling)
1918
1060
  await extractArtifactStream(tar, projectDir, workspaceRoot, verify)
1919
1061
  } catch (err) {
1920
- if (err instanceof ArchiveSecurityError || err instanceof CorruptArtifactError) throw err
1062
+ // A UserError here is the extractor naming the tree's fault (an
1063
+ // output directory that links out of the project).
1064
+ if (
1065
+ err instanceof ArchiveSecurityError ||
1066
+ err instanceof CorruptArtifactError ||
1067
+ err instanceof UserError
1068
+ ) {
1069
+ throw err
1070
+ }
1071
+ // What is on disk, not what is in the archive: a directory standing
1072
+ // where the entry holds a file, or a file where it needs a directory.
1073
+ // The clean removes everything the output globs cover, so this is a
1074
+ // stray they do not — name it as such, not as a corrupt artifact. A
1075
+ // link on the way never lands here: the extractor writes through one
1076
+ // that stays in the project, dangling or not, and names one that
1077
+ // leaves it or loops (item 748).
1078
+ const code = (err as NodeJS.ErrnoException).code
1079
+ if (code === 'EISDIR' || code === 'ENOTDIR' || code === 'EEXIST' || code === 'ENOTEMPTY') {
1080
+ throw new UserError(
1081
+ `restore of ${hash} into ${projectDir} was blocked by what is on disk (${code}: ${(err as Error).message}). ` +
1082
+ `Declared outputs are wiped before a restore, so this is a path the output globs do not cover — remove it and re-run.`,
1083
+ )
1084
+ }
1085
+ // A legal name under a destination deep enough that the two together
1086
+ // pass PATH_MAX: the artifact is fine, the workspace's location is
1087
+ // not (the parity audit's last open archive row, item 670).
1088
+ if (code === 'ENAMETOOLONG') {
1089
+ throw new UserError(
1090
+ `restore of ${hash} into ${projectDir} could not write its outputs (${code}: ${(err as Error).message}). ` +
1091
+ `An output path under this directory is longer than the file system allows — move the workspace to a shorter path.`,
1092
+ )
1093
+ }
1094
+ // Same distinction for a tree the process cannot write into — a
1095
+ // `dist/` another user owns, a read-only checkout, a full disk: the
1096
+ // artifact is intact, the tree is not the process's to change.
1097
+ if (isFsRefusal(err)) {
1098
+ const remedy = isDiskFull(err)
1099
+ ? 'Free space on that disk and re-run.'
1100
+ : 'Make the declared output paths writable by this user, or stop declaring them as outputs.'
1101
+ throw new UserError(
1102
+ `restore of ${hash} into ${projectDir} could not write its outputs (${code}: ${err.message}). ${remedy}`,
1103
+ )
1104
+ }
1105
+ // The artifact itself is gone: the read names its path (`bytes()`
1106
+ // opens it; a staged file's ENOENT names the temp below).
1107
+ if (code === 'ENOENT' && (err as NodeJS.ErrnoException).path === src) {
1108
+ throw new ArtifactVanishedError(hash)
1109
+ }
1110
+ // A staged file the restore itself just wrote is gone before its
1111
+ // commit: another process cleaned the same outputs under us (two runs
1112
+ // on one workspace, a `vx watch` beside a `vx run` — a clean landing
1113
+ // 4–32 ms into a 2,000-file restore reproduces it, 2026-09-16). The
1114
+ // artifact is intact; the tree was not ours alone.
1115
+ if (code === 'ENOENT' && (err as Error).message.includes('.vx-tmp-')) {
1116
+ throw new UserError(
1117
+ `restore of ${hash} into ${projectDir} was interrupted: a file it had just written vanished (${(err as Error).message}). ` +
1118
+ `Another vx run is using this workspace — re-run once it is done.`,
1119
+ )
1120
+ }
1921
1121
  throw new CorruptArtifactError(hash, 'artifact is not a readable archive', err)
1122
+ } finally {
1123
+ endExtract()
1922
1124
  }
1923
1125
  }
1924
1126
 
1127
+ /**
1128
+ * A run writes here on every path — the run record at its end, an
1129
+ * entry's `accessed_at` on a hit, the artifact on a miss — so a cache
1130
+ * directory this user cannot write into fails the run before the graph
1131
+ * starts, once, with the directory named, rather than every task at
1132
+ * 0 ms (or the run at its very end) with SQLite's "attempt to write a
1133
+ * readonly database" as an internal error. The readers open the same
1134
+ * directory read-only and go on; a run refuses.
1135
+ */
1136
+ assertWritable(): void {
1137
+ if (this.writeBlocked === null) return
1138
+ throw new UserError(
1139
+ `cache directory ${this.cacheDir} is not writable (${this.writeBlocked}) — every run records its ` +
1140
+ `history there; make it writable by this user, or pass --cache-dir <path>`,
1141
+ )
1142
+ }
1143
+
1925
1144
  async save(args: {
1926
1145
  hash: string
1927
1146
  /** See `CacheLayer.save` — `exitCode` is not a caller's to supply. */
@@ -1953,7 +1172,10 @@ export class Cache implements CacheLayer {
1953
1172
  // the bytes, since there's no local artifact to read off disk.
1954
1173
  if (!this.write) return
1955
1174
  if (args.skipLocalWrite === true) return
1956
- await mkdir(this.cacheDir, { recursive: true })
1175
+ // The cache directory is the constructor's (`mkdirSync`, with a
1176
+ // refusal recorded for `assertWritable`); re-creating it here and
1177
+ // again in `writeArtifactAndIndex` cost every save two thread-pool
1178
+ // round trips for an EEXIST and a stat (item 630).
1957
1179
  const endPack = span('save: pack')
1958
1180
  const compressed = await this.packArtifactToTemp(this.tempPath(args.hash), args)
1959
1181
  endPack()
@@ -1979,24 +1201,30 @@ export class Cache implements CacheLayer {
1979
1201
  * index. Used by the LayeredCache when local writes are disabled but
1980
1202
  * a remote upload still needs the artifact bytes.
1981
1203
  */
1982
- async packArtifactBytes(args: SaveArgs): Promise<Uint8Array> {
1983
- return await this.packArtifact(args)
1984
- }
1985
-
1986
- async ingest(hash: string, compressed: Uint8Array, meta: IngestMeta): Promise<void> {
1987
- await this.writeArtifactAndIndex(hash, compressed, meta)
1204
+ packArtifactBytes(args: SaveArgs): Promise<Uint8Array> {
1205
+ return this.packArtifact(args)
1988
1206
  }
1989
1207
 
1990
1208
  /**
1991
- * Collect stdout + outputs into artifact bytes, zstd-compress, return
1992
- * them. No disk write to the final cache path — that's the index
1993
- * step's job. Pure transform, so `ingest()` can skip this and hand
1994
- * its remote-supplied bytes straight to `writeArtifactAndIndex`.
1995
- *
1996
- * Entries are named directly into the archive, so there is no staging
1997
- * copy of every output byte and no `tar` subprocess (see
1998
- * `archive.ts`).
1209
+ * The remote body goes to the temp by `Bun.write`, which streams a
1210
+ * `Response` and copies a file `Blob` without collecting either, so a
1211
+ * pull never holds the artifact. A body that fails mid-stream (a dropped
1212
+ * socket) leaves a partial temp behind it, removed here; validation
1213
+ * removes its own.
1999
1214
  */
1215
+ async ingest(hash: string, body: Blob | Response, meta: IngestMeta): Promise<void> {
1216
+ const tmpPath = this.tempPath(hash)
1217
+ try {
1218
+ // Split only for the typings: Bun.write's Response and Blob overloads
1219
+ // do not accept their union.
1220
+ await (body instanceof Response ? Bun.write(tmpPath, body) : Bun.write(tmpPath, body))
1221
+ } catch (err) {
1222
+ await unlink(tmpPath).catch(() => undefined)
1223
+ throw err
1224
+ }
1225
+ await this.writeArtifactAndIndex(hash, { tmpPath }, meta)
1226
+ }
1227
+
2000
1228
  /** Archive name → absolute source path for every declared output. */
2001
1229
  private outputsOf(args: {
2002
1230
  projectDir: string
@@ -2019,9 +1247,17 @@ export class Cache implements CacheLayer {
2019
1247
  /**
2020
1248
  * The compressed artifact, in memory: the remote upload when local
2021
1249
  * writes are off. A large artifact is packed and compressed as a
2022
- * stream and collected — the seam takes bytes.
1250
+ * stream and collected — with no local artifact the bytes must be
1251
+ * captured while the outputs are still on disk.
1252
+ *
1253
+ * Both packers name entries directly into the archive — no staging
1254
+ * copy of the outputs, no `tar` subprocess (`archive.ts`) — and
1255
+ * neither writes the final cache path, which is the index step's job:
1256
+ * `ingest()` packs nothing and hands its remote-supplied temp straight
1257
+ * to `writeArtifactAndIndex`.
2023
1258
  */
2024
1259
  private async packArtifact(args: {
1260
+ hash: string
2025
1261
  entry: Omit<CacheEntry, 'hash' | 'storedAt' | 'outputFiles' | 'exitCode'>
2026
1262
  projectDir: string
2027
1263
  outputFiles: string[]
@@ -2031,13 +1267,33 @@ export class Cache implements CacheLayer {
2031
1267
  // stdout is ALWAYS present in the artifact, even if empty, so the
2032
1268
  // layout is predictable: a successful read finds `stdout` and
2033
1269
  // zero-or-more `outputs/<rel>` / `workspace-outputs/<rel>` entries.
1270
+ const plan = await this.planWithin(args)
1271
+ if (plan.size <= STREAM_DECODE_FROM)
1272
+ return await Bun.zstdCompress(await packArtifactBytes(plan))
1273
+ return bytesOf(packArtifactStream(plan).pipeThrough(zstdEncoder()))
1274
+ }
1275
+
1276
+ /**
1277
+ * The artifact's plan, refused when its tar — exactly the bytes a restore
1278
+ * decodes — is past the ceiling every restore enforces. Refused here, it
1279
+ * costs a stat per output; left to the scan, it cost the whole compress
1280
+ * and a decode of a 2.2 GB output (6 to 14 s) to learn the same thing
1281
+ * and call the task's own outputs a "corrupt artifact".
1282
+ */
1283
+ private async planWithin(args: Parameters<Cache['packArtifact']>[0]): Promise<ArtifactPlan> {
2034
1284
  const plan = await planArtifact({
1285
+ key: args.hash,
2035
1286
  stdout: args.entry.stdout ?? '',
2036
1287
  outputs: this.outputsOf(args),
1288
+ exec: usageOfEntry(args.entry),
2037
1289
  })
2038
- if (plan.size <= STREAM_DECODE_FROM)
2039
- return await Bun.zstdCompress(await packArtifactBytes(plan))
2040
- return await bytesOf(packArtifactStream(plan).pipeThrough(zstdEncoder()))
1290
+ if (plan.size > this.artifactCeiling) {
1291
+ throw new Error(
1292
+ `${args.entry.taskId} is not cached: its outputs pack to ${formatBytes(plan.size)}, past the ` +
1293
+ `${formatBytes(this.artifactCeiling)} artifact ceiling a restore enforces — narrow cache.outputs.files`,
1294
+ )
1295
+ }
1296
+ return plan
2041
1297
  }
2042
1298
 
2043
1299
  /**
@@ -2053,10 +1309,7 @@ export class Cache implements CacheLayer {
2053
1309
  tmpPath: string,
2054
1310
  args: Parameters<Cache['packArtifact']>[0],
2055
1311
  ): Promise<Uint8Array | { tmpPath: string }> {
2056
- const plan = await planArtifact({
2057
- stdout: args.entry.stdout ?? '',
2058
- outputs: this.outputsOf(args),
2059
- })
1312
+ const plan = await this.planWithin(args)
2060
1313
  if (plan.size <= STREAM_DECODE_FROM)
2061
1314
  return await Bun.zstdCompress(await packArtifactBytes(plan))
2062
1315
  const sink = Bun.file(tmpPath).writer()
@@ -2075,13 +1328,6 @@ export class Cache implements CacheLayer {
2075
1328
  return { tmpPath }
2076
1329
  }
2077
1330
 
2078
- /**
2079
- * Atomically write `compressed` to `<hash>.tar.zst` and (re)build the
2080
- * entries + output_files SQL rows from the archive itself. Shared by
2081
- * `save()` (we just packed the bytes) and `ingest()` (we got them from
2082
- * the remote layer) — both index the identical values, because both
2083
- * read them out of the artifact.
2084
- */
2085
1331
  /** tmp suffix mixes pid + hrtime + a random hex chunk so two saves of
2086
1332
  * the same hash from the same process (or from two forked workers that
2087
1333
  * happen to share a wall-clock ms) don't pick the same tmp filename and
@@ -2090,13 +1336,20 @@ export class Cache implements CacheLayer {
2090
1336
  return `${this.tarPath(hash)}.tmp-${process.pid}-${process.hrtime.bigint()}-${Math.random().toString(36).slice(2, 10)}`
2091
1337
  }
2092
1338
 
1339
+ /**
1340
+ * Atomically write `compressed` to `<hash>.tar.zst` and (re)build the
1341
+ * entries + output_files SQL rows from the archive itself. Shared by
1342
+ * `save()` (we just packed the bytes) and `ingest()` (it streamed them
1343
+ * from the remote layer into a temp) — both index the identical values, because both
1344
+ * read them out of the artifact.
1345
+ */
2093
1346
  private async writeArtifactAndIndex(
2094
1347
  hash: string,
2095
1348
  compressed: Uint8Array | { tmpPath: string },
2096
1349
  meta: IngestMeta,
2097
1350
  ): Promise<void> {
2098
1351
  // Validate BEFORE anything touches the final path. `ingest()` feeds
2099
- // us network bytes; a truncated/garbage body that went live first
1352
+ // us a temp of network bytes; a truncated/garbage body that went live first
2100
1353
  // would leave a corrupt `<hash>.tar.zst` behind (with no SQL row,
2101
1354
  // since the decompress throw aborted indexing) for every later
2102
1355
  // reader to trip over. Decompress + parse also produce the
@@ -2107,7 +1360,6 @@ export class Cache implements CacheLayer {
2107
1360
  let tmpPath: string
2108
1361
  if (compressed instanceof Uint8Array) {
2109
1362
  tmpPath = this.tempPath(hash)
2110
- await mkdir(this.cacheDir, { recursive: true })
2111
1363
  // The temp is written BEFORE validation so a large artifact can be
2112
1364
  // scanned from the file as it decodes — a file stream reads in
2113
1365
  // bounded pieces; the bytes in memory would not (see `decodedTar`).
@@ -2118,12 +1370,12 @@ export class Cache implements CacheLayer {
2118
1370
  await writeFile(tmpPath, compressed)
2119
1371
  endWrite()
2120
1372
  } else {
2121
- // `save` already streamed the artifact into its temp.
1373
+ // `save` or `ingest` already streamed the artifact into its temp.
2122
1374
  tmpPath = compressed.tmpPath
2123
1375
  }
2124
1376
  let scanned: Awaited<ReturnType<typeof scanArtifact>>
2125
1377
  try {
2126
- // ingest() is the UNTRUSTED boundary — `compressed` is bytes just
1378
+ // ingest() is the UNTRUSTED boundary — its temp holds bytes just
2127
1379
  // pulled from a remote. Refuse a bomb (declared or sizeless) before it
2128
1380
  // can expand into memory.
2129
1381
  const source =
@@ -2131,13 +1383,22 @@ export class Cache implements CacheLayer {
2131
1383
  ? compressed
2132
1384
  : Bun.file(tmpPath)
2133
1385
  const endScan = span('save: scan')
2134
- scanned = await scanArtifact(await decodedTar(source, hash))
1386
+ scanned = await scanArtifact(await decodedTar(source, hash, this.artifactCeiling))
2135
1387
  endScan()
2136
1388
  // v17 invariant: every artifact carries a `stdout` entry. Its
2137
1389
  // absence means the bytes decompressed but aren't a vx artifact.
2138
1390
  if (scanned.stdout === null) {
2139
1391
  throw new CorruptArtifactError(hash, 'missing stdout entry')
2140
1392
  }
1393
+ if (scanned.key !== hash) {
1394
+ throw new CorruptArtifactError(
1395
+ hash,
1396
+ scanned.key === undefined
1397
+ ? 'artifact records no cache key'
1398
+ : `artifact was stored under another key (${scanned.key})`,
1399
+ )
1400
+ }
1401
+ assertArtifactNames(hash, scanned.entries, meta.outputs)
2141
1402
  } catch (err) {
2142
1403
  await unlink(tmpPath).catch(() => undefined)
2143
1404
  if (err instanceof ArchiveSecurityError || err instanceof CorruptArtifactError) throw err
@@ -2191,8 +1452,7 @@ export class Cache implements CacheLayer {
2191
1452
  // (not the per-run path) so a warm all-cache-hit run — which never
2192
1453
  // saves — writes none of them.
2193
1454
  const insertEntry = this.insertEntry
2194
- const insertOutputFile = this.insertOutputFile
2195
- const deleteOutputFiles = this.deleteOutputFiles
1455
+ const outputs = this.outputs
2196
1456
  const insertEntryInput = this.insertEntryInput
2197
1457
  const inputComponents = meta.inputComponents
2198
1458
  const tx = this.db.transaction(() => {
@@ -2212,13 +1472,12 @@ export class Cache implements CacheLayer {
2212
1472
  stdoutText,
2213
1473
  now,
2214
1474
  now,
1475
+ // From the artifact, on both paths: a save indexes what it just
1476
+ // packed, an ingest what the producing machine packed.
1477
+ scanned.exec?.cpuMs ?? null,
1478
+ scanned.exec?.peakRssBytes ?? null,
2215
1479
  )
2216
- // Replace the entry's existing output_files rows (an UPDATE on
2217
- // the same hash should refresh, not append).
2218
- deleteOutputFiles.run(hash)
2219
- for (const [rel, size, mode, mtime] of outputFileRows) {
2220
- insertOutputFile.run(hash, rel, size, mode, mtime)
2221
- }
1480
+ outputs.replaceFileRows(hash, outputFileRows)
2222
1481
  // INSERT OR IGNORE: identical inputs derive this same hash, so a
2223
1482
  // re-save's rows are identical — keep the first set, skip the rest.
2224
1483
  if (inputComponents !== undefined) {
@@ -2260,55 +1519,19 @@ export class Cache implements CacheLayer {
2260
1519
  return this.db
2261
1520
  }
2262
1521
 
2263
- /**
2264
- * Content-addressed storage view over the same artifacts directory.
2265
- * Returns an `FsCASBackend` pointing at `cacheDir`, so external
2266
- * subsystems (R2 mirror, REAPI CAS bridge, analytics scanners) can
2267
- * read raw bytes with a `Digest`-keyed API without coupling to
2268
- * Cache's internal save path. Read/write semantics match what
2269
- * Cache.save writes (`<cacheDir>/<hash>.tar.zst`).
2270
- */
2271
- contentBackend(): FsCASBackend {
2272
- return new FsCASBackend(this.cacheDir)
2273
- }
2274
-
1522
+ // --- run history: delegated to `RunHistory` (see run-history.ts) ---
2275
1523
  recordRun(run: RunRecord): void {
2276
- this.insertRun.run(...bindRun(run))
1524
+ this.history.recordRun(run)
2277
1525
  }
2278
-
2279
1526
  recordRuns(runs: readonly RunRecord[]): void {
2280
- if (runs.length === 0) return
2281
- if (runs.length === 1) {
2282
- this.insertRun.run(...bindRun(runs[0]!))
2283
- return
2284
- }
2285
- // `bun:sqlite`'s `transaction()` returns a callable that wraps the
2286
- // body in BEGIN/COMMIT, fsyncing once at the end. For a 200-task
2287
- // run that's one fsync instead of 200.
2288
- const insert = this.insertRun
2289
- const tx = this.db.transaction((batch: readonly RunRecord[]) => {
2290
- for (const r of batch) insert.run(...bindRun(r))
2291
- })
2292
- tx(runs)
1527
+ this.history.recordRuns(runs)
2293
1528
  }
2294
-
2295
1529
  recordRunBundle(bundle: { runs: readonly RunRecord[]; invocation: InvocationRecord }): void {
2296
- // Whole run records atomically — one transaction, one fsync: the
2297
- // per-task `runs` rows and the one `invocations` header. The
2298
- // input-fingerprint rows do NOT live here: they're written inside
2299
- // the entry-save transaction (`save`/`ingest`) so a warm
2300
- // all-cache-hit run — which writes no `runs`-vs-`entry_inputs`
2301
- // mismatch — pays nothing for the moat it isn't refreshing.
2302
- const insertRun = this.insertRun
2303
- const insertInvocation = this.insertInvocation
2304
- this.db.transaction(() => {
2305
- for (const r of bundle.runs) insertRun.run(...bindRun(r))
2306
- insertInvocation.run(...bindInvocation(bundle.invocation))
2307
- })()
1530
+ this.history.recordRunBundle(bundle)
2308
1531
  }
2309
-
2310
1532
  stats(opts: CacheStatsOptions = {}): CacheStats {
2311
1533
  this.flushAccessed()
1534
+ this.outputs.flushOutputDirs()
2312
1535
  const project = opts.project
2313
1536
  const scoped = project !== undefined
2314
1537
  const scopeParams: string[] = project === undefined ? [] : [project]
@@ -2335,21 +1558,76 @@ export class Cache implements CacheLayer {
2335
1558
  }
2336
1559
  }
2337
1560
 
1561
+ /**
1562
+ * The workspace's `cacheRetention`, applied at the end of a run: `prune()`
1563
+ * with the same policy, but only when it would evict something, else the
1564
+ * orphan sweep alone when an hour has passed since the last one. A run
1565
+ * with nothing due pays the accessed-at flush it owed at close anyway, one
1566
+ * scan of the index and one read of the sweep's clock — never the
1567
+ * sweep's readdir. Null when nothing was due or this handle does not write.
1568
+ */
1569
+ async evictIfDue(
1570
+ policy: { maxAgeMs?: number; maxBytes?: number },
1571
+ now: number = Date.now(),
1572
+ ): Promise<PruneResult | null> {
1573
+ if (!this.write) return null
1574
+ // First: an entry this run restored still carries its old `accessed_at`
1575
+ // until the deferred bump lands, and would read as due for eviction.
1576
+ this.flushAccessed()
1577
+ const olderThanMs = policy.maxAgeMs === undefined ? undefined : now - policy.maxAgeMs
1578
+ const { oldest, bytes } = this.db
1579
+ .prepare(
1580
+ 'SELECT MIN(accessed_at) AS oldest, COALESCE(SUM(size_bytes), 0) AS bytes FROM entries',
1581
+ )
1582
+ .get() as { oldest: number | null; bytes: number }
1583
+ const ageDue = olderThanMs !== undefined && oldest !== null && oldest < olderThanMs
1584
+ const sizeDue = policy.maxBytes !== undefined && bytes > policy.maxBytes
1585
+ if (ageDue || sizeDue) {
1586
+ return this.prune({
1587
+ ...(olderThanMs !== undefined ? { olderThanMs } : {}),
1588
+ ...(policy.maxBytes !== undefined ? { maxBytes: policy.maxBytes } : {}),
1589
+ })
1590
+ }
1591
+ // Row-less artifacts are bytes the index cannot count, so the policy
1592
+ // above never sees them (upstream survey, nx#35483: 9 MiB of them sat
1593
+ // under a 1 MB limit). Listing the directory to find them costs 0.5 ms
1594
+ // per 1,000 entries (4.9 ms at 10,000), every run; the sweep's own
1595
+ // clock in `schema_meta` costs one indexed read. An orphan is reapable
1596
+ // only an hour after its last write anyway, so an hourly sweep reaps
1597
+ // it within two.
1598
+ const swept = this.db.prepare('SELECT value FROM schema_meta WHERE key = ?').get(SWEPT_AT) as {
1599
+ value: string
1600
+ } | null
1601
+ if (swept !== null && now - Number(swept.value) < ORPHAN_GRACE_MS) return null
1602
+ return { evicted: 0, bytesFreed: 0, ...(await this.reapOrphans(now)) }
1603
+ }
1604
+
2338
1605
  async prune(options: PruneOptions): Promise<PruneResult> {
2339
1606
  this.flushAccessed()
2340
- const { olderThanMs, maxBytes } = options
1607
+ // Before any entry is deleted, so a kept entry's pending snapshot is on
1608
+ // disk whatever the eviction does next. A pruned hash's snapshot cannot
1609
+ // orphan rows in either order: landed first, the cascade takes them;
1610
+ // landed after, the flush's own entry check drops them (item 628).
1611
+ this.outputs.flushOutputDirs()
1612
+ const { olderThanMs, maxBytes, dryRun = false } = options
2341
1613
  if (olderThanMs === undefined && maxBytes === undefined) {
2342
1614
  throw new Error('prune: pass at least one of `olderThanMs` or `maxBytes`')
2343
1615
  }
2344
1616
 
2345
1617
  const victims = new Set<string>()
2346
1618
  let bytesFreed = 0
1619
+ // Rows whose artifact is gone (deleted by hand, or by a disk cleaner):
1620
+ // never a hit, and their bytes are on no disk, but `--max-size` counted
1621
+ // them and evicted real entries to make room for them (item 975). They
1622
+ // are dropped, neither evicted nor freed.
1623
+ const phantoms = await this.phantomRows()
2347
1624
 
2348
1625
  if (olderThanMs !== undefined) {
2349
1626
  const rows = this.db
2350
1627
  .prepare('SELECT hash, size_bytes FROM entries WHERE accessed_at < ?')
2351
1628
  .all(olderThanMs) as Array<{ hash: string; size_bytes: number }>
2352
1629
  for (const r of rows) {
1630
+ if (phantoms.has(r.hash)) continue
2353
1631
  victims.add(r.hash)
2354
1632
  bytesFreed += r.size_bytes
2355
1633
  }
@@ -2359,7 +1637,9 @@ export class Cache implements CacheLayer {
2359
1637
  const totalRow = this.db
2360
1638
  .prepare('SELECT COALESCE(SUM(size_bytes), 0) AS bytes FROM entries')
2361
1639
  .get() as { bytes: number }
2362
- let remaining = totalRow.bytes - bytesFreed
1640
+ let phantomBytes = 0
1641
+ for (const b of phantoms.values()) phantomBytes += b
1642
+ let remaining = totalRow.bytes - phantomBytes - bytesFreed
2363
1643
  if (remaining > maxBytes) {
2364
1644
  // Exclude already-picked victims in JS, not via a SQL NOT-IN —
2365
1645
  // an IN-list over tens of thousands of TTL victims would blow
@@ -2368,7 +1648,7 @@ export class Cache implements CacheLayer {
2368
1648
  this.db
2369
1649
  .prepare('SELECT hash, size_bytes FROM entries ORDER BY accessed_at ASC')
2370
1650
  .all() as Array<{ hash: string; size_bytes: number }>
2371
- ).filter((row) => !victims.has(row.hash))
1651
+ ).filter((row) => !victims.has(row.hash) && !phantoms.has(row.hash))
2372
1652
  for (const row of candidates) {
2373
1653
  if (remaining <= maxBytes) break
2374
1654
  victims.add(row.hash)
@@ -2384,11 +1664,21 @@ export class Cache implements CacheLayer {
2384
1664
  // Promise.all over the unlinks. The IN-list is chunked at 900 like
2385
1665
  // flushAccessed so a huge eviction stays under any build's
2386
1666
  // bound-parameter ceiling.
2387
- if (victims.size > 0) {
1667
+ if (dryRun) {
1668
+ const orphans = await this.orphanStats()
1669
+ return {
1670
+ evicted: victims.size,
1671
+ bytesFreed,
1672
+ orphans: orphans.orphans,
1673
+ orphanBytes: orphans.orphanBytes,
1674
+ }
1675
+ }
1676
+ if (victims.size > 0 || phantoms.size > 0) {
2388
1677
  const hashes = [...victims]
1678
+ const rows = [...hashes, ...phantoms.keys()]
2389
1679
  this.db.transaction(() => {
2390
- for (let i = 0; i < hashes.length; i += 900) {
2391
- const chunk = hashes.slice(i, i + 900)
1680
+ for (let i = 0; i < rows.length; i += 900) {
1681
+ const chunk = rows.slice(i, i + 900)
2392
1682
  this.db
2393
1683
  .prepare(`DELETE FROM entries WHERE hash IN (${chunk.map(() => '?').join(',')})`)
2394
1684
  .run(...(chunk as readonly SQLQueryBindings[]))
@@ -2397,7 +1687,109 @@ export class Cache implements CacheLayer {
2397
1687
  await Promise.all(hashes.map((h) => rm(this.tarPath(h), { force: true })))
2398
1688
  }
2399
1689
 
2400
- return { evicted: victims.size, bytesFreed }
1690
+ const orphans = await this.reapOrphans()
1691
+ return { evicted: victims.size, bytesFreed, ...orphans }
1692
+ }
1693
+
1694
+ /**
1695
+ * Unlink artifacts the index does not know: a `<hash>.tar.zst` with no
1696
+ * `entries` row (a `SCHEMA_VERSION` drop, a `cache.db` deleted by hand)
1697
+ * and a `<hash>.tar.zst.tmp-*` a crashed save never renamed. Nothing
1698
+ * else reaps them — a lookup starts at the row, and a save of the same
1699
+ * key renames over the file, so a key that never recurs leaks its bytes
1700
+ * forever. Files younger than the grace window are left alone: a save
1701
+ * renames the artifact into place BEFORE its row commits, and its temp
1702
+ * exists while the bytes are still being written, so a fresh file
1703
+ * without a row is a save in flight, not an orphan.
1704
+ */
1705
+ private async reapOrphans(
1706
+ now: number = Date.now(),
1707
+ ): Promise<{ orphans: number; orphanBytes: number }> {
1708
+ this.db
1709
+ .prepare('INSERT OR REPLACE INTO schema_meta(key, value) VALUES (?, ?)')
1710
+ .run(SWEPT_AT, String(now))
1711
+ let orphans = 0
1712
+ let orphanBytes = 0
1713
+ await Promise.all(
1714
+ (await this.scanOrphans()).map(async (o) => {
1715
+ try {
1716
+ // unlink, not `rm({ force })`: force swallows ENOENT, and a file a
1717
+ // concurrent prune took first must not be counted as ours.
1718
+ await unlink(o.file)
1719
+ orphans += 1
1720
+ orphanBytes += o.size
1721
+ } catch {
1722
+ // Gone under us (a concurrent prune, a save's own cleanup): not ours.
1723
+ }
1724
+ }),
1725
+ )
1726
+ return { orphans, orphanBytes }
1727
+ }
1728
+
1729
+ /**
1730
+ * Index rows whose artifact is not in the directory, with their recorded
1731
+ * sizes. Only rows last written or used before the grace window: a save
1732
+ * renames its artifact in before its row commits, so a fresh row without
1733
+ * one is a listing that raced a save, not a phantom. An unreadable
1734
+ * directory judges nothing.
1735
+ */
1736
+ private async phantomRows(): Promise<Map<string, number>> {
1737
+ let names: string[]
1738
+ try {
1739
+ names = await readdir(this.cacheDir)
1740
+ } catch {
1741
+ return new Map()
1742
+ }
1743
+ const present = new Set(names)
1744
+ const rows = this.db
1745
+ .prepare('SELECT hash, size_bytes FROM entries WHERE accessed_at < ?')
1746
+ .all(Date.now() - ORPHAN_GRACE_MS) as Array<{ hash: string; size_bytes: number }>
1747
+ const out = new Map<string, number>()
1748
+ for (const r of rows) if (!present.has(`${r.hash}.tar.zst`)) out.set(r.hash, r.size_bytes)
1749
+ return out
1750
+ }
1751
+
1752
+ /** What `prune()` would reap right now, for `vx info` to say before anyone prunes. */
1753
+ async orphanStats(): Promise<{ orphans: number; orphanBytes: number }> {
1754
+ const found = await this.scanOrphans()
1755
+ let orphanBytes = 0
1756
+ for (const o of found) orphanBytes += o.size
1757
+ return { orphans: found.length, orphanBytes }
1758
+ }
1759
+
1760
+ /** Row-less artifacts and temps past the in-flight grace window: one readdir, one stat per candidate. */
1761
+ private async scanOrphans(): Promise<Array<{ file: string; size: number }>> {
1762
+ let names: string[]
1763
+ try {
1764
+ names = await readdir(this.cacheDir)
1765
+ } catch {
1766
+ return []
1767
+ }
1768
+ const indexed = new Set(
1769
+ (this.db.prepare('SELECT hash FROM entries').all() as Array<{ hash: string }>).map(
1770
+ (r) => r.hash,
1771
+ ),
1772
+ )
1773
+ const cutoff = Date.now() - ORPHAN_GRACE_MS
1774
+ const candidates: string[] = []
1775
+ for (const name of names) {
1776
+ const m = VX_ARTIFACT_NAME.exec(name)
1777
+ if (m === null) continue
1778
+ if (m[2] !== undefined || !indexed.has(m[1]!)) candidates.push(name)
1779
+ }
1780
+ const found: Array<{ file: string; size: number }> = []
1781
+ await Promise.all(
1782
+ candidates.map(async (name) => {
1783
+ const file = path.join(this.cacheDir, name)
1784
+ try {
1785
+ const st = await stat(file)
1786
+ if (st.isFile() && st.mtimeMs <= cutoff) found.push({ file, size: st.size })
1787
+ } catch {
1788
+ // Gone between readdir and stat: not an orphan any more.
1789
+ }
1790
+ }),
1791
+ )
1792
+ return found
2401
1793
  }
2402
1794
 
2403
1795
  close(): void {
@@ -2410,19 +1802,21 @@ export class Cache implements CacheLayer {
2410
1802
  // header would outlive its `runs` rows, so `vx info`/`vx last` would
2411
1803
  // list an invocation whose task detail is already gone, and the
2412
1804
  // table would grow unbounded on a long-lived checkout.
1805
+ // A reading verb and a dry run delete nothing: `vx last --list` over
1806
+ // month-old runs emptied the history it listed, and `vx cache prune
1807
+ // --dry-run` pruned it (item 1004).
2413
1808
  try {
2414
- const cutoff = Date.now() - 30 * 24 * 60 * 60 * 1000
2415
- this.db.prepare('DELETE FROM runs WHERE ended_at < ?').run(cutoff)
2416
- this.db.prepare('DELETE FROM invocations WHERE ended_at < ?').run(cutoff)
2417
- // A config that has not been loaded in 30 days was edited (its key
2418
- // moved) or its project left; either way the row is dead weight.
2419
- this.db.prepare('DELETE FROM config_evals WHERE created_at < ?').run(cutoff)
2420
- this.db.prepare('DELETE FROM config_closures WHERE created_at < ?').run(cutoff)
1809
+ if (!this.inspecting) {
1810
+ const cutoff = Date.now() - 30 * 24 * 60 * 60 * 1000
1811
+ this.history.pruneOlderThan(cutoff)
1812
+ this.configEvals.pruneOlderThan(cutoff)
1813
+ }
2421
1814
  } catch {
2422
1815
  // Retention is best-effort; never block closing the handle.
2423
1816
  }
2424
1817
  try {
2425
1818
  this.flushAccessed()
1819
+ this.outputs.flushOutputDirs()
2426
1820
  } catch {
2427
1821
  // Same contract as the retention prune above, which was already
2428
1822
  // guarded while this sibling was not: `accessed_at` is LRU
@@ -2442,70 +1836,44 @@ export class Cache implements CacheLayer {
2442
1836
  }
2443
1837
 
2444
1838
  /**
2445
- * Bind a RunRecord to the positional parameters expected by the
2446
- * `insertRun` prepared statement (17 columns). Shared between the
2447
- * single and batched record paths.
2448
- */
2449
- function bindRun(run: RunRecord): SQLQueryBindings[] {
2450
- return [
2451
- // The ONE place the no-key sentinel is applied, so the column's
2452
- // NOT NULL invariant can't be violated from a call site.
2453
- run.hash ?? '',
2454
- run.project,
2455
- run.task,
2456
- run.status,
2457
- run.exitCode,
2458
- run.durationMs,
2459
- run.forwardArgs ? JSON.stringify(run.forwardArgs) : null,
2460
- run.startedAt,
2461
- run.endedAt,
2462
- run.runId ?? null,
2463
- run.cpuMs ?? null,
2464
- run.peakRssBytes ?? null,
2465
- run.wallclockStartNs !== undefined ? run.wallclockStartNs : null,
2466
- run.wallclockEndNs !== undefined ? run.wallclockEndNs : null,
2467
- run.cacheHit === undefined ? null : run.cacheHit ? 1 : 0,
2468
- run.attempts ?? null,
2469
- ]
2470
- }
2471
-
2472
- /**
2473
- * Bind an InvocationRecord to the positional parameters of the
2474
- * `insertInvocation` prepared statement (25 columns). Booleans map to
2475
- * 0/1; null-or-bool columns (`dirty`) keep null distinct from 0.
1839
+ * Refuse an artifact whose file names no restore can materialise as one
1840
+ * tree, or (given the task's declared outputs) that names a file outside
1841
+ * them. Both reached the tree before: `outputs/out.txt` beside
1842
+ * `outputs/out.txt/x` failed every later run from the local copy, blaming
1843
+ * what was on disk, and an undeclared name overwrote an input or planted a
1844
+ * git hook under a green remote hit (item 942). Thrown at ingest, so the
1845
+ * bytes never reach the local store and the remote read is a miss.
2476
1846
  */
2477
- function bindInvocation(inv: InvocationRecord): SQLQueryBindings[] {
2478
- return [
2479
- inv.runId,
2480
- inv.command,
2481
- inv.requestedTasks,
2482
- inv.cachePolicy,
2483
- inv.concurrency,
2484
- inv.flow,
2485
- inv.startedAt,
2486
- inv.endedAt,
2487
- inv.totalDurationMs,
2488
- inv.taskCount,
2489
- inv.failedCount,
2490
- inv.hitCount,
2491
- inv.hitLocalCount,
2492
- inv.hitRemoteCount,
2493
- inv.exitOk ? 1 : 0,
2494
- inv.commitSha,
2495
- inv.branch,
2496
- inv.dirty === null ? null : inv.dirty ? 1 : 0,
2497
- inv.ci ? 1 : 0,
2498
- inv.ciProvider,
2499
- inv.host,
2500
- inv.os,
2501
- inv.arch,
2502
- inv.vxVersion,
2503
- inv.tags,
2504
- ]
2505
- }
2506
-
2507
- function splitTaskId(id: string): [string, string] {
2508
- const i = id.indexOf('#')
2509
- if (i < 0) return [id, '']
2510
- return [id.slice(0, i), id.slice(i + 1)]
1847
+ function assertArtifactNames(
1848
+ hash: string,
1849
+ entries: ReadonlyArray<{ name: string }>,
1850
+ declared: DeclaredOutputs | undefined,
1851
+ ): void {
1852
+ const names = new Set(entries.map((e) => e.name))
1853
+ for (const name of names) {
1854
+ for (let sep = name.indexOf('/'); sep !== -1; sep = name.indexOf('/', sep + 1)) {
1855
+ if (names.has(name.slice(0, sep))) {
1856
+ throw new CorruptArtifactError(
1857
+ hash,
1858
+ `artifact holds ${name.slice(0, sep)} as a file and as a directory`,
1859
+ )
1860
+ }
1861
+ }
1862
+ }
1863
+ if (declared === undefined) return
1864
+ const files = asTrees(declared.files).map(taskGlob)
1865
+ const wsFiles = asTrees(declared.workspaceFiles).map(taskGlob)
1866
+ for (const name of names) {
1867
+ const ok = name.startsWith('outputs/')
1868
+ ? files.some((g) => g.match(name.slice('outputs/'.length)))
1869
+ : name.startsWith(WORKSPACE_OUTPUT_PREFIX)
1870
+ ? wsFiles.some((g) => g.match(name.slice(WORKSPACE_OUTPUT_PREFIX.length)))
1871
+ : true
1872
+ if (!ok) {
1873
+ throw new CorruptArtifactError(
1874
+ hash,
1875
+ `artifact carries ${name}, which is not one of the task's declared outputs`,
1876
+ )
1877
+ }
1878
+ }
2511
1879
  }