@vzn/vx 0.0.0 → 0.0.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +349 -1
  3. package/launcher.mjs +92 -0
  4. package/package.json +44 -3
  5. package/src/bin.ts +25 -0
  6. package/src/cache/cache.ts +1832 -0
  7. package/src/cache/cas-backend.ts +97 -0
  8. package/src/cache/digest.ts +47 -0
  9. package/src/cache/index.ts +28 -0
  10. package/src/cache/inputs.ts +839 -0
  11. package/src/cache/layered-cache.ts +355 -0
  12. package/src/cache/tar.ts +343 -0
  13. package/src/cli/backend.ts +23 -0
  14. package/src/cli/cache.ts +96 -0
  15. package/src/cli/format.ts +15 -0
  16. package/src/cli/help.ts +129 -0
  17. package/src/cli/index.ts +85 -0
  18. package/src/cli/info.ts +80 -0
  19. package/src/cli/lock.ts +135 -0
  20. package/src/cli/mcp-rpc.ts +292 -0
  21. package/src/cli/mcp.ts +92 -0
  22. package/src/cli/migrate-nx.ts +450 -0
  23. package/src/cli/migrate-turbo.ts +364 -0
  24. package/src/cli/migrate.ts +219 -0
  25. package/src/cli/plan-format.ts +148 -0
  26. package/src/cli/run.ts +683 -0
  27. package/src/cli/show.ts +201 -0
  28. package/src/cli/upgrade.ts +103 -0
  29. package/src/cli/watch.ts +287 -0
  30. package/src/config.ts +460 -0
  31. package/src/exec/env.ts +74 -0
  32. package/src/exec/index.ts +13 -0
  33. package/src/exec/runner.ts +470 -0
  34. package/src/exec/sandbox-runtime.ts +578 -0
  35. package/src/graph/dependency-spec.ts +85 -0
  36. package/src/graph/index.ts +27 -0
  37. package/src/graph/scheduler.ts +677 -0
  38. package/src/graph/task-graph.ts +377 -0
  39. package/src/index.ts +222 -0
  40. package/src/orchestrator/colors.ts +63 -0
  41. package/src/orchestrator/devframe-surface.ts +74 -0
  42. package/src/orchestrator/events.ts +277 -0
  43. package/src/orchestrator/execute-task.ts +939 -0
  44. package/src/orchestrator/framed-output.ts +461 -0
  45. package/src/orchestrator/history.ts +148 -0
  46. package/src/orchestrator/index.ts +174 -0
  47. package/src/orchestrator/local-shortcircuit.ts +147 -0
  48. package/src/orchestrator/logger.ts +526 -0
  49. package/src/orchestrator/metrics.ts +1927 -0
  50. package/src/orchestrator/options.ts +197 -0
  51. package/src/orchestrator/plan.ts +129 -0
  52. package/src/orchestrator/plugin-host.ts +185 -0
  53. package/src/orchestrator/plugin.ts +283 -0
  54. package/src/orchestrator/predict.ts +106 -0
  55. package/src/orchestrator/prepare.ts +306 -0
  56. package/src/orchestrator/protocol.ts +156 -0
  57. package/src/orchestrator/remote-prefetch.ts +81 -0
  58. package/src/orchestrator/resources.ts +59 -0
  59. package/src/orchestrator/run-artifacts.ts +111 -0
  60. package/src/orchestrator/run-context.ts +205 -0
  61. package/src/orchestrator/run-report.ts +132 -0
  62. package/src/orchestrator/run-state.ts +145 -0
  63. package/src/orchestrator/run.ts +887 -0
  64. package/src/orchestrator/stable-keys.ts +177 -0
  65. package/src/orchestrator/status-line.ts +287 -0
  66. package/src/orchestrator/summary.ts +315 -0
  67. package/src/orchestrator/tally.ts +69 -0
  68. package/src/orchestrator/task-hash.ts +253 -0
  69. package/src/orchestrator/telemetry-host.ts +80 -0
  70. package/src/orchestrator/telemetry.ts +358 -0
  71. package/src/orchestrator/upstream.ts +121 -0
  72. package/src/orchestrator/verify.ts +164 -0
  73. package/src/orchestrator/wire-render.ts +92 -0
  74. package/src/orchestrator/wire.ts +212 -0
  75. package/src/util/errors.ts +11 -0
  76. package/src/util/hash.ts +25 -0
  77. package/src/util/index.ts +8 -0
  78. package/src/util/paths.ts +16 -0
  79. package/src/util/size.ts +16 -0
  80. package/src/util/ulid.ts +15 -0
  81. package/src/version.ts +10 -0
  82. package/src/workspace/affected.ts +118 -0
  83. package/src/workspace/filter.ts +155 -0
  84. package/src/workspace/fingerprint.ts +41 -0
  85. package/src/workspace/index.ts +27 -0
  86. package/src/workspace/lockfile.ts +118 -0
  87. package/src/workspace/nested-dirs.ts +34 -0
  88. package/src/workspace/package-graph.ts +148 -0
  89. package/src/workspace/project-loader.ts +512 -0
  90. package/src/workspace/workspace.ts +176 -0
@@ -0,0 +1,1832 @@
1
+ // Content-addressed task cache.
2
+ //
3
+ // On-disk layout:
4
+ // <cacheDir>/cache.db — SQLite index (entries + runs + file_hashes)
5
+ // <cacheDir>/<hash>.tar.zst — per-entry artifact:
6
+ // stdout (captured stdout, always present)
7
+ // outputs/ (declared output files, when any)
8
+ // workspace-outputs/ (declared outputs.workspaceFiles,
9
+ // root-relative, when any)
10
+ //
11
+ // The artifact carries ONLY replayable bytes (logs + outputs). Entry
12
+ // metadata — taskId, command, exitCode, durationMs, storedAt — lives
13
+ // in the SQLite `entries` row. The same tar.zst bytes ship to a remote
14
+ // cache server unchanged; on remote-hit, the caller supplies metadata
15
+ // via the `ingest(hash, bytes, meta)` API so the local SQL index gets
16
+ // populated without sniffing the artifact.
17
+ //
18
+ // We never cache failed runs, so stderr is dropped from the cached
19
+ // surface entirely. Live runs still stream stderr through the logger
20
+ // for the user to see — but on a cache hit there's nothing to replay
21
+ // (the original run was successful and stderr typically empty).
22
+ //
23
+ // Replace this module to plug in remote storage. The contract is:
24
+ // key() : derive a stable hash from a task's identity + inputs
25
+ // get(hash, ctx?) : retrieve a previous run's metadata, or null
26
+ // restoreOutputs : extract the artifact's outputs/ into the project dir
27
+ // save : persist outputs + stdout under a hash
28
+ // ingest : adopt an artifact produced elsewhere (remote-hit path)
29
+ // recordRun : append a row to the run history table (for stats)
30
+ // close : release the SQLite handle
31
+
32
+ import { Database, type SQLQueryBindings } from 'bun:sqlite'
33
+ import { mkdirSync, statSync } from 'node:fs'
34
+ import { mkdir, mkdtemp, rename, rm, stat, utimes } from 'node:fs/promises'
35
+ import os from 'node:os'
36
+ import path from 'node:path'
37
+ import { relPosix, UserError, xxh3, xxh3hex } from '../util/index.js'
38
+ import { FsCASBackend } from './cas-backend.js'
39
+ import { extractOutputs, parseTarHeaders, readTarText } from './tar.js'
40
+
41
+ // v17: artifact carries only logs + outputs (stdout + outputs/<rel>).
42
+ // Local and remote layers transport the SAME tar.zst bytes — no
43
+ // separate stage/meta.json/tar.gz dance for remote, no
44
+ // `cache-archive.ts`. stderr is no longer cached: we only cache
45
+ // successful runs and stderr is rarely meaningful on success.
46
+ // v19: '^task' dependsOn expansion switched from transitive-deps to
47
+ // nearest-holder frontier — upstream-hash sets shrink, so keys change.
48
+ // v20: input-file content hashes switched from xxh3 to git blob OIDs
49
+ // (Turbo parity). Clean tracked files take their OID straight from
50
+ // the index (harvested by the bulk `git ls-files -s`); dirty /
51
+ // untracked files get the identical OID computed in-process. Every
52
+ // file's hash bytes change → bump. SCHEMA_VERSION moves with it:
53
+ // pre-v20 `file_hashes.content_hash` rows hold xxh3 digests that
54
+ // must not leak into the OID domain via the mtime+size memo.
55
+ // v22: reverted the v21 output-fold "early cutoff". Downstream keys
56
+ // fold the upstream's INPUT key (its task hash) again — pure-input
57
+ // transitive hashing (Turbo/Nx model). No output content participates
58
+ // in any cache key. SCHEMA v21 drops the now-unused outputs_hash
59
+ // column. Early cutoff removed (an upstream that re-emits identical
60
+ // output still re-runs dependents) — rare, not worth the cascade.
61
+ // v23: fold cache.inputs.runtime / workspaceRuntime command output into
62
+ // the key (two namespaced sections after env-values). Command strings
63
+ // live in the resolved config (frozen by `vx lock`); the OUTPUT is
64
+ // resolved live every run, so it stays correct under --frozen. No
65
+ // SCHEMA bump — only Cache.key derivation changed.
66
+ // v24: vx-lock.json added to the always-ignored input set (cache/
67
+ // inputs.ts ALWAYS_IGNORE). A task whose globs matched the root
68
+ // lockfile (broad `**/*` on the root project) drops it from the hashed
69
+ // file set, so those keys change; tasks that never matched it are
70
+ // byte-identical. No SCHEMA bump — only the input file set changed.
71
+ //
72
+ // CACHE_VERSION stays v24 through the Tier-3 schema roll (SCHEMA v22).
73
+ // Tier 3 persists the very components already fed to `Cache.key()` —
74
+ // it adds nothing to, reorders nothing in, and reweights nothing
75
+ // inside the key fold (the new `captureInto` sink is a pure
76
+ // side-channel). A task's hash is byte-identical before and after, so
77
+ // existing artifacts stay valid and there is no cache-version bump.
78
+ const CACHE_VERSION = 'vx-cache-v24'
79
+ // SCHEMA history (drop+recreate on mismatch; pre-alpha, no migrations):
80
+ // v20: file_hashes.content_hash (git blob OIDs).
81
+ // v21: dropped the unused outputs_hash column (pure-input hashing).
82
+ // v22: Tier-3 dashboard tables — `invocations` (one header row per
83
+ // `vx run` with git/CI/host context + tags) and `entry_inputs`
84
+ // (one row per cache-key component, keyed by the cache-entry
85
+ // HASH, the input-fingerprint moat). `entry_inputs` is written
86
+ // INSIDE the entry-save transaction (only on a miss/save), so a
87
+ // warm all-cache-hit run writes nothing to it — the warm path
88
+ // does zero extra work. `invocations` is still recorded once per
89
+ // run via `recordRunBundle`. The cache KEY is unchanged
90
+ // (CACHE_VERSION not bumped) — these tables persist analytics
91
+ // derived from the same `CacheKeyInput` the key already consumes.
92
+ // v23: runs.attempts — the number of attempts a retried task took (>1),
93
+ // the DIRECT within-run flaky signal (a task that failed then
94
+ // passed under identical inputs is nondeterministic by definition;
95
+ // no cross-run inference needed). Nullable; NULL for a once-run
96
+ // task. Analytics-only — the cache KEY is unchanged.
97
+ const SCHEMA_VERSION = 'v23'
98
+
99
+ /**
100
+ * Artifact + `output_files` namespace prefix for workspace-root-
101
+ * anchored outputs (`cache.outputs.workspaceFiles`). Project outputs
102
+ * keep their bare project-relative `path` rows; workspace rows store
103
+ * the full `workspace-outputs/<rel-to-root>` tar entry name as the
104
+ * discriminator — least-invasive row format, no schema change. A
105
+ * project output dir literally named `workspace-outputs/` would
106
+ * collide with the namespace; the name is reserved.
107
+ */
108
+ export const WORKSPACE_OUTPUT_PREFIX = 'workspace-outputs/'
109
+
110
+ /**
111
+ * Independent read/write control over the two cache layers (local +
112
+ * remote). Replaces the old single `noCache` boolean: each axis can be
113
+ * toggled on its own so `--force` (re-execute but still refresh the
114
+ * cache) is distinct from `--no-cache` (disable everything).
115
+ *
116
+ * Only the task-artifact get/save path is gated. `recordRun`, `stats`,
117
+ * `prune`, key derivation, and prefetch-ingest are never affected — they
118
+ * are bookkeeping/analytics that a run policy has no business disabling.
119
+ */
120
+ export interface CachePolicy {
121
+ localRead: boolean
122
+ localWrite: boolean
123
+ remoteRead: boolean
124
+ remoteWrite: boolean
125
+ }
126
+
127
+ /** All four axes on — the default when no cache flag is passed. */
128
+ export const FULL_CACHE_POLICY: CachePolicy = {
129
+ localRead: true,
130
+ localWrite: true,
131
+ remoteRead: true,
132
+ remoteWrite: true,
133
+ }
134
+
135
+ /**
136
+ * Parse a `--cache=<spec>` value into a `CachePolicy`, starting from a
137
+ * base (defaults to {@link FULL_CACHE_POLICY}). The spec is a
138
+ * comma-separated list of `layer:flags` segments where `layer` is
139
+ * `local` or `remote` and `flags` is any subset of `r` (read) and `w`
140
+ * (write), order-independent and possibly empty.
141
+ *
142
+ * A mentioned layer is set EXACTLY to its flags (read = includes `r`,
143
+ * write = includes `w`); an unmentioned layer keeps its base value.
144
+ * So `local:rw,remote:r` = remote read-only, `remote:` = remote fully
145
+ * off, `local:r` = local read-only with remote untouched.
146
+ *
147
+ * Throws a {@link UserError} on an unknown layer, an unknown flag, a
148
+ * duplicated flag, a missing colon, or a repeated layer.
149
+ */
150
+ export function parseCachePolicy(spec: string, base: CachePolicy = FULL_CACHE_POLICY): CachePolicy {
151
+ const out: CachePolicy = { ...base }
152
+ const seen = new Set<string>()
153
+ for (const rawSeg of spec.split(',')) {
154
+ const seg = rawSeg.trim()
155
+ if (seg.length === 0) continue
156
+ const colon = seg.indexOf(':')
157
+ if (colon < 0) {
158
+ throw new UserError(`invalid --cache segment '${seg}': expected '<layer>:<flags>'`)
159
+ }
160
+ const layer = seg.slice(0, colon)
161
+ const flags = seg.slice(colon + 1)
162
+ if (layer !== 'local' && layer !== 'remote') {
163
+ throw new UserError(`invalid --cache layer '${layer}': expected 'local' or 'remote'`)
164
+ }
165
+ if (seen.has(layer)) {
166
+ throw new UserError(`--cache layer '${layer}' specified twice`)
167
+ }
168
+ seen.add(layer)
169
+ const flagSet = new Set<string>()
170
+ for (const ch of flags) {
171
+ if (ch !== 'r' && ch !== 'w') {
172
+ throw new UserError(`invalid --cache flag '${ch}' for '${layer}': expected 'r' and/or 'w'`)
173
+ }
174
+ if (flagSet.has(ch)) {
175
+ throw new UserError(`--cache flag '${ch}' repeated for '${layer}'`)
176
+ }
177
+ flagSet.add(ch)
178
+ }
179
+ if (layer === 'local') {
180
+ out.localRead = flagSet.has('r')
181
+ out.localWrite = flagSet.has('w')
182
+ } else {
183
+ out.remoteRead = flagSet.has('r')
184
+ out.remoteWrite = flagSet.has('w')
185
+ }
186
+ }
187
+ return out
188
+ }
189
+
190
+ export interface CacheKeyInput {
191
+ taskId: string
192
+ /**
193
+ * Hash of the resolved task config (post-evaluation). Folds in everything
194
+ * the user wrote — command, env declarations (passThrough names + define
195
+ * key/value pairs), dependsOn, cache.inputs declarations, outputs — including
196
+ * values that arrived via `import` at config-load time.
197
+ */
198
+ taskConfigHash: string
199
+ /**
200
+ * Runtime values of declared cache-input env names (from parent at hash
201
+ * time). Independent of `exec.env`; lives here for cache identity.
202
+ */
203
+ envValues: Array<[name: string, value: string]>
204
+ /**
205
+ * Resolved `cache.inputs.runtime` commands as [command, output] pairs
206
+ * (output = trimmed stdout+stderr, resolved live at hash time). Folded
207
+ * into the key in a namespace distinct from workspaceRuntimeValues.
208
+ */
209
+ runtimeValues?: Array<[command: string, output: string]>
210
+ /** Resolved `cache.inputs.workspaceRuntime` pairs (root-cwd commands). */
211
+ workspaceRuntimeValues?: Array<[command: string, output: string]>
212
+ /** Absolute paths to input files. */
213
+ inputFiles: string[]
214
+ workspaceRoot: string
215
+ /** Cache keys of upstream tasks this one depends on, sorted. */
216
+ upstreamHashes: string[]
217
+ /**
218
+ * Upstream hash → upstream task id, for `captureInto` row NAMING only.
219
+ * Never folded into the digest — the key already folds the sorted
220
+ * hashes. Lets the persisted `entry_inputs` row name which upstream
221
+ * task a hash came from (the diff reads better than a bare hash). When
222
+ * absent, the captured `upstream` row falls back to `name = hash`.
223
+ */
224
+ upstreamIds?: ReadonlyMap<string, string>
225
+ /**
226
+ * Workspace-level fingerprint — typically a hash of `pnpm-lock.yaml` +
227
+ * `pnpm-workspace.yaml`. Folds resolved dep versions and workspace shape
228
+ * into every task's key, so a lockfile bump invalidates everything.
229
+ */
230
+ workspaceFingerprint: string
231
+ /**
232
+ * CLI args forwarded to the task (after `--`). Folded into the key so that
233
+ * the same command with different forwarded args is treated as a distinct
234
+ * run, never a spurious cache hit.
235
+ */
236
+ forwardArgs?: readonly string[]
237
+ /**
238
+ * Hash of the project's `package.json` bytes. Folded into the key
239
+ * implicitly (Turbo / Nx parity) so dep changes invalidate every
240
+ * task in that project, even when `cache.inputs.files` doesn't
241
+ * cover package.json. Empty string when the project has no
242
+ * package.json (impossible in practice — workspace discovery
243
+ * requires one — but we don't fail-loud here).
244
+ */
245
+ projectPackageJsonHash: string
246
+ /**
247
+ * Precomputed content hashes (git blob OIDs) keyed by absolute
248
+ * path — typically the trusted-index OID map harvested by the
249
+ * run's bulk `git ls-files -s`. Paths present here skip `hashFile`
250
+ * entirely (no stat, no SQLite, no read); missing paths fall back
251
+ * to `hashFile`, which computes the byte-identical blob OID from
252
+ * disk. Pure fast path: the derived key never depends on whether a
253
+ * hash arrived via the map or the fallback.
254
+ */
255
+ fileHashes?: ReadonlyMap<string, string>
256
+ /**
257
+ * When set, `key()` pushes each component (kind, name, hash) it folds
258
+ * — at the same fold sites, in fold order. Pure SIDE-CHANNEL: it does
259
+ * not change the returned digest in any way. On a cache MISS the
260
+ * orchestrator allocates an array, passes it here, and persists the
261
+ * rows to `entry_inputs` (inside the entry-save transaction) so a
262
+ * later run can diff its inputs against this one (the Tier-3 "why did
263
+ * this re-run?" moat). Capturing at the fold sites keeps the
264
+ * recorded set in lockstep with the key by construction: a future
265
+ * component that forgets to capture is a one-line miss, not silent
266
+ * drift. The per-file OIDs are already awaited here, so file rows cost
267
+ * zero extra I/O — just array pushes.
268
+ */
269
+ captureInto?: Array<{ kind: string; name: string; hash: string }>
270
+ }
271
+
272
+ export interface CacheEntry {
273
+ hash: string
274
+ taskId: string
275
+ command: string
276
+ exitCode: number
277
+ durationMs: number
278
+ outputFiles: string[]
279
+ /** Captured stdout, always present (may be empty). stderr is not cached. */
280
+ stdout: string
281
+ storedAt: string
282
+ /**
283
+ * Where this hit was resolved from. `'local'` for a SQLite-backed
284
+ * Cache; `'remote'` when LayeredCache pulled the artifact from the
285
+ * remote layer this lookup (even though it's been materialized into
286
+ * local for next time). Lets the orchestrator surface
287
+ * `cache-hit-remote` so users see when remote caching actually saved
288
+ * them work vs. a stale-local replay.
289
+ */
290
+ source?: 'local' | 'remote'
291
+ }
292
+
293
+ export interface RunRecord {
294
+ hash: string
295
+ project: string
296
+ task: string
297
+ status: 'success' | 'failed' | 'cache-hit' | 'cache-hit-remote' | 'skipped'
298
+ exitCode: number
299
+ durationMs: number
300
+ forwardArgs?: readonly string[]
301
+ startedAt: number // ms-epoch wall clock
302
+ endedAt: number // ms-epoch wall clock
303
+ /**
304
+ * Optional analytics columns. Populated by the orchestrator/runner;
305
+ * stored as NULL on rows from older runs. Surfaced via `vx stats`
306
+ * and consumable from CI by reading cache.db directly.
307
+ */
308
+ runId?: string // ULID shared across every task in one `vx run` invocation
309
+ cpuMs?: number // sum of user + system CPU time for the child process
310
+ peakRssBytes?: number // peak resident set size of the child process
311
+ wallclockStartNs?: bigint // hrtime span relative to run t=0
312
+ wallclockEndNs?: bigint
313
+ cacheHit?: boolean // convenience for flamegraph color; derivable from status
314
+ attempts?: number // >1 when the task retried; the direct within-run flaky signal
315
+ }
316
+
317
+ /**
318
+ * One header row per `vx run` invocation (the `invocations` table). All
319
+ * fields mirror the columns; nullable VCS/host columns are `null` when
320
+ * the probe failed (not a git repo, hostname unavailable). Recorded
321
+ * once per run inside the same transaction as the per-task `runs` rows.
322
+ */
323
+ export interface InvocationRecord {
324
+ runId: string
325
+ command: string
326
+ /** JSON-serialized `string[]` of the requested task names. */
327
+ requestedTasks: string
328
+ /** Compact policy flags, e.g. `'lR,lW,rR,rW'` (an axis omitted = off). */
329
+ cachePolicy: string
330
+ concurrency: number
331
+ flow: 'focused' | 'broad' | null
332
+ startedAt: number
333
+ endedAt: number
334
+ totalDurationMs: number
335
+ taskCount: number
336
+ failedCount: number
337
+ hitCount: number
338
+ hitLocalCount: number
339
+ hitRemoteCount: number
340
+ exitOk: boolean
341
+ commitSha: string | null
342
+ branch: string | null
343
+ dirty: boolean | null
344
+ ci: boolean
345
+ ciProvider: string | null
346
+ host: string | null
347
+ os: string | null
348
+ arch: string | null
349
+ vxVersion: string
350
+ /** JSON object `{k:v}` of `--tag` pairs. */
351
+ tags: string
352
+ }
353
+
354
+ /**
355
+ * One cache-key component row for the `entry_inputs` table — keyed by
356
+ * the cache-entry HASH it belongs to, not a run. Written inside the
357
+ * entry-save transaction (`writeArtifactAndIndex`) on a miss/save; a
358
+ * cache HIT does not save, so it persists nothing (warm runs are free).
359
+ * The diff (Phase B/B1) reads these by the run's task hash
360
+ * (`runs.hash → entry_inputs[entry_hash]`).
361
+ */
362
+ export interface TaskInputRow {
363
+ entryHash: string
364
+ kind: string
365
+ name: string
366
+ hash: string
367
+ }
368
+
369
+ export interface CacheStats {
370
+ entryCount: number
371
+ totalBytes: number
372
+ runCountLast24h: number
373
+ hitCountLast24h: number
374
+ }
375
+
376
+ export interface PruneOptions {
377
+ /** Drop entries last accessed before this ms-epoch threshold. */
378
+ olderThanMs?: number
379
+ /**
380
+ * After applying olderThanMs, if the cache still exceeds this size in
381
+ * bytes, evict LRU (smallest `accessed_at` first) until under it.
382
+ */
383
+ maxBytes?: number
384
+ }
385
+
386
+ export interface PruneResult {
387
+ evicted: number
388
+ bytesFreed: number
389
+ }
390
+
391
+ /**
392
+ * Per-output-file fingerprint, scoped by the cache entry that
393
+ * produced it. Batch-loaded once at the top of a run via
394
+ * `loadOutputFilesBatch(hashes)` so the orchestrator's "is this
395
+ * tree already current?" probe becomes an in-memory Map lookup
396
+ * plus N parallel stat calls.
397
+ *
398
+ * `path` is project-relative (e.g. `dist/index.js`), matching how
399
+ * outputs are addressed under `<projectDir>/` — except workspace
400
+ * outputs, which carry the full `workspace-outputs/<rel-to-root>`
401
+ * tar entry name (see `WORKSPACE_OUTPUT_PREFIX`); callers split on
402
+ * the prefix and anchor those at the workspace root.
403
+ */
404
+ export interface OutputFileRow {
405
+ path: string
406
+ size: number
407
+ mode: number
408
+ mtimeMs: number
409
+ }
410
+
411
+ /**
412
+ * The shape every cache implementation honors. `Cache` (the local v10
413
+ * implementation) and `LayeredCache` both `implements` this so the
414
+ * orchestrator's `executeTask` can take either without a discriminated
415
+ * union and we get a compile-time guarantee the surfaces stay congruent.
416
+ */
417
+ /**
418
+ * Context passed to `get()`. Optional, but required when the lookup
419
+ * may resolve through the remote layer — the local SQL row inserted on
420
+ * remote-hit needs `taskId` + `command` to be queryable later (the
421
+ * artifact itself doesn't carry them). `Cache` (local) ignores this
422
+ * field; `LayeredCache` forwards it to `Cache.ingest`.
423
+ */
424
+ export interface CacheGetContext {
425
+ taskId: string
426
+ command: string
427
+ }
428
+
429
+ /** Metadata supplied at ingest time — values the artifact does not carry. */
430
+ export interface IngestMeta {
431
+ taskId: string
432
+ command: string
433
+ /** Wall-clock time of the original task execution. */
434
+ durationMs: number
435
+ /**
436
+ * Cache-key components (Tier-3 input fingerprint) to persist into
437
+ * `entry_inputs` in the same transaction as the entry row. Omitted on
438
+ * the remote-hit ingest path (the artifact doesn't carry them — the
439
+ * fingerprint is the saving machine's local moat). Only computed on a
440
+ * miss/save, so a cache hit never reaches here.
441
+ */
442
+ inputComponents?: readonly TaskInputRow[]
443
+ }
444
+
445
+ /**
446
+ * The supplied artifact bytes don't decompress/parse as a vx artifact.
447
+ * Thrown by `save`/`ingest` BEFORE anything reaches the final cache
448
+ * path — a rejected artifact leaves no `<hash>.tar.zst` and no SQL row.
449
+ * The LayeredCache treats this as a remote fault on the remote-hit
450
+ * path (degrades to a cache miss).
451
+ */
452
+ export class CorruptArtifactError extends Error {
453
+ constructor(
454
+ public readonly hash: string,
455
+ reason: string,
456
+ public override readonly cause?: unknown,
457
+ ) {
458
+ super(`cache: corrupt artifact for ${hash}: ${reason}`)
459
+ this.name = 'CorruptArtifactError'
460
+ }
461
+ }
462
+
463
+ /**
464
+ * A decompressed artifact above this is refused as a zstd bomb rather than
465
+ * expanded into memory. 2 GiB comfortably exceeds any real build output while
466
+ * bounding a malicious/compromised remote's ability to OOM a victim who takes
467
+ * a cache hit.
468
+ */
469
+ export const MAX_DECOMPRESSED_ARTIFACT_BYTES = 2 * 1024 * 1024 * 1024
470
+
471
+ /**
472
+ * Read a zstd frame's declared Frame_Content_Size (RFC 8878 §3.1.1) WITHOUT
473
+ * decompressing. Returns null when the frame omits it (streaming frames) or
474
+ * the header is too short to parse. vx's own producer (single-shot
475
+ * `Bun.zstdCompress` of a known buffer) always writes it, so an artifact that
476
+ * declares an enormous size can be rejected before a byte is allocated.
477
+ *
478
+ * Exported for tests: pins the per-`fcsFlag` byte layouts (incl. the 2-byte
479
+ * `+256` adjustment and the `dictIdFlag` offset) that a bomb-refusal e2e can't
480
+ * discriminate (their max declarable sizes sit below the ceiling).
481
+ */
482
+ export function zstdContentSize(b: Uint8Array): bigint | null {
483
+ if (b.length < 5) return null
484
+ // Magic_Number 0xFD2FB528, little-endian.
485
+ if (b[0] !== 0x28 || b[1] !== 0xb5 || b[2] !== 0x2f || b[3] !== 0xfd) return null
486
+ const desc = b[4]!
487
+ const fcsFlag = desc >> 6
488
+ const singleSegment = (desc >> 5) & 1
489
+ const dictIdFlag = desc & 3
490
+ let off = 5
491
+ if (singleSegment === 0) off += 1 // Window_Descriptor byte
492
+ off += dictIdFlag === 3 ? 4 : dictIdFlag // Dictionary_ID: 0,1,2,4 bytes
493
+ let fcsSize: number
494
+ if (fcsFlag === 0) fcsSize = singleSegment === 1 ? 1 : 0
495
+ else if (fcsFlag === 1) fcsSize = 2
496
+ else if (fcsFlag === 2) fcsSize = 4
497
+ else fcsSize = 8
498
+ if (fcsSize === 0 || b.length < off + fcsSize) return null
499
+ let v = 0n
500
+ for (let i = 0; i < fcsSize; i++) v |= BigInt(b[off + i]!) << BigInt(8 * i)
501
+ if (fcsSize === 2) v += 256n // per spec, the 2-byte field stores value − 256
502
+ return v
503
+ }
504
+
505
+ /**
506
+ * Decompress a zstd artifact with a hard output ceiling. `trusted` marks a
507
+ * locally-produced artifact (always carries a declared content size); an
508
+ * UNTRUSTED (remote-sourced) frame that declares no content size is refused,
509
+ * since our producer and the Turbo servers we interop with always write one —
510
+ * a sizeless frame from the network is the shape a bomb takes to dodge the
511
+ * pre-decompress check.
512
+ */
513
+ async function zstdDecompressBounded(
514
+ compressed: Uint8Array,
515
+ hash: string,
516
+ trusted: boolean,
517
+ ): Promise<Uint8Array> {
518
+ const declared = zstdContentSize(compressed)
519
+ if (declared !== null && declared > BigInt(MAX_DECOMPRESSED_ARTIFACT_BYTES)) {
520
+ throw new CorruptArtifactError(
521
+ hash,
522
+ `declares ${declared} decompressed bytes (> ${MAX_DECOMPRESSED_ARTIFACT_BYTES} cap)`,
523
+ )
524
+ }
525
+ if (declared === null && !trusted) {
526
+ throw new CorruptArtifactError(hash, 'remote zstd frame carries no declared content size')
527
+ }
528
+ const out = await Bun.zstdDecompress(compressed)
529
+ if (out.length > MAX_DECOMPRESSED_ARTIFACT_BYTES) {
530
+ throw new CorruptArtifactError(
531
+ hash,
532
+ `decompressed to ${out.length} bytes (> ${MAX_DECOMPRESSED_ARTIFACT_BYTES} cap)`,
533
+ )
534
+ }
535
+ return out
536
+ }
537
+
538
+ export interface CacheLayer {
539
+ key(input: CacheKeyInput): Promise<string>
540
+ get(hash: string, ctx?: CacheGetContext): Promise<CacheEntry | null>
541
+ /**
542
+ * Lightweight existence probe. `'local'` / `'remote'` names the layer
543
+ * that holds the artifact; `null` is a miss. NEVER moves bytes: no
544
+ * artifact read, no remote download, no local ingest, no accessed_at
545
+ * bump (the LayeredCache's remote side is an HTTP HEAD). Planning
546
+ * (`--dry` / `--graph`) predicts hits with this instead of `get` so a
547
+ * dry run can't pull N artifacts over the network. Remote errors
548
+ * degrade to `null` — an existence probe never fails anything.
549
+ */
550
+ has(hash: string): Promise<'local' | 'remote' | null>
551
+ /**
552
+ * Best-effort warm of `hash` from a slower layer (the remote cache)
553
+ * into this one, so a later `get(hash)` resolves locally without a
554
+ * round-trip on the task's critical path. Returns `true` if the
555
+ * artifact is now present locally, `false` on a miss / error
556
+ * (degrades, never throws). The local `Cache` is a no-op (nothing
557
+ * slower to warm from); `LayeredCache` owns the real implementation
558
+ * plus the in-flight de-dup so prefetch + get share ONE remote GET.
559
+ */
560
+ prefetch(hash: string, ctx?: CacheGetContext): Promise<boolean>
561
+ /**
562
+ * Batched lookup of per-output-file fingerprints for many cache
563
+ * entries in one SQL round-trip. Returns a Map keyed by entry hash.
564
+ *
565
+ * Orchestrator pattern: call this once at `prepareRun` for every
566
+ * task whose hash is known up-front, then per-task `executeCachedTask`
567
+ * does a Map.get (O(1)) + parallel stat checks to decide whether the
568
+ * on-disk tree is already current.
569
+ *
570
+ * Hashes with no rows (cache misses) are absent from the result.
571
+ */
572
+ loadOutputFilesBatch(hashes: readonly string[]): Map<string, OutputFileRow[]>
573
+ /**
574
+ * Stat each `expected` row's target under `projectDir` and return
575
+ * `true` iff every (size, mode, mtime) matches. Missing files,
576
+ * stat errors, or any mismatch → `false`.
577
+ *
578
+ * Pure FS check — no DB access. Caller batches the expected rows
579
+ * via `loadOutputFilesBatch`. Lets the orchestrator skip
580
+ * `cleanOutputs + restoreOutputs` entirely when the cached
581
+ * snapshot is already in place. Integrity-preserving: detects
582
+ * out-of-band file edits or deletions and falls through to a real
583
+ * restore.
584
+ */
585
+ isOutputsCurrent(projectDir: string, expected: readonly OutputFileRow[]): Promise<boolean>
586
+ /**
587
+ * Extract the artifact's `outputs/` entries into `projectDir` and —
588
+ * when `workspaceRoot` is given — its `workspace-outputs/` entries
589
+ * into the workspace root. Callers restoring entries that may carry
590
+ * workspace outputs must pass `workspaceRoot`.
591
+ */
592
+ restoreOutputs(hash: string, projectDir: string, workspaceRoot?: string): Promise<void>
593
+ save(args: {
594
+ hash: string
595
+ entry: Omit<CacheEntry, 'hash' | 'storedAt' | 'outputFiles'>
596
+ projectDir: string
597
+ outputFiles: string[]
598
+ /**
599
+ * Resolved `outputs.workspaceFiles` (absolute paths) + the root
600
+ * they're relative to. Packed under `workspace-outputs/<rel>`.
601
+ * Omitted → artifact bytes identical to the pre-workspaceFiles
602
+ * format.
603
+ */
604
+ workspaceOutputFiles?: string[]
605
+ workspaceRoot?: string
606
+ /**
607
+ * Cache-key components for this entry (the Tier-3 input
608
+ * fingerprint). Persisted to `entry_inputs` inside the same
609
+ * transaction as the entry row, via `INSERT OR IGNORE` (a re-save
610
+ * of the same hash is a no-op). Omitted/empty → nothing written.
611
+ * Only computed on the miss/save path — never on a hit.
612
+ */
613
+ inputComponents?: readonly TaskInputRow[]
614
+ }): Promise<void>
615
+ /**
616
+ * Adopt an artifact produced elsewhere — the remote-hit path. Writes
617
+ * the compressed bytes to `<cacheDir>/<hash>.tar.zst`, parses the
618
+ * tar headers to populate the `output_files` rows, and inserts the
619
+ * `entries` row using the caller-supplied `meta`. After this returns,
620
+ * the next `get(hash)` resolves locally.
621
+ */
622
+ ingest(hash: string, compressed: Uint8Array, meta: IngestMeta): Promise<void>
623
+ recordRun(run: RunRecord): void
624
+ /**
625
+ * Append every run in `runs` to the history in a single SQLite
626
+ * transaction. ~10× faster than calling `recordRun` in a loop when
627
+ * `runs.length > ~50` (one fsync vs. N).
628
+ */
629
+ recordRuns(runs: readonly RunRecord[]): void
630
+ /**
631
+ * Record a whole `vx run` atomically: the per-task `runs` rows and
632
+ * the one `invocations` header row — in a SINGLE transaction (one
633
+ * fsync). Replaces the bare `recordRuns` call in the orchestrator's
634
+ * end-of-run path. Input-fingerprint rows are NOT written here — they
635
+ * ride the entry-save transaction (`save`/`ingest`) so a warm
636
+ * all-cache-hit run writes nothing.
637
+ */
638
+ recordRunBundle(bundle: { runs: readonly RunRecord[]; invocation: InvocationRecord }): void
639
+ stats(): CacheStats
640
+ /**
641
+ * Content-hash a file with an mtime+size fast path. If the
642
+ * `(mtime_ms, size_bytes)` of `filePath` match a previously seen
643
+ * row, return the stored xxh3 digest instead of re-reading the
644
+ * bytes. Otherwise read + hash + upsert. The hash is byte-for-byte
645
+ * identical to what a fresh content-hash would produce — pure
646
+ * optimization, no cache-key change.
647
+ */
648
+ hashFile(filePath: string): Promise<string>
649
+ /**
650
+ * Absolute path to the on-disk outputs artifact for a hash —
651
+ * `<cacheDir>/<hash>.tar.zst`. Returns the path whether or
652
+ * not the artifact exists. Exposed for telemetry / dashboards;
653
+ * `restoreOutputs` is the canonical way to materialize the bytes.
654
+ */
655
+ outputsPath(hash: string): string
656
+ prune(options: PruneOptions): Promise<PruneResult>
657
+ close(): void
658
+ }
659
+
660
+ /**
661
+ * Convenience alias for the `save()` args. Used by `LayeredCache` to
662
+ * forward call args without redeclaring the structural shape — NOT
663
+ * part of the conceptual cache contract; consumers should call
664
+ * `CacheLayer.save({ ... })` directly.
665
+ *
666
+ * @internal
667
+ */
668
+ export type SaveArgs = Parameters<CacheLayer['save']>[0]
669
+
670
+ interface EntryRow {
671
+ hash: string
672
+ project: string
673
+ task: string
674
+ command: string
675
+ exit_code: number
676
+ duration_ms: number
677
+ size_bytes: number
678
+ stdout: string
679
+ created_at: number
680
+ accessed_at: number
681
+ }
682
+
683
+ export class Cache implements CacheLayer {
684
+ private readonly db: Database
685
+ private readonly insertEntry: ReturnType<Database['prepare']>
686
+ private readonly selectEntry: ReturnType<Database['prepare']>
687
+ private readonly bumpAccessed: ReturnType<Database['prepare']>
688
+ private readonly touched = new Set<string>()
689
+ private readonly insertRun: ReturnType<Database['prepare']>
690
+ private readonly insertInvocation: ReturnType<Database['prepare']>
691
+ private readonly insertEntryInput: ReturnType<Database['prepare']>
692
+ private readonly selectFileHash: ReturnType<Database['prepare']>
693
+ private readonly upsertFileHash: ReturnType<Database['prepare']>
694
+ private readonly insertOutputFile: ReturnType<Database['prepare']>
695
+ private readonly deleteOutputFiles: ReturnType<Database['prepare']>
696
+ /** Memoized repo object format for blob-OID hashing (lazy-detected). */
697
+ private objectFormat: 'sha1' | 'sha256' | null = null
698
+
699
+ /**
700
+ * Local-layer read/write gates for the task ARTIFACT path only.
701
+ * `get()` returns null when `!read`; `save()` skips the artifact +
702
+ * index write when `!write`. Everything else (recordRun, stats,
703
+ * prune, ingest, hashing) is unaffected — those are bookkeeping a
704
+ * run policy must not disable. Default: both true.
705
+ */
706
+ private readonly read: boolean
707
+ private readonly write: boolean
708
+
709
+ constructor(
710
+ private readonly cacheDir: string,
711
+ localPolicy: { read: boolean; write: boolean } = { read: true, write: true },
712
+ ) {
713
+ this.read = localPolicy.read
714
+ this.write = localPolicy.write
715
+ // Ensure the directory exists before opening the DB — bun:sqlite
716
+ // won't create parent dirs for us. The constructor stays sync
717
+ // because callers use `new Cache(...)` directly; `mkdirSync` keeps
718
+ // that property without a subprocess fork.
719
+ mkdirSync(cacheDir, { recursive: true })
720
+ this.db = new Database(path.join(cacheDir, 'cache.db'), { create: true })
721
+ this.db.exec('PRAGMA journal_mode = WAL')
722
+ this.db.exec('PRAGMA synchronous = NORMAL')
723
+ this.db.exec('PRAGMA foreign_keys = ON')
724
+ // busy_timeout makes concurrent writers wait for the lock instead of
725
+ // failing immediately with SQLITE_BUSY. Two parallel `vx run`
726
+ // invocations in CI is a normal pattern; without this the second one
727
+ // crashes in recordRun().
728
+ this.db.exec('PRAGMA busy_timeout = 5000')
729
+
730
+ this.db.exec(`
731
+ CREATE TABLE IF NOT EXISTS schema_meta (
732
+ key TEXT PRIMARY KEY,
733
+ value TEXT NOT NULL
734
+ );
735
+ `)
736
+
737
+ // Schema-version gate runs BEFORE the rest of the schema lands so
738
+ // a column rename (e.g. v15's `sha256` → `content_hash`) actually
739
+ // takes effect on stale DBs. Pre-alpha: no migrations, just drop
740
+ // and recreate. Outputs on disk become orphans; they'll be ignored
741
+ // on next miss and reaped by `vx cache prune`.
742
+ const meta = this.db.prepare("SELECT value FROM schema_meta WHERE key = 'version'").get() as
743
+ | { value: string }
744
+ | undefined
745
+ if (meta && meta.value !== SCHEMA_VERSION) {
746
+ this.db.exec(
747
+ 'DROP TABLE IF EXISTS entries; DROP TABLE IF EXISTS runs; DROP TABLE IF EXISTS file_hashes; DROP TABLE IF EXISTS output_files; DROP TABLE IF EXISTS invocations; DROP TABLE IF EXISTS run_task_inputs; DROP TABLE IF EXISTS entry_inputs;',
748
+ )
749
+ this.db.prepare("UPDATE schema_meta SET value = ? WHERE key = 'version'").run(SCHEMA_VERSION)
750
+ } else if (!meta) {
751
+ this.db
752
+ .prepare("INSERT INTO schema_meta(key, value) VALUES ('version', ?)")
753
+ .run(SCHEMA_VERSION)
754
+ }
755
+
756
+ this.db.exec(`
757
+ -- stdout/stderr live in the <hash>.tar.zst artifact, not here
758
+ -- (v14+) — so they survive remote round-trips. The entries
759
+ -- table is the queryable index: command, exit_code, duration,
760
+ -- size, timestamps.
761
+ CREATE TABLE IF NOT EXISTS entries (
762
+ hash TEXT PRIMARY KEY,
763
+ project TEXT NOT NULL,
764
+ task TEXT NOT NULL,
765
+ command TEXT NOT NULL,
766
+ exit_code INTEGER NOT NULL,
767
+ duration_ms INTEGER NOT NULL,
768
+ size_bytes INTEGER NOT NULL,
769
+ stdout TEXT NOT NULL DEFAULT '',
770
+ created_at INTEGER NOT NULL,
771
+ accessed_at INTEGER NOT NULL
772
+ );
773
+ CREATE TABLE IF NOT EXISTS runs (
774
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
775
+ hash TEXT NOT NULL,
776
+ project TEXT NOT NULL,
777
+ task TEXT NOT NULL,
778
+ status TEXT NOT NULL,
779
+ exit_code INTEGER NOT NULL,
780
+ duration_ms INTEGER NOT NULL,
781
+ forward_args TEXT,
782
+ started_at INTEGER NOT NULL,
783
+ ended_at INTEGER NOT NULL,
784
+ -- v11 analytics columns. Nullable until the runner / orchestrator
785
+ -- PRs populate them. Storing them now means we can swap on the
786
+ -- producer side without touching the schema again.
787
+ run_id TEXT,
788
+ cpu_ms INTEGER,
789
+ peak_rss_bytes INTEGER,
790
+ wallclock_start_ns INTEGER,
791
+ wallclock_end_ns INTEGER,
792
+ cache_hit INTEGER,
793
+ -- v23: attempts a retried task took (>1); NULL for a once-run task.
794
+ -- The direct within-run flaky signal.
795
+ attempts INTEGER
796
+ );
797
+ CREATE INDEX IF NOT EXISTS runs_hash ON runs(hash);
798
+ CREATE INDEX IF NOT EXISTS runs_started_at ON runs(started_at);
799
+ CREATE INDEX IF NOT EXISTS runs_project ON runs(project, task);
800
+ CREATE INDEX IF NOT EXISTS runs_ended ON runs(ended_at);
801
+ CREATE INDEX IF NOT EXISTS runs_run_id ON runs(run_id);
802
+ -- Per-file (mtime, size, content_hash) cache. Lets Cache.key()
803
+ -- skip the content-hash on inputs whose stat hasn't changed
804
+ -- since the last run. Pure performance optimization; the stored
805
+ -- hash is the exact same one content-hashing would compute now,
806
+ -- so the cache key derivation is unchanged.
807
+ CREATE TABLE IF NOT EXISTS file_hashes (
808
+ path TEXT PRIMARY KEY,
809
+ mtime_ms INTEGER NOT NULL,
810
+ size_bytes INTEGER NOT NULL,
811
+ content_hash TEXT NOT NULL,
812
+ seen_at INTEGER NOT NULL
813
+ );
814
+ -- v16: per-output-file fingerprints, scoped by the cache entry
815
+ -- that produced them. Lets loadOutputFilesBatch(hashes) answer
816
+ -- "for entry X, what are its outputs supposed to look like?"
817
+ -- with one SELECT, so the orchestrator can stat-and-skip the
818
+ -- whole restore when the tree's already current.
819
+ --
820
+ -- ON DELETE CASCADE keeps these rows in sync with entries:
821
+ -- a cache prune that drops an entry sweeps its output rows
822
+ -- automatically.
823
+ CREATE TABLE IF NOT EXISTS output_files (
824
+ entry_hash TEXT NOT NULL,
825
+ path TEXT NOT NULL,
826
+ size_bytes INTEGER NOT NULL,
827
+ mode INTEGER NOT NULL,
828
+ mtime_ms INTEGER NOT NULL,
829
+ PRIMARY KEY (entry_hash, path),
830
+ FOREIGN KEY (entry_hash) REFERENCES entries(hash) ON DELETE CASCADE
831
+ );
832
+ -- v22 (Tier 3): one header row per vx-run invocation. The runs
833
+ -- table is per-task; this is the per-invocation record that
834
+ -- carries git/CI/host context, the command, tags, and run-level
835
+ -- counts so the dashboard never reconstructs a header with a
836
+ -- lossy GROUP BY over runs.
837
+ CREATE TABLE IF NOT EXISTS invocations (
838
+ run_id TEXT PRIMARY KEY,
839
+ command TEXT NOT NULL,
840
+ requested_tasks TEXT NOT NULL,
841
+ cache_policy TEXT NOT NULL,
842
+ concurrency INTEGER NOT NULL,
843
+ flow TEXT,
844
+ started_at INTEGER NOT NULL,
845
+ ended_at INTEGER NOT NULL,
846
+ total_duration_ms INTEGER NOT NULL,
847
+ task_count INTEGER NOT NULL,
848
+ failed_count INTEGER NOT NULL,
849
+ hit_count INTEGER NOT NULL,
850
+ hit_local_count INTEGER NOT NULL,
851
+ hit_remote_count INTEGER NOT NULL,
852
+ exit_ok INTEGER NOT NULL,
853
+ commit_sha TEXT,
854
+ branch TEXT,
855
+ dirty INTEGER,
856
+ ci INTEGER NOT NULL,
857
+ ci_provider TEXT,
858
+ host TEXT,
859
+ os TEXT,
860
+ arch TEXT,
861
+ vx_version TEXT NOT NULL,
862
+ tags TEXT NOT NULL DEFAULT '{}'
863
+ );
864
+ CREATE INDEX IF NOT EXISTS invocations_started ON invocations(started_at);
865
+ CREATE INDEX IF NOT EXISTS invocations_branch ON invocations(branch);
866
+ CREATE INDEX IF NOT EXISTS invocations_ci ON invocations(ci);
867
+ -- v22 (Tier 3): the input-fingerprint moat. One row per cache-key
868
+ -- component, keyed by the cache-ENTRY hash it belongs to (NOT a
869
+ -- run id). Written inside the entry-save transaction — only on a
870
+ -- miss/save, never on a hit — so a warm all-cache-hit run writes
871
+ -- nothing here (the warm path does zero extra work). The
872
+ -- why-did-this-re-run diff reads two entry hashes (this run's
873
+ -- runs.hash and the previous run's) and anti-joins their rows in
874
+ -- SQL over (kind,name,hash); a JSON blob would force an app-side
875
+ -- parse + compare on every probe. ON DELETE CASCADE keeps these
876
+ -- rows in sync with entries (a prune sweeps them automatically).
877
+ CREATE TABLE IF NOT EXISTS entry_inputs (
878
+ entry_hash TEXT NOT NULL,
879
+ kind TEXT NOT NULL,
880
+ name TEXT NOT NULL,
881
+ hash TEXT NOT NULL,
882
+ PRIMARY KEY (entry_hash, kind, name),
883
+ FOREIGN KEY (entry_hash) REFERENCES entries(hash) ON DELETE CASCADE
884
+ );
885
+ `)
886
+
887
+ this.insertEntry = this.db.prepare(`
888
+ INSERT INTO entries(hash, project, task, command, exit_code, duration_ms, size_bytes, stdout, created_at, accessed_at)
889
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
890
+ ON CONFLICT(hash) DO UPDATE SET
891
+ stdout = excluded.stdout,
892
+ project = excluded.project,
893
+ task = excluded.task,
894
+ command = excluded.command,
895
+ exit_code = excluded.exit_code,
896
+ duration_ms = excluded.duration_ms,
897
+ size_bytes = excluded.size_bytes,
898
+ accessed_at = excluded.accessed_at
899
+ `)
900
+ this.selectEntry = this.db.prepare('SELECT * FROM entries WHERE hash = ?')
901
+ this.bumpAccessed = this.db.prepare('UPDATE entries SET accessed_at = ? WHERE hash = ?')
902
+ this.insertRun = this.db.prepare(`
903
+ INSERT INTO runs(
904
+ hash, project, task, status, exit_code, duration_ms, forward_args,
905
+ started_at, ended_at,
906
+ run_id, cpu_ms, peak_rss_bytes, wallclock_start_ns, wallclock_end_ns,
907
+ cache_hit, attempts
908
+ )
909
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
910
+ `)
911
+ this.insertInvocation = this.db.prepare(`
912
+ INSERT INTO invocations(
913
+ run_id, command, requested_tasks, cache_policy, concurrency, flow,
914
+ started_at, ended_at, total_duration_ms,
915
+ task_count, failed_count, hit_count, hit_local_count, hit_remote_count,
916
+ exit_ok,
917
+ commit_sha, branch, dirty, ci, ci_provider,
918
+ host, os, arch, vx_version, tags
919
+ )
920
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
921
+ ON CONFLICT(run_id) DO NOTHING
922
+ `)
923
+ // INSERT OR IGNORE: re-saving the same hash (idempotent ingest /
924
+ // overlapping concurrent saves) leaves the existing rows untouched —
925
+ // identical inputs derive the identical hash, so the rows are too.
926
+ this.insertEntryInput = this.db.prepare(`
927
+ INSERT OR IGNORE INTO entry_inputs(entry_hash, kind, name, hash)
928
+ VALUES (?, ?, ?, ?)
929
+ `)
930
+ this.selectFileHash = this.db.prepare(
931
+ 'SELECT mtime_ms, size_bytes, content_hash FROM file_hashes WHERE path = ?',
932
+ )
933
+ this.upsertFileHash = this.db.prepare(`
934
+ INSERT INTO file_hashes(path, mtime_ms, size_bytes, content_hash, seen_at)
935
+ VALUES (?, ?, ?, ?, ?)
936
+ ON CONFLICT(path) DO UPDATE SET
937
+ mtime_ms = excluded.mtime_ms,
938
+ size_bytes = excluded.size_bytes,
939
+ content_hash = excluded.content_hash,
940
+ seen_at = excluded.seen_at
941
+ `)
942
+ this.insertOutputFile = this.db.prepare(`
943
+ INSERT INTO output_files(entry_hash, path, size_bytes, mode, mtime_ms)
944
+ VALUES (?, ?, ?, ?, ?)
945
+ ON CONFLICT(entry_hash, path) DO UPDATE SET
946
+ size_bytes = excluded.size_bytes,
947
+ mode = excluded.mode,
948
+ mtime_ms = excluded.mtime_ms
949
+ `)
950
+ this.deleteOutputFiles = this.db.prepare('DELETE FROM output_files WHERE entry_hash = ?')
951
+ }
952
+
953
+ /**
954
+ * Content-hash a file (as a git blob OID, v20) with an mtime+size
955
+ * fast path. If the `file_hashes` table has a row for `path` whose
956
+ * `(mtime_ms, size_bytes)` match the current stat, we reuse the
957
+ * stored content_hash (a memory + SQLite lookup, no disk read).
958
+ * Otherwise we read + hash + upsert.
959
+ *
960
+ * The OID is byte-identical to what `git hash-object` (and the git
961
+ * index) computes for the same content, so this fallback and the
962
+ * `CacheKeyInput.fileHashes` index-OID fast path never diverge —
963
+ * a file's key contribution can't flip across dirty↔clean
964
+ * transitions.
965
+ */
966
+ async hashFile(filePath: string): Promise<string> {
967
+ // statSync intentional: a single stat is ~1.6µs (Bun 1.3); the
968
+ // async-stat equivalent adds ~75µs of Promise machinery per call.
969
+ // Promise.all over the batched callers (key derivation) gives no
970
+ // I/O parallelism benefit because the stat is faster than the
971
+ // threadpool dispatch overhead.
972
+ let st
973
+ try {
974
+ st = statSync(filePath)
975
+ } catch {
976
+ // Caller is responsible for skipping files that don't exist;
977
+ // fall through to the content-hash path which will throw with
978
+ // a more useful error.
979
+ return await this.hashFileFromDisk(filePath)
980
+ }
981
+ const mtimeMs = Math.floor(st.mtimeMs)
982
+ const size = st.size
983
+ const row = this.selectFileHash.get(filePath) as
984
+ | { mtime_ms: number; size_bytes: number; content_hash: string }
985
+ | undefined
986
+ if (row && row.mtime_ms === mtimeMs && row.size_bytes === size) {
987
+ return row.content_hash
988
+ }
989
+ const ch = await this.hashFileFromDisk(filePath)
990
+ this.upsertFileHash.run(filePath, mtimeMs, size, ch, Date.now())
991
+ return ch
992
+ }
993
+
994
+ /**
995
+ * Git blob OID of the file's bytes:
996
+ * `hex(HASH("blob " + byteLength + "\0" + content))`, where HASH is
997
+ * the repo's object format. Same value `git hash-object` prints and
998
+ * the same value the index stores. Computed in-process — no git
999
+ * spawn per file.
1000
+ */
1001
+ private async hashFileFromDisk(filePath: string): Promise<string> {
1002
+ const bytes = await Bun.file(filePath).bytes()
1003
+ const hasher = new Bun.CryptoHasher(this.objectFormat ?? this.detectObjectFormat(filePath))
1004
+ hasher.update(`blob ${bytes.byteLength}\0`)
1005
+ hasher.update(bytes)
1006
+ return hasher.digest('hex')
1007
+ }
1008
+
1009
+ /**
1010
+ * Repo object format — sha1 unless the repo was created with
1011
+ * `--object-format=sha256`. One `git rev-parse` spawn per Cache
1012
+ * lifetime, and only when at least one file misses the mtime+size
1013
+ * memo. Outside a repo (unit fixtures) we default to sha1, which is
1014
+ * still a deterministic blob-OID domain.
1015
+ */
1016
+ private detectObjectFormat(nearPath: string): 'sha1' | 'sha256' {
1017
+ let detected: 'sha1' | 'sha256' = 'sha1'
1018
+ try {
1019
+ const proc = Bun.spawnSync({
1020
+ cmd: ['git', 'rev-parse', '--show-object-format'],
1021
+ cwd: path.dirname(nearPath),
1022
+ stdout: 'pipe',
1023
+ stderr: 'pipe',
1024
+ })
1025
+ if (proc.exitCode === 0 && new TextDecoder().decode(proc.stdout).trim() === 'sha256') {
1026
+ detected = 'sha256'
1027
+ }
1028
+ } catch {
1029
+ // git unavailable → sha1 default keeps hashing deterministic.
1030
+ }
1031
+ this.objectFormat = detected
1032
+ return detected
1033
+ }
1034
+
1035
+ async key(input: CacheKeyInput): Promise<string> {
1036
+ // Seed-chained xxHash3: each step folds one field into the
1037
+ // running digest via `xxh3(part, prevDigest)`. Equivalent to the
1038
+ // old CryptoHasher.update() pattern, no intermediate buffer.
1039
+ // Field-order matters; each line is prefixed with its label so
1040
+ // adjacent fields can't collide via concat.
1041
+ // Optional capture sink — see CacheKeyInput.captureInto. Pure
1042
+ // side-channel; every push below mirrors the fold immediately
1043
+ // above/beside it so the recorded set can't drift from the key.
1044
+ const cap = input.captureInto
1045
+ let h = xxh3(CACHE_VERSION)
1046
+ h = xxh3(`task:${input.taskId}`, h)
1047
+ h = xxh3(`workspace:${input.workspaceFingerprint}`, h)
1048
+ h = xxh3(`pkg:${input.projectPackageJsonHash}`, h)
1049
+ h = xxh3(`config:${input.taskConfigHash}`, h)
1050
+ if (cap) {
1051
+ cap.push({ kind: 'workspace', name: 'fingerprint', hash: input.workspaceFingerprint })
1052
+ cap.push({ kind: 'package', name: 'package.json', hash: input.projectPackageJsonHash })
1053
+ cap.push({ kind: 'config', name: 'config', hash: input.taskConfigHash })
1054
+ }
1055
+
1056
+ // NOTE on `cap` (entry_inputs capture): the `hash` field stores a
1057
+ // DIGEST of each component, never the raw value. For value-bearing kinds
1058
+ // (env, runtime, ws-runtime, forward) the payload is a secret or
1059
+ // sensitive string (API keys via cache.inputs.env, runtime-command
1060
+ // output, args after `--`), so we push `xxh3hex(v)` — the diff consumer
1061
+ // only needs to know whether a component CHANGED, which a digest
1062
+ // preserves losslessly. cache.db must never hold plaintext secrets at
1063
+ // rest. The cache KEY (`h`) folds the plaintext separately below and is
1064
+ // unaffected.
1065
+ const forwarded = input.forwardArgs ?? []
1066
+ h = xxh3(`forward-args:${forwarded.length}`, h)
1067
+ for (const a of forwarded) h = xxh3(a, h)
1068
+ if (cap && forwarded.length > 0) {
1069
+ cap.push({ kind: 'forward', name: 'argv', hash: xxh3hex(JSON.stringify(forwarded)) })
1070
+ }
1071
+
1072
+ h = xxh3(`env-values:${input.envValues.length}`, h)
1073
+ // \0 delimiter, not `=`: names and values may themselves contain
1074
+ // `=`, and `A` + `B=C` must never fold the same bytes as `A=B` + `C`.
1075
+ for (const [n, v] of input.envValues) h = xxh3(`${n}\0${v}`, h)
1076
+ if (cap)
1077
+ for (const [n, v] of input.envValues) cap.push({ kind: 'env', name: n, hash: xxh3hex(v) })
1078
+
1079
+ const runtimeValues = input.runtimeValues ?? []
1080
+ h = xxh3(`runtime-values:${runtimeValues.length}`, h)
1081
+ for (const [c, o] of runtimeValues) h = xxh3(`${c}\0${o}`, h)
1082
+ if (cap)
1083
+ for (const [c, o] of runtimeValues) cap.push({ kind: 'runtime', name: c, hash: xxh3hex(o) })
1084
+
1085
+ const wsRuntimeValues = input.workspaceRuntimeValues ?? []
1086
+ h = xxh3(`ws-runtime-values:${wsRuntimeValues.length}`, h)
1087
+ for (const [c, o] of wsRuntimeValues) h = xxh3(`${c}\0${o}`, h)
1088
+ if (cap)
1089
+ for (const [c, o] of wsRuntimeValues)
1090
+ cap.push({ kind: 'ws-runtime', name: c, hash: xxh3hex(o) })
1091
+
1092
+ const upstream = [...input.upstreamHashes].sort()
1093
+ h = xxh3(`upstream:${upstream.length}`, h)
1094
+ for (const u of upstream) h = xxh3(u, h)
1095
+ if (cap) {
1096
+ for (const u of upstream) {
1097
+ cap.push({ kind: 'upstream', name: input.upstreamIds?.get(u) ?? u, hash: u })
1098
+ }
1099
+ }
1100
+
1101
+ const sortedInputs = [...input.inputFiles].sort()
1102
+ h = xxh3(`inputs:${sortedInputs.length}`, h)
1103
+ // Per-file hash source, in preference order: the caller-supplied
1104
+ // index-OID map (clean tracked files — zero I/O), then hashFile's
1105
+ // mtime+size memo (no read), then a full in-process blob-OID
1106
+ // computation. All three produce identical bytes for identical
1107
+ // content. The fold order is locked to `sortedInputs` so results
1108
+ // are stable across runs.
1109
+ const fileHashes = await Promise.all(
1110
+ sortedInputs.map((f) => input.fileHashes?.get(f) ?? this.hashFile(f)),
1111
+ )
1112
+ for (let i = 0; i < sortedInputs.length; i++) {
1113
+ const file = sortedInputs[i]!
1114
+ const rel = relPosix(input.workspaceRoot, file)
1115
+ const oid = fileHashes[i]!
1116
+ h = xxh3(`${rel}\0${oid}`, h)
1117
+ // The OID is already awaited — file capture is zero extra I/O.
1118
+ if (cap) cap.push({ kind: 'file', name: rel, hash: oid })
1119
+ }
1120
+
1121
+ return h.toString(16).padStart(16, '0')
1122
+ }
1123
+
1124
+ // ctx is accepted but ignored — the local layer reads metadata from
1125
+ // the entries row. It's part of the contract so LayeredCache can
1126
+ // route metadata to `ingest()` on remote-hit without a separate API.
1127
+ async get(hash: string, _ctx?: CacheGetContext): Promise<CacheEntry | null> {
1128
+ // Local reads disabled (e.g. `--force` / `--cache=local:w`): report a
1129
+ // miss so the task re-executes. The artifact + index are untouched.
1130
+ if (!this.read) return null
1131
+ const row = this.selectEntry.get(hash) as EntryRow | undefined
1132
+ if (!row) return null
1133
+
1134
+ // Verify the tar artifact actually exists. The DB and the
1135
+ // filesystem can drift if someone manually deletes the cache dir.
1136
+ if (!(await Bun.file(this.tarPath(hash)).exists())) return null
1137
+
1138
+ // Deferred: per-hit UPDATEs cost ~60 ms across 2000+ probes on a
1139
+ // full-cache run. Hashes are collected and flushed as ONE batched
1140
+ // UPDATE by flushAccessed() (called from prune/stats/close), which
1141
+ // is when accessed_at is actually read.
1142
+ this.touched.add(hash)
1143
+
1144
+ // Pure SQL: outputFiles come from the output_files rows and
1145
+ // stdout from the entries row. The artifact is NOT touched here —
1146
+ // decompressing it made hit cost scale with artifact size (73 ms
1147
+ // for four up-to-date hits on ~70 MB binaries). restoreOutputs
1148
+ // reads the artifact itself, only when extraction actually runs.
1149
+ const fileRows = this.loadOutputFilesBatch([hash]).get(hash) ?? []
1150
+
1151
+ return {
1152
+ hash: row.hash,
1153
+ taskId: `${row.project}#${row.task}`,
1154
+ command: row.command,
1155
+ exitCode: row.exit_code,
1156
+ durationMs: row.duration_ms,
1157
+ outputFiles: fileRows.map((r) => r.path),
1158
+ stdout: row.stdout,
1159
+ storedAt: new Date(row.created_at).toISOString(),
1160
+ source: 'local',
1161
+ }
1162
+ }
1163
+
1164
+ // Existence probe: SQL row + artifact-on-disk check, no byte reads
1165
+ // and no accessed_at bump (the plan path must stay read-only).
1166
+ async has(hash: string): Promise<'local' | 'remote' | null> {
1167
+ if (!this.read) return null
1168
+ const row = this.selectEntry.get(hash) as EntryRow | undefined
1169
+ if (!row) return null
1170
+ return (await Bun.file(this.tarPath(hash)).exists()) ? 'local' : null
1171
+ }
1172
+
1173
+ // Local cache has no slower layer to warm from — prefetch is a no-op.
1174
+ // The contract still resolves false so callers can treat every layer
1175
+ // uniformly (LayeredCache overrides with the real remote pull).
1176
+ async prefetch(_hash: string, _ctx?: CacheGetContext): Promise<boolean> {
1177
+ return false
1178
+ }
1179
+
1180
+ loadOutputFilesBatch(hashes: readonly string[]): Map<string, OutputFileRow[]> {
1181
+ const out = new Map<string, OutputFileRow[]>()
1182
+ if (hashes.length === 0) return out
1183
+ // Inline placeholders for an IN-list — bun:sqlite doesn't ship
1184
+ // rarray, but `IN (?, ?, …)` with N≤~999 is fast and avoids per-
1185
+ // hash select.get() overhead. `db.query` (not `db.prepare`) caches the
1186
+ // compiled statement keyed by the SQL text — so the dominant single-hash
1187
+ // warm-hit path (called up to 3× per hit) reuses one statement instead of
1188
+ // recompiling on every call.
1189
+ const placeholders = hashes.map(() => '?').join(',')
1190
+ const stmt = this.db.query(
1191
+ `SELECT entry_hash, path, size_bytes, mode, mtime_ms FROM output_files WHERE entry_hash IN (${placeholders})`,
1192
+ )
1193
+ const rows = stmt.all(...(hashes as readonly SQLQueryBindings[])) as Array<{
1194
+ entry_hash: string
1195
+ path: string
1196
+ size_bytes: number
1197
+ mode: number
1198
+ mtime_ms: number
1199
+ }>
1200
+ for (const r of rows) {
1201
+ let list = out.get(r.entry_hash)
1202
+ if (!list) {
1203
+ list = []
1204
+ out.set(r.entry_hash, list)
1205
+ }
1206
+ list.push({ path: r.path, size: r.size_bytes, mode: r.mode, mtimeMs: r.mtime_ms })
1207
+ }
1208
+ return out
1209
+ }
1210
+
1211
+ async isOutputsCurrent(projectDir: string, expected: readonly OutputFileRow[]): Promise<boolean> {
1212
+ // Empty manifest case (a task produced no outputs) → trivially
1213
+ // current; the on-disk tree under projectDir is whatever it was,
1214
+ // and nothing was supposed to land there.
1215
+ if (expected.length === 0) return true
1216
+ const results = await Promise.all(
1217
+ expected.map(async (e) => {
1218
+ try {
1219
+ const s = await stat(path.join(projectDir, e.path))
1220
+ return (
1221
+ s.size === e.size &&
1222
+ (s.mode & 0o777) === (e.mode & 0o777) &&
1223
+ // MILLISECOND comparison (sub-ms tolerance for the float
1224
+ // round-trip through utimes). Save rows carry stat-ms and
1225
+ // restoreOutputs re-syncs restored files to the row value,
1226
+ // so equality holds exactly in steady state; legacy
1227
+ // second-precision rows converge on their first restore.
1228
+ // Residual blind spot: a same-size edit landing in the SAME
1229
+ // millisecond as the recorded write, or a deliberately
1230
+ // forged mtime (touch -r) — the trade every mtime-based
1231
+ // skip check accepts.
1232
+ Math.abs(s.mtimeMs - e.mtimeMs) < 1
1233
+ )
1234
+ } catch {
1235
+ return false
1236
+ }
1237
+ }),
1238
+ )
1239
+ return results.every(Boolean)
1240
+ }
1241
+
1242
+ outputsPath(hash: string): string {
1243
+ return this.tarPath(hash)
1244
+ }
1245
+
1246
+ async restoreOutputs(hash: string, projectDir: string, workspaceRoot?: string): Promise<void> {
1247
+ // In-process tar extraction — no fork+exec on the hot path
1248
+ // (~5-10ms reclaimed per hit vs the prior subprocess `tar -xf`).
1249
+ //
1250
+ // The "tree is already current" skip-everything check happens at
1251
+ // the orchestrator level (using the batched `output_files` map),
1252
+ // BEFORE this method runs. By the time we're here we've committed
1253
+ // to a fresh extract.
1254
+ //
1255
+ // `stdout` / `stderr` entries in the archive are ignored on this
1256
+ // path — they're surfaced via `get()` for the orchestrator to
1257
+ // replay through the logger.
1258
+ const src = this.tarPath(hash)
1259
+ if (!(await Bun.file(src).exists())) return
1260
+ const compressed = await Bun.file(src).bytes()
1261
+ // Local artifact (already validated at ingest): trusted, so a missing
1262
+ // declared size is allowed; the oversize ceiling still applies.
1263
+ const tarBytes = await zstdDecompressBounded(compressed, hash, true)
1264
+
1265
+ const headers = parseTarHeaders(tarBytes)
1266
+ if (
1267
+ !headers.some(
1268
+ (h) => h.name.startsWith('outputs/') || h.name.startsWith(WORKSPACE_OUTPUT_PREFIX),
1269
+ )
1270
+ ) {
1271
+ return
1272
+ }
1273
+
1274
+ await extractOutputs(tarBytes, projectDir, workspaceRoot)
1275
+
1276
+ // Re-sync restored files' mtimes to the manifest rows (extract set
1277
+ // them from the tar's SECOND-precision headers). After this, disk
1278
+ // matches the rows at millisecond precision, so isOutputsCurrent
1279
+ // compares exactly — and any later edit moves the mtime off the
1280
+ // recorded historical value and is detected.
1281
+ const rows = this.loadOutputFilesBatch([hash]).get(hash)
1282
+ if (rows !== undefined) {
1283
+ await Promise.all(
1284
+ rows.map(async (r) => {
1285
+ try {
1286
+ const abs = r.path.startsWith(WORKSPACE_OUTPUT_PREFIX)
1287
+ ? path.join(workspaceRoot ?? projectDir, r.path.slice(WORKSPACE_OUTPUT_PREFIX.length))
1288
+ : path.join(projectDir, r.path)
1289
+ const t = r.mtimeMs / 1000
1290
+ await utimes(abs, t, t)
1291
+ } catch {
1292
+ // File absent from this extract (or unwritable) — the next
1293
+ // probe simply restores again; never fail a hit over mtimes.
1294
+ }
1295
+ }),
1296
+ )
1297
+ }
1298
+ }
1299
+
1300
+ async save(args: {
1301
+ hash: string
1302
+ entry: Omit<CacheEntry, 'hash' | 'storedAt' | 'outputFiles'>
1303
+ projectDir: string
1304
+ outputFiles: string[]
1305
+ workspaceOutputFiles?: string[]
1306
+ workspaceRoot?: string
1307
+ inputComponents?: readonly TaskInputRow[]
1308
+ }): Promise<void> {
1309
+ // Layout (v17, extended additively for workspaceFiles): one
1310
+ // `<hash>.tar.zst` per entry. Tar carries ONLY the things you'd
1311
+ // want to re-materialize on a cache hit:
1312
+ //
1313
+ // stdout — captured stdout (ALWAYS present, may be empty)
1314
+ // outputs/<rel> — declared output files (omitted when none)
1315
+ // workspace-outputs/<rel> — declared outputs.workspaceFiles,
1316
+ // rel to the WORKSPACE ROOT (omitted when none)
1317
+ //
1318
+ // Metadata (command, exitCode, durationMs, storedAt) lives in
1319
+ // SQLite, not the artifact. Remote-hit ingestion takes metadata
1320
+ // through `ingest()` arguments — the artifact stays clean bytes.
1321
+ //
1322
+ // Local writes disabled (e.g. `--cache=local:,remote:rw`): produce
1323
+ // no `<hash>.tar.zst` and no index row. The LayeredCache wrapping us
1324
+ // still uploads to remote — it calls `packArtifact` itself to get
1325
+ // the bytes, since there's no local artifact to read off disk.
1326
+ if (!this.write) return
1327
+ const compressed = await this.packArtifact(args)
1328
+ await this.writeArtifactAndIndex(
1329
+ args.hash,
1330
+ compressed,
1331
+ {
1332
+ taskId: args.entry.taskId,
1333
+ command: args.entry.command,
1334
+ durationMs: args.entry.durationMs,
1335
+ ...(args.inputComponents !== undefined ? { inputComponents: args.inputComponents } : {}),
1336
+ },
1337
+ // Save-path rows get millisecond mtimes from a fresh stat (tar
1338
+ // headers carry only seconds) — see the row loop for the guard.
1339
+ {
1340
+ projectDir: args.projectDir,
1341
+ ...(args.workspaceRoot !== undefined ? { workspaceRoot: args.workspaceRoot } : {}),
1342
+ },
1343
+ )
1344
+ }
1345
+
1346
+ /**
1347
+ * Whether this local layer persists artifacts. The LayeredCache reads
1348
+ * it to decide between uploading the on-disk artifact (write enabled)
1349
+ * vs. packing bytes in memory (write disabled) for a remote upload.
1350
+ */
1351
+ get localWritesEnabled(): boolean {
1352
+ return this.write
1353
+ }
1354
+
1355
+ /**
1356
+ * Pack the save args into tar.zst bytes WITHOUT touching disk or the
1357
+ * index. Used by the LayeredCache when local writes are disabled but
1358
+ * a remote upload still needs the artifact bytes.
1359
+ */
1360
+ async packArtifactBytes(args: SaveArgs): Promise<Uint8Array> {
1361
+ return await this.packArtifact(args)
1362
+ }
1363
+
1364
+ async ingest(hash: string, compressed: Uint8Array, meta: IngestMeta): Promise<void> {
1365
+ await this.writeArtifactAndIndex(hash, compressed, meta)
1366
+ }
1367
+
1368
+ /**
1369
+ * Stage stdout + outputs, tar them, zstd-compress, return the bytes.
1370
+ * No disk write to the final cache path — that's the index step's
1371
+ * job. Pure transform, so `ingest()` can skip this and just hand its
1372
+ * remote-supplied bytes straight to `writeArtifactAndIndex`.
1373
+ */
1374
+ private async packArtifact(args: {
1375
+ hash: string
1376
+ entry: Omit<CacheEntry, 'hash' | 'storedAt' | 'outputFiles'>
1377
+ projectDir: string
1378
+ outputFiles: string[]
1379
+ workspaceOutputFiles?: string[]
1380
+ workspaceRoot?: string
1381
+ }): Promise<Uint8Array> {
1382
+ const wsOutputFiles = args.workspaceOutputFiles ?? []
1383
+ const stage = await mkdtemp(path.join(os.tmpdir(), 'vx-save-'))
1384
+ try {
1385
+ if (args.outputFiles.length > 0) {
1386
+ const stageOutputs = path.join(stage, 'outputs')
1387
+ await Promise.all(
1388
+ args.outputFiles.map(async (f) => {
1389
+ const rel = path.relative(args.projectDir, f)
1390
+ const dest = path.join(stageOutputs, rel)
1391
+ // Bun.write creates parent dirs as needed.
1392
+ await Bun.write(dest, Bun.file(f))
1393
+ }),
1394
+ )
1395
+ }
1396
+ if (wsOutputFiles.length > 0) {
1397
+ // Caller passes workspaceRoot whenever workspaceOutputFiles is
1398
+ // non-empty; the rels are root-anchored by construction.
1399
+ const stageWs = path.join(stage, 'workspace-outputs')
1400
+ await Promise.all(
1401
+ wsOutputFiles.map(async (f) => {
1402
+ const rel = path.relative(args.workspaceRoot!, f)
1403
+ await Bun.write(path.join(stageWs, rel), Bun.file(f))
1404
+ }),
1405
+ )
1406
+ }
1407
+ // stdout is ALWAYS present in the artifact, even if empty, so the
1408
+ // archive layout is predictable: a successful read finds `stdout`
1409
+ // and zero-or-more `outputs/<rel>` / `workspace-outputs/<rel>`
1410
+ // entries.
1411
+ await Bun.write(path.join(stage, 'stdout'), args.entry.stdout ?? '')
1412
+
1413
+ const topLevel: string[] = ['stdout']
1414
+ if (wsOutputFiles.length > 0) topLevel.unshift('workspace-outputs')
1415
+ if (args.outputFiles.length > 0) topLevel.unshift('outputs')
1416
+
1417
+ // `--format=ustar` forces strict POSIX ustar — no PAX extended-
1418
+ // header records. BSD tar (macOS default) emits PAX per entry
1419
+ // by default for xattrs / mtime-nanos; those records would
1420
+ // otherwise show up as junk `PaxHeaders/<name>` entries in our
1421
+ // restored trees. GNU tar also accepts the flag (no-op on its
1422
+ // side). Names > 100 chars still work via ustar's prefix+name
1423
+ // (255 chars) or GNU longname fallback if the tar binary
1424
+ // chooses to emit one.
1425
+ const proc = Bun.spawn(['tar', '--format=ustar', '-cf', '-', '-C', stage, ...topLevel], {
1426
+ stdout: 'pipe',
1427
+ stderr: 'pipe',
1428
+ // COPYFILE_DISABLE blocks Apple's copyfile() from attaching
1429
+ // xattrs to staged files via inherited child copies; tar then
1430
+ // has nothing to emit AppleDouble for.
1431
+ env: { ...process.env, COPYFILE_DISABLE: '1' },
1432
+ })
1433
+ const [tarBytes, stderrText] = await Promise.all([
1434
+ new Response(proc.stdout).bytes(),
1435
+ new Response(proc.stderr).text(),
1436
+ ])
1437
+ await proc.exited
1438
+ if (proc.exitCode !== 0) {
1439
+ throw new Error(`save: tar exited ${proc.exitCode}: ${stderrText.trim()}`)
1440
+ }
1441
+ return await Bun.zstdCompress(tarBytes)
1442
+ } finally {
1443
+ await rm(stage, { recursive: true, force: true })
1444
+ }
1445
+ }
1446
+
1447
+ /**
1448
+ * Atomically write `compressed` to `<hash>.tar.zst` and (re)build
1449
+ * the entries + output_files SQL rows from the tar headers. Shared
1450
+ * by `save()` (we just packed the bytes) and `ingest()` (we got
1451
+ * them from the remote layer).
1452
+ */
1453
+ private async writeArtifactAndIndex(
1454
+ hash: string,
1455
+ compressed: Uint8Array,
1456
+ meta: IngestMeta,
1457
+ statBase?: { projectDir: string; workspaceRoot?: string },
1458
+ ): Promise<void> {
1459
+ // Validate BEFORE anything touches the final path. `ingest()` feeds
1460
+ // us network bytes; a truncated/garbage body that went live first
1461
+ // would leave a corrupt `<hash>.tar.zst` behind (with no SQL row,
1462
+ // since the decompress throw aborted indexing) for every later
1463
+ // reader to trip over. Decompress + parse also produce the
1464
+ // `output_files` rows: same headers extractOutputs will see on
1465
+ // restore, so the size/mode/mtime fingerprint we store matches
1466
+ // what isOutputsCurrent will compare against post-restore.
1467
+ let tarBytes: Uint8Array
1468
+ try {
1469
+ // ingest() is the UNTRUSTED boundary — `compressed` is bytes just
1470
+ // pulled from a remote. Refuse a bomb (declared or sizeless) before it
1471
+ // can expand into memory.
1472
+ tarBytes = await zstdDecompressBounded(compressed, hash, false)
1473
+ } catch (err) {
1474
+ if (err instanceof CorruptArtifactError) throw err
1475
+ throw new CorruptArtifactError(hash, 'zstd decompression failed', err)
1476
+ }
1477
+ const headers = parseTarHeaders(tarBytes)
1478
+ // v17 invariant: every artifact carries a `stdout` entry. Its
1479
+ // absence means the bytes decompressed but aren't a vx artifact.
1480
+ if (!headers.some((h) => h.name === 'stdout' && !h.isDir)) {
1481
+ throw new CorruptArtifactError(hash, 'missing stdout entry')
1482
+ }
1483
+
1484
+ const finalPath = this.tarPath(hash)
1485
+ // tmp suffix mixes pid + hrtime + a random hex chunk so two saves
1486
+ // of the same hash from the same process (or from two forked
1487
+ // workers that happen to share a wall-clock ms) don't pick the
1488
+ // same tmp filename and race on the rename.
1489
+ const tmpPath = `${finalPath}.tmp-${process.pid}-${process.hrtime.bigint()}-${Math.random().toString(36).slice(2, 10)}`
1490
+ await mkdir(this.cacheDir, { recursive: true })
1491
+ await Bun.write(tmpPath, compressed)
1492
+ // POSIX rename atomically REPLACES the destination if it exists,
1493
+ // so we don't need a pre-rm. The pre-rm was actively harmful —
1494
+ // it opened a race window where writer B could delete writer A's
1495
+ // just-renamed file BEFORE A's subsequent stat, producing a
1496
+ // spurious ENOENT. The rename itself preserves the "either-or"
1497
+ // semantics for concurrent readers.
1498
+ await rename(tmpPath, finalPath)
1499
+
1500
+ const totalBytes = compressed.byteLength
1501
+ const outputFileRows: Array<[string, number, number, number]> = []
1502
+ // Per-output-file fingerprint rows feed the skip-restore check.
1503
+ // Row paths: project entries store the bare rel (`outputs/`
1504
+ // stripped); workspace entries keep the full
1505
+ // `workspace-outputs/<rel>` name as the namespace discriminator.
1506
+ for (const h of headers) {
1507
+ if (h.isDir) continue
1508
+ let rowPath: string | null = null
1509
+ if (h.name.startsWith('outputs/')) {
1510
+ const rel = h.name.slice('outputs/'.length)
1511
+ if (rel.length > 0) rowPath = rel
1512
+ } else if (h.name.startsWith(WORKSPACE_OUTPUT_PREFIX)) {
1513
+ if (h.name.length > WORKSPACE_OUTPUT_PREFIX.length) rowPath = h.name
1514
+ }
1515
+ if (rowPath === null) continue
1516
+ let mtimeMs = Math.floor(h.mtimeMs)
1517
+ // Save path only (ingest has no filesystem to consult): tar
1518
+ // headers truncate mtimes to SECONDS, which is what made the
1519
+ // skip-restore check blind to same-second same-size edits. A
1520
+ // fresh stat refines the row to millisecond precision — but only
1521
+ // when the size still matches the packed header, so the row
1522
+ // always describes the ARTIFACT's bytes; a file mutated between
1523
+ // pack and stat keeps the coarse header time (degraded, not
1524
+ // wrong). restoreOutputs re-syncs disk mtimes to these rows.
1525
+ if (statBase !== undefined) {
1526
+ try {
1527
+ const abs = rowPath.startsWith(WORKSPACE_OUTPUT_PREFIX)
1528
+ ? path.join(
1529
+ statBase.workspaceRoot ?? statBase.projectDir,
1530
+ rowPath.slice(WORKSPACE_OUTPUT_PREFIX.length),
1531
+ )
1532
+ : path.join(statBase.projectDir, rowPath)
1533
+ const st = await stat(abs)
1534
+ if (st.size === h.size) mtimeMs = Math.floor(st.mtimeMs)
1535
+ } catch {
1536
+ // stat failed — keep the header-seconds fallback.
1537
+ }
1538
+ }
1539
+ outputFileRows.push([rowPath, h.size, h.mode & 0o777, mtimeMs])
1540
+ }
1541
+ const stdoutText = readTarText(tarBytes, headers, 'stdout')
1542
+
1543
+ const [project, task] = splitTaskId(meta.taskId)
1544
+ const now = Date.now()
1545
+
1546
+ // One transaction for the entries row + every output_files row +
1547
+ // the Tier-3 entry_inputs fingerprint rows. One fsync regardless of
1548
+ // row count. The fingerprint rows ride this save-time transaction
1549
+ // (not the per-run path) so a warm all-cache-hit run — which never
1550
+ // saves — writes none of them.
1551
+ const insertEntry = this.insertEntry
1552
+ const insertOutputFile = this.insertOutputFile
1553
+ const deleteOutputFiles = this.deleteOutputFiles
1554
+ const insertEntryInput = this.insertEntryInput
1555
+ const inputComponents = meta.inputComponents
1556
+ const tx = this.db.transaction(() => {
1557
+ insertEntry.run(
1558
+ hash,
1559
+ project,
1560
+ task,
1561
+ meta.command,
1562
+ 0, // exitCode: we never cache failures
1563
+ meta.durationMs,
1564
+ totalBytes,
1565
+ stdoutText,
1566
+ now,
1567
+ now,
1568
+ )
1569
+ // Replace the entry's existing output_files rows (an UPDATE on
1570
+ // the same hash should refresh, not append).
1571
+ deleteOutputFiles.run(hash)
1572
+ for (const [rel, size, mode, mtime] of outputFileRows) {
1573
+ insertOutputFile.run(hash, rel, size, mode, mtime)
1574
+ }
1575
+ // INSERT OR IGNORE: identical inputs derive this same hash, so a
1576
+ // re-save's rows are identical — keep the first set, skip the rest.
1577
+ if (inputComponents !== undefined) {
1578
+ for (const c of inputComponents) {
1579
+ insertEntryInput.run(hash, c.kind, c.name, c.hash)
1580
+ }
1581
+ }
1582
+ })
1583
+ tx()
1584
+ }
1585
+
1586
+ /** Apply the deferred accessed_at bumps in one statement. */
1587
+ private flushAccessed(): void {
1588
+ if (this.touched.size === 0) return
1589
+ const hashes = [...this.touched]
1590
+ this.touched.clear()
1591
+ const now = Date.now()
1592
+ // Chunked: SQLite's bound-parameter ceiling is 32k on modern
1593
+ // builds, but 900 keeps us safe on any build at negligible cost.
1594
+ for (let i = 0; i < hashes.length; i += 900) {
1595
+ const chunk = hashes.slice(i, i + 900)
1596
+ const placeholders = chunk.map(() => '?').join(',')
1597
+ this.db
1598
+ .prepare(`UPDATE entries SET accessed_at = ? WHERE hash IN (${placeholders})`)
1599
+ .run(now, ...chunk)
1600
+ }
1601
+ }
1602
+
1603
+ /**
1604
+ * Raw `bun:sqlite` Database handle. Exposed for subsystems that
1605
+ * need to issue their own queries (LocalHistoryProvider's CTE,
1606
+ * future analytics consumers) without us proxying every method.
1607
+ * Callers must NOT close the handle directly — `Cache.close()` owns
1608
+ * the lifetime.
1609
+ */
1610
+ dbHandle(): Database {
1611
+ return this.db
1612
+ }
1613
+
1614
+ /**
1615
+ * Content-addressed storage view over the same artifacts directory.
1616
+ * Returns an `FsCASBackend` pointing at `cacheDir`, so external
1617
+ * subsystems (R2 mirror, REAPI CAS bridge, analytics scanners) can
1618
+ * read raw bytes with a `Digest`-keyed API without coupling to
1619
+ * Cache's internal save path. Read/write semantics match what
1620
+ * Cache.save writes (`<cacheDir>/<hash>.tar.zst`).
1621
+ */
1622
+ contentBackend(): FsCASBackend {
1623
+ return new FsCASBackend(this.cacheDir)
1624
+ }
1625
+
1626
+ recordRun(run: RunRecord): void {
1627
+ this.insertRun.run(...bindRun(run))
1628
+ }
1629
+
1630
+ recordRuns(runs: readonly RunRecord[]): void {
1631
+ if (runs.length === 0) return
1632
+ if (runs.length === 1) {
1633
+ this.insertRun.run(...bindRun(runs[0]!))
1634
+ return
1635
+ }
1636
+ // `bun:sqlite`'s `transaction()` returns a callable that wraps the
1637
+ // body in BEGIN/COMMIT, fsyncing once at the end. For a 200-task
1638
+ // run that's one fsync instead of 200.
1639
+ const insert = this.insertRun
1640
+ const tx = this.db.transaction((batch: readonly RunRecord[]) => {
1641
+ for (const r of batch) insert.run(...bindRun(r))
1642
+ })
1643
+ tx(runs)
1644
+ }
1645
+
1646
+ recordRunBundle(bundle: { runs: readonly RunRecord[]; invocation: InvocationRecord }): void {
1647
+ // Whole run records atomically — one transaction, one fsync: the
1648
+ // per-task `runs` rows and the one `invocations` header. The
1649
+ // input-fingerprint rows do NOT live here: they're written inside
1650
+ // the entry-save transaction (`save`/`ingest`) so a warm
1651
+ // all-cache-hit run — which writes no `runs`-vs-`entry_inputs`
1652
+ // mismatch — pays nothing for the moat it isn't refreshing.
1653
+ const insertRun = this.insertRun
1654
+ const insertInvocation = this.insertInvocation
1655
+ this.db.transaction(() => {
1656
+ for (const r of bundle.runs) insertRun.run(...bindRun(r))
1657
+ insertInvocation.run(...bindInvocation(bundle.invocation))
1658
+ })()
1659
+ }
1660
+
1661
+ stats(): CacheStats {
1662
+ this.flushAccessed()
1663
+ const aggregate = this.db
1664
+ .prepare('SELECT COUNT(*) AS n, COALESCE(SUM(size_bytes), 0) AS bytes FROM entries')
1665
+ .get() as { n: number; bytes: number }
1666
+ const since = Date.now() - 24 * 60 * 60 * 1000
1667
+ const runs = this.db
1668
+ .prepare(
1669
+ "SELECT COUNT(*) AS total, COALESCE(SUM(CASE WHEN status IN ('cache-hit', 'cache-hit-remote') THEN 1 ELSE 0 END), 0) AS hits FROM runs WHERE started_at >= ?",
1670
+ )
1671
+ .get(since) as { total: number; hits: number }
1672
+ return {
1673
+ entryCount: aggregate.n,
1674
+ totalBytes: aggregate.bytes,
1675
+ runCountLast24h: runs.total,
1676
+ hitCountLast24h: runs.hits,
1677
+ }
1678
+ }
1679
+
1680
+ async prune(options: PruneOptions): Promise<PruneResult> {
1681
+ this.flushAccessed()
1682
+ const { olderThanMs, maxBytes } = options
1683
+ if (olderThanMs === undefined && maxBytes === undefined) {
1684
+ throw new Error('prune: pass at least one of `olderThanMs` or `maxBytes`')
1685
+ }
1686
+
1687
+ const victims = new Set<string>()
1688
+ let bytesFreed = 0
1689
+
1690
+ if (olderThanMs !== undefined) {
1691
+ const rows = this.db
1692
+ .prepare('SELECT hash, size_bytes FROM entries WHERE accessed_at < ?')
1693
+ .all(olderThanMs) as Array<{ hash: string; size_bytes: number }>
1694
+ for (const r of rows) {
1695
+ victims.add(r.hash)
1696
+ bytesFreed += r.size_bytes
1697
+ }
1698
+ }
1699
+
1700
+ if (maxBytes !== undefined) {
1701
+ const totalRow = this.db
1702
+ .prepare('SELECT COALESCE(SUM(size_bytes), 0) AS bytes FROM entries')
1703
+ .get() as { bytes: number }
1704
+ let remaining = totalRow.bytes - bytesFreed
1705
+ if (remaining > maxBytes) {
1706
+ // Exclude already-picked victims in JS, not via a SQL NOT-IN —
1707
+ // an IN-list over tens of thousands of TTL victims would blow
1708
+ // SQLite's bound-parameter ceiling (see flushAccessed's 900 cap).
1709
+ const candidates = (
1710
+ this.db
1711
+ .prepare('SELECT hash, size_bytes FROM entries ORDER BY accessed_at ASC')
1712
+ .all() as Array<{ hash: string; size_bytes: number }>
1713
+ ).filter((row) => !victims.has(row.hash))
1714
+ for (const row of candidates) {
1715
+ if (remaining <= maxBytes) break
1716
+ victims.add(row.hash)
1717
+ bytesFreed += row.size_bytes
1718
+ remaining -= row.size_bytes
1719
+ }
1720
+ }
1721
+ }
1722
+
1723
+ // Delete DB rows in a single transaction (one fsync; ON DELETE
1724
+ // CASCADE clears `output_files`) and unlink artifacts in parallel.
1725
+ // Replaces N round-trips + serialized rm with one transaction + a
1726
+ // Promise.all over the unlinks. The IN-list is chunked at 900 like
1727
+ // flushAccessed so a huge eviction stays under any build's
1728
+ // bound-parameter ceiling.
1729
+ if (victims.size > 0) {
1730
+ const hashes = [...victims]
1731
+ this.db.transaction(() => {
1732
+ for (let i = 0; i < hashes.length; i += 900) {
1733
+ const chunk = hashes.slice(i, i + 900)
1734
+ this.db
1735
+ .prepare(`DELETE FROM entries WHERE hash IN (${chunk.map(() => '?').join(',')})`)
1736
+ .run(...(chunk as readonly SQLQueryBindings[]))
1737
+ }
1738
+ })()
1739
+ await Promise.all(hashes.map((h) => rm(this.tarPath(h), { force: true })))
1740
+ }
1741
+
1742
+ return { evicted: victims.size, bytesFreed }
1743
+ }
1744
+
1745
+ close(): void {
1746
+ // Run-history retention: the runs table grows by one row per
1747
+ // executed task per invocation — a 2000-task repo accretes ~20k
1748
+ // rows in days, inflating insert and checkpoint cost forever.
1749
+ // 30 days comfortably covers `vx stats` (24 h windows) and any
1750
+ // CI-side analytics consumers.
1751
+ try {
1752
+ this.db
1753
+ .prepare('DELETE FROM runs WHERE ended_at < ?')
1754
+ .run(Date.now() - 30 * 24 * 60 * 60 * 1000)
1755
+ } catch {
1756
+ // Retention is best-effort; never block closing the handle.
1757
+ }
1758
+ this.flushAccessed()
1759
+ this.db.close()
1760
+ }
1761
+
1762
+ private tarPath(hash: string): string {
1763
+ return path.join(this.cacheDir, `${hash}.tar.zst`)
1764
+ }
1765
+ }
1766
+
1767
+ /**
1768
+ * Bind a RunRecord to the positional parameters expected by the
1769
+ * `insertRun` prepared statement (17 columns). Shared between the
1770
+ * single and batched record paths.
1771
+ */
1772
+ function bindRun(run: RunRecord): SQLQueryBindings[] {
1773
+ return [
1774
+ run.hash,
1775
+ run.project,
1776
+ run.task,
1777
+ run.status,
1778
+ run.exitCode,
1779
+ run.durationMs,
1780
+ run.forwardArgs ? JSON.stringify(run.forwardArgs) : null,
1781
+ run.startedAt,
1782
+ run.endedAt,
1783
+ run.runId ?? null,
1784
+ run.cpuMs ?? null,
1785
+ run.peakRssBytes ?? null,
1786
+ run.wallclockStartNs !== undefined ? run.wallclockStartNs : null,
1787
+ run.wallclockEndNs !== undefined ? run.wallclockEndNs : null,
1788
+ run.cacheHit === undefined ? null : run.cacheHit ? 1 : 0,
1789
+ run.attempts ?? null,
1790
+ ]
1791
+ }
1792
+
1793
+ /**
1794
+ * Bind an InvocationRecord to the positional parameters of the
1795
+ * `insertInvocation` prepared statement (25 columns). Booleans map to
1796
+ * 0/1; null-or-bool columns (`dirty`) keep null distinct from 0.
1797
+ */
1798
+ function bindInvocation(inv: InvocationRecord): SQLQueryBindings[] {
1799
+ return [
1800
+ inv.runId,
1801
+ inv.command,
1802
+ inv.requestedTasks,
1803
+ inv.cachePolicy,
1804
+ inv.concurrency,
1805
+ inv.flow,
1806
+ inv.startedAt,
1807
+ inv.endedAt,
1808
+ inv.totalDurationMs,
1809
+ inv.taskCount,
1810
+ inv.failedCount,
1811
+ inv.hitCount,
1812
+ inv.hitLocalCount,
1813
+ inv.hitRemoteCount,
1814
+ inv.exitOk ? 1 : 0,
1815
+ inv.commitSha,
1816
+ inv.branch,
1817
+ inv.dirty === null ? null : inv.dirty ? 1 : 0,
1818
+ inv.ci ? 1 : 0,
1819
+ inv.ciProvider,
1820
+ inv.host,
1821
+ inv.os,
1822
+ inv.arch,
1823
+ inv.vxVersion,
1824
+ inv.tags,
1825
+ ]
1826
+ }
1827
+
1828
+ function splitTaskId(id: string): [string, string] {
1829
+ const i = id.indexOf('#')
1830
+ if (i < 0) return [id, '']
1831
+ return [id.slice(0, i), id.slice(i + 1)]
1832
+ }