@vzn/vx 0.0.18 → 0.0.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/LICENSE +1 -1
  2. package/README.md +99 -65
  3. package/package.json +10 -20
  4. package/src/bin.ts +12 -2
  5. package/src/cache/archive.ts +120 -18
  6. package/src/cache/cache.ts +313 -1324
  7. package/src/cache/chained-cache.ts +0 -8
  8. package/src/cache/config-evals.ts +89 -0
  9. package/src/cache/file-hashes.ts +262 -0
  10. package/src/cache/git-inputs.ts +637 -0
  11. package/src/cache/index.ts +16 -7
  12. package/src/cache/inputs.ts +246 -733
  13. package/src/cache/layer.ts +604 -0
  14. package/src/cache/layered-cache.ts +12 -17
  15. package/src/cache/output-index.ts +224 -0
  16. package/src/cache/policy.ts +84 -0
  17. package/src/cache/run-history.ts +147 -0
  18. package/src/cache/zstd.ts +160 -0
  19. package/src/cli/cache.ts +39 -11
  20. package/src/cli/completions.ts +104 -0
  21. package/src/cli/core-alias.ts +25 -0
  22. package/src/cli/help.ts +64 -36
  23. package/src/cli/index.ts +54 -15
  24. package/src/cli/info.ts +120 -91
  25. package/src/cli/init.ts +75 -0
  26. package/src/cli/last.ts +38 -6
  27. package/src/cli/lock.ts +2 -1
  28. package/src/cli/plan-format.ts +14 -3
  29. package/src/cli/plugin-commands.ts +15 -8
  30. package/src/cli/run.ts +84 -302
  31. package/src/cli/select.ts +332 -0
  32. package/src/cli/show.ts +144 -69
  33. package/src/cli/upgrade.ts +2 -1
  34. package/src/cli/watch.ts +647 -97
  35. package/src/cli/why.ts +44 -14
  36. package/src/cli/workspace-config.ts +126 -0
  37. package/src/config.ts +156 -136
  38. package/src/exec/env.ts +11 -0
  39. package/src/exec/executor.ts +11 -26
  40. package/src/exec/index.ts +6 -2
  41. package/src/{plugins/local-executor/index.ts → exec/local-executor.ts} +12 -16
  42. package/src/exec/runner.ts +57 -18
  43. package/src/exec/sandbox-binds.ts +182 -0
  44. package/src/exec/sandbox-paths.ts +50 -0
  45. package/src/exec/sandbox-runtime.ts +582 -370
  46. package/src/exec/sandbox-violations.ts +280 -0
  47. package/src/graph/index.ts +1 -9
  48. package/src/graph/scheduler.ts +158 -203
  49. package/src/graph/task-graph.ts +2 -2
  50. package/src/index.ts +97 -102
  51. package/src/orchestrator/admission.ts +137 -0
  52. package/src/orchestrator/deferred-outputs.ts +3 -0
  53. package/src/orchestrator/doctor.ts +255 -0
  54. package/src/orchestrator/events.ts +42 -8
  55. package/src/orchestrator/execute-task.ts +145 -554
  56. package/src/orchestrator/failure-mode.ts +169 -28
  57. package/src/orchestrator/framed-output.ts +41 -36
  58. package/src/orchestrator/history.ts +118 -98
  59. package/src/orchestrator/hit-restore.ts +221 -0
  60. package/src/orchestrator/index.ts +35 -4
  61. package/src/orchestrator/lockfile-claim.ts +281 -0
  62. package/src/orchestrator/logger.ts +13 -22
  63. package/src/orchestrator/metrics.ts +53 -20
  64. package/src/orchestrator/miss-save.ts +169 -0
  65. package/src/orchestrator/options.ts +34 -31
  66. package/src/orchestrator/persistent.ts +79 -0
  67. package/src/orchestrator/placement.ts +182 -0
  68. package/src/orchestrator/plugin-host.ts +195 -38
  69. package/src/orchestrator/plugin.ts +162 -9
  70. package/src/orchestrator/prepare.ts +77 -132
  71. package/src/orchestrator/projects.ts +259 -0
  72. package/src/orchestrator/remote-prefetch.ts +1 -1
  73. package/src/orchestrator/run-artifacts.ts +25 -3
  74. package/src/orchestrator/run-context.ts +1 -1
  75. package/src/orchestrator/run-records.ts +149 -0
  76. package/src/orchestrator/run-report.ts +2 -2
  77. package/src/orchestrator/run.ts +246 -597
  78. package/src/orchestrator/sandbox-request.ts +243 -0
  79. package/src/orchestrator/save-lane.ts +47 -0
  80. package/src/orchestrator/signals.ts +111 -0
  81. package/src/orchestrator/stable-keys.ts +60 -3
  82. package/src/orchestrator/status-line.ts +2 -1
  83. package/src/orchestrator/summary.ts +68 -25
  84. package/src/orchestrator/task-hash.ts +20 -18
  85. package/src/orchestrator/task-log-buffer.ts +8 -11
  86. package/src/orchestrator/telemetry-host.ts +11 -1
  87. package/src/orchestrator/telemetry.ts +3 -17
  88. package/src/util/cgroup.ts +160 -0
  89. package/src/util/edit-distance.ts +49 -0
  90. package/src/util/index.ts +11 -3
  91. package/src/util/paths.ts +23 -2
  92. package/src/util/settle.ts +19 -0
  93. package/src/util/verbs.ts +39 -0
  94. package/src/workspace/affected.ts +111 -11
  95. package/src/workspace/config-cache.ts +11 -2
  96. package/src/workspace/config-eval.ts +11 -4
  97. package/src/workspace/config-imports.ts +5 -4
  98. package/src/workspace/config-schema.ts +837 -0
  99. package/src/workspace/filter.ts +43 -6
  100. package/src/workspace/fingerprint.ts +37 -3
  101. package/src/workspace/index.ts +30 -3
  102. package/src/workspace/lockfile.ts +12 -7
  103. package/src/{cli → workspace}/migrate-scripts.ts +26 -19
  104. package/src/{cli/migrate.ts → workspace/migration.ts} +133 -124
  105. package/src/workspace/package-graph.ts +98 -28
  106. package/src/workspace/project-loader.ts +76 -746
  107. package/src/workspace/workspace.ts +98 -13
  108. package/plugins/local-cache/index.ts +0 -2
  109. package/plugins/local-executor/index.ts +0 -2
  110. package/plugins/schedule-history/index.ts +0 -2
  111. package/src/cache/cas-backend.ts +0 -115
  112. package/src/cache/digest.ts +0 -32
  113. package/src/cli/migrate-emit.ts +0 -21
  114. package/src/cli/migrate-nx.ts +0 -473
  115. package/src/cli/migrate-turbo.ts +0 -401
  116. package/src/cli/prune.ts +0 -291
  117. package/src/orchestrator/missing-plugin.ts +0 -9
  118. package/src/orchestrator/resources.ts +0 -45
  119. package/src/orchestrator/verify.ts +0 -174
  120. package/src/plugins/local-cache/index.ts +0 -11
  121. package/src/plugins/schedule-history/index.ts +0 -109
@@ -0,0 +1,224 @@
1
+ // The output fingerprint index: per-entry `output_files` rows (size, mode,
2
+ // millisecond mtime) and `output_dirs` rows (directory mtimes), and the two
3
+ // proofs a cache hit runs against the tree before deciding not to restore.
4
+ // Owns its statements over the store's handle; `Cache` delegates, and
5
+ // writes the file rows through `replaceFileRows` inside its own save
6
+ // transaction.
7
+
8
+ /** `mtime_ms` of a recorded output prefix that did not exist when the snapshot was taken. */
9
+ const ABSENT_DIR_MTIME = -1
10
+
11
+ import type { Database, SQLQueryBindings } from 'bun:sqlite'
12
+ import { lstatSync, statSync } from 'node:fs'
13
+ import { lstat, readdir } from 'node:fs/promises'
14
+ import path from 'node:path'
15
+ import {
16
+ OUTPUT_DIRS_CAP,
17
+ OUTPUT_DIRS_RACY_MS,
18
+ type OutputDirRow,
19
+ type OutputFileRow,
20
+ } from './layer.js'
21
+
22
+ export class OutputIndex {
23
+ private readonly insertOutputFile: ReturnType<Database['prepare']>
24
+ private readonly deleteOutputFiles: ReturnType<Database['prepare']>
25
+ private readonly insertOutputDir: ReturnType<Database['prepare']>
26
+ private readonly deleteOutputDirs: ReturnType<Database['prepare']>
27
+
28
+ constructor(private readonly db: Database) {
29
+ this.insertOutputFile = this.db.prepare(`
30
+ INSERT INTO output_files(entry_hash, path, size_bytes, mode, mtime_ms)
31
+ VALUES (?, ?, ?, ?, ?)
32
+ ON CONFLICT(entry_hash, path) DO UPDATE SET
33
+ size_bytes = excluded.size_bytes,
34
+ mode = excluded.mode,
35
+ mtime_ms = excluded.mtime_ms
36
+ `)
37
+ this.deleteOutputFiles = this.db.prepare('DELETE FROM output_files WHERE entry_hash = ?')
38
+ this.insertOutputDir = this.db.prepare(
39
+ 'INSERT INTO output_dirs(entry_hash, path, mtime_ms) VALUES (?, ?, ?)',
40
+ )
41
+ this.deleteOutputDirs = this.db.prepare('DELETE FROM output_dirs WHERE entry_hash = ?')
42
+ }
43
+
44
+ /**
45
+ * Replace an entry's file rows. Called INSIDE the store's save transaction,
46
+ * so an entry and its fingerprints land together (an UPDATE on the same
47
+ * hash refreshes rather than appends).
48
+ */
49
+ replaceFileRows(hash: string, rows: ReadonlyArray<[string, number, number, number]>): void {
50
+ this.deleteOutputFiles.run(hash)
51
+ for (const [rel, size, mode, mtime] of rows) {
52
+ this.insertOutputFile.run(hash, rel, size, mode, mtime)
53
+ }
54
+ }
55
+
56
+ loadOutputFilesBatch(hashes: readonly string[]): Map<string, OutputFileRow[]> {
57
+ const out = new Map<string, OutputFileRow[]>()
58
+ if (hashes.length === 0) return out
59
+ // Inline placeholders for an IN-list — bun:sqlite doesn't ship
60
+ // rarray, but `IN (?, ?, …)` with N≤~999 is fast and avoids per-
61
+ // hash select.get() overhead. `db.query` (not `db.prepare`) caches the
62
+ // compiled statement keyed by the SQL text — so the dominant single-hash
63
+ // warm-hit path (called up to 3× per hit) reuses one statement instead of
64
+ // recompiling on every call.
65
+ const placeholders = hashes.map(() => '?').join(',')
66
+ const stmt = this.db.query(
67
+ `SELECT entry_hash, path, size_bytes, mode, mtime_ms FROM output_files WHERE entry_hash IN (${placeholders})`,
68
+ )
69
+ const rows = stmt.all(...(hashes as readonly SQLQueryBindings[])) as Array<{
70
+ entry_hash: string
71
+ path: string
72
+ size_bytes: number
73
+ mode: number
74
+ mtime_ms: number
75
+ }>
76
+ for (const r of rows) {
77
+ let list = out.get(r.entry_hash)
78
+ if (!list) {
79
+ list = []
80
+ out.set(r.entry_hash, list)
81
+ }
82
+ list.push({ path: r.path, size: r.size_bytes, mode: r.mode, mtimeMs: r.mtime_ms })
83
+ }
84
+ return out
85
+ }
86
+
87
+ async isOutputsCurrent(projectDir: string, expected: readonly OutputFileRow[]): Promise<boolean> {
88
+ // Empty manifest case (a task produced no outputs) → trivially
89
+ // current; the on-disk tree under projectDir is whatever it was,
90
+ // and nothing was supposed to land there.
91
+ if (expected.length === 0) return true
92
+ // statSync, deliberately. The async form measured faster on 2026-09-02,
93
+ // when a hit walked its whole output tree; since the directory
94
+ // short-circuit (`outputDirsCurrent`) a warm hit stats a handful of
95
+ // paths, and each async stat costs a thread-pool round trip the
96
+ // scheduler cannot overlap away (its concurrency is the worker count).
97
+ // Re-measured 2026-09-09, 1,000 warm hits, interleaved A/B: the run
98
+ // graph stage 100 → 54 ms, the whole process 422 → 368 ms.
99
+ const results = expected.map((e) => {
100
+ try {
101
+ const s = statSync(path.join(projectDir, e.path))
102
+ return (
103
+ s.size === e.size &&
104
+ (s.mode & 0o777) === (e.mode & 0o777) &&
105
+ // MILLISECOND comparison (sub-ms tolerance for the float
106
+ // round-trip through utimes). Save rows carry stat-ms and
107
+ // restoreOutputs re-syncs restored files to the row value,
108
+ // so equality holds exactly in steady state; legacy
109
+ // second-precision rows converge on their first restore.
110
+ // Residual blind spot: a same-size edit landing in the SAME
111
+ // millisecond as the recorded write, or a deliberately
112
+ // forged mtime (touch -r) — the trade every mtime-based
113
+ // skip check accepts.
114
+ Math.abs(s.mtimeMs - e.mtimeMs) < 1
115
+ )
116
+ } catch {
117
+ return false
118
+ }
119
+ })
120
+ return results.every(Boolean)
121
+ }
122
+
123
+ /**
124
+ * Snapshot every directory under each of `prefixes` for `hash`. Called
125
+ * after a save and after a restore, when the tree is known to equal the
126
+ * entry's set. Symlinked directories are not descended (the output walk
127
+ * refuses them too). Over `OUTPUT_DIRS_CAP` directories, or on any
128
+ * error, the rows are cleared and the next hit keeps the walk.
129
+ */
130
+ async recordOutputDirs(
131
+ hash: string,
132
+ projectDir: string,
133
+ prefixes: readonly string[],
134
+ ): Promise<void> {
135
+ const rows: Array<[string, number]> = []
136
+ const walk = async (rel: string, isPrefix = false): Promise<boolean> => {
137
+ const abs = path.join(projectDir, rel)
138
+ let st
139
+ try {
140
+ st = await lstat(abs)
141
+ } catch {
142
+ // A declared prefix the task never produced (`build/**` beside
143
+ // `dist/**`, medusa's `.medusa/**`) is recorded ABSENT: the check
144
+ // then asks that it still not exist. Refusing the snapshot instead
145
+ // made every such task re-glob its outputs on every warm hit — all
146
+ // 83 of medusa's, 210 ms of a 2.5 s no-op (2026-09-11).
147
+ if (isPrefix) {
148
+ rows.push([rel, ABSENT_DIR_MTIME])
149
+ return true
150
+ }
151
+ return false
152
+ }
153
+ if (!st.isDirectory()) return false
154
+ rows.push([rel, st.mtimeMs])
155
+ if (rows.length > OUTPUT_DIRS_CAP) return false
156
+ let entries
157
+ try {
158
+ entries = await readdir(abs, { withFileTypes: true })
159
+ } catch {
160
+ return false
161
+ }
162
+ for (const e of entries) {
163
+ if (e.isDirectory() && !e.isSymbolicLink()) {
164
+ if (!(await walk(`${rel}/${e.name}`))) return false
165
+ }
166
+ }
167
+ return true
168
+ }
169
+ let ok = true
170
+ for (const prefix of prefixes) {
171
+ if (!(await walk(prefix, true))) {
172
+ ok = false
173
+ break
174
+ }
175
+ }
176
+ // All or nothing: a racy directory dropped alone would leave its
177
+ // parent trusted while an addition inside it bumps only the dropped one.
178
+ const youngest = Date.now() - OUTPUT_DIRS_RACY_MS
179
+ if (rows.some(([, mtime]) => mtime > youngest)) ok = false
180
+ this.db.transaction(() => {
181
+ this.deleteOutputDirs.run(hash)
182
+ if (!ok) return
183
+ for (const [rel, mtime] of rows) this.insertOutputDir.run(hash, rel, mtime)
184
+ })()
185
+ }
186
+
187
+ loadOutputDirsBatch(hashes: readonly string[]): Map<string, OutputDirRow[]> {
188
+ const out = new Map<string, OutputDirRow[]>()
189
+ if (hashes.length === 0) return out
190
+ const placeholders = hashes.map(() => '?').join(',')
191
+ const rows = this.db
192
+ .query(
193
+ `SELECT entry_hash, path, mtime_ms FROM output_dirs WHERE entry_hash IN (${placeholders})`,
194
+ )
195
+ .all(...(hashes as readonly SQLQueryBindings[])) as Array<{
196
+ entry_hash: string
197
+ path: string
198
+ mtime_ms: number
199
+ }>
200
+ for (const r of rows) {
201
+ const list = out.get(r.entry_hash)
202
+ const row = { path: r.path, mtimeMs: r.mtime_ms }
203
+ if (list) list.push(row)
204
+ else out.set(r.entry_hash, [row])
205
+ }
206
+ return out
207
+ }
208
+
209
+ /** True iff every recorded directory exists with its recorded mtime (ms). Same forged-mtime trade as the file check. */
210
+ async outputDirsCurrent(projectDir: string, rows: readonly OutputDirRow[]): Promise<boolean> {
211
+ if (rows.length === 0) return false
212
+ const results = rows.map((r) => {
213
+ let st
214
+ try {
215
+ st = lstatSync(path.join(projectDir, r.path))
216
+ } catch {
217
+ return r.mtimeMs === ABSENT_DIR_MTIME
218
+ }
219
+ if (r.mtimeMs === ABSENT_DIR_MTIME) return false
220
+ return st.isDirectory() && Math.abs(st.mtimeMs - r.mtimeMs) < 1
221
+ })
222
+ return results.every(Boolean)
223
+ }
224
+ }
@@ -0,0 +1,84 @@
1
+ // The run-level cache policy: four axes (local/remote × read/write) and
2
+ // the `--cache=<spec>` grammar that sets them. Pure data + parsing; no I/O.
3
+
4
+ import { UserError } from '../util/index.js'
5
+
6
+ /**
7
+ * Independent read/write control over the two cache layers (local +
8
+ * remote). Replaces the old single `noCache` boolean: each axis can be
9
+ * toggled on its own so `--force` (re-execute but still refresh the
10
+ * cache) is distinct from `--no-cache` (disable everything).
11
+ *
12
+ * Only the task-artifact get/save path is gated. `recordRun`, `stats`,
13
+ * `prune`, key derivation, and prefetch-ingest are never affected — they
14
+ * are bookkeeping/analytics that a run policy has no business disabling.
15
+ */
16
+ export interface CachePolicy {
17
+ localRead: boolean
18
+ localWrite: boolean
19
+ remoteRead: boolean
20
+ remoteWrite: boolean
21
+ }
22
+
23
+ /** All four axes on — the default when no cache flag is passed. */
24
+ export const FULL_CACHE_POLICY: CachePolicy = {
25
+ localRead: true,
26
+ localWrite: true,
27
+ remoteRead: true,
28
+ remoteWrite: true,
29
+ }
30
+
31
+ /**
32
+ * Parse a `--cache=<spec>` value into a `CachePolicy`, starting from a
33
+ * base (defaults to {@link FULL_CACHE_POLICY}). The spec is a
34
+ * comma-separated list of `layer:flags` segments where `layer` is
35
+ * `local` or `remote` and `flags` is any subset of `r` (read) and `w`
36
+ * (write), order-independent and possibly empty.
37
+ *
38
+ * A mentioned layer is set EXACTLY to its flags (read = includes `r`,
39
+ * write = includes `w`); an unmentioned layer keeps its base value.
40
+ * So `local:rw,remote:r` = remote read-only, `remote:` = remote fully
41
+ * off, `local:r` = local read-only with remote untouched.
42
+ *
43
+ * Throws a {@link UserError} on an unknown layer, an unknown flag, a
44
+ * duplicated flag, a missing colon, or a repeated layer.
45
+ */
46
+ export function parseCachePolicy(spec: string, base: CachePolicy = FULL_CACHE_POLICY): CachePolicy {
47
+ const out: CachePolicy = { ...base }
48
+ const seen = new Set<string>()
49
+ for (const rawSeg of spec.split(',')) {
50
+ const seg = rawSeg.trim()
51
+ if (seg.length === 0) continue
52
+ const colon = seg.indexOf(':')
53
+ if (colon < 0) {
54
+ throw new UserError(`invalid --cache segment '${seg}': expected '<layer>:<flags>'`)
55
+ }
56
+ const layer = seg.slice(0, colon)
57
+ const flags = seg.slice(colon + 1)
58
+ if (layer !== 'local' && layer !== 'remote') {
59
+ throw new UserError(`invalid --cache layer '${layer}': expected 'local' or 'remote'`)
60
+ }
61
+ if (seen.has(layer)) {
62
+ throw new UserError(`--cache layer '${layer}' specified twice`)
63
+ }
64
+ seen.add(layer)
65
+ const flagSet = new Set<string>()
66
+ for (const ch of flags) {
67
+ if (ch !== 'r' && ch !== 'w') {
68
+ throw new UserError(`invalid --cache flag '${ch}' for '${layer}': expected 'r' and/or 'w'`)
69
+ }
70
+ if (flagSet.has(ch)) {
71
+ throw new UserError(`--cache flag '${ch}' repeated for '${layer}'`)
72
+ }
73
+ flagSet.add(ch)
74
+ }
75
+ if (layer === 'local') {
76
+ out.localRead = flagSet.has('r')
77
+ out.localWrite = flagSet.has('w')
78
+ } else {
79
+ out.remoteRead = flagSet.has('r')
80
+ out.remoteWrite = flagSet.has('w')
81
+ }
82
+ }
83
+ return out
84
+ }
@@ -0,0 +1,147 @@
1
+ // The run history: one `runs` row per task outcome and one `invocations`
2
+ // header per `vx run`, written together, plus their 30-day retention.
3
+ // What `vx why`, `vx last`, `stats` and the history reader read. Owns
4
+ // its statements over the store's handle; `Cache` delegates.
5
+
6
+ import type { Database, SQLQueryBindings } from 'bun:sqlite'
7
+ import type { InvocationRecord, RunRecord } from './layer.js'
8
+
9
+ export class RunHistory {
10
+ private readonly insertRun: ReturnType<Database['prepare']>
11
+ private readonly insertInvocation: ReturnType<Database['prepare']>
12
+
13
+ constructor(private readonly db: Database) {
14
+ this.insertRun = this.db.prepare(`
15
+ INSERT INTO runs(
16
+ hash, project, task, status, exit_code, duration_ms, forward_args,
17
+ started_at, ended_at,
18
+ run_id, cpu_ms, peak_rss_bytes, wallclock_start_ns, wallclock_end_ns,
19
+ cache_hit, attempts, cached
20
+ )
21
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
22
+ `)
23
+ this.insertInvocation = this.db.prepare(`
24
+ INSERT INTO invocations(
25
+ run_id, command, requested_tasks, cache_policy, concurrency, flow,
26
+ started_at, ended_at, total_duration_ms,
27
+ task_count, failed_count, hit_count, hit_local_count, hit_remote_count,
28
+ exit_ok,
29
+ commit_sha, branch, dirty, ci, ci_provider,
30
+ host, os, arch, vx_version, tags
31
+ )
32
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
33
+ ON CONFLICT(run_id) DO NOTHING
34
+ `)
35
+ }
36
+
37
+ recordRun(run: RunRecord): void {
38
+ this.insertRun.run(...bindRun(run))
39
+ }
40
+
41
+ recordRuns(runs: readonly RunRecord[]): void {
42
+ if (runs.length === 0) return
43
+ if (runs.length === 1) {
44
+ this.insertRun.run(...bindRun(runs[0]!))
45
+ return
46
+ }
47
+ // `bun:sqlite`'s `transaction()` returns a callable that wraps the
48
+ // body in BEGIN/COMMIT, fsyncing once at the end. For a 200-task
49
+ // run that's one fsync instead of 200.
50
+ const insert = this.insertRun
51
+ const tx = this.db.transaction((batch: readonly RunRecord[]) => {
52
+ for (const r of batch) insert.run(...bindRun(r))
53
+ })
54
+ tx(runs)
55
+ }
56
+
57
+ recordRunBundle(bundle: { runs: readonly RunRecord[]; invocation: InvocationRecord }): void {
58
+ // Whole run records atomically — one transaction, one fsync: the
59
+ // per-task `runs` rows and the one `invocations` header. The
60
+ // input-fingerprint rows do NOT live here: they're written inside
61
+ // the entry-save transaction (`save`/`ingest`) so a warm
62
+ // all-cache-hit run — which writes no `runs`-vs-`entry_inputs`
63
+ // mismatch — pays nothing for the moat it isn't refreshing.
64
+ const insertRun = this.insertRun
65
+ const insertInvocation = this.insertInvocation
66
+ this.db.transaction(() => {
67
+ for (const r of bundle.runs) insertRun.run(...bindRun(r))
68
+ insertInvocation.run(...bindInvocation(bundle.invocation))
69
+ })()
70
+ }
71
+
72
+ /**
73
+ * Retention: the runs table grows by one row per executed task per
74
+ * invocation — a 2000-task repo accretes ~20k rows in days. 30 days
75
+ * covers `vx stats` (24 h windows) and CI-side consumers. The
76
+ * `invocations` header is pruned on the SAME window, or a header would
77
+ * outlive its rows and `vx last` would list a run whose detail is gone.
78
+ */
79
+ pruneOlderThan(cutoff: number): void {
80
+ this.db.prepare('DELETE FROM runs WHERE started_at < ?').run(cutoff)
81
+ this.db.prepare('DELETE FROM invocations WHERE started_at < ?').run(cutoff)
82
+ }
83
+ }
84
+
85
+ /**
86
+ * Bind a RunRecord to the positional parameters expected by the
87
+ * `insertRun` prepared statement (17 columns). Shared between the
88
+ * single and batched record paths.
89
+ */
90
+ function bindRun(run: RunRecord): SQLQueryBindings[] {
91
+ return [
92
+ // The ONE place the no-key sentinel is applied, so the column's
93
+ // NOT NULL invariant can't be violated from a call site.
94
+ run.hash ?? '',
95
+ run.project,
96
+ run.task,
97
+ run.status,
98
+ run.exitCode,
99
+ run.durationMs,
100
+ run.forwardArgs ? JSON.stringify(run.forwardArgs) : null,
101
+ run.startedAt,
102
+ run.endedAt,
103
+ run.runId ?? null,
104
+ run.cpuMs ?? null,
105
+ run.peakRssBytes ?? null,
106
+ run.wallclockStartNs !== undefined ? run.wallclockStartNs : null,
107
+ run.wallclockEndNs !== undefined ? run.wallclockEndNs : null,
108
+ run.cacheHit === undefined ? null : run.cacheHit ? 1 : 0,
109
+ run.attempts ?? null,
110
+ run.cached === undefined ? null : run.cached ? 1 : 0,
111
+ ]
112
+ }
113
+
114
+ /**
115
+ * Bind an InvocationRecord to the positional parameters of the
116
+ * `insertInvocation` prepared statement (25 columns). Booleans map to
117
+ * 0/1; null-or-bool columns (`dirty`) keep null distinct from 0.
118
+ */
119
+ function bindInvocation(inv: InvocationRecord): SQLQueryBindings[] {
120
+ return [
121
+ inv.runId,
122
+ inv.command,
123
+ inv.requestedTasks,
124
+ inv.cachePolicy,
125
+ inv.concurrency,
126
+ inv.flow,
127
+ inv.startedAt,
128
+ inv.endedAt,
129
+ inv.totalDurationMs,
130
+ inv.taskCount,
131
+ inv.failedCount,
132
+ inv.hitCount,
133
+ inv.hitLocalCount,
134
+ inv.hitRemoteCount,
135
+ inv.exitOk ? 1 : 0,
136
+ inv.commitSha,
137
+ inv.branch,
138
+ inv.dirty === null ? null : inv.dirty ? 1 : 0,
139
+ inv.ci ? 1 : 0,
140
+ inv.ciProvider,
141
+ inv.host,
142
+ inv.os,
143
+ inv.arch,
144
+ inv.vxVersion,
145
+ inv.tags,
146
+ ]
147
+ }
@@ -0,0 +1,160 @@
1
+ // zstd framing for artifacts: the declared-size gate that refuses a
2
+ // decompression bomb before it can expand, and the bounded one-call /
3
+ // streamed decoders the store and the ingest path share.
4
+
5
+ import { CorruptArtifactError } from './layer.js'
6
+
7
+ /**
8
+ * A decompressed artifact above this is refused as a zstd bomb rather than
9
+ * expanded into memory. 2 GiB comfortably exceeds any real build output while
10
+ * bounding a malicious/compromised remote's ability to OOM a victim who takes
11
+ * a cache hit.
12
+ */
13
+ const MAX_DECOMPRESSED_ARTIFACT_BYTES = 2 * 1024 * 1024 * 1024
14
+
15
+ /**
16
+ * Read a zstd frame's declared Frame_Content_Size (RFC 8878 §3.1.1) WITHOUT
17
+ * decompressing. Returns null when the frame omits it (streaming frames) or
18
+ * the header is too short to parse. vx's own producer (single-shot
19
+ * `Bun.zstdCompress` of a known buffer) always writes it, so an artifact that
20
+ * declares an enormous size can be rejected before a byte is allocated.
21
+ *
22
+ * Exported for tests: pins the per-`fcsFlag` byte layouts (incl. the 2-byte
23
+ * `+256` adjustment and the `dictIdFlag` offset) that a bomb-refusal e2e can't
24
+ * discriminate (their max declarable sizes sit below the ceiling).
25
+ */
26
+ export function zstdContentSize(b: Uint8Array): bigint | null {
27
+ if (b.length < 5) return null
28
+ // Magic_Number 0xFD2FB528, little-endian.
29
+ if (b[0] !== 0x28 || b[1] !== 0xb5 || b[2] !== 0x2f || b[3] !== 0xfd) return null
30
+ const desc = b[4]!
31
+ const fcsFlag = desc >> 6
32
+ const singleSegment = (desc >> 5) & 1
33
+ const dictIdFlag = desc & 3
34
+ let off = 5
35
+ if (singleSegment === 0) off += 1 // Window_Descriptor byte
36
+ off += dictIdFlag === 3 ? 4 : dictIdFlag // Dictionary_ID: 0,1,2,4 bytes
37
+ let fcsSize: number
38
+ if (fcsFlag === 0) fcsSize = singleSegment === 1 ? 1 : 0
39
+ else if (fcsFlag === 1) fcsSize = 2
40
+ else if (fcsFlag === 2) fcsSize = 4
41
+ else fcsSize = 8
42
+ if (fcsSize === 0 || b.length < off + fcsSize) return null
43
+ let v = 0n
44
+ for (let i = 0; i < fcsSize; i++) v |= BigInt(b[off + i]!) << BigInt(8 * i)
45
+ if (fcsSize === 2) v += 256n // per spec, the 2-byte field stores value − 256
46
+ return v
47
+ }
48
+
49
+ /**
50
+ * Decompress a zstd artifact that DECLARES its content size, with a hard
51
+ * output ceiling: refused before a byte is allocated when the declaration
52
+ * is over the cap, and again on the actual length. A frame with no
53
+ * declaration (a streamed producer's — vx's own, above 4 MiB) never comes
54
+ * here: `decodedTar` decodes it as a stream under the running count, so
55
+ * a sizeless bomb has nowhere to expand.
56
+ */
57
+ async function zstdDecompressBounded(compressed: Uint8Array, hash: string): Promise<Uint8Array> {
58
+ assertDeclaredSize(compressed, hash)
59
+ const out = await Bun.zstdDecompress(compressed)
60
+ if (out.length > MAX_DECOMPRESSED_ARTIFACT_BYTES) {
61
+ throw new CorruptArtifactError(
62
+ hash,
63
+ `decompressed to ${out.length} bytes (> ${MAX_DECOMPRESSED_ARTIFACT_BYTES} cap)`,
64
+ )
65
+ }
66
+ return out
67
+ }
68
+
69
+ /** The pre-decompress half of the ceiling: the frame header's own claim, when it makes one. */
70
+ function assertDeclaredSize(compressed: Uint8Array, hash: string): bigint | null {
71
+ const declared = zstdContentSize(compressed)
72
+ if (declared !== null && declared > BigInt(MAX_DECOMPRESSED_ARTIFACT_BYTES)) {
73
+ throw new CorruptArtifactError(
74
+ hash,
75
+ `declares ${declared} decompressed bytes (> ${MAX_DECOMPRESSED_ARTIFACT_BYTES} cap)`,
76
+ )
77
+ }
78
+ return declared
79
+ }
80
+
81
+ /** Compressed artifacts above this size are decoded as a stream, on restore and on ingest. */
82
+ export const STREAM_DECODE_FROM = 4 * 1024 * 1024
83
+
84
+ /** Collect a byte stream; only ever used where the seam wants bytes. */
85
+ export async function bytesOf(stream: ReadableStream<Uint8Array>): Promise<Uint8Array> {
86
+ return new Uint8Array(await new Response(stream).arrayBuffer())
87
+ }
88
+
89
+ /**
90
+ * Bun's `CompressionStream` types do not satisfy `pipeThrough`'s pair
91
+ * (its readable side is typed `NonSharedUint8Array`); the runtime object
92
+ * is a plain byte transform.
93
+ */
94
+ export const zstdEncoder = (): TransformStream<Uint8Array, Uint8Array> =>
95
+ new CompressionStream('zstd') as unknown as TransformStream<Uint8Array, Uint8Array>
96
+
97
+ const oneChunk = (bytes: Uint8Array): ReadableStream<Uint8Array> =>
98
+ new ReadableStream({
99
+ start(c) {
100
+ c.enqueue(bytes)
101
+ c.close()
102
+ },
103
+ })
104
+
105
+ /**
106
+ * The decoded tar as a stream: one call for a small artifact that
107
+ * declares its size, a streamed decode otherwise — the stream setup is
108
+ * ~35 µs, which matters at a thousand one-file artifacts and nowhere
109
+ * else. Bytes in memory are only ever the small case (a large source
110
+ * must be a FILE: a Blob copies its bytes and hands the decoder
111
+ * everything at once, measured +519 MiB on 150 MiB against +448 for the
112
+ * plain decode). The 2 GiB ceiling applies either way: the declaration
113
+ * and the result length in one call, a running count on the stream.
114
+ */
115
+ export async function decodedTar(
116
+ source: Uint8Array | Bun.BunFile,
117
+ hash: string,
118
+ ): Promise<ReadableStream<Uint8Array>> {
119
+ if (source instanceof Uint8Array) {
120
+ if (assertDeclaredSize(source, hash) === null) return zstdDecodeStream(new Blob([source]), hash)
121
+ return oneChunk(await zstdDecompressBounded(source, hash))
122
+ }
123
+ if (source.size <= STREAM_DECODE_FROM) {
124
+ const bytes = await source.bytes()
125
+ if (assertDeclaredSize(bytes, hash) === null) return zstdDecodeStream(source, hash)
126
+ return oneChunk(await zstdDecompressBounded(bytes, hash))
127
+ }
128
+ assertDeclaredSize(await source.slice(0, 32).bytes(), hash)
129
+ return zstdDecodeStream(source, hash)
130
+ }
131
+
132
+ /**
133
+ * The streaming twin of `zstdDecompressBounded`: the decoded bytes as a
134
+ * stream, refused past the same ceiling. A malicious frame cannot expand
135
+ * past the cap here either — the count runs as bytes are produced, before
136
+ * any of them reach the reader's next entry.
137
+ */
138
+ function zstdDecodeStream(source: Blob, hash: string): ReadableStream<Uint8Array> {
139
+ let total = 0
140
+ return source
141
+ .stream()
142
+ .pipeThrough(new DecompressionStream('zstd'))
143
+ .pipeThrough(
144
+ new TransformStream<Uint8Array, Uint8Array>({
145
+ transform(chunk, controller) {
146
+ total += chunk.byteLength
147
+ if (total > MAX_DECOMPRESSED_ARTIFACT_BYTES) {
148
+ controller.error(
149
+ new CorruptArtifactError(
150
+ hash,
151
+ `decompresses past ${MAX_DECOMPRESSED_ARTIFACT_BYTES} bytes (cap)`,
152
+ ),
153
+ )
154
+ return
155
+ }
156
+ controller.enqueue(chunk)
157
+ },
158
+ }),
159
+ )
160
+ }