@vzn/vx 0.0.18 → 0.0.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +99 -65
- package/package.json +10 -20
- package/src/bin.ts +12 -2
- package/src/cache/archive.ts +120 -18
- package/src/cache/cache.ts +313 -1324
- package/src/cache/chained-cache.ts +0 -8
- package/src/cache/config-evals.ts +89 -0
- package/src/cache/file-hashes.ts +262 -0
- package/src/cache/git-inputs.ts +637 -0
- package/src/cache/index.ts +16 -7
- package/src/cache/inputs.ts +246 -733
- package/src/cache/layer.ts +604 -0
- package/src/cache/layered-cache.ts +12 -17
- package/src/cache/output-index.ts +224 -0
- package/src/cache/policy.ts +84 -0
- package/src/cache/run-history.ts +147 -0
- package/src/cache/zstd.ts +160 -0
- package/src/cli/cache.ts +39 -11
- package/src/cli/completions.ts +104 -0
- package/src/cli/core-alias.ts +25 -0
- package/src/cli/help.ts +64 -36
- package/src/cli/index.ts +54 -15
- package/src/cli/info.ts +120 -91
- package/src/cli/init.ts +75 -0
- package/src/cli/last.ts +38 -6
- package/src/cli/lock.ts +2 -1
- package/src/cli/plan-format.ts +14 -3
- package/src/cli/plugin-commands.ts +15 -8
- package/src/cli/run.ts +84 -302
- package/src/cli/select.ts +332 -0
- package/src/cli/show.ts +144 -69
- package/src/cli/upgrade.ts +2 -1
- package/src/cli/watch.ts +647 -97
- package/src/cli/why.ts +44 -14
- package/src/cli/workspace-config.ts +126 -0
- package/src/config.ts +156 -136
- package/src/exec/env.ts +11 -0
- package/src/exec/executor.ts +11 -26
- package/src/exec/index.ts +6 -2
- package/src/{plugins/local-executor/index.ts → exec/local-executor.ts} +12 -16
- package/src/exec/runner.ts +57 -18
- package/src/exec/sandbox-binds.ts +182 -0
- package/src/exec/sandbox-paths.ts +50 -0
- package/src/exec/sandbox-runtime.ts +582 -370
- package/src/exec/sandbox-violations.ts +280 -0
- package/src/graph/index.ts +1 -9
- package/src/graph/scheduler.ts +158 -203
- package/src/graph/task-graph.ts +2 -2
- package/src/index.ts +97 -102
- package/src/orchestrator/admission.ts +137 -0
- package/src/orchestrator/deferred-outputs.ts +3 -0
- package/src/orchestrator/doctor.ts +255 -0
- package/src/orchestrator/events.ts +42 -8
- package/src/orchestrator/execute-task.ts +145 -554
- package/src/orchestrator/failure-mode.ts +169 -28
- package/src/orchestrator/framed-output.ts +41 -36
- package/src/orchestrator/history.ts +118 -98
- package/src/orchestrator/hit-restore.ts +221 -0
- package/src/orchestrator/index.ts +35 -4
- package/src/orchestrator/lockfile-claim.ts +281 -0
- package/src/orchestrator/logger.ts +13 -22
- package/src/orchestrator/metrics.ts +53 -20
- package/src/orchestrator/miss-save.ts +169 -0
- package/src/orchestrator/options.ts +34 -31
- package/src/orchestrator/persistent.ts +79 -0
- package/src/orchestrator/placement.ts +182 -0
- package/src/orchestrator/plugin-host.ts +195 -38
- package/src/orchestrator/plugin.ts +162 -9
- package/src/orchestrator/prepare.ts +77 -132
- package/src/orchestrator/projects.ts +259 -0
- package/src/orchestrator/remote-prefetch.ts +1 -1
- package/src/orchestrator/run-artifacts.ts +25 -3
- package/src/orchestrator/run-context.ts +1 -1
- package/src/orchestrator/run-records.ts +149 -0
- package/src/orchestrator/run-report.ts +2 -2
- package/src/orchestrator/run.ts +246 -597
- package/src/orchestrator/sandbox-request.ts +243 -0
- package/src/orchestrator/save-lane.ts +47 -0
- package/src/orchestrator/signals.ts +111 -0
- package/src/orchestrator/stable-keys.ts +60 -3
- package/src/orchestrator/status-line.ts +2 -1
- package/src/orchestrator/summary.ts +68 -25
- package/src/orchestrator/task-hash.ts +20 -18
- package/src/orchestrator/task-log-buffer.ts +8 -11
- package/src/orchestrator/telemetry-host.ts +11 -1
- package/src/orchestrator/telemetry.ts +3 -17
- package/src/util/cgroup.ts +160 -0
- package/src/util/edit-distance.ts +49 -0
- package/src/util/index.ts +11 -3
- package/src/util/paths.ts +23 -2
- package/src/util/settle.ts +19 -0
- package/src/util/verbs.ts +39 -0
- package/src/workspace/affected.ts +111 -11
- package/src/workspace/config-cache.ts +11 -2
- package/src/workspace/config-eval.ts +11 -4
- package/src/workspace/config-imports.ts +5 -4
- package/src/workspace/config-schema.ts +837 -0
- package/src/workspace/filter.ts +43 -6
- package/src/workspace/fingerprint.ts +37 -3
- package/src/workspace/index.ts +30 -3
- package/src/workspace/lockfile.ts +12 -7
- package/src/{cli → workspace}/migrate-scripts.ts +26 -19
- package/src/{cli/migrate.ts → workspace/migration.ts} +133 -124
- package/src/workspace/package-graph.ts +98 -28
- package/src/workspace/project-loader.ts +76 -746
- package/src/workspace/workspace.ts +98 -13
- package/plugins/local-cache/index.ts +0 -2
- package/plugins/local-executor/index.ts +0 -2
- package/plugins/schedule-history/index.ts +0 -2
- package/src/cache/cas-backend.ts +0 -115
- package/src/cache/digest.ts +0 -32
- package/src/cli/migrate-emit.ts +0 -21
- package/src/cli/migrate-nx.ts +0 -473
- package/src/cli/migrate-turbo.ts +0 -401
- package/src/cli/prune.ts +0 -291
- package/src/orchestrator/missing-plugin.ts +0 -9
- package/src/orchestrator/resources.ts +0 -45
- package/src/orchestrator/verify.ts +0 -174
- package/src/plugins/local-cache/index.ts +0 -11
- package/src/plugins/schedule-history/index.ts +0 -109
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
// The output fingerprint index: per-entry `output_files` rows (size, mode,
|
|
2
|
+
// millisecond mtime) and `output_dirs` rows (directory mtimes), and the two
|
|
3
|
+
// proofs a cache hit runs against the tree before deciding not to restore.
|
|
4
|
+
// Owns its statements over the store's handle; `Cache` delegates, and
|
|
5
|
+
// writes the file rows through `replaceFileRows` inside its own save
|
|
6
|
+
// transaction.
|
|
7
|
+
|
|
8
|
+
/** `mtime_ms` of a recorded output prefix that did not exist when the snapshot was taken. */
|
|
9
|
+
const ABSENT_DIR_MTIME = -1
|
|
10
|
+
|
|
11
|
+
import type { Database, SQLQueryBindings } from 'bun:sqlite'
|
|
12
|
+
import { lstatSync, statSync } from 'node:fs'
|
|
13
|
+
import { lstat, readdir } from 'node:fs/promises'
|
|
14
|
+
import path from 'node:path'
|
|
15
|
+
import {
|
|
16
|
+
OUTPUT_DIRS_CAP,
|
|
17
|
+
OUTPUT_DIRS_RACY_MS,
|
|
18
|
+
type OutputDirRow,
|
|
19
|
+
type OutputFileRow,
|
|
20
|
+
} from './layer.js'
|
|
21
|
+
|
|
22
|
+
export class OutputIndex {
|
|
23
|
+
private readonly insertOutputFile: ReturnType<Database['prepare']>
|
|
24
|
+
private readonly deleteOutputFiles: ReturnType<Database['prepare']>
|
|
25
|
+
private readonly insertOutputDir: ReturnType<Database['prepare']>
|
|
26
|
+
private readonly deleteOutputDirs: ReturnType<Database['prepare']>
|
|
27
|
+
|
|
28
|
+
constructor(private readonly db: Database) {
|
|
29
|
+
this.insertOutputFile = this.db.prepare(`
|
|
30
|
+
INSERT INTO output_files(entry_hash, path, size_bytes, mode, mtime_ms)
|
|
31
|
+
VALUES (?, ?, ?, ?, ?)
|
|
32
|
+
ON CONFLICT(entry_hash, path) DO UPDATE SET
|
|
33
|
+
size_bytes = excluded.size_bytes,
|
|
34
|
+
mode = excluded.mode,
|
|
35
|
+
mtime_ms = excluded.mtime_ms
|
|
36
|
+
`)
|
|
37
|
+
this.deleteOutputFiles = this.db.prepare('DELETE FROM output_files WHERE entry_hash = ?')
|
|
38
|
+
this.insertOutputDir = this.db.prepare(
|
|
39
|
+
'INSERT INTO output_dirs(entry_hash, path, mtime_ms) VALUES (?, ?, ?)',
|
|
40
|
+
)
|
|
41
|
+
this.deleteOutputDirs = this.db.prepare('DELETE FROM output_dirs WHERE entry_hash = ?')
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Replace an entry's file rows. Called INSIDE the store's save transaction,
|
|
46
|
+
* so an entry and its fingerprints land together (an UPDATE on the same
|
|
47
|
+
* hash refreshes rather than appends).
|
|
48
|
+
*/
|
|
49
|
+
replaceFileRows(hash: string, rows: ReadonlyArray<[string, number, number, number]>): void {
|
|
50
|
+
this.deleteOutputFiles.run(hash)
|
|
51
|
+
for (const [rel, size, mode, mtime] of rows) {
|
|
52
|
+
this.insertOutputFile.run(hash, rel, size, mode, mtime)
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
loadOutputFilesBatch(hashes: readonly string[]): Map<string, OutputFileRow[]> {
|
|
57
|
+
const out = new Map<string, OutputFileRow[]>()
|
|
58
|
+
if (hashes.length === 0) return out
|
|
59
|
+
// Inline placeholders for an IN-list — bun:sqlite doesn't ship
|
|
60
|
+
// rarray, but `IN (?, ?, …)` with N≤~999 is fast and avoids per-
|
|
61
|
+
// hash select.get() overhead. `db.query` (not `db.prepare`) caches the
|
|
62
|
+
// compiled statement keyed by the SQL text — so the dominant single-hash
|
|
63
|
+
// warm-hit path (called up to 3× per hit) reuses one statement instead of
|
|
64
|
+
// recompiling on every call.
|
|
65
|
+
const placeholders = hashes.map(() => '?').join(',')
|
|
66
|
+
const stmt = this.db.query(
|
|
67
|
+
`SELECT entry_hash, path, size_bytes, mode, mtime_ms FROM output_files WHERE entry_hash IN (${placeholders})`,
|
|
68
|
+
)
|
|
69
|
+
const rows = stmt.all(...(hashes as readonly SQLQueryBindings[])) as Array<{
|
|
70
|
+
entry_hash: string
|
|
71
|
+
path: string
|
|
72
|
+
size_bytes: number
|
|
73
|
+
mode: number
|
|
74
|
+
mtime_ms: number
|
|
75
|
+
}>
|
|
76
|
+
for (const r of rows) {
|
|
77
|
+
let list = out.get(r.entry_hash)
|
|
78
|
+
if (!list) {
|
|
79
|
+
list = []
|
|
80
|
+
out.set(r.entry_hash, list)
|
|
81
|
+
}
|
|
82
|
+
list.push({ path: r.path, size: r.size_bytes, mode: r.mode, mtimeMs: r.mtime_ms })
|
|
83
|
+
}
|
|
84
|
+
return out
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
async isOutputsCurrent(projectDir: string, expected: readonly OutputFileRow[]): Promise<boolean> {
|
|
88
|
+
// Empty manifest case (a task produced no outputs) → trivially
|
|
89
|
+
// current; the on-disk tree under projectDir is whatever it was,
|
|
90
|
+
// and nothing was supposed to land there.
|
|
91
|
+
if (expected.length === 0) return true
|
|
92
|
+
// statSync, deliberately. The async form measured faster on 2026-09-02,
|
|
93
|
+
// when a hit walked its whole output tree; since the directory
|
|
94
|
+
// short-circuit (`outputDirsCurrent`) a warm hit stats a handful of
|
|
95
|
+
// paths, and each async stat costs a thread-pool round trip the
|
|
96
|
+
// scheduler cannot overlap away (its concurrency is the worker count).
|
|
97
|
+
// Re-measured 2026-09-09, 1,000 warm hits, interleaved A/B: the run
|
|
98
|
+
// graph stage 100 → 54 ms, the whole process 422 → 368 ms.
|
|
99
|
+
const results = expected.map((e) => {
|
|
100
|
+
try {
|
|
101
|
+
const s = statSync(path.join(projectDir, e.path))
|
|
102
|
+
return (
|
|
103
|
+
s.size === e.size &&
|
|
104
|
+
(s.mode & 0o777) === (e.mode & 0o777) &&
|
|
105
|
+
// MILLISECOND comparison (sub-ms tolerance for the float
|
|
106
|
+
// round-trip through utimes). Save rows carry stat-ms and
|
|
107
|
+
// restoreOutputs re-syncs restored files to the row value,
|
|
108
|
+
// so equality holds exactly in steady state; legacy
|
|
109
|
+
// second-precision rows converge on their first restore.
|
|
110
|
+
// Residual blind spot: a same-size edit landing in the SAME
|
|
111
|
+
// millisecond as the recorded write, or a deliberately
|
|
112
|
+
// forged mtime (touch -r) — the trade every mtime-based
|
|
113
|
+
// skip check accepts.
|
|
114
|
+
Math.abs(s.mtimeMs - e.mtimeMs) < 1
|
|
115
|
+
)
|
|
116
|
+
} catch {
|
|
117
|
+
return false
|
|
118
|
+
}
|
|
119
|
+
})
|
|
120
|
+
return results.every(Boolean)
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Snapshot every directory under each of `prefixes` for `hash`. Called
|
|
125
|
+
* after a save and after a restore, when the tree is known to equal the
|
|
126
|
+
* entry's set. Symlinked directories are not descended (the output walk
|
|
127
|
+
* refuses them too). Over `OUTPUT_DIRS_CAP` directories, or on any
|
|
128
|
+
* error, the rows are cleared and the next hit keeps the walk.
|
|
129
|
+
*/
|
|
130
|
+
async recordOutputDirs(
|
|
131
|
+
hash: string,
|
|
132
|
+
projectDir: string,
|
|
133
|
+
prefixes: readonly string[],
|
|
134
|
+
): Promise<void> {
|
|
135
|
+
const rows: Array<[string, number]> = []
|
|
136
|
+
const walk = async (rel: string, isPrefix = false): Promise<boolean> => {
|
|
137
|
+
const abs = path.join(projectDir, rel)
|
|
138
|
+
let st
|
|
139
|
+
try {
|
|
140
|
+
st = await lstat(abs)
|
|
141
|
+
} catch {
|
|
142
|
+
// A declared prefix the task never produced (`build/**` beside
|
|
143
|
+
// `dist/**`, medusa's `.medusa/**`) is recorded ABSENT: the check
|
|
144
|
+
// then asks that it still not exist. Refusing the snapshot instead
|
|
145
|
+
// made every such task re-glob its outputs on every warm hit — all
|
|
146
|
+
// 83 of medusa's, 210 ms of a 2.5 s no-op (2026-09-11).
|
|
147
|
+
if (isPrefix) {
|
|
148
|
+
rows.push([rel, ABSENT_DIR_MTIME])
|
|
149
|
+
return true
|
|
150
|
+
}
|
|
151
|
+
return false
|
|
152
|
+
}
|
|
153
|
+
if (!st.isDirectory()) return false
|
|
154
|
+
rows.push([rel, st.mtimeMs])
|
|
155
|
+
if (rows.length > OUTPUT_DIRS_CAP) return false
|
|
156
|
+
let entries
|
|
157
|
+
try {
|
|
158
|
+
entries = await readdir(abs, { withFileTypes: true })
|
|
159
|
+
} catch {
|
|
160
|
+
return false
|
|
161
|
+
}
|
|
162
|
+
for (const e of entries) {
|
|
163
|
+
if (e.isDirectory() && !e.isSymbolicLink()) {
|
|
164
|
+
if (!(await walk(`${rel}/${e.name}`))) return false
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
return true
|
|
168
|
+
}
|
|
169
|
+
let ok = true
|
|
170
|
+
for (const prefix of prefixes) {
|
|
171
|
+
if (!(await walk(prefix, true))) {
|
|
172
|
+
ok = false
|
|
173
|
+
break
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
// All or nothing: a racy directory dropped alone would leave its
|
|
177
|
+
// parent trusted while an addition inside it bumps only the dropped one.
|
|
178
|
+
const youngest = Date.now() - OUTPUT_DIRS_RACY_MS
|
|
179
|
+
if (rows.some(([, mtime]) => mtime > youngest)) ok = false
|
|
180
|
+
this.db.transaction(() => {
|
|
181
|
+
this.deleteOutputDirs.run(hash)
|
|
182
|
+
if (!ok) return
|
|
183
|
+
for (const [rel, mtime] of rows) this.insertOutputDir.run(hash, rel, mtime)
|
|
184
|
+
})()
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
loadOutputDirsBatch(hashes: readonly string[]): Map<string, OutputDirRow[]> {
|
|
188
|
+
const out = new Map<string, OutputDirRow[]>()
|
|
189
|
+
if (hashes.length === 0) return out
|
|
190
|
+
const placeholders = hashes.map(() => '?').join(',')
|
|
191
|
+
const rows = this.db
|
|
192
|
+
.query(
|
|
193
|
+
`SELECT entry_hash, path, mtime_ms FROM output_dirs WHERE entry_hash IN (${placeholders})`,
|
|
194
|
+
)
|
|
195
|
+
.all(...(hashes as readonly SQLQueryBindings[])) as Array<{
|
|
196
|
+
entry_hash: string
|
|
197
|
+
path: string
|
|
198
|
+
mtime_ms: number
|
|
199
|
+
}>
|
|
200
|
+
for (const r of rows) {
|
|
201
|
+
const list = out.get(r.entry_hash)
|
|
202
|
+
const row = { path: r.path, mtimeMs: r.mtime_ms }
|
|
203
|
+
if (list) list.push(row)
|
|
204
|
+
else out.set(r.entry_hash, [row])
|
|
205
|
+
}
|
|
206
|
+
return out
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/** True iff every recorded directory exists with its recorded mtime (ms). Same forged-mtime trade as the file check. */
|
|
210
|
+
async outputDirsCurrent(projectDir: string, rows: readonly OutputDirRow[]): Promise<boolean> {
|
|
211
|
+
if (rows.length === 0) return false
|
|
212
|
+
const results = rows.map((r) => {
|
|
213
|
+
let st
|
|
214
|
+
try {
|
|
215
|
+
st = lstatSync(path.join(projectDir, r.path))
|
|
216
|
+
} catch {
|
|
217
|
+
return r.mtimeMs === ABSENT_DIR_MTIME
|
|
218
|
+
}
|
|
219
|
+
if (r.mtimeMs === ABSENT_DIR_MTIME) return false
|
|
220
|
+
return st.isDirectory() && Math.abs(st.mtimeMs - r.mtimeMs) < 1
|
|
221
|
+
})
|
|
222
|
+
return results.every(Boolean)
|
|
223
|
+
}
|
|
224
|
+
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
// The run-level cache policy: four axes (local/remote × read/write) and
|
|
2
|
+
// the `--cache=<spec>` grammar that sets them. Pure data + parsing; no I/O.
|
|
3
|
+
|
|
4
|
+
import { UserError } from '../util/index.js'
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Independent read/write control over the two cache layers (local +
|
|
8
|
+
* remote). Replaces the old single `noCache` boolean: each axis can be
|
|
9
|
+
* toggled on its own so `--force` (re-execute but still refresh the
|
|
10
|
+
* cache) is distinct from `--no-cache` (disable everything).
|
|
11
|
+
*
|
|
12
|
+
* Only the task-artifact get/save path is gated. `recordRun`, `stats`,
|
|
13
|
+
* `prune`, key derivation, and prefetch-ingest are never affected — they
|
|
14
|
+
* are bookkeeping/analytics that a run policy has no business disabling.
|
|
15
|
+
*/
|
|
16
|
+
export interface CachePolicy {
|
|
17
|
+
localRead: boolean
|
|
18
|
+
localWrite: boolean
|
|
19
|
+
remoteRead: boolean
|
|
20
|
+
remoteWrite: boolean
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** All four axes on — the default when no cache flag is passed. */
|
|
24
|
+
export const FULL_CACHE_POLICY: CachePolicy = {
|
|
25
|
+
localRead: true,
|
|
26
|
+
localWrite: true,
|
|
27
|
+
remoteRead: true,
|
|
28
|
+
remoteWrite: true,
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Parse a `--cache=<spec>` value into a `CachePolicy`, starting from a
|
|
33
|
+
* base (defaults to {@link FULL_CACHE_POLICY}). The spec is a
|
|
34
|
+
* comma-separated list of `layer:flags` segments where `layer` is
|
|
35
|
+
* `local` or `remote` and `flags` is any subset of `r` (read) and `w`
|
|
36
|
+
* (write), order-independent and possibly empty.
|
|
37
|
+
*
|
|
38
|
+
* A mentioned layer is set EXACTLY to its flags (read = includes `r`,
|
|
39
|
+
* write = includes `w`); an unmentioned layer keeps its base value.
|
|
40
|
+
* So `local:rw,remote:r` = remote read-only, `remote:` = remote fully
|
|
41
|
+
* off, `local:r` = local read-only with remote untouched.
|
|
42
|
+
*
|
|
43
|
+
* Throws a {@link UserError} on an unknown layer, an unknown flag, a
|
|
44
|
+
* duplicated flag, a missing colon, or a repeated layer.
|
|
45
|
+
*/
|
|
46
|
+
export function parseCachePolicy(spec: string, base: CachePolicy = FULL_CACHE_POLICY): CachePolicy {
|
|
47
|
+
const out: CachePolicy = { ...base }
|
|
48
|
+
const seen = new Set<string>()
|
|
49
|
+
for (const rawSeg of spec.split(',')) {
|
|
50
|
+
const seg = rawSeg.trim()
|
|
51
|
+
if (seg.length === 0) continue
|
|
52
|
+
const colon = seg.indexOf(':')
|
|
53
|
+
if (colon < 0) {
|
|
54
|
+
throw new UserError(`invalid --cache segment '${seg}': expected '<layer>:<flags>'`)
|
|
55
|
+
}
|
|
56
|
+
const layer = seg.slice(0, colon)
|
|
57
|
+
const flags = seg.slice(colon + 1)
|
|
58
|
+
if (layer !== 'local' && layer !== 'remote') {
|
|
59
|
+
throw new UserError(`invalid --cache layer '${layer}': expected 'local' or 'remote'`)
|
|
60
|
+
}
|
|
61
|
+
if (seen.has(layer)) {
|
|
62
|
+
throw new UserError(`--cache layer '${layer}' specified twice`)
|
|
63
|
+
}
|
|
64
|
+
seen.add(layer)
|
|
65
|
+
const flagSet = new Set<string>()
|
|
66
|
+
for (const ch of flags) {
|
|
67
|
+
if (ch !== 'r' && ch !== 'w') {
|
|
68
|
+
throw new UserError(`invalid --cache flag '${ch}' for '${layer}': expected 'r' and/or 'w'`)
|
|
69
|
+
}
|
|
70
|
+
if (flagSet.has(ch)) {
|
|
71
|
+
throw new UserError(`--cache flag '${ch}' repeated for '${layer}'`)
|
|
72
|
+
}
|
|
73
|
+
flagSet.add(ch)
|
|
74
|
+
}
|
|
75
|
+
if (layer === 'local') {
|
|
76
|
+
out.localRead = flagSet.has('r')
|
|
77
|
+
out.localWrite = flagSet.has('w')
|
|
78
|
+
} else {
|
|
79
|
+
out.remoteRead = flagSet.has('r')
|
|
80
|
+
out.remoteWrite = flagSet.has('w')
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
return out
|
|
84
|
+
}
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
// The run history: one `runs` row per task outcome and one `invocations`
|
|
2
|
+
// header per `vx run`, written together, plus their 30-day retention.
|
|
3
|
+
// What `vx why`, `vx last`, `stats` and the history reader read. Owns
|
|
4
|
+
// its statements over the store's handle; `Cache` delegates.
|
|
5
|
+
|
|
6
|
+
import type { Database, SQLQueryBindings } from 'bun:sqlite'
|
|
7
|
+
import type { InvocationRecord, RunRecord } from './layer.js'
|
|
8
|
+
|
|
9
|
+
export class RunHistory {
|
|
10
|
+
private readonly insertRun: ReturnType<Database['prepare']>
|
|
11
|
+
private readonly insertInvocation: ReturnType<Database['prepare']>
|
|
12
|
+
|
|
13
|
+
constructor(private readonly db: Database) {
|
|
14
|
+
this.insertRun = this.db.prepare(`
|
|
15
|
+
INSERT INTO runs(
|
|
16
|
+
hash, project, task, status, exit_code, duration_ms, forward_args,
|
|
17
|
+
started_at, ended_at,
|
|
18
|
+
run_id, cpu_ms, peak_rss_bytes, wallclock_start_ns, wallclock_end_ns,
|
|
19
|
+
cache_hit, attempts, cached
|
|
20
|
+
)
|
|
21
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
22
|
+
`)
|
|
23
|
+
this.insertInvocation = this.db.prepare(`
|
|
24
|
+
INSERT INTO invocations(
|
|
25
|
+
run_id, command, requested_tasks, cache_policy, concurrency, flow,
|
|
26
|
+
started_at, ended_at, total_duration_ms,
|
|
27
|
+
task_count, failed_count, hit_count, hit_local_count, hit_remote_count,
|
|
28
|
+
exit_ok,
|
|
29
|
+
commit_sha, branch, dirty, ci, ci_provider,
|
|
30
|
+
host, os, arch, vx_version, tags
|
|
31
|
+
)
|
|
32
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
33
|
+
ON CONFLICT(run_id) DO NOTHING
|
|
34
|
+
`)
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
recordRun(run: RunRecord): void {
|
|
38
|
+
this.insertRun.run(...bindRun(run))
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
recordRuns(runs: readonly RunRecord[]): void {
|
|
42
|
+
if (runs.length === 0) return
|
|
43
|
+
if (runs.length === 1) {
|
|
44
|
+
this.insertRun.run(...bindRun(runs[0]!))
|
|
45
|
+
return
|
|
46
|
+
}
|
|
47
|
+
// `bun:sqlite`'s `transaction()` returns a callable that wraps the
|
|
48
|
+
// body in BEGIN/COMMIT, fsyncing once at the end. For a 200-task
|
|
49
|
+
// run that's one fsync instead of 200.
|
|
50
|
+
const insert = this.insertRun
|
|
51
|
+
const tx = this.db.transaction((batch: readonly RunRecord[]) => {
|
|
52
|
+
for (const r of batch) insert.run(...bindRun(r))
|
|
53
|
+
})
|
|
54
|
+
tx(runs)
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
recordRunBundle(bundle: { runs: readonly RunRecord[]; invocation: InvocationRecord }): void {
|
|
58
|
+
// Whole run records atomically — one transaction, one fsync: the
|
|
59
|
+
// per-task `runs` rows and the one `invocations` header. The
|
|
60
|
+
// input-fingerprint rows do NOT live here: they're written inside
|
|
61
|
+
// the entry-save transaction (`save`/`ingest`) so a warm
|
|
62
|
+
// all-cache-hit run — which writes no `runs`-vs-`entry_inputs`
|
|
63
|
+
// mismatch — pays nothing for the moat it isn't refreshing.
|
|
64
|
+
const insertRun = this.insertRun
|
|
65
|
+
const insertInvocation = this.insertInvocation
|
|
66
|
+
this.db.transaction(() => {
|
|
67
|
+
for (const r of bundle.runs) insertRun.run(...bindRun(r))
|
|
68
|
+
insertInvocation.run(...bindInvocation(bundle.invocation))
|
|
69
|
+
})()
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Retention: the runs table grows by one row per executed task per
|
|
74
|
+
* invocation — a 2000-task repo accretes ~20k rows in days. 30 days
|
|
75
|
+
* covers `vx stats` (24 h windows) and CI-side consumers. The
|
|
76
|
+
* `invocations` header is pruned on the SAME window, or a header would
|
|
77
|
+
* outlive its rows and `vx last` would list a run whose detail is gone.
|
|
78
|
+
*/
|
|
79
|
+
pruneOlderThan(cutoff: number): void {
|
|
80
|
+
this.db.prepare('DELETE FROM runs WHERE started_at < ?').run(cutoff)
|
|
81
|
+
this.db.prepare('DELETE FROM invocations WHERE started_at < ?').run(cutoff)
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Bind a RunRecord to the positional parameters expected by the
|
|
87
|
+
* `insertRun` prepared statement (17 columns). Shared between the
|
|
88
|
+
* single and batched record paths.
|
|
89
|
+
*/
|
|
90
|
+
function bindRun(run: RunRecord): SQLQueryBindings[] {
|
|
91
|
+
return [
|
|
92
|
+
// The ONE place the no-key sentinel is applied, so the column's
|
|
93
|
+
// NOT NULL invariant can't be violated from a call site.
|
|
94
|
+
run.hash ?? '',
|
|
95
|
+
run.project,
|
|
96
|
+
run.task,
|
|
97
|
+
run.status,
|
|
98
|
+
run.exitCode,
|
|
99
|
+
run.durationMs,
|
|
100
|
+
run.forwardArgs ? JSON.stringify(run.forwardArgs) : null,
|
|
101
|
+
run.startedAt,
|
|
102
|
+
run.endedAt,
|
|
103
|
+
run.runId ?? null,
|
|
104
|
+
run.cpuMs ?? null,
|
|
105
|
+
run.peakRssBytes ?? null,
|
|
106
|
+
run.wallclockStartNs !== undefined ? run.wallclockStartNs : null,
|
|
107
|
+
run.wallclockEndNs !== undefined ? run.wallclockEndNs : null,
|
|
108
|
+
run.cacheHit === undefined ? null : run.cacheHit ? 1 : 0,
|
|
109
|
+
run.attempts ?? null,
|
|
110
|
+
run.cached === undefined ? null : run.cached ? 1 : 0,
|
|
111
|
+
]
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Bind an InvocationRecord to the positional parameters of the
|
|
116
|
+
* `insertInvocation` prepared statement (25 columns). Booleans map to
|
|
117
|
+
* 0/1; null-or-bool columns (`dirty`) keep null distinct from 0.
|
|
118
|
+
*/
|
|
119
|
+
function bindInvocation(inv: InvocationRecord): SQLQueryBindings[] {
|
|
120
|
+
return [
|
|
121
|
+
inv.runId,
|
|
122
|
+
inv.command,
|
|
123
|
+
inv.requestedTasks,
|
|
124
|
+
inv.cachePolicy,
|
|
125
|
+
inv.concurrency,
|
|
126
|
+
inv.flow,
|
|
127
|
+
inv.startedAt,
|
|
128
|
+
inv.endedAt,
|
|
129
|
+
inv.totalDurationMs,
|
|
130
|
+
inv.taskCount,
|
|
131
|
+
inv.failedCount,
|
|
132
|
+
inv.hitCount,
|
|
133
|
+
inv.hitLocalCount,
|
|
134
|
+
inv.hitRemoteCount,
|
|
135
|
+
inv.exitOk ? 1 : 0,
|
|
136
|
+
inv.commitSha,
|
|
137
|
+
inv.branch,
|
|
138
|
+
inv.dirty === null ? null : inv.dirty ? 1 : 0,
|
|
139
|
+
inv.ci ? 1 : 0,
|
|
140
|
+
inv.ciProvider,
|
|
141
|
+
inv.host,
|
|
142
|
+
inv.os,
|
|
143
|
+
inv.arch,
|
|
144
|
+
inv.vxVersion,
|
|
145
|
+
inv.tags,
|
|
146
|
+
]
|
|
147
|
+
}
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
// zstd framing for artifacts: the declared-size gate that refuses a
|
|
2
|
+
// decompression bomb before it can expand, and the bounded one-call /
|
|
3
|
+
// streamed decoders the store and the ingest path share.
|
|
4
|
+
|
|
5
|
+
import { CorruptArtifactError } from './layer.js'
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* A decompressed artifact above this is refused as a zstd bomb rather than
|
|
9
|
+
* expanded into memory. 2 GiB comfortably exceeds any real build output while
|
|
10
|
+
* bounding a malicious/compromised remote's ability to OOM a victim who takes
|
|
11
|
+
* a cache hit.
|
|
12
|
+
*/
|
|
13
|
+
const MAX_DECOMPRESSED_ARTIFACT_BYTES = 2 * 1024 * 1024 * 1024
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Read a zstd frame's declared Frame_Content_Size (RFC 8878 §3.1.1) WITHOUT
|
|
17
|
+
* decompressing. Returns null when the frame omits it (streaming frames) or
|
|
18
|
+
* the header is too short to parse. vx's own producer (single-shot
|
|
19
|
+
* `Bun.zstdCompress` of a known buffer) always writes it, so an artifact that
|
|
20
|
+
* declares an enormous size can be rejected before a byte is allocated.
|
|
21
|
+
*
|
|
22
|
+
* Exported for tests: pins the per-`fcsFlag` byte layouts (incl. the 2-byte
|
|
23
|
+
* `+256` adjustment and the `dictIdFlag` offset) that a bomb-refusal e2e can't
|
|
24
|
+
* discriminate (their max declarable sizes sit below the ceiling).
|
|
25
|
+
*/
|
|
26
|
+
export function zstdContentSize(b: Uint8Array): bigint | null {
|
|
27
|
+
if (b.length < 5) return null
|
|
28
|
+
// Magic_Number 0xFD2FB528, little-endian.
|
|
29
|
+
if (b[0] !== 0x28 || b[1] !== 0xb5 || b[2] !== 0x2f || b[3] !== 0xfd) return null
|
|
30
|
+
const desc = b[4]!
|
|
31
|
+
const fcsFlag = desc >> 6
|
|
32
|
+
const singleSegment = (desc >> 5) & 1
|
|
33
|
+
const dictIdFlag = desc & 3
|
|
34
|
+
let off = 5
|
|
35
|
+
if (singleSegment === 0) off += 1 // Window_Descriptor byte
|
|
36
|
+
off += dictIdFlag === 3 ? 4 : dictIdFlag // Dictionary_ID: 0,1,2,4 bytes
|
|
37
|
+
let fcsSize: number
|
|
38
|
+
if (fcsFlag === 0) fcsSize = singleSegment === 1 ? 1 : 0
|
|
39
|
+
else if (fcsFlag === 1) fcsSize = 2
|
|
40
|
+
else if (fcsFlag === 2) fcsSize = 4
|
|
41
|
+
else fcsSize = 8
|
|
42
|
+
if (fcsSize === 0 || b.length < off + fcsSize) return null
|
|
43
|
+
let v = 0n
|
|
44
|
+
for (let i = 0; i < fcsSize; i++) v |= BigInt(b[off + i]!) << BigInt(8 * i)
|
|
45
|
+
if (fcsSize === 2) v += 256n // per spec, the 2-byte field stores value − 256
|
|
46
|
+
return v
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Decompress a zstd artifact that DECLARES its content size, with a hard
|
|
51
|
+
* output ceiling: refused before a byte is allocated when the declaration
|
|
52
|
+
* is over the cap, and again on the actual length. A frame with no
|
|
53
|
+
* declaration (a streamed producer's — vx's own, above 4 MiB) never comes
|
|
54
|
+
* here: `decodedTar` decodes it as a stream under the running count, so
|
|
55
|
+
* a sizeless bomb has nowhere to expand.
|
|
56
|
+
*/
|
|
57
|
+
async function zstdDecompressBounded(compressed: Uint8Array, hash: string): Promise<Uint8Array> {
|
|
58
|
+
assertDeclaredSize(compressed, hash)
|
|
59
|
+
const out = await Bun.zstdDecompress(compressed)
|
|
60
|
+
if (out.length > MAX_DECOMPRESSED_ARTIFACT_BYTES) {
|
|
61
|
+
throw new CorruptArtifactError(
|
|
62
|
+
hash,
|
|
63
|
+
`decompressed to ${out.length} bytes (> ${MAX_DECOMPRESSED_ARTIFACT_BYTES} cap)`,
|
|
64
|
+
)
|
|
65
|
+
}
|
|
66
|
+
return out
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** The pre-decompress half of the ceiling: the frame header's own claim, when it makes one. */
|
|
70
|
+
function assertDeclaredSize(compressed: Uint8Array, hash: string): bigint | null {
|
|
71
|
+
const declared = zstdContentSize(compressed)
|
|
72
|
+
if (declared !== null && declared > BigInt(MAX_DECOMPRESSED_ARTIFACT_BYTES)) {
|
|
73
|
+
throw new CorruptArtifactError(
|
|
74
|
+
hash,
|
|
75
|
+
`declares ${declared} decompressed bytes (> ${MAX_DECOMPRESSED_ARTIFACT_BYTES} cap)`,
|
|
76
|
+
)
|
|
77
|
+
}
|
|
78
|
+
return declared
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** Compressed artifacts above this size are decoded as a stream, on restore and on ingest. */
|
|
82
|
+
export const STREAM_DECODE_FROM = 4 * 1024 * 1024
|
|
83
|
+
|
|
84
|
+
/** Collect a byte stream; only ever used where the seam wants bytes. */
|
|
85
|
+
export async function bytesOf(stream: ReadableStream<Uint8Array>): Promise<Uint8Array> {
|
|
86
|
+
return new Uint8Array(await new Response(stream).arrayBuffer())
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Bun's `CompressionStream` types do not satisfy `pipeThrough`'s pair
|
|
91
|
+
* (its readable side is typed `NonSharedUint8Array`); the runtime object
|
|
92
|
+
* is a plain byte transform.
|
|
93
|
+
*/
|
|
94
|
+
export const zstdEncoder = (): TransformStream<Uint8Array, Uint8Array> =>
|
|
95
|
+
new CompressionStream('zstd') as unknown as TransformStream<Uint8Array, Uint8Array>
|
|
96
|
+
|
|
97
|
+
const oneChunk = (bytes: Uint8Array): ReadableStream<Uint8Array> =>
|
|
98
|
+
new ReadableStream({
|
|
99
|
+
start(c) {
|
|
100
|
+
c.enqueue(bytes)
|
|
101
|
+
c.close()
|
|
102
|
+
},
|
|
103
|
+
})
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* The decoded tar as a stream: one call for a small artifact that
|
|
107
|
+
* declares its size, a streamed decode otherwise — the stream setup is
|
|
108
|
+
* ~35 µs, which matters at a thousand one-file artifacts and nowhere
|
|
109
|
+
* else. Bytes in memory are only ever the small case (a large source
|
|
110
|
+
* must be a FILE: a Blob copies its bytes and hands the decoder
|
|
111
|
+
* everything at once, measured +519 MiB on 150 MiB against +448 for the
|
|
112
|
+
* plain decode). The 2 GiB ceiling applies either way: the declaration
|
|
113
|
+
* and the result length in one call, a running count on the stream.
|
|
114
|
+
*/
|
|
115
|
+
export async function decodedTar(
|
|
116
|
+
source: Uint8Array | Bun.BunFile,
|
|
117
|
+
hash: string,
|
|
118
|
+
): Promise<ReadableStream<Uint8Array>> {
|
|
119
|
+
if (source instanceof Uint8Array) {
|
|
120
|
+
if (assertDeclaredSize(source, hash) === null) return zstdDecodeStream(new Blob([source]), hash)
|
|
121
|
+
return oneChunk(await zstdDecompressBounded(source, hash))
|
|
122
|
+
}
|
|
123
|
+
if (source.size <= STREAM_DECODE_FROM) {
|
|
124
|
+
const bytes = await source.bytes()
|
|
125
|
+
if (assertDeclaredSize(bytes, hash) === null) return zstdDecodeStream(source, hash)
|
|
126
|
+
return oneChunk(await zstdDecompressBounded(bytes, hash))
|
|
127
|
+
}
|
|
128
|
+
assertDeclaredSize(await source.slice(0, 32).bytes(), hash)
|
|
129
|
+
return zstdDecodeStream(source, hash)
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* The streaming twin of `zstdDecompressBounded`: the decoded bytes as a
|
|
134
|
+
* stream, refused past the same ceiling. A malicious frame cannot expand
|
|
135
|
+
* past the cap here either — the count runs as bytes are produced, before
|
|
136
|
+
* any of them reach the reader's next entry.
|
|
137
|
+
*/
|
|
138
|
+
function zstdDecodeStream(source: Blob, hash: string): ReadableStream<Uint8Array> {
|
|
139
|
+
let total = 0
|
|
140
|
+
return source
|
|
141
|
+
.stream()
|
|
142
|
+
.pipeThrough(new DecompressionStream('zstd'))
|
|
143
|
+
.pipeThrough(
|
|
144
|
+
new TransformStream<Uint8Array, Uint8Array>({
|
|
145
|
+
transform(chunk, controller) {
|
|
146
|
+
total += chunk.byteLength
|
|
147
|
+
if (total > MAX_DECOMPRESSED_ARTIFACT_BYTES) {
|
|
148
|
+
controller.error(
|
|
149
|
+
new CorruptArtifactError(
|
|
150
|
+
hash,
|
|
151
|
+
`decompresses past ${MAX_DECOMPRESSED_ARTIFACT_BYTES} bytes (cap)`,
|
|
152
|
+
),
|
|
153
|
+
)
|
|
154
|
+
return
|
|
155
|
+
}
|
|
156
|
+
controller.enqueue(chunk)
|
|
157
|
+
},
|
|
158
|
+
}),
|
|
159
|
+
)
|
|
160
|
+
}
|