@vzn/vx 0.0.18 → 0.0.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +99 -65
- package/package.json +10 -20
- package/src/bin.ts +12 -2
- package/src/cache/archive.ts +120 -18
- package/src/cache/cache.ts +313 -1324
- package/src/cache/chained-cache.ts +0 -8
- package/src/cache/config-evals.ts +89 -0
- package/src/cache/file-hashes.ts +262 -0
- package/src/cache/git-inputs.ts +637 -0
- package/src/cache/index.ts +16 -7
- package/src/cache/inputs.ts +246 -733
- package/src/cache/layer.ts +604 -0
- package/src/cache/layered-cache.ts +12 -17
- package/src/cache/output-index.ts +224 -0
- package/src/cache/policy.ts +84 -0
- package/src/cache/run-history.ts +147 -0
- package/src/cache/zstd.ts +160 -0
- package/src/cli/cache.ts +39 -11
- package/src/cli/completions.ts +104 -0
- package/src/cli/core-alias.ts +25 -0
- package/src/cli/help.ts +64 -36
- package/src/cli/index.ts +54 -15
- package/src/cli/info.ts +120 -91
- package/src/cli/init.ts +75 -0
- package/src/cli/last.ts +38 -6
- package/src/cli/lock.ts +2 -1
- package/src/cli/plan-format.ts +14 -3
- package/src/cli/plugin-commands.ts +15 -8
- package/src/cli/run.ts +84 -302
- package/src/cli/select.ts +332 -0
- package/src/cli/show.ts +144 -69
- package/src/cli/upgrade.ts +2 -1
- package/src/cli/watch.ts +647 -97
- package/src/cli/why.ts +44 -14
- package/src/cli/workspace-config.ts +126 -0
- package/src/config.ts +156 -136
- package/src/exec/env.ts +11 -0
- package/src/exec/executor.ts +11 -26
- package/src/exec/index.ts +6 -2
- package/src/{plugins/local-executor/index.ts → exec/local-executor.ts} +12 -16
- package/src/exec/runner.ts +57 -18
- package/src/exec/sandbox-binds.ts +182 -0
- package/src/exec/sandbox-paths.ts +50 -0
- package/src/exec/sandbox-runtime.ts +582 -370
- package/src/exec/sandbox-violations.ts +280 -0
- package/src/graph/index.ts +1 -9
- package/src/graph/scheduler.ts +158 -203
- package/src/graph/task-graph.ts +2 -2
- package/src/index.ts +97 -102
- package/src/orchestrator/admission.ts +137 -0
- package/src/orchestrator/deferred-outputs.ts +3 -0
- package/src/orchestrator/doctor.ts +255 -0
- package/src/orchestrator/events.ts +42 -8
- package/src/orchestrator/execute-task.ts +145 -554
- package/src/orchestrator/failure-mode.ts +169 -28
- package/src/orchestrator/framed-output.ts +41 -36
- package/src/orchestrator/history.ts +118 -98
- package/src/orchestrator/hit-restore.ts +221 -0
- package/src/orchestrator/index.ts +35 -4
- package/src/orchestrator/lockfile-claim.ts +281 -0
- package/src/orchestrator/logger.ts +13 -22
- package/src/orchestrator/metrics.ts +53 -20
- package/src/orchestrator/miss-save.ts +169 -0
- package/src/orchestrator/options.ts +34 -31
- package/src/orchestrator/persistent.ts +79 -0
- package/src/orchestrator/placement.ts +182 -0
- package/src/orchestrator/plugin-host.ts +195 -38
- package/src/orchestrator/plugin.ts +162 -9
- package/src/orchestrator/prepare.ts +77 -132
- package/src/orchestrator/projects.ts +259 -0
- package/src/orchestrator/remote-prefetch.ts +1 -1
- package/src/orchestrator/run-artifacts.ts +25 -3
- package/src/orchestrator/run-context.ts +1 -1
- package/src/orchestrator/run-records.ts +149 -0
- package/src/orchestrator/run-report.ts +2 -2
- package/src/orchestrator/run.ts +246 -597
- package/src/orchestrator/sandbox-request.ts +243 -0
- package/src/orchestrator/save-lane.ts +47 -0
- package/src/orchestrator/signals.ts +111 -0
- package/src/orchestrator/stable-keys.ts +60 -3
- package/src/orchestrator/status-line.ts +2 -1
- package/src/orchestrator/summary.ts +68 -25
- package/src/orchestrator/task-hash.ts +20 -18
- package/src/orchestrator/task-log-buffer.ts +8 -11
- package/src/orchestrator/telemetry-host.ts +11 -1
- package/src/orchestrator/telemetry.ts +3 -17
- package/src/util/cgroup.ts +160 -0
- package/src/util/edit-distance.ts +49 -0
- package/src/util/index.ts +11 -3
- package/src/util/paths.ts +23 -2
- package/src/util/settle.ts +19 -0
- package/src/util/verbs.ts +39 -0
- package/src/workspace/affected.ts +111 -11
- package/src/workspace/config-cache.ts +11 -2
- package/src/workspace/config-eval.ts +11 -4
- package/src/workspace/config-imports.ts +5 -4
- package/src/workspace/config-schema.ts +837 -0
- package/src/workspace/filter.ts +43 -6
- package/src/workspace/fingerprint.ts +37 -3
- package/src/workspace/index.ts +30 -3
- package/src/workspace/lockfile.ts +12 -7
- package/src/{cli → workspace}/migrate-scripts.ts +26 -19
- package/src/{cli/migrate.ts → workspace/migration.ts} +133 -124
- package/src/workspace/package-graph.ts +98 -28
- package/src/workspace/project-loader.ts +76 -746
- package/src/workspace/workspace.ts +98 -13
- package/plugins/local-cache/index.ts +0 -2
- package/plugins/local-executor/index.ts +0 -2
- package/plugins/schedule-history/index.ts +0 -2
- package/src/cache/cas-backend.ts +0 -115
- package/src/cache/digest.ts +0 -32
- package/src/cli/migrate-emit.ts +0 -21
- package/src/cli/migrate-nx.ts +0 -473
- package/src/cli/migrate-turbo.ts +0 -401
- package/src/cli/prune.ts +0 -291
- package/src/orchestrator/missing-plugin.ts +0 -9
- package/src/orchestrator/resources.ts +0 -45
- package/src/orchestrator/verify.ts +0 -174
- package/src/plugins/local-cache/index.ts +0 -11
- package/src/plugins/schedule-history/index.ts +0 -109
package/src/cache/cache.ts
CHANGED
|
@@ -20,29 +20,48 @@
|
|
|
20
20
|
// for the user to see — but on a cache hit there's nothing to replay
|
|
21
21
|
// (the original run was successful and stderr typically empty).
|
|
22
22
|
//
|
|
23
|
-
//
|
|
24
|
-
//
|
|
25
|
-
//
|
|
26
|
-
//
|
|
27
|
-
// save : persist outputs + stdout under a hash
|
|
28
|
-
// ingest : adopt an artifact produced elsewhere (remote-hit path)
|
|
29
|
-
// recordRun : append a row to the run history table (for stats)
|
|
30
|
-
// close : release the SQLite handle
|
|
23
|
+
// This is core's FLOOR, not a module to replace: remote storage is a
|
|
24
|
+
// `RemoteCacheLayer` (has / get / put) that `LayeredCache` wraps around
|
|
25
|
+
// this handle, declared by a plugin's `cache` hook. The contract every
|
|
26
|
+
// layer speaks is `CacheLayer` in layer.ts; `plugin-host.ts` enforces it.
|
|
31
27
|
|
|
32
28
|
import { Database, type SQLQueryBindings } from 'bun:sqlite'
|
|
33
|
-
import { existsSync, mkdirSync,
|
|
34
|
-
import {
|
|
29
|
+
import { existsSync, mkdirSync, writeFileSync } from 'node:fs'
|
|
30
|
+
import { mkdir, readdir, rename, rm, stat, unlink, writeFile } from 'node:fs/promises'
|
|
35
31
|
import path from 'node:path'
|
|
36
32
|
import { relPosix, UserError, xxh3, xxh3hex, span } from '../util/index.js'
|
|
37
33
|
import {
|
|
38
34
|
ArchiveSecurityError,
|
|
35
|
+
type ExecUsage,
|
|
39
36
|
extractArtifactStream,
|
|
40
37
|
scanArtifact,
|
|
41
38
|
packArtifactBytes,
|
|
42
39
|
packArtifactStream,
|
|
43
40
|
planArtifact,
|
|
44
41
|
} from './archive.js'
|
|
45
|
-
import {
|
|
42
|
+
import {
|
|
43
|
+
type CacheEntry,
|
|
44
|
+
type CacheGetContext,
|
|
45
|
+
type CacheKeyInput,
|
|
46
|
+
type CacheLayer,
|
|
47
|
+
type CacheStats,
|
|
48
|
+
type CacheStatsOptions,
|
|
49
|
+
CorruptArtifactError,
|
|
50
|
+
type IngestMeta,
|
|
51
|
+
type InvocationRecord,
|
|
52
|
+
type OutputDirRow,
|
|
53
|
+
type OutputFileRow,
|
|
54
|
+
type PruneOptions,
|
|
55
|
+
type PruneResult,
|
|
56
|
+
type RunRecord,
|
|
57
|
+
type SaveArgs,
|
|
58
|
+
type TaskInputRow,
|
|
59
|
+
} from './layer.js'
|
|
60
|
+
import { bytesOf, decodedTar, STREAM_DECODE_FROM, zstdEncoder } from './zstd.js'
|
|
61
|
+
import { ConfigEvalTable } from './config-evals.js'
|
|
62
|
+
import { FileHashStore } from './file-hashes.js'
|
|
63
|
+
import { OutputIndex } from './output-index.js'
|
|
64
|
+
import { RunHistory } from './run-history.js'
|
|
46
65
|
|
|
47
66
|
// v17: artifact carries only logs + outputs (stdout + outputs/<rel>).
|
|
48
67
|
// Local and remote layers transport the SAME tar.zst bytes — no
|
|
@@ -102,7 +121,30 @@ import { FsCASBackend } from './cas-backend.js'
|
|
|
102
121
|
// `LayeredCache` uploads them, so the reach is a whole team's shared cache
|
|
103
122
|
// rather than one developer's disk. Pre-alpha, so one cold rebuild is the
|
|
104
123
|
// cheap side of that trade.
|
|
105
|
-
const CACHE_VERSION = 'vx-cache-v27'
|
|
124
|
+
export const CACHE_VERSION = 'vx-cache-v27'
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* An artifact or temp file without an `entries` row is reaped by
|
|
128
|
+
* `prune()` only once it is this old. A save renames the artifact into
|
|
129
|
+
* place and commits its row in the same tick; a crashed save's temp is
|
|
130
|
+
* stale long before this. Generous on purpose — a false orphan costs a
|
|
131
|
+
* re-run, a leaked temp costs disk.
|
|
132
|
+
*/
|
|
133
|
+
const ORPHAN_GRACE_MS = 60 * 60 * 1000
|
|
134
|
+
|
|
135
|
+
export interface SchemaReset {
|
|
136
|
+
from: string
|
|
137
|
+
to: string
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** Say once, on the channel the opener has, that an upgrade emptied the index. */
|
|
141
|
+
export function noteSchemaReset(cache: Cache, warn: (message: string) => void): void {
|
|
142
|
+
if (cache.schemaReset === null) return
|
|
143
|
+
const { from, to } = cache.schemaReset
|
|
144
|
+
warn(
|
|
145
|
+
`[vx] cache index reset: schema ${from} → ${to} (vx upgraded); every cached task misses once and re-saves, and \`vx cache prune\` reclaims the old artifacts`,
|
|
146
|
+
)
|
|
147
|
+
}
|
|
106
148
|
// SCHEMA history (drop+recreate on mismatch; pre-alpha, no migrations):
|
|
107
149
|
// v20: file_hashes.content_hash (git blob OIDs).
|
|
108
150
|
// v21: dropped the unused outputs_hash column (pure-input hashing).
|
|
@@ -133,7 +175,19 @@ const CACHE_VERSION = 'vx-cache-v27'
|
|
|
133
175
|
// the memo simply stops answering wrongly, so an affected task's
|
|
134
176
|
// key moves from a WRONG value to the right one: it misses once,
|
|
135
177
|
// re-runs, re-caches. Self-healing, never a wrong hit.
|
|
136
|
-
|
|
178
|
+
// v25: runs.cached — whether the task declared a `cache` block. An
|
|
179
|
+
// uncached task's row looked like any executed miss, so `vx why`
|
|
180
|
+
// headlined "cache key changed (inputs differ)" for a task that
|
|
181
|
+
// runs every time by design, and `vx last` could not mark it the
|
|
182
|
+
// way the terminal summary does (`no-cache`). Analytics-only —
|
|
183
|
+
// the cache KEY is unchanged.
|
|
184
|
+
// v26: entries.cpu_ms / peak_rss_bytes — what the producing execution
|
|
185
|
+
// used, read out of the artifact's sidecar at save and ingest
|
|
186
|
+
// alike, so a hit (a remote one on a fresh runner above all) can
|
|
187
|
+
// tell the history what the task needs. The cache KEY and the
|
|
188
|
+
// artifact container are unchanged (no CACHE_VERSION bump: an
|
|
189
|
+
// artifact without the field reads as before).
|
|
190
|
+
export const SCHEMA_VERSION = 'v26'
|
|
137
191
|
|
|
138
192
|
/**
|
|
139
193
|
* SQL predicate selecting `runs` rows that record an EXECUTION.
|
|
@@ -171,802 +225,12 @@ export const KEYED_RUNS_SQL = "hash <> ''"
|
|
|
171
225
|
*/
|
|
172
226
|
export const WORKSPACE_OUTPUT_PREFIX = 'workspace-outputs/'
|
|
173
227
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
*
|
|
178
|
-
*
|
|
179
|
-
|
|
180
|
-
* Only the task-artifact get/save path is gated. `recordRun`, `stats`,
|
|
181
|
-
* `prune`, key derivation, and prefetch-ingest are never affected — they
|
|
182
|
-
* are bookkeeping/analytics that a run policy has no business disabling.
|
|
183
|
-
*/
|
|
184
|
-
export interface CachePolicy {
|
|
185
|
-
localRead: boolean
|
|
186
|
-
localWrite: boolean
|
|
187
|
-
remoteRead: boolean
|
|
188
|
-
remoteWrite: boolean
|
|
189
|
-
}
|
|
190
|
-
|
|
191
|
-
/** All four axes on — the default when no cache flag is passed. */
|
|
192
|
-
export const FULL_CACHE_POLICY: CachePolicy = {
|
|
193
|
-
localRead: true,
|
|
194
|
-
localWrite: true,
|
|
195
|
-
remoteRead: true,
|
|
196
|
-
remoteWrite: true,
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
/**
|
|
200
|
-
* Parse a `--cache=<spec>` value into a `CachePolicy`, starting from a
|
|
201
|
-
* base (defaults to {@link FULL_CACHE_POLICY}). The spec is a
|
|
202
|
-
* comma-separated list of `layer:flags` segments where `layer` is
|
|
203
|
-
* `local` or `remote` and `flags` is any subset of `r` (read) and `w`
|
|
204
|
-
* (write), order-independent and possibly empty.
|
|
205
|
-
*
|
|
206
|
-
* A mentioned layer is set EXACTLY to its flags (read = includes `r`,
|
|
207
|
-
* write = includes `w`); an unmentioned layer keeps its base value.
|
|
208
|
-
* So `local:rw,remote:r` = remote read-only, `remote:` = remote fully
|
|
209
|
-
* off, `local:r` = local read-only with remote untouched.
|
|
210
|
-
*
|
|
211
|
-
* Throws a {@link UserError} on an unknown layer, an unknown flag, a
|
|
212
|
-
* duplicated flag, a missing colon, or a repeated layer.
|
|
213
|
-
*/
|
|
214
|
-
export function parseCachePolicy(spec: string, base: CachePolicy = FULL_CACHE_POLICY): CachePolicy {
|
|
215
|
-
const out: CachePolicy = { ...base }
|
|
216
|
-
const seen = new Set<string>()
|
|
217
|
-
for (const rawSeg of spec.split(',')) {
|
|
218
|
-
const seg = rawSeg.trim()
|
|
219
|
-
if (seg.length === 0) continue
|
|
220
|
-
const colon = seg.indexOf(':')
|
|
221
|
-
if (colon < 0) {
|
|
222
|
-
throw new UserError(`invalid --cache segment '${seg}': expected '<layer>:<flags>'`)
|
|
223
|
-
}
|
|
224
|
-
const layer = seg.slice(0, colon)
|
|
225
|
-
const flags = seg.slice(colon + 1)
|
|
226
|
-
if (layer !== 'local' && layer !== 'remote') {
|
|
227
|
-
throw new UserError(`invalid --cache layer '${layer}': expected 'local' or 'remote'`)
|
|
228
|
-
}
|
|
229
|
-
if (seen.has(layer)) {
|
|
230
|
-
throw new UserError(`--cache layer '${layer}' specified twice`)
|
|
231
|
-
}
|
|
232
|
-
seen.add(layer)
|
|
233
|
-
const flagSet = new Set<string>()
|
|
234
|
-
for (const ch of flags) {
|
|
235
|
-
if (ch !== 'r' && ch !== 'w') {
|
|
236
|
-
throw new UserError(`invalid --cache flag '${ch}' for '${layer}': expected 'r' and/or 'w'`)
|
|
237
|
-
}
|
|
238
|
-
if (flagSet.has(ch)) {
|
|
239
|
-
throw new UserError(`--cache flag '${ch}' repeated for '${layer}'`)
|
|
240
|
-
}
|
|
241
|
-
flagSet.add(ch)
|
|
242
|
-
}
|
|
243
|
-
if (layer === 'local') {
|
|
244
|
-
out.localRead = flagSet.has('r')
|
|
245
|
-
out.localWrite = flagSet.has('w')
|
|
246
|
-
} else {
|
|
247
|
-
out.remoteRead = flagSet.has('r')
|
|
248
|
-
out.remoteWrite = flagSet.has('w')
|
|
249
|
-
}
|
|
250
|
-
}
|
|
251
|
-
return out
|
|
252
|
-
}
|
|
253
|
-
|
|
254
|
-
export interface CacheKeyInput {
|
|
255
|
-
taskId: string
|
|
256
|
-
/**
|
|
257
|
-
* Hash of the resolved task config (post-evaluation). Folds in everything
|
|
258
|
-
* the user wrote — command, env declarations (passThrough names + define
|
|
259
|
-
* key/value pairs), dependsOn, cache.inputs declarations, outputs — including
|
|
260
|
-
* values that arrived via `import` at config-load time.
|
|
261
|
-
*/
|
|
262
|
-
taskConfigHash: string
|
|
263
|
-
/**
|
|
264
|
-
* Runtime values of declared cache-input env names (from parent at hash
|
|
265
|
-
* time). Independent of `exec.env`; lives here for cache identity.
|
|
266
|
-
*/
|
|
267
|
-
envValues: Array<[name: string, value: string]>
|
|
268
|
-
/**
|
|
269
|
-
* Resolved `cache.inputs.runtime` commands as [command, output] pairs
|
|
270
|
-
* (output = trimmed stdout+stderr, resolved live at hash time). Folded
|
|
271
|
-
* into the key in a namespace distinct from workspaceRuntimeValues.
|
|
272
|
-
*/
|
|
273
|
-
runtimeValues?: Array<[command: string, output: string]>
|
|
274
|
-
/** Resolved `cache.inputs.workspaceRuntime` pairs (root-cwd commands). */
|
|
275
|
-
workspaceRuntimeValues?: Array<[command: string, output: string]>
|
|
276
|
-
/** Absolute paths to input files. */
|
|
277
|
-
inputFiles: string[]
|
|
278
|
-
workspaceRoot: string
|
|
279
|
-
/** Cache keys of upstream tasks this one depends on, sorted. */
|
|
280
|
-
upstreamHashes: string[]
|
|
281
|
-
/**
|
|
282
|
-
* Upstream hash → upstream task id, for `captureInto` row NAMING only.
|
|
283
|
-
* Never folded into the digest — the key already folds the sorted
|
|
284
|
-
* hashes. Lets the persisted `entry_inputs` row name which upstream
|
|
285
|
-
* task a hash came from (the diff reads better than a bare hash). When
|
|
286
|
-
* absent, the captured `upstream` row falls back to `name = hash`.
|
|
287
|
-
*/
|
|
288
|
-
upstreamIds?: ReadonlyMap<string, string>
|
|
289
|
-
/**
|
|
290
|
-
* The task's DEPENDENCY closure with GROUP tasks expanded into the real
|
|
291
|
-
* tasks they stand for — what an input-shipping executor must place in the
|
|
292
|
-
* input root. A different question from what the key folds, and derived
|
|
293
|
-
* differently in both directions:
|
|
294
|
-
*
|
|
295
|
-
* - It expands groups, which contribute a synthetic roll-up hash and no
|
|
296
|
-
* outputs of their own, so a dependent of one would otherwise describe
|
|
297
|
-
* an empty closure.
|
|
298
|
-
* - It ignores `cache.inputs.tasks`. That filter says which upstream KEYS
|
|
299
|
-
* this task's key folds; what the task may READ is `dependsOn`, and
|
|
300
|
-
* locally every dependency's outputs are on disk before the command runs
|
|
301
|
-
* however the filter is written.
|
|
302
|
-
*
|
|
303
|
-
* NEVER folded into the digest — the key cascades through a group's own
|
|
304
|
-
* roll-up hash already, and folding either difference in would move every
|
|
305
|
-
* existing dependent's key without telling it anything new.
|
|
306
|
-
*/
|
|
307
|
-
upstreamGraft?: ReadonlyArray<{
|
|
308
|
-
readonly taskId: string
|
|
309
|
-
readonly hash: string
|
|
310
|
-
readonly projectDir: string
|
|
311
|
-
}>
|
|
312
|
-
/**
|
|
313
|
-
* Workspace-level fingerprint — typically a hash of `pnpm-lock.yaml` +
|
|
314
|
-
* `pnpm-workspace.yaml`. Folds resolved dep versions and workspace shape
|
|
315
|
-
* into every task's key, so a lockfile bump invalidates everything.
|
|
316
|
-
*/
|
|
317
|
-
workspaceFingerprint: string
|
|
318
|
-
/**
|
|
319
|
-
* CLI args forwarded to the task (after `--`). Folded into the key so that
|
|
320
|
-
* the same command with different forwarded args is treated as a distinct
|
|
321
|
-
* run, never a spurious cache hit.
|
|
322
|
-
*/
|
|
323
|
-
forwardArgs?: readonly string[]
|
|
324
|
-
/**
|
|
325
|
-
* Hash of the project's `package.json` bytes. Folded into the key
|
|
326
|
-
* implicitly (Turbo / Nx parity) so dep changes invalidate every
|
|
327
|
-
* task in that project, even when `cache.inputs.files` doesn't
|
|
328
|
-
* cover package.json. Empty string when the project has no
|
|
329
|
-
* package.json (impossible in practice — workspace discovery
|
|
330
|
-
* requires one — but we don't fail-loud here).
|
|
331
|
-
*/
|
|
332
|
-
projectPackageJsonHash: string
|
|
333
|
-
/**
|
|
334
|
-
* Precomputed content hashes (git blob OIDs) keyed by absolute
|
|
335
|
-
* path — typically the trusted-index OID map harvested by the
|
|
336
|
-
* run's bulk `git ls-files -s`. Paths present here skip `hashFile`
|
|
337
|
-
* entirely (no stat, no SQLite, no read); missing paths fall back
|
|
338
|
-
* to `hashFile`, which computes the byte-identical blob OID from
|
|
339
|
-
* disk. Pure fast path: the derived key never depends on whether a
|
|
340
|
-
* hash arrived via the map or the fallback.
|
|
341
|
-
*/
|
|
342
|
-
fileHashes?: ReadonlyMap<string, string>
|
|
343
|
-
/**
|
|
344
|
-
* Material a plugin's `key` stage contributed, as sorted `[name, value]`
|
|
345
|
-
* pairs. Folded only when non-empty, so a workspace without a key plugin
|
|
346
|
-
* derives exactly the key it always did.
|
|
347
|
-
*/
|
|
348
|
-
pluginParts?: ReadonlyArray<readonly [name: string, value: string]>
|
|
349
|
-
/**
|
|
350
|
-
* When set, `key()` pushes each component (kind, name, hash) it folds
|
|
351
|
-
* — at the same fold sites, in fold order. Pure SIDE-CHANNEL: it does
|
|
352
|
-
* not change the returned digest in any way. On a cache MISS the
|
|
353
|
-
* orchestrator allocates an array, passes it here, and persists the
|
|
354
|
-
* rows to `entry_inputs` (inside the entry-save transaction) so a
|
|
355
|
-
* later run can diff its inputs against this one (the Tier-3 "why did
|
|
356
|
-
* this re-run?" moat). Capturing at the fold sites keeps the
|
|
357
|
-
* recorded set in lockstep with the key by construction: a future
|
|
358
|
-
* component that forgets to capture is a one-line miss, not silent
|
|
359
|
-
* drift. The per-file OIDs are already awaited here, so file rows cost
|
|
360
|
-
* zero extra I/O — just array pushes.
|
|
361
|
-
*/
|
|
362
|
-
captureInto?: Array<{ kind: string; name: string; hash: string }>
|
|
363
|
-
}
|
|
364
|
-
|
|
365
|
-
export interface CacheEntry {
|
|
366
|
-
hash: string
|
|
367
|
-
taskId: string
|
|
368
|
-
command: string
|
|
369
|
-
exitCode: number
|
|
370
|
-
durationMs: number
|
|
371
|
-
outputFiles: string[]
|
|
372
|
-
/**
|
|
373
|
-
* The `output_files` rows behind `outputFiles` (size / mode / mtime), when
|
|
374
|
-
* the layer that produced this entry had them in hand. `restoreHit` reads
|
|
375
|
-
* them for its tree-is-current check instead of re-querying — one SQL
|
|
376
|
-
* round trip per cache hit, saved.
|
|
377
|
-
*/
|
|
378
|
-
outputRows?: OutputFileRow[]
|
|
379
|
-
/**
|
|
380
|
-
* The `output_dirs` rows behind the directory short-circuit, when the
|
|
381
|
-
* batched `getMany` loaded them — one query for the run instead of one
|
|
382
|
-
* per hit (0.12 ms each across a warm 1000-project run before this).
|
|
383
|
-
*/
|
|
384
|
-
outputDirRows?: OutputDirRow[]
|
|
385
|
-
/** Captured stdout, always present (may be empty). stderr is not cached. */
|
|
386
|
-
stdout: string
|
|
387
|
-
storedAt: string
|
|
388
|
-
/**
|
|
389
|
-
* Where this hit was resolved from. `'local'` for a SQLite-backed
|
|
390
|
-
* Cache; `'remote'` when LayeredCache pulled the artifact from the
|
|
391
|
-
* remote layer this lookup (even though it's been materialized into
|
|
392
|
-
* local for next time). Lets the orchestrator surface
|
|
393
|
-
* `cache-hit-remote` so users see when remote caching actually saved
|
|
394
|
-
* them work vs. a stale-local replay.
|
|
395
|
-
*/
|
|
396
|
-
source?: 'local' | 'remote'
|
|
397
|
-
}
|
|
398
|
-
|
|
399
|
-
export interface RunRecord {
|
|
400
|
-
/**
|
|
401
|
-
* The task's cache key. ABSENT when the outcome never derived one — a
|
|
402
|
-
* `skipped` task (its upstream failed, so it never probed) or a
|
|
403
|
-
* `persistent` one (a dev server is never cached). Such a row is still a
|
|
404
|
-
* task of the run and must be recorded, so the column keeps a `''`
|
|
405
|
-
* sentinel rather than going nullable: `''` is impossible for a real key
|
|
406
|
-
* (`Cache.key` returns 16 hex chars), the reads that must not treat it as
|
|
407
|
-
* a key already guard it, and it matches what the cloud `task_runs.hash`
|
|
408
|
-
* column has stored for the same concept since its first migration.
|
|
409
|
-
*/
|
|
410
|
-
hash?: string
|
|
411
|
-
project: string
|
|
412
|
-
task: string
|
|
413
|
-
status: 'success' | 'failed' | 'cache-hit' | 'cache-hit-remote' | 'skipped'
|
|
414
|
-
exitCode: number
|
|
415
|
-
durationMs: number
|
|
416
|
-
forwardArgs?: readonly string[]
|
|
417
|
-
startedAt: number // ms-epoch wall clock
|
|
418
|
-
endedAt: number // ms-epoch wall clock
|
|
419
|
-
/**
|
|
420
|
-
* Optional analytics columns. Populated by the orchestrator/runner;
|
|
421
|
-
* stored as NULL on rows from older runs. Surfaced via `vx stats`
|
|
422
|
-
* and consumable from CI by reading cache.db directly.
|
|
423
|
-
*/
|
|
424
|
-
runId?: string // ULID shared across every task in one `vx run` invocation
|
|
425
|
-
cpuMs?: number // sum of user + system CPU time for the child process
|
|
426
|
-
peakRssBytes?: number // peak resident set size of the child process
|
|
427
|
-
wallclockStartNs?: bigint // hrtime span relative to run t=0
|
|
428
|
-
wallclockEndNs?: bigint
|
|
429
|
-
cacheHit?: boolean // convenience for flamegraph color; derivable from status
|
|
430
|
-
attempts?: number // >1 when the task retried; the direct within-run flaky signal
|
|
431
|
-
}
|
|
432
|
-
|
|
433
|
-
/**
|
|
434
|
-
* One header row per `vx run` invocation (the `invocations` table). All
|
|
435
|
-
* fields mirror the columns; nullable VCS/host columns are `null` when
|
|
436
|
-
* the probe failed (not a git repo, hostname unavailable). Recorded
|
|
437
|
-
* once per run inside the same transaction as the per-task `runs` rows.
|
|
438
|
-
*/
|
|
439
|
-
export interface InvocationRecord {
|
|
440
|
-
runId: string
|
|
441
|
-
command: string
|
|
442
|
-
/** JSON-serialized `string[]` of the requested task names. */
|
|
443
|
-
requestedTasks: string
|
|
444
|
-
/** Compact policy flags, e.g. `'lR,lW,rR,rW'` (an axis omitted = off). */
|
|
445
|
-
cachePolicy: string
|
|
446
|
-
concurrency: number
|
|
447
|
-
flow: 'focused' | 'broad' | null
|
|
448
|
-
startedAt: number
|
|
449
|
-
endedAt: number
|
|
450
|
-
totalDurationMs: number
|
|
451
|
-
taskCount: number
|
|
452
|
-
failedCount: number
|
|
453
|
-
hitCount: number
|
|
454
|
-
hitLocalCount: number
|
|
455
|
-
hitRemoteCount: number
|
|
456
|
-
exitOk: boolean
|
|
457
|
-
commitSha: string | null
|
|
458
|
-
branch: string | null
|
|
459
|
-
dirty: boolean | null
|
|
460
|
-
ci: boolean
|
|
461
|
-
ciProvider: string | null
|
|
462
|
-
host: string | null
|
|
463
|
-
os: string | null
|
|
464
|
-
arch: string | null
|
|
465
|
-
vxVersion: string
|
|
466
|
-
/** JSON object `{k:v}` of `--tag` pairs. */
|
|
467
|
-
tags: string
|
|
468
|
-
}
|
|
469
|
-
|
|
470
|
-
/**
|
|
471
|
-
* One cache-key component row for the `entry_inputs` table — keyed by
|
|
472
|
-
* the cache-entry HASH it belongs to, not a run. Written inside the
|
|
473
|
-
* entry-save transaction (`writeArtifactAndIndex`) on a miss/save; a
|
|
474
|
-
* cache HIT does not save, so it persists nothing (warm runs are free).
|
|
475
|
-
* The diff (Phase B/B1) reads these by the run's task hash
|
|
476
|
-
* (`runs.hash → entry_inputs[entry_hash]`).
|
|
477
|
-
*/
|
|
478
|
-
export interface TaskInputRow {
|
|
479
|
-
entryHash: string
|
|
480
|
-
kind: string
|
|
481
|
-
name: string
|
|
482
|
-
hash: string
|
|
483
|
-
}
|
|
484
|
-
|
|
485
|
-
export interface CacheStats {
|
|
486
|
-
entryCount: number
|
|
487
|
-
totalBytes: number
|
|
488
|
-
runCountLast24h: number
|
|
489
|
-
hitCountLast24h: number
|
|
490
|
-
}
|
|
491
|
-
|
|
492
|
-
export interface CacheStatsOptions {
|
|
493
|
-
/** Narrow every aggregate to one project. Absent = the whole workspace. */
|
|
494
|
-
project?: string
|
|
495
|
-
}
|
|
496
|
-
|
|
497
|
-
export interface PruneOptions {
|
|
498
|
-
/** Drop entries last accessed before this ms-epoch threshold. */
|
|
499
|
-
olderThanMs?: number
|
|
500
|
-
/**
|
|
501
|
-
* After applying olderThanMs, if the cache still exceeds this size in
|
|
502
|
-
* bytes, evict LRU (smallest `accessed_at` first) until under it.
|
|
503
|
-
*/
|
|
504
|
-
maxBytes?: number
|
|
505
|
-
}
|
|
506
|
-
|
|
507
|
-
export interface PruneResult {
|
|
508
|
-
evicted: number
|
|
509
|
-
bytesFreed: number
|
|
510
|
-
}
|
|
511
|
-
|
|
512
|
-
/**
|
|
513
|
-
* Per-output-file fingerprint, scoped by the cache entry that
|
|
514
|
-
* produced it. Batch-loaded once at the top of a run via
|
|
515
|
-
* `loadOutputFilesBatch(hashes)` so the orchestrator's "is this
|
|
516
|
-
* tree already current?" probe becomes an in-memory Map lookup
|
|
517
|
-
* plus N parallel stat calls.
|
|
518
|
-
*
|
|
519
|
-
* `path` is project-relative (e.g. `dist/index.js`), matching how
|
|
520
|
-
* outputs are addressed under `<projectDir>/` — except workspace
|
|
521
|
-
* outputs, which carry the full `workspace-outputs/<rel-to-root>`
|
|
522
|
-
* tar entry name (see `WORKSPACE_OUTPUT_PREFIX`); callers split on
|
|
523
|
-
* the prefix and anchor those at the workspace root.
|
|
524
|
-
*/
|
|
525
|
-
export interface OutputFileRow {
|
|
526
|
-
path: string
|
|
527
|
-
size: number
|
|
528
|
-
mode: number
|
|
529
|
-
mtimeMs: number
|
|
530
|
-
}
|
|
531
|
-
|
|
532
|
-
/** One directory under a whole-subtree output glob, as it stood after the last save or restore on THIS machine. */
|
|
533
|
-
export interface OutputDirRow {
|
|
534
|
-
path: string
|
|
535
|
-
mtimeMs: number
|
|
536
|
-
}
|
|
537
|
-
|
|
538
|
-
/** More directories than this under a task's output prefixes: record nothing, keep the walk. */
|
|
539
|
-
export const OUTPUT_DIRS_CAP = 256
|
|
540
|
-
|
|
541
|
-
/**
|
|
542
|
-
* A directory whose mtime lies within this many ms of the snapshot is RACY
|
|
543
|
-
* and the whole snapshot is dropped. File timestamps are coarse on Linux
|
|
544
|
-
* (a kernel tick, up to 10 ms), so a write landing in the same tick as the
|
|
545
|
-
* one that made the directory's recorded mtime leaves the mtime unchanged
|
|
546
|
-
* and the change invisible — git's index distrusts stats this young for
|
|
547
|
-
* the same reason (a stray survived a hit on the ubuntu job, 2026-09-03).
|
|
548
|
-
* The next hit walks, and once the tree is older than the window it is
|
|
549
|
-
* recorded for good.
|
|
550
|
-
*/
|
|
551
|
-
export const OUTPUT_DIRS_RACY_MS = 50
|
|
552
|
-
|
|
553
|
-
/**
|
|
554
|
-
* The shape every cache implementation honors. `Cache` (the local v10
|
|
555
|
-
* implementation) and `LayeredCache` both `implements` this so the
|
|
556
|
-
* orchestrator's `executeTask` can take either without a discriminated
|
|
557
|
-
* union and we get a compile-time guarantee the surfaces stay congruent.
|
|
558
|
-
*/
|
|
559
|
-
/**
|
|
560
|
-
* Context passed to `get()`. Optional, but required when the lookup
|
|
561
|
-
* may resolve through the remote layer — the local SQL row inserted on
|
|
562
|
-
* remote-hit needs `taskId` + `command` to be queryable later (the
|
|
563
|
-
* artifact itself doesn't carry them). `Cache` (local) ignores this
|
|
564
|
-
* field; `LayeredCache` forwards it to `Cache.ingest`.
|
|
565
|
-
*/
|
|
566
|
-
export interface CacheGetContext {
|
|
567
|
-
taskId: string
|
|
568
|
-
command: string
|
|
569
|
-
}
|
|
570
|
-
|
|
571
|
-
/** Metadata supplied at ingest time — values the artifact does not carry. */
|
|
572
|
-
export interface IngestMeta {
|
|
573
|
-
taskId: string
|
|
574
|
-
command: string
|
|
575
|
-
/** Wall-clock time of the original task execution. */
|
|
576
|
-
durationMs: number
|
|
577
|
-
/**
|
|
578
|
-
* Cache-key components (Tier-3 input fingerprint) to persist into
|
|
579
|
-
* `entry_inputs` in the same transaction as the entry row. Omitted on
|
|
580
|
-
* the remote-hit ingest path (the artifact doesn't carry them — the
|
|
581
|
-
* fingerprint is the saving machine's local moat). Only computed on a
|
|
582
|
-
* miss/save, so a cache hit never reaches here.
|
|
583
|
-
*/
|
|
584
|
-
inputComponents?: readonly TaskInputRow[]
|
|
585
|
-
}
|
|
586
|
-
|
|
587
|
-
/**
|
|
588
|
-
* The supplied artifact bytes don't decompress/parse as a vx artifact.
|
|
589
|
-
* Thrown by `save`/`ingest` BEFORE anything reaches the final cache
|
|
590
|
-
* path — a rejected artifact leaves no `<hash>.tar.zst` and no SQL row.
|
|
591
|
-
* The LayeredCache treats this as a remote fault on the remote-hit
|
|
592
|
-
* path (degrades to a cache miss).
|
|
593
|
-
*/
|
|
594
|
-
export class CorruptArtifactError extends Error {
|
|
595
|
-
constructor(
|
|
596
|
-
public readonly hash: string,
|
|
597
|
-
reason: string,
|
|
598
|
-
public override readonly cause?: unknown,
|
|
599
|
-
) {
|
|
600
|
-
super(`cache: corrupt artifact for ${hash}: ${reason}`)
|
|
601
|
-
this.name = 'CorruptArtifactError'
|
|
602
|
-
}
|
|
603
|
-
}
|
|
604
|
-
|
|
605
|
-
/**
|
|
606
|
-
* A decompressed artifact above this is refused as a zstd bomb rather than
|
|
607
|
-
* expanded into memory. 2 GiB comfortably exceeds any real build output while
|
|
608
|
-
* bounding a malicious/compromised remote's ability to OOM a victim who takes
|
|
609
|
-
* a cache hit.
|
|
610
|
-
*/
|
|
611
|
-
const MAX_DECOMPRESSED_ARTIFACT_BYTES = 2 * 1024 * 1024 * 1024
|
|
612
|
-
|
|
613
|
-
/**
|
|
614
|
-
* Read a zstd frame's declared Frame_Content_Size (RFC 8878 §3.1.1) WITHOUT
|
|
615
|
-
* decompressing. Returns null when the frame omits it (streaming frames) or
|
|
616
|
-
* the header is too short to parse. vx's own producer (single-shot
|
|
617
|
-
* `Bun.zstdCompress` of a known buffer) always writes it, so an artifact that
|
|
618
|
-
* declares an enormous size can be rejected before a byte is allocated.
|
|
619
|
-
*
|
|
620
|
-
* Exported for tests: pins the per-`fcsFlag` byte layouts (incl. the 2-byte
|
|
621
|
-
* `+256` adjustment and the `dictIdFlag` offset) that a bomb-refusal e2e can't
|
|
622
|
-
* discriminate (their max declarable sizes sit below the ceiling).
|
|
623
|
-
*/
|
|
624
|
-
export function zstdContentSize(b: Uint8Array): bigint | null {
|
|
625
|
-
if (b.length < 5) return null
|
|
626
|
-
// Magic_Number 0xFD2FB528, little-endian.
|
|
627
|
-
if (b[0] !== 0x28 || b[1] !== 0xb5 || b[2] !== 0x2f || b[3] !== 0xfd) return null
|
|
628
|
-
const desc = b[4]!
|
|
629
|
-
const fcsFlag = desc >> 6
|
|
630
|
-
const singleSegment = (desc >> 5) & 1
|
|
631
|
-
const dictIdFlag = desc & 3
|
|
632
|
-
let off = 5
|
|
633
|
-
if (singleSegment === 0) off += 1 // Window_Descriptor byte
|
|
634
|
-
off += dictIdFlag === 3 ? 4 : dictIdFlag // Dictionary_ID: 0,1,2,4 bytes
|
|
635
|
-
let fcsSize: number
|
|
636
|
-
if (fcsFlag === 0) fcsSize = singleSegment === 1 ? 1 : 0
|
|
637
|
-
else if (fcsFlag === 1) fcsSize = 2
|
|
638
|
-
else if (fcsFlag === 2) fcsSize = 4
|
|
639
|
-
else fcsSize = 8
|
|
640
|
-
if (fcsSize === 0 || b.length < off + fcsSize) return null
|
|
641
|
-
let v = 0n
|
|
642
|
-
for (let i = 0; i < fcsSize; i++) v |= BigInt(b[off + i]!) << BigInt(8 * i)
|
|
643
|
-
if (fcsSize === 2) v += 256n // per spec, the 2-byte field stores value − 256
|
|
644
|
-
return v
|
|
645
|
-
}
|
|
646
|
-
|
|
647
|
-
/**
|
|
648
|
-
* Decompress a zstd artifact that DECLARES its content size, with a hard
|
|
649
|
-
* output ceiling: refused before a byte is allocated when the declaration
|
|
650
|
-
* is over the cap, and again on the actual length. A frame with no
|
|
651
|
-
* declaration (a streamed producer's — vx's own, above 4 MiB) never comes
|
|
652
|
-
* here: `decodedTar` decodes it as a stream under the running count, so
|
|
653
|
-
* a sizeless bomb has nowhere to expand.
|
|
654
|
-
*/
|
|
655
|
-
async function zstdDecompressBounded(compressed: Uint8Array, hash: string): Promise<Uint8Array> {
|
|
656
|
-
assertDeclaredSize(compressed, hash)
|
|
657
|
-
const out = await Bun.zstdDecompress(compressed)
|
|
658
|
-
if (out.length > MAX_DECOMPRESSED_ARTIFACT_BYTES) {
|
|
659
|
-
throw new CorruptArtifactError(
|
|
660
|
-
hash,
|
|
661
|
-
`decompressed to ${out.length} bytes (> ${MAX_DECOMPRESSED_ARTIFACT_BYTES} cap)`,
|
|
662
|
-
)
|
|
663
|
-
}
|
|
664
|
-
return out
|
|
665
|
-
}
|
|
666
|
-
|
|
667
|
-
/** The pre-decompress half of the ceiling: the frame header's own claim, when it makes one. */
|
|
668
|
-
function assertDeclaredSize(compressed: Uint8Array, hash: string): bigint | null {
|
|
669
|
-
const declared = zstdContentSize(compressed)
|
|
670
|
-
if (declared !== null && declared > BigInt(MAX_DECOMPRESSED_ARTIFACT_BYTES)) {
|
|
671
|
-
throw new CorruptArtifactError(
|
|
672
|
-
hash,
|
|
673
|
-
`declares ${declared} decompressed bytes (> ${MAX_DECOMPRESSED_ARTIFACT_BYTES} cap)`,
|
|
674
|
-
)
|
|
675
|
-
}
|
|
676
|
-
return declared
|
|
677
|
-
}
|
|
678
|
-
|
|
679
|
-
/** Compressed artifacts above this size are decoded as a stream, on restore and on ingest. */
|
|
680
|
-
const STREAM_DECODE_FROM = 4 * 1024 * 1024
|
|
681
|
-
|
|
682
|
-
/** Collect a byte stream; only ever used where the seam wants bytes. */
|
|
683
|
-
async function bytesOf(stream: ReadableStream<Uint8Array>): Promise<Uint8Array> {
|
|
684
|
-
return new Uint8Array(await new Response(stream).arrayBuffer())
|
|
685
|
-
}
|
|
686
|
-
|
|
687
|
-
/**
|
|
688
|
-
* Bun's `CompressionStream` types do not satisfy `pipeThrough`'s pair
|
|
689
|
-
* (its readable side is typed `NonSharedUint8Array`); the runtime object
|
|
690
|
-
* is a plain byte transform.
|
|
691
|
-
*/
|
|
692
|
-
const zstdEncoder = (): TransformStream<Uint8Array, Uint8Array> =>
|
|
693
|
-
new CompressionStream('zstd') as unknown as TransformStream<Uint8Array, Uint8Array>
|
|
694
|
-
|
|
695
|
-
const oneChunk = (bytes: Uint8Array): ReadableStream<Uint8Array> =>
|
|
696
|
-
new ReadableStream({
|
|
697
|
-
start(c) {
|
|
698
|
-
c.enqueue(bytes)
|
|
699
|
-
c.close()
|
|
700
|
-
},
|
|
701
|
-
})
|
|
702
|
-
|
|
703
|
-
/**
|
|
704
|
-
* The decoded tar as a stream: one call for a small artifact that
|
|
705
|
-
* declares its size, a streamed decode otherwise — the stream setup is
|
|
706
|
-
* ~35 µs, which matters at a thousand one-file artifacts and nowhere
|
|
707
|
-
* else. Bytes in memory are only ever the small case (a large source
|
|
708
|
-
* must be a FILE: a Blob copies its bytes and hands the decoder
|
|
709
|
-
* everything at once, measured +519 MiB on 150 MiB against +448 for the
|
|
710
|
-
* plain decode). The 2 GiB ceiling applies either way: the declaration
|
|
711
|
-
* and the result length in one call, a running count on the stream.
|
|
712
|
-
*/
|
|
713
|
-
async function decodedTar(
|
|
714
|
-
source: Uint8Array | Bun.BunFile,
|
|
715
|
-
hash: string,
|
|
716
|
-
): Promise<ReadableStream<Uint8Array>> {
|
|
717
|
-
if (source instanceof Uint8Array) {
|
|
718
|
-
if (assertDeclaredSize(source, hash) === null) return zstdDecodeStream(new Blob([source]), hash)
|
|
719
|
-
return oneChunk(await zstdDecompressBounded(source, hash))
|
|
720
|
-
}
|
|
721
|
-
if (source.size <= STREAM_DECODE_FROM) {
|
|
722
|
-
const bytes = await source.bytes()
|
|
723
|
-
if (assertDeclaredSize(bytes, hash) === null) return zstdDecodeStream(source, hash)
|
|
724
|
-
return oneChunk(await zstdDecompressBounded(bytes, hash))
|
|
725
|
-
}
|
|
726
|
-
assertDeclaredSize(await source.slice(0, 32).bytes(), hash)
|
|
727
|
-
return zstdDecodeStream(source, hash)
|
|
728
|
-
}
|
|
729
|
-
|
|
730
|
-
/**
|
|
731
|
-
* The streaming twin of `zstdDecompressBounded`: the decoded bytes as a
|
|
732
|
-
* stream, refused past the same ceiling. A malicious frame cannot expand
|
|
733
|
-
* past the cap here either — the count runs as bytes are produced, before
|
|
734
|
-
* any of them reach the reader's next entry.
|
|
735
|
-
*/
|
|
736
|
-
function zstdDecodeStream(source: Blob, hash: string): ReadableStream<Uint8Array> {
|
|
737
|
-
let total = 0
|
|
738
|
-
return source
|
|
739
|
-
.stream()
|
|
740
|
-
.pipeThrough(new DecompressionStream('zstd'))
|
|
741
|
-
.pipeThrough(
|
|
742
|
-
new TransformStream<Uint8Array, Uint8Array>({
|
|
743
|
-
transform(chunk, controller) {
|
|
744
|
-
total += chunk.byteLength
|
|
745
|
-
if (total > MAX_DECOMPRESSED_ARTIFACT_BYTES) {
|
|
746
|
-
controller.error(
|
|
747
|
-
new CorruptArtifactError(
|
|
748
|
-
hash,
|
|
749
|
-
`decompresses past ${MAX_DECOMPRESSED_ARTIFACT_BYTES} bytes (cap)`,
|
|
750
|
-
),
|
|
751
|
-
)
|
|
752
|
-
return
|
|
753
|
-
}
|
|
754
|
-
controller.enqueue(chunk)
|
|
755
|
-
},
|
|
756
|
-
}),
|
|
757
|
-
)
|
|
758
|
-
}
|
|
759
|
-
|
|
760
|
-
export interface CacheLayer {
|
|
761
|
-
/**
|
|
762
|
-
* The local handle this layer wraps, when it wraps one (`LayeredCache`).
|
|
763
|
-
* `resolveCache` uses it to drop a bare local layer another declared layer
|
|
764
|
-
* already contains, so `[cloud(), localCachePlugin()]` does not write the
|
|
765
|
-
* local store twice.
|
|
766
|
-
*/
|
|
767
|
-
readonly local?: Cache | undefined
|
|
768
|
-
/**
|
|
769
|
-
* `true` iff a REMOTE cache sits behind this layer. THE answer to "can
|
|
770
|
-
* the remote axes of the cache policy do anything on this run?" — the
|
|
771
|
-
* orchestrator clamps `remoteRead`/`remoteWrite` off when it is absent,
|
|
772
|
-
* skips the up-front local classify when it is present (that classify's
|
|
773
|
-
* `cache.get` would be a remote read-through awaited before scheduling),
|
|
774
|
-
* and drives the prefetch pass off it.
|
|
775
|
-
*
|
|
776
|
-
* A layer must answer TRUTHFULLY: identity against the local cache
|
|
777
|
-
* ("something other than the handle I passed in") is NOT the same
|
|
778
|
-
* question — an ordinary pass-through decorator with no remote at all
|
|
779
|
-
* answers yes to it. `LayeredCache` sets it, a bare `Cache` denies it,
|
|
780
|
-
* and a third-party layer opts in when, and only when, it really has a
|
|
781
|
-
* remote. Optional so a layer written before this contract keeps
|
|
782
|
-
* type-checking; absent reads as "no remote", the safe answer.
|
|
783
|
-
*/
|
|
784
|
-
readonly hasRemote?: boolean
|
|
785
|
-
/**
|
|
786
|
-
* Batch existence probe over the remote — the subset of `hashes` stored
|
|
787
|
-
* remotely, in ONE round-trip, or `null` for "no batch info; use
|
|
788
|
-
* per-hash". Only meaningful with `hasRemote`; a remote layer that can't
|
|
789
|
-
* batch omits it and the prefetch pass falls back to per-hash GETs.
|
|
790
|
-
*/
|
|
791
|
-
remoteHasMany?(hashes: readonly string[]): Promise<Set<string> | null>
|
|
792
|
-
/**
|
|
793
|
-
* Record that the remote has NO artifact for each of `hashes` (from a
|
|
794
|
-
* batch probe), so a later `get`/`prefetch` short-circuits to a miss with
|
|
795
|
-
* no round-trip. Only meaningful with `hasRemote`.
|
|
796
|
-
*/
|
|
797
|
-
markRemoteAbsent?(hashes: Iterable<string>): void
|
|
798
|
-
/**
|
|
799
|
-
* Resolve once every background write-through upload settles. The
|
|
800
|
-
* orchestrator awaits it before `close()` — an upload reading layer state
|
|
801
|
-
* after close would race. Only meaningful with `hasRemote`; a layer that
|
|
802
|
-
* uploads synchronously (or not at all) omits it.
|
|
803
|
-
*/
|
|
804
|
-
drainUploads?(): Promise<void>
|
|
805
|
-
key(input: CacheKeyInput): Promise<string>
|
|
806
|
-
get(hash: string, ctx?: CacheGetContext): Promise<CacheEntry | null>
|
|
807
|
-
/**
|
|
808
|
-
* Optional batched `get` (same answers, fewer round trips). The
|
|
809
|
-
* short-circuit probe uses it when a layer offers one; a layer without it
|
|
810
|
-
* is probed hash by hash.
|
|
811
|
-
*/
|
|
812
|
-
getMany?(hashes: readonly string[]): Promise<Map<string, CacheEntry>>
|
|
813
|
-
/**
|
|
814
|
-
* Lightweight existence probe. `'local'` / `'remote'` names the layer
|
|
815
|
-
* that holds the artifact; `null` is a miss. NEVER moves bytes: no
|
|
816
|
-
* artifact read, no remote download, no local ingest, no accessed_at
|
|
817
|
-
* bump (the LayeredCache's remote side is an HTTP HEAD). Planning
|
|
818
|
-
* (`--dry` / `--graph`) predicts hits with this instead of `get` so a
|
|
819
|
-
* dry run can't pull N artifacts over the network. Remote errors
|
|
820
|
-
* degrade to `null` — an existence probe never fails anything.
|
|
821
|
-
*/
|
|
822
|
-
has(hash: string): Promise<'local' | 'remote' | null>
|
|
823
|
-
/**
|
|
824
|
-
* Best-effort warm of `hash` from a slower layer (the remote cache)
|
|
825
|
-
* into this one, so a later `get(hash)` resolves locally without a
|
|
826
|
-
* round-trip on the task's critical path. Returns `true` if the
|
|
827
|
-
* artifact is now present locally, `false` on a miss / error
|
|
828
|
-
* (degrades, never throws). The local `Cache` is a no-op (nothing
|
|
829
|
-
* slower to warm from); `LayeredCache` owns the real implementation
|
|
830
|
-
* plus the in-flight de-dup so prefetch + get share ONE remote GET.
|
|
831
|
-
*/
|
|
832
|
-
prefetch(hash: string, ctx?: CacheGetContext): Promise<boolean>
|
|
833
|
-
/**
|
|
834
|
-
* Batched lookup of per-output-file fingerprints for many cache
|
|
835
|
-
* entries in one SQL round-trip. Returns a Map keyed by entry hash.
|
|
836
|
-
*
|
|
837
|
-
* Orchestrator pattern: call this once at `prepareRun` for every
|
|
838
|
-
* task whose hash is known up-front, then per-task `executeCachedTask`
|
|
839
|
-
* does a Map.get (O(1)) + parallel stat checks to decide whether the
|
|
840
|
-
* on-disk tree is already current.
|
|
841
|
-
*
|
|
842
|
-
* Hashes with no rows (cache misses) are absent from the result.
|
|
843
|
-
*/
|
|
844
|
-
loadOutputFilesBatch(hashes: readonly string[]): Map<string, OutputFileRow[]>
|
|
845
|
-
/**
|
|
846
|
-
* Stat each `expected` row's target under `projectDir` and return
|
|
847
|
-
* `true` iff every (size, mode, mtime) matches. Missing files,
|
|
848
|
-
* stat errors, or any mismatch → `false`.
|
|
849
|
-
*
|
|
850
|
-
* Pure FS check — no DB access. Caller batches the expected rows
|
|
851
|
-
* via `loadOutputFilesBatch`. Lets the orchestrator skip
|
|
852
|
-
* `cleanOutputs + restoreOutputs` entirely when the cached
|
|
853
|
-
* snapshot is already in place. Integrity-preserving: detects
|
|
854
|
-
* out-of-band file edits or deletions and falls through to a real
|
|
855
|
-
* restore.
|
|
856
|
-
*/
|
|
857
|
-
isOutputsCurrent(projectDir: string, expected: readonly OutputFileRow[]): Promise<boolean>
|
|
858
|
-
/**
|
|
859
|
-
* The directory-mtime short-circuit behind a warm hit (optional — a layer
|
|
860
|
-
* without it keeps the output walk). `recordOutputDirs` snapshots every
|
|
861
|
-
* directory under each of `prefixes` (project-relative, whole-subtree
|
|
862
|
-
* globs only — see `wholeSubtreePrefixes`) after a save or restore;
|
|
863
|
-
* `loadOutputDirsBatch` reads them back; `outputDirsCurrent` is true iff
|
|
864
|
-
* every recorded directory still carries its recorded mtime, which proves
|
|
865
|
-
* no file was added or removed anywhere the glob could see. Machine-local
|
|
866
|
-
* state, like the output rows: a remote ingest records none.
|
|
867
|
-
*/
|
|
868
|
-
recordOutputDirs?(hash: string, projectDir: string, prefixes: readonly string[]): Promise<void>
|
|
869
|
-
loadOutputDirsBatch?(hashes: readonly string[]): Map<string, OutputDirRow[]>
|
|
870
|
-
outputDirsCurrent?(projectDir: string, rows: readonly OutputDirRow[]): Promise<boolean>
|
|
871
|
-
/**
|
|
872
|
-
* Extract the artifact's `outputs/` entries into `projectDir` and —
|
|
873
|
-
* when `workspaceRoot` is given — its `workspace-outputs/` entries
|
|
874
|
-
* into the workspace root. Callers restoring entries that may carry
|
|
875
|
-
* workspace outputs must pass `workspaceRoot`.
|
|
876
|
-
*/
|
|
877
|
-
restoreOutputs(hash: string, projectDir: string, workspaceRoot?: string): Promise<void>
|
|
878
|
-
save(args: {
|
|
879
|
-
hash: string
|
|
880
|
-
/**
|
|
881
|
-
* `exitCode` is deliberately NOT accepted: vx caches only successes, so
|
|
882
|
-
* the stored value is pinned to 0 and there is nothing for a caller to
|
|
883
|
-
* decide. Accepting a number and discarding it invited the one shape that
|
|
884
|
-
* launders a failure into a success — cache a failing task's outputs, read
|
|
885
|
-
* the entry back as exit 0, and the hit classifies `cache-hit` while the
|
|
886
|
-
* broken build's files are restored over a good tree. Unrepresentable
|
|
887
|
-
* beats guarded.
|
|
888
|
-
*/
|
|
889
|
-
entry: Omit<CacheEntry, 'hash' | 'storedAt' | 'outputFiles' | 'exitCode'>
|
|
890
|
-
projectDir: string
|
|
891
|
-
outputFiles: string[]
|
|
892
|
-
/**
|
|
893
|
-
* Set only by ChainedCache when an EARLIER layer in the chain already
|
|
894
|
-
* wrote this artifact to the same local handle: the local pack + write
|
|
895
|
-
* is skipped and only the layer's remote side acts. A layer with no
|
|
896
|
-
* remote side treats it as a full no-op.
|
|
897
|
-
*/
|
|
898
|
-
skipLocalWrite?: boolean
|
|
899
|
-
/**
|
|
900
|
-
* Resolved `outputs.workspaceFiles` (absolute paths) + the root
|
|
901
|
-
* they're relative to. Packed under `workspace-outputs/<rel>`.
|
|
902
|
-
* Omitted → artifact bytes identical to the pre-workspaceFiles
|
|
903
|
-
* format.
|
|
904
|
-
*/
|
|
905
|
-
workspaceOutputFiles?: string[]
|
|
906
|
-
workspaceRoot?: string
|
|
907
|
-
/**
|
|
908
|
-
* Cache-key components for this entry (the Tier-3 input
|
|
909
|
-
* fingerprint). Persisted to `entry_inputs` inside the same
|
|
910
|
-
* transaction as the entry row, via `INSERT OR IGNORE` (a re-save
|
|
911
|
-
* of the same hash is a no-op). Omitted/empty → nothing written.
|
|
912
|
-
* Only computed on the miss/save path — never on a hit.
|
|
913
|
-
*/
|
|
914
|
-
inputComponents?: readonly TaskInputRow[]
|
|
915
|
-
}): Promise<void>
|
|
916
|
-
/**
|
|
917
|
-
* Adopt an artifact produced elsewhere — the remote-hit path. Writes
|
|
918
|
-
* the compressed bytes to `<cacheDir>/<hash>.tar.zst`, parses the
|
|
919
|
-
* tar headers to populate the `output_files` rows, and inserts the
|
|
920
|
-
* `entries` row using the caller-supplied `meta`. After this returns,
|
|
921
|
-
* the next `get(hash)` resolves locally.
|
|
922
|
-
*/
|
|
923
|
-
ingest(hash: string, compressed: Uint8Array, meta: IngestMeta): Promise<void>
|
|
924
|
-
recordRun(run: RunRecord): void
|
|
925
|
-
/**
|
|
926
|
-
* Append every run in `runs` to the history in a single SQLite
|
|
927
|
-
* transaction. ~10× faster than calling `recordRun` in a loop when
|
|
928
|
-
* `runs.length > ~50` (one fsync vs. N).
|
|
929
|
-
*/
|
|
930
|
-
recordRuns(runs: readonly RunRecord[]): void
|
|
931
|
-
/**
|
|
932
|
-
* Record a whole `vx run` atomically: the per-task `runs` rows and
|
|
933
|
-
* the one `invocations` header row — in a SINGLE transaction (one
|
|
934
|
-
* fsync). Replaces the bare `recordRuns` call in the orchestrator's
|
|
935
|
-
* end-of-run path. Input-fingerprint rows are NOT written here — they
|
|
936
|
-
* ride the entry-save transaction (`save`/`ingest`) so a warm
|
|
937
|
-
* all-cache-hit run writes nothing.
|
|
938
|
-
*/
|
|
939
|
-
recordRunBundle(bundle: { runs: readonly RunRecord[]; invocation: InvocationRecord }): void
|
|
940
|
-
stats(opts?: CacheStatsOptions): CacheStats
|
|
941
|
-
/**
|
|
942
|
-
* Content-hash a file with an mtime+size fast path. If the
|
|
943
|
-
* `(mtime_ms, size_bytes)` of `filePath` match a previously seen
|
|
944
|
-
* row, return the stored xxh3 digest instead of re-reading the
|
|
945
|
-
* bytes. Otherwise read + hash + upsert. The hash is byte-for-byte
|
|
946
|
-
* identical to what a fresh content-hash would produce — pure
|
|
947
|
-
* optimization, no cache-key change.
|
|
948
|
-
*/
|
|
949
|
-
hashFile(filePath: string): Promise<string>
|
|
950
|
-
/**
|
|
951
|
-
* Absolute path to the on-disk outputs artifact for a hash —
|
|
952
|
-
* `<cacheDir>/<hash>.tar.zst`. Returns the path whether or
|
|
953
|
-
* not the artifact exists. Exposed for telemetry / dashboards;
|
|
954
|
-
* `restoreOutputs` is the canonical way to materialize the bytes.
|
|
955
|
-
*/
|
|
956
|
-
outputsPath(hash: string): string
|
|
957
|
-
prune(options: PruneOptions): Promise<PruneResult>
|
|
958
|
-
close(): void
|
|
959
|
-
}
|
|
960
|
-
|
|
961
|
-
/**
|
|
962
|
-
* Convenience alias for the `save()` args. Used by `LayeredCache` to
|
|
963
|
-
* forward call args without redeclaring the structural shape — NOT
|
|
964
|
-
* part of the conceptual cache contract; consumers should call
|
|
965
|
-
* `CacheLayer.save({ ... })` directly.
|
|
966
|
-
*
|
|
967
|
-
* @internal
|
|
968
|
-
*/
|
|
969
|
-
export type SaveArgs = Parameters<CacheLayer['save']>[0]
|
|
228
|
+
// The contract, the policy grammar and the zstd framing live beside this
|
|
229
|
+
// file; they are re-exported here so an importer of `./cache.js` — the
|
|
230
|
+
// module index, the sibling layers, the tests — keeps one path.
|
|
231
|
+
export * from './layer.js'
|
|
232
|
+
export * from './policy.js'
|
|
233
|
+
export { zstdContentSize } from './zstd.js'
|
|
970
234
|
|
|
971
235
|
interface EntryRow {
|
|
972
236
|
hash: string
|
|
@@ -979,6 +243,8 @@ interface EntryRow {
|
|
|
979
243
|
stdout: string
|
|
980
244
|
created_at: number
|
|
981
245
|
accessed_at: number
|
|
246
|
+
cpu_ms: number | null
|
|
247
|
+
peak_rss_bytes: number | null
|
|
982
248
|
}
|
|
983
249
|
|
|
984
250
|
function entryOf(row: EntryRow, fileRows: OutputFileRow[]): CacheEntry {
|
|
@@ -988,6 +254,8 @@ function entryOf(row: EntryRow, fileRows: OutputFileRow[]): CacheEntry {
|
|
|
988
254
|
command: row.command,
|
|
989
255
|
exitCode: row.exit_code,
|
|
990
256
|
durationMs: row.duration_ms,
|
|
257
|
+
...(row.cpu_ms !== null ? { cpuMs: row.cpu_ms } : {}),
|
|
258
|
+
...(row.peak_rss_bytes !== null ? { peakRssBytes: row.peak_rss_bytes } : {}),
|
|
991
259
|
outputFiles: fileRows.map((r) => r.path),
|
|
992
260
|
outputRows: fileRows,
|
|
993
261
|
stdout: row.stdout,
|
|
@@ -996,6 +264,15 @@ function entryOf(row: EntryRow, fileRows: OutputFileRow[]): CacheEntry {
|
|
|
996
264
|
}
|
|
997
265
|
}
|
|
998
266
|
|
|
267
|
+
/** The sidecar's `exec` from a save's entry: only the axes the runner reported. */
|
|
268
|
+
function usageOfEntry(entry: { cpuMs?: number; peakRssBytes?: number }): ExecUsage | undefined {
|
|
269
|
+
if (entry.cpuMs === undefined && entry.peakRssBytes === undefined) return undefined
|
|
270
|
+
return {
|
|
271
|
+
...(entry.cpuMs !== undefined ? { cpuMs: entry.cpuMs } : {}),
|
|
272
|
+
...(entry.peakRssBytes !== undefined ? { peakRssBytes: entry.peakRssBytes } : {}),
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
999
276
|
export class Cache implements CacheLayer {
|
|
1000
277
|
/** This IS the local layer — there is nothing slower behind it. */
|
|
1001
278
|
readonly hasRemote = false
|
|
@@ -1005,19 +282,12 @@ export class Cache implements CacheLayer {
|
|
|
1005
282
|
private readonly selectEntry: ReturnType<Database['prepare']>
|
|
1006
283
|
private readonly bumpAccessed: ReturnType<Database['prepare']>
|
|
1007
284
|
private readonly touched = new Set<string>()
|
|
1008
|
-
private readonly insertRun: ReturnType<Database['prepare']>
|
|
1009
|
-
private readonly insertInvocation: ReturnType<Database['prepare']>
|
|
1010
285
|
private readonly insertEntryInput: ReturnType<Database['prepare']>
|
|
1011
|
-
|
|
1012
|
-
private readonly
|
|
1013
|
-
private readonly
|
|
1014
|
-
private readonly
|
|
1015
|
-
private readonly
|
|
1016
|
-
private readonly deleteOutputFiles: ReturnType<Database['prepare']>
|
|
1017
|
-
private readonly selectConfigEval: ReturnType<Database['prepare']>
|
|
1018
|
-
private readonly insertConfigEval: ReturnType<Database['prepare']>
|
|
1019
|
-
/** Memoized repo object format for blob-OID hashing (lazy-detected). */
|
|
1020
|
-
private objectFormat: 'sha1' | 'sha256' | null = null
|
|
286
|
+
/** The per-file (mtime, size) → blob-OID memo behind `hashFile`. */
|
|
287
|
+
private readonly files: FileHashStore
|
|
288
|
+
private readonly configEvals: ConfigEvalTable
|
|
289
|
+
private readonly outputs: OutputIndex
|
|
290
|
+
private readonly history: RunHistory
|
|
1021
291
|
|
|
1022
292
|
/**
|
|
1023
293
|
* Local-layer read/write gates for the task ARTIFACT path only.
|
|
@@ -1029,6 +299,16 @@ export class Cache implements CacheLayer {
|
|
|
1029
299
|
private readonly read: boolean
|
|
1030
300
|
private readonly write: boolean
|
|
1031
301
|
|
|
302
|
+
/**
|
|
303
|
+
* Set when THIS open found an index written by another `SCHEMA_VERSION`
|
|
304
|
+
* and dropped every table. The next open sees the current version and
|
|
305
|
+
* reports null, so whoever opened first is the one that can say so —
|
|
306
|
+
* `noteSchemaReset` is that one line, at every opener that has a
|
|
307
|
+
* channel. Without it an upgrade empties the cache and the history in
|
|
308
|
+
* silence, and the all-miss run that follows looks like a bug.
|
|
309
|
+
*/
|
|
310
|
+
readonly schemaReset: SchemaReset | null = null
|
|
311
|
+
|
|
1032
312
|
constructor(
|
|
1033
313
|
private readonly cacheDir: string,
|
|
1034
314
|
localPolicy: { read: boolean; write: boolean } = { read: true, write: true },
|
|
@@ -1069,12 +349,15 @@ export class Cache implements CacheLayer {
|
|
|
1069
349
|
// Schema-version gate runs BEFORE the rest of the schema lands so
|
|
1070
350
|
// a column rename (e.g. v15's `sha256` → `content_hash`) actually
|
|
1071
351
|
// takes effect on stale DBs. Pre-alpha: no migrations, just drop
|
|
1072
|
-
// and recreate.
|
|
1073
|
-
//
|
|
352
|
+
// and recreate. Artifacts on disk become orphans: a lookup never
|
|
353
|
+
// sees them (rows first, then the file), the next save of the same
|
|
354
|
+
// key renames over them, and `prune()`'s orphan sweep unlinks the
|
|
355
|
+
// rest.
|
|
1074
356
|
const meta = this.db.prepare("SELECT value FROM schema_meta WHERE key = 'version'").get() as
|
|
1075
357
|
| { value: string }
|
|
1076
358
|
| undefined
|
|
1077
359
|
if (meta && meta.value !== SCHEMA_VERSION) {
|
|
360
|
+
this.schemaReset = { from: meta.value, to: SCHEMA_VERSION }
|
|
1078
361
|
this.db.exec(
|
|
1079
362
|
'DROP TABLE IF EXISTS entries; DROP TABLE IF EXISTS runs; DROP TABLE IF EXISTS file_hashes; DROP TABLE IF EXISTS output_files; DROP TABLE IF EXISTS invocations; DROP TABLE IF EXISTS run_task_inputs; DROP TABLE IF EXISTS entry_inputs; DROP TABLE IF EXISTS config_evals;',
|
|
1080
363
|
)
|
|
@@ -1111,7 +394,10 @@ export class Cache implements CacheLayer {
|
|
|
1111
394
|
size_bytes INTEGER NOT NULL,
|
|
1112
395
|
stdout TEXT NOT NULL DEFAULT '',
|
|
1113
396
|
created_at INTEGER NOT NULL,
|
|
1114
|
-
accessed_at INTEGER NOT NULL
|
|
397
|
+
accessed_at INTEGER NOT NULL,
|
|
398
|
+
-- v26: the producing execution's usage, from the artifact's sidecar.
|
|
399
|
+
cpu_ms INTEGER,
|
|
400
|
+
peak_rss_bytes INTEGER
|
|
1115
401
|
);
|
|
1116
402
|
CREATE TABLE IF NOT EXISTS runs (
|
|
1117
403
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
@@ -1137,18 +423,37 @@ export class Cache implements CacheLayer {
|
|
|
1137
423
|
cache_hit INTEGER,
|
|
1138
424
|
-- v23: attempts a retried task took (>1); NULL for a once-run task.
|
|
1139
425
|
-- The direct within-run flaky signal.
|
|
1140
|
-
attempts INTEGER
|
|
426
|
+
attempts INTEGER,
|
|
427
|
+
-- v25: 1 when the task declared a cache block, 0 when it runs every
|
|
428
|
+
-- time by design; NULL on rows older than the column.
|
|
429
|
+
cached INTEGER
|
|
1141
430
|
);
|
|
1142
|
-
--
|
|
1143
|
-
--
|
|
1144
|
-
-- 1,000
|
|
1145
|
-
-- existing databases (a schema-meta
|
|
1146
|
-
-- an index is not one)
|
|
431
|
+
-- Two whole-table indexes only, and both APPEND: every row of a run carries the
|
|
432
|
+
-- same run_id and a started_at newer than everything before it, so
|
|
433
|
+
-- 1,000 inserts touch a handful of leaf pages. The dropped ones did
|
|
434
|
+
-- not — the DROPs shed them from existing databases (a schema-meta
|
|
435
|
+
-- bump is for stored shapes, and an index is not one):
|
|
436
|
+
-- runs_hash (2026-09-03) had no reader; 11.5 → 3.9 ms per 1,000.
|
|
437
|
+
-- runs_project (project, task) (2026-09-09) scattered every run's
|
|
438
|
+
-- rows over one leaf per pair: the record stage of a 1,000-hit
|
|
439
|
+
-- warm run was 57–79 ms with it and 14–19 ms without, at 166k
|
|
440
|
+
-- rows, and it bought its readers nothing — the history reader
|
|
441
|
+
-- now scans a rowid slice (history.ts), vx why walks
|
|
442
|
+
-- started_at newest-first and stops at the first match.
|
|
443
|
+
-- runs_ended (2026-09-09) served only the retention DELETE, which
|
|
444
|
+
-- prunes on started_at now (a row ends after it starts, so the
|
|
445
|
+
-- 30-day window moves by at most one task's duration).
|
|
1147
446
|
DROP INDEX IF EXISTS runs_hash;
|
|
447
|
+
DROP INDEX IF EXISTS runs_project;
|
|
448
|
+
DROP INDEX IF EXISTS runs_ended;
|
|
1148
449
|
CREATE INDEX IF NOT EXISTS runs_started_at ON runs(started_at);
|
|
1149
|
-
CREATE INDEX IF NOT EXISTS runs_project ON runs(project, task);
|
|
1150
|
-
CREATE INDEX IF NOT EXISTS runs_ended ON runs(ended_at);
|
|
1151
450
|
CREATE INDEX IF NOT EXISTS runs_run_id ON runs(run_id);
|
|
451
|
+
-- The one keyed index, PARTIAL over failed rows: a green run's
|
|
452
|
+
-- inserts only evaluate its predicate, so the append-only cost above
|
|
453
|
+
-- holds, and the flakiness probe after a miss (failure-mode.ts,
|
|
454
|
+
-- "did this key ever fail?") reads a handful of leaves instead of
|
|
455
|
+
-- scanning the table (2026-09-10: 10–95 ms at 170k rows without it).
|
|
456
|
+
CREATE INDEX IF NOT EXISTS runs_failed ON runs(hash) WHERE status = 'failed';
|
|
1152
457
|
-- Per-file (mtime, size, content_hash) cache. Lets Cache.key()
|
|
1153
458
|
-- skip the content-hash on inputs whose stat hasn't changed
|
|
1154
459
|
-- since the last run. Pure performance optimization; the stored
|
|
@@ -1258,41 +563,22 @@ export class Cache implements CacheLayer {
|
|
|
1258
563
|
`)
|
|
1259
564
|
|
|
1260
565
|
this.insertEntry = this.db.prepare(`
|
|
1261
|
-
INSERT INTO entries(hash, project, task, command, exit_code, duration_ms, size_bytes, stdout, created_at, accessed_at)
|
|
1262
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
566
|
+
INSERT INTO entries(hash, project, task, command, exit_code, duration_ms, size_bytes, stdout, created_at, accessed_at, cpu_ms, peak_rss_bytes)
|
|
567
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1263
568
|
ON CONFLICT(hash) DO UPDATE SET
|
|
1264
|
-
stdout
|
|
1265
|
-
project
|
|
1266
|
-
task
|
|
1267
|
-
command
|
|
1268
|
-
exit_code
|
|
1269
|
-
duration_ms
|
|
1270
|
-
size_bytes
|
|
1271
|
-
accessed_at
|
|
569
|
+
stdout = excluded.stdout,
|
|
570
|
+
project = excluded.project,
|
|
571
|
+
task = excluded.task,
|
|
572
|
+
command = excluded.command,
|
|
573
|
+
exit_code = excluded.exit_code,
|
|
574
|
+
duration_ms = excluded.duration_ms,
|
|
575
|
+
size_bytes = excluded.size_bytes,
|
|
576
|
+
accessed_at = excluded.accessed_at,
|
|
577
|
+
cpu_ms = excluded.cpu_ms,
|
|
578
|
+
peak_rss_bytes = excluded.peak_rss_bytes
|
|
1272
579
|
`)
|
|
1273
580
|
this.selectEntry = this.db.prepare('SELECT * FROM entries WHERE hash = ?')
|
|
1274
581
|
this.bumpAccessed = this.db.prepare('UPDATE entries SET accessed_at = ? WHERE hash = ?')
|
|
1275
|
-
this.insertRun = this.db.prepare(`
|
|
1276
|
-
INSERT INTO runs(
|
|
1277
|
-
hash, project, task, status, exit_code, duration_ms, forward_args,
|
|
1278
|
-
started_at, ended_at,
|
|
1279
|
-
run_id, cpu_ms, peak_rss_bytes, wallclock_start_ns, wallclock_end_ns,
|
|
1280
|
-
cache_hit, attempts
|
|
1281
|
-
)
|
|
1282
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1283
|
-
`)
|
|
1284
|
-
this.insertInvocation = this.db.prepare(`
|
|
1285
|
-
INSERT INTO invocations(
|
|
1286
|
-
run_id, command, requested_tasks, cache_policy, concurrency, flow,
|
|
1287
|
-
started_at, ended_at, total_duration_ms,
|
|
1288
|
-
task_count, failed_count, hit_count, hit_local_count, hit_remote_count,
|
|
1289
|
-
exit_ok,
|
|
1290
|
-
commit_sha, branch, dirty, ci, ci_provider,
|
|
1291
|
-
host, os, arch, vx_version, tags
|
|
1292
|
-
)
|
|
1293
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1294
|
-
ON CONFLICT(run_id) DO NOTHING
|
|
1295
|
-
`)
|
|
1296
582
|
// INSERT OR IGNORE: re-saving the same hash (idempotent ingest /
|
|
1297
583
|
// overlapping concurrent saves) leaves the existing rows untouched —
|
|
1298
584
|
// identical inputs derive the identical hash, so the rows are too.
|
|
@@ -1300,204 +586,37 @@ export class Cache implements CacheLayer {
|
|
|
1300
586
|
INSERT OR IGNORE INTO entry_inputs(entry_hash, kind, name, hash)
|
|
1301
587
|
VALUES (?, ?, ?, ?)
|
|
1302
588
|
`)
|
|
1303
|
-
this
|
|
1304
|
-
|
|
1305
|
-
)
|
|
1306
|
-
this.
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
ON CONFLICT(path) DO UPDATE SET
|
|
1310
|
-
mtime_ms = excluded.mtime_ms,
|
|
1311
|
-
size_bytes = excluded.size_bytes,
|
|
1312
|
-
ctime_ms = excluded.ctime_ms,
|
|
1313
|
-
ino = excluded.ino,
|
|
1314
|
-
content_hash = excluded.content_hash,
|
|
1315
|
-
seen_at = excluded.seen_at
|
|
1316
|
-
`)
|
|
1317
|
-
this.insertOutputFile = this.db.prepare(`
|
|
1318
|
-
INSERT INTO output_files(entry_hash, path, size_bytes, mode, mtime_ms)
|
|
1319
|
-
VALUES (?, ?, ?, ?, ?)
|
|
1320
|
-
ON CONFLICT(entry_hash, path) DO UPDATE SET
|
|
1321
|
-
size_bytes = excluded.size_bytes,
|
|
1322
|
-
mode = excluded.mode,
|
|
1323
|
-
mtime_ms = excluded.mtime_ms
|
|
1324
|
-
`)
|
|
1325
|
-
this.deleteOutputFiles = this.db.prepare('DELETE FROM output_files WHERE entry_hash = ?')
|
|
1326
|
-
this.insertOutputDir = this.db.prepare(
|
|
1327
|
-
'INSERT INTO output_dirs(entry_hash, path, mtime_ms) VALUES (?, ?, ?)',
|
|
1328
|
-
)
|
|
1329
|
-
this.deleteOutputDirs = this.db.prepare('DELETE FROM output_dirs WHERE entry_hash = ?')
|
|
1330
|
-
this.selectConfigEval = this.db.prepare('SELECT json FROM config_evals WHERE key = ?')
|
|
1331
|
-
this.insertConfigEval = this.db.prepare(
|
|
1332
|
-
'INSERT OR REPLACE INTO config_evals(key, json, created_at) VALUES (?, ?, ?)',
|
|
1333
|
-
)
|
|
589
|
+
// The slices: each owns its statements over this handle and its table(s);
|
|
590
|
+
// the schema above is the one place every table is declared.
|
|
591
|
+
this.files = new FileHashStore(this.db, cacheDir)
|
|
592
|
+
this.configEvals = new ConfigEvalTable(this.db, localPolicy)
|
|
593
|
+
this.outputs = new OutputIndex(this.db)
|
|
594
|
+
this.history = new RunHistory(this.db)
|
|
1334
595
|
}
|
|
1335
596
|
|
|
1336
|
-
|
|
597
|
+
// --- config evaluations: `ConfigEvalStore`, delegated to `ConfigEvalTable` ---
|
|
1337
598
|
getConfigEval(key: string): string | null {
|
|
1338
|
-
|
|
1339
|
-
const row = this.selectConfigEval.get(key) as { json: string } | null
|
|
1340
|
-
return row?.json ?? null
|
|
599
|
+
return this.configEvals.getConfigEval(key)
|
|
1341
600
|
}
|
|
1342
|
-
|
|
1343
|
-
/** `ConfigEvalStore`: each config's ordered closure, one `IN` query per 900 paths. */
|
|
1344
601
|
getConfigClosures(configPaths: readonly string[]): Map<string, string[]> {
|
|
1345
|
-
|
|
1346
|
-
if (!this.read || configPaths.length === 0) return out
|
|
1347
|
-
for (let i = 0; i < configPaths.length; i += 900) {
|
|
1348
|
-
const chunk = configPaths.slice(i, i + 900)
|
|
1349
|
-
const rows = this.db
|
|
1350
|
-
.query(
|
|
1351
|
-
`SELECT config_path, files_json FROM config_closures WHERE config_path IN (${chunk.map(() => '?').join(',')})`,
|
|
1352
|
-
)
|
|
1353
|
-
.all(...(chunk as readonly SQLQueryBindings[])) as Array<{
|
|
1354
|
-
config_path: string
|
|
1355
|
-
files_json: string
|
|
1356
|
-
}>
|
|
1357
|
-
for (const r of rows) out.set(r.config_path, JSON.parse(r.files_json) as string[])
|
|
1358
|
-
}
|
|
1359
|
-
return out
|
|
602
|
+
return this.configEvals.getConfigClosures(configPaths)
|
|
1360
603
|
}
|
|
1361
|
-
|
|
1362
|
-
/** `ConfigEvalStore`: remember a config's ordered closure, honouring the local WRITE axis. */
|
|
1363
604
|
putConfigClosure(configPath: string, files: readonly string[]): void {
|
|
1364
|
-
|
|
1365
|
-
this.db
|
|
1366
|
-
.prepare(
|
|
1367
|
-
'INSERT INTO config_closures(config_path, files_json, created_at) VALUES (?, ?, ?) ON CONFLICT(config_path) DO UPDATE SET files_json = excluded.files_json, created_at = excluded.created_at',
|
|
1368
|
-
)
|
|
1369
|
-
.run(configPath, JSON.stringify(files), Date.now())
|
|
605
|
+
this.configEvals.putConfigClosure(configPath, files)
|
|
1370
606
|
}
|
|
1371
|
-
|
|
1372
|
-
/** `ConfigEvalStore`: the batched read — one `IN` query per 900 keys, honouring the local READ axis. */
|
|
1373
607
|
getConfigEvals(keys: readonly string[]): Map<string, string> {
|
|
1374
|
-
|
|
1375
|
-
if (!this.read || keys.length === 0) return out
|
|
1376
|
-
for (let i = 0; i < keys.length; i += 900) {
|
|
1377
|
-
const chunk = keys.slice(i, i + 900)
|
|
1378
|
-
const rows = this.db
|
|
1379
|
-
.query(
|
|
1380
|
-
`SELECT key, json FROM config_evals WHERE key IN (${chunk.map(() => '?').join(',')})`,
|
|
1381
|
-
)
|
|
1382
|
-
.all(...(chunk as readonly SQLQueryBindings[])) as Array<{ key: string; json: string }>
|
|
1383
|
-
for (const r of rows) out.set(r.key, r.json)
|
|
1384
|
-
}
|
|
1385
|
-
return out
|
|
608
|
+
return this.configEvals.getConfigEvals(keys)
|
|
1386
609
|
}
|
|
1387
|
-
|
|
1388
|
-
/** `ConfigEvalStore`: remember a validated evaluation, honouring the local WRITE axis. */
|
|
1389
610
|
putConfigEval(key: string, json: string): void {
|
|
1390
|
-
|
|
1391
|
-
this.insertConfigEval.run(key, json, Date.now())
|
|
611
|
+
this.configEvals.putConfigEval(key, json)
|
|
1392
612
|
}
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
* fast path. If the `file_hashes` table has a row for `path` whose
|
|
1397
|
-
* `(mtime_ms, size_bytes, ctime_ms, ino)` all match the current
|
|
1398
|
-
* stat, we reuse the stored content_hash (a memory + SQLite lookup,
|
|
1399
|
-
* no disk read). Otherwise we read + hash + upsert. All four fields
|
|
1400
|
-
* are load-bearing — see the comment at the comparison.
|
|
1401
|
-
*
|
|
1402
|
-
* The OID is byte-identical to what `git hash-object` (and the git
|
|
1403
|
-
* index) computes for the same content, so this fallback and the
|
|
1404
|
-
* `CacheKeyInput.fileHashes` index-OID fast path agree on any file
|
|
1405
|
-
* git stores verbatim. They do NOT agree when a clean filter
|
|
1406
|
-
* (`text`/`eol`/`ident`) is active: the index blob is the filtered
|
|
1407
|
-
* form while this hashes the worktree bytes, so `inputs.ts` drops
|
|
1408
|
-
* the index OID for those paths and routes them here.
|
|
1409
|
-
*/
|
|
1410
|
-
async hashFile(filePath: string): Promise<string> {
|
|
1411
|
-
// statSync intentional: a single stat is ~1.6µs (Bun 1.3); the
|
|
1412
|
-
// async-stat equivalent adds ~75µs of Promise machinery per call.
|
|
1413
|
-
// Promise.all over the batched callers (key derivation) gives no
|
|
1414
|
-
// I/O parallelism benefit because the stat is faster than the
|
|
1415
|
-
// threadpool dispatch overhead.
|
|
1416
|
-
let st
|
|
1417
|
-
try {
|
|
1418
|
-
st = statSync(filePath)
|
|
1419
|
-
} catch {
|
|
1420
|
-
// Caller is responsible for skipping files that don't exist;
|
|
1421
|
-
// fall through to the content-hash path which will throw with
|
|
1422
|
-
// a more useful error.
|
|
1423
|
-
return await this.hashFileFromDisk(filePath)
|
|
1424
|
-
}
|
|
1425
|
-
const mtimeMs = Math.floor(st.mtimeMs)
|
|
1426
|
-
const size = st.size
|
|
1427
|
-
// ctime + ino are what make this memo SAFE, not merely fast. mtime is
|
|
1428
|
-
// caller-settable, so (mtime, size) alone hands back the previous run's
|
|
1429
|
-
// digest for genuinely different bytes whenever a producer preserves
|
|
1430
|
-
// mtime — `tar -x`, `unzip`, `cp -p`, `rsync --times`, any
|
|
1431
|
-
// SOURCE_DATE_EPOCH generator — which is a stale cache hit. `utimes`
|
|
1432
|
-
// cannot suppress ctime without root, and an atomic write-then-rename
|
|
1433
|
-
// changes the inode; git's own index keys on ctime+ino+dev for exactly
|
|
1434
|
-
// this reason. Both fields come from the stat we already took.
|
|
1435
|
-
const ctimeMs = Math.floor(st.ctimeMs)
|
|
1436
|
-
const ino = Number(st.ino)
|
|
1437
|
-
const row = this.selectFileHash.get(filePath) as
|
|
1438
|
-
| {
|
|
1439
|
-
mtime_ms: number
|
|
1440
|
-
size_bytes: number
|
|
1441
|
-
ctime_ms: number
|
|
1442
|
-
ino: number
|
|
1443
|
-
content_hash: string
|
|
1444
|
-
}
|
|
1445
|
-
| undefined
|
|
1446
|
-
if (
|
|
1447
|
-
row &&
|
|
1448
|
-
row.mtime_ms === mtimeMs &&
|
|
1449
|
-
row.size_bytes === size &&
|
|
1450
|
-
row.ctime_ms === ctimeMs &&
|
|
1451
|
-
row.ino === ino
|
|
1452
|
-
) {
|
|
1453
|
-
return row.content_hash
|
|
1454
|
-
}
|
|
1455
|
-
const ch = await this.hashFileFromDisk(filePath)
|
|
1456
|
-
this.upsertFileHash.run(filePath, mtimeMs, size, ctimeMs, ino, ch, Date.now())
|
|
1457
|
-
return ch
|
|
1458
|
-
}
|
|
1459
|
-
|
|
1460
|
-
/**
|
|
1461
|
-
* Git blob OID of the file's bytes:
|
|
1462
|
-
* `hex(HASH("blob " + byteLength + "\0" + content))`, where HASH is
|
|
1463
|
-
* the repo's object format. Same value `git hash-object` prints and
|
|
1464
|
-
* the same value the index stores. Computed in-process — no git
|
|
1465
|
-
* spawn per file.
|
|
1466
|
-
*/
|
|
1467
|
-
private async hashFileFromDisk(filePath: string): Promise<string> {
|
|
1468
|
-
const bytes = await Bun.file(filePath).bytes()
|
|
1469
|
-
const hasher = new Bun.CryptoHasher(this.objectFormat ?? this.detectObjectFormat(filePath))
|
|
1470
|
-
hasher.update(`blob ${bytes.byteLength}\0`)
|
|
1471
|
-
hasher.update(bytes)
|
|
1472
|
-
return hasher.digest('hex')
|
|
613
|
+
// --- input file hashes: delegated to `FileHashStore` (see file-hashes.ts) ---
|
|
614
|
+
hashFile(filePath: string): Promise<string> {
|
|
615
|
+
return this.files.hashFile(filePath)
|
|
1473
616
|
}
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
* Repo object format — sha1 unless the repo was created with
|
|
1477
|
-
* `--object-format=sha256`. One `git rev-parse` spawn per Cache
|
|
1478
|
-
* lifetime, and only when at least one file misses the mtime+size
|
|
1479
|
-
* memo. Outside a repo (unit fixtures) we default to sha1, which is
|
|
1480
|
-
* still a deterministic blob-OID domain.
|
|
1481
|
-
*/
|
|
1482
|
-
private detectObjectFormat(nearPath: string): 'sha1' | 'sha256' {
|
|
1483
|
-
let detected: 'sha1' | 'sha256' = 'sha1'
|
|
1484
|
-
try {
|
|
1485
|
-
const proc = Bun.spawnSync({
|
|
1486
|
-
cmd: ['git', 'rev-parse', '--show-object-format'],
|
|
1487
|
-
cwd: path.dirname(nearPath),
|
|
1488
|
-
stdout: 'pipe',
|
|
1489
|
-
stderr: 'pipe',
|
|
1490
|
-
})
|
|
1491
|
-
if (proc.exitCode === 0 && new TextDecoder().decode(proc.stdout).trim() === 'sha256') {
|
|
1492
|
-
detected = 'sha256'
|
|
1493
|
-
}
|
|
1494
|
-
} catch {
|
|
1495
|
-
// git unavailable → sha1 default keeps hashing deterministic.
|
|
1496
|
-
}
|
|
1497
|
-
this.objectFormat = detected
|
|
1498
|
-
return detected
|
|
617
|
+
hashFiles(paths: readonly string[]): Promise<Map<string, string>> {
|
|
618
|
+
return this.files.hashFiles(paths)
|
|
1499
619
|
}
|
|
1500
|
-
|
|
1501
620
|
async key(input: CacheKeyInput): Promise<string> {
|
|
1502
621
|
// Seed-chained xxHash3: each step folds one field into the
|
|
1503
622
|
// running digest via `xxh3(part, prevDigest)`. Equivalent to the
|
|
@@ -1607,7 +726,7 @@ export class Cache implements CacheLayer {
|
|
|
1607
726
|
// Local reads disabled (e.g. `--force` / `--cache=local:w`): report a
|
|
1608
727
|
// miss so the task re-executes. The artifact + index are untouched.
|
|
1609
728
|
if (!this.read) return null
|
|
1610
|
-
return
|
|
729
|
+
return this.readEntry(hash)
|
|
1611
730
|
}
|
|
1612
731
|
|
|
1613
732
|
/**
|
|
@@ -1619,8 +738,8 @@ export class Cache implements CacheLayer {
|
|
|
1619
738
|
* `ingest` had just written, so the task re-executed and re-uploaded on
|
|
1620
739
|
* every single run.
|
|
1621
740
|
*/
|
|
1622
|
-
|
|
1623
|
-
return
|
|
741
|
+
getIngested(hash: string): Promise<CacheEntry | null> {
|
|
742
|
+
return this.readEntry(hash)
|
|
1624
743
|
}
|
|
1625
744
|
|
|
1626
745
|
private async readEntry(hash: string): Promise<CacheEntry | null> {
|
|
@@ -1629,7 +748,7 @@ export class Cache implements CacheLayer {
|
|
|
1629
748
|
|
|
1630
749
|
// Verify the tar artifact actually exists. The DB and the
|
|
1631
750
|
// filesystem can drift if someone manually deletes the cache dir.
|
|
1632
|
-
if (!(
|
|
751
|
+
if (!existsSync(this.tarPath(hash))) return null
|
|
1633
752
|
|
|
1634
753
|
// Deferred: per-hit UPDATEs cost ~60 ms across 2000+ probes on a
|
|
1635
754
|
// full-cache run. Hashes are collected and flushed as ONE batched
|
|
@@ -1670,7 +789,7 @@ export class Cache implements CacheLayer {
|
|
|
1670
789
|
)
|
|
1671
790
|
}
|
|
1672
791
|
if (rows.length === 0) return out
|
|
1673
|
-
const present =
|
|
792
|
+
const present = rows.map((r) => existsSync(this.tarPath(r.hash)))
|
|
1674
793
|
const live = rows.filter((_r, i) => present[i])
|
|
1675
794
|
const liveHashes = live.map((r) => r.hash)
|
|
1676
795
|
const fileRows = this.loadOutputFilesBatch(liveHashes)
|
|
@@ -1690,7 +809,7 @@ export class Cache implements CacheLayer {
|
|
|
1690
809
|
if (!this.read) return null
|
|
1691
810
|
const row = this.selectEntry.get(hash) as EntryRow | undefined
|
|
1692
811
|
if (!row) return null
|
|
1693
|
-
return (
|
|
812
|
+
return existsSync(this.tarPath(hash)) ? 'local' : null
|
|
1694
813
|
}
|
|
1695
814
|
|
|
1696
815
|
// Local cache has no slower layer to warm from — prefetch is a no-op.
|
|
@@ -1700,169 +819,34 @@ export class Cache implements CacheLayer {
|
|
|
1700
819
|
return false
|
|
1701
820
|
}
|
|
1702
821
|
|
|
822
|
+
// --- output fingerprints: delegated to `OutputIndex` (see output-index.ts) ---
|
|
823
|
+
// The slices' proofs and hashes are returned as the slice's own promise —
|
|
824
|
+
// no `async` wrapper, no `return await`: each of those added a promise and
|
|
825
|
+
// a microtask hop per call, and a 1,000-hit warm run pays the proofs
|
|
826
|
+
// 2,000 times (measured 2026-09-10: +3 ms on `classify + probe`).
|
|
1703
827
|
loadOutputFilesBatch(hashes: readonly string[]): Map<string, OutputFileRow[]> {
|
|
1704
|
-
|
|
1705
|
-
if (hashes.length === 0) return out
|
|
1706
|
-
// Inline placeholders for an IN-list — bun:sqlite doesn't ship
|
|
1707
|
-
// rarray, but `IN (?, ?, …)` with N≤~999 is fast and avoids per-
|
|
1708
|
-
// hash select.get() overhead. `db.query` (not `db.prepare`) caches the
|
|
1709
|
-
// compiled statement keyed by the SQL text — so the dominant single-hash
|
|
1710
|
-
// warm-hit path (called up to 3× per hit) reuses one statement instead of
|
|
1711
|
-
// recompiling on every call.
|
|
1712
|
-
const placeholders = hashes.map(() => '?').join(',')
|
|
1713
|
-
const stmt = this.db.query(
|
|
1714
|
-
`SELECT entry_hash, path, size_bytes, mode, mtime_ms FROM output_files WHERE entry_hash IN (${placeholders})`,
|
|
1715
|
-
)
|
|
1716
|
-
const rows = stmt.all(...(hashes as readonly SQLQueryBindings[])) as Array<{
|
|
1717
|
-
entry_hash: string
|
|
1718
|
-
path: string
|
|
1719
|
-
size_bytes: number
|
|
1720
|
-
mode: number
|
|
1721
|
-
mtime_ms: number
|
|
1722
|
-
}>
|
|
1723
|
-
for (const r of rows) {
|
|
1724
|
-
let list = out.get(r.entry_hash)
|
|
1725
|
-
if (!list) {
|
|
1726
|
-
list = []
|
|
1727
|
-
out.set(r.entry_hash, list)
|
|
1728
|
-
}
|
|
1729
|
-
list.push({ path: r.path, size: r.size_bytes, mode: r.mode, mtimeMs: r.mtime_ms })
|
|
1730
|
-
}
|
|
1731
|
-
return out
|
|
828
|
+
return this.outputs.loadOutputFilesBatch(hashes)
|
|
1732
829
|
}
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
// Empty manifest case (a task produced no outputs) → trivially
|
|
1736
|
-
// current; the on-disk tree under projectDir is whatever it was,
|
|
1737
|
-
// and nothing was supposed to land there.
|
|
1738
|
-
if (expected.length === 0) return true
|
|
1739
|
-
// Promise-form stat, deliberately: statSync is ~2 µs against ~13 µs in
|
|
1740
|
-
// isolation, but this runs under the scheduler's concurrency and the
|
|
1741
|
-
// async form keeps the stats on the thread pool in parallel — the sync
|
|
1742
|
-
// version measured slower on a 1000-hit warm run (2026-09-02).
|
|
1743
|
-
const results = await Promise.all(
|
|
1744
|
-
expected.map(async (e) => {
|
|
1745
|
-
try {
|
|
1746
|
-
const s = await stat(path.join(projectDir, e.path))
|
|
1747
|
-
return (
|
|
1748
|
-
s.size === e.size &&
|
|
1749
|
-
(s.mode & 0o777) === (e.mode & 0o777) &&
|
|
1750
|
-
// MILLISECOND comparison (sub-ms tolerance for the float
|
|
1751
|
-
// round-trip through utimes). Save rows carry stat-ms and
|
|
1752
|
-
// restoreOutputs re-syncs restored files to the row value,
|
|
1753
|
-
// so equality holds exactly in steady state; legacy
|
|
1754
|
-
// second-precision rows converge on their first restore.
|
|
1755
|
-
// Residual blind spot: a same-size edit landing in the SAME
|
|
1756
|
-
// millisecond as the recorded write, or a deliberately
|
|
1757
|
-
// forged mtime (touch -r) — the trade every mtime-based
|
|
1758
|
-
// skip check accepts.
|
|
1759
|
-
Math.abs(s.mtimeMs - e.mtimeMs) < 1
|
|
1760
|
-
)
|
|
1761
|
-
} catch {
|
|
1762
|
-
return false
|
|
1763
|
-
}
|
|
1764
|
-
}),
|
|
1765
|
-
)
|
|
1766
|
-
return results.every(Boolean)
|
|
830
|
+
isOutputsCurrent(projectDir: string, expected: readonly OutputFileRow[]): Promise<boolean> {
|
|
831
|
+
return this.outputs.isOutputsCurrent(projectDir, expected)
|
|
1767
832
|
}
|
|
1768
|
-
|
|
1769
833
|
outputsPath(hash: string): string {
|
|
1770
834
|
return this.tarPath(hash)
|
|
1771
835
|
}
|
|
1772
836
|
|
|
1773
|
-
/**
|
|
1774
|
-
* Snapshot every directory under each of `prefixes` for `hash`. Called
|
|
1775
|
-
* after a save and after a restore, when the tree is known to equal the
|
|
1776
|
-
* entry's set. Symlinked directories are not descended (the output walk
|
|
1777
|
-
* refuses them too). Over `OUTPUT_DIRS_CAP` directories, or on any
|
|
1778
|
-
* error, the rows are cleared and the next hit keeps the walk.
|
|
1779
|
-
*/
|
|
1780
837
|
async recordOutputDirs(
|
|
1781
838
|
hash: string,
|
|
1782
839
|
projectDir: string,
|
|
1783
840
|
prefixes: readonly string[],
|
|
1784
841
|
): Promise<void> {
|
|
1785
|
-
|
|
1786
|
-
const walk = async (rel: string): Promise<boolean> => {
|
|
1787
|
-
const abs = path.join(projectDir, rel)
|
|
1788
|
-
let st
|
|
1789
|
-
try {
|
|
1790
|
-
st = await lstat(abs)
|
|
1791
|
-
} catch {
|
|
1792
|
-
return false
|
|
1793
|
-
}
|
|
1794
|
-
if (!st.isDirectory()) return false
|
|
1795
|
-
rows.push([rel, st.mtimeMs])
|
|
1796
|
-
if (rows.length > OUTPUT_DIRS_CAP) return false
|
|
1797
|
-
let entries
|
|
1798
|
-
try {
|
|
1799
|
-
entries = await readdir(abs, { withFileTypes: true })
|
|
1800
|
-
} catch {
|
|
1801
|
-
return false
|
|
1802
|
-
}
|
|
1803
|
-
for (const e of entries) {
|
|
1804
|
-
if (e.isDirectory() && !e.isSymbolicLink()) {
|
|
1805
|
-
if (!(await walk(`${rel}/${e.name}`))) return false
|
|
1806
|
-
}
|
|
1807
|
-
}
|
|
1808
|
-
return true
|
|
1809
|
-
}
|
|
1810
|
-
let ok = true
|
|
1811
|
-
for (const prefix of prefixes) {
|
|
1812
|
-
if (!(await walk(prefix))) {
|
|
1813
|
-
ok = false
|
|
1814
|
-
break
|
|
1815
|
-
}
|
|
1816
|
-
}
|
|
1817
|
-
// All or nothing: a racy directory dropped alone would leave its
|
|
1818
|
-
// parent trusted while an addition inside it bumps only the dropped one.
|
|
1819
|
-
const youngest = Date.now() - OUTPUT_DIRS_RACY_MS
|
|
1820
|
-
if (rows.some(([, mtime]) => mtime > youngest)) ok = false
|
|
1821
|
-
this.db.transaction(() => {
|
|
1822
|
-
this.deleteOutputDirs.run(hash)
|
|
1823
|
-
if (!ok) return
|
|
1824
|
-
for (const [rel, mtime] of rows) this.insertOutputDir.run(hash, rel, mtime)
|
|
1825
|
-
})()
|
|
842
|
+
await this.outputs.recordOutputDirs(hash, projectDir, prefixes)
|
|
1826
843
|
}
|
|
1827
|
-
|
|
1828
844
|
loadOutputDirsBatch(hashes: readonly string[]): Map<string, OutputDirRow[]> {
|
|
1829
|
-
|
|
1830
|
-
if (hashes.length === 0) return out
|
|
1831
|
-
const placeholders = hashes.map(() => '?').join(',')
|
|
1832
|
-
const rows = this.db
|
|
1833
|
-
.query(
|
|
1834
|
-
`SELECT entry_hash, path, mtime_ms FROM output_dirs WHERE entry_hash IN (${placeholders})`,
|
|
1835
|
-
)
|
|
1836
|
-
.all(...(hashes as readonly SQLQueryBindings[])) as Array<{
|
|
1837
|
-
entry_hash: string
|
|
1838
|
-
path: string
|
|
1839
|
-
mtime_ms: number
|
|
1840
|
-
}>
|
|
1841
|
-
for (const r of rows) {
|
|
1842
|
-
const list = out.get(r.entry_hash)
|
|
1843
|
-
const row = { path: r.path, mtimeMs: r.mtime_ms }
|
|
1844
|
-
if (list) list.push(row)
|
|
1845
|
-
else out.set(r.entry_hash, [row])
|
|
1846
|
-
}
|
|
1847
|
-
return out
|
|
845
|
+
return this.outputs.loadOutputDirsBatch(hashes)
|
|
1848
846
|
}
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
async outputDirsCurrent(projectDir: string, rows: readonly OutputDirRow[]): Promise<boolean> {
|
|
1852
|
-
if (rows.length === 0) return false
|
|
1853
|
-
const results = await Promise.all(
|
|
1854
|
-
rows.map(async (r) => {
|
|
1855
|
-
try {
|
|
1856
|
-
const st = await stat(path.join(projectDir, r.path))
|
|
1857
|
-
return st.isDirectory() && Math.abs(st.mtimeMs - r.mtimeMs) < 1
|
|
1858
|
-
} catch {
|
|
1859
|
-
return false
|
|
1860
|
-
}
|
|
1861
|
-
}),
|
|
1862
|
-
)
|
|
1863
|
-
return results.every(Boolean)
|
|
847
|
+
outputDirsCurrent(projectDir: string, rows: readonly OutputDirRow[]): Promise<boolean> {
|
|
848
|
+
return this.outputs.outputDirsCurrent(projectDir, rows)
|
|
1864
849
|
}
|
|
1865
|
-
|
|
1866
850
|
async restoreOutputs(hash: string, projectDir: string, workspaceRoot?: string): Promise<void> {
|
|
1867
851
|
// In-process extraction — no fork+exec on the hot path
|
|
1868
852
|
// (~5-10ms reclaimed per hit vs the prior subprocess `tar -xf`).
|
|
@@ -1881,9 +865,13 @@ export class Cache implements CacheLayer {
|
|
|
1881
865
|
// an emptied output tree. The artifact existed when `get()` probed it, so
|
|
1882
866
|
// its absence now means something removed it underneath us (a concurrent
|
|
1883
867
|
// `vx cache prune` is the documented way) — fail loud; the task re-runs.
|
|
1884
|
-
|
|
868
|
+
const endExists = span('restore: exists')
|
|
869
|
+
const exists = await Bun.file(src).exists()
|
|
870
|
+
endExists()
|
|
871
|
+
if (!exists) {
|
|
1885
872
|
throw new CorruptArtifactError(hash, 'artifact file vanished before restore')
|
|
1886
873
|
}
|
|
874
|
+
const endRows = span('restore: rows')
|
|
1887
875
|
// The index says exactly which files this entry materializes. If the
|
|
1888
876
|
// archive cannot produce one of them, restoring "successfully" leaves a
|
|
1889
877
|
// hole that no later run detects: the skip-restore check compares the
|
|
@@ -1894,6 +882,7 @@ export class Cache implements CacheLayer {
|
|
|
1894
882
|
const expected = rows
|
|
1895
883
|
.filter((r) => workspaceRoot !== undefined || !r.path.startsWith(WORKSPACE_OUTPUT_PREFIX))
|
|
1896
884
|
.map((r) => (r.path.startsWith(WORKSPACE_OUTPUT_PREFIX) ? r.path : `outputs/${r.path}`))
|
|
885
|
+
endRows()
|
|
1897
886
|
const verify = (provided: ReadonlySet<string>): void => {
|
|
1898
887
|
const missing = expected.filter((name) => !provided.has(name))
|
|
1899
888
|
if (missing.length > 0) {
|
|
@@ -1913,12 +902,31 @@ export class Cache implements CacheLayer {
|
|
|
1913
902
|
// when every artifact is a one-file `dist/` (measured: 390 vs 355 ms
|
|
1914
903
|
// per 1 000). The local artifact was validated at ingest, so a missing
|
|
1915
904
|
// declared size is allowed; the output ceiling applies to both.
|
|
905
|
+
const endExtract = span('restore: extract')
|
|
1916
906
|
try {
|
|
907
|
+
// Every step here is an async filesystem call on purpose: a
|
|
908
|
+
// synchronous restore of a small artifact is 2× faster ALONE (1.25 →
|
|
909
|
+
// 0.62 ms sequential, 2026-09-10) and 30% slower in the run, where
|
|
910
|
+
// four workers overlap their round trips and a blocking one stalls
|
|
911
|
+
// the other three (1,000-project restore row 1.1–1.2 s → 1.5 s).
|
|
1917
912
|
const tar = await decodedTar(Bun.file(src), hash)
|
|
1918
913
|
await extractArtifactStream(tar, projectDir, workspaceRoot, verify)
|
|
1919
914
|
} catch (err) {
|
|
1920
915
|
if (err instanceof ArchiveSecurityError || err instanceof CorruptArtifactError) throw err
|
|
916
|
+
// What is on disk, not what is in the archive: a directory standing
|
|
917
|
+
// where the entry holds a file, or a file where it needs a directory.
|
|
918
|
+
// The clean removes everything the output globs cover, so this is a
|
|
919
|
+
// stray they do not — name it as such, not as a corrupt artifact.
|
|
920
|
+
const code = (err as NodeJS.ErrnoException).code
|
|
921
|
+
if (code === 'EISDIR' || code === 'ENOTDIR' || code === 'EEXIST' || code === 'ENOTEMPTY') {
|
|
922
|
+
throw new UserError(
|
|
923
|
+
`restore of ${hash} into ${projectDir} was blocked by what is on disk (${code}: ${(err as Error).message}). ` +
|
|
924
|
+
`Declared outputs are wiped before a restore, so this is a path the output globs do not cover — remove it and re-run.`,
|
|
925
|
+
)
|
|
926
|
+
}
|
|
1921
927
|
throw new CorruptArtifactError(hash, 'artifact is not a readable archive', err)
|
|
928
|
+
} finally {
|
|
929
|
+
endExtract()
|
|
1922
930
|
}
|
|
1923
931
|
}
|
|
1924
932
|
|
|
@@ -1979,8 +987,8 @@ export class Cache implements CacheLayer {
|
|
|
1979
987
|
* index. Used by the LayeredCache when local writes are disabled but
|
|
1980
988
|
* a remote upload still needs the artifact bytes.
|
|
1981
989
|
*/
|
|
1982
|
-
|
|
1983
|
-
return
|
|
990
|
+
packArtifactBytes(args: SaveArgs): Promise<Uint8Array> {
|
|
991
|
+
return this.packArtifact(args)
|
|
1984
992
|
}
|
|
1985
993
|
|
|
1986
994
|
async ingest(hash: string, compressed: Uint8Array, meta: IngestMeta): Promise<void> {
|
|
@@ -2034,10 +1042,11 @@ export class Cache implements CacheLayer {
|
|
|
2034
1042
|
const plan = await planArtifact({
|
|
2035
1043
|
stdout: args.entry.stdout ?? '',
|
|
2036
1044
|
outputs: this.outputsOf(args),
|
|
1045
|
+
exec: usageOfEntry(args.entry),
|
|
2037
1046
|
})
|
|
2038
1047
|
if (plan.size <= STREAM_DECODE_FROM)
|
|
2039
1048
|
return await Bun.zstdCompress(await packArtifactBytes(plan))
|
|
2040
|
-
return
|
|
1049
|
+
return bytesOf(packArtifactStream(plan).pipeThrough(zstdEncoder()))
|
|
2041
1050
|
}
|
|
2042
1051
|
|
|
2043
1052
|
/**
|
|
@@ -2056,6 +1065,7 @@ export class Cache implements CacheLayer {
|
|
|
2056
1065
|
const plan = await planArtifact({
|
|
2057
1066
|
stdout: args.entry.stdout ?? '',
|
|
2058
1067
|
outputs: this.outputsOf(args),
|
|
1068
|
+
exec: usageOfEntry(args.entry),
|
|
2059
1069
|
})
|
|
2060
1070
|
if (plan.size <= STREAM_DECODE_FROM)
|
|
2061
1071
|
return await Bun.zstdCompress(await packArtifactBytes(plan))
|
|
@@ -2191,8 +1201,7 @@ export class Cache implements CacheLayer {
|
|
|
2191
1201
|
// (not the per-run path) so a warm all-cache-hit run — which never
|
|
2192
1202
|
// saves — writes none of them.
|
|
2193
1203
|
const insertEntry = this.insertEntry
|
|
2194
|
-
const
|
|
2195
|
-
const deleteOutputFiles = this.deleteOutputFiles
|
|
1204
|
+
const outputs = this.outputs
|
|
2196
1205
|
const insertEntryInput = this.insertEntryInput
|
|
2197
1206
|
const inputComponents = meta.inputComponents
|
|
2198
1207
|
const tx = this.db.transaction(() => {
|
|
@@ -2212,13 +1221,12 @@ export class Cache implements CacheLayer {
|
|
|
2212
1221
|
stdoutText,
|
|
2213
1222
|
now,
|
|
2214
1223
|
now,
|
|
1224
|
+
// From the artifact, on both paths: a save indexes what it just
|
|
1225
|
+
// packed, an ingest what the producing machine packed.
|
|
1226
|
+
scanned.exec?.cpuMs ?? null,
|
|
1227
|
+
scanned.exec?.peakRssBytes ?? null,
|
|
2215
1228
|
)
|
|
2216
|
-
|
|
2217
|
-
// the same hash should refresh, not append).
|
|
2218
|
-
deleteOutputFiles.run(hash)
|
|
2219
|
-
for (const [rel, size, mode, mtime] of outputFileRows) {
|
|
2220
|
-
insertOutputFile.run(hash, rel, size, mode, mtime)
|
|
2221
|
-
}
|
|
1229
|
+
outputs.replaceFileRows(hash, outputFileRows)
|
|
2222
1230
|
// INSERT OR IGNORE: identical inputs derive this same hash, so a
|
|
2223
1231
|
// re-save's rows are identical — keep the first set, skip the rest.
|
|
2224
1232
|
if (inputComponents !== undefined) {
|
|
@@ -2260,53 +1268,16 @@ export class Cache implements CacheLayer {
|
|
|
2260
1268
|
return this.db
|
|
2261
1269
|
}
|
|
2262
1270
|
|
|
2263
|
-
|
|
2264
|
-
* Content-addressed storage view over the same artifacts directory.
|
|
2265
|
-
* Returns an `FsCASBackend` pointing at `cacheDir`, so external
|
|
2266
|
-
* subsystems (R2 mirror, REAPI CAS bridge, analytics scanners) can
|
|
2267
|
-
* read raw bytes with a `Digest`-keyed API without coupling to
|
|
2268
|
-
* Cache's internal save path. Read/write semantics match what
|
|
2269
|
-
* Cache.save writes (`<cacheDir>/<hash>.tar.zst`).
|
|
2270
|
-
*/
|
|
2271
|
-
contentBackend(): FsCASBackend {
|
|
2272
|
-
return new FsCASBackend(this.cacheDir)
|
|
2273
|
-
}
|
|
2274
|
-
|
|
1271
|
+
// --- run history: delegated to `RunHistory` (see run-history.ts) ---
|
|
2275
1272
|
recordRun(run: RunRecord): void {
|
|
2276
|
-
this.
|
|
1273
|
+
this.history.recordRun(run)
|
|
2277
1274
|
}
|
|
2278
|
-
|
|
2279
1275
|
recordRuns(runs: readonly RunRecord[]): void {
|
|
2280
|
-
|
|
2281
|
-
if (runs.length === 1) {
|
|
2282
|
-
this.insertRun.run(...bindRun(runs[0]!))
|
|
2283
|
-
return
|
|
2284
|
-
}
|
|
2285
|
-
// `bun:sqlite`'s `transaction()` returns a callable that wraps the
|
|
2286
|
-
// body in BEGIN/COMMIT, fsyncing once at the end. For a 200-task
|
|
2287
|
-
// run that's one fsync instead of 200.
|
|
2288
|
-
const insert = this.insertRun
|
|
2289
|
-
const tx = this.db.transaction((batch: readonly RunRecord[]) => {
|
|
2290
|
-
for (const r of batch) insert.run(...bindRun(r))
|
|
2291
|
-
})
|
|
2292
|
-
tx(runs)
|
|
1276
|
+
this.history.recordRuns(runs)
|
|
2293
1277
|
}
|
|
2294
|
-
|
|
2295
1278
|
recordRunBundle(bundle: { runs: readonly RunRecord[]; invocation: InvocationRecord }): void {
|
|
2296
|
-
|
|
2297
|
-
// per-task `runs` rows and the one `invocations` header. The
|
|
2298
|
-
// input-fingerprint rows do NOT live here: they're written inside
|
|
2299
|
-
// the entry-save transaction (`save`/`ingest`) so a warm
|
|
2300
|
-
// all-cache-hit run — which writes no `runs`-vs-`entry_inputs`
|
|
2301
|
-
// mismatch — pays nothing for the moat it isn't refreshing.
|
|
2302
|
-
const insertRun = this.insertRun
|
|
2303
|
-
const insertInvocation = this.insertInvocation
|
|
2304
|
-
this.db.transaction(() => {
|
|
2305
|
-
for (const r of bundle.runs) insertRun.run(...bindRun(r))
|
|
2306
|
-
insertInvocation.run(...bindInvocation(bundle.invocation))
|
|
2307
|
-
})()
|
|
1279
|
+
this.history.recordRunBundle(bundle)
|
|
2308
1280
|
}
|
|
2309
|
-
|
|
2310
1281
|
stats(opts: CacheStatsOptions = {}): CacheStats {
|
|
2311
1282
|
this.flushAccessed()
|
|
2312
1283
|
const project = opts.project
|
|
@@ -2337,7 +1308,7 @@ export class Cache implements CacheLayer {
|
|
|
2337
1308
|
|
|
2338
1309
|
async prune(options: PruneOptions): Promise<PruneResult> {
|
|
2339
1310
|
this.flushAccessed()
|
|
2340
|
-
const { olderThanMs, maxBytes } = options
|
|
1311
|
+
const { olderThanMs, maxBytes, dryRun = false } = options
|
|
2341
1312
|
if (olderThanMs === undefined && maxBytes === undefined) {
|
|
2342
1313
|
throw new Error('prune: pass at least one of `olderThanMs` or `maxBytes`')
|
|
2343
1314
|
}
|
|
@@ -2384,6 +1355,15 @@ export class Cache implements CacheLayer {
|
|
|
2384
1355
|
// Promise.all over the unlinks. The IN-list is chunked at 900 like
|
|
2385
1356
|
// flushAccessed so a huge eviction stays under any build's
|
|
2386
1357
|
// bound-parameter ceiling.
|
|
1358
|
+
if (dryRun) {
|
|
1359
|
+
const orphans = await this.orphanStats()
|
|
1360
|
+
return {
|
|
1361
|
+
evicted: victims.size,
|
|
1362
|
+
bytesFreed,
|
|
1363
|
+
orphans: orphans.orphans,
|
|
1364
|
+
orphanBytes: orphans.orphanBytes,
|
|
1365
|
+
}
|
|
1366
|
+
}
|
|
2387
1367
|
if (victims.size > 0) {
|
|
2388
1368
|
const hashes = [...victims]
|
|
2389
1369
|
this.db.transaction(() => {
|
|
@@ -2397,7 +1377,83 @@ export class Cache implements CacheLayer {
|
|
|
2397
1377
|
await Promise.all(hashes.map((h) => rm(this.tarPath(h), { force: true })))
|
|
2398
1378
|
}
|
|
2399
1379
|
|
|
2400
|
-
|
|
1380
|
+
const orphans = await this.reapOrphans()
|
|
1381
|
+
return { evicted: victims.size, bytesFreed, ...orphans }
|
|
1382
|
+
}
|
|
1383
|
+
|
|
1384
|
+
/**
|
|
1385
|
+
* Unlink artifacts the index does not know: a `<hash>.tar.zst` with no
|
|
1386
|
+
* `entries` row (a `SCHEMA_VERSION` drop, a `cache.db` deleted by hand)
|
|
1387
|
+
* and a `<hash>.tar.zst.tmp-*` a crashed save never renamed. Nothing
|
|
1388
|
+
* else reaps them — a lookup starts at the row, and a save of the same
|
|
1389
|
+
* key renames over the file, so a key that never recurs leaks its bytes
|
|
1390
|
+
* forever. Files younger than the grace window are left alone: a save
|
|
1391
|
+
* renames the artifact into place BEFORE its row commits, and its temp
|
|
1392
|
+
* exists while the bytes are still being written, so a fresh file
|
|
1393
|
+
* without a row is a save in flight, not an orphan.
|
|
1394
|
+
*/
|
|
1395
|
+
private async reapOrphans(): Promise<{ orphans: number; orphanBytes: number }> {
|
|
1396
|
+
let orphans = 0
|
|
1397
|
+
let orphanBytes = 0
|
|
1398
|
+
await Promise.all(
|
|
1399
|
+
(await this.scanOrphans()).map(async (o) => {
|
|
1400
|
+
try {
|
|
1401
|
+
// unlink, not `rm({ force })`: force swallows ENOENT, and a file a
|
|
1402
|
+
// concurrent prune took first must not be counted as ours.
|
|
1403
|
+
await unlink(o.file)
|
|
1404
|
+
orphans += 1
|
|
1405
|
+
orphanBytes += o.size
|
|
1406
|
+
} catch {
|
|
1407
|
+
// Gone under us (a concurrent prune, a save's own cleanup): not ours.
|
|
1408
|
+
}
|
|
1409
|
+
}),
|
|
1410
|
+
)
|
|
1411
|
+
return { orphans, orphanBytes }
|
|
1412
|
+
}
|
|
1413
|
+
|
|
1414
|
+
/** What `prune()` would reap right now, for `vx info` to say before anyone prunes. */
|
|
1415
|
+
async orphanStats(): Promise<{ orphans: number; orphanBytes: number }> {
|
|
1416
|
+
const found = await this.scanOrphans()
|
|
1417
|
+
let orphanBytes = 0
|
|
1418
|
+
for (const o of found) orphanBytes += o.size
|
|
1419
|
+
return { orphans: found.length, orphanBytes }
|
|
1420
|
+
}
|
|
1421
|
+
|
|
1422
|
+
/** Row-less artifacts and temps past the in-flight grace window: one readdir, one stat per candidate. */
|
|
1423
|
+
private async scanOrphans(): Promise<Array<{ file: string; size: number }>> {
|
|
1424
|
+
let names: string[]
|
|
1425
|
+
try {
|
|
1426
|
+
names = await readdir(this.cacheDir)
|
|
1427
|
+
} catch {
|
|
1428
|
+
return []
|
|
1429
|
+
}
|
|
1430
|
+
const indexed = new Set(
|
|
1431
|
+
(this.db.prepare('SELECT hash FROM entries').all() as Array<{ hash: string }>).map(
|
|
1432
|
+
(r) => r.hash,
|
|
1433
|
+
),
|
|
1434
|
+
)
|
|
1435
|
+
const cutoff = Date.now() - ORPHAN_GRACE_MS
|
|
1436
|
+
const candidates: string[] = []
|
|
1437
|
+
for (const name of names) {
|
|
1438
|
+
if (name.indexOf('.tar.zst.tmp-') > 0) {
|
|
1439
|
+
candidates.push(name)
|
|
1440
|
+
} else if (name.endsWith('.tar.zst') && !indexed.has(name.slice(0, -'.tar.zst'.length))) {
|
|
1441
|
+
candidates.push(name)
|
|
1442
|
+
}
|
|
1443
|
+
}
|
|
1444
|
+
const found: Array<{ file: string; size: number }> = []
|
|
1445
|
+
await Promise.all(
|
|
1446
|
+
candidates.map(async (name) => {
|
|
1447
|
+
const file = path.join(this.cacheDir, name)
|
|
1448
|
+
try {
|
|
1449
|
+
const st = await stat(file)
|
|
1450
|
+
if (st.isFile() && st.mtimeMs <= cutoff) found.push({ file, size: st.size })
|
|
1451
|
+
} catch {
|
|
1452
|
+
// Gone between readdir and stat: not an orphan any more.
|
|
1453
|
+
}
|
|
1454
|
+
}),
|
|
1455
|
+
)
|
|
1456
|
+
return found
|
|
2401
1457
|
}
|
|
2402
1458
|
|
|
2403
1459
|
close(): void {
|
|
@@ -2412,12 +1468,8 @@ export class Cache implements CacheLayer {
|
|
|
2412
1468
|
// table would grow unbounded on a long-lived checkout.
|
|
2413
1469
|
try {
|
|
2414
1470
|
const cutoff = Date.now() - 30 * 24 * 60 * 60 * 1000
|
|
2415
|
-
this.
|
|
2416
|
-
this.
|
|
2417
|
-
// A config that has not been loaded in 30 days was edited (its key
|
|
2418
|
-
// moved) or its project left; either way the row is dead weight.
|
|
2419
|
-
this.db.prepare('DELETE FROM config_evals WHERE created_at < ?').run(cutoff)
|
|
2420
|
-
this.db.prepare('DELETE FROM config_closures WHERE created_at < ?').run(cutoff)
|
|
1471
|
+
this.history.pruneOlderThan(cutoff)
|
|
1472
|
+
this.configEvals.pruneOlderThan(cutoff)
|
|
2421
1473
|
} catch {
|
|
2422
1474
|
// Retention is best-effort; never block closing the handle.
|
|
2423
1475
|
}
|
|
@@ -2441,69 +1493,6 @@ export class Cache implements CacheLayer {
|
|
|
2441
1493
|
}
|
|
2442
1494
|
}
|
|
2443
1495
|
|
|
2444
|
-
/**
|
|
2445
|
-
* Bind a RunRecord to the positional parameters expected by the
|
|
2446
|
-
* `insertRun` prepared statement (17 columns). Shared between the
|
|
2447
|
-
* single and batched record paths.
|
|
2448
|
-
*/
|
|
2449
|
-
function bindRun(run: RunRecord): SQLQueryBindings[] {
|
|
2450
|
-
return [
|
|
2451
|
-
// The ONE place the no-key sentinel is applied, so the column's
|
|
2452
|
-
// NOT NULL invariant can't be violated from a call site.
|
|
2453
|
-
run.hash ?? '',
|
|
2454
|
-
run.project,
|
|
2455
|
-
run.task,
|
|
2456
|
-
run.status,
|
|
2457
|
-
run.exitCode,
|
|
2458
|
-
run.durationMs,
|
|
2459
|
-
run.forwardArgs ? JSON.stringify(run.forwardArgs) : null,
|
|
2460
|
-
run.startedAt,
|
|
2461
|
-
run.endedAt,
|
|
2462
|
-
run.runId ?? null,
|
|
2463
|
-
run.cpuMs ?? null,
|
|
2464
|
-
run.peakRssBytes ?? null,
|
|
2465
|
-
run.wallclockStartNs !== undefined ? run.wallclockStartNs : null,
|
|
2466
|
-
run.wallclockEndNs !== undefined ? run.wallclockEndNs : null,
|
|
2467
|
-
run.cacheHit === undefined ? null : run.cacheHit ? 1 : 0,
|
|
2468
|
-
run.attempts ?? null,
|
|
2469
|
-
]
|
|
2470
|
-
}
|
|
2471
|
-
|
|
2472
|
-
/**
|
|
2473
|
-
* Bind an InvocationRecord to the positional parameters of the
|
|
2474
|
-
* `insertInvocation` prepared statement (25 columns). Booleans map to
|
|
2475
|
-
* 0/1; null-or-bool columns (`dirty`) keep null distinct from 0.
|
|
2476
|
-
*/
|
|
2477
|
-
function bindInvocation(inv: InvocationRecord): SQLQueryBindings[] {
|
|
2478
|
-
return [
|
|
2479
|
-
inv.runId,
|
|
2480
|
-
inv.command,
|
|
2481
|
-
inv.requestedTasks,
|
|
2482
|
-
inv.cachePolicy,
|
|
2483
|
-
inv.concurrency,
|
|
2484
|
-
inv.flow,
|
|
2485
|
-
inv.startedAt,
|
|
2486
|
-
inv.endedAt,
|
|
2487
|
-
inv.totalDurationMs,
|
|
2488
|
-
inv.taskCount,
|
|
2489
|
-
inv.failedCount,
|
|
2490
|
-
inv.hitCount,
|
|
2491
|
-
inv.hitLocalCount,
|
|
2492
|
-
inv.hitRemoteCount,
|
|
2493
|
-
inv.exitOk ? 1 : 0,
|
|
2494
|
-
inv.commitSha,
|
|
2495
|
-
inv.branch,
|
|
2496
|
-
inv.dirty === null ? null : inv.dirty ? 1 : 0,
|
|
2497
|
-
inv.ci ? 1 : 0,
|
|
2498
|
-
inv.ciProvider,
|
|
2499
|
-
inv.host,
|
|
2500
|
-
inv.os,
|
|
2501
|
-
inv.arch,
|
|
2502
|
-
inv.vxVersion,
|
|
2503
|
-
inv.tags,
|
|
2504
|
-
]
|
|
2505
|
-
}
|
|
2506
|
-
|
|
2507
1496
|
function splitTaskId(id: string): [string, string] {
|
|
2508
1497
|
const i = id.indexOf('#')
|
|
2509
1498
|
if (i < 0) return [id, '']
|