@vzn/vx 0.0.16 → 0.0.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +135 -124
- package/index.ts +10 -0
- package/launcher.mjs +10 -11
- package/package.json +11 -7
- package/src/bin.ts +14 -4
- package/src/cache/archive.ts +659 -0
- package/src/cache/cache.ts +769 -1101
- package/src/cache/chained-cache.ts +194 -0
- package/src/cache/config-evals.ts +89 -0
- package/src/cache/file-hashes.ts +262 -0
- package/src/cache/git-inputs.ts +637 -0
- package/src/cache/index.ts +24 -5
- package/src/cache/inputs.ts +373 -456
- package/src/cache/layer.ts +604 -0
- package/src/cache/layered-cache.ts +96 -43
- package/src/cache/output-index.ts +224 -0
- package/src/cache/policy.ts +84 -0
- package/src/cache/run-history.ts +147 -0
- package/src/cache/tar-stream.ts +358 -0
- package/src/cache/zstd.ts +160 -0
- package/src/cli/cache.ts +75 -21
- package/src/cli/completions.ts +104 -0
- package/src/cli/core-alias.ts +25 -0
- package/src/cli/help.ts +195 -128
- package/src/cli/index.ts +116 -46
- package/src/cli/info.ts +122 -63
- package/src/cli/init.ts +75 -0
- package/src/cli/last.ts +172 -0
- package/src/cli/lock.ts +2 -1
- package/src/cli/plan-format.ts +72 -6
- package/src/cli/plugin-commands.ts +103 -0
- package/src/cli/run.ts +245 -271
- package/src/cli/select.ts +332 -0
- package/src/cli/show.ts +144 -69
- package/src/cli/upgrade.ts +9 -7
- package/src/cli/watch.ts +858 -110
- package/src/cli/why.ts +223 -0
- package/src/cli/workspace-config.ts +126 -0
- package/src/config.ts +187 -150
- package/src/exec/env.ts +18 -1
- package/src/exec/executor.ts +224 -0
- package/src/exec/index.ts +24 -2
- package/src/exec/local-executor.ts +41 -0
- package/src/exec/runner.ts +166 -40
- package/src/exec/sandbox-binds.ts +182 -0
- package/src/exec/sandbox-paths.ts +50 -0
- package/src/exec/sandbox-runtime.ts +642 -260
- package/src/exec/sandbox-violations.ts +280 -0
- package/src/graph/index.ts +4 -10
- package/src/graph/scheduler.ts +232 -203
- package/src/graph/task-graph.ts +222 -8
- package/src/index.ts +146 -153
- package/src/orchestrator/admission.ts +137 -0
- package/src/orchestrator/deferred-outputs.ts +147 -0
- package/src/orchestrator/doctor.ts +255 -0
- package/src/orchestrator/download-policy.ts +171 -0
- package/src/orchestrator/events.ts +72 -8
- package/src/orchestrator/execute-task.ts +374 -549
- package/src/orchestrator/failure-mode.ts +202 -0
- package/src/orchestrator/framed-output.ts +126 -69
- package/src/orchestrator/history.ts +126 -59
- package/src/orchestrator/hit-restore.ts +221 -0
- package/src/orchestrator/index.ts +77 -93
- package/src/orchestrator/local-shortcircuit.ts +22 -0
- package/src/orchestrator/lockfile-claim.ts +281 -0
- package/src/orchestrator/logger.ts +208 -35
- package/src/orchestrator/metrics.ts +102 -1388
- package/src/orchestrator/miss-save.ts +169 -0
- package/src/orchestrator/options.ts +55 -34
- package/src/orchestrator/persistent.ts +79 -0
- package/src/orchestrator/placement.ts +182 -0
- package/src/orchestrator/plan.ts +144 -1
- package/src/orchestrator/plugin-host.ts +357 -120
- package/src/orchestrator/plugin.ts +298 -58
- package/src/orchestrator/prepare.ts +202 -106
- package/src/orchestrator/projects.ts +259 -0
- package/src/orchestrator/remote-prefetch.ts +12 -8
- package/src/orchestrator/run-artifacts.ts +63 -16
- package/src/orchestrator/run-context.ts +167 -29
- package/src/orchestrator/run-records.ts +149 -0
- package/src/orchestrator/run-report.ts +74 -47
- package/src/orchestrator/run.ts +534 -417
- package/src/orchestrator/sandbox-request.ts +243 -0
- package/src/orchestrator/save-lane.ts +47 -0
- package/src/orchestrator/signals.ts +111 -0
- package/src/orchestrator/stable-keys.ts +129 -31
- package/src/orchestrator/status-line.ts +49 -9
- package/src/orchestrator/summary.ts +90 -26
- package/src/orchestrator/tally.ts +78 -35
- package/src/orchestrator/task-hash.ts +111 -15
- package/src/orchestrator/task-log-buffer.ts +288 -0
- package/src/orchestrator/telemetry-host.ts +34 -6
- package/src/orchestrator/telemetry.ts +182 -36
- package/src/orchestrator/upstream.ts +72 -31
- package/src/util/cgroup.ts +160 -0
- package/src/util/edit-distance.ts +70 -0
- package/src/util/errors.ts +15 -1
- package/src/util/index.ts +15 -2
- package/src/util/num.ts +37 -0
- package/src/util/paths.ts +64 -0
- package/src/util/settle.ts +67 -0
- package/src/util/size.ts +5 -1
- package/src/util/tail.ts +66 -0
- package/src/util/timing.ts +63 -0
- package/src/util/verbs.ts +39 -0
- package/src/workspace/affected.ts +250 -23
- package/src/workspace/config-cache.ts +313 -0
- package/src/workspace/config-eval.ts +217 -0
- package/src/workspace/config-imports.ts +162 -0
- package/src/workspace/config-schema.ts +837 -0
- package/src/workspace/filter.ts +65 -10
- package/src/workspace/fingerprint.ts +53 -4
- package/src/workspace/index.ts +42 -3
- package/src/workspace/lockfile.ts +25 -9
- package/src/workspace/migrate-scripts.ts +134 -0
- package/src/workspace/migration.ts +314 -0
- package/src/workspace/nested-dirs.ts +21 -7
- package/src/workspace/package-graph.ts +114 -41
- package/src/workspace/project-loader.ts +224 -450
- package/src/workspace/workspace.ts +298 -72
- package/src/cache/cas-backend.ts +0 -97
- package/src/cache/digest.ts +0 -47
- package/src/cache/tar.ts +0 -343
- package/src/cli/backend.ts +0 -23
- package/src/cli/mcp-rpc.ts +0 -292
- package/src/cli/mcp.ts +0 -92
- package/src/cli/migrate-nx.ts +0 -450
- package/src/cli/migrate-turbo.ts +0 -364
- package/src/cli/migrate.ts +0 -219
- package/src/orchestrator/devframe-surface.ts +0 -74
- package/src/orchestrator/predict.ts +0 -106
- package/src/orchestrator/protocol.ts +0 -156
- package/src/orchestrator/resources.ts +0 -59
- package/src/orchestrator/run-state.ts +0 -145
- package/src/orchestrator/verify.ts +0 -164
- package/src/orchestrator/wire-render.ts +0 -92
- package/src/orchestrator/wire.ts +0 -212
|
@@ -0,0 +1,604 @@
|
|
|
1
|
+
// The cache layer CONTRACT and its records — what every cache implementation
|
|
2
|
+
// honours (`Cache`, `LayeredCache`, `ChainedCache`, a plugin's remote layer) and
|
|
3
|
+
// the shapes that cross it: the key input, entries, run-history rows, output
|
|
4
|
+
// fingerprints, stats, prune options. No implementation lives here; the one
|
|
5
|
+
// reference to `Cache` below is a type (the local handle a layer may wrap).
|
|
6
|
+
|
|
7
|
+
import type { Cache } from './cache.js'
|
|
8
|
+
|
|
9
|
+
export interface CacheKeyInput {
|
|
10
|
+
taskId: string
|
|
11
|
+
/**
|
|
12
|
+
* Hash of the resolved task config (post-evaluation). Folds in everything
|
|
13
|
+
* the user wrote — command, env declarations (passThrough names + define
|
|
14
|
+
* key/value pairs), dependsOn, cache.inputs declarations, outputs — including
|
|
15
|
+
* values that arrived via `import` at config-load time.
|
|
16
|
+
*/
|
|
17
|
+
taskConfigHash: string
|
|
18
|
+
/**
|
|
19
|
+
* Runtime values of declared cache-input env names (from parent at hash
|
|
20
|
+
* time). Independent of `exec.env`; lives here for cache identity.
|
|
21
|
+
*/
|
|
22
|
+
envValues: Array<[name: string, value: string]>
|
|
23
|
+
/**
|
|
24
|
+
* Resolved `cache.inputs.runtime` commands as [command, output] pairs
|
|
25
|
+
* (output = trimmed stdout+stderr, resolved live at hash time). Folded
|
|
26
|
+
* into the key in a namespace distinct from workspaceRuntimeValues.
|
|
27
|
+
*/
|
|
28
|
+
runtimeValues?: Array<[command: string, output: string]>
|
|
29
|
+
/** Resolved `cache.inputs.workspaceRuntime` pairs (root-cwd commands). */
|
|
30
|
+
workspaceRuntimeValues?: Array<[command: string, output: string]>
|
|
31
|
+
/** Absolute paths to input files. */
|
|
32
|
+
inputFiles: string[]
|
|
33
|
+
workspaceRoot: string
|
|
34
|
+
/** Cache keys of upstream tasks this one depends on, sorted. */
|
|
35
|
+
upstreamHashes: string[]
|
|
36
|
+
/**
|
|
37
|
+
* Upstream hash → upstream task id, for `captureInto` row NAMING only.
|
|
38
|
+
* Never folded into the digest — the key already folds the sorted
|
|
39
|
+
* hashes. Lets the persisted `entry_inputs` row name which upstream
|
|
40
|
+
* task a hash came from (the diff reads better than a bare hash). When
|
|
41
|
+
* absent, the captured `upstream` row falls back to `name = hash`.
|
|
42
|
+
*/
|
|
43
|
+
upstreamIds?: ReadonlyMap<string, string>
|
|
44
|
+
/**
|
|
45
|
+
* The task's DEPENDENCY closure with GROUP tasks expanded into the real
|
|
46
|
+
* tasks they stand for — what an input-shipping executor must place in the
|
|
47
|
+
* input root. A different question from what the key folds, and derived
|
|
48
|
+
* differently in both directions:
|
|
49
|
+
*
|
|
50
|
+
* - It expands groups, which contribute a synthetic roll-up hash and no
|
|
51
|
+
* outputs of their own, so a dependent of one would otherwise describe
|
|
52
|
+
* an empty closure.
|
|
53
|
+
* - It ignores `cache.inputs.tasks`. That filter says which upstream KEYS
|
|
54
|
+
* this task's key folds; what the task may READ is `dependsOn`, and
|
|
55
|
+
* locally every dependency's outputs are on disk before the command runs
|
|
56
|
+
* however the filter is written.
|
|
57
|
+
*
|
|
58
|
+
* NEVER folded into the digest — the key cascades through a group's own
|
|
59
|
+
* roll-up hash already, and folding either difference in would move every
|
|
60
|
+
* existing dependent's key without telling it anything new.
|
|
61
|
+
*/
|
|
62
|
+
upstreamGraft?: ReadonlyArray<{
|
|
63
|
+
readonly taskId: string
|
|
64
|
+
readonly hash: string
|
|
65
|
+
readonly projectDir: string
|
|
66
|
+
}>
|
|
67
|
+
/**
|
|
68
|
+
* Workspace-level fingerprint — typically a hash of `pnpm-lock.yaml` +
|
|
69
|
+
* `pnpm-workspace.yaml`. Folds resolved dep versions and workspace shape
|
|
70
|
+
* into every task's key, so a lockfile bump invalidates everything.
|
|
71
|
+
*/
|
|
72
|
+
workspaceFingerprint: string
|
|
73
|
+
/**
|
|
74
|
+
* CLI args forwarded to the task (after `--`). Folded into the key so that
|
|
75
|
+
* the same command with different forwarded args is treated as a distinct
|
|
76
|
+
* run, never a spurious cache hit.
|
|
77
|
+
*/
|
|
78
|
+
forwardArgs?: readonly string[]
|
|
79
|
+
/**
|
|
80
|
+
* Hash of the project's `package.json` bytes. Folded into the key
|
|
81
|
+
* implicitly (Turbo / Nx parity) so dep changes invalidate every
|
|
82
|
+
* task in that project, even when `cache.inputs.files` doesn't
|
|
83
|
+
* cover package.json. Empty string when the project has no
|
|
84
|
+
* package.json (impossible in practice — workspace discovery
|
|
85
|
+
* requires one — but we don't fail-loud here).
|
|
86
|
+
*/
|
|
87
|
+
projectPackageJsonHash: string
|
|
88
|
+
/**
|
|
89
|
+
* Precomputed content hashes (git blob OIDs) keyed by absolute
|
|
90
|
+
* path — typically the trusted-index OID map harvested by the
|
|
91
|
+
* run's bulk `git ls-files -s`. Paths present here skip `hashFile`
|
|
92
|
+
* entirely (no stat, no SQLite, no read); missing paths fall back
|
|
93
|
+
* to `hashFile`, which computes the byte-identical blob OID from
|
|
94
|
+
* disk. Pure fast path: the derived key never depends on whether a
|
|
95
|
+
* hash arrived via the map or the fallback.
|
|
96
|
+
*/
|
|
97
|
+
fileHashes?: ReadonlyMap<string, string>
|
|
98
|
+
/**
|
|
99
|
+
* Material a plugin's `key` stage contributed, as sorted `[name, value]`
|
|
100
|
+
* pairs. Folded only when non-empty, so a workspace without a key plugin
|
|
101
|
+
* derives exactly the key it always did.
|
|
102
|
+
*/
|
|
103
|
+
pluginParts?: ReadonlyArray<readonly [name: string, value: string]>
|
|
104
|
+
/**
|
|
105
|
+
* When set, `key()` pushes each component (kind, name, hash) it folds
|
|
106
|
+
* — at the same fold sites, in fold order. Pure SIDE-CHANNEL: it does
|
|
107
|
+
* not change the returned digest in any way. On a cache MISS the
|
|
108
|
+
* orchestrator allocates an array, passes it here, and persists the
|
|
109
|
+
* rows to `entry_inputs` (inside the entry-save transaction) so a
|
|
110
|
+
* later run can diff its inputs against this one (the Tier-3 "why did
|
|
111
|
+
* this re-run?" moat). Capturing at the fold sites keeps the
|
|
112
|
+
* recorded set in lockstep with the key by construction: a future
|
|
113
|
+
* component that forgets to capture is a one-line miss, not silent
|
|
114
|
+
* drift. The per-file OIDs are already awaited here, so file rows cost
|
|
115
|
+
* zero extra I/O — just array pushes.
|
|
116
|
+
*/
|
|
117
|
+
captureInto?: Array<{ kind: string; name: string; hash: string }>
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
export interface CacheEntry {
|
|
121
|
+
hash: string
|
|
122
|
+
taskId: string
|
|
123
|
+
command: string
|
|
124
|
+
exitCode: number
|
|
125
|
+
durationMs: number
|
|
126
|
+
/**
|
|
127
|
+
* What the PRODUCING execution used — CPU time and peak RSS — carried in
|
|
128
|
+
* the artifact's sidecar, so an entry ingested from a remote knows them
|
|
129
|
+
* too. Absent when the runner reported nothing. A hit surfaces them as
|
|
130
|
+
* `storedCpuMs` / `storedPeakRssBytes` on its outcome: what the hit
|
|
131
|
+
* skipped, never what it spent.
|
|
132
|
+
*/
|
|
133
|
+
cpuMs?: number
|
|
134
|
+
peakRssBytes?: number
|
|
135
|
+
outputFiles: string[]
|
|
136
|
+
/**
|
|
137
|
+
* The `output_files` rows behind `outputFiles` (size / mode / mtime), when
|
|
138
|
+
* the layer that produced this entry had them in hand. `restoreHit` reads
|
|
139
|
+
* them for its tree-is-current check instead of re-querying — one SQL
|
|
140
|
+
* round trip per cache hit, saved.
|
|
141
|
+
*/
|
|
142
|
+
outputRows?: OutputFileRow[]
|
|
143
|
+
/**
|
|
144
|
+
* The `output_dirs` rows behind the directory short-circuit, when the
|
|
145
|
+
* batched `getMany` loaded them — one query for the run instead of one
|
|
146
|
+
* per hit (0.12 ms each across a warm 1000-project run before this).
|
|
147
|
+
*/
|
|
148
|
+
outputDirRows?: OutputDirRow[]
|
|
149
|
+
/** Captured stdout, always present (may be empty). stderr is not cached. */
|
|
150
|
+
stdout: string
|
|
151
|
+
storedAt: string
|
|
152
|
+
/**
|
|
153
|
+
* Where this hit was resolved from. `'local'` for a SQLite-backed
|
|
154
|
+
* Cache; `'remote'` when LayeredCache pulled the artifact from the
|
|
155
|
+
* remote layer this lookup (even though it's been materialized into
|
|
156
|
+
* local for next time). Lets the orchestrator surface
|
|
157
|
+
* `cache-hit-remote` so users see when remote caching actually saved
|
|
158
|
+
* them work vs. a stale-local replay.
|
|
159
|
+
*/
|
|
160
|
+
source?: 'local' | 'remote'
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
export interface RunRecord {
|
|
164
|
+
/**
|
|
165
|
+
* The task's cache key. ABSENT when the outcome never derived one — a
|
|
166
|
+
* `skipped` task (its upstream failed, so it never probed) or a
|
|
167
|
+
* `persistent` one (a dev server is never cached). Such a row is still a
|
|
168
|
+
* task of the run and must be recorded, so the column keeps a `''`
|
|
169
|
+
* sentinel rather than going nullable: `''` is impossible for a real key
|
|
170
|
+
* (`Cache.key` returns 16 hex chars), the reads that must not treat it as
|
|
171
|
+
* a key already guard it, and it matches what the cloud `task_runs.hash`
|
|
172
|
+
* column has stored for the same concept since its first migration.
|
|
173
|
+
*/
|
|
174
|
+
hash?: string
|
|
175
|
+
project: string
|
|
176
|
+
task: string
|
|
177
|
+
status: 'success' | 'failed' | 'cache-hit' | 'cache-hit-remote' | 'skipped'
|
|
178
|
+
exitCode: number
|
|
179
|
+
durationMs: number
|
|
180
|
+
forwardArgs?: readonly string[]
|
|
181
|
+
startedAt: number // ms-epoch wall clock
|
|
182
|
+
endedAt: number // ms-epoch wall clock
|
|
183
|
+
/**
|
|
184
|
+
* Optional analytics columns. Populated by the orchestrator/runner;
|
|
185
|
+
* stored as NULL on rows from older runs. Surfaced via `vx stats`
|
|
186
|
+
* and consumable from CI by reading cache.db directly.
|
|
187
|
+
*/
|
|
188
|
+
runId?: string // ULID shared across every task in one `vx run` invocation
|
|
189
|
+
cpuMs?: number // sum of user + system CPU time for the child process
|
|
190
|
+
peakRssBytes?: number // peak resident set size of the child process
|
|
191
|
+
wallclockStartNs?: bigint // hrtime span relative to run t=0
|
|
192
|
+
wallclockEndNs?: bigint
|
|
193
|
+
cacheHit?: boolean // convenience for flamegraph color; derivable from status
|
|
194
|
+
attempts?: number // >1 when the task retried; the direct within-run flaky signal
|
|
195
|
+
/**
|
|
196
|
+
* Whether the task declared a `cache` block. An uncached task executes
|
|
197
|
+
* on every run and its key exists only for dependents to fold, so a
|
|
198
|
+
* reader (`vx why`, `vx last`) must not present its re-run as a cache
|
|
199
|
+
* decision. Absent on rows written before the column existed.
|
|
200
|
+
*/
|
|
201
|
+
cached?: boolean
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/**
|
|
205
|
+
* One header row per `vx run` invocation (the `invocations` table). All
|
|
206
|
+
* fields mirror the columns; nullable VCS/host columns are `null` when
|
|
207
|
+
* the probe failed (not a git repo, hostname unavailable). Recorded
|
|
208
|
+
* once per run inside the same transaction as the per-task `runs` rows.
|
|
209
|
+
*/
|
|
210
|
+
export interface InvocationRecord {
|
|
211
|
+
runId: string
|
|
212
|
+
command: string
|
|
213
|
+
/** JSON-serialized `string[]` of the requested task names. */
|
|
214
|
+
requestedTasks: string
|
|
215
|
+
/** Compact policy flags, e.g. `'lR,lW,rR,rW'` (an axis omitted = off). */
|
|
216
|
+
cachePolicy: string
|
|
217
|
+
concurrency: number
|
|
218
|
+
flow: 'focused' | 'broad' | null
|
|
219
|
+
startedAt: number
|
|
220
|
+
endedAt: number
|
|
221
|
+
totalDurationMs: number
|
|
222
|
+
taskCount: number
|
|
223
|
+
failedCount: number
|
|
224
|
+
hitCount: number
|
|
225
|
+
hitLocalCount: number
|
|
226
|
+
hitRemoteCount: number
|
|
227
|
+
exitOk: boolean
|
|
228
|
+
commitSha: string | null
|
|
229
|
+
branch: string | null
|
|
230
|
+
dirty: boolean | null
|
|
231
|
+
ci: boolean
|
|
232
|
+
ciProvider: string | null
|
|
233
|
+
host: string | null
|
|
234
|
+
os: string | null
|
|
235
|
+
arch: string | null
|
|
236
|
+
vxVersion: string
|
|
237
|
+
/** JSON object `{k:v}` of `--tag` pairs. */
|
|
238
|
+
tags: string
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* One cache-key component row for the `entry_inputs` table — keyed by
|
|
243
|
+
* the cache-entry HASH it belongs to, not a run. Written inside the
|
|
244
|
+
* entry-save transaction (`writeArtifactAndIndex`) on a miss/save; a
|
|
245
|
+
* cache HIT does not save, so it persists nothing (warm runs are free).
|
|
246
|
+
* The diff (Phase B/B1) reads these by the run's task hash
|
|
247
|
+
* (`runs.hash → entry_inputs[entry_hash]`).
|
|
248
|
+
*/
|
|
249
|
+
export interface TaskInputRow {
|
|
250
|
+
entryHash: string
|
|
251
|
+
kind: string
|
|
252
|
+
name: string
|
|
253
|
+
hash: string
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
export interface CacheStats {
|
|
257
|
+
entryCount: number
|
|
258
|
+
totalBytes: number
|
|
259
|
+
runCountLast24h: number
|
|
260
|
+
hitCountLast24h: number
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
export interface CacheStatsOptions {
|
|
264
|
+
/** Narrow every aggregate to one project. Absent = the whole workspace. */
|
|
265
|
+
project?: string
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
export interface PruneOptions {
|
|
269
|
+
/** Drop entries last accessed before this ms-epoch threshold. */
|
|
270
|
+
olderThanMs?: number
|
|
271
|
+
/**
|
|
272
|
+
* After applying olderThanMs, if the cache still exceeds this size in
|
|
273
|
+
* bytes, evict LRU (smallest `accessed_at` first) until under it.
|
|
274
|
+
*/
|
|
275
|
+
maxBytes?: number
|
|
276
|
+
/**
|
|
277
|
+
* Pick the victims and count the orphans, delete nothing: the result
|
|
278
|
+
* says what a real prune with the same policy would reap right now.
|
|
279
|
+
*/
|
|
280
|
+
dryRun?: boolean
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
export interface PruneResult {
|
|
284
|
+
/** Index entries evicted by the age / size policy (each with its artifact). */
|
|
285
|
+
evicted: number
|
|
286
|
+
/** Bytes the evicted entries occupied, per the index. */
|
|
287
|
+
bytesFreed: number
|
|
288
|
+
/**
|
|
289
|
+
* Artifacts and temp files with no index row, unlinked by the orphan
|
|
290
|
+
* sweep that runs after eviction (a `SCHEMA_VERSION` drop, a deleted
|
|
291
|
+
* `cache.db`, a save that crashed before its rename). Files younger
|
|
292
|
+
* than one hour are never counted: they may be a save in flight.
|
|
293
|
+
*/
|
|
294
|
+
orphans: number
|
|
295
|
+
/** Bytes the reaped orphans occupied on disk. */
|
|
296
|
+
orphanBytes: number
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/**
|
|
300
|
+
* Per-output-file fingerprint, scoped by the cache entry that
|
|
301
|
+
* produced it. Batch-loaded once at the top of a run via
|
|
302
|
+
* `loadOutputFilesBatch(hashes)` so the orchestrator's "is this
|
|
303
|
+
* tree already current?" probe becomes an in-memory Map lookup
|
|
304
|
+
* plus N parallel stat calls.
|
|
305
|
+
*
|
|
306
|
+
* `path` is project-relative (e.g. `dist/index.js`), matching how
|
|
307
|
+
* outputs are addressed under `<projectDir>/` — except workspace
|
|
308
|
+
* outputs, which carry the full `workspace-outputs/<rel-to-root>`
|
|
309
|
+
* tar entry name (see `WORKSPACE_OUTPUT_PREFIX`); callers split on
|
|
310
|
+
* the prefix and anchor those at the workspace root.
|
|
311
|
+
*/
|
|
312
|
+
export interface OutputFileRow {
|
|
313
|
+
path: string
|
|
314
|
+
size: number
|
|
315
|
+
mode: number
|
|
316
|
+
mtimeMs: number
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
/** One directory under a whole-subtree output glob, as it stood after the last save or restore on THIS machine. */
|
|
320
|
+
export interface OutputDirRow {
|
|
321
|
+
path: string
|
|
322
|
+
mtimeMs: number
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
/**
|
|
326
|
+
* More directories than this under a task's output prefixes: record
|
|
327
|
+
* nothing, keep the walk. 256 refused payload's `@payloadcms/ui#build`
|
|
328
|
+
* (535 directories, 4,069 files), which then paid the refused snapshot's
|
|
329
|
+
* walk AND the output glob on every warm no-op — 65 ms of a 320 ms run
|
|
330
|
+
* (2026-09-11). Rows are 60 bytes each and the check is one stat per
|
|
331
|
+
* directory, so the cap is a bound on pathological trees, not a budget.
|
|
332
|
+
*/
|
|
333
|
+
export const OUTPUT_DIRS_CAP = 8192
|
|
334
|
+
|
|
335
|
+
/**
|
|
336
|
+
* A directory whose mtime lies within this many ms of the snapshot is RACY
|
|
337
|
+
* and the whole snapshot is dropped. File timestamps are coarse on Linux
|
|
338
|
+
* (a kernel tick, up to 10 ms), so a write landing in the same tick as the
|
|
339
|
+
* one that made the directory's recorded mtime leaves the mtime unchanged
|
|
340
|
+
* and the change invisible — git's index distrusts stats this young for
|
|
341
|
+
* the same reason (a stray survived a hit on the ubuntu job, 2026-09-03).
|
|
342
|
+
* The next hit walks, and once the tree is older than the window it is
|
|
343
|
+
* recorded for good.
|
|
344
|
+
*/
|
|
345
|
+
export const OUTPUT_DIRS_RACY_MS = 50
|
|
346
|
+
|
|
347
|
+
/** The stat memo's twin of `OUTPUT_DIRS_RACY_MS`: a file changed within this window of the stat is not memoised. */
|
|
348
|
+
export const FILE_HASH_RACY_MS = 50
|
|
349
|
+
|
|
350
|
+
/**
|
|
351
|
+
* The shape every cache implementation honors. `Cache` (the local v10
|
|
352
|
+
* implementation) and `LayeredCache` both `implements` this so the
|
|
353
|
+
* orchestrator's `executeTask` can take either without a discriminated
|
|
354
|
+
* union and we get a compile-time guarantee the surfaces stay congruent.
|
|
355
|
+
*/
|
|
356
|
+
/**
|
|
357
|
+
* Context passed to `get()`. Optional, but required when the lookup
|
|
358
|
+
* may resolve through the remote layer — the local SQL row inserted on
|
|
359
|
+
* remote-hit needs `taskId` + `command` to be queryable later (the
|
|
360
|
+
* artifact itself doesn't carry them). `Cache` (local) ignores this
|
|
361
|
+
* field; `LayeredCache` forwards it to `Cache.ingest`.
|
|
362
|
+
*/
|
|
363
|
+
export interface CacheGetContext {
|
|
364
|
+
taskId: string
|
|
365
|
+
command: string
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
/** Metadata supplied at ingest time — values the artifact does not carry. */
|
|
369
|
+
export interface IngestMeta {
|
|
370
|
+
taskId: string
|
|
371
|
+
command: string
|
|
372
|
+
/** Wall-clock time of the original task execution. */
|
|
373
|
+
durationMs: number
|
|
374
|
+
/**
|
|
375
|
+
* Cache-key components (Tier-3 input fingerprint) to persist into
|
|
376
|
+
* `entry_inputs` in the same transaction as the entry row. Omitted on
|
|
377
|
+
* the remote-hit ingest path (the artifact doesn't carry them — the
|
|
378
|
+
* fingerprint is the saving machine's local moat). Only computed on a
|
|
379
|
+
* miss/save, so a cache hit never reaches here.
|
|
380
|
+
*/
|
|
381
|
+
inputComponents?: readonly TaskInputRow[]
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
/**
|
|
385
|
+
* The supplied artifact bytes don't decompress/parse as a vx artifact.
|
|
386
|
+
* Thrown by `save`/`ingest` BEFORE anything reaches the final cache
|
|
387
|
+
* path — a rejected artifact leaves no `<hash>.tar.zst` and no SQL row.
|
|
388
|
+
* The LayeredCache treats this as a remote fault on the remote-hit
|
|
389
|
+
* path (degrades to a cache miss).
|
|
390
|
+
*/
|
|
391
|
+
export class CorruptArtifactError extends Error {
|
|
392
|
+
constructor(
|
|
393
|
+
public readonly hash: string,
|
|
394
|
+
reason: string,
|
|
395
|
+
public override readonly cause?: unknown,
|
|
396
|
+
) {
|
|
397
|
+
super(`cache: corrupt artifact for ${hash}: ${reason}`)
|
|
398
|
+
this.name = 'CorruptArtifactError'
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
export interface CacheLayer {
|
|
403
|
+
/**
|
|
404
|
+
* The local handle this layer wraps, when it wraps one (`LayeredCache`).
|
|
405
|
+
* `resolveCache` uses it to drop a bare local layer another declared layer
|
|
406
|
+
* already contains, so a plugin that layers over the local handle does
|
|
407
|
+
* not write the
|
|
408
|
+
* local store twice.
|
|
409
|
+
*/
|
|
410
|
+
readonly local?: Cache | undefined
|
|
411
|
+
/**
|
|
412
|
+
* `true` iff a REMOTE cache sits behind this layer. THE answer to "can
|
|
413
|
+
* the remote axes of the cache policy do anything on this run?" — the
|
|
414
|
+
* orchestrator clamps `remoteRead`/`remoteWrite` off when it is absent,
|
|
415
|
+
* skips the up-front local classify when it is present (that classify's
|
|
416
|
+
* `cache.get` would be a remote read-through awaited before scheduling),
|
|
417
|
+
* and drives the prefetch pass off it.
|
|
418
|
+
*
|
|
419
|
+
* A layer must answer TRUTHFULLY: identity against the local cache
|
|
420
|
+
* ("something other than the handle I passed in") is NOT the same
|
|
421
|
+
* question — an ordinary pass-through decorator with no remote at all
|
|
422
|
+
* answers yes to it. `LayeredCache` sets it, a bare `Cache` denies it,
|
|
423
|
+
* and a third-party layer opts in when, and only when, it really has a
|
|
424
|
+
* remote. Optional so a layer written before this contract keeps
|
|
425
|
+
* type-checking; absent reads as "no remote", the safe answer.
|
|
426
|
+
*/
|
|
427
|
+
readonly hasRemote?: boolean
|
|
428
|
+
/**
|
|
429
|
+
* Batch existence probe over the remote — the subset of `hashes` stored
|
|
430
|
+
* remotely, in ONE round-trip, or `null` for "no batch info; use
|
|
431
|
+
* per-hash". Only meaningful with `hasRemote`; a remote layer that can't
|
|
432
|
+
* batch omits it and the prefetch pass falls back to per-hash GETs.
|
|
433
|
+
*/
|
|
434
|
+
remoteHasMany?(hashes: readonly string[]): Promise<Set<string> | null>
|
|
435
|
+
/**
|
|
436
|
+
* Record that the remote has NO artifact for each of `hashes` (from a
|
|
437
|
+
* batch probe), so a later `get`/`prefetch` short-circuits to a miss with
|
|
438
|
+
* no round-trip. Only meaningful with `hasRemote`.
|
|
439
|
+
*/
|
|
440
|
+
markRemoteAbsent?(hashes: Iterable<string>): void
|
|
441
|
+
/**
|
|
442
|
+
* Resolve once every background write-through upload settles. The
|
|
443
|
+
* orchestrator awaits it before `close()` — an upload reading layer state
|
|
444
|
+
* after close would race. Only meaningful with `hasRemote`; a layer that
|
|
445
|
+
* uploads synchronously (or not at all) omits it.
|
|
446
|
+
*/
|
|
447
|
+
drainUploads?(): Promise<void>
|
|
448
|
+
key(input: CacheKeyInput): Promise<string>
|
|
449
|
+
get(hash: string, ctx?: CacheGetContext): Promise<CacheEntry | null>
|
|
450
|
+
/**
|
|
451
|
+
* Optional batched `get` (same answers, fewer round trips). The
|
|
452
|
+
* short-circuit probe uses it when a layer offers one; a layer without it
|
|
453
|
+
* is probed hash by hash.
|
|
454
|
+
*/
|
|
455
|
+
getMany?(hashes: readonly string[]): Promise<Map<string, CacheEntry>>
|
|
456
|
+
/**
|
|
457
|
+
* Lightweight existence probe. `'local'` / `'remote'` names the layer
|
|
458
|
+
* that holds the artifact; `null` is a miss. NEVER moves bytes: no
|
|
459
|
+
* artifact read, no remote download, no local ingest, no accessed_at
|
|
460
|
+
* bump (the LayeredCache's remote side is an HTTP HEAD). Planning
|
|
461
|
+
* (`--dry` / `--graph`) predicts hits with this instead of `get` so a
|
|
462
|
+
* dry run can't pull N artifacts over the network. Remote errors
|
|
463
|
+
* degrade to `null` — an existence probe never fails anything.
|
|
464
|
+
*/
|
|
465
|
+
has(hash: string): Promise<'local' | 'remote' | null>
|
|
466
|
+
/**
|
|
467
|
+
* Best-effort warm of `hash` from a slower layer (the remote cache)
|
|
468
|
+
* into this one, so a later `get(hash)` resolves locally without a
|
|
469
|
+
* round-trip on the task's critical path. Returns `true` if the
|
|
470
|
+
* artifact is now present locally, `false` on a miss / error
|
|
471
|
+
* (degrades, never throws). The local `Cache` is a no-op (nothing
|
|
472
|
+
* slower to warm from); `LayeredCache` owns the real implementation
|
|
473
|
+
* plus the in-flight de-dup so prefetch + get share ONE remote GET.
|
|
474
|
+
*/
|
|
475
|
+
prefetch(hash: string, ctx?: CacheGetContext): Promise<boolean>
|
|
476
|
+
/**
|
|
477
|
+
* Batched lookup of per-output-file fingerprints for many cache
|
|
478
|
+
* entries in one SQL round-trip. Returns a Map keyed by entry hash.
|
|
479
|
+
*
|
|
480
|
+
* Orchestrator pattern: call this once at `prepareRun` for every
|
|
481
|
+
* task whose hash is known up-front, then per-task `executeCachedTask`
|
|
482
|
+
* does a Map.get (O(1)) + parallel stat checks to decide whether the
|
|
483
|
+
* on-disk tree is already current.
|
|
484
|
+
*
|
|
485
|
+
* Hashes with no rows (cache misses) are absent from the result.
|
|
486
|
+
*/
|
|
487
|
+
loadOutputFilesBatch(hashes: readonly string[]): Map<string, OutputFileRow[]>
|
|
488
|
+
/**
|
|
489
|
+
* Stat each `expected` row's target under `projectDir` and return
|
|
490
|
+
* `true` iff every (size, mode, mtime) matches. Missing files,
|
|
491
|
+
* stat errors, or any mismatch → `false`.
|
|
492
|
+
*
|
|
493
|
+
* Pure FS check — no DB access. Caller batches the expected rows
|
|
494
|
+
* via `loadOutputFilesBatch`. Lets the orchestrator skip
|
|
495
|
+
* `cleanOutputs + restoreOutputs` entirely when the cached
|
|
496
|
+
* snapshot is already in place. Integrity-preserving: detects
|
|
497
|
+
* out-of-band file edits or deletions and falls through to a real
|
|
498
|
+
* restore.
|
|
499
|
+
*/
|
|
500
|
+
isOutputsCurrent(projectDir: string, expected: readonly OutputFileRow[]): Promise<boolean>
|
|
501
|
+
/**
|
|
502
|
+
* The directory-mtime short-circuit behind a warm hit (optional — a layer
|
|
503
|
+
* without it keeps the output walk). `recordOutputDirs` snapshots every
|
|
504
|
+
* directory under each of `prefixes` (project-relative, whole-subtree
|
|
505
|
+
* globs only — see `wholeSubtreePrefixes`) after a save or restore;
|
|
506
|
+
* `loadOutputDirsBatch` reads them back; `outputDirsCurrent` is true iff
|
|
507
|
+
* every recorded directory still carries its recorded mtime, which proves
|
|
508
|
+
* no file was added or removed anywhere the glob could see. Machine-local
|
|
509
|
+
* state, like the output rows: a remote ingest records none.
|
|
510
|
+
*/
|
|
511
|
+
recordOutputDirs?(hash: string, projectDir: string, prefixes: readonly string[]): Promise<void>
|
|
512
|
+
loadOutputDirsBatch?(hashes: readonly string[]): Map<string, OutputDirRow[]>
|
|
513
|
+
outputDirsCurrent?(projectDir: string, rows: readonly OutputDirRow[]): Promise<boolean>
|
|
514
|
+
/**
|
|
515
|
+
* Extract the artifact's `outputs/` entries into `projectDir` and —
|
|
516
|
+
* when `workspaceRoot` is given — its `workspace-outputs/` entries
|
|
517
|
+
* into the workspace root. Callers restoring entries that may carry
|
|
518
|
+
* workspace outputs must pass `workspaceRoot`.
|
|
519
|
+
*/
|
|
520
|
+
restoreOutputs(hash: string, projectDir: string, workspaceRoot?: string): Promise<void>
|
|
521
|
+
save(args: {
|
|
522
|
+
hash: string
|
|
523
|
+
/**
|
|
524
|
+
* `exitCode` is deliberately NOT accepted: vx caches only successes, so
|
|
525
|
+
* the stored value is pinned to 0 and there is nothing for a caller to
|
|
526
|
+
* decide. Accepting a number and discarding it invited the one shape that
|
|
527
|
+
* launders a failure into a success — cache a failing task's outputs, read
|
|
528
|
+
* the entry back as exit 0, and the hit classifies `cache-hit` while the
|
|
529
|
+
* broken build's files are restored over a good tree. Unrepresentable
|
|
530
|
+
* beats guarded.
|
|
531
|
+
*/
|
|
532
|
+
entry: Omit<CacheEntry, 'hash' | 'storedAt' | 'outputFiles' | 'exitCode'>
|
|
533
|
+
projectDir: string
|
|
534
|
+
outputFiles: string[]
|
|
535
|
+
/**
|
|
536
|
+
* Set only by ChainedCache when an EARLIER layer in the chain already
|
|
537
|
+
* wrote this artifact to the same local handle: the local pack + write
|
|
538
|
+
* is skipped and only the layer's remote side acts. A layer with no
|
|
539
|
+
* remote side treats it as a full no-op.
|
|
540
|
+
*/
|
|
541
|
+
skipLocalWrite?: boolean
|
|
542
|
+
/**
|
|
543
|
+
* Resolved `outputs.workspaceFiles` (absolute paths) + the root
|
|
544
|
+
* they're relative to. Packed under `workspace-outputs/<rel>`.
|
|
545
|
+
* Omitted → artifact bytes identical to the pre-workspaceFiles
|
|
546
|
+
* format.
|
|
547
|
+
*/
|
|
548
|
+
workspaceOutputFiles?: string[]
|
|
549
|
+
workspaceRoot?: string
|
|
550
|
+
/**
|
|
551
|
+
* Cache-key components for this entry (the Tier-3 input
|
|
552
|
+
* fingerprint). Persisted to `entry_inputs` inside the same
|
|
553
|
+
* transaction as the entry row, via `INSERT OR IGNORE` (a re-save
|
|
554
|
+
* of the same hash is a no-op). Omitted/empty → nothing written.
|
|
555
|
+
* Only computed on the miss/save path — never on a hit.
|
|
556
|
+
*/
|
|
557
|
+
inputComponents?: readonly TaskInputRow[]
|
|
558
|
+
}): Promise<void>
|
|
559
|
+
/**
|
|
560
|
+
* Adopt an artifact produced elsewhere — the remote-hit path. Writes
|
|
561
|
+
* the compressed bytes to `<cacheDir>/<hash>.tar.zst`, parses the
|
|
562
|
+
* tar headers to populate the `output_files` rows, and inserts the
|
|
563
|
+
* `entries` row using the caller-supplied `meta`. After this returns,
|
|
564
|
+
* the next `get(hash)` resolves locally.
|
|
565
|
+
*/
|
|
566
|
+
ingest(hash: string, compressed: Uint8Array, meta: IngestMeta): Promise<void>
|
|
567
|
+
/**
|
|
568
|
+
* The ONE run-history write: a whole `vx run` atomically, the per-task
|
|
569
|
+
* `runs` rows and the one `invocations` header row — in a SINGLE
|
|
570
|
+
* transaction (one fsync). Input-fingerprint rows are NOT written here — they
|
|
571
|
+
* ride the entry-save transaction (`save`/`ingest`) so a warm
|
|
572
|
+
* all-cache-hit run writes nothing.
|
|
573
|
+
*/
|
|
574
|
+
recordRunBundle(bundle: { runs: readonly RunRecord[]; invocation: InvocationRecord }): void
|
|
575
|
+
stats(opts?: CacheStatsOptions): CacheStats
|
|
576
|
+
/**
|
|
577
|
+
* Content-hash a file with an mtime+size fast path. If the
|
|
578
|
+
* `(mtime_ms, size_bytes)` of `filePath` match a previously seen
|
|
579
|
+
* row, return the stored digest (a git blob OID since CACHE_VERSION
|
|
580
|
+
* v20) instead of re-reading the bytes. Otherwise read + hash + upsert.
|
|
581
|
+
* The hash is byte-for-byte identical to what a fresh content-hash
|
|
582
|
+
* would produce — pure optimization, no cache-key change.
|
|
583
|
+
*/
|
|
584
|
+
hashFile(filePath: string): Promise<string>
|
|
585
|
+
/**
|
|
586
|
+
* Absolute path to the on-disk outputs artifact for a hash —
|
|
587
|
+
* `<cacheDir>/<hash>.tar.zst`. Returns the path whether or
|
|
588
|
+
* not the artifact exists. Exposed for telemetry / dashboards;
|
|
589
|
+
* `restoreOutputs` is the canonical way to materialize the bytes.
|
|
590
|
+
*/
|
|
591
|
+
outputsPath(hash: string): string
|
|
592
|
+
prune(options: PruneOptions): Promise<PruneResult>
|
|
593
|
+
close(): void
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
/**
|
|
597
|
+
* Convenience alias for the `save()` args. Used by `LayeredCache` to
|
|
598
|
+
* forward call args without redeclaring the structural shape — NOT
|
|
599
|
+
* part of the conceptual cache contract; consumers should call
|
|
600
|
+
* `CacheLayer.save({ ... })` directly.
|
|
601
|
+
*
|
|
602
|
+
* @internal
|
|
603
|
+
*/
|
|
604
|
+
export type SaveArgs = Parameters<CacheLayer['save']>[0]
|