@vzn/vx 0.0.16 → 0.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -89
- package/index.ts +10 -0
- package/launcher.mjs +10 -11
- package/package.json +19 -5
- package/plugins/local-cache/index.ts +2 -0
- package/plugins/local-executor/index.ts +2 -0
- package/plugins/schedule-history/index.ts +2 -0
- package/src/bin.ts +2 -2
- package/src/cache/archive.ts +557 -0
- package/src/cache/cache.ts +929 -250
- package/src/cache/cas-backend.ts +20 -2
- package/src/cache/chained-cache.ts +202 -0
- package/src/cache/digest.ts +0 -15
- package/src/cache/index.ts +13 -3
- package/src/cache/inputs.ts +445 -41
- package/src/cache/layered-cache.ts +88 -30
- package/src/cache/tar-stream.ts +358 -0
- package/src/cli/cache.ts +36 -10
- package/src/cli/help.ts +167 -128
- package/src/cli/index.ts +74 -43
- package/src/cli/info.ts +30 -0
- package/src/cli/last.ts +140 -0
- package/src/cli/migrate-emit.ts +21 -0
- package/src/cli/migrate-nx.ts +23 -0
- package/src/cli/migrate-scripts.ts +127 -0
- package/src/cli/migrate-turbo.ts +43 -6
- package/src/cli/migrate.ts +119 -33
- package/src/cli/plan-format.ts +58 -3
- package/src/cli/plugin-commands.ts +96 -0
- package/src/cli/prune.ts +291 -0
- package/src/cli/run.ts +272 -80
- package/src/cli/upgrade.ts +7 -6
- package/src/cli/watch.ts +259 -61
- package/src/cli/why.ts +193 -0
- package/src/config.ts +40 -23
- package/src/exec/env.ts +7 -1
- package/src/exec/executor.ts +239 -0
- package/src/exec/index.ts +19 -1
- package/src/exec/runner.ts +116 -29
- package/src/exec/sandbox-runtime.ts +194 -24
- package/src/graph/index.ts +3 -1
- package/src/graph/scheduler.ts +92 -18
- package/src/graph/task-graph.ts +220 -6
- package/src/index.ts +82 -84
- package/src/orchestrator/deferred-outputs.ts +144 -0
- package/src/orchestrator/download-policy.ts +171 -0
- package/src/orchestrator/events.ts +30 -0
- package/src/orchestrator/execute-task.ts +328 -94
- package/src/orchestrator/failure-mode.ts +61 -0
- package/src/orchestrator/framed-output.ts +87 -35
- package/src/orchestrator/history.ts +78 -31
- package/src/orchestrator/index.ts +43 -90
- package/src/orchestrator/local-shortcircuit.ts +22 -0
- package/src/orchestrator/logger.ts +196 -14
- package/src/orchestrator/metrics.ts +57 -1376
- package/src/orchestrator/missing-plugin.ts +9 -0
- package/src/orchestrator/options.ts +22 -4
- package/src/orchestrator/plan.ts +144 -1
- package/src/orchestrator/plugin-host.ts +199 -119
- package/src/orchestrator/plugin.ts +144 -57
- package/src/orchestrator/prepare.ts +220 -69
- package/src/orchestrator/remote-prefetch.ts +12 -8
- package/src/orchestrator/resources.ts +20 -34
- package/src/orchestrator/run-artifacts.ts +41 -16
- package/src/orchestrator/run-context.ts +167 -29
- package/src/orchestrator/run-report.ts +73 -46
- package/src/orchestrator/run.ts +590 -122
- package/src/orchestrator/stable-keys.ts +74 -33
- package/src/orchestrator/status-line.ts +47 -8
- package/src/orchestrator/summary.ts +22 -1
- package/src/orchestrator/tally.ts +78 -35
- package/src/orchestrator/task-hash.ts +105 -11
- package/src/orchestrator/task-log-buffer.ts +291 -0
- package/src/orchestrator/telemetry-host.ts +24 -6
- package/src/orchestrator/telemetry.ts +181 -21
- package/src/orchestrator/upstream.ts +72 -31
- package/src/orchestrator/verify.ts +12 -2
- package/src/plugins/local-cache/index.ts +11 -0
- package/src/plugins/local-executor/index.ts +45 -0
- package/src/plugins/schedule-history/index.ts +109 -0
- package/src/util/edit-distance.ts +21 -0
- package/src/util/errors.ts +15 -1
- package/src/util/index.ts +7 -2
- package/src/util/num.ts +37 -0
- package/src/util/paths.ts +43 -0
- package/src/util/settle.ts +48 -0
- package/src/util/size.ts +5 -1
- package/src/util/tail.ts +66 -0
- package/src/util/timing.ts +63 -0
- package/src/workspace/affected.ts +142 -15
- package/src/workspace/config-cache.ts +304 -0
- package/src/workspace/config-eval.ts +210 -0
- package/src/workspace/config-imports.ts +161 -0
- package/src/workspace/filter.ts +22 -4
- package/src/workspace/fingerprint.ts +16 -1
- package/src/workspace/index.ts +15 -3
- package/src/workspace/lockfile.ts +13 -2
- package/src/workspace/nested-dirs.ts +21 -7
- package/src/workspace/package-graph.ts +23 -20
- package/src/workspace/project-loader.ts +503 -59
- package/src/workspace/workspace.ts +213 -72
- package/src/cache/tar.ts +0 -343
- package/src/cli/backend.ts +0 -23
- package/src/cli/mcp-rpc.ts +0 -292
- package/src/cli/mcp.ts +0 -92
- package/src/orchestrator/devframe-surface.ts +0 -74
- package/src/orchestrator/predict.ts +0 -106
- package/src/orchestrator/protocol.ts +0 -156
- package/src/orchestrator/run-state.ts +0 -145
- package/src/orchestrator/wire-render.ts +0 -92
- package/src/orchestrator/wire.ts +0 -212
package/src/cache/cache.ts
CHANGED
|
@@ -30,13 +30,19 @@
|
|
|
30
30
|
// close : release the SQLite handle
|
|
31
31
|
|
|
32
32
|
import { Database, type SQLQueryBindings } from 'bun:sqlite'
|
|
33
|
-
import { mkdirSync, statSync } from 'node:fs'
|
|
34
|
-
import { mkdir,
|
|
35
|
-
import os from 'node:os'
|
|
33
|
+
import { existsSync, mkdirSync, statSync, writeFileSync } from 'node:fs'
|
|
34
|
+
import { lstat, mkdir, readdir, rename, rm, stat, unlink, writeFile } from 'node:fs/promises'
|
|
36
35
|
import path from 'node:path'
|
|
37
|
-
import { relPosix, UserError, xxh3, xxh3hex } from '../util/index.js'
|
|
36
|
+
import { relPosix, UserError, xxh3, xxh3hex, span } from '../util/index.js'
|
|
37
|
+
import {
|
|
38
|
+
ArchiveSecurityError,
|
|
39
|
+
extractArtifactStream,
|
|
40
|
+
scanArtifact,
|
|
41
|
+
packArtifactBytes,
|
|
42
|
+
packArtifactStream,
|
|
43
|
+
planArtifact,
|
|
44
|
+
} from './archive.js'
|
|
38
45
|
import { FsCASBackend } from './cas-backend.js'
|
|
39
|
-
import { extractOutputs, parseTarHeaders, readTarText } from './tar.js'
|
|
40
46
|
|
|
41
47
|
// v17: artifact carries only logs + outputs (stdout + outputs/<rel>).
|
|
42
48
|
// Local and remote layers transport the SAME tar.zst bytes — no
|
|
@@ -75,7 +81,28 @@ import { extractOutputs, parseTarHeaders, readTarText } from './tar.js'
|
|
|
75
81
|
// inside the key fold (the new `captureInto` sink is a pure
|
|
76
82
|
// side-channel). A task's hash is byte-identical before and after, so
|
|
77
83
|
// existing artifacts stay valid and there is no cache-version bump.
|
|
78
|
-
|
|
84
|
+
// v25: the ARTIFACT BYTES are wrong in every existing entry, while the
|
|
85
|
+
// key that addresses them is unchanged — the one situation a version
|
|
86
|
+
// bump exists for. Two defects: (a) `packArtifact` staged outputs with
|
|
87
|
+
// `Bun.write`, which does not carry the source mode, so every artifact
|
|
88
|
+
// records 0644 and a cache hit strips the executable bit off any binary
|
|
89
|
+
// or generated script; (b) it packed `--format=ustar`, which splits a
|
|
90
|
+
// name over 100 bytes into prefix+name, and the reader read only `name`
|
|
91
|
+
// — so those entries lost their `outputs/` prefix and were silently
|
|
92
|
+
// dropped from both the restore and the output_files index. Neither is
|
|
93
|
+
// self-healing: the stored bytes/headers are already wrong, so without a
|
|
94
|
+
// new namespace the fixed code would keep replaying them forever.
|
|
95
|
+
// v26: same situation, different producer. A task whose child was killed by
|
|
96
|
+
// a shutdown signal reported `aborted`, but `aborted` did not propagate to
|
|
97
|
+
// dependents — so a dependent ran against PARTIAL outputs, succeeded, and
|
|
98
|
+
// cached what it built. The fold takes the upstream's INPUT key, which a
|
|
99
|
+
// signal does not change, so that entry sits under exactly the key a healthy
|
|
100
|
+
// run derives and replays forever as a green hit. Fixing the propagation
|
|
101
|
+
// stops new poison but cannot reach entries already written, and a
|
|
102
|
+
// `LayeredCache` uploads them, so the reach is a whole team's shared cache
|
|
103
|
+
// rather than one developer's disk. Pre-alpha, so one cold rebuild is the
|
|
104
|
+
// cheap side of that trade.
|
|
105
|
+
const CACHE_VERSION = 'vx-cache-v27'
|
|
79
106
|
// SCHEMA history (drop+recreate on mismatch; pre-alpha, no migrations):
|
|
80
107
|
// v20: file_hashes.content_hash (git blob OIDs).
|
|
81
108
|
// v21: dropped the unused outputs_hash column (pure-input hashing).
|
|
@@ -94,7 +121,44 @@ const CACHE_VERSION = 'vx-cache-v24'
|
|
|
94
121
|
// passed under identical inputs is nondeterministic by definition;
|
|
95
122
|
// no cross-run inference needed). Nullable; NULL for a once-run
|
|
96
123
|
// task. Analytics-only — the cache KEY is unchanged.
|
|
97
|
-
|
|
124
|
+
// v24: file_hashes.ctime_ms + .ino — the stat memo's guard against a
|
|
125
|
+
// content change that PRESERVES mtime (`tar -x`, `cp -p`,
|
|
126
|
+
// `rsync --times`, SOURCE_DATE_EPOCH). (mtime, size) alone
|
|
127
|
+
// returned the previous run's digest for different bytes, i.e. a
|
|
128
|
+
// stale cache hit. `utimes` cannot suppress ctime unprivileged
|
|
129
|
+
// and an atomic write-then-rename changes the inode, so the two
|
|
130
|
+
// together close it; git's index keys on ctime+ino+dev for the
|
|
131
|
+
// same reason. Both come free from the stat already taken. The
|
|
132
|
+
// cache KEY derivation is unchanged (CACHE_VERSION not bumped) —
|
|
133
|
+
// the memo simply stops answering wrongly, so an affected task's
|
|
134
|
+
// key moves from a WRONG value to the right one: it misses once,
|
|
135
|
+
// re-runs, re-caches. Self-healing, never a wrong hit.
|
|
136
|
+
const SCHEMA_VERSION = 'v24'
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* SQL predicate selecting `runs` rows that record an EXECUTION.
|
|
140
|
+
*
|
|
141
|
+
* Every non-group, non-aborted outcome of a run gets a row, so the header's
|
|
142
|
+
* `task_count` matches `COUNT(*)` and the run-detail timeline is complete.
|
|
143
|
+
* But a `skipped` row is a task the run never executed — its upstream failed,
|
|
144
|
+
* so it has no exit of its own, no duration, and made no cache decision.
|
|
145
|
+
* Counting it in a RATE or a MEAN dilutes that figure with a non-event: a
|
|
146
|
+
* task skipped as often as it succeeds reads 50% success, and a zero-duration
|
|
147
|
+
* row drags every average toward zero.
|
|
148
|
+
*
|
|
149
|
+
* Lives here, beside the schema, because the same figure is computed in more
|
|
150
|
+
* than one place (`Cache.stats` and the run-history queries both answer
|
|
151
|
+
* "runs in the last 24h") and two copies of a rule are how they drift apart.
|
|
152
|
+
*/
|
|
153
|
+
export const EXECUTED_RUNS_SQL = "status <> 'skipped'"
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* SQL predicate selecting `runs` rows that recorded a cache key. `hash` is
|
|
157
|
+
* `''` for an outcome that never derived one (see {@link RunRecord.hash}),
|
|
158
|
+
* and `''` is not a key: it can neither corroborate flakiness nor say the
|
|
159
|
+
* inputs changed. Mirrors the `hash <> ''` guards in the cloud analytics copy.
|
|
160
|
+
*/
|
|
161
|
+
export const KEYED_RUNS_SQL = "hash <> ''"
|
|
98
162
|
|
|
99
163
|
/**
|
|
100
164
|
* Artifact + `output_files` namespace prefix for workspace-root-
|
|
@@ -222,6 +286,29 @@ export interface CacheKeyInput {
|
|
|
222
286
|
* absent, the captured `upstream` row falls back to `name = hash`.
|
|
223
287
|
*/
|
|
224
288
|
upstreamIds?: ReadonlyMap<string, string>
|
|
289
|
+
/**
|
|
290
|
+
* The task's DEPENDENCY closure with GROUP tasks expanded into the real
|
|
291
|
+
* tasks they stand for — what an input-shipping executor must place in the
|
|
292
|
+
* input root. A different question from what the key folds, and derived
|
|
293
|
+
* differently in both directions:
|
|
294
|
+
*
|
|
295
|
+
* - It expands groups, which contribute a synthetic roll-up hash and no
|
|
296
|
+
* outputs of their own, so a dependent of one would otherwise describe
|
|
297
|
+
* an empty closure.
|
|
298
|
+
* - It ignores `cache.inputs.tasks`. That filter says which upstream KEYS
|
|
299
|
+
* this task's key folds; what the task may READ is `dependsOn`, and
|
|
300
|
+
* locally every dependency's outputs are on disk before the command runs
|
|
301
|
+
* however the filter is written.
|
|
302
|
+
*
|
|
303
|
+
* NEVER folded into the digest — the key cascades through a group's own
|
|
304
|
+
* roll-up hash already, and folding either difference in would move every
|
|
305
|
+
* existing dependent's key without telling it anything new.
|
|
306
|
+
*/
|
|
307
|
+
upstreamGraft?: ReadonlyArray<{
|
|
308
|
+
readonly taskId: string
|
|
309
|
+
readonly hash: string
|
|
310
|
+
readonly projectDir: string
|
|
311
|
+
}>
|
|
225
312
|
/**
|
|
226
313
|
* Workspace-level fingerprint — typically a hash of `pnpm-lock.yaml` +
|
|
227
314
|
* `pnpm-workspace.yaml`. Folds resolved dep versions and workspace shape
|
|
@@ -253,6 +340,12 @@ export interface CacheKeyInput {
|
|
|
253
340
|
* hash arrived via the map or the fallback.
|
|
254
341
|
*/
|
|
255
342
|
fileHashes?: ReadonlyMap<string, string>
|
|
343
|
+
/**
|
|
344
|
+
* Material a plugin's `key` stage contributed, as sorted `[name, value]`
|
|
345
|
+
* pairs. Folded only when non-empty, so a workspace without a key plugin
|
|
346
|
+
* derives exactly the key it always did.
|
|
347
|
+
*/
|
|
348
|
+
pluginParts?: ReadonlyArray<readonly [name: string, value: string]>
|
|
256
349
|
/**
|
|
257
350
|
* When set, `key()` pushes each component (kind, name, hash) it folds
|
|
258
351
|
* — at the same fold sites, in fold order. Pure SIDE-CHANNEL: it does
|
|
@@ -276,6 +369,19 @@ export interface CacheEntry {
|
|
|
276
369
|
exitCode: number
|
|
277
370
|
durationMs: number
|
|
278
371
|
outputFiles: string[]
|
|
372
|
+
/**
|
|
373
|
+
* The `output_files` rows behind `outputFiles` (size / mode / mtime), when
|
|
374
|
+
* the layer that produced this entry had them in hand. `restoreHit` reads
|
|
375
|
+
* them for its tree-is-current check instead of re-querying — one SQL
|
|
376
|
+
* round trip per cache hit, saved.
|
|
377
|
+
*/
|
|
378
|
+
outputRows?: OutputFileRow[]
|
|
379
|
+
/**
|
|
380
|
+
* The `output_dirs` rows behind the directory short-circuit, when the
|
|
381
|
+
* batched `getMany` loaded them — one query for the run instead of one
|
|
382
|
+
* per hit (0.12 ms each across a warm 1000-project run before this).
|
|
383
|
+
*/
|
|
384
|
+
outputDirRows?: OutputDirRow[]
|
|
279
385
|
/** Captured stdout, always present (may be empty). stderr is not cached. */
|
|
280
386
|
stdout: string
|
|
281
387
|
storedAt: string
|
|
@@ -291,7 +397,17 @@ export interface CacheEntry {
|
|
|
291
397
|
}
|
|
292
398
|
|
|
293
399
|
export interface RunRecord {
|
|
294
|
-
|
|
400
|
+
/**
|
|
401
|
+
* The task's cache key. ABSENT when the outcome never derived one — a
|
|
402
|
+
* `skipped` task (its upstream failed, so it never probed) or a
|
|
403
|
+
* `persistent` one (a dev server is never cached). Such a row is still a
|
|
404
|
+
* task of the run and must be recorded, so the column keeps a `''`
|
|
405
|
+
* sentinel rather than going nullable: `''` is impossible for a real key
|
|
406
|
+
* (`Cache.key` returns 16 hex chars), the reads that must not treat it as
|
|
407
|
+
* a key already guard it, and it matches what the cloud `task_runs.hash`
|
|
408
|
+
* column has stored for the same concept since its first migration.
|
|
409
|
+
*/
|
|
410
|
+
hash?: string
|
|
295
411
|
project: string
|
|
296
412
|
task: string
|
|
297
413
|
status: 'success' | 'failed' | 'cache-hit' | 'cache-hit-remote' | 'skipped'
|
|
@@ -373,6 +489,11 @@ export interface CacheStats {
|
|
|
373
489
|
hitCountLast24h: number
|
|
374
490
|
}
|
|
375
491
|
|
|
492
|
+
export interface CacheStatsOptions {
|
|
493
|
+
/** Narrow every aggregate to one project. Absent = the whole workspace. */
|
|
494
|
+
project?: string
|
|
495
|
+
}
|
|
496
|
+
|
|
376
497
|
export interface PruneOptions {
|
|
377
498
|
/** Drop entries last accessed before this ms-epoch threshold. */
|
|
378
499
|
olderThanMs?: number
|
|
@@ -408,6 +529,27 @@ export interface OutputFileRow {
|
|
|
408
529
|
mtimeMs: number
|
|
409
530
|
}
|
|
410
531
|
|
|
532
|
+
/** One directory under a whole-subtree output glob, as it stood after the last save or restore on THIS machine. */
|
|
533
|
+
export interface OutputDirRow {
|
|
534
|
+
path: string
|
|
535
|
+
mtimeMs: number
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
/** More directories than this under a task's output prefixes: record nothing, keep the walk. */
|
|
539
|
+
export const OUTPUT_DIRS_CAP = 256
|
|
540
|
+
|
|
541
|
+
/**
|
|
542
|
+
* A directory whose mtime lies within this many ms of the snapshot is RACY
|
|
543
|
+
* and the whole snapshot is dropped. File timestamps are coarse on Linux
|
|
544
|
+
* (a kernel tick, up to 10 ms), so a write landing in the same tick as the
|
|
545
|
+
* one that made the directory's recorded mtime leaves the mtime unchanged
|
|
546
|
+
* and the change invisible — git's index distrusts stats this young for
|
|
547
|
+
* the same reason (a stray survived a hit on the ubuntu job, 2026-09-03).
|
|
548
|
+
* The next hit walks, and once the tree is older than the window it is
|
|
549
|
+
* recorded for good.
|
|
550
|
+
*/
|
|
551
|
+
export const OUTPUT_DIRS_RACY_MS = 50
|
|
552
|
+
|
|
411
553
|
/**
|
|
412
554
|
* The shape every cache implementation honors. `Cache` (the local v10
|
|
413
555
|
* implementation) and `LayeredCache` both `implements` this so the
|
|
@@ -466,7 +608,7 @@ export class CorruptArtifactError extends Error {
|
|
|
466
608
|
* bounding a malicious/compromised remote's ability to OOM a victim who takes
|
|
467
609
|
* a cache hit.
|
|
468
610
|
*/
|
|
469
|
-
|
|
611
|
+
const MAX_DECOMPRESSED_ARTIFACT_BYTES = 2 * 1024 * 1024 * 1024
|
|
470
612
|
|
|
471
613
|
/**
|
|
472
614
|
* Read a zstd frame's declared Frame_Content_Size (RFC 8878 §3.1.1) WITHOUT
|
|
@@ -503,18 +645,27 @@ export function zstdContentSize(b: Uint8Array): bigint | null {
|
|
|
503
645
|
}
|
|
504
646
|
|
|
505
647
|
/**
|
|
506
|
-
* Decompress a zstd artifact
|
|
507
|
-
*
|
|
508
|
-
*
|
|
509
|
-
*
|
|
510
|
-
*
|
|
511
|
-
*
|
|
648
|
+
* Decompress a zstd artifact that DECLARES its content size, with a hard
|
|
649
|
+
* output ceiling: refused before a byte is allocated when the declaration
|
|
650
|
+
* is over the cap, and again on the actual length. A frame with no
|
|
651
|
+
* declaration (a streamed producer's — vx's own, above 4 MiB) never comes
|
|
652
|
+
* here: `decodedTar` decodes it as a stream under the running count, so
|
|
653
|
+
* a sizeless bomb has nowhere to expand.
|
|
512
654
|
*/
|
|
513
|
-
async function zstdDecompressBounded(
|
|
514
|
-
compressed
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
655
|
+
async function zstdDecompressBounded(compressed: Uint8Array, hash: string): Promise<Uint8Array> {
|
|
656
|
+
assertDeclaredSize(compressed, hash)
|
|
657
|
+
const out = await Bun.zstdDecompress(compressed)
|
|
658
|
+
if (out.length > MAX_DECOMPRESSED_ARTIFACT_BYTES) {
|
|
659
|
+
throw new CorruptArtifactError(
|
|
660
|
+
hash,
|
|
661
|
+
`decompressed to ${out.length} bytes (> ${MAX_DECOMPRESSED_ARTIFACT_BYTES} cap)`,
|
|
662
|
+
)
|
|
663
|
+
}
|
|
664
|
+
return out
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
/** The pre-decompress half of the ceiling: the frame header's own claim, when it makes one. */
|
|
668
|
+
function assertDeclaredSize(compressed: Uint8Array, hash: string): bigint | null {
|
|
518
669
|
const declared = zstdContentSize(compressed)
|
|
519
670
|
if (declared !== null && declared > BigInt(MAX_DECOMPRESSED_ARTIFACT_BYTES)) {
|
|
520
671
|
throw new CorruptArtifactError(
|
|
@@ -522,22 +673,143 @@ async function zstdDecompressBounded(
|
|
|
522
673
|
`declares ${declared} decompressed bytes (> ${MAX_DECOMPRESSED_ARTIFACT_BYTES} cap)`,
|
|
523
674
|
)
|
|
524
675
|
}
|
|
525
|
-
|
|
526
|
-
|
|
676
|
+
return declared
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
/** Compressed artifacts above this size are decoded as a stream, on restore and on ingest. */
|
|
680
|
+
const STREAM_DECODE_FROM = 4 * 1024 * 1024
|
|
681
|
+
|
|
682
|
+
/** Collect a byte stream; only ever used where the seam wants bytes. */
|
|
683
|
+
async function bytesOf(stream: ReadableStream<Uint8Array>): Promise<Uint8Array> {
|
|
684
|
+
return new Uint8Array(await new Response(stream).arrayBuffer())
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
/**
|
|
688
|
+
* Bun's `CompressionStream` types do not satisfy `pipeThrough`'s pair
|
|
689
|
+
* (its readable side is typed `NonSharedUint8Array`); the runtime object
|
|
690
|
+
* is a plain byte transform.
|
|
691
|
+
*/
|
|
692
|
+
const zstdEncoder = (): TransformStream<Uint8Array, Uint8Array> =>
|
|
693
|
+
new CompressionStream('zstd') as unknown as TransformStream<Uint8Array, Uint8Array>
|
|
694
|
+
|
|
695
|
+
const oneChunk = (bytes: Uint8Array): ReadableStream<Uint8Array> =>
|
|
696
|
+
new ReadableStream({
|
|
697
|
+
start(c) {
|
|
698
|
+
c.enqueue(bytes)
|
|
699
|
+
c.close()
|
|
700
|
+
},
|
|
701
|
+
})
|
|
702
|
+
|
|
703
|
+
/**
|
|
704
|
+
* The decoded tar as a stream: one call for a small artifact that
|
|
705
|
+
* declares its size, a streamed decode otherwise — the stream setup is
|
|
706
|
+
* ~35 µs, which matters at a thousand one-file artifacts and nowhere
|
|
707
|
+
* else. Bytes in memory are only ever the small case (a large source
|
|
708
|
+
* must be a FILE: a Blob copies its bytes and hands the decoder
|
|
709
|
+
* everything at once, measured +519 MiB on 150 MiB against +448 for the
|
|
710
|
+
* plain decode). The 2 GiB ceiling applies either way: the declaration
|
|
711
|
+
* and the result length in one call, a running count on the stream.
|
|
712
|
+
*/
|
|
713
|
+
async function decodedTar(
|
|
714
|
+
source: Uint8Array | Bun.BunFile,
|
|
715
|
+
hash: string,
|
|
716
|
+
): Promise<ReadableStream<Uint8Array>> {
|
|
717
|
+
if (source instanceof Uint8Array) {
|
|
718
|
+
if (assertDeclaredSize(source, hash) === null) return zstdDecodeStream(new Blob([source]), hash)
|
|
719
|
+
return oneChunk(await zstdDecompressBounded(source, hash))
|
|
527
720
|
}
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
`decompressed to ${out.length} bytes (> ${MAX_DECOMPRESSED_ARTIFACT_BYTES} cap)`,
|
|
533
|
-
)
|
|
721
|
+
if (source.size <= STREAM_DECODE_FROM) {
|
|
722
|
+
const bytes = await source.bytes()
|
|
723
|
+
if (assertDeclaredSize(bytes, hash) === null) return zstdDecodeStream(source, hash)
|
|
724
|
+
return oneChunk(await zstdDecompressBounded(bytes, hash))
|
|
534
725
|
}
|
|
535
|
-
|
|
726
|
+
assertDeclaredSize(await source.slice(0, 32).bytes(), hash)
|
|
727
|
+
return zstdDecodeStream(source, hash)
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
/**
|
|
731
|
+
* The streaming twin of `zstdDecompressBounded`: the decoded bytes as a
|
|
732
|
+
* stream, refused past the same ceiling. A malicious frame cannot expand
|
|
733
|
+
* past the cap here either — the count runs as bytes are produced, before
|
|
734
|
+
* any of them reach the reader's next entry.
|
|
735
|
+
*/
|
|
736
|
+
function zstdDecodeStream(source: Blob, hash: string): ReadableStream<Uint8Array> {
|
|
737
|
+
let total = 0
|
|
738
|
+
return source
|
|
739
|
+
.stream()
|
|
740
|
+
.pipeThrough(new DecompressionStream('zstd'))
|
|
741
|
+
.pipeThrough(
|
|
742
|
+
new TransformStream<Uint8Array, Uint8Array>({
|
|
743
|
+
transform(chunk, controller) {
|
|
744
|
+
total += chunk.byteLength
|
|
745
|
+
if (total > MAX_DECOMPRESSED_ARTIFACT_BYTES) {
|
|
746
|
+
controller.error(
|
|
747
|
+
new CorruptArtifactError(
|
|
748
|
+
hash,
|
|
749
|
+
`decompresses past ${MAX_DECOMPRESSED_ARTIFACT_BYTES} bytes (cap)`,
|
|
750
|
+
),
|
|
751
|
+
)
|
|
752
|
+
return
|
|
753
|
+
}
|
|
754
|
+
controller.enqueue(chunk)
|
|
755
|
+
},
|
|
756
|
+
}),
|
|
757
|
+
)
|
|
536
758
|
}
|
|
537
759
|
|
|
538
760
|
export interface CacheLayer {
|
|
761
|
+
/**
|
|
762
|
+
* The local handle this layer wraps, when it wraps one (`LayeredCache`).
|
|
763
|
+
* `resolveCache` uses it to drop a bare local layer another declared layer
|
|
764
|
+
* already contains, so `[cloud(), localCachePlugin()]` does not write the
|
|
765
|
+
* local store twice.
|
|
766
|
+
*/
|
|
767
|
+
readonly local?: Cache | undefined
|
|
768
|
+
/**
|
|
769
|
+
* `true` iff a REMOTE cache sits behind this layer. THE answer to "can
|
|
770
|
+
* the remote axes of the cache policy do anything on this run?" — the
|
|
771
|
+
* orchestrator clamps `remoteRead`/`remoteWrite` off when it is absent,
|
|
772
|
+
* skips the up-front local classify when it is present (that classify's
|
|
773
|
+
* `cache.get` would be a remote read-through awaited before scheduling),
|
|
774
|
+
* and drives the prefetch pass off it.
|
|
775
|
+
*
|
|
776
|
+
* A layer must answer TRUTHFULLY: identity against the local cache
|
|
777
|
+
* ("something other than the handle I passed in") is NOT the same
|
|
778
|
+
* question — an ordinary pass-through decorator with no remote at all
|
|
779
|
+
* answers yes to it. `LayeredCache` sets it, a bare `Cache` denies it,
|
|
780
|
+
* and a third-party layer opts in when, and only when, it really has a
|
|
781
|
+
* remote. Optional so a layer written before this contract keeps
|
|
782
|
+
* type-checking; absent reads as "no remote", the safe answer.
|
|
783
|
+
*/
|
|
784
|
+
readonly hasRemote?: boolean
|
|
785
|
+
/**
|
|
786
|
+
* Batch existence probe over the remote — the subset of `hashes` stored
|
|
787
|
+
* remotely, in ONE round-trip, or `null` for "no batch info; use
|
|
788
|
+
* per-hash". Only meaningful with `hasRemote`; a remote layer that can't
|
|
789
|
+
* batch omits it and the prefetch pass falls back to per-hash GETs.
|
|
790
|
+
*/
|
|
791
|
+
remoteHasMany?(hashes: readonly string[]): Promise<Set<string> | null>
|
|
792
|
+
/**
|
|
793
|
+
* Record that the remote has NO artifact for each of `hashes` (from a
|
|
794
|
+
* batch probe), so a later `get`/`prefetch` short-circuits to a miss with
|
|
795
|
+
* no round-trip. Only meaningful with `hasRemote`.
|
|
796
|
+
*/
|
|
797
|
+
markRemoteAbsent?(hashes: Iterable<string>): void
|
|
798
|
+
/**
|
|
799
|
+
* Resolve once every background write-through upload settles. The
|
|
800
|
+
* orchestrator awaits it before `close()` — an upload reading layer state
|
|
801
|
+
* after close would race. Only meaningful with `hasRemote`; a layer that
|
|
802
|
+
* uploads synchronously (or not at all) omits it.
|
|
803
|
+
*/
|
|
804
|
+
drainUploads?(): Promise<void>
|
|
539
805
|
key(input: CacheKeyInput): Promise<string>
|
|
540
806
|
get(hash: string, ctx?: CacheGetContext): Promise<CacheEntry | null>
|
|
807
|
+
/**
|
|
808
|
+
* Optional batched `get` (same answers, fewer round trips). The
|
|
809
|
+
* short-circuit probe uses it when a layer offers one; a layer without it
|
|
810
|
+
* is probed hash by hash.
|
|
811
|
+
*/
|
|
812
|
+
getMany?(hashes: readonly string[]): Promise<Map<string, CacheEntry>>
|
|
541
813
|
/**
|
|
542
814
|
* Lightweight existence probe. `'local'` / `'remote'` names the layer
|
|
543
815
|
* that holds the artifact; `null` is a miss. NEVER moves bytes: no
|
|
@@ -583,6 +855,19 @@ export interface CacheLayer {
|
|
|
583
855
|
* restore.
|
|
584
856
|
*/
|
|
585
857
|
isOutputsCurrent(projectDir: string, expected: readonly OutputFileRow[]): Promise<boolean>
|
|
858
|
+
/**
|
|
859
|
+
* The directory-mtime short-circuit behind a warm hit (optional — a layer
|
|
860
|
+
* without it keeps the output walk). `recordOutputDirs` snapshots every
|
|
861
|
+
* directory under each of `prefixes` (project-relative, whole-subtree
|
|
862
|
+
* globs only — see `wholeSubtreePrefixes`) after a save or restore;
|
|
863
|
+
* `loadOutputDirsBatch` reads them back; `outputDirsCurrent` is true iff
|
|
864
|
+
* every recorded directory still carries its recorded mtime, which proves
|
|
865
|
+
* no file was added or removed anywhere the glob could see. Machine-local
|
|
866
|
+
* state, like the output rows: a remote ingest records none.
|
|
867
|
+
*/
|
|
868
|
+
recordOutputDirs?(hash: string, projectDir: string, prefixes: readonly string[]): Promise<void>
|
|
869
|
+
loadOutputDirsBatch?(hashes: readonly string[]): Map<string, OutputDirRow[]>
|
|
870
|
+
outputDirsCurrent?(projectDir: string, rows: readonly OutputDirRow[]): Promise<boolean>
|
|
586
871
|
/**
|
|
587
872
|
* Extract the artifact's `outputs/` entries into `projectDir` and —
|
|
588
873
|
* when `workspaceRoot` is given — its `workspace-outputs/` entries
|
|
@@ -592,9 +877,25 @@ export interface CacheLayer {
|
|
|
592
877
|
restoreOutputs(hash: string, projectDir: string, workspaceRoot?: string): Promise<void>
|
|
593
878
|
save(args: {
|
|
594
879
|
hash: string
|
|
595
|
-
|
|
880
|
+
/**
|
|
881
|
+
* `exitCode` is deliberately NOT accepted: vx caches only successes, so
|
|
882
|
+
* the stored value is pinned to 0 and there is nothing for a caller to
|
|
883
|
+
* decide. Accepting a number and discarding it invited the one shape that
|
|
884
|
+
* launders a failure into a success — cache a failing task's outputs, read
|
|
885
|
+
* the entry back as exit 0, and the hit classifies `cache-hit` while the
|
|
886
|
+
* broken build's files are restored over a good tree. Unrepresentable
|
|
887
|
+
* beats guarded.
|
|
888
|
+
*/
|
|
889
|
+
entry: Omit<CacheEntry, 'hash' | 'storedAt' | 'outputFiles' | 'exitCode'>
|
|
596
890
|
projectDir: string
|
|
597
891
|
outputFiles: string[]
|
|
892
|
+
/**
|
|
893
|
+
* Set only by ChainedCache when an EARLIER layer in the chain already
|
|
894
|
+
* wrote this artifact to the same local handle: the local pack + write
|
|
895
|
+
* is skipped and only the layer's remote side acts. A layer with no
|
|
896
|
+
* remote side treats it as a full no-op.
|
|
897
|
+
*/
|
|
898
|
+
skipLocalWrite?: boolean
|
|
598
899
|
/**
|
|
599
900
|
* Resolved `outputs.workspaceFiles` (absolute paths) + the root
|
|
600
901
|
* they're relative to. Packed under `workspace-outputs/<rel>`.
|
|
@@ -636,7 +937,7 @@ export interface CacheLayer {
|
|
|
636
937
|
* all-cache-hit run writes nothing.
|
|
637
938
|
*/
|
|
638
939
|
recordRunBundle(bundle: { runs: readonly RunRecord[]; invocation: InvocationRecord }): void
|
|
639
|
-
stats(): CacheStats
|
|
940
|
+
stats(opts?: CacheStatsOptions): CacheStats
|
|
640
941
|
/**
|
|
641
942
|
* Content-hash a file with an mtime+size fast path. If the
|
|
642
943
|
* `(mtime_ms, size_bytes)` of `filePath` match a previously seen
|
|
@@ -680,7 +981,25 @@ interface EntryRow {
|
|
|
680
981
|
accessed_at: number
|
|
681
982
|
}
|
|
682
983
|
|
|
984
|
+
function entryOf(row: EntryRow, fileRows: OutputFileRow[]): CacheEntry {
|
|
985
|
+
return {
|
|
986
|
+
hash: row.hash,
|
|
987
|
+
taskId: `${row.project}#${row.task}`,
|
|
988
|
+
command: row.command,
|
|
989
|
+
exitCode: row.exit_code,
|
|
990
|
+
durationMs: row.duration_ms,
|
|
991
|
+
outputFiles: fileRows.map((r) => r.path),
|
|
992
|
+
outputRows: fileRows,
|
|
993
|
+
stdout: row.stdout,
|
|
994
|
+
storedAt: new Date(row.created_at).toISOString(),
|
|
995
|
+
source: 'local',
|
|
996
|
+
}
|
|
997
|
+
}
|
|
998
|
+
|
|
683
999
|
export class Cache implements CacheLayer {
|
|
1000
|
+
/** This IS the local layer — there is nothing slower behind it. */
|
|
1001
|
+
readonly hasRemote = false
|
|
1002
|
+
|
|
684
1003
|
private readonly db: Database
|
|
685
1004
|
private readonly insertEntry: ReturnType<Database['prepare']>
|
|
686
1005
|
private readonly selectEntry: ReturnType<Database['prepare']>
|
|
@@ -692,7 +1011,11 @@ export class Cache implements CacheLayer {
|
|
|
692
1011
|
private readonly selectFileHash: ReturnType<Database['prepare']>
|
|
693
1012
|
private readonly upsertFileHash: ReturnType<Database['prepare']>
|
|
694
1013
|
private readonly insertOutputFile: ReturnType<Database['prepare']>
|
|
1014
|
+
private readonly insertOutputDir: ReturnType<Database['prepare']>
|
|
1015
|
+
private readonly deleteOutputDirs: ReturnType<Database['prepare']>
|
|
695
1016
|
private readonly deleteOutputFiles: ReturnType<Database['prepare']>
|
|
1017
|
+
private readonly selectConfigEval: ReturnType<Database['prepare']>
|
|
1018
|
+
private readonly insertConfigEval: ReturnType<Database['prepare']>
|
|
696
1019
|
/** Memoized repo object format for blob-OID hashing (lazy-detected). */
|
|
697
1020
|
private objectFormat: 'sha1' | 'sha256' | null = null
|
|
698
1021
|
|
|
@@ -717,6 +1040,15 @@ export class Cache implements CacheLayer {
|
|
|
717
1040
|
// because callers use `new Cache(...)` directly; `mkdirSync` keeps
|
|
718
1041
|
// that property without a subprocess fork.
|
|
719
1042
|
mkdirSync(cacheDir, { recursive: true })
|
|
1043
|
+
// Make the cache dir invisible to git, every time it is created: a
|
|
1044
|
+
// `*` .gitignore inside it (the Cargo / Nx convention). Two reasons,
|
|
1045
|
+
// both measured. A cache nobody ignored gets COMMITTED by the next
|
|
1046
|
+
// `git add -A`; and vx's own `git status -uall` walks it — 1000
|
|
1047
|
+
// artifacts doubled the enumeration on the bench workspace before
|
|
1048
|
+
// its generator ignored `.vx`. An ignored directory is skipped by the
|
|
1049
|
+
// walk entirely. Only written when absent, so a user's own file wins.
|
|
1050
|
+
const ignore = path.join(cacheDir, '.gitignore')
|
|
1051
|
+
if (!existsSync(ignore)) writeFileSync(ignore, '*\n')
|
|
720
1052
|
this.db = new Database(path.join(cacheDir, 'cache.db'), { create: true })
|
|
721
1053
|
this.db.exec('PRAGMA journal_mode = WAL')
|
|
722
1054
|
this.db.exec('PRAGMA synchronous = NORMAL')
|
|
@@ -744,7 +1076,7 @@ export class Cache implements CacheLayer {
|
|
|
744
1076
|
| undefined
|
|
745
1077
|
if (meta && meta.value !== SCHEMA_VERSION) {
|
|
746
1078
|
this.db.exec(
|
|
747
|
-
'DROP TABLE IF EXISTS entries; DROP TABLE IF EXISTS runs; DROP TABLE IF EXISTS file_hashes; DROP TABLE IF EXISTS output_files; DROP TABLE IF EXISTS invocations; DROP TABLE IF EXISTS run_task_inputs; DROP TABLE IF EXISTS entry_inputs;',
|
|
1079
|
+
'DROP TABLE IF EXISTS entries; DROP TABLE IF EXISTS runs; DROP TABLE IF EXISTS file_hashes; DROP TABLE IF EXISTS output_files; DROP TABLE IF EXISTS invocations; DROP TABLE IF EXISTS run_task_inputs; DROP TABLE IF EXISTS entry_inputs; DROP TABLE IF EXISTS config_evals;',
|
|
748
1080
|
)
|
|
749
1081
|
this.db.prepare("UPDATE schema_meta SET value = ? WHERE key = 'version'").run(SCHEMA_VERSION)
|
|
750
1082
|
} else if (!meta) {
|
|
@@ -753,6 +1085,17 @@ export class Cache implements CacheLayer {
|
|
|
753
1085
|
.run(SCHEMA_VERSION)
|
|
754
1086
|
}
|
|
755
1087
|
|
|
1088
|
+
// Cached config evaluations (workspace/config-cache.ts): the validated
|
|
1089
|
+
// config as JSON, keyed by everything the evaluation could observe. A
|
|
1090
|
+
// separate exec so the artifact schema above stays byte-identical.
|
|
1091
|
+
this.db.exec(`
|
|
1092
|
+
CREATE TABLE IF NOT EXISTS config_evals (
|
|
1093
|
+
key TEXT PRIMARY KEY,
|
|
1094
|
+
json TEXT NOT NULL,
|
|
1095
|
+
created_at INTEGER NOT NULL
|
|
1096
|
+
);
|
|
1097
|
+
`)
|
|
1098
|
+
|
|
756
1099
|
this.db.exec(`
|
|
757
1100
|
-- stdout/stderr live in the <hash>.tar.zst artifact, not here
|
|
758
1101
|
-- (v14+) — so they survive remote round-trips. The entries
|
|
@@ -772,6 +1115,8 @@ export class Cache implements CacheLayer {
|
|
|
772
1115
|
);
|
|
773
1116
|
CREATE TABLE IF NOT EXISTS runs (
|
|
774
1117
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1118
|
+
-- '' when the outcome derived no cache key (skipped / persistent).
|
|
1119
|
+
-- Every reader that must not mistake it for a key guards hash != ''.
|
|
775
1120
|
hash TEXT NOT NULL,
|
|
776
1121
|
project TEXT NOT NULL,
|
|
777
1122
|
task TEXT NOT NULL,
|
|
@@ -794,7 +1139,12 @@ export class Cache implements CacheLayer {
|
|
|
794
1139
|
-- The direct within-run flaky signal.
|
|
795
1140
|
attempts INTEGER
|
|
796
1141
|
);
|
|
797
|
-
|
|
1142
|
+
-- runs_hash had no reader: every consumer of runs.hash looks the row
|
|
1143
|
+
-- up by run_id or (project, task) first. Dropped 2026-09-03 — it cost
|
|
1144
|
+
-- 1,000 warm-run inserts 11.5 → 3.9 ms; the DROP sheds it from
|
|
1145
|
+
-- existing databases (a schema-meta bump is for stored shapes, and
|
|
1146
|
+
-- an index is not one).
|
|
1147
|
+
DROP INDEX IF EXISTS runs_hash;
|
|
798
1148
|
CREATE INDEX IF NOT EXISTS runs_started_at ON runs(started_at);
|
|
799
1149
|
CREATE INDEX IF NOT EXISTS runs_project ON runs(project, task);
|
|
800
1150
|
CREATE INDEX IF NOT EXISTS runs_ended ON runs(ended_at);
|
|
@@ -808,6 +1158,8 @@ export class Cache implements CacheLayer {
|
|
|
808
1158
|
path TEXT PRIMARY KEY,
|
|
809
1159
|
mtime_ms INTEGER NOT NULL,
|
|
810
1160
|
size_bytes INTEGER NOT NULL,
|
|
1161
|
+
ctime_ms INTEGER NOT NULL,
|
|
1162
|
+
ino INTEGER NOT NULL,
|
|
811
1163
|
content_hash TEXT NOT NULL,
|
|
812
1164
|
seen_at INTEGER NOT NULL
|
|
813
1165
|
);
|
|
@@ -829,6 +1181,27 @@ export class Cache implements CacheLayer {
|
|
|
829
1181
|
PRIMARY KEY (entry_hash, path),
|
|
830
1182
|
FOREIGN KEY (entry_hash) REFERENCES entries(hash) ON DELETE CASCADE
|
|
831
1183
|
);
|
|
1184
|
+
-- Every directory under a whole-subtree output glob, with its mtime
|
|
1185
|
+
-- as of the last save/restore on THIS machine (2026-09-03). On a warm
|
|
1186
|
+
-- hit, unchanged mtimes prove the output SET is unchanged, replacing
|
|
1187
|
+
-- the glob walk that cost 0.36 ms per hit. Machine-local: a remote
|
|
1188
|
+
-- ingest writes none, and the first hit after it walks and records.
|
|
1189
|
+
-- Each config's ORDERED import closure (the config first), so a warm
|
|
1190
|
+
-- load keys it by stat-hashing the list (the file_hashes memo) instead
|
|
1191
|
+
-- of reading and scanning every file. Machine-local; pruned with
|
|
1192
|
+
-- config_evals (2026-09-03).
|
|
1193
|
+
CREATE TABLE IF NOT EXISTS config_closures (
|
|
1194
|
+
config_path TEXT PRIMARY KEY,
|
|
1195
|
+
files_json TEXT NOT NULL,
|
|
1196
|
+
created_at INTEGER NOT NULL
|
|
1197
|
+
);
|
|
1198
|
+
CREATE TABLE IF NOT EXISTS output_dirs (
|
|
1199
|
+
entry_hash TEXT NOT NULL,
|
|
1200
|
+
path TEXT NOT NULL,
|
|
1201
|
+
mtime_ms INTEGER NOT NULL,
|
|
1202
|
+
PRIMARY KEY (entry_hash, path),
|
|
1203
|
+
FOREIGN KEY (entry_hash) REFERENCES entries(hash) ON DELETE CASCADE
|
|
1204
|
+
);
|
|
832
1205
|
-- v22 (Tier 3): one header row per vx-run invocation. The runs
|
|
833
1206
|
-- table is per-task; this is the per-invocation record that
|
|
834
1207
|
-- carries git/CI/host context, the command, tags, and run-level
|
|
@@ -928,14 +1301,16 @@ export class Cache implements CacheLayer {
|
|
|
928
1301
|
VALUES (?, ?, ?, ?)
|
|
929
1302
|
`)
|
|
930
1303
|
this.selectFileHash = this.db.prepare(
|
|
931
|
-
'SELECT mtime_ms, size_bytes, content_hash FROM file_hashes WHERE path = ?',
|
|
1304
|
+
'SELECT mtime_ms, size_bytes, ctime_ms, ino, content_hash FROM file_hashes WHERE path = ?',
|
|
932
1305
|
)
|
|
933
1306
|
this.upsertFileHash = this.db.prepare(`
|
|
934
|
-
INSERT INTO file_hashes(path, mtime_ms, size_bytes, content_hash, seen_at)
|
|
935
|
-
VALUES (?, ?, ?, ?, ?)
|
|
1307
|
+
INSERT INTO file_hashes(path, mtime_ms, size_bytes, ctime_ms, ino, content_hash, seen_at)
|
|
1308
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
936
1309
|
ON CONFLICT(path) DO UPDATE SET
|
|
937
1310
|
mtime_ms = excluded.mtime_ms,
|
|
938
1311
|
size_bytes = excluded.size_bytes,
|
|
1312
|
+
ctime_ms = excluded.ctime_ms,
|
|
1313
|
+
ino = excluded.ino,
|
|
939
1314
|
content_hash = excluded.content_hash,
|
|
940
1315
|
seen_at = excluded.seen_at
|
|
941
1316
|
`)
|
|
@@ -948,20 +1323,89 @@ export class Cache implements CacheLayer {
|
|
|
948
1323
|
mtime_ms = excluded.mtime_ms
|
|
949
1324
|
`)
|
|
950
1325
|
this.deleteOutputFiles = this.db.prepare('DELETE FROM output_files WHERE entry_hash = ?')
|
|
1326
|
+
this.insertOutputDir = this.db.prepare(
|
|
1327
|
+
'INSERT INTO output_dirs(entry_hash, path, mtime_ms) VALUES (?, ?, ?)',
|
|
1328
|
+
)
|
|
1329
|
+
this.deleteOutputDirs = this.db.prepare('DELETE FROM output_dirs WHERE entry_hash = ?')
|
|
1330
|
+
this.selectConfigEval = this.db.prepare('SELECT json FROM config_evals WHERE key = ?')
|
|
1331
|
+
this.insertConfigEval = this.db.prepare(
|
|
1332
|
+
'INSERT OR REPLACE INTO config_evals(key, json, created_at) VALUES (?, ?, ?)',
|
|
1333
|
+
)
|
|
1334
|
+
}
|
|
1335
|
+
|
|
1336
|
+
/** `ConfigEvalStore`: a cached config evaluation, honouring the local READ axis. */
|
|
1337
|
+
getConfigEval(key: string): string | null {
|
|
1338
|
+
if (!this.read) return null
|
|
1339
|
+
const row = this.selectConfigEval.get(key) as { json: string } | null
|
|
1340
|
+
return row?.json ?? null
|
|
1341
|
+
}
|
|
1342
|
+
|
|
1343
|
+
/** `ConfigEvalStore`: each config's ordered closure, one `IN` query per 900 paths. */
|
|
1344
|
+
getConfigClosures(configPaths: readonly string[]): Map<string, string[]> {
|
|
1345
|
+
const out = new Map<string, string[]>()
|
|
1346
|
+
if (!this.read || configPaths.length === 0) return out
|
|
1347
|
+
for (let i = 0; i < configPaths.length; i += 900) {
|
|
1348
|
+
const chunk = configPaths.slice(i, i + 900)
|
|
1349
|
+
const rows = this.db
|
|
1350
|
+
.query(
|
|
1351
|
+
`SELECT config_path, files_json FROM config_closures WHERE config_path IN (${chunk.map(() => '?').join(',')})`,
|
|
1352
|
+
)
|
|
1353
|
+
.all(...(chunk as readonly SQLQueryBindings[])) as Array<{
|
|
1354
|
+
config_path: string
|
|
1355
|
+
files_json: string
|
|
1356
|
+
}>
|
|
1357
|
+
for (const r of rows) out.set(r.config_path, JSON.parse(r.files_json) as string[])
|
|
1358
|
+
}
|
|
1359
|
+
return out
|
|
1360
|
+
}
|
|
1361
|
+
|
|
1362
|
+
/** `ConfigEvalStore`: remember a config's ordered closure, honouring the local WRITE axis. */
|
|
1363
|
+
putConfigClosure(configPath: string, files: readonly string[]): void {
|
|
1364
|
+
if (!this.write) return
|
|
1365
|
+
this.db
|
|
1366
|
+
.prepare(
|
|
1367
|
+
'INSERT INTO config_closures(config_path, files_json, created_at) VALUES (?, ?, ?) ON CONFLICT(config_path) DO UPDATE SET files_json = excluded.files_json, created_at = excluded.created_at',
|
|
1368
|
+
)
|
|
1369
|
+
.run(configPath, JSON.stringify(files), Date.now())
|
|
1370
|
+
}
|
|
1371
|
+
|
|
1372
|
+
/** `ConfigEvalStore`: the batched read — one `IN` query per 900 keys, honouring the local READ axis. */
|
|
1373
|
+
getConfigEvals(keys: readonly string[]): Map<string, string> {
|
|
1374
|
+
const out = new Map<string, string>()
|
|
1375
|
+
if (!this.read || keys.length === 0) return out
|
|
1376
|
+
for (let i = 0; i < keys.length; i += 900) {
|
|
1377
|
+
const chunk = keys.slice(i, i + 900)
|
|
1378
|
+
const rows = this.db
|
|
1379
|
+
.query(
|
|
1380
|
+
`SELECT key, json FROM config_evals WHERE key IN (${chunk.map(() => '?').join(',')})`,
|
|
1381
|
+
)
|
|
1382
|
+
.all(...(chunk as readonly SQLQueryBindings[])) as Array<{ key: string; json: string }>
|
|
1383
|
+
for (const r of rows) out.set(r.key, r.json)
|
|
1384
|
+
}
|
|
1385
|
+
return out
|
|
1386
|
+
}
|
|
1387
|
+
|
|
1388
|
+
/** `ConfigEvalStore`: remember a validated evaluation, honouring the local WRITE axis. */
|
|
1389
|
+
putConfigEval(key: string, json: string): void {
|
|
1390
|
+
if (!this.write) return
|
|
1391
|
+
this.insertConfigEval.run(key, json, Date.now())
|
|
951
1392
|
}
|
|
952
1393
|
|
|
953
1394
|
/**
|
|
954
1395
|
* Content-hash a file (as a git blob OID, v20) with an mtime+size
|
|
955
1396
|
* fast path. If the `file_hashes` table has a row for `path` whose
|
|
956
|
-
* `(mtime_ms, size_bytes)` match the current
|
|
957
|
-
* stored content_hash (a memory + SQLite lookup,
|
|
958
|
-
* Otherwise we read + hash + upsert.
|
|
1397
|
+
* `(mtime_ms, size_bytes, ctime_ms, ino)` all match the current
|
|
1398
|
+
* stat, we reuse the stored content_hash (a memory + SQLite lookup,
|
|
1399
|
+
* no disk read). Otherwise we read + hash + upsert. All four fields
|
|
1400
|
+
* are load-bearing — see the comment at the comparison.
|
|
959
1401
|
*
|
|
960
1402
|
* The OID is byte-identical to what `git hash-object` (and the git
|
|
961
1403
|
* index) computes for the same content, so this fallback and the
|
|
962
|
-
* `CacheKeyInput.fileHashes` index-OID fast path
|
|
963
|
-
*
|
|
964
|
-
*
|
|
1404
|
+
* `CacheKeyInput.fileHashes` index-OID fast path agree on any file
|
|
1405
|
+
* git stores verbatim. They do NOT agree when a clean filter
|
|
1406
|
+
* (`text`/`eol`/`ident`) is active: the index blob is the filtered
|
|
1407
|
+
* form while this hashes the worktree bytes, so `inputs.ts` drops
|
|
1408
|
+
* the index OID for those paths and routes them here.
|
|
965
1409
|
*/
|
|
966
1410
|
async hashFile(filePath: string): Promise<string> {
|
|
967
1411
|
// statSync intentional: a single stat is ~1.6µs (Bun 1.3); the
|
|
@@ -980,14 +1424,36 @@ export class Cache implements CacheLayer {
|
|
|
980
1424
|
}
|
|
981
1425
|
const mtimeMs = Math.floor(st.mtimeMs)
|
|
982
1426
|
const size = st.size
|
|
1427
|
+
// ctime + ino are what make this memo SAFE, not merely fast. mtime is
|
|
1428
|
+
// caller-settable, so (mtime, size) alone hands back the previous run's
|
|
1429
|
+
// digest for genuinely different bytes whenever a producer preserves
|
|
1430
|
+
// mtime — `tar -x`, `unzip`, `cp -p`, `rsync --times`, any
|
|
1431
|
+
// SOURCE_DATE_EPOCH generator — which is a stale cache hit. `utimes`
|
|
1432
|
+
// cannot suppress ctime without root, and an atomic write-then-rename
|
|
1433
|
+
// changes the inode; git's own index keys on ctime+ino+dev for exactly
|
|
1434
|
+
// this reason. Both fields come from the stat we already took.
|
|
1435
|
+
const ctimeMs = Math.floor(st.ctimeMs)
|
|
1436
|
+
const ino = Number(st.ino)
|
|
983
1437
|
const row = this.selectFileHash.get(filePath) as
|
|
984
|
-
| {
|
|
1438
|
+
| {
|
|
1439
|
+
mtime_ms: number
|
|
1440
|
+
size_bytes: number
|
|
1441
|
+
ctime_ms: number
|
|
1442
|
+
ino: number
|
|
1443
|
+
content_hash: string
|
|
1444
|
+
}
|
|
985
1445
|
| undefined
|
|
986
|
-
if (
|
|
1446
|
+
if (
|
|
1447
|
+
row &&
|
|
1448
|
+
row.mtime_ms === mtimeMs &&
|
|
1449
|
+
row.size_bytes === size &&
|
|
1450
|
+
row.ctime_ms === ctimeMs &&
|
|
1451
|
+
row.ino === ino
|
|
1452
|
+
) {
|
|
987
1453
|
return row.content_hash
|
|
988
1454
|
}
|
|
989
1455
|
const ch = await this.hashFileFromDisk(filePath)
|
|
990
|
-
this.upsertFileHash.run(filePath, mtimeMs, size, ch, Date.now())
|
|
1456
|
+
this.upsertFileHash.run(filePath, mtimeMs, size, ctimeMs, ino, ch, Date.now())
|
|
991
1457
|
return ch
|
|
992
1458
|
}
|
|
993
1459
|
|
|
@@ -1098,14 +1564,27 @@ export class Cache implements CacheLayer {
|
|
|
1098
1564
|
}
|
|
1099
1565
|
}
|
|
1100
1566
|
|
|
1567
|
+
const pluginParts = input.pluginParts ?? []
|
|
1568
|
+
if (pluginParts.length > 0) {
|
|
1569
|
+
h = xxh3(`plugin:${pluginParts.length}`, h)
|
|
1570
|
+
for (const [n, v] of pluginParts) h = xxh3(`${n}\0${v}`, h)
|
|
1571
|
+
if (cap) {
|
|
1572
|
+
for (const [n, v] of pluginParts) cap.push({ kind: 'plugin', name: n, hash: xxh3hex(v) })
|
|
1573
|
+
}
|
|
1574
|
+
}
|
|
1575
|
+
|
|
1101
1576
|
const sortedInputs = [...input.inputFiles].sort()
|
|
1102
1577
|
h = xxh3(`inputs:${sortedInputs.length}`, h)
|
|
1103
1578
|
// Per-file hash source, in preference order: the caller-supplied
|
|
1104
1579
|
// index-OID map (clean tracked files — zero I/O), then hashFile's
|
|
1105
|
-
//
|
|
1106
|
-
//
|
|
1107
|
-
//
|
|
1108
|
-
//
|
|
1580
|
+
// stat memo (no read), then a full in-process blob-OID computation.
|
|
1581
|
+
// All three describe the WORKTREE bytes, which is what the task
|
|
1582
|
+
// reads — the index-OID map is only populated for paths whose
|
|
1583
|
+
// worktree form matches their index form, because a clean filter
|
|
1584
|
+
// (`text`/`eol`/`ident`) makes the index blob a DIFFERENT sequence
|
|
1585
|
+
// of bytes and folding it would let two distinct worktree contents
|
|
1586
|
+
// share a key. The fold order is locked to `sortedInputs` so
|
|
1587
|
+
// results are stable across runs.
|
|
1109
1588
|
const fileHashes = await Promise.all(
|
|
1110
1589
|
sortedInputs.map((f) => input.fileHashes?.get(f) ?? this.hashFile(f)),
|
|
1111
1590
|
)
|
|
@@ -1128,6 +1607,23 @@ export class Cache implements CacheLayer {
|
|
|
1128
1607
|
// Local reads disabled (e.g. `--force` / `--cache=local:w`): report a
|
|
1129
1608
|
// miss so the task re-executes. The artifact + index are untouched.
|
|
1130
1609
|
if (!this.read) return null
|
|
1610
|
+
return await this.readEntry(hash)
|
|
1611
|
+
}
|
|
1612
|
+
|
|
1613
|
+
/**
|
|
1614
|
+
* Read an entry BYPASSING the local read gate. The gate means "don't
|
|
1615
|
+
* serve hits out of the PRE-EXISTING local cache" — it must not throw
|
|
1616
|
+
* away bytes this run just downloaded. `LayeredCache` calls this to
|
|
1617
|
+
* deliver an artifact it ingested from the remote; under
|
|
1618
|
+
* `--cache=local:,remote:rw` the gated `get` returned null for the row
|
|
1619
|
+
* `ingest` had just written, so the task re-executed and re-uploaded on
|
|
1620
|
+
* every single run.
|
|
1621
|
+
*/
|
|
1622
|
+
async getIngested(hash: string): Promise<CacheEntry | null> {
|
|
1623
|
+
return await this.readEntry(hash)
|
|
1624
|
+
}
|
|
1625
|
+
|
|
1626
|
+
private async readEntry(hash: string): Promise<CacheEntry | null> {
|
|
1131
1627
|
const row = this.selectEntry.get(hash) as EntryRow | undefined
|
|
1132
1628
|
if (!row) return null
|
|
1133
1629
|
|
|
@@ -1147,18 +1643,45 @@ export class Cache implements CacheLayer {
|
|
|
1147
1643
|
// for four up-to-date hits on ~70 MB binaries). restoreOutputs
|
|
1148
1644
|
// reads the artifact itself, only when extraction actually runs.
|
|
1149
1645
|
const fileRows = this.loadOutputFilesBatch([hash]).get(hash) ?? []
|
|
1646
|
+
const entry = entryOf(row, fileRows)
|
|
1647
|
+
entry.outputDirRows = this.loadOutputDirsBatch([hash]).get(hash) ?? []
|
|
1648
|
+
return entry
|
|
1649
|
+
}
|
|
1150
1650
|
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1651
|
+
/**
|
|
1652
|
+
* `get` for many hashes at once: one `entries` query and one
|
|
1653
|
+
* `output_files` query per chunk instead of two per hash, with the
|
|
1654
|
+
* artifact-existence stats in flight together. Same answers as N calls
|
|
1655
|
+
* to `get` — a hash whose artifact is gone is simply absent from the map.
|
|
1656
|
+
* The up-front short-circuit probe is the caller; per-hit `get` stays for
|
|
1657
|
+
* the lazy path.
|
|
1658
|
+
*/
|
|
1659
|
+
async getMany(hashes: readonly string[]): Promise<Map<string, CacheEntry>> {
|
|
1660
|
+
const out = new Map<string, CacheEntry>()
|
|
1661
|
+
if (!this.read || hashes.length === 0) return out
|
|
1662
|
+
const rows: EntryRow[] = []
|
|
1663
|
+
for (let i = 0; i < hashes.length; i += 900) {
|
|
1664
|
+
const chunk = hashes.slice(i, i + 900)
|
|
1665
|
+
const placeholders = chunk.map(() => '?').join(',')
|
|
1666
|
+
rows.push(
|
|
1667
|
+
...(this.db
|
|
1668
|
+
.query(`SELECT * FROM entries WHERE hash IN (${placeholders})`)
|
|
1669
|
+
.all(...(chunk as readonly SQLQueryBindings[])) as EntryRow[]),
|
|
1670
|
+
)
|
|
1671
|
+
}
|
|
1672
|
+
if (rows.length === 0) return out
|
|
1673
|
+
const present = await Promise.all(rows.map((r) => Bun.file(this.tarPath(r.hash)).exists()))
|
|
1674
|
+
const live = rows.filter((_r, i) => present[i])
|
|
1675
|
+
const liveHashes = live.map((r) => r.hash)
|
|
1676
|
+
const fileRows = this.loadOutputFilesBatch(liveHashes)
|
|
1677
|
+
const dirRows = this.loadOutputDirsBatch(liveHashes)
|
|
1678
|
+
for (const row of live) {
|
|
1679
|
+
this.touched.add(row.hash)
|
|
1680
|
+
const entry = entryOf(row, fileRows.get(row.hash) ?? [])
|
|
1681
|
+
entry.outputDirRows = dirRows.get(row.hash) ?? []
|
|
1682
|
+
out.set(row.hash, entry)
|
|
1161
1683
|
}
|
|
1684
|
+
return out
|
|
1162
1685
|
}
|
|
1163
1686
|
|
|
1164
1687
|
// Existence probe: SQL row + artifact-on-disk check, no byte reads
|
|
@@ -1213,6 +1736,10 @@ export class Cache implements CacheLayer {
|
|
|
1213
1736
|
// current; the on-disk tree under projectDir is whatever it was,
|
|
1214
1737
|
// and nothing was supposed to land there.
|
|
1215
1738
|
if (expected.length === 0) return true
|
|
1739
|
+
// Promise-form stat, deliberately: statSync is ~2 µs against ~13 µs in
|
|
1740
|
+
// isolation, but this runs under the scheduler's concurrency and the
|
|
1741
|
+
// async form keeps the stats on the thread pool in parallel — the sync
|
|
1742
|
+
// version measured slower on a 1000-hit warm run (2026-09-02).
|
|
1216
1743
|
const results = await Promise.all(
|
|
1217
1744
|
expected.map(async (e) => {
|
|
1218
1745
|
try {
|
|
@@ -1243,8 +1770,101 @@ export class Cache implements CacheLayer {
|
|
|
1243
1770
|
return this.tarPath(hash)
|
|
1244
1771
|
}
|
|
1245
1772
|
|
|
1773
|
+
/**
|
|
1774
|
+
* Snapshot every directory under each of `prefixes` for `hash`. Called
|
|
1775
|
+
* after a save and after a restore, when the tree is known to equal the
|
|
1776
|
+
* entry's set. Symlinked directories are not descended (the output walk
|
|
1777
|
+
* refuses them too). Over `OUTPUT_DIRS_CAP` directories, or on any
|
|
1778
|
+
* error, the rows are cleared and the next hit keeps the walk.
|
|
1779
|
+
*/
|
|
1780
|
+
async recordOutputDirs(
|
|
1781
|
+
hash: string,
|
|
1782
|
+
projectDir: string,
|
|
1783
|
+
prefixes: readonly string[],
|
|
1784
|
+
): Promise<void> {
|
|
1785
|
+
const rows: Array<[string, number]> = []
|
|
1786
|
+
const walk = async (rel: string): Promise<boolean> => {
|
|
1787
|
+
const abs = path.join(projectDir, rel)
|
|
1788
|
+
let st
|
|
1789
|
+
try {
|
|
1790
|
+
st = await lstat(abs)
|
|
1791
|
+
} catch {
|
|
1792
|
+
return false
|
|
1793
|
+
}
|
|
1794
|
+
if (!st.isDirectory()) return false
|
|
1795
|
+
rows.push([rel, st.mtimeMs])
|
|
1796
|
+
if (rows.length > OUTPUT_DIRS_CAP) return false
|
|
1797
|
+
let entries
|
|
1798
|
+
try {
|
|
1799
|
+
entries = await readdir(abs, { withFileTypes: true })
|
|
1800
|
+
} catch {
|
|
1801
|
+
return false
|
|
1802
|
+
}
|
|
1803
|
+
for (const e of entries) {
|
|
1804
|
+
if (e.isDirectory() && !e.isSymbolicLink()) {
|
|
1805
|
+
if (!(await walk(`${rel}/${e.name}`))) return false
|
|
1806
|
+
}
|
|
1807
|
+
}
|
|
1808
|
+
return true
|
|
1809
|
+
}
|
|
1810
|
+
let ok = true
|
|
1811
|
+
for (const prefix of prefixes) {
|
|
1812
|
+
if (!(await walk(prefix))) {
|
|
1813
|
+
ok = false
|
|
1814
|
+
break
|
|
1815
|
+
}
|
|
1816
|
+
}
|
|
1817
|
+
// All or nothing: a racy directory dropped alone would leave its
|
|
1818
|
+
// parent trusted while an addition inside it bumps only the dropped one.
|
|
1819
|
+
const youngest = Date.now() - OUTPUT_DIRS_RACY_MS
|
|
1820
|
+
if (rows.some(([, mtime]) => mtime > youngest)) ok = false
|
|
1821
|
+
this.db.transaction(() => {
|
|
1822
|
+
this.deleteOutputDirs.run(hash)
|
|
1823
|
+
if (!ok) return
|
|
1824
|
+
for (const [rel, mtime] of rows) this.insertOutputDir.run(hash, rel, mtime)
|
|
1825
|
+
})()
|
|
1826
|
+
}
|
|
1827
|
+
|
|
1828
|
+
loadOutputDirsBatch(hashes: readonly string[]): Map<string, OutputDirRow[]> {
|
|
1829
|
+
const out = new Map<string, OutputDirRow[]>()
|
|
1830
|
+
if (hashes.length === 0) return out
|
|
1831
|
+
const placeholders = hashes.map(() => '?').join(',')
|
|
1832
|
+
const rows = this.db
|
|
1833
|
+
.query(
|
|
1834
|
+
`SELECT entry_hash, path, mtime_ms FROM output_dirs WHERE entry_hash IN (${placeholders})`,
|
|
1835
|
+
)
|
|
1836
|
+
.all(...(hashes as readonly SQLQueryBindings[])) as Array<{
|
|
1837
|
+
entry_hash: string
|
|
1838
|
+
path: string
|
|
1839
|
+
mtime_ms: number
|
|
1840
|
+
}>
|
|
1841
|
+
for (const r of rows) {
|
|
1842
|
+
const list = out.get(r.entry_hash)
|
|
1843
|
+
const row = { path: r.path, mtimeMs: r.mtime_ms }
|
|
1844
|
+
if (list) list.push(row)
|
|
1845
|
+
else out.set(r.entry_hash, [row])
|
|
1846
|
+
}
|
|
1847
|
+
return out
|
|
1848
|
+
}
|
|
1849
|
+
|
|
1850
|
+
/** True iff every recorded directory exists with its recorded mtime (ms). Same forged-mtime trade as the file check. */
|
|
1851
|
+
async outputDirsCurrent(projectDir: string, rows: readonly OutputDirRow[]): Promise<boolean> {
|
|
1852
|
+
if (rows.length === 0) return false
|
|
1853
|
+
const results = await Promise.all(
|
|
1854
|
+
rows.map(async (r) => {
|
|
1855
|
+
try {
|
|
1856
|
+
const st = await stat(path.join(projectDir, r.path))
|
|
1857
|
+
return st.isDirectory() && Math.abs(st.mtimeMs - r.mtimeMs) < 1
|
|
1858
|
+
} catch {
|
|
1859
|
+
return false
|
|
1860
|
+
}
|
|
1861
|
+
}),
|
|
1862
|
+
)
|
|
1863
|
+
return results.every(Boolean)
|
|
1864
|
+
}
|
|
1865
|
+
|
|
1246
1866
|
async restoreOutputs(hash: string, projectDir: string, workspaceRoot?: string): Promise<void> {
|
|
1247
|
-
// In-process
|
|
1867
|
+
// In-process extraction — no fork+exec on the hot path
|
|
1248
1868
|
// (~5-10ms reclaimed per hit vs the prior subprocess `tar -xf`).
|
|
1249
1869
|
//
|
|
1250
1870
|
// The "tree is already current" skip-everything check happens at
|
|
@@ -1256,52 +1876,60 @@ export class Cache implements CacheLayer {
|
|
|
1256
1876
|
// path — they're surfaced via `get()` for the orchestrator to
|
|
1257
1877
|
// replay through the logger.
|
|
1258
1878
|
const src = this.tarPath(hash)
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
//
|
|
1262
|
-
//
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1879
|
+
// The caller already committed to this hit and WIPED the declared
|
|
1880
|
+
// outputs, so returning quietly here would report a green cache hit over
|
|
1881
|
+
// an emptied output tree. The artifact existed when `get()` probed it, so
|
|
1882
|
+
// its absence now means something removed it underneath us (a concurrent
|
|
1883
|
+
// `vx cache prune` is the documented way) — fail loud; the task re-runs.
|
|
1884
|
+
if (!(await Bun.file(src).exists())) {
|
|
1885
|
+
throw new CorruptArtifactError(hash, 'artifact file vanished before restore')
|
|
1886
|
+
}
|
|
1887
|
+
// The index says exactly which files this entry materializes. If the
|
|
1888
|
+
// archive cannot produce one of them, restoring "successfully" leaves a
|
|
1889
|
+
// hole that no later run detects: the skip-restore check compares the
|
|
1890
|
+
// same truncated expectation against the same truncated tree and agrees
|
|
1891
|
+
// forever. Refuse instead of silently under-restoring — checked before
|
|
1892
|
+
// anything is renamed into place.
|
|
1893
|
+
const rows = this.loadOutputFilesBatch([hash]).get(hash) ?? []
|
|
1894
|
+
const expected = rows
|
|
1895
|
+
.filter((r) => workspaceRoot !== undefined || !r.path.startsWith(WORKSPACE_OUTPUT_PREFIX))
|
|
1896
|
+
.map((r) => (r.path.startsWith(WORKSPACE_OUTPUT_PREFIX) ? r.path : `outputs/${r.path}`))
|
|
1897
|
+
const verify = (provided: ReadonlySet<string>): void => {
|
|
1898
|
+
const missing = expected.filter((name) => !provided.has(name))
|
|
1899
|
+
if (missing.length > 0) {
|
|
1900
|
+
throw new CorruptArtifactError(
|
|
1901
|
+
hash,
|
|
1902
|
+
`artifact is missing ${missing.length} recorded output(s): ${missing.slice(0, 3).join(', ')}`,
|
|
1903
|
+
)
|
|
1904
|
+
}
|
|
1272
1905
|
}
|
|
1273
1906
|
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
//
|
|
1277
|
-
//
|
|
1278
|
-
//
|
|
1279
|
-
//
|
|
1280
|
-
//
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
const t = r.mtimeMs / 1000
|
|
1290
|
-
await utimes(abs, t, t)
|
|
1291
|
-
} catch {
|
|
1292
|
-
// File absent from this extract (or unwritable) — the next
|
|
1293
|
-
// probe simply restores again; never fail a hit over mtimes.
|
|
1294
|
-
}
|
|
1295
|
-
}),
|
|
1296
|
-
)
|
|
1907
|
+
// One tar reader and one extractor either way; only the decode differs
|
|
1908
|
+
// by size. A large artifact is decoded and read as it streams, so the
|
|
1909
|
+
// restore costs a chunk of memory, not 4× the artifact (measured
|
|
1910
|
+
// 2026-09-03 on 150 MiB: +644 MiB in memory, +49 MiB streamed, same
|
|
1911
|
+
// wall time). A small one is decoded in one call: the stream setup
|
|
1912
|
+
// costs ~35 µs per artifact, which is 4% of the headline restore row
|
|
1913
|
+
// when every artifact is a one-file `dist/` (measured: 390 vs 355 ms
|
|
1914
|
+
// per 1 000). The local artifact was validated at ingest, so a missing
|
|
1915
|
+
// declared size is allowed; the output ceiling applies to both.
|
|
1916
|
+
try {
|
|
1917
|
+
const tar = await decodedTar(Bun.file(src), hash)
|
|
1918
|
+
await extractArtifactStream(tar, projectDir, workspaceRoot, verify)
|
|
1919
|
+
} catch (err) {
|
|
1920
|
+
if (err instanceof ArchiveSecurityError || err instanceof CorruptArtifactError) throw err
|
|
1921
|
+
throw new CorruptArtifactError(hash, 'artifact is not a readable archive', err)
|
|
1297
1922
|
}
|
|
1298
1923
|
}
|
|
1299
1924
|
|
|
1300
1925
|
async save(args: {
|
|
1301
1926
|
hash: string
|
|
1302
|
-
|
|
1927
|
+
/** See `CacheLayer.save` — `exitCode` is not a caller's to supply. */
|
|
1928
|
+
entry: Omit<CacheEntry, 'hash' | 'storedAt' | 'outputFiles' | 'exitCode'>
|
|
1303
1929
|
projectDir: string
|
|
1304
1930
|
outputFiles: string[]
|
|
1931
|
+
/** See `CacheLayer.save`. */
|
|
1932
|
+
skipLocalWrite?: boolean
|
|
1305
1933
|
workspaceOutputFiles?: string[]
|
|
1306
1934
|
workspaceRoot?: string
|
|
1307
1935
|
inputComponents?: readonly TaskInputRow[]
|
|
@@ -1324,23 +1952,17 @@ export class Cache implements CacheLayer {
|
|
|
1324
1952
|
// still uploads to remote — it calls `packArtifact` itself to get
|
|
1325
1953
|
// the bytes, since there's no local artifact to read off disk.
|
|
1326
1954
|
if (!this.write) return
|
|
1327
|
-
|
|
1328
|
-
await this.
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
},
|
|
1337
|
-
|
|
1338
|
-
// headers carry only seconds) — see the row loop for the guard.
|
|
1339
|
-
{
|
|
1340
|
-
projectDir: args.projectDir,
|
|
1341
|
-
...(args.workspaceRoot !== undefined ? { workspaceRoot: args.workspaceRoot } : {}),
|
|
1342
|
-
},
|
|
1343
|
-
)
|
|
1955
|
+
if (args.skipLocalWrite === true) return
|
|
1956
|
+
await mkdir(this.cacheDir, { recursive: true })
|
|
1957
|
+
const endPack = span('save: pack')
|
|
1958
|
+
const compressed = await this.packArtifactToTemp(this.tempPath(args.hash), args)
|
|
1959
|
+
endPack()
|
|
1960
|
+
await this.writeArtifactAndIndex(args.hash, compressed, {
|
|
1961
|
+
taskId: args.entry.taskId,
|
|
1962
|
+
command: args.entry.command,
|
|
1963
|
+
durationMs: args.entry.durationMs,
|
|
1964
|
+
...(args.inputComponents !== undefined ? { inputComponents: args.inputComponents } : {}),
|
|
1965
|
+
})
|
|
1344
1966
|
}
|
|
1345
1967
|
|
|
1346
1968
|
/**
|
|
@@ -1366,179 +1988,199 @@ export class Cache implements CacheLayer {
|
|
|
1366
1988
|
}
|
|
1367
1989
|
|
|
1368
1990
|
/**
|
|
1369
|
-
*
|
|
1370
|
-
* No disk write to the final cache path — that's the index
|
|
1371
|
-
* job. Pure transform, so `ingest()` can skip this and
|
|
1372
|
-
* remote-supplied bytes straight to `writeArtifactAndIndex`.
|
|
1991
|
+
* Collect stdout + outputs into artifact bytes, zstd-compress, return
|
|
1992
|
+
* them. No disk write to the final cache path — that's the index
|
|
1993
|
+
* step's job. Pure transform, so `ingest()` can skip this and hand
|
|
1994
|
+
* its remote-supplied bytes straight to `writeArtifactAndIndex`.
|
|
1995
|
+
*
|
|
1996
|
+
* Entries are named directly into the archive, so there is no staging
|
|
1997
|
+
* copy of every output byte and no `tar` subprocess (see
|
|
1998
|
+
* `archive.ts`).
|
|
1999
|
+
*/
|
|
2000
|
+
/** Archive name → absolute source path for every declared output. */
|
|
2001
|
+
private outputsOf(args: {
|
|
2002
|
+
projectDir: string
|
|
2003
|
+
outputFiles: string[]
|
|
2004
|
+
workspaceOutputFiles?: string[]
|
|
2005
|
+
workspaceRoot?: string
|
|
2006
|
+
}): Map<string, string> {
|
|
2007
|
+
const outputs = new Map<string, string>()
|
|
2008
|
+
for (const f of args.outputFiles) {
|
|
2009
|
+
outputs.set(`outputs/${path.relative(args.projectDir, f)}`, f)
|
|
2010
|
+
}
|
|
2011
|
+
// Caller passes workspaceRoot whenever workspaceOutputFiles is
|
|
2012
|
+
// non-empty; the rels are root-anchored by construction.
|
|
2013
|
+
for (const f of args.workspaceOutputFiles ?? []) {
|
|
2014
|
+
outputs.set(`${WORKSPACE_OUTPUT_PREFIX}${path.relative(args.workspaceRoot!, f)}`, f)
|
|
2015
|
+
}
|
|
2016
|
+
return outputs
|
|
2017
|
+
}
|
|
2018
|
+
|
|
2019
|
+
/**
|
|
2020
|
+
* The compressed artifact, in memory: the remote upload when local
|
|
2021
|
+
* writes are off. A large artifact is packed and compressed as a
|
|
2022
|
+
* stream and collected — the seam takes bytes.
|
|
1373
2023
|
*/
|
|
1374
2024
|
private async packArtifact(args: {
|
|
1375
|
-
|
|
1376
|
-
entry: Omit<CacheEntry, 'hash' | 'storedAt' | 'outputFiles'>
|
|
2025
|
+
entry: Omit<CacheEntry, 'hash' | 'storedAt' | 'outputFiles' | 'exitCode'>
|
|
1377
2026
|
projectDir: string
|
|
1378
2027
|
outputFiles: string[]
|
|
1379
2028
|
workspaceOutputFiles?: string[]
|
|
1380
2029
|
workspaceRoot?: string
|
|
1381
2030
|
}): Promise<Uint8Array> {
|
|
1382
|
-
|
|
1383
|
-
|
|
2031
|
+
// stdout is ALWAYS present in the artifact, even if empty, so the
|
|
2032
|
+
// layout is predictable: a successful read finds `stdout` and
|
|
2033
|
+
// zero-or-more `outputs/<rel>` / `workspace-outputs/<rel>` entries.
|
|
2034
|
+
const plan = await planArtifact({
|
|
2035
|
+
stdout: args.entry.stdout ?? '',
|
|
2036
|
+
outputs: this.outputsOf(args),
|
|
2037
|
+
})
|
|
2038
|
+
if (plan.size <= STREAM_DECODE_FROM)
|
|
2039
|
+
return await Bun.zstdCompress(await packArtifactBytes(plan))
|
|
2040
|
+
return await bytesOf(packArtifactStream(plan).pipeThrough(zstdEncoder()))
|
|
2041
|
+
}
|
|
2042
|
+
|
|
2043
|
+
/**
|
|
2044
|
+
* Pack for the local save. A large artifact is read from disk as it is
|
|
2045
|
+
* written and compressed as a stream straight into the temp, so it
|
|
2046
|
+
* never sits in memory (measured 2026-09-03 on a 150 MiB output: +705
|
|
2047
|
+
* MiB before). The streamed compressor is ~2.4× the one-call cost per
|
|
2048
|
+
* byte and a stream costs ~35 µs to set up, so a small artifact is
|
|
2049
|
+
* packed in memory and compressed in one call — returned as bytes,
|
|
2050
|
+
* nothing written yet.
|
|
2051
|
+
*/
|
|
2052
|
+
private async packArtifactToTemp(
|
|
2053
|
+
tmpPath: string,
|
|
2054
|
+
args: Parameters<Cache['packArtifact']>[0],
|
|
2055
|
+
): Promise<Uint8Array | { tmpPath: string }> {
|
|
2056
|
+
const plan = await planArtifact({
|
|
2057
|
+
stdout: args.entry.stdout ?? '',
|
|
2058
|
+
outputs: this.outputsOf(args),
|
|
2059
|
+
})
|
|
2060
|
+
if (plan.size <= STREAM_DECODE_FROM)
|
|
2061
|
+
return await Bun.zstdCompress(await packArtifactBytes(plan))
|
|
2062
|
+
const sink = Bun.file(tmpPath).writer()
|
|
1384
2063
|
try {
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
await Promise.all(
|
|
1388
|
-
args.outputFiles.map(async (f) => {
|
|
1389
|
-
const rel = path.relative(args.projectDir, f)
|
|
1390
|
-
const dest = path.join(stageOutputs, rel)
|
|
1391
|
-
// Bun.write creates parent dirs as needed.
|
|
1392
|
-
await Bun.write(dest, Bun.file(f))
|
|
1393
|
-
}),
|
|
1394
|
-
)
|
|
1395
|
-
}
|
|
1396
|
-
if (wsOutputFiles.length > 0) {
|
|
1397
|
-
// Caller passes workspaceRoot whenever workspaceOutputFiles is
|
|
1398
|
-
// non-empty; the rels are root-anchored by construction.
|
|
1399
|
-
const stageWs = path.join(stage, 'workspace-outputs')
|
|
1400
|
-
await Promise.all(
|
|
1401
|
-
wsOutputFiles.map(async (f) => {
|
|
1402
|
-
const rel = path.relative(args.workspaceRoot!, f)
|
|
1403
|
-
await Bun.write(path.join(stageWs, rel), Bun.file(f))
|
|
1404
|
-
}),
|
|
1405
|
-
)
|
|
1406
|
-
}
|
|
1407
|
-
// stdout is ALWAYS present in the artifact, even if empty, so the
|
|
1408
|
-
// archive layout is predictable: a successful read finds `stdout`
|
|
1409
|
-
// and zero-or-more `outputs/<rel>` / `workspace-outputs/<rel>`
|
|
1410
|
-
// entries.
|
|
1411
|
-
await Bun.write(path.join(stage, 'stdout'), args.entry.stdout ?? '')
|
|
1412
|
-
|
|
1413
|
-
const topLevel: string[] = ['stdout']
|
|
1414
|
-
if (wsOutputFiles.length > 0) topLevel.unshift('workspace-outputs')
|
|
1415
|
-
if (args.outputFiles.length > 0) topLevel.unshift('outputs')
|
|
1416
|
-
|
|
1417
|
-
// `--format=ustar` forces strict POSIX ustar — no PAX extended-
|
|
1418
|
-
// header records. BSD tar (macOS default) emits PAX per entry
|
|
1419
|
-
// by default for xattrs / mtime-nanos; those records would
|
|
1420
|
-
// otherwise show up as junk `PaxHeaders/<name>` entries in our
|
|
1421
|
-
// restored trees. GNU tar also accepts the flag (no-op on its
|
|
1422
|
-
// side). Names > 100 chars still work via ustar's prefix+name
|
|
1423
|
-
// (255 chars) or GNU longname fallback if the tar binary
|
|
1424
|
-
// chooses to emit one.
|
|
1425
|
-
const proc = Bun.spawn(['tar', '--format=ustar', '-cf', '-', '-C', stage, ...topLevel], {
|
|
1426
|
-
stdout: 'pipe',
|
|
1427
|
-
stderr: 'pipe',
|
|
1428
|
-
// COPYFILE_DISABLE blocks Apple's copyfile() from attaching
|
|
1429
|
-
// xattrs to staged files via inherited child copies; tar then
|
|
1430
|
-
// has nothing to emit AppleDouble for.
|
|
1431
|
-
env: { ...process.env, COPYFILE_DISABLE: '1' },
|
|
1432
|
-
})
|
|
1433
|
-
const [tarBytes, stderrText] = await Promise.all([
|
|
1434
|
-
new Response(proc.stdout).bytes(),
|
|
1435
|
-
new Response(proc.stderr).text(),
|
|
1436
|
-
])
|
|
1437
|
-
await proc.exited
|
|
1438
|
-
if (proc.exitCode !== 0) {
|
|
1439
|
-
throw new Error(`save: tar exited ${proc.exitCode}: ${stderrText.trim()}`)
|
|
2064
|
+
for await (const chunk of packArtifactStream(plan).pipeThrough(zstdEncoder())) {
|
|
2065
|
+
await sink.write(chunk)
|
|
1440
2066
|
}
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
2067
|
+
} catch (err) {
|
|
2068
|
+
// An output that vanished or changed shape between the plan's stat
|
|
2069
|
+
// and its read: the partial temp must not outlive the failure.
|
|
2070
|
+
await sink.end()
|
|
2071
|
+
await unlink(tmpPath).catch(() => undefined)
|
|
2072
|
+
throw err
|
|
1444
2073
|
}
|
|
2074
|
+
await sink.end()
|
|
2075
|
+
return { tmpPath }
|
|
1445
2076
|
}
|
|
1446
2077
|
|
|
1447
2078
|
/**
|
|
1448
|
-
* Atomically write `compressed` to `<hash>.tar.zst` and (re)build
|
|
1449
|
-
*
|
|
1450
|
-
*
|
|
1451
|
-
*
|
|
2079
|
+
* Atomically write `compressed` to `<hash>.tar.zst` and (re)build the
|
|
2080
|
+
* entries + output_files SQL rows from the archive itself. Shared by
|
|
2081
|
+
* `save()` (we just packed the bytes) and `ingest()` (we got them from
|
|
2082
|
+
* the remote layer) — both index the identical values, because both
|
|
2083
|
+
* read them out of the artifact.
|
|
1452
2084
|
*/
|
|
2085
|
+
/** tmp suffix mixes pid + hrtime + a random hex chunk so two saves of
|
|
2086
|
+
* the same hash from the same process (or from two forked workers that
|
|
2087
|
+
* happen to share a wall-clock ms) don't pick the same tmp filename and
|
|
2088
|
+
* race on the rename. */
|
|
2089
|
+
private tempPath(hash: string): string {
|
|
2090
|
+
return `${this.tarPath(hash)}.tmp-${process.pid}-${process.hrtime.bigint()}-${Math.random().toString(36).slice(2, 10)}`
|
|
2091
|
+
}
|
|
2092
|
+
|
|
1453
2093
|
private async writeArtifactAndIndex(
|
|
1454
2094
|
hash: string,
|
|
1455
|
-
compressed: Uint8Array,
|
|
2095
|
+
compressed: Uint8Array | { tmpPath: string },
|
|
1456
2096
|
meta: IngestMeta,
|
|
1457
|
-
statBase?: { projectDir: string; workspaceRoot?: string },
|
|
1458
2097
|
): Promise<void> {
|
|
1459
2098
|
// Validate BEFORE anything touches the final path. `ingest()` feeds
|
|
1460
2099
|
// us network bytes; a truncated/garbage body that went live first
|
|
1461
2100
|
// would leave a corrupt `<hash>.tar.zst` behind (with no SQL row,
|
|
1462
2101
|
// since the decompress throw aborted indexing) for every later
|
|
1463
2102
|
// reader to trip over. Decompress + parse also produce the
|
|
1464
|
-
// `output_files` rows: same headers
|
|
2103
|
+
// `output_files` rows: same headers the restore will see on
|
|
1465
2104
|
// restore, so the size/mode/mtime fingerprint we store matches
|
|
1466
2105
|
// what isOutputsCurrent will compare against post-restore.
|
|
1467
|
-
|
|
2106
|
+
const finalPath = this.tarPath(hash)
|
|
2107
|
+
let tmpPath: string
|
|
2108
|
+
if (compressed instanceof Uint8Array) {
|
|
2109
|
+
tmpPath = this.tempPath(hash)
|
|
2110
|
+
await mkdir(this.cacheDir, { recursive: true })
|
|
2111
|
+
// The temp is written BEFORE validation so a large artifact can be
|
|
2112
|
+
// scanned from the file as it decodes — a file stream reads in
|
|
2113
|
+
// bounded pieces; the bytes in memory would not (see `decodedTar`).
|
|
2114
|
+
// The final path is still untouched until the archive has passed.
|
|
2115
|
+
// node's writeFile, not Bun.write: the latter copies the buffer first
|
|
2116
|
+
// (measured on 150 MiB: +151 MiB and 33 ms against +0 and 21 ms).
|
|
2117
|
+
const endWrite = span('save: write temp')
|
|
2118
|
+
await writeFile(tmpPath, compressed)
|
|
2119
|
+
endWrite()
|
|
2120
|
+
} else {
|
|
2121
|
+
// `save` already streamed the artifact into its temp.
|
|
2122
|
+
tmpPath = compressed.tmpPath
|
|
2123
|
+
}
|
|
2124
|
+
let scanned: Awaited<ReturnType<typeof scanArtifact>>
|
|
1468
2125
|
try {
|
|
1469
2126
|
// ingest() is the UNTRUSTED boundary — `compressed` is bytes just
|
|
1470
2127
|
// pulled from a remote. Refuse a bomb (declared or sizeless) before it
|
|
1471
2128
|
// can expand into memory.
|
|
1472
|
-
|
|
2129
|
+
const source =
|
|
2130
|
+
compressed instanceof Uint8Array && compressed.byteLength <= STREAM_DECODE_FROM
|
|
2131
|
+
? compressed
|
|
2132
|
+
: Bun.file(tmpPath)
|
|
2133
|
+
const endScan = span('save: scan')
|
|
2134
|
+
scanned = await scanArtifact(await decodedTar(source, hash))
|
|
2135
|
+
endScan()
|
|
2136
|
+
// v17 invariant: every artifact carries a `stdout` entry. Its
|
|
2137
|
+
// absence means the bytes decompressed but aren't a vx artifact.
|
|
2138
|
+
if (scanned.stdout === null) {
|
|
2139
|
+
throw new CorruptArtifactError(hash, 'missing stdout entry')
|
|
2140
|
+
}
|
|
1473
2141
|
} catch (err) {
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
const headers = parseTarHeaders(tarBytes)
|
|
1478
|
-
// v17 invariant: every artifact carries a `stdout` entry. Its
|
|
1479
|
-
// absence means the bytes decompressed but aren't a vx artifact.
|
|
1480
|
-
if (!headers.some((h) => h.name === 'stdout' && !h.isDir)) {
|
|
1481
|
-
throw new CorruptArtifactError(hash, 'missing stdout entry')
|
|
2142
|
+
await unlink(tmpPath).catch(() => undefined)
|
|
2143
|
+
if (err instanceof ArchiveSecurityError || err instanceof CorruptArtifactError) throw err
|
|
2144
|
+
throw new CorruptArtifactError(hash, 'artifact is not a readable archive', err)
|
|
1482
2145
|
}
|
|
1483
|
-
|
|
1484
|
-
const finalPath = this.tarPath(hash)
|
|
1485
|
-
// tmp suffix mixes pid + hrtime + a random hex chunk so two saves
|
|
1486
|
-
// of the same hash from the same process (or from two forked
|
|
1487
|
-
// workers that happen to share a wall-clock ms) don't pick the
|
|
1488
|
-
// same tmp filename and race on the rename.
|
|
1489
|
-
const tmpPath = `${finalPath}.tmp-${process.pid}-${process.hrtime.bigint()}-${Math.random().toString(36).slice(2, 10)}`
|
|
1490
|
-
await mkdir(this.cacheDir, { recursive: true })
|
|
1491
|
-
await Bun.write(tmpPath, compressed)
|
|
2146
|
+
const { entries } = scanned
|
|
1492
2147
|
// POSIX rename atomically REPLACES the destination if it exists,
|
|
1493
2148
|
// so we don't need a pre-rm. The pre-rm was actively harmful —
|
|
1494
2149
|
// it opened a race window where writer B could delete writer A's
|
|
1495
2150
|
// just-renamed file BEFORE A's subsequent stat, producing a
|
|
1496
2151
|
// spurious ENOENT. The rename itself preserves the "either-or"
|
|
1497
2152
|
// semantics for concurrent readers.
|
|
2153
|
+
const endRename = span('save: rename')
|
|
1498
2154
|
await rename(tmpPath, finalPath)
|
|
2155
|
+
endRename()
|
|
1499
2156
|
|
|
1500
|
-
const totalBytes =
|
|
2157
|
+
const totalBytes =
|
|
2158
|
+
compressed instanceof Uint8Array ? compressed.byteLength : Bun.file(finalPath).size
|
|
1501
2159
|
const outputFileRows: Array<[string, number, number, number]> = []
|
|
1502
2160
|
// Per-output-file fingerprint rows feed the skip-restore check.
|
|
1503
2161
|
// Row paths: project entries store the bare rel (`outputs/`
|
|
1504
2162
|
// stripped); workspace entries keep the full
|
|
1505
2163
|
// `workspace-outputs/<rel>` name as the namespace discriminator.
|
|
1506
|
-
|
|
1507
|
-
|
|
2164
|
+
//
|
|
2165
|
+
// Mode and MILLISECOND mtime come from the artifact's own sidecar
|
|
2166
|
+
// (`.vx-meta.json`), written from a stat taken while packing. Both
|
|
2167
|
+
// paths — save and remote ingest — therefore index the same values,
|
|
2168
|
+
// and `restoreOutputs` materialises exactly them, so
|
|
2169
|
+
// `isOutputsCurrent` compares equal at ms precision straight after a
|
|
2170
|
+
// restore. (Tar headers carry seconds, which is why the save path
|
|
2171
|
+
// used to need a second stat pass the ingest path could not have.)
|
|
2172
|
+
for (const e of entries) {
|
|
1508
2173
|
let rowPath: string | null = null
|
|
1509
|
-
if (
|
|
1510
|
-
const rel =
|
|
2174
|
+
if (e.name.startsWith('outputs/')) {
|
|
2175
|
+
const rel = e.name.slice('outputs/'.length)
|
|
1511
2176
|
if (rel.length > 0) rowPath = rel
|
|
1512
|
-
} else if (
|
|
1513
|
-
if (
|
|
2177
|
+
} else if (e.name.startsWith(WORKSPACE_OUTPUT_PREFIX)) {
|
|
2178
|
+
if (e.name.length > WORKSPACE_OUTPUT_PREFIX.length) rowPath = e.name
|
|
1514
2179
|
}
|
|
1515
2180
|
if (rowPath === null) continue
|
|
1516
|
-
|
|
1517
|
-
// Save path only (ingest has no filesystem to consult): tar
|
|
1518
|
-
// headers truncate mtimes to SECONDS, which is what made the
|
|
1519
|
-
// skip-restore check blind to same-second same-size edits. A
|
|
1520
|
-
// fresh stat refines the row to millisecond precision — but only
|
|
1521
|
-
// when the size still matches the packed header, so the row
|
|
1522
|
-
// always describes the ARTIFACT's bytes; a file mutated between
|
|
1523
|
-
// pack and stat keeps the coarse header time (degraded, not
|
|
1524
|
-
// wrong). restoreOutputs re-syncs disk mtimes to these rows.
|
|
1525
|
-
if (statBase !== undefined) {
|
|
1526
|
-
try {
|
|
1527
|
-
const abs = rowPath.startsWith(WORKSPACE_OUTPUT_PREFIX)
|
|
1528
|
-
? path.join(
|
|
1529
|
-
statBase.workspaceRoot ?? statBase.projectDir,
|
|
1530
|
-
rowPath.slice(WORKSPACE_OUTPUT_PREFIX.length),
|
|
1531
|
-
)
|
|
1532
|
-
: path.join(statBase.projectDir, rowPath)
|
|
1533
|
-
const st = await stat(abs)
|
|
1534
|
-
if (st.size === h.size) mtimeMs = Math.floor(st.mtimeMs)
|
|
1535
|
-
} catch {
|
|
1536
|
-
// stat failed — keep the header-seconds fallback.
|
|
1537
|
-
}
|
|
1538
|
-
}
|
|
1539
|
-
outputFileRows.push([rowPath, h.size, h.mode & 0o777, mtimeMs])
|
|
2181
|
+
outputFileRows.push([rowPath, e.size, e.mode & 0o777, Math.floor(e.mtimeMs)])
|
|
1540
2182
|
}
|
|
1541
|
-
const stdoutText =
|
|
2183
|
+
const stdoutText = scanned.stdout
|
|
1542
2184
|
|
|
1543
2185
|
const [project, task] = splitTaskId(meta.taskId)
|
|
1544
2186
|
const now = Date.now()
|
|
@@ -1559,7 +2201,12 @@ export class Cache implements CacheLayer {
|
|
|
1559
2201
|
project,
|
|
1560
2202
|
task,
|
|
1561
2203
|
meta.command,
|
|
1562
|
-
|
|
2204
|
+
// exitCode. Pinned, not supplied: vx caches only successes, and neither
|
|
2205
|
+
// `save` nor `ingest` accepts one (see their arg types). The column
|
|
2206
|
+
// stays because the READ side still defends against a non-zero value —
|
|
2207
|
+
// a row from a foreign or hand-edited cache.db classifies the hit
|
|
2208
|
+
// `failed` rather than laundering a broken build into a green run.
|
|
2209
|
+
0,
|
|
1563
2210
|
meta.durationMs,
|
|
1564
2211
|
totalBytes,
|
|
1565
2212
|
stdoutText,
|
|
@@ -1580,7 +2227,9 @@ export class Cache implements CacheLayer {
|
|
|
1580
2227
|
}
|
|
1581
2228
|
}
|
|
1582
2229
|
})
|
|
2230
|
+
const endTx = span('save: index tx')
|
|
1583
2231
|
tx()
|
|
2232
|
+
endTx()
|
|
1584
2233
|
}
|
|
1585
2234
|
|
|
1586
2235
|
/** Apply the deferred accessed_at bumps in one statement. */
|
|
@@ -1658,17 +2307,26 @@ export class Cache implements CacheLayer {
|
|
|
1658
2307
|
})()
|
|
1659
2308
|
}
|
|
1660
2309
|
|
|
1661
|
-
stats(): CacheStats {
|
|
2310
|
+
stats(opts: CacheStatsOptions = {}): CacheStats {
|
|
1662
2311
|
this.flushAccessed()
|
|
2312
|
+
const project = opts.project
|
|
2313
|
+
const scoped = project !== undefined
|
|
2314
|
+
const scopeParams: string[] = project === undefined ? [] : [project]
|
|
1663
2315
|
const aggregate = this.db
|
|
1664
|
-
.prepare(
|
|
1665
|
-
|
|
2316
|
+
.prepare(
|
|
2317
|
+
`SELECT COUNT(*) AS n, COALESCE(SUM(size_bytes), 0) AS bytes FROM entries${
|
|
2318
|
+
scoped ? ' WHERE project = ?' : ''
|
|
2319
|
+
}`,
|
|
2320
|
+
)
|
|
2321
|
+
.get(...scopeParams) as { n: number; bytes: number }
|
|
1666
2322
|
const since = Date.now() - 24 * 60 * 60 * 1000
|
|
1667
2323
|
const runs = this.db
|
|
1668
2324
|
.prepare(
|
|
1669
|
-
|
|
2325
|
+
`SELECT COUNT(*) AS total, COALESCE(SUM(CASE WHEN status IN ('cache-hit', 'cache-hit-remote') THEN 1 ELSE 0 END), 0) AS hits FROM runs WHERE started_at >= ? AND ${EXECUTED_RUNS_SQL}${
|
|
2326
|
+
scoped ? ' AND project = ?' : ''
|
|
2327
|
+
}`,
|
|
1670
2328
|
)
|
|
1671
|
-
.get(since) as { total: number; hits: number }
|
|
2329
|
+
.get(since, ...scopeParams) as { total: number; hits: number }
|
|
1672
2330
|
return {
|
|
1673
2331
|
entryCount: aggregate.n,
|
|
1674
2332
|
totalBytes: aggregate.bytes,
|
|
@@ -1747,15 +2405,34 @@ export class Cache implements CacheLayer {
|
|
|
1747
2405
|
// executed task per invocation — a 2000-task repo accretes ~20k
|
|
1748
2406
|
// rows in days, inflating insert and checkpoint cost forever.
|
|
1749
2407
|
// 30 days comfortably covers `vx stats` (24 h windows) and any
|
|
1750
|
-
// CI-side analytics consumers.
|
|
2408
|
+
// CI-side analytics consumers. The `invocations` header table (one
|
|
2409
|
+
// row per `vx run`) is pruned on the SAME window — otherwise a
|
|
2410
|
+
// header would outlive its `runs` rows, so `vx info`/`vx last` would
|
|
2411
|
+
// list an invocation whose task detail is already gone, and the
|
|
2412
|
+
// table would grow unbounded on a long-lived checkout.
|
|
1751
2413
|
try {
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
2414
|
+
const cutoff = Date.now() - 30 * 24 * 60 * 60 * 1000
|
|
2415
|
+
this.db.prepare('DELETE FROM runs WHERE ended_at < ?').run(cutoff)
|
|
2416
|
+
this.db.prepare('DELETE FROM invocations WHERE ended_at < ?').run(cutoff)
|
|
2417
|
+
// A config that has not been loaded in 30 days was edited (its key
|
|
2418
|
+
// moved) or its project left; either way the row is dead weight.
|
|
2419
|
+
this.db.prepare('DELETE FROM config_evals WHERE created_at < ?').run(cutoff)
|
|
2420
|
+
this.db.prepare('DELETE FROM config_closures WHERE created_at < ?').run(cutoff)
|
|
1755
2421
|
} catch {
|
|
1756
2422
|
// Retention is best-effort; never block closing the handle.
|
|
1757
2423
|
}
|
|
1758
|
-
|
|
2424
|
+
try {
|
|
2425
|
+
this.flushAccessed()
|
|
2426
|
+
} catch {
|
|
2427
|
+
// Same contract as the retention prune above, which was already
|
|
2428
|
+
// guarded while this sibling was not: `accessed_at` is LRU
|
|
2429
|
+
// bookkeeping, never correctness, and the run's results are
|
|
2430
|
+
// already recorded by the time we get here. Letting it throw also
|
|
2431
|
+
// SKIPPED `db.close()` below, leaking the handle. Reachable when
|
|
2432
|
+
// the cache dir is removed under a live handle (a concurrent
|
|
2433
|
+
// `rm -rf .vx/cache`): macOS answers SQLITE_IOERR_VNODE on a
|
|
2434
|
+
// write to an unlinked file where Linux happily writes on.
|
|
2435
|
+
}
|
|
1759
2436
|
this.db.close()
|
|
1760
2437
|
}
|
|
1761
2438
|
|
|
@@ -1771,7 +2448,9 @@ export class Cache implements CacheLayer {
|
|
|
1771
2448
|
*/
|
|
1772
2449
|
function bindRun(run: RunRecord): SQLQueryBindings[] {
|
|
1773
2450
|
return [
|
|
1774
|
-
|
|
2451
|
+
// The ONE place the no-key sentinel is applied, so the column's
|
|
2452
|
+
// NOT NULL invariant can't be violated from a call site.
|
|
2453
|
+
run.hash ?? '',
|
|
1775
2454
|
run.project,
|
|
1776
2455
|
run.task,
|
|
1777
2456
|
run.status,
|