@vzn/vx 0.0.650 → 0.0.651
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -20
- package/package.json +5 -5
- package/schemas/last.json +15 -0
- package/src/bin.ts +9 -31
- package/src/cache/cache.ts +202 -25
- package/src/cache/file-hashes.ts +3 -0
- package/src/cache/git-inputs.ts +188 -39
- package/src/cache/index.ts +1 -0
- package/src/cache/inputs.ts +3 -0
- package/src/cache/layer.ts +11 -0
- package/src/cache/layered-cache.ts +4 -0
- package/src/cache/linked-inputs.ts +3 -0
- package/src/cache/lockfile-memo.ts +3 -2
- package/src/cache/output-index.ts +5 -2
- package/src/cache/remote-lane.ts +2 -1
- package/src/cache/run-history.ts +11 -1
- package/src/cache/schema.ts +11 -1
- package/src/cli/cache.ts +26 -16
- package/src/cli/completions.ts +4 -2
- package/src/cli/docs.ts +10 -11
- package/src/cli/help.ts +12 -23
- package/src/cli/index.ts +4 -9
- package/src/cli/info.ts +15 -7
- package/src/cli/init.ts +23 -15
- package/src/cli/last.ts +213 -203
- package/src/cli/lock.ts +55 -5
- package/src/cli/plan-format.ts +100 -118
- package/src/cli/plugin-commands.ts +8 -2
- package/src/cli/plugin-templates.ts +2 -2
- package/src/cli/run.ts +46 -27
- package/src/cli/select.ts +43 -5
- package/src/cli/show.ts +38 -42
- package/src/cli/task-verb.ts +22 -0
- package/src/cli/upgrade.ts +3 -1
- package/src/cli/watch-fs.ts +6 -1
- package/src/cli/watch-judge.ts +2 -1
- package/src/cli/watch.ts +83 -44
- package/src/cli/why.ts +123 -89
- package/src/cli/workspace-config.ts +2 -1
- package/src/exec/executor.ts +2 -1
- package/src/exec/kill-tree.ts +58 -5
- package/src/exec/local-executor.ts +1 -1
- package/src/exec/pipes.ts +152 -0
- package/src/exec/runner.ts +199 -182
- package/src/exec/sandbox-binds.ts +1 -1
- package/src/exec/sandbox-runtime.ts +11 -2
- package/src/exec/spawn-off.ts +154 -0
- package/src/exec/warm-shell.ts +146 -0
- package/src/graph/critical-path.ts +40 -28
- package/src/graph/priorities.ts +98 -44
- package/src/graph/scheduler.ts +14 -10
- package/src/graph/task-graph.ts +27 -1
- package/src/handoff.ts +10 -7
- package/src/history/history.ts +5 -0
- package/src/history/metrics.ts +9 -1
- package/src/index.ts +3 -1
- package/src/orchestrator/doctor.ts +1 -1
- package/src/orchestrator/execute-task.ts +16 -2
- package/src/orchestrator/graph-memo.ts +50 -1
- package/src/orchestrator/graph-stage.ts +6 -0
- package/src/orchestrator/index.ts +3 -0
- package/src/orchestrator/live-usage.ts +16 -11
- package/src/orchestrator/locked-digests.ts +63 -0
- package/src/orchestrator/lockfile-claim.ts +108 -29
- package/src/orchestrator/miss-save.ts +31 -3
- package/src/orchestrator/persistent-task.ts +3 -1
- package/src/orchestrator/plan.ts +5 -1
- package/src/orchestrator/plugin-host.ts +61 -0
- package/src/orchestrator/plugin.ts +47 -2
- package/src/orchestrator/prepare.ts +16 -1
- package/src/orchestrator/projects.ts +4 -1
- package/src/orchestrator/run-context.ts +29 -10
- package/src/orchestrator/run-failures.ts +1 -1
- package/src/orchestrator/run-records.ts +10 -0
- package/src/orchestrator/run.ts +53 -22
- package/src/orchestrator/sandbox-request.ts +2 -2
- package/src/orchestrator/signals.ts +8 -5
- package/src/orchestrator/task-hash.ts +17 -3
- package/src/report/events.ts +3 -11
- package/src/report/forecast.ts +10 -32
- package/src/report/framed-output.ts +272 -173
- package/src/report/index.ts +8 -3
- package/src/report/logger.ts +73 -38
- package/src/report/plain-output.ts +1 -1
- package/src/report/status-line.ts +49 -357
- package/src/report/summary.ts +100 -131
- package/src/ui/ansi.ts +48 -0
- package/src/ui/components.ts +506 -0
- package/src/ui/index.ts +79 -0
- package/src/ui/kit.ts +37 -0
- package/src/ui/live-region.ts +363 -0
- package/src/{util/refusal-notice.ts → ui/refusal.ts} +7 -5
- package/src/ui/terminal.ts +51 -0
- package/src/ui/theme.ts +224 -0
- package/src/ui/version-line.ts +28 -0
- package/src/util/config-frame.ts +65 -10
- package/src/util/edit-distance.ts +6 -1
- package/src/util/index.ts +2 -15
- package/src/util/paths.ts +20 -2
- package/src/util/secret-mask.ts +39 -3
- package/src/util/settle.ts +31 -0
- package/src/util/timing.ts +4 -16
- package/src/workspace/affected.ts +89 -30
- package/src/workspace/config-schema.ts +4 -2
- package/src/workspace/dependencies-lock.ts +191 -0
- package/src/workspace/filter.ts +28 -0
- package/src/workspace/index.ts +9 -0
- package/src/workspace/lockfile.ts +159 -6
- package/src/workspace/migration.ts +4 -3
- package/src/workspace/workspace.ts +3 -0
- package/src/util/colors.ts +0 -133
- package/src/util/report.ts +0 -44
- package/src/util/ui.ts +0 -258
- /package/src/{report/path-links.ts → ui/links.ts} +0 -0
- /package/src/{report → ui}/terminal-signals.ts +0 -0
package/README.md
CHANGED
|
@@ -142,9 +142,8 @@ not a way to run the repo. The benchmark below uses native vx config.
|
|
|
142
142
|
| Plugin API | **Yes**: every pipeline stage | No | Yes (TS-tied) |
|
|
143
143
|
| Install | **Single binary**; no Bun, Node optional | npm + Node | npm + Node |
|
|
144
144
|
|
|
145
|
-
Every gap
|
|
146
|
-
[
|
|
147
|
-
[vx, Turborepo, Nx, Bazel](https://vznjs.github.io/vx/compare/).
|
|
145
|
+
Every gap: [comparison](packages/vx/docs/comparison.md) ·
|
|
146
|
+
every tool on one suite of cases: [conformance](https://vznjs.github.io/vx/conformance/).
|
|
148
147
|
|
|
149
148
|
## Fast, without trading correctness
|
|
150
149
|
|
|
@@ -152,23 +151,9 @@ Every gap, and when another tool is the better pick:
|
|
|
152
151
|
- **No source-file reads to hash:** on a clean tree, keys come from git's index; each project's `package.json` and the lockfile are still read.
|
|
153
152
|
- **One pass:** one git walk per run; each stat and lookup happens once.
|
|
154
153
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
| ---------------------------------------------------------- | ---------- | ---------- | ---------- | --------- |
|
|
159
|
-
| Nothing changed | **799 ms** | 927 ms | 24.15 s | 12.07 s |
|
|
160
|
-
| Nothing changed, outputs restored | **1.28 s** | 2.01 s | 24.45 s | 11.86 s |
|
|
161
|
-
| Cold build | **7.57 s** | 9.63 s | 3 min 6 s | 9.01 s |
|
|
162
|
-
| Cold build: CPU the runner burns | 36.64 s | 1 min 12 s | 5 min 34 s | 25.32 s |
|
|
163
|
-
|
|
164
|
-
Time each tool adds over the ideal run; lower is better.
|
|
165
|
-
|
|
166
|
-
Benchmark workload: a synthetic monorepo of 1,601 projects and 9,603 tasks in 30 dependency levels, five core libraries a quarter of the projects use; build 300 ms, test and typecheck 150 ms, lint 75 ms, publish 30 ms; real repos with uneven task times will differ.
|
|
167
|
-
Run 2026-10-25 on linux x64, 4 cores: vx at commit 9abdfb52c, Turborepo 2.11.10, Nx 24.0.1, Vite Task (vite-plus) 1.4.0.
|
|
168
|
-
|
|
169
|
-
Same graph, commands and concurrency: [how it is measured](https://vznjs.github.io/vx/benchmarks/).
|
|
170
|
-
|
|
171
|
-
<!-- bench:end -->
|
|
154
|
+
The head-to-head against Turborepo, Nx and Vite Task, each in its own
|
|
155
|
+
native config, is re-measured on every change to vx and published on the
|
|
156
|
+
site: [benchmarks](https://vznjs.github.io/vx/benchmarks/).
|
|
172
157
|
|
|
173
158
|
## Status
|
|
174
159
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vzn/vx",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.651",
|
|
4
4
|
"description": "Task orchestration you own: correct, sandboxed, extensible, with a content-addressed build cache.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -21,10 +21,10 @@
|
|
|
21
21
|
"node": ">=18"
|
|
22
22
|
},
|
|
23
23
|
"optionalDependencies": {
|
|
24
|
-
"@vzn/vx-linux-x64": "0.0.
|
|
25
|
-
"@vzn/vx-linux-arm64": "0.0.
|
|
26
|
-
"@vzn/vx-darwin-x64": "0.0.
|
|
27
|
-
"@vzn/vx-darwin-arm64": "0.0.
|
|
24
|
+
"@vzn/vx-linux-x64": "0.0.651",
|
|
25
|
+
"@vzn/vx-linux-arm64": "0.0.651",
|
|
26
|
+
"@vzn/vx-darwin-x64": "0.0.651",
|
|
27
|
+
"@vzn/vx-darwin-arm64": "0.0.651"
|
|
28
28
|
},
|
|
29
29
|
"dependencies": {
|
|
30
30
|
"@anthropic-ai/sandbox-runtime": "^0.0.76"
|
package/schemas/last.json
CHANGED
|
@@ -242,7 +242,10 @@
|
|
|
242
242
|
"peakRssBytes",
|
|
243
243
|
"peakCores",
|
|
244
244
|
"avgRssBytes",
|
|
245
|
+
"queuedMs",
|
|
245
246
|
"placement",
|
|
247
|
+
"outputBytes",
|
|
248
|
+
"blocks",
|
|
246
249
|
"wallclockStartNs",
|
|
247
250
|
"wallclockEndNs",
|
|
248
251
|
"blockedBy",
|
|
@@ -288,10 +291,22 @@
|
|
|
288
291
|
"type": ["integer", "null"],
|
|
289
292
|
"description": "mean resident memory over the samples; null when nothing sampled"
|
|
290
293
|
},
|
|
294
|
+
"queuedMs": {
|
|
295
|
+
"type": ["integer", "null"],
|
|
296
|
+
"description": "milliseconds it waited, ready, for a slot; null when it never queued"
|
|
297
|
+
},
|
|
291
298
|
"placement": {
|
|
292
299
|
"type": ["string", "null"],
|
|
293
300
|
"description": "where an executor ran it (small#1); null when here"
|
|
294
301
|
},
|
|
302
|
+
"outputBytes": {
|
|
303
|
+
"type": ["integer", "null"],
|
|
304
|
+
"description": "bytes of its outputs: what its save stored, or its hit restored"
|
|
305
|
+
},
|
|
306
|
+
"blocks": {
|
|
307
|
+
"type": ["integer", "null"],
|
|
308
|
+
"description": "how many tasks were downstream of it"
|
|
309
|
+
},
|
|
295
310
|
"wallclockStartNs": {
|
|
296
311
|
"type": ["string", "null"],
|
|
297
312
|
"description": "decimal nanoseconds from the run's start"
|
package/src/bin.ts
CHANGED
|
@@ -21,32 +21,14 @@ async function main(): Promise<void> {
|
|
|
21
21
|
// `vx --version` needs one leaf module; the dispatcher and the util
|
|
22
22
|
// barrel it pulled in were 20 ms of its 28 (2026-10-01).
|
|
23
23
|
if (argv.length === 1 && (argv[0] === '--version' || argv[0] === 'version')) {
|
|
24
|
-
// `Bun.stdout`, not `process.stdout`: the Node stream's first touch is
|
|
25
|
-
// ~7 ms, half of this verb (2026-10-02). A reader gone (EPIPE) is no
|
|
26
|
-
// failure of the line.
|
|
27
24
|
// Handed off, it says which vx ran and which one handed it the call.
|
|
28
25
|
const from = process.env['VX_HANDED_OFF'] || undefined
|
|
29
26
|
const where = from === undefined ? '' : ` (${vxPath()}, handed off from ${from})`
|
|
30
|
-
|
|
31
|
-
// Decided here, not by util's `detectColors`: the util barrel was the
|
|
32
|
-
// 20 ms above, and `node:tty` 17 ms; fd 1 as a character device is the
|
|
33
|
-
// TTY answer. A CI's log is not painted for this one line.
|
|
34
|
-
const [{ VERSION }, { fstatSync }] = await Promise.all([
|
|
27
|
+
const [{ VERSION }, { printVersion }] = await Promise.all([
|
|
35
28
|
import('./version.js'),
|
|
36
|
-
import('
|
|
29
|
+
import('./ui/version-line.js'),
|
|
37
30
|
])
|
|
38
|
-
|
|
39
|
-
const on = env['NO_COLOR']
|
|
40
|
-
? false
|
|
41
|
-
: env['FORCE_COLOR'] !== undefined
|
|
42
|
-
? env['FORCE_COLOR'] !== '0' && env['FORCE_COLOR'] !== 'false'
|
|
43
|
-
: env['TERM'] !== 'dumb' && fstatSync(1).isCharacterDevice()
|
|
44
|
-
const sgr = (code: string, text: string): string =>
|
|
45
|
-
on ? `\x1b[${code}m${text}\x1b[0m` : text
|
|
46
|
-
await Bun.write(
|
|
47
|
-
Bun.stdout,
|
|
48
|
-
`${sgr('1', 'vx')} ${sgr('38;2;6;182;212', VERSION)}${where === '' ? '' : sgr('2', where)}\n`,
|
|
49
|
-
).catch(() => {})
|
|
31
|
+
await printVersion(VERSION, where)
|
|
50
32
|
settled = true
|
|
51
33
|
return
|
|
52
34
|
}
|
|
@@ -64,8 +46,9 @@ async function main(): Promise<void> {
|
|
|
64
46
|
// is set, and `exitOnceFlushed` exits after both streams have ended.
|
|
65
47
|
process.exitCode = code
|
|
66
48
|
} catch (err) {
|
|
49
|
+
const ui = await import('./ui/index.js')
|
|
50
|
+
const { refusalNotice } = ui
|
|
67
51
|
const {
|
|
68
|
-
detectColors,
|
|
69
52
|
errorCode,
|
|
70
53
|
errorDocument,
|
|
71
54
|
fsRefusalHint,
|
|
@@ -74,17 +57,12 @@ async function main(): Promise<void> {
|
|
|
74
57
|
isUserError,
|
|
75
58
|
maskedLine,
|
|
76
59
|
OUT_OF_FDS_HINT,
|
|
77
|
-
refusalNotice,
|
|
78
|
-
ui,
|
|
79
60
|
wantsJson,
|
|
80
61
|
} = await import('./util/index.js')
|
|
81
62
|
// A plugin's failure quotes what it was told (a remote's reply, a
|
|
82
63
|
// header), and a config's own throw is its author's text: a secret in
|
|
83
64
|
// either reached stderr whole (L-11).
|
|
84
|
-
const say = (line: string): void =>
|
|
85
|
-
void process.stderr.write(
|
|
86
|
-
`${ui.tagged('error', maskedLine(line), detectColors(process.stderr))}\n`,
|
|
87
|
-
)
|
|
65
|
+
const say = (line: string): void => ui.say('error', maskedLine(line))
|
|
88
66
|
// UserError (workspace not found, cycle, config invalid, ...) —
|
|
89
67
|
// print the message only; the stack is noise the user can't act
|
|
90
68
|
// on. Everything else gets the full stack so internal bugs are
|
|
@@ -100,7 +78,7 @@ async function main(): Promise<void> {
|
|
|
100
78
|
named === null ? err.message : err.message.slice(named[0].length),
|
|
101
79
|
err.parts,
|
|
102
80
|
)
|
|
103
|
-
|
|
81
|
+
ui.data('err', maskedLine(ui.render([block], ui.colorsOf('err'))))
|
|
104
82
|
} else if (isOutOfFds(err)) {
|
|
105
83
|
say(`vx: ${err.message} — ${OUT_OF_FDS_HINT}\n`)
|
|
106
84
|
} else if (isFsRefusal(err)) {
|
|
@@ -114,7 +92,7 @@ async function main(): Promise<void> {
|
|
|
114
92
|
// stdout got nothing to parse, and only prose on stderr to branch on.
|
|
115
93
|
if (wantsJson(argv)) {
|
|
116
94
|
const message = err instanceof Error ? err.message : String(err)
|
|
117
|
-
|
|
95
|
+
ui.data('out', errorDocument(errorCode(err), maskedLine(message)))
|
|
118
96
|
}
|
|
119
97
|
// Same reason as the success path above: a large stderr is truncated by
|
|
120
98
|
// `process.exit` too, and an error message cut in half is the one a
|
|
@@ -174,7 +152,7 @@ function withCauses(err: unknown): string {
|
|
|
174
152
|
process.on('beforeExit', () => {
|
|
175
153
|
if (settled) return
|
|
176
154
|
settled = true
|
|
177
|
-
;(require('./
|
|
155
|
+
;(require('./ui/index.js') as typeof import('./ui/index.js')).say(
|
|
178
156
|
'error',
|
|
179
157
|
'vx: the run stopped before it finished: something it awaited can never settle (a plugin hook?)',
|
|
180
158
|
)
|
package/src/cache/cache.ts
CHANGED
|
@@ -130,6 +130,9 @@ import { behindLink } from './linked-inputs.js'
|
|
|
130
130
|
* this, so the file time trails the last use by at most this much, and
|
|
131
131
|
* the policy reads the file time plus this as the last use.
|
|
132
132
|
*/
|
|
133
|
+
// About 1.3 ms of inserts per turn: a task's exit waits at most that.
|
|
134
|
+
const HELD_SLICE = 250
|
|
135
|
+
|
|
133
136
|
const ORPHAN_GRACE_MS = 60 * 60 * 1000
|
|
134
137
|
|
|
135
138
|
/**
|
|
@@ -355,7 +358,10 @@ function cantOpenIndex(dbFile: string, err: unknown): UserError | undefined {
|
|
|
355
358
|
// v50: steps, warnings, failures and lockfile_memos — what a run said
|
|
356
359
|
// and a lockfile claimant parsed, files under the cache dir until
|
|
357
360
|
// now (if it can be in the DB, it is). The cache KEY is unchanged.
|
|
358
|
-
|
|
361
|
+
// v51: runs.queued_ms, output_bytes, blocks, flaky_passes and
|
|
362
|
+
// flaky_failures — the rest of what a results row shows, so `vx
|
|
363
|
+
// last --sql` reads it. The cache KEY is unchanged.
|
|
364
|
+
export const SCHEMA_VERSION = 'v51'
|
|
359
365
|
|
|
360
366
|
/**
|
|
361
367
|
* An entry's key components as its one `entry_inputs` row (v34): JSON
|
|
@@ -942,10 +948,21 @@ export class Cache implements CacheLayer {
|
|
|
942
948
|
private adoptStats = 0
|
|
943
949
|
/** The artifact directory's names, listed once past `ADOPT_STATS`; this process's writes kept in it. */
|
|
944
950
|
private artifactNames: Set<string> | undefined
|
|
951
|
+
private heldBlobSizes: Array<[string, number]> | undefined
|
|
952
|
+
private heldTimer: ReturnType<typeof setTimeout> | undefined
|
|
953
|
+
/** Saves' index writes waiting for this turn's one transaction (`indexInBatch`). */
|
|
954
|
+
private indexQueue: Array<{ write: () => void; settle: (err?: unknown) => void }> = []
|
|
955
|
+
private heldRest: () => Array<[string, number]> = () => []
|
|
945
956
|
private readonly insertEntryInput: ReturnType<Database['prepare']>
|
|
946
957
|
/** The per-file (mtime, size) → blob-OID memo behind `hashFile`. */
|
|
947
958
|
private readonly files: FileHashStore
|
|
948
959
|
private readonly outputs: OutputIndex
|
|
960
|
+
/**
|
|
961
|
+
* Each entry `save()` wrote: its files' bytes, summed as the rows went in,
|
|
962
|
+
* for the miss row's `outputs` (a read back was one statement a miss;
|
|
963
|
+
* principle 9). Taken once, by `outputBytes`.
|
|
964
|
+
*/
|
|
965
|
+
private readonly savedBytes = new Map<string, number>()
|
|
949
966
|
private readonly folds: IndexFoldMemo
|
|
950
967
|
private readonly history: RunHistory
|
|
951
968
|
/** What the remotes said they hold, across runs (remote-hits.ts). */
|
|
@@ -1235,12 +1252,13 @@ export class Cache implements CacheLayer {
|
|
|
1235
1252
|
}
|
|
1236
1253
|
}
|
|
1237
1254
|
}
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1255
|
+
// An index that held its entries itself (no store yet: a repository
|
|
1256
|
+
// before its first commit) hands them to the store below.
|
|
1257
|
+
const holds =
|
|
1258
|
+
storeDir !== undefined &&
|
|
1259
|
+
mode === 'open' &&
|
|
1260
|
+
this.writeBlocked === null &&
|
|
1261
|
+
readable(
|
|
1244
1262
|
() =>
|
|
1245
1263
|
this.db
|
|
1246
1264
|
.prepare(
|
|
@@ -1248,16 +1266,6 @@ export class Cache implements CacheLayer {
|
|
|
1248
1266
|
)
|
|
1249
1267
|
.get() != null,
|
|
1250
1268
|
)
|
|
1251
|
-
if (holds) {
|
|
1252
|
-
this.db
|
|
1253
|
-
.transaction(() => {
|
|
1254
|
-
for (const t of [...STORE_TABLES, ...ARTIFACT_TABLES]) {
|
|
1255
|
-
this.db.exec(`DROP TABLE IF EXISTS main.${t}`)
|
|
1256
|
-
}
|
|
1257
|
-
})
|
|
1258
|
-
.immediate()
|
|
1259
|
-
}
|
|
1260
|
-
}
|
|
1261
1269
|
this.storeDir = storeDir
|
|
1262
1270
|
this.artifactDir = storeDir ?? cacheDir
|
|
1263
1271
|
if (storeDir !== undefined) {
|
|
@@ -1271,6 +1279,7 @@ export class Cache implements CacheLayer {
|
|
|
1271
1279
|
if (mode === 'open' && this.writeBlocked === null && (foreign || storeDir !== recorded)) {
|
|
1272
1280
|
readable(() => this.matchArtifactLayout(storeDir === undefined ? 'main' : 'store'))
|
|
1273
1281
|
}
|
|
1282
|
+
if (holds) this.carryEntries(cacheDir, storeDir!)
|
|
1274
1283
|
if (mode === 'open' && this.writeBlocked === null && storeDir !== recorded) {
|
|
1275
1284
|
if (storeDir === undefined)
|
|
1276
1285
|
this.db.prepare("DELETE FROM schema_meta WHERE key = 'store_dir'").run()
|
|
@@ -1396,6 +1405,68 @@ export class Cache implements CacheLayer {
|
|
|
1396
1405
|
}
|
|
1397
1406
|
}
|
|
1398
1407
|
|
|
1408
|
+
/**
|
|
1409
|
+
* The entries an index held itself, moved into the store it now opens:
|
|
1410
|
+
* the same keys over the same bytes, so the first run after a
|
|
1411
|
+
* repository's first commit replays what the run before it saved. Then
|
|
1412
|
+
* dropped from the index, where an unqualified name would shadow the
|
|
1413
|
+
* store's. An artifact file that cannot be renamed there (another
|
|
1414
|
+
* device) is left for `prune`, and its entry is a miss.
|
|
1415
|
+
*/
|
|
1416
|
+
private carryEntries(cacheDir: string, storeDir: string): void {
|
|
1417
|
+
const has = (schema: 'main' | 'store', table: string): boolean =>
|
|
1418
|
+
this.db
|
|
1419
|
+
.prepare(`SELECT 1 FROM ${schema}.sqlite_master WHERE type = 'table' AND name = ?`)
|
|
1420
|
+
.get(table) != null
|
|
1421
|
+
const layout = (schema: 'main' | 'store'): string | undefined =>
|
|
1422
|
+
has(schema, 'artifacts_meta')
|
|
1423
|
+
? (
|
|
1424
|
+
this.db
|
|
1425
|
+
.prepare(`SELECT value FROM ${schema}.artifacts_meta WHERE key = 'layout'`)
|
|
1426
|
+
.get() as { value: string } | null
|
|
1427
|
+
)?.value
|
|
1428
|
+
: undefined
|
|
1429
|
+
if (storeDir !== cacheDir) {
|
|
1430
|
+
let names: string[] = []
|
|
1431
|
+
try {
|
|
1432
|
+
names = readdirSync(cacheDir)
|
|
1433
|
+
} catch {
|
|
1434
|
+
// No directory: nothing to carry.
|
|
1435
|
+
}
|
|
1436
|
+
for (const name of names) {
|
|
1437
|
+
if (!name.endsWith('.tar.zst') && !name.endsWith('.tree')) continue
|
|
1438
|
+
const to = path.join(storeDir, name)
|
|
1439
|
+
if (existsSync(to)) continue
|
|
1440
|
+
try {
|
|
1441
|
+
renameSync(path.join(cacheDir, name), to)
|
|
1442
|
+
} catch {
|
|
1443
|
+
// Another device: the artifact stays an orphan here.
|
|
1444
|
+
}
|
|
1445
|
+
}
|
|
1446
|
+
}
|
|
1447
|
+
this.db
|
|
1448
|
+
.transaction(() => {
|
|
1449
|
+
// `entries` first: the other tables' rows reference it.
|
|
1450
|
+
for (const t of STORE_TABLES.toReversed()) {
|
|
1451
|
+
if (t !== 'store_meta' && has('main', t) && has('store', t)) {
|
|
1452
|
+
this.db.exec(`INSERT OR IGNORE INTO store.${t} SELECT * FROM main.${t}`)
|
|
1453
|
+
}
|
|
1454
|
+
}
|
|
1455
|
+
if (
|
|
1456
|
+
has('main', 'artifacts') &&
|
|
1457
|
+
has('store', 'artifacts') &&
|
|
1458
|
+
layout('main') === ARTIFACT_LAYOUT &&
|
|
1459
|
+
layout('store') === ARTIFACT_LAYOUT
|
|
1460
|
+
) {
|
|
1461
|
+
this.db.exec('INSERT OR IGNORE INTO store.artifacts SELECT * FROM main.artifacts')
|
|
1462
|
+
}
|
|
1463
|
+
for (const t of [...STORE_TABLES, ...ARTIFACT_TABLES]) {
|
|
1464
|
+
this.db.exec(`DROP TABLE IF EXISTS main.${t}`)
|
|
1465
|
+
}
|
|
1466
|
+
})
|
|
1467
|
+
.immediate()
|
|
1468
|
+
}
|
|
1469
|
+
|
|
1399
1470
|
/**
|
|
1400
1471
|
* Attach the shared store as `store`, with the pragmas the index takes:
|
|
1401
1472
|
* WAL for both, so a workspace's run and another's share the store as
|
|
@@ -1580,7 +1651,32 @@ export class Cache implements CacheLayer {
|
|
|
1580
1651
|
}
|
|
1581
1652
|
/** `BlobSizeMemo`: remember blob sizes, honouring the local WRITE axis. */
|
|
1582
1653
|
rememberBlobSizes(sizes: ReadonlyMap<string, number>): void {
|
|
1583
|
-
this.
|
|
1654
|
+
if (this.heldBlobSizes !== undefined) for (const e of sizes) this.heldBlobSizes.push(e)
|
|
1655
|
+
else this.guard(() => this.files.rememberBlobSizes(sizes))
|
|
1656
|
+
}
|
|
1657
|
+
/**
|
|
1658
|
+
* Hold the blob sizes remembered from now on until `storeHeldBlobSizes`
|
|
1659
|
+
* or `close`: a run learns them before its first task, where storing
|
|
1660
|
+
* 40,030 cost a cold 9,603-task run 217 ms (2026-11-01).
|
|
1661
|
+
*/
|
|
1662
|
+
holdBlobSizes(): void {
|
|
1663
|
+
this.heldBlobSizes ??= []
|
|
1664
|
+
}
|
|
1665
|
+
/**
|
|
1666
|
+
* Store the held blob sizes `HELD_SLICE` per event-loop turn from the
|
|
1667
|
+
* next turn on, while the run's tasks run; `close` stores the rest.
|
|
1668
|
+
*/
|
|
1669
|
+
storeHeldBlobSizes(): void {
|
|
1670
|
+
const held = this.heldBlobSizes
|
|
1671
|
+
this.heldBlobSizes = undefined
|
|
1672
|
+
if (held === undefined || held.length === 0) return
|
|
1673
|
+
let at = 0
|
|
1674
|
+
const step = (): void => {
|
|
1675
|
+
this.guard(() => this.files.rememberBlobSizes(new Map(held.slice(at, (at += HELD_SLICE)))))
|
|
1676
|
+
this.heldTimer = at < held.length ? setTimeout(step, 0) : undefined
|
|
1677
|
+
}
|
|
1678
|
+
this.heldRest = () => held.slice(at)
|
|
1679
|
+
this.heldTimer = setTimeout(step, 0)
|
|
1584
1680
|
}
|
|
1585
1681
|
/** `BlobSizeMemo`: the paths an index with this digest distrusts, when known. */
|
|
1586
1682
|
blobVerdict(digest: string): string[] | undefined {
|
|
@@ -1755,7 +1851,7 @@ export class Cache implements CacheLayer {
|
|
|
1755
1851
|
}
|
|
1756
1852
|
|
|
1757
1853
|
/** An adoption's verdict: an artifact that fails the checks is a miss. */
|
|
1758
|
-
private async adopted(indexing: Promise<
|
|
1854
|
+
private async adopted(indexing: Promise<unknown>): Promise<boolean> {
|
|
1759
1855
|
try {
|
|
1760
1856
|
await indexing
|
|
1761
1857
|
return true
|
|
@@ -1953,6 +2049,11 @@ export class Cache implements CacheLayer {
|
|
|
1953
2049
|
outputsPath(hash: string): string {
|
|
1954
2050
|
return this.tarPath(hash)
|
|
1955
2051
|
}
|
|
2052
|
+
outputBytes(hash: string): number | undefined {
|
|
2053
|
+
const bytes = this.savedBytes.get(hash)
|
|
2054
|
+
this.savedBytes.delete(hash)
|
|
2055
|
+
return bytes
|
|
2056
|
+
}
|
|
1956
2057
|
artifactSize(hash: string): number | undefined {
|
|
1957
2058
|
return this.guard(
|
|
1958
2059
|
() =>
|
|
@@ -2046,7 +2147,15 @@ export class Cache implements CacheLayer {
|
|
|
2046
2147
|
* uploaded under a fresh sum.
|
|
2047
2148
|
*/
|
|
2048
2149
|
private treePlan(hash: string, tree: string, rows: readonly OutputFileRow[]): ArtifactPlan {
|
|
2049
|
-
|
|
2150
|
+
let plan: ArtifactPlan
|
|
2151
|
+
try {
|
|
2152
|
+
plan = planOfTree(tree, this.stdoutText(hash))
|
|
2153
|
+
} catch (err) {
|
|
2154
|
+
// A sidecar cut short at rest, as a file is: named as the damage it is.
|
|
2155
|
+
if (err instanceof SyntaxError)
|
|
2156
|
+
throw new CorruptArtifactError(hash, 'its sidecar does not parse')
|
|
2157
|
+
throw err
|
|
2158
|
+
}
|
|
2050
2159
|
const size = new Map(rows.map((r) => [treeName(r.path), r.size]))
|
|
2051
2160
|
for (const i of plan.inputs) {
|
|
2052
2161
|
const want = size.get(i.name)
|
|
@@ -2488,7 +2597,16 @@ export class Cache implements CacheLayer {
|
|
|
2488
2597
|
if (!this.write) return
|
|
2489
2598
|
// A cache that reads saves only on a miss. One that does not (`--force`)
|
|
2490
2599
|
// re-executes a key it may hold: that entry is kept as it is.
|
|
2491
|
-
|
|
2600
|
+
// The kept entry's bytes are this run's too (same key): the row's
|
|
2601
|
+
// `outputs` counts them as a hit would.
|
|
2602
|
+
if (!this.read && this.guard(() => this.storedHere(args.hash))) {
|
|
2603
|
+
const rows = this.loadOutputFilesBatch([args.hash]).get(args.hash) ?? []
|
|
2604
|
+
this.savedBytes.set(
|
|
2605
|
+
args.hash,
|
|
2606
|
+
rows.reduce((n, r) => n + r.size, 0),
|
|
2607
|
+
)
|
|
2608
|
+
return
|
|
2609
|
+
}
|
|
2492
2610
|
// The cache directory is the constructor's (`mkdirSync`, with a
|
|
2493
2611
|
// refusal recorded for `assertWritable`); re-creating it here and
|
|
2494
2612
|
// again in `writeArtifactAndIndex` cost every save two thread-pool
|
|
@@ -2505,7 +2623,7 @@ export class Cache implements CacheLayer {
|
|
|
2505
2623
|
throw err
|
|
2506
2624
|
})
|
|
2507
2625
|
endPack()
|
|
2508
|
-
await this.guard(() =>
|
|
2626
|
+
const bytes = await this.guard(() =>
|
|
2509
2627
|
this.writeArtifactAndIndex(
|
|
2510
2628
|
args.hash,
|
|
2511
2629
|
'tar' in packed ? packed.compressed : 'plan' in packed ? packed.saved : packed,
|
|
@@ -2518,6 +2636,7 @@ export class Cache implements CacheLayer {
|
|
|
2518
2636
|
'tar' in packed ? packed.tar : undefined,
|
|
2519
2637
|
),
|
|
2520
2638
|
)
|
|
2639
|
+
this.savedBytes.set(args.hash, bytes)
|
|
2521
2640
|
}
|
|
2522
2641
|
|
|
2523
2642
|
/**
|
|
@@ -2827,7 +2946,7 @@ export class Cache implements CacheLayer {
|
|
|
2827
2946
|
* scanned as is rather than decoded back out of the bytes just encoded.
|
|
2828
2947
|
*/
|
|
2829
2948
|
tar?: Uint8Array,
|
|
2830
|
-
): Promise<
|
|
2949
|
+
): Promise<number> {
|
|
2831
2950
|
// Validate BEFORE anything touches the final path. `ingest()` feeds
|
|
2832
2951
|
// us a temp of network bytes; a truncated/garbage body that went live first
|
|
2833
2952
|
// would leave a corrupt `<hash>.tar.zst` behind (with no SQL row,
|
|
@@ -3049,6 +3168,7 @@ export class Cache implements CacheLayer {
|
|
|
3049
3168
|
const upsertStdout = this.upsertStdout
|
|
3050
3169
|
const deleteStdout = this.deleteStdout
|
|
3051
3170
|
const outputs = this.outputs
|
|
3171
|
+
let outputBytes = 0
|
|
3052
3172
|
const insertEntryInput = this.insertEntryInput
|
|
3053
3173
|
const inputComponents = meta.inputComponents
|
|
3054
3174
|
// The artifact goes live INSIDE the write transaction, taken
|
|
@@ -3140,7 +3260,7 @@ export class Cache implements CacheLayer {
|
|
|
3140
3260
|
hash,
|
|
3141
3261
|
stdoutText.startsWith('\uFEFF') ? utf8.encode(stdoutText) : stdoutText,
|
|
3142
3262
|
)
|
|
3143
|
-
outputs.replaceFileRows(hash, outputFileRows)
|
|
3263
|
+
outputBytes = outputs.replaceFileRows(hash, outputFileRows)
|
|
3144
3264
|
// INSERT OR IGNORE: identical inputs derive this same hash, so a
|
|
3145
3265
|
// re-save's row is identical — keep the first, skip the rest.
|
|
3146
3266
|
if (inputComponents !== undefined && inputComponents.length > 0) {
|
|
@@ -3152,7 +3272,7 @@ export class Cache implements CacheLayer {
|
|
|
3152
3272
|
// Built inside the try: on a database a stopping run closed while a
|
|
3153
3273
|
// download streamed in, building it throws, and the temp or tree
|
|
3154
3274
|
// the body went to stayed behind.
|
|
3155
|
-
this.
|
|
3275
|
+
await this.indexInBatch(index)
|
|
3156
3276
|
} catch (err) {
|
|
3157
3277
|
endTx()
|
|
3158
3278
|
const left = renamed ?? tree ?? tmpPath
|
|
@@ -3162,6 +3282,57 @@ export class Cache implements CacheLayer {
|
|
|
3162
3282
|
}
|
|
3163
3283
|
endTx()
|
|
3164
3284
|
for (const a of aside) await rm(a, { recursive: true, force: true })
|
|
3285
|
+
return outputBytes
|
|
3286
|
+
}
|
|
3287
|
+
|
|
3288
|
+
/**
|
|
3289
|
+
* Run `write` in a write transaction shared with every other save's that
|
|
3290
|
+
* arrives in this turn: settles once it has committed, or rejects with
|
|
3291
|
+
* its own error (its savepoint rolled back) or the commit's. A cold run's
|
|
3292
|
+
* saves land a few per turn, and a transaction each spent half its index
|
|
3293
|
+
* time on BEGIN and COMMIT on the main thread, the run's busiest
|
|
3294
|
+
* (2026-11-03: 81 of 164 ms over 500 saves).
|
|
3295
|
+
*/
|
|
3296
|
+
private indexInBatch(write: () => void): Promise<void> {
|
|
3297
|
+
return new Promise((resolve, reject) => {
|
|
3298
|
+
this.indexQueue.push({
|
|
3299
|
+
write,
|
|
3300
|
+
settle: (err) => (err === undefined ? resolve() : reject(err)),
|
|
3301
|
+
})
|
|
3302
|
+
if (this.indexQueue.length === 1) setImmediate(() => this.flushIndex())
|
|
3303
|
+
})
|
|
3304
|
+
}
|
|
3305
|
+
|
|
3306
|
+
private flushIndex(): void {
|
|
3307
|
+
const batch = this.indexQueue
|
|
3308
|
+
if (batch.length === 0) return
|
|
3309
|
+
this.indexQueue = []
|
|
3310
|
+
const failed = new Map<number, unknown>()
|
|
3311
|
+
try {
|
|
3312
|
+
this.db
|
|
3313
|
+
.transaction(() => {
|
|
3314
|
+
for (let i = 0; i < batch.length; i++) {
|
|
3315
|
+
// An error SQLite answers by rolling the whole transaction
|
|
3316
|
+
// back (a full disk, a busy lock) leaves none to go on in: the
|
|
3317
|
+
// rest are not run, and fail with it.
|
|
3318
|
+
if (failed.size > 0 && !this.db.inTransaction) {
|
|
3319
|
+
failed.set(i, failed.values().next().value)
|
|
3320
|
+
continue
|
|
3321
|
+
}
|
|
3322
|
+
try {
|
|
3323
|
+
// Nested: a savepoint, so one write's failure undoes only its own rows.
|
|
3324
|
+
this.db.transaction(batch[i]!.write)()
|
|
3325
|
+
} catch (err) {
|
|
3326
|
+
failed.set(i, err)
|
|
3327
|
+
}
|
|
3328
|
+
}
|
|
3329
|
+
})
|
|
3330
|
+
.immediate()
|
|
3331
|
+
} catch (err) {
|
|
3332
|
+
for (let i = 0; i < batch.length; i++) batch[i]!.settle(failed.get(i) ?? err)
|
|
3333
|
+
return
|
|
3334
|
+
}
|
|
3335
|
+
for (let i = 0; i < batch.length; i++) batch[i]!.settle(failed.get(i))
|
|
3165
3336
|
}
|
|
3166
3337
|
|
|
3167
3338
|
private tempTreePath(hash: string): string {
|
|
@@ -3515,6 +3686,12 @@ export class Cache implements CacheLayer {
|
|
|
3515
3686
|
|
|
3516
3687
|
close(): void {
|
|
3517
3688
|
try {
|
|
3689
|
+
this.flushIndex()
|
|
3690
|
+
clearTimeout(this.heldTimer)
|
|
3691
|
+
const held = this.heldBlobSizes ?? (this.heldTimer !== undefined ? this.heldRest() : [])
|
|
3692
|
+
this.heldBlobSizes = undefined
|
|
3693
|
+
this.heldTimer = undefined
|
|
3694
|
+
if (held.length > 0) this.files.rememberBlobSizes(new Map(held))
|
|
3518
3695
|
this.flushAccessed()
|
|
3519
3696
|
this.outputs.flushOutputDirs()
|
|
3520
3697
|
this.folds.flush()
|
package/src/cache/file-hashes.ts
CHANGED
|
@@ -72,6 +72,9 @@ export class FileHashStore {
|
|
|
72
72
|
/** Each of `oids` whose size a `rememberBlobSizes` stored: one `IN` query per 900. */
|
|
73
73
|
knownBlobSizes(oids: readonly string[]): Map<string, number> {
|
|
74
74
|
const out = new Map<string, number>()
|
|
75
|
+
// An empty memo (a cold cache) answers nothing: 40,030 lookups there
|
|
76
|
+
// cost a 9,603-task run 57 ms before its first task (2026-11-01).
|
|
77
|
+
if (this.db.query('SELECT 1 FROM blob_sizes LIMIT 1').get() === null) return out
|
|
75
78
|
for (let i = 0; i < oids.length; i += 900) {
|
|
76
79
|
const chunk = oids.slice(i, i + 900)
|
|
77
80
|
const rows = this.db
|