@vzn/vx 0.0.644 → 0.0.646
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +22 -48
- package/package.json +5 -5
- package/schemas/init.json +5 -0
- package/schemas/last.json +15 -0
- package/skills/vx/SKILL.md +1 -1
- package/src/bin.ts +1 -1
- package/src/cache/archive.ts +12 -76
- package/src/cache/cache.ts +72 -42
- package/src/cache/git-inputs.ts +55 -10
- package/src/cache/index.ts +5 -2
- package/src/cache/inputs.ts +61 -11
- package/src/cache/layer.ts +30 -8
- package/src/cache/layered-cache.ts +138 -44
- package/src/cache/output-index.ts +25 -11
- package/src/cache/plugin-cache.ts +9 -14
- package/src/cache/remote-lane.ts +60 -22
- package/src/cache/run-history.ts +13 -6
- package/src/cache/schema.ts +19 -2
- package/src/cache/scoped-cache.ts +3 -80
- package/src/cli/core-alias.ts +5 -2
- package/src/cli/help.ts +8 -10
- package/src/cli/index.ts +0 -12
- package/src/cli/init.ts +159 -25
- package/src/cli/last.ts +4 -2
- package/src/cli/plan-format.ts +6 -2
- package/src/cli/plugin-commands.ts +98 -6
- package/src/cli/plugin-templates.ts +2 -2
- package/src/cli/run.ts +2 -8
- package/src/cli/select.ts +5 -3
- package/src/cli/upgrade.ts +5 -6
- package/src/config.ts +2 -4
- package/src/exec/executor.ts +112 -9
- package/src/exec/index.ts +2 -0
- package/src/exec/proc-sample.ts +67 -5
- package/src/exec/runner.ts +4 -0
- package/src/exec/sandbox-runtime.ts +11 -4
- package/src/graph/scheduler.ts +59 -8
- package/src/history/history.ts +119 -7
- package/src/history/metrics.ts +53 -16
- package/src/index.ts +16 -1
- package/src/orchestrator/affected-tasks.ts +0 -18
- package/src/orchestrator/execute-args.ts +3 -0
- package/src/orchestrator/execute-task.ts +28 -7
- package/src/orchestrator/held-servers.ts +86 -0
- package/src/orchestrator/hit-restore.ts +11 -4
- package/src/orchestrator/index.ts +18 -1
- package/src/orchestrator/local-floor.ts +104 -0
- package/src/orchestrator/local-shortcircuit.ts +26 -7
- package/src/orchestrator/miss-save.ts +10 -4
- package/src/orchestrator/options.ts +0 -8
- package/src/orchestrator/persistent-task.ts +14 -0
- package/src/orchestrator/placement.ts +84 -9
- package/src/orchestrator/plan.ts +7 -1
- package/src/orchestrator/plugin-host.ts +22 -7
- package/src/orchestrator/plugin.ts +48 -7
- package/src/orchestrator/prepare.ts +14 -49
- package/src/orchestrator/run-context.ts +39 -1
- package/src/orchestrator/run-keys.ts +164 -0
- package/src/orchestrator/run-records.ts +6 -0
- package/src/orchestrator/run.ts +95 -6
- package/src/orchestrator/sandbox-request.ts +9 -3
- package/src/orchestrator/task-artifacts.ts +142 -0
- package/src/orchestrator/task-hash.ts +12 -4
- package/src/orchestrator/task-process.ts +16 -0
- package/src/orchestrator/telemetry-host.ts +2 -1
- package/src/orchestrator/telemetry.ts +60 -17
- package/src/report/forecast.ts +6 -27
- package/src/report/framed-output.ts +41 -5
- package/src/report/logger.ts +60 -20
- package/src/report/status-line.ts +38 -18
- package/src/report/summary.ts +75 -9
- package/src/settings.ts +8 -7
- package/src/util/colors.ts +16 -1
- package/src/util/index.ts +1 -0
- package/src/util/paths.ts +30 -0
- package/src/util/ui.ts +4 -1
- package/src/workspace/affected.ts +0 -39
- package/src/workspace/config-bytecode.ts +276 -0
- package/src/workspace/index.ts +0 -2
- package/src/workspace/migration.ts +4 -32
- package/src/workspace/project-loader.ts +92 -14
- package/src/cli/foreign-flags.ts +0 -727
package/LICENSE
CHANGED
package/README.md
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
|
|
13
13
|
<p align="center">
|
|
14
14
|
<a href="https://www.npmjs.com/package/@vzn/vx"><img src="https://img.shields.io/npm/v/@vzn/vx?color=7cb518" alt="npm"></a>
|
|
15
|
-
<a href="https://github.com/vznjs/vx/actions/workflows/ci.yml"><img src="https://github.com/vznjs/vx/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
|
|
15
|
+
<a href="https://github.com/vznjs/vx/actions/workflows/ci.yml?query=branch%3Amain+event%3Apush"><img src="https://github.com/vznjs/vx/actions/workflows/ci.yml/badge.svg?branch=main&event=push" alt="CI"></a>
|
|
16
16
|
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue" alt="MIT license"></a>
|
|
17
17
|
<a href="https://vznjs.github.io/vx/"><img src="https://img.shields.io/badge/docs-vznjs.github.io%2Fvx-c6f84e" alt="Documentation"></a>
|
|
18
18
|
</p>
|
|
@@ -23,38 +23,19 @@ Task orchestration with a cache you can trust, on a core you can extend.
|
|
|
23
23
|
Each task is one shell command with declared inputs and outputs; vx runs
|
|
24
24
|
them in dependency order and replays results from a content-addressed cache.
|
|
25
25
|
|
|
26
|
-
## Install
|
|
27
|
-
|
|
28
|
-
```sh
|
|
29
|
-
npm install -D @vzn/vx # or: pnpm add -D -w · yarn add -D (-W on Yarn 1) · bun add -d
|
|
30
|
-
```
|
|
31
|
-
|
|
32
|
-
A prebuilt binary for Linux (glibc, not Alpine's musl) and macOS, x64 and arm64; no Bun needed. The
|
|
33
|
-
[release](https://github.com/vznjs/vx/releases) binary needs no Node either.
|
|
34
|
-
Windows: run vx inside WSL2.
|
|
35
|
-
|
|
36
26
|
## Quick start
|
|
37
27
|
|
|
38
|
-
In a git repo
|
|
28
|
+
In a git repo:
|
|
39
29
|
|
|
40
30
|
```sh
|
|
41
|
-
|
|
42
|
-
npx @vzn/vx init # beside turbo.json or nx.json: migrates them; else a vx.config.ts per package
|
|
43
|
-
npx @vzn/vx run build --all # builds in dependency order and stores the results
|
|
44
|
-
npx @vzn/vx run build --all # nothing changed: every task up-to-date
|
|
31
|
+
bunx @vzn/vx init # installs vx, writes an empty root vx.config.ts and the lockfile plugin
|
|
45
32
|
```
|
|
46
33
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
Without Turbo or Nx, tasks run uncached until you declare what they read (below).
|
|
34
|
+
A prebuilt binary for Linux (glibc, not Alpine's musl) and macOS, x64 and
|
|
35
|
+
arm64. Windows: run vx inside WSL2.
|
|
51
36
|
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
### Without Turbo or Nx
|
|
56
|
-
|
|
57
|
-
`init` writes one task per package script, uncached. Cache one by saying what it reads and writes:
|
|
37
|
+
A task runs its tool directly, never through a package script. Cache one by
|
|
38
|
+
saying what it reads and writes:
|
|
58
39
|
|
|
59
40
|
```ts
|
|
60
41
|
// packages/app/vx.config.ts
|
|
@@ -75,10 +56,10 @@ export default defineProject({
|
|
|
75
56
|
```
|
|
76
57
|
|
|
77
58
|
```sh
|
|
78
|
-
|
|
79
|
-
|
|
59
|
+
vx run build --all # builds, in dependency order; stores the results
|
|
60
|
+
vx run build --all # nothing changed: up-to-date, in milliseconds
|
|
80
61
|
rm -rf packages/*/dist
|
|
81
|
-
|
|
62
|
+
vx run build --all # restored from the cache, not rebuilt
|
|
82
63
|
```
|
|
83
64
|
|
|
84
65
|
<img src="https://vznjs.github.io/vx/demo.svg" width="760" alt="vx run ci --all on examples/basic: cold, three tasks run; again, three up-to-date.">
|
|
@@ -87,7 +68,7 @@ npx @vzn/vx run build --all # restored from the cache, not rebuilt
|
|
|
87
68
|
|
|
88
69
|
`init` also writes `.vx/projects.lock.json`, every project's config as it evaluated.
|
|
89
70
|
Commit it. Every run reads its configs from the lock, so run
|
|
90
|
-
`
|
|
71
|
+
`vx lock` and commit the lock again after you:
|
|
91
72
|
|
|
92
73
|
- edit any `vx.config.ts` or `.vx/workspace.ts`,
|
|
93
74
|
- update dependencies or plugins,
|
|
@@ -135,7 +116,7 @@ Write your own: [plugin guide](https://vznjs.github.io/vx/guides/plugins/) ·
|
|
|
135
116
|
|
|
136
117
|
## Coming from Turbo or Nx
|
|
137
118
|
|
|
138
|
-
Beside `turbo.json` or `nx.json`, `
|
|
119
|
+
Beside `turbo.json` or `nx.json`, `bunx @vzn/vx init` runs `@vzn/vx-migrate`:
|
|
139
120
|
it writes a `vx.config.ts` per package from `turbo.json` or the Nx graph,
|
|
140
121
|
and you keep those and delete the old config. `vx init --keep` writes only
|
|
141
122
|
a `.vx/workspace.ts` that maps the old config instead, and its lock: a temporary start,
|
|
@@ -166,17 +147,17 @@ Every gap, and when another tool is the better pick:
|
|
|
166
147
|
|
|
167
148
|
<!-- bench:start — generated by packages/vx-bench/update-site.ts from results.json; do not hand-edit -->
|
|
168
149
|
|
|
169
|
-
| Time added over the ideal run, 1,601 projects, 9,603 tasks | vx
|
|
170
|
-
| ---------------------------------------------------------- |
|
|
171
|
-
| Nothing changed | **
|
|
172
|
-
| Nothing changed, outputs restored | **1.
|
|
173
|
-
| Cold build | **
|
|
174
|
-
| Cold build: CPU the runner burns |
|
|
150
|
+
| Time added over the ideal run, 1,601 projects, 9,603 tasks | vx | Turborepo | Nx | Vite Task |
|
|
151
|
+
| ---------------------------------------------------------- | ---------- | --------- | ---------- | --------- |
|
|
152
|
+
| Nothing changed | **757 ms** | 761 ms | 22.63 s | 12.26 s |
|
|
153
|
+
| Nothing changed, outputs restored | **1.27 s** | 1.76 s | 22.16 s | 11.66 s |
|
|
154
|
+
| Cold build | **6.13 s** | 8.87 s | 3 min 20 s | 7.94 s |
|
|
155
|
+
| Cold build: CPU the runner burns | 30.06 s | 1 min 8 s | 5 min 45 s | 23.91 s |
|
|
175
156
|
|
|
176
157
|
Time each tool adds over the ideal run; lower is better.
|
|
177
158
|
|
|
178
159
|
Benchmark workload: a synthetic monorepo of 1,601 projects and 9,603 tasks in 30 dependency levels, five core libraries a quarter of the projects use; build 300 ms, test and typecheck 150 ms, lint 75 ms, publish 30 ms; real repos with uneven task times will differ.
|
|
179
|
-
Run 2026-10-
|
|
160
|
+
Run 2026-10-20 on linux x64, 4 cores: vx at commit a88652b0b, Turborepo 2.11.9, Nx 23.3.0, Vite Task (vite-plus) 1.3.0.
|
|
180
161
|
|
|
181
162
|
Same graph, commands and concurrency: [how it is measured](https://vznjs.github.io/vx/benchmarks/).
|
|
182
163
|
|
|
@@ -188,17 +169,10 @@ Same graph, commands and concurrency: [how it is measured](https://vznjs.github.
|
|
|
188
169
|
vx. The schema may still change; 1.0's promise:
|
|
189
170
|
[versioning-1.0.md](packages/vx/docs/design/versioning-1.0.md).
|
|
190
171
|
|
|
191
|
-
##
|
|
192
|
-
|
|
193
|
-
```sh
|
|
194
|
-
git clone https://github.com/vznjs/vx && cd vx
|
|
195
|
-
bun install
|
|
196
|
-
bun packages/vx/src/bin.ts run ci --all # lint, test, docs build: every package
|
|
197
|
-
```
|
|
172
|
+
## Issues
|
|
198
173
|
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
[STATUS.md](packages/vx/docs/STATUS.md).
|
|
174
|
+
vx is maintained by VZN and takes issues, not pull requests: report a
|
|
175
|
+
bug or ask for a feature in [issues](https://github.com/vznjs/vx/issues).
|
|
202
176
|
|
|
203
177
|
## License
|
|
204
178
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vzn/vx",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.646",
|
|
4
4
|
"description": "Task orchestration you own: correct, sandboxed, extensible, with a content-addressed build cache.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -21,10 +21,10 @@
|
|
|
21
21
|
"node": ">=18"
|
|
22
22
|
},
|
|
23
23
|
"optionalDependencies": {
|
|
24
|
-
"@vzn/vx-linux-x64": "0.0.
|
|
25
|
-
"@vzn/vx-linux-arm64": "0.0.
|
|
26
|
-
"@vzn/vx-darwin-x64": "0.0.
|
|
27
|
-
"@vzn/vx-darwin-arm64": "0.0.
|
|
24
|
+
"@vzn/vx-linux-x64": "0.0.646",
|
|
25
|
+
"@vzn/vx-linux-arm64": "0.0.646",
|
|
26
|
+
"@vzn/vx-darwin-x64": "0.0.646",
|
|
27
|
+
"@vzn/vx-darwin-arm64": "0.0.646"
|
|
28
28
|
},
|
|
29
29
|
"dependencies": {
|
|
30
30
|
"@anthropic-ai/sandbox-runtime": "^0.0.76"
|
package/schemas/init.json
CHANGED
|
@@ -52,6 +52,11 @@
|
|
|
52
52
|
"next": {
|
|
53
53
|
"type": ["string", "null"],
|
|
54
54
|
"description": "the command to type after the real run; null where the plan names none"
|
|
55
|
+
},
|
|
56
|
+
"installs": {
|
|
57
|
+
"type": "array",
|
|
58
|
+
"items": { "type": "string" },
|
|
59
|
+
"description": "core's plan only: the install commands the real run runs (global, then the root devDependency)"
|
|
55
60
|
}
|
|
56
61
|
}
|
|
57
62
|
}
|
package/schemas/last.json
CHANGED
|
@@ -166,6 +166,9 @@
|
|
|
166
166
|
"hash",
|
|
167
167
|
"cpuMs",
|
|
168
168
|
"peakRssBytes",
|
|
169
|
+
"peakCores",
|
|
170
|
+
"avgRssBytes",
|
|
171
|
+
"placement",
|
|
169
172
|
"wallclockStartNs",
|
|
170
173
|
"wallclockEndNs",
|
|
171
174
|
"blockedBy",
|
|
@@ -199,6 +202,18 @@
|
|
|
199
202
|
"hash": { "type": "string" },
|
|
200
203
|
"cpuMs": { "type": ["number", "null"] },
|
|
201
204
|
"peakRssBytes": { "type": ["integer", "null"] },
|
|
205
|
+
"peakCores": {
|
|
206
|
+
"type": ["number", "null"],
|
|
207
|
+
"description": "most cores one sampling interval burned; null when nothing sampled"
|
|
208
|
+
},
|
|
209
|
+
"avgRssBytes": {
|
|
210
|
+
"type": ["integer", "null"],
|
|
211
|
+
"description": "mean resident memory over the samples; null when nothing sampled"
|
|
212
|
+
},
|
|
213
|
+
"placement": {
|
|
214
|
+
"type": ["string", "null"],
|
|
215
|
+
"description": "where an executor ran it (small#1); null when here"
|
|
216
|
+
},
|
|
202
217
|
"wallclockStartNs": {
|
|
203
218
|
"type": ["string", "null"],
|
|
204
219
|
"description": "decimal nanoseconds from the run's start"
|
package/skills/vx/SKILL.md
CHANGED
|
@@ -106,7 +106,7 @@ vx info --format json # vx and Bun versions, projects, config errors, plugins,
|
|
|
106
106
|
|
|
107
107
|
## Install
|
|
108
108
|
|
|
109
|
-
`
|
|
109
|
+
`bunx skills add vznjs/vx` installs this skill. To match the vx a workspace
|
|
110
110
|
has installed, copy it from there instead:
|
|
111
111
|
`cp node_modules/@vzn/vx/skills/vx/SKILL.md .claude/skills/vx/`.
|
|
112
112
|
`vx upgrade` inside a workspace moves the project's `@vzn/vx` and the
|
package/src/bin.ts
CHANGED
|
@@ -39,7 +39,7 @@ async function main(): Promise<void> {
|
|
|
39
39
|
// package, a workspace or project config — resolves to THIS core, not to
|
|
40
40
|
// a second copy from node_modules (core-alias.ts says why and what it
|
|
41
41
|
// costs). Registered before any verb runs; the façade loads on first use.
|
|
42
|
-
registerCoreAlias(() =>
|
|
42
|
+
registerCoreAlias(() => require('./index.js') as Record<string, unknown>)
|
|
43
43
|
const code = await run(argv)
|
|
44
44
|
// Not `process.exit` here: Bun drops what a pipe has not yet taken
|
|
45
45
|
// when it follows a large write (300 KB written, 64 KiB delivered;
|
package/src/cache/archive.ts
CHANGED
|
@@ -163,8 +163,6 @@ class SumCheck {
|
|
|
163
163
|
add(bytes: Uint8Array): void {
|
|
164
164
|
this.sum.add(bytes)
|
|
165
165
|
}
|
|
166
|
-
/** The checked sum's text, once `verify` passed. */
|
|
167
|
-
checked = ''
|
|
168
166
|
verify(got: string): void {
|
|
169
167
|
if (got !== this.sum.hex()) {
|
|
170
168
|
throw new TarFormatError(
|
|
@@ -172,7 +170,6 @@ class SumCheck {
|
|
|
172
170
|
)
|
|
173
171
|
}
|
|
174
172
|
this.seen = true
|
|
175
|
-
this.checked = got
|
|
176
173
|
}
|
|
177
174
|
done(): void {
|
|
178
175
|
if (!this.seen) throw new TarFormatError('artifact carries no checksum')
|
|
@@ -478,14 +475,11 @@ function streamOf(gen: AsyncGenerator<Uint8Array>): ReadableStream<Uint8Array> {
|
|
|
478
475
|
/**
|
|
479
476
|
* The tree form (`cache.store.decompress`, v47): an artifact's members
|
|
480
477
|
* as plain files under one directory, `outputs/<rel>` and
|
|
481
|
-
* `workspace-outputs/<rel>` as named in the tar, beside its sidecar
|
|
482
|
-
*
|
|
483
|
-
*
|
|
484
|
-
*
|
|
485
|
-
|
|
486
|
-
const TREE_SUM = SUM_ENTRY
|
|
487
|
-
|
|
488
|
-
/**
|
|
478
|
+
* `workspace-outputs/<rel>` as named in the tar, beside its sidecar.
|
|
479
|
+
* stdout lives in the index. A restore copies (or clones) each file out,
|
|
480
|
+
* so a hit decodes nothing; an upload packs the tree back into the
|
|
481
|
+
* artifact a remote expects.
|
|
482
|
+
*
|
|
489
483
|
* Write `plan`'s outputs into the empty `dir` as the tree form, each copied
|
|
490
484
|
* by the kernel (`copy_file_range`, or a clone where the filesystem has one)
|
|
491
485
|
* and given its sidecar mtime, and return what a scan of the plan's tar
|
|
@@ -515,24 +509,14 @@ export async function writeTree(plan: ArtifactPlan, dir: string): Promise<Artifa
|
|
|
515
509
|
}),
|
|
516
510
|
)
|
|
517
511
|
await writeFile(path.join(dir, META_ENTRY), meta)
|
|
518
|
-
await writeFile(path.join(dir, TREE_SUM), await treeSum(dir, stdoutOf(plan)))
|
|
519
512
|
return fold.result()
|
|
520
513
|
}
|
|
521
514
|
|
|
522
|
-
function stdoutOf(plan: ArtifactPlan): string {
|
|
523
|
-
return utf8.decode(plan.inputs.find((i) => i.name === STDOUT_ENTRY)!.body as Uint8Array)
|
|
524
|
-
}
|
|
525
|
-
|
|
526
515
|
/**
|
|
527
|
-
* Write a scanned artifact's sidecar
|
|
528
|
-
*
|
|
516
|
+
* Write a scanned artifact's sidecar into `dir`, whose outputs an extract
|
|
517
|
+
* just wrote: an ingested remote hit kept as a tree.
|
|
529
518
|
*/
|
|
530
|
-
export async function sealTree(
|
|
531
|
-
dir: string,
|
|
532
|
-
scan: ArtifactScan,
|
|
533
|
-
/** The archive's own sidecar text and checked sum (`extractArtifactStream`'s `onScan`). */
|
|
534
|
-
raw?: { meta: string; sum: string },
|
|
535
|
-
): Promise<void> {
|
|
519
|
+
export async function sealTree(dir: string, scan: ArtifactScan): Promise<void> {
|
|
536
520
|
// An artifact with no outputs extracts nothing, so nothing made `dir`.
|
|
537
521
|
await mkdir(dir, { recursive: true })
|
|
538
522
|
const meta: MetaFile = { version: 1, files: {} }
|
|
@@ -543,20 +527,7 @@ export async function sealTree(
|
|
|
543
527
|
if (scan.dirs.length > 0) meta.dirs = scan.dirs
|
|
544
528
|
if (scan.links.length > 0) meta.links = Object.fromEntries(scan.links)
|
|
545
529
|
if (scan.exec !== undefined) meta.exec = scan.exec
|
|
546
|
-
|
|
547
|
-
await writeFile(path.join(dir, META_ENTRY), text)
|
|
548
|
-
// The tree packs back to the archive it came from when the sidecar is
|
|
549
|
-
// the same bytes and the entries came in its order (stdout first, the
|
|
550
|
-
// sidecar last): the sum the extract checked is then the tree's, and the
|
|
551
|
-
// tree is not read back to sum it.
|
|
552
|
-
const names = Object.keys(meta.files)
|
|
553
|
-
const same =
|
|
554
|
-
raw !== undefined &&
|
|
555
|
-
raw.meta === text &&
|
|
556
|
-
scan.entries[0]?.name === STDOUT_ENTRY &&
|
|
557
|
-
scan.entries.length === names.length + 1 &&
|
|
558
|
-
names.every((n, i) => scan.entries[i + 1]!.name === n)
|
|
559
|
-
await writeFile(path.join(dir, TREE_SUM), same ? raw.sum : await treeSum(dir, scan.stdout ?? ''))
|
|
530
|
+
await writeFile(path.join(dir, META_ENTRY), JSON.stringify(meta))
|
|
560
531
|
}
|
|
561
532
|
|
|
562
533
|
/** The plan that packs the tree in `dir` back into the artifact it holds. */
|
|
@@ -579,28 +550,6 @@ export function planOfTree(dir: string, stdout: string): ArtifactPlan {
|
|
|
579
550
|
return { inputs, size: tarSize([...inputs, sumInput('0'.repeat(SUM_SIZE))]) }
|
|
580
551
|
}
|
|
581
552
|
|
|
582
|
-
/** The sum over the tree's members, as `SUM_ENTRY` defines it for the tar it packs to. */
|
|
583
|
-
async function treeSum(dir: string, stdout: string): Promise<string> {
|
|
584
|
-
const sum = new EntrySum()
|
|
585
|
-
for (const i of planOfTree(dir, stdout).inputs) {
|
|
586
|
-
sum.entry(i.name)
|
|
587
|
-
if (i.body instanceof Blob && i.size > 1 << 20)
|
|
588
|
-
for await (const c of i.body.stream()) sum.add(c)
|
|
589
|
-
else if (i.body instanceof Blob) sum.add(await i.body.bytes())
|
|
590
|
-
else sum.add(i.body as Uint8Array)
|
|
591
|
-
}
|
|
592
|
-
return sum.hex()
|
|
593
|
-
}
|
|
594
|
-
|
|
595
|
-
/** Whether the tree's bytes still sum to what was stored (`cache.store.verify: 'hash'`). */
|
|
596
|
-
export async function treeIntact(dir: string, stdout: string): Promise<boolean> {
|
|
597
|
-
try {
|
|
598
|
-
return (await treeSum(dir, stdout)) === readFileSync(path.join(dir, TREE_SUM), 'utf8')
|
|
599
|
-
} catch {
|
|
600
|
-
return false
|
|
601
|
-
}
|
|
602
|
-
}
|
|
603
|
-
|
|
604
553
|
/** What a scan of an artifact reads out of it. */
|
|
605
554
|
interface ArtifactScan {
|
|
606
555
|
entries: ArchiveEntry[]
|
|
@@ -794,12 +743,8 @@ export async function extractArtifactStream(
|
|
|
794
743
|
accept?: (name: string) => boolean,
|
|
795
744
|
/** The workspace root whose `.vx/tmp/` stages the entries; absent, each stages beside its target. */
|
|
796
745
|
stageRoot?: string,
|
|
797
|
-
/**
|
|
798
|
-
|
|
799
|
-
* has checked, with the sidecar's text and the sum when the sidecar came
|
|
800
|
-
* last: what `sealTree` may reuse rather than read the tree back.
|
|
801
|
-
*/
|
|
802
|
-
onScan?: (scan: ArtifactScan, raw: { meta: string; sum: string } | undefined) => void,
|
|
746
|
+
/** Handed what `scanArtifact` would read of the same bytes, once their sum has checked. */
|
|
747
|
+
onScan?: (scan: ArtifactScan) => void,
|
|
803
748
|
): Promise<Set<string>> {
|
|
804
749
|
// No `mkdir -p` of the destination here: `stage` creates each entry's
|
|
805
750
|
// directory chain, the destination included, and the cache restores
|
|
@@ -813,20 +758,15 @@ export async function extractArtifactStream(
|
|
|
813
758
|
let dirs: string[] = []
|
|
814
759
|
let links: Array<[string, string]> = []
|
|
815
760
|
const fold = onScan === undefined ? undefined : new ScanFold()
|
|
816
|
-
let metaText: string | undefined
|
|
817
|
-
let metaLast = false
|
|
818
761
|
try {
|
|
819
762
|
const check = new SumCheck()
|
|
820
763
|
for await (const e of tarEntries(tar)) {
|
|
821
764
|
if (e.type !== '0') continue
|
|
822
765
|
const body = await check.body(e.name, e.body)
|
|
823
766
|
if (body === null) continue
|
|
824
|
-
metaLast = false
|
|
825
767
|
if (e.name === META_ENTRY) {
|
|
826
768
|
const text = await textOf(body)
|
|
827
769
|
fold?.sidecar(text)
|
|
828
|
-
metaText = text
|
|
829
|
-
metaLast = true
|
|
830
770
|
const parsed = JSON.parse(text) as MetaFile
|
|
831
771
|
meta = parsed.files ?? {}
|
|
832
772
|
dirs = dirsOf(parsed)
|
|
@@ -855,11 +795,7 @@ export async function extractArtifactStream(
|
|
|
855
795
|
await x.stage(e.name, dest.base, target, e.size <= SMALL_ENTRY ? await bytesOf(body) : body)
|
|
856
796
|
}
|
|
857
797
|
check.done()
|
|
858
|
-
if (fold !== undefined)
|
|
859
|
-
onScan!(
|
|
860
|
-
fold.result(),
|
|
861
|
-
metaLast && metaText !== undefined ? { meta: metaText, sum: check.checked } : undefined,
|
|
862
|
-
)
|
|
798
|
+
if (fold !== undefined) onScan!(fold.result())
|
|
863
799
|
for (const name of dirs) {
|
|
864
800
|
if (accept !== undefined && !accept(name)) {
|
|
865
801
|
throw new ArchiveSecurityError(
|
package/src/cache/cache.ts
CHANGED
|
@@ -78,7 +78,6 @@ import {
|
|
|
78
78
|
planArtifact,
|
|
79
79
|
planOfTree,
|
|
80
80
|
sealTree,
|
|
81
|
-
treeIntact,
|
|
82
81
|
writeTree,
|
|
83
82
|
} from './archive.js'
|
|
84
83
|
import { readUmask, restoredMode } from './umask.js'
|
|
@@ -166,6 +165,8 @@ const VX_CACHE_NAMES = new Set([
|
|
|
166
165
|
'runs',
|
|
167
166
|
'failures',
|
|
168
167
|
'lockfile-claims',
|
|
168
|
+
// The workspace config's bytecode bundle (workspace/config-bytecode.ts).
|
|
169
|
+
'compiled',
|
|
169
170
|
])
|
|
170
171
|
|
|
171
172
|
/**
|
|
@@ -325,7 +326,16 @@ function outOfFdsAtOpen(dbFile: string, err: unknown): UserError | undefined {
|
|
|
325
326
|
// is unchanged.
|
|
326
327
|
// v42: fold_memo — each task's last key by its fold up to the files and
|
|
327
328
|
// a token of their git records. The cache KEY is unchanged.
|
|
328
|
-
|
|
329
|
+
// v43: runs.peak_cores / avg_rss_bytes — what a sampled run burned at
|
|
330
|
+
// its peak and held on average. The cache KEY is unchanged.
|
|
331
|
+
// v44: runs.placement — where an executor ran the task. The cache KEY
|
|
332
|
+
// is unchanged.
|
|
333
|
+
// v45: a 'remote' task_durations row also carries the task's measured
|
|
334
|
+
// cores, memory and saturation, for a run with no local history.
|
|
335
|
+
// The cache KEY is unchanged.
|
|
336
|
+
// v46: runs.keyless_by — the task with no inputs a key folded, so
|
|
337
|
+
// `vx why` names it. The cache KEY is unchanged.
|
|
338
|
+
export const SCHEMA_VERSION = 'v46'
|
|
329
339
|
|
|
330
340
|
/**
|
|
331
341
|
* An entry's key components as its one `entry_inputs` row (v34): JSON
|
|
@@ -814,14 +824,11 @@ export interface StoreOptions {
|
|
|
814
824
|
embeddings: number
|
|
815
825
|
/** A file-backed entry is a `<hash>.tree` of plain files; off, one `<hash>.tar.zst`. */
|
|
816
826
|
decompress: boolean
|
|
817
|
-
/** What a tree restore checks first: each file's size and mtime, every byte, or nothing. */
|
|
818
|
-
verify: 'stat' | 'hash' | 'none'
|
|
819
827
|
}
|
|
820
828
|
|
|
821
829
|
export const DEFAULT_STORE: StoreOptions = {
|
|
822
830
|
embeddings: 32 * 1024,
|
|
823
831
|
decompress: true,
|
|
824
|
-
verify: 'stat',
|
|
825
832
|
}
|
|
826
833
|
|
|
827
834
|
/** What a run's background uploads moved: totals over the ones that landed. */
|
|
@@ -859,6 +866,8 @@ export class Cache implements CacheLayer {
|
|
|
859
866
|
* so the run can report them without a method on the layer seam.
|
|
860
867
|
*/
|
|
861
868
|
readonly uploads: UploadTally = { count: 0, bytes: 0, ms: 0, failed: 0 }
|
|
869
|
+
/** The run's hits pulled from those remotes, and their bytes on the wire. */
|
|
870
|
+
readonly downloads: { count: number; bytes: number } = { count: 0, bytes: 0 }
|
|
862
871
|
/** The same remotes' traffic per key, filled by the same `LayeredCache`. */
|
|
863
872
|
readonly traffic: RemoteTraffic = { inFlight: 0, keys: new Map() }
|
|
864
873
|
|
|
@@ -1701,6 +1710,10 @@ export class Cache implements CacheLayer {
|
|
|
1701
1710
|
return this.guard(() => this.readEntry(hash))
|
|
1702
1711
|
}
|
|
1703
1712
|
|
|
1713
|
+
getSaved(hash: string): Promise<CacheEntry | null> {
|
|
1714
|
+
return this.getIngested(hash)
|
|
1715
|
+
}
|
|
1716
|
+
|
|
1704
1717
|
private async readEntry(hash: string): Promise<CacheEntry | null> {
|
|
1705
1718
|
const row = this.selectEntry.get(hash) as EntryRow | null
|
|
1706
1719
|
if (row === null) return null
|
|
@@ -1924,9 +1937,8 @@ export class Cache implements CacheLayer {
|
|
|
1924
1937
|
}
|
|
1925
1938
|
|
|
1926
1939
|
/**
|
|
1927
|
-
* Restore a tree-form entry: its files
|
|
1928
|
-
*
|
|
1929
|
-
* absent, as on every restore after a clean. Anything else (a tree to
|
|
1940
|
+
* Restore a tree-form entry: its files copied (or cloned) out one by
|
|
1941
|
+
* one where the outputs are absent, as on every restore after a clean. Anything else (a tree to
|
|
1930
1942
|
* restore into, a workspace output, a link) goes through the extractor
|
|
1931
1943
|
* the tar form uses, fed the tree packed back into its tar.
|
|
1932
1944
|
*/
|
|
@@ -1948,21 +1960,6 @@ export class Cache implements CacheLayer {
|
|
|
1948
1960
|
`its outputs hold ${formatBytes(bytes)}, past the ${formatBytes(this.artifactCeiling)} artifact ceiling`,
|
|
1949
1961
|
)
|
|
1950
1962
|
}
|
|
1951
|
-
if (this.store.verify === 'stat') {
|
|
1952
|
-
const changed = await Promise.all(
|
|
1953
|
-
files.map((r) =>
|
|
1954
|
-
stat(path.join(tree, treeName(r.path))).then(
|
|
1955
|
-
(st) => st.size !== r.size || Math.floor(st.mtimeMs) !== r.mtimeMs,
|
|
1956
|
-
() => true,
|
|
1957
|
-
),
|
|
1958
|
-
),
|
|
1959
|
-
)
|
|
1960
|
-
if (changed.includes(true)) {
|
|
1961
|
-
throw new CorruptArtifactError(hash, 'a stored file changed since it was saved')
|
|
1962
|
-
}
|
|
1963
|
-
} else if (this.store.verify === 'hash' && !(await treeIntact(tree, this.stdoutText(hash)))) {
|
|
1964
|
-
throw new CorruptArtifactError(hash, 'the stored files no longer sum to what was saved')
|
|
1965
|
-
}
|
|
1966
1963
|
if (await this.copyTree(tree, rows, files, projectDir, accept)) return
|
|
1967
1964
|
await extractArtifactStream(
|
|
1968
1965
|
packArtifactStream(planOfTree(tree, this.stdoutText(hash))),
|
|
@@ -1974,7 +1971,14 @@ export class Cache implements CacheLayer {
|
|
|
1974
1971
|
)
|
|
1975
1972
|
}
|
|
1976
1973
|
|
|
1977
|
-
/**
|
|
1974
|
+
/**
|
|
1975
|
+
* The copy-out: false, having written nothing, where it is not the case it
|
|
1976
|
+
* serves. It writes only where nothing stands: each file's place is absent,
|
|
1977
|
+
* and every directory on the way either is a real directory or is absent
|
|
1978
|
+
* and made here. solid's build outputs `store/dist/**` beside the sources
|
|
1979
|
+
* in `store/`; asking that each output's TOP be absent sent every such
|
|
1980
|
+
* restore through the tar extractor, ~25 ms a task (2026-10-22).
|
|
1981
|
+
*/
|
|
1978
1982
|
private async copyTree(
|
|
1979
1983
|
tree: string,
|
|
1980
1984
|
rows: readonly OutputFileRow[],
|
|
@@ -1982,22 +1986,49 @@ export class Cache implements CacheLayer {
|
|
|
1982
1986
|
projectDir: string,
|
|
1983
1987
|
accept: ((name: string) => boolean) | undefined,
|
|
1984
1988
|
): Promise<boolean> {
|
|
1985
|
-
const tops = new Set<string>()
|
|
1986
1989
|
for (const r of rows) {
|
|
1987
1990
|
if (r.path.startsWith(WORKSPACE_OUTPUT_PREFIX) || r.mode === LINK_MODE) return false
|
|
1988
1991
|
if (accept !== undefined && !r.path.endsWith('/') && !accept(`outputs/${r.path}`))
|
|
1989
1992
|
return false
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
)
|
|
1998
|
-
|
|
1993
|
+
}
|
|
1994
|
+
// What stands at each path, asked once: a real directory, nothing, or
|
|
1995
|
+
// anything else (a file, a link), which the extractor's rules decide.
|
|
1996
|
+
const standing = new Map<string, Promise<'dir' | 'absent' | 'other'>>()
|
|
1997
|
+
const at = (rel: string): Promise<'dir' | 'absent' | 'other'> => {
|
|
1998
|
+
let known = standing.get(rel)
|
|
1999
|
+
if (known === undefined) {
|
|
2000
|
+
known = lstat(path.join(projectDir, rel)).then(
|
|
2001
|
+
(st) => (st.isDirectory() ? 'dir' : 'other'),
|
|
2002
|
+
(err: NodeJS.ErrnoException) => (err.code === 'ENOENT' ? 'absent' : 'other'),
|
|
2003
|
+
)
|
|
2004
|
+
standing.set(rel, known)
|
|
2005
|
+
}
|
|
2006
|
+
return known
|
|
2007
|
+
}
|
|
2008
|
+
// The topmost absent directories this copy makes, and the files it
|
|
2009
|
+
// writes into a directory that stood: what a failure removes, and only that.
|
|
2010
|
+
const made = new Set<string>()
|
|
2011
|
+
const placed: string[] = []
|
|
2012
|
+
const fits = await Promise.all(
|
|
2013
|
+
rows.map(async (r) => {
|
|
2014
|
+
const segs = r.path.replace(/\/$/, '').split('/')
|
|
2015
|
+
for (let i = 1; i <= segs.length; i++) {
|
|
2016
|
+
const rel = segs.slice(0, i).join('/')
|
|
2017
|
+
const isLeaf = i === segs.length
|
|
2018
|
+
const state = await at(rel)
|
|
2019
|
+
if (state === 'absent') {
|
|
2020
|
+
if (!isLeaf || r.path.endsWith('/')) made.add(rel)
|
|
2021
|
+
else placed.push(rel)
|
|
2022
|
+
return true
|
|
2023
|
+
}
|
|
2024
|
+
if (state === 'other' || (isLeaf && !r.path.endsWith('/'))) return false
|
|
2025
|
+
}
|
|
2026
|
+
return true
|
|
2027
|
+
}),
|
|
1999
2028
|
)
|
|
2000
|
-
if (
|
|
2029
|
+
if (fits.includes(false)) return false
|
|
2030
|
+
// A directory made under another one made here is the outer one's.
|
|
2031
|
+
const roots = [...made].filter((d) => ![...made].some((o) => d.startsWith(`${o}/`)))
|
|
2001
2032
|
readUmask()
|
|
2002
2033
|
const plain = restoredMode(0o666)
|
|
2003
2034
|
try {
|
|
@@ -2017,9 +2048,10 @@ export class Cache implements CacheLayer {
|
|
|
2017
2048
|
}),
|
|
2018
2049
|
)
|
|
2019
2050
|
} catch (err) {
|
|
2020
|
-
await Promise.all(
|
|
2021
|
-
|
|
2022
|
-
|
|
2051
|
+
await Promise.all([
|
|
2052
|
+
...roots.map((d) => rm(path.join(projectDir, d), { recursive: true, force: true })),
|
|
2053
|
+
...placed.map((f) => rm(path.join(projectDir, f), { force: true })),
|
|
2054
|
+
])
|
|
2023
2055
|
throw err
|
|
2024
2056
|
}
|
|
2025
2057
|
return true
|
|
@@ -2699,7 +2731,6 @@ export class Cache implements CacheLayer {
|
|
|
2699
2731
|
const dir = this.tempTreePath(hash)
|
|
2700
2732
|
tree = dir
|
|
2701
2733
|
let scan: Awaited<ReturnType<typeof scanArtifact>> | undefined
|
|
2702
|
-
let raw: { meta: string; sum: string } | undefined
|
|
2703
2734
|
await extractArtifactStream(
|
|
2704
2735
|
await decodedTar(
|
|
2705
2736
|
compressed instanceof Uint8Array ? compressed : Bun.file(tmpPath!),
|
|
@@ -2711,13 +2742,12 @@ export class Cache implements CacheLayer {
|
|
|
2711
2742
|
undefined,
|
|
2712
2743
|
undefined,
|
|
2713
2744
|
undefined,
|
|
2714
|
-
(s
|
|
2745
|
+
(s) => {
|
|
2715
2746
|
scan = s
|
|
2716
|
-
raw = r
|
|
2717
2747
|
},
|
|
2718
2748
|
)
|
|
2719
2749
|
scanned = scan!
|
|
2720
|
-
await sealTree(dir, scanned
|
|
2750
|
+
await sealTree(dir, scanned)
|
|
2721
2751
|
} else if (tar !== undefined) {
|
|
2722
2752
|
// Our own tar, in memory: read in place, on this thread.
|
|
2723
2753
|
scanned = scanTarBytes(tar)
|