@vzn/vx 0.0.21 → 0.0.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -18
- package/package.json +6 -6
- package/src/bin.ts +52 -3
- package/src/cache/archive.ts +170 -21
- package/src/cache/cache.ts +667 -288
- package/src/cache/chained-cache.ts +14 -2
- package/src/cache/config-evals.ts +29 -6
- package/src/cache/file-hashes.ts +63 -35
- package/src/cache/git-inputs.ts +398 -69
- package/src/cache/index.ts +8 -1
- package/src/cache/inputs.ts +403 -102
- package/src/cache/key-fold.ts +232 -0
- package/src/cache/layer.ts +81 -11
- package/src/cache/layered-cache.ts +199 -32
- package/src/cache/output-index.ts +109 -10
- package/src/cache/run-history.ts +8 -3
- package/src/cache/zstd.ts +53 -30
- package/src/cli/cache.ts +51 -21
- package/src/cli/format.ts +4 -15
- package/src/cli/help.ts +13 -3
- package/src/cli/index.ts +35 -9
- package/src/cli/info.ts +49 -7
- package/src/cli/init.ts +17 -5
- package/src/cli/last.ts +96 -21
- package/src/cli/lock.ts +34 -17
- package/src/cli/plan-format.ts +4 -4
- package/src/cli/plugin-commands.ts +20 -5
- package/src/cli/run.ts +55 -12
- package/src/cli/select.ts +23 -13
- package/src/cli/show.ts +9 -4
- package/src/cli/upgrade.ts +134 -14
- package/src/cli/watch.ts +622 -102
- package/src/cli/why.ts +8 -16
- package/src/cli/workspace-config.ts +21 -9
- package/src/config.ts +46 -9
- package/src/exec/executor.ts +51 -5
- package/src/exec/index.ts +10 -0
- package/src/exec/kill-tree.ts +281 -0
- package/src/exec/local-executor.ts +1 -1
- package/src/exec/runner.ts +327 -82
- package/src/exec/sandbox-binds.ts +34 -5
- package/src/exec/sandbox-paths.ts +22 -2
- package/src/exec/sandbox-runtime.ts +627 -102
- package/src/exec/sandbox-violations.ts +36 -23
- package/src/graph/dependency-spec.ts +14 -14
- package/src/graph/index.ts +12 -2
- package/src/graph/priorities.ts +130 -0
- package/src/graph/scheduler.ts +182 -134
- package/src/graph/task-graph.ts +552 -188
- package/src/index.ts +16 -2
- package/src/orchestrator/admission.ts +41 -17
- package/src/orchestrator/colors.ts +8 -3
- package/src/orchestrator/deferred-outputs.ts +22 -7
- package/src/orchestrator/doctor.ts +122 -14
- package/src/orchestrator/events.ts +71 -2
- package/src/orchestrator/excluded-keys.ts +155 -0
- package/src/orchestrator/execute-task.ts +331 -37
- package/src/orchestrator/failure-recap.ts +121 -0
- package/src/orchestrator/fingerprint-watch.ts +89 -0
- package/src/orchestrator/framed-output.ts +81 -14
- package/src/orchestrator/history.ts +6 -7
- package/src/orchestrator/hit-restore.ts +82 -18
- package/src/orchestrator/index.ts +7 -1
- package/src/orchestrator/keyed-projects.ts +89 -0
- package/src/orchestrator/local-shortcircuit.ts +83 -23
- package/src/orchestrator/lockfile-claim.ts +152 -32
- package/src/orchestrator/logger.ts +116 -20
- package/src/orchestrator/metrics.ts +156 -60
- package/src/orchestrator/miss-save.ts +106 -43
- package/src/orchestrator/options.ts +30 -4
- package/src/orchestrator/persistent.ts +41 -21
- package/src/orchestrator/placement.ts +55 -25
- package/src/orchestrator/plan.ts +24 -11
- package/src/orchestrator/plugin-host.ts +17 -23
- package/src/orchestrator/plugin.ts +31 -8
- package/src/orchestrator/prepare.ts +99 -13
- package/src/orchestrator/projects.ts +19 -8
- package/src/orchestrator/remote-prefetch.ts +21 -1
- package/src/orchestrator/run-artifacts.ts +9 -0
- package/src/orchestrator/run-context.ts +19 -4
- package/src/orchestrator/run-lock.ts +349 -0
- package/src/orchestrator/run-records.ts +6 -23
- package/src/orchestrator/run-report.ts +4 -2
- package/src/orchestrator/run.ts +264 -57
- package/src/orchestrator/sandbox-request.ts +388 -62
- package/src/orchestrator/save-lane.ts +4 -7
- package/src/orchestrator/shell-verdict.ts +117 -0
- package/src/orchestrator/signals.ts +86 -30
- package/src/orchestrator/stable-keys.ts +202 -10
- package/src/orchestrator/status-line.ts +33 -1
- package/src/orchestrator/summary.ts +49 -2
- package/src/orchestrator/task-hash.ts +87 -7
- package/src/orchestrator/task-log-buffer.ts +7 -7
- package/src/orchestrator/telemetry-host.ts +16 -7
- package/src/orchestrator/telemetry.ts +69 -28
- package/src/orchestrator/upstream.ts +91 -45
- package/src/util/bun-version.ts +43 -0
- package/src/util/errors.ts +86 -0
- package/src/util/hash.ts +10 -1
- package/src/util/index.ts +32 -3
- package/src/util/paths.ts +147 -5
- package/src/util/procfs.ts +20 -0
- package/src/util/settle.ts +4 -2
- package/src/util/size.ts +33 -0
- package/src/util/task-id.ts +20 -0
- package/src/util/timing.ts +11 -1
- package/src/util/which.ts +48 -0
- package/src/workspace/affected.ts +272 -64
- package/src/workspace/config-cache.ts +262 -32
- package/src/workspace/config-eval.ts +49 -12
- package/src/workspace/config-imports.ts +126 -5
- package/src/workspace/config-schema.ts +237 -41
- package/src/workspace/filter.ts +33 -18
- package/src/workspace/fingerprint.ts +29 -11
- package/src/workspace/index.ts +7 -0
- package/src/workspace/json-data.ts +90 -0
- package/src/workspace/load-reads.ts +32 -0
- package/src/workspace/lockfile.ts +33 -11
- package/src/workspace/migrate-scripts.ts +60 -7
- package/src/workspace/migration.ts +40 -1
- package/src/workspace/package-graph.ts +154 -16
- package/src/workspace/project-loader.ts +227 -77
- package/src/workspace/workspace.ts +173 -23
package/README.md
CHANGED
|
@@ -67,7 +67,7 @@ export default defineProject({
|
|
|
67
67
|
vx init # scaffold vx.workspace.ts + a vx.config.ts per package from scripts
|
|
68
68
|
vx run build # cwd project + its workspace deps
|
|
69
69
|
vx run test --all # every project that declares `test`
|
|
70
|
-
vx run ci --affected #
|
|
70
|
+
vx run ci --affected # changed since origin/HEAD, and what depends on it
|
|
71
71
|
vx watch lint # re-run on file changes
|
|
72
72
|
vx run build --dry # show the plan, don't execute
|
|
73
73
|
```
|
|
@@ -136,9 +136,11 @@ vx lock --check && vx run ci --frozen # CI: audit, then run EXACTLY that gra
|
|
|
136
136
|
| `vx lock --check` | full graph | compares — catches env and import drift that byte hashes cannot |
|
|
137
137
|
|
|
138
138
|
Env values read at lock time are frozen by design — cache keys become
|
|
139
|
-
reproducible across machines.
|
|
140
|
-
|
|
141
|
-
|
|
139
|
+
reproducible across machines. Take it for that, not for speed: measured
|
|
140
|
+
on the 1,000-project bench, a plain warm run and a frozen one are a tie
|
|
141
|
+
(177 ms against 165, 2026-09-12), because the config-evaluation cache
|
|
142
|
+
already serves a pure config without evaluating it. No other runner has
|
|
143
|
+
an equivalent.
|
|
142
144
|
|
|
143
145
|
## Everything you need, nothing to configure twice
|
|
144
146
|
|
|
@@ -201,17 +203,17 @@ distributed ships in this repo; the seams are how you build it.
|
|
|
201
203
|
|
|
202
204
|
## How it compares
|
|
203
205
|
|
|
204
|
-
| | vx
|
|
205
|
-
| ------------------------- |
|
|
206
|
-
| Fully cached, 100 pkgs¹ | **144 ms**
|
|
207
|
-
| Config | TypeScript, evaluated into the cache key
|
|
208
|
-
| Output ownership | **Strict** — wiped before exec AND restore
|
|
209
|
-
| Clean-tree hashing | **Zero reads** (git index OIDs)
|
|
210
|
-
| Daemon required for speed | **No**
|
|
211
|
-
| Per-task sandbox | **Yes** — kernel-level, opt-in
|
|
212
|
-
| Plugin API | **Yes** — executor / cache / telemetry seams
|
|
213
|
-
| OTel CI/CD spans | **Yes** — `otel()` plugin, zero OTel-SDK deps
|
|
214
|
-
| Install | **Single binary** — npm or
|
|
206
|
+
| | vx | Turborepo | Nx |
|
|
207
|
+
| ------------------------- | ---------------------------------------------------------------- | ------------------------------ | ---------------- |
|
|
208
|
+
| Fully cached, 100 pkgs¹ | **144 ms** | 279 ms | 583+ ms |
|
|
209
|
+
| Config | TypeScript, evaluated into the cache key | JSON (static) | JSON (static) |
|
|
210
|
+
| Output ownership | **Strict** — wiped before exec AND restore | Additive (stale files survive) | Additive |
|
|
211
|
+
| Clean-tree hashing | **Zero reads** (git index OIDs) | git OIDs | re-hash / daemon |
|
|
212
|
+
| Daemon required for speed | **No** | Optional | Yes |
|
|
213
|
+
| Per-task sandbox | **Yes** — kernel-level, opt-in | No | No |
|
|
214
|
+
| Plugin API | **Yes** — executor / cache / telemetry seams | No | Yes (TS-tied) |
|
|
215
|
+
| OTel CI/CD spans | **Yes** — `otel()` plugin, zero OTel-SDK deps | No | Paid |
|
|
216
|
+
| Install | **Single binary** — npm, or one release file; no Node/Bun needed | npm + Node | npm + Node |
|
|
215
217
|
|
|
216
218
|
¹ Wall-clock, direct binaries, same machine and workspace — full
|
|
217
219
|
methodology and more scenarios in
|
|
@@ -306,7 +308,8 @@ maintainers' handoff is [`packages/vx/docs/STATUS.md`](packages/vx/docs/STATUS.m
|
|
|
306
308
|
## Status
|
|
307
309
|
|
|
308
310
|
**Pre-alpha.** The schema is settling; we bump `CACHE_VERSION` rather
|
|
309
|
-
than maintain back-compat.
|
|
311
|
+
than maintain back-compat. What 1.0 will promise is in
|
|
312
|
+
[`versioning-1.0.md`](packages/vx/docs/design/versioning-1.0.md). **~3,000 core tests plus the package suites; CI green on every commit**;
|
|
310
313
|
the project dogfoods itself (`vx run ci`). Published on npm:
|
|
311
314
|
[`@vzn/vx`](https://www.npmjs.com/package/@vzn/vx) (a prebuilt standalone
|
|
312
315
|
binary).
|
|
@@ -318,8 +321,8 @@ build.
|
|
|
318
321
|
|
|
319
322
|
| Surface | Maturity | Notes |
|
|
320
323
|
| -------------------------------------------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
321
|
-
| Core task runner + caching | **production-ready** | dogfooded continuously; ~
|
|
322
|
-
| Plugin pipeline (
|
|
324
|
+
| Core task runner + caching | **production-ready** | dogfooded continuously; ~3,000 core tests + the package suites, green |
|
|
325
|
+
| Plugin pipeline (13 hooks, `commands` included) | **shippable** | crash-isolated, re-validated; the local executor + cache are the floor under every plugin |
|
|
323
326
|
| `vx init` / `@vzn/vx-migrate` (scripts; Turbo, Nx) | **shippable** | one config per package, TODOs where a source cannot say |
|
|
324
327
|
| REAPI remote cache + execution (`@vzn/vx-reapi`) | **shippable** | Bazel AC + CAS + Execute; NativeLink / BuildBuddy / Buildbarn / bazel-remote |
|
|
325
328
|
| OTel export (`@vzn/vx-otel`) | **shippable** | OTLP traces + metrics + logs, zero SDK deps |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vzn/vx",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.23",
|
|
4
4
|
"description": "An open, extensible monorepo task runner.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -17,13 +17,13 @@
|
|
|
17
17
|
"node": ">=18"
|
|
18
18
|
},
|
|
19
19
|
"optionalDependencies": {
|
|
20
|
-
"@vzn/vx-linux-x64": "0.0.
|
|
21
|
-
"@vzn/vx-linux-arm64": "0.0.
|
|
22
|
-
"@vzn/vx-darwin-x64": "0.0.
|
|
23
|
-
"@vzn/vx-darwin-arm64": "0.0.
|
|
20
|
+
"@vzn/vx-linux-x64": "0.0.23",
|
|
21
|
+
"@vzn/vx-linux-arm64": "0.0.23",
|
|
22
|
+
"@vzn/vx-darwin-x64": "0.0.23",
|
|
23
|
+
"@vzn/vx-darwin-arm64": "0.0.23"
|
|
24
24
|
},
|
|
25
25
|
"dependencies": {
|
|
26
|
-
"@anthropic-ai/sandbox-runtime": "^0.0.
|
|
26
|
+
"@anthropic-ai/sandbox-runtime": "^0.0.76"
|
|
27
27
|
},
|
|
28
28
|
"files": [
|
|
29
29
|
"index.ts",
|
package/src/bin.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
2
|
import { registerCoreAlias, run } from './cli/index.js'
|
|
3
|
-
import { isUserError } from './util/index.js'
|
|
3
|
+
import { fsRefusalHint, isFsRefusal, isUserError } from './util/index.js'
|
|
4
4
|
|
|
5
5
|
// Every `import … from '@vzn/vx'` this process evaluates — a plugin
|
|
6
6
|
// package, a workspace or project config — resolves to THIS core, not to
|
|
@@ -10,9 +10,23 @@ registerCoreAlias(() => import('./index.js') as Promise<Record<string, unknown>>
|
|
|
10
10
|
|
|
11
11
|
// Wrapped in an explicit async main so `bun build --compile` accepts
|
|
12
12
|
// the file. The compile target doesn't allow top-level await.
|
|
13
|
+
let settled = false
|
|
14
|
+
|
|
13
15
|
async function main(): Promise<void> {
|
|
14
16
|
try {
|
|
15
|
-
|
|
17
|
+
const code = await run(process.argv.slice(2))
|
|
18
|
+
// NOTHING calls `process.exit` here, and that is the fix rather than a
|
|
19
|
+
// simplification. Bun drops what a pipe has not yet taken when
|
|
20
|
+
// `process.exit` follows a large write: 300 KB written then exit
|
|
21
|
+
// delivered 64 KiB (128 KiB after a tick), and `vx history --format
|
|
22
|
+
// json` on a 300-project workspace was cut mid-string (2026-09-15).
|
|
23
|
+
// `stdout.end(cb)` fixed that on 1.4.2 — and on 1.3.11 the callback
|
|
24
|
+
// still fires early: 2 MiB written, 214 KB delivered (2026-09-20).
|
|
25
|
+
// Setting the code and letting the loop drain is the one form that
|
|
26
|
+
// does not depend on when a runtime decides a pipe is flushed; it
|
|
27
|
+
// costs a hang if a verb leaves a handle open, which the suite's
|
|
28
|
+
// several hundred spawns of this binary would show at once.
|
|
29
|
+
process.exitCode = code
|
|
16
30
|
} catch (err) {
|
|
17
31
|
// UserError (workspace not found, cycle, config invalid, ...) —
|
|
18
32
|
// print the message only; the stack is noise the user can't act
|
|
@@ -24,12 +38,47 @@ async function main(): Promise<void> {
|
|
|
24
38
|
// it produced `vx: vx why: …` (walkthrough, 2026-09-04).
|
|
25
39
|
const m = err.message
|
|
26
40
|
process.stderr.write(m.startsWith('vx ') ? `${m}\n` : `vx: ${m}\n`)
|
|
41
|
+
} else if (isFsRefusal(err)) {
|
|
42
|
+
// The file system's refusal names the path; the stack would name
|
|
43
|
+
// the verb's write, which the reader cannot act on either.
|
|
44
|
+
process.stderr.write(`vx: ${err.message} — ${fsRefusalHint(err)}\n`)
|
|
27
45
|
} else {
|
|
28
46
|
const message = err instanceof Error ? (err.stack ?? err.message) : String(err)
|
|
29
47
|
process.stderr.write(`vx: ${message}\n`)
|
|
30
48
|
}
|
|
31
|
-
|
|
49
|
+
// Same reason as the success path above: a large stderr is truncated by
|
|
50
|
+
// `process.exit` too, and an error message cut in half is the one a
|
|
51
|
+
// reader most needs whole.
|
|
52
|
+
process.exitCode = 1
|
|
32
53
|
}
|
|
54
|
+
settled = true
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// The loop drained while the verb was still pending: something it awaits
|
|
58
|
+
// can never settle (a plugin hook whose promise is never resolved), and
|
|
59
|
+
// with no exit code set Bun exits 0 — a run whose task failed reported
|
|
60
|
+
// green, and its task had not even run (item 921). An exit that no
|
|
61
|
+
// verdict reached is a failure, and says so.
|
|
62
|
+
process.on('beforeExit', () => {
|
|
63
|
+
if (settled) return
|
|
64
|
+
settled = true
|
|
65
|
+
process.stderr.write(
|
|
66
|
+
'vx: the run stopped before it finished: something it awaited can never settle (a plugin hook?)\n',
|
|
67
|
+
)
|
|
68
|
+
process.exitCode = 1
|
|
69
|
+
})
|
|
70
|
+
|
|
71
|
+
// A reader that leaves is not the run's failure. `vx run build | head -1`
|
|
72
|
+
// closes the pipe after one line; every later write gets EPIPE, which Bun
|
|
73
|
+
// raises as an `error` event on the stream, and an `error` nobody listens
|
|
74
|
+
// for is an uncaught exception: the run died with a stack after its task
|
|
75
|
+
// had succeeded, exit 1, its lock directory left behind (2026-09-16). With
|
|
76
|
+
// a listener the write returns false, the run finishes, saves, releases,
|
|
77
|
+
// and exits with its own verdict; what was going to the reader goes
|
|
78
|
+
// nowhere. Same for stderr (`2>&1 | head`). Here and not in the logger:
|
|
79
|
+
// an embedder's streams are the embedder's.
|
|
80
|
+
for (const stream of [process.stdout, process.stderr]) {
|
|
81
|
+
stream.on('error', () => {})
|
|
33
82
|
}
|
|
34
83
|
|
|
35
84
|
void main()
|
package/src/cache/archive.ts
CHANGED
|
@@ -38,7 +38,7 @@
|
|
|
38
38
|
// otherwise silently loses the claim rather than acting on it.
|
|
39
39
|
|
|
40
40
|
import { chmodSync, renameSync, statSync, utimesSync } from 'node:fs'
|
|
41
|
-
import { mkdir, realpath, rmdir, stat, unlink, writeFile } from 'node:fs/promises'
|
|
41
|
+
import { mkdir, readlink, realpath, rmdir, stat, unlink, writeFile } from 'node:fs/promises'
|
|
42
42
|
import path from 'node:path'
|
|
43
43
|
import { UserError } from '../util/index.js'
|
|
44
44
|
import { TarFormatError, type TarInput, tarEntries, tarPack, tarSize } from './tar-stream.js'
|
|
@@ -47,7 +47,7 @@ import { TarFormatError, type TarInput, tarEntries, tarPack, tarSize } from './t
|
|
|
47
47
|
const META_ENTRY = '.vx-meta.json'
|
|
48
48
|
|
|
49
49
|
/** Archive entry name of the always-present stdout record. */
|
|
50
|
-
|
|
50
|
+
const STDOUT_ENTRY = 'stdout'
|
|
51
51
|
|
|
52
52
|
/**
|
|
53
53
|
* Sidecar shape. `files` maps an entry name to `[mode, mtimeMs]` — a
|
|
@@ -59,6 +59,14 @@ interface MetaFile {
|
|
|
59
59
|
/** Bumped only if the sidecar's own shape changes; the artifact
|
|
60
60
|
* container is versioned by CACHE_VERSION, which gates reads. */
|
|
61
61
|
version: 1
|
|
62
|
+
/**
|
|
63
|
+
* The cache key the artifact was packed under (v35, item 943). Ingest
|
|
64
|
+
* refuses bytes whose key is not the one it asked for: nothing else ties
|
|
65
|
+
* an artifact to its key, so a remote layer that answered one key with
|
|
66
|
+
* another's bytes (a truncated or colliding mapping) replayed the other
|
|
67
|
+
* task's outputs under a green `cache-hit-remote`.
|
|
68
|
+
*/
|
|
69
|
+
key?: string
|
|
62
70
|
files: Record<string, [mode: number, mtimeMs: number]>
|
|
63
71
|
/**
|
|
64
72
|
* What the PRODUCING execution used (2026-09-12, additive: an artifact
|
|
@@ -102,6 +110,8 @@ export class ArchiveSecurityError extends Error {
|
|
|
102
110
|
|
|
103
111
|
/** What `packArtifactStream` packs: stdout plus archive name → absolute source path. */
|
|
104
112
|
export interface PackArgs {
|
|
113
|
+
/** The cache key, recorded in the sidecar (item 943); ingest refuses an artifact without it. */
|
|
114
|
+
key?: string
|
|
105
115
|
stdout: string
|
|
106
116
|
outputs: ReadonlyMap<string, string>
|
|
107
117
|
/** The producing execution's usage, when the runner reported it. */
|
|
@@ -124,6 +134,7 @@ export interface ArtifactPlan {
|
|
|
124
134
|
*/
|
|
125
135
|
export async function planArtifact(args: PackArgs): Promise<ArtifactPlan> {
|
|
126
136
|
const meta: MetaFile = { version: 1, files: {} }
|
|
137
|
+
if (args.key !== undefined) meta.key = args.key
|
|
127
138
|
const exec = usageOf(args.exec)
|
|
128
139
|
if (exec !== undefined) meta.exec = exec
|
|
129
140
|
const files = await Promise.all(
|
|
@@ -136,14 +147,14 @@ export async function planArtifact(args: PackArgs): Promise<ArtifactPlan> {
|
|
|
136
147
|
const st = await stat(abs).catch((err: NodeJS.ErrnoException) => {
|
|
137
148
|
if (err.code === 'ENOENT') {
|
|
138
149
|
throw new UserError(
|
|
139
|
-
`output ${shown} is a dangling symlink: vx stores regular files only — emit a file, or
|
|
150
|
+
`output ${shown} is a dangling symlink: vx stores regular files only — emit a file there, or narrow cache.outputs.files to the files the task produces (output globs take no '!')`,
|
|
140
151
|
)
|
|
141
152
|
}
|
|
142
153
|
throw err
|
|
143
154
|
})
|
|
144
155
|
if (!st.isFile()) {
|
|
145
156
|
throw new UserError(
|
|
146
|
-
`output ${shown} is not a regular file (a symlink to a directory?): vx stores regular files only — emit a file, or
|
|
157
|
+
`output ${shown} is not a regular file (a symlink to a directory?): vx stores regular files only — emit a file there, or narrow cache.outputs.files to the files the task produces (output globs take no '!')`,
|
|
147
158
|
)
|
|
148
159
|
}
|
|
149
160
|
meta.files[name] = [st.mode & 0o777, Math.floor(st.mtimeMs)]
|
|
@@ -224,12 +235,16 @@ function streamOf(gen: AsyncGenerator<Uint8Array>): ReadableStream<Uint8Array> {
|
|
|
224
235
|
* name is unsafe — a partial reading of a poisoned artifact is the
|
|
225
236
|
* outcome the traversal defense exists to prevent.
|
|
226
237
|
*/
|
|
227
|
-
export async function scanArtifact(
|
|
228
|
-
|
|
229
|
-
|
|
238
|
+
export async function scanArtifact(tar: ReadableStream<Uint8Array>): Promise<{
|
|
239
|
+
entries: ArchiveEntry[]
|
|
240
|
+
stdout: string | null
|
|
241
|
+
exec: ExecUsage | undefined
|
|
242
|
+
key: string | undefined
|
|
243
|
+
}> {
|
|
230
244
|
const seen: Array<{ name: string; size: number; mtimeMs: number }> = []
|
|
231
245
|
let meta: MetaFile['files'] = {}
|
|
232
246
|
let exec: ExecUsage | undefined
|
|
247
|
+
let key: string | undefined
|
|
233
248
|
let stdout: string | null = null
|
|
234
249
|
for await (const e of tarEntries(tar)) {
|
|
235
250
|
if (e.type !== '0') continue
|
|
@@ -239,6 +254,7 @@ export async function scanArtifact(
|
|
|
239
254
|
// The ingest side is the untrusted boundary: a foreign sidecar's
|
|
240
255
|
// usage is taken only when it is a plain non-negative number.
|
|
241
256
|
exec = usageOf(parsed.exec)
|
|
257
|
+
if (typeof parsed.key === 'string') key = parsed.key
|
|
242
258
|
continue
|
|
243
259
|
}
|
|
244
260
|
assertSafeName(e.name)
|
|
@@ -255,6 +271,7 @@ export async function scanArtifact(
|
|
|
255
271
|
}),
|
|
256
272
|
stdout,
|
|
257
273
|
exec,
|
|
274
|
+
key,
|
|
258
275
|
}
|
|
259
276
|
}
|
|
260
277
|
|
|
@@ -286,7 +303,10 @@ export async function extractArtifactStream(
|
|
|
286
303
|
workspaceDest: string | undefined,
|
|
287
304
|
verify?: (provided: ReadonlySet<string>) => void,
|
|
288
305
|
): Promise<Set<string>> {
|
|
289
|
-
|
|
306
|
+
// No `mkdir -p` of the destination here: `stage` creates each entry's
|
|
307
|
+
// directory chain, the destination included, and the cache restores
|
|
308
|
+
// into a project directory that exists. The eager one cost every
|
|
309
|
+
// restore a thread-pool round trip for an EEXIST and a stat (item 627).
|
|
290
310
|
const x = new Extractor(destDir, workspaceDest)
|
|
291
311
|
const provided = new Set<string>()
|
|
292
312
|
const headerMtime = new Map<string, number>()
|
|
@@ -305,7 +325,12 @@ export async function extractArtifactStream(
|
|
|
305
325
|
await x.assertContained(dest.base, target, e.name)
|
|
306
326
|
provided.add(e.name)
|
|
307
327
|
headerMtime.set(e.name, e.mtimeMs)
|
|
308
|
-
await x.stage(
|
|
328
|
+
await x.stage(
|
|
329
|
+
e.name,
|
|
330
|
+
dest.base,
|
|
331
|
+
target,
|
|
332
|
+
e.size <= SMALL_ENTRY ? await bytesOf(e.body) : e.body,
|
|
333
|
+
)
|
|
309
334
|
}
|
|
310
335
|
verify?.(provided)
|
|
311
336
|
await x.commit((name) => {
|
|
@@ -448,22 +473,21 @@ class Extractor {
|
|
|
448
473
|
}
|
|
449
474
|
const parentResolved = path.dirname(targetResolved)
|
|
450
475
|
if (this.containedDirs.has(parentResolved)) return
|
|
451
|
-
await this.assertParentContained(base, targetResolved
|
|
476
|
+
await this.assertParentContained(base, targetResolved)
|
|
452
477
|
this.containedDirs.add(parentResolved)
|
|
453
478
|
}
|
|
454
479
|
|
|
455
|
-
private async assertParentContained(
|
|
456
|
-
base: string,
|
|
457
|
-
targetResolved: string,
|
|
458
|
-
name: string,
|
|
459
|
-
): Promise<void> {
|
|
480
|
+
private async assertParentContained(base: string, targetResolved: string): Promise<void> {
|
|
460
481
|
const baseResolved = path.resolve(base)
|
|
461
|
-
const realBase = await this.realBaseOf(base)
|
|
462
482
|
// Only ancestors strictly BELOW the base are candidates — those are the
|
|
463
483
|
// ones a poisoned entry could follow out of the tree. The walk must never
|
|
464
484
|
// climb past the base: when the base itself does not exist yet (the
|
|
465
485
|
// workspace-outputs anchor is created lazily), its parent legitimately
|
|
466
486
|
// resolves outside and comparing against it would reject every entry.
|
|
487
|
+
// The base's own real path is resolved only once an ancestor EXISTS to
|
|
488
|
+
// compare it with: after a clean pruned the output directories (the
|
|
489
|
+
// common restore) nothing below the base is there, and the memoised
|
|
490
|
+
// resolve was a round trip spent on a comparison never made (item 627).
|
|
467
491
|
let probe = path.dirname(targetResolved)
|
|
468
492
|
while (probe.startsWith(baseResolved + path.sep)) {
|
|
469
493
|
const real = await realpath(probe).then(
|
|
@@ -471,10 +495,9 @@ class Extractor {
|
|
|
471
495
|
() => null,
|
|
472
496
|
)
|
|
473
497
|
if (real !== null) {
|
|
498
|
+
const realBase = await this.realBaseOf(base)
|
|
474
499
|
if (real !== realBase && !real.startsWith(realBase + path.sep)) {
|
|
475
|
-
throw
|
|
476
|
-
`archive entry escapes destDir via a symlinked parent: ${name}`,
|
|
477
|
-
)
|
|
500
|
+
throw await linkOutError(baseResolved, probe, realBase)
|
|
478
501
|
}
|
|
479
502
|
return
|
|
480
503
|
}
|
|
@@ -484,6 +507,33 @@ class Extractor {
|
|
|
484
507
|
// pre-existing symlink to follow — the chain will be created fresh.
|
|
485
508
|
}
|
|
486
509
|
|
|
510
|
+
/**
|
|
511
|
+
* `mkdir -p` does not follow a DANGLING link: it fails EEXIST at the link,
|
|
512
|
+
* ENOENT below it, ELOOP in a cycle — and the containment walk passed it,
|
|
513
|
+
* finding nothing that resolves between it and the base. A live link that
|
|
514
|
+
* stays inside the base is written through, so a dangling one is too: the
|
|
515
|
+
* directory it names is created and the entry lands through the link,
|
|
516
|
+
* which stays the user's (never replaced, nx#37061). One that leads out of
|
|
517
|
+
* the base is the refusal a live link out gets. Only the failure pays for
|
|
518
|
+
* the walk; a clean tree never reaches it.
|
|
519
|
+
*/
|
|
520
|
+
private async mkdirThroughLink(
|
|
521
|
+
base: string,
|
|
522
|
+
dir: string,
|
|
523
|
+
err: NodeJS.ErrnoException,
|
|
524
|
+
): Promise<string | undefined> {
|
|
525
|
+
if (err.code !== 'EEXIST' && err.code !== 'ENOENT' && err.code !== 'ELOOP') throw err
|
|
526
|
+
const real = await resolveThrough(dir)
|
|
527
|
+
const realBase = await this.realBaseOf(base)
|
|
528
|
+
if (real !== realBase && !real.startsWith(realBase + path.sep)) {
|
|
529
|
+
throw await linkOutError(path.resolve(base), dir, realBase)
|
|
530
|
+
}
|
|
531
|
+
// What stands there without a link on the way (a file named like the
|
|
532
|
+
// directory) fails this mkdir as it failed the first: the caller names
|
|
533
|
+
// it as a stray.
|
|
534
|
+
return mkdir(real, { recursive: true })
|
|
535
|
+
}
|
|
536
|
+
|
|
487
537
|
/**
|
|
488
538
|
* Write the entry beside its target. A buffer is written without
|
|
489
539
|
* waiting, bounded by `INFLIGHT_BYTES`; a chunk stream goes to a file
|
|
@@ -491,13 +541,18 @@ class Extractor {
|
|
|
491
541
|
*/
|
|
492
542
|
async stage(
|
|
493
543
|
name: string,
|
|
544
|
+
base: string,
|
|
494
545
|
target: string,
|
|
495
546
|
body: Uint8Array | AsyncIterable<Uint8Array>,
|
|
496
547
|
): Promise<void> {
|
|
497
548
|
const dir = path.dirname(target)
|
|
498
549
|
let created: string | undefined
|
|
499
550
|
if (!this.ensuredDirs.has(dir)) {
|
|
500
|
-
|
|
551
|
+
try {
|
|
552
|
+
created = await mkdir(dir, { recursive: true })
|
|
553
|
+
} catch (err) {
|
|
554
|
+
created = await this.mkdirThroughLink(base, dir, err as NodeJS.ErrnoException)
|
|
555
|
+
}
|
|
501
556
|
this.ensuredDirs.add(dir)
|
|
502
557
|
}
|
|
503
558
|
// Write beside the target and RENAME into place. rename(2) replaces
|
|
@@ -517,7 +572,13 @@ class Extractor {
|
|
|
517
572
|
//
|
|
518
573
|
// A directory at the target makes the rename fail — the same
|
|
519
574
|
// fail-closed outcome the plain write had.
|
|
520
|
-
|
|
575
|
+
// The temp is a sibling with its own short name, never `<target>.suffix`:
|
|
576
|
+
// a suffix pushed a legal name of 242–255 bytes past NAME_MAX, and the
|
|
577
|
+
// restore of a valid artifact failed with ENAMETOOLONG (item 666).
|
|
578
|
+
const tmp = path.join(
|
|
579
|
+
path.dirname(target),
|
|
580
|
+
`.vx-tmp-${process.pid.toString(36)}-${(tmpSeq++).toString(36)}`,
|
|
581
|
+
)
|
|
521
582
|
this.staged.push({ name, tmp, target, created })
|
|
522
583
|
if (body instanceof Uint8Array) {
|
|
523
584
|
this.inflightBytes += body.byteLength
|
|
@@ -594,6 +655,9 @@ class Extractor {
|
|
|
594
655
|
if (s.created === undefined) continue
|
|
595
656
|
let dir = path.dirname(s.target)
|
|
596
657
|
const top = path.resolve(s.created)
|
|
658
|
+
// Created at a dangling link's resolved target (`mkdirThroughLink`),
|
|
659
|
+
// so the chain is pruned from where the link leads.
|
|
660
|
+
if (!dir.startsWith(top)) dir = await realpath(dir).catch(() => dir)
|
|
597
661
|
while (dir.startsWith(top)) {
|
|
598
662
|
if (
|
|
599
663
|
!(await rmdir(dir).then(
|
|
@@ -618,6 +682,65 @@ let tmpSeq = 0
|
|
|
618
682
|
|
|
619
683
|
const WORKSPACE_PREFIX = 'workspace-outputs/'
|
|
620
684
|
|
|
685
|
+
/**
|
|
686
|
+
* The on-disk half of containment failed: `probe`, a directory under
|
|
687
|
+
* `base`, resolves outside it, so a directory between them is a link out.
|
|
688
|
+
* That is the tree, not the artifact — an output directory made a link
|
|
689
|
+
* after the entry was saved (upstream survey, nx#37061) — so it is the
|
|
690
|
+
* user's to fix, and the refusal names the link and where it leads. The
|
|
691
|
+
* restore never writes through it and never replaces it: the link is the
|
|
692
|
+
* user's, and replacing it is the bug nx#37061 reports.
|
|
693
|
+
*/
|
|
694
|
+
async function linkOutError(base: string, probe: string, realBase: string): Promise<UserError> {
|
|
695
|
+
let dir = base
|
|
696
|
+
let real = realBase
|
|
697
|
+
for (const part of path.relative(base, probe).split(path.sep)) {
|
|
698
|
+
dir = path.join(dir, part)
|
|
699
|
+
real = await resolveThrough(dir)
|
|
700
|
+
if (real !== realBase && !real.startsWith(realBase + path.sep)) break
|
|
701
|
+
}
|
|
702
|
+
return new UserError(
|
|
703
|
+
`${dir} is a symbolic link to ${real}, outside ${base} — a cache restore never writes ` +
|
|
704
|
+
'through a link that leaves its directory. Remove the link and re-run (the restore ' +
|
|
705
|
+
'puts a real directory there), or stop declaring outputs under it.',
|
|
706
|
+
)
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
/**
|
|
710
|
+
* Where `p` leads, every link on the way followed, a dangling one included:
|
|
711
|
+
* `realpath` for the part that exists, and past the first missing name the
|
|
712
|
+
* rest as written — where `mkdir -p` would create it if it followed links.
|
|
713
|
+
* A cycle is the user's tree, not the artifact, so it is refused by name.
|
|
714
|
+
*/
|
|
715
|
+
async function resolveThrough(p: string): Promise<string> {
|
|
716
|
+
const pending = path.resolve(p).split(path.sep).filter(Boolean).reverse()
|
|
717
|
+
let at: string = path.sep
|
|
718
|
+
let hops = 0
|
|
719
|
+
while (pending.length > 0) {
|
|
720
|
+
const next = path.join(at, pending.pop()!)
|
|
721
|
+
const link = await readlink(next).catch((err: NodeJS.ErrnoException) => {
|
|
722
|
+
if (err.code === 'EINVAL') return null
|
|
723
|
+
if (err.code === 'ENOENT' || err.code === 'ENOTDIR') return undefined
|
|
724
|
+
throw err
|
|
725
|
+
})
|
|
726
|
+
if (link === undefined) return path.join(next, ...pending.reverse())
|
|
727
|
+
if (link === null) {
|
|
728
|
+
at = next
|
|
729
|
+
continue
|
|
730
|
+
}
|
|
731
|
+
// Linux's own bound on links followed in one lookup (MAXSYMLINKS).
|
|
732
|
+
if (++hops > 40) {
|
|
733
|
+
throw new UserError(
|
|
734
|
+
`${p} goes through a cycle of symbolic links (at ${next}) — a cache restore cannot ` +
|
|
735
|
+
'write through it. Remove the link and re-run.',
|
|
736
|
+
)
|
|
737
|
+
}
|
|
738
|
+
if (path.isAbsolute(link)) at = path.sep
|
|
739
|
+
pending.push(...link.split(path.sep).filter(Boolean).reverse())
|
|
740
|
+
}
|
|
741
|
+
return at
|
|
742
|
+
}
|
|
743
|
+
|
|
621
744
|
/**
|
|
622
745
|
* Path-traversal defense, applied to every entry name at read time.
|
|
623
746
|
* Mirrors Turbo's `restore.rs:check_path` normalization: an artifact is
|
|
@@ -651,8 +774,34 @@ function assertSafeName(name: string): void {
|
|
|
651
774
|
if (name.includes('\0')) {
|
|
652
775
|
throw new ArchiveSecurityError(`archive entry name contains a null byte (unsafe): ${name}`)
|
|
653
776
|
}
|
|
777
|
+
// Only a pax `path` record can carry a name this long. Left to the file
|
|
778
|
+
// system it is a raw ENAMETOOLONG, which the restore reports as an
|
|
779
|
+
// internal error rather than a refused artifact. A UTF-16 unit is at most
|
|
780
|
+
// three UTF-8 bytes, so the common short name never pays for the count.
|
|
781
|
+
if (name.length * 3 >= PATH_MAX && Buffer.byteLength(name) >= PATH_MAX) {
|
|
782
|
+
throw new ArchiveSecurityError(
|
|
783
|
+
`archive entry name is ${Buffer.byteLength(name)} bytes, past PATH_MAX (${PATH_MAX}) (unsafe): ${name.slice(0, 64)}…`,
|
|
784
|
+
)
|
|
785
|
+
}
|
|
786
|
+
// The same raw ENAMETOOLONG for one component past NAME_MAX, the same on
|
|
787
|
+
// Linux and macOS. Same cheap bound: a name under 86 units cannot hold one.
|
|
788
|
+
if (name.length * 3 > NAME_MAX) {
|
|
789
|
+
for (const part of name.split('/')) {
|
|
790
|
+
if (part.length * 3 > NAME_MAX && Buffer.byteLength(part) > NAME_MAX) {
|
|
791
|
+
throw new ArchiveSecurityError(
|
|
792
|
+
`archive entry has a ${Buffer.byteLength(part)}-byte component, past NAME_MAX (${NAME_MAX}) (unsafe): ${part.slice(0, 64)}…`,
|
|
793
|
+
)
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
}
|
|
654
797
|
}
|
|
655
798
|
|
|
799
|
+
/** Bytes one path component may hold (limits.h), on Linux and macOS alike. */
|
|
800
|
+
const NAME_MAX = 255
|
|
801
|
+
|
|
802
|
+
/** Bytes a path may hold, its terminating NUL included (limits.h). */
|
|
803
|
+
const PATH_MAX = process.platform === 'darwin' ? 1024 : 4096
|
|
804
|
+
|
|
656
805
|
function hasParentSegment(p: string): boolean {
|
|
657
806
|
if (p === '..' || p.startsWith('../') || p.endsWith('/..')) return true
|
|
658
807
|
return p.includes('/../')
|