@vzn/vx 0.0.16 → 0.0.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +135 -124
- package/index.ts +10 -0
- package/launcher.mjs +10 -11
- package/package.json +11 -7
- package/src/bin.ts +14 -4
- package/src/cache/archive.ts +659 -0
- package/src/cache/cache.ts +769 -1101
- package/src/cache/chained-cache.ts +194 -0
- package/src/cache/config-evals.ts +89 -0
- package/src/cache/file-hashes.ts +262 -0
- package/src/cache/git-inputs.ts +637 -0
- package/src/cache/index.ts +24 -5
- package/src/cache/inputs.ts +373 -456
- package/src/cache/layer.ts +604 -0
- package/src/cache/layered-cache.ts +96 -43
- package/src/cache/output-index.ts +224 -0
- package/src/cache/policy.ts +84 -0
- package/src/cache/run-history.ts +147 -0
- package/src/cache/tar-stream.ts +358 -0
- package/src/cache/zstd.ts +160 -0
- package/src/cli/cache.ts +75 -21
- package/src/cli/completions.ts +104 -0
- package/src/cli/core-alias.ts +25 -0
- package/src/cli/help.ts +195 -128
- package/src/cli/index.ts +116 -46
- package/src/cli/info.ts +122 -63
- package/src/cli/init.ts +75 -0
- package/src/cli/last.ts +172 -0
- package/src/cli/lock.ts +2 -1
- package/src/cli/plan-format.ts +72 -6
- package/src/cli/plugin-commands.ts +103 -0
- package/src/cli/run.ts +245 -271
- package/src/cli/select.ts +332 -0
- package/src/cli/show.ts +144 -69
- package/src/cli/upgrade.ts +9 -7
- package/src/cli/watch.ts +858 -110
- package/src/cli/why.ts +223 -0
- package/src/cli/workspace-config.ts +126 -0
- package/src/config.ts +187 -150
- package/src/exec/env.ts +18 -1
- package/src/exec/executor.ts +224 -0
- package/src/exec/index.ts +24 -2
- package/src/exec/local-executor.ts +41 -0
- package/src/exec/runner.ts +166 -40
- package/src/exec/sandbox-binds.ts +182 -0
- package/src/exec/sandbox-paths.ts +50 -0
- package/src/exec/sandbox-runtime.ts +642 -260
- package/src/exec/sandbox-violations.ts +280 -0
- package/src/graph/index.ts +4 -10
- package/src/graph/scheduler.ts +232 -203
- package/src/graph/task-graph.ts +222 -8
- package/src/index.ts +146 -153
- package/src/orchestrator/admission.ts +137 -0
- package/src/orchestrator/deferred-outputs.ts +147 -0
- package/src/orchestrator/doctor.ts +255 -0
- package/src/orchestrator/download-policy.ts +171 -0
- package/src/orchestrator/events.ts +72 -8
- package/src/orchestrator/execute-task.ts +374 -549
- package/src/orchestrator/failure-mode.ts +202 -0
- package/src/orchestrator/framed-output.ts +126 -69
- package/src/orchestrator/history.ts +126 -59
- package/src/orchestrator/hit-restore.ts +221 -0
- package/src/orchestrator/index.ts +77 -93
- package/src/orchestrator/local-shortcircuit.ts +22 -0
- package/src/orchestrator/lockfile-claim.ts +281 -0
- package/src/orchestrator/logger.ts +208 -35
- package/src/orchestrator/metrics.ts +102 -1388
- package/src/orchestrator/miss-save.ts +169 -0
- package/src/orchestrator/options.ts +55 -34
- package/src/orchestrator/persistent.ts +79 -0
- package/src/orchestrator/placement.ts +182 -0
- package/src/orchestrator/plan.ts +144 -1
- package/src/orchestrator/plugin-host.ts +357 -120
- package/src/orchestrator/plugin.ts +298 -58
- package/src/orchestrator/prepare.ts +202 -106
- package/src/orchestrator/projects.ts +259 -0
- package/src/orchestrator/remote-prefetch.ts +12 -8
- package/src/orchestrator/run-artifacts.ts +63 -16
- package/src/orchestrator/run-context.ts +167 -29
- package/src/orchestrator/run-records.ts +149 -0
- package/src/orchestrator/run-report.ts +74 -47
- package/src/orchestrator/run.ts +534 -417
- package/src/orchestrator/sandbox-request.ts +243 -0
- package/src/orchestrator/save-lane.ts +47 -0
- package/src/orchestrator/signals.ts +111 -0
- package/src/orchestrator/stable-keys.ts +129 -31
- package/src/orchestrator/status-line.ts +49 -9
- package/src/orchestrator/summary.ts +90 -26
- package/src/orchestrator/tally.ts +78 -35
- package/src/orchestrator/task-hash.ts +111 -15
- package/src/orchestrator/task-log-buffer.ts +288 -0
- package/src/orchestrator/telemetry-host.ts +34 -6
- package/src/orchestrator/telemetry.ts +182 -36
- package/src/orchestrator/upstream.ts +72 -31
- package/src/util/cgroup.ts +160 -0
- package/src/util/edit-distance.ts +70 -0
- package/src/util/errors.ts +15 -1
- package/src/util/index.ts +15 -2
- package/src/util/num.ts +37 -0
- package/src/util/paths.ts +64 -0
- package/src/util/settle.ts +67 -0
- package/src/util/size.ts +5 -1
- package/src/util/tail.ts +66 -0
- package/src/util/timing.ts +63 -0
- package/src/util/verbs.ts +39 -0
- package/src/workspace/affected.ts +250 -23
- package/src/workspace/config-cache.ts +313 -0
- package/src/workspace/config-eval.ts +217 -0
- package/src/workspace/config-imports.ts +162 -0
- package/src/workspace/config-schema.ts +837 -0
- package/src/workspace/filter.ts +65 -10
- package/src/workspace/fingerprint.ts +53 -4
- package/src/workspace/index.ts +42 -3
- package/src/workspace/lockfile.ts +25 -9
- package/src/workspace/migrate-scripts.ts +134 -0
- package/src/workspace/migration.ts +314 -0
- package/src/workspace/nested-dirs.ts +21 -7
- package/src/workspace/package-graph.ts +114 -41
- package/src/workspace/project-loader.ts +224 -450
- package/src/workspace/workspace.ts +298 -72
- package/src/cache/cas-backend.ts +0 -97
- package/src/cache/digest.ts +0 -47
- package/src/cache/tar.ts +0 -343
- package/src/cli/backend.ts +0 -23
- package/src/cli/mcp-rpc.ts +0 -292
- package/src/cli/mcp.ts +0 -92
- package/src/cli/migrate-nx.ts +0 -450
- package/src/cli/migrate-turbo.ts +0 -364
- package/src/cli/migrate.ts +0 -219
- package/src/orchestrator/devframe-surface.ts +0 -74
- package/src/orchestrator/predict.ts +0 -106
- package/src/orchestrator/protocol.ts +0 -156
- package/src/orchestrator/resources.ts +0 -59
- package/src/orchestrator/run-state.ts +0 -145
- package/src/orchestrator/verify.ts +0 -164
- package/src/orchestrator/wire-render.ts +0 -92
- package/src/orchestrator/wire.ts +0 -212
|
@@ -0,0 +1,659 @@
|
|
|
1
|
+
// Cache artifact container: a zstd-compressed tar, read and written by
|
|
2
|
+
// vx's own streaming tar code (`tar-stream.ts`).
|
|
3
|
+
//
|
|
4
|
+
// An artifact is a tar of:
|
|
5
|
+
// stdout always present; the vx-artifact marker
|
|
6
|
+
// outputs/<rel> project-relative outputs
|
|
7
|
+
// workspace-outputs/<rel> workspace-root-relative outputs
|
|
8
|
+
// .vx-meta.json per-output mode + millisecond mtime
|
|
9
|
+
//
|
|
10
|
+
// Tar headers carry mode and SECOND mtimes; vx needs both exactly — a
|
|
11
|
+
// lost executable bit builds cold and breaks warm, and `isOutputsCurrent`
|
|
12
|
+
// skips a restore when size+mode+mtime match at millisecond precision —
|
|
13
|
+
// so the pack side stats each output once and writes both into
|
|
14
|
+
// `.vx-meta.json`, and the restore side applies them. That also makes a
|
|
15
|
+
// REMOTE-ingested entry's index rows millisecond-accurate.
|
|
16
|
+
//
|
|
17
|
+
// Both directions stream. `packArtifactStream` reads each output as it is
|
|
18
|
+
// written; `scanArtifact` lists what an artifact holds without
|
|
19
|
+
// materialising a byte; `extractArtifactStream` restores while never
|
|
20
|
+
// holding more than a chunk. `Bun.Archive` used to do all three and
|
|
21
|
+
// needed the whole tar in memory on every side — restoring a 150 MiB
|
|
22
|
+
// artifact peaked at 3.2× its size, saving one at 4.7× (measured
|
|
23
|
+
// 2026-09-03). Every regular entry is written beside its target under a
|
|
24
|
+
// `.vx-tmp-*` name and renamed into place only once the WHOLE archive has
|
|
25
|
+
// been read and every name proven safe, so a poisoned entry anywhere —
|
|
26
|
+
// even the last one — leaves nothing behind but the empty directories it
|
|
27
|
+
// needed, which are pruned too.
|
|
28
|
+
//
|
|
29
|
+
// What stays here is the part no tar reader can decide for us: WHERE an
|
|
30
|
+
// entry may land on disk. Every name is validated and every destination
|
|
31
|
+
// is proven to stay inside its anchor before a byte is written, because a
|
|
32
|
+
// cache artifact is attacker-reachable (a poisoned remote, a tampered
|
|
33
|
+
// local entry) and "zip slip" is the class.
|
|
34
|
+
//
|
|
35
|
+
// Entries that are not regular files (symlinks, hardlinks, devices,
|
|
36
|
+
// FIFOs) are never materialised — the reader reports them only to be
|
|
37
|
+
// skipped — vx's outputs are regular files, and an artifact that claims
|
|
38
|
+
// otherwise silently loses the claim rather than acting on it.
|
|
39
|
+
|
|
40
|
+
import { chmodSync, renameSync, statSync, utimesSync } from 'node:fs'
|
|
41
|
+
import { mkdir, realpath, rmdir, stat, unlink, writeFile } from 'node:fs/promises'
|
|
42
|
+
import path from 'node:path'
|
|
43
|
+
import { UserError } from '../util/index.js'
|
|
44
|
+
import { TarFormatError, type TarInput, tarEntries, tarPack, tarSize } from './tar-stream.js'
|
|
45
|
+
|
|
46
|
+
/** Archive entry name carrying the per-output mode/mtime sidecar. */
|
|
47
|
+
const META_ENTRY = '.vx-meta.json'
|
|
48
|
+
|
|
49
|
+
/** Archive entry name of the always-present stdout record. */
|
|
50
|
+
export const STDOUT_ENTRY = 'stdout'
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Sidecar shape. `files` maps an entry name to `[mode, mtimeMs]` — a
|
|
54
|
+
* pair, not an object, because the sidecar is real entropy inside an
|
|
55
|
+
* otherwise highly compressible artifact and per-file key names would
|
|
56
|
+
* be paid for once per output.
|
|
57
|
+
*/
|
|
58
|
+
interface MetaFile {
|
|
59
|
+
/** Bumped only if the sidecar's own shape changes; the artifact
|
|
60
|
+
* container is versioned by CACHE_VERSION, which gates reads. */
|
|
61
|
+
version: 1
|
|
62
|
+
files: Record<string, [mode: number, mtimeMs: number]>
|
|
63
|
+
/**
|
|
64
|
+
* What the PRODUCING execution used (2026-09-12, additive: an artifact
|
|
65
|
+
* without it reads as before). Rides the artifact so a machine that
|
|
66
|
+
* never ran the task — a fresh CI runner on a remote hit — still learns
|
|
67
|
+
* what it needs; every wire ships the bytes verbatim, so no seam
|
|
68
|
+
* changes.
|
|
69
|
+
*/
|
|
70
|
+
exec?: ExecUsage
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** CPU time and peak RSS of the execution that produced an artifact. */
|
|
74
|
+
export interface ExecUsage {
|
|
75
|
+
cpuMs?: number
|
|
76
|
+
peakRssBytes?: number
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** One regular file in an artifact, with its restore metadata resolved. */
|
|
80
|
+
export interface ArchiveEntry {
|
|
81
|
+
/** Full archive-relative name, e.g. `outputs/dist/index.js`. */
|
|
82
|
+
name: string
|
|
83
|
+
size: number
|
|
84
|
+
/** Permission bits (0o777 mask) from the sidecar. */
|
|
85
|
+
mode: number
|
|
86
|
+
/** Modification time in ms since epoch from the sidecar. */
|
|
87
|
+
mtimeMs: number
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Thrown when an entry's name would let it escape the extraction anchor
|
|
92
|
+
* (absolute path, `..` traversal, doubled separator, Windows shapes).
|
|
93
|
+
* Callers treat this as a corrupt / malicious artifact — log it, drop
|
|
94
|
+
* the entry, re-run the task.
|
|
95
|
+
*/
|
|
96
|
+
export class ArchiveSecurityError extends Error {
|
|
97
|
+
constructor(message: string) {
|
|
98
|
+
super(message)
|
|
99
|
+
this.name = 'ArchiveSecurityError'
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** What `packArtifactStream` packs: stdout plus archive name → absolute source path. */
|
|
104
|
+
export interface PackArgs {
|
|
105
|
+
stdout: string
|
|
106
|
+
outputs: ReadonlyMap<string, string>
|
|
107
|
+
/** The producing execution's usage, when the runner reported it. */
|
|
108
|
+
exec?: ExecUsage | undefined
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** A packed artifact's plan: its entries with their stats, and the tar's exact size. */
|
|
112
|
+
export interface ArtifactPlan {
|
|
113
|
+
inputs: TarInput[]
|
|
114
|
+
size: number
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Stat every output once (size, and the sidecar's mode + millisecond
|
|
119
|
+
* mtime) and lay out the tar — stdout first, the outputs, the sidecar
|
|
120
|
+
* last — with its exact size, so the caller can choose the one-call pack
|
|
121
|
+
* for a small artifact and the streamed one for a large one before a
|
|
122
|
+
* byte is read. A symlinked output is stored as its target's content, as
|
|
123
|
+
* it always was.
|
|
124
|
+
*/
|
|
125
|
+
export async function planArtifact(args: PackArgs): Promise<ArtifactPlan> {
|
|
126
|
+
const meta: MetaFile = { version: 1, files: {} }
|
|
127
|
+
const exec = usageOf(args.exec)
|
|
128
|
+
if (exec !== undefined) meta.exec = exec
|
|
129
|
+
const files = await Promise.all(
|
|
130
|
+
[...args.outputs].map(async ([name, abs]) => {
|
|
131
|
+
// `stat` follows a symlink, so a link to a file is packed as that
|
|
132
|
+
// file's bytes and comes back as a regular file. A link to a
|
|
133
|
+
// directory, or a dangling one, has no bytes to pack: refuse loudly
|
|
134
|
+
// rather than store an entry that restores to nothing.
|
|
135
|
+
const shown = name.startsWith('outputs/') ? name.slice('outputs/'.length) : name
|
|
136
|
+
const st = await stat(abs).catch((err: NodeJS.ErrnoException) => {
|
|
137
|
+
if (err.code === 'ENOENT') {
|
|
138
|
+
throw new UserError(
|
|
139
|
+
`output ${shown} is a dangling symlink: vx stores regular files only — emit a file, or exclude it from cache.outputs`,
|
|
140
|
+
)
|
|
141
|
+
}
|
|
142
|
+
throw err
|
|
143
|
+
})
|
|
144
|
+
if (!st.isFile()) {
|
|
145
|
+
throw new UserError(
|
|
146
|
+
`output ${shown} is not a regular file (a symlink to a directory?): vx stores regular files only — emit a file, or exclude it from cache.outputs`,
|
|
147
|
+
)
|
|
148
|
+
}
|
|
149
|
+
meta.files[name] = [st.mode & 0o777, Math.floor(st.mtimeMs)]
|
|
150
|
+
return {
|
|
151
|
+
name,
|
|
152
|
+
abs,
|
|
153
|
+
size: st.size,
|
|
154
|
+
mode: st.mode & 0o777,
|
|
155
|
+
mtime: Math.floor(st.mtimeMs / 1000),
|
|
156
|
+
}
|
|
157
|
+
}),
|
|
158
|
+
)
|
|
159
|
+
const stdout = new TextEncoder().encode(args.stdout)
|
|
160
|
+
const metaBytes = new TextEncoder().encode(JSON.stringify(meta))
|
|
161
|
+
const inputs: TarInput[] = [
|
|
162
|
+
{ name: STDOUT_ENTRY, size: stdout.byteLength, body: stdout },
|
|
163
|
+
...files.map((f) => ({
|
|
164
|
+
name: f.name,
|
|
165
|
+
size: f.size,
|
|
166
|
+
mode: f.mode,
|
|
167
|
+
mtime: f.mtime,
|
|
168
|
+
body: Bun.file(f.abs) as Blob,
|
|
169
|
+
})),
|
|
170
|
+
{ name: META_ENTRY, size: metaBytes.byteLength, body: metaBytes },
|
|
171
|
+
]
|
|
172
|
+
return { inputs, size: tarSize(inputs) }
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/** The tar as a stream: each output is read from disk as it is written. */
|
|
176
|
+
export function packArtifactStream(plan: ArtifactPlan): ReadableStream<Uint8Array> {
|
|
177
|
+
return streamOf(tarPack(plan.inputs))
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* The tar in memory — the small-artifact path and the tests. Bodies are
|
|
182
|
+
* read whole (one `bytes()` each, in flight together) rather than
|
|
183
|
+
* streamed: a thousand one-file artifacts pay the stream setup
|
|
184
|
+
* otherwise (measured 2026-09-03: 238 → 293 ms per 1 000 saves).
|
|
185
|
+
*/
|
|
186
|
+
export async function packArtifactBytes(plan: ArtifactPlan): Promise<Uint8Array> {
|
|
187
|
+
const inputs = await Promise.all(
|
|
188
|
+
plan.inputs.map(async (i) =>
|
|
189
|
+
i.body instanceof Blob ? { ...i, body: await i.body.bytes() } : i,
|
|
190
|
+
),
|
|
191
|
+
)
|
|
192
|
+
const out = new Uint8Array(plan.size)
|
|
193
|
+
let off = 0
|
|
194
|
+
for await (const chunk of tarPack(inputs)) {
|
|
195
|
+
out.set(chunk, off)
|
|
196
|
+
off += chunk.byteLength
|
|
197
|
+
}
|
|
198
|
+
if (off !== plan.size) throw new TarFormatError(`packed ${off} bytes, planned ${plan.size}`)
|
|
199
|
+
return out
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/** Plan and pack in one go, in memory. */
|
|
203
|
+
export async function packArtifact(args: PackArgs): Promise<Uint8Array> {
|
|
204
|
+
return await packArtifactBytes(await planArtifact(args))
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
function streamOf(gen: AsyncGenerator<Uint8Array>): ReadableStream<Uint8Array> {
|
|
208
|
+
return new ReadableStream<Uint8Array>({
|
|
209
|
+
async pull(controller) {
|
|
210
|
+
const { done, value } = await gen.next()
|
|
211
|
+
if (done) controller.close()
|
|
212
|
+
else controller.enqueue(value)
|
|
213
|
+
},
|
|
214
|
+
cancel() {
|
|
215
|
+
void gen.return(undefined)
|
|
216
|
+
},
|
|
217
|
+
})
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* List an artifact's regular entries with their restore metadata, and
|
|
222
|
+
* its stdout text (`null` when the entry is absent — not a vx artifact),
|
|
223
|
+
* without materialising anything. Rejects the WHOLE archive when any
|
|
224
|
+
* name is unsafe — a partial reading of a poisoned artifact is the
|
|
225
|
+
* outcome the traversal defense exists to prevent.
|
|
226
|
+
*/
|
|
227
|
+
export async function scanArtifact(
|
|
228
|
+
tar: ReadableStream<Uint8Array>,
|
|
229
|
+
): Promise<{ entries: ArchiveEntry[]; stdout: string | null; exec: ExecUsage | undefined }> {
|
|
230
|
+
const seen: Array<{ name: string; size: number; mtimeMs: number }> = []
|
|
231
|
+
let meta: MetaFile['files'] = {}
|
|
232
|
+
let exec: ExecUsage | undefined
|
|
233
|
+
let stdout: string | null = null
|
|
234
|
+
for await (const e of tarEntries(tar)) {
|
|
235
|
+
if (e.type !== '0') continue
|
|
236
|
+
if (e.name === META_ENTRY) {
|
|
237
|
+
const parsed = JSON.parse(await textOf(e.body)) as MetaFile
|
|
238
|
+
meta = parsed.files ?? {}
|
|
239
|
+
// The ingest side is the untrusted boundary: a foreign sidecar's
|
|
240
|
+
// usage is taken only when it is a plain non-negative number.
|
|
241
|
+
exec = usageOf(parsed.exec)
|
|
242
|
+
continue
|
|
243
|
+
}
|
|
244
|
+
assertSafeName(e.name)
|
|
245
|
+
if (e.name === STDOUT_ENTRY) stdout = await textOf(e.body)
|
|
246
|
+
seen.push({ name: e.name, size: e.size, mtimeMs: e.mtimeMs })
|
|
247
|
+
}
|
|
248
|
+
return {
|
|
249
|
+
entries: seen.map((s) => {
|
|
250
|
+
const m = meta[s.name]
|
|
251
|
+
// A foreign artifact (or one whose sidecar lost an entry) restores
|
|
252
|
+
// readable-but-not-executable rather than failing the hit: the
|
|
253
|
+
// sidecar is metadata, never the authority on whether bytes exist.
|
|
254
|
+
return { name: s.name, size: s.size, mode: m?.[0] ?? 0o644, mtimeMs: m?.[1] ?? s.mtimeMs }
|
|
255
|
+
}),
|
|
256
|
+
stdout,
|
|
257
|
+
exec,
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/** The usage worth carrying: each axis a finite non-negative number, or absent; nothing → undefined. */
|
|
262
|
+
function usageOf(raw: unknown): ExecUsage | undefined {
|
|
263
|
+
if (typeof raw !== 'object' || raw === null) return undefined
|
|
264
|
+
const { cpuMs, peakRssBytes } = raw as { cpuMs?: unknown; peakRssBytes?: unknown }
|
|
265
|
+
const out: { cpuMs?: number; peakRssBytes?: number } = {}
|
|
266
|
+
if (typeof cpuMs === 'number' && Number.isFinite(cpuMs) && cpuMs >= 0) out.cpuMs = cpuMs
|
|
267
|
+
if (typeof peakRssBytes === 'number' && Number.isFinite(peakRssBytes) && peakRssBytes >= 0)
|
|
268
|
+
out.peakRssBytes = peakRssBytes
|
|
269
|
+
return out.cpuMs === undefined && out.peakRssBytes === undefined ? undefined : out
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/**
|
|
273
|
+
* Restore from `tar`, the DECOMPRESSED archive as a byte stream:
|
|
274
|
+
* `outputs/<rel>` lands under `destDir/<rel>` and — when `workspaceDest`
|
|
275
|
+
* is given — `workspace-outputs/<rel>` under `workspaceDest/<rel>`;
|
|
276
|
+
* entries in neither namespace are ignored. Entries are validated and staged
|
|
277
|
+
* as they arrive and renamed into place only after the archive has ended
|
|
278
|
+
* cleanly and `verify` (the caller's look at which output names the
|
|
279
|
+
* archive provided — the index's missing-output check) has passed. Any
|
|
280
|
+
* failure — an unsafe name, an escape, a truncated stream, `verify` —
|
|
281
|
+
* leaves nothing behind. Returns the provided output names.
|
|
282
|
+
*/
|
|
283
|
+
export async function extractArtifactStream(
|
|
284
|
+
tar: ReadableStream<Uint8Array>,
|
|
285
|
+
destDir: string,
|
|
286
|
+
workspaceDest: string | undefined,
|
|
287
|
+
verify?: (provided: ReadonlySet<string>) => void,
|
|
288
|
+
): Promise<Set<string>> {
|
|
289
|
+
await mkdir(destDir, { recursive: true })
|
|
290
|
+
const x = new Extractor(destDir, workspaceDest)
|
|
291
|
+
const provided = new Set<string>()
|
|
292
|
+
const headerMtime = new Map<string, number>()
|
|
293
|
+
let meta: MetaFile['files'] = {}
|
|
294
|
+
try {
|
|
295
|
+
for await (const e of tarEntries(tar)) {
|
|
296
|
+
if (e.type !== '0') continue
|
|
297
|
+
if (e.name === META_ENTRY) {
|
|
298
|
+
meta = (JSON.parse(await textOf(e.body)) as MetaFile).files ?? {}
|
|
299
|
+
continue
|
|
300
|
+
}
|
|
301
|
+
assertSafeName(e.name)
|
|
302
|
+
const dest = x.destFor(e.name)
|
|
303
|
+
if (dest === null || dest.rel.length === 0) continue
|
|
304
|
+
const target = path.join(dest.base, dest.rel)
|
|
305
|
+
await x.assertContained(dest.base, target, e.name)
|
|
306
|
+
provided.add(e.name)
|
|
307
|
+
headerMtime.set(e.name, e.mtimeMs)
|
|
308
|
+
await x.stage(e.name, target, e.size <= SMALL_ENTRY ? await bytesOf(e.body) : e.body)
|
|
309
|
+
}
|
|
310
|
+
verify?.(provided)
|
|
311
|
+
await x.commit((name) => {
|
|
312
|
+
const m = meta[name]
|
|
313
|
+
return [m?.[0] ?? 0o644, m?.[1] ?? headerMtime.get(name) ?? 0]
|
|
314
|
+
})
|
|
315
|
+
} catch (err) {
|
|
316
|
+
await x.abort()
|
|
317
|
+
throw err
|
|
318
|
+
}
|
|
319
|
+
return provided
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/**
|
|
323
|
+
* Entries up to this size are buffered and written without waiting, so a
|
|
324
|
+
* thousand tiny outputs restore in parallel; larger ones stream chunk by
|
|
325
|
+
* chunk to a file sink.
|
|
326
|
+
*/
|
|
327
|
+
const SMALL_ENTRY = 4 * 1024 * 1024
|
|
328
|
+
|
|
329
|
+
/** Buffered small writes in flight before the next one waits for them. */
|
|
330
|
+
const INFLIGHT_BYTES = 64 * 1024 * 1024
|
|
331
|
+
|
|
332
|
+
async function bytesOf(body: AsyncIterable<Uint8Array>): Promise<Uint8Array> {
|
|
333
|
+
const parts: Uint8Array[] = []
|
|
334
|
+
let n = 0
|
|
335
|
+
for await (const c of body) {
|
|
336
|
+
parts.push(new Uint8Array(c))
|
|
337
|
+
n += c.byteLength
|
|
338
|
+
}
|
|
339
|
+
if (parts.length === 1) return parts[0]!
|
|
340
|
+
const out = new Uint8Array(n)
|
|
341
|
+
let off = 0
|
|
342
|
+
for (const p of parts) {
|
|
343
|
+
out.set(p, off)
|
|
344
|
+
off += p.byteLength
|
|
345
|
+
}
|
|
346
|
+
return out
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
const textOf = async (body: AsyncIterable<Uint8Array>): Promise<string> =>
|
|
350
|
+
new TextDecoder().decode(await bytesOf(body))
|
|
351
|
+
|
|
352
|
+
/** One staged entry: written at `tmp`, renamed to `target` on commit. */
|
|
353
|
+
interface Staged {
|
|
354
|
+
name: string
|
|
355
|
+
tmp: string
|
|
356
|
+
target: string
|
|
357
|
+
/** The topmost directory `mkdir -p` created for it, pruned on abort. */
|
|
358
|
+
created: string | undefined
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
/**
|
|
362
|
+
* The staging core. Nothing reaches its final name until `commit`;
|
|
363
|
+
* `abort` removes every temp and the empty directories this extraction
|
|
364
|
+
* created.
|
|
365
|
+
*/
|
|
366
|
+
class Extractor {
|
|
367
|
+
private readonly staged: Staged[] = []
|
|
368
|
+
private inflight: Promise<unknown>[] = []
|
|
369
|
+
private inflightBytes = 0
|
|
370
|
+
private readonly realBaseCache = new Map<string, string>()
|
|
371
|
+
// Per-directory memos. An artifact holds a few files per directory
|
|
372
|
+
// (payload's ui: 4,069 files in 535 directories), and the containment
|
|
373
|
+
// walk and the `mkdir -p` are properties of the directory, not the
|
|
374
|
+
// entry: doing both once per directory took the restore of that
|
|
375
|
+
// artifact from 220–300 µs per file to the write itself. The window
|
|
376
|
+
// this opens — a symlink planted at a validated directory between two
|
|
377
|
+
// of its entries — is the same one that already lies between an
|
|
378
|
+
// entry's own check and its write.
|
|
379
|
+
private readonly containedDirs = new Set<string>()
|
|
380
|
+
private readonly ensuredDirs = new Set<string>()
|
|
381
|
+
|
|
382
|
+
constructor(
|
|
383
|
+
private readonly destDir: string,
|
|
384
|
+
private readonly workspaceDest: string | undefined,
|
|
385
|
+
) {}
|
|
386
|
+
|
|
387
|
+
/** Each namespace anchors at its own destination dir. */
|
|
388
|
+
destFor(name: string): { base: string; rel: string } | null {
|
|
389
|
+
if (name.startsWith('outputs/')) {
|
|
390
|
+
return { base: this.destDir, rel: name.slice('outputs/'.length) }
|
|
391
|
+
}
|
|
392
|
+
if (this.workspaceDest !== undefined && name.startsWith(WORKSPACE_PREFIX)) {
|
|
393
|
+
return { base: this.workspaceDest, rel: name.slice(WORKSPACE_PREFIX.length) }
|
|
394
|
+
}
|
|
395
|
+
return null
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
// The REAL path of each base dir (symlinks resolved), memoized. Used to
|
|
399
|
+
// catch a symlinked ANCESTOR under the output tree: the lexical check
|
|
400
|
+
// below stops a `..` name, but if `<dest>/dist` is already an on-disk
|
|
401
|
+
// symlink pointing outside (planted in the repo or by a dependency
|
|
402
|
+
// postinstall), a lexically-contained entry `dist/x` would follow it and
|
|
403
|
+
// escape. Both sides are realpath'd, so a legitimate symlinked ANCESTOR of
|
|
404
|
+
// the base dir itself (e.g. macOS /tmp → /private/tmp) resolves
|
|
405
|
+
// consistently and is not flagged.
|
|
406
|
+
private async realBaseOf(base: string): Promise<string> {
|
|
407
|
+
const key = path.resolve(base)
|
|
408
|
+
let r = this.realBaseCache.get(key)
|
|
409
|
+
if (r === undefined) {
|
|
410
|
+
// The base often does NOT exist yet — the workspace-outputs anchor is
|
|
411
|
+
// created lazily by the first entry. Falling back to the un-resolved
|
|
412
|
+
// path there would compare a real ancestor (`/private/tmp/...`, once
|
|
413
|
+
// mkdir has created it) against a symlinked base (`/tmp/...`) and
|
|
414
|
+
// refuse EVERY entry with a bogus security error. So resolve the
|
|
415
|
+
// deepest EXISTING ancestor and re-append the rest: the answer is the
|
|
416
|
+
// same before and after the base is created, which is what makes the
|
|
417
|
+
// memo safe.
|
|
418
|
+
r = await realpath(key).catch(async () => {
|
|
419
|
+
let probe = path.dirname(key)
|
|
420
|
+
const tail: string[] = [path.basename(key)]
|
|
421
|
+
while (probe !== path.dirname(probe)) {
|
|
422
|
+
const real = await realpath(probe).then(
|
|
423
|
+
(v) => v,
|
|
424
|
+
() => null,
|
|
425
|
+
)
|
|
426
|
+
if (real !== null) return path.join(real, ...tail.reverse())
|
|
427
|
+
tail.push(path.basename(probe))
|
|
428
|
+
probe = path.dirname(probe)
|
|
429
|
+
}
|
|
430
|
+
return key
|
|
431
|
+
})
|
|
432
|
+
this.realBaseCache.set(key, r)
|
|
433
|
+
}
|
|
434
|
+
return r
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
// Two checks per entry: lexical (the joined path stays under the base)
|
|
438
|
+
// and real (the deepest EXISTING ancestor still resolves inside the real
|
|
439
|
+
// base). Walking up to the deepest existing ancestor is what makes it
|
|
440
|
+
// sound BEFORE anything is created — checking only the immediate parent
|
|
441
|
+
// would pass for `dist/a/b` when `dist` is the symlink and `dist/a` does
|
|
442
|
+
// not exist yet.
|
|
443
|
+
async assertContained(base: string, target: string, name: string): Promise<void> {
|
|
444
|
+
const baseResolved = path.resolve(base)
|
|
445
|
+
const targetResolved = path.resolve(target)
|
|
446
|
+
if (targetResolved !== baseResolved && !targetResolved.startsWith(baseResolved + path.sep)) {
|
|
447
|
+
throw new ArchiveSecurityError(`archive entry escapes destDir (unsafe): ${name}`)
|
|
448
|
+
}
|
|
449
|
+
const parentResolved = path.dirname(targetResolved)
|
|
450
|
+
if (this.containedDirs.has(parentResolved)) return
|
|
451
|
+
await this.assertParentContained(base, targetResolved, name)
|
|
452
|
+
this.containedDirs.add(parentResolved)
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
private async assertParentContained(
|
|
456
|
+
base: string,
|
|
457
|
+
targetResolved: string,
|
|
458
|
+
name: string,
|
|
459
|
+
): Promise<void> {
|
|
460
|
+
const baseResolved = path.resolve(base)
|
|
461
|
+
const realBase = await this.realBaseOf(base)
|
|
462
|
+
// Only ancestors strictly BELOW the base are candidates — those are the
|
|
463
|
+
// ones a poisoned entry could follow out of the tree. The walk must never
|
|
464
|
+
// climb past the base: when the base itself does not exist yet (the
|
|
465
|
+
// workspace-outputs anchor is created lazily), its parent legitimately
|
|
466
|
+
// resolves outside and comparing against it would reject every entry.
|
|
467
|
+
let probe = path.dirname(targetResolved)
|
|
468
|
+
while (probe.startsWith(baseResolved + path.sep)) {
|
|
469
|
+
const real = await realpath(probe).then(
|
|
470
|
+
(r) => r,
|
|
471
|
+
() => null,
|
|
472
|
+
)
|
|
473
|
+
if (real !== null) {
|
|
474
|
+
if (real !== realBase && !real.startsWith(realBase + path.sep)) {
|
|
475
|
+
throw new ArchiveSecurityError(
|
|
476
|
+
`archive entry escapes destDir via a symlinked parent: ${name}`,
|
|
477
|
+
)
|
|
478
|
+
}
|
|
479
|
+
return
|
|
480
|
+
}
|
|
481
|
+
probe = path.dirname(probe)
|
|
482
|
+
}
|
|
483
|
+
// Nothing between the base and the target exists yet, so there is no
|
|
484
|
+
// pre-existing symlink to follow — the chain will be created fresh.
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
/**
|
|
488
|
+
* Write the entry beside its target. A buffer is written without
|
|
489
|
+
* waiting, bounded by `INFLIGHT_BYTES`; a chunk stream goes to a file
|
|
490
|
+
* sink one piece at a time.
|
|
491
|
+
*/
|
|
492
|
+
async stage(
|
|
493
|
+
name: string,
|
|
494
|
+
target: string,
|
|
495
|
+
body: Uint8Array | AsyncIterable<Uint8Array>,
|
|
496
|
+
): Promise<void> {
|
|
497
|
+
const dir = path.dirname(target)
|
|
498
|
+
let created: string | undefined
|
|
499
|
+
if (!this.ensuredDirs.has(dir)) {
|
|
500
|
+
created = await mkdir(dir, { recursive: true })
|
|
501
|
+
this.ensuredDirs.add(dir)
|
|
502
|
+
}
|
|
503
|
+
// Write beside the target and RENAME into place. rename(2) replaces
|
|
504
|
+
// the destination's directory ENTRY without following it, which is
|
|
505
|
+
// what makes this both link-safe and concurrency-safe:
|
|
506
|
+
//
|
|
507
|
+
// - a planted symlink or HARDLINK at the target is replaced, not
|
|
508
|
+
// written through, so `ln <victim> <dest>/out.txt` cannot get
|
|
509
|
+
// the artifact's bytes into <victim>;
|
|
510
|
+
// - the target is never momentarily ABSENT. The unlink-then-write
|
|
511
|
+
// version opened exactly that window, and a second extract of
|
|
512
|
+
// the same payload could delete the file between this one's
|
|
513
|
+
// write and its chmod — reproduced 3/400 locally, caught red on
|
|
514
|
+
// darwin CI where a loaded runner widened the race;
|
|
515
|
+
// - mode and mtime are applied BEFORE the file is visible, so no
|
|
516
|
+
// reader sees it with the wrong metadata.
|
|
517
|
+
//
|
|
518
|
+
// A directory at the target makes the rename fail — the same
|
|
519
|
+
// fail-closed outcome the plain write had.
|
|
520
|
+
const tmp = `${target}.vx-tmp-${process.pid.toString(36)}-${(tmpSeq++).toString(36)}`
|
|
521
|
+
this.staged.push({ name, tmp, target, created })
|
|
522
|
+
if (body instanceof Uint8Array) {
|
|
523
|
+
this.inflightBytes += body.byteLength
|
|
524
|
+
// `writeFile`, not `Bun.write`: for a buffer this size `Bun.write`
|
|
525
|
+
// does the open, write and close on the calling thread and only
|
|
526
|
+
// its promise is asynchronous, so ten restores in flight wrote
|
|
527
|
+
// their 14,430 files one after another — a serial restore of
|
|
528
|
+
// payload took the same 6–7 s as one at concurrency 10, and a CPU
|
|
529
|
+
// profile put 6.7 of its 8.0 s inside `write` (2026-09-11). The
|
|
530
|
+
// thread-pool write lets the restores overlap their I/O.
|
|
531
|
+
const write = writeFile(tmp, body)
|
|
532
|
+
// A write that fails before `commit` or `abort` awaits it must not
|
|
533
|
+
// surface as an unhandled rejection; the awaiting `Promise.all` still
|
|
534
|
+
// sees the error through the original promise.
|
|
535
|
+
write.catch(() => undefined)
|
|
536
|
+
this.inflight.push(write)
|
|
537
|
+
if (this.inflightBytes > INFLIGHT_BYTES) await this.drain()
|
|
538
|
+
return
|
|
539
|
+
}
|
|
540
|
+
const sink = Bun.file(tmp).writer()
|
|
541
|
+
try {
|
|
542
|
+
for await (const piece of body) await sink.write(piece)
|
|
543
|
+
} finally {
|
|
544
|
+
await sink.end()
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
private async drain(): Promise<void> {
|
|
549
|
+
const pending = this.inflight
|
|
550
|
+
this.inflight = []
|
|
551
|
+
this.inflightBytes = 0
|
|
552
|
+
await Promise.all(pending)
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
/**
|
|
556
|
+
* Apply each entry's mode and mtime, then rename everything into place.
|
|
557
|
+
* Synchronous calls in batches: the three metadata calls per file are a
|
|
558
|
+
* few microseconds each on the calling thread and ~40 µs each as
|
|
559
|
+
* thread-pool round trips, and a large artifact has thousands of them
|
|
560
|
+
* (measured 2026-09-11 on payload's ui, 4,069 files: 892 ms → see the
|
|
561
|
+
* archive bench). A yield every `COMMIT_BATCH` files keeps the other
|
|
562
|
+
* restores' round trips flowing. The chmod is skipped when the mode is
|
|
563
|
+
* the one the temp file was created with — the common case.
|
|
564
|
+
*/
|
|
565
|
+
async commit(metaFor: (name: string) => [mode: number, mtimeMs: number]): Promise<void> {
|
|
566
|
+
await this.drain()
|
|
567
|
+
const first = this.staged[0]
|
|
568
|
+
const createdMode = first === undefined ? -1 : statSync(first.tmp).mode & 0o777
|
|
569
|
+
let n = 0
|
|
570
|
+
for (const s of this.staged) {
|
|
571
|
+
const [mode, mtimeMs] = metaFor(s.name)
|
|
572
|
+
if (mode !== 0 && (mode & 0o777) !== createdMode) chmodSync(s.tmp, mode & 0o777)
|
|
573
|
+
if (mtimeMs > 0) {
|
|
574
|
+
const t = mtimeMs / 1000
|
|
575
|
+
utimesSync(s.tmp, t, t)
|
|
576
|
+
}
|
|
577
|
+
renameSync(s.tmp, s.target)
|
|
578
|
+
if (++n % COMMIT_BATCH === 0) await new Promise<void>((r) => setImmediate(r))
|
|
579
|
+
}
|
|
580
|
+
this.staged.length = 0
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
/**
|
|
584
|
+
* Never leave a stray `.vx-tmp-*` behind: the declared output globs would
|
|
585
|
+
* sweep it into the next artifact. Directories this extraction created
|
|
586
|
+
* are removed bottom-up while empty; one a concurrent writer has since
|
|
587
|
+
* filled is left alone.
|
|
588
|
+
*/
|
|
589
|
+
async abort(): Promise<void> {
|
|
590
|
+
await Promise.allSettled(this.inflight)
|
|
591
|
+
this.inflight = []
|
|
592
|
+
await Promise.all(this.staged.map((s) => unlink(s.tmp).catch(() => undefined)))
|
|
593
|
+
for (const s of this.staged) {
|
|
594
|
+
if (s.created === undefined) continue
|
|
595
|
+
let dir = path.dirname(s.target)
|
|
596
|
+
const top = path.resolve(s.created)
|
|
597
|
+
while (dir.startsWith(top)) {
|
|
598
|
+
if (
|
|
599
|
+
!(await rmdir(dir).then(
|
|
600
|
+
() => true,
|
|
601
|
+
() => false,
|
|
602
|
+
))
|
|
603
|
+
)
|
|
604
|
+
break
|
|
605
|
+
if (dir === top) break
|
|
606
|
+
dir = path.dirname(dir)
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
this.staged.length = 0
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
/** Files renamed into place between two yields of a commit. */
|
|
614
|
+
const COMMIT_BATCH = 256
|
|
615
|
+
|
|
616
|
+
/** Per-process counter for extract temp names; uniqueness only. */
|
|
617
|
+
let tmpSeq = 0
|
|
618
|
+
|
|
619
|
+
const WORKSPACE_PREFIX = 'workspace-outputs/'
|
|
620
|
+
|
|
621
|
+
/**
|
|
622
|
+
* Path-traversal defense, applied to every entry name at read time.
|
|
623
|
+
* Mirrors Turbo's `restore.rs:check_path` normalization: an artifact is
|
|
624
|
+
* attacker-reachable, so a name that could resolve outside the anchor is
|
|
625
|
+
* refused before anything decides where to write it.
|
|
626
|
+
*/
|
|
627
|
+
function assertSafeName(name: string): void {
|
|
628
|
+
if (name.length === 0) {
|
|
629
|
+
throw new ArchiveSecurityError('archive entry has an empty name')
|
|
630
|
+
}
|
|
631
|
+
if (name.startsWith('/')) {
|
|
632
|
+
throw new ArchiveSecurityError(`archive entry name is absolute (unsafe): ${name}`)
|
|
633
|
+
}
|
|
634
|
+
if (hasParentSegment(name)) {
|
|
635
|
+
throw new ArchiveSecurityError(`archive entry name escapes via '..' (unsafe): ${name}`)
|
|
636
|
+
}
|
|
637
|
+
if (name.includes('//')) {
|
|
638
|
+
throw new ArchiveSecurityError(`archive entry name has empty path component (unsafe): ${name}`)
|
|
639
|
+
}
|
|
640
|
+
// Windows-shaped paths are categorically unsafe to extract on POSIX
|
|
641
|
+
// hosts (and vice versa): backslash separators, drive letters,
|
|
642
|
+
// extended-length prefixes.
|
|
643
|
+
if (name.includes('\\')) {
|
|
644
|
+
throw new ArchiveSecurityError(
|
|
645
|
+
`archive entry uses backslash separators (windows-unsafe): ${name}`,
|
|
646
|
+
)
|
|
647
|
+
}
|
|
648
|
+
if (/^[A-Za-z]:[\\/]/.test(name)) {
|
|
649
|
+
throw new ArchiveSecurityError(`archive entry has windows drive prefix (unsafe): ${name}`)
|
|
650
|
+
}
|
|
651
|
+
if (name.includes('\0')) {
|
|
652
|
+
throw new ArchiveSecurityError(`archive entry name contains a null byte (unsafe): ${name}`)
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
function hasParentSegment(p: string): boolean {
|
|
657
|
+
if (p === '..' || p.startsWith('../') || p.endsWith('/..')) return true
|
|
658
|
+
return p.includes('/../')
|
|
659
|
+
}
|