@vzn/vx 0.0.640 → 0.0.642
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/launcher.sh +9 -2
- package/package.json +5 -12
- package/skills/vx/SKILL.md +4 -4
- package/src/bin.ts +8 -6
- package/src/cache/archive.ts +735 -107
- package/src/cache/cache.ts +133 -29
- package/src/cache/fold-memo.ts +58 -0
- package/src/cache/git-inputs.ts +213 -31
- package/src/cache/index.ts +9 -2
- package/src/cache/inputs.ts +244 -108
- package/src/cache/key-fold.ts +90 -69
- package/src/cache/layer.ts +9 -7
- package/src/cache/layered-cache.ts +74 -9
- package/src/cache/output-index.ts +25 -4
- package/src/cache/remote-lane.ts +27 -10
- package/src/cache/schema.ts +13 -3
- package/src/cache/scoped-cache.ts +11 -3
- package/src/cache/umask.ts +29 -0
- package/src/cli/docs.ts +71 -7
- package/src/cli/help.ts +11 -5
- package/src/cli/plan-format.ts +4 -1
- package/src/cli/run.ts +10 -3
- package/src/cli/select.ts +2 -0
- package/src/cli/upgrade.ts +37 -2
- package/src/cli/watch-filter.ts +1 -6
- package/src/cli/watch-fs.ts +2 -1
- package/src/exec/runner.ts +5 -0
- package/src/exec/sandbox-runtime.ts +81 -1
- package/src/graph/priorities.ts +14 -1
- package/src/handoff.ts +5 -0
- package/src/history/history.ts +43 -33
- package/src/index.ts +11 -1
- package/src/orchestrator/deferred-outputs.ts +3 -2
- package/src/orchestrator/execute-task.ts +20 -14
- package/src/orchestrator/hit-restore.ts +111 -29
- package/src/orchestrator/index.ts +1 -0
- package/src/orchestrator/local-shortcircuit.ts +8 -0
- package/src/orchestrator/lockfile-claim.ts +54 -8
- package/src/orchestrator/miss-save.ts +27 -17
- package/src/orchestrator/output-host.ts +9 -1
- package/src/orchestrator/plan.ts +4 -0
- package/src/orchestrator/plugin-host.ts +35 -5
- package/src/orchestrator/plugin.ts +29 -3
- package/src/orchestrator/prepare.ts +74 -5
- package/src/orchestrator/projects.ts +6 -1
- package/src/orchestrator/run-report.ts +30 -9
- package/src/orchestrator/run.ts +62 -38
- package/src/orchestrator/sandbox-request.ts +1 -48
- package/src/orchestrator/save-lane.ts +17 -10
- package/src/orchestrator/signals.ts +7 -10
- package/src/orchestrator/stable-keys.ts +83 -25
- package/src/orchestrator/task-hash.ts +46 -12
- package/src/report/events.ts +12 -0
- package/src/report/framed-output.ts +2 -1
- package/src/report/logger.ts +58 -15
- package/src/report/status-line.ts +45 -4
- package/src/report/summary.ts +11 -6
- package/src/util/errors.ts +1 -1
- package/src/util/hidden-prompt.ts +74 -0
- package/src/util/index.ts +13 -0
- package/src/util/paths.ts +58 -5
- package/src/util/secrets.ts +150 -0
- package/src/workspace/affected.ts +63 -13
- package/src/workspace/config-eval.ts +2 -2
- package/src/workspace/config-schema.ts +37 -3
- package/src/workspace/project-loader.ts +6 -4
- package/src/workspace/workspace.ts +17 -11
- package/docs/caching.md +0 -2352
- package/docs/cli.md +0 -3280
- package/docs/execution.md +0 -675
- package/docs/features.md +0 -178
- package/docs/patterns.md +0 -219
- package/docs/schema.md +0 -2461
- package/docs/security.md +0 -252
- package/src/cli/docs-corpus.ts +0 -21
- package/src/cli/docs-text.d.ts +0 -5
package/src/cache/archive.ts
CHANGED
|
@@ -20,11 +20,13 @@
|
|
|
20
20
|
// holding more than a chunk. `Bun.Archive` used to do all three and
|
|
21
21
|
// needed the whole tar in memory on every side — restoring a 150 MiB
|
|
22
22
|
// artifact peaked at 3.2× its size, saving one at 4.7× (measured
|
|
23
|
-
// 2026-09-03). Every regular entry is written
|
|
24
|
-
// `.vx
|
|
25
|
-
//
|
|
26
|
-
//
|
|
27
|
-
// needed, which
|
|
23
|
+
// 2026-09-03). Every regular entry is written to the workspace's staging
|
|
24
|
+
// directory (`.vx/tmp/<pid>-0/`, vx's own, never an output folder) and
|
|
25
|
+
// renamed over its target only once the WHOLE archive has been read and
|
|
26
|
+
// every name proven safe, so a poisoned entry anywhere — even the last
|
|
27
|
+
// one — leaves nothing behind but the empty directories it needed, which
|
|
28
|
+
// are pruned too, and a killed restore leaves its temps where the next
|
|
29
|
+
// run sweeps them (`sweepStaging`).
|
|
28
30
|
//
|
|
29
31
|
// What stays here is the part no tar reader can decide for us: WHERE an
|
|
30
32
|
// entry may land on disk. Every name is validated and every destination
|
|
@@ -32,21 +34,44 @@
|
|
|
32
34
|
// cache artifact is attacker-reachable (a poisoned remote, a tampered
|
|
33
35
|
// local entry) and "zip slip" is the class.
|
|
34
36
|
//
|
|
35
|
-
//
|
|
37
|
+
// Tar entries that are not regular files (symlinks, hardlinks, devices,
|
|
36
38
|
// FIFOs) are never materialised — the reader reports them only to be
|
|
37
|
-
// skipped
|
|
38
|
-
//
|
|
39
|
+
// skipped. An output symlink rides the sidecar instead (`links`), its
|
|
40
|
+
// target held to the workspace on both sides.
|
|
39
41
|
|
|
40
42
|
import {
|
|
41
43
|
chmodSync,
|
|
44
|
+
chownSync,
|
|
45
|
+
closeSync,
|
|
46
|
+
constants,
|
|
47
|
+
existsSync,
|
|
48
|
+
fchmodSync,
|
|
49
|
+
fstatSync,
|
|
50
|
+
futimesSync,
|
|
42
51
|
lstatSync,
|
|
52
|
+
openSync,
|
|
43
53
|
readFileSync,
|
|
54
|
+
readlinkSync,
|
|
44
55
|
realpathSync,
|
|
45
|
-
renameSync,
|
|
46
56
|
statSync,
|
|
47
57
|
utimesSync,
|
|
58
|
+
writeFileSync,
|
|
48
59
|
} from 'node:fs'
|
|
49
|
-
import {
|
|
60
|
+
import type { Stats } from 'node:fs'
|
|
61
|
+
import {
|
|
62
|
+
copyFile,
|
|
63
|
+
lstat,
|
|
64
|
+
mkdir,
|
|
65
|
+
readlink,
|
|
66
|
+
realpath,
|
|
67
|
+
rename,
|
|
68
|
+
rm,
|
|
69
|
+
rmdir,
|
|
70
|
+
stat,
|
|
71
|
+
symlink,
|
|
72
|
+
unlink,
|
|
73
|
+
writeFile,
|
|
74
|
+
} from 'node:fs/promises'
|
|
50
75
|
import path from 'node:path'
|
|
51
76
|
import { UserError } from '../util/index.js'
|
|
52
77
|
import { WORKSPACE_OUTPUT_PREFIX } from './layer.js'
|
|
@@ -60,6 +85,7 @@ import {
|
|
|
60
85
|
tarSize,
|
|
61
86
|
} from './tar-stream.js'
|
|
62
87
|
import { ON_THREAD_MAX } from './zstd.js'
|
|
88
|
+
import { readUmask, restoredMode } from './umask.js'
|
|
63
89
|
|
|
64
90
|
/** Archive entry name carrying the per-output mode/mtime sidecar. */
|
|
65
91
|
const META_ENTRY = '.vx-meta.json'
|
|
@@ -174,6 +200,18 @@ interface MetaFile {
|
|
|
174
200
|
*/
|
|
175
201
|
key?: string
|
|
176
202
|
files: Record<string, [mode: number, mtimeMs: number]>
|
|
203
|
+
/**
|
|
204
|
+
* The empty directories under the outputs, as entry names
|
|
205
|
+
* (`outputs/dist/empty`): a tar of regular files cannot hold them, and
|
|
206
|
+
* a hit that dropped them left another tree than the miss (v44).
|
|
207
|
+
*/
|
|
208
|
+
dirs?: string[]
|
|
209
|
+
/**
|
|
210
|
+
* The symlinks under the outputs, entry name → the link's target as
|
|
211
|
+
* written (relative, inside the workspace): a hit makes the link, as
|
|
212
|
+
* Turbo and Nx do, where it once packed a copy of the target (v45).
|
|
213
|
+
*/
|
|
214
|
+
links?: Record<string, string>
|
|
177
215
|
/**
|
|
178
216
|
* What the PRODUCING execution used (2026-09-12, additive: an artifact
|
|
179
217
|
* without it reads as before). Rides the artifact so a machine that
|
|
@@ -220,22 +258,12 @@ interface PackArgs {
|
|
|
220
258
|
key?: string
|
|
221
259
|
stdout: string
|
|
222
260
|
outputs: ReadonlyMap<string, string>
|
|
261
|
+
/** Entry names of the empty directories under the outputs. */
|
|
262
|
+
dirs?: readonly string[] | undefined
|
|
223
263
|
/** The producing execution's usage, when the runner reported it. */
|
|
224
264
|
exec?: ExecUsage | undefined
|
|
225
|
-
/**
|
|
226
|
-
|
|
227
|
-
* outputs outside the task's sandbox, so a link a sandboxed task planted
|
|
228
|
-
* to a file it could not read (another project's, the user's home) packed
|
|
229
|
-
* that file into the artifact and the remote (L-23). A link to another
|
|
230
|
-
* output of the same artifact is packed wherever it is.
|
|
231
|
-
*/
|
|
232
|
-
within?: string
|
|
233
|
-
/**
|
|
234
|
-
* For a sandboxed task: whether it could read this canonical path. A
|
|
235
|
-
* symlinked output inside `within` whose target it could not read is
|
|
236
|
-
* refused like one outside it.
|
|
237
|
-
*/
|
|
238
|
-
readable?: ((real: string) => boolean) | undefined
|
|
265
|
+
/** The workspace root an output symlink's target must stay inside; absent, every link is refused. */
|
|
266
|
+
root?: string
|
|
239
267
|
}
|
|
240
268
|
|
|
241
269
|
/** The largest second count ustar's 12-byte mtime field holds: 11 octal digits. */
|
|
@@ -255,8 +283,7 @@ export interface ArtifactPlan {
|
|
|
255
283
|
* mtime) and lay out the tar — stdout first, the outputs, the sidecar
|
|
256
284
|
* last — with its exact size, so the caller can choose the one-call pack
|
|
257
285
|
* for a small artifact and the streamed one for a large one before a
|
|
258
|
-
* byte is read. A symlinked output is stored as its
|
|
259
|
-
* when that target is inside `within`.
|
|
286
|
+
* byte is read. A symlinked output is stored as its link (`links`).
|
|
260
287
|
*/
|
|
261
288
|
export async function planArtifact(args: PackArgs): Promise<ArtifactPlan> {
|
|
262
289
|
const meta: MetaFile = { version: 1, files: {} }
|
|
@@ -264,29 +291,13 @@ export async function planArtifact(args: PackArgs): Promise<ArtifactPlan> {
|
|
|
264
291
|
// cost more than the stat. Many go to the pool together, as before.
|
|
265
292
|
const few = args.outputs.size <= ON_THREAD_STATS
|
|
266
293
|
const lstatOf = few ? async (p: string) => lstatSync(p) : lstat
|
|
267
|
-
const
|
|
268
|
-
const realpathOf = (p: string) => realOf(p, few)
|
|
269
|
-
let withinReal: Promise<string> | undefined
|
|
270
|
-
// Where this artifact's own non-link outputs really are: a link to one of
|
|
271
|
-
// them packs bytes the task wrote itself. A root-anchored output tree is
|
|
272
|
-
// outside every project, so `gen/latest -> v2.txt` there was refused and
|
|
273
|
-
// the task never cached (J's lead).
|
|
274
|
-
let ownReal: Promise<Set<string>> | undefined
|
|
275
|
-
const ownOutputs = (): Promise<Set<string>> =>
|
|
276
|
-
(ownReal ??= Promise.all(
|
|
277
|
-
[...args.outputs.values()].map(async (abs) =>
|
|
278
|
-
(await lstatOf(abs)).isSymbolicLink() ? null : realpathOf(abs),
|
|
279
|
-
),
|
|
280
|
-
).then((all) => new Set(all.filter((p): p is string => p !== null))))
|
|
294
|
+
const readlinkOf = few ? async (p: string) => readlinkSync(p) : readlink
|
|
281
295
|
if (args.key !== undefined) meta.key = args.key
|
|
296
|
+
if (args.dirs !== undefined && args.dirs.length > 0) meta.dirs = [...args.dirs]
|
|
282
297
|
const exec = usageOf(args.exec)
|
|
283
298
|
if (exec !== undefined) meta.exec = exec
|
|
284
299
|
const files = await Promise.all(
|
|
285
300
|
[...args.outputs].map(async ([name, abs]) => {
|
|
286
|
-
// `stat` follows a symlink, so a link to a file is packed as that
|
|
287
|
-
// file's bytes and comes back as a regular file. A link to a
|
|
288
|
-
// directory, or a dangling one, has no bytes to pack: refuse loudly
|
|
289
|
-
// rather than store an entry that restores to nothing.
|
|
290
301
|
// The path as the config spells it: a root-anchored output's entry
|
|
291
302
|
// name is vx's own `workspace-outputs/…` (M-66).
|
|
292
303
|
const shown = name.startsWith('outputs/')
|
|
@@ -304,45 +315,33 @@ export async function planArtifact(args: PackArgs): Promise<ArtifactPlan> {
|
|
|
304
315
|
`${shown} is under workspace-outputs/, a name vx's artifacts reserve for outputs.workspaceFiles — write the task's files to another directory, or take them back with a '!' entry`,
|
|
305
316
|
)
|
|
306
317
|
}
|
|
307
|
-
const
|
|
308
|
-
|
|
318
|
+
const st = await lstatOf(abs)
|
|
319
|
+
if (st.isSymbolicLink()) {
|
|
320
|
+
// Kept as the link, never read through: no byte of its target
|
|
321
|
+
// enters the artifact, so a link to a file the task could not
|
|
322
|
+
// read packs nothing of it (L-23).
|
|
323
|
+
const to = await readlinkOf(abs)
|
|
324
|
+
if (path.isAbsolute(to)) {
|
|
309
325
|
throw new UserError(
|
|
310
|
-
`${shown} is a
|
|
326
|
+
`${shown} is a symlink to an absolute path (${to}): vx restores a link as written, and an absolute one names this machine's checkout — make it relative, or take it back with a '!' entry`,
|
|
311
327
|
)
|
|
312
328
|
}
|
|
313
|
-
|
|
314
|
-
}
|
|
315
|
-
// lstat, so a regular file costs no second call; only a link is
|
|
316
|
-
// resolved, and its body is read from the path this check resolved.
|
|
317
|
-
let src = abs
|
|
318
|
-
let st = await lstatOf(abs)
|
|
319
|
-
if (st.isSymbolicLink()) {
|
|
320
|
-
src = await realpathOf(abs).catch(dangling)
|
|
321
|
-
if (args.within !== undefined) {
|
|
322
|
-
withinReal ??= realpathOf(args.within)
|
|
323
|
-
const root = await withinReal
|
|
324
|
-
if (!src.startsWith(root + path.sep) && !(await ownOutputs()).has(src)) {
|
|
325
|
-
throw new UserError(
|
|
326
|
-
`${shown} is a symlink to ${src}, outside the project: vx packs a symlinked output as its target's bytes, and a project's outputs come from its own directory — emit a file there, or take it back with a '!' entry`,
|
|
327
|
-
)
|
|
328
|
-
}
|
|
329
|
-
}
|
|
330
|
-
if (args.readable !== undefined && !args.readable(src) && !(await ownOutputs()).has(src)) {
|
|
329
|
+
if (args.root === undefined || !inside(path.resolve(path.dirname(abs), to), args.root)) {
|
|
331
330
|
throw new UserError(
|
|
332
|
-
`${shown} is a symlink to ${
|
|
331
|
+
`${shown} is a symlink to ${to}, outside the workspace: vx restores a link as written, and a project's outputs stay in its workspace — emit a file there, or take it back with a '!' entry`,
|
|
333
332
|
)
|
|
334
333
|
}
|
|
335
|
-
|
|
334
|
+
return { name, link: to }
|
|
336
335
|
}
|
|
337
336
|
if (!st.isFile()) {
|
|
338
337
|
throw new UserError(
|
|
339
|
-
`${shown} is not a regular file
|
|
338
|
+
`${shown} is not a regular file: vx stores regular files and symlinks only — emit a file there, or narrow cache.outputs.files to the files the task produces, or take it back with a '!' entry`,
|
|
340
339
|
)
|
|
341
340
|
}
|
|
342
341
|
const mode = st.mode & 0o777
|
|
343
342
|
return {
|
|
344
343
|
name,
|
|
345
|
-
abs
|
|
344
|
+
abs,
|
|
346
345
|
size: st.size,
|
|
347
346
|
mode,
|
|
348
347
|
mtimeMs: Math.floor(st.mtimeMs),
|
|
@@ -354,12 +353,19 @@ export async function planArtifact(args: PackArgs): Promise<ArtifactPlan> {
|
|
|
354
353
|
}),
|
|
355
354
|
)
|
|
356
355
|
// In the outputs' order, not as each stat settled: one tree, one artifact.
|
|
357
|
-
|
|
356
|
+
const regular: Array<Exclude<(typeof files)[number], { link: string }>> = []
|
|
357
|
+
for (const f of files) {
|
|
358
|
+
if ('link' in f) (meta.links ??= {})[f.name] = f.link
|
|
359
|
+
else {
|
|
360
|
+
meta.files[f.name] = [f.mode, f.mtimeMs]
|
|
361
|
+
regular.push(f)
|
|
362
|
+
}
|
|
363
|
+
}
|
|
358
364
|
const stdout = new TextEncoder().encode(args.stdout)
|
|
359
365
|
const metaBytes = new TextEncoder().encode(JSON.stringify(meta))
|
|
360
366
|
const inputs: TarInput[] = [
|
|
361
367
|
{ name: STDOUT_ENTRY, size: stdout.byteLength, body: stdout },
|
|
362
|
-
...
|
|
368
|
+
...regular.map((f) => ({
|
|
363
369
|
name: f.name,
|
|
364
370
|
size: f.size,
|
|
365
371
|
mode: f.mode,
|
|
@@ -468,6 +474,10 @@ function streamOf(gen: AsyncGenerator<Uint8Array>): ReadableStream<Uint8Array> {
|
|
|
468
474
|
/** What a scan of an artifact reads out of it. */
|
|
469
475
|
interface ArtifactScan {
|
|
470
476
|
entries: ArchiveEntry[]
|
|
477
|
+
/** The sidecar's empty directories, each name proven safe. */
|
|
478
|
+
dirs: string[]
|
|
479
|
+
/** The sidecar's symlinks, `[name, target]`, each name and target proven safe. */
|
|
480
|
+
links: Array<[name: string, target: string]>
|
|
471
481
|
stdout: string | null
|
|
472
482
|
exec: ExecUsage | undefined
|
|
473
483
|
key: string | undefined
|
|
@@ -477,6 +487,8 @@ interface ArtifactScan {
|
|
|
477
487
|
class ScanFold {
|
|
478
488
|
private readonly seen: Array<{ name: string; size: number; mtimeMs: number }> = []
|
|
479
489
|
private files: MetaFile['files'] = {}
|
|
490
|
+
private dirs: string[] = []
|
|
491
|
+
private links: Array<[string, string]> = []
|
|
480
492
|
private exec: ExecUsage | undefined
|
|
481
493
|
private key: string | undefined
|
|
482
494
|
stdout: string | null = null
|
|
@@ -484,6 +496,8 @@ class ScanFold {
|
|
|
484
496
|
sidecar(text: string): void {
|
|
485
497
|
const parsed = JSON.parse(text) as MetaFile
|
|
486
498
|
this.files = parsed.files ?? {}
|
|
499
|
+
this.dirs = dirsOf(parsed)
|
|
500
|
+
this.links = linksOf(parsed)
|
|
487
501
|
// The ingest side is the untrusted boundary: a foreign sidecar's
|
|
488
502
|
// usage is taken only when it is a plain non-negative number.
|
|
489
503
|
this.exec = usageOf(parsed.exec)
|
|
@@ -503,6 +517,8 @@ class ScanFold {
|
|
|
503
517
|
// sidecar is metadata, never the authority on whether bytes exist.
|
|
504
518
|
return { name: s.name, size: s.size, mode: m?.[0] ?? 0o644, mtimeMs: m?.[1] ?? s.mtimeMs }
|
|
505
519
|
}),
|
|
520
|
+
dirs: this.dirs,
|
|
521
|
+
links: this.links,
|
|
506
522
|
stdout: this.stdout,
|
|
507
523
|
exec: this.exec,
|
|
508
524
|
key: this.key,
|
|
@@ -568,6 +584,56 @@ export function scanTarBytes(tar: Uint8Array): ArtifactScan {
|
|
|
568
584
|
return fold.result()
|
|
569
585
|
}
|
|
570
586
|
|
|
587
|
+
/**
|
|
588
|
+
* The sidecar's directory names, each held to the rules a file entry's
|
|
589
|
+
* name is (the sidecar is as attacker-reachable as the tar), and only
|
|
590
|
+
* under an output namespace.
|
|
591
|
+
*/
|
|
592
|
+
function dirsOf(meta: MetaFile): string[] {
|
|
593
|
+
const raw: unknown = meta.dirs
|
|
594
|
+
if (raw === undefined) return []
|
|
595
|
+
if (!Array.isArray(raw)) throw new ArchiveSecurityError('archive sidecar dirs is not a list')
|
|
596
|
+
return raw.map((name: unknown) => {
|
|
597
|
+
if (
|
|
598
|
+
typeof name !== 'string' ||
|
|
599
|
+
!(name.startsWith('outputs/') || name.startsWith(WORKSPACE_PREFIX))
|
|
600
|
+
) {
|
|
601
|
+
throw new ArchiveSecurityError(`archive directory is not an output: ${String(name)}`)
|
|
602
|
+
}
|
|
603
|
+
assertSafeName(name)
|
|
604
|
+
return name
|
|
605
|
+
})
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
/**
|
|
609
|
+
* The sidecar's links, each name held to a file entry's rules and each
|
|
610
|
+
* target to a relative path with no NUL: where it may point is decided at
|
|
611
|
+
* the restore, which knows the workspace.
|
|
612
|
+
*/
|
|
613
|
+
function linksOf(meta: MetaFile): Array<[string, string]> {
|
|
614
|
+
const raw: unknown = meta.links
|
|
615
|
+
if (raw === undefined) return []
|
|
616
|
+
if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) {
|
|
617
|
+
throw new ArchiveSecurityError('archive sidecar links is not a map')
|
|
618
|
+
}
|
|
619
|
+
return Object.entries(raw as Record<string, unknown>).map(([name, to]) => {
|
|
620
|
+
if (!(name.startsWith('outputs/') || name.startsWith(WORKSPACE_PREFIX))) {
|
|
621
|
+
throw new ArchiveSecurityError(`archive link is not an output: ${name}`)
|
|
622
|
+
}
|
|
623
|
+
assertSafeName(name)
|
|
624
|
+
if (typeof to !== 'string' || to.length === 0 || to.includes('\0') || path.isAbsolute(to)) {
|
|
625
|
+
throw new ArchiveSecurityError(`archive link ${name} has an unsafe target: ${String(to)}`)
|
|
626
|
+
}
|
|
627
|
+
return [name, to]
|
|
628
|
+
})
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
/** `p` is `root` or under it, lexically. */
|
|
632
|
+
const inside = (p: string, root: string): boolean => {
|
|
633
|
+
const r = path.resolve(root)
|
|
634
|
+
return p === r || p.startsWith(r + path.sep)
|
|
635
|
+
}
|
|
636
|
+
|
|
571
637
|
/** The usage worth carrying: each axis a finite non-negative number, or absent; nothing → undefined. */
|
|
572
638
|
function usageOf(raw: unknown): ExecUsage | undefined {
|
|
573
639
|
if (typeof raw !== 'object' || raw === null) return undefined
|
|
@@ -595,15 +661,21 @@ export async function extractArtifactStream(
|
|
|
595
661
|
destDir: string,
|
|
596
662
|
workspaceDest: string | undefined,
|
|
597
663
|
verify?: (provided: ReadonlySet<string>) => void,
|
|
664
|
+
accept?: (name: string) => boolean,
|
|
665
|
+
/** The workspace root whose `.vx/tmp/` stages the entries; absent, each stages beside its target. */
|
|
666
|
+
stageRoot?: string,
|
|
598
667
|
): Promise<Set<string>> {
|
|
599
668
|
// No `mkdir -p` of the destination here: `stage` creates each entry's
|
|
600
669
|
// directory chain, the destination included, and the cache restores
|
|
601
670
|
// into a project directory that exists. The eager one cost every
|
|
602
671
|
// restore a thread-pool round trip for an EEXIST and a stat (item 627).
|
|
603
|
-
|
|
672
|
+
readUmask()
|
|
673
|
+
const x = new Extractor(destDir, workspaceDest, stageRoot)
|
|
604
674
|
const provided = new Set<string>()
|
|
605
675
|
const headerMtime = new Map<string, number>()
|
|
606
676
|
let meta: MetaFile['files'] = {}
|
|
677
|
+
let dirs: string[] = []
|
|
678
|
+
let links: Array<[string, string]> = []
|
|
607
679
|
try {
|
|
608
680
|
const check = new SumCheck()
|
|
609
681
|
for await (const e of tarEntries(tar)) {
|
|
@@ -611,10 +683,20 @@ export async function extractArtifactStream(
|
|
|
611
683
|
const body = await check.body(e.name, e.body)
|
|
612
684
|
if (body === null) continue
|
|
613
685
|
if (e.name === META_ENTRY) {
|
|
614
|
-
|
|
686
|
+
const parsed = JSON.parse(await textOf(body)) as MetaFile
|
|
687
|
+
meta = parsed.files ?? {}
|
|
688
|
+
dirs = dirsOf(parsed)
|
|
689
|
+
links = linksOf(parsed)
|
|
615
690
|
continue
|
|
616
691
|
}
|
|
617
692
|
assertSafeName(e.name)
|
|
693
|
+
// Before anything is staged: an entry the task never declared is not
|
|
694
|
+
// this task's save, and none of the archive lands.
|
|
695
|
+
if (accept !== undefined && !accept(e.name)) {
|
|
696
|
+
throw new ArchiveSecurityError(
|
|
697
|
+
`archive entry is not one of the task's declared outputs: ${e.name}`,
|
|
698
|
+
)
|
|
699
|
+
}
|
|
618
700
|
const dest = x.destFor(e.name)
|
|
619
701
|
if (dest === null || dest.rel.length === 0) {
|
|
620
702
|
await drain(body)
|
|
@@ -627,6 +709,38 @@ export async function extractArtifactStream(
|
|
|
627
709
|
await x.stage(e.name, dest.base, target, e.size <= SMALL_ENTRY ? await bytesOf(body) : body)
|
|
628
710
|
}
|
|
629
711
|
check.done()
|
|
712
|
+
for (const name of dirs) {
|
|
713
|
+
if (accept !== undefined && !accept(name)) {
|
|
714
|
+
throw new ArchiveSecurityError(
|
|
715
|
+
`archive directory is not one of the task's declared outputs: ${name}`,
|
|
716
|
+
)
|
|
717
|
+
}
|
|
718
|
+
const dest = x.destFor(name)
|
|
719
|
+
if (dest === null || dest.rel.length === 0) continue
|
|
720
|
+
const target = path.join(dest.base, dest.rel)
|
|
721
|
+
await x.assertContained(dest.base, target, name)
|
|
722
|
+
await x.makeDir(dest.base, target)
|
|
723
|
+
// The index rows name a directory with a trailing slash.
|
|
724
|
+
provided.add(`${name}/`)
|
|
725
|
+
}
|
|
726
|
+
// A link may point only inside the workspace (the project, without one).
|
|
727
|
+
const bound = stageRoot ?? workspaceDest ?? destDir
|
|
728
|
+
for (const [name, to] of links) {
|
|
729
|
+
if (accept !== undefined && !accept(name)) {
|
|
730
|
+
throw new ArchiveSecurityError(
|
|
731
|
+
`archive link is not one of the task's declared outputs: ${name}`,
|
|
732
|
+
)
|
|
733
|
+
}
|
|
734
|
+
const dest = x.destFor(name)
|
|
735
|
+
if (dest === null || dest.rel.length === 0) continue
|
|
736
|
+
const target = path.join(dest.base, dest.rel)
|
|
737
|
+
await x.assertContained(dest.base, target, name)
|
|
738
|
+
if (!inside(path.resolve(path.dirname(target), to), bound)) {
|
|
739
|
+
throw new ArchiveSecurityError(`archive link ${name} points out of the workspace: ${to}`)
|
|
740
|
+
}
|
|
741
|
+
await x.stageLink(name, dest.base, target, to)
|
|
742
|
+
provided.add(name)
|
|
743
|
+
}
|
|
630
744
|
verify?.(provided)
|
|
631
745
|
// The sidecar's stat is the truth, a mode of 000 and an mtime at or
|
|
632
746
|
// before the epoch included (`SOURCE_DATE_EPOCH=0`). Without one, a
|
|
@@ -644,6 +758,9 @@ export async function extractArtifactStream(
|
|
|
644
758
|
return provided
|
|
645
759
|
}
|
|
646
760
|
|
|
761
|
+
/** BSD gives every new file its directory's group, setgid or not. */
|
|
762
|
+
const INHERITS_GROUP = process.platform === 'darwin'
|
|
763
|
+
|
|
647
764
|
/**
|
|
648
765
|
* Entries up to this size are buffered and written without waiting, so a
|
|
649
766
|
* thousand tiny outputs restore in parallel; larger ones stream chunk by
|
|
@@ -674,8 +791,32 @@ async function bytesOf(body: AsyncIterable<Uint8Array>): Promise<Uint8Array> {
|
|
|
674
791
|
const textOf = async (body: AsyncIterable<Uint8Array>): Promise<string> =>
|
|
675
792
|
utf8.decode(await bytesOf(body))
|
|
676
793
|
|
|
794
|
+
/**
|
|
795
|
+
* Bring a target that already held the entry's bytes to its mode and
|
|
796
|
+
* mtime, through a descriptor opened without following a link: one
|
|
797
|
+
* planted since the compare is refused, never written through.
|
|
798
|
+
*/
|
|
799
|
+
function keep(s: Staged, ino: number, mode: number, mtimeMs: number | undefined): void {
|
|
800
|
+
const fd = openSync(s.target, constants.O_RDONLY | constants.O_NOFOLLOW)
|
|
801
|
+
try {
|
|
802
|
+
const st = fstatSync(fd)
|
|
803
|
+
if (st.ino !== ino || st.nlink !== 1) {
|
|
804
|
+
throw new Error(`${s.target} changed while it was restored`)
|
|
805
|
+
}
|
|
806
|
+
if ((st.mode & 0o7777) !== restoredMode(mode)) fchmodSync(fd, restoredMode(mode))
|
|
807
|
+
if (mtimeMs !== undefined && Math.floor(st.mtimeMs) !== mtimeMs) {
|
|
808
|
+
const t = new Date(mtimeMs)
|
|
809
|
+
futimesSync(fd, t, t)
|
|
810
|
+
}
|
|
811
|
+
} finally {
|
|
812
|
+
closeSync(fd)
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
|
|
677
816
|
/** One staged entry: written at `tmp`, renamed to `target` on commit. */
|
|
678
817
|
interface Staged {
|
|
818
|
+
/** `tmp` is in the staging directory, not beside `target`. */
|
|
819
|
+
staged: boolean
|
|
679
820
|
name: string
|
|
680
821
|
tmp: string
|
|
681
822
|
target: string
|
|
@@ -683,6 +824,24 @@ interface Staged {
|
|
|
683
824
|
created: string | undefined
|
|
684
825
|
/** Written by Bun's file writer, which creates with 0664 where `writeFile` uses 0666 (before the umask). */
|
|
685
826
|
streamed: boolean
|
|
827
|
+
/** The destination it is contained in. */
|
|
828
|
+
base: string
|
|
829
|
+
/** The target already holds the entry's bytes: nothing was written, and only its mode and mtime may change. */
|
|
830
|
+
same?: number
|
|
831
|
+
/** `tmp` was copied beside `target`, which is on another filesystem than the staging (`crossOver`). */
|
|
832
|
+
beside?: true
|
|
833
|
+
/** A symlink to this target, made at `tmp`: no mode, no mtime. */
|
|
834
|
+
link?: string
|
|
835
|
+
/** `tmp` is inside the staged copy of this missing directory, which commit renames whole. */
|
|
836
|
+
tree?: string
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
/** A directory missing in the tree, built in the staging directory and renamed in whole. */
|
|
840
|
+
interface Tree {
|
|
841
|
+
at: string
|
|
842
|
+
base: string
|
|
843
|
+
/** The empty directories under it the entry holds. */
|
|
844
|
+
dirs: string[]
|
|
686
845
|
}
|
|
687
846
|
|
|
688
847
|
/**
|
|
@@ -692,6 +851,8 @@ interface Staged {
|
|
|
692
851
|
*/
|
|
693
852
|
class Extractor {
|
|
694
853
|
private readonly staged: Staged[] = []
|
|
854
|
+
/** Empty output directories made, with the top of each chain created. */
|
|
855
|
+
private readonly madeDirs: Array<{ dir: string; created: string }> = []
|
|
695
856
|
private inflight: Promise<unknown>[] = []
|
|
696
857
|
private inflightBytes = 0
|
|
697
858
|
private readonly realBaseCache = new Map<string, string>()
|
|
@@ -705,10 +866,16 @@ class Extractor {
|
|
|
705
866
|
// entry's own check and its write.
|
|
706
867
|
private readonly containedDirs = new Set<string>()
|
|
707
868
|
private readonly ensuredDirs = new Set<string>()
|
|
869
|
+
private readonly trees = new Map<string, Tree>()
|
|
870
|
+
// Directories the containment walk found missing, which `treeFor` would
|
|
871
|
+
// otherwise probe again. A directory made since fails the tree's rename,
|
|
872
|
+
// and the entries then go in one by one.
|
|
873
|
+
private readonly seenAbsent = new Set<string>()
|
|
708
874
|
|
|
709
875
|
constructor(
|
|
710
876
|
private readonly destDir: string,
|
|
711
877
|
private readonly workspaceDest: string | undefined,
|
|
878
|
+
private readonly stageRoot: string | undefined,
|
|
712
879
|
) {}
|
|
713
880
|
|
|
714
881
|
/** Each namespace anchors at its own destination dir. */
|
|
@@ -797,6 +964,7 @@ class Extractor {
|
|
|
797
964
|
// trip and error (X-163). A dangling link is there and falls through,
|
|
798
965
|
// as does an lstat that fails otherwise: the realpath decides.
|
|
799
966
|
if (absent(probe)) {
|
|
967
|
+
this.seenAbsent.add(probe)
|
|
800
968
|
probe = path.dirname(probe)
|
|
801
969
|
continue
|
|
802
970
|
}
|
|
@@ -844,6 +1012,82 @@ class Extractor {
|
|
|
844
1012
|
return mkdir(real, { recursive: true })
|
|
845
1013
|
}
|
|
846
1014
|
|
|
1015
|
+
/**
|
|
1016
|
+
* Where an entry in `dir` is written when a directory on the way to it is
|
|
1017
|
+
* missing: inside a copy of the topmost missing one, built in the staging
|
|
1018
|
+
* directory and renamed into place whole on commit. One rename per
|
|
1019
|
+
* missing directory, not one per file: every file of a restore into a
|
|
1020
|
+
* cleaned tree renamed across directories, all out of one staging
|
|
1021
|
+
* directory, and the kernel time of a 5,000-artifact restore doubled
|
|
1022
|
+
* (2026-10-17). A reader sees the directory complete or not at all.
|
|
1023
|
+
* Undefined where the group or filesystem would differ from a write in
|
|
1024
|
+
* place: the entry is then staged and renamed alone.
|
|
1025
|
+
*/
|
|
1026
|
+
private async treeFor(base: string, dir: string): Promise<string | undefined> {
|
|
1027
|
+
if (this.stageRoot === undefined) return undefined
|
|
1028
|
+
const abs = path.resolve(dir)
|
|
1029
|
+
for (const [top, t] of this.trees) {
|
|
1030
|
+
if (abs === top || abs.startsWith(top + path.sep)) return t.at + abs.slice(top.length)
|
|
1031
|
+
}
|
|
1032
|
+
const baseResolved = path.resolve(base)
|
|
1033
|
+
let top: string | undefined
|
|
1034
|
+
let probe = abs
|
|
1035
|
+
while (
|
|
1036
|
+
probe.startsWith(baseResolved + path.sep) &&
|
|
1037
|
+
(this.seenAbsent.has(probe) || absent(probe))
|
|
1038
|
+
) {
|
|
1039
|
+
top = probe
|
|
1040
|
+
probe = path.dirname(probe)
|
|
1041
|
+
}
|
|
1042
|
+
if (top === undefined) return undefined
|
|
1043
|
+
const staging = await stagingDir(this.stageRoot)
|
|
1044
|
+
const st = statSync(staging)
|
|
1045
|
+
const parent = statSync(probe, { throwIfNoEntry: false })
|
|
1046
|
+
if (
|
|
1047
|
+
parent === undefined ||
|
|
1048
|
+
!parent.isDirectory() ||
|
|
1049
|
+
parent.dev !== st.dev ||
|
|
1050
|
+
((parent.mode | st.mode) & 0o2000) !== 0 ||
|
|
1051
|
+
(INHERITS_GROUP && parent.gid !== st.gid)
|
|
1052
|
+
) {
|
|
1053
|
+
return undefined
|
|
1054
|
+
}
|
|
1055
|
+
const at = path.join(staging, (tmpSeq++).toString(36))
|
|
1056
|
+
this.trees.set(top, { at, base, dirs: [] })
|
|
1057
|
+
return at + abs.slice(top.length)
|
|
1058
|
+
}
|
|
1059
|
+
|
|
1060
|
+
/** `dir` made, once per extraction. */
|
|
1061
|
+
private async ensure(dir: string): Promise<void> {
|
|
1062
|
+
if (this.ensuredDirs.has(dir)) return
|
|
1063
|
+
await mkdir(dir, { recursive: true })
|
|
1064
|
+
this.ensuredDirs.add(dir)
|
|
1065
|
+
}
|
|
1066
|
+
|
|
1067
|
+
/** The tree `target` is staged in. */
|
|
1068
|
+
private treeOf(target: string): string | undefined {
|
|
1069
|
+
for (const top of this.trees.keys()) if (target.startsWith(top + path.sep)) return top
|
|
1070
|
+
return undefined
|
|
1071
|
+
}
|
|
1072
|
+
|
|
1073
|
+
/** An empty output directory: made now, pruned by `abort` if it was not there. */
|
|
1074
|
+
async makeDir(base: string, dir: string): Promise<void> {
|
|
1075
|
+
const staged = await this.treeFor(base, dir)
|
|
1076
|
+
if (staged !== undefined) {
|
|
1077
|
+
await this.ensure(staged)
|
|
1078
|
+
this.trees.get(this.treeOf(dir + path.sep)!)!.dirs.push(dir)
|
|
1079
|
+
return
|
|
1080
|
+
}
|
|
1081
|
+
let created: string | undefined
|
|
1082
|
+
try {
|
|
1083
|
+
created = await mkdir(dir, { recursive: true })
|
|
1084
|
+
} catch (err) {
|
|
1085
|
+
created = await this.mkdirThroughLink(base, dir, err as NodeJS.ErrnoException)
|
|
1086
|
+
}
|
|
1087
|
+
this.ensuredDirs.add(dir)
|
|
1088
|
+
if (created !== undefined) this.madeDirs.push({ dir, created })
|
|
1089
|
+
}
|
|
1090
|
+
|
|
847
1091
|
/**
|
|
848
1092
|
* Write the entry beside its target. A buffer is written without
|
|
849
1093
|
* waiting, bounded by `INFLIGHT_BYTES`; a chunk stream goes to a file
|
|
@@ -856,6 +1100,25 @@ class Extractor {
|
|
|
856
1100
|
body: Uint8Array | AsyncIterable<Uint8Array>,
|
|
857
1101
|
): Promise<void> {
|
|
858
1102
|
const dir = path.dirname(target)
|
|
1103
|
+
const inTree = await this.treeFor(base, dir)
|
|
1104
|
+
if (inTree !== undefined) {
|
|
1105
|
+
await this.ensure(inTree)
|
|
1106
|
+
const tmp = path.join(inTree, path.basename(target))
|
|
1107
|
+
const streamed = !(body instanceof Uint8Array)
|
|
1108
|
+
const tree = this.treeOf(target)!
|
|
1109
|
+
this.staged.push({
|
|
1110
|
+
staged: true,
|
|
1111
|
+
name,
|
|
1112
|
+
tmp,
|
|
1113
|
+
target,
|
|
1114
|
+
created: undefined,
|
|
1115
|
+
streamed,
|
|
1116
|
+
base,
|
|
1117
|
+
tree,
|
|
1118
|
+
})
|
|
1119
|
+
await this.write(tmp, body)
|
|
1120
|
+
return
|
|
1121
|
+
}
|
|
859
1122
|
let created: string | undefined
|
|
860
1123
|
if (!this.ensuredDirs.has(dir)) {
|
|
861
1124
|
try {
|
|
@@ -865,7 +1128,33 @@ class Extractor {
|
|
|
865
1128
|
}
|
|
866
1129
|
this.ensuredDirs.add(dir)
|
|
867
1130
|
}
|
|
868
|
-
//
|
|
1131
|
+
// A target that already holds these bytes stays: renaming over a live
|
|
1132
|
+
// file costs ~0.4 ms on ext4 (auto_da_alloc), and a hit that changed
|
|
1133
|
+
// one of 2,000 outputs paid it 2,000 times. Bytes, not a stat stamp: a
|
|
1134
|
+
// reproducible build gives every output one mtime (item 886).
|
|
1135
|
+
if (body instanceof Uint8Array && created === undefined) {
|
|
1136
|
+
const st = lstatSync(target, { throwIfNoEntry: false })
|
|
1137
|
+
if (
|
|
1138
|
+
st !== undefined &&
|
|
1139
|
+
st.isFile() &&
|
|
1140
|
+
st.nlink === 1 &&
|
|
1141
|
+
st.size === body.byteLength &&
|
|
1142
|
+
readFileSync(target).equals(body)
|
|
1143
|
+
) {
|
|
1144
|
+
this.staged.push({
|
|
1145
|
+
staged: false,
|
|
1146
|
+
name,
|
|
1147
|
+
tmp: '',
|
|
1148
|
+
target,
|
|
1149
|
+
created,
|
|
1150
|
+
streamed: false,
|
|
1151
|
+
base,
|
|
1152
|
+
same: st.ino,
|
|
1153
|
+
})
|
|
1154
|
+
return
|
|
1155
|
+
}
|
|
1156
|
+
}
|
|
1157
|
+
// Write apart from the target and RENAME into place. rename(2) replaces
|
|
869
1158
|
// the destination's directory ENTRY without following it, which is
|
|
870
1159
|
// what makes this both link-safe and concurrency-safe:
|
|
871
1160
|
//
|
|
@@ -882,15 +1171,19 @@ class Extractor {
|
|
|
882
1171
|
//
|
|
883
1172
|
// A directory at the target makes the rename fail — the same
|
|
884
1173
|
// fail-closed outcome the plain write had.
|
|
885
|
-
//
|
|
886
|
-
//
|
|
887
|
-
//
|
|
888
|
-
const
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
1174
|
+
// Staged under `.vx/tmp/`, so no output folder ever holds a temp: a
|
|
1175
|
+
// reader or a key taken meanwhile saw `.vx-tmp-*` files beside the
|
|
1176
|
+
// outputs, and a killed restore left them there (M, 2026-10-16).
|
|
1177
|
+
const staged = this.stageRoot !== undefined
|
|
1178
|
+
const tmp = staged
|
|
1179
|
+
? path.join(await stagingDir(this.stageRoot!), (tmpSeq++).toString(36))
|
|
1180
|
+
: siblingTemp(target)
|
|
892
1181
|
const streamed = !(body instanceof Uint8Array)
|
|
893
|
-
this.staged.push({ name, tmp, target, created, streamed })
|
|
1182
|
+
this.staged.push({ staged, name, tmp, target, created, streamed, base })
|
|
1183
|
+
await this.write(tmp, body)
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
private async write(tmp: string, body: Uint8Array | AsyncIterable<Uint8Array>): Promise<void> {
|
|
894
1187
|
if (body instanceof Uint8Array) {
|
|
895
1188
|
this.inflightBytes += body.byteLength
|
|
896
1189
|
// `writeFile`, not `Bun.write`: for a buffer this size `Bun.write`
|
|
@@ -917,6 +1210,49 @@ class Extractor {
|
|
|
917
1210
|
}
|
|
918
1211
|
}
|
|
919
1212
|
|
|
1213
|
+
/** A symlink: made apart from the target like a file, renamed over it on commit. */
|
|
1214
|
+
async stageLink(name: string, base: string, target: string, to: string): Promise<void> {
|
|
1215
|
+
const dir = path.dirname(target)
|
|
1216
|
+
const inTree = await this.treeFor(base, dir)
|
|
1217
|
+
if (inTree !== undefined) {
|
|
1218
|
+
await this.ensure(inTree)
|
|
1219
|
+
const tmp = path.join(inTree, path.basename(target))
|
|
1220
|
+
const tree = this.treeOf(target)!
|
|
1221
|
+
this.staged.push({
|
|
1222
|
+
staged: true,
|
|
1223
|
+
name,
|
|
1224
|
+
tmp,
|
|
1225
|
+
target,
|
|
1226
|
+
created: undefined,
|
|
1227
|
+
streamed: false,
|
|
1228
|
+
base,
|
|
1229
|
+
link: to,
|
|
1230
|
+
tree,
|
|
1231
|
+
})
|
|
1232
|
+
await symlink(to, tmp)
|
|
1233
|
+
return
|
|
1234
|
+
}
|
|
1235
|
+
let created: string | undefined
|
|
1236
|
+
if (!this.ensuredDirs.has(dir)) {
|
|
1237
|
+
try {
|
|
1238
|
+
created = await mkdir(dir, { recursive: true })
|
|
1239
|
+
} catch (err) {
|
|
1240
|
+
created = await this.mkdirThroughLink(base, dir, err as NodeJS.ErrnoException)
|
|
1241
|
+
}
|
|
1242
|
+
this.ensuredDirs.add(dir)
|
|
1243
|
+
}
|
|
1244
|
+
if (created === undefined) {
|
|
1245
|
+
const st = lstatSync(target, { throwIfNoEntry: false })
|
|
1246
|
+
if (st?.isSymbolicLink() === true && readlinkSync(target) === to) return
|
|
1247
|
+
}
|
|
1248
|
+
const staged = this.stageRoot !== undefined
|
|
1249
|
+
const tmp = staged
|
|
1250
|
+
? path.join(await stagingDir(this.stageRoot!), (tmpSeq++).toString(36))
|
|
1251
|
+
: siblingTemp(target)
|
|
1252
|
+
this.staged.push({ staged, name, tmp, target, created, streamed: false, base, link: to })
|
|
1253
|
+
await symlink(to, tmp)
|
|
1254
|
+
}
|
|
1255
|
+
|
|
920
1256
|
private async drain(): Promise<void> {
|
|
921
1257
|
const pending = this.inflight
|
|
922
1258
|
this.inflight = []
|
|
@@ -926,12 +1262,11 @@ class Extractor {
|
|
|
926
1262
|
|
|
927
1263
|
/**
|
|
928
1264
|
* Apply each entry's mode and mtime, then rename everything into place.
|
|
929
|
-
*
|
|
930
|
-
*
|
|
931
|
-
*
|
|
932
|
-
*
|
|
933
|
-
*
|
|
934
|
-
* restores' round trips flowing. The chmod is skipped when the mode is
|
|
1265
|
+
* The metadata calls are synchronous: a few microseconds each on the
|
|
1266
|
+
* calling thread and ~40 µs each as thread-pool round trips, and a large
|
|
1267
|
+
* artifact has thousands of them (measured 2026-09-11 on payload's ui,
|
|
1268
|
+
* 4,069 files: 892 ms → see the archive bench). The renames go to the
|
|
1269
|
+
* pool, `RENAME_BATCH` at a time: each replaces a live file. The chmod is skipped when the mode is
|
|
935
1270
|
* the one the temp file was created with — the common case. The two
|
|
936
1271
|
* writers create with different modes (under umask 000: 0666 and 0664),
|
|
937
1272
|
* so each is stat'ed once.
|
|
@@ -940,38 +1275,256 @@ class Extractor {
|
|
|
940
1275
|
metaFor: (name: string) => [mode: number, mtimeMs: number | undefined],
|
|
941
1276
|
): Promise<void> {
|
|
942
1277
|
await this.drain()
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
createdModes.set(s.streamed, createdMode)
|
|
1278
|
+
// One stat per directory, shared by the filesystem check and the group.
|
|
1279
|
+
const dirStats = new Map<string, Stats>()
|
|
1280
|
+
const statDir = (dir: string): Stats => {
|
|
1281
|
+
let st = dirStats.get(dir)
|
|
1282
|
+
if (st === undefined) {
|
|
1283
|
+
st = statSync(dir)
|
|
1284
|
+
dirStats.set(dir, st)
|
|
951
1285
|
}
|
|
952
|
-
|
|
953
|
-
|
|
1286
|
+
return st
|
|
1287
|
+
}
|
|
1288
|
+
await this.crossOver(statDir)
|
|
1289
|
+
const created = new Map<boolean, { mode: number; uid: number; gid: number }>()
|
|
1290
|
+
// The group a file written in place would take, by directory: a setgid
|
|
1291
|
+
// directory's on Linux, every directory's on BSD/macOS. A file staged
|
|
1292
|
+
// elsewhere took the staging directory's.
|
|
1293
|
+
const setgid = new Map<string, number | null>()
|
|
1294
|
+
let n = 0
|
|
1295
|
+
let renames: Promise<void>[] = []
|
|
1296
|
+
const checked = new Set<string>()
|
|
1297
|
+
try {
|
|
1298
|
+
for (const s of this.staged) {
|
|
1299
|
+
if (s.tree === undefined)
|
|
1300
|
+
await this.assertStillContained(s.base, path.dirname(s.target), checked)
|
|
1301
|
+
const [mode, mtimeMs] = metaFor(s.name)
|
|
1302
|
+
if (s.same !== undefined) {
|
|
1303
|
+
keep(s, s.same, mode, mtimeMs)
|
|
1304
|
+
continue
|
|
1305
|
+
}
|
|
1306
|
+
if (s.link !== undefined) {
|
|
1307
|
+
if (s.tree !== undefined) continue
|
|
1308
|
+
renames.push(handled(this.place(s, mode, undefined)))
|
|
1309
|
+
if (++n % RENAME_BATCH === 0) {
|
|
1310
|
+
await settle(renames)
|
|
1311
|
+
renames = []
|
|
1312
|
+
}
|
|
1313
|
+
continue
|
|
1314
|
+
}
|
|
1315
|
+
if (s.beside === true) {
|
|
1316
|
+
chmodSync(s.tmp, restoredMode(mode))
|
|
1317
|
+
if (mtimeMs !== undefined) utimesSync(s.tmp, new Date(mtimeMs), new Date(mtimeMs))
|
|
1318
|
+
renames.push(handled(this.place(s, mode, undefined)))
|
|
1319
|
+
if (++n % RENAME_BATCH === 0) {
|
|
1320
|
+
await settle(renames)
|
|
1321
|
+
renames = []
|
|
1322
|
+
}
|
|
1323
|
+
continue
|
|
1324
|
+
}
|
|
1325
|
+
let made = created.get(s.streamed)
|
|
1326
|
+
if (made === undefined) {
|
|
1327
|
+
const st = statSync(s.tmp)
|
|
1328
|
+
made = { mode: st.mode & 0o777, uid: st.uid, gid: st.gid }
|
|
1329
|
+
created.set(s.streamed, made)
|
|
1330
|
+
}
|
|
1331
|
+
if (restoredMode(mode) !== made.mode) chmodSync(s.tmp, restoredMode(mode))
|
|
954
1332
|
// A Date, not seconds: Bun reads a negative number of seconds as
|
|
955
1333
|
// "now", and a Date before 1970 as itself (Bun 1.4.2).
|
|
956
|
-
const t = new Date(mtimeMs)
|
|
957
|
-
utimesSync(s.tmp, t, t)
|
|
1334
|
+
const t = mtimeMs === undefined ? undefined : new Date(mtimeMs)
|
|
1335
|
+
if (t !== undefined) utimesSync(s.tmp, t, t)
|
|
1336
|
+
if (s.tree !== undefined) continue
|
|
1337
|
+
if (s.staged) {
|
|
1338
|
+
const dir = path.dirname(s.target)
|
|
1339
|
+
let gid = setgid.get(dir)
|
|
1340
|
+
if (gid === undefined) {
|
|
1341
|
+
const st = statDir(dir)
|
|
1342
|
+
gid = INHERITS_GROUP || (st.mode & 0o2000) !== 0 ? st.gid : null
|
|
1343
|
+
setgid.set(dir, gid)
|
|
1344
|
+
}
|
|
1345
|
+
if (gid !== null && gid !== made.gid) chownSync(s.tmp, made.uid, gid)
|
|
1346
|
+
}
|
|
1347
|
+
renames.push(handled(this.place(s, mode, t)))
|
|
1348
|
+
if (++n % RENAME_BATCH === 0) {
|
|
1349
|
+
await settle(renames)
|
|
1350
|
+
renames = []
|
|
1351
|
+
}
|
|
958
1352
|
}
|
|
959
|
-
|
|
960
|
-
|
|
1353
|
+
} catch (err) {
|
|
1354
|
+
// Every rename in flight lands or fails before `abort` cleans up
|
|
1355
|
+
// (a fuzz run on macOS, 2026-10-17).
|
|
1356
|
+
await Promise.allSettled(renames)
|
|
1357
|
+
throw err
|
|
961
1358
|
}
|
|
1359
|
+
await settle(renames)
|
|
1360
|
+
for (const [top, t] of this.trees) await this.placeTree(top, t, checked)
|
|
1361
|
+
await this.assertLandedInside()
|
|
1362
|
+
this.trees.clear()
|
|
962
1363
|
this.staged.length = 0
|
|
963
1364
|
}
|
|
964
1365
|
|
|
965
1366
|
/**
|
|
966
|
-
*
|
|
967
|
-
*
|
|
1367
|
+
* Rename a staged tree to the directory it stands for. Something made
|
|
1368
|
+
* there since staging (another restore of the same outputs) takes the
|
|
1369
|
+
* tree's entries one by one instead, each renamed over its target.
|
|
1370
|
+
*/
|
|
1371
|
+
private async placeTree(top: string, t: Tree, checked: Set<string>): Promise<void> {
|
|
1372
|
+
await this.assertStillContained(t.base, path.dirname(top), checked)
|
|
1373
|
+
try {
|
|
1374
|
+
await rename(t.at, top)
|
|
1375
|
+
return
|
|
1376
|
+
} catch (err) {
|
|
1377
|
+
const code = (err as NodeJS.ErrnoException).code
|
|
1378
|
+
if (code !== 'EEXIST' && code !== 'ENOTEMPTY' && code !== 'ENOTDIR' && code !== 'EISDIR') {
|
|
1379
|
+
throw err
|
|
1380
|
+
}
|
|
1381
|
+
}
|
|
1382
|
+
for (const dir of t.dirs) await this.makeDirIn(t.base, dir)
|
|
1383
|
+
for (const s of this.staged) {
|
|
1384
|
+
if (s.tree !== top) continue
|
|
1385
|
+
const dir = path.dirname(s.target)
|
|
1386
|
+
await this.makeDirIn(t.base, dir)
|
|
1387
|
+
await this.assertStillContained(t.base, dir, checked)
|
|
1388
|
+
await rename(s.tmp, s.target)
|
|
1389
|
+
}
|
|
1390
|
+
await rm(t.at, { recursive: true, force: true })
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1393
|
+
private async makeDirIn(base: string, dir: string): Promise<void> {
|
|
1394
|
+
try {
|
|
1395
|
+
await mkdir(dir, { recursive: true })
|
|
1396
|
+
} catch (err) {
|
|
1397
|
+
await this.mkdirThroughLink(base, dir, err as NodeJS.ErrnoException)
|
|
1398
|
+
}
|
|
1399
|
+
}
|
|
1400
|
+
|
|
1401
|
+
/**
|
|
1402
|
+
* Copy every staged entry whose target is on another filesystem than
|
|
1403
|
+
* `.vx/tmp/` to a temp beside its target, before any rename: a rename
|
|
1404
|
+
* there would fail EXDEV, and copying at the rename, one file at a time,
|
|
1405
|
+
* left the files renamed before a failed copy (ENOSPC) new and the rest
|
|
1406
|
+
* old (Growth's parity run, 2026-10-16). A failed copy throws with
|
|
1407
|
+
* nothing renamed, and `abort` removes the temps made so far.
|
|
1408
|
+
*/
|
|
1409
|
+
private async crossOver(statDir: (dir: string) => Stats): Promise<void> {
|
|
1410
|
+
const dev = (dir: string): number => statDir(dir).dev
|
|
1411
|
+
const cross = this.staged.filter(
|
|
1412
|
+
(s) =>
|
|
1413
|
+
s.staged &&
|
|
1414
|
+
s.same === undefined &&
|
|
1415
|
+
s.tree === undefined &&
|
|
1416
|
+
dev(path.dirname(s.tmp)) !== dev(path.dirname(s.target)),
|
|
1417
|
+
)
|
|
1418
|
+
for (let i = 0; i < cross.length; i += RENAME_BATCH) {
|
|
1419
|
+
await settle(
|
|
1420
|
+
cross.slice(i, i + RENAME_BATCH).map(async (s) => {
|
|
1421
|
+
const side = siblingTemp(s.target)
|
|
1422
|
+
try {
|
|
1423
|
+
if (s.link !== undefined) await symlink(s.link, side)
|
|
1424
|
+
else await copyFile(s.tmp, side)
|
|
1425
|
+
} catch (err) {
|
|
1426
|
+
await unlink(side).catch(() => undefined)
|
|
1427
|
+
throw err
|
|
1428
|
+
}
|
|
1429
|
+
const staged = s.tmp
|
|
1430
|
+
s.tmp = side
|
|
1431
|
+
s.beside = true
|
|
1432
|
+
await unlink(staged).catch(() => undefined)
|
|
1433
|
+
}),
|
|
1434
|
+
)
|
|
1435
|
+
}
|
|
1436
|
+
}
|
|
1437
|
+
|
|
1438
|
+
/**
|
|
1439
|
+
* Rename `s` over its target. A target on another filesystem than
|
|
1440
|
+
* `.vx/tmp/` (EXDEV) is reached through a temp beside it, for that file
|
|
1441
|
+
* only: the rename that replaces the live file stays atomic.
|
|
1442
|
+
*/
|
|
1443
|
+
private async place(s: Staged, mode: number, t: Date | undefined): Promise<void> {
|
|
1444
|
+
try {
|
|
1445
|
+
await rename(s.tmp, s.target)
|
|
1446
|
+
} catch (err) {
|
|
1447
|
+
if (!s.staged || (err as NodeJS.ErrnoException).code !== 'EXDEV') throw err
|
|
1448
|
+
const side = siblingTemp(s.target)
|
|
1449
|
+
try {
|
|
1450
|
+
if (s.link !== undefined) await symlink(s.link, side)
|
|
1451
|
+
else {
|
|
1452
|
+
await copyFile(s.tmp, side)
|
|
1453
|
+
chmodSync(side, restoredMode(mode))
|
|
1454
|
+
if (t !== undefined) utimesSync(side, t, t)
|
|
1455
|
+
}
|
|
1456
|
+
await rename(side, s.target)
|
|
1457
|
+
} catch (e) {
|
|
1458
|
+
await unlink(side).catch(() => undefined)
|
|
1459
|
+
throw e
|
|
1460
|
+
}
|
|
1461
|
+
await unlink(s.tmp).catch(() => undefined)
|
|
1462
|
+
}
|
|
1463
|
+
}
|
|
1464
|
+
|
|
1465
|
+
/**
|
|
1466
|
+
* The staging check, again at the rename: a directory on the way swapped
|
|
1467
|
+
* for a link out of the base since then would carry the rename with it.
|
|
1468
|
+
* Bun has no `renameat` against a held directory fd, so this is by path,
|
|
1469
|
+
* once per directory per commit.
|
|
1470
|
+
*/
|
|
1471
|
+
private async assertStillContained(
|
|
1472
|
+
base: string,
|
|
1473
|
+
dir: string,
|
|
1474
|
+
checked: Set<string>,
|
|
1475
|
+
): Promise<void> {
|
|
1476
|
+
if (checked.has(dir)) return
|
|
1477
|
+
checked.add(dir)
|
|
1478
|
+
const realBase = await this.realBaseOf(base)
|
|
1479
|
+
const real = await realOf(dir, true)
|
|
1480
|
+
if (real !== realBase && !real.startsWith(realBase + path.sep)) {
|
|
1481
|
+
throw await linkOutError(path.resolve(base), dir, realBase)
|
|
1482
|
+
}
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1485
|
+
/**
|
|
1486
|
+
* The check before each rename, once more after all landed: the rename
|
|
1487
|
+
* runs on the thread pool, and a directory swapped for a link out in that
|
|
1488
|
+
* gap carried the entry with it (restore-link-swap-rename.test.ts). An
|
|
1489
|
+
* entry that landed outside is removed from there and the restore fails.
|
|
1490
|
+
*/
|
|
1491
|
+
private async assertLandedInside(): Promise<void> {
|
|
1492
|
+
const landed = new Map<string, { base: string; placed: string[] }>()
|
|
1493
|
+
const add = (base: string, dir: string, placed: string): void => {
|
|
1494
|
+
const at = landed.get(dir)
|
|
1495
|
+
if (at === undefined) landed.set(dir, { base, placed: [placed] })
|
|
1496
|
+
else at.placed.push(placed)
|
|
1497
|
+
}
|
|
1498
|
+
for (const s of this.staged) {
|
|
1499
|
+
if (s.same === undefined && s.tree === undefined)
|
|
1500
|
+
add(s.base, path.dirname(s.target), s.target)
|
|
1501
|
+
}
|
|
1502
|
+
for (const [top, t] of this.trees) add(t.base, path.dirname(top), top)
|
|
1503
|
+
for (const [dir, { base, placed }] of landed) {
|
|
1504
|
+
const realBase = await this.realBaseOf(base)
|
|
1505
|
+
const real = await realOf(dir, true).catch(() => null)
|
|
1506
|
+
if (real === null || real === realBase || real.startsWith(realBase + path.sep)) continue
|
|
1507
|
+
await Promise.allSettled(placed.map((p) => rm(p, { recursive: true, force: true })))
|
|
1508
|
+
throw await linkOutError(path.resolve(base), dir, realBase)
|
|
1509
|
+
}
|
|
1510
|
+
}
|
|
1511
|
+
|
|
1512
|
+
/**
|
|
1513
|
+
* Never leave a temp behind: beside a target, the declared output globs
|
|
1514
|
+
* would sweep it into the next artifact. Directories this extraction created
|
|
968
1515
|
* are removed bottom-up while empty; one a concurrent writer has since
|
|
969
1516
|
* filled is left alone.
|
|
970
1517
|
*/
|
|
971
1518
|
async abort(): Promise<void> {
|
|
972
1519
|
await Promise.allSettled(this.inflight)
|
|
973
1520
|
this.inflight = []
|
|
974
|
-
await Promise.all(
|
|
1521
|
+
await Promise.all(
|
|
1522
|
+
this.staged.map((s) =>
|
|
1523
|
+
s.same === undefined ? unlink(s.tmp).catch(() => undefined) : undefined,
|
|
1524
|
+
),
|
|
1525
|
+
)
|
|
1526
|
+
for (const t of this.trees.values()) await rm(t.at, { recursive: true, force: true })
|
|
1527
|
+
this.trees.clear()
|
|
975
1528
|
// Newest first: a later entry's chain can sit inside an earlier one's
|
|
976
1529
|
// (`dist/` then `dist/sub/`), never the reverse.
|
|
977
1530
|
for (const s of this.staged.toReversed()) {
|
|
@@ -993,16 +1546,91 @@ class Extractor {
|
|
|
993
1546
|
dir = path.dirname(dir)
|
|
994
1547
|
}
|
|
995
1548
|
}
|
|
1549
|
+
for (const { dir: made, created } of this.madeDirs.toReversed()) {
|
|
1550
|
+
const top = path.resolve(created)
|
|
1551
|
+
let dir = made.startsWith(top) ? made : await realOf(made).catch(() => made)
|
|
1552
|
+
while (
|
|
1553
|
+
dir.startsWith(top) &&
|
|
1554
|
+
(await rmdir(dir).then(
|
|
1555
|
+
() => true,
|
|
1556
|
+
() => false,
|
|
1557
|
+
))
|
|
1558
|
+
) {
|
|
1559
|
+
if (dir === top) break
|
|
1560
|
+
dir = path.dirname(dir)
|
|
1561
|
+
}
|
|
1562
|
+
}
|
|
1563
|
+
this.madeDirs.length = 0
|
|
996
1564
|
this.staged.length = 0
|
|
997
1565
|
}
|
|
998
1566
|
}
|
|
999
1567
|
|
|
1000
|
-
/**
|
|
1001
|
-
|
|
1568
|
+
/**
|
|
1569
|
+
* Renames in flight at once. A rename over a live file is not the few
|
|
1570
|
+
* microseconds of one into an empty directory: on ext4 replacing a file
|
|
1571
|
+
* starts writeback of the new one's data (`auto_da_alloc`), ~250 µs each,
|
|
1572
|
+
* and synchronous ones stalled the main thread. 300 one-file restores
|
|
1573
|
+
* took 25% longer than delete-then-restore (2026-10-16); on the thread
|
|
1574
|
+
* pool they tie it, and 2,000 files in one restore are faster.
|
|
1575
|
+
*/
|
|
1576
|
+
const RENAME_BATCH = 64
|
|
1577
|
+
|
|
1578
|
+
/**
|
|
1579
|
+
* `p`, its failure marked handled at once: `settle` reads it later, and a
|
|
1580
|
+
* rename that fails while the commit awaits something else is not an
|
|
1581
|
+
* unhandled rejection that fails the process.
|
|
1582
|
+
*/
|
|
1583
|
+
function handled(p: Promise<void>): Promise<void> {
|
|
1584
|
+
p.catch(() => undefined)
|
|
1585
|
+
return p
|
|
1586
|
+
}
|
|
1587
|
+
|
|
1588
|
+
/** Wait for every rename, then throw the first failure: none is left in flight for `abort`. */
|
|
1589
|
+
async function settle(renames: readonly Promise<void>[]): Promise<void> {
|
|
1590
|
+
for (const r of await Promise.allSettled(renames)) if (r.status === 'rejected') throw r.reason
|
|
1591
|
+
}
|
|
1002
1592
|
|
|
1003
1593
|
/** Per-process counter for extract temp names; uniqueness only. */
|
|
1004
1594
|
let tmpSeq = 0
|
|
1005
1595
|
|
|
1596
|
+
/** `.vx/<STAGING>/`: where restores stage, each process in its own `<pid>-0` directory. */
|
|
1597
|
+
export const STAGING = 'tmp'
|
|
1598
|
+
|
|
1599
|
+
const stagingMade = new Set<string>()
|
|
1600
|
+
|
|
1601
|
+
/**
|
|
1602
|
+
* This process's staging directory under `root`'s `.vx/tmp/`, made on its
|
|
1603
|
+
* first restore and again only if something removed it: one per restore
|
|
1604
|
+
* cost 300 restores 12% (2026-10-16). Every restore's renames empty it;
|
|
1605
|
+
* the next run's `sweepStaging` removes it once this process is gone.
|
|
1606
|
+
*/
|
|
1607
|
+
async function stagingDir(root: string): Promise<string> {
|
|
1608
|
+
const top = path.join(root, '.vx', STAGING)
|
|
1609
|
+
const own = path.join(top, `${process.pid.toString(36)}-0`)
|
|
1610
|
+
if (stagingMade.has(own) && existsSync(own)) return own
|
|
1611
|
+
await mkdir(own, { recursive: true })
|
|
1612
|
+
// Ignored by git whatever `.vx/.gitignore` an older init committed.
|
|
1613
|
+
try {
|
|
1614
|
+
writeFileSync(path.join(top, '.gitignore'), '*\n', { flag: 'wx' })
|
|
1615
|
+
} catch {
|
|
1616
|
+
// there already, or the restore fails on its first write anyway
|
|
1617
|
+
}
|
|
1618
|
+
stagingMade.add(own)
|
|
1619
|
+
return own
|
|
1620
|
+
}
|
|
1621
|
+
|
|
1622
|
+
/**
|
|
1623
|
+
* A temp beside `target`, with its own short name, never `<target>.suffix`:
|
|
1624
|
+
* a suffix pushed a legal name of 242–255 bytes past NAME_MAX, and the
|
|
1625
|
+
* restore of a valid artifact failed with ENAMETOOLONG (item 666).
|
|
1626
|
+
*/
|
|
1627
|
+
function siblingTemp(target: string): string {
|
|
1628
|
+
return path.join(
|
|
1629
|
+
path.dirname(target),
|
|
1630
|
+
`.vx-tmp-${process.pid.toString(36)}-${(tmpSeq++).toString(36)}`,
|
|
1631
|
+
)
|
|
1632
|
+
}
|
|
1633
|
+
|
|
1006
1634
|
const WORKSPACE_PREFIX = 'workspace-outputs/'
|
|
1007
1635
|
|
|
1008
1636
|
/**
|