@vzn/vx 0.0.641 → 0.0.643
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -12
- package/skills/vx/SKILL.md +4 -4
- package/src/cache/archive.ts +525 -104
- package/src/cache/cache.ts +85 -11
- package/src/cache/fold-memo.ts +58 -0
- package/src/cache/git-inputs.ts +213 -31
- package/src/cache/index.ts +12 -3
- package/src/cache/inputs.ts +169 -79
- package/src/cache/key-fold.ts +90 -69
- package/src/cache/layer.ts +4 -6
- package/src/cache/layered-cache.ts +96 -10
- package/src/cache/output-index.ts +25 -4
- package/src/cache/plugin-cache.ts +23 -16
- package/src/cache/remote-lane.ts +25 -8
- package/src/cache/run-store.ts +4 -1
- package/src/cache/schema.ts +13 -3
- package/src/cache/scoped-cache.ts +74 -237
- package/src/cache/umask.ts +29 -0
- package/src/cli/docs.ts +71 -7
- package/src/cli/help.ts +6 -8
- package/src/cli/plan-format.ts +4 -1
- package/src/cli/plugin-commands.ts +11 -6
- package/src/cli/run.ts +14 -15
- package/src/cli/select.ts +9 -9
- package/src/cli/workspace-config.ts +8 -2
- package/src/config.ts +7 -16
- package/src/exec/runner.ts +5 -0
- package/src/exec/sandbox-runtime.ts +81 -1
- package/src/graph/priorities.ts +14 -1
- package/src/history/history.ts +43 -33
- package/src/index.ts +14 -0
- package/src/orchestrator/affected-tasks.ts +23 -2
- package/src/orchestrator/deferred-outputs.ts +3 -2
- package/src/orchestrator/execute-task.ts +20 -14
- package/src/orchestrator/hit-restore.ts +10 -2
- package/src/orchestrator/index.ts +5 -0
- package/src/orchestrator/lockfile-claim.ts +54 -8
- package/src/orchestrator/miss-save.ts +27 -17
- package/src/orchestrator/options.ts +9 -1
- package/src/orchestrator/output-host.ts +9 -1
- package/src/orchestrator/plan.ts +4 -0
- package/src/orchestrator/plugin-host.ts +38 -4
- package/src/orchestrator/plugin.ts +49 -5
- package/src/orchestrator/prepare.ts +141 -22
- package/src/orchestrator/projects.ts +25 -6
- package/src/orchestrator/run-context.ts +80 -0
- package/src/orchestrator/run.ts +38 -15
- package/src/orchestrator/sandbox-request.ts +1 -48
- package/src/orchestrator/save-lane.ts +17 -10
- package/src/orchestrator/signals.ts +7 -10
- package/src/orchestrator/stable-keys.ts +83 -25
- package/src/orchestrator/task-hash.ts +52 -12
- package/src/report/events.ts +12 -0
- package/src/report/framed-output.ts +2 -1
- package/src/report/logger.ts +53 -13
- package/src/report/status-line.ts +45 -4
- package/src/report/summary.ts +8 -2
- package/src/settings.ts +11 -11
- package/src/util/errors.ts +1 -1
- package/src/util/hidden-prompt.ts +74 -0
- package/src/util/index.ts +12 -2
- package/src/util/paths.ts +33 -14
- package/src/util/secrets.ts +150 -0
- package/src/workspace/affected.ts +182 -75
- package/src/workspace/config-schema.ts +81 -19
- package/src/workspace/index.ts +3 -0
- package/src/workspace/project-loader.ts +1 -0
- package/src/workspace/workspace.ts +17 -11
- package/docs/caching.md +0 -2376
- package/docs/cli.md +0 -3285
- package/docs/execution.md +0 -676
- package/docs/features.md +0 -178
- package/docs/patterns.md +0 -219
- package/docs/schema.md +0 -2468
- package/docs/security.md +0 -253
- package/src/cli/docs-corpus.ts +0 -21
- package/src/cli/docs-text.d.ts +0 -5
package/src/cache/archive.ts
CHANGED
|
@@ -34,10 +34,10 @@
|
|
|
34
34
|
// cache artifact is attacker-reachable (a poisoned remote, a tampered
|
|
35
35
|
// local entry) and "zip slip" is the class.
|
|
36
36
|
//
|
|
37
|
-
//
|
|
37
|
+
// Tar entries that are not regular files (symlinks, hardlinks, devices,
|
|
38
38
|
// FIFOs) are never materialised — the reader reports them only to be
|
|
39
|
-
// skipped
|
|
40
|
-
//
|
|
39
|
+
// skipped. An output symlink rides the sidecar instead (`links`), its
|
|
40
|
+
// target held to the workspace on both sides.
|
|
41
41
|
|
|
42
42
|
import {
|
|
43
43
|
chmodSync,
|
|
@@ -51,11 +51,13 @@ import {
|
|
|
51
51
|
lstatSync,
|
|
52
52
|
openSync,
|
|
53
53
|
readFileSync,
|
|
54
|
+
readlinkSync,
|
|
54
55
|
realpathSync,
|
|
55
56
|
statSync,
|
|
56
57
|
utimesSync,
|
|
57
58
|
writeFileSync,
|
|
58
59
|
} from 'node:fs'
|
|
60
|
+
import type { Stats } from 'node:fs'
|
|
59
61
|
import {
|
|
60
62
|
copyFile,
|
|
61
63
|
lstat,
|
|
@@ -63,8 +65,10 @@ import {
|
|
|
63
65
|
readlink,
|
|
64
66
|
realpath,
|
|
65
67
|
rename,
|
|
68
|
+
rm,
|
|
66
69
|
rmdir,
|
|
67
70
|
stat,
|
|
71
|
+
symlink,
|
|
68
72
|
unlink,
|
|
69
73
|
writeFile,
|
|
70
74
|
} from 'node:fs/promises'
|
|
@@ -81,6 +85,7 @@ import {
|
|
|
81
85
|
tarSize,
|
|
82
86
|
} from './tar-stream.js'
|
|
83
87
|
import { ON_THREAD_MAX } from './zstd.js'
|
|
88
|
+
import { readUmask, restoredMode } from './umask.js'
|
|
84
89
|
|
|
85
90
|
/** Archive entry name carrying the per-output mode/mtime sidecar. */
|
|
86
91
|
const META_ENTRY = '.vx-meta.json'
|
|
@@ -195,6 +200,18 @@ interface MetaFile {
|
|
|
195
200
|
*/
|
|
196
201
|
key?: string
|
|
197
202
|
files: Record<string, [mode: number, mtimeMs: number]>
|
|
203
|
+
/**
|
|
204
|
+
* The empty directories under the outputs, as entry names
|
|
205
|
+
* (`outputs/dist/empty`): a tar of regular files cannot hold them, and
|
|
206
|
+
* a hit that dropped them left another tree than the miss (v44).
|
|
207
|
+
*/
|
|
208
|
+
dirs?: string[]
|
|
209
|
+
/**
|
|
210
|
+
* The symlinks under the outputs, entry name → the link's target as
|
|
211
|
+
* written (relative, inside the workspace): a hit makes the link, as
|
|
212
|
+
* Turbo and Nx do, where it once packed a copy of the target (v45).
|
|
213
|
+
*/
|
|
214
|
+
links?: Record<string, string>
|
|
198
215
|
/**
|
|
199
216
|
* What the PRODUCING execution used (2026-09-12, additive: an artifact
|
|
200
217
|
* without it reads as before). Rides the artifact so a machine that
|
|
@@ -241,22 +258,12 @@ interface PackArgs {
|
|
|
241
258
|
key?: string
|
|
242
259
|
stdout: string
|
|
243
260
|
outputs: ReadonlyMap<string, string>
|
|
261
|
+
/** Entry names of the empty directories under the outputs. */
|
|
262
|
+
dirs?: readonly string[] | undefined
|
|
244
263
|
/** The producing execution's usage, when the runner reported it. */
|
|
245
264
|
exec?: ExecUsage | undefined
|
|
246
|
-
/**
|
|
247
|
-
|
|
248
|
-
* outputs outside the task's sandbox, so a link a sandboxed task planted
|
|
249
|
-
* to a file it could not read (another project's, the user's home) packed
|
|
250
|
-
* that file into the artifact and the remote (L-23). A link to another
|
|
251
|
-
* output of the same artifact is packed wherever it is.
|
|
252
|
-
*/
|
|
253
|
-
within?: string
|
|
254
|
-
/**
|
|
255
|
-
* For a sandboxed task: whether it could read this canonical path. A
|
|
256
|
-
* symlinked output inside `within` whose target it could not read is
|
|
257
|
-
* refused like one outside it.
|
|
258
|
-
*/
|
|
259
|
-
readable?: ((real: string) => boolean) | undefined
|
|
265
|
+
/** The workspace root an output symlink's target must stay inside; absent, every link is refused. */
|
|
266
|
+
root?: string
|
|
260
267
|
}
|
|
261
268
|
|
|
262
269
|
/** The largest second count ustar's 12-byte mtime field holds: 11 octal digits. */
|
|
@@ -276,8 +283,7 @@ export interface ArtifactPlan {
|
|
|
276
283
|
* mtime) and lay out the tar — stdout first, the outputs, the sidecar
|
|
277
284
|
* last — with its exact size, so the caller can choose the one-call pack
|
|
278
285
|
* for a small artifact and the streamed one for a large one before a
|
|
279
|
-
* byte is read. A symlinked output is stored as its
|
|
280
|
-
* when that target is inside `within`.
|
|
286
|
+
* byte is read. A symlinked output is stored as its link (`links`).
|
|
281
287
|
*/
|
|
282
288
|
export async function planArtifact(args: PackArgs): Promise<ArtifactPlan> {
|
|
283
289
|
const meta: MetaFile = { version: 1, files: {} }
|
|
@@ -285,29 +291,13 @@ export async function planArtifact(args: PackArgs): Promise<ArtifactPlan> {
|
|
|
285
291
|
// cost more than the stat. Many go to the pool together, as before.
|
|
286
292
|
const few = args.outputs.size <= ON_THREAD_STATS
|
|
287
293
|
const lstatOf = few ? async (p: string) => lstatSync(p) : lstat
|
|
288
|
-
const
|
|
289
|
-
const realpathOf = (p: string) => realOf(p, few)
|
|
290
|
-
let withinReal: Promise<string> | undefined
|
|
291
|
-
// Where this artifact's own non-link outputs really are: a link to one of
|
|
292
|
-
// them packs bytes the task wrote itself. A root-anchored output tree is
|
|
293
|
-
// outside every project, so `gen/latest -> v2.txt` there was refused and
|
|
294
|
-
// the task never cached (J's lead).
|
|
295
|
-
let ownReal: Promise<Set<string>> | undefined
|
|
296
|
-
const ownOutputs = (): Promise<Set<string>> =>
|
|
297
|
-
(ownReal ??= Promise.all(
|
|
298
|
-
[...args.outputs.values()].map(async (abs) =>
|
|
299
|
-
(await lstatOf(abs)).isSymbolicLink() ? null : realpathOf(abs),
|
|
300
|
-
),
|
|
301
|
-
).then((all) => new Set(all.filter((p): p is string => p !== null))))
|
|
294
|
+
const readlinkOf = few ? async (p: string) => readlinkSync(p) : readlink
|
|
302
295
|
if (args.key !== undefined) meta.key = args.key
|
|
296
|
+
if (args.dirs !== undefined && args.dirs.length > 0) meta.dirs = [...args.dirs]
|
|
303
297
|
const exec = usageOf(args.exec)
|
|
304
298
|
if (exec !== undefined) meta.exec = exec
|
|
305
299
|
const files = await Promise.all(
|
|
306
300
|
[...args.outputs].map(async ([name, abs]) => {
|
|
307
|
-
// `stat` follows a symlink, so a link to a file is packed as that
|
|
308
|
-
// file's bytes and comes back as a regular file. A link to a
|
|
309
|
-
// directory, or a dangling one, has no bytes to pack: refuse loudly
|
|
310
|
-
// rather than store an entry that restores to nothing.
|
|
311
301
|
// The path as the config spells it: a root-anchored output's entry
|
|
312
302
|
// name is vx's own `workspace-outputs/…` (M-66).
|
|
313
303
|
const shown = name.startsWith('outputs/')
|
|
@@ -325,45 +315,33 @@ export async function planArtifact(args: PackArgs): Promise<ArtifactPlan> {
|
|
|
325
315
|
`${shown} is under workspace-outputs/, a name vx's artifacts reserve for outputs.workspaceFiles — write the task's files to another directory, or take them back with a '!' entry`,
|
|
326
316
|
)
|
|
327
317
|
}
|
|
328
|
-
const
|
|
329
|
-
|
|
318
|
+
const st = await lstatOf(abs)
|
|
319
|
+
if (st.isSymbolicLink()) {
|
|
320
|
+
// Kept as the link, never read through: no byte of its target
|
|
321
|
+
// enters the artifact, so a link to a file the task could not
|
|
322
|
+
// read packs nothing of it (L-23).
|
|
323
|
+
const to = await readlinkOf(abs)
|
|
324
|
+
if (path.isAbsolute(to)) {
|
|
330
325
|
throw new UserError(
|
|
331
|
-
`${shown} is a
|
|
326
|
+
`${shown} is a symlink to an absolute path (${to}): vx restores a link as written, and an absolute one names this machine's checkout — make it relative, or take it back with a '!' entry`,
|
|
332
327
|
)
|
|
333
328
|
}
|
|
334
|
-
|
|
335
|
-
}
|
|
336
|
-
// lstat, so a regular file costs no second call; only a link is
|
|
337
|
-
// resolved, and its body is read from the path this check resolved.
|
|
338
|
-
let src = abs
|
|
339
|
-
let st = await lstatOf(abs)
|
|
340
|
-
if (st.isSymbolicLink()) {
|
|
341
|
-
src = await realpathOf(abs).catch(dangling)
|
|
342
|
-
if (args.within !== undefined) {
|
|
343
|
-
withinReal ??= realpathOf(args.within)
|
|
344
|
-
const root = await withinReal
|
|
345
|
-
if (!src.startsWith(root + path.sep) && !(await ownOutputs()).has(src)) {
|
|
346
|
-
throw new UserError(
|
|
347
|
-
`${shown} is a symlink to ${src}, outside the project: vx packs a symlinked output as its target's bytes, and a project's outputs come from its own directory — emit a file there, or take it back with a '!' entry`,
|
|
348
|
-
)
|
|
349
|
-
}
|
|
350
|
-
}
|
|
351
|
-
if (args.readable !== undefined && !args.readable(src) && !(await ownOutputs()).has(src)) {
|
|
329
|
+
if (args.root === undefined || !inside(path.resolve(path.dirname(abs), to), args.root)) {
|
|
352
330
|
throw new UserError(
|
|
353
|
-
`${shown} is a symlink to ${
|
|
331
|
+
`${shown} is a symlink to ${to}, outside the workspace: vx restores a link as written, and a project's outputs stay in its workspace — emit a file there, or take it back with a '!' entry`,
|
|
354
332
|
)
|
|
355
333
|
}
|
|
356
|
-
|
|
334
|
+
return { name, link: to }
|
|
357
335
|
}
|
|
358
336
|
if (!st.isFile()) {
|
|
359
337
|
throw new UserError(
|
|
360
|
-
`${shown} is not a regular file
|
|
338
|
+
`${shown} is not a regular file: vx stores regular files and symlinks only — emit a file there, or narrow cache.outputs.files to the files the task produces, or take it back with a '!' entry`,
|
|
361
339
|
)
|
|
362
340
|
}
|
|
363
341
|
const mode = st.mode & 0o777
|
|
364
342
|
return {
|
|
365
343
|
name,
|
|
366
|
-
abs
|
|
344
|
+
abs,
|
|
367
345
|
size: st.size,
|
|
368
346
|
mode,
|
|
369
347
|
mtimeMs: Math.floor(st.mtimeMs),
|
|
@@ -375,12 +353,19 @@ export async function planArtifact(args: PackArgs): Promise<ArtifactPlan> {
|
|
|
375
353
|
}),
|
|
376
354
|
)
|
|
377
355
|
// In the outputs' order, not as each stat settled: one tree, one artifact.
|
|
378
|
-
|
|
356
|
+
const regular: Array<Exclude<(typeof files)[number], { link: string }>> = []
|
|
357
|
+
for (const f of files) {
|
|
358
|
+
if ('link' in f) (meta.links ??= {})[f.name] = f.link
|
|
359
|
+
else {
|
|
360
|
+
meta.files[f.name] = [f.mode, f.mtimeMs]
|
|
361
|
+
regular.push(f)
|
|
362
|
+
}
|
|
363
|
+
}
|
|
379
364
|
const stdout = new TextEncoder().encode(args.stdout)
|
|
380
365
|
const metaBytes = new TextEncoder().encode(JSON.stringify(meta))
|
|
381
366
|
const inputs: TarInput[] = [
|
|
382
367
|
{ name: STDOUT_ENTRY, size: stdout.byteLength, body: stdout },
|
|
383
|
-
...
|
|
368
|
+
...regular.map((f) => ({
|
|
384
369
|
name: f.name,
|
|
385
370
|
size: f.size,
|
|
386
371
|
mode: f.mode,
|
|
@@ -489,6 +474,10 @@ function streamOf(gen: AsyncGenerator<Uint8Array>): ReadableStream<Uint8Array> {
|
|
|
489
474
|
/** What a scan of an artifact reads out of it. */
|
|
490
475
|
interface ArtifactScan {
|
|
491
476
|
entries: ArchiveEntry[]
|
|
477
|
+
/** The sidecar's empty directories, each name proven safe. */
|
|
478
|
+
dirs: string[]
|
|
479
|
+
/** The sidecar's symlinks, `[name, target]`, each name and target proven safe. */
|
|
480
|
+
links: Array<[name: string, target: string]>
|
|
492
481
|
stdout: string | null
|
|
493
482
|
exec: ExecUsage | undefined
|
|
494
483
|
key: string | undefined
|
|
@@ -498,6 +487,8 @@ interface ArtifactScan {
|
|
|
498
487
|
class ScanFold {
|
|
499
488
|
private readonly seen: Array<{ name: string; size: number; mtimeMs: number }> = []
|
|
500
489
|
private files: MetaFile['files'] = {}
|
|
490
|
+
private dirs: string[] = []
|
|
491
|
+
private links: Array<[string, string]> = []
|
|
501
492
|
private exec: ExecUsage | undefined
|
|
502
493
|
private key: string | undefined
|
|
503
494
|
stdout: string | null = null
|
|
@@ -505,6 +496,8 @@ class ScanFold {
|
|
|
505
496
|
sidecar(text: string): void {
|
|
506
497
|
const parsed = JSON.parse(text) as MetaFile
|
|
507
498
|
this.files = parsed.files ?? {}
|
|
499
|
+
this.dirs = dirsOf(parsed)
|
|
500
|
+
this.links = linksOf(parsed)
|
|
508
501
|
// The ingest side is the untrusted boundary: a foreign sidecar's
|
|
509
502
|
// usage is taken only when it is a plain non-negative number.
|
|
510
503
|
this.exec = usageOf(parsed.exec)
|
|
@@ -524,6 +517,8 @@ class ScanFold {
|
|
|
524
517
|
// sidecar is metadata, never the authority on whether bytes exist.
|
|
525
518
|
return { name: s.name, size: s.size, mode: m?.[0] ?? 0o644, mtimeMs: m?.[1] ?? s.mtimeMs }
|
|
526
519
|
}),
|
|
520
|
+
dirs: this.dirs,
|
|
521
|
+
links: this.links,
|
|
527
522
|
stdout: this.stdout,
|
|
528
523
|
exec: this.exec,
|
|
529
524
|
key: this.key,
|
|
@@ -589,6 +584,56 @@ export function scanTarBytes(tar: Uint8Array): ArtifactScan {
|
|
|
589
584
|
return fold.result()
|
|
590
585
|
}
|
|
591
586
|
|
|
587
|
+
/**
|
|
588
|
+
* The sidecar's directory names, each held to the rules a file entry's
|
|
589
|
+
* name is (the sidecar is as attacker-reachable as the tar), and only
|
|
590
|
+
* under an output namespace.
|
|
591
|
+
*/
|
|
592
|
+
function dirsOf(meta: MetaFile): string[] {
|
|
593
|
+
const raw: unknown = meta.dirs
|
|
594
|
+
if (raw === undefined) return []
|
|
595
|
+
if (!Array.isArray(raw)) throw new ArchiveSecurityError('archive sidecar dirs is not a list')
|
|
596
|
+
return raw.map((name: unknown) => {
|
|
597
|
+
if (
|
|
598
|
+
typeof name !== 'string' ||
|
|
599
|
+
!(name.startsWith('outputs/') || name.startsWith(WORKSPACE_PREFIX))
|
|
600
|
+
) {
|
|
601
|
+
throw new ArchiveSecurityError(`archive directory is not an output: ${String(name)}`)
|
|
602
|
+
}
|
|
603
|
+
assertSafeName(name)
|
|
604
|
+
return name
|
|
605
|
+
})
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
/**
|
|
609
|
+
* The sidecar's links, each name held to a file entry's rules and each
|
|
610
|
+
* target to a relative path with no NUL: where it may point is decided at
|
|
611
|
+
* the restore, which knows the workspace.
|
|
612
|
+
*/
|
|
613
|
+
function linksOf(meta: MetaFile): Array<[string, string]> {
|
|
614
|
+
const raw: unknown = meta.links
|
|
615
|
+
if (raw === undefined) return []
|
|
616
|
+
if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) {
|
|
617
|
+
throw new ArchiveSecurityError('archive sidecar links is not a map')
|
|
618
|
+
}
|
|
619
|
+
return Object.entries(raw as Record<string, unknown>).map(([name, to]) => {
|
|
620
|
+
if (!(name.startsWith('outputs/') || name.startsWith(WORKSPACE_PREFIX))) {
|
|
621
|
+
throw new ArchiveSecurityError(`archive link is not an output: ${name}`)
|
|
622
|
+
}
|
|
623
|
+
assertSafeName(name)
|
|
624
|
+
if (typeof to !== 'string' || to.length === 0 || to.includes('\0') || path.isAbsolute(to)) {
|
|
625
|
+
throw new ArchiveSecurityError(`archive link ${name} has an unsafe target: ${String(to)}`)
|
|
626
|
+
}
|
|
627
|
+
return [name, to]
|
|
628
|
+
})
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
/** `p` is `root` or under it, lexically. */
|
|
632
|
+
const inside = (p: string, root: string): boolean => {
|
|
633
|
+
const r = path.resolve(root)
|
|
634
|
+
return p === r || p.startsWith(r + path.sep)
|
|
635
|
+
}
|
|
636
|
+
|
|
592
637
|
/** The usage worth carrying: each axis a finite non-negative number, or absent; nothing → undefined. */
|
|
593
638
|
function usageOf(raw: unknown): ExecUsage | undefined {
|
|
594
639
|
if (typeof raw !== 'object' || raw === null) return undefined
|
|
@@ -624,10 +669,13 @@ export async function extractArtifactStream(
|
|
|
624
669
|
// directory chain, the destination included, and the cache restores
|
|
625
670
|
// into a project directory that exists. The eager one cost every
|
|
626
671
|
// restore a thread-pool round trip for an EEXIST and a stat (item 627).
|
|
672
|
+
readUmask()
|
|
627
673
|
const x = new Extractor(destDir, workspaceDest, stageRoot)
|
|
628
674
|
const provided = new Set<string>()
|
|
629
675
|
const headerMtime = new Map<string, number>()
|
|
630
676
|
let meta: MetaFile['files'] = {}
|
|
677
|
+
let dirs: string[] = []
|
|
678
|
+
let links: Array<[string, string]> = []
|
|
631
679
|
try {
|
|
632
680
|
const check = new SumCheck()
|
|
633
681
|
for await (const e of tarEntries(tar)) {
|
|
@@ -635,7 +683,10 @@ export async function extractArtifactStream(
|
|
|
635
683
|
const body = await check.body(e.name, e.body)
|
|
636
684
|
if (body === null) continue
|
|
637
685
|
if (e.name === META_ENTRY) {
|
|
638
|
-
|
|
686
|
+
const parsed = JSON.parse(await textOf(body)) as MetaFile
|
|
687
|
+
meta = parsed.files ?? {}
|
|
688
|
+
dirs = dirsOf(parsed)
|
|
689
|
+
links = linksOf(parsed)
|
|
639
690
|
continue
|
|
640
691
|
}
|
|
641
692
|
assertSafeName(e.name)
|
|
@@ -658,6 +709,38 @@ export async function extractArtifactStream(
|
|
|
658
709
|
await x.stage(e.name, dest.base, target, e.size <= SMALL_ENTRY ? await bytesOf(body) : body)
|
|
659
710
|
}
|
|
660
711
|
check.done()
|
|
712
|
+
for (const name of dirs) {
|
|
713
|
+
if (accept !== undefined && !accept(name)) {
|
|
714
|
+
throw new ArchiveSecurityError(
|
|
715
|
+
`archive directory is not one of the task's declared outputs: ${name}`,
|
|
716
|
+
)
|
|
717
|
+
}
|
|
718
|
+
const dest = x.destFor(name)
|
|
719
|
+
if (dest === null || dest.rel.length === 0) continue
|
|
720
|
+
const target = path.join(dest.base, dest.rel)
|
|
721
|
+
await x.assertContained(dest.base, target, name)
|
|
722
|
+
await x.makeDir(dest.base, target)
|
|
723
|
+
// The index rows name a directory with a trailing slash.
|
|
724
|
+
provided.add(`${name}/`)
|
|
725
|
+
}
|
|
726
|
+
// A link may point only inside the workspace (the project, without one).
|
|
727
|
+
const bound = stageRoot ?? workspaceDest ?? destDir
|
|
728
|
+
for (const [name, to] of links) {
|
|
729
|
+
if (accept !== undefined && !accept(name)) {
|
|
730
|
+
throw new ArchiveSecurityError(
|
|
731
|
+
`archive link is not one of the task's declared outputs: ${name}`,
|
|
732
|
+
)
|
|
733
|
+
}
|
|
734
|
+
const dest = x.destFor(name)
|
|
735
|
+
if (dest === null || dest.rel.length === 0) continue
|
|
736
|
+
const target = path.join(dest.base, dest.rel)
|
|
737
|
+
await x.assertContained(dest.base, target, name)
|
|
738
|
+
if (!inside(path.resolve(path.dirname(target), to), bound)) {
|
|
739
|
+
throw new ArchiveSecurityError(`archive link ${name} points out of the workspace: ${to}`)
|
|
740
|
+
}
|
|
741
|
+
await x.stageLink(name, dest.base, target, to)
|
|
742
|
+
provided.add(name)
|
|
743
|
+
}
|
|
661
744
|
verify?.(provided)
|
|
662
745
|
// The sidecar's stat is the truth, a mode of 000 and an mtime at or
|
|
663
746
|
// before the epoch included (`SOURCE_DATE_EPOCH=0`). Without one, a
|
|
@@ -720,7 +803,7 @@ function keep(s: Staged, ino: number, mode: number, mtimeMs: number | undefined)
|
|
|
720
803
|
if (st.ino !== ino || st.nlink !== 1) {
|
|
721
804
|
throw new Error(`${s.target} changed while it was restored`)
|
|
722
805
|
}
|
|
723
|
-
if ((st.mode & 0o7777) !== (mode
|
|
806
|
+
if ((st.mode & 0o7777) !== restoredMode(mode)) fchmodSync(fd, restoredMode(mode))
|
|
724
807
|
if (mtimeMs !== undefined && Math.floor(st.mtimeMs) !== mtimeMs) {
|
|
725
808
|
const t = new Date(mtimeMs)
|
|
726
809
|
futimesSync(fd, t, t)
|
|
@@ -745,6 +828,20 @@ interface Staged {
|
|
|
745
828
|
base: string
|
|
746
829
|
/** The target already holds the entry's bytes: nothing was written, and only its mode and mtime may change. */
|
|
747
830
|
same?: number
|
|
831
|
+
/** `tmp` was copied beside `target`, which is on another filesystem than the staging (`crossOver`). */
|
|
832
|
+
beside?: true
|
|
833
|
+
/** A symlink to this target, made at `tmp`: no mode, no mtime. */
|
|
834
|
+
link?: string
|
|
835
|
+
/** `tmp` is inside the staged copy of this missing directory, which commit renames whole. */
|
|
836
|
+
tree?: string
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
/** A directory missing in the tree, built in the staging directory and renamed in whole. */
|
|
840
|
+
interface Tree {
|
|
841
|
+
at: string
|
|
842
|
+
base: string
|
|
843
|
+
/** The empty directories under it the entry holds. */
|
|
844
|
+
dirs: string[]
|
|
748
845
|
}
|
|
749
846
|
|
|
750
847
|
/**
|
|
@@ -754,6 +851,8 @@ interface Staged {
|
|
|
754
851
|
*/
|
|
755
852
|
class Extractor {
|
|
756
853
|
private readonly staged: Staged[] = []
|
|
854
|
+
/** Empty output directories made, with the top of each chain created. */
|
|
855
|
+
private readonly madeDirs: Array<{ dir: string; created: string }> = []
|
|
757
856
|
private inflight: Promise<unknown>[] = []
|
|
758
857
|
private inflightBytes = 0
|
|
759
858
|
private readonly realBaseCache = new Map<string, string>()
|
|
@@ -767,6 +866,11 @@ class Extractor {
|
|
|
767
866
|
// entry's own check and its write.
|
|
768
867
|
private readonly containedDirs = new Set<string>()
|
|
769
868
|
private readonly ensuredDirs = new Set<string>()
|
|
869
|
+
private readonly trees = new Map<string, Tree>()
|
|
870
|
+
// Directories the containment walk found missing, which `treeFor` would
|
|
871
|
+
// otherwise probe again. A directory made since fails the tree's rename,
|
|
872
|
+
// and the entries then go in one by one.
|
|
873
|
+
private readonly seenAbsent = new Set<string>()
|
|
770
874
|
|
|
771
875
|
constructor(
|
|
772
876
|
private readonly destDir: string,
|
|
@@ -860,6 +964,7 @@ class Extractor {
|
|
|
860
964
|
// trip and error (X-163). A dangling link is there and falls through,
|
|
861
965
|
// as does an lstat that fails otherwise: the realpath decides.
|
|
862
966
|
if (absent(probe)) {
|
|
967
|
+
this.seenAbsent.add(probe)
|
|
863
968
|
probe = path.dirname(probe)
|
|
864
969
|
continue
|
|
865
970
|
}
|
|
@@ -907,6 +1012,82 @@ class Extractor {
|
|
|
907
1012
|
return mkdir(real, { recursive: true })
|
|
908
1013
|
}
|
|
909
1014
|
|
|
1015
|
+
/**
|
|
1016
|
+
* Where an entry in `dir` is written when a directory on the way to it is
|
|
1017
|
+
* missing: inside a copy of the topmost missing one, built in the staging
|
|
1018
|
+
* directory and renamed into place whole on commit. One rename per
|
|
1019
|
+
* missing directory, not one per file: every file of a restore into a
|
|
1020
|
+
* cleaned tree renamed across directories, all out of one staging
|
|
1021
|
+
* directory, and the kernel time of a 5,000-artifact restore doubled
|
|
1022
|
+
* (2026-10-17). A reader sees the directory complete or not at all.
|
|
1023
|
+
* Undefined where the group or filesystem would differ from a write in
|
|
1024
|
+
* place: the entry is then staged and renamed alone.
|
|
1025
|
+
*/
|
|
1026
|
+
private async treeFor(base: string, dir: string): Promise<string | undefined> {
|
|
1027
|
+
if (this.stageRoot === undefined) return undefined
|
|
1028
|
+
const abs = path.resolve(dir)
|
|
1029
|
+
for (const [top, t] of this.trees) {
|
|
1030
|
+
if (abs === top || abs.startsWith(top + path.sep)) return t.at + abs.slice(top.length)
|
|
1031
|
+
}
|
|
1032
|
+
const baseResolved = path.resolve(base)
|
|
1033
|
+
let top: string | undefined
|
|
1034
|
+
let probe = abs
|
|
1035
|
+
while (
|
|
1036
|
+
probe.startsWith(baseResolved + path.sep) &&
|
|
1037
|
+
(this.seenAbsent.has(probe) || absent(probe))
|
|
1038
|
+
) {
|
|
1039
|
+
top = probe
|
|
1040
|
+
probe = path.dirname(probe)
|
|
1041
|
+
}
|
|
1042
|
+
if (top === undefined) return undefined
|
|
1043
|
+
const staging = await stagingDir(this.stageRoot)
|
|
1044
|
+
const st = statSync(staging)
|
|
1045
|
+
const parent = statSync(probe, { throwIfNoEntry: false })
|
|
1046
|
+
if (
|
|
1047
|
+
parent === undefined ||
|
|
1048
|
+
!parent.isDirectory() ||
|
|
1049
|
+
parent.dev !== st.dev ||
|
|
1050
|
+
((parent.mode | st.mode) & 0o2000) !== 0 ||
|
|
1051
|
+
(INHERITS_GROUP && parent.gid !== st.gid)
|
|
1052
|
+
) {
|
|
1053
|
+
return undefined
|
|
1054
|
+
}
|
|
1055
|
+
const at = path.join(staging, (tmpSeq++).toString(36))
|
|
1056
|
+
this.trees.set(top, { at, base, dirs: [] })
|
|
1057
|
+
return at + abs.slice(top.length)
|
|
1058
|
+
}
|
|
1059
|
+
|
|
1060
|
+
/** `dir` made, once per extraction. */
|
|
1061
|
+
private async ensure(dir: string): Promise<void> {
|
|
1062
|
+
if (this.ensuredDirs.has(dir)) return
|
|
1063
|
+
await mkdir(dir, { recursive: true })
|
|
1064
|
+
this.ensuredDirs.add(dir)
|
|
1065
|
+
}
|
|
1066
|
+
|
|
1067
|
+
/** The tree `target` is staged in. */
|
|
1068
|
+
private treeOf(target: string): string | undefined {
|
|
1069
|
+
for (const top of this.trees.keys()) if (target.startsWith(top + path.sep)) return top
|
|
1070
|
+
return undefined
|
|
1071
|
+
}
|
|
1072
|
+
|
|
1073
|
+
/** An empty output directory: made now, pruned by `abort` if it was not there. */
|
|
1074
|
+
async makeDir(base: string, dir: string): Promise<void> {
|
|
1075
|
+
const staged = await this.treeFor(base, dir)
|
|
1076
|
+
if (staged !== undefined) {
|
|
1077
|
+
await this.ensure(staged)
|
|
1078
|
+
this.trees.get(this.treeOf(dir + path.sep)!)!.dirs.push(dir)
|
|
1079
|
+
return
|
|
1080
|
+
}
|
|
1081
|
+
let created: string | undefined
|
|
1082
|
+
try {
|
|
1083
|
+
created = await mkdir(dir, { recursive: true })
|
|
1084
|
+
} catch (err) {
|
|
1085
|
+
created = await this.mkdirThroughLink(base, dir, err as NodeJS.ErrnoException)
|
|
1086
|
+
}
|
|
1087
|
+
this.ensuredDirs.add(dir)
|
|
1088
|
+
if (created !== undefined) this.madeDirs.push({ dir, created })
|
|
1089
|
+
}
|
|
1090
|
+
|
|
910
1091
|
/**
|
|
911
1092
|
* Write the entry beside its target. A buffer is written without
|
|
912
1093
|
* waiting, bounded by `INFLIGHT_BYTES`; a chunk stream goes to a file
|
|
@@ -919,6 +1100,25 @@ class Extractor {
|
|
|
919
1100
|
body: Uint8Array | AsyncIterable<Uint8Array>,
|
|
920
1101
|
): Promise<void> {
|
|
921
1102
|
const dir = path.dirname(target)
|
|
1103
|
+
const inTree = await this.treeFor(base, dir)
|
|
1104
|
+
if (inTree !== undefined) {
|
|
1105
|
+
await this.ensure(inTree)
|
|
1106
|
+
const tmp = path.join(inTree, path.basename(target))
|
|
1107
|
+
const streamed = !(body instanceof Uint8Array)
|
|
1108
|
+
const tree = this.treeOf(target)!
|
|
1109
|
+
this.staged.push({
|
|
1110
|
+
staged: true,
|
|
1111
|
+
name,
|
|
1112
|
+
tmp,
|
|
1113
|
+
target,
|
|
1114
|
+
created: undefined,
|
|
1115
|
+
streamed,
|
|
1116
|
+
base,
|
|
1117
|
+
tree,
|
|
1118
|
+
})
|
|
1119
|
+
await this.write(tmp, body)
|
|
1120
|
+
return
|
|
1121
|
+
}
|
|
922
1122
|
let created: string | undefined
|
|
923
1123
|
if (!this.ensuredDirs.has(dir)) {
|
|
924
1124
|
try {
|
|
@@ -980,6 +1180,10 @@ class Extractor {
|
|
|
980
1180
|
: siblingTemp(target)
|
|
981
1181
|
const streamed = !(body instanceof Uint8Array)
|
|
982
1182
|
this.staged.push({ staged, name, tmp, target, created, streamed, base })
|
|
1183
|
+
await this.write(tmp, body)
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
private async write(tmp: string, body: Uint8Array | AsyncIterable<Uint8Array>): Promise<void> {
|
|
983
1187
|
if (body instanceof Uint8Array) {
|
|
984
1188
|
this.inflightBytes += body.byteLength
|
|
985
1189
|
// `writeFile`, not `Bun.write`: for a buffer this size `Bun.write`
|
|
@@ -1006,6 +1210,49 @@ class Extractor {
|
|
|
1006
1210
|
}
|
|
1007
1211
|
}
|
|
1008
1212
|
|
|
1213
|
+
/** A symlink: made apart from the target like a file, renamed over it on commit. */
|
|
1214
|
+
async stageLink(name: string, base: string, target: string, to: string): Promise<void> {
|
|
1215
|
+
const dir = path.dirname(target)
|
|
1216
|
+
const inTree = await this.treeFor(base, dir)
|
|
1217
|
+
if (inTree !== undefined) {
|
|
1218
|
+
await this.ensure(inTree)
|
|
1219
|
+
const tmp = path.join(inTree, path.basename(target))
|
|
1220
|
+
const tree = this.treeOf(target)!
|
|
1221
|
+
this.staged.push({
|
|
1222
|
+
staged: true,
|
|
1223
|
+
name,
|
|
1224
|
+
tmp,
|
|
1225
|
+
target,
|
|
1226
|
+
created: undefined,
|
|
1227
|
+
streamed: false,
|
|
1228
|
+
base,
|
|
1229
|
+
link: to,
|
|
1230
|
+
tree,
|
|
1231
|
+
})
|
|
1232
|
+
await symlink(to, tmp)
|
|
1233
|
+
return
|
|
1234
|
+
}
|
|
1235
|
+
let created: string | undefined
|
|
1236
|
+
if (!this.ensuredDirs.has(dir)) {
|
|
1237
|
+
try {
|
|
1238
|
+
created = await mkdir(dir, { recursive: true })
|
|
1239
|
+
} catch (err) {
|
|
1240
|
+
created = await this.mkdirThroughLink(base, dir, err as NodeJS.ErrnoException)
|
|
1241
|
+
}
|
|
1242
|
+
this.ensuredDirs.add(dir)
|
|
1243
|
+
}
|
|
1244
|
+
if (created === undefined) {
|
|
1245
|
+
const st = lstatSync(target, { throwIfNoEntry: false })
|
|
1246
|
+
if (st?.isSymbolicLink() === true && readlinkSync(target) === to) return
|
|
1247
|
+
}
|
|
1248
|
+
const staged = this.stageRoot !== undefined
|
|
1249
|
+
const tmp = staged
|
|
1250
|
+
? path.join(await stagingDir(this.stageRoot!), (tmpSeq++).toString(36))
|
|
1251
|
+
: siblingTemp(target)
|
|
1252
|
+
this.staged.push({ staged, name, tmp, target, created, streamed: false, base, link: to })
|
|
1253
|
+
await symlink(to, tmp)
|
|
1254
|
+
}
|
|
1255
|
+
|
|
1009
1256
|
private async drain(): Promise<void> {
|
|
1010
1257
|
const pending = this.inflight
|
|
1011
1258
|
this.inflight = []
|
|
@@ -1028,6 +1275,17 @@ class Extractor {
|
|
|
1028
1275
|
metaFor: (name: string) => [mode: number, mtimeMs: number | undefined],
|
|
1029
1276
|
): Promise<void> {
|
|
1030
1277
|
await this.drain()
|
|
1278
|
+
// One stat per directory, shared by the filesystem check and the group.
|
|
1279
|
+
const dirStats = new Map<string, Stats>()
|
|
1280
|
+
const statDir = (dir: string): Stats => {
|
|
1281
|
+
let st = dirStats.get(dir)
|
|
1282
|
+
if (st === undefined) {
|
|
1283
|
+
st = statSync(dir)
|
|
1284
|
+
dirStats.set(dir, st)
|
|
1285
|
+
}
|
|
1286
|
+
return st
|
|
1287
|
+
}
|
|
1288
|
+
await this.crossOver(statDir)
|
|
1031
1289
|
const created = new Map<boolean, { mode: number; uid: number; gid: number }>()
|
|
1032
1290
|
// The group a file written in place would take, by directory: a setgid
|
|
1033
1291
|
// directory's on Linux, every directory's on BSD/macOS. A file staged
|
|
@@ -1036,44 +1294,147 @@ class Extractor {
|
|
|
1036
1294
|
let n = 0
|
|
1037
1295
|
let renames: Promise<void>[] = []
|
|
1038
1296
|
const checked = new Set<string>()
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1297
|
+
try {
|
|
1298
|
+
for (const s of this.staged) {
|
|
1299
|
+
if (s.tree === undefined)
|
|
1300
|
+
await this.assertStillContained(s.base, path.dirname(s.target), checked)
|
|
1301
|
+
const [mode, mtimeMs] = metaFor(s.name)
|
|
1302
|
+
if (s.same !== undefined) {
|
|
1303
|
+
keep(s, s.same, mode, mtimeMs)
|
|
1304
|
+
continue
|
|
1305
|
+
}
|
|
1306
|
+
if (s.link !== undefined) {
|
|
1307
|
+
if (s.tree !== undefined) continue
|
|
1308
|
+
renames.push(handled(this.place(s, mode, undefined)))
|
|
1309
|
+
if (++n % RENAME_BATCH === 0) {
|
|
1310
|
+
await settle(renames)
|
|
1311
|
+
renames = []
|
|
1312
|
+
}
|
|
1313
|
+
continue
|
|
1314
|
+
}
|
|
1315
|
+
if (s.beside === true) {
|
|
1316
|
+
chmodSync(s.tmp, restoredMode(mode))
|
|
1317
|
+
if (mtimeMs !== undefined) utimesSync(s.tmp, new Date(mtimeMs), new Date(mtimeMs))
|
|
1318
|
+
renames.push(handled(this.place(s, mode, undefined)))
|
|
1319
|
+
if (++n % RENAME_BATCH === 0) {
|
|
1320
|
+
await settle(renames)
|
|
1321
|
+
renames = []
|
|
1322
|
+
}
|
|
1323
|
+
continue
|
|
1324
|
+
}
|
|
1325
|
+
let made = created.get(s.streamed)
|
|
1326
|
+
if (made === undefined) {
|
|
1327
|
+
const st = statSync(s.tmp)
|
|
1328
|
+
made = { mode: st.mode & 0o777, uid: st.uid, gid: st.gid }
|
|
1329
|
+
created.set(s.streamed, made)
|
|
1330
|
+
}
|
|
1331
|
+
if (restoredMode(mode) !== made.mode) chmodSync(s.tmp, restoredMode(mode))
|
|
1332
|
+
// A Date, not seconds: Bun reads a negative number of seconds as
|
|
1333
|
+
// "now", and a Date before 1970 as itself (Bun 1.4.2).
|
|
1334
|
+
const t = mtimeMs === undefined ? undefined : new Date(mtimeMs)
|
|
1335
|
+
if (t !== undefined) utimesSync(s.tmp, t, t)
|
|
1336
|
+
if (s.tree !== undefined) continue
|
|
1337
|
+
if (s.staged) {
|
|
1338
|
+
const dir = path.dirname(s.target)
|
|
1339
|
+
let gid = setgid.get(dir)
|
|
1340
|
+
if (gid === undefined) {
|
|
1341
|
+
const st = statDir(dir)
|
|
1342
|
+
gid = INHERITS_GROUP || (st.mode & 0o2000) !== 0 ? st.gid : null
|
|
1343
|
+
setgid.set(dir, gid)
|
|
1344
|
+
}
|
|
1345
|
+
if (gid !== null && gid !== made.gid) chownSync(s.tmp, made.uid, gid)
|
|
1346
|
+
}
|
|
1347
|
+
renames.push(handled(this.place(s, mode, t)))
|
|
1348
|
+
if (++n % RENAME_BATCH === 0) {
|
|
1349
|
+
await settle(renames)
|
|
1350
|
+
renames = []
|
|
1064
1351
|
}
|
|
1065
|
-
if (gid !== null && gid !== made.gid) chownSync(s.tmp, made.uid, gid)
|
|
1066
|
-
}
|
|
1067
|
-
renames.push(this.place(s, mode, t))
|
|
1068
|
-
if (++n % RENAME_BATCH === 0) {
|
|
1069
|
-
await settle(renames)
|
|
1070
|
-
renames = []
|
|
1071
1352
|
}
|
|
1353
|
+
} catch (err) {
|
|
1354
|
+
// Every rename in flight lands or fails before `abort` cleans up
|
|
1355
|
+
// (a fuzz run on macOS, 2026-10-17).
|
|
1356
|
+
await Promise.allSettled(renames)
|
|
1357
|
+
throw err
|
|
1072
1358
|
}
|
|
1073
1359
|
await settle(renames)
|
|
1360
|
+
for (const [top, t] of this.trees) await this.placeTree(top, t, checked)
|
|
1361
|
+
await this.assertLandedInside()
|
|
1362
|
+
this.trees.clear()
|
|
1074
1363
|
this.staged.length = 0
|
|
1075
1364
|
}
|
|
1076
1365
|
|
|
1366
|
+
/**
|
|
1367
|
+
* Rename a staged tree to the directory it stands for. Something made
|
|
1368
|
+
* there since staging (another restore of the same outputs) takes the
|
|
1369
|
+
* tree's entries one by one instead, each renamed over its target.
|
|
1370
|
+
*/
|
|
1371
|
+
private async placeTree(top: string, t: Tree, checked: Set<string>): Promise<void> {
|
|
1372
|
+
await this.assertStillContained(t.base, path.dirname(top), checked)
|
|
1373
|
+
try {
|
|
1374
|
+
await rename(t.at, top)
|
|
1375
|
+
return
|
|
1376
|
+
} catch (err) {
|
|
1377
|
+
const code = (err as NodeJS.ErrnoException).code
|
|
1378
|
+
if (code !== 'EEXIST' && code !== 'ENOTEMPTY' && code !== 'ENOTDIR' && code !== 'EISDIR') {
|
|
1379
|
+
throw err
|
|
1380
|
+
}
|
|
1381
|
+
}
|
|
1382
|
+
for (const dir of t.dirs) await this.makeDirIn(t.base, dir)
|
|
1383
|
+
for (const s of this.staged) {
|
|
1384
|
+
if (s.tree !== top) continue
|
|
1385
|
+
const dir = path.dirname(s.target)
|
|
1386
|
+
await this.makeDirIn(t.base, dir)
|
|
1387
|
+
await this.assertStillContained(t.base, dir, checked)
|
|
1388
|
+
await rename(s.tmp, s.target)
|
|
1389
|
+
}
|
|
1390
|
+
await rm(t.at, { recursive: true, force: true })
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1393
|
+
private async makeDirIn(base: string, dir: string): Promise<void> {
|
|
1394
|
+
try {
|
|
1395
|
+
await mkdir(dir, { recursive: true })
|
|
1396
|
+
} catch (err) {
|
|
1397
|
+
await this.mkdirThroughLink(base, dir, err as NodeJS.ErrnoException)
|
|
1398
|
+
}
|
|
1399
|
+
}
|
|
1400
|
+
|
|
1401
|
+
/**
|
|
1402
|
+
* Copy every staged entry whose target is on another filesystem than
|
|
1403
|
+
* `.vx/tmp/` to a temp beside its target, before any rename: a rename
|
|
1404
|
+
* there would fail EXDEV, and copying at the rename, one file at a time,
|
|
1405
|
+
* left the files renamed before a failed copy (ENOSPC) new and the rest
|
|
1406
|
+
* old (Growth's parity run, 2026-10-16). A failed copy throws with
|
|
1407
|
+
* nothing renamed, and `abort` removes the temps made so far.
|
|
1408
|
+
*/
|
|
1409
|
+
private async crossOver(statDir: (dir: string) => Stats): Promise<void> {
|
|
1410
|
+
const dev = (dir: string): number => statDir(dir).dev
|
|
1411
|
+
const cross = this.staged.filter(
|
|
1412
|
+
(s) =>
|
|
1413
|
+
s.staged &&
|
|
1414
|
+
s.same === undefined &&
|
|
1415
|
+
s.tree === undefined &&
|
|
1416
|
+
dev(path.dirname(s.tmp)) !== dev(path.dirname(s.target)),
|
|
1417
|
+
)
|
|
1418
|
+
for (let i = 0; i < cross.length; i += RENAME_BATCH) {
|
|
1419
|
+
await settle(
|
|
1420
|
+
cross.slice(i, i + RENAME_BATCH).map(async (s) => {
|
|
1421
|
+
const side = siblingTemp(s.target)
|
|
1422
|
+
try {
|
|
1423
|
+
if (s.link !== undefined) await symlink(s.link, side)
|
|
1424
|
+
else await copyFile(s.tmp, side)
|
|
1425
|
+
} catch (err) {
|
|
1426
|
+
await unlink(side).catch(() => undefined)
|
|
1427
|
+
throw err
|
|
1428
|
+
}
|
|
1429
|
+
const staged = s.tmp
|
|
1430
|
+
s.tmp = side
|
|
1431
|
+
s.beside = true
|
|
1432
|
+
await unlink(staged).catch(() => undefined)
|
|
1433
|
+
}),
|
|
1434
|
+
)
|
|
1435
|
+
}
|
|
1436
|
+
}
|
|
1437
|
+
|
|
1077
1438
|
/**
|
|
1078
1439
|
* Rename `s` over its target. A target on another filesystem than
|
|
1079
1440
|
* `.vx/tmp/` (EXDEV) is reached through a temp beside it, for that file
|
|
@@ -1086,9 +1447,12 @@ class Extractor {
|
|
|
1086
1447
|
if (!s.staged || (err as NodeJS.ErrnoException).code !== 'EXDEV') throw err
|
|
1087
1448
|
const side = siblingTemp(s.target)
|
|
1088
1449
|
try {
|
|
1089
|
-
await
|
|
1090
|
-
|
|
1091
|
-
|
|
1450
|
+
if (s.link !== undefined) await symlink(s.link, side)
|
|
1451
|
+
else {
|
|
1452
|
+
await copyFile(s.tmp, side)
|
|
1453
|
+
chmodSync(side, restoredMode(mode))
|
|
1454
|
+
if (t !== undefined) utimesSync(side, t, t)
|
|
1455
|
+
}
|
|
1092
1456
|
await rename(side, s.target)
|
|
1093
1457
|
} catch (e) {
|
|
1094
1458
|
await unlink(side).catch(() => undefined)
|
|
@@ -1104,14 +1468,44 @@ class Extractor {
|
|
|
1104
1468
|
* Bun has no `renameat` against a held directory fd, so this is by path,
|
|
1105
1469
|
* once per directory per commit.
|
|
1106
1470
|
*/
|
|
1107
|
-
private async assertStillContained(
|
|
1108
|
-
|
|
1471
|
+
private async assertStillContained(
|
|
1472
|
+
base: string,
|
|
1473
|
+
dir: string,
|
|
1474
|
+
checked: Set<string>,
|
|
1475
|
+
): Promise<void> {
|
|
1109
1476
|
if (checked.has(dir)) return
|
|
1110
1477
|
checked.add(dir)
|
|
1111
|
-
const realBase = await this.realBaseOf(
|
|
1478
|
+
const realBase = await this.realBaseOf(base)
|
|
1112
1479
|
const real = await realOf(dir, true)
|
|
1113
1480
|
if (real !== realBase && !real.startsWith(realBase + path.sep)) {
|
|
1114
|
-
throw await linkOutError(path.resolve(
|
|
1481
|
+
throw await linkOutError(path.resolve(base), dir, realBase)
|
|
1482
|
+
}
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1485
|
+
/**
|
|
1486
|
+
* The check before each rename, once more after all landed: the rename
|
|
1487
|
+
* runs on the thread pool, and a directory swapped for a link out in that
|
|
1488
|
+
* gap carried the entry with it (restore-link-swap-rename.test.ts). An
|
|
1489
|
+
* entry that landed outside is removed from there and the restore fails.
|
|
1490
|
+
*/
|
|
1491
|
+
private async assertLandedInside(): Promise<void> {
|
|
1492
|
+
const landed = new Map<string, { base: string; placed: string[] }>()
|
|
1493
|
+
const add = (base: string, dir: string, placed: string): void => {
|
|
1494
|
+
const at = landed.get(dir)
|
|
1495
|
+
if (at === undefined) landed.set(dir, { base, placed: [placed] })
|
|
1496
|
+
else at.placed.push(placed)
|
|
1497
|
+
}
|
|
1498
|
+
for (const s of this.staged) {
|
|
1499
|
+
if (s.same === undefined && s.tree === undefined)
|
|
1500
|
+
add(s.base, path.dirname(s.target), s.target)
|
|
1501
|
+
}
|
|
1502
|
+
for (const [top, t] of this.trees) add(t.base, path.dirname(top), top)
|
|
1503
|
+
for (const [dir, { base, placed }] of landed) {
|
|
1504
|
+
const realBase = await this.realBaseOf(base)
|
|
1505
|
+
const real = await realOf(dir, true).catch(() => null)
|
|
1506
|
+
if (real === null || real === realBase || real.startsWith(realBase + path.sep)) continue
|
|
1507
|
+
await Promise.allSettled(placed.map((p) => rm(p, { recursive: true, force: true })))
|
|
1508
|
+
throw await linkOutError(path.resolve(base), dir, realBase)
|
|
1115
1509
|
}
|
|
1116
1510
|
}
|
|
1117
1511
|
|
|
@@ -1129,6 +1523,8 @@ class Extractor {
|
|
|
1129
1523
|
s.same === undefined ? unlink(s.tmp).catch(() => undefined) : undefined,
|
|
1130
1524
|
),
|
|
1131
1525
|
)
|
|
1526
|
+
for (const t of this.trees.values()) await rm(t.at, { recursive: true, force: true })
|
|
1527
|
+
this.trees.clear()
|
|
1132
1528
|
// Newest first: a later entry's chain can sit inside an earlier one's
|
|
1133
1529
|
// (`dist/` then `dist/sub/`), never the reverse.
|
|
1134
1530
|
for (const s of this.staged.toReversed()) {
|
|
@@ -1150,6 +1546,21 @@ class Extractor {
|
|
|
1150
1546
|
dir = path.dirname(dir)
|
|
1151
1547
|
}
|
|
1152
1548
|
}
|
|
1549
|
+
for (const { dir: made, created } of this.madeDirs.toReversed()) {
|
|
1550
|
+
const top = path.resolve(created)
|
|
1551
|
+
let dir = made.startsWith(top) ? made : await realOf(made).catch(() => made)
|
|
1552
|
+
while (
|
|
1553
|
+
dir.startsWith(top) &&
|
|
1554
|
+
(await rmdir(dir).then(
|
|
1555
|
+
() => true,
|
|
1556
|
+
() => false,
|
|
1557
|
+
))
|
|
1558
|
+
) {
|
|
1559
|
+
if (dir === top) break
|
|
1560
|
+
dir = path.dirname(dir)
|
|
1561
|
+
}
|
|
1562
|
+
}
|
|
1563
|
+
this.madeDirs.length = 0
|
|
1153
1564
|
this.staged.length = 0
|
|
1154
1565
|
}
|
|
1155
1566
|
}
|
|
@@ -1164,6 +1575,16 @@ class Extractor {
|
|
|
1164
1575
|
*/
|
|
1165
1576
|
const RENAME_BATCH = 64
|
|
1166
1577
|
|
|
1578
|
+
/**
|
|
1579
|
+
* `p`, its failure marked handled at once: `settle` reads it later, and a
|
|
1580
|
+
* rename that fails while the commit awaits something else is not an
|
|
1581
|
+
* unhandled rejection that fails the process.
|
|
1582
|
+
*/
|
|
1583
|
+
function handled(p: Promise<void>): Promise<void> {
|
|
1584
|
+
p.catch(() => undefined)
|
|
1585
|
+
return p
|
|
1586
|
+
}
|
|
1587
|
+
|
|
1167
1588
|
/** Wait for every rename, then throw the first failure: none is left in flight for `abort`. */
|
|
1168
1589
|
async function settle(renames: readonly Promise<void>[]): Promise<void> {
|
|
1169
1590
|
for (const r of await Promise.allSettled(renames)) if (r.status === 'rejected') throw r.reason
|