@vzn/vx 0.0.591 → 0.0.595
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/launcher.cjs +9 -2
- package/package.json +5 -5
- package/src/bin.ts +26 -11
- package/src/cache/archive.ts +18 -7
- package/src/cache/cache.ts +137 -101
- package/src/cache/config-evals.ts +19 -4
- package/src/cache/git-inputs.ts +33 -17
- package/src/cache/inputs.ts +157 -54
- package/src/cache/layered-cache.ts +25 -2
- package/src/cache/tar-stream.ts +4 -2
- package/src/cli/cache.ts +4 -16
- package/src/cli/completions.ts +42 -6
- package/src/cli/foreign-flags.ts +56 -6
- package/src/cli/help.ts +27 -8
- package/src/cli/info.ts +4 -6
- package/src/cli/init.ts +4 -4
- package/src/cli/last.ts +49 -20
- package/src/cli/lock.ts +3 -3
- package/src/cli/plan-format.ts +9 -0
- package/src/cli/plugin-templates.ts +2 -2
- package/src/cli/run.ts +80 -50
- package/src/cli/select.ts +44 -5
- package/src/cli/show.ts +47 -9
- package/src/cli/upgrade.ts +2 -3
- package/src/cli/watch-filter.ts +21 -7
- package/src/cli/watch-fs.ts +32 -8
- package/src/cli/watch-judge.ts +1 -1
- package/src/cli/watch.ts +40 -15
- package/src/cli/why.ts +4 -13
- package/src/cli/workspace-config.ts +1 -1
- package/src/config.ts +5 -5
- package/src/exec/env.ts +8 -5
- package/src/exec/index.ts +1 -0
- package/src/exec/proc-sample.ts +5 -1
- package/src/exec/runner.ts +126 -47
- package/src/exec/sandbox-deny-scan.ts +2 -2
- package/src/exec/sandbox-paths.ts +11 -5
- package/src/exec/sandbox-runtime.ts +89 -23
- package/src/exec/sandbox-violations.ts +23 -9
- package/src/graph/index.ts +1 -0
- package/src/graph/scheduler.ts +14 -4
- package/src/graph/task-graph.ts +3 -2
- package/src/index.ts +9 -0
- package/src/orchestrator/affected-tasks.ts +34 -10
- package/src/orchestrator/doctor.ts +2 -1
- package/src/orchestrator/execute-task.ts +43 -10
- package/src/orchestrator/hit-restore.ts +9 -5
- package/src/orchestrator/index.ts +1 -0
- package/src/orchestrator/local-shortcircuit.ts +4 -1
- package/src/orchestrator/lockfile-claim.ts +45 -49
- package/src/orchestrator/logger.ts +4 -1
- package/src/orchestrator/miss-save.ts +10 -3
- package/src/orchestrator/placement.ts +17 -3
- package/src/orchestrator/plan.ts +25 -1
- package/src/orchestrator/plugin-host.ts +41 -3
- package/src/orchestrator/plugin.ts +11 -14
- package/src/orchestrator/projects.ts +18 -9
- package/src/orchestrator/run-context.ts +2 -2
- package/src/orchestrator/run-lock.ts +14 -4
- package/src/orchestrator/run.ts +94 -16
- package/src/orchestrator/sandbox-request.ts +6 -10
- package/src/orchestrator/signals.ts +5 -2
- package/src/orchestrator/summary.ts +14 -1
- package/src/util/index.ts +7 -1
- package/src/util/secret-mask.ts +36 -0
- package/src/util/settle.ts +15 -0
- package/src/workspace/affected.ts +47 -19
- package/src/workspace/config-cache.ts +25 -4
- package/src/workspace/config-eval.ts +4 -0
- package/src/workspace/config-imports.ts +2 -2
- package/src/workspace/config-schema.ts +47 -10
- package/src/workspace/filter.ts +5 -4
- package/src/workspace/migrate-scripts.ts +23 -6
- package/src/workspace/package-graph.ts +15 -3
- package/src/workspace/project-loader.ts +89 -2
- package/src/workspace/workspace.ts +121 -14
package/src/cache/git-inputs.ts
CHANGED
|
@@ -8,11 +8,12 @@
|
|
|
8
8
|
import path from 'node:path'
|
|
9
9
|
import { existsSync, lstatSync, readdirSync, readFileSync, realpathSync } from 'node:fs'
|
|
10
10
|
import {
|
|
11
|
-
UserError,
|
|
12
11
|
executablePath,
|
|
13
12
|
gitSpawnRefusal,
|
|
14
13
|
isInstalledPath,
|
|
15
14
|
notAWorkTree,
|
|
15
|
+
relPosix,
|
|
16
|
+
UserError,
|
|
16
17
|
xxh3hex,
|
|
17
18
|
} from '../util/index.js'
|
|
18
19
|
import { FILE_HASH_RACY_MS, racyWindowMs } from './layer.js'
|
|
@@ -70,7 +71,9 @@ export function repoFacts(dir: string): RepoFacts | null {
|
|
|
70
71
|
// One line per flag, in order. A git that does not know
|
|
71
72
|
// `--show-object-format` echoes it back, as it does any unknown flag,
|
|
72
73
|
// which reads as sha1 — what the flag's own spawn answered there too.
|
|
73
|
-
const [prefix = '', commonDir = '', format = '', indexFile = ''] = new TextDecoder(
|
|
74
|
+
const [prefix = '', commonDir = '', format = '', indexFile = ''] = new TextDecoder('utf-8', {
|
|
75
|
+
ignoreBOM: true,
|
|
76
|
+
})
|
|
74
77
|
.decode(proc.stdout)
|
|
75
78
|
.split('\n')
|
|
76
79
|
.map((l) => l.trim())
|
|
@@ -115,10 +118,10 @@ function repoFactsFromDisk(dir: string): RepoFacts | undefined {
|
|
|
115
118
|
if (!st.isDirectory() || !existsSync(path.join(dotGit, 'HEAD'))) return undefined
|
|
116
119
|
const objectFormat = configObjectFormat(readFileSync(path.join(dotGit, 'config'), 'utf8'))
|
|
117
120
|
if (objectFormat === undefined) return undefined
|
|
118
|
-
const below =
|
|
121
|
+
const below = relPosix(at, real)
|
|
119
122
|
return {
|
|
120
123
|
prefix: below === '' ? '' : `${below}/`,
|
|
121
|
-
commonDir:
|
|
124
|
+
commonDir: relPosix(real, dotGit),
|
|
122
125
|
objectFormat,
|
|
123
126
|
indexFile: path.relative(real, path.join(dotGit, 'index')).split(path.sep).join('/'),
|
|
124
127
|
}
|
|
@@ -270,7 +273,7 @@ export class GitFilesCache extends Map<string, readonly string[]> {
|
|
|
270
273
|
for (const rel of relPaths) {
|
|
271
274
|
const abs = path.resolve(workspaceRoot, rel)
|
|
272
275
|
if (abs.startsWith(key + path.sep)) {
|
|
273
|
-
under.push(
|
|
276
|
+
under.push(relPosix(key, abs))
|
|
274
277
|
}
|
|
275
278
|
}
|
|
276
279
|
if (under.length > 0) this.recordChanged(key, under)
|
|
@@ -427,8 +430,10 @@ function decodeGitZ(bytes: Uint8Array): { text: string; undecodable: Set<string>
|
|
|
427
430
|
return { text, undecodable }
|
|
428
431
|
}
|
|
429
432
|
|
|
430
|
-
|
|
431
|
-
|
|
433
|
+
// A default decoder strips a leading U+FEFF: the first path git lists
|
|
434
|
+
// would lose it and name no file.
|
|
435
|
+
const LOSSY_UTF8 = new TextDecoder('utf-8', { ignoreBOM: true })
|
|
436
|
+
const FATAL_UTF8 = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true })
|
|
432
437
|
|
|
433
438
|
const LS_FILES_STAGE_RE = /^(?:([A-Za-z]) )?([0-7]{6}) ([0-9a-f]{40,64}) ([0-3])\t/
|
|
434
439
|
|
|
@@ -1193,7 +1198,7 @@ export function gitPathspecs(
|
|
|
1193
1198
|
projectDirs: readonly string[],
|
|
1194
1199
|
workspaceWide: boolean,
|
|
1195
1200
|
): string[] {
|
|
1196
|
-
const rels = projectDirs.map((d) =>
|
|
1201
|
+
const rels = projectDirs.map((d) => relPosix(workspaceRoot, d))
|
|
1197
1202
|
const scoped =
|
|
1198
1203
|
!workspaceWide &&
|
|
1199
1204
|
rels.length > 0 &&
|
|
@@ -1299,8 +1304,11 @@ export async function startGitEnumeration(
|
|
|
1299
1304
|
const facts = repoFacts(workspaceRoot)
|
|
1300
1305
|
// The blob-size check's verdict is a function of the index's entries, so
|
|
1301
1306
|
// the index file's bytes key it: one read, where `ls-files --debug` and a
|
|
1302
|
-
// lookup per entry cost 550 ms at 100,000 files (A-60).
|
|
1303
|
-
|
|
1307
|
+
// lookup per entry cost 550 ms at 100,000 files (A-60). The verdict's
|
|
1308
|
+
// paths are workspace-relative, so the repo→workspace prefix keys it too:
|
|
1309
|
+
// a nested workspace sharing the cache read the outer one's verdict and
|
|
1310
|
+
// trusted a resized blob.
|
|
1311
|
+
const pathspecKey = xxh3hex([facts?.prefix ?? '', ...pathspecs].join('\0'))
|
|
1304
1312
|
const indexFile =
|
|
1305
1313
|
facts === null || facts.indexFile === ''
|
|
1306
1314
|
? undefined
|
|
@@ -1516,9 +1524,10 @@ export async function applyGitEnumeration(
|
|
|
1516
1524
|
// Sort once, then each project's files are a contiguous range found
|
|
1517
1525
|
// by binary search on its `dir/` prefix — O((F+P) log F) instead of
|
|
1518
1526
|
// the O(P·F) per-project startsWith scan (54 ms at 1090 projects ×
|
|
1519
|
-
// ~9k files; ~5 ms this way).
|
|
1520
|
-
//
|
|
1521
|
-
//
|
|
1527
|
+
// ~9k files; ~5 ms this way). Every path under `dir/` sorts in
|
|
1528
|
+
// [`dir/`, `dir0`): '0' is the code unit after '/'. An upper bound of
|
|
1529
|
+
// `dir/` + U+FFFF left out a file whose name starts with U+FFFF, and
|
|
1530
|
+
// it never entered the key.
|
|
1522
1531
|
// Git lists in order; a list that already is skips the sort.
|
|
1523
1532
|
let inOrder = true
|
|
1524
1533
|
for (let i = 1; i < all.length; i++) {
|
|
@@ -1552,10 +1561,15 @@ export async function applyGitEnumeration(
|
|
|
1552
1561
|
return lo
|
|
1553
1562
|
}
|
|
1554
1563
|
for (const projectDir of projectDirs) {
|
|
1555
|
-
const
|
|
1564
|
+
const spelled =
|
|
1556
1565
|
base !== undefined && projectDir.startsWith(base) && path.normalize(projectDir) === projectDir
|
|
1557
1566
|
? projectDir.slice(base.length).replace(/\/$/, '')
|
|
1558
|
-
:
|
|
1567
|
+
: relPosix(workspaceRoot, projectDir)
|
|
1568
|
+
// macOS git reports paths NFC (core.precomposeunicode) while a dir
|
|
1569
|
+
// discovered by readdir keeps the spelling it was created with: match
|
|
1570
|
+
// NFC, and key the OIDs by the project's spelling, which is how
|
|
1571
|
+
// `resolveFiles` looks them up.
|
|
1572
|
+
const relPrefix = spelled.normalize('NFC')
|
|
1559
1573
|
if (relPrefix === '' || relPrefix === '.') {
|
|
1560
1574
|
cache.set(projectDir, all)
|
|
1561
1575
|
const rootOids = new Map<string, string>()
|
|
@@ -1565,14 +1579,16 @@ export async function applyGitEnumeration(
|
|
|
1565
1579
|
}
|
|
1566
1580
|
const prefix = `${relPrefix}/`
|
|
1567
1581
|
const start = lowerBound(prefix)
|
|
1568
|
-
const end = lowerBound(`${
|
|
1582
|
+
const end = lowerBound(`${relPrefix}0`)
|
|
1569
1583
|
const matches: string[] = []
|
|
1570
1584
|
const projOids = new Map<string, string>()
|
|
1585
|
+
const projAbs =
|
|
1586
|
+
relPrefix === spelled ? abs : (rel: string) => path.join(projectDir, rel.slice(prefix.length))
|
|
1571
1587
|
for (let i = start; i < end; i++) {
|
|
1572
1588
|
const rel = sorted[i]!
|
|
1573
1589
|
matches.push(rel.slice(prefix.length))
|
|
1574
1590
|
const oid = trusted.get(rel)
|
|
1575
|
-
if (oid !== undefined) projOids.set(
|
|
1591
|
+
if (oid !== undefined) projOids.set(projAbs(rel), oid)
|
|
1576
1592
|
}
|
|
1577
1593
|
// An empty slice is a directory git did not see, not an empty project:
|
|
1578
1594
|
// a project has at least its package.json, tracked or untracked. A
|
package/src/cache/inputs.ts
CHANGED
|
@@ -28,9 +28,10 @@ import {
|
|
|
28
28
|
isLiteralPattern,
|
|
29
29
|
normalizeGlob,
|
|
30
30
|
outputMatcher,
|
|
31
|
+
relPosix,
|
|
31
32
|
shellArgv,
|
|
32
|
-
splitNegations,
|
|
33
33
|
slashBraceExpansions,
|
|
34
|
+
splitNegations,
|
|
34
35
|
staticPrefix,
|
|
35
36
|
taskGlob,
|
|
36
37
|
UserError,
|
|
@@ -181,11 +182,14 @@ export type ProjectFilesCache = Map<
|
|
|
181
182
|
>
|
|
182
183
|
|
|
183
184
|
export async function resolveInputs(args: ResolveInputsArgs): Promise<ResolvedInputs> {
|
|
185
|
+
const projectRel = path.relative(args.workspaceRoot, args.projectDir).split(path.sep).join('/')
|
|
184
186
|
const { files: projectFiles, listing } = await resolveFiles({
|
|
185
187
|
projectDir: args.projectDir,
|
|
186
188
|
workspaceRoot: args.workspaceRoot,
|
|
187
189
|
files: args.inputs?.files,
|
|
188
190
|
ownOutputs: args.ownOutputs,
|
|
191
|
+
ownWorkspaceOutputs: args.ownWorkspaceOutputs ?? [],
|
|
192
|
+
projectRel,
|
|
189
193
|
nestedProjectDirs: args.nestedProjectDirs,
|
|
190
194
|
...(args.gitFilesCache !== undefined ? { gitFilesCache: args.gitFilesCache } : {}),
|
|
191
195
|
...(args.projectFilesCache !== undefined ? { projectFilesCache: args.projectFilesCache } : {}),
|
|
@@ -198,6 +202,8 @@ export async function resolveInputs(args: ResolveInputsArgs): Promise<ResolvedIn
|
|
|
198
202
|
workspaceRoot: args.workspaceRoot,
|
|
199
203
|
workspaceFiles: wsDecl,
|
|
200
204
|
ownWorkspaceOutputs: args.ownWorkspaceOutputs ?? [],
|
|
205
|
+
ownOutputs: args.ownOutputs,
|
|
206
|
+
projectRel,
|
|
201
207
|
...(args.gitFilesCache !== undefined ? { gitFilesCache: args.gitFilesCache } : {}),
|
|
202
208
|
...(args.workspaceFilesCache !== undefined ? { memo: args.workspaceFilesCache } : {}),
|
|
203
209
|
})
|
|
@@ -257,6 +263,9 @@ function resolveWorkspaceFiles(args: {
|
|
|
257
263
|
workspaceRoot: string
|
|
258
264
|
workspaceFiles: readonly string[]
|
|
259
265
|
ownWorkspaceOutputs: readonly string[]
|
|
266
|
+
/** The task's project-relative `outputs.files`, and its project's root-relative directory. */
|
|
267
|
+
ownOutputs: readonly string[]
|
|
268
|
+
projectRel: string
|
|
260
269
|
gitFilesCache?: GitFilesCache
|
|
261
270
|
memo?: WorkspaceFilesCache
|
|
262
271
|
}): { files: Promise<string[]>; listing: InputListing } | undefined {
|
|
@@ -273,6 +282,11 @@ function resolveWorkspaceFiles(args: {
|
|
|
273
282
|
// A path the task's own outputs take back with `!` is no output, so it
|
|
274
283
|
// stays an input (A-44).
|
|
275
284
|
const ownOutput = outputMatcher(args.ownWorkspaceOutputs)
|
|
285
|
+
// Its project outputs too, which a root-anchored glob can reach as well:
|
|
286
|
+
// left in, the task's own build moved its key, and no run was ever saved.
|
|
287
|
+
let ownProjectOutput: ((rel: string) => boolean) | undefined
|
|
288
|
+
const ownProject = (rel: string): boolean =>
|
|
289
|
+
(ownProjectOutput ??= underProject(args.projectRel, outputMatcher(args.ownOutputs)))(rel)
|
|
276
290
|
const positiveGlobs = asTrees(positive).map(globFor)
|
|
277
291
|
// Workspace-wide partition, keyed by the workspace root. Populated
|
|
278
292
|
// up-front by `populateGitFilesCache(..., workspaceWide: true)` when
|
|
@@ -292,9 +306,15 @@ function resolveWorkspaceFiles(args: {
|
|
|
292
306
|
const memoKey =
|
|
293
307
|
args.memo === undefined
|
|
294
308
|
? undefined
|
|
295
|
-
: JSON.stringify([
|
|
309
|
+
: JSON.stringify([
|
|
310
|
+
positive,
|
|
311
|
+
negative,
|
|
312
|
+
args.ownWorkspaceOutputs,
|
|
313
|
+
args.ownOutputs,
|
|
314
|
+
args.projectRel,
|
|
315
|
+
])
|
|
296
316
|
let isPositive: ((rel: string) => boolean) | undefined
|
|
297
|
-
const excluded = (rel: string): boolean => isExcluded(rel) || ownOutput(rel)
|
|
317
|
+
const excluded = (rel: string): boolean => isExcluded(rel) || ownOutput(rel) || ownProject(rel)
|
|
298
318
|
const listing: InputListing = {
|
|
299
319
|
root: args.workspaceRoot,
|
|
300
320
|
listed: gitFiles,
|
|
@@ -351,6 +371,24 @@ async function resolveWorkspaceFilesOver(
|
|
|
351
371
|
return candidates.filter((abs) => oids?.has(abs) === true || isInputOnDisk(abs)).sort()
|
|
352
372
|
}
|
|
353
373
|
|
|
374
|
+
/** A project-relative matcher asked of root-relative paths: false outside the project. */
|
|
375
|
+
function underProject(
|
|
376
|
+
projectRel: string,
|
|
377
|
+
matches: (rel: string) => boolean,
|
|
378
|
+
): (rel: string) => boolean {
|
|
379
|
+
if (projectRel === '') return matches
|
|
380
|
+
const prefix = `${projectRel}/`
|
|
381
|
+
return (rel) => rel.startsWith(prefix) && matches(rel.slice(prefix.length))
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
/** A root-relative matcher asked of project-relative paths. */
|
|
385
|
+
function inProject(
|
|
386
|
+
projectRel: string,
|
|
387
|
+
matches: (rel: string) => boolean,
|
|
388
|
+
): (rel: string) => boolean {
|
|
389
|
+
return projectRel === '' ? matches : (rel) => matches(`${projectRel}/${rel}`)
|
|
390
|
+
}
|
|
391
|
+
|
|
354
392
|
/** Anything at the path — file, directory, symlink to anything or to nothing. */
|
|
355
393
|
function existsOnDisk(abs: string): boolean {
|
|
356
394
|
try {
|
|
@@ -417,7 +455,11 @@ async function runRuntimeCommand(
|
|
|
417
455
|
owner: RuntimeMemo | undefined,
|
|
418
456
|
): Promise<string> {
|
|
419
457
|
const ambient = process.env['PATH']
|
|
420
|
-
|
|
458
|
+
// As the task's PATH does (exec/env.ts): a dir holding the delimiter
|
|
459
|
+
// splits into an entry relative to the probe's cwd.
|
|
460
|
+
const PATH = [...binDirs.filter((dir) => !dir.includes(path.delimiter)), ambient]
|
|
461
|
+
.filter((entry) => entry)
|
|
462
|
+
.join(path.delimiter)
|
|
421
463
|
let proc
|
|
422
464
|
try {
|
|
423
465
|
// vx's own `sh`, resolved on its PATH before the probe's: Bun.spawn looks
|
|
@@ -425,7 +467,7 @@ async function runRuntimeCommand(
|
|
|
425
467
|
// `node_modules/.bin`, so a dependency's `sh` bin ran every probe (J-69).
|
|
426
468
|
proc = Bun.spawn(shellArgv(command), {
|
|
427
469
|
cwd,
|
|
428
|
-
env: { ...process.env, PATH
|
|
470
|
+
env: { ...process.env, PATH },
|
|
429
471
|
stdin: 'ignore',
|
|
430
472
|
stdout: 'pipe',
|
|
431
473
|
stderr: 'pipe',
|
|
@@ -454,22 +496,49 @@ async function runRuntimeCommand(
|
|
|
454
496
|
let stdout, stderr, exitCode
|
|
455
497
|
try {
|
|
456
498
|
;[stdout, stderr, exitCode] = await Promise.all([
|
|
457
|
-
new Response(proc.stdout).
|
|
458
|
-
new Response(proc.stderr).
|
|
499
|
+
new Response(proc.stdout).bytes(),
|
|
500
|
+
new Response(proc.stderr).bytes(),
|
|
459
501
|
proc.exited,
|
|
460
502
|
])
|
|
461
503
|
} finally {
|
|
462
504
|
liveProbes.delete(proc)
|
|
463
505
|
mine?.delete(proc)
|
|
464
506
|
}
|
|
465
|
-
const output = `${stdout}${stderr}`.trim()
|
|
466
507
|
if (exitCode !== 0) {
|
|
508
|
+
const lossy = new TextDecoder()
|
|
509
|
+
const shown = `${lossy.decode(stdout)}${lossy.decode(stderr)}`.trim()
|
|
467
510
|
throw new UserError(
|
|
468
511
|
`cache.inputs runtime command exited ${exitCode}: ${command} (cwd: ${cwd})` +
|
|
469
|
-
(
|
|
512
|
+
(shown ? `\n${shown}` : ''),
|
|
470
513
|
)
|
|
471
514
|
}
|
|
472
|
-
|
|
515
|
+
let out: string
|
|
516
|
+
let err: string
|
|
517
|
+
try {
|
|
518
|
+
out = FATAL_UTF8.decode(stdout)
|
|
519
|
+
err = FATAL_UTF8.decode(stderr)
|
|
520
|
+
} catch {
|
|
521
|
+
// A lossy decode keys every invalid byte as U+FFFD: Latin-1 é and è
|
|
522
|
+
// folded the same output and replayed each other's build.
|
|
523
|
+
throw new UserError(
|
|
524
|
+
`cache.inputs runtime command printed bytes that are not UTF-8: ${command} (cwd: ${cwd}). ` +
|
|
525
|
+
`Pipe it through a hash or od.`,
|
|
526
|
+
)
|
|
527
|
+
}
|
|
528
|
+
return probeOutput(out, err)
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
/**
|
|
532
|
+
* The probe's output as the key folds it. Plain concatenation keyed stdout
|
|
533
|
+
* `ab` and stdout `a` + stderr `b` alike. A probe with no stderr and no NUL
|
|
534
|
+
* folds its trimmed stdout, as before; any other is framed with a leading
|
|
535
|
+
* NUL and stdout's length, a form no unframed output takes.
|
|
536
|
+
*/
|
|
537
|
+
function probeOutput(stdout: string, stderr: string): string {
|
|
538
|
+
const out = stdout.trim()
|
|
539
|
+
const err = stderr.trim()
|
|
540
|
+
if (err === '' && !out.includes('\0')) return out
|
|
541
|
+
return `\0${out.length}\0${out}${err}`
|
|
473
542
|
}
|
|
474
543
|
|
|
475
544
|
/**
|
|
@@ -680,43 +749,36 @@ export async function cleanOutputs(args: {
|
|
|
680
749
|
projectDir: string
|
|
681
750
|
outputs: string[]
|
|
682
751
|
nestedProjectDirs: string[]
|
|
683
|
-
/**
|
|
684
|
-
* Before a miss: keep the directory each wildcard glob is rooted at
|
|
685
|
-
* (`dist` for `dist/**`). The task writes its matches under it, so a
|
|
686
|
-
* remove there bought only an rmdir and the task's mkdir: 0.4 ms of a
|
|
687
|
-
* 4.9 ms one-file miss (B-49). A restore prunes it, since the entry's
|
|
688
|
-
* shape decides there.
|
|
689
|
-
*/
|
|
690
|
-
keepGlobRoots?: boolean
|
|
691
752
|
}): Promise<string[]> {
|
|
692
753
|
const files = await resolveOutputs(args)
|
|
693
754
|
// `force: true` makes rm tolerate ENOENT (e.g. when two output
|
|
694
755
|
// globs overlap and a sibling already deleted a path mid-iteration).
|
|
695
756
|
// A symlink is unlinked, never followed.
|
|
696
757
|
const removed = await removeAll(files, args.projectDir)
|
|
697
|
-
await pruneEmptiedDirs(
|
|
698
|
-
args.projectDir,
|
|
699
|
-
removed,
|
|
700
|
-
args.keepGlobRoots === true ? globRoots(args.projectDir, args.outputs) : undefined,
|
|
701
|
-
)
|
|
758
|
+
await pruneEmptiedDirs(args.projectDir, removed, pruneScope(args.projectDir, args.outputs))
|
|
702
759
|
// Project-relative posix paths of what was removed — the caller
|
|
703
760
|
// feeds these to GitFilesCache.markOutputsChanged after a restore.
|
|
704
|
-
return files.map((f) =>
|
|
761
|
+
return files.map((f) => relPosix(args.projectDir, f))
|
|
705
762
|
}
|
|
706
763
|
|
|
707
764
|
/**
|
|
708
765
|
* Remove exactly these project-relative paths — the recorded rows of an
|
|
709
766
|
* ADDITIVE task's own artifact (item 588), never a glob: the glob would
|
|
710
767
|
* take the upstream's files the task adds beside. Emptied directories are
|
|
711
|
-
* pruned as `cleanOutputs` prunes them,
|
|
712
|
-
* fills.
|
|
768
|
+
* pruned as `cleanOutputs` prunes them, only inside the declared `outputs`
|
|
769
|
+
* (`pruneScope`), and stop at one the upstream still fills.
|
|
713
770
|
*/
|
|
714
771
|
export async function cleanOutputPaths(args: {
|
|
715
772
|
projectDir: string
|
|
716
773
|
rels: readonly string[]
|
|
774
|
+
outputs: readonly string[]
|
|
717
775
|
}): Promise<void> {
|
|
718
776
|
const files = args.rels.map((r) => path.resolve(args.projectDir, r))
|
|
719
|
-
await pruneEmptiedDirs(
|
|
777
|
+
await pruneEmptiedDirs(
|
|
778
|
+
args.projectDir,
|
|
779
|
+
await removeAll(files, args.projectDir),
|
|
780
|
+
pruneScope(args.projectDir, args.outputs),
|
|
781
|
+
)
|
|
720
782
|
}
|
|
721
783
|
|
|
722
784
|
/**
|
|
@@ -839,7 +901,7 @@ const SYNC_CLEAN_MAX = 128
|
|
|
839
901
|
async function removeAll(all: readonly string[], root: string): Promise<string[]> {
|
|
840
902
|
const files = notThroughLink(all, root)
|
|
841
903
|
const refused = (f: string, err: NodeJS.ErrnoException): UserError => {
|
|
842
|
-
const rel =
|
|
904
|
+
const rel = relPosix(root, f)
|
|
843
905
|
return new UserError(
|
|
844
906
|
`cannot remove declared output ${rel}: ${err.code ?? err.message} — vx clears a task's ` +
|
|
845
907
|
`declared outputs before it runs and before a restore; make the path removable ` +
|
|
@@ -882,7 +944,10 @@ function notThroughLink(files: readonly string[], root: string): string[] {
|
|
|
882
944
|
return null
|
|
883
945
|
}
|
|
884
946
|
}
|
|
885
|
-
|
|
947
|
+
// Resolved only once a directory exists to compare: after a clean pruned
|
|
948
|
+
// the outputs (the common restore) nothing does, and the call was most
|
|
949
|
+
// of an empty clean.
|
|
950
|
+
let realRoot: string | undefined
|
|
886
951
|
const own = new Map<string, boolean>()
|
|
887
952
|
return files.filter((f) => {
|
|
888
953
|
const dir = path.dirname(f)
|
|
@@ -891,7 +956,8 @@ function notThroughLink(files: readonly string[], root: string): string[] {
|
|
|
891
956
|
// A directory already gone has nothing to delete through; its path
|
|
892
957
|
// stays so the prune still reaches the parents it emptied.
|
|
893
958
|
const r = real(dir)
|
|
894
|
-
ok =
|
|
959
|
+
ok =
|
|
960
|
+
r === null || r === path.join((realRoot ??= real(root) ?? root), path.relative(root, dir))
|
|
895
961
|
own.set(dir, ok)
|
|
896
962
|
}
|
|
897
963
|
return ok
|
|
@@ -905,12 +971,13 @@ function notThroughLink(files: readonly string[], root: string): string[] {
|
|
|
905
971
|
* wrote `dist/out/…` and now writes `dist/out`) blocks the rename, and an
|
|
906
972
|
* empty directory is not an output anyone declared. A directory that still
|
|
907
973
|
* holds something — a stray the globs do not cover — stays, and the restore
|
|
908
|
-
* says so if it is in the way.
|
|
974
|
+
* says so if it is in the way. With a `scope`, only a directory inside it
|
|
975
|
+
* goes (`pruneScope`).
|
|
909
976
|
*/
|
|
910
977
|
async function pruneEmptiedDirs(
|
|
911
978
|
root: string,
|
|
912
979
|
removed: readonly string[],
|
|
913
|
-
|
|
980
|
+
scope?: PruneScope,
|
|
914
981
|
): Promise<void> {
|
|
915
982
|
const rootResolved = path.resolve(root)
|
|
916
983
|
// LEVEL ORDER, not a walk-up per directory. A parent is attempted only
|
|
@@ -928,7 +995,9 @@ async function pruneEmptiedDirs(
|
|
|
928
995
|
const parents = new Set<string>()
|
|
929
996
|
const dirs = [...level].filter(
|
|
930
997
|
(dir) =>
|
|
931
|
-
dir !== rootResolved &&
|
|
998
|
+
dir !== rootResolved &&
|
|
999
|
+
dir.startsWith(rootResolved + path.sep) &&
|
|
1000
|
+
(scope === undefined || inScope(dir, scope)),
|
|
932
1001
|
)
|
|
933
1002
|
const gone = (err: NodeJS.ErrnoException): boolean => err.code === 'ENOENT'
|
|
934
1003
|
if (dirs.length <= SYNC_CLEAN_MAX) {
|
|
@@ -951,19 +1020,42 @@ async function pruneEmptiedDirs(
|
|
|
951
1020
|
}
|
|
952
1021
|
}
|
|
953
1022
|
|
|
1023
|
+
interface PruneScope {
|
|
1024
|
+
/** Wildcard globs' roots (`dist` for `dist/**`): prune strictly below. */
|
|
1025
|
+
below: string[]
|
|
1026
|
+
/** Literal outputs (`out` as a tree): prune the path itself and below. */
|
|
1027
|
+
atOrBelow: string[]
|
|
1028
|
+
}
|
|
1029
|
+
|
|
954
1030
|
/**
|
|
955
|
-
*
|
|
956
|
-
*
|
|
957
|
-
*
|
|
1031
|
+
* Where a clean, before a miss or a restore, may prune: only the trees the
|
|
1032
|
+
* task declared. A directory above a glob's root or holding a literal
|
|
1033
|
+
* output (`out` for `out/a.txt`) stays: a sibling task running beside this
|
|
1034
|
+
* one may have just made it and not yet written into it, and pruning it
|
|
1035
|
+
* failed that task "Directory nonexistent". A glob's root stays: the task
|
|
1036
|
+
* writes under it, and a remove there bought only an rmdir and the task's
|
|
1037
|
+
* mkdir (B-49). A literal's own path goes: the entry or the task may need
|
|
1038
|
+
* a file there. A glob rooted at the project prunes nothing.
|
|
958
1039
|
*/
|
|
959
|
-
function
|
|
960
|
-
const
|
|
1040
|
+
function pruneScope(base: string, outputs: readonly string[]): PruneScope {
|
|
1041
|
+
const scope: PruneScope = { below: [], atOrBelow: [] }
|
|
961
1042
|
for (const g of outputs) {
|
|
962
|
-
if (g.startsWith('!')
|
|
1043
|
+
if (g.startsWith('!')) continue
|
|
1044
|
+
if (isLiteralPattern(g)) {
|
|
1045
|
+
scope.atOrBelow.push(path.resolve(base, stripTrailingSlash(normalizeGlob(g))))
|
|
1046
|
+
continue
|
|
1047
|
+
}
|
|
963
1048
|
const prefix = staticPrefix(g)
|
|
964
|
-
if (prefix !== '.')
|
|
1049
|
+
if (prefix !== '.') scope.below.push(path.resolve(base, prefix))
|
|
965
1050
|
}
|
|
966
|
-
return
|
|
1051
|
+
return scope
|
|
1052
|
+
}
|
|
1053
|
+
|
|
1054
|
+
function inScope(dir: string, scope: PruneScope): boolean {
|
|
1055
|
+
return (
|
|
1056
|
+
scope.below.some((r) => dir.startsWith(r + path.sep)) ||
|
|
1057
|
+
scope.atOrBelow.some((r) => dir === r || dir.startsWith(r + path.sep))
|
|
1058
|
+
)
|
|
967
1059
|
}
|
|
968
1060
|
|
|
969
1061
|
/**
|
|
@@ -1001,8 +1093,12 @@ export async function cleanWorkspaceOutputs(args: {
|
|
|
1001
1093
|
outputs: string[]
|
|
1002
1094
|
}): Promise<string[]> {
|
|
1003
1095
|
const files = await resolveWorkspaceOutputs(args)
|
|
1004
|
-
await pruneEmptiedDirs(
|
|
1005
|
-
|
|
1096
|
+
await pruneEmptiedDirs(
|
|
1097
|
+
args.workspaceRoot,
|
|
1098
|
+
await removeAll(files, args.workspaceRoot),
|
|
1099
|
+
pruneScope(args.workspaceRoot, args.outputs),
|
|
1100
|
+
)
|
|
1101
|
+
return files.map((f) => relPosix(args.workspaceRoot, f))
|
|
1006
1102
|
}
|
|
1007
1103
|
|
|
1008
1104
|
function stripTrailingSlash(p: string): string {
|
|
@@ -1101,6 +1197,9 @@ interface ResolveFilesArgs {
|
|
|
1101
1197
|
workspaceRoot: string
|
|
1102
1198
|
files: string[] | undefined
|
|
1103
1199
|
ownOutputs: string[]
|
|
1200
|
+
/** Root-relative `outputs.workspaceFiles`, and the project's root-relative directory. */
|
|
1201
|
+
ownWorkspaceOutputs: readonly string[]
|
|
1202
|
+
projectRel: string
|
|
1104
1203
|
nestedProjectDirs: string[]
|
|
1105
1204
|
gitFilesCache?: GitFilesCache
|
|
1106
1205
|
projectFilesCache?: ProjectFilesCache
|
|
@@ -1136,7 +1235,7 @@ function refuseUndecodable(
|
|
|
1136
1235
|
if (undecodable === undefined || undecodable.size === 0) return
|
|
1137
1236
|
const bad = candidates.filter((abs) => undecodable.has(abs) && undecodableOnDisk(abs))
|
|
1138
1237
|
if (bad.length === 0) return
|
|
1139
|
-
const names = bad.map((abs) => JSON.stringify(
|
|
1238
|
+
const names = bad.map((abs) => JSON.stringify(relPosix(root, abs)))
|
|
1140
1239
|
throw new UserError(
|
|
1141
1240
|
`cache.inputs.${field} matched ${names.join(', ')} in ${root}: the name is not valid UTF-8 ` +
|
|
1142
1241
|
`(shown with \ufffd), and vx cannot read a file by it. Rename it, or exclude it with a ` +
|
|
@@ -1292,6 +1391,8 @@ async function resolveFiles(
|
|
|
1292
1391
|
const { positive, negative, isExcluded, ownOutput, positiveGlobs, isPositive } = plan
|
|
1293
1392
|
|
|
1294
1393
|
const nested = inNestedProject(args.projectDir, args.nestedProjectDirs)
|
|
1394
|
+
// Its root-anchored outputs that land in the project, as `files` sees them.
|
|
1395
|
+
const ownWsOutput = inProject(args.projectRel, outputMatcher(args.ownWorkspaceOutputs))
|
|
1295
1396
|
|
|
1296
1397
|
// Defer to git for the file set (Turbo / Nx parity). Nested .gitignore
|
|
1297
1398
|
// files, .git/info/exclude, and global excludes all participate
|
|
@@ -1303,7 +1404,7 @@ async function resolveFiles(
|
|
|
1303
1404
|
// duration of one orchestrator run.
|
|
1304
1405
|
// Everything below the snapshot that decides the result: the project, what
|
|
1305
1406
|
// it declares, what it excludes as its own outputs, and the boundaries.
|
|
1306
|
-
const memoKey = `${args.projectDir}\0${positive.join('\u0001')}\0${negative.join('\u0001')}\0${args.ownOutputs.join('\u0001')}\0${args.nestedProjectDirs.join('\u0001')}`
|
|
1407
|
+
const memoKey = `${args.projectDir}\0${positive.join('\u0001')}\0${negative.join('\u0001')}\0${args.ownOutputs.join('\u0001')}\0${args.ownWorkspaceOutputs.join('\u0001')}\0${args.nestedProjectDirs.join('\u0001')}`
|
|
1307
1408
|
let gitFiles = args.gitFilesCache?.snapshotFor(args.projectDir, positiveGlobs)
|
|
1308
1409
|
let undecodable = args.gitFilesCache?.undecodableNames
|
|
1309
1410
|
if (gitFiles !== undefined) {
|
|
@@ -1314,7 +1415,7 @@ async function resolveFiles(
|
|
|
1314
1415
|
if (memo !== undefined && memo.snapshot === gitFiles) {
|
|
1315
1416
|
return {
|
|
1316
1417
|
files: [...memo.result],
|
|
1317
|
-
listing: listingFor(args.projectDir, gitFiles, plan, nested),
|
|
1418
|
+
listing: listingFor(args.projectDir, gitFiles, plan, nested, ownWsOutput),
|
|
1318
1419
|
}
|
|
1319
1420
|
}
|
|
1320
1421
|
}
|
|
@@ -1365,7 +1466,7 @@ async function resolveFiles(
|
|
|
1365
1466
|
input = isPositive(rel) && !isExcluded(rel) && !ownOutput(rel)
|
|
1366
1467
|
verdicts.set(rel, input)
|
|
1367
1468
|
}
|
|
1368
|
-
if (!input || nested(rel)) continue
|
|
1469
|
+
if (!input || nested(rel) || ownWsOutput(rel)) continue
|
|
1369
1470
|
candidates.push(base === undefined ? path.resolve(args.projectDir, rel) : base + rel)
|
|
1370
1471
|
}
|
|
1371
1472
|
const unmatchedLiterals = unanswered(plan.literals, gitFiles)
|
|
@@ -1392,7 +1493,10 @@ async function resolveFiles(
|
|
|
1392
1493
|
// Stored only on the way out: a declaration whose literal named an
|
|
1393
1494
|
// invisible file threw above, and every task sharing it must throw too.
|
|
1394
1495
|
args.projectFilesCache?.set(memoKey, { snapshot: gitFiles, result: resolved })
|
|
1395
|
-
return {
|
|
1496
|
+
return {
|
|
1497
|
+
files: [...resolved],
|
|
1498
|
+
listing: listingFor(args.projectDir, gitFiles, plan, nested, ownWsOutput),
|
|
1499
|
+
}
|
|
1396
1500
|
}
|
|
1397
1501
|
|
|
1398
1502
|
function listingFor(
|
|
@@ -1400,11 +1504,12 @@ function listingFor(
|
|
|
1400
1504
|
listed: readonly string[],
|
|
1401
1505
|
plan: FilesPlan,
|
|
1402
1506
|
nested: (rel: string) => boolean,
|
|
1507
|
+
ownWsOutput: (rel: string) => boolean,
|
|
1403
1508
|
): InputListing {
|
|
1404
1509
|
return {
|
|
1405
1510
|
root,
|
|
1406
1511
|
listed,
|
|
1407
|
-
isInput: (rel) => inPlan(plan, rel),
|
|
1512
|
+
isInput: (rel) => inPlan(plan, rel) && !ownWsOutput(rel),
|
|
1408
1513
|
nested,
|
|
1409
1514
|
prefixes: plan.prefixes,
|
|
1410
1515
|
literals: plan.literals,
|
|
@@ -1448,7 +1553,7 @@ function undecodableOnDisk(abs: string): boolean {
|
|
|
1448
1553
|
} catch {
|
|
1449
1554
|
return false
|
|
1450
1555
|
}
|
|
1451
|
-
const lossy = new TextDecoder()
|
|
1556
|
+
const lossy = new TextDecoder('utf-8', { ignoreBOM: true })
|
|
1452
1557
|
for (const name of raw) {
|
|
1453
1558
|
if (lossy.decode(name) !== next) continue
|
|
1454
1559
|
try {
|
|
@@ -1589,7 +1694,7 @@ function addedTo(
|
|
|
1589
1694
|
return undefined
|
|
1590
1695
|
}
|
|
1591
1696
|
|
|
1592
|
-
const FATAL_UTF8 = new TextDecoder('utf-8', { fatal: true })
|
|
1697
|
+
const FATAL_UTF8 = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true })
|
|
1593
1698
|
|
|
1594
1699
|
/**
|
|
1595
1700
|
* Union of the OUTPUT files matching any positive pattern in `cwd`, minus
|
|
@@ -1676,9 +1781,7 @@ function globFor(pattern: string): Bun.Glob {
|
|
|
1676
1781
|
*/
|
|
1677
1782
|
function inNestedProject(projectDir: string, nestedDirs: string[]): (rel: string) => boolean {
|
|
1678
1783
|
if (nestedDirs.length === 0) return () => false
|
|
1679
|
-
const dirs = new Set(
|
|
1680
|
-
nestedDirs.map((d) => path.relative(projectDir, d).split(path.sep).join('/')),
|
|
1681
|
-
)
|
|
1784
|
+
const dirs = new Set(nestedDirs.map((d) => relPosix(projectDir, d)))
|
|
1682
1785
|
return (rel) => {
|
|
1683
1786
|
for (let i = rel.indexOf('/'); i !== -1; i = rel.indexOf('/', i + 1)) {
|
|
1684
1787
|
if (dirs.has(rel.slice(0, i))) return true
|
|
@@ -185,7 +185,27 @@ class ScopedRemote implements RemoteCacheLayer {
|
|
|
185
185
|
}
|
|
186
186
|
|
|
187
187
|
async has(hash: string): Promise<boolean> {
|
|
188
|
-
return
|
|
188
|
+
return this.fallThrough(
|
|
189
|
+
() => this.remote.has(this.scoped(hash)),
|
|
190
|
+
() => this.remote.has(hash),
|
|
191
|
+
)
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* The scope's answer, else the trusted one. A scope lookup that fails
|
|
196
|
+
* still asks the trusted key, so a hit there is not lost; its error is
|
|
197
|
+
* rethrown (and warned) when the trusted key fails or has nothing too.
|
|
198
|
+
*/
|
|
199
|
+
private async fallThrough<T>(own: () => Promise<T>, trusted: () => Promise<T>): Promise<T> {
|
|
200
|
+
let found: T
|
|
201
|
+
try {
|
|
202
|
+
found = await own()
|
|
203
|
+
} catch (err) {
|
|
204
|
+
const fallback = await trusted().catch(() => undefined)
|
|
205
|
+
if (!fallback) throw err
|
|
206
|
+
return fallback
|
|
207
|
+
}
|
|
208
|
+
return found || trusted()
|
|
189
209
|
}
|
|
190
210
|
|
|
191
211
|
private async scopedHasMany(hashes: readonly string[]): Promise<Set<string> | null> {
|
|
@@ -207,7 +227,10 @@ class ScopedRemote implements RemoteCacheLayer {
|
|
|
207
227
|
hash: string,
|
|
208
228
|
): Promise<{ body: Blob | Response; durationMs: number | undefined } | null> {
|
|
209
229
|
if (this.trustedOnly.has(hash)) return this.remote.get(hash)
|
|
210
|
-
return
|
|
230
|
+
return this.fallThrough(
|
|
231
|
+
() => this.remote.get(this.scoped(hash)),
|
|
232
|
+
() => this.remote.get(hash),
|
|
233
|
+
)
|
|
211
234
|
}
|
|
212
235
|
|
|
213
236
|
put(hash: string, body: Blob, meta: { durationMs: number }): Promise<void> {
|
package/src/cache/tar-stream.ts
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
// short entry. Non-regular entries are reported with their type and their
|
|
12
12
|
// bodies skipped; the caller decides what to materialise (nothing but `0`).
|
|
13
13
|
|
|
14
|
-
|
|
14
|
+
interface TarEntry {
|
|
15
15
|
name: string
|
|
16
16
|
size: number
|
|
17
17
|
/** POSIX typeflag: '0' regular, '5' directory, '2' symlink, … */
|
|
@@ -38,7 +38,9 @@ const BLOCK = 512
|
|
|
38
38
|
* numbers; a MiB is far past any writer's.
|
|
39
39
|
*/
|
|
40
40
|
const MAX_EXTENDED_HEADER = 1024 * 1024
|
|
41
|
-
|
|
41
|
+
// `ignoreBOM`: a default decoder drops a leading U+FEFF, and a name or
|
|
42
|
+
// prefix field that began with one read back without it.
|
|
43
|
+
const decoder = new TextDecoder('utf-8', { ignoreBOM: true })
|
|
42
44
|
|
|
43
45
|
function field(h: Uint8Array, off: number, len: number): string {
|
|
44
46
|
let end = off
|