@vzn/vx 0.0.596 → 0.0.597
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/launcher.cjs +7 -3
- package/package.json +5 -5
- package/src/cache/archive.ts +35 -6
- package/src/cache/cache.ts +31 -15
- package/src/cache/git-inputs.ts +23 -0
- package/src/cache/index.ts +2 -0
- package/src/cache/inputs.ts +87 -32
- package/src/cache/key-fold.ts +1 -1
- package/src/cache/layer.ts +2 -1
- package/src/cli/cache.ts +2 -2
- package/src/cli/foreign-flags.ts +56 -2
- package/src/cli/info.ts +1 -1
- package/src/cli/init.ts +2 -2
- package/src/cli/last.ts +6 -3
- package/src/cli/lock.ts +5 -3
- package/src/cli/plan-format.ts +4 -4
- package/src/cli/run.ts +42 -15
- package/src/cli/select.ts +6 -3
- package/src/cli/show.ts +8 -2
- package/src/cli/upgrade.ts +1 -1
- package/src/cli/watch-fs.ts +11 -3
- package/src/cli/watch.ts +14 -7
- package/src/cli/workspace-config.ts +3 -1
- package/src/config.ts +21 -0
- package/src/exec/executor.ts +1 -1
- package/src/exec/kill-tree.ts +89 -18
- package/src/exec/runner.ts +74 -36
- package/src/exec/sandbox-paths.ts +11 -0
- package/src/exec/sandbox-runtime.ts +168 -49
- package/src/exec/sandbox-violations.ts +59 -6
- package/src/graph/scheduler.ts +2 -2
- package/src/graph/task-graph.ts +145 -33
- package/src/orchestrator/affected-tasks.ts +6 -2
- package/src/orchestrator/deferred-outputs.ts +3 -2
- package/src/orchestrator/doctor.ts +2 -2
- package/src/orchestrator/excluded-keys.ts +2 -13
- package/src/orchestrator/execute-task.ts +49 -26
- package/src/orchestrator/failure-mode.ts +38 -29
- package/src/orchestrator/framed-output.ts +5 -5
- package/src/orchestrator/index.ts +0 -1
- package/src/orchestrator/keyed-projects.ts +6 -3
- package/src/orchestrator/logger.ts +10 -3
- package/src/orchestrator/miss-save.ts +7 -11
- package/src/orchestrator/plan.ts +4 -20
- package/src/orchestrator/run-artifacts.ts +2 -2
- package/src/orchestrator/run-lock.ts +25 -24
- package/src/orchestrator/run.ts +2 -1
- package/src/orchestrator/sandbox-request.ts +31 -9
- package/src/orchestrator/signals.ts +22 -4
- package/src/orchestrator/stable-keys.ts +3 -11
- package/src/orchestrator/summary.ts +5 -9
- package/src/orchestrator/task-hash.ts +13 -3
- package/src/orchestrator/upstream.ts +11 -0
- package/src/util/hangup.ts +42 -0
- package/src/util/index.ts +2 -0
- package/src/util/paths.ts +1 -1
- package/src/workspace/affected.ts +79 -42
- package/src/workspace/config-eval.ts +27 -6
- package/src/workspace/config-imports.ts +8 -55
- package/src/workspace/config-schema.ts +30 -6
- package/src/workspace/lockfile.ts +2 -1
- package/src/workspace/migrate-scripts.ts +1 -1
- package/src/workspace/project-loader.ts +15 -2
- package/src/workspace/workspace.ts +36 -14
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// task's owner can act on — inside the project, minus the loopback denial
|
|
5
5
|
// no grant can avoid, minus what the task chose to ignore.
|
|
6
6
|
|
|
7
|
-
import { lstatSync, statSync } from 'node:fs'
|
|
7
|
+
import { existsSync, lstatSync, statSync } from 'node:fs'
|
|
8
8
|
import path from 'node:path'
|
|
9
9
|
import { absolutize, atOrUnder, isUnderAny, localBindingOn, toRealPath } from './sandbox-paths.js'
|
|
10
10
|
import { bindableReads, bindableWrites } from './sandbox-binds.js'
|
|
@@ -364,6 +364,10 @@ export async function parseStraceViolations(
|
|
|
364
364
|
// workspace, never reported.
|
|
365
365
|
const pendingWrites = (args.config.pendingWrites ?? []).map((g) => new Bun.Glob(g))
|
|
366
366
|
const denyAnchors = baselines.denyRead.map((p) => toRealPath(absolutize(p)))
|
|
367
|
+
const mounted = new Set([
|
|
368
|
+
...allowAbs,
|
|
369
|
+
...bindableWrites(args.config.allowWrite).map((p) => toRealPath(absolutize(p))),
|
|
370
|
+
])
|
|
367
371
|
// A read under a widened write grant's directory is never refused, so it
|
|
368
372
|
// is reported when it succeeds: an entry that was there at the start and
|
|
369
373
|
// no grant covers is an input the key never saw, and so is the
|
|
@@ -388,12 +392,14 @@ export async function parseStraceViolations(
|
|
|
388
392
|
baselines.cwd,
|
|
389
393
|
widened.size > 0,
|
|
390
394
|
)) {
|
|
391
|
-
const
|
|
395
|
+
const lexical = absolutize(rawPath, dir ?? baselines.cwd)
|
|
396
|
+
const abs = toRealPath(lexical)
|
|
392
397
|
// Only report paths under the workspace-root deny anchor — system
|
|
393
398
|
// libs / /proc / /sys / etc. probes are not interesting violations.
|
|
394
399
|
if (!denyAnchors.some((root) => atOrUnder(abs, root))) continue
|
|
395
400
|
// Skip paths the user explicitly allowed (and their descendants).
|
|
396
|
-
if (isUnderAny(abs, allowAbs)
|
|
401
|
+
if (isUnderAny(abs, allowAbs) && !hiddenByLink(lexical, abs, mounted, denyAnchors)) continue
|
|
402
|
+
if (underGlob(abs, pendingWrites)) continue
|
|
397
403
|
if (read === true && !undeclared(abs)) continue
|
|
398
404
|
// A hidden path answers ENOENT, as a missing one does: a probe
|
|
399
405
|
// (`./gen.sh || …`, `test -x`, a PATH walk) is a refusal only where
|
|
@@ -427,6 +433,39 @@ export async function parseStraceViolations(
|
|
|
427
433
|
return out
|
|
428
434
|
}
|
|
429
435
|
|
|
436
|
+
/**
|
|
437
|
+
* A granted file the task could not open because a link on the path it
|
|
438
|
+
* opened is not in the sandbox. A grant is mounted at its real path and
|
|
439
|
+
* bwrap mounts no link, so `read: ['config.json']` over `config.json ->
|
|
440
|
+
* conf/real.json` bound the target and left the name the task opens out;
|
|
441
|
+
* judged by its real path, the ENOENT was granted and dropped, and a tool
|
|
442
|
+
* that fell back on the missing file was cached green. The link is there
|
|
443
|
+
* when the directory holding it is mounted; a target missing on the host
|
|
444
|
+
* is missing outside the sandbox too, and outside the deny anchor the
|
|
445
|
+
* host is mounted, links and all.
|
|
446
|
+
*/
|
|
447
|
+
function hiddenByLink(
|
|
448
|
+
lexical: string,
|
|
449
|
+
real: string,
|
|
450
|
+
mounted: Set<string>,
|
|
451
|
+
anchors: readonly string[],
|
|
452
|
+
): boolean {
|
|
453
|
+
if (lexical === real || !existsSync(real)) return false
|
|
454
|
+
let at = lexical
|
|
455
|
+
const links: string[] = []
|
|
456
|
+
for (let up = path.dirname(at); up !== at; at = up, up = path.dirname(at)) {
|
|
457
|
+
try {
|
|
458
|
+
if (lstatSync(at).isSymbolicLink()) links.push(at)
|
|
459
|
+
} catch {
|
|
460
|
+
// Not on the host by this spelling: no link to judge here.
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
return links.some((l) => {
|
|
464
|
+
const holder = toRealPath(path.dirname(l))
|
|
465
|
+
return anchors.some((a) => atOrUnder(holder, a)) && !isUnderAny(holder, mounted)
|
|
466
|
+
})
|
|
467
|
+
}
|
|
468
|
+
|
|
430
469
|
/**
|
|
431
470
|
* Does a violation line match something the task said to ignore?
|
|
432
471
|
*
|
|
@@ -444,9 +483,11 @@ function matchesIgnore(
|
|
|
444
483
|
ignore: NonNullable<ResolvedSandboxConfig['ignore']>,
|
|
445
484
|
): boolean {
|
|
446
485
|
if (v.target === undefined || v.ignorable === undefined) return false
|
|
486
|
+
const proxy = PROXY_DENY_RE.test(v.line)
|
|
447
487
|
for (const which of v.ignorable) {
|
|
448
|
-
const
|
|
449
|
-
if (
|
|
488
|
+
const listed = ignore[which]
|
|
489
|
+
if (listed === undefined) continue
|
|
490
|
+
const patterns = proxy ? listed.map(hostKey) : listed
|
|
450
491
|
const subject = which === 'read' || which === 'write' ? (v.path ?? v.target) : v.target
|
|
451
492
|
if (patterns.some((pat) => pat === subject || new Bun.Glob(pat).match(subject))) return true
|
|
452
493
|
}
|
|
@@ -460,7 +501,7 @@ function matchesIgnore(
|
|
|
460
501
|
*/
|
|
461
502
|
function describeMacViolation(line: string): Partial<SandboxViolation> {
|
|
462
503
|
const proxy = PROXY_DENY_RE.exec(line)
|
|
463
|
-
if (proxy !== null) return { target: proxy[1]
|
|
504
|
+
if (proxy !== null) return { target: hostKey(proxy[1]!), ignorable: ['network'] }
|
|
464
505
|
const m = /deny\(\d+\)\s+(\S+)\s+(.+?)\s*$/.exec(line)
|
|
465
506
|
if (m === null) return {}
|
|
466
507
|
const [op, target] = [m[1]!, m[2]!]
|
|
@@ -677,6 +718,18 @@ function filterIgnored(
|
|
|
677
718
|
*/
|
|
678
719
|
const PROXY_DENY_RE = /^deny network-outbound (\S+) \([^)]*\)$/
|
|
679
720
|
|
|
721
|
+
/**
|
|
722
|
+
* `<host>[:<port>]` as the proxy matches it: case-blind, no trailing dot.
|
|
723
|
+
* The record keeps the client's spelling (curl sends `EXAMPLE.Com`, curl
|
|
724
|
+
* and Bun keep `example.com.`), so an exact match against the ignore list
|
|
725
|
+
* left a refusal the task named in its report.
|
|
726
|
+
*/
|
|
727
|
+
function hostKey(target: string): string {
|
|
728
|
+
const colon = target.lastIndexOf(':')
|
|
729
|
+
const host = colon === -1 ? target : target.slice(0, colon)
|
|
730
|
+
return host.toLowerCase().replace(/\.$/, '') + (colon === -1 ? '' : target.slice(colon))
|
|
731
|
+
}
|
|
732
|
+
|
|
680
733
|
/**
|
|
681
734
|
* Linux: the connections the proxy refused, from the store records. Only
|
|
682
735
|
* the write observer's records were read there, so a task denied a host
|
package/src/graph/scheduler.ts
CHANGED
|
@@ -179,8 +179,8 @@ export interface TaskOutcome {
|
|
|
179
179
|
/** The attempts that failed and were run again, in order: when each ended (epoch ms) and how. */
|
|
180
180
|
failedAttempts?: readonly { endedAt: number; exitCode: number; timedOut?: true }[]
|
|
181
181
|
/**
|
|
182
|
-
* Set when this run proved the task flaky: its key
|
|
183
|
-
*
|
|
182
|
+
* Set when this run proved the task flaky: its key failed after it had
|
|
183
|
+
* passed (this run's included) or it needed a retry. Its row carries it.
|
|
184
184
|
*/
|
|
185
185
|
flaky?: { passes: number; failures: number }
|
|
186
186
|
/**
|
package/src/graph/task-graph.ts
CHANGED
|
@@ -334,7 +334,8 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
|
|
|
334
334
|
|
|
335
335
|
function add(node: TaskNode): void {
|
|
336
336
|
nodes.set(node.id, node)
|
|
337
|
-
if ((node.config
|
|
337
|
+
if (isKeyedGroup(node.config) && onCycle(node.projectName)) deferred.push(node)
|
|
338
|
+
else if ((node.config.dependsOn?.length ?? 0) > 0) {
|
|
338
339
|
stack.push({ node, entry: 0, added: false, pending: null, next: 0 })
|
|
339
340
|
}
|
|
340
341
|
}
|
|
@@ -444,10 +445,43 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
|
|
|
444
445
|
}
|
|
445
446
|
return component.get(start)!
|
|
446
447
|
}
|
|
448
|
+
// Default builds on a package cycle, expanded once the rest is built.
|
|
449
|
+
const deferred: TaskNode[] = []
|
|
447
450
|
const onCycle = (name: string): boolean => cyclic.has(componentOf(name))
|
|
448
451
|
const sameCycle = (a: string, b: string): boolean =>
|
|
449
452
|
onCycle(a) && componentOf(a) === componentOf(b)
|
|
450
453
|
|
|
454
|
+
// `^name` from `start`: edges to the nearest holders of `name` among the
|
|
455
|
+
// packages below it, passing through the ones `visited` holds and the
|
|
456
|
+
// ones that declare no `name`, and on past a default build on a cycle.
|
|
457
|
+
// True when it found a holder.
|
|
458
|
+
function walkHolders(frame: Frame, name: string, visited: Set<string>, start: string): boolean {
|
|
459
|
+
const frontier = [...packageGraph.directDeps(start)]
|
|
460
|
+
let held = false
|
|
461
|
+
while (frontier.length > 0) {
|
|
462
|
+
const target = frontier.pop()!
|
|
463
|
+
if (visited.has(target)) continue
|
|
464
|
+
visited.add(target)
|
|
465
|
+
if (visit(frame, target, name, false)) {
|
|
466
|
+
held = true
|
|
467
|
+
const task = declaredTask(projects.get(target)!.config, name)!
|
|
468
|
+
if (isKeyedGroup(task) && onCycle(target)) {
|
|
469
|
+
frontier.push(...packageGraph.directDeps(target))
|
|
470
|
+
}
|
|
471
|
+
} else frontier.push(...packageGraph.directDeps(target))
|
|
472
|
+
}
|
|
473
|
+
return held
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
// An edge by name (`build`, `pkg#build`) to a default build on a package
|
|
477
|
+
// cycle goes on past it as a `^build` walk does: that build folds nothing
|
|
478
|
+
// of the builds on its cycle, so `a#test` on `build` with `a` ↔ `b` both
|
|
479
|
+
// on the default build folded none of `b` and hit after `b` changed.
|
|
480
|
+
function goPast(frame: Frame, project: string, name: string, visited: Set<string>): void {
|
|
481
|
+
const task = declaredTask(projects.get(project)!.config, name)!
|
|
482
|
+
if (isKeyedGroup(task) && onCycle(project)) walkHolders(frame, name, visited, project)
|
|
483
|
+
}
|
|
484
|
+
|
|
451
485
|
// Resolves one `dependsOn` entry of `frame`'s task into its edges.
|
|
452
486
|
function resolveEntry(frame: Frame, raw: string): void {
|
|
453
487
|
const { node } = frame
|
|
@@ -497,6 +531,7 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
|
|
|
497
531
|
`Task ${id} depends on ${taskId(projectName, spec.task)} but no such task is declared`,
|
|
498
532
|
)
|
|
499
533
|
}
|
|
534
|
+
goPast(frame, projectName, spec.task, new Set([projectName]))
|
|
500
535
|
}
|
|
501
536
|
} else if (spec.kind === 'deps') {
|
|
502
537
|
// Nearest-holder frontier (Turbo/Nx direct-deps parity +
|
|
@@ -521,45 +556,35 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
|
|
|
521
556
|
// depend on itself.
|
|
522
557
|
//
|
|
523
558
|
// The default `build` sits on every project, so on a package cycle it
|
|
524
|
-
// would make a task cycle no config declares.
|
|
525
|
-
//
|
|
526
|
-
// and
|
|
527
|
-
//
|
|
528
|
-
const re = isTaskPattern(spec.task) ? compileTaskPattern(spec.task) : null
|
|
529
|
-
const keyed = isKeyedGroup(node.config)
|
|
559
|
+
// would make a task cycle no config declares. One on a cycle walks
|
|
560
|
+
// last (`resolveDeferred`), and a walk that meets it on a cycle takes
|
|
561
|
+
// the edge and goes on past it, so the builds it could not depend on
|
|
562
|
+
// still come first.
|
|
530
563
|
const visited = new Set<string>([projectName])
|
|
564
|
+
if (!isTaskPattern(spec.task)) {
|
|
565
|
+
const held = walkHolders(frame, spec.task, visited, projectName)
|
|
566
|
+
if (!held && !declaredAnywhere(projectName, spec.task)) {
|
|
567
|
+
if (undeclaredDeps === undefined) throw undeclaredDepsError(id, spec.task)
|
|
568
|
+
undeclaredDeps(id, spec.task)
|
|
569
|
+
}
|
|
570
|
+
return
|
|
571
|
+
}
|
|
572
|
+
// A pattern that matches nothing stays legal, as `build.*` does.
|
|
573
|
+
const re = compileTaskPattern(spec.task)
|
|
531
574
|
const frontier = [...packageGraph.directDeps(projectName)]
|
|
532
|
-
let held = false
|
|
533
575
|
while (frontier.length > 0) {
|
|
534
576
|
const target = frontier.pop()!
|
|
535
577
|
if (visited.has(target)) continue
|
|
536
578
|
visited.add(target)
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
const task = declaredTask(projects.get(target)!.config, spec.task)!
|
|
543
|
-
if (isKeyedGroup(task) && onCycle(target)) {
|
|
544
|
-
frontier.push(...packageGraph.directDeps(target))
|
|
545
|
-
}
|
|
546
|
-
} else frontier.push(...packageGraph.directDeps(target))
|
|
579
|
+
const names = Object.keys(projects.get(target)?.config.tasks ?? {}).filter((n) =>
|
|
580
|
+
re.test(n),
|
|
581
|
+
)
|
|
582
|
+
if (names.length > 0) {
|
|
583
|
+
for (const name of names) visit(frame, target, name, false)
|
|
547
584
|
} else {
|
|
548
|
-
|
|
549
|
-
re.test(n),
|
|
550
|
-
)
|
|
551
|
-
if (names.length > 0) {
|
|
552
|
-
for (const name of names) visit(frame, target, name, false)
|
|
553
|
-
} else {
|
|
554
|
-
frontier.push(...packageGraph.directDeps(target))
|
|
555
|
-
}
|
|
585
|
+
frontier.push(...packageGraph.directDeps(target))
|
|
556
586
|
}
|
|
557
587
|
}
|
|
558
|
-
// A pattern that matches nothing stays legal, as `build.*` does.
|
|
559
|
-
if (re === null && !held && !declaredAnywhere(projectName, spec.task)) {
|
|
560
|
-
if (undeclaredDeps === undefined) throw undeclaredDepsError(id, spec.task)
|
|
561
|
-
undeclaredDeps(id, spec.task)
|
|
562
|
-
}
|
|
563
588
|
} else {
|
|
564
589
|
// Cross-project edge: pkg#task. Missing target is a hard error
|
|
565
590
|
// because the user named the package + task explicitly.
|
|
@@ -573,11 +598,11 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
|
|
|
573
598
|
`Task ${id} depends on ${taskId(spec.project, spec.task)} but no such project or task is declared`,
|
|
574
599
|
)
|
|
575
600
|
}
|
|
601
|
+
goPast(frame, spec.project, spec.task, new Set([projectName, spec.project]))
|
|
576
602
|
}
|
|
577
603
|
}
|
|
578
604
|
|
|
579
|
-
|
|
580
|
-
visit(null, project, task, true)
|
|
605
|
+
const drain = (): void => {
|
|
581
606
|
while (stack.length > 0) {
|
|
582
607
|
const frame = stack[stack.length - 1]!
|
|
583
608
|
const { node, pending } = frame
|
|
@@ -604,10 +629,97 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
|
|
|
604
629
|
}
|
|
605
630
|
}
|
|
606
631
|
|
|
632
|
+
for (const { project, task } of requested) {
|
|
633
|
+
visit(null, project, task, true)
|
|
634
|
+
drain()
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
// A default build on a package cycle: its `^build` walk, run once the
|
|
638
|
+
// graph holds everything else. A build on its cycle that reaches it is
|
|
639
|
+
// passed through (an edge to it would close a task cycle no config
|
|
640
|
+
// declares); one that does not is a holder like any other, so the default
|
|
641
|
+
// build waits for it and folds its key. Passed through unasked, `p#test`
|
|
642
|
+
// on `p#build` ran beside a `t#build` on no `^build` and hit after `t`
|
|
643
|
+
// changed. A build no other edge brought is expanded to ask, and pruned
|
|
644
|
+
// again when it reaches back. Each walk sees the edges the ones before it
|
|
645
|
+
// took, so no edge closes a cycle.
|
|
646
|
+
let expanded = false
|
|
647
|
+
const resolveDeferred = (x: TaskNode): void => {
|
|
648
|
+
// The default build's one entry is `^` and its own name.
|
|
649
|
+
const name = x.taskName
|
|
650
|
+
const frame: Frame = { node: x, entry: 0, added: false, pending: null, next: 0 }
|
|
651
|
+
let reaching: Set<string> | null = null
|
|
652
|
+
const visited = new Set<string>([x.projectName])
|
|
653
|
+
const frontier = [...packageGraph.directDeps(x.projectName)]
|
|
654
|
+
while (frontier.length > 0) {
|
|
655
|
+
const target = frontier.pop()!
|
|
656
|
+
if (visited.has(target)) continue
|
|
657
|
+
visited.add(target)
|
|
658
|
+
const task = declaredTask(projects.get(target)?.config, name)
|
|
659
|
+
if (task === undefined) {
|
|
660
|
+
frontier.push(...packageGraph.directDeps(target))
|
|
661
|
+
continue
|
|
662
|
+
}
|
|
663
|
+
const to = taskId(target, name)
|
|
664
|
+
if (sameCycle(x.projectName, target)) {
|
|
665
|
+
if (!isKeyedGroup(task) && !nodes.has(to)) {
|
|
666
|
+
visit(null, target, name, false)
|
|
667
|
+
drain()
|
|
668
|
+
expanded = true
|
|
669
|
+
reaching = null
|
|
670
|
+
}
|
|
671
|
+
if (isKeyedGroup(task) || (reaching ??= reachingTo(nodes, x.id)).has(to)) {
|
|
672
|
+
frontier.push(...packageGraph.directDeps(target))
|
|
673
|
+
continue
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
const size = nodes.size
|
|
677
|
+
frame.added = false
|
|
678
|
+
visit(frame, target, name, false)
|
|
679
|
+
drain()
|
|
680
|
+
if (nodes.size !== size) reaching = null
|
|
681
|
+
if (isKeyedGroup(task) && onCycle(target)) frontier.push(...packageGraph.directDeps(target))
|
|
682
|
+
}
|
|
683
|
+
x.deps = [...new Set(x.deps)].sort()
|
|
684
|
+
}
|
|
685
|
+
for (let i = 0; i < deferred.length; i++) resolveDeferred(deferred[i]!)
|
|
686
|
+
if (expanded) {
|
|
687
|
+
const keep = new Set<string>()
|
|
688
|
+
const roots = [...nodes.values()].filter((n) => n.requested).map((n) => n.id)
|
|
689
|
+
while (roots.length > 0) {
|
|
690
|
+
const id = roots.pop()!
|
|
691
|
+
if (keep.has(id)) continue
|
|
692
|
+
keep.add(id)
|
|
693
|
+
roots.push(...nodes.get(id)!.deps)
|
|
694
|
+
}
|
|
695
|
+
for (const id of nodes.keys()) if (!keep.has(id)) nodes.delete(id)
|
|
696
|
+
}
|
|
697
|
+
|
|
607
698
|
checkGraph(nodes, options.workspaceRoot, options.rules)
|
|
608
699
|
return nodes
|
|
609
700
|
}
|
|
610
701
|
|
|
702
|
+
/** The ids of the tasks that reach `target` along `deps`, itself included. */
|
|
703
|
+
function reachingTo(nodes: Map<string, TaskNode>, target: string): Set<string> {
|
|
704
|
+
const dependants = new Map<string, string[]>()
|
|
705
|
+
for (const n of nodes.values()) {
|
|
706
|
+
for (const d of n.deps) {
|
|
707
|
+
const list = dependants.get(d)
|
|
708
|
+
if (list === undefined) dependants.set(d, [n.id])
|
|
709
|
+
else list.push(n.id)
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
const out = new Set<string>()
|
|
713
|
+
const stack = [target]
|
|
714
|
+
while (stack.length > 0) {
|
|
715
|
+
const id = stack.pop()!
|
|
716
|
+
if (out.has(id)) continue
|
|
717
|
+
out.add(id)
|
|
718
|
+
stack.push(...(dependants.get(id) ?? []))
|
|
719
|
+
}
|
|
720
|
+
return out
|
|
721
|
+
}
|
|
722
|
+
|
|
611
723
|
/**
|
|
612
724
|
* What a task graph must hold before it is scheduled: each node under its
|
|
613
725
|
* own id, every dep a node, no cycle, and no two tasks deleting each
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// behind `^build` runs when `ui#build` (or what it depends on) is reached,
|
|
5
5
|
// and a spec edit `ui#build`'s inputs leave out stops at `ui`'s own tasks.
|
|
6
6
|
|
|
7
|
-
import { declaresInput } from '../cache/index.js'
|
|
7
|
+
import { declaresInput, workspaceFilesReachInto } from '../cache/index.js'
|
|
8
8
|
import {
|
|
9
9
|
compileTaskPattern,
|
|
10
10
|
isGroupTask,
|
|
@@ -91,7 +91,11 @@ export function affectedRoots(
|
|
|
91
91
|
const cache = n.config.cache
|
|
92
92
|
// Asked of every node, not only the changed projects' (the
|
|
93
93
|
// `workspaceFiles` owners): a `graph` hook may have given the glob.
|
|
94
|
-
if (
|
|
94
|
+
if (
|
|
95
|
+
cache !== undefined &&
|
|
96
|
+
(changes.changed.some((rel) => declaresInput(cache, null, rel)) ||
|
|
97
|
+
(changes.nested ?? []).some((dir) => workspaceFilesReachInto(cache, dir)))
|
|
98
|
+
) {
|
|
95
99
|
return true
|
|
96
100
|
}
|
|
97
101
|
if (!changes.projects.has(n.projectName)) return false
|
|
@@ -23,7 +23,8 @@ import type { TaskNode } from '../graph/index.js'
|
|
|
23
23
|
export interface DeferredEntry {
|
|
24
24
|
/** Fetches this task's outputs onto disk. At-most-once by construction. */
|
|
25
25
|
materialize: () => Promise<void>
|
|
26
|
-
|
|
26
|
+
/** The key a fetch saves the bytes under; absent when the inputs moved after it was taken. */
|
|
27
|
+
hash?: string
|
|
27
28
|
entry: { taskId: string; command: string; durationMs: number; stdout: string }
|
|
28
29
|
}
|
|
29
30
|
|
|
@@ -149,7 +150,7 @@ export class DeferredOutputs {
|
|
|
149
150
|
// Convergence: an ordinary entry, so the next run is a plain local hit
|
|
150
151
|
// and nothing downstream learns a second shape. Policy still governs —
|
|
151
152
|
// a run that writes no local entries writes none here either.
|
|
152
|
-
if (this.args.localWrite) {
|
|
153
|
+
if (this.args.localWrite && entry.hash !== undefined) {
|
|
153
154
|
await this.args.cache.save({
|
|
154
155
|
hash: entry.hash,
|
|
155
156
|
projectDir: producer.projectDir,
|
|
@@ -97,7 +97,7 @@ export interface InfoFacts {
|
|
|
97
97
|
hits24h: number
|
|
98
98
|
/** Of `hits24h`, the hits that restored outputs; the rest were up to date. */
|
|
99
99
|
restored24h: number
|
|
100
|
-
/** Tasks the retained history shows
|
|
100
|
+
/** Tasks the retained history shows failing on a key that had passed. */
|
|
101
101
|
flakyTasks: FlakyTask[]
|
|
102
102
|
lockfile: boolean
|
|
103
103
|
/**
|
|
@@ -256,7 +256,7 @@ async function collectWorkspaceInfo(
|
|
|
256
256
|
runs24h: stats.runCountLast24h,
|
|
257
257
|
hits24h: stats.hitCountLast24h,
|
|
258
258
|
restored24h: stats.restoredCountLast24h,
|
|
259
|
-
//
|
|
259
|
+
// A key that failed after it had passed — the history's definition of flaky, over
|
|
260
260
|
// the 30 days it keeps. A run names its own findings in its footer;
|
|
261
261
|
// this is the workspace's standing list.
|
|
262
262
|
flakyTasks: flaky,
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
import type { CacheLayer, GitFilesCache } from '../cache/index.js'
|
|
17
17
|
import { isGroupTask, type TaskNode, type TaskOutcome } from '../graph/index.js'
|
|
18
18
|
import { computeGroupKey, computeTaskHash, type HashCache } from './task-hash.js'
|
|
19
|
-
import { filterUpstreamHashes, keyedDeps } from './upstream.js'
|
|
19
|
+
import { filterUpstreamHashes, keyedDeps, keyedOutcome } from './upstream.js'
|
|
20
20
|
|
|
21
21
|
export interface KeyExcludedArgs {
|
|
22
22
|
/** The scheduled graph, after `excludeDependencies`. */
|
|
@@ -50,7 +50,7 @@ export async function keyExcludedDependencies(args: KeyExcludedArgs): Promise<vo
|
|
|
50
50
|
// (item 737). The keys still resolve concurrently.
|
|
51
51
|
const keys = new Map<string, Promise<string | undefined>>()
|
|
52
52
|
const outcomes = (ids: readonly string[]): Promise<TaskOutcome[]> =>
|
|
53
|
-
Promise.all(ids.map(async (id) =>
|
|
53
|
+
Promise.all(ids.map(async (id) => keyedOutcome(nodeOf(id), await keys.get(id)!)))
|
|
54
54
|
const derive = async (node: TaskNode): Promise<string | undefined> => {
|
|
55
55
|
const upstream = await outcomes(fullDeps(node))
|
|
56
56
|
return (isGroupTask(node) ? computeGroupKey : computeTaskHash)({
|
|
@@ -139,14 +139,3 @@ function foldsExcludedKey(node: TaskNode): boolean {
|
|
|
139
139
|
.length > 0
|
|
140
140
|
)
|
|
141
141
|
}
|
|
142
|
-
|
|
143
|
-
/** A dependency as a key reads it: only `node` and `hash` matter. */
|
|
144
|
-
function synthetic(node: TaskNode, hash: string | undefined): TaskOutcome {
|
|
145
|
-
return {
|
|
146
|
-
node,
|
|
147
|
-
status: 'success',
|
|
148
|
-
exitCode: 0,
|
|
149
|
-
durationMs: 0,
|
|
150
|
-
...(hash !== undefined ? { hash } : {}),
|
|
151
|
-
}
|
|
152
|
-
}
|
|
@@ -524,11 +524,10 @@ async function executePersistentTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
524
524
|
// exited before ready keeps its own exit code rather than a made-up 1.
|
|
525
525
|
// One the readiness timeout is killing reports the signal's, as an
|
|
526
526
|
// ordinary timeout does (X-24).
|
|
527
|
+
// A readiness timeout rejects once the group is gone (runner.ts), so a
|
|
528
|
+
// server that ignored the TERM holds no port into the next `vx watch`
|
|
529
|
+
// cycle, and a second signal here would cut a one-shot handler short.
|
|
527
530
|
const ready = err instanceof PersistentReadyError ? err : undefined
|
|
528
|
-
// The timer's SIGKILL is a grace away and the shell may die on the
|
|
529
|
-
// TERM first: a server that ignores it held its port past run() into
|
|
530
|
-
// the next `vx watch` cycle. Return once the group is gone.
|
|
531
|
-
if (ready?.reason === 'timeout') await terminateChildren(() => [spawn.child])
|
|
532
531
|
return {
|
|
533
532
|
node,
|
|
534
533
|
status: 'failed',
|
|
@@ -637,7 +636,10 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
637
636
|
// `signal` stops a plugin executor on the run's stop AND on
|
|
638
637
|
// `exec.timeout`: core cannot kill a process an executor spawned, so a
|
|
639
638
|
// declared timeout meant nothing on one that kept its own clock (H-12).
|
|
640
|
-
// The local executor keeps its own timer (it signals the
|
|
639
|
+
// The local executor keeps its own timer from the spawn (it signals the
|
|
640
|
+
// process group), so its request carries no clock: armed here, it also
|
|
641
|
+
// timed the sandbox's wrap, and a wrap slower than the timeout aborted
|
|
642
|
+
// the request before the spawn — a task reported killed, never run.
|
|
641
643
|
let timeoutTimer: ReturnType<typeof setTimeout> | undefined
|
|
642
644
|
let timeoutFired = false
|
|
643
645
|
// The attempt's listener on the run's stop signal, taken off once the
|
|
@@ -645,7 +647,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
645
647
|
// grew to the task count, and each add scans it for a duplicate, ~100 ms
|
|
646
648
|
// of a 1,000-task cold run.
|
|
647
649
|
let unlistenStop: (() => void) | undefined
|
|
648
|
-
function requestSignal(): AbortSignal {
|
|
650
|
+
function requestSignal(local: boolean): AbortSignal {
|
|
649
651
|
const stop = new AbortController()
|
|
650
652
|
const abort = (): void => stop.abort(args.stopSignal?.reason)
|
|
651
653
|
unlistenStop?.()
|
|
@@ -656,9 +658,9 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
656
658
|
run.addEventListener('abort', abort, { once: true })
|
|
657
659
|
unlistenStop = () => run.removeEventListener('abort', abort)
|
|
658
660
|
}
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
661
|
+
clearTimeout(timeoutTimer)
|
|
662
|
+
timeoutFired = false
|
|
663
|
+
if (effectiveTimeout !== undefined && !local) {
|
|
662
664
|
timeoutTimer = setTimeout(() => {
|
|
663
665
|
timeoutFired = true
|
|
664
666
|
stop.abort(new Error(`timed out after ${effectiveTimeout}ms`))
|
|
@@ -808,7 +810,10 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
808
810
|
}
|
|
809
811
|
if (preProbed !== undefined) {
|
|
810
812
|
if (preProbed.hit !== null) {
|
|
811
|
-
|
|
813
|
+
// A key taken before a task rewrote the lockfile names an install
|
|
814
|
+
// the tree no longer holds, and the lazy path below never probes
|
|
815
|
+
// one (X-124): demoted like a vanished hit, it runs dep-gated.
|
|
816
|
+
const restored = fingerprintMoved() ? null : await restoreOrMiss(preProbed.hit)
|
|
812
817
|
if (restored !== null) return restored
|
|
813
818
|
// The up-front probe's hit may be restoring AHEAD of this task's
|
|
814
819
|
// deps, and a command run now would build from outputs they have
|
|
@@ -917,6 +922,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
917
922
|
async function runAttempt(): Promise<{
|
|
918
923
|
result: ExecuteResult
|
|
919
924
|
exitCode: number
|
|
925
|
+
withdrawn?: true
|
|
920
926
|
}> {
|
|
921
927
|
// A deferred task's outputs are deliberately NOT coming, so wiping the
|
|
922
928
|
// tree would replace a stale build with nothing at all. The eligibility
|
|
@@ -952,8 +958,18 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
952
958
|
}
|
|
953
959
|
violations = []
|
|
954
960
|
const endReq = span('miss: build request')
|
|
955
|
-
const req = await buildRequest()
|
|
961
|
+
const req = await buildRequest(isLocalExecutor(args.executor))
|
|
956
962
|
endReq()
|
|
963
|
+
// A sibling's failure under --continue=never can land while a retry
|
|
964
|
+
// cleans or builds its request, after the loop's own check: the retry
|
|
965
|
+
// is withdrawn and the attempt before it stands.
|
|
966
|
+
if (attempt > 1 && args.failFast?.aborted === true) {
|
|
967
|
+
clearTimeout(timeoutTimer)
|
|
968
|
+
unlistenStop?.()
|
|
969
|
+
unlistenStop = undefined
|
|
970
|
+
await sweepPlaceholders(placeholders)
|
|
971
|
+
return { result, exitCode: effectiveExitCode, withdrawn: true }
|
|
972
|
+
}
|
|
957
973
|
// An executor that THROWS produces no captured output. Rethrown: the
|
|
958
974
|
// scheduler classifies it and prints its one line into the task's own
|
|
959
975
|
// stream (run()'s onError), where the frame reads it; a copy written
|
|
@@ -984,8 +1000,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
984
1000
|
raw = new UserError(`${why}, and remote: 'only' keeps it off this machine`)
|
|
985
1001
|
} else {
|
|
986
1002
|
log.status(`[vx] ${node.id}: ${why} — running it here`)
|
|
987
|
-
|
|
988
|
-
const local = await localExecutor().execute(await buildRequest())
|
|
1003
|
+
const local = await localExecutor().execute(await buildRequest(true))
|
|
989
1004
|
assertExecuteResult('local', node.id, local)
|
|
990
1005
|
return local
|
|
991
1006
|
}
|
|
@@ -1016,11 +1031,11 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1016
1031
|
stdout: maskCaptured(res.stdout, secrets),
|
|
1017
1032
|
stderr: maskCaptured(res.stderr, secrets),
|
|
1018
1033
|
}
|
|
1019
|
-
// An executor that stopped on the timeout's abort
|
|
1020
|
-
//
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
}
|
|
1034
|
+
// An executor that stopped on the timeout's abort timed out, whatever
|
|
1035
|
+
// its exit: a child that traps the TERM and exits 0 left partial outputs
|
|
1036
|
+
// a pass would save (X-125), as the local runner's own `timedOut` says.
|
|
1037
|
+
// A local request carries no timer, so the runner alone decides there.
|
|
1038
|
+
if (timeoutFired && res.timedOut !== true) res = { ...res, timedOut: true }
|
|
1024
1039
|
violations = [...res.violations]
|
|
1025
1040
|
// A denial under a dependency the key does not answer for says only
|
|
1026
1041
|
// ENOENT; the line beside it names the package and how to key it. Only
|
|
@@ -1167,10 +1182,15 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1167
1182
|
for (;;) {
|
|
1168
1183
|
attempt++
|
|
1169
1184
|
const a = await runAttempt()
|
|
1170
|
-
forgetUndeclaredWrites(args, writeReach)
|
|
1171
|
-
if (writesFingerprint) args.fingerprintWatch?.wrote()
|
|
1172
1185
|
result = a.result
|
|
1173
1186
|
effectiveExitCode = a.exitCode
|
|
1187
|
+
if (a.withdrawn) {
|
|
1188
|
+
attempt--
|
|
1189
|
+
failedAttempts.pop()
|
|
1190
|
+
break
|
|
1191
|
+
}
|
|
1192
|
+
forgetUndeclaredWrites(args, writeReach)
|
|
1193
|
+
if (writesFingerprint) args.fingerprintWatch?.wrote()
|
|
1174
1194
|
spentMs += result.durationMs
|
|
1175
1195
|
|
|
1176
1196
|
// An attempt that ended in a shutdown never finished on its own terms —
|
|
@@ -1219,7 +1239,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1219
1239
|
)
|
|
1220
1240
|
}
|
|
1221
1241
|
|
|
1222
|
-
async function buildRequest(): Promise<ExecuteRequest> {
|
|
1242
|
+
async function buildRequest(local: boolean): Promise<ExecuteRequest> {
|
|
1223
1243
|
const out = secrets && maskedEmitter(secrets, (t) => log.taskStdout(node, t))
|
|
1224
1244
|
const err = secrets && maskedEmitter(secrets, (t) => log.taskStderr(node, t))
|
|
1225
1245
|
flushMasked = () => {
|
|
@@ -1257,10 +1277,10 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1257
1277
|
outputs: { files: outputs, workspaceFiles: wsOutputs },
|
|
1258
1278
|
...(args.terminal === true ? { terminal: true as const } : {}),
|
|
1259
1279
|
}
|
|
1260
|
-
// The signal last: it arms `exec.timeout`, and the sandbox's arming
|
|
1280
|
+
// The signal last: it arms a plugin's `exec.timeout`, and the sandbox's arming
|
|
1261
1281
|
// below is vx's work, not the task's. Armed first, a 60 ms timeout
|
|
1262
1282
|
// expired before the spawn and failed `echo` as timed out, unrun.
|
|
1263
|
-
if (!userSandbox) return { ...base, signal: requestSignal() }
|
|
1283
|
+
if (!userSandbox) return { ...base, signal: requestSignal(local) }
|
|
1264
1284
|
await args.armSandbox?.()
|
|
1265
1285
|
const sb = await sandboxRequestFor(
|
|
1266
1286
|
node,
|
|
@@ -1272,7 +1292,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1272
1292
|
)
|
|
1273
1293
|
placeholders = sb.placeholders
|
|
1274
1294
|
withheld = sb.withheld
|
|
1275
|
-
return { ...base, sandbox: sb.sandbox, signal: requestSignal() }
|
|
1295
|
+
return { ...base, sandbox: sb.sandbox, signal: requestSignal(local) }
|
|
1276
1296
|
}
|
|
1277
1297
|
|
|
1278
1298
|
const wallclockEndNs = process.hrtime.bigint() - args.runStartHrTimeNs
|
|
@@ -1283,12 +1303,15 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1283
1303
|
// record (a row with no artifact) is exactly the corrupt-entry shape
|
|
1284
1304
|
// `restoreOutputs` refuses, so writing none is the only clean answer.
|
|
1285
1305
|
// The closure is what a later local consumer — or a later eager run —
|
|
1286
|
-
// pulls the bytes with.
|
|
1306
|
+
// pulls the bytes with. Its fetch saves them under `hash` only past the
|
|
1307
|
+
// eager save's check: an input edited before the describe filed the
|
|
1308
|
+
// edit's bytes under the old key (X-123).
|
|
1287
1309
|
const materialize = result.outputs?.kind === 'deferred' ? result.outputs.materialize : undefined
|
|
1288
1310
|
if (materialize !== undefined) {
|
|
1311
|
+
const keyed = await keyStillTrue()
|
|
1289
1312
|
args.deferred?.register(node.id, {
|
|
1290
1313
|
materialize,
|
|
1291
|
-
hash,
|
|
1314
|
+
...(keyed ? { hash } : {}),
|
|
1292
1315
|
entry: {
|
|
1293
1316
|
taskId: node.id,
|
|
1294
1317
|
command: storedCommand,
|