@vzn/vx 0.0.595 → 0.0.597
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/launcher.cjs +7 -3
- package/package.json +5 -5
- package/src/cache/archive.ts +35 -6
- package/src/cache/cache.ts +170 -99
- package/src/cache/chained-cache.ts +25 -7
- package/src/cache/git-inputs.ts +23 -0
- package/src/cache/index.ts +3 -0
- package/src/cache/inputs.ts +99 -44
- package/src/cache/key-fold.ts +1 -1
- package/src/cache/layer.ts +9 -6
- package/src/cache/policy.ts +5 -0
- package/src/cache/zstd.ts +10 -4
- package/src/cli/cache.ts +12 -6
- package/src/cli/foreign-flags.ts +57 -3
- package/src/cli/info.ts +1 -1
- package/src/cli/init.ts +2 -2
- package/src/cli/last.ts +10 -4
- package/src/cli/lock.ts +5 -3
- package/src/cli/plan-format.ts +4 -4
- package/src/cli/run.ts +73 -30
- package/src/cli/select.ts +6 -3
- package/src/cli/show.ts +8 -2
- package/src/cli/upgrade.ts +1 -1
- package/src/cli/watch-filter.ts +63 -12
- package/src/cli/watch-fs.ts +11 -3
- package/src/cli/watch-judge.ts +15 -8
- package/src/cli/watch.ts +18 -11
- package/src/cli/workspace-config.ts +3 -1
- package/src/config.ts +42 -21
- package/src/exec/executor.ts +1 -1
- package/src/exec/kill-tree.ts +89 -18
- package/src/exec/proc-sample.ts +1 -1
- package/src/exec/runner.ts +74 -36
- package/src/exec/sandbox-paths.ts +12 -1
- package/src/exec/sandbox-runtime.ts +283 -88
- package/src/exec/sandbox-violations.ts +115 -15
- package/src/graph/scheduler.ts +30 -14
- package/src/graph/task-graph.ts +265 -44
- package/src/orchestrator/admission.ts +12 -3
- package/src/orchestrator/affected-tasks.ts +6 -2
- package/src/orchestrator/deferred-outputs.ts +3 -2
- package/src/orchestrator/doctor.ts +2 -2
- package/src/orchestrator/events.ts +10 -1
- package/src/orchestrator/excluded-keys.ts +2 -13
- package/src/orchestrator/execute-task.ts +85 -39
- package/src/orchestrator/failure-mode.ts +43 -28
- package/src/orchestrator/framed-output.ts +8 -8
- package/src/orchestrator/index.ts +0 -1
- package/src/orchestrator/keyed-projects.ts +6 -3
- package/src/orchestrator/logger.ts +21 -9
- package/src/orchestrator/metrics.ts +19 -8
- package/src/orchestrator/miss-save.ts +9 -13
- package/src/orchestrator/options.ts +7 -0
- package/src/orchestrator/plan.ts +7 -23
- package/src/orchestrator/plugin.ts +37 -16
- package/src/orchestrator/prepare.ts +13 -4
- package/src/orchestrator/run-artifacts.ts +4 -3
- package/src/orchestrator/run-lock.ts +25 -24
- package/src/orchestrator/run.ts +22 -5
- package/src/orchestrator/sandbox-request.ts +96 -18
- package/src/orchestrator/save-lane.ts +5 -5
- package/src/orchestrator/signals.ts +22 -4
- package/src/orchestrator/stable-keys.ts +3 -11
- package/src/orchestrator/summary.ts +7 -13
- package/src/orchestrator/task-hash.ts +13 -3
- package/src/orchestrator/telemetry.ts +16 -7
- package/src/orchestrator/upstream.ts +11 -0
- package/src/util/hangup.ts +42 -0
- package/src/util/index.ts +2 -0
- package/src/util/paths.ts +24 -4
- package/src/workspace/affected.ts +79 -42
- package/src/workspace/config-eval.ts +27 -6
- package/src/workspace/config-imports.ts +8 -53
- package/src/workspace/config-schema.ts +56 -9
- package/src/workspace/filter.ts +19 -0
- package/src/workspace/lockfile.ts +2 -1
- package/src/workspace/migrate-scripts.ts +5 -3
- package/src/workspace/migration.ts +1 -1
- package/src/workspace/project-loader.ts +19 -4
- package/src/workspace/repo-id.ts +42 -4
- package/src/workspace/workspace.ts +53 -24
package/src/graph/task-graph.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import path from 'node:path'
|
|
2
1
|
import type { ProjectConfig, TaskConfig, WorkspaceRules } from '../config.js'
|
|
3
2
|
import {
|
|
4
3
|
asTrees,
|
|
@@ -109,6 +108,11 @@ export function isGroupTask(node: TaskNode): boolean {
|
|
|
109
108
|
return node.config.exec === undefined
|
|
110
109
|
}
|
|
111
110
|
|
|
111
|
+
/** The default `build` (orchestrator/projects.ts): a config cannot key a group, so it is the one keyed group. */
|
|
112
|
+
function isKeyedGroup(task: TaskConfig): boolean {
|
|
113
|
+
return task.exec === undefined && task.cache !== undefined
|
|
114
|
+
}
|
|
115
|
+
|
|
112
116
|
/**
|
|
113
117
|
* Mark, for focused-flow display, the real tasks a requested GROUP
|
|
114
118
|
* stands for. A group has no output of its own, so `vx run build`
|
|
@@ -160,7 +164,8 @@ export function markSurfacedDeps(nodes: Map<string, TaskNode>): number {
|
|
|
160
164
|
*
|
|
161
165
|
* - Bare task names (`'build'`) → one entry per project in
|
|
162
166
|
* `candidates` that declares the task. Missing in a given project
|
|
163
|
-
* is silent (sparse tasks are normal across a workspace).
|
|
167
|
+
* is silent (sparse tasks are normal across a workspace). A task
|
|
168
|
+
* `unselected` holds (the default `build`) is no match.
|
|
164
169
|
* - Anchored entries (`'pkg#task'`) → one entry exactly, ignoring
|
|
165
170
|
* `candidates`. Silently dropped if pkg/task doesn't exist (the
|
|
166
171
|
* CLI's pre-validation catches malformed strings).
|
|
@@ -171,6 +176,7 @@ export function expandRequested(
|
|
|
171
176
|
tasks: readonly string[],
|
|
172
177
|
candidates: readonly string[],
|
|
173
178
|
projects: Map<string, ProjectEntry>,
|
|
179
|
+
unselected?: (task: TaskConfig) => boolean,
|
|
174
180
|
): Array<{ project: string; task: string }> {
|
|
175
181
|
const seen = new Set<string>()
|
|
176
182
|
const out: Array<{ project: string; task: string }> = []
|
|
@@ -189,7 +195,7 @@ export function expandRequested(
|
|
|
189
195
|
continue
|
|
190
196
|
}
|
|
191
197
|
for (const name of candidates) {
|
|
192
|
-
if (
|
|
198
|
+
if (selectsTask(projects, name, spec, unselected)) push(name, spec)
|
|
193
199
|
}
|
|
194
200
|
}
|
|
195
201
|
return out
|
|
@@ -214,6 +220,17 @@ function declaresTask(projects: Map<string, ProjectEntry>, project: string, task
|
|
|
214
220
|
return declaredTask(projects.get(project)?.config, task) !== undefined
|
|
215
221
|
}
|
|
216
222
|
|
|
223
|
+
/** What a bare name matches: a declared task `unselected` leaves alone. */
|
|
224
|
+
function selectsTask(
|
|
225
|
+
projects: Map<string, ProjectEntry>,
|
|
226
|
+
project: string,
|
|
227
|
+
task: string,
|
|
228
|
+
unselected: ((task: TaskConfig) => boolean) | undefined,
|
|
229
|
+
): boolean {
|
|
230
|
+
const config = declaredTask(projects.get(project)?.config, task)
|
|
231
|
+
return config !== undefined && unselected?.(config) !== true
|
|
232
|
+
}
|
|
233
|
+
|
|
217
234
|
/**
|
|
218
235
|
* The requested specs `expandRequested` silently dropped — each one
|
|
219
236
|
* matched NO project, so nothing it asked for will run.
|
|
@@ -230,6 +247,7 @@ export function unresolvedRequests(
|
|
|
230
247
|
tasks: readonly string[],
|
|
231
248
|
candidates: readonly string[],
|
|
232
249
|
projects: Map<string, ProjectEntry>,
|
|
250
|
+
unselected?: (task: TaskConfig) => boolean,
|
|
233
251
|
): string[] {
|
|
234
252
|
const out: string[] = []
|
|
235
253
|
const seen = new Set<string>()
|
|
@@ -239,7 +257,8 @@ export function unresolvedRequests(
|
|
|
239
257
|
const resolved =
|
|
240
258
|
idx >= 0
|
|
241
259
|
? declaresTask(projects, spec.slice(0, idx), spec.slice(idx + 1))
|
|
242
|
-
: candidates.length === 0 ||
|
|
260
|
+
: candidates.length === 0 ||
|
|
261
|
+
candidates.some((name) => selectsTask(projects, name, spec, unselected))
|
|
243
262
|
if (resolved) continue
|
|
244
263
|
seen.add(spec)
|
|
245
264
|
out.push(spec)
|
|
@@ -315,7 +334,8 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
|
|
|
315
334
|
|
|
316
335
|
function add(node: TaskNode): void {
|
|
317
336
|
nodes.set(node.id, node)
|
|
318
|
-
if ((node.config
|
|
337
|
+
if (isKeyedGroup(node.config) && onCycle(node.projectName)) deferred.push(node)
|
|
338
|
+
else if ((node.config.dependsOn?.length ?? 0) > 0) {
|
|
319
339
|
stack.push({ node, entry: 0, added: false, pending: null, next: 0 })
|
|
320
340
|
}
|
|
321
341
|
}
|
|
@@ -380,6 +400,88 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
|
|
|
380
400
|
return declared.has(name)
|
|
381
401
|
}
|
|
382
402
|
|
|
403
|
+
// Package cycles, for the default `build` (`isKeyedGroup`): each project
|
|
404
|
+
// on one maps to its strongly connected component, found on the first ask
|
|
405
|
+
// (iterative Tarjan over `directDeps`, the order the `^name` walk reads).
|
|
406
|
+
const component = new Map<string, number>()
|
|
407
|
+
const cyclic = new Set<number>()
|
|
408
|
+
let components = 0
|
|
409
|
+
function componentOf(start: string): number {
|
|
410
|
+
const known = component.get(start)
|
|
411
|
+
if (known !== undefined) return known
|
|
412
|
+
const index = new Map<string, number>()
|
|
413
|
+
const low = new Map<string, number>()
|
|
414
|
+
const open: string[] = []
|
|
415
|
+
const work: Array<[name: string, next: number]> = []
|
|
416
|
+
const enter = (name: string): void => {
|
|
417
|
+
index.set(name, index.size)
|
|
418
|
+
low.set(name, index.size - 1)
|
|
419
|
+
open.push(name)
|
|
420
|
+
work.push([name, 0])
|
|
421
|
+
}
|
|
422
|
+
enter(start)
|
|
423
|
+
while (work.length > 0) {
|
|
424
|
+
const top = work[work.length - 1]!
|
|
425
|
+
const deps = packageGraph.directDeps(top[0])
|
|
426
|
+
if (top[1] < deps.length) {
|
|
427
|
+
const dep = deps[top[1]++]!
|
|
428
|
+
if (component.has(dep)) continue
|
|
429
|
+
if (!index.has(dep)) enter(dep)
|
|
430
|
+
else low.set(top[0], Math.min(low.get(top[0])!, index.get(dep)!))
|
|
431
|
+
continue
|
|
432
|
+
}
|
|
433
|
+
work.pop()
|
|
434
|
+
const name = top[0]
|
|
435
|
+
const parent = work[work.length - 1]
|
|
436
|
+
if (parent !== undefined) low.set(parent[0], Math.min(low.get(parent[0])!, low.get(name)!))
|
|
437
|
+
if (low.get(name) !== index.get(name)) continue
|
|
438
|
+
const id = components++
|
|
439
|
+
let size = 0
|
|
440
|
+
for (let member = ''; member !== name; size++) {
|
|
441
|
+
member = open.pop()!
|
|
442
|
+
component.set(member, id)
|
|
443
|
+
}
|
|
444
|
+
if (size > 1) cyclic.add(id)
|
|
445
|
+
}
|
|
446
|
+
return component.get(start)!
|
|
447
|
+
}
|
|
448
|
+
// Default builds on a package cycle, expanded once the rest is built.
|
|
449
|
+
const deferred: TaskNode[] = []
|
|
450
|
+
const onCycle = (name: string): boolean => cyclic.has(componentOf(name))
|
|
451
|
+
const sameCycle = (a: string, b: string): boolean =>
|
|
452
|
+
onCycle(a) && componentOf(a) === componentOf(b)
|
|
453
|
+
|
|
454
|
+
// `^name` from `start`: edges to the nearest holders of `name` among the
|
|
455
|
+
// packages below it, passing through the ones `visited` holds and the
|
|
456
|
+
// ones that declare no `name`, and on past a default build on a cycle.
|
|
457
|
+
// True when it found a holder.
|
|
458
|
+
function walkHolders(frame: Frame, name: string, visited: Set<string>, start: string): boolean {
|
|
459
|
+
const frontier = [...packageGraph.directDeps(start)]
|
|
460
|
+
let held = false
|
|
461
|
+
while (frontier.length > 0) {
|
|
462
|
+
const target = frontier.pop()!
|
|
463
|
+
if (visited.has(target)) continue
|
|
464
|
+
visited.add(target)
|
|
465
|
+
if (visit(frame, target, name, false)) {
|
|
466
|
+
held = true
|
|
467
|
+
const task = declaredTask(projects.get(target)!.config, name)!
|
|
468
|
+
if (isKeyedGroup(task) && onCycle(target)) {
|
|
469
|
+
frontier.push(...packageGraph.directDeps(target))
|
|
470
|
+
}
|
|
471
|
+
} else frontier.push(...packageGraph.directDeps(target))
|
|
472
|
+
}
|
|
473
|
+
return held
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
// An edge by name (`build`, `pkg#build`) to a default build on a package
|
|
477
|
+
// cycle goes on past it as a `^build` walk does: that build folds nothing
|
|
478
|
+
// of the builds on its cycle, so `a#test` on `build` with `a` ↔ `b` both
|
|
479
|
+
// on the default build folded none of `b` and hit after `b` changed.
|
|
480
|
+
function goPast(frame: Frame, project: string, name: string, visited: Set<string>): void {
|
|
481
|
+
const task = declaredTask(projects.get(project)!.config, name)!
|
|
482
|
+
if (isKeyedGroup(task) && onCycle(project)) walkHolders(frame, name, visited, project)
|
|
483
|
+
}
|
|
484
|
+
|
|
383
485
|
// Resolves one `dependsOn` entry of `frame`'s task into its edges.
|
|
384
486
|
function resolveEntry(frame: Frame, raw: string): void {
|
|
385
487
|
const { node } = frame
|
|
@@ -429,6 +531,7 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
|
|
|
429
531
|
`Task ${id} depends on ${taskId(projectName, spec.task)} but no such task is declared`,
|
|
430
532
|
)
|
|
431
533
|
}
|
|
534
|
+
goPast(frame, projectName, spec.task, new Set([projectName]))
|
|
432
535
|
}
|
|
433
536
|
} else if (spec.kind === 'deps') {
|
|
434
537
|
// Nearest-holder frontier (Turbo/Nx direct-deps parity +
|
|
@@ -451,33 +554,37 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
|
|
|
451
554
|
// shape), and a cycle walks the frontier straight back to the
|
|
452
555
|
// origin. Mirrors the self-pattern rule above — a task can never
|
|
453
556
|
// depend on itself.
|
|
454
|
-
|
|
557
|
+
//
|
|
558
|
+
// The default `build` sits on every project, so on a package cycle it
|
|
559
|
+
// would make a task cycle no config declares. One on a cycle walks
|
|
560
|
+
// last (`resolveDeferred`), and a walk that meets it on a cycle takes
|
|
561
|
+
// the edge and goes on past it, so the builds it could not depend on
|
|
562
|
+
// still come first.
|
|
455
563
|
const visited = new Set<string>([projectName])
|
|
564
|
+
if (!isTaskPattern(spec.task)) {
|
|
565
|
+
const held = walkHolders(frame, spec.task, visited, projectName)
|
|
566
|
+
if (!held && !declaredAnywhere(projectName, spec.task)) {
|
|
567
|
+
if (undeclaredDeps === undefined) throw undeclaredDepsError(id, spec.task)
|
|
568
|
+
undeclaredDeps(id, spec.task)
|
|
569
|
+
}
|
|
570
|
+
return
|
|
571
|
+
}
|
|
572
|
+
// A pattern that matches nothing stays legal, as `build.*` does.
|
|
573
|
+
const re = compileTaskPattern(spec.task)
|
|
456
574
|
const frontier = [...packageGraph.directDeps(projectName)]
|
|
457
|
-
let held = false
|
|
458
575
|
while (frontier.length > 0) {
|
|
459
576
|
const target = frontier.pop()!
|
|
460
577
|
if (visited.has(target)) continue
|
|
461
578
|
visited.add(target)
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
579
|
+
const names = Object.keys(projects.get(target)?.config.tasks ?? {}).filter((n) =>
|
|
580
|
+
re.test(n),
|
|
581
|
+
)
|
|
582
|
+
if (names.length > 0) {
|
|
583
|
+
for (const name of names) visit(frame, target, name, false)
|
|
465
584
|
} else {
|
|
466
|
-
|
|
467
|
-
re.test(n),
|
|
468
|
-
)
|
|
469
|
-
if (names.length > 0) {
|
|
470
|
-
for (const name of names) visit(frame, target, name, false)
|
|
471
|
-
} else {
|
|
472
|
-
frontier.push(...packageGraph.directDeps(target))
|
|
473
|
-
}
|
|
585
|
+
frontier.push(...packageGraph.directDeps(target))
|
|
474
586
|
}
|
|
475
587
|
}
|
|
476
|
-
// A pattern that matches nothing stays legal, as `build.*` does.
|
|
477
|
-
if (re === null && !held && !declaredAnywhere(projectName, spec.task)) {
|
|
478
|
-
if (undeclaredDeps === undefined) throw undeclaredDepsError(id, spec.task)
|
|
479
|
-
undeclaredDeps(id, spec.task)
|
|
480
|
-
}
|
|
481
588
|
} else {
|
|
482
589
|
// Cross-project edge: pkg#task. Missing target is a hard error
|
|
483
590
|
// because the user named the package + task explicitly.
|
|
@@ -491,11 +598,11 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
|
|
|
491
598
|
`Task ${id} depends on ${taskId(spec.project, spec.task)} but no such project or task is declared`,
|
|
492
599
|
)
|
|
493
600
|
}
|
|
601
|
+
goPast(frame, spec.project, spec.task, new Set([projectName, spec.project]))
|
|
494
602
|
}
|
|
495
603
|
}
|
|
496
604
|
|
|
497
|
-
|
|
498
|
-
visit(null, project, task, true)
|
|
605
|
+
const drain = (): void => {
|
|
499
606
|
while (stack.length > 0) {
|
|
500
607
|
const frame = stack[stack.length - 1]!
|
|
501
608
|
const { node, pending } = frame
|
|
@@ -522,10 +629,97 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
|
|
|
522
629
|
}
|
|
523
630
|
}
|
|
524
631
|
|
|
632
|
+
for (const { project, task } of requested) {
|
|
633
|
+
visit(null, project, task, true)
|
|
634
|
+
drain()
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
// A default build on a package cycle: its `^build` walk, run once the
|
|
638
|
+
// graph holds everything else. A build on its cycle that reaches it is
|
|
639
|
+
// passed through (an edge to it would close a task cycle no config
|
|
640
|
+
// declares); one that does not is a holder like any other, so the default
|
|
641
|
+
// build waits for it and folds its key. Passed through unasked, `p#test`
|
|
642
|
+
// on `p#build` ran beside a `t#build` on no `^build` and hit after `t`
|
|
643
|
+
// changed. A build no other edge brought is expanded to ask, and pruned
|
|
644
|
+
// again when it reaches back. Each walk sees the edges the ones before it
|
|
645
|
+
// took, so no edge closes a cycle.
|
|
646
|
+
let expanded = false
|
|
647
|
+
const resolveDeferred = (x: TaskNode): void => {
|
|
648
|
+
// The default build's one entry is `^` and its own name.
|
|
649
|
+
const name = x.taskName
|
|
650
|
+
const frame: Frame = { node: x, entry: 0, added: false, pending: null, next: 0 }
|
|
651
|
+
let reaching: Set<string> | null = null
|
|
652
|
+
const visited = new Set<string>([x.projectName])
|
|
653
|
+
const frontier = [...packageGraph.directDeps(x.projectName)]
|
|
654
|
+
while (frontier.length > 0) {
|
|
655
|
+
const target = frontier.pop()!
|
|
656
|
+
if (visited.has(target)) continue
|
|
657
|
+
visited.add(target)
|
|
658
|
+
const task = declaredTask(projects.get(target)?.config, name)
|
|
659
|
+
if (task === undefined) {
|
|
660
|
+
frontier.push(...packageGraph.directDeps(target))
|
|
661
|
+
continue
|
|
662
|
+
}
|
|
663
|
+
const to = taskId(target, name)
|
|
664
|
+
if (sameCycle(x.projectName, target)) {
|
|
665
|
+
if (!isKeyedGroup(task) && !nodes.has(to)) {
|
|
666
|
+
visit(null, target, name, false)
|
|
667
|
+
drain()
|
|
668
|
+
expanded = true
|
|
669
|
+
reaching = null
|
|
670
|
+
}
|
|
671
|
+
if (isKeyedGroup(task) || (reaching ??= reachingTo(nodes, x.id)).has(to)) {
|
|
672
|
+
frontier.push(...packageGraph.directDeps(target))
|
|
673
|
+
continue
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
const size = nodes.size
|
|
677
|
+
frame.added = false
|
|
678
|
+
visit(frame, target, name, false)
|
|
679
|
+
drain()
|
|
680
|
+
if (nodes.size !== size) reaching = null
|
|
681
|
+
if (isKeyedGroup(task) && onCycle(target)) frontier.push(...packageGraph.directDeps(target))
|
|
682
|
+
}
|
|
683
|
+
x.deps = [...new Set(x.deps)].sort()
|
|
684
|
+
}
|
|
685
|
+
for (let i = 0; i < deferred.length; i++) resolveDeferred(deferred[i]!)
|
|
686
|
+
if (expanded) {
|
|
687
|
+
const keep = new Set<string>()
|
|
688
|
+
const roots = [...nodes.values()].filter((n) => n.requested).map((n) => n.id)
|
|
689
|
+
while (roots.length > 0) {
|
|
690
|
+
const id = roots.pop()!
|
|
691
|
+
if (keep.has(id)) continue
|
|
692
|
+
keep.add(id)
|
|
693
|
+
roots.push(...nodes.get(id)!.deps)
|
|
694
|
+
}
|
|
695
|
+
for (const id of nodes.keys()) if (!keep.has(id)) nodes.delete(id)
|
|
696
|
+
}
|
|
697
|
+
|
|
525
698
|
checkGraph(nodes, options.workspaceRoot, options.rules)
|
|
526
699
|
return nodes
|
|
527
700
|
}
|
|
528
701
|
|
|
702
|
+
/** The ids of the tasks that reach `target` along `deps`, itself included. */
|
|
703
|
+
function reachingTo(nodes: Map<string, TaskNode>, target: string): Set<string> {
|
|
704
|
+
const dependants = new Map<string, string[]>()
|
|
705
|
+
for (const n of nodes.values()) {
|
|
706
|
+
for (const d of n.deps) {
|
|
707
|
+
const list = dependants.get(d)
|
|
708
|
+
if (list === undefined) dependants.set(d, [n.id])
|
|
709
|
+
else list.push(n.id)
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
const out = new Set<string>()
|
|
713
|
+
const stack = [target]
|
|
714
|
+
while (stack.length > 0) {
|
|
715
|
+
const id = stack.pop()!
|
|
716
|
+
if (out.has(id)) continue
|
|
717
|
+
out.add(id)
|
|
718
|
+
stack.push(...(dependants.get(id) ?? []))
|
|
719
|
+
}
|
|
720
|
+
return out
|
|
721
|
+
}
|
|
722
|
+
|
|
529
723
|
/**
|
|
530
724
|
* What a task graph must hold before it is scheduled: each node under its
|
|
531
725
|
* own id, every dep a node, no cycle, and no two tasks deleting each
|
|
@@ -975,36 +1169,51 @@ const GLOB_HEAD_END = /[*?{}\\!]/
|
|
|
975
1169
|
* directory its own head names. A whole subtree meets the globs whose
|
|
976
1170
|
* literal prefix lies at or under it, found the same way from the glob's
|
|
977
1171
|
* side (item 941).
|
|
1172
|
+
*
|
|
1173
|
+
* Given `sideOf`, only the pairs whose members sit on different sides are
|
|
1174
|
+
* wanted (a reader and a writer, `detectInputOverlaps`), and each index is
|
|
1175
|
+
* kept per side, so a member looks up only the other side's: thousands of
|
|
1176
|
+
* readers sharing one input (`src/**`, a root `tsconfig.base.json`) were
|
|
1177
|
+
* paired with each other, millions of pairs no rule judges (X-101).
|
|
978
1178
|
*/
|
|
979
1179
|
function overlapCandidates<T>(
|
|
980
1180
|
tasks: readonly T[],
|
|
981
1181
|
globsOf: (n: T) => readonly string[] | undefined,
|
|
1182
|
+
sideOf?: (n: T) => boolean,
|
|
982
1183
|
): Array<[number, number]> {
|
|
983
|
-
const
|
|
984
|
-
|
|
1184
|
+
const sides = sideOf === undefined ? 1 : 2
|
|
1185
|
+
// The index a member of side `s` looks up: its own side's, or the other's.
|
|
1186
|
+
const across = (s: number): number => (sides === 1 ? s : 1 - s)
|
|
1187
|
+
const perSide = (): Array<Map<string, number[]>> =>
|
|
1188
|
+
Array.from({ length: sides }, () => new Map<string, number[]>())
|
|
1189
|
+
const globs = perSide()
|
|
1190
|
+
const globsUnder = perSide()
|
|
985
1191
|
const literals: Array<[task: number, path: string]> = []
|
|
986
1192
|
// `covers` (item 941): a whole subtree `P/**` meets every glob whose
|
|
987
1193
|
// literal prefix is P or under it, so each glob looks up the subtrees
|
|
988
1194
|
// filed at its prefix and at each of that prefix's ancestors.
|
|
989
|
-
const subtrees =
|
|
1195
|
+
const subtrees = perSide()
|
|
990
1196
|
const prefixes: Array<[task: number, prefix: string]> = []
|
|
991
1197
|
const file = (index: Map<string, number[]>, key: string, i: number): void => {
|
|
992
1198
|
const list = index.get(key)
|
|
993
1199
|
if (list === undefined) index.set(key, [i])
|
|
994
1200
|
else list.push(i)
|
|
995
1201
|
}
|
|
1202
|
+
const side: number[] = []
|
|
996
1203
|
for (let i = 0; i < tasks.length; i++) {
|
|
1204
|
+
const s = sideOf?.(tasks[i]!) === true ? 1 : 0
|
|
1205
|
+
side.push(s)
|
|
997
1206
|
for (const tree of asTrees(globsOf(tasks[i]!) ?? [])) {
|
|
998
1207
|
if (isLiteralPattern(tree)) {
|
|
999
1208
|
literals.push([i, tree])
|
|
1000
1209
|
continue
|
|
1001
1210
|
}
|
|
1002
|
-
file(globs
|
|
1211
|
+
file(globs[s]!, tree, i)
|
|
1003
1212
|
const dir = wholeSubtreePrefixes([tree])?.[0]
|
|
1004
|
-
if (dir !== undefined) file(subtrees
|
|
1213
|
+
if (dir !== undefined) file(subtrees[s]!, dir, i)
|
|
1005
1214
|
prefixes.push([i, staticPrefix(tree)])
|
|
1006
1215
|
const head = tree.slice(0, tree.search(GLOB_HEAD_END))
|
|
1007
|
-
file(globsUnder
|
|
1216
|
+
file(globsUnder[s]!, head.slice(0, Math.max(0, head.lastIndexOf('/'))), i)
|
|
1008
1217
|
}
|
|
1009
1218
|
}
|
|
1010
1219
|
const n = tasks.length
|
|
@@ -1012,34 +1221,38 @@ function overlapCandidates<T>(
|
|
|
1012
1221
|
const pair = (x: number, y: number): void => {
|
|
1013
1222
|
if (x !== y) pairs.add(x < y ? x * n + y : y * n + x)
|
|
1014
1223
|
}
|
|
1015
|
-
for (const list of globs
|
|
1224
|
+
for (const [tree, list] of globs[0]!) {
|
|
1225
|
+
const other = sides === 1 ? list : (globs[1]!.get(tree) ?? [])
|
|
1016
1226
|
for (let x = 0; x < list.length; x++) {
|
|
1017
|
-
for (let y = x + 1; y <
|
|
1227
|
+
for (let y = sides === 1 ? x + 1 : 0; y < other.length; y++) pair(list[x]!, other[y]!)
|
|
1018
1228
|
}
|
|
1019
1229
|
}
|
|
1020
1230
|
// `witnessed`: two globs meet only when one's literal prefix is the
|
|
1021
1231
|
// other's or under it, so each looks up the globs filed at its prefix
|
|
1022
1232
|
// and at each ancestor.
|
|
1023
|
-
const byPrefix =
|
|
1024
|
-
for (const [i, prefix] of prefixes) file(byPrefix
|
|
1233
|
+
const byPrefix = perSide()
|
|
1234
|
+
for (const [i, prefix] of prefixes) file(byPrefix[side[i]!]!, rootless(prefix), i)
|
|
1025
1235
|
for (const [i, raw] of prefixes) {
|
|
1236
|
+
const index = byPrefix[across(side[i]!)]!
|
|
1026
1237
|
const prefix = rootless(raw)
|
|
1027
|
-
for (const j of
|
|
1028
|
-
if (prefix !== '') for (const j of
|
|
1238
|
+
for (const j of index.get(prefix) ?? []) pair(i, j)
|
|
1239
|
+
if (prefix !== '') for (const j of index.get('') ?? []) pair(i, j)
|
|
1029
1240
|
for (let sep = prefix.indexOf('/'); sep !== -1; sep = prefix.indexOf('/', sep + 1)) {
|
|
1030
|
-
for (const j of
|
|
1241
|
+
for (const j of index.get(prefix.slice(0, sep)) ?? []) pair(i, j)
|
|
1031
1242
|
}
|
|
1032
1243
|
}
|
|
1033
1244
|
for (const [i, prefix] of prefixes) {
|
|
1034
|
-
|
|
1245
|
+
const index = subtrees[across(side[i]!)]!
|
|
1246
|
+
for (const j of index.get(prefix) ?? []) pair(i, j)
|
|
1035
1247
|
for (let sep = prefix.indexOf('/'); sep !== -1; sep = prefix.indexOf('/', sep + 1)) {
|
|
1036
|
-
for (const j of
|
|
1248
|
+
for (const j of index.get(prefix.slice(0, sep)) ?? []) pair(i, j)
|
|
1037
1249
|
}
|
|
1038
1250
|
}
|
|
1039
1251
|
for (const [i, path] of literals) {
|
|
1040
|
-
|
|
1252
|
+
const index = globsUnder[across(side[i]!)]!
|
|
1253
|
+
for (const j of index.get('') ?? []) pair(i, j)
|
|
1041
1254
|
for (let sep = path.indexOf('/'); sep !== -1; sep = path.indexOf('/', sep + 1)) {
|
|
1042
|
-
for (const j of
|
|
1255
|
+
for (const j of index.get(path.slice(0, sep)) ?? []) pair(i, j)
|
|
1043
1256
|
}
|
|
1044
1257
|
}
|
|
1045
1258
|
return [...pairs].sort((x, y) => x - y).map((p) => [Math.floor(p / n), p % n])
|
|
@@ -1236,7 +1449,11 @@ function detectInputOverlaps(nodes: Map<string, TaskNode>, workspaceRoot?: strin
|
|
|
1236
1449
|
if (projectReaders) {
|
|
1237
1450
|
for (const sides of byProject.values()) {
|
|
1238
1451
|
if (sides.length < 2) continue
|
|
1239
|
-
for (const [i, j] of overlapCandidates(
|
|
1452
|
+
for (const [i, j] of overlapCandidates(
|
|
1453
|
+
sides,
|
|
1454
|
+
(s) => s.globs,
|
|
1455
|
+
(s) => s.reads,
|
|
1456
|
+
)) {
|
|
1240
1457
|
readsOutputs(sides[i]!, sides[j]!, 'files')
|
|
1241
1458
|
}
|
|
1242
1459
|
}
|
|
@@ -1251,7 +1468,11 @@ function detectInputOverlaps(nodes: Map<string, TaskNode>, workspaceRoot?: strin
|
|
|
1251
1468
|
rooted.push({ node, globs, reads: false, rel })
|
|
1252
1469
|
}
|
|
1253
1470
|
}
|
|
1254
|
-
for (const [i, j] of overlapCandidates(
|
|
1471
|
+
for (const [i, j] of overlapCandidates(
|
|
1472
|
+
rooted,
|
|
1473
|
+
(s) => s.globs,
|
|
1474
|
+
(s) => s.reads,
|
|
1475
|
+
)) {
|
|
1255
1476
|
readsOutputs(rooted[i]!, rooted[j]!, 'workspaceFiles')
|
|
1256
1477
|
}
|
|
1257
1478
|
}
|
|
@@ -27,7 +27,10 @@ import { computeTaskHash, type ComputeHashArgs } from './task-hash.js'
|
|
|
27
27
|
|
|
28
28
|
/** What `taintTracker` answers per dispatch, and what it learns per finish. */
|
|
29
29
|
export interface TaintTracker {
|
|
30
|
-
/**
|
|
30
|
+
/**
|
|
31
|
+
* Whether `node`, dispatched with these key `upstream` outcomes, builds on
|
|
32
|
+
* bytes nothing vouches for. Its order-only deps count too.
|
|
33
|
+
*/
|
|
31
34
|
judge: (node: TaskNode, upstream: TaskOutcome[]) => boolean
|
|
32
35
|
/** Every settled outcome, the scheduler's `onFinish`. */
|
|
33
36
|
settled: (outcome: TaskOutcome) => void
|
|
@@ -102,8 +105,14 @@ export function taintTracker(
|
|
|
102
105
|
}
|
|
103
106
|
return {
|
|
104
107
|
// A restore-tier task may run before its deps and see holes here; it
|
|
105
|
-
// never saves anyway (a hit restores), so a hole is not taint.
|
|
106
|
-
|
|
108
|
+
// never saves anyway (a hit restores), so a hole is not taint. The
|
|
109
|
+
// upstream is the key's, which drops order-only edges; their failure
|
|
110
|
+
// reaches the task through the dependency `--exclude-dependencies`
|
|
111
|
+
// took out, so they are read from the settled outcomes.
|
|
112
|
+
judge: (node, upstream) =>
|
|
113
|
+
seeds.has(node.id) ||
|
|
114
|
+
upstream.some(through) ||
|
|
115
|
+
node.orderOnly?.some((d) => through(outcomes.get(d))) === true,
|
|
107
116
|
settled: (outcome) => {
|
|
108
117
|
outcomes.set(outcome.node.id, outcome)
|
|
109
118
|
},
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// behind `^build` runs when `ui#build` (or what it depends on) is reached,
|
|
5
5
|
// and a spec edit `ui#build`'s inputs leave out stops at `ui`'s own tasks.
|
|
6
6
|
|
|
7
|
-
import { declaresInput } from '../cache/index.js'
|
|
7
|
+
import { declaresInput, workspaceFilesReachInto } from '../cache/index.js'
|
|
8
8
|
import {
|
|
9
9
|
compileTaskPattern,
|
|
10
10
|
isGroupTask,
|
|
@@ -91,7 +91,11 @@ export function affectedRoots(
|
|
|
91
91
|
const cache = n.config.cache
|
|
92
92
|
// Asked of every node, not only the changed projects' (the
|
|
93
93
|
// `workspaceFiles` owners): a `graph` hook may have given the glob.
|
|
94
|
-
if (
|
|
94
|
+
if (
|
|
95
|
+
cache !== undefined &&
|
|
96
|
+
(changes.changed.some((rel) => declaresInput(cache, null, rel)) ||
|
|
97
|
+
(changes.nested ?? []).some((dir) => workspaceFilesReachInto(cache, dir)))
|
|
98
|
+
) {
|
|
95
99
|
return true
|
|
96
100
|
}
|
|
97
101
|
if (!changes.projects.has(n.projectName)) return false
|
|
@@ -23,7 +23,8 @@ import type { TaskNode } from '../graph/index.js'
|
|
|
23
23
|
export interface DeferredEntry {
|
|
24
24
|
/** Fetches this task's outputs onto disk. At-most-once by construction. */
|
|
25
25
|
materialize: () => Promise<void>
|
|
26
|
-
|
|
26
|
+
/** The key a fetch saves the bytes under; absent when the inputs moved after it was taken. */
|
|
27
|
+
hash?: string
|
|
27
28
|
entry: { taskId: string; command: string; durationMs: number; stdout: string }
|
|
28
29
|
}
|
|
29
30
|
|
|
@@ -149,7 +150,7 @@ export class DeferredOutputs {
|
|
|
149
150
|
// Convergence: an ordinary entry, so the next run is a plain local hit
|
|
150
151
|
// and nothing downstream learns a second shape. Policy still governs —
|
|
151
152
|
// a run that writes no local entries writes none here either.
|
|
152
|
-
if (this.args.localWrite) {
|
|
153
|
+
if (this.args.localWrite && entry.hash !== undefined) {
|
|
153
154
|
await this.args.cache.save({
|
|
154
155
|
hash: entry.hash,
|
|
155
156
|
projectDir: producer.projectDir,
|
|
@@ -97,7 +97,7 @@ export interface InfoFacts {
|
|
|
97
97
|
hits24h: number
|
|
98
98
|
/** Of `hits24h`, the hits that restored outputs; the rest were up to date. */
|
|
99
99
|
restored24h: number
|
|
100
|
-
/** Tasks the retained history shows
|
|
100
|
+
/** Tasks the retained history shows failing on a key that had passed. */
|
|
101
101
|
flakyTasks: FlakyTask[]
|
|
102
102
|
lockfile: boolean
|
|
103
103
|
/**
|
|
@@ -256,7 +256,7 @@ async function collectWorkspaceInfo(
|
|
|
256
256
|
runs24h: stats.runCountLast24h,
|
|
257
257
|
hits24h: stats.hitCountLast24h,
|
|
258
258
|
restored24h: stats.restoredCountLast24h,
|
|
259
|
-
//
|
|
259
|
+
// A key that failed after it had passed — the history's definition of flaky, over
|
|
260
260
|
// the 30 days it keeps. A run names its own findings in its footer;
|
|
261
261
|
// this is the workspace's standing list.
|
|
262
262
|
flakyTasks: flaky,
|
|
@@ -13,6 +13,15 @@ import { maskedCommand, secretMask } from '../util/index.js'
|
|
|
13
13
|
import type { Logger } from './logger.js'
|
|
14
14
|
import type { RunContext } from './summary.js'
|
|
15
15
|
|
|
16
|
+
/**
|
|
17
|
+
* A task no cache answered for: it declares no `cache`, or the run's policy
|
|
18
|
+
* read and wrote nothing. Its run is `no-cache`, never a miss, on every
|
|
19
|
+
* surface — `--dry`'s plan, the frame, the footer, `--report`, `--summarize`.
|
|
20
|
+
*/
|
|
21
|
+
export function ranNoCache(o: { node: TaskNode; cacheOff?: boolean }): boolean {
|
|
22
|
+
return o.node.config.cache === undefined || o.cacheOff === true
|
|
23
|
+
}
|
|
24
|
+
|
|
16
25
|
/** Payload of the `run:start` event — mirrors the Logger.runStart hook. */
|
|
17
26
|
export interface RunStartInfo {
|
|
18
27
|
total: number
|
|
@@ -373,7 +382,7 @@ export function projectOutcome(outcome: TaskOutcome): OutcomeView {
|
|
|
373
382
|
durationMs: outcome.durationMs,
|
|
374
383
|
}
|
|
375
384
|
if (isGroupTask(outcome.node)) view.isGroup = true
|
|
376
|
-
else if (outcome
|
|
385
|
+
else if (ranNoCache(outcome)) view.noCache = true
|
|
377
386
|
if (outcome.storedDurationMs !== undefined) view.storedDurationMs = outcome.storedDurationMs
|
|
378
387
|
if (outcome.storedCpuMs !== undefined) view.storedCpuMs = outcome.storedCpuMs
|
|
379
388
|
if (outcome.storedPeakRssBytes !== undefined) view.storedPeakRssBytes = outcome.storedPeakRssBytes
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
import type { CacheLayer, GitFilesCache } from '../cache/index.js'
|
|
17
17
|
import { isGroupTask, type TaskNode, type TaskOutcome } from '../graph/index.js'
|
|
18
18
|
import { computeGroupKey, computeTaskHash, type HashCache } from './task-hash.js'
|
|
19
|
-
import { filterUpstreamHashes, keyedDeps } from './upstream.js'
|
|
19
|
+
import { filterUpstreamHashes, keyedDeps, keyedOutcome } from './upstream.js'
|
|
20
20
|
|
|
21
21
|
export interface KeyExcludedArgs {
|
|
22
22
|
/** The scheduled graph, after `excludeDependencies`. */
|
|
@@ -50,7 +50,7 @@ export async function keyExcludedDependencies(args: KeyExcludedArgs): Promise<vo
|
|
|
50
50
|
// (item 737). The keys still resolve concurrently.
|
|
51
51
|
const keys = new Map<string, Promise<string | undefined>>()
|
|
52
52
|
const outcomes = (ids: readonly string[]): Promise<TaskOutcome[]> =>
|
|
53
|
-
Promise.all(ids.map(async (id) =>
|
|
53
|
+
Promise.all(ids.map(async (id) => keyedOutcome(nodeOf(id), await keys.get(id)!)))
|
|
54
54
|
const derive = async (node: TaskNode): Promise<string | undefined> => {
|
|
55
55
|
const upstream = await outcomes(fullDeps(node))
|
|
56
56
|
return (isGroupTask(node) ? computeGroupKey : computeTaskHash)({
|
|
@@ -139,14 +139,3 @@ function foldsExcludedKey(node: TaskNode): boolean {
|
|
|
139
139
|
.length > 0
|
|
140
140
|
)
|
|
141
141
|
}
|
|
142
|
-
|
|
143
|
-
/** A dependency as a key reads it: only `node` and `hash` matter. */
|
|
144
|
-
function synthetic(node: TaskNode, hash: string | undefined): TaskOutcome {
|
|
145
|
-
return {
|
|
146
|
-
node,
|
|
147
|
-
status: 'success',
|
|
148
|
-
exitCode: 0,
|
|
149
|
-
durationMs: 0,
|
|
150
|
-
...(hash !== undefined ? { hash } : {}),
|
|
151
|
-
}
|
|
152
|
-
}
|