@vzn/vx 0.0.639 → 0.0.640
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/docs/caching.md +111 -83
- package/docs/cli.md +70 -40
- package/docs/execution.md +4 -4
- package/docs/features.md +9 -8
- package/docs/patterns.md +1 -1
- package/docs/schema.md +235 -180
- package/docs/security.md +1 -1
- package/package.json +5 -5
- package/schemas/show.json +14 -14
- package/skills/vx/SKILL.md +4 -3
- package/src/cache/cache.ts +4 -1
- package/src/cache/file-hashes.ts +7 -12
- package/src/cache/fs-clock.ts +62 -0
- package/src/cache/git-inputs.ts +2 -2
- package/src/cache/index.ts +3 -0
- package/src/cache/inputs.ts +154 -83
- package/src/cache/key-fold.ts +23 -3
- package/src/cache/layer.ts +8 -7
- package/src/cache/linked-inputs.ts +87 -0
- package/src/cli/foreign-flags.ts +1 -1
- package/src/cli/init.ts +23 -11
- package/src/cli/last.ts +24 -1
- package/src/cli/plugin-templates.ts +2 -2
- package/src/cli/select.ts +16 -6
- package/src/cli/show.ts +9 -6
- package/src/cli/upgrade.ts +95 -19
- package/src/cli/watch-filter.ts +22 -6
- package/src/cli/watch-judge.ts +49 -5
- package/src/cli/watch-set.ts +18 -8
- package/src/cli/watch.ts +74 -2
- package/src/cli/why.ts +5 -3
- package/src/config.ts +19 -15
- package/src/exec/executor.ts +4 -4
- package/src/graph/dependency-spec.ts +2 -2
- package/src/graph/folds.ts +236 -0
- package/src/graph/index.ts +8 -1
- package/src/graph/scheduler.ts +41 -15
- package/src/graph/task-graph.ts +135 -73
- package/src/index.ts +1 -1
- package/src/orchestrator/affected-tasks.ts +20 -19
- package/src/orchestrator/download-policy.ts +16 -22
- package/src/orchestrator/excluded-keys.ts +4 -5
- package/src/orchestrator/execute-task.ts +29 -25
- package/src/orchestrator/index.ts +1 -0
- package/src/orchestrator/keyed-projects.ts +12 -51
- package/src/orchestrator/local-shortcircuit.ts +1 -1
- package/src/orchestrator/persistent-task.ts +3 -3
- package/src/orchestrator/placement.ts +2 -2
- package/src/orchestrator/plan.ts +1 -1
- package/src/orchestrator/plugin-host.ts +62 -22
- package/src/orchestrator/plugin.ts +12 -2
- package/src/orchestrator/prepare.ts +12 -5
- package/src/orchestrator/projects.ts +7 -10
- package/src/orchestrator/run-report.ts +124 -46
- package/src/orchestrator/run-steps.ts +57 -0
- package/src/orchestrator/run.ts +51 -21
- package/src/orchestrator/sandbox-fix.ts +1 -1
- package/src/orchestrator/sandbox-request.ts +2 -2
- package/src/orchestrator/stable-keys.ts +59 -26
- package/src/orchestrator/steps.ts +26 -6
- package/src/orchestrator/task-hash.ts +31 -9
- package/src/orchestrator/telemetry-host.ts +3 -3
- package/src/orchestrator/telemetry.ts +9 -4
- package/src/orchestrator/upstream.ts +4 -181
- package/src/report/framed-output.ts +1 -1
- package/src/report/status-line.ts +7 -3
- package/src/settings.ts +11 -17
- package/src/util/paths.ts +1 -1
- package/src/workspace/affected.ts +81 -2
- package/src/workspace/config-imports.ts +1 -1
- package/src/workspace/config-schema.ts +142 -143
- package/src/workspace/lockfile.ts +1 -1
- package/src/workspace/migration.ts +19 -3
|
@@ -14,9 +14,9 @@
|
|
|
14
14
|
// full run looks up.
|
|
15
15
|
|
|
16
16
|
import type { CacheLayer, GitFilesCache } from '../cache/index.js'
|
|
17
|
-
import { isGroupTask, type TaskNode, type TaskOutcome } from '../graph/index.js'
|
|
17
|
+
import { isGroupTask, keyedDeps, type TaskNode, type TaskOutcome } from '../graph/index.js'
|
|
18
18
|
import { computeGroupKey, computeTaskHash, type HashCache } from './task-hash.js'
|
|
19
|
-
import { filterUpstreamHashes,
|
|
19
|
+
import { filterUpstreamHashes, keyedOutcome } from './upstream.js'
|
|
20
20
|
|
|
21
21
|
export interface KeyExcludedArgs {
|
|
22
22
|
/** The scheduled graph, after `excludeDependencies`. */
|
|
@@ -124,7 +124,7 @@ export function excludedTaint(nodes: ReadonlyMap<string, TaskNode>): {
|
|
|
124
124
|
|
|
125
125
|
/**
|
|
126
126
|
* Whether `node`'s key folds a dropped dependency's key: a group folds all
|
|
127
|
-
* of its dependencies, an exec task what its `
|
|
127
|
+
* of its dependencies, an exec task what its `inputs.tasks` selects.
|
|
128
128
|
* A task with `tasks: []` folds none, so its entry claims nothing about a
|
|
129
129
|
* dependency and it saves as usual.
|
|
130
130
|
*/
|
|
@@ -133,7 +133,6 @@ function foldsExcludedKey(node: TaskNode): boolean {
|
|
|
133
133
|
if (excluded === undefined) return false
|
|
134
134
|
if (isGroupTask(node)) return true
|
|
135
135
|
return (
|
|
136
|
-
filterUpstreamHashes(excluded, node.config.
|
|
137
|
-
.length > 0
|
|
136
|
+
filterUpstreamHashes(excluded, node.config.inputs?.tasks, node.projectName, node.id).length > 0
|
|
138
137
|
)
|
|
139
138
|
}
|
|
@@ -84,6 +84,7 @@ import {
|
|
|
84
84
|
describeTaskInputs,
|
|
85
85
|
type HashCache,
|
|
86
86
|
type InputFact,
|
|
87
|
+
isKeyless,
|
|
87
88
|
movedInput,
|
|
88
89
|
type UpfrontGroupKey,
|
|
89
90
|
} from './task-hash.js'
|
|
@@ -252,9 +253,10 @@ export interface ExecuteArgs {
|
|
|
252
253
|
/**
|
|
253
254
|
* The run keeps its bytes in a remote (run-store.ts): an attempt captures
|
|
254
255
|
* its output at the entry's bounds, and an outcome that saves no entry
|
|
255
|
-
*
|
|
256
|
+
* carries it as `runLog`; a failure's only when `failures`
|
|
257
|
+
* (`cache.upload.failures`).
|
|
256
258
|
*/
|
|
257
|
-
|
|
259
|
+
keepRun?: { failures: boolean }
|
|
258
260
|
/**
|
|
259
261
|
* Whether a task has rewritten a root file a plugin claims (whose bytes
|
|
260
262
|
* the claimant folded) since the run read it (fingerprint-watch.ts): past that, no key taken
|
|
@@ -394,7 +396,7 @@ function fingerprintMoved(args: ExecuteArgs): boolean {
|
|
|
394
396
|
* through us.
|
|
395
397
|
*/
|
|
396
398
|
async function executeGroupTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
397
|
-
if (args.node.config.
|
|
399
|
+
if (args.node.config.inputs === undefined) {
|
|
398
400
|
return unkeyedGroupOutcome(
|
|
399
401
|
args.node,
|
|
400
402
|
args.upstream,
|
|
@@ -417,7 +419,7 @@ async function executeGroupTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
417
419
|
}
|
|
418
420
|
|
|
419
421
|
/**
|
|
420
|
-
* An unkeyed group's outcome (no `
|
|
422
|
+
* An unkeyed group's outcome (no `inputs`, no key of its own): its key is
|
|
421
423
|
* its upstream's alone (`computeGroupKey`), so the up-front one stands when
|
|
422
424
|
* that upstream is unchanged. Synchronous: the scheduler settles such a
|
|
423
425
|
* group in place (`ScheduleOptions.settleNow`), and `upstream` is the key's
|
|
@@ -490,7 +492,17 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
490
492
|
// exists to prevent), no output clean, no local artifact save. Its result
|
|
491
493
|
// lives in the remote executor's own store.
|
|
492
494
|
const remoteOnly = args.remoteOnly === true
|
|
493
|
-
|
|
495
|
+
// A key that folds a task with no inputs is this run's alone (`isKeyless`):
|
|
496
|
+
// nothing to read under it, and nothing a later run could find.
|
|
497
|
+
const present = upstream.filter((u): u is TaskOutcome => u !== undefined)
|
|
498
|
+
const folded = new Set(
|
|
499
|
+
filterUpstreamHashes(present, cfg.inputs?.tasks, node.projectName, node.id).map(([id]) => id),
|
|
500
|
+
)
|
|
501
|
+
const foldedTasks = expandGroupUpstream(present.filter((u) => folded.has(u.node.id)))
|
|
502
|
+
const keyless =
|
|
503
|
+
isKeyless(cfg) || foldedTasks.some((u) => u.keyless === true || isKeyless(u.node.config))
|
|
504
|
+
const reads = policy.localRead || policy.remoteRead
|
|
505
|
+
const willRead = !remoteOnly && cfgCacheable && !keyless && reads
|
|
494
506
|
const willWrite = !remoteOnly && cfgCacheable && (policy.localWrite || policy.remoteWrite)
|
|
495
507
|
// `willWrite` governs output hygiene (clean before exec); the save itself
|
|
496
508
|
// is also withheld from a task downstream of a failure (see `taintedUpstream`),
|
|
@@ -503,17 +515,9 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
503
515
|
// (`tasks: []` and a file input) is judged by its bytes. `upstream` holds
|
|
504
516
|
// a hole where a dependency did not run (a skipped one, a restore-tier
|
|
505
517
|
// task's unfinished deps).
|
|
506
|
-
const
|
|
507
|
-
const folded = new Set(
|
|
508
|
-
filterUpstreamHashes(present, cacheCfg?.inputs?.tasks, node.projectName, node.id).map(
|
|
509
|
-
([id]) => id,
|
|
510
|
-
),
|
|
511
|
-
)
|
|
512
|
-
const unkeyedUpstream = expandGroupUpstream(present.filter((u) => folded.has(u.node.id))).find(
|
|
513
|
-
(u) => u.unkeyed === true,
|
|
514
|
-
)
|
|
518
|
+
const unkeyedUpstream = foldedTasks.find((u) => u.unkeyed === true)
|
|
515
519
|
let unkeyed = unkeyedUpstream !== undefined
|
|
516
|
-
const willSave = willWrite && args.taintedUpstream !== true && !unkeyed
|
|
520
|
+
const willSave = willWrite && args.taintedUpstream !== true && !unkeyed && !keyless
|
|
517
521
|
|
|
518
522
|
// The executor retains nothing: the entry's output is the attempt's
|
|
519
523
|
// `output` log, both streams in the order they reached the logger, kept
|
|
@@ -615,7 +619,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
615
619
|
const taskStartNs = process.hrtime.bigint() - args.runStartHrTimeNs
|
|
616
620
|
|
|
617
621
|
// Hash is computed mid-run, not at prepareRun time. Tasks whose
|
|
618
|
-
// `
|
|
622
|
+
// `inputs.files` matches sibling outputs (e.g. `'**/*'` after
|
|
619
623
|
// a `codegen` step has written `generated.txt`) need the upstream
|
|
620
624
|
// outputs ALREADY on disk when their hash is computed — so we
|
|
621
625
|
// can't lift this into prepareRun. Same model as Turbo / Nx.
|
|
@@ -812,13 +816,10 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
812
816
|
// is a hit. Almost always a glob written against the wrong directory
|
|
813
817
|
// (`lib/**` in a package that builds `src/`). Said once per miss, on the
|
|
814
818
|
// run's status line — the task itself succeeds.
|
|
815
|
-
const declaredInputs = [
|
|
816
|
-
...(cacheCfg?.inputs.files ?? []),
|
|
817
|
-
...(cacheCfg?.inputs.workspaceFiles ?? []),
|
|
818
|
-
]
|
|
819
|
+
const declaredInputs = [...(cfg.inputs?.files ?? []), ...(cfg.inputs?.workspaceFiles ?? [])]
|
|
819
820
|
if (inputs !== undefined && declaredInputs.length > 0 && inputs.files.length === 0) {
|
|
820
821
|
warnOf(log)(
|
|
821
|
-
`[vx] ${node.id}:
|
|
822
|
+
`[vx] ${node.id}: inputs matched no files (${declaredInputs.join(', ')}) — ` +
|
|
822
823
|
`the key will not change when this project's source does`,
|
|
823
824
|
)
|
|
824
825
|
}
|
|
@@ -1177,7 +1178,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1177
1178
|
|
|
1178
1179
|
async function buildRequest(local: boolean): Promise<ExecuteRequest> {
|
|
1179
1180
|
const kept =
|
|
1180
|
-
willSave || args.
|
|
1181
|
+
willSave || args.keepRun !== undefined
|
|
1181
1182
|
? new BoundedCapture()
|
|
1182
1183
|
: new BoundedCapture(FAILED_OUTPUT_HEAD_CHARS, FAILED_OUTPUT_TAIL_CHARS)
|
|
1183
1184
|
output = kept
|
|
@@ -1456,11 +1457,14 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1456
1457
|
: {}),
|
|
1457
1458
|
...(inputChanges !== undefined ? { inputChanges } : {}),
|
|
1458
1459
|
...(unkeyed ? { unkeyed: true as const } : {}),
|
|
1460
|
+
...(keyless ? { keyless: true as const } : {}),
|
|
1459
1461
|
...(saved ? { saved: true as const } : {}),
|
|
1460
1462
|
...(effectiveExitCode !== 0 && output !== undefined
|
|
1461
|
-
? { failedOutput: failedOutputLog(output, willSave || args.
|
|
1463
|
+
? { failedOutput: failedOutputLog(output, willSave || args.keepRun !== undefined) }
|
|
1462
1464
|
: {}),
|
|
1463
|
-
...(args.
|
|
1465
|
+
...(args.keepRun !== undefined &&
|
|
1466
|
+
(effectiveExitCode !== 0 ? args.keepRun.failures : !willSave) &&
|
|
1467
|
+
output !== undefined
|
|
1464
1468
|
? { runLog: storedOutput() }
|
|
1465
1469
|
: {}),
|
|
1466
1470
|
...(result.timedOut === true && effectiveExitCode !== 0 ? { timedOut: true as const } : {}),
|
|
@@ -1484,7 +1488,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1484
1488
|
finalViolations,
|
|
1485
1489
|
node.projectDir,
|
|
1486
1490
|
(t) => secrets?.mask(t) ?? t,
|
|
1487
|
-
|
|
1491
|
+
cfg.inputs?.files,
|
|
1488
1492
|
),
|
|
1489
1493
|
),
|
|
1490
1494
|
}
|
|
@@ -125,5 +125,6 @@ export {
|
|
|
125
125
|
type TaskLog,
|
|
126
126
|
taskLog,
|
|
127
127
|
} from './run-failures.js'
|
|
128
|
+
export { type RunStep, runSteps } from './run-steps.js'
|
|
128
129
|
export { resolveRunId, shortRunId } from './run-id.js'
|
|
129
130
|
export { collectInfo, type CollectInfoOptions, type InfoFacts } from './doctor.js'
|
|
@@ -5,8 +5,7 @@
|
|
|
5
5
|
// is decided — on the graph, before any hash exists, through the same
|
|
6
6
|
// selection the hash path applies (`selectFoldedDeps`).
|
|
7
7
|
|
|
8
|
-
import { isGroupTask, type TaskNode } from '../graph/index.js'
|
|
9
|
-
import { keyedDeps, selectFoldedDeps, type FoldCandidate } from './upstream.js'
|
|
8
|
+
import { foldedDeps, isGroupTask, type FoldCandidate, type TaskNode } from '../graph/index.js'
|
|
10
9
|
|
|
11
10
|
/**
|
|
12
11
|
* A run's keyed-set lookup: for a task, the directories of every project
|
|
@@ -14,18 +13,17 @@ import { keyedDeps, selectFoldedDeps, type FoldCandidate } from './upstream.js'
|
|
|
14
13
|
* task itself is not counted.
|
|
15
14
|
*
|
|
16
15
|
* The fold relation is the hash path's, computed on the graph:
|
|
17
|
-
* - an exec task folds its dependencies as its own `
|
|
18
|
-
* selects them (absent
|
|
19
|
-
* - a group folds every dependency (`computeGroupKey`)
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
* - a persistent task is keyed as
|
|
16
|
+
* - an exec task folds its dependencies as its own `inputs.tasks`
|
|
17
|
+
* selects them (absent means all);
|
|
18
|
+
* - a group folds every dependency (`computeGroupKey`); a keyed one
|
|
19
|
+
* (`inputs`, the default `build`) folds its project's files as a task
|
|
20
|
+
* does;
|
|
21
|
+
* - a persistent task is keyed as any task with no `cache` (A-17).
|
|
23
22
|
*
|
|
24
|
-
* Every exec task reached contributes its project, cached or not:
|
|
25
|
-
* folds
|
|
26
|
-
*
|
|
27
|
-
* `
|
|
28
|
-
* key itself.
|
|
23
|
+
* Every exec task reached contributes its project, cached or not: its key
|
|
24
|
+
* folds its own project's files through its `inputs`, and one with no
|
|
25
|
+
* `inputs` is keyless, so a key that folds it is never stored or read.
|
|
26
|
+
* `tests/keyed-projects.test.ts` holds that to the key itself.
|
|
29
27
|
*
|
|
30
28
|
* Lazy and memoized by task id: only a sandboxed task that executes asks,
|
|
31
29
|
* so a run of hits walks nothing, and a shared subgraph is walked once.
|
|
@@ -57,7 +55,7 @@ export function keyedProjects(
|
|
|
57
55
|
stack.pop()
|
|
58
56
|
const out = new Set<string>()
|
|
59
57
|
for (const { node: dep } of deps) {
|
|
60
|
-
if (!isGroupTask(dep) || dep.config.
|
|
58
|
+
if (!isGroupTask(dep) || dep.config.inputs !== undefined) out.add(dep.projectDir)
|
|
61
59
|
for (const dir of below.get(dep.id)!) out.add(dir)
|
|
62
60
|
}
|
|
63
61
|
below.set(node.id, out)
|
|
@@ -65,40 +63,3 @@ export function keyedProjects(
|
|
|
65
63
|
return below.get(root.id)!
|
|
66
64
|
}
|
|
67
65
|
}
|
|
68
|
-
|
|
69
|
-
/**
|
|
70
|
-
* The dependencies `node`'s key folds, per the hash path's rules above, as
|
|
71
|
-
* `keyUpstream` hands them to the key: the scheduled ones less the edges
|
|
72
|
-
* `--exclude-dependencies` left for order alone, plus the dropped ones it
|
|
73
|
-
* keys all the same (`excludedUpstream`), whose own dependencies the walk
|
|
74
|
-
* reads from the run's `keyOnly` map. `node.deps` held the order-only
|
|
75
|
-
* edges and lacked the dropped ones: the sandbox granted a sibling the key
|
|
76
|
-
* does not answer for.
|
|
77
|
-
*/
|
|
78
|
-
export function foldedDeps(node: TaskNode, nodeOf: (id: string) => TaskNode): FoldCandidate[] {
|
|
79
|
-
const candidates: FoldCandidate[] = []
|
|
80
|
-
for (const id of keyedDeps(node)) {
|
|
81
|
-
const dep = nodeOf(id)
|
|
82
|
-
candidates.push({ node: dep, unit: foldUnit(dep) })
|
|
83
|
-
}
|
|
84
|
-
for (const { node: dep } of node.excludedUpstream ?? []) {
|
|
85
|
-
candidates.push({ node: dep, unit: foldUnit(dep) })
|
|
86
|
-
}
|
|
87
|
-
if (isGroupTask(node) && node.config.cache === undefined) return candidates
|
|
88
|
-
return selectFoldedDeps(candidates, node.config.cache?.inputs?.tasks, node.projectName, node.id)
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
/**
|
|
92
|
-
* What stands in for a dependency's hash in the dedup: equal exactly when
|
|
93
|
-
* the hashes are. An exec task's key folds its own id (`task:<id>`), so its
|
|
94
|
-
* id is enough; a group's hash is a roll-up of its members' ids and hashes
|
|
95
|
-
* with no id of its own, so two groups over the same members hash alike.
|
|
96
|
-
* The members are `keyUpstream`'s, which `computeGroupHash` is handed: under
|
|
97
|
-
* `--exclude-dependencies` `node.deps` holds order-only edges and lacks the
|
|
98
|
-
* dropped ones, and two groups the key told apart shared a unit.
|
|
99
|
-
*/
|
|
100
|
-
function foldUnit(node: TaskNode): string {
|
|
101
|
-
if (!isGroupTask(node) || node.config.cache !== undefined) return `task|${node.id}`
|
|
102
|
-
const members = [...keyedDeps(node), ...(node.excludedUpstream ?? []).map((u) => u.node.id)]
|
|
103
|
-
return `group|${members.sort().join('|')}`
|
|
104
|
-
}
|
|
@@ -232,7 +232,7 @@ export function restoreTierExclusions(
|
|
|
232
232
|
const dependants = new Map<string, string[]>()
|
|
233
233
|
for (const node of nodes.values()) {
|
|
234
234
|
const dir = relPosix(workspaceRoot, node.projectDir)
|
|
235
|
-
const wsInputs = node.config.
|
|
235
|
+
const wsInputs = node.config.inputs?.workspaceFiles ?? []
|
|
236
236
|
if (reaches(dir) || (wsInputs.length > 0 && workspaceInputsReach(wsInputs, prefixes))) {
|
|
237
237
|
out.add(node.id)
|
|
238
238
|
}
|
|
@@ -55,9 +55,9 @@ export async function executePersistentTask(args: ExecuteArgs): Promise<TaskOutc
|
|
|
55
55
|
const effectiveForwardArgs = node.requested ? (args.forwardArgs ?? []) : []
|
|
56
56
|
const env = taskEnv(node, step, args.workspaceRoot)
|
|
57
57
|
const wallclockStartNs = process.hrtime.bigint() - args.runStartHrTimeNs
|
|
58
|
-
// Keyed
|
|
59
|
-
//
|
|
60
|
-
// key the dependant
|
|
58
|
+
// Keyed on its inputs like any task, so a cached dependant behind a dev
|
|
59
|
+
// server re-runs when the server's sources change; with none it has no
|
|
60
|
+
// key and the dependant always runs (A-17).
|
|
61
61
|
const hash =
|
|
62
62
|
args.noDependants === true
|
|
63
63
|
? undefined
|
|
@@ -59,7 +59,7 @@ export function pinnedLocalSet(nodes: Map<string, TaskNode>): Set<string> {
|
|
|
59
59
|
}
|
|
60
60
|
|
|
61
61
|
/**
|
|
62
|
-
* A key that folds `
|
|
62
|
+
* A key that folds `inputs.runtime` / `workspaceRuntime` holds what
|
|
63
63
|
* THIS machine answered (`node -v`); a worker runs its own runtime, which no
|
|
64
64
|
* executor can prove equal, and its output saved under this key is a stale
|
|
65
65
|
* hit here (C-2). Such a task runs here. Its dependants are not pinned with
|
|
@@ -68,7 +68,7 @@ export function pinnedLocalSet(nodes: Map<string, TaskNode>): Set<string> {
|
|
|
68
68
|
*/
|
|
69
69
|
function withProbedRuntime(nodes: Map<string, TaskNode>, pinned: Set<string>): Set<string> {
|
|
70
70
|
for (const node of nodes.values()) {
|
|
71
|
-
const inputs = node.config.
|
|
71
|
+
const inputs = node.config.inputs
|
|
72
72
|
if ((inputs?.runtime?.length ?? 0) > 0 || (inputs?.workspaceRuntime?.length ?? 0) > 0) {
|
|
73
73
|
pinned.add(node.id)
|
|
74
74
|
}
|
package/src/orchestrator/plan.ts
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
// restoreOutputs. Cache probing uses the byte-free `cache.has()`
|
|
7
7
|
// existence probe (a remote layer answers it its own way — no
|
|
8
8
|
// artifact download, no local ingest), so the plan is read-only — with
|
|
9
|
-
// one deliberate exception: `
|
|
9
|
+
// one deliberate exception: `inputs.runtime` / `workspaceRuntime`
|
|
10
10
|
// probe commands DO run, because predicting a task's key requires
|
|
11
11
|
// resolving them (same as a real run). Keep runtime inputs
|
|
12
12
|
// side-effect-free pure probes.
|
|
@@ -381,41 +381,81 @@ export async function claimedAffected(
|
|
|
381
381
|
return names
|
|
382
382
|
}
|
|
383
383
|
|
|
384
|
+
/** What a run hands the `schedule` stage: a step per plugin, and where refined weights go. */
|
|
385
|
+
export interface ScheduleRun {
|
|
386
|
+
readonly step?: <T>(id: string, work: () => Promise<T>) => Promise<T>
|
|
387
|
+
readonly refined?: (merged: ReadonlyMap<string, number>) => void
|
|
388
|
+
}
|
|
389
|
+
|
|
384
390
|
/**
|
|
385
391
|
* `schedule` stage: the merged priorities map (a later plugin's weight for
|
|
386
|
-
* a task overrides an earlier one's). Empty when no plugin answered.
|
|
392
|
+
* a task overrides an earlier one's). Empty when no plugin answered. Each
|
|
393
|
+
* plugin's call is the step `<plugin>:schedule`. A plugin's `ctx.refine`
|
|
394
|
+
* replaces its own weights and hands `refined` the merge again; with no
|
|
395
|
+
* `refined` (a plan), it is ignored.
|
|
387
396
|
*/
|
|
388
397
|
export async function applyScheduleHooks(
|
|
389
398
|
plugins: readonly VxPlugin[],
|
|
390
399
|
nodes: ReadonlyMap<string, TaskNode>,
|
|
391
|
-
ctx: Omit<ScheduleHookContext, 'cache'>,
|
|
400
|
+
ctx: Omit<ScheduleHookContext, 'cache' | 'refine'>,
|
|
392
401
|
cacheFor: (plugin: string) => PluginCache = () => absentPluginCache,
|
|
402
|
+
run: ScheduleRun = {},
|
|
393
403
|
): Promise<ReadonlyMap<string, number>> {
|
|
394
|
-
const
|
|
395
|
-
|
|
404
|
+
const sets: (ReadonlyMap<string, number> | undefined)[] = []
|
|
405
|
+
const merged = (): ReadonlyMap<string, number> => {
|
|
406
|
+
const out = new Map<string, number>()
|
|
407
|
+
for (const set of sets) if (set !== undefined) for (const [id, w] of set) out.set(id, w)
|
|
408
|
+
return out
|
|
409
|
+
}
|
|
410
|
+
for (const [i, plugin] of plugins.entries()) {
|
|
396
411
|
if (plugin.schedule === undefined) continue
|
|
397
|
-
const
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
412
|
+
const refine = (weights: ReadonlyMap<string, number>): void => {
|
|
413
|
+
if (run.refined === undefined) return
|
|
414
|
+
try {
|
|
415
|
+
sets[i] = scheduleWeights(plugin, weights, nodes)
|
|
416
|
+
} catch (err) {
|
|
417
|
+
ctx.warn(`[vx] ${err instanceof Error ? err.message : String(err)}; refine ignored`)
|
|
418
|
+
return
|
|
419
|
+
}
|
|
420
|
+
run.refined(merged())
|
|
421
|
+
}
|
|
422
|
+
const own = { ...ctx, cache: cacheFor(plugin.name), refine }
|
|
423
|
+
const call = (): Promise<ReadonlyMap<string, number> | undefined> =>
|
|
424
|
+
safe(plugin, 'schedule', () => plugin.schedule!(nodes, own))
|
|
425
|
+
const weights = await (run.step === undefined
|
|
426
|
+
? call()
|
|
427
|
+
: run.step(`${plugin.name}:schedule`, call))
|
|
428
|
+
// A refine that landed while the call ran is the newer set.
|
|
429
|
+
if (weights !== undefined) sets[i] ??= scheduleWeights(plugin, weights, nodes)
|
|
430
|
+
}
|
|
431
|
+
return merged()
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
/** A `schedule` answer's weights for tasks in the graph, refused by name when it is not a Map of finite numbers. */
|
|
435
|
+
function scheduleWeights(
|
|
436
|
+
plugin: VxPlugin,
|
|
437
|
+
weights: unknown,
|
|
438
|
+
nodes: ReadonlyMap<string, TaskNode>,
|
|
439
|
+
): ReadonlyMap<string, number> {
|
|
440
|
+
// Iterating a string destructures its characters into `[id, w]` pairs
|
|
441
|
+
// that match no task — a plugin returning `'fast'` used to be a no-op
|
|
442
|
+
// with no word said.
|
|
443
|
+
if (!(weights instanceof Map)) {
|
|
444
|
+
throw new UserError(
|
|
445
|
+
`plugin '${plugin.name}' failed in schedule: returned ${describeValue(weights)}, not a Map of task id → weight`,
|
|
446
|
+
)
|
|
447
|
+
}
|
|
448
|
+
const out = new Map<string, number>()
|
|
449
|
+
for (const [id, w] of weights as Map<unknown, unknown>) {
|
|
450
|
+
if (typeof id !== 'string' || !nodes.has(id)) continue
|
|
451
|
+
if (typeof w !== 'number' || !Number.isFinite(w)) {
|
|
404
452
|
throw new UserError(
|
|
405
|
-
`plugin '${plugin.name}' failed in schedule:
|
|
453
|
+
`plugin '${plugin.name}' failed in schedule: weight for ${id} is not a finite number`,
|
|
406
454
|
)
|
|
407
455
|
}
|
|
408
|
-
|
|
409
|
-
if (!nodes.has(id)) continue
|
|
410
|
-
if (typeof w !== 'number' || !Number.isFinite(w)) {
|
|
411
|
-
throw new UserError(
|
|
412
|
-
`plugin '${plugin.name}' failed in schedule: weight for ${id} is not a finite number`,
|
|
413
|
-
)
|
|
414
|
-
}
|
|
415
|
-
merged.set(id, w)
|
|
416
|
-
}
|
|
456
|
+
out.set(id, w)
|
|
417
457
|
}
|
|
418
|
-
return
|
|
458
|
+
return out
|
|
419
459
|
}
|
|
420
460
|
|
|
421
461
|
/**
|
|
@@ -137,6 +137,9 @@ export interface VxPlugin {
|
|
|
137
137
|
* only when a task will start: a run of confirmed local hits restores
|
|
138
138
|
* in no order a weight changes. A later plugin's weight for a task
|
|
139
139
|
* overrides an earlier one's. Return undefined to leave the baseline.
|
|
140
|
+
* The run waits for this answer before its first task, so it should
|
|
141
|
+
* come from what is at hand (local history); better weights that need
|
|
142
|
+
* a fetch go to `ctx.refine` when they arrive, without holding the run.
|
|
140
143
|
*/
|
|
141
144
|
schedule?(
|
|
142
145
|
nodes: ReadonlyMap<string, TaskNode>,
|
|
@@ -244,8 +247,9 @@ export interface VxPlugin {
|
|
|
244
247
|
/**
|
|
245
248
|
* What a finished run hands each plugin's `teardown`: its facts and every
|
|
246
249
|
* task, with the names core kept the task's bytes under. Core keeps them
|
|
247
|
-
* only when a remote takes the run's writes (CI by default): a
|
|
248
|
-
* task
|
|
250
|
+
* only when a remote takes the run's writes (CI by default): the log of a
|
|
251
|
+
* task that saved no entry, and a failed task's log and the files its
|
|
252
|
+
* outputs left when `cache.upload.failures` covers the run. Read
|
|
249
253
|
* them with `ctx.cache.getRun(name)`. Core keeps no manifest of its own; a
|
|
250
254
|
* plugin that wants one writes it from here (`ctx.cache.put`).
|
|
251
255
|
*/
|
|
@@ -448,6 +452,12 @@ export interface ScheduleHookContext extends BaseContext {
|
|
|
448
452
|
readonly branch: string | null
|
|
449
453
|
/** This plugin's entries in the workspace's cache, as `setup` has them. */
|
|
450
454
|
readonly cache: PluginCache
|
|
455
|
+
/**
|
|
456
|
+
* Replace this plugin's weights while the run goes on: the tasks not yet
|
|
457
|
+
* started re-rank at once. For weights that arrive late (timings fetched
|
|
458
|
+
* from a remote). Ignored in a plan and once the run's tasks are done.
|
|
459
|
+
*/
|
|
460
|
+
refine(weights: ReadonlyMap<string, number>): void
|
|
451
461
|
}
|
|
452
462
|
|
|
453
463
|
/** A `cache` hook's context: the workspace and the run's scope, never the local store or the policy, which core owns. */
|
|
@@ -31,6 +31,7 @@ import {
|
|
|
31
31
|
MAX_SCOPED_PATHSPECS,
|
|
32
32
|
startGitEnumeration,
|
|
33
33
|
lazyGitEnumeration,
|
|
34
|
+
measureFsClockLag,
|
|
34
35
|
} from '../cache/index.js'
|
|
35
36
|
import {
|
|
36
37
|
buildPackageGraph,
|
|
@@ -50,6 +51,7 @@ import {
|
|
|
50
51
|
expandRequested,
|
|
51
52
|
type GraphRules,
|
|
52
53
|
isGroupTask,
|
|
54
|
+
refuseFiltersThroughGroups,
|
|
53
55
|
type TaskNode,
|
|
54
56
|
undeclaredDepsError,
|
|
55
57
|
unresolvedRequests,
|
|
@@ -61,6 +63,7 @@ import {
|
|
|
61
63
|
hasHook,
|
|
62
64
|
resolveCache,
|
|
63
65
|
teardownPlugins,
|
|
66
|
+
type ScheduleRun,
|
|
64
67
|
} from './plugin-host.js'
|
|
65
68
|
import {
|
|
66
69
|
discoverProjects,
|
|
@@ -83,7 +86,6 @@ import { currentBranch } from './run-context.js'
|
|
|
83
86
|
import { createHashCache, type HashCache } from './task-hash.js'
|
|
84
87
|
import { type Logger, warnOf } from '../report/index.js'
|
|
85
88
|
import type { RunOptions } from './options.js'
|
|
86
|
-
import { refuseFiltersThroughGroups } from './upstream.js'
|
|
87
89
|
import { runRules } from '../settings.js'
|
|
88
90
|
|
|
89
91
|
export interface PreparedRun {
|
|
@@ -114,9 +116,10 @@ export interface PreparedRun {
|
|
|
114
116
|
* merged over the structural baseline by the scheduler). Empty when no
|
|
115
117
|
* plugin declares the stage. Asked only by a run with a task to start:
|
|
116
118
|
* the stage reads history (148 ms at 8,002 tasks) to order work a
|
|
117
|
-
* nothing-changed run never dispatches. Runs the stage once
|
|
119
|
+
* nothing-changed run never dispatches. Runs the stage once: `run`
|
|
120
|
+
* times each plugin's call and hears its later `ctx.refine` sets.
|
|
118
121
|
*/
|
|
119
|
-
priorities: () => Promise<ReadonlyMap<string, number>>
|
|
122
|
+
priorities: (run?: ScheduleRun) => Promise<ReadonlyMap<string, number>>
|
|
120
123
|
/**
|
|
121
124
|
* The run's branch (`run.branch`, else git's, else the CI provider's):
|
|
122
125
|
* what its history records and the timings a forecast reads first.
|
|
@@ -256,6 +259,9 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
256
259
|
const cacheDir = options.cacheDir
|
|
257
260
|
? path.resolve(options.cwd, options.cacheDir)
|
|
258
261
|
: resolveCacheDir(workspaceRoot, workspaceConfig)
|
|
262
|
+
// Before any racy window is judged: a file system behind this process's
|
|
263
|
+
// clock dated a mid-command edit before the command (fs-clock.ts).
|
|
264
|
+
measureFsClockLag(cacheDir)
|
|
259
265
|
// Asked now, read when the cache opens: a repository with no remote is
|
|
260
266
|
// named by its root commit, a git spawn the discovery below overlaps.
|
|
261
267
|
const storeRoot = options.cacheDir ? null : resolveStoreRoot(workspaceRoot, workspaceConfig)
|
|
@@ -668,7 +674,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
668
674
|
let usesWorkspaceInputs = false
|
|
669
675
|
for (const n of nodes.values()) {
|
|
670
676
|
graphDirs.add(n.projectDir)
|
|
671
|
-
if ((n.config.
|
|
677
|
+
if ((n.config.inputs?.workspaceFiles?.length ?? 0) > 0) usesWorkspaceInputs = true
|
|
672
678
|
}
|
|
673
679
|
const projectDirs = [...graphDirs]
|
|
674
680
|
const enumerating =
|
|
@@ -743,7 +749,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
743
749
|
return own
|
|
744
750
|
}
|
|
745
751
|
let scheduled: Promise<ReadonlyMap<string, number>> | undefined
|
|
746
|
-
const priorities = (): Promise<ReadonlyMap<string, number>> =>
|
|
752
|
+
const priorities = (run?: ScheduleRun): Promise<ReadonlyMap<string, number>> =>
|
|
747
753
|
(scheduled ??= hasHook(plugins, 'schedule')
|
|
748
754
|
? applyScheduleHooks(
|
|
749
755
|
plugins,
|
|
@@ -756,6 +762,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
756
762
|
branch: branch(),
|
|
757
763
|
},
|
|
758
764
|
cacheFor,
|
|
765
|
+
run,
|
|
759
766
|
)
|
|
760
767
|
: Promise.resolve(new Map()))
|
|
761
768
|
|
|
@@ -258,7 +258,7 @@ export async function loadProjects(args: LoadProjectsArgs): Promise<LoadedProjec
|
|
|
258
258
|
// Turbo spells the root package `//` (`dependsOn: ['//#lint']`), as vx
|
|
259
259
|
// names the root project unless something names it; a named root's
|
|
260
260
|
// spelling is rewritten to its name once, here, for the closure, the
|
|
261
|
-
// graph and `
|
|
261
|
+
// graph and `inputs.tasks` alike. With no root project it stays,
|
|
262
262
|
// and the graph's refusal says so.
|
|
263
263
|
const root = path.resolve(workspaceRoot)
|
|
264
264
|
const rootName = args.projectMetas.find((m) => path.resolve(m.dir) === root)?.name
|
|
@@ -357,10 +357,7 @@ export async function loadProjects(args: LoadProjectsArgs): Promise<LoadedProjec
|
|
|
357
357
|
*/
|
|
358
358
|
const DEFAULT_BUILD: TaskConfig = Object.freeze({
|
|
359
359
|
dependsOn: Object.freeze(['^build']),
|
|
360
|
-
|
|
361
|
-
inputs: Object.freeze({ files: Object.freeze(['**']) }),
|
|
362
|
-
outputs: Object.freeze({ files: Object.freeze([]) }),
|
|
363
|
-
}),
|
|
360
|
+
inputs: Object.freeze({ files: Object.freeze(['**']) }),
|
|
364
361
|
})
|
|
365
362
|
|
|
366
363
|
/** The default `build`, which a menu of the tasks a user declared leaves out. */
|
|
@@ -374,7 +371,7 @@ function withDefaultBuild(config: ProjectConfig): ProjectConfig {
|
|
|
374
371
|
}
|
|
375
372
|
|
|
376
373
|
/**
|
|
377
|
-
* `config` with Turbo's `//#task` (in `dependsOn` and `
|
|
374
|
+
* `config` with Turbo's `//#task` (in `dependsOn` and `inputs.tasks`)
|
|
378
375
|
* spelled `<root>#task`; the same object when it holds none. A copy, not
|
|
379
376
|
* an edit: a loaded config may be a cached or a frozen one.
|
|
380
377
|
*/
|
|
@@ -386,14 +383,14 @@ function rootSpelled(config: ProjectConfig, rootName: string): ProjectConfig {
|
|
|
386
383
|
let tasks: NonNullable<ProjectConfig['tasks']> | undefined
|
|
387
384
|
for (const [name, task] of Object.entries(config.tasks ?? {})) {
|
|
388
385
|
const dependsOn = fix(task.dependsOn)
|
|
389
|
-
const inputTasks = fix(task.
|
|
390
|
-
if (dependsOn === task.dependsOn && inputTasks === task.
|
|
386
|
+
const inputTasks = fix(task.inputs?.tasks)
|
|
387
|
+
if (dependsOn === task.dependsOn && inputTasks === task.inputs?.tasks) continue
|
|
391
388
|
tasks ??= { ...config.tasks }
|
|
392
389
|
tasks[name] = {
|
|
393
390
|
...task,
|
|
394
391
|
...(dependsOn !== undefined ? { dependsOn: [...dependsOn] } : {}),
|
|
395
|
-
...(task.
|
|
396
|
-
? {
|
|
392
|
+
...(task.inputs !== undefined && inputTasks !== undefined
|
|
393
|
+
? { inputs: { ...task.inputs, tasks: [...inputTasks] } }
|
|
397
394
|
: {}),
|
|
398
395
|
}
|
|
399
396
|
}
|