@vzn/vx 0.0.16 → 0.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -89
- package/index.ts +10 -0
- package/launcher.mjs +10 -11
- package/package.json +19 -5
- package/plugins/local-cache/index.ts +2 -0
- package/plugins/local-executor/index.ts +2 -0
- package/plugins/schedule-history/index.ts +2 -0
- package/src/bin.ts +2 -2
- package/src/cache/archive.ts +557 -0
- package/src/cache/cache.ts +929 -250
- package/src/cache/cas-backend.ts +20 -2
- package/src/cache/chained-cache.ts +202 -0
- package/src/cache/digest.ts +0 -15
- package/src/cache/index.ts +13 -3
- package/src/cache/inputs.ts +445 -41
- package/src/cache/layered-cache.ts +88 -30
- package/src/cache/tar-stream.ts +358 -0
- package/src/cli/cache.ts +36 -10
- package/src/cli/help.ts +167 -128
- package/src/cli/index.ts +74 -43
- package/src/cli/info.ts +30 -0
- package/src/cli/last.ts +140 -0
- package/src/cli/migrate-emit.ts +21 -0
- package/src/cli/migrate-nx.ts +23 -0
- package/src/cli/migrate-scripts.ts +127 -0
- package/src/cli/migrate-turbo.ts +43 -6
- package/src/cli/migrate.ts +119 -33
- package/src/cli/plan-format.ts +58 -3
- package/src/cli/plugin-commands.ts +96 -0
- package/src/cli/prune.ts +291 -0
- package/src/cli/run.ts +272 -80
- package/src/cli/upgrade.ts +7 -6
- package/src/cli/watch.ts +259 -61
- package/src/cli/why.ts +193 -0
- package/src/config.ts +40 -23
- package/src/exec/env.ts +7 -1
- package/src/exec/executor.ts +239 -0
- package/src/exec/index.ts +19 -1
- package/src/exec/runner.ts +116 -29
- package/src/exec/sandbox-runtime.ts +194 -24
- package/src/graph/index.ts +3 -1
- package/src/graph/scheduler.ts +92 -18
- package/src/graph/task-graph.ts +220 -6
- package/src/index.ts +82 -84
- package/src/orchestrator/deferred-outputs.ts +144 -0
- package/src/orchestrator/download-policy.ts +171 -0
- package/src/orchestrator/events.ts +30 -0
- package/src/orchestrator/execute-task.ts +328 -94
- package/src/orchestrator/failure-mode.ts +61 -0
- package/src/orchestrator/framed-output.ts +87 -35
- package/src/orchestrator/history.ts +78 -31
- package/src/orchestrator/index.ts +43 -90
- package/src/orchestrator/local-shortcircuit.ts +22 -0
- package/src/orchestrator/logger.ts +196 -14
- package/src/orchestrator/metrics.ts +57 -1376
- package/src/orchestrator/missing-plugin.ts +9 -0
- package/src/orchestrator/options.ts +22 -4
- package/src/orchestrator/plan.ts +144 -1
- package/src/orchestrator/plugin-host.ts +199 -119
- package/src/orchestrator/plugin.ts +144 -57
- package/src/orchestrator/prepare.ts +220 -69
- package/src/orchestrator/remote-prefetch.ts +12 -8
- package/src/orchestrator/resources.ts +20 -34
- package/src/orchestrator/run-artifacts.ts +41 -16
- package/src/orchestrator/run-context.ts +167 -29
- package/src/orchestrator/run-report.ts +73 -46
- package/src/orchestrator/run.ts +590 -122
- package/src/orchestrator/stable-keys.ts +74 -33
- package/src/orchestrator/status-line.ts +47 -8
- package/src/orchestrator/summary.ts +22 -1
- package/src/orchestrator/tally.ts +78 -35
- package/src/orchestrator/task-hash.ts +105 -11
- package/src/orchestrator/task-log-buffer.ts +291 -0
- package/src/orchestrator/telemetry-host.ts +24 -6
- package/src/orchestrator/telemetry.ts +181 -21
- package/src/orchestrator/upstream.ts +72 -31
- package/src/orchestrator/verify.ts +12 -2
- package/src/plugins/local-cache/index.ts +11 -0
- package/src/plugins/local-executor/index.ts +45 -0
- package/src/plugins/schedule-history/index.ts +109 -0
- package/src/util/edit-distance.ts +21 -0
- package/src/util/errors.ts +15 -1
- package/src/util/index.ts +7 -2
- package/src/util/num.ts +37 -0
- package/src/util/paths.ts +43 -0
- package/src/util/settle.ts +48 -0
- package/src/util/size.ts +5 -1
- package/src/util/tail.ts +66 -0
- package/src/util/timing.ts +63 -0
- package/src/workspace/affected.ts +142 -15
- package/src/workspace/config-cache.ts +304 -0
- package/src/workspace/config-eval.ts +210 -0
- package/src/workspace/config-imports.ts +161 -0
- package/src/workspace/filter.ts +22 -4
- package/src/workspace/fingerprint.ts +16 -1
- package/src/workspace/index.ts +15 -3
- package/src/workspace/lockfile.ts +13 -2
- package/src/workspace/nested-dirs.ts +21 -7
- package/src/workspace/package-graph.ts +23 -20
- package/src/workspace/project-loader.ts +503 -59
- package/src/workspace/workspace.ts +213 -72
- package/src/cache/tar.ts +0 -343
- package/src/cli/backend.ts +0 -23
- package/src/cli/mcp-rpc.ts +0 -292
- package/src/cli/mcp.ts +0 -92
- package/src/orchestrator/devframe-surface.ts +0 -74
- package/src/orchestrator/predict.ts +0 -106
- package/src/orchestrator/protocol.ts +0 -156
- package/src/orchestrator/run-state.ts +0 -145
- package/src/orchestrator/wire-render.ts +0 -92
- package/src/orchestrator/wire.ts +0 -212
package/src/graph/task-graph.ts
CHANGED
|
@@ -23,8 +23,8 @@ export interface TaskNode {
|
|
|
23
23
|
/** Ids of tasks that must complete before this one runs. */
|
|
24
24
|
deps: string[]
|
|
25
25
|
/**
|
|
26
|
-
* True for the tasks the user actually asked for (via cwd,
|
|
27
|
-
* or `pkg#task`). False for
|
|
26
|
+
* True for the tasks the user actually asked for (via cwd, `--all`,
|
|
27
|
+
* `--filter`, or `pkg#task`). False for deps pulled in by `dependsOn`
|
|
28
28
|
* expansion. Used by the orchestrator to scope `forwardArgs` so trailing
|
|
29
29
|
* CLI args don't leak into upstream tasks the user didn't address.
|
|
30
30
|
*/
|
|
@@ -38,12 +38,38 @@ export interface TaskNode {
|
|
|
38
38
|
* them `requested`, so `forwardArgs` scoping is unaffected.
|
|
39
39
|
*/
|
|
40
40
|
surfaced?: boolean
|
|
41
|
+
/**
|
|
42
|
+
* Extra cache-key material from plugins' `key` stage, as sorted
|
|
43
|
+
* `[name, value]` pairs; set by `prepareRun`, folded by `Cache.key`.
|
|
44
|
+
* Absent when no plugin declares the stage.
|
|
45
|
+
*/
|
|
46
|
+
keyParts?: ReadonlyArray<readonly [name: string, value: string]>
|
|
41
47
|
}
|
|
42
48
|
|
|
43
49
|
export function taskId(project: string, task: string): string {
|
|
44
50
|
return `${project}#${task}`
|
|
45
51
|
}
|
|
46
52
|
|
|
53
|
+
/**
|
|
54
|
+
* The inverse of `taskId`. Splits on the FIRST `#`, so a task name that itself
|
|
55
|
+
* contains one round-trips: `taskId('a', 'b#c')` → `'a#b#c'` → `['a', 'b#c']`.
|
|
56
|
+
*
|
|
57
|
+
* This exists because it kept being written by hand as `id.split('#', 2)`,
|
|
58
|
+
* which is NOT the inverse — it discards everything after the second segment,
|
|
59
|
+
* so `'a#b#c'` reads back as task `'b'`. Six call sites had that form while
|
|
60
|
+
* `parseDependencySpec` (the surface that decides what actually runs) has
|
|
61
|
+
* always split on the first `#`, so the query layer and the graph disagreed
|
|
62
|
+
* about the identity of the same task: a lookup either found nothing or, worse,
|
|
63
|
+
* answered with a different task's history.
|
|
64
|
+
*
|
|
65
|
+
* A `#`-containing task name is legal and pinned elsewhere in the suite, so
|
|
66
|
+
* this is reachable rather than theoretical.
|
|
67
|
+
*/
|
|
68
|
+
export function splitTaskId(id: string): [project: string, task: string] {
|
|
69
|
+
const i = id.indexOf('#')
|
|
70
|
+
return i < 0 ? [id, ''] : [id.slice(0, i), id.slice(i + 1)]
|
|
71
|
+
}
|
|
72
|
+
|
|
47
73
|
/**
|
|
48
74
|
* A task is a "group" if it has no `exec` — it exists only to chain
|
|
49
75
|
* `dependsOn` (an umbrella for `vx run ci`). Group tasks:
|
|
@@ -134,16 +160,54 @@ export function expandRequested(
|
|
|
134
160
|
if (idx >= 0) {
|
|
135
161
|
const project = spec.slice(0, idx)
|
|
136
162
|
const task = spec.slice(idx + 1)
|
|
137
|
-
if (projects
|
|
163
|
+
if (declaresTask(projects, project, task)) push(project, task)
|
|
138
164
|
continue
|
|
139
165
|
}
|
|
140
166
|
for (const name of candidates) {
|
|
141
|
-
if (projects
|
|
167
|
+
if (declaresTask(projects, name, spec)) push(name, spec)
|
|
142
168
|
}
|
|
143
169
|
}
|
|
144
170
|
return out
|
|
145
171
|
}
|
|
146
172
|
|
|
173
|
+
/** Shared by `expandRequested` + `unresolvedRequests` so the two can't drift. */
|
|
174
|
+
function declaresTask(projects: Map<string, ProjectEntry>, project: string, task: string): boolean {
|
|
175
|
+
return projects.get(project)?.config.tasks?.[task] !== undefined
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* The requested specs `expandRequested` silently dropped — each one
|
|
180
|
+
* matched NO project, so nothing it asked for will run.
|
|
181
|
+
*
|
|
182
|
+
* A bare name matching only SOME projects is normal (sparse tasks across
|
|
183
|
+
* a workspace) and never reported. An empty `candidates` scope is also
|
|
184
|
+
* never reported: that is the legitimate "nothing selected" outcome
|
|
185
|
+
* (`--affected` with nothing changed), not a typo — the CLI's selection
|
|
186
|
+
* layer owns that message.
|
|
187
|
+
*
|
|
188
|
+
* Deduped so `vx run x x` names `x` once.
|
|
189
|
+
*/
|
|
190
|
+
export function unresolvedRequests(
|
|
191
|
+
tasks: readonly string[],
|
|
192
|
+
candidates: readonly string[],
|
|
193
|
+
projects: Map<string, ProjectEntry>,
|
|
194
|
+
): string[] {
|
|
195
|
+
const out: string[] = []
|
|
196
|
+
const seen = new Set<string>()
|
|
197
|
+
for (const spec of tasks) {
|
|
198
|
+
if (seen.has(spec)) continue
|
|
199
|
+
const idx = spec.indexOf('#')
|
|
200
|
+
const resolved =
|
|
201
|
+
idx >= 0
|
|
202
|
+
? declaresTask(projects, spec.slice(0, idx), spec.slice(idx + 1))
|
|
203
|
+
: candidates.length === 0 || candidates.some((name) => declaresTask(projects, name, spec))
|
|
204
|
+
if (resolved) continue
|
|
205
|
+
seen.add(spec)
|
|
206
|
+
out.push(spec)
|
|
207
|
+
}
|
|
208
|
+
return out
|
|
209
|
+
}
|
|
210
|
+
|
|
147
211
|
export interface BuildGraphOptions {
|
|
148
212
|
projects: Map<string, ProjectEntry>
|
|
149
213
|
packageGraph: PackageGraph
|
|
@@ -268,8 +332,14 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
|
|
|
268
332
|
// AT LEAST ONE matching task and it receives edges to ALL its
|
|
269
333
|
// matches — holder-ness is about declaration, so a holder still
|
|
270
334
|
// stops the walk even when every match is --excludeDependencies'd.
|
|
335
|
+
//
|
|
336
|
+
// The declaring project seeds `visited`: package graphs may legally
|
|
337
|
+
// contain cycles (the common "b devDepends on a for its tests"
|
|
338
|
+
// shape), and a cycle walks the frontier straight back to the
|
|
339
|
+
// origin. Mirrors the self-pattern rule below — a task can never
|
|
340
|
+
// depend on itself.
|
|
271
341
|
const re = isTaskPattern(spec.task) ? compileTaskPattern(spec.task) : null
|
|
272
|
-
const visited = new Set<string>()
|
|
342
|
+
const visited = new Set<string>([projectName])
|
|
273
343
|
const frontier = [...packageGraph.directDeps(projectName)]
|
|
274
344
|
while (frontier.length > 0) {
|
|
275
345
|
const target = frontier.pop()!
|
|
@@ -320,10 +390,154 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
|
|
|
320
390
|
}
|
|
321
391
|
|
|
322
392
|
detectCycle(nodes)
|
|
393
|
+
detectOutputCollisions(nodes)
|
|
323
394
|
return nodes
|
|
324
395
|
}
|
|
325
396
|
|
|
326
|
-
|
|
397
|
+
/** A glob with no wildcard — it names exactly one path. */
|
|
398
|
+
function isLiteralGlob(g: string): boolean {
|
|
399
|
+
return g.search(/[*?[\]]/) === -1
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
/**
|
|
403
|
+
* True only when two output globs PROVABLY select an overlapping set.
|
|
404
|
+
*
|
|
405
|
+
* Deliberately conservative, because the caller REFUSES the run: a false
|
|
406
|
+
* positive breaks a build that works today, which is worse than the defect
|
|
407
|
+
* being caught. So the three cases are exactly the ones that can be decided
|
|
408
|
+
* without a general glob-intersection algorithm:
|
|
409
|
+
*
|
|
410
|
+
* both literal — equal paths
|
|
411
|
+
* literal vs glob — ask the glob whether it matches the literal (exact)
|
|
412
|
+
* both globs — only identical strings; anything else is undecided
|
|
413
|
+
* here and deliberately allowed through
|
|
414
|
+
*
|
|
415
|
+
* The rejected alternative was comparing each glob's static prefix. It is
|
|
416
|
+
* cheaper and catches more, but it is UNSOUND for a refusal — measured:
|
|
417
|
+
* `dist/vx-*` and `dist/other.txt` share the prefix `dist` while matching
|
|
418
|
+
* disjoint sets, so a prefix check refuses a legitimate config. (vx's own
|
|
419
|
+
* `build.bun.*` tasks escape only because they declare distinct literals.)
|
|
420
|
+
*/
|
|
421
|
+
function outputsOverlap(a: string, b: string): boolean {
|
|
422
|
+
if (isLiteralGlob(a) && isLiteralGlob(b)) return a === b
|
|
423
|
+
if (isLiteralGlob(a)) return new Bun.Glob(b).match(a)
|
|
424
|
+
if (isLiteralGlob(b)) return new Bun.Glob(a).match(b)
|
|
425
|
+
return a === b
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
/**
|
|
429
|
+
* Refuse a graph in which two tasks declare overlapping outputs.
|
|
430
|
+
*
|
|
431
|
+
* vx cleans a task's declared outputs before it runs AND before a cache-hit
|
|
432
|
+
* restore, so that the tree ends byte-identical to the cached artifact. That
|
|
433
|
+
* makes output ownership STRICT — and two tasks claiming the same path
|
|
434
|
+
* silently destroy each other's work, in whichever order they happen to run,
|
|
435
|
+
* while the run reports success. It is data loss with a green summary.
|
|
436
|
+
*
|
|
437
|
+
* This is a hazard vx CREATED: Turbo restores additively and cannot hit it,
|
|
438
|
+
* which is why no Turbo test surfaces it and why the parity research had to
|
|
439
|
+
* reproduce it end to end.
|
|
440
|
+
*
|
|
441
|
+
* Scope follows the two namespaces' reach. `outputs.files` is
|
|
442
|
+
* project-relative, so only tasks in the SAME project can collide;
|
|
443
|
+
* `outputs.workspaceFiles` is anchored at the workspace root and ignores
|
|
444
|
+
* project boundaries by design, so ANY two tasks can. No cache key changes —
|
|
445
|
+
* this only refuses a graph that was already destroying files.
|
|
446
|
+
*/
|
|
447
|
+
function detectOutputCollisions(nodes: Map<string, TaskNode>): void {
|
|
448
|
+
// INDEX FIRST, then compare — never all-pairs over the graph. A naive
|
|
449
|
+
// pairwise loop that filters by project INSIDE the loop is quadratic in the
|
|
450
|
+
// whole graph: measured 1.6 SECONDS at this project's stated target of 1000
|
|
451
|
+
// projects x 10 tasks, on every run, against a ~120ms warm run. (Same shape
|
|
452
|
+
// as the scheduler's priority closure, which took 8.5s on a 1090-package
|
|
453
|
+
// repo before it was rewritten.)
|
|
454
|
+
//
|
|
455
|
+
// Both namespaces have a much smaller natural domain:
|
|
456
|
+
// files — project-relative, so only same-project tasks can
|
|
457
|
+
// collide. Bucketing makes this O(sum of k^2) over
|
|
458
|
+
// per-project task counts, and k is single digits.
|
|
459
|
+
// workspaceFiles — root-anchored and boundary-free, so any two tasks can
|
|
460
|
+
// collide — but only tasks that DECLARE it participate,
|
|
461
|
+
// and that set is nearly always empty.
|
|
462
|
+
const byProject = new Map<string, TaskNode[]>()
|
|
463
|
+
const wsDeclarers: TaskNode[] = []
|
|
464
|
+
for (const n of nodes.values()) {
|
|
465
|
+
const outs = n.config.cache?.outputs
|
|
466
|
+
if ((outs?.files?.length ?? 0) > 0) {
|
|
467
|
+
const bucket = byProject.get(n.projectName)
|
|
468
|
+
if (bucket) bucket.push(n)
|
|
469
|
+
else byProject.set(n.projectName, [n])
|
|
470
|
+
}
|
|
471
|
+
if ((outs?.workspaceFiles?.length ?? 0) > 0) wsDeclarers.push(n)
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
for (const bucket of byProject.values()) {
|
|
475
|
+
for (let i = 0; i < bucket.length; i++) {
|
|
476
|
+
for (let j = i + 1; j < bucket.length; j++) {
|
|
477
|
+
const a = bucket[i]!
|
|
478
|
+
const b = bucket[j]!
|
|
479
|
+
collide(a, b, a.config.cache?.outputs.files, b.config.cache?.outputs.files, 'files')
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
for (let i = 0; i < wsDeclarers.length; i++) {
|
|
484
|
+
for (let j = i + 1; j < wsDeclarers.length; j++) {
|
|
485
|
+
const a = wsDeclarers[i]!
|
|
486
|
+
const b = wsDeclarers[j]!
|
|
487
|
+
collide(
|
|
488
|
+
a,
|
|
489
|
+
b,
|
|
490
|
+
a.config.cache?.outputs.workspaceFiles,
|
|
491
|
+
b.config.cache?.outputs.workspaceFiles,
|
|
492
|
+
'workspaceFiles',
|
|
493
|
+
)
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
/**
|
|
499
|
+
* A `remote: 'only'` task never materialises its outputs on THIS machine —
|
|
500
|
+
* execute-task turns off both the read and the write axis for it, so there is
|
|
501
|
+
* no probe, no restore, no output clean and no local save. The collision
|
|
502
|
+
* hazard is entirely about local cleaning ("whichever runs second DELETES the
|
|
503
|
+
* other's output"), so a pair including one of these cannot exhibit it: there
|
|
504
|
+
* is nothing of its on disk to delete, and it deletes nothing of anyone's.
|
|
505
|
+
*
|
|
506
|
+
* Found while trying to give two projects their own install-as-an-action:
|
|
507
|
+
* both legitimately capture the workspace-root `node_modules` under a
|
|
508
|
+
* hoisting package manager, and the refusal fired on a pair that cannot
|
|
509
|
+
* exhibit the hazard. That particular layout was abandoned for an unrelated
|
|
510
|
+
* reason (identical definitions still get different cache keys), so this is
|
|
511
|
+
* not load-bearing today — it is a refusal narrowed to what it can prove,
|
|
512
|
+
* which this file's header asks for explicitly.
|
|
513
|
+
*/
|
|
514
|
+
function neverWritesLocally(n: TaskNode): boolean {
|
|
515
|
+
return n.config.exec?.remote === 'only'
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
function collide(
|
|
519
|
+
a: TaskNode,
|
|
520
|
+
b: TaskNode,
|
|
521
|
+
aGlobs: readonly string[] | undefined,
|
|
522
|
+
bGlobs: readonly string[] | undefined,
|
|
523
|
+
field: 'files' | 'workspaceFiles',
|
|
524
|
+
): void {
|
|
525
|
+
if (neverWritesLocally(a) || neverWritesLocally(b)) return
|
|
526
|
+
for (const ga of aGlobs ?? []) {
|
|
527
|
+
for (const gb of bGlobs ?? []) {
|
|
528
|
+
if (!outputsOverlap(ga, gb)) continue
|
|
529
|
+
throw new UserError(
|
|
530
|
+
`${a.id} and ${b.id} both declare the output ${JSON.stringify(ga)}` +
|
|
531
|
+
(ga === gb ? '' : ` / ${JSON.stringify(gb)}`) +
|
|
532
|
+
` in cache.outputs.${field} — vx cleans a task's declared outputs before it runs and ` +
|
|
533
|
+
`before a cache-hit restore, so whichever of these runs second DELETES the other's ` +
|
|
534
|
+
`output and the run still reports success. Give each task its own output path.`,
|
|
535
|
+
)
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
export function detectCycle(nodes: Map<string, TaskNode>): void {
|
|
327
541
|
// Iterative DFS over dense-indexed colors. Recursion + `Map<string,
|
|
328
542
|
// number>` worked, but deep workspaces (long chains of `dependsOn`)
|
|
329
543
|
// can blow V8's frame budget, and per-node Map lookups dominate the
|
package/src/index.ts
CHANGED
|
@@ -9,7 +9,14 @@
|
|
|
9
9
|
export { VERSION } from './version.js'
|
|
10
10
|
|
|
11
11
|
// Clean error type — user-input failures print a message without a stack.
|
|
12
|
-
|
|
12
|
+
// `clampInt` and `parseDecimalInt` ride along for the same reason the status
|
|
13
|
+
// predicates do: without them on the façade an integration package writes its
|
|
14
|
+
// own, and both had already happened. A bounds helper whose floor is
|
|
15
|
+
// load-bearing (a fractional SQL LIMIT is a datatype mismatch, not a smaller
|
|
16
|
+
// page) should have one implementation — and so should the ONE strict integer
|
|
17
|
+
// parser, whose entire purpose is that `Number()` silently accepts `0x10` and
|
|
18
|
+
// `1e3` at a boundary where a typo must be an error, not a different number.
|
|
19
|
+
export { clampInt, parseDecimalInt, parseSize, UserError, isUserError } from './util/index.js'
|
|
13
20
|
|
|
14
21
|
// Schema types and helpers (used by user vx.config files and presets).
|
|
15
22
|
export type {
|
|
@@ -34,15 +41,22 @@ export { defineProject, defineWorkspace } from './config.js'
|
|
|
34
41
|
// on the stability gate); `captureGitContext`/`captureWorkspaceIdentity`
|
|
35
42
|
// give agents + the submitter identity before/without a telemetry run.
|
|
36
43
|
export { run, planRun, prepareRun } from './orchestrator/index.js'
|
|
44
|
+
// The per-task duration history a `schedule` plugin learns from (see
|
|
45
|
+
// src/plugins/schedule-history — core's own plugins import core only via
|
|
46
|
+
// this façade, which is what put these here).
|
|
47
|
+
export { EmptyHistoryProvider, LocalHistoryProvider } from './orchestrator/index.js'
|
|
48
|
+
export type { HistoryProvider, HistoryTable, TaskHistory } from './orchestrator/index.js'
|
|
37
49
|
export type { PreparedRun } from './orchestrator/index.js'
|
|
38
50
|
export { computeTaskHash, createHashCache, deriveStableKeys } from './orchestrator/index.js'
|
|
39
51
|
export type { DeriveStableKeysArgs, HashCache, StableKey } from './orchestrator/index.js'
|
|
40
52
|
export {
|
|
53
|
+
captureDefaultBranch,
|
|
41
54
|
captureGitContext,
|
|
55
|
+
captureHostContext,
|
|
42
56
|
captureWorkspaceIdentity,
|
|
43
|
-
|
|
57
|
+
detectCi,
|
|
44
58
|
} from './orchestrator/index.js'
|
|
45
|
-
export type { GitContext, WorkspaceIdentity } from './orchestrator/index.js'
|
|
59
|
+
export type { CiContext, GitContext, HostContext, WorkspaceIdentity } from './orchestrator/index.js'
|
|
46
60
|
export { FULL_CACHE_POLICY, parseCachePolicy } from './orchestrator/index.js'
|
|
47
61
|
export type {
|
|
48
62
|
CachePolicy,
|
|
@@ -52,7 +66,17 @@ export type {
|
|
|
52
66
|
RunSummary,
|
|
53
67
|
} from './orchestrator/index.js'
|
|
54
68
|
export { defaultLogger, resolveOutputView } from './orchestrator/index.js'
|
|
55
|
-
|
|
69
|
+
// `splitTaskId` is on the façade because the alternative is what happened:
|
|
70
|
+
// with only `taskId()` to JOIN an id and nothing exported to SPLIT one,
|
|
71
|
+
// consumers roll their own `split('#', 2)` and drift from the graph, which
|
|
72
|
+
// splits on the FIRST '#'.
|
|
73
|
+
export {
|
|
74
|
+
buildTaskGraph,
|
|
75
|
+
expandRequested,
|
|
76
|
+
isGroupTask,
|
|
77
|
+
markSurfacedDeps,
|
|
78
|
+
splitTaskId,
|
|
79
|
+
} from './graph/index.js'
|
|
56
80
|
export type {
|
|
57
81
|
OutputFingerprint,
|
|
58
82
|
TaskNode,
|
|
@@ -86,7 +110,7 @@ export type { CacheLayer, RemoteCacheLayer, RunRecord, InvocationRecord } from '
|
|
|
86
110
|
// parser in a sibling package would drift), and the loader chain
|
|
87
111
|
// (`loadWorkspace` → `listProjectMetas` → `loadProjectConfig`) is the same
|
|
88
112
|
// one `vx show` uses. Workspace's `listProjects` re-exports as
|
|
89
|
-
// `listProjectMetas`
|
|
113
|
+
// `listProjectMetas` (the bare name once belonged to a metrics query).
|
|
90
114
|
export { findWorkspaceRoot, loadWorkspaceConfig, resolveCacheDir } from './workspace/index.js'
|
|
91
115
|
export { readLockfile, LOCKFILE_NAME } from './workspace/index.js'
|
|
92
116
|
export type { Lockfile, LockfileEntry } from './workspace/index.js'
|
|
@@ -98,58 +122,82 @@ export {
|
|
|
98
122
|
export type { ProjectMeta } from './workspace/index.js'
|
|
99
123
|
|
|
100
124
|
// Plugin API — the run-level extension points. Behavior capabilities
|
|
101
|
-
// (
|
|
125
|
+
// (executor / cache) change WHAT/HOW work runs; the observe-only `telemetry`
|
|
102
126
|
// capability is the canonical data-export path and cannot change behavior.
|
|
103
127
|
// A plugin is declared in vx.workspace.ts via defineWorkspace({ plugins }).
|
|
104
128
|
// See docs/design/observability-architecture-2026-06.md.
|
|
105
129
|
export type {
|
|
106
130
|
VxPlugin,
|
|
107
|
-
EventSink,
|
|
108
|
-
BackendContext,
|
|
109
131
|
CacheContext,
|
|
110
|
-
|
|
132
|
+
ExecutorContext,
|
|
133
|
+
CommandContext,
|
|
134
|
+
GraphHookContext,
|
|
135
|
+
KeyHookContext,
|
|
136
|
+
PluginCommand,
|
|
137
|
+
ProjectHookContext,
|
|
138
|
+
ScheduleHookContext,
|
|
139
|
+
WorkspaceHookContext,
|
|
111
140
|
PluginSetupContext,
|
|
112
141
|
} from './orchestrator/index.js'
|
|
142
|
+
// Process primitives — what an executor plugin builds on. `@vzn/vx/plugins/local-executor`
|
|
143
|
+
// is the reference implementation and imports exactly these.
|
|
144
|
+
export { runCommand, runSandboxed } from './exec/index.js'
|
|
145
|
+
// The per-task execution contract a plugin's `executor` capability returns.
|
|
146
|
+
export type {
|
|
147
|
+
ExecuteRequest,
|
|
148
|
+
ExecuteResult,
|
|
149
|
+
ExecuteSandbox,
|
|
150
|
+
ResolvedSandboxConfig,
|
|
151
|
+
TaskExecutor,
|
|
152
|
+
TaskPlacement,
|
|
153
|
+
} from './exec/index.js'
|
|
113
154
|
|
|
114
155
|
// Telemetry — THE canonical, versioned data-export contract every exporter
|
|
115
156
|
// (an OTel exporter, an HTTP sink, or any third-party consumer) reads. A sink implements TelemetrySink and is
|
|
116
157
|
// returned from VxPlugin.telemetry(); it receives immutable records and holds
|
|
117
158
|
// no run handle (observe-only by construction).
|
|
118
|
-
|
|
159
|
+
// `isPassStatus` / `isCacheHit` are on the façade for the reason the sweep
|
|
160
|
+
// that added them found: with only the raw `TaskStatus` union exported, every
|
|
161
|
+
// consumer rolls its own Set of status literals — and a Set has no
|
|
162
|
+
// compile-time tripwire when the union gains a member, so it silently answers
|
|
163
|
+
// "no" for the new one. `TASK_STATUSES` is the union at runtime, for a
|
|
164
|
+
// consumer that needs the list rather than the predicate.
|
|
165
|
+
// `escapeMarkdownCell` is on the façade for the same demonstrated need: a
|
|
166
|
+
// plugin that renders a run as a markdown table takes the same unvalidated
|
|
167
|
+
// task names core does, and the cloud job summary shipped without the escape.
|
|
168
|
+
export {
|
|
169
|
+
assembleRunSummary,
|
|
170
|
+
escapeMarkdownCell,
|
|
171
|
+
TELEMETRY_SCHEMA_VERSION,
|
|
172
|
+
deriveCacheSource,
|
|
173
|
+
isCacheHit,
|
|
174
|
+
isPassStatus,
|
|
175
|
+
TASK_STATUSES,
|
|
176
|
+
} from './orchestrator/index.js'
|
|
177
|
+
// `TaskLogBuffer` is on the façade on the same demonstrated need: EVERY
|
|
178
|
+
// telemetry sink that ships build output has to bound it, and the retention
|
|
179
|
+
// rules (per-task tail, per-run budget, failures never evicted by successes,
|
|
180
|
+
// a hit's bytes belong to the run that executed) are a decision, not an
|
|
181
|
+
// implementation detail. Two sinks rolling their own is how they fork.
|
|
182
|
+
export { LOG_WIRE_VERSION, TaskLogBuffer } from './orchestrator/index.js'
|
|
119
183
|
export type {
|
|
120
184
|
CacheSource,
|
|
121
185
|
RunContextRecord,
|
|
122
186
|
RunSummaryRecord,
|
|
187
|
+
TaskLogBundle,
|
|
188
|
+
TaskLogEntry,
|
|
123
189
|
TaskTelemetry,
|
|
124
190
|
TelemetryContext,
|
|
125
191
|
TelemetryRecord,
|
|
126
192
|
TelemetrySink,
|
|
127
193
|
} from './orchestrator/index.js'
|
|
128
194
|
|
|
129
|
-
// The
|
|
130
|
-
|
|
131
|
-
export {
|
|
132
|
-
optionsToRequest,
|
|
133
|
-
requestToOptions,
|
|
134
|
-
projectNode,
|
|
135
|
-
projectOutcome,
|
|
136
|
-
createWireRenderer,
|
|
137
|
-
} from './orchestrator/index.js'
|
|
138
|
-
export type {
|
|
139
|
-
RunBackend,
|
|
140
|
-
RunRequest,
|
|
141
|
-
RunResult,
|
|
142
|
-
ClientMessage,
|
|
143
|
-
ServerMessage,
|
|
144
|
-
} from './orchestrator/index.js'
|
|
195
|
+
// The serializable event projection.
|
|
196
|
+
export { projectNode, projectOutcome } from './orchestrator/index.js'
|
|
197
|
+
export type { RunResult } from './orchestrator/index.js'
|
|
145
198
|
|
|
146
199
|
// Event bus + wire form — adapters (otel-bridge, custom subscribers) ride this.
|
|
147
|
-
export {
|
|
148
|
-
createEventBus,
|
|
149
|
-
wireForwarder,
|
|
150
|
-
toWireEvent,
|
|
151
|
-
createVxSurface,
|
|
152
|
-
} from './orchestrator/index.js'
|
|
200
|
+
export { createEventBus, wireForwarder, toWireEvent } from './orchestrator/index.js'
|
|
153
201
|
export type {
|
|
154
202
|
EventBus,
|
|
155
203
|
RunEvent,
|
|
@@ -159,64 +207,14 @@ export type {
|
|
|
159
207
|
OutcomeView,
|
|
160
208
|
} from './orchestrator/index.js'
|
|
161
209
|
|
|
162
|
-
//
|
|
163
|
-
|
|
164
|
-
clientMessageToEnvelope,
|
|
165
|
-
decodeEnvelope,
|
|
166
|
-
encodeForNDJSON,
|
|
167
|
-
encodeForSSE,
|
|
168
|
-
encodeForWS,
|
|
169
|
-
ENVELOPE_ERRORS,
|
|
170
|
-
envelopeToClientMessage,
|
|
171
|
-
envelopeToServerMessage,
|
|
172
|
-
isEnvelope,
|
|
173
|
-
isNotification,
|
|
174
|
-
isRequest,
|
|
175
|
-
makeError,
|
|
176
|
-
makeNotification,
|
|
177
|
-
makeRequest,
|
|
178
|
-
makeResponse,
|
|
179
|
-
serverMessageToEnvelope,
|
|
180
|
-
WIRE_CHANNELS,
|
|
181
|
-
WIRE_PROTOCOL_VERSION,
|
|
182
|
-
} from './orchestrator/index.js'
|
|
183
|
-
export type {
|
|
184
|
-
Envelope,
|
|
185
|
-
ErrorResponse,
|
|
186
|
-
Notification,
|
|
187
|
-
WireRequest,
|
|
188
|
-
WireResponse,
|
|
189
|
-
WireChannel,
|
|
190
|
-
} from './orchestrator/index.js'
|
|
191
|
-
|
|
192
|
-
// Metrics / analytics query layer over cache.db — a service plugin's HTTP
|
|
193
|
-
// surface reads from these (the queries stay in core; a serve exposes them over HTTP).
|
|
210
|
+
// Run-history queries over cache.db — what `vx why` / `vx last` read. An
|
|
211
|
+
// out-of-process surface (an MCP server, a dashboard plugin) reads the same.
|
|
194
212
|
export {
|
|
195
213
|
cacheKeyDiff,
|
|
196
|
-
compareRuns,
|
|
197
214
|
explainCacheKeyQuery,
|
|
198
|
-
getBottlenecks,
|
|
199
|
-
getCacheBreakdown,
|
|
200
|
-
getCacheSavings,
|
|
201
|
-
getCacheStatsSql,
|
|
202
|
-
getFlakiestTasks,
|
|
203
|
-
getHistory,
|
|
204
|
-
getHitRateSplit,
|
|
205
215
|
getInvocation,
|
|
206
|
-
getParallelismHistory,
|
|
207
|
-
getPeriodComparison,
|
|
208
|
-
getPrunableEntries,
|
|
209
|
-
getRecentFailures,
|
|
210
|
-
getRegressions,
|
|
211
216
|
getRun,
|
|
212
|
-
getRunHeatmap,
|
|
213
|
-
getRunTrends,
|
|
214
|
-
getStorageGrowth,
|
|
215
|
-
getTaskDetail,
|
|
216
|
-
getTopTimeBurners,
|
|
217
|
-
listCacheEntries,
|
|
218
217
|
listInvocations,
|
|
219
|
-
listProjects,
|
|
220
218
|
listRuns,
|
|
221
219
|
whyDidThisRerunQuery,
|
|
222
220
|
} from './orchestrator/index.js'
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
// The run-scoped registry of tasks whose outputs were left in the remote
|
|
2
|
+
// store (`--download=none`), and the lazy materialisation that fetches
|
|
3
|
+
// them when a locally-placed task turns out to need them.
|
|
4
|
+
//
|
|
5
|
+
// See docs/design/download-policy-cas-cache-2026-08.md §5. The two
|
|
6
|
+
// invariants that keep this from becoming a second cache:
|
|
7
|
+
//
|
|
8
|
+
// - A deferred task writes NOTHING locally — no artifact, no rows. The
|
|
9
|
+
// remote store (the executor's own record) is the entry.
|
|
10
|
+
// - Materialisation CONVERGES: after the bytes land, core runs the
|
|
11
|
+
// ordinary `cache.save`, so the machine is indistinguishable from a
|
|
12
|
+
// `--download=all` run and every later reader (get, prune, verify,
|
|
13
|
+
// metrics) sees one entry shape, not two.
|
|
14
|
+
|
|
15
|
+
import {
|
|
16
|
+
cleanOutputs,
|
|
17
|
+
resolveOutputs,
|
|
18
|
+
type CacheLayer,
|
|
19
|
+
type GitFilesCache,
|
|
20
|
+
} from '../cache/index.js'
|
|
21
|
+
import type { TaskNode } from '../graph/index.js'
|
|
22
|
+
|
|
23
|
+
export interface DeferredEntry {
|
|
24
|
+
/** Fetches this task's outputs onto disk. At-most-once by construction. */
|
|
25
|
+
materialize: () => Promise<void>
|
|
26
|
+
hash: string
|
|
27
|
+
entry: { taskId: string; command: string; durationMs: number; stdout: string }
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export interface DeferredOutputsArgs {
|
|
31
|
+
nodes: Map<string, TaskNode>
|
|
32
|
+
cache: CacheLayer
|
|
33
|
+
workspaceRoot: string
|
|
34
|
+
nestedDirsByProject: Map<string, string[]>
|
|
35
|
+
gitFilesCache?: GitFilesCache
|
|
36
|
+
/** False when the run's policy writes no local entries. */
|
|
37
|
+
localWrite: boolean
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export class DeferredOutputs {
|
|
41
|
+
private readonly entries = new Map<string, DeferredEntry>()
|
|
42
|
+
private readonly inflight = new Map<string, Promise<void>>()
|
|
43
|
+
|
|
44
|
+
constructor(private readonly args: DeferredOutputsArgs) {}
|
|
45
|
+
|
|
46
|
+
register(taskId: string, entry: DeferredEntry): void {
|
|
47
|
+
this.entries.set(taskId, entry)
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Task ids whose outputs are still remote — the summary's list. An entry
|
|
52
|
+
* is removed only when materialisation SUCCEEDS, so this covers both
|
|
53
|
+
* "nothing needed them" and "fetching them failed"; the second is
|
|
54
|
+
* precisely when a user needs to be told the tree is not current, and an
|
|
55
|
+
* `inflight` filter here used to hide it.
|
|
56
|
+
*/
|
|
57
|
+
pending(): string[] {
|
|
58
|
+
return [...this.entries.keys()].sort()
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
get size(): number {
|
|
62
|
+
return this.entries.size
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Materialise every deferred producer in `node`'s TRANSITIVE dependency
|
|
67
|
+
* closure. Which upstream bytes a command reads is unknowable — that is
|
|
68
|
+
* what `dependsOn` declares — so the whole closure is taken. Each
|
|
69
|
+
* producer materialises at most once per run (memoised on the promise,
|
|
70
|
+
* so two concurrent consumers share one fetch), and they run
|
|
71
|
+
* concurrently.
|
|
72
|
+
*
|
|
73
|
+
* A failure is the CONSUMER's failure: executing against a
|
|
74
|
+
* half-materialised tree is the stale-input class with extra steps, so
|
|
75
|
+
* this throws and the caller fails the task naming the producer.
|
|
76
|
+
*/
|
|
77
|
+
async materializeFor(node: TaskNode): Promise<void> {
|
|
78
|
+
const needed: string[] = []
|
|
79
|
+
const seen = new Set<string>()
|
|
80
|
+
const walk = (id: string): void => {
|
|
81
|
+
for (const dep of this.args.nodes.get(id)?.deps ?? []) {
|
|
82
|
+
if (seen.has(dep)) continue
|
|
83
|
+
seen.add(dep)
|
|
84
|
+
if (this.entries.has(dep)) needed.push(dep)
|
|
85
|
+
walk(dep)
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
walk(node.id)
|
|
89
|
+
if (needed.length === 0) return
|
|
90
|
+
await Promise.all(needed.map((id) => this.materializeOne(id)))
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
private materializeOne(taskId: string): Promise<void> {
|
|
94
|
+
const existing = this.inflight.get(taskId)
|
|
95
|
+
if (existing !== undefined) return existing
|
|
96
|
+
const started = this.run(taskId)
|
|
97
|
+
this.inflight.set(taskId, started)
|
|
98
|
+
return started
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
private async run(taskId: string): Promise<void> {
|
|
102
|
+
const entry = this.entries.get(taskId)!
|
|
103
|
+
const producer = this.args.nodes.get(taskId)!
|
|
104
|
+
const outputs = producer.config.cache?.outputs.files ?? []
|
|
105
|
+
const nestedProjectDirs = this.args.nestedDirsByProject.get(producer.projectName) ?? []
|
|
106
|
+
const cleanArgs = { projectDir: producer.projectDir, outputs, nestedProjectDirs }
|
|
107
|
+
|
|
108
|
+
// Mirror `restoreHit`'s sequencing so the two paths cannot drift: wipe
|
|
109
|
+
// the declared outputs, write, then tell the git snapshot what moved.
|
|
110
|
+
if (outputs.length > 0) {
|
|
111
|
+
const cleaned = await cleanOutputs(cleanArgs)
|
|
112
|
+
this.args.gitFilesCache?.markOutputsChanged(producer.projectDir, cleaned)
|
|
113
|
+
}
|
|
114
|
+
try {
|
|
115
|
+
await entry.materialize()
|
|
116
|
+
} catch (err) {
|
|
117
|
+
const msg = err instanceof Error ? err.message : String(err)
|
|
118
|
+
throw new Error(
|
|
119
|
+
`could not fetch deferred outputs of ${taskId}: ${msg} — re-run it with --force, or use --download=all`,
|
|
120
|
+
)
|
|
121
|
+
}
|
|
122
|
+
const outputFiles = await resolveOutputs({
|
|
123
|
+
projectDir: producer.projectDir,
|
|
124
|
+
outputs,
|
|
125
|
+
nestedProjectDirs,
|
|
126
|
+
})
|
|
127
|
+
this.args.gitFilesCache?.markOutputsChanged(
|
|
128
|
+
producer.projectDir,
|
|
129
|
+
outputFiles.map((f) => f.slice(producer.projectDir.length + 1)),
|
|
130
|
+
)
|
|
131
|
+
// Convergence: an ordinary entry, so the next run is a plain local hit
|
|
132
|
+
// and nothing downstream learns a second shape. Policy still governs —
|
|
133
|
+
// a run that writes no local entries writes none here either.
|
|
134
|
+
if (this.args.localWrite) {
|
|
135
|
+
await this.args.cache.save({
|
|
136
|
+
hash: entry.hash,
|
|
137
|
+
projectDir: producer.projectDir,
|
|
138
|
+
outputFiles,
|
|
139
|
+
entry: entry.entry,
|
|
140
|
+
})
|
|
141
|
+
}
|
|
142
|
+
this.entries.delete(taskId)
|
|
143
|
+
}
|
|
144
|
+
}
|