@vzn/vx 0.0.18 → 0.0.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +107 -70
- package/package.json +10 -20
- package/src/bin.ts +64 -5
- package/src/cache/archive.ts +279 -28
- package/src/cache/cache.ts +952 -1584
- package/src/cache/chained-cache.ts +14 -10
- package/src/cache/config-evals.ts +112 -0
- package/src/cache/file-hashes.ts +290 -0
- package/src/cache/git-inputs.ts +966 -0
- package/src/cache/index.ts +23 -7
- package/src/cache/inputs.ts +553 -739
- package/src/cache/key-fold.ts +232 -0
- package/src/cache/layer.ts +674 -0
- package/src/cache/layered-cache.ts +211 -49
- package/src/cache/output-index.ts +323 -0
- package/src/cache/policy.ts +84 -0
- package/src/cache/run-history.ts +152 -0
- package/src/cache/zstd.ts +183 -0
- package/src/cli/cache.ts +88 -30
- package/src/cli/completions.ts +104 -0
- package/src/cli/core-alias.ts +25 -0
- package/src/cli/format.ts +4 -15
- package/src/cli/help.ts +76 -38
- package/src/cli/index.ts +82 -17
- package/src/cli/info.ts +162 -91
- package/src/cli/init.ts +87 -0
- package/src/cli/last.ts +130 -23
- package/src/cli/lock.ts +36 -18
- package/src/cli/plan-format.ts +18 -7
- package/src/cli/plugin-commands.ts +34 -12
- package/src/cli/run.ts +138 -313
- package/src/cli/select.ts +342 -0
- package/src/cli/show.ts +151 -71
- package/src/cli/upgrade.ts +136 -15
- package/src/cli/watch.ts +1175 -116
- package/src/cli/why.ts +51 -29
- package/src/cli/workspace-config.ts +138 -0
- package/src/config.ts +183 -126
- package/src/exec/env.ts +11 -0
- package/src/exec/executor.ts +62 -31
- package/src/exec/index.ts +16 -2
- package/src/exec/kill-tree.ts +281 -0
- package/src/exec/local-executor.ts +41 -0
- package/src/exec/runner.ts +361 -77
- package/src/exec/sandbox-binds.ts +211 -0
- package/src/exec/sandbox-paths.ts +70 -0
- package/src/exec/sandbox-runtime.ts +1152 -415
- package/src/exec/sandbox-violations.ts +293 -0
- package/src/graph/dependency-spec.ts +14 -14
- package/src/graph/index.ts +6 -4
- package/src/graph/priorities.ts +130 -0
- package/src/graph/scheduler.ts +332 -329
- package/src/graph/task-graph.ts +552 -188
- package/src/index.ts +111 -102
- package/src/orchestrator/admission.ts +161 -0
- package/src/orchestrator/colors.ts +8 -3
- package/src/orchestrator/deferred-outputs.ts +25 -7
- package/src/orchestrator/doctor.ts +363 -0
- package/src/orchestrator/events.ts +111 -8
- package/src/orchestrator/excluded-keys.ts +155 -0
- package/src/orchestrator/execute-task.ts +450 -565
- package/src/orchestrator/failure-mode.ts +169 -28
- package/src/orchestrator/failure-recap.ts +121 -0
- package/src/orchestrator/fingerprint-watch.ts +89 -0
- package/src/orchestrator/framed-output.ts +121 -49
- package/src/orchestrator/history.ts +123 -104
- package/src/orchestrator/hit-restore.ts +285 -0
- package/src/orchestrator/index.ts +42 -5
- package/src/orchestrator/keyed-projects.ts +89 -0
- package/src/orchestrator/local-shortcircuit.ts +83 -23
- package/src/orchestrator/lockfile-claim.ts +401 -0
- package/src/orchestrator/logger.ts +128 -41
- package/src/orchestrator/metrics.ts +201 -72
- package/src/orchestrator/miss-save.ts +232 -0
- package/src/orchestrator/options.ts +63 -34
- package/src/orchestrator/persistent.ts +99 -0
- package/src/orchestrator/placement.ts +212 -0
- package/src/orchestrator/plan.ts +24 -11
- package/src/orchestrator/plugin-host.ts +200 -49
- package/src/orchestrator/plugin.ts +190 -14
- package/src/orchestrator/prepare.ts +168 -137
- package/src/orchestrator/projects.ts +270 -0
- package/src/orchestrator/remote-prefetch.ts +22 -2
- package/src/orchestrator/run-artifacts.ts +34 -3
- package/src/orchestrator/run-context.ts +20 -5
- package/src/orchestrator/run-lock.ts +349 -0
- package/src/orchestrator/run-records.ts +132 -0
- package/src/orchestrator/run-report.ts +6 -4
- package/src/orchestrator/run.ts +467 -611
- package/src/orchestrator/sandbox-request.ts +569 -0
- package/src/orchestrator/save-lane.ts +44 -0
- package/src/orchestrator/shell-verdict.ts +117 -0
- package/src/orchestrator/signals.ts +167 -0
- package/src/orchestrator/stable-keys.ts +259 -10
- package/src/orchestrator/status-line.ts +35 -2
- package/src/orchestrator/summary.ts +115 -25
- package/src/orchestrator/task-hash.ts +103 -21
- package/src/orchestrator/task-log-buffer.ts +15 -18
- package/src/orchestrator/telemetry-host.ts +27 -8
- package/src/orchestrator/telemetry.ts +71 -44
- package/src/orchestrator/upstream.ts +91 -45
- package/src/util/bun-version.ts +43 -0
- package/src/util/cgroup.ts +160 -0
- package/src/util/edit-distance.ts +49 -0
- package/src/util/errors.ts +86 -0
- package/src/util/hash.ts +10 -1
- package/src/util/index.ts +42 -5
- package/src/util/paths.ts +168 -5
- package/src/util/procfs.ts +20 -0
- package/src/util/settle.ts +23 -2
- package/src/util/size.ts +33 -0
- package/src/util/task-id.ts +20 -0
- package/src/util/timing.ts +11 -1
- package/src/util/verbs.ts +39 -0
- package/src/util/which.ts +48 -0
- package/src/workspace/affected.ts +368 -60
- package/src/workspace/config-cache.ts +271 -32
- package/src/workspace/config-eval.ts +60 -16
- package/src/workspace/config-imports.ts +131 -9
- package/src/workspace/config-schema.ts +1033 -0
- package/src/workspace/filter.ts +69 -17
- package/src/workspace/fingerprint.ts +59 -7
- package/src/workspace/index.ts +37 -3
- package/src/workspace/json-data.ts +90 -0
- package/src/workspace/load-reads.ts +32 -0
- package/src/workspace/lockfile.ts +41 -14
- package/src/workspace/migrate-scripts.ts +187 -0
- package/src/workspace/migration.ts +353 -0
- package/src/workspace/package-graph.ts +239 -31
- package/src/workspace/project-loader.ts +295 -815
- package/src/workspace/workspace.ts +266 -31
- package/plugins/local-cache/index.ts +0 -2
- package/plugins/local-executor/index.ts +0 -2
- package/plugins/schedule-history/index.ts +0 -2
- package/src/cache/cas-backend.ts +0 -115
- package/src/cache/digest.ts +0 -32
- package/src/cli/migrate-emit.ts +0 -21
- package/src/cli/migrate-nx.ts +0 -473
- package/src/cli/migrate-scripts.ts +0 -127
- package/src/cli/migrate-turbo.ts +0 -401
- package/src/cli/migrate.ts +0 -305
- package/src/cli/prune.ts +0 -291
- package/src/orchestrator/missing-plugin.ts +0 -9
- package/src/orchestrator/resources.ts +0 -45
- package/src/orchestrator/verify.ts +0 -174
- package/src/plugins/local-cache/index.ts +0 -11
- package/src/plugins/local-executor/index.ts +0 -45
- package/src/plugins/schedule-history/index.ts +0 -109
|
@@ -1,20 +1,22 @@
|
|
|
1
1
|
// Plugin consultation for the run-level extension points (cache / executor).
|
|
2
|
-
// Each function asks the plugins in order
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
2
|
+
// Each function asks the plugins in order, and CORE IS THE TAIL: running
|
|
3
|
+
// here and caching here are what those words mean in the absence of a
|
|
4
|
+
// plugin placing work elsewhere or storing it somewhere else. A plugin
|
|
5
|
+
// goes in front — it takes what it accepts, and what it declines lands on
|
|
6
|
+
// the floor. No plugin is required for a plain local run.
|
|
6
7
|
//
|
|
7
|
-
// See docs/design/
|
|
8
|
+
// See docs/design/pipeline-2026-09.md.
|
|
8
9
|
|
|
9
10
|
import { ChainedCache, type CacheLayer } from '../cache/index.js'
|
|
10
|
-
import type
|
|
11
|
+
import { localExecutor, type TaskExecutor } from '../exec/index.js'
|
|
11
12
|
import { settleWithin, teardownTimeoutMs, UserError } from '../util/index.js'
|
|
12
13
|
import type { ProjectConfig, WorkspaceConfig } from '../config.js'
|
|
13
|
-
import {
|
|
14
|
-
import { MISSING_PLUGIN_HINT } from './missing-plugin.js'
|
|
14
|
+
import { checkGraph, type TaskNode } from '../graph/index.js'
|
|
15
15
|
import type {
|
|
16
16
|
CacheContext,
|
|
17
17
|
ExecutorContext,
|
|
18
|
+
FingerprintChange,
|
|
19
|
+
FingerprintContext,
|
|
18
20
|
GraphHookContext,
|
|
19
21
|
KeyHookContext,
|
|
20
22
|
ProjectHookContext,
|
|
@@ -23,12 +25,58 @@ import type {
|
|
|
23
25
|
WorkspaceHookContext,
|
|
24
26
|
} from './plugin.js'
|
|
25
27
|
|
|
28
|
+
/** `a string`, `an array`, `null`, `a number` — for a refusal that names what came back. */
|
|
29
|
+
function describeValue(v: unknown): string {
|
|
30
|
+
if (v === null) return 'null'
|
|
31
|
+
if (Array.isArray(v)) return 'an array'
|
|
32
|
+
const t = typeof v
|
|
33
|
+
return t === 'object' ? 'an object' : `a ${t}`
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Every method `CacheLayer` requires (the optional ones are probed with `?.`). */
|
|
37
|
+
export const CACHE_LAYER_METHODS: readonly string[] = [
|
|
38
|
+
'key',
|
|
39
|
+
'get',
|
|
40
|
+
'has',
|
|
41
|
+
'prefetch',
|
|
42
|
+
'loadOutputFilesBatch',
|
|
43
|
+
'isOutputsCurrent',
|
|
44
|
+
'restoreOutputs',
|
|
45
|
+
'save',
|
|
46
|
+
'ingest',
|
|
47
|
+
'recordRunBundle',
|
|
48
|
+
'stats',
|
|
49
|
+
'hashFile',
|
|
50
|
+
'outputsPath',
|
|
51
|
+
'prune',
|
|
52
|
+
'close',
|
|
53
|
+
]
|
|
54
|
+
|
|
55
|
+
/** What a capability hook handed back must be what the seam runs. */
|
|
56
|
+
function assertShape(
|
|
57
|
+
plugin: VxPlugin,
|
|
58
|
+
hook: string,
|
|
59
|
+
value: unknown,
|
|
60
|
+
methods: readonly string[],
|
|
61
|
+
what: string,
|
|
62
|
+
): void {
|
|
63
|
+
const missing =
|
|
64
|
+
value === null || typeof value !== 'object'
|
|
65
|
+
? methods
|
|
66
|
+
: methods.filter((m) => typeof (value as Record<string, unknown>)[m] !== 'function')
|
|
67
|
+
if (missing.length === 0) return
|
|
68
|
+
throw new UserError(
|
|
69
|
+
`plugin '${plugin.name}' returned from ${hook} something that is not ${what}: ` +
|
|
70
|
+
`missing ${missing.map((m) => `${m}()`).join(', ')}`,
|
|
71
|
+
)
|
|
72
|
+
}
|
|
73
|
+
|
|
26
74
|
/**
|
|
27
75
|
* Run a capability factory with crash isolation. A throw becomes a clean
|
|
28
|
-
* `UserError` naming the plugin + hook
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
* logs-and-skips instead
|
|
76
|
+
* `UserError` naming the plugin + hook: every stage and capability resolved
|
|
77
|
+
* here is load-bearing, so a broken one must abort with a clear message,
|
|
78
|
+
* never silently degrade. (Telemetry sinks are the observe-only exception,
|
|
79
|
+
* and telemetry-host.ts logs-and-skips them instead.)
|
|
32
80
|
*/
|
|
33
81
|
async function safe<T>(plugin: VxPlugin, hook: string, fn: () => T | Promise<T>): Promise<T> {
|
|
34
82
|
try {
|
|
@@ -49,7 +97,7 @@ async function safe<T>(plugin: VxPlugin, hook: string, fn: () => T | Promise<T>)
|
|
|
49
97
|
|
|
50
98
|
export function hasHook(
|
|
51
99
|
plugins: readonly VxPlugin[],
|
|
52
|
-
hook: 'config' | 'project' | 'graph' | 'key' | 'schedule',
|
|
100
|
+
hook: 'config' | 'project' | 'graph' | 'key' | 'schedule' | 'admit',
|
|
53
101
|
): boolean {
|
|
54
102
|
for (const p of plugins) if (p[hook] !== undefined) return true
|
|
55
103
|
return false
|
|
@@ -72,18 +120,24 @@ export async function applyProjectHooks(
|
|
|
72
120
|
plugins: readonly VxPlugin[],
|
|
73
121
|
config: ProjectConfig,
|
|
74
122
|
ctx: ProjectHookContext,
|
|
123
|
+
/** Runs after EACH plugin's edit, so a refusal can name the plugin that made it. */
|
|
124
|
+
afterEach?: (plugin: VxPlugin) => void,
|
|
75
125
|
): Promise<void> {
|
|
76
126
|
for (const plugin of plugins) {
|
|
77
127
|
if (plugin.project === undefined) continue
|
|
78
128
|
await safe(plugin, 'project', () => plugin.project!(config, ctx))
|
|
129
|
+
afterEach?.(plugin)
|
|
79
130
|
}
|
|
80
131
|
}
|
|
81
132
|
|
|
82
133
|
/**
|
|
83
134
|
* `graph` stage: every plugin edits the task graph in place, then the graph
|
|
84
|
-
* is checked the way the builder checks its own output
|
|
85
|
-
* node
|
|
86
|
-
*
|
|
135
|
+
* is checked ONCE the way the builder checks its own output (`checkGraph`:
|
|
136
|
+
* each node under its own id, every dep a node, no cycle, no two tasks
|
|
137
|
+
* deleting each other's outputs). A violation is reported
|
|
138
|
+
* against the LAST plugin that ran: usually the one whose edit made it so,
|
|
139
|
+
* but an earlier plugin's edit that a later one left in place is blamed on
|
|
140
|
+
* the later one, since nothing is checked between plugins.
|
|
87
141
|
*/
|
|
88
142
|
export async function applyGraphHooks(
|
|
89
143
|
plugins: readonly VxPlugin[],
|
|
@@ -97,16 +151,7 @@ export async function applyGraphHooks(
|
|
|
97
151
|
last = plugin
|
|
98
152
|
}
|
|
99
153
|
if (last === undefined) return
|
|
100
|
-
await safe(last, 'graph', () =>
|
|
101
|
-
for (const node of nodes.values()) {
|
|
102
|
-
for (const dep of node.deps) {
|
|
103
|
-
if (!nodes.has(dep)) {
|
|
104
|
-
throw new Error(`${node.id} depends on '${dep}', which is not a task in this run's graph`)
|
|
105
|
-
}
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
detectCycle(nodes)
|
|
109
|
-
})
|
|
154
|
+
await safe(last, 'graph', () => checkGraph(nodes))
|
|
110
155
|
}
|
|
111
156
|
|
|
112
157
|
/**
|
|
@@ -125,6 +170,14 @@ export async function applyKeyHooks(
|
|
|
125
170
|
if (plugin.key === undefined) continue
|
|
126
171
|
const material = await safe(plugin, 'key', () => plugin.key!(node, ctx))
|
|
127
172
|
if (material === undefined) continue
|
|
173
|
+
// `Object.entries` over a string yields its characters as string
|
|
174
|
+
// values, so a plugin returning `'v22'` used to fold parts named
|
|
175
|
+
// '0', '1', '2' into every key — silently, and permanently.
|
|
176
|
+
if (typeof material !== 'object' || material === null || Array.isArray(material)) {
|
|
177
|
+
throw new UserError(
|
|
178
|
+
`plugin '${plugin.name}' failed in key: returned ${describeValue(material)}, not a record of string values`,
|
|
179
|
+
)
|
|
180
|
+
}
|
|
128
181
|
for (const [name, value] of Object.entries(material)) {
|
|
129
182
|
if (typeof value !== 'string') {
|
|
130
183
|
throw new UserError(
|
|
@@ -141,6 +194,50 @@ export async function applyKeyHooks(
|
|
|
141
194
|
}
|
|
142
195
|
}
|
|
143
196
|
|
|
197
|
+
/**
|
|
198
|
+
* The fingerprint files the plugins claim (`VxPlugin.fingerprint`), each
|
|
199
|
+
* with its one claimant. The schema refused a second claimant and an
|
|
200
|
+
* unknown name, so this only indexes.
|
|
201
|
+
*/
|
|
202
|
+
export function fingerprintClaims(plugins: readonly VxPlugin[]): ReadonlyMap<string, VxPlugin> {
|
|
203
|
+
const claims = new Map<string, VxPlugin>()
|
|
204
|
+
for (const p of plugins) {
|
|
205
|
+
if (p.fingerprint === undefined) continue
|
|
206
|
+
for (const f of p.fingerprint.files) claims.set(f, p)
|
|
207
|
+
}
|
|
208
|
+
return claims
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* Ask a claimed file's plugin which projects a change to it affects.
|
|
213
|
+
* `undefined` is "every project" — the claimant could not tell — and a
|
|
214
|
+
* non-iterable answer is refused by name, so a plugin returning `'all'`
|
|
215
|
+
* cannot select the projects spelled a, l, l.
|
|
216
|
+
*/
|
|
217
|
+
export async function claimedAffected(
|
|
218
|
+
plugin: VxPlugin,
|
|
219
|
+
change: FingerprintChange,
|
|
220
|
+
ctx: FingerprintContext,
|
|
221
|
+
): Promise<Set<string> | undefined> {
|
|
222
|
+
const answer = await safe(plugin, 'fingerprint', () => plugin.fingerprint!.affected(change, ctx))
|
|
223
|
+
if (answer === undefined) return undefined
|
|
224
|
+
if (typeof answer === 'string' || typeof answer !== 'object' || answer === null) {
|
|
225
|
+
throw new UserError(
|
|
226
|
+
`plugin '${plugin.name}' failed in fingerprint: returned ${describeValue(answer)}, not a list of project names`,
|
|
227
|
+
)
|
|
228
|
+
}
|
|
229
|
+
const names = new Set<string>()
|
|
230
|
+
for (const name of answer as Iterable<unknown>) {
|
|
231
|
+
if (typeof name !== 'string') {
|
|
232
|
+
throw new UserError(
|
|
233
|
+
`plugin '${plugin.name}' failed in fingerprint: affected project ${describeValue(name)} is not a name`,
|
|
234
|
+
)
|
|
235
|
+
}
|
|
236
|
+
names.add(name)
|
|
237
|
+
}
|
|
238
|
+
return names
|
|
239
|
+
}
|
|
240
|
+
|
|
144
241
|
/**
|
|
145
242
|
* `schedule` stage: the merged priorities map (a later plugin's weight for
|
|
146
243
|
* a task overrides an earlier one's). Empty when no plugin answered.
|
|
@@ -155,6 +252,14 @@ export async function applyScheduleHooks(
|
|
|
155
252
|
if (plugin.schedule === undefined) continue
|
|
156
253
|
const weights = await safe(plugin, 'schedule', () => plugin.schedule!(nodes, ctx))
|
|
157
254
|
if (weights === undefined) continue
|
|
255
|
+
// Iterating a string destructures its characters into `[id, w]` pairs
|
|
256
|
+
// that match no task — a plugin returning `'fast'` used to be a no-op
|
|
257
|
+
// with no word said.
|
|
258
|
+
if (!(weights instanceof Map)) {
|
|
259
|
+
throw new UserError(
|
|
260
|
+
`plugin '${plugin.name}' failed in schedule: returned ${describeValue(weights)}, not a Map of task id → weight`,
|
|
261
|
+
)
|
|
262
|
+
}
|
|
158
263
|
for (const [id, w] of weights) {
|
|
159
264
|
if (!nodes.has(id)) continue
|
|
160
265
|
if (typeof w !== 'number' || !Number.isFinite(w)) {
|
|
@@ -168,13 +273,53 @@ export async function applyScheduleHooks(
|
|
|
168
273
|
return merged
|
|
169
274
|
}
|
|
170
275
|
|
|
276
|
+
/**
|
|
277
|
+
* `admit` stage, built once per run into the predicate the scheduler asks
|
|
278
|
+
* at every local dispatch: every plugin that answers must admit. A plugin
|
|
279
|
+
* that throws is reported once and admits from then on — a policy never
|
|
280
|
+
* breaks a run — so the predicate is never the reason a task hangs.
|
|
281
|
+
* Undefined when no plugin answers, which keeps the scheduler on its
|
|
282
|
+
* count-only path.
|
|
283
|
+
*/
|
|
284
|
+
export function buildAdmission(
|
|
285
|
+
plugins: readonly VxPlugin[],
|
|
286
|
+
nodes: ReadonlyMap<string, TaskNode>,
|
|
287
|
+
concurrency: number,
|
|
288
|
+
warn: (message: string) => void,
|
|
289
|
+
): ((id: string, running: ReadonlySet<string>) => boolean) | undefined {
|
|
290
|
+
const answering = plugins.filter((p) => p.admit !== undefined)
|
|
291
|
+
if (answering.length === 0) return undefined
|
|
292
|
+
const broken = new Set<VxPlugin>()
|
|
293
|
+
return (id, running) => {
|
|
294
|
+
// The scheduler asks only about ids in this same map, so the lookup
|
|
295
|
+
// always lands: an unknown-id arm here was deleted with the whole core
|
|
296
|
+
// suite green (item 651).
|
|
297
|
+
const task = nodes.get(id)!
|
|
298
|
+
const ctx = {
|
|
299
|
+
running: [...running].flatMap((r) => nodes.get(r) ?? []),
|
|
300
|
+
concurrency,
|
|
301
|
+
}
|
|
302
|
+
for (const plugin of answering) {
|
|
303
|
+
if (broken.has(plugin)) continue
|
|
304
|
+
try {
|
|
305
|
+
if (plugin.admit!(task, ctx) === false) return false
|
|
306
|
+
} catch (err) {
|
|
307
|
+
broken.add(plugin)
|
|
308
|
+
const m = err instanceof Error ? err.message : String(err)
|
|
309
|
+
warn(`plugin '${plugin.name}' failed in admit: ${m}; admitting every task from here on`)
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
return true
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
|
|
171
316
|
/**
|
|
172
317
|
* Collect every plugin's `cache` layer in declaration order. One layer is
|
|
173
318
|
* used as is; two or more are chained (lookup walks them, save reaches all;
|
|
174
319
|
* see ChainedCache). A bare local layer that another declared layer already
|
|
175
320
|
* wraps (`layer.local === ctx.localCache`) is dropped, so a remote plugin
|
|
176
|
-
* that layers over the local handle
|
|
177
|
-
*
|
|
321
|
+
* that layers over the local handle does not also write the local store
|
|
322
|
+
* directly. A plugin declaring nothing leaves that store, unwrapped.
|
|
178
323
|
*/
|
|
179
324
|
export async function resolveCache(
|
|
180
325
|
plugins: readonly VxPlugin[],
|
|
@@ -184,25 +329,30 @@ export async function resolveCache(
|
|
|
184
329
|
for (const plugin of plugins) {
|
|
185
330
|
if (plugin.cache === undefined) continue
|
|
186
331
|
const layer = await safe(plugin, 'cache', () => plugin.cache!(ctx))
|
|
187
|
-
if (layer
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
)
|
|
332
|
+
if (layer === undefined) continue
|
|
333
|
+
// The seam's WHOLE contract, checked once here: a layer missing a
|
|
334
|
+
// method otherwise failed at the first task that reached it — an
|
|
335
|
+
// internal TypeError deep in the chain, naming neither the plugin nor
|
|
336
|
+
// the hook. Five names were checked before, so a layer with those five
|
|
337
|
+
// passed and died at its first hit inside restoreOutputs.
|
|
338
|
+
assertShape(plugin, 'cache', layer, CACHE_LAYER_METHODS, 'a cache layer')
|
|
339
|
+
layers.push(layer)
|
|
194
340
|
}
|
|
341
|
+
// Core's own store is the TAIL of the chain — the floor under every
|
|
342
|
+
// lookup, not a plugin a workspace has to declare. A layer that WRAPS
|
|
343
|
+
// the local handle subsumes it below, so a remote plugin does not write
|
|
344
|
+
// the local store twice.
|
|
345
|
+
if (!layers.includes(ctx.localCache)) layers.push(ctx.localCache)
|
|
195
346
|
const wrapsLocal = layers.some((l) => l !== ctx.localCache && l.local === ctx.localCache)
|
|
196
347
|
const distinct = wrapsLocal ? layers.filter((l) => l !== ctx.localCache) : layers
|
|
197
348
|
return distinct.length === 1 ? distinct[0]! : new ChainedCache(distinct)
|
|
198
349
|
}
|
|
199
350
|
|
|
200
351
|
/**
|
|
201
|
-
* Collect every plugin's `executor`, in declaration order
|
|
202
|
-
*
|
|
203
|
-
* that accepts
|
|
204
|
-
*
|
|
205
|
-
* executor is load-bearing, not observational.
|
|
352
|
+
* Collect every plugin's `executor`, in declaration order, with core's
|
|
353
|
+
* own appended at the tail: per task, `selectExecutor` takes the first
|
|
354
|
+
* that accepts, and what every plugin declines runs here. A broken
|
|
355
|
+
* factory aborts — an executor is load-bearing, not observational.
|
|
206
356
|
*/
|
|
207
357
|
export async function resolveExecutors(
|
|
208
358
|
plugins: readonly VxPlugin[],
|
|
@@ -212,16 +362,17 @@ export async function resolveExecutors(
|
|
|
212
362
|
for (const plugin of plugins) {
|
|
213
363
|
if (plugin.executor === undefined) continue
|
|
214
364
|
const executor = await safe(plugin, 'executor', () => plugin.executor!(ctx))
|
|
215
|
-
if (executor
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
throw new UserError(
|
|
222
|
-
`no executor plugin declared${declined.length > 0 ? ` (${declined.join(', ')})` : ''}. ${MISSING_PLUGIN_HINT}`,
|
|
223
|
-
)
|
|
365
|
+
if (executor === undefined) continue
|
|
366
|
+
assertShape(plugin, 'executor', executor, ['execute'], 'an executor')
|
|
367
|
+
if (typeof executor.name !== 'string' || executor.name.length === 0) {
|
|
368
|
+
throw new UserError(`plugin '${plugin.name}' returned an executor with no name`)
|
|
369
|
+
}
|
|
370
|
+
executors.push(executor)
|
|
224
371
|
}
|
|
372
|
+
// Core's own executor is the TAIL of every list, so a plugin executor
|
|
373
|
+
// that declines a task hands it back to this machine rather than
|
|
374
|
+
// failing the run.
|
|
375
|
+
executors.push(localExecutor())
|
|
225
376
|
return executors
|
|
226
377
|
}
|
|
227
378
|
|
|
@@ -8,11 +8,15 @@
|
|
|
8
8
|
// run with a clean UserError naming plugin + hook. A plugin that throws
|
|
9
9
|
// inside a bus hook is logged and disabled for the remainder of the run.
|
|
10
10
|
|
|
11
|
+
import { readFileSync } from 'node:fs'
|
|
12
|
+
import path from 'node:path'
|
|
13
|
+
import { fileURLToPath } from 'node:url'
|
|
11
14
|
import type { Cache, CacheLayer, CachePolicy } from '../cache/index.js'
|
|
12
|
-
import type
|
|
15
|
+
import { PLUGIN_PACKAGE, type ProjectConfig, type WorkspaceConfig } from '../config.js'
|
|
13
16
|
import type { TaskExecutor } from '../exec/index.js'
|
|
14
17
|
import type { TaskNode, TaskOutcome } from '../graph/index.js'
|
|
15
18
|
import { UserError } from '../util/index.js'
|
|
19
|
+
import type { ProjectMeta } from '../workspace/index.js'
|
|
16
20
|
import type { EventBus, RunStartInfo } from './events.js'
|
|
17
21
|
import type { TelemetryContext, TelemetrySink } from './telemetry.js'
|
|
18
22
|
|
|
@@ -32,7 +36,12 @@ import type { TelemetryContext, TelemetrySink } from './telemetry.js'
|
|
|
32
36
|
* outside the list.
|
|
33
37
|
*/
|
|
34
38
|
export interface VxPlugin {
|
|
35
|
-
/**
|
|
39
|
+
/**
|
|
40
|
+
* The name of the package the plugin was defined in — set by
|
|
41
|
+
* `definePlugin` from the nearest `package.json`, never by the plugin.
|
|
42
|
+
* Heads every warning about the plugin, every `vx info` line, and the
|
|
43
|
+
* key material a `key` hook contributes.
|
|
44
|
+
*/
|
|
36
45
|
readonly name: string
|
|
37
46
|
|
|
38
47
|
// --- PIPELINE stages (shape the run before it executes — opt-in) ----------
|
|
@@ -54,7 +63,10 @@ export interface VxPlugin {
|
|
|
54
63
|
* One project's validated config, right after it loaded and before the
|
|
55
64
|
* graph is built. Add, remove or edit tasks. Runs for every loaded project
|
|
56
65
|
* on every run — a config's cached evaluation is the user's file, and the
|
|
57
|
-
* hook is applied on top of it live.
|
|
66
|
+
* hook is applied on top of it live. A package with NO config file is
|
|
67
|
+
* loaded too, as `{ tasks: {} }`, so a plugin can give tasks to packages
|
|
68
|
+
* that never wrote one (`ctx.packageJson` carries their scripts); with no
|
|
69
|
+
* `project` plugin such a package is never visited.
|
|
58
70
|
*/
|
|
59
71
|
project?(config: ProjectConfig, ctx: ProjectHookContext): void | Promise<void>
|
|
60
72
|
|
|
@@ -82,6 +94,26 @@ export interface VxPlugin {
|
|
|
82
94
|
| undefined
|
|
83
95
|
| Promise<Readonly<Record<string, string>> | undefined>
|
|
84
96
|
|
|
97
|
+
/**
|
|
98
|
+
* Workspace-root files this plugin keys on its own, taken OUT of the
|
|
99
|
+
* workspace fingerprint. Core folds every lockfile it knows into every
|
|
100
|
+
* task's key, so one `pnpm install` re-keys the whole workspace. A plugin
|
|
101
|
+
* that reads the lockfile and folds each project's own dependency
|
|
102
|
+
* closure through `key` claims the file here: core leaves it out of the
|
|
103
|
+
* fingerprint every key sees (the config-evaluation cache still keys on
|
|
104
|
+
* it — a config may import a dependency), and `--affected` asks
|
|
105
|
+
* `affected` which projects a change to it touches instead of selecting
|
|
106
|
+
* every project. A file has one claimant.
|
|
107
|
+
*
|
|
108
|
+
* A root file core does NOT fold is claimed the same way when the
|
|
109
|
+
* plugin's stages read it: `turbo()` shapes every task from
|
|
110
|
+
* `turbo.json`, so an edit to it re-keys tasks through their resolved
|
|
111
|
+
* configs while no project owns the path. Nothing is taken out;
|
|
112
|
+
* `--affected` asks `affected` (undefined: every project) and `vx watch`
|
|
113
|
+
* re-runs on an edit (item 961). A claim is a bare name at the root.
|
|
114
|
+
*/
|
|
115
|
+
readonly fingerprint?: FingerprintClaim
|
|
116
|
+
|
|
85
117
|
/**
|
|
86
118
|
* Scheduling priorities: task id → weight, higher runs first among READY
|
|
87
119
|
* tasks (merged over the structural baseline, which stays the tie-break).
|
|
@@ -93,6 +125,20 @@ export interface VxPlugin {
|
|
|
93
125
|
ctx: ScheduleHookContext,
|
|
94
126
|
): ReadonlyMap<string, number> | undefined | Promise<ReadonlyMap<string, number> | undefined>
|
|
95
127
|
|
|
128
|
+
/**
|
|
129
|
+
* Admission over the worker count: asked for every task about to start
|
|
130
|
+
* on this machine, with the tasks running here right now. Return `false`
|
|
131
|
+
* to hold it until something finishes (it is asked again then). Asked
|
|
132
|
+
* many times per run, so it must be cheap and synchronous; a throw is
|
|
133
|
+
* reported once and the plugin admits from then on — a policy never
|
|
134
|
+
* breaks a run. Restore-tier hits and tasks on an executor pool hold no
|
|
135
|
+
* local resources and are never asked. When several plugins answer, all
|
|
136
|
+
* must admit. Core keeps no notion of what a task needs: a plugin that
|
|
137
|
+
* packs memory or CPU learns or declares the numbers itself
|
|
138
|
+
* (`@vzn/vx-schedule-history` packs what past executions used).
|
|
139
|
+
*/
|
|
140
|
+
admit?(task: TaskNode, ctx: AdmitContext): boolean
|
|
141
|
+
|
|
96
142
|
// --- CLI verbs (opt-in) ---------------------------------------------------
|
|
97
143
|
|
|
98
144
|
/**
|
|
@@ -110,8 +156,9 @@ export interface VxPlugin {
|
|
|
110
156
|
* Contribute a cache layer. Returns a CacheLayer wrapping (or replacing)
|
|
111
157
|
* the local Cache, or undefined to decline. Consulted ONCE per prepareRun.
|
|
112
158
|
* Precedence: first non-undefined plugin cache wins, in declaration
|
|
113
|
-
* order
|
|
114
|
-
*
|
|
159
|
+
* order, ahead of core's own `.vx/cache` handle at the tail. A layer
|
|
160
|
+
* that WRAPS the local handle subsumes that tail, so it is not written
|
|
161
|
+
* twice.
|
|
115
162
|
*/
|
|
116
163
|
cache?(ctx: CacheContext): CacheLayer | undefined | Promise<CacheLayer | undefined>
|
|
117
164
|
|
|
@@ -119,10 +166,10 @@ export interface VxPlugin {
|
|
|
119
166
|
* Contribute a task executor — WHERE one task's command runs. Consulted
|
|
120
167
|
* ONCE per run; every contributed executor is kept, in declaration order,
|
|
121
168
|
* and per task the first whose `accepts()` passes executes it. Nothing is
|
|
122
|
-
* appended:
|
|
123
|
-
*
|
|
124
|
-
*
|
|
125
|
-
*
|
|
169
|
+
* appended in front: core's own local executor is always the TAIL, so
|
|
170
|
+
* declining a task hands it back to this machine rather than failing
|
|
171
|
+
* the run. Persistent tasks never reach an executor (local by
|
|
172
|
+
* construction).
|
|
126
173
|
*/
|
|
127
174
|
executor?(ctx: ExecutorContext): TaskExecutor | undefined | Promise<TaskExecutor | undefined>
|
|
128
175
|
|
|
@@ -146,9 +193,13 @@ export interface VxPlugin {
|
|
|
146
193
|
| Promise<TelemetrySink | TelemetrySink[] | undefined>
|
|
147
194
|
|
|
148
195
|
/**
|
|
149
|
-
* Optional one-time setup
|
|
150
|
-
*
|
|
151
|
-
*
|
|
196
|
+
* Optional one-time setup per run (validate the workspace, open a
|
|
197
|
+
* connection, read a token). Throwing aborts the run with a clean
|
|
198
|
+
* UserError naming the plugin. It is not a guard over the pipeline: the
|
|
199
|
+
* config, project, cache, graph, key and schedule stages have run by the
|
|
200
|
+
* time it is called, and a plan (`--dry`, `planRun`) never calls it. It
|
|
201
|
+
* precedes the executors, the telemetry sinks, admission and the first
|
|
202
|
+
* task (tests/plugin-e2e.test.ts).
|
|
152
203
|
*/
|
|
153
204
|
setup?(ctx: PluginSetupContext): void | Promise<void>
|
|
154
205
|
|
|
@@ -188,6 +239,18 @@ export interface ProjectHookContext extends BaseContext {
|
|
|
188
239
|
readonly dir: string
|
|
189
240
|
/** The parsed `package.json`, read-only. */
|
|
190
241
|
readonly packageJson: Readonly<Record<string, unknown>>
|
|
242
|
+
/**
|
|
243
|
+
* Every package the workspace discovered — the one being visited, the
|
|
244
|
+
* rest of the run's scope and the packages outside it, config file or
|
|
245
|
+
* not — as core read them at startup. A plugin whose mapping needs the
|
|
246
|
+
* whole workspace (a `dependsOn` is only valid against every package's
|
|
247
|
+
* scripts at once) reads it here instead of walking the workspace a
|
|
248
|
+
* second time. The same array is handed to every visit of a run and a
|
|
249
|
+
* new one to the next run — a plugin instance outlives a run under `vx
|
|
250
|
+
* watch`, so a per-run memo keys on this array's identity, never on the
|
|
251
|
+
* process.
|
|
252
|
+
*/
|
|
253
|
+
readonly projects: readonly ProjectMeta[]
|
|
191
254
|
}
|
|
192
255
|
|
|
193
256
|
export interface GraphHookContext extends BaseContext {
|
|
@@ -197,6 +260,42 @@ export interface GraphHookContext extends BaseContext {
|
|
|
197
260
|
|
|
198
261
|
export interface KeyHookContext extends BaseContext {}
|
|
199
262
|
|
|
263
|
+
/** A plugin's claim on workspace fingerprint files — see `VxPlugin.fingerprint`. */
|
|
264
|
+
export interface FingerprintClaim {
|
|
265
|
+
/** Bare names at the workspace root: a lockfile core folds (`pnpm-lock.yaml`) or a file the plugin reads (`turbo.json`). */
|
|
266
|
+
readonly files: readonly string[]
|
|
267
|
+
/**
|
|
268
|
+
* The projects (package names) a change to a claimed file affects. Called
|
|
269
|
+
* by `--affected` with the file's bytes at the base ref and in the
|
|
270
|
+
* working tree (`null` where it does not exist). Return `undefined` to
|
|
271
|
+
* select every project — the answer a claimant gives when it cannot
|
|
272
|
+
* tell, and what core does for an unclaimed file.
|
|
273
|
+
*/
|
|
274
|
+
affected(
|
|
275
|
+
change: FingerprintChange,
|
|
276
|
+
ctx: FingerprintContext,
|
|
277
|
+
): Iterable<string> | undefined | Promise<Iterable<string> | undefined>
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
export interface FingerprintChange {
|
|
281
|
+
/** The claimed file's root-relative name. */
|
|
282
|
+
readonly file: string
|
|
283
|
+
readonly before: Uint8Array | null
|
|
284
|
+
readonly after: Uint8Array | null
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
export interface FingerprintContext extends BaseContext {
|
|
288
|
+
/** Every project in the workspace: package name and absolute directory. */
|
|
289
|
+
readonly projects: ReadonlyArray<{ readonly name: string; readonly dir: string }>
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
export interface AdmitContext {
|
|
293
|
+
/** The tasks executing on this machine right now, in dispatch order. */
|
|
294
|
+
readonly running: readonly TaskNode[]
|
|
295
|
+
/** The run's worker count — the ceiling the count gate already applies. */
|
|
296
|
+
readonly concurrency: number
|
|
297
|
+
}
|
|
298
|
+
|
|
200
299
|
export interface ScheduleHookContext extends BaseContext {
|
|
201
300
|
/** The run's local cache handle — its `dbHandle()` holds the run history a policy can learn from. */
|
|
202
301
|
readonly localCache: Cache
|
|
@@ -219,7 +318,10 @@ export interface PluginContext {
|
|
|
219
318
|
readonly workspaceRoot: string
|
|
220
319
|
/** Where vx's cache lives — read-only as far as the plugin is concerned. */
|
|
221
320
|
readonly cacheDir: string
|
|
222
|
-
/**
|
|
321
|
+
/**
|
|
322
|
+
* The run event bus. A plugin can subscribe directly if its needs exceed
|
|
323
|
+
* the hooks; the subscription ends with the run, as a hook's does.
|
|
324
|
+
*/
|
|
223
325
|
readonly bus: EventBus
|
|
224
326
|
/**
|
|
225
327
|
* Convenience: register a typed handler keyed off `RunEvent.kind`.
|
|
@@ -263,6 +365,70 @@ export interface Plugin {
|
|
|
263
365
|
setup?(ctx: PluginContext): void | Promise<void>
|
|
264
366
|
}
|
|
265
367
|
|
|
368
|
+
/** What a plugin author writes: every hook, and no name. */
|
|
369
|
+
export type PluginHooks = Omit<VxPlugin, 'name'>
|
|
370
|
+
|
|
371
|
+
/**
|
|
372
|
+
* Where a plugin is defined — `import.meta` of its module. `dir` is Bun's
|
|
373
|
+
* field; `url` is the standard one, for a module evaluated elsewhere.
|
|
374
|
+
*/
|
|
375
|
+
export interface PluginOrigin {
|
|
376
|
+
readonly dir?: string
|
|
377
|
+
readonly url?: string
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
const packageNameByDir = new Map<string, string>()
|
|
381
|
+
|
|
382
|
+
/** The name of the nearest `package.json` above `dir` — the one that owns it. */
|
|
383
|
+
function pluginPackageName(dir: string): string {
|
|
384
|
+
const memo = packageNameByDir.get(dir)
|
|
385
|
+
if (memo !== undefined) return memo
|
|
386
|
+
for (let d = dir; ;) {
|
|
387
|
+
let text: string | undefined
|
|
388
|
+
try {
|
|
389
|
+
text = readFileSync(path.join(d, 'package.json'), 'utf8')
|
|
390
|
+
} catch {
|
|
391
|
+
/* not here; look one level up */
|
|
392
|
+
}
|
|
393
|
+
if (text !== undefined) {
|
|
394
|
+
const name = (JSON.parse(text) as { name?: unknown }).name
|
|
395
|
+
if (typeof name !== 'string' || name.length === 0) {
|
|
396
|
+
throw new UserError(
|
|
397
|
+
`definePlugin: ${path.join(d, 'package.json')} has no name — a plugin is a package, and its name is the package's`,
|
|
398
|
+
)
|
|
399
|
+
}
|
|
400
|
+
packageNameByDir.set(dir, name)
|
|
401
|
+
return name
|
|
402
|
+
}
|
|
403
|
+
const parent = path.dirname(d)
|
|
404
|
+
if (parent === d) {
|
|
405
|
+
throw new UserError(
|
|
406
|
+
`definePlugin: no package.json above ${dir} — a plugin is a package, and its name is the package's`,
|
|
407
|
+
)
|
|
408
|
+
}
|
|
409
|
+
d = parent
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
/**
|
|
414
|
+
* The one way to make a plugin: `definePlugin(import.meta, { ...hooks })`.
|
|
415
|
+
* The name is read from the package the calling module belongs to — the
|
|
416
|
+
* nearest `package.json` above it — and stamped where the workspace loader
|
|
417
|
+
* checks for it, so a plugin cannot be named anything but its package.
|
|
418
|
+
*/
|
|
419
|
+
export function definePlugin(origin: PluginOrigin, hooks: PluginHooks): VxPlugin {
|
|
420
|
+
if ('name' in hooks) {
|
|
421
|
+
throw new UserError(`definePlugin: a plugin's name is its package name — drop the 'name' field`)
|
|
422
|
+
}
|
|
423
|
+
const dir =
|
|
424
|
+
origin.dir ?? (origin.url !== undefined ? path.dirname(fileURLToPath(origin.url)) : undefined)
|
|
425
|
+
if (dir === undefined) {
|
|
426
|
+
throw new UserError(`definePlugin: the first argument must be the plugin module's import.meta`)
|
|
427
|
+
}
|
|
428
|
+
const name = pluginPackageName(dir)
|
|
429
|
+
return { ...hooks, name, [PLUGIN_PACKAGE]: name } as VxPlugin
|
|
430
|
+
}
|
|
431
|
+
|
|
266
432
|
export interface InstallPluginsArgs {
|
|
267
433
|
plugins: readonly Plugin[]
|
|
268
434
|
workspaceRoot: string
|
|
@@ -307,7 +473,17 @@ export async function installPlugins(args: InstallPluginsArgs): Promise<() => vo
|
|
|
307
473
|
const ctx: PluginContext = {
|
|
308
474
|
workspaceRoot,
|
|
309
475
|
cacheDir,
|
|
310
|
-
|
|
476
|
+
// A subscription made past the hooks leaves with the run like a
|
|
477
|
+
// hook's does: the bus can outlive the run (`RunOptions.bus`), and
|
|
478
|
+
// one left behind hears every later run on it (item 635).
|
|
479
|
+
bus: {
|
|
480
|
+
emit: (event) => bus.emit(event),
|
|
481
|
+
subscribe(subscriber) {
|
|
482
|
+
const dispose = bus.subscribe(subscriber)
|
|
483
|
+
disposers.push(dispose)
|
|
484
|
+
return dispose
|
|
485
|
+
},
|
|
486
|
+
},
|
|
311
487
|
on(hook, handler) {
|
|
312
488
|
const dispose = bus.subscribe((event) => {
|
|
313
489
|
if (disabled.has(plugin.name)) return
|