@vzn/vx 0.0.637 → 0.0.639
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/caching.md +84 -28
- package/docs/cli.md +273 -408
- package/docs/execution.md +19 -44
- package/docs/features.md +13 -15
- package/docs/patterns.md +5 -7
- package/docs/schema.md +68 -61
- package/package.json +5 -5
- package/schemas/last.json +7 -4
- package/schemas/plan.json +2 -2
- package/skills/vx/SKILL.md +19 -6
- package/src/bin.ts +13 -7
- package/src/cache/cache.ts +60 -11
- package/src/cache/git-inputs.ts +6 -0
- package/src/cache/index.ts +10 -2
- package/src/cache/inputs.ts +0 -24
- package/src/cache/layer.ts +32 -12
- package/src/cache/layered-cache.ts +112 -315
- package/src/cache/plugin-cache.ts +240 -0
- package/src/cache/policy.ts +9 -0
- package/src/cache/remote-lane.ts +383 -0
- package/src/cache/run-history.ts +92 -8
- package/src/cache/run-store.ts +77 -0
- package/src/cache/schema.ts +44 -13
- package/src/cache/scoped-cache.ts +22 -2
- package/src/cli/cache.ts +33 -16
- package/src/cli/completions.ts +11 -7
- package/src/cli/foreign-flags.ts +10 -22
- package/src/cli/help.ts +22 -38
- package/src/cli/index.ts +25 -11
- package/src/cli/info.ts +8 -6
- package/src/cli/init.ts +14 -15
- package/src/cli/last.ts +20 -10
- package/src/cli/lock.ts +39 -27
- package/src/cli/plan-format.ts +10 -61
- package/src/cli/plugin-commands.ts +77 -6
- package/src/cli/plugin-templates.ts +1 -1
- package/src/cli/run.ts +31 -191
- package/src/cli/select.ts +18 -9
- package/src/cli/show.ts +72 -40
- package/src/cli/task-verb.ts +4 -4
- package/src/cli/upgrade.ts +132 -39
- package/src/cli/watch-cycle.ts +2 -1
- package/src/cli/watch-filter.ts +7 -2
- package/src/cli/watch-judge.ts +6 -1
- package/src/cli/watch.ts +73 -34
- package/src/cli/why.ts +14 -5
- package/src/cli/workspace-config.ts +2 -2
- package/src/config.ts +0 -6
- package/src/exec/executor.ts +1 -1
- package/src/exec/sandbox-runtime.ts +1 -1
- package/src/exec/sandbox-violations.ts +3 -1
- package/src/graph/index.ts +9 -1
- package/src/graph/priorities.ts +1 -1
- package/src/graph/scheduler.ts +12 -16
- package/src/graph/task-graph.ts +119 -21
- package/src/graph/task-status.ts +95 -0
- package/src/{orchestrator → history}/failure-mode.ts +20 -11
- package/src/{orchestrator → history}/history.ts +61 -5
- package/src/history/index.ts +48 -0
- package/src/{orchestrator → history}/metrics.ts +14 -23
- package/src/{orchestrator → history}/miss-reason.ts +1 -1
- package/src/index.ts +21 -29
- package/src/orchestrator/affected-tasks.ts +8 -3
- package/src/orchestrator/doctor.ts +2 -2
- package/src/orchestrator/execute-args.ts +80 -0
- package/src/orchestrator/execute-task.ts +47 -429
- package/src/orchestrator/fingerprint-watch.ts +2 -2
- package/src/orchestrator/index.ts +3 -74
- package/src/orchestrator/miss-save.ts +33 -3
- package/src/orchestrator/options.ts +1 -36
- package/src/orchestrator/output-host.ts +4 -3
- package/src/orchestrator/persistent-task.ts +321 -0
- package/src/orchestrator/persistent.ts +130 -3
- package/src/orchestrator/placement.ts +3 -3
- package/src/orchestrator/plan.ts +18 -5
- package/src/orchestrator/plugin-host.ts +77 -47
- package/src/orchestrator/plugin.ts +86 -14
- package/src/orchestrator/prepare.ts +74 -31
- package/src/orchestrator/projects.ts +3 -0
- package/src/orchestrator/remote-prefetch.ts +23 -6
- package/src/orchestrator/run-artifacts.ts +5 -177
- package/src/orchestrator/run-context.ts +55 -0
- package/src/orchestrator/run-failures.ts +15 -3
- package/src/orchestrator/run-records.ts +36 -4
- package/src/orchestrator/run-report.ts +81 -179
- package/src/orchestrator/run-settings.ts +1 -16
- package/src/orchestrator/run-stop.ts +153 -0
- package/src/orchestrator/run.ts +313 -555
- package/src/orchestrator/sandbox-fix.ts +0 -16
- package/src/orchestrator/signals.ts +14 -6
- package/src/orchestrator/stable-keys.ts +107 -3
- package/src/orchestrator/steps.ts +50 -0
- package/src/orchestrator/task-hash.ts +3 -16
- package/src/orchestrator/task-log-buffer.ts +1 -1
- package/src/orchestrator/task-process.ts +94 -0
- package/src/orchestrator/telemetry-host.ts +1 -1
- package/src/orchestrator/telemetry.ts +6 -83
- package/src/{orchestrator → report}/events.ts +41 -8
- package/src/{orchestrator → report}/forecast.ts +35 -2
- package/src/{orchestrator → report}/framed-output.ts +39 -34
- package/src/report/index.ts +36 -0
- package/src/{orchestrator → report}/logger.ts +166 -21
- package/src/{orchestrator → report}/status-line.ts +84 -7
- package/src/{orchestrator → report}/summary.ts +179 -63
- package/src/report/tally.ts +83 -0
- package/src/settings.ts +13 -23
- package/src/{orchestrator → util}/colors.ts +49 -4
- package/src/util/config-frame.ts +31 -2
- package/src/util/errors.ts +32 -0
- package/src/util/git-ignored.ts +143 -0
- package/src/util/index.ts +25 -3
- package/src/util/refusal-notice.ts +111 -0
- package/src/util/report.ts +39 -0
- package/src/util/settle.ts +14 -39
- package/src/util/tail.ts +1 -1
- package/src/util/ui.ts +197 -0
- package/src/workspace/affected.ts +20 -23
- package/src/workspace/dependency-rules.ts +12 -4
- package/src/workspace/migration.ts +87 -91
- package/src/workspace/workspace.ts +5 -4
- package/schemas/summary.json +0 -133
- package/src/cache/chained-cache.ts +0 -276
- package/src/orchestrator/tally.ts +0 -112
- /package/src/{orchestrator → report}/path-links.ts +0 -0
- /package/src/{orchestrator → report}/plain-output.ts +0 -0
- /package/src/{orchestrator → report}/terminal-signals.ts +0 -0
|
@@ -3,10 +3,11 @@
|
|
|
3
3
|
// ownership moves to the run's registry, and this is the one place that
|
|
4
4
|
// decides which of them outlive the graph and how the rest go down.
|
|
5
5
|
|
|
6
|
-
import { isGroupTask, type TaskNode } from '../graph/index.js'
|
|
6
|
+
import { isGroupTask, isPassStatus, type TaskNode, type TaskOutcome } from '../graph/index.js'
|
|
7
7
|
import { killGraceMs } from '../util/index.js'
|
|
8
|
-
import { holdGroups, killTree, untilGroupsGone } from '../exec/index.js'
|
|
9
|
-
import type { HeldServer } from './options.js'
|
|
8
|
+
import { holdGroups, killTree, signalExitCode, untilGroupsGone } from '../exec/index.js'
|
|
9
|
+
import type { HeldPersistent, HeldServer } from './options.js'
|
|
10
|
+
import { terminateChildren } from './signals.js'
|
|
10
11
|
|
|
11
12
|
type Child = ReturnType<typeof Bun.spawn>
|
|
12
13
|
|
|
@@ -152,3 +153,129 @@ export function takeHeldServers(
|
|
|
152
153
|
}
|
|
153
154
|
return { kept, stale }
|
|
154
155
|
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* The kept server whose exit ends a foreground session, and its code. A
|
|
159
|
+
* server kept only as a dependency that exits 0 is a daemon that forked
|
|
160
|
+
* and returned (schema.md), not the end of the session: a
|
|
161
|
+
* `db: 'docker compose up -d'` under `app#dev` stopped the dev server
|
|
162
|
+
* right after the summary (WD-3). It ends the wait only as the last.
|
|
163
|
+
*/
|
|
164
|
+
export function sessionEnd(keepAlive: KeepAlive): Promise<{ code: number; i: number }> {
|
|
165
|
+
return new Promise((resolve) => {
|
|
166
|
+
let left = keepAlive.children.length
|
|
167
|
+
keepAlive.children.forEach((c, i) => {
|
|
168
|
+
void c.exited.then((code) => {
|
|
169
|
+
left--
|
|
170
|
+
const n = keepAlive.nodes[i]!
|
|
171
|
+
if (code !== 0 || n.requested || n.surfaced === true || left === 0) resolve({ code, i })
|
|
172
|
+
})
|
|
173
|
+
})
|
|
174
|
+
})
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
export interface HoldArgs {
|
|
178
|
+
forwardArgs: readonly string[] | undefined
|
|
179
|
+
/** Where a held server's output goes. */
|
|
180
|
+
outOf: (id: string) => HeldServer['out']
|
|
181
|
+
warn: (message: string) => void
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/** The kept servers handed to the caller (`RunOptions.hold`) instead of waited on. */
|
|
185
|
+
export function holdServers(keepAlive: KeepAlive, args: HoldArgs): HeldPersistent {
|
|
186
|
+
const held = keepAlive.children
|
|
187
|
+
// A held server that dies on its own is said, as the foreground's
|
|
188
|
+
// keep-alive says it: `vx watch` sat on "watching" over a dead dev
|
|
189
|
+
// server. The holder's own stop() is not such a death.
|
|
190
|
+
let stopping = false
|
|
191
|
+
// Taken by a later run (`RunOptions.keep`): kept or stopped there.
|
|
192
|
+
const handed = new Set<Child>()
|
|
193
|
+
const servers = new Map<string, HeldServer>()
|
|
194
|
+
keepAlive.nodes.forEach((n, i) => {
|
|
195
|
+
const child = held[i]!
|
|
196
|
+
servers.set(n.id, {
|
|
197
|
+
child,
|
|
198
|
+
config: serverConfig(n, args.forwardArgs),
|
|
199
|
+
out: args.outOf(n.id),
|
|
200
|
+
handOver: () => void handed.add(child),
|
|
201
|
+
})
|
|
202
|
+
// One that died during the graph was said then.
|
|
203
|
+
if (hasEnded(child)) return
|
|
204
|
+
void child.exited.then((code) => {
|
|
205
|
+
if (!stopping && !handed.has(child) && code !== 0)
|
|
206
|
+
args.warn(`vx: ${n.id} exited with code ${code}`)
|
|
207
|
+
})
|
|
208
|
+
})
|
|
209
|
+
return {
|
|
210
|
+
ids: keepAlive.nodes.map((n) => n.id),
|
|
211
|
+
servers,
|
|
212
|
+
stop: (signal) => {
|
|
213
|
+
stopping = true
|
|
214
|
+
return terminateChildren(() => held.filter((c) => !handed.has(c)), signal)
|
|
215
|
+
},
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* A server that died on its own before the stop failed, whatever its
|
|
221
|
+
* outcome said when it became ready: the footer counted it a success over
|
|
222
|
+
* the run's red exit (item 1071). Its code is its own, or the shell's
|
|
223
|
+
* report of the signal that took it. In place: the renderer holds this
|
|
224
|
+
* outcome (events carry live refs) and closes the server's output block
|
|
225
|
+
* from it at runEnd, where a copy left it reading `running` under its exit
|
|
226
|
+
* line (WD-10).
|
|
227
|
+
*/
|
|
228
|
+
export function failServer(
|
|
229
|
+
outcomes: ReadonlyMap<string, TaskOutcome>,
|
|
230
|
+
id: string,
|
|
231
|
+
code: number | string,
|
|
232
|
+
): void {
|
|
233
|
+
const o = outcomes.get(id)
|
|
234
|
+
if (o === undefined) return
|
|
235
|
+
o.status = 'failed'
|
|
236
|
+
o.exitCode = typeof code === 'number' ? code : signalExitCode(code)
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/**
|
|
240
|
+
* The end of the graph for its servers: which stay up (`selectKeepAlive`),
|
|
241
|
+
* the bounded shutdown of the rest, and each one that died on its own
|
|
242
|
+
* failed. A run that failed elsewhere keeps nothing: `vx run dev --all`
|
|
243
|
+
* with one server that never became ready sat on the others for good, so
|
|
244
|
+
* a script hung, and the Ctrl-C that ended it read 130 over the failure
|
|
245
|
+
* (C-60). A kept server's own crash ends the keep-alive wait as before,
|
|
246
|
+
* and `--continue=always` asked to keep going, and does. The watch loop
|
|
247
|
+
* (`hold`) still holds: a failing test in a cycle stopped the dev server
|
|
248
|
+
* the next change would restart anyway. A kept server that has already
|
|
249
|
+
* died fails too; the keep-alive wait names it.
|
|
250
|
+
*/
|
|
251
|
+
export async function disposeServers(args: {
|
|
252
|
+
registry: ReadonlyMap<string, Child>
|
|
253
|
+
nodes: ReadonlyMap<string, TaskNode>
|
|
254
|
+
outcomes: ReadonlyMap<string, TaskOutcome>
|
|
255
|
+
/** Keep the requested servers up: the CLI foreground, or a hold. */
|
|
256
|
+
keep: boolean
|
|
257
|
+
hold: boolean
|
|
258
|
+
continueAlways: boolean
|
|
259
|
+
/** The servers that had ended when a stop landed; undefined without a stop. */
|
|
260
|
+
endedBeforeStop: ReadonlySet<Child> | undefined
|
|
261
|
+
}): Promise<{ keepAlive: KeepAlive; crashed: CrashedPersistent[] }> {
|
|
262
|
+
const { registry, outcomes, endedBeforeStop } = args
|
|
263
|
+
let keepAlive = selectKeepAlive(registry, args.nodes, args.keep)
|
|
264
|
+
const failedElsewhere =
|
|
265
|
+
![...outcomes.values()].every((o) => isPassStatus(o.status)) ||
|
|
266
|
+
[...registry.values()].some(
|
|
267
|
+
(c) => hasEnded(c) && c.exitCode !== 0 && !keepAlive.children.includes(c),
|
|
268
|
+
)
|
|
269
|
+
if (failedElsewhere && !args.hold && !args.continueAlways) keepAlive = { nodes: [], children: [] }
|
|
270
|
+
const crashed = (await shutdownPersistent(registry, keepAlive.children)).filter(
|
|
271
|
+
(c) => endedBeforeStop?.has(registry.get(c.id)!) ?? true,
|
|
272
|
+
)
|
|
273
|
+
for (const c of crashed) failServer(outcomes, c.id, c.code)
|
|
274
|
+
keepAlive.nodes.forEach((n, i) => {
|
|
275
|
+
const child = keepAlive.children[i]!
|
|
276
|
+
if (!hasEnded(child) || child.exitCode === 0) return
|
|
277
|
+
if (endedBeforeStop !== undefined && !endedBeforeStop.has(child)) return
|
|
278
|
+
failServer(outcomes, n.id, child.exitCode ?? child.signalCode ?? 'unknown')
|
|
279
|
+
})
|
|
280
|
+
return { keepAlive, crashed }
|
|
281
|
+
}
|
|
@@ -11,7 +11,7 @@ import { machineParallelism, UserError } from '../util/index.js'
|
|
|
11
11
|
import { isLocalExecutor, selectExecutor, type TaskExecutor } from '../exec/index.js'
|
|
12
12
|
import { isGroupTask, type TaskNode } from '../graph/index.js'
|
|
13
13
|
import { resolveDownloadModes } from './download-policy.js'
|
|
14
|
-
import type
|
|
14
|
+
import { type Logger, warnOf } from '../report/index.js'
|
|
15
15
|
import { executorLabel, resolveExecutors } from './plugin-host.js'
|
|
16
16
|
import type { prepareRun } from './prepare.js'
|
|
17
17
|
|
|
@@ -218,11 +218,11 @@ export async function planExecutorOf(
|
|
|
218
218
|
executors = await resolveExecutors(prepared.plugins, {
|
|
219
219
|
workspaceRoot: prepared.workspaceRoot,
|
|
220
220
|
cacheDir: prepared.cacheDir,
|
|
221
|
-
warn: (
|
|
221
|
+
warn: warnOf(log),
|
|
222
222
|
concurrency: machineParallelism(),
|
|
223
223
|
})
|
|
224
224
|
} catch (err) {
|
|
225
|
-
log
|
|
225
|
+
warnOf(log)(
|
|
226
226
|
`[vx] placement not shown — ${err instanceof Error ? err.message : String(err)} (the run would refuse on it)`,
|
|
227
227
|
)
|
|
228
228
|
return {}
|
package/src/orchestrator/plan.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// Planning: walk the task graph, compute every task's cache key, and
|
|
2
2
|
// probe the cache to predict what would happen if `vx run` actually
|
|
3
|
-
// ran. Used by `--dry
|
|
3
|
+
// ran. Used by `--dry`.
|
|
4
4
|
//
|
|
5
5
|
// No TASK execution happens: no task spawn, no cleanOutputs, no
|
|
6
6
|
// restoreOutputs. Cache probing uses the byte-free `cache.has()`
|
|
@@ -13,10 +13,10 @@
|
|
|
13
13
|
|
|
14
14
|
import type { CacheLayer, CachePolicy, GitFilesCache } from '../cache/index.js'
|
|
15
15
|
import { cachesNothing, FULL_CACHE_POLICY } from '../cache/index.js'
|
|
16
|
-
import { ranNoCache } from '
|
|
16
|
+
import { ranNoCache } from '../report/index.js'
|
|
17
17
|
import { idleServers, isGroupTask, runGraph, type TaskNode } from '../graph/index.js'
|
|
18
18
|
import type { AffectedReason } from './affected-tasks.js'
|
|
19
|
-
import type { HistoryProvider } from '
|
|
19
|
+
import type { HistoryProvider } from '../history/index.js'
|
|
20
20
|
import { computeGroupKey, computeTaskHash } from './task-hash.js'
|
|
21
21
|
import { keyedOutcome, keyUpstream } from './upstream.js'
|
|
22
22
|
|
|
@@ -139,6 +139,9 @@ export interface PlanArgs {
|
|
|
139
139
|
*/
|
|
140
140
|
export async function plan(args: PlanArgs): Promise<RunPlan> {
|
|
141
141
|
const cacheStatusById = new Map<string, CacheStatus>()
|
|
142
|
+
// Keyed first, probed after: a remote that can batch (`hasMany`) is
|
|
143
|
+
// asked once for every key, one that cannot is asked per key.
|
|
144
|
+
const probed = new Map<string, string>()
|
|
142
145
|
|
|
143
146
|
const outcomes = await runGraph({
|
|
144
147
|
nodes: args.nodes,
|
|
@@ -176,13 +179,23 @@ export async function plan(args: PlanArgs): Promise<RunPlan> {
|
|
|
176
179
|
} else if (!reads) {
|
|
177
180
|
status = 'miss'
|
|
178
181
|
} else {
|
|
179
|
-
|
|
180
|
-
|
|
182
|
+
probed.set(node.id, hash)
|
|
183
|
+
return keyedOutcome(node, hash)
|
|
181
184
|
}
|
|
182
185
|
cacheStatusById.set(node.id, status)
|
|
183
186
|
return keyedOutcome(node, hash)
|
|
184
187
|
},
|
|
185
188
|
})
|
|
189
|
+
if (probed.size > 0) {
|
|
190
|
+
await args.cache.remoteHasMany?.([...new Set(probed.values())])
|
|
191
|
+
for (const [id, hash] of probed) {
|
|
192
|
+
const where = await args.cache.has(hash)
|
|
193
|
+
cacheStatusById.set(
|
|
194
|
+
id,
|
|
195
|
+
where === null ? 'miss' : where === 'remote' ? 'hit-remote' : 'hit-local',
|
|
196
|
+
)
|
|
197
|
+
}
|
|
198
|
+
}
|
|
186
199
|
|
|
187
200
|
if (args.restorable !== undefined) {
|
|
188
201
|
const restoreTier = new Set(
|
|
@@ -10,10 +10,12 @@
|
|
|
10
10
|
import path from 'node:path'
|
|
11
11
|
import {
|
|
12
12
|
type Cache,
|
|
13
|
-
ChainedCache,
|
|
14
13
|
type CacheLayer,
|
|
15
14
|
type CachePolicy,
|
|
16
15
|
LayeredCache,
|
|
16
|
+
absentPluginCache,
|
|
17
|
+
type PluginCache,
|
|
18
|
+
type ResolvedRemote,
|
|
17
19
|
} from '../cache/index.js'
|
|
18
20
|
import { localExecutor, type TaskExecutor } from '../exec/index.js'
|
|
19
21
|
import { maskedLine, settleWithin, teardownTimeoutMs, UserError } from '../util/index.js'
|
|
@@ -29,6 +31,7 @@ import type {
|
|
|
29
31
|
KeyHookContext,
|
|
30
32
|
DiscoveredProject,
|
|
31
33
|
ProjectHookContext,
|
|
34
|
+
RunReport,
|
|
32
35
|
ScheduleHookContext,
|
|
33
36
|
VxPlugin,
|
|
34
37
|
WorkspaceHookContext,
|
|
@@ -385,12 +388,14 @@ export async function claimedAffected(
|
|
|
385
388
|
export async function applyScheduleHooks(
|
|
386
389
|
plugins: readonly VxPlugin[],
|
|
387
390
|
nodes: ReadonlyMap<string, TaskNode>,
|
|
388
|
-
ctx: ScheduleHookContext,
|
|
391
|
+
ctx: Omit<ScheduleHookContext, 'cache'>,
|
|
392
|
+
cacheFor: (plugin: string) => PluginCache = () => absentPluginCache,
|
|
389
393
|
): Promise<ReadonlyMap<string, number>> {
|
|
390
394
|
const merged = new Map<string, number>()
|
|
391
395
|
for (const plugin of plugins) {
|
|
392
396
|
if (plugin.schedule === undefined) continue
|
|
393
|
-
const
|
|
397
|
+
const own = { ...ctx, cache: cacheFor(plugin.name) }
|
|
398
|
+
const weights = await safe(plugin, 'schedule', () => plugin.schedule!(nodes, own))
|
|
394
399
|
if (weights === undefined) continue
|
|
395
400
|
// Iterating a string destructures its characters into `[id, w]` pairs
|
|
396
401
|
// that match no task — a plugin returning `'fast'` used to be a no-op
|
|
@@ -478,68 +483,40 @@ export function buildAdmission(
|
|
|
478
483
|
export interface CacheHost extends CacheContext {
|
|
479
484
|
readonly localCache: Cache
|
|
480
485
|
readonly policy: CachePolicy
|
|
486
|
+
/** Filled with each remote and what its policy lets it do, for the plugins' `ctx.cache`. */
|
|
487
|
+
readonly remotes?: ResolvedRemote[]
|
|
481
488
|
}
|
|
482
489
|
|
|
483
490
|
/**
|
|
484
491
|
* Collect every plugin's remote (`cache` hook) in declaration order and
|
|
485
492
|
* layer each over the local store: a lookup asks the local store first,
|
|
486
493
|
* then each remote in order (owner, 2026-10-13), and a save reaches every
|
|
487
|
-
* remote. The run's policy, narrowed by a remote's `access`,
|
|
488
|
-
*
|
|
489
|
-
*
|
|
494
|
+
* remote. The run's policy, narrowed by a remote's `access`, gates each
|
|
495
|
+
* remote; a remote's throw is a miss and a warning naming its plugin. No
|
|
496
|
+
* remote → the local store alone.
|
|
490
497
|
*/
|
|
491
498
|
export async function resolveCache(
|
|
492
499
|
plugins: readonly VxPlugin[],
|
|
493
500
|
host: CacheHost,
|
|
494
501
|
): Promise<CacheLayer> {
|
|
495
|
-
const { localCache, policy, ...ctx } = host
|
|
496
|
-
const
|
|
497
|
-
const owners: string[] = []
|
|
502
|
+
const { localCache, policy, remotes, ...ctx } = host
|
|
503
|
+
const resolved: ResolvedRemote[] = []
|
|
498
504
|
for (const plugin of plugins) {
|
|
499
505
|
if (plugin.cache === undefined) continue
|
|
500
506
|
const remote = await safe(plugin, 'cache', () => plugin.cache!(ctx))
|
|
501
507
|
if (remote === undefined) continue
|
|
502
|
-
// The hook returned a whole layer until 0.0.637: name the fix, not the
|
|
503
|
-
// three methods such a layer happens to have.
|
|
504
|
-
if (isRecordLike(remote) && typeof remote['restoreOutputs'] === 'function') {
|
|
505
|
-
throw new UserError(
|
|
506
|
-
`plugin '${plugin.name}' returned a cache layer from cache: since vx 0.0.637 the hook ` +
|
|
507
|
-
'returns the remote alone (a RemoteCacheLayer: has, get, put) and core layers it over ' +
|
|
508
|
-
'the local cache — return the remote you passed to LayeredCache',
|
|
509
|
-
)
|
|
510
|
-
}
|
|
511
508
|
assertShape(plugin, 'cache', remote, REMOTE_CACHE_METHODS, 'a remote cache layer')
|
|
512
509
|
const access = remote.access
|
|
513
|
-
const
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
510
|
+
const entry: ResolvedRemote = {
|
|
511
|
+
remote,
|
|
512
|
+
read: policy.remoteRead && access?.read !== false,
|
|
513
|
+
write: policy.remoteWrite && access?.write !== false,
|
|
514
|
+
onError: (err) => ctx.warn(`[vx] plugin '${plugin.name}': ${err.message}`),
|
|
517
515
|
}
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
policy: own,
|
|
521
|
-
onRemoteError: (err) => ctx.warn(`[vx] plugin '${plugin.name}': ${err.message}`),
|
|
522
|
-
}),
|
|
523
|
-
)
|
|
524
|
-
owners.push(`the cache layer of plugin '${plugin.name}'`)
|
|
516
|
+
resolved.push(entry)
|
|
517
|
+
remotes?.push(entry)
|
|
525
518
|
}
|
|
526
|
-
|
|
527
|
-
if (layers.length === 1) return layers[0]!
|
|
528
|
-
// One line per layer and method: a layer that throws on every lookup
|
|
529
|
-
// would otherwise print once per task.
|
|
530
|
-
const told = new Set<string>()
|
|
531
|
-
return new ChainedCache(layers, (i, method, err) => {
|
|
532
|
-
if (told.has(`${i}\0${method}`)) return
|
|
533
|
-
told.add(`${i}\0${method}`)
|
|
534
|
-
const m = err instanceof Error ? err.message : String(err)
|
|
535
|
-
const then =
|
|
536
|
-
method === 'save' ? 'the other layers still save' : 'a miss there; the next layer answers'
|
|
537
|
-
ctx.warn(`[vx] ${owners[i]} failed in cache ${method}: ${m}; ${then}`)
|
|
538
|
-
})
|
|
539
|
-
}
|
|
540
|
-
|
|
541
|
-
function isRecordLike(v: unknown): v is Record<string, unknown> {
|
|
542
|
-
return typeof v === 'object' && v !== null
|
|
519
|
+
return resolved.length === 0 ? localCache : new LayeredCache(localCache, resolved)
|
|
543
520
|
}
|
|
544
521
|
|
|
545
522
|
/** The plugin each resolved executor came from, so a hook of it that throws is named. */
|
|
@@ -572,6 +549,55 @@ export function executorLabel(executor: TaskExecutor): string {
|
|
|
572
549
|
: `plugin '${plugin}' (executor '${executor.name}')`
|
|
573
550
|
}
|
|
574
551
|
|
|
552
|
+
/**
|
|
553
|
+
* Demand: what each executor still has left to run. An executor that
|
|
554
|
+
* provisions per task (a container, an allocation, a pod) otherwise has to
|
|
555
|
+
* guess from `capacity` alone and hold everything until teardown. Built
|
|
556
|
+
* ONLY for executors that asked: with none declared there is no map, no
|
|
557
|
+
* set, and the returned narrower does nothing.
|
|
558
|
+
*
|
|
559
|
+
* The SAME set is handed back every time, narrowed in place: a consumer
|
|
560
|
+
* reading it later sees the truth rather than a stale snapshot. A hint,
|
|
561
|
+
* as `admit` is: one that throws is named once and asked no more, where
|
|
562
|
+
* it surfaced as a bare stack, or mid-run from the completion path
|
|
563
|
+
* (item 1022). Returns the call that drops a finished task's id.
|
|
564
|
+
*/
|
|
565
|
+
export function executorDemand(
|
|
566
|
+
placed: ReadonlyMap<string, TaskExecutor>,
|
|
567
|
+
warn: (message: string) => void,
|
|
568
|
+
): (id: string) => void {
|
|
569
|
+
const demandOf = new Map<TaskExecutor, Set<string>>()
|
|
570
|
+
for (const [id, executor] of placed) {
|
|
571
|
+
if (executor.demand === undefined) continue
|
|
572
|
+
let remaining = demandOf.get(executor)
|
|
573
|
+
if (remaining === undefined) demandOf.set(executor, (remaining = new Set()))
|
|
574
|
+
remaining.add(id)
|
|
575
|
+
}
|
|
576
|
+
if (demandOf.size === 0) return () => {}
|
|
577
|
+
const tell = (executor: TaskExecutor, remaining: ReadonlySet<string>): void => {
|
|
578
|
+
const failed = (err: unknown): void => {
|
|
579
|
+
if (!demandOf.delete(executor)) return
|
|
580
|
+
const m = err instanceof Error ? err.message : String(err)
|
|
581
|
+
warn(`[vx] ${executorLabel(executor)} failed in demand: ${m}; not asked again this run`)
|
|
582
|
+
}
|
|
583
|
+
try {
|
|
584
|
+
// An `async demand` that rejects did so where no one listened: an
|
|
585
|
+
// unhandled rejection, a stack, the run killed. Its rejection is a
|
|
586
|
+
// throw like any other.
|
|
587
|
+
const ret: unknown = executor.demand!(remaining)
|
|
588
|
+
if (ret instanceof Promise) ret.catch(failed)
|
|
589
|
+
} catch (err) {
|
|
590
|
+
failed(err)
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
for (const [executor, remaining] of demandOf) tell(executor, remaining)
|
|
594
|
+
return (id) => {
|
|
595
|
+
for (const [executor, remaining] of demandOf) {
|
|
596
|
+
if (remaining.delete(id)) tell(executor, remaining)
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
|
|
575
601
|
/**
|
|
576
602
|
* Collect every plugin's `executor`, in declaration order, with core's
|
|
577
603
|
* own appended at the tail: per task, `selectExecutor` takes the first
|
|
@@ -628,12 +654,16 @@ export async function resolveExecutors(
|
|
|
628
654
|
export async function teardownPlugins(
|
|
629
655
|
plugins: readonly VxPlugin[],
|
|
630
656
|
warn: (message: string) => void,
|
|
657
|
+
report?: RunReport,
|
|
658
|
+
step: <T>(id: string, work: () => Promise<T>) => Promise<T> = (_id, work) => work(),
|
|
631
659
|
): Promise<void> {
|
|
632
660
|
for (const plugin of plugins) {
|
|
633
661
|
if (plugin.teardown === undefined) continue
|
|
634
662
|
const ms = teardownTimeoutMs()
|
|
635
663
|
try {
|
|
636
|
-
const settled = await
|
|
664
|
+
const settled = await step(`${plugin.name}:teardown`, () =>
|
|
665
|
+
settleWithin(Promise.resolve(plugin.teardown?.(report)), ms),
|
|
666
|
+
)
|
|
637
667
|
// Same rule as the flush above. A teardown that never settles has left
|
|
638
668
|
// whatever it owns un-released, and the run exits anyway — silence would
|
|
639
669
|
// present that as a clean shutdown.
|
|
@@ -11,15 +11,20 @@
|
|
|
11
11
|
import { readFileSync } from 'node:fs'
|
|
12
12
|
import path from 'node:path'
|
|
13
13
|
import { fileURLToPath } from 'node:url'
|
|
14
|
-
import
|
|
14
|
+
import {
|
|
15
|
+
absentPluginCache,
|
|
16
|
+
type Cache,
|
|
17
|
+
type PluginCache,
|
|
18
|
+
type RemoteCacheLayer,
|
|
19
|
+
} from '../cache/index.js'
|
|
15
20
|
import { PLUGIN_PACKAGE, type ProjectConfig, type WorkspaceConfig } from '../config.js'
|
|
16
21
|
import type { TaskExecutor } from '../exec/index.js'
|
|
17
|
-
import type { TaskNode, TaskOutcome } from '../graph/index.js'
|
|
22
|
+
import type { TaskNode, TaskOutcome, TaskStatus } from '../graph/index.js'
|
|
18
23
|
import { maskedLine, nearest, UserError } from '../util/index.js'
|
|
19
24
|
import type { ProjectMeta } from '../workspace/index.js'
|
|
20
|
-
import type { EventBus, RunStartInfo } from '
|
|
25
|
+
import type { EventBus, Logger, RunStartInfo } from '../report/index.js'
|
|
26
|
+
import type { CiContext } from './run-context.js'
|
|
21
27
|
import type { TelemetryContext, TelemetrySink } from './telemetry.js'
|
|
22
|
-
import type { Logger } from './logger.js'
|
|
23
28
|
|
|
24
29
|
/**
|
|
25
30
|
* A vx plugin. Contributes any subset of the run-level capabilities —
|
|
@@ -221,15 +226,51 @@ export interface VxPlugin {
|
|
|
221
226
|
* Optional one-time setup per run (validate the workspace, open a
|
|
222
227
|
* connection, read a token). Throwing aborts the run with a clean
|
|
223
228
|
* UserError naming the plugin. It is not a guard over the pipeline: the
|
|
224
|
-
* config, project, cache, graph
|
|
225
|
-
*
|
|
226
|
-
* precedes the executors, the telemetry sinks,
|
|
227
|
-
* task (tests/plugin-e2e.test.ts).
|
|
229
|
+
* config, project, cache, graph and key stages have run by the time it
|
|
230
|
+
* is called, and a plan (`--dry`, `planRun`) never calls it. It
|
|
231
|
+
* precedes the schedule stage, the executors, the telemetry sinks,
|
|
232
|
+
* admission and the first task (tests/plugin-e2e.test.ts).
|
|
228
233
|
*/
|
|
229
234
|
setup?(ctx: PluginSetupContext): void | Promise<void>
|
|
230
235
|
|
|
231
|
-
/**
|
|
232
|
-
|
|
236
|
+
/**
|
|
237
|
+
* Optional teardown at end-of-run (flush a sink, close a socket). Errors
|
|
238
|
+
* are logged, never thrown. A run that ran to its end passes its
|
|
239
|
+
* `RunReport`; every other exit (a refused setup, a stop) passes none.
|
|
240
|
+
*/
|
|
241
|
+
teardown?(run?: RunReport): void | Promise<void>
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/**
|
|
245
|
+
* What a finished run hands each plugin's `teardown`: its facts and every
|
|
246
|
+
* task, with the names core kept the task's bytes under. Core keeps them
|
|
247
|
+
* only when a remote takes the run's writes (CI by default): a failed
|
|
248
|
+
* task's log and the files its outputs left, an uncached task's log. Read
|
|
249
|
+
* them with `ctx.cache.getRun(name)`. Core keeps no manifest of its own; a
|
|
250
|
+
* plugin that wants one writes it from here (`ctx.cache.put`).
|
|
251
|
+
*/
|
|
252
|
+
export interface RunReport {
|
|
253
|
+
/** The `run_id` the run's history rows and telemetry carry. */
|
|
254
|
+
readonly runId: string
|
|
255
|
+
/** `HEAD`'s commit, null outside a git worktree. */
|
|
256
|
+
readonly commit: string | null
|
|
257
|
+
/** What the CI provider says about this run: provider, run URL, pipeline, job, attempt. */
|
|
258
|
+
readonly ciRun: CiContext
|
|
259
|
+
readonly ok: boolean
|
|
260
|
+
/** Every task the run decided, in completion order; group tasks are left out. */
|
|
261
|
+
readonly tasks: readonly RunReportTask[]
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
export interface RunReportTask {
|
|
265
|
+
readonly id: string
|
|
266
|
+
readonly status: TaskStatus
|
|
267
|
+
readonly durationMs: number
|
|
268
|
+
/** The task's cache key, when one was computed. */
|
|
269
|
+
readonly key?: string
|
|
270
|
+
/** Its output (both streams, secrets masked), when core kept it: `ctx.cache.getRun(log)`. */
|
|
271
|
+
readonly log?: string
|
|
272
|
+
/** A tar of the files its declared outputs left, when core kept it: `ctx.cache.getRun(outputs)`. */
|
|
273
|
+
readonly outputs?: string
|
|
233
274
|
}
|
|
234
275
|
|
|
235
276
|
/** Shared, read-only context every capability factory receives. */
|
|
@@ -242,6 +283,12 @@ interface BaseContext {
|
|
|
242
283
|
|
|
243
284
|
/** What `setup` receives: the run's lifecycle, observe-only. */
|
|
244
285
|
export interface PluginSetupContext extends BaseContext {
|
|
286
|
+
/** This plugin's entries in the workspace's cache (`PluginCache`). */
|
|
287
|
+
readonly cache: PluginCache
|
|
288
|
+
/** The run is on CI (`CI` set, or a provider core detects). */
|
|
289
|
+
readonly ci: boolean
|
|
290
|
+
/** The run's branch (`currentBranch`): `run.branch`, else git's, else the CI provider's. */
|
|
291
|
+
readonly branch: string | null
|
|
245
292
|
/**
|
|
246
293
|
* Subscribe a lifecycle handler; it leaves with the run. A throw or a
|
|
247
294
|
* rejection warns once and switches the plugin's handlers off for the run.
|
|
@@ -271,6 +318,12 @@ export interface CommandContext extends BaseContext {
|
|
|
271
318
|
* which `vx` a PATH holds.
|
|
272
319
|
*/
|
|
273
320
|
readonly vx: readonly string[]
|
|
321
|
+
/**
|
|
322
|
+
* This plugin's entries in the workspace's cache, as `setup` reaches them
|
|
323
|
+
* (`PluginCache`), read-only: the local store, then each remote the
|
|
324
|
+
* workspace's cache plugins supply, under the run's scope. Opened on first use.
|
|
325
|
+
*/
|
|
326
|
+
readonly cache: PluginCache
|
|
274
327
|
}
|
|
275
328
|
|
|
276
329
|
/** `config` runs before the cache dir is known — it may be what the hook changes. */
|
|
@@ -327,6 +380,13 @@ export interface ProjectHookContext extends BaseContext {
|
|
|
327
380
|
* process.
|
|
328
381
|
*/
|
|
329
382
|
readonly projects: readonly ProjectMeta[]
|
|
383
|
+
/**
|
|
384
|
+
* True when `vx lock` (or `--check`) visits: the one pass that checks the
|
|
385
|
+
* configs, so the place to report what a plugin's mapping could not say.
|
|
386
|
+
* A run checks nothing (the lock model), so such a report there repeats
|
|
387
|
+
* on every run.
|
|
388
|
+
*/
|
|
389
|
+
readonly lock: boolean
|
|
330
390
|
}
|
|
331
391
|
|
|
332
392
|
export interface GraphHookContext extends BaseContext {
|
|
@@ -386,6 +446,8 @@ export interface ScheduleHookContext extends BaseContext {
|
|
|
386
446
|
readonly localCache: Cache
|
|
387
447
|
/** The run's branch (`currentBranch`), the timings a forecast reads first: pass it to `LocalHistoryProvider`. */
|
|
388
448
|
readonly branch: string | null
|
|
449
|
+
/** This plugin's entries in the workspace's cache, as `setup` has them. */
|
|
450
|
+
readonly cache: PluginCache
|
|
389
451
|
}
|
|
390
452
|
|
|
391
453
|
/** A `cache` hook's context: the workspace and the run's scope, never the local store or the policy, which core owns. */
|
|
@@ -399,12 +461,10 @@ export interface CacheContext extends BaseContext {
|
|
|
399
461
|
}
|
|
400
462
|
|
|
401
463
|
export interface OutputContext extends BaseContext {
|
|
402
|
-
/** Core would write to a terminal: stdout
|
|
464
|
+
/** Core would write to a terminal: stdout is a TTY. */
|
|
403
465
|
readonly isTTY: boolean
|
|
404
466
|
/** The run is on CI (`CI` set, or a provider core detects). */
|
|
405
467
|
readonly ci: boolean
|
|
406
|
-
/** `--format json`: stdout carries the machine-readable result. */
|
|
407
|
-
readonly json: boolean
|
|
408
468
|
}
|
|
409
469
|
|
|
410
470
|
export interface ExecutorContext extends BaseContext {
|
|
@@ -619,6 +679,14 @@ export interface InstallPluginsArgs {
|
|
|
619
679
|
* `run:status` channel.
|
|
620
680
|
*/
|
|
621
681
|
warn?: (message: string) => void
|
|
682
|
+
/** Each plugin's `ctx.cache`, by plugin name. Default: a cache with no layer. */
|
|
683
|
+
cache?: (plugin: string) => PluginCache
|
|
684
|
+
/** Default false. */
|
|
685
|
+
ci?: boolean
|
|
686
|
+
/** Default null. */
|
|
687
|
+
branch?: string | null
|
|
688
|
+
/** Each setup as the step `<plugin>:setup`; default: untimed. */
|
|
689
|
+
step?: <T>(id: string, work: () => Promise<T>) => Promise<T>
|
|
622
690
|
}
|
|
623
691
|
|
|
624
692
|
/**
|
|
@@ -675,6 +743,9 @@ export async function installPlugins(args: InstallPluginsArgs): Promise<() => vo
|
|
|
675
743
|
workspaceRoot,
|
|
676
744
|
cacheDir,
|
|
677
745
|
warn,
|
|
746
|
+
cache: args.cache?.(plugin.name) ?? absentPluginCache,
|
|
747
|
+
ci: args.ci ?? false,
|
|
748
|
+
branch: args.branch ?? null,
|
|
678
749
|
// A subscription made past the hooks leaves with the run like a
|
|
679
750
|
// hook's does: the bus can outlive the run (`RunOptions.bus`), and
|
|
680
751
|
// one left behind hears every later run on it (item 635).
|
|
@@ -752,7 +823,8 @@ export async function installPlugins(args: InstallPluginsArgs): Promise<() => vo
|
|
|
752
823
|
}
|
|
753
824
|
|
|
754
825
|
try {
|
|
755
|
-
await plugin.setup(ctx)
|
|
826
|
+
if (args.step === undefined) await plugin.setup(ctx)
|
|
827
|
+
else await args.step(`${plugin.name}:setup`, async () => plugin.setup?.(ctx))
|
|
756
828
|
} catch (err) {
|
|
757
829
|
// Earlier plugins' subscriptions too: the caller never gets the
|
|
758
830
|
// disposer, and a bus that outlives the run would keep them.
|