@vzn/vx 0.0.636 → 0.0.638
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -11
- package/docs/caching.md +246 -213
- package/docs/cli.md +791 -705
- package/docs/execution.md +68 -76
- package/docs/features.md +47 -42
- package/docs/patterns.md +26 -25
- package/docs/schema.md +392 -205
- package/docs/security.md +76 -39
- package/launcher.sh +43 -0
- package/package.json +5 -5
- package/schemas/cache.json +41 -19
- package/schemas/info.json +23 -11
- package/schemas/last.json +48 -2
- package/schemas/lock.json +1 -1
- package/schemas/plan.json +4 -4
- package/schemas/show.json +5 -3
- package/schemas/why.json +15 -1
- package/skills/vx/SKILL.md +28 -13
- package/src/bin.ts +30 -9
- package/src/cache/archive.ts +11 -0
- package/src/cache/cache.ts +124 -53
- package/src/cache/chained-cache.ts +4 -0
- package/src/cache/git-inputs.ts +277 -63
- package/src/cache/index.ts +15 -1
- package/src/cache/inputs.ts +102 -36
- package/src/cache/key-fold.ts +8 -7
- package/src/cache/layer.ts +15 -18
- package/src/cache/layered-cache.ts +20 -102
- package/src/cache/plugin-cache.ts +248 -0
- package/src/cache/policy.ts +32 -27
- package/src/cache/run-history.ts +104 -14
- package/src/cache/run-store.ts +77 -0
- package/src/cache/schema.ts +58 -31
- package/src/cache/scoped-cache.ts +237 -0
- package/src/cli/cache.ts +173 -34
- package/src/cli/completions.ts +11 -7
- package/src/cli/core-alias.ts +1 -1
- package/src/cli/foreign-flags.ts +13 -12
- package/src/cli/help.ts +110 -70
- package/src/cli/index.ts +67 -24
- package/src/cli/info.ts +43 -27
- package/src/cli/init.ts +115 -407
- package/src/cli/last.ts +157 -9
- package/src/cli/lock.ts +356 -122
- package/src/cli/plan-format.ts +7 -5
- package/src/cli/plugin-commands.ts +81 -14
- package/src/cli/plugin-templates.ts +11 -11
- package/src/cli/run.ts +116 -160
- package/src/cli/select.ts +50 -47
- package/src/cli/show.ts +83 -59
- package/src/cli/task-verb.ts +6 -13
- package/src/cli/upgrade.ts +415 -149
- package/src/cli/watch-cycle.ts +9 -2
- package/src/cli/watch-filter.ts +39 -61
- package/src/cli/watch-fs.ts +29 -16
- package/src/cli/watch-judge.ts +18 -4
- package/src/cli/watch-set.ts +16 -42
- package/src/cli/watch.ts +227 -230
- package/src/cli/why.ts +38 -18
- package/src/cli/workspace-config.ts +33 -70
- package/src/config.ts +169 -70
- package/src/exec/env.ts +1 -31
- package/src/exec/executor.ts +1 -5
- package/src/exec/kill-tree.ts +2 -1
- package/src/exec/sandbox-runtime.ts +12 -1
- package/src/exec/sandbox-violations.ts +9 -1
- package/src/graph/dependency-spec.ts +1 -1
- package/src/graph/index.ts +1 -0
- package/src/graph/priorities.ts +1 -1
- package/src/graph/scheduler.ts +27 -7
- package/src/graph/task-graph.ts +76 -24
- package/src/handoff.ts +169 -0
- package/src/index.ts +35 -58
- package/src/orchestrator/affected-tasks.ts +15 -8
- package/src/orchestrator/command-windows.ts +44 -0
- package/src/orchestrator/deferred-outputs.ts +12 -10
- package/src/orchestrator/doctor.ts +98 -110
- package/src/orchestrator/download-policy.ts +3 -3
- package/src/orchestrator/events.ts +19 -3
- package/src/orchestrator/excluded-keys.ts +0 -2
- package/src/orchestrator/execute-task.ts +147 -69
- package/src/orchestrator/failure-mode.ts +19 -10
- package/src/orchestrator/fingerprint-watch.ts +18 -15
- package/src/orchestrator/forecast.ts +10 -2
- package/src/orchestrator/framed-output.ts +9 -5
- package/src/orchestrator/graph-stage.ts +5 -5
- package/src/orchestrator/history.ts +126 -30
- package/src/orchestrator/hit-restore.ts +15 -3
- package/src/orchestrator/index.ts +20 -5
- package/src/orchestrator/local-shortcircuit.ts +2 -1
- package/src/orchestrator/lockfile-claim.ts +79 -20
- package/src/orchestrator/logger.ts +80 -41
- package/src/orchestrator/metrics.ts +78 -9
- package/src/orchestrator/miss-save.ts +3 -0
- package/src/orchestrator/options.ts +42 -29
- package/src/orchestrator/output-host.ts +72 -0
- package/src/orchestrator/plan.ts +6 -8
- package/src/orchestrator/plugin-host.ts +100 -66
- package/src/orchestrator/plugin.ts +173 -67
- package/src/orchestrator/prepare.ts +163 -78
- package/src/orchestrator/projects.ts +105 -99
- package/src/orchestrator/remote-prefetch.ts +11 -6
- package/src/orchestrator/run-context.ts +25 -6
- package/src/orchestrator/run-failures.ts +17 -8
- package/src/orchestrator/run-lock.ts +1 -1
- package/src/orchestrator/run-records.ts +1 -1
- package/src/orchestrator/run-report.ts +9 -9
- package/src/orchestrator/run-settings.ts +48 -0
- package/src/orchestrator/run.ts +244 -119
- package/src/orchestrator/sandbox-fix.ts +39 -20
- package/src/orchestrator/sandbox-request.ts +84 -13
- package/src/orchestrator/signals.ts +1 -1
- package/src/orchestrator/stable-keys.ts +13 -11
- package/src/orchestrator/status-line.ts +49 -7
- package/src/orchestrator/summary.ts +115 -106
- package/src/orchestrator/task-hash.ts +12 -83
- package/src/orchestrator/telemetry.ts +4 -4
- package/src/settings.ts +428 -0
- package/src/{orchestrator → util}/colors.ts +5 -0
- package/src/util/config-frame.ts +31 -2
- package/src/util/errors.ts +33 -1
- package/src/util/essential-env.ts +31 -0
- package/src/util/hide-env.ts +107 -0
- package/src/util/index.ts +16 -3
- package/src/util/paths.ts +10 -0
- package/src/util/refusal-notice.ts +111 -0
- package/src/util/report.ts +31 -0
- package/src/util/settle.ts +49 -30
- package/src/util/size.ts +23 -4
- package/src/util/timing.ts +3 -2
- package/src/util/ui.ts +186 -0
- package/src/util/ulid.ts +1 -1
- package/src/util/verbs.ts +0 -15
- package/src/workspace/affected.ts +84 -232
- package/src/workspace/config-eval.ts +15 -20
- package/src/workspace/config-imports.ts +78 -3
- package/src/workspace/config-schema.ts +209 -268
- package/src/workspace/dependency-rules.ts +65 -0
- package/src/workspace/filter.ts +2 -2
- package/src/workspace/fingerprint.ts +11 -121
- package/src/workspace/index.ts +20 -27
- package/src/workspace/load-reads.ts +4 -4
- package/src/workspace/lockfile.ts +96 -140
- package/src/workspace/migration.ts +307 -215
- package/src/workspace/package-graph.ts +30 -216
- package/src/workspace/project-loader.ts +80 -285
- package/src/workspace/repo-id.ts +2 -1
- package/src/workspace/workspace.ts +363 -831
- package/src/cache/config-evals.ts +0 -130
- package/src/workspace/config-cache.ts +0 -661
- package/src/workspace/migrate-scripts.ts +0 -1121
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// so internals like prepare.ts can import them without an upward
|
|
3
3
|
// import of index.ts — the module entry must stay cycle-free.
|
|
4
4
|
|
|
5
|
-
import type { CachePolicy, RemoteCacheLayer } from '../cache/index.js'
|
|
5
|
+
import type { CacheFlags, CachePolicy, RemoteCacheLayer } from '../cache/index.js'
|
|
6
6
|
import type { ContinueMode } from '../graph/index.js'
|
|
7
7
|
import type { TaskOutcome } from '../graph/index.js'
|
|
8
8
|
import type { AffectedReason } from './affected-tasks.js'
|
|
@@ -11,6 +11,7 @@ import type { TelemetrySink } from './telemetry.js'
|
|
|
11
11
|
import type { Logger } from './logger.js'
|
|
12
12
|
import type { RunSummaryJson } from './run-artifacts.js'
|
|
13
13
|
import type { AffectedChanges, ProjectEntry, ProjectMeta } from '../workspace/index.js'
|
|
14
|
+
import type { RunSettings } from '../config.js'
|
|
14
15
|
|
|
15
16
|
export interface RunOptions {
|
|
16
17
|
cwd: string
|
|
@@ -47,7 +48,7 @@ export interface RunOptions {
|
|
|
47
48
|
affectedReasons?: Map<string, AffectedReason>
|
|
48
49
|
/**
|
|
49
50
|
* Configs a selection pass in this same process already loaded and
|
|
50
|
-
* staged with the same `cacheDir`
|
|
51
|
+
* staged with the same `cacheDir` (the CLI walks every
|
|
51
52
|
* `pkg#task` edge for `--filter 'app...'` and `--affected`). The run
|
|
52
53
|
* seeds and scopes as it always does and reuses these entries instead
|
|
53
54
|
* of evaluating and staging them again. One run only — never carry a
|
|
@@ -67,42 +68,39 @@ export interface RunOptions {
|
|
|
67
68
|
concurrency?: number
|
|
68
69
|
/**
|
|
69
70
|
* Cache directory override (`--cache-dir <path>`). Absolute, or relative
|
|
70
|
-
* to `cwd`. Takes precedence over the workspace `
|
|
71
|
+
* to `cwd`. Takes precedence over the workspace `cache.dir` field and the
|
|
71
72
|
* `.vx/cache` default. A per-run knob — never folded into a cache key.
|
|
72
73
|
*/
|
|
73
74
|
cacheDir?: string
|
|
74
75
|
/**
|
|
75
76
|
* Granular cache read/write control across the local + remote layers.
|
|
76
77
|
* Each of the four axes (localRead / localWrite / remoteRead /
|
|
77
|
-
* remoteWrite) is independent. Undefined →
|
|
78
|
-
*
|
|
79
|
-
*
|
|
78
|
+
* remoteWrite) is independent. Undefined → the workspace's
|
|
79
|
+
* `cache.policy` with `cacheFlags` applied, else {@link FULL_CACHE_POLICY}
|
|
80
|
+
* (everything on).
|
|
80
81
|
*/
|
|
81
82
|
cache?: CachePolicy
|
|
82
83
|
/**
|
|
83
|
-
*
|
|
84
|
+
* With no `cache`: `--no-cache` (all four off) and `--force` (both reads
|
|
85
|
+
* off), applied over the workspace's `cache.policy`.
|
|
86
|
+
*/
|
|
87
|
+
cacheFlags?: CacheFlags
|
|
88
|
+
/**
|
|
89
|
+
* The caller asked for a remote axis by name (`--cache-policy=remote:rw`). With
|
|
84
90
|
* no cache plugin supplying a remote layer the axes are inert, and a CI
|
|
85
91
|
* job that believes it is filling a shared cache deserves one line
|
|
86
92
|
* saying it is not.
|
|
87
93
|
*/
|
|
88
94
|
remoteRequested?: boolean
|
|
89
|
-
/** This run's `
|
|
95
|
+
/** This run's `cache.scope` (the CLI's `--cache-scope`), above `VX_CACHE_SCOPE` and `.vx/workspace`. */
|
|
90
96
|
cacheScope?: string
|
|
91
97
|
/**
|
|
92
|
-
* The
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
|
|
96
|
-
defaultCacheScope?: string
|
|
97
|
-
/**
|
|
98
|
-
* CI mode: load configs FROM the committed vx-lock.json instead of
|
|
99
|
-
* evaluating them (frozen-env reproducibility). Requires the lock
|
|
100
|
-
* to exist and pass its content-hash tripwire. Local runs default
|
|
101
|
-
* to live evaluation — byte hashes can't see a config's import
|
|
102
|
-
* closure, so a strict local mode would give false confidence.
|
|
103
|
-
* Pair with `vx lock --check` in CI for the full evaluation audit.
|
|
98
|
+
* The default policy's remote write is off (`remote:r`): the CLI passes it
|
|
99
|
+
* off CI (`CI` unset, `0` or `false`) unless `--cache-policy` or
|
|
100
|
+
* `VX_CACHE_POLICY` names the remote. Dropped when the workspace's
|
|
101
|
+
* `cache.policy` names the remote.
|
|
104
102
|
*/
|
|
105
|
-
|
|
103
|
+
remoteReadOnlyByDefault?: boolean
|
|
106
104
|
/**
|
|
107
105
|
* Explicit per-task output volume for the default logger. ALWAYS
|
|
108
106
|
* overrides the flow default. 'full': frames for work + one-liners
|
|
@@ -135,19 +133,18 @@ export interface RunOptions {
|
|
|
135
133
|
*/
|
|
136
134
|
flow?: 'focused' | 'broad'
|
|
137
135
|
/**
|
|
138
|
-
*
|
|
136
|
+
* `run.retries` from its flag (`--retries <n>`): max additional attempts
|
|
139
137
|
* after a failed attempt, for tasks that don't declare their own
|
|
140
|
-
* `exec.retries` (explicit config wins, including `retries: 0`)
|
|
141
|
-
*
|
|
142
|
-
* the same run with and without `--
|
|
138
|
+
* `exec.retries` (explicit config wins, including `retries: 0`); unset,
|
|
139
|
+
* `VX_RUN_RETRIES`, then the workspace's. Never folded into any cache
|
|
140
|
+
* key, so the same run with and without `--retries` derives identical keys.
|
|
143
141
|
* A non-negative integer; run() refuses anything else (C-61).
|
|
144
142
|
*/
|
|
145
143
|
retries?: number
|
|
146
144
|
/**
|
|
147
|
-
*
|
|
148
|
-
* `exec.timeout
|
|
149
|
-
*
|
|
150
|
-
* `exec.timeout` always wins. Threaded as an option only — never
|
|
145
|
+
* `run.timeout` in ms from its flag (`--timeout 30m`), for tasks that
|
|
146
|
+
* declare no `exec.timeout`; unset, `VX_RUN_TIMEOUT`, then the
|
|
147
|
+
* workspace's. Per-task `exec.timeout` always wins. Never
|
|
151
148
|
* folded into any cache key (a timed-out task fails and is never
|
|
152
149
|
* cached), so a `--timeout` run cache-hits a plain run's entry.
|
|
153
150
|
* A positive integer of ms up to `2 ** 31 - 1`; run() refuses anything
|
|
@@ -163,6 +160,16 @@ export interface RunOptions {
|
|
|
163
160
|
* `--continue[=never|deps-ok|always]`; bare `--continue` = 'always'.
|
|
164
161
|
*/
|
|
165
162
|
continueMode?: ContinueMode
|
|
163
|
+
/** `--branch`; unset, `VX_RUN_BRANCH`, then git's, then the CI provider's. */
|
|
164
|
+
branch?: string
|
|
165
|
+
/** `run.schedule` from its flag; unset, `VX_RUN_SCHEDULE`, then the workspace's. */
|
|
166
|
+
schedule?: NonNullable<RunSettings['schedule']>
|
|
167
|
+
/** `run.killGrace` in ms from its flag; unset, `VX_RUN_KILL_GRACE`, then the workspace's. */
|
|
168
|
+
killGrace?: number
|
|
169
|
+
/** `run.teardownTimeout` in ms from its flag; unset, `VX_RUN_TEARDOWN_TIMEOUT`, then the workspace's. */
|
|
170
|
+
teardownTimeout?: number
|
|
171
|
+
/** `run.readyNotice` in ms from its flag; unset, `VX_RUN_READY_NOTICE`, then the workspace's. */
|
|
172
|
+
readyNotice?: number
|
|
166
173
|
/**
|
|
167
174
|
* Filter `dependsOn` expansion. `'all'` drops every edge (just the
|
|
168
175
|
* requested task runs). A string array drops only those task names
|
|
@@ -190,6 +197,12 @@ export interface RunOptions {
|
|
|
190
197
|
* here. Not called on a refused run.
|
|
191
198
|
*/
|
|
192
199
|
beforeFooter?: (outcomes: readonly TaskOutcome[], ok: boolean) => string
|
|
200
|
+
/**
|
|
201
|
+
* `false`: no footer, live or printed, and no blank line opening the
|
|
202
|
+
* task log; the run is its task rows (`vx watch`, which frames each
|
|
203
|
+
* cycle with its own line).
|
|
204
|
+
*/
|
|
205
|
+
footer?: boolean
|
|
193
206
|
/**
|
|
194
207
|
* If set, write a Chrome-trace JSON profile of the run's wallclock
|
|
195
208
|
* spans. Path is cwd-relative. Default `profile.json` is selected
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
// The `output` capability: a plugin may take the terminal for a run. Sibling
|
|
2
|
+
// to telemetry-host.ts. With no plugin declaring the hook nothing here runs,
|
|
3
|
+
// and core's renderer is the run's, byte for byte.
|
|
4
|
+
|
|
5
|
+
import { terminalSubscriber, type RunEventSubscriber } from './events.js'
|
|
6
|
+
import type { Logger } from './logger.js'
|
|
7
|
+
import type { OutputContext, VxPlugin } from './plugin.js'
|
|
8
|
+
|
|
9
|
+
const REQUIRED = ['status', 'taskStdout', 'taskStderr', 'taskComplete'] as const
|
|
10
|
+
const OPTIONAL = ['runStart', 'taskStart', 'runEnd'] as const
|
|
11
|
+
|
|
12
|
+
/** Reject anything that is not Logger-shaped, naming what is missing. */
|
|
13
|
+
function checkLogger(value: unknown): Logger {
|
|
14
|
+
if (typeof value !== 'object' || value === null) {
|
|
15
|
+
throw new Error(`output must be a Logger object, got ${value === null ? 'null' : typeof value}`)
|
|
16
|
+
}
|
|
17
|
+
const o = value as Record<string, unknown>
|
|
18
|
+
const missing = REQUIRED.filter((k) => typeof o[k] !== 'function')
|
|
19
|
+
if (missing.length > 0) throw new Error(`output Logger has no ${missing.join(', ')}`)
|
|
20
|
+
const wrong = OPTIONAL.filter((k) => o[k] !== undefined && typeof o[k] !== 'function')
|
|
21
|
+
if (wrong.length > 0) throw new Error(`output Logger's ${wrong.join(', ')} must be functions`)
|
|
22
|
+
return value as Logger
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* The first plugin, in declaration order, whose `output` returns a Logger.
|
|
27
|
+
* A hook that throws or returns something else is warned about and passed
|
|
28
|
+
* over: a renderer must never fail a run.
|
|
29
|
+
*/
|
|
30
|
+
export async function claimOutput(
|
|
31
|
+
plugins: readonly VxPlugin[],
|
|
32
|
+
ctx: OutputContext,
|
|
33
|
+
): Promise<{ logger: Logger; owner: string } | undefined> {
|
|
34
|
+
for (const plugin of plugins) {
|
|
35
|
+
if (plugin.output === undefined) continue
|
|
36
|
+
try {
|
|
37
|
+
const logger = await plugin.output(ctx)
|
|
38
|
+
if (logger !== undefined) return { logger: checkLogger(logger), owner: plugin.name }
|
|
39
|
+
} catch (err) {
|
|
40
|
+
ctx.warn(
|
|
41
|
+
`[vx] plugin '${plugin.name}' output: ${err instanceof Error ? err.message : String(err)}; vx renders the run`,
|
|
42
|
+
)
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return undefined
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* The bus subscriber for a plugin's Logger. The first throw hands that
|
|
50
|
+
* event and every later one to `fallback` (core's renderer), after one
|
|
51
|
+
* status line naming the plugin.
|
|
52
|
+
*/
|
|
53
|
+
export function guardedOutput(
|
|
54
|
+
claimed: { logger: Logger; owner: string },
|
|
55
|
+
fallback: RunEventSubscriber,
|
|
56
|
+
): RunEventSubscriber {
|
|
57
|
+
const render = terminalSubscriber(claimed.logger)
|
|
58
|
+
let failed = false
|
|
59
|
+
return (event) => {
|
|
60
|
+
if (failed) return fallback(event)
|
|
61
|
+
try {
|
|
62
|
+
render(event)
|
|
63
|
+
} catch (err) {
|
|
64
|
+
failed = true
|
|
65
|
+
fallback({
|
|
66
|
+
kind: 'run:status',
|
|
67
|
+
line: `[vx] plugin '${claimed.owner}' output failed: ${err instanceof Error ? err.message : String(err)}; vx renders the rest of the run`,
|
|
68
|
+
})
|
|
69
|
+
fallback(event)
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
package/src/orchestrator/plan.ts
CHANGED
|
@@ -50,7 +50,7 @@ export interface PlannedTask {
|
|
|
50
50
|
executor?: string
|
|
51
51
|
/**
|
|
52
52
|
* `'deferred'` when this task's outputs would stay in the remote store
|
|
53
|
-
* (`--download=none` / `toplevel`). Attached only when it is TRUE — an
|
|
53
|
+
* (`--cache-download=none` / `toplevel`). Attached only when it is TRUE — an
|
|
54
54
|
* eager task is the default and saying so on every line says nothing.
|
|
55
55
|
*/
|
|
56
56
|
download?: 'deferred'
|
|
@@ -91,22 +91,21 @@ export interface RunPlan {
|
|
|
91
91
|
/** Set when `--affected` left nothing to run: the line the run would print, and why. */
|
|
92
92
|
noneAffected?: string
|
|
93
93
|
/**
|
|
94
|
-
* Producers `--download` WANTED to defer but could not, mapped to why.
|
|
95
|
-
* Without this the gate is invisible: a user asks for `--download=none`,
|
|
94
|
+
* Producers `--cache-download` WANTED to defer but could not, mapped to why.
|
|
95
|
+
* Without this the gate is invisible: a user asks for `--cache-download=none`,
|
|
96
96
|
* gets no deferral, and has nothing to read. Empty/absent under
|
|
97
|
-
* `--download=all`, which defers nothing by definition.
|
|
97
|
+
* `--cache-download=all`, which defers nothing by definition.
|
|
98
98
|
*/
|
|
99
99
|
downloadDowngrades?: ReadonlyArray<{ taskId: string; reason: string }>
|
|
100
100
|
}
|
|
101
101
|
|
|
102
102
|
export interface PlanArgs {
|
|
103
103
|
nodes: Map<string, TaskNode>
|
|
104
|
-
/** Per-task `--download` decision, when the policy is not `all`. */
|
|
104
|
+
/** Per-task `--cache-download` decision, when the policy is not `all`. */
|
|
105
105
|
downloadOf?: (id: string) => 'eager' | 'deferred' | 'never' | undefined
|
|
106
106
|
/** See `RunPlan.downloadDowngrades`. */
|
|
107
107
|
downloadDowngrades?: ReadonlyArray<{ taskId: string; reason: string }>
|
|
108
108
|
workspaceRoot: string
|
|
109
|
-
workspaceFingerprint: string
|
|
110
109
|
cache: CacheLayer
|
|
111
110
|
cachePolicy?: CachePolicy
|
|
112
111
|
forwardArgs?: readonly string[] | undefined
|
|
@@ -150,7 +149,6 @@ export async function plan(args: PlanArgs): Promise<RunPlan> {
|
|
|
150
149
|
node,
|
|
151
150
|
upstream,
|
|
152
151
|
workspaceRoot: args.workspaceRoot,
|
|
153
|
-
workspaceFingerprint: args.workspaceFingerprint,
|
|
154
152
|
cache: args.cache,
|
|
155
153
|
forwardArgs: args.forwardArgs,
|
|
156
154
|
nestedProjectDirs: args.nestedDirsByProject.get(node.projectName) ?? [],
|
|
@@ -166,7 +164,7 @@ export async function plan(args: PlanArgs): Promise<RunPlan> {
|
|
|
166
164
|
|
|
167
165
|
// Prediction keys off READS: a no-read policy re-executes every task,
|
|
168
166
|
// so nothing is probed. Whether that run is a miss or no cache at all
|
|
169
|
-
// is the WRITE axes': `--force` and `--cache=local:w` save what they
|
|
167
|
+
// is the WRITE axes': `--force` and `--cache-policy=local:w` save what they
|
|
170
168
|
// run, and the plan called them `no-cache`, the label a task with no
|
|
171
169
|
// `cache` block gets (item 991). The probe below routes through the
|
|
172
170
|
// policy-aware cache layer, which itself respects read gating.
|
|
@@ -8,11 +8,20 @@
|
|
|
8
8
|
// See docs/design/pipeline-2026-09.md.
|
|
9
9
|
|
|
10
10
|
import path from 'node:path'
|
|
11
|
-
import {
|
|
11
|
+
import {
|
|
12
|
+
type Cache,
|
|
13
|
+
ChainedCache,
|
|
14
|
+
type CacheLayer,
|
|
15
|
+
type CachePolicy,
|
|
16
|
+
LayeredCache,
|
|
17
|
+
absentPluginCache,
|
|
18
|
+
type PluginCache,
|
|
19
|
+
type PluginCacheRemote,
|
|
20
|
+
} from '../cache/index.js'
|
|
12
21
|
import { localExecutor, type TaskExecutor } from '../exec/index.js'
|
|
13
22
|
import { maskedLine, settleWithin, teardownTimeoutMs, UserError } from '../util/index.js'
|
|
14
|
-
import type { ProjectConfig, WorkspaceConfig
|
|
15
|
-
import { checkGraph, type TaskNode } from '../graph/index.js'
|
|
23
|
+
import type { ProjectConfig, WorkspaceConfig } from '../config.js'
|
|
24
|
+
import { checkGraph, type GraphRules, type TaskNode } from '../graph/index.js'
|
|
16
25
|
import type {
|
|
17
26
|
CacheContext,
|
|
18
27
|
DiscoverContext,
|
|
@@ -21,8 +30,9 @@ import type {
|
|
|
21
30
|
FingerprintContext,
|
|
22
31
|
GraphHookContext,
|
|
23
32
|
KeyHookContext,
|
|
24
|
-
|
|
33
|
+
DiscoveredProject,
|
|
25
34
|
ProjectHookContext,
|
|
35
|
+
RunReport,
|
|
26
36
|
ScheduleHookContext,
|
|
27
37
|
VxPlugin,
|
|
28
38
|
WorkspaceHookContext,
|
|
@@ -46,24 +56,8 @@ function isRecord(v: unknown): v is Record<string, unknown> {
|
|
|
46
56
|
return proto === Object.prototype || proto === null
|
|
47
57
|
}
|
|
48
58
|
|
|
49
|
-
/** Every method `
|
|
50
|
-
export const
|
|
51
|
-
'key',
|
|
52
|
-
'get',
|
|
53
|
-
'has',
|
|
54
|
-
'prefetch',
|
|
55
|
-
'loadOutputFilesBatch',
|
|
56
|
-
'isOutputsCurrent',
|
|
57
|
-
'restoreOutputs',
|
|
58
|
-
'save',
|
|
59
|
-
'ingest',
|
|
60
|
-
'recordRunBundle',
|
|
61
|
-
'stats',
|
|
62
|
-
'hashFile',
|
|
63
|
-
'outputsPath',
|
|
64
|
-
'prune',
|
|
65
|
-
'close',
|
|
66
|
-
]
|
|
59
|
+
/** Every method a `cache` hook's `RemoteCacheLayer` requires (`hasMany` is optional). */
|
|
60
|
+
export const REMOTE_CACHE_METHODS: readonly string[] = ['has', 'get', 'put']
|
|
67
61
|
|
|
68
62
|
/** What a capability hook handed back must be what the seam runs. */
|
|
69
63
|
function assertShape(
|
|
@@ -96,7 +90,7 @@ async function safe<T>(plugin: VxPlugin, hook: string, fn: () => T | Promise<T>)
|
|
|
96
90
|
return await fn()
|
|
97
91
|
} catch (err) {
|
|
98
92
|
// Masked here, not by whoever prints it: an embedder of `run()` and
|
|
99
|
-
//
|
|
93
|
+
// a plugin's verb print it as they get it, and the throw quotes what the
|
|
100
94
|
// plugin was told (a remote's reply, a header) (L-11).
|
|
101
95
|
throw new UserError(
|
|
102
96
|
maskedLine(
|
|
@@ -158,15 +152,21 @@ export async function applyConfigHooks(
|
|
|
158
152
|
|
|
159
153
|
/**
|
|
160
154
|
* `discover` stage: each plugin names directories to add to `projects`,
|
|
161
|
-
* seeing the ones earlier plugins added. `add` turns one
|
|
162
|
-
*
|
|
155
|
+
* seeing the ones earlier plugins added. `add` turns one entry into the
|
|
156
|
+
* project it names, `added` when it is new, or refuses it. Returns every
|
|
157
|
+
* entry's edges with the project and plugin they came from, for the caller
|
|
158
|
+
* to resolve once every plugin has run.
|
|
163
159
|
*/
|
|
164
160
|
export async function applyDiscoverHooks(
|
|
165
161
|
plugins: readonly VxPlugin[],
|
|
166
162
|
projects: ProjectMeta[],
|
|
167
163
|
ctx: Omit<DiscoverContext, 'projects'>,
|
|
168
|
-
add: (
|
|
169
|
-
|
|
164
|
+
add: (
|
|
165
|
+
entry: DiscoveredProject,
|
|
166
|
+
plugin: VxPlugin,
|
|
167
|
+
) => Promise<{ meta: ProjectMeta; added: boolean }>,
|
|
168
|
+
): Promise<DiscoveredEdges[]> {
|
|
169
|
+
const edges: DiscoveredEdges[] = []
|
|
170
170
|
for (const plugin of plugins) {
|
|
171
171
|
if (plugin.discover === undefined) continue
|
|
172
172
|
const named = await safe(plugin, 'discover', () => plugin.discover!({ ...ctx, projects }))
|
|
@@ -175,11 +175,23 @@ export async function applyDiscoverHooks(
|
|
|
175
175
|
`plugin '${plugin.name}' failed in discover: expected an array, got ${describeValue(named)}`,
|
|
176
176
|
)
|
|
177
177
|
}
|
|
178
|
-
for (const
|
|
179
|
-
const meta = await add(
|
|
180
|
-
if (
|
|
178
|
+
for (const entry of named as readonly DiscoveredProject[]) {
|
|
179
|
+
const { meta, added } = await add(entry, plugin)
|
|
180
|
+
if (added) projects.push(meta)
|
|
181
|
+
if (entry.deps !== undefined || entry.reach !== undefined) {
|
|
182
|
+
edges.push({ by: plugin.name, meta, deps: entry.deps, reach: entry.reach ?? [] })
|
|
183
|
+
}
|
|
181
184
|
}
|
|
182
185
|
}
|
|
186
|
+
return edges
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
export interface DiscoveredEdges {
|
|
190
|
+
readonly by: string
|
|
191
|
+
readonly meta: ProjectMeta
|
|
192
|
+
/** Absent when the entry carried none; `[]` still makes the workspace an edge source. */
|
|
193
|
+
readonly deps: readonly string[] | undefined
|
|
194
|
+
readonly reach: readonly string[]
|
|
183
195
|
}
|
|
184
196
|
|
|
185
197
|
/** `project` stage: every plugin edits one project's config in place. */
|
|
@@ -215,7 +227,7 @@ export async function applyGraphHooks(
|
|
|
215
227
|
ctx: GraphHookContext,
|
|
216
228
|
/** Runs after EACH plugin's edit, so a refused task config names the plugin that wrote it. */
|
|
217
229
|
afterEach?: (plugin: VxPlugin) => void,
|
|
218
|
-
rules?:
|
|
230
|
+
rules?: GraphRules,
|
|
219
231
|
): Promise<void> {
|
|
220
232
|
let last: VxPlugin | undefined
|
|
221
233
|
for (const plugin of plugins) {
|
|
@@ -377,12 +389,14 @@ export async function claimedAffected(
|
|
|
377
389
|
export async function applyScheduleHooks(
|
|
378
390
|
plugins: readonly VxPlugin[],
|
|
379
391
|
nodes: ReadonlyMap<string, TaskNode>,
|
|
380
|
-
ctx: ScheduleHookContext,
|
|
392
|
+
ctx: Omit<ScheduleHookContext, 'cache'>,
|
|
393
|
+
cacheFor: (plugin: string) => PluginCache = () => absentPluginCache,
|
|
381
394
|
): Promise<ReadonlyMap<string, number>> {
|
|
382
395
|
const merged = new Map<string, number>()
|
|
383
396
|
for (const plugin of plugins) {
|
|
384
397
|
if (plugin.schedule === undefined) continue
|
|
385
|
-
const
|
|
398
|
+
const own = { ...ctx, cache: cacheFor(plugin.name) }
|
|
399
|
+
const weights = await safe(plugin, 'schedule', () => plugin.schedule!(nodes, own))
|
|
386
400
|
if (weights === undefined) continue
|
|
387
401
|
// Iterating a string destructures its characters into `[id, w]` pairs
|
|
388
402
|
// that match no task — a plugin returning `'fast'` used to be a no-op
|
|
@@ -466,58 +480,77 @@ export function buildAdmission(
|
|
|
466
480
|
}
|
|
467
481
|
}
|
|
468
482
|
|
|
483
|
+
/** What `resolveCache` needs beyond a hook's context: the store and policy core owns. */
|
|
484
|
+
export interface CacheHost extends CacheContext {
|
|
485
|
+
readonly localCache: Cache
|
|
486
|
+
readonly policy: CachePolicy
|
|
487
|
+
/** Filled with each remote and what its policy lets it do, for the plugins' `ctx.cache`. */
|
|
488
|
+
readonly remotes?: PluginCacheRemote[]
|
|
489
|
+
}
|
|
490
|
+
|
|
469
491
|
/**
|
|
470
|
-
* Collect every plugin's `cache`
|
|
471
|
-
*
|
|
472
|
-
*
|
|
473
|
-
*
|
|
474
|
-
*
|
|
475
|
-
*
|
|
476
|
-
* one layer that wraps it.
|
|
492
|
+
* Collect every plugin's remote (`cache` hook) in declaration order and
|
|
493
|
+
* layer each over the local store: a lookup asks the local store first,
|
|
494
|
+
* then each remote in order (owner, 2026-10-13), and a save reaches every
|
|
495
|
+
* remote. The run's policy, narrowed by a remote's `access`, and its scope
|
|
496
|
+
* gate each remote; a remote's throw is a miss and a warning naming its
|
|
497
|
+
* plugin. No remote → the local store alone.
|
|
477
498
|
*/
|
|
478
499
|
export async function resolveCache(
|
|
479
500
|
plugins: readonly VxPlugin[],
|
|
480
|
-
|
|
501
|
+
host: CacheHost,
|
|
481
502
|
): Promise<CacheLayer> {
|
|
503
|
+
const { localCache, policy, remotes, ...ctx } = host
|
|
482
504
|
const layers: CacheLayer[] = []
|
|
483
|
-
const owners =
|
|
505
|
+
const owners: string[] = []
|
|
484
506
|
for (const plugin of plugins) {
|
|
485
507
|
if (plugin.cache === undefined) continue
|
|
486
|
-
const
|
|
487
|
-
if (
|
|
488
|
-
// The
|
|
489
|
-
//
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
508
|
+
const remote = await safe(plugin, 'cache', () => plugin.cache!(ctx))
|
|
509
|
+
if (remote === undefined) continue
|
|
510
|
+
// The hook returned a whole layer until 0.0.637: name the fix, not the
|
|
511
|
+
// three methods such a layer happens to have.
|
|
512
|
+
if (isRecordLike(remote) && typeof remote['restoreOutputs'] === 'function') {
|
|
513
|
+
throw new UserError(
|
|
514
|
+
`plugin '${plugin.name}' returned a cache layer from cache: since vx 0.0.637 the hook ` +
|
|
515
|
+
'returns the remote alone (a RemoteCacheLayer: has, get, put) and core layers it over ' +
|
|
516
|
+
'the local cache — return the remote you passed to LayeredCache',
|
|
517
|
+
)
|
|
518
|
+
}
|
|
519
|
+
assertShape(plugin, 'cache', remote, REMOTE_CACHE_METHODS, 'a remote cache layer')
|
|
520
|
+
const access = remote.access
|
|
521
|
+
const own: CachePolicy = {
|
|
522
|
+
...policy,
|
|
523
|
+
remoteRead: policy.remoteRead && access?.read !== false,
|
|
524
|
+
remoteWrite: policy.remoteWrite && access?.write !== false,
|
|
525
|
+
}
|
|
526
|
+
remotes?.push({ remote, read: own.remoteRead, write: own.remoteWrite })
|
|
527
|
+
layers.push(
|
|
528
|
+
new LayeredCache(localCache, remote, {
|
|
529
|
+
policy: own,
|
|
530
|
+
onRemoteError: (err) => ctx.warn(`[vx] plugin '${plugin.name}': ${err.message}`),
|
|
531
|
+
}),
|
|
532
|
+
)
|
|
533
|
+
owners.push(`the cache layer of plugin '${plugin.name}'`)
|
|
496
534
|
}
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
// the local handle subsumes it below, so a remote plugin does not write
|
|
500
|
-
// the local store twice.
|
|
501
|
-
if (!layers.includes(ctx.localCache)) layers.push(ctx.localCache)
|
|
502
|
-
const wrapsLocal = layers.some((l) => l !== ctx.localCache && l.local === ctx.localCache)
|
|
503
|
-
const distinct = wrapsLocal ? layers.filter((l) => l !== ctx.localCache) : layers
|
|
504
|
-
if (distinct.length === 1) return distinct[0]!
|
|
535
|
+
if (layers.length === 0) return localCache
|
|
536
|
+
if (layers.length === 1) return layers[0]!
|
|
505
537
|
// One line per layer and method: a layer that throws on every lookup
|
|
506
538
|
// would otherwise print once per task.
|
|
507
539
|
const told = new Set<string>()
|
|
508
|
-
return new ChainedCache(
|
|
509
|
-
const layer = distinct[i]!
|
|
510
|
-
const who =
|
|
511
|
-
layer === ctx.localCache ? 'the local cache' : (owners.get(layer) ?? 'a cache layer')
|
|
540
|
+
return new ChainedCache(layers, (i, method, err) => {
|
|
512
541
|
if (told.has(`${i}\0${method}`)) return
|
|
513
542
|
told.add(`${i}\0${method}`)
|
|
514
543
|
const m = err instanceof Error ? err.message : String(err)
|
|
515
544
|
const then =
|
|
516
545
|
method === 'save' ? 'the other layers still save' : 'a miss there; the next layer answers'
|
|
517
|
-
ctx.warn(`[vx] ${
|
|
546
|
+
ctx.warn(`[vx] ${owners[i]} failed in cache ${method}: ${m}; ${then}`)
|
|
518
547
|
})
|
|
519
548
|
}
|
|
520
549
|
|
|
550
|
+
function isRecordLike(v: unknown): v is Record<string, unknown> {
|
|
551
|
+
return typeof v === 'object' && v !== null
|
|
552
|
+
}
|
|
553
|
+
|
|
521
554
|
/** The plugin each resolved executor came from, so a hook of it that throws is named. */
|
|
522
555
|
const executorPlugin = new WeakMap<TaskExecutor, string>()
|
|
523
556
|
|
|
@@ -604,12 +637,13 @@ export async function resolveExecutors(
|
|
|
604
637
|
export async function teardownPlugins(
|
|
605
638
|
plugins: readonly VxPlugin[],
|
|
606
639
|
warn: (message: string) => void,
|
|
640
|
+
report?: RunReport,
|
|
607
641
|
): Promise<void> {
|
|
608
642
|
for (const plugin of plugins) {
|
|
609
643
|
if (plugin.teardown === undefined) continue
|
|
610
644
|
const ms = teardownTimeoutMs()
|
|
611
645
|
try {
|
|
612
|
-
const settled = await settleWithin(Promise.resolve(plugin.teardown()), ms)
|
|
646
|
+
const settled = await settleWithin(Promise.resolve(plugin.teardown(report)), ms)
|
|
613
647
|
// Same rule as the flush above. A teardown that never settles has left
|
|
614
648
|
// whatever it owns un-released, and the run exits anyway — silence would
|
|
615
649
|
// present that as a clean shutdown.
|