@vzn/vx 0.0.16 → 0.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -89
- package/index.ts +10 -0
- package/launcher.mjs +10 -11
- package/package.json +19 -5
- package/plugins/local-cache/index.ts +2 -0
- package/plugins/local-executor/index.ts +2 -0
- package/plugins/schedule-history/index.ts +2 -0
- package/src/bin.ts +2 -2
- package/src/cache/archive.ts +557 -0
- package/src/cache/cache.ts +929 -250
- package/src/cache/cas-backend.ts +20 -2
- package/src/cache/chained-cache.ts +202 -0
- package/src/cache/digest.ts +0 -15
- package/src/cache/index.ts +13 -3
- package/src/cache/inputs.ts +445 -41
- package/src/cache/layered-cache.ts +88 -30
- package/src/cache/tar-stream.ts +358 -0
- package/src/cli/cache.ts +36 -10
- package/src/cli/help.ts +167 -128
- package/src/cli/index.ts +74 -43
- package/src/cli/info.ts +30 -0
- package/src/cli/last.ts +140 -0
- package/src/cli/migrate-emit.ts +21 -0
- package/src/cli/migrate-nx.ts +23 -0
- package/src/cli/migrate-scripts.ts +127 -0
- package/src/cli/migrate-turbo.ts +43 -6
- package/src/cli/migrate.ts +119 -33
- package/src/cli/plan-format.ts +58 -3
- package/src/cli/plugin-commands.ts +96 -0
- package/src/cli/prune.ts +291 -0
- package/src/cli/run.ts +272 -80
- package/src/cli/upgrade.ts +7 -6
- package/src/cli/watch.ts +259 -61
- package/src/cli/why.ts +193 -0
- package/src/config.ts +40 -23
- package/src/exec/env.ts +7 -1
- package/src/exec/executor.ts +239 -0
- package/src/exec/index.ts +19 -1
- package/src/exec/runner.ts +116 -29
- package/src/exec/sandbox-runtime.ts +194 -24
- package/src/graph/index.ts +3 -1
- package/src/graph/scheduler.ts +92 -18
- package/src/graph/task-graph.ts +220 -6
- package/src/index.ts +82 -84
- package/src/orchestrator/deferred-outputs.ts +144 -0
- package/src/orchestrator/download-policy.ts +171 -0
- package/src/orchestrator/events.ts +30 -0
- package/src/orchestrator/execute-task.ts +328 -94
- package/src/orchestrator/failure-mode.ts +61 -0
- package/src/orchestrator/framed-output.ts +87 -35
- package/src/orchestrator/history.ts +78 -31
- package/src/orchestrator/index.ts +43 -90
- package/src/orchestrator/local-shortcircuit.ts +22 -0
- package/src/orchestrator/logger.ts +196 -14
- package/src/orchestrator/metrics.ts +57 -1376
- package/src/orchestrator/missing-plugin.ts +9 -0
- package/src/orchestrator/options.ts +22 -4
- package/src/orchestrator/plan.ts +144 -1
- package/src/orchestrator/plugin-host.ts +199 -119
- package/src/orchestrator/plugin.ts +144 -57
- package/src/orchestrator/prepare.ts +220 -69
- package/src/orchestrator/remote-prefetch.ts +12 -8
- package/src/orchestrator/resources.ts +20 -34
- package/src/orchestrator/run-artifacts.ts +41 -16
- package/src/orchestrator/run-context.ts +167 -29
- package/src/orchestrator/run-report.ts +73 -46
- package/src/orchestrator/run.ts +590 -122
- package/src/orchestrator/stable-keys.ts +74 -33
- package/src/orchestrator/status-line.ts +47 -8
- package/src/orchestrator/summary.ts +22 -1
- package/src/orchestrator/tally.ts +78 -35
- package/src/orchestrator/task-hash.ts +105 -11
- package/src/orchestrator/task-log-buffer.ts +291 -0
- package/src/orchestrator/telemetry-host.ts +24 -6
- package/src/orchestrator/telemetry.ts +181 -21
- package/src/orchestrator/upstream.ts +72 -31
- package/src/orchestrator/verify.ts +12 -2
- package/src/plugins/local-cache/index.ts +11 -0
- package/src/plugins/local-executor/index.ts +45 -0
- package/src/plugins/schedule-history/index.ts +109 -0
- package/src/util/edit-distance.ts +21 -0
- package/src/util/errors.ts +15 -1
- package/src/util/index.ts +7 -2
- package/src/util/num.ts +37 -0
- package/src/util/paths.ts +43 -0
- package/src/util/settle.ts +48 -0
- package/src/util/size.ts +5 -1
- package/src/util/tail.ts +66 -0
- package/src/util/timing.ts +63 -0
- package/src/workspace/affected.ts +142 -15
- package/src/workspace/config-cache.ts +304 -0
- package/src/workspace/config-eval.ts +210 -0
- package/src/workspace/config-imports.ts +161 -0
- package/src/workspace/filter.ts +22 -4
- package/src/workspace/fingerprint.ts +16 -1
- package/src/workspace/index.ts +15 -3
- package/src/workspace/lockfile.ts +13 -2
- package/src/workspace/nested-dirs.ts +21 -7
- package/src/workspace/package-graph.ts +23 -20
- package/src/workspace/project-loader.ts +503 -59
- package/src/workspace/workspace.ts +213 -72
- package/src/cache/tar.ts +0 -343
- package/src/cli/backend.ts +0 -23
- package/src/cli/mcp-rpc.ts +0 -292
- package/src/cli/mcp.ts +0 -92
- package/src/orchestrator/devframe-surface.ts +0 -74
- package/src/orchestrator/predict.ts +0 -106
- package/src/orchestrator/protocol.ts +0 -156
- package/src/orchestrator/run-state.ts +0 -145
- package/src/orchestrator/wire-render.ts +0 -92
- package/src/orchestrator/wire.ts +0 -212
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
|
|
10
10
|
import path from 'node:path'
|
|
11
11
|
import type { ProjectConfig, WorkspaceConfig } from '../config.js'
|
|
12
|
-
import { UserError } from '../util/index.js'
|
|
12
|
+
import { mark, UserError } from '../util/index.js'
|
|
13
13
|
import {
|
|
14
14
|
Cache,
|
|
15
15
|
type CacheLayer,
|
|
@@ -17,7 +17,10 @@ import {
|
|
|
17
17
|
FULL_CACHE_POLICY,
|
|
18
18
|
GitFilesCache,
|
|
19
19
|
LayeredCache,
|
|
20
|
-
|
|
20
|
+
applyGitEnumeration,
|
|
21
|
+
gitPathspecs,
|
|
22
|
+
startGitEnumeration,
|
|
23
|
+
type GitEnumeration,
|
|
21
24
|
} from '../cache/index.js'
|
|
22
25
|
import {
|
|
23
26
|
buildPackageGraph,
|
|
@@ -26,25 +29,40 @@ import {
|
|
|
26
29
|
findWorkspaceRoot,
|
|
27
30
|
frozenProjectConfig,
|
|
28
31
|
listProjects,
|
|
29
|
-
|
|
32
|
+
loadProjectConfigs,
|
|
30
33
|
loadWorkspace,
|
|
31
34
|
loadWorkspaceConfig,
|
|
32
35
|
readLockfile,
|
|
33
36
|
resolveCacheDir,
|
|
37
|
+
validateProjectConfig,
|
|
34
38
|
type ProjectEntry,
|
|
35
39
|
} from '../workspace/index.js'
|
|
36
|
-
import {
|
|
37
|
-
|
|
40
|
+
import {
|
|
41
|
+
buildTaskGraph,
|
|
42
|
+
expandRequested,
|
|
43
|
+
parseDependencySpec,
|
|
44
|
+
type TaskNode,
|
|
45
|
+
unresolvedRequests,
|
|
46
|
+
} from '../graph/index.js'
|
|
47
|
+
import {
|
|
48
|
+
applyConfigHooks,
|
|
49
|
+
applyGraphHooks,
|
|
50
|
+
applyKeyHooks,
|
|
51
|
+
applyProjectHooks,
|
|
52
|
+
applyScheduleHooks,
|
|
53
|
+
hasHook,
|
|
54
|
+
resolveCache,
|
|
55
|
+
} from './plugin-host.js'
|
|
38
56
|
import type { VxPlugin } from './plugin.js'
|
|
39
57
|
import { createHashCache, type HashCache } from './task-hash.js'
|
|
40
58
|
import type { Logger } from './logger.js'
|
|
41
59
|
import type { RunOptions } from './options.js'
|
|
42
|
-
import { type HistoryTable, LocalHistoryProvider } from './history.js'
|
|
43
|
-
import { computePredictedPriorities } from './predict.js'
|
|
44
60
|
|
|
45
61
|
export interface PreparedRun {
|
|
46
62
|
workspaceRoot: string
|
|
47
63
|
workspaceConfig: WorkspaceConfig | null
|
|
64
|
+
/** The workspace's declared plugins, in declaration order. Nothing is added. */
|
|
65
|
+
plugins: readonly VxPlugin[]
|
|
48
66
|
cacheDir: string
|
|
49
67
|
cache: CacheLayer
|
|
50
68
|
/**
|
|
@@ -54,12 +72,30 @@ export interface PreparedRun {
|
|
|
54
72
|
*/
|
|
55
73
|
localCache: Cache
|
|
56
74
|
/**
|
|
57
|
-
*
|
|
58
|
-
*
|
|
59
|
-
*
|
|
75
|
+
* True when `cache` is something other than the bare local handle — an
|
|
76
|
+
* injected remote layer, or one a plugin's `cache` capability built. The
|
|
77
|
+
* remote axes of the policy only mean anything then: without a remote
|
|
78
|
+
* layer a `remote:w` policy writes NOWHERE, so a task that believed it
|
|
79
|
+
* would be saved still cleans its outputs before executing and (under
|
|
80
|
+
* `--verify`) restores an artifact that was never written.
|
|
81
|
+
*/
|
|
82
|
+
hasRemoteLayer: boolean
|
|
83
|
+
/**
|
|
84
|
+
* Scheduling priorities from plugins' `schedule` stage (task id → weight,
|
|
85
|
+
* merged over the structural baseline by the scheduler). Empty when no
|
|
86
|
+
* plugin declares the stage.
|
|
60
87
|
*/
|
|
61
88
|
priorities: ReadonlyMap<string, number>
|
|
62
89
|
nodes: Map<string, TaskNode>
|
|
90
|
+
/**
|
|
91
|
+
* Requested task specs that matched NO project — a typo, or a stray
|
|
92
|
+
* positional (the value of an `=`-only flag written with a space).
|
|
93
|
+
* Nothing they asked for is in `nodes`, so callers must fail rather
|
|
94
|
+
* than run the remainder silently.
|
|
95
|
+
*/
|
|
96
|
+
unresolvedTasks: readonly string[]
|
|
97
|
+
/** Every discovered project — the declared task names a typo is measured against. */
|
|
98
|
+
projects: ReadonlyMap<string, ProjectEntry>
|
|
63
99
|
workspaceFingerprint: string
|
|
64
100
|
nestedDirsByProject: Map<string, string[]>
|
|
65
101
|
/**
|
|
@@ -93,6 +129,25 @@ export interface PreparedRun {
|
|
|
93
129
|
empty: null | 'no-tasks-declared' | 'empty-graph'
|
|
94
130
|
}
|
|
95
131
|
|
|
132
|
+
/** Project names named by a `pkg#task` dependsOn entry anywhere in `config`. */
|
|
133
|
+
function crossDepProjects(config: ProjectConfig): string[] {
|
|
134
|
+
const out: string[] = []
|
|
135
|
+
for (const task of Object.values(config.tasks ?? {})) {
|
|
136
|
+
for (const raw of task.dependsOn ?? []) {
|
|
137
|
+
// A malformed spec is the graph builder's error to report — it names
|
|
138
|
+
// the offending task. Here it just contributes no project.
|
|
139
|
+
if (!raw.includes('#')) continue
|
|
140
|
+
try {
|
|
141
|
+
const spec = parseDependencySpec(raw)
|
|
142
|
+
if (spec.kind === 'cross') out.push(spec.project)
|
|
143
|
+
} catch {
|
|
144
|
+
continue
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
return out
|
|
149
|
+
}
|
|
150
|
+
|
|
96
151
|
/**
|
|
97
152
|
* Build the prepared-run context: workspace discovery, project-config
|
|
98
153
|
* load, package + task graph, cache handle (local, optionally wrapped
|
|
@@ -104,19 +159,47 @@ export interface PreparedRun {
|
|
|
104
159
|
* empty plan).
|
|
105
160
|
*/
|
|
106
161
|
export async function prepareRun(options: RunOptions, log: Logger): Promise<PreparedRun> {
|
|
162
|
+
mark('startup')
|
|
107
163
|
const workspaceRoot = await findWorkspaceRoot(options.cwd)
|
|
164
|
+
// An UNSCOPED run (no explicit scope, at least one bare task name)
|
|
165
|
+
// enumerates the whole tree whatever the configs say, so git starts
|
|
166
|
+
// HERE — the walk needs only the root — and overlaps the workspace
|
|
167
|
+
// config, discovery, the cache open and the config evaluation. Its
|
|
168
|
+
// ~60 ms is the warm run's wall floor on a 1000-project tree; it used to
|
|
169
|
+
// start after discovery and the cache open, ~25 ms later. A scoped run
|
|
170
|
+
// waits: its pathspecs depend on which projects the configs pull in.
|
|
171
|
+
const unscoped =
|
|
172
|
+
options.projects === undefined && options.tasks.some((spec) => spec.indexOf('#') <= 0)
|
|
173
|
+
const earlyGit: Promise<GitEnumeration> | undefined = unscoped
|
|
174
|
+
? startGitEnumeration(workspaceRoot, ['.'])
|
|
175
|
+
: undefined
|
|
176
|
+
// A broken config below throws before this is awaited; the detached
|
|
177
|
+
// handler keeps that from surfacing as an unhandled rejection (the real
|
|
178
|
+
// await further down still sees the error).
|
|
179
|
+
earlyGit?.catch(() => {})
|
|
108
180
|
const workspace = await loadWorkspace(workspaceRoot)
|
|
109
181
|
const workspaceConfig = await loadWorkspaceConfig(workspaceRoot)
|
|
182
|
+
// The `config` stage runs before anything is derived from the workspace
|
|
183
|
+
// config; the plugin list itself is already fixed.
|
|
184
|
+
const plugins = (workspaceConfig?.plugins ?? []) as readonly VxPlugin[]
|
|
185
|
+
if (workspaceConfig !== null && hasHook(plugins, 'config')) {
|
|
186
|
+
await applyConfigHooks(plugins, workspaceConfig, {
|
|
187
|
+
workspaceRoot,
|
|
188
|
+
warn: (m) => log.status(m),
|
|
189
|
+
})
|
|
190
|
+
}
|
|
191
|
+
mark('workspace config')
|
|
110
192
|
const projectMetas = await listProjects(workspace)
|
|
193
|
+
mark('discover projects')
|
|
111
194
|
|
|
112
195
|
// SCOPED config loading: configs are programs, and evaluating 1090
|
|
113
196
|
// of them costs ~200 ms — the dominant fixed cost of small runs.
|
|
114
|
-
// Only the in-scope projects
|
|
115
|
-
//
|
|
116
|
-
//
|
|
117
|
-
// Side effect, deliberate and
|
|
118
|
-
//
|
|
119
|
-
// when that package enters
|
|
197
|
+
// Only the in-scope projects, their transitive dependency closure
|
|
198
|
+
// (which bounds '^task' frontier expansion) and any project named by
|
|
199
|
+
// a `pkg#task` dependsOn entry can contribute graph nodes, so only
|
|
200
|
+
// those configs are evaluated. Side effect, deliberate and
|
|
201
|
+
// Turbo-like: a broken config in an unrelated package no longer
|
|
202
|
+
// fails a scoped run — it surfaces when that package enters scope.
|
|
120
203
|
const packageGraph = buildPackageGraph(projectMetas)
|
|
121
204
|
const projectsWithConfigs = projectMetas.filter(
|
|
122
205
|
(m): m is typeof m & { configPath: string } =>
|
|
@@ -146,13 +229,35 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
146
229
|
for (const project of anchored) {
|
|
147
230
|
if (haveConfig.has(project)) seeds.add(project)
|
|
148
231
|
}
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
232
|
+
|
|
233
|
+
// The local cache opens BEFORE the configs load: it is also where their
|
|
234
|
+
// cached evaluations live. `--cache-dir <path>` (RunOptions.cacheDir)
|
|
235
|
+
// overrides the workspace `cacheDir` field + the `.vx/cache` default;
|
|
236
|
+
// resolved relative to cwd.
|
|
237
|
+
const policy: CachePolicy = options.cache ?? FULL_CACHE_POLICY
|
|
238
|
+
const cacheDir = options.cacheDir
|
|
239
|
+
? path.resolve(options.cwd, options.cacheDir)
|
|
240
|
+
: resolveCacheDir(workspaceRoot, workspaceConfig)
|
|
241
|
+
const localCache = new Cache(cacheDir, { read: policy.localRead, write: policy.localWrite })
|
|
242
|
+
const workspaceFingerprint = await computeWorkspaceFingerprint(workspaceRoot)
|
|
243
|
+
mark('open cache')
|
|
244
|
+
|
|
245
|
+
type ConfigMeta = (typeof projectsWithConfigs)[number]
|
|
246
|
+
const metaByName = new Map<string, ConfigMeta>(projectsWithConfigs.map((m) => [m.name, m]))
|
|
247
|
+
const needed = new Set<string>()
|
|
248
|
+
const pending: ConfigMeta[] = []
|
|
249
|
+
const consider = (name: string): void => {
|
|
250
|
+
if (needed.has(name)) return
|
|
251
|
+
needed.add(name)
|
|
252
|
+
const meta = metaByName.get(name)
|
|
253
|
+
if (meta) pending.push(meta)
|
|
254
|
+
}
|
|
255
|
+
const considerWithDeps = (name: string): void => {
|
|
256
|
+
consider(name)
|
|
257
|
+
for (const dep of packageGraph.transitiveDeps(name)) consider(dep)
|
|
152
258
|
}
|
|
259
|
+
for (const seed of seeds) considerWithDeps(seed)
|
|
153
260
|
|
|
154
|
-
const projects = new Map<string, ProjectEntry>()
|
|
155
|
-
const toLoad = projectsWithConfigs.filter((m) => needed.has(m.name))
|
|
156
261
|
// Frozen mode (--frozen, CI): configs load FROM vx-lock.json after a
|
|
157
262
|
// content-hash tripwire — no evaluation; env-dependent configs keep
|
|
158
263
|
// their locked values. Default (local) runs ALWAYS evaluate live:
|
|
@@ -166,16 +271,52 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
166
271
|
`--frozen requires vx-lock.json at the workspace root — run 'vx lock' and commit it`,
|
|
167
272
|
)
|
|
168
273
|
}
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
)
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
274
|
+
|
|
275
|
+
// Load in rounds to a fixpoint. A `pkg#task` dependsOn entry names a
|
|
276
|
+
// project the PACKAGE graph cannot reach (the cross form ignores npm
|
|
277
|
+
// deps by design), so its config has to be pulled in — and that config
|
|
278
|
+
// may declare cross edges of its own. The common case (no cross deps)
|
|
279
|
+
// is a single round, identical to loading the closure in one batch.
|
|
280
|
+
const projects = new Map<string, ProjectEntry>()
|
|
281
|
+
const projectStage = hasHook(plugins, 'project')
|
|
282
|
+
try {
|
|
283
|
+
while (pending.length > 0) {
|
|
284
|
+
const round = pending.splice(0, pending.length)
|
|
285
|
+
const configs = lock
|
|
286
|
+
? await Promise.all(round.map((m) => frozenProjectConfig(lock, m, workspaceRoot)))
|
|
287
|
+
: await loadProjectConfigs(
|
|
288
|
+
round.map((m) => m.configPath),
|
|
289
|
+
{ evalCache: { store: localCache, workspaceFingerprint } },
|
|
290
|
+
)
|
|
291
|
+
for (let i = 0; i < round.length; i++) {
|
|
292
|
+
const meta = round[i]!
|
|
293
|
+
const config = configs[i] as ProjectConfig
|
|
294
|
+
if (projectStage) {
|
|
295
|
+
// The `project` stage edits the validated object in place; core
|
|
296
|
+
// then re-validates so a plugin can only produce what the loader
|
|
297
|
+
// accepts from a user. The message names the config file — the
|
|
298
|
+
// plugin's edit is a defect in THAT project's tasks.
|
|
299
|
+
await applyProjectHooks(plugins, config, {
|
|
300
|
+
workspaceRoot,
|
|
301
|
+
cacheDir,
|
|
302
|
+
warn: (m) => log.status(m),
|
|
303
|
+
name: meta.name,
|
|
304
|
+
dir: meta.dir,
|
|
305
|
+
packageJson: meta.packageJson as unknown as Readonly<Record<string, unknown>>,
|
|
306
|
+
})
|
|
307
|
+
validateProjectConfig(config, `${meta.configPath} (after plugins)`)
|
|
308
|
+
}
|
|
309
|
+
projects.set(meta.name, { name: meta.name, dir: meta.dir, config })
|
|
310
|
+
for (const name of crossDepProjects(config)) considerWithDeps(name)
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
} catch (err) {
|
|
314
|
+
// The cache opened before the configs loaded (it holds their cached
|
|
315
|
+
// evaluations); a config error must not leak the handle.
|
|
316
|
+
localCache.close()
|
|
317
|
+
throw err
|
|
178
318
|
}
|
|
319
|
+
mark('load configs')
|
|
179
320
|
|
|
180
321
|
// Boundary geometry considers every config-bearing project in the
|
|
181
322
|
// workspace, loaded or not — an out-of-scope nested project must
|
|
@@ -189,14 +330,8 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
189
330
|
: [...projects.keys()]
|
|
190
331
|
|
|
191
332
|
const requested = expandRequested(options.tasks, candidateProjects, projects)
|
|
333
|
+
const unresolvedTasks = unresolvedRequests(options.tasks, candidateProjects, projects)
|
|
192
334
|
|
|
193
|
-
const policy: CachePolicy = options.cache ?? FULL_CACHE_POLICY
|
|
194
|
-
// `--cache-dir <path>` (RunOptions.cacheDir) overrides the workspace
|
|
195
|
-
// `cacheDir` field + the `.vx/cache` default; resolved relative to cwd.
|
|
196
|
-
const cacheDir = options.cacheDir
|
|
197
|
-
? path.resolve(options.cwd, options.cacheDir)
|
|
198
|
-
: resolveCacheDir(workspaceRoot, workspaceConfig)
|
|
199
|
-
const localCache = new Cache(cacheDir, { read: policy.localRead, write: policy.localWrite })
|
|
200
335
|
// Cache seam precedence: an EXPLICITLY injected remote layer
|
|
201
336
|
// (RunOptions.remoteCache — a distribution agent or serve that already
|
|
202
337
|
// holds a wire client) wins outright; else a plugin's `cache` capability;
|
|
@@ -204,20 +339,26 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
204
339
|
// cache is a plugin concern (native-cache-wire-2026-07). Injection
|
|
205
340
|
// winning prevents double-wrapping when the workspace also declares a
|
|
206
341
|
// cache plugin.
|
|
207
|
-
const plugins = (workspaceConfig?.plugins ?? []) as readonly VxPlugin[]
|
|
208
342
|
const cache = options.remoteCache
|
|
209
343
|
? new LayeredCache(localCache, options.remoteCache, {
|
|
210
344
|
policy,
|
|
211
345
|
onRemoteError: (err) => log.status(`[vx] remote cache: ${err.message}`),
|
|
212
346
|
})
|
|
213
|
-
: await resolveCache(
|
|
214
|
-
|
|
347
|
+
: await resolveCache(plugins, {
|
|
348
|
+
workspaceRoot,
|
|
349
|
+
cacheDir,
|
|
350
|
+
warn: (m) => log.status(m),
|
|
215
351
|
localCache,
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
352
|
+
policy,
|
|
353
|
+
})
|
|
354
|
+
// Ask the LAYER, don't infer. Identity against `localCache` answers a
|
|
355
|
+
// DIFFERENT question — "did the plugin hand back something other than the
|
|
356
|
+
// handle I passed in?" — which an ordinary pass-through decorator (a
|
|
357
|
+
// metrics wrapper, a cache-dir redirect) with no remote at all answers
|
|
358
|
+
// yes to, skipping the remote-axis clamp below. `hasRemote` is the
|
|
359
|
+
// layer's own truthful answer; `LayeredCache` sets it, a bare `Cache`
|
|
360
|
+
// doesn't, and a third-party layer opts in when it really has a remote.
|
|
361
|
+
const hasRemoteLayer = cache.hasRemote === true
|
|
221
362
|
|
|
222
363
|
const gitFilesCache = new GitFilesCache()
|
|
223
364
|
// Bulk-populate via a single `git ls-files` at the workspace root —
|
|
@@ -231,12 +372,14 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
231
372
|
(t) => (t.cache?.inputs.workspaceFiles?.length ?? 0) > 0,
|
|
232
373
|
),
|
|
233
374
|
)
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
375
|
+
const projectDirs = [...projects.values()].map((p) => p.dir)
|
|
376
|
+
const enumeration = await (earlyGit ??
|
|
377
|
+
startGitEnumeration(
|
|
378
|
+
workspaceRoot,
|
|
379
|
+
gitPathspecs(workspaceRoot, projectDirs, usesWorkspaceInputs),
|
|
380
|
+
))
|
|
381
|
+
applyGitEnumeration(enumeration, workspaceRoot, projectDirs, gitFilesCache, usesWorkspaceInputs)
|
|
382
|
+
mark('git enumeration')
|
|
240
383
|
const hashCache = createHashCache()
|
|
241
384
|
|
|
242
385
|
// Empty-cases bookkeeping. We still construct the cache + fingerprint
|
|
@@ -245,11 +388,15 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
245
388
|
return {
|
|
246
389
|
workspaceRoot,
|
|
247
390
|
workspaceConfig,
|
|
391
|
+
plugins,
|
|
248
392
|
cacheDir,
|
|
249
393
|
cache,
|
|
250
394
|
localCache,
|
|
395
|
+
hasRemoteLayer,
|
|
251
396
|
priorities: new Map(),
|
|
252
397
|
nodes: new Map(),
|
|
398
|
+
unresolvedTasks,
|
|
399
|
+
projects,
|
|
253
400
|
workspaceFingerprint,
|
|
254
401
|
nestedDirsByProject,
|
|
255
402
|
gitFilesCache,
|
|
@@ -267,35 +414,39 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
267
414
|
? { excludeDependencies: options.excludeDependencies }
|
|
268
415
|
: {}),
|
|
269
416
|
})
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
417
|
+
if (hasHook(plugins, 'graph')) {
|
|
418
|
+
await applyGraphHooks(plugins, nodes, {
|
|
419
|
+
workspaceRoot,
|
|
420
|
+
cacheDir,
|
|
421
|
+
warn: (m) => log.status(m),
|
|
422
|
+
requested: [...nodes.values()].filter((n) => n.requested).map((n) => n.id),
|
|
423
|
+
})
|
|
424
|
+
}
|
|
425
|
+
if (hasHook(plugins, 'key')) {
|
|
426
|
+
await applyKeyHooks(plugins, nodes, { workspaceRoot, cacheDir, warn: (m) => log.status(m) })
|
|
427
|
+
}
|
|
277
428
|
let priorities: ReadonlyMap<string, number> = new Map()
|
|
278
|
-
if (
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
}
|
|
285
|
-
log.status(
|
|
286
|
-
`[vx] predictive scheduling fell back to baseline: ${err instanceof Error ? err.message : String(err)}`,
|
|
287
|
-
)
|
|
288
|
-
}
|
|
429
|
+
if (hasHook(plugins, 'schedule')) {
|
|
430
|
+
priorities = await applyScheduleHooks(plugins, nodes, {
|
|
431
|
+
workspaceRoot,
|
|
432
|
+
cacheDir,
|
|
433
|
+
warn: (m) => log.status(m),
|
|
434
|
+
localCache,
|
|
435
|
+
})
|
|
289
436
|
}
|
|
290
437
|
|
|
291
438
|
return {
|
|
292
439
|
workspaceRoot,
|
|
293
440
|
workspaceConfig,
|
|
441
|
+
plugins,
|
|
294
442
|
cacheDir,
|
|
295
443
|
cache,
|
|
296
444
|
localCache,
|
|
445
|
+
hasRemoteLayer,
|
|
297
446
|
priorities,
|
|
298
447
|
nodes,
|
|
448
|
+
unresolvedTasks,
|
|
449
|
+
projects,
|
|
299
450
|
workspaceFingerprint,
|
|
300
451
|
nestedDirsByProject,
|
|
301
452
|
gitFilesCache,
|
|
@@ -14,17 +14,20 @@
|
|
|
14
14
|
// HARD SCOPE: this runs ONLY when a remote layer is configured. It
|
|
15
15
|
// NEVER touches the local cache (no local get / no isOutputsCurrent /
|
|
16
16
|
// no stat pass) — it derives keys and probes REMOTE only. Local-only
|
|
17
|
-
// runs are byte-identical to before; the caller gates on
|
|
18
|
-
//
|
|
17
|
+
// runs are byte-identical to before; the caller gates on the layer's
|
|
18
|
+
// own `hasRemote`, so a third-party remote layer gets the overlap too.
|
|
19
|
+
// The two batch-probe hooks are OPTIONAL on the contract: a remote layer
|
|
20
|
+
// that implements neither still prefetches, per-hash.
|
|
19
21
|
|
|
20
22
|
import type { TaskNode } from '../graph/index.js'
|
|
21
|
-
import type {
|
|
23
|
+
import type { CacheLayer, GitFilesCache } from '../cache/index.js'
|
|
22
24
|
import type { HashCache } from './task-hash.js'
|
|
23
25
|
import { deriveStableKeys } from './stable-keys.js'
|
|
24
26
|
|
|
25
27
|
export interface PrefetchArgs {
|
|
26
28
|
nodes: Map<string, TaskNode>
|
|
27
|
-
|
|
29
|
+
/** A layer whose `hasRemote` is true — the caller's gate. */
|
|
30
|
+
cache: CacheLayer
|
|
28
31
|
workspaceRoot: string
|
|
29
32
|
workspaceFingerprint: string
|
|
30
33
|
forwardArgs?: readonly string[] | undefined
|
|
@@ -67,13 +70,14 @@ async function runPrefetch(args: PrefetchArgs): Promise<void> {
|
|
|
67
70
|
// tells us which of the N stable hashes exist remotely, so we GET only the
|
|
68
71
|
// hits and pre-mark the misses — their lazy `get` then short-circuits with
|
|
69
72
|
// no network. This collapses N probe waves into 1 and skips every GET that
|
|
70
|
-
// would 404. When the remote can't batch (`null` — an older serve,
|
|
71
|
-
// disabled
|
|
73
|
+
// would 404. When the remote can't batch (`null` — an older serve, reads
|
|
74
|
+
// disabled, or a layer that doesn't implement the hook at all), fall back to
|
|
75
|
+
// prefetching every stable key, exactly as before.
|
|
72
76
|
let toPrefetch = stableKeys
|
|
73
77
|
const uniqueHashes = [...new Set(stableKeys.map((k) => k.hash))]
|
|
74
|
-
const present = await args.cache.remoteHasMany(uniqueHashes)
|
|
78
|
+
const present = (await args.cache.remoteHasMany?.(uniqueHashes)) ?? null
|
|
75
79
|
if (present !== null) {
|
|
76
|
-
args.cache.markRemoteAbsent(uniqueHashes.filter((h) => !present.has(h)))
|
|
80
|
+
args.cache.markRemoteAbsent?.(uniqueHashes.filter((h) => !present.has(h)))
|
|
77
81
|
toPrefetch = stableKeys.filter((k) => present.has(k.hash))
|
|
78
82
|
if (toPrefetch.length === 0) return
|
|
79
83
|
}
|
|
@@ -1,39 +1,27 @@
|
|
|
1
|
-
// Resolve `exec.resources` declarations into absolute per-task costs the
|
|
2
|
-
// scheduler's 2-D admission gate packs against. Pure
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
1
|
+
// Resolve `exec.resources` declarations into the absolute per-task costs the
|
|
2
|
+
// scheduler's 2-D admission gate packs against. Pure; the loader has already
|
|
3
|
+
// validated every form (this module trusts internal code), so an unparseable
|
|
4
|
+
// value can't reach it from a real config.
|
|
5
|
+
//
|
|
6
|
+
// The declared units are CPU CORES and MEGABYTES — the same numbers on this
|
|
7
|
+
// machine and on a remote worker, which is why they are also what
|
|
8
|
+
// `TaskPlacement.resources` hands an executor verbatim. Percent forms were
|
|
9
|
+
// removed on 2026-08-30: a percentage names a fraction of THIS run's budget,
|
|
10
|
+
// and an executor placing the task on someone else's machine has no way to
|
|
11
|
+
// mean anything by it.
|
|
7
12
|
|
|
8
13
|
import type { ResourceCost, TaskNode } from '../graph/index.js'
|
|
9
|
-
import { parseSize } from '../util/index.js'
|
|
10
14
|
|
|
11
|
-
const
|
|
15
|
+
const BYTES_PER_MB = 1024 * 1024
|
|
12
16
|
|
|
13
|
-
/**
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
* Absent → 0 = reserve nothing.
|
|
17
|
-
*/
|
|
18
|
-
export function resolveCpu(v: number | string | undefined, cpuBudget: number): number {
|
|
19
|
-
if (v === undefined) return 0
|
|
20
|
-
if (typeof v === 'number') return v
|
|
21
|
-
const m = v.match(PERCENT_RE)
|
|
22
|
-
if (m) return (Number(m[1]) / 100) * cpuBudget
|
|
23
|
-
return 0
|
|
17
|
+
/** `cpus` → CPU units. Absent → 0 = reserve nothing. */
|
|
18
|
+
export function resolveCpu(v: number | undefined): number {
|
|
19
|
+
return v ?? 0
|
|
24
20
|
}
|
|
25
21
|
|
|
26
|
-
/**
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
* budget. Absent → 0 = reserve nothing.
|
|
30
|
-
*/
|
|
31
|
-
export function resolveMem(v: number | string | undefined, memBudget: number): number {
|
|
32
|
-
if (v === undefined) return 0
|
|
33
|
-
if (typeof v === 'number') return v
|
|
34
|
-
const m = v.match(PERCENT_RE)
|
|
35
|
-
if (m) return (Number(m[1]) / 100) * memBudget
|
|
36
|
-
return parseSize(v) ?? 0
|
|
22
|
+
/** `memory` (megabytes) → bytes, which is what the budget axis counts. */
|
|
23
|
+
export function resolveMem(v: number | undefined): number {
|
|
24
|
+
return v === undefined ? 0 : v * BYTES_PER_MB
|
|
37
25
|
}
|
|
38
26
|
|
|
39
27
|
/**
|
|
@@ -44,15 +32,13 @@ export function resolveMem(v: number | string | undefined, memBudget: number): n
|
|
|
44
32
|
*/
|
|
45
33
|
export function resolveResourceCosts(
|
|
46
34
|
nodes: ReadonlyMap<string, TaskNode>,
|
|
47
|
-
cpuBudget: number,
|
|
48
|
-
memBudget: number,
|
|
49
35
|
): Map<string, ResourceCost> {
|
|
50
36
|
const out = new Map<string, ResourceCost>()
|
|
51
37
|
for (const node of nodes.values()) {
|
|
52
38
|
const declared = node.config.exec?.resources
|
|
53
39
|
if (declared === undefined) continue
|
|
54
|
-
const cpu = resolveCpu(declared.cpus
|
|
55
|
-
const mem = resolveMem(declared.memory
|
|
40
|
+
const cpu = resolveCpu(declared.cpus)
|
|
41
|
+
const mem = resolveMem(declared.memory)
|
|
56
42
|
if (cpu > 0 || mem > 0) out.set(node.id, { cpu, mem })
|
|
57
43
|
}
|
|
58
44
|
return out
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
// Both are no-ops if the corresponding RunOptions field is undefined.
|
|
15
15
|
|
|
16
16
|
import path from 'node:path'
|
|
17
|
-
import type
|
|
17
|
+
import { isGroupTask, type TaskOutcome } from '../graph/index.js'
|
|
18
18
|
import { tallyOutcomes } from './tally.js'
|
|
19
19
|
|
|
20
20
|
export interface SummarizeArgs {
|
|
@@ -26,34 +26,59 @@ export interface SummarizeArgs {
|
|
|
26
26
|
startedAtMs: number
|
|
27
27
|
endedAtMs: number
|
|
28
28
|
totalMs: number
|
|
29
|
+
/** The run's verdict — the same value the CLI turns into the exit code. */
|
|
30
|
+
ok: boolean
|
|
29
31
|
outcomes: readonly TaskOutcome[]
|
|
30
32
|
}
|
|
31
33
|
|
|
34
|
+
/** One task's entry. Shared by `tasks` and `aborted` so they read alike. */
|
|
35
|
+
function taskEntry(o: TaskOutcome): Record<string, unknown> {
|
|
36
|
+
return {
|
|
37
|
+
id: o.node.id,
|
|
38
|
+
project: o.node.projectName,
|
|
39
|
+
task: o.node.taskName,
|
|
40
|
+
status: o.status,
|
|
41
|
+
exitCode: o.exitCode,
|
|
42
|
+
durationMs: o.durationMs,
|
|
43
|
+
hash: o.hash ?? null,
|
|
44
|
+
...(o.cpuMs !== undefined ? { cpuMs: o.cpuMs } : {}),
|
|
45
|
+
...(o.peakRssBytes !== undefined ? { peakRssBytes: o.peakRssBytes } : {}),
|
|
46
|
+
// hrtime spans are bigints → emit as strings so JSON.parse on
|
|
47
|
+
// the consumer side doesn't truncate the ns precision.
|
|
48
|
+
...(o.wallclockStartNs !== undefined ? { wallclockStartNs: String(o.wallclockStartNs) } : {}),
|
|
49
|
+
...(o.wallclockEndNs !== undefined ? { wallclockEndNs: String(o.wallclockEndNs) } : {}),
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
32
53
|
export async function writeRunSummary(args: SummarizeArgs): Promise<string> {
|
|
33
54
|
const outPath =
|
|
34
55
|
args.target === ''
|
|
35
56
|
? path.join(args.cacheDir, 'runs', `${args.runId}.json`)
|
|
36
57
|
: path.resolve(args.cwd, args.target)
|
|
58
|
+
// `tasks` and `summary` must describe the same population, or one artifact
|
|
59
|
+
// contradicts itself (a group task listed in `tasks` but absent from
|
|
60
|
+
// `summary.total`). `tallyOutcomes` owns the rule; mirror its filter here.
|
|
61
|
+
const counted = args.outcomes.filter((o) => !isGroupTask(o.node) && o.status !== 'aborted')
|
|
62
|
+
// A task killed by a shutdown signal is in no bucket and no total, yet it
|
|
63
|
+
// makes the run red. Listing it separately keeps `tasks.length ===
|
|
64
|
+
// summary.total` while leaving the artifact able to explain a non-zero
|
|
65
|
+
// exit — the terminal has said so in its Aborted section all along, and a
|
|
66
|
+
// parser must not be told less than a human is.
|
|
67
|
+
const aborted = args.outcomes.filter((o) => !isGroupTask(o.node) && o.status === 'aborted')
|
|
37
68
|
const payload = {
|
|
38
69
|
runId: args.runId,
|
|
70
|
+
// The run-level verdict, first: a consumer gating on this artifact must
|
|
71
|
+
// not have to re-derive it by re-implementing the bucket rules.
|
|
72
|
+
ok: args.ok,
|
|
73
|
+
exitCode: args.ok ? 0 : 1,
|
|
39
74
|
startedAt: new Date(args.startedAtMs).toISOString(),
|
|
40
75
|
endedAt: new Date(args.endedAtMs).toISOString(),
|
|
41
76
|
totalMs: args.totalMs,
|
|
42
|
-
tasks:
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
exitCode: o.exitCode,
|
|
48
|
-
durationMs: o.durationMs,
|
|
49
|
-
hash: o.hash ?? null,
|
|
50
|
-
...(o.cpuMs !== undefined ? { cpuMs: o.cpuMs } : {}),
|
|
51
|
-
...(o.peakRssBytes !== undefined ? { peakRssBytes: o.peakRssBytes } : {}),
|
|
52
|
-
// hrtime spans are bigints → emit as strings so JSON.parse on
|
|
53
|
-
// the consumer side doesn't truncate the ns precision.
|
|
54
|
-
...(o.wallclockStartNs !== undefined ? { wallclockStartNs: String(o.wallclockStartNs) } : {}),
|
|
55
|
-
...(o.wallclockEndNs !== undefined ? { wallclockEndNs: String(o.wallclockEndNs) } : {}),
|
|
56
|
-
})),
|
|
77
|
+
tasks: counted.map(taskEntry),
|
|
78
|
+
aborted: aborted.map(taskEntry),
|
|
79
|
+
// The FULL list: `tallyOutcomes` applies the same group/aborted
|
|
80
|
+
// exclusions internally, so every counted bucket is unchanged by passing
|
|
81
|
+
// it — but `summary.aborted` is only non-zero if it sees them.
|
|
57
82
|
summary: tallyOutcomes(args.outcomes),
|
|
58
83
|
}
|
|
59
84
|
await Bun.write(outPath, JSON.stringify(payload, null, 2))
|