@vzn/vx 0.0.18 → 0.0.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +99 -65
- package/package.json +10 -20
- package/src/bin.ts +12 -2
- package/src/cache/archive.ts +120 -18
- package/src/cache/cache.ts +313 -1324
- package/src/cache/chained-cache.ts +0 -8
- package/src/cache/config-evals.ts +89 -0
- package/src/cache/file-hashes.ts +262 -0
- package/src/cache/git-inputs.ts +637 -0
- package/src/cache/index.ts +16 -7
- package/src/cache/inputs.ts +246 -733
- package/src/cache/layer.ts +604 -0
- package/src/cache/layered-cache.ts +12 -17
- package/src/cache/output-index.ts +224 -0
- package/src/cache/policy.ts +84 -0
- package/src/cache/run-history.ts +147 -0
- package/src/cache/zstd.ts +160 -0
- package/src/cli/cache.ts +39 -11
- package/src/cli/completions.ts +104 -0
- package/src/cli/core-alias.ts +25 -0
- package/src/cli/help.ts +64 -36
- package/src/cli/index.ts +54 -15
- package/src/cli/info.ts +120 -91
- package/src/cli/init.ts +75 -0
- package/src/cli/last.ts +38 -6
- package/src/cli/lock.ts +2 -1
- package/src/cli/plan-format.ts +14 -3
- package/src/cli/plugin-commands.ts +15 -8
- package/src/cli/run.ts +84 -302
- package/src/cli/select.ts +332 -0
- package/src/cli/show.ts +144 -69
- package/src/cli/upgrade.ts +2 -1
- package/src/cli/watch.ts +647 -97
- package/src/cli/why.ts +44 -14
- package/src/cli/workspace-config.ts +126 -0
- package/src/config.ts +156 -136
- package/src/exec/env.ts +11 -0
- package/src/exec/executor.ts +11 -26
- package/src/exec/index.ts +6 -2
- package/src/{plugins/local-executor/index.ts → exec/local-executor.ts} +12 -16
- package/src/exec/runner.ts +57 -18
- package/src/exec/sandbox-binds.ts +182 -0
- package/src/exec/sandbox-paths.ts +50 -0
- package/src/exec/sandbox-runtime.ts +582 -370
- package/src/exec/sandbox-violations.ts +280 -0
- package/src/graph/index.ts +1 -9
- package/src/graph/scheduler.ts +158 -203
- package/src/graph/task-graph.ts +2 -2
- package/src/index.ts +97 -102
- package/src/orchestrator/admission.ts +137 -0
- package/src/orchestrator/deferred-outputs.ts +3 -0
- package/src/orchestrator/doctor.ts +255 -0
- package/src/orchestrator/events.ts +42 -8
- package/src/orchestrator/execute-task.ts +145 -554
- package/src/orchestrator/failure-mode.ts +169 -28
- package/src/orchestrator/framed-output.ts +41 -36
- package/src/orchestrator/history.ts +118 -98
- package/src/orchestrator/hit-restore.ts +221 -0
- package/src/orchestrator/index.ts +35 -4
- package/src/orchestrator/lockfile-claim.ts +281 -0
- package/src/orchestrator/logger.ts +13 -22
- package/src/orchestrator/metrics.ts +53 -20
- package/src/orchestrator/miss-save.ts +169 -0
- package/src/orchestrator/options.ts +34 -31
- package/src/orchestrator/persistent.ts +79 -0
- package/src/orchestrator/placement.ts +182 -0
- package/src/orchestrator/plugin-host.ts +195 -38
- package/src/orchestrator/plugin.ts +162 -9
- package/src/orchestrator/prepare.ts +77 -132
- package/src/orchestrator/projects.ts +259 -0
- package/src/orchestrator/remote-prefetch.ts +1 -1
- package/src/orchestrator/run-artifacts.ts +25 -3
- package/src/orchestrator/run-context.ts +1 -1
- package/src/orchestrator/run-records.ts +149 -0
- package/src/orchestrator/run-report.ts +2 -2
- package/src/orchestrator/run.ts +246 -597
- package/src/orchestrator/sandbox-request.ts +243 -0
- package/src/orchestrator/save-lane.ts +47 -0
- package/src/orchestrator/signals.ts +111 -0
- package/src/orchestrator/stable-keys.ts +60 -3
- package/src/orchestrator/status-line.ts +2 -1
- package/src/orchestrator/summary.ts +68 -25
- package/src/orchestrator/task-hash.ts +20 -18
- package/src/orchestrator/task-log-buffer.ts +8 -11
- package/src/orchestrator/telemetry-host.ts +11 -1
- package/src/orchestrator/telemetry.ts +3 -17
- package/src/util/cgroup.ts +160 -0
- package/src/util/edit-distance.ts +49 -0
- package/src/util/index.ts +11 -3
- package/src/util/paths.ts +23 -2
- package/src/util/settle.ts +19 -0
- package/src/util/verbs.ts +39 -0
- package/src/workspace/affected.ts +111 -11
- package/src/workspace/config-cache.ts +11 -2
- package/src/workspace/config-eval.ts +11 -4
- package/src/workspace/config-imports.ts +5 -4
- package/src/workspace/config-schema.ts +837 -0
- package/src/workspace/filter.ts +43 -6
- package/src/workspace/fingerprint.ts +37 -3
- package/src/workspace/index.ts +30 -3
- package/src/workspace/lockfile.ts +12 -7
- package/src/{cli → workspace}/migrate-scripts.ts +26 -19
- package/src/{cli/migrate.ts → workspace/migration.ts} +133 -124
- package/src/workspace/package-graph.ts +98 -28
- package/src/workspace/project-loader.ts +76 -746
- package/src/workspace/workspace.ts +98 -13
- package/plugins/local-cache/index.ts +0 -2
- package/plugins/local-executor/index.ts +0 -2
- package/plugins/schedule-history/index.ts +0 -2
- package/src/cache/cas-backend.ts +0 -115
- package/src/cache/digest.ts +0 -32
- package/src/cli/migrate-emit.ts +0 -21
- package/src/cli/migrate-nx.ts +0 -473
- package/src/cli/migrate-turbo.ts +0 -401
- package/src/cli/prune.ts +0 -291
- package/src/orchestrator/missing-plugin.ts +0 -9
- package/src/orchestrator/resources.ts +0 -45
- package/src/orchestrator/verify.ts +0 -174
- package/src/plugins/local-cache/index.ts +0 -11
- package/src/plugins/schedule-history/index.ts +0 -109
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
// What a HIT leaves behind: the outputs of a confirmed cache entry
|
|
2
|
+
// materialised in the project — or proved already current and left alone —
|
|
3
|
+
// the exact changed paths marked so a same-project consumer need not
|
|
4
|
+
// re-spawn git, the stored stdout replayed, and the cache-hit outcome
|
|
5
|
+
// built. There is ONE restore path: `executeCachedTask` and the local
|
|
6
|
+
// short-circuit (a stable hit restored ahead of its deps) both come here,
|
|
7
|
+
// so the two cannot drift. Stale-hit-critical: a line moved here changes
|
|
8
|
+
// which bytes a green run replays. Moved out of execute-task.ts
|
|
9
|
+
// 2026-09-10 as pure code motion, the mirror of miss-save.ts.
|
|
10
|
+
|
|
11
|
+
import path from 'node:path'
|
|
12
|
+
import type { CacheConfig } from '../config.js'
|
|
13
|
+
import {
|
|
14
|
+
type CacheEntry,
|
|
15
|
+
cleanOutputs,
|
|
16
|
+
cleanWorkspaceOutputs,
|
|
17
|
+
resolveOutputs,
|
|
18
|
+
resolveWorkspaceOutputs,
|
|
19
|
+
WORKSPACE_OUTPUT_PREFIX,
|
|
20
|
+
} from '../cache/index.js'
|
|
21
|
+
import type { TaskOutcome } from '../graph/index.js'
|
|
22
|
+
import { span, wholeSubtreePrefixes } from '../util/index.js'
|
|
23
|
+
import type { ExecuteArgs } from './execute-task.js'
|
|
24
|
+
|
|
25
|
+
export interface RestoreHitArgs {
|
|
26
|
+
args: ExecuteArgs
|
|
27
|
+
hash: string
|
|
28
|
+
hit: CacheEntry
|
|
29
|
+
/** `performance.now()` when the cache op (probe) started — the
|
|
30
|
+
* user-perceived restore duration is measured from here. */
|
|
31
|
+
cacheOpStart: number
|
|
32
|
+
/** ns offset from run start for the outcome's wallclock window. */
|
|
33
|
+
taskStartNs: bigint
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Materialize a confirmed cache hit: decide skip-restore, clean +
|
|
38
|
+
* restore declared outputs, mark the exact changed paths so downstream
|
|
39
|
+
* same-project tasks needn't re-spawn git, replay stored stdout, and
|
|
40
|
+
* build the cache-hit `TaskOutcome`. Extracted from `executeCachedTask`
|
|
41
|
+
* so the local short-circuit can restore a stable-key hit ahead of the
|
|
42
|
+
* schedule using the IDENTICAL logic — there is one restore path, not
|
|
43
|
+
* two that could drift.
|
|
44
|
+
*/
|
|
45
|
+
export async function restoreHit(restore: RestoreHitArgs): Promise<TaskOutcome> {
|
|
46
|
+
const { args, hash, hit, cacheOpStart, taskStartNs } = restore
|
|
47
|
+
const { node, log } = args
|
|
48
|
+
const cacheCfg: CacheConfig | undefined = node.config.cache
|
|
49
|
+
const outputs = cacheCfg?.outputs.files ?? []
|
|
50
|
+
const wsOutputs = cacheCfg?.outputs.workspaceFiles ?? []
|
|
51
|
+
const anyOutputs = outputs.length > 0 || wsOutputs.length > 0
|
|
52
|
+
const dirPrefixes = wholeSubtreePrefixes(outputs)
|
|
53
|
+
const cleanArgs = {
|
|
54
|
+
projectDir: node.projectDir,
|
|
55
|
+
outputs,
|
|
56
|
+
nestedProjectDirs: args.nestedProjectDirs,
|
|
57
|
+
}
|
|
58
|
+
const wsCleanArgs = { workspaceRoot: args.workspaceRoot, outputs: wsOutputs }
|
|
59
|
+
|
|
60
|
+
// "Tree is already current" short-circuit — skip cleanOutputs
|
|
61
|
+
// + restoreOutputs when the on-disk state matches what this
|
|
62
|
+
// entry recorded at save time. Integrity-preserving: we
|
|
63
|
+
// require BOTH that the output-glob walk yields exactly the
|
|
64
|
+
// expected paths (no strays, no missing) AND that every file's
|
|
65
|
+
// (size, mode, mtime) matches the stored fingerprint. Any
|
|
66
|
+
// divergence falls through to a real clean + restore.
|
|
67
|
+
//
|
|
68
|
+
// The output-file fingerprints can't be batch-loaded at
|
|
69
|
+
// prepareRun time — non-leaf task hashes depend on upstream
|
|
70
|
+
// outputs that haven't been written yet, so hashes are
|
|
71
|
+
// necessarily computed mid-run. The short-circuit's batched probe
|
|
72
|
+
// loads the rows with the entry (`hit.outputRows`); only the lazy
|
|
73
|
+
// path pays one SELECT here. Either beats reading the manifest from
|
|
74
|
+
// the tar (decompress + parse) at the same point.
|
|
75
|
+
// A task with no declared outputs has nothing to clean or restore: its
|
|
76
|
+
// stdout replays from the row (`hit.stdout`), and its artifact holds
|
|
77
|
+
// only logs. Extracting it anyway cost an `exists` + a tar read per hit
|
|
78
|
+
// — 1.7 ms each on a four-task real repo whose `link` and `element`
|
|
79
|
+
// tasks declare `outputs: []` (2026-09-10, VX_TIMING).
|
|
80
|
+
let skipRestore = !anyOutputs
|
|
81
|
+
if (anyOutputs) {
|
|
82
|
+
const endRows = span('output rows')
|
|
83
|
+
const expected = hit.outputRows ?? args.cache.loadOutputFilesBatch([hash]).get(hash) ?? []
|
|
84
|
+
endRows()
|
|
85
|
+
if (expected.length > 0) {
|
|
86
|
+
// Two namespaces in the rows: bare rels are project outputs,
|
|
87
|
+
// `workspace-outputs/<rel>` rows anchor at the workspace root.
|
|
88
|
+
const projExpected = expected.filter((e) => !e.path.startsWith(WORKSPACE_OUTPUT_PREFIX))
|
|
89
|
+
const wsExpected = expected
|
|
90
|
+
.filter((e) => e.path.startsWith(WORKSPACE_OUTPUT_PREFIX))
|
|
91
|
+
.map((e) => ({ ...e, path: e.path.slice(WORKSPACE_OUTPUT_PREFIX.length) }))
|
|
92
|
+
// The directory short-circuit: for whole-subtree globs, unchanged
|
|
93
|
+
// mtimes on every directory recorded at the last save/restore prove
|
|
94
|
+
// the output SET is unchanged (a file added or removed anywhere the
|
|
95
|
+
// glob could see bumps a recorded directory), so the walk that cost
|
|
96
|
+
// 0.36 ms per warm hit is replaced by a few stats. The per-file
|
|
97
|
+
// fingerprint check below still runs; only the enumeration is skipped.
|
|
98
|
+
let setKnown = false
|
|
99
|
+
if (dirPrefixes !== null && wsOutputs.length === 0 && args.cache.outputDirsCurrent) {
|
|
100
|
+
const endDirs = span('output dirs')
|
|
101
|
+
// Loaded with the entry (batched by getMany, or with the lazy get).
|
|
102
|
+
const dirRows = hit.outputDirRows ?? []
|
|
103
|
+
const covers = dirPrefixes.every((pre) => dirRows.some((r) => r.path === pre))
|
|
104
|
+
setKnown = covers && (await args.cache.outputDirsCurrent!(node.projectDir, dirRows))
|
|
105
|
+
endDirs()
|
|
106
|
+
}
|
|
107
|
+
let actualAbs: string[]
|
|
108
|
+
let actualWsAbs: string[]
|
|
109
|
+
if (setKnown) {
|
|
110
|
+
actualAbs = projExpected.map((e) => path.join(node.projectDir, e.path))
|
|
111
|
+
actualWsAbs = []
|
|
112
|
+
} else {
|
|
113
|
+
const endGlob = span('output glob')
|
|
114
|
+
actualAbs = await resolveOutputs({
|
|
115
|
+
projectDir: node.projectDir,
|
|
116
|
+
outputs,
|
|
117
|
+
nestedProjectDirs: args.nestedProjectDirs,
|
|
118
|
+
})
|
|
119
|
+
actualWsAbs = await resolveWorkspaceOutputs({
|
|
120
|
+
workspaceRoot: args.workspaceRoot,
|
|
121
|
+
outputs: wsOutputs,
|
|
122
|
+
})
|
|
123
|
+
endGlob()
|
|
124
|
+
}
|
|
125
|
+
const setsMatch = (
|
|
126
|
+
actual: readonly string[],
|
|
127
|
+
exp: ReadonlyArray<{ path: string }>,
|
|
128
|
+
): boolean => {
|
|
129
|
+
const expSet = new Set(exp.map((e) => e.path))
|
|
130
|
+
return actual.length === expSet.size && actual.every((r) => expSet.has(r))
|
|
131
|
+
}
|
|
132
|
+
const actualRels = actualAbs.map((p) =>
|
|
133
|
+
path.relative(node.projectDir, p).split(path.sep).join('/'),
|
|
134
|
+
)
|
|
135
|
+
const actualWsRels = actualWsAbs.map((p) =>
|
|
136
|
+
path.relative(args.workspaceRoot, p).split(path.sep).join('/'),
|
|
137
|
+
)
|
|
138
|
+
if (setsMatch(actualRels, projExpected) && setsMatch(actualWsRels, wsExpected)) {
|
|
139
|
+
const endStat = span('output stat')
|
|
140
|
+
skipRestore =
|
|
141
|
+
(await args.cache.isOutputsCurrent(node.projectDir, projExpected)) &&
|
|
142
|
+
(await args.cache.isOutputsCurrent(args.workspaceRoot, wsExpected))
|
|
143
|
+
endStat()
|
|
144
|
+
// The walk ran and proved the tree current (rows absent, or a
|
|
145
|
+
// directory had moved — a benign touch): snapshot the directories
|
|
146
|
+
// so the next hit skips the walk.
|
|
147
|
+
if (skipRestore && !setKnown && dirPrefixes !== null) {
|
|
148
|
+
await args.cache.recordOutputDirs?.(hash, node.projectDir, dirPrefixes)
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
if (!skipRestore) {
|
|
154
|
+
let cleanedRels: string[] = []
|
|
155
|
+
let cleanedWsRels: string[] = []
|
|
156
|
+
if (outputs.length > 0) cleanedRels = await cleanOutputs(cleanArgs)
|
|
157
|
+
if (wsOutputs.length > 0) cleanedWsRels = await cleanWorkspaceOutputs(wsCleanArgs)
|
|
158
|
+
await args.cache.restoreOutputs(hash, node.projectDir, args.workspaceRoot)
|
|
159
|
+
// The directory snapshot behind the NEXT hit's skip-restore, taken at
|
|
160
|
+
// run end when the run keeps a list (as the miss path does, see
|
|
161
|
+
// miss-save.ts): the restore renamed into these directories
|
|
162
|
+
// microseconds ago, inside the snapshot's racy window, so a snapshot
|
|
163
|
+
// taken here was refused and the first warm run after every restore
|
|
164
|
+
// walked its output trees — 41 of payload's 45 tasks, 14,430 files,
|
|
165
|
+
// ~75 ms of the no-op's CPU (2026-09-11).
|
|
166
|
+
if (dirPrefixes !== null) {
|
|
167
|
+
if (args.outputDirSnapshots !== undefined) {
|
|
168
|
+
args.outputDirSnapshots.push({ hash, projectDir: node.projectDir, prefixes: dirPrefixes })
|
|
169
|
+
} else {
|
|
170
|
+
await args.cache.recordOutputDirs?.(hash, node.projectDir, dirPrefixes)
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
// Restored outputs changed the project's tree — but on this
|
|
174
|
+
// path we know the EXACT changed paths (wiped declared
|
|
175
|
+
// outputs + the artifact's files). Record them instead of
|
|
176
|
+
// dropping the snapshot; downstream same-project tasks only
|
|
177
|
+
// re-spawn git when their input globs can actually see one of
|
|
178
|
+
// these paths. The cache-miss save path keeps the
|
|
179
|
+
// unconditional drop (an executed task may write undeclared
|
|
180
|
+
// files only git can see).
|
|
181
|
+
if (outputs.length > 0) {
|
|
182
|
+
args.gitFilesCache?.markOutputsChanged(node.projectDir, [
|
|
183
|
+
...cleanedRels,
|
|
184
|
+
...hit.outputFiles.filter((p) => !p.startsWith(WORKSPACE_OUTPUT_PREFIX)),
|
|
185
|
+
])
|
|
186
|
+
}
|
|
187
|
+
if (wsOutputs.length > 0) {
|
|
188
|
+
args.gitFilesCache?.markWorkspaceOutputsChanged(args.workspaceRoot, [
|
|
189
|
+
...cleanedWsRels,
|
|
190
|
+
...hit.outputFiles
|
|
191
|
+
.filter((p) => p.startsWith(WORKSPACE_OUTPUT_PREFIX))
|
|
192
|
+
.map((p) => p.slice(WORKSPACE_OUTPUT_PREFIX.length)),
|
|
193
|
+
])
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
if (hit.stdout) log.taskStdout(node, hit.stdout)
|
|
197
|
+
const status =
|
|
198
|
+
hit.exitCode !== 0 ? 'failed' : hit.source === 'remote' ? 'cache-hit-remote' : 'cache-hit'
|
|
199
|
+
// `restored` distinguishes "we just wrote files to disk" from
|
|
200
|
+
// "disk already matched the cached snapshot". Drives the
|
|
201
|
+
// "up-to-date" vs "local-cache" / "remote-cache" label in the
|
|
202
|
+
// framed block. Only meaningful when at least one output was
|
|
203
|
+
// declared — no-outputs tasks never materialize anything, so
|
|
204
|
+
// they're vacuously up-to-date.
|
|
205
|
+
const restored = !skipRestore && anyOutputs
|
|
206
|
+
return {
|
|
207
|
+
node,
|
|
208
|
+
status,
|
|
209
|
+
exitCode: hit.exitCode,
|
|
210
|
+
// What THIS run spent: probe + restore. NOT the stored exec time — see
|
|
211
|
+
// `storedDurationMs`, which is what the hit skipped.
|
|
212
|
+
durationMs: Math.round(performance.now() - cacheOpStart),
|
|
213
|
+
storedDurationMs: hit.durationMs,
|
|
214
|
+
...(hit.cpuMs !== undefined ? { storedCpuMs: hit.cpuMs } : {}),
|
|
215
|
+
...(hit.peakRssBytes !== undefined ? { storedPeakRssBytes: hit.peakRssBytes } : {}),
|
|
216
|
+
hash,
|
|
217
|
+
restored,
|
|
218
|
+
wallclockStartNs: taskStartNs,
|
|
219
|
+
wallclockEndNs: process.hrtime.bigint() - args.runStartHrTimeNs,
|
|
220
|
+
}
|
|
221
|
+
}
|
|
@@ -3,6 +3,13 @@
|
|
|
3
3
|
|
|
4
4
|
export { run, planRun } from './run.js'
|
|
5
5
|
export { prepareRun, type PreparedRun } from './prepare.js'
|
|
6
|
+
export {
|
|
7
|
+
loadProjects,
|
|
8
|
+
loadResolvedProjects,
|
|
9
|
+
loadWorkspacePlugins,
|
|
10
|
+
type LoadedProjects,
|
|
11
|
+
type LoadProjectsArgs,
|
|
12
|
+
} from './projects.js'
|
|
6
13
|
export { computeTaskHash, createHashCache, type HashCache } from './task-hash.js'
|
|
7
14
|
export type { RunOptions, RunSummary } from './options.js'
|
|
8
15
|
// Re-surface the cache policy contract (defined in the cache module) so
|
|
@@ -19,6 +26,8 @@ export {
|
|
|
19
26
|
toWireEvent,
|
|
20
27
|
projectNode,
|
|
21
28
|
projectOutcome,
|
|
29
|
+
outcomeWord,
|
|
30
|
+
outcomeLabel,
|
|
22
31
|
} from './events.js'
|
|
23
32
|
export type {
|
|
24
33
|
EventBus,
|
|
@@ -36,6 +45,14 @@ export {
|
|
|
36
45
|
LocalHistoryProvider,
|
|
37
46
|
type TaskHistory,
|
|
38
47
|
} from './history.js'
|
|
48
|
+
export {
|
|
49
|
+
detectFlaky,
|
|
50
|
+
type FailureMode,
|
|
51
|
+
type FlakyCandidate,
|
|
52
|
+
type FlakyFinding,
|
|
53
|
+
type FlakyTask,
|
|
54
|
+
flakyTasks,
|
|
55
|
+
} from './failure-mode.js'
|
|
39
56
|
export {
|
|
40
57
|
installPlugins,
|
|
41
58
|
type CacheContext,
|
|
@@ -46,6 +63,7 @@ export {
|
|
|
46
63
|
type PluginCommand,
|
|
47
64
|
type ProjectHookContext,
|
|
48
65
|
type ScheduleHookContext,
|
|
66
|
+
type AdmitContext,
|
|
49
67
|
type WorkspaceHookContext,
|
|
50
68
|
type InstallPluginsArgs,
|
|
51
69
|
type Plugin,
|
|
@@ -53,7 +71,13 @@ export {
|
|
|
53
71
|
type PluginHookHandlers,
|
|
54
72
|
type PluginHookName,
|
|
55
73
|
type PluginSetupContext,
|
|
74
|
+
type FingerprintChange,
|
|
75
|
+
type FingerprintClaim,
|
|
76
|
+
type FingerprintContext,
|
|
56
77
|
type VxPlugin,
|
|
78
|
+
definePlugin,
|
|
79
|
+
type PluginHooks,
|
|
80
|
+
type PluginOrigin,
|
|
57
81
|
} from './plugin.js'
|
|
58
82
|
export {
|
|
59
83
|
applyConfigHooks,
|
|
@@ -61,11 +85,20 @@ export {
|
|
|
61
85
|
applyKeyHooks,
|
|
62
86
|
applyProjectHooks,
|
|
63
87
|
applyScheduleHooks,
|
|
88
|
+
claimedAffected,
|
|
89
|
+
fingerprintClaims,
|
|
64
90
|
hasHook,
|
|
91
|
+
CACHE_LAYER_METHODS,
|
|
65
92
|
resolveCache,
|
|
66
93
|
resolveExecutors,
|
|
67
94
|
} from './plugin-host.js'
|
|
68
|
-
export {
|
|
95
|
+
export {
|
|
96
|
+
lockfileClaim,
|
|
97
|
+
reachDigests,
|
|
98
|
+
type LockfileClaimHooks,
|
|
99
|
+
type LockfileClaimOptions,
|
|
100
|
+
type ReachGraph,
|
|
101
|
+
} from './lockfile-claim.js'
|
|
69
102
|
export { subscribeTelemetry, type TelemetryHandle } from './telemetry-host.js'
|
|
70
103
|
// The bounded log-capture buffer every telemetry sink shares — see the
|
|
71
104
|
// module header for why one implementation, not one per sink.
|
|
@@ -90,9 +123,6 @@ export type {
|
|
|
90
123
|
TelemetrySink,
|
|
91
124
|
TelemetrySource,
|
|
92
125
|
} from './telemetry.js'
|
|
93
|
-
// `diffOutputTrees` is the ONE tree-diff implementation — the verify verdict
|
|
94
|
-
// and a serve's cross-machine fingerprint diff must never drift on it.
|
|
95
|
-
export { diffOutputTrees } from './verify.js'
|
|
96
126
|
export { deriveStableKeys } from './stable-keys.js'
|
|
97
127
|
export type { DeriveStableKeysArgs, StableKey } from './stable-keys.js'
|
|
98
128
|
export {
|
|
@@ -125,3 +155,4 @@ export type {
|
|
|
125
155
|
RunSummaryRow,
|
|
126
156
|
WhyDidThisRerun,
|
|
127
157
|
} from './metrics.js'
|
|
158
|
+
export { collectInfo, type CollectInfoOptions, type InfoFacts } from './doctor.js'
|
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
// The claimant's shell: what every lockfile plugin needs around its
|
|
2
|
+
// parser. A plugin that keys each project on its own dependency closure
|
|
3
|
+
// from a lockfile (`@vzn/vx-lockfile`) claims the file
|
|
4
|
+
// (`VxPlugin.fingerprint`), folds one digest per project through `key`,
|
|
5
|
+
// and answers `--affected` by digesting both sides of a change. Only the
|
|
6
|
+
// parser differs per package manager; the memo, the per-run gate, the
|
|
7
|
+
// fallback for a project the file does not list and the `--affected`
|
|
8
|
+
// diff are one implementation here, so a third lockfile is a parser.
|
|
9
|
+
//
|
|
10
|
+
// The digests are memoised on disk under the cache dir by the file's
|
|
11
|
+
// xxh3, so a warm run pays one read + hash + a small JSON read, never a
|
|
12
|
+
// parse; within one process (a run, a watch cycle) the file's size and
|
|
13
|
+
// mtime gate even the read; and per run the read happens once (a WeakMap
|
|
14
|
+
// on the context core hands every `key` call of a run).
|
|
15
|
+
|
|
16
|
+
import path from 'node:path'
|
|
17
|
+
import { rename, stat } from 'node:fs/promises'
|
|
18
|
+
import { xxh3hex } from '../util/index.js'
|
|
19
|
+
import type { TaskNode } from '../graph/index.js'
|
|
20
|
+
import type {
|
|
21
|
+
FingerprintChange,
|
|
22
|
+
FingerprintClaim,
|
|
23
|
+
FingerprintContext,
|
|
24
|
+
KeyHookContext,
|
|
25
|
+
} from './plugin.js'
|
|
26
|
+
|
|
27
|
+
export interface LockfileClaimOptions {
|
|
28
|
+
/** The root-relative lockfile name, one of `WORKSPACE_FINGERPRINT_FILES`. */
|
|
29
|
+
readonly file: string
|
|
30
|
+
/**
|
|
31
|
+
* Every project's digest from the file's text: importer directory
|
|
32
|
+
* (root-relative POSIX, `.` for the root) → digest. Called once per
|
|
33
|
+
* distinct content; must be deterministic and throw on a file it
|
|
34
|
+
* cannot read (the plugin's name heads the error).
|
|
35
|
+
*/
|
|
36
|
+
readonly digest: (text: string) => ReadonlyMap<string, string>
|
|
37
|
+
/** Folded into the memo's identity: bump when `digest` folds differently. */
|
|
38
|
+
readonly version: number
|
|
39
|
+
/**
|
|
40
|
+
* `'project'` (default): each task folds its own project's digest.
|
|
41
|
+
* `'workspace'`: the whole file's hash into every task — the coarse key
|
|
42
|
+
* core would fold, through the plugin.
|
|
43
|
+
*/
|
|
44
|
+
readonly scope?: 'project' | 'workspace'
|
|
45
|
+
/**
|
|
46
|
+
* The key part's name, as `vx why` shows it under the plugin's name
|
|
47
|
+
* (`@vzn/vx-lockfile/pnpm`). Default `'deps'`.
|
|
48
|
+
*/
|
|
49
|
+
readonly part?: string
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** The two hooks a lockfile plugin spreads into `definePlugin`. */
|
|
53
|
+
export interface LockfileClaimHooks {
|
|
54
|
+
readonly fingerprint: FingerprintClaim
|
|
55
|
+
key(task: TaskNode, ctx: KeyHookContext): Promise<Readonly<Record<string, string>> | undefined>
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
interface Digests {
|
|
59
|
+
/** xxh3 of the lockfile bytes; '' when the file is absent. */
|
|
60
|
+
readonly lock: string
|
|
61
|
+
readonly importers: ReadonlyMap<string, string>
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
interface Memo {
|
|
65
|
+
version: number
|
|
66
|
+
lock: string
|
|
67
|
+
importers: Record<string, string>
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function lockfileClaim(options: LockfileClaimOptions): LockfileClaimHooks {
|
|
71
|
+
const { file, digest, version } = options
|
|
72
|
+
const scope = options.scope ?? 'project'
|
|
73
|
+
const part = options.part ?? 'deps'
|
|
74
|
+
if (scope !== 'project' && scope !== 'workspace') {
|
|
75
|
+
throw new Error(`scope must be 'project' or 'workspace', not ${JSON.stringify(scope)}`)
|
|
76
|
+
}
|
|
77
|
+
// Per workspace root: what the last read saw. `size` + `mtimeMs` gate a
|
|
78
|
+
// re-read; the content hash decides whether the digests are current.
|
|
79
|
+
const seen = new Map<string, { size: number; mtimeMs: number; digests: Digests }>()
|
|
80
|
+
// Per run: every `key` call of one run gets the same context object.
|
|
81
|
+
const perRun = new WeakMap<object, Promise<Digests>>()
|
|
82
|
+
|
|
83
|
+
const load = async (workspaceRoot: string, cacheDir: string): Promise<Digests> => {
|
|
84
|
+
const full = path.join(workspaceRoot, file)
|
|
85
|
+
let st: { size: number; mtimeMs: number }
|
|
86
|
+
try {
|
|
87
|
+
st = await stat(full)
|
|
88
|
+
} catch {
|
|
89
|
+
const digests: Digests = { lock: '', importers: new Map() }
|
|
90
|
+
seen.set(workspaceRoot, { size: -1, mtimeMs: -1, digests })
|
|
91
|
+
return digests
|
|
92
|
+
}
|
|
93
|
+
const last = seen.get(workspaceRoot)
|
|
94
|
+
if (last !== undefined && last.size === st.size && last.mtimeMs === st.mtimeMs) {
|
|
95
|
+
return last.digests
|
|
96
|
+
}
|
|
97
|
+
const bytes = await Bun.file(full).bytes()
|
|
98
|
+
const lock = xxh3hex(bytes)
|
|
99
|
+
if (last !== undefined && last.digests.lock === lock) {
|
|
100
|
+
seen.set(workspaceRoot, { size: st.size, mtimeMs: st.mtimeMs, digests: last.digests })
|
|
101
|
+
return last.digests
|
|
102
|
+
}
|
|
103
|
+
const memoFile = path.join(cacheDir, 'lockfile-claims', `${file}.json`)
|
|
104
|
+
const importers =
|
|
105
|
+
scope === 'workspace'
|
|
106
|
+
? new Map<string, string>()
|
|
107
|
+
: ((await readMemo(memoFile, version, lock)) ??
|
|
108
|
+
(await computeAndMemo(memoFile, version, lock, digest(decode(bytes)))))
|
|
109
|
+
const digests: Digests = { lock, importers }
|
|
110
|
+
seen.set(workspaceRoot, { size: st.size, mtimeMs: st.mtimeMs, digests })
|
|
111
|
+
return digests
|
|
112
|
+
}
|
|
113
|
+
const loadOnce = (ctx: KeyHookContext): Promise<Digests> => {
|
|
114
|
+
let p = perRun.get(ctx)
|
|
115
|
+
if (p === undefined) {
|
|
116
|
+
p = load(ctx.workspaceRoot, ctx.cacheDir)
|
|
117
|
+
perRun.set(ctx, p)
|
|
118
|
+
}
|
|
119
|
+
return p
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
return {
|
|
123
|
+
fingerprint: {
|
|
124
|
+
files: [file],
|
|
125
|
+
affected(change: FingerprintChange, ctx: FingerprintContext) {
|
|
126
|
+
// One side absent: the file appeared or went — every project's
|
|
127
|
+
// installed tree is in question, and core's answer (all) is right.
|
|
128
|
+
if (change.before === null || change.after === null) return undefined
|
|
129
|
+
if (scope === 'workspace') return undefined
|
|
130
|
+
const before = digest(decode(change.before))
|
|
131
|
+
const after = digest(decode(change.after))
|
|
132
|
+
const names: string[] = []
|
|
133
|
+
for (const p of ctx.projects) {
|
|
134
|
+
const importer = importerOf(ctx.workspaceRoot, p.dir)
|
|
135
|
+
if (digestFor(before, importer) !== digestFor(after, importer)) names.push(p.name)
|
|
136
|
+
}
|
|
137
|
+
return names
|
|
138
|
+
},
|
|
139
|
+
},
|
|
140
|
+
async key(task, ctx) {
|
|
141
|
+
const digests = await loadOnce(ctx)
|
|
142
|
+
if (digests.lock === '') return undefined
|
|
143
|
+
if (scope === 'workspace') return { [part]: digests.lock }
|
|
144
|
+
return {
|
|
145
|
+
[part]: digestFor(digests.importers, importerOf(ctx.workspaceRoot, task.projectDir)),
|
|
146
|
+
}
|
|
147
|
+
},
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* A project's digest: its own importer's, or — for a project the lockfile
|
|
153
|
+
* has no importer for (outside the workspace's `packages`) — the root
|
|
154
|
+
* importer's, the only installed tree it can resolve from. A file with
|
|
155
|
+
* neither folds a constant.
|
|
156
|
+
*/
|
|
157
|
+
function digestFor(importers: ReadonlyMap<string, string>, importer: string): string {
|
|
158
|
+
return importers.get(importer) ?? importers.get('.') ?? 'no-importer'
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/** The lockfile's importer path for a project directory: `.` for the root, POSIX otherwise. */
|
|
162
|
+
function importerOf(workspaceRoot: string, projectDir: string): string {
|
|
163
|
+
const rel = path.relative(workspaceRoot, projectDir)
|
|
164
|
+
return rel === '' ? '.' : rel.split(path.sep).join('/')
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
async function readMemo(
|
|
168
|
+
memoFile: string,
|
|
169
|
+
version: number,
|
|
170
|
+
lock: string,
|
|
171
|
+
): Promise<Map<string, string> | undefined> {
|
|
172
|
+
try {
|
|
173
|
+
const memo = (await Bun.file(memoFile).json()) as Memo
|
|
174
|
+
if (memo.version !== version || memo.lock !== lock) return undefined
|
|
175
|
+
return new Map(Object.entries(memo.importers))
|
|
176
|
+
} catch {
|
|
177
|
+
return undefined
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
async function computeAndMemo(
|
|
182
|
+
memoFile: string,
|
|
183
|
+
version: number,
|
|
184
|
+
lock: string,
|
|
185
|
+
importers: ReadonlyMap<string, string>,
|
|
186
|
+
): Promise<ReadonlyMap<string, string>> {
|
|
187
|
+
const memo: Memo = { version, lock, importers: Object.fromEntries(importers) }
|
|
188
|
+
// Write-then-rename: a reader never sees a half-written memo, and two
|
|
189
|
+
// concurrent runs each land a whole one.
|
|
190
|
+
const tmp = `${memoFile}.tmp-${process.pid}-${Date.now()}`
|
|
191
|
+
try {
|
|
192
|
+
await Bun.write(tmp, JSON.stringify(memo))
|
|
193
|
+
await rename(tmp, memoFile)
|
|
194
|
+
} catch {
|
|
195
|
+
// The memo is a speed-up, never a requirement: the digests are
|
|
196
|
+
// computed either way, and the next run computes them again.
|
|
197
|
+
}
|
|
198
|
+
return importers
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
function decode(bytes: Uint8Array): string {
|
|
202
|
+
return new TextDecoder().decode(bytes)
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// --- Reach digests -----------------------------------------------------------
|
|
206
|
+
|
|
207
|
+
/** A dependency graph: one material string per node and its out-edges by index. */
|
|
208
|
+
export interface ReachGraph {
|
|
209
|
+
readonly material: readonly string[]
|
|
210
|
+
readonly edges: ReadonlyArray<readonly number[]>
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* One digest per node over everything the node reaches, Merkle-style: a
|
|
215
|
+
* change anywhere in a node's reach moves its digest, a change elsewhere
|
|
216
|
+
* does not. Lockfiles carry dependency cycles, so the unit is the
|
|
217
|
+
* strongly connected component: Tarjan's walk (iterative — a dependency
|
|
218
|
+
* chain can be thousands deep) emits components children-first, and each
|
|
219
|
+
* folds its members' material (sorted) and its child components' digests
|
|
220
|
+
* (sorted). O(nodes + edges): 1000 importers over 3000 packages digest in
|
|
221
|
+
* ~20 ms where one traversal per importer took 400.
|
|
222
|
+
*/
|
|
223
|
+
export function reachDigests(g: ReachGraph): string[] {
|
|
224
|
+
const n = g.material.length
|
|
225
|
+
const idx = new Int32Array(n).fill(-1)
|
|
226
|
+
const low = new Int32Array(n)
|
|
227
|
+
const onStack = new Uint8Array(n)
|
|
228
|
+
const stack: number[] = []
|
|
229
|
+
const comp = new Int32Array(n).fill(-1)
|
|
230
|
+
const compHash: string[] = []
|
|
231
|
+
let counter = 0
|
|
232
|
+
for (let root = 0; root < n; root++) {
|
|
233
|
+
if (idx[root] !== -1) continue
|
|
234
|
+
const frames: Array<[number, number]> = [[root, 0]]
|
|
235
|
+
idx[root] = low[root] = counter++
|
|
236
|
+
stack.push(root)
|
|
237
|
+
onStack[root] = 1
|
|
238
|
+
while (frames.length > 0) {
|
|
239
|
+
const frame = frames[frames.length - 1]!
|
|
240
|
+
const v = frame[0]
|
|
241
|
+
const out = g.edges[v]!
|
|
242
|
+
if (frame[1] < out.length) {
|
|
243
|
+
const w = out[frame[1]++]!
|
|
244
|
+
if (idx[w] === -1) {
|
|
245
|
+
idx[w] = low[w] = counter++
|
|
246
|
+
stack.push(w)
|
|
247
|
+
onStack[w] = 1
|
|
248
|
+
frames.push([w, 0])
|
|
249
|
+
} else if (onStack[w] === 1) {
|
|
250
|
+
low[v] = Math.min(low[v]!, idx[w]!)
|
|
251
|
+
}
|
|
252
|
+
continue
|
|
253
|
+
}
|
|
254
|
+
frames.pop()
|
|
255
|
+
if (frames.length > 0) {
|
|
256
|
+
const u = frames[frames.length - 1]![0]
|
|
257
|
+
low[u] = Math.min(low[u]!, low[v]!)
|
|
258
|
+
}
|
|
259
|
+
if (low[v] !== idx[v]) continue
|
|
260
|
+
const members: number[] = []
|
|
261
|
+
for (;;) {
|
|
262
|
+
const w = stack.pop()!
|
|
263
|
+
onStack[w] = 0
|
|
264
|
+
comp[w] = compHash.length
|
|
265
|
+
members.push(w)
|
|
266
|
+
if (w === v) break
|
|
267
|
+
}
|
|
268
|
+
const id = compHash.length
|
|
269
|
+
const children = new Set<string>()
|
|
270
|
+
for (const m of members) {
|
|
271
|
+
for (const w of g.edges[m]!) if (comp[w] !== id) children.add(compHash[comp[w]!]!)
|
|
272
|
+
}
|
|
273
|
+
let h = Bun.hash.xxHash3(`members:${members.length}`)
|
|
274
|
+
for (const m of members.map((m) => g.material[m]!).sort()) h = Bun.hash.xxHash3(`${m}\n`, h)
|
|
275
|
+
h = Bun.hash.xxHash3(`children:${children.size}`, h)
|
|
276
|
+
for (const c of [...children].sort()) h = Bun.hash.xxHash3(`${c}\n`, h)
|
|
277
|
+
compHash.push(h.toString(16).padStart(16, '0'))
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
return Array.from({ length: n }, (_, i) => compHash[comp[i]!]!)
|
|
281
|
+
}
|