@vzn/vx 0.0.16 → 0.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -89
- package/index.ts +10 -0
- package/launcher.mjs +10 -11
- package/package.json +19 -5
- package/plugins/local-cache/index.ts +2 -0
- package/plugins/local-executor/index.ts +2 -0
- package/plugins/schedule-history/index.ts +2 -0
- package/src/bin.ts +2 -2
- package/src/cache/archive.ts +557 -0
- package/src/cache/cache.ts +929 -250
- package/src/cache/cas-backend.ts +20 -2
- package/src/cache/chained-cache.ts +202 -0
- package/src/cache/digest.ts +0 -15
- package/src/cache/index.ts +13 -3
- package/src/cache/inputs.ts +445 -41
- package/src/cache/layered-cache.ts +88 -30
- package/src/cache/tar-stream.ts +358 -0
- package/src/cli/cache.ts +36 -10
- package/src/cli/help.ts +167 -128
- package/src/cli/index.ts +74 -43
- package/src/cli/info.ts +30 -0
- package/src/cli/last.ts +140 -0
- package/src/cli/migrate-emit.ts +21 -0
- package/src/cli/migrate-nx.ts +23 -0
- package/src/cli/migrate-scripts.ts +127 -0
- package/src/cli/migrate-turbo.ts +43 -6
- package/src/cli/migrate.ts +119 -33
- package/src/cli/plan-format.ts +58 -3
- package/src/cli/plugin-commands.ts +96 -0
- package/src/cli/prune.ts +291 -0
- package/src/cli/run.ts +272 -80
- package/src/cli/upgrade.ts +7 -6
- package/src/cli/watch.ts +259 -61
- package/src/cli/why.ts +193 -0
- package/src/config.ts +40 -23
- package/src/exec/env.ts +7 -1
- package/src/exec/executor.ts +239 -0
- package/src/exec/index.ts +19 -1
- package/src/exec/runner.ts +116 -29
- package/src/exec/sandbox-runtime.ts +194 -24
- package/src/graph/index.ts +3 -1
- package/src/graph/scheduler.ts +92 -18
- package/src/graph/task-graph.ts +220 -6
- package/src/index.ts +82 -84
- package/src/orchestrator/deferred-outputs.ts +144 -0
- package/src/orchestrator/download-policy.ts +171 -0
- package/src/orchestrator/events.ts +30 -0
- package/src/orchestrator/execute-task.ts +328 -94
- package/src/orchestrator/failure-mode.ts +61 -0
- package/src/orchestrator/framed-output.ts +87 -35
- package/src/orchestrator/history.ts +78 -31
- package/src/orchestrator/index.ts +43 -90
- package/src/orchestrator/local-shortcircuit.ts +22 -0
- package/src/orchestrator/logger.ts +196 -14
- package/src/orchestrator/metrics.ts +57 -1376
- package/src/orchestrator/missing-plugin.ts +9 -0
- package/src/orchestrator/options.ts +22 -4
- package/src/orchestrator/plan.ts +144 -1
- package/src/orchestrator/plugin-host.ts +199 -119
- package/src/orchestrator/plugin.ts +144 -57
- package/src/orchestrator/prepare.ts +220 -69
- package/src/orchestrator/remote-prefetch.ts +12 -8
- package/src/orchestrator/resources.ts +20 -34
- package/src/orchestrator/run-artifacts.ts +41 -16
- package/src/orchestrator/run-context.ts +167 -29
- package/src/orchestrator/run-report.ts +73 -46
- package/src/orchestrator/run.ts +590 -122
- package/src/orchestrator/stable-keys.ts +74 -33
- package/src/orchestrator/status-line.ts +47 -8
- package/src/orchestrator/summary.ts +22 -1
- package/src/orchestrator/tally.ts +78 -35
- package/src/orchestrator/task-hash.ts +105 -11
- package/src/orchestrator/task-log-buffer.ts +291 -0
- package/src/orchestrator/telemetry-host.ts +24 -6
- package/src/orchestrator/telemetry.ts +181 -21
- package/src/orchestrator/upstream.ts +72 -31
- package/src/orchestrator/verify.ts +12 -2
- package/src/plugins/local-cache/index.ts +11 -0
- package/src/plugins/local-executor/index.ts +45 -0
- package/src/plugins/schedule-history/index.ts +109 -0
- package/src/util/edit-distance.ts +21 -0
- package/src/util/errors.ts +15 -1
- package/src/util/index.ts +7 -2
- package/src/util/num.ts +37 -0
- package/src/util/paths.ts +43 -0
- package/src/util/settle.ts +48 -0
- package/src/util/size.ts +5 -1
- package/src/util/tail.ts +66 -0
- package/src/util/timing.ts +63 -0
- package/src/workspace/affected.ts +142 -15
- package/src/workspace/config-cache.ts +304 -0
- package/src/workspace/config-eval.ts +210 -0
- package/src/workspace/config-imports.ts +161 -0
- package/src/workspace/filter.ts +22 -4
- package/src/workspace/fingerprint.ts +16 -1
- package/src/workspace/index.ts +15 -3
- package/src/workspace/lockfile.ts +13 -2
- package/src/workspace/nested-dirs.ts +21 -7
- package/src/workspace/package-graph.ts +23 -20
- package/src/workspace/project-loader.ts +503 -59
- package/src/workspace/workspace.ts +213 -72
- package/src/cache/tar.ts +0 -343
- package/src/cli/backend.ts +0 -23
- package/src/cli/mcp-rpc.ts +0 -292
- package/src/cli/mcp.ts +0 -92
- package/src/orchestrator/devframe-surface.ts +0 -74
- package/src/orchestrator/predict.ts +0 -106
- package/src/orchestrator/protocol.ts +0 -156
- package/src/orchestrator/run-state.ts +0 -145
- package/src/orchestrator/wire-render.ts +0 -92
- package/src/orchestrator/wire.ts +0 -212
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
// Stage timing for the run path, printed to stderr when `VX_TIMING` is set.
|
|
2
|
+
//
|
|
3
|
+
// The CPU profiler attributes a tight loop's cost unreliably and hides
|
|
4
|
+
// where an `await` waited; a stage table answers "where did the warm run
|
|
5
|
+
// go?" directly, and costs one boolean check per mark when off. Marks are
|
|
6
|
+
// cumulative from process start; the table shows each stage's own share.
|
|
7
|
+
|
|
8
|
+
const enabled = process.env.VX_TIMING !== undefined && process.env.VX_TIMING !== ''
|
|
9
|
+
const t0 = Bun.nanoseconds()
|
|
10
|
+
const marks: Array<[label: string, ns: number]> = []
|
|
11
|
+
|
|
12
|
+
/** Record the end of a stage. No-op unless `VX_TIMING` is set. */
|
|
13
|
+
export function mark(label: string): void {
|
|
14
|
+
if (!enabled) return
|
|
15
|
+
marks.push([label, Bun.nanoseconds() - t0])
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
const spans = new Map<string, [count: number, ns: number]>()
|
|
19
|
+
const noop = (): void => {}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Time one occurrence of a repeated operation (a per-task probe, a restore):
|
|
23
|
+
* `const end = span('probe'); …; end()`. Accumulated by label and printed
|
|
24
|
+
* under the stage table. Returns a shared no-op when off, so the hot path
|
|
25
|
+
* allocates nothing.
|
|
26
|
+
*/
|
|
27
|
+
export function span(label: string): () => void {
|
|
28
|
+
if (!enabled) return noop
|
|
29
|
+
const start = Bun.nanoseconds()
|
|
30
|
+
return () => {
|
|
31
|
+
const cur = spans.get(label)
|
|
32
|
+
const ns = Bun.nanoseconds() - start
|
|
33
|
+
if (cur === undefined) spans.set(label, [1, ns])
|
|
34
|
+
else {
|
|
35
|
+
cur[0]++
|
|
36
|
+
cur[1] += ns
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Print the stage table (once, at the end of a run). No-op unless enabled. */
|
|
42
|
+
export function printTimings(): void {
|
|
43
|
+
if (!enabled || marks.length === 0) return
|
|
44
|
+
const width = Math.max(...marks.map(([l]) => l.length))
|
|
45
|
+
let prev = 0
|
|
46
|
+
const lines = ['[vx timing] stage'.padEnd(width + 14) + ' own cumulative']
|
|
47
|
+
for (const [label, ns] of marks) {
|
|
48
|
+
const own = (ns - prev) / 1e6
|
|
49
|
+
lines.push(
|
|
50
|
+
` ${label.padEnd(width)} ${own.toFixed(1).padStart(6)}ms ${(ns / 1e6).toFixed(1).padStart(8)}ms`,
|
|
51
|
+
)
|
|
52
|
+
prev = ns
|
|
53
|
+
}
|
|
54
|
+
if (spans.size > 0) {
|
|
55
|
+
lines.push('[vx timing] accumulated total count')
|
|
56
|
+
for (const [label, [count, ns]] of [...spans].sort((a, b) => b[1][1] - a[1][1])) {
|
|
57
|
+
lines.push(
|
|
58
|
+
` ${label.padEnd(24)} ${(ns / 1e6).toFixed(1).padStart(8)}ms ${String(count).padStart(6)}`,
|
|
59
|
+
)
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
process.stderr.write(lines.join('\n') + '\n')
|
|
63
|
+
}
|
|
@@ -12,6 +12,8 @@
|
|
|
12
12
|
import path from 'node:path'
|
|
13
13
|
import { UserError } from '../util/index.js'
|
|
14
14
|
import { LOCKFILE_NAME } from './lockfile.js'
|
|
15
|
+
import { configImportOwners } from './config-imports.js'
|
|
16
|
+
import { WORKSPACE_FINGERPRINT_FILES } from './fingerprint.js'
|
|
15
17
|
import type { ProjectMeta } from './workspace.js'
|
|
16
18
|
|
|
17
19
|
export interface AffectedArgs {
|
|
@@ -19,6 +21,22 @@ export interface AffectedArgs {
|
|
|
19
21
|
/** Git ref / commit / branch to compare against. Required. */
|
|
20
22
|
since: string
|
|
21
23
|
projects: readonly ProjectMeta[]
|
|
24
|
+
/**
|
|
25
|
+
* Which projects declare a `cache.inputs.workspaceFiles` glob matching one of
|
|
26
|
+
* these workspace-relative paths.
|
|
27
|
+
*
|
|
28
|
+
* A workspace-anchored glob is the documented escape hatch for shared files
|
|
29
|
+
* that belong to NO project, so mapping changed paths to project directories
|
|
30
|
+
* structurally cannot see it — the path resolves to no project and the run
|
|
31
|
+
* selects nothing for a change that re-keyed the task.
|
|
32
|
+
*
|
|
33
|
+
* Answering needs the RESOLVED configs, which selection runs before loading;
|
|
34
|
+
* that ordering is what left this open. It is resolved by asking only when
|
|
35
|
+
* the question can matter: this is called ONLY with paths that belong to no
|
|
36
|
+
* project, so a run whose every change is inside a project — the common case
|
|
37
|
+
* — never invokes it and pays nothing.
|
|
38
|
+
*/
|
|
39
|
+
workspaceGlobOwners?: (orphanPaths: readonly string[]) => Promise<Iterable<string>>
|
|
22
40
|
}
|
|
23
41
|
|
|
24
42
|
/**
|
|
@@ -29,15 +47,113 @@ export interface AffectedArgs {
|
|
|
29
47
|
export async function affectedProjects(args: AffectedArgs): Promise<Set<string>> {
|
|
30
48
|
await verifyRef(args.workspaceRoot, args.since)
|
|
31
49
|
|
|
32
|
-
const
|
|
50
|
+
const [diffed, untracked] = await Promise.all([
|
|
33
51
|
// `--no-renames` is crucial for project-affected detection: with
|
|
34
52
|
// git's auto rename-detection on (the default in modern git), a
|
|
35
53
|
// cross-project `git mv` collapses to a single rename entry that
|
|
36
54
|
// surfaces only the destination path. We'd then miss flagging the
|
|
37
55
|
// source project as affected. Treating renames as delete+add gives
|
|
38
56
|
// us both halves so both projects get correctly marked.
|
|
39
|
-
|
|
40
|
-
cwd
|
|
57
|
+
//
|
|
58
|
+
// `--relative` prints paths relative to the cwd (the workspace root), NOT
|
|
59
|
+
// the git repo root. Without it, when the workspace root is a SUBDIR of the
|
|
60
|
+
// git repo, `git diff` emits repo-root-relative paths (`code/pkg/x`) that
|
|
61
|
+
// `path.resolve(workspaceRoot, …)` mangles into `<root>/code/…`, matching no
|
|
62
|
+
// project → the project is silently NOT flagged affected. `--relative` also
|
|
63
|
+
// (correctly) drops changes ABOVE the workspace, which belong to no project.
|
|
64
|
+
// No-op when the workspace root IS the git root.
|
|
65
|
+
//
|
|
66
|
+
// `-z` because git otherwise C-quotes and octal-escapes any path with a
|
|
67
|
+
// non-ASCII / `"` / `\` character — the quoted string then resolves to no
|
|
68
|
+
// project, while the cache-input enumeration (which uses `-z`) sees the
|
|
69
|
+
// real name and re-keys the task. The two surfaces must agree.
|
|
70
|
+
gitPaths(args.workspaceRoot, [
|
|
71
|
+
'diff',
|
|
72
|
+
'--no-renames',
|
|
73
|
+
'--relative',
|
|
74
|
+
'--name-only',
|
|
75
|
+
'-z',
|
|
76
|
+
args.since,
|
|
77
|
+
]),
|
|
78
|
+
// `git diff` never reports untracked-but-not-ignored files, but input
|
|
79
|
+
// enumeration does (`git ls-files --cached --others --exclude-standard`),
|
|
80
|
+
// so a brand-new source file changes a task's cache key. Union it in or
|
|
81
|
+
// `--affected` skips a package that genuinely has new work.
|
|
82
|
+
gitPaths(args.workspaceRoot, ['ls-files', '--others', '--exclude-standard', '-z']),
|
|
83
|
+
])
|
|
84
|
+
|
|
85
|
+
// vx-lock.json (workspace-root metadata) is excluded like a gitignored
|
|
86
|
+
// file: re-running `vx lock` must not mark every project affected.
|
|
87
|
+
const changed = [...diffed, ...untracked].filter((s) => s !== LOCKFILE_NAME)
|
|
88
|
+
|
|
89
|
+
// A lockfile or workspace-definition change re-keys EVERY task, because the
|
|
90
|
+
// workspace fingerprint folds those files into every cache key. Mapping
|
|
91
|
+
// changed paths to project directories cannot see that — they sit at the
|
|
92
|
+
// root and belong to no project — so `--affected` selected ZERO projects for
|
|
93
|
+
// a change that invalidates the entire cache. `vx run test --affected` after
|
|
94
|
+
// a `pnpm update` exited 0 having run nothing, which is precisely the
|
|
95
|
+
// failure `docs/cli.md` warns against in the sentence it states as a
|
|
96
|
+
// principle: "input hashing sees it, so `--affected` must too."
|
|
97
|
+
//
|
|
98
|
+
// Selection is not hashed, so widening it here changes no cache key.
|
|
99
|
+
if (changed.some((p) => FINGERPRINT_SET.has(p))) {
|
|
100
|
+
return new Set(args.projects.map((p) => p.name))
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const { owned, orphans } = projectsContaining(args.workspaceRoot, changed, args.projects)
|
|
104
|
+
|
|
105
|
+
// THIRD CHANNEL: a project whose `vx.config.*` IMPORTS a changed file.
|
|
106
|
+
// Resolved-config hashing folds those values into the key, so the same
|
|
107
|
+
// sentence above applies — input hashing sees it, so selection must. This
|
|
108
|
+
// runs on the FULL changed set, not just `orphans`: the common shape is a
|
|
109
|
+
// config reaching into ANOTHER project (`../../src/index.ts`), whose target
|
|
110
|
+
// is owned and therefore never an orphan at all.
|
|
111
|
+
for (const name of await configImportOwners({
|
|
112
|
+
workspaceRoot: args.workspaceRoot,
|
|
113
|
+
projects: args.projects,
|
|
114
|
+
changed,
|
|
115
|
+
skip: owned,
|
|
116
|
+
})) {
|
|
117
|
+
owned.add(name)
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
if (orphans.length === 0 || args.workspaceGlobOwners === undefined) return owned
|
|
121
|
+
for (const name of await args.workspaceGlobOwners(orphans)) owned.add(name)
|
|
122
|
+
return owned
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Does any `cache.inputs.workspaceFiles` entry in `globs` match `rel`?
|
|
127
|
+
*
|
|
128
|
+
* Mirrors `resolveWorkspaceFiles`' partition: a leading `!` is an EXCLUDE, and
|
|
129
|
+
* with no positive glob nothing matches. Paths are workspace-root-relative and
|
|
130
|
+
* POSIX-separated, which is the form both git enumeration and the glob
|
|
131
|
+
* resolver already speak.
|
|
132
|
+
*/
|
|
133
|
+
export function workspaceGlobsMatch(globs: readonly string[], rel: string): boolean {
|
|
134
|
+
const positive: Bun.Glob[] = []
|
|
135
|
+
const negative: Bun.Glob[] = []
|
|
136
|
+
for (const entry of globs) {
|
|
137
|
+
if (entry.startsWith('!')) negative.push(new Bun.Glob(entry.slice(1)))
|
|
138
|
+
else positive.push(new Bun.Glob(entry))
|
|
139
|
+
}
|
|
140
|
+
if (positive.length === 0) return false
|
|
141
|
+
if (!positive.some((g) => g.match(rel))) return false
|
|
142
|
+
return !negative.some((g) => g.match(rel))
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* The fingerprint files as workspace-root-relative paths. Read from the same
|
|
147
|
+
* constant the fingerprint itself walks, so a new lockfile format cannot be
|
|
148
|
+
* taught to one surface and not the other.
|
|
149
|
+
*/
|
|
150
|
+
const FINGERPRINT_SET: ReadonlySet<string> = new Set(WORKSPACE_FINGERPRINT_FILES)
|
|
151
|
+
|
|
152
|
+
/** Run a NUL-separated path-listing git command from the workspace root. */
|
|
153
|
+
async function gitPaths(workspaceRoot: string, cmd: string[]): Promise<string[]> {
|
|
154
|
+
const proc = Bun.spawn({
|
|
155
|
+
cmd: ['git', ...cmd],
|
|
156
|
+
cwd: workspaceRoot,
|
|
41
157
|
stdout: 'pipe',
|
|
42
158
|
stderr: 'pipe',
|
|
43
159
|
})
|
|
@@ -45,13 +161,9 @@ export async function affectedProjects(args: AffectedArgs): Promise<Set<string>>
|
|
|
45
161
|
const stderr = await new Response(proc.stderr).text()
|
|
46
162
|
const exit = await proc.exited
|
|
47
163
|
if (exit !== 0) {
|
|
48
|
-
throw new UserError(`git
|
|
164
|
+
throw new UserError(`git ${cmd[0]} failed (exit ${exit}): ${stderr.trim()}`)
|
|
49
165
|
}
|
|
50
|
-
|
|
51
|
-
// vx-lock.json (workspace-root metadata) is excluded like a gitignored
|
|
52
|
-
// file: re-running `vx lock` must not mark every project affected.
|
|
53
|
-
const changed = stdout.split('\n').filter((s) => s.length > 0 && s !== LOCKFILE_NAME)
|
|
54
|
-
return projectsContaining(args.workspaceRoot, changed, args.projects)
|
|
166
|
+
return stdout.split('\0').filter((s) => s.length > 0)
|
|
55
167
|
}
|
|
56
168
|
|
|
57
169
|
/**
|
|
@@ -79,18 +191,27 @@ async function verifyRef(workspaceRoot: string, ref: string): Promise<void> {
|
|
|
79
191
|
stdout: 'pipe',
|
|
80
192
|
stderr: 'pipe',
|
|
81
193
|
})
|
|
82
|
-
if (proc.exitCode
|
|
194
|
+
if (proc.exitCode === 0) return
|
|
195
|
+
// `--verify --quiet` exits 1 for "that ref does not exist" and 128 for
|
|
196
|
+
// "git could not run here at all" (not a repository, corrupt objects,
|
|
197
|
+
// permissions). Reporting the second as a ref problem sends you hunting
|
|
198
|
+
// for a branch name when the real fault is the repository — so only exit
|
|
199
|
+
// 1 gets the ref message; anything else surfaces what git actually said.
|
|
200
|
+
const stderr = new TextDecoder().decode(proc.stderr).trim()
|
|
201
|
+
if (proc.exitCode !== 1) {
|
|
83
202
|
throw new UserError(
|
|
84
|
-
`git
|
|
203
|
+
`git rev-parse failed (exit ${proc.exitCode}) in ${workspaceRoot}` +
|
|
204
|
+
(stderr.length > 0 ? `: ${stderr}` : ''),
|
|
85
205
|
)
|
|
86
206
|
}
|
|
207
|
+
throw new UserError(`git ref "${ref}" did not resolve. Pass a branch or commit you have locally.`)
|
|
87
208
|
}
|
|
88
209
|
|
|
89
210
|
function projectsContaining(
|
|
90
211
|
workspaceRoot: string,
|
|
91
212
|
changedRelPaths: readonly string[],
|
|
92
213
|
projects: readonly ProjectMeta[],
|
|
93
|
-
): Set<string
|
|
214
|
+
): { owned: Set<string>; orphans: string[] } {
|
|
94
215
|
// Index projects by their (canonical) dir, then for each changed path walk
|
|
95
216
|
// its ancestor dirs bottom-up until one is a project dir. The FIRST hit is
|
|
96
217
|
// the DEEPEST containing project — so a nested project still wins over its
|
|
@@ -100,19 +221,25 @@ function projectsContaining(
|
|
|
100
221
|
// pays for.
|
|
101
222
|
const dirToName = new Map<string, string>()
|
|
102
223
|
for (const p of projects) dirToName.set(p.dir, p.name)
|
|
103
|
-
const
|
|
224
|
+
const owned = new Set<string>()
|
|
225
|
+
// Paths that belong to no project. Only these can reach a task through a
|
|
226
|
+
// workspace-anchored glob, so they are the exact set worth asking about.
|
|
227
|
+
const orphans: string[] = []
|
|
104
228
|
for (const rel of changedRelPaths) {
|
|
105
229
|
let dir = path.resolve(workspaceRoot, rel)
|
|
230
|
+
let hit = false
|
|
106
231
|
for (;;) {
|
|
107
232
|
const name = dirToName.get(dir)
|
|
108
233
|
if (name !== undefined) {
|
|
109
|
-
|
|
234
|
+
owned.add(name)
|
|
235
|
+
hit = true
|
|
110
236
|
break
|
|
111
237
|
}
|
|
112
238
|
const parent = path.dirname(dir)
|
|
113
239
|
if (parent === dir) break // reached the filesystem root
|
|
114
240
|
dir = parent
|
|
115
241
|
}
|
|
242
|
+
if (!hit) orphans.push(rel)
|
|
116
243
|
}
|
|
117
|
-
return
|
|
244
|
+
return { owned, orphans }
|
|
118
245
|
}
|
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
// Config evaluation cache — skip re-evaluating a `vx.config` that cannot
|
|
2
|
+
// have changed.
|
|
3
|
+
//
|
|
4
|
+
// A config is a program, and evaluating a thousand of them is the largest
|
|
5
|
+
// fixed cost of a warm run (measured 2026-09-02: ~80 ms to import 1000
|
|
6
|
+
// synthetic configs, ~12 ms to read the same files as data). The cache
|
|
7
|
+
// stores the VALIDATED, JSON-serialised result keyed by everything the
|
|
8
|
+
// evaluation could have observed:
|
|
9
|
+
//
|
|
10
|
+
// - the config's bytes and the bytes of every file it transitively imports
|
|
11
|
+
// by RELATIVE specifier (the closure), paths included;
|
|
12
|
+
// - the workspace fingerprint (lockfiles), which covers package imports;
|
|
13
|
+
// - Bun's version, vx's version and this module's own version. A stored
|
|
14
|
+
// evaluation is served WITHOUT re-validation, so it must never outlive
|
|
15
|
+
// the validator that accepted it — a vx upgrade that tightens a rule
|
|
16
|
+
// re-evaluates every config once.
|
|
17
|
+
//
|
|
18
|
+
// It applies only to configs that are PROVABLY pure by a conservative static
|
|
19
|
+
// check: every import is relative or `@vzn/vx` (whose `defineProject` /
|
|
20
|
+
// `defineWorkspace` are identity functions), and no file in the closure
|
|
21
|
+
// mentions a global through which the environment can leak — `process`,
|
|
22
|
+
// `Bun`, `Date`, `fetch`, `import.meta`, `require`, a dynamic `import()`,
|
|
23
|
+
// `await`, … Anything else evaluates live, exactly as before. The check
|
|
24
|
+
// fails SAFE, never fast: a false negative costs one evaluation, a false
|
|
25
|
+
// positive would cost a stale key, so the deny-list is deliberately wide.
|
|
26
|
+
//
|
|
27
|
+
// JSON is already the contract for a config object — `hashTaskConfig` and
|
|
28
|
+
// `vx lock` both go through `JSON.stringify` — so a cached config derives
|
|
29
|
+
// the same cache key as a live evaluation of the same bytes.
|
|
30
|
+
|
|
31
|
+
import { realpathSync } from 'node:fs'
|
|
32
|
+
import path from 'node:path'
|
|
33
|
+
import { xxh3 } from '../util/index.js'
|
|
34
|
+
import { VERSION } from '../version.js'
|
|
35
|
+
|
|
36
|
+
/** Bump when the key derivation or the stored shape changes. */
|
|
37
|
+
export const CONFIG_EVAL_VERSION = 2
|
|
38
|
+
|
|
39
|
+
/** Where cached evaluations live; `Cache` implements it over `cache.db`. */
|
|
40
|
+
export interface ConfigEvalStore {
|
|
41
|
+
/**
|
|
42
|
+
* The warm fast path (all optional; a store without them keys by bytes).
|
|
43
|
+
* `hashFile` is the file's git blob id behind an mtime/size/ctime/inode
|
|
44
|
+
* memo — no read when unchanged; `getConfigClosures` / `putConfigClosure`
|
|
45
|
+
* keep each config's ORDERED closure (the config first, then every
|
|
46
|
+
* relative import in discovery order), so a warm load keys the config by
|
|
47
|
+
* stat-hashing that list instead of reading and scanning every file
|
|
48
|
+
* (1,000 configs: 5 ms against 15, measured 2026-09-03). Sound because
|
|
49
|
+
* closure membership can only change by editing a listed file, which
|
|
50
|
+
* changes that file's hash and so the key; the one exception, an
|
|
51
|
+
* extensionless relative import whose resolution a new file could
|
|
52
|
+
* shadow, is never indexed.
|
|
53
|
+
*/
|
|
54
|
+
hashFile?(file: string): Promise<string>
|
|
55
|
+
getConfigClosures?(configPaths: readonly string[]): Map<string, string[]>
|
|
56
|
+
putConfigClosure?(configPath: string, files: readonly string[]): void
|
|
57
|
+
getConfigEval(key: string): string | null
|
|
58
|
+
/**
|
|
59
|
+
* Many keys in one round-trip (optional; a store without it is asked per
|
|
60
|
+
* key). A warm 1000-project run paid 3.6 ms for 1,000 point lookups where
|
|
61
|
+
* one `IN` query costs 0.7 (measured 2026-09-03).
|
|
62
|
+
*/
|
|
63
|
+
getConfigEvals?(keys: readonly string[]): Map<string, string>
|
|
64
|
+
putConfigEval(key: string, json: string): void
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** What `configEvalKey` learned besides the key, for the store's closure index. */
|
|
68
|
+
export interface ConfigEvalKeyResult {
|
|
69
|
+
key: string
|
|
70
|
+
/** The config first, then every relative import in discovery order. */
|
|
71
|
+
closure: string[]
|
|
72
|
+
/** False when a relative import is extensionless — a new file could change its resolution. */
|
|
73
|
+
indexable: boolean
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export interface ConfigEvalKeyArgs {
|
|
77
|
+
configPath: string
|
|
78
|
+
/**
|
|
79
|
+
* Per-file identity, the same function the store's warm path uses. Absent
|
|
80
|
+
* (tests, a store without one), the git blob id is computed from the
|
|
81
|
+
* bytes in-process.
|
|
82
|
+
*/
|
|
83
|
+
hashFile?: (file: string) => Promise<string>
|
|
84
|
+
bytes: Uint8Array
|
|
85
|
+
workspaceFingerprint: string
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** Closure files beyond this count evaluate live — a preset tree this deep is not the case this serves. */
|
|
89
|
+
const MAX_CLOSURE_FILES = 32
|
|
90
|
+
|
|
91
|
+
// Globals through which an evaluation can observe something the file bytes
|
|
92
|
+
// do not capture. Tested against the source with string literals and
|
|
93
|
+
// comments removed (`stripLiterals`), because `node -e "process.exit(0)"`
|
|
94
|
+
// is an ordinary command, not an impure config.
|
|
95
|
+
// `global` and `self` are live objects in Bun (aliases of `globalThis`), so
|
|
96
|
+
// a computed `global['proc' + 'ess']` reaches process without ever
|
|
97
|
+
// spelling it; `Temporal` is a clock. All three were CACHED AS PURE before
|
|
98
|
+
// they were listed (2026-09-03).
|
|
99
|
+
const IMPURE_RE =
|
|
100
|
+
/\b(?:process|Bun|globalThis|global|self|fetch|Date|Temporal|Intl|crypto|performance|navigator|require|eval|Function|await|toLocale\w*)\b|import\s*\.\s*meta|Math\s*\.\s*random|\bimport\s*\(/
|
|
101
|
+
|
|
102
|
+
// Static `import … from '…'` / `export … from '…'` / `import '…'` forms.
|
|
103
|
+
// `[^;'"]*?` spans newlines, so multi-line specifier lists match.
|
|
104
|
+
const IMPORT_RE =
|
|
105
|
+
/(?:^|[\n;])\s*(?:import|export)\b[^;'"]*?\bfrom\s*['"]([^'"]+)['"]|(?:^|[\n;])\s*import\s*['"]([^'"]+)['"]/g
|
|
106
|
+
|
|
107
|
+
/** The one bare specifier a pure config may import: core's identity helpers and types. */
|
|
108
|
+
const PURE_PACKAGE = '@vzn/vx'
|
|
109
|
+
|
|
110
|
+
const decoder = new TextDecoder()
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* `source` with every string literal, template literal and comment removed
|
|
114
|
+
* (template `${…}` expressions are kept — they are code). Returns `null`
|
|
115
|
+
* when a `/` in code position is neither a comment nor a division the
|
|
116
|
+
* lexer can prove: a regex literal can contain a quote, and a lexer that
|
|
117
|
+
* misreads one would swallow real code as a string — a false SAFE, the one
|
|
118
|
+
* outcome this module must never produce. Configs do not need regexes;
|
|
119
|
+
* such a config evaluates live.
|
|
120
|
+
*/
|
|
121
|
+
export function stripLiterals(source: string): string | null {
|
|
122
|
+
let out = ''
|
|
123
|
+
let i = 0
|
|
124
|
+
const n = source.length
|
|
125
|
+
// Brace depth per open template expression, so a `}` closing the
|
|
126
|
+
// expression is told apart from one closing an object literal inside it.
|
|
127
|
+
const templateDepth: number[] = []
|
|
128
|
+
while (i < n) {
|
|
129
|
+
const c = source[i]!
|
|
130
|
+
const next = source[i + 1]
|
|
131
|
+
if (c === '/' && next === '/') {
|
|
132
|
+
while (i < n && source[i] !== '\n') i++
|
|
133
|
+
continue
|
|
134
|
+
}
|
|
135
|
+
if (c === '/' && next === '*') {
|
|
136
|
+
const end = source.indexOf('*/', i + 2)
|
|
137
|
+
if (end === -1) return null
|
|
138
|
+
i = end + 2
|
|
139
|
+
continue
|
|
140
|
+
}
|
|
141
|
+
if (c === '/') return null
|
|
142
|
+
if (c === "'" || c === '"') {
|
|
143
|
+
i++
|
|
144
|
+
while (i < n && source[i] !== c) {
|
|
145
|
+
if (source[i] === '\\') i++
|
|
146
|
+
if (source[i] === '\n') return null
|
|
147
|
+
i++
|
|
148
|
+
}
|
|
149
|
+
if (i >= n) return null
|
|
150
|
+
i++
|
|
151
|
+
out += ' '
|
|
152
|
+
continue
|
|
153
|
+
}
|
|
154
|
+
if (c === '`') {
|
|
155
|
+
i++
|
|
156
|
+
for (;;) {
|
|
157
|
+
if (i >= n) return null
|
|
158
|
+
const t = source[i]!
|
|
159
|
+
if (t === '\\') {
|
|
160
|
+
i += 2
|
|
161
|
+
continue
|
|
162
|
+
}
|
|
163
|
+
if (t === '`') {
|
|
164
|
+
i++
|
|
165
|
+
break
|
|
166
|
+
}
|
|
167
|
+
if (t === '$' && source[i + 1] === '{') {
|
|
168
|
+
templateDepth.push(0)
|
|
169
|
+
i += 2
|
|
170
|
+
out += ' '
|
|
171
|
+
break
|
|
172
|
+
}
|
|
173
|
+
i++
|
|
174
|
+
}
|
|
175
|
+
continue
|
|
176
|
+
}
|
|
177
|
+
if (templateDepth.length > 0) {
|
|
178
|
+
const top = templateDepth.length - 1
|
|
179
|
+
if (c === '{') templateDepth[top]!++
|
|
180
|
+
else if (c === '}') {
|
|
181
|
+
if (templateDepth[top] === 0) {
|
|
182
|
+
// Back into the template literal: resume scanning its text.
|
|
183
|
+
templateDepth.pop()
|
|
184
|
+
i++
|
|
185
|
+
for (;;) {
|
|
186
|
+
if (i >= n) return null
|
|
187
|
+
const t = source[i]!
|
|
188
|
+
if (t === '\\') {
|
|
189
|
+
i += 2
|
|
190
|
+
continue
|
|
191
|
+
}
|
|
192
|
+
if (t === '`') {
|
|
193
|
+
i++
|
|
194
|
+
break
|
|
195
|
+
}
|
|
196
|
+
if (t === '$' && source[i + 1] === '{') {
|
|
197
|
+
templateDepth.push(0)
|
|
198
|
+
i += 2
|
|
199
|
+
break
|
|
200
|
+
}
|
|
201
|
+
i++
|
|
202
|
+
}
|
|
203
|
+
out += ' '
|
|
204
|
+
continue
|
|
205
|
+
}
|
|
206
|
+
templateDepth[top]!--
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
out += c
|
|
210
|
+
i++
|
|
211
|
+
}
|
|
212
|
+
return out
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* The cache key for evaluating `configPath`, or `null` when the config is
|
|
217
|
+
* not provably pure (or its closure cannot be read), in which case the
|
|
218
|
+
* caller evaluates live and stores nothing.
|
|
219
|
+
*/
|
|
220
|
+
/** `git hash-object` of `bytes` (sha1 domain), the identity `Cache.hashFile` returns for a sha1 repo. */
|
|
221
|
+
export function blobOidOf(bytes: Uint8Array): string {
|
|
222
|
+
const hasher = new Bun.CryptoHasher('sha1')
|
|
223
|
+
hasher.update(`blob ${bytes.byteLength}\0`)
|
|
224
|
+
hasher.update(bytes)
|
|
225
|
+
return hasher.digest('hex')
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
function keySeed(workspaceFingerprint: string): bigint {
|
|
229
|
+
return xxh3(
|
|
230
|
+
`vx-config-eval-v${CONFIG_EVAL_VERSION}\0${VERSION}\0${Bun.version}\0${workspaceFingerprint}\0`,
|
|
231
|
+
)
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
const EXPLICIT_EXT = /\.(?:m?[jt]s|cjs|cts)$/
|
|
235
|
+
|
|
236
|
+
export async function configEvalKey(a: ConfigEvalKeyArgs): Promise<ConfigEvalKeyResult | null> {
|
|
237
|
+
let h = keySeed(a.workspaceFingerprint)
|
|
238
|
+
const hashOf = a.hashFile ?? (async (_file: string, bytes?: Uint8Array) => blobOidOf(bytes!))
|
|
239
|
+
const visited = new Set<string>([a.configPath])
|
|
240
|
+
const closure: string[] = []
|
|
241
|
+
let indexable = true
|
|
242
|
+
const queue: Array<{ file: string; bytes: Uint8Array }> = [{ file: a.configPath, bytes: a.bytes }]
|
|
243
|
+
while (queue.length > 0) {
|
|
244
|
+
const { file, bytes } = queue.shift()!
|
|
245
|
+
const source = decoder.decode(bytes)
|
|
246
|
+
const code = stripLiterals(source)
|
|
247
|
+
// A backslash in code position is an identifier escape (`\u0070rocess`
|
|
248
|
+
// IS `process`) — the one spelling the deny-list cannot see. Refuse it.
|
|
249
|
+
if (code === null || code.includes('\\') || IMPURE_RE.test(code)) return null
|
|
250
|
+
const identity = a.hashFile ? await a.hashFile(file) : await hashOf(file, bytes)
|
|
251
|
+
h = xxh3(`${file}\0${identity}`, h)
|
|
252
|
+
closure.push(file)
|
|
253
|
+
for (const m of source.matchAll(IMPORT_RE)) {
|
|
254
|
+
const spec = m[1] ?? m[2]!
|
|
255
|
+
if (spec === PURE_PACKAGE) continue
|
|
256
|
+
if (!spec.startsWith('./') && !spec.startsWith('../')) return null
|
|
257
|
+
if (!EXPLICIT_EXT.test(spec)) indexable = false
|
|
258
|
+
let resolved: string
|
|
259
|
+
try {
|
|
260
|
+
// Real-pathed: `Bun.resolveSync` answers with the symlinked spelling
|
|
261
|
+
// on one call and the real one on another (a tmp workspace under
|
|
262
|
+
// /var vs /private/var), and the key folds the path — a spelling
|
|
263
|
+
// that drifts between runs is a spurious miss and a duplicated
|
|
264
|
+
// memo row.
|
|
265
|
+
resolved = realpathSync(Bun.resolveSync(spec, path.dirname(file)))
|
|
266
|
+
} catch {
|
|
267
|
+
return null
|
|
268
|
+
}
|
|
269
|
+
if (visited.has(resolved)) continue
|
|
270
|
+
if (resolved.split(path.sep).includes('node_modules')) return null
|
|
271
|
+
visited.add(resolved)
|
|
272
|
+
if (visited.size > MAX_CLOSURE_FILES) return null
|
|
273
|
+
try {
|
|
274
|
+
queue.push({ file: resolved, bytes: await Bun.file(resolved).bytes() })
|
|
275
|
+
} catch {
|
|
276
|
+
return null
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
return { key: h.toString(16).padStart(16, '0'), closure, indexable }
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/**
|
|
284
|
+
* The warm path: the key for a config whose ordered closure the store
|
|
285
|
+
* remembers, from per-file identities alone — no read, no scan. The fold is
|
|
286
|
+
* byte-identical to `configEvalKey`'s, so the two paths share entries. A
|
|
287
|
+
* changed or vanished file changes its identity and misses, and the slow
|
|
288
|
+
* path then re-indexes.
|
|
289
|
+
*/
|
|
290
|
+
export async function configEvalKeyFromClosure(a: {
|
|
291
|
+
closure: readonly string[]
|
|
292
|
+
hashFile: (file: string) => Promise<string>
|
|
293
|
+
workspaceFingerprint: string
|
|
294
|
+
}): Promise<string | null> {
|
|
295
|
+
let h = keySeed(a.workspaceFingerprint)
|
|
296
|
+
let identities: string[]
|
|
297
|
+
try {
|
|
298
|
+
identities = await Promise.all(a.closure.map((f) => a.hashFile(f)))
|
|
299
|
+
} catch {
|
|
300
|
+
return null
|
|
301
|
+
}
|
|
302
|
+
for (let i = 0; i < a.closure.length; i++) h = xxh3(`${a.closure[i]}\0${identities[i]}`, h)
|
|
303
|
+
return h.toString(16).padStart(16, '0')
|
|
304
|
+
}
|