@vzn/vx 0.0.16 → 0.0.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +135 -124
- package/index.ts +10 -0
- package/launcher.mjs +10 -11
- package/package.json +11 -7
- package/src/bin.ts +14 -4
- package/src/cache/archive.ts +659 -0
- package/src/cache/cache.ts +769 -1101
- package/src/cache/chained-cache.ts +194 -0
- package/src/cache/config-evals.ts +89 -0
- package/src/cache/file-hashes.ts +262 -0
- package/src/cache/git-inputs.ts +637 -0
- package/src/cache/index.ts +24 -5
- package/src/cache/inputs.ts +373 -456
- package/src/cache/layer.ts +604 -0
- package/src/cache/layered-cache.ts +96 -43
- package/src/cache/output-index.ts +224 -0
- package/src/cache/policy.ts +84 -0
- package/src/cache/run-history.ts +147 -0
- package/src/cache/tar-stream.ts +358 -0
- package/src/cache/zstd.ts +160 -0
- package/src/cli/cache.ts +75 -21
- package/src/cli/completions.ts +104 -0
- package/src/cli/core-alias.ts +25 -0
- package/src/cli/help.ts +195 -128
- package/src/cli/index.ts +116 -46
- package/src/cli/info.ts +122 -63
- package/src/cli/init.ts +75 -0
- package/src/cli/last.ts +172 -0
- package/src/cli/lock.ts +2 -1
- package/src/cli/plan-format.ts +72 -6
- package/src/cli/plugin-commands.ts +103 -0
- package/src/cli/run.ts +245 -271
- package/src/cli/select.ts +332 -0
- package/src/cli/show.ts +144 -69
- package/src/cli/upgrade.ts +9 -7
- package/src/cli/watch.ts +858 -110
- package/src/cli/why.ts +223 -0
- package/src/cli/workspace-config.ts +126 -0
- package/src/config.ts +187 -150
- package/src/exec/env.ts +18 -1
- package/src/exec/executor.ts +224 -0
- package/src/exec/index.ts +24 -2
- package/src/exec/local-executor.ts +41 -0
- package/src/exec/runner.ts +166 -40
- package/src/exec/sandbox-binds.ts +182 -0
- package/src/exec/sandbox-paths.ts +50 -0
- package/src/exec/sandbox-runtime.ts +642 -260
- package/src/exec/sandbox-violations.ts +280 -0
- package/src/graph/index.ts +4 -10
- package/src/graph/scheduler.ts +232 -203
- package/src/graph/task-graph.ts +222 -8
- package/src/index.ts +146 -153
- package/src/orchestrator/admission.ts +137 -0
- package/src/orchestrator/deferred-outputs.ts +147 -0
- package/src/orchestrator/doctor.ts +255 -0
- package/src/orchestrator/download-policy.ts +171 -0
- package/src/orchestrator/events.ts +72 -8
- package/src/orchestrator/execute-task.ts +374 -549
- package/src/orchestrator/failure-mode.ts +202 -0
- package/src/orchestrator/framed-output.ts +126 -69
- package/src/orchestrator/history.ts +126 -59
- package/src/orchestrator/hit-restore.ts +221 -0
- package/src/orchestrator/index.ts +77 -93
- package/src/orchestrator/local-shortcircuit.ts +22 -0
- package/src/orchestrator/lockfile-claim.ts +281 -0
- package/src/orchestrator/logger.ts +208 -35
- package/src/orchestrator/metrics.ts +102 -1388
- package/src/orchestrator/miss-save.ts +169 -0
- package/src/orchestrator/options.ts +55 -34
- package/src/orchestrator/persistent.ts +79 -0
- package/src/orchestrator/placement.ts +182 -0
- package/src/orchestrator/plan.ts +144 -1
- package/src/orchestrator/plugin-host.ts +357 -120
- package/src/orchestrator/plugin.ts +298 -58
- package/src/orchestrator/prepare.ts +202 -106
- package/src/orchestrator/projects.ts +259 -0
- package/src/orchestrator/remote-prefetch.ts +12 -8
- package/src/orchestrator/run-artifacts.ts +63 -16
- package/src/orchestrator/run-context.ts +167 -29
- package/src/orchestrator/run-records.ts +149 -0
- package/src/orchestrator/run-report.ts +74 -47
- package/src/orchestrator/run.ts +534 -417
- package/src/orchestrator/sandbox-request.ts +243 -0
- package/src/orchestrator/save-lane.ts +47 -0
- package/src/orchestrator/signals.ts +111 -0
- package/src/orchestrator/stable-keys.ts +129 -31
- package/src/orchestrator/status-line.ts +49 -9
- package/src/orchestrator/summary.ts +90 -26
- package/src/orchestrator/tally.ts +78 -35
- package/src/orchestrator/task-hash.ts +111 -15
- package/src/orchestrator/task-log-buffer.ts +288 -0
- package/src/orchestrator/telemetry-host.ts +34 -6
- package/src/orchestrator/telemetry.ts +182 -36
- package/src/orchestrator/upstream.ts +72 -31
- package/src/util/cgroup.ts +160 -0
- package/src/util/edit-distance.ts +70 -0
- package/src/util/errors.ts +15 -1
- package/src/util/index.ts +15 -2
- package/src/util/num.ts +37 -0
- package/src/util/paths.ts +64 -0
- package/src/util/settle.ts +67 -0
- package/src/util/size.ts +5 -1
- package/src/util/tail.ts +66 -0
- package/src/util/timing.ts +63 -0
- package/src/util/verbs.ts +39 -0
- package/src/workspace/affected.ts +250 -23
- package/src/workspace/config-cache.ts +313 -0
- package/src/workspace/config-eval.ts +217 -0
- package/src/workspace/config-imports.ts +162 -0
- package/src/workspace/config-schema.ts +837 -0
- package/src/workspace/filter.ts +65 -10
- package/src/workspace/fingerprint.ts +53 -4
- package/src/workspace/index.ts +42 -3
- package/src/workspace/lockfile.ts +25 -9
- package/src/workspace/migrate-scripts.ts +134 -0
- package/src/workspace/migration.ts +314 -0
- package/src/workspace/nested-dirs.ts +21 -7
- package/src/workspace/package-graph.ts +114 -41
- package/src/workspace/project-loader.ts +224 -450
- package/src/workspace/workspace.ts +298 -72
- package/src/cache/cas-backend.ts +0 -97
- package/src/cache/digest.ts +0 -47
- package/src/cache/tar.ts +0 -343
- package/src/cli/backend.ts +0 -23
- package/src/cli/mcp-rpc.ts +0 -292
- package/src/cli/mcp.ts +0 -92
- package/src/cli/migrate-nx.ts +0 -450
- package/src/cli/migrate-turbo.ts +0 -364
- package/src/cli/migrate.ts +0 -219
- package/src/orchestrator/devframe-surface.ts +0 -74
- package/src/orchestrator/predict.ts +0 -106
- package/src/orchestrator/protocol.ts +0 -156
- package/src/orchestrator/resources.ts +0 -59
- package/src/orchestrator/run-state.ts +0 -145
- package/src/orchestrator/verify.ts +0 -164
- package/src/orchestrator/wire-render.ts +0 -92
- package/src/orchestrator/wire.ts +0 -212
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
// Config evaluation cache — skip re-evaluating a `vx.config` that cannot
|
|
2
|
+
// have changed.
|
|
3
|
+
//
|
|
4
|
+
// A config is a program, and evaluating a thousand of them is the largest
|
|
5
|
+
// fixed cost of a warm run (measured 2026-09-02: ~80 ms to import 1000
|
|
6
|
+
// synthetic configs, ~12 ms to read the same files as data). The cache
|
|
7
|
+
// stores the VALIDATED, JSON-serialised result keyed by everything the
|
|
8
|
+
// evaluation could have observed:
|
|
9
|
+
//
|
|
10
|
+
// - the config's bytes and the bytes of every file it transitively imports
|
|
11
|
+
// by RELATIVE specifier (the closure), paths included;
|
|
12
|
+
// - the workspace fingerprint (lockfiles), which covers package imports;
|
|
13
|
+
// - Bun's version, vx's version and this module's own version. A stored
|
|
14
|
+
// evaluation is served WITHOUT re-validation, so it must never outlive
|
|
15
|
+
// the validator that accepted it — a vx upgrade that tightens a rule
|
|
16
|
+
// re-evaluates every config once.
|
|
17
|
+
//
|
|
18
|
+
// It applies only to configs that are PROVABLY pure by a conservative static
|
|
19
|
+
// check: every import is relative or `@vzn/vx` (whose `defineProject` /
|
|
20
|
+
// `defineWorkspace` are identity functions), and no file in the closure
|
|
21
|
+
// mentions a global through which the environment can leak — `process`,
|
|
22
|
+
// `Bun`, `Date`, `fetch`, `import.meta`, `require`, a dynamic `import()`,
|
|
23
|
+
// `await`, … Anything else evaluates live, exactly as before. The check
|
|
24
|
+
// fails SAFE, never fast: a false negative costs one evaluation, a false
|
|
25
|
+
// positive would cost a stale key, so the deny-list is deliberately wide.
|
|
26
|
+
//
|
|
27
|
+
// JSON is already the contract for a config object — `hashTaskConfig` and
|
|
28
|
+
// `vx lock` both go through `JSON.stringify` — so a cached config derives
|
|
29
|
+
// the same cache key as a live evaluation of the same bytes.
|
|
30
|
+
|
|
31
|
+
import { realpathSync } from 'node:fs'
|
|
32
|
+
import path from 'node:path'
|
|
33
|
+
import { xxh3 } from '../util/index.js'
|
|
34
|
+
import { VERSION } from '../version.js'
|
|
35
|
+
|
|
36
|
+
/** Bump when the key derivation or the stored shape changes. */
|
|
37
|
+
export const CONFIG_EVAL_VERSION = 2
|
|
38
|
+
|
|
39
|
+
/** Where cached evaluations live; `Cache` implements it over `cache.db`. */
|
|
40
|
+
export interface ConfigEvalStore {
|
|
41
|
+
/**
|
|
42
|
+
* The warm fast path (all optional; a store without them keys by bytes).
|
|
43
|
+
* `hashFile` is the file's git blob id behind an mtime/size/ctime/inode
|
|
44
|
+
* memo — no read when unchanged; `getConfigClosures` / `putConfigClosure`
|
|
45
|
+
* keep each config's ORDERED closure (the config first, then every
|
|
46
|
+
* relative import in discovery order), so a warm load keys the config by
|
|
47
|
+
* stat-hashing that list instead of reading and scanning every file
|
|
48
|
+
* (1,000 configs: 5 ms against 15, measured 2026-09-03). Sound because
|
|
49
|
+
* closure membership can only change by editing a listed file, which
|
|
50
|
+
* changes that file's hash and so the key; the one exception, an
|
|
51
|
+
* extensionless relative import whose resolution a new file could
|
|
52
|
+
* shadow, is never indexed.
|
|
53
|
+
*/
|
|
54
|
+
hashFile?(file: string): Promise<string>
|
|
55
|
+
/**
|
|
56
|
+
* `hashFile` over many paths with one memo query; a path that cannot be
|
|
57
|
+
* stat'ed is absent. The warm load identifies every indexed closure's
|
|
58
|
+
* files through this in a single call when the store offers it.
|
|
59
|
+
*/
|
|
60
|
+
hashFiles?(files: readonly string[]): Promise<Map<string, string>>
|
|
61
|
+
getConfigClosures?(configPaths: readonly string[]): Map<string, string[]>
|
|
62
|
+
putConfigClosure?(configPath: string, files: readonly string[]): void
|
|
63
|
+
getConfigEval(key: string): string | null
|
|
64
|
+
/**
|
|
65
|
+
* Many keys in one round-trip (optional; a store without it is asked per
|
|
66
|
+
* key). A warm 1000-project run paid 3.6 ms for 1,000 point lookups where
|
|
67
|
+
* one `IN` query costs 0.7 (measured 2026-09-03).
|
|
68
|
+
*/
|
|
69
|
+
getConfigEvals?(keys: readonly string[]): Map<string, string>
|
|
70
|
+
putConfigEval(key: string, json: string): void
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** What `configEvalKey` learned besides the key, for the store's closure index. */
|
|
74
|
+
export interface ConfigEvalKeyResult {
|
|
75
|
+
key: string
|
|
76
|
+
/** The config first, then every relative import in discovery order. */
|
|
77
|
+
closure: string[]
|
|
78
|
+
/** False when a relative import is extensionless — a new file could change its resolution. */
|
|
79
|
+
indexable: boolean
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export interface ConfigEvalKeyArgs {
|
|
83
|
+
configPath: string
|
|
84
|
+
/**
|
|
85
|
+
* Per-file identity, the same function the store's warm path uses. Absent
|
|
86
|
+
* (tests, a store without one), the git blob id is computed from the
|
|
87
|
+
* bytes in-process.
|
|
88
|
+
*/
|
|
89
|
+
hashFile?: (file: string) => Promise<string>
|
|
90
|
+
bytes: Uint8Array
|
|
91
|
+
workspaceFingerprint: string
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** Closure files beyond this count evaluate live — a preset tree this deep is not the case this serves. */
|
|
95
|
+
const MAX_CLOSURE_FILES = 32
|
|
96
|
+
|
|
97
|
+
// Globals through which an evaluation can observe something the file bytes
|
|
98
|
+
// do not capture. Tested against the source with string literals and
|
|
99
|
+
// comments removed (`stripLiterals`), because `node -e "process.exit(0)"`
|
|
100
|
+
// is an ordinary command, not an impure config.
|
|
101
|
+
// `global` and `self` are live objects in Bun (aliases of `globalThis`), so
|
|
102
|
+
// a computed `global['proc' + 'ess']` reaches process without ever
|
|
103
|
+
// spelling it; `Temporal` is a clock. All three were CACHED AS PURE before
|
|
104
|
+
// they were listed (2026-09-03). `constructor` reaches `Function` through a
|
|
105
|
+
// property name (`({}).constructor.constructor('return process')()`, the
|
|
106
|
+
// body hidden in a literal the strip removes) and `localeCompare` answers
|
|
107
|
+
// by the host locale; both listed 2026-09-09.
|
|
108
|
+
const IMPURE_RE =
|
|
109
|
+
/\b(?:process|Bun|globalThis|global|self|fetch|Date|Temporal|Intl|crypto|performance|navigator|require|eval|Function|constructor|localeCompare|await|toLocale\w*)\b|import\s*\.\s*meta|Math\s*\.\s*random|\bimport\s*\(/
|
|
110
|
+
|
|
111
|
+
// Static `import … from '…'` / `export … from '…'` / `import '…'` forms.
|
|
112
|
+
// `[^;'"]*?` spans newlines, so multi-line specifier lists match.
|
|
113
|
+
const IMPORT_RE =
|
|
114
|
+
/(?:^|[\n;])\s*(?:import|export)\b[^;'"]*?\bfrom\s*['"]([^'"]+)['"]|(?:^|[\n;])\s*import\s*['"]([^'"]+)['"]/g
|
|
115
|
+
|
|
116
|
+
/** The one bare specifier a pure config may import: core's identity helpers and types. */
|
|
117
|
+
const PURE_PACKAGE = '@vzn/vx'
|
|
118
|
+
|
|
119
|
+
const decoder = new TextDecoder()
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* `source` with every string literal, template literal and comment removed
|
|
123
|
+
* (template `${…}` expressions are kept — they are code). Returns `null`
|
|
124
|
+
* when a `/` in code position is neither a comment nor a division the
|
|
125
|
+
* lexer can prove: a regex literal can contain a quote, and a lexer that
|
|
126
|
+
* misreads one would swallow real code as a string — a false SAFE, the one
|
|
127
|
+
* outcome this module must never produce. Configs do not need regexes;
|
|
128
|
+
* such a config evaluates live.
|
|
129
|
+
*/
|
|
130
|
+
export function stripLiterals(source: string): string | null {
|
|
131
|
+
let out = ''
|
|
132
|
+
let i = 0
|
|
133
|
+
const n = source.length
|
|
134
|
+
// Brace depth per open template expression, so a `}` closing the
|
|
135
|
+
// expression is told apart from one closing an object literal inside it.
|
|
136
|
+
const templateDepth: number[] = []
|
|
137
|
+
while (i < n) {
|
|
138
|
+
const c = source[i]!
|
|
139
|
+
const next = source[i + 1]
|
|
140
|
+
if (c === '/' && next === '/') {
|
|
141
|
+
while (i < n && source[i] !== '\n') i++
|
|
142
|
+
continue
|
|
143
|
+
}
|
|
144
|
+
if (c === '/' && next === '*') {
|
|
145
|
+
const end = source.indexOf('*/', i + 2)
|
|
146
|
+
if (end === -1) return null
|
|
147
|
+
i = end + 2
|
|
148
|
+
continue
|
|
149
|
+
}
|
|
150
|
+
if (c === '/') return null
|
|
151
|
+
if (c === "'" || c === '"') {
|
|
152
|
+
i++
|
|
153
|
+
while (i < n && source[i] !== c) {
|
|
154
|
+
if (source[i] === '\\') i++
|
|
155
|
+
if (source[i] === '\n') return null
|
|
156
|
+
i++
|
|
157
|
+
}
|
|
158
|
+
if (i >= n) return null
|
|
159
|
+
i++
|
|
160
|
+
out += ' '
|
|
161
|
+
continue
|
|
162
|
+
}
|
|
163
|
+
if (c === '`') {
|
|
164
|
+
i++
|
|
165
|
+
for (;;) {
|
|
166
|
+
if (i >= n) return null
|
|
167
|
+
const t = source[i]!
|
|
168
|
+
if (t === '\\') {
|
|
169
|
+
i += 2
|
|
170
|
+
continue
|
|
171
|
+
}
|
|
172
|
+
if (t === '`') {
|
|
173
|
+
i++
|
|
174
|
+
break
|
|
175
|
+
}
|
|
176
|
+
if (t === '$' && source[i + 1] === '{') {
|
|
177
|
+
templateDepth.push(0)
|
|
178
|
+
i += 2
|
|
179
|
+
out += ' '
|
|
180
|
+
break
|
|
181
|
+
}
|
|
182
|
+
i++
|
|
183
|
+
}
|
|
184
|
+
continue
|
|
185
|
+
}
|
|
186
|
+
if (templateDepth.length > 0) {
|
|
187
|
+
const top = templateDepth.length - 1
|
|
188
|
+
if (c === '{') templateDepth[top]!++
|
|
189
|
+
else if (c === '}') {
|
|
190
|
+
if (templateDepth[top] === 0) {
|
|
191
|
+
// Back into the template literal: resume scanning its text.
|
|
192
|
+
templateDepth.pop()
|
|
193
|
+
i++
|
|
194
|
+
for (;;) {
|
|
195
|
+
if (i >= n) return null
|
|
196
|
+
const t = source[i]!
|
|
197
|
+
if (t === '\\') {
|
|
198
|
+
i += 2
|
|
199
|
+
continue
|
|
200
|
+
}
|
|
201
|
+
if (t === '`') {
|
|
202
|
+
i++
|
|
203
|
+
break
|
|
204
|
+
}
|
|
205
|
+
if (t === '$' && source[i + 1] === '{') {
|
|
206
|
+
templateDepth.push(0)
|
|
207
|
+
i += 2
|
|
208
|
+
break
|
|
209
|
+
}
|
|
210
|
+
i++
|
|
211
|
+
}
|
|
212
|
+
out += ' '
|
|
213
|
+
continue
|
|
214
|
+
}
|
|
215
|
+
templateDepth[top]!--
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
out += c
|
|
219
|
+
i++
|
|
220
|
+
}
|
|
221
|
+
return out
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* The cache key for evaluating `configPath`, or `null` when the config is
|
|
226
|
+
* not provably pure (or its closure cannot be read), in which case the
|
|
227
|
+
* caller evaluates live and stores nothing.
|
|
228
|
+
*/
|
|
229
|
+
/** `git hash-object` of `bytes` (sha1 domain), the identity `Cache.hashFile` returns for a sha1 repo. */
|
|
230
|
+
export function blobOidOf(bytes: Uint8Array): string {
|
|
231
|
+
const hasher = new Bun.CryptoHasher('sha1')
|
|
232
|
+
hasher.update(`blob ${bytes.byteLength}\0`)
|
|
233
|
+
hasher.update(bytes)
|
|
234
|
+
return hasher.digest('hex')
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function keySeed(workspaceFingerprint: string): bigint {
|
|
238
|
+
return xxh3(
|
|
239
|
+
`vx-config-eval-v${CONFIG_EVAL_VERSION}\0${VERSION}\0${Bun.version}\0${workspaceFingerprint}\0`,
|
|
240
|
+
)
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
const EXPLICIT_EXT = /\.(?:m?[jt]s|cjs|cts)$/
|
|
244
|
+
|
|
245
|
+
export async function configEvalKey(a: ConfigEvalKeyArgs): Promise<ConfigEvalKeyResult | null> {
|
|
246
|
+
let h = keySeed(a.workspaceFingerprint)
|
|
247
|
+
const hashOf = a.hashFile ?? (async (_file: string, bytes?: Uint8Array) => blobOidOf(bytes!))
|
|
248
|
+
const visited = new Set<string>([a.configPath])
|
|
249
|
+
const closure: string[] = []
|
|
250
|
+
let indexable = true
|
|
251
|
+
const queue: Array<{ file: string; bytes: Uint8Array }> = [{ file: a.configPath, bytes: a.bytes }]
|
|
252
|
+
while (queue.length > 0) {
|
|
253
|
+
const { file, bytes } = queue.shift()!
|
|
254
|
+
const source = decoder.decode(bytes)
|
|
255
|
+
const code = stripLiterals(source)
|
|
256
|
+
// A backslash in code position is an identifier escape (`\u0070rocess`
|
|
257
|
+
// IS `process`) — the one spelling the deny-list cannot see. Refuse it.
|
|
258
|
+
if (code === null || code.includes('\\') || IMPURE_RE.test(code)) return null
|
|
259
|
+
const identity = a.hashFile ? await a.hashFile(file) : await hashOf(file, bytes)
|
|
260
|
+
h = xxh3(`${file}\0${identity}`, h)
|
|
261
|
+
closure.push(file)
|
|
262
|
+
for (const m of source.matchAll(IMPORT_RE)) {
|
|
263
|
+
const spec = m[1] ?? m[2]!
|
|
264
|
+
if (spec === PURE_PACKAGE) continue
|
|
265
|
+
if (!spec.startsWith('./') && !spec.startsWith('../')) return null
|
|
266
|
+
if (!EXPLICIT_EXT.test(spec)) indexable = false
|
|
267
|
+
let resolved: string
|
|
268
|
+
try {
|
|
269
|
+
// Real-pathed: `Bun.resolveSync` answers with the symlinked spelling
|
|
270
|
+
// on one call and the real one on another (a tmp workspace under
|
|
271
|
+
// /var vs /private/var), and the key folds the path — a spelling
|
|
272
|
+
// that drifts between runs is a spurious miss and a duplicated
|
|
273
|
+
// memo row.
|
|
274
|
+
resolved = realpathSync(Bun.resolveSync(spec, path.dirname(file)))
|
|
275
|
+
} catch {
|
|
276
|
+
return null
|
|
277
|
+
}
|
|
278
|
+
if (visited.has(resolved)) continue
|
|
279
|
+
if (resolved.split(path.sep).includes('node_modules')) return null
|
|
280
|
+
visited.add(resolved)
|
|
281
|
+
if (visited.size > MAX_CLOSURE_FILES) return null
|
|
282
|
+
try {
|
|
283
|
+
queue.push({ file: resolved, bytes: await Bun.file(resolved).bytes() })
|
|
284
|
+
} catch {
|
|
285
|
+
return null
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
return { key: h.toString(16).padStart(16, '0'), closure, indexable }
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/**
|
|
293
|
+
* The warm path: the key for a config whose ordered closure the store
|
|
294
|
+
* remembers, from per-file identities alone — no read, no scan. The fold is
|
|
295
|
+
* byte-identical to `configEvalKey`'s, so the two paths share entries. A
|
|
296
|
+
* changed or vanished file changes its identity and misses, and the slow
|
|
297
|
+
* path then re-indexes.
|
|
298
|
+
*/
|
|
299
|
+
export async function configEvalKeyFromClosure(a: {
|
|
300
|
+
closure: readonly string[]
|
|
301
|
+
hashFile: (file: string) => Promise<string>
|
|
302
|
+
workspaceFingerprint: string
|
|
303
|
+
}): Promise<string | null> {
|
|
304
|
+
let h = keySeed(a.workspaceFingerprint)
|
|
305
|
+
let identities: string[]
|
|
306
|
+
try {
|
|
307
|
+
identities = await Promise.all(a.closure.map((f) => a.hashFile(f)))
|
|
308
|
+
} catch {
|
|
309
|
+
return null
|
|
310
|
+
}
|
|
311
|
+
for (let i = 0; i < a.closure.length; i++) h = xxh3(`${a.closure[i]}\0${identities[i]}`, h)
|
|
312
|
+
return h.toString(16).padStart(16, '0')
|
|
313
|
+
}
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
// Re-evaluating a project config, import closure included, inside a
|
|
2
|
+
// long-lived process.
|
|
3
|
+
//
|
|
4
|
+
// The loader's query-string bust only changes the ENTRY's specifier.
|
|
5
|
+
// Bun caches an evaluated module by its RESOLVED specifier, so an
|
|
6
|
+
// `import './preset.js'` inside a config resolves to the same key no
|
|
7
|
+
// matter what query the entry carries — a busted entry re-evaluates
|
|
8
|
+
// against a STALE preset. Shared presets are the documented composition
|
|
9
|
+
// mechanism (`vx migrate` generates one), so through a whole `vx watch`
|
|
10
|
+
// session a preset edit was invisible; and because the resolved config
|
|
11
|
+
// feeds the cache key, vx answered `up-to-date` for a command that had
|
|
12
|
+
// changed on disk.
|
|
13
|
+
//
|
|
14
|
+
// A Worker gets its own module registry, so everything it imports is
|
|
15
|
+
// read from disk NOW. It is the only mechanism for this that the
|
|
16
|
+
// runtime exposes as public API: `globalThis.Loader.registry` — the
|
|
17
|
+
// obvious place to evict from — exists on Bun 1.3.11 and is GONE on
|
|
18
|
+
// 1.3.14, where an eviction-based fix degrades to no fix at all while
|
|
19
|
+
// still reporting success.
|
|
20
|
+
//
|
|
21
|
+
// The worker source is an inline data: URL, not a sibling file, because
|
|
22
|
+
// `bun build --compile` does NOT embed a Worker entry point — it
|
|
23
|
+
// resolves the URL from disk at runtime, so a sibling file would make
|
|
24
|
+
// the shipped standalone binary fail with ModuleNotFound.
|
|
25
|
+
//
|
|
26
|
+
// The config crosses back as JSON. That is already this project's
|
|
27
|
+
// contract for a config object: `hashTaskConfig` derives the cache key
|
|
28
|
+
// from `JSON.stringify(config)` and `vx lock` stores the same
|
|
29
|
+
// round-trip, so a JSON round-trip cannot change how a config hashes,
|
|
30
|
+
// locks or runs. `JSON.stringify(JSON.parse(s)) === s`, so a config
|
|
31
|
+
// re-read through a worker derives the SAME cache key as the
|
|
32
|
+
// in-process first load — which is why this needs no CACHE_VERSION bump.
|
|
33
|
+
|
|
34
|
+
import path from 'node:path'
|
|
35
|
+
import { MAX_TIMEOUT_MS } from '../util/index.js'
|
|
36
|
+
|
|
37
|
+
const WORKER_SRC = `
|
|
38
|
+
self.onmessage = async (e) => {
|
|
39
|
+
const { id, path } = e.data
|
|
40
|
+
try {
|
|
41
|
+
const ns = await import(path)
|
|
42
|
+
const mod = ns?.default
|
|
43
|
+
postMessage({
|
|
44
|
+
id,
|
|
45
|
+
ok: true,
|
|
46
|
+
json: mod !== null && typeof mod === 'object' ? JSON.stringify(mod) : null,
|
|
47
|
+
})
|
|
48
|
+
} catch (err) {
|
|
49
|
+
postMessage({
|
|
50
|
+
id,
|
|
51
|
+
ok: false,
|
|
52
|
+
name: err?.name ?? 'Error',
|
|
53
|
+
message: err?.message ?? String(err),
|
|
54
|
+
stack: err?.stack ?? null,
|
|
55
|
+
position:
|
|
56
|
+
err?.position && typeof err.position === 'object'
|
|
57
|
+
? { file: err.position.file, line: err.position.line, column: err.position.column }
|
|
58
|
+
: null,
|
|
59
|
+
})
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
`
|
|
63
|
+
|
|
64
|
+
const WORKER_URL = `data:text/javascript,${encodeURIComponent(WORKER_SRC)}`
|
|
65
|
+
|
|
66
|
+
interface WorkerReply {
|
|
67
|
+
id: number
|
|
68
|
+
ok: boolean
|
|
69
|
+
json: string | null
|
|
70
|
+
name: string
|
|
71
|
+
message: string
|
|
72
|
+
stack: string | null
|
|
73
|
+
/** A transpile error's location — `BuildMessage.position`, trimmed to what the loader reads. */
|
|
74
|
+
position: { file?: string; line?: number; column?: number } | null
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
interface Pending {
|
|
78
|
+
resolve: (json: string | null) => void
|
|
79
|
+
reject: (err: Error) => void
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* How long a single config evaluation may take before we treat the worker as
|
|
84
|
+
* wedged. Real evaluations are ~10 ms; this exists only so a killed or hung
|
|
85
|
+
* worker cannot stall a long-lived process indefinitely. Read per call so a
|
|
86
|
+
* test can drive the deadline instead of waiting it out.
|
|
87
|
+
*/
|
|
88
|
+
function workerTimeoutMs(): number {
|
|
89
|
+
const raw = process.env['VX_CONFIG_WORKER_TIMEOUT_MS']
|
|
90
|
+
// Same kind of value as the teardown deadline, so the same treatment: a
|
|
91
|
+
// BOUND on a worker that may be wedged, with no "no limit" reading. Falls
|
|
92
|
+
// back rather than clamping, because honouring ~24.8 days would hang
|
|
93
|
+
// `vx watch` forever on a worker the OS killed — and unbounded makes it 1 ms,
|
|
94
|
+
// so EVERY config load times out instead. Both ends break the same feature.
|
|
95
|
+
if (raw !== undefined && /^[0-9]+$/.test(raw)) {
|
|
96
|
+
const n = Number(raw)
|
|
97
|
+
if (n <= MAX_TIMEOUT_MS) return n
|
|
98
|
+
}
|
|
99
|
+
return 30_000
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const pending = new Map<number, Pending>()
|
|
103
|
+
let worker: Worker | null = null
|
|
104
|
+
let inFlight = 0
|
|
105
|
+
let nextId = 0
|
|
106
|
+
let workersCreated = 0
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Workers created so far. Exported solely so a test can pin that ONE
|
|
110
|
+
* worker serves a whole concurrent round — the property that keeps a
|
|
111
|
+
* 1000-project watch cycle at one worker instead of a thousand.
|
|
112
|
+
*/
|
|
113
|
+
export function configEvalWorkerCount(): number {
|
|
114
|
+
return workersCreated
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function rejectAll(err: Error): void {
|
|
118
|
+
for (const p of pending.values()) p.reject(err)
|
|
119
|
+
pending.clear()
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* The worker serving the current round. Every load that is in flight at
|
|
124
|
+
* the same moment shares it, and it is retired once the last of them
|
|
125
|
+
* settles — so a `Promise.all` round costs ONE worker, and the next
|
|
126
|
+
* round (the next watch cycle) still starts from an empty registry.
|
|
127
|
+
*
|
|
128
|
+
* Sharing within a round is also the more faithful semantics: two
|
|
129
|
+
* configs importing the same preset evaluate it once, exactly as they
|
|
130
|
+
* would in a fresh `vx run` process.
|
|
131
|
+
*/
|
|
132
|
+
function acquireWorker(): Worker {
|
|
133
|
+
if (worker !== null) return worker
|
|
134
|
+
const w = new Worker(WORKER_URL)
|
|
135
|
+
workersCreated++
|
|
136
|
+
w.onmessage = (event: MessageEvent): void => {
|
|
137
|
+
const msg = event.data as WorkerReply
|
|
138
|
+
const p = pending.get(msg.id)
|
|
139
|
+
if (p === undefined) return
|
|
140
|
+
pending.delete(msg.id)
|
|
141
|
+
if (msg.ok) {
|
|
142
|
+
p.resolve(msg.json)
|
|
143
|
+
return
|
|
144
|
+
}
|
|
145
|
+
// Rebuild the error the config actually threw. Name, message, stack
|
|
146
|
+
// and a transpile error's position all survive the hop, so a broken
|
|
147
|
+
// config reports the same text it would from an in-process import.
|
|
148
|
+
const err = new Error(msg.message) as Error & { position?: WorkerReply['position'] }
|
|
149
|
+
err.name = msg.name
|
|
150
|
+
if (msg.stack !== null) err.stack = msg.stack
|
|
151
|
+
if (msg.position !== null) err.position = msg.position
|
|
152
|
+
p.reject(err)
|
|
153
|
+
}
|
|
154
|
+
w.onerror = (event: ErrorEvent): void => {
|
|
155
|
+
rejectAll(new Error(`config worker failed: ${String(event.message)}`))
|
|
156
|
+
}
|
|
157
|
+
// A reply that cannot be deserialized would otherwise leave its caller
|
|
158
|
+
// awaiting forever — every path off this worker must settle its pending
|
|
159
|
+
// promises, including the ones that carry no usable payload.
|
|
160
|
+
w.addEventListener('messageerror', () => {
|
|
161
|
+
rejectAll(new Error('config worker sent an undeserializable reply'))
|
|
162
|
+
})
|
|
163
|
+
worker = w
|
|
164
|
+
return w
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Evaluate `configPath` against a fresh module registry and return its
|
|
169
|
+
* default export, JSON round-tripped. `null` means the module had no
|
|
170
|
+
* object default export — the caller owns that error message so it
|
|
171
|
+
* reads identically whichever path produced it.
|
|
172
|
+
*/
|
|
173
|
+
export async function evaluateConfigFresh(configPath: string): Promise<unknown> {
|
|
174
|
+
const abs = path.resolve(configPath)
|
|
175
|
+
const id = nextId++
|
|
176
|
+
inFlight++
|
|
177
|
+
// Declared out here so the `finally` can clear it on EVERY exit, including a
|
|
178
|
+
// rejection. See the note at the clear itself.
|
|
179
|
+
let timer: ReturnType<typeof setTimeout> | undefined
|
|
180
|
+
try {
|
|
181
|
+
const w = acquireWorker()
|
|
182
|
+
// A worker thread the OS kills — memory pressure on a loaded CI box, a
|
|
183
|
+
// hard crash — fires no `error` event, so without a deadline this await
|
|
184
|
+
// never settles and `vx watch` hangs forever on a cycle that would
|
|
185
|
+
// otherwise take milliseconds. Nothing else bounds it: there is no
|
|
186
|
+
// run-level timeout. The budget is enormous next to the ~10 ms a real
|
|
187
|
+
// evaluation costs, so it can only fire on a genuine wedge.
|
|
188
|
+
const budget = workerTimeoutMs()
|
|
189
|
+
const json = await new Promise<string | null>((resolve, reject) => {
|
|
190
|
+
pending.set(id, { resolve, reject })
|
|
191
|
+
timer = setTimeout(() => {
|
|
192
|
+
rejectAll(new Error(`config worker did not answer within ${budget}ms`))
|
|
193
|
+
if (worker !== null) {
|
|
194
|
+
worker.terminate()
|
|
195
|
+
worker = null
|
|
196
|
+
}
|
|
197
|
+
}, budget)
|
|
198
|
+
timer.unref?.()
|
|
199
|
+
w.postMessage({ id, path: abs })
|
|
200
|
+
})
|
|
201
|
+
return json === null ? null : (JSON.parse(json) as unknown)
|
|
202
|
+
} finally {
|
|
203
|
+
// In the `finally`, not after the await: a REJECTED evaluation — a config
|
|
204
|
+
// with a typo, the common case while editing — would otherwise skip the
|
|
205
|
+
// clear and leave its timer armed for the whole budget. When that orphan
|
|
206
|
+
// later fired it ran `rejectAll()` and terminated whatever worker was
|
|
207
|
+
// current by then, killing an unrelated healthy round with a timeout
|
|
208
|
+
// message naming a budget nobody set for it.
|
|
209
|
+
clearTimeout(timer)
|
|
210
|
+
pending.delete(id)
|
|
211
|
+
inFlight--
|
|
212
|
+
if (inFlight === 0 && worker !== null) {
|
|
213
|
+
worker.terminate()
|
|
214
|
+
worker = null
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
}
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
// The third `changed file → project` channel for `--affected`.
|
|
2
|
+
//
|
|
3
|
+
// Directory containment answers "which project owns this file", and
|
|
4
|
+
// `cache.inputs.workspaceFiles` answers "which project declared it". Neither
|
|
5
|
+
// can see the one remaining way a file reaches a task: a project's
|
|
6
|
+
// `vx.config.*` IMPORTS it. Resolved-config hashing folds the imported values
|
|
7
|
+
// into the cache key (architecture principle #4), so editing such a file
|
|
8
|
+
// re-keys the task — and `affected.ts` states the rule this exists to keep:
|
|
9
|
+
// "input hashing sees it, so `--affected` must too."
|
|
10
|
+
//
|
|
11
|
+
// This is a STATIC scan. Nothing is evaluated: `Bun.Transpiler.scanImports`
|
|
12
|
+
// reads the specifiers and `Bun.resolveSync` turns them into paths. The
|
|
13
|
+
// `project-loader.ts` note that a bust "cannot reach the config's import
|
|
14
|
+
// closure" is about `import()` at runtime, not about reading the source.
|
|
15
|
+
//
|
|
16
|
+
// Two rules keep the walk small, and the second is the one that makes it
|
|
17
|
+
// affordable at all:
|
|
18
|
+
//
|
|
19
|
+
// - RELATIVE specifiers only. A bare specifier is a package; it moves when
|
|
20
|
+
// the lockfile moves, which the workspace fingerprint already covers.
|
|
21
|
+
// - Descend only through files owned by NO project. A config reaching into
|
|
22
|
+
// another project (say a site's `vx.config.ts` importing
|
|
23
|
+
// `../core/src/index.ts`) records that edge and STOPS there — following
|
|
24
|
+
// it would drag substantially all of that project's `src/` into the
|
|
25
|
+
// closure, and the containment channel already selects the project that
|
|
26
|
+
// owns it.
|
|
27
|
+
|
|
28
|
+
import { realpath } from 'node:fs/promises'
|
|
29
|
+
import path from 'node:path'
|
|
30
|
+
import type { ProjectMeta } from './workspace.js'
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Absolute resolved targets of the RELATIVE specifiers in `source`.
|
|
34
|
+
*
|
|
35
|
+
* Paths come back realpath'd, because `Bun.resolveSync` realpaths them — see
|
|
36
|
+
* the caller, which realpaths everything it compares against for exactly that
|
|
37
|
+
* reason. `import type` is erased by `scanImports` and so contributes no edge,
|
|
38
|
+
* which is correct: an erased import cannot move a resolved value.
|
|
39
|
+
*/
|
|
40
|
+
function scanLocalImports(source: string, fromDir: string, loader: 'ts' | 'js'): string[] {
|
|
41
|
+
let specifiers: string[]
|
|
42
|
+
try {
|
|
43
|
+
specifiers = new Bun.Transpiler({ loader }).scanImports(source).map((i) => i.path)
|
|
44
|
+
} catch {
|
|
45
|
+
return [] // unparseable source contributes no edges
|
|
46
|
+
}
|
|
47
|
+
const out: string[] = []
|
|
48
|
+
for (const spec of specifiers) {
|
|
49
|
+
if (!spec.startsWith('./') && !spec.startsWith('../')) continue
|
|
50
|
+
try {
|
|
51
|
+
out.push(Bun.resolveSync(spec, fromDir))
|
|
52
|
+
} catch {
|
|
53
|
+
// Unresolvable (deleted, typo, extensionless miss) — a config that will
|
|
54
|
+
// not load cannot be shown to import anything, and failing selection
|
|
55
|
+
// over it would break a working build.
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return out
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export interface ConfigImportOwnersArgs {
|
|
62
|
+
/**
|
|
63
|
+
* Realpath'd HERE, not by the caller. `Bun.resolveSync` returns realpath'd
|
|
64
|
+
* targets, so a raw root silently fails every containment check and the
|
|
65
|
+
* scan reports "no imports" — indistinguishable from a clean tree. Owning
|
|
66
|
+
* the normalisation in one place makes that misuse unrepresentable.
|
|
67
|
+
*/
|
|
68
|
+
workspaceRoot: string
|
|
69
|
+
projects: readonly ProjectMeta[]
|
|
70
|
+
/** Workspace-relative POSIX paths — the same list containment consumed. */
|
|
71
|
+
changed: readonly string[]
|
|
72
|
+
/** Already-selected projects; their configs need no scan. */
|
|
73
|
+
skip: ReadonlySet<string>
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Project dir → name, REALPATH'D — which is why this cannot reuse the index
|
|
78
|
+
* the containment pass builds. `Bun.resolveSync` hands back realpath'd
|
|
79
|
+
* targets, and on darwin a workspace under `os.tmpdir()` lives at
|
|
80
|
+
* `/var/folders/…` while its realpath is `/private/var/folders/…`; comparing
|
|
81
|
+
* the two matches nothing and fails exactly like "found no imports".
|
|
82
|
+
*/
|
|
83
|
+
async function realDirIndex(projects: readonly ProjectMeta[]): Promise<Map<string, string>> {
|
|
84
|
+
const out = new Map<string, string>()
|
|
85
|
+
await Promise.all(
|
|
86
|
+
projects.map(async (p) => {
|
|
87
|
+
out.set(await realpath(p.dir).catch(() => p.dir), p.name)
|
|
88
|
+
}),
|
|
89
|
+
)
|
|
90
|
+
return out
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const TS_EXT = new Set(['.ts', '.mts', '.cts'])
|
|
94
|
+
|
|
95
|
+
/** The deepest project containing `file`, or undefined when none does. */
|
|
96
|
+
function ownerOf(file: string, dirToName: ReadonlyMap<string, string>): string | undefined {
|
|
97
|
+
let dir = path.dirname(file)
|
|
98
|
+
for (;;) {
|
|
99
|
+
const name = dirToName.get(dir)
|
|
100
|
+
if (name !== undefined) return name
|
|
101
|
+
const parent = path.dirname(dir)
|
|
102
|
+
if (parent === dir) return undefined
|
|
103
|
+
dir = parent
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Projects whose config file transitively imports one of `changed`. */
|
|
108
|
+
export async function configImportOwners(a: ConfigImportOwnersArgs): Promise<Set<string>> {
|
|
109
|
+
const selected = new Set<string>()
|
|
110
|
+
if (a.changed.length === 0 || a.skip.size === a.projects.length) return selected
|
|
111
|
+
const workspaceRoot = await realpath(a.workspaceRoot).catch(() => a.workspaceRoot)
|
|
112
|
+
|
|
113
|
+
const roots = new Map<string, string>() // abs config path → project name
|
|
114
|
+
await Promise.all(
|
|
115
|
+
a.projects.map(async (p) => {
|
|
116
|
+
const cfg = p.configPath
|
|
117
|
+
if (cfg === null || cfg === '' || a.skip.has(p.name)) return
|
|
118
|
+
roots.set(await realpath(cfg).catch(() => path.resolve(cfg)), p.name)
|
|
119
|
+
}),
|
|
120
|
+
)
|
|
121
|
+
if (roots.size === 0) return selected
|
|
122
|
+
const dirToName = await realDirIndex(a.projects)
|
|
123
|
+
|
|
124
|
+
// target → the files that import it. Reversed up front so one BFS from the
|
|
125
|
+
// changed set answers every root at once, instead of a walk per root.
|
|
126
|
+
const importedBy = new Map<string, string[]>()
|
|
127
|
+
const visited = new Set<string>()
|
|
128
|
+
const queue = [...roots.keys()]
|
|
129
|
+
while (queue.length > 0) {
|
|
130
|
+
const file = queue.pop()!
|
|
131
|
+
if (visited.has(file)) continue
|
|
132
|
+
visited.add(file)
|
|
133
|
+
let source: string
|
|
134
|
+
try {
|
|
135
|
+
source = await Bun.file(file).text()
|
|
136
|
+
} catch {
|
|
137
|
+
continue // unreadable: no edges, and not this pass's problem to report
|
|
138
|
+
}
|
|
139
|
+
const loader = TS_EXT.has(path.extname(file)) ? 'ts' : 'js'
|
|
140
|
+
for (const target of scanLocalImports(source, path.dirname(file), loader)) {
|
|
141
|
+
if (!target.startsWith(workspaceRoot + path.sep)) continue
|
|
142
|
+
if (target.split(path.sep).includes('node_modules')) continue
|
|
143
|
+
const list = importedBy.get(target)
|
|
144
|
+
if (list) list.push(file)
|
|
145
|
+
else importedBy.set(target, [file])
|
|
146
|
+
// Descend ONLY through unowned files — see the header.
|
|
147
|
+
if (ownerOf(target, dirToName) === undefined) queue.push(target)
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
const seen = new Set<string>()
|
|
152
|
+
const walk = [...a.changed.map((rel) => path.resolve(workspaceRoot, rel))]
|
|
153
|
+
while (walk.length > 0) {
|
|
154
|
+
const file = walk.pop()!
|
|
155
|
+
if (seen.has(file)) continue
|
|
156
|
+
seen.add(file)
|
|
157
|
+
const owner = roots.get(file)
|
|
158
|
+
if (owner !== undefined) selected.add(owner)
|
|
159
|
+
for (const importer of importedBy.get(file) ?? []) walk.push(importer)
|
|
160
|
+
}
|
|
161
|
+
return selected
|
|
162
|
+
}
|