@vzn/vx 0.0.649 → 0.0.651
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -20
- package/package.json +5 -5
- package/schemas/last.json +15 -0
- package/src/bin.ts +9 -31
- package/src/cache/cache.ts +225 -51
- package/src/cache/file-hashes.ts +3 -0
- package/src/cache/fold-memo.ts +8 -1
- package/src/cache/git-inputs.ts +188 -39
- package/src/cache/index.ts +12 -0
- package/src/cache/inputs.ts +3 -0
- package/src/cache/layer.ts +11 -0
- package/src/cache/layered-cache.ts +4 -0
- package/src/cache/linked-inputs.ts +3 -0
- package/src/cache/lockfile-memo.ts +120 -0
- package/src/cache/output-index.ts +5 -2
- package/src/cache/remote-hits.ts +9 -1
- package/src/cache/remote-lane.ts +2 -1
- package/src/cache/run-history.ts +11 -1
- package/src/cache/run-notes.ts +123 -0
- package/src/cache/schema.ts +54 -1
- package/src/cli/cache.ts +26 -16
- package/src/cli/completions.ts +4 -2
- package/src/cli/docs.ts +10 -11
- package/src/cli/help.ts +12 -23
- package/src/cli/index.ts +4 -9
- package/src/cli/info.ts +15 -7
- package/src/cli/init.ts +23 -15
- package/src/cli/last.ts +224 -223
- package/src/cli/lock.ts +55 -5
- package/src/cli/plan-format.ts +100 -118
- package/src/cli/plugin-commands.ts +10 -3
- package/src/cli/plugin-templates.ts +2 -2
- package/src/cli/run.ts +46 -27
- package/src/cli/select.ts +43 -5
- package/src/cli/show.ts +38 -42
- package/src/cli/task-verb.ts +22 -0
- package/src/cli/upgrade.ts +3 -1
- package/src/cli/watch-fs.ts +6 -1
- package/src/cli/watch-judge.ts +2 -1
- package/src/cli/watch.ts +83 -44
- package/src/cli/why.ts +126 -92
- package/src/cli/workspace-config.ts +2 -1
- package/src/exec/executor.ts +2 -1
- package/src/exec/kill-tree.ts +58 -5
- package/src/exec/local-executor.ts +1 -1
- package/src/exec/pipes.ts +152 -0
- package/src/exec/runner.ts +199 -182
- package/src/exec/sandbox-binds.ts +1 -1
- package/src/exec/sandbox-runtime.ts +11 -2
- package/src/exec/spawn-off.ts +154 -0
- package/src/exec/warm-shell.ts +146 -0
- package/src/graph/critical-path.ts +40 -28
- package/src/graph/priorities.ts +98 -44
- package/src/graph/scheduler.ts +12 -2
- package/src/graph/task-graph.ts +27 -1
- package/src/handoff.ts +10 -7
- package/src/history/history.ts +39 -2
- package/src/history/index.ts +3 -0
- package/src/history/metrics.ts +9 -1
- package/src/index.ts +12 -3
- package/src/orchestrator/doctor.ts +3 -3
- package/src/orchestrator/execute-task.ts +16 -2
- package/src/orchestrator/graph-memo.ts +50 -1
- package/src/orchestrator/graph-stage.ts +6 -0
- package/src/orchestrator/hit-restore.ts +3 -0
- package/src/orchestrator/index.ts +3 -1
- package/src/orchestrator/live-usage.ts +98 -0
- package/src/orchestrator/locked-digests.ts +63 -0
- package/src/orchestrator/lockfile-claim.ts +176 -102
- package/src/orchestrator/miss-save.ts +31 -3
- package/src/orchestrator/persistent-task.ts +3 -1
- package/src/orchestrator/plan.ts +5 -1
- package/src/orchestrator/plugin-host.ts +61 -0
- package/src/orchestrator/plugin.ts +49 -4
- package/src/orchestrator/prepare.ts +18 -2
- package/src/orchestrator/projects.ts +4 -1
- package/src/orchestrator/run-context.ts +29 -10
- package/src/orchestrator/run-failures.ts +35 -114
- package/src/orchestrator/run-records.ts +10 -0
- package/src/orchestrator/run.ts +89 -26
- package/src/orchestrator/sandbox-request.ts +2 -2
- package/src/orchestrator/signals.ts +8 -5
- package/src/orchestrator/steps.ts +2 -2
- package/src/orchestrator/task-hash.ts +17 -3
- package/src/report/forecast.ts +12 -33
- package/src/report/framed-output.ts +388 -161
- package/src/report/index.ts +8 -2
- package/src/report/logger.ts +100 -36
- package/src/report/status-line.ts +81 -365
- package/src/report/summary.ts +104 -131
- package/src/ui/ansi.ts +48 -0
- package/src/ui/components.ts +506 -0
- package/src/ui/index.ts +79 -0
- package/src/ui/kit.ts +37 -0
- package/src/ui/live-region.ts +363 -0
- package/src/{util/refusal-notice.ts → ui/refusal.ts} +7 -5
- package/src/ui/terminal.ts +51 -0
- package/src/ui/theme.ts +224 -0
- package/src/ui/version-line.ts +28 -0
- package/src/util/config-frame.ts +65 -10
- package/src/util/edit-distance.ts +6 -1
- package/src/util/hidden-prompt.ts +8 -2
- package/src/util/index.ts +2 -15
- package/src/util/paths.ts +20 -2
- package/src/util/secret-mask.ts +39 -3
- package/src/util/secrets.ts +3 -0
- package/src/util/settle.ts +31 -0
- package/src/util/timing.ts +4 -16
- package/src/workspace/affected.ts +89 -30
- package/src/workspace/config-schema.ts +4 -2
- package/src/workspace/dependencies-lock.ts +191 -0
- package/src/workspace/filter.ts +28 -0
- package/src/workspace/index.ts +9 -0
- package/src/workspace/lockfile.ts +159 -6
- package/src/workspace/migration.ts +4 -3
- package/src/workspace/workspace.ts +3 -0
- package/src/orchestrator/run-steps.ts +0 -93
- package/src/util/colors.ts +0 -130
- package/src/util/report.ts +0 -44
- package/src/util/ui.ts +0 -258
- /package/src/{report/path-links.ts → ui/links.ts} +0 -0
- /package/src/{report → ui}/terminal-signals.ts +0 -0
|
@@ -7,14 +7,22 @@
|
|
|
7
7
|
// fallback for a project the file does not list and the `--affected`
|
|
8
8
|
// diff are one implementation here, so a third lockfile is a parser.
|
|
9
9
|
//
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
10
|
+
// A run's key reads each project's digest from `.vx/dependencies.lock.json`
|
|
11
|
+
// (`KeyHookContext.locked`), never the file: only `vx lock` and
|
|
12
|
+
// `--affected` digest it, and a run whose extra file (a `link:` directory,
|
|
13
|
+
// edited with no install) moved since the lock, as the record is stale
|
|
14
|
+
// for it then. What the `discover` and `project` stages read
|
|
15
|
+
// (the edges, the recorded names) is memoised in cache.db
|
|
16
|
+
// (`lockfile_memos`, one row per lockfile and claimant) by the file's xxh3,
|
|
17
|
+
// so a warm run pays one read + hash + one row read, never a parse; within
|
|
18
|
+
// one process (a run, a watch cycle) the file's size, mtime, ctime and
|
|
19
|
+
// inode gate even the read. A run's read leaves the digests out (most of a
|
|
20
|
+
// parse: 9-25 ms on this repo's bun.lock, against 4-13 for the text), and
|
|
21
|
+
// `vx lock` adds them to the memo.
|
|
15
22
|
|
|
16
23
|
import path from 'node:path'
|
|
17
|
-
import {
|
|
24
|
+
import { stat } from 'node:fs/promises'
|
|
25
|
+
import { type LockfileMemo, readLockfileMemo, writeLockfileMemo } from '../cache/index.js'
|
|
18
26
|
import { relPosix, UserError, xxh3, xxh3hex } from '../util/index.js'
|
|
19
27
|
import type { TaskNode } from '../graph/index.js'
|
|
20
28
|
import type {
|
|
@@ -62,6 +70,12 @@ export interface LockfileClaimOptions {
|
|
|
62
70
|
* parse. Called once per distinct content, under either scope.
|
|
63
71
|
*/
|
|
64
72
|
readonly importers?: (text: string) => ReadonlyMap<string, LockfileImporter>
|
|
73
|
+
/**
|
|
74
|
+
* Each importer's recorded dependency names, from the same text, kept in
|
|
75
|
+
* the memo so a staleness check reads them without a parse. Called once
|
|
76
|
+
* per distinct content.
|
|
77
|
+
*/
|
|
78
|
+
readonly recorded?: (text: string) => ReadonlyMap<string, readonly string[]>
|
|
65
79
|
}
|
|
66
80
|
|
|
67
81
|
/**
|
|
@@ -89,33 +103,35 @@ export interface LockfileClaimHooks {
|
|
|
89
103
|
readonly workspaceRoot: string
|
|
90
104
|
readonly cacheDir: string
|
|
91
105
|
}): Promise<ReadonlyMap<string, LockfileImporter> | undefined>
|
|
106
|
+
/** `recorded`'s answer for the file as it is now, as `importers` reads it; undefined when the file is absent or `recorded` is unset. */
|
|
107
|
+
recorded(ctx: {
|
|
108
|
+
readonly workspaceRoot: string
|
|
109
|
+
readonly cacheDir: string
|
|
110
|
+
}): Promise<ReadonlyMap<string, ReadonlySet<string>> | undefined>
|
|
92
111
|
}
|
|
93
112
|
|
|
94
113
|
interface Digests {
|
|
95
|
-
/** xxh3 of the lockfile bytes; '' when the file is absent. */
|
|
114
|
+
/** xxh3 of the lockfile bytes and the extra files' hashes; '' when the file is absent. */
|
|
96
115
|
readonly lock: string
|
|
97
|
-
|
|
116
|
+
/** xxh3 of the lockfile bytes alone. */
|
|
117
|
+
readonly bytes: string
|
|
118
|
+
/** The extra files the digests read. */
|
|
119
|
+
readonly extras: readonly string[]
|
|
120
|
+
/** Each importer's digest; undefined when the read that made this left them out. */
|
|
121
|
+
readonly importers: ReadonlyMap<string, string> | undefined
|
|
98
122
|
readonly edges: ReadonlyMap<string, LockfileImporter>
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
interface Memo {
|
|
102
|
-
version: number
|
|
103
|
-
claimant: string
|
|
104
|
-
lock: string
|
|
105
|
-
/** `extraFiles` → content hash when the digests were computed. */
|
|
106
|
-
extras?: Record<string, string>
|
|
107
|
-
importers: Record<string, string>
|
|
108
|
-
edges?: Record<string, LockfileImporter>
|
|
123
|
+
readonly recorded: ReadonlyMap<string, ReadonlySet<string>> | undefined
|
|
109
124
|
}
|
|
110
125
|
|
|
111
126
|
/**
|
|
112
|
-
* A root-relative extra file's content hash, '' when absent
|
|
113
|
-
* the root is
|
|
114
|
-
*
|
|
127
|
+
* A root-relative extra file's content hash, '' when absent. A path above
|
|
128
|
+
* the root is read too: a `file:` or `link:` directory beside the checkout
|
|
129
|
+
* is what a task imports, and left unread an edit in it moved no key. One
|
|
130
|
+
* there but unreadable (EACCES, a symlink loop) refuses, as a settings file
|
|
131
|
+
* does: folded as '', it keyed as no file at all.
|
|
115
132
|
*/
|
|
116
|
-
async function hashExtra(workspaceRoot: string, rel: string): Promise<string> {
|
|
117
|
-
|
|
118
|
-
const abs = path.join(workspaceRoot, rel)
|
|
133
|
+
export async function hashExtra(workspaceRoot: string, rel: string): Promise<string> {
|
|
134
|
+
const abs = path.resolve(workspaceRoot, rel)
|
|
119
135
|
try {
|
|
120
136
|
return xxh3hex(await Bun.file(abs).bytes())
|
|
121
137
|
} catch (err) {
|
|
@@ -165,7 +181,7 @@ function identityOf(st: { size: number; mtimeMs: number; ctimeMs: number; ino: n
|
|
|
165
181
|
async function statsOf(workspaceRoot: string, rels: readonly string[]): Promise<string[]> {
|
|
166
182
|
return Promise.all(
|
|
167
183
|
rels.map((rel) =>
|
|
168
|
-
stat(path.
|
|
184
|
+
stat(path.resolve(workspaceRoot, rel)).then(
|
|
169
185
|
(st) => (st.isDirectory() ? hashExtra(workspaceRoot, rel) : identityOf(st)),
|
|
170
186
|
() => 'gone',
|
|
171
187
|
),
|
|
@@ -202,6 +218,7 @@ export function lockfileClaim(options: LockfileClaimOptions): LockfileClaimHooks
|
|
|
202
218
|
String(digest),
|
|
203
219
|
String(options.extraFiles ?? ''),
|
|
204
220
|
String(options.importers ?? ''),
|
|
221
|
+
String(options.recorded ?? ''),
|
|
205
222
|
].join('\0'),
|
|
206
223
|
)
|
|
207
224
|
if (scope !== 'project' && scope !== 'workspace') {
|
|
@@ -216,48 +233,84 @@ export function lockfileClaim(options: LockfileClaimOptions): LockfileClaimHooks
|
|
|
216
233
|
extras: readonly string[]
|
|
217
234
|
extraStats: Readonly<Record<string, string>>
|
|
218
235
|
digests: Digests
|
|
236
|
+
/** The memo a run found no index to keep in (a discover before the first run's open). */
|
|
237
|
+
unsaved?: LockfileMemo
|
|
219
238
|
}
|
|
220
239
|
>()
|
|
240
|
+
// Workspace scope with nothing else to keep has nothing a memo saves.
|
|
241
|
+
const memoises =
|
|
242
|
+
scope === 'project' || options.importers !== undefined || options.recorded !== undefined
|
|
243
|
+
const readMemo = (cacheDir: string, lock: string) => {
|
|
244
|
+
const memo = readLockfileMemo(cacheDir, file, claimant)
|
|
245
|
+
if (memo === undefined || memo.version !== version || memo.lock !== lock) return undefined
|
|
246
|
+
return memo
|
|
247
|
+
}
|
|
221
248
|
// Per run: every `key` call of one run gets the same context object.
|
|
222
249
|
const perRun = new WeakMap<object, Promise<Digests>>()
|
|
250
|
+
const lockedPerRun = new WeakMap<
|
|
251
|
+
object,
|
|
252
|
+
Promise<((dir: string) => string | undefined) | undefined>
|
|
253
|
+
>()
|
|
223
254
|
|
|
224
|
-
|
|
255
|
+
/** `withDigests`: each importer's digest too, which only `vx lock` and `--affected` read. */
|
|
256
|
+
const load = async (
|
|
257
|
+
workspaceRoot: string,
|
|
258
|
+
cacheDir: string,
|
|
259
|
+
withDigests: boolean,
|
|
260
|
+
): Promise<Digests> => {
|
|
225
261
|
const full = path.join(workspaceRoot, file)
|
|
226
262
|
let identity: string
|
|
227
263
|
try {
|
|
228
264
|
identity = identityOf(await stat(full))
|
|
229
265
|
} catch {
|
|
230
|
-
const digests: Digests = {
|
|
266
|
+
const digests: Digests = {
|
|
267
|
+
lock: '',
|
|
268
|
+
bytes: '',
|
|
269
|
+
extras: [],
|
|
270
|
+
importers: new Map(),
|
|
271
|
+
edges: new Map(),
|
|
272
|
+
recorded: undefined,
|
|
273
|
+
}
|
|
231
274
|
seen.set(workspaceRoot, { identity: 'gone', extras: [], extraStats: {}, digests })
|
|
232
275
|
return digests
|
|
233
276
|
}
|
|
234
277
|
const last = seen.get(workspaceRoot)
|
|
235
|
-
if (
|
|
278
|
+
if (
|
|
279
|
+
last !== undefined &&
|
|
280
|
+
last.identity === identity &&
|
|
281
|
+
(!withDigests || last.digests.importers !== undefined)
|
|
282
|
+
) {
|
|
236
283
|
const now = await statsOf(workspaceRoot, last.extras)
|
|
237
284
|
if (now.every((s, i) => s === last.extraStats[last.extras[i]!])) {
|
|
285
|
+
if (last.unsaved !== undefined && writeLockfileMemo(cacheDir, file, claimant, last.unsaved))
|
|
286
|
+
delete last.unsaved
|
|
238
287
|
return last.digests
|
|
239
288
|
}
|
|
240
289
|
}
|
|
241
290
|
const bytes = await Bun.file(full).bytes()
|
|
242
291
|
const lockHash = xxh3hex(bytes)
|
|
243
|
-
const memoFile = path.join(cacheDir, 'lockfile-claims', `${file}.json`)
|
|
244
292
|
// The extra files the last digest read, and their hashes now: a warm run
|
|
245
293
|
// re-hashes the files the memo names and never parses to learn them.
|
|
246
|
-
const memo =
|
|
294
|
+
const memo = memoises ? readMemo(cacheDir, lockHash) : undefined
|
|
247
295
|
let extras: Record<string, string> = {}
|
|
248
296
|
let extraStats: Record<string, string> = {}
|
|
249
297
|
let importers: ReadonlyMap<string, string> | undefined
|
|
250
298
|
let edges: ReadonlyMap<string, LockfileImporter> = new Map()
|
|
251
|
-
|
|
299
|
+
let recorded: ReadonlyMap<string, readonly string[]> | undefined
|
|
300
|
+
let unsaved: LockfileMemo | undefined
|
|
301
|
+
let served = false
|
|
302
|
+
if (memo !== undefined && (!withDigests || memo.importers !== null)) {
|
|
252
303
|
const now = await hashExtras(workspaceRoot, Object.keys(memo.extras))
|
|
253
304
|
if (Object.entries(memo.extras).every(([rel, h]) => now.hashes[rel] === h)) {
|
|
305
|
+
served = true
|
|
254
306
|
extras = memo.extras
|
|
255
307
|
extraStats = now.stats
|
|
256
|
-
importers = memo.importers
|
|
257
|
-
edges = memo.edges
|
|
308
|
+
importers = memo.importers === null ? undefined : new Map(Object.entries(memo.importers))
|
|
309
|
+
edges = new Map(Object.entries(memo.edges as Record<string, LockfileImporter>))
|
|
310
|
+
recorded = memo.recorded === undefined ? undefined : new Map(Object.entries(memo.recorded))
|
|
258
311
|
}
|
|
259
312
|
}
|
|
260
|
-
if (
|
|
313
|
+
if (!served) {
|
|
261
314
|
const text = decode(bytes)
|
|
262
315
|
const rels = [...new Set(options.extraFiles?.(text) ?? [])].sort()
|
|
263
316
|
const now = await hashExtras(workspaceRoot, rels)
|
|
@@ -266,11 +319,21 @@ export function lockfileClaim(options: LockfileClaimOptions): LockfileClaimHooks
|
|
|
266
319
|
importers =
|
|
267
320
|
scope === 'workspace'
|
|
268
321
|
? new Map<string, string>()
|
|
269
|
-
:
|
|
322
|
+
: withDigests
|
|
323
|
+
? digest(text, new Map(Object.entries(extras)))
|
|
324
|
+
: undefined
|
|
270
325
|
edges = options.importers?.(text) ?? new Map()
|
|
271
|
-
|
|
272
|
-
if (
|
|
273
|
-
|
|
326
|
+
recorded = options.recorded?.(text)
|
|
327
|
+
if (memoises) {
|
|
328
|
+
const kept: LockfileMemo = {
|
|
329
|
+
version,
|
|
330
|
+
lock: lockHash,
|
|
331
|
+
extras,
|
|
332
|
+
importers: importers === undefined ? null : Object.fromEntries(importers),
|
|
333
|
+
edges: Object.fromEntries(edges),
|
|
334
|
+
...(recorded !== undefined ? { recorded: Object.fromEntries(recorded) } : {}),
|
|
335
|
+
}
|
|
336
|
+
if (!writeLockfileMemo(cacheDir, file, claimant, kept)) unsaved = kept
|
|
274
337
|
}
|
|
275
338
|
}
|
|
276
339
|
const rels = Object.keys(extras)
|
|
@@ -278,14 +341,56 @@ export function lockfileClaim(options: LockfileClaimOptions): LockfileClaimHooks
|
|
|
278
341
|
rels.length === 0
|
|
279
342
|
? lockHash
|
|
280
343
|
: xxh3hex([lockHash, ...rels.map((r) => `${r}\0${extras[r]}`)].join('\0'))
|
|
281
|
-
const digests: Digests = {
|
|
282
|
-
|
|
344
|
+
const digests: Digests = {
|
|
345
|
+
lock,
|
|
346
|
+
bytes: lockHash,
|
|
347
|
+
extras: rels,
|
|
348
|
+
importers,
|
|
349
|
+
edges,
|
|
350
|
+
recorded:
|
|
351
|
+
recorded === undefined
|
|
352
|
+
? undefined
|
|
353
|
+
: new Map([...recorded].map(([i, names]) => [i, new Set(names)])),
|
|
354
|
+
}
|
|
355
|
+
seen.set(workspaceRoot, {
|
|
356
|
+
identity,
|
|
357
|
+
extras: rels,
|
|
358
|
+
extraStats,
|
|
359
|
+
digests,
|
|
360
|
+
...(unsaved !== undefined ? { unsaved } : {}),
|
|
361
|
+
})
|
|
283
362
|
return digests
|
|
284
363
|
}
|
|
364
|
+
/**
|
|
365
|
+
* Each project's digest for this run: the lock's, unless an extra file
|
|
366
|
+
* it records moved since (then a digest of the file as it is now, as
|
|
367
|
+
* `vx lock` would record it); undefined when the file is absent.
|
|
368
|
+
*/
|
|
369
|
+
const lockedOnce = (
|
|
370
|
+
ctx: KeyHookContext,
|
|
371
|
+
): Promise<((dir: string) => string | undefined) | undefined> => {
|
|
372
|
+
let p = lockedPerRun.get(ctx)
|
|
373
|
+
if (p === undefined) {
|
|
374
|
+
p = (async () => {
|
|
375
|
+
const rec = await ctx.locked(file)
|
|
376
|
+
if (rec === undefined) return undefined
|
|
377
|
+
const rels = Object.keys(rec.extras)
|
|
378
|
+
const now = await Promise.all(rels.map((rel) => hashExtra(ctx.workspaceRoot, rel)))
|
|
379
|
+
if (now.every((h, i) => h === rec.extras[rels[i]!])) {
|
|
380
|
+
return (dir: string) => rec.digest(dir)
|
|
381
|
+
}
|
|
382
|
+
const digests = await load(ctx.workspaceRoot, ctx.cacheDir, true)
|
|
383
|
+
const importers = digests.importers!
|
|
384
|
+
return (dir: string) => digestFor(importers, importerOf(ctx.workspaceRoot, dir))
|
|
385
|
+
})()
|
|
386
|
+
lockedPerRun.set(ctx, p)
|
|
387
|
+
}
|
|
388
|
+
return p
|
|
389
|
+
}
|
|
285
390
|
const loadOnce = (ctx: KeyHookContext): Promise<Digests> => {
|
|
286
391
|
let p = perRun.get(ctx)
|
|
287
392
|
if (p === undefined) {
|
|
288
|
-
p = load(ctx.workspaceRoot, ctx.cacheDir)
|
|
393
|
+
p = load(ctx.workspaceRoot, ctx.cacheDir, false)
|
|
289
394
|
perRun.set(ctx, p)
|
|
290
395
|
}
|
|
291
396
|
return p
|
|
@@ -333,14 +438,31 @@ export function lockfileClaim(options: LockfileClaimOptions): LockfileClaimHooks
|
|
|
333
438
|
}
|
|
334
439
|
return names
|
|
335
440
|
},
|
|
441
|
+
// What `vx lock` commits, so `--affected` reads the base's digests
|
|
442
|
+
// instead of parsing the base's file.
|
|
443
|
+
async digests(_file: string, ctx: FingerprintContext) {
|
|
444
|
+
if (scope === 'workspace') return undefined
|
|
445
|
+
const digests = await load(ctx.workspaceRoot, ctx.cacheDir, true)
|
|
446
|
+
if (digests.lock === '') return undefined
|
|
447
|
+
const importers = digests.importers!
|
|
448
|
+
return {
|
|
449
|
+
hash: digests.bytes,
|
|
450
|
+
projects: Object.fromEntries(
|
|
451
|
+
ctx.projects.map((p) => [
|
|
452
|
+
p.name,
|
|
453
|
+
digestFor(importers, importerOf(ctx.workspaceRoot, p.dir)),
|
|
454
|
+
]),
|
|
455
|
+
),
|
|
456
|
+
...(digests.extras.length > 0 ? { extras: digests.extras } : {}),
|
|
457
|
+
}
|
|
458
|
+
},
|
|
336
459
|
...(options.extraFiles !== undefined
|
|
337
460
|
? {
|
|
338
461
|
// The memo records which files these exact bytes name, for this
|
|
339
462
|
// parser: `--affected` asks on every diff, and a cold parse of
|
|
340
463
|
// a large lockfile was most of its selection's cost.
|
|
341
464
|
async extraFiles(_file: string, bytes: Uint8Array, ctx: FingerprintContext) {
|
|
342
|
-
const
|
|
343
|
-
const memo = await readMemo(memoFile, version, claimant, xxh3hex(bytes))
|
|
465
|
+
const memo = memoises ? readMemo(ctx.cacheDir, xxh3hex(bytes)) : undefined
|
|
344
466
|
return memo !== undefined
|
|
345
467
|
? Object.keys(memo.extras)
|
|
346
468
|
: options.extraFiles!(decode(bytes))
|
|
@@ -349,17 +471,22 @@ export function lockfileClaim(options: LockfileClaimOptions): LockfileClaimHooks
|
|
|
349
471
|
: {}),
|
|
350
472
|
},
|
|
351
473
|
async key(task, ctx) {
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
[part]: digestFor(digests.importers, importerOf(ctx.workspaceRoot, task.projectDir)),
|
|
474
|
+
if (scope === 'project') {
|
|
475
|
+
const digestOf = await lockedOnce(ctx)
|
|
476
|
+
const digest = digestOf?.(task.projectDir)
|
|
477
|
+
return digest === undefined ? undefined : { [part]: digest }
|
|
357
478
|
}
|
|
479
|
+
const digests = await loadOnce(ctx)
|
|
480
|
+
return digests.lock === '' ? undefined : { [part]: digests.lock }
|
|
358
481
|
},
|
|
359
482
|
async importers(ctx) {
|
|
360
|
-
const digests = await load(ctx.workspaceRoot, ctx.cacheDir)
|
|
483
|
+
const digests = await load(ctx.workspaceRoot, ctx.cacheDir, false)
|
|
361
484
|
return digests.lock === '' ? undefined : digests.edges
|
|
362
485
|
},
|
|
486
|
+
async recorded(ctx) {
|
|
487
|
+
const digests = await load(ctx.workspaceRoot, ctx.cacheDir, false)
|
|
488
|
+
return digests.lock === '' ? undefined : digests.recorded
|
|
489
|
+
},
|
|
363
490
|
}
|
|
364
491
|
}
|
|
365
492
|
|
|
@@ -394,59 +521,6 @@ function importerOf(workspaceRoot: string, projectDir: string): string {
|
|
|
394
521
|
return relPosix(workspaceRoot, projectDir) || '.'
|
|
395
522
|
}
|
|
396
523
|
|
|
397
|
-
async function readMemo(
|
|
398
|
-
memoFile: string,
|
|
399
|
-
version: number,
|
|
400
|
-
claimant: string,
|
|
401
|
-
lock: string,
|
|
402
|
-
): Promise<
|
|
403
|
-
| {
|
|
404
|
-
extras: Record<string, string>
|
|
405
|
-
importers: Map<string, string>
|
|
406
|
-
edges: Map<string, LockfileImporter>
|
|
407
|
-
}
|
|
408
|
-
| undefined
|
|
409
|
-
> {
|
|
410
|
-
try {
|
|
411
|
-
const memo = (await Bun.file(memoFile).json()) as Memo
|
|
412
|
-
if (memo.version !== version || memo.claimant !== claimant || memo.lock !== lock) {
|
|
413
|
-
return undefined
|
|
414
|
-
}
|
|
415
|
-
return {
|
|
416
|
-
extras: memo.extras ?? {},
|
|
417
|
-
importers: new Map(Object.entries(memo.importers)),
|
|
418
|
-
edges: new Map(Object.entries(memo.edges ?? {})),
|
|
419
|
-
}
|
|
420
|
-
} catch {
|
|
421
|
-
return undefined
|
|
422
|
-
}
|
|
423
|
-
}
|
|
424
|
-
|
|
425
|
-
async function writeMemo(
|
|
426
|
-
memoFile: string,
|
|
427
|
-
identity: Pick<Memo, 'version' | 'claimant' | 'lock'>,
|
|
428
|
-
extras: Record<string, string>,
|
|
429
|
-
importers: ReadonlyMap<string, string>,
|
|
430
|
-
edges: ReadonlyMap<string, LockfileImporter>,
|
|
431
|
-
): Promise<void> {
|
|
432
|
-
const memo: Memo = {
|
|
433
|
-
...identity,
|
|
434
|
-
extras,
|
|
435
|
-
importers: Object.fromEntries(importers),
|
|
436
|
-
...(edges.size > 0 ? { edges: Object.fromEntries(edges) } : {}),
|
|
437
|
-
}
|
|
438
|
-
// Write-then-rename: a reader never sees a half-written memo, and two
|
|
439
|
-
// concurrent runs each land a whole one.
|
|
440
|
-
const tmp = `${memoFile}.tmp-${process.pid}-${Date.now()}`
|
|
441
|
-
try {
|
|
442
|
-
await Bun.write(tmp, JSON.stringify(memo))
|
|
443
|
-
await rename(tmp, memoFile)
|
|
444
|
-
} catch {
|
|
445
|
-
// The memo is a speed-up, never a requirement: the digests are
|
|
446
|
-
// computed either way, and the next run computes them again.
|
|
447
|
-
}
|
|
448
|
-
}
|
|
449
|
-
|
|
450
524
|
function decode(bytes: Uint8Array): string {
|
|
451
525
|
return new TextDecoder().decode(bytes)
|
|
452
526
|
}
|
|
@@ -146,6 +146,8 @@ export interface SaveMissArgs {
|
|
|
146
146
|
export interface SaveFacts {
|
|
147
147
|
saveMs?: number
|
|
148
148
|
artifactBytes?: number
|
|
149
|
+
/** The saved files' bytes, as a hit's `outputBytes` counts them: the row's `outputs`. */
|
|
150
|
+
outputBytes?: number
|
|
149
151
|
/** The entry landed: the outcome's `saved`, set only once it has. */
|
|
150
152
|
saved?: true
|
|
151
153
|
}
|
|
@@ -239,6 +241,8 @@ export async function saveMiss(a: SaveMissArgs): Promise<{ landed: Promise<SaveF
|
|
|
239
241
|
},
|
|
240
242
|
})
|
|
241
243
|
endSave()
|
|
244
|
+
const bytes = cache.outputBytes?.(a.hash)
|
|
245
|
+
if (bytes !== undefined) facts.outputBytes = bytes
|
|
242
246
|
if (a.measure === true) {
|
|
243
247
|
facts.saveMs = Math.round(performance.now() - saveStart)
|
|
244
248
|
// No local artifact (local writes off): no size to say.
|
|
@@ -294,9 +298,26 @@ export type UnsavedArgs = Pick<
|
|
|
294
298
|
* A miss that ran here and saves nothing (it failed, the policy writes
|
|
295
299
|
* nothing, an upstream failed, its key no longer held) still wrote its
|
|
296
300
|
* outputs, and a same-project reader after it must see them as it sees a
|
|
297
|
-
* save's: resolved and marked the same way.
|
|
301
|
+
* save's: resolved and marked the same way. Returns their bytes, the
|
|
302
|
+
* row's `outputs`: blank, an unsaved miss read as one that wrote nothing.
|
|
298
303
|
*/
|
|
299
|
-
export async function markUnsaved(a: UnsavedArgs): Promise<
|
|
304
|
+
export async function markUnsaved(a: UnsavedArgs): Promise<number> {
|
|
305
|
+
const { outputFiles, wsOutputFiles } = await resolveUnsaved(a)
|
|
306
|
+
markWritten(a, outputFiles, wsOutputFiles)
|
|
307
|
+
return sizeOf(a.node.projectDir, outputFiles) + sizeOf(a.workspaceRoot, wsOutputFiles)
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/** An unsaved miss nothing reads after: only the bytes of its outputs, its row's `outputs`. */
|
|
311
|
+
export async function unsavedOutputBytes(
|
|
312
|
+
a: Pick<UnsavedArgs, 'node' | 'workspaceRoot' | 'nestedProjectDirs' | 'outputs' | 'wsOutputs'>,
|
|
313
|
+
): Promise<number> {
|
|
314
|
+
const { outputFiles, wsOutputFiles } = await resolveUnsaved(a)
|
|
315
|
+
return sizeOf(a.node.projectDir, outputFiles) + sizeOf(a.workspaceRoot, wsOutputFiles)
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
async function resolveUnsaved(
|
|
319
|
+
a: Pick<UnsavedArgs, 'node' | 'workspaceRoot' | 'nestedProjectDirs' | 'outputs' | 'wsOutputs'>,
|
|
320
|
+
): Promise<{ outputFiles: string[]; wsOutputFiles: string[] }> {
|
|
300
321
|
const endResolve = span('miss: resolve outputs')
|
|
301
322
|
const outputFiles = await resolveOutputs({
|
|
302
323
|
projectDir: a.node.projectDir,
|
|
@@ -308,7 +329,14 @@ export async function markUnsaved(a: UnsavedArgs): Promise<void> {
|
|
|
308
329
|
outputs: a.wsOutputs,
|
|
309
330
|
})
|
|
310
331
|
endResolve()
|
|
311
|
-
|
|
332
|
+
return { outputFiles, wsOutputFiles }
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
/** The bytes of `files` (absolute, or under `base`); one gone since it was listed counts none. */
|
|
336
|
+
function sizeOf(base: string, files: readonly string[]): number {
|
|
337
|
+
let n = 0
|
|
338
|
+
for (const f of files) n += statSync(path.resolve(base, f), { throwIfNoEntry: false })?.size ?? 0
|
|
339
|
+
return n
|
|
312
340
|
}
|
|
313
341
|
|
|
314
342
|
/**
|
|
@@ -283,7 +283,9 @@ export async function executePersistentTask(args: ExecuteArgs): Promise<TaskOutc
|
|
|
283
283
|
const message = err instanceof Error ? err.message : String(err)
|
|
284
284
|
// The task's OWN stream, not the process's: the frame is where a
|
|
285
285
|
// reader looks for why a task failed, and a run with a custom logger
|
|
286
|
-
// (an embedder) never saw a bare stderr write at all.
|
|
286
|
+
// (an embedder) never saw a bare stderr write at all. What waited on
|
|
287
|
+
// an open line goes first.
|
|
288
|
+
ends.end()
|
|
287
289
|
log.taskStderr(
|
|
288
290
|
node,
|
|
289
291
|
ends.own(`[vx] ${node.id}: persistent task failed to become ready: ${message}\n`),
|
package/src/orchestrator/plan.ts
CHANGED
|
@@ -225,7 +225,11 @@ export async function plan(args: PlanArgs): Promise<RunPlan> {
|
|
|
225
225
|
)
|
|
226
226
|
const dependents = new Map<string, string[]>()
|
|
227
227
|
for (const node of args.nodes.values())
|
|
228
|
-
for (const dep of node.deps)
|
|
228
|
+
for (const dep of node.deps) {
|
|
229
|
+
const ds = dependents.get(dep)
|
|
230
|
+
if (ds === undefined) dependents.set(dep, [node.id])
|
|
231
|
+
else ds.push(node.id)
|
|
232
|
+
}
|
|
229
233
|
for (const id of idleServers(args.nodes, dependents, restoreTier))
|
|
230
234
|
cacheStatusById.set(id, 'not-started')
|
|
231
235
|
}
|
|
@@ -33,6 +33,7 @@ import type {
|
|
|
33
33
|
DiscoverContext,
|
|
34
34
|
ExecutorContext,
|
|
35
35
|
FingerprintChange,
|
|
36
|
+
FingerprintDigests,
|
|
36
37
|
FingerprintContext,
|
|
37
38
|
GraphHookContext,
|
|
38
39
|
KeyHookContext,
|
|
@@ -44,6 +45,7 @@ import type {
|
|
|
44
45
|
ConfigContext,
|
|
45
46
|
} from './plugin.js'
|
|
46
47
|
import type { ProjectMeta } from '../workspace/index.js'
|
|
48
|
+
import { hashExtra } from './lockfile-claim.js'
|
|
47
49
|
|
|
48
50
|
/** `a string`, `an array`, `null`, `a number` — for a refusal that names what came back. */
|
|
49
51
|
function describeValue(v: unknown): string {
|
|
@@ -392,6 +394,65 @@ export async function claimedAffected(
|
|
|
392
394
|
ctx: FingerprintContext,
|
|
393
395
|
): Promise<Set<string> | undefined> {
|
|
394
396
|
const answer = await safe(plugin, 'fingerprint', () => plugin.fingerprint!.affected(change, ctx))
|
|
397
|
+
return projectNames(plugin, answer, ctx)
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
/**
|
|
401
|
+
* `FingerprintClaim.digests`' answer for `file`, each extra file it names
|
|
402
|
+
* with its hash now (`hashExtra`, as a run checks it); `undefined` when the
|
|
403
|
+
* plugin has none. A digest of a name no project carries, or a value that
|
|
404
|
+
* is not a string, is refused by name: `vx lock` would commit it.
|
|
405
|
+
*/
|
|
406
|
+
export async function claimedDigests(
|
|
407
|
+
plugin: VxPlugin,
|
|
408
|
+
file: string,
|
|
409
|
+
ctx: FingerprintContext,
|
|
410
|
+
): Promise<
|
|
411
|
+
| {
|
|
412
|
+
hash: string
|
|
413
|
+
projects: FingerprintDigests['projects']
|
|
414
|
+
extras?: Readonly<Record<string, string>>
|
|
415
|
+
}
|
|
416
|
+
| undefined
|
|
417
|
+
> {
|
|
418
|
+
const digests = plugin.fingerprint?.digests
|
|
419
|
+
if (digests === undefined) return undefined
|
|
420
|
+
const answer: unknown = await safe(plugin, 'fingerprint', () =>
|
|
421
|
+
digests.call(plugin.fingerprint, file, ctx),
|
|
422
|
+
)
|
|
423
|
+
if (answer === undefined) return undefined
|
|
424
|
+
const { hash, projects, extras } = (answer ?? {}) as Partial<FingerprintDigests>
|
|
425
|
+
if (
|
|
426
|
+
typeof hash !== 'string' ||
|
|
427
|
+
typeof projects !== 'object' ||
|
|
428
|
+
projects === null ||
|
|
429
|
+
Object.values(projects).some((d) => typeof d !== 'string') ||
|
|
430
|
+
(extras !== undefined &&
|
|
431
|
+
(!Array.isArray(extras) || extras.some((rel) => typeof rel !== 'string')))
|
|
432
|
+
) {
|
|
433
|
+
throw new UserError(
|
|
434
|
+
`plugin '${plugin.name}' failed in fingerprint: digests returned ${describeValue(answer)}, not { hash, projects: { name: digest }, extras?: [path] }`,
|
|
435
|
+
)
|
|
436
|
+
}
|
|
437
|
+
const known = new Set(ctx.projects.map((p) => p.name))
|
|
438
|
+
const stray = Object.keys(projects).find((n) => !known.has(n))
|
|
439
|
+
if (stray !== undefined) {
|
|
440
|
+
throw new UserError(
|
|
441
|
+
`plugin '${plugin.name}' failed in fingerprint: digest of ${JSON.stringify(stray)}, which is not a project in this workspace`,
|
|
442
|
+
)
|
|
443
|
+
}
|
|
444
|
+
if (extras === undefined || extras.length === 0) return { hash, projects }
|
|
445
|
+
const hashed = await Promise.all(
|
|
446
|
+
extras.map(async (rel) => [rel, await hashExtra(ctx.workspaceRoot, rel)] as const),
|
|
447
|
+
)
|
|
448
|
+
return { hash, projects, extras: Object.fromEntries(hashed) }
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
async function projectNames(
|
|
452
|
+
plugin: VxPlugin,
|
|
453
|
+
answer: unknown,
|
|
454
|
+
ctx: FingerprintContext,
|
|
455
|
+
): Promise<Set<string> | undefined> {
|
|
395
456
|
if (answer === undefined) return undefined
|
|
396
457
|
// A plain object is no list: `for…of` threw "is not iterable", naming
|
|
397
458
|
// neither the plugin nor the hook.
|
|
@@ -27,9 +27,9 @@ import {
|
|
|
27
27
|
type PluginSecrets,
|
|
28
28
|
SecretBroker,
|
|
29
29
|
thrownMessage,
|
|
30
|
-
ui,
|
|
31
30
|
UserError,
|
|
32
31
|
} from '../util/index.js'
|
|
32
|
+
import * as ui from '../ui/index.js'
|
|
33
33
|
import type { ProjectMeta } from '../workspace/index.js'
|
|
34
34
|
import type { EventBus, Logger, RunStartInfo } from '../report/index.js'
|
|
35
35
|
import type { CiContext } from './run-context.js'
|
|
@@ -460,7 +460,29 @@ export interface GraphHookContext extends BaseContext {
|
|
|
460
460
|
readonly requested: readonly string[]
|
|
461
461
|
}
|
|
462
462
|
|
|
463
|
-
export interface KeyHookContext extends BaseContext {
|
|
463
|
+
export interface KeyHookContext extends BaseContext {
|
|
464
|
+
/**
|
|
465
|
+
* What `vx lock` recorded of the claimed `file` in
|
|
466
|
+
* `.vx/dependencies.lock.json` (`FingerprintClaim.digests`): a run keys
|
|
467
|
+
* from the record and never parses the file. Undefined when the file is
|
|
468
|
+
* absent and unrecorded. Refuses, naming `vx lock`, when the lock is
|
|
469
|
+
* absent or records no such present file; a recorded file whose bytes
|
|
470
|
+
* moved since refused the run before the key stage.
|
|
471
|
+
*/
|
|
472
|
+
locked(file: string): Promise<LockedFile | undefined>
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
/** `KeyHookContext.locked`'s answer. */
|
|
476
|
+
export interface LockedFile {
|
|
477
|
+
/**
|
|
478
|
+
* The other files the digests read → each one's hash then (`hashExtra`).
|
|
479
|
+
* Not checked by core: a `link:` directory is edited with no install,
|
|
480
|
+
* so a claimant whose extra file moved digests anew.
|
|
481
|
+
*/
|
|
482
|
+
readonly extras: Readonly<Record<string, string>>
|
|
483
|
+
/** The project at `projectDir`'s digest; undefined when none. Refuses for a project the lock does not list. */
|
|
484
|
+
digest(projectDir: string): string | undefined
|
|
485
|
+
}
|
|
464
486
|
|
|
465
487
|
/** A plugin's claim on workspace-root files — see `VxPlugin.fingerprint`. */
|
|
466
488
|
export interface FingerprintClaim {
|
|
@@ -491,6 +513,29 @@ export interface FingerprintClaim {
|
|
|
491
513
|
bytes: Uint8Array,
|
|
492
514
|
ctx: FingerprintContext,
|
|
493
515
|
): readonly string[] | Promise<readonly string[]>
|
|
516
|
+
/**
|
|
517
|
+
* Each project's digest of the claimed `file` as it is now, by project
|
|
518
|
+
* name, with `hash`, the xxh3 of the bytes they came from (`xxh3hex`),
|
|
519
|
+
* and `extras`, the other files they read. `vx lock` records them in
|
|
520
|
+
* `.vx/dependencies.lock.json` with each extra file's hash; a run keys
|
|
521
|
+
* from that record (`KeyHookContext.locked`), and `--affected`
|
|
522
|
+
* reads the base's copy there instead of asking `affected` to parse the
|
|
523
|
+
* base side, when its `hash` is the base file's xxh3. `undefined`:
|
|
524
|
+
* nothing to record.
|
|
525
|
+
*/
|
|
526
|
+
digests?(
|
|
527
|
+
file: string,
|
|
528
|
+
ctx: FingerprintContext,
|
|
529
|
+
): FingerprintDigests | undefined | Promise<FingerprintDigests | undefined>
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
/** `FingerprintClaim.digests`' answer. */
|
|
533
|
+
export interface FingerprintDigests {
|
|
534
|
+
readonly hash: string
|
|
535
|
+
/** Project name → its digest of the file. */
|
|
536
|
+
readonly projects: Readonly<Record<string, string>>
|
|
537
|
+
/** Root-relative files besides `file` whose bytes the digests read (a `bun.lock` patch). */
|
|
538
|
+
readonly extras?: readonly string[]
|
|
494
539
|
}
|
|
495
540
|
|
|
496
541
|
export interface FingerprintChange {
|
|
@@ -527,9 +572,9 @@ export interface AdmitContext {
|
|
|
527
572
|
}
|
|
528
573
|
|
|
529
574
|
export interface ScheduleHookContext extends BaseContext {
|
|
530
|
-
/** The run's local cache
|
|
575
|
+
/** The run's local cache: `localHistory(localCache)` reads the run history a policy learns from. */
|
|
531
576
|
readonly localCache: Cache
|
|
532
|
-
/** The run's branch (`currentBranch`), the timings a forecast reads first: pass it to `
|
|
577
|
+
/** The run's branch (`currentBranch`), the timings a forecast reads first: pass it to `localHistory`. */
|
|
533
578
|
readonly branch: string | null
|
|
534
579
|
/** The refs the run reads, in order (`CacheContext.refs`): a remote's timings are read by them too. */
|
|
535
580
|
readonly refs: readonly string[]
|