@vzn/vx 0.0.650 → 0.0.651
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -20
- package/package.json +5 -5
- package/schemas/last.json +15 -0
- package/src/bin.ts +9 -31
- package/src/cache/cache.ts +202 -25
- package/src/cache/file-hashes.ts +3 -0
- package/src/cache/git-inputs.ts +188 -39
- package/src/cache/index.ts +1 -0
- package/src/cache/inputs.ts +3 -0
- package/src/cache/layer.ts +11 -0
- package/src/cache/layered-cache.ts +4 -0
- package/src/cache/linked-inputs.ts +3 -0
- package/src/cache/lockfile-memo.ts +3 -2
- package/src/cache/output-index.ts +5 -2
- package/src/cache/remote-lane.ts +2 -1
- package/src/cache/run-history.ts +11 -1
- package/src/cache/schema.ts +11 -1
- package/src/cli/cache.ts +26 -16
- package/src/cli/completions.ts +4 -2
- package/src/cli/docs.ts +10 -11
- package/src/cli/help.ts +12 -23
- package/src/cli/index.ts +4 -9
- package/src/cli/info.ts +15 -7
- package/src/cli/init.ts +23 -15
- package/src/cli/last.ts +213 -203
- package/src/cli/lock.ts +55 -5
- package/src/cli/plan-format.ts +100 -118
- package/src/cli/plugin-commands.ts +8 -2
- package/src/cli/plugin-templates.ts +2 -2
- package/src/cli/run.ts +46 -27
- package/src/cli/select.ts +43 -5
- package/src/cli/show.ts +38 -42
- package/src/cli/task-verb.ts +22 -0
- package/src/cli/upgrade.ts +3 -1
- package/src/cli/watch-fs.ts +6 -1
- package/src/cli/watch-judge.ts +2 -1
- package/src/cli/watch.ts +83 -44
- package/src/cli/why.ts +123 -89
- package/src/cli/workspace-config.ts +2 -1
- package/src/exec/executor.ts +2 -1
- package/src/exec/kill-tree.ts +58 -5
- package/src/exec/local-executor.ts +1 -1
- package/src/exec/pipes.ts +152 -0
- package/src/exec/runner.ts +199 -182
- package/src/exec/sandbox-binds.ts +1 -1
- package/src/exec/sandbox-runtime.ts +11 -2
- package/src/exec/spawn-off.ts +154 -0
- package/src/exec/warm-shell.ts +146 -0
- package/src/graph/critical-path.ts +40 -28
- package/src/graph/priorities.ts +98 -44
- package/src/graph/scheduler.ts +14 -10
- package/src/graph/task-graph.ts +27 -1
- package/src/handoff.ts +10 -7
- package/src/history/history.ts +5 -0
- package/src/history/metrics.ts +9 -1
- package/src/index.ts +3 -1
- package/src/orchestrator/doctor.ts +1 -1
- package/src/orchestrator/execute-task.ts +16 -2
- package/src/orchestrator/graph-memo.ts +50 -1
- package/src/orchestrator/graph-stage.ts +6 -0
- package/src/orchestrator/index.ts +3 -0
- package/src/orchestrator/live-usage.ts +16 -11
- package/src/orchestrator/locked-digests.ts +63 -0
- package/src/orchestrator/lockfile-claim.ts +108 -29
- package/src/orchestrator/miss-save.ts +31 -3
- package/src/orchestrator/persistent-task.ts +3 -1
- package/src/orchestrator/plan.ts +5 -1
- package/src/orchestrator/plugin-host.ts +61 -0
- package/src/orchestrator/plugin.ts +47 -2
- package/src/orchestrator/prepare.ts +16 -1
- package/src/orchestrator/projects.ts +4 -1
- package/src/orchestrator/run-context.ts +29 -10
- package/src/orchestrator/run-failures.ts +1 -1
- package/src/orchestrator/run-records.ts +10 -0
- package/src/orchestrator/run.ts +53 -22
- package/src/orchestrator/sandbox-request.ts +2 -2
- package/src/orchestrator/signals.ts +8 -5
- package/src/orchestrator/task-hash.ts +17 -3
- package/src/report/events.ts +3 -11
- package/src/report/forecast.ts +10 -32
- package/src/report/framed-output.ts +272 -173
- package/src/report/index.ts +8 -3
- package/src/report/logger.ts +73 -38
- package/src/report/plain-output.ts +1 -1
- package/src/report/status-line.ts +49 -357
- package/src/report/summary.ts +100 -131
- package/src/ui/ansi.ts +48 -0
- package/src/ui/components.ts +506 -0
- package/src/ui/index.ts +79 -0
- package/src/ui/kit.ts +37 -0
- package/src/ui/live-region.ts +363 -0
- package/src/{util/refusal-notice.ts → ui/refusal.ts} +7 -5
- package/src/ui/terminal.ts +51 -0
- package/src/ui/theme.ts +224 -0
- package/src/ui/version-line.ts +28 -0
- package/src/util/config-frame.ts +65 -10
- package/src/util/edit-distance.ts +6 -1
- package/src/util/index.ts +2 -15
- package/src/util/paths.ts +20 -2
- package/src/util/secret-mask.ts +39 -3
- package/src/util/settle.ts +31 -0
- package/src/util/timing.ts +4 -16
- package/src/workspace/affected.ts +89 -30
- package/src/workspace/config-schema.ts +4 -2
- package/src/workspace/dependencies-lock.ts +191 -0
- package/src/workspace/filter.ts +28 -0
- package/src/workspace/index.ts +9 -0
- package/src/workspace/lockfile.ts +159 -6
- package/src/workspace/migration.ts +4 -3
- package/src/workspace/workspace.ts +3 -0
- package/src/util/colors.ts +0 -133
- package/src/util/report.ts +0 -44
- package/src/util/ui.ts +0 -258
- /package/src/{report/path-links.ts → ui/links.ts} +0 -0
- /package/src/{report → ui}/terminal-signals.ts +0 -0
|
@@ -72,6 +72,7 @@ import {
|
|
|
72
72
|
bracketLiteralHint,
|
|
73
73
|
markUnsaved,
|
|
74
74
|
saveMiss,
|
|
75
|
+
unsavedOutputBytes,
|
|
75
76
|
type OutputDirSnapshot,
|
|
76
77
|
type SaveFacts,
|
|
77
78
|
} from './miss-save.js'
|
|
@@ -1347,6 +1348,8 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1347
1348
|
let movedFails: string | undefined
|
|
1348
1349
|
let own: Awaited<ReturnType<typeof ownOutputs>>
|
|
1349
1350
|
let saved = false
|
|
1351
|
+
// An unsaved miss's outputs, as a save's are counted: its row's `outputs`.
|
|
1352
|
+
let unsavedBytes: number | undefined
|
|
1350
1353
|
if (
|
|
1351
1354
|
effectiveExitCode === 0 &&
|
|
1352
1355
|
willSave &&
|
|
@@ -1408,7 +1411,17 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1408
1411
|
// entry is in; without one, the save is taken as done.
|
|
1409
1412
|
if (args.deferredSaves !== undefined) args.deferredSaves.set(node.id, landed)
|
|
1410
1413
|
else saved = true
|
|
1411
|
-
} else if (cfgCacheable && !remoteOnly && !deferralRequested && args.noDependants
|
|
1414
|
+
} else if (cfgCacheable && !remoteOnly && !deferralRequested && args.noDependants === true) {
|
|
1415
|
+
// Nothing reads after it, so nothing to mark: only its row's `outputs`.
|
|
1416
|
+
if (effectiveExitCode === 0)
|
|
1417
|
+
unsavedBytes = await unsavedOutputBytes({
|
|
1418
|
+
node,
|
|
1419
|
+
workspaceRoot: args.workspaceRoot,
|
|
1420
|
+
nestedProjectDirs: args.nestedProjectDirs,
|
|
1421
|
+
outputs,
|
|
1422
|
+
wsOutputs,
|
|
1423
|
+
})
|
|
1424
|
+
} else if (cfgCacheable && !remoteOnly && !deferralRequested) {
|
|
1412
1425
|
// Ran here and saves nothing — failed, a read-only policy, a tainted
|
|
1413
1426
|
// upstream, a key that no longer held — yet it wrote what a save would
|
|
1414
1427
|
// have marked, and may have rewritten an input the re-check before a
|
|
@@ -1422,7 +1435,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1422
1435
|
if (moved !== undefined) {
|
|
1423
1436
|
forgetUndeclaredWrites(args, wsOutputs.length > 0 ? 'workspace' : 'project')
|
|
1424
1437
|
} else {
|
|
1425
|
-
await markUnsaved({
|
|
1438
|
+
unsavedBytes = await markUnsaved({
|
|
1426
1439
|
node,
|
|
1427
1440
|
workspaceRoot: args.workspaceRoot,
|
|
1428
1441
|
nestedProjectDirs: args.nestedProjectDirs,
|
|
@@ -1572,6 +1585,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1572
1585
|
...(unkeyed ? { unkeyed: true as const } : {}),
|
|
1573
1586
|
...(keylessBy !== undefined ? { keylessBy } : {}),
|
|
1574
1587
|
...(saved ? { saved: true as const } : {}),
|
|
1588
|
+
...(unsavedBytes !== undefined ? { outputBytes: unsavedBytes } : {}),
|
|
1575
1589
|
...(effectiveExitCode !== 0 && output !== undefined
|
|
1576
1590
|
? { failedOutput: failedOutputLog(output, willSave || args.keepRun !== undefined, storeMask) }
|
|
1577
1591
|
: {}),
|
|
@@ -12,7 +12,12 @@
|
|
|
12
12
|
// it is never remembered.
|
|
13
13
|
|
|
14
14
|
import type { FoldMemo } from '../cache/index.js'
|
|
15
|
-
import {
|
|
15
|
+
import {
|
|
16
|
+
buildTaskGraph,
|
|
17
|
+
computeReverseDepCount,
|
|
18
|
+
declaredTask,
|
|
19
|
+
type TaskNode,
|
|
20
|
+
} from '../graph/index.js'
|
|
16
21
|
import { xxh3hex } from '../util/index.js'
|
|
17
22
|
import type { ProjectMeta } from '../workspace/index.js'
|
|
18
23
|
import { codeIdentity } from './run-keys.js'
|
|
@@ -138,3 +143,47 @@ function rebuilt(
|
|
|
138
143
|
}
|
|
139
144
|
return nodes
|
|
140
145
|
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* The run's dependents count, the scheduler's baseline and every row's
|
|
149
|
+
* `blocks`, read back with the graph it was counted on: the closure over a
|
|
150
|
+
* 9,603-task graph (11k nodes) was ~100 ms of every warm run, the same each
|
|
151
|
+
* time (2026-10-29). `token` names the graph's edges (`memoGraph`, undefined
|
|
152
|
+
* once anything edited them), the node ids which of them this run kept
|
|
153
|
+
* (`--affected` prunes under the same token).
|
|
154
|
+
*/
|
|
155
|
+
export function graphCounts(
|
|
156
|
+
nodes: ReadonlyMap<string, TaskNode>,
|
|
157
|
+
folds: FoldMemo | undefined,
|
|
158
|
+
token: string | undefined,
|
|
159
|
+
): Map<string, number> {
|
|
160
|
+
if (folds === undefined || token === undefined) return computeReverseDepCount(nodes)
|
|
161
|
+
// Not `counts`: those rows held a second count the rows no longer show.
|
|
162
|
+
const id = `\0dependents\0${token}`
|
|
163
|
+
const probe = xxh3hex([...nodes.keys()].join('\0'))
|
|
164
|
+
const row = folds.get(id, probe)
|
|
165
|
+
if (row !== undefined) {
|
|
166
|
+
const back = countsOf(row, nodes)
|
|
167
|
+
if (back !== undefined) return back
|
|
168
|
+
}
|
|
169
|
+
const counts = computeReverseDepCount(nodes)
|
|
170
|
+
folds.set(id, probe, JSON.stringify([...nodes.keys()].map((k) => counts.get(k)!)))
|
|
171
|
+
return counts
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function countsOf(
|
|
175
|
+
row: string,
|
|
176
|
+
nodes: ReadonlyMap<string, TaskNode>,
|
|
177
|
+
): Map<string, number> | undefined {
|
|
178
|
+
let all: number[]
|
|
179
|
+
try {
|
|
180
|
+
all = JSON.parse(row) as number[]
|
|
181
|
+
} catch {
|
|
182
|
+
return undefined
|
|
183
|
+
}
|
|
184
|
+
if (all?.length !== nodes.size) return undefined
|
|
185
|
+
const counts = new Map<string, number>()
|
|
186
|
+
let i = 0
|
|
187
|
+
for (const id of nodes.keys()) counts.set(id, all[i++]!)
|
|
188
|
+
return counts
|
|
189
|
+
}
|
|
@@ -29,6 +29,12 @@ export async function applyGraphStage(
|
|
|
29
29
|
// keyed and cleaned its outputs there: past the hard project boundary.
|
|
30
30
|
const dirs = new Map(args.projectMetas.map((m) => [m.name, path.resolve(m.dir)]))
|
|
31
31
|
const shown = (dir: string): string => relPosix(args.workspaceRoot, dir) || '.'
|
|
32
|
+
// The lock shares a task config two tasks declare alike: an edit in place
|
|
33
|
+
// works on this node's own copy.
|
|
34
|
+
if (plugins.some((p) => p.graph !== undefined)) {
|
|
35
|
+
for (const n of nodes.values())
|
|
36
|
+
if (!isDefaultBuild(n.config)) n.config = structuredClone(n.config)
|
|
37
|
+
}
|
|
32
38
|
await applyGraphHooks(
|
|
33
39
|
plugins,
|
|
34
40
|
nodes,
|
|
@@ -60,6 +60,7 @@ export {
|
|
|
60
60
|
type CommandContext,
|
|
61
61
|
type GraphHookContext,
|
|
62
62
|
type KeyHookContext,
|
|
63
|
+
type LockedFile,
|
|
63
64
|
type PluginCommand,
|
|
64
65
|
type ProjectHookContext,
|
|
65
66
|
type ScheduleHookContext,
|
|
@@ -78,6 +79,7 @@ export {
|
|
|
78
79
|
type RunReport,
|
|
79
80
|
type RunReportTask,
|
|
80
81
|
type FingerprintChange,
|
|
82
|
+
type FingerprintDigests,
|
|
81
83
|
type FingerprintClaim,
|
|
82
84
|
type FingerprintContext,
|
|
83
85
|
type VxPlugin,
|
|
@@ -95,6 +97,7 @@ export {
|
|
|
95
97
|
applyScheduleHooks,
|
|
96
98
|
claimedAffected,
|
|
97
99
|
claimedExtraFiles,
|
|
100
|
+
claimedDigests,
|
|
98
101
|
fingerprintClaims,
|
|
99
102
|
hasHook,
|
|
100
103
|
REMOTE_CACHE_METHODS,
|
|
@@ -1,6 +1,9 @@
|
|
|
1
|
-
// The live list's `cpu` and `mem`:
|
|
2
|
-
//
|
|
3
|
-
//
|
|
1
|
+
// The live list's `cpu` and `mem`: each running task's process tree's
|
|
2
|
+
// average cores since it started and its peak memory so far, the values
|
|
3
|
+
// its results row shows once it ends (2026-10-29: the live row showed the
|
|
4
|
+
// last half second's, and a task's row changed meaning as it finished).
|
|
5
|
+
// Looked at on a timer only once the list asks for a row, so a run with
|
|
6
|
+
// no live list (a pipe, CI) registers pids and reads nothing.
|
|
4
7
|
// The look is async and off the dispatch path: a slow one skips a tick.
|
|
5
8
|
|
|
6
9
|
import { sampleTrees, type TreeUsage } from '../exec/index.js'
|
|
@@ -19,7 +22,10 @@ const everyInterval: Every = (ms, fn) => {
|
|
|
19
22
|
|
|
20
23
|
interface Tracked {
|
|
21
24
|
pid: number
|
|
22
|
-
|
|
25
|
+
startedAt: number
|
|
26
|
+
/** The most CPU time the tree has shown: a descendant that exits takes its share out of the sum. */
|
|
27
|
+
cpuMs: number
|
|
28
|
+
peakRssBytes: number
|
|
23
29
|
now?: LiveUsage
|
|
24
30
|
}
|
|
25
31
|
|
|
@@ -51,13 +57,12 @@ export function createLiveUsage(
|
|
|
51
57
|
for (const [id, t] of entries) {
|
|
52
58
|
const u = usage.get(t.pid)
|
|
53
59
|
if (u === undefined || tracked.get(id) !== t) continue
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
if (
|
|
57
|
-
const cores =
|
|
58
|
-
t.now = { cores: Math.round(cores * 100) / 100, rssBytes:
|
|
60
|
+
t.cpuMs = Math.max(t.cpuMs, u.cpuMs)
|
|
61
|
+
t.peakRssBytes = Math.max(t.peakRssBytes, u.rssBytes)
|
|
62
|
+
if (ts > t.startedAt) {
|
|
63
|
+
const cores = t.cpuMs / (ts - t.startedAt)
|
|
64
|
+
t.now = { cores: Math.round(cores * 100) / 100, rssBytes: t.peakRssBytes }
|
|
59
65
|
}
|
|
60
|
-
t.last = { ts, cpuMs: u.cpuMs }
|
|
61
66
|
}
|
|
62
67
|
} catch {
|
|
63
68
|
// A failed look is a missing point, never a broken run.
|
|
@@ -71,7 +76,7 @@ export function createLiveUsage(
|
|
|
71
76
|
|
|
72
77
|
return {
|
|
73
78
|
track(taskId, pid) {
|
|
74
|
-
const t: Tracked = { pid }
|
|
79
|
+
const t: Tracked = { pid, startedAt: now(), cpuMs: 0, peakRssBytes: 0 }
|
|
75
80
|
tracked.set(taskId, t)
|
|
76
81
|
start()
|
|
77
82
|
return () => {
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
// A run keys a claimed file from what `vx lock` recorded of it in
|
|
2
|
+
// `.vx/dependencies.lock.json`, as it takes configs from the config lock:
|
|
3
|
+
// the digests are read, never derived, so no run parses a lockfile for
|
|
4
|
+
// its key. The record holds only while the file's bytes hash to what was
|
|
5
|
+
// recorded: one that moved (an install) refuses the run before any key is
|
|
6
|
+
// folded, naming the file and `vx lock`. A present file with no record,
|
|
7
|
+
// or no lock at all, refuses when a claimant asks for it: an absent record
|
|
8
|
+
// is never read as "no dependencies". The other files the digests read
|
|
9
|
+
// (`extras`: a `link:` directory, a patch) are handed to the claimant with
|
|
10
|
+
// their recorded hashes: a linked directory is edited with no install, so
|
|
11
|
+
// one that moved is the claimant's to digest anew, not a refusal.
|
|
12
|
+
|
|
13
|
+
import {
|
|
14
|
+
DEPENDENCIES_LOCK_NAME,
|
|
15
|
+
type DependenciesLock,
|
|
16
|
+
lockKey,
|
|
17
|
+
parseDependenciesLock,
|
|
18
|
+
recordedDigests,
|
|
19
|
+
} from '../workspace/index.js'
|
|
20
|
+
import { UserError, xxh3hex } from '../util/index.js'
|
|
21
|
+
import type { KeyHookContext, LockedFile } from './plugin.js'
|
|
22
|
+
|
|
23
|
+
const refused = (message: string): UserError => new UserError(message, 'VX_E_LOCK')
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* `KeyHookContext.locked` for one run, over the claimed files' bytes as
|
|
27
|
+
* the run read them (`readClaimedFiles`; absent ones are missing) and the
|
|
28
|
+
* lock's bytes (null when there is none). Each recorded claimed file is
|
|
29
|
+
* checked first: one whose bytes moved, or that is gone, throws here.
|
|
30
|
+
*/
|
|
31
|
+
export function lockedFiles(
|
|
32
|
+
workspaceRoot: string,
|
|
33
|
+
claimed: readonly string[],
|
|
34
|
+
bytes: ReadonlyMap<string, Uint8Array>,
|
|
35
|
+
lockBytes: Uint8Array | null,
|
|
36
|
+
): KeyHookContext['locked'] {
|
|
37
|
+
const lock: DependenciesLock | null = parseDependenciesLock(lockBytes)
|
|
38
|
+
const records = new Map<string, LockedFile>()
|
|
39
|
+
for (const file of claimed) {
|
|
40
|
+
const rec = lock === null ? undefined : recordedDigests(lock, file)
|
|
41
|
+
if (rec === undefined) continue
|
|
42
|
+
const now = bytes.get(file)
|
|
43
|
+
if (now === undefined || xxh3hex(now) !== rec.hash) {
|
|
44
|
+
throw refused(`${file} changed since \`vx lock\`: run \`vx lock\``)
|
|
45
|
+
}
|
|
46
|
+
records.set(file, {
|
|
47
|
+
extras: rec.extras,
|
|
48
|
+
digest(projectDir) {
|
|
49
|
+
const dir = lockKey(workspaceRoot, projectDir)
|
|
50
|
+
if (!rec.has(dir))
|
|
51
|
+
throw refused(`${dir} is not in ${DEPENDENCIES_LOCK_NAME}: run \`vx lock\``)
|
|
52
|
+
return rec.of(dir)
|
|
53
|
+
},
|
|
54
|
+
})
|
|
55
|
+
}
|
|
56
|
+
return async (file) => {
|
|
57
|
+
const rec = records.get(file)
|
|
58
|
+
if (rec !== undefined || !bytes.has(file)) return rec
|
|
59
|
+
if (lockBytes === null) throw refused(`no ${DEPENDENCIES_LOCK_NAME}: run \`vx lock\``)
|
|
60
|
+
if (lock === null) throw refused(`${DEPENDENCIES_LOCK_NAME} is invalid, run \`vx lock\``)
|
|
61
|
+
throw refused(`${file} is not in ${DEPENDENCIES_LOCK_NAME}: run \`vx lock\``)
|
|
62
|
+
}
|
|
63
|
+
}
|
|
@@ -7,11 +7,18 @@
|
|
|
7
7
|
// fallback for a project the file does not list and the `--affected`
|
|
8
8
|
// diff are one implementation here, so a third lockfile is a parser.
|
|
9
9
|
//
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
10
|
+
// A run's key reads each project's digest from `.vx/dependencies.lock.json`
|
|
11
|
+
// (`KeyHookContext.locked`), never the file: only `vx lock` and
|
|
12
|
+
// `--affected` digest it, and a run whose extra file (a `link:` directory,
|
|
13
|
+
// edited with no install) moved since the lock, as the record is stale
|
|
14
|
+
// for it then. What the `discover` and `project` stages read
|
|
15
|
+
// (the edges, the recorded names) is memoised in cache.db
|
|
16
|
+
// (`lockfile_memos`, one row per lockfile and claimant) by the file's xxh3,
|
|
17
|
+
// so a warm run pays one read + hash + one row read, never a parse; within
|
|
18
|
+
// one process (a run, a watch cycle) the file's size, mtime, ctime and
|
|
19
|
+
// inode gate even the read. A run's read leaves the digests out (most of a
|
|
20
|
+
// parse: 9-25 ms on this repo's bun.lock, against 4-13 for the text), and
|
|
21
|
+
// `vx lock` adds them to the memo.
|
|
15
22
|
|
|
16
23
|
import path from 'node:path'
|
|
17
24
|
import { stat } from 'node:fs/promises'
|
|
@@ -104,21 +111,27 @@ export interface LockfileClaimHooks {
|
|
|
104
111
|
}
|
|
105
112
|
|
|
106
113
|
interface Digests {
|
|
107
|
-
/** xxh3 of the lockfile bytes; '' when the file is absent. */
|
|
114
|
+
/** xxh3 of the lockfile bytes and the extra files' hashes; '' when the file is absent. */
|
|
108
115
|
readonly lock: string
|
|
109
|
-
|
|
116
|
+
/** xxh3 of the lockfile bytes alone. */
|
|
117
|
+
readonly bytes: string
|
|
118
|
+
/** The extra files the digests read. */
|
|
119
|
+
readonly extras: readonly string[]
|
|
120
|
+
/** Each importer's digest; undefined when the read that made this left them out. */
|
|
121
|
+
readonly importers: ReadonlyMap<string, string> | undefined
|
|
110
122
|
readonly edges: ReadonlyMap<string, LockfileImporter>
|
|
111
123
|
readonly recorded: ReadonlyMap<string, ReadonlySet<string>> | undefined
|
|
112
124
|
}
|
|
113
125
|
|
|
114
126
|
/**
|
|
115
|
-
* A root-relative extra file's content hash, '' when absent
|
|
116
|
-
* the root is
|
|
117
|
-
*
|
|
127
|
+
* A root-relative extra file's content hash, '' when absent. A path above
|
|
128
|
+
* the root is read too: a `file:` or `link:` directory beside the checkout
|
|
129
|
+
* is what a task imports, and left unread an edit in it moved no key. One
|
|
130
|
+
* there but unreadable (EACCES, a symlink loop) refuses, as a settings file
|
|
131
|
+
* does: folded as '', it keyed as no file at all.
|
|
118
132
|
*/
|
|
119
|
-
async function hashExtra(workspaceRoot: string, rel: string): Promise<string> {
|
|
120
|
-
|
|
121
|
-
const abs = path.join(workspaceRoot, rel)
|
|
133
|
+
export async function hashExtra(workspaceRoot: string, rel: string): Promise<string> {
|
|
134
|
+
const abs = path.resolve(workspaceRoot, rel)
|
|
122
135
|
try {
|
|
123
136
|
return xxh3hex(await Bun.file(abs).bytes())
|
|
124
137
|
} catch (err) {
|
|
@@ -168,7 +181,7 @@ function identityOf(st: { size: number; mtimeMs: number; ctimeMs: number; ino: n
|
|
|
168
181
|
async function statsOf(workspaceRoot: string, rels: readonly string[]): Promise<string[]> {
|
|
169
182
|
return Promise.all(
|
|
170
183
|
rels.map((rel) =>
|
|
171
|
-
stat(path.
|
|
184
|
+
stat(path.resolve(workspaceRoot, rel)).then(
|
|
172
185
|
(st) => (st.isDirectory() ? hashExtra(workspaceRoot, rel) : identityOf(st)),
|
|
173
186
|
() => 'gone',
|
|
174
187
|
),
|
|
@@ -234,8 +247,17 @@ export function lockfileClaim(options: LockfileClaimOptions): LockfileClaimHooks
|
|
|
234
247
|
}
|
|
235
248
|
// Per run: every `key` call of one run gets the same context object.
|
|
236
249
|
const perRun = new WeakMap<object, Promise<Digests>>()
|
|
250
|
+
const lockedPerRun = new WeakMap<
|
|
251
|
+
object,
|
|
252
|
+
Promise<((dir: string) => string | undefined) | undefined>
|
|
253
|
+
>()
|
|
237
254
|
|
|
238
|
-
|
|
255
|
+
/** `withDigests`: each importer's digest too, which only `vx lock` and `--affected` read. */
|
|
256
|
+
const load = async (
|
|
257
|
+
workspaceRoot: string,
|
|
258
|
+
cacheDir: string,
|
|
259
|
+
withDigests: boolean,
|
|
260
|
+
): Promise<Digests> => {
|
|
239
261
|
const full = path.join(workspaceRoot, file)
|
|
240
262
|
let identity: string
|
|
241
263
|
try {
|
|
@@ -243,6 +265,8 @@ export function lockfileClaim(options: LockfileClaimOptions): LockfileClaimHooks
|
|
|
243
265
|
} catch {
|
|
244
266
|
const digests: Digests = {
|
|
245
267
|
lock: '',
|
|
268
|
+
bytes: '',
|
|
269
|
+
extras: [],
|
|
246
270
|
importers: new Map(),
|
|
247
271
|
edges: new Map(),
|
|
248
272
|
recorded: undefined,
|
|
@@ -251,7 +275,11 @@ export function lockfileClaim(options: LockfileClaimOptions): LockfileClaimHooks
|
|
|
251
275
|
return digests
|
|
252
276
|
}
|
|
253
277
|
const last = seen.get(workspaceRoot)
|
|
254
|
-
if (
|
|
278
|
+
if (
|
|
279
|
+
last !== undefined &&
|
|
280
|
+
last.identity === identity &&
|
|
281
|
+
(!withDigests || last.digests.importers !== undefined)
|
|
282
|
+
) {
|
|
255
283
|
const now = await statsOf(workspaceRoot, last.extras)
|
|
256
284
|
if (now.every((s, i) => s === last.extraStats[last.extras[i]!])) {
|
|
257
285
|
if (last.unsaved !== undefined && writeLockfileMemo(cacheDir, file, claimant, last.unsaved))
|
|
@@ -270,17 +298,19 @@ export function lockfileClaim(options: LockfileClaimOptions): LockfileClaimHooks
|
|
|
270
298
|
let edges: ReadonlyMap<string, LockfileImporter> = new Map()
|
|
271
299
|
let recorded: ReadonlyMap<string, readonly string[]> | undefined
|
|
272
300
|
let unsaved: LockfileMemo | undefined
|
|
273
|
-
|
|
301
|
+
let served = false
|
|
302
|
+
if (memo !== undefined && (!withDigests || memo.importers !== null)) {
|
|
274
303
|
const now = await hashExtras(workspaceRoot, Object.keys(memo.extras))
|
|
275
304
|
if (Object.entries(memo.extras).every(([rel, h]) => now.hashes[rel] === h)) {
|
|
305
|
+
served = true
|
|
276
306
|
extras = memo.extras
|
|
277
307
|
extraStats = now.stats
|
|
278
|
-
importers = new Map(Object.entries(memo.importers))
|
|
308
|
+
importers = memo.importers === null ? undefined : new Map(Object.entries(memo.importers))
|
|
279
309
|
edges = new Map(Object.entries(memo.edges as Record<string, LockfileImporter>))
|
|
280
310
|
recorded = memo.recorded === undefined ? undefined : new Map(Object.entries(memo.recorded))
|
|
281
311
|
}
|
|
282
312
|
}
|
|
283
|
-
if (
|
|
313
|
+
if (!served) {
|
|
284
314
|
const text = decode(bytes)
|
|
285
315
|
const rels = [...new Set(options.extraFiles?.(text) ?? [])].sort()
|
|
286
316
|
const now = await hashExtras(workspaceRoot, rels)
|
|
@@ -289,7 +319,9 @@ export function lockfileClaim(options: LockfileClaimOptions): LockfileClaimHooks
|
|
|
289
319
|
importers =
|
|
290
320
|
scope === 'workspace'
|
|
291
321
|
? new Map<string, string>()
|
|
292
|
-
:
|
|
322
|
+
: withDigests
|
|
323
|
+
? digest(text, new Map(Object.entries(extras)))
|
|
324
|
+
: undefined
|
|
293
325
|
edges = options.importers?.(text) ?? new Map()
|
|
294
326
|
recorded = options.recorded?.(text)
|
|
295
327
|
if (memoises) {
|
|
@@ -297,7 +329,7 @@ export function lockfileClaim(options: LockfileClaimOptions): LockfileClaimHooks
|
|
|
297
329
|
version,
|
|
298
330
|
lock: lockHash,
|
|
299
331
|
extras,
|
|
300
|
-
importers: Object.fromEntries(importers),
|
|
332
|
+
importers: importers === undefined ? null : Object.fromEntries(importers),
|
|
301
333
|
edges: Object.fromEntries(edges),
|
|
302
334
|
...(recorded !== undefined ? { recorded: Object.fromEntries(recorded) } : {}),
|
|
303
335
|
}
|
|
@@ -311,6 +343,8 @@ export function lockfileClaim(options: LockfileClaimOptions): LockfileClaimHooks
|
|
|
311
343
|
: xxh3hex([lockHash, ...rels.map((r) => `${r}\0${extras[r]}`)].join('\0'))
|
|
312
344
|
const digests: Digests = {
|
|
313
345
|
lock,
|
|
346
|
+
bytes: lockHash,
|
|
347
|
+
extras: rels,
|
|
314
348
|
importers,
|
|
315
349
|
edges,
|
|
316
350
|
recorded:
|
|
@@ -327,10 +361,36 @@ export function lockfileClaim(options: LockfileClaimOptions): LockfileClaimHooks
|
|
|
327
361
|
})
|
|
328
362
|
return digests
|
|
329
363
|
}
|
|
364
|
+
/**
|
|
365
|
+
* Each project's digest for this run: the lock's, unless an extra file
|
|
366
|
+
* it records moved since (then a digest of the file as it is now, as
|
|
367
|
+
* `vx lock` would record it); undefined when the file is absent.
|
|
368
|
+
*/
|
|
369
|
+
const lockedOnce = (
|
|
370
|
+
ctx: KeyHookContext,
|
|
371
|
+
): Promise<((dir: string) => string | undefined) | undefined> => {
|
|
372
|
+
let p = lockedPerRun.get(ctx)
|
|
373
|
+
if (p === undefined) {
|
|
374
|
+
p = (async () => {
|
|
375
|
+
const rec = await ctx.locked(file)
|
|
376
|
+
if (rec === undefined) return undefined
|
|
377
|
+
const rels = Object.keys(rec.extras)
|
|
378
|
+
const now = await Promise.all(rels.map((rel) => hashExtra(ctx.workspaceRoot, rel)))
|
|
379
|
+
if (now.every((h, i) => h === rec.extras[rels[i]!])) {
|
|
380
|
+
return (dir: string) => rec.digest(dir)
|
|
381
|
+
}
|
|
382
|
+
const digests = await load(ctx.workspaceRoot, ctx.cacheDir, true)
|
|
383
|
+
const importers = digests.importers!
|
|
384
|
+
return (dir: string) => digestFor(importers, importerOf(ctx.workspaceRoot, dir))
|
|
385
|
+
})()
|
|
386
|
+
lockedPerRun.set(ctx, p)
|
|
387
|
+
}
|
|
388
|
+
return p
|
|
389
|
+
}
|
|
330
390
|
const loadOnce = (ctx: KeyHookContext): Promise<Digests> => {
|
|
331
391
|
let p = perRun.get(ctx)
|
|
332
392
|
if (p === undefined) {
|
|
333
|
-
p = load(ctx.workspaceRoot, ctx.cacheDir)
|
|
393
|
+
p = load(ctx.workspaceRoot, ctx.cacheDir, false)
|
|
334
394
|
perRun.set(ctx, p)
|
|
335
395
|
}
|
|
336
396
|
return p
|
|
@@ -378,6 +438,24 @@ export function lockfileClaim(options: LockfileClaimOptions): LockfileClaimHooks
|
|
|
378
438
|
}
|
|
379
439
|
return names
|
|
380
440
|
},
|
|
441
|
+
// What `vx lock` commits, so `--affected` reads the base's digests
|
|
442
|
+
// instead of parsing the base's file.
|
|
443
|
+
async digests(_file: string, ctx: FingerprintContext) {
|
|
444
|
+
if (scope === 'workspace') return undefined
|
|
445
|
+
const digests = await load(ctx.workspaceRoot, ctx.cacheDir, true)
|
|
446
|
+
if (digests.lock === '') return undefined
|
|
447
|
+
const importers = digests.importers!
|
|
448
|
+
return {
|
|
449
|
+
hash: digests.bytes,
|
|
450
|
+
projects: Object.fromEntries(
|
|
451
|
+
ctx.projects.map((p) => [
|
|
452
|
+
p.name,
|
|
453
|
+
digestFor(importers, importerOf(ctx.workspaceRoot, p.dir)),
|
|
454
|
+
]),
|
|
455
|
+
),
|
|
456
|
+
...(digests.extras.length > 0 ? { extras: digests.extras } : {}),
|
|
457
|
+
}
|
|
458
|
+
},
|
|
381
459
|
...(options.extraFiles !== undefined
|
|
382
460
|
? {
|
|
383
461
|
// The memo records which files these exact bytes name, for this
|
|
@@ -393,19 +471,20 @@ export function lockfileClaim(options: LockfileClaimOptions): LockfileClaimHooks
|
|
|
393
471
|
: {}),
|
|
394
472
|
},
|
|
395
473
|
async key(task, ctx) {
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
[part]: digestFor(digests.importers, importerOf(ctx.workspaceRoot, task.projectDir)),
|
|
474
|
+
if (scope === 'project') {
|
|
475
|
+
const digestOf = await lockedOnce(ctx)
|
|
476
|
+
const digest = digestOf?.(task.projectDir)
|
|
477
|
+
return digest === undefined ? undefined : { [part]: digest }
|
|
401
478
|
}
|
|
479
|
+
const digests = await loadOnce(ctx)
|
|
480
|
+
return digests.lock === '' ? undefined : { [part]: digests.lock }
|
|
402
481
|
},
|
|
403
482
|
async importers(ctx) {
|
|
404
|
-
const digests = await load(ctx.workspaceRoot, ctx.cacheDir)
|
|
483
|
+
const digests = await load(ctx.workspaceRoot, ctx.cacheDir, false)
|
|
405
484
|
return digests.lock === '' ? undefined : digests.edges
|
|
406
485
|
},
|
|
407
486
|
async recorded(ctx) {
|
|
408
|
-
const digests = await load(ctx.workspaceRoot, ctx.cacheDir)
|
|
487
|
+
const digests = await load(ctx.workspaceRoot, ctx.cacheDir, false)
|
|
409
488
|
return digests.lock === '' ? undefined : digests.recorded
|
|
410
489
|
},
|
|
411
490
|
}
|
|
@@ -146,6 +146,8 @@ export interface SaveMissArgs {
|
|
|
146
146
|
export interface SaveFacts {
|
|
147
147
|
saveMs?: number
|
|
148
148
|
artifactBytes?: number
|
|
149
|
+
/** The saved files' bytes, as a hit's `outputBytes` counts them: the row's `outputs`. */
|
|
150
|
+
outputBytes?: number
|
|
149
151
|
/** The entry landed: the outcome's `saved`, set only once it has. */
|
|
150
152
|
saved?: true
|
|
151
153
|
}
|
|
@@ -239,6 +241,8 @@ export async function saveMiss(a: SaveMissArgs): Promise<{ landed: Promise<SaveF
|
|
|
239
241
|
},
|
|
240
242
|
})
|
|
241
243
|
endSave()
|
|
244
|
+
const bytes = cache.outputBytes?.(a.hash)
|
|
245
|
+
if (bytes !== undefined) facts.outputBytes = bytes
|
|
242
246
|
if (a.measure === true) {
|
|
243
247
|
facts.saveMs = Math.round(performance.now() - saveStart)
|
|
244
248
|
// No local artifact (local writes off): no size to say.
|
|
@@ -294,9 +298,26 @@ export type UnsavedArgs = Pick<
|
|
|
294
298
|
* A miss that ran here and saves nothing (it failed, the policy writes
|
|
295
299
|
* nothing, an upstream failed, its key no longer held) still wrote its
|
|
296
300
|
* outputs, and a same-project reader after it must see them as it sees a
|
|
297
|
-
* save's: resolved and marked the same way.
|
|
301
|
+
* save's: resolved and marked the same way. Returns their bytes, the
|
|
302
|
+
* row's `outputs`: blank, an unsaved miss read as one that wrote nothing.
|
|
298
303
|
*/
|
|
299
|
-
export async function markUnsaved(a: UnsavedArgs): Promise<
|
|
304
|
+
export async function markUnsaved(a: UnsavedArgs): Promise<number> {
|
|
305
|
+
const { outputFiles, wsOutputFiles } = await resolveUnsaved(a)
|
|
306
|
+
markWritten(a, outputFiles, wsOutputFiles)
|
|
307
|
+
return sizeOf(a.node.projectDir, outputFiles) + sizeOf(a.workspaceRoot, wsOutputFiles)
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/** An unsaved miss nothing reads after: only the bytes of its outputs, its row's `outputs`. */
|
|
311
|
+
export async function unsavedOutputBytes(
|
|
312
|
+
a: Pick<UnsavedArgs, 'node' | 'workspaceRoot' | 'nestedProjectDirs' | 'outputs' | 'wsOutputs'>,
|
|
313
|
+
): Promise<number> {
|
|
314
|
+
const { outputFiles, wsOutputFiles } = await resolveUnsaved(a)
|
|
315
|
+
return sizeOf(a.node.projectDir, outputFiles) + sizeOf(a.workspaceRoot, wsOutputFiles)
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
async function resolveUnsaved(
|
|
319
|
+
a: Pick<UnsavedArgs, 'node' | 'workspaceRoot' | 'nestedProjectDirs' | 'outputs' | 'wsOutputs'>,
|
|
320
|
+
): Promise<{ outputFiles: string[]; wsOutputFiles: string[] }> {
|
|
300
321
|
const endResolve = span('miss: resolve outputs')
|
|
301
322
|
const outputFiles = await resolveOutputs({
|
|
302
323
|
projectDir: a.node.projectDir,
|
|
@@ -308,7 +329,14 @@ export async function markUnsaved(a: UnsavedArgs): Promise<void> {
|
|
|
308
329
|
outputs: a.wsOutputs,
|
|
309
330
|
})
|
|
310
331
|
endResolve()
|
|
311
|
-
|
|
332
|
+
return { outputFiles, wsOutputFiles }
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
/** The bytes of `files` (absolute, or under `base`); one gone since it was listed counts none. */
|
|
336
|
+
function sizeOf(base: string, files: readonly string[]): number {
|
|
337
|
+
let n = 0
|
|
338
|
+
for (const f of files) n += statSync(path.resolve(base, f), { throwIfNoEntry: false })?.size ?? 0
|
|
339
|
+
return n
|
|
312
340
|
}
|
|
313
341
|
|
|
314
342
|
/**
|
|
@@ -283,7 +283,9 @@ export async function executePersistentTask(args: ExecuteArgs): Promise<TaskOutc
|
|
|
283
283
|
const message = err instanceof Error ? err.message : String(err)
|
|
284
284
|
// The task's OWN stream, not the process's: the frame is where a
|
|
285
285
|
// reader looks for why a task failed, and a run with a custom logger
|
|
286
|
-
// (an embedder) never saw a bare stderr write at all.
|
|
286
|
+
// (an embedder) never saw a bare stderr write at all. What waited on
|
|
287
|
+
// an open line goes first.
|
|
288
|
+
ends.end()
|
|
287
289
|
log.taskStderr(
|
|
288
290
|
node,
|
|
289
291
|
ends.own(`[vx] ${node.id}: persistent task failed to become ready: ${message}\n`),
|
package/src/orchestrator/plan.ts
CHANGED
|
@@ -225,7 +225,11 @@ export async function plan(args: PlanArgs): Promise<RunPlan> {
|
|
|
225
225
|
)
|
|
226
226
|
const dependents = new Map<string, string[]>()
|
|
227
227
|
for (const node of args.nodes.values())
|
|
228
|
-
for (const dep of node.deps)
|
|
228
|
+
for (const dep of node.deps) {
|
|
229
|
+
const ds = dependents.get(dep)
|
|
230
|
+
if (ds === undefined) dependents.set(dep, [node.id])
|
|
231
|
+
else ds.push(node.id)
|
|
232
|
+
}
|
|
229
233
|
for (const id of idleServers(args.nodes, dependents, restoreTier))
|
|
230
234
|
cacheStatusById.set(id, 'not-started')
|
|
231
235
|
}
|