@vzn/vx 0.0.645 → 0.0.646
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -28
- package/package.json +5 -5
- package/schemas/last.json +15 -0
- package/skills/vx/SKILL.md +1 -1
- package/src/bin.ts +1 -1
- package/src/cache/cache.ts +66 -16
- package/src/cache/git-inputs.ts +55 -10
- package/src/cache/index.ts +4 -0
- package/src/cache/inputs.ts +61 -11
- package/src/cache/layer.ts +25 -2
- package/src/cache/layered-cache.ts +32 -2
- package/src/cache/output-index.ts +25 -11
- package/src/cache/remote-lane.ts +13 -4
- package/src/cache/run-history.ts +13 -6
- package/src/cache/schema.ts +19 -2
- package/src/cli/core-alias.ts +5 -2
- package/src/cli/help.ts +4 -6
- package/src/cli/index.ts +0 -12
- package/src/cli/init.ts +43 -10
- package/src/cli/last.ts +4 -2
- package/src/cli/plugin-commands.ts +96 -3
- package/src/cli/plugin-templates.ts +1 -1
- package/src/cli/run.ts +2 -7
- package/src/cli/select.ts +5 -3
- package/src/cli/upgrade.ts +4 -5
- package/src/exec/executor.ts +112 -9
- package/src/exec/index.ts +2 -0
- package/src/exec/proc-sample.ts +67 -5
- package/src/exec/runner.ts +4 -0
- package/src/exec/sandbox-runtime.ts +11 -4
- package/src/graph/scheduler.ts +59 -8
- package/src/history/history.ts +119 -7
- package/src/history/metrics.ts +53 -16
- package/src/index.ts +15 -0
- package/src/orchestrator/execute-args.ts +3 -0
- package/src/orchestrator/execute-task.ts +28 -7
- package/src/orchestrator/held-servers.ts +86 -0
- package/src/orchestrator/hit-restore.ts +11 -4
- package/src/orchestrator/index.ts +17 -1
- package/src/orchestrator/local-floor.ts +104 -0
- package/src/orchestrator/local-shortcircuit.ts +26 -7
- package/src/orchestrator/miss-save.ts +10 -4
- package/src/orchestrator/persistent-task.ts +14 -0
- package/src/orchestrator/placement.ts +84 -9
- package/src/orchestrator/plugin-host.ts +21 -3
- package/src/orchestrator/plugin.ts +37 -0
- package/src/orchestrator/prepare.ts +11 -3
- package/src/orchestrator/run-keys.ts +164 -0
- package/src/orchestrator/run-records.ts +6 -0
- package/src/orchestrator/run.ts +93 -6
- package/src/orchestrator/sandbox-request.ts +9 -3
- package/src/orchestrator/task-artifacts.ts +142 -0
- package/src/orchestrator/task-hash.ts +11 -1
- package/src/orchestrator/task-process.ts +16 -0
- package/src/orchestrator/telemetry-host.ts +2 -1
- package/src/orchestrator/telemetry.ts +60 -17
- package/src/report/forecast.ts +6 -27
- package/src/report/framed-output.ts +41 -5
- package/src/report/logger.ts +60 -20
- package/src/report/status-line.ts +38 -18
- package/src/report/summary.ts +61 -7
- package/src/settings.ts +7 -0
- package/src/util/colors.ts +16 -1
- package/src/util/index.ts +1 -0
- package/src/util/paths.ts +30 -0
- package/src/util/ui.ts +4 -1
- package/src/workspace/config-bytecode.ts +276 -0
- package/src/workspace/index.ts +0 -1
- package/src/workspace/migration.ts +4 -32
- package/src/workspace/project-loader.ts +92 -14
- package/src/cli/foreign-flags.ts +0 -727
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
// The run key memo: every up-front key a warm run derived, read back whole
|
|
2
|
+
// when nothing any of them folds has moved. Deriving them again cost 250 ms
|
|
3
|
+
// of a 1,070 ms warm run at 9,600 tasks even with every fold memo row
|
|
4
|
+
// answering (2026-10-22): the per-task walk itself, not the files.
|
|
5
|
+
//
|
|
6
|
+
// The token is everything `deriveStableKeys` reads: each node's id, edges,
|
|
7
|
+
// resolved config, plugin key parts and files token, the declared env
|
|
8
|
+
// values, the forward args and the code that derives. A run the token cannot
|
|
9
|
+
// vouch for (a keyless or runtime-probed task, a declaration no files token
|
|
10
|
+
// covers) derives as before.
|
|
11
|
+
|
|
12
|
+
import { readdirSync, statSync } from 'node:fs'
|
|
13
|
+
import path from 'node:path'
|
|
14
|
+
import type { GitFilesCache } from '../cache/index.js'
|
|
15
|
+
import type { TaskNode } from '../graph/index.js'
|
|
16
|
+
import { xxh3hex } from '../util/index.js'
|
|
17
|
+
import type { StableKey } from './stable-keys.js'
|
|
18
|
+
import { hashTaskConfig, isKeyless, taskFilesToken, type HashCache } from './task-hash.js'
|
|
19
|
+
|
|
20
|
+
/** Moves every remembered run: bump it when what a token stands for changes. */
|
|
21
|
+
const RUN_MEMO_VERSION = 'run-memo-v1'
|
|
22
|
+
|
|
23
|
+
export interface RunKeysArgs {
|
|
24
|
+
nodes: Map<string, TaskNode>
|
|
25
|
+
workspaceRoot: string
|
|
26
|
+
claimedFiles: readonly string[]
|
|
27
|
+
forwardArgs?: readonly string[] | undefined
|
|
28
|
+
nestedDirsByProject: Map<string, string[]>
|
|
29
|
+
gitFilesCache: GitFilesCache
|
|
30
|
+
hashCache: HashCache
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** What a run's up-front pass yields, as the memo holds it. */
|
|
34
|
+
export interface RunKeys {
|
|
35
|
+
stableKeys: StableKey[]
|
|
36
|
+
uncachedKeys: Map<string, string>
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* The memo row's name (the selection) and the token its keys stand for, or
|
|
41
|
+
* undefined when a node's key reads something the token cannot name.
|
|
42
|
+
*/
|
|
43
|
+
export function runKeysToken(args: RunKeysArgs): { id: string; token: string } | undefined {
|
|
44
|
+
if (args.hashCache.folds === undefined) return undefined
|
|
45
|
+
const parts: string[] = [
|
|
46
|
+
RUN_MEMO_VERSION,
|
|
47
|
+
codeIdentity(),
|
|
48
|
+
args.workspaceRoot,
|
|
49
|
+
JSON.stringify(args.forwardArgs ?? []),
|
|
50
|
+
JSON.stringify([...args.claimedFiles].sort()),
|
|
51
|
+
JSON.stringify([...args.nestedDirsByProject]),
|
|
52
|
+
// Which outputs git ignores decides which a reader may see (`readableOutputs`).
|
|
53
|
+
[...args.gitFilesCache.ignoredOutputs].sort().join('\0'),
|
|
54
|
+
]
|
|
55
|
+
const ids: string[] = []
|
|
56
|
+
const envNames = new Set<string>()
|
|
57
|
+
for (const node of args.nodes.values()) {
|
|
58
|
+
const cfg = node.config
|
|
59
|
+
if (isKeyless(cfg) || node.excludedUpstream !== undefined) return undefined
|
|
60
|
+
const inputs = cfg.inputs
|
|
61
|
+
if ((inputs?.runtime?.length ?? 0) > 0 || (inputs?.workspaceRuntime?.length ?? 0) > 0) {
|
|
62
|
+
return undefined
|
|
63
|
+
}
|
|
64
|
+
let files = ''
|
|
65
|
+
if (inputs !== undefined) {
|
|
66
|
+
if ((inputs.workspaceFiles?.length ?? 0) > 0) return undefined
|
|
67
|
+
// No positive entry resolves no files (`filesPlan`).
|
|
68
|
+
if (inputs.files === undefined || inputs.files.some((e) => !e.startsWith('!'))) {
|
|
69
|
+
const token = taskFilesToken({
|
|
70
|
+
node,
|
|
71
|
+
workspaceRoot: args.workspaceRoot,
|
|
72
|
+
nestedProjectDirs: args.nestedDirsByProject.get(node.projectName) ?? [],
|
|
73
|
+
gitFilesCache: args.gitFilesCache,
|
|
74
|
+
hashCache: args.hashCache,
|
|
75
|
+
})
|
|
76
|
+
if (token === undefined) return undefined
|
|
77
|
+
files = token
|
|
78
|
+
}
|
|
79
|
+
for (const name of inputs.env ?? []) envNames.add(name)
|
|
80
|
+
}
|
|
81
|
+
ids.push(node.id)
|
|
82
|
+
parts.push(
|
|
83
|
+
node.id,
|
|
84
|
+
node.projectName,
|
|
85
|
+
node.projectDir,
|
|
86
|
+
node.deps.join('\u0001'),
|
|
87
|
+
(node.orderOnly ?? []).join('\u0001'),
|
|
88
|
+
node.requested ? '1' : '0',
|
|
89
|
+
hashTaskConfig(cfg, args.hashCache),
|
|
90
|
+
JSON.stringify(node.keyParts ?? []),
|
|
91
|
+
JSON.stringify(node.addsToOutputsOf ?? []),
|
|
92
|
+
JSON.stringify(node.outputsAddedToBy ?? []),
|
|
93
|
+
files,
|
|
94
|
+
)
|
|
95
|
+
}
|
|
96
|
+
for (const name of [...envNames].sort()) {
|
|
97
|
+
const value = process.env[name]
|
|
98
|
+
parts.push(value === undefined ? name : `${name}=${value}`)
|
|
99
|
+
}
|
|
100
|
+
return { id: `\0run\0${xxh3hex(ids.join('\0'))}`, token: xxh3hex(parts.join('\0')) }
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** The keys remembered under `token`, mapped back onto this run's nodes. */
|
|
104
|
+
export function readRunKeys(
|
|
105
|
+
args: RunKeysArgs,
|
|
106
|
+
at: { id: string; token: string },
|
|
107
|
+
): RunKeys | undefined {
|
|
108
|
+
const row = args.hashCache.folds?.get(at.id, at.token)
|
|
109
|
+
if (row === undefined) return undefined
|
|
110
|
+
let parsed: { s: [string, string][]; u: [string, string][] }
|
|
111
|
+
try {
|
|
112
|
+
parsed = JSON.parse(row)
|
|
113
|
+
} catch {
|
|
114
|
+
return undefined
|
|
115
|
+
}
|
|
116
|
+
const stableKeys: StableKey[] = []
|
|
117
|
+
for (const [id, hash] of parsed.s) {
|
|
118
|
+
const node = args.nodes.get(id)
|
|
119
|
+
if (node === undefined) return undefined
|
|
120
|
+
stableKeys.push({ hash, node })
|
|
121
|
+
}
|
|
122
|
+
return { stableKeys, uncachedKeys: new Map(parsed.u) }
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
export function writeRunKeys(
|
|
126
|
+
args: RunKeysArgs,
|
|
127
|
+
at: { id: string; token: string },
|
|
128
|
+
keys: RunKeys,
|
|
129
|
+
): void {
|
|
130
|
+
args.hashCache.folds?.set(
|
|
131
|
+
at.id,
|
|
132
|
+
at.token,
|
|
133
|
+
JSON.stringify({
|
|
134
|
+
s: keys.stableKeys.map((k) => [k.node.id, k.hash]),
|
|
135
|
+
u: [...keys.uncachedKeys],
|
|
136
|
+
}),
|
|
137
|
+
)
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
let identity: string | undefined
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* The code that derives the keys: a compiled binary is its file; a source
|
|
144
|
+
* checkout is every module under `src/`, so an edit to the derivation from
|
|
145
|
+
* source never reads back a key the old code made.
|
|
146
|
+
*/
|
|
147
|
+
function codeIdentity(): string {
|
|
148
|
+
if (identity !== undefined) return identity
|
|
149
|
+
// `Bun.main`, not `import.meta.dir`: `--minify` inlines the build
|
|
150
|
+
// machine's source path, so an installed binary walked a `src/` that is
|
|
151
|
+
// not there, threw, and lost every warm run's short circuit (2026-10-22).
|
|
152
|
+
if (Bun.main.startsWith('/$bunfs/')) {
|
|
153
|
+
const st = statSync(process.execPath)
|
|
154
|
+
return (identity = `${process.execPath}:${st.size}:${st.mtimeMs}:${st.ino}`)
|
|
155
|
+
}
|
|
156
|
+
const src = path.resolve(import.meta.dir, '..')
|
|
157
|
+
const stamps: string[] = []
|
|
158
|
+
for (const rel of readdirSync(src, { recursive: true, encoding: 'utf8' }).sort()) {
|
|
159
|
+
if (!rel.endsWith('.ts')) continue
|
|
160
|
+
const st = statSync(path.join(src, rel))
|
|
161
|
+
stamps.push(`${rel}:${st.size}:${st.mtimeMs}`)
|
|
162
|
+
}
|
|
163
|
+
return (identity = xxh3hex(stamps.join('\0')))
|
|
164
|
+
}
|
|
@@ -113,6 +113,11 @@ export function assembleRunRecords(input: RunRecordsInput): RunRecords {
|
|
|
113
113
|
runId,
|
|
114
114
|
...(o.cpuMs !== undefined ? { cpuMs: o.cpuMs } : {}),
|
|
115
115
|
...(o.peakRssBytes !== undefined ? { peakRssBytes: o.peakRssBytes } : {}),
|
|
116
|
+
...(o.peakCores !== undefined ? { peakCores: o.peakCores } : {}),
|
|
117
|
+
...(o.avgRssBytes !== undefined ? { avgRssBytes: o.avgRssBytes } : {}),
|
|
118
|
+
...(o.where !== undefined ? { placement: o.where } : {}),
|
|
119
|
+
...(o.whereCores !== undefined ? { placementCores: o.whereCores } : {}),
|
|
120
|
+
...(o.whereMemoryBytes !== undefined ? { placementMemoryBytes: o.whereMemoryBytes } : {}),
|
|
116
121
|
...(o.wallclockStartNs !== undefined ? { wallclockStartNs: o.wallclockStartNs } : {}),
|
|
117
122
|
...(o.wallclockEndNs !== undefined ? { wallclockEndNs: o.wallclockEndNs } : {}),
|
|
118
123
|
cacheHit: isCacheHit(o.status),
|
|
@@ -126,6 +131,7 @@ export function assembleRunRecords(input: RunRecordsInput): RunRecords {
|
|
|
126
131
|
...(o.sandboxViolations !== undefined ? { sandboxViolations: o.sandboxViolations } : {}),
|
|
127
132
|
...(o.notReady !== undefined ? { notReady: o.notReady } : {}),
|
|
128
133
|
...(remoteError !== undefined ? { remoteError } : {}),
|
|
134
|
+
...(o.keylessBy !== undefined ? { keylessBy: o.keylessBy } : {}),
|
|
129
135
|
})
|
|
130
136
|
if (o.status === 'failed') failedCount++
|
|
131
137
|
if (o.status === 'cache-hit') hitLocalCount++
|
package/src/orchestrator/run.ts
CHANGED
|
@@ -24,7 +24,7 @@ import {
|
|
|
24
24
|
} from '../exec/index.js'
|
|
25
25
|
import { DeferredOutputs } from './deferred-outputs.js'
|
|
26
26
|
import { resolveDownloadModes } from './download-policy.js'
|
|
27
|
-
import type { TaskExecutor } from '../exec/index.js'
|
|
27
|
+
import type { MachineReport, TaskExecutor } from '../exec/index.js'
|
|
28
28
|
import {
|
|
29
29
|
deadServerBehind,
|
|
30
30
|
declaredTask,
|
|
@@ -51,6 +51,7 @@ import {
|
|
|
51
51
|
detectColors,
|
|
52
52
|
type ColorSupport,
|
|
53
53
|
SecretBroker,
|
|
54
|
+
type ui,
|
|
54
55
|
} from '../util/index.js'
|
|
55
56
|
import { keyedProjects } from './keyed-projects.js'
|
|
56
57
|
import { isDefaultBuild } from './projects.js'
|
|
@@ -114,6 +115,7 @@ import {
|
|
|
114
115
|
type ShortCircuit,
|
|
115
116
|
} from './local-shortcircuit.js'
|
|
116
117
|
import { deriveStableKeys, probesAfterWrites } from './stable-keys.js'
|
|
118
|
+
import { askRemote, serversToHold } from './held-servers.js'
|
|
117
119
|
|
|
118
120
|
import { writeRunFailures } from './run-failures.js'
|
|
119
121
|
import { assembleRunRecords, recordLateRemoteErrors } from './run-records.js'
|
|
@@ -136,6 +138,7 @@ import {
|
|
|
136
138
|
} from '../history/index.js'
|
|
137
139
|
import type { RunOptions, RunSummary } from './options.js'
|
|
138
140
|
import { withRunSettings } from './run-settings.js'
|
|
141
|
+
import { cacheFacts, taskArtifacts } from './task-artifacts.js'
|
|
139
142
|
import { resolveSetting } from '../settings.js'
|
|
140
143
|
|
|
141
144
|
// Per run, never shared: a `vx watch` process runs many, and a shared map
|
|
@@ -145,6 +148,7 @@ const emptyShortCircuit = (): ShortCircuit => ({
|
|
|
145
148
|
restoreTier: new Set(),
|
|
146
149
|
uncachedKeys: new Map(),
|
|
147
150
|
groupKeys: new Map(),
|
|
151
|
+
remoteAsk: new Map(),
|
|
148
152
|
})
|
|
149
153
|
|
|
150
154
|
/**
|
|
@@ -547,6 +551,8 @@ async function runOnBus(
|
|
|
547
551
|
cacheDir: prepared.cacheDir,
|
|
548
552
|
warn: warnOf(log),
|
|
549
553
|
concurrency,
|
|
554
|
+
artifacts: taskArtifacts(prepared.cache, prepared.workspaceRoot),
|
|
555
|
+
cacheFacts: () => cacheFacts(prepared.remotes, prepared.cacheRefs),
|
|
550
556
|
}),
|
|
551
557
|
)
|
|
552
558
|
// Placement: decided ONCE per task, before scheduling, so the scheduler
|
|
@@ -744,10 +750,19 @@ async function runOnBus(
|
|
|
744
750
|
n.config.exec?.persistent !== undefined ||
|
|
745
751
|
n.config.cache === undefined ||
|
|
746
752
|
!(policy.localRead || policy.remoteRead)
|
|
753
|
+
// Only what runs here arms this machine's sandbox: a `sandbox` executor
|
|
754
|
+
// enforces it where it runs the task, so a distributed run's
|
|
755
|
+
// orchestrator needs none (owner, 2026-10-22).
|
|
747
756
|
const sandboxed =
|
|
748
757
|
sandboxArmer === null
|
|
749
758
|
? []
|
|
750
|
-
: [...nodes.values()].filter(
|
|
759
|
+
: [...nodes.values()].filter(
|
|
760
|
+
(n) =>
|
|
761
|
+
n.config.exec?.sandbox !== undefined &&
|
|
762
|
+
(n.config.exec.persistent === undefined
|
|
763
|
+
? localPlaced.has(n.id)
|
|
764
|
+
: !placements.carried.has(n.id)),
|
|
765
|
+
)
|
|
751
766
|
let prewarming: Promise<void> | undefined
|
|
752
767
|
const prewarm = (): void => {
|
|
753
768
|
prewarming ??= sandboxArmer?.arm().catch(() => {})
|
|
@@ -800,7 +815,16 @@ async function runOnBus(
|
|
|
800
815
|
|
|
801
816
|
// Run context for the footer. The top-of-run header is gone — the
|
|
802
817
|
// banner now lives in the summary, where the eye lands at the end.
|
|
818
|
+
const footers = executors.filter((e) => e.footer !== undefined)
|
|
819
|
+
const placers = executors.filter((e) => e.running !== undefined)
|
|
803
820
|
const runContext = {
|
|
821
|
+
...(footers.length > 0 ? { executorRows: () => footers.flatMap(executorFooter) } : {}),
|
|
822
|
+
...(placers.length > 0
|
|
823
|
+
? {
|
|
824
|
+
whereRunning: (id: string) =>
|
|
825
|
+
placers.map((e) => executorWhere(e, id)).find((w) => w !== undefined),
|
|
826
|
+
}
|
|
827
|
+
: {}),
|
|
804
828
|
version: VERSION,
|
|
805
829
|
packageCount: packagesInScope.size,
|
|
806
830
|
remoteCacheEnabled,
|
|
@@ -816,7 +840,8 @@ async function runOnBus(
|
|
|
816
840
|
},
|
|
817
841
|
concurrency,
|
|
818
842
|
workspaceProjectCount,
|
|
819
|
-
|
|
843
|
+
// Live too: a long run is read while it runs (M, 2026-10-21).
|
|
844
|
+
runId,
|
|
820
845
|
}
|
|
821
846
|
|
|
822
847
|
// Asked only when a plugin declares `output`: a plain run pays nothing.
|
|
@@ -902,10 +927,19 @@ async function runOnBus(
|
|
|
902
927
|
shouldShortCircuit(nodes, policy) && (remote || cache === prepared.localCache)
|
|
903
928
|
? prepared.localCache.holdsNothing()
|
|
904
929
|
: false
|
|
930
|
+
// Behind a remote, a server's dependants need their keys up front even
|
|
931
|
+
// on an empty store: the remote may hold them all, and then the server
|
|
932
|
+
// never starts (`held-servers.ts`).
|
|
933
|
+
const askForServers =
|
|
934
|
+
remote &&
|
|
935
|
+
policy.remoteRead &&
|
|
936
|
+
[...nodes.values()].some(
|
|
937
|
+
(n) => n.config.exec?.persistent !== undefined && !n.requested && n.surfaced !== true,
|
|
938
|
+
)
|
|
905
939
|
// A remote may still hold what an empty local store does not: the
|
|
906
940
|
// prefetch asks it, and no confirmed miss warms the sandbox early.
|
|
907
|
-
if (coldLocal && !remote) prewarm()
|
|
908
|
-
else if (!coldLocal && shouldShortCircuit(nodes, policy)) {
|
|
941
|
+
if (coldLocal && !remote && sandboxed.length > 0) prewarm()
|
|
942
|
+
else if ((!coldLocal || askForServers) && shouldShortCircuit(nodes, policy)) {
|
|
909
943
|
shortCircuit = await step('vx:keys', () =>
|
|
910
944
|
startLocalShortCircuit({
|
|
911
945
|
nodes,
|
|
@@ -1158,6 +1192,16 @@ async function runOnBus(
|
|
|
1158
1192
|
// runs ahead of their deps as low-priority worker-slot backfill.
|
|
1159
1193
|
// Empty when the short-circuit didn't fire → byte-identical.
|
|
1160
1194
|
restoreTier: shortCircuit.restoreTier,
|
|
1195
|
+
...(askForServers
|
|
1196
|
+
? {
|
|
1197
|
+
heldServers: askRemote(
|
|
1198
|
+
serversToHold(nodes, shortCircuit.restoreTier, shortCircuit.remoteAsk),
|
|
1199
|
+
nodes,
|
|
1200
|
+
shortCircuit,
|
|
1201
|
+
cache,
|
|
1202
|
+
),
|
|
1203
|
+
}
|
|
1204
|
+
: {}),
|
|
1161
1205
|
})
|
|
1162
1206
|
stop.graphDone()
|
|
1163
1207
|
reprioritize = undefined
|
|
@@ -1297,9 +1341,15 @@ async function runOnBus(
|
|
|
1297
1341
|
recordHistory(() => cache.recordRunBundle(records))
|
|
1298
1342
|
recordFailures(list)
|
|
1299
1343
|
})
|
|
1344
|
+
} else if (stop.stoppedBy !== undefined) {
|
|
1345
|
+
// Stopped: not a run `vx last` lists (item 854), but what finished is
|
|
1346
|
+
// measured, or the next run knows none of it (M, 2026-10-21).
|
|
1347
|
+
const ended = list.filter((o) => o.status !== 'aborted' && o.status !== 'skipped')
|
|
1348
|
+
stepSync('vx:history', () =>
|
|
1349
|
+
recordHistory(() => prepared.localCache.recordRuns(recordsOf(ended, false).runs)),
|
|
1350
|
+
)
|
|
1300
1351
|
}
|
|
1301
1352
|
mark('record history')
|
|
1302
|
-
runContext.runId = runId
|
|
1303
1353
|
// Drain any still-in-flight background prefetches before closing the
|
|
1304
1354
|
// cache handle — a prefetch ingesting into a closed SQLite DB would
|
|
1305
1355
|
// throw. Tasks that resolved as local hits never awaited their
|
|
@@ -1345,6 +1395,7 @@ async function runOnBus(
|
|
|
1345
1395
|
abortedCount: list.filter((o) => o.status === 'aborted' && !isGroupTask(o.node)).length,
|
|
1346
1396
|
stages: stageTimes(),
|
|
1347
1397
|
uploads: prepared.localCache.uploads,
|
|
1398
|
+
machines: executors.flatMap(executorMachines),
|
|
1348
1399
|
})
|
|
1349
1400
|
telemetry.emitSummary(summary)
|
|
1350
1401
|
const sinks = telemetry
|
|
@@ -1816,3 +1867,39 @@ function canonical(dir: string): string {
|
|
|
1816
1867
|
return path.resolve(dir)
|
|
1817
1868
|
}
|
|
1818
1869
|
}
|
|
1870
|
+
|
|
1871
|
+
/** An executor's footer rows; a plugin's throw or bad shape shows none. */
|
|
1872
|
+
function executorFooter(e: TaskExecutor): (readonly [string, ui.Inline])[] {
|
|
1873
|
+
try {
|
|
1874
|
+
const rows = e.footer?.()
|
|
1875
|
+
if (!Array.isArray(rows)) return []
|
|
1876
|
+
return rows.filter(
|
|
1877
|
+
(r): r is readonly [string, ui.Inline] =>
|
|
1878
|
+
Array.isArray(r) &&
|
|
1879
|
+
typeof r[0] === 'string' &&
|
|
1880
|
+
(typeof r[1] === 'string' || (typeof r[1] === 'object' && r[1] !== null)),
|
|
1881
|
+
)
|
|
1882
|
+
} catch {
|
|
1883
|
+
return []
|
|
1884
|
+
}
|
|
1885
|
+
}
|
|
1886
|
+
|
|
1887
|
+
function executorMachines(e: TaskExecutor): readonly MachineReport[] {
|
|
1888
|
+
try {
|
|
1889
|
+
const m = e.machines?.()
|
|
1890
|
+
return Array.isArray(m) ? m : []
|
|
1891
|
+
} catch {
|
|
1892
|
+
return []
|
|
1893
|
+
}
|
|
1894
|
+
}
|
|
1895
|
+
|
|
1896
|
+
/** Where `e` says a task runs, or `null` while it waits; a throw or anything else says nothing. */
|
|
1897
|
+
function executorWhere(e: TaskExecutor, id: string): string | null | undefined {
|
|
1898
|
+
try {
|
|
1899
|
+
const w = e.running?.(id)
|
|
1900
|
+
if (w === null) return null
|
|
1901
|
+
return typeof w === 'string' && w !== '' ? w : undefined
|
|
1902
|
+
} catch {
|
|
1903
|
+
return undefined
|
|
1904
|
+
}
|
|
1905
|
+
}
|
|
@@ -45,6 +45,11 @@ export interface SandboxArmer {
|
|
|
45
45
|
readonly armed: boolean
|
|
46
46
|
}
|
|
47
47
|
|
|
48
|
+
/** What the run's sandbox union reads of a task: a `TaskNode`, or an agent's order. */
|
|
49
|
+
export interface SandboxedNode {
|
|
50
|
+
readonly config: { readonly exec?: { readonly sandbox?: ExecConfig['sandbox'] } }
|
|
51
|
+
}
|
|
52
|
+
|
|
48
53
|
/**
|
|
49
54
|
* What SRT's run-wide `initialize()` is armed with, folded from every
|
|
50
55
|
* sandboxed task in the graph — or `null` when the run has none.
|
|
@@ -71,7 +76,7 @@ export interface SandboxRunUnion {
|
|
|
71
76
|
weakerNested: boolean
|
|
72
77
|
}
|
|
73
78
|
|
|
74
|
-
export function sandboxRunUnion(nodes: Iterable<
|
|
79
|
+
export function sandboxRunUnion(nodes: Iterable<SandboxedNode>): SandboxRunUnion | null {
|
|
75
80
|
const sandboxed = [...nodes].filter((n) => n.config.exec?.sandbox !== undefined)
|
|
76
81
|
if (sandboxed.length === 0) return null
|
|
77
82
|
const weakerNested = sandboxed.every((n) => n.config.exec?.sandbox?.weakerWhenNested === true)
|
|
@@ -119,7 +124,7 @@ export function sandboxRunUnion(nodes: Iterable<TaskNode>): SandboxRunUnion | nu
|
|
|
119
124
|
* port list (its bridge is a unix socket the task's side creates), lifts
|
|
120
125
|
* it for the run. macOS keeps per-task precision through vx's own rules.
|
|
121
126
|
*/
|
|
122
|
-
export function prepareSandbox(nodes: Iterable<
|
|
127
|
+
export function prepareSandbox(nodes: Iterable<SandboxedNode>): SandboxArmer | null {
|
|
123
128
|
const union = sandboxRunUnion(nodes)
|
|
124
129
|
if (union === null) return null
|
|
125
130
|
const { domains, deniedDomains, unixSockets, gitConfig, weakerNested } = union
|
|
@@ -180,7 +185,7 @@ export function prepareSandbox(nodes: Iterable<TaskNode>): SandboxArmer | null {
|
|
|
180
185
|
* with no `cache`, which has no key to be stale.
|
|
181
186
|
*/
|
|
182
187
|
export async function sandboxRequestFor(
|
|
183
|
-
node: TaskNode,
|
|
188
|
+
node: Pick<TaskNode, 'projectDir'>,
|
|
184
189
|
sandbox: NonNullable<ExecConfig['sandbox']>,
|
|
185
190
|
workspaceRoot: string,
|
|
186
191
|
keyed: ReadonlySet<string> | undefined,
|
|
@@ -285,6 +290,7 @@ export async function sandboxRequestFor(
|
|
|
285
290
|
// withholding exists to stop, not the wall.
|
|
286
291
|
reportLinked: links.withheld.map((w) => w.dir),
|
|
287
292
|
config,
|
|
293
|
+
...(keyed === undefined ? {} : { keyed: [...keyed] }),
|
|
288
294
|
}
|
|
289
295
|
return { sandbox: request, placeholders, withheld: links.withheld }
|
|
290
296
|
}
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
// Task entries by key for an executor that runs a task on another machine
|
|
2
|
+
// (`ExecutorContext.artifacts`, `CommandContext.artifacts`): the machine
|
|
3
|
+
// that ran it saves its outputs and output under the key the order named,
|
|
4
|
+
// and the one that needs them restores them, through the workspace's own
|
|
5
|
+
// cache (design packages/vx/docs/design/dte-2026-10.md, "Files").
|
|
6
|
+
|
|
7
|
+
import {
|
|
8
|
+
type CacheLayer,
|
|
9
|
+
type ResolvedRemote,
|
|
10
|
+
resolveOutputTree,
|
|
11
|
+
resolveWorkspaceOutputs,
|
|
12
|
+
} from '../cache/index.js'
|
|
13
|
+
import { secretMask } from '../util/index.js'
|
|
14
|
+
import { decodeOutputLog, encodeOutputLog } from './output-log.js'
|
|
15
|
+
|
|
16
|
+
/** The task a `TaskArtifacts` call names: where its outputs are and what ran. */
|
|
17
|
+
export interface ArtifactTask {
|
|
18
|
+
/** `project#task`. */
|
|
19
|
+
readonly id: string
|
|
20
|
+
/** Absolute. */
|
|
21
|
+
readonly projectDir: string
|
|
22
|
+
readonly command: string
|
|
23
|
+
/** Its `cache.outputs`: project-relative `files`, root-relative `workspaceFiles`. */
|
|
24
|
+
readonly outputs: {
|
|
25
|
+
readonly files: readonly string[]
|
|
26
|
+
readonly workspaceFiles: readonly string[]
|
|
27
|
+
}
|
|
28
|
+
/** The projects inside `projectDir` (absolute), whose files are not its outputs. */
|
|
29
|
+
readonly nestedProjectDirs?: readonly string[]
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** What a task's run left besides its files: its output, and its time and usage. */
|
|
33
|
+
export interface ArtifactRun {
|
|
34
|
+
readonly stdout: string
|
|
35
|
+
readonly stderr: string
|
|
36
|
+
readonly durationMs: number
|
|
37
|
+
readonly cpuMs?: number
|
|
38
|
+
readonly peakRssBytes?: number
|
|
39
|
+
/** Its environment: a secret value in it is masked out of the stored output. */
|
|
40
|
+
readonly env?: Readonly<Record<string, string | undefined>>
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export interface TaskArtifacts {
|
|
44
|
+
/**
|
|
45
|
+
* Keeps the files `task`'s outputs match, and its output, as entry
|
|
46
|
+
* `key`: here, and on each remote the workspace writes. Resolves once
|
|
47
|
+
* every upload has settled; a remote that fails degrades as a run's
|
|
48
|
+
* upload does.
|
|
49
|
+
*/
|
|
50
|
+
save(key: string, task: ArtifactTask, run: ArtifactRun): Promise<void>
|
|
51
|
+
/**
|
|
52
|
+
* Lays entry `key`'s files into the workspace, from the local store or
|
|
53
|
+
* else the first remote holding it, and resolves its output; null when
|
|
54
|
+
* none holds it. A key restored here and then saved is not uploaded
|
|
55
|
+
* again.
|
|
56
|
+
*/
|
|
57
|
+
restore(key: string, task: ArtifactTask): Promise<{ stdout: string; stderr: string } | null>
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export function taskArtifacts(cache: CacheLayer, workspaceRoot: string): TaskArtifacts {
|
|
61
|
+
return {
|
|
62
|
+
async save(key, task, run) {
|
|
63
|
+
const [tree, wsFiles] = await Promise.all([
|
|
64
|
+
resolveOutputTree({
|
|
65
|
+
projectDir: task.projectDir,
|
|
66
|
+
outputs: task.outputs.files,
|
|
67
|
+
nestedProjectDirs: [...(task.nestedProjectDirs ?? [])],
|
|
68
|
+
}),
|
|
69
|
+
resolveWorkspaceOutputs({ workspaceRoot, outputs: task.outputs.workspaceFiles }),
|
|
70
|
+
])
|
|
71
|
+
const mask = secretMask([process.env, run.env ?? {}])
|
|
72
|
+
const log = encodeOutputLog([
|
|
73
|
+
{ text: run.stdout, err: false },
|
|
74
|
+
{ text: run.stderr, err: true },
|
|
75
|
+
])
|
|
76
|
+
await cache.save({
|
|
77
|
+
hash: key,
|
|
78
|
+
projectDir: task.projectDir,
|
|
79
|
+
outputFiles: tree.files,
|
|
80
|
+
...(tree.emptyDirs.length > 0 ? { outputDirs: tree.emptyDirs } : {}),
|
|
81
|
+
workspaceRoot,
|
|
82
|
+
...(wsFiles.length > 0 ? { workspaceOutputFiles: wsFiles } : {}),
|
|
83
|
+
entry: {
|
|
84
|
+
taskId: task.id,
|
|
85
|
+
command: mask?.mask(task.command) ?? task.command,
|
|
86
|
+
durationMs: run.durationMs,
|
|
87
|
+
stdout: mask?.mask(log) ?? log,
|
|
88
|
+
...(run.cpuMs !== undefined ? { cpuMs: run.cpuMs } : {}),
|
|
89
|
+
...(run.peakRssBytes !== undefined ? { peakRssBytes: run.peakRssBytes } : {}),
|
|
90
|
+
},
|
|
91
|
+
})
|
|
92
|
+
await cache.drainUploads?.()
|
|
93
|
+
},
|
|
94
|
+
|
|
95
|
+
async restore(key, task) {
|
|
96
|
+
const entry = await (cache.getSaved ? cache.getSaved(key) : cache.get(key))
|
|
97
|
+
if (entry === null) return null
|
|
98
|
+
await cache.restoreOutputs(key, task.projectDir, workspaceRoot, task.outputs)
|
|
99
|
+
let stdout = ''
|
|
100
|
+
let stderr = ''
|
|
101
|
+
for (const c of decodeOutputLog(entry.stdout)) {
|
|
102
|
+
if (c.err) stderr += c.text
|
|
103
|
+
else stdout += c.text
|
|
104
|
+
}
|
|
105
|
+
return { stdout, stderr }
|
|
106
|
+
},
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* What a run's cache is read under, for a machine that runs its tasks
|
|
112
|
+
* (`ExecutorContext.cacheFacts`, `CommandContext.artifactsFor`): the
|
|
113
|
+
* refs the run worked out, so that machine reads as the run would, and
|
|
114
|
+
* the access each remote lent, by plugin.
|
|
115
|
+
*/
|
|
116
|
+
export interface CacheFacts {
|
|
117
|
+
/** `CacheContext.refs`. */
|
|
118
|
+
readonly refs: readonly string[]
|
|
119
|
+
/** Plugin name → the token its remote's `lend` gave. */
|
|
120
|
+
readonly lent: Readonly<Record<string, string>>
|
|
121
|
+
/** Epoch ms: the first lent token's end, before which to ask again. */
|
|
122
|
+
readonly expiresAt?: number
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** The run's `CacheFacts`: each readable remote's `lend`, asked now; one that throws lends nothing. */
|
|
126
|
+
export async function cacheFacts(
|
|
127
|
+
remotes: readonly ResolvedRemote[],
|
|
128
|
+
refs: readonly string[],
|
|
129
|
+
): Promise<CacheFacts> {
|
|
130
|
+
const lent: Record<string, string> = {}
|
|
131
|
+
let expiresAt: number | undefined
|
|
132
|
+
await Promise.all(
|
|
133
|
+
remotes.map(async (r) => {
|
|
134
|
+
if (r.plugin === undefined || r.remote.lend === undefined || !r.read) return
|
|
135
|
+
const got = await r.remote.lend().catch(() => undefined)
|
|
136
|
+
if (got === undefined) return
|
|
137
|
+
lent[r.plugin] = got.token
|
|
138
|
+
if (got.expiresAt !== undefined) expiresAt = Math.min(expiresAt ?? Infinity, got.expiresAt)
|
|
139
|
+
}),
|
|
140
|
+
)
|
|
141
|
+
return { refs, lent, ...(expiresAt === undefined ? {} : { expiresAt }) }
|
|
142
|
+
}
|
|
@@ -297,6 +297,16 @@ export async function movedInput(
|
|
|
297
297
|
return undefined
|
|
298
298
|
}
|
|
299
299
|
|
|
300
|
+
/** A task's `filesToken`, as its key's fold memo probe takes it. */
|
|
301
|
+
export function taskFilesToken(
|
|
302
|
+
args: Pick<
|
|
303
|
+
ComputeHashArgs,
|
|
304
|
+
'node' | 'workspaceRoot' | 'nestedProjectDirs' | 'gitFilesCache' | 'hashCache'
|
|
305
|
+
>,
|
|
306
|
+
): string | undefined {
|
|
307
|
+
return filesToken(inputsArgs({ ...args, upstream: [], cache: undefined as never }))
|
|
308
|
+
}
|
|
309
|
+
|
|
300
310
|
/** What `resolveInputs` is asked for a task's key. */
|
|
301
311
|
function inputsArgs(args: ComputeHashArgs): ResolveInputsArgs {
|
|
302
312
|
const cfg = args.node.config
|
|
@@ -411,7 +421,7 @@ async function resolveKeyInput(
|
|
|
411
421
|
* each task's config object is created once per run, so a hit there
|
|
412
422
|
* skips the JSON.stringify + xxh3 entirely.
|
|
413
423
|
*/
|
|
414
|
-
function hashTaskConfig(cfg: TaskConfig, hashCache?: HashCache): string {
|
|
424
|
+
export function hashTaskConfig(cfg: TaskConfig, hashCache?: HashCache): string {
|
|
415
425
|
if (hashCache) {
|
|
416
426
|
const cached = hashCache.taskConfig.get(cfg)
|
|
417
427
|
if (cached !== undefined) return cached
|
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
PM_EXEC_ENV,
|
|
12
12
|
VX_RUN_TASK_ENV,
|
|
13
13
|
VX_RUN_WORKSPACE_ENV,
|
|
14
|
+
type ServerSpec,
|
|
14
15
|
} from '../exec/index.js'
|
|
15
16
|
import type { TaskNode } from '../graph/index.js'
|
|
16
17
|
import { parseDuration } from '../util/index.js'
|
|
@@ -89,6 +90,21 @@ export function taskTimeoutMs(step: ExecConfig): number | undefined {
|
|
|
89
90
|
return step.timeout === undefined ? undefined : parseDuration(step.timeout)!
|
|
90
91
|
}
|
|
91
92
|
|
|
93
|
+
/** Persistent task `node` as a `servers` executor starts it: no forwarded args, which a dependency never gets. */
|
|
94
|
+
export function serverSpec(node: TaskNode, workspaceRoot: string): ServerSpec {
|
|
95
|
+
const step = node.config.exec!
|
|
96
|
+
const timeoutMs = taskTimeoutMs(step)
|
|
97
|
+
const readyWhen = step.persistent?.readyWhen
|
|
98
|
+
return {
|
|
99
|
+
taskId: node.id,
|
|
100
|
+
command: step.command,
|
|
101
|
+
cwd: node.projectDir,
|
|
102
|
+
env: taskEnv(node, step, workspaceRoot),
|
|
103
|
+
...(readyWhen === undefined ? {} : { readyWhen }),
|
|
104
|
+
...(timeoutMs === undefined ? {} : { timeoutMs }),
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
92
108
|
export function isAborted(signal: AbortSignal | undefined): boolean {
|
|
93
109
|
return signal?.aborted === true
|
|
94
110
|
}
|
|
@@ -18,6 +18,7 @@ import type {
|
|
|
18
18
|
RunSummaryRecord,
|
|
19
19
|
TelemetryContext,
|
|
20
20
|
TelemetrySink,
|
|
21
|
+
TreeLoad,
|
|
21
22
|
} from './telemetry.js'
|
|
22
23
|
import { settleWithin, teardownTimeoutMs } from '../util/index.js'
|
|
23
24
|
import { createTelemetrySource } from './telemetry.js'
|
|
@@ -32,7 +33,7 @@ export interface TelemetryHandle {
|
|
|
32
33
|
/** Remove the bus subscription. Idempotent. */
|
|
33
34
|
dispose(): void
|
|
34
35
|
/** Sample a running task's process tree; see `TelemetrySource.track`. */
|
|
35
|
-
readonly track?: (taskId: string, pid: number) => () =>
|
|
36
|
+
readonly track?: (taskId: string, pid: number) => () => TreeLoad | undefined
|
|
36
37
|
}
|
|
37
38
|
|
|
38
39
|
/** Reject anything that is not sink-shaped, naming what arrived. */
|