@vzn/vx 0.0.644 → 0.0.646
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +22 -48
- package/package.json +5 -5
- package/schemas/init.json +5 -0
- package/schemas/last.json +15 -0
- package/skills/vx/SKILL.md +1 -1
- package/src/bin.ts +1 -1
- package/src/cache/archive.ts +12 -76
- package/src/cache/cache.ts +72 -42
- package/src/cache/git-inputs.ts +55 -10
- package/src/cache/index.ts +5 -2
- package/src/cache/inputs.ts +61 -11
- package/src/cache/layer.ts +30 -8
- package/src/cache/layered-cache.ts +138 -44
- package/src/cache/output-index.ts +25 -11
- package/src/cache/plugin-cache.ts +9 -14
- package/src/cache/remote-lane.ts +60 -22
- package/src/cache/run-history.ts +13 -6
- package/src/cache/schema.ts +19 -2
- package/src/cache/scoped-cache.ts +3 -80
- package/src/cli/core-alias.ts +5 -2
- package/src/cli/help.ts +8 -10
- package/src/cli/index.ts +0 -12
- package/src/cli/init.ts +159 -25
- package/src/cli/last.ts +4 -2
- package/src/cli/plan-format.ts +6 -2
- package/src/cli/plugin-commands.ts +98 -6
- package/src/cli/plugin-templates.ts +2 -2
- package/src/cli/run.ts +2 -8
- package/src/cli/select.ts +5 -3
- package/src/cli/upgrade.ts +5 -6
- package/src/config.ts +2 -4
- package/src/exec/executor.ts +112 -9
- package/src/exec/index.ts +2 -0
- package/src/exec/proc-sample.ts +67 -5
- package/src/exec/runner.ts +4 -0
- package/src/exec/sandbox-runtime.ts +11 -4
- package/src/graph/scheduler.ts +59 -8
- package/src/history/history.ts +119 -7
- package/src/history/metrics.ts +53 -16
- package/src/index.ts +16 -1
- package/src/orchestrator/affected-tasks.ts +0 -18
- package/src/orchestrator/execute-args.ts +3 -0
- package/src/orchestrator/execute-task.ts +28 -7
- package/src/orchestrator/held-servers.ts +86 -0
- package/src/orchestrator/hit-restore.ts +11 -4
- package/src/orchestrator/index.ts +18 -1
- package/src/orchestrator/local-floor.ts +104 -0
- package/src/orchestrator/local-shortcircuit.ts +26 -7
- package/src/orchestrator/miss-save.ts +10 -4
- package/src/orchestrator/options.ts +0 -8
- package/src/orchestrator/persistent-task.ts +14 -0
- package/src/orchestrator/placement.ts +84 -9
- package/src/orchestrator/plan.ts +7 -1
- package/src/orchestrator/plugin-host.ts +22 -7
- package/src/orchestrator/plugin.ts +48 -7
- package/src/orchestrator/prepare.ts +14 -49
- package/src/orchestrator/run-context.ts +39 -1
- package/src/orchestrator/run-keys.ts +164 -0
- package/src/orchestrator/run-records.ts +6 -0
- package/src/orchestrator/run.ts +95 -6
- package/src/orchestrator/sandbox-request.ts +9 -3
- package/src/orchestrator/task-artifacts.ts +142 -0
- package/src/orchestrator/task-hash.ts +12 -4
- package/src/orchestrator/task-process.ts +16 -0
- package/src/orchestrator/telemetry-host.ts +2 -1
- package/src/orchestrator/telemetry.ts +60 -17
- package/src/report/forecast.ts +6 -27
- package/src/report/framed-output.ts +41 -5
- package/src/report/logger.ts +60 -20
- package/src/report/status-line.ts +38 -18
- package/src/report/summary.ts +75 -9
- package/src/settings.ts +8 -7
- package/src/util/colors.ts +16 -1
- package/src/util/index.ts +1 -0
- package/src/util/paths.ts +30 -0
- package/src/util/ui.ts +4 -1
- package/src/workspace/affected.ts +0 -39
- package/src/workspace/config-bytecode.ts +276 -0
- package/src/workspace/index.ts +0 -2
- package/src/workspace/migration.ts +4 -32
- package/src/workspace/project-loader.ts +92 -14
- package/src/cli/foreign-flags.ts +0 -727
package/src/orchestrator/run.ts
CHANGED
|
@@ -24,7 +24,7 @@ import {
|
|
|
24
24
|
} from '../exec/index.js'
|
|
25
25
|
import { DeferredOutputs } from './deferred-outputs.js'
|
|
26
26
|
import { resolveDownloadModes } from './download-policy.js'
|
|
27
|
-
import type { TaskExecutor } from '../exec/index.js'
|
|
27
|
+
import type { MachineReport, TaskExecutor } from '../exec/index.js'
|
|
28
28
|
import {
|
|
29
29
|
deadServerBehind,
|
|
30
30
|
declaredTask,
|
|
@@ -51,6 +51,7 @@ import {
|
|
|
51
51
|
detectColors,
|
|
52
52
|
type ColorSupport,
|
|
53
53
|
SecretBroker,
|
|
54
|
+
type ui,
|
|
54
55
|
} from '../util/index.js'
|
|
55
56
|
import { keyedProjects } from './keyed-projects.js'
|
|
56
57
|
import { isDefaultBuild } from './projects.js'
|
|
@@ -114,6 +115,7 @@ import {
|
|
|
114
115
|
type ShortCircuit,
|
|
115
116
|
} from './local-shortcircuit.js'
|
|
116
117
|
import { deriveStableKeys, probesAfterWrites } from './stable-keys.js'
|
|
118
|
+
import { askRemote, serversToHold } from './held-servers.js'
|
|
117
119
|
|
|
118
120
|
import { writeRunFailures } from './run-failures.js'
|
|
119
121
|
import { assembleRunRecords, recordLateRemoteErrors } from './run-records.js'
|
|
@@ -136,6 +138,7 @@ import {
|
|
|
136
138
|
} from '../history/index.js'
|
|
137
139
|
import type { RunOptions, RunSummary } from './options.js'
|
|
138
140
|
import { withRunSettings } from './run-settings.js'
|
|
141
|
+
import { cacheFacts, taskArtifacts } from './task-artifacts.js'
|
|
139
142
|
import { resolveSetting } from '../settings.js'
|
|
140
143
|
|
|
141
144
|
// Per run, never shared: a `vx watch` process runs many, and a shared map
|
|
@@ -145,6 +148,7 @@ const emptyShortCircuit = (): ShortCircuit => ({
|
|
|
145
148
|
restoreTier: new Set(),
|
|
146
149
|
uncachedKeys: new Map(),
|
|
147
150
|
groupKeys: new Map(),
|
|
151
|
+
remoteAsk: new Map(),
|
|
148
152
|
})
|
|
149
153
|
|
|
150
154
|
/**
|
|
@@ -547,6 +551,8 @@ async function runOnBus(
|
|
|
547
551
|
cacheDir: prepared.cacheDir,
|
|
548
552
|
warn: warnOf(log),
|
|
549
553
|
concurrency,
|
|
554
|
+
artifacts: taskArtifacts(prepared.cache, prepared.workspaceRoot),
|
|
555
|
+
cacheFacts: () => cacheFacts(prepared.remotes, prepared.cacheRefs),
|
|
550
556
|
}),
|
|
551
557
|
)
|
|
552
558
|
// Placement: decided ONCE per task, before scheduling, so the scheduler
|
|
@@ -744,10 +750,19 @@ async function runOnBus(
|
|
|
744
750
|
n.config.exec?.persistent !== undefined ||
|
|
745
751
|
n.config.cache === undefined ||
|
|
746
752
|
!(policy.localRead || policy.remoteRead)
|
|
753
|
+
// Only what runs here arms this machine's sandbox: a `sandbox` executor
|
|
754
|
+
// enforces it where it runs the task, so a distributed run's
|
|
755
|
+
// orchestrator needs none (owner, 2026-10-22).
|
|
747
756
|
const sandboxed =
|
|
748
757
|
sandboxArmer === null
|
|
749
758
|
? []
|
|
750
|
-
: [...nodes.values()].filter(
|
|
759
|
+
: [...nodes.values()].filter(
|
|
760
|
+
(n) =>
|
|
761
|
+
n.config.exec?.sandbox !== undefined &&
|
|
762
|
+
(n.config.exec.persistent === undefined
|
|
763
|
+
? localPlaced.has(n.id)
|
|
764
|
+
: !placements.carried.has(n.id)),
|
|
765
|
+
)
|
|
751
766
|
let prewarming: Promise<void> | undefined
|
|
752
767
|
const prewarm = (): void => {
|
|
753
768
|
prewarming ??= sandboxArmer?.arm().catch(() => {})
|
|
@@ -800,7 +815,16 @@ async function runOnBus(
|
|
|
800
815
|
|
|
801
816
|
// Run context for the footer. The top-of-run header is gone — the
|
|
802
817
|
// banner now lives in the summary, where the eye lands at the end.
|
|
818
|
+
const footers = executors.filter((e) => e.footer !== undefined)
|
|
819
|
+
const placers = executors.filter((e) => e.running !== undefined)
|
|
803
820
|
const runContext = {
|
|
821
|
+
...(footers.length > 0 ? { executorRows: () => footers.flatMap(executorFooter) } : {}),
|
|
822
|
+
...(placers.length > 0
|
|
823
|
+
? {
|
|
824
|
+
whereRunning: (id: string) =>
|
|
825
|
+
placers.map((e) => executorWhere(e, id)).find((w) => w !== undefined),
|
|
826
|
+
}
|
|
827
|
+
: {}),
|
|
804
828
|
version: VERSION,
|
|
805
829
|
packageCount: packagesInScope.size,
|
|
806
830
|
remoteCacheEnabled,
|
|
@@ -810,11 +834,14 @@ async function runOnBus(
|
|
|
810
834
|
localWrite: policy.localWrite,
|
|
811
835
|
remote: remoteCacheEnabled && (policy.remoteRead || policy.remoteWrite),
|
|
812
836
|
traffic: prepared.localCache.traffic,
|
|
837
|
+
downloads: prepared.localCache.downloads,
|
|
838
|
+
uploads: prepared.localCache.uploads,
|
|
813
839
|
notDownloaded: () => deferredOutputs.pending().length,
|
|
814
840
|
},
|
|
815
841
|
concurrency,
|
|
816
842
|
workspaceProjectCount,
|
|
817
|
-
|
|
843
|
+
// Live too: a long run is read while it runs (M, 2026-10-21).
|
|
844
|
+
runId,
|
|
818
845
|
}
|
|
819
846
|
|
|
820
847
|
// Asked only when a plugin declares `output`: a plain run pays nothing.
|
|
@@ -900,10 +927,19 @@ async function runOnBus(
|
|
|
900
927
|
shouldShortCircuit(nodes, policy) && (remote || cache === prepared.localCache)
|
|
901
928
|
? prepared.localCache.holdsNothing()
|
|
902
929
|
: false
|
|
930
|
+
// Behind a remote, a server's dependants need their keys up front even
|
|
931
|
+
// on an empty store: the remote may hold them all, and then the server
|
|
932
|
+
// never starts (`held-servers.ts`).
|
|
933
|
+
const askForServers =
|
|
934
|
+
remote &&
|
|
935
|
+
policy.remoteRead &&
|
|
936
|
+
[...nodes.values()].some(
|
|
937
|
+
(n) => n.config.exec?.persistent !== undefined && !n.requested && n.surfaced !== true,
|
|
938
|
+
)
|
|
903
939
|
// A remote may still hold what an empty local store does not: the
|
|
904
940
|
// prefetch asks it, and no confirmed miss warms the sandbox early.
|
|
905
|
-
if (coldLocal && !remote) prewarm()
|
|
906
|
-
else if (!coldLocal && shouldShortCircuit(nodes, policy)) {
|
|
941
|
+
if (coldLocal && !remote && sandboxed.length > 0) prewarm()
|
|
942
|
+
else if ((!coldLocal || askForServers) && shouldShortCircuit(nodes, policy)) {
|
|
907
943
|
shortCircuit = await step('vx:keys', () =>
|
|
908
944
|
startLocalShortCircuit({
|
|
909
945
|
nodes,
|
|
@@ -1156,6 +1192,16 @@ async function runOnBus(
|
|
|
1156
1192
|
// runs ahead of their deps as low-priority worker-slot backfill.
|
|
1157
1193
|
// Empty when the short-circuit didn't fire → byte-identical.
|
|
1158
1194
|
restoreTier: shortCircuit.restoreTier,
|
|
1195
|
+
...(askForServers
|
|
1196
|
+
? {
|
|
1197
|
+
heldServers: askRemote(
|
|
1198
|
+
serversToHold(nodes, shortCircuit.restoreTier, shortCircuit.remoteAsk),
|
|
1199
|
+
nodes,
|
|
1200
|
+
shortCircuit,
|
|
1201
|
+
cache,
|
|
1202
|
+
),
|
|
1203
|
+
}
|
|
1204
|
+
: {}),
|
|
1159
1205
|
})
|
|
1160
1206
|
stop.graphDone()
|
|
1161
1207
|
reprioritize = undefined
|
|
@@ -1295,9 +1341,15 @@ async function runOnBus(
|
|
|
1295
1341
|
recordHistory(() => cache.recordRunBundle(records))
|
|
1296
1342
|
recordFailures(list)
|
|
1297
1343
|
})
|
|
1344
|
+
} else if (stop.stoppedBy !== undefined) {
|
|
1345
|
+
// Stopped: not a run `vx last` lists (item 854), but what finished is
|
|
1346
|
+
// measured, or the next run knows none of it (M, 2026-10-21).
|
|
1347
|
+
const ended = list.filter((o) => o.status !== 'aborted' && o.status !== 'skipped')
|
|
1348
|
+
stepSync('vx:history', () =>
|
|
1349
|
+
recordHistory(() => prepared.localCache.recordRuns(recordsOf(ended, false).runs)),
|
|
1350
|
+
)
|
|
1298
1351
|
}
|
|
1299
1352
|
mark('record history')
|
|
1300
|
-
runContext.runId = runId
|
|
1301
1353
|
// Drain any still-in-flight background prefetches before closing the
|
|
1302
1354
|
// cache handle — a prefetch ingesting into a closed SQLite DB would
|
|
1303
1355
|
// throw. Tasks that resolved as local hits never awaited their
|
|
@@ -1343,6 +1395,7 @@ async function runOnBus(
|
|
|
1343
1395
|
abortedCount: list.filter((o) => o.status === 'aborted' && !isGroupTask(o.node)).length,
|
|
1344
1396
|
stages: stageTimes(),
|
|
1345
1397
|
uploads: prepared.localCache.uploads,
|
|
1398
|
+
machines: executors.flatMap(executorMachines),
|
|
1346
1399
|
})
|
|
1347
1400
|
telemetry.emitSummary(summary)
|
|
1348
1401
|
const sinks = telemetry
|
|
@@ -1814,3 +1867,39 @@ function canonical(dir: string): string {
|
|
|
1814
1867
|
return path.resolve(dir)
|
|
1815
1868
|
}
|
|
1816
1869
|
}
|
|
1870
|
+
|
|
1871
|
+
/** An executor's footer rows; a plugin's throw or bad shape shows none. */
|
|
1872
|
+
function executorFooter(e: TaskExecutor): (readonly [string, ui.Inline])[] {
|
|
1873
|
+
try {
|
|
1874
|
+
const rows = e.footer?.()
|
|
1875
|
+
if (!Array.isArray(rows)) return []
|
|
1876
|
+
return rows.filter(
|
|
1877
|
+
(r): r is readonly [string, ui.Inline] =>
|
|
1878
|
+
Array.isArray(r) &&
|
|
1879
|
+
typeof r[0] === 'string' &&
|
|
1880
|
+
(typeof r[1] === 'string' || (typeof r[1] === 'object' && r[1] !== null)),
|
|
1881
|
+
)
|
|
1882
|
+
} catch {
|
|
1883
|
+
return []
|
|
1884
|
+
}
|
|
1885
|
+
}
|
|
1886
|
+
|
|
1887
|
+
function executorMachines(e: TaskExecutor): readonly MachineReport[] {
|
|
1888
|
+
try {
|
|
1889
|
+
const m = e.machines?.()
|
|
1890
|
+
return Array.isArray(m) ? m : []
|
|
1891
|
+
} catch {
|
|
1892
|
+
return []
|
|
1893
|
+
}
|
|
1894
|
+
}
|
|
1895
|
+
|
|
1896
|
+
/** Where `e` says a task runs, or `null` while it waits; a throw or anything else says nothing. */
|
|
1897
|
+
function executorWhere(e: TaskExecutor, id: string): string | null | undefined {
|
|
1898
|
+
try {
|
|
1899
|
+
const w = e.running?.(id)
|
|
1900
|
+
if (w === null) return null
|
|
1901
|
+
return typeof w === 'string' && w !== '' ? w : undefined
|
|
1902
|
+
} catch {
|
|
1903
|
+
return undefined
|
|
1904
|
+
}
|
|
1905
|
+
}
|
|
@@ -45,6 +45,11 @@ export interface SandboxArmer {
|
|
|
45
45
|
readonly armed: boolean
|
|
46
46
|
}
|
|
47
47
|
|
|
48
|
+
/** What the run's sandbox union reads of a task: a `TaskNode`, or an agent's order. */
|
|
49
|
+
export interface SandboxedNode {
|
|
50
|
+
readonly config: { readonly exec?: { readonly sandbox?: ExecConfig['sandbox'] } }
|
|
51
|
+
}
|
|
52
|
+
|
|
48
53
|
/**
|
|
49
54
|
* What SRT's run-wide `initialize()` is armed with, folded from every
|
|
50
55
|
* sandboxed task in the graph — or `null` when the run has none.
|
|
@@ -71,7 +76,7 @@ export interface SandboxRunUnion {
|
|
|
71
76
|
weakerNested: boolean
|
|
72
77
|
}
|
|
73
78
|
|
|
74
|
-
export function sandboxRunUnion(nodes: Iterable<
|
|
79
|
+
export function sandboxRunUnion(nodes: Iterable<SandboxedNode>): SandboxRunUnion | null {
|
|
75
80
|
const sandboxed = [...nodes].filter((n) => n.config.exec?.sandbox !== undefined)
|
|
76
81
|
if (sandboxed.length === 0) return null
|
|
77
82
|
const weakerNested = sandboxed.every((n) => n.config.exec?.sandbox?.weakerWhenNested === true)
|
|
@@ -119,7 +124,7 @@ export function sandboxRunUnion(nodes: Iterable<TaskNode>): SandboxRunUnion | nu
|
|
|
119
124
|
* port list (its bridge is a unix socket the task's side creates), lifts
|
|
120
125
|
* it for the run. macOS keeps per-task precision through vx's own rules.
|
|
121
126
|
*/
|
|
122
|
-
export function prepareSandbox(nodes: Iterable<
|
|
127
|
+
export function prepareSandbox(nodes: Iterable<SandboxedNode>): SandboxArmer | null {
|
|
123
128
|
const union = sandboxRunUnion(nodes)
|
|
124
129
|
if (union === null) return null
|
|
125
130
|
const { domains, deniedDomains, unixSockets, gitConfig, weakerNested } = union
|
|
@@ -180,7 +185,7 @@ export function prepareSandbox(nodes: Iterable<TaskNode>): SandboxArmer | null {
|
|
|
180
185
|
* with no `cache`, which has no key to be stale.
|
|
181
186
|
*/
|
|
182
187
|
export async function sandboxRequestFor(
|
|
183
|
-
node: TaskNode,
|
|
188
|
+
node: Pick<TaskNode, 'projectDir'>,
|
|
184
189
|
sandbox: NonNullable<ExecConfig['sandbox']>,
|
|
185
190
|
workspaceRoot: string,
|
|
186
191
|
keyed: ReadonlySet<string> | undefined,
|
|
@@ -285,6 +290,7 @@ export async function sandboxRequestFor(
|
|
|
285
290
|
// withholding exists to stop, not the wall.
|
|
286
291
|
reportLinked: links.withheld.map((w) => w.dir),
|
|
287
292
|
config,
|
|
293
|
+
...(keyed === undefined ? {} : { keyed: [...keyed] }),
|
|
288
294
|
}
|
|
289
295
|
return { sandbox: request, placeholders, withheld: links.withheld }
|
|
290
296
|
}
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
// Task entries by key for an executor that runs a task on another machine
|
|
2
|
+
// (`ExecutorContext.artifacts`, `CommandContext.artifacts`): the machine
|
|
3
|
+
// that ran it saves its outputs and output under the key the order named,
|
|
4
|
+
// and the one that needs them restores them, through the workspace's own
|
|
5
|
+
// cache (design packages/vx/docs/design/dte-2026-10.md, "Files").
|
|
6
|
+
|
|
7
|
+
import {
|
|
8
|
+
type CacheLayer,
|
|
9
|
+
type ResolvedRemote,
|
|
10
|
+
resolveOutputTree,
|
|
11
|
+
resolveWorkspaceOutputs,
|
|
12
|
+
} from '../cache/index.js'
|
|
13
|
+
import { secretMask } from '../util/index.js'
|
|
14
|
+
import { decodeOutputLog, encodeOutputLog } from './output-log.js'
|
|
15
|
+
|
|
16
|
+
/** The task a `TaskArtifacts` call names: where its outputs are and what ran. */
|
|
17
|
+
export interface ArtifactTask {
|
|
18
|
+
/** `project#task`. */
|
|
19
|
+
readonly id: string
|
|
20
|
+
/** Absolute. */
|
|
21
|
+
readonly projectDir: string
|
|
22
|
+
readonly command: string
|
|
23
|
+
/** Its `cache.outputs`: project-relative `files`, root-relative `workspaceFiles`. */
|
|
24
|
+
readonly outputs: {
|
|
25
|
+
readonly files: readonly string[]
|
|
26
|
+
readonly workspaceFiles: readonly string[]
|
|
27
|
+
}
|
|
28
|
+
/** The projects inside `projectDir` (absolute), whose files are not its outputs. */
|
|
29
|
+
readonly nestedProjectDirs?: readonly string[]
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** What a task's run left besides its files: its output, and its time and usage. */
|
|
33
|
+
export interface ArtifactRun {
|
|
34
|
+
readonly stdout: string
|
|
35
|
+
readonly stderr: string
|
|
36
|
+
readonly durationMs: number
|
|
37
|
+
readonly cpuMs?: number
|
|
38
|
+
readonly peakRssBytes?: number
|
|
39
|
+
/** Its environment: a secret value in it is masked out of the stored output. */
|
|
40
|
+
readonly env?: Readonly<Record<string, string | undefined>>
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export interface TaskArtifacts {
|
|
44
|
+
/**
|
|
45
|
+
* Keeps the files `task`'s outputs match, and its output, as entry
|
|
46
|
+
* `key`: here, and on each remote the workspace writes. Resolves once
|
|
47
|
+
* every upload has settled; a remote that fails degrades as a run's
|
|
48
|
+
* upload does.
|
|
49
|
+
*/
|
|
50
|
+
save(key: string, task: ArtifactTask, run: ArtifactRun): Promise<void>
|
|
51
|
+
/**
|
|
52
|
+
* Lays entry `key`'s files into the workspace, from the local store or
|
|
53
|
+
* else the first remote holding it, and resolves its output; null when
|
|
54
|
+
* none holds it. A key restored here and then saved is not uploaded
|
|
55
|
+
* again.
|
|
56
|
+
*/
|
|
57
|
+
restore(key: string, task: ArtifactTask): Promise<{ stdout: string; stderr: string } | null>
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export function taskArtifacts(cache: CacheLayer, workspaceRoot: string): TaskArtifacts {
|
|
61
|
+
return {
|
|
62
|
+
async save(key, task, run) {
|
|
63
|
+
const [tree, wsFiles] = await Promise.all([
|
|
64
|
+
resolveOutputTree({
|
|
65
|
+
projectDir: task.projectDir,
|
|
66
|
+
outputs: task.outputs.files,
|
|
67
|
+
nestedProjectDirs: [...(task.nestedProjectDirs ?? [])],
|
|
68
|
+
}),
|
|
69
|
+
resolveWorkspaceOutputs({ workspaceRoot, outputs: task.outputs.workspaceFiles }),
|
|
70
|
+
])
|
|
71
|
+
const mask = secretMask([process.env, run.env ?? {}])
|
|
72
|
+
const log = encodeOutputLog([
|
|
73
|
+
{ text: run.stdout, err: false },
|
|
74
|
+
{ text: run.stderr, err: true },
|
|
75
|
+
])
|
|
76
|
+
await cache.save({
|
|
77
|
+
hash: key,
|
|
78
|
+
projectDir: task.projectDir,
|
|
79
|
+
outputFiles: tree.files,
|
|
80
|
+
...(tree.emptyDirs.length > 0 ? { outputDirs: tree.emptyDirs } : {}),
|
|
81
|
+
workspaceRoot,
|
|
82
|
+
...(wsFiles.length > 0 ? { workspaceOutputFiles: wsFiles } : {}),
|
|
83
|
+
entry: {
|
|
84
|
+
taskId: task.id,
|
|
85
|
+
command: mask?.mask(task.command) ?? task.command,
|
|
86
|
+
durationMs: run.durationMs,
|
|
87
|
+
stdout: mask?.mask(log) ?? log,
|
|
88
|
+
...(run.cpuMs !== undefined ? { cpuMs: run.cpuMs } : {}),
|
|
89
|
+
...(run.peakRssBytes !== undefined ? { peakRssBytes: run.peakRssBytes } : {}),
|
|
90
|
+
},
|
|
91
|
+
})
|
|
92
|
+
await cache.drainUploads?.()
|
|
93
|
+
},
|
|
94
|
+
|
|
95
|
+
async restore(key, task) {
|
|
96
|
+
const entry = await (cache.getSaved ? cache.getSaved(key) : cache.get(key))
|
|
97
|
+
if (entry === null) return null
|
|
98
|
+
await cache.restoreOutputs(key, task.projectDir, workspaceRoot, task.outputs)
|
|
99
|
+
let stdout = ''
|
|
100
|
+
let stderr = ''
|
|
101
|
+
for (const c of decodeOutputLog(entry.stdout)) {
|
|
102
|
+
if (c.err) stderr += c.text
|
|
103
|
+
else stdout += c.text
|
|
104
|
+
}
|
|
105
|
+
return { stdout, stderr }
|
|
106
|
+
},
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* What a run's cache is read under, for a machine that runs its tasks
|
|
112
|
+
* (`ExecutorContext.cacheFacts`, `CommandContext.artifactsFor`): the
|
|
113
|
+
* refs the run worked out, so that machine reads as the run would, and
|
|
114
|
+
* the access each remote lent, by plugin.
|
|
115
|
+
*/
|
|
116
|
+
export interface CacheFacts {
|
|
117
|
+
/** `CacheContext.refs`. */
|
|
118
|
+
readonly refs: readonly string[]
|
|
119
|
+
/** Plugin name → the token its remote's `lend` gave. */
|
|
120
|
+
readonly lent: Readonly<Record<string, string>>
|
|
121
|
+
/** Epoch ms: the first lent token's end, before which to ask again. */
|
|
122
|
+
readonly expiresAt?: number
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** The run's `CacheFacts`: each readable remote's `lend`, asked now; one that throws lends nothing. */
|
|
126
|
+
export async function cacheFacts(
|
|
127
|
+
remotes: readonly ResolvedRemote[],
|
|
128
|
+
refs: readonly string[],
|
|
129
|
+
): Promise<CacheFacts> {
|
|
130
|
+
const lent: Record<string, string> = {}
|
|
131
|
+
let expiresAt: number | undefined
|
|
132
|
+
await Promise.all(
|
|
133
|
+
remotes.map(async (r) => {
|
|
134
|
+
if (r.plugin === undefined || r.remote.lend === undefined || !r.read) return
|
|
135
|
+
const got = await r.remote.lend().catch(() => undefined)
|
|
136
|
+
if (got === undefined) return
|
|
137
|
+
lent[r.plugin] = got.token
|
|
138
|
+
if (got.expiresAt !== undefined) expiresAt = Math.min(expiresAt ?? Infinity, got.expiresAt)
|
|
139
|
+
}),
|
|
140
|
+
)
|
|
141
|
+
return { refs, lent, ...(expiresAt === undefined ? {} : { expiresAt }) }
|
|
142
|
+
}
|
|
@@ -97,9 +97,7 @@ export interface ComputeHashArgs {
|
|
|
97
97
|
*/
|
|
98
98
|
export async function computeTaskHash(args: ComputeHashArgs): Promise<string> {
|
|
99
99
|
if (isKeyless(args.node.config)) return keylessHash(args.node.id)
|
|
100
|
-
|
|
101
|
-
args.cache.noteKey?.(args.node.id, key)
|
|
102
|
-
return key
|
|
100
|
+
return taskHash(args)
|
|
103
101
|
}
|
|
104
102
|
|
|
105
103
|
async function taskHash(args: ComputeHashArgs): Promise<string> {
|
|
@@ -299,6 +297,16 @@ export async function movedInput(
|
|
|
299
297
|
return undefined
|
|
300
298
|
}
|
|
301
299
|
|
|
300
|
+
/** A task's `filesToken`, as its key's fold memo probe takes it. */
|
|
301
|
+
export function taskFilesToken(
|
|
302
|
+
args: Pick<
|
|
303
|
+
ComputeHashArgs,
|
|
304
|
+
'node' | 'workspaceRoot' | 'nestedProjectDirs' | 'gitFilesCache' | 'hashCache'
|
|
305
|
+
>,
|
|
306
|
+
): string | undefined {
|
|
307
|
+
return filesToken(inputsArgs({ ...args, upstream: [], cache: undefined as never }))
|
|
308
|
+
}
|
|
309
|
+
|
|
302
310
|
/** What `resolveInputs` is asked for a task's key. */
|
|
303
311
|
function inputsArgs(args: ComputeHashArgs): ResolveInputsArgs {
|
|
304
312
|
const cfg = args.node.config
|
|
@@ -413,7 +421,7 @@ async function resolveKeyInput(
|
|
|
413
421
|
* each task's config object is created once per run, so a hit there
|
|
414
422
|
* skips the JSON.stringify + xxh3 entirely.
|
|
415
423
|
*/
|
|
416
|
-
function hashTaskConfig(cfg: TaskConfig, hashCache?: HashCache): string {
|
|
424
|
+
export function hashTaskConfig(cfg: TaskConfig, hashCache?: HashCache): string {
|
|
417
425
|
if (hashCache) {
|
|
418
426
|
const cached = hashCache.taskConfig.get(cfg)
|
|
419
427
|
if (cached !== undefined) return cached
|
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
PM_EXEC_ENV,
|
|
12
12
|
VX_RUN_TASK_ENV,
|
|
13
13
|
VX_RUN_WORKSPACE_ENV,
|
|
14
|
+
type ServerSpec,
|
|
14
15
|
} from '../exec/index.js'
|
|
15
16
|
import type { TaskNode } from '../graph/index.js'
|
|
16
17
|
import { parseDuration } from '../util/index.js'
|
|
@@ -89,6 +90,21 @@ export function taskTimeoutMs(step: ExecConfig): number | undefined {
|
|
|
89
90
|
return step.timeout === undefined ? undefined : parseDuration(step.timeout)!
|
|
90
91
|
}
|
|
91
92
|
|
|
93
|
+
/** Persistent task `node` as a `servers` executor starts it: no forwarded args, which a dependency never gets. */
|
|
94
|
+
export function serverSpec(node: TaskNode, workspaceRoot: string): ServerSpec {
|
|
95
|
+
const step = node.config.exec!
|
|
96
|
+
const timeoutMs = taskTimeoutMs(step)
|
|
97
|
+
const readyWhen = step.persistent?.readyWhen
|
|
98
|
+
return {
|
|
99
|
+
taskId: node.id,
|
|
100
|
+
command: step.command,
|
|
101
|
+
cwd: node.projectDir,
|
|
102
|
+
env: taskEnv(node, step, workspaceRoot),
|
|
103
|
+
...(readyWhen === undefined ? {} : { readyWhen }),
|
|
104
|
+
...(timeoutMs === undefined ? {} : { timeoutMs }),
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
92
108
|
export function isAborted(signal: AbortSignal | undefined): boolean {
|
|
93
109
|
return signal?.aborted === true
|
|
94
110
|
}
|
|
@@ -18,6 +18,7 @@ import type {
|
|
|
18
18
|
RunSummaryRecord,
|
|
19
19
|
TelemetryContext,
|
|
20
20
|
TelemetrySink,
|
|
21
|
+
TreeLoad,
|
|
21
22
|
} from './telemetry.js'
|
|
22
23
|
import { settleWithin, teardownTimeoutMs } from '../util/index.js'
|
|
23
24
|
import { createTelemetrySource } from './telemetry.js'
|
|
@@ -32,7 +33,7 @@ export interface TelemetryHandle {
|
|
|
32
33
|
/** Remove the bus subscription. Idempotent. */
|
|
33
34
|
dispose(): void
|
|
34
35
|
/** Sample a running task's process tree; see `TelemetrySource.track`. */
|
|
35
|
-
readonly track?: (taskId: string, pid: number) => () =>
|
|
36
|
+
readonly track?: (taskId: string, pid: number) => () => TreeLoad | undefined
|
|
36
37
|
}
|
|
37
38
|
|
|
38
39
|
/** Reject anything that is not sink-shaped, naming what arrived. */
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
// what or how tasks run. Contrast `cache`/`executor`, which return objects
|
|
17
17
|
// core calls INTO; those are the behavior capabilities, kept separate.
|
|
18
18
|
|
|
19
|
-
import { sampleTrees } from '../exec/index.js'
|
|
19
|
+
import { sampleTrees, type MachineReport } from '../exec/index.js'
|
|
20
20
|
import {
|
|
21
21
|
type CacheSource,
|
|
22
22
|
deriveCacheSource,
|
|
@@ -274,6 +274,12 @@ export interface RunSummaryRecord {
|
|
|
274
274
|
* Absent when nothing was uploaded or tried. Additive.
|
|
275
275
|
*/
|
|
276
276
|
uploads?: { count: number; bytes: number; ms: number; failed: number }
|
|
277
|
+
/**
|
|
278
|
+
* The machines an executor plugin ran tasks on, with what each did
|
|
279
|
+
* (`TaskExecutor.machines`); a task's `where` names one. Absent when no
|
|
280
|
+
* executor reports any. Additive.
|
|
281
|
+
*/
|
|
282
|
+
machines?: readonly MachineReport[]
|
|
277
283
|
}
|
|
278
284
|
|
|
279
285
|
/** One stage of a run, a wall window in epoch ms. */
|
|
@@ -305,6 +311,7 @@ export function assembleRunSummary(
|
|
|
305
311
|
abortedCount: number
|
|
306
312
|
stages?: readonly RunStage[]
|
|
307
313
|
uploads?: { count: number; bytes: number; ms: number; failed: number }
|
|
314
|
+
machines?: readonly MachineReport[]
|
|
308
315
|
},
|
|
309
316
|
): RunSummaryRecord {
|
|
310
317
|
let failedCount = 0
|
|
@@ -343,6 +350,9 @@ export function assembleRunSummary(
|
|
|
343
350
|
...(timing.uploads !== undefined && timing.uploads.count + timing.uploads.failed > 0
|
|
344
351
|
? { uploads: { ...timing.uploads } }
|
|
345
352
|
: {}),
|
|
353
|
+
...(timing.machines !== undefined && timing.machines.length > 0
|
|
354
|
+
? { machines: timing.machines }
|
|
355
|
+
: {}),
|
|
346
356
|
}
|
|
347
357
|
}
|
|
348
358
|
|
|
@@ -400,8 +410,19 @@ export interface TelemetrySource {
|
|
|
400
410
|
* Sample this task's process tree every `SAMPLE_MS` until the returned
|
|
401
411
|
* function is called, the root exits, or the run ends. Present only when
|
|
402
412
|
* a sink wants `task.sample`: nobody asking costs no timer and no read.
|
|
413
|
+
* The returned stop answers what the samples saw, for the run history.
|
|
403
414
|
*/
|
|
404
|
-
readonly track?: (taskId: string, pid: number) => () =>
|
|
415
|
+
readonly track?: (taskId: string, pid: number) => () => TreeLoad | undefined
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
/**
|
|
419
|
+
* What a tracked task's samples saw: the most cores one interval between
|
|
420
|
+
* looks burned, and the mean resident memory over the looks. Undefined
|
|
421
|
+
* when no look landed (a task shorter than `SAMPLE_MS`).
|
|
422
|
+
*/
|
|
423
|
+
export interface TreeLoad {
|
|
424
|
+
peakCores: number
|
|
425
|
+
avgRssBytes: number
|
|
405
426
|
}
|
|
406
427
|
|
|
407
428
|
/** How often a running task's process tree is sampled. */
|
|
@@ -476,10 +497,18 @@ export function createTelemetrySource(args: {
|
|
|
476
497
|
const wantsLog = sinks.some((s) => (s.wants ?? DEFAULT_KINDS).includes('task.log'))
|
|
477
498
|
const wantsSample = sinks.some((s) => (s.wants ?? DEFAULT_KINDS).includes('task.sample'))
|
|
478
499
|
|
|
479
|
-
// taskId → the
|
|
480
|
-
// only while one runs; a tick still reading when the
|
|
481
|
-
// doubled.
|
|
482
|
-
|
|
500
|
+
// taskId → its running root and what the looks at it saw. One timer for
|
|
501
|
+
// every task, alive only while one runs; a tick still reading when the
|
|
502
|
+
// next is due is not doubled.
|
|
503
|
+
interface Tracked {
|
|
504
|
+
pid: number
|
|
505
|
+
lastTs: number
|
|
506
|
+
lastCpuMs: number
|
|
507
|
+
peakCores: number
|
|
508
|
+
rssSum: number
|
|
509
|
+
looks: number
|
|
510
|
+
}
|
|
511
|
+
const tracked = new Map<string, Tracked>()
|
|
483
512
|
const every = args.every ?? everyInterval
|
|
484
513
|
let stopEvery: (() => void) | undefined
|
|
485
514
|
let sampling = false
|
|
@@ -488,8 +517,8 @@ export function createTelemetrySource(args: {
|
|
|
488
517
|
stopEvery?.()
|
|
489
518
|
stopEvery = undefined
|
|
490
519
|
}
|
|
491
|
-
const untrack = (taskId: string,
|
|
492
|
-
if (tracked.get(taskId) ===
|
|
520
|
+
const untrack = (taskId: string, t: Tracked): void => {
|
|
521
|
+
if (tracked.get(taskId) === t) tracked.delete(taskId)
|
|
493
522
|
if (tracked.size === 0) stopTimer()
|
|
494
523
|
}
|
|
495
524
|
const tick = async (): Promise<void> => {
|
|
@@ -497,15 +526,23 @@ export function createTelemetrySource(args: {
|
|
|
497
526
|
sampling = true
|
|
498
527
|
try {
|
|
499
528
|
const entries = [...tracked]
|
|
500
|
-
const usage = await sampleTrees(entries.map(([,
|
|
529
|
+
const usage = await sampleTrees(entries.map(([, t]) => t.pid))
|
|
501
530
|
const ts = Date.now()
|
|
502
|
-
for (const [taskId,
|
|
503
|
-
if (ended || tracked.get(taskId) !==
|
|
504
|
-
const u = usage.get(pid)
|
|
531
|
+
for (const [taskId, t] of entries) {
|
|
532
|
+
if (ended || tracked.get(taskId) !== t) continue
|
|
533
|
+
const u = usage.get(t.pid)
|
|
505
534
|
if (u === undefined) {
|
|
506
|
-
untrack(taskId,
|
|
535
|
+
untrack(taskId, t)
|
|
507
536
|
continue
|
|
508
537
|
}
|
|
538
|
+
// A descendant that exited takes its CPU out of the live sum, so an
|
|
539
|
+
// interval can read negative: it says nothing of the peak.
|
|
540
|
+
const ms = ts - t.lastTs
|
|
541
|
+
if (ms > 0) t.peakCores = Math.max(t.peakCores, (u.cpuMs - t.lastCpuMs) / ms)
|
|
542
|
+
t.lastTs = ts
|
|
543
|
+
t.lastCpuMs = u.cpuMs
|
|
544
|
+
t.rssSum += u.rssBytes
|
|
545
|
+
t.looks++
|
|
509
546
|
deliver({
|
|
510
547
|
v: TELEMETRY_SCHEMA_VERSION,
|
|
511
548
|
kind: 'task.sample',
|
|
@@ -522,11 +559,17 @@ export function createTelemetrySource(args: {
|
|
|
522
559
|
sampling = false
|
|
523
560
|
}
|
|
524
561
|
}
|
|
525
|
-
const track = (taskId: string, pid: number): (() =>
|
|
526
|
-
if (ended) return () =>
|
|
527
|
-
|
|
562
|
+
const track = (taskId: string, pid: number): (() => TreeLoad | undefined) => {
|
|
563
|
+
if (ended) return () => undefined
|
|
564
|
+
const t: Tracked = { pid, lastTs: Date.now(), lastCpuMs: 0, peakCores: 0, rssSum: 0, looks: 0 }
|
|
565
|
+
tracked.set(taskId, t)
|
|
528
566
|
stopEvery ??= every(SAMPLE_MS, tick)
|
|
529
|
-
return () =>
|
|
567
|
+
return () => {
|
|
568
|
+
untrack(taskId, t)
|
|
569
|
+
return t.looks === 0
|
|
570
|
+
? undefined
|
|
571
|
+
: { peakCores: t.peakCores, avgRssBytes: Math.round(t.rssSum / t.looks) }
|
|
572
|
+
}
|
|
530
573
|
}
|
|
531
574
|
|
|
532
575
|
function deliver(record: TelemetryRecord): void {
|