@vzn/vx 0.0.15 → 0.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +91 -130
- package/index.ts +10 -0
- package/launcher.mjs +10 -11
- package/package.json +19 -5
- package/plugins/local-cache/index.ts +2 -0
- package/plugins/local-executor/index.ts +2 -0
- package/plugins/schedule-history/index.ts +2 -0
- package/src/bin.ts +2 -2
- package/src/cache/archive.ts +557 -0
- package/src/cache/cache.ts +929 -250
- package/src/cache/cas-backend.ts +20 -2
- package/src/cache/chained-cache.ts +202 -0
- package/src/cache/digest.ts +0 -15
- package/src/cache/index.ts +13 -3
- package/src/cache/inputs.ts +445 -41
- package/src/cache/layered-cache.ts +129 -30
- package/src/cache/tar-stream.ts +358 -0
- package/src/cli/cache.ts +36 -10
- package/src/cli/help.ts +167 -128
- package/src/cli/index.ts +74 -43
- package/src/cli/info.ts +30 -0
- package/src/cli/last.ts +140 -0
- package/src/cli/migrate-emit.ts +21 -0
- package/src/cli/migrate-nx.ts +23 -0
- package/src/cli/migrate-scripts.ts +127 -0
- package/src/cli/migrate-turbo.ts +43 -6
- package/src/cli/migrate.ts +119 -33
- package/src/cli/plan-format.ts +58 -3
- package/src/cli/plugin-commands.ts +96 -0
- package/src/cli/prune.ts +291 -0
- package/src/cli/run.ts +272 -80
- package/src/cli/upgrade.ts +7 -6
- package/src/cli/watch.ts +259 -61
- package/src/cli/why.ts +193 -0
- package/src/config.ts +40 -23
- package/src/exec/env.ts +7 -1
- package/src/exec/executor.ts +239 -0
- package/src/exec/index.ts +19 -1
- package/src/exec/runner.ts +116 -29
- package/src/exec/sandbox-runtime.ts +194 -24
- package/src/graph/index.ts +3 -1
- package/src/graph/scheduler.ts +92 -18
- package/src/graph/task-graph.ts +220 -6
- package/src/index.ts +82 -84
- package/src/orchestrator/deferred-outputs.ts +144 -0
- package/src/orchestrator/download-policy.ts +171 -0
- package/src/orchestrator/events.ts +30 -0
- package/src/orchestrator/execute-task.ts +328 -94
- package/src/orchestrator/failure-mode.ts +61 -0
- package/src/orchestrator/framed-output.ts +87 -35
- package/src/orchestrator/history.ts +78 -31
- package/src/orchestrator/index.ts +43 -90
- package/src/orchestrator/local-shortcircuit.ts +22 -0
- package/src/orchestrator/logger.ts +196 -14
- package/src/orchestrator/metrics.ts +57 -1376
- package/src/orchestrator/missing-plugin.ts +9 -0
- package/src/orchestrator/options.ts +22 -4
- package/src/orchestrator/plan.ts +144 -1
- package/src/orchestrator/plugin-host.ts +199 -119
- package/src/orchestrator/plugin.ts +144 -57
- package/src/orchestrator/prepare.ts +220 -69
- package/src/orchestrator/remote-prefetch.ts +27 -8
- package/src/orchestrator/resources.ts +20 -34
- package/src/orchestrator/run-artifacts.ts +41 -16
- package/src/orchestrator/run-context.ts +167 -29
- package/src/orchestrator/run-report.ts +73 -46
- package/src/orchestrator/run.ts +590 -122
- package/src/orchestrator/stable-keys.ts +74 -33
- package/src/orchestrator/status-line.ts +47 -8
- package/src/orchestrator/summary.ts +22 -1
- package/src/orchestrator/tally.ts +78 -35
- package/src/orchestrator/task-hash.ts +105 -11
- package/src/orchestrator/task-log-buffer.ts +291 -0
- package/src/orchestrator/telemetry-host.ts +24 -6
- package/src/orchestrator/telemetry.ts +181 -21
- package/src/orchestrator/upstream.ts +72 -31
- package/src/orchestrator/verify.ts +12 -2
- package/src/plugins/local-cache/index.ts +11 -0
- package/src/plugins/local-executor/index.ts +45 -0
- package/src/plugins/schedule-history/index.ts +109 -0
- package/src/util/edit-distance.ts +21 -0
- package/src/util/errors.ts +15 -1
- package/src/util/index.ts +7 -2
- package/src/util/num.ts +37 -0
- package/src/util/paths.ts +43 -0
- package/src/util/settle.ts +48 -0
- package/src/util/size.ts +5 -1
- package/src/util/tail.ts +66 -0
- package/src/util/timing.ts +63 -0
- package/src/workspace/affected.ts +142 -15
- package/src/workspace/config-cache.ts +304 -0
- package/src/workspace/config-eval.ts +210 -0
- package/src/workspace/config-imports.ts +161 -0
- package/src/workspace/filter.ts +22 -4
- package/src/workspace/fingerprint.ts +16 -1
- package/src/workspace/index.ts +15 -3
- package/src/workspace/lockfile.ts +13 -2
- package/src/workspace/nested-dirs.ts +21 -7
- package/src/workspace/package-graph.ts +23 -20
- package/src/workspace/project-loader.ts +503 -59
- package/src/workspace/workspace.ts +213 -72
- package/src/cache/tar.ts +0 -343
- package/src/cli/backend.ts +0 -23
- package/src/cli/mcp-rpc.ts +0 -292
- package/src/cli/mcp.ts +0 -92
- package/src/orchestrator/devframe-surface.ts +0 -74
- package/src/orchestrator/predict.ts +0 -106
- package/src/orchestrator/protocol.ts +0 -156
- package/src/orchestrator/run-state.ts +0 -145
- package/src/orchestrator/wire-render.ts +0 -92
- package/src/orchestrator/wire.ts +0 -212
|
@@ -47,16 +47,45 @@ export async function deriveStableKeys(args: DeriveStableKeysArgs): Promise<Stab
|
|
|
47
47
|
const order = topoOrder(args.nodes)
|
|
48
48
|
const keyById = new Map<string, string>()
|
|
49
49
|
const unstableById = new Set<string>()
|
|
50
|
+
// Transitive-upstream output producers, accumulated in topo order — the
|
|
51
|
+
// inputs to the stability gate. A task's key is preliminary if an upstream
|
|
52
|
+
// writes where its inputs read, and that reach is TRANSITIVE: a producer
|
|
53
|
+
// reached through a no-output intermediate still poisons the key, so each
|
|
54
|
+
// dep's accumulated producers fold forward.
|
|
55
|
+
// - outputProjects: the project names of every upstream task declaring
|
|
56
|
+
// cache.outputs.files (project-relative outputs land in the producer's
|
|
57
|
+
// own dir).
|
|
58
|
+
// - wsOutputUpstream: any upstream declares cache.outputs.workspaceFiles
|
|
59
|
+
// (root-anchored, boundary-ignoring outputs).
|
|
60
|
+
const outputProjectsById = new Map<string, ReadonlySet<string>>()
|
|
61
|
+
const wsOutputUpstreamById = new Map<string, boolean>()
|
|
50
62
|
const stableKeys: StableKey[] = []
|
|
51
63
|
|
|
52
64
|
for (const id of order) {
|
|
53
65
|
const node = args.nodes.get(id)!
|
|
54
66
|
const upstream = synthUpstream(node, args.nodes, keyById)
|
|
55
67
|
|
|
68
|
+
// Fold every dep's accumulated producers + the dep's own declared
|
|
69
|
+
// outputs into this node's transitive-upstream producer sets.
|
|
70
|
+
const outputProjects = new Set<string>()
|
|
71
|
+
let wsOutputUpstream = false
|
|
72
|
+
for (const dep of node.deps) {
|
|
73
|
+
const depNode = args.nodes.get(dep)
|
|
74
|
+
if (!depNode) continue
|
|
75
|
+
for (const p of outputProjectsById.get(dep) ?? []) outputProjects.add(p)
|
|
76
|
+
if (wsOutputUpstreamById.get(dep) === true) wsOutputUpstream = true
|
|
77
|
+
const depOut = depNode.config.cache?.outputs
|
|
78
|
+
if ((depOut?.files?.length ?? 0) > 0) outputProjects.add(depNode.projectName)
|
|
79
|
+
if ((depOut?.workspaceFiles?.length ?? 0) > 0) wsOutputUpstream = true
|
|
80
|
+
}
|
|
81
|
+
outputProjectsById.set(id, outputProjects)
|
|
82
|
+
wsOutputUpstreamById.set(id, wsOutputUpstream)
|
|
83
|
+
|
|
56
84
|
if (isGroupTask(node)) {
|
|
57
85
|
// Groups have no exec/cache; they only fold upstream keys so
|
|
58
86
|
// dependents that filter inputs.tasks through the group still
|
|
59
|
-
// cascade
|
|
87
|
+
// cascade (and forward their producers, above). They inherit
|
|
88
|
+
// instability from any unstable member.
|
|
60
89
|
keyById.set(id, computeGroupHash(upstream))
|
|
61
90
|
if (node.deps.some((d) => unstableById.has(d))) unstableById.add(id)
|
|
62
91
|
continue
|
|
@@ -76,7 +105,8 @@ export async function deriveStableKeys(args: DeriveStableKeysArgs): Promise<Stab
|
|
|
76
105
|
keyById.set(id, hash)
|
|
77
106
|
|
|
78
107
|
const unstable =
|
|
79
|
-
node.deps.some((d) => unstableById.has(d)) ||
|
|
108
|
+
node.deps.some((d) => unstableById.has(d)) ||
|
|
109
|
+
dependsOnSiblingOutputs(node, outputProjects, wsOutputUpstream)
|
|
80
110
|
if (unstable) unstableById.add(id)
|
|
81
111
|
|
|
82
112
|
const cacheEnabled = node.config.cache !== undefined
|
|
@@ -90,7 +120,7 @@ export async function deriveStableKeys(args: DeriveStableKeysArgs): Promise<Stab
|
|
|
90
120
|
* are read by filterUpstreamHashes / computeGroupHash; the rest of
|
|
91
121
|
* TaskOutcome is irrelevant here, so we cast a minimal shape.
|
|
92
122
|
*/
|
|
93
|
-
|
|
123
|
+
function synthUpstream(
|
|
94
124
|
node: TaskNode,
|
|
95
125
|
nodes: Map<string, TaskNode>,
|
|
96
126
|
keyById: Map<string, string>,
|
|
@@ -111,44 +141,55 @@ export function synthUpstream(
|
|
|
111
141
|
}
|
|
112
142
|
|
|
113
143
|
/**
|
|
114
|
-
* Conservative stability check: does any
|
|
115
|
-
* could land where this task reads its inputs make this task's
|
|
116
|
-
* preliminary?
|
|
144
|
+
* Conservative stability check: does any TRANSITIVE-upstream task whose
|
|
145
|
+
* outputs could land where this task reads its inputs make this task's
|
|
146
|
+
* key preliminary? The producers are pre-folded by `deriveStableKeys`
|
|
147
|
+
* (`upstreamOutputProjects` = projects declaring `cache.outputs.files`
|
|
148
|
+
* upstream; `hasWsOutputUpstream` = any `cache.outputs.workspaceFiles`
|
|
149
|
+
* upstream), so a producer reached through a no-output intermediate is
|
|
150
|
+
* caught too.
|
|
117
151
|
*
|
|
118
|
-
* -
|
|
119
|
-
*
|
|
120
|
-
*
|
|
121
|
-
*
|
|
122
|
-
*
|
|
123
|
-
*
|
|
124
|
-
*
|
|
125
|
-
*
|
|
126
|
-
* outputs
|
|
127
|
-
*
|
|
152
|
+
* - Project-relative inputs (default `**` / anything project-relative)
|
|
153
|
+
* read THIS project's dir, so a same-project upstream's
|
|
154
|
+
* `outputs.files` — direct or transitive — makes the key
|
|
155
|
+
* preliminary. Project boundaries are hard, so only SAME-project
|
|
156
|
+
* outputs can reach a project-relative input.
|
|
157
|
+
* - `cache.inputs.workspaceFiles` is boundary-free: it can read ANY
|
|
158
|
+
* project's dir (so any upstream `outputs.files`, in any project,
|
|
159
|
+
* matters) or a root-anchored location (so any upstream
|
|
160
|
+
* `outputs.workspaceFiles` matters). Either → preliminary key. This
|
|
161
|
+
* is the reach the earlier per-dep check missed — it only compared a
|
|
162
|
+
* dep's `outputs.workspaceFiles`, never a dep's `outputs.files`.
|
|
128
163
|
*
|
|
129
|
-
* Either case → unstable → skip prefetch (
|
|
130
|
-
*
|
|
164
|
+
* Either case → unstable → skip prefetch AND probe reuse (execute-task
|
|
165
|
+
* recomputes the key lazily once the upstream ran). When in doubt,
|
|
166
|
+
* unstable.
|
|
131
167
|
*/
|
|
132
|
-
export function dependsOnSiblingOutputs(
|
|
133
|
-
|
|
168
|
+
export function dependsOnSiblingOutputs(
|
|
169
|
+
node: TaskNode,
|
|
170
|
+
upstreamOutputProjects: ReadonlySet<string>,
|
|
171
|
+
hasWsOutputUpstream: boolean,
|
|
172
|
+
): boolean {
|
|
173
|
+
const cache = node.config.cache
|
|
134
174
|
// A cache-disabled task has no key to prefetch anyway; treat as
|
|
135
175
|
// unstable-irrelevant (caller filters on cacheEnabled).
|
|
136
|
-
if (
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
176
|
+
if (cache === undefined) return false
|
|
177
|
+
if (upstreamOutputProjects.has(node.projectName)) return true
|
|
178
|
+
// A root-anchored output is boundary-IGNORING by design, so it can land
|
|
179
|
+
// inside THIS task's own project dir — where an ordinary project-relative
|
|
180
|
+
// input reads it. Neither other clause sees that: `upstreamOutputProjects`
|
|
181
|
+
// holds the PRODUCER's project, not this one, and the reader clause below
|
|
182
|
+
// requires this task to read workspace-anchored inputs, which it need not.
|
|
183
|
+
// That gap was a real stale hit (a producer writing `pkgs/app/gen/**` while
|
|
184
|
+
// `app` read `gen/**` project-relative), so the mere presence of a
|
|
185
|
+
// workspace-output producer upstream makes the key preliminary.
|
|
186
|
+
if (hasWsOutputUpstream) return true
|
|
187
|
+
const readsWorkspaceFiles = (cache.inputs?.workspaceFiles?.length ?? 0) > 0
|
|
188
|
+
if (readsWorkspaceFiles && upstreamOutputProjects.size > 0) return true
|
|
148
189
|
return false
|
|
149
190
|
}
|
|
150
191
|
|
|
151
|
-
|
|
192
|
+
function topoOrder(nodes: Map<string, TaskNode>): string[] {
|
|
152
193
|
const indegree = new Map<string, number>()
|
|
153
194
|
const dependents = new Map<string, string[]>()
|
|
154
195
|
for (const node of nodes.values()) {
|
|
@@ -19,6 +19,12 @@ const CLEAR = '\x1b[2K\r'
|
|
|
19
19
|
export interface StatusStream {
|
|
20
20
|
write(chunk: string): unknown
|
|
21
21
|
isTTY?: boolean
|
|
22
|
+
/**
|
|
23
|
+
* Terminal width in columns. Read fresh on every draw — `process.stdout`
|
|
24
|
+
* updates it on SIGWINCH, so a mid-run resize is picked up. Absent (not a
|
|
25
|
+
* TTY) means the region is never drawn anyway.
|
|
26
|
+
*/
|
|
27
|
+
columns?: number
|
|
22
28
|
}
|
|
23
29
|
|
|
24
30
|
export interface OutputWriter {
|
|
@@ -68,7 +74,9 @@ export function createOutputWriter(
|
|
|
68
74
|
|
|
69
75
|
let current: readonly string[] | null = null
|
|
70
76
|
let shown = false
|
|
71
|
-
|
|
77
|
+
// PHYSICAL rows the shown region occupies, not logical lines — see
|
|
78
|
+
// `regionRows`.
|
|
79
|
+
let shownRows = 0
|
|
72
80
|
let dead = false
|
|
73
81
|
let lastDraw = -Infinity
|
|
74
82
|
// Trailing draw scheduled when a forced set lands inside the floor.
|
|
@@ -85,10 +93,41 @@ export function createOutputWriter(
|
|
|
85
93
|
}
|
|
86
94
|
}
|
|
87
95
|
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
96
|
+
/**
|
|
97
|
+
* How many PHYSICAL terminal rows `lines` occupies. The terminal wraps
|
|
98
|
+
* any line wider than the viewport, so a region of N logical lines can
|
|
99
|
+
* cover more than N rows — and erasing by the logical count moves the
|
|
100
|
+
* cursor up too few rows, leaving the top of the region on screen. It
|
|
101
|
+
* accumulates, because every redraw is short by the same amount.
|
|
102
|
+
*
|
|
103
|
+
* Both triggers are ordinary: the summary section is a fixed 62 visible
|
|
104
|
+
* columns (narrower terminal ⇒ every bar row wraps), and a task id is
|
|
105
|
+
* deliberately never truncated (long id ⇒ the worker row wraps at any
|
|
106
|
+
* width). Width is read per draw so a mid-run resize is handled. Width
|
|
107
|
+
* must be VISIBLE width — these lines carry ANSI, which occupies no
|
|
108
|
+
* column.
|
|
109
|
+
*
|
|
110
|
+
* Unknown width falls back to the logical count, which is a KNOWN
|
|
111
|
+
* residual rather than a guarantee: a wrapped line then under-erases
|
|
112
|
+
* exactly as it did before this function existed. It is deliberate,
|
|
113
|
+
* because the alternatives are worse — guessing a width over-erases on a
|
|
114
|
+
* wider terminal, and `ESC[J` clears to end of SCREEN, so over-erasing
|
|
115
|
+
* destroys output above the region instead of merely leaving junk below
|
|
116
|
+
* it. Reachable only on a TTY whose winsize ioctl fails, since a non-TTY
|
|
117
|
+
* disables the region outright.
|
|
118
|
+
*/
|
|
119
|
+
const regionRows = (lines: readonly string[]): number => {
|
|
120
|
+
const cols = stream.columns
|
|
121
|
+
if (cols === undefined || cols <= 0) return lines.length
|
|
122
|
+
let rows = 0
|
|
123
|
+
for (const line of lines) rows += Math.max(1, Math.ceil(Bun.stringWidth(line) / cols))
|
|
124
|
+
return rows
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
// Erase sequence for whatever is currently shown. A single row keeps
|
|
128
|
+
// the exact legacy bytes (ESC[2K\r); anything taller moves to its
|
|
129
|
+
// top row and clears to end of screen.
|
|
130
|
+
const eraseSeq = (): string => (shownRows > 1 ? `\r\x1b[${shownRows - 1}A\x1b[J` : CLEAR)
|
|
92
131
|
|
|
93
132
|
const draw = (): void => {
|
|
94
133
|
// Whatever was pending is now painted — the trailing draw would
|
|
@@ -97,7 +136,7 @@ export function createOutputWriter(
|
|
|
97
136
|
const erase = shown ? eraseSeq() : CLEAR
|
|
98
137
|
stream.write(erase + current!.join('\n'))
|
|
99
138
|
shown = true
|
|
100
|
-
|
|
139
|
+
shownRows = regionRows(current!)
|
|
101
140
|
lastDraw = now()
|
|
102
141
|
}
|
|
103
142
|
const set = (lines: readonly string[], o: { force?: boolean }): void => {
|
|
@@ -136,7 +175,7 @@ export function createOutputWriter(
|
|
|
136
175
|
if (shown) {
|
|
137
176
|
stream.write(eraseSeq())
|
|
138
177
|
shown = false
|
|
139
|
-
|
|
178
|
+
shownRows = 0
|
|
140
179
|
}
|
|
141
180
|
stream.write(chunk)
|
|
142
181
|
if (chunk.length > 0) atLineStart = chunk.endsWith('\n')
|
|
@@ -156,7 +195,7 @@ export function createOutputWriter(
|
|
|
156
195
|
if (shown) {
|
|
157
196
|
stream.write(eraseSeq())
|
|
158
197
|
shown = false
|
|
159
|
-
|
|
198
|
+
shownRows = 0
|
|
160
199
|
}
|
|
161
200
|
},
|
|
162
201
|
}
|
|
@@ -28,7 +28,7 @@ const GRADIENT_TO = [0xf4, 0x72, 0xb6] as const
|
|
|
28
28
|
const RULE_DASHES = 57
|
|
29
29
|
|
|
30
30
|
/** Total visible rule width: `\u2500 <mark> ` + dashes. Shared by summary + header. */
|
|
31
|
-
|
|
31
|
+
function gradientRule(colors: ColorSupport, mark = 'vx'): string {
|
|
32
32
|
const head = `\u2500 `
|
|
33
33
|
const dashes = '\u2500'.repeat(Math.max(8, RULE_DASHES + 2 - mark.length))
|
|
34
34
|
if (!colors.enabled) return `${head}${mark} ${dashes}`
|
|
@@ -309,6 +309,27 @@ export function formatRunSummary(
|
|
|
309
309
|
)
|
|
310
310
|
}
|
|
311
311
|
|
|
312
|
+
/**
|
|
313
|
+
* Post-summary section naming the tasks a shutdown signal killed. An aborted
|
|
314
|
+
* task did no work, so it is in no tally bucket and no history row — but the
|
|
315
|
+
* run still exits non-zero, and without this the user reads a red exit over a
|
|
316
|
+
* fully green summary that names nothing. The interactive Ctrl-C path never
|
|
317
|
+
* gets here (the signal handler exits before any outcome lands), so this only
|
|
318
|
+
* ever prints for a run that reached its summary WITH a task killed by some
|
|
319
|
+
* other signal — an external `kill`, a supervisor, a self-terminating child.
|
|
320
|
+
* Empty when nothing aborted.
|
|
321
|
+
*/
|
|
322
|
+
export function formatAbortedSection(outcomes: readonly TaskOutcome[]): string[] {
|
|
323
|
+
const aborted = outcomes.filter((o) => o.status === 'aborted')
|
|
324
|
+
if (aborted.length === 0) return []
|
|
325
|
+
const lines = [
|
|
326
|
+
'',
|
|
327
|
+
` Aborted: ${aborted.length} task${aborted.length === 1 ? '' : 's'} killed by a shutdown signal — not counted above`,
|
|
328
|
+
]
|
|
329
|
+
for (const o of aborted) lines.push(` ✗ ${o.node.id} — exit ${o.exitCode}, nothing cached`)
|
|
330
|
+
return lines
|
|
331
|
+
}
|
|
332
|
+
|
|
312
333
|
export function formatDuration(ms: number): string {
|
|
313
334
|
if (ms < 1000) return `${Math.round(ms)}ms`
|
|
314
335
|
return `${(ms / 1000).toFixed(2)}s`
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
// Shared outcome-tally helper.
|
|
2
|
-
//
|
|
3
|
-
// numbers; centralising
|
|
1
|
+
// Shared outcome-tally helper. `summary.ts` (end-of-run terminal block),
|
|
2
|
+
// `run-artifacts.ts` (`--summarize` JSON) and `run-report.ts`
|
|
3
|
+
// (`--report=markdown`) all count the same numbers; centralising the bucket
|
|
4
|
+
// rules keeps the three surfaces from disagreeing about one run.
|
|
4
5
|
|
|
5
6
|
import { isGroupTask, type TaskOutcome } from '../graph/index.js'
|
|
6
7
|
|
|
@@ -16,25 +17,31 @@ export interface Tally {
|
|
|
16
17
|
/** Cache hits whose tree already matched — nothing was written. */
|
|
17
18
|
upToDate: number
|
|
18
19
|
cachedRemote: number
|
|
20
|
+
/**
|
|
21
|
+
* Killed by a shutdown signal. No work, so it joins no outcome bucket and
|
|
22
|
+
* not `total` — but it is counted, because it makes the run red: a red run
|
|
23
|
+
* whose every counted task succeeded is undiagnosable without this.
|
|
24
|
+
*/
|
|
25
|
+
aborted: number
|
|
19
26
|
/** Tasks counted toward `total`. Group tasks are excluded by default. */
|
|
20
27
|
total: number
|
|
21
28
|
}
|
|
22
29
|
|
|
23
30
|
/**
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
* `success` → +successful
|
|
31
|
-
* `cache-hit` → +successful, +cachedLocal
|
|
32
|
-
* `cache-hit-remote` → +successful, +cachedRemote
|
|
33
|
-
* `failed` → +failed
|
|
34
|
-
* `skipped` → +skipped
|
|
31
|
+
* The minimum an outcome must expose to be bucketed. The in-process
|
|
32
|
+
* `TaskOutcome` and the serializable `OutcomeView` both satisfy it; they
|
|
33
|
+
* differ only in how they carry group-ness (a live node ref vs. a projected
|
|
34
|
+
* boolean), so each entry point supplies that and the RULE lives in one
|
|
35
|
+
* place. Declared structurally here rather than importing `OutcomeView`,
|
|
36
|
+
* which would close a module cycle (events → summary → tally).
|
|
35
37
|
*/
|
|
36
|
-
export
|
|
37
|
-
|
|
38
|
+
export interface TallyItem {
|
|
39
|
+
status: TaskOutcome['status']
|
|
40
|
+
restored?: boolean
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function emptyTally(): Tally {
|
|
44
|
+
return {
|
|
38
45
|
successful: 0,
|
|
39
46
|
failed: 0,
|
|
40
47
|
skipped: 0,
|
|
@@ -43,27 +50,63 @@ export function tallyOutcomes(outcomes: readonly TaskOutcome[]): Tally {
|
|
|
43
50
|
restoredRemote: 0,
|
|
44
51
|
upToDate: 0,
|
|
45
52
|
cachedRemote: 0,
|
|
53
|
+
aborted: 0,
|
|
46
54
|
total: 0,
|
|
47
55
|
}
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Fold ONE outcome into the tally — the single owner of the bucket rules.
|
|
60
|
+
*
|
|
61
|
+
* Group tasks (no `exec`) aren't real work; including them inflates the
|
|
62
|
+
* per-run "N total" count in ways that mislead ("3 of 4 cached" when one was
|
|
63
|
+
* a group that ran nothing).
|
|
64
|
+
*
|
|
65
|
+
* Status bucket rules:
|
|
66
|
+
* `success` → +successful
|
|
67
|
+
* `cache-hit` → +successful, +cachedLocal
|
|
68
|
+
* `cache-hit-remote` → +successful, +cachedRemote
|
|
69
|
+
* `failed` → +failed
|
|
70
|
+
* `skipped` → +skipped
|
|
71
|
+
* `aborted` → +aborted only (no bucket, no total)
|
|
72
|
+
*/
|
|
73
|
+
function fold(t: Tally, o: TallyItem, isGroup: boolean): void {
|
|
74
|
+
if (isGroup) return
|
|
75
|
+
if (o.status === 'aborted') {
|
|
76
|
+
t.aborted++
|
|
77
|
+
return
|
|
67
78
|
}
|
|
79
|
+
t.total++
|
|
80
|
+
if (o.status === 'success') t.successful++
|
|
81
|
+
else if (o.status === 'cache-hit') {
|
|
82
|
+
t.successful++
|
|
83
|
+
t.cachedLocal++
|
|
84
|
+
if (o.restored === true) t.restoredLocal++
|
|
85
|
+
else t.upToDate++
|
|
86
|
+
} else if (o.status === 'cache-hit-remote') {
|
|
87
|
+
t.successful++
|
|
88
|
+
t.cachedRemote++
|
|
89
|
+
if (o.restored === true) t.restoredRemote++
|
|
90
|
+
else t.upToDate++
|
|
91
|
+
} else if (o.status === 'failed') t.failed++
|
|
92
|
+
else if (o.status === 'skipped') t.skipped++
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** Walk live outcomes and return a `Tally`. */
|
|
96
|
+
export function tallyOutcomes(outcomes: readonly TaskOutcome[]): Tally {
|
|
97
|
+
const t = emptyTally()
|
|
98
|
+
for (const o of outcomes) fold(t, o, isGroupTask(o.node))
|
|
99
|
+
return t
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* The same tally over the serializable projection (`OutcomeView`), which
|
|
104
|
+
* carries group-ness as a flag because it has no node back-ref. An absent
|
|
105
|
+
* flag reads as "not a group" — the only outcomes without one predate the
|
|
106
|
+
* field and none of them are groups.
|
|
107
|
+
*/
|
|
108
|
+
export function tallyViews(views: readonly (TallyItem & { isGroup?: boolean })[]): Tally {
|
|
109
|
+
const t = emptyTally()
|
|
110
|
+
for (const v of views) fold(t, v, v.isGroup === true)
|
|
68
111
|
return t
|
|
69
112
|
}
|
|
@@ -1,9 +1,16 @@
|
|
|
1
1
|
import path from 'node:path'
|
|
2
2
|
import type { TaskConfig, CacheConfig } from '../config.js'
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
type CacheKeyInput,
|
|
5
|
+
type CacheLayer,
|
|
6
|
+
resolveInputs,
|
|
7
|
+
type GitFilesCache,
|
|
8
|
+
WORKSPACE_OUTPUT_PREFIX,
|
|
9
|
+
} from '../cache/index.js'
|
|
10
|
+
import type { InputFile, TaskInputs } from '../exec/index.js'
|
|
4
11
|
import type { TaskNode, TaskOutcome } from '../graph/index.js'
|
|
5
|
-
import { xxh3hex } from '../util/index.js'
|
|
6
|
-
import { filterUpstreamHashes } from './upstream.js'
|
|
12
|
+
import { span, relPosix, xxh3hex } from '../util/index.js'
|
|
13
|
+
import { expandGroupUpstream, filterUpstreamHashes } from './upstream.js'
|
|
7
14
|
|
|
8
15
|
/**
|
|
9
16
|
* Per-run memoization caches for the two derived hashes that don't
|
|
@@ -83,6 +90,73 @@ export interface ComputeHashArgs {
|
|
|
83
90
|
* `computeGroupHash` is exported for that purpose.
|
|
84
91
|
*/
|
|
85
92
|
export async function computeTaskHash(args: ComputeHashArgs): Promise<string> {
|
|
93
|
+
const end = span('task hash')
|
|
94
|
+
try {
|
|
95
|
+
return await args.cache.key(await resolveKeyInput(args))
|
|
96
|
+
} finally {
|
|
97
|
+
end()
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* The key AND the structured input set behind it, for the executor seam.
|
|
103
|
+
* Miss path only: it re-runs the (memoized) resolution `computeTaskHash`
|
|
104
|
+
* did for the probe and retains the VALUES the key folded — env, runtime
|
|
105
|
+
* output, per-file digests — which `captureInto` deliberately reduces to
|
|
106
|
+
* digests because its rows are persisted. Per-file digests come from the
|
|
107
|
+
* same sources as the fold: the index-OID map for clean tracked files,
|
|
108
|
+
* else `hashFile`'s stat memo (already warm after `key()`).
|
|
109
|
+
*/
|
|
110
|
+
export async function describeTaskInputs(
|
|
111
|
+
args: ComputeHashArgs,
|
|
112
|
+
): Promise<{ hash: string; inputs: TaskInputs }> {
|
|
113
|
+
const input = await resolveKeyInput(args)
|
|
114
|
+
const hash = await args.cache.key(input)
|
|
115
|
+
const sorted = [...input.inputFiles].sort()
|
|
116
|
+
const digests = await Promise.all(
|
|
117
|
+
sorted.map((f) => input.fileHashes?.get(f) ?? args.cache.hashFile(f)),
|
|
118
|
+
)
|
|
119
|
+
const files: InputFile[] = sorted.map((f, i) => ({
|
|
120
|
+
path: relPosix(input.workspaceRoot, f),
|
|
121
|
+
digest: digests[i]!,
|
|
122
|
+
}))
|
|
123
|
+
// Groups expanded: this list describes the INPUT ROOT, not the key, and a
|
|
124
|
+
// group produces nothing of its own (see `CacheKeyInput.upstreamGraft`).
|
|
125
|
+
// Sorted by hash so the list — and any action digest derived from it — is
|
|
126
|
+
// independent of dependency declaration order.
|
|
127
|
+
const graft = [...(input.upstreamGraft ?? [])].sort((a, b) =>
|
|
128
|
+
a.hash < b.hash ? -1 : a.hash > b.hash ? 1 : 0,
|
|
129
|
+
)
|
|
130
|
+
// Output lists come from the local index — one SELECT for every upstream
|
|
131
|
+
// entry. An upstream with no entry (non-cacheable) contributes an empty list.
|
|
132
|
+
const rows = args.cache.loadOutputFilesBatch(graft.map((g) => g.hash))
|
|
133
|
+
const upstream = graft.map((g) => {
|
|
134
|
+
const outputs = (rows.get(g.hash) ?? []).map((r) =>
|
|
135
|
+
r.path.startsWith(WORKSPACE_OUTPUT_PREFIX)
|
|
136
|
+
? r.path.slice(WORKSPACE_OUTPUT_PREFIX.length)
|
|
137
|
+
: relPosix(input.workspaceRoot, path.join(g.projectDir, r.path)),
|
|
138
|
+
)
|
|
139
|
+
return { taskId: g.taskId, hash: g.hash, outputs }
|
|
140
|
+
})
|
|
141
|
+
return {
|
|
142
|
+
hash,
|
|
143
|
+
inputs: {
|
|
144
|
+
files,
|
|
145
|
+
env: input.envValues.map(([name, value]) => ({ name, value })),
|
|
146
|
+
runtime: (input.runtimeValues ?? []).map(([command, output]) => ({ command, output })),
|
|
147
|
+
workspaceRuntime: (input.workspaceRuntimeValues ?? []).map(([command, output]) => ({
|
|
148
|
+
command,
|
|
149
|
+
output,
|
|
150
|
+
})),
|
|
151
|
+
upstream,
|
|
152
|
+
packageJsonDigest: input.projectPackageJsonHash,
|
|
153
|
+
configDigest: input.taskConfigHash,
|
|
154
|
+
workspaceFingerprint: input.workspaceFingerprint,
|
|
155
|
+
},
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
async function resolveKeyInput(args: ComputeHashArgs): Promise<CacheKeyInput> {
|
|
86
160
|
const cfg = args.node.config
|
|
87
161
|
const cacheCfg: CacheConfig | undefined = cfg.cache
|
|
88
162
|
const outputs = cacheCfg?.outputs.files ?? []
|
|
@@ -113,6 +187,16 @@ export async function computeTaskHash(args: ComputeHashArgs): Promise<string> {
|
|
|
113
187
|
const upstreamHashes = upstreamPairs.map(([, hash]) => hash)
|
|
114
188
|
// hash → upstream task id, for capture-row naming only (not folded).
|
|
115
189
|
const upstreamIds = new Map(upstreamPairs.map(([id, hash]) => [hash, id]))
|
|
190
|
+
// The DEPENDENCY closure, groups expanded — for the executor's input root.
|
|
191
|
+
// Deliberately NOT the filtered set above: `cache.inputs.tasks` says which
|
|
192
|
+
// upstream KEYS this task's key folds, and what a task may READ is
|
|
193
|
+
// `dependsOn`. Locally every dependency's outputs are on disk before the
|
|
194
|
+
// command runs however the filter is written, so an executor that ships the
|
|
195
|
+
// input closure has to ship the same thing or the task sees less remotely
|
|
196
|
+
// than it does here. Not folded — see `upstreamGraft`.
|
|
197
|
+
const upstreamGraft = expandGroupUpstream(args.upstream)
|
|
198
|
+
.filter((u) => u.hash !== undefined)
|
|
199
|
+
.map((u) => ({ taskId: u.node.id, hash: u.hash!, projectDir: u.node.projectDir }))
|
|
116
200
|
// Trusted index-OID map for this project (populated by the run's
|
|
117
201
|
// bulk `git ls-files -s` + `git status` pass). Mapped files skip
|
|
118
202
|
// hashFile entirely; everything else falls back to the identical
|
|
@@ -136,7 +220,7 @@ export async function computeTaskHash(args: ComputeHashArgs): Promise<string> {
|
|
|
136
220
|
|
|
137
221
|
const effectiveForwardArgs = args.node.requested ? (args.forwardArgs ?? []) : []
|
|
138
222
|
|
|
139
|
-
return
|
|
223
|
+
return {
|
|
140
224
|
taskId: args.node.id,
|
|
141
225
|
taskConfigHash,
|
|
142
226
|
projectPackageJsonHash,
|
|
@@ -147,11 +231,13 @@ export async function computeTaskHash(args: ComputeHashArgs): Promise<string> {
|
|
|
147
231
|
workspaceRoot: args.workspaceRoot,
|
|
148
232
|
upstreamHashes,
|
|
149
233
|
upstreamIds,
|
|
234
|
+
upstreamGraft,
|
|
150
235
|
workspaceFingerprint: args.workspaceFingerprint,
|
|
151
236
|
forwardArgs: effectiveForwardArgs,
|
|
152
237
|
...(fileHashes !== undefined ? { fileHashes } : {}),
|
|
238
|
+
...(args.node.keyParts !== undefined ? { pluginParts: args.node.keyParts } : {}),
|
|
153
239
|
...(args.captureInto !== undefined ? { captureInto: args.captureInto } : {}),
|
|
154
|
-
}
|
|
240
|
+
}
|
|
155
241
|
}
|
|
156
242
|
|
|
157
243
|
/**
|
|
@@ -178,17 +264,25 @@ function hashTaskConfig(cfg: TaskConfig, hashCache?: HashCache): string {
|
|
|
178
264
|
}
|
|
179
265
|
|
|
180
266
|
/**
|
|
181
|
-
* Project the config for hashing: `exec.resources`
|
|
182
|
-
*
|
|
183
|
-
*
|
|
184
|
-
*
|
|
267
|
+
* Project the config for hashing: `exec.resources` (cores, memory AND the
|
|
268
|
+
* image a task must be matched to) and `exec.remote` are pure PLACEMENT
|
|
269
|
+
* hints with zero effect on outputs, so they're stripped — tuning a
|
|
270
|
+
* reservation, pinning a task to this machine, or naming the container it
|
|
271
|
+
* runs in never busts a cache. `remote` in particular must not: the whole
|
|
272
|
+
* contract of a remote executor is that the same command over the same
|
|
273
|
+
* inputs produces the same outputs, so a key that moved when placement
|
|
274
|
+
* changed would split a laptop from a worker pool over nothing. `image`
|
|
275
|
+
* rides the same rule one step further: a task that ran locally and the
|
|
276
|
+
* same task on a worker must share an entry, and a task whose output really
|
|
277
|
+
* does depend on its toolchain says so with `cache.inputs.runtime`, which
|
|
278
|
+
* works on both. A config that declares neither takes the fast path and stringifies byte-identically to before the fields existed
|
|
185
279
|
* (why this needs no CACHE_VERSION bump). `timeout`/`retries` stay folded
|
|
186
280
|
* — their keys are distinct by design (see the decision log); stripping
|
|
187
281
|
* them retroactively would bump CACHE_VERSION.
|
|
188
282
|
*/
|
|
189
283
|
function hashableConfig(cfg: TaskConfig): unknown {
|
|
190
|
-
if (cfg.exec?.resources === undefined) return cfg
|
|
191
|
-
const { resources: _resources, ...execRest } = cfg.exec
|
|
284
|
+
if (cfg.exec?.resources === undefined && cfg.exec?.remote === undefined) return cfg
|
|
285
|
+
const { resources: _resources, remote: _remote, ...execRest } = cfg.exec
|
|
192
286
|
return { ...cfg, exec: execRest }
|
|
193
287
|
}
|
|
194
288
|
|