@vzn/vx 0.0.614 → 0.0.616
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -5
- package/src/graph/task-graph.ts +100 -6
- package/src/orchestrator/execute-task.ts +37 -12
- package/src/orchestrator/local-shortcircuit.ts +15 -7
- package/src/orchestrator/run.ts +29 -15
- package/src/orchestrator/stable-keys.ts +32 -15
- package/src/orchestrator/task-hash.ts +10 -0
- package/src/orchestrator/upstream.ts +3 -7
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vzn/vx",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.616",
|
|
4
4
|
"description": "The fastest task runner and build cache for JavaScript monorepos.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -21,10 +21,10 @@
|
|
|
21
21
|
"node": ">=18"
|
|
22
22
|
},
|
|
23
23
|
"optionalDependencies": {
|
|
24
|
-
"@vzn/vx-linux-x64": "0.0.
|
|
25
|
-
"@vzn/vx-linux-arm64": "0.0.
|
|
26
|
-
"@vzn/vx-darwin-x64": "0.0.
|
|
27
|
-
"@vzn/vx-darwin-arm64": "0.0.
|
|
24
|
+
"@vzn/vx-linux-x64": "0.0.616",
|
|
25
|
+
"@vzn/vx-linux-arm64": "0.0.616",
|
|
26
|
+
"@vzn/vx-darwin-x64": "0.0.616",
|
|
27
|
+
"@vzn/vx-darwin-arm64": "0.0.616"
|
|
28
28
|
},
|
|
29
29
|
"dependencies": {
|
|
30
30
|
"@anthropic-ai/sandbox-runtime": "^0.0.76"
|
package/src/graph/task-graph.ts
CHANGED
|
@@ -2,6 +2,7 @@ import type { ProjectConfig, TaskConfig, WorkspaceRules } from '../config.js'
|
|
|
2
2
|
import {
|
|
3
3
|
asTrees,
|
|
4
4
|
isLiteralPattern,
|
|
5
|
+
normalizeGlob,
|
|
5
6
|
relPosix,
|
|
6
7
|
splitNegations,
|
|
7
8
|
staticPrefix,
|
|
@@ -1152,6 +1153,64 @@ function collideAcross(
|
|
|
1152
1153
|
*/
|
|
1153
1154
|
const GLOB_HEAD_END = /[*?{}\\!]/
|
|
1154
1155
|
|
|
1156
|
+
/** `firstSegment`'s answer for a glob that can start anywhere. */
|
|
1157
|
+
const ANYWHERE = ''
|
|
1158
|
+
|
|
1159
|
+
/** Where a glob's first segment ends: a separator, or anything `GLOB_HEAD_END` stops at. */
|
|
1160
|
+
const FIRST_SEGMENT_END = /[*?{}\\!/]/
|
|
1161
|
+
|
|
1162
|
+
const firstSegments = new Map<string, string>()
|
|
1163
|
+
|
|
1164
|
+
/**
|
|
1165
|
+
* The first path segment every path a glob (or its `asTrees` twin) can
|
|
1166
|
+
* match starts with, or `ANYWHERE`. A task glob matches whole paths, so
|
|
1167
|
+
* two globs with different first segments select disjoint sets, and none
|
|
1168
|
+
* of `outputsOverlap`'s three cases can hold for them. Memoised by the
|
|
1169
|
+
* raw string: a pure function, and one glob (`src/**`) recurs in every
|
|
1170
|
+
* project.
|
|
1171
|
+
*/
|
|
1172
|
+
function firstSegment(raw: string): string {
|
|
1173
|
+
let seg = firstSegments.get(raw)
|
|
1174
|
+
if (seg !== undefined) return seg
|
|
1175
|
+
const g = normalizeGlob(raw)
|
|
1176
|
+
const cut = g.search(FIRST_SEGMENT_END)
|
|
1177
|
+
seg = cut === -1 ? g : g[cut] === '/' ? g.slice(0, cut) : ANYWHERE
|
|
1178
|
+
if (seg === '.' || seg === '..') seg = ANYWHERE
|
|
1179
|
+
firstSegments.set(raw, seg)
|
|
1180
|
+
return seg
|
|
1181
|
+
}
|
|
1182
|
+
|
|
1183
|
+
/**
|
|
1184
|
+
* False only when no pair `overlapCandidates` could return can overlap:
|
|
1185
|
+
* every glob starts in a named segment and no two members (on different
|
|
1186
|
+
* sides, given `sideOf`) share one. The per-project buckets are nearly
|
|
1187
|
+
* all of this shape (`src/**` read, `dist/**` written), and building the
|
|
1188
|
+
* indexes for each cost the warm run's graph stage tens of milliseconds
|
|
1189
|
+
* at 1,090 projects (X-148).
|
|
1190
|
+
*/
|
|
1191
|
+
function mayOverlap<T>(
|
|
1192
|
+
tasks: readonly T[],
|
|
1193
|
+
globsOf: (n: T) => readonly string[] | undefined,
|
|
1194
|
+
sideOf?: (n: T) => boolean,
|
|
1195
|
+
): boolean {
|
|
1196
|
+
// Per segment: the first member seen (one side) or the sides seen as a mask.
|
|
1197
|
+
const seen = new Map<string, number>()
|
|
1198
|
+
for (let i = 0; i < tasks.length; i++) {
|
|
1199
|
+
const mark = sideOf === undefined ? i : sideOf(tasks[i]!) ? 2 : 1
|
|
1200
|
+
for (const g of globsOf(tasks[i]!) ?? []) {
|
|
1201
|
+
const seg = firstSegment(g)
|
|
1202
|
+
if (seg === ANYWHERE) return true
|
|
1203
|
+
const was = seen.get(seg)
|
|
1204
|
+
if (was === undefined) seen.set(seg, mark)
|
|
1205
|
+
else if (sideOf === undefined) {
|
|
1206
|
+
if (was !== i) return true
|
|
1207
|
+
} else if ((was | mark) === 3) return true
|
|
1208
|
+
else seen.set(seg, was | mark)
|
|
1209
|
+
}
|
|
1210
|
+
}
|
|
1211
|
+
return false
|
|
1212
|
+
}
|
|
1213
|
+
|
|
1155
1214
|
/**
|
|
1156
1215
|
* The pairs `[i, j]`, `i < j`, of `tasks` whose declared outputs CAN
|
|
1157
1216
|
* overlap, ascending by `i` then `j`: a superset of the pairs
|
|
@@ -1181,6 +1240,7 @@ function overlapCandidates<T>(
|
|
|
1181
1240
|
globsOf: (n: T) => readonly string[] | undefined,
|
|
1182
1241
|
sideOf?: (n: T) => boolean,
|
|
1183
1242
|
): Array<[number, number]> {
|
|
1243
|
+
if (!mayOverlap(tasks, globsOf, sideOf)) return []
|
|
1184
1244
|
const sides = sideOf === undefined ? 1 : 2
|
|
1185
1245
|
// The index a member of side `s` looks up: its own side's, or the other's.
|
|
1186
1246
|
const across = (s: number): number => (sides === 1 ? s : 1 - s)
|
|
@@ -1356,6 +1416,7 @@ function refuseSelfClean(node: TaskNode): void {
|
|
|
1356
1416
|
const cache = node.config.cache
|
|
1357
1417
|
if (cache === undefined) return
|
|
1358
1418
|
for (const field of ['files', 'workspaceFiles'] as const) {
|
|
1419
|
+
if (!coversAny(cache.inputs[field], cache.outputs[field])) continue
|
|
1359
1420
|
const inputs = splitNegations(cache.inputs[field] ?? [])
|
|
1360
1421
|
const out = splitNegations(cache.outputs[field] ?? [])
|
|
1361
1422
|
const outputs = out.positive
|
|
@@ -1384,6 +1445,39 @@ function refuseSelfClean(node: TaskNode): void {
|
|
|
1384
1445
|
}
|
|
1385
1446
|
}
|
|
1386
1447
|
|
|
1448
|
+
/**
|
|
1449
|
+
* False only when no positive output can cover a positive input: each
|
|
1450
|
+
* starts in a named first segment and no input shares one with an output
|
|
1451
|
+
* (`firstSegment`). The common task reads `src/**` and writes `dist/**`.
|
|
1452
|
+
*/
|
|
1453
|
+
function coversAny(
|
|
1454
|
+
inputs: readonly string[] | undefined,
|
|
1455
|
+
outputs: readonly string[] | undefined,
|
|
1456
|
+
): boolean {
|
|
1457
|
+
if (inputs === undefined || outputs === undefined) return false
|
|
1458
|
+
let anyOutput = false
|
|
1459
|
+
for (const go of outputs) {
|
|
1460
|
+
if (go.startsWith('!')) continue
|
|
1461
|
+
anyOutput = true
|
|
1462
|
+
if (firstSegment(go) === ANYWHERE) return true
|
|
1463
|
+
}
|
|
1464
|
+
if (!anyOutput) return false
|
|
1465
|
+
for (const gi of inputs) {
|
|
1466
|
+
if (gi.startsWith('!')) continue
|
|
1467
|
+
const seg = firstSegment(gi)
|
|
1468
|
+
if (seg === ANYWHERE) return true
|
|
1469
|
+
for (const go of outputs) if (!go.startsWith('!') && firstSegment(go) === seg) return true
|
|
1470
|
+
}
|
|
1471
|
+
return false
|
|
1472
|
+
}
|
|
1473
|
+
|
|
1474
|
+
/** A glob list's positive entries: the list itself when it takes nothing back. */
|
|
1475
|
+
function positives(globs: readonly string[] | undefined): readonly string[] {
|
|
1476
|
+
if (globs === undefined) return []
|
|
1477
|
+
for (const g of globs) if (g.startsWith('!')) return splitNegations(globs).positive
|
|
1478
|
+
return globs
|
|
1479
|
+
}
|
|
1480
|
+
|
|
1387
1481
|
/** One task's inputs (`reads`) or outputs in one namespace, for `detectInputOverlaps`. */
|
|
1388
1482
|
interface Side {
|
|
1389
1483
|
node: TaskNode
|
|
@@ -1431,8 +1525,8 @@ function detectInputOverlaps(nodes: Map<string, TaskNode>, workspaceRoot?: strin
|
|
|
1431
1525
|
for (const n of nodes.values()) {
|
|
1432
1526
|
const cache = n.config.cache
|
|
1433
1527
|
if (cache === undefined) continue
|
|
1434
|
-
const outFiles =
|
|
1435
|
-
const outWs =
|
|
1528
|
+
const outFiles = positives(cache.outputs.files)
|
|
1529
|
+
const outWs = positives(cache.outputs.workspaceFiles ?? [])
|
|
1436
1530
|
if (outFiles.length > 0) {
|
|
1437
1531
|
bucket(n.projectName).push({ node: n, globs: outFiles, reads: false })
|
|
1438
1532
|
filesWriters.push(n)
|
|
@@ -1443,8 +1537,8 @@ function detectInputOverlaps(nodes: Map<string, TaskNode>, workspaceRoot?: strin
|
|
|
1443
1537
|
wsWriters.push(side)
|
|
1444
1538
|
}
|
|
1445
1539
|
if (isGroupTask(n)) continue
|
|
1446
|
-
const inFiles =
|
|
1447
|
-
const inWs =
|
|
1540
|
+
const inFiles = positives(cache.inputs.files)
|
|
1541
|
+
const inWs = positives(cache.inputs.workspaceFiles ?? [])
|
|
1448
1542
|
if (inFiles.length > 0) {
|
|
1449
1543
|
bucket(n.projectName).push({ node: n, globs: inFiles, reads: true })
|
|
1450
1544
|
filesReaders.push(n)
|
|
@@ -1471,7 +1565,7 @@ function detectInputOverlaps(nodes: Map<string, TaskNode>, workspaceRoot?: strin
|
|
|
1471
1565
|
const sides = [...wsWriters]
|
|
1472
1566
|
for (const node of filesReaders) {
|
|
1473
1567
|
const rel = relPosix(workspaceRoot, node.projectDir)
|
|
1474
|
-
const globs =
|
|
1568
|
+
const globs = positives(node.config.cache!.inputs.files).map((g) =>
|
|
1475
1569
|
rel === '' ? g : `${rel}/${g}`,
|
|
1476
1570
|
)
|
|
1477
1571
|
sides.push({ node, globs, reads: true, rel })
|
|
@@ -1488,7 +1582,7 @@ function detectInputOverlaps(nodes: Map<string, TaskNode>, workspaceRoot?: strin
|
|
|
1488
1582
|
if (workspaceRoot !== undefined) {
|
|
1489
1583
|
for (const node of filesWriters) {
|
|
1490
1584
|
const rel = relPosix(workspaceRoot, node.projectDir)
|
|
1491
|
-
const globs =
|
|
1585
|
+
const globs = positives(node.config.cache!.outputs.files).map((g) =>
|
|
1492
1586
|
rel === '' ? g : `${rel}/${g}`,
|
|
1493
1587
|
)
|
|
1494
1588
|
rooted.push({ node, globs, reads: false, rel })
|
|
@@ -76,12 +76,13 @@ export { restoreHit, type RestoreHitArgs } from './hit-restore.js'
|
|
|
76
76
|
import type { DeferredOutputs } from './deferred-outputs.js'
|
|
77
77
|
import type { Logger } from './logger.js'
|
|
78
78
|
import {
|
|
79
|
-
|
|
79
|
+
computeGroupHash,
|
|
80
80
|
computeTaskHash,
|
|
81
81
|
describeTaskInputs,
|
|
82
82
|
type HashCache,
|
|
83
83
|
movedInput,
|
|
84
84
|
type TaskInputComponent,
|
|
85
|
+
type UpfrontGroupKey,
|
|
85
86
|
} from './task-hash.js'
|
|
86
87
|
import { getContext } from './remote-prefetch.js'
|
|
87
88
|
import { expandGroupUpstream, filterUpstreamHashes } from './upstream.js'
|
|
@@ -190,6 +191,11 @@ export interface ExecuteArgs {
|
|
|
190
191
|
* used verbatim instead of deriving the same key again.
|
|
191
192
|
*/
|
|
192
193
|
upfrontKey?: string
|
|
194
|
+
/**
|
|
195
|
+
* An unkeyed group's up-front key (`deriveStableKeys`' `groupKeys`): used
|
|
196
|
+
* verbatim when the live upstream folds the same ids and keys.
|
|
197
|
+
*/
|
|
198
|
+
upfrontGroupKey?: UpfrontGroupKey
|
|
193
199
|
/**
|
|
194
200
|
* Start the sandbox runtime, on the first task that executes inside one
|
|
195
201
|
* (run.ts, `prepareSandbox`). Absent when no task in the run declares a
|
|
@@ -263,17 +269,25 @@ export async function executeTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
263
269
|
*/
|
|
264
270
|
async function executeGroupTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
265
271
|
const wallclockNs = process.hrtime.bigint() - args.runStartHrTimeNs
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
272
|
+
// `computeGroupKey`: an unkeyed group's key is its upstream's alone, so
|
|
273
|
+
// the up-front one stands when that upstream is unchanged.
|
|
274
|
+
const upfront = args.upfrontGroupKey
|
|
275
|
+
const hash =
|
|
276
|
+
args.node.config.cache === undefined
|
|
277
|
+
? upfront !== undefined && sameUpstream(args.upstream, upfront.upstream)
|
|
278
|
+
? upfront.key
|
|
279
|
+
: computeGroupHash(args.upstream)
|
|
280
|
+
: await computeTaskHash({
|
|
281
|
+
node: args.node,
|
|
282
|
+
upstream: args.upstream,
|
|
283
|
+
workspaceRoot: args.workspaceRoot,
|
|
284
|
+
workspaceFingerprint: args.workspaceFingerprint,
|
|
285
|
+
cache: args.cache,
|
|
286
|
+
forwardArgs: args.forwardArgs,
|
|
287
|
+
nestedProjectDirs: args.nestedProjectDirs,
|
|
288
|
+
...(args.gitFilesCache !== undefined ? { gitFilesCache: args.gitFilesCache } : {}),
|
|
289
|
+
...(args.hashCache !== undefined ? { hashCache: args.hashCache } : {}),
|
|
290
|
+
})
|
|
277
291
|
return {
|
|
278
292
|
node: args.node,
|
|
279
293
|
status: 'success',
|
|
@@ -288,6 +302,17 @@ async function executeGroupTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
288
302
|
}
|
|
289
303
|
}
|
|
290
304
|
|
|
305
|
+
/** The same ids with the same keys, in order: what `computeGroupHash` reads. */
|
|
306
|
+
function sameUpstream(live: TaskOutcome[], upfront: TaskOutcome[]): boolean {
|
|
307
|
+
if (live.length !== upfront.length) return false
|
|
308
|
+
for (let i = 0; i < live.length; i++) {
|
|
309
|
+
const a = live[i]
|
|
310
|
+
const b = upfront[i]!
|
|
311
|
+
if (a === undefined || a.node.id !== b.node.id || a.hash !== b.hash) return false
|
|
312
|
+
}
|
|
313
|
+
return true
|
|
314
|
+
}
|
|
315
|
+
|
|
291
316
|
/**
|
|
292
317
|
* Persistent task: dev server / file watcher / daemon. Spawn, wait
|
|
293
318
|
* for ready (regex match or immediate), stash the subprocess in the
|
|
@@ -45,7 +45,7 @@ import { normalizeGlob, relPosix, span, staticPrefix } from '../util/index.js'
|
|
|
45
45
|
import type { CacheEntry, CacheLayer, GitFilesCache } from '../cache/index.js'
|
|
46
46
|
import type { TaskNode } from '../graph/index.js'
|
|
47
47
|
import { deriveStableKeys, workspaceInputsReach } from './stable-keys.js'
|
|
48
|
-
import type { HashCache } from './task-hash.js'
|
|
48
|
+
import type { HashCache, UpfrontGroupKey } from './task-hash.js'
|
|
49
49
|
import { getContext } from './remote-prefetch.js'
|
|
50
50
|
|
|
51
51
|
export interface ShortCircuitArgs {
|
|
@@ -91,6 +91,8 @@ export interface ShortCircuit {
|
|
|
91
91
|
* again (`deriveStableKeys`' `uncachedKeys`).
|
|
92
92
|
*/
|
|
93
93
|
uncachedKeys: Map<string, string>
|
|
94
|
+
/** Unkeyed group id → its up-front key (`deriveStableKeys`' `groupKeys`). */
|
|
95
|
+
groupKeys: Map<string, UpfrontGroupKey>
|
|
94
96
|
}
|
|
95
97
|
|
|
96
98
|
/**
|
|
@@ -105,11 +107,17 @@ export interface ShortCircuit {
|
|
|
105
107
|
export async function startLocalShortCircuit(args: ShortCircuitArgs): Promise<ShortCircuit> {
|
|
106
108
|
let stableKeys
|
|
107
109
|
const uncachedKeys = new Map<string, string>()
|
|
110
|
+
const groupKeys = new Map<string, UpfrontGroupKey>()
|
|
108
111
|
const endKeys = span('stable keys')
|
|
109
112
|
try {
|
|
110
|
-
stableKeys = await deriveStableKeys({ ...args, uncachedKeys })
|
|
113
|
+
stableKeys = await deriveStableKeys({ ...args, uncachedKeys, groupKeys })
|
|
111
114
|
} catch {
|
|
112
|
-
return {
|
|
115
|
+
return {
|
|
116
|
+
preProbed: new Map(),
|
|
117
|
+
restoreTier: new Set(),
|
|
118
|
+
uncachedKeys: new Map(),
|
|
119
|
+
groupKeys: new Map(),
|
|
120
|
+
}
|
|
113
121
|
} finally {
|
|
114
122
|
endKeys()
|
|
115
123
|
}
|
|
@@ -120,14 +128,14 @@ export async function startLocalShortCircuit(args: ShortCircuitArgs): Promise<Sh
|
|
|
120
128
|
const preProbed = new Map<string, ProbedEntry>()
|
|
121
129
|
const restoreTier = new Set<string>()
|
|
122
130
|
if (candidates.length === 0 && uncachedKeys.size === 0) {
|
|
123
|
-
return { preProbed, restoreTier, uncachedKeys }
|
|
131
|
+
return { preProbed, restoreTier, uncachedKeys, groupKeys }
|
|
124
132
|
}
|
|
125
133
|
|
|
126
134
|
const keptOut = restoreTierExclusions(args.nodes, args.workspaceRoot)
|
|
127
135
|
// A root-anchored output may land in a project no edge leads from, so an
|
|
128
136
|
// uncached key there is derived again once the writer may have run.
|
|
129
137
|
for (const id of keptOut) uncachedKeys.delete(id)
|
|
130
|
-
if (candidates.length === 0) return { preProbed, restoreTier, uncachedKeys }
|
|
138
|
+
if (candidates.length === 0) return { preProbed, restoreTier, uncachedKeys, groupKeys }
|
|
131
139
|
|
|
132
140
|
// Bounded pool over the stable candidates: probe local ONCE each. A
|
|
133
141
|
// confirmed hit becomes restore-tier (unless workspace outputs disable
|
|
@@ -153,7 +161,7 @@ export async function startLocalShortCircuit(args: ShortCircuitArgs): Promise<Sh
|
|
|
153
161
|
preProbed.set(node.id, { hash, hit })
|
|
154
162
|
if (hit !== null && !keptOut.has(node.id)) restoreTier.add(node.id)
|
|
155
163
|
}
|
|
156
|
-
return { preProbed, restoreTier, uncachedKeys }
|
|
164
|
+
return { preProbed, restoreTier, uncachedKeys, groupKeys }
|
|
157
165
|
} catch {
|
|
158
166
|
// Fall through to the per-hash pool, which isolates a failing probe
|
|
159
167
|
// to its own task.
|
|
@@ -176,7 +184,7 @@ export async function startLocalShortCircuit(args: ShortCircuitArgs): Promise<Sh
|
|
|
176
184
|
}
|
|
177
185
|
await Promise.all(Array.from({ length: workers }, () => pump()))
|
|
178
186
|
|
|
179
|
-
return { preProbed, restoreTier, uncachedKeys }
|
|
187
|
+
return { preProbed, restoreTier, uncachedKeys, groupKeys }
|
|
180
188
|
}
|
|
181
189
|
|
|
182
190
|
/**
|
package/src/orchestrator/run.ts
CHANGED
|
@@ -54,6 +54,7 @@ import { isDefaultBuild } from './projects.js'
|
|
|
54
54
|
import { prepareSandbox } from './sandbox-request.js'
|
|
55
55
|
import type { OutputDirSnapshot, SaveFacts } from './miss-save.js'
|
|
56
56
|
import { admitTasks, taintTracker } from './admission.js'
|
|
57
|
+
import type { ExecuteArgs } from './execute-task.js'
|
|
57
58
|
import { excludedTaint } from './excluded-keys.js'
|
|
58
59
|
import { keyUpstream } from './upstream.js'
|
|
59
60
|
import { busLogger, createEventBus, terminalSubscriber, type EventBus } from './events.js'
|
|
@@ -116,6 +117,7 @@ const emptyShortCircuit = (): ShortCircuit => ({
|
|
|
116
117
|
preProbed: new Map(),
|
|
117
118
|
restoreTier: new Set(),
|
|
118
119
|
uncachedKeys: new Map(),
|
|
120
|
+
groupKeys: new Map(),
|
|
119
121
|
})
|
|
120
122
|
|
|
121
123
|
/**
|
|
@@ -938,9 +940,19 @@ async function runOnBus(
|
|
|
938
940
|
// The scheduler's fail-fast stop, as the retry loop of a task already
|
|
939
941
|
// in flight sees it.
|
|
940
942
|
const failFast = new AbortController()
|
|
941
|
-
|
|
943
|
+
// A literal of the fields every task carries, then each optional one
|
|
944
|
+
// assigned only when present: twelve conditional spreads per task cost
|
|
945
|
+
// ~18 ms of object copying on a 1,090-package warm run (X-149).
|
|
946
|
+
const track = telemetry?.track
|
|
947
|
+
const armSandbox = sandboxArmer !== null ? () => sandboxArmer.arm() : undefined
|
|
948
|
+
const buildExecuteArgs = (
|
|
949
|
+
node: TaskNode,
|
|
950
|
+
upstream: TaskOutcome[],
|
|
951
|
+
reuseProbe = true,
|
|
952
|
+
): ExecuteArgs => {
|
|
942
953
|
const probe = reuseProbe ? shortCircuit.preProbed.get(node.id) : undefined
|
|
943
954
|
const upfrontKey = shortCircuit.uncachedKeys.get(node.id)
|
|
955
|
+
const upfrontGroupKey = shortCircuit.groupKeys.get(node.id)
|
|
944
956
|
if (
|
|
945
957
|
options.continueMode === 'always' &&
|
|
946
958
|
node.deps.some((d) => deadServerBehind(nodes, serverDied, d) !== undefined)
|
|
@@ -948,7 +960,7 @@ async function runOnBus(
|
|
|
948
960
|
taintSeeds.add(node.id)
|
|
949
961
|
const tainted = taint.judge(node, upstream)
|
|
950
962
|
if (tainted) taintedRan.add(node.id)
|
|
951
|
-
|
|
963
|
+
const a: ExecuteArgs = {
|
|
952
964
|
node,
|
|
953
965
|
upstream,
|
|
954
966
|
workspaceRoot,
|
|
@@ -956,31 +968,18 @@ async function runOnBus(
|
|
|
956
968
|
cache,
|
|
957
969
|
cachePolicy: policy,
|
|
958
970
|
forwardArgs: options.forwardArgs,
|
|
959
|
-
...(options.retries !== undefined ? { retries: options.retries } : {}),
|
|
960
|
-
...(taskTimeoutDefault !== undefined ? { timeout: taskTimeoutDefault } : {}),
|
|
961
971
|
log,
|
|
962
972
|
executor: placements.executors.get(node.id) ?? UNPLACED_EXECUTOR,
|
|
963
|
-
...(download.modeOf.get(node.id) === 'deferred' ? { download: 'deferred' as const } : {}),
|
|
964
973
|
deferred: deferredOutputs,
|
|
965
|
-
...(placements.remoteOnlyNoop.has(node.id) ? { remoteOnlyNoop: true } : {}),
|
|
966
|
-
...(placements.remoteOnly.has(node.id) ? { remoteOnly: true } : {}),
|
|
967
974
|
nestedProjectDirs: nestedDirsByProject.get(node.projectName) ?? [],
|
|
968
975
|
cacheDir,
|
|
969
976
|
runStartHrTimeNs,
|
|
970
977
|
persistentRegistry,
|
|
971
978
|
liveChildren,
|
|
972
|
-
...(telemetry?.track !== undefined ? { track: telemetry.track } : {}),
|
|
973
|
-
...(explainMiss !== undefined ? { explainMiss } : {}),
|
|
974
979
|
gitFilesCache,
|
|
975
980
|
hashCache,
|
|
976
981
|
probesAfterWrites: lateProbes,
|
|
977
|
-
...(probe !== undefined ? { preProbed: probe } : {}),
|
|
978
|
-
...(upfrontKey !== undefined ? { upfrontKey } : {}),
|
|
979
|
-
...(tainted ? { taintedUpstream: true } : {}),
|
|
980
|
-
...(dependedOn.has(node.id) ? {} : { noDependants: true as const }),
|
|
981
|
-
...(holders.has(node.id) ? { terminal: true as const } : {}),
|
|
982
982
|
fingerprintWatch,
|
|
983
|
-
...(sandboxArmer !== null ? { armSandbox: () => sandboxArmer.arm() } : {}),
|
|
984
983
|
keyedProjects: keyed,
|
|
985
984
|
outputDirSnapshots,
|
|
986
985
|
deferSave: saveLane.defer,
|
|
@@ -988,6 +987,21 @@ async function runOnBus(
|
|
|
988
987
|
stopSignal: stopRun.signal,
|
|
989
988
|
failFast: failFast.signal,
|
|
990
989
|
}
|
|
990
|
+
if (options.retries !== undefined) a.retries = options.retries
|
|
991
|
+
if (taskTimeoutDefault !== undefined) a.timeout = taskTimeoutDefault
|
|
992
|
+
if (download.modeOf.get(node.id) === 'deferred') a.download = 'deferred'
|
|
993
|
+
if (placements.remoteOnlyNoop.has(node.id)) a.remoteOnlyNoop = true
|
|
994
|
+
if (placements.remoteOnly.has(node.id)) a.remoteOnly = true
|
|
995
|
+
if (track !== undefined) a.track = track
|
|
996
|
+
if (explainMiss !== undefined) a.explainMiss = explainMiss
|
|
997
|
+
if (probe !== undefined) a.preProbed = probe
|
|
998
|
+
if (upfrontKey !== undefined) a.upfrontKey = upfrontKey
|
|
999
|
+
if (upfrontGroupKey !== undefined) a.upfrontGroupKey = upfrontGroupKey
|
|
1000
|
+
if (tainted) a.taintedUpstream = true
|
|
1001
|
+
if (!dependedOn.has(node.id)) a.noDependants = true
|
|
1002
|
+
if (holders.has(node.id)) a.terminal = true
|
|
1003
|
+
if (armSandbox !== undefined) a.armSandbox = armSandbox
|
|
1004
|
+
return a
|
|
991
1005
|
}
|
|
992
1006
|
|
|
993
1007
|
const executeWithDedup = admitTasks({
|
|
@@ -16,7 +16,12 @@ import type { CacheLayer, GitFilesCache } from '../cache/index.js'
|
|
|
16
16
|
import { isLiteralPattern, normalizeGlob, relPosix } from '../util/index.js'
|
|
17
17
|
import { commandWriteReach, mayWriteFingerprint, undeclaredWriteReach } from './sandbox-request.js'
|
|
18
18
|
import { foldedDeps } from './keyed-projects.js'
|
|
19
|
-
import {
|
|
19
|
+
import {
|
|
20
|
+
computeGroupHash,
|
|
21
|
+
computeTaskHash,
|
|
22
|
+
type HashCache,
|
|
23
|
+
type UpfrontGroupKey,
|
|
24
|
+
} from './task-hash.js'
|
|
20
25
|
import { filterUpstreamHashes, keyedOutcome, keyUpstream } from './upstream.js'
|
|
21
26
|
|
|
22
27
|
export interface DeriveStableKeysArgs {
|
|
@@ -33,6 +38,11 @@ export interface DeriveStableKeysArgs {
|
|
|
33
38
|
* that key reads, so execute-task need not derive it a second time.
|
|
34
39
|
*/
|
|
35
40
|
uncachedKeys?: Map<string, string>
|
|
41
|
+
/**
|
|
42
|
+
* Filled with each unkeyed group's key and the upstream it folded:
|
|
43
|
+
* execute-task reuses the key when the live upstream matches.
|
|
44
|
+
*/
|
|
45
|
+
groupKeys?: Map<string, UpfrontGroupKey>
|
|
36
46
|
}
|
|
37
47
|
|
|
38
48
|
export interface StableKey {
|
|
@@ -217,20 +227,27 @@ export async function deriveStableKeys(args: DeriveStableKeysArgs): Promise<Stab
|
|
|
217
227
|
// dependents that filter inputs.tasks through the group still
|
|
218
228
|
// cascade (and forward their producers, above). They inherit
|
|
219
229
|
// instability from any unstable member.
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
230
|
+
// `computeGroupKey`, with the unkeyed branch kept synchronous.
|
|
231
|
+
if (node.config.cache === undefined) {
|
|
232
|
+
const key = computeGroupHash(upstream)
|
|
233
|
+
keyById.set(id, key)
|
|
234
|
+
args.groupKeys?.set(id, { key, upstream })
|
|
235
|
+
} else {
|
|
236
|
+
keyById.set(
|
|
237
|
+
id,
|
|
238
|
+
await computeTaskHash({
|
|
239
|
+
node,
|
|
240
|
+
upstream,
|
|
241
|
+
workspaceRoot: args.workspaceRoot,
|
|
242
|
+
workspaceFingerprint: args.workspaceFingerprint,
|
|
243
|
+
cache: args.cache,
|
|
244
|
+
forwardArgs: args.forwardArgs,
|
|
245
|
+
nestedProjectDirs: args.nestedDirsByProject.get(node.projectName) ?? [],
|
|
246
|
+
gitFilesCache: args.gitFilesCache,
|
|
247
|
+
hashCache: args.hashCache,
|
|
248
|
+
}),
|
|
249
|
+
)
|
|
250
|
+
}
|
|
234
251
|
if (node.deps.some((d) => unstableById.has(d))) unstableById.add(id)
|
|
235
252
|
continue
|
|
236
253
|
}
|
|
@@ -432,6 +432,16 @@ export async function computeGroupKey(args: ComputeHashArgs): Promise<string> {
|
|
|
432
432
|
return computeTaskHash(args)
|
|
433
433
|
}
|
|
434
434
|
|
|
435
|
+
/**
|
|
436
|
+
* An unkeyed group's key is `computeGroupHash` of its upstream alone, so a
|
|
437
|
+
* live upstream with the same ids and keys in the same order derives these
|
|
438
|
+
* bytes (execute-task's `sameUpstream`).
|
|
439
|
+
*/
|
|
440
|
+
export interface UpfrontGroupKey {
|
|
441
|
+
key: string
|
|
442
|
+
upstream: TaskOutcome[]
|
|
443
|
+
}
|
|
444
|
+
|
|
435
445
|
/**
|
|
436
446
|
* Derive a stable hash for a group task (no `exec`) from its upstream
|
|
437
447
|
* outcomes. Lets downstream tasks that filter `inputs.tasks` to
|
|
@@ -30,13 +30,9 @@ export function keyUpstream(node: TaskNode, upstream: TaskOutcome[]): TaskOutcom
|
|
|
30
30
|
|
|
31
31
|
/** A dependency as a key reads it: only `node` and `hash` matter. */
|
|
32
32
|
export function keyedOutcome(node: TaskNode, hash: string | undefined): TaskOutcome {
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
exitCode: 0,
|
|
37
|
-
durationMs: 0,
|
|
38
|
-
...(hash !== undefined ? { hash } : {}),
|
|
39
|
-
}
|
|
33
|
+
const o: TaskOutcome = { node, status: 'success', exitCode: 0, durationMs: 0 }
|
|
34
|
+
if (hash !== undefined) o.hash = hash
|
|
35
|
+
return o
|
|
40
36
|
}
|
|
41
37
|
|
|
42
38
|
/** `node.deps` less its ordering-only edges: the dependencies a key may fold. */
|