@vzn/vx 0.0.639 → 0.0.641
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/docs/caching.md +158 -106
- package/docs/cli.md +80 -45
- package/docs/execution.md +11 -10
- package/docs/features.md +9 -8
- package/docs/patterns.md +1 -1
- package/docs/schema.md +252 -190
- package/docs/security.md +5 -4
- package/launcher.sh +9 -2
- package/package.json +5 -5
- package/schemas/show.json +14 -14
- package/skills/vx/SKILL.md +4 -3
- package/src/bin.ts +8 -6
- package/src/cache/archive.ts +248 -41
- package/src/cache/cache.ts +46 -19
- package/src/cache/file-hashes.ts +7 -12
- package/src/cache/fs-clock.ts +62 -0
- package/src/cache/git-inputs.ts +2 -2
- package/src/cache/index.ts +6 -0
- package/src/cache/inputs.ts +228 -111
- package/src/cache/key-fold.ts +23 -3
- package/src/cache/layer.ts +15 -8
- package/src/cache/layered-cache.ts +8 -7
- package/src/cache/linked-inputs.ts +87 -0
- package/src/cache/scoped-cache.ts +8 -2
- package/src/cli/foreign-flags.ts +1 -1
- package/src/cli/help.ts +7 -1
- package/src/cli/init.ts +23 -11
- package/src/cli/last.ts +24 -1
- package/src/cli/plugin-templates.ts +2 -2
- package/src/cli/select.ts +16 -6
- package/src/cli/show.ts +9 -6
- package/src/cli/upgrade.ts +132 -21
- package/src/cli/watch-filter.ts +23 -12
- package/src/cli/watch-fs.ts +2 -1
- package/src/cli/watch-judge.ts +49 -5
- package/src/cli/watch-set.ts +18 -8
- package/src/cli/watch.ts +74 -2
- package/src/cli/why.ts +5 -3
- package/src/config.ts +19 -15
- package/src/exec/executor.ts +4 -4
- package/src/graph/dependency-spec.ts +2 -2
- package/src/graph/folds.ts +236 -0
- package/src/graph/index.ts +8 -1
- package/src/graph/scheduler.ts +41 -15
- package/src/graph/task-graph.ts +135 -73
- package/src/handoff.ts +5 -0
- package/src/index.ts +2 -2
- package/src/orchestrator/affected-tasks.ts +20 -19
- package/src/orchestrator/download-policy.ts +16 -22
- package/src/orchestrator/excluded-keys.ts +4 -5
- package/src/orchestrator/execute-task.ts +29 -25
- package/src/orchestrator/hit-restore.ts +101 -27
- package/src/orchestrator/index.ts +1 -0
- package/src/orchestrator/keyed-projects.ts +12 -51
- package/src/orchestrator/local-shortcircuit.ts +9 -1
- package/src/orchestrator/persistent-task.ts +3 -3
- package/src/orchestrator/placement.ts +2 -2
- package/src/orchestrator/plan.ts +1 -1
- package/src/orchestrator/plugin-host.ts +66 -27
- package/src/orchestrator/plugin.ts +12 -2
- package/src/orchestrator/prepare.ts +12 -5
- package/src/orchestrator/projects.ts +7 -10
- package/src/orchestrator/run-report.ts +145 -46
- package/src/orchestrator/run-steps.ts +57 -0
- package/src/orchestrator/run.ts +76 -44
- package/src/orchestrator/sandbox-fix.ts +1 -1
- package/src/orchestrator/sandbox-request.ts +2 -2
- package/src/orchestrator/stable-keys.ts +59 -26
- package/src/orchestrator/steps.ts +26 -6
- package/src/orchestrator/task-hash.ts +31 -9
- package/src/orchestrator/telemetry-host.ts +3 -3
- package/src/orchestrator/telemetry.ts +9 -4
- package/src/orchestrator/upstream.ts +4 -181
- package/src/report/framed-output.ts +1 -1
- package/src/report/logger.ts +5 -2
- package/src/report/status-line.ts +7 -3
- package/src/report/summary.ts +6 -4
- package/src/settings.ts +11 -17
- package/src/util/index.ts +3 -0
- package/src/util/paths.ts +39 -5
- package/src/workspace/affected.ts +81 -2
- package/src/workspace/config-eval.ts +2 -2
- package/src/workspace/config-imports.ts +1 -1
- package/src/workspace/config-schema.ts +142 -143
- package/src/workspace/lockfile.ts +1 -1
- package/src/workspace/migration.ts +19 -3
- package/src/workspace/project-loader.ts +5 -4
|
@@ -84,6 +84,7 @@ import {
|
|
|
84
84
|
describeTaskInputs,
|
|
85
85
|
type HashCache,
|
|
86
86
|
type InputFact,
|
|
87
|
+
isKeyless,
|
|
87
88
|
movedInput,
|
|
88
89
|
type UpfrontGroupKey,
|
|
89
90
|
} from './task-hash.js'
|
|
@@ -252,9 +253,10 @@ export interface ExecuteArgs {
|
|
|
252
253
|
/**
|
|
253
254
|
* The run keeps its bytes in a remote (run-store.ts): an attempt captures
|
|
254
255
|
* its output at the entry's bounds, and an outcome that saves no entry
|
|
255
|
-
*
|
|
256
|
+
* carries it as `runLog`; a failure's only when `failures`
|
|
257
|
+
* (`cache.upload.failures`).
|
|
256
258
|
*/
|
|
257
|
-
|
|
259
|
+
keepRun?: { failures: boolean }
|
|
258
260
|
/**
|
|
259
261
|
* Whether a task has rewritten a root file a plugin claims (whose bytes
|
|
260
262
|
* the claimant folded) since the run read it (fingerprint-watch.ts): past that, no key taken
|
|
@@ -394,7 +396,7 @@ function fingerprintMoved(args: ExecuteArgs): boolean {
|
|
|
394
396
|
* through us.
|
|
395
397
|
*/
|
|
396
398
|
async function executeGroupTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
397
|
-
if (args.node.config.
|
|
399
|
+
if (args.node.config.inputs === undefined) {
|
|
398
400
|
return unkeyedGroupOutcome(
|
|
399
401
|
args.node,
|
|
400
402
|
args.upstream,
|
|
@@ -417,7 +419,7 @@ async function executeGroupTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
417
419
|
}
|
|
418
420
|
|
|
419
421
|
/**
|
|
420
|
-
* An unkeyed group's outcome (no `
|
|
422
|
+
* An unkeyed group's outcome (no `inputs`, no key of its own): its key is
|
|
421
423
|
* its upstream's alone (`computeGroupKey`), so the up-front one stands when
|
|
422
424
|
* that upstream is unchanged. Synchronous: the scheduler settles such a
|
|
423
425
|
* group in place (`ScheduleOptions.settleNow`), and `upstream` is the key's
|
|
@@ -490,7 +492,17 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
490
492
|
// exists to prevent), no output clean, no local artifact save. Its result
|
|
491
493
|
// lives in the remote executor's own store.
|
|
492
494
|
const remoteOnly = args.remoteOnly === true
|
|
493
|
-
|
|
495
|
+
// A key that folds a task with no inputs is this run's alone (`isKeyless`):
|
|
496
|
+
// nothing to read under it, and nothing a later run could find.
|
|
497
|
+
const present = upstream.filter((u): u is TaskOutcome => u !== undefined)
|
|
498
|
+
const folded = new Set(
|
|
499
|
+
filterUpstreamHashes(present, cfg.inputs?.tasks, node.projectName, node.id).map(([id]) => id),
|
|
500
|
+
)
|
|
501
|
+
const foldedTasks = expandGroupUpstream(present.filter((u) => folded.has(u.node.id)))
|
|
502
|
+
const keyless =
|
|
503
|
+
isKeyless(cfg) || foldedTasks.some((u) => u.keyless === true || isKeyless(u.node.config))
|
|
504
|
+
const reads = policy.localRead || policy.remoteRead
|
|
505
|
+
const willRead = !remoteOnly && cfgCacheable && !keyless && reads
|
|
494
506
|
const willWrite = !remoteOnly && cfgCacheable && (policy.localWrite || policy.remoteWrite)
|
|
495
507
|
// `willWrite` governs output hygiene (clean before exec); the save itself
|
|
496
508
|
// is also withheld from a task downstream of a failure (see `taintedUpstream`),
|
|
@@ -503,17 +515,9 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
503
515
|
// (`tasks: []` and a file input) is judged by its bytes. `upstream` holds
|
|
504
516
|
// a hole where a dependency did not run (a skipped one, a restore-tier
|
|
505
517
|
// task's unfinished deps).
|
|
506
|
-
const
|
|
507
|
-
const folded = new Set(
|
|
508
|
-
filterUpstreamHashes(present, cacheCfg?.inputs?.tasks, node.projectName, node.id).map(
|
|
509
|
-
([id]) => id,
|
|
510
|
-
),
|
|
511
|
-
)
|
|
512
|
-
const unkeyedUpstream = expandGroupUpstream(present.filter((u) => folded.has(u.node.id))).find(
|
|
513
|
-
(u) => u.unkeyed === true,
|
|
514
|
-
)
|
|
518
|
+
const unkeyedUpstream = foldedTasks.find((u) => u.unkeyed === true)
|
|
515
519
|
let unkeyed = unkeyedUpstream !== undefined
|
|
516
|
-
const willSave = willWrite && args.taintedUpstream !== true && !unkeyed
|
|
520
|
+
const willSave = willWrite && args.taintedUpstream !== true && !unkeyed && !keyless
|
|
517
521
|
|
|
518
522
|
// The executor retains nothing: the entry's output is the attempt's
|
|
519
523
|
// `output` log, both streams in the order they reached the logger, kept
|
|
@@ -615,7 +619,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
615
619
|
const taskStartNs = process.hrtime.bigint() - args.runStartHrTimeNs
|
|
616
620
|
|
|
617
621
|
// Hash is computed mid-run, not at prepareRun time. Tasks whose
|
|
618
|
-
// `
|
|
622
|
+
// `inputs.files` matches sibling outputs (e.g. `'**/*'` after
|
|
619
623
|
// a `codegen` step has written `generated.txt`) need the upstream
|
|
620
624
|
// outputs ALREADY on disk when their hash is computed — so we
|
|
621
625
|
// can't lift this into prepareRun. Same model as Turbo / Nx.
|
|
@@ -812,13 +816,10 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
812
816
|
// is a hit. Almost always a glob written against the wrong directory
|
|
813
817
|
// (`lib/**` in a package that builds `src/`). Said once per miss, on the
|
|
814
818
|
// run's status line — the task itself succeeds.
|
|
815
|
-
const declaredInputs = [
|
|
816
|
-
...(cacheCfg?.inputs.files ?? []),
|
|
817
|
-
...(cacheCfg?.inputs.workspaceFiles ?? []),
|
|
818
|
-
]
|
|
819
|
+
const declaredInputs = [...(cfg.inputs?.files ?? []), ...(cfg.inputs?.workspaceFiles ?? [])]
|
|
819
820
|
if (inputs !== undefined && declaredInputs.length > 0 && inputs.files.length === 0) {
|
|
820
821
|
warnOf(log)(
|
|
821
|
-
`[vx] ${node.id}:
|
|
822
|
+
`[vx] ${node.id}: inputs matched no files (${declaredInputs.join(', ')}) — ` +
|
|
822
823
|
`the key will not change when this project's source does`,
|
|
823
824
|
)
|
|
824
825
|
}
|
|
@@ -1177,7 +1178,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1177
1178
|
|
|
1178
1179
|
async function buildRequest(local: boolean): Promise<ExecuteRequest> {
|
|
1179
1180
|
const kept =
|
|
1180
|
-
willSave || args.
|
|
1181
|
+
willSave || args.keepRun !== undefined
|
|
1181
1182
|
? new BoundedCapture()
|
|
1182
1183
|
: new BoundedCapture(FAILED_OUTPUT_HEAD_CHARS, FAILED_OUTPUT_TAIL_CHARS)
|
|
1183
1184
|
output = kept
|
|
@@ -1456,11 +1457,14 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1456
1457
|
: {}),
|
|
1457
1458
|
...(inputChanges !== undefined ? { inputChanges } : {}),
|
|
1458
1459
|
...(unkeyed ? { unkeyed: true as const } : {}),
|
|
1460
|
+
...(keyless ? { keyless: true as const } : {}),
|
|
1459
1461
|
...(saved ? { saved: true as const } : {}),
|
|
1460
1462
|
...(effectiveExitCode !== 0 && output !== undefined
|
|
1461
|
-
? { failedOutput: failedOutputLog(output, willSave || args.
|
|
1463
|
+
? { failedOutput: failedOutputLog(output, willSave || args.keepRun !== undefined) }
|
|
1462
1464
|
: {}),
|
|
1463
|
-
...(args.
|
|
1465
|
+
...(args.keepRun !== undefined &&
|
|
1466
|
+
(effectiveExitCode !== 0 ? args.keepRun.failures : !willSave) &&
|
|
1467
|
+
output !== undefined
|
|
1464
1468
|
? { runLog: storedOutput() }
|
|
1465
1469
|
: {}),
|
|
1466
1470
|
...(result.timedOut === true && effectiveExitCode !== 0 ? { timedOut: true as const } : {}),
|
|
@@ -1484,7 +1488,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1484
1488
|
finalViolations,
|
|
1485
1489
|
node.projectDir,
|
|
1486
1490
|
(t) => secrets?.mask(t) ?? t,
|
|
1487
|
-
|
|
1491
|
+
cfg.inputs?.files,
|
|
1488
1492
|
),
|
|
1489
1493
|
),
|
|
1490
1494
|
}
|
|
@@ -12,11 +12,10 @@ import type { CacheConfig } from '../config.js'
|
|
|
12
12
|
import {
|
|
13
13
|
type CacheEntry,
|
|
14
14
|
cleanOutputPaths,
|
|
15
|
-
cleanOutputs,
|
|
16
|
-
cleanWorkspaceOutputs,
|
|
17
15
|
committedFiles,
|
|
18
16
|
resolveOutputs,
|
|
19
17
|
resolveWorkspaceOutputs,
|
|
18
|
+
sweepRestoreTemps,
|
|
20
19
|
WORKSPACE_OUTPUT_PREFIX,
|
|
21
20
|
} from '../cache/index.js'
|
|
22
21
|
import type { TaskOutcome } from '../graph/index.js'
|
|
@@ -232,34 +231,74 @@ export async function restoreHit(restore: RestoreHitArgs): Promise<TaskOutcome>
|
|
|
232
231
|
}
|
|
233
232
|
}
|
|
234
233
|
if (!skipRestore) {
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
234
|
+
const heldRels = hit.outputFiles.filter((p) => !p.startsWith(WORKSPACE_OUTPUT_PREFIX))
|
|
235
|
+
const heldWsRels = hit.outputFiles
|
|
236
|
+
.filter((p) => p.startsWith(WORKSPACE_OUTPUT_PREFIX))
|
|
237
|
+
.map((p) => p.slice(WORKSPACE_OUTPUT_PREFIX.length))
|
|
238
|
+
// In place, never clean-then-restore: each file is renamed over the one
|
|
239
|
+
// it replaces, so a reader of the tree never finds a declared output
|
|
240
|
+
// missing, and a restore that fails before its renames (a full disk,
|
|
241
|
+
// a damaged entry) leaves the outputs it found as they were. Only what blocks a rename
|
|
242
|
+
// goes first; what the entry does not hold goes after. The clean used
|
|
243
|
+
// to run first: a reader polling a 64 MB output while ten hits restored
|
|
244
|
+
// met ENOENT on 153,336 of 204,411 reads (2026-10-16). An ADDITIVE task
|
|
245
|
+
// owns only its rows (item 588), and the restore rewrites exactly those.
|
|
246
|
+
let foundRels: string[] = []
|
|
247
|
+
let foundWsRels: string[] = []
|
|
248
|
+
let staleRels: string[] = []
|
|
249
|
+
let staleWsRels: string[] = []
|
|
250
|
+
if (!additive) {
|
|
251
|
+
sweepRestoreTemps(node.projectDir, outputs)
|
|
252
|
+
sweepRestoreTemps(args.workspaceRoot, wsOutputs)
|
|
253
|
+
if (outputs.length > 0) {
|
|
254
|
+
const abs = walked?.proj ?? (await resolveOutputs(cleanArgs))
|
|
255
|
+
foundRels = abs.map((f) => relPosix(node.projectDir, f))
|
|
256
|
+
const blocking = inTheWay(foundRels, heldRels)
|
|
257
|
+
staleRels = notHeld(foundRels, heldRels).filter((f) => !blocking.has(f))
|
|
258
|
+
if (blocking.size > 0) {
|
|
259
|
+
await cleanOutputPaths({
|
|
260
|
+
projectDir: node.projectDir,
|
|
261
|
+
rels: [...blocking],
|
|
262
|
+
outputs,
|
|
263
|
+
committed: cleanArgs.committed,
|
|
264
|
+
})
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
if (wsOutputs.length > 0) {
|
|
268
|
+
const abs = walked?.ws ?? (await resolveWorkspaceOutputs(wsCleanArgs))
|
|
269
|
+
foundWsRels = abs.map((f) => relPosix(args.workspaceRoot, f))
|
|
270
|
+
const blocking = inTheWay(foundWsRels, heldWsRels)
|
|
271
|
+
staleWsRels = notHeld(foundWsRels, heldWsRels).filter((f) => !blocking.has(f))
|
|
272
|
+
if (blocking.size > 0) {
|
|
273
|
+
await cleanOutputPaths({
|
|
274
|
+
projectDir: args.workspaceRoot,
|
|
275
|
+
rels: [...blocking],
|
|
276
|
+
outputs: wsOutputs,
|
|
277
|
+
committed: wsCleanArgs.committed,
|
|
278
|
+
})
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
await args.cache.restoreOutputs(hash, node.projectDir, args.workspaceRoot, {
|
|
283
|
+
files: outputs,
|
|
284
|
+
workspaceFiles: wsOutputs,
|
|
285
|
+
})
|
|
286
|
+
if (staleRels.length > 0) {
|
|
239
287
|
await cleanOutputPaths({
|
|
240
288
|
projectDir: node.projectDir,
|
|
241
|
-
rels:
|
|
289
|
+
rels: staleRels,
|
|
242
290
|
outputs,
|
|
243
291
|
committed: cleanArgs.committed,
|
|
244
292
|
})
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
.map((p) => p.slice(WORKSPACE_OUTPUT_PREFIX.length))
|
|
293
|
+
}
|
|
294
|
+
if (staleWsRels.length > 0) {
|
|
248
295
|
await cleanOutputPaths({
|
|
249
296
|
projectDir: args.workspaceRoot,
|
|
250
|
-
rels:
|
|
297
|
+
rels: staleWsRels,
|
|
251
298
|
outputs: wsOutputs,
|
|
252
299
|
committed: wsCleanArgs.committed,
|
|
253
300
|
})
|
|
254
|
-
} else {
|
|
255
|
-
if (outputs.length > 0) {
|
|
256
|
-
cleanedRels = await cleanOutputs({ ...cleanArgs, resolved: walked?.proj })
|
|
257
|
-
}
|
|
258
|
-
if (wsOutputs.length > 0) {
|
|
259
|
-
cleanedWsRels = await cleanWorkspaceOutputs({ ...wsCleanArgs, resolved: walked?.ws })
|
|
260
|
-
}
|
|
261
301
|
}
|
|
262
|
-
await args.cache.restoreOutputs(hash, node.projectDir, args.workspaceRoot)
|
|
263
302
|
// The tree now holds this entry's bytes: stamp them for the next hit
|
|
264
303
|
// (item 886).
|
|
265
304
|
args.cache.recordOutputStamps?.(hash, node.projectDir, args.workspaceRoot)
|
|
@@ -288,17 +327,12 @@ export async function restoreHit(restore: RestoreHitArgs): Promise<TaskOutcome>
|
|
|
288
327
|
// unconditional drop (an executed task may write undeclared
|
|
289
328
|
// files only git can see).
|
|
290
329
|
if (outputs.length > 0) {
|
|
291
|
-
args.gitFilesCache?.markOutputsChanged(node.projectDir, [
|
|
292
|
-
...cleanedRels,
|
|
293
|
-
...hit.outputFiles.filter((p) => !p.startsWith(WORKSPACE_OUTPUT_PREFIX)),
|
|
294
|
-
])
|
|
330
|
+
args.gitFilesCache?.markOutputsChanged(node.projectDir, [...foundRels, ...heldRels])
|
|
295
331
|
}
|
|
296
332
|
if (wsOutputs.length > 0) {
|
|
297
333
|
args.gitFilesCache?.markWorkspaceOutputsChanged(args.workspaceRoot, [
|
|
298
|
-
...
|
|
299
|
-
...
|
|
300
|
-
.filter((p) => p.startsWith(WORKSPACE_OUTPUT_PREFIX))
|
|
301
|
-
.map((p) => p.slice(WORKSPACE_OUTPUT_PREFIX.length)),
|
|
334
|
+
...foundWsRels,
|
|
335
|
+
...heldWsRels,
|
|
302
336
|
])
|
|
303
337
|
}
|
|
304
338
|
}
|
|
@@ -361,3 +395,43 @@ function dirPrefixesOf(outputs: readonly string[]): readonly string[] | null {
|
|
|
361
395
|
}
|
|
362
396
|
|
|
363
397
|
const prefixMemo = new Map<string, readonly string[] | null>()
|
|
398
|
+
|
|
399
|
+
/**
|
|
400
|
+
* On a case-insensitive volume (macOS's default) a rename onto `out.txt`
|
|
401
|
+
* replaces `Out.txt`: the two are one file, and removing the found name
|
|
402
|
+
* after the restore would remove the restored one.
|
|
403
|
+
*/
|
|
404
|
+
const fold: (rel: string) => string =
|
|
405
|
+
process.platform === 'darwin' ? (rel) => rel.normalize('NFC').toLowerCase() : (rel) => rel
|
|
406
|
+
|
|
407
|
+
/**
|
|
408
|
+
* The found outputs a rename cannot replace: a file where the entry needs a
|
|
409
|
+
* directory (`dist/out` for `dist/out/a.js`) and a file under a path the
|
|
410
|
+
* entry holds as a file (`dist/out/a.js` for `dist/out`), whose directory
|
|
411
|
+
* the prune then takes.
|
|
412
|
+
*/
|
|
413
|
+
function inTheWay(found: readonly string[], held: readonly string[]): Set<string> {
|
|
414
|
+
if (found.length === 0 || held.length === 0) return new Set()
|
|
415
|
+
const files = new Set<string>()
|
|
416
|
+
const dirs = new Set<string>()
|
|
417
|
+
for (const h of held) {
|
|
418
|
+
const key = fold(h)
|
|
419
|
+
files.add(key)
|
|
420
|
+
for (let i = key.indexOf('/'); i !== -1; i = key.indexOf('/', i + 1)) dirs.add(key.slice(0, i))
|
|
421
|
+
}
|
|
422
|
+
const out = new Set<string>()
|
|
423
|
+
for (const f of found) {
|
|
424
|
+
const key = fold(f)
|
|
425
|
+
if (dirs.has(key)) out.add(f)
|
|
426
|
+
for (let i = key.indexOf('/'); i !== -1; i = key.indexOf('/', i + 1)) {
|
|
427
|
+
if (files.has(key.slice(0, i))) out.add(f)
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
return out
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
/** The found outputs the entry does not hold: the stale ones, once it has restored. */
|
|
434
|
+
function notHeld(found: readonly string[], held: readonly string[]): string[] {
|
|
435
|
+
const keep = new Set(held.map(fold))
|
|
436
|
+
return found.filter((f) => !keep.has(fold(f)))
|
|
437
|
+
}
|
|
@@ -125,5 +125,6 @@ export {
|
|
|
125
125
|
type TaskLog,
|
|
126
126
|
taskLog,
|
|
127
127
|
} from './run-failures.js'
|
|
128
|
+
export { type RunStep, runSteps } from './run-steps.js'
|
|
128
129
|
export { resolveRunId, shortRunId } from './run-id.js'
|
|
129
130
|
export { collectInfo, type CollectInfoOptions, type InfoFacts } from './doctor.js'
|
|
@@ -5,8 +5,7 @@
|
|
|
5
5
|
// is decided — on the graph, before any hash exists, through the same
|
|
6
6
|
// selection the hash path applies (`selectFoldedDeps`).
|
|
7
7
|
|
|
8
|
-
import { isGroupTask, type TaskNode } from '../graph/index.js'
|
|
9
|
-
import { keyedDeps, selectFoldedDeps, type FoldCandidate } from './upstream.js'
|
|
8
|
+
import { foldedDeps, isGroupTask, type FoldCandidate, type TaskNode } from '../graph/index.js'
|
|
10
9
|
|
|
11
10
|
/**
|
|
12
11
|
* A run's keyed-set lookup: for a task, the directories of every project
|
|
@@ -14,18 +13,17 @@ import { keyedDeps, selectFoldedDeps, type FoldCandidate } from './upstream.js'
|
|
|
14
13
|
* task itself is not counted.
|
|
15
14
|
*
|
|
16
15
|
* The fold relation is the hash path's, computed on the graph:
|
|
17
|
-
* - an exec task folds its dependencies as its own `
|
|
18
|
-
* selects them (absent
|
|
19
|
-
* - a group folds every dependency (`computeGroupKey`)
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
* - a persistent task is keyed as
|
|
16
|
+
* - an exec task folds its dependencies as its own `inputs.tasks`
|
|
17
|
+
* selects them (absent means all);
|
|
18
|
+
* - a group folds every dependency (`computeGroupKey`); a keyed one
|
|
19
|
+
* (`inputs`, the default `build`) folds its project's files as a task
|
|
20
|
+
* does;
|
|
21
|
+
* - a persistent task is keyed as any task with no `cache` (A-17).
|
|
23
22
|
*
|
|
24
|
-
* Every exec task reached contributes its project, cached or not:
|
|
25
|
-
* folds
|
|
26
|
-
*
|
|
27
|
-
* `
|
|
28
|
-
* key itself.
|
|
23
|
+
* Every exec task reached contributes its project, cached or not: its key
|
|
24
|
+
* folds its own project's files through its `inputs`, and one with no
|
|
25
|
+
* `inputs` is keyless, so a key that folds it is never stored or read.
|
|
26
|
+
* `tests/keyed-projects.test.ts` holds that to the key itself.
|
|
29
27
|
*
|
|
30
28
|
* Lazy and memoized by task id: only a sandboxed task that executes asks,
|
|
31
29
|
* so a run of hits walks nothing, and a shared subgraph is walked once.
|
|
@@ -57,7 +55,7 @@ export function keyedProjects(
|
|
|
57
55
|
stack.pop()
|
|
58
56
|
const out = new Set<string>()
|
|
59
57
|
for (const { node: dep } of deps) {
|
|
60
|
-
if (!isGroupTask(dep) || dep.config.
|
|
58
|
+
if (!isGroupTask(dep) || dep.config.inputs !== undefined) out.add(dep.projectDir)
|
|
61
59
|
for (const dir of below.get(dep.id)!) out.add(dir)
|
|
62
60
|
}
|
|
63
61
|
below.set(node.id, out)
|
|
@@ -65,40 +63,3 @@ export function keyedProjects(
|
|
|
65
63
|
return below.get(root.id)!
|
|
66
64
|
}
|
|
67
65
|
}
|
|
68
|
-
|
|
69
|
-
/**
|
|
70
|
-
* The dependencies `node`'s key folds, per the hash path's rules above, as
|
|
71
|
-
* `keyUpstream` hands them to the key: the scheduled ones less the edges
|
|
72
|
-
* `--exclude-dependencies` left for order alone, plus the dropped ones it
|
|
73
|
-
* keys all the same (`excludedUpstream`), whose own dependencies the walk
|
|
74
|
-
* reads from the run's `keyOnly` map. `node.deps` held the order-only
|
|
75
|
-
* edges and lacked the dropped ones: the sandbox granted a sibling the key
|
|
76
|
-
* does not answer for.
|
|
77
|
-
*/
|
|
78
|
-
export function foldedDeps(node: TaskNode, nodeOf: (id: string) => TaskNode): FoldCandidate[] {
|
|
79
|
-
const candidates: FoldCandidate[] = []
|
|
80
|
-
for (const id of keyedDeps(node)) {
|
|
81
|
-
const dep = nodeOf(id)
|
|
82
|
-
candidates.push({ node: dep, unit: foldUnit(dep) })
|
|
83
|
-
}
|
|
84
|
-
for (const { node: dep } of node.excludedUpstream ?? []) {
|
|
85
|
-
candidates.push({ node: dep, unit: foldUnit(dep) })
|
|
86
|
-
}
|
|
87
|
-
if (isGroupTask(node) && node.config.cache === undefined) return candidates
|
|
88
|
-
return selectFoldedDeps(candidates, node.config.cache?.inputs?.tasks, node.projectName, node.id)
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
/**
|
|
92
|
-
* What stands in for a dependency's hash in the dedup: equal exactly when
|
|
93
|
-
* the hashes are. An exec task's key folds its own id (`task:<id>`), so its
|
|
94
|
-
* id is enough; a group's hash is a roll-up of its members' ids and hashes
|
|
95
|
-
* with no id of its own, so two groups over the same members hash alike.
|
|
96
|
-
* The members are `keyUpstream`'s, which `computeGroupHash` is handed: under
|
|
97
|
-
* `--exclude-dependencies` `node.deps` holds order-only edges and lacks the
|
|
98
|
-
* dropped ones, and two groups the key told apart shared a unit.
|
|
99
|
-
*/
|
|
100
|
-
function foldUnit(node: TaskNode): string {
|
|
101
|
-
if (!isGroupTask(node) || node.config.cache !== undefined) return `task|${node.id}`
|
|
102
|
-
const members = [...keyedDeps(node), ...(node.excludedUpstream ?? []).map((u) => u.node.id)]
|
|
103
|
-
return `group|${members.sort().join('|')}`
|
|
104
|
-
}
|
|
@@ -60,6 +60,12 @@ export interface ShortCircuitArgs {
|
|
|
60
60
|
hashCache: HashCache
|
|
61
61
|
/** Cap on concurrent probes — the run's concurrency. */
|
|
62
62
|
concurrency: number
|
|
63
|
+
/**
|
|
64
|
+
* `cache` is the local store in front of a remote: a hit is certain, a
|
|
65
|
+
* miss is not (the remote may hold it), so only hits are recorded and
|
|
66
|
+
* execute probes every other task through the layers.
|
|
67
|
+
*/
|
|
68
|
+
localOnly?: boolean | undefined
|
|
63
69
|
}
|
|
64
70
|
|
|
65
71
|
/** Per-task result of the up-front probe. `hit` null = a confirmed
|
|
@@ -159,6 +165,7 @@ export async function startLocalShortCircuit(args: ShortCircuitArgs): Promise<Sh
|
|
|
159
165
|
endProbe()
|
|
160
166
|
for (const { hash, node } of candidates) {
|
|
161
167
|
const hit = hits.get(hash) ?? null
|
|
168
|
+
if (hit === null && args.localOnly === true) continue
|
|
162
169
|
preProbed.set(node.id, { hash, hit })
|
|
163
170
|
if (hit !== null && !keptOut.has(node.id)) restoreTier.add(node.id)
|
|
164
171
|
}
|
|
@@ -175,6 +182,7 @@ export async function startLocalShortCircuit(args: ShortCircuitArgs): Promise<Sh
|
|
|
175
182
|
const { hash, node } = candidates[next++]!
|
|
176
183
|
try {
|
|
177
184
|
const hit = await args.cache.get(hash, getContext(node, node.config.exec?.command ?? ''))
|
|
185
|
+
if (hit === null && args.localOnly === true) continue
|
|
178
186
|
preProbed.set(node.id, { hash, hit })
|
|
179
187
|
if (hit !== null && !keptOut.has(node.id)) restoreTier.add(node.id)
|
|
180
188
|
} catch {
|
|
@@ -232,7 +240,7 @@ export function restoreTierExclusions(
|
|
|
232
240
|
const dependants = new Map<string, string[]>()
|
|
233
241
|
for (const node of nodes.values()) {
|
|
234
242
|
const dir = relPosix(workspaceRoot, node.projectDir)
|
|
235
|
-
const wsInputs = node.config.
|
|
243
|
+
const wsInputs = node.config.inputs?.workspaceFiles ?? []
|
|
236
244
|
if (reaches(dir) || (wsInputs.length > 0 && workspaceInputsReach(wsInputs, prefixes))) {
|
|
237
245
|
out.add(node.id)
|
|
238
246
|
}
|
|
@@ -55,9 +55,9 @@ export async function executePersistentTask(args: ExecuteArgs): Promise<TaskOutc
|
|
|
55
55
|
const effectiveForwardArgs = node.requested ? (args.forwardArgs ?? []) : []
|
|
56
56
|
const env = taskEnv(node, step, args.workspaceRoot)
|
|
57
57
|
const wallclockStartNs = process.hrtime.bigint() - args.runStartHrTimeNs
|
|
58
|
-
// Keyed
|
|
59
|
-
//
|
|
60
|
-
// key the dependant
|
|
58
|
+
// Keyed on its inputs like any task, so a cached dependant behind a dev
|
|
59
|
+
// server re-runs when the server's sources change; with none it has no
|
|
60
|
+
// key and the dependant always runs (A-17).
|
|
61
61
|
const hash =
|
|
62
62
|
args.noDependants === true
|
|
63
63
|
? undefined
|
|
@@ -59,7 +59,7 @@ export function pinnedLocalSet(nodes: Map<string, TaskNode>): Set<string> {
|
|
|
59
59
|
}
|
|
60
60
|
|
|
61
61
|
/**
|
|
62
|
-
* A key that folds `
|
|
62
|
+
* A key that folds `inputs.runtime` / `workspaceRuntime` holds what
|
|
63
63
|
* THIS machine answered (`node -v`); a worker runs its own runtime, which no
|
|
64
64
|
* executor can prove equal, and its output saved under this key is a stale
|
|
65
65
|
* hit here (C-2). Such a task runs here. Its dependants are not pinned with
|
|
@@ -68,7 +68,7 @@ export function pinnedLocalSet(nodes: Map<string, TaskNode>): Set<string> {
|
|
|
68
68
|
*/
|
|
69
69
|
function withProbedRuntime(nodes: Map<string, TaskNode>, pinned: Set<string>): Set<string> {
|
|
70
70
|
for (const node of nodes.values()) {
|
|
71
|
-
const inputs = node.config.
|
|
71
|
+
const inputs = node.config.inputs
|
|
72
72
|
if ((inputs?.runtime?.length ?? 0) > 0 || (inputs?.workspaceRuntime?.length ?? 0) > 0) {
|
|
73
73
|
pinned.add(node.id)
|
|
74
74
|
}
|
package/src/orchestrator/plan.ts
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
// restoreOutputs. Cache probing uses the byte-free `cache.has()`
|
|
7
7
|
// existence probe (a remote layer answers it its own way — no
|
|
8
8
|
// artifact download, no local ingest), so the plan is read-only — with
|
|
9
|
-
// one deliberate exception: `
|
|
9
|
+
// one deliberate exception: `inputs.runtime` / `workspaceRuntime`
|
|
10
10
|
// probe commands DO run, because predicting a task's key requires
|
|
11
11
|
// resolving them (same as a real run). Keep runtime inputs
|
|
12
12
|
// side-effect-free pure probes.
|
|
@@ -381,41 +381,81 @@ export async function claimedAffected(
|
|
|
381
381
|
return names
|
|
382
382
|
}
|
|
383
383
|
|
|
384
|
+
/** What a run hands the `schedule` stage: a step per plugin, and where refined weights go. */
|
|
385
|
+
export interface ScheduleRun {
|
|
386
|
+
readonly step?: <T>(id: string, work: () => Promise<T>) => Promise<T>
|
|
387
|
+
readonly refined?: (merged: ReadonlyMap<string, number>) => void
|
|
388
|
+
}
|
|
389
|
+
|
|
384
390
|
/**
|
|
385
391
|
* `schedule` stage: the merged priorities map (a later plugin's weight for
|
|
386
|
-
* a task overrides an earlier one's). Empty when no plugin answered.
|
|
392
|
+
* a task overrides an earlier one's). Empty when no plugin answered. Each
|
|
393
|
+
* plugin's call is the step `<plugin>:schedule`. A plugin's `ctx.refine`
|
|
394
|
+
* replaces its own weights and hands `refined` the merge again; with no
|
|
395
|
+
* `refined` (a plan), it is ignored.
|
|
387
396
|
*/
|
|
388
397
|
export async function applyScheduleHooks(
|
|
389
398
|
plugins: readonly VxPlugin[],
|
|
390
399
|
nodes: ReadonlyMap<string, TaskNode>,
|
|
391
|
-
ctx: Omit<ScheduleHookContext, 'cache'>,
|
|
400
|
+
ctx: Omit<ScheduleHookContext, 'cache' | 'refine'>,
|
|
392
401
|
cacheFor: (plugin: string) => PluginCache = () => absentPluginCache,
|
|
402
|
+
run: ScheduleRun = {},
|
|
393
403
|
): Promise<ReadonlyMap<string, number>> {
|
|
394
|
-
const
|
|
395
|
-
|
|
404
|
+
const sets: (ReadonlyMap<string, number> | undefined)[] = []
|
|
405
|
+
const merged = (): ReadonlyMap<string, number> => {
|
|
406
|
+
const out = new Map<string, number>()
|
|
407
|
+
for (const set of sets) if (set !== undefined) for (const [id, w] of set) out.set(id, w)
|
|
408
|
+
return out
|
|
409
|
+
}
|
|
410
|
+
for (const [i, plugin] of plugins.entries()) {
|
|
396
411
|
if (plugin.schedule === undefined) continue
|
|
397
|
-
const
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
412
|
+
const refine = (weights: ReadonlyMap<string, number>): void => {
|
|
413
|
+
if (run.refined === undefined) return
|
|
414
|
+
try {
|
|
415
|
+
sets[i] = scheduleWeights(plugin, weights, nodes)
|
|
416
|
+
} catch (err) {
|
|
417
|
+
ctx.warn(`[vx] ${err instanceof Error ? err.message : String(err)}; refine ignored`)
|
|
418
|
+
return
|
|
419
|
+
}
|
|
420
|
+
run.refined(merged())
|
|
421
|
+
}
|
|
422
|
+
const own = { ...ctx, cache: cacheFor(plugin.name), refine }
|
|
423
|
+
const call = (): Promise<ReadonlyMap<string, number> | undefined> =>
|
|
424
|
+
safe(plugin, 'schedule', () => plugin.schedule!(nodes, own))
|
|
425
|
+
const weights = await (run.step === undefined
|
|
426
|
+
? call()
|
|
427
|
+
: run.step(`${plugin.name}:schedule`, call))
|
|
428
|
+
// A refine that landed while the call ran is the newer set.
|
|
429
|
+
if (weights !== undefined) sets[i] ??= scheduleWeights(plugin, weights, nodes)
|
|
430
|
+
}
|
|
431
|
+
return merged()
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
/** A `schedule` answer's weights for tasks in the graph, refused by name when it is not a Map of finite numbers. */
|
|
435
|
+
function scheduleWeights(
|
|
436
|
+
plugin: VxPlugin,
|
|
437
|
+
weights: unknown,
|
|
438
|
+
nodes: ReadonlyMap<string, TaskNode>,
|
|
439
|
+
): ReadonlyMap<string, number> {
|
|
440
|
+
// Iterating a string destructures its characters into `[id, w]` pairs
|
|
441
|
+
// that match no task — a plugin returning `'fast'` used to be a no-op
|
|
442
|
+
// with no word said.
|
|
443
|
+
if (!(weights instanceof Map)) {
|
|
444
|
+
throw new UserError(
|
|
445
|
+
`plugin '${plugin.name}' failed in schedule: returned ${describeValue(weights)}, not a Map of task id → weight`,
|
|
446
|
+
)
|
|
447
|
+
}
|
|
448
|
+
const out = new Map<string, number>()
|
|
449
|
+
for (const [id, w] of weights as Map<unknown, unknown>) {
|
|
450
|
+
if (typeof id !== 'string' || !nodes.has(id)) continue
|
|
451
|
+
if (typeof w !== 'number' || !Number.isFinite(w)) {
|
|
404
452
|
throw new UserError(
|
|
405
|
-
`plugin '${plugin.name}' failed in schedule:
|
|
453
|
+
`plugin '${plugin.name}' failed in schedule: weight for ${id} is not a finite number`,
|
|
406
454
|
)
|
|
407
455
|
}
|
|
408
|
-
|
|
409
|
-
if (!nodes.has(id)) continue
|
|
410
|
-
if (typeof w !== 'number' || !Number.isFinite(w)) {
|
|
411
|
-
throw new UserError(
|
|
412
|
-
`plugin '${plugin.name}' failed in schedule: weight for ${id} is not a finite number`,
|
|
413
|
-
)
|
|
414
|
-
}
|
|
415
|
-
merged.set(id, w)
|
|
416
|
-
}
|
|
456
|
+
out.set(id, w)
|
|
417
457
|
}
|
|
418
|
-
return
|
|
458
|
+
return out
|
|
419
459
|
}
|
|
420
460
|
|
|
421
461
|
/**
|
|
@@ -491,8 +531,8 @@ export interface CacheHost extends CacheContext {
|
|
|
491
531
|
* Collect every plugin's remote (`cache` hook) in declaration order and
|
|
492
532
|
* layer each over the local store: a lookup asks the local store first,
|
|
493
533
|
* then each remote in order (owner, 2026-10-13), and a save reaches every
|
|
494
|
-
* remote. The run's policy
|
|
495
|
-
*
|
|
534
|
+
* remote. The run's policy alone gates each remote (M, 2026-10-16: a
|
|
535
|
+
* plugin moves bytes, it never decides reads or writes); a remote's throw is a miss and a warning naming its plugin. No
|
|
496
536
|
* remote → the local store alone.
|
|
497
537
|
*/
|
|
498
538
|
export async function resolveCache(
|
|
@@ -506,11 +546,10 @@ export async function resolveCache(
|
|
|
506
546
|
const remote = await safe(plugin, 'cache', () => plugin.cache!(ctx))
|
|
507
547
|
if (remote === undefined) continue
|
|
508
548
|
assertShape(plugin, 'cache', remote, REMOTE_CACHE_METHODS, 'a remote cache layer')
|
|
509
|
-
const access = remote.access
|
|
510
549
|
const entry: ResolvedRemote = {
|
|
511
550
|
remote,
|
|
512
|
-
read: policy.remoteRead
|
|
513
|
-
write: policy.remoteWrite
|
|
551
|
+
read: policy.remoteRead,
|
|
552
|
+
write: policy.remoteWrite,
|
|
514
553
|
onError: (err) => ctx.warn(`[vx] plugin '${plugin.name}': ${err.message}`),
|
|
515
554
|
}
|
|
516
555
|
resolved.push(entry)
|
|
@@ -137,6 +137,9 @@ export interface VxPlugin {
|
|
|
137
137
|
* only when a task will start: a run of confirmed local hits restores
|
|
138
138
|
* in no order a weight changes. A later plugin's weight for a task
|
|
139
139
|
* overrides an earlier one's. Return undefined to leave the baseline.
|
|
140
|
+
* The run waits for this answer before its first task, so it should
|
|
141
|
+
* come from what is at hand (local history); better weights that need
|
|
142
|
+
* a fetch go to `ctx.refine` when they arrive, without holding the run.
|
|
140
143
|
*/
|
|
141
144
|
schedule?(
|
|
142
145
|
nodes: ReadonlyMap<string, TaskNode>,
|
|
@@ -244,8 +247,9 @@ export interface VxPlugin {
|
|
|
244
247
|
/**
|
|
245
248
|
* What a finished run hands each plugin's `teardown`: its facts and every
|
|
246
249
|
* task, with the names core kept the task's bytes under. Core keeps them
|
|
247
|
-
* only when a remote takes the run's writes (CI by default): a
|
|
248
|
-
* task
|
|
250
|
+
* only when a remote takes the run's writes (CI by default): the log of a
|
|
251
|
+
* task that saved no entry, and a failed task's log and the files its
|
|
252
|
+
* outputs left when `cache.upload.failures` covers the run. Read
|
|
249
253
|
* them with `ctx.cache.getRun(name)`. Core keeps no manifest of its own; a
|
|
250
254
|
* plugin that wants one writes it from here (`ctx.cache.put`).
|
|
251
255
|
*/
|
|
@@ -448,6 +452,12 @@ export interface ScheduleHookContext extends BaseContext {
|
|
|
448
452
|
readonly branch: string | null
|
|
449
453
|
/** This plugin's entries in the workspace's cache, as `setup` has them. */
|
|
450
454
|
readonly cache: PluginCache
|
|
455
|
+
/**
|
|
456
|
+
* Replace this plugin's weights while the run goes on: the tasks not yet
|
|
457
|
+
* started re-rank at once. For weights that arrive late (timings fetched
|
|
458
|
+
* from a remote). Ignored in a plan and once the run's tasks are done.
|
|
459
|
+
*/
|
|
460
|
+
refine(weights: ReadonlyMap<string, number>): void
|
|
451
461
|
}
|
|
452
462
|
|
|
453
463
|
/** A `cache` hook's context: the workspace and the run's scope, never the local store or the policy, which core owns. */
|