@vzn/vx 0.0.633 → 0.0.635
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -52
- package/docs/caching.md +11 -2
- package/docs/cli.md +192 -56
- package/docs/execution.md +1 -1
- package/docs/features.md +6 -3
- package/docs/patterns.md +8 -5
- package/docs/schema.md +48 -2
- package/docs/security.md +6 -0
- package/launcher.cjs +0 -0
- package/launcher.sh +50 -0
- package/package.json +17 -8
- package/schemas/error.json +5 -1
- package/schemas/last.json +5 -1
- package/schemas/plan.json +4 -0
- package/schemas/show.json +2 -1
- package/schemas/why.json +4 -1
- package/skills/vx/SKILL.md +7 -1
- package/src/cache/cache.ts +60 -5
- package/src/cache/inputs.ts +66 -30
- package/src/cache/layered-cache.ts +3 -1
- package/src/cli/docs.ts +9 -3
- package/src/cli/help.ts +2 -1
- package/src/cli/init.ts +9 -2
- package/src/cli/last.ts +9 -1
- package/src/cli/plan-format.ts +3 -1
- package/src/cli/plugin-templates.ts +1 -1
- package/src/cli/run.ts +17 -8
- package/src/cli/select.ts +17 -11
- package/src/cli/show.ts +2 -2
- package/src/config.ts +23 -2
- package/src/graph/priorities.ts +72 -6
- package/src/graph/scheduler.ts +31 -7
- package/src/graph/task-graph.ts +75 -20
- package/src/index.ts +6 -1
- package/src/orchestrator/execute-task.ts +30 -1
- package/src/orchestrator/framed-output.ts +27 -1
- package/src/orchestrator/history.ts +75 -1
- package/src/orchestrator/index.ts +1 -0
- package/src/orchestrator/path-links.ts +5 -3
- package/src/orchestrator/plugin.ts +3 -1
- package/src/orchestrator/prepare.ts +15 -12
- package/src/orchestrator/remote-prefetch.ts +8 -2
- package/src/orchestrator/run-failures.ts +20 -3
- package/src/orchestrator/run.ts +24 -2
- package/src/orchestrator/sandbox-fix.ts +123 -0
- package/src/orchestrator/task-hash.ts +7 -1
- package/src/util/errors.ts +9 -2
- package/src/util/paths.ts +32 -0
- package/src/workspace/affected.ts +16 -4
- package/src/workspace/config-schema.ts +30 -2
- package/src/workspace/lockfile.ts +29 -2
- package/src/workspace/migration.ts +51 -4
- package/src/workspace/project-loader.ts +3 -1
package/src/graph/task-graph.ts
CHANGED
|
@@ -302,6 +302,7 @@ export interface BuildGraphOptions {
|
|
|
302
302
|
export function undeclaredDepsError(taskId: string, name: string): UserError {
|
|
303
303
|
return new UserError(
|
|
304
304
|
`Task ${taskId} depends on ^${name} but no project in the workspace declares ${name}`,
|
|
305
|
+
'VX_E_DEPENDS_ON',
|
|
305
306
|
)
|
|
306
307
|
}
|
|
307
308
|
|
|
@@ -492,7 +493,7 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
|
|
|
492
493
|
spec = parseDependencySpec(raw)
|
|
493
494
|
} catch (err) {
|
|
494
495
|
if (err instanceof DependencySpecError) {
|
|
495
|
-
throw new UserError(`Task ${id}: ${err.message}
|
|
496
|
+
throw new UserError(`Task ${id}: ${err.message}`, 'VX_E_DEPENDS_ON')
|
|
496
497
|
}
|
|
497
498
|
throw err
|
|
498
499
|
}
|
|
@@ -503,14 +504,21 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
|
|
|
503
504
|
// operations. PARTIAL patterns (`build.*`, `^build.*`) are legal:
|
|
504
505
|
// they name a namespace of tasks to add (Nx 19.5 parity).
|
|
505
506
|
if (spec.kind === 'wildcardSelf' || spec.kind === 'wildcardDeps') {
|
|
506
|
-
throw new UserError(
|
|
507
|
+
throw new UserError(
|
|
508
|
+
`Task ${id}: dependsOn does not accept bare wildcards (got "${raw}")`,
|
|
509
|
+
'VX_E_DEPENDS_ON',
|
|
510
|
+
)
|
|
507
511
|
}
|
|
508
512
|
if (spec.negated) {
|
|
509
|
-
throw new UserError(
|
|
513
|
+
throw new UserError(
|
|
514
|
+
`Task ${id}: dependsOn does not accept negation (got "${raw}")`,
|
|
515
|
+
'VX_E_DEPENDS_ON',
|
|
516
|
+
)
|
|
510
517
|
}
|
|
511
518
|
if (spec.kind === 'cross' && (isTaskPattern(spec.task) || isTaskPattern(spec.project))) {
|
|
512
519
|
throw new UserError(
|
|
513
520
|
`Task ${id}: dependsOn patterns are not supported in the "pkg#task" form (got "${raw}")`,
|
|
521
|
+
'VX_E_DEPENDS_ON',
|
|
514
522
|
)
|
|
515
523
|
}
|
|
516
524
|
if (spec.kind === 'self') {
|
|
@@ -530,6 +538,7 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
|
|
|
530
538
|
if (!visit(frame, projectName, spec.task, false)) {
|
|
531
539
|
throw new UserError(
|
|
532
540
|
`Task ${id} depends on ${taskId(projectName, spec.task)} but no such task is declared`,
|
|
541
|
+
'VX_E_DEPENDS_ON',
|
|
533
542
|
)
|
|
534
543
|
}
|
|
535
544
|
goPast(frame, projectName, spec.task, new Set([projectName]))
|
|
@@ -594,9 +603,11 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
|
|
|
594
603
|
if (spec.project === '//')
|
|
595
604
|
throw new UserError(
|
|
596
605
|
`Task ${id} depends on //#${spec.task}, Turbo's root package, but the workspace root is no project here: give it a package.json name and a vx.config with that task`,
|
|
606
|
+
'VX_E_DEPENDS_ON',
|
|
597
607
|
)
|
|
598
608
|
throw new UserError(
|
|
599
609
|
`Task ${id} depends on ${taskId(spec.project, spec.task)} but no such project or task is declared`,
|
|
610
|
+
'VX_E_DEPENDS_ON',
|
|
600
611
|
)
|
|
601
612
|
}
|
|
602
613
|
goPast(frame, spec.project, spec.task, new Set([projectName, spec.project]))
|
|
@@ -1109,7 +1120,7 @@ function collideAcross(
|
|
|
1109
1120
|
`cache.outputs.files — vx cleans a task's declared outputs before it runs and before a ` +
|
|
1110
1121
|
`cache-hit restore, so whichever of these runs second DELETES the other's output`
|
|
1111
1122
|
if (exclusive && (reaches(files.id, ws.id) || reaches(ws.id, files.id))) {
|
|
1112
|
-
throw new UserError(`${head}. ${EXCLUSIVE_OUTPUTS_FIX}
|
|
1123
|
+
throw new UserError(`${head}. ${EXCLUSIVE_OUTPUTS_FIX}`, 'VX_E_OUTPUT_OVERLAP')
|
|
1113
1124
|
}
|
|
1114
1125
|
if (reaches(files.id, ws.id)) {
|
|
1115
1126
|
;(files.addsToOutputsOf ??= []).push(ws.id)
|
|
@@ -1132,11 +1143,13 @@ function collideAcross(
|
|
|
1132
1143
|
`${head}. ${ws.id} depends on ${files.id}, but adding to its output needs every ` +
|
|
1133
1144
|
`cache.outputs.workspaceFiles entry inside ${rel === '' ? 'the workspace root' : rel}, ` +
|
|
1134
1145
|
`and ${JSON.stringify(outside)} is not. Give each task its own output path.`,
|
|
1146
|
+
'VX_E_OUTPUT_OVERLAP',
|
|
1135
1147
|
)
|
|
1136
1148
|
}
|
|
1137
1149
|
throw new UserError(
|
|
1138
1150
|
`${head} and the run still reports success. Give each task its own output path` +
|
|
1139
1151
|
(exclusive ? '.' : ', or make one depend on the other.'),
|
|
1152
|
+
'VX_E_OUTPUT_OVERLAP',
|
|
1140
1153
|
)
|
|
1141
1154
|
}
|
|
1142
1155
|
}
|
|
@@ -1373,7 +1386,8 @@ function collide(
|
|
|
1373
1386
|
// `rules.exclusiveOutputs` is off (X-53). Without an edge the two run
|
|
1374
1387
|
// in either order, and the refusal below stands.
|
|
1375
1388
|
const [up, down] = reaches(b.id, a.id) ? [a, b] : reaches(a.id, b.id) ? [b, a] : []
|
|
1376
|
-
if (up !== undefined && exclusive)
|
|
1389
|
+
if (up !== undefined && exclusive)
|
|
1390
|
+
throw new UserError(`${head}. ${EXCLUSIVE_OUTPUTS_FIX}`, 'VX_E_OUTPUT_OVERLAP')
|
|
1377
1391
|
if (up !== undefined && down !== undefined) {
|
|
1378
1392
|
const downGlobs = splitNegations(
|
|
1379
1393
|
(field === 'files'
|
|
@@ -1386,6 +1400,7 @@ function collide(
|
|
|
1386
1400
|
}
|
|
1387
1401
|
throw new UserError(
|
|
1388
1402
|
`${head} and the run still reports success. Give each task its own output path.`,
|
|
1403
|
+
'VX_E_OUTPUT_OVERLAP',
|
|
1389
1404
|
)
|
|
1390
1405
|
}
|
|
1391
1406
|
}
|
|
@@ -1440,6 +1455,7 @@ function refuseSelfClean(node: TaskNode): void {
|
|
|
1440
1455
|
`its own output — vx removes a task's outputs before it runs, so the task would ` +
|
|
1441
1456
|
`delete its own sources. A task that rewrites files in place (a formatter) declares ` +
|
|
1442
1457
|
`no outputs.`,
|
|
1458
|
+
'VX_E_INPUT_IS_OUTPUT',
|
|
1443
1459
|
)
|
|
1444
1460
|
}
|
|
1445
1461
|
}
|
|
@@ -1517,6 +1533,7 @@ function detectInputOverlaps(nodes: Map<string, TaskNode>, workspaceRoot?: strin
|
|
|
1517
1533
|
const filesWriters: TaskNode[] = []
|
|
1518
1534
|
let projectReaders = false
|
|
1519
1535
|
let rootReaders = false
|
|
1536
|
+
const memo: TakeBackMemo = { bySide: new Map(), byList: new Map() }
|
|
1520
1537
|
const bucket = (name: string): Side[] => {
|
|
1521
1538
|
let list = byProject.get(name)
|
|
1522
1539
|
if (list === undefined) byProject.set(name, (list = []))
|
|
@@ -1557,7 +1574,7 @@ function detectInputOverlaps(nodes: Map<string, TaskNode>, workspaceRoot?: strin
|
|
|
1557
1574
|
(s) => s.globs,
|
|
1558
1575
|
(s) => s.reads,
|
|
1559
1576
|
)) {
|
|
1560
|
-
readsOutputs(sides[i]!, sides[j]!, 'files')
|
|
1577
|
+
readsOutputs(memo, sides[i]!, sides[j]!, 'files')
|
|
1561
1578
|
}
|
|
1562
1579
|
}
|
|
1563
1580
|
}
|
|
@@ -1575,7 +1592,7 @@ function detectInputOverlaps(nodes: Map<string, TaskNode>, workspaceRoot?: strin
|
|
|
1575
1592
|
(s) => s.globs,
|
|
1576
1593
|
(s) => s.reads,
|
|
1577
1594
|
)) {
|
|
1578
|
-
readsOutputs(sides[i]!, sides[j]!, 'files')
|
|
1595
|
+
readsOutputs(memo, sides[i]!, sides[j]!, 'files')
|
|
1579
1596
|
}
|
|
1580
1597
|
}
|
|
1581
1598
|
if (!rootReaders) return
|
|
@@ -1593,26 +1610,63 @@ function detectInputOverlaps(nodes: Map<string, TaskNode>, workspaceRoot?: strin
|
|
|
1593
1610
|
(s) => s.globs,
|
|
1594
1611
|
(s) => s.reads,
|
|
1595
1612
|
)) {
|
|
1596
|
-
readsOutputs(rooted[i]!, rooted[j]!, 'workspaceFiles')
|
|
1613
|
+
readsOutputs(memo, rooted[i]!, rooted[j]!, 'workspaceFiles')
|
|
1597
1614
|
}
|
|
1598
1615
|
}
|
|
1599
1616
|
|
|
1600
|
-
|
|
1617
|
+
/**
|
|
1618
|
+
* Whether a reader's take-back list covers an output glob, asked once per
|
|
1619
|
+
* list and glob: a Turbo-mapped repo gives two thousand readers one
|
|
1620
|
+
* 400-entry list, and asking it per reader and writer pair was 2.7 s of an
|
|
1621
|
+
* astro plan (2026-10-09). One per `detectInputOverlaps` call.
|
|
1622
|
+
*/
|
|
1623
|
+
interface TakeBack {
|
|
1624
|
+
list: string[]
|
|
1625
|
+
verdicts: Map<string, boolean>
|
|
1626
|
+
}
|
|
1627
|
+
type TakeBackMemo = { bySide: Map<Side, TakeBack>; byList: Map<string, TakeBack> }
|
|
1628
|
+
|
|
1629
|
+
function takenBack(
|
|
1630
|
+
memo: TakeBackMemo,
|
|
1631
|
+
reader: Side,
|
|
1632
|
+
field: 'files' | 'workspaceFiles',
|
|
1633
|
+
go: string,
|
|
1634
|
+
): boolean {
|
|
1635
|
+
let tb = memo.bySide.get(reader)
|
|
1636
|
+
if (tb === undefined) {
|
|
1637
|
+
const cache = reader.node.config.cache!
|
|
1638
|
+
// A reader rebased to the root (X-135) speaks in its project's terms.
|
|
1639
|
+
const at = reader.rel === undefined || reader.rel === '' ? '' : `${reader.rel}/`
|
|
1640
|
+
// What the reader's key never reads: its `!` entries and its own outputs.
|
|
1641
|
+
const list = [
|
|
1642
|
+
...splitNegations(
|
|
1643
|
+
field === 'files' ? cache.inputs.files : (cache.inputs.workspaceFiles ?? []),
|
|
1644
|
+
).negative,
|
|
1645
|
+
...splitNegations(
|
|
1646
|
+
field === 'files' ? cache.outputs.files : (cache.outputs.workspaceFiles ?? []),
|
|
1647
|
+
).positive,
|
|
1648
|
+
].map((g) => at + g)
|
|
1649
|
+
const key = list.join('\0')
|
|
1650
|
+
tb = memo.byList.get(key)
|
|
1651
|
+
if (tb === undefined) memo.byList.set(key, (tb = { list, verdicts: new Map() }))
|
|
1652
|
+
memo.bySide.set(reader, tb)
|
|
1653
|
+
}
|
|
1654
|
+
let verdict = tb.verdicts.get(go)
|
|
1655
|
+
if (verdict === undefined) tb.verdicts.set(go, (verdict = outputTakenBack(go, tb.list)))
|
|
1656
|
+
return verdict
|
|
1657
|
+
}
|
|
1658
|
+
|
|
1659
|
+
function readsOutputs(
|
|
1660
|
+
memo: TakeBackMemo,
|
|
1661
|
+
x: Side,
|
|
1662
|
+
y: Side,
|
|
1663
|
+
field: 'files' | 'workspaceFiles',
|
|
1664
|
+
): void {
|
|
1601
1665
|
if (x.reads === y.reads || x.node === y.node) return
|
|
1602
1666
|
const [reader, writer] = x.reads ? [x, y] : [y, x]
|
|
1603
|
-
const cache = reader.node.config.cache!
|
|
1604
|
-
// A reader rebased to the root (X-135) speaks in its project's terms.
|
|
1605
1667
|
const at = reader.rel === undefined || reader.rel === '' ? '' : `${reader.rel}/`
|
|
1606
|
-
// What the reader's key never reads: its `!` entries and its own outputs.
|
|
1607
|
-
const takeBack = [
|
|
1608
|
-
...splitNegations(field === 'files' ? cache.inputs.files : (cache.inputs.workspaceFiles ?? []))
|
|
1609
|
-
.negative,
|
|
1610
|
-
...splitNegations(
|
|
1611
|
-
field === 'files' ? cache.outputs.files : (cache.outputs.workspaceFiles ?? []),
|
|
1612
|
-
).positive,
|
|
1613
|
-
].map((g) => at + g)
|
|
1614
1668
|
for (const go of writer.globs) {
|
|
1615
|
-
if (
|
|
1669
|
+
if (takenBack(memo, reader, field, go)) continue
|
|
1616
1670
|
for (const gi of reader.globs) {
|
|
1617
1671
|
if (!outputsOverlap(gi, go)) continue
|
|
1618
1672
|
const shown =
|
|
@@ -1625,6 +1679,7 @@ function readsOutputs(x: Side, y: Side, field: 'files' | 'workspaceFiles'): void
|
|
|
1625
1679
|
`Exclude it: add ${JSON.stringify(`!${out}`)} to ${reader.node.id}'s ` +
|
|
1626
1680
|
`cache.inputs.${field}, or set rules: { upfrontKeys: false } in vx.workspace.ts to ` +
|
|
1627
1681
|
`let it wait for its producer.`,
|
|
1682
|
+
'VX_E_INPUT_IS_OUTPUT',
|
|
1628
1683
|
)
|
|
1629
1684
|
}
|
|
1630
1685
|
}
|
package/src/index.ts
CHANGED
|
@@ -65,7 +65,12 @@ export { run, planRun, prepareRun } from './orchestrator/index.js'
|
|
|
65
65
|
// (`@vzn/vx-schedule-history` does; a plugin package reaches core only
|
|
66
66
|
// through this façade, which is what put these here).
|
|
67
67
|
export { LocalHistoryProvider } from './orchestrator/index.js'
|
|
68
|
-
export type {
|
|
68
|
+
export type {
|
|
69
|
+
HistoryProvider,
|
|
70
|
+
HistoryTable,
|
|
71
|
+
TaskHistory,
|
|
72
|
+
TaskResources,
|
|
73
|
+
} from './orchestrator/index.js'
|
|
69
74
|
export type { PreparedRun } from './orchestrator/index.js'
|
|
70
75
|
// `RunOptions.affected`: what `--affected`'s diff changed, per project.
|
|
71
76
|
export type { AffectedChanges } from './workspace/index.js'
|
|
@@ -78,6 +78,7 @@ import type { FingerprintWatch } from './fingerprint-watch.js'
|
|
|
78
78
|
import { restoreHit } from './hit-restore.js'
|
|
79
79
|
import type { MissExplainer } from './miss-reason.js'
|
|
80
80
|
import { shellVerdict } from './shell-verdict.js'
|
|
81
|
+
import { sandboxFix } from './sandbox-fix.js'
|
|
81
82
|
// The hit path's entry stays importable from here (tests).
|
|
82
83
|
export { restoreHit, type RestoreHitArgs } from './hit-restore.js'
|
|
83
84
|
import type { DeferredOutputs } from './deferred-outputs.js'
|
|
@@ -336,6 +337,18 @@ async function restoreOrMiss(
|
|
|
336
337
|
}
|
|
337
338
|
}
|
|
338
339
|
|
|
340
|
+
/**
|
|
341
|
+
* Does a write to an input during the command count as unchanged when it
|
|
342
|
+
* left the bytes the key folded (`movedInput`'s `writesByBytes`)? Only when
|
|
343
|
+
* the task says `compare`, or says nothing and `VX_REWRITTEN_INPUTS` is
|
|
344
|
+
* `compare`: by default any such write withholds the save, on CI too,
|
|
345
|
+
* since a concurrent task can change an input and put it back (owner,
|
|
346
|
+
* 2026-10-12).
|
|
347
|
+
*/
|
|
348
|
+
function rewritesByBytes(cache: CacheConfig | undefined): boolean {
|
|
349
|
+
return (cache?.rewrittenInputs ?? process.env['VX_REWRITTEN_INPUTS']) === 'compare'
|
|
350
|
+
}
|
|
351
|
+
|
|
339
352
|
/**
|
|
340
353
|
* The workspace fingerprint every key here folded has moved (a task
|
|
341
354
|
* rewrote the lockfile), said once per run.
|
|
@@ -1558,7 +1571,14 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1558
1571
|
*/
|
|
1559
1572
|
async function movedSinceKey(): Promise<string | null | undefined> {
|
|
1560
1573
|
if (described!.hash !== hash) return null
|
|
1561
|
-
return (
|
|
1574
|
+
return (
|
|
1575
|
+
(await movedInput(
|
|
1576
|
+
described!.facts,
|
|
1577
|
+
cache,
|
|
1578
|
+
described!.describedAt,
|
|
1579
|
+
rewritesByBytes(args.node.config.cache),
|
|
1580
|
+
)) ?? described!.added()
|
|
1581
|
+
)
|
|
1562
1582
|
}
|
|
1563
1583
|
|
|
1564
1584
|
/**
|
|
@@ -1608,6 +1628,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1608
1628
|
}
|
|
1609
1629
|
|
|
1610
1630
|
const finalViolations = violations
|
|
1631
|
+
const fixOf = (fix: TaskOutcome['sandboxFix']) => (fix === undefined ? {} : { sandboxFix: fix })
|
|
1611
1632
|
|
|
1612
1633
|
return {
|
|
1613
1634
|
node,
|
|
@@ -1640,6 +1661,14 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1640
1661
|
sandboxViolations: finalViolations.filter((v) => v.hint !== true).length,
|
|
1641
1662
|
// A denied path is the task's own spelling, a secret in it too.
|
|
1642
1663
|
sandboxViolationLines: finalViolations.map((v) => secrets?.mask(v.line) ?? v.line),
|
|
1664
|
+
...fixOf(
|
|
1665
|
+
sandboxFix(
|
|
1666
|
+
finalViolations,
|
|
1667
|
+
node.projectDir,
|
|
1668
|
+
(t) => secrets?.mask(t) ?? t,
|
|
1669
|
+
cacheCfg?.inputs.files,
|
|
1670
|
+
),
|
|
1671
|
+
),
|
|
1643
1672
|
}
|
|
1644
1673
|
: {}),
|
|
1645
1674
|
}
|
|
@@ -32,6 +32,7 @@ import { paint, type ColorSupport } from './colors.js'
|
|
|
32
32
|
import { neverStarted } from './summary.js'
|
|
33
33
|
import { outcomeLabel, ranNoCache, skippedReason } from './events.js'
|
|
34
34
|
import { fileMemo, linkPaths } from './path-links.js'
|
|
35
|
+
import { sandboxFixLine } from './sandbox-fix.js'
|
|
35
36
|
|
|
36
37
|
const NO_COLOR: ColorSupport = { enabled: false }
|
|
37
38
|
|
|
@@ -529,7 +530,32 @@ function violationSection(outcome: TaskOutcome, colors: ColorSupport): string[]
|
|
|
529
530
|
// every record for telemetry.
|
|
530
531
|
const body = [...new Set(vlines)]
|
|
531
532
|
// Red, not amber: a violation fails the task, so it reads as a failure.
|
|
532
|
-
|
|
533
|
+
// vx's note, after the evidence: the grants that would have allowed it.
|
|
534
|
+
// A read the output depends on is a key input too; the sandbox never
|
|
535
|
+
// adds one (owner, 2026-10-09).
|
|
536
|
+
const fix = outcome.sandboxFix
|
|
537
|
+
const line = fix === undefined ? undefined : sandboxFixLine(fix)
|
|
538
|
+
const dim = (text: string): string => paint('', text, colors, { dim: true })
|
|
539
|
+
const note =
|
|
540
|
+
fix === undefined
|
|
541
|
+
? []
|
|
542
|
+
: [
|
|
543
|
+
...(line !== undefined ? [dim(`fix: ${line}`)] : []),
|
|
544
|
+
...(fix.read !== undefined
|
|
545
|
+
? [dim('a read the output depends on belongs in cache.inputs.files too')]
|
|
546
|
+
: []),
|
|
547
|
+
...(fix.withheld !== undefined
|
|
548
|
+
? [dim(`credential path, grant it only on purpose: ${fix.withheld.join(', ')}`)]
|
|
549
|
+
: []),
|
|
550
|
+
'',
|
|
551
|
+
]
|
|
552
|
+
return [
|
|
553
|
+
sectionLine(`SANDBOX VIOLATIONS (${body.length})`, ERROR, colors),
|
|
554
|
+
'',
|
|
555
|
+
...body,
|
|
556
|
+
'',
|
|
557
|
+
...note,
|
|
558
|
+
]
|
|
533
559
|
}
|
|
534
560
|
|
|
535
561
|
export function formatFrameClose(
|
|
@@ -21,7 +21,8 @@ const DEFAULT_RECENT = 50
|
|
|
21
21
|
|
|
22
22
|
// A hit is recorded as `cache-hit` / `cache-hit-remote` and replays a
|
|
23
23
|
// success; the list is derived so a new status cannot miss it.
|
|
24
|
-
|
|
24
|
+
/** The pass set as SQL, `(…)`: one definition, `isPassStatus`. */
|
|
25
|
+
export const PASS_STATUSES = `(${TASK_STATUSES.filter(isPassStatus)
|
|
25
26
|
.map((s) => `'${s}'`)
|
|
26
27
|
.join(', ')})`
|
|
27
28
|
|
|
@@ -58,6 +59,12 @@ export interface TaskHistory {
|
|
|
58
59
|
/** Map keyed by `project#task`. */
|
|
59
60
|
export type HistoryTable = ReadonlyMap<string, TaskHistory>
|
|
60
61
|
|
|
62
|
+
/** What a scheduler learns from a task's history: its p50 and its resource maxima. */
|
|
63
|
+
export type TaskResources = Pick<
|
|
64
|
+
TaskHistory,
|
|
65
|
+
'p50DurationMs' | 'maxPeakRssBytes' | 'maxCpuParallelism'
|
|
66
|
+
>
|
|
67
|
+
|
|
61
68
|
export interface HistoryProvider {
|
|
62
69
|
loadFor(taskIds: readonly string[]): Promise<HistoryTable>
|
|
63
70
|
/**
|
|
@@ -213,6 +220,73 @@ export class LocalHistoryProvider implements HistoryProvider {
|
|
|
213
220
|
return out
|
|
214
221
|
}
|
|
215
222
|
|
|
223
|
+
/**
|
|
224
|
+
* `loadFor`'s p50 and resource maxima over the same window, without its
|
|
225
|
+
* rates. The rates group every row of the window and join each hit to
|
|
226
|
+
* its entry: 407 ms at 168,000 rows (8,002 tasks, 21 runs), most of a
|
|
227
|
+
* nothing-changed run's schedule stage. Here the executed successes are
|
|
228
|
+
* grouped alone, and the hits only as distinct keys, each joined to its
|
|
229
|
+
* entry once: 148 ms on the same table, the same numbers.
|
|
230
|
+
*/
|
|
231
|
+
async resourcesFor(taskIds: readonly string[]): Promise<ReadonlyMap<string, TaskResources>> {
|
|
232
|
+
const out = new Map<string, TaskResources>()
|
|
233
|
+
if (taskIds.length === 0) return out
|
|
234
|
+
const floor = this.floor()
|
|
235
|
+
const wanted = new Set(taskIds)
|
|
236
|
+
type Executed = {
|
|
237
|
+
project: string
|
|
238
|
+
task: string
|
|
239
|
+
ds: string
|
|
240
|
+
rss: number | null
|
|
241
|
+
cpu: number | null
|
|
242
|
+
}
|
|
243
|
+
const executed = this.db
|
|
244
|
+
.query(
|
|
245
|
+
`SELECT project, task, GROUP_CONCAT(duration_ms) AS ds, MAX(peak_rss_bytes) AS rss,
|
|
246
|
+
MAX(CASE WHEN cpu_ms IS NOT NULL AND duration_ms > 0
|
|
247
|
+
THEN cpu_ms * 1.0 / duration_ms END) AS cpu
|
|
248
|
+
FROM runs
|
|
249
|
+
WHERE id >= ? AND status = 'success' AND (cache_hit IS NULL OR cache_hit = 0)
|
|
250
|
+
GROUP BY project, task`,
|
|
251
|
+
)
|
|
252
|
+
.all(floor) as Executed[]
|
|
253
|
+
for (const row of executed) {
|
|
254
|
+
const key = `${row.project}#${row.task}`
|
|
255
|
+
if (!wanted.has(key)) continue
|
|
256
|
+
const durations = row.ds.split(',').map(Number)
|
|
257
|
+
durations.sort((a, b) => a - b)
|
|
258
|
+
out.set(key, {
|
|
259
|
+
p50DurationMs: pickPercentile(durations, 0.5),
|
|
260
|
+
...(row.rss !== null ? { maxPeakRssBytes: row.rss } : {}),
|
|
261
|
+
...(row.cpu !== null ? { maxCpuParallelism: row.cpu } : {}),
|
|
262
|
+
})
|
|
263
|
+
}
|
|
264
|
+
type Hit = { project: string; task: string; rss: number | null; cpu: number | null }
|
|
265
|
+
const hits = this.db
|
|
266
|
+
.query(
|
|
267
|
+
`SELECT h.project, h.task, MAX(e.peak_rss_bytes) AS rss,
|
|
268
|
+
MAX(CASE WHEN e.cpu_ms IS NOT NULL AND e.duration_ms > 0
|
|
269
|
+
THEN e.cpu_ms * 1.0 / e.duration_ms END) AS cpu
|
|
270
|
+
FROM (SELECT DISTINCT project, task, hash FROM runs WHERE id >= ? AND cache_hit = 1) h
|
|
271
|
+
JOIN entries e ON e.hash = h.hash
|
|
272
|
+
GROUP BY h.project, h.task`,
|
|
273
|
+
)
|
|
274
|
+
.all(floor) as Hit[]
|
|
275
|
+
for (const row of hits) {
|
|
276
|
+
const key = `${row.project}#${row.task}`
|
|
277
|
+
if (!wanted.has(key) || (row.rss === null && row.cpu === null)) continue
|
|
278
|
+
const had = out.get(key)
|
|
279
|
+
const rss = Math.max(had?.maxPeakRssBytes ?? -Infinity, row.rss ?? -Infinity)
|
|
280
|
+
const cpu = Math.max(had?.maxCpuParallelism ?? -Infinity, row.cpu ?? -Infinity)
|
|
281
|
+
out.set(key, {
|
|
282
|
+
p50DurationMs: had?.p50DurationMs,
|
|
283
|
+
...(rss !== -Infinity ? { maxPeakRssBytes: rss } : {}),
|
|
284
|
+
...(cpu !== -Infinity ? { maxCpuParallelism: cpu } : {}),
|
|
285
|
+
})
|
|
286
|
+
}
|
|
287
|
+
return out
|
|
288
|
+
}
|
|
289
|
+
|
|
216
290
|
/** The first `runs` id of the newest `recent` invocations (see `loadFor`). */
|
|
217
291
|
private floor(): number {
|
|
218
292
|
const row = this.db
|
|
@@ -35,7 +35,9 @@ export function linkPaths(text: string, dir: string, isFile: (abs: string) => bo
|
|
|
35
35
|
|
|
36
36
|
/**
|
|
37
37
|
* The files `text` names that exist (resolved against `dir`), each with the
|
|
38
|
-
* line and column it gave, unique, in order of first appearance.
|
|
38
|
+
* line and column it gave, unique, in order of first appearance. A file also
|
|
39
|
+
* named with a line drops its bare mention: bun test heads a failure with
|
|
40
|
+
* `app.test.js:` and gives the line only in the stack below it.
|
|
39
41
|
*/
|
|
40
42
|
export function fileLocations(
|
|
41
43
|
text: string,
|
|
@@ -56,9 +58,9 @@ export function fileLocations(
|
|
|
56
58
|
...(line !== undefined ? { line: Number(line) } : {}),
|
|
57
59
|
...(col !== undefined ? { col: Number(col) } : {}),
|
|
58
60
|
})
|
|
59
|
-
if (out.length === LOCATIONS_CAP) break
|
|
60
61
|
}
|
|
61
|
-
|
|
62
|
+
const lined = new Set(out.filter((l) => l.line !== undefined).map((l) => l.file))
|
|
63
|
+
return out.filter((l) => l.line !== undefined || !lined.has(l.file)).slice(0, LOCATIONS_CAP)
|
|
62
64
|
}
|
|
63
65
|
|
|
64
66
|
/** A file test that asks the file system once per path. */
|
|
@@ -127,7 +127,9 @@ export interface VxPlugin {
|
|
|
127
127
|
/**
|
|
128
128
|
* Scheduling priorities: task id → weight, higher runs first among READY
|
|
129
129
|
* tasks (merged over the structural baseline, which stays the tie-break).
|
|
130
|
-
* Runs once, after the graph is final
|
|
130
|
+
* Runs once, after the graph is final and the local hits are known, and
|
|
131
|
+
* only when a task will start: a run of confirmed local hits restores
|
|
132
|
+
* in no order a weight changes. A later plugin's weight for a task
|
|
131
133
|
* overrides an earlier one's. Return undefined to leave the baseline.
|
|
132
134
|
*/
|
|
133
135
|
schedule?(
|
|
@@ -102,9 +102,11 @@ export interface PreparedRun {
|
|
|
102
102
|
/**
|
|
103
103
|
* Scheduling priorities from plugins' `schedule` stage (task id → weight,
|
|
104
104
|
* merged over the structural baseline by the scheduler). Empty when no
|
|
105
|
-
* plugin declares the stage.
|
|
105
|
+
* plugin declares the stage. Asked only by a run with a task to start:
|
|
106
|
+
* the stage reads history (148 ms at 8,002 tasks) to order work a
|
|
107
|
+
* nothing-changed run never dispatches. Runs the stage once.
|
|
106
108
|
*/
|
|
107
|
-
priorities: ReadonlyMap<string, number
|
|
109
|
+
priorities: () => Promise<ReadonlyMap<string, number>>
|
|
108
110
|
nodes: Map<string, TaskNode>
|
|
109
111
|
/**
|
|
110
112
|
* The tasks `--exclude-dependencies` took out of the schedule: keyed as a
|
|
@@ -491,7 +493,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
491
493
|
localCache,
|
|
492
494
|
hasRemoteLayer,
|
|
493
495
|
cachePolicy: policy,
|
|
494
|
-
priorities: new Map(),
|
|
496
|
+
priorities: () => Promise.resolve(new Map()),
|
|
495
497
|
nodes: new Map(),
|
|
496
498
|
keyOnly: new Map(),
|
|
497
499
|
unresolvedTasks,
|
|
@@ -687,15 +689,16 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
687
689
|
})
|
|
688
690
|
}
|
|
689
691
|
}
|
|
690
|
-
let
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
692
|
+
let scheduled: Promise<ReadonlyMap<string, number>> | undefined
|
|
693
|
+
const priorities = (): Promise<ReadonlyMap<string, number>> =>
|
|
694
|
+
(scheduled ??= hasHook(plugins, 'schedule')
|
|
695
|
+
? applyScheduleHooks(plugins, nodes, {
|
|
696
|
+
workspaceRoot,
|
|
697
|
+
cacheDir,
|
|
698
|
+
warn: (m) => log.status(m),
|
|
699
|
+
localCache,
|
|
700
|
+
})
|
|
701
|
+
: Promise.resolve(new Map()))
|
|
699
702
|
|
|
700
703
|
return {
|
|
701
704
|
workspaceRoot,
|
|
@@ -113,11 +113,17 @@ async function runPrefetch(args: PrefetchArgs): Promise<void> {
|
|
|
113
113
|
*/
|
|
114
114
|
export function getContext(node: TaskNode, command: string) {
|
|
115
115
|
const outputs = node.config.cache?.outputs
|
|
116
|
+
let masked: string | undefined
|
|
116
117
|
return {
|
|
117
118
|
taskId: node.id,
|
|
118
119
|
// Stored in the entry row a remote hit inserts: masked as a save's is,
|
|
119
|
-
// or the token a config inlined sat in cache.db (L-27).
|
|
120
|
-
|
|
120
|
+
// or the token a config inlined sat in cache.db (L-27). Masked on the
|
|
121
|
+
// first read: only an adopt or a remote hit reads it, and masking every
|
|
122
|
+
// lookup's command cost a cold 1,090-package run ~20 ms before its
|
|
123
|
+
// first task.
|
|
124
|
+
get command() {
|
|
125
|
+
return (masked ??= maskedCommand(command, node.config.exec?.env))
|
|
126
|
+
},
|
|
121
127
|
outputs: { files: outputs?.files ?? [], workspaceFiles: outputs?.workspaceFiles ?? [] },
|
|
122
128
|
}
|
|
123
129
|
}
|
|
@@ -9,6 +9,7 @@ import { mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync
|
|
|
9
9
|
import path from 'node:path'
|
|
10
10
|
import { isGroupTask, type TaskOutcome } from '../graph/index.js'
|
|
11
11
|
import { decodeOutputLog } from './output-log.js'
|
|
12
|
+
import { PASS_STATUSES } from './history.js'
|
|
12
13
|
import { splitTaskId } from '../util/index.js'
|
|
13
14
|
import { fileLocations, fileMemo, type OutputLocation } from './path-links.js'
|
|
14
15
|
|
|
@@ -23,6 +24,10 @@ export interface TaskFailure {
|
|
|
23
24
|
/** Plain text, bounded, secrets masked; '' when the run kept none. */
|
|
24
25
|
output: string
|
|
25
26
|
locations: OutputLocation[]
|
|
27
|
+
/** The `exec.sandbox.allow` entries that would have granted its denials. */
|
|
28
|
+
sandboxFix?: NonNullable<TaskOutcome['sandboxFix']>
|
|
29
|
+
/** The first later run where this task passed (ran or hit): the failure is history. */
|
|
30
|
+
fixedIn?: string
|
|
26
31
|
}
|
|
27
32
|
|
|
28
33
|
export interface RunFailures {
|
|
@@ -56,6 +61,7 @@ function failuresOf(outcomes: readonly TaskOutcome[]): StoredFailure[] {
|
|
|
56
61
|
...(o.timedOut === true ? { timedOut: true as const } : {}),
|
|
57
62
|
output,
|
|
58
63
|
locations: fileLocations(output, o.node.projectDir, isFile),
|
|
64
|
+
...(o.sandboxFix !== undefined ? { sandboxFix: o.sandboxFix } : {}),
|
|
59
65
|
})
|
|
60
66
|
}
|
|
61
67
|
return out
|
|
@@ -110,10 +116,19 @@ export function runFailures(cacheDir: string, db: Database, runId?: string): Run
|
|
|
110
116
|
}
|
|
111
117
|
const rows = db
|
|
112
118
|
.query(
|
|
113
|
-
`SELECT project, task, exit_code AS exitCode, timed_out AS timedOut
|
|
114
|
-
|
|
119
|
+
`SELECT r.project, r.task, r.exit_code AS exitCode, r.timed_out AS timedOut,
|
|
120
|
+
(SELECT s.run_id FROM runs s WHERE s.project = r.project AND s.task = r.task
|
|
121
|
+
AND s.id > r.id AND s.status IN ${PASS_STATUSES}
|
|
122
|
+
ORDER BY s.id LIMIT 1) AS fixedIn
|
|
123
|
+
FROM runs r WHERE r.run_id = ? AND r.status = 'failed' ORDER BY r.id`,
|
|
115
124
|
)
|
|
116
|
-
.all(id) as {
|
|
125
|
+
.all(id) as {
|
|
126
|
+
project: string
|
|
127
|
+
task: string
|
|
128
|
+
exitCode: number
|
|
129
|
+
timedOut: number | null
|
|
130
|
+
fixedIn: string | null
|
|
131
|
+
}[]
|
|
117
132
|
return {
|
|
118
133
|
runId: id,
|
|
119
134
|
tasks: rows.map((r) => {
|
|
@@ -125,6 +140,8 @@ export function runFailures(cacheDir: string, db: Database, runId?: string): Run
|
|
|
125
140
|
...(r.timedOut === 1 ? { timedOut: true as const } : {}),
|
|
126
141
|
output: k?.output ?? '',
|
|
127
142
|
locations: k?.locations ?? [],
|
|
143
|
+
...(k?.sandboxFix !== undefined ? { sandboxFix: k.sandboxFix } : {}),
|
|
144
|
+
...(r.fixedIn !== null ? { fixedIn: r.fixedIn } : {}),
|
|
128
145
|
}
|
|
129
146
|
}),
|
|
130
147
|
}
|
package/src/orchestrator/run.ts
CHANGED
|
@@ -945,7 +945,15 @@ async function runOnBus(
|
|
|
945
945
|
// shouldShortCircuit (local reads on, no remote layer); when off, both
|
|
946
946
|
// maps are empty and the run is byte-identical.
|
|
947
947
|
let shortCircuit: ShortCircuit = emptyShortCircuit()
|
|
948
|
-
|
|
948
|
+
// An empty local cache answers every probe with a miss: each task
|
|
949
|
+
// derives its key as it starts instead (~100 ms before a cold 1,090-
|
|
950
|
+
// package run's first spawn), and every sandboxed task runs.
|
|
951
|
+
const coldLocal =
|
|
952
|
+
cache === prepared.localCache &&
|
|
953
|
+
shouldShortCircuit(nodes, policy, cache) &&
|
|
954
|
+
prepared.localCache.holdsNothing()
|
|
955
|
+
if (coldLocal) prewarm()
|
|
956
|
+
else if (shouldShortCircuit(nodes, policy, cache)) {
|
|
949
957
|
shortCircuit = await startLocalShortCircuit({
|
|
950
958
|
nodes,
|
|
951
959
|
cache,
|
|
@@ -1081,6 +1089,14 @@ async function runOnBus(
|
|
|
1081
1089
|
})
|
|
1082
1090
|
|
|
1083
1091
|
mark('classify + probe')
|
|
1092
|
+
// A run whose every task is a confirmed local hit dispatches only
|
|
1093
|
+
// restores, which no weight orders: the `schedule` stage is not asked.
|
|
1094
|
+
const priorities = [...nodes.values()].every(
|
|
1095
|
+
(n) => n.config.exec?.command === undefined || shortCircuit.restoreTier.has(n.id),
|
|
1096
|
+
)
|
|
1097
|
+
? new Map<string, number>()
|
|
1098
|
+
: await prepared.priorities()
|
|
1099
|
+
mark('schedule')
|
|
1084
1100
|
// Judged as each task finishes, against the history BEFORE this run's
|
|
1085
1101
|
// rows land, so the row that prints next carries the verdict: nothing
|
|
1086
1102
|
// prints below the footer. A hit or a skip asks nothing; a green miss
|
|
@@ -1164,7 +1180,11 @@ async function runOnBus(
|
|
|
1164
1180
|
: {}),
|
|
1165
1181
|
// A `schedule` plugin's weights; the scheduler keeps its structural
|
|
1166
1182
|
// baseline as the tie-break. Empty map → baseline only.
|
|
1167
|
-
...(
|
|
1183
|
+
...(priorities.size > 0 ? { priorities } : {}),
|
|
1184
|
+
// The baseline under them; `most-work` when the workspace names none.
|
|
1185
|
+
...(prepared.workspaceConfig?.schedule !== undefined
|
|
1186
|
+
? { strategy: prepared.workspaceConfig.schedule }
|
|
1187
|
+
: {}),
|
|
1168
1188
|
// Local short-circuit: confirmed stable local hits the scheduler
|
|
1169
1189
|
// runs ahead of their deps as low-priority worker-slot backfill.
|
|
1170
1190
|
// Empty when the short-circuit didn't fire → byte-identical.
|
|
@@ -1706,6 +1726,8 @@ export async function planRun(options: RunOptions): Promise<RunPlan> {
|
|
|
1706
1726
|
if (prepared.empty === 'none-affected')
|
|
1707
1727
|
return { tasks: [], noneAffected: noneAffected(prepared, options) }
|
|
1708
1728
|
if (prepared.empty !== null) return { tasks: [] }
|
|
1729
|
+
// The plan orders nothing, but it refuses what the run would refuse.
|
|
1730
|
+
await prepared.priorities()
|
|
1709
1731
|
// Its own mark: a dry run's plan (every task's hash, the cache lookups,
|
|
1710
1732
|
// the history p50s) was booked under `close`, the next mark, and read
|
|
1711
1733
|
// as 105 ms of closing a cache at 1,000 projects (item 601).
|