@vzn/vx 0.0.12 → 0.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +106 -89
- package/launcher.mjs +36 -28
- package/package.json +5 -5
- package/src/cache/cas-backend.ts +2 -2
- package/src/cache/index.ts +1 -2
- package/src/cache/inputs.ts +6 -0
- package/src/cache/layered-cache.ts +22 -2
- package/src/cli/cache.ts +5 -10
- package/src/cli/help.ts +8 -2
- package/src/cli/info.ts +0 -3
- package/src/cli/run.ts +32 -10
- package/src/config.ts +40 -1
- package/src/graph/dependency-spec.ts +16 -0
- package/src/graph/index.ts +10 -1
- package/src/graph/scheduler.ts +230 -35
- package/src/graph/task-graph.ts +69 -17
- package/src/index.ts +29 -4
- package/src/orchestrator/execute-task.ts +66 -10
- package/src/orchestrator/index.ts +11 -0
- package/src/orchestrator/metrics.ts +351 -0
- package/src/orchestrator/options.ts +26 -5
- package/src/orchestrator/plugin-host.ts +5 -10
- package/src/orchestrator/prepare.ts +20 -11
- package/src/orchestrator/protocol.ts +16 -2
- package/src/orchestrator/resources.ts +59 -0
- package/src/orchestrator/run.ts +45 -0
- package/src/orchestrator/summary.ts +24 -0
- package/src/orchestrator/task-hash.ts +17 -2
- package/src/orchestrator/telemetry.ts +9 -1
- package/src/orchestrator/upstream.ts +34 -6
- package/src/orchestrator/verify.ts +34 -11
- package/src/util/index.ts +1 -0
- package/src/util/size.ts +16 -0
- package/src/workspace/project-loader.ts +43 -1
- package/src/cache/remote-cache.ts +0 -251
- package/src/orchestrator/remote-cache-setup.ts +0 -60
|
@@ -1380,6 +1380,357 @@ export function getFlakiestTasks(db: Database, limit = 25): FlakyTask[] {
|
|
|
1380
1380
|
.slice(0, clampInt(limit, 1, 200))
|
|
1381
1381
|
}
|
|
1382
1382
|
|
|
1383
|
+
// ---------------------------------------------------------------------------
|
|
1384
|
+
// Regressions — "which tasks just started failing across branches?"
|
|
1385
|
+
// ---------------------------------------------------------------------------
|
|
1386
|
+
|
|
1387
|
+
/**
|
|
1388
|
+
* A task that is currently failing on one or more branches and USED to pass —
|
|
1389
|
+
* a regression, distinct from a flaky task (nondeterministic) or a task that
|
|
1390
|
+
* has always been broken. "Across branches" is the key signal: a task failing
|
|
1391
|
+
* on several branches at once points at a real break in that task or in shared
|
|
1392
|
+
* code, not one developer's work-in-progress branch.
|
|
1393
|
+
*/
|
|
1394
|
+
export interface RegressedTask {
|
|
1395
|
+
id: string
|
|
1396
|
+
project: string
|
|
1397
|
+
task: string
|
|
1398
|
+
/** Distinct branches whose MOST-RECENT run in the window failed. */
|
|
1399
|
+
branchesFailing: number
|
|
1400
|
+
/** Distinct branches the task ran on in the window. */
|
|
1401
|
+
branchesTotal: number
|
|
1402
|
+
/** The currently-failing branch names (capped). */
|
|
1403
|
+
branches: string[]
|
|
1404
|
+
/**
|
|
1405
|
+
* True if the task has any prior successful run — it regressed, rather than
|
|
1406
|
+
* being perpetually broken. A regression is the more urgent signal.
|
|
1407
|
+
*/
|
|
1408
|
+
regressed: boolean
|
|
1409
|
+
/** Earliest failed run in the window (ms epoch) — ≈ when it started failing. */
|
|
1410
|
+
firstFailedAt: number
|
|
1411
|
+
/** Most-recent run in the window (ms epoch). */
|
|
1412
|
+
lastRunAt: number
|
|
1413
|
+
/** Failed runs in the window. */
|
|
1414
|
+
failures: number
|
|
1415
|
+
/** Total runs in the window. */
|
|
1416
|
+
runs: number
|
|
1417
|
+
}
|
|
1418
|
+
|
|
1419
|
+
export interface RegressionArgs {
|
|
1420
|
+
/** Look-back window in days. Default 7. */
|
|
1421
|
+
sinceDays?: number
|
|
1422
|
+
/** Minimum distinct currently-failing branches to surface. Default 2
|
|
1423
|
+
* ("across branches"); pass 1 to include single-branch regressions. */
|
|
1424
|
+
minBranches?: number
|
|
1425
|
+
limit?: number
|
|
1426
|
+
}
|
|
1427
|
+
|
|
1428
|
+
const PASS_STATUSES = "('success', 'cache-hit', 'cache-hit-remote')"
|
|
1429
|
+
const BRANCH_CAP = 12
|
|
1430
|
+
|
|
1431
|
+
export function getRegressions(db: Database, args: RegressionArgs = {}): RegressedTask[] {
|
|
1432
|
+
const sinceDays = args.sinceDays ?? 7
|
|
1433
|
+
const minBranches = Math.max(1, args.minBranches ?? 2)
|
|
1434
|
+
const limit = clampInt(args.limit ?? 25, 1, 200)
|
|
1435
|
+
const since = Date.now() - sinceDays * 86_400_000
|
|
1436
|
+
|
|
1437
|
+
// The most-recent non-skipped run per (task, branch) in the window: its
|
|
1438
|
+
// status is that task's CURRENT state on that branch. Skipped/aborted runs
|
|
1439
|
+
// never finished on their own terms, so they're excluded from the state.
|
|
1440
|
+
const latest = db
|
|
1441
|
+
.query(
|
|
1442
|
+
`WITH windowed AS (
|
|
1443
|
+
SELECT r.project AS project, r.task AS task, inv.branch AS branch,
|
|
1444
|
+
r.status AS status,
|
|
1445
|
+
ROW_NUMBER() OVER (
|
|
1446
|
+
PARTITION BY r.project, r.task, inv.branch
|
|
1447
|
+
ORDER BY r.started_at DESC, r.run_id DESC
|
|
1448
|
+
) AS rn
|
|
1449
|
+
FROM runs r JOIN invocations inv ON r.run_id = inv.run_id
|
|
1450
|
+
WHERE inv.branch IS NOT NULL
|
|
1451
|
+
AND r.started_at >= ?
|
|
1452
|
+
AND r.status IN ('success', 'failed', 'cache-hit', 'cache-hit-remote')
|
|
1453
|
+
)
|
|
1454
|
+
SELECT project, task, branch, status FROM windowed WHERE rn = 1`,
|
|
1455
|
+
)
|
|
1456
|
+
.all(since) as { project: string; task: string; branch: string; status: string }[]
|
|
1457
|
+
|
|
1458
|
+
// Aggregate the latest-per-branch rows into per-task failing/total branch
|
|
1459
|
+
// sets. `failing` = the task's most recent run on that branch failed.
|
|
1460
|
+
const byTask = new Map<
|
|
1461
|
+
string,
|
|
1462
|
+
{ project: string; task: string; failing: string[]; total: Set<string> }
|
|
1463
|
+
>()
|
|
1464
|
+
for (const r of latest) {
|
|
1465
|
+
const id = `${r.project}#${r.task}`
|
|
1466
|
+
let agg = byTask.get(id)
|
|
1467
|
+
if (agg === undefined) {
|
|
1468
|
+
agg = { project: r.project, task: r.task, failing: [], total: new Set() }
|
|
1469
|
+
byTask.set(id, agg)
|
|
1470
|
+
}
|
|
1471
|
+
agg.total.add(r.branch)
|
|
1472
|
+
if (r.status === 'failed') agg.failing.push(r.branch)
|
|
1473
|
+
}
|
|
1474
|
+
|
|
1475
|
+
const out: RegressedTask[] = []
|
|
1476
|
+
for (const [id, agg] of byTask) {
|
|
1477
|
+
if (agg.failing.length < minBranches) continue
|
|
1478
|
+
// Per-task follow-ups (the getFlakiestTasks pattern) — the regressed set
|
|
1479
|
+
// is small, so a couple of point queries each is cheap.
|
|
1480
|
+
const win = db
|
|
1481
|
+
.query(
|
|
1482
|
+
`SELECT COUNT(*) AS runs,
|
|
1483
|
+
SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) AS failures,
|
|
1484
|
+
MIN(CASE WHEN status = 'failed' THEN started_at END) AS first_failed,
|
|
1485
|
+
MAX(started_at) AS last_run
|
|
1486
|
+
FROM runs WHERE project = ? AND task = ? AND started_at >= ?`,
|
|
1487
|
+
)
|
|
1488
|
+
.get(agg.project, agg.task, since) as {
|
|
1489
|
+
runs: number
|
|
1490
|
+
failures: number | null
|
|
1491
|
+
first_failed: number | null
|
|
1492
|
+
last_run: number | null
|
|
1493
|
+
}
|
|
1494
|
+
const everPassed =
|
|
1495
|
+
db
|
|
1496
|
+
.query(
|
|
1497
|
+
`SELECT 1 FROM runs WHERE project = ? AND task = ? AND status IN ${PASS_STATUSES} LIMIT 1`,
|
|
1498
|
+
)
|
|
1499
|
+
.get(agg.project, agg.task) !== null
|
|
1500
|
+
out.push({
|
|
1501
|
+
id,
|
|
1502
|
+
project: agg.project,
|
|
1503
|
+
task: agg.task,
|
|
1504
|
+
branchesFailing: agg.failing.length,
|
|
1505
|
+
branchesTotal: agg.total.size,
|
|
1506
|
+
branches: agg.failing.sort().slice(0, BRANCH_CAP),
|
|
1507
|
+
regressed: everPassed,
|
|
1508
|
+
firstFailedAt: win.first_failed ?? 0,
|
|
1509
|
+
lastRunAt: win.last_run ?? 0,
|
|
1510
|
+
failures: win.failures ?? 0,
|
|
1511
|
+
runs: win.runs,
|
|
1512
|
+
})
|
|
1513
|
+
}
|
|
1514
|
+
// Regressions (used-to-pass) first, then most branches affected, then the
|
|
1515
|
+
// most recently-started failures — the "act on this now" ordering.
|
|
1516
|
+
return out
|
|
1517
|
+
.sort(
|
|
1518
|
+
(a, b) =>
|
|
1519
|
+
Number(b.regressed) - Number(a.regressed) ||
|
|
1520
|
+
b.branchesFailing - a.branchesFailing ||
|
|
1521
|
+
b.firstFailedAt - a.firstFailedAt,
|
|
1522
|
+
)
|
|
1523
|
+
.slice(0, limit)
|
|
1524
|
+
}
|
|
1525
|
+
|
|
1526
|
+
// ---------------------------------------------------------------------------
|
|
1527
|
+
// Period-over-period analysis — "how is CI trending vs the previous window,
|
|
1528
|
+
// and which tasks moved the most?" Two adjacent equal-length windows.
|
|
1529
|
+
// ---------------------------------------------------------------------------
|
|
1530
|
+
|
|
1531
|
+
export interface PeriodStats {
|
|
1532
|
+
/** Distinct runs (invocations) in the window. */
|
|
1533
|
+
runs: number
|
|
1534
|
+
/** Task executions recorded (rows). */
|
|
1535
|
+
taskRuns: number
|
|
1536
|
+
/** Executions that actually ran (not a cache hit). */
|
|
1537
|
+
executed: number
|
|
1538
|
+
/** Failed task executions. */
|
|
1539
|
+
failures: number
|
|
1540
|
+
/** Cache-hit task executions (local or remote). */
|
|
1541
|
+
cacheHits: number
|
|
1542
|
+
/** Sum of executed-task durations (ms). */
|
|
1543
|
+
totalDurationMs: number
|
|
1544
|
+
/** Mean duration of successful executed tasks (ms). */
|
|
1545
|
+
avgDurationMs: number
|
|
1546
|
+
p50DurationMs: number | undefined
|
|
1547
|
+
p95DurationMs: number | undefined
|
|
1548
|
+
/** failures / taskRuns. */
|
|
1549
|
+
failureRate: number
|
|
1550
|
+
/** cacheHits / taskRuns. */
|
|
1551
|
+
cacheHitRate: number
|
|
1552
|
+
}
|
|
1553
|
+
|
|
1554
|
+
export interface TaskMover {
|
|
1555
|
+
id: string
|
|
1556
|
+
project: string
|
|
1557
|
+
task: string
|
|
1558
|
+
currentAvgMs: number
|
|
1559
|
+
previousAvgMs: number
|
|
1560
|
+
/** current − previous (positive = slower / regressed). */
|
|
1561
|
+
deltaMs: number
|
|
1562
|
+
/** (current − previous) / previous, as a fraction. */
|
|
1563
|
+
deltaPct: number
|
|
1564
|
+
currentRuns: number
|
|
1565
|
+
previousRuns: number
|
|
1566
|
+
}
|
|
1567
|
+
|
|
1568
|
+
export interface PeriodComparison {
|
|
1569
|
+
windowDays: number
|
|
1570
|
+
current: { from: number; to: number; stats: PeriodStats }
|
|
1571
|
+
previous: { from: number; to: number; stats: PeriodStats }
|
|
1572
|
+
/**
|
|
1573
|
+
* Tasks whose average executed duration moved the most between the two
|
|
1574
|
+
* windows, by absolute impact (biggest ms shift first; positive = slower).
|
|
1575
|
+
* Only tasks with >= minRuns successful executions in BOTH windows qualify,
|
|
1576
|
+
* so a mover reflects a real trend, not one-off noise.
|
|
1577
|
+
*/
|
|
1578
|
+
movers: TaskMover[]
|
|
1579
|
+
}
|
|
1580
|
+
|
|
1581
|
+
export interface PeriodComparisonArgs {
|
|
1582
|
+
/** Length of each window in days. Default 7 (this week vs last week). */
|
|
1583
|
+
windowDays?: number
|
|
1584
|
+
/** End of the CURRENT window (ms). Default now — override for tests. */
|
|
1585
|
+
endMs?: number
|
|
1586
|
+
/** Min successful executions in EACH window for a mover to qualify. Default 3. */
|
|
1587
|
+
minRuns?: number
|
|
1588
|
+
/** Max movers returned. Default 8. */
|
|
1589
|
+
limit?: number
|
|
1590
|
+
/** Scope to one project — the project-detail "did MY project trend?" view. */
|
|
1591
|
+
project?: string
|
|
1592
|
+
/** Scope to one task within `project` — the task-detail trend. */
|
|
1593
|
+
task?: string
|
|
1594
|
+
}
|
|
1595
|
+
|
|
1596
|
+
/** Optional per-project/task scoping shared by the two window queries. */
|
|
1597
|
+
interface PeriodScope {
|
|
1598
|
+
project?: string
|
|
1599
|
+
task?: string
|
|
1600
|
+
}
|
|
1601
|
+
|
|
1602
|
+
function scopeSql(scope: PeriodScope): { sql: string; params: string[] } {
|
|
1603
|
+
let sql = ''
|
|
1604
|
+
const params: string[] = []
|
|
1605
|
+
if (scope.project !== undefined) {
|
|
1606
|
+
sql += ' AND project = ?'
|
|
1607
|
+
params.push(scope.project)
|
|
1608
|
+
}
|
|
1609
|
+
if (scope.task !== undefined) {
|
|
1610
|
+
sql += ' AND task = ?'
|
|
1611
|
+
params.push(scope.task)
|
|
1612
|
+
}
|
|
1613
|
+
return { sql, params }
|
|
1614
|
+
}
|
|
1615
|
+
|
|
1616
|
+
function periodStats(db: Database, from: number, to: number, scope: PeriodScope): PeriodStats {
|
|
1617
|
+
const { sql, params } = scopeSql(scope)
|
|
1618
|
+
const agg = db
|
|
1619
|
+
.query(
|
|
1620
|
+
// COALESCE every SUM: over an empty window SQLite SUM() returns NULL, and
|
|
1621
|
+
// the previous window is empty for any workspace younger than the window
|
|
1622
|
+
// (a fresh serve, a quiet prior week) — a bare SUM would ship `null` where
|
|
1623
|
+
// PeriodStats declares `number`, throwing on the client's `.toFixed()`.
|
|
1624
|
+
`SELECT COUNT(*) AS taskRuns,
|
|
1625
|
+
COUNT(DISTINCT run_id) AS runs,
|
|
1626
|
+
COALESCE(SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END), 0) AS failures,
|
|
1627
|
+
COALESCE(SUM(CASE WHEN cache_hit = 1 THEN 1 ELSE 0 END), 0) AS cacheHits,
|
|
1628
|
+
COALESCE(SUM(CASE WHEN cache_hit IS NULL OR cache_hit = 0 THEN 1 ELSE 0 END), 0) AS executed,
|
|
1629
|
+
COALESCE(SUM(CASE WHEN cache_hit IS NULL OR cache_hit = 0 THEN duration_ms ELSE 0 END), 0) AS totalDurationMs
|
|
1630
|
+
FROM runs WHERE started_at >= ? AND started_at < ?${sql}`,
|
|
1631
|
+
)
|
|
1632
|
+
.get(from, to, ...params) as {
|
|
1633
|
+
taskRuns: number
|
|
1634
|
+
runs: number
|
|
1635
|
+
failures: number
|
|
1636
|
+
cacheHits: number
|
|
1637
|
+
executed: number
|
|
1638
|
+
totalDurationMs: number
|
|
1639
|
+
}
|
|
1640
|
+
const durs = (
|
|
1641
|
+
db
|
|
1642
|
+
.query(
|
|
1643
|
+
`SELECT duration_ms AS d FROM runs
|
|
1644
|
+
WHERE started_at >= ? AND started_at < ?
|
|
1645
|
+
AND (cache_hit IS NULL OR cache_hit = 0) AND status = 'success'${sql}
|
|
1646
|
+
ORDER BY duration_ms`,
|
|
1647
|
+
)
|
|
1648
|
+
.all(from, to, ...params) as { d: number }[]
|
|
1649
|
+
).map((r) => r.d)
|
|
1650
|
+
const taskRuns = agg.taskRuns
|
|
1651
|
+
return {
|
|
1652
|
+
runs: agg.runs,
|
|
1653
|
+
taskRuns,
|
|
1654
|
+
executed: agg.executed,
|
|
1655
|
+
failures: agg.failures,
|
|
1656
|
+
cacheHits: agg.cacheHits,
|
|
1657
|
+
totalDurationMs: agg.totalDurationMs,
|
|
1658
|
+
avgDurationMs: durs.length > 0 ? Math.round(durs.reduce((a, b) => a + b, 0) / durs.length) : 0,
|
|
1659
|
+
p50DurationMs: pickPercentile(durs, 0.5),
|
|
1660
|
+
p95DurationMs: pickPercentile(durs, 0.95),
|
|
1661
|
+
failureRate: taskRuns > 0 ? agg.failures / taskRuns : 0,
|
|
1662
|
+
cacheHitRate: taskRuns > 0 ? agg.cacheHits / taskRuns : 0,
|
|
1663
|
+
}
|
|
1664
|
+
}
|
|
1665
|
+
|
|
1666
|
+
function avgByTask(
|
|
1667
|
+
db: Database,
|
|
1668
|
+
from: number,
|
|
1669
|
+
to: number,
|
|
1670
|
+
scope: PeriodScope,
|
|
1671
|
+
): Map<string, { avg: number; runs: number; project: string; task: string }> {
|
|
1672
|
+
const { sql, params } = scopeSql(scope)
|
|
1673
|
+
const rows = db
|
|
1674
|
+
.query(
|
|
1675
|
+
`SELECT project, task, AVG(duration_ms) AS avg, COUNT(*) AS runs
|
|
1676
|
+
FROM runs
|
|
1677
|
+
WHERE started_at >= ? AND started_at < ?
|
|
1678
|
+
AND (cache_hit IS NULL OR cache_hit = 0) AND status = 'success'${sql}
|
|
1679
|
+
GROUP BY project, task`,
|
|
1680
|
+
)
|
|
1681
|
+
.all(from, to, ...params) as { project: string; task: string; avg: number; runs: number }[]
|
|
1682
|
+
return new Map(
|
|
1683
|
+
rows.map((r) => [
|
|
1684
|
+
`${r.project}#${r.task}`,
|
|
1685
|
+
{ avg: r.avg, runs: r.runs, project: r.project, task: r.task },
|
|
1686
|
+
]),
|
|
1687
|
+
)
|
|
1688
|
+
}
|
|
1689
|
+
|
|
1690
|
+
export function getPeriodComparison(
|
|
1691
|
+
db: Database,
|
|
1692
|
+
args: PeriodComparisonArgs = {},
|
|
1693
|
+
): PeriodComparison {
|
|
1694
|
+
const windowDays = Math.max(1, args.windowDays ?? 7)
|
|
1695
|
+
const minRuns = Math.max(1, args.minRuns ?? 3)
|
|
1696
|
+
const limit = clampInt(args.limit ?? 8, 1, 100)
|
|
1697
|
+
const scope: PeriodScope = {
|
|
1698
|
+
...(args.project !== undefined ? { project: args.project } : {}),
|
|
1699
|
+
...(args.task !== undefined ? { task: args.task } : {}),
|
|
1700
|
+
}
|
|
1701
|
+
const to = args.endMs ?? Date.now()
|
|
1702
|
+
const win = windowDays * 86_400_000
|
|
1703
|
+
const curFrom = to - win
|
|
1704
|
+
const prevTo = curFrom
|
|
1705
|
+
const prevFrom = curFrom - win
|
|
1706
|
+
|
|
1707
|
+
const cur = avgByTask(db, curFrom, to, scope)
|
|
1708
|
+
const prev = avgByTask(db, prevFrom, prevTo, scope)
|
|
1709
|
+
const movers: TaskMover[] = []
|
|
1710
|
+
for (const [id, c] of cur) {
|
|
1711
|
+
const p = prev.get(id)
|
|
1712
|
+
if (p === undefined || c.runs < minRuns || p.runs < minRuns) continue
|
|
1713
|
+
movers.push({
|
|
1714
|
+
id,
|
|
1715
|
+
project: c.project,
|
|
1716
|
+
task: c.task,
|
|
1717
|
+
currentAvgMs: Math.round(c.avg),
|
|
1718
|
+
previousAvgMs: Math.round(p.avg),
|
|
1719
|
+
deltaMs: Math.round(c.avg - p.avg),
|
|
1720
|
+
deltaPct: p.avg > 0 ? (c.avg - p.avg) / p.avg : 0,
|
|
1721
|
+
currentRuns: c.runs,
|
|
1722
|
+
previousRuns: p.runs,
|
|
1723
|
+
})
|
|
1724
|
+
}
|
|
1725
|
+
movers.sort((a, b) => Math.abs(b.deltaMs) - Math.abs(a.deltaMs))
|
|
1726
|
+
return {
|
|
1727
|
+
windowDays,
|
|
1728
|
+
current: { from: curFrom, to, stats: periodStats(db, curFrom, to, scope) },
|
|
1729
|
+
previous: { from: prevFrom, to: prevTo, stats: periodStats(db, prevFrom, prevTo, scope) },
|
|
1730
|
+
movers: movers.slice(0, limit),
|
|
1731
|
+
}
|
|
1732
|
+
}
|
|
1733
|
+
|
|
1383
1734
|
// ---------------------------------------------------------------------------
|
|
1384
1735
|
// Bottlenecks — "if you sped up X, you'd save Y per week"
|
|
1385
1736
|
// ---------------------------------------------------------------------------
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// so internals like prepare.ts can import them without an upward
|
|
3
3
|
// import of index.ts — the module entry must stay cycle-free.
|
|
4
4
|
|
|
5
|
-
import type { CachePolicy } from '../cache/index.js'
|
|
5
|
+
import type { CachePolicy, RemoteCacheLayer } from '../cache/index.js'
|
|
6
6
|
import type { ContinueMode } from '../graph/index.js'
|
|
7
7
|
import type { TaskOutcome } from '../graph/index.js'
|
|
8
8
|
import type { EventBus } from './events.js'
|
|
@@ -72,14 +72,17 @@ export interface RunOptions {
|
|
|
72
72
|
* executed + cacheable task is re-run after its save and its outputs are
|
|
73
73
|
* content-compared: a divergence means the task is non-hermetic and its
|
|
74
74
|
* cache entry is provably unsafe (see docs/design/cache-correctness).
|
|
75
|
-
* `
|
|
76
|
-
* tasks
|
|
77
|
-
*
|
|
78
|
-
*
|
|
75
|
+
* `fingerprint` computes + ships an output-tree fingerprint on executed
|
|
76
|
+
* tasks (no re-run — the cross-machine diff feed; `--verify`/`=all` set
|
|
77
|
+
* it too, for free). `allow` (from `--verify-allow=<pkg#task>,…`) exempts
|
|
78
|
+
* known-nondeterministic tasks from failing the run. A pure side-channel —
|
|
79
|
+
* never folded into a cache key; the re-run never saves. Undefined = off
|
|
80
|
+
* (a plain run is byte-identical).
|
|
79
81
|
*/
|
|
80
82
|
verify?: {
|
|
81
83
|
determinism: boolean
|
|
82
84
|
inputs: boolean
|
|
85
|
+
fingerprint: boolean
|
|
83
86
|
allow: ReadonlySet<string>
|
|
84
87
|
}
|
|
85
88
|
/**
|
|
@@ -91,6 +94,14 @@ export interface RunOptions {
|
|
|
91
94
|
* cached), so a `--timeout` run cache-hits a plain run's entry.
|
|
92
95
|
*/
|
|
93
96
|
timeout?: number
|
|
97
|
+
/**
|
|
98
|
+
* Memory budget (bytes) that per-task `exec.resources.memory`
|
|
99
|
+
* reservations pack against (`--memory <size>`). Defaults to
|
|
100
|
+
* `os.totalmem()` — override it in cgroup-limited containers, where
|
|
101
|
+
* totalmem() reports the HOST's RAM, not the limit. A per-run
|
|
102
|
+
* scheduling knob only — never folded into any cache key.
|
|
103
|
+
*/
|
|
104
|
+
memory?: number
|
|
94
105
|
/**
|
|
95
106
|
* Failure propagation: 'never' stops dispatch on the first failure
|
|
96
107
|
* (in-flight tasks finish; everything queued skips), 'deps-ok'
|
|
@@ -168,6 +179,16 @@ export interface RunOptions {
|
|
|
168
179
|
* record something useful.
|
|
169
180
|
*/
|
|
170
181
|
command?: string
|
|
182
|
+
/**
|
|
183
|
+
* An injected remote cache layer — the embedder seam mirroring
|
|
184
|
+
* `telemetrySinks`: a host that already holds a wire client (a
|
|
185
|
+
* distribution agent, a serve executing on behalf of a submitter)
|
|
186
|
+
* passes it here and run() composes `LayeredCache(local, injected)`.
|
|
187
|
+
* Explicit injection WINS over the plugin `cache` capability (the
|
|
188
|
+
* host knows best; prevents double-wrapping when the workspace also
|
|
189
|
+
* declares a cache plugin). Undefined → identical to before.
|
|
190
|
+
*/
|
|
191
|
+
remoteCache?: RemoteCacheLayer
|
|
171
192
|
}
|
|
172
193
|
|
|
173
194
|
export interface RunSummary {
|
|
@@ -56,26 +56,21 @@ export async function resolveBackend(
|
|
|
56
56
|
|
|
57
57
|
/**
|
|
58
58
|
* Resolve the cache layer. First plugin returning a non-undefined `cache`
|
|
59
|
-
* wins; otherwise the caller's `fallback` (
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
* (explicit beats ambient). A broken cache factory aborts.
|
|
59
|
+
* wins; otherwise the caller's `fallback` (the plain local cache — core
|
|
60
|
+
* ships no wire client; a remote layer comes from a plugin or from
|
|
61
|
+
* `RunOptions.remoteCache`). A broken cache factory aborts.
|
|
63
62
|
*/
|
|
64
63
|
export async function resolveCache(
|
|
65
64
|
plugins: readonly VxPlugin[],
|
|
66
65
|
_localCache: Cache,
|
|
67
66
|
ctx: CacheContext,
|
|
68
|
-
|
|
67
|
+
_log: Logger,
|
|
69
68
|
fallback: () => CacheLayer,
|
|
70
69
|
): Promise<CacheLayer> {
|
|
71
70
|
for (const plugin of plugins) {
|
|
72
71
|
if (plugin.cache === undefined) continue
|
|
73
72
|
const cache = await safe(plugin, 'cache', () => plugin.cache!(ctx))
|
|
74
|
-
if (cache !== undefined)
|
|
75
|
-
if (process.env.VX_REMOTE_CACHE_URL)
|
|
76
|
-
log.status(`[vx] plugin '${plugin.name}' cache overrides VX_REMOTE_CACHE_*`)
|
|
77
|
-
return cache
|
|
78
|
-
}
|
|
73
|
+
if (cache !== undefined) return cache
|
|
79
74
|
}
|
|
80
75
|
return fallback()
|
|
81
76
|
}
|
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
type CachePolicy,
|
|
17
17
|
FULL_CACHE_POLICY,
|
|
18
18
|
GitFilesCache,
|
|
19
|
+
LayeredCache,
|
|
19
20
|
populateGitFilesCache,
|
|
20
21
|
} from '../cache/index.js'
|
|
21
22
|
import {
|
|
@@ -33,7 +34,6 @@ import {
|
|
|
33
34
|
type ProjectEntry,
|
|
34
35
|
} from '../workspace/index.js'
|
|
35
36
|
import { buildTaskGraph, expandRequested, type TaskNode } from '../graph/index.js'
|
|
36
|
-
import { wrapWithRemoteCache } from './remote-cache-setup.js'
|
|
37
37
|
import { resolveCache } from './plugin-host.js'
|
|
38
38
|
import type { VxPlugin } from './plugin.js'
|
|
39
39
|
import { createHashCache, type HashCache } from './task-hash.js'
|
|
@@ -197,17 +197,26 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
197
197
|
? path.resolve(options.cwd, options.cacheDir)
|
|
198
198
|
: resolveCacheDir(workspaceRoot, workspaceConfig)
|
|
199
199
|
const localCache = new Cache(cacheDir, { read: policy.localRead, write: policy.localWrite })
|
|
200
|
-
// Cache seam:
|
|
201
|
-
//
|
|
202
|
-
//
|
|
200
|
+
// Cache seam precedence: an EXPLICITLY injected remote layer
|
|
201
|
+
// (RunOptions.remoteCache — a distribution agent or serve that already
|
|
202
|
+
// holds a wire client) wins outright; else a plugin's `cache` capability;
|
|
203
|
+
// else the local cache alone. Core ships no wire client — the remote
|
|
204
|
+
// cache is a plugin concern (native-cache-wire-2026-07). Injection
|
|
205
|
+
// winning prevents double-wrapping when the workspace also declares a
|
|
206
|
+
// cache plugin.
|
|
203
207
|
const plugins = (workspaceConfig?.plugins ?? []) as readonly VxPlugin[]
|
|
204
|
-
const cache =
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
208
|
+
const cache = options.remoteCache
|
|
209
|
+
? new LayeredCache(localCache, options.remoteCache, {
|
|
210
|
+
policy,
|
|
211
|
+
onRemoteError: (err) => log.status(`[vx] remote cache: ${err.message}`),
|
|
212
|
+
})
|
|
213
|
+
: await resolveCache(
|
|
214
|
+
plugins,
|
|
215
|
+
localCache,
|
|
216
|
+
{ workspaceRoot, cacheDir, warn: (m) => log.status(m), localCache, policy },
|
|
217
|
+
log,
|
|
218
|
+
() => localCache,
|
|
219
|
+
)
|
|
211
220
|
const workspaceFingerprint = await computeWorkspaceFingerprint(workspaceRoot)
|
|
212
221
|
|
|
213
222
|
const gitFilesCache = new GitFilesCache()
|
|
@@ -28,9 +28,19 @@ export interface RunRequest {
|
|
|
28
28
|
retries?: number
|
|
29
29
|
/** Run-level task-timeout default (ms); per-task `exec.timeout` wins. */
|
|
30
30
|
timeout?: number
|
|
31
|
+
/** Explicit `--memory` budget (bytes) for `exec.resources` packing; the
|
|
32
|
+
* default (executing side's os.totalmem()) resolves where tasks run. */
|
|
33
|
+
memory?: number
|
|
31
34
|
/** Cache-correctness verification (`--verify`); `allow` as an array (Sets
|
|
32
|
-
* don't serialize).
|
|
33
|
-
|
|
35
|
+
* don't serialize). `fingerprint` is additive-optional — an old serve
|
|
36
|
+
* ignores it, so a delegated `=fingerprint` run against an old serve
|
|
37
|
+
* degrades to a harmless no-op verify. */
|
|
38
|
+
verify?: {
|
|
39
|
+
determinism: boolean
|
|
40
|
+
inputs: boolean
|
|
41
|
+
fingerprint?: boolean
|
|
42
|
+
allow: readonly string[]
|
|
43
|
+
}
|
|
34
44
|
flow?: 'focused' | 'broad'
|
|
35
45
|
outputLogs?: 'full' | 'errors-only' | 'none'
|
|
36
46
|
excludeDependencies?: 'all' | readonly string[]
|
|
@@ -89,10 +99,12 @@ export function optionsToRequest(options: RunOptions): RunRequest {
|
|
|
89
99
|
if (options.frozen !== undefined) req.frozen = options.frozen
|
|
90
100
|
if (options.retries !== undefined) req.retries = options.retries
|
|
91
101
|
if (options.timeout !== undefined) req.timeout = options.timeout
|
|
102
|
+
if (options.memory !== undefined) req.memory = options.memory
|
|
92
103
|
if (options.verify !== undefined)
|
|
93
104
|
req.verify = {
|
|
94
105
|
determinism: options.verify.determinism,
|
|
95
106
|
inputs: options.verify.inputs,
|
|
107
|
+
fingerprint: options.verify.fingerprint,
|
|
96
108
|
allow: [...options.verify.allow],
|
|
97
109
|
}
|
|
98
110
|
if (options.flow !== undefined) req.flow = options.flow
|
|
@@ -122,10 +134,12 @@ export function requestToOptions(request: RunRequest): RunOptions {
|
|
|
122
134
|
if (request.frozen !== undefined) options.frozen = request.frozen
|
|
123
135
|
if (request.retries !== undefined) options.retries = request.retries
|
|
124
136
|
if (request.timeout !== undefined) options.timeout = request.timeout
|
|
137
|
+
if (request.memory !== undefined) options.memory = request.memory
|
|
125
138
|
if (request.verify !== undefined)
|
|
126
139
|
options.verify = {
|
|
127
140
|
determinism: request.verify.determinism,
|
|
128
141
|
inputs: request.verify.inputs,
|
|
142
|
+
fingerprint: request.verify.fingerprint ?? false,
|
|
129
143
|
allow: new Set(request.verify.allow),
|
|
130
144
|
}
|
|
131
145
|
if (request.flow !== undefined) options.flow = request.flow
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
// Resolve `exec.resources` declarations into absolute per-task costs the
|
|
2
|
+
// scheduler's 2-D admission gate packs against. Pure and budget-
|
|
3
|
+
// parameterized — percent forms resolve here, ONCE per run, so the
|
|
4
|
+
// scheduler's inner loop only ever does a Map.get. The loader has already
|
|
5
|
+
// validated every form (this module trusts internal code); an
|
|
6
|
+
// unparseable value can't reach it from a real config.
|
|
7
|
+
|
|
8
|
+
import type { ResourceCost, TaskNode } from '../graph/index.js'
|
|
9
|
+
import { parseSize } from '../util/index.js'
|
|
10
|
+
|
|
11
|
+
const PERCENT_RE = /^(\d+(?:\.\d+)?)%$/
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* `cpus` → CPU units. A number is itself (fractional ok); `"<n>%"` is
|
|
15
|
+
* percent of the CPU budget, kept fractional (rounding is display-only).
|
|
16
|
+
* Absent → 0 = reserve nothing.
|
|
17
|
+
*/
|
|
18
|
+
export function resolveCpu(v: number | string | undefined, cpuBudget: number): number {
|
|
19
|
+
if (v === undefined) return 0
|
|
20
|
+
if (typeof v === 'number') return v
|
|
21
|
+
const m = v.match(PERCENT_RE)
|
|
22
|
+
if (m) return (Number(m[1]) / 100) * cpuBudget
|
|
23
|
+
return 0
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* `memory` → bytes. A number is bytes; a size string goes through
|
|
28
|
+
* `parseSize` (`"2GB"`, `"512MB"`); `"<n>%"` is percent of the memory
|
|
29
|
+
* budget. Absent → 0 = reserve nothing.
|
|
30
|
+
*/
|
|
31
|
+
export function resolveMem(v: number | string | undefined, memBudget: number): number {
|
|
32
|
+
if (v === undefined) return 0
|
|
33
|
+
if (typeof v === 'number') return v
|
|
34
|
+
const m = v.match(PERCENT_RE)
|
|
35
|
+
if (m) return (Number(m[1]) / 100) * memBudget
|
|
36
|
+
return parseSize(v) ?? 0
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Resolve the whole graph's declared reservations, OMITTING zero-cost
|
|
41
|
+
* tasks: an empty map means "no reservations declared", the single gate
|
|
42
|
+
* the scheduler (and run.ts's option threading) keys off to keep every
|
|
43
|
+
* current run byte-identical.
|
|
44
|
+
*/
|
|
45
|
+
export function resolveResourceCosts(
|
|
46
|
+
nodes: ReadonlyMap<string, TaskNode>,
|
|
47
|
+
cpuBudget: number,
|
|
48
|
+
memBudget: number,
|
|
49
|
+
): Map<string, ResourceCost> {
|
|
50
|
+
const out = new Map<string, ResourceCost>()
|
|
51
|
+
for (const node of nodes.values()) {
|
|
52
|
+
const declared = node.config.exec?.resources
|
|
53
|
+
if (declared === undefined) continue
|
|
54
|
+
const cpu = resolveCpu(declared.cpus, cpuBudget)
|
|
55
|
+
const mem = resolveMem(declared.memory, memBudget)
|
|
56
|
+
if (cpu > 0 || mem > 0) out.set(node.id, { cpu, mem })
|
|
57
|
+
}
|
|
58
|
+
return out
|
|
59
|
+
}
|