@vzn/vx 0.0.18 → 0.0.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +99 -65
- package/package.json +10 -20
- package/src/bin.ts +12 -2
- package/src/cache/archive.ts +120 -18
- package/src/cache/cache.ts +313 -1324
- package/src/cache/chained-cache.ts +0 -8
- package/src/cache/config-evals.ts +89 -0
- package/src/cache/file-hashes.ts +262 -0
- package/src/cache/git-inputs.ts +637 -0
- package/src/cache/index.ts +16 -7
- package/src/cache/inputs.ts +246 -733
- package/src/cache/layer.ts +604 -0
- package/src/cache/layered-cache.ts +12 -17
- package/src/cache/output-index.ts +224 -0
- package/src/cache/policy.ts +84 -0
- package/src/cache/run-history.ts +147 -0
- package/src/cache/zstd.ts +160 -0
- package/src/cli/cache.ts +39 -11
- package/src/cli/completions.ts +104 -0
- package/src/cli/core-alias.ts +25 -0
- package/src/cli/help.ts +64 -36
- package/src/cli/index.ts +54 -15
- package/src/cli/info.ts +120 -91
- package/src/cli/init.ts +75 -0
- package/src/cli/last.ts +38 -6
- package/src/cli/lock.ts +2 -1
- package/src/cli/plan-format.ts +14 -3
- package/src/cli/plugin-commands.ts +15 -8
- package/src/cli/run.ts +84 -302
- package/src/cli/select.ts +332 -0
- package/src/cli/show.ts +144 -69
- package/src/cli/upgrade.ts +2 -1
- package/src/cli/watch.ts +647 -97
- package/src/cli/why.ts +44 -14
- package/src/cli/workspace-config.ts +126 -0
- package/src/config.ts +156 -136
- package/src/exec/env.ts +11 -0
- package/src/exec/executor.ts +11 -26
- package/src/exec/index.ts +6 -2
- package/src/{plugins/local-executor/index.ts → exec/local-executor.ts} +12 -16
- package/src/exec/runner.ts +57 -18
- package/src/exec/sandbox-binds.ts +182 -0
- package/src/exec/sandbox-paths.ts +50 -0
- package/src/exec/sandbox-runtime.ts +582 -370
- package/src/exec/sandbox-violations.ts +280 -0
- package/src/graph/index.ts +1 -9
- package/src/graph/scheduler.ts +158 -203
- package/src/graph/task-graph.ts +2 -2
- package/src/index.ts +97 -102
- package/src/orchestrator/admission.ts +137 -0
- package/src/orchestrator/deferred-outputs.ts +3 -0
- package/src/orchestrator/doctor.ts +255 -0
- package/src/orchestrator/events.ts +42 -8
- package/src/orchestrator/execute-task.ts +145 -554
- package/src/orchestrator/failure-mode.ts +169 -28
- package/src/orchestrator/framed-output.ts +41 -36
- package/src/orchestrator/history.ts +118 -98
- package/src/orchestrator/hit-restore.ts +221 -0
- package/src/orchestrator/index.ts +35 -4
- package/src/orchestrator/lockfile-claim.ts +281 -0
- package/src/orchestrator/logger.ts +13 -22
- package/src/orchestrator/metrics.ts +53 -20
- package/src/orchestrator/miss-save.ts +169 -0
- package/src/orchestrator/options.ts +34 -31
- package/src/orchestrator/persistent.ts +79 -0
- package/src/orchestrator/placement.ts +182 -0
- package/src/orchestrator/plugin-host.ts +195 -38
- package/src/orchestrator/plugin.ts +162 -9
- package/src/orchestrator/prepare.ts +77 -132
- package/src/orchestrator/projects.ts +259 -0
- package/src/orchestrator/remote-prefetch.ts +1 -1
- package/src/orchestrator/run-artifacts.ts +25 -3
- package/src/orchestrator/run-context.ts +1 -1
- package/src/orchestrator/run-records.ts +149 -0
- package/src/orchestrator/run-report.ts +2 -2
- package/src/orchestrator/run.ts +246 -597
- package/src/orchestrator/sandbox-request.ts +243 -0
- package/src/orchestrator/save-lane.ts +47 -0
- package/src/orchestrator/signals.ts +111 -0
- package/src/orchestrator/stable-keys.ts +60 -3
- package/src/orchestrator/status-line.ts +2 -1
- package/src/orchestrator/summary.ts +68 -25
- package/src/orchestrator/task-hash.ts +20 -18
- package/src/orchestrator/task-log-buffer.ts +8 -11
- package/src/orchestrator/telemetry-host.ts +11 -1
- package/src/orchestrator/telemetry.ts +3 -17
- package/src/util/cgroup.ts +160 -0
- package/src/util/edit-distance.ts +49 -0
- package/src/util/index.ts +11 -3
- package/src/util/paths.ts +23 -2
- package/src/util/settle.ts +19 -0
- package/src/util/verbs.ts +39 -0
- package/src/workspace/affected.ts +111 -11
- package/src/workspace/config-cache.ts +11 -2
- package/src/workspace/config-eval.ts +11 -4
- package/src/workspace/config-imports.ts +5 -4
- package/src/workspace/config-schema.ts +837 -0
- package/src/workspace/filter.ts +43 -6
- package/src/workspace/fingerprint.ts +37 -3
- package/src/workspace/index.ts +30 -3
- package/src/workspace/lockfile.ts +12 -7
- package/src/{cli → workspace}/migrate-scripts.ts +26 -19
- package/src/{cli/migrate.ts → workspace/migration.ts} +133 -124
- package/src/workspace/package-graph.ts +98 -28
- package/src/workspace/project-loader.ts +76 -746
- package/src/workspace/workspace.ts +98 -13
- package/plugins/local-cache/index.ts +0 -2
- package/plugins/local-executor/index.ts +0 -2
- package/plugins/schedule-history/index.ts +0 -2
- package/src/cache/cas-backend.ts +0 -115
- package/src/cache/digest.ts +0 -32
- package/src/cli/migrate-emit.ts +0 -21
- package/src/cli/migrate-nx.ts +0 -473
- package/src/cli/migrate-turbo.ts +0 -401
- package/src/cli/prune.ts +0 -291
- package/src/orchestrator/missing-plugin.ts +0 -9
- package/src/orchestrator/resources.ts +0 -45
- package/src/orchestrator/verify.ts +0 -174
- package/src/plugins/local-cache/index.ts +0 -11
- package/src/plugins/schedule-history/index.ts +0 -109
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
// The flakiness verdict, in ONE place. Two
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
1
|
+
// The flakiness verdict, in ONE place. Two readers classify `runs` rows —
|
|
2
|
+
// the all-time per-task query here and `LocalHistoryProvider`'s bounded
|
|
3
|
+
// window — and they used to encode the rule independently, which let them
|
|
4
|
+
// drift into opposite verdicts on identical data. Both build on
|
|
5
|
+
// `mixedOutcomeKeysSql` + `failureModeOf`, so the rule cannot fork again.
|
|
6
|
+
// The per-run detector (`detectFlaky`) and the doctor's list (`flakyTasks`)
|
|
7
|
+
// read the same per-key projection (`keyOutcomesSql`).
|
|
7
8
|
|
|
8
9
|
import type { Database } from 'bun:sqlite'
|
|
9
10
|
import { KEYED_RUNS_SQL } from '../cache/index.js'
|
|
@@ -19,43 +20,183 @@ const HIT_STATUSES = `(${TASK_STATUSES.filter(isCacheHit)
|
|
|
19
20
|
export type FailureMode = 'stable' | 'flaky-recoverable' | 'flaky-fatal'
|
|
20
21
|
|
|
21
22
|
/**
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
23
|
+
* Per-key outcome projection: every keyed (project, task, hash) under
|
|
24
|
+
* `source` (a relation with `project`, `task`, `hash`, `status`, `cache_hit`
|
|
25
|
+
* columns) with how often it failed and how often it passed — a pass being
|
|
26
|
+
* an executed success or a cache hit, which replays a success. Every
|
|
27
|
+
* flakiness reader is a filter over this one projection.
|
|
26
28
|
*/
|
|
29
|
+
function keyOutcomesSql(source: string, where = ''): string {
|
|
30
|
+
return `SELECT project, task, hash,
|
|
31
|
+
SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) AS failures,
|
|
32
|
+
SUM(CASE WHEN status = 'success' OR status IN ${HIT_STATUSES} OR cache_hit = 1
|
|
33
|
+
THEN 1 ELSE 0 END) AS passes
|
|
34
|
+
FROM ${source}
|
|
35
|
+
WHERE ${KEYED_RUNS_SQL}${where}
|
|
36
|
+
GROUP BY project, task, hash`
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Per-key outcome subquery: the distinct cache keys under `source` that
|
|
41
|
+
* produced BOTH a failure and a success — the definitional flake: identical
|
|
42
|
+
* inputs, different outcomes. A failure whose key never succeeded is a
|
|
43
|
+
* legitimate break (a changed input that fails), which belongs to the
|
|
44
|
+
* regressions surface, not flakiness. One projection of the rule serves the
|
|
45
|
+
* all-time query below and the windowed one in `history.ts`.
|
|
46
|
+
*/
|
|
47
|
+
export function mixedOutcomeKeysSql(source: string, where = ''): string {
|
|
48
|
+
return `SELECT project, task FROM (${keyOutcomesSql(source, where)})
|
|
49
|
+
WHERE failures > 0 AND passes > 0`
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Mixed-outcome keys of one (project, task) over its WHOLE recorded history. */
|
|
27
53
|
export function mixedOutcomeKeyCount(db: Database, project: string, task: string): number {
|
|
28
54
|
const row = db
|
|
29
55
|
.query(
|
|
30
|
-
`SELECT COUNT(*) AS n FROM (
|
|
31
|
-
SELECT hash FROM runs
|
|
32
|
-
WHERE project = ? AND task = ? AND ${KEYED_RUNS_SQL}
|
|
33
|
-
GROUP BY hash
|
|
34
|
-
HAVING SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) > 0
|
|
35
|
-
AND SUM(CASE WHEN status = 'success' OR status IN ${HIT_STATUSES} OR cache_hit = 1
|
|
36
|
-
THEN 1 ELSE 0 END) > 0
|
|
37
|
-
)`,
|
|
56
|
+
`SELECT COUNT(*) AS n FROM (${mixedOutcomeKeysSql('runs', ' AND project = ? AND task = ?')})`,
|
|
38
57
|
)
|
|
39
58
|
.get(project, task) as { n: number }
|
|
40
59
|
return row.n
|
|
41
60
|
}
|
|
42
61
|
|
|
43
62
|
/**
|
|
44
|
-
* Flaky requires a NONDETERMINISM signal: a within-run retry,
|
|
45
|
-
* that both failed and succeeded. Failures
|
|
46
|
-
* legitimate breaks, however many there
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
* path: a task that never failed is stable without touching the DB.
|
|
63
|
+
* The verdict. Flaky requires a NONDETERMINISM signal: a within-run retry,
|
|
64
|
+
* or a cache key that both failed and succeeded (`mixedKeys`). Failures
|
|
65
|
+
* alone — each on its own key — are legitimate breaks, however many there
|
|
66
|
+
* are. `mixedKeys` is a thunk so a task that never failed is stable without
|
|
67
|
+
* the key count being computed at all.
|
|
50
68
|
*/
|
|
69
|
+
export function failureModeOf(
|
|
70
|
+
counts: { total: number; failures: number; retried: number },
|
|
71
|
+
mixedKeys: () => number,
|
|
72
|
+
): FailureMode {
|
|
73
|
+
const flakySignal = counts.retried > 0 || (counts.failures > 0 && mixedKeys() > 0)
|
|
74
|
+
if (!flakySignal) return 'stable'
|
|
75
|
+
return counts.failures < counts.total / 5 ? 'flaky-recoverable' : 'flaky-fatal'
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** `failureModeOf` over a task's whole recorded history (the DB is not touched unless it failed). */
|
|
51
79
|
export function classifyFailureMode(
|
|
52
80
|
db: Database,
|
|
53
81
|
project: string,
|
|
54
82
|
task: string,
|
|
55
83
|
counts: { total: number; failures: number; retried: number },
|
|
56
84
|
): FailureMode {
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
85
|
+
return failureModeOf(counts, () => mixedOutcomeKeyCount(db, project, task))
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** An executed, keyed outcome of the run in progress — what `detectFlaky` judges. */
|
|
89
|
+
export interface FlakyCandidate {
|
|
90
|
+
project: string
|
|
91
|
+
task: string
|
|
92
|
+
hash: string
|
|
93
|
+
status: 'success' | 'failed'
|
|
94
|
+
/** Attempts this run took; > 1 is a within-run retry. */
|
|
95
|
+
attempts: number
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** One task the run just proved nondeterministic. */
|
|
99
|
+
export interface FlakyFinding extends FlakyCandidate {
|
|
100
|
+
/** `project#task`. */
|
|
101
|
+
taskId: string
|
|
102
|
+
/** Outcomes on record for this exact key, this run's included. */
|
|
103
|
+
passes: number
|
|
104
|
+
failures: number
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// Bound parameters per statement: SQLite's compile-time cap is 32 766 in
|
|
108
|
+
// every build Bun ships, but a run can execute more tasks than that only in
|
|
109
|
+
// theory — chunking keeps the query shape identical at any size.
|
|
110
|
+
const IN_CHUNK = 500
|
|
111
|
+
|
|
112
|
+
function chunked<T>(items: readonly T[], each: (chunk: readonly T[]) => void): void {
|
|
113
|
+
for (let i = 0; i < items.length; i += IN_CHUNK) each(items.slice(i, i + IN_CHUNK))
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const keyOf = (c: { project: string; task: string; hash: string }): string =>
|
|
117
|
+
`${c.project}\0${c.task}\0${c.hash}`
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Which of this run's executed tasks are flaky, judged BEFORE the run's own
|
|
121
|
+
* rows are recorded: a pass on a key that has failed before, a failure on
|
|
122
|
+
* a key that has passed before, or a within-run retry. The counts fold this
|
|
123
|
+
* run in.
|
|
124
|
+
*
|
|
125
|
+
* The cost follows the run's colour. No candidate (every task a hit or a
|
|
126
|
+
* skip): no query. A green miss: one probe of the `runs_failed` partial
|
|
127
|
+
* index for the keys that ever failed — failures are the rare rows, so the
|
|
128
|
+
* probe is microseconds however long the history. Only a key that DID fail
|
|
129
|
+
* before, or a task failing now, pays the projection scan over its own
|
|
130
|
+
* keys, and a run that is failing has more to look at than a scan.
|
|
131
|
+
*/
|
|
132
|
+
export function detectFlaky(db: Database, candidates: readonly FlakyCandidate[]): FlakyFinding[] {
|
|
133
|
+
if (candidates.length === 0) return []
|
|
134
|
+
const everFailed = new Set<string>()
|
|
135
|
+
const passedNow = candidates.filter((c) => c.status === 'success' && c.attempts <= 1)
|
|
136
|
+
chunked(passedNow, (chunk) => {
|
|
137
|
+
const rows = db
|
|
138
|
+
.query(
|
|
139
|
+
`SELECT DISTINCT project, task, hash FROM runs
|
|
140
|
+
WHERE status = 'failed' AND hash IN (${chunk.map(() => '?').join(', ')})`,
|
|
141
|
+
)
|
|
142
|
+
.all(...chunk.map((c) => c.hash)) as { project: string; task: string; hash: string }[]
|
|
143
|
+
for (const r of rows) everFailed.add(keyOf(r))
|
|
144
|
+
})
|
|
145
|
+
const suspects = candidates.filter(
|
|
146
|
+
(c) => c.status === 'failed' || c.attempts > 1 || everFailed.has(keyOf(c)),
|
|
147
|
+
)
|
|
148
|
+
if (suspects.length === 0) return []
|
|
149
|
+
const counts = new Map<string, { passes: number; failures: number }>()
|
|
150
|
+
chunked(suspects, (chunk) => {
|
|
151
|
+
const rows = db
|
|
152
|
+
.query(keyOutcomesSql('runs', ` AND hash IN (${chunk.map(() => '?').join(', ')})`))
|
|
153
|
+
.all(...chunk.map((c) => c.hash)) as {
|
|
154
|
+
project: string
|
|
155
|
+
task: string
|
|
156
|
+
hash: string
|
|
157
|
+
passes: number
|
|
158
|
+
failures: number
|
|
159
|
+
}[]
|
|
160
|
+
for (const r of rows) counts.set(keyOf(r), { passes: r.passes, failures: r.failures })
|
|
161
|
+
})
|
|
162
|
+
const out: FlakyFinding[] = []
|
|
163
|
+
for (const c of suspects) {
|
|
164
|
+
const before = counts.get(keyOf(c)) ?? { passes: 0, failures: 0 }
|
|
165
|
+
const passes = before.passes + (c.status === 'success' ? 1 : 0)
|
|
166
|
+
const failures = before.failures + (c.status === 'failed' ? 1 : 0)
|
|
167
|
+
if (!(passes > 0 && failures > 0) && c.attempts <= 1) continue
|
|
168
|
+
out.push({ ...c, taskId: `${c.project}#${c.task}`, passes, failures })
|
|
169
|
+
}
|
|
170
|
+
return out
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/** One task the recorded history shows both passing and failing on unchanged inputs. */
|
|
174
|
+
export interface FlakyTask {
|
|
175
|
+
/** `project#task`. */
|
|
176
|
+
taskId: string
|
|
177
|
+
project: string
|
|
178
|
+
task: string
|
|
179
|
+
/** Distinct cache keys that both passed and failed. */
|
|
180
|
+
keys: number
|
|
181
|
+
/** Outcomes over those keys. */
|
|
182
|
+
passes: number
|
|
183
|
+
failures: number
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Every task with a mixed-outcome key anywhere in the retained history
|
|
188
|
+
* (30 days, `RunHistory.pruneOlderThan`), most failures first: the
|
|
189
|
+
* doctor's list. A full scan of `runs`, which an inspection verb affords.
|
|
190
|
+
*/
|
|
191
|
+
export function flakyTasks(db: Database): FlakyTask[] {
|
|
192
|
+
const rows = db
|
|
193
|
+
.query(
|
|
194
|
+
`SELECT project, task, COUNT(*) AS keys, SUM(passes) AS passes, SUM(failures) AS failures
|
|
195
|
+
FROM (${keyOutcomesSql('runs')})
|
|
196
|
+
WHERE failures > 0 AND passes > 0
|
|
197
|
+
GROUP BY project, task
|
|
198
|
+
ORDER BY failures DESC, passes DESC, project, task`,
|
|
199
|
+
)
|
|
200
|
+
.all() as { project: string; task: string; keys: number; passes: number; failures: number }[]
|
|
201
|
+
return rows.map((r) => ({ ...r, taskId: `${r.project}#${r.task}` }))
|
|
61
202
|
}
|
|
@@ -28,6 +28,7 @@
|
|
|
28
28
|
import { isGroupTask, type TaskNode, type TaskOutcome } from '../graph/index.js'
|
|
29
29
|
import { paint, type ColorSupport } from './colors.js'
|
|
30
30
|
import { formatDuration } from './summary.js'
|
|
31
|
+
import { outcomeLabel } from './events.js'
|
|
31
32
|
|
|
32
33
|
const NO_COLOR: ColorSupport = { enabled: false }
|
|
33
34
|
|
|
@@ -92,7 +93,6 @@ export function formatTaskBlock(
|
|
|
92
93
|
|
|
93
94
|
const idPainted = paintTaskId(node, colors, { bold: true })
|
|
94
95
|
const corner = (s: string) => paint('', s, colors, { dim: true })
|
|
95
|
-
const section = (title: string, color?: string) => sectionLine(title, color ?? '', colors)
|
|
96
96
|
const header = formatBlockHeader(outcome, colors)
|
|
97
97
|
const lines: string[] = [`${corner('┌─')} ${idPainted} ${corner('>')} ${header}`]
|
|
98
98
|
|
|
@@ -109,15 +109,7 @@ export function formatTaskBlock(
|
|
|
109
109
|
pushStreamSection(lines, stdout, 'STDOUT', SUCCESS, body.droppedStdout ?? 0, colors)
|
|
110
110
|
pushStreamSection(lines, stderr, 'STDERR', ERROR, body.droppedStderr ?? 0, colors)
|
|
111
111
|
|
|
112
|
-
|
|
113
|
-
// user sees them in context with the failing task, not as loose
|
|
114
|
-
// status output above the box.
|
|
115
|
-
const vlines = outcome.sandboxViolationLines
|
|
116
|
-
if (vlines && vlines.length > 0) {
|
|
117
|
-
lines.push(section(`SANDBOX VIOLATIONS (${vlines.length})`, WARN), '')
|
|
118
|
-
for (const v of vlines) lines.push(v)
|
|
119
|
-
lines.push('')
|
|
120
|
-
}
|
|
112
|
+
lines.push(...violationSection(outcome, colors))
|
|
121
113
|
|
|
122
114
|
lines.push(`${corner('└─')} ${idPainted} ${corner('──')}${formatBlockFooter(outcome, colors)}`)
|
|
123
115
|
return lines.join('\n') + '\n'
|
|
@@ -220,7 +212,7 @@ function paintTaskId(node: TaskNode, colors: ColorSupport, opts: { bold?: boolea
|
|
|
220
212
|
// the row. All detail (exit code, output) lives in the framed block.
|
|
221
213
|
export const TIME_COL = 7 // "0ms" … "999.99s"
|
|
222
214
|
const STATUS_COL = 7 // "success" / "running" / "skipped"
|
|
223
|
-
const CACHE_COL =
|
|
215
|
+
const CACHE_COL = 8 // "no-cache" / "remote" / "local" / "fresh" / "miss"
|
|
224
216
|
|
|
225
217
|
/** Right-align the duration in a TIME_COL cell (pad on the left). */
|
|
226
218
|
function timeCell(ms: number | null, colors: ColorSupport): string {
|
|
@@ -278,7 +270,8 @@ function cacheOf(o: TaskOutcome): { word: string; color: string } {
|
|
|
278
270
|
switch (o.status) {
|
|
279
271
|
case 'success':
|
|
280
272
|
case 'failed':
|
|
281
|
-
|
|
273
|
+
// A task with no `cache` block never consulted the cache: not a miss.
|
|
274
|
+
return { word: o.node.config.cache === undefined ? 'no-cache' : 'miss', color: '' } // dim
|
|
282
275
|
case 'cache-hit':
|
|
283
276
|
return o.restored === false
|
|
284
277
|
? { word: 'fresh', color: SUCCESS }
|
|
@@ -385,6 +378,27 @@ export function formatFrameOpen(node: TaskNode, colors: ColorSupport = NO_COLOR)
|
|
|
385
378
|
return `${corner('┌─')} ${mark}${paintTaskId(node, colors, { bold: true })} ${corner('>')} $ ${cmd}`
|
|
386
379
|
}
|
|
387
380
|
|
|
381
|
+
/**
|
|
382
|
+
* Sandbox violations get a dedicated section inside the frame so the user
|
|
383
|
+
* sees them in context with the failing task, not as loose status output
|
|
384
|
+
* above the box. Shared by BOTH renderers: the focused/live frame used to
|
|
385
|
+
* omit it, so a single-task run — exactly how someone debugging one task
|
|
386
|
+
* invokes vx — showed a failure with no violations at all (owner,
|
|
387
|
+
* 2026-09-05).
|
|
388
|
+
*/
|
|
389
|
+
function violationSection(outcome: TaskOutcome, colors: ColorSupport): string[] {
|
|
390
|
+
const vlines = outcome.sandboxViolationLines
|
|
391
|
+
if (!vlines || vlines.length === 0) return []
|
|
392
|
+
// Unique lines only — a task's children trip the same denial many times
|
|
393
|
+
// and the repeats bury everything else. The lines themselves are printed
|
|
394
|
+
// VERBATIM: a violation is evidence, and evidence that has been rewritten
|
|
395
|
+
// for display is not the thing the sandbox reported. The outcome keeps
|
|
396
|
+
// every record for telemetry.
|
|
397
|
+
const body = [...new Set(vlines)]
|
|
398
|
+
// Red, not amber: a violation fails the task, so it reads as a failure.
|
|
399
|
+
return [sectionLine(`SANDBOX VIOLATIONS (${body.length})`, ERROR, colors), '', ...body, '']
|
|
400
|
+
}
|
|
401
|
+
|
|
388
402
|
export function formatFrameClose(
|
|
389
403
|
node: TaskNode,
|
|
390
404
|
outcome: TaskOutcome,
|
|
@@ -400,7 +414,9 @@ export function formatFrameClose(
|
|
|
400
414
|
persistent && outcome.status === 'success'
|
|
401
415
|
? ` ${paint('', `(${formatDuration(outcome.durationMs)})`, colors, { dim: true })} ${paint(ACCENT, 'running', colors)}`
|
|
402
416
|
: formatBlockFooter(outcome, colors)
|
|
403
|
-
|
|
417
|
+
const close = `${corner('└─')} ${mark}${paintTaskId(node, colors, { bold: true })} ${corner('──')}${tail}`
|
|
418
|
+
const violations = violationSection(outcome, colors)
|
|
419
|
+
return violations.length === 0 ? close : `${violations.join('\n')}\n${close}`
|
|
404
420
|
}
|
|
405
421
|
|
|
406
422
|
function isPersistentNode(node: TaskNode): boolean {
|
|
@@ -454,27 +470,21 @@ export function formatPersistentList(
|
|
|
454
470
|
function formatBlockHeader(o: TaskOutcome, colors: ColorSupport): string {
|
|
455
471
|
const shortHash = o.hash ? o.hash.slice(0, 8) : ''
|
|
456
472
|
const dim = (s: string) => paint('', s, colors, { dim: true })
|
|
473
|
+
const word = outcomeLabel(o)
|
|
457
474
|
switch (o.status) {
|
|
458
475
|
case 'cache-hit':
|
|
459
|
-
|
|
460
|
-
return `${paint(SUCCESS, 'up-to-date', colors)} ${dim(`• ${shortHash}`)}`
|
|
461
|
-
}
|
|
462
|
-
return `${paint(SUCCESS, 'restored-local', colors)} ${dim(`• ${shortHash}`)}`
|
|
476
|
+
return `${paint(SUCCESS, word, colors)} ${dim(`• ${shortHash}`)}`
|
|
463
477
|
case 'cache-hit-remote':
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
}
|
|
467
|
-
return `${paint(ACCENT, 'restored-remote', colors)} ${dim(`• ${shortHash}`)}`
|
|
478
|
+
// A remote restore is the one hit painted accent: bytes came off the wire.
|
|
479
|
+
return `${paint(o.restored === false ? SUCCESS : ACCENT, word, colors)} ${dim(`• ${shortHash}`)}`
|
|
468
480
|
case 'failed':
|
|
469
481
|
// The command lives in its own `├─ command` section; the header
|
|
470
482
|
// carries the outcome like every other status.
|
|
471
|
-
return paint(ERROR,
|
|
483
|
+
return paint(ERROR, word, colors, { bold: true })
|
|
472
484
|
case 'skipped':
|
|
473
485
|
return paint(WARN, 'skipped (upstream failed)', colors)
|
|
474
|
-
case 'success':
|
|
475
|
-
return dim('success')
|
|
476
486
|
default:
|
|
477
|
-
return
|
|
487
|
+
return dim(word)
|
|
478
488
|
}
|
|
479
489
|
}
|
|
480
490
|
|
|
@@ -492,21 +502,16 @@ function formatBlockFooter(o: TaskOutcome, colors: ColorSupport): string {
|
|
|
492
502
|
}
|
|
493
503
|
|
|
494
504
|
function formatStatusTag(o: TaskOutcome, colors: ColorSupport): string {
|
|
505
|
+
const word = outcomeLabel(o)
|
|
495
506
|
switch (o.status) {
|
|
507
|
+
case 'failed':
|
|
508
|
+
return paint(ERROR, word, colors, { bold: true })
|
|
509
|
+
case 'skipped':
|
|
510
|
+
return paint(WARN, word, colors)
|
|
496
511
|
case 'cache-hit':
|
|
497
|
-
return paint('', o.restored === false ? 'up-to-date' : 'restored-local', colors, {
|
|
498
|
-
dim: true,
|
|
499
|
-
})
|
|
500
512
|
case 'cache-hit-remote':
|
|
501
|
-
return paint('', o.restored === false ? 'up-to-date' : 'restored-remote', colors, {
|
|
502
|
-
dim: true,
|
|
503
|
-
})
|
|
504
513
|
case 'success':
|
|
505
|
-
return paint('',
|
|
506
|
-
case 'failed':
|
|
507
|
-
return paint(ERROR, `failed (exit ${o.exitCode})`, colors, { bold: true })
|
|
508
|
-
case 'skipped':
|
|
509
|
-
return paint(WARN, 'skipped', colors)
|
|
514
|
+
return paint('', word, colors, { dim: true })
|
|
510
515
|
default:
|
|
511
516
|
return o.status
|
|
512
517
|
}
|
|
@@ -2,9 +2,10 @@
|
|
|
2
2
|
// uses for predictive priority and `vx info --history` surfaces.
|
|
3
3
|
//
|
|
4
4
|
// The data has been in cache.db.runs since schema v11; what's new is
|
|
5
|
-
// surfacing it.
|
|
6
|
-
// (project, task) pair
|
|
7
|
-
// the run's lifetime. Loaded once
|
|
5
|
+
// surfacing it. One SQL statement per call pulls the last N executed rows
|
|
6
|
+
// per (project, task) pair out of a rowid-bounded slice of the table; the
|
|
7
|
+
// result becomes a read-only snapshot for the run's lifetime. Loaded once
|
|
8
|
+
// by the `schedule` stage (or `--dry`); never mutated mid-run.
|
|
8
9
|
//
|
|
9
10
|
// Two providers:
|
|
10
11
|
// LocalHistoryProvider — reads cache.db directly (zero-config).
|
|
@@ -13,7 +14,7 @@
|
|
|
13
14
|
|
|
14
15
|
import type { Database } from 'bun:sqlite'
|
|
15
16
|
import { EXECUTED_RUNS_SQL } from '../cache/index.js'
|
|
16
|
-
import {
|
|
17
|
+
import { failureModeOf, mixedOutcomeKeysSql } from './failure-mode.js'
|
|
17
18
|
import type { FailureMode } from './failure-mode.js'
|
|
18
19
|
|
|
19
20
|
const DEFAULT_RECENT = 50
|
|
@@ -30,9 +31,22 @@ export interface TaskHistory {
|
|
|
30
31
|
successRate: number
|
|
31
32
|
/** Cache hit rate over the recent window ([0, 1]). */
|
|
32
33
|
hitRate: number
|
|
33
|
-
/** Failure mode classification
|
|
34
|
-
* the
|
|
34
|
+
/** Failure mode classification over the same window (`failureModeOf`,
|
|
35
|
+
* the one rule `vx why`'s all-time query also applies). */
|
|
35
36
|
failureMode: FailureMode
|
|
37
|
+
/**
|
|
38
|
+
* Largest peak RSS (bytes) any successful execution in the window
|
|
39
|
+
* reported; undefined when none did (hits report nothing, and a
|
|
40
|
+
* platform may not expose rusage). The floor a memory reservation
|
|
41
|
+
* learned from history packs on.
|
|
42
|
+
*/
|
|
43
|
+
maxPeakRssBytes?: number
|
|
44
|
+
/**
|
|
45
|
+
* The most CPU parallelism any successful execution in the window
|
|
46
|
+
* showed: cpu time over wall time, so 2.0 means two cores busy for the
|
|
47
|
+
* whole task. Undefined when no row carried both numbers.
|
|
48
|
+
*/
|
|
49
|
+
maxCpuParallelism?: number
|
|
36
50
|
}
|
|
37
51
|
|
|
38
52
|
/** Map keyed by `project#task`. */
|
|
@@ -56,138 +70,144 @@ export class LocalHistoryProvider implements HistoryProvider {
|
|
|
56
70
|
private readonly recent: number = DEFAULT_RECENT,
|
|
57
71
|
) {}
|
|
58
72
|
|
|
73
|
+
/**
|
|
74
|
+
* The window is the last `recent` INVOCATIONS, read as one rowid slice.
|
|
75
|
+
*
|
|
76
|
+
* `runs` has no (project, task) index: it cost every run a scattered
|
|
77
|
+
* B-tree write per task (2026-09-09, 1,000 warm hits: the record stage
|
|
78
|
+
* 57–79 ms with it, 14–19 ms without) and bought this reader nothing —
|
|
79
|
+
* ranking every pair's rows newest-first (`ROW_NUMBER` over a partition)
|
|
80
|
+
* sorted the whole table either way, 230 ms at 116k rows. Rows are
|
|
81
|
+
* appended in time order, one contiguous block per invocation, so the
|
|
82
|
+
* rows of the newest `recent` invocations are exactly `id >= MIN(id)` of
|
|
83
|
+
* the `recent`-th newest header — a primary-key range, no sort, plain
|
|
84
|
+
* aggregates, and at most one row per pair per invocation, so "the last
|
|
85
|
+
* `recent` invocations" IS the last `recent` runs for a pair that runs
|
|
86
|
+
* every time and whatever fewer it has for one that runs less often. A
|
|
87
|
+
* hint for ordering and prediction, not a ledger; `vx why` / `vx last`
|
|
88
|
+
* read by `run_id` and stay exact.
|
|
89
|
+
*/
|
|
59
90
|
async loadFor(taskIds: readonly string[]): Promise<HistoryTable> {
|
|
60
91
|
const out = new Map<string, TaskHistory>()
|
|
61
92
|
if (taskIds.length === 0) return out
|
|
62
93
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
// task names never contain `#`, and a project name (a package name) never
|
|
68
|
-
// does either.
|
|
69
|
-
const pairs = (taskIds as string[]).map(splitTaskId)
|
|
70
|
-
const tupleParams = pairs.flat()
|
|
71
|
-
const tuplePlaceholders = pairs.map(() => '(?,?)').join(',')
|
|
72
|
-
|
|
73
|
-
// One CTE per call: rank rows per (project, task) descending by
|
|
74
|
-
// started_at, keep the top `recent`, aggregate. Cache-hit rows
|
|
75
|
-
// (cache_hit = 1) are excluded from the duration percentiles so
|
|
76
|
-
// p50/p99 reflect work the runner actually did. successRate +
|
|
77
|
-
// hitRate are computed over ALL recent rows.
|
|
78
|
-
//
|
|
79
|
-
// `skipped` rows are excluded from the window entirely: a skip is a task
|
|
80
|
-
// the run never executed, so it neither belongs in a success/hit RATE nor
|
|
81
|
-
// deserves to occupy one of the `recent` slots — a task whose upstream
|
|
82
|
-
// keeps breaking would otherwise push its own real history out of view.
|
|
83
|
-
const sql = `
|
|
84
|
-
WITH recent AS (
|
|
85
|
-
SELECT
|
|
86
|
-
project,
|
|
87
|
-
task,
|
|
88
|
-
status,
|
|
89
|
-
duration_ms,
|
|
90
|
-
cache_hit,
|
|
91
|
-
attempts,
|
|
92
|
-
ROW_NUMBER() OVER (PARTITION BY project, task ORDER BY started_at DESC) AS rn
|
|
93
|
-
FROM runs
|
|
94
|
-
WHERE (project, task) IN (VALUES ${tuplePlaceholders})
|
|
95
|
-
AND ${EXECUTED_RUNS_SQL}
|
|
94
|
+
const floorRow = this.db
|
|
95
|
+
.query(
|
|
96
|
+
`SELECT MIN(id) AS id FROM runs WHERE run_id =
|
|
97
|
+
(SELECT run_id FROM invocations ORDER BY started_at DESC LIMIT 1 OFFSET ?)`,
|
|
96
98
|
)
|
|
99
|
+
.get(this.recent - 1) as { id: number | null }
|
|
100
|
+
// Fewer invocations than the window (or none): the whole table is the window.
|
|
101
|
+
const floor = floorRow.id ?? 0
|
|
102
|
+
|
|
103
|
+
// `skipped` rows are excluded: a skip is a task the run never executed,
|
|
104
|
+
// so it belongs in no success/hit RATE and no duration. Percentiles are
|
|
105
|
+
// over the executed-success rows of the slice — work the runner actually
|
|
106
|
+
// did — while the rates count every executed row.
|
|
107
|
+
const sql = `
|
|
97
108
|
SELECT
|
|
98
|
-
project,
|
|
99
|
-
task,
|
|
109
|
+
r.project,
|
|
110
|
+
r.task,
|
|
100
111
|
COUNT(*) AS total,
|
|
101
|
-
SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) AS successes,
|
|
102
|
-
SUM(CASE WHEN cache_hit = 1 THEN 1 ELSE 0 END) AS hits,
|
|
103
|
-
SUM(CASE WHEN
|
|
104
|
-
SUM(CASE WHEN
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
112
|
+
SUM(CASE WHEN r.status = 'success' THEN 1 ELSE 0 END) AS successes,
|
|
113
|
+
SUM(CASE WHEN r.cache_hit = 1 THEN 1 ELSE 0 END) AS hits,
|
|
114
|
+
SUM(CASE WHEN r.status = 'failed' THEN 1 ELSE 0 END) AS failures,
|
|
115
|
+
SUM(CASE WHEN r.attempts > 1 THEN 1 ELSE 0 END) AS retried,
|
|
116
|
+
GROUP_CONCAT(CASE WHEN (r.cache_hit IS NULL OR r.cache_hit = 0) AND r.status = 'success'
|
|
117
|
+
THEN r.duration_ms END) AS ds,
|
|
118
|
+
MAX(CASE WHEN (r.cache_hit IS NULL OR r.cache_hit = 0) AND r.status = 'success'
|
|
119
|
+
THEN r.peak_rss_bytes
|
|
120
|
+
WHEN r.cache_hit = 1 THEN e.peak_rss_bytes END) AS rss,
|
|
121
|
+
MAX(CASE WHEN (r.cache_hit IS NULL OR r.cache_hit = 0) AND r.status = 'success'
|
|
122
|
+
AND r.cpu_ms IS NOT NULL AND r.duration_ms > 0
|
|
123
|
+
THEN r.cpu_ms * 1.0 / r.duration_ms
|
|
124
|
+
WHEN r.cache_hit = 1 AND e.cpu_ms IS NOT NULL AND e.duration_ms > 0
|
|
125
|
+
THEN e.cpu_ms * 1.0 / e.duration_ms END) AS cpu
|
|
126
|
+
FROM runs r
|
|
127
|
+
-- A hit executed nothing here, but its entry carries what the
|
|
128
|
+
-- PRODUCING execution used (the artifact's sidecar, indexed at save
|
|
129
|
+
-- and ingest alike), so a task this machine has only ever restored
|
|
130
|
+
-- — a fresh runner behind a remote cache — still has a number. The
|
|
131
|
+
-- join is by primary key and only for hit rows; an entry pruned
|
|
132
|
+
-- since contributes nothing.
|
|
133
|
+
LEFT JOIN entries e ON r.cache_hit = 1 AND e.hash = r.hash
|
|
134
|
+
WHERE r.id >= ? AND ${EXECUTED_RUNS_SQL}
|
|
135
|
+
GROUP BY r.project, r.task
|
|
109
136
|
`
|
|
110
|
-
|
|
111
137
|
type Row = {
|
|
112
138
|
project: string
|
|
113
139
|
task: string
|
|
114
140
|
total: number
|
|
115
141
|
successes: number
|
|
116
142
|
hits: number
|
|
117
|
-
executed: number
|
|
118
143
|
failures: number
|
|
119
144
|
retried: number
|
|
145
|
+
ds: string | null
|
|
146
|
+
rss: number | null
|
|
147
|
+
cpu: number | null
|
|
120
148
|
}
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
//
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
149
|
+
// Every pair in the slice is grouped (the slice is bounded, so this is
|
|
150
|
+
// cheap) and the asked-for ones are picked out here — a per-row IN-list
|
|
151
|
+
// probe in SQL cost more than the grouping it would have saved.
|
|
152
|
+
const wanted = new Set(taskIds)
|
|
153
|
+
const rows = (this.db.query(sql).all(floor) as Row[]).filter((r) =>
|
|
154
|
+
wanted.has(`${r.project}#${r.task}`),
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
// The flakiness signal needs per-key outcomes, which the aggregate above
|
|
158
|
+
// folded away — one more pass over the slice, only for the pairs whose
|
|
159
|
+
// verdict actually depends on it (failed, and no retry already proving
|
|
160
|
+
// nondeterminism); a green history never pays for it.
|
|
161
|
+
const mixed = this.mixedOutcomeKeys(
|
|
162
|
+
floor,
|
|
163
|
+
rows.filter((r) => (r.failures || 0) > 0 && (r.retried || 0) === 0),
|
|
164
|
+
)
|
|
129
165
|
|
|
130
166
|
for (const row of rows) {
|
|
131
167
|
const key = `${row.project}#${row.task}`
|
|
132
168
|
const total = row.total || 0
|
|
133
|
-
const
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
retried: row.retried || 0,
|
|
137
|
-
})
|
|
138
|
-
const durations = durationsByTask.get(key)
|
|
169
|
+
const counts = { total, failures: row.failures || 0, retried: row.retried || 0 }
|
|
170
|
+
const durations = row.ds === null ? undefined : row.ds.split(',').map(Number)
|
|
171
|
+
if (durations !== undefined) durations.sort((a, b) => a - b)
|
|
139
172
|
out.set(key, {
|
|
140
173
|
runs: total,
|
|
141
174
|
p50DurationMs: durations ? pickPercentile(durations, 0.5) : undefined,
|
|
142
175
|
p99DurationMs: durations ? pickPercentile(durations, 0.99) : undefined,
|
|
143
176
|
successRate: total > 0 ? (row.successes || 0) / total : 0,
|
|
144
177
|
hitRate: total > 0 ? (row.hits || 0) / total : 0,
|
|
145
|
-
failureMode,
|
|
178
|
+
failureMode: failureModeOf(counts, () => mixed.get(key) ?? 0),
|
|
179
|
+
...(row.rss !== null ? { maxPeakRssBytes: row.rss } : {}),
|
|
180
|
+
...(row.cpu !== null ? { maxCpuParallelism: row.cpu } : {}),
|
|
146
181
|
})
|
|
147
182
|
}
|
|
148
183
|
return out
|
|
149
184
|
}
|
|
150
185
|
|
|
151
|
-
/**
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
const
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
AND status = 'success'
|
|
186
|
+
/** Mixed-outcome key count per `project#task`, over the slice, for `pairs` only. */
|
|
187
|
+
private mixedOutcomeKeys(
|
|
188
|
+
floor: number,
|
|
189
|
+
pairs: readonly { project: string; task: string }[],
|
|
190
|
+
): Map<string, number> {
|
|
191
|
+
const out = new Map<string, number>()
|
|
192
|
+
if (pairs.length === 0) return out
|
|
193
|
+
const placeholders = pairs.map(() => '(?,?)').join(',')
|
|
194
|
+
const rows = this.db
|
|
195
|
+
.query(
|
|
196
|
+
`SELECT project, task, COUNT(*) AS n FROM (${mixedOutcomeKeysSql(
|
|
197
|
+
'runs',
|
|
198
|
+
` AND id >= ? AND (project, task) IN (VALUES ${placeholders})`,
|
|
199
|
+
)}) GROUP BY project, task`,
|
|
166
200
|
)
|
|
167
|
-
|
|
168
|
-
const rows = this.db.query(sql).all(...pairs.flat()) as {
|
|
201
|
+
.all(floor, ...pairs.flatMap((p) => [p.project, p.task])) as {
|
|
169
202
|
project: string
|
|
170
203
|
task: string
|
|
171
|
-
|
|
204
|
+
n: number
|
|
172
205
|
}[]
|
|
173
|
-
const
|
|
174
|
-
for (const r of rows) {
|
|
175
|
-
const key = `${r.project}#${r.task}`
|
|
176
|
-
const list = out.get(key)
|
|
177
|
-
if (list) list.push(r.duration_ms)
|
|
178
|
-
else out.set(key, [r.duration_ms])
|
|
179
|
-
}
|
|
180
|
-
for (const list of out.values()) list.sort((a, b) => a - b)
|
|
206
|
+
for (const r of rows) out.set(`${r.project}#${r.task}`, r.n)
|
|
181
207
|
return out
|
|
182
208
|
}
|
|
183
209
|
}
|
|
184
210
|
|
|
185
|
-
/** Split a `project#task` id on its FIRST `#` (task names contain no `#`). */
|
|
186
|
-
function splitTaskId(id: string): [string, string] {
|
|
187
|
-
const i = id.indexOf('#')
|
|
188
|
-
return i < 0 ? [id, ''] : [id.slice(0, i), id.slice(i + 1)]
|
|
189
|
-
}
|
|
190
|
-
|
|
191
211
|
function pickPercentile(sorted: number[], q: number): number {
|
|
192
212
|
if (sorted.length === 0) return 0
|
|
193
213
|
const idx = Math.min(sorted.length - 1, Math.floor(q * sorted.length))
|