@vzn/vx 0.0.651 → 0.0.652
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -3
- package/package.json +5 -5
- package/schemas/show.json +14 -20
- package/src/bin.ts +12 -0
- package/src/cache/archive.ts +1 -1
- package/src/cache/cache.ts +19 -5
- package/src/cache/history-stream.ts +134 -0
- package/src/cache/history-writer.ts +76 -0
- package/src/cache/index.ts +2 -0
- package/src/cache/inputs.ts +74 -14
- package/src/cache/layer.ts +6 -6
- package/src/cache/lockfile-memo.ts +2 -1
- package/src/cache/run-history.ts +46 -17
- package/src/cache/schema.ts +4 -1
- package/src/cli/cache.ts +2 -2
- package/src/cli/help.ts +2 -2
- package/src/cli/index.ts +24 -0
- package/src/cli/info.ts +1 -1
- package/src/cli/last.ts +34 -2
- package/src/cli/plugin-templates.ts +36 -45
- package/src/cli/show.ts +7 -10
- package/src/cli/watch-judge.ts +1 -1
- package/src/cli/watch-set.ts +4 -4
- package/src/cli/watch.ts +11 -8
- package/src/config.ts +47 -30
- package/src/exec/executor.ts +40 -1
- package/src/exec/index.ts +1 -0
- package/src/exec/sandbox-paths.ts +3 -1
- package/src/exec/sandbox-runtime.ts +314 -29
- package/src/exec/task-proxy.ts +93 -0
- package/src/graph/definition.ts +16 -3
- package/src/graph/scheduler.ts +3 -1
- package/src/graph/task-graph.ts +24 -23
- package/src/history/failure-mode.ts +24 -1
- package/src/history/index.ts +1 -0
- package/src/history/metrics.ts +5 -5
- package/src/index.ts +4 -2
- package/src/orchestrator/admission.ts +1 -1
- package/src/orchestrator/command-windows.ts +1 -1
- package/src/orchestrator/deferred-outputs.ts +1 -1
- package/src/orchestrator/download-policy.ts +5 -7
- package/src/orchestrator/excluded-keys.ts +1 -1
- package/src/orchestrator/execute-task.ts +51 -18
- package/src/orchestrator/hit-restore.ts +5 -6
- package/src/orchestrator/keyed-projects.ts +28 -0
- package/src/orchestrator/local-shortcircuit.ts +2 -2
- package/src/orchestrator/miss-save.ts +27 -8
- package/src/orchestrator/persistent-task.ts +2 -0
- package/src/orchestrator/placement.ts +1 -1
- package/src/orchestrator/plan.ts +2 -2
- package/src/orchestrator/plugin-host.ts +6 -2
- package/src/orchestrator/plugin.ts +6 -0
- package/src/orchestrator/remote-prefetch.ts +1 -1
- package/src/orchestrator/run-failures.ts +61 -12
- package/src/orchestrator/run-keys.ts +1 -1
- package/src/orchestrator/run-records.ts +86 -67
- package/src/orchestrator/run-report.ts +1 -1
- package/src/orchestrator/run.ts +92 -19
- package/src/orchestrator/sandbox-request.ts +50 -7
- package/src/orchestrator/shell-verdict.ts +9 -0
- package/src/orchestrator/stable-keys.ts +17 -18
- package/src/orchestrator/task-artifacts.ts +1 -1
- package/src/orchestrator/task-hash.ts +4 -4
- package/src/orchestrator/telemetry.ts +5 -1
- package/src/report/events.ts +3 -3
- package/src/report/framed-output.ts +16 -7
- package/src/report/logger.ts +1 -1
- package/src/report/plain-output.ts +2 -0
- package/src/report/summary.ts +5 -5
- package/src/settings.ts +8 -8
- package/src/util/config-frame.ts +128 -6
- package/src/util/paths.ts +1 -1
- package/src/util/settle.ts +2 -1
- package/src/util/tail.ts +4 -1
- package/src/workspace/bracket-literal.ts +2 -2
- package/src/workspace/config-schema.ts +66 -61
- package/src/workspace/filter.ts +8 -1
- package/src/workspace/migration.ts +12 -6
|
@@ -63,3 +63,31 @@ export function keyedProjects(
|
|
|
63
63
|
return below.get(root.id)!
|
|
64
64
|
}
|
|
65
65
|
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* A run's lookup of the exec tasks whose outputs a sandboxed task may read:
|
|
69
|
+
* those its key folds, through a group to its members, never past an exec
|
|
70
|
+
* task. Their outputs are what their keys answer for, and this key folds
|
|
71
|
+
* those keys, so a read of them can never replay stale (the toolchain
|
|
72
|
+
* rule: a `tools` project builds Node, every dependant runs it). A
|
|
73
|
+
* dependency `inputs.tasks` leaves out is ordered, not read.
|
|
74
|
+
*/
|
|
75
|
+
export function readableDeps(
|
|
76
|
+
nodes: ReadonlyMap<string, TaskNode>,
|
|
77
|
+
keyOnly: ReadonlyMap<string, TaskNode> = new Map(),
|
|
78
|
+
): (node: TaskNode) => readonly TaskNode[] {
|
|
79
|
+
const nodeOf = (id: string): TaskNode => nodes.get(id) ?? keyOnly.get(id)!
|
|
80
|
+
return (root) => {
|
|
81
|
+
const out = new Map<string, TaskNode>()
|
|
82
|
+
const seen = new Set<string>()
|
|
83
|
+
const stack = foldedDeps(root, nodeOf).map((c) => c.node)
|
|
84
|
+
while (stack.length > 0) {
|
|
85
|
+
const dep = stack.pop()!
|
|
86
|
+
if (seen.has(dep.id)) continue
|
|
87
|
+
seen.add(dep.id)
|
|
88
|
+
if (!isGroupTask(dep)) out.set(dep.id, dep)
|
|
89
|
+
else for (const { node } of foldedDeps(dep, nodeOf)) stack.push(node)
|
|
90
|
+
}
|
|
91
|
+
return [...out.values()]
|
|
92
|
+
}
|
|
93
|
+
}
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
// - Only STABLE-key, cacheable, local-read tasks are classified. A
|
|
33
33
|
// stable miss stays in the normal (dep-gated) schedule; an unstable
|
|
34
34
|
// task is never probed here.
|
|
35
|
-
// - A task that declares `
|
|
35
|
+
// - A task that declares `outputs.workspaceFiles` (the
|
|
36
36
|
// boundary-ignoring escape hatch) can write where a restore touches,
|
|
37
37
|
// so every task whose project directory a workspace-output glob's
|
|
38
38
|
// static prefix REACHES stays out of the restore tier, and so does
|
|
@@ -261,7 +261,7 @@ export function restoreTierExclusions(
|
|
|
261
261
|
const prefixes: string[] = []
|
|
262
262
|
let everything = false
|
|
263
263
|
for (const node of nodes.values()) {
|
|
264
|
-
for (const raw of node.config.
|
|
264
|
+
for (const raw of node.config.outputs?.workspaceFiles ?? []) {
|
|
265
265
|
// A `!` entry needs no skip: its prefix (`!gen`) names no directory,
|
|
266
266
|
// so it excludes nothing (A-44; the `continue` that stood here was
|
|
267
267
|
// dropped in item 640).
|
|
@@ -109,7 +109,7 @@ export interface SaveMissArgs {
|
|
|
109
109
|
workspaceRoot: string
|
|
110
110
|
nestedProjectDirs: string[]
|
|
111
111
|
gitFilesCache?: GitFilesCache | undefined
|
|
112
|
-
/** Declared `
|
|
112
|
+
/** Declared `outputs.files` / `.workspaceFiles`. */
|
|
113
113
|
outputs: readonly string[]
|
|
114
114
|
wsOutputs: readonly string[]
|
|
115
115
|
/**
|
|
@@ -119,7 +119,7 @@ export interface SaveMissArgs {
|
|
|
119
119
|
* would also take the upstream's files the task adds beside.
|
|
120
120
|
*/
|
|
121
121
|
ownOutputFiles?: string[] | undefined
|
|
122
|
-
/** `ownOutputFiles` for `
|
|
122
|
+
/** `ownOutputFiles` for `outputs.workspaceFiles` (A-43). */
|
|
123
123
|
ownWsOutputFiles?: string[] | undefined
|
|
124
124
|
/** The Tier-3 input fingerprint rows captured by the pre-exec describe. */
|
|
125
125
|
captured: readonly TaskInputComponent[]
|
|
@@ -162,19 +162,38 @@ export async function saveMiss(a: SaveMissArgs): Promise<{ landed: Promise<SaveF
|
|
|
162
162
|
// An additive task owns only the files it wrote (item 588), no directory.
|
|
163
163
|
const tree =
|
|
164
164
|
a.ownOutputFiles !== undefined
|
|
165
|
-
? { files: a.ownOutputFiles, emptyDirs: [] }
|
|
165
|
+
? { files: a.ownOutputFiles, emptyDirs: [], special: [] }
|
|
166
166
|
: await resolveOutputTree({
|
|
167
167
|
projectDir: node.projectDir,
|
|
168
168
|
outputs: a.outputs,
|
|
169
169
|
nestedProjectDirs: a.nestedProjectDirs,
|
|
170
|
+
special: true,
|
|
170
171
|
})
|
|
171
172
|
const outputFiles = tree.files
|
|
172
173
|
const wsTree =
|
|
173
174
|
a.ownWsOutputFiles !== undefined
|
|
174
|
-
? { files: a.ownWsOutputFiles, emptyDirs: [] }
|
|
175
|
-
: await resolveWorkspaceOutputTree({
|
|
175
|
+
? { files: a.ownWsOutputFiles, emptyDirs: [], special: [] }
|
|
176
|
+
: await resolveWorkspaceOutputTree({
|
|
177
|
+
workspaceRoot: a.workspaceRoot,
|
|
178
|
+
outputs: a.wsOutputs,
|
|
179
|
+
special: true,
|
|
180
|
+
})
|
|
176
181
|
const wsOutputFiles = wsTree.files
|
|
177
182
|
endResolve()
|
|
183
|
+
// A FIFO, socket or device holds no bytes to store (opening a FIFO waits
|
|
184
|
+
// for a writer, nx#35228), so it stays out of the entry; said, because a
|
|
185
|
+
// hit restores the tree without it.
|
|
186
|
+
const special = [
|
|
187
|
+
...tree.special.map((p) => relPosix(node.projectDir, p)),
|
|
188
|
+
...wsTree.special.map((p) => relPosix(a.workspaceRoot, p)),
|
|
189
|
+
]
|
|
190
|
+
if (special.length > 0) {
|
|
191
|
+
warnOf(log)(
|
|
192
|
+
`[vx] ${node.id}: ${special.join(', ')} ${special.length === 1 ? 'is' : 'are'} not a regular file or symlink ` +
|
|
193
|
+
`(a FIFO, socket or device): vx does not cache ${special.length === 1 ? 'it' : 'them'}, so a hit restores the outputs without ${special.length === 1 ? 'it' : 'them'} ` +
|
|
194
|
+
`— emit a file there, or take ${special.length === 1 ? 'it' : 'them'} back with a '!' entry`,
|
|
195
|
+
)
|
|
196
|
+
}
|
|
178
197
|
// The mirror of the input warning: declared outputs that resolve to
|
|
179
198
|
// nothing save an empty artifact, and the next hit "restores" it —
|
|
180
199
|
// the build that ran nowhere looks like a build that ran. `files: []`
|
|
@@ -191,14 +210,14 @@ export async function saveMiss(a: SaveMissArgs): Promise<{ landed: Promise<SaveF
|
|
|
191
210
|
// (project dir === workspace root) the write SUCCEEDS into that
|
|
192
211
|
// scratch and the task exits 0, so this warning is the only thing that
|
|
193
212
|
// says the build produced nothing (item 444). A declared
|
|
194
|
-
// `
|
|
213
|
+
// `outputs` is not a write grant; `exec.sandbox.allow.write` is.
|
|
195
214
|
const sandboxed = node.config.exec?.sandbox !== undefined
|
|
196
215
|
const grantsWrite = (node.config.exec?.sandbox?.allow?.write?.length ?? 0) > 0
|
|
197
216
|
// A `workspaceFiles` directory linked out of the workspace drops every
|
|
198
217
|
// file (M-65); a project one is refused by the resolver (X-88).
|
|
199
218
|
const linkedOut = outputDirLinkedOut(a.workspaceRoot, a.wsOutputs)
|
|
200
219
|
warnOf(log)(
|
|
201
|
-
`[vx] ${node.id}:
|
|
220
|
+
`[vx] ${node.id}: outputs matched no files (${[...a.outputs, ...a.wsOutputs].join(', ')}) — ` +
|
|
202
221
|
`an empty artifact is saved; a later hit restores nothing` +
|
|
203
222
|
(linkedOut !== undefined
|
|
204
223
|
? ` — ${linkedOut.dir} is a symlink to ${linkedOut.target}, outside ${linkedOut.base}, and vx keeps only outputs inside it: make ${linkedOut.dir} a directory`
|
|
@@ -347,7 +366,7 @@ function sizeOf(base: string, files: readonly string[]): number {
|
|
|
347
366
|
* these paths. On a 1000-package cold run this removes ~one synchronous
|
|
348
367
|
* `git ls-files` spawn per project (the single largest cold-run cost —
|
|
349
368
|
* 22% of CPU in profiling). Contract: outputs must be declared — an
|
|
350
|
-
* executed task that writes files outside `
|
|
369
|
+
* executed task that writes files outside `outputs.files` which a
|
|
351
370
|
* same-project downstream task reads is undeclared behavior (the restore
|
|
352
371
|
* path already assumes it).
|
|
353
372
|
*/
|
|
@@ -19,6 +19,7 @@ import { warnOf } from '../report/index.js'
|
|
|
19
19
|
import type { ExecuteArgs } from './execute-task.js'
|
|
20
20
|
import {
|
|
21
21
|
commandWriteReach,
|
|
22
|
+
dependencyReads,
|
|
22
23
|
mayWriteFingerprint,
|
|
23
24
|
type Placeholder,
|
|
24
25
|
placeholderSweeper,
|
|
@@ -135,6 +136,7 @@ export async function executePersistentTask(args: ExecuteArgs): Promise<TaskOutc
|
|
|
135
136
|
undefined,
|
|
136
137
|
args.nestedProjectDirs,
|
|
137
138
|
args.cacheDir,
|
|
139
|
+
await dependencyReads(args.readableDeps?.(node) ?? [], args.workspaceRoot),
|
|
138
140
|
)
|
|
139
141
|
placeholders = sb.placeholders
|
|
140
142
|
serverSandbox = sb.sandbox
|
|
@@ -337,7 +337,7 @@ export function placeTasks(
|
|
|
337
337
|
projectDir: node.projectDir,
|
|
338
338
|
command: node.config.exec!.command,
|
|
339
339
|
pinnedLocal: pinned.has(node.id),
|
|
340
|
-
cacheable: node.config.cache
|
|
340
|
+
cacheable: node.config.cache === true,
|
|
341
341
|
},
|
|
342
342
|
executorLabel,
|
|
343
343
|
(e) => pinsOf(e).has(node.id) || !routes(e),
|
package/src/orchestrator/plan.ts
CHANGED
|
@@ -25,7 +25,7 @@ export type CacheStatus =
|
|
|
25
25
|
| 'hit-local' // entry exists in local cache
|
|
26
26
|
| 'hit-remote' // entry exists in the remote layer (would be fetched)
|
|
27
27
|
| 'miss' // caching enabled but no entry — would execute
|
|
28
|
-
| 'no-cache' // task opts out of caching (no `cache`
|
|
28
|
+
| 'no-cache' // task opts out of caching (no `cache: true`) or --no-cache
|
|
29
29
|
| 'group' // no `exec`; just an aggregator
|
|
30
30
|
| 'not-started' // a server every dependant of which restores early: the run never spawns it
|
|
31
31
|
| 'skipped' // `exec.platforms` leave this machine out, or a task it depends on is skipped
|
|
@@ -186,7 +186,7 @@ export async function plan(args: PlanArgs): Promise<RunPlan> {
|
|
|
186
186
|
// so nothing is probed. Whether that run is a miss or no cache at all
|
|
187
187
|
// is the WRITE axes': `--force` and `--cache-policy=local:w` save what they
|
|
188
188
|
// run, and the plan called them `no-cache`, the label a task with no
|
|
189
|
-
// `cache`
|
|
189
|
+
// `cache: true` gets (item 991). The probe below routes through the
|
|
190
190
|
// policy-aware cache layer, which itself respects read gating.
|
|
191
191
|
const policy = args.cachePolicy ?? FULL_CACHE_POLICY
|
|
192
192
|
const reads = policy.localRead || policy.remoteRead
|
|
@@ -521,6 +521,8 @@ export async function claimedExtraFiles(
|
|
|
521
521
|
export interface ScheduleRun {
|
|
522
522
|
readonly step?: <T>(id: string, work: () => Promise<T>) => Promise<T>
|
|
523
523
|
readonly refined?: (merged: ReadonlyMap<string, number>) => void
|
|
524
|
+
/** Where `ctx.flaky` names go; none in a plan. */
|
|
525
|
+
readonly flaky?: (taskIds: Iterable<string>) => void
|
|
524
526
|
}
|
|
525
527
|
|
|
526
528
|
/**
|
|
@@ -533,7 +535,7 @@ export interface ScheduleRun {
|
|
|
533
535
|
export async function applyScheduleHooks(
|
|
534
536
|
plugins: readonly VxPlugin[],
|
|
535
537
|
nodes: ReadonlyMap<string, TaskNode>,
|
|
536
|
-
ctx: Omit<ScheduleHookContext, 'cache' | 'refine' | 'onRefine' | 'refining'>,
|
|
538
|
+
ctx: Omit<ScheduleHookContext, 'cache' | 'refine' | 'onRefine' | 'refining' | 'flaky'>,
|
|
537
539
|
cacheFor: (plugin: string) => PluginCache = () => absentPluginCache,
|
|
538
540
|
run: ScheduleRun = {},
|
|
539
541
|
): Promise<ReadonlyMap<string, number>> {
|
|
@@ -586,7 +588,8 @@ export async function applyScheduleHooks(
|
|
|
586
588
|
const onRefine = (listener: () => void): void => {
|
|
587
589
|
if (run.refined !== undefined) listeners.push({ plugin, i, listener })
|
|
588
590
|
}
|
|
589
|
-
const
|
|
591
|
+
const flaky = (taskIds: Iterable<string>): void => run.flaky?.(taskIds)
|
|
592
|
+
const own = { ...ctx, cache: cacheFor(plugin.name), refine, onRefine, refining, flaky }
|
|
590
593
|
const call = (): Promise<ReadonlyMap<string, number> | undefined> =>
|
|
591
594
|
safe(plugin, 'schedule', () => plugin.schedule!(nodes, own))
|
|
592
595
|
const weights = await (run.step === undefined
|
|
@@ -888,6 +891,7 @@ const EXECUTOR_HOOKS = [
|
|
|
888
891
|
'footer',
|
|
889
892
|
'machines',
|
|
890
893
|
'idealMs',
|
|
894
|
+
'advice',
|
|
891
895
|
'profile',
|
|
892
896
|
'running',
|
|
893
897
|
] as const
|
|
@@ -603,6 +603,12 @@ export interface ScheduleHookContext extends BaseContext {
|
|
|
603
603
|
* in a plan.
|
|
604
604
|
*/
|
|
605
605
|
onRefine(listener: () => void): void
|
|
606
|
+
/**
|
|
607
|
+
* Name tasks a shared record knows as flaky (a history server's): one
|
|
608
|
+
* that fails gets the one retry a known flake gets, as the local
|
|
609
|
+
* record's do. Ids not in the run are ignored; ignored in a plan.
|
|
610
|
+
*/
|
|
611
|
+
flaky(taskIds: Iterable<string>): void
|
|
606
612
|
}
|
|
607
613
|
|
|
608
614
|
/** A `cache` hook's context: the workspace and the run's scope, never the local store or the policy, which core owns. */
|
|
@@ -231,7 +231,7 @@ function fetchPool(
|
|
|
231
231
|
* lookup, the other one that can reach a remote.
|
|
232
232
|
*/
|
|
233
233
|
export function getContext(node: TaskNode, command: string) {
|
|
234
|
-
const outputs = node.config.
|
|
234
|
+
const outputs = node.config.outputs
|
|
235
235
|
let masked: string | undefined
|
|
236
236
|
return {
|
|
237
237
|
taskId: node.id,
|
|
@@ -46,17 +46,16 @@ export function writeRunFailures(
|
|
|
46
46
|
const failures = []
|
|
47
47
|
for (const o of outcomes) {
|
|
48
48
|
// A task failed by a moved input may have printed nothing: the input
|
|
49
|
-
// is still why, and an agent reads it here.
|
|
50
|
-
|
|
49
|
+
// is still why, and an agent reads it here. A pass that took a retry
|
|
50
|
+
// keeps its failed attempts.
|
|
51
|
+
if (
|
|
52
|
+
(o.failedOutput === undefined &&
|
|
53
|
+
o.inputMoved === undefined &&
|
|
54
|
+
o.attemptOutputs === undefined) ||
|
|
55
|
+
isGroupTask(o.node)
|
|
56
|
+
)
|
|
51
57
|
continue
|
|
52
|
-
const output =
|
|
53
|
-
o.failedOutput === undefined
|
|
54
|
-
? ''
|
|
55
|
-
: Bun.stripANSI(
|
|
56
|
-
decodeOutputLog(o.failedOutput)
|
|
57
|
-
.map((c) => c.text)
|
|
58
|
-
.join(''),
|
|
59
|
-
)
|
|
58
|
+
const output = attemptsText(o)
|
|
60
59
|
failures.push({
|
|
61
60
|
taskId: o.node.id,
|
|
62
61
|
...(o.inputMoved !== undefined ? { inputMoved: o.inputMoved } : {}),
|
|
@@ -74,6 +73,40 @@ export function writeRunFailures(
|
|
|
74
73
|
}
|
|
75
74
|
}
|
|
76
75
|
|
|
76
|
+
const plain = (log: string): string =>
|
|
77
|
+
Bun.stripANSI(
|
|
78
|
+
decodeOutputLog(log)
|
|
79
|
+
.map((c) => c.text)
|
|
80
|
+
.join(''),
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
/** How attempt `n` of `of` ended, the line above its output. */
|
|
84
|
+
const attemptHeading = (n: number, of: number, how: string): string =>
|
|
85
|
+
`── attempt ${n} of ${of} · ${how} ──\n`
|
|
86
|
+
|
|
87
|
+
const ended = (a: { exitCode: number; timedOut?: true }): string =>
|
|
88
|
+
a.timedOut === true ? 'timed out' : `exit ${a.exitCode}`
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* What a failure row keeps of `o`: its output, after each failed attempt's
|
|
92
|
+
* under a heading when it retried. A pass that retried keeps only the
|
|
93
|
+
* failed attempts: its own output is its cache entry's.
|
|
94
|
+
*/
|
|
95
|
+
function attemptsText(o: TaskOutcome): string {
|
|
96
|
+
const last = o.failedOutput === undefined ? '' : plain(o.failedOutput)
|
|
97
|
+
const before = o.attemptOutputs ?? []
|
|
98
|
+
if (before.length === 0) return last
|
|
99
|
+
const of = before.length + 1
|
|
100
|
+
let text = ''
|
|
101
|
+
before.forEach((log, i) => {
|
|
102
|
+
const out = plain(log)
|
|
103
|
+
text += attemptHeading(i + 1, of, ended(o.failedAttempts![i]!)) + out
|
|
104
|
+
if (out !== '' && !out.endsWith('\n')) text += '\n'
|
|
105
|
+
})
|
|
106
|
+
if (o.status === 'failed') text += attemptHeading(of, of, ended(o)) + last
|
|
107
|
+
return text
|
|
108
|
+
}
|
|
109
|
+
|
|
77
110
|
/**
|
|
78
111
|
* The failed tasks of `runId`, or of the latest failed run when omitted;
|
|
79
112
|
* null when there is no such run. A failed task with no kept row (a run
|
|
@@ -167,6 +200,19 @@ export function taskLog(db: Database, taskId: string, runId?: string): TaskLog |
|
|
|
167
200
|
const kept = keptFailures(db, row.runId).get(taskId)
|
|
168
201
|
if (kept !== undefined) return { ...base, source: 'failure', output: kept.output }
|
|
169
202
|
} else if (row.hash !== '') {
|
|
203
|
+
// A pass that took a retry: its failed attempts, then its own output.
|
|
204
|
+
const before = keptFailures(db, row.runId).get(taskId)?.output
|
|
205
|
+
const tries = (
|
|
206
|
+
db
|
|
207
|
+
.query(
|
|
208
|
+
'SELECT attempts FROM runs WHERE run_id = ? AND project = ? AND task = ? ORDER BY id DESC LIMIT 1',
|
|
209
|
+
)
|
|
210
|
+
.get(row.runId, project, task) as { attempts: number | null } | null
|
|
211
|
+
)?.attempts
|
|
212
|
+
const prefix =
|
|
213
|
+
before === undefined || tries === null || tries === undefined
|
|
214
|
+
? ''
|
|
215
|
+
: `${before}${attemptHeading(tries, tries, 'passed')}`
|
|
170
216
|
const stored = db.query('SELECT stdout FROM entry_stdout WHERE hash = ?').get(row.hash) as {
|
|
171
217
|
stdout: string
|
|
172
218
|
} | null
|
|
@@ -182,12 +228,15 @@ export function taskLog(db: Database, taskId: string, runId?: string): TaskLog |
|
|
|
182
228
|
.join(''),
|
|
183
229
|
),
|
|
184
230
|
)
|
|
185
|
-
return { ...base, source: 'cache', output }
|
|
231
|
+
return { ...base, source: 'cache', output: prefix + output }
|
|
186
232
|
}
|
|
187
233
|
// An entry that printed nothing stores no stdout row: kept, and empty.
|
|
188
234
|
if (db.query('SELECT 1 FROM entries WHERE hash = ?').get(row.hash) !== null) {
|
|
189
|
-
return { ...base, source: 'cache', output:
|
|
235
|
+
return { ...base, source: 'cache', output: prefix }
|
|
190
236
|
}
|
|
191
237
|
}
|
|
238
|
+
// A pass with no entry that took a retry still kept its failed attempts.
|
|
239
|
+
const kept = row.status === 'failed' ? undefined : keptFailures(db, row.runId).get(taskId)
|
|
240
|
+
if (kept !== undefined) return { ...base, source: 'failure', output: kept.output }
|
|
192
241
|
return { ...base, source: null, output: '' }
|
|
193
242
|
}
|
|
@@ -89,7 +89,7 @@ export function runKeysToken(args: RunKeysArgs): RunKeysAt | undefined {
|
|
|
89
89
|
if ((inputs.workspaceFiles?.length ?? 0) > 0) return undefined
|
|
90
90
|
// No positive entry resolves no files (`filesPlan`).
|
|
91
91
|
if (inputs.files === undefined || inputs.files.some((e) => !e.startsWith('!'))) {
|
|
92
|
-
const outputs = cfg.
|
|
92
|
+
const outputs = cfg.outputs
|
|
93
93
|
const declared = `${node.projectName}\0${JSON.stringify([inputs.files, outputs?.files, outputs?.workspaceFiles])}`
|
|
94
94
|
let token = tokens.get(declared)
|
|
95
95
|
if (token === undefined) {
|
|
@@ -42,8 +42,16 @@ export interface RunRecordsInput {
|
|
|
42
42
|
blocks?: ReadonlyMap<string, number>
|
|
43
43
|
/** The run's remote failures by key, so far (`Cache.traffic`). */
|
|
44
44
|
remoteTraffic?: ReadonlyMap<string, { failures: readonly string[] }>
|
|
45
|
+
/** Outcomes whose rows the history stream already wrote: counted, not written again. */
|
|
46
|
+
landed?: ReadonlySet<TaskOutcome>
|
|
45
47
|
}
|
|
46
48
|
|
|
49
|
+
/** What a task's `runs` row reads besides its outcome. */
|
|
50
|
+
export type RunRecordContext = Pick<
|
|
51
|
+
RunRecordsInput,
|
|
52
|
+
'tainted' | 'runId' | 'startedAtMs' | 'endedAtMs' | 'forwardArgs' | 'blocks' | 'remoteTraffic'
|
|
53
|
+
>
|
|
54
|
+
|
|
47
55
|
export interface RunRecords {
|
|
48
56
|
runs: RunRecord[]
|
|
49
57
|
invocation: InvocationRecord
|
|
@@ -72,6 +80,7 @@ export function assembleRunRecords(input: RunRecordsInput): RunRecords {
|
|
|
72
80
|
const { outcomes, runId, startedAtMs, endedAtMs } = input
|
|
73
81
|
const runs: RunRecord[] = []
|
|
74
82
|
const telemetryTasks: TaskTelemetry[] = []
|
|
83
|
+
let taskCount = 0
|
|
75
84
|
let failedCount = 0
|
|
76
85
|
let hitLocalCount = 0
|
|
77
86
|
let hitRemoteCount = 0
|
|
@@ -83,72 +92,8 @@ export function assembleRunRecords(input: RunRecordsInput): RunRecords {
|
|
|
83
92
|
// aborted (killed by a shutdown signal) isn't a real run.
|
|
84
93
|
if (o.status === 'aborted') continue
|
|
85
94
|
if (input.withTelemetry) telemetryTasks.push(taskTelemetryOf(o))
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
// Built field by field: a spread per optional field was ~2 µs of each
|
|
89
|
-
// of a 1,200-hit warm run's rows.
|
|
90
|
-
const hit = isCacheHit(o.status)
|
|
91
|
-
const r: RunRecord = {
|
|
92
|
-
project: o.node.projectName,
|
|
93
|
-
task: o.node.taskName,
|
|
94
|
-
status: o.status,
|
|
95
|
-
exitCode: o.exitCode,
|
|
96
|
-
durationMs: o.durationMs,
|
|
97
|
-
// Anchor to the REAL per-task wall-clock window: run-start wall time +
|
|
98
|
-
// the task's ns offset (captured for hits and executed tasks alike).
|
|
99
|
-
// The `end - duration` fallback applies to outcomes without an offset —
|
|
100
|
-
// today only `skipped`, which the scheduler finishes synchronously with
|
|
101
|
-
// no span, so it collapses to a zero-width mark at the run's end. Using
|
|
102
|
-
// run-end-minus-duration for EVERYTHING was the old bug that piled every
|
|
103
|
-
// task at the right edge of the timeline.
|
|
104
|
-
startedAt:
|
|
105
|
-
o.wallclockStartNs !== undefined
|
|
106
|
-
? startedAtMs + Math.round(Number(o.wallclockStartNs) / 1e6)
|
|
107
|
-
: endedAtMs - o.durationMs,
|
|
108
|
-
endedAt:
|
|
109
|
-
o.wallclockEndNs !== undefined
|
|
110
|
-
? startedAtMs + Math.round(Number(o.wallclockEndNs) / 1e6)
|
|
111
|
-
: endedAtMs,
|
|
112
|
-
runId,
|
|
113
|
-
cacheHit: hit,
|
|
114
|
-
cached: o.node.config.cache !== undefined,
|
|
115
|
-
}
|
|
116
|
-
if (o.hash !== undefined) r.hash = o.hash
|
|
117
|
-
// A run forwards its args to the tasks it was asked for, never to a
|
|
118
|
-
// dependency: the row says which ran with them.
|
|
119
|
-
if (input.forwardArgs !== undefined && input.forwardArgs.length > 0 && o.node.requested)
|
|
120
|
-
r.forwardArgs = input.forwardArgs
|
|
121
|
-
if (o.cpuMs !== undefined) r.cpuMs = o.cpuMs
|
|
122
|
-
if (o.peakRssBytes !== undefined) r.peakRssBytes = o.peakRssBytes
|
|
123
|
-
if (o.peakCores !== undefined) r.peakCores = o.peakCores
|
|
124
|
-
if (o.avgRssBytes !== undefined) r.avgRssBytes = o.avgRssBytes
|
|
125
|
-
if (o.where !== undefined) r.placement = o.where
|
|
126
|
-
if (o.whereCores !== undefined) r.placementCores = o.whereCores
|
|
127
|
-
if (o.whereMemoryBytes !== undefined) r.placementMemoryBytes = o.whereMemoryBytes
|
|
128
|
-
if (o.cpuShare !== undefined) r.cpuShare = o.cpuShare
|
|
129
|
-
if (o.queuedMs !== undefined) r.queuedMs = o.queuedMs
|
|
130
|
-
if (o.outputBytes !== undefined) r.outputBytes = o.outputBytes
|
|
131
|
-
const blocks = input.blocks?.get(o.node.id)
|
|
132
|
-
if (blocks !== undefined) r.blocks = blocks
|
|
133
|
-
if (o.flaky !== undefined) {
|
|
134
|
-
r.flakyPasses = o.flaky.passes
|
|
135
|
-
r.flakyFailures = o.flaky.failures
|
|
136
|
-
}
|
|
137
|
-
if (o.wallclockStartNs !== undefined) r.wallclockStartNs = o.wallclockStartNs
|
|
138
|
-
if (o.wallclockEndNs !== undefined) r.wallclockEndNs = o.wallclockEndNs
|
|
139
|
-
if (hit) r.restored = o.restored === true
|
|
140
|
-
if (o.attempts !== undefined) r.attempts = o.attempts
|
|
141
|
-
// A task that ran behind a failure names it too, so the flakiness
|
|
142
|
-
// projection counts its outcome on no key (failure-mode.ts).
|
|
143
|
-
if (blockedBy !== undefined) r.blockedBy = blockedBy
|
|
144
|
-
if (o.timedOut === true) r.timedOut = true
|
|
145
|
-
if (o.sandboxViolations !== undefined) r.sandboxViolations = o.sandboxViolations
|
|
146
|
-
if (o.notReady !== undefined) r.notReady = o.notReady
|
|
147
|
-
if (remoteError !== undefined) r.remoteError = remoteError
|
|
148
|
-
if (o.keylessBy !== undefined) r.keylessBy = o.keylessBy
|
|
149
|
-
const def = taskDefinition(o.node)
|
|
150
|
-
if (def !== undefined) r.def = def
|
|
151
|
-
runs.push(r)
|
|
95
|
+
taskCount++
|
|
96
|
+
if (input.landed?.has(o) !== true) runs.push(runRecordOf(o as RecordedOutcome, input))
|
|
152
97
|
if (o.status === 'failed') failedCount++
|
|
153
98
|
if (o.status === 'cache-hit') hitLocalCount++
|
|
154
99
|
if (o.status === 'cache-hit-remote') hitRemoteCount++
|
|
@@ -168,7 +113,7 @@ export function assembleRunRecords(input: RunRecordsInput): RunRecords {
|
|
|
168
113
|
startedAt: startedAtMs,
|
|
169
114
|
endedAt: endedAtMs,
|
|
170
115
|
totalDurationMs: Math.round(input.totalMs),
|
|
171
|
-
taskCount
|
|
116
|
+
taskCount,
|
|
172
117
|
failedCount,
|
|
173
118
|
hitCount: hitLocalCount + hitRemoteCount,
|
|
174
119
|
hitLocalCount,
|
|
@@ -190,6 +135,80 @@ export function assembleRunRecords(input: RunRecordsInput): RunRecords {
|
|
|
190
135
|
return { runs, invocation, telemetryTasks }
|
|
191
136
|
}
|
|
192
137
|
|
|
138
|
+
/** An outcome a `runs` row records: every status but `aborted`. */
|
|
139
|
+
export type RecordedOutcome = TaskOutcome & { status: RunRecord['status'] }
|
|
140
|
+
|
|
141
|
+
/** One outcome's `runs` row. */
|
|
142
|
+
export function runRecordOf(o: RecordedOutcome, input: RunRecordContext): RunRecord {
|
|
143
|
+
const { runId, startedAtMs, endedAtMs } = input
|
|
144
|
+
const blockedBy = o.blockedBy ?? input.tainted?.get(o.node.id)
|
|
145
|
+
const remoteError = remoteErrorOf(input.remoteTraffic, o.hash)
|
|
146
|
+
// Built field by field: a spread per optional field was ~2 µs of each
|
|
147
|
+
// of a 1,200-hit warm run's rows.
|
|
148
|
+
const hit = isCacheHit(o.status)
|
|
149
|
+
const r: RunRecord = {
|
|
150
|
+
project: o.node.projectName,
|
|
151
|
+
task: o.node.taskName,
|
|
152
|
+
status: o.status,
|
|
153
|
+
exitCode: o.exitCode,
|
|
154
|
+
durationMs: o.durationMs,
|
|
155
|
+
// Anchor to the REAL per-task wall-clock window: run-start wall time +
|
|
156
|
+
// the task's ns offset (captured for hits and executed tasks alike).
|
|
157
|
+
// The `end - duration` fallback applies to outcomes without an offset —
|
|
158
|
+
// today only `skipped`, which the scheduler finishes synchronously with
|
|
159
|
+
// no span, so it collapses to a zero-width mark at the run's end. Using
|
|
160
|
+
// run-end-minus-duration for EVERYTHING was the old bug that piled every
|
|
161
|
+
// task at the right edge of the timeline.
|
|
162
|
+
startedAt:
|
|
163
|
+
o.wallclockStartNs !== undefined
|
|
164
|
+
? startedAtMs + Math.round(Number(o.wallclockStartNs) / 1e6)
|
|
165
|
+
: endedAtMs - o.durationMs,
|
|
166
|
+
endedAt:
|
|
167
|
+
o.wallclockEndNs !== undefined
|
|
168
|
+
? startedAtMs + Math.round(Number(o.wallclockEndNs) / 1e6)
|
|
169
|
+
: endedAtMs,
|
|
170
|
+
runId,
|
|
171
|
+
cacheHit: hit,
|
|
172
|
+
cached: o.node.config.cache === true,
|
|
173
|
+
}
|
|
174
|
+
if (o.hash !== undefined) r.hash = o.hash
|
|
175
|
+
// A run forwards its args to the tasks it was asked for, never to a
|
|
176
|
+
// dependency: the row says which ran with them.
|
|
177
|
+
if (input.forwardArgs !== undefined && input.forwardArgs.length > 0 && o.node.requested)
|
|
178
|
+
r.forwardArgs = input.forwardArgs
|
|
179
|
+
if (o.cpuMs !== undefined) r.cpuMs = o.cpuMs
|
|
180
|
+
if (o.peakRssBytes !== undefined) r.peakRssBytes = o.peakRssBytes
|
|
181
|
+
if (o.peakCores !== undefined) r.peakCores = o.peakCores
|
|
182
|
+
if (o.avgRssBytes !== undefined) r.avgRssBytes = o.avgRssBytes
|
|
183
|
+
if (o.where !== undefined) r.placement = o.where
|
|
184
|
+
if (o.whereCores !== undefined) r.placementCores = o.whereCores
|
|
185
|
+
if (o.whereMemoryBytes !== undefined) r.placementMemoryBytes = o.whereMemoryBytes
|
|
186
|
+
if (o.cpuShare !== undefined) r.cpuShare = o.cpuShare
|
|
187
|
+
if (o.queuedMs !== undefined) r.queuedMs = o.queuedMs
|
|
188
|
+
if (o.outputBytes !== undefined) r.outputBytes = o.outputBytes
|
|
189
|
+
const blocks = input.blocks?.get(o.node.id)
|
|
190
|
+
if (blocks !== undefined) r.blocks = blocks
|
|
191
|
+
if (o.flaky !== undefined) {
|
|
192
|
+
r.flakyPasses = o.flaky.passes
|
|
193
|
+
r.flakyFailures = o.flaky.failures
|
|
194
|
+
}
|
|
195
|
+
if (o.wallclockStartNs !== undefined) r.wallclockStartNs = o.wallclockStartNs
|
|
196
|
+
if (o.wallclockEndNs !== undefined) r.wallclockEndNs = o.wallclockEndNs
|
|
197
|
+
if (hit) r.restored = o.restored === true
|
|
198
|
+
if (o.attempts !== undefined) r.attempts = o.attempts
|
|
199
|
+
// A task that ran behind a failure names it too, so the flakiness
|
|
200
|
+
// projection counts its outcome on no key (failure-mode.ts).
|
|
201
|
+
if (blockedBy !== undefined) r.blockedBy = blockedBy
|
|
202
|
+
if (o.timedOut === true) r.timedOut = true
|
|
203
|
+
if (o.sandboxViolations !== undefined) r.sandboxViolations = o.sandboxViolations
|
|
204
|
+
if (o.notReady !== undefined) r.notReady = o.notReady
|
|
205
|
+
if (remoteError !== undefined) r.remoteError = remoteError
|
|
206
|
+
if (o.keylessBy !== undefined) r.keylessBy = o.keylessBy
|
|
207
|
+
const def = taskDefinition(o.node)
|
|
208
|
+
if (def !== undefined) r.def = def
|
|
209
|
+
return r
|
|
210
|
+
}
|
|
211
|
+
|
|
193
212
|
/**
|
|
194
213
|
* The remote failures uploads met after the run's rows went in (the uploads
|
|
195
214
|
* drain later): each row whose text is now short of its key's is rewritten.
|
|
@@ -49,7 +49,7 @@ async function outputsLeft(
|
|
|
49
49
|
workspaceRoot: string,
|
|
50
50
|
nestedProjectDirs: string[],
|
|
51
51
|
): Promise<string[]> {
|
|
52
|
-
const declared = node.config.
|
|
52
|
+
const declared = node.config.outputs
|
|
53
53
|
if (declared === undefined) return []
|
|
54
54
|
const [own, ws] = await Promise.all([
|
|
55
55
|
resolveOutputs({ projectDir: node.projectDir, outputs: declared.files, nestedProjectDirs }),
|