@vzn/vx 0.0.650 → 0.0.652

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. package/README.md +7 -23
  2. package/package.json +5 -5
  3. package/schemas/last.json +15 -0
  4. package/schemas/show.json +14 -20
  5. package/src/bin.ts +21 -31
  6. package/src/cache/archive.ts +1 -1
  7. package/src/cache/cache.ts +221 -30
  8. package/src/cache/file-hashes.ts +3 -0
  9. package/src/cache/git-inputs.ts +188 -39
  10. package/src/cache/history-stream.ts +134 -0
  11. package/src/cache/history-writer.ts +76 -0
  12. package/src/cache/index.ts +3 -0
  13. package/src/cache/inputs.ts +77 -14
  14. package/src/cache/layer.ts +17 -6
  15. package/src/cache/layered-cache.ts +4 -0
  16. package/src/cache/linked-inputs.ts +3 -0
  17. package/src/cache/lockfile-memo.ts +5 -3
  18. package/src/cache/output-index.ts +5 -2
  19. package/src/cache/remote-lane.ts +2 -1
  20. package/src/cache/run-history.ts +56 -17
  21. package/src/cache/schema.ts +15 -2
  22. package/src/cli/cache.ts +28 -18
  23. package/src/cli/completions.ts +4 -2
  24. package/src/cli/docs.ts +10 -11
  25. package/src/cli/help.ts +14 -25
  26. package/src/cli/index.ts +28 -9
  27. package/src/cli/info.ts +16 -8
  28. package/src/cli/init.ts +23 -15
  29. package/src/cli/last.ts +246 -204
  30. package/src/cli/lock.ts +55 -5
  31. package/src/cli/plan-format.ts +100 -118
  32. package/src/cli/plugin-commands.ts +8 -2
  33. package/src/cli/plugin-templates.ts +36 -45
  34. package/src/cli/run.ts +46 -27
  35. package/src/cli/select.ts +43 -5
  36. package/src/cli/show.ts +45 -52
  37. package/src/cli/task-verb.ts +22 -0
  38. package/src/cli/upgrade.ts +3 -1
  39. package/src/cli/watch-fs.ts +6 -1
  40. package/src/cli/watch-judge.ts +3 -2
  41. package/src/cli/watch-set.ts +4 -4
  42. package/src/cli/watch.ts +94 -52
  43. package/src/cli/why.ts +123 -89
  44. package/src/cli/workspace-config.ts +2 -1
  45. package/src/config.ts +47 -30
  46. package/src/exec/executor.ts +42 -2
  47. package/src/exec/index.ts +1 -0
  48. package/src/exec/kill-tree.ts +58 -5
  49. package/src/exec/local-executor.ts +1 -1
  50. package/src/exec/pipes.ts +152 -0
  51. package/src/exec/runner.ts +199 -182
  52. package/src/exec/sandbox-binds.ts +1 -1
  53. package/src/exec/sandbox-paths.ts +3 -1
  54. package/src/exec/sandbox-runtime.ts +317 -23
  55. package/src/exec/spawn-off.ts +154 -0
  56. package/src/exec/task-proxy.ts +93 -0
  57. package/src/exec/warm-shell.ts +146 -0
  58. package/src/graph/critical-path.ts +40 -28
  59. package/src/graph/definition.ts +16 -3
  60. package/src/graph/priorities.ts +98 -44
  61. package/src/graph/scheduler.ts +17 -11
  62. package/src/graph/task-graph.ts +51 -24
  63. package/src/handoff.ts +10 -7
  64. package/src/history/failure-mode.ts +24 -1
  65. package/src/history/history.ts +5 -0
  66. package/src/history/index.ts +1 -0
  67. package/src/history/metrics.ts +14 -6
  68. package/src/index.ts +7 -3
  69. package/src/orchestrator/admission.ts +1 -1
  70. package/src/orchestrator/command-windows.ts +1 -1
  71. package/src/orchestrator/deferred-outputs.ts +1 -1
  72. package/src/orchestrator/doctor.ts +1 -1
  73. package/src/orchestrator/download-policy.ts +5 -7
  74. package/src/orchestrator/excluded-keys.ts +1 -1
  75. package/src/orchestrator/execute-task.ts +67 -20
  76. package/src/orchestrator/graph-memo.ts +50 -1
  77. package/src/orchestrator/graph-stage.ts +6 -0
  78. package/src/orchestrator/hit-restore.ts +5 -6
  79. package/src/orchestrator/index.ts +3 -0
  80. package/src/orchestrator/keyed-projects.ts +28 -0
  81. package/src/orchestrator/live-usage.ts +16 -11
  82. package/src/orchestrator/local-shortcircuit.ts +2 -2
  83. package/src/orchestrator/locked-digests.ts +63 -0
  84. package/src/orchestrator/lockfile-claim.ts +108 -29
  85. package/src/orchestrator/miss-save.ts +58 -11
  86. package/src/orchestrator/persistent-task.ts +5 -1
  87. package/src/orchestrator/placement.ts +1 -1
  88. package/src/orchestrator/plan.ts +7 -3
  89. package/src/orchestrator/plugin-host.ts +67 -2
  90. package/src/orchestrator/plugin.ts +53 -2
  91. package/src/orchestrator/prepare.ts +16 -1
  92. package/src/orchestrator/projects.ts +4 -1
  93. package/src/orchestrator/remote-prefetch.ts +1 -1
  94. package/src/orchestrator/run-context.ts +29 -10
  95. package/src/orchestrator/run-failures.ts +62 -13
  96. package/src/orchestrator/run-keys.ts +1 -1
  97. package/src/orchestrator/run-records.ts +88 -59
  98. package/src/orchestrator/run-report.ts +1 -1
  99. package/src/orchestrator/run.ts +145 -41
  100. package/src/orchestrator/sandbox-request.ts +52 -9
  101. package/src/orchestrator/shell-verdict.ts +9 -0
  102. package/src/orchestrator/signals.ts +8 -5
  103. package/src/orchestrator/stable-keys.ts +17 -18
  104. package/src/orchestrator/task-artifacts.ts +1 -1
  105. package/src/orchestrator/task-hash.ts +21 -7
  106. package/src/orchestrator/telemetry.ts +5 -1
  107. package/src/report/events.ts +6 -14
  108. package/src/report/forecast.ts +10 -32
  109. package/src/report/framed-output.ts +283 -175
  110. package/src/report/index.ts +8 -3
  111. package/src/report/logger.ts +74 -39
  112. package/src/report/plain-output.ts +3 -1
  113. package/src/report/status-line.ts +49 -357
  114. package/src/report/summary.ts +105 -136
  115. package/src/settings.ts +8 -8
  116. package/src/ui/ansi.ts +48 -0
  117. package/src/ui/components.ts +506 -0
  118. package/src/ui/index.ts +79 -0
  119. package/src/ui/kit.ts +37 -0
  120. package/src/ui/live-region.ts +363 -0
  121. package/src/{util/refusal-notice.ts → ui/refusal.ts} +7 -5
  122. package/src/ui/terminal.ts +51 -0
  123. package/src/ui/theme.ts +224 -0
  124. package/src/ui/version-line.ts +28 -0
  125. package/src/util/config-frame.ts +191 -14
  126. package/src/util/edit-distance.ts +6 -1
  127. package/src/util/index.ts +2 -15
  128. package/src/util/paths.ts +21 -3
  129. package/src/util/secret-mask.ts +39 -3
  130. package/src/util/settle.ts +33 -1
  131. package/src/util/tail.ts +4 -1
  132. package/src/util/timing.ts +4 -16
  133. package/src/workspace/affected.ts +89 -30
  134. package/src/workspace/bracket-literal.ts +2 -2
  135. package/src/workspace/config-schema.ts +70 -63
  136. package/src/workspace/dependencies-lock.ts +191 -0
  137. package/src/workspace/filter.ts +36 -1
  138. package/src/workspace/index.ts +9 -0
  139. package/src/workspace/lockfile.ts +159 -6
  140. package/src/workspace/migration.ts +16 -9
  141. package/src/workspace/workspace.ts +3 -0
  142. package/src/util/colors.ts +0 -133
  143. package/src/util/report.ts +0 -44
  144. package/src/util/ui.ts +0 -258
  145. /package/src/{report/path-links.ts → ui/links.ts} +0 -0
  146. /package/src/{report → ui}/terminal-signals.ts +0 -0
@@ -27,9 +27,9 @@ import {
27
27
  type PluginSecrets,
28
28
  SecretBroker,
29
29
  thrownMessage,
30
- ui,
31
30
  UserError,
32
31
  } from '../util/index.js'
32
+ import * as ui from '../ui/index.js'
33
33
  import type { ProjectMeta } from '../workspace/index.js'
34
34
  import type { EventBus, Logger, RunStartInfo } from '../report/index.js'
35
35
  import type { CiContext } from './run-context.js'
@@ -460,7 +460,29 @@ export interface GraphHookContext extends BaseContext {
460
460
  readonly requested: readonly string[]
461
461
  }
462
462
 
463
- export interface KeyHookContext extends BaseContext {}
463
+ export interface KeyHookContext extends BaseContext {
464
+ /**
465
+ * What `vx lock` recorded of the claimed `file` in
466
+ * `.vx/dependencies.lock.json` (`FingerprintClaim.digests`): a run keys
467
+ * from the record and never parses the file. Undefined when the file is
468
+ * absent and unrecorded. Refuses, naming `vx lock`, when the lock is
469
+ * absent or records no such present file; a recorded file whose bytes
470
+ * moved since refused the run before the key stage.
471
+ */
472
+ locked(file: string): Promise<LockedFile | undefined>
473
+ }
474
+
475
+ /** `KeyHookContext.locked`'s answer. */
476
+ export interface LockedFile {
477
+ /**
478
+ * The other files the digests read → each one's hash then (`hashExtra`).
479
+ * Not checked by core: a `link:` directory is edited with no install,
480
+ * so a claimant whose extra file moved digests anew.
481
+ */
482
+ readonly extras: Readonly<Record<string, string>>
483
+ /** The project at `projectDir`'s digest; undefined when none. Refuses for a project the lock does not list. */
484
+ digest(projectDir: string): string | undefined
485
+ }
464
486
 
465
487
  /** A plugin's claim on workspace-root files — see `VxPlugin.fingerprint`. */
466
488
  export interface FingerprintClaim {
@@ -491,6 +513,29 @@ export interface FingerprintClaim {
491
513
  bytes: Uint8Array,
492
514
  ctx: FingerprintContext,
493
515
  ): readonly string[] | Promise<readonly string[]>
516
+ /**
517
+ * Each project's digest of the claimed `file` as it is now, by project
518
+ * name, with `hash`, the xxh3 of the bytes they came from (`xxh3hex`),
519
+ * and `extras`, the other files they read. `vx lock` records them in
520
+ * `.vx/dependencies.lock.json` with each extra file's hash; a run keys
521
+ * from that record (`KeyHookContext.locked`), and `--affected`
522
+ * reads the base's copy there instead of asking `affected` to parse the
523
+ * base side, when its `hash` is the base file's xxh3. `undefined`:
524
+ * nothing to record.
525
+ */
526
+ digests?(
527
+ file: string,
528
+ ctx: FingerprintContext,
529
+ ): FingerprintDigests | undefined | Promise<FingerprintDigests | undefined>
530
+ }
531
+
532
+ /** `FingerprintClaim.digests`' answer. */
533
+ export interface FingerprintDigests {
534
+ readonly hash: string
535
+ /** Project name → its digest of the file. */
536
+ readonly projects: Readonly<Record<string, string>>
537
+ /** Root-relative files besides `file` whose bytes the digests read (a `bun.lock` patch). */
538
+ readonly extras?: readonly string[]
494
539
  }
495
540
 
496
541
  export interface FingerprintChange {
@@ -558,6 +603,12 @@ export interface ScheduleHookContext extends BaseContext {
558
603
  * in a plan.
559
604
  */
560
605
  onRefine(listener: () => void): void
606
+ /**
607
+ * Name tasks a shared record knows as flaky (a history server's): one
608
+ * that fails gets the one retry a known flake gets, as the local
609
+ * record's do. Ids not in the run are ignored; ignored in a plan.
610
+ */
611
+ flaky(taskIds: Iterable<string>): void
561
612
  }
562
613
 
563
614
  /** A `cache` hook's context: the workspace and the run's scope, never the local store or the policy, which core owns. */
@@ -39,6 +39,7 @@ import {
39
39
  gitPathspecs,
40
40
  MAX_SCOPED_PATHSPECS,
41
41
  startGitEnumeration,
42
+ checkBlobSizes,
42
43
  lazyGitEnumeration,
43
44
  measureFsClockLag,
44
45
  DEFAULT_STORE,
@@ -52,6 +53,7 @@ import {
52
53
  lockDigest,
53
54
  PROJECT_CONFIG_FILENAMES,
54
55
  readClaimedFiles,
56
+ readDependenciesLock,
55
57
  resolveCacheDir,
56
58
  resolveStoreRoot,
57
59
  type ProjectEntry,
@@ -76,6 +78,7 @@ import {
76
78
  teardownPlugins,
77
79
  type ScheduleRun,
78
80
  } from './plugin-host.js'
81
+ import { lockedFiles } from './locked-digests.js'
79
82
  import {
80
83
  discoverProjects,
81
84
  gitOfDiscovery,
@@ -382,6 +385,16 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
382
385
  const fingerprintsAt = Date.now()
383
386
  const claimed = [...fingerprintClaims(plugins).keys()]
384
387
  const claimedBytes = await readClaimedFiles(workspaceRoot, claimed, reads)
388
+ // What `vx lock` recorded of them: a run keys from it (locked-digests.ts).
389
+ const dependenciesLock = claimed.length > 0 ? readDependenciesLock(workspaceRoot, reads) : null
390
+ dependenciesLock?.catch(() => {})
391
+ // A whole-tree run's blob-size check starts with the cache it reads its
392
+ // verdict from, beside the config load and graph build: after them, a cold
393
+ // 100-package run waited 40-50 ms on it before its first task (2026-11-03).
394
+ if (git.started !== undefined) {
395
+ localCache.holdBlobSizes()
396
+ git.started.then((e) => checkBlobSizes(e, localCache)).catch(() => {})
397
+ }
385
398
  mark('open cache')
386
399
 
387
400
  // Configs load FROM .vx/projects.lock.json with no evaluation and no check of
@@ -776,6 +789,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
776
789
  // 1,000 projects when it ran before the spawn.
777
790
  const settleIgnored = askIgnoredOutputs(gatedOutputs(nodes, workspaceRoot), workspaceRoot)
778
791
  const enumeration = await enumerating
792
+ localCache.holdBlobSizes()
779
793
  await applyGitEnumeration(
780
794
  enumeration,
781
795
  workspaceRoot,
@@ -800,7 +814,8 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
800
814
  }
801
815
  }
802
816
  if (hasHook(plugins, 'key')) {
803
- await applyKeyHooks(plugins, nodes, { workspaceRoot, cacheDir, warn: warnOf(log) })
817
+ const locked = lockedFiles(workspaceRoot, claimed, claimedBytes, await dependenciesLock)
818
+ await applyKeyHooks(plugins, nodes, { workspaceRoot, cacheDir, warn: warnOf(log), locked })
804
819
  }
805
820
  refuseFiltersThroughGroups(nodes)
806
821
  const exclude = options.excludeDependencies
@@ -370,8 +370,11 @@ export async function loadProjects(args: LoadProjectsArgs): Promise<LoadedProjec
370
370
  }
371
371
  continue
372
372
  }
373
- const config: ProjectConfig =
373
+ const frozen: ProjectConfig =
374
374
  meta.configPath === null ? { tasks: {} } : (loaded[next++] as ProjectConfig)
375
+ // The lock shares a task two projects declare alike: an edit in place
376
+ // works on this project's own copy.
377
+ const config = projectStage ? structuredClone(frozen) : frozen
375
378
  if (projectStage) {
376
379
  // The `project` stage edits the validated object in place; core
377
380
  // re-validates after EACH plugin so a plugin can only produce what
@@ -231,7 +231,7 @@ function fetchPool(
231
231
  * lookup, the other one that can reach a remote.
232
232
  */
233
233
  export function getContext(node: TaskNode, command: string) {
234
- const outputs = node.config.cache?.outputs
234
+ const outputs = node.config.outputs
235
235
  let masked: string | undefined
236
236
  return {
237
237
  taskId: node.id,
@@ -30,7 +30,7 @@ import type { RunContextRecord } from './telemetry.js'
30
30
  export interface GitContext {
31
31
  /** `git rev-parse HEAD`, or null outside a repo / on failure. */
32
32
  commitSha: string | null
33
- /** `git rev-parse --abbrev-ref HEAD`, or null. */
33
+ /** The branch HEAD names (`refs/heads/` stripped), with or without a commit; null when detached. */
34
34
  branch: string | null
35
35
  /** True if `git status --porcelain` was non-empty; null on failure. */
36
36
  dirty: boolean | null
@@ -61,9 +61,11 @@ export interface HostContext {
61
61
 
62
62
  /**
63
63
  * One short `git` spawn per run (not per task), behind try/catch.
64
- * `git rev-parse HEAD --abbrev-ref HEAD` returns the commit on line 1
65
- * and the branch on line 2 in a SINGLE invocation — half the spawns of
66
- * two separate `rev-parse` calls. `dirty` is NOT probed here: the run's
64
+ * `git rev-parse HEAD --symbolic-full-name HEAD` returns the commit on
65
+ * line 1 and the ref on line 2 in a SINGLE invocation — half the spawns
66
+ * of two separate `rev-parse` calls. The full name, not `--abbrev-ref`:
67
+ * a tag named like the branch made that answer `heads/main`
68
+ * (vercel/turborepo#13882). `dirty` is NOT probed here: the run's
67
69
  * `GitFilesCache` populate already ran `git status --porcelain` for
68
70
  * input enumeration, so the orchestrator passes that aggregate in
69
71
  * (`dirty`) rather than paying for a second status spawn. Pass
@@ -95,7 +97,7 @@ export function captureGitContext(
95
97
  workspaceRoot,
96
98
  'rev-parse',
97
99
  'HEAD',
98
- '--abbrev-ref',
100
+ '--symbolic-full-name',
99
101
  'HEAD',
100
102
  ],
101
103
  env: gitEnv(),
@@ -107,15 +109,19 @@ export function captureGitContext(
107
109
  const sha = lines[0]?.trim()
108
110
  const br = lines[1]?.trim()
109
111
  if (sha) commitSha = sha
110
- // `--abbrev-ref HEAD` answers the literal string `HEAD` when nothing is
111
- // checked out. Recording that verbatim INVENTS a branch: every detached
112
+ // A detached HEAD answers the literal string `HEAD`, never `refs/heads/…`.
113
+ // Recording that verbatim INVENTS a branch: every detached
112
114
  // run shares a pseudo-branch "HEAD", which is how `actions/checkout`
113
115
  // leaves a pull_request build. The trust boundary (`branch ===
114
116
  // defaultBranch`) means it is at least never mistaken for trunk, but
115
117
  // every PR collapses into one scope — so one PR's timings feed another's
116
118
  // baseline, cross-branch regression detection counts them as a single
117
119
  // branch, and a consumer grouping runs by branch reads "HEAD".
118
- if (br && br !== 'HEAD') branch = br
120
+ if (br?.startsWith('refs/heads/')) branch = br.slice('refs/heads/'.length)
121
+ } else {
122
+ // No commit yet: `rev-parse HEAD` fails, yet HEAD already names the
123
+ // branch the first commit lands on (`git branch --show-current`).
124
+ branch = unbornBranch(workspaceRoot)
119
125
  }
120
126
  } catch {
121
127
  // git unavailable / non-repo: both fields stay null.
@@ -129,6 +135,18 @@ export function captureGitContext(
129
135
  return { commitSha, branch, dirty }
130
136
  }
131
137
 
138
+ /** The branch HEAD names, asked of git only after `rev-parse HEAD` failed. */
139
+ function unbornBranch(workspaceRoot: string): string | null {
140
+ const proc = Bun.spawnSync({
141
+ cmd: [executablePath('git'), '-C', workspaceRoot, 'symbolic-ref', '--quiet', 'HEAD'],
142
+ env: gitEnv(),
143
+ stdout: 'pipe',
144
+ stderr: 'ignore',
145
+ })
146
+ const ref = proc.exitCode === 0 ? new TextDecoder().decode(proc.stdout).trim() : ''
147
+ return ref.startsWith('refs/heads/') ? ref.slice('refs/heads/'.length) : null
148
+ }
149
+
132
150
  /**
133
151
  * `HEAD`'s commit and branch from the `.git` files, or null when anything
134
152
  * about the layout is unfamiliar (the caller then spawns git). Handles a
@@ -138,7 +156,7 @@ export function captureGitContext(
138
156
  */
139
157
  function readHeadDirect(
140
158
  workspaceRoot: string,
141
- ): { commitSha: string; branch: string | null } | null {
159
+ ): { commitSha: string | null; branch: string | null } | null {
142
160
  try {
143
161
  const dirs = refDirsOf(workspaceRoot)
144
162
  if (dirs === null) return null
@@ -166,7 +184,8 @@ function readHeadDirect(
166
184
  }
167
185
  }
168
186
  }
169
- return null
187
+ // Neither a loose nor a packed ref: the branch has no commit yet.
188
+ return { commitSha: null, branch }
170
189
  } catch {
171
190
  return null
172
191
  }
@@ -8,7 +8,7 @@ import { isGroupTask, type TaskOutcome } from '../graph/index.js'
8
8
  import { decodeOutputLog } from './output-log.js'
9
9
  import { PASS_STATUSES } from '../history/index.js'
10
10
  import { maskedLine, splitTaskId } from '../util/index.js'
11
- import { fileLocations, fileMemo, type OutputLocation } from '../report/index.js'
11
+ import { fileLocations, fileMemo, type OutputLocation } from '../ui/index.js'
12
12
 
13
13
  /** A failed task of a run and what it said. */
14
14
  export interface TaskFailure {
@@ -46,17 +46,16 @@ export function writeRunFailures(
46
46
  const failures = []
47
47
  for (const o of outcomes) {
48
48
  // A task failed by a moved input may have printed nothing: the input
49
- // is still why, and an agent reads it here.
50
- if ((o.failedOutput === undefined && o.inputMoved === undefined) || isGroupTask(o.node))
49
+ // is still why, and an agent reads it here. A pass that took a retry
50
+ // keeps its failed attempts.
51
+ if (
52
+ (o.failedOutput === undefined &&
53
+ o.inputMoved === undefined &&
54
+ o.attemptOutputs === undefined) ||
55
+ isGroupTask(o.node)
56
+ )
51
57
  continue
52
- const output =
53
- o.failedOutput === undefined
54
- ? ''
55
- : Bun.stripANSI(
56
- decodeOutputLog(o.failedOutput)
57
- .map((c) => c.text)
58
- .join(''),
59
- )
58
+ const output = attemptsText(o)
60
59
  failures.push({
61
60
  taskId: o.node.id,
62
61
  ...(o.inputMoved !== undefined ? { inputMoved: o.inputMoved } : {}),
@@ -74,6 +73,40 @@ export function writeRunFailures(
74
73
  }
75
74
  }
76
75
 
76
+ const plain = (log: string): string =>
77
+ Bun.stripANSI(
78
+ decodeOutputLog(log)
79
+ .map((c) => c.text)
80
+ .join(''),
81
+ )
82
+
83
+ /** How attempt `n` of `of` ended, the line above its output. */
84
+ const attemptHeading = (n: number, of: number, how: string): string =>
85
+ `── attempt ${n} of ${of} · ${how} ──\n`
86
+
87
+ const ended = (a: { exitCode: number; timedOut?: true }): string =>
88
+ a.timedOut === true ? 'timed out' : `exit ${a.exitCode}`
89
+
90
+ /**
91
+ * What a failure row keeps of `o`: its output, after each failed attempt's
92
+ * under a heading when it retried. A pass that retried keeps only the
93
+ * failed attempts: its own output is its cache entry's.
94
+ */
95
+ function attemptsText(o: TaskOutcome): string {
96
+ const last = o.failedOutput === undefined ? '' : plain(o.failedOutput)
97
+ const before = o.attemptOutputs ?? []
98
+ if (before.length === 0) return last
99
+ const of = before.length + 1
100
+ let text = ''
101
+ before.forEach((log, i) => {
102
+ const out = plain(log)
103
+ text += attemptHeading(i + 1, of, ended(o.failedAttempts![i]!)) + out
104
+ if (out !== '' && !out.endsWith('\n')) text += '\n'
105
+ })
106
+ if (o.status === 'failed') text += attemptHeading(of, of, ended(o)) + last
107
+ return text
108
+ }
109
+
77
110
  /**
78
111
  * The failed tasks of `runId`, or of the latest failed run when omitted;
79
112
  * null when there is no such run. A failed task with no kept row (a run
@@ -167,6 +200,19 @@ export function taskLog(db: Database, taskId: string, runId?: string): TaskLog |
167
200
  const kept = keptFailures(db, row.runId).get(taskId)
168
201
  if (kept !== undefined) return { ...base, source: 'failure', output: kept.output }
169
202
  } else if (row.hash !== '') {
203
+ // A pass that took a retry: its failed attempts, then its own output.
204
+ const before = keptFailures(db, row.runId).get(taskId)?.output
205
+ const tries = (
206
+ db
207
+ .query(
208
+ 'SELECT attempts FROM runs WHERE run_id = ? AND project = ? AND task = ? ORDER BY id DESC LIMIT 1',
209
+ )
210
+ .get(row.runId, project, task) as { attempts: number | null } | null
211
+ )?.attempts
212
+ const prefix =
213
+ before === undefined || tries === null || tries === undefined
214
+ ? ''
215
+ : `${before}${attemptHeading(tries, tries, 'passed')}`
170
216
  const stored = db.query('SELECT stdout FROM entry_stdout WHERE hash = ?').get(row.hash) as {
171
217
  stdout: string
172
218
  } | null
@@ -182,12 +228,15 @@ export function taskLog(db: Database, taskId: string, runId?: string): TaskLog |
182
228
  .join(''),
183
229
  ),
184
230
  )
185
- return { ...base, source: 'cache', output }
231
+ return { ...base, source: 'cache', output: prefix + output }
186
232
  }
187
233
  // An entry that printed nothing stores no stdout row: kept, and empty.
188
234
  if (db.query('SELECT 1 FROM entries WHERE hash = ?').get(row.hash) !== null) {
189
- return { ...base, source: 'cache', output: '' }
235
+ return { ...base, source: 'cache', output: prefix }
190
236
  }
191
237
  }
238
+ // A pass with no entry that took a retry still kept its failed attempts.
239
+ const kept = row.status === 'failed' ? undefined : keptFailures(db, row.runId).get(taskId)
240
+ if (kept !== undefined) return { ...base, source: 'failure', output: kept.output }
192
241
  return { ...base, source: null, output: '' }
193
242
  }
@@ -89,7 +89,7 @@ export function runKeysToken(args: RunKeysArgs): RunKeysAt | undefined {
89
89
  if ((inputs.workspaceFiles?.length ?? 0) > 0) return undefined
90
90
  // No positive entry resolves no files (`filesPlan`).
91
91
  if (inputs.files === undefined || inputs.files.some((e) => !e.startsWith('!'))) {
92
- const outputs = cfg.cache?.outputs
92
+ const outputs = cfg.outputs
93
93
  const declared = `${node.projectName}\0${JSON.stringify([inputs.files, outputs?.files, outputs?.workspaceFiles])}`
94
94
  let token = tokens.get(declared)
95
95
  if (token === undefined) {
@@ -38,10 +38,20 @@ export interface RunRecordsInput {
38
38
  host: HostContext
39
39
  /** Build the telemetry mirror; a run with no sink allocates none. */
40
40
  withTelemetry: boolean
41
+ /** Each task's count of the tasks and groups downstream of it (the scheduler's): the rows' `blocks`. */
42
+ blocks?: ReadonlyMap<string, number>
41
43
  /** The run's remote failures by key, so far (`Cache.traffic`). */
42
44
  remoteTraffic?: ReadonlyMap<string, { failures: readonly string[] }>
45
+ /** Outcomes whose rows the history stream already wrote: counted, not written again. */
46
+ landed?: ReadonlySet<TaskOutcome>
43
47
  }
44
48
 
49
+ /** What a task's `runs` row reads besides its outcome. */
50
+ export type RunRecordContext = Pick<
51
+ RunRecordsInput,
52
+ 'tainted' | 'runId' | 'startedAtMs' | 'endedAtMs' | 'forwardArgs' | 'blocks' | 'remoteTraffic'
53
+ >
54
+
45
55
  export interface RunRecords {
46
56
  runs: RunRecord[]
47
57
  invocation: InvocationRecord
@@ -70,6 +80,7 @@ export function assembleRunRecords(input: RunRecordsInput): RunRecords {
70
80
  const { outcomes, runId, startedAtMs, endedAtMs } = input
71
81
  const runs: RunRecord[] = []
72
82
  const telemetryTasks: TaskTelemetry[] = []
83
+ let taskCount = 0
73
84
  let failedCount = 0
74
85
  let hitLocalCount = 0
75
86
  let hitRemoteCount = 0
@@ -81,64 +92,8 @@ export function assembleRunRecords(input: RunRecordsInput): RunRecords {
81
92
  // aborted (killed by a shutdown signal) isn't a real run.
82
93
  if (o.status === 'aborted') continue
83
94
  if (input.withTelemetry) telemetryTasks.push(taskTelemetryOf(o))
84
- const blockedBy = o.blockedBy ?? input.tainted?.get(o.node.id)
85
- const remoteError = remoteErrorOf(input.remoteTraffic, o.hash)
86
- // Built field by field: a spread per optional field was ~2 µs of each
87
- // of a 1,200-hit warm run's rows.
88
- const hit = isCacheHit(o.status)
89
- const r: RunRecord = {
90
- project: o.node.projectName,
91
- task: o.node.taskName,
92
- status: o.status,
93
- exitCode: o.exitCode,
94
- durationMs: o.durationMs,
95
- // Anchor to the REAL per-task wall-clock window: run-start wall time +
96
- // the task's ns offset (captured for hits and executed tasks alike).
97
- // The `end - duration` fallback applies to outcomes without an offset —
98
- // today only `skipped`, which the scheduler finishes synchronously with
99
- // no span, so it collapses to a zero-width mark at the run's end. Using
100
- // run-end-minus-duration for EVERYTHING was the old bug that piled every
101
- // task at the right edge of the timeline.
102
- startedAt:
103
- o.wallclockStartNs !== undefined
104
- ? startedAtMs + Math.round(Number(o.wallclockStartNs) / 1e6)
105
- : endedAtMs - o.durationMs,
106
- endedAt:
107
- o.wallclockEndNs !== undefined
108
- ? startedAtMs + Math.round(Number(o.wallclockEndNs) / 1e6)
109
- : endedAtMs,
110
- runId,
111
- cacheHit: hit,
112
- cached: o.node.config.cache !== undefined,
113
- }
114
- if (o.hash !== undefined) r.hash = o.hash
115
- // A run forwards its args to the tasks it was asked for, never to a
116
- // dependency: the row says which ran with them.
117
- if (input.forwardArgs !== undefined && input.forwardArgs.length > 0 && o.node.requested)
118
- r.forwardArgs = input.forwardArgs
119
- if (o.cpuMs !== undefined) r.cpuMs = o.cpuMs
120
- if (o.peakRssBytes !== undefined) r.peakRssBytes = o.peakRssBytes
121
- if (o.peakCores !== undefined) r.peakCores = o.peakCores
122
- if (o.avgRssBytes !== undefined) r.avgRssBytes = o.avgRssBytes
123
- if (o.where !== undefined) r.placement = o.where
124
- if (o.whereCores !== undefined) r.placementCores = o.whereCores
125
- if (o.whereMemoryBytes !== undefined) r.placementMemoryBytes = o.whereMemoryBytes
126
- if (o.cpuShare !== undefined) r.cpuShare = o.cpuShare
127
- if (o.wallclockStartNs !== undefined) r.wallclockStartNs = o.wallclockStartNs
128
- if (o.wallclockEndNs !== undefined) r.wallclockEndNs = o.wallclockEndNs
129
- if (hit) r.restored = o.restored === true
130
- if (o.attempts !== undefined) r.attempts = o.attempts
131
- // A task that ran behind a failure names it too, so the flakiness
132
- // projection counts its outcome on no key (failure-mode.ts).
133
- if (blockedBy !== undefined) r.blockedBy = blockedBy
134
- if (o.timedOut === true) r.timedOut = true
135
- if (o.sandboxViolations !== undefined) r.sandboxViolations = o.sandboxViolations
136
- if (o.notReady !== undefined) r.notReady = o.notReady
137
- if (remoteError !== undefined) r.remoteError = remoteError
138
- if (o.keylessBy !== undefined) r.keylessBy = o.keylessBy
139
- const def = taskDefinition(o.node)
140
- if (def !== undefined) r.def = def
141
- runs.push(r)
95
+ taskCount++
96
+ if (input.landed?.has(o) !== true) runs.push(runRecordOf(o as RecordedOutcome, input))
142
97
  if (o.status === 'failed') failedCount++
143
98
  if (o.status === 'cache-hit') hitLocalCount++
144
99
  if (o.status === 'cache-hit-remote') hitRemoteCount++
@@ -158,7 +113,7 @@ export function assembleRunRecords(input: RunRecordsInput): RunRecords {
158
113
  startedAt: startedAtMs,
159
114
  endedAt: endedAtMs,
160
115
  totalDurationMs: Math.round(input.totalMs),
161
- taskCount: runs.length,
116
+ taskCount,
162
117
  failedCount,
163
118
  hitCount: hitLocalCount + hitRemoteCount,
164
119
  hitLocalCount,
@@ -180,6 +135,80 @@ export function assembleRunRecords(input: RunRecordsInput): RunRecords {
180
135
  return { runs, invocation, telemetryTasks }
181
136
  }
182
137
 
138
+ /** An outcome a `runs` row records: every status but `aborted`. */
139
+ export type RecordedOutcome = TaskOutcome & { status: RunRecord['status'] }
140
+
141
+ /** One outcome's `runs` row. */
142
+ export function runRecordOf(o: RecordedOutcome, input: RunRecordContext): RunRecord {
143
+ const { runId, startedAtMs, endedAtMs } = input
144
+ const blockedBy = o.blockedBy ?? input.tainted?.get(o.node.id)
145
+ const remoteError = remoteErrorOf(input.remoteTraffic, o.hash)
146
+ // Built field by field: a spread per optional field was ~2 µs of each
147
+ // of a 1,200-hit warm run's rows.
148
+ const hit = isCacheHit(o.status)
149
+ const r: RunRecord = {
150
+ project: o.node.projectName,
151
+ task: o.node.taskName,
152
+ status: o.status,
153
+ exitCode: o.exitCode,
154
+ durationMs: o.durationMs,
155
+ // Anchor to the REAL per-task wall-clock window: run-start wall time +
156
+ // the task's ns offset (captured for hits and executed tasks alike).
157
+ // The `end - duration` fallback applies to outcomes without an offset —
158
+ // today only `skipped`, which the scheduler finishes synchronously with
159
+ // no span, so it collapses to a zero-width mark at the run's end. Using
160
+ // run-end-minus-duration for EVERYTHING was the old bug that piled every
161
+ // task at the right edge of the timeline.
162
+ startedAt:
163
+ o.wallclockStartNs !== undefined
164
+ ? startedAtMs + Math.round(Number(o.wallclockStartNs) / 1e6)
165
+ : endedAtMs - o.durationMs,
166
+ endedAt:
167
+ o.wallclockEndNs !== undefined
168
+ ? startedAtMs + Math.round(Number(o.wallclockEndNs) / 1e6)
169
+ : endedAtMs,
170
+ runId,
171
+ cacheHit: hit,
172
+ cached: o.node.config.cache === true,
173
+ }
174
+ if (o.hash !== undefined) r.hash = o.hash
175
+ // A run forwards its args to the tasks it was asked for, never to a
176
+ // dependency: the row says which ran with them.
177
+ if (input.forwardArgs !== undefined && input.forwardArgs.length > 0 && o.node.requested)
178
+ r.forwardArgs = input.forwardArgs
179
+ if (o.cpuMs !== undefined) r.cpuMs = o.cpuMs
180
+ if (o.peakRssBytes !== undefined) r.peakRssBytes = o.peakRssBytes
181
+ if (o.peakCores !== undefined) r.peakCores = o.peakCores
182
+ if (o.avgRssBytes !== undefined) r.avgRssBytes = o.avgRssBytes
183
+ if (o.where !== undefined) r.placement = o.where
184
+ if (o.whereCores !== undefined) r.placementCores = o.whereCores
185
+ if (o.whereMemoryBytes !== undefined) r.placementMemoryBytes = o.whereMemoryBytes
186
+ if (o.cpuShare !== undefined) r.cpuShare = o.cpuShare
187
+ if (o.queuedMs !== undefined) r.queuedMs = o.queuedMs
188
+ if (o.outputBytes !== undefined) r.outputBytes = o.outputBytes
189
+ const blocks = input.blocks?.get(o.node.id)
190
+ if (blocks !== undefined) r.blocks = blocks
191
+ if (o.flaky !== undefined) {
192
+ r.flakyPasses = o.flaky.passes
193
+ r.flakyFailures = o.flaky.failures
194
+ }
195
+ if (o.wallclockStartNs !== undefined) r.wallclockStartNs = o.wallclockStartNs
196
+ if (o.wallclockEndNs !== undefined) r.wallclockEndNs = o.wallclockEndNs
197
+ if (hit) r.restored = o.restored === true
198
+ if (o.attempts !== undefined) r.attempts = o.attempts
199
+ // A task that ran behind a failure names it too, so the flakiness
200
+ // projection counts its outcome on no key (failure-mode.ts).
201
+ if (blockedBy !== undefined) r.blockedBy = blockedBy
202
+ if (o.timedOut === true) r.timedOut = true
203
+ if (o.sandboxViolations !== undefined) r.sandboxViolations = o.sandboxViolations
204
+ if (o.notReady !== undefined) r.notReady = o.notReady
205
+ if (remoteError !== undefined) r.remoteError = remoteError
206
+ if (o.keylessBy !== undefined) r.keylessBy = o.keylessBy
207
+ const def = taskDefinition(o.node)
208
+ if (def !== undefined) r.def = def
209
+ return r
210
+ }
211
+
183
212
  /**
184
213
  * The remote failures uploads met after the run's rows went in (the uploads
185
214
  * drain later): each row whose text is now short of its key's is rewritten.
@@ -49,7 +49,7 @@ async function outputsLeft(
49
49
  workspaceRoot: string,
50
50
  nestedProjectDirs: string[],
51
51
  ): Promise<string[]> {
52
- const declared = node.config.cache?.outputs
52
+ const declared = node.config.outputs
53
53
  if (declared === undefined) return []
54
54
  const [own, ws] = await Promise.all([
55
55
  resolveOutputs({ projectDir: node.projectDir, outputs: declared.files, nestedProjectDirs }),