@vzn/vx 0.0.595 → 0.0.597

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/README.md +1 -1
  2. package/launcher.cjs +7 -3
  3. package/package.json +5 -5
  4. package/src/cache/archive.ts +35 -6
  5. package/src/cache/cache.ts +170 -99
  6. package/src/cache/chained-cache.ts +25 -7
  7. package/src/cache/git-inputs.ts +23 -0
  8. package/src/cache/index.ts +3 -0
  9. package/src/cache/inputs.ts +99 -44
  10. package/src/cache/key-fold.ts +1 -1
  11. package/src/cache/layer.ts +9 -6
  12. package/src/cache/policy.ts +5 -0
  13. package/src/cache/zstd.ts +10 -4
  14. package/src/cli/cache.ts +12 -6
  15. package/src/cli/foreign-flags.ts +57 -3
  16. package/src/cli/info.ts +1 -1
  17. package/src/cli/init.ts +2 -2
  18. package/src/cli/last.ts +10 -4
  19. package/src/cli/lock.ts +5 -3
  20. package/src/cli/plan-format.ts +4 -4
  21. package/src/cli/run.ts +73 -30
  22. package/src/cli/select.ts +6 -3
  23. package/src/cli/show.ts +8 -2
  24. package/src/cli/upgrade.ts +1 -1
  25. package/src/cli/watch-filter.ts +63 -12
  26. package/src/cli/watch-fs.ts +11 -3
  27. package/src/cli/watch-judge.ts +15 -8
  28. package/src/cli/watch.ts +18 -11
  29. package/src/cli/workspace-config.ts +3 -1
  30. package/src/config.ts +42 -21
  31. package/src/exec/executor.ts +1 -1
  32. package/src/exec/kill-tree.ts +89 -18
  33. package/src/exec/proc-sample.ts +1 -1
  34. package/src/exec/runner.ts +74 -36
  35. package/src/exec/sandbox-paths.ts +12 -1
  36. package/src/exec/sandbox-runtime.ts +283 -88
  37. package/src/exec/sandbox-violations.ts +115 -15
  38. package/src/graph/scheduler.ts +30 -14
  39. package/src/graph/task-graph.ts +265 -44
  40. package/src/orchestrator/admission.ts +12 -3
  41. package/src/orchestrator/affected-tasks.ts +6 -2
  42. package/src/orchestrator/deferred-outputs.ts +3 -2
  43. package/src/orchestrator/doctor.ts +2 -2
  44. package/src/orchestrator/events.ts +10 -1
  45. package/src/orchestrator/excluded-keys.ts +2 -13
  46. package/src/orchestrator/execute-task.ts +85 -39
  47. package/src/orchestrator/failure-mode.ts +43 -28
  48. package/src/orchestrator/framed-output.ts +8 -8
  49. package/src/orchestrator/index.ts +0 -1
  50. package/src/orchestrator/keyed-projects.ts +6 -3
  51. package/src/orchestrator/logger.ts +21 -9
  52. package/src/orchestrator/metrics.ts +19 -8
  53. package/src/orchestrator/miss-save.ts +9 -13
  54. package/src/orchestrator/options.ts +7 -0
  55. package/src/orchestrator/plan.ts +7 -23
  56. package/src/orchestrator/plugin.ts +37 -16
  57. package/src/orchestrator/prepare.ts +13 -4
  58. package/src/orchestrator/run-artifacts.ts +4 -3
  59. package/src/orchestrator/run-lock.ts +25 -24
  60. package/src/orchestrator/run.ts +22 -5
  61. package/src/orchestrator/sandbox-request.ts +96 -18
  62. package/src/orchestrator/save-lane.ts +5 -5
  63. package/src/orchestrator/signals.ts +22 -4
  64. package/src/orchestrator/stable-keys.ts +3 -11
  65. package/src/orchestrator/summary.ts +7 -13
  66. package/src/orchestrator/task-hash.ts +13 -3
  67. package/src/orchestrator/telemetry.ts +16 -7
  68. package/src/orchestrator/upstream.ts +11 -0
  69. package/src/util/hangup.ts +42 -0
  70. package/src/util/index.ts +2 -0
  71. package/src/util/paths.ts +24 -4
  72. package/src/workspace/affected.ts +79 -42
  73. package/src/workspace/config-eval.ts +27 -6
  74. package/src/workspace/config-imports.ts +8 -53
  75. package/src/workspace/config-schema.ts +56 -9
  76. package/src/workspace/filter.ts +19 -0
  77. package/src/workspace/lockfile.ts +2 -1
  78. package/src/workspace/migrate-scripts.ts +5 -3
  79. package/src/workspace/migration.ts +1 -1
  80. package/src/workspace/project-loader.ts +19 -4
  81. package/src/workspace/repo-id.ts +42 -4
  82. package/src/workspace/workspace.ts +53 -24
@@ -362,12 +362,17 @@ async function executePersistentTask(args: ExecuteArgs): Promise<TaskOutcome> {
362
362
  args.cacheDir,
363
363
  )
364
364
  placeholders = sb.placeholders
365
+ // A wrap that refuses (a port the host holds) leaves the server unspawned
366
+ // and the placeholders vx made for it in the project for every later run.
365
367
  const wrapped = await wrapSandboxedCommand({
366
368
  command: plainCommand,
367
369
  cwd: node.projectDir,
368
370
  env,
369
371
  ...sb.sandbox,
370
372
  server: true,
373
+ }).catch(async (err: unknown) => {
374
+ await sweepPlaceholders(placeholders)
375
+ throw err
371
376
  })
372
377
  command = wrapped.wrapped
373
378
  bridgeTag = wrapped.tag
@@ -519,11 +524,10 @@ async function executePersistentTask(args: ExecuteArgs): Promise<TaskOutcome> {
519
524
  // exited before ready keeps its own exit code rather than a made-up 1.
520
525
  // One the readiness timeout is killing reports the signal's, as an
521
526
  // ordinary timeout does (X-24).
527
+ // A readiness timeout rejects once the group is gone (runner.ts), so a
528
+ // server that ignored the TERM holds no port into the next `vx watch`
529
+ // cycle, and a second signal here would cut a one-shot handler short.
522
530
  const ready = err instanceof PersistentReadyError ? err : undefined
523
- // The timer's SIGKILL is a grace away and the shell may die on the
524
- // TERM first: a server that ignores it held its port past run() into
525
- // the next `vx watch` cycle. Return once the group is gone.
526
- if (ready?.reason === 'timeout') await terminateChildren(() => [spawn.child])
527
531
  return {
528
532
  node,
529
533
  status: 'failed',
@@ -632,7 +636,10 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
632
636
  // `signal` stops a plugin executor on the run's stop AND on
633
637
  // `exec.timeout`: core cannot kill a process an executor spawned, so a
634
638
  // declared timeout meant nothing on one that kept its own clock (H-12).
635
- // The local executor keeps its own timer (it signals the process group).
639
+ // The local executor keeps its own timer from the spawn (it signals the
640
+ // process group), so its request carries no clock: armed here, it also
641
+ // timed the sandbox's wrap, and a wrap slower than the timeout aborted
642
+ // the request before the spawn — a task reported killed, never run.
636
643
  let timeoutTimer: ReturnType<typeof setTimeout> | undefined
637
644
  let timeoutFired = false
638
645
  // The attempt's listener on the run's stop signal, taken off once the
@@ -640,7 +647,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
640
647
  // grew to the task count, and each add scans it for a duplicate, ~100 ms
641
648
  // of a 1,000-task cold run.
642
649
  let unlistenStop: (() => void) | undefined
643
- function requestSignal(): AbortSignal {
650
+ function requestSignal(local: boolean): AbortSignal {
644
651
  const stop = new AbortController()
645
652
  const abort = (): void => stop.abort(args.stopSignal?.reason)
646
653
  unlistenStop?.()
@@ -651,9 +658,9 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
651
658
  run.addEventListener('abort', abort, { once: true })
652
659
  unlistenStop = () => run.removeEventListener('abort', abort)
653
660
  }
654
- if (effectiveTimeout !== undefined) {
655
- clearTimeout(timeoutTimer)
656
- timeoutFired = false
661
+ clearTimeout(timeoutTimer)
662
+ timeoutFired = false
663
+ if (effectiveTimeout !== undefined && !local) {
657
664
  timeoutTimer = setTimeout(() => {
658
665
  timeoutFired = true
659
666
  stop.abort(new Error(`timed out after ${effectiveTimeout}ms`))
@@ -803,7 +810,10 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
803
810
  }
804
811
  if (preProbed !== undefined) {
805
812
  if (preProbed.hit !== null) {
806
- const restored = await restoreOrMiss(preProbed.hit)
813
+ // A key taken before a task rewrote the lockfile names an install
814
+ // the tree no longer holds, and the lazy path below never probes
815
+ // one (X-124): demoted like a vanished hit, it runs dep-gated.
816
+ const restored = fingerprintMoved() ? null : await restoreOrMiss(preProbed.hit)
807
817
  if (restored !== null) return restored
808
818
  // The up-front probe's hit may be restoring AHEAD of this task's
809
819
  // deps, and a command run now would build from outputs they have
@@ -912,6 +922,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
912
922
  async function runAttempt(): Promise<{
913
923
  result: ExecuteResult
914
924
  exitCode: number
925
+ withdrawn?: true
915
926
  }> {
916
927
  // A deferred task's outputs are deliberately NOT coming, so wiping the
917
928
  // tree would replace a stale build with nothing at all. The eligibility
@@ -947,17 +958,33 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
947
958
  }
948
959
  violations = []
949
960
  const endReq = span('miss: build request')
950
- const req = await buildRequest()
961
+ const req = await buildRequest(isLocalExecutor(args.executor))
951
962
  endReq()
952
- // An executor that THROWS produces no captured output, so the task's
953
- // frame would print the command and nothing else while the reason went
954
- // straight to stderr and scrolled away in a broad run. Put it in the
955
- // task's own stream first: the frame is where a reader looks for why a
956
- // task failed, and a remote executor's failures are exactly the ones with
957
- // no other trace. Rethrown unchanged — the scheduler still classifies it,
958
- // and still prints it plainly for a UserError.
963
+ // A sibling's failure under --continue=never can land while a retry
964
+ // cleans or builds its request, after the loop's own check: the retry
965
+ // is withdrawn and the attempt before it stands.
966
+ if (attempt > 1 && args.failFast?.aborted === true) {
967
+ clearTimeout(timeoutTimer)
968
+ unlistenStop?.()
969
+ unlistenStop = undefined
970
+ await sweepPlaceholders(placeholders)
971
+ return { result, exitCode: effectiveExitCode, withdrawn: true }
972
+ }
973
+ // An executor that THROWS produces no captured output. Rethrown: the
974
+ // scheduler classifies it and prints its one line into the task's own
975
+ // stream (run()'s onError), where the frame reads it; a copy written
976
+ // here too printed the reason twice.
959
977
  const endExec = span('miss: execute')
960
- let res = await boundAfterAbort(args.executor.execute(req), req.signal)
978
+ // A plugin's execute may throw before returning or return a bare result:
979
+ // the throw is its rejection, the value its resolution, so both reach
980
+ // the naming catch and the cleanup in finally below.
981
+ let running: Promise<unknown>
982
+ try {
983
+ running = Promise.resolve(args.executor.execute(req))
984
+ } catch (thrown) {
985
+ running = Promise.reject(thrown)
986
+ }
987
+ let res = await boundAfterAbort(running, req.signal)
961
988
  .then((r: unknown) => {
962
989
  assertExecuteResult(args.executor.name, node.id, r)
963
990
  return r
@@ -973,23 +1000,19 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
973
1000
  raw = new UserError(`${why}, and remote: 'only' keeps it off this machine`)
974
1001
  } else {
975
1002
  log.status(`[vx] ${node.id}: ${why} — running it here`)
976
- clearTimeout(timeoutTimer)
977
- const local = await localExecutor().execute(await buildRequest())
1003
+ const local = await localExecutor().execute(await buildRequest(true))
978
1004
  assertExecuteResult('local', node.id, local)
979
1005
  return local
980
1006
  }
981
1007
  }
982
1008
  const err = nameExecutorFailure(args.executor, raw)
983
1009
  // A remote executor's message carries the server's own text, which
984
- // may echo the env it was sent: masked here, where it is printed,
985
- // and on the error the scheduler prints with its cause (L-39).
1010
+ // may echo the env it was sent: masked on the error the scheduler
1011
+ // prints with its cause (L-39).
986
1012
  if (secrets !== null) {
987
1013
  for (const e of [err, err instanceof Error ? err.cause : undefined])
988
1014
  if (e instanceof Error) e.message = secrets.mask(e.message)
989
1015
  }
990
- const message =
991
- err instanceof Error ? err.message : (secrets?.mask(String(err)) ?? String(err))
992
- log.taskStderr(node, `${message}\n`)
993
1016
  await sweepPlaceholders(placeholders)
994
1017
  throw err
995
1018
  })
@@ -1008,11 +1031,11 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
1008
1031
  stdout: maskCaptured(res.stdout, secrets),
1009
1032
  stderr: maskCaptured(res.stderr, secrets),
1010
1033
  }
1011
- // An executor that stopped on the timeout's abort exits non-zero; say
1012
- // why, so the frame, the retry line and `timedOut` read as a timeout.
1013
- if (timeoutFired && res.exitCode !== 0 && res.timedOut !== true) {
1014
- res = { ...res, timedOut: true }
1015
- }
1034
+ // An executor that stopped on the timeout's abort timed out, whatever
1035
+ // its exit: a child that traps the TERM and exits 0 left partial outputs
1036
+ // a pass would save (X-125), as the local runner's own `timedOut` says.
1037
+ // A local request carries no timer, so the runner alone decides there.
1038
+ if (timeoutFired && res.timedOut !== true) res = { ...res, timedOut: true }
1016
1039
  violations = [...res.violations]
1017
1040
  // A denial under a dependency the key does not answer for says only
1018
1041
  // ENOENT; the line beside it names the package and how to key it. Only
@@ -1141,13 +1164,33 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
1141
1164
  args.executor.remote === true ? 'none' : undeclaredWriteReach(node, args.workspaceRoot)
1142
1165
  const writesFingerprint =
1143
1166
  args.executor.remote !== true && mayWriteFingerprint(node, args.workspaceRoot)
1167
+ // A stop that landed during the awaits above (the key, the probe, the
1168
+ // input description): the first attempt would wipe the last build and
1169
+ // hand the executor a request after the run had stopped. Through a call,
1170
+ // so the loop's own reads of the stop are not narrowed to false.
1171
+ if (isAborted(args.stopSignal)) {
1172
+ return {
1173
+ node,
1174
+ status: 'aborted',
1175
+ exitCode: signalExitCode(forwardedSignal(args.stopSignal?.reason)),
1176
+ durationMs: 0,
1177
+ hash,
1178
+ wallclockStartNs,
1179
+ wallclockEndNs: process.hrtime.bigint() - args.runStartHrTimeNs,
1180
+ }
1181
+ }
1144
1182
  for (;;) {
1145
1183
  attempt++
1146
1184
  const a = await runAttempt()
1147
- forgetUndeclaredWrites(args, writeReach)
1148
- if (writesFingerprint) args.fingerprintWatch?.wrote()
1149
1185
  result = a.result
1150
1186
  effectiveExitCode = a.exitCode
1187
+ if (a.withdrawn) {
1188
+ attempt--
1189
+ failedAttempts.pop()
1190
+ break
1191
+ }
1192
+ forgetUndeclaredWrites(args, writeReach)
1193
+ if (writesFingerprint) args.fingerprintWatch?.wrote()
1151
1194
  spentMs += result.durationMs
1152
1195
 
1153
1196
  // An attempt that ended in a shutdown never finished on its own terms —
@@ -1196,7 +1239,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
1196
1239
  )
1197
1240
  }
1198
1241
 
1199
- async function buildRequest(): Promise<ExecuteRequest> {
1242
+ async function buildRequest(local: boolean): Promise<ExecuteRequest> {
1200
1243
  const out = secrets && maskedEmitter(secrets, (t) => log.taskStdout(node, t))
1201
1244
  const err = secrets && maskedEmitter(secrets, (t) => log.taskStderr(node, t))
1202
1245
  flushMasked = () => {
@@ -1234,10 +1277,10 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
1234
1277
  outputs: { files: outputs, workspaceFiles: wsOutputs },
1235
1278
  ...(args.terminal === true ? { terminal: true as const } : {}),
1236
1279
  }
1237
- // The signal last: it arms `exec.timeout`, and the sandbox's arming
1280
+ // The signal last: it arms a plugin's `exec.timeout`, and the sandbox's arming
1238
1281
  // below is vx's work, not the task's. Armed first, a 60 ms timeout
1239
1282
  // expired before the spawn and failed `echo` as timed out, unrun.
1240
- if (!userSandbox) return { ...base, signal: requestSignal() }
1283
+ if (!userSandbox) return { ...base, signal: requestSignal(local) }
1241
1284
  await args.armSandbox?.()
1242
1285
  const sb = await sandboxRequestFor(
1243
1286
  node,
@@ -1249,7 +1292,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
1249
1292
  )
1250
1293
  placeholders = sb.placeholders
1251
1294
  withheld = sb.withheld
1252
- return { ...base, sandbox: sb.sandbox, signal: requestSignal() }
1295
+ return { ...base, sandbox: sb.sandbox, signal: requestSignal(local) }
1253
1296
  }
1254
1297
 
1255
1298
  const wallclockEndNs = process.hrtime.bigint() - args.runStartHrTimeNs
@@ -1260,12 +1303,15 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
1260
1303
  // record (a row with no artifact) is exactly the corrupt-entry shape
1261
1304
  // `restoreOutputs` refuses, so writing none is the only clean answer.
1262
1305
  // The closure is what a later local consumer — or a later eager run —
1263
- // pulls the bytes with.
1306
+ // pulls the bytes with. Its fetch saves them under `hash` only past the
1307
+ // eager save's check: an input edited before the describe filed the
1308
+ // edit's bytes under the old key (X-123).
1264
1309
  const materialize = result.outputs?.kind === 'deferred' ? result.outputs.materialize : undefined
1265
1310
  if (materialize !== undefined) {
1311
+ const keyed = await keyStillTrue()
1266
1312
  args.deferred?.register(node.id, {
1267
1313
  materialize,
1268
- hash,
1314
+ ...(keyed ? { hash } : {}),
1269
1315
  entry: {
1270
1316
  taskId: node.id,
1271
1317
  command: storedCommand,
@@ -21,32 +21,40 @@ export type FailureMode = 'stable' | 'flaky-recoverable' | 'flaky-fatal'
21
21
 
22
22
  /**
23
23
  * Per-key outcome projection: every keyed (project, task, hash) under
24
- * `source` (a relation with `project`, `task`, `hash`, `status`, `cache_hit`
25
- * columns) with how often it failed and how often it passed — a pass being
26
- * an executed success or a cache hit, which replays a success. Every
24
+ * `source` (`runs`, or a relation with its `id`, `project`, `task`, `hash`,
25
+ * `status`, `cache_hit` columns) with how often it failed, how often it
26
+ * passed — an executed success or a cache hit, which replays a success —
27
+ * and how often it failed AFTER its first pass (`relapses`). Every
27
28
  * flakiness reader is a filter over this one projection.
28
29
  */
29
30
  function keyOutcomesSql(source: string, where = ''): string {
30
- return `SELECT project, task, hash,
31
- SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) AS failures,
32
- SUM(CASE WHEN status = 'success' OR status IN ${HIT_STATUSES} OR cache_hit = 1
33
- THEN 1 ELSE 0 END) AS passes
34
- FROM ${source}
35
- WHERE ${KEYED_RUNS_SQL}${where}
31
+ return `SELECT project, task, hash, SUM(failed) AS failures, SUM(passed) AS passes,
32
+ SUM(retried) AS retried_failures,
33
+ SUM(CASE WHEN failed = 1 AND id > first_pass THEN 1 ELSE 0 END) AS relapses
34
+ FROM (SELECT project, task, hash, id, failed, passed, retried,
35
+ MIN(CASE WHEN passed = 1 THEN id END) OVER (PARTITION BY project, task, hash) AS first_pass
36
+ FROM (SELECT project, task, hash, id,
37
+ CASE WHEN status = 'failed' THEN 1 ELSE 0 END AS failed,
38
+ CASE WHEN status = 'success' AND attempts > 1 THEN attempts - 1 ELSE 0 END AS retried,
39
+ CASE WHEN status = 'success' OR status IN ${HIT_STATUSES} OR cache_hit = 1
40
+ THEN 1 ELSE 0 END AS passed
41
+ FROM ${source}
42
+ WHERE ${KEYED_RUNS_SQL}${where}))
36
43
  GROUP BY project, task, hash`
37
44
  }
38
45
 
39
46
  /**
40
47
  * Per-key outcome subquery: the distinct cache keys under `source` that
41
- * produced BOTH a failure and a success — the definitional flake: identical
42
- * inputs, different outcomes. A failure whose key never succeeded is a
43
- * legitimate break (a changed input that fails), which belongs to the
44
- * regressions surface, not flakiness. One projection of the rule serves the
48
+ * FAILED AFTER THEY HAD PASSED — the definitional flake: identical inputs,
49
+ * a green result, then a red one. A failure whose key never succeeded is a
50
+ * legitimate break (a changed input that fails), and failures that end in
51
+ * a first pass are a recovery (a missing tool installed, a service brought
52
+ * up): neither is flakiness. One projection of the rule serves the
45
53
  * all-time query below and the windowed one in `history.ts`.
46
54
  */
47
55
  export function mixedOutcomeKeysSql(source: string, where = ''): string {
48
56
  return `SELECT project, task FROM (${keyOutcomesSql(source, where)})
49
- WHERE failures > 0 AND passes > 0`
57
+ WHERE relapses > 0`
50
58
  }
51
59
 
52
60
  /** Mixed-outcome keys of one (project, task) over its WHOLE recorded history. */
@@ -61,7 +69,7 @@ export function mixedOutcomeKeyCount(db: Database, project: string, task: string
61
69
 
62
70
  /**
63
71
  * The verdict. Flaky requires a NONDETERMINISM signal: a within-run retry,
64
- * or a cache key that both failed and succeeded (`mixedKeys`). Failures
72
+ * or a cache key that failed after it had passed (`mixedKeys`). Failures
65
73
  * alone — each on its own key — are legitimate breaks, however many there
66
74
  * are. `mixedKeys` is a thunk so a task that never failed is stable without
67
75
  * the key count being computed at all.
@@ -118,9 +126,9 @@ const keyOf = (c: { project: string; task: string; hash: string }): string =>
118
126
 
119
127
  /**
120
128
  * Which of this run's executed tasks are flaky, judged BEFORE the run's own
121
- * rows are recorded: a pass on a key that has failed before, a failure on
122
- * a key that has passed before, or a within-run retry. The counts fold this
123
- * run in.
129
+ * rows are recorded: a failure on a key that has passed before, a pass on a
130
+ * key that already did so, or a within-run retry. A first pass after only
131
+ * failures is a recovery, not a flake. The counts fold this run in.
124
132
  *
125
133
  * The cost follows the run's colour. No candidate (every task a hit or a
126
134
  * skip): no query. A green miss: one probe of the `runs_failed` partial
@@ -146,7 +154,7 @@ export function detectFlaky(db: Database, candidates: readonly FlakyCandidate[])
146
154
  (c) => c.status === 'failed' || c.attempts > 1 || everFailed.has(keyOf(c)),
147
155
  )
148
156
  if (suspects.length === 0) return []
149
- const counts = new Map<string, { passes: number; failures: number }>()
157
+ const counts = new Map<string, { passes: number; failures: number; relapses: number }>()
150
158
  chunked(suspects, (chunk) => {
151
159
  const rows = db
152
160
  .query(keyOutcomesSql('runs', ` AND hash IN (${chunk.map(() => '?').join(', ')})`))
@@ -156,27 +164,30 @@ export function detectFlaky(db: Database, candidates: readonly FlakyCandidate[])
156
164
  hash: string
157
165
  passes: number
158
166
  failures: number
167
+ relapses: number
159
168
  }[]
160
- for (const r of rows) counts.set(keyOf(r), { passes: r.passes, failures: r.failures })
169
+ for (const r of rows)
170
+ counts.set(keyOf(r), { passes: r.passes, failures: r.failures, relapses: r.relapses })
161
171
  })
162
172
  const out: FlakyFinding[] = []
163
173
  for (const c of suspects) {
164
- const before = counts.get(keyOf(c)) ?? { passes: 0, failures: 0 }
174
+ const before = counts.get(keyOf(c)) ?? { passes: 0, failures: 0, relapses: 0 }
175
+ const relapsed = c.status === 'failed' ? before.passes > 0 : before.relapses > 0
176
+ if (!relapsed && c.attempts <= 1) continue
165
177
  const passes = before.passes + (c.status === 'success' ? 1 : 0)
166
178
  const failures = before.failures + (c.status === 'failed' ? 1 : 0)
167
- if (!(passes > 0 && failures > 0) && c.attempts <= 1) continue
168
179
  out.push({ ...c, taskId: `${c.project}#${c.task}`, passes, failures })
169
180
  }
170
181
  return out
171
182
  }
172
183
 
173
- /** One task the recorded history shows both passing and failing on unchanged inputs. */
184
+ /** One task the recorded history shows failing on a key that had passed. */
174
185
  export interface FlakyTask {
175
186
  /** `project#task`. */
176
187
  taskId: string
177
188
  project: string
178
189
  task: string
179
- /** Distinct cache keys that both passed and failed. */
190
+ /** Distinct cache keys that failed after they had passed. */
180
191
  keys: number
181
192
  /** Outcomes over those keys. */
182
193
  passes: number
@@ -184,16 +195,20 @@ export interface FlakyTask {
184
195
  }
185
196
 
186
197
  /**
187
- * Every task with a mixed-outcome key anywhere in the retained history
198
+ * Every task with a key that failed after it had passed anywhere in the retained history
188
199
  * (30 days, `RunHistory.pruneOlderThan`), most failures first: the
189
- * doctor's list. A full scan of `runs`, which an inspection verb affords.
200
+ * doctor's list. A pass that took a retry mixes its key on its own: it is
201
+ * one `success` row whose failed attempts count as failures (`vx info`
202
+ * said `none` over one). A full scan of `runs`, which an inspection verb
203
+ * affords.
190
204
  */
191
205
  export function flakyTasks(db: Database): FlakyTask[] {
192
206
  const rows = db
193
207
  .query(
194
208
  `SELECT project, task, COUNT(*) AS keys, SUM(passes) AS passes, SUM(failures) AS failures
195
- FROM (${keyOutcomesSql('runs')})
196
- WHERE failures > 0 AND passes > 0
209
+ FROM (SELECT project, task, passes, failures + retried_failures AS failures
210
+ FROM (${keyOutcomesSql('runs')})
211
+ WHERE relapses > 0 OR retried_failures > 0)
197
212
  GROUP BY project, task
198
213
  ORDER BY failures DESC, passes DESC, project, task`,
199
214
  )
@@ -27,10 +27,10 @@
27
27
 
28
28
  import { withForwardArgs } from '../exec/index.js'
29
29
  import { isGroupTask, type TaskNode, type TaskOutcome } from '../graph/index.js'
30
- import { maskedCommand } from '../util/index.js'
30
+ import { formatElapsed, maskedCommand } from '../util/index.js'
31
31
  import { paint, type ColorSupport } from './colors.js'
32
- import { formatDuration, neverStarted } from './summary.js'
33
- import { outcomeLabel, skippedReason } from './events.js'
32
+ import { neverStarted } from './summary.js'
33
+ import { outcomeLabel, ranNoCache, skippedReason } from './events.js'
34
34
 
35
35
  const NO_COLOR: ColorSupport = { enabled: false }
36
36
 
@@ -220,7 +220,7 @@ const CACHE_COL = 8 // "no-cache" / "remote" / "local" / "fresh" / "miss"
220
220
 
221
221
  /** Right-align the duration in a TIME_COL cell (pad on the left). */
222
222
  function timeCell(ms: number | null, colors: ColorSupport): string {
223
- const raw = ms === null ? '' : formatDuration(ms)
223
+ const raw = ms === null ? '' : formatElapsed(ms)
224
224
  return ' '.repeat(Math.max(0, TIME_COL - raw.length)) + paint('', raw, colors, { dim: true })
225
225
  }
226
226
 
@@ -274,8 +274,8 @@ function cacheOf(o: TaskOutcome): { word: string; color: string } {
274
274
  switch (o.status) {
275
275
  case 'success':
276
276
  case 'failed':
277
- // A task with no `cache` block never consulted the cache: not a miss.
278
- return { word: o.node.config.cache === undefined ? 'no-cache' : 'miss', color: '' } // dim
277
+ // A task no cache answered for never consulted one: not a miss.
278
+ return { word: ranNoCache(o) ? 'no-cache' : 'miss', color: '' } // dim
279
279
  case 'cache-hit':
280
280
  return o.restored === false
281
281
  ? { word: 'fresh', color: SUCCESS }
@@ -487,7 +487,7 @@ export function formatFrameClose(
487
487
  // both ends of the frame as a long-lived task.
488
488
  const tail =
489
489
  persistent && outcome.status === 'success'
490
- ? ` ${paint('', `(${formatDuration(outcome.durationMs)})`, colors, { dim: true })} ${paint(ACCENT, 'running', colors)}`
490
+ ? ` ${paint('', `(${formatElapsed(outcome.durationMs)})`, colors, { dim: true })} ${paint(ACCENT, 'running', colors)}`
491
491
  : formatBlockFooter(outcome, colors)
492
492
  const close = `${corner('└─')} ${mark}${paintTaskId(node, colors, { bold: true })} ${corner('──')}${tail}`
493
493
  const violations = violationSection(outcome, colors)
@@ -594,7 +594,7 @@ function formatBlockFooter(o: TaskOutcome, colors: ColorSupport): string {
594
594
  // saved" on the strength of it. The stored time lives on
595
595
  // `TaskOutcome.storedDurationMs`.) Status differs by outcome — see
596
596
  // formatStatusTag.
597
- const dur = paint('', `(${formatDuration(o.durationMs)})`, colors, { dim: true })
597
+ const dur = paint('', `(${formatElapsed(o.durationMs)})`, colors, { dim: true })
598
598
  const tag = formatStatusTag(o, colors)
599
599
  return ` ${dur} ${tag}${flakyNote(o, colors)}`
600
600
  }
@@ -30,7 +30,6 @@ export { defaultLogger, resolveOutputView } from './logger.js'
30
30
  export { detectColors, paint, type ColorSupport } from './colors.js'
31
31
  export type { Logger, OutputView } from './logger.js'
32
32
  export type { RunPlan, PlannedTask, PlanPrediction, CacheStatus } from './plan.js'
33
- export { formatDuration } from './summary.js'
34
33
  export {
35
34
  createEventBus,
36
35
  wireForwarder,
@@ -93,9 +93,12 @@ export function foldedDeps(node: TaskNode, nodeOf: (id: string) => TaskNode): Fo
93
93
  * the hashes are. An exec task's key folds its own id (`task:<id>`), so its
94
94
  * id is enough; a group's hash is a roll-up of its members' ids and hashes
95
95
  * with no id of its own, so two groups over the same members hash alike.
96
+ * The members are `keyUpstream`'s, which `computeGroupHash` is handed: under
97
+ * `--exclude-dependencies` `node.deps` holds order-only edges and lacks the
98
+ * dropped ones, and two groups the key told apart shared a unit.
96
99
  */
97
100
  function foldUnit(node: TaskNode): string {
98
- return isGroupTask(node) && node.config.cache === undefined
99
- ? `group|${[...node.deps].sort().join('|')}`
100
- : `task|${node.id}`
101
+ if (!isGroupTask(node) || node.config.cache !== undefined) return `task|${node.id}`
102
+ const members = [...keyedDeps(node), ...(node.excludedUpstream ?? []).map((u) => u.node.id)]
103
+ return `group|${members.sort().join('|')}`
101
104
  }
@@ -19,11 +19,18 @@ import {
19
19
  type StatusStream,
20
20
  type WorkerSlot,
21
21
  } from './status-line.js'
22
- import { formatDuration, formatSummarySection, neverStarted, type RunContext } from './summary.js'
22
+ import { formatSummarySection, neverStarted, type RunContext } from './summary.js'
23
23
  import { isGroupTask } from '../graph/index.js'
24
- import { appendTail, createTail, resetTail, tailText, type Tail } from '../util/index.js'
24
+ import {
25
+ appendTail,
26
+ createTail,
27
+ formatElapsed,
28
+ resetTail,
29
+ tailText,
30
+ type Tail,
31
+ } from '../util/index.js'
25
32
  import { isCacheHit } from './telemetry.js'
26
- import { failedLabel, outcomeWord } from './events.js'
33
+ import { failedLabel, outcomeWord, ranNoCache } from './events.js'
27
34
 
28
35
  export interface Logger {
29
36
  /** Header / footer / status text. Written verbatim, one trailing \n added. */
@@ -139,8 +146,7 @@ function ghaFence(body: string, token: string): string {
139
146
  return `::stop-commands::${token}\n${body}::${token}::\n`
140
147
  }
141
148
 
142
- const cacheWordOf = (n: TaskNode): 'miss' | 'no-cache' =>
143
- n.config.cache === undefined ? 'no-cache' : 'miss'
149
+ const cacheWordOf = (o: TaskOutcome): 'miss' | 'no-cache' => (ranNoCache(o) ? 'no-cache' : 'miss')
144
150
 
145
151
  export function resolveOutputView(
146
152
  options: {
@@ -440,7 +446,7 @@ export function defaultLogger(
440
446
  (view.mode === 'focused' && isPrimary(node) && !isGroupTask(node) && requestedCount <= 1)
441
447
 
442
448
  const failureRow = (node: TaskNode, outcome: TaskOutcome): string =>
443
- formatFailureLine(node.id, outcome.durationMs, colors, cacheWordOf(node)) +
449
+ formatFailureLine(node.id, outcome.durationMs, colors, cacheWordOf(outcome)) +
444
450
  flakyNote(outcome, colors)
445
451
 
446
452
  return {
@@ -453,6 +459,12 @@ export function defaultLogger(
453
459
  // A kept server's held partial line is older than this line: held
454
460
  // past it, a server's last words printed below its own exit notice.
455
461
  flushKept()
462
+ // A live-streamed server keeps writing after its frame closes, so a
463
+ // summary row or "exited with code" can follow its partial line.
464
+ if (streamMidLine) {
465
+ writer.write('\n')
466
+ streamMidLine = false
467
+ }
456
468
  writer.write(`${line}\n`)
457
469
  },
458
470
  runStart(info) {
@@ -654,7 +666,7 @@ export function defaultLogger(
654
666
  done++
655
667
  if (outcome.status === 'failed') {
656
668
  failed++
657
- if (node.config.cache === undefined) noCache++
669
+ if (ranNoCache(outcome)) noCache++
658
670
  } else if (node.config.exec?.persistent !== undefined && outcome.status === 'success') {
659
671
  // A persistent task's outcome arrives at READY; the child
660
672
  // keeps running until the orchestrator SIGTERMs it at run
@@ -678,7 +690,7 @@ export function defaultLogger(
678
690
  switch (outcome.status) {
679
691
  case 'success':
680
692
  succeeded++
681
- if (node.config.cache === undefined) noCache++
693
+ if (ranNoCache(outcome)) noCache++
682
694
  break
683
695
  case 'cache-hit':
684
696
  if (outcome.restored === false) upToDate++
@@ -875,7 +887,7 @@ export function defaultLogger(
875
887
  } else {
876
888
  emitBlock(
877
889
  `::group::${ghaData(
878
- `${node.id} (${outcomeWord(outcome)} ${formatDuration(outcome.durationMs)})`,
890
+ `${node.id} (${outcomeWord(outcome)} ${formatElapsed(outcome.durationMs)})`,
879
891
  )}\n${body}::endgroup::\n`,
880
892
  )
881
893
  }
@@ -407,11 +407,18 @@ function unchangedKeyNote(
407
407
  at: number
408
408
  } | null
409
409
  )?.at
410
- // Both runs executed the task and succeeded with the cache writable, no
411
- // entry holds the key, and each ran beside a failure: the continue-taint
412
- // (admission.ts), which runs a task downstream of a failure and never
413
- // saves it. Blaming `--no-cache` / `--force` named a flag nobody passed.
414
- if (createdAt === undefined && this_.status === 'success' && prev.status === 'success') {
410
+ // The previous run executed the task and succeeded with the cache
411
+ // writable beside a failure, and no entry older than this run holds the
412
+ // key: the continue-taint (admission.ts), which runs a task downstream of
413
+ // a failure and never saves it. Blaming `--no-cache` / `--force` named a
414
+ // flag nobody passed, and a prune when this run saved the key named an
415
+ // eviction that never happened.
416
+ if (
417
+ (createdAt === undefined || createdAt >= this_.startedAt) &&
418
+ this_.status === 'success' &&
419
+ prev.status === 'success' &&
420
+ prev.cacheHit === 0
421
+ ) {
415
422
  const failedWithWrites = (id: string | null): number => {
416
423
  if (id === null) return 0
417
424
  const row = db
@@ -420,9 +427,13 @@ function unchangedKeyNote(
420
427
  const writes = row?.p.split(',').some((axis) => axis === 'lW' || axis === 'rW') === true
421
428
  return writes ? (row?.f ?? 0) : 0
422
429
  }
423
- const now = failedWithWrites(runId)
424
- if (prev.cacheHit === 0 && now > 0 && failedWithWrites(prev.runId) > 0) {
425
- return `cache key unchanged — neither run saved it: each ran beside a failed task (${now} failed in this one), and a task run past a failed dependency (--continue) is never cached`
430
+ const before = failedWithWrites(prev.runId)
431
+ if (before > 0) {
432
+ const now = failedWithWrites(runId)
433
+ if (createdAt === undefined && now > 0) {
434
+ return `cache key unchanged — neither run saved it: each ran beside a failed task (${now} failed in this one), and a task run past a failed dependency (--continue) is never cached`
435
+ }
436
+ return `cache key unchanged — the previous run on this key ran beside a failed task (${before} failed in it) and was not saved: a task run past a failed dependency (--continue) is never cached, so there was nothing to hit`
426
437
  }
427
438
  }
428
439
  if (createdAt !== undefined && createdAt >= this_.startedAt) {