omp-conductor 0.20.0 → 0.20.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/settlement.ts CHANGED
@@ -25,7 +25,15 @@ import {
25
25
  analyseSettlement,
26
26
  deriveChangedLine,
27
27
  } from "./diff-flags.ts";
28
- import { SPINNING_CAP_CLASSES, classifyRun, normalise, type ClassifyFacts } from "./failure-class.ts";
28
+ import {
29
+ EFFECTIVE_BUDGET_SAMPLE_RUNS,
30
+ SPINNING_CAP_CLASSES,
31
+ COMPOSE_DEPENDENCY_STARTUP_SIGNATURE,
32
+ classifyRun,
33
+ normalise,
34
+ observeTurnBudget,
35
+ type ClassifyFacts,
36
+ } from "./failure-class.ts";
29
37
  import { GhPrMissingError } from "./tracker/github.ts";
30
38
  import { formatModelsTried, modelsTried } from "./model-fallback.ts";
31
39
  import { PR_LOOKUP_WINDOW_MS } from "./decisions.ts";
@@ -1510,6 +1518,13 @@ export async function classifyAndRecover(d: SettlementDeps): Promise<number> {
1510
1518
  // merged is settled here when the settle sweep could not establish identity
1511
1519
  // (#497). Every other exit returns 0.
1512
1520
  let settled = 0;
1521
+ // One read for the whole sweep (#1063): the completed-run sample the
1522
+ // effective turn budget is derived from. Each row is then judged against the
1523
+ // budget *excluding itself*, so a just-killed run's own extreme pace cannot
1524
+ // drag the average it is being explained against.
1525
+ const latencySamples = store.recentLatencySamples(project.name, EFFECTIVE_BUDGET_SAMPLE_RUNS);
1526
+ const observedBudgetFor = (runId: string) =>
1527
+ observeTurnBudget(latencySamples, caps, project.workerModel, runId);
1513
1528
  for (const run of store.runsNeedingClassification(project.name, CLASSIFY_BATCH)) {
1514
1529
  const facts: ClassifyFacts = {};
1515
1530
  let classifiedRun = run;
@@ -1558,6 +1573,26 @@ export async function classifyAndRecover(d: SettlementDeps): Promise<number> {
1558
1573
  if (firstFailure?.link !== undefined) {
1559
1574
  facts.failingLog = await tracker.checkLog(firstFailure.link);
1560
1575
  }
1576
+ // A Compose dependency-startup failure ("docker compose up" aborted
1577
+ // because a dependency container failed its healthcheck) is only
1578
+ // outside the diff when the PR touched no container configuration —
1579
+ // a PR that breaks its own `docker-compose.yml` produces this exact
1580
+ // sentence and must still charge its attempt (#1059). So the
1581
+ // classifier needs the PR's changed-file list, derived from the
1582
+ // diff this settlement already knows how to fetch
1583
+ // (`Tracker.prDiff`, the same call the settlement audit uses) —
1584
+ // one fetch, and only for a row whose log actually carries
1585
+ // Compose's sentence; every other row pays nothing. A diff that
1586
+ // cannot be read or was cut short (`truncated`) stays undefined: an
1587
+ // unknown list must never waive an attempt.
1588
+ if (
1589
+ facts.failingLog !== undefined &&
1590
+ facts.failingLog.toLowerCase().includes(COMPOSE_DEPENDENCY_STARTUP_SIGNATURE)
1591
+ ) {
1592
+ const diff = await tracker.prDiff(run.prUrl);
1593
+ facts.changedFiles =
1594
+ diff === undefined || diff.truncated ? undefined : diff.files.map((f) => f.path);
1595
+ }
1561
1596
  }
1562
1597
  }
1563
1598
  } catch (err) {
@@ -1579,8 +1614,12 @@ export async function classifyAndRecover(d: SettlementDeps): Promise<number> {
1579
1614
  continue;
1580
1615
  }
1581
1616
  }
1582
-
1583
- const { cls, recovery, evidence } = classifyRun(classifiedRun, facts, caps);
1617
+ const { cls, recovery, evidence } = classifyRun(
1618
+ classifiedRun,
1619
+ facts,
1620
+ caps,
1621
+ observedBudgetFor(run.id),
1622
+ );
1584
1623
 
1585
1624
  // A healthy green PR is not a failure of any class. Leaving the row
1586
1625
  // unclassified is what keeps it eligible for the sweep on the tick where its
@@ -1669,9 +1708,27 @@ async function recoverRun(
1669
1708
  // candidate admission can never accept — only the failed label comes off,
1670
1709
  // and the exhaustion reaches a human (#490, #348).
1671
1710
  if (cls === "wall-clock-cap-progress") {
1711
+ // Same split the requeue branch applies (#1080): an issue the tracker
1712
+ // reports closed was resolved by another route — a duplicate, a
1713
+ // supersede, a decomposition — so continuing it would dispatch work
1714
+ // nobody asked for. That verdict is terminal: release the lifecycle
1715
+ // label(s) the row could be holding (the kill path swaps in-progress
1716
+ // for failed; a crash between those two writes leaves either) and
1717
+ // stamp `recoveredAt`, or `selectUnclassified` re-offers this row on
1718
+ // every pass forever (#1095). An unreadable answer is the tracker
1719
+ // failing to answer, not answering no — it stays transient and retries.
1672
1720
  const state = await tracker.issueState(run.issue).catch(() => undefined);
1721
+ if (state === "closed") {
1722
+ store.enqueueLabelOps(project.name, [
1723
+ { issue: run.issue, op: "remove", label: project.stateLabels.failed },
1724
+ { issue: run.issue, op: "remove", label: inProgress },
1725
+ ]);
1726
+ store.updateRun(run.id, { recoveredAt: Date.now() });
1727
+ log(`#${run.issue} not continued from ${cls}: issue is closed`);
1728
+ return;
1729
+ }
1673
1730
  if (state !== "open") {
1674
- log(`#${run.issue} not continued from ${cls}: issue is ${state ?? "unreadable"}`);
1731
+ log(`#${run.issue} not continued from ${cls}: issue is unreadable (retrying)`);
1675
1732
  return;
1676
1733
  }
1677
1734
  const continuation = store.continuationsFor(project.name, run.issue);
@@ -1706,6 +1763,42 @@ async function recoverRun(
1706
1763
  return;
1707
1764
  }
1708
1765
 
1766
+ // `progress-stall`: the watch settled a hung session (#1086), which keeps
1767
+ // the in-progress label held the way an orphan row does. Work to continue
1768
+ // from hands that label back for a continuation brief; a budget already
1769
+ // spent releases the issue to a human instead of re-offering a candidate
1770
+ // admission can never accept — the same ceiling the orphan path honours.
1771
+ if (cls === "progress-stall") {
1772
+ const state = await tracker.issueState(run.issue).catch(() => undefined);
1773
+ if (state !== "open") {
1774
+ log(`#${run.issue} not continued from ${cls}: issue is ${state ?? "unreadable"}`);
1775
+ return;
1776
+ }
1777
+ const continuation = store.continuationsFor(project.name, run.issue);
1778
+ if (hasContinuationBudget(continuation, caps.maxContinuationsPerIssue)) {
1779
+ await swapToQueue(d, run.issue, inProgress);
1780
+ store.updateRun(run.id, { recoveredAt: Date.now() });
1781
+ log(`#${run.issue} requeued after a progress stall: ${evidence}`);
1782
+ } else {
1783
+ store.enqueueLabelOps(project.name, [{ issue: run.issue, op: "remove", label: inProgress }]);
1784
+ store.updateRun(run.id, { recoveredAt: Date.now() });
1785
+ await safeEscalate(d, {
1786
+ tier: 1,
1787
+ project: project.name,
1788
+ issue: run.issue,
1789
+ summary: `#${run.issue} exhausted its continuation budget on progress stalls`,
1790
+ detail: [
1791
+ `Attempt ${run.attempt} was settled silent with work to continue from, but ${continuation} continuation(s) are already charged against a cap of ${caps.maxContinuationsPerIssue}.`,
1792
+ evidence,
1793
+ `Work to continue: branch ${run.branch} at ${run.headSha ?? run.salvageSha}${run.prUrl === undefined ? "" : ` — ${run.prUrl}`}.`,
1794
+ "Sessions on this issue keep hanging, so another run would burn a worker slot for the same outcome. Re-scope it, raise maxContinuationsPerIssue for it, or finish the remaining work by hand.",
1795
+ ].join("\n"),
1796
+ });
1797
+ log(`#${run.issue} not continued from ${cls}: continuation budget exhausted`);
1798
+ }
1799
+ return;
1800
+ }
1801
+
1709
1802
  // `model-empty-stop`: the provider answered with empty turns until the
1710
1803
  // harness ended the session. The work is not at fault and the row is not a
1711
1804
  // failed attempt (the counters exclude the class), so this hands the queue
@@ -1714,9 +1807,22 @@ async function recoverRun(
1714
1807
  // wall-clock path is: a provider stuck in that state must reach a human
1715
1808
  // instead of consuming the issue forever, one free retry at a time.
1716
1809
  if (cls === "model-empty-stop") {
1810
+ // Same split as the wall-clock branch above (#1080, #1095): a closed
1811
+ // issue is terminal — release the lifecycle label(s), stamp
1812
+ // `recoveredAt`, stop being offered — while an unreadable tracker keeps
1813
+ // the row unstamped and retrying on a later pass.
1717
1814
  const state = await tracker.issueState(run.issue).catch(() => undefined);
1815
+ if (state === "closed") {
1816
+ store.enqueueLabelOps(project.name, [
1817
+ { issue: run.issue, op: "remove", label: project.stateLabels.failed },
1818
+ { issue: run.issue, op: "remove", label: inProgress },
1819
+ ]);
1820
+ store.updateRun(run.id, { recoveredAt: Date.now() });
1821
+ log(`#${run.issue} not continued from ${cls}: issue is closed`);
1822
+ return;
1823
+ }
1718
1824
  if (state !== "open") {
1719
- log(`#${run.issue} not continued from ${cls}: issue is ${state ?? "unreadable"}`);
1825
+ log(`#${run.issue} not continued from ${cls}: issue is unreadable (retrying)`);
1720
1826
  return;
1721
1827
  }
1722
1828
  const continuation = store.continuationsFor(project.name, run.issue);
@@ -1845,11 +1951,22 @@ async function recoverRun(
1845
1951
  return;
1846
1952
  }
1847
1953
  // Only when the tracker still shows this issue as ours to hand back. An
1848
- // issue that is closed, or has no state label, was resolved by another route
1849
- // and requeueing it would dispatch work nobody asked for.
1954
+ // issue that is closed was resolved by another route and requeueing it
1955
+ // would dispatch work nobody asked for — so that verdict is terminal
1956
+ // (#1080): release the dispatcher-owned in-progress label and stamp
1957
+ // `recoveredAt`, exactly like the operator-withdrawal branch below, or the
1958
+ // sweep re-offers this row on every pass forever. An unreadable answer is
1959
+ // a different thing in kind — the tracker failing to answer, not answering
1960
+ // no — so it stays transient and keeps retrying.
1850
1961
  const state = await tracker.issueState(run.issue).catch(() => undefined);
1962
+ if (state === "closed") {
1963
+ store.enqueueLabelOps(project.name, [{ issue: run.issue, op: "remove", label: inProgress }]);
1964
+ store.updateRun(run.id, { recoveredAt: Date.now() });
1965
+ log(`#${run.issue} not requeued from ${cls}: issue is closed`);
1966
+ return;
1967
+ }
1851
1968
  if (state !== "open") {
1852
- log(`#${run.issue} not requeued from ${cls}: issue is ${state ?? "unreadable"}`);
1969
+ log(`#${run.issue} not requeued from ${cls}: issue is unreadable (retrying)`);
1853
1970
  return;
1854
1971
  }
1855
1972
  // A clean orphan whose queue label is already absent is a deliberate
@@ -1990,10 +2107,14 @@ async function recoverRun(
1990
2107
  return;
1991
2108
  }
1992
2109
 
1993
- // `hold` (orphan-dirty) and `none`: recorded, nothing performed. The existing
1994
- // unsalvaged-WIP admission hold already fails dispatch closed until an
1995
- // operator acknowledges the tree, which is the only safe move when the
1996
- // worktree holds the only copy of real work.
2110
+ // `observe` (awaiting-observation), `hold` (orphan-dirty) and `none`:
2111
+ // recorded, nothing performed here. `observe`'s recovery IS the later observation —
2112
+ // the settle sweep re-offers the blocked row until its PR resolves and settles it
2113
+ // the way it settles `settlement-stuck`, so no action exists to perform today and
2114
+ // `recoveredAt` stays NULL by design (#1068). The existing unsalvaged-WIP
2115
+ // admission hold already fails dispatch closed until an operator acknowledges
2116
+ // the tree, which is the only safe move when the worktree holds the only copy
2117
+ // of real work.
1997
2118
  }
1998
2119
 
1999
2120
  /**
@@ -254,12 +254,14 @@ const MECHANICAL_GROOMING_REASONS: Record<string, true> = {
254
254
  };
255
255
 
256
256
  /**
257
- * The to-spec refusal classes persisted as blocked rows (#772) — a result
258
- * that failed validation is a mechanical block, never a verdict. Mirrors the
259
- * failure kinds of `ToSpecFailure` in `to-spec.ts`.
257
+ * The to-spec refusal classes persisted as blocked rows (#772, #1064) — a
258
+ * result that failed validation, or a pass that produced no answer at all, is
259
+ * a mechanical block, never a verdict. Mirrors the failure kinds of
260
+ * `ToSpecFailure` in `to-spec.ts`.
260
261
  */
261
262
  const REFUSED_GROOMING_REASONS: Record<string, true> = {
262
263
  malformed: true,
264
+ "no-answer": true,
263
265
  "missing-source": true,
264
266
  "stale-source": true,
265
267
  };
@@ -289,10 +291,12 @@ const REFUSED_GROOMING_REASONS: Record<string, true> = {
289
291
  * (`blocked` rows whose reason is a groomer verdict or a product-judgement
290
292
  * label, e.g. `needs-product-decision`).
291
293
  * - `mechanically blocked` — admission's lane/dependency holds.
292
- * - `refused` — to-spec results that failed validation (`malformed`,
293
- * `missing-source`, `stale-source`), told apart from the holds so an
294
- * operator sees whether the runway cannot move or a result cannot be
295
- * trusted.
294
+ * - `refused` — to-spec passes that produced no usable verdict (`malformed`,
295
+ * `no-answer`, `missing-source`, `stale-source`), told apart from the
296
+ * holds so an operator sees whether the runway cannot move or a result
297
+ * cannot be trusted. Each row's line names its own class (`#19 malformed`,
298
+ * `#22 no-answer`), so a run of identical refusals is visible as a pattern
299
+ * rather than a wall of one word.
296
300
  * - `in-flight` — a launched batch is running (#777).
297
301
  * - `operator-parked` — the dispatch snapshot's parked count (#507).
298
302
  *
@@ -486,6 +490,9 @@ export function formatFleetStatus(
486
490
  lastStop: DaemonStop | undefined = undefined,
487
491
  siblings: { project: string; live: number }[] = [],
488
492
  grooming: string | undefined = undefined,
493
+ /** One line naming the flat-chat deliveries this project's stale pin caused
494
+ * (#1094), or undefined when there is nothing outstanding. */
495
+ misrouteNote: string | undefined = undefined,
489
496
  ): string {
490
497
  const tickLine =
491
498
  layers.ticksDetail === undefined
@@ -591,6 +598,7 @@ export function formatFleetStatus(
591
598
  recoveryLine,
592
599
  herdrLine,
593
600
  telegramLine,
601
+ ...(misrouteNote === undefined ? [] : [misrouteNote]),
594
602
  ...(brief === undefined ? [] : [brief]),
595
603
  ...(decisions === undefined ? [] : [decisions]),
596
604
  ...(failureClasses === undefined ? [] : [failureClasses]),
@@ -807,6 +815,18 @@ export function formatReviewCorrections(rounds: readonly ReviewCorrectionRound[]
807
815
  });
808
816
  }
809
817
 
818
+ /**
819
+ * The #1063 effective-turn-budget annotation for the `new worker turns` row:
820
+ * shown whenever observed per-turn latency makes the configured ceiling
821
+ * unreachable at all — the break-even pace is exactly wall clock ÷ max turns,
822
+ * so anything slower is material, anything faster is noise.
823
+ */
824
+ function effectiveTurnBudgetSuffix(s: StatusSnapshot): string {
825
+ const observed = s.workerTurnBudgetObserved;
826
+ if (observed === undefined || observed.effectiveTurns >= s.caps.workerMaxTurns) return "";
827
+ return ` (effective ~${observed.effectiveTurns} at ${observed.minutesPerTurn.toFixed(2)} min/turn, last ${observed.sampleSize} runs)`;
828
+ }
829
+
810
830
 
811
831
  function formatProjectBody(
812
832
  s: StatusSnapshot,
@@ -912,7 +932,7 @@ function formatProjectBody(
912
932
  : `${Math.max(1, Math.round(s.labelOps.oldestAgeMs / 1000))}s`
913
933
  })`,
914
934
  ]),
915
- ` new worker turns ${s.caps.workerMaxTurns}`,
935
+ ` new worker turns ${s.caps.workerMaxTurns}${effectiveTurnBudgetSuffix(s)}`,
916
936
  ...s.turnOverrides.map(
917
937
  ({ issue, maxTurns }) => ` turn override #${issue} → ${maxTurns} (next attempt)`,
918
938
  ),
@@ -1078,6 +1098,15 @@ function formatProjectBody(
1078
1098
  // the phase the line is naming, and a number attributed to the wrong phase
1079
1099
  // is worse than no number. The attempt's own cap remains visible as
1080
1100
  // `N/M turns cumulative`.
1101
+ // Transcript silence (#1086), from the progress watch's recorded write
1102
+ // instant — read off the row, never probed at render time (#919). This
1103
+ // is where a stall is visible forming, before the pass settles it.
1104
+ // Skipped under pause, whose silence is the pause working (#938), and
1105
+ // for rows no longer live, whose transcripts have stopped forever.
1106
+ const silent =
1107
+ paused || r.state !== "running" || r.lastProgressAt === undefined
1108
+ ? ""
1109
+ : ` silent ${formatDownDuration(Math.max(0, now - r.lastProgressAt))}`;
1081
1110
  const progress = paused
1082
1111
  ? ""
1083
1112
  : round !== undefined
@@ -1089,7 +1118,7 @@ function formatProjectBody(
1089
1118
  ).toFixed(1)} turns/min avg` +
1090
1119
  ` projects ${Math.round(
1091
1120
  (r.turns * s.caps.workerWallClockMs) / Math.max(elapsedMs, 1_000),
1092
- )} turns at cap`;
1121
+ )} turns at cap` + silent;
1093
1122
  lines.push(
1094
1123
  ` #${r.issue} ${r.repo} ${state} attempt ${r.attempt} ` +
1095
1124
  `${r.turns}/${r.maxTurns} turns${progress} ${r.spendUsd.toFixed(2)} ${r.branch}` +
package/src/store.ts CHANGED
@@ -221,6 +221,7 @@ const UPDATABLE_COLUMNS: Record<string, true> = {
221
221
  failureCharges: true,
222
222
  continuationCharges: true,
223
223
  terminalEvidence: true,
224
+ lastProgressAt: true,
224
225
  };
225
226
 
226
227
  /** Everything SQLite will accept from us. */
@@ -281,6 +282,8 @@ interface RunRow {
281
282
  graphTools: string | null;
282
283
  failureCharges: number | null;
283
284
  continuationCharges: number | null;
285
+ /** NULL for a row the progress pass has not observed, or one with no transcript yet (#1086). */
286
+ lastProgressAt: number | null;
284
287
  }
285
288
 
286
289
  /** The `base_health` table exactly as SQLite hands it back. */
@@ -887,7 +890,11 @@ CREATE TABLE IF NOT EXISTS runs (
887
890
  model TEXT,
888
891
  graphTools TEXT,
889
892
  failureCharges INTEGER,
890
- continuationCharges INTEGER
893
+ continuationCharges INTEGER,
894
+ -- When the run's transcript was last observed to grow, as epoch ms (#1086).
895
+ -- Written by the daemon's progress watch from the file's own mtime; NULL
896
+ -- reads as "not observed yet", never "no progress".
897
+ lastProgressAt INTEGER
891
898
  );
892
899
  CREATE INDEX IF NOT EXISTS runs_project_issue ON runs (project, issue);
893
900
  CREATE INDEX IF NOT EXISTS runs_project_state ON runs (project, state);
@@ -1752,6 +1759,7 @@ function toRecord(row: RunRow): RunRecord {
1752
1759
  };
1753
1760
  if (row.spendReservedUsd !== null) record.spendReservedUsd = row.spendReservedUsd;
1754
1761
  if (row.sessionFile !== null) record.sessionFile = row.sessionFile;
1762
+ if (row.lastProgressAt !== null) record.lastProgressAt = row.lastProgressAt;
1755
1763
  if (row.workerPid !== null) record.workerPid = row.workerPid;
1756
1764
  if (row.paneId !== null) record.paneId = row.paneId;
1757
1765
  if (row.paneLabel !== null) record.paneLabel = row.paneLabel;
@@ -2321,6 +2329,12 @@ export function openStore(dbPath: string): Store {
2321
2329
  if (!columns.some((column) => column.name === "terminalEvidence")) {
2322
2330
  db.exec("ALTER TABLE runs ADD COLUMN terminalEvidence TEXT");
2323
2331
  }
2332
+ // The progress watch's last-observed transcript write (#1086). Rows written
2333
+ // before the column were never observed, so NULL is the honest reading — no
2334
+ // backfill, because the instant only the file's own mtime holds.
2335
+ if (!columns.some((column) => column.name === "lastProgressAt")) {
2336
+ db.exec("ALTER TABLE runs ADD COLUMN lastProgressAt INTEGER");
2337
+ }
2324
2338
  // Every row written before #132 is unclassified, and NULL is the honest
2325
2339
  // reading of that: the budget counters below deliberately still count an
2326
2340
  // unclassified terminal row exactly as this release's predecessor did, so an
@@ -2686,8 +2700,8 @@ export function openStore(dbPath: string): Store {
2686
2700
  id, project, issue, repo, branch, worktree, state, attempt, turns,
2687
2701
  maxTurns, spendUsd, spendReservedUsd, sessionFile, resumedFromRunId, lane, prUrl, headSha, mergeSha, baseRef,
2688
2702
  baseCheck, baseCheckAt, salvageSha, salvageError, salvageAckAt, startedAt,
2689
- endedAt, lastError, settlementFlags, report, graphTools
2690
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
2703
+ endedAt, lastError, settlementFlags, report, graphTools, lastProgressAt
2704
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
2691
2705
  );
2692
2706
  const selectRun = db.query<RunRow, [string]>(`SELECT * FROM runs WHERE id = ?`);
2693
2707
  const selectGraphToolsObs = db.query<{ graphTools: string }, [string]>(
@@ -2973,6 +2987,17 @@ export function openStore(dbPath: string): Store {
2973
2987
  // "retrying next tick" is a lie and the row is stranded with a class and no
2974
2988
  // action. `hold` is excluded because it is *recorded only* by design — its
2975
2989
  // `recoveredAt` stays NULL forever, and re-offering it would spin the sweep.
2990
+ //
2991
+ // One deliberate exception re-enters the sweep after classification: a
2992
+ // blocked row with a PR. It is the observation half of the
2993
+ // `awaiting-observation` recovery (#1068) — the sweep re-reads the PR every
2994
+ // pass, and the classifier's merged-PR branch names `settlement-stuck` the
2995
+ // way it does for any other stuck row, so a blocked run whose PR later merges
2996
+ // (the #1062 shape: merged at 15:02, row still `blocked`/`question`) settles
2997
+ // instead of listing for Duty 1 triage forever. A genuine question row is
2998
+ // re-offered too and its escalation is not repeated: the notifications ledger
2999
+ // dedupes on the class+run summary of the stable evidence. Rows without a PR
3000
+ // have nothing to observe and are only re-offered by the ordinary clauses.
2976
3001
  const selectUnclassified = db.query<RunRow, [string, number]>(
2977
3002
  `SELECT * FROM runs
2978
3003
  WHERE project = ?
@@ -2980,6 +3005,7 @@ export function openStore(dbPath: string): Store {
2980
3005
  AND (
2981
3006
  failureClass IS NULL
2982
3007
  OR (recoveredAt IS NULL AND recoveryAction IN ('settle', 'continue', 'requeue', 'rerun-checks'))
3008
+ OR (state = 'blocked' AND prUrl IS NOT NULL)
2983
3009
  )
2984
3010
  ORDER BY startedAt DESC, rowid DESC
2985
3011
  LIMIT ?`,
@@ -3058,11 +3084,21 @@ export function openStore(dbPath: string): Store {
3058
3084
  ORDER BY endedAt DESC, rowid DESC
3059
3085
  LIMIT ?`,
3060
3086
  );
3087
+ // Newest runs that can yield a per-turn pace (#1063): rows with turns and
3088
+ // elapsed time recorded. Same over-sample discipline as the spend query
3089
+ // above — the caller filters by model attribution, so this stays unfiltered
3090
+ // beyond what no row can answer without.
3091
+ const selectLatencySamples = db.query<RunRow, [string, number]>(
3092
+ `SELECT * FROM runs
3093
+ WHERE project = ? AND turns > 0 AND endedAt IS NOT NULL
3094
+ ORDER BY endedAt DESC, rowid DESC
3095
+ LIMIT ?`,
3096
+ );
3061
3097
  const selectFailureClassCounts = db.query<{ cls: string; n: number }, [string]>(
3062
3098
  `SELECT failureClass AS cls, COUNT(*) AS n FROM runs
3063
3099
  WHERE project = ? AND failureClass IS NOT NULL AND recoveredAt IS NULL
3064
3100
  AND state IN ('blocked', 'killed', 'failed', 'orphaned', 'pushed-green')
3065
- AND recoveryAction NOT IN ('none', 'hold')
3101
+ AND recoveryAction NOT IN ('none', 'hold', 'observe')
3066
3102
  GROUP BY failureClass
3067
3103
  ORDER BY n DESC, failureClass ASC`,
3068
3104
  );
@@ -3070,7 +3106,7 @@ export function openStore(dbPath: string): Store {
3070
3106
  `SELECT failureClass AS cls, COUNT(*) AS n FROM runs
3071
3107
  WHERE project = ? AND failureClass IS NOT NULL AND recoveredAt IS NULL
3072
3108
  AND state IN ('blocked', 'killed', 'failed', 'orphaned', 'pushed-green')
3073
- AND recoveryAction IN ('none', 'hold')
3109
+ AND recoveryAction IN ('none', 'hold', 'observe')
3074
3110
  GROUP BY failureClass
3075
3111
  ORDER BY n DESC, failureClass ASC`,
3076
3112
  );
@@ -4469,11 +4505,12 @@ export function openStore(dbPath: string): Store {
4469
4505
  },
4470
4506
  );
4471
4507
  // The states the review verb admits and the dispatch pass may therefore
4472
- // claim (#795): a settled green run, or a capped/failed run that pushed one
4473
- // (`failed` / `killed`). The terminal set is closed on purpose a live row
4474
- // (`running` / `claimed`) is already doing its own work, a `pushed-pending`
4475
- // PR is not green yet, and a `blocked` / `orphaned` / `stopped` / `merged`
4476
- // row is not work the orchestrator returned for revision.
4508
+ // claim (#795, #1101): a settled green run, a capped/failed run that pushed
4509
+ // one (`failed` / `killed`), and since #1101 a `stopped` run that had
4510
+ // already pushed when the operator stopped it. The set is closed on purpose
4511
+ // a live row (`running` / `claimed`) is already doing its own work, a
4512
+ // `pushed-pending` PR is not green yet, and a `blocked` / `orphaned` /
4513
+ // `merged` row is not work the orchestrator returned for revision.
4477
4514
  //
4478
4515
  // The terminal leg does two things in the same atomic statement (#795 review
4479
4516
  // rounds 1-2): it PRESERVES the budget charge the row's terminal event was
@@ -4510,13 +4547,21 @@ export function openStore(dbPath: string): Store {
4510
4547
  const claimRunForReviewGreen = db.query<unknown, [string]>(
4511
4548
  `UPDATE runs SET state = 'running', endedAt = NULL WHERE id = ? AND state = 'pushed-green'`,
4512
4549
  );
4513
- // Exactly one leg can match (the row is either terminal or pushed-green at
4514
- // the moment of the claim), a repeated claim matches neither, and the two
4515
- // statements plus the read of `changes` are one transaction the dispatch
4516
- // pass can never see a half-claimed row.
4550
+ // The stopped leg (#1101): reclaiming an operator-stopped run spends no
4551
+ // budget an operator stop never charged one and `prUrl IS NOT NULL`
4552
+ // keeps a stop before any push unclaimable: only a run whose row records
4553
+ // the reviewed PR may be woken for its revision.
4554
+ const claimRunForReviewStopped = db.query<unknown, [string]>(
4555
+ `UPDATE runs SET state = 'running', endedAt = NULL WHERE id = ? AND state = 'stopped' AND prUrl IS NOT NULL`,
4556
+ );
4557
+ // Exactly one leg can match (the row is in exactly one state at the moment
4558
+ // of the claim), a repeated claim matches none, and the three statements
4559
+ // plus the read of `changes` are one transaction — the dispatch pass can
4560
+ // never see a half-claimed row.
4517
4561
  const claimRunForReviewTx = db.transaction((runId: string): boolean => {
4518
4562
  if (claimRunForReviewTerminal.run(runId).changes > 0) return true;
4519
- return claimRunForReviewGreen.run(runId).changes > 0;
4563
+ if (claimRunForReviewGreen.run(runId).changes > 0) return true;
4564
+ return claimRunForReviewStopped.run(runId).changes > 0;
4520
4565
  });
4521
4566
 
4522
4567
  // Appending is a single-statement concatenation so even a write from a
@@ -4760,6 +4805,7 @@ export function openStore(dbPath: string): Store {
4760
4805
  toSql(record.settlementFlags),
4761
4806
  toSql(record.report),
4762
4807
  toSql(record.graphTools),
4808
+ toSql(record.lastProgressAt),
4763
4809
  );
4764
4810
  return record;
4765
4811
  };
@@ -5877,6 +5923,9 @@ export function openStore(dbPath: string): Store {
5877
5923
  recentSpendSamples(project: string, limit: number): { turns: number; spendUsd: number }[] {
5878
5924
  return selectSpendSamples.all(project, limit).map((row) => ({ ...row }));
5879
5925
  },
5926
+ recentLatencySamples(project: string, limit: number): RunRecord[] {
5927
+ return selectLatencySamples.all(project, limit).map(toRecord);
5928
+ },
5880
5929
 
5881
5930
  failureClassCounts(project: string): { cls: FailureClass; n: number }[] {
5882
5931
  return selectFailureClassCounts