omp-conductor 0.20.0 → 0.20.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/schema/config.schema.json +24 -0
- package/src/admission.ts +58 -14
- package/src/arm-challenge.ts +54 -3
- package/src/briefs/console.md +10 -5
- package/src/commands/arm.ts +7 -5
- package/src/commands/companion.ts +52 -16
- package/src/commands/drain.ts +12 -5
- package/src/commands/worker.ts +5 -0
- package/src/config-schema.ts +8 -0
- package/src/config.ts +17 -2
- package/src/daemon/drain.ts +33 -0
- package/src/daemon/groom-pass.ts +16 -6
- package/src/daemon/http.ts +28 -0
- package/src/daemon/review.ts +11 -11
- package/src/daemon/runtime.ts +55 -3
- package/src/daemon/settle-pass.ts +19 -2
- package/src/daemon/supervision.ts +144 -1
- package/src/daemon/tick.ts +31 -14
- package/src/daemon/views.ts +17 -0
- package/src/daemon.ts +4 -3
- package/src/decisions.ts +20 -9
- package/src/diff-flags.ts +204 -28
- package/src/doctor.ts +27 -9
- package/src/escalate.ts +187 -15
- package/src/failure-class.ts +358 -5
- package/src/fleet.ts +55 -25
- package/src/graph-health.ts +17 -1
- package/src/groom.ts +11 -0
- package/src/orchestrator-tick.ts +302 -24
- package/src/reports.ts +4 -1
- package/src/settlement.ts +133 -12
- package/src/status-render.ts +38 -9
- package/src/store.ts +64 -15
- package/src/to-spec.ts +285 -24
- package/src/tracker/github.ts +132 -10
- package/src/types.ts +49 -3
- package/src/verbs/server.ts +16 -12
- package/src/worker.ts +149 -35
package/src/settlement.ts
CHANGED
|
@@ -25,7 +25,15 @@ import {
|
|
|
25
25
|
analyseSettlement,
|
|
26
26
|
deriveChangedLine,
|
|
27
27
|
} from "./diff-flags.ts";
|
|
28
|
-
import {
|
|
28
|
+
import {
|
|
29
|
+
EFFECTIVE_BUDGET_SAMPLE_RUNS,
|
|
30
|
+
SPINNING_CAP_CLASSES,
|
|
31
|
+
COMPOSE_DEPENDENCY_STARTUP_SIGNATURE,
|
|
32
|
+
classifyRun,
|
|
33
|
+
normalise,
|
|
34
|
+
observeTurnBudget,
|
|
35
|
+
type ClassifyFacts,
|
|
36
|
+
} from "./failure-class.ts";
|
|
29
37
|
import { GhPrMissingError } from "./tracker/github.ts";
|
|
30
38
|
import { formatModelsTried, modelsTried } from "./model-fallback.ts";
|
|
31
39
|
import { PR_LOOKUP_WINDOW_MS } from "./decisions.ts";
|
|
@@ -1510,6 +1518,13 @@ export async function classifyAndRecover(d: SettlementDeps): Promise<number> {
|
|
|
1510
1518
|
// merged is settled here when the settle sweep could not establish identity
|
|
1511
1519
|
// (#497). Every other exit returns 0.
|
|
1512
1520
|
let settled = 0;
|
|
1521
|
+
// One read for the whole sweep (#1063): the completed-run sample the
|
|
1522
|
+
// effective turn budget is derived from. Each row is then judged against the
|
|
1523
|
+
// budget *excluding itself*, so a just-killed run's own extreme pace cannot
|
|
1524
|
+
// drag the average it is being explained against.
|
|
1525
|
+
const latencySamples = store.recentLatencySamples(project.name, EFFECTIVE_BUDGET_SAMPLE_RUNS);
|
|
1526
|
+
const observedBudgetFor = (runId: string) =>
|
|
1527
|
+
observeTurnBudget(latencySamples, caps, project.workerModel, runId);
|
|
1513
1528
|
for (const run of store.runsNeedingClassification(project.name, CLASSIFY_BATCH)) {
|
|
1514
1529
|
const facts: ClassifyFacts = {};
|
|
1515
1530
|
let classifiedRun = run;
|
|
@@ -1558,6 +1573,26 @@ export async function classifyAndRecover(d: SettlementDeps): Promise<number> {
|
|
|
1558
1573
|
if (firstFailure?.link !== undefined) {
|
|
1559
1574
|
facts.failingLog = await tracker.checkLog(firstFailure.link);
|
|
1560
1575
|
}
|
|
1576
|
+
// A Compose dependency-startup failure ("docker compose up" aborted
|
|
1577
|
+
// because a dependency container failed its healthcheck) is only
|
|
1578
|
+
// outside the diff when the PR touched no container configuration —
|
|
1579
|
+
// a PR that breaks its own `docker-compose.yml` produces this exact
|
|
1580
|
+
// sentence and must still charge its attempt (#1059). So the
|
|
1581
|
+
// classifier needs the PR's changed-file list, derived from the
|
|
1582
|
+
// diff this settlement already knows how to fetch
|
|
1583
|
+
// (`Tracker.prDiff`, the same call the settlement audit uses) —
|
|
1584
|
+
// one fetch, and only for a row whose log actually carries
|
|
1585
|
+
// Compose's sentence; every other row pays nothing. A diff that
|
|
1586
|
+
// cannot be read or was cut short (`truncated`) stays undefined: an
|
|
1587
|
+
// unknown list must never waive an attempt.
|
|
1588
|
+
if (
|
|
1589
|
+
facts.failingLog !== undefined &&
|
|
1590
|
+
facts.failingLog.toLowerCase().includes(COMPOSE_DEPENDENCY_STARTUP_SIGNATURE)
|
|
1591
|
+
) {
|
|
1592
|
+
const diff = await tracker.prDiff(run.prUrl);
|
|
1593
|
+
facts.changedFiles =
|
|
1594
|
+
diff === undefined || diff.truncated ? undefined : diff.files.map((f) => f.path);
|
|
1595
|
+
}
|
|
1561
1596
|
}
|
|
1562
1597
|
}
|
|
1563
1598
|
} catch (err) {
|
|
@@ -1579,8 +1614,12 @@ export async function classifyAndRecover(d: SettlementDeps): Promise<number> {
|
|
|
1579
1614
|
continue;
|
|
1580
1615
|
}
|
|
1581
1616
|
}
|
|
1582
|
-
|
|
1583
|
-
|
|
1617
|
+
const { cls, recovery, evidence } = classifyRun(
|
|
1618
|
+
classifiedRun,
|
|
1619
|
+
facts,
|
|
1620
|
+
caps,
|
|
1621
|
+
observedBudgetFor(run.id),
|
|
1622
|
+
);
|
|
1584
1623
|
|
|
1585
1624
|
// A healthy green PR is not a failure of any class. Leaving the row
|
|
1586
1625
|
// unclassified is what keeps it eligible for the sweep on the tick where its
|
|
@@ -1669,9 +1708,27 @@ async function recoverRun(
|
|
|
1669
1708
|
// candidate admission can never accept — only the failed label comes off,
|
|
1670
1709
|
// and the exhaustion reaches a human (#490, #348).
|
|
1671
1710
|
if (cls === "wall-clock-cap-progress") {
|
|
1711
|
+
// Same split the requeue branch applies (#1080): an issue the tracker
|
|
1712
|
+
// reports closed was resolved by another route — a duplicate, a
|
|
1713
|
+
// supersede, a decomposition — so continuing it would dispatch work
|
|
1714
|
+
// nobody asked for. That verdict is terminal: release the lifecycle
|
|
1715
|
+
// label(s) the row could be holding (the kill path swaps in-progress
|
|
1716
|
+
// for failed; a crash between those two writes leaves either) and
|
|
1717
|
+
// stamp `recoveredAt`, or `selectUnclassified` re-offers this row on
|
|
1718
|
+
// every pass forever (#1095). An unreadable answer is the tracker
|
|
1719
|
+
// failing to answer, not answering no — it stays transient and retries.
|
|
1672
1720
|
const state = await tracker.issueState(run.issue).catch(() => undefined);
|
|
1721
|
+
if (state === "closed") {
|
|
1722
|
+
store.enqueueLabelOps(project.name, [
|
|
1723
|
+
{ issue: run.issue, op: "remove", label: project.stateLabels.failed },
|
|
1724
|
+
{ issue: run.issue, op: "remove", label: inProgress },
|
|
1725
|
+
]);
|
|
1726
|
+
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
1727
|
+
log(`#${run.issue} not continued from ${cls}: issue is closed`);
|
|
1728
|
+
return;
|
|
1729
|
+
}
|
|
1673
1730
|
if (state !== "open") {
|
|
1674
|
-
log(`#${run.issue} not continued from ${cls}: issue is
|
|
1731
|
+
log(`#${run.issue} not continued from ${cls}: issue is unreadable (retrying)`);
|
|
1675
1732
|
return;
|
|
1676
1733
|
}
|
|
1677
1734
|
const continuation = store.continuationsFor(project.name, run.issue);
|
|
@@ -1706,6 +1763,42 @@ async function recoverRun(
|
|
|
1706
1763
|
return;
|
|
1707
1764
|
}
|
|
1708
1765
|
|
|
1766
|
+
// `progress-stall`: the watch settled a hung session (#1086), which keeps
|
|
1767
|
+
// the in-progress label held the way an orphan row does. Work to continue
|
|
1768
|
+
// from hands that label back for a continuation brief; a budget already
|
|
1769
|
+
// spent releases the issue to a human instead of re-offering a candidate
|
|
1770
|
+
// admission can never accept — the same ceiling the orphan path honours.
|
|
1771
|
+
if (cls === "progress-stall") {
|
|
1772
|
+
const state = await tracker.issueState(run.issue).catch(() => undefined);
|
|
1773
|
+
if (state !== "open") {
|
|
1774
|
+
log(`#${run.issue} not continued from ${cls}: issue is ${state ?? "unreadable"}`);
|
|
1775
|
+
return;
|
|
1776
|
+
}
|
|
1777
|
+
const continuation = store.continuationsFor(project.name, run.issue);
|
|
1778
|
+
if (hasContinuationBudget(continuation, caps.maxContinuationsPerIssue)) {
|
|
1779
|
+
await swapToQueue(d, run.issue, inProgress);
|
|
1780
|
+
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
1781
|
+
log(`#${run.issue} requeued after a progress stall: ${evidence}`);
|
|
1782
|
+
} else {
|
|
1783
|
+
store.enqueueLabelOps(project.name, [{ issue: run.issue, op: "remove", label: inProgress }]);
|
|
1784
|
+
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
1785
|
+
await safeEscalate(d, {
|
|
1786
|
+
tier: 1,
|
|
1787
|
+
project: project.name,
|
|
1788
|
+
issue: run.issue,
|
|
1789
|
+
summary: `#${run.issue} exhausted its continuation budget on progress stalls`,
|
|
1790
|
+
detail: [
|
|
1791
|
+
`Attempt ${run.attempt} was settled silent with work to continue from, but ${continuation} continuation(s) are already charged against a cap of ${caps.maxContinuationsPerIssue}.`,
|
|
1792
|
+
evidence,
|
|
1793
|
+
`Work to continue: branch ${run.branch} at ${run.headSha ?? run.salvageSha}${run.prUrl === undefined ? "" : ` — ${run.prUrl}`}.`,
|
|
1794
|
+
"Sessions on this issue keep hanging, so another run would burn a worker slot for the same outcome. Re-scope it, raise maxContinuationsPerIssue for it, or finish the remaining work by hand.",
|
|
1795
|
+
].join("\n"),
|
|
1796
|
+
});
|
|
1797
|
+
log(`#${run.issue} not continued from ${cls}: continuation budget exhausted`);
|
|
1798
|
+
}
|
|
1799
|
+
return;
|
|
1800
|
+
}
|
|
1801
|
+
|
|
1709
1802
|
// `model-empty-stop`: the provider answered with empty turns until the
|
|
1710
1803
|
// harness ended the session. The work is not at fault and the row is not a
|
|
1711
1804
|
// failed attempt (the counters exclude the class), so this hands the queue
|
|
@@ -1714,9 +1807,22 @@ async function recoverRun(
|
|
|
1714
1807
|
// wall-clock path is: a provider stuck in that state must reach a human
|
|
1715
1808
|
// instead of consuming the issue forever, one free retry at a time.
|
|
1716
1809
|
if (cls === "model-empty-stop") {
|
|
1810
|
+
// Same split as the wall-clock branch above (#1080, #1095): a closed
|
|
1811
|
+
// issue is terminal — release the lifecycle label(s), stamp
|
|
1812
|
+
// `recoveredAt`, stop being offered — while an unreadable tracker keeps
|
|
1813
|
+
// the row unstamped and retrying on a later pass.
|
|
1717
1814
|
const state = await tracker.issueState(run.issue).catch(() => undefined);
|
|
1815
|
+
if (state === "closed") {
|
|
1816
|
+
store.enqueueLabelOps(project.name, [
|
|
1817
|
+
{ issue: run.issue, op: "remove", label: project.stateLabels.failed },
|
|
1818
|
+
{ issue: run.issue, op: "remove", label: inProgress },
|
|
1819
|
+
]);
|
|
1820
|
+
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
1821
|
+
log(`#${run.issue} not continued from ${cls}: issue is closed`);
|
|
1822
|
+
return;
|
|
1823
|
+
}
|
|
1718
1824
|
if (state !== "open") {
|
|
1719
|
-
log(`#${run.issue} not continued from ${cls}: issue is
|
|
1825
|
+
log(`#${run.issue} not continued from ${cls}: issue is unreadable (retrying)`);
|
|
1720
1826
|
return;
|
|
1721
1827
|
}
|
|
1722
1828
|
const continuation = store.continuationsFor(project.name, run.issue);
|
|
@@ -1845,11 +1951,22 @@ async function recoverRun(
|
|
|
1845
1951
|
return;
|
|
1846
1952
|
}
|
|
1847
1953
|
// Only when the tracker still shows this issue as ours to hand back. An
|
|
1848
|
-
// issue that is closed
|
|
1849
|
-
//
|
|
1954
|
+
// issue that is closed was resolved by another route and requeueing it
|
|
1955
|
+
// would dispatch work nobody asked for — so that verdict is terminal
|
|
1956
|
+
// (#1080): release the dispatcher-owned in-progress label and stamp
|
|
1957
|
+
// `recoveredAt`, exactly like the operator-withdrawal branch below, or the
|
|
1958
|
+
// sweep re-offers this row on every pass forever. An unreadable answer is
|
|
1959
|
+
// a different thing in kind — the tracker failing to answer, not answering
|
|
1960
|
+
// no — so it stays transient and keeps retrying.
|
|
1850
1961
|
const state = await tracker.issueState(run.issue).catch(() => undefined);
|
|
1962
|
+
if (state === "closed") {
|
|
1963
|
+
store.enqueueLabelOps(project.name, [{ issue: run.issue, op: "remove", label: inProgress }]);
|
|
1964
|
+
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
1965
|
+
log(`#${run.issue} not requeued from ${cls}: issue is closed`);
|
|
1966
|
+
return;
|
|
1967
|
+
}
|
|
1851
1968
|
if (state !== "open") {
|
|
1852
|
-
log(`#${run.issue} not requeued from ${cls}: issue is
|
|
1969
|
+
log(`#${run.issue} not requeued from ${cls}: issue is unreadable (retrying)`);
|
|
1853
1970
|
return;
|
|
1854
1971
|
}
|
|
1855
1972
|
// A clean orphan whose queue label is already absent is a deliberate
|
|
@@ -1990,10 +2107,14 @@ async function recoverRun(
|
|
|
1990
2107
|
return;
|
|
1991
2108
|
}
|
|
1992
2109
|
|
|
1993
|
-
// `hold` (orphan-dirty) and `none`:
|
|
1994
|
-
//
|
|
1995
|
-
//
|
|
1996
|
-
//
|
|
2110
|
+
// `observe` (awaiting-observation), `hold` (orphan-dirty) and `none`:
|
|
2111
|
+
// recorded, nothing performed here. `observe`'s recovery IS the later observation —
|
|
2112
|
+
// the settle sweep re-offers the blocked row until its PR resolves and settles it
|
|
2113
|
+
// the way it settles `settlement-stuck`, so no action exists to perform today and
|
|
2114
|
+
// `recoveredAt` stays NULL by design (#1068). The existing unsalvaged-WIP
|
|
2115
|
+
// admission hold already fails dispatch closed until an operator acknowledges
|
|
2116
|
+
// the tree, which is the only safe move when the worktree holds the only copy
|
|
2117
|
+
// of real work.
|
|
1997
2118
|
}
|
|
1998
2119
|
|
|
1999
2120
|
/**
|
package/src/status-render.ts
CHANGED
|
@@ -254,12 +254,14 @@ const MECHANICAL_GROOMING_REASONS: Record<string, true> = {
|
|
|
254
254
|
};
|
|
255
255
|
|
|
256
256
|
/**
|
|
257
|
-
* The to-spec refusal classes persisted as blocked rows (#772) — a
|
|
258
|
-
* that failed validation
|
|
259
|
-
*
|
|
257
|
+
* The to-spec refusal classes persisted as blocked rows (#772, #1064) — a
|
|
258
|
+
* result that failed validation, or a pass that produced no answer at all, is
|
|
259
|
+
* a mechanical block, never a verdict. Mirrors the failure kinds of
|
|
260
|
+
* `ToSpecFailure` in `to-spec.ts`.
|
|
260
261
|
*/
|
|
261
262
|
const REFUSED_GROOMING_REASONS: Record<string, true> = {
|
|
262
263
|
malformed: true,
|
|
264
|
+
"no-answer": true,
|
|
263
265
|
"missing-source": true,
|
|
264
266
|
"stale-source": true,
|
|
265
267
|
};
|
|
@@ -289,10 +291,12 @@ const REFUSED_GROOMING_REASONS: Record<string, true> = {
|
|
|
289
291
|
* (`blocked` rows whose reason is a groomer verdict or a product-judgement
|
|
290
292
|
* label, e.g. `needs-product-decision`).
|
|
291
293
|
* - `mechanically blocked` — admission's lane/dependency holds.
|
|
292
|
-
* - `refused` — to-spec
|
|
293
|
-
* `missing-source`, `stale-source`), told apart from the
|
|
294
|
-
* operator sees whether the runway cannot move or a result
|
|
295
|
-
* trusted.
|
|
294
|
+
* - `refused` — to-spec passes that produced no usable verdict (`malformed`,
|
|
295
|
+
* `no-answer`, `missing-source`, `stale-source`), told apart from the
|
|
296
|
+
* holds so an operator sees whether the runway cannot move or a result
|
|
297
|
+
* cannot be trusted. Each row's line names its own class (`#19 malformed`,
|
|
298
|
+
* `#22 no-answer`), so a run of identical refusals is visible as a pattern
|
|
299
|
+
* rather than a wall of one word.
|
|
296
300
|
* - `in-flight` — a launched batch is running (#777).
|
|
297
301
|
* - `operator-parked` — the dispatch snapshot's parked count (#507).
|
|
298
302
|
*
|
|
@@ -486,6 +490,9 @@ export function formatFleetStatus(
|
|
|
486
490
|
lastStop: DaemonStop | undefined = undefined,
|
|
487
491
|
siblings: { project: string; live: number }[] = [],
|
|
488
492
|
grooming: string | undefined = undefined,
|
|
493
|
+
/** One line naming the flat-chat deliveries this project's stale pin caused
|
|
494
|
+
* (#1094), or undefined when there is nothing outstanding. */
|
|
495
|
+
misrouteNote: string | undefined = undefined,
|
|
489
496
|
): string {
|
|
490
497
|
const tickLine =
|
|
491
498
|
layers.ticksDetail === undefined
|
|
@@ -591,6 +598,7 @@ export function formatFleetStatus(
|
|
|
591
598
|
recoveryLine,
|
|
592
599
|
herdrLine,
|
|
593
600
|
telegramLine,
|
|
601
|
+
...(misrouteNote === undefined ? [] : [misrouteNote]),
|
|
594
602
|
...(brief === undefined ? [] : [brief]),
|
|
595
603
|
...(decisions === undefined ? [] : [decisions]),
|
|
596
604
|
...(failureClasses === undefined ? [] : [failureClasses]),
|
|
@@ -807,6 +815,18 @@ export function formatReviewCorrections(rounds: readonly ReviewCorrectionRound[]
|
|
|
807
815
|
});
|
|
808
816
|
}
|
|
809
817
|
|
|
818
|
+
/**
|
|
819
|
+
* The #1063 effective-turn-budget annotation for the `new worker turns` row:
|
|
820
|
+
* shown whenever observed per-turn latency makes the configured ceiling
|
|
821
|
+
* unreachable at all — the break-even pace is exactly wall clock ÷ max turns,
|
|
822
|
+
* so anything slower is material, anything faster is noise.
|
|
823
|
+
*/
|
|
824
|
+
function effectiveTurnBudgetSuffix(s: StatusSnapshot): string {
|
|
825
|
+
const observed = s.workerTurnBudgetObserved;
|
|
826
|
+
if (observed === undefined || observed.effectiveTurns >= s.caps.workerMaxTurns) return "";
|
|
827
|
+
return ` (effective ~${observed.effectiveTurns} at ${observed.minutesPerTurn.toFixed(2)} min/turn, last ${observed.sampleSize} runs)`;
|
|
828
|
+
}
|
|
829
|
+
|
|
810
830
|
|
|
811
831
|
function formatProjectBody(
|
|
812
832
|
s: StatusSnapshot,
|
|
@@ -912,7 +932,7 @@ function formatProjectBody(
|
|
|
912
932
|
: `${Math.max(1, Math.round(s.labelOps.oldestAgeMs / 1000))}s`
|
|
913
933
|
})`,
|
|
914
934
|
]),
|
|
915
|
-
` new worker turns ${s.caps.workerMaxTurns}`,
|
|
935
|
+
` new worker turns ${s.caps.workerMaxTurns}${effectiveTurnBudgetSuffix(s)}`,
|
|
916
936
|
...s.turnOverrides.map(
|
|
917
937
|
({ issue, maxTurns }) => ` turn override #${issue} → ${maxTurns} (next attempt)`,
|
|
918
938
|
),
|
|
@@ -1078,6 +1098,15 @@ function formatProjectBody(
|
|
|
1078
1098
|
// the phase the line is naming, and a number attributed to the wrong phase
|
|
1079
1099
|
// is worse than no number. The attempt's own cap remains visible as
|
|
1080
1100
|
// `N/M turns cumulative`.
|
|
1101
|
+
// Transcript silence (#1086), from the progress watch's recorded write
|
|
1102
|
+
// instant — read off the row, never probed at render time (#919). This
|
|
1103
|
+
// is where a stall is visible forming, before the pass settles it.
|
|
1104
|
+
// Skipped under pause, whose silence is the pause working (#938), and
|
|
1105
|
+
// for rows no longer live, whose transcripts have stopped forever.
|
|
1106
|
+
const silent =
|
|
1107
|
+
paused || r.state !== "running" || r.lastProgressAt === undefined
|
|
1108
|
+
? ""
|
|
1109
|
+
: ` silent ${formatDownDuration(Math.max(0, now - r.lastProgressAt))}`;
|
|
1081
1110
|
const progress = paused
|
|
1082
1111
|
? ""
|
|
1083
1112
|
: round !== undefined
|
|
@@ -1089,7 +1118,7 @@ function formatProjectBody(
|
|
|
1089
1118
|
).toFixed(1)} turns/min avg` +
|
|
1090
1119
|
` projects ${Math.round(
|
|
1091
1120
|
(r.turns * s.caps.workerWallClockMs) / Math.max(elapsedMs, 1_000),
|
|
1092
|
-
)} turns at cap
|
|
1121
|
+
)} turns at cap` + silent;
|
|
1093
1122
|
lines.push(
|
|
1094
1123
|
` #${r.issue} ${r.repo} ${state} attempt ${r.attempt} ` +
|
|
1095
1124
|
`${r.turns}/${r.maxTurns} turns${progress} ${r.spendUsd.toFixed(2)} ${r.branch}` +
|
package/src/store.ts
CHANGED
|
@@ -221,6 +221,7 @@ const UPDATABLE_COLUMNS: Record<string, true> = {
|
|
|
221
221
|
failureCharges: true,
|
|
222
222
|
continuationCharges: true,
|
|
223
223
|
terminalEvidence: true,
|
|
224
|
+
lastProgressAt: true,
|
|
224
225
|
};
|
|
225
226
|
|
|
226
227
|
/** Everything SQLite will accept from us. */
|
|
@@ -281,6 +282,8 @@ interface RunRow {
|
|
|
281
282
|
graphTools: string | null;
|
|
282
283
|
failureCharges: number | null;
|
|
283
284
|
continuationCharges: number | null;
|
|
285
|
+
/** NULL for a row the progress pass has not observed, or one with no transcript yet (#1086). */
|
|
286
|
+
lastProgressAt: number | null;
|
|
284
287
|
}
|
|
285
288
|
|
|
286
289
|
/** The `base_health` table exactly as SQLite hands it back. */
|
|
@@ -887,7 +890,11 @@ CREATE TABLE IF NOT EXISTS runs (
|
|
|
887
890
|
model TEXT,
|
|
888
891
|
graphTools TEXT,
|
|
889
892
|
failureCharges INTEGER,
|
|
890
|
-
continuationCharges INTEGER
|
|
893
|
+
continuationCharges INTEGER,
|
|
894
|
+
-- When the run's transcript was last observed to grow, as epoch ms (#1086).
|
|
895
|
+
-- Written by the daemon's progress watch from the file's own mtime; NULL
|
|
896
|
+
-- reads as "not observed yet", never "no progress".
|
|
897
|
+
lastProgressAt INTEGER
|
|
891
898
|
);
|
|
892
899
|
CREATE INDEX IF NOT EXISTS runs_project_issue ON runs (project, issue);
|
|
893
900
|
CREATE INDEX IF NOT EXISTS runs_project_state ON runs (project, state);
|
|
@@ -1752,6 +1759,7 @@ function toRecord(row: RunRow): RunRecord {
|
|
|
1752
1759
|
};
|
|
1753
1760
|
if (row.spendReservedUsd !== null) record.spendReservedUsd = row.spendReservedUsd;
|
|
1754
1761
|
if (row.sessionFile !== null) record.sessionFile = row.sessionFile;
|
|
1762
|
+
if (row.lastProgressAt !== null) record.lastProgressAt = row.lastProgressAt;
|
|
1755
1763
|
if (row.workerPid !== null) record.workerPid = row.workerPid;
|
|
1756
1764
|
if (row.paneId !== null) record.paneId = row.paneId;
|
|
1757
1765
|
if (row.paneLabel !== null) record.paneLabel = row.paneLabel;
|
|
@@ -2321,6 +2329,12 @@ export function openStore(dbPath: string): Store {
|
|
|
2321
2329
|
if (!columns.some((column) => column.name === "terminalEvidence")) {
|
|
2322
2330
|
db.exec("ALTER TABLE runs ADD COLUMN terminalEvidence TEXT");
|
|
2323
2331
|
}
|
|
2332
|
+
// The progress watch's last-observed transcript write (#1086). Rows written
|
|
2333
|
+
// before the column were never observed, so NULL is the honest reading — no
|
|
2334
|
+
// backfill, because the instant only the file's own mtime holds.
|
|
2335
|
+
if (!columns.some((column) => column.name === "lastProgressAt")) {
|
|
2336
|
+
db.exec("ALTER TABLE runs ADD COLUMN lastProgressAt INTEGER");
|
|
2337
|
+
}
|
|
2324
2338
|
// Every row written before #132 is unclassified, and NULL is the honest
|
|
2325
2339
|
// reading of that: the budget counters below deliberately still count an
|
|
2326
2340
|
// unclassified terminal row exactly as this release's predecessor did, so an
|
|
@@ -2686,8 +2700,8 @@ export function openStore(dbPath: string): Store {
|
|
|
2686
2700
|
id, project, issue, repo, branch, worktree, state, attempt, turns,
|
|
2687
2701
|
maxTurns, spendUsd, spendReservedUsd, sessionFile, resumedFromRunId, lane, prUrl, headSha, mergeSha, baseRef,
|
|
2688
2702
|
baseCheck, baseCheckAt, salvageSha, salvageError, salvageAckAt, startedAt,
|
|
2689
|
-
endedAt, lastError, settlementFlags, report, graphTools
|
|
2690
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
2703
|
+
endedAt, lastError, settlementFlags, report, graphTools, lastProgressAt
|
|
2704
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
2691
2705
|
);
|
|
2692
2706
|
const selectRun = db.query<RunRow, [string]>(`SELECT * FROM runs WHERE id = ?`);
|
|
2693
2707
|
const selectGraphToolsObs = db.query<{ graphTools: string }, [string]>(
|
|
@@ -2973,6 +2987,17 @@ export function openStore(dbPath: string): Store {
|
|
|
2973
2987
|
// "retrying next tick" is a lie and the row is stranded with a class and no
|
|
2974
2988
|
// action. `hold` is excluded because it is *recorded only* by design — its
|
|
2975
2989
|
// `recoveredAt` stays NULL forever, and re-offering it would spin the sweep.
|
|
2990
|
+
//
|
|
2991
|
+
// One deliberate exception re-enters the sweep after classification: a
|
|
2992
|
+
// blocked row with a PR. It is the observation half of the
|
|
2993
|
+
// `awaiting-observation` recovery (#1068) — the sweep re-reads the PR every
|
|
2994
|
+
// pass, and the classifier's merged-PR branch names `settlement-stuck` the
|
|
2995
|
+
// way it does for any other stuck row, so a blocked run whose PR later merges
|
|
2996
|
+
// (the #1062 shape: merged at 15:02, row still `blocked`/`question`) settles
|
|
2997
|
+
// instead of listing for Duty 1 triage forever. A genuine question row is
|
|
2998
|
+
// re-offered too and its escalation is not repeated: the notifications ledger
|
|
2999
|
+
// dedupes on the class+run summary of the stable evidence. Rows without a PR
|
|
3000
|
+
// have nothing to observe and are only re-offered by the ordinary clauses.
|
|
2976
3001
|
const selectUnclassified = db.query<RunRow, [string, number]>(
|
|
2977
3002
|
`SELECT * FROM runs
|
|
2978
3003
|
WHERE project = ?
|
|
@@ -2980,6 +3005,7 @@ export function openStore(dbPath: string): Store {
|
|
|
2980
3005
|
AND (
|
|
2981
3006
|
failureClass IS NULL
|
|
2982
3007
|
OR (recoveredAt IS NULL AND recoveryAction IN ('settle', 'continue', 'requeue', 'rerun-checks'))
|
|
3008
|
+
OR (state = 'blocked' AND prUrl IS NOT NULL)
|
|
2983
3009
|
)
|
|
2984
3010
|
ORDER BY startedAt DESC, rowid DESC
|
|
2985
3011
|
LIMIT ?`,
|
|
@@ -3058,11 +3084,21 @@ export function openStore(dbPath: string): Store {
|
|
|
3058
3084
|
ORDER BY endedAt DESC, rowid DESC
|
|
3059
3085
|
LIMIT ?`,
|
|
3060
3086
|
);
|
|
3087
|
+
// Newest runs that can yield a per-turn pace (#1063): rows with turns and
|
|
3088
|
+
// elapsed time recorded. Same over-sample discipline as the spend query
|
|
3089
|
+
// above — the caller filters by model attribution, so this stays unfiltered
|
|
3090
|
+
// beyond what no row can answer without.
|
|
3091
|
+
const selectLatencySamples = db.query<RunRow, [string, number]>(
|
|
3092
|
+
`SELECT * FROM runs
|
|
3093
|
+
WHERE project = ? AND turns > 0 AND endedAt IS NOT NULL
|
|
3094
|
+
ORDER BY endedAt DESC, rowid DESC
|
|
3095
|
+
LIMIT ?`,
|
|
3096
|
+
);
|
|
3061
3097
|
const selectFailureClassCounts = db.query<{ cls: string; n: number }, [string]>(
|
|
3062
3098
|
`SELECT failureClass AS cls, COUNT(*) AS n FROM runs
|
|
3063
3099
|
WHERE project = ? AND failureClass IS NOT NULL AND recoveredAt IS NULL
|
|
3064
3100
|
AND state IN ('blocked', 'killed', 'failed', 'orphaned', 'pushed-green')
|
|
3065
|
-
AND recoveryAction NOT IN ('none', 'hold')
|
|
3101
|
+
AND recoveryAction NOT IN ('none', 'hold', 'observe')
|
|
3066
3102
|
GROUP BY failureClass
|
|
3067
3103
|
ORDER BY n DESC, failureClass ASC`,
|
|
3068
3104
|
);
|
|
@@ -3070,7 +3106,7 @@ export function openStore(dbPath: string): Store {
|
|
|
3070
3106
|
`SELECT failureClass AS cls, COUNT(*) AS n FROM runs
|
|
3071
3107
|
WHERE project = ? AND failureClass IS NOT NULL AND recoveredAt IS NULL
|
|
3072
3108
|
AND state IN ('blocked', 'killed', 'failed', 'orphaned', 'pushed-green')
|
|
3073
|
-
AND recoveryAction IN ('none', 'hold')
|
|
3109
|
+
AND recoveryAction IN ('none', 'hold', 'observe')
|
|
3074
3110
|
GROUP BY failureClass
|
|
3075
3111
|
ORDER BY n DESC, failureClass ASC`,
|
|
3076
3112
|
);
|
|
@@ -4469,11 +4505,12 @@ export function openStore(dbPath: string): Store {
|
|
|
4469
4505
|
},
|
|
4470
4506
|
);
|
|
4471
4507
|
// The states the review verb admits and the dispatch pass may therefore
|
|
4472
|
-
// claim (#795): a settled green run,
|
|
4473
|
-
// (`failed` / `killed`)
|
|
4474
|
-
//
|
|
4475
|
-
//
|
|
4476
|
-
//
|
|
4508
|
+
// claim (#795, #1101): a settled green run, a capped/failed run that pushed
|
|
4509
|
+
// one (`failed` / `killed`), and — since #1101 — a `stopped` run that had
|
|
4510
|
+
// already pushed when the operator stopped it. The set is closed on purpose
|
|
4511
|
+
// — a live row (`running` / `claimed`) is already doing its own work, a
|
|
4512
|
+
// `pushed-pending` PR is not green yet, and a `blocked` / `orphaned` /
|
|
4513
|
+
// `merged` row is not work the orchestrator returned for revision.
|
|
4477
4514
|
//
|
|
4478
4515
|
// The terminal leg does two things in the same atomic statement (#795 review
|
|
4479
4516
|
// rounds 1-2): it PRESERVES the budget charge the row's terminal event was
|
|
@@ -4510,13 +4547,21 @@ export function openStore(dbPath: string): Store {
|
|
|
4510
4547
|
const claimRunForReviewGreen = db.query<unknown, [string]>(
|
|
4511
4548
|
`UPDATE runs SET state = 'running', endedAt = NULL WHERE id = ? AND state = 'pushed-green'`,
|
|
4512
4549
|
);
|
|
4513
|
-
//
|
|
4514
|
-
//
|
|
4515
|
-
//
|
|
4516
|
-
//
|
|
4550
|
+
// The stopped leg (#1101): reclaiming an operator-stopped run spends no
|
|
4551
|
+
// budget — an operator stop never charged one — and `prUrl IS NOT NULL`
|
|
4552
|
+
// keeps a stop before any push unclaimable: only a run whose row records
|
|
4553
|
+
// the reviewed PR may be woken for its revision.
|
|
4554
|
+
const claimRunForReviewStopped = db.query<unknown, [string]>(
|
|
4555
|
+
`UPDATE runs SET state = 'running', endedAt = NULL WHERE id = ? AND state = 'stopped' AND prUrl IS NOT NULL`,
|
|
4556
|
+
);
|
|
4557
|
+
// Exactly one leg can match (the row is in exactly one state at the moment
|
|
4558
|
+
// of the claim), a repeated claim matches none, and the three statements
|
|
4559
|
+
// plus the read of `changes` are one transaction — the dispatch pass can
|
|
4560
|
+
// never see a half-claimed row.
|
|
4517
4561
|
const claimRunForReviewTx = db.transaction((runId: string): boolean => {
|
|
4518
4562
|
if (claimRunForReviewTerminal.run(runId).changes > 0) return true;
|
|
4519
|
-
|
|
4563
|
+
if (claimRunForReviewGreen.run(runId).changes > 0) return true;
|
|
4564
|
+
return claimRunForReviewStopped.run(runId).changes > 0;
|
|
4520
4565
|
});
|
|
4521
4566
|
|
|
4522
4567
|
// Appending is a single-statement concatenation so even a write from a
|
|
@@ -4760,6 +4805,7 @@ export function openStore(dbPath: string): Store {
|
|
|
4760
4805
|
toSql(record.settlementFlags),
|
|
4761
4806
|
toSql(record.report),
|
|
4762
4807
|
toSql(record.graphTools),
|
|
4808
|
+
toSql(record.lastProgressAt),
|
|
4763
4809
|
);
|
|
4764
4810
|
return record;
|
|
4765
4811
|
};
|
|
@@ -5877,6 +5923,9 @@ export function openStore(dbPath: string): Store {
|
|
|
5877
5923
|
recentSpendSamples(project: string, limit: number): { turns: number; spendUsd: number }[] {
|
|
5878
5924
|
return selectSpendSamples.all(project, limit).map((row) => ({ ...row }));
|
|
5879
5925
|
},
|
|
5926
|
+
recentLatencySamples(project: string, limit: number): RunRecord[] {
|
|
5927
|
+
return selectLatencySamples.all(project, limit).map(toRecord);
|
|
5928
|
+
},
|
|
5880
5929
|
|
|
5881
5930
|
failureClassCounts(project: string): { cls: FailureClass; n: number }[] {
|
|
5882
5931
|
return selectFailureClassCounts
|