omp-conductor 0.6.0 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/daemon.ts CHANGED
@@ -102,6 +102,11 @@ const GRAPH_HEALTH_INTERVAL_MS = 60_000;
102
102
  * report an operator is waiting on must not sit in the outbox for the length of
103
103
  * a poll interval, and delivery is owed even while claiming is paused (#123). */
104
104
  const REPORT_DELIVERY_INTERVAL_MS = 30_000;
105
+ /** A dispatch-infra run (turn-0 git failure) is requeued so the next tick
106
+ * retries — but only a bounded number of times. Three strikes for one issue
107
+ * means the mirror itself is broken, not unlucky, and the sweep escalates
108
+ * instead of burning a turn-0 run per tick forever (#168, #177). */
109
+ const DISPATCH_INFRA_MAX_STRIKES = 3;
105
110
  const DEFAULT_PORT = 8787;
106
111
  const BRIEF_TEMPLATE_PATH = join(import.meta.dir, "briefs", "worker.md");
107
112
 
@@ -121,8 +126,13 @@ export interface DaemonOpts {
121
126
  project?: string;
122
127
  }
123
128
 
124
- /** Everything one tick touches, resolved once at startup so a tick never
125
- * re-reads config mid-flight and changes its own limits underneath itself. */
129
+ /** Everything one tick touches. `project` and `caps` are re-resolved at each
130
+ * tick boundary so an operator's config edit applies on the next tick rather
131
+ * than the next daemon restart (#170); a tick and the runs it admits see one
132
+ * consistent snapshot, and a mid-run edit never changes a live run's labels,
133
+ * model or caps — `handleIssue` destructures them at dispatch time. The rest
134
+ * are resolved once at startup so a tick never re-reads config mid-flight and
135
+ * changes its own limits underneath itself. */
126
136
  interface Deps {
127
137
  project: ProjectConfig;
128
138
  caps: Caps;
@@ -176,6 +186,7 @@ export function verbDeps(d: Pick<Deps, "project" | "store" | "tracker" | "verbAc
176
186
  isPaused()
177
187
  ? "claiming is paused for this fleet (omp-conductor pause, hold or halt)"
178
188
  : undefined,
189
+ pausedAt,
179
190
  log,
180
191
  now: () => Date.now(),
181
192
  };
@@ -288,6 +299,30 @@ export function isPaused(): boolean {
288
299
  return existsSync(join(stateDir(), "paused"));
289
300
  }
290
301
 
302
+ /**
303
+ * The epoch-ms timestamp at which the current pause began, read from the same
304
+ * sentinel file {@link setPaused} writes (`<stateDir()>/paused`). Returns
305
+ * `undefined` when the fleet is not paused, or when the file's first line does
306
+ * not parse as a date — a legacy/blank sentinel keeps today's refuse-everything
307
+ * behavior, because a run admitted before an *unknown* pause cannot be proven
308
+ * innocent. {@link isPaused} is the authority on *whether*; this answers
309
+ * *since when*.
310
+ */
311
+ export function pausedAt(): number | undefined {
312
+ const f = join(stateDir(), "paused");
313
+ if (!existsSync(f)) return undefined;
314
+ try {
315
+ const first = readFileSync(f, "utf8").split("\n")[0]?.trim();
316
+ if (first === undefined || first === "") return undefined;
317
+ const t = Date.parse(first);
318
+ return Number.isNaN(t) ? undefined : t;
319
+ } catch {
320
+ // Unreadable sentinel (permissions, corruption): fail closed like an
321
+ // unparseable line — refuse mutations while the pause is unprovable.
322
+ return undefined;
323
+ }
324
+ }
325
+
291
326
  export function setPaused(v: boolean): void {
292
327
  const f = join(stateDir(), "paused");
293
328
  if (v) {
@@ -1709,8 +1744,19 @@ export async function admitCandidates(
1709
1744
  slots: number,
1710
1745
  ): Promise<AdmissionPass> {
1711
1746
  const { project, caps, tracker, store } = d;
1712
- const busyIssues = store.activeRuns(project.name).map((r) => r.issue);
1747
+ const activeRuns = store.activeRuns(project.name);
1748
+ const busyIssues = activeRuns.map((r) => r.issue);
1713
1749
  const busy = new Set(busyIssues);
1750
+ // issue -> its active run rows, for the pushed-green admission bypass (#175):
1751
+ // only a worker-free pushed-green row may be bypassed, and only when *every*
1752
+ // active run for the issue is worker-free. A live (claimed/running) row still
1753
+ // holds unconditionally.
1754
+ const activeByIssue = new Map<number, RunRecord[]>();
1755
+ for (const run of activeRuns) {
1756
+ const list = activeByIssue.get(run.issue);
1757
+ if (list === undefined) activeByIssue.set(run.issue, [run]);
1758
+ else list.push(run);
1759
+ }
1714
1760
  const holds: AdmissionHold[] = [];
1715
1761
  const hold = (issue: number, reason: AdmissionHoldReason): void => {
1716
1762
  holds.push({ issue, reason });
@@ -1769,8 +1815,17 @@ export async function admitCandidates(
1769
1815
  continue;
1770
1816
  }
1771
1817
  if (busy.has(issue)) {
1772
- hold(issue, "issue-active");
1773
- continue;
1818
+ // A pushed-green row is worker-free by definition (it is not in
1819
+ // LIVE_STATES): its PR is live but no process is writing to its branch.
1820
+ // So an issue whose active runs are ALL pushed-green is not actually
1821
+ // occupied — the corrective attempt the operator unblocked may be
1822
+ // admitted as a continuation of that PR, and the open-PR gate below
1823
+ // decides the identity. Any live row still holds (#175).
1824
+ const allWorkerFree = (activeByIssue.get(issue) ?? []).every((r) => r.state === "pushed-green");
1825
+ if (!allWorkerFree) {
1826
+ hold(issue, "issue-active");
1827
+ continue;
1828
+ }
1774
1829
  }
1775
1830
 
1776
1831
  const priorRuns = store.attemptsFor(project.name, issue);
@@ -1888,7 +1943,8 @@ export async function admitCandidates(
1888
1943
  latest?.state === "blocked" ||
1889
1944
  latest?.state === "failed" ||
1890
1945
  latest?.state === "killed" ||
1891
- latest?.state === "orphaned"
1946
+ latest?.state === "orphaned" ||
1947
+ latest?.state === "pushed-green"
1892
1948
  ? latest
1893
1949
  : undefined;
1894
1950
  // The second half asks "is this open PR our retained work", and accepts
@@ -1966,6 +2022,24 @@ export async function dispatchAdmissions(
1966
2022
  // ----------------------------------------------------------------------- a tick
1967
2023
 
1968
2024
  export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
2025
+ // A config edit takes effect on the next tick, not the next daemon restart
2026
+ // (#170). Re-resolve the project and its caps at the tick boundary so a tick
2027
+ // and every run it admits see one consistent snapshot; a failed read keeps
2028
+ // the boot values rather than wedging the tick, and the next tick tries
2029
+ // again.
2030
+ try {
2031
+ const cfg = loadConfig();
2032
+ const fresh = findProject(cfg, d.project.name);
2033
+ const freshCaps = resolveCaps(fresh, cfg.defaults);
2034
+ if (JSON.stringify(fresh) !== JSON.stringify(d.project) || JSON.stringify(freshCaps) !== JSON.stringify(d.caps)) {
2035
+ log(`config reloaded: project ${d.project.name} changed on disk — applying from this tick`);
2036
+ }
2037
+ d.project = fresh;
2038
+ d.caps = freshCaps;
2039
+ } catch (err) {
2040
+ log(`config reload failed (${errText(err)}) — continuing with the values loaded at boot`);
2041
+ }
2042
+
1969
2043
  // Before the pause check, deliberately. This one is not about dispatch: the
1970
2044
  // orchestrator is a different process, and it can be wedged while this fleet
1971
2045
  // is paused — which is exactly the state the reference fleet was in when the
@@ -2643,6 +2717,14 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
2643
2717
  }
2644
2718
  if (run.state === "failed" && facts.pr === "open") {
2645
2719
  facts.checks = await tracker.checkConclusions(run.prUrl);
2720
+ // When a check failed with a reachable log, pull its tail so the
2721
+ // table can tell an infra outage (#177) from a real test failure by
2722
+ // the log's own words. First failure wins; a log that cannot be
2723
+ // fetched is left undefined and classification stays conservative.
2724
+ const firstFailure = facts.checks.find((c) => c.state === "failure" && c.link !== undefined);
2725
+ if (firstFailure?.link !== undefined) {
2726
+ facts.failingLog = await tracker.checkLog(firstFailure.link);
2727
+ }
2646
2728
  }
2647
2729
  }
2648
2730
  } catch (err) {
@@ -2730,6 +2812,26 @@ async function recoverRun(
2730
2812
  }
2731
2813
 
2732
2814
  if (recovery === "requeue") {
2815
+ // A dispatch-infra requeue that keeps landing on the same issue means the
2816
+ // mirror for its repo is persistently broken — a ref-lock that retry already
2817
+ // exhausted, a dead remote. Requeueing forever spends a turn-0 run per tick
2818
+ // with no chance of success, so after a bounded number of strikes this
2819
+ // escalates to a human instead (#168, #177).
2820
+ if (cls === "dispatch-infra" && store.classCountFor(project.name, run.issue, "dispatch-infra") >= DISPATCH_INFRA_MAX_STRIKES) {
2821
+ await safeEscalate(d, {
2822
+ tier: 1,
2823
+ project: project.name,
2824
+ issue: run.issue,
2825
+ summary: `[dispatch-infra] #${run.issue}: the mirror for ${run.repo} is failing persistently — ${evidence}`,
2826
+ detail: [
2827
+ `The dispatcher could not provision a worktree for #${run.issue} ${DISPATCH_INFRA_MAX_STRIKES} times in a row, all before the worker's first turn.`,
2828
+ "The mirror on this host needs attention (check disk, SSH/HTTPS credentials, and the mirror root).",
2829
+ ].join("\n"),
2830
+ });
2831
+ store.updateRun(run.id, { recoveredAt: Date.now() });
2832
+ log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
2833
+ return;
2834
+ }
2733
2835
  // Only when the tracker still shows this issue as ours to hand back. An
2734
2836
  // issue that is closed, or has no state label, was resolved by another route
2735
2837
  // and requeueing it would dispatch work nobody asked for.
@@ -2772,7 +2874,15 @@ async function recoverRun(
2772
2874
  );
2773
2875
  }
2774
2876
  if (run.lastError !== undefined && cls !== "question") detail.push(run.lastError);
2775
- detail.push(`Session: ${run.sessionFile ?? "(no transcript)"}`);
2877
+ // #172: an unwritten transcript is "the run died before it flushed", not a
2878
+ // link to a file the operator will open and find missing.
2879
+ detail.push(
2880
+ run.sessionFile === undefined
2881
+ ? "Session: (no transcript)"
2882
+ : existsSync(run.sessionFile)
2883
+ ? `Session: ${run.sessionFile}`
2884
+ : `Session: ${run.sessionFile} (file was never written — the run died before its transcript was flushed)`,
2885
+ );
2776
2886
  // The class and the run are in the summary, which is what the notifications
2777
2887
  // ledger dedupes on — so one class escalates once per run rather than every
2778
2888
  // five minutes.
@@ -22,6 +22,10 @@ export interface ClassifyFacts {
22
22
  pr?: "open" | "merged" | "closed";
23
23
  mergeable?: "conflicting" | "clean" | "unknown";
24
24
  checks?: { name: string; state: string; link?: string }[];
25
+ /** Tail (ANSI-stripped) of the first failed check's log, when one was
26
+ * reachable. Lets the table tell an infrastructure outage (#177) from a
27
+ * deterministic test failure by the log's own words. */
28
+ failingLog?: string;
25
29
  }
26
30
 
27
31
  export interface Classification {
@@ -50,6 +54,24 @@ const INFRA_CHECK_STATES: Record<string, true> = {
50
54
  /** States that mean "this check has a verdict and it is good". */
51
55
  const SUCCESS_CHECK_STATES: Record<string, true> = { success: true, neutral: true };
52
56
 
57
+ /**
58
+ * Substrings in a failed check's log that prove the failure was infrastructure,
59
+ * not the diff (#177). Each is a registry/docker/runner fault a worker cannot
60
+ * have introduced: a rate limit, an image-manifest resolution failure, a runner
61
+ * being torn down under the job, or a DNS failure. Matched lowercased against
62
+ * the log tail.
63
+ *
64
+ * Deliberately *not* matching bare `failed to solve:` — a docker build failure
65
+ * often prints it with a real resolution error, so the closing words carry the
66
+ * signal, not the phrase.
67
+ */
68
+ const INFRA_LOG_SIGNATURES = [
69
+ "429 too many requests",
70
+ "failed to resolve source metadata for",
71
+ "the runner has received a shutdown signal",
72
+ "could not resolve host",
73
+ ];
74
+
53
75
  function normalise(state: string): string {
54
76
  return state.trim().toLowerCase();
55
77
  }
@@ -127,6 +149,35 @@ export function neverStarted(
127
149
  return undefined;
128
150
  }
129
151
 
152
+ /**
153
+ * Evidence that the conductor's own git path failed before the worker's first
154
+ * turn, or `undefined` when this row is something else.
155
+ *
156
+ * A mirror ref-lock (#168), a dead mirror, a transiently unreachable host: the
157
+ * worker never touched the issue, so none of this should charge an
158
+ * implementation attempt. `lastError` of this shape is written only by the
159
+ * dispatch catch (`errText(err)` = the stack, which prefixes the thrown
160
+ * "git … exited N: …" message with "Error: ") and by state messages — never by
161
+ * a worker — so it is proof of the daemon's own git rather than a worker's
162
+ * failed push.
163
+ *
164
+ * Exported for the same reason {@link neverStarted} is: the store's one-time
165
+ * repair has to recognise exactly what the classifier does, or history and
166
+ * future disagree about which rows were the daemon's fault.
167
+ */
168
+ export function dispatchInfra(
169
+ run: Pick<RunRecord, "turns" | "lastError" | "prUrl" | "headSha" | "salvageSha">,
170
+ ): string | undefined {
171
+ if (run.turns !== 0) return undefined;
172
+ if (run.prUrl !== undefined || run.headSha !== undefined || run.salvageSha !== undefined) return undefined;
173
+ if (run.lastError === undefined || !DISPATCH_GIT_ERROR.test(run.lastError)) return undefined;
174
+ return `dispatch failed in the conductor's own git path before turn 1: ${run.lastError.split("\n")[0]}`;
175
+ }
176
+
177
+ // See {@link dispatchInfra}: the "Error: " prefix is what `errText` puts in
178
+ // front of the thrown message, so a bare /^git / would never match a real row.
179
+ const DISPATCH_GIT_ERROR = /^(?:Error: )?git .+ exited \d+/s;
180
+
130
181
  export function classifyRun(run: RunRecord, facts: ClassifyFacts): Classification {
131
182
  const hasArtifacts = run.prUrl !== undefined || run.headSha !== undefined || run.salvageSha !== undefined;
132
183
 
@@ -161,6 +212,16 @@ export function classifyRun(run: RunRecord, facts: ClassifyFacts): Classificatio
161
212
  }
162
213
  }
163
214
 
215
+ // Dispatch infra: the conductor's own git path failed before the worker's
216
+ // first turn. See {@link dispatchInfra} — the worker never touched the issue,
217
+ // so this must not charge an implementation attempt.
218
+ if (run.state === "failed") {
219
+ const evidence = dispatchInfra(run);
220
+ if (evidence !== undefined) {
221
+ return { cls: "dispatch-infra", recovery: "requeue", evidence };
222
+ }
223
+ }
224
+
164
225
  if (run.state === "blocked") {
165
226
  return {
166
227
  cls: "question",
@@ -215,6 +276,24 @@ export function classifyRun(run: RunRecord, facts: ClassifyFacts): Classificatio
215
276
  const checks = facts.checks ?? [];
216
277
  const unresolved = checks.filter((c) => !SUCCESS_CHECK_STATES[normalise(c.state)] === true);
217
278
  if (checks.length > 0 && unresolved.length > 0) {
279
+ // A failed check whose *log* smells like infrastructure — a registry 429,
280
+ // a runner shutdown, a DNS failure (#177). The check has a verdict, so
281
+ // the check-state table above cannot call it infra; only the log can. It
282
+ // beats ci-deterministic because charging an implementation attempt for a
283
+ // rate limit is exactly the waste that class exists to prevent.
284
+ if (facts.failingLog !== undefined) {
285
+ const lower = facts.failingLog.toLowerCase();
286
+ for (const signature of INFRA_LOG_SIGNATURES) {
287
+ if (lower.includes(signature)) {
288
+ const check = checks.find((c) => normalise(c.state) === "failure");
289
+ return {
290
+ cls: "ci-infra",
291
+ recovery: "rerun-checks",
292
+ evidence: `infrastructure signature in ${check?.name === undefined ? "failed check" : check.name} log: "${signature}"`,
293
+ };
294
+ }
295
+ }
296
+ }
218
297
  const failing = unresolved.filter((c) => normalise(c.state) === "failure");
219
298
  if (failing.length > 0) {
220
299
  return {
@@ -400,6 +400,16 @@ export const TICK_SCOPE_CONSTRAINTS: { readonly [K in ReportScope]: string } = {
400
400
  export const TICK_DELIVERY_RULE =
401
401
  "This tick was injected locally, not sent from Telegram, so your end-of-turn text does NOT reach your operator. Hand anything reportable this turn to the durable outbox by running `omp-conductor report --text \"<the whole report>\"` (add `--kind digest` for the daily digest) and confirming it printed a report id; the daemon then owns delivery and retries until it lands. Never claim a report was sent otherwise, and never use telegram_send for a report -- that path leaves no record that it went out.";
402
402
 
403
+ /** The {@link TICK_DELIVERY_RULE} variant for a fleet whose bridge actually
404
+ * delivers the tick's ending text (#169). That is a narrower class than the
405
+ * approval surface being ready: `notifyMode: "always"` has to be set, and the
406
+ * profile has to be `interactive` — a `daemon` profile suppresses the
407
+ * `agent_end` notify post, so its end-of-turn text reaches nobody and the
408
+ * durable-outbox rule above is the honest one there. Everything here keeps the
409
+ * outbox as the message; only the opening claim about delivery differs. */
410
+ export const TICK_DELIVERY_RULE_BRIDGED =
411
+ "Your end-of-turn text IS delivered to the operator by the Telegram bridge (notifyMode: always). Still hand anything reportable to the durable outbox with omp-conductor report — and then close with at most one short line. The report is the message; never restate it in your closing text.";
412
+
403
413
  /** Re-exported so the tick's own contract stays readable from one file: the
404
414
  * constant itself lives beside the check that decides whether it is callable. */
405
415
  export { TELEGRAM_APPROVAL_TOOL };
@@ -1195,6 +1205,29 @@ function tick(pi: TickApi, ctx: TickContext, config: TickConfig, session: TickSe
1195
1205
  refreshComposedBriefBestEffort();
1196
1206
 
1197
1207
  const scope = resolveTickScope();
1208
+ // The transport contract, read once from the same file at the same moment so
1209
+ // the approval line, the delivery rule and the narration line cannot disagree
1210
+ // (#169, #179). No access file means no fleet channel to judge; a session
1211
+ // with no bound bridge token means omp-telegram posts nothing, so both halves
1212
+ // fail toward "not delivered" — the durable outbox instruction — rather than
1213
+ // asserting a delivery the bridge cannot make.
1214
+ const approval =
1215
+ config.accessFile === undefined
1216
+ ? undefined
1217
+ : session.bridgeTokenAtStart
1218
+ ? readApprovalSurface(config.accessFile)
1219
+ : ({
1220
+ kind: "missing",
1221
+ reason:
1222
+ `${TELEGRAM_APPROVAL_TOOL} unavailable on local ticks: omp-telegram had no bot token when this ` +
1223
+ "session started, so it bound none and its tools stay dead however complete the access file looks " +
1224
+ "now — run `/telegram on` in this session, or restart it, to rebind the bridge",
1225
+ } as const);
1226
+ const profile =
1227
+ config.accessFile === undefined || !session.bridgeTokenAtStart
1228
+ ? undefined
1229
+ : readDaemonProfile(config.accessFile);
1230
+
1198
1231
  // A configured message owns the ordinary reporting and delivery clauses.
1199
1232
  // Mechanical evidence is different: release-policy drift and repeated
1200
1233
  // operational friction must not disappear because an operator customized the
@@ -1205,7 +1238,8 @@ function tick(pi: TickApi, ctx: TickContext, config: TickConfig, session: TickSe
1205
1238
  session.scopeFallbackLogged = true;
1206
1239
  pi.logger.info(`[omp-conductor] tick reporting scope: using ${DEFAULT_REPORT_SCOPE} — ${scope.fallback}`);
1207
1240
  }
1208
- content = `${defaultTickMessage(new Date(), scope.briefPath, scope.policyPath)}\n${TICK_SCOPE_CONSTRAINTS[scope.scope]}\n${TICK_DELIVERY_RULE}`;
1241
+ const bridged = approval?.kind === "ready" && approval.notifyMode === "always" && profile?.kind === "interactive";
1242
+ content = `${defaultTickMessage(new Date(), scope.briefPath, scope.policyPath)}\n${TICK_SCOPE_CONSTRAINTS[scope.scope]}\n${bridged ? TICK_DELIVERY_RULE_BRIDGED : TICK_DELIVERY_RULE}`;
1209
1243
  }
1210
1244
  let frictionStore: Store | undefined;
1211
1245
  let frictionSignals: FrictionSignal[] = [];
@@ -1275,18 +1309,6 @@ function tick(pi: TickApi, ctx: TickContext, config: TickConfig, session: TickSe
1275
1309
  // `telegram_ask` and `telegram_send` dead until `/telegram on`. Trusting the
1276
1310
  // file alone there puts the tick straight back to mandating a call its surface
1277
1311
  // cannot make, which is #114 exactly.
1278
- const approval =
1279
- config.accessFile === undefined
1280
- ? undefined
1281
- : session.bridgeTokenAtStart
1282
- ? readApprovalSurface(config.accessFile)
1283
- : ({
1284
- kind: "missing",
1285
- reason:
1286
- `${TELEGRAM_APPROVAL_TOOL} unavailable on local ticks: omp-telegram had no bot token when this ` +
1287
- "session started, so it bound none and its tools stay dead however complete the access file looks " +
1288
- "now — run `/telegram on` in this session, or restart it, to rebind the bridge",
1289
- } as const);
1290
1312
  if (approval?.kind === "missing") {
1291
1313
  content = `${content}\n${TICK_APPROVAL_UNAVAILABLE_RULE}`;
1292
1314
  if (!session.approvalToolMissingLogged) {
@@ -1318,10 +1340,6 @@ function tick(pi: TickApi, ctx: TickContext, config: TickConfig, session: TickSe
1318
1340
  // live bridge whose profile is not `daemon`.
1319
1341
  //
1320
1342
  // No `accessFile` means no fleet bridge to judge, exactly as above.
1321
- const profile =
1322
- config.accessFile === undefined || !session.bridgeTokenAtStart
1323
- ? undefined
1324
- : readDaemonProfile(config.accessFile);
1325
1343
  if (profile?.kind === "interactive") content = `${content}\n${TICK_NARRATION_RULE}`;
1326
1344
 
1327
1345
  try {
package/src/store.ts CHANGED
@@ -15,7 +15,7 @@ import { dirname, join } from "node:path";
15
15
 
16
16
  import { stateDir } from "./config.ts";
17
17
  import { DECISION_TTL_MS, DEFAULT_CAPS } from "./types.ts";
18
- import { neverStarted } from "./failure-class.ts";
18
+ import { dispatchInfra, neverStarted } from "./failure-class.ts";
19
19
  import type {
20
20
  DecisionDraft,
21
21
  FailureClass,
@@ -61,7 +61,7 @@ export const LIVE_STATES: readonly RunState[] = ["claimed", "running"];
61
61
  * worktrees removed, so they must not consume slots, while their live PRs must
62
62
  * still block duplicate attempts.
63
63
  */
64
- const ACTIVE_STATES: readonly RunState[] = [...LIVE_STATES, "pushed-pending", "pushed-green"];
64
+ export const ACTIVE_STATES: readonly RunState[] = [...LIVE_STATES, "pushed-pending", "pushed-green"];
65
65
 
66
66
  const LIVE_PLACEHOLDERS = LIVE_STATES.map(() => "?").join(", ");
67
67
  const ACTIVE_PLACEHOLDERS = ACTIVE_STATES.map(() => "?").join(", ");
@@ -726,6 +726,48 @@ export function openStore(dbPath: string): Store {
726
726
  for (const row of misread) reclassify.run(row.id);
727
727
  }
728
728
 
729
+ // The same repair, for the case the one above deliberately leaves alone: its
730
+ // comment names it exactly ("a turn-0 row carrying an unrecognised error (a
731
+ // failed mirror fetch, say) is untouched"). #177 gave that fault a name, but
732
+ // only for rows written after it existed. Every earlier one was logged
733
+ // `unknown`, whose recovery is `escalate` — so `runsNeedingClassification`
734
+ // never revisits it and `countFailures` counts it forever. With
735
+ // `maxAttemptsPerIssue` defaulting to 2, a single mirror ref-lock is half an
736
+ // issue's entire budget spent on a fault that was never in its code.
737
+ //
738
+ // Narrow the same way: exactly `unknown`, state `failed`, and the
739
+ // CLASSIFIER's own predicate rather than a second copy of the rule.
740
+ // Deliberately only `failureClass` — not `recoveryAction`/`recoveredAt`.
741
+ // Relieving the budget is the point; re-animating a months-old run into the
742
+ // requeue sweep is not, and clearing the recovery columns would do exactly
743
+ // that. Idempotent for free: these rows then read `dispatch-infra`, which the
744
+ // WHERE no longer matches.
745
+ const mirrorBroke = db
746
+ .query<
747
+ { id: string; lastError: string | null; prUrl: string | null; headSha: string | null; salvageSha: string | null },
748
+ []
749
+ >(
750
+ `SELECT id, lastError, prUrl, headSha, salvageSha FROM runs
751
+ WHERE failureClass = 'unknown' AND turns = 0 AND state = 'failed'`,
752
+ )
753
+ .all()
754
+ .filter(
755
+ (row) =>
756
+ dispatchInfra({
757
+ turns: 0,
758
+ ...(row.lastError === null ? {} : { lastError: row.lastError }),
759
+ ...(row.prUrl === null ? {} : { prUrl: row.prUrl }),
760
+ ...(row.headSha === null ? {} : { headSha: row.headSha }),
761
+ ...(row.salvageSha === null ? {} : { salvageSha: row.salvageSha }),
762
+ }) !== undefined,
763
+ );
764
+ if (mirrorBroke.length > 0) {
765
+ const reclassify = db.query<unknown, [string]>(
766
+ `UPDATE runs SET failureClass = 'dispatch-infra' WHERE id = ?`,
767
+ );
768
+ for (const row of mirrorBroke) reclassify.run(row.id);
769
+ }
770
+
729
771
  const insertRun = db.query<unknown, SqlValue[]>(
730
772
  `INSERT INTO runs (
731
773
  id, project, issue, repo, branch, worktree, state, attempt, turns,
@@ -773,12 +815,19 @@ export function openStore(dbPath: string): Store {
773
815
  const countFailures = db.query<{ n: number }, [string, number]>(
774
816
  `SELECT COUNT(*) AS n FROM runs
775
817
  WHERE project = ? AND issue = ? AND state = 'failed'
776
- AND (failureClass IS NULL OR failureClass NOT IN ('ci-infra', 'settlement-stuck', 'env-start-failure'))`,
818
+ AND (failureClass IS NULL OR failureClass NOT IN ('ci-infra', 'settlement-stuck', 'env-start-failure', 'dispatch-infra'))`,
777
819
  );
778
820
  const countContinuations = db.query<{ n: number }, [string, number]>(
779
821
  `SELECT COUNT(*) AS n FROM runs
780
822
  WHERE project = ? AND issue = ? AND state IN ('killed', 'orphaned', 'blocked')
781
- AND (failureClass IS NULL OR failureClass NOT IN ('admin-kill', 'settlement-stuck', 'env-start-failure'))`,
823
+ AND (failureClass IS NULL OR failureClass NOT IN ('admin-kill', 'settlement-stuck', 'env-start-failure', 'dispatch-infra'))`,
824
+ );
825
+ // How many times one issue reached a given class. Recovery uses it to bound a
826
+ // retry loop whose cause is persistent (e.g. a mirror that will not refresh):
827
+ // after `DISPATCH_INFRA_MAX_STRIKES` the daemon escalates instead of looping.
828
+ const countByClass = db.query<{ n: number }, [string, number, FailureClass]>(
829
+ `SELECT COUNT(*) AS n FROM runs
830
+ WHERE project = ? AND issue = ? AND failureClass = ?`,
782
831
  );
783
832
  // Newest first, and bounded: every row this returns costs `gh` calls to gather
784
833
  // facts for, so a fleet with a long unclassified history classifies over
@@ -1142,6 +1191,10 @@ export function openStore(dbPath: string): Store {
1142
1191
  return countContinuations.get(project, issue)?.n ?? 0;
1143
1192
  },
1144
1193
 
1194
+ classCountFor(project: string, issue: number, cls: FailureClass): number {
1195
+ return countByClass.get(project, issue, cls)?.n ?? 0;
1196
+ },
1197
+
1145
1198
  latestRun(project: string, issue: number): RunRecord | undefined {
1146
1199
  const row = selectLatestRun.get(project, issue);
1147
1200
  return row ? toRecord(row) : undefined;
@@ -695,6 +695,22 @@ export function makeTracker(p: ProjectConfig, runGh: typeof gh = gh): Tracker {
695
695
  }
696
696
  },
697
697
 
698
+ async checkLog(detailsUrl: string): Promise<string | undefined> {
699
+ const args = runLogArgs(detailsUrl);
700
+ if (args === undefined) return undefined;
701
+ try {
702
+ const raw = await runGh(args);
703
+ const lines = raw.replaceAll(/\u001b\[[0-9;]*m/g, "").split("\n");
704
+ return lines.slice(-400).join("\n");
705
+ } catch {
706
+ // Best-effort, never throws: a revoked token, a deleted run or a flaky
707
+ // network must not error the classifier out of gathering facts. The
708
+ // caller treats `undefined` as "could not read the log" and classifies
709
+ // conservatively (#177).
710
+ return undefined;
711
+ }
712
+ },
713
+
698
714
  async mergeable(prUrl: string): Promise<"conflicting" | "clean" | "unknown"> {
699
715
  if (!PR_URL.test(prUrl)) return "unknown";
700
716
  try {
package/src/types.ts CHANGED
@@ -766,6 +766,12 @@ export interface Tracker {
766
766
  * port: nothing downstream may read it as "no checks failed".
767
767
  */
768
768
  checkConclusions(prUrl: string): Promise<{ name: string; state: string; link?: string }[]>;
769
+ /** The tail of a failed workflow run's log, ANSI stripped, or `undefined`
770
+ * when it cannot be fetched. Best-effort, never throws: the classifier uses
771
+ * it to tell an infrastructure outage from a deterministic test failure
772
+ * (#177), and "could not read the log" must fall through to the existing
773
+ * conservative classification rather than error out of fact-gathering. */
774
+ checkLog(detailsUrl: string): Promise<string | undefined>;
769
775
  /** Whether the PR can merge into its base. `unknown` on any doubt, so a
770
776
  * mergeability nobody could read never becomes a conflict recovery. */
771
777
  mergeable(prUrl: string): Promise<"conflicting" | "clean" | "unknown">;
@@ -803,6 +809,7 @@ export const FAILURE_CLASSES = [
803
809
  "admin-kill",
804
810
  "ci-infra",
805
811
  "ci-deterministic",
812
+ "dispatch-infra",
806
813
  "merge-conflict",
807
814
  "question",
808
815
  "orphan-clean",
@@ -1155,6 +1162,10 @@ export interface Store {
1155
1162
  failuresFor(project: string, issue: number): number;
1156
1163
  /** Operational stops that require a bounded continuation resume. */
1157
1164
  continuationsFor(project: string, issue: number): number;
1165
+ /** How many times one issue has reached a given failure class. A recovery
1166
+ * that keeps landing on the same class (e.g. a persistently broken mirror)
1167
+ * uses this to escalate instead of retrying forever. */
1168
+ classCountFor(project: string, issue: number, cls: FailureClass): number;
1158
1169
  /** Newest attempt for one issue, whatever state it reached. `omp-conductor
1159
1170
  * tail` resolves an issue number to a transcript through this; the number is
1160
1171
  * what an operator has, the run id is not. */
@@ -1321,6 +1332,7 @@ export const VERB_NAMES = [
1321
1332
  "conductor_push",
1322
1333
  "conductor_pr_create",
1323
1334
  "conductor_pr_update_branch",
1335
+ "conductor_pr_update",
1324
1336
  "conductor_pr_merge",
1325
1337
  "conductor_label",
1326
1338
  "conductor_release",
package/src/unblock.ts CHANGED
@@ -40,6 +40,10 @@ export interface UnblockOutcome {
40
40
  continuationsUsed: number;
41
41
  /** Newest attempt, when the store has one for this issue at all. */
42
42
  latest?: RunRecord;
43
+ /** Whether any run for the issue is still in an active state — the answer
44
+ * to "will the dispatcher hold this issue as issue-active?" that #178
45
+ * found this verb guessing at. */
46
+ active: boolean;
43
47
  /** Set when nothing was cleared because the newest attempt's work exists
44
48
  * only in its worktree. Carries the salvage failure verbatim. */
45
49
  refused?: string;
@@ -92,10 +96,12 @@ export async function unblockIssue(
92
96
  // for clearing in-progress, so the row that carries it decides the set.
93
97
  const latest = store.latestRun(project.name, issue);
94
98
  const terminal = latest !== undefined && !LIVE_STATES.includes(latest.state);
99
+ const active = store.activeRuns(project.name).some((r) => r.issue === issue);
95
100
  const counts = {
96
101
  attemptsUsed: store.attemptsFor(project.name, issue),
97
102
  failuresUsed: store.failuresFor(project.name, issue),
98
103
  continuationsUsed: store.continuationsFor(project.name, issue),
104
+ active,
99
105
  };
100
106
 
101
107
  // The one case where this verb refuses. Clearing the labels here re-queues an
@@ -205,8 +211,31 @@ export function formatUnblock(
205
211
  `run row to prove the worker is gone, "${project.stateLabels.inProgress}" was left in place, and ` +
206
212
  "on its own it keeps the issue ineligible",
207
213
  );
214
+ } else if (latest.state === "pushed-green" || latest.state === "pushed-pending") {
215
+ // #178: an unblocked issue whose newest run pushed a green PR is not "free
216
+ // for a fresh attempt" — the dispatcher continues the pushed run's branch,
217
+ // and only once the PR resolves does the issue settle (#175). Say that
218
+ // instead of promising a re-claim the next tick withholds.
219
+ lines.push(
220
+ latest.prUrl === undefined
221
+ ? ` next tick eligible again, as long as the issue still carries "${project.queueLabel}" ` +
222
+ `(newest run is ${latest.state} with no recorded PR; the dispatcher still applies its open-PR check at claim time)`
223
+ : ` next tick eligible as a continuation of ${latest.prUrl} — the pushed run stays active until ` +
224
+ "that PR resolves; dispatch continues on its branch",
225
+ );
226
+ } else if (o.active) {
227
+ // A run other than the newest is still active (e.g. a live sibling under a
228
+ // terminal newest row), so the dispatcher will hold the issue as
229
+ // issue-active whatever the labels now say. #178's misleading case.
230
+ lines.push(
231
+ ` in flight a run for this issue is still active, so the dispatcher holds the issue until it settles — ` +
232
+ `nothing is re-claimed before then ("${project.stateLabels.inProgress}" stays unless already released)`,
233
+ );
208
234
  } else {
209
- lines.push(` next tick eligible again, as long as the issue still carries "${project.queueLabel}"`);
235
+ lines.push(
236
+ ` next tick eligible again, as long as the issue still carries "${project.queueLabel}" ` +
237
+ "(the dispatcher still applies its open-PR check at claim time)",
238
+ );
210
239
  }
211
240
 
212
241
  return lines.join("\n");
@@ -97,6 +97,14 @@ export function githubVerbActions(project: ProjectConfig, run: CommandRunner = s
97
97
 
98
98
  updatePrBranch: (prUrl) => gh(["pr", "update-branch", prUrl]),
99
99
 
100
+ updatePr: async (prUrl, fields) => {
101
+ const argv = ["pr", "edit", prUrl];
102
+ if (fields.title !== undefined) argv.push("--title", fields.title);
103
+ if (fields.body !== undefined) argv.push("--body", fields.body);
104
+ const outcome = await gh(argv);
105
+ return outcome.ok ? { ok: true } : { ok: false, stderr: outcome.stderr };
106
+ },
107
+
100
108
  // `--match-head-commit` is the server-side half of the exact-head rule: the
101
109
  // daemon already re-read the head and refused a stale one, and this makes
102
110
  // GitHub refuse too if the branch moved in the milliseconds between. Belt