omp-conductor 0.15.11 → 0.15.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/config.ts CHANGED
@@ -355,8 +355,11 @@ export function findProject(c: ConductorConfig, name?: string): ProjectConfig {
355
355
  if (name === undefined) {
356
356
  const only = c.projects[0];
357
357
  if (c.projects.length !== 1 || only === undefined) {
358
+ // #514: every operator hitting this has to guess the spelling of the
359
+ // flag that names a project — say it outright. Doctor's project finding
360
+ // quotes this text, so keep the "name one explicitly" phrase.
358
361
  throw new Error(
359
- `Ambiguous project: config has ${c.projects.length} projects (${names.join(", ") || "none"}) — name one explicitly.`,
362
+ `Ambiguous project: config has ${c.projects.length} projects (${names.join(", ") || "none"}) — name one explicitly with --project NAME.`,
360
363
  );
361
364
  }
362
365
  return only;
@@ -1080,6 +1083,34 @@ function finalizeProject(
1080
1083
  const workerModel =
1081
1084
  typeof rawWorkerModel === "string" && rawWorkerModel.trim() !== "" ? rawWorkerModel : undefined;
1082
1085
 
1086
+ // The failover chain is a hint like workerModel, not a ceiling: an unusable
1087
+ // entry or a malformed threshold is dropped rather than failing the load,
1088
+ // and absent/empty `modelFallbacks` keeps today's dispatch byte for byte.
1089
+ const rawFallbacks = p["modelFallbacks"];
1090
+ const modelFallbacks = Array.isArray(rawFallbacks)
1091
+ ? rawFallbacks
1092
+ .filter((entry): entry is string => typeof entry === "string")
1093
+ .map((entry) => entry.trim())
1094
+ .filter((entry) => entry !== "")
1095
+ : [];
1096
+ const rawThreshold = p["modelFallbackThreshold"];
1097
+ const modelFallbackThreshold =
1098
+ typeof rawThreshold === "number" && Number.isInteger(rawThreshold) && rawThreshold >= 1
1099
+ ? rawThreshold
1100
+ : undefined;
1101
+
1102
+ // Marked critical-base/safety markers (commit SHAs or refs). A continuation
1103
+ // branch must contain every one before the dispatcher may reattach it;
1104
+ // unusable entries are dropped like `modelFallbacks` and an absent or empty
1105
+ // list keeps today's behaviour byte for byte.
1106
+ const rawCriticalBase = p["criticalBase"];
1107
+ const criticalBase = Array.isArray(rawCriticalBase)
1108
+ ? rawCriticalBase
1109
+ .filter((entry): entry is string => typeof entry === "string")
1110
+ .map((entry) => entry.trim())
1111
+ .filter((entry) => entry !== "")
1112
+ : [];
1113
+
1083
1114
  if (problems.length > before) return undefined;
1084
1115
 
1085
1116
  return {
@@ -1095,12 +1126,15 @@ function finalizeProject(
1095
1126
  routing: { labelPrefix, repos },
1096
1127
  caps,
1097
1128
  ...(workerModel === undefined ? {} : { workerModel }),
1129
+ ...(modelFallbacks.length === 0 ? {} : { modelFallbacks }),
1130
+ ...(modelFallbackThreshold === undefined ? {} : { modelFallbackThreshold }),
1098
1131
  escalation,
1099
1132
  authority,
1100
1133
  releasePolicy,
1101
1134
  policy,
1102
1135
  ...(recoveryMerges === undefined ? {} : { recoveryMerges }),
1103
1136
  reporting,
1137
+ ...(criticalBase.length === 0 ? {} : { criticalBase }),
1104
1138
  workspaceRoot: expandHome(pickString(p["workspaceRoot"], defaultWorkspaceRoot())),
1105
1139
  mirrorRoot: expandHome(pickString(p["mirrorRoot"], defaultMirrorRoot())),
1106
1140
  };
package/src/daemon.ts CHANGED
@@ -47,12 +47,21 @@ import {
47
47
  } from "./reports.ts";
48
48
  import {
49
49
  recordReleaseBlock,
50
+ type GateShape,
50
51
  type ReleaseBlockContext,
51
52
  } from "./release-policy.ts";
52
53
  import { branchName, effectiveLabels, route } from "./routing.ts";
53
54
  import type { Routed, UnroutableReason } from "./routing.ts";
54
55
  import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
55
56
  import { classifyRun, providerCreditRefusal, providerTransientFault, type ClassifyFacts } from "./failure-class.ts";
57
+ import {
58
+ DEFAULT_MODEL_FALLBACK_THRESHOLD,
59
+ fallbackClause,
60
+ formatModelsTried,
61
+ modelsTried,
62
+ providerFailureFacts,
63
+ resolveDispatchModel,
64
+ } from "./model-fallback.ts";
56
65
  import { projectLabels } from "./label-projection.ts";
57
66
  import { dbPath, LIVE_STATES, openStore, utcDay } from "./store.ts";
58
67
  import { makeTracker, type RateLimitStatus } from "./tracker/github.ts";
@@ -64,6 +73,7 @@ import type {
64
73
  DispatchSummary,
65
74
  DigestBacklog,
66
75
  Escalation,
76
+ IssueComment,
67
77
  IssueSnapshot,
68
78
  MergedPrInfo,
69
79
  OpenCloser,
@@ -122,7 +132,14 @@ import {
122
132
  } from "./verbs/socket.ts";
123
133
  import { homedir } from "node:os";
124
134
 
125
- import { pushRunBranch, readBaseChain, type RunRepoRef } from "./gitops.ts";
135
+ import {
136
+ probeCriticalBase,
137
+ pushRunBranch,
138
+ readBaseChain,
139
+ type CriticalBaseProbe,
140
+ type CriticalBaseVerdict,
141
+ type RunRepoRef,
142
+ } from "./gitops.ts";
126
143
  import {
127
144
  planUsageLine,
128
145
  readPlanUsage,
@@ -233,6 +250,14 @@ interface Deps {
233
250
  * without touching the host's packages, journal or outbox.
234
251
  */
235
252
  upgradeVerifier?: (d: Deps) => Promise<void>;
253
+ /**
254
+ * Answers whether a preserved continuation branch contains every configured
255
+ * critical-base marker, for the stale-base admission hold (#428). Wired by
256
+ * `runDaemon` to the mirror-backed {@link probeCriticalBase}; a test injects
257
+ * a fake. Absent, the stale-base gate fails closed whenever a project names
258
+ * a marker (a safety interlock must not silently weaken).
259
+ */
260
+ probeCriticalBase?: CriticalBaseProbe;
236
261
  }
237
262
 
238
263
  /**
@@ -682,6 +707,64 @@ function acceptanceCriteria(issue: ReadyIssue): string {
682
707
  : "_The issue body is empty. Read the issue and its comments, and escalate if it is genuinely underspecified._";
683
708
  }
684
709
 
710
+ /**
711
+ * How much of a comment thread one brief may hold. Grooming notes are short,
712
+ * and a thread that outgrows this is truncated with an explicit marker rather
713
+ * than a silently dropped tail — the disappearance this fix exists to prevent.
714
+ */
715
+ const DISCUSSION_CHARS_BUDGET = 8_000;
716
+
717
+ /**
718
+ * The issue's comments as a brief section: attributed, numbered, oldest
719
+ * first, so a later correction visibly supersedes an earlier note. Rendered
720
+ * beside the body at dispatch, because the orchestrator's grooming is posted
721
+ * as comments and a worker must never depend on a runtime read to see it.
722
+ *
723
+ * `"unread"` is not "empty": when the tracker refused at dispatch the section
724
+ * says so, instead of silently reading as "no discussion" — which is exactly
725
+ * the #517 failure. An empty comment list renders nothing, so a commentless
726
+ * issue's brief stays byte-identical to what this package has always shipped.
727
+ */
728
+ function renderDiscussion(comments: IssueComment[] | "unread"): string {
729
+ if (comments === "unread") {
730
+ return [
731
+ "## Discussion",
732
+ "",
733
+ "_The issue's comments could not be read at dispatch time. The live read below is_",
734
+ "_the only path to them; if it prints nothing, that is a failed read, not an_",
735
+ "_absence of discussion._",
736
+ "",
737
+ ].join("\n");
738
+ }
739
+ if (comments.length === 0) return "";
740
+ const total = comments.length;
741
+ const lines = [
742
+ "## Discussion",
743
+ "",
744
+ `${total} comment${total === 1 ? "" : "s"} on the issue at dispatch, oldest first — a later`,
745
+ "comment supersedes an earlier one.",
746
+ "",
747
+ ];
748
+ let chars = lines.join("\n").length;
749
+ let shown = 0;
750
+ for (const comment of comments) {
751
+ const block = `**@${comment.author} — comment ${shown + 1}:**\n\n${comment.body}\n\n`;
752
+ if (shown > 0 && chars + block.length > DISCUSSION_CHARS_BUDGET) break;
753
+ lines.push(`**@${comment.author} — comment ${shown + 1}:**`, "", comment.body, "");
754
+ chars += block.length;
755
+ shown += 1;
756
+ }
757
+ if (shown < total) {
758
+ const omitted = total - shown;
759
+ lines.push(
760
+ `… ${omitted} comment${omitted === 1 ? "" : "s"} omitted — the discussion exceeded ${DISCUSSION_CHARS_BUDGET} characters.`,
761
+ "Read the issue for the tail.",
762
+ "",
763
+ );
764
+ }
765
+ return lines.join("\n");
766
+ }
767
+
685
768
  /**
686
769
  * Record a state-label swap for projection (#201). The add enqueues before the
687
770
  * remove — the reverse order would leave a window where the issue carries no
@@ -934,7 +1017,15 @@ export async function buildBrief(
934
1017
  r: Routed,
935
1018
  branch: string,
936
1019
  worktree: string,
937
- opts: { continuation?: boolean; defaultBranch?: string; salvagedSha?: string } = {},
1020
+ opts: {
1021
+ continuation?: boolean;
1022
+ defaultBranch?: string;
1023
+ salvagedSha?: string;
1024
+ /** The issue's comments, rendered into the Discussion section. `"unread"`
1025
+ * when the tracker refused at dispatch — rare, but it must never read as
1026
+ * "no comments" (the #517 failure mode). Absent means an empty list. */
1027
+ comments?: IssueComment[] | "unread";
1028
+ } = {},
938
1029
  ): Promise<string> {
939
1030
  // Read per dispatch rather than caching: editing the brief then takes effect
940
1031
  // on the next issue instead of needing a daemon restart.
@@ -979,6 +1070,7 @@ export async function buildBrief(
979
1070
  BRANCH: branch,
980
1071
  WORKTREE: worktree,
981
1072
  ACCEPTANCE_CRITERIA: acceptanceCriteria(r.issue),
1073
+ ISSUE_COMMENTS: renderDiscussion(opts.comments ?? []),
982
1074
  GATES: gatesBlock(r.repo),
983
1075
  // Empty for a repo with no `graphProject`, and empty means *nothing*: the
984
1076
  // placeholder sits flush against the next list item in the template, so an
@@ -1045,6 +1137,135 @@ function newestContinuableRun(runs: readonly RunRecord[]): RunRecord | undefined
1045
1137
  );
1046
1138
  }
1047
1139
 
1140
+ /** The fenced-block info string that marks an exhaustion postmortem comment, so
1141
+ * a grooming scout re-slicing the issue can find and parse the whole block by
1142
+ * grepping for it. */
1143
+ export const POSTMORTEM_MARKER = "conductor-postmortem";
1144
+
1145
+ /** How one issue's attempt chain failed, as "3× ci-deterministic, 1× …" — the
1146
+ * digest shape for naming what the exhaustion was. Groups every row by its
1147
+ * failure class, whether or not it charged the continuation budget, because
1148
+ * the postmortem tells the whole story and not just the budget half (#290). */
1149
+ export function attemptClassBreakdown(runs: readonly RunRecord[]): string {
1150
+ const perClass = new Map<string, number>();
1151
+ for (const r of runs) {
1152
+ const cls = r.failureClass ?? "unclassified";
1153
+ perClass.set(cls, (perClass.get(cls) ?? 0) + 1);
1154
+ }
1155
+ if (perClass.size === 0) return "unclassified";
1156
+ return Array.from(perClass, ([cls, n]) => `${n}× ${cls}`).join(", ");
1157
+ }
1158
+
1159
+ /** Wall-clock duration of one run as a compact human string ("45m", "1h30m"). */
1160
+ export function humanDuration(ms: number): string {
1161
+ const seconds = Math.max(0, Math.round(ms / 1_000));
1162
+ if (seconds < 60) return `${seconds}s`;
1163
+ const minutes = Math.round(seconds / 60);
1164
+ if (minutes < 60) return `${minutes}m`;
1165
+ const hours = Math.floor(minutes / 60);
1166
+ const rest = minutes % 60;
1167
+ return rest === 0 ? `${hours}h` : `${hours}h${rest}m`;
1168
+ }
1169
+
1170
+ /** Flatten and bound a run's last error to one greppable table line. */
1171
+ export function oneLineBrief(text: string | undefined): string | undefined {
1172
+ if (text === undefined || text.trim() === "") return undefined;
1173
+ const flat = text.replace(/\s+/g, " ").trim();
1174
+ return flat.length > 90 ? `${flat.slice(0, 89)}…` : flat;
1175
+ }
1176
+
1177
+ /**
1178
+ * The exhaustion postmortem block: one greppable fenced block covering every
1179
+ * attempt in the chain — continuation rows included — with per-attempt turns,
1180
+ * wall clock, failure class and a one-line last error, the explicit salvage
1181
+ * state, the spend total and the transcript paths for local inspection.
1182
+ *
1183
+ * Pure so the tests hold the shape, not the transport: the writer below owns
1184
+ * the once-only guarantee, this owns what "once" looks like.
1185
+ */
1186
+ export function formatExhaustionPostmortem(args: {
1187
+ issue: number;
1188
+ runs: readonly RunRecord[];
1189
+ reason: string;
1190
+ }): string {
1191
+ const { issue, runs, reason } = args;
1192
+ const artifact = newestContinuableRun(runs);
1193
+ const totalSpend = runs.reduce((sum, r) => sum + r.spendUsd, 0);
1194
+ const spend = `$${totalSpend.toFixed(2)}`;
1195
+ const attemptLines = runs.map((r) => {
1196
+ const wall = r.endedAt === undefined ? "—" : humanDuration(r.endedAt - r.startedAt);
1197
+ const cls = r.failureClass ?? "unclassified";
1198
+ const error = oneLineBrief(r.lastError) ?? "—";
1199
+ return (
1200
+ ` attempt ${r.attempt} ${r.state.padEnd(12)} turns ${r.turns}/${r.maxTurns} ` +
1201
+ `${cls.padEnd(24)} ${wall.padStart(4)} last error: ${error}`
1202
+ );
1203
+ });
1204
+ const salvage =
1205
+ artifact === undefined
1206
+ ? "Salvaged WIP: absent — no attempt preserved a branch, head SHA or pull request."
1207
+ : `Salvaged WIP: present — branch ${artifact.branch} at ${artifact.headSha ?? artifact.salvageSha}` +
1208
+ `${artifact.prUrl === undefined ? "" : ` (PR ${artifact.prUrl})`}.`;
1209
+ return [
1210
+ `\`\`\`${POSTMORTEM_MARKER}`,
1211
+ `#${issue} exhausted: ${attemptClassBreakdown(runs)}`,
1212
+ reason,
1213
+ "",
1214
+ `Attempts (${runs.length} total, ${spend} spend):`,
1215
+ ...attemptLines,
1216
+ salvage,
1217
+ `Spend total: ${spend} across ${runs.length} attempts.`,
1218
+ "Transcripts:",
1219
+ ...runs.map((r) => (r.sessionFile === undefined ? " (none)" : ` ${r.sessionFile}`)),
1220
+ "```",
1221
+ ].join("\n");
1222
+ }
1223
+
1224
+ /** Dedupe key prefix for the exhaustion postmortem comment, per issue. */
1225
+ function postmortemDedupeKey(project: string, issue: number): string {
1226
+ return `${project}:postmortem:${issue}`;
1227
+ }
1228
+
1229
+ /**
1230
+ * The exhaustion postmortem: written exactly once per issue, at the point the
1231
+ * continuation budget is spent and the issue is settled toward a human.
1232
+ *
1233
+ * The comment and the material event are both gated by the store's notification
1234
+ * ledger — the same idempotence guard the escalator uses — so re-settling an
1235
+ * already-postmortemed issue posts nothing and records nothing. A body-string
1236
+ * match on the issue would be the wrong guard: an issue re-scoped and re-run
1237
+ * would still carry the old block, and the guarantee asked of this is "decided
1238
+ * once", not "deduped against what is already written".
1239
+ *
1240
+ * The digest must name the exhaustion even if the comment write fails, so the
1241
+ * material event is recorded before the write and unconditionally (the ledger
1242
+ * is append-only, and the gate above already ran once). The comment failure is
1243
+ * logged rather than taking the sweep down with it.
1244
+ */
1245
+ async function postExhaustionPostmortem(d: Deps, run: RunRecord, reason: string): Promise<void> {
1246
+ const { project, tracker, store } = d;
1247
+ const key = postmortemDedupeKey(project.name, run.issue);
1248
+ if (store.wasNotified(key)) return;
1249
+ const runs = store.runsForIssue(project.name, run.issue);
1250
+ const body = formatExhaustionPostmortem({ issue: run.issue, runs, reason });
1251
+ const occurredAt = Date.now();
1252
+ store.recordMaterialEvent({
1253
+ project: project.name,
1254
+ category: "exhaustion",
1255
+ summary: `#${run.issue} exhausted: ${attemptClassBreakdown(runs)}`,
1256
+ evidence: body,
1257
+ occurredAt,
1258
+ recordedAt: occurredAt,
1259
+ });
1260
+ try {
1261
+ await tracker.comment(run.issue, body);
1262
+ store.markNotified(key);
1263
+ log(`#${run.issue} posted exhaustion postmortem (${runs.length} attempts)`);
1264
+ } catch (err) {
1265
+ log(`#${run.issue} postmortem comment could not be posted (${errText(err)})`);
1266
+ }
1267
+ }
1268
+
1048
1269
  export type ExtendTurnLimitResult =
1049
1270
  | { kind: "extended"; runId: string; maxTurns: number }
1050
1271
  | { kind: "not-increase"; runId: string; maxTurns: number }
@@ -1329,7 +1550,7 @@ export function workerReleaseBlockRecorder(
1329
1550
  issue: number,
1330
1551
  runId: string,
1331
1552
  root = stateDir(),
1332
- ): (shape: ReleaseShape, context: ReleaseBlockContext) => void {
1553
+ ): (shape: GateShape, context: ReleaseBlockContext) => void {
1333
1554
  return (shape, context) =>
1334
1555
  recordReleaseBlock(
1335
1556
  project,
@@ -1544,6 +1765,11 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
1544
1765
  // attempt whose work this one inherits.
1545
1766
  const prior = store.latestRun(project.name, issue);
1546
1767
  const priorSalvage = prior?.salvageSha;
1768
+ // The failover decision is made from the store the same way admission is:
1769
+ // counts of this issue's own rows, so the switch is sticky per chain and a
1770
+ // fresh issue always starts on the primary (#286). Read before the claim
1771
+ // writes this attempt's row, which would otherwise break the streak.
1772
+ const chainFacts = providerFailureFacts(store.runsForIssue(project.name, issue));
1547
1773
  run = store.createRun({
1548
1774
  project: project.name,
1549
1775
  issue,
@@ -1657,17 +1883,50 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
1657
1883
 
1658
1884
  store.updateRun(runId, { worktree: worktreePath, state: "running" });
1659
1885
 
1886
+ // Where this attempt goes, and (when the failover fired) the clause that
1887
+ // makes it attributable: with no `modelFallbacks` configured this is the
1888
+ // primary model — or none, for an unconfigured project — and today's
1889
+ // dispatch is byte for byte what it has always been.
1890
+ const chainConfigured = (project.modelFallbacks?.length ?? 0) > 0;
1891
+ const choice = resolveDispatchModel({
1892
+ workerModel: project.workerModel,
1893
+ modelFallbacks: project.modelFallbacks,
1894
+ threshold: project.modelFallbackThreshold ?? DEFAULT_MODEL_FALLBACK_THRESHOLD,
1895
+ streak: chainFacts.streak,
1896
+ });
1897
+ // Recorded before the launch, so even a run killed mid-flight leaves the
1898
+ // model it chose on its row. Only a chain-configured project writes the
1899
+ // column: absent `modelFallbacks` must preserve today's rows byte for byte.
1900
+ if (chainConfigured && choice.model !== undefined) {
1901
+ store.updateRun(runId, { model: choice.model });
1902
+ }
1903
+ const clause = fallbackClause(choice, chainFacts, project.workerModel);
1904
+
1660
1905
  log(
1661
- `#${issue} attempt ${attempt} → ${r.repo.name} ${branch}` +
1906
+ `#${issue} attempt ${attempt}${clause === undefined ? "" : ` ${clause}`} → ${r.repo.name} ${branch}` +
1662
1907
  (provisioned.reattached ? " (continuation: reattached existing branch)" : ""),
1663
1908
  );
1664
1909
 
1910
+ // The discussion is rendered at dispatch so a worker never depends on a
1911
+ // runtime `gh` read to see the orchestrator's grooming (#517). The read is
1912
+ // best-effort, but its failure is not silent: an unreadable tracker names
1913
+ // itself in the brief's Discussion section instead of reading as "no
1914
+ // comments" — the exact confusion this fix removes.
1915
+ let comments: IssueComment[] | "unread";
1916
+ try {
1917
+ comments = await tracker.listComments(issue);
1918
+ } catch (err) {
1919
+ log(`#${issue} issue comments unreadable at dispatch; the brief will say so: ${errText(err)}`);
1920
+ comments = "unread";
1921
+ }
1922
+
1665
1923
  const brief = await buildBrief(project, r, branch, worktreePath, {
1666
1924
  continuation: provisioned.reattached,
1667
1925
  defaultBranch: r.repo.defaultBranch,
1668
1926
  ...(provisioned.reattached && priorSalvage !== undefined
1669
1927
  ? { salvagedSha: priorSalvage }
1670
1928
  : {}),
1929
+ comments,
1671
1930
  });
1672
1931
  if (await settleStopBeforeSession()) return;
1673
1932
  if (await settleDrainBeforeSession()) return;
@@ -1702,7 +1961,7 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
1702
1961
  onChildLog: (line) => {
1703
1962
  log(`#${issue} ${line}`);
1704
1963
  },
1705
- ...(project.workerModel === undefined ? {} : { model: project.workerModel }),
1964
+ ...(choice.model === undefined ? {} : { model: choice.model }),
1706
1965
  releaseGrants: resolveReleaseGrants(project),
1707
1966
  onReleaseBlocked: workerReleaseBlockRecorder(project.name, issue, runId),
1708
1967
  onTurn: (n) => store.updateRun(runId, { turns: n }),
@@ -2886,7 +3145,7 @@ function planUsageEscalation(project: string, plan: PlanUsageStatus): Escalation
2886
3145
  * that grows a field has no business breaking these tests.
2887
3146
  */
2888
3147
  export async function admitCandidates(
2889
- d: Pick<Deps, "project" | "caps" | "tracker" | "store" | "escalate" | "usage">,
3148
+ d: Pick<Deps, "project" | "caps" | "tracker" | "store" | "escalate" | "usage" | "probeCriticalBase">,
2890
3149
  routed: Routed[],
2891
3150
  slots: number,
2892
3151
  ): Promise<AdmissionPass> {
@@ -3066,6 +3325,81 @@ export async function admitCandidates(
3066
3325
  continue;
3067
3326
  }
3068
3327
 
3328
+ // #428 half (a): a preserved continuation that predates a configured
3329
+ // critical-base/safety marker must not be reattached. A base safety fix
3330
+ // protects only branches forked after it landed — a continuation forked
3331
+ // before it still carries the dangerous test/runtime code, and re-running
3332
+ // it on the shared host is what SIGTERMed the production daemon. Fail
3333
+ // closed: only a probe that proves every marker is in the reattach
3334
+ // source's ancestry admits, and a project that names a marker but has no
3335
+ // probe wired (never happens outside tests) holds. Both the hold and the
3336
+ // escalation are durable across restart and orphan recovery because this
3337
+ // gate runs every admission pass; the branch is re-admitted automatically
3338
+ // once the operator updates it to contain the marker, without losing work.
3339
+ const markers = project.criticalBase ?? [];
3340
+ if (markers.length > 0) {
3341
+ const branch = branchName(r.issue);
3342
+ let verdict: CriticalBaseVerdict;
3343
+ if (d.probeCriticalBase === undefined) {
3344
+ verdict = { state: "unknown", error: "no critical-base probe is wired in this deployment" };
3345
+ } else {
3346
+ try {
3347
+ verdict = await d.probeCriticalBase(r.repo, markers, branch);
3348
+ } catch (err) {
3349
+ verdict = { state: "unknown", error: errText(err) };
3350
+ }
3351
+ }
3352
+ if (verdict.state === "stale") {
3353
+ hold(issue, "stale-base");
3354
+ log(
3355
+ `#${issue} held (stale-base): continuation branch ${branch} predates critical-base marker ${verdict.marker}`,
3356
+ );
3357
+ await safeEscalate(d, {
3358
+ tier: 1,
3359
+ project: project.name,
3360
+ issue,
3361
+ summary: `#${issue} continuation branch predates a critical base safety commit and is held (stale-base)`,
3362
+ detail: [
3363
+ r.issue.title,
3364
+ r.issue.url,
3365
+ `The retained branch ${branch} does not contain critical-base marker ${verdict.marker}.`,
3366
+ ...(verdict.range.length > 0
3367
+ ? [`Base commits the branch is missing: ${verdict.range.join(", ")}`]
3368
+ : []),
3369
+ "Recovery: merge current base into the branch so it contains the marker, and the next",
3370
+ "admission pass re-admits it automatically without losing the branch's work; or review",
3371
+ "the branch by hand and clear the hold once the fix is present.",
3372
+ ].join("\n"),
3373
+ });
3374
+ continue;
3375
+ }
3376
+ if (verdict.state === "unknown") {
3377
+ // Fail closed: a branch that cannot be *proven* to contain the marker
3378
+ // is refused, and the reason names the unverifiable marker so the
3379
+ // operator can fix the fetch or the marker rather than guess.
3380
+ hold(issue, "stale-base");
3381
+ log(
3382
+ `#${issue} held (stale-base): continuation branch ${branch} could not be verified ` +
3383
+ `against critical-base marker(s) ${markers.join(", ")} (${verdict.error})`,
3384
+ );
3385
+ await safeEscalate(d, {
3386
+ tier: 1,
3387
+ project: project.name,
3388
+ issue,
3389
+ summary: `#${issue} continuation branch could not be verified against a critical base safety commit and is held (stale-base)`,
3390
+ detail: [
3391
+ r.issue.title,
3392
+ r.issue.url,
3393
+ `The retained branch ${branch} could not be verified against critical-base marker(s) ${markers.join(", ")}: ${verdict.error}`,
3394
+ "Recovery: merge current base into the branch so it contains the marker, and the next",
3395
+ "admission pass re-admits it automatically without losing the branch's work; or review",
3396
+ "the branch by hand and clear the hold once the fix is present.",
3397
+ ].join("\n"),
3398
+ });
3399
+ continue;
3400
+ }
3401
+ }
3402
+
3069
3403
  // Soft concurrency per epic, per repository: at most one in-flight child of
3070
3404
  // a given parent in each repo. Children of one epic in *different* repos
3071
3405
  // parallelise freely — `repo-active` / `maxConcurrentWorkersPerRepo` owns
@@ -4677,6 +5011,11 @@ async function recoverRun(
4677
5011
  "The issue cannot finish inside the wall-clock cap, so another run would burn a worker slot for the same outcome. Re-scope it, raise maxContinuationsPerIssue for it, or finish the remaining work by hand.",
4678
5012
  ].join("\n"),
4679
5013
  });
5014
+ await postExhaustionPostmortem(
5015
+ d,
5016
+ run,
5017
+ `Attempt ${run.attempt} hit the wall-clock cap with work to continue from, but ${continuation} continuation(s) are already charged against a cap of ${caps.maxContinuationsPerIssue}.`,
5018
+ );
4680
5019
  log(`#${run.issue} not continued from ${cls}: continuation budget exhausted`);
4681
5020
  }
4682
5021
  return;
@@ -4731,11 +5070,14 @@ async function recoverRun(
4731
5070
  // Same bound for provider-transient: an issue whose stream keeps stalling
4732
5071
  // mid-run is requeued free (no attempt, no continuation charged) — but a
4733
5072
  // provider that aborts three times for one issue is down, and a human has
4734
- // to check its status before hand-requeueing (#220).
5073
+ // to check its status before hand-requeueing (#220). The escalation names
5074
+ // every model the chain tried, so a merged branch built on a different
5075
+ // model is attributable (#286).
4735
5076
  if (
4736
5077
  cls === "provider-transient" &&
4737
5078
  store.classCountFor(project.name, run.issue, "provider-transient") >= PROVIDER_TRANSIENT_MAX_STRIKES
4738
5079
  ) {
5080
+ const tried = formatModelsTried(modelsTried(store.runsForIssue(project.name, run.issue)));
4739
5081
  await safeEscalate(d, {
4740
5082
  tier: 1,
4741
5083
  project: project.name,
@@ -4743,6 +5085,9 @@ async function recoverRun(
4743
5085
  summary: `[provider-transient] #${run.issue}: the provider keeps aborting mid-stream — ${evidence}`,
4744
5086
  detail: [
4745
5087
  `The provider aborted the stream for #${run.issue} ${PROVIDER_TRANSIENT_MAX_STRIKES} times without the run ever producing a verdict (0 tokens billed each time).`,
5088
+ ...(tried === ""
5089
+ ? []
5090
+ : [`Models tried: ${tried}.`]),
4746
5091
  "Check provider status before requeueing by hand.",
4747
5092
  ].join("\n"),
4748
5093
  });
@@ -4816,6 +5161,11 @@ async function recoverRun(
4816
5161
  "What you can do: raise maxContinuationsPerIssue for this issue, re-scope it, or continue from the preserved work (or start clean if none).",
4817
5162
  ].join("\n"),
4818
5163
  });
5164
+ await postExhaustionPostmortem(
5165
+ d,
5166
+ run,
5167
+ `Attempt ${run.attempt} is not requeued: ${continuations} continuation(s) charged against a cap of ${caps.maxContinuationsPerIssue}.`,
5168
+ );
4819
5169
  log(`#${run.issue} not requeued from orphan-clean: continuation budget exhausted`);
4820
5170
  return;
4821
5171
  }
@@ -5336,6 +5686,8 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
5336
5686
  integrity,
5337
5687
  stall: { paged: false },
5338
5688
  cleanup: { next: 0 },
5689
+ probeCriticalBase: (repo, markers, branch) =>
5690
+ probeCriticalBase(project, repo, branch, markers),
5339
5691
  ...(verbPeerReader === undefined ? {} : { verbPeerReader }),
5340
5692
  verbActions,
5341
5693
  };