omp-conductor 0.15.11 → 0.15.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +98 -56
- package/package.json +1 -1
- package/schema/config.schema.json +3 -0
- package/src/briefs/orchestrator.md +64 -11
- package/src/briefs/policy.md +19 -3
- package/src/briefs/worker.md +11 -8
- package/src/cli.ts +41 -21
- package/src/commands/context.ts +102 -1
- package/src/commands/doctor.ts +4 -2
- package/src/commands/intake.ts +26 -5
- package/src/commands/message.ts +80 -32
- package/src/commands/report.ts +38 -2
- package/src/commands/setup.ts +61 -11
- package/src/commands/upgrade-rollback.ts +9 -0
- package/src/config-schema.ts +9 -0
- package/src/config.ts +35 -1
- package/src/daemon.ts +359 -7
- package/src/dashboard/app.js +398 -59
- package/src/dashboard/index.html +27 -0
- package/src/dashboard/server.ts +219 -5
- package/src/dashboard/style.css +169 -1
- package/src/doctor.ts +179 -42
- package/src/failure-class.ts +37 -0
- package/src/gitops.ts +157 -0
- package/src/model-fallback.ts +177 -0
- package/src/omp.ts +91 -13
- package/src/orchestrator-tick.ts +101 -5
- package/src/orchestrator.ts +4 -4
- package/src/privileged.ts +10 -0
- package/src/release-policy.ts +212 -12
- package/src/session-host.ts +11 -5
- package/src/setup-host.ts +286 -60
- package/src/setup-install.ts +237 -28
- package/src/setup-wizard.ts +67 -19
- package/src/setup.ts +25 -0
- package/src/store.ts +13 -1
- package/src/tracker/github.ts +47 -0
- package/src/types.ts +71 -0
- package/src/upgrade.ts +90 -32
- package/src/worker.ts +3 -3
- package/systemd/omp-conductor.service.example +7 -3
package/src/config.ts
CHANGED
|
@@ -355,8 +355,11 @@ export function findProject(c: ConductorConfig, name?: string): ProjectConfig {
|
|
|
355
355
|
if (name === undefined) {
|
|
356
356
|
const only = c.projects[0];
|
|
357
357
|
if (c.projects.length !== 1 || only === undefined) {
|
|
358
|
+
// #514: every operator hitting this has to guess the spelling of the
|
|
359
|
+
// flag that names a project — say it outright. Doctor's project finding
|
|
360
|
+
// quotes this text, so keep the "name one explicitly" phrase.
|
|
358
361
|
throw new Error(
|
|
359
|
-
`Ambiguous project: config has ${c.projects.length} projects (${names.join(", ") || "none"}) — name one explicitly.`,
|
|
362
|
+
`Ambiguous project: config has ${c.projects.length} projects (${names.join(", ") || "none"}) — name one explicitly with --project NAME.`,
|
|
360
363
|
);
|
|
361
364
|
}
|
|
362
365
|
return only;
|
|
@@ -1080,6 +1083,34 @@ function finalizeProject(
|
|
|
1080
1083
|
const workerModel =
|
|
1081
1084
|
typeof rawWorkerModel === "string" && rawWorkerModel.trim() !== "" ? rawWorkerModel : undefined;
|
|
1082
1085
|
|
|
1086
|
+
// The failover chain is a hint like workerModel, not a ceiling: an unusable
|
|
1087
|
+
// entry or a malformed threshold is dropped rather than failing the load,
|
|
1088
|
+
// and absent/empty `modelFallbacks` keeps today's dispatch byte for byte.
|
|
1089
|
+
const rawFallbacks = p["modelFallbacks"];
|
|
1090
|
+
const modelFallbacks = Array.isArray(rawFallbacks)
|
|
1091
|
+
? rawFallbacks
|
|
1092
|
+
.filter((entry): entry is string => typeof entry === "string")
|
|
1093
|
+
.map((entry) => entry.trim())
|
|
1094
|
+
.filter((entry) => entry !== "")
|
|
1095
|
+
: [];
|
|
1096
|
+
const rawThreshold = p["modelFallbackThreshold"];
|
|
1097
|
+
const modelFallbackThreshold =
|
|
1098
|
+
typeof rawThreshold === "number" && Number.isInteger(rawThreshold) && rawThreshold >= 1
|
|
1099
|
+
? rawThreshold
|
|
1100
|
+
: undefined;
|
|
1101
|
+
|
|
1102
|
+
// Marked critical-base/safety markers (commit SHAs or refs). A continuation
|
|
1103
|
+
// branch must contain every one before the dispatcher may reattach it;
|
|
1104
|
+
// unusable entries are dropped like `modelFallbacks` and an absent or empty
|
|
1105
|
+
// list keeps today's behaviour byte for byte.
|
|
1106
|
+
const rawCriticalBase = p["criticalBase"];
|
|
1107
|
+
const criticalBase = Array.isArray(rawCriticalBase)
|
|
1108
|
+
? rawCriticalBase
|
|
1109
|
+
.filter((entry): entry is string => typeof entry === "string")
|
|
1110
|
+
.map((entry) => entry.trim())
|
|
1111
|
+
.filter((entry) => entry !== "")
|
|
1112
|
+
: [];
|
|
1113
|
+
|
|
1083
1114
|
if (problems.length > before) return undefined;
|
|
1084
1115
|
|
|
1085
1116
|
return {
|
|
@@ -1095,12 +1126,15 @@ function finalizeProject(
|
|
|
1095
1126
|
routing: { labelPrefix, repos },
|
|
1096
1127
|
caps,
|
|
1097
1128
|
...(workerModel === undefined ? {} : { workerModel }),
|
|
1129
|
+
...(modelFallbacks.length === 0 ? {} : { modelFallbacks }),
|
|
1130
|
+
...(modelFallbackThreshold === undefined ? {} : { modelFallbackThreshold }),
|
|
1098
1131
|
escalation,
|
|
1099
1132
|
authority,
|
|
1100
1133
|
releasePolicy,
|
|
1101
1134
|
policy,
|
|
1102
1135
|
...(recoveryMerges === undefined ? {} : { recoveryMerges }),
|
|
1103
1136
|
reporting,
|
|
1137
|
+
...(criticalBase.length === 0 ? {} : { criticalBase }),
|
|
1104
1138
|
workspaceRoot: expandHome(pickString(p["workspaceRoot"], defaultWorkspaceRoot())),
|
|
1105
1139
|
mirrorRoot: expandHome(pickString(p["mirrorRoot"], defaultMirrorRoot())),
|
|
1106
1140
|
};
|
package/src/daemon.ts
CHANGED
|
@@ -47,12 +47,21 @@ import {
|
|
|
47
47
|
} from "./reports.ts";
|
|
48
48
|
import {
|
|
49
49
|
recordReleaseBlock,
|
|
50
|
+
type GateShape,
|
|
50
51
|
type ReleaseBlockContext,
|
|
51
52
|
} from "./release-policy.ts";
|
|
52
53
|
import { branchName, effectiveLabels, route } from "./routing.ts";
|
|
53
54
|
import type { Routed, UnroutableReason } from "./routing.ts";
|
|
54
55
|
import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
|
|
55
56
|
import { classifyRun, providerCreditRefusal, providerTransientFault, type ClassifyFacts } from "./failure-class.ts";
|
|
57
|
+
import {
|
|
58
|
+
DEFAULT_MODEL_FALLBACK_THRESHOLD,
|
|
59
|
+
fallbackClause,
|
|
60
|
+
formatModelsTried,
|
|
61
|
+
modelsTried,
|
|
62
|
+
providerFailureFacts,
|
|
63
|
+
resolveDispatchModel,
|
|
64
|
+
} from "./model-fallback.ts";
|
|
56
65
|
import { projectLabels } from "./label-projection.ts";
|
|
57
66
|
import { dbPath, LIVE_STATES, openStore, utcDay } from "./store.ts";
|
|
58
67
|
import { makeTracker, type RateLimitStatus } from "./tracker/github.ts";
|
|
@@ -64,6 +73,7 @@ import type {
|
|
|
64
73
|
DispatchSummary,
|
|
65
74
|
DigestBacklog,
|
|
66
75
|
Escalation,
|
|
76
|
+
IssueComment,
|
|
67
77
|
IssueSnapshot,
|
|
68
78
|
MergedPrInfo,
|
|
69
79
|
OpenCloser,
|
|
@@ -122,7 +132,14 @@ import {
|
|
|
122
132
|
} from "./verbs/socket.ts";
|
|
123
133
|
import { homedir } from "node:os";
|
|
124
134
|
|
|
125
|
-
import {
|
|
135
|
+
import {
|
|
136
|
+
probeCriticalBase,
|
|
137
|
+
pushRunBranch,
|
|
138
|
+
readBaseChain,
|
|
139
|
+
type CriticalBaseProbe,
|
|
140
|
+
type CriticalBaseVerdict,
|
|
141
|
+
type RunRepoRef,
|
|
142
|
+
} from "./gitops.ts";
|
|
126
143
|
import {
|
|
127
144
|
planUsageLine,
|
|
128
145
|
readPlanUsage,
|
|
@@ -233,6 +250,14 @@ interface Deps {
|
|
|
233
250
|
* without touching the host's packages, journal or outbox.
|
|
234
251
|
*/
|
|
235
252
|
upgradeVerifier?: (d: Deps) => Promise<void>;
|
|
253
|
+
/**
|
|
254
|
+
* Answers whether a preserved continuation branch contains every configured
|
|
255
|
+
* critical-base marker, for the stale-base admission hold (#428). Wired by
|
|
256
|
+
* `runDaemon` to the mirror-backed {@link probeCriticalBase}; a test injects
|
|
257
|
+
* a fake. Absent, the stale-base gate fails closed whenever a project names
|
|
258
|
+
* a marker (a safety interlock must not silently weaken).
|
|
259
|
+
*/
|
|
260
|
+
probeCriticalBase?: CriticalBaseProbe;
|
|
236
261
|
}
|
|
237
262
|
|
|
238
263
|
/**
|
|
@@ -682,6 +707,64 @@ function acceptanceCriteria(issue: ReadyIssue): string {
|
|
|
682
707
|
: "_The issue body is empty. Read the issue and its comments, and escalate if it is genuinely underspecified._";
|
|
683
708
|
}
|
|
684
709
|
|
|
710
|
+
/**
|
|
711
|
+
* How much of a comment thread one brief may hold. Grooming notes are short,
|
|
712
|
+
* and a thread that outgrows this is truncated with an explicit marker rather
|
|
713
|
+
* than a silently dropped tail — the disappearance this fix exists to prevent.
|
|
714
|
+
*/
|
|
715
|
+
const DISCUSSION_CHARS_BUDGET = 8_000;
|
|
716
|
+
|
|
717
|
+
/**
|
|
718
|
+
* The issue's comments as a brief section: attributed, numbered, oldest
|
|
719
|
+
* first, so a later correction visibly supersedes an earlier note. Rendered
|
|
720
|
+
* beside the body at dispatch, because the orchestrator's grooming is posted
|
|
721
|
+
* as comments and a worker must never depend on a runtime read to see it.
|
|
722
|
+
*
|
|
723
|
+
* `"unread"` is not "empty": when the tracker refused at dispatch the section
|
|
724
|
+
* says so, instead of silently reading as "no discussion" — which is exactly
|
|
725
|
+
* the #517 failure. An empty comment list renders nothing, so a commentless
|
|
726
|
+
* issue's brief stays byte-identical to what this package has always shipped.
|
|
727
|
+
*/
|
|
728
|
+
function renderDiscussion(comments: IssueComment[] | "unread"): string {
|
|
729
|
+
if (comments === "unread") {
|
|
730
|
+
return [
|
|
731
|
+
"## Discussion",
|
|
732
|
+
"",
|
|
733
|
+
"_The issue's comments could not be read at dispatch time. The live read below is_",
|
|
734
|
+
"_the only path to them; if it prints nothing, that is a failed read, not an_",
|
|
735
|
+
"_absence of discussion._",
|
|
736
|
+
"",
|
|
737
|
+
].join("\n");
|
|
738
|
+
}
|
|
739
|
+
if (comments.length === 0) return "";
|
|
740
|
+
const total = comments.length;
|
|
741
|
+
const lines = [
|
|
742
|
+
"## Discussion",
|
|
743
|
+
"",
|
|
744
|
+
`${total} comment${total === 1 ? "" : "s"} on the issue at dispatch, oldest first — a later`,
|
|
745
|
+
"comment supersedes an earlier one.",
|
|
746
|
+
"",
|
|
747
|
+
];
|
|
748
|
+
let chars = lines.join("\n").length;
|
|
749
|
+
let shown = 0;
|
|
750
|
+
for (const comment of comments) {
|
|
751
|
+
const block = `**@${comment.author} — comment ${shown + 1}:**\n\n${comment.body}\n\n`;
|
|
752
|
+
if (shown > 0 && chars + block.length > DISCUSSION_CHARS_BUDGET) break;
|
|
753
|
+
lines.push(`**@${comment.author} — comment ${shown + 1}:**`, "", comment.body, "");
|
|
754
|
+
chars += block.length;
|
|
755
|
+
shown += 1;
|
|
756
|
+
}
|
|
757
|
+
if (shown < total) {
|
|
758
|
+
const omitted = total - shown;
|
|
759
|
+
lines.push(
|
|
760
|
+
`… ${omitted} comment${omitted === 1 ? "" : "s"} omitted — the discussion exceeded ${DISCUSSION_CHARS_BUDGET} characters.`,
|
|
761
|
+
"Read the issue for the tail.",
|
|
762
|
+
"",
|
|
763
|
+
);
|
|
764
|
+
}
|
|
765
|
+
return lines.join("\n");
|
|
766
|
+
}
|
|
767
|
+
|
|
685
768
|
/**
|
|
686
769
|
* Record a state-label swap for projection (#201). The add enqueues before the
|
|
687
770
|
* remove — the reverse order would leave a window where the issue carries no
|
|
@@ -934,7 +1017,15 @@ export async function buildBrief(
|
|
|
934
1017
|
r: Routed,
|
|
935
1018
|
branch: string,
|
|
936
1019
|
worktree: string,
|
|
937
|
-
opts: {
|
|
1020
|
+
opts: {
|
|
1021
|
+
continuation?: boolean;
|
|
1022
|
+
defaultBranch?: string;
|
|
1023
|
+
salvagedSha?: string;
|
|
1024
|
+
/** The issue's comments, rendered into the Discussion section. `"unread"`
|
|
1025
|
+
* when the tracker refused at dispatch — rare, but it must never read as
|
|
1026
|
+
* "no comments" (the #517 failure mode). Absent means an empty list. */
|
|
1027
|
+
comments?: IssueComment[] | "unread";
|
|
1028
|
+
} = {},
|
|
938
1029
|
): Promise<string> {
|
|
939
1030
|
// Read per dispatch rather than caching: editing the brief then takes effect
|
|
940
1031
|
// on the next issue instead of needing a daemon restart.
|
|
@@ -979,6 +1070,7 @@ export async function buildBrief(
|
|
|
979
1070
|
BRANCH: branch,
|
|
980
1071
|
WORKTREE: worktree,
|
|
981
1072
|
ACCEPTANCE_CRITERIA: acceptanceCriteria(r.issue),
|
|
1073
|
+
ISSUE_COMMENTS: renderDiscussion(opts.comments ?? []),
|
|
982
1074
|
GATES: gatesBlock(r.repo),
|
|
983
1075
|
// Empty for a repo with no `graphProject`, and empty means *nothing*: the
|
|
984
1076
|
// placeholder sits flush against the next list item in the template, so an
|
|
@@ -1045,6 +1137,135 @@ function newestContinuableRun(runs: readonly RunRecord[]): RunRecord | undefined
|
|
|
1045
1137
|
);
|
|
1046
1138
|
}
|
|
1047
1139
|
|
|
1140
|
+
/** The fenced-block info string that marks an exhaustion postmortem comment, so
|
|
1141
|
+
* a grooming scout re-slicing the issue can find and parse the whole block by
|
|
1142
|
+
* grepping for it. */
|
|
1143
|
+
export const POSTMORTEM_MARKER = "conductor-postmortem";
|
|
1144
|
+
|
|
1145
|
+
/** How one issue's attempt chain failed, as "3× ci-deterministic, 1× …" — the
|
|
1146
|
+
* digest shape for naming what the exhaustion was. Groups every row by its
|
|
1147
|
+
* failure class, whether or not it charged the continuation budget, because
|
|
1148
|
+
* the postmortem tells the whole story and not just the budget half (#290). */
|
|
1149
|
+
export function attemptClassBreakdown(runs: readonly RunRecord[]): string {
|
|
1150
|
+
const perClass = new Map<string, number>();
|
|
1151
|
+
for (const r of runs) {
|
|
1152
|
+
const cls = r.failureClass ?? "unclassified";
|
|
1153
|
+
perClass.set(cls, (perClass.get(cls) ?? 0) + 1);
|
|
1154
|
+
}
|
|
1155
|
+
if (perClass.size === 0) return "unclassified";
|
|
1156
|
+
return Array.from(perClass, ([cls, n]) => `${n}× ${cls}`).join(", ");
|
|
1157
|
+
}
|
|
1158
|
+
|
|
1159
|
+
/** Wall-clock duration of one run as a compact human string ("45m", "1h30m"). */
|
|
1160
|
+
export function humanDuration(ms: number): string {
|
|
1161
|
+
const seconds = Math.max(0, Math.round(ms / 1_000));
|
|
1162
|
+
if (seconds < 60) return `${seconds}s`;
|
|
1163
|
+
const minutes = Math.round(seconds / 60);
|
|
1164
|
+
if (minutes < 60) return `${minutes}m`;
|
|
1165
|
+
const hours = Math.floor(minutes / 60);
|
|
1166
|
+
const rest = minutes % 60;
|
|
1167
|
+
return rest === 0 ? `${hours}h` : `${hours}h${rest}m`;
|
|
1168
|
+
}
|
|
1169
|
+
|
|
1170
|
+
/** Flatten and bound a run's last error to one greppable table line. */
|
|
1171
|
+
export function oneLineBrief(text: string | undefined): string | undefined {
|
|
1172
|
+
if (text === undefined || text.trim() === "") return undefined;
|
|
1173
|
+
const flat = text.replace(/\s+/g, " ").trim();
|
|
1174
|
+
return flat.length > 90 ? `${flat.slice(0, 89)}…` : flat;
|
|
1175
|
+
}
|
|
1176
|
+
|
|
1177
|
+
/**
|
|
1178
|
+
* The exhaustion postmortem block: one greppable fenced block covering every
|
|
1179
|
+
* attempt in the chain — continuation rows included — with per-attempt turns,
|
|
1180
|
+
* wall clock, failure class and a one-line last error, the explicit salvage
|
|
1181
|
+
* state, the spend total and the transcript paths for local inspection.
|
|
1182
|
+
*
|
|
1183
|
+
* Pure so the tests hold the shape, not the transport: the writer below owns
|
|
1184
|
+
* the once-only guarantee, this owns what "once" looks like.
|
|
1185
|
+
*/
|
|
1186
|
+
export function formatExhaustionPostmortem(args: {
|
|
1187
|
+
issue: number;
|
|
1188
|
+
runs: readonly RunRecord[];
|
|
1189
|
+
reason: string;
|
|
1190
|
+
}): string {
|
|
1191
|
+
const { issue, runs, reason } = args;
|
|
1192
|
+
const artifact = newestContinuableRun(runs);
|
|
1193
|
+
const totalSpend = runs.reduce((sum, r) => sum + r.spendUsd, 0);
|
|
1194
|
+
const spend = `$${totalSpend.toFixed(2)}`;
|
|
1195
|
+
const attemptLines = runs.map((r) => {
|
|
1196
|
+
const wall = r.endedAt === undefined ? "—" : humanDuration(r.endedAt - r.startedAt);
|
|
1197
|
+
const cls = r.failureClass ?? "unclassified";
|
|
1198
|
+
const error = oneLineBrief(r.lastError) ?? "—";
|
|
1199
|
+
return (
|
|
1200
|
+
` attempt ${r.attempt} ${r.state.padEnd(12)} turns ${r.turns}/${r.maxTurns} ` +
|
|
1201
|
+
`${cls.padEnd(24)} ${wall.padStart(4)} last error: ${error}`
|
|
1202
|
+
);
|
|
1203
|
+
});
|
|
1204
|
+
const salvage =
|
|
1205
|
+
artifact === undefined
|
|
1206
|
+
? "Salvaged WIP: absent — no attempt preserved a branch, head SHA or pull request."
|
|
1207
|
+
: `Salvaged WIP: present — branch ${artifact.branch} at ${artifact.headSha ?? artifact.salvageSha}` +
|
|
1208
|
+
`${artifact.prUrl === undefined ? "" : ` (PR ${artifact.prUrl})`}.`;
|
|
1209
|
+
return [
|
|
1210
|
+
`\`\`\`${POSTMORTEM_MARKER}`,
|
|
1211
|
+
`#${issue} exhausted: ${attemptClassBreakdown(runs)}`,
|
|
1212
|
+
reason,
|
|
1213
|
+
"",
|
|
1214
|
+
`Attempts (${runs.length} total, ${spend} spend):`,
|
|
1215
|
+
...attemptLines,
|
|
1216
|
+
salvage,
|
|
1217
|
+
`Spend total: ${spend} across ${runs.length} attempts.`,
|
|
1218
|
+
"Transcripts:",
|
|
1219
|
+
...runs.map((r) => (r.sessionFile === undefined ? " (none)" : ` ${r.sessionFile}`)),
|
|
1220
|
+
"```",
|
|
1221
|
+
].join("\n");
|
|
1222
|
+
}
|
|
1223
|
+
|
|
1224
|
+
/** Dedupe key prefix for the exhaustion postmortem comment, per issue. */
|
|
1225
|
+
function postmortemDedupeKey(project: string, issue: number): string {
|
|
1226
|
+
return `${project}:postmortem:${issue}`;
|
|
1227
|
+
}
|
|
1228
|
+
|
|
1229
|
+
/**
|
|
1230
|
+
* The exhaustion postmortem: written exactly once per issue, at the point the
|
|
1231
|
+
* continuation budget is spent and the issue is settled toward a human.
|
|
1232
|
+
*
|
|
1233
|
+
* The comment and the material event are both gated by the store's notification
|
|
1234
|
+
* ledger — the same idempotence guard the escalator uses — so re-settling an
|
|
1235
|
+
* already-postmortemed issue posts nothing and records nothing. A body-string
|
|
1236
|
+
* match on the issue would be the wrong guard: an issue re-scoped and re-run
|
|
1237
|
+
* would still carry the old block, and the guarantee asked of this is "decided
|
|
1238
|
+
* once", not "deduped against what is already written".
|
|
1239
|
+
*
|
|
1240
|
+
* The digest must name the exhaustion even if the comment write fails, so the
|
|
1241
|
+
* material event is recorded before the write and unconditionally (the ledger
|
|
1242
|
+
* is append-only, and the gate above already ran once). The comment failure is
|
|
1243
|
+
* logged rather than taking the sweep down with it.
|
|
1244
|
+
*/
|
|
1245
|
+
async function postExhaustionPostmortem(d: Deps, run: RunRecord, reason: string): Promise<void> {
|
|
1246
|
+
const { project, tracker, store } = d;
|
|
1247
|
+
const key = postmortemDedupeKey(project.name, run.issue);
|
|
1248
|
+
if (store.wasNotified(key)) return;
|
|
1249
|
+
const runs = store.runsForIssue(project.name, run.issue);
|
|
1250
|
+
const body = formatExhaustionPostmortem({ issue: run.issue, runs, reason });
|
|
1251
|
+
const occurredAt = Date.now();
|
|
1252
|
+
store.recordMaterialEvent({
|
|
1253
|
+
project: project.name,
|
|
1254
|
+
category: "exhaustion",
|
|
1255
|
+
summary: `#${run.issue} exhausted: ${attemptClassBreakdown(runs)}`,
|
|
1256
|
+
evidence: body,
|
|
1257
|
+
occurredAt,
|
|
1258
|
+
recordedAt: occurredAt,
|
|
1259
|
+
});
|
|
1260
|
+
try {
|
|
1261
|
+
await tracker.comment(run.issue, body);
|
|
1262
|
+
store.markNotified(key);
|
|
1263
|
+
log(`#${run.issue} posted exhaustion postmortem (${runs.length} attempts)`);
|
|
1264
|
+
} catch (err) {
|
|
1265
|
+
log(`#${run.issue} postmortem comment could not be posted (${errText(err)})`);
|
|
1266
|
+
}
|
|
1267
|
+
}
|
|
1268
|
+
|
|
1048
1269
|
export type ExtendTurnLimitResult =
|
|
1049
1270
|
| { kind: "extended"; runId: string; maxTurns: number }
|
|
1050
1271
|
| { kind: "not-increase"; runId: string; maxTurns: number }
|
|
@@ -1329,7 +1550,7 @@ export function workerReleaseBlockRecorder(
|
|
|
1329
1550
|
issue: number,
|
|
1330
1551
|
runId: string,
|
|
1331
1552
|
root = stateDir(),
|
|
1332
|
-
): (shape:
|
|
1553
|
+
): (shape: GateShape, context: ReleaseBlockContext) => void {
|
|
1333
1554
|
return (shape, context) =>
|
|
1334
1555
|
recordReleaseBlock(
|
|
1335
1556
|
project,
|
|
@@ -1544,6 +1765,11 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
1544
1765
|
// attempt whose work this one inherits.
|
|
1545
1766
|
const prior = store.latestRun(project.name, issue);
|
|
1546
1767
|
const priorSalvage = prior?.salvageSha;
|
|
1768
|
+
// The failover decision is made from the store the same way admission is:
|
|
1769
|
+
// counts of this issue's own rows, so the switch is sticky per chain and a
|
|
1770
|
+
// fresh issue always starts on the primary (#286). Read before the claim
|
|
1771
|
+
// writes this attempt's row, which would otherwise break the streak.
|
|
1772
|
+
const chainFacts = providerFailureFacts(store.runsForIssue(project.name, issue));
|
|
1547
1773
|
run = store.createRun({
|
|
1548
1774
|
project: project.name,
|
|
1549
1775
|
issue,
|
|
@@ -1657,17 +1883,50 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
1657
1883
|
|
|
1658
1884
|
store.updateRun(runId, { worktree: worktreePath, state: "running" });
|
|
1659
1885
|
|
|
1886
|
+
// Where this attempt goes, and (when the failover fired) the clause that
|
|
1887
|
+
// makes it attributable: with no `modelFallbacks` configured this is the
|
|
1888
|
+
// primary model — or none, for an unconfigured project — and today's
|
|
1889
|
+
// dispatch is byte for byte what it has always been.
|
|
1890
|
+
const chainConfigured = (project.modelFallbacks?.length ?? 0) > 0;
|
|
1891
|
+
const choice = resolveDispatchModel({
|
|
1892
|
+
workerModel: project.workerModel,
|
|
1893
|
+
modelFallbacks: project.modelFallbacks,
|
|
1894
|
+
threshold: project.modelFallbackThreshold ?? DEFAULT_MODEL_FALLBACK_THRESHOLD,
|
|
1895
|
+
streak: chainFacts.streak,
|
|
1896
|
+
});
|
|
1897
|
+
// Recorded before the launch, so even a run killed mid-flight leaves the
|
|
1898
|
+
// model it chose on its row. Only a chain-configured project writes the
|
|
1899
|
+
// column: absent `modelFallbacks` must preserve today's rows byte for byte.
|
|
1900
|
+
if (chainConfigured && choice.model !== undefined) {
|
|
1901
|
+
store.updateRun(runId, { model: choice.model });
|
|
1902
|
+
}
|
|
1903
|
+
const clause = fallbackClause(choice, chainFacts, project.workerModel);
|
|
1904
|
+
|
|
1660
1905
|
log(
|
|
1661
|
-
`#${issue} attempt ${attempt} → ${r.repo.name} ${branch}` +
|
|
1906
|
+
`#${issue} attempt ${attempt}${clause === undefined ? "" : ` ${clause}`} → ${r.repo.name} ${branch}` +
|
|
1662
1907
|
(provisioned.reattached ? " (continuation: reattached existing branch)" : ""),
|
|
1663
1908
|
);
|
|
1664
1909
|
|
|
1910
|
+
// The discussion is rendered at dispatch so a worker never depends on a
|
|
1911
|
+
// runtime `gh` read to see the orchestrator's grooming (#517). The read is
|
|
1912
|
+
// best-effort, but its failure is not silent: an unreadable tracker names
|
|
1913
|
+
// itself in the brief's Discussion section instead of reading as "no
|
|
1914
|
+
// comments" — the exact confusion this fix removes.
|
|
1915
|
+
let comments: IssueComment[] | "unread";
|
|
1916
|
+
try {
|
|
1917
|
+
comments = await tracker.listComments(issue);
|
|
1918
|
+
} catch (err) {
|
|
1919
|
+
log(`#${issue} issue comments unreadable at dispatch; the brief will say so: ${errText(err)}`);
|
|
1920
|
+
comments = "unread";
|
|
1921
|
+
}
|
|
1922
|
+
|
|
1665
1923
|
const brief = await buildBrief(project, r, branch, worktreePath, {
|
|
1666
1924
|
continuation: provisioned.reattached,
|
|
1667
1925
|
defaultBranch: r.repo.defaultBranch,
|
|
1668
1926
|
...(provisioned.reattached && priorSalvage !== undefined
|
|
1669
1927
|
? { salvagedSha: priorSalvage }
|
|
1670
1928
|
: {}),
|
|
1929
|
+
comments,
|
|
1671
1930
|
});
|
|
1672
1931
|
if (await settleStopBeforeSession()) return;
|
|
1673
1932
|
if (await settleDrainBeforeSession()) return;
|
|
@@ -1702,7 +1961,7 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
1702
1961
|
onChildLog: (line) => {
|
|
1703
1962
|
log(`#${issue} ${line}`);
|
|
1704
1963
|
},
|
|
1705
|
-
...(
|
|
1964
|
+
...(choice.model === undefined ? {} : { model: choice.model }),
|
|
1706
1965
|
releaseGrants: resolveReleaseGrants(project),
|
|
1707
1966
|
onReleaseBlocked: workerReleaseBlockRecorder(project.name, issue, runId),
|
|
1708
1967
|
onTurn: (n) => store.updateRun(runId, { turns: n }),
|
|
@@ -2886,7 +3145,7 @@ function planUsageEscalation(project: string, plan: PlanUsageStatus): Escalation
|
|
|
2886
3145
|
* that grows a field has no business breaking these tests.
|
|
2887
3146
|
*/
|
|
2888
3147
|
export async function admitCandidates(
|
|
2889
|
-
d: Pick<Deps, "project" | "caps" | "tracker" | "store" | "escalate" | "usage">,
|
|
3148
|
+
d: Pick<Deps, "project" | "caps" | "tracker" | "store" | "escalate" | "usage" | "probeCriticalBase">,
|
|
2890
3149
|
routed: Routed[],
|
|
2891
3150
|
slots: number,
|
|
2892
3151
|
): Promise<AdmissionPass> {
|
|
@@ -3066,6 +3325,81 @@ export async function admitCandidates(
|
|
|
3066
3325
|
continue;
|
|
3067
3326
|
}
|
|
3068
3327
|
|
|
3328
|
+
// #428 half (a): a preserved continuation that predates a configured
|
|
3329
|
+
// critical-base/safety marker must not be reattached. A base safety fix
|
|
3330
|
+
// protects only branches forked after it landed — a continuation forked
|
|
3331
|
+
// before it still carries the dangerous test/runtime code, and re-running
|
|
3332
|
+
// it on the shared host is what SIGTERMed the production daemon. Fail
|
|
3333
|
+
// closed: only a probe that proves every marker is in the reattach
|
|
3334
|
+
// source's ancestry admits, and a project that names a marker but has no
|
|
3335
|
+
// probe wired (never happens outside tests) holds. Both the hold and the
|
|
3336
|
+
// escalation are durable across restart and orphan recovery because this
|
|
3337
|
+
// gate runs every admission pass; the branch is re-admitted automatically
|
|
3338
|
+
// once the operator updates it to contain the marker, without losing work.
|
|
3339
|
+
const markers = project.criticalBase ?? [];
|
|
3340
|
+
if (markers.length > 0) {
|
|
3341
|
+
const branch = branchName(r.issue);
|
|
3342
|
+
let verdict: CriticalBaseVerdict;
|
|
3343
|
+
if (d.probeCriticalBase === undefined) {
|
|
3344
|
+
verdict = { state: "unknown", error: "no critical-base probe is wired in this deployment" };
|
|
3345
|
+
} else {
|
|
3346
|
+
try {
|
|
3347
|
+
verdict = await d.probeCriticalBase(r.repo, markers, branch);
|
|
3348
|
+
} catch (err) {
|
|
3349
|
+
verdict = { state: "unknown", error: errText(err) };
|
|
3350
|
+
}
|
|
3351
|
+
}
|
|
3352
|
+
if (verdict.state === "stale") {
|
|
3353
|
+
hold(issue, "stale-base");
|
|
3354
|
+
log(
|
|
3355
|
+
`#${issue} held (stale-base): continuation branch ${branch} predates critical-base marker ${verdict.marker}`,
|
|
3356
|
+
);
|
|
3357
|
+
await safeEscalate(d, {
|
|
3358
|
+
tier: 1,
|
|
3359
|
+
project: project.name,
|
|
3360
|
+
issue,
|
|
3361
|
+
summary: `#${issue} continuation branch predates a critical base safety commit and is held (stale-base)`,
|
|
3362
|
+
detail: [
|
|
3363
|
+
r.issue.title,
|
|
3364
|
+
r.issue.url,
|
|
3365
|
+
`The retained branch ${branch} does not contain critical-base marker ${verdict.marker}.`,
|
|
3366
|
+
...(verdict.range.length > 0
|
|
3367
|
+
? [`Base commits the branch is missing: ${verdict.range.join(", ")}`]
|
|
3368
|
+
: []),
|
|
3369
|
+
"Recovery: merge current base into the branch so it contains the marker, and the next",
|
|
3370
|
+
"admission pass re-admits it automatically without losing the branch's work; or review",
|
|
3371
|
+
"the branch by hand and clear the hold once the fix is present.",
|
|
3372
|
+
].join("\n"),
|
|
3373
|
+
});
|
|
3374
|
+
continue;
|
|
3375
|
+
}
|
|
3376
|
+
if (verdict.state === "unknown") {
|
|
3377
|
+
// Fail closed: a branch that cannot be *proven* to contain the marker
|
|
3378
|
+
// is refused, and the reason names the unverifiable marker so the
|
|
3379
|
+
// operator can fix the fetch or the marker rather than guess.
|
|
3380
|
+
hold(issue, "stale-base");
|
|
3381
|
+
log(
|
|
3382
|
+
`#${issue} held (stale-base): continuation branch ${branch} could not be verified ` +
|
|
3383
|
+
`against critical-base marker(s) ${markers.join(", ")} (${verdict.error})`,
|
|
3384
|
+
);
|
|
3385
|
+
await safeEscalate(d, {
|
|
3386
|
+
tier: 1,
|
|
3387
|
+
project: project.name,
|
|
3388
|
+
issue,
|
|
3389
|
+
summary: `#${issue} continuation branch could not be verified against a critical base safety commit and is held (stale-base)`,
|
|
3390
|
+
detail: [
|
|
3391
|
+
r.issue.title,
|
|
3392
|
+
r.issue.url,
|
|
3393
|
+
`The retained branch ${branch} could not be verified against critical-base marker(s) ${markers.join(", ")}: ${verdict.error}`,
|
|
3394
|
+
"Recovery: merge current base into the branch so it contains the marker, and the next",
|
|
3395
|
+
"admission pass re-admits it automatically without losing the branch's work; or review",
|
|
3396
|
+
"the branch by hand and clear the hold once the fix is present.",
|
|
3397
|
+
].join("\n"),
|
|
3398
|
+
});
|
|
3399
|
+
continue;
|
|
3400
|
+
}
|
|
3401
|
+
}
|
|
3402
|
+
|
|
3069
3403
|
// Soft concurrency per epic, per repository: at most one in-flight child of
|
|
3070
3404
|
// a given parent in each repo. Children of one epic in *different* repos
|
|
3071
3405
|
// parallelise freely — `repo-active` / `maxConcurrentWorkersPerRepo` owns
|
|
@@ -4677,6 +5011,11 @@ async function recoverRun(
|
|
|
4677
5011
|
"The issue cannot finish inside the wall-clock cap, so another run would burn a worker slot for the same outcome. Re-scope it, raise maxContinuationsPerIssue for it, or finish the remaining work by hand.",
|
|
4678
5012
|
].join("\n"),
|
|
4679
5013
|
});
|
|
5014
|
+
await postExhaustionPostmortem(
|
|
5015
|
+
d,
|
|
5016
|
+
run,
|
|
5017
|
+
`Attempt ${run.attempt} hit the wall-clock cap with work to continue from, but ${continuation} continuation(s) are already charged against a cap of ${caps.maxContinuationsPerIssue}.`,
|
|
5018
|
+
);
|
|
4680
5019
|
log(`#${run.issue} not continued from ${cls}: continuation budget exhausted`);
|
|
4681
5020
|
}
|
|
4682
5021
|
return;
|
|
@@ -4731,11 +5070,14 @@ async function recoverRun(
|
|
|
4731
5070
|
// Same bound for provider-transient: an issue whose stream keeps stalling
|
|
4732
5071
|
// mid-run is requeued free (no attempt, no continuation charged) — but a
|
|
4733
5072
|
// provider that aborts three times for one issue is down, and a human has
|
|
4734
|
-
// to check its status before hand-requeueing (#220).
|
|
5073
|
+
// to check its status before hand-requeueing (#220). The escalation names
|
|
5074
|
+
// every model the chain tried, so a merged branch built on a different
|
|
5075
|
+
// model is attributable (#286).
|
|
4735
5076
|
if (
|
|
4736
5077
|
cls === "provider-transient" &&
|
|
4737
5078
|
store.classCountFor(project.name, run.issue, "provider-transient") >= PROVIDER_TRANSIENT_MAX_STRIKES
|
|
4738
5079
|
) {
|
|
5080
|
+
const tried = formatModelsTried(modelsTried(store.runsForIssue(project.name, run.issue)));
|
|
4739
5081
|
await safeEscalate(d, {
|
|
4740
5082
|
tier: 1,
|
|
4741
5083
|
project: project.name,
|
|
@@ -4743,6 +5085,9 @@ async function recoverRun(
|
|
|
4743
5085
|
summary: `[provider-transient] #${run.issue}: the provider keeps aborting mid-stream — ${evidence}`,
|
|
4744
5086
|
detail: [
|
|
4745
5087
|
`The provider aborted the stream for #${run.issue} ${PROVIDER_TRANSIENT_MAX_STRIKES} times without the run ever producing a verdict (0 tokens billed each time).`,
|
|
5088
|
+
...(tried === ""
|
|
5089
|
+
? []
|
|
5090
|
+
: [`Models tried: ${tried}.`]),
|
|
4746
5091
|
"Check provider status before requeueing by hand.",
|
|
4747
5092
|
].join("\n"),
|
|
4748
5093
|
});
|
|
@@ -4816,6 +5161,11 @@ async function recoverRun(
|
|
|
4816
5161
|
"What you can do: raise maxContinuationsPerIssue for this issue, re-scope it, or continue from the preserved work (or start clean if none).",
|
|
4817
5162
|
].join("\n"),
|
|
4818
5163
|
});
|
|
5164
|
+
await postExhaustionPostmortem(
|
|
5165
|
+
d,
|
|
5166
|
+
run,
|
|
5167
|
+
`Attempt ${run.attempt} is not requeued: ${continuations} continuation(s) charged against a cap of ${caps.maxContinuationsPerIssue}.`,
|
|
5168
|
+
);
|
|
4819
5169
|
log(`#${run.issue} not requeued from orphan-clean: continuation budget exhausted`);
|
|
4820
5170
|
return;
|
|
4821
5171
|
}
|
|
@@ -5336,6 +5686,8 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
5336
5686
|
integrity,
|
|
5337
5687
|
stall: { paged: false },
|
|
5338
5688
|
cleanup: { next: 0 },
|
|
5689
|
+
probeCriticalBase: (repo, markers, branch) =>
|
|
5690
|
+
probeCriticalBase(project, repo, branch, markers),
|
|
5339
5691
|
...(verbPeerReader === undefined ? {} : { verbPeerReader }),
|
|
5340
5692
|
verbActions,
|
|
5341
5693
|
};
|