omp-conductor 0.15.10 → 0.15.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +273 -2544
- package/REFERENCE.md +2680 -0
- package/package.json +3 -2
- package/schema/config.schema.json +11 -23
- package/src/arm-challenge.ts +112 -0
- package/src/ask.ts +434 -0
- package/src/board.ts +81 -15
- package/src/brief-upgrade.ts +114 -8
- package/src/briefs/orchestrator.md +113 -34
- package/src/briefs/policy.md +33 -8
- package/src/briefs/worker.md +18 -9
- package/src/chain-check.ts +1 -1
- package/src/check-trailing-newlines.ts +82 -0
- package/src/cli.ts +225 -1406
- package/src/commands/arm.ts +21 -0
- package/src/commands/board.ts +23 -0
- package/src/commands/brief-upgrade.ts +186 -0
- package/src/commands/context.ts +150 -0
- package/src/commands/daemon.ts +71 -0
- package/src/commands/dashboard.ts +74 -0
- package/src/commands/decision.ts +103 -0
- package/src/commands/disarm.ts +21 -0
- package/src/commands/doctor.ts +100 -0
- package/src/commands/event.ts +62 -0
- package/src/commands/extend.ts +64 -0
- package/src/commands/friction.ts +56 -0
- package/src/commands/help.ts +9 -0
- package/src/commands/hold.ts +26 -0
- package/src/commands/intake.ts +155 -0
- package/src/commands/ledger.ts +69 -0
- package/src/commands/message.ts +96 -0
- package/src/commands/report.ts +206 -0
- package/src/commands/restart.ts +76 -0
- package/src/commands/resume.ts +58 -0
- package/src/commands/setup.ts +143 -0
- package/src/commands/start.ts +23 -0
- package/src/commands/stats.ts +131 -0
- package/src/commands/status.ts +48 -0
- package/src/commands/stop.ts +51 -0
- package/src/commands/tail.ts +109 -0
- package/src/commands/unblock.ts +39 -0
- package/src/commands/upgrade-install.ts +31 -0
- package/src/commands/upgrade-rollback.ts +32 -0
- package/src/commands/upgrade.ts +25 -0
- package/src/commands/verb.ts +83 -0
- package/src/commands/version.ts +30 -0
- package/src/commands/worker.ts +100 -0
- package/src/config-schema.ts +47 -1
- package/src/config.ts +45 -4
- package/src/daemon.ts +972 -101
- package/src/dashboard/app.js +459 -0
- package/src/dashboard/index.html +61 -0
- package/src/dashboard/server.ts +481 -0
- package/src/dashboard/style.css +348 -0
- package/src/decisions.ts +39 -14
- package/src/diff-flags.ts +131 -241
- package/src/doctor.ts +932 -0
- package/src/escalate.ts +2 -2
- package/src/failure-class.ts +66 -3
- package/src/fleet.ts +58 -1
- package/src/gitops.ts +157 -0
- package/src/graph-health.ts +1 -1
- package/src/label-projection.ts +1 -1
- package/src/lifecycle.ts +198 -2
- package/src/model-fallback.ts +177 -0
- package/src/notices.ts +9 -0
- package/src/omp.ts +93 -13
- package/src/orchestrator-tick.ts +414 -20
- package/src/orchestrator.ts +4 -4
- package/src/privileged.ts +10 -0
- package/src/release-policy.ts +342 -30
- package/src/reports.ts +19 -5
- package/src/session-host.ts +11 -5
- package/src/setup-host.ts +663 -25
- package/src/setup-install.ts +292 -28
- package/src/setup-wizard.ts +255 -74
- package/src/setup.ts +156 -35
- package/src/stats.ts +331 -0
- package/src/store.ts +219 -22
- package/src/tracker/github.ts +74 -4
- package/src/types.ts +215 -31
- package/src/unblock.ts +55 -11
- package/src/upgrade-journal.ts +220 -0
- package/src/upgrade-verify.ts +506 -0
- package/src/upgrade.ts +385 -58
- package/src/verbs/actions.ts +73 -1
- package/src/verbs/protocol.ts +29 -4
- package/src/verbs/server.ts +183 -20
- package/src/worker.ts +3 -3
- package/systemd/omp-conductor-recover.sh +433 -0
- package/systemd/omp-conductor.service.example +14 -3
- package/systemd/recover-unit-test.sh +428 -0
package/src/daemon.ts
CHANGED
|
@@ -20,16 +20,23 @@ import {
|
|
|
20
20
|
} from "./config.ts";
|
|
21
21
|
import { availabilityState, type AvailabilityState } from "./availability.ts";
|
|
22
22
|
import {
|
|
23
|
+
UNREADABLE_TREE_FLAG,
|
|
23
24
|
analyseSettlement,
|
|
25
|
+
deriveChangedLine,
|
|
24
26
|
formatSettlementFlags,
|
|
25
27
|
settlementFlagSummary,
|
|
28
|
+
withDerivedChangedLine,
|
|
26
29
|
} from "./diff-flags.ts";
|
|
27
30
|
import { digestScheduleState, type DigestScheduleState } from "./digest-schedule.ts";
|
|
28
31
|
import { createEscalator, escalationIssueRef } from "./escalate.ts";
|
|
29
32
|
import { pendingCodeGraph, probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
|
|
30
33
|
import { graphHint } from "./graph.ts";
|
|
31
|
-
import { livingDaemon } from "./lifecycle.ts";
|
|
34
|
+
import { acquireOnceLease, healthCheck, livingDaemon } from "./lifecycle.ts";
|
|
32
35
|
import { requestImmediateTick, resolveTickConfigCwd, STALL_MARKER_FILE } from "./orchestrator-tick.ts";
|
|
36
|
+
import { runDoctor } from "./doctor.ts";
|
|
37
|
+
import { appendJournal, launchTransientUnit, readUpgradeJournal } from "./upgrade-journal.ts";
|
|
38
|
+
import { runCommand, verifyPendingUpgrade, type UpgradeVerifyDeps } from "./upgrade-verify.ts";
|
|
39
|
+
import { fleetLayers } from "./fleet.ts";
|
|
33
40
|
import { startOrchestrator } from "./orchestrator.ts";
|
|
34
41
|
import type { OrchestratorHandle } from "./orchestrator.ts";
|
|
35
42
|
import {
|
|
@@ -40,12 +47,21 @@ import {
|
|
|
40
47
|
} from "./reports.ts";
|
|
41
48
|
import {
|
|
42
49
|
recordReleaseBlock,
|
|
50
|
+
type GateShape,
|
|
43
51
|
type ReleaseBlockContext,
|
|
44
52
|
} from "./release-policy.ts";
|
|
45
53
|
import { branchName, effectiveLabels, route } from "./routing.ts";
|
|
46
54
|
import type { Routed, UnroutableReason } from "./routing.ts";
|
|
47
55
|
import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
|
|
48
56
|
import { classifyRun, providerCreditRefusal, providerTransientFault, type ClassifyFacts } from "./failure-class.ts";
|
|
57
|
+
import {
|
|
58
|
+
DEFAULT_MODEL_FALLBACK_THRESHOLD,
|
|
59
|
+
fallbackClause,
|
|
60
|
+
formatModelsTried,
|
|
61
|
+
modelsTried,
|
|
62
|
+
providerFailureFacts,
|
|
63
|
+
resolveDispatchModel,
|
|
64
|
+
} from "./model-fallback.ts";
|
|
49
65
|
import { projectLabels } from "./label-projection.ts";
|
|
50
66
|
import { dbPath, LIVE_STATES, openStore, utcDay } from "./store.ts";
|
|
51
67
|
import { makeTracker, type RateLimitStatus } from "./tracker/github.ts";
|
|
@@ -57,6 +73,7 @@ import type {
|
|
|
57
73
|
DispatchSummary,
|
|
58
74
|
DigestBacklog,
|
|
59
75
|
Escalation,
|
|
76
|
+
IssueComment,
|
|
60
77
|
IssueSnapshot,
|
|
61
78
|
MergedPrInfo,
|
|
62
79
|
OpenCloser,
|
|
@@ -115,7 +132,14 @@ import {
|
|
|
115
132
|
} from "./verbs/socket.ts";
|
|
116
133
|
import { homedir } from "node:os";
|
|
117
134
|
|
|
118
|
-
import {
|
|
135
|
+
import {
|
|
136
|
+
probeCriticalBase,
|
|
137
|
+
pushRunBranch,
|
|
138
|
+
readBaseChain,
|
|
139
|
+
type CriticalBaseProbe,
|
|
140
|
+
type CriticalBaseVerdict,
|
|
141
|
+
type RunRepoRef,
|
|
142
|
+
} from "./gitops.ts";
|
|
119
143
|
import {
|
|
120
144
|
planUsageLine,
|
|
121
145
|
readPlanUsage,
|
|
@@ -219,6 +243,21 @@ interface Deps {
|
|
|
219
243
|
* the real one.
|
|
220
244
|
*/
|
|
221
245
|
verbActions?: VerbActions;
|
|
246
|
+
/**
|
|
247
|
+
* The fleet-installs-itself verification pass (#486), run on the first tick
|
|
248
|
+
* after a detached upgrade. The default is the journal-gated
|
|
249
|
+
* {@link verifyPendingUpgradeTick}; a test injects its own to drive the tick
|
|
250
|
+
* without touching the host's packages, journal or outbox.
|
|
251
|
+
*/
|
|
252
|
+
upgradeVerifier?: (d: Deps) => Promise<void>;
|
|
253
|
+
/**
|
|
254
|
+
* Answers whether a preserved continuation branch contains every configured
|
|
255
|
+
* critical-base marker, for the stale-base admission hold (#428). Wired by
|
|
256
|
+
* `runDaemon` to the mirror-backed {@link probeCriticalBase}; a test injects
|
|
257
|
+
* a fake. Absent, the stale-base gate fails closed whenever a project names
|
|
258
|
+
* a marker (a safety interlock must not silently weaken).
|
|
259
|
+
*/
|
|
260
|
+
probeCriticalBase?: CriticalBaseProbe;
|
|
222
261
|
}
|
|
223
262
|
|
|
224
263
|
/**
|
|
@@ -668,6 +707,64 @@ function acceptanceCriteria(issue: ReadyIssue): string {
|
|
|
668
707
|
: "_The issue body is empty. Read the issue and its comments, and escalate if it is genuinely underspecified._";
|
|
669
708
|
}
|
|
670
709
|
|
|
710
|
+
/**
|
|
711
|
+
* How much of a comment thread one brief may hold. Grooming notes are short,
|
|
712
|
+
* and a thread that outgrows this is truncated with an explicit marker rather
|
|
713
|
+
* than a silently dropped tail — the disappearance this fix exists to prevent.
|
|
714
|
+
*/
|
|
715
|
+
const DISCUSSION_CHARS_BUDGET = 8_000;
|
|
716
|
+
|
|
717
|
+
/**
|
|
718
|
+
* The issue's comments as a brief section: attributed, numbered, oldest
|
|
719
|
+
* first, so a later correction visibly supersedes an earlier note. Rendered
|
|
720
|
+
* beside the body at dispatch, because the orchestrator's grooming is posted
|
|
721
|
+
* as comments and a worker must never depend on a runtime read to see it.
|
|
722
|
+
*
|
|
723
|
+
* `"unread"` is not "empty": when the tracker refused at dispatch the section
|
|
724
|
+
* says so, instead of silently reading as "no discussion" — which is exactly
|
|
725
|
+
* the #517 failure. An empty comment list renders nothing, so a commentless
|
|
726
|
+
* issue's brief stays byte-identical to what this package has always shipped.
|
|
727
|
+
*/
|
|
728
|
+
function renderDiscussion(comments: IssueComment[] | "unread"): string {
|
|
729
|
+
if (comments === "unread") {
|
|
730
|
+
return [
|
|
731
|
+
"## Discussion",
|
|
732
|
+
"",
|
|
733
|
+
"_The issue's comments could not be read at dispatch time. The live read below is_",
|
|
734
|
+
"_the only path to them; if it prints nothing, that is a failed read, not an_",
|
|
735
|
+
"_absence of discussion._",
|
|
736
|
+
"",
|
|
737
|
+
].join("\n");
|
|
738
|
+
}
|
|
739
|
+
if (comments.length === 0) return "";
|
|
740
|
+
const total = comments.length;
|
|
741
|
+
const lines = [
|
|
742
|
+
"## Discussion",
|
|
743
|
+
"",
|
|
744
|
+
`${total} comment${total === 1 ? "" : "s"} on the issue at dispatch, oldest first — a later`,
|
|
745
|
+
"comment supersedes an earlier one.",
|
|
746
|
+
"",
|
|
747
|
+
];
|
|
748
|
+
let chars = lines.join("\n").length;
|
|
749
|
+
let shown = 0;
|
|
750
|
+
for (const comment of comments) {
|
|
751
|
+
const block = `**@${comment.author} — comment ${shown + 1}:**\n\n${comment.body}\n\n`;
|
|
752
|
+
if (shown > 0 && chars + block.length > DISCUSSION_CHARS_BUDGET) break;
|
|
753
|
+
lines.push(`**@${comment.author} — comment ${shown + 1}:**`, "", comment.body, "");
|
|
754
|
+
chars += block.length;
|
|
755
|
+
shown += 1;
|
|
756
|
+
}
|
|
757
|
+
if (shown < total) {
|
|
758
|
+
const omitted = total - shown;
|
|
759
|
+
lines.push(
|
|
760
|
+
`… ${omitted} comment${omitted === 1 ? "" : "s"} omitted — the discussion exceeded ${DISCUSSION_CHARS_BUDGET} characters.`,
|
|
761
|
+
"Read the issue for the tail.",
|
|
762
|
+
"",
|
|
763
|
+
);
|
|
764
|
+
}
|
|
765
|
+
return lines.join("\n");
|
|
766
|
+
}
|
|
767
|
+
|
|
671
768
|
/**
|
|
672
769
|
* Record a state-label swap for projection (#201). The add enqueues before the
|
|
673
770
|
* remove — the reverse order would leave a window where the issue carries no
|
|
@@ -920,7 +1017,15 @@ export async function buildBrief(
|
|
|
920
1017
|
r: Routed,
|
|
921
1018
|
branch: string,
|
|
922
1019
|
worktree: string,
|
|
923
|
-
opts: {
|
|
1020
|
+
opts: {
|
|
1021
|
+
continuation?: boolean;
|
|
1022
|
+
defaultBranch?: string;
|
|
1023
|
+
salvagedSha?: string;
|
|
1024
|
+
/** The issue's comments, rendered into the Discussion section. `"unread"`
|
|
1025
|
+
* when the tracker refused at dispatch — rare, but it must never read as
|
|
1026
|
+
* "no comments" (the #517 failure mode). Absent means an empty list. */
|
|
1027
|
+
comments?: IssueComment[] | "unread";
|
|
1028
|
+
} = {},
|
|
924
1029
|
): Promise<string> {
|
|
925
1030
|
// Read per dispatch rather than caching: editing the brief then takes effect
|
|
926
1031
|
// on the next issue instead of needing a daemon restart.
|
|
@@ -965,6 +1070,7 @@ export async function buildBrief(
|
|
|
965
1070
|
BRANCH: branch,
|
|
966
1071
|
WORKTREE: worktree,
|
|
967
1072
|
ACCEPTANCE_CRITERIA: acceptanceCriteria(r.issue),
|
|
1073
|
+
ISSUE_COMMENTS: renderDiscussion(opts.comments ?? []),
|
|
968
1074
|
GATES: gatesBlock(r.repo),
|
|
969
1075
|
// Empty for a repo with no `graphProject`, and empty means *nothing*: the
|
|
970
1076
|
// placeholder sits flush against the next list item in the template, so an
|
|
@@ -981,6 +1087,185 @@ export function hasContinuationBudget(stops: number, maxContinuations: number):
|
|
|
981
1087
|
return stops <= maxContinuations;
|
|
982
1088
|
}
|
|
983
1089
|
|
|
1090
|
+
/** True while unspent failed-implementation attempts remain. This is the
|
|
1091
|
+
* dispatcher's admission gate: once every `maxAttemptsPerIssue` slot is
|
|
1092
|
+
* spent, the issue is held as `failed-attempts` forever, and the `unblock`
|
|
1093
|
+
* verb withholds the queue label on the same predicate (#348). */
|
|
1094
|
+
export function hasFailedAttemptBudget(failures: number, maxAttempts: number): boolean {
|
|
1095
|
+
return failures < maxAttempts;
|
|
1096
|
+
}
|
|
1097
|
+
|
|
1098
|
+
/** The failure classes `countContinuations` deliberately does not charge — the
|
|
1099
|
+
* inverted copy of its exclusions, kept beside the breakdown that consumes it
|
|
1100
|
+
* so the two can only drift together (#439). `orphan-clean` is absent on
|
|
1101
|
+
* purpose: daemon orphans consume the continuation budget, which is exactly
|
|
1102
|
+
* why the requeue side has to respect the ceiling instead of racing it. */
|
|
1103
|
+
const NON_CONTINUATION_CLASSES: Partial<Record<FailureClass, true>> = {
|
|
1104
|
+
"admin-kill": true,
|
|
1105
|
+
"settlement-stuck": true,
|
|
1106
|
+
"env-start-failure": true,
|
|
1107
|
+
"dispatch-infra": true,
|
|
1108
|
+
"provider-credit": true,
|
|
1109
|
+
"provider-transient": true,
|
|
1110
|
+
};
|
|
1111
|
+
|
|
1112
|
+
/** How one issue spent its continuation budget, grouped by failure class —
|
|
1113
|
+
* the exact rows `continuationsFor` charges, so an exhaustion escalation
|
|
1114
|
+
* reports the same budget it says is spent. `unclassified` groups rows that
|
|
1115
|
+
* charged before the class was written (a pre-upgrade NULL). */
|
|
1116
|
+
function continuationBreakdown(runs: readonly RunRecord[]): Map<string, number> {
|
|
1117
|
+
const perClass = new Map<string, number>();
|
|
1118
|
+
for (const r of runs) {
|
|
1119
|
+
const chargedAsKilledOrOrphaned =
|
|
1120
|
+
(r.state === "killed" || r.state === "orphaned" || r.state === "blocked") &&
|
|
1121
|
+
(r.failureClass === undefined || NON_CONTINUATION_CLASSES[r.failureClass] === undefined);
|
|
1122
|
+
const chargedAsReturned = r.state === "failed" && r.failureClass === "returned-for-revision";
|
|
1123
|
+
if (!chargedAsKilledOrOrphaned && !chargedAsReturned) continue;
|
|
1124
|
+
const cls = r.failureClass ?? "unclassified";
|
|
1125
|
+
perClass.set(cls, (perClass.get(cls) ?? 0) + 1);
|
|
1126
|
+
}
|
|
1127
|
+
return perClass;
|
|
1128
|
+
}
|
|
1129
|
+
|
|
1130
|
+
/** The newest attempt's preserved work, if any, so an exhaustion escalation can
|
|
1131
|
+
* say whether continuing is worthwhile: `salvageSha`/`headSha`/`prUrl` are the
|
|
1132
|
+
* three artifacts a run can leave, and all null on a branch nothing reached. */
|
|
1133
|
+
function newestContinuableRun(runs: readonly RunRecord[]): RunRecord | undefined {
|
|
1134
|
+
const newestFirst = [...runs].reverse();
|
|
1135
|
+
return newestFirst.find(
|
|
1136
|
+
(r) => r.salvageSha !== undefined || r.headSha !== undefined || r.prUrl !== undefined,
|
|
1137
|
+
);
|
|
1138
|
+
}
|
|
1139
|
+
|
|
1140
|
+
/** The fenced-block info string that marks an exhaustion postmortem comment, so
|
|
1141
|
+
* a grooming scout re-slicing the issue can find and parse the whole block by
|
|
1142
|
+
* grepping for it. */
|
|
1143
|
+
export const POSTMORTEM_MARKER = "conductor-postmortem";
|
|
1144
|
+
|
|
1145
|
+
/** How one issue's attempt chain failed, as "3× ci-deterministic, 1× …" — the
|
|
1146
|
+
* digest shape for naming what the exhaustion was. Groups every row by its
|
|
1147
|
+
* failure class, whether or not it charged the continuation budget, because
|
|
1148
|
+
* the postmortem tells the whole story and not just the budget half (#290). */
|
|
1149
|
+
export function attemptClassBreakdown(runs: readonly RunRecord[]): string {
|
|
1150
|
+
const perClass = new Map<string, number>();
|
|
1151
|
+
for (const r of runs) {
|
|
1152
|
+
const cls = r.failureClass ?? "unclassified";
|
|
1153
|
+
perClass.set(cls, (perClass.get(cls) ?? 0) + 1);
|
|
1154
|
+
}
|
|
1155
|
+
if (perClass.size === 0) return "unclassified";
|
|
1156
|
+
return Array.from(perClass, ([cls, n]) => `${n}× ${cls}`).join(", ");
|
|
1157
|
+
}
|
|
1158
|
+
|
|
1159
|
+
/** Wall-clock duration of one run as a compact human string ("45m", "1h30m"). */
|
|
1160
|
+
export function humanDuration(ms: number): string {
|
|
1161
|
+
const seconds = Math.max(0, Math.round(ms / 1_000));
|
|
1162
|
+
if (seconds < 60) return `${seconds}s`;
|
|
1163
|
+
const minutes = Math.round(seconds / 60);
|
|
1164
|
+
if (minutes < 60) return `${minutes}m`;
|
|
1165
|
+
const hours = Math.floor(minutes / 60);
|
|
1166
|
+
const rest = minutes % 60;
|
|
1167
|
+
return rest === 0 ? `${hours}h` : `${hours}h${rest}m`;
|
|
1168
|
+
}
|
|
1169
|
+
|
|
1170
|
+
/** Flatten and bound a run's last error to one greppable table line. */
|
|
1171
|
+
export function oneLineBrief(text: string | undefined): string | undefined {
|
|
1172
|
+
if (text === undefined || text.trim() === "") return undefined;
|
|
1173
|
+
const flat = text.replace(/\s+/g, " ").trim();
|
|
1174
|
+
return flat.length > 90 ? `${flat.slice(0, 89)}…` : flat;
|
|
1175
|
+
}
|
|
1176
|
+
|
|
1177
|
+
/**
|
|
1178
|
+
* The exhaustion postmortem block: one greppable fenced block covering every
|
|
1179
|
+
* attempt in the chain — continuation rows included — with per-attempt turns,
|
|
1180
|
+
* wall clock, failure class and a one-line last error, the explicit salvage
|
|
1181
|
+
* state, the spend total and the transcript paths for local inspection.
|
|
1182
|
+
*
|
|
1183
|
+
* Pure so the tests hold the shape, not the transport: the writer below owns
|
|
1184
|
+
* the once-only guarantee, this owns what "once" looks like.
|
|
1185
|
+
*/
|
|
1186
|
+
export function formatExhaustionPostmortem(args: {
|
|
1187
|
+
issue: number;
|
|
1188
|
+
runs: readonly RunRecord[];
|
|
1189
|
+
reason: string;
|
|
1190
|
+
}): string {
|
|
1191
|
+
const { issue, runs, reason } = args;
|
|
1192
|
+
const artifact = newestContinuableRun(runs);
|
|
1193
|
+
const totalSpend = runs.reduce((sum, r) => sum + r.spendUsd, 0);
|
|
1194
|
+
const spend = `$${totalSpend.toFixed(2)}`;
|
|
1195
|
+
const attemptLines = runs.map((r) => {
|
|
1196
|
+
const wall = r.endedAt === undefined ? "—" : humanDuration(r.endedAt - r.startedAt);
|
|
1197
|
+
const cls = r.failureClass ?? "unclassified";
|
|
1198
|
+
const error = oneLineBrief(r.lastError) ?? "—";
|
|
1199
|
+
return (
|
|
1200
|
+
` attempt ${r.attempt} ${r.state.padEnd(12)} turns ${r.turns}/${r.maxTurns} ` +
|
|
1201
|
+
`${cls.padEnd(24)} ${wall.padStart(4)} last error: ${error}`
|
|
1202
|
+
);
|
|
1203
|
+
});
|
|
1204
|
+
const salvage =
|
|
1205
|
+
artifact === undefined
|
|
1206
|
+
? "Salvaged WIP: absent — no attempt preserved a branch, head SHA or pull request."
|
|
1207
|
+
: `Salvaged WIP: present — branch ${artifact.branch} at ${artifact.headSha ?? artifact.salvageSha}` +
|
|
1208
|
+
`${artifact.prUrl === undefined ? "" : ` (PR ${artifact.prUrl})`}.`;
|
|
1209
|
+
return [
|
|
1210
|
+
`\`\`\`${POSTMORTEM_MARKER}`,
|
|
1211
|
+
`#${issue} exhausted: ${attemptClassBreakdown(runs)}`,
|
|
1212
|
+
reason,
|
|
1213
|
+
"",
|
|
1214
|
+
`Attempts (${runs.length} total, ${spend} spend):`,
|
|
1215
|
+
...attemptLines,
|
|
1216
|
+
salvage,
|
|
1217
|
+
`Spend total: ${spend} across ${runs.length} attempts.`,
|
|
1218
|
+
"Transcripts:",
|
|
1219
|
+
...runs.map((r) => (r.sessionFile === undefined ? " (none)" : ` ${r.sessionFile}`)),
|
|
1220
|
+
"```",
|
|
1221
|
+
].join("\n");
|
|
1222
|
+
}
|
|
1223
|
+
|
|
1224
|
+
/** Dedupe key prefix for the exhaustion postmortem comment, per issue. */
|
|
1225
|
+
function postmortemDedupeKey(project: string, issue: number): string {
|
|
1226
|
+
return `${project}:postmortem:${issue}`;
|
|
1227
|
+
}
|
|
1228
|
+
|
|
1229
|
+
/**
|
|
1230
|
+
* The exhaustion postmortem: written exactly once per issue, at the point the
|
|
1231
|
+
* continuation budget is spent and the issue is settled toward a human.
|
|
1232
|
+
*
|
|
1233
|
+
* The comment and the material event are both gated by the store's notification
|
|
1234
|
+
* ledger — the same idempotence guard the escalator uses — so re-settling an
|
|
1235
|
+
* already-postmortemed issue posts nothing and records nothing. A body-string
|
|
1236
|
+
* match on the issue would be the wrong guard: an issue re-scoped and re-run
|
|
1237
|
+
* would still carry the old block, and the guarantee asked of this is "decided
|
|
1238
|
+
* once", not "deduped against what is already written".
|
|
1239
|
+
*
|
|
1240
|
+
* The digest must name the exhaustion even if the comment write fails, so the
|
|
1241
|
+
* material event is recorded before the write and unconditionally (the ledger
|
|
1242
|
+
* is append-only, and the gate above already ran once). The comment failure is
|
|
1243
|
+
* logged rather than taking the sweep down with it.
|
|
1244
|
+
*/
|
|
1245
|
+
async function postExhaustionPostmortem(d: Deps, run: RunRecord, reason: string): Promise<void> {
|
|
1246
|
+
const { project, tracker, store } = d;
|
|
1247
|
+
const key = postmortemDedupeKey(project.name, run.issue);
|
|
1248
|
+
if (store.wasNotified(key)) return;
|
|
1249
|
+
const runs = store.runsForIssue(project.name, run.issue);
|
|
1250
|
+
const body = formatExhaustionPostmortem({ issue: run.issue, runs, reason });
|
|
1251
|
+
const occurredAt = Date.now();
|
|
1252
|
+
store.recordMaterialEvent({
|
|
1253
|
+
project: project.name,
|
|
1254
|
+
category: "exhaustion",
|
|
1255
|
+
summary: `#${run.issue} exhausted: ${attemptClassBreakdown(runs)}`,
|
|
1256
|
+
evidence: body,
|
|
1257
|
+
occurredAt,
|
|
1258
|
+
recordedAt: occurredAt,
|
|
1259
|
+
});
|
|
1260
|
+
try {
|
|
1261
|
+
await tracker.comment(run.issue, body);
|
|
1262
|
+
store.markNotified(key);
|
|
1263
|
+
log(`#${run.issue} posted exhaustion postmortem (${runs.length} attempts)`);
|
|
1264
|
+
} catch (err) {
|
|
1265
|
+
log(`#${run.issue} postmortem comment could not be posted (${errText(err)})`);
|
|
1266
|
+
}
|
|
1267
|
+
}
|
|
1268
|
+
|
|
984
1269
|
export type ExtendTurnLimitResult =
|
|
985
1270
|
| { kind: "extended"; runId: string; maxTurns: number }
|
|
986
1271
|
| { kind: "not-increase"; runId: string; maxTurns: number }
|
|
@@ -1184,6 +1469,19 @@ export async function verifyPushedGreenClaim(
|
|
|
1184
1469
|
};
|
|
1185
1470
|
}
|
|
1186
1471
|
|
|
1472
|
+
/** What the settlement audit of one green run produced: the advisory flags
|
|
1473
|
+
* (test weakening only — the file-list disclosure is derived, not flagged),
|
|
1474
|
+
* whether the diff was cut short, and the `changed:` line composed from the
|
|
1475
|
+
* PR's own diff. */
|
|
1476
|
+
export interface SettlementAuditResult {
|
|
1477
|
+
flags: SettlementFlag[];
|
|
1478
|
+
truncated: boolean;
|
|
1479
|
+
/** The `changed:` file list derived from the PR's diff, present whenever the
|
|
1480
|
+
* diff could be read. Absent means the tree could not be read, and `flags`
|
|
1481
|
+
* then carries exactly {@link UNREADABLE_TREE_FLAG}. */
|
|
1482
|
+
changedLine?: string;
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1187
1485
|
/**
|
|
1188
1486
|
* Audit a worker's own account of its work against the pull request it pushed.
|
|
1189
1487
|
*
|
|
@@ -1192,27 +1490,21 @@ export async function verifyPushedGreenClaim(
|
|
|
1192
1490
|
* shares none of its authority — that function decides a run's state, this one
|
|
1193
1491
|
* cannot, by construction. It returns evidence and the caller appends it.
|
|
1194
1492
|
*
|
|
1195
|
-
*
|
|
1196
|
-
*
|
|
1197
|
-
*
|
|
1198
|
-
* "did not look", and the report says so rather than presenting silence as a
|
|
1199
|
-
* clean bill.
|
|
1493
|
+
* A diff that cannot be read is a finding (`changed-line-missing`), never an
|
|
1494
|
+
* empty flag list: nothing was derived and nothing was checked, and that must
|
|
1495
|
+
* not read as a clean bill.
|
|
1200
1496
|
*/
|
|
1201
1497
|
export async function collectSettlementFlags(
|
|
1202
1498
|
tracker: Pick<Tracker, "prDiff">,
|
|
1203
|
-
claim: { prUrl?: string;
|
|
1204
|
-
): Promise<
|
|
1205
|
-
if (claim.prUrl === undefined) return
|
|
1499
|
+
claim: { prUrl?: string; issueText: string },
|
|
1500
|
+
): Promise<SettlementAuditResult> {
|
|
1501
|
+
if (claim.prUrl === undefined) return { flags: [UNREADABLE_TREE_FLAG], truncated: false };
|
|
1206
1502
|
const diff = await tracker.prDiff(claim.prUrl);
|
|
1207
|
-
if (diff === undefined) return
|
|
1503
|
+
if (diff === undefined) return { flags: [UNREADABLE_TREE_FLAG], truncated: false };
|
|
1208
1504
|
return {
|
|
1209
|
-
flags: analyseSettlement({
|
|
1210
|
-
report: claim.report,
|
|
1211
|
-
issueText: claim.issueText,
|
|
1212
|
-
diff,
|
|
1213
|
-
priorReports: claim.priorReports,
|
|
1214
|
-
}),
|
|
1505
|
+
flags: analyseSettlement({ issueText: claim.issueText, diff }),
|
|
1215
1506
|
truncated: diff.truncated,
|
|
1507
|
+
changedLine: deriveChangedLine(diff),
|
|
1216
1508
|
};
|
|
1217
1509
|
}
|
|
1218
1510
|
|
|
@@ -1258,7 +1550,7 @@ export function workerReleaseBlockRecorder(
|
|
|
1258
1550
|
issue: number,
|
|
1259
1551
|
runId: string,
|
|
1260
1552
|
root = stateDir(),
|
|
1261
|
-
): (shape:
|
|
1553
|
+
): (shape: GateShape, context: ReleaseBlockContext) => void {
|
|
1262
1554
|
return (shape, context) =>
|
|
1263
1555
|
recordReleaseBlock(
|
|
1264
1556
|
project,
|
|
@@ -1273,6 +1565,48 @@ export function workerReleaseBlockRecorder(
|
|
|
1273
1565
|
);
|
|
1274
1566
|
}
|
|
1275
1567
|
|
|
1568
|
+
/**
|
|
1569
|
+
* The exact PR a new claim inherits from its terminal predecessor (#434).
|
|
1570
|
+
*
|
|
1571
|
+
* `recentRuns` shows only the newest attempt per issue, and the mediated verbs
|
|
1572
|
+
* resolve a PR's owner through the store — so the moment an orphan-clean
|
|
1573
|
+
* requeue creates its continuation row, the predecessor's green PR (recorded
|
|
1574
|
+
* on the older row) stops resolving and every mediated verb refuses it as "not
|
|
1575
|
+
* a pull request any run opened". Seeding the continuation row with the same
|
|
1576
|
+
* PR at claim time closes that window: ownership never depends on a worker
|
|
1577
|
+
* rediscovering and re-reporting the PR.
|
|
1578
|
+
*
|
|
1579
|
+
* The checks are deliberately the exact counterpart of the store query's scope:
|
|
1580
|
+
* the claim's project/issue/repo/branch must all match the predecessor, the
|
|
1581
|
+
* predecessor must be terminal (a live row is still doing its own work, and a
|
|
1582
|
+
* claim beside one is already a double dispatch), and the tracker must confirm
|
|
1583
|
+
* the PR is still open — a closed or merged predecessor PR is settled by the
|
|
1584
|
+
* sweep against the row that owns it, never carried onto a fresh attempt. An
|
|
1585
|
+
* unreadable tracker fails closed: the new row simply starts without PR
|
|
1586
|
+
* ownership, exactly as before #434, and adoption or the worker's own
|
|
1587
|
+
* pr_create recovers the link.
|
|
1588
|
+
*/
|
|
1589
|
+
export async function inheritedPrForContinuation(
|
|
1590
|
+
tracker: Tracker,
|
|
1591
|
+
prior: RunRecord | undefined,
|
|
1592
|
+
claim: { project: string; issue: number; repo: string; branch: string },
|
|
1593
|
+
): Promise<{ prUrl: string; headSha?: string } | undefined> {
|
|
1594
|
+
if (prior === undefined || prior.prUrl === undefined) return undefined;
|
|
1595
|
+
// Terminal only. A live predecessor owns its PR outright and is not done
|
|
1596
|
+
// with it; inheriting would launder a double dispatch's row.
|
|
1597
|
+
if (prior.state === "claimed" || prior.state === "running") return undefined;
|
|
1598
|
+
if (prior.project !== claim.project || prior.issue !== claim.issue) return undefined;
|
|
1599
|
+
if (prior.repo !== claim.repo || prior.branch !== claim.branch) return undefined;
|
|
1600
|
+
let state: PrState | undefined;
|
|
1601
|
+
try {
|
|
1602
|
+
state = await tracker.prState(prior.prUrl);
|
|
1603
|
+
} catch {
|
|
1604
|
+
return undefined;
|
|
1605
|
+
}
|
|
1606
|
+
if (state !== "open") return undefined;
|
|
1607
|
+
return { prUrl: prior.prUrl, ...(prior.headSha === undefined ? {} : { headSha: prior.headSha }) };
|
|
1608
|
+
}
|
|
1609
|
+
|
|
1276
1610
|
export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
1277
1611
|
const { project, caps, tracker, store } = d;
|
|
1278
1612
|
const issue = r.issue.number;
|
|
@@ -1429,7 +1763,13 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
1429
1763
|
// (#201).
|
|
1430
1764
|
// Read before this attempt's own row exists, so `latestRun` still means the
|
|
1431
1765
|
// attempt whose work this one inherits.
|
|
1432
|
-
const
|
|
1766
|
+
const prior = store.latestRun(project.name, issue);
|
|
1767
|
+
const priorSalvage = prior?.salvageSha;
|
|
1768
|
+
// The failover decision is made from the store the same way admission is:
|
|
1769
|
+
// counts of this issue's own rows, so the switch is sticky per chain and a
|
|
1770
|
+
// fresh issue always starts on the primary (#286). Read before the claim
|
|
1771
|
+
// writes this attempt's row, which would otherwise break the streak.
|
|
1772
|
+
const chainFacts = providerFailureFacts(store.runsForIssue(project.name, issue));
|
|
1433
1773
|
run = store.createRun({
|
|
1434
1774
|
project: project.name,
|
|
1435
1775
|
issue,
|
|
@@ -1443,6 +1783,27 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
1443
1783
|
maxTurns: caps.workerMaxTurns,
|
|
1444
1784
|
startedAt: Date.now(),
|
|
1445
1785
|
});
|
|
1786
|
+
// #434: carry a terminal predecessor's open PR onto the continuation row.
|
|
1787
|
+
// The claim itself stays synchronous — `handleIssue` claims at the top of
|
|
1788
|
+
// its dispatch burst, the window the shutdown fence and its tests rely on
|
|
1789
|
+
// (#374) — so the tuple-exact, tracker-verified inheritance lands here,
|
|
1790
|
+
// still before any provisioning, network call or verb socket. The new row
|
|
1791
|
+
// is `recentRuns`' newest attempt for the issue the moment it exists, so
|
|
1792
|
+
// without this a requeue hides the green PR the orphaned attempt opened
|
|
1793
|
+
// until a worker happens to rediscover it.
|
|
1794
|
+
const inherited = await inheritedPrForContinuation(tracker, prior, {
|
|
1795
|
+
project: project.name,
|
|
1796
|
+
issue,
|
|
1797
|
+
repo: r.repo.name,
|
|
1798
|
+
branch,
|
|
1799
|
+
});
|
|
1800
|
+
if (inherited !== undefined && prior !== undefined) {
|
|
1801
|
+
store.updateRun(run.id, {
|
|
1802
|
+
prUrl: inherited.prUrl,
|
|
1803
|
+
...(inherited.headSha === undefined ? {} : { headSha: inherited.headSha }),
|
|
1804
|
+
});
|
|
1805
|
+
log(`#${issue} inherited PR ${inherited.prUrl} from attempt ${prior.attempt} (${prior.state})`);
|
|
1806
|
+
}
|
|
1446
1807
|
const maxTurns = run.maxTurns;
|
|
1447
1808
|
const turnOverride = maxTurns > caps.workerMaxTurns ? maxTurns : undefined;
|
|
1448
1809
|
const runId = run.id;
|
|
@@ -1522,17 +1883,50 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
1522
1883
|
|
|
1523
1884
|
store.updateRun(runId, { worktree: worktreePath, state: "running" });
|
|
1524
1885
|
|
|
1886
|
+
// Where this attempt goes, and (when the failover fired) the clause that
|
|
1887
|
+
// makes it attributable: with no `modelFallbacks` configured this is the
|
|
1888
|
+
// primary model — or none, for an unconfigured project — and today's
|
|
1889
|
+
// dispatch is byte for byte what it has always been.
|
|
1890
|
+
const chainConfigured = (project.modelFallbacks?.length ?? 0) > 0;
|
|
1891
|
+
const choice = resolveDispatchModel({
|
|
1892
|
+
workerModel: project.workerModel,
|
|
1893
|
+
modelFallbacks: project.modelFallbacks,
|
|
1894
|
+
threshold: project.modelFallbackThreshold ?? DEFAULT_MODEL_FALLBACK_THRESHOLD,
|
|
1895
|
+
streak: chainFacts.streak,
|
|
1896
|
+
});
|
|
1897
|
+
// Recorded before the launch, so even a run killed mid-flight leaves the
|
|
1898
|
+
// model it chose on its row. Only a chain-configured project writes the
|
|
1899
|
+
// column: absent `modelFallbacks` must preserve today's rows byte for byte.
|
|
1900
|
+
if (chainConfigured && choice.model !== undefined) {
|
|
1901
|
+
store.updateRun(runId, { model: choice.model });
|
|
1902
|
+
}
|
|
1903
|
+
const clause = fallbackClause(choice, chainFacts, project.workerModel);
|
|
1904
|
+
|
|
1525
1905
|
log(
|
|
1526
|
-
`#${issue} attempt ${attempt} → ${r.repo.name} ${branch}` +
|
|
1906
|
+
`#${issue} attempt ${attempt}${clause === undefined ? "" : ` ${clause}`} → ${r.repo.name} ${branch}` +
|
|
1527
1907
|
(provisioned.reattached ? " (continuation: reattached existing branch)" : ""),
|
|
1528
1908
|
);
|
|
1529
1909
|
|
|
1910
|
+
// The discussion is rendered at dispatch so a worker never depends on a
|
|
1911
|
+
// runtime `gh` read to see the orchestrator's grooming (#517). The read is
|
|
1912
|
+
// best-effort, but its failure is not silent: an unreadable tracker names
|
|
1913
|
+
// itself in the brief's Discussion section instead of reading as "no
|
|
1914
|
+
// comments" — the exact confusion this fix removes.
|
|
1915
|
+
let comments: IssueComment[] | "unread";
|
|
1916
|
+
try {
|
|
1917
|
+
comments = await tracker.listComments(issue);
|
|
1918
|
+
} catch (err) {
|
|
1919
|
+
log(`#${issue} issue comments unreadable at dispatch; the brief will say so: ${errText(err)}`);
|
|
1920
|
+
comments = "unread";
|
|
1921
|
+
}
|
|
1922
|
+
|
|
1530
1923
|
const brief = await buildBrief(project, r, branch, worktreePath, {
|
|
1531
1924
|
continuation: provisioned.reattached,
|
|
1532
1925
|
defaultBranch: r.repo.defaultBranch,
|
|
1533
1926
|
...(provisioned.reattached && priorSalvage !== undefined
|
|
1534
1927
|
? { salvagedSha: priorSalvage }
|
|
1535
1928
|
: {}),
|
|
1929
|
+
comments,
|
|
1536
1930
|
});
|
|
1537
1931
|
if (await settleStopBeforeSession()) return;
|
|
1538
1932
|
if (await settleDrainBeforeSession()) return;
|
|
@@ -1567,7 +1961,7 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
1567
1961
|
onChildLog: (line) => {
|
|
1568
1962
|
log(`#${issue} ${line}`);
|
|
1569
1963
|
},
|
|
1570
|
-
...(
|
|
1964
|
+
...(choice.model === undefined ? {} : { model: choice.model }),
|
|
1571
1965
|
releaseGrants: resolveReleaseGrants(project),
|
|
1572
1966
|
onReleaseBlocked: workerReleaseBlockRecorder(project.name, issue, runId),
|
|
1573
1967
|
onTurn: (n) => store.updateRun(runId, { turns: n }),
|
|
@@ -1618,39 +2012,37 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
1618
2012
|
// The other half of not believing a worker about its own run (#128). The
|
|
1619
2013
|
// claim being audited is `state: pushed-green`, so the audit runs on the
|
|
1620
2014
|
// worker's claim rather than on what verification made of it: a claim that
|
|
1621
|
-
// GitHub then contradicted is exactly the report whose
|
|
1622
|
-
//
|
|
1623
|
-
//
|
|
2015
|
+
// GitHub then contradicted is exactly the report whose details matter
|
|
2016
|
+
// most. Advisory throughout — nothing below reads `flags` when deciding
|
|
2017
|
+
// `state`, and the ordering here makes that checkable.
|
|
1624
2018
|
const audit =
|
|
1625
2019
|
result.state === "pushed-green"
|
|
1626
2020
|
? await collectSettlementFlags(tracker, {
|
|
1627
2021
|
prUrl: result.prUrl,
|
|
1628
|
-
report: result.report,
|
|
1629
|
-
// Earlier attempts' reports pool as disclosure coverage (#199). The
|
|
1630
|
-
// current row has not settled yet, so this is exactly the prior set;
|
|
1631
|
-
// the strict `< attempt` guard keeps an already-persisted copy of
|
|
1632
|
-
// this attempt out just in case. Pre-#199 rows have NULL reports and
|
|
1633
|
-
// contribute nothing — today's behaviour.
|
|
1634
|
-
priorReports: store
|
|
1635
|
-
.attemptReports(project.name, issue)
|
|
1636
|
-
.filter((r) => r.attempt < attempt)
|
|
1637
|
-
.map((r) => r.report),
|
|
1638
2022
|
issueText: `${r.issue.title}\n${r.issue.body}`,
|
|
1639
2023
|
})
|
|
1640
2024
|
: undefined;
|
|
1641
|
-
if (result.state === "pushed-green") {
|
|
1642
|
-
|
|
1643
|
-
else if (audit.truncated) log(`#${issue} settlement audit read a truncated PR diff`);
|
|
2025
|
+
if (result.state === "pushed-green" && audit?.truncated) {
|
|
2026
|
+
log(`#${issue} settlement audit read a truncated PR diff`);
|
|
1644
2027
|
}
|
|
1645
2028
|
const auditLines =
|
|
1646
2029
|
audit === undefined
|
|
1647
2030
|
? []
|
|
1648
|
-
: formatSettlementFlags(audit.flags, { truncated: audit.truncated
|
|
2031
|
+
: formatSettlementFlags(audit.flags, { truncated: audit.truncated });
|
|
2032
|
+
// The report's file list is derived from the PR's own diff, not retyped
|
|
2033
|
+
// from the worker's memory of the session: the narrative stays the
|
|
2034
|
+
// worker's, the disclosure becomes the diff's (#488). A diff that could
|
|
2035
|
+
// not be read leaves the worker's text untouched and the audit's
|
|
2036
|
+
// `changed-line-missing` flag says so.
|
|
2037
|
+
const settlementReport =
|
|
2038
|
+
audit?.changedLine === undefined
|
|
2039
|
+
? result.report
|
|
2040
|
+
: withDerivedChangedLine(result.report, audit.changedLine);
|
|
1649
2041
|
|
|
1650
2042
|
const finalReport = [
|
|
1651
2043
|
...(verified.reason === undefined ? [] : [verified.reason, ""]),
|
|
1652
2044
|
...(auditLines.length === 0 ? [] : [...auditLines, ""]),
|
|
1653
|
-
|
|
2045
|
+
settlementReport,
|
|
1654
2046
|
].join("\n");
|
|
1655
2047
|
|
|
1656
2048
|
// What becomes of the tree, decided once, before any label or page. A
|
|
@@ -1695,12 +2087,18 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
1695
2087
|
endedAt: Date.now(),
|
|
1696
2088
|
turns: result.turns,
|
|
1697
2089
|
spendUsd: result.spendUsd,
|
|
1698
|
-
|
|
1699
|
-
|
|
2090
|
+
// The worker only reports these when it actually established them; a kill
|
|
2091
|
+
// or a settle whose report named no PR must not wipe what a verb recorded
|
|
2092
|
+
// earlier in the same run (#468). The sink in `updateRun` skips undefined
|
|
2093
|
+
// too, but a settle should not even build such a patch.
|
|
2094
|
+
...(result.prUrl === undefined ? {} : { prUrl: result.prUrl }),
|
|
2095
|
+
...(result.headSha === undefined ? {} : { headSha: result.headSha }),
|
|
1700
2096
|
sessionFile: result.sessionFile,
|
|
1701
|
-
// Every terminal state persists the worker's report
|
|
1702
|
-
//
|
|
1703
|
-
|
|
2097
|
+
// Every terminal state persists the worker's report — with the `changed:`
|
|
2098
|
+
// file list derived from the PR's diff where one could be read — not
|
|
2099
|
+
// just a green push: a stopped attempt's partial report is still part of
|
|
2100
|
+
// its audit trail.
|
|
2101
|
+
report: settlementReport,
|
|
1704
2102
|
...settlement?.patch,
|
|
1705
2103
|
...(audit === undefined || audit.flags.length === 0
|
|
1706
2104
|
? {}
|
|
@@ -2275,8 +2673,13 @@ export async function watchBaseHealth(
|
|
|
2275
2673
|
|
|
2276
2674
|
export async function settlePushedGreen(
|
|
2277
2675
|
d: Pick<Deps, "project" | "tracker" | "store">,
|
|
2278
|
-
): Promise<
|
|
2676
|
+
): Promise<number> {
|
|
2279
2677
|
const { project, tracker, store } = d;
|
|
2678
|
+
// Runs the sweep resolved by terminalising the row. Every terminal write
|
|
2679
|
+
// below increments it; the tick attributes it as `settled` on the pass's
|
|
2680
|
+
// dispatch record, so a held pass is visible as work done, not just as a
|
|
2681
|
+
// clock that moved (#497).
|
|
2682
|
+
let settled = 0;
|
|
2280
2683
|
// Filtered from the active set rather than asked for with a new query: active
|
|
2281
2684
|
// is live workers plus these, so the list is bounded by the worker cap plus
|
|
2282
2685
|
// the number of PRs awaiting a merge — a handful, by construction. A fleet
|
|
@@ -2343,6 +2746,7 @@ export async function settlePushedGreen(
|
|
|
2343
2746
|
patch.recoveryAction = "none";
|
|
2344
2747
|
}
|
|
2345
2748
|
store.updateRun(run.id, patch);
|
|
2749
|
+
settled += 1;
|
|
2346
2750
|
log(`#${run.issue} settled: ${settlement.reason}`);
|
|
2347
2751
|
continue;
|
|
2348
2752
|
}
|
|
@@ -2357,7 +2761,7 @@ export async function settlePushedGreen(
|
|
|
2357
2761
|
}
|
|
2358
2762
|
if (verification === undefined) continue;
|
|
2359
2763
|
if (verification.status === "green") {
|
|
2360
|
-
store.updateRun(run.id, { state: "pushed-green", lastError:
|
|
2764
|
+
store.updateRun(run.id, { state: "pushed-green", lastError: null });
|
|
2361
2765
|
log(`#${run.issue} checks settled: ${verification.reason}`);
|
|
2362
2766
|
} else if (verification.status === "failed") {
|
|
2363
2767
|
// Equally terminal, so the release is enqueued before the row writes,
|
|
@@ -2366,11 +2770,13 @@ export async function settlePushedGreen(
|
|
|
2366
2770
|
// and its live PR is exactly the work the label must keep guarding.
|
|
2367
2771
|
releaseInProgress(d, run.issue, verification.reason);
|
|
2368
2772
|
store.updateRun(run.id, { state: "failed", lastError: verification.reason });
|
|
2773
|
+
settled += 1;
|
|
2369
2774
|
log(`#${run.issue} checks failed: ${verification.reason}`);
|
|
2370
2775
|
} else {
|
|
2371
2776
|
store.updateRun(run.id, { lastError: verification.reason });
|
|
2372
2777
|
}
|
|
2373
2778
|
}
|
|
2779
|
+
return settled;
|
|
2374
2780
|
}
|
|
2375
2781
|
|
|
2376
2782
|
const ADOPTABLE_PR_STATES: Partial<Record<RunState, true>> = {
|
|
@@ -2612,6 +3018,7 @@ export function summarizeDispatch(
|
|
|
2612
3018
|
admitted: number,
|
|
2613
3019
|
holds: readonly AdmissionHold[],
|
|
2614
3020
|
completedAt = Date.now(),
|
|
3021
|
+
settled = 0,
|
|
2615
3022
|
): DispatchSummary {
|
|
2616
3023
|
const groups = new Map<AdmissionHoldReason, { count: number; issues: number[] }>();
|
|
2617
3024
|
for (const hold of holds) {
|
|
@@ -2630,6 +3037,27 @@ export function summarizeDispatch(
|
|
|
2630
3037
|
holds: [...groups]
|
|
2631
3038
|
.sort(([a], [b]) => a.localeCompare(b))
|
|
2632
3039
|
.map(([reason, group]) => ({ reason, ...group })),
|
|
3040
|
+
settled,
|
|
3041
|
+
};
|
|
3042
|
+
}
|
|
3043
|
+
|
|
3044
|
+
/**
|
|
3045
|
+
* The record of a held pass (#497): the loop ran — settling and reconciling
|
|
3046
|
+
* above the pause gate — but routing never happened and nothing was admitted.
|
|
3047
|
+
* `ready`/`routed`/`claimed` stay 0 because they were never computed; the
|
|
3048
|
+
* `paused` flag is what keeps a reader from reading them as an empty queue.
|
|
3049
|
+
*/
|
|
3050
|
+
export function summarizeHeldPass(settled: number, completedAt = Date.now()): DispatchSummary {
|
|
3051
|
+
return {
|
|
3052
|
+
completedAt,
|
|
3053
|
+
ready: 0,
|
|
3054
|
+
routed: 0,
|
|
3055
|
+
claimed: 0,
|
|
3056
|
+
admitted: 0,
|
|
3057
|
+
degraded: false,
|
|
3058
|
+
holds: [],
|
|
3059
|
+
settled,
|
|
3060
|
+
paused: true,
|
|
2633
3061
|
};
|
|
2634
3062
|
}
|
|
2635
3063
|
|
|
@@ -2717,7 +3145,7 @@ function planUsageEscalation(project: string, plan: PlanUsageStatus): Escalation
|
|
|
2717
3145
|
* that grows a field has no business breaking these tests.
|
|
2718
3146
|
*/
|
|
2719
3147
|
export async function admitCandidates(
|
|
2720
|
-
d: Pick<Deps, "project" | "caps" | "tracker" | "store" | "escalate" | "usage">,
|
|
3148
|
+
d: Pick<Deps, "project" | "caps" | "tracker" | "store" | "escalate" | "usage" | "probeCriticalBase">,
|
|
2721
3149
|
routed: Routed[],
|
|
2722
3150
|
slots: number,
|
|
2723
3151
|
): Promise<AdmissionPass> {
|
|
@@ -2837,7 +3265,7 @@ export async function admitCandidates(
|
|
|
2837
3265
|
|
|
2838
3266
|
const priorRuns = store.attemptsFor(project.name, issue);
|
|
2839
3267
|
const failures = store.failuresFor(project.name, issue);
|
|
2840
|
-
if (failures
|
|
3268
|
+
if (!hasFailedAttemptBudget(failures, caps.maxAttemptsPerIssue)) {
|
|
2841
3269
|
hold(issue, "failed-attempts");
|
|
2842
3270
|
await safeEscalate(d, {
|
|
2843
3271
|
tier: 1,
|
|
@@ -2897,6 +3325,81 @@ export async function admitCandidates(
|
|
|
2897
3325
|
continue;
|
|
2898
3326
|
}
|
|
2899
3327
|
|
|
3328
|
+
// #428 half (a): a preserved continuation that predates a configured
|
|
3329
|
+
// critical-base/safety marker must not be reattached. A base safety fix
|
|
3330
|
+
// protects only branches forked after it landed — a continuation forked
|
|
3331
|
+
// before it still carries the dangerous test/runtime code, and re-running
|
|
3332
|
+
// it on the shared host is what SIGTERMed the production daemon. Fail
|
|
3333
|
+
// closed: only a probe that proves every marker is in the reattach
|
|
3334
|
+
// source's ancestry admits, and a project that names a marker but has no
|
|
3335
|
+
// probe wired (never happens outside tests) holds. Both the hold and the
|
|
3336
|
+
// escalation are durable across restart and orphan recovery because this
|
|
3337
|
+
// gate runs every admission pass; the branch is re-admitted automatically
|
|
3338
|
+
// once the operator updates it to contain the marker, without losing work.
|
|
3339
|
+
const markers = project.criticalBase ?? [];
|
|
3340
|
+
if (markers.length > 0) {
|
|
3341
|
+
const branch = branchName(r.issue);
|
|
3342
|
+
let verdict: CriticalBaseVerdict;
|
|
3343
|
+
if (d.probeCriticalBase === undefined) {
|
|
3344
|
+
verdict = { state: "unknown", error: "no critical-base probe is wired in this deployment" };
|
|
3345
|
+
} else {
|
|
3346
|
+
try {
|
|
3347
|
+
verdict = await d.probeCriticalBase(r.repo, markers, branch);
|
|
3348
|
+
} catch (err) {
|
|
3349
|
+
verdict = { state: "unknown", error: errText(err) };
|
|
3350
|
+
}
|
|
3351
|
+
}
|
|
3352
|
+
if (verdict.state === "stale") {
|
|
3353
|
+
hold(issue, "stale-base");
|
|
3354
|
+
log(
|
|
3355
|
+
`#${issue} held (stale-base): continuation branch ${branch} predates critical-base marker ${verdict.marker}`,
|
|
3356
|
+
);
|
|
3357
|
+
await safeEscalate(d, {
|
|
3358
|
+
tier: 1,
|
|
3359
|
+
project: project.name,
|
|
3360
|
+
issue,
|
|
3361
|
+
summary: `#${issue} continuation branch predates a critical base safety commit and is held (stale-base)`,
|
|
3362
|
+
detail: [
|
|
3363
|
+
r.issue.title,
|
|
3364
|
+
r.issue.url,
|
|
3365
|
+
`The retained branch ${branch} does not contain critical-base marker ${verdict.marker}.`,
|
|
3366
|
+
...(verdict.range.length > 0
|
|
3367
|
+
? [`Base commits the branch is missing: ${verdict.range.join(", ")}`]
|
|
3368
|
+
: []),
|
|
3369
|
+
"Recovery: merge current base into the branch so it contains the marker, and the next",
|
|
3370
|
+
"admission pass re-admits it automatically without losing the branch's work; or review",
|
|
3371
|
+
"the branch by hand and clear the hold once the fix is present.",
|
|
3372
|
+
].join("\n"),
|
|
3373
|
+
});
|
|
3374
|
+
continue;
|
|
3375
|
+
}
|
|
3376
|
+
if (verdict.state === "unknown") {
|
|
3377
|
+
// Fail closed: a branch that cannot be *proven* to contain the marker
|
|
3378
|
+
// is refused, and the reason names the unverifiable marker so the
|
|
3379
|
+
// operator can fix the fetch or the marker rather than guess.
|
|
3380
|
+
hold(issue, "stale-base");
|
|
3381
|
+
log(
|
|
3382
|
+
`#${issue} held (stale-base): continuation branch ${branch} could not be verified ` +
|
|
3383
|
+
`against critical-base marker(s) ${markers.join(", ")} (${verdict.error})`,
|
|
3384
|
+
);
|
|
3385
|
+
await safeEscalate(d, {
|
|
3386
|
+
tier: 1,
|
|
3387
|
+
project: project.name,
|
|
3388
|
+
issue,
|
|
3389
|
+
summary: `#${issue} continuation branch could not be verified against a critical base safety commit and is held (stale-base)`,
|
|
3390
|
+
detail: [
|
|
3391
|
+
r.issue.title,
|
|
3392
|
+
r.issue.url,
|
|
3393
|
+
`The retained branch ${branch} could not be verified against critical-base marker(s) ${markers.join(", ")}: ${verdict.error}`,
|
|
3394
|
+
"Recovery: merge current base into the branch so it contains the marker, and the next",
|
|
3395
|
+
"admission pass re-admits it automatically without losing the branch's work; or review",
|
|
3396
|
+
"the branch by hand and clear the hold once the fix is present.",
|
|
3397
|
+
].join("\n"),
|
|
3398
|
+
});
|
|
3399
|
+
continue;
|
|
3400
|
+
}
|
|
3401
|
+
}
|
|
3402
|
+
|
|
2900
3403
|
// Soft concurrency per epic, per repository: at most one in-flight child of
|
|
2901
3404
|
// a given parent in each repo. Children of one epic in *different* repos
|
|
2902
3405
|
// parallelise freely — `repo-active` / `maxConcurrentWorkersPerRepo` owns
|
|
@@ -3103,6 +3606,70 @@ export function wakeOrchestratorForMetConditions(
|
|
|
3103
3606
|
}
|
|
3104
3607
|
}
|
|
3105
3608
|
|
|
3609
|
+
|
|
3610
|
+
/**
|
|
3611
|
+
* The first-tick verification for a fleet-initiated upgrade (#486).
|
|
3612
|
+
*
|
|
3613
|
+
* The detached install unit journals its progress and stops, leaving dispatch
|
|
3614
|
+
* paused; whichever daemon comes back after the restart runs this on its
|
|
3615
|
+
* first tick — above every gate that a pause or an integrity mismatch could
|
|
3616
|
+
* close, because the install's own pause is the thing under verification.
|
|
3617
|
+
* A day with no journal costs one file read; only a pending request pays for
|
|
3618
|
+
* the version/health/ticks/pane/doctor checks and the outbox rows.
|
|
3619
|
+
*/
|
|
3620
|
+
export function upgradeVerifyDepsFor(d: Pick<Deps, "project" | "store">): UpgradeVerifyDeps {
|
|
3621
|
+
return {
|
|
3622
|
+
projectName: d.project.name,
|
|
3623
|
+
journal: (entry) => appendJournal(stateDir(), entry),
|
|
3624
|
+
run: runCommand,
|
|
3625
|
+
layers: (project) => fleetLayers(project),
|
|
3626
|
+
health: healthCheck,
|
|
3627
|
+
doctor: (name) => runDoctor(name),
|
|
3628
|
+
enqueue: (draft) => {
|
|
3629
|
+
try {
|
|
3630
|
+
d.store.enqueueReport(draft);
|
|
3631
|
+
} catch (err) {
|
|
3632
|
+
log(`upgrade verification report could not be queued: ${errText(err)}`);
|
|
3633
|
+
}
|
|
3634
|
+
},
|
|
3635
|
+
pause: pauseInstance,
|
|
3636
|
+
resume: (project) => setPaused(false, undefined, project),
|
|
3637
|
+
launchRollback: async (version) => {
|
|
3638
|
+
const launched = await launchTransientUnit(
|
|
3639
|
+
(argv) => runCommand(argv[0]!, argv.slice(1)),
|
|
3640
|
+
process.env,
|
|
3641
|
+
{ kind: "rollback", version },
|
|
3642
|
+
);
|
|
3643
|
+
return launched.ok === true ? { ok: true, unit: launched.unit } : { ok: false, stderr: launched.stderr };
|
|
3644
|
+
},
|
|
3645
|
+
log,
|
|
3646
|
+
now: () => Date.now(),
|
|
3647
|
+
};
|
|
3648
|
+
}
|
|
3649
|
+
|
|
3650
|
+
async function verifyPendingUpgradeTick(
|
|
3651
|
+
d: Deps,
|
|
3652
|
+
run = verifyPendingUpgrade,
|
|
3653
|
+
): Promise<void> {
|
|
3654
|
+
// Cheap pre-test: no journal, no pending request, nothing to touch.
|
|
3655
|
+
if (readUpgradeJournal(stateDir()).length === 0) return;
|
|
3656
|
+
const verifyDeps = upgradeVerifyDepsFor(d);
|
|
3657
|
+
try {
|
|
3658
|
+
const outcome = await run(verifyDeps);
|
|
3659
|
+
if (outcome.handled !== "none" && outcome.handled !== "already-closed") {
|
|
3660
|
+
log(
|
|
3661
|
+
`upgrade journal handled: ${outcome.handled}` +
|
|
3662
|
+
`${outcome.version === undefined ? "" : ` for omp-conductor@${outcome.version}`}` +
|
|
3663
|
+
`${outcome.detail === undefined ? "" : ` — ${outcome.detail}`}`,
|
|
3664
|
+
);
|
|
3665
|
+
}
|
|
3666
|
+
} catch (err) {
|
|
3667
|
+
// A torn journal must not take the tick down: the operator still reads it
|
|
3668
|
+
// through `status`, and the next tick retries the same pass.
|
|
3669
|
+
log(`upgrade verification pass failed: ${errText(err)}`);
|
|
3670
|
+
}
|
|
3671
|
+
}
|
|
3672
|
+
|
|
3106
3673
|
export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
3107
3674
|
// A config edit takes effect on the next tick, not the next daemon restart
|
|
3108
3675
|
// (#170). Re-resolve the project and its caps at the tick boundary so a tick
|
|
@@ -3124,6 +3691,16 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
3124
3691
|
d.deliveryPolicyValid = false;
|
|
3125
3692
|
}
|
|
3126
3693
|
|
|
3694
|
+
// The fleet-installs-itself transaction (#486): the detached install unit
|
|
3695
|
+
// paused dispatch and stopped; the first tick after the restart verifies
|
|
3696
|
+
// against its journal and closes or rolls back. Above the pause gate on
|
|
3697
|
+
// purpose — the install's own pause is exactly the state being verified.
|
|
3698
|
+
try {
|
|
3699
|
+
await (d.upgradeVerifier ?? verifyPendingUpgradeTick)(d);
|
|
3700
|
+
} catch (err) {
|
|
3701
|
+
log(`upgrade journal pass failed: ${errText(err)}`);
|
|
3702
|
+
}
|
|
3703
|
+
|
|
3127
3704
|
// Availability-held notices are already durable. Once the freshly reloaded
|
|
3128
3705
|
// policy opens (or newly allows their category), atomically hand a bounded
|
|
3129
3706
|
// batch to the report outbox. Pauses do not suppress delivery.
|
|
@@ -3150,8 +3727,10 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
3150
3727
|
// Settlement is maintenance, not dispatch. Run it before every gate that can
|
|
3151
3728
|
// stop claiming — pause, integrity, spend, and capacity — so status converges
|
|
3152
3729
|
// while the fleet is parked or workers are still active. Resident workers run
|
|
3153
|
-
// through the pool without blocking this five-minute tick.
|
|
3154
|
-
|
|
3730
|
+
// through the pool without blocking this five-minute tick. The return value
|
|
3731
|
+
// is what this pass settled; a held pass records it so the paused state reads
|
|
3732
|
+
// as work done, not just a clock that moved (#497).
|
|
3733
|
+
let settled = await settlePushedGreen(d);
|
|
3155
3734
|
try {
|
|
3156
3735
|
await watchMergedBase(d);
|
|
3157
3736
|
} catch (err) {
|
|
@@ -3177,7 +3756,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
3177
3756
|
// Each is guarded on its own: a tracker that fails mid-classification must not
|
|
3178
3757
|
// stop the label reconcile, and neither may stop the tick.
|
|
3179
3758
|
try {
|
|
3180
|
-
await classifyAndRecover(d);
|
|
3759
|
+
settled += await classifyAndRecover(d);
|
|
3181
3760
|
} catch (err) {
|
|
3182
3761
|
log(`classification sweep failed: ${errText(err)}`);
|
|
3183
3762
|
}
|
|
@@ -3228,8 +3807,15 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
3228
3807
|
});
|
|
3229
3808
|
|
|
3230
3809
|
// A paused fleet claims nothing. Checked first so pausing takes effect on the
|
|
3231
|
-
// next tick without signalling the process.
|
|
3232
|
-
|
|
3810
|
+
// next tick without signalling the process. But the pass still ran, and the
|
|
3811
|
+
// operator has to be able to see it: record it as a held pass — the work the
|
|
3812
|
+
// maintenance sweeps settled above this gate, and nothing admitted below it —
|
|
3813
|
+
// so `last dispatch` keeps moving while the fleet is deliberately parked. A
|
|
3814
|
+
// frozen clock is then a stall report, not a hold (#497).
|
|
3815
|
+
if (isPaused(d.project.name)) {
|
|
3816
|
+
d.store.recordDispatch(d.project.name, summarizeHeldPass(settled));
|
|
3817
|
+
return;
|
|
3818
|
+
}
|
|
3233
3819
|
|
|
3234
3820
|
const { project, caps, store } = d;
|
|
3235
3821
|
|
|
@@ -3319,7 +3905,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
3319
3905
|
const recordDispatch = (admitted: number, holds: readonly AdmissionHold[]): void => {
|
|
3320
3906
|
store.recordDispatch(
|
|
3321
3907
|
project.name,
|
|
3322
|
-
summarizeDispatch(ready.length, routed.length, claimed, admitted, holds),
|
|
3908
|
+
summarizeDispatch(ready.length, routed.length, claimed, admitted, holds, Date.now(), settled),
|
|
3323
3909
|
);
|
|
3324
3910
|
};
|
|
3325
3911
|
|
|
@@ -3577,7 +4163,7 @@ export async function turnLimitResponse(
|
|
|
3577
4163
|
export async function workerControlResponse(
|
|
3578
4164
|
req: Request,
|
|
3579
4165
|
project: string,
|
|
3580
|
-
store: Pick<Store, "latestRun">,
|
|
4166
|
+
store: Pick<Store, "latestRun" | "updateRun">,
|
|
3581
4167
|
registry: WorkerControlRegistry,
|
|
3582
4168
|
): Promise<Response | undefined> {
|
|
3583
4169
|
const url = new URL(req.url);
|
|
@@ -3671,6 +4257,28 @@ export async function workerControlResponse(
|
|
|
3671
4257
|
state: latest.state,
|
|
3672
4258
|
});
|
|
3673
4259
|
}
|
|
4260
|
+
// A live-state row whose controller is absent from this process is a ghost
|
|
4261
|
+
// (#431): a session controller lives in the daemon that launched it, and once
|
|
4262
|
+
// ownership is exclusive (the once lease makes a drill hold the same pidfile
|
|
4263
|
+
// as the long dispatcher), a row that is live but has no controller in *this*
|
|
4264
|
+
// owner is a row whose owner already died — not work another live daemon is
|
|
4265
|
+
// still writing to. Refuse nothing: recover it as an orphan (worktree kept
|
|
4266
|
+
// for triage, labels untouched, live slot freed) so the issue can be
|
|
4267
|
+
// dispatched again and the row stops counting as a live worker.
|
|
4268
|
+
if (action === "stop") {
|
|
4269
|
+
store.updateRun(latest.id, {
|
|
4270
|
+
state: "orphaned",
|
|
4271
|
+
endedAt: Date.now(),
|
|
4272
|
+
lastError: reason,
|
|
4273
|
+
});
|
|
4274
|
+
return Response.json({
|
|
4275
|
+
outcome: "stopped",
|
|
4276
|
+
runId: latest.id,
|
|
4277
|
+
state: "orphaned",
|
|
4278
|
+
reason: reason!,
|
|
4279
|
+
worktree: latest.worktree,
|
|
4280
|
+
});
|
|
4281
|
+
}
|
|
3674
4282
|
return Response.json(
|
|
3675
4283
|
{
|
|
3676
4284
|
error:
|
|
@@ -3683,7 +4291,7 @@ export async function workerControlResponse(
|
|
|
3683
4291
|
|
|
3684
4292
|
export interface DaemonHttpProjectDeps {
|
|
3685
4293
|
project: string;
|
|
3686
|
-
store: Pick<Store, "latestRun" | "setTurnOverride">;
|
|
4294
|
+
store: Pick<Store, "latestRun" | "setTurnOverride" | "updateRun">;
|
|
3687
4295
|
caps: () => Pick<Caps, "workerMaxTurns" | "workerMaxTurnsCeiling">;
|
|
3688
4296
|
}
|
|
3689
4297
|
|
|
@@ -3692,21 +4300,32 @@ export interface DaemonHttpDeps {
|
|
|
3692
4300
|
turnLimits: TurnLimitRegistry;
|
|
3693
4301
|
workerControls: WorkerControlRegistry;
|
|
3694
4302
|
health: () => DaemonHealth;
|
|
4303
|
+
/** Request an immediate dispatch pass (`resume`'s wake, #380). */
|
|
4304
|
+
wake: () => void;
|
|
3695
4305
|
}
|
|
3696
4306
|
|
|
3697
4307
|
/**
|
|
3698
4308
|
* The whole HTTP surface, in one named function so a test can pin what is *not*
|
|
3699
4309
|
* on it.
|
|
3700
4310
|
*
|
|
3701
|
-
*
|
|
3702
|
-
* pause/resume/stop control. Everything else is 404. The
|
|
3703
|
-
* daemon-owned live sessions; tracker and repository
|
|
3704
|
-
* authenticated per-run channel described at the
|
|
4311
|
+
* Four route families: the health read, the dispatch wake, turn-limit control,
|
|
4312
|
+
* and live-worker pause/resume/stop control. Everything else is 404. The
|
|
4313
|
+
* controls mutate only daemon-owned live sessions; tracker and repository
|
|
4314
|
+
* mutations stay on the authenticated per-run channel described at the
|
|
4315
|
+
* `Bun.serve` call (#126).
|
|
3705
4316
|
*/
|
|
3706
4317
|
export async function daemonHttpResponse(req: Request, d: DaemonHttpDeps): Promise<Response> {
|
|
3707
4318
|
const url = new URL(req.url);
|
|
3708
4319
|
if (req.method === "GET" && url.pathname === "/healthz") return Response.json(d.health());
|
|
3709
4320
|
|
|
4321
|
+
if (req.method === "POST" && url.pathname === "/wake") {
|
|
4322
|
+
// A resume wake: daemon-wide and loopback-only, like /healthz. The pass it
|
|
4323
|
+
// produces re-reads every project's pause file, so a project that is still
|
|
4324
|
+
// held stays held — the wake shortens the wait, it never bypasses a gate.
|
|
4325
|
+
d.wake();
|
|
4326
|
+
return new Response(null, { status: 204 });
|
|
4327
|
+
}
|
|
4328
|
+
|
|
3710
4329
|
let selected = d.projects[0];
|
|
3711
4330
|
if (
|
|
3712
4331
|
req.method === "PUT" &&
|
|
@@ -3903,21 +4522,31 @@ export function formatDispatchSummary(summary?: DispatchSummary): string {
|
|
|
3903
4522
|
if (summary === undefined) return "last dispatch (none recorded)";
|
|
3904
4523
|
const lines = [
|
|
3905
4524
|
`last dispatch ${new Date(summary.completedAt).toISOString()}${summary.degraded ? " DEGRADED" : ""}`,
|
|
3906
|
-
` candidates ${summary.ready} ready / ${summary.claimed ?? 0} in flight / ${summary.routed} spare`,
|
|
3907
|
-
` admitted ${summary.admitted}`,
|
|
3908
4525
|
];
|
|
3909
|
-
if (summary.
|
|
3910
|
-
|
|
4526
|
+
if (summary.paused === true) {
|
|
4527
|
+
// A held pass never routed the queue: printing zero candidates here would
|
|
4528
|
+
// read as "the queue was empty". Say what the pass was and attribute what
|
|
4529
|
+
// it did instead (#497).
|
|
4530
|
+
lines.push(` pass held — nothing admitted`);
|
|
3911
4531
|
} else {
|
|
3912
|
-
lines.push(
|
|
3913
|
-
|
|
3914
|
-
|
|
3915
|
-
|
|
3916
|
-
|
|
3917
|
-
|
|
3918
|
-
|
|
4532
|
+
lines.push(
|
|
4533
|
+
` candidates ${summary.ready} ready / ${summary.claimed ?? 0} in flight / ${summary.routed} spare`,
|
|
4534
|
+
` admitted ${summary.admitted}`,
|
|
4535
|
+
);
|
|
4536
|
+
if (summary.holds.length === 0) {
|
|
4537
|
+
lines.push(" held 0");
|
|
4538
|
+
} else {
|
|
4539
|
+
lines.push(" held");
|
|
4540
|
+
for (const hold of summary.holds) {
|
|
4541
|
+
const sample =
|
|
4542
|
+
hold.issues.length === 0
|
|
4543
|
+
? ""
|
|
4544
|
+
: ` (#${hold.issues.join(", #")}${hold.count > hold.issues.length ? ", …" : ""})`;
|
|
4545
|
+
lines.push(` ${hold.reason} ${hold.count}${sample}`);
|
|
4546
|
+
}
|
|
3919
4547
|
}
|
|
3920
4548
|
}
|
|
4549
|
+
if ((summary.settled ?? 0) > 0) lines.push(` settled ${summary.settled}`);
|
|
3921
4550
|
return lines.join("\n");
|
|
3922
4551
|
}
|
|
3923
4552
|
|
|
@@ -4237,8 +4866,12 @@ export function readSessionError(sessionFile: string | undefined): SessionError
|
|
|
4237
4866
|
* it is healthy, and writing a class onto it would take it out of this sweep for
|
|
4238
4867
|
* good.
|
|
4239
4868
|
*/
|
|
4240
|
-
export async function classifyAndRecover(d: Deps): Promise<
|
|
4241
|
-
const { project, tracker, store } = d;
|
|
4869
|
+
export async function classifyAndRecover(d: Deps): Promise<number> {
|
|
4870
|
+
const { project, caps, tracker, store } = d;
|
|
4871
|
+
// Settle recoveries, counted for the pass's dispatch record — a row whose PR
|
|
4872
|
+
// merged is settled here when the settle sweep could not establish identity
|
|
4873
|
+
// (#497). Every other exit returns 0.
|
|
4874
|
+
let settled = 0;
|
|
4242
4875
|
for (const run of store.runsNeedingClassification(project.name, CLASSIFY_BATCH)) {
|
|
4243
4876
|
const facts: ClassifyFacts = {};
|
|
4244
4877
|
let classifiedRun = run;
|
|
@@ -4280,7 +4913,7 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
|
|
|
4280
4913
|
continue;
|
|
4281
4914
|
}
|
|
4282
4915
|
|
|
4283
|
-
const { cls, recovery, evidence } = classifyRun(classifiedRun, facts);
|
|
4916
|
+
const { cls, recovery, evidence } = classifyRun(classifiedRun, facts, caps);
|
|
4284
4917
|
|
|
4285
4918
|
// A healthy green PR is not a failure of any class. Leaving the row
|
|
4286
4919
|
// unclassified is what keeps it eligible for the sweep on the tick where its
|
|
@@ -4294,8 +4927,10 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
|
|
|
4294
4927
|
? `#${run.issue} retrying ${recovery} for ${cls}: ${evidence}`
|
|
4295
4928
|
: `#${run.issue} classified ${cls} → ${recovery}: ${evidence}`,
|
|
4296
4929
|
);
|
|
4930
|
+
if (recovery === "settle") settled += 1;
|
|
4297
4931
|
await recoverRun(d, classifiedRun, cls, recovery, evidence);
|
|
4298
4932
|
}
|
|
4933
|
+
return settled;
|
|
4299
4934
|
}
|
|
4300
4935
|
|
|
4301
4936
|
/** Performs the one action a class names. Never chooses one of its own. */
|
|
@@ -4306,7 +4941,7 @@ async function recoverRun(
|
|
|
4306
4941
|
recovery: RecoveryAction,
|
|
4307
4942
|
evidence: string,
|
|
4308
4943
|
): Promise<void> {
|
|
4309
|
-
const { project, tracker, store } = d;
|
|
4944
|
+
const { project, caps, tracker, store } = d;
|
|
4310
4945
|
const inProgress = project.stateLabels.inProgress;
|
|
4311
4946
|
|
|
4312
4947
|
if (recovery === "settle") {
|
|
@@ -4337,6 +4972,55 @@ async function recoverRun(
|
|
|
4337
4972
|
return;
|
|
4338
4973
|
}
|
|
4339
4974
|
|
|
4975
|
+
// `wall-clock-cap-progress`: the completion path only auto-continues
|
|
4976
|
+
// turn-cap kills, so a wall-clock kill with work to show reaches this
|
|
4977
|
+
// sweep still holding the failed label. Swap it for the queue — the branch
|
|
4978
|
+
// is retained, so the next dispatch reattaches it and briefs a resume from
|
|
4979
|
+
// the recorded work. Killed rows gathered no tracker facts, so the
|
|
4980
|
+
// issue-open guard mirrors the requeue path's. The continuation gate is
|
|
4981
|
+
// the same one the turns path applies at kill time
|
|
4982
|
+
// (`shouldContinueAfterTurnsCap`): the row is already charged, and once
|
|
4983
|
+
// the ceiling is spent, handing back the queue label would offer a
|
|
4984
|
+
// candidate admission can never accept — only the failed label comes off,
|
|
4985
|
+
// and the exhaustion reaches a human (#490, #348).
|
|
4986
|
+
if (cls === "wall-clock-cap-progress") {
|
|
4987
|
+
const state = await tracker.issueState(run.issue).catch(() => undefined);
|
|
4988
|
+
if (state !== "open") {
|
|
4989
|
+
log(`#${run.issue} not continued from ${cls}: issue is ${state ?? "unreadable"}`);
|
|
4990
|
+
return;
|
|
4991
|
+
}
|
|
4992
|
+
const continuation = store.continuationsFor(project.name, run.issue);
|
|
4993
|
+
if (hasContinuationBudget(continuation, caps.maxContinuationsPerIssue)) {
|
|
4994
|
+
swapToQueue(d, run.issue, project.stateLabels.failed);
|
|
4995
|
+
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
4996
|
+
log(`#${run.issue} requeued for a wall-clock continuation: ${evidence}`);
|
|
4997
|
+
} else {
|
|
4998
|
+
store.enqueueLabelOps(project.name, [
|
|
4999
|
+
{ issue: run.issue, op: "remove", label: project.stateLabels.failed },
|
|
5000
|
+
]);
|
|
5001
|
+
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
5002
|
+
await safeEscalate(d, {
|
|
5003
|
+
tier: 1,
|
|
5004
|
+
project: project.name,
|
|
5005
|
+
issue: run.issue,
|
|
5006
|
+
summary: `#${run.issue} exhausted its continuation budget on wall-clock cap kills`,
|
|
5007
|
+
detail: [
|
|
5008
|
+
`Attempt ${run.attempt} hit the wall-clock cap with work to continue from, but ${continuation} continuation(s) are already charged against a cap of ${caps.maxContinuationsPerIssue}.`,
|
|
5009
|
+
evidence,
|
|
5010
|
+
`Work to continue: branch ${run.branch} at ${run.headSha ?? run.salvageSha}${run.prUrl === undefined ? "" : ` — ${run.prUrl}`}.`,
|
|
5011
|
+
"The issue cannot finish inside the wall-clock cap, so another run would burn a worker slot for the same outcome. Re-scope it, raise maxContinuationsPerIssue for it, or finish the remaining work by hand.",
|
|
5012
|
+
].join("\n"),
|
|
5013
|
+
});
|
|
5014
|
+
await postExhaustionPostmortem(
|
|
5015
|
+
d,
|
|
5016
|
+
run,
|
|
5017
|
+
`Attempt ${run.attempt} hit the wall-clock cap with work to continue from, but ${continuation} continuation(s) are already charged against a cap of ${caps.maxContinuationsPerIssue}.`,
|
|
5018
|
+
);
|
|
5019
|
+
log(`#${run.issue} not continued from ${cls}: continuation budget exhausted`);
|
|
5020
|
+
}
|
|
5021
|
+
return;
|
|
5022
|
+
}
|
|
5023
|
+
|
|
4340
5024
|
// `merge-conflict`: the branch is retained and its PR is open, so #50's
|
|
4341
5025
|
// continuation guard admits it and the next tick briefs a rebase.
|
|
4342
5026
|
//
|
|
@@ -4386,11 +5070,14 @@ async function recoverRun(
|
|
|
4386
5070
|
// Same bound for provider-transient: an issue whose stream keeps stalling
|
|
4387
5071
|
// mid-run is requeued free (no attempt, no continuation charged) — but a
|
|
4388
5072
|
// provider that aborts three times for one issue is down, and a human has
|
|
4389
|
-
// to check its status before hand-requeueing (#220).
|
|
5073
|
+
// to check its status before hand-requeueing (#220). The escalation names
|
|
5074
|
+
// every model the chain tried, so a merged branch built on a different
|
|
5075
|
+
// model is attributable (#286).
|
|
4390
5076
|
if (
|
|
4391
5077
|
cls === "provider-transient" &&
|
|
4392
5078
|
store.classCountFor(project.name, run.issue, "provider-transient") >= PROVIDER_TRANSIENT_MAX_STRIKES
|
|
4393
5079
|
) {
|
|
5080
|
+
const tried = formatModelsTried(modelsTried(store.runsForIssue(project.name, run.issue)));
|
|
4394
5081
|
await safeEscalate(d, {
|
|
4395
5082
|
tier: 1,
|
|
4396
5083
|
project: project.name,
|
|
@@ -4398,6 +5085,9 @@ async function recoverRun(
|
|
|
4398
5085
|
summary: `[provider-transient] #${run.issue}: the provider keeps aborting mid-stream — ${evidence}`,
|
|
4399
5086
|
detail: [
|
|
4400
5087
|
`The provider aborted the stream for #${run.issue} ${PROVIDER_TRANSIENT_MAX_STRIKES} times without the run ever producing a verdict (0 tokens billed each time).`,
|
|
5088
|
+
...(tried === ""
|
|
5089
|
+
? []
|
|
5090
|
+
: [`Models tried: ${tried}.`]),
|
|
4401
5091
|
"Check provider status before requeueing by hand.",
|
|
4402
5092
|
].join("\n"),
|
|
4403
5093
|
});
|
|
@@ -4413,6 +5103,73 @@ async function recoverRun(
|
|
|
4413
5103
|
log(`#${run.issue} not requeued from ${cls}: issue is ${state ?? "unreadable"}`);
|
|
4414
5104
|
return;
|
|
4415
5105
|
}
|
|
5106
|
+
// A clean orphan whose queue label is already absent is a deliberate
|
|
5107
|
+
// withdrawal — the operator took the issue out of the queue (e.g. so a
|
|
5108
|
+
// daemon restart could not re-dispatch work known to be unsafe) — and
|
|
5109
|
+
// recovery must not recreate that intent (#423). Read the live label set;
|
|
5110
|
+
// when the queue label is gone, release the dispatcher-owned in-progress
|
|
5111
|
+
// label and stop, leaving the operator's withdrawal to survive recovery.
|
|
5112
|
+
if (cls === "orphan-clean") {
|
|
5113
|
+
const snapshot = await tracker.issueSnapshot(run.issue).catch(() => undefined);
|
|
5114
|
+
if (snapshot === undefined) {
|
|
5115
|
+
log(`#${run.issue} not requeued from orphan-clean: cannot confirm ${project.queueLabel} (unreadable, retrying)`);
|
|
5116
|
+
return;
|
|
5117
|
+
}
|
|
5118
|
+
if (!snapshot.labels.includes(project.queueLabel)) {
|
|
5119
|
+
store.enqueueLabelOps(project.name, [{ issue: run.issue, op: "remove", label: inProgress }]);
|
|
5120
|
+
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
5121
|
+
log(`#${run.issue} not requeued from orphan-clean: ${project.queueLabel} removed before recovery (operator withdrawal)`);
|
|
5122
|
+
return;
|
|
5123
|
+
}
|
|
5124
|
+
// #439: an orphan-clean row charges the continuation budget, so once
|
|
5125
|
+
// `hasContinuationBudget` is spent — exactly the predicate `admitCandidates`
|
|
5126
|
+
// holds the issue on — requeueing re-adds a queue label for a candidate the
|
|
5127
|
+
// dispatcher can never admit. Stop handing it back and hold it instead: the
|
|
5128
|
+
// queue label comes off (so admission never re-holds on every dispatch),
|
|
5129
|
+
// the in-progress label is released, and a single diagnosis escalates once.
|
|
5130
|
+
// `orphan-clean` is deliberately NOT a global exclusion from the budget (a
|
|
5131
|
+
// worker that genuinely keeps dying mid-work must still be bounded); this is
|
|
5132
|
+
// the missing ceiling check this path never had (#348's invariant).
|
|
5133
|
+
const continuations = store.continuationsFor(project.name, run.issue);
|
|
5134
|
+
if (!hasContinuationBudget(continuations, caps.maxContinuationsPerIssue)) {
|
|
5135
|
+
const runs = store.runsForIssue(project.name, run.issue);
|
|
5136
|
+
const breakdown = continuationBreakdown(runs);
|
|
5137
|
+
const artifact = newestContinuableRun(runs);
|
|
5138
|
+
const onlyDaemonStops = breakdown.size === 1 && breakdown.get("orphan-clean") === continuations;
|
|
5139
|
+
const classLine = Array.from(breakdown, ([cls, n]) => `${n} ${cls}`).join(", ");
|
|
5140
|
+
const artifactLine =
|
|
5141
|
+
artifact === undefined
|
|
5142
|
+
? "The attempts left no salvage commit, head SHA or PR — the branch is empty, so start clean from a re-scope rather than continuing from nothing."
|
|
5143
|
+
: `Work to continue: branch ${artifact.branch} at ${artifact.headSha ?? artifact.salvageSha}${artifact.prUrl === undefined ? "" : ` — ${artifact.prUrl}`}.`;
|
|
5144
|
+
store.enqueueLabelOps(project.name, [
|
|
5145
|
+
{ issue: run.issue, op: "remove", label: inProgress },
|
|
5146
|
+
{ issue: run.issue, op: "remove", label: project.queueLabel },
|
|
5147
|
+
]);
|
|
5148
|
+
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
5149
|
+
await safeEscalate(d, {
|
|
5150
|
+
tier: 1,
|
|
5151
|
+
project: project.name,
|
|
5152
|
+
issue: run.issue,
|
|
5153
|
+
summary: `#${run.issue} exhausted its ${caps.maxContinuationsPerIssue}-continuation budget on ${cls}`,
|
|
5154
|
+
detail: [
|
|
5155
|
+
`Attempt ${run.attempt} is not requeued: ${continuations} continuation(s) charged against a cap of ${caps.maxContinuationsPerIssue}.`,
|
|
5156
|
+
`Continuations by failure class: ${classLine}.`,
|
|
5157
|
+
onlyDaemonStops
|
|
5158
|
+
? "Every continuation was a daemon stop — the work never failed; the budget was spent by daemon deaths, not the issue."
|
|
5159
|
+
: "Continuations span real work — inspect what each attempt left behind before continuing.",
|
|
5160
|
+
artifactLine,
|
|
5161
|
+
"What you can do: raise maxContinuationsPerIssue for this issue, re-scope it, or continue from the preserved work (or start clean if none).",
|
|
5162
|
+
].join("\n"),
|
|
5163
|
+
});
|
|
5164
|
+
await postExhaustionPostmortem(
|
|
5165
|
+
d,
|
|
5166
|
+
run,
|
|
5167
|
+
`Attempt ${run.attempt} is not requeued: ${continuations} continuation(s) charged against a cap of ${caps.maxContinuationsPerIssue}.`,
|
|
5168
|
+
);
|
|
5169
|
+
log(`#${run.issue} not requeued from orphan-clean: continuation budget exhausted`);
|
|
5170
|
+
return;
|
|
5171
|
+
}
|
|
5172
|
+
}
|
|
4416
5173
|
const label = cls === "orphan-clean" ? inProgress : project.stateLabels.failed;
|
|
4417
5174
|
swapToQueue(d, run.issue, label);
|
|
4418
5175
|
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
@@ -4431,14 +5188,14 @@ async function recoverRun(
|
|
|
4431
5188
|
// Back to pending rather than green: the existing settle sweep re-verifies
|
|
4432
5189
|
// it against the recorded head on a later tick, so nothing here has to guess
|
|
4433
5190
|
// whether the re-run passed.
|
|
4434
|
-
store.updateRun(run.id, { state: "pushed-pending", lastError:
|
|
5191
|
+
store.updateRun(run.id, { state: "pushed-pending", lastError: null, recoveredAt: Date.now() });
|
|
4435
5192
|
log(`#${run.issue} re-ran infrastructure checks: ${evidence}`);
|
|
4436
5193
|
return;
|
|
4437
5194
|
}
|
|
4438
5195
|
|
|
4439
5196
|
if (recovery === "escalate") {
|
|
4440
5197
|
const detail = [evidence];
|
|
4441
|
-
if (cls === "turn-cap-spinning") {
|
|
5198
|
+
if (cls === "turn-cap-spinning" || cls === "wall-clock-cap-spinning") {
|
|
4442
5199
|
const calls = lastToolCalls(run.sessionFile);
|
|
4443
5200
|
detail.push(
|
|
4444
5201
|
calls.length === 0
|
|
@@ -4669,7 +5426,112 @@ function orchestratorStandingOrders(project: ProjectConfig): {
|
|
|
4669
5426
|
};
|
|
4670
5427
|
}
|
|
4671
5428
|
|
|
5429
|
+
/**
|
|
5430
|
+
* Pacing for the resident dispatch loop, interruptible so an operator action —
|
|
5431
|
+
* `resume` — does not wait out the full interval (#380).
|
|
5432
|
+
*
|
|
5433
|
+
* `requestWake` is sticky until a pass absorbs it: while the long sleep is in
|
|
5434
|
+
* flight it is cut short, and when a pass is mid-flight the loop skips the
|
|
5435
|
+
* *following* sleep. Either way the next pass starts without the interval
|
|
5436
|
+
* wait. The single boolean coalesces any number of near-simultaneous wakes
|
|
5437
|
+
* into one prompt pass, and the strictly sequential loop means a wake can
|
|
5438
|
+
* never start a pass on top of one already running.
|
|
5439
|
+
*/
|
|
5440
|
+
export interface DispatchPace {
|
|
5441
|
+
/** Ask for a prompt pass; cuts an in-flight sleep short. Coalesces. */
|
|
5442
|
+
requestWake(): void;
|
|
5443
|
+
/** Absorb a pending request — true when the pass that just finished should
|
|
5444
|
+
* skip its sleep. */
|
|
5445
|
+
takeWake(): boolean;
|
|
5446
|
+
/** The long interval sleep; resolves early when {@link requestWake} fires. */
|
|
5447
|
+
sleep(ms: number): Promise<void>;
|
|
5448
|
+
}
|
|
5449
|
+
|
|
5450
|
+
export function createDispatchPace(): DispatchPace {
|
|
5451
|
+
let pendingWake = false;
|
|
5452
|
+
let interruptSleep: (() => void) | undefined;
|
|
5453
|
+
return {
|
|
5454
|
+
requestWake() {
|
|
5455
|
+
pendingWake = true;
|
|
5456
|
+
interruptSleep?.();
|
|
5457
|
+
},
|
|
5458
|
+
takeWake() {
|
|
5459
|
+
const pending = pendingWake;
|
|
5460
|
+
pendingWake = false;
|
|
5461
|
+
return pending;
|
|
5462
|
+
},
|
|
5463
|
+
async sleep(ms) {
|
|
5464
|
+
// A wake that landed in the gap between the loop's takeWake check and
|
|
5465
|
+
// this call is absorbed here: skip the interval, and consume the flag
|
|
5466
|
+
// so the pass that follows it is the only prompt one.
|
|
5467
|
+
if (pendingWake) {
|
|
5468
|
+
pendingWake = false;
|
|
5469
|
+
return;
|
|
5470
|
+
}
|
|
5471
|
+
await new Promise<void>((resolve) => {
|
|
5472
|
+
const timer = setTimeout(resolve, ms);
|
|
5473
|
+
interruptSleep = () => {
|
|
5474
|
+
// The interruption *is* the absorption: the pass that follows the
|
|
5475
|
+
// sleep is the prompt pass this wake asked for, so the loop sleeps
|
|
5476
|
+
// normally after it instead of running a second immediate pass.
|
|
5477
|
+
pendingWake = false;
|
|
5478
|
+
clearTimeout(timer);
|
|
5479
|
+
resolve();
|
|
5480
|
+
};
|
|
5481
|
+
// A wake can land between the loop's takeWake check and this
|
|
5482
|
+
// constructor; resolve immediately instead of letting it wait out
|
|
5483
|
+
// the full interval.
|
|
5484
|
+
if (pendingWake) interruptSleep();
|
|
5485
|
+
});
|
|
5486
|
+
interruptSleep = undefined;
|
|
5487
|
+
},
|
|
5488
|
+
};
|
|
5489
|
+
}
|
|
5490
|
+
|
|
5491
|
+
export interface DispatchLoopOptions {
|
|
5492
|
+
/** One dispatch pass over every configured project runtime. */
|
|
5493
|
+
tick(): Promise<void>;
|
|
5494
|
+
/** True once a stop is requested; checked between passes. */
|
|
5495
|
+
shouldStop(): boolean;
|
|
5496
|
+
pace: DispatchPace;
|
|
5497
|
+
/** The scheduled interval between passes (`TICK_INTERVAL_MS` in production). */
|
|
5498
|
+
tickIntervalMs: number;
|
|
5499
|
+
}
|
|
5500
|
+
|
|
5501
|
+
/**
|
|
5502
|
+
* The resident daemon's dispatch loop: one pass immediately, then a
|
|
5503
|
+
* `tickIntervalMs` sleep the {@link DispatchPace} can cut short. Exported so
|
|
5504
|
+
* the resume-wake behaviour can be pinned deterministically without a live
|
|
5505
|
+
* daemon (#380).
|
|
5506
|
+
*/
|
|
5507
|
+
export async function runDispatchLoop(o: DispatchLoopOptions): Promise<void> {
|
|
5508
|
+
const { tick, shouldStop, pace, tickIntervalMs } = o;
|
|
5509
|
+
while (!shouldStop()) {
|
|
5510
|
+
await tick();
|
|
5511
|
+
if (shouldStop()) break;
|
|
5512
|
+
// A wake that landed while the pass was in flight skips the following
|
|
5513
|
+
// sleep: that pass made its admission decisions against pre-resume state,
|
|
5514
|
+
// and the resumed project must not wait a full interval for the next one.
|
|
5515
|
+
if (pace.takeWake()) continue;
|
|
5516
|
+
await pace.sleep(tickIntervalMs);
|
|
5517
|
+
}
|
|
5518
|
+
}
|
|
5519
|
+
|
|
4672
5520
|
export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
5521
|
+
// A `--once` tick is still a dispatcher: it settles rows, projects labels,
|
|
5522
|
+
// admits and launches workers, so it must hold the same exclusive daemon/
|
|
5523
|
+
// state ownership as the long-running dispatcher for its whole lifecycle
|
|
5524
|
+
// (#431). This is the one choke point every once tick passes through (both
|
|
5525
|
+
// `daemon --once` and the `setup host` smoke), so ownership is acquired here
|
|
5526
|
+
// — before any settlement, label, admission, worktree or worker mutation —
|
|
5527
|
+
// and refused (having done none of those) while another live daemon owns the
|
|
5528
|
+
// project or another live `--once` holds the lease.
|
|
5529
|
+
const onceLease = o.once ? acquireOnceLease({ port: o.port, project: o.project }) : undefined;
|
|
5530
|
+
if (onceLease !== undefined && !onceLease.ok) {
|
|
5531
|
+
throw new Error(
|
|
5532
|
+
`another daemon is alive (pid ${onceLease.pid}); stop it first — a --once tick refuses to run beside it`,
|
|
5533
|
+
);
|
|
5534
|
+
}
|
|
4673
5535
|
const cfg = loadConfig();
|
|
4674
5536
|
const projects = o.project === undefined ? cfg.projects : [findProject(cfg, o.project)];
|
|
4675
5537
|
const store = openStore(dbPath());
|
|
@@ -4824,6 +5686,8 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
4824
5686
|
integrity,
|
|
4825
5687
|
stall: { paged: false },
|
|
4826
5688
|
cleanup: { next: 0 },
|
|
5689
|
+
probeCriticalBase: (repo, markers, branch) =>
|
|
5690
|
+
probeCriticalBase(project, repo, branch, markers),
|
|
4827
5691
|
...(verbPeerReader === undefined ? {} : { verbPeerReader }),
|
|
4828
5692
|
verbActions,
|
|
4829
5693
|
};
|
|
@@ -4843,6 +5707,10 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
4843
5707
|
await workers.drain();
|
|
4844
5708
|
for (const runtime of runtimes) await runtime.outbox.deliverDue();
|
|
4845
5709
|
} finally {
|
|
5710
|
+
// Drop exclusivity only while it is still ours; a manager that claimed
|
|
5711
|
+
// the record mid-tick is left alone. A lease acquired but never released
|
|
5712
|
+
// (config/lease failure before this branch) is stale recovery's job.
|
|
5713
|
+
if (onceLease !== undefined && onceLease.ok) onceLease.lease.release();
|
|
4846
5714
|
for (const runtime of runtimes.toReversed()) {
|
|
4847
5715
|
await runtime.orchestrator?.dispose();
|
|
4848
5716
|
await runtime.orchestratorVerbs?.close();
|
|
@@ -4891,7 +5759,10 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
4891
5759
|
}, REPORT_DELIVERY_INTERVAL_MS);
|
|
4892
5760
|
|
|
4893
5761
|
let stopping = false;
|
|
4894
|
-
|
|
5762
|
+
// The wake surface (#380): `resume` POSTs /wake, `stop` interrupts the same
|
|
5763
|
+
// sleep. A wake request is sticky and coalescing — several resumes in quick
|
|
5764
|
+
// succession produce one prompt pass, never an overlapping one.
|
|
5765
|
+
const pace = createDispatchPace();
|
|
4895
5766
|
const stop = (): void => {
|
|
4896
5767
|
if (stopping) return;
|
|
4897
5768
|
stopping = true;
|
|
@@ -4901,7 +5772,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
4901
5772
|
// wait for (#374).
|
|
4902
5773
|
drain.draining = true;
|
|
4903
5774
|
log("shutting down after active workers finish");
|
|
4904
|
-
|
|
5775
|
+
pace.requestWake();
|
|
4905
5776
|
};
|
|
4906
5777
|
process.on("SIGINT", stop);
|
|
4907
5778
|
process.on("SIGTERM", stop);
|
|
@@ -4921,6 +5792,10 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
4921
5792
|
})),
|
|
4922
5793
|
turnLimits,
|
|
4923
5794
|
workerControls,
|
|
5795
|
+
wake: () => {
|
|
5796
|
+
log("dispatch wake requested — an immediate pass is starting");
|
|
5797
|
+
pace.requestWake();
|
|
5798
|
+
},
|
|
4924
5799
|
health: () =>
|
|
4925
5800
|
daemonHealth(
|
|
4926
5801
|
runtimes.map((runtime) =>
|
|
@@ -4942,27 +5817,23 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
4942
5817
|
);
|
|
4943
5818
|
|
|
4944
5819
|
try {
|
|
4945
|
-
|
|
4946
|
-
|
|
4947
|
-
|
|
4948
|
-
|
|
4949
|
-
|
|
4950
|
-
|
|
4951
|
-
|
|
4952
|
-
|
|
4953
|
-
|
|
5820
|
+
await runDispatchLoop({
|
|
5821
|
+
tick: async () => {
|
|
5822
|
+
for (const runtime of runtimes) {
|
|
5823
|
+
try {
|
|
5824
|
+
await tick(runtime.d, workers);
|
|
5825
|
+
} catch (err) {
|
|
5826
|
+
log(
|
|
5827
|
+
`${projects.length === 1 ? "" : `[${runtime.d.project.name}] `}` +
|
|
5828
|
+
`tick failed: ${errText(err)}`,
|
|
5829
|
+
);
|
|
5830
|
+
}
|
|
4954
5831
|
}
|
|
4955
|
-
}
|
|
4956
|
-
|
|
4957
|
-
|
|
4958
|
-
|
|
4959
|
-
|
|
4960
|
-
clearTimeout(timer);
|
|
4961
|
-
resolve();
|
|
4962
|
-
};
|
|
4963
|
-
});
|
|
4964
|
-
wake = undefined;
|
|
4965
|
-
}
|
|
5832
|
+
},
|
|
5833
|
+
shouldStop: () => stopping,
|
|
5834
|
+
pace,
|
|
5835
|
+
tickIntervalMs: TICK_INTERVAL_MS,
|
|
5836
|
+
});
|
|
4966
5837
|
} finally {
|
|
4967
5838
|
process.off("SIGINT", stop);
|
|
4968
5839
|
process.off("SIGTERM", stop);
|