omp-conductor 0.10.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +73 -19
- package/package.json +1 -1
- package/src/board.ts +125 -11
- package/src/briefs/orchestrator.md +50 -4
- package/src/briefs/policy.md +4 -1
- package/src/chain-check.ts +157 -0
- package/src/cli.ts +188 -19
- package/src/config.ts +195 -15
- package/src/daemon.ts +1017 -119
- package/src/diff-flags.ts +48 -3
- package/src/digest-schedule.ts +59 -0
- package/src/escalate.ts +17 -0
- package/src/failure-class.ts +31 -4
- package/src/fleet.ts +8 -3
- package/src/gitops.ts +49 -0
- package/src/omp.ts +44 -9
- package/src/orchestrator-tick.ts +94 -9
- package/src/orchestrator.ts +3 -2
- package/src/plugin.ts +46 -4
- package/src/release-policy.ts +66 -6
- package/src/reports.ts +5 -6
- package/src/session-host.ts +23 -6
- package/src/setup.ts +43 -10
- package/src/store.ts +263 -31
- package/src/tracker/github.ts +261 -56
- package/src/types.ts +251 -32
- package/src/verbs/actions.ts +127 -15
- package/src/verbs/protocol.ts +7 -6
- package/src/verbs/server.ts +144 -13
- package/src/worker.ts +183 -17
- package/src/worktree.ts +5 -0
package/src/daemon.ts
CHANGED
|
@@ -31,13 +31,16 @@ import { STALL_MARKER_FILE } from "./orchestrator-tick.ts";
|
|
|
31
31
|
import { startOrchestrator } from "./orchestrator.ts";
|
|
32
32
|
import type { OrchestratorHandle } from "./orchestrator.ts";
|
|
33
33
|
import { createReportOutbox, formatOpenReports } from "./reports.ts";
|
|
34
|
-
import {
|
|
34
|
+
import {
|
|
35
|
+
recordReleaseBlock,
|
|
36
|
+
type ReleaseBlockContext,
|
|
37
|
+
} from "./release-policy.ts";
|
|
35
38
|
import { branchName, effectiveLabels, route } from "./routing.ts";
|
|
36
39
|
import type { Routed, UnroutableReason } from "./routing.ts";
|
|
37
40
|
import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
|
|
38
|
-
import { classifyRun, providerCreditRefusal, type ClassifyFacts } from "./failure-class.ts";
|
|
41
|
+
import { classifyRun, providerCreditRefusal, providerTransientFault, type ClassifyFacts } from "./failure-class.ts";
|
|
39
42
|
import { projectLabels } from "./label-projection.ts";
|
|
40
|
-
import { dbPath, openStore, utcDay } from "./store.ts";
|
|
43
|
+
import { dbPath, LIVE_STATES, openStore, utcDay } from "./store.ts";
|
|
41
44
|
import { makeTracker, type RateLimitStatus } from "./tracker/github.ts";
|
|
42
45
|
import { RELEASE_SHAPES } from "./types.ts";
|
|
43
46
|
import type {
|
|
@@ -45,7 +48,10 @@ import type {
|
|
|
45
48
|
Caps,
|
|
46
49
|
DispatchSummary,
|
|
47
50
|
Escalation,
|
|
51
|
+
IssueSnapshot,
|
|
52
|
+
MergedPrInfo,
|
|
48
53
|
OpenCloser,
|
|
54
|
+
ReleaseShape,
|
|
49
55
|
PrState,
|
|
50
56
|
ProjectConfig,
|
|
51
57
|
ReadyIssue,
|
|
@@ -60,8 +66,17 @@ import type {
|
|
|
60
66
|
Store,
|
|
61
67
|
Tracker,
|
|
62
68
|
VerbLedgerEntry,
|
|
69
|
+
TurnOverride,
|
|
63
70
|
} from "./types.ts";
|
|
64
|
-
import {
|
|
71
|
+
import {
|
|
72
|
+
type KilledBy,
|
|
73
|
+
type WorkerPauseControl,
|
|
74
|
+
type WorkerPausePhase,
|
|
75
|
+
type WorkerResult,
|
|
76
|
+
type RunWorkerDeps,
|
|
77
|
+
renderBrief,
|
|
78
|
+
runWorker,
|
|
79
|
+
} from "./worker.ts";
|
|
65
80
|
import {
|
|
66
81
|
addRunRepo,
|
|
67
82
|
cleanupRetainedWorktree,
|
|
@@ -75,8 +90,13 @@ import {
|
|
|
75
90
|
} from "./worktree.ts";
|
|
76
91
|
import { githubVerbActions } from "./verbs/actions.ts";
|
|
77
92
|
import { formatVerbLedger, STATUS_LEDGER_SCAN } from "./verbs/ledger.ts";
|
|
78
|
-
import {
|
|
79
|
-
|
|
93
|
+
import {
|
|
94
|
+
listenVerbChannel,
|
|
95
|
+
PR_LOOKUP_WINDOW_MS,
|
|
96
|
+
type VerbActions,
|
|
97
|
+
type VerbDeps,
|
|
98
|
+
type VerbListener,
|
|
99
|
+
} from "./verbs/server.ts";
|
|
80
100
|
import {
|
|
81
101
|
ensureVerbSocketDir,
|
|
82
102
|
peerCredentialReader,
|
|
@@ -86,7 +106,7 @@ import {
|
|
|
86
106
|
} from "./verbs/socket.ts";
|
|
87
107
|
import { homedir } from "node:os";
|
|
88
108
|
|
|
89
|
-
import { pushRunBranch, type RunRepoRef } from "./gitops.ts";
|
|
109
|
+
import { pushRunBranch, readBaseChain, type RunRepoRef } from "./gitops.ts";
|
|
90
110
|
import {
|
|
91
111
|
planUsageLine,
|
|
92
112
|
readPlanUsage,
|
|
@@ -108,6 +128,14 @@ const REPORT_DELIVERY_INTERVAL_MS = 30_000;
|
|
|
108
128
|
* means the mirror itself is broken, not unlucky, and the sweep escalates
|
|
109
129
|
* instead of burning a turn-0 run per tick forever (#168, #177). */
|
|
110
130
|
const DISPATCH_INFRA_MAX_STRIKES = 3;
|
|
131
|
+
/** A provider-transient requeue (stream stalled mid-run) is retried, but only a
|
|
132
|
+
* bounded number of times: three aborted streams for one issue means the
|
|
133
|
+
* provider itself is degraded, not unlucky, and the sweep escalates to a
|
|
134
|
+
* human instead of requeueing into a down provider forever (#220). */
|
|
135
|
+
const PROVIDER_TRANSIENT_MAX_STRIKES = 3;
|
|
136
|
+
/** Terminal rows inspected for a salvaged PR per tick. GitHub provenance reads
|
|
137
|
+
* are maintenance, but a backlog must not turn one tick into an API burst. */
|
|
138
|
+
const SALVAGED_PR_ADOPTION_BATCH = 10;
|
|
111
139
|
const DEFAULT_PORT = 8787;
|
|
112
140
|
const BRIEF_TEMPLATE_PATH = join(import.meta.dir, "briefs", "worker.md");
|
|
113
141
|
|
|
@@ -144,6 +172,9 @@ interface Deps {
|
|
|
144
172
|
usage: UsageSource;
|
|
145
173
|
escalate(e: Escalation): Promise<void>;
|
|
146
174
|
turnLimits: TurnLimitRegistry;
|
|
175
|
+
workerControls: WorkerControlRegistry;
|
|
176
|
+
/** Session seam for lifecycle integration tests; production uses the real harness. */
|
|
177
|
+
workerDeps?: RunWorkerDeps;
|
|
147
178
|
integrity: IntegrityGate;
|
|
148
179
|
stall: StallGate;
|
|
149
180
|
cleanup?: RetainedCleanupCursor;
|
|
@@ -190,6 +221,7 @@ export function verbDeps(d: Pick<Deps, "project" | "store" | "tracker" | "verbAc
|
|
|
190
221
|
pausedAt,
|
|
191
222
|
log,
|
|
192
223
|
now: () => Date.now(),
|
|
224
|
+
chain: { readBaseChain },
|
|
193
225
|
};
|
|
194
226
|
}
|
|
195
227
|
|
|
@@ -198,18 +230,25 @@ export function verbDeps(d: Pick<Deps, "project" | "store" | "tracker" | "verbAc
|
|
|
198
230
|
|
|
199
231
|
// ------------------------------------------------------- orchestrator liveness
|
|
200
232
|
|
|
233
|
+
/** No more than one reminder per hour while the same marker remains. */
|
|
234
|
+
const STALL_RENOTIFY_MS = 60 * 60_000;
|
|
235
|
+
|
|
201
236
|
/**
|
|
202
|
-
* Whether the wedged-orchestrator page has
|
|
203
|
-
* currently on disk.
|
|
204
|
-
* consumed, so paging per five minutes would be paging forever.
|
|
237
|
+
* Whether the wedged-orchestrator page has gone out, and when it last went out,
|
|
238
|
+
* for the stall currently on disk.
|
|
205
239
|
*/
|
|
206
240
|
export interface StallGate {
|
|
207
241
|
paged: boolean;
|
|
242
|
+
lastPagedAt?: number;
|
|
208
243
|
}
|
|
209
244
|
|
|
210
245
|
export interface StallVerdict {
|
|
211
246
|
/** The marker's own line, when one is there. */
|
|
212
247
|
since?: string;
|
|
248
|
+
/** Count written by the stalled tick producer, when its marker is readable. */
|
|
249
|
+
unconsumedTicks?: number;
|
|
250
|
+
/** Whole hours elapsed since the marker timestamp, when it is parseable. */
|
|
251
|
+
stalledHours?: number;
|
|
213
252
|
page: boolean;
|
|
214
253
|
}
|
|
215
254
|
|
|
@@ -232,50 +271,84 @@ export interface StallVerdict {
|
|
|
232
271
|
*
|
|
233
272
|
* Resets when the marker disappears, so a second stall days later pages again.
|
|
234
273
|
*/
|
|
235
|
-
export function checkStall(gate: StallGate, marker: string): StallVerdict {
|
|
274
|
+
export function checkStall(gate: StallGate, marker: string, now = Date.now()): StallVerdict {
|
|
236
275
|
if (!existsSync(marker)) {
|
|
237
276
|
gate.paged = false;
|
|
277
|
+
delete gate.lastPagedAt;
|
|
238
278
|
return { page: false };
|
|
239
279
|
}
|
|
240
|
-
const page =
|
|
280
|
+
const page =
|
|
281
|
+
!gate.paged ||
|
|
282
|
+
gate.lastPagedAt === undefined ||
|
|
283
|
+
now - gate.lastPagedAt >= STALL_RENOTIFY_MS;
|
|
241
284
|
let since: string | undefined;
|
|
285
|
+
let unconsumedTicks: number | undefined;
|
|
286
|
+
let stalledHours: number | undefined;
|
|
242
287
|
try {
|
|
243
288
|
const body = readFileSync(marker, "utf8").split("\n")[0]?.trim();
|
|
244
|
-
if (body !== undefined && body !== "")
|
|
289
|
+
if (body !== undefined && body !== "") {
|
|
290
|
+
since = body;
|
|
291
|
+
const ticks = /\b(\d+) ticks queued unconsumed\b/.exec(body)?.[1];
|
|
292
|
+
if (ticks !== undefined) unconsumedTicks = Number(ticks);
|
|
293
|
+
const startedAt = Date.parse(body.split(/\s+/, 1)[0] ?? "");
|
|
294
|
+
if (Number.isFinite(startedAt)) {
|
|
295
|
+
stalledHours = Math.max(0, Math.floor((now - startedAt) / STALL_RENOTIFY_MS));
|
|
296
|
+
}
|
|
297
|
+
}
|
|
245
298
|
} catch {
|
|
246
|
-
// An unreadable marker still means stalled;
|
|
299
|
+
// An unreadable marker still means stalled; its evidence is a nicety.
|
|
247
300
|
}
|
|
248
|
-
return {
|
|
301
|
+
return {
|
|
302
|
+
...(since === undefined ? {} : { since }),
|
|
303
|
+
...(unconsumedTicks === undefined ? {} : { unconsumedTicks }),
|
|
304
|
+
...(stalledHours === undefined ? {} : { stalledHours }),
|
|
305
|
+
page,
|
|
306
|
+
};
|
|
249
307
|
}
|
|
250
308
|
|
|
251
309
|
/**
|
|
252
|
-
* Pages tier 2
|
|
310
|
+
* Pages tier 2 when the orchestrator session stops draining its queue.
|
|
253
311
|
*
|
|
254
312
|
* Deliberately does not restart anything. A wedge lands mid-turn, this process
|
|
255
313
|
* cannot tell a half-applied edit from an idle loop, and killing the session
|
|
256
314
|
* could destroy work an operator would rather read first — the same refusal to
|
|
257
315
|
* guess that the recovery plugin is built on.
|
|
258
316
|
*/
|
|
259
|
-
async function watchOrchestrator(d: Deps): Promise<void> {
|
|
317
|
+
export async function watchOrchestrator(d: Deps, now = Date.now()): Promise<void> {
|
|
260
318
|
const marker = join(stateDir(), STALL_MARKER_FILE);
|
|
261
|
-
const
|
|
319
|
+
const repeat = d.stall.paged;
|
|
320
|
+
const verdict = checkStall(d.stall, marker, now);
|
|
262
321
|
if (!verdict.page) return;
|
|
263
322
|
|
|
323
|
+
const greenRuns = d.store.activeRuns(d.project.name).filter((run) => run.state === "pushed-green");
|
|
324
|
+
const repeatSuffix = repeat
|
|
325
|
+
? ` — still stalled (${
|
|
326
|
+
verdict.stalledHours === undefined
|
|
327
|
+
? `${new Date(now).toISOString().slice(0, 13)}Z`
|
|
328
|
+
: `${verdict.stalledHours}h`
|
|
329
|
+
})`
|
|
330
|
+
: "";
|
|
331
|
+
|
|
264
332
|
log(`ERROR: the orchestrator session is not draining its queue — ${verdict.since ?? "no timestamp"}`);
|
|
265
333
|
const delivered = await safeEscalate(d, {
|
|
266
334
|
tier: 2,
|
|
335
|
+
category: "confirmed-failure",
|
|
336
|
+
urgent: true,
|
|
267
337
|
project: d.project.name,
|
|
268
338
|
issue: NO_ISSUE,
|
|
269
339
|
// Keyed on the marker's own timestamp, not the date. The dedup ledger keys
|
|
270
340
|
// on this summary, and two wedges in one day is not a hypothetical — the
|
|
271
341
|
// failure mode is a session that gets stuck, gets restarted, and gets stuck
|
|
272
|
-
// again on the same cause an hour later.
|
|
273
|
-
//
|
|
342
|
+
// again on the same cause an hour later. The hourly suffix makes bounded
|
|
343
|
+
// reminders distinct without allowing every five-minute tick through.
|
|
274
344
|
summary:
|
|
275
345
|
`Orchestrator session wedged (${verdict.since ?? `marker at ${marker}`}) — ` +
|
|
276
|
-
`it has stopped reading its queue (${d.project.name})`,
|
|
346
|
+
`it has stopped reading its queue (${d.project.name})${repeatSuffix}`,
|
|
277
347
|
detail: [
|
|
278
348
|
verdict.since ?? "Marker present with no readable timestamp.",
|
|
349
|
+
`Unconsumed ticks: ${verdict.unconsumedTicks ?? "unknown"}`,
|
|
350
|
+
`Open green worker PRs: ${greenRuns.length}`,
|
|
351
|
+
...greenRuns.map((run) => `- ${run.prUrl ?? `#${run.issue} (PR URL unavailable)`}`),
|
|
279
352
|
`Marker: ${marker}`,
|
|
280
353
|
"",
|
|
281
354
|
"Its process and its herdr agent label are both healthy, which is why nothing else noticed:",
|
|
@@ -287,7 +360,7 @@ async function watchOrchestrator(d: Deps): Promise<void> {
|
|
|
287
360
|
"Dispatch is unaffected: workers keep running. What stops is drain, groom, report and merge.",
|
|
288
361
|
].join("\n"),
|
|
289
362
|
});
|
|
290
|
-
markPaged(d.stall, delivered);
|
|
363
|
+
markPaged(d.stall, delivered, now);
|
|
291
364
|
}
|
|
292
365
|
|
|
293
366
|
/**
|
|
@@ -304,10 +377,10 @@ export function isPaused(): boolean {
|
|
|
304
377
|
* The epoch-ms timestamp at which the current pause began, read from the same
|
|
305
378
|
* sentinel file {@link setPaused} writes (`<stateDir()>/paused`). Returns
|
|
306
379
|
* `undefined` when the fleet is not paused, or when the file's first line does
|
|
307
|
-
* not parse as a date
|
|
308
|
-
*
|
|
309
|
-
* innocent. {@link isPaused} is the authority on *whether*; this answers
|
|
310
|
-
*
|
|
380
|
+
* not parse as a date. A legacy/blank sentinel keeps completion mutations
|
|
381
|
+
* fail-closed because a run admitted before an *unknown* pause cannot be proven
|
|
382
|
+
* innocent. {@link isPaused} is the authority on *whether*; this answers *since
|
|
383
|
+
* when*.
|
|
311
384
|
*/
|
|
312
385
|
export function pausedAt(): number | undefined {
|
|
313
386
|
const f = join(stateDir(), "paused");
|
|
@@ -318,8 +391,8 @@ export function pausedAt(): number | undefined {
|
|
|
318
391
|
const t = Date.parse(first);
|
|
319
392
|
return Number.isNaN(t) ? undefined : t;
|
|
320
393
|
} catch {
|
|
321
|
-
// Unreadable sentinel (permissions, corruption): fail
|
|
322
|
-
//
|
|
394
|
+
// Unreadable sentinel (permissions, corruption): fail completion mutations
|
|
395
|
+
// closed like an unparseable line while the pause time is unprovable.
|
|
323
396
|
return undefined;
|
|
324
397
|
}
|
|
325
398
|
}
|
|
@@ -344,8 +417,8 @@ export function pauseProvenance(): { source: string; reason?: string } | undefin
|
|
|
344
417
|
const reason = match[2];
|
|
345
418
|
return { source, ...(reason === undefined ? {} : { reason }) };
|
|
346
419
|
} catch {
|
|
347
|
-
// Unreadable sentinel: no provenance to name
|
|
348
|
-
//
|
|
420
|
+
// Unreadable sentinel: no provenance to name. Completion mutations still
|
|
421
|
+
// fail closed because the pause time is unknown.
|
|
349
422
|
return undefined;
|
|
350
423
|
}
|
|
351
424
|
}
|
|
@@ -456,9 +529,17 @@ export function checkIntegrity(gate: IntegrityGate, current: Map<string, string>
|
|
|
456
529
|
return { diff, pause: true, page: !gate.paged };
|
|
457
530
|
}
|
|
458
531
|
|
|
459
|
-
/** Latch a
|
|
460
|
-
export function markPaged(gate:
|
|
461
|
-
|
|
532
|
+
/** Latch a page after delivery is confirmed, and never before. */
|
|
533
|
+
export function markPaged(gate: StallGate, delivered: boolean, now: number): void;
|
|
534
|
+
export function markPaged(gate: IntegrityGate, delivered: boolean): void;
|
|
535
|
+
export function markPaged(
|
|
536
|
+
gate: { paged: boolean; lastPagedAt?: number },
|
|
537
|
+
delivered: boolean,
|
|
538
|
+
now?: number,
|
|
539
|
+
): void {
|
|
540
|
+
if (!delivered) return;
|
|
541
|
+
gate.paged = true;
|
|
542
|
+
if (now !== undefined) gate.lastPagedAt = now;
|
|
462
543
|
}
|
|
463
544
|
|
|
464
545
|
// ---------------------------------------------------------------------- helpers
|
|
@@ -528,6 +609,32 @@ function swapLabel(store: Store, projectName: string, issue: number, from: strin
|
|
|
528
609
|
{ issue, op: "remove", label: from },
|
|
529
610
|
]);
|
|
530
611
|
}
|
|
612
|
+
/**
|
|
613
|
+
* Persist the operator-stop transition before releasing its live controller.
|
|
614
|
+
* The row is terminal first, then its in-progress label is removed through the
|
|
615
|
+
* same durable projection outbox as every other lifecycle transition.
|
|
616
|
+
*/
|
|
617
|
+
export function recordOperatorStop(
|
|
618
|
+
store: Pick<Store, "updateRun" | "enqueueLabelOps">,
|
|
619
|
+
args: {
|
|
620
|
+
project: string;
|
|
621
|
+
issue: number;
|
|
622
|
+
runId: string;
|
|
623
|
+
inProgress: string;
|
|
624
|
+
reason: string;
|
|
625
|
+
patch: Partial<RunRecord>;
|
|
626
|
+
},
|
|
627
|
+
): void {
|
|
628
|
+
store.updateRun(args.runId, {
|
|
629
|
+
...args.patch,
|
|
630
|
+
state: "stopped",
|
|
631
|
+
lastError: `operator stopped: ${args.reason}`,
|
|
632
|
+
});
|
|
633
|
+
store.enqueueLabelOps(args.project, [
|
|
634
|
+
{ issue: args.issue, op: "remove", label: args.inProgress },
|
|
635
|
+
]);
|
|
636
|
+
}
|
|
637
|
+
|
|
531
638
|
|
|
532
639
|
/**
|
|
533
640
|
* The escalator throws when no transport is configured or Telegram rejects, and
|
|
@@ -549,6 +656,38 @@ async function safeEscalate(d: Pick<Deps, "escalate">, e: Escalation): Promise<b
|
|
|
549
656
|
}
|
|
550
657
|
}
|
|
551
658
|
|
|
659
|
+
async function reactToProviderCredit(
|
|
660
|
+
d: Deps,
|
|
661
|
+
issue: number,
|
|
662
|
+
message: string,
|
|
663
|
+
sessionFile: string | undefined,
|
|
664
|
+
): Promise<void> {
|
|
665
|
+
const { project } = d;
|
|
666
|
+
const alreadyPaused = isPaused();
|
|
667
|
+
if (!alreadyPaused) setPaused(true, { source: "provider-credit", reason: message });
|
|
668
|
+
log(
|
|
669
|
+
`#${issue} provider refused for credit — dispatch ${alreadyPaused ? "remains paused" : "paused"}: ${message}`,
|
|
670
|
+
);
|
|
671
|
+
// Fleet-scoped and run-independent on purpose. The notification ledger
|
|
672
|
+
// dedupes on `project:issue:tier:summary`, so `NO_ISSUE` plus a summary
|
|
673
|
+
// carrying no run or attempt pages once for the fleet, not once per run.
|
|
674
|
+
await safeEscalate(d, {
|
|
675
|
+
tier: 2,
|
|
676
|
+
category: "fleet-stopped",
|
|
677
|
+
project: project.name,
|
|
678
|
+
issue: NO_ISSUE,
|
|
679
|
+
summary: `Model provider refused for credit — ${project.name} is paused`,
|
|
680
|
+
detail: [
|
|
681
|
+
message,
|
|
682
|
+
"",
|
|
683
|
+
"No implementation attempt was charged: this is a billing state, not a",
|
|
684
|
+
"failed implementation. Each affected issue keeps its queue label and",
|
|
685
|
+
"re-dispatches on `omp-conductor resume` once the provider has credit.",
|
|
686
|
+
`Session: ${sessionFile ?? "(no transcript)"}`,
|
|
687
|
+
].join("\n"),
|
|
688
|
+
});
|
|
689
|
+
}
|
|
690
|
+
|
|
552
691
|
/**
|
|
553
692
|
* What a salvage attempt contributes to the escalation: where the work went, or
|
|
554
693
|
* that it went nowhere. Split from the effects below for the same reason
|
|
@@ -823,6 +962,135 @@ export function createTurnLimitRegistry(
|
|
|
823
962
|
};
|
|
824
963
|
}
|
|
825
964
|
|
|
965
|
+
export type WorkerControlResult =
|
|
966
|
+
| { kind: "ok"; runId: string; phase: WorkerPausePhase }
|
|
967
|
+
| { kind: "stopped"; runId: string; reason: string }
|
|
968
|
+
| { kind: "refused"; runId: string; error: string }
|
|
969
|
+
| { kind: "not-active" };
|
|
970
|
+
|
|
971
|
+
export interface WorkerControlSlot {
|
|
972
|
+
install(control: WorkerPauseControl): void;
|
|
973
|
+
/** Stop accepted before the session controller exists. */
|
|
974
|
+
requestedStop(): string | undefined;
|
|
975
|
+
close(): void;
|
|
976
|
+
}
|
|
977
|
+
|
|
978
|
+
export interface WorkerControlRegistry {
|
|
979
|
+
open(project: string, issue: number, runId: string): WorkerControlSlot;
|
|
980
|
+
pause(project: string, issue: number): Promise<WorkerControlResult>;
|
|
981
|
+
resume(project: string, issue: number): WorkerControlResult;
|
|
982
|
+
stop(project: string, issue: number, reason: string): Promise<WorkerControlResult>;
|
|
983
|
+
/** Live runs whose phase is not `running` — what /healthz and the board show. */
|
|
984
|
+
snapshot(project: string): { issue: number; runId: string; phase: WorkerPausePhase }[];
|
|
985
|
+
}
|
|
986
|
+
|
|
987
|
+
/** Authoritative controls for sessions owned by this daemon process. */
|
|
988
|
+
export function createWorkerControlRegistry(): WorkerControlRegistry {
|
|
989
|
+
interface Entry {
|
|
990
|
+
project: string;
|
|
991
|
+
issue: number;
|
|
992
|
+
runId: string;
|
|
993
|
+
control?: WorkerPauseControl;
|
|
994
|
+
stopReason?: string;
|
|
995
|
+
stopError?: string;
|
|
996
|
+
finished: PromiseWithResolvers<void>;
|
|
997
|
+
}
|
|
998
|
+
|
|
999
|
+
const active = new Map<string, Entry>();
|
|
1000
|
+
const key = (project: string, issue: number): string => `${project}\0${issue}`;
|
|
1001
|
+
return {
|
|
1002
|
+
open(project, issue, runId) {
|
|
1003
|
+
const k = key(project, issue);
|
|
1004
|
+
if (active.has(k)) throw new Error(`#${issue} already has a live worker controller`);
|
|
1005
|
+
const entry: Entry = {
|
|
1006
|
+
project,
|
|
1007
|
+
issue,
|
|
1008
|
+
runId,
|
|
1009
|
+
finished: Promise.withResolvers<void>(),
|
|
1010
|
+
};
|
|
1011
|
+
active.set(k, entry);
|
|
1012
|
+
return {
|
|
1013
|
+
install: (control) => {
|
|
1014
|
+
if (active.get(k) !== entry) return;
|
|
1015
|
+
entry.control = control;
|
|
1016
|
+
if (entry.stopReason === undefined) return;
|
|
1017
|
+
try {
|
|
1018
|
+
control.stop(entry.stopReason);
|
|
1019
|
+
} catch (err) {
|
|
1020
|
+
entry.stopError = err instanceof Error ? err.message : String(err);
|
|
1021
|
+
}
|
|
1022
|
+
},
|
|
1023
|
+
requestedStop: () => active.get(k) === entry ? entry.stopReason : undefined,
|
|
1024
|
+
close: () => {
|
|
1025
|
+
if (active.get(k) === entry) active.delete(k);
|
|
1026
|
+
entry.finished.resolve();
|
|
1027
|
+
},
|
|
1028
|
+
};
|
|
1029
|
+
},
|
|
1030
|
+
async pause(project, issue) {
|
|
1031
|
+
const entry = active.get(key(project, issue));
|
|
1032
|
+
if (entry?.control === undefined) return { kind: "not-active" };
|
|
1033
|
+
try {
|
|
1034
|
+
await entry.control.pause();
|
|
1035
|
+
return { kind: "ok", runId: entry.runId, phase: entry.control.phase() };
|
|
1036
|
+
} catch (err) {
|
|
1037
|
+
return {
|
|
1038
|
+
kind: "refused",
|
|
1039
|
+
runId: entry.runId,
|
|
1040
|
+
error: err instanceof Error ? err.message : String(err),
|
|
1041
|
+
};
|
|
1042
|
+
}
|
|
1043
|
+
},
|
|
1044
|
+
resume(project, issue) {
|
|
1045
|
+
const entry = active.get(key(project, issue));
|
|
1046
|
+
if (entry?.control === undefined) return { kind: "not-active" };
|
|
1047
|
+
try {
|
|
1048
|
+
entry.control.resume();
|
|
1049
|
+
return { kind: "ok", runId: entry.runId, phase: entry.control.phase() };
|
|
1050
|
+
} catch (err) {
|
|
1051
|
+
return {
|
|
1052
|
+
kind: "refused",
|
|
1053
|
+
runId: entry.runId,
|
|
1054
|
+
error: err instanceof Error ? err.message : String(err),
|
|
1055
|
+
};
|
|
1056
|
+
}
|
|
1057
|
+
},
|
|
1058
|
+
async stop(project, issue, reason) {
|
|
1059
|
+
const entry = active.get(key(project, issue));
|
|
1060
|
+
if (entry === undefined) return { kind: "not-active" };
|
|
1061
|
+
if (entry.stopReason === undefined) {
|
|
1062
|
+
entry.stopReason = reason;
|
|
1063
|
+
if (entry.control !== undefined) {
|
|
1064
|
+
try {
|
|
1065
|
+
entry.control.stop(reason);
|
|
1066
|
+
} catch (err) {
|
|
1067
|
+
entry.stopReason = undefined;
|
|
1068
|
+
return {
|
|
1069
|
+
kind: "refused",
|
|
1070
|
+
runId: entry.runId,
|
|
1071
|
+
error: err instanceof Error ? err.message : String(err),
|
|
1072
|
+
};
|
|
1073
|
+
}
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
1076
|
+
await entry.finished.promise;
|
|
1077
|
+
if (entry.stopError !== undefined) {
|
|
1078
|
+
return { kind: "refused", runId: entry.runId, error: entry.stopError };
|
|
1079
|
+
}
|
|
1080
|
+
return { kind: "stopped", runId: entry.runId, reason: entry.stopReason! };
|
|
1081
|
+
},
|
|
1082
|
+
snapshot(project) {
|
|
1083
|
+
const workers: { issue: number; runId: string; phase: WorkerPausePhase }[] = [];
|
|
1084
|
+
for (const entry of active.values()) {
|
|
1085
|
+
if (entry.project !== project || entry.control === undefined) continue;
|
|
1086
|
+
const phase = entry.control.phase();
|
|
1087
|
+
if (phase !== "running") workers.push({ issue: entry.issue, runId: entry.runId, phase });
|
|
1088
|
+
}
|
|
1089
|
+
return workers;
|
|
1090
|
+
},
|
|
1091
|
+
};
|
|
1092
|
+
}
|
|
1093
|
+
|
|
826
1094
|
export async function verifyPushedGreenClaim(
|
|
827
1095
|
tracker: Pick<Tracker, "verifyPr">,
|
|
828
1096
|
claim: Pick<WorkerResult, "prUrl" | "headSha">,
|
|
@@ -912,7 +1180,28 @@ export function shouldContinueAfterTurnsCap(f: {
|
|
|
912
1180
|
return hasContinuationBudget(f.continuation, f.maxContinuations);
|
|
913
1181
|
}
|
|
914
1182
|
|
|
915
|
-
|
|
1183
|
+
/** Bind a worker's run identity to every release-policy block it emits. */
|
|
1184
|
+
export function workerReleaseBlockRecorder(
|
|
1185
|
+
project: string,
|
|
1186
|
+
issue: number,
|
|
1187
|
+
runId: string,
|
|
1188
|
+
root = stateDir(),
|
|
1189
|
+
): (shape: ReleaseShape, context: ReleaseBlockContext) => void {
|
|
1190
|
+
return (shape, context) =>
|
|
1191
|
+
recordReleaseBlock(
|
|
1192
|
+
project,
|
|
1193
|
+
"worker",
|
|
1194
|
+
shape,
|
|
1195
|
+
{
|
|
1196
|
+
...context,
|
|
1197
|
+
issue,
|
|
1198
|
+
runId,
|
|
1199
|
+
},
|
|
1200
|
+
root,
|
|
1201
|
+
);
|
|
1202
|
+
}
|
|
1203
|
+
|
|
1204
|
+
export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
916
1205
|
const { project, caps, tracker, store } = d;
|
|
917
1206
|
const issue = r.issue.number;
|
|
918
1207
|
const branch = branchName(r.issue);
|
|
@@ -926,6 +1215,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
926
1215
|
const mirrorPath = mirrorPathFor(r.repo, project.mirrorRoot);
|
|
927
1216
|
let worktreePath: string | undefined;
|
|
928
1217
|
let turnLimit: TurnLimitController | undefined;
|
|
1218
|
+
let workerControl: WorkerControlSlot | undefined;
|
|
1219
|
+
let workerSessionInstalled = false;
|
|
929
1220
|
// The run's own repository. Hoisted for the same reason `worktreePath` is —
|
|
930
1221
|
// the catch and finally paths have to publish the branch.
|
|
931
1222
|
let runRepo: RunRepoRef | undefined;
|
|
@@ -947,6 +1238,53 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
947
1238
|
return pushRunBranch(project, runRepo);
|
|
948
1239
|
};
|
|
949
1240
|
|
|
1241
|
+
/**
|
|
1242
|
+
* A stop can arrive after the run is claimed but before `runWorker` exposes
|
|
1243
|
+
* its controller. Settle that run here instead of turning an operator action
|
|
1244
|
+
* into a dispatch failure or making the HTTP request wait for a session that
|
|
1245
|
+
* will never exist.
|
|
1246
|
+
*/
|
|
1247
|
+
const settleStopBeforeSession = async (): Promise<boolean> => {
|
|
1248
|
+
const reason = workerControl?.requestedStop();
|
|
1249
|
+
if (reason === undefined || run === undefined || workerSessionInstalled) return false;
|
|
1250
|
+
turnLimit?.close();
|
|
1251
|
+
turnLimit = undefined;
|
|
1252
|
+
const settlement =
|
|
1253
|
+
worktreePath === undefined
|
|
1254
|
+
? undefined
|
|
1255
|
+
: await settleWorktree({
|
|
1256
|
+
issue,
|
|
1257
|
+
attempt,
|
|
1258
|
+
ending: `stopped by the operator: ${reason}`,
|
|
1259
|
+
worktree: worktreePath,
|
|
1260
|
+
branch,
|
|
1261
|
+
publish,
|
|
1262
|
+
tree: "remove",
|
|
1263
|
+
mirrorPath,
|
|
1264
|
+
});
|
|
1265
|
+
recordOperatorStop(store, {
|
|
1266
|
+
project: project.name,
|
|
1267
|
+
issue,
|
|
1268
|
+
runId: run.id,
|
|
1269
|
+
inProgress,
|
|
1270
|
+
reason,
|
|
1271
|
+
patch: {
|
|
1272
|
+
endedAt: Date.now(),
|
|
1273
|
+
turns: run.turns,
|
|
1274
|
+
spendUsd: run.spendUsd,
|
|
1275
|
+
worktree: worktreePath ?? run.worktree,
|
|
1276
|
+
report: [
|
|
1277
|
+
"Operator stopped the run before its worker session started.",
|
|
1278
|
+
`Reason: ${reason}`,
|
|
1279
|
+
...(settlement?.lines ?? []),
|
|
1280
|
+
].join("\n"),
|
|
1281
|
+
...settlement?.patch,
|
|
1282
|
+
},
|
|
1283
|
+
});
|
|
1284
|
+
log(`#${issue} stopped by operator before its worker session started: ${reason}`);
|
|
1285
|
+
return true;
|
|
1286
|
+
};
|
|
1287
|
+
|
|
950
1288
|
try {
|
|
951
1289
|
// Claim on the STORE first, before anything that can fail. The run row —
|
|
952
1290
|
// not the label — is the crash-safe guard against double dispatch: rows
|
|
@@ -962,8 +1300,6 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
962
1300
|
// Read before this attempt's own row exists, so `latestRun` still means the
|
|
963
1301
|
// attempt whose work this one inherits.
|
|
964
1302
|
const priorSalvage = store.latestRun(project.name, issue)?.salvageSha;
|
|
965
|
-
|
|
966
|
-
|
|
967
1303
|
run = store.createRun({
|
|
968
1304
|
project: project.name,
|
|
969
1305
|
issue,
|
|
@@ -977,10 +1313,20 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
977
1313
|
maxTurns: caps.workerMaxTurns,
|
|
978
1314
|
startedAt: Date.now(),
|
|
979
1315
|
});
|
|
1316
|
+
const maxTurns = run.maxTurns;
|
|
1317
|
+
const turnOverride = maxTurns > caps.workerMaxTurns ? maxTurns : undefined;
|
|
980
1318
|
const runId = run.id;
|
|
1319
|
+
if (turnOverride !== undefined) {
|
|
1320
|
+
log(
|
|
1321
|
+
`#${issue} claimed with turn override ${turnOverride} ` +
|
|
1322
|
+
`(base ${caps.workerMaxTurns})`,
|
|
1323
|
+
);
|
|
1324
|
+
}
|
|
981
1325
|
store.enqueueLabelOps(project.name, [{ issue, op: "add", label: inProgress }]);
|
|
982
1326
|
claimed = true;
|
|
983
|
-
turnLimit = d.turnLimits.open(project.name, issue, runId,
|
|
1327
|
+
turnLimit = d.turnLimits.open(project.name, issue, runId, maxTurns);
|
|
1328
|
+
workerControl = d.workerControls.open(project.name, issue, runId);
|
|
1329
|
+
if (await settleStopBeforeSession()) return;
|
|
984
1330
|
|
|
985
1331
|
// A run's tree is <workspaceRoot>/<issue> and addRunRepo refuses to reuse
|
|
986
1332
|
// an existing path, so a retry — or a tree kept from a failed attempt — has
|
|
@@ -989,6 +1335,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
989
1335
|
// first attempt. addRunRepo does its own ensureMirror; calling it here too
|
|
990
1336
|
// would cost a second network fetch per attempt.
|
|
991
1337
|
await removeWorktree(mirrorPath, worktreePathFor(project.workspaceRoot, issue));
|
|
1338
|
+
if (await settleStopBeforeSession()) return;
|
|
992
1339
|
const provisioned = await addRunRepo(
|
|
993
1340
|
r.repo,
|
|
994
1341
|
project.mirrorRoot,
|
|
@@ -998,6 +1345,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
998
1345
|
);
|
|
999
1346
|
worktreePath = provisioned.path;
|
|
1000
1347
|
runRepo = { repo: r.repo, runRepoPath: worktreePath, branch };
|
|
1348
|
+
if (await settleStopBeforeSession()) return;
|
|
1001
1349
|
|
|
1002
1350
|
// The SDK names the transcript itself, so the daemon supplies the parent
|
|
1003
1351
|
// directory and learns the real path back from the result. Inventing one
|
|
@@ -1036,6 +1384,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1036
1384
|
},
|
|
1037
1385
|
{ ...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }) },
|
|
1038
1386
|
);
|
|
1387
|
+
if (await settleStopBeforeSession()) return;
|
|
1039
1388
|
|
|
1040
1389
|
store.updateRun(runId, { worktree: worktreePath, state: "running" });
|
|
1041
1390
|
|
|
@@ -1044,19 +1393,26 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1044
1393
|
(provisioned.reattached ? " (continuation: reattached existing branch)" : ""),
|
|
1045
1394
|
);
|
|
1046
1395
|
|
|
1396
|
+
const brief = await buildBrief(project, r, branch, worktreePath, {
|
|
1397
|
+
continuation: provisioned.reattached,
|
|
1398
|
+
defaultBranch: r.repo.defaultBranch,
|
|
1399
|
+
...(provisioned.reattached && priorSalvage !== undefined
|
|
1400
|
+
? { salvagedSha: priorSalvage }
|
|
1401
|
+
: {}),
|
|
1402
|
+
});
|
|
1403
|
+
if (await settleStopBeforeSession()) return;
|
|
1404
|
+
|
|
1047
1405
|
let result: WorkerResult;
|
|
1048
1406
|
try {
|
|
1049
1407
|
result = await runWorker({
|
|
1050
|
-
brief
|
|
1051
|
-
continuation: provisioned.reattached,
|
|
1052
|
-
defaultBranch: r.repo.defaultBranch,
|
|
1053
|
-
...(provisioned.reattached && priorSalvage !== undefined
|
|
1054
|
-
? { salvagedSha: priorSalvage }
|
|
1055
|
-
: {}),
|
|
1056
|
-
}),
|
|
1408
|
+
brief,
|
|
1057
1409
|
cwd: worktreePath,
|
|
1058
1410
|
caps,
|
|
1059
|
-
maxTurns: () => turnLimit?.maxTurns() ??
|
|
1411
|
+
maxTurns: () => turnLimit?.maxTurns() ?? maxTurns,
|
|
1412
|
+
onPauseControl: (control) => {
|
|
1413
|
+
workerSessionInstalled = true;
|
|
1414
|
+
workerControl?.install(control);
|
|
1415
|
+
},
|
|
1060
1416
|
sessionDir,
|
|
1061
1417
|
// The session's control socket, under the daemon's own state directory —
|
|
1062
1418
|
// a child process of the daemon reaches it directly.
|
|
@@ -1075,7 +1431,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1075
1431
|
},
|
|
1076
1432
|
...(project.workerModel === undefined ? {} : { model: project.workerModel }),
|
|
1077
1433
|
releaseGrants: resolveReleaseGrants(project),
|
|
1078
|
-
onReleaseBlocked: (
|
|
1434
|
+
onReleaseBlocked: workerReleaseBlockRecorder(project.name, issue, runId),
|
|
1079
1435
|
onTurn: (n) => store.updateRun(runId, { turns: n }),
|
|
1080
1436
|
onSpend: (usd) => store.updateRun(runId, { spendUsd: usd }),
|
|
1081
1437
|
onKilled: () => {
|
|
@@ -1087,7 +1443,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1087
1443
|
// and a path written at completion is a path nobody can follow live. The
|
|
1088
1444
|
// completion-time update below writes the same value again, harmlessly.
|
|
1089
1445
|
onSessionFile: (f) => store.updateRun(runId, { sessionFile: f }),
|
|
1090
|
-
});
|
|
1446
|
+
}, d.workerDeps);
|
|
1091
1447
|
} finally {
|
|
1092
1448
|
// This is the authoritative settlement edge for `extend`: close before
|
|
1093
1449
|
// PR verification or terminal row writes can leave stale `running` state.
|
|
@@ -1110,13 +1466,10 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1110
1466
|
|
|
1111
1467
|
// Read before the row is written so `lastError` carries the provider's own
|
|
1112
1468
|
// message — it names the fix and the URL, which no classification can.
|
|
1113
|
-
const
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
return err === undefined ? undefined : providerCreditRefusal(err);
|
|
1118
|
-
})()
|
|
1119
|
-
: undefined;
|
|
1469
|
+
const sessionErr = state === "failed" || state === "killed" ? readSessionError(result.sessionFile) : undefined;
|
|
1470
|
+
const providerCredit = sessionErr === undefined ? undefined : providerCreditRefusal(sessionErr);
|
|
1471
|
+
const providerTransient =
|
|
1472
|
+
providerCredit !== undefined || sessionErr === undefined ? undefined : providerTransientFault(sessionErr);
|
|
1120
1473
|
|
|
1121
1474
|
// The other half of not believing a worker about its own run (#128). The
|
|
1122
1475
|
// claim being audited is `state: pushed-green`, so the audit runs on the
|
|
@@ -1160,8 +1513,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1160
1513
|
// `pushed-*` run is the only end that does not salvage: its deliverable is
|
|
1161
1514
|
// already on a remote branch, whatever is left loose in the tree is by the
|
|
1162
1515
|
// worker's own account not part of it, and appending a WIP commit would
|
|
1163
|
-
// turn the green PR this daemon just verified red. Every other end
|
|
1164
|
-
//
|
|
1516
|
+
// turn the green PR this daemon just verified red. Every other end may
|
|
1517
|
+
// contain work, so it is salvaged before the tree's final fate is decided.
|
|
1165
1518
|
const settlement =
|
|
1166
1519
|
state === "pushed-green" || state === "pushed-pending" || state === "merged"
|
|
1167
1520
|
? undefined
|
|
@@ -1169,7 +1522,11 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1169
1522
|
issue,
|
|
1170
1523
|
attempt,
|
|
1171
1524
|
ending:
|
|
1172
|
-
state === "blocked"
|
|
1525
|
+
state === "blocked"
|
|
1526
|
+
? "blocked for an operator decision"
|
|
1527
|
+
: state === "stopped"
|
|
1528
|
+
? `stopped by the operator: ${result.stoppedReason ?? "no reason recorded"}`
|
|
1529
|
+
: endedBy(result.killedBy),
|
|
1173
1530
|
worktree: worktreePath,
|
|
1174
1531
|
branch,
|
|
1175
1532
|
publish,
|
|
@@ -1190,8 +1547,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1190
1547
|
await removeWorktree(mirrorPath, worktreePath);
|
|
1191
1548
|
}
|
|
1192
1549
|
|
|
1193
|
-
|
|
1194
|
-
state,
|
|
1550
|
+
const terminalPatch: Partial<RunRecord> = {
|
|
1195
1551
|
endedAt: Date.now(),
|
|
1196
1552
|
turns: result.turns,
|
|
1197
1553
|
spendUsd: result.spendUsd,
|
|
@@ -1199,23 +1555,41 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1199
1555
|
headSha: result.headSha,
|
|
1200
1556
|
sessionFile: result.sessionFile,
|
|
1201
1557
|
// Every terminal state persists the worker's report, not just a green
|
|
1202
|
-
// push:
|
|
1203
|
-
// continuation must pool its disclosures from (#199).
|
|
1558
|
+
// push: a stopped attempt's partial report is still part of its audit trail.
|
|
1204
1559
|
report: result.report,
|
|
1205
|
-
...(providerCredit !== undefined
|
|
1206
|
-
? { lastError: providerCredit }
|
|
1207
|
-
: verified.reason === undefined
|
|
1208
|
-
? {}
|
|
1209
|
-
: { lastError: verified.reason }),
|
|
1210
1560
|
...settlement?.patch,
|
|
1211
1561
|
...(audit === undefined || audit.flags.length === 0
|
|
1212
1562
|
? {}
|
|
1213
1563
|
: { settlementFlags: audit.flags }),
|
|
1214
|
-
}
|
|
1564
|
+
};
|
|
1565
|
+
if (state === "stopped") {
|
|
1566
|
+
recordOperatorStop(store, {
|
|
1567
|
+
project: project.name,
|
|
1568
|
+
issue,
|
|
1569
|
+
runId,
|
|
1570
|
+
inProgress,
|
|
1571
|
+
reason: result.stoppedReason ?? "no reason recorded",
|
|
1572
|
+
patch: terminalPatch,
|
|
1573
|
+
});
|
|
1574
|
+
} else {
|
|
1575
|
+
const lastError = completionLastError(
|
|
1576
|
+
providerCredit,
|
|
1577
|
+
providerTransient,
|
|
1578
|
+
verified.reason,
|
|
1579
|
+
sessionErr,
|
|
1580
|
+
);
|
|
1581
|
+
store.updateRun(runId, {
|
|
1582
|
+
...terminalPatch,
|
|
1583
|
+
state,
|
|
1584
|
+
...(lastError === undefined ? {} : { lastError }),
|
|
1585
|
+
});
|
|
1586
|
+
}
|
|
1215
1587
|
|
|
1216
1588
|
const salvaged = settlement?.lines ?? [];
|
|
1217
1589
|
|
|
1218
|
-
if (state === "
|
|
1590
|
+
if (state === "stopped") {
|
|
1591
|
+
log(`#${issue} stopped by operator on attempt ${attempt}: ${result.stoppedReason}`);
|
|
1592
|
+
} else if (state === "blocked") {
|
|
1219
1593
|
swapLabel(store, project.name, issue, inProgress, project.stateLabels.blocked);
|
|
1220
1594
|
await safeEscalate(d, {
|
|
1221
1595
|
tier: 1,
|
|
@@ -1239,30 +1613,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1239
1613
|
});
|
|
1240
1614
|
|
|
1241
1615
|
if (providerCredit !== undefined) {
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
// the first 402 and a human noticing (#220).
|
|
1245
|
-
setPaused(true, { source: "provider-credit", reason: providerCredit });
|
|
1246
|
-
log(`#${issue} provider refused for credit — dispatch paused: ${providerCredit}`);
|
|
1247
|
-
swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
|
|
1248
|
-
// Fleet-scoped and run-independent on purpose. The notification ledger
|
|
1249
|
-
// dedupes on `project:issue:tier:summary`, so `NO_ISSUE` plus a summary
|
|
1250
|
-
// carrying no run or attempt is what makes this page once for the fleet
|
|
1251
|
-
// instead of once per affected run.
|
|
1252
|
-
await safeEscalate(d, {
|
|
1253
|
-
tier: 2,
|
|
1254
|
-
project: project.name,
|
|
1255
|
-
issue: NO_ISSUE,
|
|
1256
|
-
summary: `Model provider refused for credit — ${project.name} is paused`,
|
|
1257
|
-
detail: [
|
|
1258
|
-
providerCredit,
|
|
1259
|
-
"",
|
|
1260
|
-
"No implementation attempt was charged: this is a billing state, not a",
|
|
1261
|
-
"failed implementation. Each affected issue keeps its queue label and",
|
|
1262
|
-
"re-dispatches on `omp-conductor resume` once the provider has credit.",
|
|
1263
|
-
`Session: ${result.sessionFile ?? "(no transcript)"}`,
|
|
1264
|
-
].join("\n"),
|
|
1265
|
-
});
|
|
1616
|
+
await reactToProviderCredit(d, issue, providerCredit, result.sessionFile);
|
|
1617
|
+
swapToQueue(d, issue, inProgress);
|
|
1266
1618
|
} else if (continueTurns) {
|
|
1267
1619
|
// Requeue as one ordered pair: the in-progress removal before the
|
|
1268
1620
|
// queue add, exactly the order the projector will apply them in (#201).
|
|
@@ -1364,6 +1716,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1364
1716
|
// inner settlement guard exists. Latch it before any terminal write or await.
|
|
1365
1717
|
turnLimit?.close();
|
|
1366
1718
|
turnLimit = undefined;
|
|
1719
|
+
if (await settleStopBeforeSession()) return;
|
|
1367
1720
|
const detail = errText(err);
|
|
1368
1721
|
log(`#${issue} errored: ${detail}`);
|
|
1369
1722
|
// A crash lands anywhere, including mid-edit in a tree holding the only
|
|
@@ -1421,6 +1774,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1421
1774
|
}
|
|
1422
1775
|
verbListener = undefined;
|
|
1423
1776
|
}
|
|
1777
|
+
workerControl?.close();
|
|
1778
|
+
workerControl = undefined;
|
|
1424
1779
|
}
|
|
1425
1780
|
}
|
|
1426
1781
|
|
|
@@ -1538,6 +1893,129 @@ export function releaseInProgress(
|
|
|
1538
1893
|
* in the busy set throughout, so no second worker can be sent at the issue while
|
|
1539
1894
|
* it waits.
|
|
1540
1895
|
*/
|
|
1896
|
+
const BASE_CHECK_BATCH = 20;
|
|
1897
|
+
const BASE_CHECK_WINDOW_MS = 24 * 60 * 60 * 1_000;
|
|
1898
|
+
const BASE_STATUS_WINDOW_MS = 7 * 24 * 60 * 60 * 1_000;
|
|
1899
|
+
|
|
1900
|
+
const SUCCESSFUL_WORKFLOW_CONCLUSIONS = new Set(["success", "neutral", "skipped"]);
|
|
1901
|
+
|
|
1902
|
+
const FAILING_WORKFLOW_CONCLUSIONS = new Set([
|
|
1903
|
+
"failure",
|
|
1904
|
+
"cancelled",
|
|
1905
|
+
"timed_out",
|
|
1906
|
+
"action_required",
|
|
1907
|
+
"startup_failure",
|
|
1908
|
+
"stale",
|
|
1909
|
+
]);
|
|
1910
|
+
|
|
1911
|
+
function appendSettlementFlag(run: RunRecord, flag: SettlementFlag): SettlementFlag[] {
|
|
1912
|
+
const flags = run.settlementFlags ?? [];
|
|
1913
|
+
return flags.some((existing) => existing.kind === flag.kind && existing.detail === flag.detail)
|
|
1914
|
+
? flags
|
|
1915
|
+
: [...flags, flag];
|
|
1916
|
+
}
|
|
1917
|
+
|
|
1918
|
+
/**
|
|
1919
|
+
* Observe Actions on exact merge commits for up to one day. A running workflow
|
|
1920
|
+
* stays quiet and pending; a failure becomes durable evidence on the merged row
|
|
1921
|
+
* and pages exactly once because the row leaves `pending` before delivery.
|
|
1922
|
+
*/
|
|
1923
|
+
export async function watchMergedBase(d: Pick<Deps, "project" | "tracker" | "store" | "escalate">): Promise<void> {
|
|
1924
|
+
const now = Date.now();
|
|
1925
|
+
for (const run of d.store.runsNeedingBaseCheck(d.project.name, BASE_CHECK_BATCH)) {
|
|
1926
|
+
if (
|
|
1927
|
+
run.endedAt === undefined ||
|
|
1928
|
+
now - run.endedAt > BASE_CHECK_WINDOW_MS ||
|
|
1929
|
+
run.mergeSha === undefined ||
|
|
1930
|
+
run.baseRef === undefined
|
|
1931
|
+
) {
|
|
1932
|
+
d.store.updateRun(run.id, { baseCheck: "unknown", baseCheckAt: now });
|
|
1933
|
+
log(`#${run.issue} base check unknown: merge identity is absent or older than 24h`);
|
|
1934
|
+
continue;
|
|
1935
|
+
}
|
|
1936
|
+
|
|
1937
|
+
const repo = d.project.routing.repos[run.repo];
|
|
1938
|
+
const repoIdentity = repo === undefined ? undefined : githubRepo(repo.cloneUrl);
|
|
1939
|
+
if (repoIdentity === undefined) {
|
|
1940
|
+
d.store.updateRun(run.id, { baseCheck: "unknown", baseCheckAt: now });
|
|
1941
|
+
log(`#${run.issue} base check unknown: routed repository ${run.repo} has no GitHub identity`);
|
|
1942
|
+
continue;
|
|
1943
|
+
}
|
|
1944
|
+
|
|
1945
|
+
let workflows;
|
|
1946
|
+
try {
|
|
1947
|
+
workflows = await d.tracker.workflowRunsAt(repoIdentity, run.mergeSha);
|
|
1948
|
+
} catch (err) {
|
|
1949
|
+
log(`#${run.issue} base check unavailable (${errText(err)}) — retrying next tick`);
|
|
1950
|
+
continue;
|
|
1951
|
+
}
|
|
1952
|
+
if (workflows === undefined || workflows.length === 0) {
|
|
1953
|
+
log(`#${run.issue} base check unavailable for ${run.mergeSha} — retrying next tick`);
|
|
1954
|
+
continue;
|
|
1955
|
+
}
|
|
1956
|
+
if (workflows.some((workflow) => workflow.status !== "completed")) continue;
|
|
1957
|
+
|
|
1958
|
+
const failed = workflows.find(
|
|
1959
|
+
(workflow) =>
|
|
1960
|
+
workflow.conclusion !== undefined &&
|
|
1961
|
+
FAILING_WORKFLOW_CONCLUSIONS.has(workflow.conclusion),
|
|
1962
|
+
);
|
|
1963
|
+
if (failed === undefined) {
|
|
1964
|
+
const unknown = workflows.find(
|
|
1965
|
+
(workflow) =>
|
|
1966
|
+
workflow.conclusion === undefined ||
|
|
1967
|
+
!SUCCESSFUL_WORKFLOW_CONCLUSIONS.has(workflow.conclusion),
|
|
1968
|
+
);
|
|
1969
|
+
if (unknown !== undefined) {
|
|
1970
|
+
log(
|
|
1971
|
+
`#${run.issue} base check has unrecognised completed conclusion ` +
|
|
1972
|
+
`${JSON.stringify(unknown.conclusion)} for ${unknown.name} — retrying next tick`,
|
|
1973
|
+
);
|
|
1974
|
+
continue;
|
|
1975
|
+
}
|
|
1976
|
+
d.store.updateRun(run.id, { baseCheck: "green", baseCheckAt: now });
|
|
1977
|
+
log(`#${run.issue} base ${run.baseRef} green at ${run.mergeSha}`);
|
|
1978
|
+
continue;
|
|
1979
|
+
}
|
|
1980
|
+
|
|
1981
|
+
let previous;
|
|
1982
|
+
try {
|
|
1983
|
+
previous = await d.tracker.previousWorkflowRun(
|
|
1984
|
+
repoIdentity,
|
|
1985
|
+
failed.workflowId,
|
|
1986
|
+
run.baseRef,
|
|
1987
|
+
failed.createdAt,
|
|
1988
|
+
);
|
|
1989
|
+
} catch (err) {
|
|
1990
|
+
log(`#${run.issue} previous ${failed.name} run unavailable (${errText(err)}) — retrying next tick`);
|
|
1991
|
+
continue;
|
|
1992
|
+
}
|
|
1993
|
+
if (previous === undefined || (previous !== null && previous.status !== "completed")) continue;
|
|
1994
|
+
const preexisting =
|
|
1995
|
+
previous !== null &&
|
|
1996
|
+
previous.conclusion !== undefined &&
|
|
1997
|
+
FAILING_WORKFLOW_CONCLUSIONS.has(previous.conclusion);
|
|
1998
|
+
const detail =
|
|
1999
|
+
`${failed.name} failed at ${run.mergeSha} — ${failed.url}` +
|
|
2000
|
+
(preexisting ? " (already red before this merge)" : "");
|
|
2001
|
+
const flag: SettlementFlag = { kind: "base-branch-red", file: "(base branch)", detail };
|
|
2002
|
+
const delivered = await safeEscalate(d, {
|
|
2003
|
+
tier: 1,
|
|
2004
|
+
project: d.project.name,
|
|
2005
|
+
issue: run.issue,
|
|
2006
|
+
runId: run.id,
|
|
2007
|
+
summary: `Base branch ${run.baseRef} is red after merge`,
|
|
2008
|
+
detail,
|
|
2009
|
+
});
|
|
2010
|
+
if (!delivered) continue;
|
|
2011
|
+
d.store.updateRun(run.id, {
|
|
2012
|
+
baseCheck: preexisting ? "red-preexisting" : "red",
|
|
2013
|
+
baseCheckAt: now,
|
|
2014
|
+
settlementFlags: appendSettlementFlag(run, flag),
|
|
2015
|
+
});
|
|
2016
|
+
}
|
|
2017
|
+
}
|
|
2018
|
+
|
|
1541
2019
|
export async function settlePushedGreen(
|
|
1542
2020
|
d: Pick<Deps, "project" | "tracker" | "store">,
|
|
1543
2021
|
): Promise<void> {
|
|
@@ -1569,16 +2047,39 @@ export async function settlePushedGreen(
|
|
|
1569
2047
|
|
|
1570
2048
|
const settlement = settlementFor(pr, run.prUrl);
|
|
1571
2049
|
if (settlement !== undefined) {
|
|
2050
|
+
// A mediated merge enters a second, bounded observation phase. Record the
|
|
2051
|
+
// exact merge commit before the row leaves the active set; if GitHub cannot
|
|
2052
|
+
// supply it yet, retry this settlement next tick rather than create a
|
|
2053
|
+
// merged row whose base result can never be attributed.
|
|
2054
|
+
let merged: MergedPrInfo | undefined;
|
|
2055
|
+
if (settlement.state === "merged") {
|
|
2056
|
+
try {
|
|
2057
|
+
merged = await tracker.mergedPrInfo(run.prUrl);
|
|
2058
|
+
} catch (err) {
|
|
2059
|
+
log(`#${run.issue} not settled: merge identity lookup failed (${errText(err)}) — retrying next tick`);
|
|
2060
|
+
continue;
|
|
2061
|
+
}
|
|
2062
|
+
if (merged === undefined) {
|
|
2063
|
+
log(`#${run.issue} not settled: merge identity unavailable — retrying next tick`);
|
|
2064
|
+
continue;
|
|
2065
|
+
}
|
|
2066
|
+
}
|
|
2067
|
+
|
|
1572
2068
|
// The label removal and the terminal row are one fact again (#201): the
|
|
1573
2069
|
// release is enqueued — a durable local write that cannot fail on the
|
|
1574
|
-
// tracker — in the same breath as the row is terminalised
|
|
1575
|
-
// no window in which a row beyond every later tick still owes its label.
|
|
1576
|
-
// The projector applies it with retry; while pending, the eligibility
|
|
1577
|
-
// overlay treats the label as already gone, so #18's
|
|
1578
|
-
// permanent-`agent:in-progress` cannot re-form even when GitHub refuses
|
|
1579
|
-
// the write.
|
|
2070
|
+
// tracker — in the same breath as the row is terminalised.
|
|
1580
2071
|
releaseInProgress(d, run.issue, settlement.reason);
|
|
1581
|
-
const patch: Partial<RunRecord> = {
|
|
2072
|
+
const patch: Partial<RunRecord> = {
|
|
2073
|
+
state: settlement.state,
|
|
2074
|
+
endedAt: Date.now(),
|
|
2075
|
+
...(merged === undefined
|
|
2076
|
+
? {}
|
|
2077
|
+
: {
|
|
2078
|
+
mergeSha: merged.mergeSha,
|
|
2079
|
+
baseRef: merged.baseRef,
|
|
2080
|
+
baseCheck: "pending",
|
|
2081
|
+
}),
|
|
2082
|
+
};
|
|
1582
2083
|
if (settlement.state === "failed") patch.lastError = settlement.reason;
|
|
1583
2084
|
store.updateRun(run.id, patch);
|
|
1584
2085
|
log(`#${run.issue} settled: ${settlement.reason}`);
|
|
@@ -1611,6 +2112,121 @@ export async function settlePushedGreen(
|
|
|
1611
2112
|
}
|
|
1612
2113
|
}
|
|
1613
2114
|
|
|
2115
|
+
const ADOPTABLE_PR_STATES: Partial<Record<RunState, true>> = {
|
|
2116
|
+
failed: true,
|
|
2117
|
+
killed: true,
|
|
2118
|
+
orphaned: true,
|
|
2119
|
+
blocked: true,
|
|
2120
|
+
};
|
|
2121
|
+
|
|
2122
|
+
/**
|
|
2123
|
+
* Reattaches a recovered PR to the terminal run that owns it (#245).
|
|
2124
|
+
*
|
|
2125
|
+
* A worker can fail before its completion report records `prUrl`, then have its
|
|
2126
|
+
* dirty tree committed and pushed by salvage. If that branch already has a PR,
|
|
2127
|
+
* the orchestrator otherwise has no policy-compliant path to inspect or merge
|
|
2128
|
+
* it: ownership is store-backed. Adoption is deliberately stricter than
|
|
2129
|
+
* admission. The tracker query proves the PR closes this run's issue; exact
|
|
2130
|
+
* branch and canonical repository matches prove it is this run's recovered
|
|
2131
|
+
* work, not an unrelated closer. Missing identity is refusal, never a guess.
|
|
2132
|
+
*
|
|
2133
|
+
* The newest run per issue is inspected, at most ten per tick and only inside
|
|
2134
|
+
* the same 30-day window as mediated PR verbs. The cursor advances through the
|
|
2135
|
+
* full eligible set so persistent non-matches cannot starve older recovered
|
|
2136
|
+
* work. Successful adoption is idempotent because the row gains `prUrl`;
|
|
2137
|
+
* non-matches are logged once per daemon process.
|
|
2138
|
+
*/
|
|
2139
|
+
const rejectedSalvagedPrRuns = new Set<string>();
|
|
2140
|
+
const salvagedPrCursor = new Map<string, string>();
|
|
2141
|
+
|
|
2142
|
+
export async function adoptSalvagedPrs(
|
|
2143
|
+
d: Pick<Deps, "project" | "tracker" | "store">,
|
|
2144
|
+
now = Date.now(),
|
|
2145
|
+
): Promise<void> {
|
|
2146
|
+
const { project, tracker, store } = d;
|
|
2147
|
+
const eligible = store
|
|
2148
|
+
.recentRuns(project.name, now - PR_LOOKUP_WINDOW_MS)
|
|
2149
|
+
.filter(
|
|
2150
|
+
(run) =>
|
|
2151
|
+
ADOPTABLE_PR_STATES[run.state] === true &&
|
|
2152
|
+
run.prUrl === undefined &&
|
|
2153
|
+
run.branch.trim() !== "",
|
|
2154
|
+
);
|
|
2155
|
+
const previous = salvagedPrCursor.get(project.name);
|
|
2156
|
+
const previousIndex =
|
|
2157
|
+
previous === undefined ? -1 : eligible.findIndex((run) => run.id === previous);
|
|
2158
|
+
const start = previousIndex === -1 ? 0 : (previousIndex + 1) % eligible.length;
|
|
2159
|
+
const candidates = Array.from(
|
|
2160
|
+
{ length: Math.min(SALVAGED_PR_ADOPTION_BATCH, eligible.length) },
|
|
2161
|
+
(_, offset) => eligible[(start + offset) % eligible.length]!,
|
|
2162
|
+
);
|
|
2163
|
+
const last = candidates.at(-1);
|
|
2164
|
+
if (last !== undefined) salvagedPrCursor.set(project.name, last.id);
|
|
2165
|
+
|
|
2166
|
+
for (const run of candidates) {
|
|
2167
|
+
const repo = project.routing.repos[run.repo];
|
|
2168
|
+
const repoIdentity = repo === undefined ? undefined : githubRepo(repo.cloneUrl);
|
|
2169
|
+
let closers: OpenCloser[];
|
|
2170
|
+
try {
|
|
2171
|
+
closers = await tracker.openClosersFor(run.issue);
|
|
2172
|
+
} catch (err) {
|
|
2173
|
+
log(`#${run.issue} PR adoption lookup failed (${errText(err)}) — retrying next tick`);
|
|
2174
|
+
continue;
|
|
2175
|
+
}
|
|
2176
|
+
|
|
2177
|
+
const closer = closers.find(
|
|
2178
|
+
(candidate) =>
|
|
2179
|
+
candidate.headRefName !== "" &&
|
|
2180
|
+
candidate.headRefName === run.branch &&
|
|
2181
|
+
repo !== undefined &&
|
|
2182
|
+
candidate.repo !== "" &&
|
|
2183
|
+
candidate.repo === repoIdentity,
|
|
2184
|
+
);
|
|
2185
|
+
if (closer === undefined) {
|
|
2186
|
+
if (!rejectedSalvagedPrRuns.has(run.id)) {
|
|
2187
|
+
const observed = closers[0];
|
|
2188
|
+
const reason =
|
|
2189
|
+
observed === undefined
|
|
2190
|
+
? "no open closing PR"
|
|
2191
|
+
: observed.headRefName === ""
|
|
2192
|
+
? "closer has no head branch identity"
|
|
2193
|
+
: observed.headRefName !== run.branch
|
|
2194
|
+
? `closer head ${observed.headRefName} does not match retained branch ${run.branch}`
|
|
2195
|
+
: observed.repo === ""
|
|
2196
|
+
? "closer has no repository identity"
|
|
2197
|
+
: `closer repository ${observed.repo} does not match routed repository`;
|
|
2198
|
+
log(`#${run.issue} PR not adopted onto attempt ${run.attempt}: ${reason}`);
|
|
2199
|
+
rejectedSalvagedPrRuns.add(run.id);
|
|
2200
|
+
}
|
|
2201
|
+
continue;
|
|
2202
|
+
}
|
|
2203
|
+
|
|
2204
|
+
const flag: SettlementFlag = {
|
|
2205
|
+
kind: "pr-adopted",
|
|
2206
|
+
file: "(recovery)",
|
|
2207
|
+
detail: `${closer.url} matched retained branch ${run.branch} in ${closer.repo}`,
|
|
2208
|
+
};
|
|
2209
|
+
store.updateRun(run.id, {
|
|
2210
|
+
prUrl: closer.url,
|
|
2211
|
+
settlementFlags: [...(run.settlementFlags ?? []), flag],
|
|
2212
|
+
});
|
|
2213
|
+
rejectedSalvagedPrRuns.delete(run.id);
|
|
2214
|
+
log(
|
|
2215
|
+
`#${run.issue} adopted PR ${closer.url} onto attempt ${run.attempt}` +
|
|
2216
|
+
(run.salvageSha === undefined ? "" : ` (salvaged head ${run.salvageSha})`),
|
|
2217
|
+
);
|
|
2218
|
+
}
|
|
2219
|
+
}
|
|
2220
|
+
|
|
2221
|
+
/** Canonical `owner/repo` identity from a configured network clone URL. */
|
|
2222
|
+
function githubRepo(cloneUrl: string): string | undefined {
|
|
2223
|
+
const normalized = cloneUrl.replace(/\/$/, "").replace(/\.git$/, "");
|
|
2224
|
+
const match = /^(?:https?:\/\/[^/]+\/|ssh:\/\/git@[^/]+\/|git@[^:]+:)([^/\s]+\/[^/\s]+)$/.exec(
|
|
2225
|
+
normalized,
|
|
2226
|
+
);
|
|
2227
|
+
return match?.[1];
|
|
2228
|
+
}
|
|
2229
|
+
|
|
1614
2230
|
const RETAINED_CLEANUP_BATCH = 10;
|
|
1615
2231
|
|
|
1616
2232
|
export interface RetainedCleanupCursor {
|
|
@@ -1724,12 +2340,14 @@ const HOLD_SAMPLE_SIZE = 5;
|
|
|
1724
2340
|
const DEGRADED_HOLDS: ReadonlySet<AdmissionHoldReason> = new Set([
|
|
1725
2341
|
"parent-lookup-error",
|
|
1726
2342
|
"open-pr-lookup-error",
|
|
2343
|
+
"issue-state-lookup-error",
|
|
1727
2344
|
]);
|
|
1728
2345
|
|
|
1729
2346
|
/** Groups transient decisions into the bounded record exposed by status. */
|
|
1730
2347
|
export function summarizeDispatch(
|
|
1731
2348
|
ready: number,
|
|
1732
2349
|
routed: number,
|
|
2350
|
+
claimed: number,
|
|
1733
2351
|
admitted: number,
|
|
1734
2352
|
holds: readonly AdmissionHold[],
|
|
1735
2353
|
completedAt = Date.now(),
|
|
@@ -1745,6 +2363,7 @@ export function summarizeDispatch(
|
|
|
1745
2363
|
completedAt,
|
|
1746
2364
|
ready,
|
|
1747
2365
|
routed,
|
|
2366
|
+
claimed,
|
|
1748
2367
|
admitted,
|
|
1749
2368
|
degraded: holds.some((hold) => DEGRADED_HOLDS.has(hold.reason)),
|
|
1750
2369
|
holds: [...groups]
|
|
@@ -1792,6 +2411,7 @@ function planUsageEscalation(project: string, plan: PlanUsageStatus): Escalation
|
|
|
1792
2411
|
return {
|
|
1793
2412
|
...base,
|
|
1794
2413
|
tier: 2,
|
|
2414
|
+
category: "fleet-stopped",
|
|
1795
2415
|
// Dated: a meter that breaks again next month is a new incident, not a
|
|
1796
2416
|
// repeat of this one.
|
|
1797
2417
|
summary: `Plan usage source unreadable — ${project} is not claiming new work (${new Date().toISOString().slice(0, 10)})`,
|
|
@@ -1810,6 +2430,7 @@ function planUsageEscalation(project: string, plan: PlanUsageStatus): Escalation
|
|
|
1810
2430
|
return {
|
|
1811
2431
|
...base,
|
|
1812
2432
|
tier: 2,
|
|
2433
|
+
category: "fleet-stopped",
|
|
1813
2434
|
summary: `Plan usage cap names an unusable window "${plan.cap?.windowId ?? "?"}" — ${project} is not claiming new work`,
|
|
1814
2435
|
detail: [
|
|
1815
2436
|
plan.detail,
|
|
@@ -2104,6 +2725,32 @@ export async function admitCandidates(
|
|
|
2104
2725
|
log(`#${issue} continuing retained PR ${closer.url} ${resume}`);
|
|
2105
2726
|
}
|
|
2106
2727
|
|
|
2728
|
+
// The queue comes from GitHub's eventually-consistent search index. Re-read
|
|
2729
|
+
// state and labels directly at the last possible moment so a just-closed or
|
|
2730
|
+
// explicitly dequeued issue cannot turn a stale candidate into another
|
|
2731
|
+
// attempt (#247).
|
|
2732
|
+
let snapshot: IssueSnapshot | undefined;
|
|
2733
|
+
try {
|
|
2734
|
+
snapshot = await tracker.issueSnapshot(issue);
|
|
2735
|
+
} catch {
|
|
2736
|
+
snapshot = undefined;
|
|
2737
|
+
}
|
|
2738
|
+
if (snapshot === undefined) {
|
|
2739
|
+
hold(issue, "issue-state-lookup-error");
|
|
2740
|
+
log(`#${issue} held: issue snapshot check failed — retrying next tick`);
|
|
2741
|
+
continue;
|
|
2742
|
+
}
|
|
2743
|
+
if (snapshot.state === "closed") {
|
|
2744
|
+
hold(issue, "issue-closed");
|
|
2745
|
+
log(`#${issue} skipped: issue is closed (search index lag)`);
|
|
2746
|
+
continue;
|
|
2747
|
+
}
|
|
2748
|
+
if (!snapshot.labels.includes(project.queueLabel)) {
|
|
2749
|
+
hold(issue, "issue-dequeued");
|
|
2750
|
+
log(`#${issue} skipped: queue label ${project.queueLabel} was removed (search index lag)`);
|
|
2751
|
+
continue;
|
|
2752
|
+
}
|
|
2753
|
+
|
|
2107
2754
|
admitted.push({ r, attempt: priorRuns + 1 });
|
|
2108
2755
|
liveByRepo.set(r.repo.name, (liveByRepo.get(r.repo.name) ?? 0) + 1);
|
|
2109
2756
|
if (parent !== undefined) {
|
|
@@ -2192,6 +2839,16 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
2192
2839
|
// while the fleet is parked or workers are still active. Resident workers run
|
|
2193
2840
|
// through the pool without blocking this five-minute tick.
|
|
2194
2841
|
await settlePushedGreen(d);
|
|
2842
|
+
try {
|
|
2843
|
+
await watchMergedBase(d);
|
|
2844
|
+
} catch (err) {
|
|
2845
|
+
log(`base-branch check sweep failed: ${errText(err)}`);
|
|
2846
|
+
}
|
|
2847
|
+
try {
|
|
2848
|
+
await adoptSalvagedPrs(d);
|
|
2849
|
+
} catch (err) {
|
|
2850
|
+
log(`salvaged PR adoption sweep failed: ${errText(err)}`);
|
|
2851
|
+
}
|
|
2195
2852
|
|
|
2196
2853
|
// Immediately after settlement and before any routing, so a class is on the
|
|
2197
2854
|
// row before the next dispatch decision reads its budgets (#132). Above the
|
|
@@ -2284,6 +2941,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
2284
2941
|
if (integrity.page) {
|
|
2285
2942
|
const delivered = await safeEscalate(d, {
|
|
2286
2943
|
tier: 2,
|
|
2944
|
+
category: "fleet-stopped",
|
|
2287
2945
|
project: project.name,
|
|
2288
2946
|
issue: NO_ISSUE,
|
|
2289
2947
|
// Dated for the same reason the spend cap is: the dedup key is the
|
|
@@ -2327,6 +2985,10 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
2327
2985
|
return pending.length === 0 ? issue : { ...issue, labels: effectiveLabels(issue.labels, pending) };
|
|
2328
2986
|
});
|
|
2329
2987
|
const { routed, unroutable } = route(effective, project);
|
|
2988
|
+
// Ready issues that route() dropped without a reason carry a state label —
|
|
2989
|
+
// claimed by a live or settling run — so the claimed count is subtraction
|
|
2990
|
+
// rather than a routed-state signal (#228).
|
|
2991
|
+
const claimed = effective.length - routed.length - unroutable.length;
|
|
2330
2992
|
const routingHolds: AdmissionHold[] = unroutable.map((u) => ({
|
|
2331
2993
|
issue: u.issue.number,
|
|
2332
2994
|
reason: `unroutable:${u.reason}`,
|
|
@@ -2334,7 +2996,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
2334
2996
|
const recordDispatch = (admitted: number, holds: readonly AdmissionHold[]): void => {
|
|
2335
2997
|
store.recordDispatch(
|
|
2336
2998
|
project.name,
|
|
2337
|
-
summarizeDispatch(ready.length, routed.length, admitted, holds),
|
|
2999
|
+
summarizeDispatch(ready.length, routed.length, claimed, admitted, holds),
|
|
2338
3000
|
);
|
|
2339
3001
|
};
|
|
2340
3002
|
|
|
@@ -2368,6 +3030,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
2368
3030
|
setPaused(true, { source: "spend-cap", reason: `daily spend reached $${caps.dailySpendUsd}` });
|
|
2369
3031
|
await safeEscalate(d, {
|
|
2370
3032
|
tier: 2,
|
|
3033
|
+
category: "fleet-stopped",
|
|
2371
3034
|
project: project.name,
|
|
2372
3035
|
issue: NO_ISSUE,
|
|
2373
3036
|
// Dated so the same cap pages again tomorrow, but only once per day.
|
|
@@ -2429,10 +3092,14 @@ export interface DaemonHealthSnapshot {
|
|
|
2429
3092
|
paused: boolean;
|
|
2430
3093
|
activeRuns: number;
|
|
2431
3094
|
project: string;
|
|
3095
|
+
/** One-shot issue ceilings waiting for the next claim. */
|
|
3096
|
+
turnOverrides: TurnOverride[];
|
|
2432
3097
|
/** Resident set of this daemon; workers are in-process omp sessions. */
|
|
2433
3098
|
rssBytes: number;
|
|
2434
3099
|
dispatch?: DispatchSummary;
|
|
2435
3100
|
codeGraph?: CodeGraphHealth;
|
|
3101
|
+
/** Live workers in a non-running pause phase; absent/empty = nothing paused. */
|
|
3102
|
+
workers?: { issue: number; runId: string; phase: WorkerPausePhase }[];
|
|
2436
3103
|
}
|
|
2437
3104
|
|
|
2438
3105
|
export function daemonHealthSnapshot(
|
|
@@ -2441,24 +3108,35 @@ export function daemonHealthSnapshot(
|
|
|
2441
3108
|
paused = isPaused(),
|
|
2442
3109
|
rssBytes = process.memoryUsage().rss,
|
|
2443
3110
|
codeGraph?: CodeGraphHealth,
|
|
3111
|
+
workerControls?: WorkerControlRegistry,
|
|
2444
3112
|
): DaemonHealthSnapshot {
|
|
2445
3113
|
const dispatch = store.latestDispatch(project);
|
|
2446
3114
|
return {
|
|
2447
3115
|
ok: true,
|
|
2448
3116
|
paused,
|
|
2449
3117
|
activeRuns: store.activeRuns(project).length,
|
|
3118
|
+
turnOverrides: store.listTurnOverrides(project),
|
|
2450
3119
|
project,
|
|
2451
3120
|
rssBytes,
|
|
2452
3121
|
...(dispatch === undefined ? {} : { dispatch }),
|
|
2453
3122
|
...(codeGraph?.configured === true ? { codeGraph } : {}),
|
|
3123
|
+
...(workerControls === undefined ? {} : { workers: workerControls.snapshot(project) }),
|
|
2454
3124
|
};
|
|
2455
3125
|
}
|
|
2456
3126
|
|
|
3127
|
+
const TURN_OVERRIDE_STATES: ReadonlySet<RunState> = new Set([
|
|
3128
|
+
"failed",
|
|
3129
|
+
"killed",
|
|
3130
|
+
"orphaned",
|
|
3131
|
+
"blocked",
|
|
3132
|
+
]);
|
|
3133
|
+
|
|
2457
3134
|
export async function turnLimitResponse(
|
|
2458
3135
|
req: Request,
|
|
2459
3136
|
project: string,
|
|
2460
|
-
store: Pick<Store, "latestRun">,
|
|
3137
|
+
store: Pick<Store, "latestRun" | "setTurnOverride">,
|
|
2461
3138
|
registry: TurnLimitRegistry,
|
|
3139
|
+
caps: Pick<Caps, "workerMaxTurns" | "workerMaxTurnsCeiling">,
|
|
2462
3140
|
): Promise<Response | undefined> {
|
|
2463
3141
|
const url = new URL(req.url);
|
|
2464
3142
|
const match = /^\/runs\/(\d+)\/turn-limit$/.exec(url.pathname);
|
|
@@ -2492,6 +3170,16 @@ export async function turnLimitResponse(
|
|
|
2492
3170
|
}
|
|
2493
3171
|
|
|
2494
3172
|
const issue = Number(match[1]);
|
|
3173
|
+
if ((maxTurns as number) > caps.workerMaxTurnsCeiling) {
|
|
3174
|
+
return Response.json(
|
|
3175
|
+
{
|
|
3176
|
+
error:
|
|
3177
|
+
`#${issue} turn budget ${maxTurns as number} exceeds the ` +
|
|
3178
|
+
`${caps.workerMaxTurnsCeiling}-turn caps.workerMaxTurnsCeiling`,
|
|
3179
|
+
},
|
|
3180
|
+
{ status: 422 },
|
|
3181
|
+
);
|
|
3182
|
+
}
|
|
2495
3183
|
const outcome = registry.extend(project, issue, maxTurns as number);
|
|
2496
3184
|
if (outcome.kind === "extended") return Response.json(outcome);
|
|
2497
3185
|
if (outcome.kind === "not-increase") {
|
|
@@ -2505,6 +3193,132 @@ export async function turnLimitResponse(
|
|
|
2505
3193
|
if (latest === undefined) {
|
|
2506
3194
|
return Response.json({ error: `no run recorded for #${issue}` }, { status: 404 });
|
|
2507
3195
|
}
|
|
3196
|
+
if (!TURN_OVERRIDE_STATES.has(latest.state)) {
|
|
3197
|
+
return Response.json(
|
|
3198
|
+
{
|
|
3199
|
+
error:
|
|
3200
|
+
`#${issue} has no live worker controller; its session already settled ` +
|
|
3201
|
+
`or belongs to another daemon (stored state: ${latest.state})`,
|
|
3202
|
+
},
|
|
3203
|
+
{ status: 409 },
|
|
3204
|
+
);
|
|
3205
|
+
}
|
|
3206
|
+
if ((maxTurns as number) <= caps.workerMaxTurns) {
|
|
3207
|
+
return Response.json(
|
|
3208
|
+
{
|
|
3209
|
+
error:
|
|
3210
|
+
`#${issue} next-attempt turn budget must exceed the ` +
|
|
3211
|
+
`${caps.workerMaxTurns}-turn caps.workerMaxTurns base`,
|
|
3212
|
+
},
|
|
3213
|
+
{ status: 409 },
|
|
3214
|
+
);
|
|
3215
|
+
}
|
|
3216
|
+
store.setTurnOverride(project, issue, maxTurns as number);
|
|
3217
|
+
return Response.json({
|
|
3218
|
+
kind: "next-attempt",
|
|
3219
|
+
issue,
|
|
3220
|
+
nextAttemptMaxTurns: maxTurns,
|
|
3221
|
+
baseMaxTurns: caps.workerMaxTurns,
|
|
3222
|
+
});
|
|
3223
|
+
}
|
|
3224
|
+
|
|
3225
|
+
export async function workerControlResponse(
|
|
3226
|
+
req: Request,
|
|
3227
|
+
project: string,
|
|
3228
|
+
store: Pick<Store, "latestRun">,
|
|
3229
|
+
registry: WorkerControlRegistry,
|
|
3230
|
+
): Promise<Response | undefined> {
|
|
3231
|
+
const url = new URL(req.url);
|
|
3232
|
+
const match = /^\/runs\/(\d+)\/(pause|resume|stop)$/.exec(url.pathname);
|
|
3233
|
+
if (req.method !== "PUT" || match === null) return undefined;
|
|
3234
|
+
if (!req.headers.get("content-type")?.startsWith("application/json")) {
|
|
3235
|
+
return Response.json({ error: "content-type must be application/json" }, { status: 415 });
|
|
3236
|
+
}
|
|
3237
|
+
|
|
3238
|
+
let body: unknown;
|
|
3239
|
+
try {
|
|
3240
|
+
body = await req.json();
|
|
3241
|
+
} catch {
|
|
3242
|
+
return Response.json({ error: "request body must be valid JSON" }, { status: 400 });
|
|
3243
|
+
}
|
|
3244
|
+
if (body === null || typeof body !== "object") {
|
|
3245
|
+
return Response.json({ error: "request body must be a JSON object" }, { status: 400 });
|
|
3246
|
+
}
|
|
3247
|
+
const requestedProject = Reflect.get(body, "project");
|
|
3248
|
+
if (typeof requestedProject !== "string" || requestedProject.length === 0) {
|
|
3249
|
+
return Response.json({ error: "project must be a non-empty string" }, { status: 400 });
|
|
3250
|
+
}
|
|
3251
|
+
if (requestedProject !== project) {
|
|
3252
|
+
return Response.json(
|
|
3253
|
+
{ error: `daemon serves project "${project}", not requested project "${requestedProject}"` },
|
|
3254
|
+
{ status: 409 },
|
|
3255
|
+
);
|
|
3256
|
+
}
|
|
3257
|
+
const action = match[2] as "pause" | "resume" | "stop";
|
|
3258
|
+
let reason: string | undefined;
|
|
3259
|
+
if (action === "stop") {
|
|
3260
|
+
const rawReason = Reflect.get(body, "reason");
|
|
3261
|
+
if (typeof rawReason !== "string" || rawReason.trim() === "") {
|
|
3262
|
+
return Response.json({ error: "reason must be a non-empty string" }, { status: 400 });
|
|
3263
|
+
}
|
|
3264
|
+
reason = rawReason.trim().replace(/\s+/g, " ");
|
|
3265
|
+
if (reason.length > 500) {
|
|
3266
|
+
return Response.json({ error: "reason must be at most 500 characters" }, { status: 400 });
|
|
3267
|
+
}
|
|
3268
|
+
}
|
|
3269
|
+
|
|
3270
|
+
|
|
3271
|
+
const issue = Number(match[1]);
|
|
3272
|
+
const outcome =
|
|
3273
|
+
action === "pause"
|
|
3274
|
+
? await registry.pause(project, issue)
|
|
3275
|
+
: action === "resume"
|
|
3276
|
+
? registry.resume(project, issue)
|
|
3277
|
+
: await registry.stop(project, issue, reason!);
|
|
3278
|
+
if (outcome.kind === "ok") {
|
|
3279
|
+
return Response.json({ runId: outcome.runId, phase: outcome.phase });
|
|
3280
|
+
}
|
|
3281
|
+
if (outcome.kind === "refused") {
|
|
3282
|
+
return Response.json({ error: `#${issue}: ${outcome.error}` }, { status: 409 });
|
|
3283
|
+
}
|
|
3284
|
+
if (outcome.kind === "stopped") {
|
|
3285
|
+
const stopped = store.latestRun(project, issue);
|
|
3286
|
+
if (stopped === undefined || stopped.id !== outcome.runId) {
|
|
3287
|
+
return Response.json(
|
|
3288
|
+
{ error: `#${issue} stopped, but its terminal run record is unavailable` },
|
|
3289
|
+
{ status: 500 },
|
|
3290
|
+
);
|
|
3291
|
+
}
|
|
3292
|
+
if (stopped.state !== "stopped") {
|
|
3293
|
+
return Response.json({
|
|
3294
|
+
outcome: "already-terminal",
|
|
3295
|
+
runId: stopped.id,
|
|
3296
|
+
state: stopped.state,
|
|
3297
|
+
});
|
|
3298
|
+
}
|
|
3299
|
+
return Response.json({
|
|
3300
|
+
outcome: "stopped",
|
|
3301
|
+
runId: stopped.id,
|
|
3302
|
+
state: stopped.state,
|
|
3303
|
+
reason: outcome.reason,
|
|
3304
|
+
...(stopped.salvageSha === undefined ? {} : { salvageSha: stopped.salvageSha }),
|
|
3305
|
+
...(stopped.salvageError === undefined ? {} : { salvageError: stopped.salvageError }),
|
|
3306
|
+
worktree: stopped.worktree,
|
|
3307
|
+
});
|
|
3308
|
+
}
|
|
3309
|
+
|
|
3310
|
+
|
|
3311
|
+
const latest = store.latestRun(project, issue);
|
|
3312
|
+
if (latest === undefined) {
|
|
3313
|
+
return Response.json({ error: `no run recorded for #${issue}` }, { status: 404 });
|
|
3314
|
+
}
|
|
3315
|
+
if (action === "stop" && !LIVE_STATES.includes(latest.state)) {
|
|
3316
|
+
return Response.json({
|
|
3317
|
+
outcome: "already-terminal",
|
|
3318
|
+
runId: latest.id,
|
|
3319
|
+
state: latest.state,
|
|
3320
|
+
});
|
|
3321
|
+
}
|
|
2508
3322
|
return Response.json(
|
|
2509
3323
|
{
|
|
2510
3324
|
error:
|
|
@@ -2517,8 +3331,10 @@ export async function turnLimitResponse(
|
|
|
2517
3331
|
|
|
2518
3332
|
export interface DaemonHttpDeps {
|
|
2519
3333
|
project: string;
|
|
2520
|
-
store: Pick<Store, "latestRun">;
|
|
3334
|
+
store: Pick<Store, "latestRun" | "setTurnOverride">;
|
|
3335
|
+
caps: () => Pick<Caps, "workerMaxTurns" | "workerMaxTurnsCeiling">;
|
|
2521
3336
|
turnLimits: TurnLimitRegistry;
|
|
3337
|
+
workerControls: WorkerControlRegistry;
|
|
2522
3338
|
health: () => DaemonHealthSnapshot;
|
|
2523
3339
|
}
|
|
2524
3340
|
|
|
@@ -2526,17 +3342,18 @@ export interface DaemonHttpDeps {
|
|
|
2526
3342
|
* The whole HTTP surface, in one named function so a test can pin what is *not*
|
|
2527
3343
|
* on it.
|
|
2528
3344
|
*
|
|
2529
|
-
*
|
|
2530
|
-
*
|
|
2531
|
-
*
|
|
2532
|
-
*
|
|
2533
|
-
* a test asserts rather than something a reviewer has to notice (#126).
|
|
3345
|
+
* Three route families: the health read, turn-limit control, and live-worker
|
|
3346
|
+
* pause/resume/stop control. Everything else is 404. The controls mutate only
|
|
3347
|
+
* daemon-owned live sessions; tracker and repository mutations stay on the
|
|
3348
|
+
* authenticated per-run channel described at the `Bun.serve` call (#126).
|
|
2534
3349
|
*/
|
|
2535
3350
|
export async function daemonHttpResponse(req: Request, d: DaemonHttpDeps): Promise<Response> {
|
|
2536
3351
|
const url = new URL(req.url);
|
|
2537
3352
|
if (req.method === "GET" && url.pathname === "/healthz") return Response.json(d.health());
|
|
2538
|
-
const
|
|
2539
|
-
|
|
3353
|
+
const turnLimit = await turnLimitResponse(req, d.project, d.store, d.turnLimits, d.caps());
|
|
3354
|
+
if (turnLimit !== undefined) return turnLimit;
|
|
3355
|
+
const workerControl = await workerControlResponse(req, d.project, d.store, d.workerControls);
|
|
3356
|
+
return workerControl ?? new Response("not found\n", { status: 404 });
|
|
2540
3357
|
}
|
|
2541
3358
|
|
|
2542
3359
|
export interface StatusSnapshot {
|
|
@@ -2563,6 +3380,8 @@ export interface StatusSnapshot {
|
|
|
2563
3380
|
/** Newest attempts holding a preserved WIP tip, or a tree that is still the
|
|
2564
3381
|
* only copy of work the daemon could not save. */
|
|
2565
3382
|
salvagedRuns: RunRecord[];
|
|
3383
|
+
/** One-shot issue ceilings waiting for the next claim. */
|
|
3384
|
+
turnOverrides: TurnOverride[];
|
|
2566
3385
|
/** Reports the operator has not provably received: pending, in-flight with an
|
|
2567
3386
|
* unknown outcome, or written off. An empty list is the only honest way to
|
|
2568
3387
|
* say "everything authored this cycle actually went out" (#123). */
|
|
@@ -2574,6 +3393,8 @@ export interface StatusSnapshot {
|
|
|
2574
3393
|
* config does not let it, and an operator who has to know to go looking is an
|
|
2575
3394
|
* operator who finds out from the tracker instead.
|
|
2576
3395
|
*/
|
|
3396
|
+
/** Newest post-merge base verdict per routed repository within seven days. */
|
|
3397
|
+
baseChecks: RunRecord[];
|
|
2577
3398
|
verbLedger: VerbLedgerEntry[];
|
|
2578
3399
|
/** Runs backed by a worker process — the number capacity compares against. */
|
|
2579
3400
|
liveWorkers: number;
|
|
@@ -2637,6 +3458,7 @@ export function statusSnapshotFromStore(
|
|
|
2637
3458
|
releaseGrants: resolveReleaseGrants(p),
|
|
2638
3459
|
activeRuns: store.activeRuns(p.name),
|
|
2639
3460
|
salvagedRuns: store.salvagedRuns(p.name),
|
|
3461
|
+
turnOverrides: store.listTurnOverrides(p.name),
|
|
2640
3462
|
openReports: store.openReports(p.name),
|
|
2641
3463
|
verbLedger: store.verbLedger(p.name, { limit: STATUS_LEDGER_SCAN }),
|
|
2642
3464
|
liveWorkers: store.liveRuns(p.name).length,
|
|
@@ -2651,6 +3473,7 @@ export function statusSnapshotFromStore(
|
|
|
2651
3473
|
...(labelOpsPending === 0 || oldestLabelOpAt === undefined
|
|
2652
3474
|
? {}
|
|
2653
3475
|
: { labelOps: { pending: labelOpsPending, oldestAgeMs: Date.now() - oldestLabelOpAt } }),
|
|
3476
|
+
baseChecks: store.latestBaseChecks(p.name, Date.now() - BASE_STATUS_WINDOW_MS),
|
|
2654
3477
|
};
|
|
2655
3478
|
}
|
|
2656
3479
|
|
|
@@ -2672,7 +3495,7 @@ export function formatDispatchSummary(summary?: DispatchSummary): string {
|
|
|
2672
3495
|
if (summary === undefined) return "last dispatch (none recorded)";
|
|
2673
3496
|
const lines = [
|
|
2674
3497
|
`last dispatch ${new Date(summary.completedAt).toISOString()}${summary.degraded ? " DEGRADED" : ""}`,
|
|
2675
|
-
` candidates ${summary.ready} ready / ${summary.routed}
|
|
3498
|
+
` candidates ${summary.ready} ready / ${summary.claimed ?? 0} in flight / ${summary.routed} spare`,
|
|
2676
3499
|
` admitted ${summary.admitted}`,
|
|
2677
3500
|
];
|
|
2678
3501
|
if (summary.holds.length === 0) {
|
|
@@ -2733,6 +3556,21 @@ export function formatReleaseGrants(grants: ResolvedGrants): string[] {
|
|
|
2733
3556
|
...RELEASE_SHAPES.map((shape) => ` ${shape.padEnd(19)}${grants[shape]}`),
|
|
2734
3557
|
];
|
|
2735
3558
|
}
|
|
3559
|
+
export function formatBaseChecks(runs: readonly RunRecord[]): string[] {
|
|
3560
|
+
return runs.flatMap((run) => {
|
|
3561
|
+
if (run.baseCheck === undefined) return [];
|
|
3562
|
+
const branch = run.baseRef ?? "?";
|
|
3563
|
+
const flag = run.settlementFlags?.find((candidate) => candidate.kind === "base-branch-red");
|
|
3564
|
+
const verdict =
|
|
3565
|
+
run.baseCheck === "red"
|
|
3566
|
+
? `RED — ${flag?.detail ?? "workflow failed after merge"}`
|
|
3567
|
+
: run.baseCheck === "red-preexisting"
|
|
3568
|
+
? `red before merge${flag === undefined ? "" : ` — ${flag.detail}`}`
|
|
3569
|
+
: run.baseCheck;
|
|
3570
|
+
return [`base ${run.repo}/${branch} ${verdict}`];
|
|
3571
|
+
});
|
|
3572
|
+
}
|
|
3573
|
+
|
|
2736
3574
|
|
|
2737
3575
|
export function formatStatus(s: StatusSnapshot): string {
|
|
2738
3576
|
const lines = [
|
|
@@ -2751,6 +3589,9 @@ export function formatStatus(s: StatusSnapshot): string {
|
|
|
2751
3589
|
// fleet (#110).
|
|
2752
3590
|
` plan usage ${planUsageLine(s.planUsage)}`,
|
|
2753
3591
|
` new worker turns ${s.caps.workerMaxTurns}`,
|
|
3592
|
+
...s.turnOverrides.map(
|
|
3593
|
+
({ issue, maxTurns }) => ` turn override #${issue} → ${maxTurns} (next attempt)`,
|
|
3594
|
+
),
|
|
2754
3595
|
` worker wall clock ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
|
|
2755
3596
|
` failed attempts ${s.caps.maxAttemptsPerIssue}`,
|
|
2756
3597
|
` continuations ${s.caps.maxContinuationsPerIssue}`,
|
|
@@ -2777,6 +3618,7 @@ export function formatStatus(s: StatusSnapshot): string {
|
|
|
2777
3618
|
if (flagged !== undefined) lines.push(` ${flagged}`);
|
|
2778
3619
|
}
|
|
2779
3620
|
}
|
|
3621
|
+
lines.push(...formatBaseChecks(s.baseChecks));
|
|
2780
3622
|
lines.push(...formatSalvagedRuns(s.salvagedRuns));
|
|
2781
3623
|
lines.push(...formatOpenReports(s.openReports));
|
|
2782
3624
|
lines.push(...formatVerbLedger(s.verbLedger));
|
|
@@ -2912,6 +3754,15 @@ export interface SessionError {
|
|
|
2912
3754
|
message: string;
|
|
2913
3755
|
}
|
|
2914
3756
|
|
|
3757
|
+
export function completionLastError(
|
|
3758
|
+
providerCredit: string | undefined,
|
|
3759
|
+
providerTransient: string | undefined,
|
|
3760
|
+
verifiedReason: string | undefined,
|
|
3761
|
+
sessionErr: SessionError | undefined,
|
|
3762
|
+
): string | undefined {
|
|
3763
|
+
return providerCredit ?? providerTransient ?? verifiedReason ?? sessionErr?.message;
|
|
3764
|
+
}
|
|
3765
|
+
|
|
2915
3766
|
/**
|
|
2916
3767
|
* The last error a transcript recorded, or undefined when it recorded none.
|
|
2917
3768
|
*
|
|
@@ -2978,6 +3829,19 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
|
|
|
2978
3829
|
const { project, tracker, store } = d;
|
|
2979
3830
|
for (const run of store.runsNeedingClassification(project.name, CLASSIFY_BATCH)) {
|
|
2980
3831
|
const facts: ClassifyFacts = {};
|
|
3832
|
+
let classifiedRun = run;
|
|
3833
|
+
if (run.state === "failed" || run.state === "killed") {
|
|
3834
|
+
const sessionError = readSessionError(run.sessionFile);
|
|
3835
|
+
if (sessionError !== undefined) {
|
|
3836
|
+
if (run.lastError === undefined || run.lastError === sessionError.message) {
|
|
3837
|
+
facts.sessionError = sessionError;
|
|
3838
|
+
}
|
|
3839
|
+
if (run.lastError === undefined) {
|
|
3840
|
+
store.updateRun(run.id, { lastError: sessionError.message });
|
|
3841
|
+
classifiedRun = { ...run, lastError: sessionError.message };
|
|
3842
|
+
}
|
|
3843
|
+
}
|
|
3844
|
+
}
|
|
2981
3845
|
try {
|
|
2982
3846
|
if (run.prUrl !== undefined) {
|
|
2983
3847
|
const pr = await tracker.prState(run.prUrl);
|
|
@@ -3004,7 +3868,7 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
|
|
|
3004
3868
|
continue;
|
|
3005
3869
|
}
|
|
3006
3870
|
|
|
3007
|
-
const { cls, recovery, evidence } = classifyRun(
|
|
3871
|
+
const { cls, recovery, evidence } = classifyRun(classifiedRun, facts);
|
|
3008
3872
|
|
|
3009
3873
|
// A healthy green PR is not a failure of any class. Leaving the row
|
|
3010
3874
|
// unclassified is what keeps it eligible for the sweep on the tick where its
|
|
@@ -3018,7 +3882,7 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
|
|
|
3018
3882
|
? `#${run.issue} retrying ${recovery} for ${cls}: ${evidence}`
|
|
3019
3883
|
: `#${run.issue} classified ${cls} → ${recovery}: ${evidence}`,
|
|
3020
3884
|
);
|
|
3021
|
-
await recoverRun(d,
|
|
3885
|
+
await recoverRun(d, classifiedRun, cls, recovery, evidence);
|
|
3022
3886
|
}
|
|
3023
3887
|
}
|
|
3024
3888
|
|
|
@@ -3084,6 +3948,9 @@ async function recoverRun(
|
|
|
3084
3948
|
}
|
|
3085
3949
|
|
|
3086
3950
|
if (recovery === "requeue") {
|
|
3951
|
+
if (cls === "provider-credit") {
|
|
3952
|
+
await reactToProviderCredit(d, run.issue, evidence, run.sessionFile);
|
|
3953
|
+
}
|
|
3087
3954
|
// A dispatch-infra requeue that keeps landing on the same issue means the
|
|
3088
3955
|
// mirror for its repo is persistently broken — a ref-lock that retry already
|
|
3089
3956
|
// exhausted, a dead remote. Requeueing forever spends a turn-0 run per tick
|
|
@@ -3104,6 +3971,28 @@ async function recoverRun(
|
|
|
3104
3971
|
log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
|
|
3105
3972
|
return;
|
|
3106
3973
|
}
|
|
3974
|
+
// Same bound for provider-transient: an issue whose stream keeps stalling
|
|
3975
|
+
// mid-run is requeued free (no attempt, no continuation charged) — but a
|
|
3976
|
+
// provider that aborts three times for one issue is down, and a human has
|
|
3977
|
+
// to check its status before hand-requeueing (#220).
|
|
3978
|
+
if (
|
|
3979
|
+
cls === "provider-transient" &&
|
|
3980
|
+
store.classCountFor(project.name, run.issue, "provider-transient") >= PROVIDER_TRANSIENT_MAX_STRIKES
|
|
3981
|
+
) {
|
|
3982
|
+
await safeEscalate(d, {
|
|
3983
|
+
tier: 1,
|
|
3984
|
+
project: project.name,
|
|
3985
|
+
issue: run.issue,
|
|
3986
|
+
summary: `[provider-transient] #${run.issue}: the provider keeps aborting mid-stream — ${evidence}`,
|
|
3987
|
+
detail: [
|
|
3988
|
+
`The provider aborted the stream for #${run.issue} ${PROVIDER_TRANSIENT_MAX_STRIKES} times without the run ever producing a verdict (0 tokens billed each time).`,
|
|
3989
|
+
"Check provider status before requeueing by hand.",
|
|
3990
|
+
].join("\n"),
|
|
3991
|
+
});
|
|
3992
|
+
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
3993
|
+
log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
|
|
3994
|
+
return;
|
|
3995
|
+
}
|
|
3107
3996
|
// Only when the tracker still shows this issue as ours to hand back. An
|
|
3108
3997
|
// issue that is closed, or has no state label, was resolved by another route
|
|
3109
3998
|
// and requeueing it would dispatch work nobody asked for.
|
|
@@ -3476,7 +4365,8 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
3476
4365
|
onChildLog: (line) => {
|
|
3477
4366
|
log(`orchestrator ${line}`);
|
|
3478
4367
|
},
|
|
3479
|
-
onReleaseBlocked: (shape) =>
|
|
4368
|
+
onReleaseBlocked: (shape, context) =>
|
|
4369
|
+
recordReleaseBlock(project.name, "orchestrator", shape, context),
|
|
3480
4370
|
});
|
|
3481
4371
|
const transcript = orchestrator.sessionFile();
|
|
3482
4372
|
log(`orchestrator session ready${transcript === undefined ? "" : ` · ${transcript}`}`);
|
|
@@ -3521,6 +4411,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
3521
4411
|
const turnLimits = createTurnLimitRegistry((runId, maxTurns) => {
|
|
3522
4412
|
store.updateRun(runId, { maxTurns });
|
|
3523
4413
|
});
|
|
4414
|
+
const workerControls = createWorkerControlRegistry();
|
|
3524
4415
|
const d: Deps = {
|
|
3525
4416
|
project,
|
|
3526
4417
|
caps,
|
|
@@ -3531,6 +4422,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
3531
4422
|
usage: sharedUsageSource(),
|
|
3532
4423
|
escalate: (e) => escalator.escalate(e),
|
|
3533
4424
|
turnLimits,
|
|
4425
|
+
workerControls,
|
|
3534
4426
|
integrity,
|
|
3535
4427
|
// Fresh per daemon run, like the integrity gate: a restart is entitled to
|
|
3536
4428
|
// page again about a stall that is still on disk.
|
|
@@ -3607,14 +4499,17 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
3607
4499
|
process.on("SIGINT", stop);
|
|
3608
4500
|
process.on("SIGTERM", stop);
|
|
3609
4501
|
|
|
3610
|
-
// ── NO MUTATION
|
|
4502
|
+
// ── NO TRACKER OR REPOSITORY MUTATION BELONGS ON THIS PORT ───────────────
|
|
3611
4503
|
//
|
|
3612
4504
|
// This is unauthenticated loopback TCP. Every local user can reach it, it
|
|
3613
4505
|
// carries no credential of any kind, and it cannot tell one caller from
|
|
3614
|
-
// another: `127.0.0.1` is not an identity.
|
|
3615
|
-
//
|
|
3616
|
-
// shape "identity from the payload" takes when nobody is watching
|
|
3617
|
-
// tolerable only because
|
|
4506
|
+
// another: `127.0.0.1` is not an identity. The turn-limit and worker
|
|
4507
|
+
// pause/resume controls trust a body-supplied `project`, which is exactly the
|
|
4508
|
+
// shape "identity from the payload" takes when nobody is watching. They stay
|
|
4509
|
+
// tolerable only because every effect is bounded: pauses are reversible,
|
|
4510
|
+
// live extensions touch one controller, and a persisted next-attempt
|
|
4511
|
+
// override can only raise the project base up to its configured ceiling and
|
|
4512
|
+
// is consumed by one claim.
|
|
3618
4513
|
//
|
|
3619
4514
|
// A merge, a push, a release or a label is none of those things. Do not add
|
|
3620
4515
|
// one here, and do not add "just a small one" behind a shared secret either:
|
|
@@ -3629,8 +4524,11 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
3629
4524
|
daemonHttpResponse(req, {
|
|
3630
4525
|
project: project.name,
|
|
3631
4526
|
store,
|
|
4527
|
+
caps: () => d.caps,
|
|
3632
4528
|
turnLimits,
|
|
3633
|
-
|
|
4529
|
+
workerControls,
|
|
4530
|
+
health: () =>
|
|
4531
|
+
daemonHealthSnapshot(store, project.name, undefined, undefined, codeGraph, workerControls),
|
|
3634
4532
|
}),
|
|
3635
4533
|
});
|
|
3636
4534
|
log(`serving /healthz on :${server.port}, project ${project.name}`);
|