omp-conductor 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -19
- package/package.json +1 -1
- package/src/board.ts +1 -1
- package/src/briefs/orchestrator.md +32 -2
- package/src/cli.ts +104 -23
- package/src/config.ts +10 -2
- package/src/daemon.ts +818 -131
- package/src/diff-flags.ts +4 -0
- package/src/escalate.ts +5 -5
- package/src/failure-class.ts +7 -5
- package/src/fleet.ts +7 -2
- package/src/omp.ts +8 -4
- package/src/orchestrator-tick.ts +35 -20
- package/src/orchestrator.ts +3 -2
- package/src/plugin.ts +28 -2
- package/src/release-policy.ts +66 -6
- package/src/session-host.ts +4 -3
- package/src/setup.ts +4 -2
- package/src/store.ts +190 -29
- package/src/tracker/github.ts +261 -56
- package/src/types.ts +141 -28
- package/src/verbs/actions.ts +127 -15
- package/src/verbs/protocol.ts +7 -6
- package/src/verbs/server.ts +95 -13
- package/src/worker.ts +33 -5
- package/src/worktree.ts +5 -0
package/src/daemon.ts
CHANGED
|
@@ -31,13 +31,16 @@ import { STALL_MARKER_FILE } from "./orchestrator-tick.ts";
|
|
|
31
31
|
import { startOrchestrator } from "./orchestrator.ts";
|
|
32
32
|
import type { OrchestratorHandle } from "./orchestrator.ts";
|
|
33
33
|
import { createReportOutbox, formatOpenReports } from "./reports.ts";
|
|
34
|
-
import {
|
|
34
|
+
import {
|
|
35
|
+
recordReleaseBlock,
|
|
36
|
+
type ReleaseBlockContext,
|
|
37
|
+
} from "./release-policy.ts";
|
|
35
38
|
import { branchName, effectiveLabels, route } from "./routing.ts";
|
|
36
39
|
import type { Routed, UnroutableReason } from "./routing.ts";
|
|
37
40
|
import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
|
|
38
41
|
import { classifyRun, providerCreditRefusal, providerTransientFault, type ClassifyFacts } from "./failure-class.ts";
|
|
39
42
|
import { projectLabels } from "./label-projection.ts";
|
|
40
|
-
import { dbPath, openStore, utcDay } from "./store.ts";
|
|
43
|
+
import { dbPath, LIVE_STATES, openStore, utcDay } from "./store.ts";
|
|
41
44
|
import { makeTracker, type RateLimitStatus } from "./tracker/github.ts";
|
|
42
45
|
import { RELEASE_SHAPES } from "./types.ts";
|
|
43
46
|
import type {
|
|
@@ -45,7 +48,10 @@ import type {
|
|
|
45
48
|
Caps,
|
|
46
49
|
DispatchSummary,
|
|
47
50
|
Escalation,
|
|
51
|
+
IssueSnapshot,
|
|
52
|
+
MergedPrInfo,
|
|
48
53
|
OpenCloser,
|
|
54
|
+
ReleaseShape,
|
|
49
55
|
PrState,
|
|
50
56
|
ProjectConfig,
|
|
51
57
|
ReadyIssue,
|
|
@@ -60,12 +66,14 @@ import type {
|
|
|
60
66
|
Store,
|
|
61
67
|
Tracker,
|
|
62
68
|
VerbLedgerEntry,
|
|
69
|
+
TurnOverride,
|
|
63
70
|
} from "./types.ts";
|
|
64
71
|
import {
|
|
65
72
|
type KilledBy,
|
|
66
73
|
type WorkerPauseControl,
|
|
67
74
|
type WorkerPausePhase,
|
|
68
75
|
type WorkerResult,
|
|
76
|
+
type RunWorkerDeps,
|
|
69
77
|
renderBrief,
|
|
70
78
|
runWorker,
|
|
71
79
|
} from "./worker.ts";
|
|
@@ -82,8 +90,13 @@ import {
|
|
|
82
90
|
} from "./worktree.ts";
|
|
83
91
|
import { githubVerbActions } from "./verbs/actions.ts";
|
|
84
92
|
import { formatVerbLedger, STATUS_LEDGER_SCAN } from "./verbs/ledger.ts";
|
|
85
|
-
import {
|
|
86
|
-
|
|
93
|
+
import {
|
|
94
|
+
listenVerbChannel,
|
|
95
|
+
PR_LOOKUP_WINDOW_MS,
|
|
96
|
+
type VerbActions,
|
|
97
|
+
type VerbDeps,
|
|
98
|
+
type VerbListener,
|
|
99
|
+
} from "./verbs/server.ts";
|
|
87
100
|
import {
|
|
88
101
|
ensureVerbSocketDir,
|
|
89
102
|
peerCredentialReader,
|
|
@@ -120,6 +133,9 @@ const DISPATCH_INFRA_MAX_STRIKES = 3;
|
|
|
120
133
|
* provider itself is degraded, not unlucky, and the sweep escalates to a
|
|
121
134
|
* human instead of requeueing into a down provider forever (#220). */
|
|
122
135
|
const PROVIDER_TRANSIENT_MAX_STRIKES = 3;
|
|
136
|
+
/** Terminal rows inspected for a salvaged PR per tick. GitHub provenance reads
|
|
137
|
+
* are maintenance, but a backlog must not turn one tick into an API burst. */
|
|
138
|
+
const SALVAGED_PR_ADOPTION_BATCH = 10;
|
|
123
139
|
const DEFAULT_PORT = 8787;
|
|
124
140
|
const BRIEF_TEMPLATE_PATH = join(import.meta.dir, "briefs", "worker.md");
|
|
125
141
|
|
|
@@ -157,6 +173,8 @@ interface Deps {
|
|
|
157
173
|
escalate(e: Escalation): Promise<void>;
|
|
158
174
|
turnLimits: TurnLimitRegistry;
|
|
159
175
|
workerControls: WorkerControlRegistry;
|
|
176
|
+
/** Session seam for lifecycle integration tests; production uses the real harness. */
|
|
177
|
+
workerDeps?: RunWorkerDeps;
|
|
160
178
|
integrity: IntegrityGate;
|
|
161
179
|
stall: StallGate;
|
|
162
180
|
cleanup?: RetainedCleanupCursor;
|
|
@@ -212,18 +230,25 @@ export function verbDeps(d: Pick<Deps, "project" | "store" | "tracker" | "verbAc
|
|
|
212
230
|
|
|
213
231
|
// ------------------------------------------------------- orchestrator liveness
|
|
214
232
|
|
|
233
|
+
/** No more than one reminder per hour while the same marker remains. */
|
|
234
|
+
const STALL_RENOTIFY_MS = 60 * 60_000;
|
|
235
|
+
|
|
215
236
|
/**
|
|
216
|
-
* Whether the wedged-orchestrator page has
|
|
217
|
-
* currently on disk.
|
|
218
|
-
* consumed, so paging per five minutes would be paging forever.
|
|
237
|
+
* Whether the wedged-orchestrator page has gone out, and when it last went out,
|
|
238
|
+
* for the stall currently on disk.
|
|
219
239
|
*/
|
|
220
240
|
export interface StallGate {
|
|
221
241
|
paged: boolean;
|
|
242
|
+
lastPagedAt?: number;
|
|
222
243
|
}
|
|
223
244
|
|
|
224
245
|
export interface StallVerdict {
|
|
225
246
|
/** The marker's own line, when one is there. */
|
|
226
247
|
since?: string;
|
|
248
|
+
/** Count written by the stalled tick producer, when its marker is readable. */
|
|
249
|
+
unconsumedTicks?: number;
|
|
250
|
+
/** Whole hours elapsed since the marker timestamp, when it is parseable. */
|
|
251
|
+
stalledHours?: number;
|
|
227
252
|
page: boolean;
|
|
228
253
|
}
|
|
229
254
|
|
|
@@ -246,51 +271,84 @@ export interface StallVerdict {
|
|
|
246
271
|
*
|
|
247
272
|
* Resets when the marker disappears, so a second stall days later pages again.
|
|
248
273
|
*/
|
|
249
|
-
export function checkStall(gate: StallGate, marker: string): StallVerdict {
|
|
274
|
+
export function checkStall(gate: StallGate, marker: string, now = Date.now()): StallVerdict {
|
|
250
275
|
if (!existsSync(marker)) {
|
|
251
276
|
gate.paged = false;
|
|
277
|
+
delete gate.lastPagedAt;
|
|
252
278
|
return { page: false };
|
|
253
279
|
}
|
|
254
|
-
const page =
|
|
280
|
+
const page =
|
|
281
|
+
!gate.paged ||
|
|
282
|
+
gate.lastPagedAt === undefined ||
|
|
283
|
+
now - gate.lastPagedAt >= STALL_RENOTIFY_MS;
|
|
255
284
|
let since: string | undefined;
|
|
285
|
+
let unconsumedTicks: number | undefined;
|
|
286
|
+
let stalledHours: number | undefined;
|
|
256
287
|
try {
|
|
257
288
|
const body = readFileSync(marker, "utf8").split("\n")[0]?.trim();
|
|
258
|
-
if (body !== undefined && body !== "")
|
|
289
|
+
if (body !== undefined && body !== "") {
|
|
290
|
+
since = body;
|
|
291
|
+
const ticks = /\b(\d+) ticks queued unconsumed\b/.exec(body)?.[1];
|
|
292
|
+
if (ticks !== undefined) unconsumedTicks = Number(ticks);
|
|
293
|
+
const startedAt = Date.parse(body.split(/\s+/, 1)[0] ?? "");
|
|
294
|
+
if (Number.isFinite(startedAt)) {
|
|
295
|
+
stalledHours = Math.max(0, Math.floor((now - startedAt) / STALL_RENOTIFY_MS));
|
|
296
|
+
}
|
|
297
|
+
}
|
|
259
298
|
} catch {
|
|
260
|
-
// An unreadable marker still means stalled;
|
|
299
|
+
// An unreadable marker still means stalled; its evidence is a nicety.
|
|
261
300
|
}
|
|
262
|
-
return {
|
|
301
|
+
return {
|
|
302
|
+
...(since === undefined ? {} : { since }),
|
|
303
|
+
...(unconsumedTicks === undefined ? {} : { unconsumedTicks }),
|
|
304
|
+
...(stalledHours === undefined ? {} : { stalledHours }),
|
|
305
|
+
page,
|
|
306
|
+
};
|
|
263
307
|
}
|
|
264
308
|
|
|
265
309
|
/**
|
|
266
|
-
* Pages tier 2
|
|
310
|
+
* Pages tier 2 when the orchestrator session stops draining its queue.
|
|
267
311
|
*
|
|
268
312
|
* Deliberately does not restart anything. A wedge lands mid-turn, this process
|
|
269
313
|
* cannot tell a half-applied edit from an idle loop, and killing the session
|
|
270
314
|
* could destroy work an operator would rather read first — the same refusal to
|
|
271
315
|
* guess that the recovery plugin is built on.
|
|
272
316
|
*/
|
|
273
|
-
async function watchOrchestrator(d: Deps): Promise<void> {
|
|
317
|
+
export async function watchOrchestrator(d: Deps, now = Date.now()): Promise<void> {
|
|
274
318
|
const marker = join(stateDir(), STALL_MARKER_FILE);
|
|
275
|
-
const
|
|
319
|
+
const repeat = d.stall.paged;
|
|
320
|
+
const verdict = checkStall(d.stall, marker, now);
|
|
276
321
|
if (!verdict.page) return;
|
|
277
322
|
|
|
323
|
+
const greenRuns = d.store.activeRuns(d.project.name).filter((run) => run.state === "pushed-green");
|
|
324
|
+
const repeatSuffix = repeat
|
|
325
|
+
? ` — still stalled (${
|
|
326
|
+
verdict.stalledHours === undefined
|
|
327
|
+
? `${new Date(now).toISOString().slice(0, 13)}Z`
|
|
328
|
+
: `${verdict.stalledHours}h`
|
|
329
|
+
})`
|
|
330
|
+
: "";
|
|
331
|
+
|
|
278
332
|
log(`ERROR: the orchestrator session is not draining its queue — ${verdict.since ?? "no timestamp"}`);
|
|
279
333
|
const delivered = await safeEscalate(d, {
|
|
280
334
|
tier: 2,
|
|
281
335
|
category: "confirmed-failure",
|
|
336
|
+
urgent: true,
|
|
282
337
|
project: d.project.name,
|
|
283
338
|
issue: NO_ISSUE,
|
|
284
339
|
// Keyed on the marker's own timestamp, not the date. The dedup ledger keys
|
|
285
340
|
// on this summary, and two wedges in one day is not a hypothetical — the
|
|
286
341
|
// failure mode is a session that gets stuck, gets restarted, and gets stuck
|
|
287
|
-
// again on the same cause an hour later.
|
|
288
|
-
//
|
|
342
|
+
// again on the same cause an hour later. The hourly suffix makes bounded
|
|
343
|
+
// reminders distinct without allowing every five-minute tick through.
|
|
289
344
|
summary:
|
|
290
345
|
`Orchestrator session wedged (${verdict.since ?? `marker at ${marker}`}) — ` +
|
|
291
|
-
`it has stopped reading its queue (${d.project.name})`,
|
|
346
|
+
`it has stopped reading its queue (${d.project.name})${repeatSuffix}`,
|
|
292
347
|
detail: [
|
|
293
348
|
verdict.since ?? "Marker present with no readable timestamp.",
|
|
349
|
+
`Unconsumed ticks: ${verdict.unconsumedTicks ?? "unknown"}`,
|
|
350
|
+
`Open green worker PRs: ${greenRuns.length}`,
|
|
351
|
+
...greenRuns.map((run) => `- ${run.prUrl ?? `#${run.issue} (PR URL unavailable)`}`),
|
|
294
352
|
`Marker: ${marker}`,
|
|
295
353
|
"",
|
|
296
354
|
"Its process and its herdr agent label are both healthy, which is why nothing else noticed:",
|
|
@@ -302,7 +360,7 @@ async function watchOrchestrator(d: Deps): Promise<void> {
|
|
|
302
360
|
"Dispatch is unaffected: workers keep running. What stops is drain, groom, report and merge.",
|
|
303
361
|
].join("\n"),
|
|
304
362
|
});
|
|
305
|
-
markPaged(d.stall, delivered);
|
|
363
|
+
markPaged(d.stall, delivered, now);
|
|
306
364
|
}
|
|
307
365
|
|
|
308
366
|
/**
|
|
@@ -319,10 +377,10 @@ export function isPaused(): boolean {
|
|
|
319
377
|
* The epoch-ms timestamp at which the current pause began, read from the same
|
|
320
378
|
* sentinel file {@link setPaused} writes (`<stateDir()>/paused`). Returns
|
|
321
379
|
* `undefined` when the fleet is not paused, or when the file's first line does
|
|
322
|
-
* not parse as a date
|
|
323
|
-
*
|
|
324
|
-
* innocent. {@link isPaused} is the authority on *whether*; this answers
|
|
325
|
-
*
|
|
380
|
+
* not parse as a date. A legacy/blank sentinel keeps completion mutations
|
|
381
|
+
* fail-closed because a run admitted before an *unknown* pause cannot be proven
|
|
382
|
+
* innocent. {@link isPaused} is the authority on *whether*; this answers *since
|
|
383
|
+
* when*.
|
|
326
384
|
*/
|
|
327
385
|
export function pausedAt(): number | undefined {
|
|
328
386
|
const f = join(stateDir(), "paused");
|
|
@@ -333,8 +391,8 @@ export function pausedAt(): number | undefined {
|
|
|
333
391
|
const t = Date.parse(first);
|
|
334
392
|
return Number.isNaN(t) ? undefined : t;
|
|
335
393
|
} catch {
|
|
336
|
-
// Unreadable sentinel (permissions, corruption): fail
|
|
337
|
-
//
|
|
394
|
+
// Unreadable sentinel (permissions, corruption): fail completion mutations
|
|
395
|
+
// closed like an unparseable line while the pause time is unprovable.
|
|
338
396
|
return undefined;
|
|
339
397
|
}
|
|
340
398
|
}
|
|
@@ -359,8 +417,8 @@ export function pauseProvenance(): { source: string; reason?: string } | undefin
|
|
|
359
417
|
const reason = match[2];
|
|
360
418
|
return { source, ...(reason === undefined ? {} : { reason }) };
|
|
361
419
|
} catch {
|
|
362
|
-
// Unreadable sentinel: no provenance to name
|
|
363
|
-
//
|
|
420
|
+
// Unreadable sentinel: no provenance to name. Completion mutations still
|
|
421
|
+
// fail closed because the pause time is unknown.
|
|
364
422
|
return undefined;
|
|
365
423
|
}
|
|
366
424
|
}
|
|
@@ -471,9 +529,17 @@ export function checkIntegrity(gate: IntegrityGate, current: Map<string, string>
|
|
|
471
529
|
return { diff, pause: true, page: !gate.paged };
|
|
472
530
|
}
|
|
473
531
|
|
|
474
|
-
/** Latch a
|
|
475
|
-
export function markPaged(gate:
|
|
476
|
-
|
|
532
|
+
/** Latch a page after delivery is confirmed, and never before. */
|
|
533
|
+
export function markPaged(gate: StallGate, delivered: boolean, now: number): void;
|
|
534
|
+
export function markPaged(gate: IntegrityGate, delivered: boolean): void;
|
|
535
|
+
export function markPaged(
|
|
536
|
+
gate: { paged: boolean; lastPagedAt?: number },
|
|
537
|
+
delivered: boolean,
|
|
538
|
+
now?: number,
|
|
539
|
+
): void {
|
|
540
|
+
if (!delivered) return;
|
|
541
|
+
gate.paged = true;
|
|
542
|
+
if (now !== undefined) gate.lastPagedAt = now;
|
|
477
543
|
}
|
|
478
544
|
|
|
479
545
|
// ---------------------------------------------------------------------- helpers
|
|
@@ -543,6 +609,32 @@ function swapLabel(store: Store, projectName: string, issue: number, from: strin
|
|
|
543
609
|
{ issue, op: "remove", label: from },
|
|
544
610
|
]);
|
|
545
611
|
}
|
|
612
|
+
/**
|
|
613
|
+
* Persist the operator-stop transition before releasing its live controller.
|
|
614
|
+
* The row is terminal first, then its in-progress label is removed through the
|
|
615
|
+
* same durable projection outbox as every other lifecycle transition.
|
|
616
|
+
*/
|
|
617
|
+
export function recordOperatorStop(
|
|
618
|
+
store: Pick<Store, "updateRun" | "enqueueLabelOps">,
|
|
619
|
+
args: {
|
|
620
|
+
project: string;
|
|
621
|
+
issue: number;
|
|
622
|
+
runId: string;
|
|
623
|
+
inProgress: string;
|
|
624
|
+
reason: string;
|
|
625
|
+
patch: Partial<RunRecord>;
|
|
626
|
+
},
|
|
627
|
+
): void {
|
|
628
|
+
store.updateRun(args.runId, {
|
|
629
|
+
...args.patch,
|
|
630
|
+
state: "stopped",
|
|
631
|
+
lastError: `operator stopped: ${args.reason}`,
|
|
632
|
+
});
|
|
633
|
+
store.enqueueLabelOps(args.project, [
|
|
634
|
+
{ issue: args.issue, op: "remove", label: args.inProgress },
|
|
635
|
+
]);
|
|
636
|
+
}
|
|
637
|
+
|
|
546
638
|
|
|
547
639
|
/**
|
|
548
640
|
* The escalator throws when no transport is configured or Telegram rejects, and
|
|
@@ -564,6 +656,38 @@ async function safeEscalate(d: Pick<Deps, "escalate">, e: Escalation): Promise<b
|
|
|
564
656
|
}
|
|
565
657
|
}
|
|
566
658
|
|
|
659
|
+
async function reactToProviderCredit(
|
|
660
|
+
d: Deps,
|
|
661
|
+
issue: number,
|
|
662
|
+
message: string,
|
|
663
|
+
sessionFile: string | undefined,
|
|
664
|
+
): Promise<void> {
|
|
665
|
+
const { project } = d;
|
|
666
|
+
const alreadyPaused = isPaused();
|
|
667
|
+
if (!alreadyPaused) setPaused(true, { source: "provider-credit", reason: message });
|
|
668
|
+
log(
|
|
669
|
+
`#${issue} provider refused for credit — dispatch ${alreadyPaused ? "remains paused" : "paused"}: ${message}`,
|
|
670
|
+
);
|
|
671
|
+
// Fleet-scoped and run-independent on purpose. The notification ledger
|
|
672
|
+
// dedupes on `project:issue:tier:summary`, so `NO_ISSUE` plus a summary
|
|
673
|
+
// carrying no run or attempt pages once for the fleet, not once per run.
|
|
674
|
+
await safeEscalate(d, {
|
|
675
|
+
tier: 2,
|
|
676
|
+
category: "fleet-stopped",
|
|
677
|
+
project: project.name,
|
|
678
|
+
issue: NO_ISSUE,
|
|
679
|
+
summary: `Model provider refused for credit — ${project.name} is paused`,
|
|
680
|
+
detail: [
|
|
681
|
+
message,
|
|
682
|
+
"",
|
|
683
|
+
"No implementation attempt was charged: this is a billing state, not a",
|
|
684
|
+
"failed implementation. Each affected issue keeps its queue label and",
|
|
685
|
+
"re-dispatches on `omp-conductor resume` once the provider has credit.",
|
|
686
|
+
`Session: ${sessionFile ?? "(no transcript)"}`,
|
|
687
|
+
].join("\n"),
|
|
688
|
+
});
|
|
689
|
+
}
|
|
690
|
+
|
|
567
691
|
/**
|
|
568
692
|
* What a salvage attempt contributes to the escalation: where the work went, or
|
|
569
693
|
* that it went nowhere. Split from the effects below for the same reason
|
|
@@ -840,11 +964,14 @@ export function createTurnLimitRegistry(
|
|
|
840
964
|
|
|
841
965
|
export type WorkerControlResult =
|
|
842
966
|
| { kind: "ok"; runId: string; phase: WorkerPausePhase }
|
|
967
|
+
| { kind: "stopped"; runId: string; reason: string }
|
|
843
968
|
| { kind: "refused"; runId: string; error: string }
|
|
844
969
|
| { kind: "not-active" };
|
|
845
970
|
|
|
846
971
|
export interface WorkerControlSlot {
|
|
847
972
|
install(control: WorkerPauseControl): void;
|
|
973
|
+
/** Stop accepted before the session controller exists. */
|
|
974
|
+
requestedStop(): string | undefined;
|
|
848
975
|
close(): void;
|
|
849
976
|
}
|
|
850
977
|
|
|
@@ -852,34 +979,51 @@ export interface WorkerControlRegistry {
|
|
|
852
979
|
open(project: string, issue: number, runId: string): WorkerControlSlot;
|
|
853
980
|
pause(project: string, issue: number): Promise<WorkerControlResult>;
|
|
854
981
|
resume(project: string, issue: number): WorkerControlResult;
|
|
982
|
+
stop(project: string, issue: number, reason: string): Promise<WorkerControlResult>;
|
|
855
983
|
/** Live runs whose phase is not `running` — what /healthz and the board show. */
|
|
856
984
|
snapshot(project: string): { issue: number; runId: string; phase: WorkerPausePhase }[];
|
|
857
985
|
}
|
|
858
986
|
|
|
859
987
|
/** Authoritative controls for sessions owned by this daemon process. */
|
|
860
988
|
export function createWorkerControlRegistry(): WorkerControlRegistry {
|
|
861
|
-
|
|
862
|
-
string
|
|
863
|
-
|
|
864
|
-
|
|
989
|
+
interface Entry {
|
|
990
|
+
project: string;
|
|
991
|
+
issue: number;
|
|
992
|
+
runId: string;
|
|
993
|
+
control?: WorkerPauseControl;
|
|
994
|
+
stopReason?: string;
|
|
995
|
+
stopError?: string;
|
|
996
|
+
finished: PromiseWithResolvers<void>;
|
|
997
|
+
}
|
|
998
|
+
|
|
999
|
+
const active = new Map<string, Entry>();
|
|
865
1000
|
const key = (project: string, issue: number): string => `${project}\0${issue}`;
|
|
866
1001
|
return {
|
|
867
1002
|
open(project, issue, runId) {
|
|
868
1003
|
const k = key(project, issue);
|
|
869
1004
|
if (active.has(k)) throw new Error(`#${issue} already has a live worker controller`);
|
|
870
|
-
const entry = {
|
|
871
|
-
project
|
|
872
|
-
issue
|
|
873
|
-
runId
|
|
874
|
-
|
|
1005
|
+
const entry: Entry = {
|
|
1006
|
+
project,
|
|
1007
|
+
issue,
|
|
1008
|
+
runId,
|
|
1009
|
+
finished: Promise.withResolvers<void>(),
|
|
875
1010
|
};
|
|
876
1011
|
active.set(k, entry);
|
|
877
1012
|
return {
|
|
878
1013
|
install: (control) => {
|
|
879
|
-
if (active.get(k)
|
|
1014
|
+
if (active.get(k) !== entry) return;
|
|
1015
|
+
entry.control = control;
|
|
1016
|
+
if (entry.stopReason === undefined) return;
|
|
1017
|
+
try {
|
|
1018
|
+
control.stop(entry.stopReason);
|
|
1019
|
+
} catch (err) {
|
|
1020
|
+
entry.stopError = err instanceof Error ? err.message : String(err);
|
|
1021
|
+
}
|
|
880
1022
|
},
|
|
1023
|
+
requestedStop: () => active.get(k) === entry ? entry.stopReason : undefined,
|
|
881
1024
|
close: () => {
|
|
882
1025
|
if (active.get(k) === entry) active.delete(k);
|
|
1026
|
+
entry.finished.resolve();
|
|
883
1027
|
},
|
|
884
1028
|
};
|
|
885
1029
|
},
|
|
@@ -911,6 +1055,30 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
|
|
|
911
1055
|
};
|
|
912
1056
|
}
|
|
913
1057
|
},
|
|
1058
|
+
async stop(project, issue, reason) {
|
|
1059
|
+
const entry = active.get(key(project, issue));
|
|
1060
|
+
if (entry === undefined) return { kind: "not-active" };
|
|
1061
|
+
if (entry.stopReason === undefined) {
|
|
1062
|
+
entry.stopReason = reason;
|
|
1063
|
+
if (entry.control !== undefined) {
|
|
1064
|
+
try {
|
|
1065
|
+
entry.control.stop(reason);
|
|
1066
|
+
} catch (err) {
|
|
1067
|
+
entry.stopReason = undefined;
|
|
1068
|
+
return {
|
|
1069
|
+
kind: "refused",
|
|
1070
|
+
runId: entry.runId,
|
|
1071
|
+
error: err instanceof Error ? err.message : String(err),
|
|
1072
|
+
};
|
|
1073
|
+
}
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
1076
|
+
await entry.finished.promise;
|
|
1077
|
+
if (entry.stopError !== undefined) {
|
|
1078
|
+
return { kind: "refused", runId: entry.runId, error: entry.stopError };
|
|
1079
|
+
}
|
|
1080
|
+
return { kind: "stopped", runId: entry.runId, reason: entry.stopReason! };
|
|
1081
|
+
},
|
|
914
1082
|
snapshot(project) {
|
|
915
1083
|
const workers: { issue: number; runId: string; phase: WorkerPausePhase }[] = [];
|
|
916
1084
|
for (const entry of active.values()) {
|
|
@@ -1012,7 +1180,28 @@ export function shouldContinueAfterTurnsCap(f: {
|
|
|
1012
1180
|
return hasContinuationBudget(f.continuation, f.maxContinuations);
|
|
1013
1181
|
}
|
|
1014
1182
|
|
|
1015
|
-
|
|
1183
|
+
/** Bind a worker's run identity to every release-policy block it emits. */
|
|
1184
|
+
export function workerReleaseBlockRecorder(
|
|
1185
|
+
project: string,
|
|
1186
|
+
issue: number,
|
|
1187
|
+
runId: string,
|
|
1188
|
+
root = stateDir(),
|
|
1189
|
+
): (shape: ReleaseShape, context: ReleaseBlockContext) => void {
|
|
1190
|
+
return (shape, context) =>
|
|
1191
|
+
recordReleaseBlock(
|
|
1192
|
+
project,
|
|
1193
|
+
"worker",
|
|
1194
|
+
shape,
|
|
1195
|
+
{
|
|
1196
|
+
...context,
|
|
1197
|
+
issue,
|
|
1198
|
+
runId,
|
|
1199
|
+
},
|
|
1200
|
+
root,
|
|
1201
|
+
);
|
|
1202
|
+
}
|
|
1203
|
+
|
|
1204
|
+
export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
1016
1205
|
const { project, caps, tracker, store } = d;
|
|
1017
1206
|
const issue = r.issue.number;
|
|
1018
1207
|
const branch = branchName(r.issue);
|
|
@@ -1027,6 +1216,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1027
1216
|
let worktreePath: string | undefined;
|
|
1028
1217
|
let turnLimit: TurnLimitController | undefined;
|
|
1029
1218
|
let workerControl: WorkerControlSlot | undefined;
|
|
1219
|
+
let workerSessionInstalled = false;
|
|
1030
1220
|
// The run's own repository. Hoisted for the same reason `worktreePath` is —
|
|
1031
1221
|
// the catch and finally paths have to publish the branch.
|
|
1032
1222
|
let runRepo: RunRepoRef | undefined;
|
|
@@ -1048,6 +1238,53 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1048
1238
|
return pushRunBranch(project, runRepo);
|
|
1049
1239
|
};
|
|
1050
1240
|
|
|
1241
|
+
/**
|
|
1242
|
+
* A stop can arrive after the run is claimed but before `runWorker` exposes
|
|
1243
|
+
* its controller. Settle that run here instead of turning an operator action
|
|
1244
|
+
* into a dispatch failure or making the HTTP request wait for a session that
|
|
1245
|
+
* will never exist.
|
|
1246
|
+
*/
|
|
1247
|
+
const settleStopBeforeSession = async (): Promise<boolean> => {
|
|
1248
|
+
const reason = workerControl?.requestedStop();
|
|
1249
|
+
if (reason === undefined || run === undefined || workerSessionInstalled) return false;
|
|
1250
|
+
turnLimit?.close();
|
|
1251
|
+
turnLimit = undefined;
|
|
1252
|
+
const settlement =
|
|
1253
|
+
worktreePath === undefined
|
|
1254
|
+
? undefined
|
|
1255
|
+
: await settleWorktree({
|
|
1256
|
+
issue,
|
|
1257
|
+
attempt,
|
|
1258
|
+
ending: `stopped by the operator: ${reason}`,
|
|
1259
|
+
worktree: worktreePath,
|
|
1260
|
+
branch,
|
|
1261
|
+
publish,
|
|
1262
|
+
tree: "remove",
|
|
1263
|
+
mirrorPath,
|
|
1264
|
+
});
|
|
1265
|
+
recordOperatorStop(store, {
|
|
1266
|
+
project: project.name,
|
|
1267
|
+
issue,
|
|
1268
|
+
runId: run.id,
|
|
1269
|
+
inProgress,
|
|
1270
|
+
reason,
|
|
1271
|
+
patch: {
|
|
1272
|
+
endedAt: Date.now(),
|
|
1273
|
+
turns: run.turns,
|
|
1274
|
+
spendUsd: run.spendUsd,
|
|
1275
|
+
worktree: worktreePath ?? run.worktree,
|
|
1276
|
+
report: [
|
|
1277
|
+
"Operator stopped the run before its worker session started.",
|
|
1278
|
+
`Reason: ${reason}`,
|
|
1279
|
+
...(settlement?.lines ?? []),
|
|
1280
|
+
].join("\n"),
|
|
1281
|
+
...settlement?.patch,
|
|
1282
|
+
},
|
|
1283
|
+
});
|
|
1284
|
+
log(`#${issue} stopped by operator before its worker session started: ${reason}`);
|
|
1285
|
+
return true;
|
|
1286
|
+
};
|
|
1287
|
+
|
|
1051
1288
|
try {
|
|
1052
1289
|
// Claim on the STORE first, before anything that can fail. The run row —
|
|
1053
1290
|
// not the label — is the crash-safe guard against double dispatch: rows
|
|
@@ -1063,8 +1300,6 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1063
1300
|
// Read before this attempt's own row exists, so `latestRun` still means the
|
|
1064
1301
|
// attempt whose work this one inherits.
|
|
1065
1302
|
const priorSalvage = store.latestRun(project.name, issue)?.salvageSha;
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
1303
|
run = store.createRun({
|
|
1069
1304
|
project: project.name,
|
|
1070
1305
|
issue,
|
|
@@ -1078,11 +1313,20 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1078
1313
|
maxTurns: caps.workerMaxTurns,
|
|
1079
1314
|
startedAt: Date.now(),
|
|
1080
1315
|
});
|
|
1316
|
+
const maxTurns = run.maxTurns;
|
|
1317
|
+
const turnOverride = maxTurns > caps.workerMaxTurns ? maxTurns : undefined;
|
|
1081
1318
|
const runId = run.id;
|
|
1319
|
+
if (turnOverride !== undefined) {
|
|
1320
|
+
log(
|
|
1321
|
+
`#${issue} claimed with turn override ${turnOverride} ` +
|
|
1322
|
+
`(base ${caps.workerMaxTurns})`,
|
|
1323
|
+
);
|
|
1324
|
+
}
|
|
1082
1325
|
store.enqueueLabelOps(project.name, [{ issue, op: "add", label: inProgress }]);
|
|
1083
1326
|
claimed = true;
|
|
1084
|
-
turnLimit = d.turnLimits.open(project.name, issue, runId,
|
|
1327
|
+
turnLimit = d.turnLimits.open(project.name, issue, runId, maxTurns);
|
|
1085
1328
|
workerControl = d.workerControls.open(project.name, issue, runId);
|
|
1329
|
+
if (await settleStopBeforeSession()) return;
|
|
1086
1330
|
|
|
1087
1331
|
// A run's tree is <workspaceRoot>/<issue> and addRunRepo refuses to reuse
|
|
1088
1332
|
// an existing path, so a retry — or a tree kept from a failed attempt — has
|
|
@@ -1091,6 +1335,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1091
1335
|
// first attempt. addRunRepo does its own ensureMirror; calling it here too
|
|
1092
1336
|
// would cost a second network fetch per attempt.
|
|
1093
1337
|
await removeWorktree(mirrorPath, worktreePathFor(project.workspaceRoot, issue));
|
|
1338
|
+
if (await settleStopBeforeSession()) return;
|
|
1094
1339
|
const provisioned = await addRunRepo(
|
|
1095
1340
|
r.repo,
|
|
1096
1341
|
project.mirrorRoot,
|
|
@@ -1100,6 +1345,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1100
1345
|
);
|
|
1101
1346
|
worktreePath = provisioned.path;
|
|
1102
1347
|
runRepo = { repo: r.repo, runRepoPath: worktreePath, branch };
|
|
1348
|
+
if (await settleStopBeforeSession()) return;
|
|
1103
1349
|
|
|
1104
1350
|
// The SDK names the transcript itself, so the daemon supplies the parent
|
|
1105
1351
|
// directory and learns the real path back from the result. Inventing one
|
|
@@ -1138,6 +1384,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1138
1384
|
},
|
|
1139
1385
|
{ ...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }) },
|
|
1140
1386
|
);
|
|
1387
|
+
if (await settleStopBeforeSession()) return;
|
|
1141
1388
|
|
|
1142
1389
|
store.updateRun(runId, { worktree: worktreePath, state: "running" });
|
|
1143
1390
|
|
|
@@ -1146,20 +1393,26 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1146
1393
|
(provisioned.reattached ? " (continuation: reattached existing branch)" : ""),
|
|
1147
1394
|
);
|
|
1148
1395
|
|
|
1396
|
+
const brief = await buildBrief(project, r, branch, worktreePath, {
|
|
1397
|
+
continuation: provisioned.reattached,
|
|
1398
|
+
defaultBranch: r.repo.defaultBranch,
|
|
1399
|
+
...(provisioned.reattached && priorSalvage !== undefined
|
|
1400
|
+
? { salvagedSha: priorSalvage }
|
|
1401
|
+
: {}),
|
|
1402
|
+
});
|
|
1403
|
+
if (await settleStopBeforeSession()) return;
|
|
1404
|
+
|
|
1149
1405
|
let result: WorkerResult;
|
|
1150
1406
|
try {
|
|
1151
1407
|
result = await runWorker({
|
|
1152
|
-
brief
|
|
1153
|
-
continuation: provisioned.reattached,
|
|
1154
|
-
defaultBranch: r.repo.defaultBranch,
|
|
1155
|
-
...(provisioned.reattached && priorSalvage !== undefined
|
|
1156
|
-
? { salvagedSha: priorSalvage }
|
|
1157
|
-
: {}),
|
|
1158
|
-
}),
|
|
1408
|
+
brief,
|
|
1159
1409
|
cwd: worktreePath,
|
|
1160
1410
|
caps,
|
|
1161
|
-
maxTurns: () => turnLimit?.maxTurns() ??
|
|
1162
|
-
onPauseControl: (control) =>
|
|
1411
|
+
maxTurns: () => turnLimit?.maxTurns() ?? maxTurns,
|
|
1412
|
+
onPauseControl: (control) => {
|
|
1413
|
+
workerSessionInstalled = true;
|
|
1414
|
+
workerControl?.install(control);
|
|
1415
|
+
},
|
|
1163
1416
|
sessionDir,
|
|
1164
1417
|
// The session's control socket, under the daemon's own state directory —
|
|
1165
1418
|
// a child process of the daemon reaches it directly.
|
|
@@ -1178,28 +1431,24 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1178
1431
|
},
|
|
1179
1432
|
...(project.workerModel === undefined ? {} : { model: project.workerModel }),
|
|
1180
1433
|
releaseGrants: resolveReleaseGrants(project),
|
|
1181
|
-
onReleaseBlocked: (
|
|
1434
|
+
onReleaseBlocked: workerReleaseBlockRecorder(project.name, issue, runId),
|
|
1182
1435
|
onTurn: (n) => store.updateRun(runId, { turns: n }),
|
|
1183
1436
|
onSpend: (usd) => store.updateRun(runId, { spendUsd: usd }),
|
|
1184
1437
|
onKilled: () => {
|
|
1185
1438
|
turnLimit?.close();
|
|
1186
1439
|
turnLimit = undefined;
|
|
1187
|
-
workerControl?.close();
|
|
1188
|
-
workerControl = undefined;
|
|
1189
1440
|
},
|
|
1190
1441
|
// Recorded the moment the session opens its transcript, not when the run
|
|
1191
1442
|
// ends: `omp-conductor tail` resolves an issue to a file through this row,
|
|
1192
1443
|
// and a path written at completion is a path nobody can follow live. The
|
|
1193
1444
|
// completion-time update below writes the same value again, harmlessly.
|
|
1194
1445
|
onSessionFile: (f) => store.updateRun(runId, { sessionFile: f }),
|
|
1195
|
-
});
|
|
1446
|
+
}, d.workerDeps);
|
|
1196
1447
|
} finally {
|
|
1197
1448
|
// This is the authoritative settlement edge for `extend`: close before
|
|
1198
1449
|
// PR verification or terminal row writes can leave stale `running` state.
|
|
1199
1450
|
turnLimit?.close();
|
|
1200
1451
|
turnLimit = undefined;
|
|
1201
|
-
workerControl?.close();
|
|
1202
|
-
workerControl = undefined;
|
|
1203
1452
|
}
|
|
1204
1453
|
|
|
1205
1454
|
// A configured model the harness could not honour means this run was done by
|
|
@@ -1264,8 +1513,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1264
1513
|
// `pushed-*` run is the only end that does not salvage: its deliverable is
|
|
1265
1514
|
// already on a remote branch, whatever is left loose in the tree is by the
|
|
1266
1515
|
// worker's own account not part of it, and appending a WIP commit would
|
|
1267
|
-
// turn the green PR this daemon just verified red. Every other end
|
|
1268
|
-
//
|
|
1516
|
+
// turn the green PR this daemon just verified red. Every other end may
|
|
1517
|
+
// contain work, so it is salvaged before the tree's final fate is decided.
|
|
1269
1518
|
const settlement =
|
|
1270
1519
|
state === "pushed-green" || state === "pushed-pending" || state === "merged"
|
|
1271
1520
|
? undefined
|
|
@@ -1273,7 +1522,11 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1273
1522
|
issue,
|
|
1274
1523
|
attempt,
|
|
1275
1524
|
ending:
|
|
1276
|
-
state === "blocked"
|
|
1525
|
+
state === "blocked"
|
|
1526
|
+
? "blocked for an operator decision"
|
|
1527
|
+
: state === "stopped"
|
|
1528
|
+
? `stopped by the operator: ${result.stoppedReason ?? "no reason recorded"}`
|
|
1529
|
+
: endedBy(result.killedBy),
|
|
1277
1530
|
worktree: worktreePath,
|
|
1278
1531
|
branch,
|
|
1279
1532
|
publish,
|
|
@@ -1294,8 +1547,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1294
1547
|
await removeWorktree(mirrorPath, worktreePath);
|
|
1295
1548
|
}
|
|
1296
1549
|
|
|
1297
|
-
|
|
1298
|
-
state,
|
|
1550
|
+
const terminalPatch: Partial<RunRecord> = {
|
|
1299
1551
|
endedAt: Date.now(),
|
|
1300
1552
|
turns: result.turns,
|
|
1301
1553
|
spendUsd: result.spendUsd,
|
|
@@ -1303,23 +1555,41 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1303
1555
|
headSha: result.headSha,
|
|
1304
1556
|
sessionFile: result.sessionFile,
|
|
1305
1557
|
// Every terminal state persists the worker's report, not just a green
|
|
1306
|
-
// push:
|
|
1307
|
-
// continuation must pool its disclosures from (#199).
|
|
1558
|
+
// push: a stopped attempt's partial report is still part of its audit trail.
|
|
1308
1559
|
report: result.report,
|
|
1309
|
-
...(providerCredit !== undefined || providerTransient !== undefined
|
|
1310
|
-
? { lastError: providerCredit ?? providerTransient }
|
|
1311
|
-
: verified.reason === undefined
|
|
1312
|
-
? {}
|
|
1313
|
-
: { lastError: verified.reason }),
|
|
1314
1560
|
...settlement?.patch,
|
|
1315
1561
|
...(audit === undefined || audit.flags.length === 0
|
|
1316
1562
|
? {}
|
|
1317
1563
|
: { settlementFlags: audit.flags }),
|
|
1318
|
-
}
|
|
1564
|
+
};
|
|
1565
|
+
if (state === "stopped") {
|
|
1566
|
+
recordOperatorStop(store, {
|
|
1567
|
+
project: project.name,
|
|
1568
|
+
issue,
|
|
1569
|
+
runId,
|
|
1570
|
+
inProgress,
|
|
1571
|
+
reason: result.stoppedReason ?? "no reason recorded",
|
|
1572
|
+
patch: terminalPatch,
|
|
1573
|
+
});
|
|
1574
|
+
} else {
|
|
1575
|
+
const lastError = completionLastError(
|
|
1576
|
+
providerCredit,
|
|
1577
|
+
providerTransient,
|
|
1578
|
+
verified.reason,
|
|
1579
|
+
sessionErr,
|
|
1580
|
+
);
|
|
1581
|
+
store.updateRun(runId, {
|
|
1582
|
+
...terminalPatch,
|
|
1583
|
+
state,
|
|
1584
|
+
...(lastError === undefined ? {} : { lastError }),
|
|
1585
|
+
});
|
|
1586
|
+
}
|
|
1319
1587
|
|
|
1320
1588
|
const salvaged = settlement?.lines ?? [];
|
|
1321
1589
|
|
|
1322
|
-
if (state === "
|
|
1590
|
+
if (state === "stopped") {
|
|
1591
|
+
log(`#${issue} stopped by operator on attempt ${attempt}: ${result.stoppedReason}`);
|
|
1592
|
+
} else if (state === "blocked") {
|
|
1323
1593
|
swapLabel(store, project.name, issue, inProgress, project.stateLabels.blocked);
|
|
1324
1594
|
await safeEscalate(d, {
|
|
1325
1595
|
tier: 1,
|
|
@@ -1343,31 +1613,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1343
1613
|
});
|
|
1344
1614
|
|
|
1345
1615
|
if (providerCredit !== undefined) {
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
// the first 402 and a human noticing (#220).
|
|
1349
|
-
setPaused(true, { source: "provider-credit", reason: providerCredit });
|
|
1350
|
-
log(`#${issue} provider refused for credit — dispatch paused: ${providerCredit}`);
|
|
1351
|
-
swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
|
|
1352
|
-
// Fleet-scoped and run-independent on purpose. The notification ledger
|
|
1353
|
-
// dedupes on `project:issue:tier:summary`, so `NO_ISSUE` plus a summary
|
|
1354
|
-
// carrying no run or attempt is what makes this page once for the fleet
|
|
1355
|
-
// instead of once per affected run.
|
|
1356
|
-
await safeEscalate(d, {
|
|
1357
|
-
tier: 2,
|
|
1358
|
-
category: "fleet-stopped",
|
|
1359
|
-
project: project.name,
|
|
1360
|
-
issue: NO_ISSUE,
|
|
1361
|
-
summary: `Model provider refused for credit — ${project.name} is paused`,
|
|
1362
|
-
detail: [
|
|
1363
|
-
providerCredit,
|
|
1364
|
-
"",
|
|
1365
|
-
"No implementation attempt was charged: this is a billing state, not a",
|
|
1366
|
-
"failed implementation. Each affected issue keeps its queue label and",
|
|
1367
|
-
"re-dispatches on `omp-conductor resume` once the provider has credit.",
|
|
1368
|
-
`Session: ${result.sessionFile ?? "(no transcript)"}`,
|
|
1369
|
-
].join("\n"),
|
|
1370
|
-
});
|
|
1616
|
+
await reactToProviderCredit(d, issue, providerCredit, result.sessionFile);
|
|
1617
|
+
swapToQueue(d, issue, inProgress);
|
|
1371
1618
|
} else if (continueTurns) {
|
|
1372
1619
|
// Requeue as one ordered pair: the in-progress removal before the
|
|
1373
1620
|
// queue add, exactly the order the projector will apply them in (#201).
|
|
@@ -1469,8 +1716,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1469
1716
|
// inner settlement guard exists. Latch it before any terminal write or await.
|
|
1470
1717
|
turnLimit?.close();
|
|
1471
1718
|
turnLimit = undefined;
|
|
1472
|
-
|
|
1473
|
-
workerControl = undefined;
|
|
1719
|
+
if (await settleStopBeforeSession()) return;
|
|
1474
1720
|
const detail = errText(err);
|
|
1475
1721
|
log(`#${issue} errored: ${detail}`);
|
|
1476
1722
|
// A crash lands anywhere, including mid-edit in a tree holding the only
|
|
@@ -1516,8 +1762,6 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1516
1762
|
// a failure path, and whatever it still held is now a commit on the branch.
|
|
1517
1763
|
} finally {
|
|
1518
1764
|
turnLimit?.close();
|
|
1519
|
-
workerControl?.close();
|
|
1520
|
-
workerControl = undefined;
|
|
1521
1765
|
// The run is over, so its channel is too. Closed here rather than beside
|
|
1522
1766
|
// the session so the crash path closes it as well: a listener left bound
|
|
1523
1767
|
// after its run settled is a socket whose `run-not-live` check is the only
|
|
@@ -1530,6 +1774,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1530
1774
|
}
|
|
1531
1775
|
verbListener = undefined;
|
|
1532
1776
|
}
|
|
1777
|
+
workerControl?.close();
|
|
1778
|
+
workerControl = undefined;
|
|
1533
1779
|
}
|
|
1534
1780
|
}
|
|
1535
1781
|
|
|
@@ -1647,6 +1893,129 @@ export function releaseInProgress(
|
|
|
1647
1893
|
* in the busy set throughout, so no second worker can be sent at the issue while
|
|
1648
1894
|
* it waits.
|
|
1649
1895
|
*/
|
|
1896
|
+
const BASE_CHECK_BATCH = 20;
|
|
1897
|
+
const BASE_CHECK_WINDOW_MS = 24 * 60 * 60 * 1_000;
|
|
1898
|
+
const BASE_STATUS_WINDOW_MS = 7 * 24 * 60 * 60 * 1_000;
|
|
1899
|
+
|
|
1900
|
+
const SUCCESSFUL_WORKFLOW_CONCLUSIONS = new Set(["success", "neutral", "skipped"]);
|
|
1901
|
+
|
|
1902
|
+
const FAILING_WORKFLOW_CONCLUSIONS = new Set([
|
|
1903
|
+
"failure",
|
|
1904
|
+
"cancelled",
|
|
1905
|
+
"timed_out",
|
|
1906
|
+
"action_required",
|
|
1907
|
+
"startup_failure",
|
|
1908
|
+
"stale",
|
|
1909
|
+
]);
|
|
1910
|
+
|
|
1911
|
+
function appendSettlementFlag(run: RunRecord, flag: SettlementFlag): SettlementFlag[] {
|
|
1912
|
+
const flags = run.settlementFlags ?? [];
|
|
1913
|
+
return flags.some((existing) => existing.kind === flag.kind && existing.detail === flag.detail)
|
|
1914
|
+
? flags
|
|
1915
|
+
: [...flags, flag];
|
|
1916
|
+
}
|
|
1917
|
+
|
|
1918
|
+
/**
|
|
1919
|
+
* Observe Actions on exact merge commits for up to one day. A running workflow
|
|
1920
|
+
* stays quiet and pending; a failure becomes durable evidence on the merged row
|
|
1921
|
+
* and pages exactly once because the row leaves `pending` before delivery.
|
|
1922
|
+
*/
|
|
1923
|
+
export async function watchMergedBase(d: Pick<Deps, "project" | "tracker" | "store" | "escalate">): Promise<void> {
|
|
1924
|
+
const now = Date.now();
|
|
1925
|
+
for (const run of d.store.runsNeedingBaseCheck(d.project.name, BASE_CHECK_BATCH)) {
|
|
1926
|
+
if (
|
|
1927
|
+
run.endedAt === undefined ||
|
|
1928
|
+
now - run.endedAt > BASE_CHECK_WINDOW_MS ||
|
|
1929
|
+
run.mergeSha === undefined ||
|
|
1930
|
+
run.baseRef === undefined
|
|
1931
|
+
) {
|
|
1932
|
+
d.store.updateRun(run.id, { baseCheck: "unknown", baseCheckAt: now });
|
|
1933
|
+
log(`#${run.issue} base check unknown: merge identity is absent or older than 24h`);
|
|
1934
|
+
continue;
|
|
1935
|
+
}
|
|
1936
|
+
|
|
1937
|
+
const repo = d.project.routing.repos[run.repo];
|
|
1938
|
+
const repoIdentity = repo === undefined ? undefined : githubRepo(repo.cloneUrl);
|
|
1939
|
+
if (repoIdentity === undefined) {
|
|
1940
|
+
d.store.updateRun(run.id, { baseCheck: "unknown", baseCheckAt: now });
|
|
1941
|
+
log(`#${run.issue} base check unknown: routed repository ${run.repo} has no GitHub identity`);
|
|
1942
|
+
continue;
|
|
1943
|
+
}
|
|
1944
|
+
|
|
1945
|
+
let workflows;
|
|
1946
|
+
try {
|
|
1947
|
+
workflows = await d.tracker.workflowRunsAt(repoIdentity, run.mergeSha);
|
|
1948
|
+
} catch (err) {
|
|
1949
|
+
log(`#${run.issue} base check unavailable (${errText(err)}) — retrying next tick`);
|
|
1950
|
+
continue;
|
|
1951
|
+
}
|
|
1952
|
+
if (workflows === undefined || workflows.length === 0) {
|
|
1953
|
+
log(`#${run.issue} base check unavailable for ${run.mergeSha} — retrying next tick`);
|
|
1954
|
+
continue;
|
|
1955
|
+
}
|
|
1956
|
+
if (workflows.some((workflow) => workflow.status !== "completed")) continue;
|
|
1957
|
+
|
|
1958
|
+
const failed = workflows.find(
|
|
1959
|
+
(workflow) =>
|
|
1960
|
+
workflow.conclusion !== undefined &&
|
|
1961
|
+
FAILING_WORKFLOW_CONCLUSIONS.has(workflow.conclusion),
|
|
1962
|
+
);
|
|
1963
|
+
if (failed === undefined) {
|
|
1964
|
+
const unknown = workflows.find(
|
|
1965
|
+
(workflow) =>
|
|
1966
|
+
workflow.conclusion === undefined ||
|
|
1967
|
+
!SUCCESSFUL_WORKFLOW_CONCLUSIONS.has(workflow.conclusion),
|
|
1968
|
+
);
|
|
1969
|
+
if (unknown !== undefined) {
|
|
1970
|
+
log(
|
|
1971
|
+
`#${run.issue} base check has unrecognised completed conclusion ` +
|
|
1972
|
+
`${JSON.stringify(unknown.conclusion)} for ${unknown.name} — retrying next tick`,
|
|
1973
|
+
);
|
|
1974
|
+
continue;
|
|
1975
|
+
}
|
|
1976
|
+
d.store.updateRun(run.id, { baseCheck: "green", baseCheckAt: now });
|
|
1977
|
+
log(`#${run.issue} base ${run.baseRef} green at ${run.mergeSha}`);
|
|
1978
|
+
continue;
|
|
1979
|
+
}
|
|
1980
|
+
|
|
1981
|
+
let previous;
|
|
1982
|
+
try {
|
|
1983
|
+
previous = await d.tracker.previousWorkflowRun(
|
|
1984
|
+
repoIdentity,
|
|
1985
|
+
failed.workflowId,
|
|
1986
|
+
run.baseRef,
|
|
1987
|
+
failed.createdAt,
|
|
1988
|
+
);
|
|
1989
|
+
} catch (err) {
|
|
1990
|
+
log(`#${run.issue} previous ${failed.name} run unavailable (${errText(err)}) — retrying next tick`);
|
|
1991
|
+
continue;
|
|
1992
|
+
}
|
|
1993
|
+
if (previous === undefined || (previous !== null && previous.status !== "completed")) continue;
|
|
1994
|
+
const preexisting =
|
|
1995
|
+
previous !== null &&
|
|
1996
|
+
previous.conclusion !== undefined &&
|
|
1997
|
+
FAILING_WORKFLOW_CONCLUSIONS.has(previous.conclusion);
|
|
1998
|
+
const detail =
|
|
1999
|
+
`${failed.name} failed at ${run.mergeSha} — ${failed.url}` +
|
|
2000
|
+
(preexisting ? " (already red before this merge)" : "");
|
|
2001
|
+
const flag: SettlementFlag = { kind: "base-branch-red", file: "(base branch)", detail };
|
|
2002
|
+
const delivered = await safeEscalate(d, {
|
|
2003
|
+
tier: 1,
|
|
2004
|
+
project: d.project.name,
|
|
2005
|
+
issue: run.issue,
|
|
2006
|
+
runId: run.id,
|
|
2007
|
+
summary: `Base branch ${run.baseRef} is red after merge`,
|
|
2008
|
+
detail,
|
|
2009
|
+
});
|
|
2010
|
+
if (!delivered) continue;
|
|
2011
|
+
d.store.updateRun(run.id, {
|
|
2012
|
+
baseCheck: preexisting ? "red-preexisting" : "red",
|
|
2013
|
+
baseCheckAt: now,
|
|
2014
|
+
settlementFlags: appendSettlementFlag(run, flag),
|
|
2015
|
+
});
|
|
2016
|
+
}
|
|
2017
|
+
}
|
|
2018
|
+
|
|
1650
2019
|
export async function settlePushedGreen(
|
|
1651
2020
|
d: Pick<Deps, "project" | "tracker" | "store">,
|
|
1652
2021
|
): Promise<void> {
|
|
@@ -1678,16 +2047,39 @@ export async function settlePushedGreen(
|
|
|
1678
2047
|
|
|
1679
2048
|
const settlement = settlementFor(pr, run.prUrl);
|
|
1680
2049
|
if (settlement !== undefined) {
|
|
2050
|
+
// A mediated merge enters a second, bounded observation phase. Record the
|
|
2051
|
+
// exact merge commit before the row leaves the active set; if GitHub cannot
|
|
2052
|
+
// supply it yet, retry this settlement next tick rather than create a
|
|
2053
|
+
// merged row whose base result can never be attributed.
|
|
2054
|
+
let merged: MergedPrInfo | undefined;
|
|
2055
|
+
if (settlement.state === "merged") {
|
|
2056
|
+
try {
|
|
2057
|
+
merged = await tracker.mergedPrInfo(run.prUrl);
|
|
2058
|
+
} catch (err) {
|
|
2059
|
+
log(`#${run.issue} not settled: merge identity lookup failed (${errText(err)}) — retrying next tick`);
|
|
2060
|
+
continue;
|
|
2061
|
+
}
|
|
2062
|
+
if (merged === undefined) {
|
|
2063
|
+
log(`#${run.issue} not settled: merge identity unavailable — retrying next tick`);
|
|
2064
|
+
continue;
|
|
2065
|
+
}
|
|
2066
|
+
}
|
|
2067
|
+
|
|
1681
2068
|
// The label removal and the terminal row are one fact again (#201): the
|
|
1682
2069
|
// release is enqueued — a durable local write that cannot fail on the
|
|
1683
|
-
// tracker — in the same breath as the row is terminalised
|
|
1684
|
-
// no window in which a row beyond every later tick still owes its label.
|
|
1685
|
-
// The projector applies it with retry; while pending, the eligibility
|
|
1686
|
-
// overlay treats the label as already gone, so #18's
|
|
1687
|
-
// permanent-`agent:in-progress` cannot re-form even when GitHub refuses
|
|
1688
|
-
// the write.
|
|
2070
|
+
// tracker — in the same breath as the row is terminalised.
|
|
1689
2071
|
releaseInProgress(d, run.issue, settlement.reason);
|
|
1690
|
-
const patch: Partial<RunRecord> = {
|
|
2072
|
+
const patch: Partial<RunRecord> = {
|
|
2073
|
+
state: settlement.state,
|
|
2074
|
+
endedAt: Date.now(),
|
|
2075
|
+
...(merged === undefined
|
|
2076
|
+
? {}
|
|
2077
|
+
: {
|
|
2078
|
+
mergeSha: merged.mergeSha,
|
|
2079
|
+
baseRef: merged.baseRef,
|
|
2080
|
+
baseCheck: "pending",
|
|
2081
|
+
}),
|
|
2082
|
+
};
|
|
1691
2083
|
if (settlement.state === "failed") patch.lastError = settlement.reason;
|
|
1692
2084
|
store.updateRun(run.id, patch);
|
|
1693
2085
|
log(`#${run.issue} settled: ${settlement.reason}`);
|
|
@@ -1720,6 +2112,121 @@ export async function settlePushedGreen(
|
|
|
1720
2112
|
}
|
|
1721
2113
|
}
|
|
1722
2114
|
|
|
2115
|
+
const ADOPTABLE_PR_STATES: Partial<Record<RunState, true>> = {
|
|
2116
|
+
failed: true,
|
|
2117
|
+
killed: true,
|
|
2118
|
+
orphaned: true,
|
|
2119
|
+
blocked: true,
|
|
2120
|
+
};
|
|
2121
|
+
|
|
2122
|
+
/**
|
|
2123
|
+
* Reattaches a recovered PR to the terminal run that owns it (#245).
|
|
2124
|
+
*
|
|
2125
|
+
* A worker can fail before its completion report records `prUrl`, then have its
|
|
2126
|
+
* dirty tree committed and pushed by salvage. If that branch already has a PR,
|
|
2127
|
+
* the orchestrator otherwise has no policy-compliant path to inspect or merge
|
|
2128
|
+
* it: ownership is store-backed. Adoption is deliberately stricter than
|
|
2129
|
+
* admission. The tracker query proves the PR closes this run's issue; exact
|
|
2130
|
+
* branch and canonical repository matches prove it is this run's recovered
|
|
2131
|
+
* work, not an unrelated closer. Missing identity is refusal, never a guess.
|
|
2132
|
+
*
|
|
2133
|
+
* The newest run per issue is inspected, at most ten per tick and only inside
|
|
2134
|
+
* the same 30-day window as mediated PR verbs. The cursor advances through the
|
|
2135
|
+
* full eligible set so persistent non-matches cannot starve older recovered
|
|
2136
|
+
* work. Successful adoption is idempotent because the row gains `prUrl`;
|
|
2137
|
+
* non-matches are logged once per daemon process.
|
|
2138
|
+
*/
|
|
2139
|
+
const rejectedSalvagedPrRuns = new Set<string>();
|
|
2140
|
+
const salvagedPrCursor = new Map<string, string>();
|
|
2141
|
+
|
|
2142
|
+
export async function adoptSalvagedPrs(
|
|
2143
|
+
d: Pick<Deps, "project" | "tracker" | "store">,
|
|
2144
|
+
now = Date.now(),
|
|
2145
|
+
): Promise<void> {
|
|
2146
|
+
const { project, tracker, store } = d;
|
|
2147
|
+
const eligible = store
|
|
2148
|
+
.recentRuns(project.name, now - PR_LOOKUP_WINDOW_MS)
|
|
2149
|
+
.filter(
|
|
2150
|
+
(run) =>
|
|
2151
|
+
ADOPTABLE_PR_STATES[run.state] === true &&
|
|
2152
|
+
run.prUrl === undefined &&
|
|
2153
|
+
run.branch.trim() !== "",
|
|
2154
|
+
);
|
|
2155
|
+
const previous = salvagedPrCursor.get(project.name);
|
|
2156
|
+
const previousIndex =
|
|
2157
|
+
previous === undefined ? -1 : eligible.findIndex((run) => run.id === previous);
|
|
2158
|
+
const start = previousIndex === -1 ? 0 : (previousIndex + 1) % eligible.length;
|
|
2159
|
+
const candidates = Array.from(
|
|
2160
|
+
{ length: Math.min(SALVAGED_PR_ADOPTION_BATCH, eligible.length) },
|
|
2161
|
+
(_, offset) => eligible[(start + offset) % eligible.length]!,
|
|
2162
|
+
);
|
|
2163
|
+
const last = candidates.at(-1);
|
|
2164
|
+
if (last !== undefined) salvagedPrCursor.set(project.name, last.id);
|
|
2165
|
+
|
|
2166
|
+
for (const run of candidates) {
|
|
2167
|
+
const repo = project.routing.repos[run.repo];
|
|
2168
|
+
const repoIdentity = repo === undefined ? undefined : githubRepo(repo.cloneUrl);
|
|
2169
|
+
let closers: OpenCloser[];
|
|
2170
|
+
try {
|
|
2171
|
+
closers = await tracker.openClosersFor(run.issue);
|
|
2172
|
+
} catch (err) {
|
|
2173
|
+
log(`#${run.issue} PR adoption lookup failed (${errText(err)}) — retrying next tick`);
|
|
2174
|
+
continue;
|
|
2175
|
+
}
|
|
2176
|
+
|
|
2177
|
+
const closer = closers.find(
|
|
2178
|
+
(candidate) =>
|
|
2179
|
+
candidate.headRefName !== "" &&
|
|
2180
|
+
candidate.headRefName === run.branch &&
|
|
2181
|
+
repo !== undefined &&
|
|
2182
|
+
candidate.repo !== "" &&
|
|
2183
|
+
candidate.repo === repoIdentity,
|
|
2184
|
+
);
|
|
2185
|
+
if (closer === undefined) {
|
|
2186
|
+
if (!rejectedSalvagedPrRuns.has(run.id)) {
|
|
2187
|
+
const observed = closers[0];
|
|
2188
|
+
const reason =
|
|
2189
|
+
observed === undefined
|
|
2190
|
+
? "no open closing PR"
|
|
2191
|
+
: observed.headRefName === ""
|
|
2192
|
+
? "closer has no head branch identity"
|
|
2193
|
+
: observed.headRefName !== run.branch
|
|
2194
|
+
? `closer head ${observed.headRefName} does not match retained branch ${run.branch}`
|
|
2195
|
+
: observed.repo === ""
|
|
2196
|
+
? "closer has no repository identity"
|
|
2197
|
+
: `closer repository ${observed.repo} does not match routed repository`;
|
|
2198
|
+
log(`#${run.issue} PR not adopted onto attempt ${run.attempt}: ${reason}`);
|
|
2199
|
+
rejectedSalvagedPrRuns.add(run.id);
|
|
2200
|
+
}
|
|
2201
|
+
continue;
|
|
2202
|
+
}
|
|
2203
|
+
|
|
2204
|
+
const flag: SettlementFlag = {
|
|
2205
|
+
kind: "pr-adopted",
|
|
2206
|
+
file: "(recovery)",
|
|
2207
|
+
detail: `${closer.url} matched retained branch ${run.branch} in ${closer.repo}`,
|
|
2208
|
+
};
|
|
2209
|
+
store.updateRun(run.id, {
|
|
2210
|
+
prUrl: closer.url,
|
|
2211
|
+
settlementFlags: [...(run.settlementFlags ?? []), flag],
|
|
2212
|
+
});
|
|
2213
|
+
rejectedSalvagedPrRuns.delete(run.id);
|
|
2214
|
+
log(
|
|
2215
|
+
`#${run.issue} adopted PR ${closer.url} onto attempt ${run.attempt}` +
|
|
2216
|
+
(run.salvageSha === undefined ? "" : ` (salvaged head ${run.salvageSha})`),
|
|
2217
|
+
);
|
|
2218
|
+
}
|
|
2219
|
+
}
|
|
2220
|
+
|
|
2221
|
+
/** Canonical `owner/repo` identity from a configured network clone URL. */
|
|
2222
|
+
function githubRepo(cloneUrl: string): string | undefined {
|
|
2223
|
+
const normalized = cloneUrl.replace(/\/$/, "").replace(/\.git$/, "");
|
|
2224
|
+
const match = /^(?:https?:\/\/[^/]+\/|ssh:\/\/git@[^/]+\/|git@[^:]+:)([^/\s]+\/[^/\s]+)$/.exec(
|
|
2225
|
+
normalized,
|
|
2226
|
+
);
|
|
2227
|
+
return match?.[1];
|
|
2228
|
+
}
|
|
2229
|
+
|
|
1723
2230
|
const RETAINED_CLEANUP_BATCH = 10;
|
|
1724
2231
|
|
|
1725
2232
|
export interface RetainedCleanupCursor {
|
|
@@ -1833,6 +2340,7 @@ const HOLD_SAMPLE_SIZE = 5;
|
|
|
1833
2340
|
const DEGRADED_HOLDS: ReadonlySet<AdmissionHoldReason> = new Set([
|
|
1834
2341
|
"parent-lookup-error",
|
|
1835
2342
|
"open-pr-lookup-error",
|
|
2343
|
+
"issue-state-lookup-error",
|
|
1836
2344
|
]);
|
|
1837
2345
|
|
|
1838
2346
|
/** Groups transient decisions into the bounded record exposed by status. */
|
|
@@ -2217,6 +2725,32 @@ export async function admitCandidates(
|
|
|
2217
2725
|
log(`#${issue} continuing retained PR ${closer.url} ${resume}`);
|
|
2218
2726
|
}
|
|
2219
2727
|
|
|
2728
|
+
// The queue comes from GitHub's eventually-consistent search index. Re-read
|
|
2729
|
+
// state and labels directly at the last possible moment so a just-closed or
|
|
2730
|
+
// explicitly dequeued issue cannot turn a stale candidate into another
|
|
2731
|
+
// attempt (#247).
|
|
2732
|
+
let snapshot: IssueSnapshot | undefined;
|
|
2733
|
+
try {
|
|
2734
|
+
snapshot = await tracker.issueSnapshot(issue);
|
|
2735
|
+
} catch {
|
|
2736
|
+
snapshot = undefined;
|
|
2737
|
+
}
|
|
2738
|
+
if (snapshot === undefined) {
|
|
2739
|
+
hold(issue, "issue-state-lookup-error");
|
|
2740
|
+
log(`#${issue} held: issue snapshot check failed — retrying next tick`);
|
|
2741
|
+
continue;
|
|
2742
|
+
}
|
|
2743
|
+
if (snapshot.state === "closed") {
|
|
2744
|
+
hold(issue, "issue-closed");
|
|
2745
|
+
log(`#${issue} skipped: issue is closed (search index lag)`);
|
|
2746
|
+
continue;
|
|
2747
|
+
}
|
|
2748
|
+
if (!snapshot.labels.includes(project.queueLabel)) {
|
|
2749
|
+
hold(issue, "issue-dequeued");
|
|
2750
|
+
log(`#${issue} skipped: queue label ${project.queueLabel} was removed (search index lag)`);
|
|
2751
|
+
continue;
|
|
2752
|
+
}
|
|
2753
|
+
|
|
2220
2754
|
admitted.push({ r, attempt: priorRuns + 1 });
|
|
2221
2755
|
liveByRepo.set(r.repo.name, (liveByRepo.get(r.repo.name) ?? 0) + 1);
|
|
2222
2756
|
if (parent !== undefined) {
|
|
@@ -2305,6 +2839,16 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
2305
2839
|
// while the fleet is parked or workers are still active. Resident workers run
|
|
2306
2840
|
// through the pool without blocking this five-minute tick.
|
|
2307
2841
|
await settlePushedGreen(d);
|
|
2842
|
+
try {
|
|
2843
|
+
await watchMergedBase(d);
|
|
2844
|
+
} catch (err) {
|
|
2845
|
+
log(`base-branch check sweep failed: ${errText(err)}`);
|
|
2846
|
+
}
|
|
2847
|
+
try {
|
|
2848
|
+
await adoptSalvagedPrs(d);
|
|
2849
|
+
} catch (err) {
|
|
2850
|
+
log(`salvaged PR adoption sweep failed: ${errText(err)}`);
|
|
2851
|
+
}
|
|
2308
2852
|
|
|
2309
2853
|
// Immediately after settlement and before any routing, so a class is on the
|
|
2310
2854
|
// row before the next dispatch decision reads its budgets (#132). Above the
|
|
@@ -2548,6 +3092,8 @@ export interface DaemonHealthSnapshot {
|
|
|
2548
3092
|
paused: boolean;
|
|
2549
3093
|
activeRuns: number;
|
|
2550
3094
|
project: string;
|
|
3095
|
+
/** One-shot issue ceilings waiting for the next claim. */
|
|
3096
|
+
turnOverrides: TurnOverride[];
|
|
2551
3097
|
/** Resident set of this daemon; workers are in-process omp sessions. */
|
|
2552
3098
|
rssBytes: number;
|
|
2553
3099
|
dispatch?: DispatchSummary;
|
|
@@ -2569,6 +3115,7 @@ export function daemonHealthSnapshot(
|
|
|
2569
3115
|
ok: true,
|
|
2570
3116
|
paused,
|
|
2571
3117
|
activeRuns: store.activeRuns(project).length,
|
|
3118
|
+
turnOverrides: store.listTurnOverrides(project),
|
|
2572
3119
|
project,
|
|
2573
3120
|
rssBytes,
|
|
2574
3121
|
...(dispatch === undefined ? {} : { dispatch }),
|
|
@@ -2577,11 +3124,19 @@ export function daemonHealthSnapshot(
|
|
|
2577
3124
|
};
|
|
2578
3125
|
}
|
|
2579
3126
|
|
|
3127
|
+
const TURN_OVERRIDE_STATES: ReadonlySet<RunState> = new Set([
|
|
3128
|
+
"failed",
|
|
3129
|
+
"killed",
|
|
3130
|
+
"orphaned",
|
|
3131
|
+
"blocked",
|
|
3132
|
+
]);
|
|
3133
|
+
|
|
2580
3134
|
export async function turnLimitResponse(
|
|
2581
3135
|
req: Request,
|
|
2582
3136
|
project: string,
|
|
2583
|
-
store: Pick<Store, "latestRun">,
|
|
3137
|
+
store: Pick<Store, "latestRun" | "setTurnOverride">,
|
|
2584
3138
|
registry: TurnLimitRegistry,
|
|
3139
|
+
caps: Pick<Caps, "workerMaxTurns" | "workerMaxTurnsCeiling">,
|
|
2585
3140
|
): Promise<Response | undefined> {
|
|
2586
3141
|
const url = new URL(req.url);
|
|
2587
3142
|
const match = /^\/runs\/(\d+)\/turn-limit$/.exec(url.pathname);
|
|
@@ -2615,6 +3170,16 @@ export async function turnLimitResponse(
|
|
|
2615
3170
|
}
|
|
2616
3171
|
|
|
2617
3172
|
const issue = Number(match[1]);
|
|
3173
|
+
if ((maxTurns as number) > caps.workerMaxTurnsCeiling) {
|
|
3174
|
+
return Response.json(
|
|
3175
|
+
{
|
|
3176
|
+
error:
|
|
3177
|
+
`#${issue} turn budget ${maxTurns as number} exceeds the ` +
|
|
3178
|
+
`${caps.workerMaxTurnsCeiling}-turn caps.workerMaxTurnsCeiling`,
|
|
3179
|
+
},
|
|
3180
|
+
{ status: 422 },
|
|
3181
|
+
);
|
|
3182
|
+
}
|
|
2618
3183
|
const outcome = registry.extend(project, issue, maxTurns as number);
|
|
2619
3184
|
if (outcome.kind === "extended") return Response.json(outcome);
|
|
2620
3185
|
if (outcome.kind === "not-increase") {
|
|
@@ -2628,14 +3193,33 @@ export async function turnLimitResponse(
|
|
|
2628
3193
|
if (latest === undefined) {
|
|
2629
3194
|
return Response.json({ error: `no run recorded for #${issue}` }, { status: 404 });
|
|
2630
3195
|
}
|
|
2631
|
-
|
|
2632
|
-
|
|
2633
|
-
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
|
|
3196
|
+
if (!TURN_OVERRIDE_STATES.has(latest.state)) {
|
|
3197
|
+
return Response.json(
|
|
3198
|
+
{
|
|
3199
|
+
error:
|
|
3200
|
+
`#${issue} has no live worker controller; its session already settled ` +
|
|
3201
|
+
`or belongs to another daemon (stored state: ${latest.state})`,
|
|
3202
|
+
},
|
|
3203
|
+
{ status: 409 },
|
|
3204
|
+
);
|
|
3205
|
+
}
|
|
3206
|
+
if ((maxTurns as number) <= caps.workerMaxTurns) {
|
|
3207
|
+
return Response.json(
|
|
3208
|
+
{
|
|
3209
|
+
error:
|
|
3210
|
+
`#${issue} next-attempt turn budget must exceed the ` +
|
|
3211
|
+
`${caps.workerMaxTurns}-turn caps.workerMaxTurns base`,
|
|
3212
|
+
},
|
|
3213
|
+
{ status: 409 },
|
|
3214
|
+
);
|
|
3215
|
+
}
|
|
3216
|
+
store.setTurnOverride(project, issue, maxTurns as number);
|
|
3217
|
+
return Response.json({
|
|
3218
|
+
kind: "next-attempt",
|
|
3219
|
+
issue,
|
|
3220
|
+
nextAttemptMaxTurns: maxTurns,
|
|
3221
|
+
baseMaxTurns: caps.workerMaxTurns,
|
|
3222
|
+
});
|
|
2639
3223
|
}
|
|
2640
3224
|
|
|
2641
3225
|
export async function workerControlResponse(
|
|
@@ -2645,7 +3229,7 @@ export async function workerControlResponse(
|
|
|
2645
3229
|
registry: WorkerControlRegistry,
|
|
2646
3230
|
): Promise<Response | undefined> {
|
|
2647
3231
|
const url = new URL(req.url);
|
|
2648
|
-
const match = /^\/runs\/(\d+)\/(pause|resume)$/.exec(url.pathname);
|
|
3232
|
+
const match = /^\/runs\/(\d+)\/(pause|resume|stop)$/.exec(url.pathname);
|
|
2649
3233
|
if (req.method !== "PUT" || match === null) return undefined;
|
|
2650
3234
|
if (!req.headers.get("content-type")?.startsWith("application/json")) {
|
|
2651
3235
|
return Response.json({ error: "content-type must be application/json" }, { status: 415 });
|
|
@@ -2670,23 +3254,71 @@ export async function workerControlResponse(
|
|
|
2670
3254
|
{ status: 409 },
|
|
2671
3255
|
);
|
|
2672
3256
|
}
|
|
3257
|
+
const action = match[2] as "pause" | "resume" | "stop";
|
|
3258
|
+
let reason: string | undefined;
|
|
3259
|
+
if (action === "stop") {
|
|
3260
|
+
const rawReason = Reflect.get(body, "reason");
|
|
3261
|
+
if (typeof rawReason !== "string" || rawReason.trim() === "") {
|
|
3262
|
+
return Response.json({ error: "reason must be a non-empty string" }, { status: 400 });
|
|
3263
|
+
}
|
|
3264
|
+
reason = rawReason.trim().replace(/\s+/g, " ");
|
|
3265
|
+
if (reason.length > 500) {
|
|
3266
|
+
return Response.json({ error: "reason must be at most 500 characters" }, { status: 400 });
|
|
3267
|
+
}
|
|
3268
|
+
}
|
|
3269
|
+
|
|
2673
3270
|
|
|
2674
3271
|
const issue = Number(match[1]);
|
|
2675
3272
|
const outcome =
|
|
2676
|
-
|
|
3273
|
+
action === "pause"
|
|
2677
3274
|
? await registry.pause(project, issue)
|
|
2678
|
-
:
|
|
3275
|
+
: action === "resume"
|
|
3276
|
+
? registry.resume(project, issue)
|
|
3277
|
+
: await registry.stop(project, issue, reason!);
|
|
2679
3278
|
if (outcome.kind === "ok") {
|
|
2680
3279
|
return Response.json({ runId: outcome.runId, phase: outcome.phase });
|
|
2681
3280
|
}
|
|
2682
3281
|
if (outcome.kind === "refused") {
|
|
2683
3282
|
return Response.json({ error: `#${issue}: ${outcome.error}` }, { status: 409 });
|
|
2684
3283
|
}
|
|
3284
|
+
if (outcome.kind === "stopped") {
|
|
3285
|
+
const stopped = store.latestRun(project, issue);
|
|
3286
|
+
if (stopped === undefined || stopped.id !== outcome.runId) {
|
|
3287
|
+
return Response.json(
|
|
3288
|
+
{ error: `#${issue} stopped, but its terminal run record is unavailable` },
|
|
3289
|
+
{ status: 500 },
|
|
3290
|
+
);
|
|
3291
|
+
}
|
|
3292
|
+
if (stopped.state !== "stopped") {
|
|
3293
|
+
return Response.json({
|
|
3294
|
+
outcome: "already-terminal",
|
|
3295
|
+
runId: stopped.id,
|
|
3296
|
+
state: stopped.state,
|
|
3297
|
+
});
|
|
3298
|
+
}
|
|
3299
|
+
return Response.json({
|
|
3300
|
+
outcome: "stopped",
|
|
3301
|
+
runId: stopped.id,
|
|
3302
|
+
state: stopped.state,
|
|
3303
|
+
reason: outcome.reason,
|
|
3304
|
+
...(stopped.salvageSha === undefined ? {} : { salvageSha: stopped.salvageSha }),
|
|
3305
|
+
...(stopped.salvageError === undefined ? {} : { salvageError: stopped.salvageError }),
|
|
3306
|
+
worktree: stopped.worktree,
|
|
3307
|
+
});
|
|
3308
|
+
}
|
|
3309
|
+
|
|
2685
3310
|
|
|
2686
3311
|
const latest = store.latestRun(project, issue);
|
|
2687
3312
|
if (latest === undefined) {
|
|
2688
3313
|
return Response.json({ error: `no run recorded for #${issue}` }, { status: 404 });
|
|
2689
3314
|
}
|
|
3315
|
+
if (action === "stop" && !LIVE_STATES.includes(latest.state)) {
|
|
3316
|
+
return Response.json({
|
|
3317
|
+
outcome: "already-terminal",
|
|
3318
|
+
runId: latest.id,
|
|
3319
|
+
state: latest.state,
|
|
3320
|
+
});
|
|
3321
|
+
}
|
|
2690
3322
|
return Response.json(
|
|
2691
3323
|
{
|
|
2692
3324
|
error:
|
|
@@ -2699,7 +3331,8 @@ export async function workerControlResponse(
|
|
|
2699
3331
|
|
|
2700
3332
|
export interface DaemonHttpDeps {
|
|
2701
3333
|
project: string;
|
|
2702
|
-
store: Pick<Store, "latestRun">;
|
|
3334
|
+
store: Pick<Store, "latestRun" | "setTurnOverride">;
|
|
3335
|
+
caps: () => Pick<Caps, "workerMaxTurns" | "workerMaxTurnsCeiling">;
|
|
2703
3336
|
turnLimits: TurnLimitRegistry;
|
|
2704
3337
|
workerControls: WorkerControlRegistry;
|
|
2705
3338
|
health: () => DaemonHealthSnapshot;
|
|
@@ -2709,15 +3342,15 @@ export interface DaemonHttpDeps {
|
|
|
2709
3342
|
* The whole HTTP surface, in one named function so a test can pin what is *not*
|
|
2710
3343
|
* on it.
|
|
2711
3344
|
*
|
|
2712
|
-
* Three route families: the health read, turn-limit control, and worker
|
|
2713
|
-
* pause/resume control. Everything else is 404. The controls mutate only
|
|
3345
|
+
* Three route families: the health read, turn-limit control, and live-worker
|
|
3346
|
+
* pause/resume/stop control. Everything else is 404. The controls mutate only
|
|
2714
3347
|
* daemon-owned live sessions; tracker and repository mutations stay on the
|
|
2715
3348
|
* authenticated per-run channel described at the `Bun.serve` call (#126).
|
|
2716
3349
|
*/
|
|
2717
3350
|
export async function daemonHttpResponse(req: Request, d: DaemonHttpDeps): Promise<Response> {
|
|
2718
3351
|
const url = new URL(req.url);
|
|
2719
3352
|
if (req.method === "GET" && url.pathname === "/healthz") return Response.json(d.health());
|
|
2720
|
-
const turnLimit = await turnLimitResponse(req, d.project, d.store, d.turnLimits);
|
|
3353
|
+
const turnLimit = await turnLimitResponse(req, d.project, d.store, d.turnLimits, d.caps());
|
|
2721
3354
|
if (turnLimit !== undefined) return turnLimit;
|
|
2722
3355
|
const workerControl = await workerControlResponse(req, d.project, d.store, d.workerControls);
|
|
2723
3356
|
return workerControl ?? new Response("not found\n", { status: 404 });
|
|
@@ -2747,6 +3380,8 @@ export interface StatusSnapshot {
|
|
|
2747
3380
|
/** Newest attempts holding a preserved WIP tip, or a tree that is still the
|
|
2748
3381
|
* only copy of work the daemon could not save. */
|
|
2749
3382
|
salvagedRuns: RunRecord[];
|
|
3383
|
+
/** One-shot issue ceilings waiting for the next claim. */
|
|
3384
|
+
turnOverrides: TurnOverride[];
|
|
2750
3385
|
/** Reports the operator has not provably received: pending, in-flight with an
|
|
2751
3386
|
* unknown outcome, or written off. An empty list is the only honest way to
|
|
2752
3387
|
* say "everything authored this cycle actually went out" (#123). */
|
|
@@ -2758,6 +3393,8 @@ export interface StatusSnapshot {
|
|
|
2758
3393
|
* config does not let it, and an operator who has to know to go looking is an
|
|
2759
3394
|
* operator who finds out from the tracker instead.
|
|
2760
3395
|
*/
|
|
3396
|
+
/** Newest post-merge base verdict per routed repository within seven days. */
|
|
3397
|
+
baseChecks: RunRecord[];
|
|
2761
3398
|
verbLedger: VerbLedgerEntry[];
|
|
2762
3399
|
/** Runs backed by a worker process — the number capacity compares against. */
|
|
2763
3400
|
liveWorkers: number;
|
|
@@ -2821,6 +3458,7 @@ export function statusSnapshotFromStore(
|
|
|
2821
3458
|
releaseGrants: resolveReleaseGrants(p),
|
|
2822
3459
|
activeRuns: store.activeRuns(p.name),
|
|
2823
3460
|
salvagedRuns: store.salvagedRuns(p.name),
|
|
3461
|
+
turnOverrides: store.listTurnOverrides(p.name),
|
|
2824
3462
|
openReports: store.openReports(p.name),
|
|
2825
3463
|
verbLedger: store.verbLedger(p.name, { limit: STATUS_LEDGER_SCAN }),
|
|
2826
3464
|
liveWorkers: store.liveRuns(p.name).length,
|
|
@@ -2835,6 +3473,7 @@ export function statusSnapshotFromStore(
|
|
|
2835
3473
|
...(labelOpsPending === 0 || oldestLabelOpAt === undefined
|
|
2836
3474
|
? {}
|
|
2837
3475
|
: { labelOps: { pending: labelOpsPending, oldestAgeMs: Date.now() - oldestLabelOpAt } }),
|
|
3476
|
+
baseChecks: store.latestBaseChecks(p.name, Date.now() - BASE_STATUS_WINDOW_MS),
|
|
2838
3477
|
};
|
|
2839
3478
|
}
|
|
2840
3479
|
|
|
@@ -2917,6 +3556,21 @@ export function formatReleaseGrants(grants: ResolvedGrants): string[] {
|
|
|
2917
3556
|
...RELEASE_SHAPES.map((shape) => ` ${shape.padEnd(19)}${grants[shape]}`),
|
|
2918
3557
|
];
|
|
2919
3558
|
}
|
|
3559
|
+
export function formatBaseChecks(runs: readonly RunRecord[]): string[] {
|
|
3560
|
+
return runs.flatMap((run) => {
|
|
3561
|
+
if (run.baseCheck === undefined) return [];
|
|
3562
|
+
const branch = run.baseRef ?? "?";
|
|
3563
|
+
const flag = run.settlementFlags?.find((candidate) => candidate.kind === "base-branch-red");
|
|
3564
|
+
const verdict =
|
|
3565
|
+
run.baseCheck === "red"
|
|
3566
|
+
? `RED — ${flag?.detail ?? "workflow failed after merge"}`
|
|
3567
|
+
: run.baseCheck === "red-preexisting"
|
|
3568
|
+
? `red before merge${flag === undefined ? "" : ` — ${flag.detail}`}`
|
|
3569
|
+
: run.baseCheck;
|
|
3570
|
+
return [`base ${run.repo}/${branch} ${verdict}`];
|
|
3571
|
+
});
|
|
3572
|
+
}
|
|
3573
|
+
|
|
2920
3574
|
|
|
2921
3575
|
export function formatStatus(s: StatusSnapshot): string {
|
|
2922
3576
|
const lines = [
|
|
@@ -2935,6 +3589,9 @@ export function formatStatus(s: StatusSnapshot): string {
|
|
|
2935
3589
|
// fleet (#110).
|
|
2936
3590
|
` plan usage ${planUsageLine(s.planUsage)}`,
|
|
2937
3591
|
` new worker turns ${s.caps.workerMaxTurns}`,
|
|
3592
|
+
...s.turnOverrides.map(
|
|
3593
|
+
({ issue, maxTurns }) => ` turn override #${issue} → ${maxTurns} (next attempt)`,
|
|
3594
|
+
),
|
|
2938
3595
|
` worker wall clock ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
|
|
2939
3596
|
` failed attempts ${s.caps.maxAttemptsPerIssue}`,
|
|
2940
3597
|
` continuations ${s.caps.maxContinuationsPerIssue}`,
|
|
@@ -2961,6 +3618,7 @@ export function formatStatus(s: StatusSnapshot): string {
|
|
|
2961
3618
|
if (flagged !== undefined) lines.push(` ${flagged}`);
|
|
2962
3619
|
}
|
|
2963
3620
|
}
|
|
3621
|
+
lines.push(...formatBaseChecks(s.baseChecks));
|
|
2964
3622
|
lines.push(...formatSalvagedRuns(s.salvagedRuns));
|
|
2965
3623
|
lines.push(...formatOpenReports(s.openReports));
|
|
2966
3624
|
lines.push(...formatVerbLedger(s.verbLedger));
|
|
@@ -3096,6 +3754,15 @@ export interface SessionError {
|
|
|
3096
3754
|
message: string;
|
|
3097
3755
|
}
|
|
3098
3756
|
|
|
3757
|
+
export function completionLastError(
|
|
3758
|
+
providerCredit: string | undefined,
|
|
3759
|
+
providerTransient: string | undefined,
|
|
3760
|
+
verifiedReason: string | undefined,
|
|
3761
|
+
sessionErr: SessionError | undefined,
|
|
3762
|
+
): string | undefined {
|
|
3763
|
+
return providerCredit ?? providerTransient ?? verifiedReason ?? sessionErr?.message;
|
|
3764
|
+
}
|
|
3765
|
+
|
|
3099
3766
|
/**
|
|
3100
3767
|
* The last error a transcript recorded, or undefined when it recorded none.
|
|
3101
3768
|
*
|
|
@@ -3162,6 +3829,19 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
|
|
|
3162
3829
|
const { project, tracker, store } = d;
|
|
3163
3830
|
for (const run of store.runsNeedingClassification(project.name, CLASSIFY_BATCH)) {
|
|
3164
3831
|
const facts: ClassifyFacts = {};
|
|
3832
|
+
let classifiedRun = run;
|
|
3833
|
+
if (run.state === "failed" || run.state === "killed") {
|
|
3834
|
+
const sessionError = readSessionError(run.sessionFile);
|
|
3835
|
+
if (sessionError !== undefined) {
|
|
3836
|
+
if (run.lastError === undefined || run.lastError === sessionError.message) {
|
|
3837
|
+
facts.sessionError = sessionError;
|
|
3838
|
+
}
|
|
3839
|
+
if (run.lastError === undefined) {
|
|
3840
|
+
store.updateRun(run.id, { lastError: sessionError.message });
|
|
3841
|
+
classifiedRun = { ...run, lastError: sessionError.message };
|
|
3842
|
+
}
|
|
3843
|
+
}
|
|
3844
|
+
}
|
|
3165
3845
|
try {
|
|
3166
3846
|
if (run.prUrl !== undefined) {
|
|
3167
3847
|
const pr = await tracker.prState(run.prUrl);
|
|
@@ -3188,7 +3868,7 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
|
|
|
3188
3868
|
continue;
|
|
3189
3869
|
}
|
|
3190
3870
|
|
|
3191
|
-
const { cls, recovery, evidence } = classifyRun(
|
|
3871
|
+
const { cls, recovery, evidence } = classifyRun(classifiedRun, facts);
|
|
3192
3872
|
|
|
3193
3873
|
// A healthy green PR is not a failure of any class. Leaving the row
|
|
3194
3874
|
// unclassified is what keeps it eligible for the sweep on the tick where its
|
|
@@ -3202,7 +3882,7 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
|
|
|
3202
3882
|
? `#${run.issue} retrying ${recovery} for ${cls}: ${evidence}`
|
|
3203
3883
|
: `#${run.issue} classified ${cls} → ${recovery}: ${evidence}`,
|
|
3204
3884
|
);
|
|
3205
|
-
await recoverRun(d,
|
|
3885
|
+
await recoverRun(d, classifiedRun, cls, recovery, evidence);
|
|
3206
3886
|
}
|
|
3207
3887
|
}
|
|
3208
3888
|
|
|
@@ -3268,6 +3948,9 @@ async function recoverRun(
|
|
|
3268
3948
|
}
|
|
3269
3949
|
|
|
3270
3950
|
if (recovery === "requeue") {
|
|
3951
|
+
if (cls === "provider-credit") {
|
|
3952
|
+
await reactToProviderCredit(d, run.issue, evidence, run.sessionFile);
|
|
3953
|
+
}
|
|
3271
3954
|
// A dispatch-infra requeue that keeps landing on the same issue means the
|
|
3272
3955
|
// mirror for its repo is persistently broken — a ref-lock that retry already
|
|
3273
3956
|
// exhausted, a dead remote. Requeueing forever spends a turn-0 run per tick
|
|
@@ -3682,7 +4365,8 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
3682
4365
|
onChildLog: (line) => {
|
|
3683
4366
|
log(`orchestrator ${line}`);
|
|
3684
4367
|
},
|
|
3685
|
-
onReleaseBlocked: (shape) =>
|
|
4368
|
+
onReleaseBlocked: (shape, context) =>
|
|
4369
|
+
recordReleaseBlock(project.name, "orchestrator", shape, context),
|
|
3686
4370
|
});
|
|
3687
4371
|
const transcript = orchestrator.sessionFile();
|
|
3688
4372
|
log(`orchestrator session ready${transcript === undefined ? "" : ` · ${transcript}`}`);
|
|
@@ -3821,9 +4505,11 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
3821
4505
|
// carries no credential of any kind, and it cannot tell one caller from
|
|
3822
4506
|
// another: `127.0.0.1` is not an identity. The turn-limit and worker
|
|
3823
4507
|
// pause/resume controls trust a body-supplied `project`, which is exactly the
|
|
3824
|
-
// shape "identity from the payload" takes when nobody is watching.
|
|
3825
|
-
//
|
|
3826
|
-
//
|
|
4508
|
+
// shape "identity from the payload" takes when nobody is watching. They stay
|
|
4509
|
+
// tolerable only because every effect is bounded: pauses are reversible,
|
|
4510
|
+
// live extensions touch one controller, and a persisted next-attempt
|
|
4511
|
+
// override can only raise the project base up to its configured ceiling and
|
|
4512
|
+
// is consumed by one claim.
|
|
3827
4513
|
//
|
|
3828
4514
|
// A merge, a push, a release or a label is none of those things. Do not add
|
|
3829
4515
|
// one here, and do not add "just a small one" behind a shared secret either:
|
|
@@ -3838,6 +4524,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
3838
4524
|
daemonHttpResponse(req, {
|
|
3839
4525
|
project: project.name,
|
|
3840
4526
|
store,
|
|
4527
|
+
caps: () => d.caps,
|
|
3841
4528
|
turnLimits,
|
|
3842
4529
|
workerControls,
|
|
3843
4530
|
health: () =>
|