omp-conductor 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +253 -84
- package/package.json +1 -1
- package/src/availability.ts +165 -0
- package/src/board.ts +1 -1
- package/src/briefs/orchestrator.md +95 -28
- package/src/briefs/policy.md +44 -32
- package/src/cli.ts +226 -37
- package/src/config.ts +123 -7
- package/src/daemon.ts +1018 -145
- package/src/diff-flags.ts +77 -4
- package/src/digest-schedule.ts +92 -24
- package/src/escalate.ts +46 -19
- package/src/failure-class.ts +7 -5
- package/src/fleet.ts +39 -3
- package/src/omp.ts +8 -4
- package/src/orchestrator-tick.ts +471 -39
- package/src/orchestrator.ts +3 -2
- package/src/plugin.ts +166 -14
- package/src/release-policy.ts +66 -6
- package/src/reports.ts +202 -5
- package/src/session-host.ts +4 -3
- package/src/setup.ts +197 -35
- package/src/store.ts +785 -112
- package/src/tracker/github.ts +299 -56
- package/src/types.ts +289 -37
- package/src/verbs/actions.ts +245 -15
- package/src/verbs/protocol.ts +7 -6
- package/src/verbs/server.ts +94 -13
- package/src/worker.ts +51 -11
- package/src/worktree.ts +5 -0
package/src/daemon.ts
CHANGED
|
@@ -18,11 +18,13 @@ import {
|
|
|
18
18
|
resolveReleaseGrants,
|
|
19
19
|
stateDir,
|
|
20
20
|
} from "./config.ts";
|
|
21
|
+
import { availabilityState, type AvailabilityState } from "./availability.ts";
|
|
21
22
|
import {
|
|
22
23
|
analyseSettlement,
|
|
23
24
|
formatSettlementFlags,
|
|
24
25
|
settlementFlagSummary,
|
|
25
26
|
} from "./diff-flags.ts";
|
|
27
|
+
import { digestScheduleState, type DigestScheduleState } from "./digest-schedule.ts";
|
|
26
28
|
import { createEscalator, escalationIssueRef } from "./escalate.ts";
|
|
27
29
|
import { pendingCodeGraph, probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
|
|
28
30
|
import { graphHint } from "./graph.ts";
|
|
@@ -30,22 +32,34 @@ import { livingDaemon } from "./lifecycle.ts";
|
|
|
30
32
|
import { STALL_MARKER_FILE } from "./orchestrator-tick.ts";
|
|
31
33
|
import { startOrchestrator } from "./orchestrator.ts";
|
|
32
34
|
import type { OrchestratorHandle } from "./orchestrator.ts";
|
|
33
|
-
import {
|
|
34
|
-
|
|
35
|
+
import {
|
|
36
|
+
createReportOutbox,
|
|
37
|
+
enqueueAvailableHeldNotices,
|
|
38
|
+
formatOpenReports,
|
|
39
|
+
} from "./reports.ts";
|
|
40
|
+
import {
|
|
41
|
+
recordReleaseBlock,
|
|
42
|
+
type ReleaseBlockContext,
|
|
43
|
+
} from "./release-policy.ts";
|
|
35
44
|
import { branchName, effectiveLabels, route } from "./routing.ts";
|
|
36
45
|
import type { Routed, UnroutableReason } from "./routing.ts";
|
|
37
46
|
import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
|
|
38
47
|
import { classifyRun, providerCreditRefusal, providerTransientFault, type ClassifyFacts } from "./failure-class.ts";
|
|
39
48
|
import { projectLabels } from "./label-projection.ts";
|
|
40
|
-
import { dbPath, openStore, utcDay } from "./store.ts";
|
|
49
|
+
import { dbPath, LIVE_STATES, openStore, utcDay } from "./store.ts";
|
|
41
50
|
import { makeTracker, type RateLimitStatus } from "./tracker/github.ts";
|
|
42
|
-
import { RELEASE_SHAPES } from "./types.ts";
|
|
51
|
+
import { DEFAULT_REPORT_POLICY, RELEASE_SHAPES } from "./types.ts";
|
|
43
52
|
import type {
|
|
53
|
+
BaseHealth,
|
|
44
54
|
AdmissionHoldReason,
|
|
45
55
|
Caps,
|
|
46
56
|
DispatchSummary,
|
|
57
|
+
DigestBacklog,
|
|
47
58
|
Escalation,
|
|
59
|
+
IssueSnapshot,
|
|
60
|
+
MergedPrInfo,
|
|
48
61
|
OpenCloser,
|
|
62
|
+
ReleaseShape,
|
|
49
63
|
PrState,
|
|
50
64
|
ProjectConfig,
|
|
51
65
|
ReadyIssue,
|
|
@@ -60,12 +74,14 @@ import type {
|
|
|
60
74
|
Store,
|
|
61
75
|
Tracker,
|
|
62
76
|
VerbLedgerEntry,
|
|
77
|
+
TurnOverride,
|
|
63
78
|
} from "./types.ts";
|
|
64
79
|
import {
|
|
65
80
|
type KilledBy,
|
|
66
81
|
type WorkerPauseControl,
|
|
67
82
|
type WorkerPausePhase,
|
|
68
83
|
type WorkerResult,
|
|
84
|
+
type RunWorkerDeps,
|
|
69
85
|
renderBrief,
|
|
70
86
|
runWorker,
|
|
71
87
|
} from "./worker.ts";
|
|
@@ -82,8 +98,13 @@ import {
|
|
|
82
98
|
} from "./worktree.ts";
|
|
83
99
|
import { githubVerbActions } from "./verbs/actions.ts";
|
|
84
100
|
import { formatVerbLedger, STATUS_LEDGER_SCAN } from "./verbs/ledger.ts";
|
|
85
|
-
import {
|
|
86
|
-
|
|
101
|
+
import {
|
|
102
|
+
listenVerbChannel,
|
|
103
|
+
PR_LOOKUP_WINDOW_MS,
|
|
104
|
+
type VerbActions,
|
|
105
|
+
type VerbDeps,
|
|
106
|
+
type VerbListener,
|
|
107
|
+
} from "./verbs/server.ts";
|
|
87
108
|
import {
|
|
88
109
|
ensureVerbSocketDir,
|
|
89
110
|
peerCredentialReader,
|
|
@@ -120,6 +141,9 @@ const DISPATCH_INFRA_MAX_STRIKES = 3;
|
|
|
120
141
|
* provider itself is degraded, not unlucky, and the sweep escalates to a
|
|
121
142
|
* human instead of requeueing into a down provider forever (#220). */
|
|
122
143
|
const PROVIDER_TRANSIENT_MAX_STRIKES = 3;
|
|
144
|
+
/** Terminal rows inspected for a salvaged PR per tick. GitHub provenance reads
|
|
145
|
+
* are maintenance, but a backlog must not turn one tick into an API burst. */
|
|
146
|
+
const SALVAGED_PR_ADOPTION_BATCH = 10;
|
|
123
147
|
const DEFAULT_PORT = 8787;
|
|
124
148
|
const BRIEF_TEMPLATE_PATH = join(import.meta.dir, "briefs", "worker.md");
|
|
125
149
|
|
|
@@ -151,12 +175,17 @@ interface Deps {
|
|
|
151
175
|
caps: Caps;
|
|
152
176
|
tracker: Tracker;
|
|
153
177
|
store: Store;
|
|
178
|
+
/** False after a live config reload fails; autonomous delivery then holds
|
|
179
|
+
* fail-closed until a later tick validates the config again. */
|
|
180
|
+
deliveryPolicyValid?: boolean;
|
|
154
181
|
/** Provider-reported plan allowance, cached with a TTL. Resolved once at
|
|
155
182
|
* startup like every other dep so a tick cannot swap its own meter. */
|
|
156
183
|
usage: UsageSource;
|
|
157
184
|
escalate(e: Escalation): Promise<void>;
|
|
158
185
|
turnLimits: TurnLimitRegistry;
|
|
159
186
|
workerControls: WorkerControlRegistry;
|
|
187
|
+
/** Session seam for lifecycle integration tests; production uses the real harness. */
|
|
188
|
+
workerDeps?: RunWorkerDeps;
|
|
160
189
|
integrity: IntegrityGate;
|
|
161
190
|
stall: StallGate;
|
|
162
191
|
cleanup?: RetainedCleanupCursor;
|
|
@@ -212,18 +241,25 @@ export function verbDeps(d: Pick<Deps, "project" | "store" | "tracker" | "verbAc
|
|
|
212
241
|
|
|
213
242
|
// ------------------------------------------------------- orchestrator liveness
|
|
214
243
|
|
|
244
|
+
/** No more than one reminder per hour while the same marker remains. */
|
|
245
|
+
const STALL_RENOTIFY_MS = 60 * 60_000;
|
|
246
|
+
|
|
215
247
|
/**
|
|
216
|
-
* Whether the wedged-orchestrator page has
|
|
217
|
-
* currently on disk.
|
|
218
|
-
* consumed, so paging per five minutes would be paging forever.
|
|
248
|
+
* Whether the wedged-orchestrator page has gone out, and when it last went out,
|
|
249
|
+
* for the stall currently on disk.
|
|
219
250
|
*/
|
|
220
251
|
export interface StallGate {
|
|
221
252
|
paged: boolean;
|
|
253
|
+
lastPagedAt?: number;
|
|
222
254
|
}
|
|
223
255
|
|
|
224
256
|
export interface StallVerdict {
|
|
225
257
|
/** The marker's own line, when one is there. */
|
|
226
258
|
since?: string;
|
|
259
|
+
/** Count written by the stalled tick producer, when its marker is readable. */
|
|
260
|
+
unconsumedTicks?: number;
|
|
261
|
+
/** Whole hours elapsed since the marker timestamp, when it is parseable. */
|
|
262
|
+
stalledHours?: number;
|
|
227
263
|
page: boolean;
|
|
228
264
|
}
|
|
229
265
|
|
|
@@ -246,51 +282,84 @@ export interface StallVerdict {
|
|
|
246
282
|
*
|
|
247
283
|
* Resets when the marker disappears, so a second stall days later pages again.
|
|
248
284
|
*/
|
|
249
|
-
export function checkStall(gate: StallGate, marker: string): StallVerdict {
|
|
285
|
+
export function checkStall(gate: StallGate, marker: string, now = Date.now()): StallVerdict {
|
|
250
286
|
if (!existsSync(marker)) {
|
|
251
287
|
gate.paged = false;
|
|
288
|
+
delete gate.lastPagedAt;
|
|
252
289
|
return { page: false };
|
|
253
290
|
}
|
|
254
|
-
const page =
|
|
291
|
+
const page =
|
|
292
|
+
!gate.paged ||
|
|
293
|
+
gate.lastPagedAt === undefined ||
|
|
294
|
+
now - gate.lastPagedAt >= STALL_RENOTIFY_MS;
|
|
255
295
|
let since: string | undefined;
|
|
296
|
+
let unconsumedTicks: number | undefined;
|
|
297
|
+
let stalledHours: number | undefined;
|
|
256
298
|
try {
|
|
257
299
|
const body = readFileSync(marker, "utf8").split("\n")[0]?.trim();
|
|
258
|
-
if (body !== undefined && body !== "")
|
|
300
|
+
if (body !== undefined && body !== "") {
|
|
301
|
+
since = body;
|
|
302
|
+
const ticks = /\b(\d+) ticks queued unconsumed\b/.exec(body)?.[1];
|
|
303
|
+
if (ticks !== undefined) unconsumedTicks = Number(ticks);
|
|
304
|
+
const startedAt = Date.parse(body.split(/\s+/, 1)[0] ?? "");
|
|
305
|
+
if (Number.isFinite(startedAt)) {
|
|
306
|
+
stalledHours = Math.max(0, Math.floor((now - startedAt) / STALL_RENOTIFY_MS));
|
|
307
|
+
}
|
|
308
|
+
}
|
|
259
309
|
} catch {
|
|
260
|
-
// An unreadable marker still means stalled;
|
|
310
|
+
// An unreadable marker still means stalled; its evidence is a nicety.
|
|
261
311
|
}
|
|
262
|
-
return {
|
|
312
|
+
return {
|
|
313
|
+
...(since === undefined ? {} : { since }),
|
|
314
|
+
...(unconsumedTicks === undefined ? {} : { unconsumedTicks }),
|
|
315
|
+
...(stalledHours === undefined ? {} : { stalledHours }),
|
|
316
|
+
page,
|
|
317
|
+
};
|
|
263
318
|
}
|
|
264
319
|
|
|
265
320
|
/**
|
|
266
|
-
* Pages tier 2
|
|
321
|
+
* Pages tier 2 when the orchestrator session stops draining its queue.
|
|
267
322
|
*
|
|
268
323
|
* Deliberately does not restart anything. A wedge lands mid-turn, this process
|
|
269
324
|
* cannot tell a half-applied edit from an idle loop, and killing the session
|
|
270
325
|
* could destroy work an operator would rather read first — the same refusal to
|
|
271
326
|
* guess that the recovery plugin is built on.
|
|
272
327
|
*/
|
|
273
|
-
async function watchOrchestrator(d: Deps): Promise<void> {
|
|
328
|
+
export async function watchOrchestrator(d: Deps, now = Date.now()): Promise<void> {
|
|
274
329
|
const marker = join(stateDir(), STALL_MARKER_FILE);
|
|
275
|
-
const
|
|
330
|
+
const repeat = d.stall.paged;
|
|
331
|
+
const verdict = checkStall(d.stall, marker, now);
|
|
276
332
|
if (!verdict.page) return;
|
|
277
333
|
|
|
334
|
+
const greenRuns = d.store.activeRuns(d.project.name).filter((run) => run.state === "pushed-green");
|
|
335
|
+
const repeatSuffix = repeat
|
|
336
|
+
? ` — still stalled (${
|
|
337
|
+
verdict.stalledHours === undefined
|
|
338
|
+
? `${new Date(now).toISOString().slice(0, 13)}Z`
|
|
339
|
+
: `${verdict.stalledHours}h`
|
|
340
|
+
})`
|
|
341
|
+
: "";
|
|
342
|
+
|
|
278
343
|
log(`ERROR: the orchestrator session is not draining its queue — ${verdict.since ?? "no timestamp"}`);
|
|
279
344
|
const delivered = await safeEscalate(d, {
|
|
280
345
|
tier: 2,
|
|
281
346
|
category: "confirmed-failure",
|
|
347
|
+
urgent: true,
|
|
282
348
|
project: d.project.name,
|
|
283
349
|
issue: NO_ISSUE,
|
|
284
350
|
// Keyed on the marker's own timestamp, not the date. The dedup ledger keys
|
|
285
351
|
// on this summary, and two wedges in one day is not a hypothetical — the
|
|
286
352
|
// failure mode is a session that gets stuck, gets restarted, and gets stuck
|
|
287
|
-
// again on the same cause an hour later.
|
|
288
|
-
//
|
|
353
|
+
// again on the same cause an hour later. The hourly suffix makes bounded
|
|
354
|
+
// reminders distinct without allowing every five-minute tick through.
|
|
289
355
|
summary:
|
|
290
356
|
`Orchestrator session wedged (${verdict.since ?? `marker at ${marker}`}) — ` +
|
|
291
|
-
`it has stopped reading its queue (${d.project.name})`,
|
|
357
|
+
`it has stopped reading its queue (${d.project.name})${repeatSuffix}`,
|
|
292
358
|
detail: [
|
|
293
359
|
verdict.since ?? "Marker present with no readable timestamp.",
|
|
360
|
+
`Unconsumed ticks: ${verdict.unconsumedTicks ?? "unknown"}`,
|
|
361
|
+
`Open green worker PRs: ${greenRuns.length}`,
|
|
362
|
+
...greenRuns.map((run) => `- ${run.prUrl ?? `#${run.issue} (PR URL unavailable)`}`),
|
|
294
363
|
`Marker: ${marker}`,
|
|
295
364
|
"",
|
|
296
365
|
"Its process and its herdr agent label are both healthy, which is why nothing else noticed:",
|
|
@@ -302,7 +371,7 @@ async function watchOrchestrator(d: Deps): Promise<void> {
|
|
|
302
371
|
"Dispatch is unaffected: workers keep running. What stops is drain, groom, report and merge.",
|
|
303
372
|
].join("\n"),
|
|
304
373
|
});
|
|
305
|
-
markPaged(d.stall, delivered);
|
|
374
|
+
markPaged(d.stall, delivered, now);
|
|
306
375
|
}
|
|
307
376
|
|
|
308
377
|
/**
|
|
@@ -319,10 +388,10 @@ export function isPaused(): boolean {
|
|
|
319
388
|
* The epoch-ms timestamp at which the current pause began, read from the same
|
|
320
389
|
* sentinel file {@link setPaused} writes (`<stateDir()>/paused`). Returns
|
|
321
390
|
* `undefined` when the fleet is not paused, or when the file's first line does
|
|
322
|
-
* not parse as a date
|
|
323
|
-
*
|
|
324
|
-
* innocent. {@link isPaused} is the authority on *whether*; this answers
|
|
325
|
-
*
|
|
391
|
+
* not parse as a date. A legacy/blank sentinel keeps completion mutations
|
|
392
|
+
* fail-closed because a run admitted before an *unknown* pause cannot be proven
|
|
393
|
+
* innocent. {@link isPaused} is the authority on *whether*; this answers *since
|
|
394
|
+
* when*.
|
|
326
395
|
*/
|
|
327
396
|
export function pausedAt(): number | undefined {
|
|
328
397
|
const f = join(stateDir(), "paused");
|
|
@@ -333,8 +402,8 @@ export function pausedAt(): number | undefined {
|
|
|
333
402
|
const t = Date.parse(first);
|
|
334
403
|
return Number.isNaN(t) ? undefined : t;
|
|
335
404
|
} catch {
|
|
336
|
-
// Unreadable sentinel (permissions, corruption): fail
|
|
337
|
-
//
|
|
405
|
+
// Unreadable sentinel (permissions, corruption): fail completion mutations
|
|
406
|
+
// closed like an unparseable line while the pause time is unprovable.
|
|
338
407
|
return undefined;
|
|
339
408
|
}
|
|
340
409
|
}
|
|
@@ -359,8 +428,8 @@ export function pauseProvenance(): { source: string; reason?: string } | undefin
|
|
|
359
428
|
const reason = match[2];
|
|
360
429
|
return { source, ...(reason === undefined ? {} : { reason }) };
|
|
361
430
|
} catch {
|
|
362
|
-
// Unreadable sentinel: no provenance to name
|
|
363
|
-
//
|
|
431
|
+
// Unreadable sentinel: no provenance to name. Completion mutations still
|
|
432
|
+
// fail closed because the pause time is unknown.
|
|
364
433
|
return undefined;
|
|
365
434
|
}
|
|
366
435
|
}
|
|
@@ -471,9 +540,17 @@ export function checkIntegrity(gate: IntegrityGate, current: Map<string, string>
|
|
|
471
540
|
return { diff, pause: true, page: !gate.paged };
|
|
472
541
|
}
|
|
473
542
|
|
|
474
|
-
/** Latch a
|
|
475
|
-
export function markPaged(gate:
|
|
476
|
-
|
|
543
|
+
/** Latch a page after delivery is confirmed, and never before. */
|
|
544
|
+
export function markPaged(gate: StallGate, delivered: boolean, now: number): void;
|
|
545
|
+
export function markPaged(gate: IntegrityGate, delivered: boolean): void;
|
|
546
|
+
export function markPaged(
|
|
547
|
+
gate: { paged: boolean; lastPagedAt?: number },
|
|
548
|
+
delivered: boolean,
|
|
549
|
+
now?: number,
|
|
550
|
+
): void {
|
|
551
|
+
if (!delivered) return;
|
|
552
|
+
gate.paged = true;
|
|
553
|
+
if (now !== undefined) gate.lastPagedAt = now;
|
|
477
554
|
}
|
|
478
555
|
|
|
479
556
|
// ---------------------------------------------------------------------- helpers
|
|
@@ -543,6 +620,32 @@ function swapLabel(store: Store, projectName: string, issue: number, from: strin
|
|
|
543
620
|
{ issue, op: "remove", label: from },
|
|
544
621
|
]);
|
|
545
622
|
}
|
|
623
|
+
/**
|
|
624
|
+
* Persist the operator-stop transition before releasing its live controller.
|
|
625
|
+
* The row is terminal first, then its in-progress label is removed through the
|
|
626
|
+
* same durable projection outbox as every other lifecycle transition.
|
|
627
|
+
*/
|
|
628
|
+
export function recordOperatorStop(
|
|
629
|
+
store: Pick<Store, "updateRun" | "enqueueLabelOps">,
|
|
630
|
+
args: {
|
|
631
|
+
project: string;
|
|
632
|
+
issue: number;
|
|
633
|
+
runId: string;
|
|
634
|
+
inProgress: string;
|
|
635
|
+
reason: string;
|
|
636
|
+
patch: Partial<RunRecord>;
|
|
637
|
+
},
|
|
638
|
+
): void {
|
|
639
|
+
store.updateRun(args.runId, {
|
|
640
|
+
...args.patch,
|
|
641
|
+
state: "stopped",
|
|
642
|
+
lastError: `operator stopped: ${args.reason}`,
|
|
643
|
+
});
|
|
644
|
+
store.enqueueLabelOps(args.project, [
|
|
645
|
+
{ issue: args.issue, op: "remove", label: args.inProgress },
|
|
646
|
+
]);
|
|
647
|
+
}
|
|
648
|
+
|
|
546
649
|
|
|
547
650
|
/**
|
|
548
651
|
* The escalator throws when no transport is configured or Telegram rejects, and
|
|
@@ -564,6 +667,38 @@ async function safeEscalate(d: Pick<Deps, "escalate">, e: Escalation): Promise<b
|
|
|
564
667
|
}
|
|
565
668
|
}
|
|
566
669
|
|
|
670
|
+
async function reactToProviderCredit(
|
|
671
|
+
d: Deps,
|
|
672
|
+
issue: number,
|
|
673
|
+
message: string,
|
|
674
|
+
sessionFile: string | undefined,
|
|
675
|
+
): Promise<void> {
|
|
676
|
+
const { project } = d;
|
|
677
|
+
const alreadyPaused = isPaused();
|
|
678
|
+
if (!alreadyPaused) setPaused(true, { source: "provider-credit", reason: message });
|
|
679
|
+
log(
|
|
680
|
+
`#${issue} provider refused for credit — dispatch ${alreadyPaused ? "remains paused" : "paused"}: ${message}`,
|
|
681
|
+
);
|
|
682
|
+
// Fleet-scoped and run-independent on purpose. The notification ledger
|
|
683
|
+
// dedupes on `project:issue:tier:summary`, so `NO_ISSUE` plus a summary
|
|
684
|
+
// carrying no run or attempt pages once for the fleet, not once per run.
|
|
685
|
+
await safeEscalate(d, {
|
|
686
|
+
tier: 2,
|
|
687
|
+
category: "fleet-stopped",
|
|
688
|
+
project: project.name,
|
|
689
|
+
issue: NO_ISSUE,
|
|
690
|
+
summary: `Model provider refused for credit — ${project.name} is paused`,
|
|
691
|
+
detail: [
|
|
692
|
+
message,
|
|
693
|
+
"",
|
|
694
|
+
"No implementation attempt was charged: this is a billing state, not a",
|
|
695
|
+
"failed implementation. Each affected issue keeps its queue label and",
|
|
696
|
+
"re-dispatches on `omp-conductor resume` once the provider has credit.",
|
|
697
|
+
`Session: ${sessionFile ?? "(no transcript)"}`,
|
|
698
|
+
].join("\n"),
|
|
699
|
+
});
|
|
700
|
+
}
|
|
701
|
+
|
|
567
702
|
/**
|
|
568
703
|
* What a salvage attempt contributes to the escalation: where the work went, or
|
|
569
704
|
* that it went nowhere. Split from the effects below for the same reason
|
|
@@ -840,11 +975,14 @@ export function createTurnLimitRegistry(
|
|
|
840
975
|
|
|
841
976
|
export type WorkerControlResult =
|
|
842
977
|
| { kind: "ok"; runId: string; phase: WorkerPausePhase }
|
|
978
|
+
| { kind: "stopped"; runId: string; reason: string }
|
|
843
979
|
| { kind: "refused"; runId: string; error: string }
|
|
844
980
|
| { kind: "not-active" };
|
|
845
981
|
|
|
846
982
|
export interface WorkerControlSlot {
|
|
847
983
|
install(control: WorkerPauseControl): void;
|
|
984
|
+
/** Stop accepted before the session controller exists. */
|
|
985
|
+
requestedStop(): string | undefined;
|
|
848
986
|
close(): void;
|
|
849
987
|
}
|
|
850
988
|
|
|
@@ -852,34 +990,51 @@ export interface WorkerControlRegistry {
|
|
|
852
990
|
open(project: string, issue: number, runId: string): WorkerControlSlot;
|
|
853
991
|
pause(project: string, issue: number): Promise<WorkerControlResult>;
|
|
854
992
|
resume(project: string, issue: number): WorkerControlResult;
|
|
993
|
+
stop(project: string, issue: number, reason: string): Promise<WorkerControlResult>;
|
|
855
994
|
/** Live runs whose phase is not `running` — what /healthz and the board show. */
|
|
856
995
|
snapshot(project: string): { issue: number; runId: string; phase: WorkerPausePhase }[];
|
|
857
996
|
}
|
|
858
997
|
|
|
859
998
|
/** Authoritative controls for sessions owned by this daemon process. */
|
|
860
999
|
export function createWorkerControlRegistry(): WorkerControlRegistry {
|
|
861
|
-
|
|
862
|
-
string
|
|
863
|
-
|
|
864
|
-
|
|
1000
|
+
interface Entry {
|
|
1001
|
+
project: string;
|
|
1002
|
+
issue: number;
|
|
1003
|
+
runId: string;
|
|
1004
|
+
control?: WorkerPauseControl;
|
|
1005
|
+
stopReason?: string;
|
|
1006
|
+
stopError?: string;
|
|
1007
|
+
finished: PromiseWithResolvers<void>;
|
|
1008
|
+
}
|
|
1009
|
+
|
|
1010
|
+
const active = new Map<string, Entry>();
|
|
865
1011
|
const key = (project: string, issue: number): string => `${project}\0${issue}`;
|
|
866
1012
|
return {
|
|
867
1013
|
open(project, issue, runId) {
|
|
868
1014
|
const k = key(project, issue);
|
|
869
1015
|
if (active.has(k)) throw new Error(`#${issue} already has a live worker controller`);
|
|
870
|
-
const entry = {
|
|
871
|
-
project
|
|
872
|
-
issue
|
|
873
|
-
runId
|
|
874
|
-
|
|
1016
|
+
const entry: Entry = {
|
|
1017
|
+
project,
|
|
1018
|
+
issue,
|
|
1019
|
+
runId,
|
|
1020
|
+
finished: Promise.withResolvers<void>(),
|
|
875
1021
|
};
|
|
876
1022
|
active.set(k, entry);
|
|
877
1023
|
return {
|
|
878
1024
|
install: (control) => {
|
|
879
|
-
if (active.get(k)
|
|
1025
|
+
if (active.get(k) !== entry) return;
|
|
1026
|
+
entry.control = control;
|
|
1027
|
+
if (entry.stopReason === undefined) return;
|
|
1028
|
+
try {
|
|
1029
|
+
control.stop(entry.stopReason);
|
|
1030
|
+
} catch (err) {
|
|
1031
|
+
entry.stopError = err instanceof Error ? err.message : String(err);
|
|
1032
|
+
}
|
|
880
1033
|
},
|
|
1034
|
+
requestedStop: () => active.get(k) === entry ? entry.stopReason : undefined,
|
|
881
1035
|
close: () => {
|
|
882
1036
|
if (active.get(k) === entry) active.delete(k);
|
|
1037
|
+
entry.finished.resolve();
|
|
883
1038
|
},
|
|
884
1039
|
};
|
|
885
1040
|
},
|
|
@@ -911,6 +1066,30 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
|
|
|
911
1066
|
};
|
|
912
1067
|
}
|
|
913
1068
|
},
|
|
1069
|
+
async stop(project, issue, reason) {
|
|
1070
|
+
const entry = active.get(key(project, issue));
|
|
1071
|
+
if (entry === undefined) return { kind: "not-active" };
|
|
1072
|
+
if (entry.stopReason === undefined) {
|
|
1073
|
+
entry.stopReason = reason;
|
|
1074
|
+
if (entry.control !== undefined) {
|
|
1075
|
+
try {
|
|
1076
|
+
entry.control.stop(reason);
|
|
1077
|
+
} catch (err) {
|
|
1078
|
+
entry.stopReason = undefined;
|
|
1079
|
+
return {
|
|
1080
|
+
kind: "refused",
|
|
1081
|
+
runId: entry.runId,
|
|
1082
|
+
error: err instanceof Error ? err.message : String(err),
|
|
1083
|
+
};
|
|
1084
|
+
}
|
|
1085
|
+
}
|
|
1086
|
+
}
|
|
1087
|
+
await entry.finished.promise;
|
|
1088
|
+
if (entry.stopError !== undefined) {
|
|
1089
|
+
return { kind: "refused", runId: entry.runId, error: entry.stopError };
|
|
1090
|
+
}
|
|
1091
|
+
return { kind: "stopped", runId: entry.runId, reason: entry.stopReason! };
|
|
1092
|
+
},
|
|
914
1093
|
snapshot(project) {
|
|
915
1094
|
const workers: { issue: number; runId: string; phase: WorkerPausePhase }[] = [];
|
|
916
1095
|
for (const entry of active.values()) {
|
|
@@ -1012,7 +1191,28 @@ export function shouldContinueAfterTurnsCap(f: {
|
|
|
1012
1191
|
return hasContinuationBudget(f.continuation, f.maxContinuations);
|
|
1013
1192
|
}
|
|
1014
1193
|
|
|
1015
|
-
|
|
1194
|
+
/** Bind a worker's run identity to every release-policy block it emits. */
|
|
1195
|
+
export function workerReleaseBlockRecorder(
|
|
1196
|
+
project: string,
|
|
1197
|
+
issue: number,
|
|
1198
|
+
runId: string,
|
|
1199
|
+
root = stateDir(),
|
|
1200
|
+
): (shape: ReleaseShape, context: ReleaseBlockContext) => void {
|
|
1201
|
+
return (shape, context) =>
|
|
1202
|
+
recordReleaseBlock(
|
|
1203
|
+
project,
|
|
1204
|
+
"worker",
|
|
1205
|
+
shape,
|
|
1206
|
+
{
|
|
1207
|
+
...context,
|
|
1208
|
+
issue,
|
|
1209
|
+
runId,
|
|
1210
|
+
},
|
|
1211
|
+
root,
|
|
1212
|
+
);
|
|
1213
|
+
}
|
|
1214
|
+
|
|
1215
|
+
export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
1016
1216
|
const { project, caps, tracker, store } = d;
|
|
1017
1217
|
const issue = r.issue.number;
|
|
1018
1218
|
const branch = branchName(r.issue);
|
|
@@ -1027,6 +1227,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1027
1227
|
let worktreePath: string | undefined;
|
|
1028
1228
|
let turnLimit: TurnLimitController | undefined;
|
|
1029
1229
|
let workerControl: WorkerControlSlot | undefined;
|
|
1230
|
+
let workerSessionInstalled = false;
|
|
1030
1231
|
// The run's own repository. Hoisted for the same reason `worktreePath` is —
|
|
1031
1232
|
// the catch and finally paths have to publish the branch.
|
|
1032
1233
|
let runRepo: RunRepoRef | undefined;
|
|
@@ -1048,6 +1249,53 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1048
1249
|
return pushRunBranch(project, runRepo);
|
|
1049
1250
|
};
|
|
1050
1251
|
|
|
1252
|
+
/**
|
|
1253
|
+
* A stop can arrive after the run is claimed but before `runWorker` exposes
|
|
1254
|
+
* its controller. Settle that run here instead of turning an operator action
|
|
1255
|
+
* into a dispatch failure or making the HTTP request wait for a session that
|
|
1256
|
+
* will never exist.
|
|
1257
|
+
*/
|
|
1258
|
+
const settleStopBeforeSession = async (): Promise<boolean> => {
|
|
1259
|
+
const reason = workerControl?.requestedStop();
|
|
1260
|
+
if (reason === undefined || run === undefined || workerSessionInstalled) return false;
|
|
1261
|
+
turnLimit?.close();
|
|
1262
|
+
turnLimit = undefined;
|
|
1263
|
+
const settlement =
|
|
1264
|
+
worktreePath === undefined
|
|
1265
|
+
? undefined
|
|
1266
|
+
: await settleWorktree({
|
|
1267
|
+
issue,
|
|
1268
|
+
attempt,
|
|
1269
|
+
ending: `stopped by the operator: ${reason}`,
|
|
1270
|
+
worktree: worktreePath,
|
|
1271
|
+
branch,
|
|
1272
|
+
publish,
|
|
1273
|
+
tree: "remove",
|
|
1274
|
+
mirrorPath,
|
|
1275
|
+
});
|
|
1276
|
+
recordOperatorStop(store, {
|
|
1277
|
+
project: project.name,
|
|
1278
|
+
issue,
|
|
1279
|
+
runId: run.id,
|
|
1280
|
+
inProgress,
|
|
1281
|
+
reason,
|
|
1282
|
+
patch: {
|
|
1283
|
+
endedAt: Date.now(),
|
|
1284
|
+
turns: run.turns,
|
|
1285
|
+
spendUsd: run.spendUsd,
|
|
1286
|
+
worktree: worktreePath ?? run.worktree,
|
|
1287
|
+
report: [
|
|
1288
|
+
"Operator stopped the run before its worker session started.",
|
|
1289
|
+
`Reason: ${reason}`,
|
|
1290
|
+
...(settlement?.lines ?? []),
|
|
1291
|
+
].join("\n"),
|
|
1292
|
+
...settlement?.patch,
|
|
1293
|
+
},
|
|
1294
|
+
});
|
|
1295
|
+
log(`#${issue} stopped by operator before its worker session started: ${reason}`);
|
|
1296
|
+
return true;
|
|
1297
|
+
};
|
|
1298
|
+
|
|
1051
1299
|
try {
|
|
1052
1300
|
// Claim on the STORE first, before anything that can fail. The run row —
|
|
1053
1301
|
// not the label — is the crash-safe guard against double dispatch: rows
|
|
@@ -1063,8 +1311,6 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1063
1311
|
// Read before this attempt's own row exists, so `latestRun` still means the
|
|
1064
1312
|
// attempt whose work this one inherits.
|
|
1065
1313
|
const priorSalvage = store.latestRun(project.name, issue)?.salvageSha;
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
1314
|
run = store.createRun({
|
|
1069
1315
|
project: project.name,
|
|
1070
1316
|
issue,
|
|
@@ -1078,11 +1324,20 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1078
1324
|
maxTurns: caps.workerMaxTurns,
|
|
1079
1325
|
startedAt: Date.now(),
|
|
1080
1326
|
});
|
|
1327
|
+
const maxTurns = run.maxTurns;
|
|
1328
|
+
const turnOverride = maxTurns > caps.workerMaxTurns ? maxTurns : undefined;
|
|
1081
1329
|
const runId = run.id;
|
|
1330
|
+
if (turnOverride !== undefined) {
|
|
1331
|
+
log(
|
|
1332
|
+
`#${issue} claimed with turn override ${turnOverride} ` +
|
|
1333
|
+
`(base ${caps.workerMaxTurns})`,
|
|
1334
|
+
);
|
|
1335
|
+
}
|
|
1082
1336
|
store.enqueueLabelOps(project.name, [{ issue, op: "add", label: inProgress }]);
|
|
1083
1337
|
claimed = true;
|
|
1084
|
-
turnLimit = d.turnLimits.open(project.name, issue, runId,
|
|
1338
|
+
turnLimit = d.turnLimits.open(project.name, issue, runId, maxTurns);
|
|
1085
1339
|
workerControl = d.workerControls.open(project.name, issue, runId);
|
|
1340
|
+
if (await settleStopBeforeSession()) return;
|
|
1086
1341
|
|
|
1087
1342
|
// A run's tree is <workspaceRoot>/<issue> and addRunRepo refuses to reuse
|
|
1088
1343
|
// an existing path, so a retry — or a tree kept from a failed attempt — has
|
|
@@ -1091,6 +1346,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1091
1346
|
// first attempt. addRunRepo does its own ensureMirror; calling it here too
|
|
1092
1347
|
// would cost a second network fetch per attempt.
|
|
1093
1348
|
await removeWorktree(mirrorPath, worktreePathFor(project.workspaceRoot, issue));
|
|
1349
|
+
if (await settleStopBeforeSession()) return;
|
|
1094
1350
|
const provisioned = await addRunRepo(
|
|
1095
1351
|
r.repo,
|
|
1096
1352
|
project.mirrorRoot,
|
|
@@ -1100,6 +1356,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1100
1356
|
);
|
|
1101
1357
|
worktreePath = provisioned.path;
|
|
1102
1358
|
runRepo = { repo: r.repo, runRepoPath: worktreePath, branch };
|
|
1359
|
+
if (await settleStopBeforeSession()) return;
|
|
1103
1360
|
|
|
1104
1361
|
// The SDK names the transcript itself, so the daemon supplies the parent
|
|
1105
1362
|
// directory and learns the real path back from the result. Inventing one
|
|
@@ -1138,6 +1395,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1138
1395
|
},
|
|
1139
1396
|
{ ...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }) },
|
|
1140
1397
|
);
|
|
1398
|
+
if (await settleStopBeforeSession()) return;
|
|
1141
1399
|
|
|
1142
1400
|
store.updateRun(runId, { worktree: worktreePath, state: "running" });
|
|
1143
1401
|
|
|
@@ -1146,20 +1404,29 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1146
1404
|
(provisioned.reattached ? " (continuation: reattached existing branch)" : ""),
|
|
1147
1405
|
);
|
|
1148
1406
|
|
|
1407
|
+
const brief = await buildBrief(project, r, branch, worktreePath, {
|
|
1408
|
+
continuation: provisioned.reattached,
|
|
1409
|
+
defaultBranch: r.repo.defaultBranch,
|
|
1410
|
+
...(provisioned.reattached && priorSalvage !== undefined
|
|
1411
|
+
? { salvagedSha: priorSalvage }
|
|
1412
|
+
: {}),
|
|
1413
|
+
});
|
|
1414
|
+
if (await settleStopBeforeSession()) return;
|
|
1415
|
+
|
|
1416
|
+
const repoSlug = githubRepo(r.repo.cloneUrl);
|
|
1417
|
+
|
|
1149
1418
|
let result: WorkerResult;
|
|
1150
1419
|
try {
|
|
1151
1420
|
result = await runWorker({
|
|
1152
|
-
brief
|
|
1153
|
-
continuation: provisioned.reattached,
|
|
1154
|
-
defaultBranch: r.repo.defaultBranch,
|
|
1155
|
-
...(provisioned.reattached && priorSalvage !== undefined
|
|
1156
|
-
? { salvagedSha: priorSalvage }
|
|
1157
|
-
: {}),
|
|
1158
|
-
}),
|
|
1421
|
+
brief,
|
|
1159
1422
|
cwd: worktreePath,
|
|
1160
1423
|
caps,
|
|
1161
|
-
|
|
1162
|
-
|
|
1424
|
+
...(repoSlug === undefined ? {} : { repoSlug }),
|
|
1425
|
+
maxTurns: () => turnLimit?.maxTurns() ?? maxTurns,
|
|
1426
|
+
onPauseControl: (control) => {
|
|
1427
|
+
workerSessionInstalled = true;
|
|
1428
|
+
workerControl?.install(control);
|
|
1429
|
+
},
|
|
1163
1430
|
sessionDir,
|
|
1164
1431
|
// The session's control socket, under the daemon's own state directory —
|
|
1165
1432
|
// a child process of the daemon reaches it directly.
|
|
@@ -1178,28 +1445,24 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1178
1445
|
},
|
|
1179
1446
|
...(project.workerModel === undefined ? {} : { model: project.workerModel }),
|
|
1180
1447
|
releaseGrants: resolveReleaseGrants(project),
|
|
1181
|
-
onReleaseBlocked: (
|
|
1448
|
+
onReleaseBlocked: workerReleaseBlockRecorder(project.name, issue, runId),
|
|
1182
1449
|
onTurn: (n) => store.updateRun(runId, { turns: n }),
|
|
1183
1450
|
onSpend: (usd) => store.updateRun(runId, { spendUsd: usd }),
|
|
1184
1451
|
onKilled: () => {
|
|
1185
1452
|
turnLimit?.close();
|
|
1186
1453
|
turnLimit = undefined;
|
|
1187
|
-
workerControl?.close();
|
|
1188
|
-
workerControl = undefined;
|
|
1189
1454
|
},
|
|
1190
1455
|
// Recorded the moment the session opens its transcript, not when the run
|
|
1191
1456
|
// ends: `omp-conductor tail` resolves an issue to a file through this row,
|
|
1192
1457
|
// and a path written at completion is a path nobody can follow live. The
|
|
1193
1458
|
// completion-time update below writes the same value again, harmlessly.
|
|
1194
1459
|
onSessionFile: (f) => store.updateRun(runId, { sessionFile: f }),
|
|
1195
|
-
});
|
|
1460
|
+
}, d.workerDeps);
|
|
1196
1461
|
} finally {
|
|
1197
1462
|
// This is the authoritative settlement edge for `extend`: close before
|
|
1198
1463
|
// PR verification or terminal row writes can leave stale `running` state.
|
|
1199
1464
|
turnLimit?.close();
|
|
1200
1465
|
turnLimit = undefined;
|
|
1201
|
-
workerControl?.close();
|
|
1202
|
-
workerControl = undefined;
|
|
1203
1466
|
}
|
|
1204
1467
|
|
|
1205
1468
|
// A configured model the harness could not honour means this run was done by
|
|
@@ -1264,8 +1527,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1264
1527
|
// `pushed-*` run is the only end that does not salvage: its deliverable is
|
|
1265
1528
|
// already on a remote branch, whatever is left loose in the tree is by the
|
|
1266
1529
|
// worker's own account not part of it, and appending a WIP commit would
|
|
1267
|
-
// turn the green PR this daemon just verified red. Every other end
|
|
1268
|
-
//
|
|
1530
|
+
// turn the green PR this daemon just verified red. Every other end may
|
|
1531
|
+
// contain work, so it is salvaged before the tree's final fate is decided.
|
|
1269
1532
|
const settlement =
|
|
1270
1533
|
state === "pushed-green" || state === "pushed-pending" || state === "merged"
|
|
1271
1534
|
? undefined
|
|
@@ -1273,7 +1536,11 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1273
1536
|
issue,
|
|
1274
1537
|
attempt,
|
|
1275
1538
|
ending:
|
|
1276
|
-
state === "blocked"
|
|
1539
|
+
state === "blocked"
|
|
1540
|
+
? "blocked for an operator decision"
|
|
1541
|
+
: state === "stopped"
|
|
1542
|
+
? `stopped by the operator: ${result.stoppedReason ?? "no reason recorded"}`
|
|
1543
|
+
: endedBy(result.killedBy),
|
|
1277
1544
|
worktree: worktreePath,
|
|
1278
1545
|
branch,
|
|
1279
1546
|
publish,
|
|
@@ -1294,8 +1561,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1294
1561
|
await removeWorktree(mirrorPath, worktreePath);
|
|
1295
1562
|
}
|
|
1296
1563
|
|
|
1297
|
-
|
|
1298
|
-
state,
|
|
1564
|
+
const terminalPatch: Partial<RunRecord> = {
|
|
1299
1565
|
endedAt: Date.now(),
|
|
1300
1566
|
turns: result.turns,
|
|
1301
1567
|
spendUsd: result.spendUsd,
|
|
@@ -1303,23 +1569,41 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1303
1569
|
headSha: result.headSha,
|
|
1304
1570
|
sessionFile: result.sessionFile,
|
|
1305
1571
|
// Every terminal state persists the worker's report, not just a green
|
|
1306
|
-
// push:
|
|
1307
|
-
// continuation must pool its disclosures from (#199).
|
|
1572
|
+
// push: a stopped attempt's partial report is still part of its audit trail.
|
|
1308
1573
|
report: result.report,
|
|
1309
|
-
...(providerCredit !== undefined || providerTransient !== undefined
|
|
1310
|
-
? { lastError: providerCredit ?? providerTransient }
|
|
1311
|
-
: verified.reason === undefined
|
|
1312
|
-
? {}
|
|
1313
|
-
: { lastError: verified.reason }),
|
|
1314
1574
|
...settlement?.patch,
|
|
1315
1575
|
...(audit === undefined || audit.flags.length === 0
|
|
1316
1576
|
? {}
|
|
1317
1577
|
: { settlementFlags: audit.flags }),
|
|
1318
|
-
}
|
|
1578
|
+
};
|
|
1579
|
+
if (state === "stopped") {
|
|
1580
|
+
recordOperatorStop(store, {
|
|
1581
|
+
project: project.name,
|
|
1582
|
+
issue,
|
|
1583
|
+
runId,
|
|
1584
|
+
inProgress,
|
|
1585
|
+
reason: result.stoppedReason ?? "no reason recorded",
|
|
1586
|
+
patch: terminalPatch,
|
|
1587
|
+
});
|
|
1588
|
+
} else {
|
|
1589
|
+
const lastError = completionLastError(
|
|
1590
|
+
providerCredit,
|
|
1591
|
+
providerTransient,
|
|
1592
|
+
verified.reason,
|
|
1593
|
+
sessionErr,
|
|
1594
|
+
);
|
|
1595
|
+
store.updateRun(runId, {
|
|
1596
|
+
...terminalPatch,
|
|
1597
|
+
state,
|
|
1598
|
+
...(lastError === undefined ? {} : { lastError }),
|
|
1599
|
+
});
|
|
1600
|
+
}
|
|
1319
1601
|
|
|
1320
1602
|
const salvaged = settlement?.lines ?? [];
|
|
1321
1603
|
|
|
1322
|
-
if (state === "
|
|
1604
|
+
if (state === "stopped") {
|
|
1605
|
+
log(`#${issue} stopped by operator on attempt ${attempt}: ${result.stoppedReason}`);
|
|
1606
|
+
} else if (state === "blocked") {
|
|
1323
1607
|
swapLabel(store, project.name, issue, inProgress, project.stateLabels.blocked);
|
|
1324
1608
|
await safeEscalate(d, {
|
|
1325
1609
|
tier: 1,
|
|
@@ -1343,31 +1627,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1343
1627
|
});
|
|
1344
1628
|
|
|
1345
1629
|
if (providerCredit !== undefined) {
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
// the first 402 and a human noticing (#220).
|
|
1349
|
-
setPaused(true, { source: "provider-credit", reason: providerCredit });
|
|
1350
|
-
log(`#${issue} provider refused for credit — dispatch paused: ${providerCredit}`);
|
|
1351
|
-
swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
|
|
1352
|
-
// Fleet-scoped and run-independent on purpose. The notification ledger
|
|
1353
|
-
// dedupes on `project:issue:tier:summary`, so `NO_ISSUE` plus a summary
|
|
1354
|
-
// carrying no run or attempt is what makes this page once for the fleet
|
|
1355
|
-
// instead of once per affected run.
|
|
1356
|
-
await safeEscalate(d, {
|
|
1357
|
-
tier: 2,
|
|
1358
|
-
category: "fleet-stopped",
|
|
1359
|
-
project: project.name,
|
|
1360
|
-
issue: NO_ISSUE,
|
|
1361
|
-
summary: `Model provider refused for credit — ${project.name} is paused`,
|
|
1362
|
-
detail: [
|
|
1363
|
-
providerCredit,
|
|
1364
|
-
"",
|
|
1365
|
-
"No implementation attempt was charged: this is a billing state, not a",
|
|
1366
|
-
"failed implementation. Each affected issue keeps its queue label and",
|
|
1367
|
-
"re-dispatches on `omp-conductor resume` once the provider has credit.",
|
|
1368
|
-
`Session: ${result.sessionFile ?? "(no transcript)"}`,
|
|
1369
|
-
].join("\n"),
|
|
1370
|
-
});
|
|
1630
|
+
await reactToProviderCredit(d, issue, providerCredit, result.sessionFile);
|
|
1631
|
+
swapToQueue(d, issue, inProgress);
|
|
1371
1632
|
} else if (continueTurns) {
|
|
1372
1633
|
// Requeue as one ordered pair: the in-progress removal before the
|
|
1373
1634
|
// queue add, exactly the order the projector will apply them in (#201).
|
|
@@ -1469,8 +1730,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1469
1730
|
// inner settlement guard exists. Latch it before any terminal write or await.
|
|
1470
1731
|
turnLimit?.close();
|
|
1471
1732
|
turnLimit = undefined;
|
|
1472
|
-
|
|
1473
|
-
workerControl = undefined;
|
|
1733
|
+
if (await settleStopBeforeSession()) return;
|
|
1474
1734
|
const detail = errText(err);
|
|
1475
1735
|
log(`#${issue} errored: ${detail}`);
|
|
1476
1736
|
// A crash lands anywhere, including mid-edit in a tree holding the only
|
|
@@ -1516,8 +1776,6 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1516
1776
|
// a failure path, and whatever it still held is now a commit on the branch.
|
|
1517
1777
|
} finally {
|
|
1518
1778
|
turnLimit?.close();
|
|
1519
|
-
workerControl?.close();
|
|
1520
|
-
workerControl = undefined;
|
|
1521
1779
|
// The run is over, so its channel is too. Closed here rather than beside
|
|
1522
1780
|
// the session so the crash path closes it as well: a listener left bound
|
|
1523
1781
|
// after its run settled is a socket whose `run-not-live` check is the only
|
|
@@ -1530,6 +1788,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1530
1788
|
}
|
|
1531
1789
|
verbListener = undefined;
|
|
1532
1790
|
}
|
|
1791
|
+
workerControl?.close();
|
|
1792
|
+
workerControl = undefined;
|
|
1533
1793
|
}
|
|
1534
1794
|
}
|
|
1535
1795
|
|
|
@@ -1558,12 +1818,12 @@ export interface Settlement {
|
|
|
1558
1818
|
*
|
|
1559
1819
|
* - `merged` — the work landed. That is what `merged` was reserved for.
|
|
1560
1820
|
* - `closed` — a human read the work and said no. Leaving it `pushed-green`
|
|
1561
|
-
*
|
|
1562
|
-
*
|
|
1563
|
-
*
|
|
1564
|
-
*
|
|
1565
|
-
*
|
|
1566
|
-
*
|
|
1821
|
+
* forever is a lie; `failed` records that it did not land and releases the
|
|
1822
|
+
* busy guard, so an issue a human re-queues can be attempted again. A row
|
|
1823
|
+
* that had reached `pushed-green` or `pushed-pending` is classified
|
|
1824
|
+
* `returned-for-revision` at settlement. A review decision asks for another
|
|
1825
|
+
* implementation pass, not a failure, so it consumes the continuation budget
|
|
1826
|
+
* instead of the failed-attempt budget.
|
|
1567
1827
|
* - `open`, and undefined — nothing changes. Undefined is "could not tell": a
|
|
1568
1828
|
* flaky network, a revoked token, a deleted PR. Settling on it would record a
|
|
1569
1829
|
* merge that never happened, and the next tick asks again for free. An
|
|
@@ -1647,6 +1907,242 @@ export function releaseInProgress(
|
|
|
1647
1907
|
* in the busy set throughout, so no second worker can be sent at the issue while
|
|
1648
1908
|
* it waits.
|
|
1649
1909
|
*/
|
|
1910
|
+
const BASE_CHECK_BATCH = 20;
|
|
1911
|
+
const BASE_CHECK_WINDOW_MS = 24 * 60 * 60 * 1_000;
|
|
1912
|
+
const BASE_STATUS_WINDOW_MS = 7 * 24 * 60 * 60 * 1_000;
|
|
1913
|
+
|
|
1914
|
+
const SUCCESSFUL_WORKFLOW_CONCLUSIONS = new Set(["success", "neutral", "skipped"]);
|
|
1915
|
+
|
|
1916
|
+
const FAILING_WORKFLOW_CONCLUSIONS = new Set([
|
|
1917
|
+
"failure",
|
|
1918
|
+
"cancelled",
|
|
1919
|
+
"timed_out",
|
|
1920
|
+
"action_required",
|
|
1921
|
+
"startup_failure",
|
|
1922
|
+
"stale",
|
|
1923
|
+
]);
|
|
1924
|
+
|
|
1925
|
+
function appendSettlementFlag(run: RunRecord, flag: SettlementFlag): SettlementFlag[] {
|
|
1926
|
+
const flags = run.settlementFlags ?? [];
|
|
1927
|
+
return flags.some((existing) => existing.kind === flag.kind && existing.detail === flag.detail)
|
|
1928
|
+
? flags
|
|
1929
|
+
: [...flags, flag];
|
|
1930
|
+
}
|
|
1931
|
+
|
|
1932
|
+
/**
|
|
1933
|
+
* Observe Actions on exact merge commits for up to one day. A running workflow
|
|
1934
|
+
* stays quiet and pending; a failure becomes durable evidence on the merged row
|
|
1935
|
+
* and pages exactly once because the row leaves `pending` before delivery.
|
|
1936
|
+
*/
|
|
1937
|
+
export async function watchMergedBase(d: Pick<Deps, "project" | "tracker" | "store" | "escalate">): Promise<void> {
|
|
1938
|
+
const now = Date.now();
|
|
1939
|
+
for (const run of d.store.runsNeedingBaseCheck(d.project.name, BASE_CHECK_BATCH)) {
|
|
1940
|
+
if (
|
|
1941
|
+
run.endedAt === undefined ||
|
|
1942
|
+
now - run.endedAt > BASE_CHECK_WINDOW_MS ||
|
|
1943
|
+
run.mergeSha === undefined ||
|
|
1944
|
+
run.baseRef === undefined
|
|
1945
|
+
) {
|
|
1946
|
+
d.store.updateRun(run.id, { baseCheck: "unknown", baseCheckAt: now });
|
|
1947
|
+
log(`#${run.issue} base check unknown: merge identity is absent or older than 24h`);
|
|
1948
|
+
continue;
|
|
1949
|
+
}
|
|
1950
|
+
|
|
1951
|
+
const repo = d.project.routing.repos[run.repo];
|
|
1952
|
+
const repoIdentity = repo === undefined ? undefined : githubRepo(repo.cloneUrl);
|
|
1953
|
+
if (repoIdentity === undefined) {
|
|
1954
|
+
d.store.updateRun(run.id, { baseCheck: "unknown", baseCheckAt: now });
|
|
1955
|
+
log(`#${run.issue} base check unknown: routed repository ${run.repo} has no GitHub identity`);
|
|
1956
|
+
continue;
|
|
1957
|
+
}
|
|
1958
|
+
|
|
1959
|
+
let workflows;
|
|
1960
|
+
try {
|
|
1961
|
+
workflows = await d.tracker.workflowRunsAt(repoIdentity, run.mergeSha, {
|
|
1962
|
+
event: "push",
|
|
1963
|
+
branch: run.baseRef,
|
|
1964
|
+
});
|
|
1965
|
+
} catch (err) {
|
|
1966
|
+
log(`#${run.issue} base check unavailable (${errText(err)}) — retrying next tick`);
|
|
1967
|
+
continue;
|
|
1968
|
+
}
|
|
1969
|
+
if (workflows === undefined) {
|
|
1970
|
+
log(`#${run.issue} base check unavailable for ${run.mergeSha} — retrying next tick`);
|
|
1971
|
+
continue;
|
|
1972
|
+
}
|
|
1973
|
+
if (workflows.length === 0) {
|
|
1974
|
+
log(
|
|
1975
|
+
`#${run.issue} base check: no push-triggered run yet for ${run.mergeSha} — retrying next tick`,
|
|
1976
|
+
);
|
|
1977
|
+
continue;
|
|
1978
|
+
}
|
|
1979
|
+
if (workflows.some((workflow) => workflow.status !== "completed")) continue;
|
|
1980
|
+
|
|
1981
|
+
const failed = workflows.find(
|
|
1982
|
+
(workflow) =>
|
|
1983
|
+
workflow.conclusion !== undefined &&
|
|
1984
|
+
FAILING_WORKFLOW_CONCLUSIONS.has(workflow.conclusion),
|
|
1985
|
+
);
|
|
1986
|
+
if (failed === undefined) {
|
|
1987
|
+
const unknown = workflows.find(
|
|
1988
|
+
(workflow) =>
|
|
1989
|
+
workflow.conclusion === undefined ||
|
|
1990
|
+
!SUCCESSFUL_WORKFLOW_CONCLUSIONS.has(workflow.conclusion),
|
|
1991
|
+
);
|
|
1992
|
+
if (unknown !== undefined) {
|
|
1993
|
+
log(
|
|
1994
|
+
`#${run.issue} base check has unrecognised completed conclusion ` +
|
|
1995
|
+
`${JSON.stringify(unknown.conclusion)} for ${unknown.name} — retrying next tick`,
|
|
1996
|
+
);
|
|
1997
|
+
continue;
|
|
1998
|
+
}
|
|
1999
|
+
d.store.updateRun(run.id, { baseCheck: "green", baseCheckAt: now });
|
|
2000
|
+
log(`#${run.issue} base ${run.baseRef} green at ${run.mergeSha}`);
|
|
2001
|
+
continue;
|
|
2002
|
+
}
|
|
2003
|
+
|
|
2004
|
+
let previous;
|
|
2005
|
+
try {
|
|
2006
|
+
previous = await d.tracker.previousWorkflowRun(
|
|
2007
|
+
repoIdentity,
|
|
2008
|
+
failed.workflowId,
|
|
2009
|
+
run.baseRef,
|
|
2010
|
+
failed.createdAt,
|
|
2011
|
+
);
|
|
2012
|
+
} catch (err) {
|
|
2013
|
+
log(`#${run.issue} previous ${failed.name} run unavailable (${errText(err)}) — retrying next tick`);
|
|
2014
|
+
continue;
|
|
2015
|
+
}
|
|
2016
|
+
if (previous === undefined || (previous !== null && previous.status !== "completed")) continue;
|
|
2017
|
+
const preexisting =
|
|
2018
|
+
previous !== null &&
|
|
2019
|
+
previous.conclusion !== undefined &&
|
|
2020
|
+
FAILING_WORKFLOW_CONCLUSIONS.has(previous.conclusion);
|
|
2021
|
+
const detail =
|
|
2022
|
+
`${failed.name} failed at ${run.mergeSha} — ${failed.url}` +
|
|
2023
|
+
(preexisting ? " (already red before this merge)" : "");
|
|
2024
|
+
const flag: SettlementFlag = { kind: "base-branch-red", file: "(base branch)", detail };
|
|
2025
|
+
const delivered = await safeEscalate(d, {
|
|
2026
|
+
tier: 1,
|
|
2027
|
+
project: d.project.name,
|
|
2028
|
+
issue: run.issue,
|
|
2029
|
+
runId: run.id,
|
|
2030
|
+
summary: `Base branch ${run.baseRef} is red after merge`,
|
|
2031
|
+
detail,
|
|
2032
|
+
});
|
|
2033
|
+
if (!delivered) continue;
|
|
2034
|
+
d.store.updateRun(run.id, {
|
|
2035
|
+
baseCheck: preexisting ? "red-preexisting" : "red",
|
|
2036
|
+
baseCheckAt: now,
|
|
2037
|
+
settlementFlags: appendSettlementFlag(run, flag),
|
|
2038
|
+
});
|
|
2039
|
+
}
|
|
2040
|
+
}
|
|
2041
|
+
|
|
2042
|
+
/**
|
|
2043
|
+
* Refresh current base-branch health at each recently merged repository's live
|
|
2044
|
+
* head. This is status and release-gate evidence only; the per-merge audit
|
|
2045
|
+
* above remains the sole path that attributes and escalates a regression.
|
|
2046
|
+
*/
|
|
2047
|
+
export async function watchBaseHealth(
|
|
2048
|
+
d: Pick<Deps, "project" | "tracker" | "store">,
|
|
2049
|
+
): Promise<void> {
|
|
2050
|
+
const now = Date.now();
|
|
2051
|
+
const previousByRepo = new Map(
|
|
2052
|
+
d.store.baseHealth(d.project.name).map((row) => [row.repo, row] as const),
|
|
2053
|
+
);
|
|
2054
|
+
for (const { repo, baseRef } of d.store.mergedRepoBranches(
|
|
2055
|
+
d.project.name,
|
|
2056
|
+
now - BASE_STATUS_WINDOW_MS,
|
|
2057
|
+
)) {
|
|
2058
|
+
const target = d.project.routing.repos[repo];
|
|
2059
|
+
if (target === undefined) {
|
|
2060
|
+
log(`base health skipped: routed repository ${repo} is no longer configured`);
|
|
2061
|
+
continue;
|
|
2062
|
+
}
|
|
2063
|
+
const identity = githubRepo(target.cloneUrl);
|
|
2064
|
+
if (identity === undefined) {
|
|
2065
|
+
log(`base health skipped: routed repository ${repo} has no GitHub identity`);
|
|
2066
|
+
continue;
|
|
2067
|
+
}
|
|
2068
|
+
const branch = baseRef ?? target.defaultBranch;
|
|
2069
|
+
|
|
2070
|
+
let head: string | undefined;
|
|
2071
|
+
try {
|
|
2072
|
+
head = await d.tracker.branchHead(identity, branch);
|
|
2073
|
+
} catch (err) {
|
|
2074
|
+
log(`base ${repo}/${branch} head unavailable (${errText(err)}) — keeping previous health`);
|
|
2075
|
+
continue;
|
|
2076
|
+
}
|
|
2077
|
+
if (head === undefined) {
|
|
2078
|
+
log(`base ${repo}/${branch} head unavailable — keeping previous health`);
|
|
2079
|
+
continue;
|
|
2080
|
+
}
|
|
2081
|
+
|
|
2082
|
+
const previous = previousByRepo.get(repo);
|
|
2083
|
+
if (
|
|
2084
|
+
previous?.branch === branch &&
|
|
2085
|
+
previous.headSha === head &&
|
|
2086
|
+
(previous.verdict === "green" || previous.verdict === "red")
|
|
2087
|
+
) {
|
|
2088
|
+
continue;
|
|
2089
|
+
}
|
|
2090
|
+
|
|
2091
|
+
let runs;
|
|
2092
|
+
try {
|
|
2093
|
+
runs = await d.tracker.workflowRunsAt(identity, head, { event: "push", branch });
|
|
2094
|
+
} catch (err) {
|
|
2095
|
+
log(`base ${repo}/${branch} workflows unavailable (${errText(err)}) — keeping previous health`);
|
|
2096
|
+
continue;
|
|
2097
|
+
}
|
|
2098
|
+
if (runs === undefined) {
|
|
2099
|
+
log(`base ${repo}/${branch} workflows unavailable — keeping previous health`);
|
|
2100
|
+
continue;
|
|
2101
|
+
}
|
|
2102
|
+
|
|
2103
|
+
let verdict: BaseHealth["verdict"];
|
|
2104
|
+
let detail: string | undefined;
|
|
2105
|
+
if (runs.length === 0) {
|
|
2106
|
+
verdict = "unknown";
|
|
2107
|
+
detail = `no push-triggered workflow run for ${head.slice(0, 8)}`;
|
|
2108
|
+
} else if (runs.some((run) => run.status !== "completed")) {
|
|
2109
|
+
verdict = "pending";
|
|
2110
|
+
} else {
|
|
2111
|
+
const failed = runs.find(
|
|
2112
|
+
(run) =>
|
|
2113
|
+
run.conclusion !== undefined &&
|
|
2114
|
+
FAILING_WORKFLOW_CONCLUSIONS.has(run.conclusion),
|
|
2115
|
+
);
|
|
2116
|
+
if (failed !== undefined) {
|
|
2117
|
+
verdict = "red";
|
|
2118
|
+
detail = `${failed.name} failed at ${head.slice(0, 8)} — ${failed.url}`;
|
|
2119
|
+
} else if (
|
|
2120
|
+
runs.some(
|
|
2121
|
+
(run) =>
|
|
2122
|
+
run.conclusion === undefined ||
|
|
2123
|
+
!SUCCESSFUL_WORKFLOW_CONCLUSIONS.has(run.conclusion),
|
|
2124
|
+
)
|
|
2125
|
+
) {
|
|
2126
|
+
verdict = "pending";
|
|
2127
|
+
} else {
|
|
2128
|
+
verdict = "green";
|
|
2129
|
+
}
|
|
2130
|
+
}
|
|
2131
|
+
|
|
2132
|
+
const health: BaseHealth = {
|
|
2133
|
+
repo,
|
|
2134
|
+
branch,
|
|
2135
|
+
headSha: head,
|
|
2136
|
+
verdict,
|
|
2137
|
+
runsCount: runs.length,
|
|
2138
|
+
checkedAt: now,
|
|
2139
|
+
...(detail === undefined ? {} : { detail }),
|
|
2140
|
+
};
|
|
2141
|
+
d.store.upsertBaseHealth(d.project.name, health);
|
|
2142
|
+
previousByRepo.set(repo, health);
|
|
2143
|
+
}
|
|
2144
|
+
}
|
|
2145
|
+
|
|
1650
2146
|
export async function settlePushedGreen(
|
|
1651
2147
|
d: Pick<Deps, "project" | "tracker" | "store">,
|
|
1652
2148
|
): Promise<void> {
|
|
@@ -1678,17 +2174,44 @@ export async function settlePushedGreen(
|
|
|
1678
2174
|
|
|
1679
2175
|
const settlement = settlementFor(pr, run.prUrl);
|
|
1680
2176
|
if (settlement !== undefined) {
|
|
2177
|
+
// A mediated merge enters a second, bounded observation phase. Record the
|
|
2178
|
+
// exact merge commit before the row leaves the active set; if GitHub cannot
|
|
2179
|
+
// supply it yet, retry this settlement next tick rather than create a
|
|
2180
|
+
// merged row whose base result can never be attributed.
|
|
2181
|
+
let merged: MergedPrInfo | undefined;
|
|
2182
|
+
if (settlement.state === "merged") {
|
|
2183
|
+
try {
|
|
2184
|
+
merged = await tracker.mergedPrInfo(run.prUrl);
|
|
2185
|
+
} catch (err) {
|
|
2186
|
+
log(`#${run.issue} not settled: merge identity lookup failed (${errText(err)}) — retrying next tick`);
|
|
2187
|
+
continue;
|
|
2188
|
+
}
|
|
2189
|
+
if (merged === undefined) {
|
|
2190
|
+
log(`#${run.issue} not settled: merge identity unavailable — retrying next tick`);
|
|
2191
|
+
continue;
|
|
2192
|
+
}
|
|
2193
|
+
}
|
|
2194
|
+
|
|
1681
2195
|
// The label removal and the terminal row are one fact again (#201): the
|
|
1682
2196
|
// release is enqueued — a durable local write that cannot fail on the
|
|
1683
|
-
// tracker — in the same breath as the row is terminalised
|
|
1684
|
-
// no window in which a row beyond every later tick still owes its label.
|
|
1685
|
-
// The projector applies it with retry; while pending, the eligibility
|
|
1686
|
-
// overlay treats the label as already gone, so #18's
|
|
1687
|
-
// permanent-`agent:in-progress` cannot re-form even when GitHub refuses
|
|
1688
|
-
// the write.
|
|
2197
|
+
// tracker — in the same breath as the row is terminalised.
|
|
1689
2198
|
releaseInProgress(d, run.issue, settlement.reason);
|
|
1690
|
-
const patch: Partial<RunRecord> = {
|
|
1691
|
-
|
|
2199
|
+
const patch: Partial<RunRecord> = {
|
|
2200
|
+
state: settlement.state,
|
|
2201
|
+
endedAt: Date.now(),
|
|
2202
|
+
...(merged === undefined
|
|
2203
|
+
? {}
|
|
2204
|
+
: {
|
|
2205
|
+
mergeSha: merged.mergeSha,
|
|
2206
|
+
baseRef: merged.baseRef,
|
|
2207
|
+
baseCheck: "pending",
|
|
2208
|
+
}),
|
|
2209
|
+
};
|
|
2210
|
+
if (settlement.state === "failed") {
|
|
2211
|
+
patch.lastError = settlement.reason;
|
|
2212
|
+
patch.failureClass = "returned-for-revision";
|
|
2213
|
+
patch.recoveryAction = "none";
|
|
2214
|
+
}
|
|
1692
2215
|
store.updateRun(run.id, patch);
|
|
1693
2216
|
log(`#${run.issue} settled: ${settlement.reason}`);
|
|
1694
2217
|
continue;
|
|
@@ -1720,6 +2243,121 @@ export async function settlePushedGreen(
|
|
|
1720
2243
|
}
|
|
1721
2244
|
}
|
|
1722
2245
|
|
|
2246
|
+
const ADOPTABLE_PR_STATES: Partial<Record<RunState, true>> = {
|
|
2247
|
+
failed: true,
|
|
2248
|
+
killed: true,
|
|
2249
|
+
orphaned: true,
|
|
2250
|
+
blocked: true,
|
|
2251
|
+
};
|
|
2252
|
+
|
|
2253
|
+
/**
|
|
2254
|
+
* Reattaches a recovered PR to the terminal run that owns it (#245).
|
|
2255
|
+
*
|
|
2256
|
+
* A worker can fail before its completion report records `prUrl`, then have its
|
|
2257
|
+
* dirty tree committed and pushed by salvage. If that branch already has a PR,
|
|
2258
|
+
* the orchestrator otherwise has no policy-compliant path to inspect or merge
|
|
2259
|
+
* it: ownership is store-backed. Adoption is deliberately stricter than
|
|
2260
|
+
* admission. The tracker query proves the PR closes this run's issue; exact
|
|
2261
|
+
* branch and canonical repository matches prove it is this run's recovered
|
|
2262
|
+
* work, not an unrelated closer. Missing identity is refusal, never a guess.
|
|
2263
|
+
*
|
|
2264
|
+
* The newest run per issue is inspected, at most ten per tick and only inside
|
|
2265
|
+
* the same 30-day window as mediated PR verbs. The cursor advances through the
|
|
2266
|
+
* full eligible set so persistent non-matches cannot starve older recovered
|
|
2267
|
+
* work. Successful adoption is idempotent because the row gains `prUrl`;
|
|
2268
|
+
* non-matches are logged once per daemon process.
|
|
2269
|
+
*/
|
|
2270
|
+
const rejectedSalvagedPrRuns = new Set<string>();
|
|
2271
|
+
const salvagedPrCursor = new Map<string, string>();
|
|
2272
|
+
|
|
2273
|
+
export async function adoptSalvagedPrs(
|
|
2274
|
+
d: Pick<Deps, "project" | "tracker" | "store">,
|
|
2275
|
+
now = Date.now(),
|
|
2276
|
+
): Promise<void> {
|
|
2277
|
+
const { project, tracker, store } = d;
|
|
2278
|
+
const eligible = store
|
|
2279
|
+
.recentRuns(project.name, now - PR_LOOKUP_WINDOW_MS)
|
|
2280
|
+
.filter(
|
|
2281
|
+
(run) =>
|
|
2282
|
+
ADOPTABLE_PR_STATES[run.state] === true &&
|
|
2283
|
+
run.prUrl === undefined &&
|
|
2284
|
+
run.branch.trim() !== "",
|
|
2285
|
+
);
|
|
2286
|
+
const previous = salvagedPrCursor.get(project.name);
|
|
2287
|
+
const previousIndex =
|
|
2288
|
+
previous === undefined ? -1 : eligible.findIndex((run) => run.id === previous);
|
|
2289
|
+
const start = previousIndex === -1 ? 0 : (previousIndex + 1) % eligible.length;
|
|
2290
|
+
const candidates = Array.from(
|
|
2291
|
+
{ length: Math.min(SALVAGED_PR_ADOPTION_BATCH, eligible.length) },
|
|
2292
|
+
(_, offset) => eligible[(start + offset) % eligible.length]!,
|
|
2293
|
+
);
|
|
2294
|
+
const last = candidates.at(-1);
|
|
2295
|
+
if (last !== undefined) salvagedPrCursor.set(project.name, last.id);
|
|
2296
|
+
|
|
2297
|
+
for (const run of candidates) {
|
|
2298
|
+
const repo = project.routing.repos[run.repo];
|
|
2299
|
+
const repoIdentity = repo === undefined ? undefined : githubRepo(repo.cloneUrl);
|
|
2300
|
+
let closers: OpenCloser[];
|
|
2301
|
+
try {
|
|
2302
|
+
closers = await tracker.openClosersFor(run.issue);
|
|
2303
|
+
} catch (err) {
|
|
2304
|
+
log(`#${run.issue} PR adoption lookup failed (${errText(err)}) — retrying next tick`);
|
|
2305
|
+
continue;
|
|
2306
|
+
}
|
|
2307
|
+
|
|
2308
|
+
const closer = closers.find(
|
|
2309
|
+
(candidate) =>
|
|
2310
|
+
candidate.headRefName !== "" &&
|
|
2311
|
+
candidate.headRefName === run.branch &&
|
|
2312
|
+
repo !== undefined &&
|
|
2313
|
+
candidate.repo !== "" &&
|
|
2314
|
+
candidate.repo === repoIdentity,
|
|
2315
|
+
);
|
|
2316
|
+
if (closer === undefined) {
|
|
2317
|
+
if (!rejectedSalvagedPrRuns.has(run.id)) {
|
|
2318
|
+
const observed = closers[0];
|
|
2319
|
+
const reason =
|
|
2320
|
+
observed === undefined
|
|
2321
|
+
? "no open closing PR"
|
|
2322
|
+
: observed.headRefName === ""
|
|
2323
|
+
? "closer has no head branch identity"
|
|
2324
|
+
: observed.headRefName !== run.branch
|
|
2325
|
+
? `closer head ${observed.headRefName} does not match retained branch ${run.branch}`
|
|
2326
|
+
: observed.repo === ""
|
|
2327
|
+
? "closer has no repository identity"
|
|
2328
|
+
: `closer repository ${observed.repo} does not match routed repository`;
|
|
2329
|
+
log(`#${run.issue} PR not adopted onto attempt ${run.attempt}: ${reason}`);
|
|
2330
|
+
rejectedSalvagedPrRuns.add(run.id);
|
|
2331
|
+
}
|
|
2332
|
+
continue;
|
|
2333
|
+
}
|
|
2334
|
+
|
|
2335
|
+
const flag: SettlementFlag = {
|
|
2336
|
+
kind: "pr-adopted",
|
|
2337
|
+
file: "(recovery)",
|
|
2338
|
+
detail: `${closer.url} matched retained branch ${run.branch} in ${closer.repo}`,
|
|
2339
|
+
};
|
|
2340
|
+
store.updateRun(run.id, {
|
|
2341
|
+
prUrl: closer.url,
|
|
2342
|
+
settlementFlags: [...(run.settlementFlags ?? []), flag],
|
|
2343
|
+
});
|
|
2344
|
+
rejectedSalvagedPrRuns.delete(run.id);
|
|
2345
|
+
log(
|
|
2346
|
+
`#${run.issue} adopted PR ${closer.url} onto attempt ${run.attempt}` +
|
|
2347
|
+
(run.salvageSha === undefined ? "" : ` (salvaged head ${run.salvageSha})`),
|
|
2348
|
+
);
|
|
2349
|
+
}
|
|
2350
|
+
}
|
|
2351
|
+
|
|
2352
|
+
/** Canonical `owner/repo` identity from a configured network clone URL. */
|
|
2353
|
+
function githubRepo(cloneUrl: string): string | undefined {
|
|
2354
|
+
const normalized = cloneUrl.replace(/\/$/, "").replace(/\.git$/, "");
|
|
2355
|
+
const match = /^(?:https?:\/\/[^/]+\/|ssh:\/\/git@[^/]+\/|git@[^:]+:)([^/\s]+\/[^/\s]+)$/.exec(
|
|
2356
|
+
normalized,
|
|
2357
|
+
);
|
|
2358
|
+
return match?.[1];
|
|
2359
|
+
}
|
|
2360
|
+
|
|
1723
2361
|
const RETAINED_CLEANUP_BATCH = 10;
|
|
1724
2362
|
|
|
1725
2363
|
export interface RetainedCleanupCursor {
|
|
@@ -1833,6 +2471,7 @@ const HOLD_SAMPLE_SIZE = 5;
|
|
|
1833
2471
|
const DEGRADED_HOLDS: ReadonlySet<AdmissionHoldReason> = new Set([
|
|
1834
2472
|
"parent-lookup-error",
|
|
1835
2473
|
"open-pr-lookup-error",
|
|
2474
|
+
"issue-state-lookup-error",
|
|
1836
2475
|
]);
|
|
1837
2476
|
|
|
1838
2477
|
/** Groups transient decisions into the bounded record exposed by status. */
|
|
@@ -2217,6 +2856,32 @@ export async function admitCandidates(
|
|
|
2217
2856
|
log(`#${issue} continuing retained PR ${closer.url} ${resume}`);
|
|
2218
2857
|
}
|
|
2219
2858
|
|
|
2859
|
+
// The queue comes from GitHub's eventually-consistent search index. Re-read
|
|
2860
|
+
// state and labels directly at the last possible moment so a just-closed or
|
|
2861
|
+
// explicitly dequeued issue cannot turn a stale candidate into another
|
|
2862
|
+
// attempt (#247).
|
|
2863
|
+
let snapshot: IssueSnapshot | undefined;
|
|
2864
|
+
try {
|
|
2865
|
+
snapshot = await tracker.issueSnapshot(issue);
|
|
2866
|
+
} catch {
|
|
2867
|
+
snapshot = undefined;
|
|
2868
|
+
}
|
|
2869
|
+
if (snapshot === undefined) {
|
|
2870
|
+
hold(issue, "issue-state-lookup-error");
|
|
2871
|
+
log(`#${issue} held: issue snapshot check failed — retrying next tick`);
|
|
2872
|
+
continue;
|
|
2873
|
+
}
|
|
2874
|
+
if (snapshot.state === "closed") {
|
|
2875
|
+
hold(issue, "issue-closed");
|
|
2876
|
+
log(`#${issue} skipped: issue is closed (search index lag)`);
|
|
2877
|
+
continue;
|
|
2878
|
+
}
|
|
2879
|
+
if (!snapshot.labels.includes(project.queueLabel)) {
|
|
2880
|
+
hold(issue, "issue-dequeued");
|
|
2881
|
+
log(`#${issue} skipped: queue label ${project.queueLabel} was removed (search index lag)`);
|
|
2882
|
+
continue;
|
|
2883
|
+
}
|
|
2884
|
+
|
|
2220
2885
|
admitted.push({ r, attempt: priorRuns + 1 });
|
|
2221
2886
|
liveByRepo.set(r.repo.name, (liveByRepo.get(r.repo.name) ?? 0) + 1);
|
|
2222
2887
|
if (parent !== undefined) {
|
|
@@ -2289,8 +2954,26 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
2289
2954
|
}
|
|
2290
2955
|
d.project = fresh;
|
|
2291
2956
|
d.caps = freshCaps;
|
|
2957
|
+
d.deliveryPolicyValid = true;
|
|
2292
2958
|
} catch (err) {
|
|
2293
|
-
log(`config reload failed (${errText(err)}) —
|
|
2959
|
+
log(`config reload failed (${errText(err)}) — retaining boot values but blocking autonomous delivery`);
|
|
2960
|
+
d.deliveryPolicyValid = false;
|
|
2961
|
+
}
|
|
2962
|
+
|
|
2963
|
+
// Availability-held notices are already durable. Once the freshly reloaded
|
|
2964
|
+
// policy opens (or newly allows their category), atomically hand a bounded
|
|
2965
|
+
// batch to the report outbox. Pauses do not suppress delivery.
|
|
2966
|
+
if (d.deliveryPolicyValid !== false) {
|
|
2967
|
+
try {
|
|
2968
|
+
const catchUp = enqueueAvailableHeldNotices(d.project, d.store, Date.now());
|
|
2969
|
+
if (catchUp !== undefined && !catchUp.deduped) {
|
|
2970
|
+
log(`availability catch-up ${catchUp.report.id} queued for ${d.project.name}`);
|
|
2971
|
+
}
|
|
2972
|
+
} catch (err) {
|
|
2973
|
+
// The daily digest may have claimed the same rows from another process
|
|
2974
|
+
// between selection and association. Either way the ledger still owns them.
|
|
2975
|
+
log(`availability catch-up handoff deferred (${errText(err)}) — retrying next tick`);
|
|
2976
|
+
}
|
|
2294
2977
|
}
|
|
2295
2978
|
|
|
2296
2979
|
// Before the pause check, deliberately. This one is not about dispatch: the
|
|
@@ -2305,6 +2988,21 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
2305
2988
|
// while the fleet is parked or workers are still active. Resident workers run
|
|
2306
2989
|
// through the pool without blocking this five-minute tick.
|
|
2307
2990
|
await settlePushedGreen(d);
|
|
2991
|
+
try {
|
|
2992
|
+
await watchMergedBase(d);
|
|
2993
|
+
} catch (err) {
|
|
2994
|
+
log(`base-branch check sweep failed: ${errText(err)}`);
|
|
2995
|
+
}
|
|
2996
|
+
try {
|
|
2997
|
+
await watchBaseHealth(d);
|
|
2998
|
+
} catch (err) {
|
|
2999
|
+
log(`current base-health sweep failed: ${errText(err)}`);
|
|
3000
|
+
}
|
|
3001
|
+
try {
|
|
3002
|
+
await adoptSalvagedPrs(d);
|
|
3003
|
+
} catch (err) {
|
|
3004
|
+
log(`salvaged PR adoption sweep failed: ${errText(err)}`);
|
|
3005
|
+
}
|
|
2308
3006
|
|
|
2309
3007
|
// Immediately after settlement and before any routing, so a class is on the
|
|
2310
3008
|
// row before the next dispatch decision reads its budgets (#132). Above the
|
|
@@ -2548,6 +3246,8 @@ export interface DaemonHealthSnapshot {
|
|
|
2548
3246
|
paused: boolean;
|
|
2549
3247
|
activeRuns: number;
|
|
2550
3248
|
project: string;
|
|
3249
|
+
/** One-shot issue ceilings waiting for the next claim. */
|
|
3250
|
+
turnOverrides: TurnOverride[];
|
|
2551
3251
|
/** Resident set of this daemon; workers are in-process omp sessions. */
|
|
2552
3252
|
rssBytes: number;
|
|
2553
3253
|
dispatch?: DispatchSummary;
|
|
@@ -2569,6 +3269,7 @@ export function daemonHealthSnapshot(
|
|
|
2569
3269
|
ok: true,
|
|
2570
3270
|
paused,
|
|
2571
3271
|
activeRuns: store.activeRuns(project).length,
|
|
3272
|
+
turnOverrides: store.listTurnOverrides(project),
|
|
2572
3273
|
project,
|
|
2573
3274
|
rssBytes,
|
|
2574
3275
|
...(dispatch === undefined ? {} : { dispatch }),
|
|
@@ -2577,11 +3278,19 @@ export function daemonHealthSnapshot(
|
|
|
2577
3278
|
};
|
|
2578
3279
|
}
|
|
2579
3280
|
|
|
3281
|
+
const TURN_OVERRIDE_STATES: ReadonlySet<RunState> = new Set([
|
|
3282
|
+
"failed",
|
|
3283
|
+
"killed",
|
|
3284
|
+
"orphaned",
|
|
3285
|
+
"blocked",
|
|
3286
|
+
]);
|
|
3287
|
+
|
|
2580
3288
|
export async function turnLimitResponse(
|
|
2581
3289
|
req: Request,
|
|
2582
3290
|
project: string,
|
|
2583
|
-
store: Pick<Store, "latestRun">,
|
|
3291
|
+
store: Pick<Store, "latestRun" | "setTurnOverride">,
|
|
2584
3292
|
registry: TurnLimitRegistry,
|
|
3293
|
+
caps: Pick<Caps, "workerMaxTurns" | "workerMaxTurnsCeiling">,
|
|
2585
3294
|
): Promise<Response | undefined> {
|
|
2586
3295
|
const url = new URL(req.url);
|
|
2587
3296
|
const match = /^\/runs\/(\d+)\/turn-limit$/.exec(url.pathname);
|
|
@@ -2615,6 +3324,16 @@ export async function turnLimitResponse(
|
|
|
2615
3324
|
}
|
|
2616
3325
|
|
|
2617
3326
|
const issue = Number(match[1]);
|
|
3327
|
+
if ((maxTurns as number) > caps.workerMaxTurnsCeiling) {
|
|
3328
|
+
return Response.json(
|
|
3329
|
+
{
|
|
3330
|
+
error:
|
|
3331
|
+
`#${issue} turn budget ${maxTurns as number} exceeds the ` +
|
|
3332
|
+
`${caps.workerMaxTurnsCeiling}-turn caps.workerMaxTurnsCeiling`,
|
|
3333
|
+
},
|
|
3334
|
+
{ status: 422 },
|
|
3335
|
+
);
|
|
3336
|
+
}
|
|
2618
3337
|
const outcome = registry.extend(project, issue, maxTurns as number);
|
|
2619
3338
|
if (outcome.kind === "extended") return Response.json(outcome);
|
|
2620
3339
|
if (outcome.kind === "not-increase") {
|
|
@@ -2628,14 +3347,33 @@ export async function turnLimitResponse(
|
|
|
2628
3347
|
if (latest === undefined) {
|
|
2629
3348
|
return Response.json({ error: `no run recorded for #${issue}` }, { status: 404 });
|
|
2630
3349
|
}
|
|
2631
|
-
|
|
2632
|
-
|
|
2633
|
-
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
|
|
3350
|
+
if (!TURN_OVERRIDE_STATES.has(latest.state)) {
|
|
3351
|
+
return Response.json(
|
|
3352
|
+
{
|
|
3353
|
+
error:
|
|
3354
|
+
`#${issue} has no live worker controller; its session already settled ` +
|
|
3355
|
+
`or belongs to another daemon (stored state: ${latest.state})`,
|
|
3356
|
+
},
|
|
3357
|
+
{ status: 409 },
|
|
3358
|
+
);
|
|
3359
|
+
}
|
|
3360
|
+
if ((maxTurns as number) <= caps.workerMaxTurns) {
|
|
3361
|
+
return Response.json(
|
|
3362
|
+
{
|
|
3363
|
+
error:
|
|
3364
|
+
`#${issue} next-attempt turn budget must exceed the ` +
|
|
3365
|
+
`${caps.workerMaxTurns}-turn caps.workerMaxTurns base`,
|
|
3366
|
+
},
|
|
3367
|
+
{ status: 409 },
|
|
3368
|
+
);
|
|
3369
|
+
}
|
|
3370
|
+
store.setTurnOverride(project, issue, maxTurns as number);
|
|
3371
|
+
return Response.json({
|
|
3372
|
+
kind: "next-attempt",
|
|
3373
|
+
issue,
|
|
3374
|
+
nextAttemptMaxTurns: maxTurns,
|
|
3375
|
+
baseMaxTurns: caps.workerMaxTurns,
|
|
3376
|
+
});
|
|
2639
3377
|
}
|
|
2640
3378
|
|
|
2641
3379
|
export async function workerControlResponse(
|
|
@@ -2645,7 +3383,7 @@ export async function workerControlResponse(
|
|
|
2645
3383
|
registry: WorkerControlRegistry,
|
|
2646
3384
|
): Promise<Response | undefined> {
|
|
2647
3385
|
const url = new URL(req.url);
|
|
2648
|
-
const match = /^\/runs\/(\d+)\/(pause|resume)$/.exec(url.pathname);
|
|
3386
|
+
const match = /^\/runs\/(\d+)\/(pause|resume|stop)$/.exec(url.pathname);
|
|
2649
3387
|
if (req.method !== "PUT" || match === null) return undefined;
|
|
2650
3388
|
if (!req.headers.get("content-type")?.startsWith("application/json")) {
|
|
2651
3389
|
return Response.json({ error: "content-type must be application/json" }, { status: 415 });
|
|
@@ -2670,23 +3408,71 @@ export async function workerControlResponse(
|
|
|
2670
3408
|
{ status: 409 },
|
|
2671
3409
|
);
|
|
2672
3410
|
}
|
|
3411
|
+
const action = match[2] as "pause" | "resume" | "stop";
|
|
3412
|
+
let reason: string | undefined;
|
|
3413
|
+
if (action === "stop") {
|
|
3414
|
+
const rawReason = Reflect.get(body, "reason");
|
|
3415
|
+
if (typeof rawReason !== "string" || rawReason.trim() === "") {
|
|
3416
|
+
return Response.json({ error: "reason must be a non-empty string" }, { status: 400 });
|
|
3417
|
+
}
|
|
3418
|
+
reason = rawReason.trim().replace(/\s+/g, " ");
|
|
3419
|
+
if (reason.length > 500) {
|
|
3420
|
+
return Response.json({ error: "reason must be at most 500 characters" }, { status: 400 });
|
|
3421
|
+
}
|
|
3422
|
+
}
|
|
3423
|
+
|
|
2673
3424
|
|
|
2674
3425
|
const issue = Number(match[1]);
|
|
2675
3426
|
const outcome =
|
|
2676
|
-
|
|
3427
|
+
action === "pause"
|
|
2677
3428
|
? await registry.pause(project, issue)
|
|
2678
|
-
:
|
|
3429
|
+
: action === "resume"
|
|
3430
|
+
? registry.resume(project, issue)
|
|
3431
|
+
: await registry.stop(project, issue, reason!);
|
|
2679
3432
|
if (outcome.kind === "ok") {
|
|
2680
3433
|
return Response.json({ runId: outcome.runId, phase: outcome.phase });
|
|
2681
3434
|
}
|
|
2682
3435
|
if (outcome.kind === "refused") {
|
|
2683
3436
|
return Response.json({ error: `#${issue}: ${outcome.error}` }, { status: 409 });
|
|
2684
3437
|
}
|
|
3438
|
+
if (outcome.kind === "stopped") {
|
|
3439
|
+
const stopped = store.latestRun(project, issue);
|
|
3440
|
+
if (stopped === undefined || stopped.id !== outcome.runId) {
|
|
3441
|
+
return Response.json(
|
|
3442
|
+
{ error: `#${issue} stopped, but its terminal run record is unavailable` },
|
|
3443
|
+
{ status: 500 },
|
|
3444
|
+
);
|
|
3445
|
+
}
|
|
3446
|
+
if (stopped.state !== "stopped") {
|
|
3447
|
+
return Response.json({
|
|
3448
|
+
outcome: "already-terminal",
|
|
3449
|
+
runId: stopped.id,
|
|
3450
|
+
state: stopped.state,
|
|
3451
|
+
});
|
|
3452
|
+
}
|
|
3453
|
+
return Response.json({
|
|
3454
|
+
outcome: "stopped",
|
|
3455
|
+
runId: stopped.id,
|
|
3456
|
+
state: stopped.state,
|
|
3457
|
+
reason: outcome.reason,
|
|
3458
|
+
...(stopped.salvageSha === undefined ? {} : { salvageSha: stopped.salvageSha }),
|
|
3459
|
+
...(stopped.salvageError === undefined ? {} : { salvageError: stopped.salvageError }),
|
|
3460
|
+
worktree: stopped.worktree,
|
|
3461
|
+
});
|
|
3462
|
+
}
|
|
3463
|
+
|
|
2685
3464
|
|
|
2686
3465
|
const latest = store.latestRun(project, issue);
|
|
2687
3466
|
if (latest === undefined) {
|
|
2688
3467
|
return Response.json({ error: `no run recorded for #${issue}` }, { status: 404 });
|
|
2689
3468
|
}
|
|
3469
|
+
if (action === "stop" && !LIVE_STATES.includes(latest.state)) {
|
|
3470
|
+
return Response.json({
|
|
3471
|
+
outcome: "already-terminal",
|
|
3472
|
+
runId: latest.id,
|
|
3473
|
+
state: latest.state,
|
|
3474
|
+
});
|
|
3475
|
+
}
|
|
2690
3476
|
return Response.json(
|
|
2691
3477
|
{
|
|
2692
3478
|
error:
|
|
@@ -2699,7 +3485,8 @@ export async function workerControlResponse(
|
|
|
2699
3485
|
|
|
2700
3486
|
export interface DaemonHttpDeps {
|
|
2701
3487
|
project: string;
|
|
2702
|
-
store: Pick<Store, "latestRun">;
|
|
3488
|
+
store: Pick<Store, "latestRun" | "setTurnOverride">;
|
|
3489
|
+
caps: () => Pick<Caps, "workerMaxTurns" | "workerMaxTurnsCeiling">;
|
|
2703
3490
|
turnLimits: TurnLimitRegistry;
|
|
2704
3491
|
workerControls: WorkerControlRegistry;
|
|
2705
3492
|
health: () => DaemonHealthSnapshot;
|
|
@@ -2709,15 +3496,15 @@ export interface DaemonHttpDeps {
|
|
|
2709
3496
|
* The whole HTTP surface, in one named function so a test can pin what is *not*
|
|
2710
3497
|
* on it.
|
|
2711
3498
|
*
|
|
2712
|
-
* Three route families: the health read, turn-limit control, and worker
|
|
2713
|
-
* pause/resume control. Everything else is 404. The controls mutate only
|
|
3499
|
+
* Three route families: the health read, turn-limit control, and live-worker
|
|
3500
|
+
* pause/resume/stop control. Everything else is 404. The controls mutate only
|
|
2714
3501
|
* daemon-owned live sessions; tracker and repository mutations stay on the
|
|
2715
3502
|
* authenticated per-run channel described at the `Bun.serve` call (#126).
|
|
2716
3503
|
*/
|
|
2717
3504
|
export async function daemonHttpResponse(req: Request, d: DaemonHttpDeps): Promise<Response> {
|
|
2718
3505
|
const url = new URL(req.url);
|
|
2719
3506
|
if (req.method === "GET" && url.pathname === "/healthz") return Response.json(d.health());
|
|
2720
|
-
const turnLimit = await turnLimitResponse(req, d.project, d.store, d.turnLimits);
|
|
3507
|
+
const turnLimit = await turnLimitResponse(req, d.project, d.store, d.turnLimits, d.caps());
|
|
2721
3508
|
if (turnLimit !== undefined) return turnLimit;
|
|
2722
3509
|
const workerControl = await workerControlResponse(req, d.project, d.store, d.workerControls);
|
|
2723
3510
|
return workerControl ?? new Response("not found\n", { status: 404 });
|
|
@@ -2734,6 +3521,10 @@ export interface StatusSnapshot {
|
|
|
2734
3521
|
* reading like a mistake (#220).
|
|
2735
3522
|
*/
|
|
2736
3523
|
pauseReason?: string;
|
|
3524
|
+
/** Mechanical operator availability at the moment this snapshot was read. */
|
|
3525
|
+
availability?: AvailabilityState;
|
|
3526
|
+
/** Next digest opportunity under the same predicate that gates submission. */
|
|
3527
|
+
digestSchedule?: DigestScheduleState;
|
|
2737
3528
|
caps: Caps;
|
|
2738
3529
|
/**
|
|
2739
3530
|
* The effective per-shape release grants. On the snapshot rather than re-read
|
|
@@ -2747,10 +3538,15 @@ export interface StatusSnapshot {
|
|
|
2747
3538
|
/** Newest attempts holding a preserved WIP tip, or a tree that is still the
|
|
2748
3539
|
* only copy of work the daemon could not save. */
|
|
2749
3540
|
salvagedRuns: RunRecord[];
|
|
3541
|
+
/** One-shot issue ceilings waiting for the next claim. */
|
|
3542
|
+
turnOverrides: TurnOverride[];
|
|
2750
3543
|
/** Reports the operator has not provably received: pending, in-flight with an
|
|
2751
3544
|
* unknown outcome, or written off. An empty list is the only honest way to
|
|
2752
3545
|
* say "everything authored this cycle actually went out" (#123). */
|
|
2753
3546
|
openReports: ReportRecord[];
|
|
3547
|
+
/** Ordinary outcomes and deferred escalations not yet associated with an
|
|
3548
|
+
* accepted digest report. */
|
|
3549
|
+
digestBacklog: DigestBacklog;
|
|
2754
3550
|
/**
|
|
2755
3551
|
* The most recent conductor-verb calls and how the daemon decided them
|
|
2756
3552
|
* (#126). On `status` rather than only behind `omp-conductor ledger` because
|
|
@@ -2758,6 +3554,8 @@ export interface StatusSnapshot {
|
|
|
2758
3554
|
* config does not let it, and an operator who has to know to go looking is an
|
|
2759
3555
|
* operator who finds out from the tracker instead.
|
|
2760
3556
|
*/
|
|
3557
|
+
/** Current live-head push-workflow verdict per recently merged repository. */
|
|
3558
|
+
baseHealth: BaseHealth[];
|
|
2761
3559
|
verbLedger: VerbLedgerEntry[];
|
|
2762
3560
|
/** Runs backed by a worker process — the number capacity compares against. */
|
|
2763
3561
|
liveWorkers: number;
|
|
@@ -2804,6 +3602,10 @@ export function statusSnapshotFromStore(
|
|
|
2804
3602
|
store: Store,
|
|
2805
3603
|
planUsage?: PlanUsageStatus,
|
|
2806
3604
|
): StatusSnapshot {
|
|
3605
|
+
const now = Date.now();
|
|
3606
|
+
const lastDigestKey = store.lastDigestDedupeKey(p.name);
|
|
3607
|
+
const lastDigestDay =
|
|
3608
|
+
lastDigestKey === undefined ? undefined : lastDigestKey.slice("digest:".length);
|
|
2807
3609
|
const since = startOfToday();
|
|
2808
3610
|
const dispatch = store.latestDispatch(p.name);
|
|
2809
3611
|
const labelOpsPending = store.countPendingLabelOps(p.name);
|
|
@@ -2817,11 +3619,15 @@ export function statusSnapshotFromStore(
|
|
|
2817
3619
|
stateDir: stateDir(),
|
|
2818
3620
|
paused: isPaused(),
|
|
2819
3621
|
...(reason === undefined ? {} : { pauseReason: reason }),
|
|
3622
|
+
availability: availabilityState(p.reporting, now),
|
|
3623
|
+
digestSchedule: digestScheduleState(p.reporting ?? DEFAULT_REPORT_POLICY, lastDigestDay, now),
|
|
2820
3624
|
caps,
|
|
2821
3625
|
releaseGrants: resolveReleaseGrants(p),
|
|
2822
3626
|
activeRuns: store.activeRuns(p.name),
|
|
2823
3627
|
salvagedRuns: store.salvagedRuns(p.name),
|
|
3628
|
+
turnOverrides: store.listTurnOverrides(p.name),
|
|
2824
3629
|
openReports: store.openReports(p.name),
|
|
3630
|
+
digestBacklog: store.digestBacklog(p.name),
|
|
2825
3631
|
verbLedger: store.verbLedger(p.name, { limit: STATUS_LEDGER_SCAN }),
|
|
2826
3632
|
liveWorkers: store.liveRuns(p.name).length,
|
|
2827
3633
|
runsToday: store.runsStartedSince(p.name, since),
|
|
@@ -2830,11 +3636,12 @@ export function statusSnapshotFromStore(
|
|
|
2830
3636
|
...(planUsage === undefined ? {} : { planUsage }),
|
|
2831
3637
|
// Written by the tracker's hooks rather than polled, so the renderer does
|
|
2832
3638
|
// not re-read GitHub to know it is being refused (#198).
|
|
2833
|
-
ghRefusals: store.ghRefusalsSince?.(
|
|
3639
|
+
ghRefusals: store.ghRefusalsSince?.(now - 5 * 60_000),
|
|
2834
3640
|
ghCallsToday: store.ghCallsToday?.(utcDay()),
|
|
2835
3641
|
...(labelOpsPending === 0 || oldestLabelOpAt === undefined
|
|
2836
3642
|
? {}
|
|
2837
|
-
: { labelOps: { pending: labelOpsPending, oldestAgeMs:
|
|
3643
|
+
: { labelOps: { pending: labelOpsPending, oldestAgeMs: now - oldestLabelOpAt } }),
|
|
3644
|
+
baseHealth: store.baseHealth(p.name),
|
|
2838
3645
|
};
|
|
2839
3646
|
}
|
|
2840
3647
|
|
|
@@ -2917,6 +3724,25 @@ export function formatReleaseGrants(grants: ResolvedGrants): string[] {
|
|
|
2917
3724
|
...RELEASE_SHAPES.map((shape) => ` ${shape.padEnd(19)}${grants[shape]}`),
|
|
2918
3725
|
];
|
|
2919
3726
|
}
|
|
3727
|
+
export function formatBaseHealth(rows: readonly BaseHealth[]): string[] {
|
|
3728
|
+
return rows.map((row) => {
|
|
3729
|
+
const head = row.headSha.slice(0, 8);
|
|
3730
|
+
if (row.verdict === "green") {
|
|
3731
|
+
return `base ${row.repo}/${row.branch} green (${row.runsCount} run(s)) at ${head}`;
|
|
3732
|
+
}
|
|
3733
|
+
if (row.verdict === "red") {
|
|
3734
|
+
return `base ${row.repo}/${row.branch} RED — ${row.detail ?? `workflow failed at ${head}`}`;
|
|
3735
|
+
}
|
|
3736
|
+
if (row.verdict === "pending") {
|
|
3737
|
+
return `base ${row.repo}/${row.branch} pending (${row.runsCount} run(s)) at ${head}`;
|
|
3738
|
+
}
|
|
3739
|
+
return (
|
|
3740
|
+
`base ${row.repo}/${row.branch} unknown — ` +
|
|
3741
|
+
(row.detail ?? `no push-triggered workflow run for ${head}`)
|
|
3742
|
+
);
|
|
3743
|
+
});
|
|
3744
|
+
}
|
|
3745
|
+
|
|
2920
3746
|
|
|
2921
3747
|
export function formatStatus(s: StatusSnapshot): string {
|
|
2922
3748
|
const lines = [
|
|
@@ -2935,6 +3761,9 @@ export function formatStatus(s: StatusSnapshot): string {
|
|
|
2935
3761
|
// fleet (#110).
|
|
2936
3762
|
` plan usage ${planUsageLine(s.planUsage)}`,
|
|
2937
3763
|
` new worker turns ${s.caps.workerMaxTurns}`,
|
|
3764
|
+
...s.turnOverrides.map(
|
|
3765
|
+
({ issue, maxTurns }) => ` turn override #${issue} → ${maxTurns} (next attempt)`,
|
|
3766
|
+
),
|
|
2938
3767
|
` worker wall clock ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
|
|
2939
3768
|
` failed attempts ${s.caps.maxAttemptsPerIssue}`,
|
|
2940
3769
|
` continuations ${s.caps.maxContinuationsPerIssue}`,
|
|
@@ -2961,6 +3790,7 @@ export function formatStatus(s: StatusSnapshot): string {
|
|
|
2961
3790
|
if (flagged !== undefined) lines.push(` ${flagged}`);
|
|
2962
3791
|
}
|
|
2963
3792
|
}
|
|
3793
|
+
lines.push(...formatBaseHealth(s.baseHealth));
|
|
2964
3794
|
lines.push(...formatSalvagedRuns(s.salvagedRuns));
|
|
2965
3795
|
lines.push(...formatOpenReports(s.openReports));
|
|
2966
3796
|
lines.push(...formatVerbLedger(s.verbLedger));
|
|
@@ -3096,6 +3926,15 @@ export interface SessionError {
|
|
|
3096
3926
|
message: string;
|
|
3097
3927
|
}
|
|
3098
3928
|
|
|
3929
|
+
export function completionLastError(
|
|
3930
|
+
providerCredit: string | undefined,
|
|
3931
|
+
providerTransient: string | undefined,
|
|
3932
|
+
verifiedReason: string | undefined,
|
|
3933
|
+
sessionErr: SessionError | undefined,
|
|
3934
|
+
): string | undefined {
|
|
3935
|
+
return providerCredit ?? providerTransient ?? verifiedReason ?? sessionErr?.message;
|
|
3936
|
+
}
|
|
3937
|
+
|
|
3099
3938
|
/**
|
|
3100
3939
|
* The last error a transcript recorded, or undefined when it recorded none.
|
|
3101
3940
|
*
|
|
@@ -3162,6 +4001,19 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
|
|
|
3162
4001
|
const { project, tracker, store } = d;
|
|
3163
4002
|
for (const run of store.runsNeedingClassification(project.name, CLASSIFY_BATCH)) {
|
|
3164
4003
|
const facts: ClassifyFacts = {};
|
|
4004
|
+
let classifiedRun = run;
|
|
4005
|
+
if (run.state === "failed" || run.state === "killed") {
|
|
4006
|
+
const sessionError = readSessionError(run.sessionFile);
|
|
4007
|
+
if (sessionError !== undefined) {
|
|
4008
|
+
if (run.lastError === undefined || run.lastError === sessionError.message) {
|
|
4009
|
+
facts.sessionError = sessionError;
|
|
4010
|
+
}
|
|
4011
|
+
if (run.lastError === undefined) {
|
|
4012
|
+
store.updateRun(run.id, { lastError: sessionError.message });
|
|
4013
|
+
classifiedRun = { ...run, lastError: sessionError.message };
|
|
4014
|
+
}
|
|
4015
|
+
}
|
|
4016
|
+
}
|
|
3165
4017
|
try {
|
|
3166
4018
|
if (run.prUrl !== undefined) {
|
|
3167
4019
|
const pr = await tracker.prState(run.prUrl);
|
|
@@ -3188,7 +4040,7 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
|
|
|
3188
4040
|
continue;
|
|
3189
4041
|
}
|
|
3190
4042
|
|
|
3191
|
-
const { cls, recovery, evidence } = classifyRun(
|
|
4043
|
+
const { cls, recovery, evidence } = classifyRun(classifiedRun, facts);
|
|
3192
4044
|
|
|
3193
4045
|
// A healthy green PR is not a failure of any class. Leaving the row
|
|
3194
4046
|
// unclassified is what keeps it eligible for the sweep on the tick where its
|
|
@@ -3202,7 +4054,7 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
|
|
|
3202
4054
|
? `#${run.issue} retrying ${recovery} for ${cls}: ${evidence}`
|
|
3203
4055
|
: `#${run.issue} classified ${cls} → ${recovery}: ${evidence}`,
|
|
3204
4056
|
);
|
|
3205
|
-
await recoverRun(d,
|
|
4057
|
+
await recoverRun(d, classifiedRun, cls, recovery, evidence);
|
|
3206
4058
|
}
|
|
3207
4059
|
}
|
|
3208
4060
|
|
|
@@ -3268,6 +4120,9 @@ async function recoverRun(
|
|
|
3268
4120
|
}
|
|
3269
4121
|
|
|
3270
4122
|
if (recovery === "requeue") {
|
|
4123
|
+
if (cls === "provider-credit") {
|
|
4124
|
+
await reactToProviderCredit(d, run.issue, evidence, run.sessionFile);
|
|
4125
|
+
}
|
|
3271
4126
|
// A dispatch-infra requeue that keeps landing on the same issue means the
|
|
3272
4127
|
// mirror for its repo is persistently broken — a ref-lock that retry already
|
|
3273
4128
|
// exhausted, a dead remote. Requeueing forever spends a turn-0 run per tick
|
|
@@ -3682,7 +4537,8 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
3682
4537
|
onChildLog: (line) => {
|
|
3683
4538
|
log(`orchestrator ${line}`);
|
|
3684
4539
|
},
|
|
3685
|
-
onReleaseBlocked: (shape) =>
|
|
4540
|
+
onReleaseBlocked: (shape, context) =>
|
|
4541
|
+
recordReleaseBlock(project.name, "orchestrator", shape, context),
|
|
3686
4542
|
});
|
|
3687
4543
|
const transcript = orchestrator.sessionFile();
|
|
3688
4544
|
log(`orchestrator session ready${transcript === undefined ? "" : ` · ${transcript}`}`);
|
|
@@ -3696,7 +4552,17 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
3696
4552
|
}
|
|
3697
4553
|
}
|
|
3698
4554
|
|
|
3699
|
-
|
|
4555
|
+
let runtimeDeps: Deps | undefined;
|
|
4556
|
+
const currentProject = (): ProjectConfig => runtimeDeps?.project ?? project;
|
|
4557
|
+
const deliveryPolicyValid = (): boolean => runtimeDeps?.deliveryPolicyValid === true;
|
|
4558
|
+
const escalator = createEscalator(
|
|
4559
|
+
currentProject,
|
|
4560
|
+
tracker,
|
|
4561
|
+
store,
|
|
4562
|
+
orchestrator,
|
|
4563
|
+
Date.now,
|
|
4564
|
+
deliveryPolicyValid,
|
|
4565
|
+
);
|
|
3700
4566
|
|
|
3701
4567
|
// Report delivery is the daemon's, not the model's (#123). Built beside the
|
|
3702
4568
|
// escalator because a report nobody can deliver pages through it, and driven
|
|
@@ -3704,10 +4570,11 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
3704
4570
|
// but still owes its operator the report it was handed, and five minutes is a
|
|
3705
4571
|
// long time to sit on a page.
|
|
3706
4572
|
const outbox = createReportOutbox({
|
|
3707
|
-
project,
|
|
4573
|
+
project: currentProject,
|
|
3708
4574
|
store,
|
|
3709
4575
|
escalate: (e) => escalator.escalate(e),
|
|
3710
4576
|
log,
|
|
4577
|
+
deliveryAllowed: deliveryPolicyValid,
|
|
3711
4578
|
});
|
|
3712
4579
|
|
|
3713
4580
|
// Every row still `sending` when a daemon boots belonged to a process that is
|
|
@@ -3733,6 +4600,8 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
3733
4600
|
caps,
|
|
3734
4601
|
tracker,
|
|
3735
4602
|
store,
|
|
4603
|
+
// Fail closed until the first tick re-reads and validates the live config.
|
|
4604
|
+
deliveryPolicyValid: false,
|
|
3736
4605
|
// Process-wide, so a `status` served off this daemon's own HTTP surface
|
|
3737
4606
|
// reuses the tick's reading instead of shelling out again.
|
|
3738
4607
|
usage: sharedUsageSource(),
|
|
@@ -3747,6 +4616,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
3747
4616
|
...(verbPeerReader === undefined ? {} : { verbPeerReader }),
|
|
3748
4617
|
verbActions,
|
|
3749
4618
|
};
|
|
4619
|
+
runtimeDeps = d;
|
|
3750
4620
|
|
|
3751
4621
|
if (o.once) {
|
|
3752
4622
|
try {
|
|
@@ -3821,9 +4691,11 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
3821
4691
|
// carries no credential of any kind, and it cannot tell one caller from
|
|
3822
4692
|
// another: `127.0.0.1` is not an identity. The turn-limit and worker
|
|
3823
4693
|
// pause/resume controls trust a body-supplied `project`, which is exactly the
|
|
3824
|
-
// shape "identity from the payload" takes when nobody is watching.
|
|
3825
|
-
//
|
|
3826
|
-
//
|
|
4694
|
+
// shape "identity from the payload" takes when nobody is watching. They stay
|
|
4695
|
+
// tolerable only because every effect is bounded: pauses are reversible,
|
|
4696
|
+
// live extensions touch one controller, and a persisted next-attempt
|
|
4697
|
+
// override can only raise the project base up to its configured ceiling and
|
|
4698
|
+
// is consumed by one claim.
|
|
3827
4699
|
//
|
|
3828
4700
|
// A merge, a push, a release or a label is none of those things. Do not add
|
|
3829
4701
|
// one here, and do not add "just a small one" behind a shared secret either:
|
|
@@ -3838,6 +4710,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
3838
4710
|
daemonHttpResponse(req, {
|
|
3839
4711
|
project: project.name,
|
|
3840
4712
|
store,
|
|
4713
|
+
caps: () => d.caps,
|
|
3841
4714
|
turnLimits,
|
|
3842
4715
|
workerControls,
|
|
3843
4716
|
health: () =>
|