omp-conductor 0.3.18 → 0.3.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +352 -182
- package/package.json +1 -1
- package/skills/conductor-onboarding/SKILL.md +43 -50
- package/skills/conductor-update/SKILL.md +15 -165
- package/src/board.ts +734 -0
- package/src/brief-upgrade.ts +102 -19
- package/src/briefs/orchestrator.md +32 -16
- package/src/briefs/worker.md +7 -2
- package/src/cli.ts +138 -53
- package/src/config.ts +22 -6
- package/src/daemon.ts +673 -126
- package/src/fleet.ts +64 -6
- package/src/graph-health.ts +296 -0
- package/src/graph.ts +6 -6
- package/src/omp.ts +15 -4
- package/src/orchestrator-tick.ts +157 -10
- package/src/orchestrator.ts +8 -1
- package/src/plugin.ts +173 -45
- package/src/release-policy.ts +202 -0
- package/src/setup-host.ts +285 -0
- package/src/setup.ts +24 -20
- package/src/store.ts +373 -13
- package/src/tracker/github.ts +171 -3
- package/src/transcript.ts +45 -0
- package/src/types.ts +145 -2
- package/src/unblock.ts +26 -14
- package/src/upgrade.ts +537 -0
- package/src/worker.ts +67 -30
- package/src/worktree.ts +66 -0
- package/systemd/omp-conductor.service.example +5 -3
package/src/daemon.ts
CHANGED
|
@@ -10,34 +10,41 @@
|
|
|
10
10
|
import { createHash } from "node:crypto";
|
|
11
11
|
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
12
12
|
import { dirname, join, relative } from "node:path";
|
|
13
|
-
import { configPath, findProject, loadConfig, resolveCaps, stateDir } from "./config.ts";
|
|
13
|
+
import { configPath, findProject, loadConfig, resolveCaps, resolveReleasePolicy, stateDir } from "./config.ts";
|
|
14
14
|
import { createEscalator, escalationIssueRef } from "./escalate.ts";
|
|
15
|
+
import { pendingCodeGraph, probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
|
|
15
16
|
import { graphHint } from "./graph.ts";
|
|
16
17
|
import { livingDaemon } from "./lifecycle.ts";
|
|
17
18
|
import { STALL_MARKER_FILE } from "./orchestrator-tick.ts";
|
|
18
19
|
import { startOrchestrator } from "./orchestrator.ts";
|
|
19
20
|
import type { OrchestratorHandle } from "./orchestrator.ts";
|
|
21
|
+
import { recordReleaseBlock } from "./release-policy.ts";
|
|
20
22
|
import { branchName, route } from "./routing.ts";
|
|
21
23
|
import type { Routed, UnroutableReason } from "./routing.ts";
|
|
22
|
-
import { openStore } from "./store.ts";
|
|
24
|
+
import { dbPath, openStore } from "./store.ts";
|
|
23
25
|
import { makeTracker } from "./tracker/github.ts";
|
|
24
26
|
import type {
|
|
27
|
+
AdmissionHoldReason,
|
|
25
28
|
Caps,
|
|
29
|
+
DispatchSummary,
|
|
26
30
|
Escalation,
|
|
27
31
|
PrState,
|
|
28
32
|
ProjectConfig,
|
|
29
33
|
ReadyIssue,
|
|
30
34
|
RepoTarget,
|
|
31
35
|
RunRecord,
|
|
36
|
+
RunState,
|
|
32
37
|
Store,
|
|
33
38
|
Tracker,
|
|
34
39
|
} from "./types.ts";
|
|
35
|
-
import { type KilledBy, renderBrief, runWorker } from "./worker.ts";
|
|
40
|
+
import { type KilledBy, type WorkerResult, renderBrief, runWorker } from "./worker.ts";
|
|
36
41
|
import {
|
|
37
42
|
addWorktree,
|
|
43
|
+
cleanupRetainedWorktree,
|
|
38
44
|
mirrorPathFor,
|
|
39
45
|
removeWorktree,
|
|
40
46
|
salvageWip,
|
|
47
|
+
type RetainedWorktreeCleanup,
|
|
41
48
|
type SalvageOutcome,
|
|
42
49
|
worktreePathFor,
|
|
43
50
|
} from "./worktree.ts";
|
|
@@ -45,6 +52,7 @@ import {
|
|
|
45
52
|
/** Long enough that the tracker is not polled raw, short enough that a human
|
|
46
53
|
* who labels an issue sees it picked up within a coffee break. */
|
|
47
54
|
const TICK_INTERVAL_MS = 5 * 60_000;
|
|
55
|
+
const GRAPH_HEALTH_INTERVAL_MS = 60_000;
|
|
48
56
|
const DEFAULT_PORT = 8787;
|
|
49
57
|
const BRIEF_TEMPLATE_PATH = join(import.meta.dir, "briefs", "worker.md");
|
|
50
58
|
|
|
@@ -72,16 +80,14 @@ interface Deps {
|
|
|
72
80
|
tracker: Tracker;
|
|
73
81
|
store: Store;
|
|
74
82
|
escalate(e: Escalation): Promise<void>;
|
|
83
|
+
turnLimits: TurnLimitRegistry;
|
|
75
84
|
integrity: IntegrityGate;
|
|
76
85
|
stall: StallGate;
|
|
86
|
+
cleanup?: RetainedCleanupCursor;
|
|
77
87
|
}
|
|
78
88
|
|
|
79
89
|
// ---------------------------------------------------------------- paths & pause
|
|
80
90
|
|
|
81
|
-
/** Single database for every project; the store partitions by project name. */
|
|
82
|
-
export function dbPath(): string {
|
|
83
|
-
return join(stateDir(), "conductor.db");
|
|
84
|
-
}
|
|
85
91
|
|
|
86
92
|
// ------------------------------------------------------- orchestrator liveness
|
|
87
93
|
|
|
@@ -501,6 +507,85 @@ export async function buildBrief(
|
|
|
501
507
|
|
|
502
508
|
// ------------------------------------------------------------------- one issue
|
|
503
509
|
|
|
510
|
+
/** `stops` are the operational ends that each require one resume. */
|
|
511
|
+
export function hasContinuationBudget(stops: number, maxContinuations: number): boolean {
|
|
512
|
+
return stops <= maxContinuations;
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
export type ExtendTurnLimitResult =
|
|
516
|
+
| { kind: "extended"; runId: string; maxTurns: number }
|
|
517
|
+
| { kind: "not-increase"; runId: string; maxTurns: number }
|
|
518
|
+
| { kind: "not-active" };
|
|
519
|
+
|
|
520
|
+
export interface TurnLimitController {
|
|
521
|
+
maxTurns(): number;
|
|
522
|
+
close(): void;
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
export interface TurnLimitRegistry {
|
|
526
|
+
open(project: string, issue: number, runId: string, maxTurns: number): TurnLimitController;
|
|
527
|
+
extend(project: string, issue: number, maxTurns: number): ExtendTurnLimitResult;
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
/**
|
|
531
|
+
* Authoritative live-run turn controls. Persistence happens synchronously
|
|
532
|
+
* before the in-memory ceiling changes, so no turn event can interleave.
|
|
533
|
+
*/
|
|
534
|
+
export function createTurnLimitRegistry(
|
|
535
|
+
persist: (runId: string, maxTurns: number) => void,
|
|
536
|
+
): TurnLimitRegistry {
|
|
537
|
+
const active = new Map<string, { runId: string; maxTurns: number }>();
|
|
538
|
+
const key = (project: string, issue: number): string => `${project}\0${issue}`;
|
|
539
|
+
return {
|
|
540
|
+
open(project, issue, runId, maxTurns) {
|
|
541
|
+
const k = key(project, issue);
|
|
542
|
+
if (active.has(k)) throw new Error(`#${issue} already has a live turn controller`);
|
|
543
|
+
const entry = { runId, maxTurns };
|
|
544
|
+
active.set(k, entry);
|
|
545
|
+
return {
|
|
546
|
+
maxTurns: () => entry.maxTurns,
|
|
547
|
+
close: () => {
|
|
548
|
+
if (active.get(k) === entry) active.delete(k);
|
|
549
|
+
},
|
|
550
|
+
};
|
|
551
|
+
},
|
|
552
|
+
extend(project, issue, maxTurns) {
|
|
553
|
+
if (!Number.isSafeInteger(maxTurns) || maxTurns < 1) {
|
|
554
|
+
throw new RangeError(`turn ceiling must be a positive integer, got ${maxTurns}`);
|
|
555
|
+
}
|
|
556
|
+
const entry = active.get(key(project, issue));
|
|
557
|
+
if (entry === undefined) return { kind: "not-active" };
|
|
558
|
+
if (maxTurns <= entry.maxTurns) {
|
|
559
|
+
return { kind: "not-increase", runId: entry.runId, maxTurns: entry.maxTurns };
|
|
560
|
+
}
|
|
561
|
+
persist(entry.runId, maxTurns);
|
|
562
|
+
entry.maxTurns = maxTurns;
|
|
563
|
+
return { kind: "extended", runId: entry.runId, maxTurns };
|
|
564
|
+
},
|
|
565
|
+
};
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
export async function verifyPushedGreenClaim(
|
|
569
|
+
tracker: Pick<Tracker, "verifyPr">,
|
|
570
|
+
claim: Pick<WorkerResult, "prUrl" | "headSha">,
|
|
571
|
+
): Promise<{
|
|
572
|
+
state: "pushed-green" | "pushed-pending" | "failed";
|
|
573
|
+
reason?: string;
|
|
574
|
+
}> {
|
|
575
|
+
if (claim.prUrl === undefined || claim.headSha === undefined) {
|
|
576
|
+
return { state: "failed", reason: "Worker did not report a PR URL and observed head SHA" };
|
|
577
|
+
}
|
|
578
|
+
const verification = await tracker.verifyPr(claim.prUrl, claim.headSha);
|
|
579
|
+
if (verification === undefined) {
|
|
580
|
+
return { state: "pushed-pending", reason: "GitHub PR verification unavailable; retrying" };
|
|
581
|
+
}
|
|
582
|
+
if (verification.status === "green") return { state: "pushed-green" };
|
|
583
|
+
return {
|
|
584
|
+
state: verification.status === "pending" ? "pushed-pending" : "failed",
|
|
585
|
+
reason: verification.reason,
|
|
586
|
+
};
|
|
587
|
+
}
|
|
588
|
+
|
|
504
589
|
/**
|
|
505
590
|
* One attempt at one issue, from claim to terminal state. Everything is inside
|
|
506
591
|
* a single try/catch so that a bad issue costs its own run and nothing else.
|
|
@@ -517,6 +602,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
517
602
|
// mid-dispatch is one of the non-graceful ends whose uncommitted work has to
|
|
518
603
|
// be salvaged too, and it is the path least likely to have committed first.
|
|
519
604
|
let worktreePath: string | undefined;
|
|
605
|
+
let turnLimit: TurnLimitController | undefined;
|
|
520
606
|
|
|
521
607
|
try {
|
|
522
608
|
// Claim on the tracker FIRST, before any local work. The label — not the
|
|
@@ -536,9 +622,11 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
536
622
|
attempt,
|
|
537
623
|
turns: 0,
|
|
538
624
|
spendUsd: 0,
|
|
625
|
+
maxTurns: caps.workerMaxTurns,
|
|
539
626
|
startedAt: Date.now(),
|
|
540
627
|
});
|
|
541
628
|
const runId = run.id;
|
|
629
|
+
turnLimit = d.turnLimits.open(project.name, issue, runId, caps.workerMaxTurns);
|
|
542
630
|
|
|
543
631
|
// A run's tree is <workspaceRoot>/<issue> and addWorktree refuses to reuse
|
|
544
632
|
// an existing path, so a retry — or a tree kept from a failed attempt — has
|
|
@@ -569,22 +657,38 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
569
657
|
(provisioned.reattached ? " (continuation: reattached existing branch)" : ""),
|
|
570
658
|
);
|
|
571
659
|
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
660
|
+
let result: WorkerResult;
|
|
661
|
+
try {
|
|
662
|
+
result = await runWorker({
|
|
663
|
+
brief: await buildBrief(project, r, branch, worktreePath, {
|
|
664
|
+
continuation: provisioned.reattached,
|
|
665
|
+
defaultBranch: r.repo.defaultBranch,
|
|
666
|
+
}),
|
|
667
|
+
cwd: worktreePath,
|
|
668
|
+
caps,
|
|
669
|
+
maxTurns: () => turnLimit?.maxTurns() ?? caps.workerMaxTurns,
|
|
670
|
+
sessionDir,
|
|
671
|
+
...(project.workerModel === undefined ? {} : { model: project.workerModel }),
|
|
672
|
+
releasePolicy: resolveReleasePolicy(project),
|
|
673
|
+
onReleaseBlocked: (shape) => recordReleaseBlock(project.name, "worker", shape),
|
|
674
|
+
onTurn: (n) => store.updateRun(runId, { turns: n }),
|
|
675
|
+
onSpend: (usd) => store.updateRun(runId, { spendUsd: usd }),
|
|
676
|
+
onKilled: () => {
|
|
677
|
+
turnLimit?.close();
|
|
678
|
+
turnLimit = undefined;
|
|
679
|
+
},
|
|
680
|
+
// Recorded the moment the session opens its transcript, not when the run
|
|
681
|
+
// ends: `omp-conductor tail` resolves an issue to a file through this row,
|
|
682
|
+
// and a path written at completion is a path nobody can follow live. The
|
|
683
|
+
// completion-time update below writes the same value again, harmlessly.
|
|
684
|
+
onSessionFile: (f) => store.updateRun(runId, { sessionFile: f }),
|
|
685
|
+
});
|
|
686
|
+
} finally {
|
|
687
|
+
// This is the authoritative settlement edge for `extend`: close before
|
|
688
|
+
// PR verification or terminal row writes can leave stale `running` state.
|
|
689
|
+
turnLimit?.close();
|
|
690
|
+
turnLimit = undefined;
|
|
691
|
+
}
|
|
588
692
|
|
|
589
693
|
// A configured model the harness could not honour means this run was done by
|
|
590
694
|
// a different model than the operator chose. Logged per run, because it is
|
|
@@ -593,16 +697,26 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
593
697
|
log(`#${issue} model fallback: ${result.modelFallbackMessage}`);
|
|
594
698
|
}
|
|
595
699
|
|
|
700
|
+
const verified: { state: RunState; reason?: string } =
|
|
701
|
+
result.state === "pushed-green"
|
|
702
|
+
? await verifyPushedGreenClaim(tracker, result)
|
|
703
|
+
: { state: result.state };
|
|
704
|
+
const state = verified.state;
|
|
705
|
+
const finalReport =
|
|
706
|
+
verified.reason === undefined ? result.report : `${verified.reason}\n\n${result.report}`;
|
|
707
|
+
|
|
596
708
|
store.updateRun(runId, {
|
|
597
|
-
state
|
|
709
|
+
state,
|
|
598
710
|
endedAt: Date.now(),
|
|
599
711
|
turns: result.turns,
|
|
600
712
|
spendUsd: result.spendUsd,
|
|
601
713
|
prUrl: result.prUrl,
|
|
714
|
+
headSha: result.headSha,
|
|
602
715
|
sessionFile: result.sessionFile,
|
|
716
|
+
...(verified.reason === undefined ? {} : { lastError: verified.reason }),
|
|
603
717
|
});
|
|
604
718
|
|
|
605
|
-
if (
|
|
719
|
+
if (state === "blocked") {
|
|
606
720
|
await swapLabel(tracker, issue, inProgress, project.stateLabels.blocked);
|
|
607
721
|
await safeEscalate(d, {
|
|
608
722
|
tier: 1,
|
|
@@ -612,20 +726,22 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
612
726
|
summary: `#${issue} is blocked on attempt ${attempt} and needs a decision`,
|
|
613
727
|
detail: [`${r.issue.title}`, r.issue.url, "", result.report].join("\n"),
|
|
614
728
|
});
|
|
615
|
-
} else if (
|
|
616
|
-
//
|
|
617
|
-
//
|
|
618
|
-
//
|
|
729
|
+
} else if (state === "failed" || state === "killed") {
|
|
730
|
+
// A turns cap consumes the independent continuation budget, not an
|
|
731
|
+
// implementation-failure attempt. The row is already `killed`, so this
|
|
732
|
+
// count includes the segment that just ended.
|
|
733
|
+
const continuation = store.continuationsFor(project.name, issue);
|
|
619
734
|
const continueTurns =
|
|
620
|
-
result.killedBy === "turns" &&
|
|
735
|
+
result.killedBy === "turns" &&
|
|
736
|
+
hasContinuationBudget(continuation, caps.maxContinuationsPerIssue);
|
|
621
737
|
const salvaged = await salvage(issue, attempt, endedBy(result.killedBy), worktreePath);
|
|
622
738
|
|
|
623
739
|
if (continueTurns) {
|
|
624
740
|
await tracker.removeLabel(issue, inProgress);
|
|
625
741
|
await tracker.addLabel(issue, project.queueLabel);
|
|
626
742
|
log(
|
|
627
|
-
`#${issue} turns-cap on
|
|
628
|
-
|
|
743
|
+
`#${issue} turns-cap on run ${attempt}, continuation ` +
|
|
744
|
+
`${continuation}/${caps.maxContinuationsPerIssue} — salvaged and re-queued`,
|
|
629
745
|
);
|
|
630
746
|
await safeEscalate(d, {
|
|
631
747
|
tier: 1,
|
|
@@ -664,24 +780,27 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
664
780
|
...salvaged,
|
|
665
781
|
`Session: ${result.sessionFile ?? "(no transcript)"}`,
|
|
666
782
|
"",
|
|
667
|
-
|
|
783
|
+
finalReport,
|
|
668
784
|
].join("\n"),
|
|
669
785
|
});
|
|
670
786
|
}
|
|
671
787
|
} else {
|
|
672
|
-
//
|
|
673
|
-
//
|
|
674
|
-
|
|
675
|
-
log(`#${issue} ${result.state}${result.prUrl ? ` ${result.prUrl}` : ""}`);
|
|
788
|
+
// A verified or still-pending PR keeps the in-progress label until its
|
|
789
|
+
// checks or merge settle, preventing another worker from duplicating it.
|
|
790
|
+
log(`#${issue} ${state}${result.prUrl ? ` ${result.prUrl}` : ""}`);
|
|
676
791
|
}
|
|
677
792
|
|
|
678
793
|
// A failed or killed tree is evidence — keep it. Anything else is just
|
|
679
794
|
// disk, and the mirror means re-provisioning is cheap. (The kept tree is
|
|
680
795
|
// wiped by the next attempt, not left to accumulate forever.)
|
|
681
|
-
if (
|
|
796
|
+
if (state !== "failed" && state !== "killed") {
|
|
682
797
|
await removeWorktree(mirrorPath, worktreePath);
|
|
683
798
|
}
|
|
684
799
|
} catch (err) {
|
|
800
|
+
// Dispatch setup can fail after the controller opens but before runWorker's
|
|
801
|
+
// inner settlement guard exists. Latch it before any terminal write or await.
|
|
802
|
+
turnLimit?.close();
|
|
803
|
+
turnLimit = undefined;
|
|
685
804
|
const detail = errText(err);
|
|
686
805
|
log(`#${issue} errored: ${detail}`);
|
|
687
806
|
if (run) {
|
|
@@ -713,6 +832,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
713
832
|
});
|
|
714
833
|
// The worktree, if one was created, is deliberately left in place: this is
|
|
715
834
|
// a failure path, and whatever it still held is now a commit on the branch.
|
|
835
|
+
} finally {
|
|
836
|
+
turnLimit?.close();
|
|
716
837
|
}
|
|
717
838
|
}
|
|
718
839
|
|
|
@@ -782,12 +903,13 @@ export async function settlePushedGreen(
|
|
|
782
903
|
// is live workers plus these, so the list is bounded by the worker cap plus
|
|
783
904
|
// the number of PRs awaiting a merge — a handful, by construction. A fleet
|
|
784
905
|
// where that is not a handful has a merge problem, not a dispatch one.
|
|
785
|
-
const pending = store
|
|
906
|
+
const pending = store
|
|
907
|
+
.activeRuns(project.name)
|
|
908
|
+
.filter((r) => r.state === "pushed-green" || r.state === "pushed-pending");
|
|
786
909
|
|
|
787
910
|
for (const run of pending) {
|
|
788
|
-
// Nothing to ask about. A
|
|
789
|
-
//
|
|
790
|
-
// forever to be told nothing.
|
|
911
|
+
// Nothing to ask about. A pushed result requires a PR, so a malformed row
|
|
912
|
+
// must not buy a `gh` call every five minutes forever.
|
|
791
913
|
if (run.prUrl === undefined) continue;
|
|
792
914
|
|
|
793
915
|
let pr: PrState | undefined;
|
|
@@ -803,12 +925,123 @@ export async function settlePushedGreen(
|
|
|
803
925
|
}
|
|
804
926
|
|
|
805
927
|
const settlement = settlementFor(pr, run.prUrl);
|
|
806
|
-
if (settlement
|
|
928
|
+
if (settlement !== undefined) {
|
|
929
|
+
const patch: Partial<RunRecord> = { state: settlement.state, endedAt: Date.now() };
|
|
930
|
+
if (settlement.state === "failed") patch.lastError = settlement.reason;
|
|
931
|
+
store.updateRun(run.id, patch);
|
|
932
|
+
log(`#${run.issue} settled: ${settlement.reason}`);
|
|
933
|
+
continue;
|
|
934
|
+
}
|
|
935
|
+
|
|
936
|
+
if (run.state !== "pushed-pending" || pr !== "open" || run.headSha === undefined) continue;
|
|
937
|
+
let verification;
|
|
938
|
+
try {
|
|
939
|
+
verification = await tracker.verifyPr(run.prUrl, run.headSha);
|
|
940
|
+
} catch (err) {
|
|
941
|
+
log(`#${run.issue} checks not settled (${errText(err)}) — retrying next tick`);
|
|
942
|
+
continue;
|
|
943
|
+
}
|
|
944
|
+
if (verification === undefined) continue;
|
|
945
|
+
if (verification.status === "green") {
|
|
946
|
+
store.updateRun(run.id, { state: "pushed-green", lastError: undefined });
|
|
947
|
+
log(`#${run.issue} checks settled: ${verification.reason}`);
|
|
948
|
+
} else if (verification.status === "failed") {
|
|
949
|
+
store.updateRun(run.id, { state: "failed", lastError: verification.reason });
|
|
950
|
+
log(`#${run.issue} checks failed: ${verification.reason}`);
|
|
951
|
+
} else {
|
|
952
|
+
store.updateRun(run.id, { lastError: verification.reason });
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
const RETAINED_CLEANUP_BATCH = 10;
|
|
958
|
+
|
|
959
|
+
export interface RetainedCleanupCursor {
|
|
960
|
+
next: number;
|
|
961
|
+
}
|
|
962
|
+
|
|
963
|
+
type CleanupRetainedWorktree = (
|
|
964
|
+
mirrorPath: string,
|
|
965
|
+
worktreePath: string,
|
|
966
|
+
branch: string,
|
|
967
|
+
) => Promise<RetainedWorktreeCleanup>;
|
|
968
|
+
|
|
969
|
+
/**
|
|
970
|
+
* Bounded, rotating cleanup for failure-path trees. Tracker state proves the
|
|
971
|
+
* run is terminal; local git state independently proves deletion cannot erase
|
|
972
|
+
* dirty or uniquely unpushed work.
|
|
973
|
+
*/
|
|
974
|
+
export async function cleanupRetainedRuns(
|
|
975
|
+
d: Pick<Deps, "project" | "tracker" | "store">,
|
|
976
|
+
queuedIssues: ReadonlySet<number>,
|
|
977
|
+
cursor: RetainedCleanupCursor,
|
|
978
|
+
cleanup: CleanupRetainedWorktree = cleanupRetainedWorktree,
|
|
979
|
+
): Promise<void> {
|
|
980
|
+
const { project, tracker, store } = d;
|
|
981
|
+
const candidates = store.retainedRuns(project.name);
|
|
982
|
+
if (candidates.length === 0) {
|
|
983
|
+
cursor.next = 0;
|
|
984
|
+
return;
|
|
985
|
+
}
|
|
986
|
+
|
|
987
|
+
const occupied = new Set(store.activeRuns(project.name).map((run) => run.issue));
|
|
988
|
+
const liveRepos = new Set(store.liveRuns(project.name).map((run) => run.repo));
|
|
989
|
+
const start = cursor.next % candidates.length;
|
|
990
|
+
const count = Math.min(RETAINED_CLEANUP_BATCH, candidates.length);
|
|
991
|
+
const batch = Array.from({ length: count }, (_, offset) => candidates[(start + offset) % candidates.length]!);
|
|
992
|
+
cursor.next = (start + count) % candidates.length;
|
|
993
|
+
|
|
994
|
+
for (const run of batch) {
|
|
995
|
+
if (occupied.has(run.issue) || queuedIssues.has(run.issue) || liveRepos.has(run.repo)) continue;
|
|
996
|
+
// Attempts reuse one physical path and deterministic branch. An older row
|
|
997
|
+
// cannot authorize deleting the newest failed attempt's evidence merely
|
|
998
|
+
// because its own PR resolved first.
|
|
999
|
+
const latest = store.latestRun(project.name, run.issue);
|
|
1000
|
+
if (
|
|
1001
|
+
latest !== undefined &&
|
|
1002
|
+
latest.id !== run.id &&
|
|
1003
|
+
latest.state !== "merged" &&
|
|
1004
|
+
latest.worktree !== ""
|
|
1005
|
+
) {
|
|
1006
|
+
continue;
|
|
1007
|
+
}
|
|
807
1008
|
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
1009
|
+
let terminal = false;
|
|
1010
|
+
try {
|
|
1011
|
+
if (run.prUrl !== undefined) {
|
|
1012
|
+
const pr = await tracker.prState(run.prUrl);
|
|
1013
|
+
if (pr === undefined) continue;
|
|
1014
|
+
if (pr === "merged" || pr === "closed") {
|
|
1015
|
+
terminal = true;
|
|
1016
|
+
} else {
|
|
1017
|
+
const issue = await tracker.issueState(run.issue);
|
|
1018
|
+
if (issue === undefined) continue;
|
|
1019
|
+
terminal = issue === "closed";
|
|
1020
|
+
}
|
|
1021
|
+
} else {
|
|
1022
|
+
const issue = await tracker.issueState(run.issue);
|
|
1023
|
+
if (issue === undefined) continue;
|
|
1024
|
+
terminal = issue === "closed";
|
|
1025
|
+
}
|
|
1026
|
+
} catch (err) {
|
|
1027
|
+
log(`#${run.issue} retained cleanup deferred: tracker state failed (${errText(err)})`);
|
|
1028
|
+
continue;
|
|
1029
|
+
}
|
|
1030
|
+
if (!terminal) continue;
|
|
1031
|
+
|
|
1032
|
+
const repo = Object.values(project.routing.repos).find((candidate) => candidate.name === run.repo);
|
|
1033
|
+
if (repo === undefined) {
|
|
1034
|
+
log(`#${run.issue} retained cleanup deferred: repo ${run.repo} is no longer configured`);
|
|
1035
|
+
continue;
|
|
1036
|
+
}
|
|
1037
|
+
|
|
1038
|
+
const outcome = await cleanup(mirrorPathFor(repo, project.mirrorRoot), run.worktree, run.branch);
|
|
1039
|
+
if (outcome.kind === "removed") {
|
|
1040
|
+
store.updateRun(run.id, { worktree: "" });
|
|
1041
|
+
log(`#${run.issue} retained worktree reaped: ${run.worktree} (${run.branch})`);
|
|
1042
|
+
} else {
|
|
1043
|
+
log(`#${run.issue} retained worktree kept (${outcome.reason}): ${outcome.detail}`);
|
|
1044
|
+
}
|
|
812
1045
|
}
|
|
813
1046
|
}
|
|
814
1047
|
|
|
@@ -820,9 +1053,52 @@ export interface Admission {
|
|
|
820
1053
|
attempt: number;
|
|
821
1054
|
}
|
|
822
1055
|
|
|
1056
|
+
export interface AdmissionHold {
|
|
1057
|
+
issue: number;
|
|
1058
|
+
reason: AdmissionHoldReason;
|
|
1059
|
+
}
|
|
1060
|
+
|
|
1061
|
+
export interface AdmissionPass {
|
|
1062
|
+
admitted: Admission[];
|
|
1063
|
+
holds: AdmissionHold[];
|
|
1064
|
+
}
|
|
1065
|
+
|
|
1066
|
+
const HOLD_SAMPLE_SIZE = 5;
|
|
1067
|
+
const DEGRADED_HOLDS: ReadonlySet<AdmissionHoldReason> = new Set([
|
|
1068
|
+
"parent-lookup-error",
|
|
1069
|
+
"open-pr-lookup-error",
|
|
1070
|
+
]);
|
|
1071
|
+
|
|
1072
|
+
/** Groups transient decisions into the bounded record exposed by status. */
|
|
1073
|
+
export function summarizeDispatch(
|
|
1074
|
+
ready: number,
|
|
1075
|
+
routed: number,
|
|
1076
|
+
admitted: number,
|
|
1077
|
+
holds: readonly AdmissionHold[],
|
|
1078
|
+
completedAt = Date.now(),
|
|
1079
|
+
): DispatchSummary {
|
|
1080
|
+
const groups = new Map<AdmissionHoldReason, { count: number; issues: number[] }>();
|
|
1081
|
+
for (const hold of holds) {
|
|
1082
|
+
const group = groups.get(hold.reason) ?? { count: 0, issues: [] };
|
|
1083
|
+
group.count += 1;
|
|
1084
|
+
if (group.issues.length < HOLD_SAMPLE_SIZE) group.issues.push(hold.issue);
|
|
1085
|
+
groups.set(hold.reason, group);
|
|
1086
|
+
}
|
|
1087
|
+
return {
|
|
1088
|
+
completedAt,
|
|
1089
|
+
ready,
|
|
1090
|
+
routed,
|
|
1091
|
+
admitted,
|
|
1092
|
+
degraded: holds.some((hold) => DEGRADED_HOLDS.has(hold.reason)),
|
|
1093
|
+
holds: [...groups]
|
|
1094
|
+
.sort(([a], [b]) => a.localeCompare(b))
|
|
1095
|
+
.map(([reason, group]) => ({ reason, ...group })),
|
|
1096
|
+
};
|
|
1097
|
+
}
|
|
1098
|
+
|
|
823
1099
|
/**
|
|
824
1100
|
* Which routed candidates get a worker this tick — in queue order, never more
|
|
825
|
-
* than `slots` of them.
|
|
1101
|
+
* than `slots` of them. Every non-admission receives a stable reason code.
|
|
826
1102
|
*
|
|
827
1103
|
* Exported so the admission rules can be pinned without spawning a worker.
|
|
828
1104
|
* Every one of them exists because of a live incident, and each guards a
|
|
@@ -837,10 +1113,14 @@ export async function admitCandidates(
|
|
|
837
1113
|
d: Pick<Deps, "project" | "caps" | "tracker" | "store" | "escalate">,
|
|
838
1114
|
routed: Routed[],
|
|
839
1115
|
slots: number,
|
|
840
|
-
): Promise<
|
|
1116
|
+
): Promise<AdmissionPass> {
|
|
841
1117
|
const { project, caps, tracker, store } = d;
|
|
842
1118
|
const busyIssues = store.activeRuns(project.name).map((r) => r.issue);
|
|
843
1119
|
const busy = new Set(busyIssues);
|
|
1120
|
+
const holds: AdmissionHold[] = [];
|
|
1121
|
+
const hold = (issue: number, reason: AdmissionHoldReason): void => {
|
|
1122
|
+
holds.push({ issue, reason });
|
|
1123
|
+
};
|
|
844
1124
|
|
|
845
1125
|
// parent -> blocking issue. Seeded from active runs (including pushed-green),
|
|
846
1126
|
// then extended by candidates admitted earlier in this same pass so two
|
|
@@ -870,43 +1150,69 @@ export async function admitCandidates(
|
|
|
870
1150
|
|
|
871
1151
|
const admitted: Admission[] = [];
|
|
872
1152
|
for (const r of routed) {
|
|
873
|
-
|
|
874
|
-
if (
|
|
1153
|
+
const issue = r.issue.number;
|
|
1154
|
+
if (admitted.length >= slots) {
|
|
1155
|
+
hold(issue, "capacity");
|
|
1156
|
+
continue;
|
|
1157
|
+
}
|
|
1158
|
+
if (busy.has(issue)) {
|
|
1159
|
+
hold(issue, "issue-active");
|
|
1160
|
+
continue;
|
|
1161
|
+
}
|
|
875
1162
|
|
|
876
|
-
const
|
|
877
|
-
|
|
1163
|
+
const priorRuns = store.attemptsFor(project.name, issue);
|
|
1164
|
+
const failures = store.failuresFor(project.name, issue);
|
|
1165
|
+
if (failures >= caps.maxAttemptsPerIssue) {
|
|
1166
|
+
hold(issue, "failed-attempts");
|
|
878
1167
|
await safeEscalate(d, {
|
|
879
1168
|
tier: 1,
|
|
880
1169
|
project: project.name,
|
|
881
|
-
issue
|
|
882
|
-
summary: `#${
|
|
1170
|
+
issue,
|
|
1171
|
+
summary: `#${issue} has used all ${caps.maxAttemptsPerIssue} failed attempts`,
|
|
883
1172
|
detail: [
|
|
884
1173
|
r.issue.title,
|
|
885
1174
|
r.issue.url,
|
|
886
|
-
"Another attempt almost always means the issue itself is underspecified.",
|
|
1175
|
+
"Another implementation attempt almost always means the issue itself is underspecified.",
|
|
887
1176
|
"Rewrite the acceptance criteria, or take it off the queue.",
|
|
888
1177
|
].join("\n"),
|
|
889
1178
|
});
|
|
890
1179
|
continue;
|
|
891
1180
|
}
|
|
892
1181
|
|
|
1182
|
+
const continuations = store.continuationsFor(project.name, issue);
|
|
1183
|
+
if (!hasContinuationBudget(continuations, caps.maxContinuationsPerIssue)) {
|
|
1184
|
+
hold(issue, "continuations");
|
|
1185
|
+
await safeEscalate(d, {
|
|
1186
|
+
tier: 1,
|
|
1187
|
+
project: project.name,
|
|
1188
|
+
issue,
|
|
1189
|
+
summary: `#${issue} exceeded its ${caps.maxContinuationsPerIssue}-continuation budget`,
|
|
1190
|
+
detail: [
|
|
1191
|
+
r.issue.title,
|
|
1192
|
+
r.issue.url,
|
|
1193
|
+
"Repeated cap kills, daemon orphans, or answered blocks need an operator to inspect progress.",
|
|
1194
|
+
].join("\n"),
|
|
1195
|
+
});
|
|
1196
|
+
continue;
|
|
1197
|
+
}
|
|
1198
|
+
|
|
893
1199
|
// Soft concurrency per epic: at most one in-flight child of a given parent.
|
|
894
1200
|
// No parent means today's concurrent admission. Cheap local filters already
|
|
895
1201
|
// ran; this sits before the open-PR API call so a held sibling frees the
|
|
896
1202
|
// slot for unrelated work without spending a closers query.
|
|
897
1203
|
let parent: number | undefined;
|
|
898
1204
|
try {
|
|
899
|
-
parent = await resolveParent(
|
|
1205
|
+
parent = await resolveParent(issue);
|
|
900
1206
|
} catch (err) {
|
|
901
|
-
|
|
1207
|
+
hold(issue, "parent-lookup-error");
|
|
1208
|
+
log(`#${issue} held: parent check failed (${errText(err)}) — retrying next tick`);
|
|
902
1209
|
continue;
|
|
903
1210
|
}
|
|
904
1211
|
if (parent !== undefined) {
|
|
905
1212
|
const blocker = occupiedParents.get(parent);
|
|
906
1213
|
if (blocker !== undefined) {
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
);
|
|
1214
|
+
hold(issue, "sibling-active");
|
|
1215
|
+
log(`#${issue} skipped: sibling #${blocker} in flight under epic #${parent}`);
|
|
910
1216
|
continue;
|
|
911
1217
|
}
|
|
912
1218
|
}
|
|
@@ -917,36 +1223,85 @@ export async function admitCandidates(
|
|
|
917
1223
|
// like fresh work, and a worker sent at it re-implements a finished PR. The
|
|
918
1224
|
// tracker is the only party that remembers, so it is asked. The cost is
|
|
919
1225
|
// bounded by free slots, not by queue depth: the call sits behind the two
|
|
920
|
-
// cheap local filters and
|
|
1226
|
+
// cheap local filters and candidates beyond capacity skip it.
|
|
921
1227
|
let closer: string | undefined;
|
|
922
1228
|
try {
|
|
923
|
-
closer = await tracker.openCloserFor(
|
|
1229
|
+
closer = await tracker.openCloserFor(issue);
|
|
924
1230
|
} catch (err) {
|
|
925
1231
|
// Fail closed, per candidate. An API error means "unknown whether
|
|
926
1232
|
// finished work exists", and admitting on unknown recreates precisely the
|
|
927
1233
|
// duplicate-work failure this guard exists to kill: the worst case of
|
|
928
1234
|
// holding is a five-minute delay, the worst case of admitting is a burned
|
|
929
1235
|
// attempt and a second PR on the same issue. Holding one candidate rather
|
|
930
|
-
// than aborting the loop
|
|
931
|
-
//
|
|
932
|
-
|
|
1236
|
+
// than aborting the loop keeps a transient GitHub failure from deadlocking
|
|
1237
|
+
// the whole dispatcher; the next tick retries by itself.
|
|
1238
|
+
hold(issue, "open-pr-lookup-error");
|
|
1239
|
+
log(`#${issue} held: open-PR check failed (${errText(err)}) — retrying next tick`);
|
|
933
1240
|
continue;
|
|
934
1241
|
}
|
|
935
1242
|
if (closer !== undefined) {
|
|
936
|
-
|
|
937
|
-
|
|
1243
|
+
const latest = store.latestRun(project.name, issue);
|
|
1244
|
+
const retainedContinuation =
|
|
1245
|
+
latest?.prUrl === closer &&
|
|
1246
|
+
(latest.state === "blocked" ||
|
|
1247
|
+
latest.state === "failed" ||
|
|
1248
|
+
latest.state === "killed" ||
|
|
1249
|
+
latest.state === "orphaned");
|
|
1250
|
+
if (!retainedContinuation) {
|
|
1251
|
+
hold(issue, "open-pr");
|
|
1252
|
+
log(`#${issue} skipped: open PR ${closer} already closes it`);
|
|
1253
|
+
continue;
|
|
1254
|
+
}
|
|
1255
|
+
log(`#${issue} continuing retained PR ${closer} from ${latest.state} run`);
|
|
938
1256
|
}
|
|
939
1257
|
|
|
940
|
-
admitted.push({ r, attempt:
|
|
941
|
-
if (parent !== undefined) occupiedParents.set(parent,
|
|
1258
|
+
admitted.push({ r, attempt: priorRuns + 1 });
|
|
1259
|
+
if (parent !== undefined) occupiedParents.set(parent, issue);
|
|
942
1260
|
}
|
|
943
1261
|
|
|
944
|
-
return admitted;
|
|
1262
|
+
return { admitted, holds };
|
|
1263
|
+
}
|
|
1264
|
+
|
|
1265
|
+
export interface WorkerPool {
|
|
1266
|
+
launch(work: Promise<void>): void;
|
|
1267
|
+
activeCount(): number;
|
|
1268
|
+
drain(): Promise<void>;
|
|
1269
|
+
}
|
|
1270
|
+
|
|
1271
|
+
/** Keeps background workers alive without making the five-minute tick await them. */
|
|
1272
|
+
export function createWorkerPool(): WorkerPool {
|
|
1273
|
+
const active = new Set<Promise<void>>();
|
|
1274
|
+
return {
|
|
1275
|
+
launch(work) {
|
|
1276
|
+
active.add(work);
|
|
1277
|
+
void work.then(
|
|
1278
|
+
() => active.delete(work),
|
|
1279
|
+
() => active.delete(work),
|
|
1280
|
+
);
|
|
1281
|
+
},
|
|
1282
|
+
activeCount: () => active.size,
|
|
1283
|
+
async drain() {
|
|
1284
|
+
await Promise.allSettled(active);
|
|
1285
|
+
},
|
|
1286
|
+
};
|
|
1287
|
+
}
|
|
1288
|
+
|
|
1289
|
+
/** `--once` awaits workers; the resident daemon registers them for shutdown. */
|
|
1290
|
+
export async function dispatchAdmissions(
|
|
1291
|
+
admitted: readonly Admission[],
|
|
1292
|
+
run: (admission: Admission) => Promise<void>,
|
|
1293
|
+
pool?: WorkerPool,
|
|
1294
|
+
): Promise<void> {
|
|
1295
|
+
if (pool !== undefined) {
|
|
1296
|
+
for (const admission of admitted) pool.launch(run(admission));
|
|
1297
|
+
return;
|
|
1298
|
+
}
|
|
1299
|
+
await Promise.allSettled(admitted.map(run));
|
|
945
1300
|
}
|
|
946
1301
|
|
|
947
1302
|
// ----------------------------------------------------------------------- a tick
|
|
948
1303
|
|
|
949
|
-
async function tick(d: Deps): Promise<void> {
|
|
1304
|
+
export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
950
1305
|
// Before the pause check, deliberately. This one is not about dispatch: the
|
|
951
1306
|
// orchestrator is a different process, and it can be wedged while this fleet
|
|
952
1307
|
// is paused — which is exactly the state the reference fleet was in when the
|
|
@@ -954,6 +1309,12 @@ async function tick(d: Deps): Promise<void> {
|
|
|
954
1309
|
// know their supervising session stopped reading its queue.
|
|
955
1310
|
await watchOrchestrator(d);
|
|
956
1311
|
|
|
1312
|
+
// Settlement is maintenance, not dispatch. Run it before every gate that can
|
|
1313
|
+
// stop claiming — pause, integrity, spend, and capacity — so status converges
|
|
1314
|
+
// while the fleet is parked or workers are still active. Resident workers run
|
|
1315
|
+
// through the pool without blocking this five-minute tick.
|
|
1316
|
+
await settlePushedGreen(d);
|
|
1317
|
+
|
|
957
1318
|
// A paused fleet claims nothing. Checked first so pausing takes effect on the
|
|
958
1319
|
// next tick without signalling the process.
|
|
959
1320
|
if (isPaused()) return;
|
|
@@ -1011,16 +1372,25 @@ async function tick(d: Deps): Promise<void> {
|
|
|
1011
1372
|
return;
|
|
1012
1373
|
}
|
|
1013
1374
|
|
|
1014
|
-
// Above admission on purpose: a row settled this tick frees its issue for
|
|
1015
|
-
// this same tick, so a merge and a re-queue no longer cost five minutes each.
|
|
1016
|
-
// Above the spend cap too, which returns early — settling is bookkeeping about
|
|
1017
|
-
// work already paid for, and a fleet that halts itself is exactly when an
|
|
1018
|
-
// operator reads `status` and needs it to be true.
|
|
1019
|
-
await settlePushedGreen(d);
|
|
1020
|
-
|
|
1021
1375
|
// route() filters the queue through isEligible() itself, so anything already
|
|
1022
1376
|
// carrying a state label is gone before it gets here.
|
|
1023
|
-
const
|
|
1377
|
+
const ready = await d.tracker.listReady();
|
|
1378
|
+
await cleanupRetainedRuns(
|
|
1379
|
+
d,
|
|
1380
|
+
new Set(ready.map((issue) => issue.number)),
|
|
1381
|
+
d.cleanup ?? { next: 0 },
|
|
1382
|
+
);
|
|
1383
|
+
const { routed, unroutable } = route(ready, project);
|
|
1384
|
+
const routingHolds: AdmissionHold[] = unroutable.map((u) => ({
|
|
1385
|
+
issue: u.issue.number,
|
|
1386
|
+
reason: `unroutable:${u.reason}`,
|
|
1387
|
+
}));
|
|
1388
|
+
const recordDispatch = (admitted: number, holds: readonly AdmissionHold[]): void => {
|
|
1389
|
+
store.recordDispatch(
|
|
1390
|
+
project.name,
|
|
1391
|
+
summarizeDispatch(ready.length, routed.length, admitted, holds),
|
|
1392
|
+
);
|
|
1393
|
+
};
|
|
1024
1394
|
|
|
1025
1395
|
// An issue nobody can route never reaches a worker: guessing the target repo
|
|
1026
1396
|
// is exactly the kind of improvisation this system exists to prevent. The
|
|
@@ -1061,6 +1431,10 @@ async function tick(d: Deps): Promise<void> {
|
|
|
1061
1431
|
"No further work will be claimed until `omp-conductor resume` (or /conductor resume).",
|
|
1062
1432
|
].join("\n"),
|
|
1063
1433
|
});
|
|
1434
|
+
recordDispatch(0, [
|
|
1435
|
+
...routingHolds,
|
|
1436
|
+
...routed.map((r) => ({ issue: r.issue.number, reason: "daily-spend-cap" as const })),
|
|
1437
|
+
]);
|
|
1064
1438
|
return;
|
|
1065
1439
|
}
|
|
1066
1440
|
|
|
@@ -1072,20 +1446,119 @@ async function tick(d: Deps): Promise<void> {
|
|
|
1072
1446
|
const slots = caps.maxConcurrentWorkers - live.length;
|
|
1073
1447
|
if (slots <= 0) {
|
|
1074
1448
|
log(`at capacity: ${live.length}/${caps.maxConcurrentWorkers} workers`);
|
|
1449
|
+
recordDispatch(0, [
|
|
1450
|
+
...routingHolds,
|
|
1451
|
+
...routed.map((r) => ({ issue: r.issue.number, reason: "capacity" as const })),
|
|
1452
|
+
]);
|
|
1075
1453
|
return;
|
|
1076
1454
|
}
|
|
1077
1455
|
|
|
1078
|
-
const
|
|
1456
|
+
const pass = await admitCandidates(d, routed, slots);
|
|
1457
|
+
recordDispatch(pass.admitted.length, [...routingHolds, ...pass.holds]);
|
|
1079
1458
|
|
|
1080
|
-
if (admitted.length === 0) return;
|
|
1459
|
+
if (pass.admitted.length === 0) return;
|
|
1081
1460
|
|
|
1082
|
-
log(`dispatching ${admitted.map((a) => `#${a.r.issue.number}`).join(" ")}`);
|
|
1083
|
-
|
|
1084
|
-
|
|
1461
|
+
log(`dispatching ${pass.admitted.map((a) => `#${a.r.issue.number}`).join(" ")}`);
|
|
1462
|
+
await dispatchAdmissions(
|
|
1463
|
+
pass.admitted,
|
|
1464
|
+
(a) => handleIssue(d, a.r, a.attempt),
|
|
1465
|
+
workers,
|
|
1466
|
+
);
|
|
1085
1467
|
}
|
|
1086
1468
|
|
|
1087
1469
|
// --------------------------------------------------------------- read-only views
|
|
1088
1470
|
|
|
1471
|
+
export interface DaemonHealthSnapshot {
|
|
1472
|
+
ok: true;
|
|
1473
|
+
paused: boolean;
|
|
1474
|
+
activeRuns: number;
|
|
1475
|
+
project: string;
|
|
1476
|
+
/** Resident set of this daemon; workers are in-process omp sessions. */
|
|
1477
|
+
rssBytes: number;
|
|
1478
|
+
dispatch?: DispatchSummary;
|
|
1479
|
+
codeGraph?: CodeGraphHealth;
|
|
1480
|
+
}
|
|
1481
|
+
|
|
1482
|
+
export function daemonHealthSnapshot(
|
|
1483
|
+
store: Store,
|
|
1484
|
+
project: string,
|
|
1485
|
+
paused = isPaused(),
|
|
1486
|
+
rssBytes = process.memoryUsage().rss,
|
|
1487
|
+
codeGraph?: CodeGraphHealth,
|
|
1488
|
+
): DaemonHealthSnapshot {
|
|
1489
|
+
const dispatch = store.latestDispatch(project);
|
|
1490
|
+
return {
|
|
1491
|
+
ok: true,
|
|
1492
|
+
paused,
|
|
1493
|
+
activeRuns: store.activeRuns(project).length,
|
|
1494
|
+
project,
|
|
1495
|
+
rssBytes,
|
|
1496
|
+
...(dispatch === undefined ? {} : { dispatch }),
|
|
1497
|
+
...(codeGraph?.configured === true ? { codeGraph } : {}),
|
|
1498
|
+
};
|
|
1499
|
+
}
|
|
1500
|
+
|
|
1501
|
+
export async function turnLimitResponse(
|
|
1502
|
+
req: Request,
|
|
1503
|
+
project: string,
|
|
1504
|
+
store: Pick<Store, "latestRun">,
|
|
1505
|
+
registry: TurnLimitRegistry,
|
|
1506
|
+
): Promise<Response | undefined> {
|
|
1507
|
+
const url = new URL(req.url);
|
|
1508
|
+
const match = /^\/runs\/(\d+)\/turn-limit$/.exec(url.pathname);
|
|
1509
|
+
if (req.method !== "PUT" || match === null) return undefined;
|
|
1510
|
+
if (!req.headers.get("content-type")?.startsWith("application/json")) {
|
|
1511
|
+
return Response.json({ error: "content-type must be application/json" }, { status: 415 });
|
|
1512
|
+
}
|
|
1513
|
+
|
|
1514
|
+
let body: unknown;
|
|
1515
|
+
try {
|
|
1516
|
+
body = await req.json();
|
|
1517
|
+
} catch {
|
|
1518
|
+
return Response.json({ error: "request body must be valid JSON" }, { status: 400 });
|
|
1519
|
+
}
|
|
1520
|
+
if (body === null || typeof body !== "object") {
|
|
1521
|
+
return Response.json({ error: "request body must be a JSON object" }, { status: 400 });
|
|
1522
|
+
}
|
|
1523
|
+
const requestedProject = Reflect.get(body, "project");
|
|
1524
|
+
if (typeof requestedProject !== "string" || requestedProject.length === 0) {
|
|
1525
|
+
return Response.json({ error: "project must be a non-empty string" }, { status: 400 });
|
|
1526
|
+
}
|
|
1527
|
+
if (requestedProject !== project) {
|
|
1528
|
+
return Response.json(
|
|
1529
|
+
{ error: `daemon serves project "${project}", not requested project "${requestedProject}"` },
|
|
1530
|
+
{ status: 409 },
|
|
1531
|
+
);
|
|
1532
|
+
}
|
|
1533
|
+
const maxTurns = Reflect.get(body, "maxTurns");
|
|
1534
|
+
if (!Number.isSafeInteger(maxTurns) || (maxTurns as number) < 1) {
|
|
1535
|
+
return Response.json({ error: "maxTurns must be a positive integer" }, { status: 400 });
|
|
1536
|
+
}
|
|
1537
|
+
|
|
1538
|
+
const issue = Number(match[1]);
|
|
1539
|
+
const outcome = registry.extend(project, issue, maxTurns as number);
|
|
1540
|
+
if (outcome.kind === "extended") return Response.json(outcome);
|
|
1541
|
+
if (outcome.kind === "not-increase") {
|
|
1542
|
+
return Response.json(
|
|
1543
|
+
{ error: `#${issue} already has a ${outcome.maxTurns}-turn ceiling` },
|
|
1544
|
+
{ status: 409 },
|
|
1545
|
+
);
|
|
1546
|
+
}
|
|
1547
|
+
|
|
1548
|
+
const latest = store.latestRun(project, issue);
|
|
1549
|
+
if (latest === undefined) {
|
|
1550
|
+
return Response.json({ error: `no run recorded for #${issue}` }, { status: 404 });
|
|
1551
|
+
}
|
|
1552
|
+
return Response.json(
|
|
1553
|
+
{
|
|
1554
|
+
error:
|
|
1555
|
+
`#${issue} has no live worker controller; its session already settled ` +
|
|
1556
|
+
`or belongs to another daemon (stored state: ${latest.state})`,
|
|
1557
|
+
},
|
|
1558
|
+
{ status: 409 },
|
|
1559
|
+
);
|
|
1560
|
+
}
|
|
1561
|
+
|
|
1089
1562
|
export interface StatusSnapshot {
|
|
1090
1563
|
project: string;
|
|
1091
1564
|
configPath: string;
|
|
@@ -1098,33 +1571,65 @@ export interface StatusSnapshot {
|
|
|
1098
1571
|
liveWorkers: number;
|
|
1099
1572
|
runsToday: number;
|
|
1100
1573
|
spendTodayUsd: number;
|
|
1574
|
+
dispatch?: DispatchSummary;
|
|
1101
1575
|
}
|
|
1102
1576
|
|
|
1103
|
-
/**
|
|
1104
|
-
*
|
|
1105
|
-
*
|
|
1577
|
+
/** Builds a status reading from an already-open store. Long-lived operator
|
|
1578
|
+
* surfaces use this path so a one-second refresh does not repeatedly open and
|
|
1579
|
+
* initialise SQLite connections. */
|
|
1580
|
+
export function statusSnapshotFromStore(p: ProjectConfig, caps: Caps, store: Store): StatusSnapshot {
|
|
1581
|
+
const since = startOfToday();
|
|
1582
|
+
const dispatch = store.latestDispatch(p.name);
|
|
1583
|
+
return {
|
|
1584
|
+
project: p.name,
|
|
1585
|
+
configPath: configPath(),
|
|
1586
|
+
stateDir: stateDir(),
|
|
1587
|
+
paused: isPaused(),
|
|
1588
|
+
caps,
|
|
1589
|
+
activeRuns: store.activeRuns(p.name),
|
|
1590
|
+
liveWorkers: store.liveRuns(p.name).length,
|
|
1591
|
+
runsToday: store.runsStartedSince(p.name, since),
|
|
1592
|
+
spendTodayUsd: store.spendSince(p.name, since),
|
|
1593
|
+
...(dispatch === undefined ? {} : { dispatch }),
|
|
1594
|
+
};
|
|
1595
|
+
}
|
|
1596
|
+
|
|
1597
|
+
/** Opens and closes its own store handle so one-shot CLI and plugin readers can
|
|
1598
|
+
* read status while a daemon in another process is writing (the store runs in
|
|
1599
|
+
* WAL mode for exactly this). */
|
|
1106
1600
|
export function statusSnapshot(project?: string): StatusSnapshot {
|
|
1107
1601
|
const cfg = loadConfig();
|
|
1108
1602
|
const p = findProject(cfg, project);
|
|
1109
1603
|
const store = openStore(dbPath());
|
|
1110
1604
|
try {
|
|
1111
|
-
|
|
1112
|
-
return {
|
|
1113
|
-
project: p.name,
|
|
1114
|
-
configPath: configPath(),
|
|
1115
|
-
stateDir: stateDir(),
|
|
1116
|
-
paused: isPaused(),
|
|
1117
|
-
caps: resolveCaps(p, cfg.defaults),
|
|
1118
|
-
activeRuns: store.activeRuns(p.name),
|
|
1119
|
-
liveWorkers: store.liveRuns(p.name).length,
|
|
1120
|
-
runsToday: store.runsStartedSince(p.name, since),
|
|
1121
|
-
spendTodayUsd: store.spendSince(p.name, since),
|
|
1122
|
-
};
|
|
1605
|
+
return statusSnapshotFromStore(p, resolveCaps(p, cfg.defaults), store);
|
|
1123
1606
|
} finally {
|
|
1124
1607
|
store.close();
|
|
1125
1608
|
}
|
|
1126
1609
|
}
|
|
1127
1610
|
|
|
1611
|
+
export function formatDispatchSummary(summary?: DispatchSummary): string {
|
|
1612
|
+
if (summary === undefined) return "last dispatch (none recorded)";
|
|
1613
|
+
const lines = [
|
|
1614
|
+
`last dispatch ${new Date(summary.completedAt).toISOString()}${summary.degraded ? " DEGRADED" : ""}`,
|
|
1615
|
+
` candidates ${summary.ready} ready / ${summary.routed} routed`,
|
|
1616
|
+
` admitted ${summary.admitted}`,
|
|
1617
|
+
];
|
|
1618
|
+
if (summary.holds.length === 0) {
|
|
1619
|
+
lines.push(" held 0");
|
|
1620
|
+
} else {
|
|
1621
|
+
lines.push(" held");
|
|
1622
|
+
for (const hold of summary.holds) {
|
|
1623
|
+
const sample =
|
|
1624
|
+
hold.issues.length === 0
|
|
1625
|
+
? ""
|
|
1626
|
+
: ` (#${hold.issues.join(", #")}${hold.count > hold.issues.length ? ", …" : ""})`;
|
|
1627
|
+
lines.push(` ${hold.reason} ${hold.count}${sample}`);
|
|
1628
|
+
}
|
|
1629
|
+
}
|
|
1630
|
+
return lines.join("\n");
|
|
1631
|
+
}
|
|
1632
|
+
|
|
1128
1633
|
export function formatStatus(s: StatusSnapshot): string {
|
|
1129
1634
|
const lines = [
|
|
1130
1635
|
`project ${s.project}${s.paused ? " (PAUSED)" : ""}`,
|
|
@@ -1137,9 +1642,12 @@ export function formatStatus(s: StatusSnapshot): string {
|
|
|
1137
1642
|
s.caps.dailySpendUsd === null
|
|
1138
1643
|
? ` spend today $${s.spendTodayUsd.toFixed(2)} (no daily cap)`
|
|
1139
1644
|
: ` spend today $${s.spendTodayUsd.toFixed(2)} / $${s.caps.dailySpendUsd.toFixed(2)}`,
|
|
1140
|
-
` worker
|
|
1645
|
+
` new worker turns ${s.caps.workerMaxTurns}`,
|
|
1141
1646
|
` worker wall clock ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
|
|
1142
|
-
` attempts
|
|
1647
|
+
` failed attempts ${s.caps.maxAttemptsPerIssue}`,
|
|
1648
|
+
` continuations ${s.caps.maxContinuationsPerIssue}`,
|
|
1649
|
+
"",
|
|
1650
|
+
formatDispatchSummary(s.dispatch),
|
|
1143
1651
|
"",
|
|
1144
1652
|
];
|
|
1145
1653
|
if (s.activeRuns.length === 0) {
|
|
@@ -1149,7 +1657,7 @@ export function formatStatus(s: StatusSnapshot): string {
|
|
|
1149
1657
|
for (const r of s.activeRuns) {
|
|
1150
1658
|
lines.push(
|
|
1151
1659
|
` #${r.issue} ${r.repo} ${r.state} attempt ${r.attempt} ` +
|
|
1152
|
-
`${r.turns} turns $${r.spendUsd.toFixed(2)} ${r.branch}` +
|
|
1660
|
+
`${r.turns}/${r.maxTurns} turns $${r.spendUsd.toFixed(2)} ${r.branch}` +
|
|
1153
1661
|
(r.prUrl ? ` ${r.prUrl}` : ""),
|
|
1154
1662
|
);
|
|
1155
1663
|
}
|
|
@@ -1180,14 +1688,15 @@ export interface QueuePreview {
|
|
|
1180
1688
|
* label, run row or worktree. This is what makes `/conductor setup` honest: the
|
|
1181
1689
|
* dry run is the same routing code the loop uses, not a description of it.
|
|
1182
1690
|
*/
|
|
1183
|
-
export async function
|
|
1184
|
-
|
|
1185
|
-
|
|
1691
|
+
export async function previewProject(
|
|
1692
|
+
p: ProjectConfig,
|
|
1693
|
+
path: string = configPath(),
|
|
1694
|
+
): Promise<QueuePreview> {
|
|
1186
1695
|
const { routed, unroutable } = route(await makeTracker(p).listReady(), p);
|
|
1187
1696
|
const states = Object.values(p.stateLabels).join(", ");
|
|
1188
1697
|
return {
|
|
1189
1698
|
project: p.name,
|
|
1190
|
-
configPath:
|
|
1699
|
+
configPath: path,
|
|
1191
1700
|
queueDescription:
|
|
1192
1701
|
`open issues in ${p.tracker.repo} labelled "${p.queueLabel}", ` +
|
|
1193
1702
|
`minus anything already labelled ${states}, ` +
|
|
@@ -1208,14 +1717,20 @@ export async function previewQueue(project?: string): Promise<QueuePreview> {
|
|
|
1208
1717
|
};
|
|
1209
1718
|
}
|
|
1210
1719
|
|
|
1720
|
+
export async function previewQueue(project?: string): Promise<QueuePreview> {
|
|
1721
|
+
const cfg = loadConfig();
|
|
1722
|
+
return previewProject(findProject(cfg, project));
|
|
1723
|
+
}
|
|
1724
|
+
|
|
1211
1725
|
/**
|
|
1212
|
-
*
|
|
1213
|
-
*
|
|
1214
|
-
*
|
|
1726
|
+
* Creates the state store and holds dispatch while setup verifies the host.
|
|
1727
|
+
*
|
|
1728
|
+
* Setup calls this immediately after consent. Every later setup error therefore
|
|
1729
|
+
* leaves the fleet paused instead of exposing a partially written runtime.
|
|
1215
1730
|
*/
|
|
1216
|
-
export function
|
|
1731
|
+
export function prepareConductor(): void {
|
|
1217
1732
|
openStore(dbPath()).close();
|
|
1218
|
-
setPaused(
|
|
1733
|
+
setPaused(true);
|
|
1219
1734
|
}
|
|
1220
1735
|
|
|
1221
1736
|
/**
|
|
@@ -1317,6 +1832,10 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
1317
1832
|
"a time, freshness-checked against the base branch, per the Releases section of your POLICY.md."
|
|
1318
1833
|
: "You never edit product code, push a branch, or merge a PR — a worker session edits and pushes, and a " +
|
|
1319
1834
|
"human merges.",
|
|
1835
|
+
`Release tool gate: releasePolicy=${resolveReleasePolicy(project)}. ` +
|
|
1836
|
+
(resolveReleasePolicy(project) === "none"
|
|
1837
|
+
? "Release and deploy tool calls are mechanically blocked."
|
|
1838
|
+
: "Release and deploy tool calls are permitted only by the operator brief."),
|
|
1320
1839
|
"Handle each escalation below before the next one.",
|
|
1321
1840
|
].join("\n");
|
|
1322
1841
|
|
|
@@ -1334,7 +1853,12 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
1334
1853
|
log("orchestrator: external — tier-1 escalations post as issue comments for the external session's drain duty");
|
|
1335
1854
|
} else {
|
|
1336
1855
|
try {
|
|
1337
|
-
orchestrator = await startOrchestrator({
|
|
1856
|
+
orchestrator = await startOrchestrator({
|
|
1857
|
+
cwd: stateDir(),
|
|
1858
|
+
brief,
|
|
1859
|
+
releasePolicy: resolveReleasePolicy(project),
|
|
1860
|
+
onReleaseBlocked: (shape) => recordReleaseBlock(project.name, "orchestrator", shape),
|
|
1861
|
+
});
|
|
1338
1862
|
const transcript = orchestrator.sessionFile();
|
|
1339
1863
|
log(`orchestrator session ready${transcript === undefined ? "" : ` · ${transcript}`}`);
|
|
1340
1864
|
} catch (err) {
|
|
@@ -1348,16 +1872,21 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
1348
1872
|
}
|
|
1349
1873
|
|
|
1350
1874
|
const escalator = createEscalator(project, tracker, store, orchestrator);
|
|
1875
|
+
const turnLimits = createTurnLimitRegistry((runId, maxTurns) => {
|
|
1876
|
+
store.updateRun(runId, { maxTurns });
|
|
1877
|
+
});
|
|
1351
1878
|
const d: Deps = {
|
|
1352
1879
|
project,
|
|
1353
1880
|
caps,
|
|
1354
1881
|
tracker,
|
|
1355
1882
|
store,
|
|
1356
1883
|
escalate: (e) => escalator.escalate(e),
|
|
1884
|
+
turnLimits,
|
|
1357
1885
|
integrity,
|
|
1358
1886
|
// Fresh per daemon run, like the integrity gate: a restart is entitled to
|
|
1359
1887
|
// page again about a stall that is still on disk.
|
|
1360
1888
|
stall: { paged: false },
|
|
1889
|
+
cleanup: { next: 0 },
|
|
1361
1890
|
};
|
|
1362
1891
|
|
|
1363
1892
|
if (o.once) {
|
|
@@ -1370,33 +1899,49 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
1370
1899
|
return;
|
|
1371
1900
|
}
|
|
1372
1901
|
|
|
1902
|
+
// Graphs are optional, so their bounded probes run beside dispatch and feed
|
|
1903
|
+
// a cache. /healthz remains an in-memory answer and never blocks liveness on
|
|
1904
|
+
// the indexer or systemd.
|
|
1905
|
+
let codeGraph = pendingCodeGraph(project);
|
|
1906
|
+
let graphProbe: Promise<void> | undefined;
|
|
1907
|
+
const refreshCodeGraph = (): void => {
|
|
1908
|
+
if (graphProbe !== undefined) return;
|
|
1909
|
+
graphProbe = probeCodeGraph(project)
|
|
1910
|
+
.then((health) => {
|
|
1911
|
+
codeGraph = health;
|
|
1912
|
+
})
|
|
1913
|
+
.catch(() => {
|
|
1914
|
+
log("code-graph health probe failed unexpectedly; retaining the previous bounded result");
|
|
1915
|
+
})
|
|
1916
|
+
.finally(() => {
|
|
1917
|
+
graphProbe = undefined;
|
|
1918
|
+
});
|
|
1919
|
+
};
|
|
1920
|
+
refreshCodeGraph();
|
|
1921
|
+
const graphTimer = setInterval(refreshCodeGraph, GRAPH_HEALTH_INTERVAL_MS);
|
|
1922
|
+
|
|
1923
|
+
const workers = createWorkerPool();
|
|
1373
1924
|
let stopping = false;
|
|
1374
1925
|
let wake: (() => void) | undefined;
|
|
1375
1926
|
const stop = (): void => {
|
|
1376
1927
|
if (stopping) return;
|
|
1377
1928
|
stopping = true;
|
|
1378
|
-
log("shutting down after
|
|
1929
|
+
log("shutting down after active workers finish");
|
|
1379
1930
|
wake?.();
|
|
1380
1931
|
};
|
|
1381
1932
|
process.on("SIGINT", stop);
|
|
1382
1933
|
process.on("SIGTERM", stop);
|
|
1383
1934
|
|
|
1384
1935
|
const server = Bun.serve({
|
|
1936
|
+
hostname: "127.0.0.1",
|
|
1385
1937
|
port: o.port ?? DEFAULT_PORT,
|
|
1386
|
-
fetch(req) {
|
|
1938
|
+
async fetch(req) {
|
|
1387
1939
|
const url = new URL(req.url);
|
|
1388
1940
|
if (req.method === "GET" && url.pathname === "/healthz") {
|
|
1389
|
-
return Response.json(
|
|
1390
|
-
ok: true,
|
|
1391
|
-
paused: isPaused(),
|
|
1392
|
-
activeRuns: store.activeRuns(project.name).length,
|
|
1393
|
-
project: project.name,
|
|
1394
|
-
// Resident set of *this* process: workers are in-process omp sessions,
|
|
1395
|
-
// so the unit's Memory peak is this number, not a separate worker pid.
|
|
1396
|
-
rssBytes: process.memoryUsage().rss,
|
|
1397
|
-
});
|
|
1941
|
+
return Response.json(daemonHealthSnapshot(store, project.name, undefined, undefined, codeGraph));
|
|
1398
1942
|
}
|
|
1399
|
-
|
|
1943
|
+
const control = await turnLimitResponse(req, project.name, store, turnLimits);
|
|
1944
|
+
return control ?? new Response("not found\n", { status: 404 });
|
|
1400
1945
|
},
|
|
1401
1946
|
});
|
|
1402
1947
|
log(`serving /healthz on :${server.port}, project ${project.name}`);
|
|
@@ -1404,7 +1949,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
1404
1949
|
try {
|
|
1405
1950
|
while (!stopping) {
|
|
1406
1951
|
try {
|
|
1407
|
-
await tick(d);
|
|
1952
|
+
await tick(d, workers);
|
|
1408
1953
|
} catch (err) {
|
|
1409
1954
|
// A tick that blows up outside an issue (the tracker is down, say) must
|
|
1410
1955
|
// not end the daemon; the next one will retry.
|
|
@@ -1423,6 +1968,8 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
1423
1968
|
} finally {
|
|
1424
1969
|
process.off("SIGINT", stop);
|
|
1425
1970
|
process.off("SIGTERM", stop);
|
|
1971
|
+
clearInterval(graphTimer);
|
|
1972
|
+
await workers.drain();
|
|
1426
1973
|
await server.stop(true);
|
|
1427
1974
|
// Before the store closes: a queued injection that rejects on the way out
|
|
1428
1975
|
// falls back to an issue comment, and that path writes the dedup marker.
|