omp-conductor 0.3.18 → 0.3.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/daemon.ts CHANGED
@@ -10,34 +10,41 @@
10
10
  import { createHash } from "node:crypto";
11
11
  import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
12
12
  import { dirname, join, relative } from "node:path";
13
- import { configPath, findProject, loadConfig, resolveCaps, stateDir } from "./config.ts";
13
+ import { configPath, findProject, loadConfig, resolveCaps, resolveReleasePolicy, stateDir } from "./config.ts";
14
14
  import { createEscalator, escalationIssueRef } from "./escalate.ts";
15
+ import { pendingCodeGraph, probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
15
16
  import { graphHint } from "./graph.ts";
16
17
  import { livingDaemon } from "./lifecycle.ts";
17
18
  import { STALL_MARKER_FILE } from "./orchestrator-tick.ts";
18
19
  import { startOrchestrator } from "./orchestrator.ts";
19
20
  import type { OrchestratorHandle } from "./orchestrator.ts";
21
+ import { recordReleaseBlock } from "./release-policy.ts";
20
22
  import { branchName, route } from "./routing.ts";
21
23
  import type { Routed, UnroutableReason } from "./routing.ts";
22
- import { openStore } from "./store.ts";
24
+ import { dbPath, openStore } from "./store.ts";
23
25
  import { makeTracker } from "./tracker/github.ts";
24
26
  import type {
27
+ AdmissionHoldReason,
25
28
  Caps,
29
+ DispatchSummary,
26
30
  Escalation,
27
31
  PrState,
28
32
  ProjectConfig,
29
33
  ReadyIssue,
30
34
  RepoTarget,
31
35
  RunRecord,
36
+ RunState,
32
37
  Store,
33
38
  Tracker,
34
39
  } from "./types.ts";
35
- import { type KilledBy, renderBrief, runWorker } from "./worker.ts";
40
+ import { type KilledBy, type WorkerResult, renderBrief, runWorker } from "./worker.ts";
36
41
  import {
37
42
  addWorktree,
43
+ cleanupRetainedWorktree,
38
44
  mirrorPathFor,
39
45
  removeWorktree,
40
46
  salvageWip,
47
+ type RetainedWorktreeCleanup,
41
48
  type SalvageOutcome,
42
49
  worktreePathFor,
43
50
  } from "./worktree.ts";
@@ -45,6 +52,7 @@ import {
45
52
  /** Long enough that the tracker is not polled raw, short enough that a human
46
53
  * who labels an issue sees it picked up within a coffee break. */
47
54
  const TICK_INTERVAL_MS = 5 * 60_000;
55
+ const GRAPH_HEALTH_INTERVAL_MS = 60_000;
48
56
  const DEFAULT_PORT = 8787;
49
57
  const BRIEF_TEMPLATE_PATH = join(import.meta.dir, "briefs", "worker.md");
50
58
 
@@ -72,16 +80,14 @@ interface Deps {
72
80
  tracker: Tracker;
73
81
  store: Store;
74
82
  escalate(e: Escalation): Promise<void>;
83
+ turnLimits: TurnLimitRegistry;
75
84
  integrity: IntegrityGate;
76
85
  stall: StallGate;
86
+ cleanup?: RetainedCleanupCursor;
77
87
  }
78
88
 
79
89
  // ---------------------------------------------------------------- paths & pause
80
90
 
81
- /** Single database for every project; the store partitions by project name. */
82
- export function dbPath(): string {
83
- return join(stateDir(), "conductor.db");
84
- }
85
91
 
86
92
  // ------------------------------------------------------- orchestrator liveness
87
93
 
@@ -501,6 +507,85 @@ export async function buildBrief(
501
507
 
502
508
  // ------------------------------------------------------------------- one issue
503
509
 
510
+ /** `stops` are the operational ends that each require one resume. */
511
+ export function hasContinuationBudget(stops: number, maxContinuations: number): boolean {
512
+ return stops <= maxContinuations;
513
+ }
514
+
515
+ export type ExtendTurnLimitResult =
516
+ | { kind: "extended"; runId: string; maxTurns: number }
517
+ | { kind: "not-increase"; runId: string; maxTurns: number }
518
+ | { kind: "not-active" };
519
+
520
+ export interface TurnLimitController {
521
+ maxTurns(): number;
522
+ close(): void;
523
+ }
524
+
525
+ export interface TurnLimitRegistry {
526
+ open(project: string, issue: number, runId: string, maxTurns: number): TurnLimitController;
527
+ extend(project: string, issue: number, maxTurns: number): ExtendTurnLimitResult;
528
+ }
529
+
530
+ /**
531
+ * Authoritative live-run turn controls. Persistence happens synchronously
532
+ * before the in-memory ceiling changes, so no turn event can interleave.
533
+ */
534
+ export function createTurnLimitRegistry(
535
+ persist: (runId: string, maxTurns: number) => void,
536
+ ): TurnLimitRegistry {
537
+ const active = new Map<string, { runId: string; maxTurns: number }>();
538
+ const key = (project: string, issue: number): string => `${project}\0${issue}`;
539
+ return {
540
+ open(project, issue, runId, maxTurns) {
541
+ const k = key(project, issue);
542
+ if (active.has(k)) throw new Error(`#${issue} already has a live turn controller`);
543
+ const entry = { runId, maxTurns };
544
+ active.set(k, entry);
545
+ return {
546
+ maxTurns: () => entry.maxTurns,
547
+ close: () => {
548
+ if (active.get(k) === entry) active.delete(k);
549
+ },
550
+ };
551
+ },
552
+ extend(project, issue, maxTurns) {
553
+ if (!Number.isSafeInteger(maxTurns) || maxTurns < 1) {
554
+ throw new RangeError(`turn ceiling must be a positive integer, got ${maxTurns}`);
555
+ }
556
+ const entry = active.get(key(project, issue));
557
+ if (entry === undefined) return { kind: "not-active" };
558
+ if (maxTurns <= entry.maxTurns) {
559
+ return { kind: "not-increase", runId: entry.runId, maxTurns: entry.maxTurns };
560
+ }
561
+ persist(entry.runId, maxTurns);
562
+ entry.maxTurns = maxTurns;
563
+ return { kind: "extended", runId: entry.runId, maxTurns };
564
+ },
565
+ };
566
+ }
567
+
568
+ export async function verifyPushedGreenClaim(
569
+ tracker: Pick<Tracker, "verifyPr">,
570
+ claim: Pick<WorkerResult, "prUrl" | "headSha">,
571
+ ): Promise<{
572
+ state: "pushed-green" | "pushed-pending" | "failed";
573
+ reason?: string;
574
+ }> {
575
+ if (claim.prUrl === undefined || claim.headSha === undefined) {
576
+ return { state: "failed", reason: "Worker did not report a PR URL and observed head SHA" };
577
+ }
578
+ const verification = await tracker.verifyPr(claim.prUrl, claim.headSha);
579
+ if (verification === undefined) {
580
+ return { state: "pushed-pending", reason: "GitHub PR verification unavailable; retrying" };
581
+ }
582
+ if (verification.status === "green") return { state: "pushed-green" };
583
+ return {
584
+ state: verification.status === "pending" ? "pushed-pending" : "failed",
585
+ reason: verification.reason,
586
+ };
587
+ }
588
+
504
589
  /**
505
590
  * One attempt at one issue, from claim to terminal state. Everything is inside
506
591
  * a single try/catch so that a bad issue costs its own run and nothing else.
@@ -517,6 +602,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
517
602
  // mid-dispatch is one of the non-graceful ends whose uncommitted work has to
518
603
  // be salvaged too, and it is the path least likely to have committed first.
519
604
  let worktreePath: string | undefined;
605
+ let turnLimit: TurnLimitController | undefined;
520
606
 
521
607
  try {
522
608
  // Claim on the tracker FIRST, before any local work. The label — not the
@@ -536,9 +622,11 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
536
622
  attempt,
537
623
  turns: 0,
538
624
  spendUsd: 0,
625
+ maxTurns: caps.workerMaxTurns,
539
626
  startedAt: Date.now(),
540
627
  });
541
628
  const runId = run.id;
629
+ turnLimit = d.turnLimits.open(project.name, issue, runId, caps.workerMaxTurns);
542
630
 
543
631
  // A run's tree is <workspaceRoot>/<issue> and addWorktree refuses to reuse
544
632
  // an existing path, so a retry — or a tree kept from a failed attempt — has
@@ -569,22 +657,38 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
569
657
  (provisioned.reattached ? " (continuation: reattached existing branch)" : ""),
570
658
  );
571
659
 
572
- const result = await runWorker({
573
- brief: await buildBrief(project, r, branch, worktreePath, {
574
- continuation: provisioned.reattached,
575
- defaultBranch: r.repo.defaultBranch,
576
- }),
577
- cwd: worktreePath,
578
- caps,
579
- sessionDir,
580
- ...(project.workerModel === undefined ? {} : { model: project.workerModel }),
581
- onTurn: (n) => store.updateRun(runId, { turns: n }),
582
- // Recorded the moment the session opens its transcript, not when the run
583
- // ends: `omp-conductor tail` resolves an issue to a file through this row,
584
- // and a path written at completion is a path nobody can follow live. The
585
- // completion-time update below writes the same value again, harmlessly.
586
- onSessionFile: (f) => store.updateRun(runId, { sessionFile: f }),
587
- });
660
+ let result: WorkerResult;
661
+ try {
662
+ result = await runWorker({
663
+ brief: await buildBrief(project, r, branch, worktreePath, {
664
+ continuation: provisioned.reattached,
665
+ defaultBranch: r.repo.defaultBranch,
666
+ }),
667
+ cwd: worktreePath,
668
+ caps,
669
+ maxTurns: () => turnLimit?.maxTurns() ?? caps.workerMaxTurns,
670
+ sessionDir,
671
+ ...(project.workerModel === undefined ? {} : { model: project.workerModel }),
672
+ releasePolicy: resolveReleasePolicy(project),
673
+ onReleaseBlocked: (shape) => recordReleaseBlock(project.name, "worker", shape),
674
+ onTurn: (n) => store.updateRun(runId, { turns: n }),
675
+ onSpend: (usd) => store.updateRun(runId, { spendUsd: usd }),
676
+ onKilled: () => {
677
+ turnLimit?.close();
678
+ turnLimit = undefined;
679
+ },
680
+ // Recorded the moment the session opens its transcript, not when the run
681
+ // ends: `omp-conductor tail` resolves an issue to a file through this row,
682
+ // and a path written at completion is a path nobody can follow live. The
683
+ // completion-time update below writes the same value again, harmlessly.
684
+ onSessionFile: (f) => store.updateRun(runId, { sessionFile: f }),
685
+ });
686
+ } finally {
687
+ // This is the authoritative settlement edge for `extend`: close before
688
+ // PR verification or terminal row writes can leave stale `running` state.
689
+ turnLimit?.close();
690
+ turnLimit = undefined;
691
+ }
588
692
 
589
693
  // A configured model the harness could not honour means this run was done by
590
694
  // a different model than the operator chose. Logged per run, because it is
@@ -593,16 +697,26 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
593
697
  log(`#${issue} model fallback: ${result.modelFallbackMessage}`);
594
698
  }
595
699
 
700
+ const verified: { state: RunState; reason?: string } =
701
+ result.state === "pushed-green"
702
+ ? await verifyPushedGreenClaim(tracker, result)
703
+ : { state: result.state };
704
+ const state = verified.state;
705
+ const finalReport =
706
+ verified.reason === undefined ? result.report : `${verified.reason}\n\n${result.report}`;
707
+
596
708
  store.updateRun(runId, {
597
- state: result.state,
709
+ state,
598
710
  endedAt: Date.now(),
599
711
  turns: result.turns,
600
712
  spendUsd: result.spendUsd,
601
713
  prUrl: result.prUrl,
714
+ headSha: result.headSha,
602
715
  sessionFile: result.sessionFile,
716
+ ...(verified.reason === undefined ? {} : { lastError: verified.reason }),
603
717
  });
604
718
 
605
- if (result.state === "blocked") {
719
+ if (state === "blocked") {
606
720
  await swapLabel(tracker, issue, inProgress, project.stateLabels.blocked);
607
721
  await safeEscalate(d, {
608
722
  tier: 1,
@@ -612,20 +726,22 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
612
726
  summary: `#${issue} is blocked on attempt ${attempt} and needs a decision`,
613
727
  detail: [`${r.issue.title}`, r.issue.url, "", result.report].join("\n"),
614
728
  });
615
- } else if (result.state === "failed" || result.state === "killed") {
616
- // Turns-cap with attempts left: salvage, put the issue back on the queue,
617
- // and skip the failed label so the next tick reclaims as a continuation
618
- // instead of burning a human triage cycle (#50 / #21).
729
+ } else if (state === "failed" || state === "killed") {
730
+ // A turns cap consumes the independent continuation budget, not an
731
+ // implementation-failure attempt. The row is already `killed`, so this
732
+ // count includes the segment that just ended.
733
+ const continuation = store.continuationsFor(project.name, issue);
619
734
  const continueTurns =
620
- result.killedBy === "turns" && attempt < caps.maxAttemptsPerIssue;
735
+ result.killedBy === "turns" &&
736
+ hasContinuationBudget(continuation, caps.maxContinuationsPerIssue);
621
737
  const salvaged = await salvage(issue, attempt, endedBy(result.killedBy), worktreePath);
622
738
 
623
739
  if (continueTurns) {
624
740
  await tracker.removeLabel(issue, inProgress);
625
741
  await tracker.addLabel(issue, project.queueLabel);
626
742
  log(
627
- `#${issue} turns-cap on attempt ${attempt}/${caps.maxAttemptsPerIssue} ` +
628
- `salvaged and re-queued for continuation`,
743
+ `#${issue} turns-cap on run ${attempt}, continuation ` +
744
+ `${continuation}/${caps.maxContinuationsPerIssue} — salvaged and re-queued`,
629
745
  );
630
746
  await safeEscalate(d, {
631
747
  tier: 1,
@@ -664,24 +780,27 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
664
780
  ...salvaged,
665
781
  `Session: ${result.sessionFile ?? "(no transcript)"}`,
666
782
  "",
667
- result.report,
783
+ finalReport,
668
784
  ].join("\n"),
669
785
  });
670
786
  }
671
787
  } else {
672
- // pushed-green: the PR belongs to a human now. The in-progress label
673
- // stays on until the merge closes the issue, which is also what keeps
674
- // the next tick from re-claiming it.
675
- log(`#${issue} ${result.state}${result.prUrl ? ` ${result.prUrl}` : ""}`);
788
+ // A verified or still-pending PR keeps the in-progress label until its
789
+ // checks or merge settle, preventing another worker from duplicating it.
790
+ log(`#${issue} ${state}${result.prUrl ? ` ${result.prUrl}` : ""}`);
676
791
  }
677
792
 
678
793
  // A failed or killed tree is evidence — keep it. Anything else is just
679
794
  // disk, and the mirror means re-provisioning is cheap. (The kept tree is
680
795
  // wiped by the next attempt, not left to accumulate forever.)
681
- if (result.state !== "failed" && result.state !== "killed") {
796
+ if (state !== "failed" && state !== "killed") {
682
797
  await removeWorktree(mirrorPath, worktreePath);
683
798
  }
684
799
  } catch (err) {
800
+ // Dispatch setup can fail after the controller opens but before runWorker's
801
+ // inner settlement guard exists. Latch it before any terminal write or await.
802
+ turnLimit?.close();
803
+ turnLimit = undefined;
685
804
  const detail = errText(err);
686
805
  log(`#${issue} errored: ${detail}`);
687
806
  if (run) {
@@ -713,6 +832,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
713
832
  });
714
833
  // The worktree, if one was created, is deliberately left in place: this is
715
834
  // a failure path, and whatever it still held is now a commit on the branch.
835
+ } finally {
836
+ turnLimit?.close();
716
837
  }
717
838
  }
718
839
 
@@ -782,12 +903,13 @@ export async function settlePushedGreen(
782
903
  // is live workers plus these, so the list is bounded by the worker cap plus
783
904
  // the number of PRs awaiting a merge — a handful, by construction. A fleet
784
905
  // where that is not a handful has a merge problem, not a dispatch one.
785
- const pending = store.activeRuns(project.name).filter((r) => r.state === "pushed-green");
906
+ const pending = store
907
+ .activeRuns(project.name)
908
+ .filter((r) => r.state === "pushed-green" || r.state === "pushed-pending");
786
909
 
787
910
  for (const run of pending) {
788
- // Nothing to ask about. A green push means a PR, so this row should not
789
- // exist; if one ever does, it must not buy a `gh` call every five minutes
790
- // forever to be told nothing.
911
+ // Nothing to ask about. A pushed result requires a PR, so a malformed row
912
+ // must not buy a `gh` call every five minutes forever.
791
913
  if (run.prUrl === undefined) continue;
792
914
 
793
915
  let pr: PrState | undefined;
@@ -803,12 +925,123 @@ export async function settlePushedGreen(
803
925
  }
804
926
 
805
927
  const settlement = settlementFor(pr, run.prUrl);
806
- if (settlement === undefined) continue;
928
+ if (settlement !== undefined) {
929
+ const patch: Partial<RunRecord> = { state: settlement.state, endedAt: Date.now() };
930
+ if (settlement.state === "failed") patch.lastError = settlement.reason;
931
+ store.updateRun(run.id, patch);
932
+ log(`#${run.issue} settled: ${settlement.reason}`);
933
+ continue;
934
+ }
935
+
936
+ if (run.state !== "pushed-pending" || pr !== "open" || run.headSha === undefined) continue;
937
+ let verification;
938
+ try {
939
+ verification = await tracker.verifyPr(run.prUrl, run.headSha);
940
+ } catch (err) {
941
+ log(`#${run.issue} checks not settled (${errText(err)}) — retrying next tick`);
942
+ continue;
943
+ }
944
+ if (verification === undefined) continue;
945
+ if (verification.status === "green") {
946
+ store.updateRun(run.id, { state: "pushed-green", lastError: undefined });
947
+ log(`#${run.issue} checks settled: ${verification.reason}`);
948
+ } else if (verification.status === "failed") {
949
+ store.updateRun(run.id, { state: "failed", lastError: verification.reason });
950
+ log(`#${run.issue} checks failed: ${verification.reason}`);
951
+ } else {
952
+ store.updateRun(run.id, { lastError: verification.reason });
953
+ }
954
+ }
955
+ }
956
+
957
+ const RETAINED_CLEANUP_BATCH = 10;
958
+
959
+ export interface RetainedCleanupCursor {
960
+ next: number;
961
+ }
962
+
963
+ type CleanupRetainedWorktree = (
964
+ mirrorPath: string,
965
+ worktreePath: string,
966
+ branch: string,
967
+ ) => Promise<RetainedWorktreeCleanup>;
968
+
969
+ /**
970
+ * Bounded, rotating cleanup for failure-path trees. Tracker state proves the
971
+ * run is terminal; local git state independently proves deletion cannot erase
972
+ * dirty or uniquely unpushed work.
973
+ */
974
+ export async function cleanupRetainedRuns(
975
+ d: Pick<Deps, "project" | "tracker" | "store">,
976
+ queuedIssues: ReadonlySet<number>,
977
+ cursor: RetainedCleanupCursor,
978
+ cleanup: CleanupRetainedWorktree = cleanupRetainedWorktree,
979
+ ): Promise<void> {
980
+ const { project, tracker, store } = d;
981
+ const candidates = store.retainedRuns(project.name);
982
+ if (candidates.length === 0) {
983
+ cursor.next = 0;
984
+ return;
985
+ }
986
+
987
+ const occupied = new Set(store.activeRuns(project.name).map((run) => run.issue));
988
+ const liveRepos = new Set(store.liveRuns(project.name).map((run) => run.repo));
989
+ const start = cursor.next % candidates.length;
990
+ const count = Math.min(RETAINED_CLEANUP_BATCH, candidates.length);
991
+ const batch = Array.from({ length: count }, (_, offset) => candidates[(start + offset) % candidates.length]!);
992
+ cursor.next = (start + count) % candidates.length;
993
+
994
+ for (const run of batch) {
995
+ if (occupied.has(run.issue) || queuedIssues.has(run.issue) || liveRepos.has(run.repo)) continue;
996
+ // Attempts reuse one physical path and deterministic branch. An older row
997
+ // cannot authorize deleting the newest failed attempt's evidence merely
998
+ // because its own PR resolved first.
999
+ const latest = store.latestRun(project.name, run.issue);
1000
+ if (
1001
+ latest !== undefined &&
1002
+ latest.id !== run.id &&
1003
+ latest.state !== "merged" &&
1004
+ latest.worktree !== ""
1005
+ ) {
1006
+ continue;
1007
+ }
807
1008
 
808
- const patch: Partial<RunRecord> = { state: settlement.state, endedAt: Date.now() };
809
- if (settlement.state === "failed") patch.lastError = settlement.reason;
810
- store.updateRun(run.id, patch);
811
- log(`#${run.issue} settled: ${settlement.reason}`);
1009
+ let terminal = false;
1010
+ try {
1011
+ if (run.prUrl !== undefined) {
1012
+ const pr = await tracker.prState(run.prUrl);
1013
+ if (pr === undefined) continue;
1014
+ if (pr === "merged" || pr === "closed") {
1015
+ terminal = true;
1016
+ } else {
1017
+ const issue = await tracker.issueState(run.issue);
1018
+ if (issue === undefined) continue;
1019
+ terminal = issue === "closed";
1020
+ }
1021
+ } else {
1022
+ const issue = await tracker.issueState(run.issue);
1023
+ if (issue === undefined) continue;
1024
+ terminal = issue === "closed";
1025
+ }
1026
+ } catch (err) {
1027
+ log(`#${run.issue} retained cleanup deferred: tracker state failed (${errText(err)})`);
1028
+ continue;
1029
+ }
1030
+ if (!terminal) continue;
1031
+
1032
+ const repo = Object.values(project.routing.repos).find((candidate) => candidate.name === run.repo);
1033
+ if (repo === undefined) {
1034
+ log(`#${run.issue} retained cleanup deferred: repo ${run.repo} is no longer configured`);
1035
+ continue;
1036
+ }
1037
+
1038
+ const outcome = await cleanup(mirrorPathFor(repo, project.mirrorRoot), run.worktree, run.branch);
1039
+ if (outcome.kind === "removed") {
1040
+ store.updateRun(run.id, { worktree: "" });
1041
+ log(`#${run.issue} retained worktree reaped: ${run.worktree} (${run.branch})`);
1042
+ } else {
1043
+ log(`#${run.issue} retained worktree kept (${outcome.reason}): ${outcome.detail}`);
1044
+ }
812
1045
  }
813
1046
  }
814
1047
 
@@ -820,9 +1053,52 @@ export interface Admission {
820
1053
  attempt: number;
821
1054
  }
822
1055
 
1056
+ export interface AdmissionHold {
1057
+ issue: number;
1058
+ reason: AdmissionHoldReason;
1059
+ }
1060
+
1061
+ export interface AdmissionPass {
1062
+ admitted: Admission[];
1063
+ holds: AdmissionHold[];
1064
+ }
1065
+
1066
+ const HOLD_SAMPLE_SIZE = 5;
1067
+ const DEGRADED_HOLDS: ReadonlySet<AdmissionHoldReason> = new Set([
1068
+ "parent-lookup-error",
1069
+ "open-pr-lookup-error",
1070
+ ]);
1071
+
1072
+ /** Groups transient decisions into the bounded record exposed by status. */
1073
+ export function summarizeDispatch(
1074
+ ready: number,
1075
+ routed: number,
1076
+ admitted: number,
1077
+ holds: readonly AdmissionHold[],
1078
+ completedAt = Date.now(),
1079
+ ): DispatchSummary {
1080
+ const groups = new Map<AdmissionHoldReason, { count: number; issues: number[] }>();
1081
+ for (const hold of holds) {
1082
+ const group = groups.get(hold.reason) ?? { count: 0, issues: [] };
1083
+ group.count += 1;
1084
+ if (group.issues.length < HOLD_SAMPLE_SIZE) group.issues.push(hold.issue);
1085
+ groups.set(hold.reason, group);
1086
+ }
1087
+ return {
1088
+ completedAt,
1089
+ ready,
1090
+ routed,
1091
+ admitted,
1092
+ degraded: holds.some((hold) => DEGRADED_HOLDS.has(hold.reason)),
1093
+ holds: [...groups]
1094
+ .sort(([a], [b]) => a.localeCompare(b))
1095
+ .map(([reason, group]) => ({ reason, ...group })),
1096
+ };
1097
+ }
1098
+
823
1099
  /**
824
1100
  * Which routed candidates get a worker this tick — in queue order, never more
825
- * than `slots` of them.
1101
+ * than `slots` of them. Every non-admission receives a stable reason code.
826
1102
  *
827
1103
  * Exported so the admission rules can be pinned without spawning a worker.
828
1104
  * Every one of them exists because of a live incident, and each guards a
@@ -837,10 +1113,14 @@ export async function admitCandidates(
837
1113
  d: Pick<Deps, "project" | "caps" | "tracker" | "store" | "escalate">,
838
1114
  routed: Routed[],
839
1115
  slots: number,
840
- ): Promise<Admission[]> {
1116
+ ): Promise<AdmissionPass> {
841
1117
  const { project, caps, tracker, store } = d;
842
1118
  const busyIssues = store.activeRuns(project.name).map((r) => r.issue);
843
1119
  const busy = new Set(busyIssues);
1120
+ const holds: AdmissionHold[] = [];
1121
+ const hold = (issue: number, reason: AdmissionHoldReason): void => {
1122
+ holds.push({ issue, reason });
1123
+ };
844
1124
 
845
1125
  // parent -> blocking issue. Seeded from active runs (including pushed-green),
846
1126
  // then extended by candidates admitted earlier in this same pass so two
@@ -870,43 +1150,69 @@ export async function admitCandidates(
870
1150
 
871
1151
  const admitted: Admission[] = [];
872
1152
  for (const r of routed) {
873
- if (admitted.length >= slots) break;
874
- if (busy.has(r.issue.number)) continue;
1153
+ const issue = r.issue.number;
1154
+ if (admitted.length >= slots) {
1155
+ hold(issue, "capacity");
1156
+ continue;
1157
+ }
1158
+ if (busy.has(issue)) {
1159
+ hold(issue, "issue-active");
1160
+ continue;
1161
+ }
875
1162
 
876
- const prior = store.attemptsFor(project.name, r.issue.number);
877
- if (prior >= caps.maxAttemptsPerIssue) {
1163
+ const priorRuns = store.attemptsFor(project.name, issue);
1164
+ const failures = store.failuresFor(project.name, issue);
1165
+ if (failures >= caps.maxAttemptsPerIssue) {
1166
+ hold(issue, "failed-attempts");
878
1167
  await safeEscalate(d, {
879
1168
  tier: 1,
880
1169
  project: project.name,
881
- issue: r.issue.number,
882
- summary: `#${r.issue.number} has used all ${caps.maxAttemptsPerIssue} attempts`,
1170
+ issue,
1171
+ summary: `#${issue} has used all ${caps.maxAttemptsPerIssue} failed attempts`,
883
1172
  detail: [
884
1173
  r.issue.title,
885
1174
  r.issue.url,
886
- "Another attempt almost always means the issue itself is underspecified.",
1175
+ "Another implementation attempt almost always means the issue itself is underspecified.",
887
1176
  "Rewrite the acceptance criteria, or take it off the queue.",
888
1177
  ].join("\n"),
889
1178
  });
890
1179
  continue;
891
1180
  }
892
1181
 
1182
+ const continuations = store.continuationsFor(project.name, issue);
1183
+ if (!hasContinuationBudget(continuations, caps.maxContinuationsPerIssue)) {
1184
+ hold(issue, "continuations");
1185
+ await safeEscalate(d, {
1186
+ tier: 1,
1187
+ project: project.name,
1188
+ issue,
1189
+ summary: `#${issue} exceeded its ${caps.maxContinuationsPerIssue}-continuation budget`,
1190
+ detail: [
1191
+ r.issue.title,
1192
+ r.issue.url,
1193
+ "Repeated cap kills, daemon orphans, or answered blocks need an operator to inspect progress.",
1194
+ ].join("\n"),
1195
+ });
1196
+ continue;
1197
+ }
1198
+
893
1199
  // Soft concurrency per epic: at most one in-flight child of a given parent.
894
1200
  // No parent means today's concurrent admission. Cheap local filters already
895
1201
  // ran; this sits before the open-PR API call so a held sibling frees the
896
1202
  // slot for unrelated work without spending a closers query.
897
1203
  let parent: number | undefined;
898
1204
  try {
899
- parent = await resolveParent(r.issue.number);
1205
+ parent = await resolveParent(issue);
900
1206
  } catch (err) {
901
- log(`#${r.issue.number} held: parent check failed (${errText(err)}) — retrying next tick`);
1207
+ hold(issue, "parent-lookup-error");
1208
+ log(`#${issue} held: parent check failed (${errText(err)}) — retrying next tick`);
902
1209
  continue;
903
1210
  }
904
1211
  if (parent !== undefined) {
905
1212
  const blocker = occupiedParents.get(parent);
906
1213
  if (blocker !== undefined) {
907
- log(
908
- `#${r.issue.number} skipped: sibling #${blocker} in flight under epic #${parent}`,
909
- );
1214
+ hold(issue, "sibling-active");
1215
+ log(`#${issue} skipped: sibling #${blocker} in flight under epic #${parent}`);
910
1216
  continue;
911
1217
  }
912
1218
  }
@@ -917,36 +1223,85 @@ export async function admitCandidates(
917
1223
  // like fresh work, and a worker sent at it re-implements a finished PR. The
918
1224
  // tracker is the only party that remembers, so it is asked. The cost is
919
1225
  // bounded by free slots, not by queue depth: the call sits behind the two
920
- // cheap local filters and the loop stops once the slots are full.
1226
+ // cheap local filters and candidates beyond capacity skip it.
921
1227
  let closer: string | undefined;
922
1228
  try {
923
- closer = await tracker.openCloserFor(r.issue.number);
1229
+ closer = await tracker.openCloserFor(issue);
924
1230
  } catch (err) {
925
1231
  // Fail closed, per candidate. An API error means "unknown whether
926
1232
  // finished work exists", and admitting on unknown recreates precisely the
927
1233
  // duplicate-work failure this guard exists to kill: the worst case of
928
1234
  // holding is a five-minute delay, the worst case of admitting is a burned
929
1235
  // attempt and a second PR on the same issue. Holding one candidate rather
930
- // than aborting the loop is what keeps a transient GitHub failure from
931
- // deadlocking the whole dispatcher; the next tick retries by itself.
932
- log(`#${r.issue.number} held: open-PR check failed (${errText(err)}) — retrying next tick`);
1236
+ // than aborting the loop keeps a transient GitHub failure from deadlocking
1237
+ // the whole dispatcher; the next tick retries by itself.
1238
+ hold(issue, "open-pr-lookup-error");
1239
+ log(`#${issue} held: open-PR check failed (${errText(err)}) — retrying next tick`);
933
1240
  continue;
934
1241
  }
935
1242
  if (closer !== undefined) {
936
- log(`#${r.issue.number} skipped: open PR ${closer} already closes it`);
937
- continue;
1243
+ const latest = store.latestRun(project.name, issue);
1244
+ const retainedContinuation =
1245
+ latest?.prUrl === closer &&
1246
+ (latest.state === "blocked" ||
1247
+ latest.state === "failed" ||
1248
+ latest.state === "killed" ||
1249
+ latest.state === "orphaned");
1250
+ if (!retainedContinuation) {
1251
+ hold(issue, "open-pr");
1252
+ log(`#${issue} skipped: open PR ${closer} already closes it`);
1253
+ continue;
1254
+ }
1255
+ log(`#${issue} continuing retained PR ${closer} from ${latest.state} run`);
938
1256
  }
939
1257
 
940
- admitted.push({ r, attempt: prior + 1 });
941
- if (parent !== undefined) occupiedParents.set(parent, r.issue.number);
1258
+ admitted.push({ r, attempt: priorRuns + 1 });
1259
+ if (parent !== undefined) occupiedParents.set(parent, issue);
942
1260
  }
943
1261
 
944
- return admitted;
1262
+ return { admitted, holds };
1263
+ }
1264
+
1265
+ export interface WorkerPool {
1266
+ launch(work: Promise<void>): void;
1267
+ activeCount(): number;
1268
+ drain(): Promise<void>;
1269
+ }
1270
+
1271
+ /** Keeps background workers alive without making the five-minute tick await them. */
1272
+ export function createWorkerPool(): WorkerPool {
1273
+ const active = new Set<Promise<void>>();
1274
+ return {
1275
+ launch(work) {
1276
+ active.add(work);
1277
+ void work.then(
1278
+ () => active.delete(work),
1279
+ () => active.delete(work),
1280
+ );
1281
+ },
1282
+ activeCount: () => active.size,
1283
+ async drain() {
1284
+ await Promise.allSettled(active);
1285
+ },
1286
+ };
1287
+ }
1288
+
1289
+ /** `--once` awaits workers; the resident daemon registers them for shutdown. */
1290
+ export async function dispatchAdmissions(
1291
+ admitted: readonly Admission[],
1292
+ run: (admission: Admission) => Promise<void>,
1293
+ pool?: WorkerPool,
1294
+ ): Promise<void> {
1295
+ if (pool !== undefined) {
1296
+ for (const admission of admitted) pool.launch(run(admission));
1297
+ return;
1298
+ }
1299
+ await Promise.allSettled(admitted.map(run));
945
1300
  }
946
1301
 
947
1302
  // ----------------------------------------------------------------------- a tick
948
1303
 
949
- async function tick(d: Deps): Promise<void> {
1304
+ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
950
1305
  // Before the pause check, deliberately. This one is not about dispatch: the
951
1306
  // orchestrator is a different process, and it can be wedged while this fleet
952
1307
  // is paused — which is exactly the state the reference fleet was in when the
@@ -954,6 +1309,12 @@ async function tick(d: Deps): Promise<void> {
954
1309
  // know their supervising session stopped reading its queue.
955
1310
  await watchOrchestrator(d);
956
1311
 
1312
+ // Settlement is maintenance, not dispatch. Run it before every gate that can
1313
+ // stop claiming — pause, integrity, spend, and capacity — so status converges
1314
+ // while the fleet is parked or workers are still active. Resident workers run
1315
+ // through the pool without blocking this five-minute tick.
1316
+ await settlePushedGreen(d);
1317
+
957
1318
  // A paused fleet claims nothing. Checked first so pausing takes effect on the
958
1319
  // next tick without signalling the process.
959
1320
  if (isPaused()) return;
@@ -1011,16 +1372,25 @@ async function tick(d: Deps): Promise<void> {
1011
1372
  return;
1012
1373
  }
1013
1374
 
1014
- // Above admission on purpose: a row settled this tick frees its issue for
1015
- // this same tick, so a merge and a re-queue no longer cost five minutes each.
1016
- // Above the spend cap too, which returns early — settling is bookkeeping about
1017
- // work already paid for, and a fleet that halts itself is exactly when an
1018
- // operator reads `status` and needs it to be true.
1019
- await settlePushedGreen(d);
1020
-
1021
1375
  // route() filters the queue through isEligible() itself, so anything already
1022
1376
  // carrying a state label is gone before it gets here.
1023
- const { routed, unroutable } = route(await d.tracker.listReady(), project);
1377
+ const ready = await d.tracker.listReady();
1378
+ await cleanupRetainedRuns(
1379
+ d,
1380
+ new Set(ready.map((issue) => issue.number)),
1381
+ d.cleanup ?? { next: 0 },
1382
+ );
1383
+ const { routed, unroutable } = route(ready, project);
1384
+ const routingHolds: AdmissionHold[] = unroutable.map((u) => ({
1385
+ issue: u.issue.number,
1386
+ reason: `unroutable:${u.reason}`,
1387
+ }));
1388
+ const recordDispatch = (admitted: number, holds: readonly AdmissionHold[]): void => {
1389
+ store.recordDispatch(
1390
+ project.name,
1391
+ summarizeDispatch(ready.length, routed.length, admitted, holds),
1392
+ );
1393
+ };
1024
1394
 
1025
1395
  // An issue nobody can route never reaches a worker: guessing the target repo
1026
1396
  // is exactly the kind of improvisation this system exists to prevent. The
@@ -1061,6 +1431,10 @@ async function tick(d: Deps): Promise<void> {
1061
1431
  "No further work will be claimed until `omp-conductor resume` (or /conductor resume).",
1062
1432
  ].join("\n"),
1063
1433
  });
1434
+ recordDispatch(0, [
1435
+ ...routingHolds,
1436
+ ...routed.map((r) => ({ issue: r.issue.number, reason: "daily-spend-cap" as const })),
1437
+ ]);
1064
1438
  return;
1065
1439
  }
1066
1440
 
@@ -1072,20 +1446,119 @@ async function tick(d: Deps): Promise<void> {
1072
1446
  const slots = caps.maxConcurrentWorkers - live.length;
1073
1447
  if (slots <= 0) {
1074
1448
  log(`at capacity: ${live.length}/${caps.maxConcurrentWorkers} workers`);
1449
+ recordDispatch(0, [
1450
+ ...routingHolds,
1451
+ ...routed.map((r) => ({ issue: r.issue.number, reason: "capacity" as const })),
1452
+ ]);
1075
1453
  return;
1076
1454
  }
1077
1455
 
1078
- const admitted = await admitCandidates(d, routed, slots);
1456
+ const pass = await admitCandidates(d, routed, slots);
1457
+ recordDispatch(pass.admitted.length, [...routingHolds, ...pass.holds]);
1079
1458
 
1080
- if (admitted.length === 0) return;
1459
+ if (pass.admitted.length === 0) return;
1081
1460
 
1082
- log(`dispatching ${admitted.map((a) => `#${a.r.issue.number}`).join(" ")}`);
1083
- // handleIssue never rejects; allSettled is the belt to that braces.
1084
- await Promise.allSettled(admitted.map((a) => handleIssue(d, a.r, a.attempt)));
1461
+ log(`dispatching ${pass.admitted.map((a) => `#${a.r.issue.number}`).join(" ")}`);
1462
+ await dispatchAdmissions(
1463
+ pass.admitted,
1464
+ (a) => handleIssue(d, a.r, a.attempt),
1465
+ workers,
1466
+ );
1085
1467
  }
1086
1468
 
1087
1469
  // --------------------------------------------------------------- read-only views
1088
1470
 
1471
+ export interface DaemonHealthSnapshot {
1472
+ ok: true;
1473
+ paused: boolean;
1474
+ activeRuns: number;
1475
+ project: string;
1476
+ /** Resident set of this daemon; workers are in-process omp sessions. */
1477
+ rssBytes: number;
1478
+ dispatch?: DispatchSummary;
1479
+ codeGraph?: CodeGraphHealth;
1480
+ }
1481
+
1482
+ export function daemonHealthSnapshot(
1483
+ store: Store,
1484
+ project: string,
1485
+ paused = isPaused(),
1486
+ rssBytes = process.memoryUsage().rss,
1487
+ codeGraph?: CodeGraphHealth,
1488
+ ): DaemonHealthSnapshot {
1489
+ const dispatch = store.latestDispatch(project);
1490
+ return {
1491
+ ok: true,
1492
+ paused,
1493
+ activeRuns: store.activeRuns(project).length,
1494
+ project,
1495
+ rssBytes,
1496
+ ...(dispatch === undefined ? {} : { dispatch }),
1497
+ ...(codeGraph?.configured === true ? { codeGraph } : {}),
1498
+ };
1499
+ }
1500
+
1501
+ export async function turnLimitResponse(
1502
+ req: Request,
1503
+ project: string,
1504
+ store: Pick<Store, "latestRun">,
1505
+ registry: TurnLimitRegistry,
1506
+ ): Promise<Response | undefined> {
1507
+ const url = new URL(req.url);
1508
+ const match = /^\/runs\/(\d+)\/turn-limit$/.exec(url.pathname);
1509
+ if (req.method !== "PUT" || match === null) return undefined;
1510
+ if (!req.headers.get("content-type")?.startsWith("application/json")) {
1511
+ return Response.json({ error: "content-type must be application/json" }, { status: 415 });
1512
+ }
1513
+
1514
+ let body: unknown;
1515
+ try {
1516
+ body = await req.json();
1517
+ } catch {
1518
+ return Response.json({ error: "request body must be valid JSON" }, { status: 400 });
1519
+ }
1520
+ if (body === null || typeof body !== "object") {
1521
+ return Response.json({ error: "request body must be a JSON object" }, { status: 400 });
1522
+ }
1523
+ const requestedProject = Reflect.get(body, "project");
1524
+ if (typeof requestedProject !== "string" || requestedProject.length === 0) {
1525
+ return Response.json({ error: "project must be a non-empty string" }, { status: 400 });
1526
+ }
1527
+ if (requestedProject !== project) {
1528
+ return Response.json(
1529
+ { error: `daemon serves project "${project}", not requested project "${requestedProject}"` },
1530
+ { status: 409 },
1531
+ );
1532
+ }
1533
+ const maxTurns = Reflect.get(body, "maxTurns");
1534
+ if (!Number.isSafeInteger(maxTurns) || (maxTurns as number) < 1) {
1535
+ return Response.json({ error: "maxTurns must be a positive integer" }, { status: 400 });
1536
+ }
1537
+
1538
+ const issue = Number(match[1]);
1539
+ const outcome = registry.extend(project, issue, maxTurns as number);
1540
+ if (outcome.kind === "extended") return Response.json(outcome);
1541
+ if (outcome.kind === "not-increase") {
1542
+ return Response.json(
1543
+ { error: `#${issue} already has a ${outcome.maxTurns}-turn ceiling` },
1544
+ { status: 409 },
1545
+ );
1546
+ }
1547
+
1548
+ const latest = store.latestRun(project, issue);
1549
+ if (latest === undefined) {
1550
+ return Response.json({ error: `no run recorded for #${issue}` }, { status: 404 });
1551
+ }
1552
+ return Response.json(
1553
+ {
1554
+ error:
1555
+ `#${issue} has no live worker controller; its session already settled ` +
1556
+ `or belongs to another daemon (stored state: ${latest.state})`,
1557
+ },
1558
+ { status: 409 },
1559
+ );
1560
+ }
1561
+
1089
1562
  export interface StatusSnapshot {
1090
1563
  project: string;
1091
1564
  configPath: string;
@@ -1098,33 +1571,65 @@ export interface StatusSnapshot {
1098
1571
  liveWorkers: number;
1099
1572
  runsToday: number;
1100
1573
  spendTodayUsd: number;
1574
+ dispatch?: DispatchSummary;
1101
1575
  }
1102
1576
 
1103
- /** Opens and closes its own store handle so the CLI and the plugin can read
1104
- * status while a daemon in another process is writing (the store runs in WAL
1105
- * mode for exactly this). */
1577
+ /** Builds a status reading from an already-open store. Long-lived operator
1578
+ * surfaces use this path so a one-second refresh does not repeatedly open and
1579
+ * initialise SQLite connections. */
1580
+ export function statusSnapshotFromStore(p: ProjectConfig, caps: Caps, store: Store): StatusSnapshot {
1581
+ const since = startOfToday();
1582
+ const dispatch = store.latestDispatch(p.name);
1583
+ return {
1584
+ project: p.name,
1585
+ configPath: configPath(),
1586
+ stateDir: stateDir(),
1587
+ paused: isPaused(),
1588
+ caps,
1589
+ activeRuns: store.activeRuns(p.name),
1590
+ liveWorkers: store.liveRuns(p.name).length,
1591
+ runsToday: store.runsStartedSince(p.name, since),
1592
+ spendTodayUsd: store.spendSince(p.name, since),
1593
+ ...(dispatch === undefined ? {} : { dispatch }),
1594
+ };
1595
+ }
1596
+
1597
+ /** Opens and closes its own store handle so one-shot CLI and plugin readers can
1598
+ * read status while a daemon in another process is writing (the store runs in
1599
+ * WAL mode for exactly this). */
1106
1600
  export function statusSnapshot(project?: string): StatusSnapshot {
1107
1601
  const cfg = loadConfig();
1108
1602
  const p = findProject(cfg, project);
1109
1603
  const store = openStore(dbPath());
1110
1604
  try {
1111
- const since = startOfToday();
1112
- return {
1113
- project: p.name,
1114
- configPath: configPath(),
1115
- stateDir: stateDir(),
1116
- paused: isPaused(),
1117
- caps: resolveCaps(p, cfg.defaults),
1118
- activeRuns: store.activeRuns(p.name),
1119
- liveWorkers: store.liveRuns(p.name).length,
1120
- runsToday: store.runsStartedSince(p.name, since),
1121
- spendTodayUsd: store.spendSince(p.name, since),
1122
- };
1605
+ return statusSnapshotFromStore(p, resolveCaps(p, cfg.defaults), store);
1123
1606
  } finally {
1124
1607
  store.close();
1125
1608
  }
1126
1609
  }
1127
1610
 
1611
+ export function formatDispatchSummary(summary?: DispatchSummary): string {
1612
+ if (summary === undefined) return "last dispatch (none recorded)";
1613
+ const lines = [
1614
+ `last dispatch ${new Date(summary.completedAt).toISOString()}${summary.degraded ? " DEGRADED" : ""}`,
1615
+ ` candidates ${summary.ready} ready / ${summary.routed} routed`,
1616
+ ` admitted ${summary.admitted}`,
1617
+ ];
1618
+ if (summary.holds.length === 0) {
1619
+ lines.push(" held 0");
1620
+ } else {
1621
+ lines.push(" held");
1622
+ for (const hold of summary.holds) {
1623
+ const sample =
1624
+ hold.issues.length === 0
1625
+ ? ""
1626
+ : ` (#${hold.issues.join(", #")}${hold.count > hold.issues.length ? ", …" : ""})`;
1627
+ lines.push(` ${hold.reason} ${hold.count}${sample}`);
1628
+ }
1629
+ }
1630
+ return lines.join("\n");
1631
+ }
1632
+
1128
1633
  export function formatStatus(s: StatusSnapshot): string {
1129
1634
  const lines = [
1130
1635
  `project ${s.project}${s.paused ? " (PAUSED)" : ""}`,
@@ -1137,9 +1642,12 @@ export function formatStatus(s: StatusSnapshot): string {
1137
1642
  s.caps.dailySpendUsd === null
1138
1643
  ? ` spend today $${s.spendTodayUsd.toFixed(2)} (no daily cap)`
1139
1644
  : ` spend today $${s.spendTodayUsd.toFixed(2)} / $${s.caps.dailySpendUsd.toFixed(2)}`,
1140
- ` worker max turns ${s.caps.workerMaxTurns}`,
1645
+ ` new worker turns ${s.caps.workerMaxTurns}`,
1141
1646
  ` worker wall clock ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
1142
- ` attempts per issue ${s.caps.maxAttemptsPerIssue}`,
1647
+ ` failed attempts ${s.caps.maxAttemptsPerIssue}`,
1648
+ ` continuations ${s.caps.maxContinuationsPerIssue}`,
1649
+ "",
1650
+ formatDispatchSummary(s.dispatch),
1143
1651
  "",
1144
1652
  ];
1145
1653
  if (s.activeRuns.length === 0) {
@@ -1149,7 +1657,7 @@ export function formatStatus(s: StatusSnapshot): string {
1149
1657
  for (const r of s.activeRuns) {
1150
1658
  lines.push(
1151
1659
  ` #${r.issue} ${r.repo} ${r.state} attempt ${r.attempt} ` +
1152
- `${r.turns} turns $${r.spendUsd.toFixed(2)} ${r.branch}` +
1660
+ `${r.turns}/${r.maxTurns} turns $${r.spendUsd.toFixed(2)} ${r.branch}` +
1153
1661
  (r.prUrl ? ` ${r.prUrl}` : ""),
1154
1662
  );
1155
1663
  }
@@ -1180,14 +1688,15 @@ export interface QueuePreview {
1180
1688
  * label, run row or worktree. This is what makes `/conductor setup` honest: the
1181
1689
  * dry run is the same routing code the loop uses, not a description of it.
1182
1690
  */
1183
- export async function previewQueue(project?: string): Promise<QueuePreview> {
1184
- const cfg = loadConfig();
1185
- const p = findProject(cfg, project);
1691
+ export async function previewProject(
1692
+ p: ProjectConfig,
1693
+ path: string = configPath(),
1694
+ ): Promise<QueuePreview> {
1186
1695
  const { routed, unroutable } = route(await makeTracker(p).listReady(), p);
1187
1696
  const states = Object.values(p.stateLabels).join(", ");
1188
1697
  return {
1189
1698
  project: p.name,
1190
- configPath: configPath(),
1699
+ configPath: path,
1191
1700
  queueDescription:
1192
1701
  `open issues in ${p.tracker.repo} labelled "${p.queueLabel}", ` +
1193
1702
  `minus anything already labelled ${states}, ` +
@@ -1208,14 +1717,20 @@ export async function previewQueue(project?: string): Promise<QueuePreview> {
1208
1717
  };
1209
1718
  }
1210
1719
 
1720
+ export async function previewQueue(project?: string): Promise<QueuePreview> {
1721
+ const cfg = loadConfig();
1722
+ return previewProject(findProject(cfg, project));
1723
+ }
1724
+
1211
1725
  /**
1212
- * The one mutation `/conductor setup` performs, and only after the operator has
1213
- * seen the dry run: create the state directory and schema, then clear the pause
1214
- * flag so the daemon is allowed to claim work.
1726
+ * Creates the state store and holds dispatch while setup verifies the host.
1727
+ *
1728
+ * Setup calls this immediately after consent. Every later setup error therefore
1729
+ * leaves the fleet paused instead of exposing a partially written runtime.
1215
1730
  */
1216
- export function armConductor(): void {
1731
+ export function prepareConductor(): void {
1217
1732
  openStore(dbPath()).close();
1218
- setPaused(false);
1733
+ setPaused(true);
1219
1734
  }
1220
1735
 
1221
1736
  /**
@@ -1317,6 +1832,10 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
1317
1832
  "a time, freshness-checked against the base branch, per the Releases section of your POLICY.md."
1318
1833
  : "You never edit product code, push a branch, or merge a PR — a worker session edits and pushes, and a " +
1319
1834
  "human merges.",
1835
+ `Release tool gate: releasePolicy=${resolveReleasePolicy(project)}. ` +
1836
+ (resolveReleasePolicy(project) === "none"
1837
+ ? "Release and deploy tool calls are mechanically blocked."
1838
+ : "Release and deploy tool calls are permitted only by the operator brief."),
1320
1839
  "Handle each escalation below before the next one.",
1321
1840
  ].join("\n");
1322
1841
 
@@ -1334,7 +1853,12 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
1334
1853
  log("orchestrator: external — tier-1 escalations post as issue comments for the external session's drain duty");
1335
1854
  } else {
1336
1855
  try {
1337
- orchestrator = await startOrchestrator({ cwd: stateDir(), brief });
1856
+ orchestrator = await startOrchestrator({
1857
+ cwd: stateDir(),
1858
+ brief,
1859
+ releasePolicy: resolveReleasePolicy(project),
1860
+ onReleaseBlocked: (shape) => recordReleaseBlock(project.name, "orchestrator", shape),
1861
+ });
1338
1862
  const transcript = orchestrator.sessionFile();
1339
1863
  log(`orchestrator session ready${transcript === undefined ? "" : ` · ${transcript}`}`);
1340
1864
  } catch (err) {
@@ -1348,16 +1872,21 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
1348
1872
  }
1349
1873
 
1350
1874
  const escalator = createEscalator(project, tracker, store, orchestrator);
1875
+ const turnLimits = createTurnLimitRegistry((runId, maxTurns) => {
1876
+ store.updateRun(runId, { maxTurns });
1877
+ });
1351
1878
  const d: Deps = {
1352
1879
  project,
1353
1880
  caps,
1354
1881
  tracker,
1355
1882
  store,
1356
1883
  escalate: (e) => escalator.escalate(e),
1884
+ turnLimits,
1357
1885
  integrity,
1358
1886
  // Fresh per daemon run, like the integrity gate: a restart is entitled to
1359
1887
  // page again about a stall that is still on disk.
1360
1888
  stall: { paged: false },
1889
+ cleanup: { next: 0 },
1361
1890
  };
1362
1891
 
1363
1892
  if (o.once) {
@@ -1370,33 +1899,49 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
1370
1899
  return;
1371
1900
  }
1372
1901
 
1902
+ // Graphs are optional, so their bounded probes run beside dispatch and feed
1903
+ // a cache. /healthz remains an in-memory answer and never blocks liveness on
1904
+ // the indexer or systemd.
1905
+ let codeGraph = pendingCodeGraph(project);
1906
+ let graphProbe: Promise<void> | undefined;
1907
+ const refreshCodeGraph = (): void => {
1908
+ if (graphProbe !== undefined) return;
1909
+ graphProbe = probeCodeGraph(project)
1910
+ .then((health) => {
1911
+ codeGraph = health;
1912
+ })
1913
+ .catch(() => {
1914
+ log("code-graph health probe failed unexpectedly; retaining the previous bounded result");
1915
+ })
1916
+ .finally(() => {
1917
+ graphProbe = undefined;
1918
+ });
1919
+ };
1920
+ refreshCodeGraph();
1921
+ const graphTimer = setInterval(refreshCodeGraph, GRAPH_HEALTH_INTERVAL_MS);
1922
+
1923
+ const workers = createWorkerPool();
1373
1924
  let stopping = false;
1374
1925
  let wake: (() => void) | undefined;
1375
1926
  const stop = (): void => {
1376
1927
  if (stopping) return;
1377
1928
  stopping = true;
1378
- log("shutting down after the current tick");
1929
+ log("shutting down after active workers finish");
1379
1930
  wake?.();
1380
1931
  };
1381
1932
  process.on("SIGINT", stop);
1382
1933
  process.on("SIGTERM", stop);
1383
1934
 
1384
1935
  const server = Bun.serve({
1936
+ hostname: "127.0.0.1",
1385
1937
  port: o.port ?? DEFAULT_PORT,
1386
- fetch(req) {
1938
+ async fetch(req) {
1387
1939
  const url = new URL(req.url);
1388
1940
  if (req.method === "GET" && url.pathname === "/healthz") {
1389
- return Response.json({
1390
- ok: true,
1391
- paused: isPaused(),
1392
- activeRuns: store.activeRuns(project.name).length,
1393
- project: project.name,
1394
- // Resident set of *this* process: workers are in-process omp sessions,
1395
- // so the unit's Memory peak is this number, not a separate worker pid.
1396
- rssBytes: process.memoryUsage().rss,
1397
- });
1941
+ return Response.json(daemonHealthSnapshot(store, project.name, undefined, undefined, codeGraph));
1398
1942
  }
1399
- return new Response("not found\n", { status: 404 });
1943
+ const control = await turnLimitResponse(req, project.name, store, turnLimits);
1944
+ return control ?? new Response("not found\n", { status: 404 });
1400
1945
  },
1401
1946
  });
1402
1947
  log(`serving /healthz on :${server.port}, project ${project.name}`);
@@ -1404,7 +1949,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
1404
1949
  try {
1405
1950
  while (!stopping) {
1406
1951
  try {
1407
- await tick(d);
1952
+ await tick(d, workers);
1408
1953
  } catch (err) {
1409
1954
  // A tick that blows up outside an issue (the tracker is down, say) must
1410
1955
  // not end the daemon; the next one will retry.
@@ -1423,6 +1968,8 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
1423
1968
  } finally {
1424
1969
  process.off("SIGINT", stop);
1425
1970
  process.off("SIGTERM", stop);
1971
+ clearInterval(graphTimer);
1972
+ await workers.drain();
1426
1973
  await server.stop(true);
1427
1974
  // Before the store closes: a queued injection that rejects on the way out
1428
1975
  // falls back to an issue comment, and that path writes the dedup marker.