omp-conductor 0.2.2 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/daemon.ts CHANGED
@@ -7,11 +7,13 @@
7
7
  * of it, so concurrency, daily volume, spend and per-issue attempts are counted
8
8
  * here and enforced before anything is claimed.
9
9
  */
10
- import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
11
- import { dirname, join } from "node:path";
10
+ import { createHash } from "node:crypto";
11
+ import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
12
+ import { dirname, join, relative } from "node:path";
12
13
  import { configPath, findProject, loadConfig, resolveCaps, stateDir } from "./config.ts";
13
14
  import { createEscalator } from "./escalate.ts";
14
15
  import { livingDaemon } from "./lifecycle.ts";
16
+ import { STALL_MARKER_FILE } from "./orchestrator-tick.ts";
15
17
  import { startOrchestrator } from "./orchestrator.ts";
16
18
  import type { OrchestratorHandle } from "./orchestrator.ts";
17
19
  import { branchName, route } from "./routing.ts";
@@ -28,8 +30,15 @@ import type {
28
30
  Store,
29
31
  Tracker,
30
32
  } from "./types.ts";
31
- import { renderBrief, runWorker } from "./worker.ts";
32
- import { addWorktree, mirrorPathFor, removeWorktree, worktreePathFor } from "./worktree.ts";
33
+ import { type KilledBy, renderBrief, runWorker } from "./worker.ts";
34
+ import {
35
+ addWorktree,
36
+ mirrorPathFor,
37
+ removeWorktree,
38
+ salvageWip,
39
+ type SalvageOutcome,
40
+ worktreePathFor,
41
+ } from "./worktree.ts";
33
42
 
34
43
  /** Long enough that the tracker is not polled raw, short enough that a human
35
44
  * who labels an issue sees it picked up within a coffee break. */
@@ -61,6 +70,8 @@ interface Deps {
61
70
  tracker: Tracker;
62
71
  store: Store;
63
72
  escalate(e: Escalation): Promise<void>;
73
+ integrity: IntegrityGate;
74
+ stall: StallGate;
64
75
  }
65
76
 
66
77
  // ---------------------------------------------------------------- paths & pause
@@ -70,6 +81,97 @@ export function dbPath(): string {
70
81
  return join(stateDir(), "conductor.db");
71
82
  }
72
83
 
84
+ // ------------------------------------------------------- orchestrator liveness
85
+
86
+ /**
87
+ * Whether the wedged-orchestrator page has already gone out for the stall
88
+ * currently on disk. One page per episode: the marker persists until a tick is
89
+ * consumed, so paging per five minutes would be paging forever.
90
+ */
91
+ export interface StallGate {
92
+ paged: boolean;
93
+ }
94
+
95
+ export interface StallVerdict {
96
+ /** The marker's own line, when one is there. */
97
+ since?: string;
98
+ page: boolean;
99
+ }
100
+
101
+ /**
102
+ * Reads the orchestrator's stall marker and decides whether this tick pages.
103
+ *
104
+ * The marker is written by the tick extension inside the orchestrator session
105
+ * ({@link STALL_MARKER_FILE}) when two of its own prompts go unconsumed — the
106
+ * one signal that separates "the process is alive" from "the loop is reading
107
+ * its queue". Every other guard in this system reads healthy through a wedge:
108
+ * the herdr recovery plugin tests for a live process and an agent label, both
109
+ * of which survive it, and `/healthz` describes this daemon, which is a
110
+ * different process entirely.
111
+ *
112
+ * The daemon is the natural watcher precisely because it is that different
113
+ * process: it already wakes every five minutes, it owns a working escalation
114
+ * path, and nothing about its health depends on the session that is stuck. A
115
+ * wedged loop cannot page for itself, and the herdr plugin only runs on session
116
+ * lifecycle events — a session that stays alive and stops working emits none.
117
+ *
118
+ * Resets when the marker disappears, so a second stall days later pages again.
119
+ */
120
+ export function checkStall(gate: StallGate, marker: string): StallVerdict {
121
+ if (!existsSync(marker)) {
122
+ gate.paged = false;
123
+ return { page: false };
124
+ }
125
+ const page = !gate.paged;
126
+ let since: string | undefined;
127
+ try {
128
+ const body = readFileSync(marker, "utf8").split("\n")[0]?.trim();
129
+ if (body !== undefined && body !== "") since = body;
130
+ } catch {
131
+ // An unreadable marker still means stalled; the timestamp is a nicety.
132
+ }
133
+ return { ...(since === undefined ? {} : { since }), page };
134
+ }
135
+
136
+ /**
137
+ * Pages tier 2 once when the orchestrator session stops draining its queue.
138
+ *
139
+ * Deliberately does not restart anything. A wedge lands mid-turn, this process
140
+ * cannot tell a half-applied edit from an idle loop, and killing the session
141
+ * could destroy work an operator would rather read first — the same refusal to
142
+ * guess that the recovery plugin is built on.
143
+ */
144
+ async function watchOrchestrator(d: Deps): Promise<void> {
145
+ const marker = join(stateDir(), STALL_MARKER_FILE);
146
+ const verdict = checkStall(d.stall, marker);
147
+ if (!verdict.page) return;
148
+
149
+ log(`ERROR: the orchestrator session is not draining its queue — ${verdict.since ?? "no timestamp"}`);
150
+ const delivered = await safeEscalate(d, {
151
+ tier: 2,
152
+ project: d.project.name,
153
+ issue: NO_ISSUE,
154
+ // Dated like the other tier-2 summaries: the dedup key is the summary, and
155
+ // a second wedge next month must not read as a repeat of this one.
156
+ summary:
157
+ `Orchestrator session wedged on ${new Date().toISOString().slice(0, 10)} — ` +
158
+ `it has stopped reading its queue (${d.project.name})`,
159
+ detail: [
160
+ verdict.since ?? "Marker present with no readable timestamp.",
161
+ `Marker: ${marker}`,
162
+ "",
163
+ "Its process and its herdr agent label are both healthy, which is why nothing else noticed:",
164
+ "the loop is alive and consuming nothing, so ticks and your messages queue behind it unread.",
165
+ "",
166
+ "Attach and look before you act — a wedge lands mid-turn. Then SIGTERM the omp process:",
167
+ "herdr-conductor resumes it by exact identity, and the first consumed tick clears this marker.",
168
+ "",
169
+ "Dispatch is unaffected: workers keep running. What stops is drain, groom, report and merge.",
170
+ ].join("\n"),
171
+ });
172
+ markPaged(d.stall, delivered);
173
+ }
174
+
73
175
  /**
74
176
  * Pause is a file rather than process state on purpose: `omp-conductor pause`
75
177
  * and `/conductor pause` run in a different process from the daemon, and a flag
@@ -90,6 +192,99 @@ export function setPaused(v: boolean): void {
90
192
  }
91
193
  }
92
194
 
195
+ // ----------------------------------------------------------- package integrity
196
+
197
+ /** Enough differing paths to tell a deploy from a tamper at a glance; the full
198
+ * list is on the host, and the answer is always "go look at the host". */
199
+ const INTEGRITY_SAMPLE = 5;
200
+
201
+ /**
202
+ * What the daemon booted with, and whether it has already paged about losing
203
+ * it. Lives exactly as long as one `runDaemon()` call — which is the whole
204
+ * trick: a restart re-records both.
205
+ */
206
+ export interface IntegrityGate {
207
+ baseline: Map<string, string>;
208
+ paged: boolean;
209
+ }
210
+
211
+ export interface IntegrityVerdict {
212
+ /** Labelled, sorted differences; empty when the package is untouched. */
213
+ diff: string[];
214
+ /** Any difference at all stops the fleet. */
215
+ pause: boolean;
216
+ /** First divergent tick only — a page every five minutes is a page nobody reads. */
217
+ page: boolean;
218
+ }
219
+
220
+ /**
221
+ * sha256 of every source file the running package is made of, keyed by path
222
+ * relative to `root`.
223
+ *
224
+ * `import.meta.dir` is the installed `src/` of the code executing right now, so
225
+ * this is a self-portrait: what was actually deployed, not what some checkout
226
+ * on disk happens to contain. `.ts` and `.md` because both are executable in
227
+ * this package — the briefs under `src/briefs/` are the sessions' instructions,
228
+ * and rewriting one of those buys more than rewriting the dispatcher does.
229
+ * (A checkout also carries `*.test.ts`, which the published package excludes, so
230
+ * a daemon started from one is watching its tests too. That is the honest
231
+ * answer — its code did change — and it costs nothing on a real install.)
232
+ *
233
+ * Walking and hashing the ~30 files of this package measures 0.6 ms warm, once
234
+ * per five-minute tick, so a tick does it inline. No cache and no mtime
235
+ * shortcut on purpose: a cache is a second thing that can be wrong, and mtime
236
+ * is the first field anyone covering their tracks restores.
237
+ */
238
+ export function packageManifest(root: string = import.meta.dir): Map<string, string> {
239
+ const out = new Map<string, string>();
240
+ const walk = (dir: string): void => {
241
+ for (const e of readdirSync(dir, { withFileTypes: true })) {
242
+ const full = join(dir, e.name);
243
+ if (e.isDirectory()) walk(full);
244
+ else if (e.isFile() && (e.name.endsWith(".ts") || e.name.endsWith(".md")))
245
+ out.set(relative(root, full), createHash("sha256").update(readFileSync(full)).digest("hex"));
246
+ }
247
+ };
248
+ walk(root);
249
+ return out;
250
+ }
251
+
252
+ /**
253
+ * Labelled rather than three arrays because every consumer — the log line, the
254
+ * page, the test — wants one readable list of what moved.
255
+ */
256
+ export function manifestDiff(before: Map<string, string>, after: Map<string, string>): string[] {
257
+ const out: string[] = [];
258
+ for (const [path, hash] of before) {
259
+ const now = after.get(path);
260
+ if (now === undefined) out.push(`removed ${path}`);
261
+ else if (now !== hash) out.push(`changed ${path}`);
262
+ }
263
+ for (const path of after.keys()) if (!before.has(path)) out.push(`added ${path}`);
264
+ return out.sort();
265
+ }
266
+
267
+ /**
268
+ * The tick's decision, split from its effects so the once-only page is a thing
269
+ * a test can hold.
270
+ *
271
+ * `pause` stays true on every divergent tick, deliberately: an operator who
272
+ * resumes without restarting gets re-paused, because the boundary is still
273
+ * broken. `page` asks whether this tick should *try* — the caller latches the
274
+ * gate with {@link markPaged} only once a page actually went out, so a Telegram
275
+ * outage during the one tick that noticed does not buy permanent silence.
276
+ */
277
+ export function checkIntegrity(gate: IntegrityGate, current: Map<string, string>): IntegrityVerdict {
278
+ const diff = manifestDiff(gate.baseline, current);
279
+ if (diff.length === 0) return { diff, pause: false, page: false };
280
+ return { diff, pause: true, page: !gate.paged };
281
+ }
282
+
283
+ /** Latch a once-only page, after delivery is confirmed and never before. */
284
+ export function markPaged(gate: { paged: boolean }, delivered: boolean): void {
285
+ if (delivered) gate.paged = true;
286
+ }
287
+
93
288
  // ---------------------------------------------------------------------- helpers
94
289
 
95
290
  function log(msg: string): void {
@@ -158,13 +353,76 @@ async function swapLabel(tracker: Tracker, issue: number, from: string, to: stri
158
353
  * The escalator throws when no transport is configured or Telegram rejects, and
159
354
  * only records the dedup marker on success. A page that cannot be delivered
160
355
  * must not take the tick down with it — log it and let the next tick retry.
356
+ *
357
+ * Returns whether it actually went out, because "page once" and "page once
358
+ * *successfully*" are different promises: a caller that latches a once-only
359
+ * gate on the attempt turns one failed delivery into permanent silence about a
360
+ * condition that is still true.
161
361
  */
162
- async function safeEscalate(d: Deps, e: Escalation): Promise<void> {
362
+ async function safeEscalate(d: Pick<Deps, "escalate">, e: Escalation): Promise<boolean> {
163
363
  try {
164
364
  await d.escalate(e);
365
+ return true;
165
366
  } catch (err) {
166
367
  log(`escalation for #${e.issue} could not be delivered: ${errText(err)}`);
368
+ return false;
369
+ }
370
+ }
371
+
372
+ /**
373
+ * What a salvage attempt contributes to the escalation: where the work went, or
374
+ * that it went nowhere. Split from the effects below for the same reason
375
+ * `checkIntegrity` is — this wording is the whole thing a human acts on, so it
376
+ * is worth a test holding it, and the sha in it is the only pointer to work
377
+ * that no longer has any other copy.
378
+ */
379
+ export function salvageLines(outcome: SalvageOutcome, worktree: string): string[] {
380
+ const kept = `Worktree kept for inspection: ${worktree}`;
381
+
382
+ if (outcome.kind === "nothing") return [`${kept} — nothing uncommitted to salvage`];
383
+
384
+ if (outcome.kind === "failed") {
385
+ return [
386
+ `WIP SALVAGE FAILED: ${outcome.error}`,
387
+ `Uncommitted work in ${worktree} is the only copy of it, and the next attempt removes that tree.`,
388
+ ];
167
389
  }
390
+
391
+ return [
392
+ `WIP committed to ${outcome.branch} @ ${outcome.sha}` +
393
+ (outcome.pushed
394
+ ? " and pushed — the work outlives this worktree"
395
+ : ` but NOT pushed (${outcome.pushError ?? "no reason given"}) — it lives only in this host's mirror`),
396
+ kept,
397
+ ];
398
+ }
399
+
400
+ /**
401
+ * Commits and pushes whatever a dead run left uncommitted, logs the outcome and
402
+ * returns the escalation lines that say where that work now lives.
403
+ *
404
+ * Only ever called on a non-graceful end — a cap kill, a crashed session, a
405
+ * dispatch error. A `blocked` run stopped on purpose, with turns still in hand
406
+ * and a brief that tells it to report rather than push, so nothing is committed
407
+ * behind its back. The rest never got the chance: the kill is external and
408
+ * lands mid-edit, in the tree the next attempt removes `--force`.
409
+ */
410
+ async function salvage(
411
+ issue: number,
412
+ attempt: number,
413
+ reason: string,
414
+ worktree: string,
415
+ ): Promise<string[]> {
416
+ const lines = salvageLines(await salvageWip(worktree, issue, attempt, reason), worktree);
417
+ log(`#${issue} salvage: ${lines.join(" ")}`);
418
+ return lines;
419
+ }
420
+
421
+ /** How a run's end is named — in the salvage commit, and to whoever reads it. */
422
+ function endedBy(killedBy: KilledBy | undefined): string {
423
+ if (killedBy === "turns") return "the turns cap";
424
+ if (killedBy === "wallclock") return "the wall-clock cap";
425
+ return "a failed run";
168
426
  }
169
427
 
170
428
  async function buildBrief(
@@ -202,6 +460,10 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
202
460
 
203
461
  let claimed = false;
204
462
  let run: RunRecord | undefined;
463
+ // Hoisted out of the try so the catch path can still name the tree: a crash
464
+ // mid-dispatch is one of the non-graceful ends whose uncommitted work has to
465
+ // be salvaged too, and it is the path least likely to have committed first.
466
+ let worktreePath: string | undefined;
205
467
 
206
468
  try {
207
469
  // Claim on the tracker FIRST, before any local work. The label — not the
@@ -234,7 +496,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
234
496
  const mirrorPath = mirrorPathFor(r.repo, project.mirrorRoot);
235
497
  await removeWorktree(mirrorPath, worktreePathFor(project.workspaceRoot, issue));
236
498
 
237
- const worktreePath = await addWorktree(
499
+ worktreePath = await addWorktree(
238
500
  r.repo,
239
501
  project.mirrorRoot,
240
502
  project.workspaceRoot,
@@ -258,6 +520,11 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
258
520
  sessionDir,
259
521
  ...(project.workerModel === undefined ? {} : { model: project.workerModel }),
260
522
  onTurn: (n) => store.updateRun(runId, { turns: n }),
523
+ // Recorded the moment the session opens its transcript, not when the run
524
+ // ends: `omp-conductor tail` resolves an issue to a file through this row,
525
+ // and a path written at completion is a path nobody can follow live. The
526
+ // completion-time update below writes the same value again, harmlessly.
527
+ onSessionFile: (f) => store.updateRun(runId, { sessionFile: f }),
261
528
  });
262
529
 
263
530
  // A configured model the harness could not honour means this run was done by
@@ -288,6 +555,9 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
288
555
  });
289
556
  } else if (result.state === "failed" || result.state === "killed") {
290
557
  await swapLabel(tracker, issue, inProgress, project.stateLabels.failed);
558
+ // Before the escalation is composed, so it can say where the work went —
559
+ // and long before the next attempt provisions over this tree.
560
+ const salvaged = await salvage(issue, attempt, endedBy(result.killedBy), worktreePath);
291
561
  await safeEscalate(d, {
292
562
  tier: 1,
293
563
  project: project.name,
@@ -302,7 +572,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
302
572
  detail: [
303
573
  `${r.issue.title}`,
304
574
  r.issue.url,
305
- `Worktree kept for inspection: ${worktreePath}`,
575
+ ...salvaged,
306
576
  `Session: ${result.sessionFile ?? "(no transcript)"}`,
307
577
  "",
308
578
  result.report,
@@ -336,28 +606,166 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
336
606
  log(`#${issue} could not be relabelled: ${errText(relabelErr)}`);
337
607
  }
338
608
  }
609
+ // A crash lands anywhere, including mid-edit in a tree holding the only
610
+ // copy of real work. Nothing else on this path so much as looks at it.
611
+ const salvaged =
612
+ worktreePath === undefined
613
+ ? []
614
+ : await salvage(issue, attempt, "a dispatch error", worktreePath);
615
+
339
616
  await safeEscalate(d, {
340
617
  tier: 1,
341
618
  project: project.name,
342
619
  issue,
343
620
  runId: run?.id,
344
621
  summary: `#${issue} could not be dispatched on attempt ${attempt}`,
345
- detail,
622
+ detail: salvaged.length === 0 ? detail : [detail, "", ...salvaged].join("\n"),
346
623
  });
347
624
  // The worktree, if one was created, is deliberately left in place: this is
348
- // a failure path.
625
+ // a failure path, and whatever it still held is now a commit on the branch.
626
+ }
627
+ }
628
+
629
+ // -------------------------------------------------------------------- admission
630
+
631
+ /** A candidate cleared for dispatch, with the attempt number it will run as. */
632
+ export interface Admission {
633
+ r: Routed;
634
+ attempt: number;
635
+ }
636
+
637
+ /**
638
+ * Which routed candidates get a worker this tick — in queue order, never more
639
+ * than `slots` of them.
640
+ *
641
+ * Exported so the admission rules can be pinned without spawning a worker.
642
+ * Every one of them exists because of a live incident, and each guards a
643
+ * different way the same issue gets worked twice.
644
+ *
645
+ * Takes the slice of `Deps` it actually reads rather than the whole thing: what
646
+ * admission is allowed to consult is the point of the function, and a `Deps`
647
+ * that grows a field has no business breaking these tests.
648
+ */
649
+ export async function admitCandidates(
650
+ d: Pick<Deps, "project" | "caps" | "tracker" | "store" | "escalate">,
651
+ routed: Routed[],
652
+ slots: number,
653
+ ): Promise<Admission[]> {
654
+ const { project, caps, tracker, store } = d;
655
+ const busy = new Set(store.activeRuns(project.name).map((r) => r.issue));
656
+
657
+ const admitted: Admission[] = [];
658
+ for (const r of routed) {
659
+ if (admitted.length >= slots) break;
660
+ if (busy.has(r.issue.number)) continue;
661
+
662
+ const prior = store.attemptsFor(project.name, r.issue.number);
663
+ if (prior >= caps.maxAttemptsPerIssue) {
664
+ await safeEscalate(d, {
665
+ tier: 1,
666
+ project: project.name,
667
+ issue: r.issue.number,
668
+ summary: `#${r.issue.number} has used all ${caps.maxAttemptsPerIssue} attempts`,
669
+ detail: [
670
+ r.issue.title,
671
+ r.issue.url,
672
+ "Another attempt almost always means the issue itself is underspecified.",
673
+ "Rewrite the acceptance criteria, or take it off the queue.",
674
+ ].join("\n"),
675
+ });
676
+ continue;
677
+ }
678
+
679
+ // The busy set is built from run rows, so it can only speak for work this
680
+ // database recorded. Work pushed before this store existed — a migration, a
681
+ // wiped or relocated state dir, a restore onto a new host — looks exactly
682
+ // like fresh work, and a worker sent at it re-implements a finished PR. The
683
+ // tracker is the only party that remembers, so it is asked. The cost is
684
+ // bounded by free slots, not by queue depth: the call sits behind the two
685
+ // cheap local filters and the loop stops once the slots are full.
686
+ let closer: string | undefined;
687
+ try {
688
+ closer = await tracker.openCloserFor(r.issue.number);
689
+ } catch (err) {
690
+ // Fail closed, per candidate. An API error means "unknown whether
691
+ // finished work exists", and admitting on unknown recreates precisely the
692
+ // duplicate-work failure this guard exists to kill: the worst case of
693
+ // holding is a five-minute delay, the worst case of admitting is a burned
694
+ // attempt and a second PR on the same issue. Holding one candidate rather
695
+ // than aborting the loop is what keeps a transient GitHub failure from
696
+ // deadlocking the whole dispatcher; the next tick retries by itself.
697
+ log(`#${r.issue.number} held: open-PR check failed (${errText(err)}) — retrying next tick`);
698
+ continue;
699
+ }
700
+ if (closer !== undefined) {
701
+ log(`#${r.issue.number} skipped: open PR ${closer} already closes it`);
702
+ continue;
703
+ }
704
+
705
+ admitted.push({ r, attempt: prior + 1 });
349
706
  }
707
+
708
+ return admitted;
350
709
  }
351
710
 
352
711
  // ----------------------------------------------------------------------- a tick
353
712
 
354
713
  async function tick(d: Deps): Promise<void> {
714
+ // Before the pause check, deliberately. This one is not about dispatch: the
715
+ // orchestrator is a different process, and it can be wedged while this fleet
716
+ // is paused — which is exactly the state the reference fleet was in when the
717
+ // failure happened. A pause silences claiming, not the operator's right to
718
+ // know their supervising session stopped reading its queue.
719
+ await watchOrchestrator(d);
720
+
355
721
  // A paused fleet claims nothing. Checked first so pausing takes effect on the
356
722
  // next tick without signalling the process.
357
723
  if (isPaused()) return;
358
724
 
359
725
  const { project, caps, store } = d;
360
726
 
727
+ // "Nobody patches the running conductor" is a hard boundary in both briefs —
728
+ // which makes it prompt text, and prompt text is a request. This is the half
729
+ // that does not negotiate: the package that dispatched the last worker must
730
+ // still be the package on disk, or nothing else this tick does is
731
+ // attributable. A legitimate deploy never trips it, because installing a new
732
+ // build and restarting the unit re-records the baseline from the new files;
733
+ // only an edit *underneath* a live daemon diverges from it.
734
+ const integrity = checkIntegrity(d.integrity, packageManifest());
735
+ if (integrity.pause) {
736
+ const shown = integrity.diff.slice(0, INTEGRITY_SAMPLE);
737
+ log(
738
+ `ERROR: the installed conductor changed under this daemon — ${integrity.diff.length} file(s) differ ` +
739
+ `(${shown.join(", ")}${integrity.diff.length > shown.length ? ", …" : ""}) — pausing`,
740
+ );
741
+ setPaused(true);
742
+ if (integrity.page) {
743
+ const delivered = await safeEscalate(d, {
744
+ tier: 2,
745
+ project: project.name,
746
+ issue: NO_ISSUE,
747
+ // Dated for the same reason the spend cap is: the dedup key is the
748
+ // summary, and a second tamper months later must not be swallowed as a
749
+ // repeat of the first.
750
+ summary:
751
+ `Installed conductor changed under a running daemon on ${new Date().toISOString().slice(0, 10)}: ` +
752
+ `${integrity.diff.length} file(s) differ (first: ${integrity.diff[0]}) — ${project.name} is paused`,
753
+ detail: [
754
+ `Package root: ${import.meta.dir}`,
755
+ ...shown,
756
+ ...(integrity.diff.length > shown.length ? [`… and ${integrity.diff.length - shown.length} more`] : []),
757
+ "",
758
+ "If you deployed a new build, restart the daemon — the restart re-records the baseline.",
759
+ "If you did not, the host edited itself while it was dispatching work: treat every run since",
760
+ "the last known-good restart as unattributable before resuming.",
761
+ "`omp-conductor resume` alone will not hold — the next tick re-pauses while the files differ.",
762
+ ].join("\n"),
763
+ });
764
+ markPaged(d.integrity, delivered);
765
+ }
766
+ return;
767
+ }
768
+
361
769
  // route() filters the queue through isEligible() itself, so anything already
362
770
  // carrying a state label is gone before it gets here.
363
771
  const { routed, unroutable } = route(await d.tracker.listReady(), project);
@@ -405,8 +813,8 @@ async function tick(d: Deps): Promise<void> {
405
813
 
406
814
  // Two different questions, deliberately two queries. Capacity counts worker
407
815
  // *processes*, so a green PR awaiting a human merge must not consume a slot —
408
- // two of those would otherwise stop the fleet. The busy set protects *issues*,
409
- // so that same green PR must be in it, or a second attempt lands on a live PR.
816
+ // two of those would otherwise stop the fleet. That same PR's *issue* must
817
+ // still be occupied, which is what `admitCandidates`' busy set is for.
410
818
  const live = store.liveRuns(project.name);
411
819
  const slots = caps.maxConcurrentWorkers - live.length;
412
820
  if (slots <= 0) {
@@ -414,32 +822,7 @@ async function tick(d: Deps): Promise<void> {
414
822
  return;
415
823
  }
416
824
 
417
- const busy = new Set(store.activeRuns(project.name).map((r) => r.issue));
418
-
419
- const admitted: { r: Routed; attempt: number }[] = [];
420
- for (const r of routed) {
421
- if (admitted.length >= slots) break;
422
- if (busy.has(r.issue.number)) continue;
423
-
424
- const prior = store.attemptsFor(project.name, r.issue.number);
425
- if (prior >= caps.maxAttemptsPerIssue) {
426
- await safeEscalate(d, {
427
- tier: 1,
428
- project: project.name,
429
- issue: r.issue.number,
430
- summary: `#${r.issue.number} has used all ${caps.maxAttemptsPerIssue} attempts`,
431
- detail: [
432
- r.issue.title,
433
- r.issue.url,
434
- "Another attempt almost always means the issue itself is underspecified.",
435
- "Rewrite the acceptance criteria, or take it off the queue.",
436
- ].join("\n"),
437
- });
438
- continue;
439
- }
440
-
441
- admitted.push({ r, attempt: prior + 1 });
442
- }
825
+ const admitted = await admitCandidates(d, routed, slots);
443
826
 
444
827
  if (admitted.length === 0) return;
445
828
 
@@ -612,6 +995,15 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
612
995
  const store = openStore(dbPath());
613
996
  const tracker = makeTracker(project);
614
997
 
998
+ // Recorded here, before a single tick runs, so that the deploy an operator
999
+ // *means* to do never trips the tripwire: installing a new build and
1000
+ // restarting the unit re-records this from the new files. What it catches is
1001
+ // the other thing — the package changing while the daemon that dispatches
1002
+ // work is holding it open, whether that is a worker that wandered out of its
1003
+ // worktree or a human editing the live install "just to test something".
1004
+ const integrity: IntegrityGate = { baseline: packageManifest(), paged: false };
1005
+ log(`package integrity baseline: ${integrity.baseline.size} files under ${import.meta.dir}`);
1006
+
615
1007
  // Before the first tick, settle what the last process left behind — unless
616
1008
  // another daemon is alive (a foreground `daemon --once` beside a running
617
1009
  // daemon must not orphan that daemon's real, live workers).
@@ -641,8 +1033,16 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
641
1033
  "fails twice, its gates stay red, its branch conflicts, or a tripwire fires.",
642
1034
  "Your job when that happens: re-brief the issue (comment what the next worker must do",
643
1035
  `differently, then put ${project.queueLabel} back on it), file follow-up issues, or promote to`,
644
- "tier 2 and let the human decide. You never edit product code, push a branch, or merge a PR —",
645
- "a worker session does all of that. Handle each escalation below before the next one.",
1036
+ "tier 2 and let the human decide.",
1037
+ // Worded from `authority.merge` rather than fixed, so the standing orders
1038
+ // and the Releases section of the rendered brief cannot disagree about who
1039
+ // is holding the merge button. The daemon still merges nothing itself.
1040
+ project.authority.merge === "orchestrator"
1041
+ ? "You never edit product code or push a branch — a worker session does that. Merging is yours: one PR at " +
1042
+ "a time, freshness-checked against the base branch, per the Releases section of your ORCHESTRATOR.md."
1043
+ : "You never edit product code, push a branch, or merge a PR — a worker session edits and pushes, and a " +
1044
+ "human merges.",
1045
+ "Handle each escalation below before the next one.",
646
1046
  ].join("\n");
647
1047
 
648
1048
  // One orchestrator per daemon run, not per tick: it is a persistent session
@@ -651,17 +1051,25 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
651
1051
  // directory, deliberately not a checkout — the orchestrator re-briefs workers
652
1052
  // and talks to the tracker, it does not edit product code.
653
1053
  let orchestrator: OrchestratorHandle | undefined;
654
- try {
655
- orchestrator = await startOrchestrator({ cwd: stateDir(), brief });
656
- const transcript = orchestrator.sessionFile();
657
- log(`orchestrator session ready${transcript === undefined ? "" : ` · ${transcript}`}`);
658
- } catch (err) {
659
- // Loudly, but not fatally: tier-1 escalations degrade to issue comments,
660
- // which a human still reads. A dispatcher that refuses to run because its
661
- // re-briefing channel is down helps nobody.
662
- log(
663
- `WARNING: orchestrator session failed to start; tier-1 escalations will fall back to issue comments: ${errText(err)}`,
664
- );
1054
+ if (project.escalation.orchestrator === "external") {
1055
+ // An operator already runs the brain — typically a visible TUI session that
1056
+ // drains `blocked`/`failed` off the tracker as one of its standing duties.
1057
+ // Starting a second one here would re-triage the same issues from a
1058
+ // transcript nobody is watching, and the two would undo each other.
1059
+ log("orchestrator: external tier-1 escalations post as issue comments for the external session's drain duty");
1060
+ } else {
1061
+ try {
1062
+ orchestrator = await startOrchestrator({ cwd: stateDir(), brief });
1063
+ const transcript = orchestrator.sessionFile();
1064
+ log(`orchestrator session ready${transcript === undefined ? "" : ` · ${transcript}`}`);
1065
+ } catch (err) {
1066
+ // Loudly, but not fatally: tier-1 escalations degrade to issue comments,
1067
+ // which a human still reads. A dispatcher that refuses to run because its
1068
+ // re-briefing channel is down helps nobody.
1069
+ log(
1070
+ `WARNING: orchestrator session failed to start; tier-1 escalations will fall back to issue comments: ${errText(err)}`,
1071
+ );
1072
+ }
665
1073
  }
666
1074
 
667
1075
  const escalator = createEscalator(project, tracker, store, orchestrator);
@@ -671,6 +1079,10 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
671
1079
  tracker,
672
1080
  store,
673
1081
  escalate: (e) => escalator.escalate(e),
1082
+ integrity,
1083
+ // Fresh per daemon run, like the integrity gate: a restart is entitled to
1084
+ // page again about a stall that is still on disk.
1085
+ stall: { paged: false },
674
1086
  };
675
1087
 
676
1088
  if (o.once) {
package/src/lifecycle.ts CHANGED
@@ -22,11 +22,13 @@ import { join } from "node:path";
22
22
 
23
23
  /**
24
24
  * Mirrors `DEFAULT_PORT` in ./daemon.ts. Duplicated rather than imported so
25
- * this module stays free of the dispatcher's dependency tree.
26
- * // ponytail: two constants that must agree. If a third caller ever needs it,
27
- * // move it into ./types.ts and import it in both places.
25
+ * this module stays free of the dispatcher's dependency tree; exported so the
26
+ * CLI can name the port a foreground daemon will serve on without adding a
27
+ * third literal.
28
+ * // ponytail: two constants that must agree. If a third *definition* ever
29
+ * // appears, move it into ./types.ts and import it everywhere.
28
30
  */
29
- const DEFAULT_PORT = 8787;
31
+ export const DEFAULT_PORT = 8787;
30
32
 
31
33
  /** How long `startDaemon` waits for the first successful `/healthz`. */
32
34
  const READY_TIMEOUT_MS = 15_000;