omp-conductor 0.2.2 → 0.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/daemon.ts CHANGED
@@ -7,11 +7,13 @@
7
7
  * of it, so concurrency, daily volume, spend and per-issue attempts are counted
8
8
  * here and enforced before anything is claimed.
9
9
  */
10
- import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
11
- import { dirname, join } from "node:path";
10
+ import { createHash } from "node:crypto";
11
+ import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
12
+ import { dirname, join, relative } from "node:path";
12
13
  import { configPath, findProject, loadConfig, resolveCaps, stateDir } from "./config.ts";
13
14
  import { createEscalator } from "./escalate.ts";
14
15
  import { livingDaemon } from "./lifecycle.ts";
16
+ import { STALL_MARKER_FILE } from "./orchestrator-tick.ts";
15
17
  import { startOrchestrator } from "./orchestrator.ts";
16
18
  import type { OrchestratorHandle } from "./orchestrator.ts";
17
19
  import { branchName, route } from "./routing.ts";
@@ -28,8 +30,15 @@ import type {
28
30
  Store,
29
31
  Tracker,
30
32
  } from "./types.ts";
31
- import { renderBrief, runWorker } from "./worker.ts";
32
- import { addWorktree, mirrorPathFor, removeWorktree, worktreePathFor } from "./worktree.ts";
33
+ import { type KilledBy, renderBrief, runWorker } from "./worker.ts";
34
+ import {
35
+ addWorktree,
36
+ mirrorPathFor,
37
+ removeWorktree,
38
+ salvageWip,
39
+ type SalvageOutcome,
40
+ worktreePathFor,
41
+ } from "./worktree.ts";
33
42
 
34
43
  /** Long enough that the tracker is not polled raw, short enough that a human
35
44
  * who labels an issue sees it picked up within a coffee break. */
@@ -61,6 +70,8 @@ interface Deps {
61
70
  tracker: Tracker;
62
71
  store: Store;
63
72
  escalate(e: Escalation): Promise<void>;
73
+ integrity: IntegrityGate;
74
+ stall: StallGate;
64
75
  }
65
76
 
66
77
  // ---------------------------------------------------------------- paths & pause
@@ -70,6 +81,100 @@ export function dbPath(): string {
70
81
  return join(stateDir(), "conductor.db");
71
82
  }
72
83
 
84
+ // ------------------------------------------------------- orchestrator liveness
85
+
86
+ /**
87
+ * Whether the wedged-orchestrator page has already gone out for the stall
88
+ * currently on disk. One page per episode: the marker persists until a tick is
89
+ * consumed, so paging per five minutes would be paging forever.
90
+ */
91
+ export interface StallGate {
92
+ paged: boolean;
93
+ }
94
+
95
+ export interface StallVerdict {
96
+ /** The marker's own line, when one is there. */
97
+ since?: string;
98
+ page: boolean;
99
+ }
100
+
101
+ /**
102
+ * Reads the orchestrator's stall marker and decides whether this tick pages.
103
+ *
104
+ * The marker is written by the tick extension inside the orchestrator session
105
+ * ({@link STALL_MARKER_FILE}) when two of its own prompts go unconsumed — the
106
+ * one signal that separates "the process is alive" from "the loop is reading
107
+ * its queue". Every other guard in this system reads healthy through a wedge:
108
+ * the herdr recovery plugin tests for a live process and an agent label, both
109
+ * of which survive it, and `/healthz` describes this daemon, which is a
110
+ * different process entirely.
111
+ *
112
+ * The daemon is the natural watcher precisely because it is that different
113
+ * process: it already wakes every five minutes, it owns a working escalation
114
+ * path, and nothing about its health depends on the session that is stuck. A
115
+ * wedged loop cannot page for itself, and the herdr plugin only runs on session
116
+ * lifecycle events — a session that stays alive and stops working emits none.
117
+ *
118
+ * Resets when the marker disappears, so a second stall days later pages again.
119
+ */
120
+ export function checkStall(gate: StallGate, marker: string): StallVerdict {
121
+ if (!existsSync(marker)) {
122
+ gate.paged = false;
123
+ return { page: false };
124
+ }
125
+ const page = !gate.paged;
126
+ let since: string | undefined;
127
+ try {
128
+ const body = readFileSync(marker, "utf8").split("\n")[0]?.trim();
129
+ if (body !== undefined && body !== "") since = body;
130
+ } catch {
131
+ // An unreadable marker still means stalled; the timestamp is a nicety.
132
+ }
133
+ return { ...(since === undefined ? {} : { since }), page };
134
+ }
135
+
136
+ /**
137
+ * Pages tier 2 once when the orchestrator session stops draining its queue.
138
+ *
139
+ * Deliberately does not restart anything. A wedge lands mid-turn, this process
140
+ * cannot tell a half-applied edit from an idle loop, and killing the session
141
+ * could destroy work an operator would rather read first — the same refusal to
142
+ * guess that the recovery plugin is built on.
143
+ */
144
+ async function watchOrchestrator(d: Deps): Promise<void> {
145
+ const marker = join(stateDir(), STALL_MARKER_FILE);
146
+ const verdict = checkStall(d.stall, marker);
147
+ if (!verdict.page) return;
148
+
149
+ log(`ERROR: the orchestrator session is not draining its queue — ${verdict.since ?? "no timestamp"}`);
150
+ const delivered = await safeEscalate(d, {
151
+ tier: 2,
152
+ project: d.project.name,
153
+ issue: NO_ISSUE,
154
+ // Keyed on the marker's own timestamp, not the date. The dedup ledger keys
155
+ // on this summary, and two wedges in one day is not a hypothetical — the
156
+ // failure mode is a session that gets stuck, gets restarted, and gets stuck
157
+ // again on the same cause an hour later. A day-keyed summary would report
158
+ // the first and silently swallow every one after it.
159
+ summary:
160
+ `Orchestrator session wedged (${verdict.since ?? `marker at ${marker}`}) — ` +
161
+ `it has stopped reading its queue (${d.project.name})`,
162
+ detail: [
163
+ verdict.since ?? "Marker present with no readable timestamp.",
164
+ `Marker: ${marker}`,
165
+ "",
166
+ "Its process and its herdr agent label are both healthy, which is why nothing else noticed:",
167
+ "the loop is alive and consuming nothing, so ticks and your messages queue behind it unread.",
168
+ "",
169
+ "Attach and look before you act — a wedge lands mid-turn. Then SIGTERM the omp process:",
170
+ "herdr-conductor resumes it by exact identity, and the first consumed tick clears this marker.",
171
+ "",
172
+ "Dispatch is unaffected: workers keep running. What stops is drain, groom, report and merge.",
173
+ ].join("\n"),
174
+ });
175
+ markPaged(d.stall, delivered);
176
+ }
177
+
73
178
  /**
74
179
  * Pause is a file rather than process state on purpose: `omp-conductor pause`
75
180
  * and `/conductor pause` run in a different process from the daemon, and a flag
@@ -90,6 +195,99 @@ export function setPaused(v: boolean): void {
90
195
  }
91
196
  }
92
197
 
198
+ // ----------------------------------------------------------- package integrity
199
+
200
+ /** Enough differing paths to tell a deploy from a tamper at a glance; the full
201
+ * list is on the host, and the answer is always "go look at the host". */
202
+ const INTEGRITY_SAMPLE = 5;
203
+
204
+ /**
205
+ * What the daemon booted with, and whether it has already paged about losing
206
+ * it. Lives exactly as long as one `runDaemon()` call — which is the whole
207
+ * trick: a restart re-records both.
208
+ */
209
+ export interface IntegrityGate {
210
+ baseline: Map<string, string>;
211
+ paged: boolean;
212
+ }
213
+
214
+ export interface IntegrityVerdict {
215
+ /** Labelled, sorted differences; empty when the package is untouched. */
216
+ diff: string[];
217
+ /** Any difference at all stops the fleet. */
218
+ pause: boolean;
219
+ /** First divergent tick only — a page every five minutes is a page nobody reads. */
220
+ page: boolean;
221
+ }
222
+
223
+ /**
224
+ * sha256 of every source file the running package is made of, keyed by path
225
+ * relative to `root`.
226
+ *
227
+ * `import.meta.dir` is the installed `src/` of the code executing right now, so
228
+ * this is a self-portrait: what was actually deployed, not what some checkout
229
+ * on disk happens to contain. `.ts` and `.md` because both are executable in
230
+ * this package — the briefs under `src/briefs/` are the sessions' instructions,
231
+ * and rewriting one of those buys more than rewriting the dispatcher does.
232
+ * (A checkout also carries `*.test.ts`, which the published package excludes, so
233
+ * a daemon started from one is watching its tests too. That is the honest
234
+ * answer — its code did change — and it costs nothing on a real install.)
235
+ *
236
+ * Walking and hashing the ~30 files of this package measures 0.6 ms warm, once
237
+ * per five-minute tick, so a tick does it inline. No cache and no mtime
238
+ * shortcut on purpose: a cache is a second thing that can be wrong, and mtime
239
+ * is the first field anyone covering their tracks restores.
240
+ */
241
+ export function packageManifest(root: string = import.meta.dir): Map<string, string> {
242
+ const out = new Map<string, string>();
243
+ const walk = (dir: string): void => {
244
+ for (const e of readdirSync(dir, { withFileTypes: true })) {
245
+ const full = join(dir, e.name);
246
+ if (e.isDirectory()) walk(full);
247
+ else if (e.isFile() && (e.name.endsWith(".ts") || e.name.endsWith(".md")))
248
+ out.set(relative(root, full), createHash("sha256").update(readFileSync(full)).digest("hex"));
249
+ }
250
+ };
251
+ walk(root);
252
+ return out;
253
+ }
254
+
255
+ /**
256
+ * Labelled rather than three arrays because every consumer — the log line, the
257
+ * page, the test — wants one readable list of what moved.
258
+ */
259
+ export function manifestDiff(before: Map<string, string>, after: Map<string, string>): string[] {
260
+ const out: string[] = [];
261
+ for (const [path, hash] of before) {
262
+ const now = after.get(path);
263
+ if (now === undefined) out.push(`removed ${path}`);
264
+ else if (now !== hash) out.push(`changed ${path}`);
265
+ }
266
+ for (const path of after.keys()) if (!before.has(path)) out.push(`added ${path}`);
267
+ return out.sort();
268
+ }
269
+
270
+ /**
271
+ * The tick's decision, split from its effects so the once-only page is a thing
272
+ * a test can hold.
273
+ *
274
+ * `pause` stays true on every divergent tick, deliberately: an operator who
275
+ * resumes without restarting gets re-paused, because the boundary is still
276
+ * broken. `page` asks whether this tick should *try* — the caller latches the
277
+ * gate with {@link markPaged} only once a page actually went out, so a Telegram
278
+ * outage during the one tick that noticed does not buy permanent silence.
279
+ */
280
+ export function checkIntegrity(gate: IntegrityGate, current: Map<string, string>): IntegrityVerdict {
281
+ const diff = manifestDiff(gate.baseline, current);
282
+ if (diff.length === 0) return { diff, pause: false, page: false };
283
+ return { diff, pause: true, page: !gate.paged };
284
+ }
285
+
286
+ /** Latch a once-only page, after delivery is confirmed and never before. */
287
+ export function markPaged(gate: { paged: boolean }, delivered: boolean): void {
288
+ if (delivered) gate.paged = true;
289
+ }
290
+
93
291
  // ---------------------------------------------------------------------- helpers
94
292
 
95
293
  function log(msg: string): void {
@@ -158,13 +356,76 @@ async function swapLabel(tracker: Tracker, issue: number, from: string, to: stri
158
356
  * The escalator throws when no transport is configured or Telegram rejects, and
159
357
  * only records the dedup marker on success. A page that cannot be delivered
160
358
  * must not take the tick down with it — log it and let the next tick retry.
359
+ *
360
+ * Returns whether it actually went out, because "page once" and "page once
361
+ * *successfully*" are different promises: a caller that latches a once-only
362
+ * gate on the attempt turns one failed delivery into permanent silence about a
363
+ * condition that is still true.
161
364
  */
162
- async function safeEscalate(d: Deps, e: Escalation): Promise<void> {
365
+ async function safeEscalate(d: Pick<Deps, "escalate">, e: Escalation): Promise<boolean> {
163
366
  try {
164
367
  await d.escalate(e);
368
+ return true;
165
369
  } catch (err) {
166
370
  log(`escalation for #${e.issue} could not be delivered: ${errText(err)}`);
371
+ return false;
372
+ }
373
+ }
374
+
375
+ /**
376
+ * What a salvage attempt contributes to the escalation: where the work went, or
377
+ * that it went nowhere. Split from the effects below for the same reason
378
+ * `checkIntegrity` is — this wording is the whole thing a human acts on, so it
379
+ * is worth a test holding it, and the sha in it is the only pointer to work
380
+ * that no longer has any other copy.
381
+ */
382
+ export function salvageLines(outcome: SalvageOutcome, worktree: string): string[] {
383
+ const kept = `Worktree kept for inspection: ${worktree}`;
384
+
385
+ if (outcome.kind === "nothing") return [`${kept} — nothing uncommitted to salvage`];
386
+
387
+ if (outcome.kind === "failed") {
388
+ return [
389
+ `WIP SALVAGE FAILED: ${outcome.error}`,
390
+ `Uncommitted work in ${worktree} is the only copy of it, and the next attempt removes that tree.`,
391
+ ];
167
392
  }
393
+
394
+ return [
395
+ `WIP committed to ${outcome.branch} @ ${outcome.sha}` +
396
+ (outcome.pushed
397
+ ? " and pushed — the work outlives this worktree"
398
+ : ` but NOT pushed (${outcome.pushError ?? "no reason given"}) — it lives only in this host's mirror`),
399
+ kept,
400
+ ];
401
+ }
402
+
403
+ /**
404
+ * Commits and pushes whatever a dead run left uncommitted, logs the outcome and
405
+ * returns the escalation lines that say where that work now lives.
406
+ *
407
+ * Only ever called on a non-graceful end — a cap kill, a crashed session, a
408
+ * dispatch error. A `blocked` run stopped on purpose, with turns still in hand
409
+ * and a brief that tells it to report rather than push, so nothing is committed
410
+ * behind its back. The rest never got the chance: the kill is external and
411
+ * lands mid-edit, in the tree the next attempt removes `--force`.
412
+ */
413
+ async function salvage(
414
+ issue: number,
415
+ attempt: number,
416
+ reason: string,
417
+ worktree: string,
418
+ ): Promise<string[]> {
419
+ const lines = salvageLines(await salvageWip(worktree, issue, attempt, reason), worktree);
420
+ log(`#${issue} salvage: ${lines.join(" ")}`);
421
+ return lines;
422
+ }
423
+
424
+ /** How a run's end is named — in the salvage commit, and to whoever reads it. */
425
+ function endedBy(killedBy: KilledBy | undefined): string {
426
+ if (killedBy === "turns") return "the turns cap";
427
+ if (killedBy === "wallclock") return "the wall-clock cap";
428
+ return "a failed run";
168
429
  }
169
430
 
170
431
  async function buildBrief(
@@ -202,6 +463,10 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
202
463
 
203
464
  let claimed = false;
204
465
  let run: RunRecord | undefined;
466
+ // Hoisted out of the try so the catch path can still name the tree: a crash
467
+ // mid-dispatch is one of the non-graceful ends whose uncommitted work has to
468
+ // be salvaged too, and it is the path least likely to have committed first.
469
+ let worktreePath: string | undefined;
205
470
 
206
471
  try {
207
472
  // Claim on the tracker FIRST, before any local work. The label — not the
@@ -234,7 +499,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
234
499
  const mirrorPath = mirrorPathFor(r.repo, project.mirrorRoot);
235
500
  await removeWorktree(mirrorPath, worktreePathFor(project.workspaceRoot, issue));
236
501
 
237
- const worktreePath = await addWorktree(
502
+ worktreePath = await addWorktree(
238
503
  r.repo,
239
504
  project.mirrorRoot,
240
505
  project.workspaceRoot,
@@ -258,6 +523,11 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
258
523
  sessionDir,
259
524
  ...(project.workerModel === undefined ? {} : { model: project.workerModel }),
260
525
  onTurn: (n) => store.updateRun(runId, { turns: n }),
526
+ // Recorded the moment the session opens its transcript, not when the run
527
+ // ends: `omp-conductor tail` resolves an issue to a file through this row,
528
+ // and a path written at completion is a path nobody can follow live. The
529
+ // completion-time update below writes the same value again, harmlessly.
530
+ onSessionFile: (f) => store.updateRun(runId, { sessionFile: f }),
261
531
  });
262
532
 
263
533
  // A configured model the harness could not honour means this run was done by
@@ -288,6 +558,9 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
288
558
  });
289
559
  } else if (result.state === "failed" || result.state === "killed") {
290
560
  await swapLabel(tracker, issue, inProgress, project.stateLabels.failed);
561
+ // Before the escalation is composed, so it can say where the work went —
562
+ // and long before the next attempt provisions over this tree.
563
+ const salvaged = await salvage(issue, attempt, endedBy(result.killedBy), worktreePath);
291
564
  await safeEscalate(d, {
292
565
  tier: 1,
293
566
  project: project.name,
@@ -302,7 +575,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
302
575
  detail: [
303
576
  `${r.issue.title}`,
304
577
  r.issue.url,
305
- `Worktree kept for inspection: ${worktreePath}`,
578
+ ...salvaged,
306
579
  `Session: ${result.sessionFile ?? "(no transcript)"}`,
307
580
  "",
308
581
  result.report,
@@ -336,28 +609,175 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
336
609
  log(`#${issue} could not be relabelled: ${errText(relabelErr)}`);
337
610
  }
338
611
  }
612
+ // A crash lands anywhere, including mid-edit in a tree holding the only
613
+ // copy of real work. Nothing else on this path so much as looks at it.
614
+ const salvaged =
615
+ worktreePath === undefined
616
+ ? []
617
+ : await salvage(issue, attempt, "a dispatch error", worktreePath);
618
+
339
619
  await safeEscalate(d, {
340
620
  tier: 1,
341
621
  project: project.name,
342
622
  issue,
343
623
  runId: run?.id,
344
624
  summary: `#${issue} could not be dispatched on attempt ${attempt}`,
345
- detail,
625
+ detail: salvaged.length === 0 ? detail : [detail, "", ...salvaged].join("\n"),
346
626
  });
347
627
  // The worktree, if one was created, is deliberately left in place: this is
348
- // a failure path.
628
+ // a failure path, and whatever it still held is now a commit on the branch.
629
+ }
630
+ }
631
+
632
+ // -------------------------------------------------------------------- admission
633
+
634
+ /** A candidate cleared for dispatch, with the attempt number it will run as. */
635
+ export interface Admission {
636
+ r: Routed;
637
+ attempt: number;
638
+ }
639
+
640
+ /**
641
+ * Which routed candidates get a worker this tick — in queue order, never more
642
+ * than `slots` of them.
643
+ *
644
+ * Exported so the admission rules can be pinned without spawning a worker.
645
+ * Every one of them exists because of a live incident, and each guards a
646
+ * different way the same issue gets worked twice.
647
+ *
648
+ * Takes the slice of `Deps` it actually reads rather than the whole thing: what
649
+ * admission is allowed to consult is the point of the function, and a `Deps`
650
+ * that grows a field has no business breaking these tests.
651
+ */
652
+ export async function admitCandidates(
653
+ d: Pick<Deps, "project" | "caps" | "tracker" | "store" | "escalate">,
654
+ routed: Routed[],
655
+ slots: number,
656
+ ): Promise<Admission[]> {
657
+ const { project, caps, tracker, store } = d;
658
+ const busy = new Set(store.activeRuns(project.name).map((r) => r.issue));
659
+
660
+ const admitted: Admission[] = [];
661
+ for (const r of routed) {
662
+ if (admitted.length >= slots) break;
663
+ if (busy.has(r.issue.number)) continue;
664
+
665
+ const prior = store.attemptsFor(project.name, r.issue.number);
666
+ if (prior >= caps.maxAttemptsPerIssue) {
667
+ await safeEscalate(d, {
668
+ tier: 1,
669
+ project: project.name,
670
+ issue: r.issue.number,
671
+ summary: `#${r.issue.number} has used all ${caps.maxAttemptsPerIssue} attempts`,
672
+ detail: [
673
+ r.issue.title,
674
+ r.issue.url,
675
+ "Another attempt almost always means the issue itself is underspecified.",
676
+ "Rewrite the acceptance criteria, or take it off the queue.",
677
+ ].join("\n"),
678
+ });
679
+ continue;
680
+ }
681
+
682
+ // The busy set is built from run rows, so it can only speak for work this
683
+ // database recorded. Work pushed before this store existed — a migration, a
684
+ // wiped or relocated state dir, a restore onto a new host — looks exactly
685
+ // like fresh work, and a worker sent at it re-implements a finished PR. The
686
+ // tracker is the only party that remembers, so it is asked. The cost is
687
+ // bounded by free slots, not by queue depth: the call sits behind the two
688
+ // cheap local filters and the loop stops once the slots are full.
689
+ let closer: string | undefined;
690
+ try {
691
+ closer = await tracker.openCloserFor(r.issue.number);
692
+ } catch (err) {
693
+ // Fail closed, per candidate. An API error means "unknown whether
694
+ // finished work exists", and admitting on unknown recreates precisely the
695
+ // duplicate-work failure this guard exists to kill: the worst case of
696
+ // holding is a five-minute delay, the worst case of admitting is a burned
697
+ // attempt and a second PR on the same issue. Holding one candidate rather
698
+ // than aborting the loop is what keeps a transient GitHub failure from
699
+ // deadlocking the whole dispatcher; the next tick retries by itself.
700
+ log(`#${r.issue.number} held: open-PR check failed (${errText(err)}) — retrying next tick`);
701
+ continue;
702
+ }
703
+ if (closer !== undefined) {
704
+ log(`#${r.issue.number} skipped: open PR ${closer} already closes it`);
705
+ continue;
706
+ }
707
+
708
+ admitted.push({ r, attempt: prior + 1 });
349
709
  }
710
+
711
+ return admitted;
350
712
  }
351
713
 
352
714
  // ----------------------------------------------------------------------- a tick
353
715
 
354
716
  async function tick(d: Deps): Promise<void> {
717
+ // Before the pause check, deliberately. This one is not about dispatch: the
718
+ // orchestrator is a different process, and it can be wedged while this fleet
719
+ // is paused — which is exactly the state the reference fleet was in when the
720
+ // failure happened. A pause silences claiming, not the operator's right to
721
+ // know their supervising session stopped reading its queue.
722
+ await watchOrchestrator(d);
723
+
355
724
  // A paused fleet claims nothing. Checked first so pausing takes effect on the
356
725
  // next tick without signalling the process.
357
726
  if (isPaused()) return;
358
727
 
359
728
  const { project, caps, store } = d;
360
729
 
730
+ // "Nobody patches the running conductor" is a hard boundary in both briefs —
731
+ // which makes it prompt text, and prompt text is a request. This is the half
732
+ // that does not negotiate: the package that dispatched the last worker must
733
+ // still be the package on disk, or nothing else this tick does is
734
+ // attributable. A legitimate deploy never trips it, because installing a new
735
+ // build and restarting the unit re-records the baseline from the new files;
736
+ // only an edit *underneath* a live daemon diverges from it.
737
+ //
738
+ // Below the pause gate on purpose, unlike the stall watch above. The property
739
+ // being defended is that no work is dispatched under a package the operator
740
+ // did not install — and a paused fleet dispatches nothing, so nothing needs
741
+ // attributing yet. Tampering during a pause is not missed, only deferred: the
742
+ // baseline is boot's, so the first tick after `resume` compares against it and
743
+ // pauses again before claiming anything. Checking above the gate instead would
744
+ // page on every legitimate build an operator deploys into a parked fleet,
745
+ // which is exactly when they deploy them.
746
+ const integrity = checkIntegrity(d.integrity, packageManifest());
747
+ if (integrity.pause) {
748
+ const shown = integrity.diff.slice(0, INTEGRITY_SAMPLE);
749
+ log(
750
+ `ERROR: the installed conductor changed under this daemon — ${integrity.diff.length} file(s) differ ` +
751
+ `(${shown.join(", ")}${integrity.diff.length > shown.length ? ", …" : ""}) — pausing`,
752
+ );
753
+ setPaused(true);
754
+ if (integrity.page) {
755
+ const delivered = await safeEscalate(d, {
756
+ tier: 2,
757
+ project: project.name,
758
+ issue: NO_ISSUE,
759
+ // Dated for the same reason the spend cap is: the dedup key is the
760
+ // summary, and a second tamper months later must not be swallowed as a
761
+ // repeat of the first.
762
+ summary:
763
+ `Installed conductor changed under a running daemon on ${new Date().toISOString().slice(0, 10)}: ` +
764
+ `${integrity.diff.length} file(s) differ (first: ${integrity.diff[0]}) — ${project.name} is paused`,
765
+ detail: [
766
+ `Package root: ${import.meta.dir}`,
767
+ ...shown,
768
+ ...(integrity.diff.length > shown.length ? [`… and ${integrity.diff.length - shown.length} more`] : []),
769
+ "",
770
+ "If you deployed a new build, restart the daemon — the restart re-records the baseline.",
771
+ "If you did not, the host edited itself while it was dispatching work: treat every run since",
772
+ "the last known-good restart as unattributable before resuming.",
773
+ "`omp-conductor resume` alone will not hold — the next tick re-pauses while the files differ.",
774
+ ].join("\n"),
775
+ });
776
+ markPaged(d.integrity, delivered);
777
+ }
778
+ return;
779
+ }
780
+
361
781
  // route() filters the queue through isEligible() itself, so anything already
362
782
  // carrying a state label is gone before it gets here.
363
783
  const { routed, unroutable } = route(await d.tracker.listReady(), project);
@@ -405,8 +825,8 @@ async function tick(d: Deps): Promise<void> {
405
825
 
406
826
  // Two different questions, deliberately two queries. Capacity counts worker
407
827
  // *processes*, so a green PR awaiting a human merge must not consume a slot —
408
- // two of those would otherwise stop the fleet. The busy set protects *issues*,
409
- // so that same green PR must be in it, or a second attempt lands on a live PR.
828
+ // two of those would otherwise stop the fleet. That same PR's *issue* must
829
+ // still be occupied, which is what `admitCandidates`' busy set is for.
410
830
  const live = store.liveRuns(project.name);
411
831
  const slots = caps.maxConcurrentWorkers - live.length;
412
832
  if (slots <= 0) {
@@ -414,32 +834,7 @@ async function tick(d: Deps): Promise<void> {
414
834
  return;
415
835
  }
416
836
 
417
- const busy = new Set(store.activeRuns(project.name).map((r) => r.issue));
418
-
419
- const admitted: { r: Routed; attempt: number }[] = [];
420
- for (const r of routed) {
421
- if (admitted.length >= slots) break;
422
- if (busy.has(r.issue.number)) continue;
423
-
424
- const prior = store.attemptsFor(project.name, r.issue.number);
425
- if (prior >= caps.maxAttemptsPerIssue) {
426
- await safeEscalate(d, {
427
- tier: 1,
428
- project: project.name,
429
- issue: r.issue.number,
430
- summary: `#${r.issue.number} has used all ${caps.maxAttemptsPerIssue} attempts`,
431
- detail: [
432
- r.issue.title,
433
- r.issue.url,
434
- "Another attempt almost always means the issue itself is underspecified.",
435
- "Rewrite the acceptance criteria, or take it off the queue.",
436
- ].join("\n"),
437
- });
438
- continue;
439
- }
440
-
441
- admitted.push({ r, attempt: prior + 1 });
442
- }
837
+ const admitted = await admitCandidates(d, routed, slots);
443
838
 
444
839
  if (admitted.length === 0) return;
445
840
 
@@ -612,6 +1007,15 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
612
1007
  const store = openStore(dbPath());
613
1008
  const tracker = makeTracker(project);
614
1009
 
1010
+ // Recorded here, before a single tick runs, so that the deploy an operator
1011
+ // *means* to do never trips the tripwire: installing a new build and
1012
+ // restarting the unit re-records this from the new files. What it catches is
1013
+ // the other thing — the package changing while the daemon that dispatches
1014
+ // work is holding it open, whether that is a worker that wandered out of its
1015
+ // worktree or a human editing the live install "just to test something".
1016
+ const integrity: IntegrityGate = { baseline: packageManifest(), paged: false };
1017
+ log(`package integrity baseline: ${integrity.baseline.size} files under ${import.meta.dir}`);
1018
+
615
1019
  // Before the first tick, settle what the last process left behind — unless
616
1020
  // another daemon is alive (a foreground `daemon --once` beside a running
617
1021
  // daemon must not orphan that daemon's real, live workers).
@@ -641,8 +1045,16 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
641
1045
  "fails twice, its gates stay red, its branch conflicts, or a tripwire fires.",
642
1046
  "Your job when that happens: re-brief the issue (comment what the next worker must do",
643
1047
  `differently, then put ${project.queueLabel} back on it), file follow-up issues, or promote to`,
644
- "tier 2 and let the human decide. You never edit product code, push a branch, or merge a PR —",
645
- "a worker session does all of that. Handle each escalation below before the next one.",
1048
+ "tier 2 and let the human decide.",
1049
+ // Worded from `authority.merge` rather than fixed, so the standing orders
1050
+ // and the Releases section of the rendered brief cannot disagree about who
1051
+ // is holding the merge button. The daemon still merges nothing itself.
1052
+ project.authority.merge === "orchestrator"
1053
+ ? "You never edit product code or push a branch — a worker session does that. Merging is yours: one PR at " +
1054
+ "a time, freshness-checked against the base branch, per the Releases section of your ORCHESTRATOR.md."
1055
+ : "You never edit product code, push a branch, or merge a PR — a worker session edits and pushes, and a " +
1056
+ "human merges.",
1057
+ "Handle each escalation below before the next one.",
646
1058
  ].join("\n");
647
1059
 
648
1060
  // One orchestrator per daemon run, not per tick: it is a persistent session
@@ -651,17 +1063,25 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
651
1063
  // directory, deliberately not a checkout — the orchestrator re-briefs workers
652
1064
  // and talks to the tracker, it does not edit product code.
653
1065
  let orchestrator: OrchestratorHandle | undefined;
654
- try {
655
- orchestrator = await startOrchestrator({ cwd: stateDir(), brief });
656
- const transcript = orchestrator.sessionFile();
657
- log(`orchestrator session ready${transcript === undefined ? "" : ` · ${transcript}`}`);
658
- } catch (err) {
659
- // Loudly, but not fatally: tier-1 escalations degrade to issue comments,
660
- // which a human still reads. A dispatcher that refuses to run because its
661
- // re-briefing channel is down helps nobody.
662
- log(
663
- `WARNING: orchestrator session failed to start; tier-1 escalations will fall back to issue comments: ${errText(err)}`,
664
- );
1066
+ if (project.escalation.orchestrator === "external") {
1067
+ // An operator already runs the brain — typically a visible TUI session that
1068
+ // drains `blocked`/`failed` off the tracker as one of its standing duties.
1069
+ // Starting a second one here would re-triage the same issues from a
1070
+ // transcript nobody is watching, and the two would undo each other.
1071
+ log("orchestrator: external tier-1 escalations post as issue comments for the external session's drain duty");
1072
+ } else {
1073
+ try {
1074
+ orchestrator = await startOrchestrator({ cwd: stateDir(), brief });
1075
+ const transcript = orchestrator.sessionFile();
1076
+ log(`orchestrator session ready${transcript === undefined ? "" : ` · ${transcript}`}`);
1077
+ } catch (err) {
1078
+ // Loudly, but not fatally: tier-1 escalations degrade to issue comments,
1079
+ // which a human still reads. A dispatcher that refuses to run because its
1080
+ // re-briefing channel is down helps nobody.
1081
+ log(
1082
+ `WARNING: orchestrator session failed to start; tier-1 escalations will fall back to issue comments: ${errText(err)}`,
1083
+ );
1084
+ }
665
1085
  }
666
1086
 
667
1087
  const escalator = createEscalator(project, tracker, store, orchestrator);
@@ -671,6 +1091,10 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
671
1091
  tracker,
672
1092
  store,
673
1093
  escalate: (e) => escalator.escalate(e),
1094
+ integrity,
1095
+ // Fresh per daemon run, like the integrity gate: a restart is entitled to
1096
+ // page again about a stall that is still on disk.
1097
+ stall: { paged: false },
674
1098
  };
675
1099
 
676
1100
  if (o.once) {