omp-conductor 0.13.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/README.md +549 -234
  2. package/package.json +8 -5
  3. package/schema/config.schema.json +609 -0
  4. package/src/availability.ts +165 -0
  5. package/src/board.ts +19 -32
  6. package/src/brief-upgrade.ts +1 -1
  7. package/src/briefs/orchestrator.md +72 -31
  8. package/src/briefs/policy.md +48 -36
  9. package/src/briefs/probes/gates.md +51 -0
  10. package/src/briefs/probes/project-context.md +59 -0
  11. package/src/briefs/probes/release-procedure.md +81 -0
  12. package/src/cli.ts +356 -212
  13. package/src/config-schema.ts +352 -0
  14. package/src/config.ts +1037 -679
  15. package/src/confinement.ts +54 -0
  16. package/src/daemon.ts +644 -390
  17. package/src/diff-flags.ts +73 -4
  18. package/src/digest-schedule.ts +92 -24
  19. package/src/escalate.ts +89 -22
  20. package/src/fleet.ts +351 -46
  21. package/src/generate-schema.ts +21 -0
  22. package/src/graph.ts +3 -3
  23. package/src/host.ts +16 -0
  24. package/src/omp.ts +21 -1
  25. package/src/orchestrator-tick.ts +732 -56
  26. package/src/privileged.ts +264 -0
  27. package/src/reports.ts +203 -6
  28. package/src/session-host.ts +3 -0
  29. package/src/setup-host.ts +209 -24
  30. package/src/setup-install.ts +320 -0
  31. package/src/setup-probe.ts +412 -0
  32. package/src/setup-wizard.ts +1946 -0
  33. package/src/setup.ts +457 -53
  34. package/src/store.ts +610 -98
  35. package/src/tracker/github.ts +43 -5
  36. package/src/types.ts +153 -14
  37. package/src/upgrade.ts +44 -10
  38. package/src/verbs/actions.ts +131 -13
  39. package/src/verbs/server.ts +40 -18
  40. package/src/wizard-ui.ts +249 -0
  41. package/src/worker.ts +24 -7
  42. package/skills/conductor-onboarding/SKILL.md +0 -748
  43. package/skills/conductor-update/SKILL.md +0 -51
  44. package/src/plugin.ts +0 -1495
package/src/daemon.ts CHANGED
@@ -18,19 +18,26 @@ import {
18
18
  resolveReleaseGrants,
19
19
  stateDir,
20
20
  } from "./config.ts";
21
+ import { availabilityState, type AvailabilityState } from "./availability.ts";
21
22
  import {
22
23
  analyseSettlement,
23
24
  formatSettlementFlags,
24
25
  settlementFlagSummary,
25
26
  } from "./diff-flags.ts";
27
+ import { digestScheduleState, type DigestScheduleState } from "./digest-schedule.ts";
26
28
  import { createEscalator, escalationIssueRef } from "./escalate.ts";
27
29
  import { pendingCodeGraph, probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
28
30
  import { graphHint } from "./graph.ts";
29
31
  import { livingDaemon } from "./lifecycle.ts";
30
- import { STALL_MARKER_FILE } from "./orchestrator-tick.ts";
32
+ import { requestImmediateTick, resolveTickConfigCwd, STALL_MARKER_FILE } from "./orchestrator-tick.ts";
31
33
  import { startOrchestrator } from "./orchestrator.ts";
32
34
  import type { OrchestratorHandle } from "./orchestrator.ts";
33
- import { createReportOutbox, formatOpenReports } from "./reports.ts";
35
+ import {
36
+ createReportOutbox,
37
+ enqueueAvailableHeldNotices,
38
+ formatOpenReports,
39
+ type ReportOutbox,
40
+ } from "./reports.ts";
34
41
  import {
35
42
  recordReleaseBlock,
36
43
  type ReleaseBlockContext,
@@ -42,11 +49,13 @@ import { classifyRun, providerCreditRefusal, providerTransientFault, type Classi
42
49
  import { projectLabels } from "./label-projection.ts";
43
50
  import { dbPath, LIVE_STATES, openStore, utcDay } from "./store.ts";
44
51
  import { makeTracker, type RateLimitStatus } from "./tracker/github.ts";
45
- import { RELEASE_SHAPES } from "./types.ts";
52
+ import { DEFAULT_REPORT_POLICY, RELEASE_SHAPES } from "./types.ts";
46
53
  import type {
54
+ BaseHealth,
47
55
  AdmissionHoldReason,
48
56
  Caps,
49
57
  DispatchSummary,
58
+ DigestBacklog,
50
59
  Escalation,
51
60
  IssueSnapshot,
52
61
  MergedPrInfo,
@@ -167,6 +176,9 @@ interface Deps {
167
176
  caps: Caps;
168
177
  tracker: Tracker;
169
178
  store: Store;
179
+ /** False after a live config reload fails; autonomous delivery then holds
180
+ * fail-closed until a later tick validates the config again. */
181
+ deliveryPolicyValid?: boolean;
170
182
  /** Provider-reported plan allowance, cached with a TTL. Resolved once at
171
183
  * startup like every other dep so a tick cannot swap its own meter. */
172
184
  usage: UsageSource;
@@ -215,10 +227,10 @@ export function verbDeps(d: Pick<Deps, "project" | "store" | "tracker" | "verbAc
215
227
  tracker: d.tracker,
216
228
  actions: d.verbActions ?? githubVerbActions(d.project),
217
229
  fleetStop: () =>
218
- isPaused()
219
- ? "claiming is paused for this fleet (omp-conductor pause, hold or halt)"
230
+ isPaused(d.project.name)
231
+ ? "claiming is paused for this fleet (omp-conductor hold or stop)"
220
232
  : undefined,
221
- pausedAt,
233
+ pausedAt: () => pausedAt(d.project.name),
222
234
  log,
223
235
  now: () => Date.now(),
224
236
  chain: { readBaseChain },
@@ -364,80 +376,92 @@ export async function watchOrchestrator(d: Deps, now = Date.now()): Promise<void
364
376
  }
365
377
 
366
378
  /**
367
- * Pause is a file rather than process state on purpose: `omp-conductor pause`
368
- * and `/conductor pause` run in a different process from the daemon, and a flag
379
+ * Pause is a file rather than process state on purpose: `omp-conductor hold`
380
+ * runs in a different process from the daemon, and a flag
369
381
  * on disk needs no IPC and survives a restart. A daemon that crashed while
370
382
  * paused comes back paused.
371
383
  */
372
- export function isPaused(): boolean {
373
- return existsSync(join(stateDir(), "paused"));
384
+ export function pausedPath(project?: string): string {
385
+ return join(stateDir(), project === undefined ? "paused" : `paused-${project}`);
386
+ }
387
+
388
+ function activePausePaths(project?: string): string[] {
389
+ const paths = project === undefined ? [pausedPath()] : [pausedPath(project), pausedPath()];
390
+ return paths.filter((path) => existsSync(path));
391
+ }
392
+
393
+ export function isPaused(project?: string): boolean {
394
+ return activePausePaths(project).length !== 0;
374
395
  }
375
396
 
376
397
  /**
377
- * The epoch-ms timestamp at which the current pause began, read from the same
378
- * sentinel file {@link setPaused} writes (`<stateDir()>/paused`). Returns
379
- * `undefined` when the fleet is not paused, or when the file's first line does
380
- * not parse as a date. A legacy/blank sentinel keeps completion mutations
381
- * fail-closed because a run admitted before an *unknown* pause cannot be proven
382
- * innocent. {@link isPaused} is the authority on *whether*; this answers *since
383
- * when*.
398
+ * The epoch-ms timestamp at which the current pause began. A project pause also
399
+ * observes the legacy bare sentinel, which pauses every project. If any active
400
+ * sentinel is unreadable or unparseable, the timestamp is unknown so callers
401
+ * continue to fail closed.
384
402
  */
385
- export function pausedAt(): number | undefined {
386
- const f = join(stateDir(), "paused");
387
- if (!existsSync(f)) return undefined;
388
- try {
389
- const first = readFileSync(f, "utf8").split("\n")[0]?.trim();
390
- if (first === undefined || first === "") return undefined;
391
- const t = Date.parse(first);
392
- return Number.isNaN(t) ? undefined : t;
393
- } catch {
394
- // Unreadable sentinel (permissions, corruption): fail completion mutations
395
- // closed like an unparseable line while the pause time is unprovable.
396
- return undefined;
403
+ export function pausedAt(project?: string): number | undefined {
404
+ const paths = activePausePaths(project);
405
+ if (paths.length === 0) return undefined;
406
+ const times: number[] = [];
407
+ for (const path of paths) {
408
+ try {
409
+ const first = readFileSync(path, "utf8").split("\n")[0]?.trim();
410
+ if (first === undefined || first === "") return undefined;
411
+ const time = Date.parse(first);
412
+ if (Number.isNaN(time)) return undefined;
413
+ times.push(time);
414
+ } catch {
415
+ return undefined;
416
+ }
397
417
  }
418
+ return Math.min(...times);
398
419
  }
399
420
 
400
421
  /**
401
- * Who paused the fleet and why, read from line 2 of the same sentinel
402
- * {@link setPaused} writes — `undefined` when the fleet is not paused or the
403
- * file has no (parseable) line 2. Provenance lives on its own line so line 1
404
- * stays a pure ISO timestamp that {@link pausedAt} can `Date.parse`; the `armed
405
- * <ISO> owner=<id>` marker `armTicks` writes is the same one-key-per-line
406
- * precedent.
422
+ * Who paused the project and why. Per-project provenance wins when both its
423
+ * sentinel and the legacy all-project sentinel are active.
407
424
  */
408
- export function pauseProvenance(): { source: string; reason?: string } | undefined {
409
- const f = join(stateDir(), "paused");
410
- if (!existsSync(f)) return undefined;
425
+ export function pauseProvenance(
426
+ project?: string,
427
+ ): { source: string; reason?: string } | undefined {
428
+ const paths =
429
+ project === undefined
430
+ ? [pausedPath()]
431
+ : [pausedPath(project), pausedPath()];
432
+ const path = paths.find((candidate) => existsSync(candidate));
433
+ if (path === undefined) return undefined;
411
434
  try {
412
- const second = readFileSync(f, "utf8").split("\n")[1];
435
+ const second = readFileSync(path, "utf8").split("\n")[1];
413
436
  if (second === undefined) return undefined;
414
437
  const match = /^source=(\S+)(?: reason="(.*)")?$/.exec(second.trim());
415
438
  if (match === null) return undefined;
416
- const source = match[1]!; // the regex guarantees group 1 on a match
439
+ const source = match[1]!;
417
440
  const reason = match[2];
418
441
  return { source, ...(reason === undefined ? {} : { reason }) };
419
442
  } catch {
420
- // Unreadable sentinel: no provenance to name. Completion mutations still
421
- // fail closed because the pause time is unknown.
422
443
  return undefined;
423
444
  }
424
445
  }
425
446
 
426
- export function setPaused(v: boolean, why?: { source: string; reason?: string }): void {
427
- const f = join(stateDir(), "paused");
447
+ export function setPaused(
448
+ v: boolean,
449
+ why?: { source: string; reason?: string },
450
+ project?: string,
451
+ ): void {
452
+ const path = pausedPath(project);
428
453
  if (v) {
429
- mkdirSync(dirname(f), { recursive: true });
454
+ mkdirSync(dirname(path), { recursive: true });
430
455
  const line1 = `${new Date().toISOString()}\n`;
431
456
  if (why === undefined) {
432
- writeFileSync(f, line1);
457
+ writeFileSync(path, line1);
433
458
  } else {
434
- // Quotes are stripped before embedding so a reason cannot break out of
435
- // the `reason="..."` field of line 2.
436
459
  const reason = why.reason === undefined ? "" : ` reason="${why.reason.replaceAll('"', "")}"`;
437
- writeFileSync(f, `${line1}source=${why.source}${reason}\n`);
460
+ writeFileSync(path, `${line1}source=${why.source}${reason}\n`);
438
461
  }
439
462
  } else {
440
- rmSync(f, { force: true });
463
+ rmSync(path, { force: true });
464
+ if (project !== undefined) rmSync(pausedPath(), { force: true });
441
465
  }
442
466
  }
443
467
 
@@ -663,8 +687,8 @@ async function reactToProviderCredit(
663
687
  sessionFile: string | undefined,
664
688
  ): Promise<void> {
665
689
  const { project } = d;
666
- const alreadyPaused = isPaused();
667
- if (!alreadyPaused) setPaused(true, { source: "provider-credit", reason: message });
690
+ const alreadyPaused = isPaused(project.name);
691
+ if (!alreadyPaused) setPaused(true, { source: "provider-credit", reason: message }, project.name);
668
692
  log(
669
693
  `#${issue} provider refused for credit — dispatch ${alreadyPaused ? "remains paused" : "paused"}: ${message}`,
670
694
  );
@@ -1402,12 +1426,15 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
1402
1426
  });
1403
1427
  if (await settleStopBeforeSession()) return;
1404
1428
 
1429
+ const repoSlug = githubRepo(r.repo.cloneUrl);
1430
+
1405
1431
  let result: WorkerResult;
1406
1432
  try {
1407
1433
  result = await runWorker({
1408
1434
  brief,
1409
1435
  cwd: worktreePath,
1410
1436
  caps,
1437
+ ...(repoSlug === undefined ? {} : { repoSlug }),
1411
1438
  maxTurns: () => turnLimit?.maxTurns() ?? maxTurns,
1412
1439
  onPauseControl: (control) => {
1413
1440
  workerSessionInstalled = true;
@@ -1804,12 +1831,12 @@ export interface Settlement {
1804
1831
  *
1805
1832
  * - `merged` — the work landed. That is what `merged` was reserved for.
1806
1833
  * - `closed` — a human read the work and said no. Leaving it `pushed-green`
1807
- * strands the issue forever behind a PR nobody will ever merge, and calling it
1808
- * `merged` is simply a lie about work that does not exist on the base branch.
1809
- * `failed` is true the attempt did not land — and it releases the busy guard,
1810
- * so an issue a human re-queues can be attempted again. The attempt counter is
1811
- * untouched either way: this row was a real attempt, and pretending otherwise
1812
- * would let a rejected issue cycle past `maxAttemptsPerIssue`.
1834
+ * forever is a lie; `failed` records that it did not land and releases the
1835
+ * busy guard, so an issue a human re-queues can be attempted again. A row
1836
+ * that had reached `pushed-green` or `pushed-pending` is classified
1837
+ * `returned-for-revision` at settlement. A review decision asks for another
1838
+ * implementation pass, not a failure, so it consumes the continuation budget
1839
+ * instead of the failed-attempt budget.
1813
1840
  * - `open`, and undefined — nothing changes. Undefined is "could not tell": a
1814
1841
  * flaky network, a revoked token, a deleted PR. Settling on it would record a
1815
1842
  * merge that never happened, and the next tick asks again for free. An
@@ -1944,15 +1971,24 @@ export async function watchMergedBase(d: Pick<Deps, "project" | "tracker" | "sto
1944
1971
 
1945
1972
  let workflows;
1946
1973
  try {
1947
- workflows = await d.tracker.workflowRunsAt(repoIdentity, run.mergeSha);
1974
+ workflows = await d.tracker.workflowRunsAt(repoIdentity, run.mergeSha, {
1975
+ event: "push",
1976
+ branch: run.baseRef,
1977
+ });
1948
1978
  } catch (err) {
1949
1979
  log(`#${run.issue} base check unavailable (${errText(err)}) — retrying next tick`);
1950
1980
  continue;
1951
1981
  }
1952
- if (workflows === undefined || workflows.length === 0) {
1982
+ if (workflows === undefined) {
1953
1983
  log(`#${run.issue} base check unavailable for ${run.mergeSha} — retrying next tick`);
1954
1984
  continue;
1955
1985
  }
1986
+ if (workflows.length === 0) {
1987
+ log(
1988
+ `#${run.issue} base check: no push-triggered run yet for ${run.mergeSha} — retrying next tick`,
1989
+ );
1990
+ continue;
1991
+ }
1956
1992
  if (workflows.some((workflow) => workflow.status !== "completed")) continue;
1957
1993
 
1958
1994
  const failed = workflows.find(
@@ -2016,6 +2052,110 @@ export async function watchMergedBase(d: Pick<Deps, "project" | "tracker" | "sto
2016
2052
  }
2017
2053
  }
2018
2054
 
2055
+ /**
2056
+ * Refresh current base-branch health at each recently merged repository's live
2057
+ * head. This is status and release-gate evidence only; the per-merge audit
2058
+ * above remains the sole path that attributes and escalates a regression.
2059
+ */
2060
+ export async function watchBaseHealth(
2061
+ d: Pick<Deps, "project" | "tracker" | "store">,
2062
+ ): Promise<void> {
2063
+ const now = Date.now();
2064
+ const previousByRepo = new Map(
2065
+ d.store.baseHealth(d.project.name).map((row) => [row.repo, row] as const),
2066
+ );
2067
+ for (const { repo, baseRef } of d.store.mergedRepoBranches(
2068
+ d.project.name,
2069
+ now - BASE_STATUS_WINDOW_MS,
2070
+ )) {
2071
+ const target = d.project.routing.repos[repo];
2072
+ if (target === undefined) {
2073
+ log(`base health skipped: routed repository ${repo} is no longer configured`);
2074
+ continue;
2075
+ }
2076
+ const identity = githubRepo(target.cloneUrl);
2077
+ if (identity === undefined) {
2078
+ log(`base health skipped: routed repository ${repo} has no GitHub identity`);
2079
+ continue;
2080
+ }
2081
+ const branch = baseRef ?? target.defaultBranch;
2082
+
2083
+ let head: string | undefined;
2084
+ try {
2085
+ head = await d.tracker.branchHead(identity, branch);
2086
+ } catch (err) {
2087
+ log(`base ${repo}/${branch} head unavailable (${errText(err)}) — keeping previous health`);
2088
+ continue;
2089
+ }
2090
+ if (head === undefined) {
2091
+ log(`base ${repo}/${branch} head unavailable — keeping previous health`);
2092
+ continue;
2093
+ }
2094
+
2095
+ const previous = previousByRepo.get(repo);
2096
+ if (
2097
+ previous?.branch === branch &&
2098
+ previous.headSha === head &&
2099
+ (previous.verdict === "green" || previous.verdict === "red")
2100
+ ) {
2101
+ continue;
2102
+ }
2103
+
2104
+ let runs;
2105
+ try {
2106
+ runs = await d.tracker.workflowRunsAt(identity, head, { event: "push", branch });
2107
+ } catch (err) {
2108
+ log(`base ${repo}/${branch} workflows unavailable (${errText(err)}) — keeping previous health`);
2109
+ continue;
2110
+ }
2111
+ if (runs === undefined) {
2112
+ log(`base ${repo}/${branch} workflows unavailable — keeping previous health`);
2113
+ continue;
2114
+ }
2115
+
2116
+ let verdict: BaseHealth["verdict"];
2117
+ let detail: string | undefined;
2118
+ if (runs.length === 0) {
2119
+ verdict = "unknown";
2120
+ detail = `no push-triggered workflow run for ${head.slice(0, 8)}`;
2121
+ } else if (runs.some((run) => run.status !== "completed")) {
2122
+ verdict = "pending";
2123
+ } else {
2124
+ const failed = runs.find(
2125
+ (run) =>
2126
+ run.conclusion !== undefined &&
2127
+ FAILING_WORKFLOW_CONCLUSIONS.has(run.conclusion),
2128
+ );
2129
+ if (failed !== undefined) {
2130
+ verdict = "red";
2131
+ detail = `${failed.name} failed at ${head.slice(0, 8)} — ${failed.url}`;
2132
+ } else if (
2133
+ runs.some(
2134
+ (run) =>
2135
+ run.conclusion === undefined ||
2136
+ !SUCCESSFUL_WORKFLOW_CONCLUSIONS.has(run.conclusion),
2137
+ )
2138
+ ) {
2139
+ verdict = "pending";
2140
+ } else {
2141
+ verdict = "green";
2142
+ }
2143
+ }
2144
+
2145
+ const health: BaseHealth = {
2146
+ repo,
2147
+ branch,
2148
+ headSha: head,
2149
+ verdict,
2150
+ runsCount: runs.length,
2151
+ checkedAt: now,
2152
+ ...(detail === undefined ? {} : { detail }),
2153
+ };
2154
+ d.store.upsertBaseHealth(d.project.name, health);
2155
+ previousByRepo.set(repo, health);
2156
+ }
2157
+ }
2158
+
2019
2159
  export async function settlePushedGreen(
2020
2160
  d: Pick<Deps, "project" | "tracker" | "store">,
2021
2161
  ): Promise<void> {
@@ -2080,7 +2220,11 @@ export async function settlePushedGreen(
2080
2220
  baseCheck: "pending",
2081
2221
  }),
2082
2222
  };
2083
- if (settlement.state === "failed") patch.lastError = settlement.reason;
2223
+ if (settlement.state === "failed") {
2224
+ patch.lastError = settlement.reason;
2225
+ patch.failureClass = "returned-for-revision";
2226
+ patch.recoveryAction = "none";
2227
+ }
2084
2228
  store.updateRun(run.id, patch);
2085
2229
  log(`#${run.issue} settled: ${settlement.reason}`);
2086
2230
  continue;
@@ -2808,6 +2952,40 @@ export async function dispatchAdmissions(
2808
2952
 
2809
2953
  // ----------------------------------------------------------------------- a tick
2810
2954
 
2955
+ /**
2956
+ * After a false→true condition transition, poke the orchestrator heartbeat so
2957
+ * it does not wait a full interval (#329). Best-effort: missing tick config or
2958
+ * a failed write leaves the row flagged in the ledger for the next heartbeat.
2959
+ */
2960
+ export function wakeOrchestratorForMetConditions(
2961
+ projectName: string,
2962
+ met: readonly { id: string; condition?: string }[],
2963
+ writeLog: (line: string) => void = log,
2964
+ ): void {
2965
+ if (met.length === 0) return;
2966
+ for (const decision of met) {
2967
+ writeLog(
2968
+ `decision ${decision.id} condition met (${decision.condition ?? "?"}) — requesting orchestrator tick`,
2969
+ );
2970
+ }
2971
+ const tickCwd = resolveTickConfigCwd(projectName);
2972
+ if (tickCwd === undefined) {
2973
+ writeLog(
2974
+ `decision condition met for ${met.map((m) => m.id).join(", ")} but no tick config cwd — heartbeat will surface them on its next interval`,
2975
+ );
2976
+ return;
2977
+ }
2978
+ const ids = met.map((m) => m.id).join(",");
2979
+ const reason = `condition-met ${ids}`;
2980
+ if (requestImmediateTick(tickCwd, reason)) {
2981
+ writeLog(`requested immediate tick at ${tickCwd}: ${reason}`);
2982
+ } else {
2983
+ writeLog(
2984
+ `could not write tick request under ${tickCwd}; conditions ${ids} wait for the next heartbeat`,
2985
+ );
2986
+ }
2987
+ }
2988
+
2811
2989
  export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
2812
2990
  // A config edit takes effect on the next tick, not the next daemon restart
2813
2991
  // (#170). Re-resolve the project and its caps at the tick boundary so a tick
@@ -2823,8 +3001,26 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
2823
3001
  }
2824
3002
  d.project = fresh;
2825
3003
  d.caps = freshCaps;
3004
+ d.deliveryPolicyValid = true;
2826
3005
  } catch (err) {
2827
- log(`config reload failed (${errText(err)}) — continuing with the values loaded at boot`);
3006
+ log(`config reload failed (${errText(err)}) — retaining boot values but blocking autonomous delivery`);
3007
+ d.deliveryPolicyValid = false;
3008
+ }
3009
+
3010
+ // Availability-held notices are already durable. Once the freshly reloaded
3011
+ // policy opens (or newly allows their category), atomically hand a bounded
3012
+ // batch to the report outbox. Pauses do not suppress delivery.
3013
+ if (d.deliveryPolicyValid !== false) {
3014
+ try {
3015
+ const catchUp = enqueueAvailableHeldNotices(d.project, d.store, Date.now());
3016
+ if (catchUp !== undefined && !catchUp.deduped) {
3017
+ log(`availability catch-up ${catchUp.report.id} queued for ${d.project.name}`);
3018
+ }
3019
+ } catch (err) {
3020
+ // The daily digest may have claimed the same rows from another process
3021
+ // between selection and association. Either way the ledger still owns them.
3022
+ log(`availability catch-up handoff deferred (${errText(err)}) — retrying next tick`);
3023
+ }
2828
3024
  }
2829
3025
 
2830
3026
  // Before the pause check, deliberately. This one is not about dispatch: the
@@ -2844,6 +3040,11 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
2844
3040
  } catch (err) {
2845
3041
  log(`base-branch check sweep failed: ${errText(err)}`);
2846
3042
  }
3043
+ try {
3044
+ await watchBaseHealth(d);
3045
+ } catch (err) {
3046
+ log(`current base-health sweep failed: ${errText(err)}`);
3047
+ }
2847
3048
  try {
2848
3049
  await adoptSalvagedPrs(d);
2849
3050
  } catch (err) {
@@ -2884,11 +3085,14 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
2884
3085
  // Ledger maintenance, above the pause gate for the same reason the stall watch
2885
3086
  // is: a paused fleet still owes its operator the questions it asked, and a
2886
3087
  // condition that came true while dispatch was parked is exactly the thing the
2887
- // orchestrator has to see on its next tick.
3088
+ // orchestrator has to see promptly (#329).
2888
3089
  //
2889
3090
  // Expiry is synchronous (one UPDATE); condition evaluation is fire-and-forget
2890
3091
  // because it shells out to `gh` and `npm`, and a registry that hangs must cost
2891
- // one unevaluated condition rather than the tick.
3092
+ // one unevaluated condition rather than the tick. A false→true transition
3093
+ // writes the same immediate-tick poke recover uses so the heartbeat does not
3094
+ // wait a full interval; repeated sweeps while the condition stays true never
3095
+ // re-enter `met` (store mark is idempotent).
2892
3096
  for (const expired of d.store.expireDueDecisions(d.project.name, Date.now())) {
2893
3097
  log(`decision ${expired.id} expired unanswered after seven days: ${expired.question}`);
2894
3098
  }
@@ -2900,9 +3104,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
2900
3104
  Date.now,
2901
3105
  )
2902
3106
  .then((met) => {
2903
- for (const decision of met) {
2904
- log(`decision ${decision.id} condition met (${decision.condition ?? "?"}) — surfacing on the next tick`);
2905
- }
3107
+ wakeOrchestratorForMetConditions(d.project.name, met);
2906
3108
  })
2907
3109
  .catch((err: unknown) => {
2908
3110
  log(`decision condition pass failed: ${errText(err)}`);
@@ -2910,7 +3112,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
2910
3112
 
2911
3113
  // A paused fleet claims nothing. Checked first so pausing takes effect on the
2912
3114
  // next tick without signalling the process.
2913
- if (isPaused()) return;
3115
+ if (isPaused(d.project.name)) return;
2914
3116
 
2915
3117
  const { project, caps, store } = d;
2916
3118
 
@@ -2937,7 +3139,11 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
2937
3139
  `ERROR: the installed conductor changed under this daemon — ${integrity.diff.length} file(s) differ ` +
2938
3140
  `(${shown.join(", ")}${integrity.diff.length > shown.length ? ", …" : ""}) — pausing`,
2939
3141
  );
2940
- setPaused(true, { source: "integrity", reason: "installed package changed under the daemon" });
3142
+ setPaused(
3143
+ true,
3144
+ { source: "integrity", reason: "installed package changed under the daemon" },
3145
+ project.name,
3146
+ );
2941
3147
  if (integrity.page) {
2942
3148
  const delivered = await safeEscalate(d, {
2943
3149
  tier: 2,
@@ -3027,7 +3233,11 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
3027
3233
  // operator opted out — turns and wall-clock still brake every run (#46).
3028
3234
  const spent = store.spendSince(project.name, since);
3029
3235
  if (caps.dailySpendUsd !== null && spent >= caps.dailySpendUsd) {
3030
- setPaused(true, { source: "spend-cap", reason: `daily spend reached $${caps.dailySpendUsd}` });
3236
+ setPaused(
3237
+ true,
3238
+ { source: "spend-cap", reason: `daily spend reached $${caps.dailySpendUsd}` },
3239
+ project.name,
3240
+ );
3031
3241
  await safeEscalate(d, {
3032
3242
  tier: 2,
3033
3243
  category: "fleet-stopped",
@@ -3037,7 +3247,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
3037
3247
  summary: `Daily spend cap reached on ${new Date().toISOString().slice(0, 10)} — ${project.name} is paused`,
3038
3248
  detail: [
3039
3249
  `Spent $${spent.toFixed(2)} of the $${caps.dailySpendUsd.toFixed(2)} daily cap.`,
3040
- "No further work will be claimed until `omp-conductor resume` (or /conductor resume).",
3250
+ "No further work will be claimed until `omp-conductor resume`.",
3041
3251
  ].join("\n"),
3042
3252
  });
3043
3253
  recordDispatch(0, [
@@ -3088,25 +3298,29 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
3088
3298
  // --------------------------------------------------------------- read-only views
3089
3299
 
3090
3300
  export interface DaemonHealthSnapshot {
3091
- ok: true;
3301
+ ok: boolean;
3092
3302
  paused: boolean;
3093
3303
  activeRuns: number;
3094
3304
  project: string;
3095
3305
  /** One-shot issue ceilings waiting for the next claim. */
3096
3306
  turnOverrides: TurnOverride[];
3097
- /** Resident set of this daemon; workers are in-process omp sessions. */
3098
- rssBytes: number;
3099
3307
  dispatch?: DispatchSummary;
3100
3308
  codeGraph?: CodeGraphHealth;
3101
3309
  /** Live workers in a non-running pause phase; absent/empty = nothing paused. */
3102
3310
  workers?: { issue: number; runId: string; phase: WorkerPausePhase }[];
3103
3311
  }
3104
3312
 
3313
+ export interface DaemonHealth {
3314
+ ok: boolean;
3315
+ /** Resident set of this daemon; workers are in-process omp sessions. */
3316
+ rssBytes: number;
3317
+ projects: DaemonHealthSnapshot[];
3318
+ }
3319
+
3105
3320
  export function daemonHealthSnapshot(
3106
3321
  store: Store,
3107
3322
  project: string,
3108
- paused = isPaused(),
3109
- rssBytes = process.memoryUsage().rss,
3323
+ paused = isPaused(project),
3110
3324
  codeGraph?: CodeGraphHealth,
3111
3325
  workerControls?: WorkerControlRegistry,
3112
3326
  ): DaemonHealthSnapshot {
@@ -3117,13 +3331,19 @@ export function daemonHealthSnapshot(
3117
3331
  activeRuns: store.activeRuns(project).length,
3118
3332
  turnOverrides: store.listTurnOverrides(project),
3119
3333
  project,
3120
- rssBytes,
3121
3334
  ...(dispatch === undefined ? {} : { dispatch }),
3122
3335
  ...(codeGraph?.configured === true ? { codeGraph } : {}),
3123
3336
  ...(workerControls === undefined ? {} : { workers: workerControls.snapshot(project) }),
3124
3337
  };
3125
3338
  }
3126
3339
 
3340
+ export function daemonHealth(
3341
+ projects: DaemonHealthSnapshot[],
3342
+ rssBytes = process.memoryUsage().rss,
3343
+ ): DaemonHealth {
3344
+ return { ok: projects.every((project) => project.ok), rssBytes, projects };
3345
+ }
3346
+
3127
3347
  const TURN_OVERRIDE_STATES: ReadonlySet<RunState> = new Set([
3128
3348
  "failed",
3129
3349
  "killed",
@@ -3329,13 +3549,17 @@ export async function workerControlResponse(
3329
3549
  );
3330
3550
  }
3331
3551
 
3332
- export interface DaemonHttpDeps {
3552
+ export interface DaemonHttpProjectDeps {
3333
3553
  project: string;
3334
3554
  store: Pick<Store, "latestRun" | "setTurnOverride">;
3335
3555
  caps: () => Pick<Caps, "workerMaxTurns" | "workerMaxTurnsCeiling">;
3556
+ }
3557
+
3558
+ export interface DaemonHttpDeps {
3559
+ projects: readonly DaemonHttpProjectDeps[];
3336
3560
  turnLimits: TurnLimitRegistry;
3337
3561
  workerControls: WorkerControlRegistry;
3338
- health: () => DaemonHealthSnapshot;
3562
+ health: () => DaemonHealth;
3339
3563
  }
3340
3564
 
3341
3565
  /**
@@ -3350,9 +3574,47 @@ export interface DaemonHttpDeps {
3350
3574
  export async function daemonHttpResponse(req: Request, d: DaemonHttpDeps): Promise<Response> {
3351
3575
  const url = new URL(req.url);
3352
3576
  if (req.method === "GET" && url.pathname === "/healthz") return Response.json(d.health());
3353
- const turnLimit = await turnLimitResponse(req, d.project, d.store, d.turnLimits, d.caps());
3577
+
3578
+ let selected = d.projects[0];
3579
+ if (
3580
+ req.method === "PUT" &&
3581
+ /^\/runs\/\d+\/(?:turn-limit|pause|resume|stop)$/.test(url.pathname) &&
3582
+ req.headers.get("content-type")?.startsWith("application/json")
3583
+ ) {
3584
+ try {
3585
+ const body = await req.clone().json();
3586
+ if (body !== null && typeof body === "object") {
3587
+ const requested = Reflect.get(body, "project");
3588
+ if (typeof requested === "string" && requested.length > 0) {
3589
+ selected = d.projects.find(({ project }) => project === requested);
3590
+ if (selected === undefined) {
3591
+ return Response.json(
3592
+ { error: `daemon does not serve requested project "${requested}"` },
3593
+ { status: 409 },
3594
+ );
3595
+ }
3596
+ }
3597
+ }
3598
+ } catch {
3599
+ // The route handler below owns the public malformed-body response.
3600
+ }
3601
+ }
3602
+ if (selected === undefined) return new Response("not found\n", { status: 404 });
3603
+
3604
+ const turnLimit = await turnLimitResponse(
3605
+ req,
3606
+ selected.project,
3607
+ selected.store,
3608
+ d.turnLimits,
3609
+ selected.caps(),
3610
+ );
3354
3611
  if (turnLimit !== undefined) return turnLimit;
3355
- const workerControl = await workerControlResponse(req, d.project, d.store, d.workerControls);
3612
+ const workerControl = await workerControlResponse(
3613
+ req,
3614
+ selected.project,
3615
+ selected.store,
3616
+ d.workerControls,
3617
+ );
3356
3618
  return workerControl ?? new Response("not found\n", { status: 404 });
3357
3619
  }
3358
3620
 
@@ -3367,6 +3629,10 @@ export interface StatusSnapshot {
3367
3629
  * reading like a mistake (#220).
3368
3630
  */
3369
3631
  pauseReason?: string;
3632
+ /** Mechanical operator availability at the moment this snapshot was read. */
3633
+ availability?: AvailabilityState;
3634
+ /** Next digest opportunity under the same predicate that gates submission. */
3635
+ digestSchedule?: DigestScheduleState;
3370
3636
  caps: Caps;
3371
3637
  /**
3372
3638
  * The effective per-shape release grants. On the snapshot rather than re-read
@@ -3386,6 +3652,9 @@ export interface StatusSnapshot {
3386
3652
  * unknown outcome, or written off. An empty list is the only honest way to
3387
3653
  * say "everything authored this cycle actually went out" (#123). */
3388
3654
  openReports: ReportRecord[];
3655
+ /** Ordinary outcomes and deferred escalations not yet associated with an
3656
+ * accepted digest report. */
3657
+ digestBacklog: DigestBacklog;
3389
3658
  /**
3390
3659
  * The most recent conductor-verb calls and how the daemon decided them
3391
3660
  * (#126). On `status` rather than only behind `omp-conductor ledger` because
@@ -3393,8 +3662,8 @@ export interface StatusSnapshot {
3393
3662
  * config does not let it, and an operator who has to know to go looking is an
3394
3663
  * operator who finds out from the tracker instead.
3395
3664
  */
3396
- /** Newest post-merge base verdict per routed repository within seven days. */
3397
- baseChecks: RunRecord[];
3665
+ /** Current live-head push-workflow verdict per recently merged repository. */
3666
+ baseHealth: BaseHealth[];
3398
3667
  verbLedger: VerbLedgerEntry[];
3399
3668
  /** Runs backed by a worker process — the number capacity compares against. */
3400
3669
  liveWorkers: number;
@@ -3441,25 +3710,32 @@ export function statusSnapshotFromStore(
3441
3710
  store: Store,
3442
3711
  planUsage?: PlanUsageStatus,
3443
3712
  ): StatusSnapshot {
3713
+ const now = Date.now();
3714
+ const lastDigestKey = store.lastDigestDedupeKey(p.name);
3715
+ const lastDigestDay =
3716
+ lastDigestKey === undefined ? undefined : lastDigestKey.slice("digest:".length);
3444
3717
  const since = startOfToday();
3445
3718
  const dispatch = store.latestDispatch(p.name);
3446
3719
  const labelOpsPending = store.countPendingLabelOps(p.name);
3447
3720
  const oldestLabelOpAt = store.oldestPendingLabelOpAt(p.name);
3448
3721
  // Read once: the provenance read touches the filesystem, and the renderer
3449
3722
  // should never pay for it twice per status.
3450
- const reason = pauseProvenance()?.reason;
3723
+ const reason = pauseProvenance(p.name)?.reason;
3451
3724
  return {
3452
3725
  project: p.name,
3453
3726
  configPath: configPath(),
3454
3727
  stateDir: stateDir(),
3455
- paused: isPaused(),
3728
+ paused: isPaused(p.name),
3456
3729
  ...(reason === undefined ? {} : { pauseReason: reason }),
3730
+ availability: availabilityState(p.reporting, now),
3731
+ digestSchedule: digestScheduleState(p.reporting ?? DEFAULT_REPORT_POLICY, lastDigestDay, now),
3457
3732
  caps,
3458
3733
  releaseGrants: resolveReleaseGrants(p),
3459
3734
  activeRuns: store.activeRuns(p.name),
3460
3735
  salvagedRuns: store.salvagedRuns(p.name),
3461
3736
  turnOverrides: store.listTurnOverrides(p.name),
3462
3737
  openReports: store.openReports(p.name),
3738
+ digestBacklog: store.digestBacklog(p.name),
3463
3739
  verbLedger: store.verbLedger(p.name, { limit: STATUS_LEDGER_SCAN }),
3464
3740
  liveWorkers: store.liveRuns(p.name).length,
3465
3741
  runsToday: store.runsStartedSince(p.name, since),
@@ -3468,12 +3744,12 @@ export function statusSnapshotFromStore(
3468
3744
  ...(planUsage === undefined ? {} : { planUsage }),
3469
3745
  // Written by the tracker's hooks rather than polled, so the renderer does
3470
3746
  // not re-read GitHub to know it is being refused (#198).
3471
- ghRefusals: store.ghRefusalsSince?.(Date.now() - 5 * 60_000),
3747
+ ghRefusals: store.ghRefusalsSince?.(now - 5 * 60_000),
3472
3748
  ghCallsToday: store.ghCallsToday?.(utcDay()),
3473
3749
  ...(labelOpsPending === 0 || oldestLabelOpAt === undefined
3474
3750
  ? {}
3475
- : { labelOps: { pending: labelOpsPending, oldestAgeMs: Date.now() - oldestLabelOpAt } }),
3476
- baseChecks: store.latestBaseChecks(p.name, Date.now() - BASE_STATUS_WINDOW_MS),
3751
+ : { labelOps: { pending: labelOpsPending, oldestAgeMs: now - oldestLabelOpAt } }),
3752
+ baseHealth: store.baseHealth(p.name),
3477
3753
  };
3478
3754
  }
3479
3755
 
@@ -3556,18 +3832,22 @@ export function formatReleaseGrants(grants: ResolvedGrants): string[] {
3556
3832
  ...RELEASE_SHAPES.map((shape) => ` ${shape.padEnd(19)}${grants[shape]}`),
3557
3833
  ];
3558
3834
  }
3559
- export function formatBaseChecks(runs: readonly RunRecord[]): string[] {
3560
- return runs.flatMap((run) => {
3561
- if (run.baseCheck === undefined) return [];
3562
- const branch = run.baseRef ?? "?";
3563
- const flag = run.settlementFlags?.find((candidate) => candidate.kind === "base-branch-red");
3564
- const verdict =
3565
- run.baseCheck === "red"
3566
- ? `RED — ${flag?.detail ?? "workflow failed after merge"}`
3567
- : run.baseCheck === "red-preexisting"
3568
- ? `red before merge${flag === undefined ? "" : ` — ${flag.detail}`}`
3569
- : run.baseCheck;
3570
- return [`base ${run.repo}/${branch} ${verdict}`];
3835
+ export function formatBaseHealth(rows: readonly BaseHealth[]): string[] {
3836
+ return rows.map((row) => {
3837
+ const head = row.headSha.slice(0, 8);
3838
+ if (row.verdict === "green") {
3839
+ return `base ${row.repo}/${row.branch} green (${row.runsCount} run(s)) at ${head}`;
3840
+ }
3841
+ if (row.verdict === "red") {
3842
+ return `base ${row.repo}/${row.branch} RED — ${row.detail ?? `workflow failed at ${head}`}`;
3843
+ }
3844
+ if (row.verdict === "pending") {
3845
+ return `base ${row.repo}/${row.branch} pending (${row.runsCount} run(s)) at ${head}`;
3846
+ }
3847
+ return (
3848
+ `base ${row.repo}/${row.branch} unknown — ` +
3849
+ (row.detail ?? `no push-triggered workflow run for ${head}`)
3850
+ );
3571
3851
  });
3572
3852
  }
3573
3853
 
@@ -3618,7 +3898,7 @@ export function formatStatus(s: StatusSnapshot): string {
3618
3898
  if (flagged !== undefined) lines.push(` ${flagged}`);
3619
3899
  }
3620
3900
  }
3621
- lines.push(...formatBaseChecks(s.baseChecks));
3901
+ lines.push(...formatBaseHealth(s.baseHealth));
3622
3902
  lines.push(...formatSalvagedRuns(s.salvagedRuns));
3623
3903
  lines.push(...formatOpenReports(s.openReports));
3624
3904
  lines.push(...formatVerbLedger(s.verbLedger));
@@ -3645,7 +3925,7 @@ export interface QueuePreview {
3645
3925
 
3646
3926
  /**
3647
3927
  * Exactly what the next tick would pick up, computed without touching a single
3648
- * label, run row or worktree. This is what makes `/conductor setup` honest: the
3928
+ * label, run row or worktree. This is what makes `omp-conductor setup` honest: the
3649
3929
  * dry run is the same routing code the loop uses, not a description of it.
3650
3930
  */
3651
3931
  export async function previewProject(
@@ -3661,7 +3941,7 @@ export async function previewProject(
3661
3941
  `open issues in ${p.tracker.repo} labelled "${p.queueLabel}", ` +
3662
3942
  `minus anything already labelled ${states}, ` +
3663
3943
  `routed by one "${p.routing.labelPrefix}<repo>" label`,
3664
- paused: isPaused(),
3944
+ paused: isPaused(p.name),
3665
3945
  ready: routed.map((r) => ({
3666
3946
  number: r.issue.number,
3667
3947
  title: r.issue.title,
@@ -3688,9 +3968,9 @@ export async function previewQueue(project?: string): Promise<QueuePreview> {
3688
3968
  * Setup calls this immediately after consent. Every later setup error therefore
3689
3969
  * leaves the fleet paused instead of exposing a partially written runtime.
3690
3970
  */
3691
- export function prepareConductor(): void {
3971
+ export function prepareConductor(project?: string): void {
3692
3972
  openStore(dbPath()).close();
3693
- setPaused(true, { source: "setup" });
3973
+ setPaused(true, { source: "setup" }, project);
3694
3974
  }
3695
3975
 
3696
3976
  /** Bounded per tick: each row costs tracker calls to gather facts for. */
@@ -4211,283 +4491,264 @@ export async function reconcileOrphanedRuns(
4211
4491
 
4212
4492
  // ------------------------------------------------------------------- the daemon
4213
4493
 
4494
+ interface ProjectRuntime {
4495
+ d: Deps;
4496
+ outbox: ReportOutbox;
4497
+ orchestrator?: OrchestratorHandle;
4498
+ orchestratorVerbs?: VerbListener;
4499
+ codeGraph: CodeGraphHealth;
4500
+ graphProbe?: Promise<void>;
4501
+ reportPass?: Promise<void>;
4502
+ }
4503
+
4504
+ function orchestratorStandingOrders(project: ProjectConfig): {
4505
+ brief: string;
4506
+ releaseGrants: ResolvedGrants;
4507
+ } {
4508
+ const releaseGrants = resolveReleaseGrants(project);
4509
+ const grantedShapes = RELEASE_SHAPES.filter((shape) => releaseGrants[shape] === "orchestrator");
4510
+ const deniedShapes = RELEASE_SHAPES.filter((shape) => releaseGrants[shape] !== "orchestrator");
4511
+ return {
4512
+ releaseGrants,
4513
+ brief: [
4514
+ `You are the omp-conductor orchestrator for project "${project.name}".`,
4515
+ `Tracker: ${project.tracker.repo}. Pass --repo ${project.tracker.repo} to every gh command:`,
4516
+ "this working directory is the conductor's state directory, not a checkout.",
4517
+ `Labels: queue=${project.queueLabel}, running=${project.stateLabels.inProgress}, ` +
4518
+ `blocked=${project.stateLabels.blocked}, failed=${project.stateLabels.failed}.`,
4519
+ "The dispatcher claims queue-labelled issues, runs one worker session per attempt in its own",
4520
+ "worktree under hard turn/wallclock/spend caps, and escalates to you when a worker blocks or",
4521
+ "fails twice, its gates stay red, its branch conflicts, or a tripwire fires.",
4522
+ "Your job when that happens: re-brief the issue (comment what the next worker must do",
4523
+ `differently, then put ${project.queueLabel} back on it), file follow-up issues, or promote to`,
4524
+ "tier 2 and let the human decide.",
4525
+ project.authority.merge === "orchestrator"
4526
+ ? "You never edit product code or push a branch — a worker session does that. Merging is yours: one PR at " +
4527
+ "a time, freshness-checked against the base branch, per the Releases section of your POLICY.md."
4528
+ : "You never edit product code, push a branch, or merge a PR — a worker session edits and pushes, and a " +
4529
+ "human merges.",
4530
+ `Release tool gate: ${
4531
+ grantedShapes.length === 0
4532
+ ? "every release and deploy shape is mechanically blocked for you"
4533
+ : `you may invoke ${grantedShapes.join(", ")} — and only by the procedure in your POLICY.md`
4534
+ }.` + (deniedShapes.length === 0 ? "" : ` Blocked: ${deniedShapes.join(", ")}.`),
4535
+ "Handle each escalation below before the next one.",
4536
+ ].join("\n"),
4537
+ };
4538
+ }
4539
+
4214
4540
  export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
4215
4541
  const cfg = loadConfig();
4216
- const project = findProject(cfg, o.project);
4217
- const caps = resolveCaps(project, cfg.defaults);
4542
+ const projects = o.project === undefined ? cfg.projects : [findProject(cfg, o.project)];
4218
4543
  const store = openStore(dbPath());
4219
- // The tracker's single `gh` funnel is bound to the store so the operator sees
4220
- // observed truth (#198): every spawn is counted per UTC day, and every
4221
- // rate-limit refusal is recorded for `status`'s 5m window. Board's ad-hoc
4222
- // trackers and the polled `fetchRateLimit` probe are deliberately not bound —
4223
- // this row is the daemon's own traffic.
4224
- const tracker = makeTracker(project, undefined, {
4225
- onCall: () => store.bumpGhCalls?.(utcDay(), "daemon"),
4226
- onRefusal: (at) => store.recordGhRefusal?.(at),
4227
- // A conditional 304 revalidation is a spawn but not a billed read (#203);
4228
- // counted separately so `status`'s call row keeps telling the truth once
4229
- // most spawns are free revalidations.
4230
- onNotModified: () => store.bumpGhCalls?.(utcDay(), "daemon-304"),
4231
- });
4232
-
4233
- // #126's transport, stated at startup rather than guessed at first use. The
4234
- // banner names what this host can actually enforce — whether the kernel will
4235
- // vouch for a caller's uid, and whether runs get distinct principals at all —
4236
- // because "peer credentials asserted" is a claim, and a claim nobody printed
4237
- // is one nobody can check against the host it is running on.
4238
4544
  const verbPeerReader = peerCredentialReader();
4239
- const verbActions = githubVerbActions(project);
4240
- let orchestratorVerbs: VerbListener | undefined;
4241
- log(`verb transport: ${transportBanner(ensureVerbSocketDir(stateDir()), verbPeerReader)}`);
4242
-
4243
- // Recorded here, before a single tick runs, so that the deploy an operator
4244
- // *means* to do never trips the tripwire: installing a new build and
4245
- // restarting the unit re-records this from the new files. What it catches is
4246
- // the other thing — the package changing while the daemon that dispatches
4247
- // work is holding it open, whether that is a worker that wandered out of its
4248
- // worktree or a human editing the live install "just to test something".
4545
+ const verbDir = ensureVerbSocketDir(stateDir());
4249
4546
  const integrity: IntegrityGate = { baseline: packageManifest(), paged: false };
4547
+ const usage = sharedUsageSource();
4548
+ const turnLimits = createTurnLimitRegistry((runId, maxTurns) => {
4549
+ store.updateRun(runId, { maxTurns });
4550
+ });
4551
+ const workerControls = createWorkerControlRegistry();
4552
+ const workers = createWorkerPool();
4553
+ const alive = livingDaemon();
4554
+ const runtimes: ProjectRuntime[] = [];
4555
+
4556
+ log(`verb transport: ${transportBanner(verbDir, verbPeerReader)}`);
4250
4557
  log(`package integrity baseline: ${integrity.baseline.size} files under ${import.meta.dir}`);
4251
4558
 
4252
- // Before the first tick, settle what the last process left behind — unless
4253
- // another daemon is alive (a foreground `daemon --once` beside a running
4254
- // daemon must not orphan that daemon's real, live workers).
4255
- const alive = livingDaemon();
4256
- if (alive === undefined || alive.pid === process.pid) {
4257
- // The orphan path salvages, and #121 requires that salvage to reach
4258
- // GitHub. Since the run's commits now live in its own repository, the hop
4259
- // is the daemon's: resolve the routed repo by name off the run row.
4260
- const orphanPublisher = (r: RunRecord): RunPublisher => {
4261
- const repo = project.routing.repos[r.repo];
4262
- return async (branch) =>
4263
- repo === undefined
4264
- ? { ok: false, stderr: `run ${String(r.id)} names repo "${r.repo}", which this project no longer routes` }
4265
- : pushRunBranch(project, { repo, runRepoPath: r.worktree, branch });
4559
+ for (const project of projects) {
4560
+ const projectLog = (message: string): void => {
4561
+ log(projects.length === 1 ? message : `[${project.name}] ${message}`);
4266
4562
  };
4267
- for (const r of await reconcileOrphanedRuns(store, project.name, orphanPublisher)) {
4268
- log(
4269
- `#${r.issue} orphaned by a previous daemon (attempt ${r.attempt}, was ${r.state}, worktree ${r.worktree}) — ` +
4270
- `slot freed; the ${project.stateLabels.inProgress} label stays until the orchestrator triages what the worker left`,
4271
- );
4563
+ const caps = resolveCaps(project, cfg.defaults);
4564
+ const tracker = makeTracker(project, undefined, {
4565
+ onCall: () => store.bumpGhCalls?.(utcDay(), "daemon"),
4566
+ onRefusal: (at) => store.recordGhRefusal?.(at),
4567
+ onNotModified: () => store.bumpGhCalls?.(utcDay(), "daemon-304"),
4568
+ });
4569
+ const verbActions = githubVerbActions(project);
4570
+
4571
+ if (alive === undefined || alive.pid === process.pid) {
4572
+ const orphanPublisher = (run: RunRecord): RunPublisher => {
4573
+ const repo = project.routing.repos[run.repo];
4574
+ return async (branch) =>
4575
+ repo === undefined
4576
+ ? {
4577
+ ok: false,
4578
+ stderr:
4579
+ `run ${String(run.id)} names repo "${run.repo}", ` +
4580
+ `which project "${project.name}" no longer routes`,
4581
+ }
4582
+ : pushRunBranch(project, { repo, runRepoPath: run.worktree, branch });
4583
+ };
4584
+ for (const run of await reconcileOrphanedRuns(store, project.name, orphanPublisher)) {
4585
+ projectLog(
4586
+ `#${run.issue} orphaned by a previous daemon (attempt ${run.attempt}, was ${run.state}, ` +
4587
+ `worktree ${run.worktree}) — slot freed; the ${project.stateLabels.inProgress} label stays ` +
4588
+ "until the orchestrator triages what the worker left",
4589
+ );
4590
+ }
4591
+ } else if (runtimes.length === 0) {
4592
+ log(`skipping orphan reconciliation: daemon pid ${alive.pid} is alive and owns the active runs`);
4272
4593
  }
4273
- } else {
4274
- log(`skipping orphan reconciliation: daemon pid ${alive.pid} is alive and owns the active runs`);
4275
- }
4276
4594
 
4277
- // Standing orders. The orchestrator holds none of this file's context, so
4278
- // everything it needs to act — which tracker, which labels, what the fleet
4279
- // does has to be said once, in words.
4280
- const releaseGrants = resolveReleaseGrants(project);
4281
- const grantedShapes = RELEASE_SHAPES.filter((shape) => releaseGrants[shape] === "orchestrator");
4282
- const deniedShapes = RELEASE_SHAPES.filter((shape) => releaseGrants[shape] !== "orchestrator");
4283
- const brief = [
4284
- `You are the omp-conductor orchestrator for project "${project.name}".`,
4285
- `Tracker: ${project.tracker.repo}. Pass --repo ${project.tracker.repo} to every gh command:`,
4286
- "this working directory is the conductor's state directory, not a checkout.",
4287
- `Labels: queue=${project.queueLabel}, running=${project.stateLabels.inProgress}, ` +
4288
- `blocked=${project.stateLabels.blocked}, failed=${project.stateLabels.failed}.`,
4289
- "The dispatcher claims queue-labelled issues, runs one worker session per attempt in its own",
4290
- "worktree under hard turn/wallclock/spend caps, and escalates to you when a worker blocks or",
4291
- "fails twice, its gates stay red, its branch conflicts, or a tripwire fires.",
4292
- "Your job when that happens: re-brief the issue (comment what the next worker must do",
4293
- `differently, then put ${project.queueLabel} back on it), file follow-up issues, or promote to`,
4294
- "tier 2 and let the human decide.",
4295
- // Worded from `authority.merge` rather than fixed, so the standing orders
4296
- // and the Releases section of the rendered brief cannot disagree about who
4297
- // is holding the merge button. The daemon still merges nothing itself.
4298
- project.authority.merge === "orchestrator"
4299
- ? "You never edit product code or push a branch — a worker session does that. Merging is yours: one PR at " +
4300
- "a time, freshness-checked against the base branch, per the Releases section of your POLICY.md."
4301
- : "You never edit product code, push a branch, or merge a PR — a worker session edits and pushes, and a " +
4302
- "human merges.",
4303
- // Named shape by shape rather than as one policy word, so a stale grant is
4304
- // legible in the transcript instead of only in the config file — the #122
4305
- // incident began with a grant that no longer matched anyone's intent.
4306
- `Release tool gate: ${
4307
- grantedShapes.length === 0
4308
- ? "every release and deploy shape is mechanically blocked for you"
4309
- : `you may invoke ${grantedShapes.join(", ")} — and only by the procedure in your POLICY.md`
4310
- }.` + (deniedShapes.length === 0 ? "" : ` Blocked: ${deniedShapes.join(", ")}.`),
4311
- "Handle each escalation below before the next one.",
4312
- ].join("\n");
4313
-
4314
- // One orchestrator per daemon run, not per tick: it is a persistent session
4315
- // whose whole value is remembering what it has already escalated, and a fresh
4316
- // one every five minutes would remember nothing. Its cwd is the state
4317
- // directory, deliberately not a checkout — the orchestrator re-briefs workers
4318
- // and talks to the tracker, it does not edit product code.
4319
- let orchestrator: OrchestratorHandle | undefined;
4320
- if (project.escalation.orchestrator === "external") {
4321
- // An operator already runs the brain — typically a visible TUI session that
4322
- // drains `blocked`/`failed` off the tracker as one of its standing duties.
4323
- // Starting a second one here would re-triage the same issues from a
4324
- // transcript nobody is watching, and the two would undo each other.
4325
- log("orchestrator: external — tier-1 escalations post as issue comments for the external session's drain duty");
4326
- } else {
4327
- try {
4328
- // The orchestrator is a child process of this daemon running as its own
4329
- // user. Nothing mechanically stops it reading a run checkout — what holds
4330
- // it is its brief and the verb ledger (#143).
4331
- const orchTreeRoot = stateDir();
4332
- const orchCwd = join(orchTreeRoot, "orchestrator");
4333
- mkdirSync(orchCwd, { recursive: true });
4334
- // A third socket, distinct from every run's, in the same daemon-owned
4335
- // 0711 parent. It makes the orchestrator's authority a property of the
4336
- // channel rather than of a payload: no argument list can move a worker's
4337
- // call onto this one (#126).
4338
- //
4339
- // It does NOT authenticate the session. Sessions share this daemon's uid,
4340
- // so one could list this directory and connect here, and the ledger would
4341
- // record the orchestrator's role because that is the channel's. See
4342
- // conductor#163 for the pid binding that would close it.
4343
- orchestratorVerbs = await listenVerbChannel(
4344
- verbDeps({ project, store, tracker, verbActions }),
4345
- {
4346
- kind: "orchestrator",
4347
- path: verbSocketPath(ensureVerbSocketDir(orchTreeRoot), "orchestrator"),
4348
- project: project.name,
4349
- role: "orchestrator",
4350
- },
4351
- { ...(verbPeerReader === undefined ? {} : { peerReader: verbPeerReader }) },
4352
- );
4353
- orchestrator = await startOrchestrator({
4354
- cwd: orchCwd,
4355
- brief,
4356
- releaseGrants,
4357
- socketPath: join(orchCwd, "ipc.sock"),
4358
- verbSocketPath: orchestratorVerbs.path,
4359
- // The orchestrator's channel is the one that matters most: it is the
4360
- // session authorised to merge, so an unbound channel here is a worker's
4361
- // route to that authority (#163).
4362
- onSpawn: (pid) => {
4363
- orchestratorVerbs?.bindPid(pid);
4364
- },
4365
- onChildLog: (line) => {
4366
- log(`orchestrator ${line}`);
4367
- },
4368
- onReleaseBlocked: (shape, context) =>
4369
- recordReleaseBlock(project.name, "orchestrator", shape, context),
4370
- });
4371
- const transcript = orchestrator.sessionFile();
4372
- log(`orchestrator session ready${transcript === undefined ? "" : ` · ${transcript}`}`);
4373
- } catch (err) {
4374
- // Loudly, but not fatally: tier-1 escalations degrade to issue comments,
4375
- // which a human still reads. A dispatcher that refuses to run because its
4376
- // re-briefing channel is down helps nobody.
4377
- log(
4378
- `WARNING: orchestrator session failed to start; tier-1 escalations will fall back to issue comments: ${errText(err)}`,
4595
+ const { brief, releaseGrants } = orchestratorStandingOrders(project);
4596
+ let orchestrator: OrchestratorHandle | undefined;
4597
+ let orchestratorVerbs: VerbListener | undefined;
4598
+ if (project.escalation.orchestrator === "external") {
4599
+ projectLog(
4600
+ "orchestrator: external tier-1 escalations post as issue comments for the external session's drain duty",
4379
4601
  );
4602
+ } else {
4603
+ try {
4604
+ const orchCwd = join(
4605
+ stateDir(),
4606
+ projects.length === 1 ? "orchestrator" : `orchestrator-${project.name}`,
4607
+ );
4608
+ mkdirSync(orchCwd, { recursive: true });
4609
+ orchestratorVerbs = await listenVerbChannel(
4610
+ verbDeps({ project, store, tracker, verbActions }),
4611
+ {
4612
+ kind: "orchestrator",
4613
+ path: verbSocketPath(verbDir, `orchestrator-${project.name}`),
4614
+ project: project.name,
4615
+ role: "orchestrator",
4616
+ },
4617
+ { ...(verbPeerReader === undefined ? {} : { peerReader: verbPeerReader }) },
4618
+ );
4619
+ orchestrator = await startOrchestrator({
4620
+ cwd: orchCwd,
4621
+ brief,
4622
+ releaseGrants,
4623
+ socketPath: join(orchCwd, "ipc.sock"),
4624
+ verbSocketPath: orchestratorVerbs.path,
4625
+ onSpawn: (pid) => {
4626
+ orchestratorVerbs?.bindPid(pid);
4627
+ },
4628
+ onChildLog: (line) => {
4629
+ projectLog(`orchestrator ${line}`);
4630
+ },
4631
+ onReleaseBlocked: (shape, context) =>
4632
+ recordReleaseBlock(project.name, "orchestrator", shape, context),
4633
+ });
4634
+ const transcript = orchestrator.sessionFile();
4635
+ projectLog(`orchestrator session ready${transcript === undefined ? "" : ` · ${transcript}`}`);
4636
+ } catch (err) {
4637
+ projectLog(
4638
+ "WARNING: orchestrator session failed to start; tier-1 escalations will fall back to issue " +
4639
+ `comments: ${errText(err)}`,
4640
+ );
4641
+ await orchestratorVerbs?.close();
4642
+ orchestratorVerbs = undefined;
4643
+ }
4380
4644
  }
4381
- }
4382
-
4383
- const escalator = createEscalator(project, tracker, store, orchestrator);
4384
4645
 
4385
- // Report delivery is the daemon's, not the model's (#123). Built beside the
4386
- // escalator because a report nobody can deliver pages through it, and driven
4387
- // on its own timer rather than inside `tick()`: a paused fleet claims nothing
4388
- // but still owes its operator the report it was handed, and five minutes is a
4389
- // long time to sit on a page.
4390
- const outbox = createReportOutbox({
4391
- project,
4392
- store,
4393
- escalate: (e) => escalator.escalate(e),
4394
- log,
4395
- });
4396
-
4397
- // Every row still `sending` when a daemon boots belonged to a process that is
4398
- // gone, so its outcome will never be learned — retry it and say so in the
4399
- // message. Guarded exactly like the orphan reconciliation above and for the
4400
- // same reason: a row belonging to a *live* daemon is genuinely in flight, and
4401
- // stealing it would page the operator twice for one report.
4402
- if (alive === undefined || alive.pid === process.pid) {
4403
- for (const r of outbox.recover(Date.now())) {
4404
- log(
4405
- `report ${r.id} was left mid-send by a previous daemon (attempt ${r.attempts}) — ` +
4406
- `retrying; its message will say it may be a repeat`,
4407
- );
4646
+ let runtimeDeps: Deps | undefined;
4647
+ const currentProject = (): ProjectConfig => runtimeDeps?.project ?? project;
4648
+ const deliveryPolicyValid = (): boolean => runtimeDeps?.deliveryPolicyValid === true;
4649
+ const escalator = createEscalator(
4650
+ currentProject,
4651
+ tracker,
4652
+ store,
4653
+ orchestrator,
4654
+ Date.now,
4655
+ deliveryPolicyValid,
4656
+ );
4657
+ const outbox = createReportOutbox({
4658
+ project: currentProject,
4659
+ store,
4660
+ escalate: (event) => escalator.escalate(event),
4661
+ log: projectLog,
4662
+ deliveryAllowed: deliveryPolicyValid,
4663
+ });
4664
+ if (alive === undefined || alive.pid === process.pid) {
4665
+ for (const report of outbox.recover(Date.now())) {
4666
+ projectLog(
4667
+ `report ${report.id} was left mid-send by a previous daemon (attempt ${report.attempts}) — ` +
4668
+ "retrying; its message will say it may be a repeat",
4669
+ );
4670
+ }
4408
4671
  }
4409
- }
4410
4672
 
4411
- const turnLimits = createTurnLimitRegistry((runId, maxTurns) => {
4412
- store.updateRun(runId, { maxTurns });
4413
- });
4414
- const workerControls = createWorkerControlRegistry();
4415
- const d: Deps = {
4416
- project,
4417
- caps,
4418
- tracker,
4419
- store,
4420
- // Process-wide, so a `status` served off this daemon's own HTTP surface
4421
- // reuses the tick's reading instead of shelling out again.
4422
- usage: sharedUsageSource(),
4423
- escalate: (e) => escalator.escalate(e),
4424
- turnLimits,
4425
- workerControls,
4426
- integrity,
4427
- // Fresh per daemon run, like the integrity gate: a restart is entitled to
4428
- // page again about a stall that is still on disk.
4429
- stall: { paged: false },
4430
- cleanup: { next: 0 },
4431
- ...(verbPeerReader === undefined ? {} : { verbPeerReader }),
4432
- verbActions,
4433
- };
4673
+ const d: Deps = {
4674
+ project,
4675
+ caps,
4676
+ tracker,
4677
+ store,
4678
+ deliveryPolicyValid: false,
4679
+ usage,
4680
+ escalate: (event) => escalator.escalate(event),
4681
+ turnLimits,
4682
+ workerControls,
4683
+ integrity,
4684
+ stall: { paged: false },
4685
+ cleanup: { next: 0 },
4686
+ ...(verbPeerReader === undefined ? {} : { verbPeerReader }),
4687
+ verbActions,
4688
+ };
4689
+ runtimeDeps = d;
4690
+ runtimes.push({
4691
+ d,
4692
+ outbox,
4693
+ ...(orchestrator === undefined ? {} : { orchestrator }),
4694
+ ...(orchestratorVerbs === undefined ? {} : { orchestratorVerbs }),
4695
+ codeGraph: pendingCodeGraph(project),
4696
+ });
4697
+ }
4434
4698
 
4435
4699
  if (o.once) {
4436
4700
  try {
4437
- await tick(d);
4438
- // A single tick still owes the outbox a pass: a drill that leaves a
4439
- // report undelivered teaches an operator the wrong thing about the
4440
- // mechanism it is drilling.
4441
- await outbox.deliverDue();
4701
+ for (const runtime of runtimes) await tick(runtime.d, workers);
4702
+ await workers.drain();
4703
+ for (const runtime of runtimes) await runtime.outbox.deliverDue();
4442
4704
  } finally {
4443
- await orchestrator?.dispose();
4444
- await orchestratorVerbs?.close();
4705
+ for (const runtime of runtimes.toReversed()) {
4706
+ await runtime.orchestrator?.dispose();
4707
+ await runtime.orchestratorVerbs?.close();
4708
+ }
4445
4709
  store.close();
4446
4710
  }
4447
4711
  return;
4448
4712
  }
4449
4713
 
4450
- // Graphs are optional, so their bounded probes run beside dispatch and feed
4451
- // a cache. /healthz remains an in-memory answer and never blocks liveness on
4452
- // the indexer or systemd.
4453
- let codeGraph = pendingCodeGraph(project);
4454
- let graphProbe: Promise<void> | undefined;
4455
- const refreshCodeGraph = (): void => {
4456
- if (graphProbe !== undefined) return;
4457
- graphProbe = probeCodeGraph(project)
4714
+ const refreshCodeGraph = (runtime: ProjectRuntime): void => {
4715
+ if (runtime.graphProbe !== undefined) return;
4716
+ runtime.graphProbe = probeCodeGraph(runtime.d.project)
4458
4717
  .then((health) => {
4459
- codeGraph = health;
4718
+ runtime.codeGraph = health;
4460
4719
  })
4461
4720
  .catch(() => {
4462
- log("code-graph health probe failed unexpectedly; retaining the previous bounded result");
4721
+ log(
4722
+ `[${runtime.d.project.name}] code-graph health probe failed unexpectedly; ` +
4723
+ "retaining the previous bounded result",
4724
+ );
4463
4725
  })
4464
4726
  .finally(() => {
4465
- graphProbe = undefined;
4727
+ runtime.graphProbe = undefined;
4466
4728
  });
4467
4729
  };
4468
- refreshCodeGraph();
4469
- const graphTimer = setInterval(refreshCodeGraph, GRAPH_HEALTH_INTERVAL_MS);
4470
-
4471
- // Overlap-guarded like the graph probe: a pass that is still waiting on
4472
- // Telegram must not have a second pass started on top of it, or one report
4473
- // would be claimed, reclaimed and sent twice by this process alone.
4474
- let reportPass: Promise<void> | undefined;
4475
- const drainReports = (): void => {
4476
- if (reportPass !== undefined) return;
4477
- reportPass = outbox
4730
+ for (const runtime of runtimes) refreshCodeGraph(runtime);
4731
+ const graphTimer = setInterval(() => {
4732
+ for (const runtime of runtimes) refreshCodeGraph(runtime);
4733
+ }, GRAPH_HEALTH_INTERVAL_MS);
4734
+
4735
+ const drainReports = (runtime: ProjectRuntime): void => {
4736
+ if (runtime.reportPass !== undefined) return;
4737
+ runtime.reportPass = runtime.outbox
4478
4738
  .deliverDue()
4479
4739
  .then(() => {})
4480
4740
  .catch((err: unknown) => {
4481
- log(`report delivery pass failed: ${errText(err)}`);
4741
+ log(`[${runtime.d.project.name}] report delivery pass failed: ${errText(err)}`);
4482
4742
  })
4483
4743
  .finally(() => {
4484
- reportPass = undefined;
4744
+ runtime.reportPass = undefined;
4485
4745
  });
4486
4746
  };
4487
- drainReports();
4488
- const reportTimer = setInterval(drainReports, REPORT_DELIVERY_INTERVAL_MS);
4747
+ for (const runtime of runtimes) drainReports(runtime);
4748
+ const reportTimer = setInterval(() => {
4749
+ for (const runtime of runtimes) drainReports(runtime);
4750
+ }, REPORT_DELIVERY_INTERVAL_MS);
4489
4751
 
4490
- const workers = createWorkerPool();
4491
4752
  let stopping = false;
4492
4753
  let wake: (() => void) | undefined;
4493
4754
  const stop = (): void => {
@@ -4499,54 +4760,58 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
4499
4760
  process.on("SIGINT", stop);
4500
4761
  process.on("SIGTERM", stop);
4501
4762
 
4502
- // ── NO TRACKER OR REPOSITORY MUTATION BELONGS ON THIS PORT ───────────────
4503
- //
4504
- // This is unauthenticated loopback TCP. Every local user can reach it, it
4505
- // carries no credential of any kind, and it cannot tell one caller from
4506
- // another: `127.0.0.1` is not an identity. The turn-limit and worker
4507
- // pause/resume controls trust a body-supplied `project`, which is exactly the
4508
- // shape "identity from the payload" takes when nobody is watching. They stay
4509
- // tolerable only because every effect is bounded: pauses are reversible,
4510
- // live extensions touch one controller, and a persisted next-attempt
4511
- // override can only raise the project base up to its configured ceiling and
4512
- // is consumed by one claim.
4513
- //
4514
- // A merge, a push, a release or a label is none of those things. Do not add
4515
- // one here, and do not add "just a small one" behind a shared secret either:
4516
- // a secret readable by the process that would be attacking you is not
4517
- // authentication. Mutations go over the per-run unix sockets in
4518
- // `verbs/socket.ts`, where the kernel says who the caller is and the daemon
4519
- // derives project, run and role from the channel rather than the body (#126).
4763
+ // This unauthenticated loopback surface exposes liveness and bounded live-run
4764
+ // controls only. Repository and tracker mutations stay on credentialled verb
4765
+ // sockets, where project and role come from the channel rather than a payload.
4520
4766
  const server = Bun.serve({
4521
4767
  hostname: "127.0.0.1",
4522
4768
  port: o.port ?? DEFAULT_PORT,
4523
4769
  fetch: (req) =>
4524
4770
  daemonHttpResponse(req, {
4525
- project: project.name,
4526
- store,
4527
- caps: () => d.caps,
4771
+ projects: runtimes.map((runtime) => ({
4772
+ project: runtime.d.project.name,
4773
+ store,
4774
+ caps: () => runtime.d.caps,
4775
+ })),
4528
4776
  turnLimits,
4529
4777
  workerControls,
4530
4778
  health: () =>
4531
- daemonHealthSnapshot(store, project.name, undefined, undefined, codeGraph, workerControls),
4779
+ daemonHealth(
4780
+ runtimes.map((runtime) =>
4781
+ daemonHealthSnapshot(
4782
+ store,
4783
+ runtime.d.project.name,
4784
+ isPaused(runtime.d.project.name),
4785
+ runtime.codeGraph,
4786
+ workerControls,
4787
+ ),
4788
+ ),
4789
+ ),
4532
4790
  }),
4533
4791
  });
4534
- log(`serving /healthz on :${server.port}, project ${project.name}`);
4792
+ log(
4793
+ projects.length === 1
4794
+ ? `serving /healthz on :${server.port}, project ${projects[0]!.name}`
4795
+ : `serving /healthz on :${server.port}, projects ${projects.map(({ name }) => name).join(", ")}`,
4796
+ );
4535
4797
 
4536
4798
  try {
4537
4799
  while (!stopping) {
4538
- try {
4539
- await tick(d, workers);
4540
- } catch (err) {
4541
- // A tick that blows up outside an issue (the tracker is down, say) must
4542
- // not end the daemon; the next one will retry.
4543
- log(`tick failed: ${errText(err)}`);
4800
+ for (const runtime of runtimes) {
4801
+ try {
4802
+ await tick(runtime.d, workers);
4803
+ } catch (err) {
4804
+ log(
4805
+ `${projects.length === 1 ? "" : `[${runtime.d.project.name}] `}` +
4806
+ `tick failed: ${errText(err)}`,
4807
+ );
4808
+ }
4544
4809
  }
4545
4810
  if (stopping) break;
4546
4811
  await new Promise<void>((resolve) => {
4547
- const t = setTimeout(resolve, TICK_INTERVAL_MS);
4812
+ const timer = setTimeout(resolve, TICK_INTERVAL_MS);
4548
4813
  wake = () => {
4549
- clearTimeout(t);
4814
+ clearTimeout(timer);
4550
4815
  resolve();
4551
4816
  };
4552
4817
  });
@@ -4557,26 +4822,15 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
4557
4822
  process.off("SIGTERM", stop);
4558
4823
  clearInterval(graphTimer);
4559
4824
  clearInterval(reportTimer);
4560
- // Before the store closes, like the orchestrator below: a pass mid-send has
4561
- // a `markReportDelivered` still to write, and losing that write is exactly
4562
- // how a delivered report comes back as an ambiguous one on the next boot.
4563
- await reportPass;
4825
+ for (const runtime of runtimes) await runtime.reportPass;
4564
4826
  await workers.drain();
4565
4827
  await server.stop(true);
4566
- // Before the store closes: a queued injection that rejects on the way out
4567
- // falls back to an issue comment, and that path writes the dedup marker.
4568
- await orchestrator?.dispose();
4569
- // After the session it belongs to is gone. A bound socket outliving its
4570
- // orchestrator is a channel accepting merges for a session that no longer
4571
- // exists.
4572
- await orchestratorVerbs?.close();
4828
+ for (const runtime of runtimes.toReversed()) {
4829
+ await runtime.orchestrator?.dispose();
4830
+ await runtime.orchestratorVerbs?.close();
4831
+ }
4573
4832
  store.close();
4574
4833
  log("stopped");
4575
- // A handled SIGTERM still leaves some runtimes with a non-zero default
4576
- // (historically 128+signal). Under systemd `Restart=on-failure` that looks
4577
- // like a crash and the unit comes straight back — the exact failure mode
4578
- // `omp-conductor stop` hit on the reference fleet. Force success so a
4579
- // graceful drain is not a restart.
4580
4834
  process.exitCode = 0;
4581
4835
  }
4582
4836
  }