omp-conductor 0.15.13 → 0.16.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/REFERENCE.md +72 -2
  2. package/package.json +2 -1
  3. package/schema/config.schema.json +7 -0
  4. package/src/admission.ts +849 -0
  5. package/src/ask.ts +47 -0
  6. package/src/backups.ts +19 -7
  7. package/src/board.ts +1 -2
  8. package/src/briefs/orchestrator.md +62 -4
  9. package/src/cli.ts +26 -0
  10. package/src/commands/context.ts +3 -0
  11. package/src/commands/decision.ts +10 -1
  12. package/src/commands/doctor.ts +2 -0
  13. package/src/commands/message.ts +8 -1
  14. package/src/commands/restart.ts +15 -3
  15. package/src/commands/restore-db.ts +146 -0
  16. package/src/commands/stop.ts +24 -15
  17. package/src/commands/tail.ts +204 -44
  18. package/src/commands/unfreeze.ts +56 -0
  19. package/src/commands/watch.ts +77 -0
  20. package/src/config-schema.ts +13 -0
  21. package/src/config.ts +54 -0
  22. package/src/daemon.ts +255 -530
  23. package/src/dashboard/server.ts +2 -1
  24. package/src/decisions.ts +32 -7
  25. package/src/depends-on.ts +122 -0
  26. package/src/doctor.ts +297 -5
  27. package/src/escalate.ts +191 -19
  28. package/src/failure-class.ts +47 -0
  29. package/src/fleet.ts +168 -452
  30. package/src/gitops.ts +86 -1
  31. package/src/log.ts +40 -0
  32. package/src/model-fallback.ts +3 -2
  33. package/src/omp-settings.ts +114 -0
  34. package/src/omp.ts +39 -0
  35. package/src/orchestrator-tick.ts +7 -1
  36. package/src/reports.ts +124 -12
  37. package/src/session-host.ts +6 -0
  38. package/src/setup-wizard.ts +36 -0
  39. package/src/setup.ts +58 -1
  40. package/src/status-render.ts +445 -0
  41. package/src/stop-provenance.ts +53 -0
  42. package/src/store.ts +352 -11
  43. package/src/transcript.ts +1 -1
  44. package/src/types.ts +187 -4
  45. package/src/unblock.ts +1 -1
  46. package/src/upgrade-verify.ts +1 -1
  47. package/src/upgrade.ts +1 -2
  48. package/src/verbs/server.ts +25 -0
  49. package/src/worker.ts +358 -10
  50. package/src/worktree.ts +13 -1
package/src/setup.ts CHANGED
@@ -23,6 +23,7 @@
23
23
  import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
24
24
  import { homedir } from "node:os";
25
25
  import { dirname, join } from "node:path";
26
+ import { parse as parseYaml } from "yaml";
26
27
  import {
27
28
  COMPOSE_BANNER,
28
29
  ORCHESTRATOR_BRIEF_NAME,
@@ -46,6 +47,7 @@ import {
46
47
  stateDir,
47
48
  } from "./config.ts";
48
49
  import { graphProjectPath, graphRepos } from "./graph.ts";
50
+ import { ompSettingsOverlay } from "./omp-settings.ts";
49
51
  import {
50
52
  CONFIG_VERSION,
51
53
  DEFAULT_AUTHORITY,
@@ -163,6 +165,15 @@ export interface SetupAnswers {
163
165
  */
164
166
  modelFallbacks?: string[];
165
167
  modelFallbackThreshold?: number;
168
+ /**
169
+ * The project's omp settings overlay (#537): an opaque map layered into
170
+ * every worker session via the fleet-owned settings channel. The wizard asks
171
+ * for it as one free-form YAML answer (the only prompt this map gets — the
172
+ * keys inside it are omp's vocabulary, not a list conductor can offer); it
173
+ * also survives an amend of any other area unchanged, like
174
+ * {@link modelFallbacks}.
175
+ */
176
+ ompSettings?: Record<string, unknown>;
166
177
  telegramChatId?: string;
167
178
  /** Forum topic for tier-2 Telegram pages; absent keeps flat-chat 0.13 behaviour. */
168
179
  telegramTopicId?: number;
@@ -855,6 +866,9 @@ export function buildProject(a: SetupAnswers): ProjectConfig {
855
866
  // would pin every run of the project onto a dead provider again (#286).
856
867
  ...(a.modelFallbacks === undefined ? {} : { modelFallbacks: [...a.modelFallbacks] }),
857
868
  ...(a.modelFallbackThreshold === undefined ? {} : { modelFallbackThreshold: a.modelFallbackThreshold }),
869
+ // The omp settings overlay is an opaque map the wizard collects as free-form
870
+ // YAML; an unrelated amend must not delete it (#537).
871
+ ...(a.ompSettings === undefined ? {} : { ompSettings: a.ompSettings }),
858
872
  escalation,
859
873
  authority: { ...a.authority },
860
874
  // Written out in full, never as the legacy string: the file then says which
@@ -1061,6 +1075,7 @@ export function answersFromProject(p: ProjectConfig): SetupAnswers {
1061
1075
  if (p.groomBelow !== undefined) answers.groomBelow = p.groomBelow;
1062
1076
  if (p.modelFallbacks !== undefined) answers.modelFallbacks = [...p.modelFallbacks];
1063
1077
  if (p.modelFallbackThreshold !== undefined) answers.modelFallbackThreshold = p.modelFallbackThreshold;
1078
+ if (p.ompSettings !== undefined) answers.ompSettings = { ...p.ompSettings };
1064
1079
  if (p.escalation.telegramChatId !== undefined) answers.telegramChatId = p.escalation.telegramChatId;
1065
1080
  if (p.escalation.telegramTopicId !== undefined) answers.telegramTopicId = p.escalation.telegramTopicId;
1066
1081
  if (p.recoveryMerges !== undefined) {
@@ -1682,19 +1697,47 @@ export function formatGates(gates: readonly { cmd: string; cwd: string }[]): str
1682
1697
  return gates.map((g) => (g.cwd === "." ? g.cmd : `${g.cmd} @ ${g.cwd}`)).join(", ");
1683
1698
  }
1684
1699
 
1700
+ /**
1701
+ * Parses the wizard's free-form YAML answer for a project's omp settings
1702
+ * overlay (#537) and validates YAML shape only — the same boundary the config
1703
+ * loader enforces. A mapping at the document root is the whole contract:
1704
+ * everything inside it is omp's schema to own, so an unknown key or a
1705
+ * wrong-typed value is omp's to reject, never this module's to understand
1706
+ * (non-string keys are coerced by the YAML parser exactly as omp's own loader
1707
+ * coerces them, so a `true: x` mapping stays YAML-valid on both sides). Blank
1708
+ * input is a valid "no overlay" answer.
1709
+ */
1710
+ export function parseOmpSettingsYaml(
1711
+ text: string,
1712
+ ): { ok: true; value: Record<string, unknown> } | { ok: false; problem: string } {
1713
+ const trimmed = text.trim();
1714
+ if (trimmed.length === 0) return { ok: true, value: {} };
1715
+ let parsed: unknown;
1716
+ try {
1717
+ parsed = parseYaml(trimmed);
1718
+ } catch (err) {
1719
+ return { ok: false, problem: `"${text}" is not valid YAML: ${err instanceof Error ? err.message : String(err)}` };
1720
+ }
1721
+ if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
1722
+ return { ok: false, problem: "the omp settings overlay must be a YAML mapping at the root, e.g. `modelRoles: { worker: \"@slow\" }`" };
1723
+ }
1724
+ return { ok: true, value: parsed as Record<string, unknown> };
1725
+ }
1726
+
1685
1727
  /**
1686
1728
  * The wizard's questions, grouped as the areas a re-run can amend one of, in the
1687
1729
  * order the full interview asks them.
1688
1730
  *
1689
1731
  * Data rather than a switch so the menu, the CLI's positional area vocabulary,
1690
1732
  * ./setup-wizard.ts's `AREA_ASKERS` table and the amend summary all enumerate the
1691
- * same nine areas: an added area fails to compile until it has a name, a current
1733
+ * same ten areas: an added area fails to compile until it has a name, a current
1692
1734
  * value and a set of questions.
1693
1735
  */
1694
1736
  export const AMEND_AREA_IDS = [
1695
1737
  "tracker",
1696
1738
  "gates",
1697
1739
  "caps",
1740
+ "omp-settings",
1698
1741
  "code-graph",
1699
1742
  "authority",
1700
1743
  "policy",
@@ -1765,6 +1808,20 @@ export const AMEND_AREAS: {
1765
1808
  );
1766
1809
  },
1767
1810
  },
1811
+ "omp-settings": {
1812
+ // The free-form omp settings overlay (#537), the sibling per-worker knob to
1813
+ // the model: one area no menu offers is a setting only a full re-interview
1814
+ // can reach, so this one has its own row like the caps.
1815
+ name: "omp settings overlay",
1816
+ asks: "the free-form YAML map layered into every worker session's omp settings — omp's schema, not conductor's",
1817
+ // The *effective* overlay — `ompSettings` plus the retry keys derived from
1818
+ // `modelFallbacks` — so the menu says what a worker would actually load,
1819
+ // matching `doctor` and the dispatcher rather than echoing raw config.
1820
+ describe: (p) => {
1821
+ const overlay = ompSettingsOverlay(p);
1822
+ return overlay === undefined ? "no overlay — workers inherit global settings" : JSON.stringify(overlay);
1823
+ },
1824
+ },
1768
1825
  "code-graph": {
1769
1826
  name: "code graph",
1770
1827
  asks: "whether workers query a code-graph index, and the root its one-clone-per-repo lives under",
@@ -0,0 +1,445 @@
1
+ /**
2
+ * The status-rendering half of the fleet operator surface (`fleet.ts`
3
+ * composes; this owns the rendered text).
4
+ *
5
+ * Extracted from `fleet.ts` so the region every issue that surfaces something
6
+ * an operator reads must touch is a lane of its own: #459, #582, #318, #283
7
+ * and #484 all render a new line here, and a diff touching only this module
8
+ * must not have to touch the composition root. Behavior is deliberately
9
+ * untouched by the move — this is a move, not a rewrite: the module takes a
10
+ * data structure and returns text, and reads nothing itself.
11
+ *
12
+ * What status rendering is allowed to consult is the point of the module. It
13
+ * does not import `fleet.ts` back, and it owns the structural input it
14
+ * consumes (`FleetLayers`, `TelegramHealth`, `FleetDaemonProbe`) rather than
15
+ * reaching into the module that builds them — the same seam #580 used for
16
+ * `AdmissionDeps`. Adding a field to `fleetLayers` must not break these
17
+ * functions.
18
+ */
19
+
20
+ import { formatZonedMinute } from "./availability.ts";
21
+ import type { DaemonStop } from "./types.ts";
22
+ import { settlementFlagSummary } from "./diff-flags.ts";
23
+ import type { CodeGraphHealth } from "./graph-health.ts";
24
+ import { formatDigestBacklog, formatOpenReports } from "./reports.ts";
25
+ import { formatOrchestratorDown } from "./orchestrator-down.ts";
26
+ import { planUsageLine } from "./usage.ts";
27
+ import { SYSTEMD_UNIT, type UnitOwnership } from "./lifecycle.ts";
28
+ import type { WorkerPausePhase } from "./worker.ts";
29
+ import { formatRss, rssBytesFromHealthz } from "./host.ts";
30
+ import {
31
+ formatBaseHealth,
32
+ formatDispatchSummary,
33
+ formatFreezes,
34
+ formatReleaseGrants,
35
+ formatSalvagedRuns,
36
+ isPaused,
37
+ pausedAt,
38
+ pauseProvenance,
39
+ type StatusSnapshot,
40
+ } from "./daemon.ts";
41
+
42
+ // layered status
43
+ // ---------------------------------------------------------------------------
44
+
45
+ export type DispatchLayer = "running" | "paused" | "stopped";
46
+ export type TicksLayer =
47
+ | "armed"
48
+ | "disarmed"
49
+ | "no-heartbeat-config"
50
+ | "invalid-heartbeat-config"
51
+ | "ungated";
52
+ export type PaneLayer = "live" | "missing" | "unknown";
53
+ /** `unpinnable`: no tick config, so FLEET_CWD — the only path recovery reads — is unknown. */
54
+ export type RecoveryLayer = "pinned" | "clear" | "unpinnable";
55
+ export type HerdrLayer = "active" | "inactive" | "unknown";
56
+ export type TelegramLayer = "ok" | "degraded" | "down" | "unconfigured" | "unprobed";
57
+
58
+ export interface TelegramHealth {
59
+ kind: TelegramLayer;
60
+ detail?: string;
61
+ }
62
+
63
+ export interface FleetLayers {
64
+ dispatch: DispatchLayer;
65
+ ticks: TicksLayer;
66
+ ticksDetail?: string;
67
+ nextTickAt?: string;
68
+ pane: PaneLayer;
69
+ paneDetail?: string;
70
+ recovery: RecoveryLayer;
71
+ recoveryDetail?: string;
72
+ herdr: HerdrLayer;
73
+ herdrDetail?: string;
74
+ armedPath?: string;
75
+ tickConfigPath?: string;
76
+ paneHaltPath?: string;
77
+ paused: boolean;
78
+ daemon: { running: boolean; pid?: number; port?: number };
79
+ }
80
+
81
+ export function formatCodeGraphHealth(graph: CodeGraphHealth, now = Date.now()): string | undefined {
82
+ if (!graph.configured) return undefined;
83
+ const indexed = graph.repos.filter((repo) => repo.index === "present").length;
84
+ let refresh: string = graph.refresh.result;
85
+ if (graph.refresh.lastSuccessAt !== undefined) {
86
+ const ageMs = Math.max(0, now - Date.parse(graph.refresh.lastSuccessAt));
87
+ refresh = `${graph.refresh.lastSuccessAt} (${Math.max(1, Math.ceil(ageMs / 60_000))}m ago)`;
88
+ }
89
+ return [
90
+ `code graph ${graph.status} ${indexed}/${graph.repos.length} repos indexed`,
91
+ ` indexer ${graph.prerequisites.indexer}`,
92
+ ` MCP mount ${graph.prerequisites.mcpMount}`,
93
+ ` timer ${graph.timer.enabled} / ${graph.timer.active}`,
94
+ ` refresh ${refresh}`,
95
+ ...graph.reasons.map((reason) => ` - ${reason}`),
96
+ ].join("\n");
97
+ }
98
+
99
+
100
+ /**
101
+ * Project-scoped reading of a living daemon record + `/healthz` body.
102
+ *
103
+ * Board and `status` share this so a host running one daemon for several
104
+ * projects cannot be called healthy for a project it does not serve (#379).
105
+ * A third interpretation convention is forbidden.
106
+ */
107
+ export type DaemonProjectHealth =
108
+ | { kind: "stopped" }
109
+ | { kind: "ok" }
110
+ | { kind: "unreachable" }
111
+ | { kind: "other-project"; serves?: string };
112
+
113
+ /**
114
+ * Facts `formatFleetStatus` needs about the living daemon. Pure input — the
115
+ * caller probes; the formatter never shells out (#379).
116
+ */
117
+ export type FleetDaemonProbe = {
118
+ project: DaemonProjectHealth;
119
+ /** `/healthz` body when the probe is project-ok (rss / overlays). */
120
+ body?: string;
121
+ /** systemd ownership of the living record pid; omit when unprobed. */
122
+ unit?: UnitOwnership;
123
+ };
124
+
125
+ function formatDaemonHealthz(probe: FleetDaemonProbe | undefined): string {
126
+ if (probe === undefined) return "unprobed";
127
+ switch (probe.project.kind) {
128
+ case "stopped":
129
+ return "stopped";
130
+ case "ok":
131
+ return "ok";
132
+ case "unreachable":
133
+ return "unreachable — the process is up but not serving";
134
+ case "other-project":
135
+ return probe.project.serves === undefined
136
+ ? "other-project"
137
+ : `other-project — serves ${probe.project.serves}`;
138
+ }
139
+ }
140
+
141
+ /**
142
+ * Unit line for a living record. The bare unit name alone used to imply
143
+ * systemd ownership that was never checked (#379); every branch states the
144
+ * probe result, and `unknown` is never rendered as owned.
145
+ */
146
+ function formatDaemonUnit(unit: UnitOwnership | undefined, recordPid: number): string {
147
+ if (unit === undefined) return ` unit ${SYSTEMD_UNIT} unprobed`;
148
+ switch (unit.kind) {
149
+ case "active":
150
+ if (unit.pid === recordPid) {
151
+ return ` unit ${SYSTEMD_UNIT} systemd-owned`;
152
+ }
153
+ // Live unit, different MainPID — the record pid is not systemd's.
154
+ return ` unit ${SYSTEMD_UNIT} unmanaged — MainPID ${unit.pid}`;
155
+ case "failed":
156
+ return ` unit ${SYSTEMD_UNIT} unmanaged — unit failed`;
157
+ case "inactive":
158
+ return ` unit ${SYSTEMD_UNIT} inactive`;
159
+ case "unknown":
160
+ return ` unit ${SYSTEMD_UNIT} unknown — ${unit.reason}`;
161
+ }
162
+ }
163
+
164
+ /**
165
+ * The newest daemon stop/restart provenance as status lines (#378).
166
+ *
167
+ * Rendered under the daemon block whether the daemon is down or has since
168
+ * been restarted — the whole point is that the debrief survives the restart.
169
+ * An unattributed record says so explicitly: the external-signal fallback
170
+ * records what the receiving daemon knew (that it was unattributed, when,
171
+ * which projects it served with live counts) rather than guessing a caller.
172
+ */
173
+ function formatLastStop(lastStop: DaemonStop | undefined): string[] {
174
+ if (lastStop === undefined) return [];
175
+ const caller = lastStop.unattributed
176
+ ? "unattributed — no mediated request (external signal)"
177
+ : `pid ${lastStop.callerPid ?? "?"}` +
178
+ (lastStop.callerUid === undefined ? "" : ` uid ${lastStop.callerUid}`) +
179
+ (lastStop.role === undefined ? "" : ` (${lastStop.role})`);
180
+ return [
181
+ ` last stop ${new Date(lastStop.at).toISOString()} ${lastStop.controlPath}`,
182
+ ` caller ${caller}`,
183
+ ` scope ${lastStop.scope}${lastStop.project === undefined ? "" : `: ${lastStop.project}`}`,
184
+ ` affects ${lastStop.affected.map((a) => `${a.project} (${a.live} live)`).join(", ")}`,
185
+ ` reason ${lastStop.reason}`,
186
+ ];
187
+ }
188
+
189
+
190
+ export function formatFleetStatus(
191
+ s: StatusSnapshot,
192
+ layers: FleetLayers,
193
+ daemon: FleetDaemonProbe | undefined = undefined,
194
+ telegram: TelegramHealth = { kind: "unprobed" },
195
+ now = Date.now(),
196
+ codeGraph: CodeGraphHealth = { configured: false },
197
+ brief: string | undefined = undefined,
198
+ decisions: string | undefined = undefined,
199
+ failureClasses: string | undefined = undefined,
200
+ workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
201
+ intake: string | undefined = undefined,
202
+ lastStop: DaemonStop | undefined = undefined,
203
+ siblings: { project: string; live: number }[] = [],
204
+ ): string {
205
+ const tickLine =
206
+ layers.ticksDetail === undefined
207
+ ? `ticks ${layers.ticks}`
208
+ : `ticks ${layers.ticks} (${layers.ticksDetail})`;
209
+ let nextTickLine: string | undefined;
210
+ if (layers.nextTickAt !== undefined) {
211
+ const delta = Date.parse(layers.nextTickAt) - now;
212
+ const minutes = Math.max(1, Math.ceil(Math.abs(delta) / 60_000));
213
+ nextTickLine =
214
+ `next tick ${layers.nextTickAt} ` +
215
+ `(${delta >= 0 ? `in ${minutes}m` : `overdue by ${minutes}m`})`;
216
+ }
217
+ const paneLine =
218
+ layers.paneDetail === undefined
219
+ ? `pane ${layers.pane}`
220
+ : `pane ${layers.pane} (${layers.paneDetail})`;
221
+ const recoveryLine =
222
+ layers.recoveryDetail === undefined
223
+ ? `recovery ${layers.recovery}`
224
+ : `recovery ${layers.recovery} (${layers.recoveryDetail})`;
225
+ const herdrLine =
226
+ layers.herdrDetail === undefined
227
+ ? `herdr ${layers.herdr}`
228
+ : `herdr ${layers.herdr} (${layers.herdrDetail})`;
229
+ const telegramLine =
230
+ telegram.detail === undefined
231
+ ? `telegram ${telegram.kind}`
232
+ : `telegram ${telegram.kind} (${telegram.detail})`;
233
+
234
+ let daemonBlock: string;
235
+ if (!layers.daemon.running || layers.daemon.pid === undefined) {
236
+ daemonBlock = "daemon not running";
237
+ } else {
238
+ // Only trust rss from a project-ok body — a foreign payload's bytes are
239
+ // not this project's daemon facts (#379).
240
+ const rss =
241
+ daemon?.project.kind === "ok" ? rssBytesFromHealthz(daemon.body) : undefined;
242
+ daemonBlock = [
243
+ "daemon",
244
+ ` pid ${layers.daemon.pid}`,
245
+ ` port ${layers.daemon.port ?? "?"}`,
246
+ ...(rss === undefined ? [] : [` rss ${formatRss(rss)}`]),
247
+ ` healthz ${formatDaemonHealthz(daemon)}`,
248
+ formatDaemonUnit(daemon?.unit, layers.daemon.pid),
249
+ ].join("\n");
250
+ }
251
+
252
+ const graphBlock = formatCodeGraphHealth(codeGraph, now);
253
+
254
+ // Pause provenance (`pause --reason`, an integrity/spend-cap/upgrade pause,
255
+ // a halt) answers "who stopped the fleet" without opening a file (#185). An
256
+ // unparseable sentinel — paused but with no datable line 1 — is itself news:
257
+ // it means a run admitted before an *unknown* pause cannot prove innocence
258
+ // (#174), so completion mutations fail closed while release gates remain usable.
259
+ const dispatchLine =
260
+ layers.dispatch === "paused"
261
+ ? (() => {
262
+ const prov = pauseProvenance(s.project);
263
+ if (prov !== undefined) {
264
+ const reason = prov.reason === undefined ? "" : ` — "${prov.reason}"`;
265
+ return `dispatch paused (source: ${prov.source}${reason})`;
266
+ }
267
+ return isPaused(s.project) && pausedAt(s.project) === undefined
268
+ ? "dispatch paused (unparseable sentinel — new work and completion mutations refused; release gates remain available)"
269
+ : "dispatch paused";
270
+ })()
271
+ : `dispatch ${layers.dispatch}`;
272
+
273
+ return [
274
+ dispatchLine,
275
+ tickLine,
276
+ ...(nextTickLine === undefined ? [] : [nextTickLine]),
277
+ paneLine,
278
+ recoveryLine,
279
+ herdrLine,
280
+ telegramLine,
281
+ ...(brief === undefined ? [] : [brief]),
282
+ ...(decisions === undefined ? [] : [decisions]),
283
+ ...(failureClasses === undefined ? [] : [failureClasses]),
284
+ ...(intake === undefined ? [] : [intake]),
285
+ ...(graphBlock === undefined ? [] : [graphBlock]),
286
+ daemonBlock,
287
+ ...formatLastStop(lastStop),
288
+ "",
289
+ formatProjectBody(s, workerPhases, now, siblings),
290
+ ].join("\n");
291
+ }
292
+
293
+
294
+ function formatAvailabilityStatus(s: StatusSnapshot): string[] {
295
+ const availability = s.availability;
296
+ if (availability === undefined) return [];
297
+ if (availability.mode === "always") {
298
+ return ["availability 24-hour interrupts (no weekly window)"];
299
+ }
300
+ const mode =
301
+ availability.nextTransitionAt === undefined || availability.timezone === undefined
302
+ ? `${availability.mode}; next transition could not be calculated`
303
+ : `${availability.mode} until ${formatZonedMinute(availability.nextTransitionAt, availability.timezone)}`;
304
+ const bypass = availability.bypass.length === 0 ? "none" : availability.bypass.join(", ");
305
+ return [`availability ${mode}; quiet-hours bypass ${bypass}`];
306
+ }
307
+
308
+
309
+ function formatDigestScheduleStatus(s: StatusSnapshot): string[] {
310
+ const schedule = s.digestSchedule;
311
+ if (schedule === undefined) return [];
312
+ if (schedule.mode === "disabled") return ["next digest disabled"];
313
+ if (schedule.mode === "per-tick") return ["next digest every tick"];
314
+ if (schedule.mode === "due") return ["next digest due now"];
315
+ return [
316
+ schedule.nextAt === undefined
317
+ ? "next digest could not be calculated"
318
+ : `next digest ${formatZonedMinute(schedule.nextAt, schedule.timezone)}`,
319
+ ];
320
+ }
321
+
322
+ /**
323
+ * The shared-daemon visibility row (#545). When the daemon also serves sibling
324
+ * projects, tell the reader how many runs each has live, so they can tell "my
325
+ * fleet is idle" from "the process I am about to stop is busy" — the count is
326
+ * the point, so a "shared daemon" line that omits it is the bug re-expressed.
327
+ */
328
+ function formatSiblingLive(siblings: { project: string; live: number }[]): string | undefined {
329
+ if (siblings.length === 0) return undefined;
330
+ const counts = siblings.map((s) => `${s.project} (${s.live} live)`).join(", ");
331
+ return `shared daemon also serves ${siblings.length} other project(s): ${counts}`;
332
+ }
333
+
334
+
335
+ function formatProjectBody(
336
+ s: StatusSnapshot,
337
+ workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
338
+ now = Date.now(),
339
+ siblings: { project: string; live: number }[] = [],
340
+ ): string {
341
+ const siblingLine = formatSiblingLive(siblings);
342
+ const lines = [
343
+ `project ${s.project}${s.paused ? " (PAUSED)" : ""}`,
344
+ ...(s.pauseReason === undefined ? [] : [`paused ${s.pauseReason}`]),
345
+ `config ${s.configPath}`,
346
+ `state ${s.stateDir}`,
347
+ ...(siblingLine === undefined ? [] : [siblingLine]),
348
+ // The orchestrator-down degrade row: first-class in the body, present only
349
+ // while the incident is open, so recovery drops it (#288).
350
+ ...(s.orchestratorDown === undefined
351
+ ? []
352
+ : formatOrchestratorDown(s.orchestratorDown, now)),
353
+ ...formatAvailabilityStatus(s),
354
+ ...formatDigestScheduleStatus(s),
355
+ "",
356
+ "caps",
357
+ ` workers ${s.liveWorkers} / ${s.caps.maxConcurrentWorkers}`,
358
+ ` issues today ${s.runsToday}`,
359
+ s.caps.dailySpendUsd === null
360
+ ? ` spend today $${s.spendTodayUsd.toFixed(2)} (no daily cap)`
361
+ : ` spend today $${s.spendTodayUsd.toFixed(2)} / $${s.caps.dailySpendUsd.toFixed(2)}`,
362
+ // Its own row beside the spend row, never folded into it: they are two
363
+ // independent controls and an operator has to see which one stopped the
364
+ // fleet (#110).
365
+ ` plan usage ${planUsageLine(s.planUsage)}`,
366
+ // The tracker's API budget, when the renderer could read it. Absent on a
367
+ // broken `gh`: one missing row, never a broken report (#188). Next to it,
368
+ // the refusals the tracker actually observed, so a budget that looks
369
+ // healthy next to every call being refused is still visible (#198).
370
+ ...(s.github === undefined
371
+ ? []
372
+ : [
373
+ ` github graphql ${s.github.graphql.remaining}/${s.github.graphql.limit}, core ${s.github.core.remaining}/${s.github.core.limit} (graphql resets ${Math.max(0, Math.round((s.github.graphql.reset * 1000 - Date.now()) / 60_000))}m)` +
374
+ (s.ghRefusals === undefined || s.ghRefusals.count === 0
375
+ ? ""
376
+ : ` — ${s.ghRefusals.count} refusal(s) in last 5m (last ${new Date(s.ghRefusals.latestAt ?? Date.now()).toISOString().slice(11, 19)}Z)`),
377
+ ]),
378
+ // The daemon's own observed github traffic today (#198): a separate row
379
+ // from the polled budget above, so one missing `status` read never hides
380
+ // the other. `onCall` counts every spawn including ones that end 304, so
381
+ // `daemon` is spawns and `daemon-304` is the unbilled subset — billed ≈
382
+ // difference, and the polled budget row stays the authority (#203).
383
+ ` github calls daemon ${s.ghCallsToday?.find((c) => c.source === "daemon")?.calls ?? 0} today` +
384
+ ((s.ghCallsToday?.find((c) => c.source === "daemon-304")?.calls ?? 0) === 0
385
+ ? ""
386
+ : ` (${s.ghCallsToday!.find((c) => c.source === "daemon-304")!.calls} free 304s)`),
387
+ // Label-projection ops the tracker has not applied yet (#201): GitHub is
388
+ // behind what the store decided, and the operator can see the lag instead
389
+ // of discovering it as a stale label or a missing one.
390
+ ...(s.labelOps === undefined
391
+ ? []
392
+ : [
393
+ ` labels projection ${s.labelOps.pending} pending (oldest ${
394
+ s.labelOps.oldestAgeMs >= 60_000
395
+ ? `${Math.round(s.labelOps.oldestAgeMs / 60_000)}m`
396
+ : `${Math.max(1, Math.round(s.labelOps.oldestAgeMs / 1000))}s`
397
+ })`,
398
+ ]),
399
+ ` new worker turns ${s.caps.workerMaxTurns}`,
400
+ ...s.turnOverrides.map(
401
+ ({ issue, maxTurns }) => ` turn override #${issue} → ${maxTurns} (next attempt)`,
402
+ ),
403
+ ` worker wall clock ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
404
+ ` failed attempts ${s.caps.maxAttemptsPerIssue}`,
405
+ ` continuations ${s.caps.maxContinuationsPerIssue}`,
406
+ "",
407
+ ...formatReleaseGrants(s.releaseGrants),
408
+ "",
409
+ formatDispatchSummary(s.dispatch),
410
+ "",
411
+ ];
412
+ if (s.activeRuns.length === 0) {
413
+ lines.push("active runs (none)");
414
+ } else {
415
+ lines.push("active runs");
416
+ for (const r of s.activeRuns) {
417
+ const phase = workerPhases.get(r.issue);
418
+ const state = phase === "pausing" || phase === "paused" ? phase : r.state;
419
+ lines.push(
420
+ ` #${r.issue} ${r.repo} ${state} attempt ${r.attempt} ` +
421
+ `${r.turns}/${r.maxTurns} turns ${r.spendUsd.toFixed(2)} ${r.branch}` +
422
+ (r.prUrl ? ` ${r.prUrl}` : ""),
423
+ );
424
+ // The orchestrator's Duty 1 reads this command, and a flagged run's
425
+ // escalation is deduplicated after one delivery — so this is where a
426
+ // flagged PR stays visible for as long as it is still open (#128).
427
+ const flagged = settlementFlagSummary(r.settlementFlags);
428
+ if (flagged !== undefined) lines.push(` ${flagged}`);
429
+ }
430
+ }
431
+ lines.push(...formatBaseHealth(s.baseHealth));
432
+ lines.push(...formatFreezes(s.freezes));
433
+ lines.push(...formatSalvagedRuns(s.salvagedRuns));
434
+ lines.push(...formatOpenReports(s.openReports));
435
+ lines.push(...formatDigestBacklog(s.digestBacklog));
436
+ if (s.liveWorkers > 0) {
437
+ lines.push(
438
+ "",
439
+ `deploy ${s.liveWorkers} live worker(s) — restart salvages dirty trees then orphans the rows; ` +
440
+ `hold and wait for workers 0/${s.caps.maxConcurrentWorkers} when you can drain instead`,
441
+ );
442
+ }
443
+ return lines.join("\n");
444
+ }
445
+
@@ -37,6 +37,59 @@ export interface StopProvenanceSpec {
37
37
  * entry for drain-style restarts whose signal may never arrive, and at the
38
38
  * lifecycle chokepoint immediately before signalling for every mediated stop).
39
39
  */
40
+ /**
41
+ * One configured project's live workload at a stop/restart refusal window.
42
+ */
43
+ export interface LiveWorkload {
44
+ project: string;
45
+ /** Live (claimed/running) run count for this project. */
46
+ live: number;
47
+ /** The issues currently live, so a refusal can name exactly what would be orphaned. */
48
+ issues: number[];
49
+ }
50
+
51
+ /**
52
+ * Every configured project's live workload. The refusal half of #545: the
53
+ * shared daemon serves all projects, so a stop/restart must account for every
54
+ * live run — not just the one the request happens to name — and it has to name
55
+ * the issues, not just a count, so an operator can see what `--force` would
56
+ * orphan. Read-only over the store; never writes.
57
+ */
58
+ export function liveWorkload(
59
+ store: { liveRuns(project: string): readonly { issue: number }[] },
60
+ ): LiveWorkload[] {
61
+ const cfg = loadConfig();
62
+ return cfg.projects.map((project) => {
63
+ const runs = store.liveRuns(project.name);
64
+ return { project: project.name, live: runs.length, issues: runs.map((r) => r.issue) };
65
+ });
66
+ }
67
+
68
+ /**
69
+ * The refusal gate for a stop/restart of the shared daemon (#545).
70
+ *
71
+ * Refuses while ANY configured project — the named one and its siblings alike
72
+ * — has a live run, since the process is global and stopping it orphans every
73
+ * project's workers. Throws with the live project(s) and issue numbers so the
74
+ * operator sees the whole blast radius, never a project-local count.
75
+ *
76
+ * `override` (an explicit `--force`) is the escape hatch that keeps a wedged
77
+ * daemon stoppable: the caller skips this gate on `--force` and records the
78
+ * override in the stop provenance rather than silencing it.
79
+ */
80
+ export function refuseBusyDaemon(
81
+ workload: LiveWorkload[],
82
+ o: { override: boolean; verb: "stop" | "restart" },
83
+ ): void {
84
+ const busy = workload.filter((w) => w.live > 0);
85
+ if (busy.length === 0 || o.override) return;
86
+ const live = busy.map((w) => `project ${w.project}: #${w.issues.join(", #")}`).join("; ");
87
+ throw new Error(
88
+ `refusing to ${o.verb} the shared daemon while work is live — ${live}. ` +
89
+ `Re-run with --force to ${o.verb} anyway (the override is recorded in the daemon-stop provenance).`,
90
+ );
91
+ }
92
+
40
93
  export function buildStopProvenance(spec: StopProvenanceSpec): DaemonStopDraft {
41
94
  const cfg = loadConfig();
42
95
  const store = openStore(dbPath());