omp-conductor 0.14.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/escalate.ts CHANGED
@@ -269,7 +269,7 @@ export function createEscalator(
269
269
  // the tick re-raises it. A *report* has no such source — it exists
270
270
  // once, in the model's head, and nothing regenerates it — which is
271
271
  // why that one needed a ledger and this one does not.
272
- await sendTelegram(token, chatId, text);
272
+ await sendTelegram(token, chatId, text, { topicId: p.escalation.telegramTopicId });
273
273
  store.markNotified(key);
274
274
  return;
275
275
  }
@@ -357,14 +357,52 @@ export function readTelegramToken(): string | undefined {
357
357
  * Telegram's own message id when it returns one, and throws on every *known*
358
358
  * failure — connection refused, HTTP error, `{"ok":false}` — which is what lets
359
359
  * a caller treat a throw as "nobody has this" and a crash as "nobody knows".
360
+ *
361
+ * Optional `topicId` pins the message to a forum topic (`message_thread_id`).
362
+ * A definitive missing-thread reject retries once as a flat chat and warns, so a
363
+ * deleted topic degrades instead of silently losing the page (#318).
360
364
  */
361
365
  export async function sendTelegram(
362
366
  token: string,
363
367
  chatId: string,
364
368
  text: string,
369
+ opts?: { topicId?: number },
370
+ ): Promise<number | undefined> {
371
+ const topicId =
372
+ opts?.topicId !== undefined && Number.isFinite(opts.topicId) && Number.isSafeInteger(opts.topicId)
373
+ ? opts.topicId
374
+ : undefined;
375
+ try {
376
+ return await postTelegramMessage(token, chatId, text, topicId);
377
+ } catch (err) {
378
+ if (
379
+ topicId === undefined ||
380
+ !(err instanceof TelegramSendError) ||
381
+ err.outcome !== "definitive" ||
382
+ !isMissingTelegramThread(err.message)
383
+ ) {
384
+ throw err;
385
+ }
386
+ warn(
387
+ `escalation.telegramTopicId=${topicId} is stale (${err.message}); retrying flat chat`,
388
+ );
389
+ return await postTelegramMessage(token, chatId, text, undefined);
390
+ }
391
+ }
392
+
393
+ /** Telegram's definitive "that forum topic is gone" answers. */
394
+ function isMissingTelegramThread(diagnostic: string): boolean {
395
+ return /message thread not found|topic_id_invalid/i.test(diagnostic);
396
+ }
397
+
398
+ async function postTelegramMessage(
399
+ token: string,
400
+ chatId: string,
401
+ text: string,
402
+ topicId: number | undefined,
365
403
  ): Promise<number | undefined> {
366
404
  const url = `https://api.telegram.org/bot${token}/sendMessage`;
367
- const body = JSON.stringify({
405
+ const payload: Record<string, unknown> = {
368
406
  chat_id: chatId,
369
407
  // ponytail: hard truncation rather than splitting across messages — the
370
408
  // tail of a stack trace is rarely the interesting part. Upgrade path is to
@@ -372,7 +410,9 @@ export async function sendTelegram(
372
410
  text:
373
411
  text.length > TELEGRAM_TEXT_LIMIT ? `${text.slice(0, TELEGRAM_TEXT_LIMIT)}\n[truncated]` : text,
374
412
  disable_web_page_preview: true,
375
- });
413
+ };
414
+ if (topicId !== undefined) payload.message_thread_id = topicId;
415
+ const body = JSON.stringify(payload);
376
416
 
377
417
  let res: Response;
378
418
  try {
package/src/fleet.ts CHANGED
@@ -10,7 +10,7 @@
10
10
  * never systemctl stop herdr-fleet
11
11
  * - arm / disarm — first-class armed marker (arm is inbound Telegram proof)
12
12
  * - releaseHold — clear pause only; never re-arms
13
- * - release-pane — clear the halt --pane recovery pin
13
+ * - resume — clear pause and the stop --pane recovery pin
14
14
  */
15
15
 
16
16
  import { spawnSync } from "node:child_process";
@@ -29,6 +29,7 @@ import { homedir } from "node:os";
29
29
  import { dirname, join } from "node:path";
30
30
  import { formatZonedMinute } from "./availability.ts";
31
31
  import { findProject, loadConfig, stateDir } from "./config.ts";
32
+ import { sendTelegram } from "./escalate.ts";
32
33
  import { planUsageLine, readPlanUsage, sharedUsageSource } from "./usage.ts";
33
34
  import { readApprovalSurface, readDaemonProfile } from "./approval-surface.ts";
34
35
  import { inspectBriefLayout } from "./brief-upgrade.ts";
@@ -59,11 +60,16 @@ import {
59
60
  SYSTEMD_UNIT,
60
61
  } from "./lifecycle.ts";
61
62
  import { formatRss, rssBytesFromHealthz } from "./host.ts";
63
+ import type { WorkerPausePhase } from "./worker.ts";
62
64
  import { fetchRateLimit } from "./tracker/github.ts";
63
65
  import {
66
+ LEGACY_ARM_MARKER_DETAIL,
67
+ legacyArmedMarkerPath,
64
68
  readTickConfig,
65
69
  readTickRuntimeStatus,
70
+ resolveArmState,
66
71
  TICK_CONFIG_FILE,
72
+ tickConfigMatchesProject,
67
73
  type TickConfig,
68
74
  type TickConfigResult,
69
75
  } from "./orchestrator-tick.ts";
@@ -71,6 +77,14 @@ import {
71
77
  export const PANE_HALT_FILE = ".conductor-pane-halted";
72
78
  export const DEFAULT_HERDR_UNIT = "herdr-fleet.service";
73
79
 
80
+ /**
81
+ * Default Herdr session name for conductor fleets. Renamed from `fleet` in
82
+ * #320; `HERDR_SESSION` still wins when set. Legacy hosts that still run a
83
+ * session named `fleet` are bridged for one release by
84
+ * {@link resolveHerdrSession}.
85
+ */
86
+ export const DEFAULT_HERDR_SESSION = "conductor";
87
+
74
88
  /**
75
89
  * Agent name assumed when no tick config names one. Mirrors
76
90
  * `herdr-conductor`'s own `AGENT_NAME=${AGENT_NAME:-fleet}` default — a
@@ -80,6 +94,89 @@ export const DEFAULT_HERDR_UNIT = "herdr-fleet.service";
80
94
  export const DEFAULT_FLEET_AGENT_NAME = "fleet";
81
95
  export const ARM_CHALLENGE_TIMEOUT_MS = 300_000;
82
96
 
97
+
98
+ /** Pre-#320 session name. Bridged for one release when `conductor` is empty. */
99
+ const LEGACY_HERDR_SESSION = "fleet";
100
+
101
+ export const LEGACY_HERDR_SESSION_HINT =
102
+ 'herdr session "fleet" found — rename it to "conductor" or set HERDR_SESSION=fleet (remove this bridge next minor)';
103
+
104
+ let legacyHerdrSessionHintPrinted = false;
105
+
106
+ function noteLegacyHerdrSession(log?: (message: string) => void): void {
107
+ if (legacyHerdrSessionHintPrinted) return;
108
+ legacyHerdrSessionHintPrinted = true;
109
+ if (log) log(LEGACY_HERDR_SESSION_HINT);
110
+ else console.warn(LEGACY_HERDR_SESSION_HINT);
111
+ }
112
+
113
+ /** Reset the once-per-process rename hint (tests). */
114
+ export function resetLegacyHerdrSessionHintForTests(): void {
115
+ legacyHerdrSessionHintPrinted = false;
116
+ }
117
+
118
+ /**
119
+ * Resolve the Herdr session name. `HERDR_SESSION` is always authoritative.
120
+ * When unset, default to {@link DEFAULT_HERDR_SESSION}. The one-release bridge
121
+ * that prefers a populated legacy `fleet` session lives in
122
+ * {@link resolveHerdrSessionWithBridge} — call that at probe sites.
123
+ */
124
+ export function resolveHerdrSession(env: NodeJS.ProcessEnv = process.env): string {
125
+ const explicit = env["HERDR_SESSION"];
126
+ if (explicit !== undefined && explicit.length > 0) return explicit;
127
+ return DEFAULT_HERDR_SESSION;
128
+ }
129
+
130
+ function herdrAgentListRaw(
131
+ session: string,
132
+ bin: string,
133
+ env: NodeJS.ProcessEnv,
134
+ ): { ok: true; agents: HerdrAgent[] } | { ok: false } {
135
+ const res = spawnSync(bin, ["--session", session, "agent", "list"], {
136
+ encoding: "utf8",
137
+ timeout: 8_000,
138
+ env,
139
+ });
140
+ if (res.error || res.status !== 0) return { ok: false };
141
+ try {
142
+ return { ok: true, agents: parseHerdrAgentList(res.stdout ?? "") };
143
+ } catch {
144
+ return { ok: false };
145
+ }
146
+ }
147
+
148
+ /**
149
+ * One-release bridge (#320): when `HERDR_SESSION` is unset, a `conductor`
150
+ * session with no agents while a `fleet` session answers → use `fleet` and
151
+ * print the rename hint once. Remove next minor.
152
+ */
153
+ export function resolveHerdrSessionWithBridge(opts: {
154
+ env?: NodeJS.ProcessEnv;
155
+ herdrBin?: string;
156
+ log?: (message: string) => void;
157
+ } = {}): string {
158
+ const env = opts.env ?? process.env;
159
+ const explicit = env["HERDR_SESSION"];
160
+ if (explicit !== undefined && explicit.length > 0) return explicit;
161
+
162
+ const bin = opts.herdrBin ?? "herdr";
163
+ const primary = herdrAgentListRaw(DEFAULT_HERDR_SESSION, bin, env);
164
+ if (primary.ok && primary.agents.length > 0) return DEFAULT_HERDR_SESSION;
165
+
166
+ const legacy = herdrAgentListRaw(LEGACY_HERDR_SESSION, bin, env);
167
+ if (legacy.ok && legacy.agents.length > 0) {
168
+ noteLegacyHerdrSession(opts.log);
169
+ return LEGACY_HERDR_SESSION;
170
+ }
171
+ // conductor answered (even empty) wins over a dead legacy session.
172
+ if (primary.ok) return DEFAULT_HERDR_SESSION;
173
+ if (legacy.ok) {
174
+ noteLegacyHerdrSession(opts.log);
175
+ return LEGACY_HERDR_SESSION;
176
+ }
177
+ return DEFAULT_HERDR_SESSION;
178
+ }
179
+
83
180
  export function telegramStateDir(): string {
84
181
  const override = process.env["OMP_TELEGRAM_STATE_DIR"];
85
182
  if (override !== undefined && override.length > 0) return override;
@@ -104,25 +201,69 @@ export type ResolvedTick =
104
201
  | { kind: "invalid"; path: string; cwd: string; problem: string }
105
202
  | { kind: "ok"; path: string; cwd: string; config: TickConfig };
106
203
 
204
+ /**
205
+ * The tick config that belongs to this project, skipping any that names another
206
+ * one. The search roots overlap between projects — `stateDir()` and the shared
207
+ * parent of two fleet cwds are read for every one of them — so a stamped config
208
+ * sitting in a shared root would otherwise answer for whichever project asked
209
+ * first. An unstamped (pre-multi-project) config still matches anything, which
210
+ * is what keeps a single-project fleet on the file it already has.
211
+ */
107
212
  export function resolveTickConfig(projectName?: string): ResolvedTick {
108
213
  for (const cwd of tickConfigSearchRoots(projectName)) {
109
214
  const r: TickConfigResult = readTickConfig(cwd);
110
- if (r.kind === "ok") return { kind: "ok", path: r.path, cwd, config: r.config };
215
+ if (r.kind === "ok") {
216
+ if (!tickConfigMatchesProject(r.config, projectName)) continue;
217
+ return { kind: "ok", path: r.path, cwd, config: r.config };
218
+ }
111
219
  if (r.kind === "invalid") return { kind: "invalid", path: r.path, cwd, problem: r.problem };
112
220
  }
113
221
  return { kind: "absent" };
114
222
  }
115
223
 
224
+ /**
225
+ * Where this project's arm marker lives. The per-project fallback is the point:
226
+ * one shared `<stateDir>/armed` meant arming any project armed all of them. The
227
+ * bare name survives only for the un-named single-project call, which is the
228
+ * pre-multi-project spelling.
229
+ */
116
230
  export function armedMarkerPath(projectName?: string): string {
231
+ return armGate(projectName).path;
232
+ }
233
+
234
+ /**
235
+ * This project's arm marker together with the project name that identifies it —
236
+ * the pair every arm decision needs, from one tick-config read. `named` prefers
237
+ * the stamped {@link TickConfig.project} over the caller's `--project`, because
238
+ * the stamp is what the marker was generated from.
239
+ */
240
+ function armGate(projectName?: string): { path: string; named?: string } {
117
241
  const tick = resolveTickConfig(projectName);
118
- if (tick.kind === "ok" && tick.config.armedFile !== undefined) return tick.config.armedFile;
119
- return join(stateDir(), "armed");
242
+ const configured = tick.kind === "ok" ? tick.config : undefined;
243
+ const named = configured?.project ?? projectName;
244
+ const path =
245
+ configured?.armedFile ??
246
+ (projectName === undefined ? legacyArmedMarkerPath() : join(stateDir(), `armed-${projectName}`));
247
+ return named === undefined ? { path } : { path, named };
120
248
  }
121
249
 
250
+ /**
251
+ * Clears the arm gate for this project — and `wasArmed` is the gate the
252
+ * heartbeat reads, not merely one file's presence.
253
+ *
254
+ * The second removal is the whole reason this is not a one-line `rmSync`. While
255
+ * the shared pre-per-project marker is still honoured (single-project host, an
256
+ * `armedFile` just restamped to `armed-<name>`), it holds the gate open as soon
257
+ * as this project's own marker is gone — so a disarm that removed only the
258
+ * per-project file would leave the fleet ticking, which is exactly what `hold`
259
+ * exists to prevent. A *stranded* shared marker arms nothing, so it is not this
260
+ * command's to remove.
261
+ */
122
262
  export function disarmTicks(projectName?: string): { path: string; wasArmed: boolean } {
123
- const path = armedMarkerPath(projectName);
124
- const wasArmed = existsSync(path);
263
+ const { path, named } = armGate(projectName);
264
+ const wasArmed = resolveArmState(path, named).armed;
125
265
  rmSync(path, { force: true });
266
+ if (resolveArmState(path, named).legacy === "honoured") rmSync(legacyArmedMarkerPath(), { force: true });
126
267
  return { path, wasArmed };
127
268
  }
128
269
 
@@ -134,7 +275,7 @@ export interface ArmResult {
134
275
  }
135
276
 
136
277
  export interface ArmDeps {
137
- sendChallenge?: (token: string, owner: string, text: string) => Promise<void>;
278
+ sendChallenge?: (token: string, owner: string, text: string, topicId?: number) => Promise<void>;
138
279
  /**
139
280
  * Waits for the challenge to appear as a user turn somewhere under the
140
281
  * session directory. The waiter owns transcript discovery — not the caller —
@@ -190,18 +331,38 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
190
331
  }
191
332
 
192
333
  const path = tick.config.armedFile;
193
- const alreadyArmed = existsSync(path);
334
+ // The gate as the heartbeat reads it, so "replaced previous marker" is not a
335
+ // lie about a fleet the shared marker was arming, and so the write below knows
336
+ // whether it is superseding that marker.
337
+ const arm = resolveArmState(path, tick.config.project ?? projectName);
338
+ const alreadyArmed = arm.armed;
194
339
  const code = makeChallengeCode();
340
+ // One bot, one chat, and — once a host runs more than one fleet — more than
341
+ // one pane that can ask. The challenge names which one, or the operator is
342
+ // answering a question they cannot attribute.
343
+ const named = tick.config.project ?? projectName;
195
344
  const text =
196
- `Fleet arming check. Reply to this chat with exactly:\n${code}\n` +
345
+ `Fleet arming check${named === undefined ? "" : ` — project ${named}`}. ` +
346
+ `Reply to this chat with exactly:\n${code}\n` +
197
347
  `Nothing will be dispatched until that reply is seen in the orchestrator session.`;
198
348
 
349
+ // Prefer the project's configured forum topic so arm challenges land where
350
+ // escalations already do (#318). Missing project config keeps flat-chat 0.13.
351
+ let topicId: number | undefined;
352
+ if (named !== undefined) {
353
+ try {
354
+ topicId = findProject(loadConfig(), named).escalation.telegramTopicId;
355
+ } catch {
356
+ /* no project config */
357
+ }
358
+ }
359
+
199
360
  const send = deps.sendChallenge ?? sendTelegramMessage;
200
361
  // Read before the send, not after: a transcript untouched since this instant
201
362
  // cannot contain the reply, and that is what the waiter filters on.
202
363
  const sentAt = (deps.now ?? Date.now)();
203
364
  try {
204
- await send(token, channel.owner, text);
365
+ await send(token, channel.owner, text, topicId);
205
366
  } catch (err) {
206
367
  throw new Error(
207
368
  `arm: outbound sendMessage failed — NOT armed: ${err instanceof Error ? err.message : String(err)}`,
@@ -232,6 +393,11 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
232
393
 
233
394
  mkdirSync(dirname(path), { recursive: true });
234
395
  writeFileSync(path, `armed ${new Date().toISOString()} owner=${channel.owner}\n`, { mode: 0o600 });
396
+ // This project now has its own marker, so the shared one it was borrowing has
397
+ // done its last job. Left in place it would survive the next `disarm` as a
398
+ // marker that re-arms the fleet, and turn into a meaningless legacy warning
399
+ // the moment a second project is configured.
400
+ if (arm.legacy === "honoured") rmSync(legacyArmedMarkerPath(), { force: true });
235
401
  return { path, alreadyArmed, owner: channel.owner, challenge: code };
236
402
  }
237
403
 
@@ -258,13 +424,13 @@ export interface HaltWithPaneResult extends HaltResult {
258
424
  }
259
425
 
260
426
  export function hold(projectName?: string, source: string = "hold"): HoldResult {
261
- const wasPaused = isPaused();
262
- setPaused(true, { source });
427
+ const wasPaused = isPaused(projectName);
428
+ setPaused(true, { source }, projectName);
263
429
  return { wasPaused, disarmed: disarmTicks(projectName) };
264
430
  }
265
431
 
266
- export function releaseHold(): void {
267
- setPaused(false);
432
+ export function releaseHold(projectName?: string): void {
433
+ setPaused(false, undefined, projectName);
268
434
  }
269
435
 
270
436
  export async function halt(projectName?: string): Promise<HaltResult> {
@@ -312,11 +478,11 @@ export function resolvePaneHaltPath(projectName?: string): ResolvedPaneHalt {
312
478
  export function paneHaltPath(projectName?: string): string {
313
479
  const resolved = resolvePaneHaltPath(projectName);
314
480
  if (resolved.kind === "unresolved") {
315
- // Verb-neutral: this is also the `release-pane` path.
481
+ // Verb-neutral: this is also `resume`'s pin-clearing path.
316
482
  throw new Error(
317
483
  `cannot locate the pane recovery pin — ${resolved.reason}. ` +
318
484
  `Drop a tick config beside the pane, or stop the agent by hand; ` +
319
- `\`halt\` without \`--pane\` still stops the dispatch daemon.`,
485
+ `\`stop\` without \`--pane\` still stops the dispatch daemon.`,
320
486
  );
321
487
  }
322
488
  return resolved.path;
@@ -328,9 +494,9 @@ export function pinPaneHalt(projectName?: string): { path: string } {
328
494
  writeFileSync(
329
495
  path,
330
496
  [
331
- `# Written by omp-conductor halt --pane at ${new Date().toISOString()}`,
497
+ `# Written by omp-conductor stop --pane at ${new Date().toISOString()}`,
332
498
  `# herdr-conductor recover.sh must not resume the fleet agent while this file exists.`,
333
- `# Clear with: omp-conductor release-pane (or rm this file)`,
499
+ `# Clear with: omp-conductor resume (or rm this file)`,
334
500
  "",
335
501
  ].join("\n"),
336
502
  { mode: 0o600 },
@@ -338,6 +504,23 @@ export function pinPaneHalt(projectName?: string): { path: string } {
338
504
  return { path };
339
505
  }
340
506
 
507
+ /**
508
+ * Clears the pin when its location is known, and reports plainly when it is not.
509
+ *
510
+ * Distinct from {@link clearPaneHalt}, which refuses an unresolvable location
511
+ * because pinning somewhere recovery never reads is worse than failing. `resume`
512
+ * needs the softer answer: it folds in the pin clear, and an operator resuming a
513
+ * fleet from a host with no tick config has no pin to clear rather than an error
514
+ * to work around.
515
+ */
516
+ export function clearPaneHaltIfResolvable(projectName?: string): { path?: string; wasHalted: boolean } {
517
+ const resolved = resolvePaneHaltPath(projectName);
518
+ if (resolved.kind !== "ok") return { wasHalted: false };
519
+ const wasHalted = existsSync(resolved.path);
520
+ rmSync(resolved.path, { force: true });
521
+ return { path: resolved.path, wasHalted };
522
+ }
523
+
341
524
  export function clearPaneHalt(projectName?: string): { path: string; wasHalted: boolean } {
342
525
  const path = paneHaltPath(projectName);
343
526
  const wasHalted = existsSync(path);
@@ -656,7 +839,9 @@ export async function stopConductorPane(
656
839
 
657
840
  async function herdrAgentList(deps: PaneStopDeps): Promise<HerdrAgent[]> {
658
841
  const bin = deps.herdrBin ?? "herdr";
659
- const session = deps.herdrSession ?? process.env["HERDR_SESSION"] ?? "fleet";
842
+ const session =
843
+ deps.herdrSession ??
844
+ resolveHerdrSessionWithBridge({ env: process.env, herdrBin: bin });
660
845
  const res = spawnSync(bin, ["--session", session, "agent", "list"], {
661
846
  encoding: "utf8",
662
847
  timeout: 8_000,
@@ -730,7 +915,9 @@ export function parseHerdrAgentList(rawOutput: string): HerdrAgent[] {
730
915
 
731
916
  async function herdrOmpForegroundPids(paneId: string, deps: PaneStopDeps): Promise<number[]> {
732
917
  const bin = deps.herdrBin ?? "herdr";
733
- const session = deps.herdrSession ?? process.env["HERDR_SESSION"] ?? "fleet";
918
+ const session =
919
+ deps.herdrSession ??
920
+ resolveHerdrSessionWithBridge({ env: process.env, herdrBin: bin });
734
921
  const res = spawnSync(bin, ["--session", session, "pane", "process-info", "--pane", paneId], {
735
922
  encoding: "utf8",
736
923
  timeout: 8_000,
@@ -836,7 +1023,7 @@ export interface FleetLayers {
836
1023
 
837
1024
  export function fleetLayers(projectName?: string): FleetLayers {
838
1025
  const rec = livingDaemon();
839
- const paused = isPaused();
1026
+ const paused = isPaused(projectName);
840
1027
  const dispatch: DispatchLayer = rec === undefined ? "stopped" : paused ? "paused" : "running";
841
1028
 
842
1029
  const tick = resolveTickConfig(projectName);
@@ -857,12 +1044,21 @@ export function fleetLayers(projectName?: string): FleetLayers {
857
1044
  if (tick.config.armedFile === undefined) {
858
1045
  ticks = "ungated";
859
1046
  ticksDetail = "no armedFile in tick config — heartbeat sends without an arm marker";
860
- } else if (existsSync(tick.config.armedFile)) {
861
- ticks = "armed";
862
- ticksDetail = tick.config.armedFile;
863
1047
  } else {
864
- ticks = "disarmed";
865
- ticksDetail = tick.config.armedFile;
1048
+ // The same decision the heartbeat itself makes, so status can never claim
1049
+ // armed while the tick skips (or the reverse) — including the two shared
1050
+ // marker cases: honoured through an upgrade on a single-project host,
1051
+ // stranded once a second project exists.
1052
+ const arm = resolveArmState(tick.config.armedFile, tick.config.project ?? projectName);
1053
+ ticks = arm.armed ? "armed" : "disarmed";
1054
+ ticksDetail =
1055
+ arm.legacy === undefined
1056
+ ? tick.config.armedFile
1057
+ : `${tick.config.armedFile} — ${legacyArmedMarkerPath()}: ${
1058
+ arm.legacy === "honoured"
1059
+ ? "legacy global arm marker still honoured; re-arm this project to replace it"
1060
+ : LEGACY_ARM_MARKER_DETAIL
1061
+ }`;
866
1062
  }
867
1063
  const runtime = readTickRuntimeStatus(tick.cwd);
868
1064
  if (runtime !== undefined && isAlive(runtime.pid)) nextTickAt = runtime.nextTickAt;
@@ -916,8 +1112,27 @@ export function codeGraphFromHealthz(body: string | undefined, project: string):
916
1112
  if (body === undefined) return undefined;
917
1113
  try {
918
1114
  const parsed = JSON.parse(body) as Record<string, unknown>;
919
- if (parsed["project"] !== project) return undefined;
920
- const graph = parsed["codeGraph"];
1115
+ let block: Record<string, unknown> | undefined;
1116
+ if (parsed["project"] === project) {
1117
+ block = parsed;
1118
+ } else {
1119
+ const projects = parsed["projects"];
1120
+ if (Array.isArray(projects)) {
1121
+ for (const entry of projects) {
1122
+ if (
1123
+ entry !== null &&
1124
+ typeof entry === "object" &&
1125
+ !Array.isArray(entry) &&
1126
+ (entry as Record<string, unknown>)["project"] === project
1127
+ ) {
1128
+ block = entry as Record<string, unknown>;
1129
+ break;
1130
+ }
1131
+ }
1132
+ }
1133
+ }
1134
+ if (block === undefined) return undefined;
1135
+ const graph = block["codeGraph"];
921
1136
  if (graph === null || typeof graph !== "object" || Array.isArray(graph)) return undefined;
922
1137
  const value = graph as Record<string, unknown>;
923
1138
  if (value["configured"] === false) return { configured: false };
@@ -942,6 +1157,57 @@ export function codeGraphFromHealthz(body: string | undefined, project: string):
942
1157
  }
943
1158
  }
944
1159
 
1160
+
1161
+ /** Live worker pause phases from a daemon `/healthz` body for one project. */
1162
+ export function workerPhasesFromHealthz(
1163
+ body: string | undefined,
1164
+ project: string,
1165
+ ): ReadonlyMap<number, WorkerPausePhase> {
1166
+ const phases = new Map<number, WorkerPausePhase>();
1167
+ if (body === undefined) return phases;
1168
+ try {
1169
+ const payload = JSON.parse(body) as unknown;
1170
+ if (payload === null || typeof payload !== "object") return phases;
1171
+ let block: object | undefined;
1172
+ if (Reflect.get(payload, "project") === project) {
1173
+ block = payload;
1174
+ } else {
1175
+ const projects = Reflect.get(payload, "projects");
1176
+ if (Array.isArray(projects)) {
1177
+ for (const entry of projects) {
1178
+ if (
1179
+ entry !== null &&
1180
+ typeof entry === "object" &&
1181
+ Reflect.get(entry, "project") === project
1182
+ ) {
1183
+ block = entry;
1184
+ break;
1185
+ }
1186
+ }
1187
+ }
1188
+ }
1189
+ if (block === undefined) return phases;
1190
+ const workers = Reflect.get(block, "workers");
1191
+ if (!Array.isArray(workers)) return phases;
1192
+ for (const worker of workers) {
1193
+ if (worker === null || typeof worker !== "object") continue;
1194
+ const issue = Reflect.get(worker, "issue");
1195
+ const phase = Reflect.get(worker, "phase");
1196
+ if (
1197
+ Number.isSafeInteger(issue) &&
1198
+ (issue as number) > 0 &&
1199
+ (phase === "pausing" || phase === "paused")
1200
+ ) {
1201
+ phases.set(issue as number, phase);
1202
+ }
1203
+ }
1204
+ } catch {
1205
+ // An unreadable health body means no trustworthy pause phase.
1206
+ }
1207
+ return phases;
1208
+ }
1209
+
1210
+
945
1211
  export function formatCodeGraphHealth(graph: CodeGraphHealth, now = Date.now()): string | undefined {
946
1212
  if (!graph.configured) return undefined;
947
1213
  const indexed = graph.repos.filter((repo) => repo.index === "present").length;
@@ -971,6 +1237,7 @@ export function formatFleetStatus(
971
1237
  brief: string | undefined = undefined,
972
1238
  decisions: string | undefined = undefined,
973
1239
  failureClasses: string | undefined = undefined,
1240
+ workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
974
1241
  ): string {
975
1242
  const tickLine =
976
1243
  layers.ticksDetail === undefined
@@ -1032,12 +1299,12 @@ export function formatFleetStatus(
1032
1299
  const dispatchLine =
1033
1300
  layers.dispatch === "paused"
1034
1301
  ? (() => {
1035
- const prov = pauseProvenance();
1302
+ const prov = pauseProvenance(s.project);
1036
1303
  if (prov !== undefined) {
1037
1304
  const reason = prov.reason === undefined ? "" : ` — "${prov.reason}"`;
1038
1305
  return `dispatch paused (source: ${prov.source}${reason})`;
1039
1306
  }
1040
- return isPaused() && pausedAt() === undefined
1307
+ return isPaused(s.project) && pausedAt(s.project) === undefined
1041
1308
  ? "dispatch paused (unparseable sentinel — new work and completion mutations refused; release gates remain available)"
1042
1309
  : "dispatch paused";
1043
1310
  })()
@@ -1057,7 +1324,7 @@ export function formatFleetStatus(
1057
1324
  ...(graphBlock === undefined ? [] : [graphBlock]),
1058
1325
  daemonBlock,
1059
1326
  "",
1060
- formatProjectBody(s),
1327
+ formatProjectBody(s, workerPhases),
1061
1328
  ].join("\n");
1062
1329
  }
1063
1330
 
@@ -1088,7 +1355,10 @@ function formatDigestScheduleStatus(s: StatusSnapshot): string[] {
1088
1355
  ];
1089
1356
  }
1090
1357
 
1091
- function formatProjectBody(s: StatusSnapshot): string {
1358
+ function formatProjectBody(
1359
+ s: StatusSnapshot,
1360
+ workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
1361
+ ): string {
1092
1362
  const lines = [
1093
1363
  `project ${s.project}${s.paused ? " (PAUSED)" : ""}`,
1094
1364
  ...(s.pauseReason === undefined ? [] : [`paused ${s.pauseReason}`]),
@@ -1158,9 +1428,11 @@ function formatProjectBody(s: StatusSnapshot): string {
1158
1428
  } else {
1159
1429
  lines.push("active runs");
1160
1430
  for (const r of s.activeRuns) {
1431
+ const phase = workerPhases.get(r.issue);
1432
+ const state = phase === "pausing" || phase === "paused" ? phase : r.state;
1161
1433
  lines.push(
1162
- ` #${r.issue} ${r.repo} ${r.state} attempt ${r.attempt} ` +
1163
- `${r.turns}/${r.maxTurns} turns $${r.spendUsd.toFixed(2)} ${r.branch}` +
1434
+ ` #${r.issue} ${r.repo} ${state} attempt ${r.attempt} ` +
1435
+ `${r.turns}/${r.maxTurns} turns ${r.spendUsd.toFixed(2)} ${r.branch}` +
1164
1436
  (r.prUrl ? ` ${r.prUrl}` : ""),
1165
1437
  );
1166
1438
  // The orchestrator's Duty 1 reads this command, and a flagged run's
@@ -1203,6 +1475,7 @@ export async function renderStatus(projectName?: string): Promise<string> {
1203
1475
  ]);
1204
1476
  const cached = codeGraphFromHealthz(health?.body, project.name);
1205
1477
  const codeGraph = cached ?? (await probeCodeGraph(project));
1478
+ const workerPhases = workerPhasesFromHealthz(health?.body, project.name);
1206
1479
  return formatFleetStatus(
1207
1480
  { ...s, planUsage, github },
1208
1481
  layers,
@@ -1213,6 +1486,7 @@ export async function renderStatus(projectName?: string): Promise<string> {
1213
1486
  briefStatusLine(project),
1214
1487
  decisionStatusLine(project.name),
1215
1488
  failureClassBlock(project.name),
1489
+ workerPhases,
1216
1490
  );
1217
1491
  }
1218
1492
 
@@ -1433,14 +1707,14 @@ function makeChallengeCode(): string {
1433
1707
  return `FLEET-${hex}`;
1434
1708
  }
1435
1709
 
1436
- async function sendTelegramMessage(token: string, owner: string, text: string): Promise<void> {
1437
- const url = `https://api.telegram.org/bot${token}/sendMessage`;
1438
- const body = new URLSearchParams({ chat_id: owner, text });
1439
- const res = await fetch(url, { method: "POST", body, signal: AbortSignal.timeout(20_000) });
1440
- const json = (await res.json()) as { ok?: boolean; description?: string };
1441
- if (!res.ok || json.ok !== true) {
1442
- throw new Error(json.description ?? `HTTP ${res.status}`);
1443
- }
1710
+ async function sendTelegramMessage(
1711
+ token: string,
1712
+ owner: string,
1713
+ text: string,
1714
+ topicId?: number,
1715
+ ): Promise<void> {
1716
+ // Shared transport: stale-topic retry + message_thread_id live in one place.
1717
+ await sendTelegram(token, owner, text, { topicId });
1444
1718
  }
1445
1719
 
1446
1720
  interface SessionScan {
@@ -1607,7 +1881,7 @@ function probeOmpPane(
1607
1881
  }
1608
1882
  const agentName =
1609
1883
  tick.kind === "ok" ? (tick.config.agentName ?? DEFAULT_FLEET_AGENT_NAME) : DEFAULT_FLEET_AGENT_NAME;
1610
- const session = process.env["HERDR_SESSION"] ?? "fleet";
1884
+ const session = resolveHerdrSessionWithBridge({ env: process.env });
1611
1885
  try {
1612
1886
  const res = spawnSync("herdr", ["--session", session, "agent", "list"], {
1613
1887
  encoding: "utf8",
@@ -0,0 +1,21 @@
1
+ /**
2
+ * Regenerator for `schema/config.schema.json`.
3
+ *
4
+ * Run via `bun run schema` from the package root. Writes the draft 2020-12 JSON
5
+ * Schema rendering of `ConfigSchema` (see `config-schema.ts`) to
6
+ * `schema/config.schema.json`, which ships in the package so editors can
7
+ * validate a hand-written config. The `config.test.ts` freshness lock fails
8
+ * when a committed schema drifts from what this script produces.
9
+ */
10
+
11
+ import { mkdirSync, writeFileSync } from "node:fs";
12
+ import { dirname, join } from "node:path";
13
+ import { fileURLToPath } from "node:url";
14
+ import { configJsonSchema } from "./config-schema.ts";
15
+
16
+ const packageDir = join(dirname(fileURLToPath(import.meta.url)), "..");
17
+ const schemaDir = join(packageDir, "schema");
18
+ mkdirSync(schemaDir, { recursive: true });
19
+ const out = join(schemaDir, "config.schema.json");
20
+ writeFileSync(out, `${JSON.stringify(configJsonSchema(), null, 2)}\n`);
21
+ console.log(`wrote ${out}`);