omp-conductor 0.15.13 → 0.16.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/REFERENCE.md +72 -2
  2. package/package.json +2 -1
  3. package/schema/config.schema.json +7 -0
  4. package/src/admission.ts +849 -0
  5. package/src/ask.ts +47 -0
  6. package/src/backups.ts +19 -7
  7. package/src/board.ts +1 -2
  8. package/src/briefs/orchestrator.md +62 -4
  9. package/src/cli.ts +26 -0
  10. package/src/commands/context.ts +3 -0
  11. package/src/commands/decision.ts +10 -1
  12. package/src/commands/doctor.ts +2 -0
  13. package/src/commands/message.ts +8 -1
  14. package/src/commands/restart.ts +15 -3
  15. package/src/commands/restore-db.ts +146 -0
  16. package/src/commands/stop.ts +24 -15
  17. package/src/commands/tail.ts +204 -44
  18. package/src/commands/unfreeze.ts +56 -0
  19. package/src/commands/watch.ts +77 -0
  20. package/src/config-schema.ts +13 -0
  21. package/src/config.ts +54 -0
  22. package/src/daemon.ts +255 -530
  23. package/src/dashboard/server.ts +2 -1
  24. package/src/decisions.ts +32 -7
  25. package/src/depends-on.ts +122 -0
  26. package/src/doctor.ts +297 -5
  27. package/src/escalate.ts +191 -19
  28. package/src/failure-class.ts +47 -0
  29. package/src/fleet.ts +168 -452
  30. package/src/gitops.ts +86 -1
  31. package/src/log.ts +40 -0
  32. package/src/model-fallback.ts +3 -2
  33. package/src/omp-settings.ts +114 -0
  34. package/src/omp.ts +39 -0
  35. package/src/orchestrator-tick.ts +7 -1
  36. package/src/reports.ts +124 -12
  37. package/src/session-host.ts +6 -0
  38. package/src/setup-wizard.ts +36 -0
  39. package/src/setup.ts +58 -1
  40. package/src/status-render.ts +445 -0
  41. package/src/stop-provenance.ts +53 -0
  42. package/src/store.ts +352 -11
  43. package/src/transcript.ts +1 -1
  44. package/src/types.ts +187 -4
  45. package/src/unblock.ts +1 -1
  46. package/src/upgrade-verify.ts +1 -1
  47. package/src/upgrade.ts +1 -2
  48. package/src/verbs/server.ts +25 -0
  49. package/src/worker.ts +358 -10
  50. package/src/worktree.ts +13 -1
package/src/fleet.ts CHANGED
@@ -26,33 +26,18 @@ import {
26
26
  } from "node:fs";
27
27
  import { createInterface } from "node:readline";
28
28
  import { homedir } from "node:os";
29
- import { dirname, join } from "node:path";
30
- import { formatZonedMinute } from "./availability.ts";
29
+ import { dirname, join, sep } from "node:path";
31
30
  import { findProject, loadConfig, stateDir } from "./config.ts";
32
31
  import { clearArmChallenge, recordArmChallenge } from "./arm-challenge.ts";
33
- import { resolveProjectTopicId, sendTelegram } from "./escalate.ts";
34
- import { planUsageLine, readPlanUsage, sharedUsageSource } from "./usage.ts";
32
+ import { resolveClaimedSessionFile, resolveProjectTopicId, sendTelegram } from "./escalate.ts";
33
+ import { readPlanUsage, sharedUsageSource } from "./usage.ts";
35
34
  import { readApprovalSurface, readDaemonProfile } from "./approval-surface.ts";
36
35
  import { inspectBriefLayout } from "./brief-upgrade.ts";
37
36
  import { dbPath, openStore } from "./store.ts";
38
37
  import { renderBriefForProject } from "./setup.ts";
39
38
  import type { DaemonStop, ProjectConfig, Store } from "./types.ts";
40
- import { settlementFlagSummary } from "./diff-flags.ts";
41
39
  import { probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
42
- import { formatDigestBacklog, formatOpenReports } from "./reports.ts";
43
- import {
44
- formatBaseHealth,
45
- formatDispatchSummary,
46
- formatReleaseGrants,
47
- formatSalvagedRuns,
48
- isPaused,
49
- pauseProvenance,
50
- pausedAt,
51
- setPaused,
52
- statusSnapshot,
53
- type StatusSnapshot,
54
- } from "./daemon.ts";
55
- import { formatOrchestratorDown } from "./orchestrator-down.ts";
40
+ import { isPaused, setPaused, statusSnapshot } from "./daemon.ts";
56
41
  import {
57
42
  healthCheck,
58
43
  isAlive,
@@ -61,11 +46,19 @@ import {
61
46
  runSystemctl,
62
47
  stopDaemon,
63
48
  type StopResult,
64
- type UnitOwnership,
65
- SYSTEMD_UNIT,
66
49
  } from "./lifecycle.ts";
67
- import { formatRss, rssBytesFromHealthz } from "./host.ts";
68
50
  import type { WorkerPausePhase } from "./worker.ts";
51
+ import {
52
+ formatFleetStatus,
53
+ type DaemonProjectHealth,
54
+ type DispatchLayer,
55
+ type FleetDaemonProbe,
56
+ type FleetLayers,
57
+ type HerdrLayer,
58
+ type PaneLayer,
59
+ type TelegramHealth,
60
+ type TicksLayer,
61
+ } from "./status-render.ts";
69
62
  import { fetchRateLimit } from "./tracker/github.ts";
70
63
  import {
71
64
  LEGACY_ARM_MARKER_DETAIL,
@@ -294,13 +287,22 @@ export interface ArmResult {
294
287
  export interface ArmDeps {
295
288
  sendChallenge?: (token: string, owner: string, text: string, topicId?: number) => Promise<void>;
296
289
  /**
297
- * Waits for the challenge to appear as a user turn somewhere under the
298
- * session directory. The waiter owns transcript discovery not the caller —
290
+ * The orchestrator session file the live omp-telegram claim names for this
291
+ * project, or undefined when there is no (readable) claim. The default
292
+ * resolves the claim the same way the send does; tests inject a fixture.
293
+ * Absent → arm watches the cwd-derived session directory, the pre-claim
294
+ * behaviour.
295
+ */
296
+ claimedSessionFile?: () => string | undefined;
297
+ /**
298
+ * Waits for the challenge to appear as a user turn somewhere under the scan
299
+ * directories. The waiter owns transcript discovery — not the caller —
299
300
  * because the reply may land in a session that starts *after* the send, so a
300
301
  * path resolved before the challenge went out can be the wrong file by the
301
- * time the operator answers (#142).
302
+ * time the operator answers (#142), and because the claimed session file can
303
+ * sit in a different directory than the tick cwd implies (#600).
302
304
  */
303
- waitForUserTurn?: (dir: string, code: string, sentAt: number, timeoutMs: number) => Promise<boolean>;
305
+ waitForUserTurn?: (dirs: readonly string[], code: string, sentAt: number, timeoutMs: number) => Promise<boolean>;
304
306
  now?: () => number;
305
307
  sleep?: (ms: number) => Promise<void>;
306
308
  timeoutMs?: number;
@@ -339,10 +341,23 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
339
341
  );
340
342
  }
341
343
 
342
- const dir = sessionDirForCwd(tick.cwd);
343
- if (!existsSync(dir)) {
344
+ // One bot, one chat, and — once a host runs more than one fleet — more than
345
+ // one pane that can ask. The challenge names which one, or the operator is
346
+ // answering a question they cannot attribute.
347
+ const named = tick.config.project ?? projectName;
348
+
349
+ // Resolve the orchestrator's live session file *before* the challenge goes
350
+ // out — the same claim the send follows (#600). A pane resumed from a
351
+ // session created elsewhere (herdr pins it to the original transcript)
352
+ // writes a session file outside the directory the tick cwd implies, and a
353
+ // cwd-derived scan would poll the one place the reply is guaranteed not to
354
+ // be. A claim outside the session tree arm scans can never be answered, so
355
+ // that is a stop, not five minutes of polling.
356
+ const claimed = deps.claimedSessionFile !== undefined ? deps.claimedSessionFile() : claimedOrchestratorSessionFile(named);
357
+ const dirs = armSessionScanDirs(tick.cwd, claimed);
358
+ if (dirs.every((d) => !existsSync(d))) {
344
359
  throw new Error(
345
- `no orchestrator session directory at ${dir} — ` +
360
+ `no orchestrator session directory under ${dirs.join(" or ")} — ` +
346
361
  `the inbound proof is read from a user turn in a transcript there. Start the pane orchestrator, let it settle, then arm again`,
347
362
  );
348
363
  }
@@ -354,10 +369,6 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
354
369
  const arm = resolveArmState(path, tick.config.project ?? projectName);
355
370
  const alreadyArmed = arm.armed;
356
371
  const code = makeChallengeCode();
357
- // One bot, one chat, and — once a host runs more than one fleet — more than
358
- // one pane that can ask. The challenge names which one, or the operator is
359
- // answering a question they cannot attribute.
360
- const named = tick.config.project ?? projectName;
361
372
  const text =
362
373
  `Fleet arming check${named === undefined ? "" : ` — project ${named}`}. ` +
363
374
  `Reply to this chat with exactly:\n${code}\n` +
@@ -398,25 +409,37 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
398
409
 
399
410
  const injected = deps.waitForUserTurn;
400
411
  const scan: SessionScan = injected
401
- ? { seen: await injected(dir, code, sentAt, timeoutMs), scanned: [], ignored: [] }
402
- : await waitForChallengeInSessions(dir, code, sentAt, timeoutMs, deps);
412
+ ? { seen: await injected(dirs, code, sentAt, timeoutMs), scanned: [], ignored: [] }
413
+ : await waitForChallengeInSessions(dirs, code, sentAt, timeoutMs, deps);
403
414
  if (!scan.seen) {
404
415
  // The proof missed its window: a lookalike reply must not stay classifiable
405
416
  // after arming gave up, so the record is cleared before the failure lands.
406
417
  clearArmChallenge(named);
418
+ // The claimed session file is named before the bridge/token/chat block: a
419
+ // rotation under the window is visible from this error alone, and on the
420
+ // host this guard exists for the claimed file is the one that held the
421
+ // answer (#600).
407
422
  const listing = [
408
- `session dir: ${dir}`,
423
+ ...(claimed === undefined
424
+ ? []
425
+ : [
426
+ `claimed orchestrator session file: ${claimed}` +
427
+ (dirs.includes(dirname(claimed))
428
+ ? " (watched)"
429
+ : ` (NOT under any watched dir — a reply there can never be seen)`),
430
+ ]),
431
+ `session dir: ${dirs.join(", ")}`,
409
432
  ...scan.scanned.map((f) => ` watched: ${f}`),
410
433
  ...scan.ignored.map((f) => ` ignored (stale, last written before the challenge): ${f}`),
411
434
  ].join("\n");
412
435
  throw new Error(
413
436
  `arm: the challenge never arrived as a user turn in time — NOT armed.\n` +
414
- `Inbound Telegram is not reaching the omp session. Check, in order:\n` +
437
+ listing +
438
+ `\nInbound Telegram is not reaching the omp session. Check, in order:\n` +
415
439
  ` * is the bridge polling? attach and run: /telegram status\n` +
416
440
  ` * is another process holding this bot token? Telegram allows exactly one\n` +
417
441
  ` getUpdates consumer and rejects the second with HTTP 409.\n` +
418
- ` * did you reply in the DM with the bot, not another chat?\n` +
419
- listing,
442
+ ` * did you reply in the DM with the bot, not another chat?\n`,
420
443
  );
421
444
  }
422
445
 
@@ -1023,46 +1046,6 @@ function isOmpProcess(proc: ForegroundProc): boolean {
1023
1046
  return false;
1024
1047
  }
1025
1048
 
1026
- // ---------------------------------------------------------------------------
1027
- // layered status
1028
- // ---------------------------------------------------------------------------
1029
-
1030
- export type DispatchLayer = "running" | "paused" | "stopped";
1031
- export type TicksLayer =
1032
- | "armed"
1033
- | "disarmed"
1034
- | "no-heartbeat-config"
1035
- | "invalid-heartbeat-config"
1036
- | "ungated";
1037
- export type PaneLayer = "live" | "missing" | "unknown";
1038
- /** `unpinnable`: no tick config, so FLEET_CWD — the only path recovery reads — is unknown. */
1039
- export type RecoveryLayer = "pinned" | "clear" | "unpinnable";
1040
- export type HerdrLayer = "active" | "inactive" | "unknown";
1041
- export type TelegramLayer = "ok" | "degraded" | "down" | "unconfigured" | "unprobed";
1042
-
1043
- export interface TelegramHealth {
1044
- kind: TelegramLayer;
1045
- detail?: string;
1046
- }
1047
-
1048
- export interface FleetLayers {
1049
- dispatch: DispatchLayer;
1050
- ticks: TicksLayer;
1051
- ticksDetail?: string;
1052
- nextTickAt?: string;
1053
- pane: PaneLayer;
1054
- paneDetail?: string;
1055
- recovery: RecoveryLayer;
1056
- recoveryDetail?: string;
1057
- herdr: HerdrLayer;
1058
- herdrDetail?: string;
1059
- armedPath?: string;
1060
- tickConfigPath?: string;
1061
- paneHaltPath?: string;
1062
- paused: boolean;
1063
- daemon: { running: boolean; pid?: number; port?: number };
1064
- }
1065
-
1066
1049
  /**
1067
1050
  * The project a bare read means, when the config leaves no doubt.
1068
1051
  *
@@ -1270,50 +1253,6 @@ export function workerPhasesFromHealthz(
1270
1253
  }
1271
1254
 
1272
1255
 
1273
- export function formatCodeGraphHealth(graph: CodeGraphHealth, now = Date.now()): string | undefined {
1274
- if (!graph.configured) return undefined;
1275
- const indexed = graph.repos.filter((repo) => repo.index === "present").length;
1276
- let refresh: string = graph.refresh.result;
1277
- if (graph.refresh.lastSuccessAt !== undefined) {
1278
- const ageMs = Math.max(0, now - Date.parse(graph.refresh.lastSuccessAt));
1279
- refresh = `${graph.refresh.lastSuccessAt} (${Math.max(1, Math.ceil(ageMs / 60_000))}m ago)`;
1280
- }
1281
- return [
1282
- `code graph ${graph.status} ${indexed}/${graph.repos.length} repos indexed`,
1283
- ` indexer ${graph.prerequisites.indexer}`,
1284
- ` MCP mount ${graph.prerequisites.mcpMount}`,
1285
- ` timer ${graph.timer.enabled} / ${graph.timer.active}`,
1286
- ` refresh ${refresh}`,
1287
- ...graph.reasons.map((reason) => ` - ${reason}`),
1288
- ].join("\n");
1289
- }
1290
-
1291
-
1292
- /**
1293
- * Project-scoped reading of a living daemon record + `/healthz` body.
1294
- *
1295
- * Board and `status` share this so a host running one daemon for several
1296
- * projects cannot be called healthy for a project it does not serve (#379).
1297
- * A third interpretation convention is forbidden.
1298
- */
1299
- export type DaemonProjectHealth =
1300
- | { kind: "stopped" }
1301
- | { kind: "ok" }
1302
- | { kind: "unreachable" }
1303
- | { kind: "other-project"; serves?: string };
1304
-
1305
- /**
1306
- * Facts `formatFleetStatus` needs about the living daemon. Pure input — the
1307
- * caller probes; the formatter never shells out (#379).
1308
- */
1309
- export type FleetDaemonProbe = {
1310
- project: DaemonProjectHealth;
1311
- /** `/healthz` body when the probe is project-ok (rss / overlays). */
1312
- body?: string;
1313
- /** systemd ownership of the living record pid; omit when unprobed. */
1314
- unit?: UnitOwnership;
1315
- };
1316
-
1317
1256
  /**
1318
1257
  * Decide whether a living record + optional `/healthz` answer serve `project`.
1319
1258
  *
@@ -1372,306 +1311,6 @@ function servedProjectName(payload: object): string | undefined {
1372
1311
  return undefined;
1373
1312
  }
1374
1313
 
1375
- function formatDaemonHealthz(probe: FleetDaemonProbe | undefined): string {
1376
- if (probe === undefined) return "unprobed";
1377
- switch (probe.project.kind) {
1378
- case "stopped":
1379
- return "stopped";
1380
- case "ok":
1381
- return "ok";
1382
- case "unreachable":
1383
- return "unreachable — the process is up but not serving";
1384
- case "other-project":
1385
- return probe.project.serves === undefined
1386
- ? "other-project"
1387
- : `other-project — serves ${probe.project.serves}`;
1388
- }
1389
- }
1390
-
1391
- /**
1392
- * Unit line for a living record. The bare unit name alone used to imply
1393
- * systemd ownership that was never checked (#379); every branch states the
1394
- * probe result, and `unknown` is never rendered as owned.
1395
- */
1396
- function formatDaemonUnit(unit: UnitOwnership | undefined, recordPid: number): string {
1397
- if (unit === undefined) return ` unit ${SYSTEMD_UNIT} unprobed`;
1398
- switch (unit.kind) {
1399
- case "active":
1400
- if (unit.pid === recordPid) {
1401
- return ` unit ${SYSTEMD_UNIT} systemd-owned`;
1402
- }
1403
- // Live unit, different MainPID — the record pid is not systemd's.
1404
- return ` unit ${SYSTEMD_UNIT} unmanaged — MainPID ${unit.pid}`;
1405
- case "failed":
1406
- return ` unit ${SYSTEMD_UNIT} unmanaged — unit failed`;
1407
- case "inactive":
1408
- return ` unit ${SYSTEMD_UNIT} inactive`;
1409
- case "unknown":
1410
- return ` unit ${SYSTEMD_UNIT} unknown — ${unit.reason}`;
1411
- }
1412
- }
1413
-
1414
- /**
1415
- * The newest daemon stop/restart provenance as status lines (#378).
1416
- *
1417
- * Rendered under the daemon block whether the daemon is down or has since
1418
- * been restarted — the whole point is that the debrief survives the restart.
1419
- * An unattributed record says so explicitly: the external-signal fallback
1420
- * records what the receiving daemon knew (that it was unattributed, when,
1421
- * which projects it served with live counts) rather than guessing a caller.
1422
- */
1423
- function formatLastStop(lastStop: DaemonStop | undefined): string[] {
1424
- if (lastStop === undefined) return [];
1425
- const caller = lastStop.unattributed
1426
- ? "unattributed — no mediated request (external signal)"
1427
- : `pid ${lastStop.callerPid ?? "?"}` +
1428
- (lastStop.callerUid === undefined ? "" : ` uid ${lastStop.callerUid}`) +
1429
- (lastStop.role === undefined ? "" : ` (${lastStop.role})`);
1430
- return [
1431
- ` last stop ${new Date(lastStop.at).toISOString()} ${lastStop.controlPath}`,
1432
- ` caller ${caller}`,
1433
- ` scope ${lastStop.scope}${lastStop.project === undefined ? "" : `: ${lastStop.project}`}`,
1434
- ` affects ${lastStop.affected.map((a) => `${a.project} (${a.live} live)`).join(", ")}`,
1435
- ` reason ${lastStop.reason}`,
1436
- ];
1437
- }
1438
-
1439
- export function formatFleetStatus(
1440
- s: StatusSnapshot,
1441
- layers: FleetLayers,
1442
- daemon: FleetDaemonProbe | undefined = undefined,
1443
- telegram: TelegramHealth = { kind: "unprobed" },
1444
- now = Date.now(),
1445
- codeGraph: CodeGraphHealth = { configured: false },
1446
- brief: string | undefined = undefined,
1447
- decisions: string | undefined = undefined,
1448
- failureClasses: string | undefined = undefined,
1449
- workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
1450
- intake: string | undefined = undefined,
1451
- lastStop: DaemonStop | undefined = undefined,
1452
- ): string {
1453
- const tickLine =
1454
- layers.ticksDetail === undefined
1455
- ? `ticks ${layers.ticks}`
1456
- : `ticks ${layers.ticks} (${layers.ticksDetail})`;
1457
- let nextTickLine: string | undefined;
1458
- if (layers.nextTickAt !== undefined) {
1459
- const delta = Date.parse(layers.nextTickAt) - now;
1460
- const minutes = Math.max(1, Math.ceil(Math.abs(delta) / 60_000));
1461
- nextTickLine =
1462
- `next tick ${layers.nextTickAt} ` +
1463
- `(${delta >= 0 ? `in ${minutes}m` : `overdue by ${minutes}m`})`;
1464
- }
1465
- const paneLine =
1466
- layers.paneDetail === undefined
1467
- ? `pane ${layers.pane}`
1468
- : `pane ${layers.pane} (${layers.paneDetail})`;
1469
- const recoveryLine =
1470
- layers.recoveryDetail === undefined
1471
- ? `recovery ${layers.recovery}`
1472
- : `recovery ${layers.recovery} (${layers.recoveryDetail})`;
1473
- const herdrLine =
1474
- layers.herdrDetail === undefined
1475
- ? `herdr ${layers.herdr}`
1476
- : `herdr ${layers.herdr} (${layers.herdrDetail})`;
1477
- const telegramLine =
1478
- telegram.detail === undefined
1479
- ? `telegram ${telegram.kind}`
1480
- : `telegram ${telegram.kind} (${telegram.detail})`;
1481
-
1482
- let daemonBlock: string;
1483
- if (!layers.daemon.running || layers.daemon.pid === undefined) {
1484
- daemonBlock = "daemon not running";
1485
- } else {
1486
- // Only trust rss from a project-ok body — a foreign payload's bytes are
1487
- // not this project's daemon facts (#379).
1488
- const rss =
1489
- daemon?.project.kind === "ok" ? rssBytesFromHealthz(daemon.body) : undefined;
1490
- daemonBlock = [
1491
- "daemon",
1492
- ` pid ${layers.daemon.pid}`,
1493
- ` port ${layers.daemon.port ?? "?"}`,
1494
- ...(rss === undefined ? [] : [` rss ${formatRss(rss)}`]),
1495
- ` healthz ${formatDaemonHealthz(daemon)}`,
1496
- formatDaemonUnit(daemon?.unit, layers.daemon.pid),
1497
- ].join("\n");
1498
- }
1499
-
1500
- const graphBlock = formatCodeGraphHealth(codeGraph, now);
1501
-
1502
- // Pause provenance (`pause --reason`, an integrity/spend-cap/upgrade pause,
1503
- // a halt) answers "who stopped the fleet" without opening a file (#185). An
1504
- // unparseable sentinel — paused but with no datable line 1 — is itself news:
1505
- // it means a run admitted before an *unknown* pause cannot prove innocence
1506
- // (#174), so completion mutations fail closed while release gates remain usable.
1507
- const dispatchLine =
1508
- layers.dispatch === "paused"
1509
- ? (() => {
1510
- const prov = pauseProvenance(s.project);
1511
- if (prov !== undefined) {
1512
- const reason = prov.reason === undefined ? "" : ` — "${prov.reason}"`;
1513
- return `dispatch paused (source: ${prov.source}${reason})`;
1514
- }
1515
- return isPaused(s.project) && pausedAt(s.project) === undefined
1516
- ? "dispatch paused (unparseable sentinel — new work and completion mutations refused; release gates remain available)"
1517
- : "dispatch paused";
1518
- })()
1519
- : `dispatch ${layers.dispatch}`;
1520
-
1521
- return [
1522
- dispatchLine,
1523
- tickLine,
1524
- ...(nextTickLine === undefined ? [] : [nextTickLine]),
1525
- paneLine,
1526
- recoveryLine,
1527
- herdrLine,
1528
- telegramLine,
1529
- ...(brief === undefined ? [] : [brief]),
1530
- ...(decisions === undefined ? [] : [decisions]),
1531
- ...(failureClasses === undefined ? [] : [failureClasses]),
1532
- ...(intake === undefined ? [] : [intake]),
1533
- ...(graphBlock === undefined ? [] : [graphBlock]),
1534
- daemonBlock,
1535
- ...formatLastStop(lastStop),
1536
- "",
1537
- formatProjectBody(s, workerPhases, now),
1538
- ].join("\n");
1539
- }
1540
-
1541
- function formatAvailabilityStatus(s: StatusSnapshot): string[] {
1542
- const availability = s.availability;
1543
- if (availability === undefined) return [];
1544
- if (availability.mode === "always") {
1545
- return ["availability 24-hour interrupts (no weekly window)"];
1546
- }
1547
- const mode =
1548
- availability.nextTransitionAt === undefined || availability.timezone === undefined
1549
- ? `${availability.mode}; next transition could not be calculated`
1550
- : `${availability.mode} until ${formatZonedMinute(availability.nextTransitionAt, availability.timezone)}`;
1551
- const bypass = availability.bypass.length === 0 ? "none" : availability.bypass.join(", ");
1552
- return [`availability ${mode}; quiet-hours bypass ${bypass}`];
1553
- }
1554
-
1555
- function formatDigestScheduleStatus(s: StatusSnapshot): string[] {
1556
- const schedule = s.digestSchedule;
1557
- if (schedule === undefined) return [];
1558
- if (schedule.mode === "disabled") return ["next digest disabled"];
1559
- if (schedule.mode === "per-tick") return ["next digest every tick"];
1560
- if (schedule.mode === "due") return ["next digest due now"];
1561
- return [
1562
- schedule.nextAt === undefined
1563
- ? "next digest could not be calculated"
1564
- : `next digest ${formatZonedMinute(schedule.nextAt, schedule.timezone)}`,
1565
- ];
1566
- }
1567
-
1568
- function formatProjectBody(
1569
- s: StatusSnapshot,
1570
- workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
1571
- now = Date.now(),
1572
- ): string {
1573
- const lines = [
1574
- `project ${s.project}${s.paused ? " (PAUSED)" : ""}`,
1575
- ...(s.pauseReason === undefined ? [] : [`paused ${s.pauseReason}`]),
1576
- `config ${s.configPath}`,
1577
- `state ${s.stateDir}`,
1578
- // The orchestrator-down degrade row: first-class in the body, present only
1579
- // while the incident is open, so recovery drops it (#288).
1580
- ...(s.orchestratorDown === undefined
1581
- ? []
1582
- : formatOrchestratorDown(s.orchestratorDown, now)),
1583
- ...formatAvailabilityStatus(s),
1584
- ...formatDigestScheduleStatus(s),
1585
- "",
1586
- "caps",
1587
- ` workers ${s.liveWorkers} / ${s.caps.maxConcurrentWorkers}`,
1588
- ` issues today ${s.runsToday}`,
1589
- s.caps.dailySpendUsd === null
1590
- ? ` spend today $${s.spendTodayUsd.toFixed(2)} (no daily cap)`
1591
- : ` spend today $${s.spendTodayUsd.toFixed(2)} / $${s.caps.dailySpendUsd.toFixed(2)}`,
1592
- // Its own row beside the spend row, never folded into it: they are two
1593
- // independent controls and an operator has to see which one stopped the
1594
- // fleet (#110).
1595
- ` plan usage ${planUsageLine(s.planUsage)}`,
1596
- // The tracker's API budget, when the renderer could read it. Absent on a
1597
- // broken `gh`: one missing row, never a broken report (#188). Next to it,
1598
- // the refusals the tracker actually observed, so a budget that looks
1599
- // healthy next to every call being refused is still visible (#198).
1600
- ...(s.github === undefined
1601
- ? []
1602
- : [
1603
- ` github graphql ${s.github.graphql.remaining}/${s.github.graphql.limit}, core ${s.github.core.remaining}/${s.github.core.limit} (graphql resets ${Math.max(0, Math.round((s.github.graphql.reset * 1000 - Date.now()) / 60_000))}m)` +
1604
- (s.ghRefusals === undefined || s.ghRefusals.count === 0
1605
- ? ""
1606
- : ` — ${s.ghRefusals.count} refusal(s) in last 5m (last ${new Date(s.ghRefusals.latestAt ?? Date.now()).toISOString().slice(11, 19)}Z)`),
1607
- ]),
1608
- // The daemon's own observed github traffic today (#198): a separate row
1609
- // from the polled budget above, so one missing `status` read never hides
1610
- // the other. `onCall` counts every spawn including ones that end 304, so
1611
- // `daemon` is spawns and `daemon-304` is the unbilled subset — billed ≈
1612
- // difference, and the polled budget row stays the authority (#203).
1613
- ` github calls daemon ${s.ghCallsToday?.find((c) => c.source === "daemon")?.calls ?? 0} today` +
1614
- ((s.ghCallsToday?.find((c) => c.source === "daemon-304")?.calls ?? 0) === 0
1615
- ? ""
1616
- : ` (${s.ghCallsToday!.find((c) => c.source === "daemon-304")!.calls} free 304s)`),
1617
- // Label-projection ops the tracker has not applied yet (#201): GitHub is
1618
- // behind what the store decided, and the operator can see the lag instead
1619
- // of discovering it as a stale label or a missing one.
1620
- ...(s.labelOps === undefined
1621
- ? []
1622
- : [
1623
- ` labels projection ${s.labelOps.pending} pending (oldest ${
1624
- s.labelOps.oldestAgeMs >= 60_000
1625
- ? `${Math.round(s.labelOps.oldestAgeMs / 60_000)}m`
1626
- : `${Math.max(1, Math.round(s.labelOps.oldestAgeMs / 1000))}s`
1627
- })`,
1628
- ]),
1629
- ` new worker turns ${s.caps.workerMaxTurns}`,
1630
- ...s.turnOverrides.map(
1631
- ({ issue, maxTurns }) => ` turn override #${issue} → ${maxTurns} (next attempt)`,
1632
- ),
1633
- ` worker wall clock ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
1634
- ` failed attempts ${s.caps.maxAttemptsPerIssue}`,
1635
- ` continuations ${s.caps.maxContinuationsPerIssue}`,
1636
- "",
1637
- ...formatReleaseGrants(s.releaseGrants),
1638
- "",
1639
- formatDispatchSummary(s.dispatch),
1640
- "",
1641
- ];
1642
- if (s.activeRuns.length === 0) {
1643
- lines.push("active runs (none)");
1644
- } else {
1645
- lines.push("active runs");
1646
- for (const r of s.activeRuns) {
1647
- const phase = workerPhases.get(r.issue);
1648
- const state = phase === "pausing" || phase === "paused" ? phase : r.state;
1649
- lines.push(
1650
- ` #${r.issue} ${r.repo} ${state} attempt ${r.attempt} ` +
1651
- `${r.turns}/${r.maxTurns} turns ${r.spendUsd.toFixed(2)} ${r.branch}` +
1652
- (r.prUrl ? ` ${r.prUrl}` : ""),
1653
- );
1654
- // The orchestrator's Duty 1 reads this command, and a flagged run's
1655
- // escalation is deduplicated after one delivery — so this is where a
1656
- // flagged PR stays visible for as long as it is still open (#128).
1657
- const flagged = settlementFlagSummary(r.settlementFlags);
1658
- if (flagged !== undefined) lines.push(` ${flagged}`);
1659
- }
1660
- }
1661
- lines.push(...formatBaseHealth(s.baseHealth));
1662
- lines.push(...formatSalvagedRuns(s.salvagedRuns));
1663
- lines.push(...formatOpenReports(s.openReports));
1664
- lines.push(...formatDigestBacklog(s.digestBacklog));
1665
- if (s.liveWorkers > 0) {
1666
- lines.push(
1667
- "",
1668
- `deploy ${s.liveWorkers} live worker(s) — restart salvages dirty trees then orphans the rows; ` +
1669
- `hold and wait for workers 0/${s.caps.maxConcurrentWorkers} when you can drain instead`,
1670
- );
1671
- }
1672
- return lines.join("\n");
1673
- }
1674
-
1675
1314
  export async function renderStatus(projectName?: string): Promise<string> {
1676
1315
  const s = statusSnapshot(projectName);
1677
1316
  const layers = fleetLayers(projectName);
@@ -1716,8 +1355,16 @@ export async function renderStatus(projectName?: string): Promise<string> {
1716
1355
  // project and the uninvolved one must see who stopped it too.
1717
1356
  const store = openStore(dbPath());
1718
1357
  let lastStop: DaemonStop | undefined;
1358
+ let siblings: { project: string; live: number }[] = [];
1719
1359
  try {
1720
1360
  lastStop = store.latestDaemonStop();
1361
+ // Shared-daemon visibility (#545): every configured project other than the
1362
+ // one being viewed, with its live-run count. Read from the same open store
1363
+ // as the provenance above, so a status reader can tell "my fleet is idle"
1364
+ // from "the process I am about to stop is busy".
1365
+ siblings = loadConfig()
1366
+ .projects.filter((p) => p.name !== project.name)
1367
+ .map((p) => ({ project: p.name, live: store.liveRuns(p.name).length }));
1721
1368
  } finally {
1722
1369
  store.close();
1723
1370
  }
@@ -1734,6 +1381,7 @@ export async function renderStatus(projectName?: string): Promise<string> {
1734
1381
  workerPhases,
1735
1382
  intakeStatusLine(project.name),
1736
1383
  lastStop,
1384
+ siblings,
1737
1385
  );
1738
1386
  }
1739
1387
 
@@ -1793,18 +1441,32 @@ function failureClassBlock(projectName: string): string | undefined {
1793
1441
  * the ambiguity this ledger exists to remove. Omitted on any throw — a status
1794
1442
  * that fails to render because the store is mid-migration is worse than one
1795
1443
  * missing a row.
1444
+ *
1445
+ * `decisions N open` counts only rows a human must answer (#459). Watches the
1446
+ * orchestrator set for itself are reported separately — a fleet with three
1447
+ * watches behind GitHub's checks is not a fleet waiting on its operator, and
1448
+ * lumping them in made `decisions 3 open` read as three unanswered questions.
1796
1449
  */
1797
- function decisionStatusLine(projectName: string): string | undefined {
1450
+ export function decisionStatusLine(projectName: string): string | undefined {
1798
1451
  const path = dbPath();
1799
1452
  if (!existsSync(path)) return undefined;
1800
1453
  let store: Store | undefined;
1801
1454
  try {
1802
1455
  store = openStore(path);
1803
1456
  const open = store.openDecisions(projectName);
1804
- if (open.length === 0) return "decisions none open";
1805
- const oldest = open[0]!;
1806
- const hours = Math.max(0, Math.round((Date.now() - oldest.askedAt) / 3_600_000));
1807
- return `decisions ${open.length} open (oldest ${hours}h)`;
1457
+ const questions = open.filter((d) => d.kind !== "watch");
1458
+ const watches = open.filter((d) => d.kind === "watch");
1459
+ if (questions.length === 0 && watches.length === 0) return "decisions none open";
1460
+ let line: string;
1461
+ if (questions.length === 0) {
1462
+ line = "decisions none open";
1463
+ } else {
1464
+ const oldest = questions[0]!;
1465
+ const hours = Math.max(0, Math.round((Date.now() - oldest.askedAt) / 3_600_000));
1466
+ line = `decisions ${questions.length} open (oldest ${hours}h)`;
1467
+ }
1468
+ if (watches.length > 0) line += ` · watches ${watches.length}`;
1469
+ return line;
1808
1470
  } catch {
1809
1471
  return undefined;
1810
1472
  } finally {
@@ -1973,6 +1635,56 @@ export function sessionDirForCwd(cwd: string): string {
1973
1635
  return join(home, ".omp", "agent", "sessions", slug.replaceAll("/", "-"));
1974
1636
  }
1975
1637
 
1638
+ /** The session tree arm scans: every transcript under it, whatever the cwd slug. */
1639
+ export function sessionsRoot(): string {
1640
+ return join(homedir(), ".omp", "agent", "sessions");
1641
+ }
1642
+
1643
+ /**
1644
+ * The orchestrator session file omp-telegram's live claim names for this
1645
+ * project — the transcript a resumed/restored pane actually writes, which can
1646
+ * live in a different session directory than the tick cwd implies (#600).
1647
+ * Never throws: no claim, or an unreadable config, means undefined and arm
1648
+ * falls back to the cwd-derived directory.
1649
+ */
1650
+ function claimedOrchestratorSessionFile(named: string | undefined): string | undefined {
1651
+ if (named === undefined) return undefined;
1652
+ try {
1653
+ return resolveClaimedSessionFile(findProject(loadConfig(), named));
1654
+ } catch {
1655
+ return undefined;
1656
+ }
1657
+ }
1658
+
1659
+ /**
1660
+ * The session directories `arm` watches for the reply — the cwd-derived one
1661
+ * plus, when the live claim's file lives elsewhere, that file's directory.
1662
+ *
1663
+ * The cwd-derived directory is where a *fresh* session started from the tick
1664
+ * cwd writes; the claimed file's directory is where the *restored* pane keeps
1665
+ * writing (herdr pins it to the original transcript). Watching both covers a
1666
+ * resumed session, a rotated one, and a session that starts after the send.
1667
+ *
1668
+ * A claim outside the session tree is a stop, not a slower failure: arm polls
1669
+ * only transcripts under {@link sessionsRoot}, so a claimed file elsewhere can
1670
+ * never satisfy the challenge, and five minutes of polling cannot discover a
1671
+ * file that is not in the search set. The throw names both paths.
1672
+ */
1673
+ function armSessionScanDirs(cwd: string, claimed: string | undefined): string[] {
1674
+ const cwdDir = sessionDirForCwd(cwd);
1675
+ if (claimed === undefined) return [cwdDir];
1676
+ const root = sessionsRoot();
1677
+ const claimDir = dirname(claimed);
1678
+ if (claimDir !== root && !claimDir.startsWith(join(root, sep))) {
1679
+ throw new Error(
1680
+ `arm: the orchestrator's claimed session file ${claimed} is outside the session tree arm scans (${root}) — ` +
1681
+ `no transcript there can satisfy the challenge. Resume the pane under ${root}, or start it from ${cwd}; ` +
1682
+ `arm would otherwise poll ${cwdDir} until the timer runs out`,
1683
+ );
1684
+ }
1685
+ return claimDir === cwdDir ? [cwdDir] : [cwdDir, claimDir];
1686
+ }
1687
+
1976
1688
  function makeChallengeCode(): string {
1977
1689
  const bytes = new Uint8Array(4);
1978
1690
  crypto.getRandomValues(bytes);
@@ -1999,20 +1711,23 @@ interface SessionScan {
1999
1711
  }
2000
1712
 
2001
1713
  /**
2002
- * Polls the session directory — re-read on every pass, never snapshotted —
1714
+ * Polls the session directories — re-read on every pass, never snapshotted —
2003
1715
  * until the challenge shows up as a user turn or the deadline passes.
2004
1716
  *
2005
1717
  * Discovery lives here because the reply can land in a transcript that does not
2006
1718
  * exist yet when the challenge is sent: a rotated session, or the first one of
2007
- * a pane started right after arming (#142). A waiter handed one path polls a
2008
- * file the answer will never be written to, and arming becomes impossible.
1719
+ * a pane started right after arming (#142). And the scan set can be two
1720
+ * directories when the live claim's session file lives outside the tick-cwd
1721
+ * directory: the restored pane keeps writing the claimed file, so its
1722
+ * directory is watched alongside the cwd-derived one (#600).
2009
1723
  *
2010
1724
  * Files untouched since just before the send are named, not parsed: an append
2011
1725
  * bumps mtime, so a transcript older than the challenge cannot hold the reply,
2012
- * and skipping it keeps a large stale session out of every 5 s pass.
1726
+ * and skipping it keeps a large stale session out of every 5 s pass. A missing
1727
+ * directory is not an error — it can vanish under a rotation and reappear.
2013
1728
  */
2014
1729
  async function waitForChallengeInSessions(
2015
- dir: string,
1730
+ dirs: readonly string[],
2016
1731
  code: string,
2017
1732
  sentAt: number,
2018
1733
  timeoutMs: number,
@@ -2024,28 +1739,30 @@ async function waitForChallengeInSessions(
2024
1739
  for (;;) {
2025
1740
  const scanned: string[] = [];
2026
1741
  const ignored: string[] = [];
2027
- let names: string[] = [];
2028
- try {
2029
- names = readdirSync(dir);
2030
- } catch {
2031
- /* the directory can go away under a rotation; the next pass re-reads it */
2032
- }
2033
- for (const name of names.sort()) {
2034
- if (!name.endsWith(".jsonl")) continue;
2035
- const path = join(dir, name);
2036
- let mtimeMs: number;
1742
+ for (const dir of dirs) {
1743
+ let names: string[] = [];
2037
1744
  try {
2038
- mtimeMs = statSync(path).mtimeMs;
1745
+ names = readdirSync(dir);
2039
1746
  } catch {
2040
- continue; /* race */
1747
+ /* the directory can go away under a rotation; the next pass re-reads it */
2041
1748
  }
2042
- const label = `${name} (mtime ${new Date(mtimeMs).toISOString()})`;
2043
- if (mtimeMs < sentAt - 1_000) {
2044
- ignored.push(label);
2045
- continue;
1749
+ for (const name of names.sort()) {
1750
+ if (!name.endsWith(".jsonl")) continue;
1751
+ const path = join(dir, name);
1752
+ let mtimeMs: number;
1753
+ try {
1754
+ mtimeMs = statSync(path).mtimeMs;
1755
+ } catch {
1756
+ continue; /* race */
1757
+ }
1758
+ const label = `${name} (mtime ${new Date(mtimeMs).toISOString()})`;
1759
+ if (mtimeMs < sentAt - 1_000) {
1760
+ ignored.push(label);
1761
+ continue;
1762
+ }
1763
+ scanned.push(label);
1764
+ if (await transcriptHasUserCode(path, code)) return { seen: true, scanned, ignored };
2046
1765
  }
2047
- scanned.push(label);
2048
- if (await transcriptHasUserCode(path, code)) return { seen: true, scanned, ignored };
2049
1766
  }
2050
1767
  if (now() >= deadline) return { seen: false, scanned, ignored };
2051
1768
  await sleep(5_000);
@@ -2167,4 +1884,3 @@ function probeOmpPane(
2167
1884
  return { kind: "unknown", reason: err instanceof Error ? err.message : String(err) };
2168
1885
  }
2169
1886
  }
2170
-