omp-conductor 0.18.2 → 0.19.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/README.md +105 -40
  2. package/REFERENCE.md +865 -30
  3. package/package.json +1 -1
  4. package/schema/config.schema.json +26 -0
  5. package/src/admission.ts +212 -26
  6. package/src/ask.ts +288 -1
  7. package/src/briefs/orchestrator.md +6 -5
  8. package/src/cli.ts +5 -1
  9. package/src/command-help.ts +9 -1
  10. package/src/command-manifest.ts +36 -3
  11. package/src/commands/arm.ts +5 -1
  12. package/src/commands/context.ts +2 -0
  13. package/src/commands/message.ts +26 -2
  14. package/src/commands/reconcile-units.ts +104 -0
  15. package/src/commands/release-composition.ts +232 -0
  16. package/src/commands/resume.ts +2 -27
  17. package/src/commands/setup.ts +101 -16
  18. package/src/commands/stats.ts +11 -30
  19. package/src/commands/tail.ts +31 -1
  20. package/src/commands/upgrade.ts +20 -3
  21. package/src/commands/verb.ts +2 -1
  22. package/src/config-schema.ts +19 -0
  23. package/src/config.ts +80 -0
  24. package/src/credential-class.ts +366 -0
  25. package/src/daemon.ts +1218 -288
  26. package/src/dashboard/app.js +504 -2
  27. package/src/dashboard/controls.ts +336 -0
  28. package/src/dashboard/index.html +30 -0
  29. package/src/dashboard/server.ts +271 -30
  30. package/src/dashboard/style.css +116 -0
  31. package/src/dashboard/transcript.ts +173 -0
  32. package/src/doctor.ts +379 -22
  33. package/src/failure-class.ts +59 -0
  34. package/src/fleet.ts +511 -101
  35. package/src/host.ts +6 -130
  36. package/src/omp.ts +29 -0
  37. package/src/orchestrator-tick.ts +343 -88
  38. package/src/pause.ts +233 -0
  39. package/src/settlement.ts +159 -2
  40. package/src/setup-answers.ts +97 -0
  41. package/src/setup-host.ts +325 -1159
  42. package/src/setup-install.ts +204 -27
  43. package/src/setup-wizard.ts +111 -50
  44. package/src/setup.ts +33 -0
  45. package/src/spend-telemetry.ts +117 -0
  46. package/src/stats.ts +35 -0
  47. package/src/status-render.ts +348 -19
  48. package/src/store.ts +1229 -55
  49. package/src/telegram-freshness.ts +269 -0
  50. package/src/to-spec.ts +27 -0
  51. package/src/types.ts +697 -4
  52. package/src/unblock.ts +22 -0
  53. package/src/unit-reconcile.ts +303 -0
  54. package/src/upgrade-verify.ts +8 -1
  55. package/src/upgrade.ts +326 -47
  56. package/src/verbs/actions.ts +124 -10
  57. package/src/verbs/protocol.ts +70 -2
  58. package/src/verbs/server.ts +447 -8
  59. package/src/wake.ts +48 -0
  60. package/src/worker.ts +403 -3
package/src/fleet.ts CHANGED
@@ -105,10 +105,11 @@ export const PANE_HALT_FILE = ".conductor-pane-halted";
105
105
  export const DEFAULT_HERDR_UNIT = "herdr-fleet.service";
106
106
 
107
107
  /**
108
- * Default Herdr session name for conductor fleets. Renamed from `fleet` in
109
- * #320; `HERDR_SESSION` still wins when set. Legacy hosts that still run a
110
- * session named `fleet` are bridged for one release by
111
- * {@link resolveHerdrSession}.
108
+ * The Herdr session conductor fleets run in. `HERDR_SESSION` wins when set;
109
+ * this is the only other name any surface may resolve. #320 renamed it from
110
+ * `fleet` and shipped a one-release probe that preferred a populated legacy
111
+ * session; #976 removed that probe, because a name resolved two ways is a
112
+ * name two surfaces can disagree about — and they did, in `upgrade`.
112
113
  */
113
114
  export const DEFAULT_HERDR_SESSION = "conductor";
114
115
 
@@ -121,32 +122,13 @@ export const DEFAULT_HERDR_SESSION = "conductor";
121
122
  export const DEFAULT_FLEET_AGENT_NAME = "fleet";
122
123
  export const ARM_CHALLENGE_TIMEOUT_MS = 300_000;
123
124
 
124
-
125
- /** Pre-#320 session name. Bridged for one release when `conductor` is empty. */
126
- const LEGACY_HERDR_SESSION = "fleet";
127
-
128
- export const LEGACY_HERDR_SESSION_HINT =
129
- 'herdr session "fleet" found — rename it to "conductor" or run omp-conductor setup host to pin HERDR_SESSION=fleet (remove this bridge next minor)';
130
-
131
- let legacyHerdrSessionHintPrinted = false;
132
-
133
- function noteLegacyHerdrSession(log?: (message: string) => void): void {
134
- if (legacyHerdrSessionHintPrinted) return;
135
- legacyHerdrSessionHintPrinted = true;
136
- if (log) log(LEGACY_HERDR_SESSION_HINT);
137
- else console.warn(LEGACY_HERDR_SESSION_HINT);
138
- }
139
-
140
- /** Reset the once-per-process rename hint (tests). */
141
- export function resetLegacyHerdrSessionHintForTests(): void {
142
- legacyHerdrSessionHintPrinted = false;
143
- }
144
-
145
125
  /**
146
- * Resolve the Herdr session name. `HERDR_SESSION` is always authoritative.
147
- * When unset, default to {@link DEFAULT_HERDR_SESSION}. The one-release bridge
148
- * that prefers a populated legacy `fleet` session lives in
149
- * {@link resolveHerdrSessionWithBridge} call that at probe sites.
126
+ * Resolve the Herdr session name the single resolution every surface uses.
127
+ * `HERDR_SESSION` is authoritative; otherwise {@link DEFAULT_HERDR_SESSION}.
128
+ * Never probe for an alternative name here: a session that answers is not
129
+ * evidence of which session this fleet is, and the probe that assumed it was
130
+ * is what made `upgrade` install against `fleet` and verify against
131
+ * `conductor` (#976).
150
132
  */
151
133
  export function resolveHerdrSession(env: NodeJS.ProcessEnv = process.env): string {
152
134
  const explicit = env["HERDR_SESSION"];
@@ -154,56 +136,6 @@ export function resolveHerdrSession(env: NodeJS.ProcessEnv = process.env): strin
154
136
  return DEFAULT_HERDR_SESSION;
155
137
  }
156
138
 
157
- function herdrAgentListRaw(
158
- session: string,
159
- bin: string,
160
- env: NodeJS.ProcessEnv,
161
- ): { ok: true; agents: HerdrAgent[] } | { ok: false } {
162
- const res = spawnSync(bin, ["--session", session, "agent", "list"], {
163
- encoding: "utf8",
164
- timeout: 8_000,
165
- env,
166
- });
167
- if (res.error || res.status !== 0) return { ok: false };
168
- try {
169
- return { ok: true, agents: parseHerdrAgentList(res.stdout ?? "") };
170
- } catch {
171
- return { ok: false };
172
- }
173
- }
174
-
175
- /**
176
- * One-release bridge (#320): when `HERDR_SESSION` is unset, a `conductor`
177
- * session with no agents while a `fleet` session answers → use `fleet` and
178
- * print the rename hint once. Remove next minor.
179
- */
180
- export function resolveHerdrSessionWithBridge(opts: {
181
- env?: NodeJS.ProcessEnv;
182
- herdrBin?: string;
183
- log?: (message: string) => void;
184
- } = {}): string {
185
- const env = opts.env ?? process.env;
186
- const explicit = env["HERDR_SESSION"];
187
- if (explicit !== undefined && explicit.length > 0) return explicit;
188
-
189
- const bin = opts.herdrBin ?? "herdr";
190
- const primary = herdrAgentListRaw(DEFAULT_HERDR_SESSION, bin, env);
191
- if (primary.ok && primary.agents.length > 0) return DEFAULT_HERDR_SESSION;
192
-
193
- const legacy = herdrAgentListRaw(LEGACY_HERDR_SESSION, bin, env);
194
- if (legacy.ok && legacy.agents.length > 0) {
195
- noteLegacyHerdrSession(opts.log);
196
- return LEGACY_HERDR_SESSION;
197
- }
198
- // conductor answered (even empty) wins over a dead legacy session.
199
- if (primary.ok) return DEFAULT_HERDR_SESSION;
200
- if (legacy.ok) {
201
- noteLegacyHerdrSession(opts.log);
202
- return LEGACY_HERDR_SESSION;
203
- }
204
- return DEFAULT_HERDR_SESSION;
205
- }
206
-
207
139
  export function telegramStateDir(): string {
208
140
  const override = process.env["OMP_TELEGRAM_STATE_DIR"];
209
141
  if (override !== undefined && override.length > 0) return override;
@@ -343,6 +275,14 @@ export interface ArmDeps {
343
275
  pidAlive?: (pid: number) => boolean;
344
276
  lockPidAlive?: (pid: number) => boolean;
345
277
  lockFresh?: (mtimeMs: number) => boolean;
278
+ /**
279
+ * Where the pending-proof heartbeat is written (#861). The challenge proof
280
+ * waits up to five minutes on a human, inside a fence that holds dispatch:
281
+ * without this, that wait is silent and a healthy process is indistinguishable
282
+ * from a dead one. Absent means no reporting — the callers that have a surface
283
+ * (the CLI, the wizard) pass theirs.
284
+ */
285
+ progress?: (line: string) => void;
346
286
  }
347
287
 
348
288
  export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promise<ArmResult> {
@@ -484,6 +424,14 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
484
424
  `arm: outbound sendMessage failed — NOT armed: ${err instanceof Error ? err.message : String(err)}`,
485
425
  );
486
426
  }
427
+ // What is being waited on, before the wait starts: the window, the chat the
428
+ // reply has to land in, and the fact that dispatch is held until it does
429
+ // (#861). One line, so a five-minute wait opens with an explanation rather
430
+ // than with silence.
431
+ deps.progress?.(
432
+ `arm: challenge sent to ${channel.owner}${sendTopic === undefined ? "" : ` (topic ${sendTopic})`} — ` +
433
+ `waiting up to ${armClock(timeoutMs)} for the reply. Dispatch stays held until it arrives.`,
434
+ );
487
435
 
488
436
  // Wait for the orchestrator's own acknowledgement — conductor state written
489
437
  // by the inbound user-turn adapter when the real reply lands (#614). No
@@ -553,10 +501,14 @@ export interface HaltWithPaneResult extends HaltResult {
553
501
  export function hold(
554
502
  projectName?: string,
555
503
  source: string = "hold",
556
- opts: { keepTicks?: boolean } = {},
504
+ opts: { keepTicks?: boolean; reason?: string } = {},
557
505
  ): HoldResult {
558
506
  const wasPaused = isPaused(projectName);
559
- setPaused(true, { source }, projectName);
507
+ // `source` is a space-free token by contract (`pauseSourceToken`) — the fence
508
+ // matches on it, and whitespace there makes a pause the fence cannot prove
509
+ // (#552). The human sentence belongs in `reason=`, which is why this is a
510
+ // separate field rather than something a caller concatenates into `source`.
511
+ setPaused(true, { source, ...(opts.reason === undefined ? {} : { reason: opts.reason }) }, projectName);
560
512
  if (opts.keepTicks === true) return { wasPaused };
561
513
  return { wasPaused, disarmed: disarmTicks(projectName) };
562
514
  }
@@ -717,6 +669,407 @@ export interface HerdrAgent {
717
669
  sessionPath?: string;
718
670
  }
719
671
 
672
+ /**
673
+ * The Herdr representation of one live conductor worker (#840).
674
+ *
675
+ * ## Why this shape, and not `herdr agent start --kind omp`
676
+ *
677
+ * The authoritative worker is the `session-host` child conductor spawns itself:
678
+ * it holds the typed control socket, the mediated verb socket, the worktree, the
679
+ * transcript, the accounting and the output-schema settlement. Starting a second
680
+ * OMP process in a pane would look right in the workspace and be authoritative
681
+ * for nothing — the named silent fake of #840 — so nothing here starts an agent.
682
+ *
683
+ * Herdr already has the verb for exactly this case: an *external* supervisor
684
+ * reports lifecycle onto a pane it owns (`pane report-agent`,
685
+ * `pane report-agent-session`, `pane release-agent`). So conductor keeps the
686
+ * child it already spawned, and reports that child's identity and state onto a
687
+ * pane whose only job is display.
688
+ *
689
+ * ## What is displayed, and why that is not "a transcript-tail pane"
690
+ *
691
+ * The pane runs `omp-conductor tail <issue>`, which is the repository's existing
692
+ * read-only follower. The distinction #840 draws is about *authority*, not about
693
+ * pixels: nothing here parses that output, and no run state, settlement, turn,
694
+ * spend or model fact is derived from it. Identity comes from the exact child
695
+ * pid handed over by the spawn path, and state comes from the typed events the
696
+ * proxy already emits.
697
+ *
698
+ * ## Observation-only is structural, not a gate
699
+ *
700
+ * The authoritative child is spawned with `stdin: "ignore"` — there is no file
701
+ * descriptor for a keystroke to travel down, from a pane or anywhere else. The
702
+ * pane's own process is a follower whose stdin reaches only itself. So "input
703
+ * cannot reach OMP, change its prompt, invoke a verb, or settle the run" is a
704
+ * property of the process tree rather than a check that could be forgotten.
705
+ */
706
+ export interface WorkerPaneIdentity {
707
+ project: string;
708
+ issue: number;
709
+ attempt: number;
710
+ /** The run row's id — the durable identity the pane is reported under. */
711
+ runId: string;
712
+ /** The exact `session-host` pid, from the spawn path's own `onSpawn`. */
713
+ pid: number;
714
+ /** The transcript the child opened, when it has already reported one. */
715
+ sessionFile?: string;
716
+ }
717
+
718
+ /** A tracked pane, or the explicit reason there is none. Never a silent claim. */
719
+ export type WorkerPaneOutcome =
720
+ | { kind: "tracked"; paneId: string; label: string; pid: number }
721
+ | { kind: "unavailable"; reason: string };
722
+
723
+ /** One `herdr` invocation, injected so every path is testable with no terminal. */
724
+ export type HerdrRun = (args: readonly string[]) => { ok: boolean; stdout: string; stderr: string };
725
+
726
+ export interface WorkerPaneDeps {
727
+ run?: HerdrRun;
728
+ session?: string;
729
+ /** The follower command the pane displays; the CLI by default. */
730
+ viewer?: (identity: WorkerPaneIdentity) => readonly string[];
731
+ }
732
+
733
+ /**
734
+ * The pane's unique, human-readable name (#840: "uniquely named").
735
+ *
736
+ * Keyed by run id rather than issue: an issue can have several attempts, and two
737
+ * attempts must never resolve to one pane. The issue and attempt ride along
738
+ * because the operator reads this label in a workspace, not a database.
739
+ */
740
+ /**
741
+ * The external-supervisor source every conductor pane report carries.
742
+ *
743
+ * This, not the label, is what proves a pane is conductor's: it is written by
744
+ * `openWorkerPane` and read back by {@link listWorkerPanes}, so the two halves
745
+ * of reconciliation cannot come to disagree about what "ours" means.
746
+ */
747
+ export const WORKER_PANE_SOURCE = "omp-conductor";
748
+
749
+ export function workerPaneLabel(identity: WorkerPaneIdentity): string {
750
+ return `worker-${identity.project}-${identity.issue}-a${identity.attempt}-${identity.runId.slice(0, 8)}`;
751
+ }
752
+
753
+ const realHerdrRun: HerdrRun = (args) => {
754
+ const res = spawnSync("herdr", [...args], { encoding: "utf8", timeout: 8_000, env: process.env });
755
+ return {
756
+ ok: !res.error && res.status === 0,
757
+ stdout: res.stdout ?? "",
758
+ stderr: res.stderr ?? (res.error === undefined ? "" : String(res.error.message)),
759
+ };
760
+ };
761
+
762
+ /** `omp-conductor tail` — read-only by construction, and never parsed here. */
763
+ function defaultViewer(identity: WorkerPaneIdentity): readonly string[] {
764
+ return ["omp-conductor", "tail", String(identity.issue), "--project", identity.project];
765
+ }
766
+
767
+ /**
768
+ * Create the pane, run the follower in it, and report the child's identity and
769
+ * initial state — or say exactly why it could not.
770
+ *
771
+ * Every step is checked, and the first failure returns `unavailable` with the
772
+ * reason: a partially established representation (a pane with no agent identity,
773
+ * say) is worse than none, because it is a pane the operator would read as a
774
+ * tracked worker. What this deliberately does NOT do is decide what a failure
775
+ * means for the launch — failing closed versus running degraded is #841's
776
+ * policy, and inventing it here would pre-empt it.
777
+ */
778
+ export function openWorkerPane(identity: WorkerPaneIdentity, deps: WorkerPaneDeps = {}): WorkerPaneOutcome {
779
+ const run = deps.run ?? realHerdrRun;
780
+ const session = deps.session ?? resolveHerdrSession();
781
+ const label = workerPaneLabel(identity);
782
+ const base = ["--session", session, "pane"];
783
+
784
+ const split = run([...base, "split", "--direction", "down", "--ratio", "0.3"]);
785
+ if (!split.ok) {
786
+ return { kind: "unavailable", reason: `herdr pane split failed: ${firstLine(split.stderr) || "no output"}` };
787
+ }
788
+ const paneId = firstLine(split.stdout);
789
+ if (paneId === "") {
790
+ return { kind: "unavailable", reason: "herdr pane split reported no pane id" };
791
+ }
792
+
793
+ const named = run([...base, "rename", paneId, label]);
794
+ if (!named.ok) {
795
+ return { kind: "unavailable", reason: `herdr pane rename failed: ${firstLine(named.stderr) || "no output"}` };
796
+ }
797
+
798
+ // Identity before display: the pane must be attributable to this exact run
799
+ // before it shows anything, so a pane that appears is never a pane nobody can
800
+ // trace back to a worker.
801
+ const identified = run([
802
+ ...base,
803
+ "report-agent-session",
804
+ paneId,
805
+ "--source",
806
+ WORKER_PANE_SOURCE,
807
+ "--agent",
808
+ label,
809
+ "--agent-session-id",
810
+ identity.runId,
811
+ ...(identity.sessionFile === undefined ? [] : ["--agent-session-path", identity.sessionFile]),
812
+ "--session-start-source",
813
+ WORKER_PANE_SOURCE,
814
+ ]);
815
+ if (!identified.ok) {
816
+ return {
817
+ kind: "unavailable",
818
+ reason: `herdr pane report-agent-session failed: ${firstLine(identified.stderr) || "no output"}`,
819
+ };
820
+ }
821
+
822
+ const started = run([...base, "run", paneId, ...deps.viewer?.(identity) ?? defaultViewer(identity)]);
823
+ if (!started.ok) {
824
+ return { kind: "unavailable", reason: `herdr pane run failed: ${firstLine(started.stderr) || "no output"}` };
825
+ }
826
+
827
+ // A worker that has just spawned is working by definition. The ongoing
828
+ // projection of turn/pause/blocked transitions is #842's, from the same typed
829
+ // events — never from the pane's output.
830
+ const reported = reportWorkerPaneState(paneId, label, "working", { run, session });
831
+ if (!reported.ok) {
832
+ return { kind: "unavailable", reason: reported.reason };
833
+ }
834
+ return { kind: "tracked", paneId, label, pid: identity.pid };
835
+ }
836
+
837
+ /** Report one lifecycle state for a tracked pane. Monotonic `seq` is the caller's. */
838
+ export function reportWorkerPaneState(
839
+ paneId: string,
840
+ label: string,
841
+ state: "working" | "idle" | "blocked" | "unknown",
842
+ deps: { run?: HerdrRun; session?: string; seq?: number; message?: string } = {},
843
+ ): { ok: true } | { ok: false; reason: string } {
844
+ const run = deps.run ?? realHerdrRun;
845
+ const session = deps.session ?? resolveHerdrSession();
846
+ const res = run([
847
+ "--session",
848
+ session,
849
+ "pane",
850
+ "report-agent",
851
+ paneId,
852
+ "--source",
853
+ WORKER_PANE_SOURCE,
854
+ "--agent",
855
+ label,
856
+ "--state",
857
+ state,
858
+ ...(deps.seq === undefined ? [] : ["--seq", String(deps.seq)]),
859
+ ...(deps.message === undefined ? [] : ["--message", deps.message]),
860
+ ]);
861
+ return res.ok
862
+ ? { ok: true }
863
+ : { ok: false, reason: `herdr pane report-agent failed: ${firstLine(res.stderr) || "no output"}` };
864
+ }
865
+
866
+ /**
867
+ * Hand lifecycle authority back when the worker is gone.
868
+ *
869
+ * Release, never close: whether a settled worker's pane is closed, kept, or
870
+ * retained for N settlements is #841's documented policy, and a slice that
871
+ * closed panes here would decide it by accident. Releasing says only "conductor
872
+ * no longer speaks for this agent", which is exactly what is true.
873
+ */
874
+ export function releaseWorkerPane(
875
+ paneId: string,
876
+ label: string,
877
+ deps: { run?: HerdrRun; session?: string; seq?: number } = {},
878
+ ): { ok: true } | { ok: false; reason: string } {
879
+ const run = deps.run ?? realHerdrRun;
880
+ const session = deps.session ?? resolveHerdrSession();
881
+ const res = run([
882
+ "--session",
883
+ session,
884
+ "pane",
885
+ "release-agent",
886
+ paneId,
887
+ "--source",
888
+ WORKER_PANE_SOURCE,
889
+ "--agent",
890
+ label,
891
+ ...(deps.seq === undefined ? [] : ["--seq", String(deps.seq)]),
892
+ ]);
893
+ return res.ok
894
+ ? { ok: true }
895
+ : { ok: false, reason: `herdr pane release-agent failed: ${firstLine(res.stderr) || "no output"}` };
896
+ }
897
+
898
+ /**
899
+ * Hand back the pane a dead worker left behind (#842).
900
+ *
901
+ * Called for every run a restart reaps. The recorded pid is deliberately NOT
902
+ * consulted for liveness: a `session-host` child dies with the daemon that owned
903
+ * its verb socket, so an orphaned row's worker is gone whatever pid it carries —
904
+ * and pids are reused, so checking one is how a stranger's process comes to read
905
+ * as a live worker. A run with no recorded pane is a no-op, not a failure: it
906
+ * never had one to release.
907
+ */
908
+ export function releaseOrphanedWorkerPane(
909
+ run: { paneId?: string; paneLabel?: string },
910
+ deps: { run?: HerdrRun; session?: string; seq?: number } = {},
911
+ ): { kind: "none" } | { kind: "released"; paneId: string } | { kind: "failed"; paneId: string; reason: string } {
912
+ if (run.paneId === undefined || run.paneLabel === undefined) return { kind: "none" };
913
+ const released = releaseWorkerPane(run.paneId, run.paneLabel, deps);
914
+ return released.ok
915
+ ? { kind: "released", paneId: run.paneId }
916
+ : { kind: "failed", paneId: run.paneId, reason: released.reason };
917
+ }
918
+
919
+ /**
920
+ * Every conductor-owned pane Herdr currently has, keyed by the run id it was
921
+ * reported under (#841).
922
+ *
923
+ * Read from `pane list`'s own agent-session record, never from a label pattern:
924
+ * the label is what a human reads, and matching on it is exactly how a stale
925
+ * lookalike (a renamed pane, a pane from a previous fleet) gets mistaken for a
926
+ * live worker. `source` proves conductor reported it; `value` is the run id.
927
+ */
928
+ export function listWorkerPanes(
929
+ deps: { run?: HerdrRun; session?: string } = {},
930
+ ): { ok: true; panes: { paneId: string; runId: string; label?: string }[] } | { ok: false; reason: string } {
931
+ const run = deps.run ?? realHerdrRun;
932
+ const session = deps.session ?? resolveHerdrSession();
933
+ const res = run(["--session", session, "pane", "list"]);
934
+ if (!res.ok) {
935
+ return { ok: false, reason: `herdr pane list failed: ${firstLine(res.stderr) || "no output"}` };
936
+ }
937
+ let parsed: unknown;
938
+ try {
939
+ parsed = JSON.parse(res.stdout);
940
+ } catch (err) {
941
+ return { ok: false, reason: `herdr pane list was unreadable: ${err instanceof Error ? err.message : String(err)}` };
942
+ }
943
+ const panes = (parsed as { result?: { panes?: unknown[] } }).result?.panes;
944
+ if (!Array.isArray(panes)) return { ok: false, reason: "herdr pane list carried no pane array" };
945
+ const owned: { paneId: string; runId: string; label?: string }[] = [];
946
+ for (const pane of panes) {
947
+ const p = pane as {
948
+ pane_id?: unknown;
949
+ agent?: unknown;
950
+ agent_session?: { source?: unknown; value?: unknown };
951
+ };
952
+ if (typeof p.pane_id !== "string") continue;
953
+ if (p.agent_session?.source !== WORKER_PANE_SOURCE) continue;
954
+ const runId = p.agent_session.value;
955
+ // Ours by source but carrying no run id: an identity nobody can resolve is
956
+ // not an identity. Reported as a pane with an empty run id so the caller
957
+ // treats it as stale rather than silently ignoring it.
958
+ owned.push({
959
+ paneId: p.pane_id,
960
+ runId: typeof runId === "string" ? runId : "",
961
+ ...(typeof p.agent === "string" ? { label: p.agent } : {}),
962
+ });
963
+ }
964
+ return { ok: true, panes: owned };
965
+ }
966
+
967
+ /** One live worker, as the reconciler needs to see it. */
968
+ export interface LiveWorkerPane {
969
+ runId: string;
970
+ issue: number;
971
+ attempt: number;
972
+ project: string;
973
+ /** The recorded session-host pid; absent when nothing ever reported one. */
974
+ pid?: number;
975
+ paneId?: string;
976
+ paneLabel?: string;
977
+ sessionFile?: string;
978
+ }
979
+
980
+ export type WorkerPaneReconciliation =
981
+ /** The recorded pane is still there and still carries this run — nothing done. */
982
+ | { kind: "intact"; runId: string; paneId: string }
983
+ /** Herdr lost the pane (a restart); a new one now represents the same child. */
984
+ | { kind: "reassociated"; runId: string; paneId: string; label: string }
985
+ /** No representation, and the reason. The run keeps working regardless. */
986
+ | { kind: "untracked"; runId: string; reason: string }
987
+ /** A conductor pane whose run is not live: handed back to Herdr. */
988
+ | { kind: "stale-released"; paneId: string; runId: string }
989
+ | { kind: "stale-release-failed"; paneId: string; runId: string; reason: string };
990
+
991
+ /**
992
+ * Make Herdr's conductor-owned panes agree with the live run set (#841).
993
+ *
994
+ * Idempotent by construction: a second pass over an already-reconciled fleet
995
+ * returns `intact` for every live run and finds no stale panes, so repeated
996
+ * daemon or Herdr restarts converge rather than accumulate.
997
+ *
998
+ * Three rules, and each exists to refuse a specific way this goes wrong:
999
+ *
1000
+ * - **A live run's pane is re-created, never duplicated.** Re-association is
1001
+ * keyed on the run id Herdr itself reports, so a pane that is still there is
1002
+ * left alone. Only a run whose pane Herdr no longer has gets a new one.
1003
+ * - **Cleanup is by exact identity, never by name or age.** A pane is stale only
1004
+ * when the run id it carries is absent from the live set. A worker's own pane
1005
+ * can therefore never be released while its run is live, whatever it is called
1006
+ * and however old it is.
1007
+ * - **Nothing here can stop a worker.** The only mutation is `release-agent`,
1008
+ * which hands lifecycle authority back to Herdr; the authoritative child is
1009
+ * never signalled, and its pane is never closed.
1010
+ */
1011
+ export function reconcileWorkerPanes(
1012
+ live: readonly LiveWorkerPane[],
1013
+ deps: WorkerPaneDeps = {},
1014
+ ): { ok: true; outcomes: WorkerPaneReconciliation[] } | { ok: false; reason: string } {
1015
+ const listed = listWorkerPanes(deps);
1016
+ if (!listed.ok) return { ok: false, reason: listed.reason };
1017
+ const byRun = new Map(listed.panes.map((pane) => [pane.runId, pane]));
1018
+ const liveIds = new Set(live.map((worker) => worker.runId));
1019
+ const outcomes: WorkerPaneReconciliation[] = [];
1020
+
1021
+ for (const worker of live) {
1022
+ const held = byRun.get(worker.runId);
1023
+ if (held !== undefined) {
1024
+ outcomes.push({ kind: "intact", runId: worker.runId, paneId: held.paneId });
1025
+ continue;
1026
+ }
1027
+ // Herdr does not have this run's pane. Only the pid makes a replacement
1028
+ // honest: the pane represents an exact child, so without one there is
1029
+ // nothing to represent and inventing a pane would be the silent claim this
1030
+ // whole surface exists to avoid.
1031
+ if (worker.pid === undefined) {
1032
+ outcomes.push({
1033
+ kind: "untracked",
1034
+ runId: worker.runId,
1035
+ reason: "no session-host pid was ever recorded for this run",
1036
+ });
1037
+ continue;
1038
+ }
1039
+ const opened = openWorkerPane(
1040
+ {
1041
+ project: worker.project,
1042
+ issue: worker.issue,
1043
+ attempt: worker.attempt,
1044
+ runId: worker.runId,
1045
+ pid: worker.pid,
1046
+ ...(worker.sessionFile === undefined ? {} : { sessionFile: worker.sessionFile }),
1047
+ },
1048
+ deps,
1049
+ );
1050
+ outcomes.push(
1051
+ opened.kind === "tracked"
1052
+ ? { kind: "reassociated", runId: worker.runId, paneId: opened.paneId, label: opened.label }
1053
+ : { kind: "untracked", runId: worker.runId, reason: opened.reason },
1054
+ );
1055
+ }
1056
+
1057
+ for (const pane of listed.panes) {
1058
+ if (liveIds.has(pane.runId)) continue;
1059
+ const released = releaseWorkerPane(pane.paneId, pane.label ?? "", deps);
1060
+ outcomes.push(
1061
+ released.ok
1062
+ ? { kind: "stale-released", paneId: pane.paneId, runId: pane.runId }
1063
+ : { kind: "stale-release-failed", paneId: pane.paneId, runId: pane.runId, reason: released.reason },
1064
+ );
1065
+ }
1066
+ return { ok: true, outcomes };
1067
+ }
1068
+
1069
+ function firstLine(text: string): string {
1070
+ return text.split("\n", 1)[0]?.trim() ?? "";
1071
+ }
1072
+
720
1073
  /** The kill syscall, injectable so error mapping is testable without one. */
721
1074
  export type KillFn = (pid: number, sig: NodeJS.Signals | 0) => void;
722
1075
 
@@ -961,9 +1314,7 @@ export async function stopConductorPane(
961
1314
 
962
1315
  async function herdrAgentList(deps: PaneStopDeps): Promise<HerdrAgent[]> {
963
1316
  const bin = deps.herdrBin ?? "herdr";
964
- const session =
965
- deps.herdrSession ??
966
- resolveHerdrSessionWithBridge({ env: process.env, herdrBin: bin });
1317
+ const session = deps.herdrSession ?? resolveHerdrSession(process.env);
967
1318
  const res = spawnSync(bin, ["--session", session, "agent", "list"], {
968
1319
  encoding: "utf8",
969
1320
  timeout: 8_000,
@@ -1064,9 +1415,7 @@ export function parseHerdrProcessInfo(stdout: string, paneId: string): ProcessIn
1064
1415
 
1065
1416
  async function herdrOmpForegroundPids(paneId: string, deps: PaneStopDeps): Promise<number[]> {
1066
1417
  const bin = deps.herdrBin ?? "herdr";
1067
- const session =
1068
- deps.herdrSession ??
1069
- resolveHerdrSessionWithBridge({ env: process.env, herdrBin: bin });
1418
+ const session = deps.herdrSession ?? resolveHerdrSession(process.env);
1070
1419
  const res = spawnSync(bin, ["--session", session, "pane", "process-info", "--pane", paneId], {
1071
1420
  encoding: "utf8",
1072
1421
  timeout: 8_000,
@@ -1642,13 +1991,22 @@ function briefStatusLine(project: ProjectConfig): string | undefined {
1642
1991
  }
1643
1992
 
1644
1993
  /**
1645
- * Unrecovered runs grouped by failure class (#132), or nothing when there are
1646
- * none.
1994
+ * Failure classes, in two blocks that mean different things (#132).
1647
1995
  *
1648
1996
  * Reported as classes rather than as row states because a row state is not an
1649
1997
  * issue state: the FAILED column counted four completed issues on this fleet
1650
- * while the genuinely stuck ones were invisible (#109). A class says which of
1651
- * those it is, and `recoveredAt` is what keeps a recovered row out of the list.
1998
+ * while the genuinely stuck ones were invisible (#109).
1999
+ *
2000
+ * That defect then reappeared one level up. A single "unrecovered" list keyed on
2001
+ * `recoveredAt IS NULL` swept in the two recorded-only actions, which never
2002
+ * stamp it — so this fleet's status read `returned-for-revision 44`, growing by
2003
+ * one on every review return, while every class an operator could actually act
2004
+ * on sat at zero and invisible beneath it. A count nobody can act on, printed
2005
+ * where the actionable ones go, is the same lie in a new column.
2006
+ *
2007
+ * So: `awaiting recovery` is what the sweep will pick up, and it is omitted when
2008
+ * empty because empty is the healthy answer. `classified, no action` is the
2009
+ * signal — review returns and holds — named as such, never as a backlog.
1652
2010
  */
1653
2011
  function failureClassBlock(projectName: string): string | undefined {
1654
2012
  const path = dbPath();
@@ -1656,12 +2014,23 @@ function failureClassBlock(projectName: string): string | undefined {
1656
2014
  let store: Store | undefined;
1657
2015
  try {
1658
2016
  store = openStore(path);
1659
- const counts = store.failureClassCounts(projectName);
1660
- if (counts.length === 0) return undefined;
1661
- return [
1662
- "failure classes (unrecovered)",
1663
- ...counts.map((c) => ` ${c.cls.padEnd(18)}${c.n}`),
1664
- ].join("\n");
2017
+ const awaiting = store.failureClassCounts(projectName);
2018
+ const recorded = store.recordedOnlyClassCounts(projectName);
2019
+ if (awaiting.length === 0 && recorded.length === 0) return undefined;
2020
+ const lines: string[] = [];
2021
+ if (awaiting.length > 0) {
2022
+ lines.push(
2023
+ "failure classes (awaiting recovery)",
2024
+ ...awaiting.map((c) => ` ${c.cls.padEnd(24)}${c.n}`),
2025
+ );
2026
+ }
2027
+ if (recorded.length > 0) {
2028
+ lines.push(
2029
+ "failure classes (classified, no action by design)",
2030
+ ...recorded.map((c) => ` ${c.cls.padEnd(24)}${c.n}`),
2031
+ );
2032
+ }
2033
+ return lines.join("\n");
1665
2034
  } catch {
1666
2035
  return undefined;
1667
2036
  } finally {
@@ -1901,6 +2270,23 @@ function makeChallengeCode(): string {
1901
2270
  return `FLEET-${hex}`;
1902
2271
  }
1903
2272
 
2273
+ /**
2274
+ * mm:ss for a window whose whole length is five minutes. Deliberately not
2275
+ * `formatDownDuration`, which rounds to whole minutes because it reports
2276
+ * hours-to-days outages: rounded to the minute, the last 30 seconds of this
2277
+ * window would read "5m of 5m left" while the wait was nearly over, which is
2278
+ * the exact ambiguity the progress lines exist to remove (#861).
2279
+ */
2280
+ function armClock(ms: number): string {
2281
+ const total = Math.max(0, Math.round(ms / 1000));
2282
+ const minutes = Math.floor(total / 60);
2283
+ const seconds = total % 60;
2284
+ return minutes === 0 ? `${seconds}s` : `${minutes}m${String(seconds).padStart(2, "0")}s`;
2285
+ }
2286
+
2287
+ /** How often the wait says it is still waiting. Six lines across the window. */
2288
+ const ARM_PROGRESS_INTERVAL_MS = 30_000;
2289
+
1904
2290
  /**
1905
2291
  * Polls the acknowledgement record for one exact challenge id until the
1906
2292
  * orchestrator's inbound adapter writes it or the deadline passes (#614).
@@ -1910,6 +2296,14 @@ function makeChallengeCode(): string {
1910
2296
  * orchestrator process. Only a record naming this exact id satisfies the
1911
2297
  * wait — an acknowledgement cut for a replaced challenge is inert here by
1912
2298
  * construction, and no transcript anywhere is opened.
2299
+ *
2300
+ * It also *says* it is waiting (#861). Measured 2026-08-21: an arm proof sat
2301
+ * silent for five minutes and was reported as a hung setup — the process was
2302
+ * healthy and the operator had no way to tell. A silent five-minute wait
2303
+ * inside a fence that holds dispatch is indistinguishable from a dead one, so
2304
+ * the elapsed/remaining line lands every {@link ARM_PROGRESS_INTERVAL_MS}
2305
+ * regardless of the (much shorter) poll cadence, and the terminal outcome is
2306
+ * always printed.
1913
2307
  */
1914
2308
  async function waitForArmAcknowledgement(
1915
2309
  challengeId: string,
@@ -1918,10 +2312,26 @@ async function waitForArmAcknowledgement(
1918
2312
  ): Promise<boolean> {
1919
2313
  const now = deps.now ?? Date.now;
1920
2314
  const sleep = deps.sleep ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)));
1921
- const deadline = now() + timeoutMs;
2315
+ const report = deps.progress;
2316
+ const startedAt = now();
2317
+ const deadline = startedAt + timeoutMs;
2318
+ let nextReportAt = startedAt + ARM_PROGRESS_INTERVAL_MS;
1922
2319
  for (;;) {
1923
- if (readArmAcknowledgement(challengeId) !== undefined) return true;
1924
- if (now() >= deadline) return false;
2320
+ if (readArmAcknowledgement(challengeId) !== undefined) {
2321
+ report?.(`arm: reply acknowledged after ${armClock(now() - startedAt)} the fleet is armed.`);
2322
+ return true;
2323
+ }
2324
+ const at = now();
2325
+ if (at >= deadline) return false;
2326
+ if (report !== undefined && at >= nextReportAt) {
2327
+ report(
2328
+ `arm: still waiting for the reply — ${armClock(at - startedAt)} elapsed, ` +
2329
+ `${armClock(deadline - at)} left. Nothing is stuck: reply in the Telegram chat with the code.`,
2330
+ );
2331
+ // Anchored to the clock, not to this pass, so a slow pass cannot make the
2332
+ // cadence drift into silence.
2333
+ while (nextReportAt <= at) nextReportAt += ARM_PROGRESS_INTERVAL_MS;
2334
+ }
1925
2335
  await sleep(5_000);
1926
2336
  }
1927
2337
  }
@@ -2057,7 +2467,7 @@ function probeOmpPane(
2057
2467
  }
2058
2468
  const agentName =
2059
2469
  tick.kind === "ok" ? (tick.config.agentName ?? DEFAULT_FLEET_AGENT_NAME) : DEFAULT_FLEET_AGENT_NAME;
2060
- const session = resolveHerdrSessionWithBridge({ env: process.env });
2470
+ const session = resolveHerdrSession(process.env);
2061
2471
  try {
2062
2472
  const res = spawnSync("herdr", ["--session", session, "agent", "list"], {
2063
2473
  encoding: "utf8",