omp-conductor 0.18.2 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/README.md +105 -40
  2. package/REFERENCE.md +865 -30
  3. package/package.json +1 -1
  4. package/schema/config.schema.json +26 -0
  5. package/src/admission.ts +212 -26
  6. package/src/ask.ts +288 -1
  7. package/src/briefs/orchestrator.md +6 -5
  8. package/src/cli.ts +5 -1
  9. package/src/command-help.ts +9 -1
  10. package/src/command-manifest.ts +36 -3
  11. package/src/commands/arm.ts +5 -1
  12. package/src/commands/context.ts +2 -0
  13. package/src/commands/message.ts +26 -2
  14. package/src/commands/reconcile-units.ts +104 -0
  15. package/src/commands/release-composition.ts +232 -0
  16. package/src/commands/resume.ts +2 -27
  17. package/src/commands/setup.ts +101 -16
  18. package/src/commands/stats.ts +11 -30
  19. package/src/commands/tail.ts +31 -1
  20. package/src/commands/upgrade.ts +20 -3
  21. package/src/commands/verb.ts +2 -1
  22. package/src/config-schema.ts +19 -0
  23. package/src/config.ts +80 -0
  24. package/src/credential-class.ts +366 -0
  25. package/src/daemon.ts +1218 -288
  26. package/src/dashboard/app.js +504 -2
  27. package/src/dashboard/controls.ts +336 -0
  28. package/src/dashboard/index.html +30 -0
  29. package/src/dashboard/server.ts +271 -30
  30. package/src/dashboard/style.css +116 -0
  31. package/src/dashboard/transcript.ts +173 -0
  32. package/src/doctor.ts +377 -20
  33. package/src/failure-class.ts +59 -0
  34. package/src/fleet.ts +497 -15
  35. package/src/host.ts +6 -130
  36. package/src/omp.ts +29 -0
  37. package/src/orchestrator-tick.ts +343 -88
  38. package/src/pause.ts +233 -0
  39. package/src/settlement.ts +159 -2
  40. package/src/setup-answers.ts +97 -0
  41. package/src/setup-host.ts +321 -1155
  42. package/src/setup-install.ts +204 -27
  43. package/src/setup-wizard.ts +111 -50
  44. package/src/setup.ts +33 -0
  45. package/src/spend-telemetry.ts +117 -0
  46. package/src/stats.ts +35 -0
  47. package/src/status-render.ts +348 -19
  48. package/src/store.ts +1229 -55
  49. package/src/telegram-freshness.ts +269 -0
  50. package/src/to-spec.ts +27 -0
  51. package/src/types.ts +697 -4
  52. package/src/unblock.ts +22 -0
  53. package/src/unit-reconcile.ts +303 -0
  54. package/src/upgrade-verify.ts +8 -1
  55. package/src/upgrade.ts +299 -12
  56. package/src/verbs/actions.ts +124 -10
  57. package/src/verbs/protocol.ts +70 -2
  58. package/src/verbs/server.ts +447 -8
  59. package/src/wake.ts +48 -0
  60. package/src/worker.ts +403 -3
package/src/fleet.ts CHANGED
@@ -343,6 +343,14 @@ export interface ArmDeps {
343
343
  pidAlive?: (pid: number) => boolean;
344
344
  lockPidAlive?: (pid: number) => boolean;
345
345
  lockFresh?: (mtimeMs: number) => boolean;
346
+ /**
347
+ * Where the pending-proof heartbeat is written (#861). The challenge proof
348
+ * waits up to five minutes on a human, inside a fence that holds dispatch:
349
+ * without this, that wait is silent and a healthy process is indistinguishable
350
+ * from a dead one. Absent means no reporting — the callers that have a surface
351
+ * (the CLI, the wizard) pass theirs.
352
+ */
353
+ progress?: (line: string) => void;
346
354
  }
347
355
 
348
356
  export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promise<ArmResult> {
@@ -484,6 +492,14 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
484
492
  `arm: outbound sendMessage failed — NOT armed: ${err instanceof Error ? err.message : String(err)}`,
485
493
  );
486
494
  }
495
+ // What is being waited on, before the wait starts: the window, the chat the
496
+ // reply has to land in, and the fact that dispatch is held until it does
497
+ // (#861). One line, so a five-minute wait opens with an explanation rather
498
+ // than with silence.
499
+ deps.progress?.(
500
+ `arm: challenge sent to ${channel.owner}${sendTopic === undefined ? "" : ` (topic ${sendTopic})`} — ` +
501
+ `waiting up to ${armClock(timeoutMs)} for the reply. Dispatch stays held until it arrives.`,
502
+ );
487
503
 
488
504
  // Wait for the orchestrator's own acknowledgement — conductor state written
489
505
  // by the inbound user-turn adapter when the real reply lands (#614). No
@@ -553,10 +569,14 @@ export interface HaltWithPaneResult extends HaltResult {
553
569
  export function hold(
554
570
  projectName?: string,
555
571
  source: string = "hold",
556
- opts: { keepTicks?: boolean } = {},
572
+ opts: { keepTicks?: boolean; reason?: string } = {},
557
573
  ): HoldResult {
558
574
  const wasPaused = isPaused(projectName);
559
- setPaused(true, { source }, projectName);
575
+ // `source` is a space-free token by contract (`pauseSourceToken`) — the fence
576
+ // matches on it, and whitespace there makes a pause the fence cannot prove
577
+ // (#552). The human sentence belongs in `reason=`, which is why this is a
578
+ // separate field rather than something a caller concatenates into `source`.
579
+ setPaused(true, { source, ...(opts.reason === undefined ? {} : { reason: opts.reason }) }, projectName);
560
580
  if (opts.keepTicks === true) return { wasPaused };
561
581
  return { wasPaused, disarmed: disarmTicks(projectName) };
562
582
  }
@@ -717,6 +737,407 @@ export interface HerdrAgent {
717
737
  sessionPath?: string;
718
738
  }
719
739
 
740
+ /**
741
+ * The Herdr representation of one live conductor worker (#840).
742
+ *
743
+ * ## Why this shape, and not `herdr agent start --kind omp`
744
+ *
745
+ * The authoritative worker is the `session-host` child conductor spawns itself:
746
+ * it holds the typed control socket, the mediated verb socket, the worktree, the
747
+ * transcript, the accounting and the output-schema settlement. Starting a second
748
+ * OMP process in a pane would look right in the workspace and be authoritative
749
+ * for nothing — the named silent fake of #840 — so nothing here starts an agent.
750
+ *
751
+ * Herdr already has the verb for exactly this case: an *external* supervisor
752
+ * reports lifecycle onto a pane it owns (`pane report-agent`,
753
+ * `pane report-agent-session`, `pane release-agent`). So conductor keeps the
754
+ * child it already spawned, and reports that child's identity and state onto a
755
+ * pane whose only job is display.
756
+ *
757
+ * ## What is displayed, and why that is not "a transcript-tail pane"
758
+ *
759
+ * The pane runs `omp-conductor tail <issue>`, which is the repository's existing
760
+ * read-only follower. The distinction #840 draws is about *authority*, not about
761
+ * pixels: nothing here parses that output, and no run state, settlement, turn,
762
+ * spend or model fact is derived from it. Identity comes from the exact child
763
+ * pid handed over by the spawn path, and state comes from the typed events the
764
+ * proxy already emits.
765
+ *
766
+ * ## Observation-only is structural, not a gate
767
+ *
768
+ * The authoritative child is spawned with `stdin: "ignore"` — there is no file
769
+ * descriptor for a keystroke to travel down, from a pane or anywhere else. The
770
+ * pane's own process is a follower whose stdin reaches only itself. So "input
771
+ * cannot reach OMP, change its prompt, invoke a verb, or settle the run" is a
772
+ * property of the process tree rather than a check that could be forgotten.
773
+ */
774
+ export interface WorkerPaneIdentity {
775
+ project: string;
776
+ issue: number;
777
+ attempt: number;
778
+ /** The run row's id — the durable identity the pane is reported under. */
779
+ runId: string;
780
+ /** The exact `session-host` pid, from the spawn path's own `onSpawn`. */
781
+ pid: number;
782
+ /** The transcript the child opened, when it has already reported one. */
783
+ sessionFile?: string;
784
+ }
785
+
786
+ /** A tracked pane, or the explicit reason there is none. Never a silent claim. */
787
+ export type WorkerPaneOutcome =
788
+ | { kind: "tracked"; paneId: string; label: string; pid: number }
789
+ | { kind: "unavailable"; reason: string };
790
+
791
+ /** One `herdr` invocation, injected so every path is testable with no terminal. */
792
+ export type HerdrRun = (args: readonly string[]) => { ok: boolean; stdout: string; stderr: string };
793
+
794
+ export interface WorkerPaneDeps {
795
+ run?: HerdrRun;
796
+ session?: string;
797
+ /** The follower command the pane displays; the CLI by default. */
798
+ viewer?: (identity: WorkerPaneIdentity) => readonly string[];
799
+ }
800
+
801
+ /**
802
+ * The pane's unique, human-readable name (#840: "uniquely named").
803
+ *
804
+ * Keyed by run id rather than issue: an issue can have several attempts, and two
805
+ * attempts must never resolve to one pane. The issue and attempt ride along
806
+ * because the operator reads this label in a workspace, not a database.
807
+ */
808
+ /**
809
+ * The external-supervisor source every conductor pane report carries.
810
+ *
811
+ * This, not the label, is what proves a pane is conductor's: it is written by
812
+ * `openWorkerPane` and read back by {@link listWorkerPanes}, so the two halves
813
+ * of reconciliation cannot come to disagree about what "ours" means.
814
+ */
815
+ export const WORKER_PANE_SOURCE = "omp-conductor";
816
+
817
+ export function workerPaneLabel(identity: WorkerPaneIdentity): string {
818
+ return `worker-${identity.project}-${identity.issue}-a${identity.attempt}-${identity.runId.slice(0, 8)}`;
819
+ }
820
+
821
+ const realHerdrRun: HerdrRun = (args) => {
822
+ const res = spawnSync("herdr", [...args], { encoding: "utf8", timeout: 8_000, env: process.env });
823
+ return {
824
+ ok: !res.error && res.status === 0,
825
+ stdout: res.stdout ?? "",
826
+ stderr: res.stderr ?? (res.error === undefined ? "" : String(res.error.message)),
827
+ };
828
+ };
829
+
830
+ /** `omp-conductor tail` — read-only by construction, and never parsed here. */
831
+ function defaultViewer(identity: WorkerPaneIdentity): readonly string[] {
832
+ return ["omp-conductor", "tail", String(identity.issue), "--project", identity.project];
833
+ }
834
+
835
+ /**
836
+ * Create the pane, run the follower in it, and report the child's identity and
837
+ * initial state — or say exactly why it could not.
838
+ *
839
+ * Every step is checked, and the first failure returns `unavailable` with the
840
+ * reason: a partially established representation (a pane with no agent identity,
841
+ * say) is worse than none, because it is a pane the operator would read as a
842
+ * tracked worker. What this deliberately does NOT do is decide what a failure
843
+ * means for the launch — failing closed versus running degraded is #841's
844
+ * policy, and inventing it here would pre-empt it.
845
+ */
846
+ export function openWorkerPane(identity: WorkerPaneIdentity, deps: WorkerPaneDeps = {}): WorkerPaneOutcome {
847
+ const run = deps.run ?? realHerdrRun;
848
+ const session = deps.session ?? resolveHerdrSession();
849
+ const label = workerPaneLabel(identity);
850
+ const base = ["--session", session, "pane"];
851
+
852
+ const split = run([...base, "split", "--direction", "down", "--ratio", "0.3"]);
853
+ if (!split.ok) {
854
+ return { kind: "unavailable", reason: `herdr pane split failed: ${firstLine(split.stderr) || "no output"}` };
855
+ }
856
+ const paneId = firstLine(split.stdout);
857
+ if (paneId === "") {
858
+ return { kind: "unavailable", reason: "herdr pane split reported no pane id" };
859
+ }
860
+
861
+ const named = run([...base, "rename", paneId, label]);
862
+ if (!named.ok) {
863
+ return { kind: "unavailable", reason: `herdr pane rename failed: ${firstLine(named.stderr) || "no output"}` };
864
+ }
865
+
866
+ // Identity before display: the pane must be attributable to this exact run
867
+ // before it shows anything, so a pane that appears is never a pane nobody can
868
+ // trace back to a worker.
869
+ const identified = run([
870
+ ...base,
871
+ "report-agent-session",
872
+ paneId,
873
+ "--source",
874
+ WORKER_PANE_SOURCE,
875
+ "--agent",
876
+ label,
877
+ "--agent-session-id",
878
+ identity.runId,
879
+ ...(identity.sessionFile === undefined ? [] : ["--agent-session-path", identity.sessionFile]),
880
+ "--session-start-source",
881
+ WORKER_PANE_SOURCE,
882
+ ]);
883
+ if (!identified.ok) {
884
+ return {
885
+ kind: "unavailable",
886
+ reason: `herdr pane report-agent-session failed: ${firstLine(identified.stderr) || "no output"}`,
887
+ };
888
+ }
889
+
890
+ const started = run([...base, "run", paneId, ...deps.viewer?.(identity) ?? defaultViewer(identity)]);
891
+ if (!started.ok) {
892
+ return { kind: "unavailable", reason: `herdr pane run failed: ${firstLine(started.stderr) || "no output"}` };
893
+ }
894
+
895
+ // A worker that has just spawned is working by definition. The ongoing
896
+ // projection of turn/pause/blocked transitions is #842's, from the same typed
897
+ // events — never from the pane's output.
898
+ const reported = reportWorkerPaneState(paneId, label, "working", { run, session });
899
+ if (!reported.ok) {
900
+ return { kind: "unavailable", reason: reported.reason };
901
+ }
902
+ return { kind: "tracked", paneId, label, pid: identity.pid };
903
+ }
904
+
905
+ /** Report one lifecycle state for a tracked pane. Monotonic `seq` is the caller's. */
906
+ export function reportWorkerPaneState(
907
+ paneId: string,
908
+ label: string,
909
+ state: "working" | "idle" | "blocked" | "unknown",
910
+ deps: { run?: HerdrRun; session?: string; seq?: number; message?: string } = {},
911
+ ): { ok: true } | { ok: false; reason: string } {
912
+ const run = deps.run ?? realHerdrRun;
913
+ const session = deps.session ?? resolveHerdrSession();
914
+ const res = run([
915
+ "--session",
916
+ session,
917
+ "pane",
918
+ "report-agent",
919
+ paneId,
920
+ "--source",
921
+ WORKER_PANE_SOURCE,
922
+ "--agent",
923
+ label,
924
+ "--state",
925
+ state,
926
+ ...(deps.seq === undefined ? [] : ["--seq", String(deps.seq)]),
927
+ ...(deps.message === undefined ? [] : ["--message", deps.message]),
928
+ ]);
929
+ return res.ok
930
+ ? { ok: true }
931
+ : { ok: false, reason: `herdr pane report-agent failed: ${firstLine(res.stderr) || "no output"}` };
932
+ }
933
+
934
+ /**
935
+ * Hand lifecycle authority back when the worker is gone.
936
+ *
937
+ * Release, never close: whether a settled worker's pane is closed, kept, or
938
+ * retained for N settlements is #841's documented policy, and a slice that
939
+ * closed panes here would decide it by accident. Releasing says only "conductor
940
+ * no longer speaks for this agent", which is exactly what is true.
941
+ */
942
+ export function releaseWorkerPane(
943
+ paneId: string,
944
+ label: string,
945
+ deps: { run?: HerdrRun; session?: string; seq?: number } = {},
946
+ ): { ok: true } | { ok: false; reason: string } {
947
+ const run = deps.run ?? realHerdrRun;
948
+ const session = deps.session ?? resolveHerdrSession();
949
+ const res = run([
950
+ "--session",
951
+ session,
952
+ "pane",
953
+ "release-agent",
954
+ paneId,
955
+ "--source",
956
+ WORKER_PANE_SOURCE,
957
+ "--agent",
958
+ label,
959
+ ...(deps.seq === undefined ? [] : ["--seq", String(deps.seq)]),
960
+ ]);
961
+ return res.ok
962
+ ? { ok: true }
963
+ : { ok: false, reason: `herdr pane release-agent failed: ${firstLine(res.stderr) || "no output"}` };
964
+ }
965
+
966
+ /**
967
+ * Hand back the pane a dead worker left behind (#842).
968
+ *
969
+ * Called for every run a restart reaps. The recorded pid is deliberately NOT
970
+ * consulted for liveness: a `session-host` child dies with the daemon that owned
971
+ * its verb socket, so an orphaned row's worker is gone whatever pid it carries —
972
+ * and pids are reused, so checking one is how a stranger's process comes to read
973
+ * as a live worker. A run with no recorded pane is a no-op, not a failure: it
974
+ * never had one to release.
975
+ */
976
+ export function releaseOrphanedWorkerPane(
977
+ run: { paneId?: string; paneLabel?: string },
978
+ deps: { run?: HerdrRun; session?: string; seq?: number } = {},
979
+ ): { kind: "none" } | { kind: "released"; paneId: string } | { kind: "failed"; paneId: string; reason: string } {
980
+ if (run.paneId === undefined || run.paneLabel === undefined) return { kind: "none" };
981
+ const released = releaseWorkerPane(run.paneId, run.paneLabel, deps);
982
+ return released.ok
983
+ ? { kind: "released", paneId: run.paneId }
984
+ : { kind: "failed", paneId: run.paneId, reason: released.reason };
985
+ }
986
+
987
+ /**
988
+ * Every conductor-owned pane Herdr currently has, keyed by the run id it was
989
+ * reported under (#841).
990
+ *
991
+ * Read from `pane list`'s own agent-session record, never from a label pattern:
992
+ * the label is what a human reads, and matching on it is exactly how a stale
993
+ * lookalike (a renamed pane, a pane from a previous fleet) gets mistaken for a
994
+ * live worker. `source` proves conductor reported it; `value` is the run id.
995
+ */
996
+ export function listWorkerPanes(
997
+ deps: { run?: HerdrRun; session?: string } = {},
998
+ ): { ok: true; panes: { paneId: string; runId: string; label?: string }[] } | { ok: false; reason: string } {
999
+ const run = deps.run ?? realHerdrRun;
1000
+ const session = deps.session ?? resolveHerdrSession();
1001
+ const res = run(["--session", session, "pane", "list"]);
1002
+ if (!res.ok) {
1003
+ return { ok: false, reason: `herdr pane list failed: ${firstLine(res.stderr) || "no output"}` };
1004
+ }
1005
+ let parsed: unknown;
1006
+ try {
1007
+ parsed = JSON.parse(res.stdout);
1008
+ } catch (err) {
1009
+ return { ok: false, reason: `herdr pane list was unreadable: ${err instanceof Error ? err.message : String(err)}` };
1010
+ }
1011
+ const panes = (parsed as { result?: { panes?: unknown[] } }).result?.panes;
1012
+ if (!Array.isArray(panes)) return { ok: false, reason: "herdr pane list carried no pane array" };
1013
+ const owned: { paneId: string; runId: string; label?: string }[] = [];
1014
+ for (const pane of panes) {
1015
+ const p = pane as {
1016
+ pane_id?: unknown;
1017
+ agent?: unknown;
1018
+ agent_session?: { source?: unknown; value?: unknown };
1019
+ };
1020
+ if (typeof p.pane_id !== "string") continue;
1021
+ if (p.agent_session?.source !== WORKER_PANE_SOURCE) continue;
1022
+ const runId = p.agent_session.value;
1023
+ // Ours by source but carrying no run id: an identity nobody can resolve is
1024
+ // not an identity. Reported as a pane with an empty run id so the caller
1025
+ // treats it as stale rather than silently ignoring it.
1026
+ owned.push({
1027
+ paneId: p.pane_id,
1028
+ runId: typeof runId === "string" ? runId : "",
1029
+ ...(typeof p.agent === "string" ? { label: p.agent } : {}),
1030
+ });
1031
+ }
1032
+ return { ok: true, panes: owned };
1033
+ }
1034
+
1035
+ /** One live worker, as the reconciler needs to see it. */
1036
+ export interface LiveWorkerPane {
1037
+ runId: string;
1038
+ issue: number;
1039
+ attempt: number;
1040
+ project: string;
1041
+ /** The recorded session-host pid; absent when nothing ever reported one. */
1042
+ pid?: number;
1043
+ paneId?: string;
1044
+ paneLabel?: string;
1045
+ sessionFile?: string;
1046
+ }
1047
+
1048
+ export type WorkerPaneReconciliation =
1049
+ /** The recorded pane is still there and still carries this run — nothing done. */
1050
+ | { kind: "intact"; runId: string; paneId: string }
1051
+ /** Herdr lost the pane (a restart); a new one now represents the same child. */
1052
+ | { kind: "reassociated"; runId: string; paneId: string; label: string }
1053
+ /** No representation, and the reason. The run keeps working regardless. */
1054
+ | { kind: "untracked"; runId: string; reason: string }
1055
+ /** A conductor pane whose run is not live: handed back to Herdr. */
1056
+ | { kind: "stale-released"; paneId: string; runId: string }
1057
+ | { kind: "stale-release-failed"; paneId: string; runId: string; reason: string };
1058
+
1059
+ /**
1060
+ * Make Herdr's conductor-owned panes agree with the live run set (#841).
1061
+ *
1062
+ * Idempotent by construction: a second pass over an already-reconciled fleet
1063
+ * returns `intact` for every live run and finds no stale panes, so repeated
1064
+ * daemon or Herdr restarts converge rather than accumulate.
1065
+ *
1066
+ * Three rules, and each exists to refuse a specific way this goes wrong:
1067
+ *
1068
+ * - **A live run's pane is re-created, never duplicated.** Re-association is
1069
+ * keyed on the run id Herdr itself reports, so a pane that is still there is
1070
+ * left alone. Only a run whose pane Herdr no longer has gets a new one.
1071
+ * - **Cleanup is by exact identity, never by name or age.** A pane is stale only
1072
+ * when the run id it carries is absent from the live set. A worker's own pane
1073
+ * can therefore never be released while its run is live, whatever it is called
1074
+ * and however old it is.
1075
+ * - **Nothing here can stop a worker.** The only mutation is `release-agent`,
1076
+ * which hands lifecycle authority back to Herdr; the authoritative child is
1077
+ * never signalled, and its pane is never closed.
1078
+ */
1079
+ export function reconcileWorkerPanes(
1080
+ live: readonly LiveWorkerPane[],
1081
+ deps: WorkerPaneDeps = {},
1082
+ ): { ok: true; outcomes: WorkerPaneReconciliation[] } | { ok: false; reason: string } {
1083
+ const listed = listWorkerPanes(deps);
1084
+ if (!listed.ok) return { ok: false, reason: listed.reason };
1085
+ const byRun = new Map(listed.panes.map((pane) => [pane.runId, pane]));
1086
+ const liveIds = new Set(live.map((worker) => worker.runId));
1087
+ const outcomes: WorkerPaneReconciliation[] = [];
1088
+
1089
+ for (const worker of live) {
1090
+ const held = byRun.get(worker.runId);
1091
+ if (held !== undefined) {
1092
+ outcomes.push({ kind: "intact", runId: worker.runId, paneId: held.paneId });
1093
+ continue;
1094
+ }
1095
+ // Herdr does not have this run's pane. Only the pid makes a replacement
1096
+ // honest: the pane represents an exact child, so without one there is
1097
+ // nothing to represent and inventing a pane would be the silent claim this
1098
+ // whole surface exists to avoid.
1099
+ if (worker.pid === undefined) {
1100
+ outcomes.push({
1101
+ kind: "untracked",
1102
+ runId: worker.runId,
1103
+ reason: "no session-host pid was ever recorded for this run",
1104
+ });
1105
+ continue;
1106
+ }
1107
+ const opened = openWorkerPane(
1108
+ {
1109
+ project: worker.project,
1110
+ issue: worker.issue,
1111
+ attempt: worker.attempt,
1112
+ runId: worker.runId,
1113
+ pid: worker.pid,
1114
+ ...(worker.sessionFile === undefined ? {} : { sessionFile: worker.sessionFile }),
1115
+ },
1116
+ deps,
1117
+ );
1118
+ outcomes.push(
1119
+ opened.kind === "tracked"
1120
+ ? { kind: "reassociated", runId: worker.runId, paneId: opened.paneId, label: opened.label }
1121
+ : { kind: "untracked", runId: worker.runId, reason: opened.reason },
1122
+ );
1123
+ }
1124
+
1125
+ for (const pane of listed.panes) {
1126
+ if (liveIds.has(pane.runId)) continue;
1127
+ const released = releaseWorkerPane(pane.paneId, pane.label ?? "", deps);
1128
+ outcomes.push(
1129
+ released.ok
1130
+ ? { kind: "stale-released", paneId: pane.paneId, runId: pane.runId }
1131
+ : { kind: "stale-release-failed", paneId: pane.paneId, runId: pane.runId, reason: released.reason },
1132
+ );
1133
+ }
1134
+ return { ok: true, outcomes };
1135
+ }
1136
+
1137
+ function firstLine(text: string): string {
1138
+ return text.split("\n", 1)[0]?.trim() ?? "";
1139
+ }
1140
+
720
1141
  /** The kill syscall, injectable so error mapping is testable without one. */
721
1142
  export type KillFn = (pid: number, sig: NodeJS.Signals | 0) => void;
722
1143
 
@@ -1642,13 +2063,22 @@ function briefStatusLine(project: ProjectConfig): string | undefined {
1642
2063
  }
1643
2064
 
1644
2065
  /**
1645
- * Unrecovered runs grouped by failure class (#132), or nothing when there are
1646
- * none.
2066
+ * Failure classes, in two blocks that mean different things (#132).
1647
2067
  *
1648
2068
  * Reported as classes rather than as row states because a row state is not an
1649
2069
  * issue state: the FAILED column counted four completed issues on this fleet
1650
- * while the genuinely stuck ones were invisible (#109). A class says which of
1651
- * those it is, and `recoveredAt` is what keeps a recovered row out of the list.
2070
+ * while the genuinely stuck ones were invisible (#109).
2071
+ *
2072
+ * That defect then reappeared one level up. A single "unrecovered" list keyed on
2073
+ * `recoveredAt IS NULL` swept in the two recorded-only actions, which never
2074
+ * stamp it — so this fleet's status read `returned-for-revision 44`, growing by
2075
+ * one on every review return, while every class an operator could actually act
2076
+ * on sat at zero and invisible beneath it. A count nobody can act on, printed
2077
+ * where the actionable ones go, is the same lie in a new column.
2078
+ *
2079
+ * So: `awaiting recovery` is what the sweep will pick up, and it is omitted when
2080
+ * empty because empty is the healthy answer. `classified, no action` is the
2081
+ * signal — review returns and holds — named as such, never as a backlog.
1652
2082
  */
1653
2083
  function failureClassBlock(projectName: string): string | undefined {
1654
2084
  const path = dbPath();
@@ -1656,12 +2086,23 @@ function failureClassBlock(projectName: string): string | undefined {
1656
2086
  let store: Store | undefined;
1657
2087
  try {
1658
2088
  store = openStore(path);
1659
- const counts = store.failureClassCounts(projectName);
1660
- if (counts.length === 0) return undefined;
1661
- return [
1662
- "failure classes (unrecovered)",
1663
- ...counts.map((c) => ` ${c.cls.padEnd(18)}${c.n}`),
1664
- ].join("\n");
2089
+ const awaiting = store.failureClassCounts(projectName);
2090
+ const recorded = store.recordedOnlyClassCounts(projectName);
2091
+ if (awaiting.length === 0 && recorded.length === 0) return undefined;
2092
+ const lines: string[] = [];
2093
+ if (awaiting.length > 0) {
2094
+ lines.push(
2095
+ "failure classes (awaiting recovery)",
2096
+ ...awaiting.map((c) => ` ${c.cls.padEnd(24)}${c.n}`),
2097
+ );
2098
+ }
2099
+ if (recorded.length > 0) {
2100
+ lines.push(
2101
+ "failure classes (classified, no action by design)",
2102
+ ...recorded.map((c) => ` ${c.cls.padEnd(24)}${c.n}`),
2103
+ );
2104
+ }
2105
+ return lines.join("\n");
1665
2106
  } catch {
1666
2107
  return undefined;
1667
2108
  } finally {
@@ -1901,6 +2342,23 @@ function makeChallengeCode(): string {
1901
2342
  return `FLEET-${hex}`;
1902
2343
  }
1903
2344
 
2345
+ /**
2346
+ * mm:ss for a window whose whole length is five minutes. Deliberately not
2347
+ * `formatDownDuration`, which rounds to whole minutes because it reports
2348
+ * hours-to-days outages: rounded to the minute, the last 30 seconds of this
2349
+ * window would read "5m of 5m left" while the wait was nearly over, which is
2350
+ * the exact ambiguity the progress lines exist to remove (#861).
2351
+ */
2352
+ function armClock(ms: number): string {
2353
+ const total = Math.max(0, Math.round(ms / 1000));
2354
+ const minutes = Math.floor(total / 60);
2355
+ const seconds = total % 60;
2356
+ return minutes === 0 ? `${seconds}s` : `${minutes}m${String(seconds).padStart(2, "0")}s`;
2357
+ }
2358
+
2359
+ /** How often the wait says it is still waiting. Six lines across the window. */
2360
+ const ARM_PROGRESS_INTERVAL_MS = 30_000;
2361
+
1904
2362
  /**
1905
2363
  * Polls the acknowledgement record for one exact challenge id until the
1906
2364
  * orchestrator's inbound adapter writes it or the deadline passes (#614).
@@ -1910,6 +2368,14 @@ function makeChallengeCode(): string {
1910
2368
  * orchestrator process. Only a record naming this exact id satisfies the
1911
2369
  * wait — an acknowledgement cut for a replaced challenge is inert here by
1912
2370
  * construction, and no transcript anywhere is opened.
2371
+ *
2372
+ * It also *says* it is waiting (#861). Measured 2026-08-21: an arm proof sat
2373
+ * silent for five minutes and was reported as a hung setup — the process was
2374
+ * healthy and the operator had no way to tell. A silent five-minute wait
2375
+ * inside a fence that holds dispatch is indistinguishable from a dead one, so
2376
+ * the elapsed/remaining line lands every {@link ARM_PROGRESS_INTERVAL_MS}
2377
+ * regardless of the (much shorter) poll cadence, and the terminal outcome is
2378
+ * always printed.
1913
2379
  */
1914
2380
  async function waitForArmAcknowledgement(
1915
2381
  challengeId: string,
@@ -1918,10 +2384,26 @@ async function waitForArmAcknowledgement(
1918
2384
  ): Promise<boolean> {
1919
2385
  const now = deps.now ?? Date.now;
1920
2386
  const sleep = deps.sleep ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)));
1921
- const deadline = now() + timeoutMs;
2387
+ const report = deps.progress;
2388
+ const startedAt = now();
2389
+ const deadline = startedAt + timeoutMs;
2390
+ let nextReportAt = startedAt + ARM_PROGRESS_INTERVAL_MS;
1922
2391
  for (;;) {
1923
- if (readArmAcknowledgement(challengeId) !== undefined) return true;
1924
- if (now() >= deadline) return false;
2392
+ if (readArmAcknowledgement(challengeId) !== undefined) {
2393
+ report?.(`arm: reply acknowledged after ${armClock(now() - startedAt)} the fleet is armed.`);
2394
+ return true;
2395
+ }
2396
+ const at = now();
2397
+ if (at >= deadline) return false;
2398
+ if (report !== undefined && at >= nextReportAt) {
2399
+ report(
2400
+ `arm: still waiting for the reply — ${armClock(at - startedAt)} elapsed, ` +
2401
+ `${armClock(deadline - at)} left. Nothing is stuck: reply in the Telegram chat with the code.`,
2402
+ );
2403
+ // Anchored to the clock, not to this pass, so a slow pass cannot make the
2404
+ // cadence drift into silence.
2405
+ while (nextReportAt <= at) nextReportAt += ARM_PROGRESS_INTERVAL_MS;
2406
+ }
1925
2407
  await sleep(5_000);
1926
2408
  }
1927
2409
  }