omp-conductor 0.18.2 → 0.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +105 -40
- package/REFERENCE.md +865 -30
- package/package.json +1 -1
- package/schema/config.schema.json +26 -0
- package/src/admission.ts +212 -26
- package/src/ask.ts +288 -1
- package/src/briefs/orchestrator.md +6 -5
- package/src/cli.ts +5 -1
- package/src/command-help.ts +9 -1
- package/src/command-manifest.ts +36 -3
- package/src/commands/arm.ts +5 -1
- package/src/commands/context.ts +2 -0
- package/src/commands/message.ts +26 -2
- package/src/commands/reconcile-units.ts +104 -0
- package/src/commands/release-composition.ts +232 -0
- package/src/commands/resume.ts +2 -27
- package/src/commands/setup.ts +101 -16
- package/src/commands/stats.ts +11 -30
- package/src/commands/tail.ts +31 -1
- package/src/commands/upgrade.ts +20 -3
- package/src/commands/verb.ts +2 -1
- package/src/config-schema.ts +19 -0
- package/src/config.ts +80 -0
- package/src/credential-class.ts +366 -0
- package/src/daemon.ts +1218 -288
- package/src/dashboard/app.js +504 -2
- package/src/dashboard/controls.ts +336 -0
- package/src/dashboard/index.html +30 -0
- package/src/dashboard/server.ts +271 -30
- package/src/dashboard/style.css +116 -0
- package/src/dashboard/transcript.ts +173 -0
- package/src/doctor.ts +379 -22
- package/src/failure-class.ts +59 -0
- package/src/fleet.ts +511 -101
- package/src/host.ts +6 -130
- package/src/omp.ts +29 -0
- package/src/orchestrator-tick.ts +343 -88
- package/src/pause.ts +233 -0
- package/src/settlement.ts +159 -2
- package/src/setup-answers.ts +97 -0
- package/src/setup-host.ts +325 -1159
- package/src/setup-install.ts +204 -27
- package/src/setup-wizard.ts +111 -50
- package/src/setup.ts +33 -0
- package/src/spend-telemetry.ts +117 -0
- package/src/stats.ts +35 -0
- package/src/status-render.ts +348 -19
- package/src/store.ts +1229 -55
- package/src/telegram-freshness.ts +269 -0
- package/src/to-spec.ts +27 -0
- package/src/types.ts +697 -4
- package/src/unblock.ts +22 -0
- package/src/unit-reconcile.ts +303 -0
- package/src/upgrade-verify.ts +8 -1
- package/src/upgrade.ts +326 -47
- package/src/verbs/actions.ts +124 -10
- package/src/verbs/protocol.ts +70 -2
- package/src/verbs/server.ts +447 -8
- package/src/wake.ts +48 -0
- package/src/worker.ts +403 -3
package/src/fleet.ts
CHANGED
|
@@ -105,10 +105,11 @@ export const PANE_HALT_FILE = ".conductor-pane-halted";
|
|
|
105
105
|
export const DEFAULT_HERDR_UNIT = "herdr-fleet.service";
|
|
106
106
|
|
|
107
107
|
/**
|
|
108
|
-
*
|
|
109
|
-
*
|
|
110
|
-
*
|
|
111
|
-
*
|
|
108
|
+
* The Herdr session conductor fleets run in. `HERDR_SESSION` wins when set;
|
|
109
|
+
* this is the only other name any surface may resolve. #320 renamed it from
|
|
110
|
+
* `fleet` and shipped a one-release probe that preferred a populated legacy
|
|
111
|
+
* session; #976 removed that probe, because a name resolved two ways is a
|
|
112
|
+
* name two surfaces can disagree about — and they did, in `upgrade`.
|
|
112
113
|
*/
|
|
113
114
|
export const DEFAULT_HERDR_SESSION = "conductor";
|
|
114
115
|
|
|
@@ -121,32 +122,13 @@ export const DEFAULT_HERDR_SESSION = "conductor";
|
|
|
121
122
|
export const DEFAULT_FLEET_AGENT_NAME = "fleet";
|
|
122
123
|
export const ARM_CHALLENGE_TIMEOUT_MS = 300_000;
|
|
123
124
|
|
|
124
|
-
|
|
125
|
-
/** Pre-#320 session name. Bridged for one release when `conductor` is empty. */
|
|
126
|
-
const LEGACY_HERDR_SESSION = "fleet";
|
|
127
|
-
|
|
128
|
-
export const LEGACY_HERDR_SESSION_HINT =
|
|
129
|
-
'herdr session "fleet" found — rename it to "conductor" or run omp-conductor setup host to pin HERDR_SESSION=fleet (remove this bridge next minor)';
|
|
130
|
-
|
|
131
|
-
let legacyHerdrSessionHintPrinted = false;
|
|
132
|
-
|
|
133
|
-
function noteLegacyHerdrSession(log?: (message: string) => void): void {
|
|
134
|
-
if (legacyHerdrSessionHintPrinted) return;
|
|
135
|
-
legacyHerdrSessionHintPrinted = true;
|
|
136
|
-
if (log) log(LEGACY_HERDR_SESSION_HINT);
|
|
137
|
-
else console.warn(LEGACY_HERDR_SESSION_HINT);
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
/** Reset the once-per-process rename hint (tests). */
|
|
141
|
-
export function resetLegacyHerdrSessionHintForTests(): void {
|
|
142
|
-
legacyHerdrSessionHintPrinted = false;
|
|
143
|
-
}
|
|
144
|
-
|
|
145
125
|
/**
|
|
146
|
-
* Resolve the Herdr session name
|
|
147
|
-
*
|
|
148
|
-
*
|
|
149
|
-
*
|
|
126
|
+
* Resolve the Herdr session name — the single resolution every surface uses.
|
|
127
|
+
* `HERDR_SESSION` is authoritative; otherwise {@link DEFAULT_HERDR_SESSION}.
|
|
128
|
+
* Never probe for an alternative name here: a session that answers is not
|
|
129
|
+
* evidence of which session this fleet is, and the probe that assumed it was
|
|
130
|
+
* is what made `upgrade` install against `fleet` and verify against
|
|
131
|
+
* `conductor` (#976).
|
|
150
132
|
*/
|
|
151
133
|
export function resolveHerdrSession(env: NodeJS.ProcessEnv = process.env): string {
|
|
152
134
|
const explicit = env["HERDR_SESSION"];
|
|
@@ -154,56 +136,6 @@ export function resolveHerdrSession(env: NodeJS.ProcessEnv = process.env): strin
|
|
|
154
136
|
return DEFAULT_HERDR_SESSION;
|
|
155
137
|
}
|
|
156
138
|
|
|
157
|
-
function herdrAgentListRaw(
|
|
158
|
-
session: string,
|
|
159
|
-
bin: string,
|
|
160
|
-
env: NodeJS.ProcessEnv,
|
|
161
|
-
): { ok: true; agents: HerdrAgent[] } | { ok: false } {
|
|
162
|
-
const res = spawnSync(bin, ["--session", session, "agent", "list"], {
|
|
163
|
-
encoding: "utf8",
|
|
164
|
-
timeout: 8_000,
|
|
165
|
-
env,
|
|
166
|
-
});
|
|
167
|
-
if (res.error || res.status !== 0) return { ok: false };
|
|
168
|
-
try {
|
|
169
|
-
return { ok: true, agents: parseHerdrAgentList(res.stdout ?? "") };
|
|
170
|
-
} catch {
|
|
171
|
-
return { ok: false };
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
/**
|
|
176
|
-
* One-release bridge (#320): when `HERDR_SESSION` is unset, a `conductor`
|
|
177
|
-
* session with no agents while a `fleet` session answers → use `fleet` and
|
|
178
|
-
* print the rename hint once. Remove next minor.
|
|
179
|
-
*/
|
|
180
|
-
export function resolveHerdrSessionWithBridge(opts: {
|
|
181
|
-
env?: NodeJS.ProcessEnv;
|
|
182
|
-
herdrBin?: string;
|
|
183
|
-
log?: (message: string) => void;
|
|
184
|
-
} = {}): string {
|
|
185
|
-
const env = opts.env ?? process.env;
|
|
186
|
-
const explicit = env["HERDR_SESSION"];
|
|
187
|
-
if (explicit !== undefined && explicit.length > 0) return explicit;
|
|
188
|
-
|
|
189
|
-
const bin = opts.herdrBin ?? "herdr";
|
|
190
|
-
const primary = herdrAgentListRaw(DEFAULT_HERDR_SESSION, bin, env);
|
|
191
|
-
if (primary.ok && primary.agents.length > 0) return DEFAULT_HERDR_SESSION;
|
|
192
|
-
|
|
193
|
-
const legacy = herdrAgentListRaw(LEGACY_HERDR_SESSION, bin, env);
|
|
194
|
-
if (legacy.ok && legacy.agents.length > 0) {
|
|
195
|
-
noteLegacyHerdrSession(opts.log);
|
|
196
|
-
return LEGACY_HERDR_SESSION;
|
|
197
|
-
}
|
|
198
|
-
// conductor answered (even empty) wins over a dead legacy session.
|
|
199
|
-
if (primary.ok) return DEFAULT_HERDR_SESSION;
|
|
200
|
-
if (legacy.ok) {
|
|
201
|
-
noteLegacyHerdrSession(opts.log);
|
|
202
|
-
return LEGACY_HERDR_SESSION;
|
|
203
|
-
}
|
|
204
|
-
return DEFAULT_HERDR_SESSION;
|
|
205
|
-
}
|
|
206
|
-
|
|
207
139
|
export function telegramStateDir(): string {
|
|
208
140
|
const override = process.env["OMP_TELEGRAM_STATE_DIR"];
|
|
209
141
|
if (override !== undefined && override.length > 0) return override;
|
|
@@ -343,6 +275,14 @@ export interface ArmDeps {
|
|
|
343
275
|
pidAlive?: (pid: number) => boolean;
|
|
344
276
|
lockPidAlive?: (pid: number) => boolean;
|
|
345
277
|
lockFresh?: (mtimeMs: number) => boolean;
|
|
278
|
+
/**
|
|
279
|
+
* Where the pending-proof heartbeat is written (#861). The challenge proof
|
|
280
|
+
* waits up to five minutes on a human, inside a fence that holds dispatch:
|
|
281
|
+
* without this, that wait is silent and a healthy process is indistinguishable
|
|
282
|
+
* from a dead one. Absent means no reporting — the callers that have a surface
|
|
283
|
+
* (the CLI, the wizard) pass theirs.
|
|
284
|
+
*/
|
|
285
|
+
progress?: (line: string) => void;
|
|
346
286
|
}
|
|
347
287
|
|
|
348
288
|
export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promise<ArmResult> {
|
|
@@ -484,6 +424,14 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
484
424
|
`arm: outbound sendMessage failed — NOT armed: ${err instanceof Error ? err.message : String(err)}`,
|
|
485
425
|
);
|
|
486
426
|
}
|
|
427
|
+
// What is being waited on, before the wait starts: the window, the chat the
|
|
428
|
+
// reply has to land in, and the fact that dispatch is held until it does
|
|
429
|
+
// (#861). One line, so a five-minute wait opens with an explanation rather
|
|
430
|
+
// than with silence.
|
|
431
|
+
deps.progress?.(
|
|
432
|
+
`arm: challenge sent to ${channel.owner}${sendTopic === undefined ? "" : ` (topic ${sendTopic})`} — ` +
|
|
433
|
+
`waiting up to ${armClock(timeoutMs)} for the reply. Dispatch stays held until it arrives.`,
|
|
434
|
+
);
|
|
487
435
|
|
|
488
436
|
// Wait for the orchestrator's own acknowledgement — conductor state written
|
|
489
437
|
// by the inbound user-turn adapter when the real reply lands (#614). No
|
|
@@ -553,10 +501,14 @@ export interface HaltWithPaneResult extends HaltResult {
|
|
|
553
501
|
export function hold(
|
|
554
502
|
projectName?: string,
|
|
555
503
|
source: string = "hold",
|
|
556
|
-
opts: { keepTicks?: boolean } = {},
|
|
504
|
+
opts: { keepTicks?: boolean; reason?: string } = {},
|
|
557
505
|
): HoldResult {
|
|
558
506
|
const wasPaused = isPaused(projectName);
|
|
559
|
-
|
|
507
|
+
// `source` is a space-free token by contract (`pauseSourceToken`) — the fence
|
|
508
|
+
// matches on it, and whitespace there makes a pause the fence cannot prove
|
|
509
|
+
// (#552). The human sentence belongs in `reason=`, which is why this is a
|
|
510
|
+
// separate field rather than something a caller concatenates into `source`.
|
|
511
|
+
setPaused(true, { source, ...(opts.reason === undefined ? {} : { reason: opts.reason }) }, projectName);
|
|
560
512
|
if (opts.keepTicks === true) return { wasPaused };
|
|
561
513
|
return { wasPaused, disarmed: disarmTicks(projectName) };
|
|
562
514
|
}
|
|
@@ -717,6 +669,407 @@ export interface HerdrAgent {
|
|
|
717
669
|
sessionPath?: string;
|
|
718
670
|
}
|
|
719
671
|
|
|
672
|
+
/**
|
|
673
|
+
* The Herdr representation of one live conductor worker (#840).
|
|
674
|
+
*
|
|
675
|
+
* ## Why this shape, and not `herdr agent start --kind omp`
|
|
676
|
+
*
|
|
677
|
+
* The authoritative worker is the `session-host` child conductor spawns itself:
|
|
678
|
+
* it holds the typed control socket, the mediated verb socket, the worktree, the
|
|
679
|
+
* transcript, the accounting and the output-schema settlement. Starting a second
|
|
680
|
+
* OMP process in a pane would look right in the workspace and be authoritative
|
|
681
|
+
* for nothing — the named silent fake of #840 — so nothing here starts an agent.
|
|
682
|
+
*
|
|
683
|
+
* Herdr already has the verb for exactly this case: an *external* supervisor
|
|
684
|
+
* reports lifecycle onto a pane it owns (`pane report-agent`,
|
|
685
|
+
* `pane report-agent-session`, `pane release-agent`). So conductor keeps the
|
|
686
|
+
* child it already spawned, and reports that child's identity and state onto a
|
|
687
|
+
* pane whose only job is display.
|
|
688
|
+
*
|
|
689
|
+
* ## What is displayed, and why that is not "a transcript-tail pane"
|
|
690
|
+
*
|
|
691
|
+
* The pane runs `omp-conductor tail <issue>`, which is the repository's existing
|
|
692
|
+
* read-only follower. The distinction #840 draws is about *authority*, not about
|
|
693
|
+
* pixels: nothing here parses that output, and no run state, settlement, turn,
|
|
694
|
+
* spend or model fact is derived from it. Identity comes from the exact child
|
|
695
|
+
* pid handed over by the spawn path, and state comes from the typed events the
|
|
696
|
+
* proxy already emits.
|
|
697
|
+
*
|
|
698
|
+
* ## Observation-only is structural, not a gate
|
|
699
|
+
*
|
|
700
|
+
* The authoritative child is spawned with `stdin: "ignore"` — there is no file
|
|
701
|
+
* descriptor for a keystroke to travel down, from a pane or anywhere else. The
|
|
702
|
+
* pane's own process is a follower whose stdin reaches only itself. So "input
|
|
703
|
+
* cannot reach OMP, change its prompt, invoke a verb, or settle the run" is a
|
|
704
|
+
* property of the process tree rather than a check that could be forgotten.
|
|
705
|
+
*/
|
|
706
|
+
export interface WorkerPaneIdentity {
|
|
707
|
+
project: string;
|
|
708
|
+
issue: number;
|
|
709
|
+
attempt: number;
|
|
710
|
+
/** The run row's id — the durable identity the pane is reported under. */
|
|
711
|
+
runId: string;
|
|
712
|
+
/** The exact `session-host` pid, from the spawn path's own `onSpawn`. */
|
|
713
|
+
pid: number;
|
|
714
|
+
/** The transcript the child opened, when it has already reported one. */
|
|
715
|
+
sessionFile?: string;
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
/** A tracked pane, or the explicit reason there is none. Never a silent claim. */
|
|
719
|
+
export type WorkerPaneOutcome =
|
|
720
|
+
| { kind: "tracked"; paneId: string; label: string; pid: number }
|
|
721
|
+
| { kind: "unavailable"; reason: string };
|
|
722
|
+
|
|
723
|
+
/** One `herdr` invocation, injected so every path is testable with no terminal. */
|
|
724
|
+
export type HerdrRun = (args: readonly string[]) => { ok: boolean; stdout: string; stderr: string };
|
|
725
|
+
|
|
726
|
+
export interface WorkerPaneDeps {
|
|
727
|
+
run?: HerdrRun;
|
|
728
|
+
session?: string;
|
|
729
|
+
/** The follower command the pane displays; the CLI by default. */
|
|
730
|
+
viewer?: (identity: WorkerPaneIdentity) => readonly string[];
|
|
731
|
+
}
|
|
732
|
+
|
|
733
|
+
/**
|
|
734
|
+
* The pane's unique, human-readable name (#840: "uniquely named").
|
|
735
|
+
*
|
|
736
|
+
* Keyed by run id rather than issue: an issue can have several attempts, and two
|
|
737
|
+
* attempts must never resolve to one pane. The issue and attempt ride along
|
|
738
|
+
* because the operator reads this label in a workspace, not a database.
|
|
739
|
+
*/
|
|
740
|
+
/**
|
|
741
|
+
* The external-supervisor source every conductor pane report carries.
|
|
742
|
+
*
|
|
743
|
+
* This, not the label, is what proves a pane is conductor's: it is written by
|
|
744
|
+
* `openWorkerPane` and read back by {@link listWorkerPanes}, so the two halves
|
|
745
|
+
* of reconciliation cannot come to disagree about what "ours" means.
|
|
746
|
+
*/
|
|
747
|
+
export const WORKER_PANE_SOURCE = "omp-conductor";
|
|
748
|
+
|
|
749
|
+
export function workerPaneLabel(identity: WorkerPaneIdentity): string {
|
|
750
|
+
return `worker-${identity.project}-${identity.issue}-a${identity.attempt}-${identity.runId.slice(0, 8)}`;
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
const realHerdrRun: HerdrRun = (args) => {
|
|
754
|
+
const res = spawnSync("herdr", [...args], { encoding: "utf8", timeout: 8_000, env: process.env });
|
|
755
|
+
return {
|
|
756
|
+
ok: !res.error && res.status === 0,
|
|
757
|
+
stdout: res.stdout ?? "",
|
|
758
|
+
stderr: res.stderr ?? (res.error === undefined ? "" : String(res.error.message)),
|
|
759
|
+
};
|
|
760
|
+
};
|
|
761
|
+
|
|
762
|
+
/** `omp-conductor tail` — read-only by construction, and never parsed here. */
|
|
763
|
+
function defaultViewer(identity: WorkerPaneIdentity): readonly string[] {
|
|
764
|
+
return ["omp-conductor", "tail", String(identity.issue), "--project", identity.project];
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
/**
|
|
768
|
+
* Create the pane, run the follower in it, and report the child's identity and
|
|
769
|
+
* initial state — or say exactly why it could not.
|
|
770
|
+
*
|
|
771
|
+
* Every step is checked, and the first failure returns `unavailable` with the
|
|
772
|
+
* reason: a partially established representation (a pane with no agent identity,
|
|
773
|
+
* say) is worse than none, because it is a pane the operator would read as a
|
|
774
|
+
* tracked worker. What this deliberately does NOT do is decide what a failure
|
|
775
|
+
* means for the launch — failing closed versus running degraded is #841's
|
|
776
|
+
* policy, and inventing it here would pre-empt it.
|
|
777
|
+
*/
|
|
778
|
+
export function openWorkerPane(identity: WorkerPaneIdentity, deps: WorkerPaneDeps = {}): WorkerPaneOutcome {
|
|
779
|
+
const run = deps.run ?? realHerdrRun;
|
|
780
|
+
const session = deps.session ?? resolveHerdrSession();
|
|
781
|
+
const label = workerPaneLabel(identity);
|
|
782
|
+
const base = ["--session", session, "pane"];
|
|
783
|
+
|
|
784
|
+
const split = run([...base, "split", "--direction", "down", "--ratio", "0.3"]);
|
|
785
|
+
if (!split.ok) {
|
|
786
|
+
return { kind: "unavailable", reason: `herdr pane split failed: ${firstLine(split.stderr) || "no output"}` };
|
|
787
|
+
}
|
|
788
|
+
const paneId = firstLine(split.stdout);
|
|
789
|
+
if (paneId === "") {
|
|
790
|
+
return { kind: "unavailable", reason: "herdr pane split reported no pane id" };
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
const named = run([...base, "rename", paneId, label]);
|
|
794
|
+
if (!named.ok) {
|
|
795
|
+
return { kind: "unavailable", reason: `herdr pane rename failed: ${firstLine(named.stderr) || "no output"}` };
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
// Identity before display: the pane must be attributable to this exact run
|
|
799
|
+
// before it shows anything, so a pane that appears is never a pane nobody can
|
|
800
|
+
// trace back to a worker.
|
|
801
|
+
const identified = run([
|
|
802
|
+
...base,
|
|
803
|
+
"report-agent-session",
|
|
804
|
+
paneId,
|
|
805
|
+
"--source",
|
|
806
|
+
WORKER_PANE_SOURCE,
|
|
807
|
+
"--agent",
|
|
808
|
+
label,
|
|
809
|
+
"--agent-session-id",
|
|
810
|
+
identity.runId,
|
|
811
|
+
...(identity.sessionFile === undefined ? [] : ["--agent-session-path", identity.sessionFile]),
|
|
812
|
+
"--session-start-source",
|
|
813
|
+
WORKER_PANE_SOURCE,
|
|
814
|
+
]);
|
|
815
|
+
if (!identified.ok) {
|
|
816
|
+
return {
|
|
817
|
+
kind: "unavailable",
|
|
818
|
+
reason: `herdr pane report-agent-session failed: ${firstLine(identified.stderr) || "no output"}`,
|
|
819
|
+
};
|
|
820
|
+
}
|
|
821
|
+
|
|
822
|
+
const started = run([...base, "run", paneId, ...deps.viewer?.(identity) ?? defaultViewer(identity)]);
|
|
823
|
+
if (!started.ok) {
|
|
824
|
+
return { kind: "unavailable", reason: `herdr pane run failed: ${firstLine(started.stderr) || "no output"}` };
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
// A worker that has just spawned is working by definition. The ongoing
|
|
828
|
+
// projection of turn/pause/blocked transitions is #842's, from the same typed
|
|
829
|
+
// events — never from the pane's output.
|
|
830
|
+
const reported = reportWorkerPaneState(paneId, label, "working", { run, session });
|
|
831
|
+
if (!reported.ok) {
|
|
832
|
+
return { kind: "unavailable", reason: reported.reason };
|
|
833
|
+
}
|
|
834
|
+
return { kind: "tracked", paneId, label, pid: identity.pid };
|
|
835
|
+
}
|
|
836
|
+
|
|
837
|
+
/** Report one lifecycle state for a tracked pane. Monotonic `seq` is the caller's. */
|
|
838
|
+
export function reportWorkerPaneState(
|
|
839
|
+
paneId: string,
|
|
840
|
+
label: string,
|
|
841
|
+
state: "working" | "idle" | "blocked" | "unknown",
|
|
842
|
+
deps: { run?: HerdrRun; session?: string; seq?: number; message?: string } = {},
|
|
843
|
+
): { ok: true } | { ok: false; reason: string } {
|
|
844
|
+
const run = deps.run ?? realHerdrRun;
|
|
845
|
+
const session = deps.session ?? resolveHerdrSession();
|
|
846
|
+
const res = run([
|
|
847
|
+
"--session",
|
|
848
|
+
session,
|
|
849
|
+
"pane",
|
|
850
|
+
"report-agent",
|
|
851
|
+
paneId,
|
|
852
|
+
"--source",
|
|
853
|
+
WORKER_PANE_SOURCE,
|
|
854
|
+
"--agent",
|
|
855
|
+
label,
|
|
856
|
+
"--state",
|
|
857
|
+
state,
|
|
858
|
+
...(deps.seq === undefined ? [] : ["--seq", String(deps.seq)]),
|
|
859
|
+
...(deps.message === undefined ? [] : ["--message", deps.message]),
|
|
860
|
+
]);
|
|
861
|
+
return res.ok
|
|
862
|
+
? { ok: true }
|
|
863
|
+
: { ok: false, reason: `herdr pane report-agent failed: ${firstLine(res.stderr) || "no output"}` };
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
/**
|
|
867
|
+
* Hand lifecycle authority back when the worker is gone.
|
|
868
|
+
*
|
|
869
|
+
* Release, never close: whether a settled worker's pane is closed, kept, or
|
|
870
|
+
* retained for N settlements is #841's documented policy, and a slice that
|
|
871
|
+
* closed panes here would decide it by accident. Releasing says only "conductor
|
|
872
|
+
* no longer speaks for this agent", which is exactly what is true.
|
|
873
|
+
*/
|
|
874
|
+
export function releaseWorkerPane(
|
|
875
|
+
paneId: string,
|
|
876
|
+
label: string,
|
|
877
|
+
deps: { run?: HerdrRun; session?: string; seq?: number } = {},
|
|
878
|
+
): { ok: true } | { ok: false; reason: string } {
|
|
879
|
+
const run = deps.run ?? realHerdrRun;
|
|
880
|
+
const session = deps.session ?? resolveHerdrSession();
|
|
881
|
+
const res = run([
|
|
882
|
+
"--session",
|
|
883
|
+
session,
|
|
884
|
+
"pane",
|
|
885
|
+
"release-agent",
|
|
886
|
+
paneId,
|
|
887
|
+
"--source",
|
|
888
|
+
WORKER_PANE_SOURCE,
|
|
889
|
+
"--agent",
|
|
890
|
+
label,
|
|
891
|
+
...(deps.seq === undefined ? [] : ["--seq", String(deps.seq)]),
|
|
892
|
+
]);
|
|
893
|
+
return res.ok
|
|
894
|
+
? { ok: true }
|
|
895
|
+
: { ok: false, reason: `herdr pane release-agent failed: ${firstLine(res.stderr) || "no output"}` };
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
/**
|
|
899
|
+
* Hand back the pane a dead worker left behind (#842).
|
|
900
|
+
*
|
|
901
|
+
* Called for every run a restart reaps. The recorded pid is deliberately NOT
|
|
902
|
+
* consulted for liveness: a `session-host` child dies with the daemon that owned
|
|
903
|
+
* its verb socket, so an orphaned row's worker is gone whatever pid it carries —
|
|
904
|
+
* and pids are reused, so checking one is how a stranger's process comes to read
|
|
905
|
+
* as a live worker. A run with no recorded pane is a no-op, not a failure: it
|
|
906
|
+
* never had one to release.
|
|
907
|
+
*/
|
|
908
|
+
export function releaseOrphanedWorkerPane(
|
|
909
|
+
run: { paneId?: string; paneLabel?: string },
|
|
910
|
+
deps: { run?: HerdrRun; session?: string; seq?: number } = {},
|
|
911
|
+
): { kind: "none" } | { kind: "released"; paneId: string } | { kind: "failed"; paneId: string; reason: string } {
|
|
912
|
+
if (run.paneId === undefined || run.paneLabel === undefined) return { kind: "none" };
|
|
913
|
+
const released = releaseWorkerPane(run.paneId, run.paneLabel, deps);
|
|
914
|
+
return released.ok
|
|
915
|
+
? { kind: "released", paneId: run.paneId }
|
|
916
|
+
: { kind: "failed", paneId: run.paneId, reason: released.reason };
|
|
917
|
+
}
|
|
918
|
+
|
|
919
|
+
/**
|
|
920
|
+
* Every conductor-owned pane Herdr currently has, keyed by the run id it was
|
|
921
|
+
* reported under (#841).
|
|
922
|
+
*
|
|
923
|
+
* Read from `pane list`'s own agent-session record, never from a label pattern:
|
|
924
|
+
* the label is what a human reads, and matching on it is exactly how a stale
|
|
925
|
+
* lookalike (a renamed pane, a pane from a previous fleet) gets mistaken for a
|
|
926
|
+
* live worker. `source` proves conductor reported it; `value` is the run id.
|
|
927
|
+
*/
|
|
928
|
+
export function listWorkerPanes(
|
|
929
|
+
deps: { run?: HerdrRun; session?: string } = {},
|
|
930
|
+
): { ok: true; panes: { paneId: string; runId: string; label?: string }[] } | { ok: false; reason: string } {
|
|
931
|
+
const run = deps.run ?? realHerdrRun;
|
|
932
|
+
const session = deps.session ?? resolveHerdrSession();
|
|
933
|
+
const res = run(["--session", session, "pane", "list"]);
|
|
934
|
+
if (!res.ok) {
|
|
935
|
+
return { ok: false, reason: `herdr pane list failed: ${firstLine(res.stderr) || "no output"}` };
|
|
936
|
+
}
|
|
937
|
+
let parsed: unknown;
|
|
938
|
+
try {
|
|
939
|
+
parsed = JSON.parse(res.stdout);
|
|
940
|
+
} catch (err) {
|
|
941
|
+
return { ok: false, reason: `herdr pane list was unreadable: ${err instanceof Error ? err.message : String(err)}` };
|
|
942
|
+
}
|
|
943
|
+
const panes = (parsed as { result?: { panes?: unknown[] } }).result?.panes;
|
|
944
|
+
if (!Array.isArray(panes)) return { ok: false, reason: "herdr pane list carried no pane array" };
|
|
945
|
+
const owned: { paneId: string; runId: string; label?: string }[] = [];
|
|
946
|
+
for (const pane of panes) {
|
|
947
|
+
const p = pane as {
|
|
948
|
+
pane_id?: unknown;
|
|
949
|
+
agent?: unknown;
|
|
950
|
+
agent_session?: { source?: unknown; value?: unknown };
|
|
951
|
+
};
|
|
952
|
+
if (typeof p.pane_id !== "string") continue;
|
|
953
|
+
if (p.agent_session?.source !== WORKER_PANE_SOURCE) continue;
|
|
954
|
+
const runId = p.agent_session.value;
|
|
955
|
+
// Ours by source but carrying no run id: an identity nobody can resolve is
|
|
956
|
+
// not an identity. Reported as a pane with an empty run id so the caller
|
|
957
|
+
// treats it as stale rather than silently ignoring it.
|
|
958
|
+
owned.push({
|
|
959
|
+
paneId: p.pane_id,
|
|
960
|
+
runId: typeof runId === "string" ? runId : "",
|
|
961
|
+
...(typeof p.agent === "string" ? { label: p.agent } : {}),
|
|
962
|
+
});
|
|
963
|
+
}
|
|
964
|
+
return { ok: true, panes: owned };
|
|
965
|
+
}
|
|
966
|
+
|
|
967
|
+
/** One live worker, as the reconciler needs to see it. */
|
|
968
|
+
export interface LiveWorkerPane {
|
|
969
|
+
runId: string;
|
|
970
|
+
issue: number;
|
|
971
|
+
attempt: number;
|
|
972
|
+
project: string;
|
|
973
|
+
/** The recorded session-host pid; absent when nothing ever reported one. */
|
|
974
|
+
pid?: number;
|
|
975
|
+
paneId?: string;
|
|
976
|
+
paneLabel?: string;
|
|
977
|
+
sessionFile?: string;
|
|
978
|
+
}
|
|
979
|
+
|
|
980
|
+
export type WorkerPaneReconciliation =
|
|
981
|
+
/** The recorded pane is still there and still carries this run — nothing done. */
|
|
982
|
+
| { kind: "intact"; runId: string; paneId: string }
|
|
983
|
+
/** Herdr lost the pane (a restart); a new one now represents the same child. */
|
|
984
|
+
| { kind: "reassociated"; runId: string; paneId: string; label: string }
|
|
985
|
+
/** No representation, and the reason. The run keeps working regardless. */
|
|
986
|
+
| { kind: "untracked"; runId: string; reason: string }
|
|
987
|
+
/** A conductor pane whose run is not live: handed back to Herdr. */
|
|
988
|
+
| { kind: "stale-released"; paneId: string; runId: string }
|
|
989
|
+
| { kind: "stale-release-failed"; paneId: string; runId: string; reason: string };
|
|
990
|
+
|
|
991
|
+
/**
|
|
992
|
+
* Make Herdr's conductor-owned panes agree with the live run set (#841).
|
|
993
|
+
*
|
|
994
|
+
* Idempotent by construction: a second pass over an already-reconciled fleet
|
|
995
|
+
* returns `intact` for every live run and finds no stale panes, so repeated
|
|
996
|
+
* daemon or Herdr restarts converge rather than accumulate.
|
|
997
|
+
*
|
|
998
|
+
* Three rules, and each exists to refuse a specific way this goes wrong:
|
|
999
|
+
*
|
|
1000
|
+
* - **A live run's pane is re-created, never duplicated.** Re-association is
|
|
1001
|
+
* keyed on the run id Herdr itself reports, so a pane that is still there is
|
|
1002
|
+
* left alone. Only a run whose pane Herdr no longer has gets a new one.
|
|
1003
|
+
* - **Cleanup is by exact identity, never by name or age.** A pane is stale only
|
|
1004
|
+
* when the run id it carries is absent from the live set. A worker's own pane
|
|
1005
|
+
* can therefore never be released while its run is live, whatever it is called
|
|
1006
|
+
* and however old it is.
|
|
1007
|
+
* - **Nothing here can stop a worker.** The only mutation is `release-agent`,
|
|
1008
|
+
* which hands lifecycle authority back to Herdr; the authoritative child is
|
|
1009
|
+
* never signalled, and its pane is never closed.
|
|
1010
|
+
*/
|
|
1011
|
+
export function reconcileWorkerPanes(
|
|
1012
|
+
live: readonly LiveWorkerPane[],
|
|
1013
|
+
deps: WorkerPaneDeps = {},
|
|
1014
|
+
): { ok: true; outcomes: WorkerPaneReconciliation[] } | { ok: false; reason: string } {
|
|
1015
|
+
const listed = listWorkerPanes(deps);
|
|
1016
|
+
if (!listed.ok) return { ok: false, reason: listed.reason };
|
|
1017
|
+
const byRun = new Map(listed.panes.map((pane) => [pane.runId, pane]));
|
|
1018
|
+
const liveIds = new Set(live.map((worker) => worker.runId));
|
|
1019
|
+
const outcomes: WorkerPaneReconciliation[] = [];
|
|
1020
|
+
|
|
1021
|
+
for (const worker of live) {
|
|
1022
|
+
const held = byRun.get(worker.runId);
|
|
1023
|
+
if (held !== undefined) {
|
|
1024
|
+
outcomes.push({ kind: "intact", runId: worker.runId, paneId: held.paneId });
|
|
1025
|
+
continue;
|
|
1026
|
+
}
|
|
1027
|
+
// Herdr does not have this run's pane. Only the pid makes a replacement
|
|
1028
|
+
// honest: the pane represents an exact child, so without one there is
|
|
1029
|
+
// nothing to represent and inventing a pane would be the silent claim this
|
|
1030
|
+
// whole surface exists to avoid.
|
|
1031
|
+
if (worker.pid === undefined) {
|
|
1032
|
+
outcomes.push({
|
|
1033
|
+
kind: "untracked",
|
|
1034
|
+
runId: worker.runId,
|
|
1035
|
+
reason: "no session-host pid was ever recorded for this run",
|
|
1036
|
+
});
|
|
1037
|
+
continue;
|
|
1038
|
+
}
|
|
1039
|
+
const opened = openWorkerPane(
|
|
1040
|
+
{
|
|
1041
|
+
project: worker.project,
|
|
1042
|
+
issue: worker.issue,
|
|
1043
|
+
attempt: worker.attempt,
|
|
1044
|
+
runId: worker.runId,
|
|
1045
|
+
pid: worker.pid,
|
|
1046
|
+
...(worker.sessionFile === undefined ? {} : { sessionFile: worker.sessionFile }),
|
|
1047
|
+
},
|
|
1048
|
+
deps,
|
|
1049
|
+
);
|
|
1050
|
+
outcomes.push(
|
|
1051
|
+
opened.kind === "tracked"
|
|
1052
|
+
? { kind: "reassociated", runId: worker.runId, paneId: opened.paneId, label: opened.label }
|
|
1053
|
+
: { kind: "untracked", runId: worker.runId, reason: opened.reason },
|
|
1054
|
+
);
|
|
1055
|
+
}
|
|
1056
|
+
|
|
1057
|
+
for (const pane of listed.panes) {
|
|
1058
|
+
if (liveIds.has(pane.runId)) continue;
|
|
1059
|
+
const released = releaseWorkerPane(pane.paneId, pane.label ?? "", deps);
|
|
1060
|
+
outcomes.push(
|
|
1061
|
+
released.ok
|
|
1062
|
+
? { kind: "stale-released", paneId: pane.paneId, runId: pane.runId }
|
|
1063
|
+
: { kind: "stale-release-failed", paneId: pane.paneId, runId: pane.runId, reason: released.reason },
|
|
1064
|
+
);
|
|
1065
|
+
}
|
|
1066
|
+
return { ok: true, outcomes };
|
|
1067
|
+
}
|
|
1068
|
+
|
|
1069
|
+
function firstLine(text: string): string {
|
|
1070
|
+
return text.split("\n", 1)[0]?.trim() ?? "";
|
|
1071
|
+
}
|
|
1072
|
+
|
|
720
1073
|
/** The kill syscall, injectable so error mapping is testable without one. */
|
|
721
1074
|
export type KillFn = (pid: number, sig: NodeJS.Signals | 0) => void;
|
|
722
1075
|
|
|
@@ -961,9 +1314,7 @@ export async function stopConductorPane(
|
|
|
961
1314
|
|
|
962
1315
|
async function herdrAgentList(deps: PaneStopDeps): Promise<HerdrAgent[]> {
|
|
963
1316
|
const bin = deps.herdrBin ?? "herdr";
|
|
964
|
-
const session =
|
|
965
|
-
deps.herdrSession ??
|
|
966
|
-
resolveHerdrSessionWithBridge({ env: process.env, herdrBin: bin });
|
|
1317
|
+
const session = deps.herdrSession ?? resolveHerdrSession(process.env);
|
|
967
1318
|
const res = spawnSync(bin, ["--session", session, "agent", "list"], {
|
|
968
1319
|
encoding: "utf8",
|
|
969
1320
|
timeout: 8_000,
|
|
@@ -1064,9 +1415,7 @@ export function parseHerdrProcessInfo(stdout: string, paneId: string): ProcessIn
|
|
|
1064
1415
|
|
|
1065
1416
|
async function herdrOmpForegroundPids(paneId: string, deps: PaneStopDeps): Promise<number[]> {
|
|
1066
1417
|
const bin = deps.herdrBin ?? "herdr";
|
|
1067
|
-
const session =
|
|
1068
|
-
deps.herdrSession ??
|
|
1069
|
-
resolveHerdrSessionWithBridge({ env: process.env, herdrBin: bin });
|
|
1418
|
+
const session = deps.herdrSession ?? resolveHerdrSession(process.env);
|
|
1070
1419
|
const res = spawnSync(bin, ["--session", session, "pane", "process-info", "--pane", paneId], {
|
|
1071
1420
|
encoding: "utf8",
|
|
1072
1421
|
timeout: 8_000,
|
|
@@ -1642,13 +1991,22 @@ function briefStatusLine(project: ProjectConfig): string | undefined {
|
|
|
1642
1991
|
}
|
|
1643
1992
|
|
|
1644
1993
|
/**
|
|
1645
|
-
*
|
|
1646
|
-
* none.
|
|
1994
|
+
* Failure classes, in two blocks that mean different things (#132).
|
|
1647
1995
|
*
|
|
1648
1996
|
* Reported as classes rather than as row states because a row state is not an
|
|
1649
1997
|
* issue state: the FAILED column counted four completed issues on this fleet
|
|
1650
|
-
* while the genuinely stuck ones were invisible (#109).
|
|
1651
|
-
*
|
|
1998
|
+
* while the genuinely stuck ones were invisible (#109).
|
|
1999
|
+
*
|
|
2000
|
+
* That defect then reappeared one level up. A single "unrecovered" list keyed on
|
|
2001
|
+
* `recoveredAt IS NULL` swept in the two recorded-only actions, which never
|
|
2002
|
+
* stamp it — so this fleet's status read `returned-for-revision 44`, growing by
|
|
2003
|
+
* one on every review return, while every class an operator could actually act
|
|
2004
|
+
* on sat at zero and invisible beneath it. A count nobody can act on, printed
|
|
2005
|
+
* where the actionable ones go, is the same lie in a new column.
|
|
2006
|
+
*
|
|
2007
|
+
* So: `awaiting recovery` is what the sweep will pick up, and it is omitted when
|
|
2008
|
+
* empty because empty is the healthy answer. `classified, no action` is the
|
|
2009
|
+
* signal — review returns and holds — named as such, never as a backlog.
|
|
1652
2010
|
*/
|
|
1653
2011
|
function failureClassBlock(projectName: string): string | undefined {
|
|
1654
2012
|
const path = dbPath();
|
|
@@ -1656,12 +2014,23 @@ function failureClassBlock(projectName: string): string | undefined {
|
|
|
1656
2014
|
let store: Store | undefined;
|
|
1657
2015
|
try {
|
|
1658
2016
|
store = openStore(path);
|
|
1659
|
-
const
|
|
1660
|
-
|
|
1661
|
-
return
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
2017
|
+
const awaiting = store.failureClassCounts(projectName);
|
|
2018
|
+
const recorded = store.recordedOnlyClassCounts(projectName);
|
|
2019
|
+
if (awaiting.length === 0 && recorded.length === 0) return undefined;
|
|
2020
|
+
const lines: string[] = [];
|
|
2021
|
+
if (awaiting.length > 0) {
|
|
2022
|
+
lines.push(
|
|
2023
|
+
"failure classes (awaiting recovery)",
|
|
2024
|
+
...awaiting.map((c) => ` ${c.cls.padEnd(24)}${c.n}`),
|
|
2025
|
+
);
|
|
2026
|
+
}
|
|
2027
|
+
if (recorded.length > 0) {
|
|
2028
|
+
lines.push(
|
|
2029
|
+
"failure classes (classified, no action by design)",
|
|
2030
|
+
...recorded.map((c) => ` ${c.cls.padEnd(24)}${c.n}`),
|
|
2031
|
+
);
|
|
2032
|
+
}
|
|
2033
|
+
return lines.join("\n");
|
|
1665
2034
|
} catch {
|
|
1666
2035
|
return undefined;
|
|
1667
2036
|
} finally {
|
|
@@ -1901,6 +2270,23 @@ function makeChallengeCode(): string {
|
|
|
1901
2270
|
return `FLEET-${hex}`;
|
|
1902
2271
|
}
|
|
1903
2272
|
|
|
2273
|
+
/**
|
|
2274
|
+
* mm:ss for a window whose whole length is five minutes. Deliberately not
|
|
2275
|
+
* `formatDownDuration`, which rounds to whole minutes because it reports
|
|
2276
|
+
* hours-to-days outages: rounded to the minute, the last 30 seconds of this
|
|
2277
|
+
* window would read "5m of 5m left" while the wait was nearly over, which is
|
|
2278
|
+
* the exact ambiguity the progress lines exist to remove (#861).
|
|
2279
|
+
*/
|
|
2280
|
+
function armClock(ms: number): string {
|
|
2281
|
+
const total = Math.max(0, Math.round(ms / 1000));
|
|
2282
|
+
const minutes = Math.floor(total / 60);
|
|
2283
|
+
const seconds = total % 60;
|
|
2284
|
+
return minutes === 0 ? `${seconds}s` : `${minutes}m${String(seconds).padStart(2, "0")}s`;
|
|
2285
|
+
}
|
|
2286
|
+
|
|
2287
|
+
/** How often the wait says it is still waiting. Six lines across the window. */
|
|
2288
|
+
const ARM_PROGRESS_INTERVAL_MS = 30_000;
|
|
2289
|
+
|
|
1904
2290
|
/**
|
|
1905
2291
|
* Polls the acknowledgement record for one exact challenge id until the
|
|
1906
2292
|
* orchestrator's inbound adapter writes it or the deadline passes (#614).
|
|
@@ -1910,6 +2296,14 @@ function makeChallengeCode(): string {
|
|
|
1910
2296
|
* orchestrator process. Only a record naming this exact id satisfies the
|
|
1911
2297
|
* wait — an acknowledgement cut for a replaced challenge is inert here by
|
|
1912
2298
|
* construction, and no transcript anywhere is opened.
|
|
2299
|
+
*
|
|
2300
|
+
* It also *says* it is waiting (#861). Measured 2026-08-21: an arm proof sat
|
|
2301
|
+
* silent for five minutes and was reported as a hung setup — the process was
|
|
2302
|
+
* healthy and the operator had no way to tell. A silent five-minute wait
|
|
2303
|
+
* inside a fence that holds dispatch is indistinguishable from a dead one, so
|
|
2304
|
+
* the elapsed/remaining line lands every {@link ARM_PROGRESS_INTERVAL_MS}
|
|
2305
|
+
* regardless of the (much shorter) poll cadence, and the terminal outcome is
|
|
2306
|
+
* always printed.
|
|
1913
2307
|
*/
|
|
1914
2308
|
async function waitForArmAcknowledgement(
|
|
1915
2309
|
challengeId: string,
|
|
@@ -1918,10 +2312,26 @@ async function waitForArmAcknowledgement(
|
|
|
1918
2312
|
): Promise<boolean> {
|
|
1919
2313
|
const now = deps.now ?? Date.now;
|
|
1920
2314
|
const sleep = deps.sleep ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)));
|
|
1921
|
-
const
|
|
2315
|
+
const report = deps.progress;
|
|
2316
|
+
const startedAt = now();
|
|
2317
|
+
const deadline = startedAt + timeoutMs;
|
|
2318
|
+
let nextReportAt = startedAt + ARM_PROGRESS_INTERVAL_MS;
|
|
1922
2319
|
for (;;) {
|
|
1923
|
-
if (readArmAcknowledgement(challengeId) !== undefined)
|
|
1924
|
-
|
|
2320
|
+
if (readArmAcknowledgement(challengeId) !== undefined) {
|
|
2321
|
+
report?.(`arm: reply acknowledged after ${armClock(now() - startedAt)} — the fleet is armed.`);
|
|
2322
|
+
return true;
|
|
2323
|
+
}
|
|
2324
|
+
const at = now();
|
|
2325
|
+
if (at >= deadline) return false;
|
|
2326
|
+
if (report !== undefined && at >= nextReportAt) {
|
|
2327
|
+
report(
|
|
2328
|
+
`arm: still waiting for the reply — ${armClock(at - startedAt)} elapsed, ` +
|
|
2329
|
+
`${armClock(deadline - at)} left. Nothing is stuck: reply in the Telegram chat with the code.`,
|
|
2330
|
+
);
|
|
2331
|
+
// Anchored to the clock, not to this pass, so a slow pass cannot make the
|
|
2332
|
+
// cadence drift into silence.
|
|
2333
|
+
while (nextReportAt <= at) nextReportAt += ARM_PROGRESS_INTERVAL_MS;
|
|
2334
|
+
}
|
|
1925
2335
|
await sleep(5_000);
|
|
1926
2336
|
}
|
|
1927
2337
|
}
|
|
@@ -2057,7 +2467,7 @@ function probeOmpPane(
|
|
|
2057
2467
|
}
|
|
2058
2468
|
const agentName =
|
|
2059
2469
|
tick.kind === "ok" ? (tick.config.agentName ?? DEFAULT_FLEET_AGENT_NAME) : DEFAULT_FLEET_AGENT_NAME;
|
|
2060
|
-
const session =
|
|
2470
|
+
const session = resolveHerdrSession(process.env);
|
|
2061
2471
|
try {
|
|
2062
2472
|
const res = spawnSync("herdr", ["--session", session, "agent", "list"], {
|
|
2063
2473
|
encoding: "utf8",
|