omp-conductor 0.18.2 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +105 -40
- package/REFERENCE.md +865 -30
- package/package.json +1 -1
- package/schema/config.schema.json +26 -0
- package/src/admission.ts +212 -26
- package/src/ask.ts +288 -1
- package/src/briefs/orchestrator.md +6 -5
- package/src/cli.ts +5 -1
- package/src/command-help.ts +9 -1
- package/src/command-manifest.ts +36 -3
- package/src/commands/arm.ts +5 -1
- package/src/commands/context.ts +2 -0
- package/src/commands/message.ts +26 -2
- package/src/commands/reconcile-units.ts +104 -0
- package/src/commands/release-composition.ts +232 -0
- package/src/commands/resume.ts +2 -27
- package/src/commands/setup.ts +101 -16
- package/src/commands/stats.ts +11 -30
- package/src/commands/tail.ts +31 -1
- package/src/commands/upgrade.ts +20 -3
- package/src/commands/verb.ts +2 -1
- package/src/config-schema.ts +19 -0
- package/src/config.ts +80 -0
- package/src/credential-class.ts +366 -0
- package/src/daemon.ts +1218 -288
- package/src/dashboard/app.js +504 -2
- package/src/dashboard/controls.ts +336 -0
- package/src/dashboard/index.html +30 -0
- package/src/dashboard/server.ts +271 -30
- package/src/dashboard/style.css +116 -0
- package/src/dashboard/transcript.ts +173 -0
- package/src/doctor.ts +377 -20
- package/src/failure-class.ts +59 -0
- package/src/fleet.ts +497 -15
- package/src/host.ts +6 -130
- package/src/omp.ts +29 -0
- package/src/orchestrator-tick.ts +343 -88
- package/src/pause.ts +233 -0
- package/src/settlement.ts +159 -2
- package/src/setup-answers.ts +97 -0
- package/src/setup-host.ts +321 -1155
- package/src/setup-install.ts +204 -27
- package/src/setup-wizard.ts +111 -50
- package/src/setup.ts +33 -0
- package/src/spend-telemetry.ts +117 -0
- package/src/stats.ts +35 -0
- package/src/status-render.ts +348 -19
- package/src/store.ts +1229 -55
- package/src/telegram-freshness.ts +269 -0
- package/src/to-spec.ts +27 -0
- package/src/types.ts +697 -4
- package/src/unblock.ts +22 -0
- package/src/unit-reconcile.ts +303 -0
- package/src/upgrade-verify.ts +8 -1
- package/src/upgrade.ts +299 -12
- package/src/verbs/actions.ts +124 -10
- package/src/verbs/protocol.ts +70 -2
- package/src/verbs/server.ts +447 -8
- package/src/wake.ts +48 -0
- package/src/worker.ts +403 -3
package/src/fleet.ts
CHANGED
|
@@ -343,6 +343,14 @@ export interface ArmDeps {
|
|
|
343
343
|
pidAlive?: (pid: number) => boolean;
|
|
344
344
|
lockPidAlive?: (pid: number) => boolean;
|
|
345
345
|
lockFresh?: (mtimeMs: number) => boolean;
|
|
346
|
+
/**
|
|
347
|
+
* Where the pending-proof heartbeat is written (#861). The challenge proof
|
|
348
|
+
* waits up to five minutes on a human, inside a fence that holds dispatch:
|
|
349
|
+
* without this, that wait is silent and a healthy process is indistinguishable
|
|
350
|
+
* from a dead one. Absent means no reporting — the callers that have a surface
|
|
351
|
+
* (the CLI, the wizard) pass theirs.
|
|
352
|
+
*/
|
|
353
|
+
progress?: (line: string) => void;
|
|
346
354
|
}
|
|
347
355
|
|
|
348
356
|
export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promise<ArmResult> {
|
|
@@ -484,6 +492,14 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
484
492
|
`arm: outbound sendMessage failed — NOT armed: ${err instanceof Error ? err.message : String(err)}`,
|
|
485
493
|
);
|
|
486
494
|
}
|
|
495
|
+
// What is being waited on, before the wait starts: the window, the chat the
|
|
496
|
+
// reply has to land in, and the fact that dispatch is held until it does
|
|
497
|
+
// (#861). One line, so a five-minute wait opens with an explanation rather
|
|
498
|
+
// than with silence.
|
|
499
|
+
deps.progress?.(
|
|
500
|
+
`arm: challenge sent to ${channel.owner}${sendTopic === undefined ? "" : ` (topic ${sendTopic})`} — ` +
|
|
501
|
+
`waiting up to ${armClock(timeoutMs)} for the reply. Dispatch stays held until it arrives.`,
|
|
502
|
+
);
|
|
487
503
|
|
|
488
504
|
// Wait for the orchestrator's own acknowledgement — conductor state written
|
|
489
505
|
// by the inbound user-turn adapter when the real reply lands (#614). No
|
|
@@ -553,10 +569,14 @@ export interface HaltWithPaneResult extends HaltResult {
|
|
|
553
569
|
export function hold(
|
|
554
570
|
projectName?: string,
|
|
555
571
|
source: string = "hold",
|
|
556
|
-
opts: { keepTicks?: boolean } = {},
|
|
572
|
+
opts: { keepTicks?: boolean; reason?: string } = {},
|
|
557
573
|
): HoldResult {
|
|
558
574
|
const wasPaused = isPaused(projectName);
|
|
559
|
-
|
|
575
|
+
// `source` is a space-free token by contract (`pauseSourceToken`) — the fence
|
|
576
|
+
// matches on it, and whitespace there makes a pause the fence cannot prove
|
|
577
|
+
// (#552). The human sentence belongs in `reason=`, which is why this is a
|
|
578
|
+
// separate field rather than something a caller concatenates into `source`.
|
|
579
|
+
setPaused(true, { source, ...(opts.reason === undefined ? {} : { reason: opts.reason }) }, projectName);
|
|
560
580
|
if (opts.keepTicks === true) return { wasPaused };
|
|
561
581
|
return { wasPaused, disarmed: disarmTicks(projectName) };
|
|
562
582
|
}
|
|
@@ -717,6 +737,407 @@ export interface HerdrAgent {
|
|
|
717
737
|
sessionPath?: string;
|
|
718
738
|
}
|
|
719
739
|
|
|
740
|
+
/**
|
|
741
|
+
* The Herdr representation of one live conductor worker (#840).
|
|
742
|
+
*
|
|
743
|
+
* ## Why this shape, and not `herdr agent start --kind omp`
|
|
744
|
+
*
|
|
745
|
+
* The authoritative worker is the `session-host` child conductor spawns itself:
|
|
746
|
+
* it holds the typed control socket, the mediated verb socket, the worktree, the
|
|
747
|
+
* transcript, the accounting and the output-schema settlement. Starting a second
|
|
748
|
+
* OMP process in a pane would look right in the workspace and be authoritative
|
|
749
|
+
* for nothing — the named silent fake of #840 — so nothing here starts an agent.
|
|
750
|
+
*
|
|
751
|
+
* Herdr already has the verb for exactly this case: an *external* supervisor
|
|
752
|
+
* reports lifecycle onto a pane it owns (`pane report-agent`,
|
|
753
|
+
* `pane report-agent-session`, `pane release-agent`). So conductor keeps the
|
|
754
|
+
* child it already spawned, and reports that child's identity and state onto a
|
|
755
|
+
* pane whose only job is display.
|
|
756
|
+
*
|
|
757
|
+
* ## What is displayed, and why that is not "a transcript-tail pane"
|
|
758
|
+
*
|
|
759
|
+
* The pane runs `omp-conductor tail <issue>`, which is the repository's existing
|
|
760
|
+
* read-only follower. The distinction #840 draws is about *authority*, not about
|
|
761
|
+
* pixels: nothing here parses that output, and no run state, settlement, turn,
|
|
762
|
+
* spend or model fact is derived from it. Identity comes from the exact child
|
|
763
|
+
* pid handed over by the spawn path, and state comes from the typed events the
|
|
764
|
+
* proxy already emits.
|
|
765
|
+
*
|
|
766
|
+
* ## Observation-only is structural, not a gate
|
|
767
|
+
*
|
|
768
|
+
* The authoritative child is spawned with `stdin: "ignore"` — there is no file
|
|
769
|
+
* descriptor for a keystroke to travel down, from a pane or anywhere else. The
|
|
770
|
+
* pane's own process is a follower whose stdin reaches only itself. So "input
|
|
771
|
+
* cannot reach OMP, change its prompt, invoke a verb, or settle the run" is a
|
|
772
|
+
* property of the process tree rather than a check that could be forgotten.
|
|
773
|
+
*/
|
|
774
|
+
export interface WorkerPaneIdentity {
|
|
775
|
+
project: string;
|
|
776
|
+
issue: number;
|
|
777
|
+
attempt: number;
|
|
778
|
+
/** The run row's id — the durable identity the pane is reported under. */
|
|
779
|
+
runId: string;
|
|
780
|
+
/** The exact `session-host` pid, from the spawn path's own `onSpawn`. */
|
|
781
|
+
pid: number;
|
|
782
|
+
/** The transcript the child opened, when it has already reported one. */
|
|
783
|
+
sessionFile?: string;
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
/** A tracked pane, or the explicit reason there is none. Never a silent claim. */
|
|
787
|
+
export type WorkerPaneOutcome =
|
|
788
|
+
| { kind: "tracked"; paneId: string; label: string; pid: number }
|
|
789
|
+
| { kind: "unavailable"; reason: string };
|
|
790
|
+
|
|
791
|
+
/** One `herdr` invocation, injected so every path is testable with no terminal. */
|
|
792
|
+
export type HerdrRun = (args: readonly string[]) => { ok: boolean; stdout: string; stderr: string };
|
|
793
|
+
|
|
794
|
+
export interface WorkerPaneDeps {
|
|
795
|
+
run?: HerdrRun;
|
|
796
|
+
session?: string;
|
|
797
|
+
/** The follower command the pane displays; the CLI by default. */
|
|
798
|
+
viewer?: (identity: WorkerPaneIdentity) => readonly string[];
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
/**
|
|
802
|
+
* The pane's unique, human-readable name (#840: "uniquely named").
|
|
803
|
+
*
|
|
804
|
+
* Keyed by run id rather than issue: an issue can have several attempts, and two
|
|
805
|
+
* attempts must never resolve to one pane. The issue and attempt ride along
|
|
806
|
+
* because the operator reads this label in a workspace, not a database.
|
|
807
|
+
*/
|
|
808
|
+
/**
|
|
809
|
+
* The external-supervisor source every conductor pane report carries.
|
|
810
|
+
*
|
|
811
|
+
* This, not the label, is what proves a pane is conductor's: it is written by
|
|
812
|
+
* `openWorkerPane` and read back by {@link listWorkerPanes}, so the two halves
|
|
813
|
+
* of reconciliation cannot come to disagree about what "ours" means.
|
|
814
|
+
*/
|
|
815
|
+
export const WORKER_PANE_SOURCE = "omp-conductor";
|
|
816
|
+
|
|
817
|
+
export function workerPaneLabel(identity: WorkerPaneIdentity): string {
|
|
818
|
+
return `worker-${identity.project}-${identity.issue}-a${identity.attempt}-${identity.runId.slice(0, 8)}`;
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
const realHerdrRun: HerdrRun = (args) => {
|
|
822
|
+
const res = spawnSync("herdr", [...args], { encoding: "utf8", timeout: 8_000, env: process.env });
|
|
823
|
+
return {
|
|
824
|
+
ok: !res.error && res.status === 0,
|
|
825
|
+
stdout: res.stdout ?? "",
|
|
826
|
+
stderr: res.stderr ?? (res.error === undefined ? "" : String(res.error.message)),
|
|
827
|
+
};
|
|
828
|
+
};
|
|
829
|
+
|
|
830
|
+
/** `omp-conductor tail` — read-only by construction, and never parsed here. */
|
|
831
|
+
function defaultViewer(identity: WorkerPaneIdentity): readonly string[] {
|
|
832
|
+
return ["omp-conductor", "tail", String(identity.issue), "--project", identity.project];
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
/**
|
|
836
|
+
* Create the pane, run the follower in it, and report the child's identity and
|
|
837
|
+
* initial state — or say exactly why it could not.
|
|
838
|
+
*
|
|
839
|
+
* Every step is checked, and the first failure returns `unavailable` with the
|
|
840
|
+
* reason: a partially established representation (a pane with no agent identity,
|
|
841
|
+
* say) is worse than none, because it is a pane the operator would read as a
|
|
842
|
+
* tracked worker. What this deliberately does NOT do is decide what a failure
|
|
843
|
+
* means for the launch — failing closed versus running degraded is #841's
|
|
844
|
+
* policy, and inventing it here would pre-empt it.
|
|
845
|
+
*/
|
|
846
|
+
export function openWorkerPane(identity: WorkerPaneIdentity, deps: WorkerPaneDeps = {}): WorkerPaneOutcome {
|
|
847
|
+
const run = deps.run ?? realHerdrRun;
|
|
848
|
+
const session = deps.session ?? resolveHerdrSession();
|
|
849
|
+
const label = workerPaneLabel(identity);
|
|
850
|
+
const base = ["--session", session, "pane"];
|
|
851
|
+
|
|
852
|
+
const split = run([...base, "split", "--direction", "down", "--ratio", "0.3"]);
|
|
853
|
+
if (!split.ok) {
|
|
854
|
+
return { kind: "unavailable", reason: `herdr pane split failed: ${firstLine(split.stderr) || "no output"}` };
|
|
855
|
+
}
|
|
856
|
+
const paneId = firstLine(split.stdout);
|
|
857
|
+
if (paneId === "") {
|
|
858
|
+
return { kind: "unavailable", reason: "herdr pane split reported no pane id" };
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
const named = run([...base, "rename", paneId, label]);
|
|
862
|
+
if (!named.ok) {
|
|
863
|
+
return { kind: "unavailable", reason: `herdr pane rename failed: ${firstLine(named.stderr) || "no output"}` };
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
// Identity before display: the pane must be attributable to this exact run
|
|
867
|
+
// before it shows anything, so a pane that appears is never a pane nobody can
|
|
868
|
+
// trace back to a worker.
|
|
869
|
+
const identified = run([
|
|
870
|
+
...base,
|
|
871
|
+
"report-agent-session",
|
|
872
|
+
paneId,
|
|
873
|
+
"--source",
|
|
874
|
+
WORKER_PANE_SOURCE,
|
|
875
|
+
"--agent",
|
|
876
|
+
label,
|
|
877
|
+
"--agent-session-id",
|
|
878
|
+
identity.runId,
|
|
879
|
+
...(identity.sessionFile === undefined ? [] : ["--agent-session-path", identity.sessionFile]),
|
|
880
|
+
"--session-start-source",
|
|
881
|
+
WORKER_PANE_SOURCE,
|
|
882
|
+
]);
|
|
883
|
+
if (!identified.ok) {
|
|
884
|
+
return {
|
|
885
|
+
kind: "unavailable",
|
|
886
|
+
reason: `herdr pane report-agent-session failed: ${firstLine(identified.stderr) || "no output"}`,
|
|
887
|
+
};
|
|
888
|
+
}
|
|
889
|
+
|
|
890
|
+
const started = run([...base, "run", paneId, ...deps.viewer?.(identity) ?? defaultViewer(identity)]);
|
|
891
|
+
if (!started.ok) {
|
|
892
|
+
return { kind: "unavailable", reason: `herdr pane run failed: ${firstLine(started.stderr) || "no output"}` };
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
// A worker that has just spawned is working by definition. The ongoing
|
|
896
|
+
// projection of turn/pause/blocked transitions is #842's, from the same typed
|
|
897
|
+
// events — never from the pane's output.
|
|
898
|
+
const reported = reportWorkerPaneState(paneId, label, "working", { run, session });
|
|
899
|
+
if (!reported.ok) {
|
|
900
|
+
return { kind: "unavailable", reason: reported.reason };
|
|
901
|
+
}
|
|
902
|
+
return { kind: "tracked", paneId, label, pid: identity.pid };
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
/** Report one lifecycle state for a tracked pane. Monotonic `seq` is the caller's. */
|
|
906
|
+
export function reportWorkerPaneState(
|
|
907
|
+
paneId: string,
|
|
908
|
+
label: string,
|
|
909
|
+
state: "working" | "idle" | "blocked" | "unknown",
|
|
910
|
+
deps: { run?: HerdrRun; session?: string; seq?: number; message?: string } = {},
|
|
911
|
+
): { ok: true } | { ok: false; reason: string } {
|
|
912
|
+
const run = deps.run ?? realHerdrRun;
|
|
913
|
+
const session = deps.session ?? resolveHerdrSession();
|
|
914
|
+
const res = run([
|
|
915
|
+
"--session",
|
|
916
|
+
session,
|
|
917
|
+
"pane",
|
|
918
|
+
"report-agent",
|
|
919
|
+
paneId,
|
|
920
|
+
"--source",
|
|
921
|
+
WORKER_PANE_SOURCE,
|
|
922
|
+
"--agent",
|
|
923
|
+
label,
|
|
924
|
+
"--state",
|
|
925
|
+
state,
|
|
926
|
+
...(deps.seq === undefined ? [] : ["--seq", String(deps.seq)]),
|
|
927
|
+
...(deps.message === undefined ? [] : ["--message", deps.message]),
|
|
928
|
+
]);
|
|
929
|
+
return res.ok
|
|
930
|
+
? { ok: true }
|
|
931
|
+
: { ok: false, reason: `herdr pane report-agent failed: ${firstLine(res.stderr) || "no output"}` };
|
|
932
|
+
}
|
|
933
|
+
|
|
934
|
+
/**
|
|
935
|
+
* Hand lifecycle authority back when the worker is gone.
|
|
936
|
+
*
|
|
937
|
+
* Release, never close: whether a settled worker's pane is closed, kept, or
|
|
938
|
+
* retained for N settlements is #841's documented policy, and a slice that
|
|
939
|
+
* closed panes here would decide it by accident. Releasing says only "conductor
|
|
940
|
+
* no longer speaks for this agent", which is exactly what is true.
|
|
941
|
+
*/
|
|
942
|
+
export function releaseWorkerPane(
|
|
943
|
+
paneId: string,
|
|
944
|
+
label: string,
|
|
945
|
+
deps: { run?: HerdrRun; session?: string; seq?: number } = {},
|
|
946
|
+
): { ok: true } | { ok: false; reason: string } {
|
|
947
|
+
const run = deps.run ?? realHerdrRun;
|
|
948
|
+
const session = deps.session ?? resolveHerdrSession();
|
|
949
|
+
const res = run([
|
|
950
|
+
"--session",
|
|
951
|
+
session,
|
|
952
|
+
"pane",
|
|
953
|
+
"release-agent",
|
|
954
|
+
paneId,
|
|
955
|
+
"--source",
|
|
956
|
+
WORKER_PANE_SOURCE,
|
|
957
|
+
"--agent",
|
|
958
|
+
label,
|
|
959
|
+
...(deps.seq === undefined ? [] : ["--seq", String(deps.seq)]),
|
|
960
|
+
]);
|
|
961
|
+
return res.ok
|
|
962
|
+
? { ok: true }
|
|
963
|
+
: { ok: false, reason: `herdr pane release-agent failed: ${firstLine(res.stderr) || "no output"}` };
|
|
964
|
+
}
|
|
965
|
+
|
|
966
|
+
/**
|
|
967
|
+
* Hand back the pane a dead worker left behind (#842).
|
|
968
|
+
*
|
|
969
|
+
* Called for every run a restart reaps. The recorded pid is deliberately NOT
|
|
970
|
+
* consulted for liveness: a `session-host` child dies with the daemon that owned
|
|
971
|
+
* its verb socket, so an orphaned row's worker is gone whatever pid it carries —
|
|
972
|
+
* and pids are reused, so checking one is how a stranger's process comes to read
|
|
973
|
+
* as a live worker. A run with no recorded pane is a no-op, not a failure: it
|
|
974
|
+
* never had one to release.
|
|
975
|
+
*/
|
|
976
|
+
export function releaseOrphanedWorkerPane(
|
|
977
|
+
run: { paneId?: string; paneLabel?: string },
|
|
978
|
+
deps: { run?: HerdrRun; session?: string; seq?: number } = {},
|
|
979
|
+
): { kind: "none" } | { kind: "released"; paneId: string } | { kind: "failed"; paneId: string; reason: string } {
|
|
980
|
+
if (run.paneId === undefined || run.paneLabel === undefined) return { kind: "none" };
|
|
981
|
+
const released = releaseWorkerPane(run.paneId, run.paneLabel, deps);
|
|
982
|
+
return released.ok
|
|
983
|
+
? { kind: "released", paneId: run.paneId }
|
|
984
|
+
: { kind: "failed", paneId: run.paneId, reason: released.reason };
|
|
985
|
+
}
|
|
986
|
+
|
|
987
|
+
/**
|
|
988
|
+
* Every conductor-owned pane Herdr currently has, keyed by the run id it was
|
|
989
|
+
* reported under (#841).
|
|
990
|
+
*
|
|
991
|
+
* Read from `pane list`'s own agent-session record, never from a label pattern:
|
|
992
|
+
* the label is what a human reads, and matching on it is exactly how a stale
|
|
993
|
+
* lookalike (a renamed pane, a pane from a previous fleet) gets mistaken for a
|
|
994
|
+
* live worker. `source` proves conductor reported it; `value` is the run id.
|
|
995
|
+
*/
|
|
996
|
+
export function listWorkerPanes(
|
|
997
|
+
deps: { run?: HerdrRun; session?: string } = {},
|
|
998
|
+
): { ok: true; panes: { paneId: string; runId: string; label?: string }[] } | { ok: false; reason: string } {
|
|
999
|
+
const run = deps.run ?? realHerdrRun;
|
|
1000
|
+
const session = deps.session ?? resolveHerdrSession();
|
|
1001
|
+
const res = run(["--session", session, "pane", "list"]);
|
|
1002
|
+
if (!res.ok) {
|
|
1003
|
+
return { ok: false, reason: `herdr pane list failed: ${firstLine(res.stderr) || "no output"}` };
|
|
1004
|
+
}
|
|
1005
|
+
let parsed: unknown;
|
|
1006
|
+
try {
|
|
1007
|
+
parsed = JSON.parse(res.stdout);
|
|
1008
|
+
} catch (err) {
|
|
1009
|
+
return { ok: false, reason: `herdr pane list was unreadable: ${err instanceof Error ? err.message : String(err)}` };
|
|
1010
|
+
}
|
|
1011
|
+
const panes = (parsed as { result?: { panes?: unknown[] } }).result?.panes;
|
|
1012
|
+
if (!Array.isArray(panes)) return { ok: false, reason: "herdr pane list carried no pane array" };
|
|
1013
|
+
const owned: { paneId: string; runId: string; label?: string }[] = [];
|
|
1014
|
+
for (const pane of panes) {
|
|
1015
|
+
const p = pane as {
|
|
1016
|
+
pane_id?: unknown;
|
|
1017
|
+
agent?: unknown;
|
|
1018
|
+
agent_session?: { source?: unknown; value?: unknown };
|
|
1019
|
+
};
|
|
1020
|
+
if (typeof p.pane_id !== "string") continue;
|
|
1021
|
+
if (p.agent_session?.source !== WORKER_PANE_SOURCE) continue;
|
|
1022
|
+
const runId = p.agent_session.value;
|
|
1023
|
+
// Ours by source but carrying no run id: an identity nobody can resolve is
|
|
1024
|
+
// not an identity. Reported as a pane with an empty run id so the caller
|
|
1025
|
+
// treats it as stale rather than silently ignoring it.
|
|
1026
|
+
owned.push({
|
|
1027
|
+
paneId: p.pane_id,
|
|
1028
|
+
runId: typeof runId === "string" ? runId : "",
|
|
1029
|
+
...(typeof p.agent === "string" ? { label: p.agent } : {}),
|
|
1030
|
+
});
|
|
1031
|
+
}
|
|
1032
|
+
return { ok: true, panes: owned };
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
/** One live worker, as the reconciler needs to see it. */
|
|
1036
|
+
export interface LiveWorkerPane {
|
|
1037
|
+
runId: string;
|
|
1038
|
+
issue: number;
|
|
1039
|
+
attempt: number;
|
|
1040
|
+
project: string;
|
|
1041
|
+
/** The recorded session-host pid; absent when nothing ever reported one. */
|
|
1042
|
+
pid?: number;
|
|
1043
|
+
paneId?: string;
|
|
1044
|
+
paneLabel?: string;
|
|
1045
|
+
sessionFile?: string;
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
export type WorkerPaneReconciliation =
|
|
1049
|
+
/** The recorded pane is still there and still carries this run — nothing done. */
|
|
1050
|
+
| { kind: "intact"; runId: string; paneId: string }
|
|
1051
|
+
/** Herdr lost the pane (a restart); a new one now represents the same child. */
|
|
1052
|
+
| { kind: "reassociated"; runId: string; paneId: string; label: string }
|
|
1053
|
+
/** No representation, and the reason. The run keeps working regardless. */
|
|
1054
|
+
| { kind: "untracked"; runId: string; reason: string }
|
|
1055
|
+
/** A conductor pane whose run is not live: handed back to Herdr. */
|
|
1056
|
+
| { kind: "stale-released"; paneId: string; runId: string }
|
|
1057
|
+
| { kind: "stale-release-failed"; paneId: string; runId: string; reason: string };
|
|
1058
|
+
|
|
1059
|
+
/**
|
|
1060
|
+
* Make Herdr's conductor-owned panes agree with the live run set (#841).
|
|
1061
|
+
*
|
|
1062
|
+
* Idempotent by construction: a second pass over an already-reconciled fleet
|
|
1063
|
+
* returns `intact` for every live run and finds no stale panes, so repeated
|
|
1064
|
+
* daemon or Herdr restarts converge rather than accumulate.
|
|
1065
|
+
*
|
|
1066
|
+
* Three rules, and each exists to refuse a specific way this goes wrong:
|
|
1067
|
+
*
|
|
1068
|
+
* - **A live run's pane is re-created, never duplicated.** Re-association is
|
|
1069
|
+
* keyed on the run id Herdr itself reports, so a pane that is still there is
|
|
1070
|
+
* left alone. Only a run whose pane Herdr no longer has gets a new one.
|
|
1071
|
+
* - **Cleanup is by exact identity, never by name or age.** A pane is stale only
|
|
1072
|
+
* when the run id it carries is absent from the live set. A worker's own pane
|
|
1073
|
+
* can therefore never be released while its run is live, whatever it is called
|
|
1074
|
+
* and however old it is.
|
|
1075
|
+
* - **Nothing here can stop a worker.** The only mutation is `release-agent`,
|
|
1076
|
+
* which hands lifecycle authority back to Herdr; the authoritative child is
|
|
1077
|
+
* never signalled, and its pane is never closed.
|
|
1078
|
+
*/
|
|
1079
|
+
export function reconcileWorkerPanes(
|
|
1080
|
+
live: readonly LiveWorkerPane[],
|
|
1081
|
+
deps: WorkerPaneDeps = {},
|
|
1082
|
+
): { ok: true; outcomes: WorkerPaneReconciliation[] } | { ok: false; reason: string } {
|
|
1083
|
+
const listed = listWorkerPanes(deps);
|
|
1084
|
+
if (!listed.ok) return { ok: false, reason: listed.reason };
|
|
1085
|
+
const byRun = new Map(listed.panes.map((pane) => [pane.runId, pane]));
|
|
1086
|
+
const liveIds = new Set(live.map((worker) => worker.runId));
|
|
1087
|
+
const outcomes: WorkerPaneReconciliation[] = [];
|
|
1088
|
+
|
|
1089
|
+
for (const worker of live) {
|
|
1090
|
+
const held = byRun.get(worker.runId);
|
|
1091
|
+
if (held !== undefined) {
|
|
1092
|
+
outcomes.push({ kind: "intact", runId: worker.runId, paneId: held.paneId });
|
|
1093
|
+
continue;
|
|
1094
|
+
}
|
|
1095
|
+
// Herdr does not have this run's pane. Only the pid makes a replacement
|
|
1096
|
+
// honest: the pane represents an exact child, so without one there is
|
|
1097
|
+
// nothing to represent and inventing a pane would be the silent claim this
|
|
1098
|
+
// whole surface exists to avoid.
|
|
1099
|
+
if (worker.pid === undefined) {
|
|
1100
|
+
outcomes.push({
|
|
1101
|
+
kind: "untracked",
|
|
1102
|
+
runId: worker.runId,
|
|
1103
|
+
reason: "no session-host pid was ever recorded for this run",
|
|
1104
|
+
});
|
|
1105
|
+
continue;
|
|
1106
|
+
}
|
|
1107
|
+
const opened = openWorkerPane(
|
|
1108
|
+
{
|
|
1109
|
+
project: worker.project,
|
|
1110
|
+
issue: worker.issue,
|
|
1111
|
+
attempt: worker.attempt,
|
|
1112
|
+
runId: worker.runId,
|
|
1113
|
+
pid: worker.pid,
|
|
1114
|
+
...(worker.sessionFile === undefined ? {} : { sessionFile: worker.sessionFile }),
|
|
1115
|
+
},
|
|
1116
|
+
deps,
|
|
1117
|
+
);
|
|
1118
|
+
outcomes.push(
|
|
1119
|
+
opened.kind === "tracked"
|
|
1120
|
+
? { kind: "reassociated", runId: worker.runId, paneId: opened.paneId, label: opened.label }
|
|
1121
|
+
: { kind: "untracked", runId: worker.runId, reason: opened.reason },
|
|
1122
|
+
);
|
|
1123
|
+
}
|
|
1124
|
+
|
|
1125
|
+
for (const pane of listed.panes) {
|
|
1126
|
+
if (liveIds.has(pane.runId)) continue;
|
|
1127
|
+
const released = releaseWorkerPane(pane.paneId, pane.label ?? "", deps);
|
|
1128
|
+
outcomes.push(
|
|
1129
|
+
released.ok
|
|
1130
|
+
? { kind: "stale-released", paneId: pane.paneId, runId: pane.runId }
|
|
1131
|
+
: { kind: "stale-release-failed", paneId: pane.paneId, runId: pane.runId, reason: released.reason },
|
|
1132
|
+
);
|
|
1133
|
+
}
|
|
1134
|
+
return { ok: true, outcomes };
|
|
1135
|
+
}
|
|
1136
|
+
|
|
1137
|
+
function firstLine(text: string): string {
|
|
1138
|
+
return text.split("\n", 1)[0]?.trim() ?? "";
|
|
1139
|
+
}
|
|
1140
|
+
|
|
720
1141
|
/** The kill syscall, injectable so error mapping is testable without one. */
|
|
721
1142
|
export type KillFn = (pid: number, sig: NodeJS.Signals | 0) => void;
|
|
722
1143
|
|
|
@@ -1642,13 +2063,22 @@ function briefStatusLine(project: ProjectConfig): string | undefined {
|
|
|
1642
2063
|
}
|
|
1643
2064
|
|
|
1644
2065
|
/**
|
|
1645
|
-
*
|
|
1646
|
-
* none.
|
|
2066
|
+
* Failure classes, in two blocks that mean different things (#132).
|
|
1647
2067
|
*
|
|
1648
2068
|
* Reported as classes rather than as row states because a row state is not an
|
|
1649
2069
|
* issue state: the FAILED column counted four completed issues on this fleet
|
|
1650
|
-
* while the genuinely stuck ones were invisible (#109).
|
|
1651
|
-
*
|
|
2070
|
+
* while the genuinely stuck ones were invisible (#109).
|
|
2071
|
+
*
|
|
2072
|
+
* That defect then reappeared one level up. A single "unrecovered" list keyed on
|
|
2073
|
+
* `recoveredAt IS NULL` swept in the two recorded-only actions, which never
|
|
2074
|
+
* stamp it — so this fleet's status read `returned-for-revision 44`, growing by
|
|
2075
|
+
* one on every review return, while every class an operator could actually act
|
|
2076
|
+
* on sat at zero and invisible beneath it. A count nobody can act on, printed
|
|
2077
|
+
* where the actionable ones go, is the same lie in a new column.
|
|
2078
|
+
*
|
|
2079
|
+
* So: `awaiting recovery` is what the sweep will pick up, and it is omitted when
|
|
2080
|
+
* empty because empty is the healthy answer. `classified, no action` is the
|
|
2081
|
+
* signal — review returns and holds — named as such, never as a backlog.
|
|
1652
2082
|
*/
|
|
1653
2083
|
function failureClassBlock(projectName: string): string | undefined {
|
|
1654
2084
|
const path = dbPath();
|
|
@@ -1656,12 +2086,23 @@ function failureClassBlock(projectName: string): string | undefined {
|
|
|
1656
2086
|
let store: Store | undefined;
|
|
1657
2087
|
try {
|
|
1658
2088
|
store = openStore(path);
|
|
1659
|
-
const
|
|
1660
|
-
|
|
1661
|
-
return
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
2089
|
+
const awaiting = store.failureClassCounts(projectName);
|
|
2090
|
+
const recorded = store.recordedOnlyClassCounts(projectName);
|
|
2091
|
+
if (awaiting.length === 0 && recorded.length === 0) return undefined;
|
|
2092
|
+
const lines: string[] = [];
|
|
2093
|
+
if (awaiting.length > 0) {
|
|
2094
|
+
lines.push(
|
|
2095
|
+
"failure classes (awaiting recovery)",
|
|
2096
|
+
...awaiting.map((c) => ` ${c.cls.padEnd(24)}${c.n}`),
|
|
2097
|
+
);
|
|
2098
|
+
}
|
|
2099
|
+
if (recorded.length > 0) {
|
|
2100
|
+
lines.push(
|
|
2101
|
+
"failure classes (classified, no action by design)",
|
|
2102
|
+
...recorded.map((c) => ` ${c.cls.padEnd(24)}${c.n}`),
|
|
2103
|
+
);
|
|
2104
|
+
}
|
|
2105
|
+
return lines.join("\n");
|
|
1665
2106
|
} catch {
|
|
1666
2107
|
return undefined;
|
|
1667
2108
|
} finally {
|
|
@@ -1901,6 +2342,23 @@ function makeChallengeCode(): string {
|
|
|
1901
2342
|
return `FLEET-${hex}`;
|
|
1902
2343
|
}
|
|
1903
2344
|
|
|
2345
|
+
/**
|
|
2346
|
+
* mm:ss for a window whose whole length is five minutes. Deliberately not
|
|
2347
|
+
* `formatDownDuration`, which rounds to whole minutes because it reports
|
|
2348
|
+
* hours-to-days outages: rounded to the minute, the last 30 seconds of this
|
|
2349
|
+
* window would read "5m of 5m left" while the wait was nearly over, which is
|
|
2350
|
+
* the exact ambiguity the progress lines exist to remove (#861).
|
|
2351
|
+
*/
|
|
2352
|
+
function armClock(ms: number): string {
|
|
2353
|
+
const total = Math.max(0, Math.round(ms / 1000));
|
|
2354
|
+
const minutes = Math.floor(total / 60);
|
|
2355
|
+
const seconds = total % 60;
|
|
2356
|
+
return minutes === 0 ? `${seconds}s` : `${minutes}m${String(seconds).padStart(2, "0")}s`;
|
|
2357
|
+
}
|
|
2358
|
+
|
|
2359
|
+
/** How often the wait says it is still waiting. Six lines across the window. */
|
|
2360
|
+
const ARM_PROGRESS_INTERVAL_MS = 30_000;
|
|
2361
|
+
|
|
1904
2362
|
/**
|
|
1905
2363
|
* Polls the acknowledgement record for one exact challenge id until the
|
|
1906
2364
|
* orchestrator's inbound adapter writes it or the deadline passes (#614).
|
|
@@ -1910,6 +2368,14 @@ function makeChallengeCode(): string {
|
|
|
1910
2368
|
* orchestrator process. Only a record naming this exact id satisfies the
|
|
1911
2369
|
* wait — an acknowledgement cut for a replaced challenge is inert here by
|
|
1912
2370
|
* construction, and no transcript anywhere is opened.
|
|
2371
|
+
*
|
|
2372
|
+
* It also *says* it is waiting (#861). Measured 2026-08-21: an arm proof sat
|
|
2373
|
+
* silent for five minutes and was reported as a hung setup — the process was
|
|
2374
|
+
* healthy and the operator had no way to tell. A silent five-minute wait
|
|
2375
|
+
* inside a fence that holds dispatch is indistinguishable from a dead one, so
|
|
2376
|
+
* the elapsed/remaining line lands every {@link ARM_PROGRESS_INTERVAL_MS}
|
|
2377
|
+
* regardless of the (much shorter) poll cadence, and the terminal outcome is
|
|
2378
|
+
* always printed.
|
|
1913
2379
|
*/
|
|
1914
2380
|
async function waitForArmAcknowledgement(
|
|
1915
2381
|
challengeId: string,
|
|
@@ -1918,10 +2384,26 @@ async function waitForArmAcknowledgement(
|
|
|
1918
2384
|
): Promise<boolean> {
|
|
1919
2385
|
const now = deps.now ?? Date.now;
|
|
1920
2386
|
const sleep = deps.sleep ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)));
|
|
1921
|
-
const
|
|
2387
|
+
const report = deps.progress;
|
|
2388
|
+
const startedAt = now();
|
|
2389
|
+
const deadline = startedAt + timeoutMs;
|
|
2390
|
+
let nextReportAt = startedAt + ARM_PROGRESS_INTERVAL_MS;
|
|
1922
2391
|
for (;;) {
|
|
1923
|
-
if (readArmAcknowledgement(challengeId) !== undefined)
|
|
1924
|
-
|
|
2392
|
+
if (readArmAcknowledgement(challengeId) !== undefined) {
|
|
2393
|
+
report?.(`arm: reply acknowledged after ${armClock(now() - startedAt)} — the fleet is armed.`);
|
|
2394
|
+
return true;
|
|
2395
|
+
}
|
|
2396
|
+
const at = now();
|
|
2397
|
+
if (at >= deadline) return false;
|
|
2398
|
+
if (report !== undefined && at >= nextReportAt) {
|
|
2399
|
+
report(
|
|
2400
|
+
`arm: still waiting for the reply — ${armClock(at - startedAt)} elapsed, ` +
|
|
2401
|
+
`${armClock(deadline - at)} left. Nothing is stuck: reply in the Telegram chat with the code.`,
|
|
2402
|
+
);
|
|
2403
|
+
// Anchored to the clock, not to this pass, so a slow pass cannot make the
|
|
2404
|
+
// cadence drift into silence.
|
|
2405
|
+
while (nextReportAt <= at) nextReportAt += ARM_PROGRESS_INTERVAL_MS;
|
|
2406
|
+
}
|
|
1925
2407
|
await sleep(5_000);
|
|
1926
2408
|
}
|
|
1927
2409
|
}
|