omp-conductor 0.15.12 → 0.15.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +9 -4
- package/package.json +1 -1
- package/src/commands/restart.ts +81 -54
- package/src/commands/stop.ts +45 -22
- package/src/daemon.ts +253 -54
- package/src/doctor.ts +241 -4
- package/src/escalate.ts +8 -0
- package/src/fleet.ts +49 -2
- package/src/lifecycle.ts +113 -2
- package/src/omp.ts +24 -0
- package/src/orchestrator-down.ts +231 -0
- package/src/orchestrator-tick.ts +7 -0
- package/src/orchestrator.ts +14 -0
- package/src/release-policy.ts +163 -18
- package/src/setup-host.ts +386 -17
- package/src/setup-install.ts +40 -2
- package/src/setup-wizard.ts +278 -113
- package/src/stop-provenance.ts +66 -0
- package/src/store.ts +181 -0
- package/src/types.ts +111 -0
- package/src/upgrade.ts +26 -6
- package/src/verbs/protocol.ts +16 -3
- package/src/verbs/server.ts +27 -1
- package/src/wizard-ui.ts +261 -46
- package/src/worker.ts +21 -0
package/src/doctor.ts
CHANGED
|
@@ -27,18 +27,38 @@
|
|
|
27
27
|
|
|
28
28
|
import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
|
|
29
29
|
import { spawnSync } from "node:child_process";
|
|
30
|
+
import { homedir } from "node:os";
|
|
30
31
|
import { join } from "node:path";
|
|
31
32
|
import type { Stats } from "node:fs";
|
|
32
33
|
import { Database } from "bun:sqlite";
|
|
33
34
|
|
|
34
35
|
import { configBackupDir, configPath, findProject, loadConfig, resolveCaps, stateDir } from "./config.ts";
|
|
35
|
-
import {
|
|
36
|
-
|
|
36
|
+
import {
|
|
37
|
+
DEFAULT_HERDR_UNIT,
|
|
38
|
+
probeTelegramHealth,
|
|
39
|
+
resolveHerdrSessionWithBridge,
|
|
40
|
+
telegramStateDir,
|
|
41
|
+
type TelegramHealth,
|
|
42
|
+
} from "./fleet.ts";
|
|
43
|
+
import {
|
|
44
|
+
herdrConductorPluginConfigDir,
|
|
45
|
+
planHostRuntime,
|
|
46
|
+
STAGED_SERVICE_NAME,
|
|
47
|
+
SYSTEMD_UNIT_DIR,
|
|
48
|
+
tickCwdForProject,
|
|
49
|
+
totalConfiguredWorkers,
|
|
50
|
+
} from "./setup-host.ts";
|
|
37
51
|
import { checkTokenScopes, type ScopeCheck } from "./setup.ts";
|
|
38
52
|
import { dbPath, LIVE_STATES } from "./store.ts";
|
|
39
53
|
import { telegramReportSend, type ReportSend } from "./reports.ts";
|
|
40
54
|
import { fetchRateLimit, GhError, gh } from "./tracker/github.ts";
|
|
41
55
|
import { repoSlugFor } from "./gitops.ts";
|
|
56
|
+
import {
|
|
57
|
+
DEFAULT_FLEET_AGENT_NAME,
|
|
58
|
+
parseHerdrAgents,
|
|
59
|
+
readTickConfig,
|
|
60
|
+
type HerdrAgentList,
|
|
61
|
+
} from "./orchestrator-tick.ts";
|
|
42
62
|
import type { ConductorConfig, ProjectConfig, RepoTarget, RunState } from "./types.ts";
|
|
43
63
|
|
|
44
64
|
/**
|
|
@@ -148,6 +168,20 @@ export interface DoctorDeps {
|
|
|
148
168
|
telegramSend?: (project: ProjectConfig) => ReportSend;
|
|
149
169
|
/** The canonical units to compare the installed units against. */
|
|
150
170
|
canonicalUnits?: (project: ProjectConfig, cfg: ConductorConfig) => CanonicalUnits;
|
|
171
|
+
/** Whether herdr is installed on this host (a `herdr` on PATH). */
|
|
172
|
+
herdrInstalled?: () => boolean;
|
|
173
|
+
/** Live `herdr --session <s> agent list`, parsed through the tick's own
|
|
174
|
+
* parser (the same "one JSON line on stdout" contract recover.sh reads). */
|
|
175
|
+
herdrAgents?: (session: string) => HerdrAgentList;
|
|
176
|
+
/** The fleet session name, for the rename fix line. */
|
|
177
|
+
herdrSession?: () => string;
|
|
178
|
+
/** The live herdr config at `~/.config/herdr/config.toml`, or undefined. */
|
|
179
|
+
herdrConfig?: () => string | undefined;
|
|
180
|
+
/** The live herdr-conductor plugin `config.env`, or undefined. */
|
|
181
|
+
herdrEnv?: () => string | undefined;
|
|
182
|
+
/** The fleet agent name the tick config of one project names, or undefined
|
|
183
|
+
* when there is no (readable) tick — the expected live herdr pane identity. */
|
|
184
|
+
tickAgentName?: (project: ProjectConfig) => string | undefined;
|
|
151
185
|
/** Clock, so a run is deterministic in tests. */
|
|
152
186
|
now?: () => number;
|
|
153
187
|
/** The one opt-in side effect: send one self-identified Telegram probe. */
|
|
@@ -790,7 +824,190 @@ function spendProbe(rows: RunSpendRow[], limit: number): Finding {
|
|
|
790
824
|
return passFinding("spend-telemetry", `spend observed on the last ${window.length} completed runs ($${total.toFixed(2)} total)`);
|
|
791
825
|
}
|
|
792
826
|
|
|
793
|
-
|
|
827
|
+
/** Live `herdr --session <session> agent list` through the project's own
|
|
828
|
+
* parser — the published "one JSON line on stdout" interface, never the
|
|
829
|
+
* socket. Same bound as the tick's own query so a hung herdr cannot hang
|
|
830
|
+
* doctor. */
|
|
831
|
+
function defaultHerdrAgents(session: string): HerdrAgentList {
|
|
832
|
+
try {
|
|
833
|
+
const run = spawnSync("herdr", ["--session", session, "agent", "list"], {
|
|
834
|
+
encoding: "utf8",
|
|
835
|
+
timeout: 3000,
|
|
836
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
837
|
+
});
|
|
838
|
+
if (run.error !== undefined) return { kind: "unavailable", problem: run.error.message };
|
|
839
|
+
if (run.status !== 0) {
|
|
840
|
+
return {
|
|
841
|
+
kind: "unavailable",
|
|
842
|
+
problem: `herdr agent list exited ${String(run.status)}: ${(run.stderr ?? "").trim().split("\n")[0] ?? ""}`,
|
|
843
|
+
};
|
|
844
|
+
}
|
|
845
|
+
return parseHerdrAgents(run.stdout ?? "");
|
|
846
|
+
} catch (err) {
|
|
847
|
+
return { kind: "unavailable", problem: err instanceof Error ? err.message : String(err) };
|
|
848
|
+
}
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
/** Live herdr config, defaulting to the fleet account's own path. */
|
|
852
|
+
function defaultHerdrConfig(): string | undefined {
|
|
853
|
+
try {
|
|
854
|
+
return readFileSync(join(homedir(), ".config", "herdr", "config.toml"), "utf8");
|
|
855
|
+
} catch {
|
|
856
|
+
return undefined;
|
|
857
|
+
}
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
/** Live herdr-conductor plugin `config.env`, same path setup provisions. */
|
|
861
|
+
function defaultHerdrEnv(): string | undefined {
|
|
862
|
+
try {
|
|
863
|
+
return readFileSync(join(herdrConductorPluginConfigDir(homedir()), "config.env"), "utf8");
|
|
864
|
+
} catch {
|
|
865
|
+
return undefined;
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
/**
|
|
870
|
+
* #541 check 1 — live herdr agent name equals the tick config's agentName.
|
|
871
|
+
*
|
|
872
|
+
* The paneOwnership decline the fleet actually hit names the exact fix (`herdr
|
|
873
|
+
* agent rename <pane> <name>`); doctor checks the same premise the tick does
|
|
874
|
+
* (is a pane registered under the configured agent name?) and reports drift
|
|
875
|
+
* with the one-line remedy.
|
|
876
|
+
*/
|
|
877
|
+
function herdrAgentNameProbe(probes: Probes, p: ProjectConfig): Finding {
|
|
878
|
+
if (!probes.herdrInstalled()) {
|
|
879
|
+
return passFinding("herdr-agent-name", `[${p.name}] herdr not installed — nothing to check`);
|
|
880
|
+
}
|
|
881
|
+
const want = probes.tickAgentName(p);
|
|
882
|
+
if (want === undefined) {
|
|
883
|
+
return passFinding("herdr-agent-name", `[${p.name}] no readable tick config — nothing to check`);
|
|
884
|
+
}
|
|
885
|
+
const session = probes.herdrSession();
|
|
886
|
+
const agents = probes.herdrAgents(session);
|
|
887
|
+
if (agents.kind !== "ok") {
|
|
888
|
+
return failFinding(
|
|
889
|
+
"herdr-agent-name",
|
|
890
|
+
`[${p.name}] could not read the live herdr agent list (${agents.problem}) — cannot verify the fleet agent name`,
|
|
891
|
+
`restore the herdr session (check \`herdr status\`), then re-run doctor`,
|
|
892
|
+
);
|
|
893
|
+
}
|
|
894
|
+
const holders = agents.agents.filter((a) => a.name === want);
|
|
895
|
+
if (holders.length > 0) {
|
|
896
|
+
return passFinding("herdr-agent-name", `[${p.name}] live pane(s) ${holders.map((a) => a.paneId).join(", ")} are agent "${want}"`);
|
|
897
|
+
}
|
|
898
|
+
// Drift: no pane carries the configured identity. Exactly one pane pinned to
|
|
899
|
+
// the shared default is the safe candidate from the decline turnover — name
|
|
900
|
+
// it when unambiguous, else give the generic command.
|
|
901
|
+
const stale = agents.agents.filter((a) => a.name === DEFAULT_FLEET_AGENT_NAME);
|
|
902
|
+
const pane = stale.length === 1 ? stale[0]!.paneId : "<pane>";
|
|
903
|
+
return failFinding(
|
|
904
|
+
"herdr-agent-name",
|
|
905
|
+
`[${p.name}] no live pane is agent "${want}" — every tick will be declined until the fleet pane carries this name (the 2026-08-15 restamp class)`,
|
|
906
|
+
`herdr --session ${session} agent rename ${pane} ${want}`,
|
|
907
|
+
);
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
/**
|
|
911
|
+
* #541 check 2 — `[session] resume_agents_on_restore` on the live herdr config.
|
|
912
|
+
*
|
|
913
|
+
* herdr's own README: the default `true` leaves a restored omp pane with a
|
|
914
|
+
* deferred resume plan and no live terminal, which is the configuration that
|
|
915
|
+
* pages instead of recovering. Absent ~ the unrecoverable default; an explicit
|
|
916
|
+
* `true` is a deliberate (desktop) choice and only warned.
|
|
917
|
+
*/
|
|
918
|
+
function herdrResumeProbe(probes: Probes): Finding {
|
|
919
|
+
if (!probes.herdrInstalled()) {
|
|
920
|
+
return passFinding("herdr-resume", "herdr not installed — nothing to check");
|
|
921
|
+
}
|
|
922
|
+
const live = probes.herdrConfig();
|
|
923
|
+
if (live === undefined) {
|
|
924
|
+
return failFinding(
|
|
925
|
+
"herdr-resume",
|
|
926
|
+
"no herdr config at " + join(homedir(), ".config", "herdr", "config.toml") + " — a restored omp pane gets no live terminal (herdr's own README)",
|
|
927
|
+
"run \`omp-conductor setup host\` from the fleet account (writes [session] resume_agents_on_restore = false), or add it by hand",
|
|
928
|
+
);
|
|
929
|
+
}
|
|
930
|
+
let parsed: { session?: { resume_agents_on_restore?: unknown } };
|
|
931
|
+
try {
|
|
932
|
+
parsed = Bun.TOML.parse(live) as { session?: { resume_agents_on_restore?: unknown } };
|
|
933
|
+
} catch (err) {
|
|
934
|
+
return failFinding(
|
|
935
|
+
"herdr-resume",
|
|
936
|
+
`the live herdr config does not parse (${err instanceof Error ? err.message : String(err)}) — herdr itself refuses it`,
|
|
937
|
+
"run `herdr config check` to name the line, fix, then re-run doctor",
|
|
938
|
+
);
|
|
939
|
+
}
|
|
940
|
+
const value = parsed.session?.resume_agents_on_restore;
|
|
941
|
+
if (value === false) {
|
|
942
|
+
return passFinding("herdr-resume", "resume_agents_on_restore is false — restored panes come up as shells, not deferred plans");
|
|
943
|
+
}
|
|
944
|
+
if (value === true) {
|
|
945
|
+
return warnFinding(
|
|
946
|
+
"herdr-resume",
|
|
947
|
+
"resume_agents_on_restore is explicitly true — possibly a deliberate desktop config, but a headless fleet restores panes with no live terminal",
|
|
948
|
+
"set it to false for a headless host ([session] resume_agents_on_restore = false), or run `omp-conductor setup host`",
|
|
949
|
+
);
|
|
950
|
+
}
|
|
951
|
+
return failFinding(
|
|
952
|
+
"herdr-resume",
|
|
953
|
+
"resume_agents_on_restore is absent — herdr defaults it to true, the configuration its own README calls unrecoverable",
|
|
954
|
+
"run `omp-conductor setup host` from the fleet account (writes the key), or add [session] resume_agents_on_restore = false by hand",
|
|
955
|
+
);
|
|
956
|
+
}
|
|
957
|
+
|
|
958
|
+
/**
|
|
959
|
+
* #541 check 3 — herdr-conductor `config.env` FLEET_CWDS vs the configured
|
|
960
|
+
* projects. Absent, a virgin host recovers against legacy `/root/fleet` paths
|
|
961
|
+
* that do not exist: recovery still runs, only the page silently goes nowhere.
|
|
962
|
+
*/
|
|
963
|
+
function herdrEnvProbe(probes: Probes, cfg: ConductorConfig | undefined): Finding {
|
|
964
|
+
if (!probes.herdrInstalled()) {
|
|
965
|
+
return passFinding("herdr-config-env", "herdr not installed — nothing to check");
|
|
966
|
+
}
|
|
967
|
+
if (cfg === undefined) {
|
|
968
|
+
return passFinding("herdr-config-env", "config unreadable — nothing to compare");
|
|
969
|
+
}
|
|
970
|
+
const live = probes.herdrEnv();
|
|
971
|
+
if (live === undefined) {
|
|
972
|
+
return failFinding(
|
|
973
|
+
"herdr-config-env",
|
|
974
|
+
"no herdr-conductor config.env — recovery falls back to legacy /root-based single-tenant paths this host does not use, so pages silently skip",
|
|
975
|
+
"run `omp-conductor setup host` from the fleet account (writes config.env)",
|
|
976
|
+
);
|
|
977
|
+
}
|
|
978
|
+
const expected = new Set(cfg.projects.map((p) => tickCwdForProject(p)));
|
|
979
|
+
const actual = parseFleetCwds(live);
|
|
980
|
+
const coherent =
|
|
981
|
+
actual !== undefined && actual.size === expected.size && [...actual].every((cwd) => expected.has(cwd));
|
|
982
|
+
if (coherent) {
|
|
983
|
+
return passFinding("herdr-config-env", `config.env FLEET_CWDS matches the configured fleets (${[...expected].join(":")})`);
|
|
984
|
+
}
|
|
985
|
+
return failFinding(
|
|
986
|
+
"herdr-config-env",
|
|
987
|
+
`config.env FLEET_CWDS (${actual === undefined ? "unset" : [...actual].join(":")}) does not match the configured fleets (${[...expected].join(":")})`,
|
|
988
|
+
"re-run `omp-conductor setup host`, which derives FLEET_CWDS from the configured projects",
|
|
989
|
+
);
|
|
990
|
+
}
|
|
991
|
+
|
|
992
|
+
/** FLEET_CWDS (colon- or space-separated, per recover.sh) or undefined. */
|
|
993
|
+
function parseFleetCwds(text: string): Set<string> | undefined {
|
|
994
|
+
const value = parseEnvKey(text, "FLEET_CWDS");
|
|
995
|
+
if (value !== undefined) {
|
|
996
|
+
return new Set(value.split(/[\s:]+/).filter((w) => w.length > 0));
|
|
997
|
+
}
|
|
998
|
+
// Legacy single-fleet form, the same fallback recover.sh applies.
|
|
999
|
+
const legacy = parseEnvKey(text, "FLEET_CWD");
|
|
1000
|
+
return legacy === undefined ? undefined : new Set([legacy]);
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
/** The first unquoted-or-quoted `KEY=value` on its own line, comments ignored. */
|
|
1004
|
+
function parseEnvKey(text: string, key: string): string | undefined {
|
|
1005
|
+
for (const line of text.split("\n")) {
|
|
1006
|
+
const match = new RegExp(`^[ \\t]*${key}[ \\t]*=[ \\t]*"?([^"\\n]*)"?[ \\t]*$`).exec(line);
|
|
1007
|
+
if (match !== null) return match[1];
|
|
1008
|
+
}
|
|
1009
|
+
return undefined;
|
|
1010
|
+
}
|
|
794
1011
|
|
|
795
1012
|
/**
|
|
796
1013
|
* Run every probe and assemble the stable report.
|
|
@@ -873,8 +1090,14 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
|
|
|
873
1090
|
findings.push(
|
|
874
1091
|
passFinding("labels", projectProblem === undefined ? "no project resolved — nothing to check" : `labels uncheckable: ${projectProblem}`),
|
|
875
1092
|
);
|
|
1093
|
+
findings.push(
|
|
1094
|
+
passFinding("herdr-agent-name", projectProblem === undefined ? "no project resolved — nothing to check" : `agent name uncheckable: ${projectProblem}`),
|
|
1095
|
+
);
|
|
876
1096
|
} else {
|
|
877
|
-
for (const p of projects)
|
|
1097
|
+
for (const p of projects) {
|
|
1098
|
+
findings.push(await labelProbe(probes, p));
|
|
1099
|
+
findings.push(herdrAgentNameProbe(probes, p));
|
|
1100
|
+
}
|
|
878
1101
|
}
|
|
879
1102
|
// The installed-unit check is host-global: one shared daemon. Any project
|
|
880
1103
|
// renders the same canonical units (the daemon/units carry no project), so
|
|
@@ -882,6 +1105,10 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
|
|
|
882
1105
|
findings.push(unitProbe(probes, projects[0], cfg));
|
|
883
1106
|
findings.push(recoveryProbe(probes));
|
|
884
1107
|
findings.push(ownershipProbe(probes));
|
|
1108
|
+
// #541 seam checks, host-global: the live herdr config and the plugin's
|
|
1109
|
+
// config.env are single files on the host, not per-project facts.
|
|
1110
|
+
findings.push(herdrResumeProbe(probes));
|
|
1111
|
+
findings.push(herdrEnvProbe(probes, cfg));
|
|
885
1112
|
if (projects.length === 0) {
|
|
886
1113
|
findings.push(timezoneProbe(undefined));
|
|
887
1114
|
findings.push(await telegramProbe(probes, undefined, checkedAt));
|
|
@@ -926,6 +1153,16 @@ export function defaultProbes(): Probes {
|
|
|
926
1153
|
telegramHealth: (projectName) => probeTelegramHealth(projectName),
|
|
927
1154
|
telegramSend: telegramReportSend,
|
|
928
1155
|
canonicalUnits: defaultCanonicalUnits,
|
|
1156
|
+
herdrInstalled: () => Bun.which("herdr") !== null,
|
|
1157
|
+
herdrAgents: defaultHerdrAgents,
|
|
1158
|
+
herdrSession: () => resolveHerdrSessionWithBridge(),
|
|
1159
|
+
herdrConfig: defaultHerdrConfig,
|
|
1160
|
+
herdrEnv: defaultHerdrEnv,
|
|
1161
|
+
tickAgentName: (p) => {
|
|
1162
|
+
const tick = readTickConfig(tickCwdForProject(p));
|
|
1163
|
+
if (tick.kind !== "ok") return undefined;
|
|
1164
|
+
return tick.config.agentName ?? DEFAULT_FLEET_AGENT_NAME;
|
|
1165
|
+
},
|
|
929
1166
|
now: Date.now,
|
|
930
1167
|
probeTelegram: false,
|
|
931
1168
|
};
|
package/src/escalate.ts
CHANGED
|
@@ -144,6 +144,7 @@ export function createEscalator(
|
|
|
144
144
|
orchestrator?: OrchestratorHandle,
|
|
145
145
|
now: () => number = Date.now,
|
|
146
146
|
deliveryAllowed: () => boolean = () => true,
|
|
147
|
+
onTier1Diverted?: (e: Escalation) => void,
|
|
147
148
|
): Escalator {
|
|
148
149
|
const currentProject = typeof source === "function" ? source : (): ProjectConfig => source;
|
|
149
150
|
return {
|
|
@@ -278,6 +279,13 @@ export function createEscalator(
|
|
|
278
279
|
// Tier 1 lands here with no orchestrator, or with one that would not take
|
|
279
280
|
// the injection; tier 2 lands here when omp-telegram is not installed or
|
|
280
281
|
// the project never configured a chat id.
|
|
282
|
+
//
|
|
283
|
+
// A tier-1 event reaching this point was diverted from the orchestrator to
|
|
284
|
+
// an issue comment. That is the exact condition the down incident counts:
|
|
285
|
+
// the daemon hooks this to accumulate the orchestrator-down incident's
|
|
286
|
+
// diverted tally (a reader that owns its own dedupe and no-ops for a
|
|
287
|
+
// healthy or external orchestrator).
|
|
288
|
+
if (e.tier === 1) onTier1Diverted?.(e);
|
|
281
289
|
if (!p.escalation.fallbackToIssueComment) {
|
|
282
290
|
throw new Error(
|
|
283
291
|
`no escalation transport configured for project "${p.name}": tier ${e.tier} ` +
|
package/src/fleet.ts
CHANGED
|
@@ -36,7 +36,7 @@ import { readApprovalSurface, readDaemonProfile } from "./approval-surface.ts";
|
|
|
36
36
|
import { inspectBriefLayout } from "./brief-upgrade.ts";
|
|
37
37
|
import { dbPath, openStore } from "./store.ts";
|
|
38
38
|
import { renderBriefForProject } from "./setup.ts";
|
|
39
|
-
import type { ProjectConfig, Store } from "./types.ts";
|
|
39
|
+
import type { DaemonStop, ProjectConfig, Store } from "./types.ts";
|
|
40
40
|
import { settlementFlagSummary } from "./diff-flags.ts";
|
|
41
41
|
import { probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
|
|
42
42
|
import { formatDigestBacklog, formatOpenReports } from "./reports.ts";
|
|
@@ -52,6 +52,7 @@ import {
|
|
|
52
52
|
statusSnapshot,
|
|
53
53
|
type StatusSnapshot,
|
|
54
54
|
} from "./daemon.ts";
|
|
55
|
+
import { formatOrchestratorDown } from "./orchestrator-down.ts";
|
|
55
56
|
import {
|
|
56
57
|
healthCheck,
|
|
57
58
|
isAlive,
|
|
@@ -1410,6 +1411,31 @@ function formatDaemonUnit(unit: UnitOwnership | undefined, recordPid: number): s
|
|
|
1410
1411
|
}
|
|
1411
1412
|
}
|
|
1412
1413
|
|
|
1414
|
+
/**
|
|
1415
|
+
* The newest daemon stop/restart provenance as status lines (#378).
|
|
1416
|
+
*
|
|
1417
|
+
* Rendered under the daemon block whether the daemon is down or has since
|
|
1418
|
+
* been restarted — the whole point is that the debrief survives the restart.
|
|
1419
|
+
* An unattributed record says so explicitly: the external-signal fallback
|
|
1420
|
+
* records what the receiving daemon knew (that it was unattributed, when,
|
|
1421
|
+
* which projects it served with live counts) rather than guessing a caller.
|
|
1422
|
+
*/
|
|
1423
|
+
function formatLastStop(lastStop: DaemonStop | undefined): string[] {
|
|
1424
|
+
if (lastStop === undefined) return [];
|
|
1425
|
+
const caller = lastStop.unattributed
|
|
1426
|
+
? "unattributed — no mediated request (external signal)"
|
|
1427
|
+
: `pid ${lastStop.callerPid ?? "?"}` +
|
|
1428
|
+
(lastStop.callerUid === undefined ? "" : ` uid ${lastStop.callerUid}`) +
|
|
1429
|
+
(lastStop.role === undefined ? "" : ` (${lastStop.role})`);
|
|
1430
|
+
return [
|
|
1431
|
+
` last stop ${new Date(lastStop.at).toISOString()} ${lastStop.controlPath}`,
|
|
1432
|
+
` caller ${caller}`,
|
|
1433
|
+
` scope ${lastStop.scope}${lastStop.project === undefined ? "" : `: ${lastStop.project}`}`,
|
|
1434
|
+
` affects ${lastStop.affected.map((a) => `${a.project} (${a.live} live)`).join(", ")}`,
|
|
1435
|
+
` reason ${lastStop.reason}`,
|
|
1436
|
+
];
|
|
1437
|
+
}
|
|
1438
|
+
|
|
1413
1439
|
export function formatFleetStatus(
|
|
1414
1440
|
s: StatusSnapshot,
|
|
1415
1441
|
layers: FleetLayers,
|
|
@@ -1422,6 +1448,7 @@ export function formatFleetStatus(
|
|
|
1422
1448
|
failureClasses: string | undefined = undefined,
|
|
1423
1449
|
workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
|
|
1424
1450
|
intake: string | undefined = undefined,
|
|
1451
|
+
lastStop: DaemonStop | undefined = undefined,
|
|
1425
1452
|
): string {
|
|
1426
1453
|
const tickLine =
|
|
1427
1454
|
layers.ticksDetail === undefined
|
|
@@ -1505,8 +1532,9 @@ export function formatFleetStatus(
|
|
|
1505
1532
|
...(intake === undefined ? [] : [intake]),
|
|
1506
1533
|
...(graphBlock === undefined ? [] : [graphBlock]),
|
|
1507
1534
|
daemonBlock,
|
|
1535
|
+
...formatLastStop(lastStop),
|
|
1508
1536
|
"",
|
|
1509
|
-
formatProjectBody(s, workerPhases),
|
|
1537
|
+
formatProjectBody(s, workerPhases, now),
|
|
1510
1538
|
].join("\n");
|
|
1511
1539
|
}
|
|
1512
1540
|
|
|
@@ -1540,12 +1568,18 @@ function formatDigestScheduleStatus(s: StatusSnapshot): string[] {
|
|
|
1540
1568
|
function formatProjectBody(
|
|
1541
1569
|
s: StatusSnapshot,
|
|
1542
1570
|
workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
|
|
1571
|
+
now = Date.now(),
|
|
1543
1572
|
): string {
|
|
1544
1573
|
const lines = [
|
|
1545
1574
|
`project ${s.project}${s.paused ? " (PAUSED)" : ""}`,
|
|
1546
1575
|
...(s.pauseReason === undefined ? [] : [`paused ${s.pauseReason}`]),
|
|
1547
1576
|
`config ${s.configPath}`,
|
|
1548
1577
|
`state ${s.stateDir}`,
|
|
1578
|
+
// The orchestrator-down degrade row: first-class in the body, present only
|
|
1579
|
+
// while the incident is open, so recovery drops it (#288).
|
|
1580
|
+
...(s.orchestratorDown === undefined
|
|
1581
|
+
? []
|
|
1582
|
+
: formatOrchestratorDown(s.orchestratorDown, now)),
|
|
1549
1583
|
...formatAvailabilityStatus(s),
|
|
1550
1584
|
...formatDigestScheduleStatus(s),
|
|
1551
1585
|
"",
|
|
@@ -1675,6 +1709,18 @@ export async function renderStatus(projectName?: string): Promise<string> {
|
|
|
1675
1709
|
const cached = codeGraphFromHealthz(healthBody, project.name);
|
|
1676
1710
|
const codeGraph = cached ?? (await probeCodeGraph(project));
|
|
1677
1711
|
const workerPhases = workerPhasesFromHealthz(healthBody, project.name);
|
|
1712
|
+
// The newest host-wide stop/restart provenance (#378). Read here — not in
|
|
1713
|
+
// `statusSnapshot`, which is synchronous and belongs to the daemon module —
|
|
1714
|
+
// and rendered identically from either project: the daemon_stops table is
|
|
1715
|
+
// deliberately not partitioned by project, because the daemon serves every
|
|
1716
|
+
// project and the uninvolved one must see who stopped it too.
|
|
1717
|
+
const store = openStore(dbPath());
|
|
1718
|
+
let lastStop: DaemonStop | undefined;
|
|
1719
|
+
try {
|
|
1720
|
+
lastStop = store.latestDaemonStop();
|
|
1721
|
+
} finally {
|
|
1722
|
+
store.close();
|
|
1723
|
+
}
|
|
1678
1724
|
return formatFleetStatus(
|
|
1679
1725
|
{ ...s, planUsage, github },
|
|
1680
1726
|
layers,
|
|
@@ -1687,6 +1733,7 @@ export async function renderStatus(projectName?: string): Promise<string> {
|
|
|
1687
1733
|
failureClassBlock(project.name),
|
|
1688
1734
|
workerPhases,
|
|
1689
1735
|
intakeStatusLine(project.name),
|
|
1736
|
+
lastStop,
|
|
1690
1737
|
);
|
|
1691
1738
|
}
|
|
1692
1739
|
|
package/src/lifecycle.ts
CHANGED
|
@@ -25,6 +25,10 @@ import { spawn, spawnSync } from "node:child_process";
|
|
|
25
25
|
import { chmodSync, closeSync, mkdirSync, openSync, readFileSync, readSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
26
26
|
import { homedir } from "node:os";
|
|
27
27
|
import { join } from "node:path";
|
|
28
|
+
// Type-only: erased at runtime, so the "free of every other module" property
|
|
29
|
+
// below survives — this module still opens no store, loads no config and runs
|
|
30
|
+
// no `gh`, and `stop`/`status` keep working when the config is broken.
|
|
31
|
+
import type { DaemonStopDraft } from "./types.ts";
|
|
28
32
|
|
|
29
33
|
/**
|
|
30
34
|
* The systemd unit name operators are expected to install for a supervised
|
|
@@ -480,6 +484,58 @@ export async function startDaemon(
|
|
|
480
484
|
);
|
|
481
485
|
}
|
|
482
486
|
|
|
487
|
+
/**
|
|
488
|
+
* The delivery facts a mediated stop/restart records at the exact chokepoint:
|
|
489
|
+
* the request's own provenance, how the stop is about to be delivered, and the
|
|
490
|
+
* daemon pid it targets (#378). The recorder is injected by the CLI callers
|
|
491
|
+
* (which own the store); this module stays free of it.
|
|
492
|
+
*/
|
|
493
|
+
export interface StopDelivery {
|
|
494
|
+
/** The request facts built by the CLI (commands/stop.ts, commands/restart.ts). */
|
|
495
|
+
provenance: DaemonStopDraft;
|
|
496
|
+
/** Whether the stop is about to go through `systemctl` or a raw signal. */
|
|
497
|
+
via: "systemctl" | "signal";
|
|
498
|
+
/** The daemon pid being stopped, when one was known. */
|
|
499
|
+
pid?: number;
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
/** The injected recorder invoked immediately before a stop/restart is signalled. */
|
|
503
|
+
export type StopDeliveryFn = (stop: StopDelivery) => void;
|
|
504
|
+
|
|
505
|
+
/**
|
|
506
|
+
* The honest provenance for a daemon that received a stop with no mediated
|
|
507
|
+
* request anywhere: unattributed by construction — Linux exposes no sender
|
|
508
|
+
* identity for a signal, so the record must say "unattributed" rather than
|
|
509
|
+
* guess one — plus what the receiving process actually knows: its own pid,
|
|
510
|
+
* the moment (stamped by the store), the runtime directory, and the projects
|
|
511
|
+
* it serves with their live-run counts.
|
|
512
|
+
*
|
|
513
|
+
* Storage belongs at the caller: the daemon's SIGINT/SIGTERM path writes the
|
|
514
|
+
* returned draft through the store before/while draining. This builds the
|
|
515
|
+
* facts, store-free, so a raw `kill`/out-of-band `systemctl stop` — no
|
|
516
|
+
* conductor CLI in the delivery path at all — still leaves a durable row the
|
|
517
|
+
* next `omp-conductor status` can show.
|
|
518
|
+
*/
|
|
519
|
+
export function externalStopProvenance(o: {
|
|
520
|
+
/** The daemon pid that received the signal, when known. */
|
|
521
|
+
daemonPid?: number;
|
|
522
|
+
/** The daemon's runtime directory (host paths only — never secrets). */
|
|
523
|
+
runtimeDir: string;
|
|
524
|
+
/** Every served project with its live-run count at signal time. */
|
|
525
|
+
affected: { project: string; live: number }[];
|
|
526
|
+
reason?: string;
|
|
527
|
+
}): DaemonStopDraft {
|
|
528
|
+
return {
|
|
529
|
+
controlPath: "external signal",
|
|
530
|
+
scope: "global",
|
|
531
|
+
...(o.daemonPid === undefined ? {} : { daemonPid: o.daemonPid }),
|
|
532
|
+
runtimeDir: o.runtimeDir,
|
|
533
|
+
affected: o.affected,
|
|
534
|
+
reason: o.reason ?? "external signal — no mediated stop request was recorded",
|
|
535
|
+
unattributed: true,
|
|
536
|
+
};
|
|
537
|
+
}
|
|
538
|
+
|
|
483
539
|
/**
|
|
484
540
|
* How the last stop actually landed. Callers print this so an operator can
|
|
485
541
|
* tell a supervised stop from a bare SIGTERM without reading the journal.
|
|
@@ -510,7 +566,35 @@ export interface RestartResult {
|
|
|
510
566
|
* a stopped unit). The grace period is a deadline, not a clean drain: a tick
|
|
511
567
|
* with a worker in flight can run for that worker's whole wall clock.
|
|
512
568
|
*/
|
|
513
|
-
export async function stopDaemon(
|
|
569
|
+
export async function stopDaemon(
|
|
570
|
+
o: {
|
|
571
|
+
timeoutMs?: number;
|
|
572
|
+
/**
|
|
573
|
+
* The request facts for this stop, when a mediated caller carries them.
|
|
574
|
+
* With `record`, the durable row is written immediately before the signal
|
|
575
|
+
* is sent — the exact ordering "provenance precedes signalling" refers to
|
|
576
|
+
* — and only when a stop actually lands: a request that finds no daemon
|
|
577
|
+
* stops nothing and records nothing. Drain-style callers whose restart may
|
|
578
|
+
* never execute additionally record the request at entry themselves.
|
|
579
|
+
*/
|
|
580
|
+
provenance?: DaemonStopDraft;
|
|
581
|
+
/** Invoked just before the daemon is signalled, with the delivery method. */
|
|
582
|
+
record?: StopDeliveryFn;
|
|
583
|
+
} = {},
|
|
584
|
+
): Promise<StopResult> {
|
|
585
|
+
const recordStop = (via: "systemctl" | "signal", pid: number | undefined): void => {
|
|
586
|
+
if (o.provenance === undefined || o.record === undefined) return;
|
|
587
|
+
o.record({
|
|
588
|
+
provenance: {
|
|
589
|
+
...o.provenance,
|
|
590
|
+
controlPath: `${o.provenance.controlPath} via ${via}`,
|
|
591
|
+
...(pid === undefined ? {} : { daemonPid: pid }),
|
|
592
|
+
},
|
|
593
|
+
via,
|
|
594
|
+
pid,
|
|
595
|
+
});
|
|
596
|
+
};
|
|
597
|
+
|
|
514
598
|
const rec = livingDaemon();
|
|
515
599
|
if (rec === undefined) {
|
|
516
600
|
// `livingDaemon` already cleared a stale file; this covers the unparseable
|
|
@@ -521,6 +605,7 @@ export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<StopRe
|
|
|
521
605
|
throw new Error(ownershipUnknown("stop", decision.reason));
|
|
522
606
|
}
|
|
523
607
|
if (decision.kind === "stop") {
|
|
608
|
+
recordStop("systemctl", decision.mainPid);
|
|
524
609
|
await runSystemdStop(decision.mainPid, o.timeoutMs);
|
|
525
610
|
clearRecord();
|
|
526
611
|
return { kind: "stopped", pid: decision.mainPid, via: "systemctl" };
|
|
@@ -536,6 +621,7 @@ export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<StopRe
|
|
|
536
621
|
throw new Error(ownershipUnknown("stop", decision.reason));
|
|
537
622
|
}
|
|
538
623
|
if (decision.kind === "stop") {
|
|
624
|
+
recordStop("systemctl", rec.pid);
|
|
539
625
|
await runSystemdStop(rec.pid, o.timeoutMs);
|
|
540
626
|
clearRecord();
|
|
541
627
|
return { kind: "stopped", pid: rec.pid, via: "systemctl" };
|
|
@@ -544,6 +630,7 @@ export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<StopRe
|
|
|
544
630
|
// decision.kind === "not-ours": confirmed no unit, inactive unit, no systemd
|
|
545
631
|
// binary, or a unit whose MainPID is somebody else. Only a *confirmed*
|
|
546
632
|
// negative is safe to signal.
|
|
633
|
+
recordStop("signal", rec.pid);
|
|
547
634
|
const gone = await terminate(rec.pid, o.timeoutMs ?? STOP_TIMEOUT_MS);
|
|
548
635
|
if (!gone) {
|
|
549
636
|
// The record stays: something is still holding that pid, and forgetting
|
|
@@ -577,8 +664,29 @@ export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<StopRe
|
|
|
577
664
|
* Returns the record of the process that is now answering `/healthz`.
|
|
578
665
|
*/
|
|
579
666
|
export async function restartDaemon(
|
|
580
|
-
o: {
|
|
667
|
+
o: {
|
|
668
|
+
port?: number;
|
|
669
|
+
project?: string;
|
|
670
|
+
timeoutMs?: number;
|
|
671
|
+
/** Request facts and recorder, same semantics as {@link stopDaemon}: the
|
|
672
|
+
* durable row is written immediately before the restarting signal. */
|
|
673
|
+
provenance?: DaemonStopDraft;
|
|
674
|
+
record?: StopDeliveryFn;
|
|
675
|
+
} = {},
|
|
581
676
|
): Promise<RestartResult> {
|
|
677
|
+
const recordStop = (via: "systemctl" | "signal", pid: number | undefined): void => {
|
|
678
|
+
if (o.provenance === undefined || o.record === undefined) return;
|
|
679
|
+
o.record({
|
|
680
|
+
provenance: {
|
|
681
|
+
...o.provenance,
|
|
682
|
+
controlPath: `${o.provenance.controlPath} via ${via}`,
|
|
683
|
+
...(pid === undefined ? {} : { daemonPid: pid }),
|
|
684
|
+
},
|
|
685
|
+
via,
|
|
686
|
+
pid,
|
|
687
|
+
});
|
|
688
|
+
};
|
|
689
|
+
|
|
582
690
|
const previous = livingDaemon();
|
|
583
691
|
const ownership = probeUnit();
|
|
584
692
|
if (ownership.kind === "unknown") {
|
|
@@ -591,6 +699,7 @@ export async function restartDaemon(
|
|
|
591
699
|
// startDaemon() — that leaves the unit failed while handing the operator
|
|
592
700
|
// a healthy-looking but unmanaged pid (the 12:17Z incident in #376).
|
|
593
701
|
// Any manager refusal is terminal; so is unproven ownership afterwards.
|
|
702
|
+
recordStop("systemctl", previous?.pid);
|
|
594
703
|
const record = await restoreFailedUnit(o.timeoutMs, o.project);
|
|
595
704
|
return { previous, record, via: "systemctl" };
|
|
596
705
|
}
|
|
@@ -602,6 +711,7 @@ export async function restartDaemon(
|
|
|
602
711
|
// Ownership is proven. A refused/timed-out restart must not fall through
|
|
603
712
|
// to stopDaemon's signal path — that is the exact bounce this module exists
|
|
604
713
|
// to prevent (SIGTERM → exit 143 → Restart=on-failure → new MainPID).
|
|
714
|
+
recordStop("systemctl", ownership.pid);
|
|
605
715
|
const ran = systemctl(["restart", SYSTEMD_UNIT]);
|
|
606
716
|
if (!ran.ok) {
|
|
607
717
|
throw new Error(systemctlFailure("restart", ran));
|
|
@@ -615,6 +725,7 @@ export async function restartDaemon(
|
|
|
615
725
|
|
|
616
726
|
// Confirmed unmanaged: no unit, an inactive unit, or a unit whose MainPID
|
|
617
727
|
// is somebody else. The detached CLI daemon is the only path left.
|
|
728
|
+
recordStop("signal", previous?.pid);
|
|
618
729
|
await stopDaemon({ timeoutMs: o.timeoutMs });
|
|
619
730
|
const record = await startDaemon({ port: o.port ?? previous?.port, project: o.project ?? previous?.project });
|
|
620
731
|
return { previous, record, via: "cli" };
|
package/src/omp.ts
CHANGED
|
@@ -44,6 +44,14 @@ export interface AgentSessionLike {
|
|
|
44
44
|
* `"*"` for every event. The payload is the harness's own event union, which
|
|
45
45
|
* this package cannot name without the peer dependency, so it arrives as
|
|
46
46
|
* `unknown` and each caller narrows the two or three fields it reads.
|
|
47
|
+
*
|
|
48
|
+
* One event is not the harness's at all: `"session_exit"` fires when the
|
|
49
|
+
* underlying session *process* has terminated, carrying
|
|
50
|
+
* `{ type: "session_exit", code?: number | null }`. It is the single real
|
|
51
|
+
* terminal signal — a crashed or killed session is never confused with one
|
|
52
|
+
* that merely stopped streaming (`"agent_end"`). A supervisor may receive it
|
|
53
|
+
* after its own `dispose()`, and must decide what an exit means from its own
|
|
54
|
+
* state rather than assuming every exit is a crash.
|
|
47
55
|
*/
|
|
48
56
|
on(event: string, cb: (e: unknown) => void): void;
|
|
49
57
|
abort(): void;
|
|
@@ -714,8 +722,24 @@ export async function createSession(opts: CreateSessionOptions): Promise<AgentSe
|
|
|
714
722
|
pending.clear();
|
|
715
723
|
};
|
|
716
724
|
|
|
725
|
+
// The one terminal event this proxy owns outright: the session process is
|
|
726
|
+
// gone, whether by crash, kill, or a clean dispose of its own. Delivered to
|
|
727
|
+
// subscribers *before* any failure is surfaced, so a supervisor learning of
|
|
728
|
+
// the death can also read its exit code off the same event. `disposing` is
|
|
729
|
+
// the proxy's own intent (its disposer asked the child to stop), not the
|
|
730
|
+
// subscriber's, so the event is emitted in both branches and each observer
|
|
731
|
+
// decides what an exit means from its own state.
|
|
732
|
+
const emitSessionExit = (code: number | null): void => {
|
|
733
|
+
const type = "session_exit";
|
|
734
|
+
const event = { type, code };
|
|
735
|
+
for (const cb of handlers.get(type) ?? []) cb(event);
|
|
736
|
+
for (const cb of handlers.get("*") ?? []) cb(event);
|
|
737
|
+
};
|
|
738
|
+
|
|
717
739
|
void child.exited.then(async (code) => {
|
|
718
740
|
onExit();
|
|
741
|
+
const exitCode = typeof code === "number" ? code : null;
|
|
742
|
+
emitSessionExit(exitCode);
|
|
719
743
|
// A child that cannot start writes `start-error` over the socket, not the
|
|
720
744
|
// pipes — and its exit can be dispatched before the accept of a connection
|
|
721
745
|
// that already completed in the kernel. Let the socket settle before the
|