omp-conductor 0.3.13 → 0.3.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +227 -70
- package/package.json +3 -2
- package/skills/conductor-update/SKILL.md +157 -0
- package/src/brief-upgrade.ts +1 -1
- package/src/briefs/orchestrator.md +3 -2
- package/src/briefs/worker.md +1 -1
- package/src/cli.ts +161 -40
- package/src/confinement.ts +123 -0
- package/src/daemon.ts +55 -2
- package/src/fleet.ts +1271 -0
- package/src/host.ts +90 -0
- package/src/lifecycle.ts +182 -77
- package/src/omp.ts +12 -0
- package/src/orchestrator-tick.ts +64 -3
- package/src/plugin.ts +110 -17
- package/src/tracker/github.ts +25 -1
- package/src/types.ts +10 -0
- package/src/worker.ts +2 -0
- package/systemd/omp-conductor.service.example +54 -0
package/src/fleet.ts
ADDED
|
@@ -0,0 +1,1271 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Operator surface for the two-process fleet: dispatch daemon + orchestrator
|
|
3
|
+
* pane. Issue #61 — one answer to "stop the conductor" without naming four
|
|
4
|
+
* files and units.
|
|
5
|
+
*
|
|
6
|
+
* Verbs:
|
|
7
|
+
* - hold — pause claiming AND disarm ticks; processes stay up
|
|
8
|
+
* - halt — hold, then stop the dispatch daemon (systemctl-aware)
|
|
9
|
+
* - halt --pane — pin recovery first, stop the exact omp conductor agent,
|
|
10
|
+
* never systemctl stop herdr-fleet
|
|
11
|
+
* - arm / disarm — first-class armed marker (arm is inbound Telegram proof)
|
|
12
|
+
* - releaseHold — clear pause only; never re-arms
|
|
13
|
+
* - release-pane — clear the halt --pane recovery pin
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { spawnSync } from "node:child_process";
|
|
17
|
+
import {
|
|
18
|
+
createReadStream,
|
|
19
|
+
existsSync,
|
|
20
|
+
mkdirSync,
|
|
21
|
+
readdirSync,
|
|
22
|
+
readFileSync,
|
|
23
|
+
rmSync,
|
|
24
|
+
statSync,
|
|
25
|
+
writeFileSync,
|
|
26
|
+
} from "node:fs";
|
|
27
|
+
import { createInterface } from "node:readline";
|
|
28
|
+
import { homedir } from "node:os";
|
|
29
|
+
import { dirname, join } from "node:path";
|
|
30
|
+
import { findProject, loadConfig, stateDir } from "./config.ts";
|
|
31
|
+
import { isPaused, setPaused, statusSnapshot, type StatusSnapshot } from "./daemon.ts";
|
|
32
|
+
import {
|
|
33
|
+
healthCheck,
|
|
34
|
+
isAlive,
|
|
35
|
+
livingDaemon,
|
|
36
|
+
stopDaemon,
|
|
37
|
+
type StopResult,
|
|
38
|
+
SYSTEMD_UNIT,
|
|
39
|
+
} from "./lifecycle.ts";
|
|
40
|
+
import { formatRss, rssBytesFromHealthz } from "./host.ts";
|
|
41
|
+
import {
|
|
42
|
+
readTickConfig,
|
|
43
|
+
readTickRuntimeStatus,
|
|
44
|
+
TICK_CONFIG_FILE,
|
|
45
|
+
type TickConfig,
|
|
46
|
+
type TickConfigResult,
|
|
47
|
+
} from "./orchestrator-tick.ts";
|
|
48
|
+
|
|
49
|
+
export const PANE_HALT_FILE = ".conductor-pane-halted";
|
|
50
|
+
export const DEFAULT_HERDR_UNIT = "herdr-fleet.service";
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Agent name assumed when no tick config names one. Mirrors
|
|
54
|
+
* `herdr-conductor`'s own `AGENT_NAME=${AGENT_NAME:-fleet}` default — a
|
|
55
|
+
* documented default, not a guess. An *invalid* config is a different thing
|
|
56
|
+
* and makes {@link stopConductorPane} refuse.
|
|
57
|
+
*/
|
|
58
|
+
export const DEFAULT_FLEET_AGENT_NAME = "fleet";
|
|
59
|
+
export const ARM_CHALLENGE_TIMEOUT_MS = 300_000;
|
|
60
|
+
|
|
61
|
+
export function telegramStateDir(): string {
|
|
62
|
+
const override = process.env["OMP_TELEGRAM_STATE_DIR"];
|
|
63
|
+
if (override !== undefined && override.length > 0) return override;
|
|
64
|
+
return join(homedir(), ".omp", "agent", "telegram");
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export function tickConfigSearchRoots(projectName?: string): string[] {
|
|
68
|
+
const roots: string[] = [stateDir()];
|
|
69
|
+
try {
|
|
70
|
+
const p = findProject(loadConfig(), projectName);
|
|
71
|
+
const parent = dirname(p.workspaceRoot);
|
|
72
|
+
if (parent !== roots[0]) roots.push(parent);
|
|
73
|
+
if (p.workspaceRoot !== roots[0] && p.workspaceRoot !== parent) roots.push(p.workspaceRoot);
|
|
74
|
+
} catch {
|
|
75
|
+
/* no config */
|
|
76
|
+
}
|
|
77
|
+
return roots;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export type ResolvedTick =
|
|
81
|
+
| { kind: "absent" }
|
|
82
|
+
| { kind: "invalid"; path: string; cwd: string; problem: string }
|
|
83
|
+
| { kind: "ok"; path: string; cwd: string; config: TickConfig };
|
|
84
|
+
|
|
85
|
+
export function resolveTickConfig(projectName?: string): ResolvedTick {
|
|
86
|
+
for (const cwd of tickConfigSearchRoots(projectName)) {
|
|
87
|
+
const r: TickConfigResult = readTickConfig(cwd);
|
|
88
|
+
if (r.kind === "ok") return { kind: "ok", path: r.path, cwd, config: r.config };
|
|
89
|
+
if (r.kind === "invalid") return { kind: "invalid", path: r.path, cwd, problem: r.problem };
|
|
90
|
+
}
|
|
91
|
+
return { kind: "absent" };
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export function armedMarkerPath(projectName?: string): string {
|
|
95
|
+
const tick = resolveTickConfig(projectName);
|
|
96
|
+
if (tick.kind === "ok" && tick.config.armedFile !== undefined) return tick.config.armedFile;
|
|
97
|
+
return join(stateDir(), "armed");
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export function disarmTicks(projectName?: string): { path: string; wasArmed: boolean } {
|
|
101
|
+
const path = armedMarkerPath(projectName);
|
|
102
|
+
const wasArmed = existsSync(path);
|
|
103
|
+
rmSync(path, { force: true });
|
|
104
|
+
return { path, wasArmed };
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export interface ArmResult {
|
|
108
|
+
path: string;
|
|
109
|
+
alreadyArmed: boolean;
|
|
110
|
+
owner: string;
|
|
111
|
+
challenge: string;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
export interface ArmDeps {
|
|
115
|
+
sendChallenge?: (token: string, owner: string, text: string) => Promise<void>;
|
|
116
|
+
waitForUserTurn?: (transcript: string, code: string, timeoutMs: number) => Promise<boolean>;
|
|
117
|
+
now?: () => number;
|
|
118
|
+
sleep?: (ms: number) => Promise<void>;
|
|
119
|
+
timeoutMs?: number;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promise<ArmResult> {
|
|
123
|
+
const tick = resolveTickConfig(projectName);
|
|
124
|
+
if (tick.kind === "invalid") {
|
|
125
|
+
throw new Error(`tick config invalid at ${tick.path}: ${tick.problem}`);
|
|
126
|
+
}
|
|
127
|
+
if (tick.kind === "absent") {
|
|
128
|
+
throw new Error(
|
|
129
|
+
`no ${TICK_CONFIG_FILE} under ${tickConfigSearchRoots(projectName).join(" or ")} — ` +
|
|
130
|
+
`nothing would read an arm marker; drop a tick config first`,
|
|
131
|
+
);
|
|
132
|
+
}
|
|
133
|
+
if (tick.config.armedFile === undefined) {
|
|
134
|
+
throw new Error(`${tick.path} has no armedFile — this heartbeat is ungated; add armedFile before arming`);
|
|
135
|
+
}
|
|
136
|
+
if (tick.config.accessFile === undefined) {
|
|
137
|
+
throw new Error(`${tick.path} has no accessFile — arm cannot prove an inbound channel without it`);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
const channel = readPairedChannel(tick.config.accessFile);
|
|
141
|
+
if (channel.kind === "down") {
|
|
142
|
+
throw new Error(
|
|
143
|
+
`escalation channel is not up (${tick.config.accessFile}): ${channel.reason} — ` +
|
|
144
|
+
`pair the bot (/telegram pair) and enable the bridge (/telegram on) before arming`,
|
|
145
|
+
);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
const token = readBotToken();
|
|
149
|
+
if (token === undefined) {
|
|
150
|
+
throw new Error(
|
|
151
|
+
`no TELEGRAM_BOT_TOKEN in ${join(telegramStateDir(), ".env")} — the arm challenge cannot send without it`,
|
|
152
|
+
);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
const transcript = newestSessionTranscript(tick.cwd);
|
|
156
|
+
if (transcript === undefined) {
|
|
157
|
+
throw new Error(
|
|
158
|
+
`no orchestrator session transcript under ${sessionDirForCwd(tick.cwd)} — ` +
|
|
159
|
+
`the inbound proof is read from a user turn there. Start the pane orchestrator, let it settle, then arm again`,
|
|
160
|
+
);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
const path = tick.config.armedFile;
|
|
164
|
+
const alreadyArmed = existsSync(path);
|
|
165
|
+
const code = makeChallengeCode();
|
|
166
|
+
const text =
|
|
167
|
+
`Fleet arming check. Reply to this chat with exactly:\n${code}\n` +
|
|
168
|
+
`Nothing will be dispatched until that reply is seen in the orchestrator session.`;
|
|
169
|
+
|
|
170
|
+
const send = deps.sendChallenge ?? sendTelegramMessage;
|
|
171
|
+
try {
|
|
172
|
+
await send(token, channel.owner, text);
|
|
173
|
+
} catch (err) {
|
|
174
|
+
throw new Error(
|
|
175
|
+
`arm: outbound sendMessage failed — NOT armed: ${err instanceof Error ? err.message : String(err)}`,
|
|
176
|
+
);
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
const timeoutMs = deps.timeoutMs ?? ARM_CHALLENGE_TIMEOUT_MS;
|
|
180
|
+
const wait = deps.waitForUserTurn ?? ((tr, c, ms) => waitForChallengeInTranscript(tr, c, ms, deps));
|
|
181
|
+
const seen = await wait(transcript, code, timeoutMs);
|
|
182
|
+
if (!seen) {
|
|
183
|
+
throw new Error(
|
|
184
|
+
`arm: the challenge never arrived as a user turn in time — NOT armed.\n` +
|
|
185
|
+
`Inbound Telegram is not reaching the omp session. Check, in order:\n` +
|
|
186
|
+
` * is the bridge polling? attach and run: /telegram status\n` +
|
|
187
|
+
` * is another process holding this bot token? Telegram allows exactly one\n` +
|
|
188
|
+
` getUpdates consumer and rejects the second with HTTP 409.\n` +
|
|
189
|
+
` * did you reply in the DM with the bot, not another chat?\n` +
|
|
190
|
+
`transcript: ${transcript}`,
|
|
191
|
+
);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
195
|
+
writeFileSync(path, `armed ${new Date().toISOString()} owner=${channel.owner}\n`, { mode: 0o600 });
|
|
196
|
+
return { path, alreadyArmed, owner: channel.owner, challenge: code };
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
export interface HoldResult {
|
|
200
|
+
wasPaused: boolean;
|
|
201
|
+
disarmed: { path: string; wasArmed: boolean };
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
export interface HaltResult {
|
|
205
|
+
hold: HoldResult;
|
|
206
|
+
stop: StopResult;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
export interface PaneStopResult {
|
|
210
|
+
pinPath: string;
|
|
211
|
+
/** How the pane was confirmed gone. Failures throw — never silent success. */
|
|
212
|
+
stopped: "herdr-agent" | "already-gone";
|
|
213
|
+
detail: string;
|
|
214
|
+
agentName: string;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
export interface HaltWithPaneResult extends HaltResult {
|
|
218
|
+
pane: PaneStopResult;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
export function hold(projectName?: string): HoldResult {
|
|
222
|
+
const wasPaused = isPaused();
|
|
223
|
+
setPaused(true);
|
|
224
|
+
return { wasPaused, disarmed: disarmTicks(projectName) };
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
export function releaseHold(): void {
|
|
228
|
+
setPaused(false);
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
export async function halt(projectName?: string): Promise<HaltResult> {
|
|
232
|
+
const held = hold(projectName);
|
|
233
|
+
const stop = await stopDaemon();
|
|
234
|
+
return { hold: held, stop };
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* Where `halt --pane` pins recovery off, or why we cannot say.
|
|
239
|
+
*
|
|
240
|
+
* `herdr-conductor`'s `recover.sh` reads exactly one path —
|
|
241
|
+
* `$FLEET_CWD/.conductor-pane-halted`. The only directory this package *knows*
|
|
242
|
+
* is `FLEET_CWD` is the one holding `.conductor-tick.json`: the heartbeat
|
|
243
|
+
* extension activates on that file in the session cwd, so the pane's cwd and
|
|
244
|
+
* the tick config's directory are the same by construction.
|
|
245
|
+
*
|
|
246
|
+
* An *invalid* tick config still names that directory, so it resolves. An
|
|
247
|
+
* *absent* one does not: the state dir is a guess, and a pin written there is
|
|
248
|
+
* one `recover.sh` never reads — the agent would be respawned seconds after a
|
|
249
|
+
* "successful" halt.
|
|
250
|
+
*/
|
|
251
|
+
export type ResolvedPaneHalt =
|
|
252
|
+
| { kind: "ok"; path: string }
|
|
253
|
+
| { kind: "unresolved"; reason: string };
|
|
254
|
+
|
|
255
|
+
export function resolvePaneHaltPath(projectName?: string): ResolvedPaneHalt {
|
|
256
|
+
const tick = resolveTickConfig(projectName);
|
|
257
|
+
if (tick.kind === "ok" || tick.kind === "invalid") {
|
|
258
|
+
return { kind: "ok", path: join(tick.cwd, PANE_HALT_FILE) };
|
|
259
|
+
}
|
|
260
|
+
return {
|
|
261
|
+
kind: "unresolved",
|
|
262
|
+
reason:
|
|
263
|
+
`no ${TICK_CONFIG_FILE} under ${tickConfigSearchRoots(projectName).join(" or ")} — ` +
|
|
264
|
+
`the pane's own directory (FLEET_CWD) is unknown, and ${PANE_HALT_FILE} anywhere else ` +
|
|
265
|
+
`is a file herdr-conductor never reads`,
|
|
266
|
+
};
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* {@link resolvePaneHaltPath}, refusing rather than guessing. Used by every
|
|
271
|
+
* path that writes, clears or promises a pin.
|
|
272
|
+
*/
|
|
273
|
+
export function paneHaltPath(projectName?: string): string {
|
|
274
|
+
const resolved = resolvePaneHaltPath(projectName);
|
|
275
|
+
if (resolved.kind === "unresolved") {
|
|
276
|
+
// Verb-neutral: this is also the `release-pane` path.
|
|
277
|
+
throw new Error(
|
|
278
|
+
`cannot locate the pane recovery pin — ${resolved.reason}. ` +
|
|
279
|
+
`Drop a tick config beside the pane, or stop the agent by hand; ` +
|
|
280
|
+
`\`halt\` without \`--pane\` still stops the dispatch daemon.`,
|
|
281
|
+
);
|
|
282
|
+
}
|
|
283
|
+
return resolved.path;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
export function pinPaneHalt(projectName?: string): { path: string } {
|
|
287
|
+
const path = paneHaltPath(projectName);
|
|
288
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
289
|
+
writeFileSync(
|
|
290
|
+
path,
|
|
291
|
+
[
|
|
292
|
+
`# Written by omp-conductor halt --pane at ${new Date().toISOString()}`,
|
|
293
|
+
`# herdr-conductor recover.sh must not resume the fleet agent while this file exists.`,
|
|
294
|
+
`# Clear with: omp-conductor release-pane (or rm this file)`,
|
|
295
|
+
"",
|
|
296
|
+
].join("\n"),
|
|
297
|
+
{ mode: 0o600 },
|
|
298
|
+
);
|
|
299
|
+
return { path };
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
export function clearPaneHalt(projectName?: string): { path: string; wasHalted: boolean } {
|
|
303
|
+
const path = paneHaltPath(projectName);
|
|
304
|
+
const wasHalted = existsSync(path);
|
|
305
|
+
rmSync(path, { force: true });
|
|
306
|
+
return { path, wasHalted };
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
export type HerdrStartResult =
|
|
310
|
+
| { kind: "active"; unit: string; recoveryReleased: boolean }
|
|
311
|
+
| { kind: "unmanaged"; unit: string; reason: string };
|
|
312
|
+
|
|
313
|
+
export interface HerdrStartDeps {
|
|
314
|
+
systemctl?: (args: string[]) => { ok: boolean; stdout: string; stderr: string; missing?: boolean };
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* Starts the dedicated Herdr fleet unit when it is installed. Hosts without
|
|
319
|
+
* systemd or without that optional unit keep the standalone daemon behaviour.
|
|
320
|
+
*/
|
|
321
|
+
export function startHerdrFleet(projectName?: string, deps: HerdrStartDeps = {}): HerdrStartResult {
|
|
322
|
+
const run: NonNullable<HerdrStartDeps["systemctl"]> =
|
|
323
|
+
deps.systemctl ??
|
|
324
|
+
((args: string[]) => {
|
|
325
|
+
const res = spawnSync("systemctl", args, { encoding: "utf8", timeout: 15_000, env: process.env });
|
|
326
|
+
if (res.error) {
|
|
327
|
+
const err = res.error as NodeJS.ErrnoException;
|
|
328
|
+
return {
|
|
329
|
+
ok: false,
|
|
330
|
+
stdout: "",
|
|
331
|
+
stderr: err.message,
|
|
332
|
+
...(err.code === "ENOENT" ? { missing: true } : {}),
|
|
333
|
+
};
|
|
334
|
+
}
|
|
335
|
+
return { ok: res.status === 0, stdout: res.stdout ?? "", stderr: res.stderr ?? "" };
|
|
336
|
+
});
|
|
337
|
+
|
|
338
|
+
const shown = run(["show", DEFAULT_HERDR_UNIT, "--property=LoadState", "--value"]);
|
|
339
|
+
if (shown.missing) return { kind: "unmanaged", unit: DEFAULT_HERDR_UNIT, reason: "no systemctl" };
|
|
340
|
+
if (!shown.ok) {
|
|
341
|
+
const detail = (shown.stderr.trim() || shown.stdout.trim() || "systemctl show failed").split("\n")[0]!;
|
|
342
|
+
throw new Error(`cannot inspect ${DEFAULT_HERDR_UNIT}: ${detail}`);
|
|
343
|
+
}
|
|
344
|
+
const loadState = shown.stdout.trim();
|
|
345
|
+
if (loadState === "not-found" || loadState === "") {
|
|
346
|
+
return { kind: "unmanaged", unit: DEFAULT_HERDR_UNIT, reason: "unit not installed" };
|
|
347
|
+
}
|
|
348
|
+
if (loadState !== "loaded") {
|
|
349
|
+
throw new Error(`${DEFAULT_HERDR_UNIT} is ${loadState}, not startable`);
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
const halt = resolvePaneHaltPath(projectName);
|
|
353
|
+
const recoveryReleased = halt.kind === "ok" && existsSync(halt.path);
|
|
354
|
+
if (halt.kind === "ok") rmSync(halt.path, { force: true });
|
|
355
|
+
|
|
356
|
+
const started = run(["start", DEFAULT_HERDR_UNIT]);
|
|
357
|
+
if (!started.ok) {
|
|
358
|
+
const detail = (started.stderr.trim() || started.stdout.trim() || "systemctl start failed").split("\n")[0]!;
|
|
359
|
+
throw new Error(`systemctl start ${DEFAULT_HERDR_UNIT} failed: ${detail}`);
|
|
360
|
+
}
|
|
361
|
+
const active = run(["is-active", DEFAULT_HERDR_UNIT]);
|
|
362
|
+
if (!active.ok || active.stdout.trim() !== "active") {
|
|
363
|
+
const detail = (active.stderr.trim() || active.stdout.trim() || "not active").split("\n")[0]!;
|
|
364
|
+
throw new Error(`${DEFAULT_HERDR_UNIT} did not become active: ${detail}`);
|
|
365
|
+
}
|
|
366
|
+
return { kind: "active", unit: DEFAULT_HERDR_UNIT, recoveryReleased };
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
export interface HerdrAgent {
|
|
370
|
+
name: string;
|
|
371
|
+
paneId: string;
|
|
372
|
+
agent?: string;
|
|
373
|
+
sessionPath?: string;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
/** The kill syscall, injectable so error mapping is testable without one. */
|
|
377
|
+
export type KillFn = (pid: number, sig: NodeJS.Signals | 0) => void;
|
|
378
|
+
|
|
379
|
+
const realKill: KillFn = (pid, sig) => {
|
|
380
|
+
process.kill(pid, sig);
|
|
381
|
+
};
|
|
382
|
+
|
|
383
|
+
/**
|
|
384
|
+
* What a failed `kill` proves.
|
|
385
|
+
*
|
|
386
|
+
* Only `ESRCH` — "no such process" — proves the process is gone. `EPERM` means
|
|
387
|
+
* it *exists* and merely is not ours to signal; every other errno is an answer
|
|
388
|
+
* we cannot read. Collapsing either into "gone" is how a live conductor gets
|
|
389
|
+
* reported as stopped.
|
|
390
|
+
*/
|
|
391
|
+
export function classifyKillError(err: unknown): "gone" | "unknown" {
|
|
392
|
+
return (err as NodeJS.ErrnoException | undefined)?.code === "ESRCH" ? "gone" : "unknown";
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
/** Whether `pid` is still running. Throws when the answer cannot be read. */
|
|
396
|
+
export function pidLiveness(pid: number, kill: KillFn = realKill): boolean {
|
|
397
|
+
try {
|
|
398
|
+
kill(pid, 0);
|
|
399
|
+
return true;
|
|
400
|
+
} catch (err) {
|
|
401
|
+
if (classifyKillError(err) === "gone") return false;
|
|
402
|
+
const code = (err as NodeJS.ErrnoException).code;
|
|
403
|
+
throw new Error(`cannot probe pid ${pid} (${code ?? String(err)}) — liveness unknown, not dead`);
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
/**
|
|
408
|
+
* Deliver `sig` to `pid`. An already-gone process is success — that is the
|
|
409
|
+
* outcome we wanted. Every other failure throws: an undelivered signal must
|
|
410
|
+
* never read as a kill.
|
|
411
|
+
*/
|
|
412
|
+
export function deliverSignal(pid: number, sig: NodeJS.Signals, kill: KillFn = realKill): boolean {
|
|
413
|
+
try {
|
|
414
|
+
kill(pid, sig);
|
|
415
|
+
return true;
|
|
416
|
+
} catch (err) {
|
|
417
|
+
if (classifyKillError(err) === "gone") return true;
|
|
418
|
+
const code = (err as NodeJS.ErrnoException).code;
|
|
419
|
+
throw new Error(`cannot send ${sig} to pid ${pid} (${code ?? String(err)})`);
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
export interface PaneStopDeps {
|
|
424
|
+
herdrBin?: string;
|
|
425
|
+
herdrSession?: string;
|
|
426
|
+
/** Deliver a signal. Return `false` or throw when delivery is unproven. */
|
|
427
|
+
signalPid?: (pid: number, sig: NodeJS.Signals) => boolean;
|
|
428
|
+
/** Liveness probe. MUST throw when it cannot tell — never answer `false`. */
|
|
429
|
+
isAlive?: (pid: number) => boolean;
|
|
430
|
+
/** Clock for deterministic deadline tests. */
|
|
431
|
+
now?: () => number;
|
|
432
|
+
sleep?: (ms: number) => Promise<void>;
|
|
433
|
+
listAgents?: () => Promise<HerdrAgent[]>;
|
|
434
|
+
panePids?: (paneId: string) => Promise<number[]>;
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
/**
|
|
438
|
+
* hold + pin recovery first + stop exact omp conductor agent.
|
|
439
|
+
* Never systemctl stop herdr-fleet.
|
|
440
|
+
*
|
|
441
|
+
* Pin is written even when the pane stop throws — recovery stays off so a
|
|
442
|
+
* failed kill cannot be undone by herdr-conductor respawning the agent. The
|
|
443
|
+
* command still fails (throws) so operators never see a green "halted" while
|
|
444
|
+
* the pane is still alive.
|
|
445
|
+
*/
|
|
446
|
+
export async function haltWithPane(
|
|
447
|
+
projectName?: string,
|
|
448
|
+
deps: PaneStopDeps = {},
|
|
449
|
+
): Promise<HaltWithPaneResult> {
|
|
450
|
+
const pin = pinPaneHalt(projectName);
|
|
451
|
+
const result = await halt(projectName);
|
|
452
|
+
// Throws on uncertainty or if the agent survives SIGTERM+SIGKILL.
|
|
453
|
+
const stopped = await stopConductorPane(projectName, deps);
|
|
454
|
+
return { ...result, pane: { pinPath: pin.path, ...stopped } };
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
/**
|
|
458
|
+
* Stop the configured conductor agent only. Recovery must already be pinned.
|
|
459
|
+
*
|
|
460
|
+
* Success returns only when the pane is confirmed gone (`herdr-agent` or
|
|
461
|
+
* `already-gone`). Any uncertainty **throws**: an unparseable tick config (the
|
|
462
|
+
* agent name is then a guess), herdr unreachable, output we cannot read as an
|
|
463
|
+
* explicit agent list, ambiguous identity, a live omp claim with no recognized
|
|
464
|
+
* PID, a liveness probe or signal delivery that fails (`EPERM` is "exists but
|
|
465
|
+
* not ours", never "dead"), or a process that survives SIGKILL. Callers must not treat a thrown
|
|
466
|
+
* error as "maybe stopped".
|
|
467
|
+
*/
|
|
468
|
+
export async function stopConductorPane(
|
|
469
|
+
projectName?: string,
|
|
470
|
+
deps: PaneStopDeps = {},
|
|
471
|
+
): Promise<Omit<PaneStopResult, "pinPath">> {
|
|
472
|
+
const tick = resolveTickConfig(projectName);
|
|
473
|
+
if (tick.kind === "invalid") {
|
|
474
|
+
// The file that names the agent does not parse, so the identity we would
|
|
475
|
+
// stop is a guess. Stopping the wrong pane is worse than refusing.
|
|
476
|
+
throw new Error(
|
|
477
|
+
`halt --pane: tick config invalid at ${tick.path} (${tick.problem}) — ` +
|
|
478
|
+
`the conductor agent name cannot be read, so refusing to stop a guessed identity. ` +
|
|
479
|
+
`Recovery pin was written; fix the config and re-run.`,
|
|
480
|
+
);
|
|
481
|
+
}
|
|
482
|
+
const agentName =
|
|
483
|
+
tick.kind === "ok" && tick.config.agentName !== undefined
|
|
484
|
+
? tick.config.agentName
|
|
485
|
+
: DEFAULT_FLEET_AGENT_NAME;
|
|
486
|
+
const sleep = deps.sleep ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)));
|
|
487
|
+
const now = deps.now ?? Date.now;
|
|
488
|
+
const signalPid = deps.signalPid ?? deliverSignal;
|
|
489
|
+
const alive = deps.isAlive ?? pidLiveness;
|
|
490
|
+
const pinned = "Recovery pin was written";
|
|
491
|
+
|
|
492
|
+
/** Any-alive across `pids`; an unreadable probe refuses instead of "dead". */
|
|
493
|
+
const anyAlive = (pids: number[], stage: string): boolean => {
|
|
494
|
+
try {
|
|
495
|
+
return pids.some((pid) => alive(pid));
|
|
496
|
+
} catch (err) {
|
|
497
|
+
throw new Error(
|
|
498
|
+
`halt --pane: cannot tell whether the conductor agent is still running ${stage} ` +
|
|
499
|
+
`(${err instanceof Error ? err.message : String(err)}) — refusing to report success. ` +
|
|
500
|
+
`${pinned}; check the pane by hand.`,
|
|
501
|
+
);
|
|
502
|
+
}
|
|
503
|
+
};
|
|
504
|
+
|
|
505
|
+
/** Send `sig` to every pid, refusing on any delivery we cannot prove. */
|
|
506
|
+
const deliver = (pids: number[], sig: NodeJS.Signals): void => {
|
|
507
|
+
for (const pid of pids) {
|
|
508
|
+
let delivered: boolean;
|
|
509
|
+
try {
|
|
510
|
+
delivered = signalPid(pid, sig);
|
|
511
|
+
} catch (err) {
|
|
512
|
+
throw new Error(
|
|
513
|
+
`halt --pane: ${sig} to pid ${pid} failed ` +
|
|
514
|
+
`(${err instanceof Error ? err.message : String(err)}) — refusing to report success. ${pinned}.`,
|
|
515
|
+
);
|
|
516
|
+
}
|
|
517
|
+
if (!delivered) {
|
|
518
|
+
throw new Error(
|
|
519
|
+
`halt --pane: ${sig} to pid ${pid} was not delivered — refusing to report success. ${pinned}.`,
|
|
520
|
+
);
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
};
|
|
524
|
+
|
|
525
|
+
let agents: HerdrAgent[];
|
|
526
|
+
try {
|
|
527
|
+
agents = deps.listAgents ? await deps.listAgents() : await herdrAgentList(deps);
|
|
528
|
+
} catch (err) {
|
|
529
|
+
throw new Error(
|
|
530
|
+
`halt --pane: herdr agent list failed — refusing to guess a pane ` +
|
|
531
|
+
`(${err instanceof Error ? err.message : String(err)}). ` +
|
|
532
|
+
`Recovery pin was written; fix herdr and re-run halt --pane or kill the agent by hand.`,
|
|
533
|
+
);
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
const matches = agents.filter((a) => a.name === agentName);
|
|
537
|
+
if (matches.length === 0) {
|
|
538
|
+
return { stopped: "already-gone", detail: `no herdr agent named ${agentName}`, agentName };
|
|
539
|
+
}
|
|
540
|
+
if (matches.length > 1) {
|
|
541
|
+
throw new Error(
|
|
542
|
+
`halt --pane: refusing to stop — ${matches.length} herdr agents named ${agentName} (not unique). ` +
|
|
543
|
+
`Recovery pin was written; resolve the identity and re-run or kill by hand.`,
|
|
544
|
+
);
|
|
545
|
+
}
|
|
546
|
+
const claim = matches[0]!;
|
|
547
|
+
if (claim.agent !== "omp") {
|
|
548
|
+
if (claim.agent === undefined || claim.agent === "") {
|
|
549
|
+
// Sticky name after exit — herdr still lists the claim but no live agent.
|
|
550
|
+
return {
|
|
551
|
+
stopped: "already-gone",
|
|
552
|
+
detail: `agent ${agentName} pane ${claim.paneId} has no live agent label (sticky name only)`,
|
|
553
|
+
agentName,
|
|
554
|
+
};
|
|
555
|
+
}
|
|
556
|
+
throw new Error(
|
|
557
|
+
`halt --pane: agent ${agentName} pane ${claim.paneId} is ${claim.agent}, not omp — refusing. ` +
|
|
558
|
+
`Recovery pin was written.`,
|
|
559
|
+
);
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
let pids: number[];
|
|
563
|
+
try {
|
|
564
|
+
pids = deps.panePids
|
|
565
|
+
? await deps.panePids(claim.paneId)
|
|
566
|
+
: await herdrOmpForegroundPids(claim.paneId, deps);
|
|
567
|
+
} catch (err) {
|
|
568
|
+
throw new Error(
|
|
569
|
+
`halt --pane: pane process-info failed for ${claim.paneId} ` +
|
|
570
|
+
`(${err instanceof Error ? err.message : String(err)}). ` +
|
|
571
|
+
`Live omp claim exists but PIDs are unknown — refusing to report success. Recovery pin was written.`,
|
|
572
|
+
);
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
const live = pids.filter((pid) => anyAlive([pid], `for pane ${claim.paneId}`));
|
|
576
|
+
if (live.length === 0) {
|
|
577
|
+
// Live agent=omp claim but no recognizable omp PID: we cannot prove gone.
|
|
578
|
+
// Treating this as already-gone would exit 0 while the pane may still run.
|
|
579
|
+
throw new Error(
|
|
580
|
+
`halt --pane: agent ${agentName} pane ${claim.paneId} is live omp but no omp foreground PID ` +
|
|
581
|
+
`was recognized — refusing to report success. Recovery pin was written; inspect the pane and kill by hand if needed.`,
|
|
582
|
+
);
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
deliver(live, "SIGTERM");
|
|
586
|
+
const softDeadline = now() + 10_000;
|
|
587
|
+
while (now() < softDeadline) {
|
|
588
|
+
if (!anyAlive(live, "after SIGTERM")) {
|
|
589
|
+
return {
|
|
590
|
+
stopped: "herdr-agent",
|
|
591
|
+
detail: `SIGTERM omp in agent ${agentName} pane ${claim.paneId} pids ${live.join(",")}`,
|
|
592
|
+
agentName,
|
|
593
|
+
};
|
|
594
|
+
}
|
|
595
|
+
await sleep(100);
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
deliver(live, "SIGKILL");
|
|
599
|
+
const hardDeadline = now() + 2_000;
|
|
600
|
+
while (now() < hardDeadline) {
|
|
601
|
+
if (!anyAlive(live, "after SIGKILL")) {
|
|
602
|
+
return {
|
|
603
|
+
stopped: "herdr-agent",
|
|
604
|
+
detail: `SIGKILL omp in agent ${agentName} pane ${claim.paneId} pids ${live.join(",")} after SIGTERM grace`,
|
|
605
|
+
agentName,
|
|
606
|
+
};
|
|
607
|
+
}
|
|
608
|
+
await sleep(50);
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
const survivors = live.filter((pid) => anyAlive([pid], "after SIGKILL"));
|
|
612
|
+
throw new Error(
|
|
613
|
+
`halt --pane: agent ${agentName} pane ${claim.paneId} still alive after SIGTERM+SIGKILL ` +
|
|
614
|
+
`(pids ${survivors.join(",")}). Recovery pin was written; kill by hand before trusting a quiet fleet.`,
|
|
615
|
+
);
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
async function herdrAgentList(deps: PaneStopDeps): Promise<HerdrAgent[]> {
|
|
619
|
+
const bin = deps.herdrBin ?? "herdr";
|
|
620
|
+
const session = deps.herdrSession ?? process.env["HERDR_SESSION"] ?? "fleet";
|
|
621
|
+
const res = spawnSync(bin, ["--session", session, "agent", "list"], {
|
|
622
|
+
encoding: "utf8",
|
|
623
|
+
timeout: 8_000,
|
|
624
|
+
env: process.env,
|
|
625
|
+
});
|
|
626
|
+
if (res.error) throw res.error;
|
|
627
|
+
if (res.status !== 0) {
|
|
628
|
+
throw new Error((res.stderr ?? res.stdout ?? `herdr exit ${String(res.status)}`).trim());
|
|
629
|
+
}
|
|
630
|
+
return parseHerdrAgentList(res.stdout ?? "");
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
export function parseHerdrAgentList(rawOutput: string): HerdrAgent[] {
|
|
634
|
+
// Only an explicit `agents: []` means no agents. Every malformed answer is
|
|
635
|
+
// uncertainty, never permission to report the conductor pane missing.
|
|
636
|
+
const raw = rawOutput.trim();
|
|
637
|
+
if (raw.length === 0) {
|
|
638
|
+
throw new Error("herdr agent list printed nothing — cannot tell whether the pane is running");
|
|
639
|
+
}
|
|
640
|
+
let parsed: { result?: { agents?: unknown }; agents?: unknown };
|
|
641
|
+
try {
|
|
642
|
+
parsed = JSON.parse(raw) as { result?: { agents?: unknown }; agents?: unknown };
|
|
643
|
+
} catch (err) {
|
|
644
|
+
throw new Error(
|
|
645
|
+
`herdr agent list output is not JSON (${err instanceof Error ? err.message : String(err)})`,
|
|
646
|
+
);
|
|
647
|
+
}
|
|
648
|
+
const list = parsed.result?.agents ?? parsed.agents;
|
|
649
|
+
if (!Array.isArray(list)) {
|
|
650
|
+
throw new Error("herdr agent list output has no `agents` array — unrecognized schema");
|
|
651
|
+
}
|
|
652
|
+
const out: HerdrAgent[] = [];
|
|
653
|
+
for (const row of list) {
|
|
654
|
+
if (row === null || typeof row !== "object") {
|
|
655
|
+
throw new Error("herdr agent list contains a non-object agent row — unrecognized schema");
|
|
656
|
+
}
|
|
657
|
+
const agent = row as { readonly [key: string]: unknown };
|
|
658
|
+
const name = agent["name"];
|
|
659
|
+
const paneId = agent["pane_id"];
|
|
660
|
+
if (typeof name !== "string" || typeof paneId !== "string") {
|
|
661
|
+
throw new Error(
|
|
662
|
+
"herdr agent list row is missing a string `name`/`pane_id` — " +
|
|
663
|
+
"cannot tell whether it is the conductor pane",
|
|
664
|
+
);
|
|
665
|
+
}
|
|
666
|
+
const rawAgent = agent["agent"];
|
|
667
|
+
if (rawAgent !== undefined && rawAgent !== null && typeof rawAgent !== "string") {
|
|
668
|
+
throw new Error(
|
|
669
|
+
`herdr agent list row for ${name} has a non-string \`agent\` (${typeof rawAgent}) — ` +
|
|
670
|
+
`cannot tell whether an agent is live`,
|
|
671
|
+
);
|
|
672
|
+
}
|
|
673
|
+
const liveAgent = typeof rawAgent === "string" ? rawAgent : undefined;
|
|
674
|
+
let sessionPath: string | undefined;
|
|
675
|
+
const session = agent["agent_session"];
|
|
676
|
+
if (session !== null && typeof session === "object") {
|
|
677
|
+
const value = session as { readonly [key: string]: unknown };
|
|
678
|
+
if (value["source"] === "herdr:omp" && typeof value["value"] === "string") {
|
|
679
|
+
sessionPath = value["value"];
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
out.push({
|
|
683
|
+
name,
|
|
684
|
+
paneId,
|
|
685
|
+
...(liveAgent === undefined ? {} : { agent: liveAgent }),
|
|
686
|
+
...(sessionPath === undefined ? {} : { sessionPath }),
|
|
687
|
+
});
|
|
688
|
+
}
|
|
689
|
+
return out;
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
async function herdrOmpForegroundPids(paneId: string, deps: PaneStopDeps): Promise<number[]> {
|
|
693
|
+
const bin = deps.herdrBin ?? "herdr";
|
|
694
|
+
const session = deps.herdrSession ?? process.env["HERDR_SESSION"] ?? "fleet";
|
|
695
|
+
const res = spawnSync(bin, ["--session", session, "pane", "process-info", "--pane", paneId], {
|
|
696
|
+
encoding: "utf8",
|
|
697
|
+
timeout: 8_000,
|
|
698
|
+
env: process.env,
|
|
699
|
+
});
|
|
700
|
+
if (res.error) throw res.error;
|
|
701
|
+
if (res.status !== 0) {
|
|
702
|
+
throw new Error((res.stderr ?? res.stdout ?? `process-info exit ${String(res.status)}`).trim());
|
|
703
|
+
}
|
|
704
|
+
const raw = (res.stdout ?? "").trim();
|
|
705
|
+
if (raw.length === 0) {
|
|
706
|
+
throw new Error(`herdr pane process-info printed nothing for ${paneId}`);
|
|
707
|
+
}
|
|
708
|
+
let parsed: { result?: { process_info?: ProcessInfo }; process_info?: ProcessInfo };
|
|
709
|
+
try {
|
|
710
|
+
parsed = JSON.parse(raw) as { result?: { process_info?: ProcessInfo }; process_info?: ProcessInfo };
|
|
711
|
+
} catch (err) {
|
|
712
|
+
throw new Error(
|
|
713
|
+
`herdr pane process-info output is not JSON (${err instanceof Error ? err.message : String(err)})`,
|
|
714
|
+
);
|
|
715
|
+
}
|
|
716
|
+
const info = parsed.result?.process_info ?? parsed.process_info;
|
|
717
|
+
if (info === undefined) {
|
|
718
|
+
throw new Error(`herdr pane process-info has no process_info for ${paneId} — unrecognized schema`);
|
|
719
|
+
}
|
|
720
|
+
return ompPidsFromProcessInfo(info);
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
interface ProcessInfo {
|
|
724
|
+
shell_pid?: number;
|
|
725
|
+
foreground_processes?: ForegroundProc[];
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
interface ForegroundProc {
|
|
729
|
+
pid?: number;
|
|
730
|
+
name?: string;
|
|
731
|
+
argv0?: string;
|
|
732
|
+
argv?: string[];
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
export function ompPidsFromProcessInfo(info: ProcessInfo): number[] {
|
|
736
|
+
const shell = typeof info.shell_pid === "number" ? info.shell_pid : undefined;
|
|
737
|
+
const out: number[] = [];
|
|
738
|
+
for (const proc of info.foreground_processes ?? []) {
|
|
739
|
+
if (typeof proc.pid !== "number" || !Number.isInteger(proc.pid) || proc.pid <= 1) continue;
|
|
740
|
+
if (shell !== undefined && proc.pid === shell) continue;
|
|
741
|
+
if (!isOmpProcess(proc)) continue;
|
|
742
|
+
out.push(proc.pid);
|
|
743
|
+
}
|
|
744
|
+
return out;
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
function isOmpProcess(proc: ForegroundProc): boolean {
|
|
748
|
+
const name = (proc.name ?? "").toLowerCase();
|
|
749
|
+
const argv0 = (proc.argv0 ?? "").toLowerCase();
|
|
750
|
+
const argv = (proc.argv ?? []).map((a) => a.toLowerCase());
|
|
751
|
+
if (name === "omp" || argv0 === "omp" || argv0.endsWith("/omp")) return true;
|
|
752
|
+
if (name === "bun" || argv0 === "bun" || argv0.endsWith("/bun")) {
|
|
753
|
+
if (argv.some((a) => a === "omp" || a.endsWith("/omp") || /(^|\/)omp$/.test(a))) return true;
|
|
754
|
+
}
|
|
755
|
+
return false;
|
|
756
|
+
}
|
|
757
|
+
|
|
758
|
+
// ---------------------------------------------------------------------------
|
|
759
|
+
// layered status
|
|
760
|
+
// ---------------------------------------------------------------------------
|
|
761
|
+
|
|
762
|
+
export type DispatchLayer = "running" | "paused" | "stopped";
|
|
763
|
+
export type TicksLayer =
|
|
764
|
+
| "armed"
|
|
765
|
+
| "disarmed"
|
|
766
|
+
| "no-heartbeat-config"
|
|
767
|
+
| "invalid-heartbeat-config"
|
|
768
|
+
| "ungated";
|
|
769
|
+
export type PaneLayer = "live" | "missing" | "unknown";
|
|
770
|
+
/** `unpinnable`: no tick config, so FLEET_CWD — the only path recovery reads — is unknown. */
|
|
771
|
+
export type RecoveryLayer = "pinned" | "clear" | "unpinnable";
|
|
772
|
+
export type HerdrLayer = "active" | "inactive" | "unknown";
|
|
773
|
+
export type TelegramLayer = "ok" | "degraded" | "down" | "unconfigured" | "unprobed";
|
|
774
|
+
|
|
775
|
+
export interface TelegramHealth {
|
|
776
|
+
kind: TelegramLayer;
|
|
777
|
+
detail?: string;
|
|
778
|
+
}
|
|
779
|
+
|
|
780
|
+
export interface FleetLayers {
|
|
781
|
+
dispatch: DispatchLayer;
|
|
782
|
+
ticks: TicksLayer;
|
|
783
|
+
ticksDetail?: string;
|
|
784
|
+
nextTickAt?: string;
|
|
785
|
+
pane: PaneLayer;
|
|
786
|
+
paneDetail?: string;
|
|
787
|
+
recovery: RecoveryLayer;
|
|
788
|
+
recoveryDetail?: string;
|
|
789
|
+
herdr: HerdrLayer;
|
|
790
|
+
herdrDetail?: string;
|
|
791
|
+
armedPath?: string;
|
|
792
|
+
tickConfigPath?: string;
|
|
793
|
+
paneHaltPath?: string;
|
|
794
|
+
paused: boolean;
|
|
795
|
+
daemon: { running: boolean; pid?: number; port?: number };
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
export function fleetLayers(projectName?: string): FleetLayers {
|
|
799
|
+
const rec = livingDaemon();
|
|
800
|
+
const paused = isPaused();
|
|
801
|
+
const dispatch: DispatchLayer = rec === undefined ? "stopped" : paused ? "paused" : "running";
|
|
802
|
+
|
|
803
|
+
const tick = resolveTickConfig(projectName);
|
|
804
|
+
let ticks: TicksLayer;
|
|
805
|
+
let ticksDetail: string | undefined;
|
|
806
|
+
let nextTickAt: string | undefined;
|
|
807
|
+
let armedPath: string | undefined;
|
|
808
|
+
let tickConfigPath: string | undefined;
|
|
809
|
+
if (tick.kind === "absent") {
|
|
810
|
+
ticks = "no-heartbeat-config";
|
|
811
|
+
} else if (tick.kind === "invalid") {
|
|
812
|
+
ticks = "invalid-heartbeat-config";
|
|
813
|
+
ticksDetail = `${tick.path}: ${tick.problem}`;
|
|
814
|
+
tickConfigPath = tick.path;
|
|
815
|
+
} else {
|
|
816
|
+
tickConfigPath = tick.path;
|
|
817
|
+
armedPath = tick.config.armedFile;
|
|
818
|
+
if (tick.config.armedFile === undefined) {
|
|
819
|
+
ticks = "ungated";
|
|
820
|
+
ticksDetail = "no armedFile in tick config — heartbeat sends without an arm marker";
|
|
821
|
+
} else if (existsSync(tick.config.armedFile)) {
|
|
822
|
+
ticks = "armed";
|
|
823
|
+
ticksDetail = tick.config.armedFile;
|
|
824
|
+
} else {
|
|
825
|
+
ticks = "disarmed";
|
|
826
|
+
ticksDetail = tick.config.armedFile;
|
|
827
|
+
}
|
|
828
|
+
const runtime = readTickRuntimeStatus(tick.cwd);
|
|
829
|
+
if (runtime !== undefined && isAlive(runtime.pid)) nextTickAt = runtime.nextTickAt;
|
|
830
|
+
}
|
|
831
|
+
if (armedPath === undefined) armedPath = armedMarkerPath(projectName);
|
|
832
|
+
|
|
833
|
+
// Status reports; it never refuses. An unresolvable pin location is its own
|
|
834
|
+
// answer — "clear" would claim recovery is armed and ready to be pinned.
|
|
835
|
+
const resolvedHalt = resolvePaneHaltPath(projectName);
|
|
836
|
+
const haltPath = resolvedHalt.kind === "ok" ? resolvedHalt.path : undefined;
|
|
837
|
+
const recoveryPinned = haltPath !== undefined && existsSync(haltPath);
|
|
838
|
+
const omp = probeOmpPane(projectName);
|
|
839
|
+
let pane: PaneLayer;
|
|
840
|
+
let paneDetail: string | undefined;
|
|
841
|
+
if (omp.kind === "live") {
|
|
842
|
+
pane = "live";
|
|
843
|
+
paneDetail = omp.summary;
|
|
844
|
+
} else if (omp.kind === "missing") {
|
|
845
|
+
pane = "missing";
|
|
846
|
+
} else {
|
|
847
|
+
pane = "unknown";
|
|
848
|
+
paneDetail = omp.reason;
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
const herdr = probeHerdrUnit();
|
|
852
|
+
|
|
853
|
+
return {
|
|
854
|
+
dispatch,
|
|
855
|
+
ticks,
|
|
856
|
+
...(ticksDetail === undefined ? {} : { ticksDetail }),
|
|
857
|
+
...(nextTickAt === undefined ? {} : { nextTickAt }),
|
|
858
|
+
pane,
|
|
859
|
+
...(paneDetail === undefined ? {} : { paneDetail }),
|
|
860
|
+
recovery: haltPath === undefined ? "unpinnable" : recoveryPinned ? "pinned" : "clear",
|
|
861
|
+
...(haltPath === undefined
|
|
862
|
+
? { recoveryDetail: resolvedHalt.kind === "unresolved" ? resolvedHalt.reason : undefined }
|
|
863
|
+
: recoveryPinned
|
|
864
|
+
? { recoveryDetail: haltPath }
|
|
865
|
+
: {}),
|
|
866
|
+
herdr: herdr.kind,
|
|
867
|
+
...(herdr.detail === undefined ? {} : { herdrDetail: herdr.detail }),
|
|
868
|
+
armedPath,
|
|
869
|
+
...(tickConfigPath === undefined ? {} : { tickConfigPath }),
|
|
870
|
+
...(haltPath === undefined ? {} : { paneHaltPath: haltPath }),
|
|
871
|
+
paused,
|
|
872
|
+
daemon: rec === undefined ? { running: false } : { running: true, pid: rec.pid, port: rec.port },
|
|
873
|
+
};
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
export function formatFleetStatus(
|
|
877
|
+
s: StatusSnapshot,
|
|
878
|
+
layers: FleetLayers,
|
|
879
|
+
daemonHealth?: { ok: boolean; body?: string },
|
|
880
|
+
telegram: TelegramHealth = { kind: "unprobed" },
|
|
881
|
+
now = Date.now(),
|
|
882
|
+
): string {
|
|
883
|
+
const tickLine =
|
|
884
|
+
layers.ticksDetail === undefined
|
|
885
|
+
? `ticks ${layers.ticks}`
|
|
886
|
+
: `ticks ${layers.ticks} (${layers.ticksDetail})`;
|
|
887
|
+
let nextTickLine: string | undefined;
|
|
888
|
+
if (layers.nextTickAt !== undefined) {
|
|
889
|
+
const delta = Date.parse(layers.nextTickAt) - now;
|
|
890
|
+
const minutes = Math.max(1, Math.ceil(Math.abs(delta) / 60_000));
|
|
891
|
+
nextTickLine =
|
|
892
|
+
`next tick ${layers.nextTickAt} ` +
|
|
893
|
+
`(${delta >= 0 ? `in ${minutes}m` : `overdue by ${minutes}m`})`;
|
|
894
|
+
}
|
|
895
|
+
const paneLine =
|
|
896
|
+
layers.paneDetail === undefined
|
|
897
|
+
? `pane ${layers.pane}`
|
|
898
|
+
: `pane ${layers.pane} (${layers.paneDetail})`;
|
|
899
|
+
const recoveryLine =
|
|
900
|
+
layers.recoveryDetail === undefined
|
|
901
|
+
? `recovery ${layers.recovery}`
|
|
902
|
+
: `recovery ${layers.recovery} (${layers.recoveryDetail})`;
|
|
903
|
+
const herdrLine =
|
|
904
|
+
layers.herdrDetail === undefined
|
|
905
|
+
? `herdr ${layers.herdr}`
|
|
906
|
+
: `herdr ${layers.herdr} (${layers.herdrDetail})`;
|
|
907
|
+
const telegramLine =
|
|
908
|
+
telegram.detail === undefined
|
|
909
|
+
? `telegram ${telegram.kind}`
|
|
910
|
+
: `telegram ${telegram.kind} (${telegram.detail})`;
|
|
911
|
+
|
|
912
|
+
let daemonBlock: string;
|
|
913
|
+
if (!layers.daemon.running || layers.daemon.pid === undefined) {
|
|
914
|
+
daemonBlock = "daemon not running";
|
|
915
|
+
} else {
|
|
916
|
+
const hz =
|
|
917
|
+
daemonHealth === undefined
|
|
918
|
+
? "unprobed"
|
|
919
|
+
: daemonHealth.ok
|
|
920
|
+
? `ok ${daemonHealth.body ?? ""}`.trimEnd()
|
|
921
|
+
: "unreachable — the process is up but not serving";
|
|
922
|
+
const rss = rssBytesFromHealthz(daemonHealth?.body);
|
|
923
|
+
daemonBlock = [
|
|
924
|
+
"daemon",
|
|
925
|
+
` pid ${layers.daemon.pid}`,
|
|
926
|
+
` port ${layers.daemon.port ?? "?"}`,
|
|
927
|
+
...(rss === undefined ? [] : [` rss ${formatRss(rss)}`]),
|
|
928
|
+
` healthz ${hz}`,
|
|
929
|
+
` unit ${SYSTEMD_UNIT}`,
|
|
930
|
+
].join("\n");
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
return [
|
|
934
|
+
`dispatch ${layers.dispatch}`,
|
|
935
|
+
tickLine,
|
|
936
|
+
...(nextTickLine === undefined ? [] : [nextTickLine]),
|
|
937
|
+
paneLine,
|
|
938
|
+
recoveryLine,
|
|
939
|
+
herdrLine,
|
|
940
|
+
telegramLine,
|
|
941
|
+
daemonBlock,
|
|
942
|
+
"",
|
|
943
|
+
formatProjectBody(s),
|
|
944
|
+
].join("\n");
|
|
945
|
+
}
|
|
946
|
+
|
|
947
|
+
function formatProjectBody(s: StatusSnapshot): string {
|
|
948
|
+
const lines = [
|
|
949
|
+
`project ${s.project}${s.paused ? " (PAUSED)" : ""}`,
|
|
950
|
+
`config ${s.configPath}`,
|
|
951
|
+
`state ${s.stateDir}`,
|
|
952
|
+
"",
|
|
953
|
+
"caps",
|
|
954
|
+
` workers ${s.liveWorkers} / ${s.caps.maxConcurrentWorkers}`,
|
|
955
|
+
` issues today ${s.runsToday}`,
|
|
956
|
+
s.caps.dailySpendUsd === null
|
|
957
|
+
? ` spend today $${s.spendTodayUsd.toFixed(2)} (no daily cap)`
|
|
958
|
+
: ` spend today $${s.spendTodayUsd.toFixed(2)} / $${s.caps.dailySpendUsd.toFixed(2)}`,
|
|
959
|
+
` worker max turns ${s.caps.workerMaxTurns}`,
|
|
960
|
+
` worker wall clock ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
|
|
961
|
+
` attempts per issue ${s.caps.maxAttemptsPerIssue}`,
|
|
962
|
+
"",
|
|
963
|
+
];
|
|
964
|
+
if (s.activeRuns.length === 0) {
|
|
965
|
+
lines.push("active runs (none)");
|
|
966
|
+
} else {
|
|
967
|
+
lines.push("active runs");
|
|
968
|
+
for (const r of s.activeRuns) {
|
|
969
|
+
lines.push(
|
|
970
|
+
` #${r.issue} ${r.repo} ${r.state} attempt ${r.attempt} ` +
|
|
971
|
+
`${r.turns} turns $${r.spendUsd.toFixed(2)} ${r.branch}` +
|
|
972
|
+
(r.prUrl ? ` ${r.prUrl}` : ""),
|
|
973
|
+
);
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
if (s.liveWorkers > 0) {
|
|
977
|
+
lines.push(
|
|
978
|
+
"",
|
|
979
|
+
`deploy ${s.liveWorkers} live worker(s) — restart salvages dirty trees then orphans the rows; ` +
|
|
980
|
+
`hold and wait for workers 0/${s.caps.maxConcurrentWorkers} when you can drain instead`,
|
|
981
|
+
);
|
|
982
|
+
}
|
|
983
|
+
return lines.join("\n");
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
export async function renderStatus(projectName?: string): Promise<string> {
|
|
987
|
+
const s = statusSnapshot(projectName);
|
|
988
|
+
const layers = fleetLayers(projectName);
|
|
989
|
+
const rec = livingDaemon();
|
|
990
|
+
const [health, telegram] = await Promise.all([
|
|
991
|
+
rec === undefined ? undefined : healthCheck(rec.port),
|
|
992
|
+
probeTelegramHealth(projectName),
|
|
993
|
+
]);
|
|
994
|
+
return formatFleetStatus(s, layers, health, telegram);
|
|
995
|
+
}
|
|
996
|
+
|
|
997
|
+
// ---------------------------------------------------------------------------
|
|
998
|
+
// arm proof helpers
|
|
999
|
+
// ---------------------------------------------------------------------------
|
|
1000
|
+
|
|
1001
|
+
type Channel = { kind: "up"; owner: string } | { kind: "down"; reason: string };
|
|
1002
|
+
|
|
1003
|
+
function readPairedChannel(path: string): Channel {
|
|
1004
|
+
let parsed: unknown;
|
|
1005
|
+
try {
|
|
1006
|
+
parsed = JSON.parse(readFileSync(path, "utf8"));
|
|
1007
|
+
} catch {
|
|
1008
|
+
return { kind: "down", reason: "unreadable or missing access.json" };
|
|
1009
|
+
}
|
|
1010
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
1011
|
+
return { kind: "down", reason: "access.json is not an object" };
|
|
1012
|
+
}
|
|
1013
|
+
const access = parsed as { readonly [key: string]: unknown };
|
|
1014
|
+
if (access["enabled"] !== true) return { kind: "down", reason: "bridge disabled (enabled !== true)" };
|
|
1015
|
+
const allowFrom = access["allowFrom"];
|
|
1016
|
+
if (!Array.isArray(allowFrom) || allowFrom.length !== 1) {
|
|
1017
|
+
return { kind: "down", reason: "allowFrom must hold exactly one paired owner id" };
|
|
1018
|
+
}
|
|
1019
|
+
const owner = allowFrom[0];
|
|
1020
|
+
if (typeof owner !== "string" && typeof owner !== "number") {
|
|
1021
|
+
return { kind: "down", reason: "paired owner id is not a string or number" };
|
|
1022
|
+
}
|
|
1023
|
+
return { kind: "up", owner: String(owner) };
|
|
1024
|
+
}
|
|
1025
|
+
|
|
1026
|
+
function readBotToken(): string | undefined {
|
|
1027
|
+
const env = process.env["TELEGRAM_BOT_TOKEN"];
|
|
1028
|
+
if (env !== undefined && env.length > 0) return env;
|
|
1029
|
+
const file = join(telegramStateDir(), ".env");
|
|
1030
|
+
if (!existsSync(file)) return undefined;
|
|
1031
|
+
try {
|
|
1032
|
+
for (const line of readFileSync(file, "utf8").split("\n")) {
|
|
1033
|
+
const m = /^TELEGRAM_BOT_TOKEN=(.*)$/.exec(line.trim());
|
|
1034
|
+
if (m?.[1]) return m[1].replace(/^["']|["']$/g, "");
|
|
1035
|
+
}
|
|
1036
|
+
} catch {
|
|
1037
|
+
return undefined;
|
|
1038
|
+
}
|
|
1039
|
+
return undefined;
|
|
1040
|
+
}
|
|
1041
|
+
|
|
1042
|
+
export async function probeTelegramHealth(
|
|
1043
|
+
projectName?: string,
|
|
1044
|
+
request: (input: string, init?: RequestInit) => Promise<Response> = fetch,
|
|
1045
|
+
): Promise<TelegramHealth> {
|
|
1046
|
+
const tick = resolveTickConfig(projectName);
|
|
1047
|
+
const accessPath =
|
|
1048
|
+
tick.kind === "ok" && tick.config.accessFile !== undefined
|
|
1049
|
+
? tick.config.accessFile
|
|
1050
|
+
: join(telegramStateDir(), "access.json");
|
|
1051
|
+
const channel = readPairedChannel(accessPath);
|
|
1052
|
+
const token = readBotToken();
|
|
1053
|
+
if (token === undefined) {
|
|
1054
|
+
return {
|
|
1055
|
+
kind: "unconfigured",
|
|
1056
|
+
detail: `no TELEGRAM_BOT_TOKEN; inbound ${channel.kind === "up" ? "configured" : channel.reason}`,
|
|
1057
|
+
};
|
|
1058
|
+
}
|
|
1059
|
+
|
|
1060
|
+
let response: Response;
|
|
1061
|
+
let body: unknown;
|
|
1062
|
+
try {
|
|
1063
|
+
response = await request(`https://api.telegram.org/bot${token}/getMe`, {
|
|
1064
|
+
signal: AbortSignal.timeout(5_000),
|
|
1065
|
+
});
|
|
1066
|
+
body = await response.json();
|
|
1067
|
+
} catch {
|
|
1068
|
+
return {
|
|
1069
|
+
kind: "down",
|
|
1070
|
+
detail: `Telegram API request failed; inbound ${channel.kind === "up" ? "configured" : channel.reason}`,
|
|
1071
|
+
};
|
|
1072
|
+
}
|
|
1073
|
+
|
|
1074
|
+
if (body === null || typeof body !== "object" || Array.isArray(body)) {
|
|
1075
|
+
return { kind: "down", detail: "Telegram API returned an invalid response" };
|
|
1076
|
+
}
|
|
1077
|
+
const result = body as Record<string, unknown>;
|
|
1078
|
+
if (!response.ok || result["ok"] !== true) {
|
|
1079
|
+
const description =
|
|
1080
|
+
typeof result["description"] === "string" ? result["description"] : `HTTP ${response.status}`;
|
|
1081
|
+
return { kind: "down", detail: `${description}; inbound ${channel.kind === "up" ? "configured" : channel.reason}` };
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
const user =
|
|
1085
|
+
result["result"] !== null && typeof result["result"] === "object" && !Array.isArray(result["result"])
|
|
1086
|
+
? (result["result"] as Record<string, unknown>)
|
|
1087
|
+
: undefined;
|
|
1088
|
+
const username = typeof user?.["username"] === "string" ? `@${user["username"]}` : "authenticated";
|
|
1089
|
+
if (channel.kind === "down") return { kind: "degraded", detail: `${username}; inbound ${channel.reason}` };
|
|
1090
|
+
return { kind: "ok", detail: `${username}; inbound configured` };
|
|
1091
|
+
}
|
|
1092
|
+
|
|
1093
|
+
export function sessionDirForCwd(cwd: string): string {
|
|
1094
|
+
const home = homedir();
|
|
1095
|
+
const slug = cwd.startsWith(home) ? cwd.slice(home.length) : cwd;
|
|
1096
|
+
return join(home, ".omp", "agent", "sessions", slug.replaceAll("/", "-"));
|
|
1097
|
+
}
|
|
1098
|
+
|
|
1099
|
+
function newestSessionTranscript(cwd: string): string | undefined {
|
|
1100
|
+
const dir = sessionDirForCwd(cwd);
|
|
1101
|
+
if (!existsSync(dir)) return undefined;
|
|
1102
|
+
let best: { path: string; mtime: number } | undefined;
|
|
1103
|
+
for (const name of readdirSync(dir)) {
|
|
1104
|
+
if (!name.endsWith(".jsonl")) continue;
|
|
1105
|
+
const path = join(dir, name);
|
|
1106
|
+
try {
|
|
1107
|
+
const mtime = statSync(path).mtimeMs;
|
|
1108
|
+
if (best === undefined || mtime > best.mtime) best = { path, mtime };
|
|
1109
|
+
} catch {
|
|
1110
|
+
/* race */
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1113
|
+
return best?.path;
|
|
1114
|
+
}
|
|
1115
|
+
|
|
1116
|
+
function makeChallengeCode(): string {
|
|
1117
|
+
const bytes = new Uint8Array(4);
|
|
1118
|
+
crypto.getRandomValues(bytes);
|
|
1119
|
+
const hex = [...bytes].map((b) => b.toString(16).padStart(2, "0")).join("").toUpperCase();
|
|
1120
|
+
return `FLEET-${hex}`;
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
async function sendTelegramMessage(token: string, owner: string, text: string): Promise<void> {
|
|
1124
|
+
const url = `https://api.telegram.org/bot${token}/sendMessage`;
|
|
1125
|
+
const body = new URLSearchParams({ chat_id: owner, text });
|
|
1126
|
+
const res = await fetch(url, { method: "POST", body, signal: AbortSignal.timeout(20_000) });
|
|
1127
|
+
const json = (await res.json()) as { ok?: boolean; description?: string };
|
|
1128
|
+
if (!res.ok || json.ok !== true) {
|
|
1129
|
+
throw new Error(json.description ?? `HTTP ${res.status}`);
|
|
1130
|
+
}
|
|
1131
|
+
}
|
|
1132
|
+
|
|
1133
|
+
async function waitForChallengeInTranscript(
|
|
1134
|
+
transcript: string,
|
|
1135
|
+
code: string,
|
|
1136
|
+
timeoutMs: number,
|
|
1137
|
+
deps: ArmDeps,
|
|
1138
|
+
): Promise<boolean> {
|
|
1139
|
+
const now = deps.now ?? Date.now;
|
|
1140
|
+
const sleep = deps.sleep ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)));
|
|
1141
|
+
const deadline = now() + timeoutMs;
|
|
1142
|
+
while (now() < deadline) {
|
|
1143
|
+
if (await transcriptHasUserCode(transcript, code)) return true;
|
|
1144
|
+
await sleep(5_000);
|
|
1145
|
+
}
|
|
1146
|
+
return false;
|
|
1147
|
+
}
|
|
1148
|
+
|
|
1149
|
+
export async function transcriptHasUserCode(path: string, code: string): Promise<boolean> {
|
|
1150
|
+
if (!existsSync(path)) return false;
|
|
1151
|
+
const rl = createInterface({ input: createReadStream(path, { encoding: "utf8" }), crlfDelay: Infinity });
|
|
1152
|
+
try {
|
|
1153
|
+
for await (const line of rl) {
|
|
1154
|
+
if (line.length === 0) continue;
|
|
1155
|
+
let row: unknown;
|
|
1156
|
+
try {
|
|
1157
|
+
row = JSON.parse(line);
|
|
1158
|
+
} catch {
|
|
1159
|
+
continue;
|
|
1160
|
+
}
|
|
1161
|
+
if (row === null || typeof row !== "object") continue;
|
|
1162
|
+
const rec = row as { readonly [key: string]: unknown };
|
|
1163
|
+
if (rec["type"] !== "message") continue;
|
|
1164
|
+
const message = rec["message"];
|
|
1165
|
+
if (message === null || typeof message !== "object") continue;
|
|
1166
|
+
const msg = message as { readonly [key: string]: unknown };
|
|
1167
|
+
if (msg["role"] !== "user") continue;
|
|
1168
|
+
const content = msg["content"];
|
|
1169
|
+
if (!Array.isArray(content)) continue;
|
|
1170
|
+
for (const part of content) {
|
|
1171
|
+
if (part === null || typeof part !== "object") continue;
|
|
1172
|
+
const p = part as { readonly [key: string]: unknown };
|
|
1173
|
+
if (p["type"] === "text" && typeof p["text"] === "string" && p["text"].includes(code)) {
|
|
1174
|
+
return true;
|
|
1175
|
+
}
|
|
1176
|
+
}
|
|
1177
|
+
}
|
|
1178
|
+
} finally {
|
|
1179
|
+
rl.close();
|
|
1180
|
+
}
|
|
1181
|
+
return false;
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
// ---------------------------------------------------------------------------
|
|
1185
|
+
// probes
|
|
1186
|
+
// ---------------------------------------------------------------------------
|
|
1187
|
+
|
|
1188
|
+
function probeHerdrUnit(unit = DEFAULT_HERDR_UNIT): { kind: HerdrLayer; detail?: string } {
|
|
1189
|
+
try {
|
|
1190
|
+
const res = spawnSync("systemctl", ["is-active", unit], {
|
|
1191
|
+
encoding: "utf8",
|
|
1192
|
+
timeout: 5_000,
|
|
1193
|
+
env: process.env,
|
|
1194
|
+
});
|
|
1195
|
+
if (res.error) {
|
|
1196
|
+
const err = res.error as NodeJS.ErrnoException;
|
|
1197
|
+
if (err.code === "ENOENT") return { kind: "unknown", detail: "no systemctl" };
|
|
1198
|
+
return { kind: "unknown", detail: err.message };
|
|
1199
|
+
}
|
|
1200
|
+
const out = (res.stdout ?? "").trim();
|
|
1201
|
+
if (out === "active") return { kind: "active", detail: unit };
|
|
1202
|
+
if (out === "inactive" || out === "failed" || out === "dead") {
|
|
1203
|
+
return { kind: "inactive", detail: `${unit} ${out}` };
|
|
1204
|
+
}
|
|
1205
|
+
return { kind: "unknown", detail: `${unit} ${out || `exit ${String(res.status)}`}` };
|
|
1206
|
+
} catch (err) {
|
|
1207
|
+
return { kind: "unknown", detail: err instanceof Error ? err.message : String(err) };
|
|
1208
|
+
}
|
|
1209
|
+
}
|
|
1210
|
+
|
|
1211
|
+
export function paneLayerFromAgents(
|
|
1212
|
+
agents: HerdrAgent[],
|
|
1213
|
+
agentName: string,
|
|
1214
|
+
):
|
|
1215
|
+
| { kind: "live"; summary: string }
|
|
1216
|
+
| { kind: "missing" }
|
|
1217
|
+
| { kind: "unknown"; reason: string } {
|
|
1218
|
+
const matches = agents.filter((agent) => agent.name === agentName);
|
|
1219
|
+
if (matches.length === 0) return { kind: "missing" };
|
|
1220
|
+
if (matches.length > 1) {
|
|
1221
|
+
return { kind: "unknown", reason: `${matches.length} herdr agents named ${agentName}` };
|
|
1222
|
+
}
|
|
1223
|
+
const match = matches[0]!;
|
|
1224
|
+
if (match.agent === "omp") {
|
|
1225
|
+
return { kind: "live", summary: `agent ${agentName} pane ${match.paneId}` };
|
|
1226
|
+
}
|
|
1227
|
+
if (match.agent === undefined || match.agent === "") return { kind: "missing" };
|
|
1228
|
+
return {
|
|
1229
|
+
kind: "unknown",
|
|
1230
|
+
reason: `agent ${agentName} pane ${match.paneId} is ${match.agent}, not omp`,
|
|
1231
|
+
};
|
|
1232
|
+
}
|
|
1233
|
+
|
|
1234
|
+
function probeOmpPane(
|
|
1235
|
+
projectName?: string,
|
|
1236
|
+
):
|
|
1237
|
+
| { kind: "live"; summary: string }
|
|
1238
|
+
| { kind: "missing" }
|
|
1239
|
+
| { kind: "unknown"; reason: string } {
|
|
1240
|
+
const tick = resolveTickConfig(projectName);
|
|
1241
|
+
if (tick.kind === "invalid") {
|
|
1242
|
+
return {
|
|
1243
|
+
kind: "unknown",
|
|
1244
|
+
reason:
|
|
1245
|
+
`tick config invalid at ${tick.path} (${tick.problem}) — ` +
|
|
1246
|
+
"the conductor agent name cannot be read",
|
|
1247
|
+
};
|
|
1248
|
+
}
|
|
1249
|
+
const agentName =
|
|
1250
|
+
tick.kind === "ok" ? (tick.config.agentName ?? DEFAULT_FLEET_AGENT_NAME) : DEFAULT_FLEET_AGENT_NAME;
|
|
1251
|
+
const session = process.env["HERDR_SESSION"] ?? "fleet";
|
|
1252
|
+
try {
|
|
1253
|
+
const res = spawnSync("herdr", ["--session", session, "agent", "list"], {
|
|
1254
|
+
encoding: "utf8",
|
|
1255
|
+
timeout: 8_000,
|
|
1256
|
+
env: process.env,
|
|
1257
|
+
});
|
|
1258
|
+
if (res.error) {
|
|
1259
|
+
const err = res.error as NodeJS.ErrnoException;
|
|
1260
|
+
if (err.code === "ENOENT") return { kind: "unknown", reason: "no herdr" };
|
|
1261
|
+
return { kind: "unknown", reason: err.message };
|
|
1262
|
+
}
|
|
1263
|
+
if (res.status !== 0) {
|
|
1264
|
+
const detail = (res.stderr ?? res.stdout ?? `herdr exit ${String(res.status)}`).trim();
|
|
1265
|
+
return { kind: "unknown", reason: detail };
|
|
1266
|
+
}
|
|
1267
|
+
return paneLayerFromAgents(parseHerdrAgentList(res.stdout ?? ""), agentName);
|
|
1268
|
+
} catch (err) {
|
|
1269
|
+
return { kind: "unknown", reason: err instanceof Error ? err.message : String(err) };
|
|
1270
|
+
}
|
|
1271
|
+
}
|