omp-conductor 0.14.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +336 -169
- package/package.json +8 -5
- package/schema/config.schema.json +609 -0
- package/src/board.ts +19 -32
- package/src/brief-upgrade.ts +1 -1
- package/src/briefs/orchestrator.md +9 -5
- package/src/briefs/policy.md +4 -4
- package/src/briefs/probes/gates.md +51 -0
- package/src/briefs/probes/project-context.md +59 -0
- package/src/briefs/probes/release-procedure.md +81 -0
- package/src/cli.ts +235 -199
- package/src/config-schema.ts +352 -0
- package/src/config.ts +1046 -796
- package/src/confinement.ts +54 -0
- package/src/daemon.ts +442 -374
- package/src/escalate.ts +43 -3
- package/src/fleet.ts +317 -43
- package/src/generate-schema.ts +21 -0
- package/src/graph.ts +3 -3
- package/src/host.ts +16 -0
- package/src/omp.ts +21 -1
- package/src/orchestrator-tick.ts +298 -39
- package/src/privileged.ts +264 -0
- package/src/reports.ts +1 -1
- package/src/session-host.ts +3 -0
- package/src/setup-host.ts +209 -24
- package/src/setup-install.ts +320 -0
- package/src/setup-probe.ts +412 -0
- package/src/{plugin.ts → setup-wizard.ts} +790 -465
- package/src/setup.ts +264 -20
- package/src/types.ts +2 -2
- package/src/upgrade.ts +44 -10
- package/src/verbs/server.ts +32 -9
- package/src/wizard-ui.ts +249 -0
- package/src/worker.ts +6 -1
- package/skills/conductor-onboarding/SKILL.md +0 -748
- package/skills/conductor-update/SKILL.md +0 -51
package/src/escalate.ts
CHANGED
|
@@ -269,7 +269,7 @@ export function createEscalator(
|
|
|
269
269
|
// the tick re-raises it. A *report* has no such source — it exists
|
|
270
270
|
// once, in the model's head, and nothing regenerates it — which is
|
|
271
271
|
// why that one needed a ledger and this one does not.
|
|
272
|
-
await sendTelegram(token, chatId, text);
|
|
272
|
+
await sendTelegram(token, chatId, text, { topicId: p.escalation.telegramTopicId });
|
|
273
273
|
store.markNotified(key);
|
|
274
274
|
return;
|
|
275
275
|
}
|
|
@@ -357,14 +357,52 @@ export function readTelegramToken(): string | undefined {
|
|
|
357
357
|
* Telegram's own message id when it returns one, and throws on every *known*
|
|
358
358
|
* failure — connection refused, HTTP error, `{"ok":false}` — which is what lets
|
|
359
359
|
* a caller treat a throw as "nobody has this" and a crash as "nobody knows".
|
|
360
|
+
*
|
|
361
|
+
* Optional `topicId` pins the message to a forum topic (`message_thread_id`).
|
|
362
|
+
* A definitive missing-thread reject retries once as a flat chat and warns, so a
|
|
363
|
+
* deleted topic degrades instead of silently losing the page (#318).
|
|
360
364
|
*/
|
|
361
365
|
export async function sendTelegram(
|
|
362
366
|
token: string,
|
|
363
367
|
chatId: string,
|
|
364
368
|
text: string,
|
|
369
|
+
opts?: { topicId?: number },
|
|
370
|
+
): Promise<number | undefined> {
|
|
371
|
+
const topicId =
|
|
372
|
+
opts?.topicId !== undefined && Number.isFinite(opts.topicId) && Number.isSafeInteger(opts.topicId)
|
|
373
|
+
? opts.topicId
|
|
374
|
+
: undefined;
|
|
375
|
+
try {
|
|
376
|
+
return await postTelegramMessage(token, chatId, text, topicId);
|
|
377
|
+
} catch (err) {
|
|
378
|
+
if (
|
|
379
|
+
topicId === undefined ||
|
|
380
|
+
!(err instanceof TelegramSendError) ||
|
|
381
|
+
err.outcome !== "definitive" ||
|
|
382
|
+
!isMissingTelegramThread(err.message)
|
|
383
|
+
) {
|
|
384
|
+
throw err;
|
|
385
|
+
}
|
|
386
|
+
warn(
|
|
387
|
+
`escalation.telegramTopicId=${topicId} is stale (${err.message}); retrying flat chat`,
|
|
388
|
+
);
|
|
389
|
+
return await postTelegramMessage(token, chatId, text, undefined);
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
/** Telegram's definitive "that forum topic is gone" answers. */
|
|
394
|
+
function isMissingTelegramThread(diagnostic: string): boolean {
|
|
395
|
+
return /message thread not found|topic_id_invalid/i.test(diagnostic);
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
async function postTelegramMessage(
|
|
399
|
+
token: string,
|
|
400
|
+
chatId: string,
|
|
401
|
+
text: string,
|
|
402
|
+
topicId: number | undefined,
|
|
365
403
|
): Promise<number | undefined> {
|
|
366
404
|
const url = `https://api.telegram.org/bot${token}/sendMessage`;
|
|
367
|
-
const
|
|
405
|
+
const payload: Record<string, unknown> = {
|
|
368
406
|
chat_id: chatId,
|
|
369
407
|
// ponytail: hard truncation rather than splitting across messages — the
|
|
370
408
|
// tail of a stack trace is rarely the interesting part. Upgrade path is to
|
|
@@ -372,7 +410,9 @@ export async function sendTelegram(
|
|
|
372
410
|
text:
|
|
373
411
|
text.length > TELEGRAM_TEXT_LIMIT ? `${text.slice(0, TELEGRAM_TEXT_LIMIT)}\n[truncated]` : text,
|
|
374
412
|
disable_web_page_preview: true,
|
|
375
|
-
}
|
|
413
|
+
};
|
|
414
|
+
if (topicId !== undefined) payload.message_thread_id = topicId;
|
|
415
|
+
const body = JSON.stringify(payload);
|
|
376
416
|
|
|
377
417
|
let res: Response;
|
|
378
418
|
try {
|
package/src/fleet.ts
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* never systemctl stop herdr-fleet
|
|
11
11
|
* - arm / disarm — first-class armed marker (arm is inbound Telegram proof)
|
|
12
12
|
* - releaseHold — clear pause only; never re-arms
|
|
13
|
-
* -
|
|
13
|
+
* - resume — clear pause and the stop --pane recovery pin
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
16
|
import { spawnSync } from "node:child_process";
|
|
@@ -29,6 +29,7 @@ import { homedir } from "node:os";
|
|
|
29
29
|
import { dirname, join } from "node:path";
|
|
30
30
|
import { formatZonedMinute } from "./availability.ts";
|
|
31
31
|
import { findProject, loadConfig, stateDir } from "./config.ts";
|
|
32
|
+
import { sendTelegram } from "./escalate.ts";
|
|
32
33
|
import { planUsageLine, readPlanUsage, sharedUsageSource } from "./usage.ts";
|
|
33
34
|
import { readApprovalSurface, readDaemonProfile } from "./approval-surface.ts";
|
|
34
35
|
import { inspectBriefLayout } from "./brief-upgrade.ts";
|
|
@@ -59,11 +60,16 @@ import {
|
|
|
59
60
|
SYSTEMD_UNIT,
|
|
60
61
|
} from "./lifecycle.ts";
|
|
61
62
|
import { formatRss, rssBytesFromHealthz } from "./host.ts";
|
|
63
|
+
import type { WorkerPausePhase } from "./worker.ts";
|
|
62
64
|
import { fetchRateLimit } from "./tracker/github.ts";
|
|
63
65
|
import {
|
|
66
|
+
LEGACY_ARM_MARKER_DETAIL,
|
|
67
|
+
legacyArmedMarkerPath,
|
|
64
68
|
readTickConfig,
|
|
65
69
|
readTickRuntimeStatus,
|
|
70
|
+
resolveArmState,
|
|
66
71
|
TICK_CONFIG_FILE,
|
|
72
|
+
tickConfigMatchesProject,
|
|
67
73
|
type TickConfig,
|
|
68
74
|
type TickConfigResult,
|
|
69
75
|
} from "./orchestrator-tick.ts";
|
|
@@ -71,6 +77,14 @@ import {
|
|
|
71
77
|
export const PANE_HALT_FILE = ".conductor-pane-halted";
|
|
72
78
|
export const DEFAULT_HERDR_UNIT = "herdr-fleet.service";
|
|
73
79
|
|
|
80
|
+
/**
|
|
81
|
+
* Default Herdr session name for conductor fleets. Renamed from `fleet` in
|
|
82
|
+
* #320; `HERDR_SESSION` still wins when set. Legacy hosts that still run a
|
|
83
|
+
* session named `fleet` are bridged for one release by
|
|
84
|
+
* {@link resolveHerdrSession}.
|
|
85
|
+
*/
|
|
86
|
+
export const DEFAULT_HERDR_SESSION = "conductor";
|
|
87
|
+
|
|
74
88
|
/**
|
|
75
89
|
* Agent name assumed when no tick config names one. Mirrors
|
|
76
90
|
* `herdr-conductor`'s own `AGENT_NAME=${AGENT_NAME:-fleet}` default — a
|
|
@@ -80,6 +94,89 @@ export const DEFAULT_HERDR_UNIT = "herdr-fleet.service";
|
|
|
80
94
|
export const DEFAULT_FLEET_AGENT_NAME = "fleet";
|
|
81
95
|
export const ARM_CHALLENGE_TIMEOUT_MS = 300_000;
|
|
82
96
|
|
|
97
|
+
|
|
98
|
+
/** Pre-#320 session name. Bridged for one release when `conductor` is empty. */
|
|
99
|
+
const LEGACY_HERDR_SESSION = "fleet";
|
|
100
|
+
|
|
101
|
+
export const LEGACY_HERDR_SESSION_HINT =
|
|
102
|
+
'herdr session "fleet" found — rename it to "conductor" or set HERDR_SESSION=fleet (remove this bridge next minor)';
|
|
103
|
+
|
|
104
|
+
let legacyHerdrSessionHintPrinted = false;
|
|
105
|
+
|
|
106
|
+
function noteLegacyHerdrSession(log?: (message: string) => void): void {
|
|
107
|
+
if (legacyHerdrSessionHintPrinted) return;
|
|
108
|
+
legacyHerdrSessionHintPrinted = true;
|
|
109
|
+
if (log) log(LEGACY_HERDR_SESSION_HINT);
|
|
110
|
+
else console.warn(LEGACY_HERDR_SESSION_HINT);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** Reset the once-per-process rename hint (tests). */
|
|
114
|
+
export function resetLegacyHerdrSessionHintForTests(): void {
|
|
115
|
+
legacyHerdrSessionHintPrinted = false;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Resolve the Herdr session name. `HERDR_SESSION` is always authoritative.
|
|
120
|
+
* When unset, default to {@link DEFAULT_HERDR_SESSION}. The one-release bridge
|
|
121
|
+
* that prefers a populated legacy `fleet` session lives in
|
|
122
|
+
* {@link resolveHerdrSessionWithBridge} — call that at probe sites.
|
|
123
|
+
*/
|
|
124
|
+
export function resolveHerdrSession(env: NodeJS.ProcessEnv = process.env): string {
|
|
125
|
+
const explicit = env["HERDR_SESSION"];
|
|
126
|
+
if (explicit !== undefined && explicit.length > 0) return explicit;
|
|
127
|
+
return DEFAULT_HERDR_SESSION;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function herdrAgentListRaw(
|
|
131
|
+
session: string,
|
|
132
|
+
bin: string,
|
|
133
|
+
env: NodeJS.ProcessEnv,
|
|
134
|
+
): { ok: true; agents: HerdrAgent[] } | { ok: false } {
|
|
135
|
+
const res = spawnSync(bin, ["--session", session, "agent", "list"], {
|
|
136
|
+
encoding: "utf8",
|
|
137
|
+
timeout: 8_000,
|
|
138
|
+
env,
|
|
139
|
+
});
|
|
140
|
+
if (res.error || res.status !== 0) return { ok: false };
|
|
141
|
+
try {
|
|
142
|
+
return { ok: true, agents: parseHerdrAgentList(res.stdout ?? "") };
|
|
143
|
+
} catch {
|
|
144
|
+
return { ok: false };
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* One-release bridge (#320): when `HERDR_SESSION` is unset, a `conductor`
|
|
150
|
+
* session with no agents while a `fleet` session answers → use `fleet` and
|
|
151
|
+
* print the rename hint once. Remove next minor.
|
|
152
|
+
*/
|
|
153
|
+
export function resolveHerdrSessionWithBridge(opts: {
|
|
154
|
+
env?: NodeJS.ProcessEnv;
|
|
155
|
+
herdrBin?: string;
|
|
156
|
+
log?: (message: string) => void;
|
|
157
|
+
} = {}): string {
|
|
158
|
+
const env = opts.env ?? process.env;
|
|
159
|
+
const explicit = env["HERDR_SESSION"];
|
|
160
|
+
if (explicit !== undefined && explicit.length > 0) return explicit;
|
|
161
|
+
|
|
162
|
+
const bin = opts.herdrBin ?? "herdr";
|
|
163
|
+
const primary = herdrAgentListRaw(DEFAULT_HERDR_SESSION, bin, env);
|
|
164
|
+
if (primary.ok && primary.agents.length > 0) return DEFAULT_HERDR_SESSION;
|
|
165
|
+
|
|
166
|
+
const legacy = herdrAgentListRaw(LEGACY_HERDR_SESSION, bin, env);
|
|
167
|
+
if (legacy.ok && legacy.agents.length > 0) {
|
|
168
|
+
noteLegacyHerdrSession(opts.log);
|
|
169
|
+
return LEGACY_HERDR_SESSION;
|
|
170
|
+
}
|
|
171
|
+
// conductor answered (even empty) wins over a dead legacy session.
|
|
172
|
+
if (primary.ok) return DEFAULT_HERDR_SESSION;
|
|
173
|
+
if (legacy.ok) {
|
|
174
|
+
noteLegacyHerdrSession(opts.log);
|
|
175
|
+
return LEGACY_HERDR_SESSION;
|
|
176
|
+
}
|
|
177
|
+
return DEFAULT_HERDR_SESSION;
|
|
178
|
+
}
|
|
179
|
+
|
|
83
180
|
export function telegramStateDir(): string {
|
|
84
181
|
const override = process.env["OMP_TELEGRAM_STATE_DIR"];
|
|
85
182
|
if (override !== undefined && override.length > 0) return override;
|
|
@@ -104,25 +201,69 @@ export type ResolvedTick =
|
|
|
104
201
|
| { kind: "invalid"; path: string; cwd: string; problem: string }
|
|
105
202
|
| { kind: "ok"; path: string; cwd: string; config: TickConfig };
|
|
106
203
|
|
|
204
|
+
/**
|
|
205
|
+
* The tick config that belongs to this project, skipping any that names another
|
|
206
|
+
* one. The search roots overlap between projects — `stateDir()` and the shared
|
|
207
|
+
* parent of two fleet cwds are read for every one of them — so a stamped config
|
|
208
|
+
* sitting in a shared root would otherwise answer for whichever project asked
|
|
209
|
+
* first. An unstamped (pre-multi-project) config still matches anything, which
|
|
210
|
+
* is what keeps a single-project fleet on the file it already has.
|
|
211
|
+
*/
|
|
107
212
|
export function resolveTickConfig(projectName?: string): ResolvedTick {
|
|
108
213
|
for (const cwd of tickConfigSearchRoots(projectName)) {
|
|
109
214
|
const r: TickConfigResult = readTickConfig(cwd);
|
|
110
|
-
if (r.kind === "ok")
|
|
215
|
+
if (r.kind === "ok") {
|
|
216
|
+
if (!tickConfigMatchesProject(r.config, projectName)) continue;
|
|
217
|
+
return { kind: "ok", path: r.path, cwd, config: r.config };
|
|
218
|
+
}
|
|
111
219
|
if (r.kind === "invalid") return { kind: "invalid", path: r.path, cwd, problem: r.problem };
|
|
112
220
|
}
|
|
113
221
|
return { kind: "absent" };
|
|
114
222
|
}
|
|
115
223
|
|
|
224
|
+
/**
|
|
225
|
+
* Where this project's arm marker lives. The per-project fallback is the point:
|
|
226
|
+
* one shared `<stateDir>/armed` meant arming any project armed all of them. The
|
|
227
|
+
* bare name survives only for the un-named single-project call, which is the
|
|
228
|
+
* pre-multi-project spelling.
|
|
229
|
+
*/
|
|
116
230
|
export function armedMarkerPath(projectName?: string): string {
|
|
231
|
+
return armGate(projectName).path;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/**
|
|
235
|
+
* This project's arm marker together with the project name that identifies it —
|
|
236
|
+
* the pair every arm decision needs, from one tick-config read. `named` prefers
|
|
237
|
+
* the stamped {@link TickConfig.project} over the caller's `--project`, because
|
|
238
|
+
* the stamp is what the marker was generated from.
|
|
239
|
+
*/
|
|
240
|
+
function armGate(projectName?: string): { path: string; named?: string } {
|
|
117
241
|
const tick = resolveTickConfig(projectName);
|
|
118
|
-
|
|
119
|
-
|
|
242
|
+
const configured = tick.kind === "ok" ? tick.config : undefined;
|
|
243
|
+
const named = configured?.project ?? projectName;
|
|
244
|
+
const path =
|
|
245
|
+
configured?.armedFile ??
|
|
246
|
+
(projectName === undefined ? legacyArmedMarkerPath() : join(stateDir(), `armed-${projectName}`));
|
|
247
|
+
return named === undefined ? { path } : { path, named };
|
|
120
248
|
}
|
|
121
249
|
|
|
250
|
+
/**
|
|
251
|
+
* Clears the arm gate for this project — and `wasArmed` is the gate the
|
|
252
|
+
* heartbeat reads, not merely one file's presence.
|
|
253
|
+
*
|
|
254
|
+
* The second removal is the whole reason this is not a one-line `rmSync`. While
|
|
255
|
+
* the shared pre-per-project marker is still honoured (single-project host, an
|
|
256
|
+
* `armedFile` just restamped to `armed-<name>`), it holds the gate open as soon
|
|
257
|
+
* as this project's own marker is gone — so a disarm that removed only the
|
|
258
|
+
* per-project file would leave the fleet ticking, which is exactly what `hold`
|
|
259
|
+
* exists to prevent. A *stranded* shared marker arms nothing, so it is not this
|
|
260
|
+
* command's to remove.
|
|
261
|
+
*/
|
|
122
262
|
export function disarmTicks(projectName?: string): { path: string; wasArmed: boolean } {
|
|
123
|
-
const path =
|
|
124
|
-
const wasArmed =
|
|
263
|
+
const { path, named } = armGate(projectName);
|
|
264
|
+
const wasArmed = resolveArmState(path, named).armed;
|
|
125
265
|
rmSync(path, { force: true });
|
|
266
|
+
if (resolveArmState(path, named).legacy === "honoured") rmSync(legacyArmedMarkerPath(), { force: true });
|
|
126
267
|
return { path, wasArmed };
|
|
127
268
|
}
|
|
128
269
|
|
|
@@ -134,7 +275,7 @@ export interface ArmResult {
|
|
|
134
275
|
}
|
|
135
276
|
|
|
136
277
|
export interface ArmDeps {
|
|
137
|
-
sendChallenge?: (token: string, owner: string, text: string) => Promise<void>;
|
|
278
|
+
sendChallenge?: (token: string, owner: string, text: string, topicId?: number) => Promise<void>;
|
|
138
279
|
/**
|
|
139
280
|
* Waits for the challenge to appear as a user turn somewhere under the
|
|
140
281
|
* session directory. The waiter owns transcript discovery — not the caller —
|
|
@@ -190,18 +331,38 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
190
331
|
}
|
|
191
332
|
|
|
192
333
|
const path = tick.config.armedFile;
|
|
193
|
-
|
|
334
|
+
// The gate as the heartbeat reads it, so "replaced previous marker" is not a
|
|
335
|
+
// lie about a fleet the shared marker was arming, and so the write below knows
|
|
336
|
+
// whether it is superseding that marker.
|
|
337
|
+
const arm = resolveArmState(path, tick.config.project ?? projectName);
|
|
338
|
+
const alreadyArmed = arm.armed;
|
|
194
339
|
const code = makeChallengeCode();
|
|
340
|
+
// One bot, one chat, and — once a host runs more than one fleet — more than
|
|
341
|
+
// one pane that can ask. The challenge names which one, or the operator is
|
|
342
|
+
// answering a question they cannot attribute.
|
|
343
|
+
const named = tick.config.project ?? projectName;
|
|
195
344
|
const text =
|
|
196
|
-
`Fleet arming check
|
|
345
|
+
`Fleet arming check${named === undefined ? "" : ` — project ${named}`}. ` +
|
|
346
|
+
`Reply to this chat with exactly:\n${code}\n` +
|
|
197
347
|
`Nothing will be dispatched until that reply is seen in the orchestrator session.`;
|
|
198
348
|
|
|
349
|
+
// Prefer the project's configured forum topic so arm challenges land where
|
|
350
|
+
// escalations already do (#318). Missing project config keeps flat-chat 0.13.
|
|
351
|
+
let topicId: number | undefined;
|
|
352
|
+
if (named !== undefined) {
|
|
353
|
+
try {
|
|
354
|
+
topicId = findProject(loadConfig(), named).escalation.telegramTopicId;
|
|
355
|
+
} catch {
|
|
356
|
+
/* no project config */
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
|
|
199
360
|
const send = deps.sendChallenge ?? sendTelegramMessage;
|
|
200
361
|
// Read before the send, not after: a transcript untouched since this instant
|
|
201
362
|
// cannot contain the reply, and that is what the waiter filters on.
|
|
202
363
|
const sentAt = (deps.now ?? Date.now)();
|
|
203
364
|
try {
|
|
204
|
-
await send(token, channel.owner, text);
|
|
365
|
+
await send(token, channel.owner, text, topicId);
|
|
205
366
|
} catch (err) {
|
|
206
367
|
throw new Error(
|
|
207
368
|
`arm: outbound sendMessage failed — NOT armed: ${err instanceof Error ? err.message : String(err)}`,
|
|
@@ -232,6 +393,11 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
232
393
|
|
|
233
394
|
mkdirSync(dirname(path), { recursive: true });
|
|
234
395
|
writeFileSync(path, `armed ${new Date().toISOString()} owner=${channel.owner}\n`, { mode: 0o600 });
|
|
396
|
+
// This project now has its own marker, so the shared one it was borrowing has
|
|
397
|
+
// done its last job. Left in place it would survive the next `disarm` as a
|
|
398
|
+
// marker that re-arms the fleet, and turn into a meaningless legacy warning
|
|
399
|
+
// the moment a second project is configured.
|
|
400
|
+
if (arm.legacy === "honoured") rmSync(legacyArmedMarkerPath(), { force: true });
|
|
235
401
|
return { path, alreadyArmed, owner: channel.owner, challenge: code };
|
|
236
402
|
}
|
|
237
403
|
|
|
@@ -258,13 +424,13 @@ export interface HaltWithPaneResult extends HaltResult {
|
|
|
258
424
|
}
|
|
259
425
|
|
|
260
426
|
export function hold(projectName?: string, source: string = "hold"): HoldResult {
|
|
261
|
-
const wasPaused = isPaused();
|
|
262
|
-
setPaused(true, { source });
|
|
427
|
+
const wasPaused = isPaused(projectName);
|
|
428
|
+
setPaused(true, { source }, projectName);
|
|
263
429
|
return { wasPaused, disarmed: disarmTicks(projectName) };
|
|
264
430
|
}
|
|
265
431
|
|
|
266
|
-
export function releaseHold(): void {
|
|
267
|
-
setPaused(false);
|
|
432
|
+
export function releaseHold(projectName?: string): void {
|
|
433
|
+
setPaused(false, undefined, projectName);
|
|
268
434
|
}
|
|
269
435
|
|
|
270
436
|
export async function halt(projectName?: string): Promise<HaltResult> {
|
|
@@ -312,11 +478,11 @@ export function resolvePaneHaltPath(projectName?: string): ResolvedPaneHalt {
|
|
|
312
478
|
export function paneHaltPath(projectName?: string): string {
|
|
313
479
|
const resolved = resolvePaneHaltPath(projectName);
|
|
314
480
|
if (resolved.kind === "unresolved") {
|
|
315
|
-
// Verb-neutral: this is also
|
|
481
|
+
// Verb-neutral: this is also `resume`'s pin-clearing path.
|
|
316
482
|
throw new Error(
|
|
317
483
|
`cannot locate the pane recovery pin — ${resolved.reason}. ` +
|
|
318
484
|
`Drop a tick config beside the pane, or stop the agent by hand; ` +
|
|
319
|
-
`\`
|
|
485
|
+
`\`stop\` without \`--pane\` still stops the dispatch daemon.`,
|
|
320
486
|
);
|
|
321
487
|
}
|
|
322
488
|
return resolved.path;
|
|
@@ -328,9 +494,9 @@ export function pinPaneHalt(projectName?: string): { path: string } {
|
|
|
328
494
|
writeFileSync(
|
|
329
495
|
path,
|
|
330
496
|
[
|
|
331
|
-
`# Written by omp-conductor
|
|
497
|
+
`# Written by omp-conductor stop --pane at ${new Date().toISOString()}`,
|
|
332
498
|
`# herdr-conductor recover.sh must not resume the fleet agent while this file exists.`,
|
|
333
|
-
`# Clear with: omp-conductor
|
|
499
|
+
`# Clear with: omp-conductor resume (or rm this file)`,
|
|
334
500
|
"",
|
|
335
501
|
].join("\n"),
|
|
336
502
|
{ mode: 0o600 },
|
|
@@ -338,6 +504,23 @@ export function pinPaneHalt(projectName?: string): { path: string } {
|
|
|
338
504
|
return { path };
|
|
339
505
|
}
|
|
340
506
|
|
|
507
|
+
/**
|
|
508
|
+
* Clears the pin when its location is known, and reports plainly when it is not.
|
|
509
|
+
*
|
|
510
|
+
* Distinct from {@link clearPaneHalt}, which refuses an unresolvable location
|
|
511
|
+
* because pinning somewhere recovery never reads is worse than failing. `resume`
|
|
512
|
+
* needs the softer answer: it folds in the pin clear, and an operator resuming a
|
|
513
|
+
* fleet from a host with no tick config has no pin to clear rather than an error
|
|
514
|
+
* to work around.
|
|
515
|
+
*/
|
|
516
|
+
export function clearPaneHaltIfResolvable(projectName?: string): { path?: string; wasHalted: boolean } {
|
|
517
|
+
const resolved = resolvePaneHaltPath(projectName);
|
|
518
|
+
if (resolved.kind !== "ok") return { wasHalted: false };
|
|
519
|
+
const wasHalted = existsSync(resolved.path);
|
|
520
|
+
rmSync(resolved.path, { force: true });
|
|
521
|
+
return { path: resolved.path, wasHalted };
|
|
522
|
+
}
|
|
523
|
+
|
|
341
524
|
export function clearPaneHalt(projectName?: string): { path: string; wasHalted: boolean } {
|
|
342
525
|
const path = paneHaltPath(projectName);
|
|
343
526
|
const wasHalted = existsSync(path);
|
|
@@ -656,7 +839,9 @@ export async function stopConductorPane(
|
|
|
656
839
|
|
|
657
840
|
async function herdrAgentList(deps: PaneStopDeps): Promise<HerdrAgent[]> {
|
|
658
841
|
const bin = deps.herdrBin ?? "herdr";
|
|
659
|
-
const session =
|
|
842
|
+
const session =
|
|
843
|
+
deps.herdrSession ??
|
|
844
|
+
resolveHerdrSessionWithBridge({ env: process.env, herdrBin: bin });
|
|
660
845
|
const res = spawnSync(bin, ["--session", session, "agent", "list"], {
|
|
661
846
|
encoding: "utf8",
|
|
662
847
|
timeout: 8_000,
|
|
@@ -730,7 +915,9 @@ export function parseHerdrAgentList(rawOutput: string): HerdrAgent[] {
|
|
|
730
915
|
|
|
731
916
|
async function herdrOmpForegroundPids(paneId: string, deps: PaneStopDeps): Promise<number[]> {
|
|
732
917
|
const bin = deps.herdrBin ?? "herdr";
|
|
733
|
-
const session =
|
|
918
|
+
const session =
|
|
919
|
+
deps.herdrSession ??
|
|
920
|
+
resolveHerdrSessionWithBridge({ env: process.env, herdrBin: bin });
|
|
734
921
|
const res = spawnSync(bin, ["--session", session, "pane", "process-info", "--pane", paneId], {
|
|
735
922
|
encoding: "utf8",
|
|
736
923
|
timeout: 8_000,
|
|
@@ -836,7 +1023,7 @@ export interface FleetLayers {
|
|
|
836
1023
|
|
|
837
1024
|
export function fleetLayers(projectName?: string): FleetLayers {
|
|
838
1025
|
const rec = livingDaemon();
|
|
839
|
-
const paused = isPaused();
|
|
1026
|
+
const paused = isPaused(projectName);
|
|
840
1027
|
const dispatch: DispatchLayer = rec === undefined ? "stopped" : paused ? "paused" : "running";
|
|
841
1028
|
|
|
842
1029
|
const tick = resolveTickConfig(projectName);
|
|
@@ -857,12 +1044,21 @@ export function fleetLayers(projectName?: string): FleetLayers {
|
|
|
857
1044
|
if (tick.config.armedFile === undefined) {
|
|
858
1045
|
ticks = "ungated";
|
|
859
1046
|
ticksDetail = "no armedFile in tick config — heartbeat sends without an arm marker";
|
|
860
|
-
} else if (existsSync(tick.config.armedFile)) {
|
|
861
|
-
ticks = "armed";
|
|
862
|
-
ticksDetail = tick.config.armedFile;
|
|
863
1047
|
} else {
|
|
864
|
-
|
|
865
|
-
|
|
1048
|
+
// The same decision the heartbeat itself makes, so status can never claim
|
|
1049
|
+
// armed while the tick skips (or the reverse) — including the two shared
|
|
1050
|
+
// marker cases: honoured through an upgrade on a single-project host,
|
|
1051
|
+
// stranded once a second project exists.
|
|
1052
|
+
const arm = resolveArmState(tick.config.armedFile, tick.config.project ?? projectName);
|
|
1053
|
+
ticks = arm.armed ? "armed" : "disarmed";
|
|
1054
|
+
ticksDetail =
|
|
1055
|
+
arm.legacy === undefined
|
|
1056
|
+
? tick.config.armedFile
|
|
1057
|
+
: `${tick.config.armedFile} — ${legacyArmedMarkerPath()}: ${
|
|
1058
|
+
arm.legacy === "honoured"
|
|
1059
|
+
? "legacy global arm marker still honoured; re-arm this project to replace it"
|
|
1060
|
+
: LEGACY_ARM_MARKER_DETAIL
|
|
1061
|
+
}`;
|
|
866
1062
|
}
|
|
867
1063
|
const runtime = readTickRuntimeStatus(tick.cwd);
|
|
868
1064
|
if (runtime !== undefined && isAlive(runtime.pid)) nextTickAt = runtime.nextTickAt;
|
|
@@ -916,8 +1112,27 @@ export function codeGraphFromHealthz(body: string | undefined, project: string):
|
|
|
916
1112
|
if (body === undefined) return undefined;
|
|
917
1113
|
try {
|
|
918
1114
|
const parsed = JSON.parse(body) as Record<string, unknown>;
|
|
919
|
-
|
|
920
|
-
|
|
1115
|
+
let block: Record<string, unknown> | undefined;
|
|
1116
|
+
if (parsed["project"] === project) {
|
|
1117
|
+
block = parsed;
|
|
1118
|
+
} else {
|
|
1119
|
+
const projects = parsed["projects"];
|
|
1120
|
+
if (Array.isArray(projects)) {
|
|
1121
|
+
for (const entry of projects) {
|
|
1122
|
+
if (
|
|
1123
|
+
entry !== null &&
|
|
1124
|
+
typeof entry === "object" &&
|
|
1125
|
+
!Array.isArray(entry) &&
|
|
1126
|
+
(entry as Record<string, unknown>)["project"] === project
|
|
1127
|
+
) {
|
|
1128
|
+
block = entry as Record<string, unknown>;
|
|
1129
|
+
break;
|
|
1130
|
+
}
|
|
1131
|
+
}
|
|
1132
|
+
}
|
|
1133
|
+
}
|
|
1134
|
+
if (block === undefined) return undefined;
|
|
1135
|
+
const graph = block["codeGraph"];
|
|
921
1136
|
if (graph === null || typeof graph !== "object" || Array.isArray(graph)) return undefined;
|
|
922
1137
|
const value = graph as Record<string, unknown>;
|
|
923
1138
|
if (value["configured"] === false) return { configured: false };
|
|
@@ -942,6 +1157,57 @@ export function codeGraphFromHealthz(body: string | undefined, project: string):
|
|
|
942
1157
|
}
|
|
943
1158
|
}
|
|
944
1159
|
|
|
1160
|
+
|
|
1161
|
+
/** Live worker pause phases from a daemon `/healthz` body for one project. */
|
|
1162
|
+
export function workerPhasesFromHealthz(
|
|
1163
|
+
body: string | undefined,
|
|
1164
|
+
project: string,
|
|
1165
|
+
): ReadonlyMap<number, WorkerPausePhase> {
|
|
1166
|
+
const phases = new Map<number, WorkerPausePhase>();
|
|
1167
|
+
if (body === undefined) return phases;
|
|
1168
|
+
try {
|
|
1169
|
+
const payload = JSON.parse(body) as unknown;
|
|
1170
|
+
if (payload === null || typeof payload !== "object") return phases;
|
|
1171
|
+
let block: object | undefined;
|
|
1172
|
+
if (Reflect.get(payload, "project") === project) {
|
|
1173
|
+
block = payload;
|
|
1174
|
+
} else {
|
|
1175
|
+
const projects = Reflect.get(payload, "projects");
|
|
1176
|
+
if (Array.isArray(projects)) {
|
|
1177
|
+
for (const entry of projects) {
|
|
1178
|
+
if (
|
|
1179
|
+
entry !== null &&
|
|
1180
|
+
typeof entry === "object" &&
|
|
1181
|
+
Reflect.get(entry, "project") === project
|
|
1182
|
+
) {
|
|
1183
|
+
block = entry;
|
|
1184
|
+
break;
|
|
1185
|
+
}
|
|
1186
|
+
}
|
|
1187
|
+
}
|
|
1188
|
+
}
|
|
1189
|
+
if (block === undefined) return phases;
|
|
1190
|
+
const workers = Reflect.get(block, "workers");
|
|
1191
|
+
if (!Array.isArray(workers)) return phases;
|
|
1192
|
+
for (const worker of workers) {
|
|
1193
|
+
if (worker === null || typeof worker !== "object") continue;
|
|
1194
|
+
const issue = Reflect.get(worker, "issue");
|
|
1195
|
+
const phase = Reflect.get(worker, "phase");
|
|
1196
|
+
if (
|
|
1197
|
+
Number.isSafeInteger(issue) &&
|
|
1198
|
+
(issue as number) > 0 &&
|
|
1199
|
+
(phase === "pausing" || phase === "paused")
|
|
1200
|
+
) {
|
|
1201
|
+
phases.set(issue as number, phase);
|
|
1202
|
+
}
|
|
1203
|
+
}
|
|
1204
|
+
} catch {
|
|
1205
|
+
// An unreadable health body means no trustworthy pause phase.
|
|
1206
|
+
}
|
|
1207
|
+
return phases;
|
|
1208
|
+
}
|
|
1209
|
+
|
|
1210
|
+
|
|
945
1211
|
export function formatCodeGraphHealth(graph: CodeGraphHealth, now = Date.now()): string | undefined {
|
|
946
1212
|
if (!graph.configured) return undefined;
|
|
947
1213
|
const indexed = graph.repos.filter((repo) => repo.index === "present").length;
|
|
@@ -971,6 +1237,7 @@ export function formatFleetStatus(
|
|
|
971
1237
|
brief: string | undefined = undefined,
|
|
972
1238
|
decisions: string | undefined = undefined,
|
|
973
1239
|
failureClasses: string | undefined = undefined,
|
|
1240
|
+
workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
|
|
974
1241
|
): string {
|
|
975
1242
|
const tickLine =
|
|
976
1243
|
layers.ticksDetail === undefined
|
|
@@ -1032,12 +1299,12 @@ export function formatFleetStatus(
|
|
|
1032
1299
|
const dispatchLine =
|
|
1033
1300
|
layers.dispatch === "paused"
|
|
1034
1301
|
? (() => {
|
|
1035
|
-
const prov = pauseProvenance();
|
|
1302
|
+
const prov = pauseProvenance(s.project);
|
|
1036
1303
|
if (prov !== undefined) {
|
|
1037
1304
|
const reason = prov.reason === undefined ? "" : ` — "${prov.reason}"`;
|
|
1038
1305
|
return `dispatch paused (source: ${prov.source}${reason})`;
|
|
1039
1306
|
}
|
|
1040
|
-
return isPaused() && pausedAt() === undefined
|
|
1307
|
+
return isPaused(s.project) && pausedAt(s.project) === undefined
|
|
1041
1308
|
? "dispatch paused (unparseable sentinel — new work and completion mutations refused; release gates remain available)"
|
|
1042
1309
|
: "dispatch paused";
|
|
1043
1310
|
})()
|
|
@@ -1057,7 +1324,7 @@ export function formatFleetStatus(
|
|
|
1057
1324
|
...(graphBlock === undefined ? [] : [graphBlock]),
|
|
1058
1325
|
daemonBlock,
|
|
1059
1326
|
"",
|
|
1060
|
-
formatProjectBody(s),
|
|
1327
|
+
formatProjectBody(s, workerPhases),
|
|
1061
1328
|
].join("\n");
|
|
1062
1329
|
}
|
|
1063
1330
|
|
|
@@ -1088,7 +1355,10 @@ function formatDigestScheduleStatus(s: StatusSnapshot): string[] {
|
|
|
1088
1355
|
];
|
|
1089
1356
|
}
|
|
1090
1357
|
|
|
1091
|
-
function formatProjectBody(
|
|
1358
|
+
function formatProjectBody(
|
|
1359
|
+
s: StatusSnapshot,
|
|
1360
|
+
workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
|
|
1361
|
+
): string {
|
|
1092
1362
|
const lines = [
|
|
1093
1363
|
`project ${s.project}${s.paused ? " (PAUSED)" : ""}`,
|
|
1094
1364
|
...(s.pauseReason === undefined ? [] : [`paused ${s.pauseReason}`]),
|
|
@@ -1158,9 +1428,11 @@ function formatProjectBody(s: StatusSnapshot): string {
|
|
|
1158
1428
|
} else {
|
|
1159
1429
|
lines.push("active runs");
|
|
1160
1430
|
for (const r of s.activeRuns) {
|
|
1431
|
+
const phase = workerPhases.get(r.issue);
|
|
1432
|
+
const state = phase === "pausing" || phase === "paused" ? phase : r.state;
|
|
1161
1433
|
lines.push(
|
|
1162
|
-
` #${r.issue} ${r.repo} ${
|
|
1163
|
-
`${r.turns}/${r.maxTurns} turns
|
|
1434
|
+
` #${r.issue} ${r.repo} ${state} attempt ${r.attempt} ` +
|
|
1435
|
+
`${r.turns}/${r.maxTurns} turns ${r.spendUsd.toFixed(2)} ${r.branch}` +
|
|
1164
1436
|
(r.prUrl ? ` ${r.prUrl}` : ""),
|
|
1165
1437
|
);
|
|
1166
1438
|
// The orchestrator's Duty 1 reads this command, and a flagged run's
|
|
@@ -1203,6 +1475,7 @@ export async function renderStatus(projectName?: string): Promise<string> {
|
|
|
1203
1475
|
]);
|
|
1204
1476
|
const cached = codeGraphFromHealthz(health?.body, project.name);
|
|
1205
1477
|
const codeGraph = cached ?? (await probeCodeGraph(project));
|
|
1478
|
+
const workerPhases = workerPhasesFromHealthz(health?.body, project.name);
|
|
1206
1479
|
return formatFleetStatus(
|
|
1207
1480
|
{ ...s, planUsage, github },
|
|
1208
1481
|
layers,
|
|
@@ -1213,6 +1486,7 @@ export async function renderStatus(projectName?: string): Promise<string> {
|
|
|
1213
1486
|
briefStatusLine(project),
|
|
1214
1487
|
decisionStatusLine(project.name),
|
|
1215
1488
|
failureClassBlock(project.name),
|
|
1489
|
+
workerPhases,
|
|
1216
1490
|
);
|
|
1217
1491
|
}
|
|
1218
1492
|
|
|
@@ -1433,14 +1707,14 @@ function makeChallengeCode(): string {
|
|
|
1433
1707
|
return `FLEET-${hex}`;
|
|
1434
1708
|
}
|
|
1435
1709
|
|
|
1436
|
-
async function sendTelegramMessage(
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
}
|
|
1710
|
+
async function sendTelegramMessage(
|
|
1711
|
+
token: string,
|
|
1712
|
+
owner: string,
|
|
1713
|
+
text: string,
|
|
1714
|
+
topicId?: number,
|
|
1715
|
+
): Promise<void> {
|
|
1716
|
+
// Shared transport: stale-topic retry + message_thread_id live in one place.
|
|
1717
|
+
await sendTelegram(token, owner, text, { topicId });
|
|
1444
1718
|
}
|
|
1445
1719
|
|
|
1446
1720
|
interface SessionScan {
|
|
@@ -1607,7 +1881,7 @@ function probeOmpPane(
|
|
|
1607
1881
|
}
|
|
1608
1882
|
const agentName =
|
|
1609
1883
|
tick.kind === "ok" ? (tick.config.agentName ?? DEFAULT_FLEET_AGENT_NAME) : DEFAULT_FLEET_AGENT_NAME;
|
|
1610
|
-
const session = process.env
|
|
1884
|
+
const session = resolveHerdrSessionWithBridge({ env: process.env });
|
|
1611
1885
|
try {
|
|
1612
1886
|
const res = spawnSync("herdr", ["--session", session, "agent", "list"], {
|
|
1613
1887
|
encoding: "utf8",
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Regenerator for `schema/config.schema.json`.
|
|
3
|
+
*
|
|
4
|
+
* Run via `bun run schema` from the package root. Writes the draft 2020-12 JSON
|
|
5
|
+
* Schema rendering of `ConfigSchema` (see `config-schema.ts`) to
|
|
6
|
+
* `schema/config.schema.json`, which ships in the package so editors can
|
|
7
|
+
* validate a hand-written config. The `config.test.ts` freshness lock fails
|
|
8
|
+
* when a committed schema drifts from what this script produces.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { mkdirSync, writeFileSync } from "node:fs";
|
|
12
|
+
import { dirname, join } from "node:path";
|
|
13
|
+
import { fileURLToPath } from "node:url";
|
|
14
|
+
import { configJsonSchema } from "./config-schema.ts";
|
|
15
|
+
|
|
16
|
+
const packageDir = join(dirname(fileURLToPath(import.meta.url)), "..");
|
|
17
|
+
const schemaDir = join(packageDir, "schema");
|
|
18
|
+
mkdirSync(schemaDir, { recursive: true });
|
|
19
|
+
const out = join(schemaDir, "config.schema.json");
|
|
20
|
+
writeFileSync(out, `${JSON.stringify(configJsonSchema(), null, 2)}\n`);
|
|
21
|
+
console.log(`wrote ${out}`);
|