omp-conductor 0.13.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +549 -234
- package/package.json +8 -5
- package/schema/config.schema.json +609 -0
- package/src/availability.ts +165 -0
- package/src/board.ts +19 -32
- package/src/brief-upgrade.ts +1 -1
- package/src/briefs/orchestrator.md +72 -31
- package/src/briefs/policy.md +48 -36
- package/src/briefs/probes/gates.md +51 -0
- package/src/briefs/probes/project-context.md +59 -0
- package/src/briefs/probes/release-procedure.md +81 -0
- package/src/cli.ts +356 -212
- package/src/config-schema.ts +352 -0
- package/src/config.ts +1037 -679
- package/src/confinement.ts +54 -0
- package/src/daemon.ts +644 -390
- package/src/diff-flags.ts +73 -4
- package/src/digest-schedule.ts +92 -24
- package/src/escalate.ts +89 -22
- package/src/fleet.ts +351 -46
- package/src/generate-schema.ts +21 -0
- package/src/graph.ts +3 -3
- package/src/host.ts +16 -0
- package/src/omp.ts +21 -1
- package/src/orchestrator-tick.ts +732 -56
- package/src/privileged.ts +264 -0
- package/src/reports.ts +203 -6
- package/src/session-host.ts +3 -0
- package/src/setup-host.ts +209 -24
- package/src/setup-install.ts +320 -0
- package/src/setup-probe.ts +412 -0
- package/src/setup-wizard.ts +1946 -0
- package/src/setup.ts +457 -53
- package/src/store.ts +610 -98
- package/src/tracker/github.ts +43 -5
- package/src/types.ts +153 -14
- package/src/upgrade.ts +44 -10
- package/src/verbs/actions.ts +131 -13
- package/src/verbs/server.ts +40 -18
- package/src/wizard-ui.ts +249 -0
- package/src/worker.ts +24 -7
- package/skills/conductor-onboarding/SKILL.md +0 -748
- package/skills/conductor-update/SKILL.md +0 -51
- package/src/plugin.ts +0 -1495
package/src/fleet.ts
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* never systemctl stop herdr-fleet
|
|
11
11
|
* - arm / disarm — first-class armed marker (arm is inbound Telegram proof)
|
|
12
12
|
* - releaseHold — clear pause only; never re-arms
|
|
13
|
-
* -
|
|
13
|
+
* - resume — clear pause and the stop --pane recovery pin
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
16
|
import { spawnSync } from "node:child_process";
|
|
@@ -27,7 +27,9 @@ import {
|
|
|
27
27
|
import { createInterface } from "node:readline";
|
|
28
28
|
import { homedir } from "node:os";
|
|
29
29
|
import { dirname, join } from "node:path";
|
|
30
|
+
import { formatZonedMinute } from "./availability.ts";
|
|
30
31
|
import { findProject, loadConfig, stateDir } from "./config.ts";
|
|
32
|
+
import { sendTelegram } from "./escalate.ts";
|
|
31
33
|
import { planUsageLine, readPlanUsage, sharedUsageSource } from "./usage.ts";
|
|
32
34
|
import { readApprovalSurface, readDaemonProfile } from "./approval-surface.ts";
|
|
33
35
|
import { inspectBriefLayout } from "./brief-upgrade.ts";
|
|
@@ -36,9 +38,9 @@ import { renderBriefForProject } from "./setup.ts";
|
|
|
36
38
|
import type { ProjectConfig, Store } from "./types.ts";
|
|
37
39
|
import { settlementFlagSummary } from "./diff-flags.ts";
|
|
38
40
|
import { probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
|
|
39
|
-
import { formatOpenReports } from "./reports.ts";
|
|
41
|
+
import { formatDigestBacklog, formatOpenReports } from "./reports.ts";
|
|
40
42
|
import {
|
|
41
|
-
|
|
43
|
+
formatBaseHealth,
|
|
42
44
|
formatDispatchSummary,
|
|
43
45
|
formatReleaseGrants,
|
|
44
46
|
formatSalvagedRuns,
|
|
@@ -58,11 +60,16 @@ import {
|
|
|
58
60
|
SYSTEMD_UNIT,
|
|
59
61
|
} from "./lifecycle.ts";
|
|
60
62
|
import { formatRss, rssBytesFromHealthz } from "./host.ts";
|
|
63
|
+
import type { WorkerPausePhase } from "./worker.ts";
|
|
61
64
|
import { fetchRateLimit } from "./tracker/github.ts";
|
|
62
65
|
import {
|
|
66
|
+
LEGACY_ARM_MARKER_DETAIL,
|
|
67
|
+
legacyArmedMarkerPath,
|
|
63
68
|
readTickConfig,
|
|
64
69
|
readTickRuntimeStatus,
|
|
70
|
+
resolveArmState,
|
|
65
71
|
TICK_CONFIG_FILE,
|
|
72
|
+
tickConfigMatchesProject,
|
|
66
73
|
type TickConfig,
|
|
67
74
|
type TickConfigResult,
|
|
68
75
|
} from "./orchestrator-tick.ts";
|
|
@@ -70,6 +77,14 @@ import {
|
|
|
70
77
|
export const PANE_HALT_FILE = ".conductor-pane-halted";
|
|
71
78
|
export const DEFAULT_HERDR_UNIT = "herdr-fleet.service";
|
|
72
79
|
|
|
80
|
+
/**
|
|
81
|
+
* Default Herdr session name for conductor fleets. Renamed from `fleet` in
|
|
82
|
+
* #320; `HERDR_SESSION` still wins when set. Legacy hosts that still run a
|
|
83
|
+
* session named `fleet` are bridged for one release by
|
|
84
|
+
* {@link resolveHerdrSession}.
|
|
85
|
+
*/
|
|
86
|
+
export const DEFAULT_HERDR_SESSION = "conductor";
|
|
87
|
+
|
|
73
88
|
/**
|
|
74
89
|
* Agent name assumed when no tick config names one. Mirrors
|
|
75
90
|
* `herdr-conductor`'s own `AGENT_NAME=${AGENT_NAME:-fleet}` default — a
|
|
@@ -79,6 +94,89 @@ export const DEFAULT_HERDR_UNIT = "herdr-fleet.service";
|
|
|
79
94
|
export const DEFAULT_FLEET_AGENT_NAME = "fleet";
|
|
80
95
|
export const ARM_CHALLENGE_TIMEOUT_MS = 300_000;
|
|
81
96
|
|
|
97
|
+
|
|
98
|
+
/** Pre-#320 session name. Bridged for one release when `conductor` is empty. */
|
|
99
|
+
const LEGACY_HERDR_SESSION = "fleet";
|
|
100
|
+
|
|
101
|
+
export const LEGACY_HERDR_SESSION_HINT =
|
|
102
|
+
'herdr session "fleet" found — rename it to "conductor" or set HERDR_SESSION=fleet (remove this bridge next minor)';
|
|
103
|
+
|
|
104
|
+
let legacyHerdrSessionHintPrinted = false;
|
|
105
|
+
|
|
106
|
+
function noteLegacyHerdrSession(log?: (message: string) => void): void {
|
|
107
|
+
if (legacyHerdrSessionHintPrinted) return;
|
|
108
|
+
legacyHerdrSessionHintPrinted = true;
|
|
109
|
+
if (log) log(LEGACY_HERDR_SESSION_HINT);
|
|
110
|
+
else console.warn(LEGACY_HERDR_SESSION_HINT);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** Reset the once-per-process rename hint (tests). */
|
|
114
|
+
export function resetLegacyHerdrSessionHintForTests(): void {
|
|
115
|
+
legacyHerdrSessionHintPrinted = false;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Resolve the Herdr session name. `HERDR_SESSION` is always authoritative.
|
|
120
|
+
* When unset, default to {@link DEFAULT_HERDR_SESSION}. The one-release bridge
|
|
121
|
+
* that prefers a populated legacy `fleet` session lives in
|
|
122
|
+
* {@link resolveHerdrSessionWithBridge} — call that at probe sites.
|
|
123
|
+
*/
|
|
124
|
+
export function resolveHerdrSession(env: NodeJS.ProcessEnv = process.env): string {
|
|
125
|
+
const explicit = env["HERDR_SESSION"];
|
|
126
|
+
if (explicit !== undefined && explicit.length > 0) return explicit;
|
|
127
|
+
return DEFAULT_HERDR_SESSION;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function herdrAgentListRaw(
|
|
131
|
+
session: string,
|
|
132
|
+
bin: string,
|
|
133
|
+
env: NodeJS.ProcessEnv,
|
|
134
|
+
): { ok: true; agents: HerdrAgent[] } | { ok: false } {
|
|
135
|
+
const res = spawnSync(bin, ["--session", session, "agent", "list"], {
|
|
136
|
+
encoding: "utf8",
|
|
137
|
+
timeout: 8_000,
|
|
138
|
+
env,
|
|
139
|
+
});
|
|
140
|
+
if (res.error || res.status !== 0) return { ok: false };
|
|
141
|
+
try {
|
|
142
|
+
return { ok: true, agents: parseHerdrAgentList(res.stdout ?? "") };
|
|
143
|
+
} catch {
|
|
144
|
+
return { ok: false };
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* One-release bridge (#320): when `HERDR_SESSION` is unset, a `conductor`
|
|
150
|
+
* session with no agents while a `fleet` session answers → use `fleet` and
|
|
151
|
+
* print the rename hint once. Remove next minor.
|
|
152
|
+
*/
|
|
153
|
+
export function resolveHerdrSessionWithBridge(opts: {
|
|
154
|
+
env?: NodeJS.ProcessEnv;
|
|
155
|
+
herdrBin?: string;
|
|
156
|
+
log?: (message: string) => void;
|
|
157
|
+
} = {}): string {
|
|
158
|
+
const env = opts.env ?? process.env;
|
|
159
|
+
const explicit = env["HERDR_SESSION"];
|
|
160
|
+
if (explicit !== undefined && explicit.length > 0) return explicit;
|
|
161
|
+
|
|
162
|
+
const bin = opts.herdrBin ?? "herdr";
|
|
163
|
+
const primary = herdrAgentListRaw(DEFAULT_HERDR_SESSION, bin, env);
|
|
164
|
+
if (primary.ok && primary.agents.length > 0) return DEFAULT_HERDR_SESSION;
|
|
165
|
+
|
|
166
|
+
const legacy = herdrAgentListRaw(LEGACY_HERDR_SESSION, bin, env);
|
|
167
|
+
if (legacy.ok && legacy.agents.length > 0) {
|
|
168
|
+
noteLegacyHerdrSession(opts.log);
|
|
169
|
+
return LEGACY_HERDR_SESSION;
|
|
170
|
+
}
|
|
171
|
+
// conductor answered (even empty) wins over a dead legacy session.
|
|
172
|
+
if (primary.ok) return DEFAULT_HERDR_SESSION;
|
|
173
|
+
if (legacy.ok) {
|
|
174
|
+
noteLegacyHerdrSession(opts.log);
|
|
175
|
+
return LEGACY_HERDR_SESSION;
|
|
176
|
+
}
|
|
177
|
+
return DEFAULT_HERDR_SESSION;
|
|
178
|
+
}
|
|
179
|
+
|
|
82
180
|
export function telegramStateDir(): string {
|
|
83
181
|
const override = process.env["OMP_TELEGRAM_STATE_DIR"];
|
|
84
182
|
if (override !== undefined && override.length > 0) return override;
|
|
@@ -103,25 +201,69 @@ export type ResolvedTick =
|
|
|
103
201
|
| { kind: "invalid"; path: string; cwd: string; problem: string }
|
|
104
202
|
| { kind: "ok"; path: string; cwd: string; config: TickConfig };
|
|
105
203
|
|
|
204
|
+
/**
|
|
205
|
+
* The tick config that belongs to this project, skipping any that names another
|
|
206
|
+
* one. The search roots overlap between projects — `stateDir()` and the shared
|
|
207
|
+
* parent of two fleet cwds are read for every one of them — so a stamped config
|
|
208
|
+
* sitting in a shared root would otherwise answer for whichever project asked
|
|
209
|
+
* first. An unstamped (pre-multi-project) config still matches anything, which
|
|
210
|
+
* is what keeps a single-project fleet on the file it already has.
|
|
211
|
+
*/
|
|
106
212
|
export function resolveTickConfig(projectName?: string): ResolvedTick {
|
|
107
213
|
for (const cwd of tickConfigSearchRoots(projectName)) {
|
|
108
214
|
const r: TickConfigResult = readTickConfig(cwd);
|
|
109
|
-
if (r.kind === "ok")
|
|
215
|
+
if (r.kind === "ok") {
|
|
216
|
+
if (!tickConfigMatchesProject(r.config, projectName)) continue;
|
|
217
|
+
return { kind: "ok", path: r.path, cwd, config: r.config };
|
|
218
|
+
}
|
|
110
219
|
if (r.kind === "invalid") return { kind: "invalid", path: r.path, cwd, problem: r.problem };
|
|
111
220
|
}
|
|
112
221
|
return { kind: "absent" };
|
|
113
222
|
}
|
|
114
223
|
|
|
224
|
+
/**
|
|
225
|
+
* Where this project's arm marker lives. The per-project fallback is the point:
|
|
226
|
+
* one shared `<stateDir>/armed` meant arming any project armed all of them. The
|
|
227
|
+
* bare name survives only for the un-named single-project call, which is the
|
|
228
|
+
* pre-multi-project spelling.
|
|
229
|
+
*/
|
|
115
230
|
export function armedMarkerPath(projectName?: string): string {
|
|
231
|
+
return armGate(projectName).path;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/**
|
|
235
|
+
* This project's arm marker together with the project name that identifies it —
|
|
236
|
+
* the pair every arm decision needs, from one tick-config read. `named` prefers
|
|
237
|
+
* the stamped {@link TickConfig.project} over the caller's `--project`, because
|
|
238
|
+
* the stamp is what the marker was generated from.
|
|
239
|
+
*/
|
|
240
|
+
function armGate(projectName?: string): { path: string; named?: string } {
|
|
116
241
|
const tick = resolveTickConfig(projectName);
|
|
117
|
-
|
|
118
|
-
|
|
242
|
+
const configured = tick.kind === "ok" ? tick.config : undefined;
|
|
243
|
+
const named = configured?.project ?? projectName;
|
|
244
|
+
const path =
|
|
245
|
+
configured?.armedFile ??
|
|
246
|
+
(projectName === undefined ? legacyArmedMarkerPath() : join(stateDir(), `armed-${projectName}`));
|
|
247
|
+
return named === undefined ? { path } : { path, named };
|
|
119
248
|
}
|
|
120
249
|
|
|
250
|
+
/**
|
|
251
|
+
* Clears the arm gate for this project — and `wasArmed` is the gate the
|
|
252
|
+
* heartbeat reads, not merely one file's presence.
|
|
253
|
+
*
|
|
254
|
+
* The second removal is the whole reason this is not a one-line `rmSync`. While
|
|
255
|
+
* the shared pre-per-project marker is still honoured (single-project host, an
|
|
256
|
+
* `armedFile` just restamped to `armed-<name>`), it holds the gate open as soon
|
|
257
|
+
* as this project's own marker is gone — so a disarm that removed only the
|
|
258
|
+
* per-project file would leave the fleet ticking, which is exactly what `hold`
|
|
259
|
+
* exists to prevent. A *stranded* shared marker arms nothing, so it is not this
|
|
260
|
+
* command's to remove.
|
|
261
|
+
*/
|
|
121
262
|
export function disarmTicks(projectName?: string): { path: string; wasArmed: boolean } {
|
|
122
|
-
const path =
|
|
123
|
-
const wasArmed =
|
|
263
|
+
const { path, named } = armGate(projectName);
|
|
264
|
+
const wasArmed = resolveArmState(path, named).armed;
|
|
124
265
|
rmSync(path, { force: true });
|
|
266
|
+
if (resolveArmState(path, named).legacy === "honoured") rmSync(legacyArmedMarkerPath(), { force: true });
|
|
125
267
|
return { path, wasArmed };
|
|
126
268
|
}
|
|
127
269
|
|
|
@@ -133,7 +275,7 @@ export interface ArmResult {
|
|
|
133
275
|
}
|
|
134
276
|
|
|
135
277
|
export interface ArmDeps {
|
|
136
|
-
sendChallenge?: (token: string, owner: string, text: string) => Promise<void>;
|
|
278
|
+
sendChallenge?: (token: string, owner: string, text: string, topicId?: number) => Promise<void>;
|
|
137
279
|
/**
|
|
138
280
|
* Waits for the challenge to appear as a user turn somewhere under the
|
|
139
281
|
* session directory. The waiter owns transcript discovery — not the caller —
|
|
@@ -189,18 +331,38 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
189
331
|
}
|
|
190
332
|
|
|
191
333
|
const path = tick.config.armedFile;
|
|
192
|
-
|
|
334
|
+
// The gate as the heartbeat reads it, so "replaced previous marker" is not a
|
|
335
|
+
// lie about a fleet the shared marker was arming, and so the write below knows
|
|
336
|
+
// whether it is superseding that marker.
|
|
337
|
+
const arm = resolveArmState(path, tick.config.project ?? projectName);
|
|
338
|
+
const alreadyArmed = arm.armed;
|
|
193
339
|
const code = makeChallengeCode();
|
|
340
|
+
// One bot, one chat, and — once a host runs more than one fleet — more than
|
|
341
|
+
// one pane that can ask. The challenge names which one, or the operator is
|
|
342
|
+
// answering a question they cannot attribute.
|
|
343
|
+
const named = tick.config.project ?? projectName;
|
|
194
344
|
const text =
|
|
195
|
-
`Fleet arming check
|
|
345
|
+
`Fleet arming check${named === undefined ? "" : ` — project ${named}`}. ` +
|
|
346
|
+
`Reply to this chat with exactly:\n${code}\n` +
|
|
196
347
|
`Nothing will be dispatched until that reply is seen in the orchestrator session.`;
|
|
197
348
|
|
|
349
|
+
// Prefer the project's configured forum topic so arm challenges land where
|
|
350
|
+
// escalations already do (#318). Missing project config keeps flat-chat 0.13.
|
|
351
|
+
let topicId: number | undefined;
|
|
352
|
+
if (named !== undefined) {
|
|
353
|
+
try {
|
|
354
|
+
topicId = findProject(loadConfig(), named).escalation.telegramTopicId;
|
|
355
|
+
} catch {
|
|
356
|
+
/* no project config */
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
|
|
198
360
|
const send = deps.sendChallenge ?? sendTelegramMessage;
|
|
199
361
|
// Read before the send, not after: a transcript untouched since this instant
|
|
200
362
|
// cannot contain the reply, and that is what the waiter filters on.
|
|
201
363
|
const sentAt = (deps.now ?? Date.now)();
|
|
202
364
|
try {
|
|
203
|
-
await send(token, channel.owner, text);
|
|
365
|
+
await send(token, channel.owner, text, topicId);
|
|
204
366
|
} catch (err) {
|
|
205
367
|
throw new Error(
|
|
206
368
|
`arm: outbound sendMessage failed — NOT armed: ${err instanceof Error ? err.message : String(err)}`,
|
|
@@ -231,6 +393,11 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
231
393
|
|
|
232
394
|
mkdirSync(dirname(path), { recursive: true });
|
|
233
395
|
writeFileSync(path, `armed ${new Date().toISOString()} owner=${channel.owner}\n`, { mode: 0o600 });
|
|
396
|
+
// This project now has its own marker, so the shared one it was borrowing has
|
|
397
|
+
// done its last job. Left in place it would survive the next `disarm` as a
|
|
398
|
+
// marker that re-arms the fleet, and turn into a meaningless legacy warning
|
|
399
|
+
// the moment a second project is configured.
|
|
400
|
+
if (arm.legacy === "honoured") rmSync(legacyArmedMarkerPath(), { force: true });
|
|
234
401
|
return { path, alreadyArmed, owner: channel.owner, challenge: code };
|
|
235
402
|
}
|
|
236
403
|
|
|
@@ -257,13 +424,13 @@ export interface HaltWithPaneResult extends HaltResult {
|
|
|
257
424
|
}
|
|
258
425
|
|
|
259
426
|
export function hold(projectName?: string, source: string = "hold"): HoldResult {
|
|
260
|
-
const wasPaused = isPaused();
|
|
261
|
-
setPaused(true, { source });
|
|
427
|
+
const wasPaused = isPaused(projectName);
|
|
428
|
+
setPaused(true, { source }, projectName);
|
|
262
429
|
return { wasPaused, disarmed: disarmTicks(projectName) };
|
|
263
430
|
}
|
|
264
431
|
|
|
265
|
-
export function releaseHold(): void {
|
|
266
|
-
setPaused(false);
|
|
432
|
+
export function releaseHold(projectName?: string): void {
|
|
433
|
+
setPaused(false, undefined, projectName);
|
|
267
434
|
}
|
|
268
435
|
|
|
269
436
|
export async function halt(projectName?: string): Promise<HaltResult> {
|
|
@@ -311,11 +478,11 @@ export function resolvePaneHaltPath(projectName?: string): ResolvedPaneHalt {
|
|
|
311
478
|
export function paneHaltPath(projectName?: string): string {
|
|
312
479
|
const resolved = resolvePaneHaltPath(projectName);
|
|
313
480
|
if (resolved.kind === "unresolved") {
|
|
314
|
-
// Verb-neutral: this is also
|
|
481
|
+
// Verb-neutral: this is also `resume`'s pin-clearing path.
|
|
315
482
|
throw new Error(
|
|
316
483
|
`cannot locate the pane recovery pin — ${resolved.reason}. ` +
|
|
317
484
|
`Drop a tick config beside the pane, or stop the agent by hand; ` +
|
|
318
|
-
`\`
|
|
485
|
+
`\`stop\` without \`--pane\` still stops the dispatch daemon.`,
|
|
319
486
|
);
|
|
320
487
|
}
|
|
321
488
|
return resolved.path;
|
|
@@ -327,9 +494,9 @@ export function pinPaneHalt(projectName?: string): { path: string } {
|
|
|
327
494
|
writeFileSync(
|
|
328
495
|
path,
|
|
329
496
|
[
|
|
330
|
-
`# Written by omp-conductor
|
|
497
|
+
`# Written by omp-conductor stop --pane at ${new Date().toISOString()}`,
|
|
331
498
|
`# herdr-conductor recover.sh must not resume the fleet agent while this file exists.`,
|
|
332
|
-
`# Clear with: omp-conductor
|
|
499
|
+
`# Clear with: omp-conductor resume (or rm this file)`,
|
|
333
500
|
"",
|
|
334
501
|
].join("\n"),
|
|
335
502
|
{ mode: 0o600 },
|
|
@@ -337,6 +504,23 @@ export function pinPaneHalt(projectName?: string): { path: string } {
|
|
|
337
504
|
return { path };
|
|
338
505
|
}
|
|
339
506
|
|
|
507
|
+
/**
|
|
508
|
+
* Clears the pin when its location is known, and reports plainly when it is not.
|
|
509
|
+
*
|
|
510
|
+
* Distinct from {@link clearPaneHalt}, which refuses an unresolvable location
|
|
511
|
+
* because pinning somewhere recovery never reads is worse than failing. `resume`
|
|
512
|
+
* needs the softer answer: it folds in the pin clear, and an operator resuming a
|
|
513
|
+
* fleet from a host with no tick config has no pin to clear rather than an error
|
|
514
|
+
* to work around.
|
|
515
|
+
*/
|
|
516
|
+
export function clearPaneHaltIfResolvable(projectName?: string): { path?: string; wasHalted: boolean } {
|
|
517
|
+
const resolved = resolvePaneHaltPath(projectName);
|
|
518
|
+
if (resolved.kind !== "ok") return { wasHalted: false };
|
|
519
|
+
const wasHalted = existsSync(resolved.path);
|
|
520
|
+
rmSync(resolved.path, { force: true });
|
|
521
|
+
return { path: resolved.path, wasHalted };
|
|
522
|
+
}
|
|
523
|
+
|
|
340
524
|
export function clearPaneHalt(projectName?: string): { path: string; wasHalted: boolean } {
|
|
341
525
|
const path = paneHaltPath(projectName);
|
|
342
526
|
const wasHalted = existsSync(path);
|
|
@@ -655,7 +839,9 @@ export async function stopConductorPane(
|
|
|
655
839
|
|
|
656
840
|
async function herdrAgentList(deps: PaneStopDeps): Promise<HerdrAgent[]> {
|
|
657
841
|
const bin = deps.herdrBin ?? "herdr";
|
|
658
|
-
const session =
|
|
842
|
+
const session =
|
|
843
|
+
deps.herdrSession ??
|
|
844
|
+
resolveHerdrSessionWithBridge({ env: process.env, herdrBin: bin });
|
|
659
845
|
const res = spawnSync(bin, ["--session", session, "agent", "list"], {
|
|
660
846
|
encoding: "utf8",
|
|
661
847
|
timeout: 8_000,
|
|
@@ -729,7 +915,9 @@ export function parseHerdrAgentList(rawOutput: string): HerdrAgent[] {
|
|
|
729
915
|
|
|
730
916
|
async function herdrOmpForegroundPids(paneId: string, deps: PaneStopDeps): Promise<number[]> {
|
|
731
917
|
const bin = deps.herdrBin ?? "herdr";
|
|
732
|
-
const session =
|
|
918
|
+
const session =
|
|
919
|
+
deps.herdrSession ??
|
|
920
|
+
resolveHerdrSessionWithBridge({ env: process.env, herdrBin: bin });
|
|
733
921
|
const res = spawnSync(bin, ["--session", session, "pane", "process-info", "--pane", paneId], {
|
|
734
922
|
encoding: "utf8",
|
|
735
923
|
timeout: 8_000,
|
|
@@ -835,7 +1023,7 @@ export interface FleetLayers {
|
|
|
835
1023
|
|
|
836
1024
|
export function fleetLayers(projectName?: string): FleetLayers {
|
|
837
1025
|
const rec = livingDaemon();
|
|
838
|
-
const paused = isPaused();
|
|
1026
|
+
const paused = isPaused(projectName);
|
|
839
1027
|
const dispatch: DispatchLayer = rec === undefined ? "stopped" : paused ? "paused" : "running";
|
|
840
1028
|
|
|
841
1029
|
const tick = resolveTickConfig(projectName);
|
|
@@ -856,12 +1044,21 @@ export function fleetLayers(projectName?: string): FleetLayers {
|
|
|
856
1044
|
if (tick.config.armedFile === undefined) {
|
|
857
1045
|
ticks = "ungated";
|
|
858
1046
|
ticksDetail = "no armedFile in tick config — heartbeat sends without an arm marker";
|
|
859
|
-
} else if (existsSync(tick.config.armedFile)) {
|
|
860
|
-
ticks = "armed";
|
|
861
|
-
ticksDetail = tick.config.armedFile;
|
|
862
1047
|
} else {
|
|
863
|
-
|
|
864
|
-
|
|
1048
|
+
// The same decision the heartbeat itself makes, so status can never claim
|
|
1049
|
+
// armed while the tick skips (or the reverse) — including the two shared
|
|
1050
|
+
// marker cases: honoured through an upgrade on a single-project host,
|
|
1051
|
+
// stranded once a second project exists.
|
|
1052
|
+
const arm = resolveArmState(tick.config.armedFile, tick.config.project ?? projectName);
|
|
1053
|
+
ticks = arm.armed ? "armed" : "disarmed";
|
|
1054
|
+
ticksDetail =
|
|
1055
|
+
arm.legacy === undefined
|
|
1056
|
+
? tick.config.armedFile
|
|
1057
|
+
: `${tick.config.armedFile} — ${legacyArmedMarkerPath()}: ${
|
|
1058
|
+
arm.legacy === "honoured"
|
|
1059
|
+
? "legacy global arm marker still honoured; re-arm this project to replace it"
|
|
1060
|
+
: LEGACY_ARM_MARKER_DETAIL
|
|
1061
|
+
}`;
|
|
865
1062
|
}
|
|
866
1063
|
const runtime = readTickRuntimeStatus(tick.cwd);
|
|
867
1064
|
if (runtime !== undefined && isAlive(runtime.pid)) nextTickAt = runtime.nextTickAt;
|
|
@@ -915,8 +1112,27 @@ export function codeGraphFromHealthz(body: string | undefined, project: string):
|
|
|
915
1112
|
if (body === undefined) return undefined;
|
|
916
1113
|
try {
|
|
917
1114
|
const parsed = JSON.parse(body) as Record<string, unknown>;
|
|
918
|
-
|
|
919
|
-
|
|
1115
|
+
let block: Record<string, unknown> | undefined;
|
|
1116
|
+
if (parsed["project"] === project) {
|
|
1117
|
+
block = parsed;
|
|
1118
|
+
} else {
|
|
1119
|
+
const projects = parsed["projects"];
|
|
1120
|
+
if (Array.isArray(projects)) {
|
|
1121
|
+
for (const entry of projects) {
|
|
1122
|
+
if (
|
|
1123
|
+
entry !== null &&
|
|
1124
|
+
typeof entry === "object" &&
|
|
1125
|
+
!Array.isArray(entry) &&
|
|
1126
|
+
(entry as Record<string, unknown>)["project"] === project
|
|
1127
|
+
) {
|
|
1128
|
+
block = entry as Record<string, unknown>;
|
|
1129
|
+
break;
|
|
1130
|
+
}
|
|
1131
|
+
}
|
|
1132
|
+
}
|
|
1133
|
+
}
|
|
1134
|
+
if (block === undefined) return undefined;
|
|
1135
|
+
const graph = block["codeGraph"];
|
|
920
1136
|
if (graph === null || typeof graph !== "object" || Array.isArray(graph)) return undefined;
|
|
921
1137
|
const value = graph as Record<string, unknown>;
|
|
922
1138
|
if (value["configured"] === false) return { configured: false };
|
|
@@ -941,6 +1157,57 @@ export function codeGraphFromHealthz(body: string | undefined, project: string):
|
|
|
941
1157
|
}
|
|
942
1158
|
}
|
|
943
1159
|
|
|
1160
|
+
|
|
1161
|
+
/** Live worker pause phases from a daemon `/healthz` body for one project. */
|
|
1162
|
+
export function workerPhasesFromHealthz(
|
|
1163
|
+
body: string | undefined,
|
|
1164
|
+
project: string,
|
|
1165
|
+
): ReadonlyMap<number, WorkerPausePhase> {
|
|
1166
|
+
const phases = new Map<number, WorkerPausePhase>();
|
|
1167
|
+
if (body === undefined) return phases;
|
|
1168
|
+
try {
|
|
1169
|
+
const payload = JSON.parse(body) as unknown;
|
|
1170
|
+
if (payload === null || typeof payload !== "object") return phases;
|
|
1171
|
+
let block: object | undefined;
|
|
1172
|
+
if (Reflect.get(payload, "project") === project) {
|
|
1173
|
+
block = payload;
|
|
1174
|
+
} else {
|
|
1175
|
+
const projects = Reflect.get(payload, "projects");
|
|
1176
|
+
if (Array.isArray(projects)) {
|
|
1177
|
+
for (const entry of projects) {
|
|
1178
|
+
if (
|
|
1179
|
+
entry !== null &&
|
|
1180
|
+
typeof entry === "object" &&
|
|
1181
|
+
Reflect.get(entry, "project") === project
|
|
1182
|
+
) {
|
|
1183
|
+
block = entry;
|
|
1184
|
+
break;
|
|
1185
|
+
}
|
|
1186
|
+
}
|
|
1187
|
+
}
|
|
1188
|
+
}
|
|
1189
|
+
if (block === undefined) return phases;
|
|
1190
|
+
const workers = Reflect.get(block, "workers");
|
|
1191
|
+
if (!Array.isArray(workers)) return phases;
|
|
1192
|
+
for (const worker of workers) {
|
|
1193
|
+
if (worker === null || typeof worker !== "object") continue;
|
|
1194
|
+
const issue = Reflect.get(worker, "issue");
|
|
1195
|
+
const phase = Reflect.get(worker, "phase");
|
|
1196
|
+
if (
|
|
1197
|
+
Number.isSafeInteger(issue) &&
|
|
1198
|
+
(issue as number) > 0 &&
|
|
1199
|
+
(phase === "pausing" || phase === "paused")
|
|
1200
|
+
) {
|
|
1201
|
+
phases.set(issue as number, phase);
|
|
1202
|
+
}
|
|
1203
|
+
}
|
|
1204
|
+
} catch {
|
|
1205
|
+
// An unreadable health body means no trustworthy pause phase.
|
|
1206
|
+
}
|
|
1207
|
+
return phases;
|
|
1208
|
+
}
|
|
1209
|
+
|
|
1210
|
+
|
|
944
1211
|
export function formatCodeGraphHealth(graph: CodeGraphHealth, now = Date.now()): string | undefined {
|
|
945
1212
|
if (!graph.configured) return undefined;
|
|
946
1213
|
const indexed = graph.repos.filter((repo) => repo.index === "present").length;
|
|
@@ -970,6 +1237,7 @@ export function formatFleetStatus(
|
|
|
970
1237
|
brief: string | undefined = undefined,
|
|
971
1238
|
decisions: string | undefined = undefined,
|
|
972
1239
|
failureClasses: string | undefined = undefined,
|
|
1240
|
+
workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
|
|
973
1241
|
): string {
|
|
974
1242
|
const tickLine =
|
|
975
1243
|
layers.ticksDetail === undefined
|
|
@@ -1031,12 +1299,12 @@ export function formatFleetStatus(
|
|
|
1031
1299
|
const dispatchLine =
|
|
1032
1300
|
layers.dispatch === "paused"
|
|
1033
1301
|
? (() => {
|
|
1034
|
-
const prov = pauseProvenance();
|
|
1302
|
+
const prov = pauseProvenance(s.project);
|
|
1035
1303
|
if (prov !== undefined) {
|
|
1036
1304
|
const reason = prov.reason === undefined ? "" : ` — "${prov.reason}"`;
|
|
1037
1305
|
return `dispatch paused (source: ${prov.source}${reason})`;
|
|
1038
1306
|
}
|
|
1039
|
-
return isPaused() && pausedAt() === undefined
|
|
1307
|
+
return isPaused(s.project) && pausedAt(s.project) === undefined
|
|
1040
1308
|
? "dispatch paused (unparseable sentinel — new work and completion mutations refused; release gates remain available)"
|
|
1041
1309
|
: "dispatch paused";
|
|
1042
1310
|
})()
|
|
@@ -1056,16 +1324,48 @@ export function formatFleetStatus(
|
|
|
1056
1324
|
...(graphBlock === undefined ? [] : [graphBlock]),
|
|
1057
1325
|
daemonBlock,
|
|
1058
1326
|
"",
|
|
1059
|
-
formatProjectBody(s),
|
|
1327
|
+
formatProjectBody(s, workerPhases),
|
|
1060
1328
|
].join("\n");
|
|
1061
1329
|
}
|
|
1062
1330
|
|
|
1063
|
-
function
|
|
1331
|
+
function formatAvailabilityStatus(s: StatusSnapshot): string[] {
|
|
1332
|
+
const availability = s.availability;
|
|
1333
|
+
if (availability === undefined) return [];
|
|
1334
|
+
if (availability.mode === "always") {
|
|
1335
|
+
return ["availability 24-hour interrupts (no weekly window)"];
|
|
1336
|
+
}
|
|
1337
|
+
const mode =
|
|
1338
|
+
availability.nextTransitionAt === undefined || availability.timezone === undefined
|
|
1339
|
+
? `${availability.mode}; next transition could not be calculated`
|
|
1340
|
+
: `${availability.mode} until ${formatZonedMinute(availability.nextTransitionAt, availability.timezone)}`;
|
|
1341
|
+
const bypass = availability.bypass.length === 0 ? "none" : availability.bypass.join(", ");
|
|
1342
|
+
return [`availability ${mode}; quiet-hours bypass ${bypass}`];
|
|
1343
|
+
}
|
|
1344
|
+
|
|
1345
|
+
function formatDigestScheduleStatus(s: StatusSnapshot): string[] {
|
|
1346
|
+
const schedule = s.digestSchedule;
|
|
1347
|
+
if (schedule === undefined) return [];
|
|
1348
|
+
if (schedule.mode === "disabled") return ["next digest disabled"];
|
|
1349
|
+
if (schedule.mode === "per-tick") return ["next digest every tick"];
|
|
1350
|
+
if (schedule.mode === "due") return ["next digest due now"];
|
|
1351
|
+
return [
|
|
1352
|
+
schedule.nextAt === undefined
|
|
1353
|
+
? "next digest could not be calculated"
|
|
1354
|
+
: `next digest ${formatZonedMinute(schedule.nextAt, schedule.timezone)}`,
|
|
1355
|
+
];
|
|
1356
|
+
}
|
|
1357
|
+
|
|
1358
|
+
function formatProjectBody(
|
|
1359
|
+
s: StatusSnapshot,
|
|
1360
|
+
workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
|
|
1361
|
+
): string {
|
|
1064
1362
|
const lines = [
|
|
1065
1363
|
`project ${s.project}${s.paused ? " (PAUSED)" : ""}`,
|
|
1066
1364
|
...(s.pauseReason === undefined ? [] : [`paused ${s.pauseReason}`]),
|
|
1067
1365
|
`config ${s.configPath}`,
|
|
1068
1366
|
`state ${s.stateDir}`,
|
|
1367
|
+
...formatAvailabilityStatus(s),
|
|
1368
|
+
...formatDigestScheduleStatus(s),
|
|
1069
1369
|
"",
|
|
1070
1370
|
"caps",
|
|
1071
1371
|
` workers ${s.liveWorkers} / ${s.caps.maxConcurrentWorkers}`,
|
|
@@ -1128,9 +1428,11 @@ function formatProjectBody(s: StatusSnapshot): string {
|
|
|
1128
1428
|
} else {
|
|
1129
1429
|
lines.push("active runs");
|
|
1130
1430
|
for (const r of s.activeRuns) {
|
|
1431
|
+
const phase = workerPhases.get(r.issue);
|
|
1432
|
+
const state = phase === "pausing" || phase === "paused" ? phase : r.state;
|
|
1131
1433
|
lines.push(
|
|
1132
|
-
` #${r.issue} ${r.repo} ${
|
|
1133
|
-
`${r.turns}/${r.maxTurns} turns
|
|
1434
|
+
` #${r.issue} ${r.repo} ${state} attempt ${r.attempt} ` +
|
|
1435
|
+
`${r.turns}/${r.maxTurns} turns ${r.spendUsd.toFixed(2)} ${r.branch}` +
|
|
1134
1436
|
(r.prUrl ? ` ${r.prUrl}` : ""),
|
|
1135
1437
|
);
|
|
1136
1438
|
// The orchestrator's Duty 1 reads this command, and a flagged run's
|
|
@@ -1140,9 +1442,10 @@ function formatProjectBody(s: StatusSnapshot): string {
|
|
|
1140
1442
|
if (flagged !== undefined) lines.push(` ${flagged}`);
|
|
1141
1443
|
}
|
|
1142
1444
|
}
|
|
1143
|
-
lines.push(...
|
|
1445
|
+
lines.push(...formatBaseHealth(s.baseHealth));
|
|
1144
1446
|
lines.push(...formatSalvagedRuns(s.salvagedRuns));
|
|
1145
1447
|
lines.push(...formatOpenReports(s.openReports));
|
|
1448
|
+
lines.push(...formatDigestBacklog(s.digestBacklog));
|
|
1146
1449
|
if (s.liveWorkers > 0) {
|
|
1147
1450
|
lines.push(
|
|
1148
1451
|
"",
|
|
@@ -1172,6 +1475,7 @@ export async function renderStatus(projectName?: string): Promise<string> {
|
|
|
1172
1475
|
]);
|
|
1173
1476
|
const cached = codeGraphFromHealthz(health?.body, project.name);
|
|
1174
1477
|
const codeGraph = cached ?? (await probeCodeGraph(project));
|
|
1478
|
+
const workerPhases = workerPhasesFromHealthz(health?.body, project.name);
|
|
1175
1479
|
return formatFleetStatus(
|
|
1176
1480
|
{ ...s, planUsage, github },
|
|
1177
1481
|
layers,
|
|
@@ -1182,6 +1486,7 @@ export async function renderStatus(projectName?: string): Promise<string> {
|
|
|
1182
1486
|
briefStatusLine(project),
|
|
1183
1487
|
decisionStatusLine(project.name),
|
|
1184
1488
|
failureClassBlock(project.name),
|
|
1489
|
+
workerPhases,
|
|
1185
1490
|
);
|
|
1186
1491
|
}
|
|
1187
1492
|
|
|
@@ -1402,14 +1707,14 @@ function makeChallengeCode(): string {
|
|
|
1402
1707
|
return `FLEET-${hex}`;
|
|
1403
1708
|
}
|
|
1404
1709
|
|
|
1405
|
-
async function sendTelegramMessage(
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
}
|
|
1710
|
+
async function sendTelegramMessage(
|
|
1711
|
+
token: string,
|
|
1712
|
+
owner: string,
|
|
1713
|
+
text: string,
|
|
1714
|
+
topicId?: number,
|
|
1715
|
+
): Promise<void> {
|
|
1716
|
+
// Shared transport: stale-topic retry + message_thread_id live in one place.
|
|
1717
|
+
await sendTelegram(token, owner, text, { topicId });
|
|
1413
1718
|
}
|
|
1414
1719
|
|
|
1415
1720
|
interface SessionScan {
|
|
@@ -1576,7 +1881,7 @@ function probeOmpPane(
|
|
|
1576
1881
|
}
|
|
1577
1882
|
const agentName =
|
|
1578
1883
|
tick.kind === "ok" ? (tick.config.agentName ?? DEFAULT_FLEET_AGENT_NAME) : DEFAULT_FLEET_AGENT_NAME;
|
|
1579
|
-
const session = process.env
|
|
1884
|
+
const session = resolveHerdrSessionWithBridge({ env: process.env });
|
|
1580
1885
|
try {
|
|
1581
1886
|
const res = spawnSync("herdr", ["--session", session, "agent", "list"], {
|
|
1582
1887
|
encoding: "utf8",
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Regenerator for `schema/config.schema.json`.
|
|
3
|
+
*
|
|
4
|
+
* Run via `bun run schema` from the package root. Writes the draft 2020-12 JSON
|
|
5
|
+
* Schema rendering of `ConfigSchema` (see `config-schema.ts`) to
|
|
6
|
+
* `schema/config.schema.json`, which ships in the package so editors can
|
|
7
|
+
* validate a hand-written config. The `config.test.ts` freshness lock fails
|
|
8
|
+
* when a committed schema drifts from what this script produces.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { mkdirSync, writeFileSync } from "node:fs";
|
|
12
|
+
import { dirname, join } from "node:path";
|
|
13
|
+
import { fileURLToPath } from "node:url";
|
|
14
|
+
import { configJsonSchema } from "./config-schema.ts";
|
|
15
|
+
|
|
16
|
+
const packageDir = join(dirname(fileURLToPath(import.meta.url)), "..");
|
|
17
|
+
const schemaDir = join(packageDir, "schema");
|
|
18
|
+
mkdirSync(schemaDir, { recursive: true });
|
|
19
|
+
const out = join(schemaDir, "config.schema.json");
|
|
20
|
+
writeFileSync(out, `${JSON.stringify(configJsonSchema(), null, 2)}\n`);
|
|
21
|
+
console.log(`wrote ${out}`);
|