omp-conductor 0.15.12 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +81 -6
- package/package.json +2 -1
- package/schema/config.schema.json +6 -0
- package/src/admission.ts +745 -0
- package/src/ask.ts +47 -0
- package/src/backups.ts +19 -7
- package/src/board.ts +1 -2
- package/src/briefs/orchestrator.md +62 -4
- package/src/cli.ts +26 -0
- package/src/commands/context.ts +3 -0
- package/src/commands/decision.ts +10 -1
- package/src/commands/doctor.ts +2 -0
- package/src/commands/message.ts +8 -1
- package/src/commands/restart.ts +93 -54
- package/src/commands/restore-db.ts +146 -0
- package/src/commands/stop.ts +66 -34
- package/src/commands/unfreeze.ts +56 -0
- package/src/commands/watch.ts +77 -0
- package/src/config-schema.ts +9 -0
- package/src/config.ts +24 -0
- package/src/daemon.ts +485 -577
- package/src/dashboard/server.ts +2 -1
- package/src/decisions.ts +32 -7
- package/src/depends-on.ts +73 -0
- package/src/doctor.ts +418 -8
- package/src/escalate.ts +122 -15
- package/src/failure-class.ts +47 -0
- package/src/fleet.ts +55 -377
- package/src/gitops.ts +86 -1
- package/src/lifecycle.ts +113 -2
- package/src/log.ts +40 -0
- package/src/model-fallback.ts +3 -2
- package/src/omp-settings.ts +114 -0
- package/src/omp.ts +63 -0
- package/src/orchestrator-down.ts +231 -0
- package/src/orchestrator-tick.ts +14 -1
- package/src/orchestrator.ts +14 -0
- package/src/release-policy.ts +163 -18
- package/src/reports.ts +124 -12
- package/src/session-host.ts +6 -0
- package/src/setup-host.ts +386 -17
- package/src/setup-install.ts +40 -2
- package/src/setup-wizard.ts +314 -113
- package/src/setup.ts +58 -1
- package/src/status-render.ts +445 -0
- package/src/stop-provenance.ts +119 -0
- package/src/store.ts +533 -11
- package/src/types.ts +298 -4
- package/src/unblock.ts +1 -1
- package/src/upgrade-verify.ts +1 -1
- package/src/upgrade.ts +27 -8
- package/src/verbs/protocol.ts +16 -3
- package/src/verbs/server.ts +52 -1
- package/src/wizard-ui.ts +261 -46
- package/src/worker.ts +183 -10
package/src/dashboard/server.ts
CHANGED
|
@@ -28,7 +28,8 @@ import { chmodSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync }
|
|
|
28
28
|
import { join } from "node:path";
|
|
29
29
|
import { loadConfig, stateDir } from "../config.ts";
|
|
30
30
|
import { healthCheck, livingDaemon, type DaemonRecord } from "../lifecycle.ts";
|
|
31
|
-
import { classifyDaemonProjectHealth, fleetLayers
|
|
31
|
+
import { classifyDaemonProjectHealth, fleetLayers } from "../fleet.ts";
|
|
32
|
+
import type { FleetLayers } from "../status-render.ts";
|
|
32
33
|
import { statusSnapshot, type StatusSnapshot } from "../daemon.ts";
|
|
33
34
|
import { boardJson, boardSnapshotOnce } from "../board.ts";
|
|
34
35
|
import { dbPath, openStore } from "../store.ts";
|
package/src/decisions.ts
CHANGED
|
@@ -235,15 +235,40 @@ function age(since: number, now: number): string {
|
|
|
235
235
|
* being a memory question. A row whose condition has come true is flagged, not
|
|
236
236
|
* merely listed — that is the difference between a parked question and one to
|
|
237
237
|
* act on now.
|
|
238
|
+
*
|
|
239
|
+
* The two kinds render under their own headings (#459). Only rows a human must
|
|
240
|
+
* answer — `kind === "question"`, whatever condition they carry — appear under
|
|
241
|
+
* the operator heading; a watch the orchestrator set for itself goes under the
|
|
242
|
+
* watch heading with no instruction to resolve it, so a fleet at rest behind
|
|
243
|
+
* GitHub's checks is never mistaken for a fleet that is waiting on its
|
|
244
|
+
* operator. A met watch still surfaces to the orchestrator with its note and
|
|
245
|
+
* the same `[CONDITION MET]` flag a met question gets.
|
|
238
246
|
*/
|
|
239
247
|
export function formatDecisionDigest(open: readonly DecisionRecord[], now = Date.now()): string {
|
|
240
|
-
|
|
241
|
-
const
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
248
|
+
const questions = open.filter((d) => d.kind !== "watch");
|
|
249
|
+
const watches = open.filter((d) => d.kind === "watch");
|
|
250
|
+
const lines: string[] = [];
|
|
251
|
+
if (questions.length > 0) {
|
|
252
|
+
lines.push(
|
|
253
|
+
`Open operator decisions (${questions.length}) — resolve or withdraw each with omp-conductor decision resolve|withdraw <id>:`,
|
|
254
|
+
);
|
|
255
|
+
for (const d of questions) {
|
|
256
|
+
lines.push(decisionLine(d, now));
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
if (watches.length > 0) {
|
|
260
|
+
lines.push(
|
|
261
|
+
`Watches (${watches.length}) — conditions the orchestrator set for itself; no operator action needed:`,
|
|
262
|
+
);
|
|
263
|
+
for (const d of watches) {
|
|
264
|
+
lines.push(decisionLine(d, now));
|
|
265
|
+
}
|
|
247
266
|
}
|
|
248
267
|
return lines.join("\n");
|
|
249
268
|
}
|
|
269
|
+
|
|
270
|
+
/** One digest row: id, age, what it blocks, the met flag, and the note. */
|
|
271
|
+
function decisionLine(d: DecisionRecord, now: number): string {
|
|
272
|
+
const flag = d.conditionMetAt === undefined ? "" : " [CONDITION MET — act on this now]";
|
|
273
|
+
return `- ${d.id} (${age(d.askedAt, now)}, blocks ${d.blocks ?? "nothing"})${flag}: ${d.question}`;
|
|
274
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `Depends-on:` declaration parser (epic #321, slice #419).
|
|
3
|
+
*
|
|
4
|
+
* A candidate issue can declare same-repo prerequisites it must not be
|
|
5
|
+
* dispatched before: `Depends-on: #123` (repeatable, one or more per line).
|
|
6
|
+
* Admission reads every referenced issue's live tracker state and holds the
|
|
7
|
+
* candidate while any prerequisite is open, so a worker is never sent at work
|
|
8
|
+
* whose base is not yet merged.
|
|
9
|
+
*
|
|
10
|
+
* The parser is deliberately small and strict, the two halves of the contract:
|
|
11
|
+
* the *marker* is matched case-insensitively (so `DEPENDS-ON`, `Depends-On`,
|
|
12
|
+
* `depends-on` all declare), while the *references* are strict bare issue
|
|
13
|
+
* numbers (`#123`). A marker line that carries no strict reference — an empty
|
|
14
|
+
* `Depends-on:`, or a bare non-numeric `Depends-on: #abc` — is never a usable
|
|
15
|
+
* prerequisite, so it is reported as `malformed` for the grooming flag rather
|
|
16
|
+
* than guessed at, and the declaration it belongs to is ignored.
|
|
17
|
+
*
|
|
18
|
+
* Cross-repo references (a qualifying `owner/repo#n` or `repo#n` form) and
|
|
19
|
+
* graph cycles are later slices (#420/#421) and are deliberately out of scope
|
|
20
|
+
* here: only plain `#<n>` same-repo forms are resolved.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
export interface DependsOnDecl {
|
|
24
|
+
/** Referenced prerequisite issue numbers, deduplicated, first-seen order. */
|
|
25
|
+
readonly refs: readonly number[];
|
|
26
|
+
/** Marker-bearing lines that carried no strict `#<n>` reference (e.g.
|
|
27
|
+
* `Depends-on: #abc`). Bounded to what the body actually said, for the
|
|
28
|
+
* grooming flag's detail. */
|
|
29
|
+
readonly malformed: readonly string[];
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* How a line is spelled: `depends-on`/`depends on`, optional whitespace, then
|
|
34
|
+
* a `:` or `=` separator. The marker is matched case-insensitively; the value
|
|
35
|
+
* (everything after the separator) is what the strict-reference rules apply
|
|
36
|
+
* to. Anchored at the start of the (trimmed) line so a prose mention of
|
|
37
|
+
* "Depends-on" mid-sentence is not a declaration.
|
|
38
|
+
*/
|
|
39
|
+
const MARKER = /^depends[-\s]?on\s*[:=]\s*(.*)$/i;
|
|
40
|
+
|
|
41
|
+
/** A strict reference is a bare `#` followed by digits — no repo qualifier,
|
|
42
|
+
* no words. Everything else is not a prerequisite this slice resolves. */
|
|
43
|
+
const REF = /#(\d+)/g;
|
|
44
|
+
|
|
45
|
+
export function parseDependsOn(body: string): DependsOnDecl {
|
|
46
|
+
const refs: number[] = [];
|
|
47
|
+
const seen = new Set<number>();
|
|
48
|
+
const malformed: string[] = [];
|
|
49
|
+
for (const raw of body.split("\n")) {
|
|
50
|
+
const line = raw.trim();
|
|
51
|
+
if (line === "") continue;
|
|
52
|
+
const match = MARKER.exec(line);
|
|
53
|
+
if (match === null) continue;
|
|
54
|
+
const value = match[1] ?? "";
|
|
55
|
+
REF.lastIndex = 0;
|
|
56
|
+
const found = [...value.matchAll(REF)];
|
|
57
|
+
if (found.length === 0) {
|
|
58
|
+
// A marker line with no strict reference is a malformed declaration:
|
|
59
|
+
// never crash, never silently claim — the declaration is ignored and
|
|
60
|
+
// surfaced for grooming.
|
|
61
|
+
malformed.push(line);
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
for (const group of found) {
|
|
65
|
+
const n = Number(group[1]!);
|
|
66
|
+
if (Number.isSafeInteger(n) && !seen.has(n)) {
|
|
67
|
+
seen.add(n);
|
|
68
|
+
refs.push(n);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return { refs, malformed };
|
|
73
|
+
}
|
package/src/doctor.ts
CHANGED
|
@@ -25,20 +25,57 @@
|
|
|
25
25
|
* (the #399 boundary).
|
|
26
26
|
*/
|
|
27
27
|
|
|
28
|
-
import {
|
|
28
|
+
import {
|
|
29
|
+
accessSync,
|
|
30
|
+
constants,
|
|
31
|
+
existsSync,
|
|
32
|
+
readdirSync,
|
|
33
|
+
readFileSync,
|
|
34
|
+
statSync,
|
|
35
|
+
} from "node:fs";
|
|
29
36
|
import { spawnSync } from "node:child_process";
|
|
30
|
-
import {
|
|
37
|
+
import { homedir } from "node:os";
|
|
38
|
+
import { dirname, join } from "node:path";
|
|
31
39
|
import type { Stats } from "node:fs";
|
|
32
40
|
import { Database } from "bun:sqlite";
|
|
41
|
+
import { stringify } from "yaml";
|
|
33
42
|
|
|
34
|
-
import {
|
|
35
|
-
|
|
36
|
-
|
|
43
|
+
import {
|
|
44
|
+
configBackupDir,
|
|
45
|
+
configPath,
|
|
46
|
+
dbBackupDirFor,
|
|
47
|
+
findProject,
|
|
48
|
+
loadConfig,
|
|
49
|
+
resolveCaps,
|
|
50
|
+
stateDir,
|
|
51
|
+
} from "./config.ts";
|
|
52
|
+
import { ompSettingsOverlay, sessionRootDir } from "./omp-settings.ts";
|
|
53
|
+
import {
|
|
54
|
+
DEFAULT_HERDR_UNIT,
|
|
55
|
+
probeTelegramHealth,
|
|
56
|
+
resolveHerdrSessionWithBridge,
|
|
57
|
+
telegramStateDir,
|
|
58
|
+
} from "./fleet.ts";
|
|
59
|
+
import type { TelegramHealth } from "./status-render.ts";
|
|
60
|
+
import {
|
|
61
|
+
herdrConductorPluginConfigDir,
|
|
62
|
+
planHostRuntime,
|
|
63
|
+
STAGED_SERVICE_NAME,
|
|
64
|
+
SYSTEMD_UNIT_DIR,
|
|
65
|
+
tickCwdForProject,
|
|
66
|
+
totalConfiguredWorkers,
|
|
67
|
+
} from "./setup-host.ts";
|
|
37
68
|
import { checkTokenScopes, type ScopeCheck } from "./setup.ts";
|
|
38
|
-
import { dbPath, LIVE_STATES } from "./store.ts";
|
|
69
|
+
import { DB_SNAPSHOT_STEM, dbPath, LIVE_STATES } from "./store.ts";
|
|
39
70
|
import { telegramReportSend, type ReportSend } from "./reports.ts";
|
|
40
71
|
import { fetchRateLimit, GhError, gh } from "./tracker/github.ts";
|
|
41
72
|
import { repoSlugFor } from "./gitops.ts";
|
|
73
|
+
import {
|
|
74
|
+
DEFAULT_FLEET_AGENT_NAME,
|
|
75
|
+
parseHerdrAgents,
|
|
76
|
+
readTickConfig,
|
|
77
|
+
type HerdrAgentList,
|
|
78
|
+
} from "./orchestrator-tick.ts";
|
|
42
79
|
import type { ConductorConfig, ProjectConfig, RepoTarget, RunState } from "./types.ts";
|
|
43
80
|
|
|
44
81
|
/**
|
|
@@ -139,6 +176,14 @@ export interface DoctorDeps {
|
|
|
139
176
|
uidOf?: (name: string) => number | undefined;
|
|
140
177
|
/** `PRAGMA integrity_check` over the store, read-only. ok=true for "ok". */
|
|
141
178
|
dbIntegrity?: (path: string) => { ok: boolean; detail?: string };
|
|
179
|
+
/** Whether the configured db-snapshot directory exists and is writable, so a
|
|
180
|
+
* snapshot can land. Injected so a test never chmods the host (#399). */
|
|
181
|
+
snapshotDirState?: (dir: string) => { exists: boolean; writable: boolean };
|
|
182
|
+
/** Whether the run-session root — where the omp settings overlay is
|
|
183
|
+
* materialised at dispatch (#537) — can hold a new `run-<id>` directory. A
|
|
184
|
+
* missing `sessions/` parent is not a fault (dispatch's recursive `mkdir`
|
|
185
|
+
* creates it), so the writable question falls back to the state dir itself. */
|
|
186
|
+
sessionRootState?: (root: string) => { exists: boolean; writable: boolean };
|
|
142
187
|
/** The most recent completed runs for one project, newest first, limited. */
|
|
143
188
|
recentRuns?: (project: string, limit: number) => RunSpendRow[];
|
|
144
189
|
/** The bot health probe `status` uses (getMe etc.). */
|
|
@@ -148,6 +193,20 @@ export interface DoctorDeps {
|
|
|
148
193
|
telegramSend?: (project: ProjectConfig) => ReportSend;
|
|
149
194
|
/** The canonical units to compare the installed units against. */
|
|
150
195
|
canonicalUnits?: (project: ProjectConfig, cfg: ConductorConfig) => CanonicalUnits;
|
|
196
|
+
/** Whether herdr is installed on this host (a `herdr` on PATH). */
|
|
197
|
+
herdrInstalled?: () => boolean;
|
|
198
|
+
/** Live `herdr --session <s> agent list`, parsed through the tick's own
|
|
199
|
+
* parser (the same "one JSON line on stdout" contract recover.sh reads). */
|
|
200
|
+
herdrAgents?: (session: string) => HerdrAgentList;
|
|
201
|
+
/** The fleet session name, for the rename fix line. */
|
|
202
|
+
herdrSession?: () => string;
|
|
203
|
+
/** The live herdr config at `~/.config/herdr/config.toml`, or undefined. */
|
|
204
|
+
herdrConfig?: () => string | undefined;
|
|
205
|
+
/** The live herdr-conductor plugin `config.env`, or undefined. */
|
|
206
|
+
herdrEnv?: () => string | undefined;
|
|
207
|
+
/** The fleet agent name the tick config of one project names, or undefined
|
|
208
|
+
* when there is no (readable) tick — the expected live herdr pane identity. */
|
|
209
|
+
tickAgentName?: (project: ProjectConfig) => string | undefined;
|
|
151
210
|
/** Clock, so a run is deterministic in tests. */
|
|
152
211
|
now?: () => number;
|
|
153
212
|
/** The one opt-in side effect: send one self-identified Telegram probe. */
|
|
@@ -252,6 +311,55 @@ function defaultDbIntegrity(path: string): { ok: boolean; detail?: string } {
|
|
|
252
311
|
}
|
|
253
312
|
}
|
|
254
313
|
|
|
314
|
+
/**
|
|
315
|
+
* Whether the configured db-snapshot directory exists and is writable. No
|
|
316
|
+
* write is performed — the check is stat + access, keeping doctor read-only.
|
|
317
|
+
* A directory that does not exist reads `writable: false` and lets the probe
|
|
318
|
+
* report it as "no snapshot yet" rather than inventing one, while a read-only
|
|
319
|
+
* or non-directory path reads as a genuine unwritable failure.
|
|
320
|
+
*/
|
|
321
|
+
function defaultSnapshotDirState(dir: string): { exists: boolean; writable: boolean } {
|
|
322
|
+
try {
|
|
323
|
+
const st = statSync(dir);
|
|
324
|
+
if (!st.isDirectory()) return { exists: true, writable: false };
|
|
325
|
+
try {
|
|
326
|
+
accessSync(dir, constants.W_OK);
|
|
327
|
+
return { exists: true, writable: true };
|
|
328
|
+
} catch {
|
|
329
|
+
return { exists: true, writable: false };
|
|
330
|
+
}
|
|
331
|
+
} catch {
|
|
332
|
+
return { exists: false, writable: false };
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
/**
|
|
337
|
+
* Whether the run-session root can hold a new session directory. No write is
|
|
338
|
+
* performed — the same stat + access as {@link defaultSnapshotDirState}, which
|
|
339
|
+
* keeps `doctor` read-only. A missing `sessions/` parent is normal on a fresh
|
|
340
|
+
* host (dispatch's recursive `mkdir` creates it), so the writable question
|
|
341
|
+
* falls back to the state dir it would be created under; a state dir that
|
|
342
|
+
* cannot even be read is a genuine unwritable failure.
|
|
343
|
+
*/
|
|
344
|
+
function defaultSessionRootState(root: string): { exists: boolean; writable: boolean } {
|
|
345
|
+
const writable = (dir: string): boolean => {
|
|
346
|
+
try {
|
|
347
|
+
const st = statSync(dir);
|
|
348
|
+
if (!st.isDirectory()) return false;
|
|
349
|
+
accessSync(dir, constants.W_OK);
|
|
350
|
+
return true;
|
|
351
|
+
} catch {
|
|
352
|
+
return false;
|
|
353
|
+
}
|
|
354
|
+
};
|
|
355
|
+
try {
|
|
356
|
+
const st = statSync(root);
|
|
357
|
+
return { exists: true, writable: st.isDirectory() && writable(root) };
|
|
358
|
+
} catch {
|
|
359
|
+
return { exists: false, writable: writable(dirname(root)) };
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
|
|
255
363
|
/**
|
|
256
364
|
* The most recent completed runs for one project, newest first — read on the
|
|
257
365
|
* same read-only connection the integrity check opens, because the store's own
|
|
@@ -385,6 +493,57 @@ function dbProbe(probes: Probes): Finding {
|
|
|
385
493
|
);
|
|
386
494
|
}
|
|
387
495
|
|
|
496
|
+
/**
|
|
497
|
+
* The conductor.db snapshot store (#579): the configured dir is writable so a
|
|
498
|
+
* snapshot can land, and a snapshot at least as fresh as the store protects
|
|
499
|
+
* it. The store is the carrier of the verb ledger, the decision rows, the run
|
|
500
|
+
* rows and the material-event ledger; a store with no restorable copy has no
|
|
501
|
+
* recovery path, so absence is a warning and an unwritable configured dir a
|
|
502
|
+
* failure.
|
|
503
|
+
*/
|
|
504
|
+
function dbBackupProbe(probes: Probes, cfg: ConductorConfig | undefined): Finding {
|
|
505
|
+
const dir = dbBackupDirFor(cfg);
|
|
506
|
+
const state = probes.snapshotDirState(dir);
|
|
507
|
+
if (state.exists && !state.writable) {
|
|
508
|
+
return failFinding(
|
|
509
|
+
"db-backup",
|
|
510
|
+
`db backup dir ${dir} is not writable — a conductor.db snapshot cannot land there`,
|
|
511
|
+
`make ${dir} writable by the conductor account, or set dbBackupDir in config to a writable directory`,
|
|
512
|
+
);
|
|
513
|
+
}
|
|
514
|
+
const snapshots = (() => {
|
|
515
|
+
try {
|
|
516
|
+
return readdirSync(dir).filter((name) => name.startsWith(DB_SNAPSHOT_STEM));
|
|
517
|
+
} catch {
|
|
518
|
+
return [];
|
|
519
|
+
}
|
|
520
|
+
})();
|
|
521
|
+
const store = probes.stat(dbPath());
|
|
522
|
+
if (store === undefined) {
|
|
523
|
+
return snapshots.length === 0
|
|
524
|
+
? passFinding("db-backup", "no conductor.db yet — nothing to snapshot")
|
|
525
|
+
: passFinding("db-backup", `${snapshots.length} conductor.db snapshot(s) present`);
|
|
526
|
+
}
|
|
527
|
+
if (snapshots.length === 0) {
|
|
528
|
+
return warnFinding(
|
|
529
|
+
"db-backup",
|
|
530
|
+
"conductor.db has no snapshot — losing it loses the audit trail with no recovery path",
|
|
531
|
+
"take a conductor.db snapshot (the daemon's cadence files one; restore-db needs one to restore from)",
|
|
532
|
+
);
|
|
533
|
+
}
|
|
534
|
+
const newestMtime = snapshots
|
|
535
|
+
.map((name) => probes.stat(join(dir, name))?.mtimeMs ?? 0)
|
|
536
|
+
.reduce((max, m) => Math.max(max, m), 0);
|
|
537
|
+
if (store.mtimeMs > newestMtime + 60_000) {
|
|
538
|
+
return warnFinding(
|
|
539
|
+
"db-backup",
|
|
540
|
+
`conductor.db is newer than every snapshot (${snapshots.length} file(s)) — the newest writes were never snapshot`,
|
|
541
|
+
"take a fresh conductor.db snapshot so restore-db has the current state",
|
|
542
|
+
);
|
|
543
|
+
}
|
|
544
|
+
return passFinding("db-backup", `conductor.db snapshot is fresh (${snapshots.length} file(s))`);
|
|
545
|
+
}
|
|
546
|
+
|
|
388
547
|
/** Every repo the config names: each project's tracker plus routing targets,
|
|
389
548
|
* as GitHub slugs, deduplicated, in config order. */
|
|
390
549
|
function configuredRepos(cfg: ConductorConfig | undefined): string[] {
|
|
@@ -720,6 +879,42 @@ export function isKnownTimezone(tz: string): boolean {
|
|
|
720
879
|
}
|
|
721
880
|
}
|
|
722
881
|
|
|
882
|
+
/**
|
|
883
|
+
* The fleet-owned omp settings overlay (#537): reports the *effective* overlay
|
|
884
|
+
* every worker session of the project will load — the `ompSettings` map plus
|
|
885
|
+
* the retry keys derived from `modelFallbacks`, rendered exactly as the
|
|
886
|
+
* materialiser writes them, never the raw config snippet `setup` would echo —
|
|
887
|
+
* and turns red when that overlay cannot land. The file is written at dispatch
|
|
888
|
+
* under `<stateDir>/sessions/run-<id>/`, so doctor cannot prove the write
|
|
889
|
+
* without writing: it probes the root the materialisation would land in with
|
|
890
|
+
* the same stat + access the db-backup probe uses, keeping the check
|
|
891
|
+
* read-only (#287). The session root is a host-wide fact, probed once and
|
|
892
|
+
* shared across the per-project findings.
|
|
893
|
+
*/
|
|
894
|
+
function ompSettingsProbe(
|
|
895
|
+
project: ProjectConfig,
|
|
896
|
+
sessionRoot: { exists: boolean; writable: boolean },
|
|
897
|
+
): Finding {
|
|
898
|
+
const overlay = ompSettingsOverlay(project);
|
|
899
|
+
if (overlay === undefined) {
|
|
900
|
+
return passFinding(
|
|
901
|
+
"omp-settings",
|
|
902
|
+
`[${project.name}] no omp settings overlay configured — workers inherit the daemon account's global settings unchanged`,
|
|
903
|
+
);
|
|
904
|
+
}
|
|
905
|
+
if (!sessionRoot.writable) {
|
|
906
|
+
return failFinding(
|
|
907
|
+
"omp-settings",
|
|
908
|
+
`[${project.name}] the omp settings overlay cannot be materialised: ${sessionRootDir()} cannot hold a session directory, so every worker would dispatch without its staged settings`,
|
|
909
|
+
`make ${sessionRootDir()} writable by the conductor account`,
|
|
910
|
+
);
|
|
911
|
+
}
|
|
912
|
+
return passFinding(
|
|
913
|
+
"omp-settings",
|
|
914
|
+
`[${project.name}] worker sessions load this omp settings overlay:\n${stringify(overlay).trimEnd()}`,
|
|
915
|
+
);
|
|
916
|
+
}
|
|
917
|
+
|
|
723
918
|
/** The self-identified probe message `--probe-telegram` delivers through the
|
|
724
919
|
* same report transport a real report would take. */
|
|
725
920
|
function probeMessage(project: string, checkedAt: string): string {
|
|
@@ -790,7 +985,190 @@ function spendProbe(rows: RunSpendRow[], limit: number): Finding {
|
|
|
790
985
|
return passFinding("spend-telemetry", `spend observed on the last ${window.length} completed runs ($${total.toFixed(2)} total)`);
|
|
791
986
|
}
|
|
792
987
|
|
|
793
|
-
|
|
988
|
+
/** Live `herdr --session <session> agent list` through the project's own
|
|
989
|
+
* parser — the published "one JSON line on stdout" interface, never the
|
|
990
|
+
* socket. Same bound as the tick's own query so a hung herdr cannot hang
|
|
991
|
+
* doctor. */
|
|
992
|
+
function defaultHerdrAgents(session: string): HerdrAgentList {
|
|
993
|
+
try {
|
|
994
|
+
const run = spawnSync("herdr", ["--session", session, "agent", "list"], {
|
|
995
|
+
encoding: "utf8",
|
|
996
|
+
timeout: 3000,
|
|
997
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
998
|
+
});
|
|
999
|
+
if (run.error !== undefined) return { kind: "unavailable", problem: run.error.message };
|
|
1000
|
+
if (run.status !== 0) {
|
|
1001
|
+
return {
|
|
1002
|
+
kind: "unavailable",
|
|
1003
|
+
problem: `herdr agent list exited ${String(run.status)}: ${(run.stderr ?? "").trim().split("\n")[0] ?? ""}`,
|
|
1004
|
+
};
|
|
1005
|
+
}
|
|
1006
|
+
return parseHerdrAgents(run.stdout ?? "");
|
|
1007
|
+
} catch (err) {
|
|
1008
|
+
return { kind: "unavailable", problem: err instanceof Error ? err.message : String(err) };
|
|
1009
|
+
}
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
/** Live herdr config, defaulting to the fleet account's own path. */
|
|
1013
|
+
function defaultHerdrConfig(): string | undefined {
|
|
1014
|
+
try {
|
|
1015
|
+
return readFileSync(join(homedir(), ".config", "herdr", "config.toml"), "utf8");
|
|
1016
|
+
} catch {
|
|
1017
|
+
return undefined;
|
|
1018
|
+
}
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
/** Live herdr-conductor plugin `config.env`, same path setup provisions. */
|
|
1022
|
+
function defaultHerdrEnv(): string | undefined {
|
|
1023
|
+
try {
|
|
1024
|
+
return readFileSync(join(herdrConductorPluginConfigDir(homedir()), "config.env"), "utf8");
|
|
1025
|
+
} catch {
|
|
1026
|
+
return undefined;
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
/**
|
|
1031
|
+
* #541 check 1 — live herdr agent name equals the tick config's agentName.
|
|
1032
|
+
*
|
|
1033
|
+
* The paneOwnership decline the fleet actually hit names the exact fix (`herdr
|
|
1034
|
+
* agent rename <pane> <name>`); doctor checks the same premise the tick does
|
|
1035
|
+
* (is a pane registered under the configured agent name?) and reports drift
|
|
1036
|
+
* with the one-line remedy.
|
|
1037
|
+
*/
|
|
1038
|
+
function herdrAgentNameProbe(probes: Probes, p: ProjectConfig): Finding {
|
|
1039
|
+
if (!probes.herdrInstalled()) {
|
|
1040
|
+
return passFinding("herdr-agent-name", `[${p.name}] herdr not installed — nothing to check`);
|
|
1041
|
+
}
|
|
1042
|
+
const want = probes.tickAgentName(p);
|
|
1043
|
+
if (want === undefined) {
|
|
1044
|
+
return passFinding("herdr-agent-name", `[${p.name}] no readable tick config — nothing to check`);
|
|
1045
|
+
}
|
|
1046
|
+
const session = probes.herdrSession();
|
|
1047
|
+
const agents = probes.herdrAgents(session);
|
|
1048
|
+
if (agents.kind !== "ok") {
|
|
1049
|
+
return failFinding(
|
|
1050
|
+
"herdr-agent-name",
|
|
1051
|
+
`[${p.name}] could not read the live herdr agent list (${agents.problem}) — cannot verify the fleet agent name`,
|
|
1052
|
+
`restore the herdr session (check \`herdr status\`), then re-run doctor`,
|
|
1053
|
+
);
|
|
1054
|
+
}
|
|
1055
|
+
const holders = agents.agents.filter((a) => a.name === want);
|
|
1056
|
+
if (holders.length > 0) {
|
|
1057
|
+
return passFinding("herdr-agent-name", `[${p.name}] live pane(s) ${holders.map((a) => a.paneId).join(", ")} are agent "${want}"`);
|
|
1058
|
+
}
|
|
1059
|
+
// Drift: no pane carries the configured identity. Exactly one pane pinned to
|
|
1060
|
+
// the shared default is the safe candidate from the decline turnover — name
|
|
1061
|
+
// it when unambiguous, else give the generic command.
|
|
1062
|
+
const stale = agents.agents.filter((a) => a.name === DEFAULT_FLEET_AGENT_NAME);
|
|
1063
|
+
const pane = stale.length === 1 ? stale[0]!.paneId : "<pane>";
|
|
1064
|
+
return failFinding(
|
|
1065
|
+
"herdr-agent-name",
|
|
1066
|
+
`[${p.name}] no live pane is agent "${want}" — every tick will be declined until the fleet pane carries this name (the 2026-08-15 restamp class)`,
|
|
1067
|
+
`herdr --session ${session} agent rename ${pane} ${want}`,
|
|
1068
|
+
);
|
|
1069
|
+
}
|
|
1070
|
+
|
|
1071
|
+
/**
|
|
1072
|
+
* #541 check 2 — `[session] resume_agents_on_restore` on the live herdr config.
|
|
1073
|
+
*
|
|
1074
|
+
* herdr's own README: the default `true` leaves a restored omp pane with a
|
|
1075
|
+
* deferred resume plan and no live terminal, which is the configuration that
|
|
1076
|
+
* pages instead of recovering. Absent ~ the unrecoverable default; an explicit
|
|
1077
|
+
* `true` is a deliberate (desktop) choice and only warned.
|
|
1078
|
+
*/
|
|
1079
|
+
function herdrResumeProbe(probes: Probes): Finding {
|
|
1080
|
+
if (!probes.herdrInstalled()) {
|
|
1081
|
+
return passFinding("herdr-resume", "herdr not installed — nothing to check");
|
|
1082
|
+
}
|
|
1083
|
+
const live = probes.herdrConfig();
|
|
1084
|
+
if (live === undefined) {
|
|
1085
|
+
return failFinding(
|
|
1086
|
+
"herdr-resume",
|
|
1087
|
+
"no herdr config at " + join(homedir(), ".config", "herdr", "config.toml") + " — a restored omp pane gets no live terminal (herdr's own README)",
|
|
1088
|
+
"run \`omp-conductor setup host\` from the fleet account (writes [session] resume_agents_on_restore = false), or add it by hand",
|
|
1089
|
+
);
|
|
1090
|
+
}
|
|
1091
|
+
let parsed: { session?: { resume_agents_on_restore?: unknown } };
|
|
1092
|
+
try {
|
|
1093
|
+
parsed = Bun.TOML.parse(live) as { session?: { resume_agents_on_restore?: unknown } };
|
|
1094
|
+
} catch (err) {
|
|
1095
|
+
return failFinding(
|
|
1096
|
+
"herdr-resume",
|
|
1097
|
+
`the live herdr config does not parse (${err instanceof Error ? err.message : String(err)}) — herdr itself refuses it`,
|
|
1098
|
+
"run `herdr config check` to name the line, fix, then re-run doctor",
|
|
1099
|
+
);
|
|
1100
|
+
}
|
|
1101
|
+
const value = parsed.session?.resume_agents_on_restore;
|
|
1102
|
+
if (value === false) {
|
|
1103
|
+
return passFinding("herdr-resume", "resume_agents_on_restore is false — restored panes come up as shells, not deferred plans");
|
|
1104
|
+
}
|
|
1105
|
+
if (value === true) {
|
|
1106
|
+
return warnFinding(
|
|
1107
|
+
"herdr-resume",
|
|
1108
|
+
"resume_agents_on_restore is explicitly true — possibly a deliberate desktop config, but a headless fleet restores panes with no live terminal",
|
|
1109
|
+
"set it to false for a headless host ([session] resume_agents_on_restore = false), or run `omp-conductor setup host`",
|
|
1110
|
+
);
|
|
1111
|
+
}
|
|
1112
|
+
return failFinding(
|
|
1113
|
+
"herdr-resume",
|
|
1114
|
+
"resume_agents_on_restore is absent — herdr defaults it to true, the configuration its own README calls unrecoverable",
|
|
1115
|
+
"run `omp-conductor setup host` from the fleet account (writes the key), or add [session] resume_agents_on_restore = false by hand",
|
|
1116
|
+
);
|
|
1117
|
+
}
|
|
1118
|
+
|
|
1119
|
+
/**
|
|
1120
|
+
* #541 check 3 — herdr-conductor `config.env` FLEET_CWDS vs the configured
|
|
1121
|
+
* projects. Absent, a virgin host recovers against legacy `/root/fleet` paths
|
|
1122
|
+
* that do not exist: recovery still runs, only the page silently goes nowhere.
|
|
1123
|
+
*/
|
|
1124
|
+
function herdrEnvProbe(probes: Probes, cfg: ConductorConfig | undefined): Finding {
|
|
1125
|
+
if (!probes.herdrInstalled()) {
|
|
1126
|
+
return passFinding("herdr-config-env", "herdr not installed — nothing to check");
|
|
1127
|
+
}
|
|
1128
|
+
if (cfg === undefined) {
|
|
1129
|
+
return passFinding("herdr-config-env", "config unreadable — nothing to compare");
|
|
1130
|
+
}
|
|
1131
|
+
const live = probes.herdrEnv();
|
|
1132
|
+
if (live === undefined) {
|
|
1133
|
+
return failFinding(
|
|
1134
|
+
"herdr-config-env",
|
|
1135
|
+
"no herdr-conductor config.env — recovery falls back to legacy /root-based single-tenant paths this host does not use, so pages silently skip",
|
|
1136
|
+
"run `omp-conductor setup host` from the fleet account (writes config.env)",
|
|
1137
|
+
);
|
|
1138
|
+
}
|
|
1139
|
+
const expected = new Set(cfg.projects.map((p) => tickCwdForProject(p)));
|
|
1140
|
+
const actual = parseFleetCwds(live);
|
|
1141
|
+
const coherent =
|
|
1142
|
+
actual !== undefined && actual.size === expected.size && [...actual].every((cwd) => expected.has(cwd));
|
|
1143
|
+
if (coherent) {
|
|
1144
|
+
return passFinding("herdr-config-env", `config.env FLEET_CWDS matches the configured fleets (${[...expected].join(":")})`);
|
|
1145
|
+
}
|
|
1146
|
+
return failFinding(
|
|
1147
|
+
"herdr-config-env",
|
|
1148
|
+
`config.env FLEET_CWDS (${actual === undefined ? "unset" : [...actual].join(":")}) does not match the configured fleets (${[...expected].join(":")})`,
|
|
1149
|
+
"re-run `omp-conductor setup host`, which derives FLEET_CWDS from the configured projects",
|
|
1150
|
+
);
|
|
1151
|
+
}
|
|
1152
|
+
|
|
1153
|
+
/** FLEET_CWDS (colon- or space-separated, per recover.sh) or undefined. */
|
|
1154
|
+
function parseFleetCwds(text: string): Set<string> | undefined {
|
|
1155
|
+
const value = parseEnvKey(text, "FLEET_CWDS");
|
|
1156
|
+
if (value !== undefined) {
|
|
1157
|
+
return new Set(value.split(/[\s:]+/).filter((w) => w.length > 0));
|
|
1158
|
+
}
|
|
1159
|
+
// Legacy single-fleet form, the same fallback recover.sh applies.
|
|
1160
|
+
const legacy = parseEnvKey(text, "FLEET_CWD");
|
|
1161
|
+
return legacy === undefined ? undefined : new Set([legacy]);
|
|
1162
|
+
}
|
|
1163
|
+
|
|
1164
|
+
/** The first unquoted-or-quoted `KEY=value` on its own line, comments ignored. */
|
|
1165
|
+
function parseEnvKey(text: string, key: string): string | undefined {
|
|
1166
|
+
for (const line of text.split("\n")) {
|
|
1167
|
+
const match = new RegExp(`^[ \\t]*${key}[ \\t]*=[ \\t]*"?([^"\\n]*)"?[ \\t]*$`).exec(line);
|
|
1168
|
+
if (match !== null) return match[1];
|
|
1169
|
+
}
|
|
1170
|
+
return undefined;
|
|
1171
|
+
}
|
|
794
1172
|
|
|
795
1173
|
/**
|
|
796
1174
|
* Run every probe and assemble the stable report.
|
|
@@ -859,7 +1237,13 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
|
|
|
859
1237
|
// difference behind a constant count).
|
|
860
1238
|
findings.push(backupProbe(probes));
|
|
861
1239
|
findings.push(dbProbe(probes));
|
|
1240
|
+
findings.push(dbBackupProbe(probes, cfg));
|
|
862
1241
|
findings.push(await ghAuthProbe(probes, configuredRepos(cfg)));
|
|
1242
|
+
// The run-session root is one host-wide fact — where every run's session dir
|
|
1243
|
+
// (and the omp settings overlay inside it) lands — probed once and shared
|
|
1244
|
+
// across the per-project omp-settings findings.
|
|
1245
|
+
const sessionRoot = sessionRootDir();
|
|
1246
|
+
const sessionRootState = probes.sessionRootState(sessionRoot);
|
|
863
1247
|
// The collected run sample across the resolved project set, newest-first;
|
|
864
1248
|
// spend telemetry is a single host-wide finding, not one per project.
|
|
865
1249
|
const spendRows: RunSpendRow[] = [];
|
|
@@ -873,8 +1257,18 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
|
|
|
873
1257
|
findings.push(
|
|
874
1258
|
passFinding("labels", projectProblem === undefined ? "no project resolved — nothing to check" : `labels uncheckable: ${projectProblem}`),
|
|
875
1259
|
);
|
|
1260
|
+
findings.push(
|
|
1261
|
+
passFinding("herdr-agent-name", projectProblem === undefined ? "no project resolved — nothing to check" : `agent name uncheckable: ${projectProblem}`),
|
|
1262
|
+
);
|
|
1263
|
+
findings.push(
|
|
1264
|
+
passFinding("omp-settings", projectProblem === undefined ? "no project resolved — nothing to check" : `omp settings overlay uncheckable: ${projectProblem}`),
|
|
1265
|
+
);
|
|
876
1266
|
} else {
|
|
877
|
-
for (const p of projects)
|
|
1267
|
+
for (const p of projects) {
|
|
1268
|
+
findings.push(await labelProbe(probes, p));
|
|
1269
|
+
findings.push(herdrAgentNameProbe(probes, p));
|
|
1270
|
+
findings.push(ompSettingsProbe(p, sessionRootState));
|
|
1271
|
+
}
|
|
878
1272
|
}
|
|
879
1273
|
// The installed-unit check is host-global: one shared daemon. Any project
|
|
880
1274
|
// renders the same canonical units (the daemon/units carry no project), so
|
|
@@ -882,6 +1276,10 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
|
|
|
882
1276
|
findings.push(unitProbe(probes, projects[0], cfg));
|
|
883
1277
|
findings.push(recoveryProbe(probes));
|
|
884
1278
|
findings.push(ownershipProbe(probes));
|
|
1279
|
+
// #541 seam checks, host-global: the live herdr config and the plugin's
|
|
1280
|
+
// config.env are single files on the host, not per-project facts.
|
|
1281
|
+
findings.push(herdrResumeProbe(probes));
|
|
1282
|
+
findings.push(herdrEnvProbe(probes, cfg));
|
|
885
1283
|
if (projects.length === 0) {
|
|
886
1284
|
findings.push(timezoneProbe(undefined));
|
|
887
1285
|
findings.push(await telegramProbe(probes, undefined, checkedAt));
|
|
@@ -922,10 +1320,22 @@ export function defaultProbes(): Probes {
|
|
|
922
1320
|
stat: defaultStat,
|
|
923
1321
|
uidOf: defaultUidOf,
|
|
924
1322
|
dbIntegrity: defaultDbIntegrity,
|
|
1323
|
+
snapshotDirState: defaultSnapshotDirState,
|
|
1324
|
+
sessionRootState: defaultSessionRootState,
|
|
925
1325
|
recentRuns: defaultRecentRuns,
|
|
926
1326
|
telegramHealth: (projectName) => probeTelegramHealth(projectName),
|
|
927
1327
|
telegramSend: telegramReportSend,
|
|
928
1328
|
canonicalUnits: defaultCanonicalUnits,
|
|
1329
|
+
herdrInstalled: () => Bun.which("herdr") !== null,
|
|
1330
|
+
herdrAgents: defaultHerdrAgents,
|
|
1331
|
+
herdrSession: () => resolveHerdrSessionWithBridge(),
|
|
1332
|
+
herdrConfig: defaultHerdrConfig,
|
|
1333
|
+
herdrEnv: defaultHerdrEnv,
|
|
1334
|
+
tickAgentName: (p) => {
|
|
1335
|
+
const tick = readTickConfig(tickCwdForProject(p));
|
|
1336
|
+
if (tick.kind !== "ok") return undefined;
|
|
1337
|
+
return tick.config.agentName ?? DEFAULT_FLEET_AGENT_NAME;
|
|
1338
|
+
},
|
|
929
1339
|
now: Date.now,
|
|
930
1340
|
probeTelegram: false,
|
|
931
1341
|
};
|