omp-conductor 0.15.13 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +72 -2
- package/package.json +2 -1
- package/schema/config.schema.json +7 -0
- package/src/admission.ts +849 -0
- package/src/ask.ts +47 -0
- package/src/backups.ts +19 -7
- package/src/board.ts +1 -2
- package/src/briefs/orchestrator.md +62 -4
- package/src/cli.ts +26 -0
- package/src/commands/context.ts +3 -0
- package/src/commands/decision.ts +10 -1
- package/src/commands/doctor.ts +2 -0
- package/src/commands/message.ts +8 -1
- package/src/commands/restart.ts +15 -3
- package/src/commands/restore-db.ts +146 -0
- package/src/commands/stop.ts +24 -15
- package/src/commands/tail.ts +204 -44
- package/src/commands/unfreeze.ts +56 -0
- package/src/commands/watch.ts +77 -0
- package/src/config-schema.ts +13 -0
- package/src/config.ts +54 -0
- package/src/daemon.ts +255 -530
- package/src/dashboard/server.ts +2 -1
- package/src/decisions.ts +32 -7
- package/src/depends-on.ts +122 -0
- package/src/doctor.ts +297 -5
- package/src/escalate.ts +191 -19
- package/src/failure-class.ts +47 -0
- package/src/fleet.ts +168 -452
- package/src/gitops.ts +86 -1
- package/src/log.ts +40 -0
- package/src/model-fallback.ts +3 -2
- package/src/omp-settings.ts +114 -0
- package/src/omp.ts +39 -0
- package/src/orchestrator-tick.ts +7 -1
- package/src/reports.ts +124 -12
- package/src/session-host.ts +6 -0
- package/src/setup-wizard.ts +36 -0
- package/src/setup.ts +58 -1
- package/src/status-render.ts +445 -0
- package/src/stop-provenance.ts +53 -0
- package/src/store.ts +352 -11
- package/src/transcript.ts +1 -1
- package/src/types.ts +187 -4
- package/src/unblock.ts +1 -1
- package/src/upgrade-verify.ts +1 -1
- package/src/upgrade.ts +1 -2
- package/src/verbs/server.ts +25 -0
- package/src/worker.ts +358 -10
- package/src/worktree.ts +13 -1
package/src/dashboard/server.ts
CHANGED
|
@@ -28,7 +28,8 @@ import { chmodSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync }
|
|
|
28
28
|
import { join } from "node:path";
|
|
29
29
|
import { loadConfig, stateDir } from "../config.ts";
|
|
30
30
|
import { healthCheck, livingDaemon, type DaemonRecord } from "../lifecycle.ts";
|
|
31
|
-
import { classifyDaemonProjectHealth, fleetLayers
|
|
31
|
+
import { classifyDaemonProjectHealth, fleetLayers } from "../fleet.ts";
|
|
32
|
+
import type { FleetLayers } from "../status-render.ts";
|
|
32
33
|
import { statusSnapshot, type StatusSnapshot } from "../daemon.ts";
|
|
33
34
|
import { boardJson, boardSnapshotOnce } from "../board.ts";
|
|
34
35
|
import { dbPath, openStore } from "../store.ts";
|
package/src/decisions.ts
CHANGED
|
@@ -235,15 +235,40 @@ function age(since: number, now: number): string {
|
|
|
235
235
|
* being a memory question. A row whose condition has come true is flagged, not
|
|
236
236
|
* merely listed — that is the difference between a parked question and one to
|
|
237
237
|
* act on now.
|
|
238
|
+
*
|
|
239
|
+
* The two kinds render under their own headings (#459). Only rows a human must
|
|
240
|
+
* answer — `kind === "question"`, whatever condition they carry — appear under
|
|
241
|
+
* the operator heading; a watch the orchestrator set for itself goes under the
|
|
242
|
+
* watch heading with no instruction to resolve it, so a fleet at rest behind
|
|
243
|
+
* GitHub's checks is never mistaken for a fleet that is waiting on its
|
|
244
|
+
* operator. A met watch still surfaces to the orchestrator with its note and
|
|
245
|
+
* the same `[CONDITION MET]` flag a met question gets.
|
|
238
246
|
*/
|
|
239
247
|
export function formatDecisionDigest(open: readonly DecisionRecord[], now = Date.now()): string {
|
|
240
|
-
|
|
241
|
-
const
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
248
|
+
const questions = open.filter((d) => d.kind !== "watch");
|
|
249
|
+
const watches = open.filter((d) => d.kind === "watch");
|
|
250
|
+
const lines: string[] = [];
|
|
251
|
+
if (questions.length > 0) {
|
|
252
|
+
lines.push(
|
|
253
|
+
`Open operator decisions (${questions.length}) — resolve or withdraw each with omp-conductor decision resolve|withdraw <id>:`,
|
|
254
|
+
);
|
|
255
|
+
for (const d of questions) {
|
|
256
|
+
lines.push(decisionLine(d, now));
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
if (watches.length > 0) {
|
|
260
|
+
lines.push(
|
|
261
|
+
`Watches (${watches.length}) — conditions the orchestrator set for itself; no operator action needed:`,
|
|
262
|
+
);
|
|
263
|
+
for (const d of watches) {
|
|
264
|
+
lines.push(decisionLine(d, now));
|
|
265
|
+
}
|
|
247
266
|
}
|
|
248
267
|
return lines.join("\n");
|
|
249
268
|
}
|
|
269
|
+
|
|
270
|
+
/** One digest row: id, age, what it blocks, the met flag, and the note. */
|
|
271
|
+
function decisionLine(d: DecisionRecord, now: number): string {
|
|
272
|
+
const flag = d.conditionMetAt === undefined ? "" : " [CONDITION MET — act on this now]";
|
|
273
|
+
return `- ${d.id} (${age(d.askedAt, now)}, blocks ${d.blocks ?? "nothing"})${flag}: ${d.question}`;
|
|
274
|
+
}
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `Depends-on:` declaration parser (epic #321, slices #419/#420).
|
|
3
|
+
*
|
|
4
|
+
* A candidate issue can declare prerequisites it must not be dispatched
|
|
5
|
+
* before: `Depends-on: #123` (a same-repo issue) or `Depends-on:
|
|
6
|
+
* owner/repo#123` (an issue in a routed repository), repeatable one or more
|
|
7
|
+
* per line. Admission reads every referenced issue's live tracker state and
|
|
8
|
+
* holds the candidate while any prerequisite is open, so a worker is never
|
|
9
|
+
* sent at work whose base is not yet merged.
|
|
10
|
+
*
|
|
11
|
+
* The parser is deliberately small and strict, the two halves of the contract:
|
|
12
|
+
* the *marker* is matched case-insensitively (so `DEPENDS-ON`, `Depends-On`,
|
|
13
|
+
* `depends-on` all declare), while the *references* are strict — a bare issue
|
|
14
|
+
* number (`#123`) or a qualifying `owner/repo#123` with exactly two path
|
|
15
|
+
* segments glued to the number. A marker line that carries no strict
|
|
16
|
+
* reference — an empty `Depends-on:`, a bare non-numeric `Depends-on: #abc`,
|
|
17
|
+
* or a slashed-but-unqualified form that is not `owner/repo#n` — is never a
|
|
18
|
+
* usable prerequisite, so it is reported as `malformed` for the grooming flag
|
|
19
|
+
* rather than guessed at, and the declaration it belongs to is ignored.
|
|
20
|
+
*
|
|
21
|
+
* Graph cycles are a later slice (#421) and stay out of scope here.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
export interface DependsOnDecl {
|
|
25
|
+
/** Referenced same-repo prerequisite issue numbers, deduplicated,
|
|
26
|
+
* first-seen order. */
|
|
27
|
+
readonly refs: readonly number[];
|
|
28
|
+
/** Referenced cross-repo prerequisites (`owner/repo#n`), deduplicated,
|
|
29
|
+
* first-seen order. Admission resolves these against the fleet's routed
|
|
30
|
+
* repositories, never against the candidate's own repo (#420). */
|
|
31
|
+
readonly crossRefs: readonly DependsOnCrossRef[];
|
|
32
|
+
/** Marker-bearing lines that carried no strict reference (e.g.
|
|
33
|
+
* `Depends-on: #abc`). Bounded to what the body actually said, for the
|
|
34
|
+
* grooming flag's detail. */
|
|
35
|
+
readonly malformed: readonly string[];
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* A strict cross-repo reference. `repo` is the canonical `owner/repo` spelling
|
|
40
|
+
* from the body, untouched, so admission can match it by identity against the
|
|
41
|
+
* fleet's routed repositories rather than by guess.
|
|
42
|
+
*/
|
|
43
|
+
export interface DependsOnCrossRef {
|
|
44
|
+
readonly repo: string;
|
|
45
|
+
readonly issue: number;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* How a line is spelled: `depends-on`/`depends on`, optional whitespace, then
|
|
50
|
+
* a `:` or `=` separator. The marker is matched case-insensitively; the value
|
|
51
|
+
* (everything after the separator) is what the strict-reference rules apply
|
|
52
|
+
* to. Anchored at the start of the (trimmed) line so a prose mention of
|
|
53
|
+
* "Depends-on" mid-sentence is not a declaration.
|
|
54
|
+
*/
|
|
55
|
+
const MARKER = /^depends[-\s]?on\s*[:=]\s*(.*)$/i;
|
|
56
|
+
|
|
57
|
+
/** A strict same-repo reference is a bare `#` followed by digits — no repo
|
|
58
|
+
* qualifier, no words. The negative lookbehind means a `#<n>` glued to a
|
|
59
|
+
* word (e.g. `web#7` or `issue#5`) is not a bare reference: without the
|
|
60
|
+
* `owner/` segment it is neither a valid cross-repo ref nor a valid
|
|
61
|
+
* same-repo one, so it is flagged malformed rather than silently resolved in
|
|
62
|
+
* the candidate's own repo. */
|
|
63
|
+
const REF = /(?<![A-Za-z0-9._/-])#(\d+)/g;
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* A strict cross-repo reference is `owner/repo#<n>` — exactly two path
|
|
67
|
+
* segments (each a GitHub owner/repository charset run) glued to a bare issue
|
|
68
|
+
* number, no whitespace. Scanned before {@link REF} and blanked out of the
|
|
69
|
+
* value, so the trailing `#<n>` is never re-read as a *same-repo* reference:
|
|
70
|
+
* the whole point of the qualified form is that it names a different
|
|
71
|
+
* repository, and resolving it in the candidate's own repo is the #420 fake.
|
|
72
|
+
*/
|
|
73
|
+
const CROSS_REF = /([A-Za-z0-9][A-Za-z0-9._-]*)\/([A-Za-z0-9][A-Za-z0-9._-]*)#(\d+)/g;
|
|
74
|
+
|
|
75
|
+
export function parseDependsOn(body: string): DependsOnDecl {
|
|
76
|
+
const refs: number[] = [];
|
|
77
|
+
const seen = new Set<number>();
|
|
78
|
+
const crossRefs: DependsOnCrossRef[] = [];
|
|
79
|
+
const seenCross = new Set<string>();
|
|
80
|
+
const malformed: string[] = [];
|
|
81
|
+
for (const raw of body.split("\n")) {
|
|
82
|
+
const line = raw.trim();
|
|
83
|
+
if (line === "") continue;
|
|
84
|
+
const match = MARKER.exec(line);
|
|
85
|
+
if (match === null) continue;
|
|
86
|
+
const value = match[1] ?? "";
|
|
87
|
+
|
|
88
|
+
// Cross-repo references first, blanked out of the value so their `#<n>`
|
|
89
|
+
// cannot also be collected as a same-repo reference below.
|
|
90
|
+
let lineHadRef = false;
|
|
91
|
+
const masked = value.replace(CROSS_REF, (whole, owner, name, number) => {
|
|
92
|
+
const repo = `${owner}/${name}`;
|
|
93
|
+
const n = Number(number);
|
|
94
|
+
if (Number.isSafeInteger(n) && !seenCross.has(`${repo}#${n}`)) {
|
|
95
|
+
seenCross.add(`${repo}#${n}`);
|
|
96
|
+
crossRefs.push({ repo, issue: n });
|
|
97
|
+
}
|
|
98
|
+
lineHadRef = true;
|
|
99
|
+
return " ".repeat(whole.length);
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
REF.lastIndex = 0;
|
|
103
|
+
const found = [...masked.matchAll(REF)];
|
|
104
|
+
for (const group of found) {
|
|
105
|
+
const n = Number(group[1]!);
|
|
106
|
+
if (Number.isSafeInteger(n) && !seen.has(n)) {
|
|
107
|
+
seen.add(n);
|
|
108
|
+
refs.push(n);
|
|
109
|
+
}
|
|
110
|
+
lineHadRef = true;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
if (!lineHadRef) {
|
|
114
|
+
// A marker line with no strict reference — same-repo or cross-repo — is
|
|
115
|
+
// a malformed declaration: never crash, never silently claim — the
|
|
116
|
+
// declaration is ignored and surfaced for grooming.
|
|
117
|
+
malformed.push(line);
|
|
118
|
+
continue;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
return { refs, crossRefs, malformed };
|
|
122
|
+
}
|
package/src/doctor.ts
CHANGED
|
@@ -25,21 +25,46 @@
|
|
|
25
25
|
* (the #399 boundary).
|
|
26
26
|
*/
|
|
27
27
|
|
|
28
|
-
import {
|
|
28
|
+
import {
|
|
29
|
+
accessSync,
|
|
30
|
+
constants,
|
|
31
|
+
existsSync,
|
|
32
|
+
readdirSync,
|
|
33
|
+
readFileSync,
|
|
34
|
+
statSync,
|
|
35
|
+
} from "node:fs";
|
|
29
36
|
import { spawnSync } from "node:child_process";
|
|
30
37
|
import { homedir } from "node:os";
|
|
31
|
-
import { join } from "node:path";
|
|
38
|
+
import { dirname, join, sep } from "node:path";
|
|
32
39
|
import type { Stats } from "node:fs";
|
|
33
40
|
import { Database } from "bun:sqlite";
|
|
41
|
+
import { stringify } from "yaml";
|
|
34
42
|
|
|
35
|
-
import {
|
|
43
|
+
import {
|
|
44
|
+
configBackupDir,
|
|
45
|
+
configPath,
|
|
46
|
+
dbBackupDirFor,
|
|
47
|
+
findProject,
|
|
48
|
+
loadConfig,
|
|
49
|
+
resolveCaps,
|
|
50
|
+
stateDir,
|
|
51
|
+
} from "./config.ts";
|
|
52
|
+
import { ompSettingsOverlay, sessionRootDir } from "./omp-settings.ts";
|
|
36
53
|
import {
|
|
37
54
|
DEFAULT_HERDR_UNIT,
|
|
38
55
|
probeTelegramHealth,
|
|
39
56
|
resolveHerdrSessionWithBridge,
|
|
57
|
+
sessionsRoot,
|
|
58
|
+
sessionDirForCwd,
|
|
40
59
|
telegramStateDir,
|
|
41
|
-
type TelegramHealth,
|
|
42
60
|
} from "./fleet.ts";
|
|
61
|
+
import type { TelegramHealth } from "./status-render.ts";
|
|
62
|
+
import {
|
|
63
|
+
claimForProject,
|
|
64
|
+
claimedTelegramTopics,
|
|
65
|
+
telegramTopicsTidy,
|
|
66
|
+
type ClaimedTopic,
|
|
67
|
+
} from "./escalate.ts";
|
|
43
68
|
import {
|
|
44
69
|
herdrConductorPluginConfigDir,
|
|
45
70
|
planHostRuntime,
|
|
@@ -49,7 +74,7 @@ import {
|
|
|
49
74
|
totalConfiguredWorkers,
|
|
50
75
|
} from "./setup-host.ts";
|
|
51
76
|
import { checkTokenScopes, type ScopeCheck } from "./setup.ts";
|
|
52
|
-
import { dbPath, LIVE_STATES } from "./store.ts";
|
|
77
|
+
import { DB_SNAPSHOT_STEM, dbPath, LIVE_STATES } from "./store.ts";
|
|
53
78
|
import { telegramReportSend, type ReportSend } from "./reports.ts";
|
|
54
79
|
import { fetchRateLimit, GhError, gh } from "./tracker/github.ts";
|
|
55
80
|
import { repoSlugFor } from "./gitops.ts";
|
|
@@ -159,6 +184,14 @@ export interface DoctorDeps {
|
|
|
159
184
|
uidOf?: (name: string) => number | undefined;
|
|
160
185
|
/** `PRAGMA integrity_check` over the store, read-only. ok=true for "ok". */
|
|
161
186
|
dbIntegrity?: (path: string) => { ok: boolean; detail?: string };
|
|
187
|
+
/** Whether the configured db-snapshot directory exists and is writable, so a
|
|
188
|
+
* snapshot can land. Injected so a test never chmods the host (#399). */
|
|
189
|
+
snapshotDirState?: (dir: string) => { exists: boolean; writable: boolean };
|
|
190
|
+
/** Whether the run-session root — where the omp settings overlay is
|
|
191
|
+
* materialised at dispatch (#537) — can hold a new `run-<id>` directory. A
|
|
192
|
+
* missing `sessions/` parent is not a fault (dispatch's recursive `mkdir`
|
|
193
|
+
* creates it), so the writable question falls back to the state dir itself. */
|
|
194
|
+
sessionRootState?: (root: string) => { exists: boolean; writable: boolean };
|
|
162
195
|
/** The most recent completed runs for one project, newest first, limited. */
|
|
163
196
|
recentRuns?: (project: string, limit: number) => RunSpendRow[];
|
|
164
197
|
/** The bot health probe `status` uses (getMe etc.). */
|
|
@@ -179,6 +212,10 @@ export interface DoctorDeps {
|
|
|
179
212
|
herdrConfig?: () => string | undefined;
|
|
180
213
|
/** The live herdr-conductor plugin `config.env`, or undefined. */
|
|
181
214
|
herdrEnv?: () => string | undefined;
|
|
215
|
+
/** Live omp-telegram topic claims (threads.json), for the #600 probes. */
|
|
216
|
+
claimedTopics?: () => ClaimedTopic[];
|
|
217
|
+
/** Whether the bridge's access.json runs per-session topic tidy on. */
|
|
218
|
+
topicsTidy?: () => boolean;
|
|
182
219
|
/** The fleet agent name the tick config of one project names, or undefined
|
|
183
220
|
* when there is no (readable) tick — the expected live herdr pane identity. */
|
|
184
221
|
tickAgentName?: (project: ProjectConfig) => string | undefined;
|
|
@@ -286,6 +323,55 @@ function defaultDbIntegrity(path: string): { ok: boolean; detail?: string } {
|
|
|
286
323
|
}
|
|
287
324
|
}
|
|
288
325
|
|
|
326
|
+
/**
|
|
327
|
+
* Whether the configured db-snapshot directory exists and is writable. No
|
|
328
|
+
* write is performed — the check is stat + access, keeping doctor read-only.
|
|
329
|
+
* A directory that does not exist reads `writable: false` and lets the probe
|
|
330
|
+
* report it as "no snapshot yet" rather than inventing one, while a read-only
|
|
331
|
+
* or non-directory path reads as a genuine unwritable failure.
|
|
332
|
+
*/
|
|
333
|
+
function defaultSnapshotDirState(dir: string): { exists: boolean; writable: boolean } {
|
|
334
|
+
try {
|
|
335
|
+
const st = statSync(dir);
|
|
336
|
+
if (!st.isDirectory()) return { exists: true, writable: false };
|
|
337
|
+
try {
|
|
338
|
+
accessSync(dir, constants.W_OK);
|
|
339
|
+
return { exists: true, writable: true };
|
|
340
|
+
} catch {
|
|
341
|
+
return { exists: true, writable: false };
|
|
342
|
+
}
|
|
343
|
+
} catch {
|
|
344
|
+
return { exists: false, writable: false };
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
/**
|
|
349
|
+
* Whether the run-session root can hold a new session directory. No write is
|
|
350
|
+
* performed — the same stat + access as {@link defaultSnapshotDirState}, which
|
|
351
|
+
* keeps `doctor` read-only. A missing `sessions/` parent is normal on a fresh
|
|
352
|
+
* host (dispatch's recursive `mkdir` creates it), so the writable question
|
|
353
|
+
* falls back to the state dir it would be created under; a state dir that
|
|
354
|
+
* cannot even be read is a genuine unwritable failure.
|
|
355
|
+
*/
|
|
356
|
+
function defaultSessionRootState(root: string): { exists: boolean; writable: boolean } {
|
|
357
|
+
const writable = (dir: string): boolean => {
|
|
358
|
+
try {
|
|
359
|
+
const st = statSync(dir);
|
|
360
|
+
if (!st.isDirectory()) return false;
|
|
361
|
+
accessSync(dir, constants.W_OK);
|
|
362
|
+
return true;
|
|
363
|
+
} catch {
|
|
364
|
+
return false;
|
|
365
|
+
}
|
|
366
|
+
};
|
|
367
|
+
try {
|
|
368
|
+
const st = statSync(root);
|
|
369
|
+
return { exists: true, writable: st.isDirectory() && writable(root) };
|
|
370
|
+
} catch {
|
|
371
|
+
return { exists: false, writable: writable(dirname(root)) };
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
|
|
289
375
|
/**
|
|
290
376
|
* The most recent completed runs for one project, newest first — read on the
|
|
291
377
|
* same read-only connection the integrity check opens, because the store's own
|
|
@@ -419,6 +505,57 @@ function dbProbe(probes: Probes): Finding {
|
|
|
419
505
|
);
|
|
420
506
|
}
|
|
421
507
|
|
|
508
|
+
/**
|
|
509
|
+
* The conductor.db snapshot store (#579): the configured dir is writable so a
|
|
510
|
+
* snapshot can land, and a snapshot at least as fresh as the store protects
|
|
511
|
+
* it. The store is the carrier of the verb ledger, the decision rows, the run
|
|
512
|
+
* rows and the material-event ledger; a store with no restorable copy has no
|
|
513
|
+
* recovery path, so absence is a warning and an unwritable configured dir a
|
|
514
|
+
* failure.
|
|
515
|
+
*/
|
|
516
|
+
function dbBackupProbe(probes: Probes, cfg: ConductorConfig | undefined): Finding {
|
|
517
|
+
const dir = dbBackupDirFor(cfg);
|
|
518
|
+
const state = probes.snapshotDirState(dir);
|
|
519
|
+
if (state.exists && !state.writable) {
|
|
520
|
+
return failFinding(
|
|
521
|
+
"db-backup",
|
|
522
|
+
`db backup dir ${dir} is not writable — a conductor.db snapshot cannot land there`,
|
|
523
|
+
`make ${dir} writable by the conductor account, or set dbBackupDir in config to a writable directory`,
|
|
524
|
+
);
|
|
525
|
+
}
|
|
526
|
+
const snapshots = (() => {
|
|
527
|
+
try {
|
|
528
|
+
return readdirSync(dir).filter((name) => name.startsWith(DB_SNAPSHOT_STEM));
|
|
529
|
+
} catch {
|
|
530
|
+
return [];
|
|
531
|
+
}
|
|
532
|
+
})();
|
|
533
|
+
const store = probes.stat(dbPath());
|
|
534
|
+
if (store === undefined) {
|
|
535
|
+
return snapshots.length === 0
|
|
536
|
+
? passFinding("db-backup", "no conductor.db yet — nothing to snapshot")
|
|
537
|
+
: passFinding("db-backup", `${snapshots.length} conductor.db snapshot(s) present`);
|
|
538
|
+
}
|
|
539
|
+
if (snapshots.length === 0) {
|
|
540
|
+
return warnFinding(
|
|
541
|
+
"db-backup",
|
|
542
|
+
"conductor.db has no snapshot — losing it loses the audit trail with no recovery path",
|
|
543
|
+
"take a conductor.db snapshot (the daemon's cadence files one; restore-db needs one to restore from)",
|
|
544
|
+
);
|
|
545
|
+
}
|
|
546
|
+
const newestMtime = snapshots
|
|
547
|
+
.map((name) => probes.stat(join(dir, name))?.mtimeMs ?? 0)
|
|
548
|
+
.reduce((max, m) => Math.max(max, m), 0);
|
|
549
|
+
if (store.mtimeMs > newestMtime + 60_000) {
|
|
550
|
+
return warnFinding(
|
|
551
|
+
"db-backup",
|
|
552
|
+
`conductor.db is newer than every snapshot (${snapshots.length} file(s)) — the newest writes were never snapshot`,
|
|
553
|
+
"take a fresh conductor.db snapshot so restore-db has the current state",
|
|
554
|
+
);
|
|
555
|
+
}
|
|
556
|
+
return passFinding("db-backup", `conductor.db snapshot is fresh (${snapshots.length} file(s))`);
|
|
557
|
+
}
|
|
558
|
+
|
|
422
559
|
/** Every repo the config names: each project's tracker plus routing targets,
|
|
423
560
|
* as GitHub slugs, deduplicated, in config order. */
|
|
424
561
|
function configuredRepos(cfg: ConductorConfig | undefined): string[] {
|
|
@@ -754,6 +891,42 @@ export function isKnownTimezone(tz: string): boolean {
|
|
|
754
891
|
}
|
|
755
892
|
}
|
|
756
893
|
|
|
894
|
+
/**
|
|
895
|
+
* The fleet-owned omp settings overlay (#537): reports the *effective* overlay
|
|
896
|
+
* every worker session of the project will load — the `ompSettings` map plus
|
|
897
|
+
* the retry keys derived from `modelFallbacks`, rendered exactly as the
|
|
898
|
+
* materialiser writes them, never the raw config snippet `setup` would echo —
|
|
899
|
+
* and turns red when that overlay cannot land. The file is written at dispatch
|
|
900
|
+
* under `<stateDir>/sessions/run-<id>/`, so doctor cannot prove the write
|
|
901
|
+
* without writing: it probes the root the materialisation would land in with
|
|
902
|
+
* the same stat + access the db-backup probe uses, keeping the check
|
|
903
|
+
* read-only (#287). The session root is a host-wide fact, probed once and
|
|
904
|
+
* shared across the per-project findings.
|
|
905
|
+
*/
|
|
906
|
+
function ompSettingsProbe(
|
|
907
|
+
project: ProjectConfig,
|
|
908
|
+
sessionRoot: { exists: boolean; writable: boolean },
|
|
909
|
+
): Finding {
|
|
910
|
+
const overlay = ompSettingsOverlay(project);
|
|
911
|
+
if (overlay === undefined) {
|
|
912
|
+
return passFinding(
|
|
913
|
+
"omp-settings",
|
|
914
|
+
`[${project.name}] no omp settings overlay configured — workers inherit the daemon account's global settings unchanged`,
|
|
915
|
+
);
|
|
916
|
+
}
|
|
917
|
+
if (!sessionRoot.writable) {
|
|
918
|
+
return failFinding(
|
|
919
|
+
"omp-settings",
|
|
920
|
+
`[${project.name}] the omp settings overlay cannot be materialised: ${sessionRootDir()} cannot hold a session directory, so every worker would dispatch without its staged settings`,
|
|
921
|
+
`make ${sessionRootDir()} writable by the conductor account`,
|
|
922
|
+
);
|
|
923
|
+
}
|
|
924
|
+
return passFinding(
|
|
925
|
+
"omp-settings",
|
|
926
|
+
`[${project.name}] worker sessions load this omp settings overlay:\n${stringify(overlay).trimEnd()}`,
|
|
927
|
+
);
|
|
928
|
+
}
|
|
929
|
+
|
|
757
930
|
/** The self-identified probe message `--probe-telegram` delivers through the
|
|
758
931
|
* same report transport a real report would take. */
|
|
759
932
|
function probeMessage(project: string, checkedAt: string): string {
|
|
@@ -907,6 +1080,103 @@ function herdrAgentNameProbe(probes: Probes, p: ProjectConfig): Finding {
|
|
|
907
1080
|
);
|
|
908
1081
|
}
|
|
909
1082
|
|
|
1083
|
+
/**
|
|
1084
|
+
* #600 check — the orchestrator's claimed session file vs the directory arm
|
|
1085
|
+
* scans.
|
|
1086
|
+
*
|
|
1087
|
+
* A pane resumed from a session created elsewhere keeps the original transcript
|
|
1088
|
+
* path (herdr pins the pane to it), so the claimed file can live outside the
|
|
1089
|
+
* session directory the tick cwd implies. arm follows the claim, so the
|
|
1090
|
+
* mismatch alone is handled — but it is exactly the shape of host that once
|
|
1091
|
+
* made arming silently impossible, and an operator reading transcript paths
|
|
1092
|
+
* should not have to rediscover it. A claimed file *outside the session tree*
|
|
1093
|
+
* is worse: arm refuses to arm there (the reply could never be seen), so
|
|
1094
|
+
* doctor fails rather than letting the fleet sit disarmed.
|
|
1095
|
+
*/
|
|
1096
|
+
function armSessionDirProbe(probes: Probes, p: ProjectConfig): Finding {
|
|
1097
|
+
const claims = probes.claimedTopics();
|
|
1098
|
+
if (claims.length === 0) {
|
|
1099
|
+
return passFinding(
|
|
1100
|
+
"arm-session-dir",
|
|
1101
|
+
`[${p.name}] no live omp-telegram claim — arm scans the tick-cwd session directory`,
|
|
1102
|
+
);
|
|
1103
|
+
}
|
|
1104
|
+
const claim = claimForProject(claims, p.name);
|
|
1105
|
+
if (claim === undefined || claim.sessionFile === undefined) {
|
|
1106
|
+
return passFinding(
|
|
1107
|
+
"arm-session-dir",
|
|
1108
|
+
`[${p.name}] the live claim names no session file — arm scans the tick-cwd session directory`,
|
|
1109
|
+
);
|
|
1110
|
+
}
|
|
1111
|
+
const root = sessionsRoot();
|
|
1112
|
+
const claimDir = dirname(claim.sessionFile);
|
|
1113
|
+
const scanDir = sessionDirForCwd(tickCwdForProject(p));
|
|
1114
|
+
if (claimDir !== root && !claimDir.startsWith(join(root, sep))) {
|
|
1115
|
+
return failFinding(
|
|
1116
|
+
"arm-session-dir",
|
|
1117
|
+
`[${p.name}] the orchestrator's claimed session file ${claim.sessionFile} is outside the session tree arm scans (${root}) — ` +
|
|
1118
|
+
`arm cannot arm while the reply can only land there`,
|
|
1119
|
+
`start the orchestrator pane's session under ${root}, or from the tick cwd ${scanDir}`,
|
|
1120
|
+
);
|
|
1121
|
+
}
|
|
1122
|
+
if (claimDir !== scanDir) {
|
|
1123
|
+
return warnFinding(
|
|
1124
|
+
"arm-session-dir",
|
|
1125
|
+
`[${p.name}] the orchestrator's claimed session file ${claim.sessionFile} lives in ${claimDir}, not the tick-cwd session directory ${scanDir} — ` +
|
|
1126
|
+
`arm follows the claim, so arming works, but this is the resumed-pane shape #600 names`,
|
|
1127
|
+
`optional tidy-up: start the pane's session from the tick cwd so transcript and cwd agree again`,
|
|
1128
|
+
);
|
|
1129
|
+
}
|
|
1130
|
+
return passFinding(
|
|
1131
|
+
"arm-session-dir",
|
|
1132
|
+
`[${p.name}] the orchestrator's claimed session file is inside the session directory arm scans`,
|
|
1133
|
+
);
|
|
1134
|
+
}
|
|
1135
|
+
|
|
1136
|
+
/**
|
|
1137
|
+
* #600 check — a pinned escalation.telegramTopicId that no live claim carries.
|
|
1138
|
+
*
|
|
1139
|
+
* The wizard pins from the claims at setup time, but the bridge re-claims
|
|
1140
|
+
* topics across restarts and conductor restarts the service itself (#407), so
|
|
1141
|
+
* the pin drifts. On a host with `topicsTidy` on it drifts *structurally*:
|
|
1142
|
+
* every pane exit closes the topic, so a pinned id can never be live again.
|
|
1143
|
+
* There the pin is vestigial — reporting a dead id as the live destination is
|
|
1144
|
+
* the exact false belief this check exists to remove.
|
|
1145
|
+
*/
|
|
1146
|
+
function topicPinProbe(probes: Probes, p: ProjectConfig): Finding {
|
|
1147
|
+
const pinned = p.escalation.telegramTopicId;
|
|
1148
|
+
if (pinned === undefined) {
|
|
1149
|
+
return passFinding(
|
|
1150
|
+
"topic-pin",
|
|
1151
|
+
`[${p.name}] no pinned topic id — sends follow the live claim or the flat chat`,
|
|
1152
|
+
);
|
|
1153
|
+
}
|
|
1154
|
+
const claims = probes.claimedTopics();
|
|
1155
|
+
if (claims.length === 0) {
|
|
1156
|
+
return passFinding(
|
|
1157
|
+
"topic-pin",
|
|
1158
|
+
`[${p.name}] pinned topic ${pinned} — no live claims to compare (bridge state unreadable, or not a forum host)`,
|
|
1159
|
+
);
|
|
1160
|
+
}
|
|
1161
|
+
if (claims.some((claim) => claim.threadId === pinned)) {
|
|
1162
|
+
return passFinding("topic-pin", `[${p.name}] pinned topic ${pinned} is a live claim`);
|
|
1163
|
+
}
|
|
1164
|
+
if (probes.topicsTidy()) {
|
|
1165
|
+
return warnFinding(
|
|
1166
|
+
"topic-pin",
|
|
1167
|
+
`[${p.name}] pinned topic ${pinned} is not among omp-telegram's live claims, and this host runs topicsTidy — ` +
|
|
1168
|
+
`the pin is vestigial and can never be live again (every pane exit closes its topic)`,
|
|
1169
|
+
`remove escalation.telegramTopicId from ${p.name}'s config (sends already follow the live claim), or turn topicsTidy off`,
|
|
1170
|
+
);
|
|
1171
|
+
}
|
|
1172
|
+
return warnFinding(
|
|
1173
|
+
"topic-pin",
|
|
1174
|
+
`[${p.name}] pinned topic ${pinned} is not among omp-telegram's live claims (${claims.map((c) => c.threadId).join(", ")}) — ` +
|
|
1175
|
+
`sends fall back to the live claim or the flat chat`,
|
|
1176
|
+
`update or remove escalation.telegramTopicId in ${p.name}'s config`,
|
|
1177
|
+
);
|
|
1178
|
+
}
|
|
1179
|
+
|
|
910
1180
|
/**
|
|
911
1181
|
* #541 check 2 — `[session] resume_agents_on_restore` on the live herdr config.
|
|
912
1182
|
*
|
|
@@ -1076,7 +1346,13 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
|
|
|
1076
1346
|
// difference behind a constant count).
|
|
1077
1347
|
findings.push(backupProbe(probes));
|
|
1078
1348
|
findings.push(dbProbe(probes));
|
|
1349
|
+
findings.push(dbBackupProbe(probes, cfg));
|
|
1079
1350
|
findings.push(await ghAuthProbe(probes, configuredRepos(cfg)));
|
|
1351
|
+
// The run-session root is one host-wide fact — where every run's session dir
|
|
1352
|
+
// (and the omp settings overlay inside it) lands — probed once and shared
|
|
1353
|
+
// across the per-project omp-settings findings.
|
|
1354
|
+
const sessionRoot = sessionRootDir();
|
|
1355
|
+
const sessionRootState = probes.sessionRootState(sessionRoot);
|
|
1080
1356
|
// The collected run sample across the resolved project set, newest-first;
|
|
1081
1357
|
// spend telemetry is a single host-wide finding, not one per project.
|
|
1082
1358
|
const spendRows: RunSpendRow[] = [];
|
|
@@ -1093,10 +1369,22 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
|
|
|
1093
1369
|
findings.push(
|
|
1094
1370
|
passFinding("herdr-agent-name", projectProblem === undefined ? "no project resolved — nothing to check" : `agent name uncheckable: ${projectProblem}`),
|
|
1095
1371
|
);
|
|
1372
|
+
findings.push(
|
|
1373
|
+
passFinding("arm-session-dir", projectProblem === undefined ? "no project resolved — nothing to check" : `arm session dir uncheckable: ${projectProblem}`),
|
|
1374
|
+
);
|
|
1375
|
+
findings.push(
|
|
1376
|
+
passFinding("topic-pin", projectProblem === undefined ? "no project resolved — nothing to check" : `topic pin uncheckable: ${projectProblem}`),
|
|
1377
|
+
);
|
|
1378
|
+
findings.push(
|
|
1379
|
+
passFinding("omp-settings", projectProblem === undefined ? "no project resolved — nothing to check" : `omp settings overlay uncheckable: ${projectProblem}`),
|
|
1380
|
+
);
|
|
1096
1381
|
} else {
|
|
1097
1382
|
for (const p of projects) {
|
|
1098
1383
|
findings.push(await labelProbe(probes, p));
|
|
1099
1384
|
findings.push(herdrAgentNameProbe(probes, p));
|
|
1385
|
+
findings.push(armSessionDirProbe(probes, p));
|
|
1386
|
+
findings.push(topicPinProbe(probes, p));
|
|
1387
|
+
findings.push(ompSettingsProbe(p, sessionRootState));
|
|
1100
1388
|
}
|
|
1101
1389
|
}
|
|
1102
1390
|
// The installed-unit check is host-global: one shared daemon. Any project
|
|
@@ -1149,6 +1437,8 @@ export function defaultProbes(): Probes {
|
|
|
1149
1437
|
stat: defaultStat,
|
|
1150
1438
|
uidOf: defaultUidOf,
|
|
1151
1439
|
dbIntegrity: defaultDbIntegrity,
|
|
1440
|
+
snapshotDirState: defaultSnapshotDirState,
|
|
1441
|
+
sessionRootState: defaultSessionRootState,
|
|
1152
1442
|
recentRuns: defaultRecentRuns,
|
|
1153
1443
|
telegramHealth: (projectName) => probeTelegramHealth(projectName),
|
|
1154
1444
|
telegramSend: telegramReportSend,
|
|
@@ -1158,6 +1448,8 @@ export function defaultProbes(): Probes {
|
|
|
1158
1448
|
herdrSession: () => resolveHerdrSessionWithBridge(),
|
|
1159
1449
|
herdrConfig: defaultHerdrConfig,
|
|
1160
1450
|
herdrEnv: defaultHerdrEnv,
|
|
1451
|
+
claimedTopics: () => claimedTelegramTopics(),
|
|
1452
|
+
topicsTidy: () => telegramTopicsTidy(),
|
|
1161
1453
|
tickAgentName: (p) => {
|
|
1162
1454
|
const tick = readTickConfig(tickCwdForProject(p));
|
|
1163
1455
|
if (tick.kind !== "ok") return undefined;
|