omp-conductor 0.15.13 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +72 -2
- package/package.json +2 -1
- package/schema/config.schema.json +6 -0
- package/src/admission.ts +745 -0
- package/src/ask.ts +47 -0
- package/src/backups.ts +19 -7
- package/src/board.ts +1 -2
- package/src/briefs/orchestrator.md +62 -4
- package/src/cli.ts +26 -0
- package/src/commands/context.ts +3 -0
- package/src/commands/decision.ts +10 -1
- package/src/commands/doctor.ts +2 -0
- package/src/commands/message.ts +8 -1
- package/src/commands/restart.ts +15 -3
- package/src/commands/restore-db.ts +146 -0
- package/src/commands/stop.ts +24 -15
- package/src/commands/unfreeze.ts +56 -0
- package/src/commands/watch.ts +77 -0
- package/src/config-schema.ts +9 -0
- package/src/config.ts +24 -0
- package/src/daemon.ts +239 -530
- package/src/dashboard/server.ts +2 -1
- package/src/decisions.ts +32 -7
- package/src/depends-on.ts +73 -0
- package/src/doctor.ts +178 -5
- package/src/escalate.ts +114 -15
- package/src/failure-class.ts +47 -0
- package/src/fleet.ts +41 -410
- package/src/gitops.ts +86 -1
- package/src/log.ts +40 -0
- package/src/model-fallback.ts +3 -2
- package/src/omp-settings.ts +114 -0
- package/src/omp.ts +39 -0
- package/src/orchestrator-tick.ts +7 -1
- package/src/reports.ts +124 -12
- package/src/session-host.ts +6 -0
- package/src/setup-wizard.ts +36 -0
- package/src/setup.ts +58 -1
- package/src/status-render.ts +445 -0
- package/src/stop-provenance.ts +53 -0
- package/src/store.ts +352 -11
- package/src/types.ts +187 -4
- package/src/unblock.ts +1 -1
- package/src/upgrade-verify.ts +1 -1
- package/src/upgrade.ts +1 -2
- package/src/verbs/server.ts +25 -0
- package/src/worker.ts +162 -10
package/src/dashboard/server.ts
CHANGED
|
@@ -28,7 +28,8 @@ import { chmodSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync }
|
|
|
28
28
|
import { join } from "node:path";
|
|
29
29
|
import { loadConfig, stateDir } from "../config.ts";
|
|
30
30
|
import { healthCheck, livingDaemon, type DaemonRecord } from "../lifecycle.ts";
|
|
31
|
-
import { classifyDaemonProjectHealth, fleetLayers
|
|
31
|
+
import { classifyDaemonProjectHealth, fleetLayers } from "../fleet.ts";
|
|
32
|
+
import type { FleetLayers } from "../status-render.ts";
|
|
32
33
|
import { statusSnapshot, type StatusSnapshot } from "../daemon.ts";
|
|
33
34
|
import { boardJson, boardSnapshotOnce } from "../board.ts";
|
|
34
35
|
import { dbPath, openStore } from "../store.ts";
|
package/src/decisions.ts
CHANGED
|
@@ -235,15 +235,40 @@ function age(since: number, now: number): string {
|
|
|
235
235
|
* being a memory question. A row whose condition has come true is flagged, not
|
|
236
236
|
* merely listed — that is the difference between a parked question and one to
|
|
237
237
|
* act on now.
|
|
238
|
+
*
|
|
239
|
+
* The two kinds render under their own headings (#459). Only rows a human must
|
|
240
|
+
* answer — `kind === "question"`, whatever condition they carry — appear under
|
|
241
|
+
* the operator heading; a watch the orchestrator set for itself goes under the
|
|
242
|
+
* watch heading with no instruction to resolve it, so a fleet at rest behind
|
|
243
|
+
* GitHub's checks is never mistaken for a fleet that is waiting on its
|
|
244
|
+
* operator. A met watch still surfaces to the orchestrator with its note and
|
|
245
|
+
* the same `[CONDITION MET]` flag a met question gets.
|
|
238
246
|
*/
|
|
239
247
|
export function formatDecisionDigest(open: readonly DecisionRecord[], now = Date.now()): string {
|
|
240
|
-
|
|
241
|
-
const
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
248
|
+
const questions = open.filter((d) => d.kind !== "watch");
|
|
249
|
+
const watches = open.filter((d) => d.kind === "watch");
|
|
250
|
+
const lines: string[] = [];
|
|
251
|
+
if (questions.length > 0) {
|
|
252
|
+
lines.push(
|
|
253
|
+
`Open operator decisions (${questions.length}) — resolve or withdraw each with omp-conductor decision resolve|withdraw <id>:`,
|
|
254
|
+
);
|
|
255
|
+
for (const d of questions) {
|
|
256
|
+
lines.push(decisionLine(d, now));
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
if (watches.length > 0) {
|
|
260
|
+
lines.push(
|
|
261
|
+
`Watches (${watches.length}) — conditions the orchestrator set for itself; no operator action needed:`,
|
|
262
|
+
);
|
|
263
|
+
for (const d of watches) {
|
|
264
|
+
lines.push(decisionLine(d, now));
|
|
265
|
+
}
|
|
247
266
|
}
|
|
248
267
|
return lines.join("\n");
|
|
249
268
|
}
|
|
269
|
+
|
|
270
|
+
/** One digest row: id, age, what it blocks, the met flag, and the note. */
|
|
271
|
+
function decisionLine(d: DecisionRecord, now: number): string {
|
|
272
|
+
const flag = d.conditionMetAt === undefined ? "" : " [CONDITION MET — act on this now]";
|
|
273
|
+
return `- ${d.id} (${age(d.askedAt, now)}, blocks ${d.blocks ?? "nothing"})${flag}: ${d.question}`;
|
|
274
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `Depends-on:` declaration parser (epic #321, slice #419).
|
|
3
|
+
*
|
|
4
|
+
* A candidate issue can declare same-repo prerequisites it must not be
|
|
5
|
+
* dispatched before: `Depends-on: #123` (repeatable, one or more per line).
|
|
6
|
+
* Admission reads every referenced issue's live tracker state and holds the
|
|
7
|
+
* candidate while any prerequisite is open, so a worker is never sent at work
|
|
8
|
+
* whose base is not yet merged.
|
|
9
|
+
*
|
|
10
|
+
* The parser is deliberately small and strict, the two halves of the contract:
|
|
11
|
+
* the *marker* is matched case-insensitively (so `DEPENDS-ON`, `Depends-On`,
|
|
12
|
+
* `depends-on` all declare), while the *references* are strict bare issue
|
|
13
|
+
* numbers (`#123`). A marker line that carries no strict reference — an empty
|
|
14
|
+
* `Depends-on:`, or a bare non-numeric `Depends-on: #abc` — is never a usable
|
|
15
|
+
* prerequisite, so it is reported as `malformed` for the grooming flag rather
|
|
16
|
+
* than guessed at, and the declaration it belongs to is ignored.
|
|
17
|
+
*
|
|
18
|
+
* Cross-repo references (a qualifying `owner/repo#n` or `repo#n` form) and
|
|
19
|
+
* graph cycles are later slices (#420/#421) and are deliberately out of scope
|
|
20
|
+
* here: only plain `#<n>` same-repo forms are resolved.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
export interface DependsOnDecl {
|
|
24
|
+
/** Referenced prerequisite issue numbers, deduplicated, first-seen order. */
|
|
25
|
+
readonly refs: readonly number[];
|
|
26
|
+
/** Marker-bearing lines that carried no strict `#<n>` reference (e.g.
|
|
27
|
+
* `Depends-on: #abc`). Bounded to what the body actually said, for the
|
|
28
|
+
* grooming flag's detail. */
|
|
29
|
+
readonly malformed: readonly string[];
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* How a line is spelled: `depends-on`/`depends on`, optional whitespace, then
|
|
34
|
+
* a `:` or `=` separator. The marker is matched case-insensitively; the value
|
|
35
|
+
* (everything after the separator) is what the strict-reference rules apply
|
|
36
|
+
* to. Anchored at the start of the (trimmed) line so a prose mention of
|
|
37
|
+
* "Depends-on" mid-sentence is not a declaration.
|
|
38
|
+
*/
|
|
39
|
+
const MARKER = /^depends[-\s]?on\s*[:=]\s*(.*)$/i;
|
|
40
|
+
|
|
41
|
+
/** A strict reference is a bare `#` followed by digits — no repo qualifier,
|
|
42
|
+
* no words. Everything else is not a prerequisite this slice resolves. */
|
|
43
|
+
const REF = /#(\d+)/g;
|
|
44
|
+
|
|
45
|
+
export function parseDependsOn(body: string): DependsOnDecl {
|
|
46
|
+
const refs: number[] = [];
|
|
47
|
+
const seen = new Set<number>();
|
|
48
|
+
const malformed: string[] = [];
|
|
49
|
+
for (const raw of body.split("\n")) {
|
|
50
|
+
const line = raw.trim();
|
|
51
|
+
if (line === "") continue;
|
|
52
|
+
const match = MARKER.exec(line);
|
|
53
|
+
if (match === null) continue;
|
|
54
|
+
const value = match[1] ?? "";
|
|
55
|
+
REF.lastIndex = 0;
|
|
56
|
+
const found = [...value.matchAll(REF)];
|
|
57
|
+
if (found.length === 0) {
|
|
58
|
+
// A marker line with no strict reference is a malformed declaration:
|
|
59
|
+
// never crash, never silently claim — the declaration is ignored and
|
|
60
|
+
// surfaced for grooming.
|
|
61
|
+
malformed.push(line);
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
for (const group of found) {
|
|
65
|
+
const n = Number(group[1]!);
|
|
66
|
+
if (Number.isSafeInteger(n) && !seen.has(n)) {
|
|
67
|
+
seen.add(n);
|
|
68
|
+
refs.push(n);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return { refs, malformed };
|
|
73
|
+
}
|
package/src/doctor.ts
CHANGED
|
@@ -25,21 +25,38 @@
|
|
|
25
25
|
* (the #399 boundary).
|
|
26
26
|
*/
|
|
27
27
|
|
|
28
|
-
import {
|
|
28
|
+
import {
|
|
29
|
+
accessSync,
|
|
30
|
+
constants,
|
|
31
|
+
existsSync,
|
|
32
|
+
readdirSync,
|
|
33
|
+
readFileSync,
|
|
34
|
+
statSync,
|
|
35
|
+
} from "node:fs";
|
|
29
36
|
import { spawnSync } from "node:child_process";
|
|
30
37
|
import { homedir } from "node:os";
|
|
31
|
-
import { join } from "node:path";
|
|
38
|
+
import { dirname, join } from "node:path";
|
|
32
39
|
import type { Stats } from "node:fs";
|
|
33
40
|
import { Database } from "bun:sqlite";
|
|
41
|
+
import { stringify } from "yaml";
|
|
34
42
|
|
|
35
|
-
import {
|
|
43
|
+
import {
|
|
44
|
+
configBackupDir,
|
|
45
|
+
configPath,
|
|
46
|
+
dbBackupDirFor,
|
|
47
|
+
findProject,
|
|
48
|
+
loadConfig,
|
|
49
|
+
resolveCaps,
|
|
50
|
+
stateDir,
|
|
51
|
+
} from "./config.ts";
|
|
52
|
+
import { ompSettingsOverlay, sessionRootDir } from "./omp-settings.ts";
|
|
36
53
|
import {
|
|
37
54
|
DEFAULT_HERDR_UNIT,
|
|
38
55
|
probeTelegramHealth,
|
|
39
56
|
resolveHerdrSessionWithBridge,
|
|
40
57
|
telegramStateDir,
|
|
41
|
-
type TelegramHealth,
|
|
42
58
|
} from "./fleet.ts";
|
|
59
|
+
import type { TelegramHealth } from "./status-render.ts";
|
|
43
60
|
import {
|
|
44
61
|
herdrConductorPluginConfigDir,
|
|
45
62
|
planHostRuntime,
|
|
@@ -49,7 +66,7 @@ import {
|
|
|
49
66
|
totalConfiguredWorkers,
|
|
50
67
|
} from "./setup-host.ts";
|
|
51
68
|
import { checkTokenScopes, type ScopeCheck } from "./setup.ts";
|
|
52
|
-
import { dbPath, LIVE_STATES } from "./store.ts";
|
|
69
|
+
import { DB_SNAPSHOT_STEM, dbPath, LIVE_STATES } from "./store.ts";
|
|
53
70
|
import { telegramReportSend, type ReportSend } from "./reports.ts";
|
|
54
71
|
import { fetchRateLimit, GhError, gh } from "./tracker/github.ts";
|
|
55
72
|
import { repoSlugFor } from "./gitops.ts";
|
|
@@ -159,6 +176,14 @@ export interface DoctorDeps {
|
|
|
159
176
|
uidOf?: (name: string) => number | undefined;
|
|
160
177
|
/** `PRAGMA integrity_check` over the store, read-only. ok=true for "ok". */
|
|
161
178
|
dbIntegrity?: (path: string) => { ok: boolean; detail?: string };
|
|
179
|
+
/** Whether the configured db-snapshot directory exists and is writable, so a
|
|
180
|
+
* snapshot can land. Injected so a test never chmods the host (#399). */
|
|
181
|
+
snapshotDirState?: (dir: string) => { exists: boolean; writable: boolean };
|
|
182
|
+
/** Whether the run-session root — where the omp settings overlay is
|
|
183
|
+
* materialised at dispatch (#537) — can hold a new `run-<id>` directory. A
|
|
184
|
+
* missing `sessions/` parent is not a fault (dispatch's recursive `mkdir`
|
|
185
|
+
* creates it), so the writable question falls back to the state dir itself. */
|
|
186
|
+
sessionRootState?: (root: string) => { exists: boolean; writable: boolean };
|
|
162
187
|
/** The most recent completed runs for one project, newest first, limited. */
|
|
163
188
|
recentRuns?: (project: string, limit: number) => RunSpendRow[];
|
|
164
189
|
/** The bot health probe `status` uses (getMe etc.). */
|
|
@@ -286,6 +311,55 @@ function defaultDbIntegrity(path: string): { ok: boolean; detail?: string } {
|
|
|
286
311
|
}
|
|
287
312
|
}
|
|
288
313
|
|
|
314
|
+
/**
|
|
315
|
+
* Whether the configured db-snapshot directory exists and is writable. No
|
|
316
|
+
* write is performed — the check is stat + access, keeping doctor read-only.
|
|
317
|
+
* A directory that does not exist reads `writable: false` and lets the probe
|
|
318
|
+
* report it as "no snapshot yet" rather than inventing one, while a read-only
|
|
319
|
+
* or non-directory path reads as a genuine unwritable failure.
|
|
320
|
+
*/
|
|
321
|
+
function defaultSnapshotDirState(dir: string): { exists: boolean; writable: boolean } {
|
|
322
|
+
try {
|
|
323
|
+
const st = statSync(dir);
|
|
324
|
+
if (!st.isDirectory()) return { exists: true, writable: false };
|
|
325
|
+
try {
|
|
326
|
+
accessSync(dir, constants.W_OK);
|
|
327
|
+
return { exists: true, writable: true };
|
|
328
|
+
} catch {
|
|
329
|
+
return { exists: true, writable: false };
|
|
330
|
+
}
|
|
331
|
+
} catch {
|
|
332
|
+
return { exists: false, writable: false };
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
/**
|
|
337
|
+
* Whether the run-session root can hold a new session directory. No write is
|
|
338
|
+
* performed — the same stat + access as {@link defaultSnapshotDirState}, which
|
|
339
|
+
* keeps `doctor` read-only. A missing `sessions/` parent is normal on a fresh
|
|
340
|
+
* host (dispatch's recursive `mkdir` creates it), so the writable question
|
|
341
|
+
* falls back to the state dir it would be created under; a state dir that
|
|
342
|
+
* cannot even be read is a genuine unwritable failure.
|
|
343
|
+
*/
|
|
344
|
+
function defaultSessionRootState(root: string): { exists: boolean; writable: boolean } {
|
|
345
|
+
const writable = (dir: string): boolean => {
|
|
346
|
+
try {
|
|
347
|
+
const st = statSync(dir);
|
|
348
|
+
if (!st.isDirectory()) return false;
|
|
349
|
+
accessSync(dir, constants.W_OK);
|
|
350
|
+
return true;
|
|
351
|
+
} catch {
|
|
352
|
+
return false;
|
|
353
|
+
}
|
|
354
|
+
};
|
|
355
|
+
try {
|
|
356
|
+
const st = statSync(root);
|
|
357
|
+
return { exists: true, writable: st.isDirectory() && writable(root) };
|
|
358
|
+
} catch {
|
|
359
|
+
return { exists: false, writable: writable(dirname(root)) };
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
|
|
289
363
|
/**
|
|
290
364
|
* The most recent completed runs for one project, newest first — read on the
|
|
291
365
|
* same read-only connection the integrity check opens, because the store's own
|
|
@@ -419,6 +493,57 @@ function dbProbe(probes: Probes): Finding {
|
|
|
419
493
|
);
|
|
420
494
|
}
|
|
421
495
|
|
|
496
|
+
/**
|
|
497
|
+
* The conductor.db snapshot store (#579): the configured dir is writable so a
|
|
498
|
+
* snapshot can land, and a snapshot at least as fresh as the store protects
|
|
499
|
+
* it. The store is the carrier of the verb ledger, the decision rows, the run
|
|
500
|
+
* rows and the material-event ledger; a store with no restorable copy has no
|
|
501
|
+
* recovery path, so absence is a warning and an unwritable configured dir a
|
|
502
|
+
* failure.
|
|
503
|
+
*/
|
|
504
|
+
function dbBackupProbe(probes: Probes, cfg: ConductorConfig | undefined): Finding {
|
|
505
|
+
const dir = dbBackupDirFor(cfg);
|
|
506
|
+
const state = probes.snapshotDirState(dir);
|
|
507
|
+
if (state.exists && !state.writable) {
|
|
508
|
+
return failFinding(
|
|
509
|
+
"db-backup",
|
|
510
|
+
`db backup dir ${dir} is not writable — a conductor.db snapshot cannot land there`,
|
|
511
|
+
`make ${dir} writable by the conductor account, or set dbBackupDir in config to a writable directory`,
|
|
512
|
+
);
|
|
513
|
+
}
|
|
514
|
+
const snapshots = (() => {
|
|
515
|
+
try {
|
|
516
|
+
return readdirSync(dir).filter((name) => name.startsWith(DB_SNAPSHOT_STEM));
|
|
517
|
+
} catch {
|
|
518
|
+
return [];
|
|
519
|
+
}
|
|
520
|
+
})();
|
|
521
|
+
const store = probes.stat(dbPath());
|
|
522
|
+
if (store === undefined) {
|
|
523
|
+
return snapshots.length === 0
|
|
524
|
+
? passFinding("db-backup", "no conductor.db yet — nothing to snapshot")
|
|
525
|
+
: passFinding("db-backup", `${snapshots.length} conductor.db snapshot(s) present`);
|
|
526
|
+
}
|
|
527
|
+
if (snapshots.length === 0) {
|
|
528
|
+
return warnFinding(
|
|
529
|
+
"db-backup",
|
|
530
|
+
"conductor.db has no snapshot — losing it loses the audit trail with no recovery path",
|
|
531
|
+
"take a conductor.db snapshot (the daemon's cadence files one; restore-db needs one to restore from)",
|
|
532
|
+
);
|
|
533
|
+
}
|
|
534
|
+
const newestMtime = snapshots
|
|
535
|
+
.map((name) => probes.stat(join(dir, name))?.mtimeMs ?? 0)
|
|
536
|
+
.reduce((max, m) => Math.max(max, m), 0);
|
|
537
|
+
if (store.mtimeMs > newestMtime + 60_000) {
|
|
538
|
+
return warnFinding(
|
|
539
|
+
"db-backup",
|
|
540
|
+
`conductor.db is newer than every snapshot (${snapshots.length} file(s)) — the newest writes were never snapshot`,
|
|
541
|
+
"take a fresh conductor.db snapshot so restore-db has the current state",
|
|
542
|
+
);
|
|
543
|
+
}
|
|
544
|
+
return passFinding("db-backup", `conductor.db snapshot is fresh (${snapshots.length} file(s))`);
|
|
545
|
+
}
|
|
546
|
+
|
|
422
547
|
/** Every repo the config names: each project's tracker plus routing targets,
|
|
423
548
|
* as GitHub slugs, deduplicated, in config order. */
|
|
424
549
|
function configuredRepos(cfg: ConductorConfig | undefined): string[] {
|
|
@@ -754,6 +879,42 @@ export function isKnownTimezone(tz: string): boolean {
|
|
|
754
879
|
}
|
|
755
880
|
}
|
|
756
881
|
|
|
882
|
+
/**
|
|
883
|
+
* The fleet-owned omp settings overlay (#537): reports the *effective* overlay
|
|
884
|
+
* every worker session of the project will load — the `ompSettings` map plus
|
|
885
|
+
* the retry keys derived from `modelFallbacks`, rendered exactly as the
|
|
886
|
+
* materialiser writes them, never the raw config snippet `setup` would echo —
|
|
887
|
+
* and turns red when that overlay cannot land. The file is written at dispatch
|
|
888
|
+
* under `<stateDir>/sessions/run-<id>/`, so doctor cannot prove the write
|
|
889
|
+
* without writing: it probes the root the materialisation would land in with
|
|
890
|
+
* the same stat + access the db-backup probe uses, keeping the check
|
|
891
|
+
* read-only (#287). The session root is a host-wide fact, probed once and
|
|
892
|
+
* shared across the per-project findings.
|
|
893
|
+
*/
|
|
894
|
+
function ompSettingsProbe(
|
|
895
|
+
project: ProjectConfig,
|
|
896
|
+
sessionRoot: { exists: boolean; writable: boolean },
|
|
897
|
+
): Finding {
|
|
898
|
+
const overlay = ompSettingsOverlay(project);
|
|
899
|
+
if (overlay === undefined) {
|
|
900
|
+
return passFinding(
|
|
901
|
+
"omp-settings",
|
|
902
|
+
`[${project.name}] no omp settings overlay configured — workers inherit the daemon account's global settings unchanged`,
|
|
903
|
+
);
|
|
904
|
+
}
|
|
905
|
+
if (!sessionRoot.writable) {
|
|
906
|
+
return failFinding(
|
|
907
|
+
"omp-settings",
|
|
908
|
+
`[${project.name}] the omp settings overlay cannot be materialised: ${sessionRootDir()} cannot hold a session directory, so every worker would dispatch without its staged settings`,
|
|
909
|
+
`make ${sessionRootDir()} writable by the conductor account`,
|
|
910
|
+
);
|
|
911
|
+
}
|
|
912
|
+
return passFinding(
|
|
913
|
+
"omp-settings",
|
|
914
|
+
`[${project.name}] worker sessions load this omp settings overlay:\n${stringify(overlay).trimEnd()}`,
|
|
915
|
+
);
|
|
916
|
+
}
|
|
917
|
+
|
|
757
918
|
/** The self-identified probe message `--probe-telegram` delivers through the
|
|
758
919
|
* same report transport a real report would take. */
|
|
759
920
|
function probeMessage(project: string, checkedAt: string): string {
|
|
@@ -1076,7 +1237,13 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
|
|
|
1076
1237
|
// difference behind a constant count).
|
|
1077
1238
|
findings.push(backupProbe(probes));
|
|
1078
1239
|
findings.push(dbProbe(probes));
|
|
1240
|
+
findings.push(dbBackupProbe(probes, cfg));
|
|
1079
1241
|
findings.push(await ghAuthProbe(probes, configuredRepos(cfg)));
|
|
1242
|
+
// The run-session root is one host-wide fact — where every run's session dir
|
|
1243
|
+
// (and the omp settings overlay inside it) lands — probed once and shared
|
|
1244
|
+
// across the per-project omp-settings findings.
|
|
1245
|
+
const sessionRoot = sessionRootDir();
|
|
1246
|
+
const sessionRootState = probes.sessionRootState(sessionRoot);
|
|
1080
1247
|
// The collected run sample across the resolved project set, newest-first;
|
|
1081
1248
|
// spend telemetry is a single host-wide finding, not one per project.
|
|
1082
1249
|
const spendRows: RunSpendRow[] = [];
|
|
@@ -1093,10 +1260,14 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
|
|
|
1093
1260
|
findings.push(
|
|
1094
1261
|
passFinding("herdr-agent-name", projectProblem === undefined ? "no project resolved — nothing to check" : `agent name uncheckable: ${projectProblem}`),
|
|
1095
1262
|
);
|
|
1263
|
+
findings.push(
|
|
1264
|
+
passFinding("omp-settings", projectProblem === undefined ? "no project resolved — nothing to check" : `omp settings overlay uncheckable: ${projectProblem}`),
|
|
1265
|
+
);
|
|
1096
1266
|
} else {
|
|
1097
1267
|
for (const p of projects) {
|
|
1098
1268
|
findings.push(await labelProbe(probes, p));
|
|
1099
1269
|
findings.push(herdrAgentNameProbe(probes, p));
|
|
1270
|
+
findings.push(ompSettingsProbe(p, sessionRootState));
|
|
1100
1271
|
}
|
|
1101
1272
|
}
|
|
1102
1273
|
// The installed-unit check is host-global: one shared daemon. Any project
|
|
@@ -1149,6 +1320,8 @@ export function defaultProbes(): Probes {
|
|
|
1149
1320
|
stat: defaultStat,
|
|
1150
1321
|
uidOf: defaultUidOf,
|
|
1151
1322
|
dbIntegrity: defaultDbIntegrity,
|
|
1323
|
+
snapshotDirState: defaultSnapshotDirState,
|
|
1324
|
+
sessionRootState: defaultSessionRootState,
|
|
1152
1325
|
recentRuns: defaultRecentRuns,
|
|
1153
1326
|
telegramHealth: (projectName) => probeTelegramHealth(projectName),
|
|
1154
1327
|
telegramSend: telegramReportSend,
|
package/src/escalate.ts
CHANGED
|
@@ -29,8 +29,79 @@ import { heldNoticeId } from "./notices.ts";
|
|
|
29
29
|
import type { OrchestratorHandle } from "./orchestrator.ts";
|
|
30
30
|
import type { Escalation, ProjectConfig, Store, Tracker } from "./types.ts";
|
|
31
31
|
|
|
32
|
-
/**
|
|
33
|
-
|
|
32
|
+
/**
|
|
33
|
+
* Telegram rejects `sendMessage` over 4096 characters. Longer text is split
|
|
34
|
+
* into labelled parts ({@link telegramTextParts}) instead of truncated, so the
|
|
35
|
+
* tail of a digest or a tier-2 escalation arrives instead of being silently
|
|
36
|
+
* dropped (#566).
|
|
37
|
+
*/
|
|
38
|
+
export const TELEGRAM_TEXT_LIMIT = 4096;
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Width reserved for the `(i/n)` label prepended to every part of a split
|
|
42
|
+
* message, ported from omp-telegram's `PART_LABEL_RESERVE`. Parts are re-split
|
|
43
|
+
* against `TELEGRAM_TEXT_LIMIT - TELEGRAM_PART_LABEL_RESERVE` so the label can
|
|
44
|
+
* never push a part past the wire limit.
|
|
45
|
+
*/
|
|
46
|
+
export const TELEGRAM_PART_LABEL_RESERVE = 16;
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Split text into Telegram-sized parts. Ports omp-telegram 0.12.1's
|
|
50
|
+
* `chunkLabeled` semantics (the plugin this transport bypasses, #566) so a
|
|
51
|
+
* digest copies the same readability contract that plugin ships:
|
|
52
|
+
*
|
|
53
|
+
* - newline-preferred boundaries: a cut prefers the last paragraph break
|
|
54
|
+
* (`\n\n`), then line break, then the last space, each only when it lands in
|
|
55
|
+
* the second half of the window so parts stay large; an unsplittable run is
|
|
56
|
+
* hard-cut rather than dropped.
|
|
57
|
+
* - numbered labels: when a message takes more than one part, every part is
|
|
58
|
+
* prefixed `(i/N)\n` so a reader sees the message continues and knows how
|
|
59
|
+
* many parts to expect. Under the label budget, so no part exceeds the limit.
|
|
60
|
+
* - lossless: a split message reassembles byte-for-byte to its input
|
|
61
|
+
* (strip the labels and concatenate). Deliberately *not* ported from
|
|
62
|
+
* upstream: fence repair and leading-newline stripping are rendering
|
|
63
|
+
* flourishes for markdown-mode sends, and this transport sends plain text
|
|
64
|
+
* (`parse_mode` is never set) — both would rewrite bytes the operator never
|
|
65
|
+
* wrote.
|
|
66
|
+
*
|
|
67
|
+
* Text at or under the limit is returned whole and untouched, so the
|
|
68
|
+
* single-message path keeps sending byte-identical payloads.
|
|
69
|
+
*/
|
|
70
|
+
export function telegramTextParts(text: string, limit: number = TELEGRAM_TEXT_LIMIT): string[] {
|
|
71
|
+
if (text.length === 0 || text.length <= limit) return [text];
|
|
72
|
+
// `splitToLimit` returns at least two pieces here (text is over the limit);
|
|
73
|
+
// the guard is for future-proofing against a limit rewrite.
|
|
74
|
+
const parts = splitToLimit(text, Math.max(1, limit));
|
|
75
|
+
if (parts.length === 1) return parts;
|
|
76
|
+
// Re-split under the label budget so `(i/N)\n` never pushes a part past the
|
|
77
|
+
// wire limit.
|
|
78
|
+
const labelled = splitToLimit(text, Math.max(1, limit - TELEGRAM_PART_LABEL_RESERVE));
|
|
79
|
+
return labelled.map((part, index) => `(${index + 1}/${labelled.length})\n${part}`);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* The base splitter, ported from omp-telegram's `splitToLimit`. Prefers a
|
|
84
|
+
* paragraph break, then a line break, then a space — each only when it sits in
|
|
85
|
+
* the second half of the window, so a digest splits on natural boundaries
|
|
86
|
+
* rather than mid-word. Never drops a byte: the fallback is a hard cut at
|
|
87
|
+
* `limit`, which split a run with no whitespace in the window in the middle,
|
|
88
|
+
* and the parts still reassemble to the input.
|
|
89
|
+
*/
|
|
90
|
+
function splitToLimit(text: string, limit: number): string[] {
|
|
91
|
+
if (text.length <= limit) return [text];
|
|
92
|
+
const out: string[] = [];
|
|
93
|
+
let rest = text;
|
|
94
|
+
while (rest.length > limit) {
|
|
95
|
+
const para = rest.lastIndexOf("\n\n", limit);
|
|
96
|
+
const line = rest.lastIndexOf("\n", limit);
|
|
97
|
+
const space = rest.lastIndexOf(" ", limit);
|
|
98
|
+
const cut = para > limit / 2 ? para : line > limit / 2 ? line : space > 0 ? space : limit;
|
|
99
|
+
out.push(rest.slice(0, cut));
|
|
100
|
+
rest = rest.slice(cut);
|
|
101
|
+
}
|
|
102
|
+
out.push(rest);
|
|
103
|
+
return out;
|
|
104
|
+
}
|
|
34
105
|
|
|
35
106
|
/**
|
|
36
107
|
* Whether a failed send settles the question of delivery. This distinction is
|
|
@@ -471,26 +542,39 @@ export function claimedTelegramTopics(stateDir?: string): ClaimedTopic[] {
|
|
|
471
542
|
* The one Telegram send in this package. Exported so the report outbox (#123)
|
|
472
543
|
* reuses it rather than forking it: the response handling below is load-bearing
|
|
473
544
|
* and was paid for once already (see the comment on the parse). Resolves with
|
|
474
|
-
* Telegram's own message id
|
|
475
|
-
*
|
|
476
|
-
*
|
|
545
|
+
* Telegram's own message id for every accepted part, in order — one id for a
|
|
546
|
+
* message under the limit, one per message once a split is needed. Throws on
|
|
547
|
+
* every *known* failure — connection refused, HTTP error, `{"ok":false}` —
|
|
548
|
+
* which is what lets a caller treat a throw as "nobody has this" and a crash
|
|
549
|
+
* as "nobody knows".
|
|
550
|
+
*
|
|
551
|
+
* Splitting happens here, at the transport, so every message class that rides
|
|
552
|
+
* this seam — tier-2 escalations, report parts, fleet pages, operator
|
|
553
|
+
* messages — is delivered whole: a report's tail is the fleet state and open
|
|
554
|
+
* items, and a tier-2 escalation's tail may be the question itself (#566).
|
|
555
|
+
* Parts are sent strictly in order, one `sendMessage` at a time, so they never
|
|
556
|
+
* interleave with each other.
|
|
477
557
|
*
|
|
478
558
|
* Optional `topicId` pins the message to a forum topic (`message_thread_id`).
|
|
479
|
-
* A definitive missing-thread reject retries once as a flat chat and warns, so
|
|
480
|
-
* deleted topic degrades instead of silently losing the page (#318).
|
|
559
|
+
* A definitive missing-thread reject retries once as a flat chat and warns, so
|
|
560
|
+
* a deleted topic degrades instead of silently losing the page (#318). The
|
|
561
|
+
* flat retry re-sends the whole split: parts already accepted went into a
|
|
562
|
+
* thread Telegram has just told us is gone, so nothing is readable there that
|
|
563
|
+
* a flat resend would duplicate.
|
|
481
564
|
*/
|
|
482
565
|
export async function sendTelegram(
|
|
483
566
|
token: string,
|
|
484
567
|
chatId: string,
|
|
485
568
|
text: string,
|
|
486
569
|
opts?: { topicId?: number },
|
|
487
|
-
): Promise<number
|
|
570
|
+
): Promise<number[]> {
|
|
488
571
|
const topicId =
|
|
489
572
|
opts?.topicId !== undefined && Number.isFinite(opts.topicId) && Number.isSafeInteger(opts.topicId)
|
|
490
573
|
? opts.topicId
|
|
491
574
|
: undefined;
|
|
575
|
+
const parts = telegramTextParts(text);
|
|
492
576
|
try {
|
|
493
|
-
return await
|
|
577
|
+
return await postTelegramParts(token, chatId, parts, topicId);
|
|
494
578
|
} catch (err) {
|
|
495
579
|
if (
|
|
496
580
|
topicId === undefined ||
|
|
@@ -503,8 +587,23 @@ export async function sendTelegram(
|
|
|
503
587
|
warn(
|
|
504
588
|
`escalation.telegramTopicId=${topicId} is stale (${err.message}); retrying flat chat`,
|
|
505
589
|
);
|
|
506
|
-
return await
|
|
590
|
+
return await postTelegramParts(token, chatId, parts, undefined);
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
/** One part per `sendMessage`, in order, collecting the ids Telegram returns. */
|
|
595
|
+
async function postTelegramParts(
|
|
596
|
+
token: string,
|
|
597
|
+
chatId: string,
|
|
598
|
+
parts: readonly string[],
|
|
599
|
+
topicId: number | undefined,
|
|
600
|
+
): Promise<number[]> {
|
|
601
|
+
const ids: number[] = [];
|
|
602
|
+
for (const part of parts) {
|
|
603
|
+
const id = await postTelegramMessage(token, chatId, part, topicId);
|
|
604
|
+
if (id !== undefined) ids.push(id);
|
|
507
605
|
}
|
|
606
|
+
return ids;
|
|
508
607
|
}
|
|
509
608
|
|
|
510
609
|
/** Telegram's definitive "that forum topic is gone" answers. */
|
|
@@ -521,11 +620,11 @@ async function postTelegramMessage(
|
|
|
521
620
|
const url = `https://api.telegram.org/bot${token}/sendMessage`;
|
|
522
621
|
const payload: Record<string, unknown> = {
|
|
523
622
|
chat_id: chatId,
|
|
524
|
-
//
|
|
525
|
-
//
|
|
526
|
-
//
|
|
527
|
-
|
|
528
|
-
|
|
623
|
+
// `sendTelegram` splits long text into labelled parts above this call, so
|
|
624
|
+
// the text handed here is always a whole message: a part under the wire
|
|
625
|
+
// limit, or a message that never needed splitting. Nothing is truncated —
|
|
626
|
+
// a hard slice here is how the digest lost its tail (#566).
|
|
627
|
+
text,
|
|
529
628
|
disable_web_page_preview: true,
|
|
530
629
|
};
|
|
531
630
|
if (topicId !== undefined) payload.message_thread_id = topicId;
|
package/src/failure-class.ts
CHANGED
|
@@ -157,6 +157,35 @@ export function providerTransientFault(error: { status?: number; message: string
|
|
|
157
157
|
return error.message.split("\n")[0]?.trim() ?? error.message;
|
|
158
158
|
}
|
|
159
159
|
|
|
160
|
+
/**
|
|
161
|
+
* How many in-session provider 429s read as *sustained* rate limiting rather
|
|
162
|
+
* than a retried blip (#573).
|
|
163
|
+
*
|
|
164
|
+
* A single 429 the harness retried and recovered from is noise — counting it
|
|
165
|
+
* would reclassify every run that ever hit one throttle as a provider fault
|
|
166
|
+
* and let real failures escape the failed-attempt budget. Only a run that kept
|
|
167
|
+
* hitting the throttle never gets clear of it, and that is the signature the
|
|
168
|
+
* classifier needs. Three matches the "three strikes" motif this file already
|
|
169
|
+
* uses (`DISPATCH_INFRA_MAX_STRIKES`, `PROVIDER_TRANSIENT_MAX_STRIKES`).
|
|
170
|
+
*/
|
|
171
|
+
const SUSTAINED_RATE_LIMIT_COUNT = 3;
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Evidence that this run drowned in in-session provider rate limiting, or
|
|
175
|
+
* `undefined` when it did not.
|
|
176
|
+
*
|
|
177
|
+
* Reads the count the worker recorded while the run was live — never a string
|
|
178
|
+
* match over the transcript, and never `lastError`: omp retried the 429s
|
|
179
|
+
* internally, so they never surfaced one `lastError`, which is exactly why the
|
|
180
|
+
* run was previously unclassifiable (#573). 429 is deliberately its own class:
|
|
181
|
+
* a generic 429 is rate limiting, a 402 is credit, a stream abort is
|
|
182
|
+
* transient — three failures with three remedies (#220).
|
|
183
|
+
*/
|
|
184
|
+
export function providerCapacityFault(count: number | undefined): string | undefined {
|
|
185
|
+
if (count === undefined || count < SUSTAINED_RATE_LIMIT_COUNT) return undefined;
|
|
186
|
+
return `${count} in-session provider rate limits (HTTP 429) — the harness retried and was exhausted`;
|
|
187
|
+
}
|
|
188
|
+
|
|
160
189
|
/**
|
|
161
190
|
* Evidence that this run never started, or `undefined` when something did happen.
|
|
162
191
|
*
|
|
@@ -300,6 +329,24 @@ export function classifyRun(
|
|
|
300
329
|
}
|
|
301
330
|
}
|
|
302
331
|
|
|
332
|
+
// Sustained in-session provider rate limiting: the run drowned in HTTP 429s
|
|
333
|
+
// the harness retried and never surfaced one as `lastError`, so neither the
|
|
334
|
+
// credit nor the transient branch sees it and it used to fall through to
|
|
335
|
+
// `unknown`, spending an implementation attempt on the provider's capacity
|
|
336
|
+
// (#573). Only the worker-recorded count — above a *sustained* threshold, so
|
|
337
|
+
// a single retried 429 on an otherwise healthy run stays untouched — reads as
|
|
338
|
+
// this. Distinct from the two provider classes on purpose and requeued free,
|
|
339
|
+
// bounded by its own strike cap in the daemon so a permanently throttled
|
|
340
|
+
// provider escalates instead of looping. Ahead of `neverStarted` for the same
|
|
341
|
+
// reason as the credit branch: a turn-0 run that immediately drowned in 429s
|
|
342
|
+
// would otherwise be absorbed by `env-start-failure` and lose its cause.
|
|
343
|
+
if (run.state === "failed" || run.state === "killed") {
|
|
344
|
+
const capacity = providerCapacityFault(run.provider429Count);
|
|
345
|
+
if (capacity !== undefined) {
|
|
346
|
+
return { cls: "provider-capacity", recovery: "requeue", evidence: capacity };
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
|
|
303
350
|
// A run that never started is the most classifiable failure there is, and the
|
|
304
351
|
// least deserving of an implementation attempt: the session did not get as far
|
|
305
352
|
// as reading the issue. See {@link neverStarted} for the two shapes and why the
|