omp-conductor 0.15.13 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/REFERENCE.md +72 -2
  2. package/package.json +2 -1
  3. package/schema/config.schema.json +6 -0
  4. package/src/admission.ts +745 -0
  5. package/src/ask.ts +47 -0
  6. package/src/backups.ts +19 -7
  7. package/src/board.ts +1 -2
  8. package/src/briefs/orchestrator.md +62 -4
  9. package/src/cli.ts +26 -0
  10. package/src/commands/context.ts +3 -0
  11. package/src/commands/decision.ts +10 -1
  12. package/src/commands/doctor.ts +2 -0
  13. package/src/commands/message.ts +8 -1
  14. package/src/commands/restart.ts +15 -3
  15. package/src/commands/restore-db.ts +146 -0
  16. package/src/commands/stop.ts +24 -15
  17. package/src/commands/unfreeze.ts +56 -0
  18. package/src/commands/watch.ts +77 -0
  19. package/src/config-schema.ts +9 -0
  20. package/src/config.ts +24 -0
  21. package/src/daemon.ts +239 -530
  22. package/src/dashboard/server.ts +2 -1
  23. package/src/decisions.ts +32 -7
  24. package/src/depends-on.ts +73 -0
  25. package/src/doctor.ts +178 -5
  26. package/src/escalate.ts +114 -15
  27. package/src/failure-class.ts +47 -0
  28. package/src/fleet.ts +41 -410
  29. package/src/gitops.ts +86 -1
  30. package/src/log.ts +40 -0
  31. package/src/model-fallback.ts +3 -2
  32. package/src/omp-settings.ts +114 -0
  33. package/src/omp.ts +39 -0
  34. package/src/orchestrator-tick.ts +7 -1
  35. package/src/reports.ts +124 -12
  36. package/src/session-host.ts +6 -0
  37. package/src/setup-wizard.ts +36 -0
  38. package/src/setup.ts +58 -1
  39. package/src/status-render.ts +445 -0
  40. package/src/stop-provenance.ts +53 -0
  41. package/src/store.ts +352 -11
  42. package/src/types.ts +187 -4
  43. package/src/unblock.ts +1 -1
  44. package/src/upgrade-verify.ts +1 -1
  45. package/src/upgrade.ts +1 -2
  46. package/src/verbs/server.ts +25 -0
  47. package/src/worker.ts +162 -10
@@ -28,7 +28,8 @@ import { chmodSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync }
28
28
  import { join } from "node:path";
29
29
  import { loadConfig, stateDir } from "../config.ts";
30
30
  import { healthCheck, livingDaemon, type DaemonRecord } from "../lifecycle.ts";
31
- import { classifyDaemonProjectHealth, fleetLayers, type FleetLayers } from "../fleet.ts";
31
+ import { classifyDaemonProjectHealth, fleetLayers } from "../fleet.ts";
32
+ import type { FleetLayers } from "../status-render.ts";
32
33
  import { statusSnapshot, type StatusSnapshot } from "../daemon.ts";
33
34
  import { boardJson, boardSnapshotOnce } from "../board.ts";
34
35
  import { dbPath, openStore } from "../store.ts";
package/src/decisions.ts CHANGED
@@ -235,15 +235,40 @@ function age(since: number, now: number): string {
235
235
  * being a memory question. A row whose condition has come true is flagged, not
236
236
  * merely listed — that is the difference between a parked question and one to
237
237
  * act on now.
238
+ *
239
+ * The two kinds render under their own headings (#459). Only rows a human must
240
+ * answer — `kind === "question"`, whatever condition they carry — appear under
241
+ * the operator heading; a watch the orchestrator set for itself goes under the
242
+ * watch heading with no instruction to resolve it, so a fleet at rest behind
243
+ * GitHub's checks is never mistaken for a fleet that is waiting on its
244
+ * operator. A met watch still surfaces to the orchestrator with its note and
245
+ * the same `[CONDITION MET]` flag a met question gets.
238
246
  */
239
247
  export function formatDecisionDigest(open: readonly DecisionRecord[], now = Date.now()): string {
240
- if (open.length === 0) return "";
241
- const lines = [
242
- `Open operator decisions (${open.length}) — resolve or withdraw each with omp-conductor decision resolve|withdraw <id>:`,
243
- ];
244
- for (const d of open) {
245
- const flag = d.conditionMetAt === undefined ? "" : " [CONDITION MET act on this now]";
246
- lines.push(`- ${d.id} (${age(d.askedAt, now)}, blocks ${d.blocks ?? "nothing"})${flag}: ${d.question}`);
248
+ const questions = open.filter((d) => d.kind !== "watch");
249
+ const watches = open.filter((d) => d.kind === "watch");
250
+ const lines: string[] = [];
251
+ if (questions.length > 0) {
252
+ lines.push(
253
+ `Open operator decisions (${questions.length}) resolve or withdraw each with omp-conductor decision resolve|withdraw <id>:`,
254
+ );
255
+ for (const d of questions) {
256
+ lines.push(decisionLine(d, now));
257
+ }
258
+ }
259
+ if (watches.length > 0) {
260
+ lines.push(
261
+ `Watches (${watches.length}) — conditions the orchestrator set for itself; no operator action needed:`,
262
+ );
263
+ for (const d of watches) {
264
+ lines.push(decisionLine(d, now));
265
+ }
247
266
  }
248
267
  return lines.join("\n");
249
268
  }
269
+
270
+ /** One digest row: id, age, what it blocks, the met flag, and the note. */
271
+ function decisionLine(d: DecisionRecord, now: number): string {
272
+ const flag = d.conditionMetAt === undefined ? "" : " [CONDITION MET — act on this now]";
273
+ return `- ${d.id} (${age(d.askedAt, now)}, blocks ${d.blocks ?? "nothing"})${flag}: ${d.question}`;
274
+ }
@@ -0,0 +1,73 @@
1
+ /**
2
+ * The `Depends-on:` declaration parser (epic #321, slice #419).
3
+ *
4
+ * A candidate issue can declare same-repo prerequisites it must not be
5
+ * dispatched before: `Depends-on: #123` (repeatable, one or more per line).
6
+ * Admission reads every referenced issue's live tracker state and holds the
7
+ * candidate while any prerequisite is open, so a worker is never sent at work
8
+ * whose base is not yet merged.
9
+ *
10
+ * The parser is deliberately small and strict, the two halves of the contract:
11
+ * the *marker* is matched case-insensitively (so `DEPENDS-ON`, `Depends-On`,
12
+ * `depends-on` all declare), while the *references* are strict bare issue
13
+ * numbers (`#123`). A marker line that carries no strict reference — an empty
14
+ * `Depends-on:`, or a bare non-numeric `Depends-on: #abc` — is never a usable
15
+ * prerequisite, so it is reported as `malformed` for the grooming flag rather
16
+ * than guessed at, and the declaration it belongs to is ignored.
17
+ *
18
+ * Cross-repo references (a qualifying `owner/repo#n` or `repo#n` form) and
19
+ * graph cycles are later slices (#420/#421) and are deliberately out of scope
20
+ * here: only plain `#<n>` same-repo forms are resolved.
21
+ */
22
+
23
+ export interface DependsOnDecl {
24
+ /** Referenced prerequisite issue numbers, deduplicated, first-seen order. */
25
+ readonly refs: readonly number[];
26
+ /** Marker-bearing lines that carried no strict `#<n>` reference (e.g.
27
+ * `Depends-on: #abc`). Bounded to what the body actually said, for the
28
+ * grooming flag's detail. */
29
+ readonly malformed: readonly string[];
30
+ }
31
+
32
+ /**
33
+ * How a line is spelled: `depends-on`/`depends on`, optional whitespace, then
34
+ * a `:` or `=` separator. The marker is matched case-insensitively; the value
35
+ * (everything after the separator) is what the strict-reference rules apply
36
+ * to. Anchored at the start of the (trimmed) line so a prose mention of
37
+ * "Depends-on" mid-sentence is not a declaration.
38
+ */
39
+ const MARKER = /^depends[-\s]?on\s*[:=]\s*(.*)$/i;
40
+
41
+ /** A strict reference is a bare `#` followed by digits — no repo qualifier,
42
+ * no words. Everything else is not a prerequisite this slice resolves. */
43
+ const REF = /#(\d+)/g;
44
+
45
+ export function parseDependsOn(body: string): DependsOnDecl {
46
+ const refs: number[] = [];
47
+ const seen = new Set<number>();
48
+ const malformed: string[] = [];
49
+ for (const raw of body.split("\n")) {
50
+ const line = raw.trim();
51
+ if (line === "") continue;
52
+ const match = MARKER.exec(line);
53
+ if (match === null) continue;
54
+ const value = match[1] ?? "";
55
+ REF.lastIndex = 0;
56
+ const found = [...value.matchAll(REF)];
57
+ if (found.length === 0) {
58
+ // A marker line with no strict reference is a malformed declaration:
59
+ // never crash, never silently claim — the declaration is ignored and
60
+ // surfaced for grooming.
61
+ malformed.push(line);
62
+ continue;
63
+ }
64
+ for (const group of found) {
65
+ const n = Number(group[1]!);
66
+ if (Number.isSafeInteger(n) && !seen.has(n)) {
67
+ seen.add(n);
68
+ refs.push(n);
69
+ }
70
+ }
71
+ }
72
+ return { refs, malformed };
73
+ }
package/src/doctor.ts CHANGED
@@ -25,21 +25,38 @@
25
25
  * (the #399 boundary).
26
26
  */
27
27
 
28
- import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
28
+ import {
29
+ accessSync,
30
+ constants,
31
+ existsSync,
32
+ readdirSync,
33
+ readFileSync,
34
+ statSync,
35
+ } from "node:fs";
29
36
  import { spawnSync } from "node:child_process";
30
37
  import { homedir } from "node:os";
31
- import { join } from "node:path";
38
+ import { dirname, join } from "node:path";
32
39
  import type { Stats } from "node:fs";
33
40
  import { Database } from "bun:sqlite";
41
+ import { stringify } from "yaml";
34
42
 
35
- import { configBackupDir, configPath, findProject, loadConfig, resolveCaps, stateDir } from "./config.ts";
43
+ import {
44
+ configBackupDir,
45
+ configPath,
46
+ dbBackupDirFor,
47
+ findProject,
48
+ loadConfig,
49
+ resolveCaps,
50
+ stateDir,
51
+ } from "./config.ts";
52
+ import { ompSettingsOverlay, sessionRootDir } from "./omp-settings.ts";
36
53
  import {
37
54
  DEFAULT_HERDR_UNIT,
38
55
  probeTelegramHealth,
39
56
  resolveHerdrSessionWithBridge,
40
57
  telegramStateDir,
41
- type TelegramHealth,
42
58
  } from "./fleet.ts";
59
+ import type { TelegramHealth } from "./status-render.ts";
43
60
  import {
44
61
  herdrConductorPluginConfigDir,
45
62
  planHostRuntime,
@@ -49,7 +66,7 @@ import {
49
66
  totalConfiguredWorkers,
50
67
  } from "./setup-host.ts";
51
68
  import { checkTokenScopes, type ScopeCheck } from "./setup.ts";
52
- import { dbPath, LIVE_STATES } from "./store.ts";
69
+ import { DB_SNAPSHOT_STEM, dbPath, LIVE_STATES } from "./store.ts";
53
70
  import { telegramReportSend, type ReportSend } from "./reports.ts";
54
71
  import { fetchRateLimit, GhError, gh } from "./tracker/github.ts";
55
72
  import { repoSlugFor } from "./gitops.ts";
@@ -159,6 +176,14 @@ export interface DoctorDeps {
159
176
  uidOf?: (name: string) => number | undefined;
160
177
  /** `PRAGMA integrity_check` over the store, read-only. ok=true for "ok". */
161
178
  dbIntegrity?: (path: string) => { ok: boolean; detail?: string };
179
+ /** Whether the configured db-snapshot directory exists and is writable, so a
180
+ * snapshot can land. Injected so a test never chmods the host (#399). */
181
+ snapshotDirState?: (dir: string) => { exists: boolean; writable: boolean };
182
+ /** Whether the run-session root — where the omp settings overlay is
183
+ * materialised at dispatch (#537) — can hold a new `run-<id>` directory. A
184
+ * missing `sessions/` parent is not a fault (dispatch's recursive `mkdir`
185
+ * creates it), so the writable question falls back to the state dir itself. */
186
+ sessionRootState?: (root: string) => { exists: boolean; writable: boolean };
162
187
  /** The most recent completed runs for one project, newest first, limited. */
163
188
  recentRuns?: (project: string, limit: number) => RunSpendRow[];
164
189
  /** The bot health probe `status` uses (getMe etc.). */
@@ -286,6 +311,55 @@ function defaultDbIntegrity(path: string): { ok: boolean; detail?: string } {
286
311
  }
287
312
  }
288
313
 
314
+ /**
315
+ * Whether the configured db-snapshot directory exists and is writable. No
316
+ * write is performed — the check is stat + access, keeping doctor read-only.
317
+ * A directory that does not exist reads `writable: false` and lets the probe
318
+ * report it as "no snapshot yet" rather than inventing one, while a read-only
319
+ * or non-directory path reads as a genuine unwritable failure.
320
+ */
321
+ function defaultSnapshotDirState(dir: string): { exists: boolean; writable: boolean } {
322
+ try {
323
+ const st = statSync(dir);
324
+ if (!st.isDirectory()) return { exists: true, writable: false };
325
+ try {
326
+ accessSync(dir, constants.W_OK);
327
+ return { exists: true, writable: true };
328
+ } catch {
329
+ return { exists: true, writable: false };
330
+ }
331
+ } catch {
332
+ return { exists: false, writable: false };
333
+ }
334
+ }
335
+
336
+ /**
337
+ * Whether the run-session root can hold a new session directory. No write is
338
+ * performed — the same stat + access as {@link defaultSnapshotDirState}, which
339
+ * keeps `doctor` read-only. A missing `sessions/` parent is normal on a fresh
340
+ * host (dispatch's recursive `mkdir` creates it), so the writable question
341
+ * falls back to the state dir it would be created under; a state dir that
342
+ * cannot even be read is a genuine unwritable failure.
343
+ */
344
+ function defaultSessionRootState(root: string): { exists: boolean; writable: boolean } {
345
+ const writable = (dir: string): boolean => {
346
+ try {
347
+ const st = statSync(dir);
348
+ if (!st.isDirectory()) return false;
349
+ accessSync(dir, constants.W_OK);
350
+ return true;
351
+ } catch {
352
+ return false;
353
+ }
354
+ };
355
+ try {
356
+ const st = statSync(root);
357
+ return { exists: true, writable: st.isDirectory() && writable(root) };
358
+ } catch {
359
+ return { exists: false, writable: writable(dirname(root)) };
360
+ }
361
+ }
362
+
289
363
  /**
290
364
  * The most recent completed runs for one project, newest first — read on the
291
365
  * same read-only connection the integrity check opens, because the store's own
@@ -419,6 +493,57 @@ function dbProbe(probes: Probes): Finding {
419
493
  );
420
494
  }
421
495
 
496
+ /**
497
+ * The conductor.db snapshot store (#579): the configured dir is writable so a
498
+ * snapshot can land, and a snapshot at least as fresh as the store protects
499
+ * it. The store is the carrier of the verb ledger, the decision rows, the run
500
+ * rows and the material-event ledger; a store with no restorable copy has no
501
+ * recovery path, so absence is a warning and an unwritable configured dir a
502
+ * failure.
503
+ */
504
+ function dbBackupProbe(probes: Probes, cfg: ConductorConfig | undefined): Finding {
505
+ const dir = dbBackupDirFor(cfg);
506
+ const state = probes.snapshotDirState(dir);
507
+ if (state.exists && !state.writable) {
508
+ return failFinding(
509
+ "db-backup",
510
+ `db backup dir ${dir} is not writable — a conductor.db snapshot cannot land there`,
511
+ `make ${dir} writable by the conductor account, or set dbBackupDir in config to a writable directory`,
512
+ );
513
+ }
514
+ const snapshots = (() => {
515
+ try {
516
+ return readdirSync(dir).filter((name) => name.startsWith(DB_SNAPSHOT_STEM));
517
+ } catch {
518
+ return [];
519
+ }
520
+ })();
521
+ const store = probes.stat(dbPath());
522
+ if (store === undefined) {
523
+ return snapshots.length === 0
524
+ ? passFinding("db-backup", "no conductor.db yet — nothing to snapshot")
525
+ : passFinding("db-backup", `${snapshots.length} conductor.db snapshot(s) present`);
526
+ }
527
+ if (snapshots.length === 0) {
528
+ return warnFinding(
529
+ "db-backup",
530
+ "conductor.db has no snapshot — losing it loses the audit trail with no recovery path",
531
+ "take a conductor.db snapshot (the daemon's cadence files one; restore-db needs one to restore from)",
532
+ );
533
+ }
534
+ const newestMtime = snapshots
535
+ .map((name) => probes.stat(join(dir, name))?.mtimeMs ?? 0)
536
+ .reduce((max, m) => Math.max(max, m), 0);
537
+ if (store.mtimeMs > newestMtime + 60_000) {
538
+ return warnFinding(
539
+ "db-backup",
540
+ `conductor.db is newer than every snapshot (${snapshots.length} file(s)) — the newest writes were never snapshot`,
541
+ "take a fresh conductor.db snapshot so restore-db has the current state",
542
+ );
543
+ }
544
+ return passFinding("db-backup", `conductor.db snapshot is fresh (${snapshots.length} file(s))`);
545
+ }
546
+
422
547
  /** Every repo the config names: each project's tracker plus routing targets,
423
548
  * as GitHub slugs, deduplicated, in config order. */
424
549
  function configuredRepos(cfg: ConductorConfig | undefined): string[] {
@@ -754,6 +879,42 @@ export function isKnownTimezone(tz: string): boolean {
754
879
  }
755
880
  }
756
881
 
882
+ /**
883
+ * The fleet-owned omp settings overlay (#537): reports the *effective* overlay
884
+ * every worker session of the project will load — the `ompSettings` map plus
885
+ * the retry keys derived from `modelFallbacks`, rendered exactly as the
886
+ * materialiser writes them, never the raw config snippet `setup` would echo —
887
+ * and turns red when that overlay cannot land. The file is written at dispatch
888
+ * under `<stateDir>/sessions/run-<id>/`, so doctor cannot prove the write
889
+ * without writing: it probes the root the materialisation would land in with
890
+ * the same stat + access the db-backup probe uses, keeping the check
891
+ * read-only (#287). The session root is a host-wide fact, probed once and
892
+ * shared across the per-project findings.
893
+ */
894
+ function ompSettingsProbe(
895
+ project: ProjectConfig,
896
+ sessionRoot: { exists: boolean; writable: boolean },
897
+ ): Finding {
898
+ const overlay = ompSettingsOverlay(project);
899
+ if (overlay === undefined) {
900
+ return passFinding(
901
+ "omp-settings",
902
+ `[${project.name}] no omp settings overlay configured — workers inherit the daemon account's global settings unchanged`,
903
+ );
904
+ }
905
+ if (!sessionRoot.writable) {
906
+ return failFinding(
907
+ "omp-settings",
908
+ `[${project.name}] the omp settings overlay cannot be materialised: ${sessionRootDir()} cannot hold a session directory, so every worker would dispatch without its staged settings`,
909
+ `make ${sessionRootDir()} writable by the conductor account`,
910
+ );
911
+ }
912
+ return passFinding(
913
+ "omp-settings",
914
+ `[${project.name}] worker sessions load this omp settings overlay:\n${stringify(overlay).trimEnd()}`,
915
+ );
916
+ }
917
+
757
918
  /** The self-identified probe message `--probe-telegram` delivers through the
758
919
  * same report transport a real report would take. */
759
920
  function probeMessage(project: string, checkedAt: string): string {
@@ -1076,7 +1237,13 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
1076
1237
  // difference behind a constant count).
1077
1238
  findings.push(backupProbe(probes));
1078
1239
  findings.push(dbProbe(probes));
1240
+ findings.push(dbBackupProbe(probes, cfg));
1079
1241
  findings.push(await ghAuthProbe(probes, configuredRepos(cfg)));
1242
+ // The run-session root is one host-wide fact — where every run's session dir
1243
+ // (and the omp settings overlay inside it) lands — probed once and shared
1244
+ // across the per-project omp-settings findings.
1245
+ const sessionRoot = sessionRootDir();
1246
+ const sessionRootState = probes.sessionRootState(sessionRoot);
1080
1247
  // The collected run sample across the resolved project set, newest-first;
1081
1248
  // spend telemetry is a single host-wide finding, not one per project.
1082
1249
  const spendRows: RunSpendRow[] = [];
@@ -1093,10 +1260,14 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
1093
1260
  findings.push(
1094
1261
  passFinding("herdr-agent-name", projectProblem === undefined ? "no project resolved — nothing to check" : `agent name uncheckable: ${projectProblem}`),
1095
1262
  );
1263
+ findings.push(
1264
+ passFinding("omp-settings", projectProblem === undefined ? "no project resolved — nothing to check" : `omp settings overlay uncheckable: ${projectProblem}`),
1265
+ );
1096
1266
  } else {
1097
1267
  for (const p of projects) {
1098
1268
  findings.push(await labelProbe(probes, p));
1099
1269
  findings.push(herdrAgentNameProbe(probes, p));
1270
+ findings.push(ompSettingsProbe(p, sessionRootState));
1100
1271
  }
1101
1272
  }
1102
1273
  // The installed-unit check is host-global: one shared daemon. Any project
@@ -1149,6 +1320,8 @@ export function defaultProbes(): Probes {
1149
1320
  stat: defaultStat,
1150
1321
  uidOf: defaultUidOf,
1151
1322
  dbIntegrity: defaultDbIntegrity,
1323
+ snapshotDirState: defaultSnapshotDirState,
1324
+ sessionRootState: defaultSessionRootState,
1152
1325
  recentRuns: defaultRecentRuns,
1153
1326
  telegramHealth: (projectName) => probeTelegramHealth(projectName),
1154
1327
  telegramSend: telegramReportSend,
package/src/escalate.ts CHANGED
@@ -29,8 +29,79 @@ import { heldNoticeId } from "./notices.ts";
29
29
  import type { OrchestratorHandle } from "./orchestrator.ts";
30
30
  import type { Escalation, ProjectConfig, Store, Tracker } from "./types.ts";
31
31
 
32
- /** Telegram rejects `sendMessage` over 4096 chars; leave room for the marker. */
33
- const TELEGRAM_TEXT_LIMIT = 4000;
32
+ /**
33
+ * Telegram rejects `sendMessage` over 4096 characters. Longer text is split
34
+ * into labelled parts ({@link telegramTextParts}) instead of truncated, so the
35
+ * tail of a digest or a tier-2 escalation arrives instead of being silently
36
+ * dropped (#566).
37
+ */
38
+ export const TELEGRAM_TEXT_LIMIT = 4096;
39
+
40
+ /**
41
+ * Width reserved for the `(i/n)` label prepended to every part of a split
42
+ * message, ported from omp-telegram's `PART_LABEL_RESERVE`. Parts are re-split
43
+ * against `TELEGRAM_TEXT_LIMIT - TELEGRAM_PART_LABEL_RESERVE` so the label can
44
+ * never push a part past the wire limit.
45
+ */
46
+ export const TELEGRAM_PART_LABEL_RESERVE = 16;
47
+
48
+ /**
49
+ * Split text into Telegram-sized parts. Ports omp-telegram 0.12.1's
50
+ * `chunkLabeled` semantics (the plugin this transport bypasses, #566) so a
51
+ * digest copies the same readability contract that plugin ships:
52
+ *
53
+ * - newline-preferred boundaries: a cut prefers the last paragraph break
54
+ * (`\n\n`), then line break, then the last space, each only when it lands in
55
+ * the second half of the window so parts stay large; an unsplittable run is
56
+ * hard-cut rather than dropped.
57
+ * - numbered labels: when a message takes more than one part, every part is
58
+ * prefixed `(i/N)\n` so a reader sees the message continues and knows how
59
+ * many parts to expect. Under the label budget, so no part exceeds the limit.
60
+ * - lossless: a split message reassembles byte-for-byte to its input
61
+ * (strip the labels and concatenate). Deliberately *not* ported from
62
+ * upstream: fence repair and leading-newline stripping are rendering
63
+ * flourishes for markdown-mode sends, and this transport sends plain text
64
+ * (`parse_mode` is never set) — both would rewrite bytes the operator never
65
+ * wrote.
66
+ *
67
+ * Text at or under the limit is returned whole and untouched, so the
68
+ * single-message path keeps sending byte-identical payloads.
69
+ */
70
+ export function telegramTextParts(text: string, limit: number = TELEGRAM_TEXT_LIMIT): string[] {
71
+ if (text.length === 0 || text.length <= limit) return [text];
72
+ // `splitToLimit` returns at least two pieces here (text is over the limit);
73
+ // the guard is for future-proofing against a limit rewrite.
74
+ const parts = splitToLimit(text, Math.max(1, limit));
75
+ if (parts.length === 1) return parts;
76
+ // Re-split under the label budget so `(i/N)\n` never pushes a part past the
77
+ // wire limit.
78
+ const labelled = splitToLimit(text, Math.max(1, limit - TELEGRAM_PART_LABEL_RESERVE));
79
+ return labelled.map((part, index) => `(${index + 1}/${labelled.length})\n${part}`);
80
+ }
81
+
82
+ /**
83
+ * The base splitter, ported from omp-telegram's `splitToLimit`. Prefers a
84
+ * paragraph break, then a line break, then a space — each only when it sits in
85
+ * the second half of the window, so a digest splits on natural boundaries
86
+ * rather than mid-word. Never drops a byte: the fallback is a hard cut at
87
+ * `limit`, which split a run with no whitespace in the window in the middle,
88
+ * and the parts still reassemble to the input.
89
+ */
90
+ function splitToLimit(text: string, limit: number): string[] {
91
+ if (text.length <= limit) return [text];
92
+ const out: string[] = [];
93
+ let rest = text;
94
+ while (rest.length > limit) {
95
+ const para = rest.lastIndexOf("\n\n", limit);
96
+ const line = rest.lastIndexOf("\n", limit);
97
+ const space = rest.lastIndexOf(" ", limit);
98
+ const cut = para > limit / 2 ? para : line > limit / 2 ? line : space > 0 ? space : limit;
99
+ out.push(rest.slice(0, cut));
100
+ rest = rest.slice(cut);
101
+ }
102
+ out.push(rest);
103
+ return out;
104
+ }
34
105
 
35
106
  /**
36
107
  * Whether a failed send settles the question of delivery. This distinction is
@@ -471,26 +542,39 @@ export function claimedTelegramTopics(stateDir?: string): ClaimedTopic[] {
471
542
  * The one Telegram send in this package. Exported so the report outbox (#123)
472
543
  * reuses it rather than forking it: the response handling below is load-bearing
473
544
  * and was paid for once already (see the comment on the parse). Resolves with
474
- * Telegram's own message id when it returns one, and throws on every *known*
475
- * failure connection refused, HTTP error, `{"ok":false}` which is what lets
476
- * a caller treat a throw as "nobody has this" and a crash as "nobody knows".
545
+ * Telegram's own message id for every accepted part, in order one id for a
546
+ * message under the limit, one per message once a split is needed. Throws on
547
+ * every *known* failure connection refused, HTTP error, `{"ok":false}`
548
+ * which is what lets a caller treat a throw as "nobody has this" and a crash
549
+ * as "nobody knows".
550
+ *
551
+ * Splitting happens here, at the transport, so every message class that rides
552
+ * this seam — tier-2 escalations, report parts, fleet pages, operator
553
+ * messages — is delivered whole: a report's tail is the fleet state and open
554
+ * items, and a tier-2 escalation's tail may be the question itself (#566).
555
+ * Parts are sent strictly in order, one `sendMessage` at a time, so they never
556
+ * interleave with each other.
477
557
  *
478
558
  * Optional `topicId` pins the message to a forum topic (`message_thread_id`).
479
- * A definitive missing-thread reject retries once as a flat chat and warns, so a
480
- * deleted topic degrades instead of silently losing the page (#318).
559
+ * A definitive missing-thread reject retries once as a flat chat and warns, so
560
+ * a deleted topic degrades instead of silently losing the page (#318). The
561
+ * flat retry re-sends the whole split: parts already accepted went into a
562
+ * thread Telegram has just told us is gone, so nothing is readable there that
563
+ * a flat resend would duplicate.
481
564
  */
482
565
  export async function sendTelegram(
483
566
  token: string,
484
567
  chatId: string,
485
568
  text: string,
486
569
  opts?: { topicId?: number },
487
- ): Promise<number | undefined> {
570
+ ): Promise<number[]> {
488
571
  const topicId =
489
572
  opts?.topicId !== undefined && Number.isFinite(opts.topicId) && Number.isSafeInteger(opts.topicId)
490
573
  ? opts.topicId
491
574
  : undefined;
575
+ const parts = telegramTextParts(text);
492
576
  try {
493
- return await postTelegramMessage(token, chatId, text, topicId);
577
+ return await postTelegramParts(token, chatId, parts, topicId);
494
578
  } catch (err) {
495
579
  if (
496
580
  topicId === undefined ||
@@ -503,8 +587,23 @@ export async function sendTelegram(
503
587
  warn(
504
588
  `escalation.telegramTopicId=${topicId} is stale (${err.message}); retrying flat chat`,
505
589
  );
506
- return await postTelegramMessage(token, chatId, text, undefined);
590
+ return await postTelegramParts(token, chatId, parts, undefined);
591
+ }
592
+ }
593
+
594
+ /** One part per `sendMessage`, in order, collecting the ids Telegram returns. */
595
+ async function postTelegramParts(
596
+ token: string,
597
+ chatId: string,
598
+ parts: readonly string[],
599
+ topicId: number | undefined,
600
+ ): Promise<number[]> {
601
+ const ids: number[] = [];
602
+ for (const part of parts) {
603
+ const id = await postTelegramMessage(token, chatId, part, topicId);
604
+ if (id !== undefined) ids.push(id);
507
605
  }
606
+ return ids;
508
607
  }
509
608
 
510
609
  /** Telegram's definitive "that forum topic is gone" answers. */
@@ -521,11 +620,11 @@ async function postTelegramMessage(
521
620
  const url = `https://api.telegram.org/bot${token}/sendMessage`;
522
621
  const payload: Record<string, unknown> = {
523
622
  chat_id: chatId,
524
- // ponytail: hard truncation rather than splitting across messages the
525
- // tail of a stack trace is rarely the interesting part. Upgrade path is to
526
- // attach the overflow as a file via sendDocument.
527
- text:
528
- text.length > TELEGRAM_TEXT_LIMIT ? `${text.slice(0, TELEGRAM_TEXT_LIMIT)}\n[truncated]` : text,
623
+ // `sendTelegram` splits long text into labelled parts above this call, so
624
+ // the text handed here is always a whole message: a part under the wire
625
+ // limit, or a message that never needed splitting. Nothing is truncated —
626
+ // a hard slice here is how the digest lost its tail (#566).
627
+ text,
529
628
  disable_web_page_preview: true,
530
629
  };
531
630
  if (topicId !== undefined) payload.message_thread_id = topicId;
@@ -157,6 +157,35 @@ export function providerTransientFault(error: { status?: number; message: string
157
157
  return error.message.split("\n")[0]?.trim() ?? error.message;
158
158
  }
159
159
 
160
+ /**
161
+ * How many in-session provider 429s read as *sustained* rate limiting rather
162
+ * than a retried blip (#573).
163
+ *
164
+ * A single 429 the harness retried and recovered from is noise — counting it
165
+ * would reclassify every run that ever hit one throttle as a provider fault
166
+ * and let real failures escape the failed-attempt budget. Only a run that kept
167
+ * hitting the throttle never gets clear of it, and that is the signature the
168
+ * classifier needs. Three matches the "three strikes" motif this file already
169
+ * uses (`DISPATCH_INFRA_MAX_STRIKES`, `PROVIDER_TRANSIENT_MAX_STRIKES`).
170
+ */
171
+ const SUSTAINED_RATE_LIMIT_COUNT = 3;
172
+
173
+ /**
174
+ * Evidence that this run drowned in in-session provider rate limiting, or
175
+ * `undefined` when it did not.
176
+ *
177
+ * Reads the count the worker recorded while the run was live — never a string
178
+ * match over the transcript, and never `lastError`: omp retried the 429s
179
+ * internally, so they never surfaced one `lastError`, which is exactly why the
180
+ * run was previously unclassifiable (#573). 429 is deliberately its own class:
181
+ * a generic 429 is rate limiting, a 402 is credit, a stream abort is
182
+ * transient — three failures with three remedies (#220).
183
+ */
184
+ export function providerCapacityFault(count: number | undefined): string | undefined {
185
+ if (count === undefined || count < SUSTAINED_RATE_LIMIT_COUNT) return undefined;
186
+ return `${count} in-session provider rate limits (HTTP 429) — the harness retried and was exhausted`;
187
+ }
188
+
160
189
  /**
161
190
  * Evidence that this run never started, or `undefined` when something did happen.
162
191
  *
@@ -300,6 +329,24 @@ export function classifyRun(
300
329
  }
301
330
  }
302
331
 
332
+ // Sustained in-session provider rate limiting: the run drowned in HTTP 429s
333
+ // the harness retried and never surfaced one as `lastError`, so neither the
334
+ // credit nor the transient branch sees it and it used to fall through to
335
+ // `unknown`, spending an implementation attempt on the provider's capacity
336
+ // (#573). Only the worker-recorded count — above a *sustained* threshold, so
337
+ // a single retried 429 on an otherwise healthy run stays untouched — reads as
338
+ // this. Distinct from the two provider classes on purpose and requeued free,
339
+ // bounded by its own strike cap in the daemon so a permanently throttled
340
+ // provider escalates instead of looping. Ahead of `neverStarted` for the same
341
+ // reason as the credit branch: a turn-0 run that immediately drowned in 429s
342
+ // would otherwise be absorbed by `env-start-failure` and lose its cause.
343
+ if (run.state === "failed" || run.state === "killed") {
344
+ const capacity = providerCapacityFault(run.provider429Count);
345
+ if (capacity !== undefined) {
346
+ return { cls: "provider-capacity", recovery: "requeue", evidence: capacity };
347
+ }
348
+ }
349
+
303
350
  // A run that never started is the most classifiable failure there is, and the
304
351
  // least deserving of an implementation attempt: the session did not get as far
305
352
  // as reading the issue. See {@link neverStarted} for the two shapes and why the