omp-conductor 0.15.12 → 0.16.0

package/src/escalate.ts

@@ -29,8 +29,79 @@ import { heldNoticeId } from "./notices.ts";
 import type { OrchestratorHandle } from "./orchestrator.ts";
 import type { Escalation, ProjectConfig, Store, Tracker } from "./types.ts";
 
-/** Telegram rejects `sendMessage` over 4096 chars; leave room for the marker. */
-const TELEGRAM_TEXT_LIMIT = 4000;
+/**
+ * Telegram rejects `sendMessage` over 4096 characters. Longer text is split
+ * into labelled parts ({@link telegramTextParts}) instead of truncated, so the
+ * tail of a digest or a tier-2 escalation arrives instead of being silently
+ * dropped (#566).
+ */
+export const TELEGRAM_TEXT_LIMIT = 4096;
+
+/**
+ * Width reserved for the `(i/n)` label prepended to every part of a split
+ * message, ported from omp-telegram's `PART_LABEL_RESERVE`. Parts are re-split
+ * against `TELEGRAM_TEXT_LIMIT - TELEGRAM_PART_LABEL_RESERVE` so the label can
+ * never push a part past the wire limit.
+ */
+export const TELEGRAM_PART_LABEL_RESERVE = 16;
+
+/**
+ * Split text into Telegram-sized parts. Ports omp-telegram 0.12.1's
+ * `chunkLabeled` semantics (the plugin this transport bypasses, #566) so a
+ * digest copies the same readability contract that plugin ships:
+ *
+ * - newline-preferred boundaries: a cut prefers the last paragraph break
+ *   (`\n\n`), then line break, then the last space, each only when it lands in
+ *   the second half of the window so parts stay large; an unsplittable run is
+ *   hard-cut rather than dropped.
+ * - numbered labels: when a message takes more than one part, every part is
+ *   prefixed `(i/N)\n` so a reader sees the message continues and knows how
+ *   many parts to expect. Under the label budget, so no part exceeds the limit.
+ * - lossless: a split message reassembles byte-for-byte to its input
+ *   (strip the labels and concatenate). Deliberately *not* ported from
+ *   upstream: fence repair and leading-newline stripping are rendering
+ *   flourishes for markdown-mode sends, and this transport sends plain text
+ *   (`parse_mode` is never set) — both would rewrite bytes the operator never
+ *   wrote.
+ *
+ * Text at or under the limit is returned whole and untouched, so the
+ * single-message path keeps sending byte-identical payloads.
+ */
+export function telegramTextParts(text: string, limit: number = TELEGRAM_TEXT_LIMIT): string[] {
+  if (text.length === 0 || text.length <= limit) return [text];
+  // `splitToLimit` returns at least two pieces here (text is over the limit);
+  // the guard is for future-proofing against a limit rewrite.
+  const parts = splitToLimit(text, Math.max(1, limit));
+  if (parts.length === 1) return parts;
+  // Re-split under the label budget so `(i/N)\n` never pushes a part past the
+  // wire limit.
+  const labelled = splitToLimit(text, Math.max(1, limit - TELEGRAM_PART_LABEL_RESERVE));
+  return labelled.map((part, index) => `(${index + 1}/${labelled.length})\n${part}`);
+}
+
+/**
+ * The base splitter, ported from omp-telegram's `splitToLimit`. Prefers a
+ * paragraph break, then a line break, then a space — each only when it sits in
+ * the second half of the window, so a digest splits on natural boundaries
+ * rather than mid-word. Never drops a byte: the fallback is a hard cut at
+ * `limit`, which split a run with no whitespace in the window in the middle,
+ * and the parts still reassemble to the input.
+ */
+function splitToLimit(text: string, limit: number): string[] {
+  if (text.length <= limit) return [text];
+  const out: string[] = [];
+  let rest = text;
+  while (rest.length > limit) {
+    const para = rest.lastIndexOf("\n\n", limit);
+    const line = rest.lastIndexOf("\n", limit);
+    const space = rest.lastIndexOf(" ", limit);
+    const cut = para > limit / 2 ? para : line > limit / 2 ? line : space > 0 ? space : limit;
+    out.push(rest.slice(0, cut));
+    rest = rest.slice(cut);
+  }
+  out.push(rest);
+  return out;
+}
 
 /**
  * Whether a failed send settles the question of delivery. This distinction is
@@ -144,6 +215,7 @@ export function createEscalator(
   orchestrator?: OrchestratorHandle,
   now: () => number = Date.now,
   deliveryAllowed: () => boolean = () => true,
+  onTier1Diverted?: (e: Escalation) => void,
 ): Escalator {
   const currentProject = typeof source === "function" ? source : (): ProjectConfig => source;
   return {
@@ -278,6 +350,13 @@ export function createEscalator(
       // Tier 1 lands here with no orchestrator, or with one that would not take
       // the injection; tier 2 lands here when omp-telegram is not installed or
       // the project never configured a chat id.
+      //
+      // A tier-1 event reaching this point was diverted from the orchestrator to
+      // an issue comment. That is the exact condition the down incident counts:
+      // the daemon hooks this to accumulate the orchestrator-down incident's
+      // diverted tally (a reader that owns its own dedupe and no-ops for a
+      // healthy or external orchestrator).
+      if (e.tier === 1) onTier1Diverted?.(e);
       if (!p.escalation.fallbackToIssueComment) {
         throw new Error(
           `no escalation transport configured for project "${p.name}": tier ${e.tier} ` +
@@ -463,26 +542,39 @@ export function claimedTelegramTopics(stateDir?: string): ClaimedTopic[] {
  * The one Telegram send in this package. Exported so the report outbox (#123)
  * reuses it rather than forking it: the response handling below is load-bearing
  * and was paid for once already (see the comment on the parse). Resolves with
- * Telegram's own message id when it returns one, and throws on every *known*
- * failure — connection refused, HTTP error, `{"ok":false}` — which is what lets
- * a caller treat a throw as "nobody has this" and a crash as "nobody knows".
+ * Telegram's own message id for every accepted part, in order — one id for a
+ * message under the limit, one per message once a split is needed. Throws on
+ * every *known* failure — connection refused, HTTP error, `{"ok":false}` —
+ * which is what lets a caller treat a throw as "nobody has this" and a crash
+ * as "nobody knows".
+ *
+ * Splitting happens here, at the transport, so every message class that rides
+ * this seam — tier-2 escalations, report parts, fleet pages, operator
+ * messages — is delivered whole: a report's tail is the fleet state and open
+ * items, and a tier-2 escalation's tail may be the question itself (#566).
+ * Parts are sent strictly in order, one `sendMessage` at a time, so they never
+ * interleave with each other.
  *
  * Optional `topicId` pins the message to a forum topic (`message_thread_id`).
- * A definitive missing-thread reject retries once as a flat chat and warns, so a
- * deleted topic degrades instead of silently losing the page (#318).
+ * A definitive missing-thread reject retries once as a flat chat and warns, so
+ * a deleted topic degrades instead of silently losing the page (#318). The
+ * flat retry re-sends the whole split: parts already accepted went into a
+ * thread Telegram has just told us is gone, so nothing is readable there that
+ * a flat resend would duplicate.
  */
 export async function sendTelegram(
   token: string,
   chatId: string,
   text: string,
   opts?: { topicId?: number },
-): Promise<number | undefined> {
+): Promise<number[]> {
   const topicId =
     opts?.topicId !== undefined && Number.isFinite(opts.topicId) && Number.isSafeInteger(opts.topicId)
       ? opts.topicId
       : undefined;
+  const parts = telegramTextParts(text);
   try {
-    return await postTelegramMessage(token, chatId, text, topicId);
+    return await postTelegramParts(token, chatId, parts, topicId);
   } catch (err) {
     if (
       topicId === undefined ||
@@ -495,8 +587,23 @@ export async function sendTelegram(
     warn(
       `escalation.telegramTopicId=${topicId} is stale (${err.message}); retrying flat chat`,
     );
-    return await postTelegramMessage(token, chatId, text, undefined);
+    return await postTelegramParts(token, chatId, parts, undefined);
+  }
+}
+
+/** One part per `sendMessage`, in order, collecting the ids Telegram returns. */
+async function postTelegramParts(
+  token: string,
+  chatId: string,
+  parts: readonly string[],
+  topicId: number | undefined,
+): Promise<number[]> {
+  const ids: number[] = [];
+  for (const part of parts) {
+    const id = await postTelegramMessage(token, chatId, part, topicId);
+    if (id !== undefined) ids.push(id);
   }
+  return ids;
 }
 
 /** Telegram's definitive "that forum topic is gone" answers. */
@@ -513,11 +620,11 @@ async function postTelegramMessage(
   const url = `https://api.telegram.org/bot${token}/sendMessage`;
   const payload: Record<string, unknown> = {
     chat_id: chatId,
-    // ponytail: hard truncation rather than splitting across messages — the
-    // tail of a stack trace is rarely the interesting part. Upgrade path is to
-    // attach the overflow as a file via sendDocument.
-    text:
-      text.length > TELEGRAM_TEXT_LIMIT ? `${text.slice(0, TELEGRAM_TEXT_LIMIT)}\n[truncated]` : text,
+    // `sendTelegram` splits long text into labelled parts above this call, so
+    // the text handed here is always a whole message: a part under the wire
+    // limit, or a message that never needed splitting. Nothing is truncated —
+    // a hard slice here is how the digest lost its tail (#566).
+    text,
     disable_web_page_preview: true,
   };
   if (topicId !== undefined) payload.message_thread_id = topicId;

package/src/failure-class.ts

@@ -157,6 +157,35 @@ export function providerTransientFault(error: { status?: number; message: string
   return error.message.split("\n")[0]?.trim() ?? error.message;
 }
 
+/**
+ * How many in-session provider 429s read as *sustained* rate limiting rather
+ * than a retried blip (#573).
+ *
+ * A single 429 the harness retried and recovered from is noise — counting it
+ * would reclassify every run that ever hit one throttle as a provider fault
+ * and let real failures escape the failed-attempt budget. Only a run that kept
+ * hitting the throttle never gets clear of it, and that is the signature the
+ * classifier needs. Three matches the "three strikes" motif this file already
+ * uses (`DISPATCH_INFRA_MAX_STRIKES`, `PROVIDER_TRANSIENT_MAX_STRIKES`).
+ */
+const SUSTAINED_RATE_LIMIT_COUNT = 3;
+
+/**
+ * Evidence that this run drowned in in-session provider rate limiting, or
+ * `undefined` when it did not.
+ *
+ * Reads the count the worker recorded while the run was live — never a string
+ * match over the transcript, and never `lastError`: omp retried the 429s
+ * internally, so they never surfaced one `lastError`, which is exactly why the
+ * run was previously unclassifiable (#573). 429 is deliberately its own class:
+ * a generic 429 is rate limiting, a 402 is credit, a stream abort is
+ * transient — three failures with three remedies (#220).
+ */
+export function providerCapacityFault(count: number | undefined): string | undefined {
+  if (count === undefined || count < SUSTAINED_RATE_LIMIT_COUNT) return undefined;
+  return `${count} in-session provider rate limits (HTTP 429) — the harness retried and was exhausted`;
+}
+
 /**
  * Evidence that this run never started, or `undefined` when something did happen.
  *
@@ -300,6 +329,24 @@ export function classifyRun(
     }
   }
 
+  // Sustained in-session provider rate limiting: the run drowned in HTTP 429s
+  // the harness retried and never surfaced one as `lastError`, so neither the
+  // credit nor the transient branch sees it and it used to fall through to
+  // `unknown`, spending an implementation attempt on the provider's capacity
+  // (#573). Only the worker-recorded count — above a *sustained* threshold, so
+  // a single retried 429 on an otherwise healthy run stays untouched — reads as
+  // this. Distinct from the two provider classes on purpose and requeued free,
+  // bounded by its own strike cap in the daemon so a permanently throttled
+  // provider escalates instead of looping. Ahead of `neverStarted` for the same
+  // reason as the credit branch: a turn-0 run that immediately drowned in 429s
+  // would otherwise be absorbed by `env-start-failure` and lose its cause.
+  if (run.state === "failed" || run.state === "killed") {
+    const capacity = providerCapacityFault(run.provider429Count);
+    if (capacity !== undefined) {
+      return { cls: "provider-capacity", recovery: "requeue", evidence: capacity };
+    }
+  }
+
   // A run that never started is the most classifiable failure there is, and the
   // least deserving of an implementation attempt: the session did not get as far
   // as reading the issue. See {@link neverStarted} for the two shapes and why the