omp-conductor 0.15.13 → 0.16.1

package/src/escalate.ts

@@ -29,8 +29,79 @@ import { heldNoticeId } from "./notices.ts";
 import type { OrchestratorHandle } from "./orchestrator.ts";
 import type { Escalation, ProjectConfig, Store, Tracker } from "./types.ts";
 
-/** Telegram rejects `sendMessage` over 4096 chars; leave room for the marker. */
-const TELEGRAM_TEXT_LIMIT = 4000;
+/**
+ * Telegram rejects `sendMessage` over 4096 characters. Longer text is split
+ * into labelled parts ({@link telegramTextParts}) instead of truncated, so the
+ * tail of a digest or a tier-2 escalation arrives instead of being silently
+ * dropped (#566).
+ */
+export const TELEGRAM_TEXT_LIMIT = 4096;
+
+/**
+ * Width reserved for the `(i/n)` label prepended to every part of a split
+ * message, ported from omp-telegram's `PART_LABEL_RESERVE`. Parts are re-split
+ * against `TELEGRAM_TEXT_LIMIT - TELEGRAM_PART_LABEL_RESERVE` so the label can
+ * never push a part past the wire limit.
+ */
+export const TELEGRAM_PART_LABEL_RESERVE = 16;
+
+/**
+ * Split text into Telegram-sized parts. Ports omp-telegram 0.12.1's
+ * `chunkLabeled` semantics (the plugin this transport bypasses, #566) so a
+ * digest copies the same readability contract that plugin ships:
+ *
+ * - newline-preferred boundaries: a cut prefers the last paragraph break
+ *   (`\n\n`), then line break, then the last space, each only when it lands in
+ *   the second half of the window so parts stay large; an unsplittable run is
+ *   hard-cut rather than dropped.
+ * - numbered labels: when a message takes more than one part, every part is
+ *   prefixed `(i/N)\n` so a reader sees the message continues and knows how
+ *   many parts to expect. Under the label budget, so no part exceeds the limit.
+ * - lossless: a split message reassembles byte-for-byte to its input
+ *   (strip the labels and concatenate). Deliberately *not* ported from
+ *   upstream: fence repair and leading-newline stripping are rendering
+ *   flourishes for markdown-mode sends, and this transport sends plain text
+ *   (`parse_mode` is never set) — both would rewrite bytes the operator never
+ *   wrote.
+ *
+ * Text at or under the limit is returned whole and untouched, so the
+ * single-message path keeps sending byte-identical payloads.
+ */
+export function telegramTextParts(text: string, limit: number = TELEGRAM_TEXT_LIMIT): string[] {
+  if (text.length === 0 || text.length <= limit) return [text];
+  // `splitToLimit` returns at least two pieces here (text is over the limit);
+  // the guard is for future-proofing against a limit rewrite.
+  const parts = splitToLimit(text, Math.max(1, limit));
+  if (parts.length === 1) return parts;
+  // Re-split under the label budget so `(i/N)\n` never pushes a part past the
+  // wire limit.
+  const labelled = splitToLimit(text, Math.max(1, limit - TELEGRAM_PART_LABEL_RESERVE));
+  return labelled.map((part, index) => `(${index + 1}/${labelled.length})\n${part}`);
+}
+
+/**
+ * The base splitter, ported from omp-telegram's `splitToLimit`. Prefers a
+ * paragraph break, then a line break, then a space — each only when it sits in
+ * the second half of the window, so a digest splits on natural boundaries
+ * rather than mid-word. Never drops a byte: the fallback is a hard cut at
+ * `limit`, which split a run with no whitespace in the window in the middle,
+ * and the parts still reassemble to the input.
+ */
+function splitToLimit(text: string, limit: number): string[] {
+  if (text.length <= limit) return [text];
+  const out: string[] = [];
+  let rest = text;
+  while (rest.length > limit) {
+    const para = rest.lastIndexOf("\n\n", limit);
+    const line = rest.lastIndexOf("\n", limit);
+    const space = rest.lastIndexOf(" ", limit);
+    const cut = para > limit / 2 ? para : line > limit / 2 ? line : space > 0 ? space : limit;
+    out.push(rest.slice(0, cut));
+    rest = rest.slice(cut);
+  }
+  out.push(rest);
+  return out;
+}
 
 /**
  * Whether a failed send settles the question of delivery. This distinction is
@@ -365,6 +436,13 @@ export interface ClaimedTopic {
   name: string;
   /** The herdr space the claiming pane sits in, when the bridge captured one. */
   workspaceLabel?: string;
+  /**
+   * The transcript the claiming pane actually writes, when the bridge captured
+   * one. This is the *live* session file — herdr pins a restored pane to it
+   * (`omp --resume=<that path>`) — so it can sit in a different session
+   * directory than the pane's cwd implies (#600).
+   */
+  sessionFile?: string;
 }
 
 /**
@@ -399,17 +477,47 @@ export function resolveProjectTopicId(project: ProjectConfig): number | undefine
   const claims = claimedTelegramTopics();
   if (claims.length === 0) return pinned;
   if (claims.some((claim) => claim.threadId === pinned)) return pinned;
-  const bySpace = soleClaim(claims, (claim) => claim.workspaceLabel === project.name);
-  const match = bySpace ?? soleClaim(claims, (claim) => claim.name === project.name);
+  const match = claimForProject(claims, project.name);
   if (match === undefined) return pinned;
   warn(
     `escalation.telegramTopicId for ${project.name} is no longer a claimed topic; ` +
       `following omp-telegram's current "${project.name}" ` +
-      `${bySpace === undefined ? "claim" : "herdr space claim"} instead`,
+      `${match.workspaceLabel === project.name ? "herdr space claim" : "claim"} instead`,
   );
   return match.threadId;
 }
 
+/**
+ * The one claim answering to a project's identity — herdr space first, title
+ * second — or nothing when none or several do (#412).
+ */
+export function claimForProject(
+  claims: readonly ClaimedTopic[],
+  projectName: string,
+): ClaimedTopic | undefined {
+  const bySpace = soleClaim(claims, (claim) => claim.workspaceLabel === projectName);
+  return bySpace ?? soleClaim(claims, (claim) => claim.name === projectName);
+}
+
+/**
+ * The transcript this project's live omp-telegram claim names — where the
+ * orchestrator pane's reply actually lands.
+ *
+ * `arm` derives its scan directory from the tick cwd, but a pane resumed from
+ * a session created elsewhere keeps the original transcript path (herdr pins
+ * the pane to it), so the cwd-derived directory is the one place the reply is
+ * guaranteed *not* to be (#600). The claim's `sessionFile` is the live answer;
+ * identity resolves exactly as {@link resolveProjectTopicId}'s substitution
+ * does, via {@link claimForProject}. Absent when the bridge has no claim or
+ * the claim names no file — the caller then falls back to the cwd-derived
+ * directory, which is the honest answer for a host that predates claims.
+ */
+export function resolveClaimedSessionFile(project: ProjectConfig): string | undefined {
+  const claims = claimedTelegramTopics();
+  if (claims.length === 0) return undefined;
+  return claimForProject(claims, project.name)?.sessionFile;
+}
+
 /**
  * The one claim answering to `predicate`, or nothing when none or several do.
  *
@@ -452,6 +560,7 @@ export function claimedTelegramTopics(stateDir?: string): ClaimedTopic[] {
     if (!Number.isFinite(threadId) || !Number.isSafeInteger(threadId)) continue;
     let name = id;
     let workspaceLabel: string | undefined;
+    let sessionFile: string | undefined;
     if (typeof entry === "object" && entry !== null) {
       if ("name" in entry) {
         const candidate = entry.name;
@@ -461,36 +570,84 @@ export function claimedTelegramTopics(stateDir?: string): ClaimedTopic[] {
         const candidate = entry.workspaceLabel;
         if (typeof candidate === "string" && candidate.trim() !== "") workspaceLabel = candidate.trim();
       }
+      if ("sessionFile" in entry) {
+        const candidate = entry.sessionFile;
+        if (typeof candidate === "string" && candidate.trim() !== "") sessionFile = candidate.trim();
+      }
     }
-    out.push(workspaceLabel === undefined ? { threadId, name } : { threadId, name, workspaceLabel });
+    out.push(
+      workspaceLabel === undefined && sessionFile === undefined
+        ? { threadId, name }
+        : workspaceLabel === undefined
+          ? { threadId, name, sessionFile }
+          : sessionFile === undefined
+            ? { threadId, name, workspaceLabel }
+            : { threadId, name, workspaceLabel, sessionFile },
+    );
   }
   return out;
 }
 
+/**
+ * Whether the paired bridge runs with per-session topic tidy on (access.json
+ * `topicsTidy: true` — `/telegram topics tidy on`).
+ *
+ * On a tidy host every pane exit closes (forum) or deletes (DM) its own topic,
+ * so a pinned `escalation.telegramTopicId` is structurally stale: it can never
+ * be among the live claims again, and treating it as a live pin is the false
+ * belief #600 names. The bridge's own setting is read here so `doctor` can say
+ * "vestigial" rather than "moved once".
+ */
+export function telegramTopicsTidy(stateDirPath?: string): boolean {
+  const override = stateDirPath?.trim() ?? process.env.OMP_TELEGRAM_STATE_DIR?.trim();
+  const dir = override ? override : join(homedir(), ".omp", "agent", "telegram");
+  try {
+    const parsed = JSON.parse(readFileSync(join(dir, "access.json"), "utf8")) as {
+      readonly topicsTidy?: unknown;
+    };
+    return parsed.topicsTidy === true;
+  } catch {
+    return false;
+  }
+}
+
 /**
  * The one Telegram send in this package. Exported so the report outbox (#123)
  * reuses it rather than forking it: the response handling below is load-bearing
  * and was paid for once already (see the comment on the parse). Resolves with
- * Telegram's own message id when it returns one, and throws on every *known*
- * failure — connection refused, HTTP error, `{"ok":false}` — which is what lets
- * a caller treat a throw as "nobody has this" and a crash as "nobody knows".
+ * Telegram's own message id for every accepted part, in order — one id for a
+ * message under the limit, one per message once a split is needed. Throws on
+ * every *known* failure — connection refused, HTTP error, `{"ok":false}` —
+ * which is what lets a caller treat a throw as "nobody has this" and a crash
+ * as "nobody knows".
+ *
+ * Splitting happens here, at the transport, so every message class that rides
+ * this seam — tier-2 escalations, report parts, fleet pages, operator
+ * messages — is delivered whole: a report's tail is the fleet state and open
+ * items, and a tier-2 escalation's tail may be the question itself (#566).
+ * Parts are sent strictly in order, one `sendMessage` at a time, so they never
+ * interleave with each other.
  *
  * Optional `topicId` pins the message to a forum topic (`message_thread_id`).
- * A definitive missing-thread reject retries once as a flat chat and warns, so a
- * deleted topic degrades instead of silently losing the page (#318).
+ * A definitive missing-thread reject retries once as a flat chat and warns, so
+ * a deleted topic degrades instead of silently losing the page (#318). The
+ * flat retry re-sends the whole split: parts already accepted went into a
+ * thread Telegram has just told us is gone, so nothing is readable there that
+ * a flat resend would duplicate.
  */
 export async function sendTelegram(
   token: string,
   chatId: string,
   text: string,
   opts?: { topicId?: number },
-): Promise<number | undefined> {
+): Promise<number[]> {
   const topicId =
     opts?.topicId !== undefined && Number.isFinite(opts.topicId) && Number.isSafeInteger(opts.topicId)
       ? opts.topicId
       : undefined;
+  const parts = telegramTextParts(text);
   try {
-    return await postTelegramMessage(token, chatId, text, topicId);
+    return await postTelegramParts(token, chatId, parts, topicId);
   } catch (err) {
     if (
       topicId === undefined ||
@@ -503,8 +660,23 @@ export async function sendTelegram(
     warn(
       `escalation.telegramTopicId=${topicId} is stale (${err.message}); retrying flat chat`,
     );
-    return await postTelegramMessage(token, chatId, text, undefined);
+    return await postTelegramParts(token, chatId, parts, undefined);
+  }
+}
+
+/** One part per `sendMessage`, in order, collecting the ids Telegram returns. */
+async function postTelegramParts(
+  token: string,
+  chatId: string,
+  parts: readonly string[],
+  topicId: number | undefined,
+): Promise<number[]> {
+  const ids: number[] = [];
+  for (const part of parts) {
+    const id = await postTelegramMessage(token, chatId, part, topicId);
+    if (id !== undefined) ids.push(id);
   }
+  return ids;
 }
 
 /** Telegram's definitive "that forum topic is gone" answers. */
@@ -521,11 +693,11 @@ async function postTelegramMessage(
   const url = `https://api.telegram.org/bot${token}/sendMessage`;
   const payload: Record<string, unknown> = {
     chat_id: chatId,
-    // ponytail: hard truncation rather than splitting across messages — the
-    // tail of a stack trace is rarely the interesting part. Upgrade path is to
-    // attach the overflow as a file via sendDocument.
-    text:
-      text.length > TELEGRAM_TEXT_LIMIT ? `${text.slice(0, TELEGRAM_TEXT_LIMIT)}\n[truncated]` : text,
+    // `sendTelegram` splits long text into labelled parts above this call, so
+    // the text handed here is always a whole message: a part under the wire
+    // limit, or a message that never needed splitting. Nothing is truncated —
+    // a hard slice here is how the digest lost its tail (#566).
+    text,
     disable_web_page_preview: true,
   };
   if (topicId !== undefined) payload.message_thread_id = topicId;

package/src/failure-class.ts

@@ -157,6 +157,35 @@ export function providerTransientFault(error: { status?: number; message: string
   return error.message.split("\n")[0]?.trim() ?? error.message;
 }
 
+/**
+ * How many in-session provider 429s read as *sustained* rate limiting rather
+ * than a retried blip (#573).
+ *
+ * A single 429 the harness retried and recovered from is noise — counting it
+ * would reclassify every run that ever hit one throttle as a provider fault
+ * and let real failures escape the failed-attempt budget. Only a run that kept
+ * hitting the throttle never gets clear of it, and that is the signature the
+ * classifier needs. Three matches the "three strikes" motif this file already
+ * uses (`DISPATCH_INFRA_MAX_STRIKES`, `PROVIDER_TRANSIENT_MAX_STRIKES`).
+ */
+const SUSTAINED_RATE_LIMIT_COUNT = 3;
+
+/**
+ * Evidence that this run drowned in in-session provider rate limiting, or
+ * `undefined` when it did not.
+ *
+ * Reads the count the worker recorded while the run was live — never a string
+ * match over the transcript, and never `lastError`: omp retried the 429s
+ * internally, so they never surfaced one `lastError`, which is exactly why the
+ * run was previously unclassifiable (#573). 429 is deliberately its own class:
+ * a generic 429 is rate limiting, a 402 is credit, a stream abort is
+ * transient — three failures with three remedies (#220).
+ */
+export function providerCapacityFault(count: number | undefined): string | undefined {
+  if (count === undefined || count < SUSTAINED_RATE_LIMIT_COUNT) return undefined;
+  return `${count} in-session provider rate limits (HTTP 429) — the harness retried and was exhausted`;
+}
+
 /**
  * Evidence that this run never started, or `undefined` when something did happen.
  *
@@ -300,6 +329,24 @@ export function classifyRun(
     }
   }
 
+  // Sustained in-session provider rate limiting: the run drowned in HTTP 429s
+  // the harness retried and never surfaced one as `lastError`, so neither the
+  // credit nor the transient branch sees it and it used to fall through to
+  // `unknown`, spending an implementation attempt on the provider's capacity
+  // (#573). Only the worker-recorded count — above a *sustained* threshold, so
+  // a single retried 429 on an otherwise healthy run stays untouched — reads as
+  // this. Distinct from the two provider classes on purpose and requeued free,
+  // bounded by its own strike cap in the daemon so a permanently throttled
+  // provider escalates instead of looping. Ahead of `neverStarted` for the same
+  // reason as the credit branch: a turn-0 run that immediately drowned in 429s
+  // would otherwise be absorbed by `env-start-failure` and lose its cause.
+  if (run.state === "failed" || run.state === "killed") {
+    const capacity = providerCapacityFault(run.provider429Count);
+    if (capacity !== undefined) {
+      return { cls: "provider-capacity", recovery: "requeue", evidence: capacity };
+    }
+  }
+
   // A run that never started is the most classifiable failure there is, and the
   // least deserving of an implementation attempt: the session did not get as far
   // as reading the issue. See {@link neverStarted} for the two shapes and why the