omp-conductor 0.3.25 → 0.4.0

package/src/escalate.ts

@@ -31,6 +31,68 @@ import type { Escalation, ProjectConfig, Store, Tracker } from "./types.ts";
 /** Telegram rejects `sendMessage` over 4096 chars; leave room for the marker. */
 const TELEGRAM_TEXT_LIMIT = 4000;
 
+/**
+ * Whether a failed send settles the question of delivery. This distinction is
+ * the whole of #123's honesty: both classes retry, but only one of them may be
+ * producing a *second* copy of a message Telegram already accepted, and only
+ * that one is allowed to say so.
+ *
+ * - `definitive` — Telegram answered and did not accept the message, or the
+ *   request provably never left this host. Nobody has it. A retry is a first
+ *   attempt, not a possible duplicate.
+ * - `unknown` — the request left and its fate was never learned. Telegram may
+ *   be holding the message. A retry may duplicate, and must carry that warning.
+ */
+export type TelegramSendOutcome = "definitive" | "unknown";
+
+/** A send failure that knows which of the two it is. */
+export class TelegramSendError extends Error {
+  readonly outcome: TelegramSendOutcome;
+  constructor(message: string, outcome: TelegramSendOutcome) {
+    super(message);
+    this.name = "TelegramSendError";
+    this.outcome = outcome;
+  }
+}
+
+/**
+ * Error codes that prove no request was ever put on the wire. Bun reports both
+ * a refused connection and a DNS failure as `ConnectionRefused`; the `E*` codes
+ * are what Node/undici produce, kept so this classification does not silently
+ * invert if the runtime underneath ever changes.
+ */
+const NEVER_CONNECTED: ReadonlySet<string> = new Set([
+  "ConnectionRefused",
+  "FailedToOpenSocket",
+  "ECONNREFUSED",
+  "ENOTFOUND",
+  "EAI_AGAIN",
+  "EHOSTUNREACH",
+  "ENETUNREACH",
+]);
+
+/**
+ * Classifies a `fetch` rejection, defaulting to `unknown`.
+ *
+ * The default is the entire point and must not be "tidied" into the simpler
+ * branch: an aborted request, a socket reset or an `EPIPE` all happen *after*
+ * the bytes went out, and Telegram may well have accepted the message before
+ * the connection died. Guessing `definitive` there re-sends it as though
+ * nothing had happened, which is how a delivered page is posted twice with
+ * nothing saying so. Guessing `unknown` costs a possible-repeat marker on a
+ * message that was never sent — a cosmetic wrong, traded against a silent
+ * duplicate. When in doubt, doubt out loud.
+ */
+export function telegramFailureOutcome(cause: unknown): TelegramSendOutcome {
+  let error: unknown = cause;
+  for (let depth = 0; error instanceof Error && depth < 4; depth += 1) {
+    const code: unknown = (error as { code?: unknown }).code;
+    if (typeof code === "string" && NEVER_CONNECTED.has(code)) return "definitive";
+    error = error.cause;
+  }
+  return "unknown";
+}
+
 export interface Escalator {
   escalate(e: Escalation): Promise<void>;
 }
@@ -152,9 +214,17 @@ export function createEscalator(
         const token = readTelegramToken();
         if (token) {
           // A send failure throws: `markNotified` stays uncalled so the next
-          // poll retries instead of writing the event off as delivered.
-          // ponytail: no retry/backoff in here — the dispatcher tick is the
-          // retry. Upgrade path is a durable outbox table in the store.
+          // poll retries instead of writing the event off as delivered. No
+          // backoff in here — the dispatcher tick *is* the retry, and an
+          // escalation the loop keeps rediscovering is re-derived from the
+          // world rather than replayed from a queue.
+          //
+          // #123 built the durable outbox this comment used to name as the
+          // upgrade path, and deliberately did not move escalations into it:
+          // the condition that raises one is still true on the next tick, so
+          // the tick re-raises it. A *report* has no such source — it exists
+          // once, in the model's head, and nothing regenerates it — which is
+          // why that one needed a ledger and this one does not.
           await sendTelegram(token, chatId, text);
           store.markNotified(key);
           return;
@@ -207,8 +277,12 @@ function errText(e: unknown): string {
  * omp-telegram owns `<state dir>/.env`; conductor only borrows the token, so a
  * user already running that bot gets tier-2 pings with no extra configuration.
  * Absence is not an error — it just means tier 2 degrades to the fallback.
+ *
+ * Exported for the report outbox (#123), which pages over the same bot and must
+ * resolve the token the same way. Two readers of one `.env` is fine; two
+ * *implementations* of the parse below is how they drift.
  */
-function readTelegramToken(): string | undefined {
+export function readTelegramToken(): string | undefined {
   const override = process.env.OMP_TELEGRAM_STATE_DIR?.trim();
   const dir = override ? override : join(homedir(), ".omp", "agent", "telegram");
   let raw: string;
@@ -232,7 +306,19 @@ function readTelegramToken(): string | undefined {
   return undefined;
 }
 
-async function sendTelegram(token: string, chatId: string, text: string): Promise<void> {
+/**
+ * The one Telegram send in this package. Exported so the report outbox (#123)
+ * reuses it rather than forking it: the response handling below is load-bearing
+ * and was paid for once already (see the comment on the parse). Resolves with
+ * Telegram's own message id when it returns one, and throws on every *known*
+ * failure — connection refused, HTTP error, `{"ok":false}` — which is what lets
+ * a caller treat a throw as "nobody has this" and a crash as "nobody knows".
+ */
+export async function sendTelegram(
+  token: string,
+  chatId: string,
+  text: string,
+): Promise<number | undefined> {
   const url = `https://api.telegram.org/bot${token}/sendMessage`;
   const body = JSON.stringify({
     chat_id: chatId,
@@ -252,7 +338,10 @@ async function sendTelegram(token: string, chatId: string, text: string): Promis
     // at you — redact before this string reaches a log or an issue comment.
     const nested = cause instanceof Error && cause.cause instanceof Error ? `: ${cause.cause.message}` : "";
     const reason = cause instanceof Error ? `${cause.message}${nested}` : String(cause);
-    throw new Error(`telegram sendMessage failed: ${redact(reason, token)}`);
+    throw new TelegramSendError(
+      `telegram sendMessage failed: ${redact(reason, token)}`,
+      telegramFailureOutcome(cause),
+    );
   }
 
   // The raw body is what gets parsed; `diagnostic` is only ever for humans.
@@ -264,24 +353,62 @@ async function sendTelegram(token: string, chatId: string, text: string): Promis
   // written on success, that also re-sent the same page every tick. Redaction
   // stays out of the parse for the same reason: it rewrites the very bytes the
   // decision is read from.
-  const raw = await res.text().catch(() => "");
+  let raw: string;
+  let bodyUnread: unknown;
+  try {
+    raw = await res.text();
+  } catch (cause) {
+    raw = "";
+    bodyUnread = cause;
+  }
   const diagnostic = redact(raw, token).slice(0, 400);
   if (!res.ok) {
-    throw new Error(`telegram sendMessage failed: HTTP ${res.status} ${diagnostic}`);
+    throw new TelegramSendError(
+      `telegram sendMessage failed: HTTP ${res.status} ${diagnostic}`,
+      "definitive",
+    );
+  }
+  // A 200 whose body never arrived is the one case where the status reads as
+  // encouraging and proves nothing: Telegram had already decided, and the answer
+  // was lost coming back. Reported as `unknown` rather than rejected so the
+  // retry carries a possible-repeat marker instead of quietly posting a second
+  // copy of a message that very probably landed. Checked after the status,
+  // because a non-ok status settles the question on its own.
+  if (bodyUnread !== undefined) {
+    const reason = bodyUnread instanceof Error ? bodyUnread.message : String(bodyUnread);
+    throw new TelegramSendError(
+      `telegram sendMessage left but its response body could not be read: ${redact(reason, token)}`,
+      "unknown",
+    );
   }
 
   // Telegram answers 200 with `{"ok":false}` for plenty of real failures
   // (kicked from the chat, bad chat_id), so the status alone proves nothing.
   let ok = false;
+  let messageId: number | undefined;
   try {
     const parsed: unknown = JSON.parse(raw);
     ok = typeof parsed === "object" && parsed !== null && "ok" in parsed && parsed.ok === true;
+    // Read from the same parse the ok/not-ok decision comes from, and only from
+    // the body — the outbox records this id as the durable proof of *which*
+    // attempt landed, and this transport offers no other reconciliation. A
+    // missing or odd `result` is not a delivery failure: `ok: true` is the
+    // verdict, the id is a courtesy, and inventing one would be worse than
+    // recording none.
+    if (typeof parsed === "object" && parsed !== null && "result" in parsed) {
+      const result: unknown = parsed.result;
+      if (typeof result === "object" && result !== null && "message_id" in result) {
+        const id: unknown = result.message_id;
+        if (typeof id === "number" && Number.isSafeInteger(id)) messageId = id;
+      }
+    }
   } catch {
     ok = false;
   }
   if (!ok) {
-    throw new Error(`telegram sendMessage rejected: ${diagnostic}`);
+    throw new TelegramSendError(`telegram sendMessage rejected: ${diagnostic}`, "definitive");
   }
+  return messageId;
 }
 
 /**

package/src/fleet.ts

@@ -27,11 +27,17 @@ import {
 import { createInterface } from "node:readline";
 import { homedir } from "node:os";
 import { dirname, join } from "node:path";
-import { findProject, loadConfig, stateDir } from "./config.ts";
+import { findProject, loadConfig, resolveCredentials, stateDir } from "./config.ts";
+import { describeBoundary, probeHost } from "./credentials.ts";
+import { planUsageLine, readPlanUsage, sharedUsageSource } from "./usage.ts";
 import { readApprovalSurface } from "./approval-surface.ts";
+import { confinementRefusalsToday, type ConfinementRefusalSummary } from "./confinement.ts";
+import { settlementFlagSummary } from "./diff-flags.ts";
 import { probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
+import { formatOpenReports } from "./reports.ts";
 import {
   formatDispatchSummary,
+  formatReleaseGrants,
   formatSalvagedRuns,
   isPaused,
   setPaused,
@@ -930,6 +936,16 @@ export function formatCodeGraphHealth(graph: CodeGraphHealth, now = Date.now()):
   ].join("\n");
 }
 
+/**
+ * The credential boundary row (#125): which mechanism is actually live, and the
+ * residuals it does *not* close. Passed in rather than probed inside the
+ * formatter so the wording is testable without a host.
+ */
+export interface BoundaryStatus {
+  headline: string;
+  detail: string[];
+}
+
 export function formatFleetStatus(
   s: StatusSnapshot,
   layers: FleetLayers,
@@ -937,6 +953,8 @@ export function formatFleetStatus(
   telegram: TelegramHealth = { kind: "unprobed" },
   now = Date.now(),
   codeGraph: CodeGraphHealth = { configured: false },
+  refusals: ConfinementRefusalSummary | undefined = undefined,
+  boundary: BoundaryStatus | undefined = undefined,
 ): string {
   const tickLine =
     layers.ticksDetail === undefined
@@ -989,6 +1007,24 @@ export function formatFleetStatus(
   }
 
   const graphBlock = formatCodeGraphHealth(codeGraph, now);
+  // Silent when there were none: a line reading "0" every day is one nobody
+  // reads on the day it says 40. A repeatedly-refused orchestrator is
+  // misbriefed, and that is the operator's problem to see (#127).
+  const confinementLine =
+    refusals === undefined
+      ? undefined
+      : `confine   ${refusals.count} orchestrator refusal(s) today  ` +
+        `(latest: ${refusals.latest.tool} ${refusals.latest.path} — ${refusals.latest.kind})`;
+  // Reported every time, never only when it is bad. An operator reading this
+  // has to be able to see "unprotected" on the day they assumed otherwise, and
+  // a line that appears only in the failure case is one whose absence means
+  // nothing (#125). `undefined` here is a status rendered without a host probe,
+  // which is itself worth saying rather than silently omitting.
+  const boundaryLines =
+    boundary === undefined
+      ? ["boundary  unprobed  (no host capability probe was run for this status)"]
+      : [`boundary  ${boundary.headline}`, ...boundary.detail.map((d) => `          ${d}`)];
+
   return [
     `dispatch  ${layers.dispatch}`,
     tickLine,
@@ -997,6 +1033,8 @@ export function formatFleetStatus(
     recoveryLine,
     herdrLine,
     telegramLine,
+    ...boundaryLines,
+    ...(confinementLine === undefined ? [] : [confinementLine]),
     ...(graphBlock === undefined ? [] : [graphBlock]),
     daemonBlock,
     "",
@@ -1016,11 +1054,17 @@ function formatProjectBody(s: StatusSnapshot): string {
     s.caps.dailySpendUsd === null
       ? `  spend today        $${s.spendTodayUsd.toFixed(2)} (no daily cap)`
       : `  spend today        $${s.spendTodayUsd.toFixed(2)} / $${s.caps.dailySpendUsd.toFixed(2)}`,
+    // Its own row beside the spend row, never folded into it: they are two
+    // independent controls and an operator has to see which one stopped the
+    // fleet (#110).
+    `  plan usage         ${planUsageLine(s.planUsage)}`,
     `  new worker turns   ${s.caps.workerMaxTurns}`,
     `  worker wall clock  ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
     `  failed attempts    ${s.caps.maxAttemptsPerIssue}`,
     `  continuations      ${s.caps.maxContinuationsPerIssue}`,
     "",
+    ...formatReleaseGrants(s.releaseGrants),
+    "",
     formatDispatchSummary(s.dispatch),
     "",
   ];
@@ -1034,9 +1078,15 @@ function formatProjectBody(s: StatusSnapshot): string {
           `${r.turns}/${r.maxTurns} turns  $${r.spendUsd.toFixed(2)}  ${r.branch}` +
           (r.prUrl ? `  ${r.prUrl}` : ""),
       );
+      // The orchestrator's Duty 1 reads this command, and a flagged run's
+      // escalation is deduplicated after one delivery — so this is where a
+      // flagged PR stays visible for as long as it is still open (#128).
+      const flagged = settlementFlagSummary(r.settlementFlags);
+      if (flagged !== undefined) lines.push(`      ${flagged}`);
     }
   }
   lines.push(...formatSalvagedRuns(s.salvagedRuns));
+  lines.push(...formatOpenReports(s.openReports));
   if (s.liveWorkers > 0) {
     lines.push(
       "",
@@ -1052,13 +1102,31 @@ export async function renderStatus(projectName?: string): Promise<string> {
   const layers = fleetLayers(projectName);
   const project = findProject(loadConfig(), projectName);
   const rec = livingDaemon();
-  const [health, telegram] = await Promise.all([
+  const [health, telegram, planUsage] = await Promise.all([
     rec === undefined ? undefined : healthCheck(rec.port),
     probeTelegramHealth(projectName),
+    // Read here rather than in `statusSnapshot`, which is synchronous and used
+    // by callers that must not shell out. An unmetered project never spawns
+    // the provider at all.
+    readPlanUsage(s.caps.planUsage, sharedUsageSource()),
   ]);
   const cached = codeGraphFromHealthz(health?.body, project.name);
   const codeGraph = cached ?? (await probeCodeGraph(project));
-  return formatFleetStatus(s, layers, health, telegram, Date.now(), codeGraph);
+  // Probed here, per `status` call, rather than read off the daemon: an
+  // operator asking whether their fleet is protected must get the answer for
+  // the host they are standing on, including when no daemon is running.
+  const credentials = resolveCredentials(project);
+  const probe = await probeHost({ slots: s.caps.maxConcurrentWorkers });
+  return formatFleetStatus(
+    { ...s, planUsage },
+    layers,
+    health,
+    telegram,
+    Date.now(),
+    codeGraph,
+    confinementRefusalsToday(),
+    describeBoundary(credentials.isolation, probe),
+  );
 }
 
 // ---------------------------------------------------------------------------