mercury-agent 0.23.2 → 0.24.0

package/src/core/operator-alerts.ts

@@ -40,6 +40,7 @@ import {
   formatSystemMessage,
   type MessageLocale,
   resolveLocale,
+  type SystemMessageKey,
 } from "./system-messages.js";
 
 /**
@@ -94,17 +95,29 @@ export interface MissedRunsAlert {
   name: string | null;
   /** Who created the task — the alert's first-choice recipient. */
   createdBy: string;
-  /** How many occurrences were skipped. Always >= 1. */
+  /**
+   * How many occurrences the host slept through. May be 0 when the report is
+   * only about an interrupted run; `missed + interrupted` is always >= 1.
+   */
   missed: number;
+  /**
+   * 1 when the occurrence at `oldest` was not missed but *interrupted*: it
+   * started and a restart, crash or host death cut it off. Not a failure — the
+   * report says "interrupted by a restart" rather than "missed" for it.
+   */
+  interrupted: 0 | 1;
   /**
    * True when the count is a floor rather than a total — the scheduler's walk
    * hit its scan limit. Rendered as "at least N" so a lower bound is never
    * presented as a fact.
    */
   capped: boolean;
-  /** Epoch ms of the oldest missed occurrence. */
+  /**
+   * Epoch ms of the oldest occurrence that did not run to completion — the
+   * interrupted one when `interrupted` is 1, else the oldest missed one.
+   */
   oldest: number;
-  /** Epoch ms of the newest missed occurrence. */
+  /** Epoch ms of the newest occurrence that did not run to completion. */
   newest: number;
   /** The task's timezone, so the alert names times the way the task is written. */
   timezone: string | null;
@@ -512,6 +525,58 @@ export function createOperatorAlerter(
     newest: formatOccurrence(alert.newest, alert.timezone),
   });
 
+  /**
+   * Which sentence a report gets. Three shapes: only missed (the host slept
+   * through it), only interrupted (a restart cut the run off — not a failure,
+   * and not "missed"), or both (cut off, then down for more occurrences).
+   * Each in a long form and a list line, each by policy.
+   */
+  const missedRunKey = (
+    alert: MissedRunsAlert,
+    form: "single" | "line",
+  ): SystemMessageKey => {
+    const caughtUp = alert.policy === "caught-up";
+    const shape =
+      alert.interrupted === 0
+        ? "missed"
+        : alert.missed === 0
+          ? "interrupted"
+          : "interrupted_then_missed";
+    const keys: Record<
+      typeof shape,
+      Record<typeof form, [SystemMessageKey, SystemMessageKey]>
+    > = {
+      missed: {
+        single: ["alert_missed_runs_skipped", "alert_missed_runs_caught_up"],
+        line: [
+          "alert_missed_runs_line_skipped",
+          "alert_missed_runs_line_caught_up",
+        ],
+      },
+      interrupted: {
+        single: [
+          "alert_interrupted_run_skipped",
+          "alert_interrupted_run_caught_up",
+        ],
+        line: [
+          "alert_missed_runs_line_interrupted_skipped",
+          "alert_missed_runs_line_interrupted_caught_up",
+        ],
+      },
+      interrupted_then_missed: {
+        single: [
+          "alert_interrupted_then_missed_skipped",
+          "alert_interrupted_then_missed_caught_up",
+        ],
+        line: [
+          "alert_missed_runs_line_interrupted_then_missed_skipped",
+          "alert_missed_runs_line_interrupted_then_missed_caught_up",
+        ],
+      },
+    };
+    return keys[shape][form][caughtUp ? 1 : 0];
+  };
+
   /**
    * One message for everything this recipient lost. A single task keeps the
    * long form, which has room to explain the policy; several become a list,
@@ -527,9 +592,7 @@ export function createOperatorAlerter(
     if (reports.length === 1) {
       return formatSystemMessage(
         locale,
-        first.policy === "caught-up"
-          ? "alert_missed_runs_caught_up"
-          : "alert_missed_runs_skipped",
+        missedRunKey(first, "single"),
         missedRunFields(first, locale),
       );
     }
@@ -537,9 +600,7 @@ export function createOperatorAlerter(
       .map((report) =>
         formatSystemMessage(
           locale,
-          report.policy === "caught-up"
-            ? "alert_missed_runs_line_caught_up"
-            : "alert_missed_runs_line_skipped",
+          missedRunKey(report, "line"),
           missedRunFields(report, locale),
         ),
       )
@@ -809,13 +870,22 @@ export function createOperatorAlerter(
         for (const report of reports) {
           const recipient = sender ? resolveMissedRunRecipient(report) : null;
           if (!recipient) {
+            const what = [
+              report.interrupted ? "had a run interrupted by a restart" : "",
+              report.missed > 0
+                ? `missed ${report.capped ? "at least " : ""}${report.missed} scheduled run(s)`
+                : "",
+            ]
+              .filter(Boolean)
+              .join(" and ");
             logger.error(
-              `Missed ${report.capped ? "at least " : ""}${report.missed} scheduled run(s) of task ${report.taskId} and could not tell anyone — ${sender ? "neither the task's creator nor its space has a conversation to deliver into" : "no message sender is bound yet"}. The only record is this log line.`,
+              `Task ${report.taskId} ${what} and could not tell anyone — ${sender ? "neither the task's creator nor its space has a conversation to deliver into" : "no message sender is bound yet"}. The only record is this log line.`,
               {
                 taskId: report.taskId,
                 spaceId: report.spaceId,
                 createdBy: report.createdBy,
                 missed: report.missed,
+                interrupted: report.interrupted,
                 capped: report.capped,
                 oldest: new Date(report.oldest).toISOString(),
                 newest: new Date(report.newest).toISOString(),

package/src/core/pending-ingress.ts

@@ -0,0 +1,298 @@
+/**
+ * Debounced chat messages that survive a restart.
+ *
+ * `MessageDebouncer` holds inbound messages in memory until its idle timer
+ * fires, and the user message is first written to `messages` only when the
+ * run starts. A restart used to drop every held message with no trace at all:
+ * the sender believed the bot saw it, and the bot never did.
+ *
+ * Flushing them at shutdown cannot work — the queue is cancelled and the
+ * containers killed before shutdown hooks run, inside a 10 s budget — so they
+ * are **persisted** at shutdown (`pending_ingress`, one row per debounce key)
+ * and **replayed** by the next boot through the handler's live dispatch path.
+ *
+ * The boot side is the shape of every other kind of interrupted work (see
+ * `boot-reconcile.ts`): {@link pendingIngressReconciler} runs on the one boot
+ * pass, before any adapter, and drops what is too old or unreadable;
+ * {@link replayPendingIngress} runs after the adapters and hands each live row
+ * to its handler. The row stays in the table between the two, so a crash in
+ * that window replays it on the next boot instead of losing it.
+ */
+
+import { logger } from "../logger.js";
+import type { Db, PendingIngressRecord } from "../storage/db.js";
+import type { IngressMessage } from "../types.js";
+import type { BootReconciler, InterruptedWork } from "./boot-reconcile.js";
+
+export const PENDING_INGRESS_KIND = "pending_ingress";
+
+export type PendingReason =
+  /** Waiting in a debounce batch for its idle timer. */
+  | "batch"
+  /** Parked behind an in-flight run for the same key. */
+  | "queued-behind-run"
+  /** Reached the handler after shutdown had begun. */
+  | "arrived-during-shutdown";
+
+const PENDING_REASONS: ReadonlySet<string> = new Set<PendingReason>([
+  "batch",
+  "queued-behind-run",
+  "arrived-during-shutdown",
+]);
+
+export interface PendingIngressRow {
+  id: number;
+  platform: string;
+  /** For logs and operator queries; replay re-derives the key from the messages. */
+  debounceKey: string;
+  spaceId: string;
+  reason: PendingReason;
+  /** In arrival order, NOT merged. */
+  ingresses: IngressMessage[];
+  savedAt: number;
+}
+
+/** Saved longer ago than this at boot → dropped and logged, not replayed. */
+export const PENDING_INGRESS_MAX_AGE_MS = 15 * 60_000; // owner decision 2026-09-22
+
+export function encodePendingIngress(
+  row: Omit<PendingIngressRow, "id">,
+): Omit<PendingIngressRecord, "id"> {
+  return {
+    platform: row.platform,
+    debounceKey: row.debounceKey,
+    spaceId: row.spaceId,
+    reason: row.reason,
+    ingressesJson: JSON.stringify(row.ingresses),
+    savedAt: row.savedAt,
+  };
+}
+
+function isIngress(value: unknown): value is IngressMessage {
+  if (typeof value !== "object" || value === null) return false;
+  const m = value as Record<string, unknown>;
+  return (
+    typeof m.platform === "string" &&
+    typeof m.spaceId === "string" &&
+    typeof m.threadId === "string" &&
+    m.threadId.length > 0 &&
+    typeof m.conversationExternalId === "string" &&
+    typeof m.callerId === "string" &&
+    typeof m.text === "string" &&
+    typeof m.isDM === "boolean" &&
+    typeof m.isReplyToBot === "boolean" &&
+    Array.isArray(m.attachments)
+  );
+}
+
+/**
+ * The stored row, or null when it cannot be replayed safely. Every message
+ * must carry its own thread (the reply target is read off the message, never
+ * off the row) and all of them must belong to the row's platform and space.
+ */
+export function decodePendingIngress(
+  record: PendingIngressRecord,
+): PendingIngressRow | null {
+  if (!PENDING_REASONS.has(record.reason)) return null;
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(record.ingressesJson);
+  } catch {
+    return null;
+  }
+  if (!Array.isArray(parsed) || parsed.length === 0) return null;
+  if (!parsed.every(isIngress)) return null;
+  const ingresses = parsed as IngressMessage[];
+  if (
+    !ingresses.every(
+      (m) =>
+        m.platform === record.platform &&
+        m.spaceId === record.spaceId &&
+        m.threadId === ingresses[0].threadId &&
+        m.callerId === ingresses[0].callerId,
+    )
+  ) {
+    return null;
+  }
+  return {
+    id: record.id,
+    platform: record.platform,
+    debounceKey: record.debounceKey,
+    spaceId: record.spaceId,
+    reason: record.reason as PendingReason,
+    ingresses,
+    savedAt: record.savedAt,
+  };
+}
+
+export interface PendingIngressReplayer {
+  replay(row: PendingIngressRow): void;
+}
+
+/** The `detail` of a `pending_ingress` item — counts and keys, never text. */
+export interface PendingIngressDetail {
+  platform: string;
+  key: string;
+  reason: PendingReason;
+  messages: number;
+}
+
+function preview(record: PendingIngressRecord): string {
+  return record.ingressesJson.slice(0, 80);
+}
+
+/**
+ * Boot pass: every row a previous process saved (`savedAt < bootAt`) is either
+ * dropped — unreadable (error), or saved more than
+ * {@link PENDING_INGRESS_MAX_AGE_MS} before this boot (warn: the chat has moved
+ * on and a late answer would be noise) — or returned as a live item. Live rows
+ * are left in the table; {@link replayPendingIngress} claims them.
+ */
+export const pendingIngressReconciler: BootReconciler = {
+  name: PENDING_INGRESS_KIND,
+  reconcile({ db, bootAt }) {
+    const live: InterruptedWork[] = [];
+    let stale = 0;
+    let malformed = 0;
+    for (const record of db.listPendingIngress()) {
+      if (record.savedAt >= bootAt) continue;
+      const row = decodePendingIngress(record);
+      if (!row) {
+        if (db.claimPendingIngress(record.id)) {
+          malformed++;
+          logger.error("Boot: pending ingress row is malformed; dropped", {
+            id: record.id,
+            key: record.debounceKey,
+            reason: record.reason,
+            preview: preview(record),
+          });
+        }
+        continue;
+      }
+      const ageMs = bootAt - row.savedAt;
+      if (ageMs > PENDING_INGRESS_MAX_AGE_MS) {
+        if (db.claimPendingIngress(row.id)) {
+          stale++;
+          logger.warn("Boot: pending ingress too old to answer; dropped", {
+            key: row.debounceKey,
+            reason: row.reason,
+            count: row.ingresses.length,
+            ageMs,
+            textPreview: row.ingresses[0].text.slice(0, 80),
+          });
+        }
+        continue;
+      }
+      live.push({
+        kind: PENDING_INGRESS_KIND,
+        id: row.id,
+        spaceId: row.spaceId,
+        startedAt: row.savedAt,
+        detail: {
+          platform: row.platform,
+          key: row.debounceKey,
+          reason: row.reason,
+          messages: row.ingresses.length,
+        } satisfies PendingIngressDetail,
+      });
+    }
+    if (stale > 0 || malformed > 0) {
+      logger.info("Boot: pending ingress dropped", { stale, malformed });
+    }
+    return live;
+  },
+};
+
+export interface PendingIngressReplayCounts {
+  replayed: number;
+  /** No handler for the row's platform: kept, aged out by a later boot. */
+  unowned: number;
+  /** A shadow: nothing replayed, rows untouched. */
+  skipped: number;
+  /** Row gone or unreadable by the time of replay (claimed elsewhere). */
+  missing: number;
+  /** Claimed, and the handler threw: the one loss, logged in full. */
+  failed: number;
+}
+
+/**
+ * After the adapters: hand each live `pending_ingress` item back to the
+ * handler for its platform, oldest first. A row is dispatched only after
+ * `claimPendingIngress` returned true, so it is answered at most once.
+ *
+ * `shadow` is the literal `config.shadow`: a shadow replays nothing and leaves
+ * every row in place.
+ */
+export function replayPendingIngress(opts: {
+  items: readonly InterruptedWork[];
+  db: Db;
+  handlers: ReadonlyMap<string, PendingIngressReplayer>;
+  shadow: boolean;
+}): PendingIngressReplayCounts {
+  const { db, handlers, shadow } = opts;
+  const items = opts.items.filter((i) => i.kind === PENDING_INGRESS_KIND);
+  const counts: PendingIngressReplayCounts = {
+    replayed: 0,
+    unowned: 0,
+    skipped: 0,
+    missing: 0,
+    failed: 0,
+  };
+
+  if (shadow) {
+    counts.skipped = items.length;
+    logger.info("Boot: pending ingress", { ...counts, shadow: true });
+    return counts;
+  }
+
+  for (const item of items) {
+    const record = db.getPendingIngress(item.id);
+    const row = record ? decodePendingIngress(record) : null;
+    if (!row) {
+      counts.missing++;
+      logger.warn("Boot: pending ingress row vanished before replay", {
+        id: item.id,
+        spaceId: item.spaceId,
+      });
+      continue;
+    }
+
+    const handler = handlers.get(row.platform);
+    if (!handler) {
+      counts.unowned++;
+      logger.warn("Boot: pending ingress for an unconfigured platform; kept", {
+        platform: row.platform,
+        key: row.debounceKey,
+        count: row.ingresses.length,
+      });
+      continue;
+    }
+
+    if (!db.claimPendingIngress(row.id)) {
+      counts.missing++;
+      logger.info("Boot: pending ingress already claimed elsewhere", {
+        id: row.id,
+        key: row.debounceKey,
+      });
+      continue;
+    }
+    try {
+      handler.replay(row);
+      counts.replayed++;
+    } catch (err) {
+      // Claimed and not handed over: the one place a message can still be
+      // lost, so it is named in full.
+      counts.failed++;
+      logger.error("Boot: pending ingress replay threw; messages lost", {
+        key: row.debounceKey,
+        reason: row.reason,
+        count: row.ingresses.length,
+        textPreview: row.ingresses[0].text.slice(0, 80),
+        error: err instanceof Error ? err.message : String(err),
+      });
+    }
+  }
+
+  logger.info("Boot: pending ingress", counts);
+  return counts;
+}

package/src/core/routes/dashboard.ts

@@ -50,6 +50,16 @@ const SKIP_REASON_LABEL: Record<SkipReason, string> = {
   "no-index": "no index.ts",
 };
 
+/**
+ * The two ledger states a run can be in without having an outcome. One
+ * rendering each, shared by the task list and the run history, so the two
+ * cannot drift into saying different things about the same row.
+ */
+const RUNNING_BADGE =
+  '<span class="badge blue" title="In flight now.">running</span>';
+const INTERRUPTED_BADGE =
+  '<span class="badge amber" title="The process stopped mid-run (restart, crash, or host death). Not a failure.">interrupted</span>';
+
 const VOICE_TRANSCRIBE_EXT = "voice-transcribe";
 const VT_KEY = {
   provider: `${VOICE_TRANSCRIBE_EXT}.provider`,
@@ -1786,12 +1796,18 @@ export function createDashboardRoutes(ctx: DashboardContext) {
               // `failed` still comes from the snapshot, so the badge means the
               // same thing it did before the ledger existed. `quiet` is the new
               // answer — a run that succeeded and deliberately said nothing.
+              // `running` / `interrupted` come from the newest row, which the
+              // snapshot (last *completed* attempt) never reflects.
               const outcomeBadge =
                 t.lastStatus === "error"
                   ? ' <span class="badge">failed</span>'
-                  : lastRun?.outcome === "no_update"
-                    ? ' <span class="badge muted">quiet</span>'
-                    : "";
+                  : lastRun?.status === "running"
+                    ? ` ${RUNNING_BADGE}`
+                    : lastRun?.status === "interrupted"
+                      ? ` ${INTERRUPTED_BADGE}`
+                      : lastRun?.outcome === "no_update"
+                        ? ' <span class="badge muted">quiet</span>'
+                        : "";
               return `
               <tr data-space-id="${escapeHtml(t.spaceId || "")}">
                 <td class="mono">#${t.id}</td>
@@ -1901,6 +1917,8 @@ export function createDashboardRoutes(ctx: DashboardContext) {
 
     const outcomeCell = (run: (typeof runs)[number]): string => {
       if (run.status === "error") return '<span class="badge">failed</span>';
+      if (run.status === "running") return RUNNING_BADGE;
+      if (run.status === "interrupted") return INTERRUPTED_BADGE;
       if (run.outcome === "no_update")
         return '<span class="badge muted">quiet</span>';
       if (run.outcome === "posted")
@@ -1913,9 +1931,13 @@ export function createDashboardRoutes(ctx: DashboardContext) {
         ? runs
             .map((run) => {
               // A clock moved backwards mid-run makes this negative. Say
-              // nothing rather than report a negative duration as fact.
-              const ms = run.finishedAt - run.startedAt;
-              const duration = ms >= 0 ? `${Math.round(ms / 1000)}s` : "—";
+              // nothing rather than report a negative duration as fact. No
+              // finish time — still running, or interrupted by a hard kill —
+              // has no duration either.
+              const ms =
+                run.finishedAt === null ? null : run.finishedAt - run.startedAt;
+              const duration =
+                ms !== null && ms >= 0 ? `${Math.round(ms / 1000)}s` : "—";
               return `
               <tr>
                 <td class="muted">${formatRelativeTime(run.startedAt)}</td>

package/src/core/routes/media.ts

@@ -3,6 +3,7 @@ import { readdir } from "node:fs/promises";
 import path from "node:path";
 import { Hono } from "hono";
 import { checkPerm, type Env, getApiCtx, getAuth } from "../api-types.js";
+import { getStorageInfo } from "./storage.js";
 
 export const media = new Hono<Env>();
 
@@ -50,6 +51,27 @@ media.post("/purge", async (c) => {
   });
 });
 
+/**
+ * GET /media/storage — disk, database and per-space media usage, for `mrctl disk`.
+ *
+ * The caller-token route to what `/api/console/storage` serves the operator,
+ * byte for byte (same helper, same inputs). Gated on `media.purge` because
+ * the response lists EVERY space's inbox/outbox: it is a host-wide view, the
+ * one an operator reads before a purge, and `media.purge` is admin-only by
+ * default. Members could read it before only because every container held
+ * the operator secret.
+ */
+media.get("/storage", async (c) => {
+  const denied = checkPerm(c, "media.purge");
+  if (denied) return denied;
+  const { config } = getApiCtx(c);
+  const info = await getStorageInfo({
+    spacesDir: config.spacesDir,
+    dbPath: config.dbPath,
+  });
+  return c.json(info);
+});
+
 /** Remove all files inside a directory (non-recursive into subdirs). Returns count of removed entries. */
 async function purgeDir(dir: string): Promise<number> {
   let entries: string[];

package/src/core/routes/shadow.ts

@@ -53,12 +53,36 @@ import {
 /** The two members the route uses; a test hands in a fake with exactly these. */
 export type ShadowSimulateSeam = Pick<IngressSimulator, "check" | "simulate">;
 
+/**
+ * `Bun.serve`'s default `idleTimeout`, in ms: a connection with no bytes in
+ * either direction for this long is closed by the server itself, with
+ * `[Bun.serve]: request timed out after 10 seconds` in the journal. None of
+ * the three `Bun.serve` sites (`main.ts` TCP, `main.ts` unix socket,
+ * `smoke/boot.ts`) sets the knob, so this is the number the route streams
+ * against. `tests/shadow-route.test.ts` pins it to the installed bun-types.
+ */
+export const BUN_SERVE_DEFAULT_IDLE_TIMEOUT_MS = 10_000;
+
+/**
+ * Default gap between `: keepalive` comments on the simulate stream. Well
+ * under the idle timeout, not just under it: the host runs on one thread,
+ * and a tick delayed by a blocking Docker or child-process call still has to
+ * land before Bun's clock runs out. The first "idle proxy" that closed this
+ * stream was `Bun.serve` — the default was 15 s against its 10 s.
+ */
+export const DEFAULT_SHADOW_KEEPALIVE_MS = 4_000;
+
 export interface ShadowRouteOptions {
   /** Undefined when the simulator failed to construct at boot: every call is a 503. */
   simulator: ShadowSimulateSeam | undefined;
   /** `config.shadowApiSecret`. Undefined ⇒ every call is a 503. */
   secret: string | undefined;
-  /** Seconds between `: keepalive` comments while a run is in flight. */
+  /**
+   * Milliseconds between `: keepalive` comments while a run is in flight.
+   * Default `DEFAULT_SHADOW_KEEPALIVE_MS`; must stay well under
+   * `BUN_SERVE_DEFAULT_IDLE_TIMEOUT_MS` (the test pins the default at half
+   * or less) or the server closes the stream before the first comment.
+   */
   keepaliveMs?: number;
 }
 
@@ -319,7 +343,7 @@ async function parseSimulateRequest(
 
 export function createShadowRoute(opts: ShadowRouteOptions): Hono {
   const app = new Hono();
-  const keepaliveMs = opts.keepaliveMs ?? 15_000;
+  const keepaliveMs = opts.keepaliveMs ?? DEFAULT_SHADOW_KEEPALIVE_MS;
 
   app.post("/simulate", async (c) => {
     // Auth first, before the body is read: the secret is the shadow's own,
@@ -432,7 +456,8 @@ export function createShadowRoute(opts: ShadowRouteOptions): Hono {
         return tail;
       };
       // Comments are SSE's idle signal; a container run is minutes and a
-      // silent stream is what an idle proxy closes.
+      // silent stream is what an idle proxy closes — `Bun.serve` itself
+      // first, at `BUN_SERVE_DEFAULT_IDLE_TIMEOUT_MS`.
       const keepalive = setInterval(() => {
         if (gone || stream.closed) return;
         void stream.write(": keepalive\n\n").catch(() => {