mercury-agent 0.23.2 → 0.24.0

package/examples/extensions/feed-watch/fixtures.ts

@@ -0,0 +1,210 @@
+/**
+ * Shadow fixtures — a chosen feed body served in place of the network.
+ *
+ * A feed-watch change could be checked only by waiting for the world to
+ * publish the right headline. In a **shadow** (a rehearsal instance, never the
+ * live bot) an operator can drop a file named after a target's key into
+ * `<dataDir>/fixtures/feeds/`, and the next tick reads that file instead of the
+ * target's URL. Rehearsal-bench story M1.7, `docs/archive/2026-09-24-shadow-feed-fixtures.md`.
+ *
+ * **The fixture replaces only the transport.** What this module hands back is a
+ * `FetchImpl`; the kind's parser, the redirect policy, the byte and item caps,
+ * `HTTP <status>` on a non-ok answer, item ids, matching, `seen` and the verify
+ * task all stay the real code in `feeds.ts` / `watch.ts`. A fixture that
+ * exercised a parallel path would prove nothing about the live one.
+ *
+ * No logger and no `ctx` here — pure functions plus one async reader, like
+ * `feeds.ts`. `index.ts` owns the wiring and every log line.
+ */
+
+import { randomInt } from "node:crypto";
+import { readdir, readFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import type { FetchImpl } from "./feeds.js";
+
+/** `<dataDir>/fixtures/feeds` — the one place a shadow looks. */
+export const FIXTURE_SUBDIR = ["fixtures", "feeds"] as const;
+
+/** A fixture whose whole content (trimmed) is this fails its source as HTTP 503. */
+export const DEAD_MARKER = "dead";
+
+/** The status a `dead` fixture answers with — the throw a real 503 makes. */
+const DEAD_STATUS = 503;
+
+/**
+ * The fixture directory, or `null` when fixtures are off.
+ *
+ * **Absence denies.** Only a literal `shadow === true` turns this on — a
+ * missing, false or non-boolean value (`"true"` from a hand-built config) is
+ * off — and a shadow without a non-empty `dataDir` has nowhere to look. On
+ * the live bot this is one strict equality and the directory is never listed
+ * or read.
+ *
+ * `resolve()` gives the path `loadConfig`'s `resolveProjectPath` gave the
+ * shadow data-dir guard (same cwd; `tests/feed-watch-fixtures.test.ts` pins
+ * the equivalence). The extension cannot import that function: feed-watch
+ * imports nothing from `src/`, and its installed snapshot could not resolve
+ * such an import. Never `~/mercury-shadow/current`, which `up` can move, and
+ * never `os.homedir()` or an env var.
+ */
+export function shadowFixtureDir(config: {
+  shadow?: unknown;
+  dataDir?: unknown;
+}): string | null {
+  if (config.shadow !== true) return null;
+  const dataDir = config.dataDir;
+  if (typeof dataDir !== "string" || dataDir.length === 0) return null;
+  return join(resolve(dataDir), ...FIXTURE_SUBDIR);
+}
+
+/**
+ * The file name a target key is served from: `encodeURIComponent`, no
+ * extension. It round-trips exactly and never contains `/`.
+ */
+export function fixtureFileName(targetKey: string): string {
+  return encodeURIComponent(targetKey);
+}
+
+/**
+ * The target key a file name encodes, or `null` for a dotfile (`.gitkeep`, an
+ * editor swap file — ignored) or a name `decodeURIComponent` rejects
+ * (reported as unknown by the caller).
+ */
+export function fixtureKeyOf(fileName: string): string | null {
+  if (fileName.startsWith(".")) return null;
+  try {
+    return decodeURIComponent(fileName);
+  } catch {
+    // A malformed escape is not a key; the caller warns about it as an
+    // unknown file, which is where the operator learns the name is wrong.
+    return null;
+  }
+}
+
+/**
+ * Nine decimal digits, drawn once per process.
+ *
+ * Digits only: a Telegram fixture puts it in `data-post="<channel>/{{nonce}}"`,
+ * and `TG_POST` in `feeds.ts` accepts `\d{1,12}` and nothing else.
+ */
+export function newNonce(): string {
+  return String(randomInt(100_000_000, 1_000_000_000));
+}
+
+/**
+ * Substitute the template tokens, every occurrence. Anything else in double
+ * braces is left as written.
+ *
+ * - `{{nonce}}` — the per-process nonce (item ids: `<guid>`, `<id>`, `data-post`)
+ * - `{{now}}` — RFC 822 (`<pubDate>`)
+ * - `{{now_iso}}` — ISO 8601 (Atom `<updated>`, Telegram `datetime`)
+ */
+export function renderFixture(
+  body: string,
+  t: { nowMs: number; nonce: string },
+): string {
+  const now = new Date(t.nowMs);
+  return body
+    .replaceAll("{{nonce}}", t.nonce)
+    .replaceAll("{{now_iso}}", now.toISOString())
+    .replaceAll("{{now}}", now.toUTCString());
+}
+
+export type FixtureHit =
+  | { kind: "body"; body: string; file: string }
+  | { kind: "dead"; file: string };
+
+function isEnoent(err: unknown): boolean {
+  return (
+    typeof err === "object" &&
+    err !== null &&
+    (err as { code?: unknown }).code === "ENOENT"
+  );
+}
+
+function errorCode(err: unknown): string {
+  const code =
+    typeof err === "object" && err !== null
+      ? (err as { code?: unknown }).code
+      : undefined;
+  if (typeof code === "string" && code.length > 0) return code;
+  return err instanceof Error ? err.message : String(err);
+}
+
+/**
+ * The reader over one fixture directory.
+ *
+ * `node:fs/promises` only: no sync read on the host thread. A fixture is an
+ * operator-written local file in a directory only a shadow reads, so it is not
+ * read in bounded chunks; `fetchFeed`'s byte cap still applies to what it
+ * serves.
+ */
+export function createFixtureSource(dir: string): {
+  lookup(targetKey: string): Promise<FixtureHit | null>;
+  list(): Promise<string[]>;
+} {
+  return {
+    /**
+     * `null` when no file of that name exists (ENOENT) — the request goes to
+     * the network unchanged. **Any other read error rejects** with
+     * `fixture unreadable: <code>`: the source then fails visibly and backs
+     * off, and never silently turns into a network fetch the operator did not
+     * ask for.
+     */
+    async lookup(targetKey) {
+      const file = fixtureFileName(targetKey);
+      let body: string;
+      try {
+        body = await readFile(join(dir, file), "utf8");
+      } catch (err) {
+        if (isEnoent(err)) return null;
+        throw new Error(`fixture unreadable: ${errorCode(err)}`);
+      }
+      if (body.trim() === DEAD_MARKER) return { kind: "dead", file };
+      return { kind: "body", body, file };
+    },
+    /** Fixture file names, dotfiles excluded; `[]` when the directory is absent. */
+    async list() {
+      let names: string[];
+      try {
+        names = await readdir(dir);
+      } catch (err) {
+        if (isEnoent(err)) return [];
+        throw err;
+      }
+      return names.filter((n) => !n.startsWith(".")).sort();
+    },
+  };
+}
+
+/**
+ * The `FetchImpl` that answers one hit. It ignores the URL: the file was
+ * chosen by target key, not by URL, and the kind's parser and redirect policy
+ * are still chosen by `fetchForTarget` exactly as for the network.
+ */
+export function fixtureFetchImpl(
+  hit: FixtureHit,
+  t: { nowMs: number; nonce: string },
+): FetchImpl {
+  if (hit.kind === "dead") {
+    return async () => ({
+      ok: false,
+      status: DEAD_STATUS,
+      text: async () => "",
+    });
+  }
+  const rendered = renderFixture(hit.body, t);
+  return async () => ({ ok: true, status: 200, text: async () => rendered });
+}
+
+/** Files whose decoded key is not in `knownKeys` (undecodable names included). */
+export function unknownFixtures(
+  files: readonly string[],
+  knownKeys: ReadonlySet<string>,
+): string[] {
+  return files.filter((f) => {
+    if (f.startsWith(".")) return false;
+    const key = fixtureKeyOf(f);
+    return key === null || !knownKeys.has(key);
+  });
+}

package/examples/extensions/feed-watch/index.ts

@@ -23,7 +23,7 @@
 
 import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { dirname, join } from "node:path";
-import { CONFIG_KEYS } from "./config.js";
+import { CONFIG_KEYS, readSpaceConfig } from "./config.js";
 import {
   type DigestFileIO,
   handleAfterContainer,
@@ -38,8 +38,18 @@ import {
   type OnTruncate,
   parseTelegramPreview,
 } from "./feeds.js";
+import {
+  createFixtureSource,
+  fixtureFetchImpl,
+  fixtureFileName,
+  fixtureKeyOf,
+  newNonce,
+  shadowFixtureDir,
+  unknownFixtures,
+} from "./fixtures.js";
 import type { PostedReply } from "./posted.js";
 import {
+  expandSources,
   type FetchTarget,
   type PollDeps,
   pollSpace,
@@ -59,6 +69,13 @@ export const POLL_INTERVAL_MS = 180_000;
  */
 export const LAST_TICK_KEY = "last_tick";
 
+/**
+ * How many `{ key, file }` pairs the unknown-fixture warning carries. A space
+ * with a gnews fan-out has one key per term, and a warn line is not the place
+ * for all of them; `more` says how many were left out.
+ */
+export const FIXTURE_KNOWN_KEYS_SHOWN = 40;
+
 // Ships alongside this extension, so `import.meta.dir` is the installed
 // extension directory — the same arrangement longview uses for its prompt.
 const VERIFY_PROMPT_PATH = join(import.meta.dir, "prompts", "verify.md");
@@ -98,7 +115,9 @@ interface ExtensionDb {
 
 interface ExtensionContext {
   db: ExtensionDb;
-  config: { defaultTimezone?: string };
+  // `shadow` and `dataDir` are read only by the fixture lookup, and only a
+  // literal `shadow === true` turns it on (`shadowFixtureDir`).
+  config: { defaultTimezone?: string; shadow?: boolean; dataDir?: string };
   // Mirrors `Logger` in `src/logger.ts`: every level takes a message and one
   // optional `extra`. An extension declares its own structural type and the
   // loader hands it an untyped object, so nothing would catch a third argument
@@ -281,6 +300,22 @@ export default function (mercury: MercuryApi) {
    */
   let missingReadWarned = false;
 
+  // ── Shadow fixtures ──────────────────────────────────────────────────────
+
+  /**
+   * Drawn once per process: every tick of one shadow renders a fixture to the
+   * same item ids, so the second tick finds the item in `seen` and opens no
+   * second run; the next `shadow up` draws a new one, so the same file is new
+   * again against a snapshot that already saw it. No state is deleted.
+   */
+  const fixtureNonce = newNonce();
+
+  /** The `shadow fixtures active` line is said on the first tick only. */
+  let fixturesAnnounced = false;
+
+  /** Unknown fixture file names already warned about — once per name per process. */
+  const fixtureWarned = new Set<string>();
+
   /**
    * The one place the host's `listRepliesSince` is reached. Both consumers —
    * the poller's link drop and the `<recent_posts>` hook — go through it, so
@@ -304,13 +339,16 @@ export default function (mercury: MercuryApi) {
     };
   }
 
-  function buildDeps(ctx: ExtensionContext): Omit<PollDeps, "log"> & {
+  function buildDeps(
+    ctx: ExtensionContext,
+    fixtures: ReturnType<typeof createFixtureSource> | null,
+  ): Omit<PollDeps, "log"> & {
     log: WatchLogger;
   } {
     return {
       getConfig: (spaceId, key) => ctx.getConfig(spaceId, key),
       store,
-      fetchFeed: (target: FetchTarget): Promise<FeedItem[]> => {
+      fetchFeed: async (target: FetchTarget): Promise<FeedItem[]> => {
         // Hoisted now that there are two kinds — M1.2 left it inline in the
         // `xml` case so that case would diff against the pre-seam closure as
         // whitespace only. That debt is paid by the second caller, not by a
@@ -325,10 +363,23 @@ export default function (mercury: MercuryApi) {
             ...detail,
           });
         };
-        return fetchForTarget(target, {
-          fetchImpl: globalThis.fetch as unknown as FetchImpl,
-          onTruncate,
-        });
+        let fetchImpl = globalThis.fetch as unknown as FetchImpl;
+        // Shadow only. A lookup that rejects (anything but ENOENT) rejects
+        // this fetch, so the source fails visibly as `fixture unreadable`
+        // rather than quietly reading the network instead.
+        const hit = fixtures ? await fixtures.lookup(target.key) : null;
+        if (hit) {
+          ctx.log.info("feed-watch: serving fixture", {
+            source: target.key,
+            file: hit.file,
+            dead: hit.kind === "dead",
+          });
+          fetchImpl = fixtureFetchImpl(hit, {
+            nowMs: Date.now(),
+            nonce: fixtureNonce,
+          });
+        }
+        return fetchForTarget(target, { fetchImpl, onTruncate });
       },
       createTask: (spaceId, prompt, runAs) =>
         ctx.db.createTask(
@@ -355,6 +406,71 @@ export default function (mercury: MercuryApi) {
     };
   }
 
+  /**
+   * The two fixture lines a shadow tick owes its operator: the directory on
+   * the first tick (even at `files: 0` — it proves the lookup is on and names
+   * the path to copy into; `setup` has no `ctx`, so the first tick is the
+   * earliest `ctx.config` can be read), and each file name that matches no
+   * target of any enabled space, once per name per process.
+   *
+   * Known keys come from the functions `pollSpace` itself uses —
+   * `readSpaceConfig` → `expandSources` — so the warning cannot disagree with
+   * the poller about what a key is. Not from fetched targets: a target in
+   * backoff (one served `dead`, say) is not fetched and would read as unknown.
+   * `readSpaceConfig` is called without a logger because `pollSpace` has
+   * already said whatever is wrong with the config this tick.
+   */
+  async function reportFixtures(
+    ctx: ExtensionContext,
+    dir: string,
+    fixtures: ReturnType<typeof createFixtureSource>,
+  ): Promise<void> {
+    const files = await fixtures.list();
+    if (!fixturesAnnounced) {
+      fixturesAnnounced = true;
+      ctx.log.info("feed-watch: shadow fixtures active", {
+        dir,
+        files: files.length,
+      });
+    }
+    if (files.every((f) => fixtureWarned.has(f))) return;
+
+    const known = new Set<string>();
+    let enabledSpaces = 0;
+    for (const space of ctx.db.listSpaces()) {
+      const cfg = readSpaceConfig(
+        (spaceId, key) => ctx.getConfig(spaceId, key),
+        space.id,
+      );
+      if (!cfg.enabled) continue;
+      enabledSpaces++;
+      for (const target of expandSources(cfg)) known.add(target.key);
+    }
+    // With nothing enabled every file is "unknown", and saying so once would
+    // use up the warning before the operator enables the space it was for.
+    if (enabledSpaces === 0) return;
+
+    const unknown = unknownFixtures(files, known).filter(
+      (f) => !fixtureWarned.has(f),
+    );
+    if (unknown.length === 0) return;
+    const keys = [...known].sort();
+    const shown = keys
+      .slice(0, FIXTURE_KNOWN_KEYS_SHOWN)
+      .map((key) => ({ key, file: fixtureFileName(key) }));
+    for (const file of unknown) {
+      fixtureWarned.add(file);
+      ctx.log.warn("feed-watch: fixture matches no source", {
+        file,
+        decoded: fixtureKeyOf(file),
+        known: shown,
+        ...(keys.length > shown.length
+          ? { more: keys.length - shown.length }
+          : {}),
+      });
+    }
+  }
+
   mercury.job("poll", {
     interval: POLL_INTERVAL_MS,
     run: async (ctx) => {
@@ -379,7 +495,9 @@ export default function (mercury: MercuryApi) {
         deferred: [] as string[],
       };
       try {
-        const deps = buildDeps(ctx);
+        const fixtureDir = shadowFixtureDir(ctx.config);
+        const fixtures = fixtureDir ? createFixtureSource(fixtureDir) : null;
+        const deps = buildDeps(ctx, fixtures);
         for (const space of ctx.db.listSpaces()) {
           tally.spaces++;
           try {
@@ -405,6 +523,19 @@ export default function (mercury: MercuryApi) {
             });
           }
         }
+        if (fixtureDir && fixtures) {
+          try {
+            await reportFixtures(ctx, fixtureDir, fixtures);
+          } catch (err) {
+            // Counted like a space's failure, and said: a fixture directory
+            // that cannot be listed is the operator's whole rehearsal.
+            tally.errors++;
+            ctx.log.error("feed-watch: fixture directory unreadable", {
+              dir: fixtureDir,
+              error: err instanceof Error ? err.message : String(err),
+            });
+          }
+        }
       } finally {
         inFlight = false;
         const elapsedMs = Date.now() - started;

package/examples/extensions/office-365-mail/README.md

@@ -0,0 +1,132 @@
+# office-365-mail — Outlook / Microsoft 365 mail over Microsoft Graph
+
+Read, search, triage, send and reply to a Microsoft 365 (Office 365 / Outlook)
+mailbox from a chat. Third adopter of `mercury.capability()`: the Entra client
+id, the refresh token and every rotated successor stay in the Mercury host
+process and never enter an agent container.
+
+Microsoft has switched off basic-auth IMAP/SMTP for Microsoft 365, so this
+extension speaks Microsoft Graph over delegated OAuth2. No CLI is installed in
+the image and there are no runtime dependencies — plain `fetch`.
+
+## What this example shows
+
+- `mercury.capability()` with ten verbs the agent calls via `mrctl capability office-365-mail <action> '<json>'`
+- three `hostOnly` env claims, so no `M365_*` value reaches a container
+- a `mercury.connection()` whose daily `healthCheck` is a forced token exchange through the same single-flight path the verbs use
+- a credential that **rotates on every use** — see "How the refresh token is stored"
+
+## Install
+
+```bash
+mercury add ./examples/extensions/office-365-mail
+```
+
+No CLI is declared, so no Docker image rebuild is needed.
+
+## One-time setup
+
+### 1. Register a public-client app in Microsoft Entra
+
+In the [Entra admin center](https://entra.microsoft.com) → **App registrations** → **New registration**:
+
+- Name: anything (e.g. `Mercury mail`)
+- Supported account types: *Accounts in this organizational directory only* is enough for one work mailbox
+- Redirect URI: leave empty (the device-code flow needs none)
+
+Then, on the new registration:
+
+- **Authentication** → *Advanced settings* → **Allow public client flows: Yes** → Save. Without this the device-code login fails with `AADSTS7000218` / `invalid_client`.
+- **API permissions** → *Add a permission* → *Microsoft Graph* → *Delegated* → add `Mail.ReadWrite`, `Mail.Send`, `User.Read`, `offline_access`. Some tenants require an admin to press **Grant admin consent** for these; if the login step below says consent is required, that is the button.
+- Copy the **Application (client) ID** from *Overview*.
+
+No client secret is created — the app is a public client and the login proves the user, not the app.
+
+### 2. Sign in once, on the host
+
+```bash
+bun examples/extensions/office-365-mail/scripts/login.ts --client-id <application-client-id>
+```
+
+(After `mercury add`, the same script lives at `.mercury/extensions/office-365-mail/scripts/login.ts`.)
+
+It prints a URL and a short code. Open the URL in any browser, sign in as the
+mailbox owner, enter the code, accept the permissions. The script then prints:
+
+```
+MERCURY_M365_CLIENT_ID=…
+MERCURY_M365_TENANT=organizations
+MERCURY_M365_REFRESH_TOKEN=…
+```
+
+Add those lines to Mercury's `.env` and restart Mercury. All three are claimed
+`hostOnly`.
+
+`--tenant` defaults to `organizations` (any work or school account). If Microsoft
+answers `AADSTS90133` ("device code flow is not supported under this
+endpoint"), pass your tenant id or primary domain: `--tenant contoso.onmicrosoft.com`.
+For a personal outlook.com mailbox use `--tenant consumers` (untested).
+
+### 3. Verify
+
+```bash
+mercury chat --space main --caller "<admin-caller-id>" \
+  "run exactly: mrctl capability office-365-mail list-folders '{}'"
+```
+
+A non-admin caller is blocked by RBAC (`Forbidden: requires 'office-365-mail' permission`).
+The console's Connections page shows **Office 365 Mail** as connected.
+
+## How the refresh token is stored
+
+Microsoft's token endpoint returns a **new refresh token on every exchange**.
+The `.env` value is therefore only a seed:
+
+- The newest token lives in Mercury's extension state (`state.db`), under a
+  fingerprint of the seed it descends from.
+- An unchanged seed defers to its stored descendant. A **changed** seed (you
+  re-ran `login.ts`) wins outright and the old descendant is dropped.
+- Removing the seed from `.env` after the first rotation does **not** break the
+  connection — the extension loads on `MERCURY_M365_CLIENT_ID`, not the seed.
+- A failed exchange never deletes the stored token. Microsoft keeps the previous
+  token valid, so a transient outage costs nothing.
+
+Consequence worth knowing: `state.db` now carries a live mailbox credential.
+Treat its backups the way you treat `.env`.
+
+## When it needs re-authentication
+
+The daily connection-health pass does one token exchange. If Microsoft rejects
+the token (`invalid_grant` — password changed, sessions revoked, 90 days of
+inactivity, conditional-access policy), the operator alert names the fix and
+the Connections page shows **needs-reauth**. Re-run step 2, replace the three
+`.env` lines, restart.
+
+## Verbs
+
+| Action | Body | Returns |
+|---|---|---|
+| `list-folders` | `{}` | `{ folders: [{ id, name, total, unread }] }` |
+| `list-inbox` | `{ limit?, folder? }` | `{ folder, total, messages: [...] }` |
+| `search` | `{ query, limit?, folder? }` | `{ query, folder, total, messages: [...] }` — `query` is KQL (`from:ann subject:invoice`) |
+| `read` | `{ id }` | message with text `body`, `cc`, `attachments: [{ name, size }]`, `conversationId` |
+| `move` | `{ id, destination }` | `{ ok, id: <new id>, to }` — the id changes |
+| `delete` | `{ id }` | `{ ok, id: <new id>, folder: "deleteditems" }` — a move, recoverable |
+| `mark-read` / `mark-unread` | `{ id }` | `{ ok, id, marked }` |
+| `send` | `{ to, subject, body, cc? }` | `{ ok, to, subject }` |
+| `reply` | `{ id, body }` | `{ ok, id, subject }` |
+
+Folders: the well-known names `inbox`, `drafts`, `sentitems`, `deleteditems`,
+`junkemail`, `archive` work in any mailbox language; anything else is matched by
+display name. `limit` defaults to 20 and is capped at 100.
+
+Every failure is returned as JSON with Microsoft's own words, never thrown. A
+`401` carries `"reauth": true`.
+
+## Not implemented
+
+- Calendar, contacts, OneDrive — different scopes; a separate extension.
+- Attachments beyond names and sizes (no upload on send, no download on read).
+- Application (client-credentials) permissions and shared mailboxes — one delegated mailbox per host.
+- A dashboard "Connect Microsoft" button — the console has no OAuth redirect flow; the host-side login is the operator step, as with `gws auth login`.
+- Nested folders — `list-folders` lists the root level only, and a subfolder's display name is not resolved by `list-inbox`, `search` or `move`. The well-known names and root-level folders cover the common case.