mercury-agent 0.16.2 → 0.17.0

package/examples/extensions/feed-watch/digest.ts

@@ -17,6 +17,7 @@ import { type GetConfig, readSpaceConfig } from "./config.js";
 import {
   formatItemLines,
   formatLocal,
+  isAggregatorLink,
   resolveTimezone,
   type StoredItem,
 } from "./items.js";
@@ -36,6 +37,20 @@ export const MARKER_VERIFY = "[feed-watch:verify]";
 /** Path of the human-readable digest log inside a space workspace. */
 export const DIGEST_FILE = "knowledge/feed-digest.md";
 
+/**
+ * Path of the *current* window, written before the article runs and overwritten
+ * every time.
+ *
+ * `DIGEST_FILE` is the archive and is written on the way out, so during a run it
+ * holds every window except the one the agent is being asked about. This file is
+ * the one the block can honestly point at: full lines, every item, every URL —
+ * including the aggregator redirects the block drops and anything the item cap
+ * left out. The prompt carries the summary; the workspace carries the detail,
+ * which is the arrangement `CLAUDE.md` asks for (facts come from tools, not
+ * prose) and the reason dropping links from the block costs nothing.
+ */
+export const DIGEST_WINDOW_FILE = "knowledge/feed-digest-window.md";
+
 /** How many past windows the workspace file keeps. */
 export const DIGEST_FILE_WINDOWS = 30;
 
@@ -55,13 +70,90 @@ export const pendingDigestKey = (spaceId: string) => `digest_until:${spaceId}`;
 // ---------------------------------------------------------------------------
 
 export interface Digest {
-  /** The `<feed_watch_digest>` block, ready to append to a prompt. */
+  /**
+   * The `<feed_watch_digest>` block plus a trailer that promises no workspace
+   * file — `body` and `renderDigestTrailer` composed for a caller that writes
+   * nothing. `handleBeforeContainer` composes its own, because only a file
+   * that is actually on disk may be named.
+   */
   block: string;
+  /** The block itself: header, item lines, closing tag. No trailer. */
+  body: string;
   /** ISO cursor this window starts at, or null when there was none. */
   since: string | null;
   /** ISO instant the window ends at. */
   until: string;
+  /**
+   * The whole window, uncapped and with every link intact.
+   *
+   * `block` is a *rendering* of this and may be smaller in two ways — the item
+   * cap and dropped aggregator URLs. Everything downstream that is not a prompt
+   * (the cursor, the log line, the workspace file) uses this, so the archive
+   * keeps what the prompt could not afford to carry.
+   */
   items: StoredItem[];
+  /** How many of `items` the block omitted because the cap bit. */
+  omitted: number;
+  /** How many of `items` the block actually lists. */
+  shownCount: number;
+  /** Whether any listed item lost its URL for being aggregator-carried. */
+  hasAggregatorLinks: boolean;
+}
+
+/**
+ * The prose after the block: what the window is, and where the detail the
+ * block traded away can be found.
+ *
+ * Separate from `buildDigest` on purpose. `windowFile` is not an intention —
+ * it is a fact about the filesystem, and the only caller that can establish it
+ * is the one that just tried the write. Deciding it up front is how a run ended
+ * up telling the model to read `knowledge/feed-digest-window.md` on
+ * 2026-08-31 after the write for that same run had thrown.
+ */
+export function renderDigestTrailer(
+  digest: Pick<
+    Digest,
+    "items" | "omitted" | "shownCount" | "hasAggregatorLinks"
+  >,
+  windowFile: boolean,
+): string {
+  return [
+    "These are the only items observed by the feed watcher in this window. " +
+      "Prefer them over anything remembered from earlier conversation; an item " +
+      "not listed here and not verifiable today from a source dated today is " +
+      "not news.",
+    // Said only when it is true, so the article is never told to go looking for
+    // a file that has nothing extra in it.
+    digest.omitted > 0
+      ? `Only the ${digest.shownCount} most recent of ${digest.items.length} matched items are listed here` +
+        (windowFile
+          ? `; the whole window is in \`${DIGEST_WINDOW_FILE}\`.`
+          : ".")
+      : "",
+    digest.hasAggregatorLinks
+      ? "Items carried by a news aggregator are listed without a URL: search " +
+        "the named outlet for the headline" +
+        (windowFile
+          ? `, or read the full lines — every item, every URL — from \`${DIGEST_WINDOW_FILE}\`.`
+          : ".")
+      : "",
+  ]
+    .filter(Boolean)
+    .join(" ");
+}
+
+export interface BuildDigestOptions {
+  /**
+   * Close the window at this instant instead of `now`. The `after_container`
+   * pass uses it to rebuild the *same* window the prompt carried, minutes after
+   * the fact — see `handleAfterContainer`.
+   */
+  untilOverride?: string;
+  /**
+   * Most items to render into `block`, newest first. Omitted items stay in
+   * `items`, so the workspace file still receives them.
+   */
+  maxItems?: number;
 }
 
 /**
@@ -79,13 +171,9 @@ export function buildDigest(
   now: number,
   zone: string,
   log: WatchLogger,
-  /**
-   * Close the window at this instant instead of `now`. The `after_container`
-   * pass uses it to rebuild the *same* window the prompt carried, minutes after
-   * the fact — see `handleAfterContainer`.
-   */
-  untilOverride?: string,
+  opts: BuildDigestOptions = {},
 ): Digest | undefined {
+  const { untilOverride, maxItems } = opts;
   const raw = store.get(bufferKey(spaceId));
   if (raw === null) return undefined;
 
@@ -112,16 +200,41 @@ export function buildDigest(
     return !Number.isFinite(untilMs) || seen <= untilMs;
   });
 
-  const header = `<feed_watch_digest since="${since ?? ""}" until="${until}" items="${items.length}">`;
-  const lines = items.length > 0 ? `\n${formatItemLines(items, zone)}` : "";
-  const block =
-    `${header}${lines}\n</feed_watch_digest>\n` +
-    "These are the only items observed by the feed watcher in this window. " +
-    "Prefer them over anything remembered from earlier conversation; an item " +
-    "not listed here and not verifiable today from a source dated today is " +
-    "not news.";
-
-  return { block, since, until, items };
+  // Newest first, then cut. `formatItemLines` sorts the same way, so the lines
+  // and the cut agree about which items are the freshest.
+  const byRecency = [...items].sort(
+    (a, b) =>
+      Date.parse(b.publishedAt ?? b.seenAt) -
+      Date.parse(a.publishedAt ?? a.seenAt),
+  );
+  const cap = maxItems && maxItems > 0 ? maxItems : byRecency.length;
+  const shown = byRecency.slice(0, cap);
+  const omitted = byRecency.length - shown.length;
+
+  const shownAttr = omitted > 0 ? ` shown="${shown.length}"` : "";
+  const header = `<feed_watch_digest since="${since ?? ""}" until="${until}" items="${items.length}"${shownAttr}>`;
+  const lines =
+    shown.length > 0
+      ? `\n${formatItemLines(shown, zone, { dropAggregatorLinks: true })}`
+      : "";
+
+  const body = `${header}${lines}\n</feed_watch_digest>`;
+  const counts = {
+    items,
+    omitted,
+    shownCount: shown.length,
+    hasAggregatorLinks: shown.some((i) => isAggregatorLink(i.link ?? "")),
+  };
+
+  return {
+    ...counts,
+    body,
+    // Composed for a caller that writes no window file. The one caller that
+    // does re-composes with the outcome of its own write.
+    block: `${body}\n${renderDigestTrailer(counts, false)}`,
+    since,
+    until,
+  };
 }
 
 // ---------------------------------------------------------------------------
@@ -141,10 +254,16 @@ export interface DigestDeps {
   defaultTimezone?: string;
   /** Injectable clock, so a test does not have to reason about wall time. */
   now?: () => number;
+  /**
+   * Writes `DIGEST_WINDOW_FILE`. Optional: a caller that does not supply it
+   * still gets the block, and the block then says nothing about a file it
+   * cannot promise exists.
+   */
+  writeWindow?: (workspace: string, content: string) => void;
 }
 
 /**
- * Append the digest to a prompt that asked for it.
+ * Append the digest to a prompt that asked for it, **for this run only**.
  *
  * Three ways to decline, all logged at debug and all leaving the run untouched:
  * the prompt did not ask, the space is switched off, or the watcher has no
@@ -153,7 +272,7 @@ export interface DigestDeps {
 export function handleBeforeContainer(
   event: DigestEventLike,
   deps: DigestDeps,
-): { promptAppend: string } | undefined {
+): { ephemeralAppend: string } | undefined {
   try {
     if (!event.prompt.includes(MARKER_DIGEST)) return undefined;
 
@@ -167,7 +286,9 @@ export function handleBeforeContainer(
 
     const now = deps.now?.() ?? Date.now();
     const zone = resolveTimezone(cfg.timezone, deps.defaultTimezone, deps.log);
-    const digest = buildDigest(deps.store, event.spaceId, now, zone, deps.log);
+    const digest = buildDigest(deps.store, event.spaceId, now, zone, deps.log, {
+      maxItems: cfg.digestMaxItems,
+    });
     if (!digest) {
       deps.log.debug("feed-watch: no buffer yet, digest omitted", {
         spaceId: event.spaceId,
@@ -175,6 +296,40 @@ export function handleBeforeContainer(
       return undefined;
     }
 
+    // A cap nobody can see is the failure this whole bug was made of. Said at
+    // info, with both numbers, so "the article stopped seeing everything" is a
+    // grep away rather than a re-derivation.
+    if (digest.omitted > 0) {
+      deps.log.info("feed-watch: digest capped, older items left to the file", {
+        spaceId: event.spaceId,
+        window: digest.items.length,
+        shown: digest.items.length - digest.omitted,
+        omitted: digest.omitted,
+        cap: cfg.digestMaxItems,
+      });
+    }
+
+    // The detail the block traded away, put where the agent can fetch it. Wrapped
+    // on its own because the governing rule of this module is that the digest is
+    // an input, not a dependency: a workspace that cannot be written still gets
+    // the article, just without the lookup.
+    //
+    // The trailer is composed from `windowWritten` *after* this, never from the
+    // presence of a writer before it — see `renderDigestTrailer`.
+    let windowWritten = false;
+    if (deps.writeWindow) {
+      try {
+        deps.writeWindow(event.workspace, renderWindowFile(digest, zone, now));
+        windowWritten = true;
+      } catch (err) {
+        deps.log.warn("feed-watch: could not write the digest window file", {
+          spaceId: event.spaceId,
+          error: err instanceof Error ? err.message : String(err),
+        });
+      }
+    }
+    const block = `${digest.body}\n${renderDigestTrailer(digest, windowWritten)}`;
+
     // Remember exactly where this window closed. `after_container` promotes it
     // into the cursor, so the cursor can never move past an item the agent was
     // not shown.
@@ -183,10 +338,17 @@ export function handleBeforeContainer(
     deps.log.info("feed-watch: digest appended", {
       spaceId: event.spaceId,
       items: digest.items.length,
+      blockChars: block.length,
+      windowFile: windowWritten,
       since: digest.since,
       until: digest.until,
     });
-    return { promptAppend: digest.block };
+    // `ephemeralAppend`, not `promptAppend`: the article's turn is stored with
+    // the `[feed-watch:digest]` marker it was written with, and the expansion
+    // lives only in the prompt this one run receives. Stored, it would be
+    // replayed into every later run by `context.mode = context` — which is
+    // exactly what happened, at 103 KB a turn.
+    return { ephemeralAppend: block };
   } catch (err) {
     // The article is worth more than the digest. Whatever went wrong here, the
     // run proceeds with the prompt it already had.
@@ -264,14 +426,11 @@ export function handleAfterContainer(
     // Rebuild the window the prompt carried — while `lastDigestKey` still
     // holds the *old* cursor, since that is the window's lower bound. Only
     // then advance it.
-    const digest = buildDigest(
-      deps.store,
-      event.spaceId,
-      now,
-      zone,
-      deps.log,
-      shownUntil,
-    );
+    // No `maxItems` here on purpose: this rebuild feeds the cursor log and the
+    // workspace file, and the file is where the capped items have to land.
+    const digest = buildDigest(deps.store, event.spaceId, now, zone, deps.log, {
+      untilOverride: shownUntil,
+    });
 
     // Advance and clear unconditionally. Writing the workspace file is
     // best-effort; consuming the window is not, and a buffer that has become
@@ -294,6 +453,47 @@ export function handleAfterContainer(
   }
 }
 
+/**
+ * Human label for the start of the window.
+ *
+ * A cursor that will not parse costs the pretty label and nothing else. On
+ * 2026-08-31 a malformed cursor threw out of `formatLocal` and took both
+ * workspace writes with it — the window file and the archive — while the block
+ * it belonged to went out promising the window file was there.
+ */
+function sinceLabelOf(digest: Digest, zone: string): string {
+  if (!digest.since) return "the beginning";
+  const ms = Date.parse(digest.since);
+  return Number.isFinite(ms) ? formatLocal(ms, zone) : digest.since;
+}
+
+/**
+ * The whole current window as a standalone file: every item, every URL, newest
+ * first. Overwritten each article run — it describes one window, not a history,
+ * and `DIGEST_FILE` is where windows accumulate.
+ */
+export function renderWindowFile(
+  digest: Digest,
+  zone: string,
+  now: number,
+): string {
+  const sinceLabel = sinceLabelOf(digest, zone);
+  const body =
+    digest.items.length > 0
+      ? formatItemLines(digest.items, zone)
+      : "- (nothing matched in this window)";
+  return [
+    `# Feed watcher — current window (${digest.items.length} items)`,
+    "",
+    `Since ${sinceLabel}, up to ${formatLocal(now, zone)}. Overwritten every run.`,
+    "Full lines with URLs, including items the prompt's digest summarised or",
+    "left out. Grep it for a headline when you need the link.",
+    "",
+    body,
+    "",
+  ].join("\n");
+}
+
 /** Render one window as a section and prepend it to the workspace file. */
 export function renderDigestSection(
   digest: Digest,
@@ -301,9 +501,7 @@ export function renderDigestSection(
   now: number,
 ): string {
   const stamp = formatLocal(now, zone);
-  const sinceLabel = digest.since
-    ? formatLocal(Date.parse(digest.since), zone)
-    : "the beginning";
+  const sinceLabel = sinceLabelOf(digest, zone);
   const body =
     digest.items.length > 0
       ? formatItemLines(digest.items, zone)

package/examples/extensions/feed-watch/index.ts

@@ -26,6 +26,7 @@ import { dirname, join } from "node:path";
 import { CONFIG_KEYS } from "./config.js";
 import {
   DIGEST_FILE,
+  DIGEST_WINDOW_FILE,
   type DigestFileIO,
   handleAfterContainer,
   handleBeforeContainer,
@@ -90,6 +91,8 @@ interface ContainerEvent {
 
 interface HookResult {
   promptAppend?: string;
+  /** Appended to this run's prompt but kept out of the stored turn. */
+  ephemeralAppend?: string;
 }
 
 interface MercuryApi {
@@ -285,6 +288,11 @@ export default function (mercury: MercuryApi) {
       store,
       log: ctx.log,
       defaultTimezone: ctx.config.defaultTimezone,
+      writeWindow: (workspace, content) => {
+        const path = join(workspace, DIGEST_WINDOW_FILE);
+        mkdirSync(dirname(path), { recursive: true });
+        writeFileSync(path, content, "utf8");
+      },
     });
   });
 

package/examples/extensions/feed-watch/items.ts

@@ -6,6 +6,12 @@
  * that has learned to read one has learned to read the other. That is why this
  * lives apart from `watch.ts` and `digest.ts` rather than in whichever of them
  * happened to need it first.
+ *
+ * There is exactly one deliberate divergence, `FormatOptions.dropAggregatorLinks`,
+ * and it is about volume rather than meaning: a verify run carries two to four
+ * items and exists to open them, while the digest carries a whole day's window
+ * into one prompt, where a 330-character redirect URL per item is most of the
+ * payload. Any further divergence should be argued the same way or not at all.
  */
 
 import type { FeedItem } from "./feeds.js";
@@ -131,6 +137,34 @@ export function inWindow(hhmm: string, start: string, end: string): boolean {
 // Rendering
 // ---------------------------------------------------------------------------
 
+/**
+ * Is this link an aggregator redirect rather than a page?
+ *
+ * Google News RSS gives every item a `news.google.com/rss/articles/CBMi…` URL:
+ * an opaque base64 blob averaging **330 characters** that resolves to the
+ * outlet only sometimes. `prompts/verify.md` already tells the agent what to do
+ * when one fails — "search the outlet named on the line for the headline
+ * instead" — so the outlet name and title, which are always present, are the
+ * real handle on these items. The URL is mostly weight.
+ */
+export function isAggregatorLink(link: string): boolean {
+  return /^https?:\/\/(?:[a-z0-9-]+\.)*news\.google\.[a-z.]+\//i.test(link);
+}
+
+export interface FormatOptions {
+  /**
+   * Drop aggregator redirect URLs from the line. The digest sets this; a verify
+   * prompt does not.
+   *
+   * Measured on `football-friends`, 2026-08-31: of 345 buffered items, 203 carried
+   * a Google News redirect totalling 67 KB (avg 330 chars) while 142 carried a
+   * direct outlet URL totalling 11.5 KB (avg 81). Dropping only the redirects
+   * takes a full window from 124 KB to 57 KB and costs nothing the agent could
+   * rely on.
+   */
+  dropAggregatorLinks?: boolean;
+}
+
 /**
  * One item, one line:
  *
@@ -141,12 +175,18 @@ export function inWindow(hhmm: string, start: string, end: string): boolean {
  * clock presented as a publication date is exactly the staleness bug this
  * feature exists to end, so the two are never allowed to look alike.
  */
-export function formatItemLine(item: StoredItem, zone: string): string {
+export function formatItemLine(
+  item: StoredItem,
+  zone: string,
+  opts: FormatOptions = {},
+): string {
   const when = item.publishedAt
     ? formatLocal(Date.parse(item.publishedAt), zone)
     : "?";
   const parts = [when, item.source, item.title];
-  if (item.link) parts.push(item.link);
+  const hideLink =
+    opts.dropAggregatorLinks && isAggregatorLink(item.link ?? "");
+  if (item.link && !hideLink) parts.push(item.link);
   parts.push(`terms: ${item.terms.join(", ")}`);
   return `- ${parts.join(" | ")}`;
 }
@@ -162,10 +202,14 @@ export function formatItemLine(item: StoredItem, zone: string): string {
  * where it did not — an undated item's `seenAt` is the closest honest stand-in,
  * and it keeps the order total rather than leaving undated items bunched.
  */
-export function formatItemLines(items: StoredItem[], zone: string): string {
+export function formatItemLines(
+  items: StoredItem[],
+  zone: string,
+  opts: FormatOptions = {},
+): string {
   const at = (i: StoredItem) => Date.parse(i.publishedAt ?? i.seenAt);
   return [...items]
     .sort((a, b) => at(b) - at(a))
-    .map((i) => formatItemLine(i, zone))
+    .map((i) => formatItemLine(i, zone, opts))
     .join("\n");
 }