@animalabs/agent-framework 0.22.2 → 0.22.4

package/src/recovery/offline-branch.ts

@@ -1,6 +1,7 @@
 import { JsStore } from '@animalabs/chronicle';
 import { ContextManager, continuesBody, type BodyShard } from '@animalabs/context-manager';
 import { removeStoredInterval } from '../stored-removal.js';
+import { firstFreeBranchName } from '../branch-names.js';
 import {
   DEFAULT_DISCORD_AWARENESS_EMOJI,
   DiscordAwarenessOutbox,
@@ -22,6 +23,7 @@ export interface OfflineRecoveryBranchOptions {
   suppressMessageIds?: string[];
   /** Inclusive context-order ranges between Discord message endpoints. */
   suppressRanges?: Array<{ fromMessageId: string; toMessageId: string }>;
+  /** Defaults to `recovery/{agentName}/{ms}`, or with `-2`, `-3`, … appended when the store already holds that name. */
   branchName?: string;
   /** Defaults to the primary-agent namespace `agents/{agentName}`. */
   namespace?: string;
@@ -161,8 +163,9 @@ export async function createOfflineRecoveryBranch(
     refs = dedupeDiscordRefs([...refs, ...suppression.refs]);
 
     const sourceBranch = store.currentBranch().name;
+    const held = new Set(store.listBranches().map((branch) => branch.name));
     const targetBranch = options.branchName
-      ?? `recovery/${options.agentName}/${Date.now()}`;
+      ?? firstFreeBranchName(`recovery/${options.agentName}/${Date.now()}`, (name) => held.has(name));
     if (targetBranch === sourceBranch) {
       throw new Error('Recovery branch name must differ from the active source branch');
     }

package/src/safe-slice.ts

@@ -2,9 +2,17 @@
  * Slice a string without splitting surrogate pairs.
  *
  * Regular String.prototype.slice can cut in the middle of a surrogate pair,
- * producing invalid UTF-16. This function adjusts the end index to avoid that.
+ * producing invalid UTF-16. Here no half of a pair faces a cut: a start on a
+ * low surrogate moves forward past it, and an end just after a high surrogate
+ * moves back before it, whether or not the other half is there. A start of 0
+ * and an end at or past the length cut nothing, so the string's own first and
+ * last units stay as they are.
  */
 export function safeSlice(str: string, start: number, end: number): string {
+  // A low surrogate where the start cuts is a pair's second half, or a lone half: either way, skip it
+  const first = str.charCodeAt(start);
+  if (start > 0 && first >= 0xdc00 && first <= 0xdfff) start += 1;
+
   if (end >= str.length) return str.slice(start);
 
   // If we're about to cut between a high and low surrogate, back up one char

package/src/thinking-token-stamp.ts

@@ -26,9 +26,13 @@ import type { ContentBlock } from '@animalabs/membrane';
  * the hidden thinking is the bulk of the output, and this stamp is what makes
  * the compiled budget honest (see context-manager `SIGNATURE_CHARS_PER_TOKEN`).
  *
- * Wire safety: membrane's Anthropic formatter builds thinking blocks field
- * by field (`type`, `thinking`, `signature`), so the stamp never reaches the
- * provider; the signature itself is copied, never edited.
+ * Wire safety: the stamp is context-manager's pricing metadata, and no field
+ * a provider takes. The Messages API refuses a block that carries it
+ * ("Extra inputs are not permitted"), and membrane (0.5.87 and 0.6.0) sends a
+ * `redacted_thinking` block to Anthropic and Bedrock with every field it has.
+ * So an agent's request builders drop the stamp from the compiled messages
+ * (`withoutTokenEstimates`), and the stored block, which context-manager
+ * prices, keeps it. The signature itself is copied, never edited.
  */
 const PROSE_CHARS_PER_TOKEN = 2.9;
 const DENSE_CHARS_PER_TOKEN = 2.3;
@@ -93,3 +97,30 @@ export function stampThinkingTokenEstimates(blocks: ContentBlock[], outputTokens
     return { ...block, tokenEstimate } as ContentBlock;
   });
 }
+
+/**
+ * A request's messages with no top-level block carrying `tokenEstimate`. Both
+ * of an agent's request builders call it on the compiled messages: a
+ * framework turn's (`Agent.assembleActivationRequest`) and the direct API's
+ * (`Agent.runInferenceWithInjections`). A block that carries the stamp is
+ * copied without it, and so is the message that holds it. Every other
+ * message and block is passed on as it is, and nothing is changed in place,
+ * so the stored block keeps its stamp.
+ */
+export function withoutTokenEstimates<T extends { content: ContentBlock[] }>(messages: T[]): T[] {
+  return messages.map((message) => {
+    if (!message.content.some(isStamped)) return message;
+    return {
+      ...message,
+      content: message.content.map((block) => {
+        if (!isStamped(block)) return block;
+        const { tokenEstimate: _stamp, ...unstamped } = block as ContentBlock & { tokenEstimate?: unknown };
+        return unstamped as ContentBlock;
+      }),
+    };
+  });
+}
+
+function isStamped(block: ContentBlock): boolean {
+  return typeof block === 'object' && block !== null && 'tokenEstimate' in block;
+}

package/src/tool-image-ledger.ts

@@ -27,6 +27,7 @@
  */
 
 import { createHash, randomBytes } from 'node:crypto';
+import { REFERENCE_STUB_MAX_CHARS, referenceTypeLabel } from './mcpl/references.js';
 import { safeSlice } from './safe-slice.js';
 
 export interface RetainedToolImage {
@@ -189,8 +190,9 @@ function mintLedgerNonce(): string {
 /** Grammar of a ref this module mints: `img_<ledger nonce>_<sequence>`. */
 export const TOOL_IMAGE_REF_RE = /^img_[0-9a-z]+_\d+$/;
 
-/** RFC 6838 type/subtype token characters (lowercased); no parameters. */
-const MEDIA_TYPE_ESSENCE_RE = /^[a-z0-9!#$&^_.+-]+\/[a-z0-9!#$&^_.+-]+$/;
+/** RFC 6838 type/subtype token characters (lowercased), each name at most
+ *  127 characters (its section 4.2); no parameters. */
+const MEDIA_TYPE_ESSENCE_RE = /^[a-z0-9!#$&^_.+-]{1,127}\/[a-z0-9!#$&^_.+-]{1,127}$/;
 export const FALLBACK_MEDIA_TYPE = 'application/octet-stream';
 
 /**
@@ -228,21 +230,32 @@ export type ParsedImagePlaceholder =
     /** The placeholder exactly as it appears in the text. */
     text: string;
   }
-  /** An RFC-005 reference stub whose testimony says image — bytes live
+  /** An RFC-005 reference stub whose text shows an image type — bytes live
    *  behind fetch_reference, never in this ledger. Occupies a slot so the
    *  index cannot slide past it. */
   | {
     kind: 'reference';
     refId: string;
-    /** Mime sniffed from the stub text; null when the stub names none. */
-    mediaType: string | null;
+    /** The image type the stub shows, as a stub shows a type (sanitized and
+     *  bounded), lowercased: the first `image/` token in its text, which can
+     *  come from its name. */
+    mediaType: string;
     offset: number;
     /** The stub line exactly as it appears in the text. */
     text: string;
   };
 
-const PLACEHOLDER_RE = /\[image: ([a-z0-9!#$&^_.+-]+\/[a-z0-9!#$&^_.+-]+), (~?[\d.]+(?:B|KB|MB))(?:, ref (img_[0-9a-z]+_\d+))?\]/g;
-/** `[ref_1_ab3d] cam.png — image/png, 1.2MB claimed — from tool result — …` (one line). */
+/**
+ * `[image: image/png, ~691KB, ref img_k7x3q2_7]`. Each part is bounded by
+ * what formatToolImagePlaceholder writes, so text shaped like a placeholder
+ * but longer is no slot: a type and a subtype name at RFC 6838's 127
+ * characters each (normalizeMediaType writes none longer), and a size's
+ * number, a ledger nonce and a sequence at 16 characters each, a safe
+ * integer's digits.
+ */
+const PLACEHOLDER_RE = /\[image: ([a-z0-9!#$&^_.+-]{1,127}\/[a-z0-9!#$&^_.+-]{1,127}), (~?[\d.]{1,16}(?:B|KB|MB))(?:, ref (img_[0-9a-z]{1,16}_\d{1,16}))?\]/g;
+/** `[ref_1_ab3d] cam.png — image/png, 1.2MB claimed — from tool result — …`
+ *  (one line, at most REFERENCE_STUB_MAX_CHARS long: see parseImagePlaceholders). */
 const REFERENCE_STUB_RE = /^\[(ref_[0-9a-z]+_[0-9a-z]+)\] ([^\n]*)$/gm;
 const IMAGE_MIME_RE = /\bimage\/[a-z0-9!#$&^_.+-]+/i;
 
@@ -252,6 +265,13 @@ const IMAGE_MIME_RE = /\bimage\/[a-z0-9!#$&^_.+-]+/i;
  * untrusted — a tool result can quote someone else's placeholder verbatim —
  * so the caller MUST cross-check a resolved ref's provenance against the
  * tool_result block it was found in before handing out bytes.
+ *
+ * Decided from the text alone, so a stored result counts the same however
+ * long ago it was written, across restarts and registry evictions. A
+ * reference stub is a slot when its text shows an image type, and a
+ * non-image reference is no slot. A line shaped like a stub but longer than
+ * buildReferenceStub writes (REFERENCE_STUB_MAX_CHARS) is quoted text and no
+ * slot either, so a slot's text is bounded wherever it is kept.
  */
 export function parseImagePlaceholders(text: string): ParsedImagePlaceholder[] {
   const out: ParsedImagePlaceholder[] = [];
@@ -259,43 +279,142 @@ export function parseImagePlaceholders(text: string): ParsedImagePlaceholder[] {
     out.push({ kind: 'inline', mediaType: m[1]!, sizeLabel: m[2]!, ref: m[3] ?? null, offset: m.index ?? 0, text: m[0] });
   }
   for (const m of text.matchAll(REFERENCE_STUB_RE)) {
-    const mime = m[2]!.match(IMAGE_MIME_RE);
-    out.push({ kind: 'reference', refId: m[1]!, mediaType: mime ? mime[0].toLowerCase() : null, offset: m.index ?? 0, text: m[0] });
+    if (m[0].length > REFERENCE_STUB_MAX_CHARS) continue;
+    const shown = m[2]!.match(IMAGE_MIME_RE);
+    if (!shown) continue;
+    out.push({ kind: 'reference', refId: m[1]!, mediaType: referenceTypeLabel(shown[0]).toLowerCase(), offset: m.index ?? 0, text: m[0] });
   }
   out.sort((a, b) => a.offset - b.offset);
   return out;
 }
 
 /**
- * Cut `text` to at most `cap` chars WITHOUT losing or splitting an image
- * slot. The stored tool_result text is the only place the inventory
- * discovers tool images, while the live wire still delivers every image
- * regardless of text length — so a slot dropped by truncation is an image
- * the resident saw that the index silently slides past (the #104 failure,
- * one representation over). Returns the head to keep and the slots that
- * fell at or past the cut, to be re-appended verbatim after the truncation
- * notice. A slot straddling the cut moves the cut back to its start, so the
- * head never contains a partial slot (a partial reference-stub line would
- * otherwise still re-parse and be counted twice).
+ * The most image slots a stored copy keeps past its cut: a bound on what it
+ * carries there, set at the ledger's entry limit, the most images it holds
+ * bytes for. Serializing a result retains its images in order before their
+ * placeholders are written, and every agent's ledger is built with the
+ * defaults, so once a result is stored a save can reach bytes for at most
+ * its newest DEFAULT_TOOL_IMAGE_LEDGER_MAX_ENTRIES images, fewer when the
+ * ledger's limit on held characters binds first, and an inline image older
+ * than those can only fail. Image stubs and quoted placeholders count toward
+ * the bound too, so where they follow a result's images past the cut, the
+ * newest slots kept can leave out an image whose bytes the ledger still
+ * holds.
+ */
+export const MAX_KEPT_IMAGE_SLOTS = DEFAULT_TOOL_IMAGE_LEDGER_MAX_ENTRIES;
+
+/** The ranges `[start, end)` of a tool result's text that its live copy
+ *  showed beside its images: each text block up to where the live copy cut it
+ *  at the inline cap (toolResultCutAt), and every other block whole. They
+ *  come in text order and never overlap, as the blocks do, and
+ *  cutStoredToolResult relies on that. */
+export type ShownRanges = ReadonlyArray<readonly [number, number]>;
+
+/** A tool result's stored copy, cut at its inline cap. */
+export interface StoredCut {
+  /** The text before the cut. */
+  head: string;
+  /** The image slots past the cut that the stored copy re-appends after its
+   *  truncation notice, in text order: the newest MAX_KEPT_IMAGE_SLOTS of
+   *  those the live copy showed. */
+  kept: string[];
+  /** How many image slots past the cut the live copy showed. */
+  fell: number;
+}
+
+/**
+ * Where a tool result's text longer than `cap` is cut: by its stored copy
+ * (cutStoredToolResult), by history's own truncation (truncateForHistory),
+ * and by its live copy, for each text block beside an image
+ * (tryNativeToolResultContent, framework.ts), so the stored copy and the live
+ * copy cut alike. The text's length when it isn't longer than `cap`.
+ *
+ * The cut is made at the cap, or before it when the cap would split a
+ * surrogate pair (safeSlice's rule), and that is where the moves back are
+ * measured from. It moves back to the start of a placeholder it would split,
+ * and then to the start of a stub-shaped line it would split when at most
+ * REFERENCE_STUB_MAX_CHARS of the line lie before it, since that part could
+ * still read as a slot. So no slot straddles the cut, and the text before it
+ * never ends in part of one. A longer part, which can't parse as a slot,
+ * stays before the cut. Each move back lands on a `[`, so it splits no pair.
  */
-export function splitPreservingImageSlots(text: string, cap: number): { head: string; tail: string[] } {
-  const slots = parseImagePlaceholders(text);
-  let cutAt = cap;
-  for (const slot of slots) {
-    if (slot.offset < cutAt && slot.offset + slot.text.length > cutAt) cutAt = slot.offset;
+export function toolResultCutAt(text: string, cap: number): number {
+  if (text.length <= cap) return text.length;
+  let cutAt = safeSlice(text, 0, cap).length;
+  // Neither grammar crosses a newline, so only the line holding the cut can
+  // be split; it is read whole, as the parser reads it.
+  const lineStart = text.lastIndexOf('\n', cutAt - 1) + 1;
+  const newline = text.indexOf('\n', cutAt);
+  const line = text.slice(lineStart, newline === -1 ? text.length : newline);
+  for (const m of line.matchAll(PLACEHOLDER_RE)) {
+    const start = lineStart + (m.index ?? 0);
+    if (start < cutAt && start + m[0].length > cutAt) cutAt = start;
   }
-  return {
-    head: safeSlice(text, 0, cutAt),
-    tail: slots.filter((slot) => slot.offset >= cutAt).map((slot) => slot.text),
-  };
+  for (const m of line.matchAll(REFERENCE_STUB_RE)) {
+    const start = lineStart + (m.index ?? 0);
+    if (start < cutAt && start + m[0].length > cutAt && cutAt - start <= REFERENCE_STUB_MAX_CHARS) cutAt = start;
+  }
+  return cutAt;
 }
 
-/** Text to append after a truncation notice so the preserved slots stay
- *  parseable (each reference stub on its own line). Empty when none. */
-export function formatPreservedImageSlots(tail: string[]): string {
-  if (tail.length === 0) return '';
-  return `\n[${tail.length} image slot(s) fell past the cut; kept here in order so save_recent_image still counts them:]\n`
-    + tail.join('\n');
+/**
+ * Cut a tool result's text at its inline `cap` as its stored copy keeps it
+ * (toolResultCutAt): the stored copy keeps what the live copy showed. A text
+ * within the cap is all head, with nothing kept after it.
+ *
+ * `shown` is what the live copy showed beside the stored text: null when the
+ * live copy was the text itself, as for every result that carried no inline
+ * image. Then what the cut keeps is exactly what the resident saw, and
+ * nothing past it is kept (#403). A result that carried an inline image went
+ * to the model as native blocks (tryNativeToolResultContent, framework.ts),
+ * every image whatever the text's length, so its stored text is the only
+ * place save_recent_image's inventory finds them, and a slot dropped by the
+ * cut would be an image the resident saw that the index slides past (#104).
+ * The image slots past the cut that lie wholly within what the live copy
+ * showed are kept, the newest MAX_KEPT_IMAGE_SLOTS of them, since the
+ * inventory counts from the end.
+ */
+export function cutStoredToolResult(text: string, cap: number, shown: ShownRanges | null): StoredCut {
+  const cutAt = toolResultCutAt(text, cap);
+  const head = text.slice(0, cutAt);
+  if (shown === null) return { head, kept: [], fell: 0 };
+  // The slots and the ranges both come in text order, so one pass walks
+  // them together: a range that ends where a slot starts, or before, holds
+  // no later slot either.
+  const past: ParsedImagePlaceholder[] = [];
+  let r = 0;
+  for (const slot of parseImagePlaceholders(text)) {
+    if (slot.offset < cutAt) continue;
+    while (r < shown.length && shown[r]![1] <= slot.offset) r++;
+    const range = shown[r];
+    if (range && slot.offset >= range[0] && slot.offset + slot.text.length <= range[1]) past.push(slot);
+  }
+  return { head, kept: past.slice(-MAX_KEPT_IMAGE_SLOTS).map((slot) => slot.text), fell: past.length };
+}
+
+/** What a stored copy re-appends after its truncation notice: the kept
+ *  slots, each on its own line so a reference stub stays parseable, under a
+ *  note saying how many fell past the cut, and how many of those were left
+ *  out and aren't counted. Empty when none fell past the cut. */
+export function formatKeptImageSlots(cut: StoredCut): string {
+  if (cut.fell === 0) return '';
+  const leftOut = cut.fell - cut.kept.length;
+  const note = leftOut === 0
+    ? `${cut.fell} image slot(s) fell past the cut; kept here in order so save_recent_image still counts them:`
+    : `${cut.fell} image slot(s) fell past the cut. The last ${cut.kept.length} are kept here in order so `
+      + `save_recent_image still counts them; the ${leftOut} before them are left out, and it doesn't count them:`;
+  return `\n[${note}]\n${cut.kept.join('\n')}`;
+}
+
+/**
+ * The image slots a tool result's stored copy holds: its text's, or, when the
+ * text is longer than `cap`, those the cut keeps, in head and after the
+ * notice (cutStoredToolResult). The notice itself holds none. For counting a
+ * result before its stored copy is written, as it will count after.
+ */
+export function storedImageSlots(text: string, cap: number, shown: ShownRanges | null): ParsedImagePlaceholder[] {
+  const cut = cutStoredToolResult(text, cap, shown);
+  return parseImagePlaceholders(cut.head + formatKeptImageSlots(cut));
 }
 
 export function sha256Hex(bytes: Buffer): string {