@animalabs/membrane 0.5.80 → 0.5.81

package/src/formatters/anthropic-xml.ts

@@ -34,6 +34,8 @@ import {
   type ToolDefinitionForPrompt,
 } from '../utils/tool-parser.js';
 import { IncrementalXmlParser } from '../utils/stream-parser.js';
+import { clampCacheMarkers } from '../utils/cache-marker-budget.js';
+import { lastCacheableBlockIndex } from './native.js';
 import { isAcceptedImageMediaType, strippedImagePlaceholder } from '../utils/image-media.js';
 
 // ============================================================================
@@ -118,9 +120,13 @@ export class AnthropicXmlFormatter implements PrefillFormatter {
   readonly name = 'anthropic-xml';
   readonly usesPrefill = true;
 
+  /** See PrefillFormatter.configuredToolMode — undefined when the caller left the mode to Membrane. */
+  readonly configuredToolMode: 'xml' | 'native' | undefined;
+
   private config: Required<AnthropicXmlFormatterConfig>;
 
   constructor(config: AnthropicXmlFormatterConfig = {}) {
+    this.configuredToolMode = config.toolMode;
     this.config = {
       toolMode: config.toolMode ?? 'xml',
       toolInjectionMode: config.toolInjectionMode ?? 'conversation',
@@ -149,6 +155,10 @@ export class AnthropicXmlFormatter implements PrefillFormatter {
       hasCacheMarker,
     } = options;
 
+    // Membrane resolves the mode per request and passes it here; the
+    // constructor-time mode is the fallback for direct callers only.
+    const toolMode = options.toolMode ?? this.config.toolMode;
+
     // Build cache_control object (with optional TTL for extended caching)
     const cacheControl: Record<string, unknown> = { type: 'ephemeral' };
     if (cacheTtl) {
@@ -165,42 +175,66 @@ export class AnthropicXmlFormatter implements PrefillFormatter {
     // message continues the same turn, so it must not get a fresh label.
     let lastWasToolResults = false;
 
-    // Track cache markers applied
-    let cacheMarkersApplied = 0;
 
     // Calculate tool injection point
     const totalMessages = messages.length;
     const toolInjectionIndex = Math.max(0, totalMessages - this.config.toolInjectionPosition);
     let toolsInjected = false;
     const hasToolsForConversation =
-      this.config.toolMode === 'xml' &&
+      toolMode === 'xml' &&
       this.config.toolInjectionMode === 'conversation' &&
       tools &&
       tools.length > 0;
     const toolsText = hasToolsForConversation ? this.formatToolsForInjection(tools!) : '';
 
-    // Build system content
-    let systemText = typeof systemPrompt === 'string' ? systemPrompt : '';
-    if (Array.isArray(systemPrompt)) {
-      systemText = systemPrompt
-        .filter((b): b is ContentBlock & { type: 'text' } => b.type === 'text')
-        .map(b => b.text)
-        .join('\n');
-    }
+    // Build system content. A caller-supplied system ARRAY keeps its block
+    // structure and its per-block cache_control: `request.system` explicitly
+    // accepts caller-marked blocks, and flattening them into one text block
+    // discarded every marker the caller placed (three in, one out) — the
+    // caller's stable prefixes then re-paid full input price forever.
+    const callerSystemBlocks = Array.isArray(systemPrompt)
+      ? systemPrompt
+          .filter((b): b is ContentBlock & { type: 'text' } => b.type === 'text')
+          .map((b) => {
+            const callerMarker = (b as unknown as { cache_control?: unknown }).cache_control;
+            const block: Record<string, unknown> = { type: 'text', text: b.text };
+            if (callerMarker) block.cache_control = callerMarker;
+            return block;
+          })
+      : undefined;
+    const systemBlocks = callerSystemBlocks?.length ? callerSystemBlocks : undefined;
+
+    let systemText = typeof systemPrompt === 'string'
+      ? systemPrompt
+      : (systemBlocks?.map((b) => b.text as string).join('\n') ?? '');
 
     // Inject tools into system if configured
-    if (this.config.toolMode === 'xml' && this.config.toolInjectionMode === 'system' && tools?.length) {
+    if (toolMode === 'xml' && this.config.toolInjectionMode === 'system' && tools?.length) {
       const toolsXml = this.formatToolDefinitionsXml(tools);
       systemText = this.injectToolsIntoSystem(systemText, toolsXml);
+      if (systemBlocks) {
+        // Append to the LAST block only — appending to the join would
+        // collapse the array and take every earlier block's marker with it.
+        const tail = systemBlocks[systemBlocks.length - 1]!;
+        tail.text = this.injectToolsIntoSystem(tail.text as string, toolsXml);
+      }
     }
 
     // Build system content with optional cache control
     let systemContent: unknown;
-    if (systemText) {
+    if (systemBlocks) {
+      // The caller's own markers are authoritative: adding one beside them
+      // spends a slot the caller already allocated.
+      const callerMarkedAny = systemBlocks.some((b) => b.cache_control);
+      if (promptCaching && !callerMarkedAny) {
+        const bpIdx = lastCacheableBlockIndex(systemBlocks);
+        if (bpIdx >= 0) systemBlocks[bpIdx]!.cache_control = cacheControl;
+      }
+      systemContent = systemBlocks;
+    } else if (systemText) {
       const systemBlock: Record<string, unknown> = { type: 'text', text: systemText };
       if (promptCaching) {
         systemBlock.cache_control = cacheControl;
-        cacheMarkersApplied++;
       }
       systemContent = [systemBlock];
     }
@@ -210,7 +244,6 @@ export class AnthropicXmlFormatter implements PrefillFormatter {
       const prefixBlock: Record<string, unknown> = { type: 'text', text: contextPrefix };
       if (promptCaching) {
         prefixBlock.cache_control = cacheControl;
-        cacheMarkersApplied++;
       }
       providerMessages.push({
         role: 'assistant',
@@ -276,7 +309,6 @@ export class AnthropicXmlFormatter implements PrefillFormatter {
           const content = currentConversation.join(joiner);
           const contentBlock: Record<string, unknown> = { type: 'text', text: content };
           contentBlock.cache_control = cacheControl;
-          cacheMarkersApplied++;
           providerMessages.push({
             role: 'assistant',
             content: [contentBlock],
@@ -344,7 +376,6 @@ export class AnthropicXmlFormatter implements PrefillFormatter {
         const content = currentConversation.join(joiner);
         const contentBlock: Record<string, unknown> = { type: 'text', text: content };
         contentBlock.cache_control = cacheControl;
-        cacheMarkersApplied++;
         providerMessages.push({
           role: 'assistant',
           content: [contentBlock],
@@ -396,7 +427,6 @@ export class AnthropicXmlFormatter implements PrefillFormatter {
         };
         if (promptCaching) {
           cliSystemBlock.cache_control = cacheControl;
-          cacheMarkersApplied++;
         }
         systemContent = [cliSystemBlock];
         providerMessages.unshift({
@@ -416,10 +446,22 @@ export class AnthropicXmlFormatter implements PrefillFormatter {
     const stopSequences = this.buildStopSequences(messages, assistantParticipant, options);
 
     // Native tools if configured
-    const nativeTools = this.config.toolMode === 'native' && tools?.length
+    const nativeTools = toolMode === 'native' && tools?.length
       ? this.convertToNativeTools(tools)
       : undefined;
 
+    // Budget. Five sites above attach cache_control (system, contextPrefix,
+    // hasCacheMarker flush, cacheBreakpoint flush, CLI-simulation system) and
+    // multiple cacheBreakpoints are documented input, so a prefill turn with
+    // three marked messages reaches five markers — one over Anthropic's hard
+    // limit, which rejects the request outright. Clamping here, once, on the
+    // finished artifacts is the only count that can see all five sites; the
+    // reported tally is that same recount, so it can never drift from the wire.
+    const budget = clampCacheMarkers(
+      { messages: providerMessages, system: systemContent, tools: nativeTools },
+      'anthropic-xml'
+    );
+
     return {
       messages: providerMessages,
       systemContent,
@@ -428,7 +470,7 @@ export class AnthropicXmlFormatter implements PrefillFormatter {
         : undefined,
       stopSequences,
       nativeTools,
-      cacheMarkersApplied,
+      cacheMarkersApplied: budget.total,
     };
   }
 

package/src/formatters/normalize-tool-pairs.ts

@@ -43,6 +43,55 @@ import type { NormalizeEvent } from './types.js';
  */
 export type ProviderBlock = Record<string, unknown> & { type: string };
 
+/**
+ * Does this repair REWRITE prefix bytes, or only re-shape envelopes?
+ *
+ * The two `true` entries are placeholders that get rewritten again the moment
+ * the real pairing arrives — the synthetic `[pending]` result when its
+ * tool_result lands, the textified orphan when its tool_use does — so a cache
+ * breakpoint placed at or past one caches a prefix that is about to change,
+ * poisoning every subsequent read.
+ *
+ * EVERY kind is listed, and the type is a `Record` over the whole union
+ * precisely so the compiler forces that: a new `NormalizeEvent` kind is a
+ * missing-property error here until someone decides which side it falls on.
+ * The previous shape (`[...] satisfies Array<NormalizeEvent['kind']>`) only
+ * checked that the listed kinds were REAL, and a `satisfies` on an array
+ * accepts any subset — so a new prefix-rewriting repair could be added to the
+ * union, never listed, and silently escape the cache gate below while the
+ * comment claimed a compile error that did not exist.
+ *
+ * NOTE FOR COMPOSE: the `stray_tool_result_textified` kind landing from the
+ * stray-orphan lane belongs here as `true` — it rewrites prefix bytes for the
+ * same reason `orphan_tool_result_textified` does.
+ */
+const NORMALIZE_EVENT_REWRITES_PREFIX: Record<NormalizeEvent['kind'], boolean> = {
+  block_re_roled: false,
+  tool_result_hoisted: false,
+  interloper_deferred: false,
+  synthetic_pending_result: true,
+  orphan_tool_result_textified: true,
+  pending_in_flight: false,
+  cache_suppressed_for_synthetic: false,
+  leading_user_synthesized: false,
+};
+
+function deriveKindsThatRewritePrefix(): ReadonlySet<NormalizeEvent['kind']> {
+  const kinds = Object.entries(NORMALIZE_EVENT_REWRITES_PREFIX)
+    .filter(([, rewritesPrefix]) => rewritesPrefix)
+    .map(([kind]) => kind as NormalizeEvent['kind']);
+  return new Set(kinds);
+}
+
+/**
+ * The repairs that rewrite prefix bytes, derived from the exhaustive
+ * classification above. Consumers gate cache placement on this SET rather
+ * than on a single kind, so a normalizer that grows another prefix-rewriting
+ * repair is covered at every cache site by classifying it once.
+ */
+export const PREFIX_REWRITING_NORMALIZE_EVENT_KINDS: ReadonlySet<NormalizeEvent['kind']> =
+  deriveKindsThatRewritePrefix();
+
 export interface NormalizeOptions {
   /** See `BuildOptions.pendingToolCallIds`. */
   pendingToolCallIds?: ReadonlySet<string>;

package/src/formatters/types.ts

@@ -49,6 +49,16 @@ export interface BuildOptions {
   /** Tool definitions to include */
   tools?: ToolDefinition[];
 
+  /**
+   * The tool mode Membrane RESOLVED for this request (see
+   * `Membrane.resolveToolMode`), which already accounts for `request.toolMode`,
+   * the formatter's own configured mode, and provider/formatter derivation.
+   * A formatter that supports both shapes must build for THIS mode; its
+   * constructor-time mode is only the fallback for direct `buildMessages`
+   * callers that resolve nothing.
+   */
+  toolMode?: 'xml' | 'native';
+
   /** Whether thinking is enabled */
   thinking?: { enabled: boolean; budgetTokens?: number };
 
@@ -163,6 +173,18 @@ export interface BuildResult {
   /** Number of cache control markers applied (for Anthropic prompt caching) */
   cacheMarkersApplied?: number;
 
+  /**
+   * Offset into the turn's accumulated assistant text at which the CURRENT
+   * last message of `messages` begins. Zero (or absent) for an ordinary
+   * build: the whole accumulated document is the trailing assistant prefill.
+   *
+   * A split-turn image injection persists its three messages here and moves
+   * this watermark to the image seam, so later continuations replace only
+   * the trailing assistant message and never re-flatten the pre-image text
+   * over the user turn that carries the image.
+   */
+  accumulatedBaseOffset?: number;
+
   /**
    * `false` only when the tool-pair normalizer detected a trailing
    * unmatched tool_use whose id is in `pendingToolCallIds` — i.e. the
@@ -258,6 +280,16 @@ export interface PrefillFormatter {
   /** Whether this formatter uses prefill (vs native pass-through) */
   readonly usesPrefill: boolean;
 
+  /**
+   * The tool mode this formatter instance was EXPLICITLY constructed with, if
+   * any. Read by `Membrane.resolveToolMode` as the fallback under an explicit
+   * `request.toolMode`: a formatter that can build either shape carries its
+   * caller's configured choice here so resolution honors it instead of
+   * re-deriving one from the formatter's name. Left undefined by formatters
+   * that build exactly one shape.
+   */
+  readonly configuredToolMode?: 'xml' | 'native';
+
   // ==========================================================================
   // REQUEST BUILDING
   // ==========================================================================