@animalabs/membrane 0.5.80 → 0.5.82

package/src/cache-wire-receipt.ts

@@ -0,0 +1,125 @@
+import { createHash } from 'node:crypto';
+
+export interface CacheWireMarkerReceipt {
+  ordinal: number;
+  prefixHash: string;
+  estimatedOffset: number;
+}
+
+export interface CacheWireReceipt {
+  requestHash: string;
+  markers: CacheWireMarkerReceipt[];
+}
+
+/** Hash the exact post-format/post-hook provider request and every marked
+ * prefix. Token offsets are estimates; provider usage reconciles them later. */
+export function computeCacheWireReceipt(rawRequest: unknown): CacheWireReceipt {
+  const requestHash = sha(stableStringify(rawRequest));
+  const blocks = flattenWireBlocks(rawRequest);
+  const prefix: unknown[] = [];
+  const markers: CacheWireMarkerReceipt[] = [];
+  let chars = 0;
+  for (const unit of blocks) {
+    prefix.push(unit.identity);
+    chars += unit.estimatedChars;
+    if (unit.marked) {
+      markers.push({
+        ordinal: markers.length,
+        prefixHash: sha(stableStringify(prefix)),
+        estimatedOffset: Math.ceil(chars / 4),
+      });
+    }
+  }
+  return { requestHash, markers };
+}
+
+interface ReceiptUnit {
+  identity: unknown;
+  marked: boolean;
+  estimatedChars: number;
+}
+
+function flattenWireBlocks(rawRequest: unknown): ReceiptUnit[] {
+  if (!rawRequest || typeof rawRequest !== 'object') {
+    return [{ identity: rawRequest, marked: false, estimatedChars: stableStringify(rawRequest).length }];
+  }
+  const request = rawRequest as Record<string, unknown>;
+  const out: ReceiptUnit[] = [];
+  if (Array.isArray(request.tools)) {
+    request.tools.forEach((tool, index) => out.push({
+      identity: { surface: 'tool', index, tool },
+      marked: hasCacheControl(tool),
+      estimatedChars: stableStringify(tool).length,
+    }));
+  } else if (request.tools !== undefined) {
+    out.push({
+      identity: { surface: 'tools', value: request.tools },
+      marked: hasCacheControl(request.tools),
+      estimatedChars: stableStringify(request.tools).length,
+    });
+  }
+  if (request.system !== undefined) {
+    const system = request.system;
+    if (Array.isArray(system)) {
+      system.forEach((block, index) => out.push({
+        identity: { surface: 'system', index, block },
+        marked: hasCacheControl(block),
+        estimatedChars: stableStringify(block).length,
+      }));
+    } else {
+      out.push({
+        identity: { surface: 'system', value: system },
+        marked: hasCacheControl(system),
+        estimatedChars: stableStringify(system).length,
+      });
+    }
+  }
+  if (Array.isArray(request.messages)) {
+    request.messages.forEach((message, messageIndex) => {
+      if (!message || typeof message !== 'object') {
+        out.push({
+          identity: { surface: 'message', messageIndex, value: message },
+          marked: false,
+          estimatedChars: stableStringify(message).length,
+        });
+        return;
+      }
+      const record = message as Record<string, unknown>;
+      const content = (message as Record<string, unknown>).content;
+      if (Array.isArray(content)) {
+        content.forEach((block, blockIndex) => out.push({
+          identity: {
+            surface: 'message-block', messageIndex, blockIndex,
+            role: record.role, block,
+          },
+          marked: hasCacheControl(block),
+          estimatedChars: stableStringify({ role: record.role, content: [block] }).length,
+        }));
+      } else {
+        out.push({
+          identity: { surface: 'message', messageIndex, role: record.role, content },
+          marked: hasCacheControl(message),
+          estimatedChars: stableStringify(message).length,
+        });
+      }
+    });
+  }
+  return out;
+}
+
+function hasCacheControl(value: unknown): boolean {
+  return Boolean(value && typeof value === 'object' && (value as Record<string, unknown>).cache_control);
+}
+
+function stableStringify(value: unknown): string {
+  if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`;
+  if (value && typeof value === 'object') {
+    const record = value as Record<string, unknown>;
+    return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(record[key])}`).join(',')}}`;
+  }
+  return JSON.stringify(value) ?? 'null';
+}
+
+function sha(value: string): string {
+  return createHash('sha256').update(value).digest('hex');
+}

package/src/context/index.ts

@@ -25,6 +25,7 @@ export {
   createInitialState,
   defaultTokenEstimator,
   DEFAULT_CONTEXT_CONFIG,
+  MembraneContextIdentityError,
 } from './types.js';
 
 export type {

package/src/context/process.ts

@@ -17,8 +17,26 @@ import {
   createInitialState,
   defaultTokenEstimator,
   DEFAULT_CONTEXT_CONFIG,
+  MembraneContextIdentityError,
 } from './types.js';
 
+/**
+ * Ceiling on markers this module places, regardless of `cache.points`.
+ *
+ * Anthropic accepts at most 4 `cache_control` blocks per request, and the
+ * request builders spend from that same budget (a system/tools fallback
+ * block, a `contextPrefix` block when that option is set, the floating
+ * tool-loop marker). Nothing reconciles those spends against
+ * `cache.points`, so the module keeps one slot free rather than risk a 400
+ * on the default XML path, which always marks the system block. A caller
+ * combining this module with MORE than one formatter spend (e.g. system
+ * prompt AND `contextPrefix`) can still exceed the provider budget: the
+ * module cannot see the formatter's choices from here, which is why the
+ * request-side wire clamp (the cache-marker budget PR) is the layer that
+ * makes the ceiling safe by construction rather than by convention.
+ */
+const MAX_MODULE_CACHE_POINTS = 3;
+
 // ============================================================================
 // Main Entry Point
 // ============================================================================
@@ -48,6 +66,10 @@ export async function processContext(
   const contextConfig = mergeConfig(input.context);
   const tokenEstimator = contextConfig.tokenEstimator ?? defaultTokenEstimator;
   
+  // Stable identity is a precondition, not a nicety: without it every call
+  // looks like a new conversation and rolling/caching silently stop working.
+  assertStableMessageIds(input.messages);
+  
   // Initialize or continue state
   let currentState = state ?? createInitialState();
   
@@ -87,12 +109,16 @@ export async function processContext(
       messageTokens,
       rollDecision.targetTokens,
       rollDecision.targetMessages,
-      contextConfig
+      contextConfig,
+      rollDecision.targetCharacters
     );
     
     keptMessages = truncateResult.kept.map(m => m.message);
     messagesDropped = truncateResult.dropped;
-    didRoll = true;
+    // A roll that dropped nothing is not a roll: reporting it as one both
+    // lies to the caller and resets the roll counters every call, which is
+    // exactly when threshold rolling is needed most.
+    didRoll = messagesDropped > 0;
     hardLimitHit = rollDecision.reason === 'hard_limit';
   }
   
@@ -104,6 +130,15 @@ export async function processContext(
   }));
   const keptTotalTokens = keptTokens.reduce((sum, m) => sum + m.tokens, 0);
   
+  // Re-assert the hard limits against the truncated window. The window is
+  // floored at one message, so a single oversize message survives every
+  // truncation - the caller is told rather than handed an empty array.
+  const residualOverflow = measureResidualOverflow(
+    keptMessages,
+    keptTotalTokens,
+    contextConfig
+  );
+  
   // Place cache markers
   const cacheMarkers = placeCacheMarkers(
     keptMessages,
@@ -168,6 +203,7 @@ export async function processContext(
     totalTokens: keptTotalTokens,
     hardLimitHit,
     cachedStartMessageId,
+    ...(residualOverflow ? { residualOverflow } : {}),
   };
   
   return { response, state: newState, info };
@@ -178,7 +214,13 @@ export async function processContext(
 // ============================================================================
 
 function mergeConfig(config: ContextConfig): ContextConfig {
+  // The caller's config is the base, so a top-level field is carried through
+  // by default and only the three sub-objects that have defaults are merged.
+  // Enumerating the survivors instead silently dropped assistantParticipant:
+  // every helper test passed a config straight in, so the loss was invisible
+  // until an end-to-end call classified the configured assistant as a user.
   return {
+    ...config,
     rolling: {
       ...DEFAULT_CONTEXT_CONFIG.rolling,
       ...config.rolling,
@@ -191,7 +233,6 @@ function mergeConfig(config: ContextConfig): ContextConfig {
       ...DEFAULT_CONTEXT_CONFIG.cache,
       ...config.cache,
     },
-    tokenEstimator: config.tokenEstimator,
   };
 }
 
@@ -199,6 +240,61 @@ function getMessageId(message: NormalizedMessage): string {
   return message.metadata?.sourceId ?? `msg-${Math.random().toString(36).slice(2)}`;
 }
 
+function assertStableMessageIds(messages: NormalizedMessage[]): void {
+  const messageIndicesWithoutSourceId: number[] = [];
+  
+  messages.forEach((message, index) => {
+    const sourceId = message.metadata?.sourceId;
+    if (typeof sourceId !== 'string' || sourceId.length === 0) {
+      messageIndicesWithoutSourceId.push(index);
+    }
+  });
+  
+  if (messageIndicesWithoutSourceId.length === 0) {
+    return;
+  }
+  
+  const shown = messageIndicesWithoutSourceId.slice(0, 10).join(', ');
+  const ellipsis = messageIndicesWithoutSourceId.length > 10 ? ', ...' : '';
+  throw new MembraneContextIdentityError(
+    `processContext requires stable message identity: ` +
+    `${messageIndicesWithoutSourceId.length} of ${messages.length} messages carry no ` +
+    `metadata.sourceId (indices ${shown}${ellipsis}). Without it every call is detected ` +
+    `as a new conversation, so the roll threshold never accumulates, cache markers never ` +
+    `stay stable, and cachedStartMessageId is meaningless. Populate metadata.sourceId ` +
+    `with the originating system's message id.`,
+    messageIndicesWithoutSourceId
+  );
+}
+
+function measureResidualOverflow(
+  keptMessages: NormalizedMessage[],
+  keptTotalTokens: number,
+  config: ContextConfig
+): ContextInfo['residualOverflow'] {
+  const limits = config.limits;
+  if (!limits) {
+    return undefined;
+  }
+  
+  if (limits.maxCharacters) {
+    const keptCharacters = calculateCharacters(keptMessages);
+    if (keptCharacters > limits.maxCharacters) {
+      return { unit: 'characters', limit: limits.maxCharacters, actual: keptCharacters };
+    }
+  }
+  
+  if (limits.maxTokens && keptTotalTokens > limits.maxTokens) {
+    return { unit: 'tokens', limit: limits.maxTokens, actual: keptTotalTokens };
+  }
+  
+  if (limits.maxMessages && keptMessages.length > limits.maxMessages) {
+    return { unit: 'messages', limit: limits.maxMessages, actual: keptMessages.length };
+  }
+  
+  return undefined;
+}
+
 function detectDiscontinuity(
   messages: NormalizedMessage[],
   state: ContextState
@@ -237,6 +333,8 @@ export interface RollDecision {
   reason?: 'threshold' | 'grace_exceeded' | 'hard_limit';
   targetTokens?: number;
   targetMessages?: number;
+  /** Character budget the kept window must fit (set by the maxCharacters limit). */
+  targetCharacters?: number;
   enteredGrace: boolean;
 }
 
@@ -261,6 +359,7 @@ export function shouldRoll(
       reason: 'hard_limit',
       targetTokens: limits.maxTokens,
       targetMessages: limits.maxMessages,
+      targetCharacters: limits.maxCharacters,
       enteredGrace: false,
     };
   }
@@ -271,6 +370,7 @@ export function shouldRoll(
       reason: 'hard_limit',
       targetTokens: limits.maxTokens,
       targetMessages: limits.maxMessages,
+      targetCharacters: limits.maxCharacters,
       enteredGrace: false,
     };
   }
@@ -281,12 +381,16 @@ export function shouldRoll(
       reason: 'hard_limit',
       targetTokens: limits.maxTokens,
       targetMessages: limits.maxMessages,
+      targetCharacters: limits.maxCharacters,
       enteredGrace: false,
     };
   }
   
-  // Check rolling threshold
-  const current = unit === 'messages' ? state.messagesSinceRoll : state.tokensSinceRoll;
+  // Check rolling threshold against the MEASURED window. state.messagesSinceRoll
+  // counts calls (not messages) and state.tokensSinceRoll re-adds the whole
+  // window every call, so both cross any threshold as a function of call count
+  // alone; they stay as telemetry and no longer decide the roll.
+  const current = unit === 'messages' ? messageCount : totalTokens;
   
   if (current >= maxThreshold) {
     // Exceeded grace, must roll
@@ -323,54 +427,162 @@ export function truncateMessages(
   messages: MessageWithTokens[],
   targetTokens?: number,
   targetMessages?: number,
-  config?: ContextConfig
+  config?: ContextConfig,
+  targetCharacters?: number
 ): { kept: MessageWithTokens[]; dropped: number } {
-  // Truncate from the beginning, keeping most recent
+  // Truncate from the beginning, keeping most recent.
+  // Every supplied target contributes a candidate start index; the window has
+  // to satisfy all of them, so the deepest cut wins.
   
-  if (targetMessages && messages.length > targetMessages) {
-    const startIdx = messages.length - targetMessages;
-    return {
-      kept: messages.slice(startIdx),
-      dropped: startIdx,
-    };
+  if (messages.length === 0) {
+    return { kept: messages, dropped: 0 };
   }
   
-  if (targetTokens) {
-    let tokenSum = 0;
-    let startIdx = messages.length;
+  const candidateStartIndices: number[] = [];
+  
+  if (targetMessages !== undefined && messages.length > targetMessages) {
+    candidateStartIndices.push(messages.length - targetMessages);
+  }
+  
+  if (targetTokens !== undefined) {
+    candidateStartIndices.push(
+      startIndexForBudget(messages, targetTokens, m => m.tokens)
+    );
+  }
+  
+  if (targetCharacters !== undefined) {
+    candidateStartIndices.push(
+      startIndexForBudget(messages, targetCharacters, m => calculateCharacters([m.message]))
+    );
+  }
+  
+  if (candidateStartIndices.length === 0) {
+    // Default: use buffer from config
+    const buffer = config?.rolling.buffer ?? 20;
+    const unit = config?.rolling.unit ?? 'messages';
     
-    // Count from end backwards
-    for (let i = messages.length - 1; i >= 0; i--) {
-      tokenSum += messages[i]!.tokens;
-      if (tokenSum > targetTokens) {
-        startIdx = i + 1;
-        break;
+    if (unit === 'messages') {
+      const targetCount = Math.max(buffer * 2, messages.length - buffer);
+      if (messages.length > targetCount) {
+        candidateStartIndices.push(messages.length - targetCount);
       }
-      startIdx = i;
     }
-    
-    return {
-      kept: messages.slice(startIdx),
-      dropped: startIdx,
-    };
   }
   
-  // Default: use buffer from config
-  const buffer = config?.rolling.buffer ?? 20;
-  const unit = config?.rolling.unit ?? 'messages';
+  if (candidateStartIndices.length === 0) {
+    return { kept: messages, dropped: 0 };
+  }
   
-  if (unit === 'messages') {
-    const targetCount = Math.max(buffer * 2, messages.length - buffer);
-    if (messages.length > targetCount) {
-      const startIdx = messages.length - targetCount;
-      return {
-        kept: messages.slice(startIdx),
-        dropped: startIdx,
-      };
+  const startIdx = resolveWindowStart(messages, Math.max(0, ...candidateStartIndices));
+  
+  return {
+    kept: messages.slice(startIdx),
+    dropped: startIdx,
+  };
+}
+
+/**
+ * Walk backwards from the newest message, accumulating cost, and return the
+ * first index whose window fits the budget. Mirrors the original token walk:
+ * the message that tips the sum past the budget is excluded.
+ */
+function startIndexForBudget(
+  messages: MessageWithTokens[],
+  budget: number,
+  costOf: (message: MessageWithTokens) => number
+): number {
+  let sum = 0;
+  let startIdx = messages.length;
+  
+  for (let i = messages.length - 1; i >= 0; i--) {
+    sum += costOf(messages[i]!);
+    if (sum > budget) {
+      return i + 1;
     }
+    startIdx = i;
   }
   
-  return { kept: messages, dropped: 0 };
+  return startIdx;
+}
+
+/**
+ * Resolve the index the kept window actually starts at: the deepest cut at or
+ * after the requested one whose window is both non-empty and free of
+ * tool_results whose tool_use was dropped.
+ *
+ * Snapping forward and flooring at one message used to be two independent
+ * steps that could undo each other. When the newest message is a tool_result,
+ * the forward snap runs off the end of the array and the floor pulls the index
+ * back onto that same orphan - so truncating a final tool cycle to one message
+ * kept exactly the result, which is what the snap exists to prevent. A cut that
+ * cannot go forward now walks BACKWARD to the call that opens the final cycle:
+ * the window overshoots the requested size (reported through
+ * ContextInfo.residualOverflow) rather than shipping a corrupt transcript.
+ */
+function resolveWindowStart(
+  messages: MessageWithTokens[],
+  desiredStartIdx: number
+): number {
+  const lastIdx = messages.length - 1;
+  
+  if (desiredStartIdx <= 0) {
+    return 0;
+  }
+  
+  const windowIsCleanAt = markCleanWindowStarts(messages);
+  const cappedStartIdx = Math.min(desiredStartIdx, lastIdx);
+  
+  for (let idx = cappedStartIdx; idx <= lastIdx; idx++) {
+    if (windowIsCleanAt[idx]) {
+      return idx;
+    }
+  }
+  
+  for (let idx = cappedStartIdx - 1; idx >= 0; idx--) {
+    if (windowIsCleanAt[idx]) {
+      return idx;
+    }
+  }
+  
+  // No clean boundary exists anywhere: the supplied history itself carries a
+  // tool_result whose tool_use is not in it. Dropping messages cannot repair
+  // that, so the requested cut stands, floored at one message.
+  return cappedStartIdx;
+}
+
+/**
+ * For every index, whether the window starting there contains no tool_result
+ * whose tool_use is missing from it. A result is matched by a tool_use in an
+ * earlier message or earlier in its own message, so one backward pass carrying
+ * the still-unmatched result ids answers every index in linear time.
+ */
+function markCleanWindowStarts(messages: MessageWithTokens[]): boolean[] {
+  const windowIsCleanAt = new Array<boolean>(messages.length).fill(false);
+  const resultIdsAwaitingTheirCall = new Set<string>();
+  
+  for (let idx = messages.length - 1; idx >= 0; idx--) {
+    const callIdsInMessage = new Set<string>();
+    const resultIdsNeedingAnEarlierCall: string[] = [];
+    
+    for (const block of messages[idx]!.message.content) {
+      if (block.type === 'tool_use') {
+        callIdsInMessage.add(block.id);
+      } else if (block.type === 'tool_result' && !callIdsInMessage.has(block.toolUseId)) {
+        resultIdsNeedingAnEarlierCall.push(block.toolUseId);
+      }
+    }
+    
+    for (const callId of callIdsInMessage) {
+      resultIdsAwaitingTheirCall.delete(callId);
+    }
+    for (const resultId of resultIdsNeedingAnEarlierCall) {
+      resultIdsAwaitingTheirCall.add(resultId);
+    }
+    
+    windowIsCleanAt[idx] = resultIdsAwaitingTheirCall.size === 0;
+  }
+  
+  return windowIsCleanAt;
 }
 
 export function placeCacheMarkers(
@@ -386,7 +598,7 @@ export function placeCacheMarkers(
     return [];
   }
   
-  const numPoints = cacheConfig.points ?? 1;
+  const numPoints = Math.min(cacheConfig.points ?? 1, MAX_MODULE_CACHE_POINTS);
   const minTokens = cacheConfig.minTokens ?? 1024;
   const preferUser = cacheConfig.preferUserMessages ?? true;
   
@@ -403,19 +615,30 @@ export function placeCacheMarkers(
     const validMarkers = state.cacheMarkers.filter(m => currentIds.has(m.messageId));
     
     if (validMarkers.length > 0) {
-      // Recalculate token estimates for valid markers
-      return validMarkers.map(marker => {
-        const idx = messages.findIndex(m => getMessageId(m) === marker.messageId);
-        const tokenEstimate = messageTokens
-          .slice(0, idx + 1)
-          .reduce((sum, m) => sum + m.tokens, 0);
-        
-        return {
-          messageId: marker.messageId,
-          messageIndex: idx,
-          tokenEstimate,
-        };
-      });
+      // Recalculate token estimates for valid markers, then apply the same cap
+      // fresh placement obeys. The cap is a property of the REQUEST (Anthropic
+      // accepts four cache_control blocks and the builders spend from the same
+      // budget), so honouring it only on the fresh path let a state written
+      // before the cap existed - or by a caller that asked for four points -
+      // keep re-spending four slots on every subsequent call, which is where
+      // the builders' own marker turns the request into a 400. The deepest
+      // markers cover the longest prefixes, so those are the ones kept.
+      const retainedMarkers = validMarkers
+        .map(marker => {
+          const idx = messages.findIndex(m => getMessageId(m) === marker.messageId);
+          const tokenEstimate = messageTokens
+            .slice(0, idx + 1)
+            .reduce((sum, m) => sum + m.tokens, 0);
+          
+          return {
+            messageId: marker.messageId,
+            messageIndex: idx,
+            tokenEstimate,
+          };
+        })
+        .sort((a, b) => a.messageIndex - b.messageIndex);
+      
+      return retainedMarkers.slice(-MAX_MODULE_CACHE_POINTS);
     }
   }
   
@@ -455,7 +678,12 @@ export function placeCacheMarkers(
     
     // Adjust to user message if preferred
     if (preferUser) {
-      const adjusted = findNearestUserMessage(messages, markerIdx, messageTokens);
+      const adjusted = findNearestUserMessage(
+        messages,
+        markerIdx,
+        messageTokens,
+        config.assistantParticipant
+      );
       if (adjusted) {
         markerIdx = adjusted.index;
         markerTokens = adjusted.tokens;
@@ -482,21 +710,35 @@ export function placeCacheMarkers(
   return markers;
 }
 
+/** Assistant names assumed when the deployment configures none. */
+const LEGACY_ASSISTANT_PARTICIPANTS = ['claude', 'assistant', 'bot', 'ai'];
+
 function findNearestUserMessage(
   messages: NormalizedMessage[],
   startIdx: number,
-  messageTokens: MessageWithTokens[]
+  messageTokens: MessageWithTokens[],
+  assistantParticipant?: string
 ): { index: number; tokens: number } | null {
   // Search backwards for a user message (non-assistant participant)
   const maxSearch = 5;
   
+  // A deployment whose assistant is named anything else (Sol, a persona name)
+  // had every assistant turn classified as a user turn by the legacy list.
+  const assistantNames = assistantParticipant
+    ? [assistantParticipant.toLowerCase()]
+    : LEGACY_ASSISTANT_PARTICIPANTS;
+  
   let tokens = messageTokens.slice(0, startIdx + 1).reduce((sum, m) => sum + m.tokens, 0);
   
   for (let i = startIdx; i >= Math.max(0, startIdx - maxSearch); i--) {
     const msg = messages[i]!;
-    // Heuristic: if participant isn't a common assistant name, it's probably a user
-    const participant = msg.participant.toLowerCase();
-    const isUser = !['claude', 'assistant', 'bot', 'ai'].includes(participant);
+    // Heuristic: if participant isn't a known assistant name, it's probably a
+    // user. A message with no participant at all (role-shaped producers) falls
+    // through the same way rather than crashing the whole call.
+    const participant = typeof msg.participant === 'string'
+      ? msg.participant.toLowerCase()
+      : '';
+    const isUser = !assistantNames.includes(participant);
     
     if (isUser) {
       return { index: i, tokens };
@@ -522,6 +764,10 @@ export function applyCacheMarkers(
     if (markerIndices.has(idx)) {
       return {
         ...msg,
+        // cacheBreakpoint is the field every request builder reads; the
+        // metadata.cacheControl write has no reader inside membrane and is
+        // kept only for external consumers that may already read it.
+        cacheBreakpoint: true,
         metadata: {
           ...msg.metadata,
           cacheControl: { type: 'ephemeral' as const },