@animalabs/membrane 0.5.78 → 0.5.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cache-keepalive.d.ts +115 -0
- package/dist/cache-keepalive.d.ts.map +1 -0
- package/dist/cache-keepalive.js +0 -0
- package/dist/cache-keepalive.js.map +1 -0
- package/dist/cache-keepalive.test.d.ts +2 -0
- package/dist/cache-keepalive.test.d.ts.map +1 -0
- package/dist/cache-keepalive.test.js +206 -0
- package/dist/cache-keepalive.test.js.map +1 -0
- package/dist/floating-cache-marker.test.d.ts +2 -0
- package/dist/floating-cache-marker.test.d.ts.map +1 -0
- package/dist/floating-cache-marker.test.js +242 -0
- package/dist/floating-cache-marker.test.js.map +1 -0
- package/dist/formatters/anthropic-xml.d.ts.map +1 -1
- package/dist/formatters/anthropic-xml.js +30 -2
- package/dist/formatters/anthropic-xml.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/membrane.d.ts +7 -1
- package/dist/membrane.d.ts.map +1 -1
- package/dist/membrane.js +148 -5
- package/dist/membrane.js.map +1 -1
- package/dist/providers/anthropic.d.ts +20 -0
- package/dist/providers/anthropic.d.ts.map +1 -1
- package/dist/providers/anthropic.js +23 -3
- package/dist/providers/anthropic.js.map +1 -1
- package/dist/providers/bedrock.d.ts.map +1 -1
- package/dist/providers/bedrock.js +31 -2
- package/dist/providers/bedrock.js.map +1 -1
- package/dist/types/config.d.ts +5 -0
- package/dist/types/config.d.ts.map +1 -1
- package/dist/types/config.js.map +1 -1
- package/dist/types/content.d.ts +5 -0
- package/dist/types/content.d.ts.map +1 -1
- package/dist/types/content.js.map +1 -1
- package/dist/types/request.d.ts +13 -0
- package/dist/types/request.d.ts.map +1 -1
- package/dist/types/tools.d.ts +19 -0
- package/dist/types/tools.d.ts.map +1 -1
- package/dist/utils/tool-parser.d.ts +16 -2
- package/dist/utils/tool-parser.d.ts.map +1 -1
- package/dist/utils/tool-parser.js +143 -35
- package/dist/utils/tool-parser.js.map +1 -1
- package/package.json +3 -2
- package/src/cache-keepalive.test.ts +244 -0
- package/src/cache-keepalive.ts +385 -0
- package/src/floating-cache-marker.test.ts +261 -0
- package/src/formatters/anthropic-xml.ts +30 -2
- package/src/index.ts +13 -0
- package/src/membrane.ts +143 -5
- package/src/providers/anthropic.ts +46 -3
- package/src/providers/bedrock.ts +32 -1
- package/src/types/config.ts +6 -0
- package/src/types/content.ts +5 -0
- package/src/types/request.ts +14 -0
- package/src/types/tools.ts +20 -0
- package/src/utils/tool-parser.ts +147 -36
|
@@ -104,6 +104,7 @@ function toToolResult(block: ToolResultContent): ToolResult {
|
|
|
104
104
|
}
|
|
105
105
|
return {
|
|
106
106
|
toolUseId: block.toolUseId,
|
|
107
|
+
toolName: block.toolName,
|
|
107
108
|
content,
|
|
108
109
|
isError: block.isError ?? false,
|
|
109
110
|
};
|
|
@@ -160,6 +161,9 @@ export class AnthropicXmlFormatter implements PrefillFormatter {
|
|
|
160
161
|
// Track conversation state
|
|
161
162
|
let currentConversation: string[] = [];
|
|
162
163
|
let lastNonEmptyParticipant: string | null = null;
|
|
164
|
+
// True right after an unlabeled tool-results glue — the next assistant
|
|
165
|
+
// message continues the same turn, so it must not get a fresh label.
|
|
166
|
+
let lastWasToolResults = false;
|
|
163
167
|
|
|
164
168
|
// Track cache markers applied
|
|
165
169
|
let cacheMarkersApplied = 0;
|
|
@@ -298,15 +302,39 @@ export class AnthropicXmlFormatter implements PrefillFormatter {
|
|
|
298
302
|
const isBotMessage = message.participant === assistantParticipant;
|
|
299
303
|
const isContinuation = isBotMessage && lastNonEmptyParticipant === assistantParticipant && !hasToolResult;
|
|
300
304
|
|
|
305
|
+
const isPureToolResults =
|
|
306
|
+
hasToolResult && message.content.every((c) => c.type === 'tool_result');
|
|
307
|
+
|
|
301
308
|
if (isContinuation && isLastMessage) {
|
|
302
309
|
// Bot continuation - don't add prefix
|
|
303
310
|
continue;
|
|
304
311
|
} else if (isLastMessage && isEmpty) {
|
|
305
312
|
// Completion target - prefix added below
|
|
306
313
|
} else if (text) {
|
|
307
|
-
|
|
308
|
-
|
|
314
|
+
if (isPureToolResults) {
|
|
315
|
+
// Tool results are not speech: replay them exactly as they were
|
|
316
|
+
// injected live — inside the assistant flow, unlabeled (legacy
|
|
317
|
+
// convention). A participant prefix here re-attributes the
|
|
318
|
+
// harness's injection as someone's utterance, and the model then
|
|
319
|
+
// reads the same result in two attributions across compiles
|
|
320
|
+
// (D2 of the Evander 2026-08-08 scaffold-leak analysis).
|
|
321
|
+
currentConversation.push(`${text}${this.config.messageDelimiter}`);
|
|
322
|
+
lastWasToolResults = true;
|
|
323
|
+
} else if (isBotMessage && lastWasToolResults) {
|
|
324
|
+
// The round after injected results continues the same assistant
|
|
325
|
+
// turn — no fresh label mid-turn, matching what the model lived.
|
|
326
|
+
// (If a turn ended exactly on a results injection, the next
|
|
327
|
+
// assistant turn glues here unlabeled — a minor cost the message
|
|
328
|
+
// model can't distinguish; a turn id would be needed.)
|
|
329
|
+
currentConversation.push(`${text}${this.config.messageDelimiter}`);
|
|
330
|
+
lastWasToolResults = false;
|
|
309
331
|
lastNonEmptyParticipant = message.participant;
|
|
332
|
+
} else {
|
|
333
|
+
currentConversation.push(`${message.participant}: ${text}${this.config.messageDelimiter}`);
|
|
334
|
+
lastWasToolResults = false;
|
|
335
|
+
if (!hasToolResult) {
|
|
336
|
+
lastNonEmptyParticipant = message.participant;
|
|
337
|
+
}
|
|
310
338
|
}
|
|
311
339
|
}
|
|
312
340
|
|
package/src/index.ts
CHANGED
|
@@ -24,3 +24,16 @@ export * from './formatters/index.js';
|
|
|
24
24
|
|
|
25
25
|
// Context management
|
|
26
26
|
export * from './context/index.js';
|
|
27
|
+
|
|
28
|
+
// Prompt-cache keepalive (Anthropic 1h cache)
|
|
29
|
+
export {
|
|
30
|
+
CacheKeepalive,
|
|
31
|
+
ineligibleReason as cacheKeepaliveIneligibleReason,
|
|
32
|
+
lineageKey as cacheLineageKey,
|
|
33
|
+
} from './cache-keepalive.js';
|
|
34
|
+
export type {
|
|
35
|
+
CacheKeepaliveConfig,
|
|
36
|
+
KeepaliveEvent,
|
|
37
|
+
KeepaliveLane,
|
|
38
|
+
KeepaliveSend,
|
|
39
|
+
} from './cache-keepalive.js';
|
package/src/membrane.ts
CHANGED
|
@@ -408,10 +408,16 @@ export class Membrane {
|
|
|
408
408
|
// Track the initial prefill length so we can extract only NEW content for response
|
|
409
409
|
// Also track what block type we're inside at the end of prefill
|
|
410
410
|
let initialPrefillLength = 0;
|
|
411
|
+
// Watermark for per-round delta text (ToolContext.roundPreamble): start of
|
|
412
|
+
// the CURRENT round's model text in parser-accumulated coordinates.
|
|
413
|
+
// Advanced past each round's injected results push, so injected
|
|
414
|
+
// <function_results> XML never enters a round's delta.
|
|
415
|
+
let roundStartLen = 0;
|
|
411
416
|
let initialBlockType: 'thinking' | 'tool_call' | 'tool_result' | null = null;
|
|
412
417
|
if (prefillResult.assistantPrefill) {
|
|
413
418
|
parser.push(prefillResult.assistantPrefill);
|
|
414
419
|
initialPrefillLength = prefillResult.assistantPrefill.length;
|
|
420
|
+
roundStartLen = initialPrefillLength;
|
|
415
421
|
// Capture what block type we're inside after prefill (if any)
|
|
416
422
|
if (parser.isInsideBlock()) {
|
|
417
423
|
const blockType = parser.getCurrentBlockType();
|
|
@@ -699,6 +705,7 @@ export class Membrane {
|
|
|
699
705
|
const context: ToolContext = {
|
|
700
706
|
rawText: parsed.fullMatch,
|
|
701
707
|
preamble: parsed.beforeText.slice(initialPrefillLength),
|
|
708
|
+
roundPreamble: parsed.beforeText.slice(roundStartLen),
|
|
702
709
|
depth: toolDepth,
|
|
703
710
|
previousResults: executedToolResults,
|
|
704
711
|
accumulated: parser.getAccumulated().slice(initialPrefillLength),
|
|
@@ -711,6 +718,14 @@ export class Membrane {
|
|
|
711
718
|
);
|
|
712
719
|
}
|
|
713
720
|
|
|
721
|
+
// Backfill tool names for the legacy XML result rendering
|
|
722
|
+
// (<result><tool_name>…</tool_name><stdout>…) when the executor
|
|
723
|
+
// didn't supply them.
|
|
724
|
+
const callNames = new Map(parsed.calls.map((c) => [c.id, c.name]));
|
|
725
|
+
for (const r of results) {
|
|
726
|
+
if (!r.toolName) r.toolName = callNames.get(r.toolUseId);
|
|
727
|
+
}
|
|
728
|
+
|
|
714
729
|
// Track the tool results
|
|
715
730
|
executedToolResults.push(...results);
|
|
716
731
|
|
|
@@ -832,6 +847,10 @@ export class Membrane {
|
|
|
832
847
|
);
|
|
833
848
|
}
|
|
834
849
|
|
|
850
|
+
// Next round's model text starts after everything injected this
|
|
851
|
+
// round (results XML, image-split tags, thinking opener).
|
|
852
|
+
roundStartLen = parser.getAccumulated().length;
|
|
853
|
+
|
|
835
854
|
// Reset parser state for new streaming iteration. Tool rounds
|
|
836
855
|
// are the caller's work — they count against maxToolDepth only,
|
|
837
856
|
// never against the resumption guards (issue #39 review).
|
|
@@ -986,7 +1005,7 @@ export class Membrane {
|
|
|
986
1005
|
// Tool execution loop
|
|
987
1006
|
while (toolDepth <= maxToolDepth) {
|
|
988
1007
|
// Build provider request with native tools
|
|
989
|
-
const providerRequest = this.buildNativeToolRequest(request, messages);
|
|
1008
|
+
const providerRequest = this.buildNativeToolRequest(request, messages, toolDepth > 0);
|
|
990
1009
|
|
|
991
1010
|
// Stream from provider
|
|
992
1011
|
let textAccumulated = '';
|
|
@@ -1178,12 +1197,20 @@ export class Membrane {
|
|
|
1178
1197
|
}
|
|
1179
1198
|
}
|
|
1180
1199
|
|
|
1200
|
+
/** See the floating-cache-marker block in buildNativeToolRequest. */
|
|
1201
|
+
private floatBudgetWarned = false;
|
|
1202
|
+
|
|
1181
1203
|
/**
|
|
1182
|
-
* Build a provider request with native tool support
|
|
1204
|
+
* Build a provider request with native tool support.
|
|
1205
|
+
*
|
|
1206
|
+
* `toolLoopRebuild` is true when this build is a tool-loop continuation
|
|
1207
|
+
* (toolDepth > 0) rather than the turn's first request — the only case
|
|
1208
|
+
* where the floating cache marker applies.
|
|
1183
1209
|
*/
|
|
1184
1210
|
private buildNativeToolRequest(
|
|
1185
1211
|
request: NormalizedRequest,
|
|
1186
|
-
messages: typeof request.messages
|
|
1212
|
+
messages: typeof request.messages,
|
|
1213
|
+
toolLoopRebuild = false
|
|
1187
1214
|
): any {
|
|
1188
1215
|
// Provider-native formatters own their complete input-item shape. The
|
|
1189
1216
|
// legacy implementation below is intentionally Anthropic-specific; using
|
|
@@ -1335,7 +1362,17 @@ export class Membrane {
|
|
|
1335
1362
|
// tool_results to `messages`. Any unmatched tool_use that reaches
|
|
1336
1363
|
// this splice is upstream stranding (the bug class this fix exists
|
|
1337
1364
|
// to catch) — `[pending]` is exactly the right synthesis.
|
|
1338
|
-
|
|
1365
|
+
// A synthesized [pending] tool_result's bytes are rewritten when the
|
|
1366
|
+
// real result lands — the floating-marker block below must not cache
|
|
1367
|
+
// past one. `synthetic_pending_result` (not the downstream
|
|
1368
|
+
// cache_suppressed_for_synthetic, which only fires when a marker was
|
|
1369
|
+
// actually stripped) is the root condition.
|
|
1370
|
+
let pendingResultSynthesized = false;
|
|
1371
|
+
const normalized = normalizeToolPairs(providerMessages, {
|
|
1372
|
+
onEvent: (e) => {
|
|
1373
|
+
if (e.kind === 'synthetic_pending_result') pendingResultSynthesized = true;
|
|
1374
|
+
},
|
|
1375
|
+
});
|
|
1339
1376
|
const mergedMessages = mergeConsecutiveRoles(normalized.messages);
|
|
1340
1377
|
|
|
1341
1378
|
// Convert tools to provider format.
|
|
@@ -1368,6 +1405,90 @@ export class Membrane {
|
|
|
1368
1405
|
);
|
|
1369
1406
|
}
|
|
1370
1407
|
|
|
1408
|
+
// ------------------------------------------------------------------
|
|
1409
|
+
// Floating cache marker: incremental prompt caching inside the native
|
|
1410
|
+
// tool loop. Message breakpoints are placed by the context strategy at
|
|
1411
|
+
// compile time — once per turn — but this builder re-runs on every
|
|
1412
|
+
// tool round with that round's messages appended, so the deepest
|
|
1413
|
+
// upstream marker stays glued to the turn-start snapshot and each
|
|
1414
|
+
// rebuild re-pays the entire appended suffix at full input price
|
|
1415
|
+
// (qa-ops incident, 2026-08-20: two subagents re-sent a suffix growing
|
|
1416
|
+
// to ~118k tokens ~30 times each — ~5.3M uncached tokens in 18 min —
|
|
1417
|
+
// with their one marker sitting on message 2 of 61).
|
|
1418
|
+
//
|
|
1419
|
+
// The tool loop only ever appends, so a marker riding the newest
|
|
1420
|
+
// message yields the intended incremental pattern: each round writes
|
|
1421
|
+
// its delta and cache-reads everything before it.
|
|
1422
|
+
//
|
|
1423
|
+
// Authority contract: the float spends only the RESIDUAL breakpoint
|
|
1424
|
+
// budget (Anthropic allows 4 cache_control including tools/system).
|
|
1425
|
+
// Upstream markers are never displaced or stripped — if they fill all
|
|
1426
|
+
// 4 slots the float is withheld (with a warning) and behavior is
|
|
1427
|
+
// exactly pre-float. With 2+ slots free, the previous round's
|
|
1428
|
+
// endpoint is marked too: a wide parallel-tool round can append more
|
|
1429
|
+
// blocks than the provider's ~20-block backward search covers, which
|
|
1430
|
+
// would orphan the previous round's cache entry behind an unmarked
|
|
1431
|
+
// boundary.
|
|
1432
|
+
//
|
|
1433
|
+
// Skipped when the normalizer synthesized a [pending] tool_result:
|
|
1434
|
+
// those bytes are rewritten when the real result lands, and caching
|
|
1435
|
+
// past them poisons the prefix — the same rationale as the
|
|
1436
|
+
// normalizer's phase 5.5 cache suppression.
|
|
1437
|
+
// ------------------------------------------------------------------
|
|
1438
|
+
const floatingEnabled =
|
|
1439
|
+
request.floatingCacheMarker ?? this.config.defaultFloatingCacheMarker ?? true;
|
|
1440
|
+
if (toolLoopRebuild && floatingEnabled && cacheControl && !pendingResultSynthesized) {
|
|
1441
|
+
// Residuum from a RECOUNT of the constructed wire artifacts, not the
|
|
1442
|
+
// running messageBreakpoints tally — the tally diverges from the wire
|
|
1443
|
+
// in both directions (mirrors NativeFormatter's recount, same bug
|
|
1444
|
+
// class as the Sill 2026-07-25 wedge): a message-level breakpoint
|
|
1445
|
+
// landing on a block already carrying stale cache_control is one
|
|
1446
|
+
// physical marker counted twice, and a pre-marked system block is a
|
|
1447
|
+
// real wire marker the tally never sees. Counted post-fallback and
|
|
1448
|
+
// post-normalize, so fallback spend and phase-5.5 suppression are
|
|
1449
|
+
// both reflected.
|
|
1450
|
+
let wireMarkers = 0;
|
|
1451
|
+
for (const m of mergedMessages) {
|
|
1452
|
+
if (!Array.isArray(m.content)) continue;
|
|
1453
|
+
for (const b of m.content as Array<Record<string, unknown>>) {
|
|
1454
|
+
if (b.cache_control) wireMarkers++;
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
if (tools) for (const t of tools) { if (t.cache_control) wireMarkers++; }
|
|
1458
|
+
if (Array.isArray(system)) {
|
|
1459
|
+
for (const b of system as Array<Record<string, unknown>>) {
|
|
1460
|
+
if (b.cache_control) wireMarkers++;
|
|
1461
|
+
}
|
|
1462
|
+
}
|
|
1463
|
+
let residuum = 4 - wireMarkers;
|
|
1464
|
+
if (residuum <= 0) {
|
|
1465
|
+
if (!this.floatBudgetWarned) {
|
|
1466
|
+
this.floatBudgetWarned = true;
|
|
1467
|
+
console.warn(
|
|
1468
|
+
`[membrane] floating cache marker withheld: upstream markers already ` +
|
|
1469
|
+
`occupy all 4 cache_control slots (${wireMarkers} on the wire). ` +
|
|
1470
|
+
`Tool-round suffixes will not cache incrementally.`
|
|
1471
|
+
);
|
|
1472
|
+
}
|
|
1473
|
+
} else {
|
|
1474
|
+
// Newest message first; then the previous round's endpoint (two
|
|
1475
|
+
// wire messages back: [..., prevResults, assistant, results]).
|
|
1476
|
+
const targets = [mergedMessages.length - 1, mergedMessages.length - 3];
|
|
1477
|
+
for (const mi of targets) {
|
|
1478
|
+
if (residuum <= 0 || mi < 0) continue;
|
|
1479
|
+
const content = mergedMessages[mi]?.content;
|
|
1480
|
+
if (!Array.isArray(content) || content.length === 0) continue;
|
|
1481
|
+
const bpIdx = lastCacheableBlockIndex(content as Array<Record<string, unknown>>);
|
|
1482
|
+
if (bpIdx < 0) continue;
|
|
1483
|
+
// Already a breakpoint here (e.g. the strategy's own end marker
|
|
1484
|
+
// on the turn's first rebuild) — nothing to add.
|
|
1485
|
+
if ((content[bpIdx] as Record<string, unknown>).cache_control) continue;
|
|
1486
|
+
(content[bpIdx] as Record<string, unknown>).cache_control = cacheControl;
|
|
1487
|
+
residuum--;
|
|
1488
|
+
}
|
|
1489
|
+
}
|
|
1490
|
+
}
|
|
1491
|
+
|
|
1371
1492
|
// Build thinking config for native extended thinking (budget clamped to max_tokens)
|
|
1372
1493
|
// Fable/Mythos models: thinking is always on and unconfigurable; sampling params are removed.
|
|
1373
1494
|
// Sending thinking config or temperature returns a 400 — omit both entirely.
|
|
@@ -2314,10 +2435,14 @@ export class Membrane {
|
|
|
2314
2435
|
|
|
2315
2436
|
// Initialize parser with prefill content
|
|
2316
2437
|
let initialPrefillLength = 0;
|
|
2438
|
+
// Watermark for per-round delta text (ToolContext.roundPreamble) — see
|
|
2439
|
+
// streamWithXmlTools for rationale. Advanced past each results push.
|
|
2440
|
+
let roundStartLen = 0;
|
|
2317
2441
|
let initialBlockType: 'thinking' | 'tool_call' | 'tool_result' | null = null;
|
|
2318
2442
|
if (prefillResult.assistantPrefill) {
|
|
2319
2443
|
parser.push(prefillResult.assistantPrefill);
|
|
2320
2444
|
initialPrefillLength = prefillResult.assistantPrefill.length;
|
|
2445
|
+
roundStartLen = initialPrefillLength;
|
|
2321
2446
|
if (parser.isInsideBlock()) {
|
|
2322
2447
|
const blockType = parser.getCurrentBlockType();
|
|
2323
2448
|
if (blockType === 'thinking' || blockType === 'tool_call' || blockType === 'tool_result') {
|
|
@@ -2564,6 +2689,7 @@ export class Membrane {
|
|
|
2564
2689
|
const context: ToolContext = {
|
|
2565
2690
|
rawText: parsed.fullMatch,
|
|
2566
2691
|
preamble: parsed.beforeText.slice(initialPrefillLength),
|
|
2692
|
+
roundPreamble: parsed.beforeText.slice(roundStartLen),
|
|
2567
2693
|
depth: toolDepth,
|
|
2568
2694
|
previousResults: executedToolResults,
|
|
2569
2695
|
accumulated: parser.getAccumulated().slice(initialPrefillLength),
|
|
@@ -2578,6 +2704,14 @@ export class Membrane {
|
|
|
2578
2704
|
|
|
2579
2705
|
const { results, injectedMessages } = await stream.requestToolExecution(toolCallsEvent);
|
|
2580
2706
|
|
|
2707
|
+
// Backfill tool names for the legacy XML result rendering
|
|
2708
|
+
// (<result><tool_name>…</tool_name><stdout>…) when the executor
|
|
2709
|
+
// didn't supply them.
|
|
2710
|
+
const yieldCallNames = new Map(parsed.calls.map((c) => [c.id, c.name]));
|
|
2711
|
+
for (const r of results) {
|
|
2712
|
+
if (!r.toolName) r.toolName = yieldCallNames.get(r.toolUseId);
|
|
2713
|
+
}
|
|
2714
|
+
|
|
2581
2715
|
// Mid-turn injected messages are not supported on the XML prefill
|
|
2582
2716
|
// path: the continuation is an assistant prefill over an XML
|
|
2583
2717
|
// transcript, not a message array, so there is no user envelope
|
|
@@ -2725,6 +2859,10 @@ export class Membrane {
|
|
|
2725
2859
|
);
|
|
2726
2860
|
}
|
|
2727
2861
|
|
|
2862
|
+
// Next round's model text starts after everything injected this
|
|
2863
|
+
// round (results XML, image-split tags, thinking opener).
|
|
2864
|
+
roundStartLen = parser.getAccumulated().length;
|
|
2865
|
+
|
|
2728
2866
|
// Tool rounds are the caller's work — they count against
|
|
2729
2867
|
// maxToolDepth only, never against the resumption guards
|
|
2730
2868
|
// (issue #39 review: the uncapped tool-loop contract stands).
|
|
@@ -2877,7 +3015,7 @@ export class Membrane {
|
|
|
2877
3015
|
}
|
|
2878
3016
|
|
|
2879
3017
|
// Build provider request with native tools
|
|
2880
|
-
const providerRequest = this.buildNativeToolRequest(request, messages);
|
|
3018
|
+
const providerRequest = this.buildNativeToolRequest(request, messages, toolDepth > 0);
|
|
2881
3019
|
|
|
2882
3020
|
// Stream from provider
|
|
2883
3021
|
let textAccumulated = '';
|
|
@@ -22,6 +22,7 @@ import {
|
|
|
22
22
|
abortError,
|
|
23
23
|
} from '../types/index.js';
|
|
24
24
|
import { flattenRootSchemaUnion } from './anthropic-tool-schema.js';
|
|
25
|
+
import { CacheKeepalive, type CacheKeepaliveConfig } from '../cache-keepalive.js';
|
|
25
26
|
|
|
26
27
|
// ============================================================================
|
|
27
28
|
// Model capability gates
|
|
@@ -146,6 +147,15 @@ export interface AnthropicAdapterConfig {
|
|
|
146
147
|
|
|
147
148
|
/** Default max tokens */
|
|
148
149
|
defaultMaxTokens?: number;
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Prompt-cache keepalive: hold this agent's cached prefix warm across idle
|
|
153
|
+
* gaps by replaying the last request with `max_tokens: 0`, which refreshes
|
|
154
|
+
* the entry's TTL at cache-READ price instead of letting it expire into a
|
|
155
|
+
* 2x cache write on the next wake. See `../cache-keepalive.ts`.
|
|
156
|
+
* Pass `{ enabled: false }` to turn off.
|
|
157
|
+
*/
|
|
158
|
+
cacheKeepalive?: CacheKeepaliveConfig;
|
|
149
159
|
}
|
|
150
160
|
|
|
151
161
|
// ============================================================================
|
|
@@ -161,6 +171,8 @@ export class AnthropicAdapter implements ProviderAdapter {
|
|
|
161
171
|
* the SDK rather than merging, so when we add a per-request beta we must
|
|
162
172
|
* re-carry this one alongside it or auth breaks. */
|
|
163
173
|
private defaultBeta: string | undefined;
|
|
174
|
+
/** Holds idle agents' cached prefixes warm; undefined when disabled. */
|
|
175
|
+
readonly cacheKeepalive: CacheKeepalive | undefined;
|
|
164
176
|
|
|
165
177
|
constructor(config: AnthropicAdapterConfig = {}) {
|
|
166
178
|
const clientOptions: ClientOptions = {
|
|
@@ -178,6 +190,19 @@ export class AnthropicAdapter implements ProviderAdapter {
|
|
|
178
190
|
|
|
179
191
|
this.client = new Anthropic(clientOptions);
|
|
180
192
|
this.defaultMaxTokens = config.defaultMaxTokens ?? 4096;
|
|
193
|
+
|
|
194
|
+
this.cacheKeepalive = config.cacheKeepalive?.enabled === false
|
|
195
|
+
? undefined
|
|
196
|
+
: new CacheKeepalive(
|
|
197
|
+
// Replay path. Deliberately bypasses buildRequest(): the payload is
|
|
198
|
+
// the already-built wire request from a real call, and rebuilding it
|
|
199
|
+
// risks a byte diff that silently converts a 0.1x read into a 2x write.
|
|
200
|
+
async (wire, headers) => await this.client.messages.create(
|
|
201
|
+
wire as unknown as Anthropic.MessageCreateParamsNonStreaming,
|
|
202
|
+
headers ? { headers } : undefined,
|
|
203
|
+
),
|
|
204
|
+
config.cacheKeepalive ?? {},
|
|
205
|
+
);
|
|
181
206
|
}
|
|
182
207
|
|
|
183
208
|
supportsModel(modelId: string): boolean {
|
|
@@ -192,10 +217,15 @@ export class AnthropicAdapter implements ProviderAdapter {
|
|
|
192
217
|
const fullRequest = { ...anthropicRequest, stream: false as const };
|
|
193
218
|
options?.onRequest?.(fullRequest);
|
|
194
219
|
|
|
220
|
+
const headers = this.betaHeaders(request);
|
|
221
|
+
this.cacheKeepalive?.record(
|
|
222
|
+
fullRequest as unknown as Record<string, unknown>, headers, 'complete',
|
|
223
|
+
);
|
|
224
|
+
|
|
195
225
|
try {
|
|
196
226
|
const response = await this.client.messages.create(fullRequest, {
|
|
197
227
|
signal: options?.signal,
|
|
198
|
-
headers
|
|
228
|
+
headers,
|
|
199
229
|
});
|
|
200
230
|
|
|
201
231
|
return this.parseResponse(response, fullRequest);
|
|
@@ -214,6 +244,14 @@ export class AnthropicAdapter implements ProviderAdapter {
|
|
|
214
244
|
const fullRequest = { ...anthropicRequest, stream: true };
|
|
215
245
|
options?.onRequest?.(fullRequest);
|
|
216
246
|
|
|
247
|
+
// Snapshot the primary lane's prefix so it can be held warm across idle
|
|
248
|
+
// gaps. `stream: true` is dropped at replay time (transport, not cache key).
|
|
249
|
+
this.cacheKeepalive?.record(
|
|
250
|
+
fullRequest as unknown as Record<string, unknown>,
|
|
251
|
+
this.betaHeaders(request),
|
|
252
|
+
'stream',
|
|
253
|
+
);
|
|
254
|
+
|
|
217
255
|
// Idle timeout: abort if no SSE event arrives within the deadline.
|
|
218
256
|
// The SDK's timeout only covers the initial HTTP response headers;
|
|
219
257
|
// once streaming starts, a silently dropped connection waits forever.
|
|
@@ -664,7 +702,12 @@ export class AnthropicAdapter implements ProviderAdapter {
|
|
|
664
702
|
return authError(message, error, rawRequest);
|
|
665
703
|
}
|
|
666
704
|
|
|
667
|
-
|
|
705
|
+
// Context-length is a client-side request-shape problem — it only ever
|
|
706
|
+
// arrives as a 400 (invalid_request_error). Without the status guard, a
|
|
707
|
+
// transient 5xx whose body happens to contain "context" or "too long"
|
|
708
|
+
// (e.g. "Internal error: context processing failed") was misclassified
|
|
709
|
+
// as non-retryable context_length, silently suppressing retries.
|
|
710
|
+
if (status === 400 && (message.includes('context') || message.includes('too long'))) {
|
|
668
711
|
return contextLengthError(message, error, rawRequest);
|
|
669
712
|
}
|
|
670
713
|
|
|
@@ -780,7 +823,7 @@ function toAnthropicToolResultContent(
|
|
|
780
823
|
* can lose or mislabel mediaType (e.g. a PNG tagged image/jpeg), which the
|
|
781
824
|
* Anthropic API rejects with a 400. Trust the bytes; fall back to the declared
|
|
782
825
|
* type, then jpeg. */
|
|
783
|
-
function detectImageMediaType(data: string | undefined, fallback?: string): string {
|
|
826
|
+
export function detectImageMediaType(data: string | undefined, fallback?: string): string {
|
|
784
827
|
try {
|
|
785
828
|
const b = Buffer.from((data || "").slice(0, 24), "base64");
|
|
786
829
|
if (b[0]===0x89&&b[1]===0x50&&b[2]===0x4e&&b[3]===0x47) return "image/png";
|
package/src/providers/bedrock.ts
CHANGED
|
@@ -25,6 +25,7 @@ import {
|
|
|
25
25
|
INTERLEAVED_THINKING_BETA,
|
|
26
26
|
needsInterleavedThinkingBeta,
|
|
27
27
|
thinkingEnabled,
|
|
28
|
+
detectImageMediaType,
|
|
28
29
|
} from './anthropic.js';
|
|
29
30
|
|
|
30
31
|
// ============================================================================
|
|
@@ -404,6 +405,25 @@ export class BedrockAdapter implements ProviderAdapter {
|
|
|
404
405
|
// caching works without the field, so strip just the ttl and keep the
|
|
405
406
|
// breakpoint. Transport quirks belong to the transport, not to every
|
|
406
407
|
// caller that sets cacheTtl. (Connectome issue #35.)
|
|
408
|
+
// Bedrock is strict about the wire shape of image sources: internal blocks
|
|
409
|
+
// carry camelCase `mediaType`, the API requires snake_case `media_type`.
|
|
410
|
+
// The Anthropic adapter converts on both paths (toAnthropicContent /
|
|
411
|
+
// toAnthropicToolResultContent); without the same conversion here a
|
|
412
|
+
// tool-returned image 400s with "media_type: Field required" mid-turn
|
|
413
|
+
// (observed on eidoverse snapshot results, 2026-08-08). Applies to
|
|
414
|
+
// top-level image blocks AND images nested inside tool_result content.
|
|
415
|
+
const toWireImage = (block: any): any => {
|
|
416
|
+
const source = block.source;
|
|
417
|
+
if (!source || source.type !== 'base64') return block;
|
|
418
|
+
const { mediaType, media_type, ...restSource } = source;
|
|
419
|
+
return {
|
|
420
|
+
...block,
|
|
421
|
+
source: {
|
|
422
|
+
...restSource,
|
|
423
|
+
media_type: detectImageMediaType(source.data, (media_type ?? mediaType) as string),
|
|
424
|
+
},
|
|
425
|
+
};
|
|
426
|
+
};
|
|
407
427
|
const sanitizedMessages = (request.messages as any[]).map((msg: any) => {
|
|
408
428
|
if (!Array.isArray(msg.content)) return msg;
|
|
409
429
|
return {
|
|
@@ -411,7 +431,18 @@ export class BedrockAdapter implements ProviderAdapter {
|
|
|
411
431
|
content: msg.content.map((block: any) => {
|
|
412
432
|
if (block.type === 'image' && block.sourceUrl !== undefined) {
|
|
413
433
|
const { sourceUrl, ...rest } = block;
|
|
414
|
-
return stripCacheTtl(rest);
|
|
434
|
+
return stripCacheTtl(toWireImage(rest));
|
|
435
|
+
}
|
|
436
|
+
if (block.type === 'image') {
|
|
437
|
+
return stripCacheTtl(toWireImage(block));
|
|
438
|
+
}
|
|
439
|
+
if (block.type === 'tool_result' && Array.isArray(block.content)) {
|
|
440
|
+
return stripCacheTtl({
|
|
441
|
+
...block,
|
|
442
|
+
content: block.content.map((inner: any) =>
|
|
443
|
+
inner?.type === 'image' ? toWireImage(inner) : inner,
|
|
444
|
+
),
|
|
445
|
+
});
|
|
415
446
|
}
|
|
416
447
|
return stripCacheTtl(block);
|
|
417
448
|
}),
|
package/src/types/config.ts
CHANGED
|
@@ -180,6 +180,12 @@ export interface MembraneConfig {
|
|
|
180
180
|
*/
|
|
181
181
|
defaultPromptCaching?: boolean;
|
|
182
182
|
|
|
183
|
+
/**
|
|
184
|
+
* Default for request.floatingCacheMarker when the request doesn't set it.
|
|
185
|
+
* Default: true. See NormalizedRequest.floatingCacheMarker.
|
|
186
|
+
*/
|
|
187
|
+
defaultFloatingCacheMarker?: boolean;
|
|
188
|
+
|
|
183
189
|
/**
|
|
184
190
|
* Prefill formatter for message serialization and response parsing.
|
|
185
191
|
* Controls how messages are formatted for the API and how responses are parsed.
|
package/src/types/content.ts
CHANGED
|
@@ -128,6 +128,11 @@ export interface ToolUseContent {
|
|
|
128
128
|
export interface ToolResultContent {
|
|
129
129
|
type: 'tool_result';
|
|
130
130
|
toolUseId: string;
|
|
131
|
+
/**
|
|
132
|
+
* Tool name, persisted so XML replay can reconstruct the legacy
|
|
133
|
+
* `<tool_name>` element byte-identically to the live injection.
|
|
134
|
+
*/
|
|
135
|
+
toolName?: string;
|
|
131
136
|
content: string | ContentBlock[];
|
|
132
137
|
isError?: boolean;
|
|
133
138
|
/**
|
package/src/types/request.ts
CHANGED
|
@@ -159,6 +159,20 @@ export interface NormalizedRequest {
|
|
|
159
159
|
*/
|
|
160
160
|
cacheTtl?: '5m' | '1h';
|
|
161
161
|
|
|
162
|
+
/**
|
|
163
|
+
* Float a trailing cache_control marker onto the newest message when the
|
|
164
|
+
* native tool loop rebuilds the request between tool-execution rounds, so
|
|
165
|
+
* the growing tool-round suffix caches incrementally (each round writes
|
|
166
|
+
* only its delta and cache-reads everything before it). Placed only from
|
|
167
|
+
* the request's *residual* breakpoint budget — the marker is withheld when
|
|
168
|
+
* upstream markers already occupy all 4 Anthropic cache_control slots —
|
|
169
|
+
* so upstream breakpoints are never displaced or stripped.
|
|
170
|
+
* Defaults to true (when promptCaching is enabled). Set false for context
|
|
171
|
+
* strategies whose request prefix churns between rounds, where a trailing
|
|
172
|
+
* marker would be pure cache-write cost.
|
|
173
|
+
*/
|
|
174
|
+
floatingCacheMarker?: boolean;
|
|
175
|
+
|
|
162
176
|
/**
|
|
163
177
|
* Context prefix for simulacrum seeding.
|
|
164
178
|
* Injected as first assistant message (before conversation history).
|
package/src/types/tools.ts
CHANGED
|
@@ -37,6 +37,13 @@ export interface ToolCall {
|
|
|
37
37
|
|
|
38
38
|
export interface ToolResult {
|
|
39
39
|
toolUseId: string;
|
|
40
|
+
/**
|
|
41
|
+
* Tool name, for the legacy XML result rendering
|
|
42
|
+
* (`<result><tool_name>…</tool_name><stdout>…</stdout></result>`).
|
|
43
|
+
* Optional: XML paths backfill it from the round's parsed calls when the
|
|
44
|
+
* executor didn't supply it.
|
|
45
|
+
*/
|
|
46
|
+
toolName?: string;
|
|
40
47
|
/**
|
|
41
48
|
* Result content - can be string or structured content blocks (for images).
|
|
42
49
|
* For XML mode, images are noted in text. For native mode, passed as content blocks.
|
|
@@ -63,6 +70,19 @@ export interface ToolContext {
|
|
|
63
70
|
/** Text before the tool calls (already streamed to user) */
|
|
64
71
|
preamble: string;
|
|
65
72
|
|
|
73
|
+
/**
|
|
74
|
+
* XML mode only: THIS round's model-authored text — the slice of the
|
|
75
|
+
* turn between the end of the previous round's injected results and this
|
|
76
|
+
* round's <function_calls> opener. Unlike `preamble` (cumulative: the
|
|
77
|
+
* whole turn so far, including harness-injected <function_results> XML),
|
|
78
|
+
* this never repeats earlier rounds and never contains injected results.
|
|
79
|
+
* Consumers persisting per-round assistant text must prefer this field —
|
|
80
|
+
* persisting the cumulative `preamble` per round stores each round's text
|
|
81
|
+
* N times and re-persists injected results as model text (the Evander
|
|
82
|
+
* 2026-08-08 scaffold-leak pyramid).
|
|
83
|
+
*/
|
|
84
|
+
roundPreamble?: string;
|
|
85
|
+
|
|
66
86
|
/** Current depth in tool execution loop */
|
|
67
87
|
depth: number;
|
|
68
88
|
|