@animalabs/membrane 0.5.80 → 0.5.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cache-wire-receipt.d.ts +13 -0
- package/dist/cache-wire-receipt.d.ts.map +1 -0
- package/dist/cache-wire-receipt.js +108 -0
- package/dist/cache-wire-receipt.js.map +1 -0
- package/dist/context/index.d.ts +1 -1
- package/dist/context/index.d.ts.map +1 -1
- package/dist/context/index.js +1 -1
- package/dist/context/index.js.map +1 -1
- package/dist/context/process.d.ts +3 -1
- package/dist/context/process.d.ts.map +1 -1
- package/dist/context/process.js +230 -49
- package/dist/context/process.js.map +1 -1
- package/dist/context/types.d.ts +30 -0
- package/dist/context/types.d.ts.map +1 -1
- package/dist/context/types.js +16 -0
- package/dist/context/types.js.map +1 -1
- package/dist/floating-cache-marker.test.js +21 -0
- package/dist/floating-cache-marker.test.js.map +1 -1
- package/dist/formatters/anthropic-xml.d.ts +2 -0
- package/dist/formatters/anthropic-xml.d.ts.map +1 -1
- package/dist/formatters/anthropic-xml.js +64 -23
- package/dist/formatters/anthropic-xml.js.map +1 -1
- package/dist/formatters/native.d.ts.map +1 -1
- package/dist/formatters/native.js +16 -3
- package/dist/formatters/native.js.map +1 -1
- package/dist/formatters/normalize-tool-pairs.d.ts +7 -0
- package/dist/formatters/normalize-tool-pairs.d.ts.map +1 -1
- package/dist/formatters/normalize-tool-pairs.js +45 -0
- package/dist/formatters/normalize-tool-pairs.js.map +1 -1
- package/dist/formatters/types.d.ts +31 -0
- package/dist/formatters/types.d.ts.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/membrane.d.ts +137 -8
- package/dist/membrane.d.ts.map +1 -1
- package/dist/membrane.js +823 -310
- package/dist/membrane.js.map +1 -1
- package/dist/providers/anthropic.d.ts +37 -0
- package/dist/providers/anthropic.d.ts.map +1 -1
- package/dist/providers/anthropic.js +142 -25
- package/dist/providers/anthropic.js.map +1 -1
- package/dist/providers/bedrock.d.ts +8 -0
- package/dist/providers/bedrock.d.ts.map +1 -1
- package/dist/providers/bedrock.js +24 -1
- package/dist/providers/bedrock.js.map +1 -1
- package/dist/providers/gemini.d.ts +11 -0
- package/dist/providers/gemini.d.ts.map +1 -1
- package/dist/providers/gemini.js +104 -82
- package/dist/providers/gemini.js.map +1 -1
- package/dist/providers/index.d.ts +1 -1
- package/dist/providers/index.d.ts.map +1 -1
- package/dist/providers/index.js.map +1 -1
- package/dist/providers/mock.d.ts +2 -0
- package/dist/providers/mock.d.ts.map +1 -1
- package/dist/providers/mock.js +2 -0
- package/dist/providers/mock.js.map +1 -1
- package/dist/providers/openai-compatible.d.ts +7 -0
- package/dist/providers/openai-compatible.d.ts.map +1 -1
- package/dist/providers/openai-compatible.js +99 -48
- package/dist/providers/openai-compatible.js.map +1 -1
- package/dist/providers/openai-completions.d.ts +5 -0
- package/dist/providers/openai-completions.d.ts.map +1 -1
- package/dist/providers/openai-completions.js +107 -44
- package/dist/providers/openai-completions.js.map +1 -1
- package/dist/providers/openai-responses-api.d.ts +16 -0
- package/dist/providers/openai-responses-api.d.ts.map +1 -1
- package/dist/providers/openai-responses-api.js +38 -10
- package/dist/providers/openai-responses-api.js.map +1 -1
- package/dist/providers/openai-responses.d.ts +2 -0
- package/dist/providers/openai-responses.d.ts.map +1 -1
- package/dist/providers/openai-responses.js +8 -1
- package/dist/providers/openai-responses.js.map +1 -1
- package/dist/providers/openai.d.ts +7 -0
- package/dist/providers/openai.d.ts.map +1 -1
- package/dist/providers/openai.js +92 -44
- package/dist/providers/openai.js.map +1 -1
- package/dist/providers/openrouter.d.ts +9 -0
- package/dist/providers/openrouter.d.ts.map +1 -1
- package/dist/providers/openrouter.js +118 -62
- package/dist/providers/openrouter.js.map +1 -1
- package/dist/providers/utils.d.ts +74 -2
- package/dist/providers/utils.d.ts.map +1 -1
- package/dist/providers/utils.js +184 -3
- package/dist/providers/utils.js.map +1 -1
- package/dist/registry/default-pricing.d.ts +22 -0
- package/dist/registry/default-pricing.d.ts.map +1 -1
- package/dist/registry/default-pricing.js +162 -28
- package/dist/registry/default-pricing.js.map +1 -1
- package/dist/types/content.d.ts +13 -0
- package/dist/types/content.d.ts.map +1 -1
- package/dist/types/content.js.map +1 -1
- package/dist/types/errors.d.ts +15 -0
- package/dist/types/errors.d.ts.map +1 -1
- package/dist/types/errors.js +26 -0
- package/dist/types/errors.js.map +1 -1
- package/dist/types/index.d.ts +3 -3
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/index.js +1 -1
- package/dist/types/index.js.map +1 -1
- package/dist/types/provider.d.ts +50 -0
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/request.d.ts +6 -0
- package/dist/types/request.d.ts.map +1 -1
- package/dist/types/response.d.ts +85 -4
- package/dist/types/response.d.ts.map +1 -1
- package/dist/types/response.js.map +1 -1
- package/dist/types/streaming.d.ts +6 -1
- package/dist/types/streaming.d.ts.map +1 -1
- package/dist/utils/cache-marker-budget.d.ts +91 -0
- package/dist/utils/cache-marker-budget.d.ts.map +1 -0
- package/dist/utils/cache-marker-budget.js +208 -0
- package/dist/utils/cache-marker-budget.js.map +1 -0
- package/dist/utils/cost.d.ts +22 -0
- package/dist/utils/cost.d.ts.map +1 -1
- package/dist/utils/cost.js +55 -0
- package/dist/utils/cost.js.map +1 -1
- package/dist/utils/thinking-carriers.d.ts +80 -0
- package/dist/utils/thinking-carriers.d.ts.map +1 -0
- package/dist/utils/thinking-carriers.js +115 -0
- package/dist/utils/thinking-carriers.js.map +1 -0
- package/dist/utils/tool-parser.d.ts +24 -0
- package/dist/utils/tool-parser.d.ts.map +1 -1
- package/dist/utils/tool-parser.js +367 -161
- package/dist/utils/tool-parser.js.map +1 -1
- package/dist/utils/usage.d.ts +71 -0
- package/dist/utils/usage.d.ts.map +1 -0
- package/dist/utils/usage.js +161 -0
- package/dist/utils/usage.js.map +1 -0
- package/dist/yielding-stream.d.ts +8 -0
- package/dist/yielding-stream.d.ts.map +1 -1
- package/dist/yielding-stream.js +49 -3
- package/dist/yielding-stream.js.map +1 -1
- package/package.json +1 -1
- package/src/cache-wire-receipt.ts +125 -0
- package/src/context/index.ts +1 -0
- package/src/context/process.ts +305 -59
- package/src/context/types.ts +38 -0
- package/src/floating-cache-marker.test.ts +24 -0
- package/src/formatters/anthropic-xml.ts +66 -23
- package/src/formatters/native.ts +23 -2
- package/src/formatters/normalize-tool-pairs.ts +49 -0
- package/src/formatters/types.ts +35 -0
- package/src/index.ts +6 -0
- package/src/membrane.ts +974 -313
- package/src/providers/anthropic.ts +173 -19
- package/src/providers/bedrock.ts +25 -1
- package/src/providers/gemini.ts +125 -87
- package/src/providers/index.ts +1 -0
- package/src/providers/mock.ts +3 -0
- package/src/providers/openai-compatible.ts +102 -45
- package/src/providers/openai-completions.ts +112 -46
- package/src/providers/openai-responses-api.ts +45 -15
- package/src/providers/openai-responses.ts +8 -1
- package/src/providers/openai.ts +95 -42
- package/src/providers/openrouter.ts +125 -63
- package/src/providers/utils.ts +227 -6
- package/src/registry/default-pricing.ts +165 -28
- package/src/types/content.ts +14 -0
- package/src/types/errors.ts +28 -0
- package/src/types/index.ts +6 -0
- package/src/types/provider.ts +55 -0
- package/src/types/request.ts +7 -0
- package/src/types/response.ts +109 -8
- package/src/types/streaming.ts +8 -2
- package/src/utils/cache-marker-budget.ts +237 -0
- package/src/utils/cost.ts +64 -0
- package/src/utils/thinking-carriers.ts +125 -0
- package/src/utils/tool-parser.ts +473 -174
- package/src/utils/usage.ts +184 -0
- package/src/yielding-stream.ts +54 -3
package/dist/membrane.js
CHANGED
|
@@ -4,17 +4,105 @@
|
|
|
4
4
|
* A selective boundary that transforms what passes through.
|
|
5
5
|
*/
|
|
6
6
|
import { lastCacheableBlockIndex } from './formatters/native.js';
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
7
|
+
import { sameThinkingText, findSpanningProviderRun, thinkingCarrierKey, stripThinkingForPrefill, } from './utils/thinking-carriers.js';
|
|
8
|
+
import { assertCacheMarkersWithinLimit, countWireCacheMarkers, clampCacheMarkers, ownSystemBlocks, MAX_CACHE_BREAKPOINTS, } from './utils/cache-marker-budget.js';
|
|
9
|
+
import { DEFAULT_RETRY_CONFIG, MembraneError, classifyError, isOverloadedError, isTimeoutAbortError, unsupportedError, } from './types/index.js';
|
|
10
|
+
import { computeCacheWireReceipt } from './cache-wire-receipt.js';
|
|
11
|
+
import { parseToolCalls, formatToolResults, parseAccumulatedIntoBlocks, endsWithPartialToolBlock, hasImageInToolResults, formatToolResultsForSplitTurn, } from './utils/tool-parser.js';
|
|
9
12
|
import { AnthropicXmlFormatter } from './formatters/anthropic-xml.js';
|
|
10
|
-
import { normalizeToolPairs, mergeConsecutiveRoles } from './formatters/normalize-tool-pairs.js';
|
|
13
|
+
import { normalizeToolPairs, mergeConsecutiveRoles, PREFIX_REWRITING_NORMALIZE_EVENT_KINDS, } from './formatters/normalize-tool-pairs.js';
|
|
11
14
|
import { YieldingStreamImpl } from './yielding-stream.js';
|
|
12
|
-
import { calculateCost } from './utils/cost.js';
|
|
15
|
+
import { calculateCost, warnUnpricedModel } from './utils/cost.js';
|
|
16
|
+
import { TurnUsageAccumulator, calculateCacheHitRatio, normalizeUsageToCacheExcluded, warnUnconvertibleProviderItem, } from './utils/usage.js';
|
|
13
17
|
import { isAcceptedImageMediaType, strippedImagePlaceholder, shedImagesToFitByteBudget, assertWithinByteBudget, } from './utils/image-media.js';
|
|
14
18
|
import { getDefaultPricing } from './registry/default-pricing.js';
|
|
15
19
|
// ============================================================================
|
|
16
20
|
// Membrane Class
|
|
17
21
|
// ============================================================================
|
|
22
|
+
/**
|
|
23
|
+
* Block-lifecycle tracking shared by the two native-tools streaming paths
|
|
24
|
+
* (`streamWithNativeTools` and `runNativeToolsYielding`).
|
|
25
|
+
*
|
|
26
|
+
* Providers signal blocks through `onContentBlock(index, block)`, but not all
|
|
27
|
+
* of them the same way: the Anthropic and Bedrock adapters fire it twice per
|
|
28
|
+
* index (content_block_start with an empty block, content_block_stop with the
|
|
29
|
+
* finalised one), while the OpenAI Responses adapter fires it ONCE per block,
|
|
30
|
+
* already finalised, after the stream has ended. Treating "second sighting"
|
|
31
|
+
* as the only completion signal therefore left single-callback adapters with
|
|
32
|
+
* `block_start` events that never completed (#63 review). The tracker keeps
|
|
33
|
+
* the paired semantics and adds `flush()`, which the caller runs once the
|
|
34
|
+
* provider stream has returned: every started block that never saw a second
|
|
35
|
+
* callback is completed from the last block payload seen for it.
|
|
36
|
+
*/
|
|
37
|
+
class NativeBlockTracker {
|
|
38
|
+
emit;
|
|
39
|
+
currentType = 'text';
|
|
40
|
+
blockIndex = 0;
|
|
41
|
+
started = new Map();
|
|
42
|
+
completed = new Set();
|
|
43
|
+
lastSeen = new Map();
|
|
44
|
+
constructor(emit) {
|
|
45
|
+
this.emit = emit;
|
|
46
|
+
}
|
|
47
|
+
static mapApiBlockType(apiType) {
|
|
48
|
+
if (apiType === 'thinking' || apiType === 'redacted_thinking' || apiType === 'reasoning')
|
|
49
|
+
return 'thinking';
|
|
50
|
+
if (apiType === 'tool_use' || apiType === 'function_call' || apiType === 'tool_call')
|
|
51
|
+
return 'tool_call';
|
|
52
|
+
return 'text';
|
|
53
|
+
}
|
|
54
|
+
/** Provider block callback: first sighting of an index starts it, a second completes it. */
|
|
55
|
+
onProviderBlock(index, block) {
|
|
56
|
+
this.lastSeen.set(index, block);
|
|
57
|
+
if (!this.started.has(index)) {
|
|
58
|
+
const mbType = NativeBlockTracker.mapApiBlockType(block?.type);
|
|
59
|
+
this.started.set(index, mbType);
|
|
60
|
+
this.currentType = mbType;
|
|
61
|
+
this.blockIndex = index;
|
|
62
|
+
this.emit?.({ event: 'block_start', index, block: { type: mbType } });
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
this.complete(index, block);
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Complete every started block that never received its second callback.
|
|
69
|
+
* Run after the provider stream has returned; idempotent, and a no-op for
|
|
70
|
+
* paired-callback adapters.
|
|
71
|
+
*/
|
|
72
|
+
flush() {
|
|
73
|
+
for (const index of this.started.keys()) {
|
|
74
|
+
if (!this.completed.has(index))
|
|
75
|
+
this.complete(index, this.lastSeen.get(index));
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
/** Discard tracking state (refusal retry rolled the attempt back). */
|
|
79
|
+
reset() {
|
|
80
|
+
this.currentType = 'text';
|
|
81
|
+
this.blockIndex = 0;
|
|
82
|
+
this.started.clear();
|
|
83
|
+
this.completed.clear();
|
|
84
|
+
this.lastSeen.clear();
|
|
85
|
+
}
|
|
86
|
+
complete(index, block) {
|
|
87
|
+
if (this.completed.has(index))
|
|
88
|
+
return;
|
|
89
|
+
this.completed.add(index);
|
|
90
|
+
const mbType = this.started.get(index)
|
|
91
|
+
?? NativeBlockTracker.mapApiBlockType(block?.type);
|
|
92
|
+
const apiBlock = block;
|
|
93
|
+
const mb = { type: mbType };
|
|
94
|
+
if (mbType === 'text')
|
|
95
|
+
mb.content = apiBlock?.text;
|
|
96
|
+
else if (mbType === 'thinking')
|
|
97
|
+
mb.content = apiBlock?.thinking;
|
|
98
|
+
else if (mbType === 'tool_call') {
|
|
99
|
+
mb.toolId = apiBlock?.id;
|
|
100
|
+
mb.toolName = apiBlock?.name;
|
|
101
|
+
mb.input = apiBlock?.input;
|
|
102
|
+
}
|
|
103
|
+
this.emit?.({ event: 'block_complete', index, block: mb });
|
|
104
|
+
}
|
|
105
|
+
}
|
|
18
106
|
export class Membrane {
|
|
19
107
|
adapter;
|
|
20
108
|
registry;
|
|
@@ -47,17 +135,33 @@ export class Membrane {
|
|
|
47
135
|
// refusal is a successful HTTP call with an unwanted verdict, and letting
|
|
48
136
|
// it consume error retries would couple two unrelated budgets.
|
|
49
137
|
let refusalRetriesUsed = 0;
|
|
138
|
+
// Spend on attempts we threw away. A refused attempt is a completed,
|
|
139
|
+
// billed HTTP call; reporting only the surviving attempt's usage
|
|
140
|
+
// under-reports the turn by one full call per retry.
|
|
141
|
+
let discardedUsage;
|
|
142
|
+
// One selection for the whole call: mode resolution and the build must
|
|
143
|
+
// name the same formatter instance (see resolveActiveFormatter).
|
|
144
|
+
const activeFormatter = this.resolveActiveFormatter(options.formatter);
|
|
50
145
|
while (true) {
|
|
51
146
|
attempts++;
|
|
52
147
|
try {
|
|
53
|
-
const { providerRequest, prefillResult } = this.transformRequest(request,
|
|
148
|
+
const { providerRequest, prefillResult } = this.transformRequest(request, activeFormatter);
|
|
54
149
|
// Route through the single canonical hook helper so any future
|
|
55
150
|
// change to hook semantics (logging, retry interaction, error
|
|
56
151
|
// handling) applies to both complete() and the streaming paths.
|
|
57
152
|
// Cast back to the local provider-request shape: the hook returns
|
|
58
153
|
// `unknown` deliberately, and we acknowledge the cast at the boundary.
|
|
59
154
|
const finalRequest = (await this.applyBeforeRequestHook(request, providerRequest));
|
|
60
|
-
|
|
155
|
+
// Last exit before the adapter: the only place that sees EVERY
|
|
156
|
+
// contribution (builder, formatter, passthrough, float, hook).
|
|
157
|
+
if (request.cacheMarkers === 'cm-owned') {
|
|
158
|
+
assertCacheMarkersWithinLimit(finalRequest, 'complete');
|
|
159
|
+
}
|
|
160
|
+
else {
|
|
161
|
+
clampCacheMarkers(finalRequest, 'complete');
|
|
162
|
+
}
|
|
163
|
+
request.onCacheWireReceipt?.(computeCacheWireReceipt(finalRequest));
|
|
164
|
+
const rawProviderResponse = await this.adapter.complete(finalRequest, {
|
|
61
165
|
signal: options.signal,
|
|
62
166
|
timeoutMs: options.timeoutMs,
|
|
63
167
|
onRequest: (req) => {
|
|
@@ -65,6 +169,11 @@ export class Membrane {
|
|
|
65
169
|
options.onRequest?.(req);
|
|
66
170
|
},
|
|
67
171
|
});
|
|
172
|
+
// Restate usage in the one convention before any ratio or price sees it.
|
|
173
|
+
const providerResponse = {
|
|
174
|
+
...rawProviderResponse,
|
|
175
|
+
usage: normalizeUsageToCacheExcluded(rawProviderResponse.usage, this.adapter.name, this.adapter.usageCacheConvention),
|
|
176
|
+
};
|
|
68
177
|
// Call onResponse callback with raw response from API
|
|
69
178
|
options.onResponse?.(providerResponse.raw);
|
|
70
179
|
const response = this.transformResponse(providerResponse, request, prefillResult, startTime, attempts, rawRequest);
|
|
@@ -76,8 +185,16 @@ export class Membrane {
|
|
|
76
185
|
if (response.stopReason === 'refusal' &&
|
|
77
186
|
refusalRetriesUsed < Math.max(0, options.refusalRetries ?? 0)) {
|
|
78
187
|
refusalRetriesUsed++;
|
|
188
|
+
discardedUsage = this.mergeDiscardedAttempts(discardedUsage, this.discardedAttemptFrom(response.usage));
|
|
79
189
|
continue;
|
|
80
190
|
}
|
|
191
|
+
// Report what the discarded attempts cost. Set BEFORE afterResponse
|
|
192
|
+
// so a hook that logs spend sees the whole turn, not just the
|
|
193
|
+
// attempt that stands.
|
|
194
|
+
if (discardedUsage) {
|
|
195
|
+
response.details.usage.discardedAttempts =
|
|
196
|
+
this.pricedDiscardedAttempts(discardedUsage, request.config.model);
|
|
197
|
+
}
|
|
81
198
|
// Call afterResponse hook
|
|
82
199
|
if (this.config.hooks?.afterResponse) {
|
|
83
200
|
return await this.config.hooks.afterResponse(response, providerResponse.raw);
|
|
@@ -111,9 +228,17 @@ export class Membrane {
|
|
|
111
228
|
throw new MembraneError(errorInfo);
|
|
112
229
|
}
|
|
113
230
|
}
|
|
114
|
-
// Wait before retry (abort-aware)
|
|
231
|
+
// Wait before retry (abort-aware). An abort landing inside the
|
|
232
|
+
// sleep must fail like every other failure of this method — a
|
|
233
|
+
// MembraneError — rather than escaping the loop as a raw
|
|
234
|
+
// DOMException whose shape no caller of complete() expects.
|
|
115
235
|
const delay = this.calculateRetryDelay(attempts, isOverloaded);
|
|
116
|
-
|
|
236
|
+
try {
|
|
237
|
+
await this.sleep(delay, options.signal);
|
|
238
|
+
}
|
|
239
|
+
catch (sleepError) {
|
|
240
|
+
throw this.attachRawRequest(sleepError, rawRequest);
|
|
241
|
+
}
|
|
117
242
|
continue;
|
|
118
243
|
}
|
|
119
244
|
throw new MembraneError(errorInfo);
|
|
@@ -142,6 +267,18 @@ export class Membrane {
|
|
|
142
267
|
// If streaming is explicitly disabled on the request, fall back to complete()
|
|
143
268
|
// and synthesize the streaming callbacks from the full response
|
|
144
269
|
if (request.streaming === false) {
|
|
270
|
+
// complete() has no tool loop, and neither branch of this fallback can
|
|
271
|
+
// build one: honouring onToolCalls here would mean re-implementing the
|
|
272
|
+
// whole XML/native continuation machinery. Silently dropping it turned
|
|
273
|
+
// a working agent into one that narrates tool calls it never makes —
|
|
274
|
+
// the raw <function_calls> XML lands in the returned text and the turn
|
|
275
|
+
// ends. Refuse where the option is passed, before spending a call.
|
|
276
|
+
if (options.onToolCalls) {
|
|
277
|
+
throw unsupportedError('stream() cannot execute tools with streaming: false — the non-streaming ' +
|
|
278
|
+
'fallback routes to complete(), which has no tool loop, so onToolCalls ' +
|
|
279
|
+
'would never run. Leave streaming enabled (or drive the loop yourself ' +
|
|
280
|
+
'with complete() per round).');
|
|
281
|
+
}
|
|
145
282
|
const response = await this.complete(request, options);
|
|
146
283
|
// Synthesize onChunk callbacks so callers that depend on them still work
|
|
147
284
|
if (options.onChunk && 'content' in response) {
|
|
@@ -158,8 +295,9 @@ export class Membrane {
|
|
|
158
295
|
}
|
|
159
296
|
return response;
|
|
160
297
|
}
|
|
161
|
-
// Determine tool mode
|
|
162
|
-
const
|
|
298
|
+
// Determine tool mode against the formatter that will build the request
|
|
299
|
+
const activeFormatter = this.resolveActiveFormatter(options.formatter);
|
|
300
|
+
const toolMode = this.resolveToolMode(request, activeFormatter);
|
|
163
301
|
const useNative = toolMode === 'native' && !!request.tools && request.tools.length > 0;
|
|
164
302
|
// Overloaded (529) pre-emission retry. The streaming paths have no retry
|
|
165
303
|
// loop of their own, so a capacity error used to kill the turn outright —
|
|
@@ -186,14 +324,16 @@ export class Membrane {
|
|
|
186
324
|
};
|
|
187
325
|
try {
|
|
188
326
|
const result = useNative
|
|
189
|
-
? await this.streamWithNativeTools(request, tracked)
|
|
190
|
-
: await this.streamWithXmlTools(request, tracked);
|
|
191
|
-
// The inner paths
|
|
192
|
-
//
|
|
193
|
-
//
|
|
194
|
-
//
|
|
327
|
+
? await this.streamWithNativeTools(request, tracked, activeFormatter)
|
|
328
|
+
: await this.streamWithXmlTools(request, tracked, activeFormatter);
|
|
329
|
+
// The inner paths count their own provider calls but cannot see this
|
|
330
|
+
// wrapper's discarded attempts. Each failed attempt here died before
|
|
331
|
+
// emitting anything (that is the precondition for retrying), so it
|
|
332
|
+
// cost at least the one call it failed on — ADD those to the inner
|
|
333
|
+
// count rather than overwriting it, or a turn that retried twice and
|
|
334
|
+
// then ran three tool rounds would report 2 calls instead of 5.
|
|
195
335
|
if (attempts > 1 && 'details' in result) {
|
|
196
|
-
result.details.timing.attempts
|
|
336
|
+
result.details.timing.attempts += attempts - 1;
|
|
197
337
|
result.details.timing.retryDelaysMs = retryDelaysMs;
|
|
198
338
|
}
|
|
199
339
|
return result;
|
|
@@ -218,7 +358,21 @@ export class Membrane {
|
|
|
218
358
|
}
|
|
219
359
|
const delay = this.calculateRetryDelay(attempts, true);
|
|
220
360
|
retryDelaysMs.push(delay);
|
|
221
|
-
|
|
361
|
+
// An abort during the backoff window is still a cancellation of
|
|
362
|
+
// this stream, and stream() documents cancellation as an
|
|
363
|
+
// AbortedResponse. Letting the sleep's rejection escape made that
|
|
364
|
+
// contract depend on which millisecond the abort landed in.
|
|
365
|
+
// Nothing has been emitted on this path (that is the precondition
|
|
366
|
+
// for retrying at all), so there is no partial content to report.
|
|
367
|
+
try {
|
|
368
|
+
await this.sleep(delay, options.signal);
|
|
369
|
+
}
|
|
370
|
+
catch (sleepError) {
|
|
371
|
+
if (this.isAbortError(sleepError)) {
|
|
372
|
+
return this.buildAbortedResponse('', { inputTokens: 0, outputTokens: 0 }, [], [], this.abortReason(sleepError, options.signal));
|
|
373
|
+
}
|
|
374
|
+
throw sleepError;
|
|
375
|
+
}
|
|
222
376
|
continue;
|
|
223
377
|
}
|
|
224
378
|
throw error;
|
|
@@ -226,17 +380,62 @@ export class Membrane {
|
|
|
226
380
|
}
|
|
227
381
|
}
|
|
228
382
|
/**
|
|
229
|
-
*
|
|
383
|
+
* Select the ACTIVE formatter for a request: the one instance that resolves
|
|
384
|
+
* its tool mode, builds its provider request, and parses its stream.
|
|
385
|
+
*
|
|
386
|
+
* A per-request override (`CompleteOptions.formatter` /
|
|
387
|
+
* `StreamOptions.formatter`) wins over the instance formatter, with ONE
|
|
388
|
+
* transport exception: the Responses adapter's input is a provider-native
|
|
389
|
+
* item array, and a generic override (for example Context Manager's
|
|
390
|
+
* NativeFormatter) produces Anthropic-style `{ role, content: [{ type:
|
|
391
|
+
* 'text' }] }` envelopes the Responses API rejects before inference — so a
|
|
392
|
+
* configured Responses formatter stays authoritative there.
|
|
393
|
+
*
|
|
394
|
+
* The exception is why this selection is a method rather than a `??` at each
|
|
395
|
+
* call site: while it lived inside transformRequest alone, the BUILD honored
|
|
396
|
+
* it and every other formatter reader resolved against a different instance,
|
|
397
|
+
* which is the split resolveToolMode exists to prevent, one layer down.
|
|
398
|
+
* Every entry point selects once, here, and threads the result.
|
|
399
|
+
*/
|
|
400
|
+
resolveActiveFormatter(requestFormatter) {
|
|
401
|
+
if (this.adapter.name === 'openai-responses-api' && this.formatter.name === 'openai-responses') {
|
|
402
|
+
return this.formatter;
|
|
403
|
+
}
|
|
404
|
+
return requestFormatter ?? this.formatter;
|
|
405
|
+
}
|
|
406
|
+
/**
|
|
407
|
+
* Determine the effective tool mode.
|
|
408
|
+
*
|
|
409
|
+
* THE single source of truth for the mode: both complete() (via
|
|
410
|
+
* transformRequest → BuildOptions.toolMode) and the streaming paths (via
|
|
411
|
+
* their native-vs-XML path choice) resolve here, so a given request resolves
|
|
412
|
+
* to the same mode whichever entry point it arrives through.
|
|
413
|
+
*
|
|
414
|
+
* Precedence, strongest first:
|
|
415
|
+
* 1. an explicit non-'auto' `request.toolMode`
|
|
416
|
+
* 2. the mode the BUILDING formatter was explicitly constructed with
|
|
417
|
+
* (`AnthropicXmlFormatter({ toolMode: 'native' })`) — a caller's stated
|
|
418
|
+
* choice, not a derivation
|
|
419
|
+
* 3. formatter/provider derivation
|
|
420
|
+
*
|
|
421
|
+
* `formatter` is the formatter that will actually build the request — the
|
|
422
|
+
* instance `resolveActiveFormatter` selected for this call — because
|
|
423
|
+
* resolving against one formatter while building with another is exactly the
|
|
424
|
+
* split this method exists to prevent.
|
|
230
425
|
*/
|
|
231
|
-
resolveToolMode(request) {
|
|
426
|
+
resolveToolMode(request, formatter = this.formatter) {
|
|
232
427
|
// Explicit mode takes precedence
|
|
233
428
|
if (request.toolMode && request.toolMode !== 'auto') {
|
|
234
429
|
return request.toolMode;
|
|
235
430
|
}
|
|
431
|
+
// A formatter constructed with an explicit mode states its caller's choice
|
|
432
|
+
if (formatter.configuredToolMode) {
|
|
433
|
+
return formatter.configuredToolMode;
|
|
434
|
+
}
|
|
236
435
|
// Auto mode: choose based on formatter
|
|
237
436
|
// NativeFormatter → native tools via API
|
|
238
437
|
// AnthropicXmlFormatter (default) → XML tools in prefill
|
|
239
|
-
if (
|
|
438
|
+
if (formatter.name === 'native' || formatter.name === 'openai-responses') {
|
|
240
439
|
return 'native';
|
|
241
440
|
}
|
|
242
441
|
// Also handle known native-tool providers regardless of formatter
|
|
@@ -253,16 +452,25 @@ export class Membrane {
|
|
|
253
452
|
* - False-positive stop sequence detection (e.g., "\nUser:" inside tool results)
|
|
254
453
|
* - Structured block events for UI
|
|
255
454
|
*/
|
|
256
|
-
async streamWithXmlTools(request, options) {
|
|
455
|
+
async streamWithXmlTools(request, options, activeFormatter = this.resolveActiveFormatter(options.formatter)) {
|
|
257
456
|
const startTime = Date.now();
|
|
258
|
-
const { onChunk, onContentBlockUpdate, onToolCalls, onPreToolContent, onUsage, onBlock, onRequest, onResponse, maxToolDepth = 10, signal,
|
|
259
|
-
//
|
|
260
|
-
|
|
457
|
+
const { onChunk, onContentBlockUpdate, onToolCalls, onPreToolContent, onUsage, onBlock, onRequest, onResponse, maxToolDepth = 10, signal, timeoutMs, idleTimeoutMs, } = options;
|
|
458
|
+
// The formatter stream() selected: the same instance that resolved the
|
|
459
|
+
// mode and will build the request, so the parser can never be reading a
|
|
460
|
+
// different format than the one on the wire.
|
|
461
|
+
const formatter = activeFormatter;
|
|
261
462
|
// Initialize parser from formatter for format-specific tracking
|
|
262
463
|
const parser = formatter.createStreamParser();
|
|
263
464
|
let toolDepth = 0;
|
|
264
|
-
|
|
265
|
-
|
|
465
|
+
// Each round is priced under the model that served THAT round and the
|
|
466
|
+
// costs are summed: a routed turn can change models mid-turn, and pricing
|
|
467
|
+
// the whole accumulated usage at the latest rate re-bills every earlier
|
|
468
|
+
// round at a price it was never charged.
|
|
469
|
+
const turnUsage = new TurnUsageAccumulator(request.config.model, (servedModel) => this.resolvePricing(request.config.model, servedModel));
|
|
470
|
+
// Honest turn telemetry: provider calls actually made (including refusal
|
|
471
|
+
// re-issues inside streamOnce) and continuation rounds.
|
|
472
|
+
let providerCalls = 0;
|
|
473
|
+
let rounds = 0;
|
|
266
474
|
const contentBlocks = [];
|
|
267
475
|
let lastStopReason = 'end_turn';
|
|
268
476
|
let lastStopSequence;
|
|
@@ -337,13 +545,13 @@ export class Membrane {
|
|
|
337
545
|
resumptionRounds++;
|
|
338
546
|
if (resumptionRounds === RESUMPTION_WARN_ROUNDS) {
|
|
339
547
|
warnLog.warn(`[membrane] automatic resumption at round ${resumptionRounds} ` +
|
|
340
|
-
`(${
|
|
548
|
+
`(${turnUsage.total.inputTokens} input tokens so far this turn) — ` +
|
|
341
549
|
`a spin shows up here before it shows up on the bill`);
|
|
342
550
|
}
|
|
343
551
|
if (resumptionRounds > maxResumptionRounds) {
|
|
344
552
|
warnLog.warn(`[membrane] automatic resumption cap (${maxResumptionRounds}) reached — ` +
|
|
345
553
|
`ending turn with stopReason 'round_limit'. ` +
|
|
346
|
-
`${
|
|
554
|
+
`${turnUsage.total.inputTokens} input tokens spent this turn.`);
|
|
347
555
|
return false;
|
|
348
556
|
}
|
|
349
557
|
return true;
|
|
@@ -407,6 +615,8 @@ export class Membrane {
|
|
|
407
615
|
: undefined,
|
|
408
616
|
}, {
|
|
409
617
|
signal,
|
|
618
|
+
timeoutMs,
|
|
619
|
+
idleTimeoutMs,
|
|
410
620
|
normalizedRequest: request,
|
|
411
621
|
// The tag-based parser tracks thinking via <thinking> tags — ask the
|
|
412
622
|
// provider to wrap native thinking deltas so they don't stream as
|
|
@@ -417,6 +627,8 @@ export class Membrane {
|
|
|
417
627
|
onRequest?.(req);
|
|
418
628
|
},
|
|
419
629
|
});
|
|
630
|
+
rounds++;
|
|
631
|
+
providerCalls += streamResult.providerCalls;
|
|
420
632
|
// If we detected stop sequence manually, fix up the parser and result
|
|
421
633
|
if (detectedStopSequence && truncatedAccumulated !== null) {
|
|
422
634
|
parser.reset();
|
|
@@ -446,18 +658,11 @@ export class Membrane {
|
|
|
446
658
|
onResponse?.(rawResponse);
|
|
447
659
|
lastStopReason = this.mapStopReason(streamResult.stopReason);
|
|
448
660
|
lastStopSequence = streamResult.stopSequence ?? undefined;
|
|
449
|
-
// Accumulate usage (including cache metrics)
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
}
|
|
455
|
-
if (streamResult.usage.cacheReadTokens) {
|
|
456
|
-
totalUsage.cacheReadTokens = (totalUsage.cacheReadTokens ?? 0) + streamResult.usage.cacheReadTokens;
|
|
457
|
-
}
|
|
458
|
-
if (pricing)
|
|
459
|
-
totalUsage.estimatedCost = calculateCost(totalUsage, pricing);
|
|
460
|
-
onUsage?.(totalUsage);
|
|
661
|
+
// Accumulate usage (including cache metrics), priced at this round's
|
|
662
|
+
// model. NOT inlined into the optional call — `onUsage?.(addRound())`
|
|
663
|
+
// skips evaluating its argument entirely when no callback is set.
|
|
664
|
+
const usageSoFar = turnUsage.addRound(streamResult.model, streamResult.usage);
|
|
665
|
+
onUsage?.(usageSoFar);
|
|
461
666
|
// Flush the parser to complete any in-progress streaming block
|
|
462
667
|
const flushResult = parser.flush();
|
|
463
668
|
for (const emission of flushResult.emissions) {
|
|
@@ -485,7 +690,7 @@ export class Membrane {
|
|
|
485
690
|
warnLog.warn(`[membrane] ${consecutiveStalledResumptions} consecutive automatic resumptions ` +
|
|
486
691
|
`made no progress (${streamedThisRound} chars this round, stop ` +
|
|
487
692
|
`${JSON.stringify(lastStopSequence ?? null)} repeated) — ending turn with ` +
|
|
488
|
-
`stopReason 'no_progress'. ${
|
|
693
|
+
`stopReason 'no_progress'. ${turnUsage.total.inputTokens} input tokens spent this turn.`);
|
|
489
694
|
lastStopReason = 'no_progress';
|
|
490
695
|
break;
|
|
491
696
|
}
|
|
@@ -718,14 +923,14 @@ export class Membrane {
|
|
|
718
923
|
// The full accumulated text is still available in raw.response
|
|
719
924
|
const fullAccumulated = parser.getAccumulated();
|
|
720
925
|
const newContent = fullAccumulated.slice(initialPrefillLength);
|
|
721
|
-
const response = this.buildFinalResponse(newContent, contentBlocks, lastStopReason,
|
|
722
|
-
rawRequest, rawResponse, executedToolCalls, executedToolResults, initialBlockType, lastStopSequence);
|
|
926
|
+
const response = this.buildFinalResponse(newContent, contentBlocks, lastStopReason, turnUsage, request, prefillResult, startTime, providerCalls, rawRequest, rawResponse, executedToolCalls, executedToolResults, initialBlockType, lastStopSequence);
|
|
723
927
|
// Append non-text content blocks (e.g., generated_image) that the XML parser can't handle
|
|
724
928
|
if (extraContentBlocks.length > 0) {
|
|
725
929
|
response.content.push(...extraContentBlocks);
|
|
726
930
|
}
|
|
727
931
|
// Merge provider thinking signatures into parser-derived thinking blocks
|
|
728
932
|
this.mergeProviderThinkingBlocks(response.content, providerThinkingBlocks);
|
|
933
|
+
response.details.timing.rounds = rounds;
|
|
729
934
|
return response;
|
|
730
935
|
}
|
|
731
936
|
catch (error) {
|
|
@@ -734,7 +939,7 @@ export class Membrane {
|
|
|
734
939
|
// Only use NEW content (after initial prefill) for partial content
|
|
735
940
|
const fullAccumulated = parser.getAccumulated();
|
|
736
941
|
const newContent = fullAccumulated.slice(initialPrefillLength);
|
|
737
|
-
return this.buildAbortedResponse(newContent,
|
|
942
|
+
return this.buildAbortedResponse(newContent, turnUsage.total, executedToolCalls, executedToolResults, this.abortReason(error, signal), initialBlockType);
|
|
738
943
|
}
|
|
739
944
|
// Re-throw with rawRequest attached for logging
|
|
740
945
|
throw this.attachRawRequest(error, rawRequest);
|
|
@@ -743,12 +948,17 @@ export class Membrane {
|
|
|
743
948
|
/**
|
|
744
949
|
* Stream with native API tool execution
|
|
745
950
|
*/
|
|
746
|
-
async streamWithNativeTools(request, options) {
|
|
951
|
+
async streamWithNativeTools(request, options, activeFormatter = this.resolveActiveFormatter(options.formatter)) {
|
|
747
952
|
const startTime = Date.now();
|
|
748
|
-
const { onChunk, onContentBlockUpdate, onToolCalls, onPreToolContent, onUsage, onRequest, onResponse, maxToolDepth = 10, signal, } = options;
|
|
953
|
+
const { onChunk, onContentBlockUpdate, onBlock, onToolCalls, onPreToolContent, onUsage, onRequest, onResponse, maxToolDepth = 10, signal, timeoutMs, idleTimeoutMs, } = options;
|
|
749
954
|
let toolDepth = 0;
|
|
750
|
-
|
|
751
|
-
|
|
955
|
+
// See streamWithXmlTools: one accumulator per turn, pricing each round
|
|
956
|
+
// under the model that served it.
|
|
957
|
+
const turnUsage = new TurnUsageAccumulator(request.config.model, (servedModel) => this.resolvePricing(request.config.model, servedModel));
|
|
958
|
+
// Honest turn telemetry: provider calls actually made (including refusal
|
|
959
|
+
// re-issues inside streamOnce) and continuation rounds.
|
|
960
|
+
let providerCalls = 0;
|
|
961
|
+
let rounds = 0;
|
|
752
962
|
let lastStopReason = 'end_turn';
|
|
753
963
|
let lastStopSequence;
|
|
754
964
|
let rawRequest;
|
|
@@ -761,55 +971,69 @@ export class Membrane {
|
|
|
761
971
|
// Build messages array that we'll update with tool results
|
|
762
972
|
let messages = [...request.messages];
|
|
763
973
|
let allContentBlocks = [];
|
|
974
|
+
let markersInLastRequest = 0;
|
|
764
975
|
try {
|
|
765
976
|
// Tool execution loop
|
|
766
977
|
while (toolDepth <= maxToolDepth) {
|
|
767
978
|
// Build provider request with native tools
|
|
768
|
-
const providerRequest = this.buildNativeToolRequest(request, messages, toolDepth > 0);
|
|
979
|
+
const providerRequest = this.buildNativeToolRequest(request, messages, toolDepth > 0, activeFormatter);
|
|
769
980
|
// Stream from provider
|
|
770
981
|
let textAccumulated = '';
|
|
771
|
-
|
|
982
|
+
// Tag every token chunk with the membrane block it belongs to and
|
|
983
|
+
// surface the block lifecycle through onBlock — the same shape
|
|
984
|
+
// runNativeToolsYielding uses (#19). Before this, meta.type was
|
|
985
|
+
// hardcoded to 'text' on every chunk and onBlock was never invoked
|
|
986
|
+
// from this path (#20).
|
|
987
|
+
const tracker = new NativeBlockTracker(onBlock ? (event) => onBlock(event) : undefined);
|
|
772
988
|
const streamResult = await this.streamOnce(providerRequest, {
|
|
773
989
|
onChunk: (chunk) => {
|
|
774
990
|
textAccumulated += chunk;
|
|
775
991
|
allTextAccumulated += chunk;
|
|
776
|
-
// For native mode, emit text chunks with basic metadata
|
|
777
|
-
// TODO: Use native API content_block events for richer metadata
|
|
778
992
|
const meta = {
|
|
779
|
-
type:
|
|
780
|
-
visible:
|
|
781
|
-
blockIndex,
|
|
993
|
+
type: tracker.currentType,
|
|
994
|
+
visible: tracker.currentType === 'text',
|
|
995
|
+
blockIndex: tracker.blockIndex,
|
|
782
996
|
};
|
|
783
997
|
onChunk?.(chunk, meta);
|
|
784
998
|
},
|
|
785
|
-
onContentBlock:
|
|
786
|
-
|
|
787
|
-
|
|
999
|
+
onContentBlock: (index, block) => {
|
|
1000
|
+
tracker.onProviderBlock(index, block);
|
|
1001
|
+
// Deprecated pass-through, kept for callers still on it.
|
|
1002
|
+
onContentBlockUpdate?.(index, block);
|
|
1003
|
+
},
|
|
788
1004
|
}, {
|
|
789
1005
|
signal,
|
|
1006
|
+
timeoutMs,
|
|
1007
|
+
idleTimeoutMs,
|
|
790
1008
|
normalizedRequest: request,
|
|
791
1009
|
onRequest: (req) => {
|
|
792
1010
|
rawRequest = req;
|
|
793
1011
|
onRequest?.(req);
|
|
794
1012
|
},
|
|
1013
|
+
// Telemetry reports what this request actually SHIPPED with —
|
|
1014
|
+
// builder breakpoints, stale passthrough, fallback, float, plus
|
|
1015
|
+
// whatever the beforeRequest hook and the wire clamp did after
|
|
1016
|
+
// the build. Both native paths used to hardcode 0, and counting
|
|
1017
|
+
// at build time reported a number no request ever had.
|
|
1018
|
+
onWireCacheMarkers: (markerCount) => {
|
|
1019
|
+
markersInLastRequest = markerCount;
|
|
1020
|
+
},
|
|
795
1021
|
});
|
|
1022
|
+
// Single-callback adapters (OpenAI Responses) report each finalised
|
|
1023
|
+
// block once, after the stream: complete whatever never saw a stop.
|
|
1024
|
+
tracker.flush();
|
|
1025
|
+
rounds++;
|
|
1026
|
+
providerCalls += streamResult.providerCalls;
|
|
796
1027
|
rawResponse = streamResult.raw;
|
|
797
1028
|
// Call onResponse callback with raw response from API
|
|
798
1029
|
onResponse?.(rawResponse);
|
|
799
1030
|
lastStopReason = this.mapStopReason(streamResult.stopReason);
|
|
800
1031
|
lastStopSequence = streamResult.stopSequence ?? undefined;
|
|
801
|
-
// Accumulate usage (including cache metrics)
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
}
|
|
807
|
-
if (streamResult.usage.cacheReadTokens) {
|
|
808
|
-
totalUsage.cacheReadTokens = (totalUsage.cacheReadTokens ?? 0) + streamResult.usage.cacheReadTokens;
|
|
809
|
-
}
|
|
810
|
-
if (pricing)
|
|
811
|
-
totalUsage.estimatedCost = calculateCost(totalUsage, pricing);
|
|
812
|
-
onUsage?.(totalUsage);
|
|
1032
|
+
// Accumulate usage (including cache metrics), priced at this round's
|
|
1033
|
+
// model. NOT inlined into the optional call — `onUsage?.(addRound())`
|
|
1034
|
+
// skips evaluating its argument entirely when no callback is set.
|
|
1035
|
+
const usageSoFar = turnUsage.addRound(streamResult.model, streamResult.usage);
|
|
1036
|
+
onUsage?.(usageSoFar);
|
|
813
1037
|
// Parse content blocks from response
|
|
814
1038
|
const responseBlocks = this.parseProviderContent(streamResult.content);
|
|
815
1039
|
allContentBlocks.push(...responseBlocks);
|
|
@@ -879,6 +1103,7 @@ export class Membrane {
|
|
|
879
1103
|
break;
|
|
880
1104
|
}
|
|
881
1105
|
const durationMs = Date.now() - startTime;
|
|
1106
|
+
const totalUsage = turnUsage.total;
|
|
882
1107
|
return {
|
|
883
1108
|
content: allContentBlocks,
|
|
884
1109
|
rawAssistantText: allTextAccumulated,
|
|
@@ -895,15 +1120,17 @@ export class Membrane {
|
|
|
895
1120
|
usage: { ...totalUsage },
|
|
896
1121
|
timing: {
|
|
897
1122
|
totalDurationMs: durationMs,
|
|
898
|
-
attempts:
|
|
1123
|
+
attempts: providerCalls,
|
|
1124
|
+
rounds,
|
|
899
1125
|
},
|
|
900
1126
|
model: {
|
|
901
1127
|
requested: request.config.model,
|
|
902
|
-
actual: request.config.model,
|
|
1128
|
+
actual: turnUsage.lastServedModel || request.config.model,
|
|
903
1129
|
provider: this.adapter.name,
|
|
1130
|
+
perRound: turnUsage.perRound,
|
|
904
1131
|
},
|
|
905
1132
|
cache: {
|
|
906
|
-
markersInRequest:
|
|
1133
|
+
markersInRequest: markersInLastRequest,
|
|
907
1134
|
tokensCreated: totalUsage.cacheCreationTokens ?? 0,
|
|
908
1135
|
tokensRead: totalUsage.cacheReadTokens ?? 0,
|
|
909
1136
|
hitRatio: this.calculateCacheHitRatio(totalUsage),
|
|
@@ -918,28 +1145,60 @@ export class Membrane {
|
|
|
918
1145
|
catch (error) {
|
|
919
1146
|
// Check if this is an abort error
|
|
920
1147
|
if (this.isAbortError(error)) {
|
|
921
|
-
return this.buildAbortedResponse(allTextAccumulated,
|
|
1148
|
+
return this.buildAbortedResponse(allTextAccumulated, turnUsage.total, executedToolCalls, executedToolResults, this.abortReason(error, signal));
|
|
922
1149
|
}
|
|
923
1150
|
// Re-throw with rawRequest attached for logging
|
|
924
1151
|
throw this.attachRawRequest(error, rawRequest);
|
|
925
1152
|
}
|
|
926
1153
|
}
|
|
927
|
-
/**
|
|
928
|
-
|
|
1154
|
+
/**
|
|
1155
|
+
* Rate-limit state for the float's budget warning. See the
|
|
1156
|
+
* floating-cache-marker block in buildNativeToolRequest.
|
|
1157
|
+
*
|
|
1158
|
+
* A once-per-instance latch made the ONLY observable of an over-budget wire
|
|
1159
|
+
* go quiet for the life of the process: a long-lived Membrane warns for the
|
|
1160
|
+
* first agent that trips it and never again, so the condition looks like it
|
|
1161
|
+
* healed. Warn on the first occurrence, then at most once per interval,
|
|
1162
|
+
* carrying the count of what was suppressed in between.
|
|
1163
|
+
*/
|
|
1164
|
+
floatBudgetWarnState = { lastWarnedAtMs: 0, suppressedSinceWarn: 0 };
|
|
1165
|
+
static FLOAT_BUDGET_WARN_INTERVAL_MS = 60_000;
|
|
1166
|
+
warnFloatBudgetExhausted(wireMarkers) {
|
|
1167
|
+
const now = Date.now();
|
|
1168
|
+
const state = this.floatBudgetWarnState;
|
|
1169
|
+
const elapsed = now - state.lastWarnedAtMs;
|
|
1170
|
+
if (state.lastWarnedAtMs !== 0 && elapsed < Membrane.FLOAT_BUDGET_WARN_INTERVAL_MS) {
|
|
1171
|
+
state.suppressedSinceWarn++;
|
|
1172
|
+
return;
|
|
1173
|
+
}
|
|
1174
|
+
const suppressed = state.suppressedSinceWarn;
|
|
1175
|
+
state.lastWarnedAtMs = now;
|
|
1176
|
+
state.suppressedSinceWarn = 0;
|
|
1177
|
+
console.warn(`[membrane] floating cache marker withheld: upstream markers already ` +
|
|
1178
|
+
`occupy all ${MAX_CACHE_BREAKPOINTS} cache_control slots (${wireMarkers} on the wire). ` +
|
|
1179
|
+
`Tool-round suffixes will not cache incrementally.` +
|
|
1180
|
+
(suppressed > 0 ? ` (${suppressed} further occurrences suppressed since the last warning.)` : ''));
|
|
1181
|
+
}
|
|
929
1182
|
/**
|
|
930
1183
|
* Build a provider request with native tool support.
|
|
931
1184
|
*
|
|
932
1185
|
* `toolLoopRebuild` is true when this build is a tool-loop continuation
|
|
933
1186
|
* (toolDepth > 0) rather than the turn's first request — the only case
|
|
934
1187
|
* where the floating cache marker applies.
|
|
1188
|
+
*
|
|
1189
|
+
* `activeFormatter` is the formatter the caller selected for the request
|
|
1190
|
+
* (see resolveActiveFormatter). Reading `this.formatter` here instead made
|
|
1191
|
+
* the native loop build through the instance formatter while the mode had
|
|
1192
|
+
* been resolved against a per-request override — the two disagreeing about
|
|
1193
|
+
* which formatter is active.
|
|
935
1194
|
*/
|
|
936
|
-
buildNativeToolRequest(request, messages, toolLoopRebuild = false) {
|
|
1195
|
+
buildNativeToolRequest(request, messages, toolLoopRebuild = false, activeFormatter = this.formatter) {
|
|
937
1196
|
// Provider-native formatters own their complete input-item shape. The
|
|
938
1197
|
// legacy implementation below is intentionally Anthropic-specific; using
|
|
939
1198
|
// it for Responses would normalize away item IDs, encrypted reasoning,
|
|
940
1199
|
// assistant phases, and compaction items.
|
|
941
|
-
if (
|
|
942
|
-
return this.transformRequest({ ...request, messages },
|
|
1200
|
+
if (activeFormatter.name === 'openai-responses') {
|
|
1201
|
+
return this.transformRequest({ ...request, messages }, activeFormatter).providerRequest;
|
|
943
1202
|
}
|
|
944
1203
|
// Convert messages to provider format
|
|
945
1204
|
const providerMessages = [];
|
|
@@ -950,10 +1209,13 @@ export class Membrane {
|
|
|
950
1209
|
// Anthropic allows at most 4 cache_control breakpoints per request. The
|
|
951
1210
|
// message breakpoints are the valuable ones (they cache the longest prefixes,
|
|
952
1211
|
// and every one already includes tools+system at the front of the request).
|
|
953
|
-
// So tools/system get a breakpoint only as a FALLBACK — when no
|
|
954
|
-
//
|
|
955
|
-
// total past 4, which the API hard-rejects (the agent goes
|
|
956
|
-
|
|
1212
|
+
// So tools/system get a breakpoint only as a FALLBACK — when no marker
|
|
1213
|
+
// exists anywhere on the wire — otherwise they're redundant and would push
|
|
1214
|
+
// the total past 4, which the API hard-rejects (the agent goes
|
|
1215
|
+
// unresponsive). The fallback gate reads a RECOUNT of the built artifacts
|
|
1216
|
+
// (see below), never a running tally: a running tally cannot see a
|
|
1217
|
+
// caller-marked system block, and double-counts a message breakpoint that
|
|
1218
|
+
// lands on a block already carrying stale cache_control.
|
|
957
1219
|
for (const msg of messages) {
|
|
958
1220
|
const isAssistant = msg.participant === assistantName;
|
|
959
1221
|
const role = isAssistant ? 'assistant' : 'user';
|
|
@@ -974,14 +1236,13 @@ export class Membrane {
|
|
|
974
1236
|
}
|
|
975
1237
|
const textBlock = { type: 'text', text };
|
|
976
1238
|
if (block.cache_control) {
|
|
977
|
-
textBlock.cache_control = block.cache_control;
|
|
978
1239
|
// A block-level passthrough occupies one of the 4 breakpoint slots
|
|
979
|
-
// exactly like a marked message
|
|
980
|
-
//
|
|
981
|
-
//
|
|
982
|
-
//
|
|
983
|
-
//
|
|
984
|
-
|
|
1240
|
+
// exactly like a marked message; the recount below sees it.
|
|
1241
|
+
// (Imported/seeded conversations carry stale request-time
|
|
1242
|
+
// cache_control on stored blocks — first seen wedging Sill
|
|
1243
|
+
// 2026-07-25: 3 cm markers + 2 stale Arc-export blocks = 5 → hard
|
|
1244
|
+
// 400 on every inference.)
|
|
1245
|
+
textBlock.cache_control = block.cache_control;
|
|
985
1246
|
}
|
|
986
1247
|
content.push(textBlock);
|
|
987
1248
|
}
|
|
@@ -1058,7 +1319,6 @@ export class Membrane {
|
|
|
1058
1319
|
const bpIdx = lastCacheableBlockIndex(content);
|
|
1059
1320
|
if (bpIdx >= 0) {
|
|
1060
1321
|
content[bpIdx].cache_control = cacheControl;
|
|
1061
|
-
messageBreakpoints++;
|
|
1062
1322
|
}
|
|
1063
1323
|
}
|
|
1064
1324
|
providerMessages.push({ role, content });
|
|
@@ -1087,14 +1347,31 @@ export class Membrane {
|
|
|
1087
1347
|
// past one. `synthetic_pending_result` (not the downstream
|
|
1088
1348
|
// cache_suppressed_for_synthetic, which only fires when a marker was
|
|
1089
1349
|
// actually stripped) is the root condition.
|
|
1090
|
-
|
|
1350
|
+
// Every repair that REWRITES prefix bytes stands the float down, not just
|
|
1351
|
+
// the synthetic [pending] result: a textified orphan tool_result is
|
|
1352
|
+
// rewritten the same way when its real pairing arrives, so caching at or
|
|
1353
|
+
// past one poisons the prefix identically. The kinds live in one exported
|
|
1354
|
+
// set so a normalizer that grows a new prefix-rewriting repair cannot
|
|
1355
|
+
// silently escape this guard.
|
|
1356
|
+
let prefixRewritten = false;
|
|
1091
1357
|
const normalized = normalizeToolPairs(providerMessages, {
|
|
1092
1358
|
onEvent: (e) => {
|
|
1093
|
-
if (e.kind
|
|
1094
|
-
|
|
1359
|
+
if (PREFIX_REWRITING_NORMALIZE_EVENT_KINDS.has(e.kind))
|
|
1360
|
+
prefixRewritten = true;
|
|
1095
1361
|
},
|
|
1096
1362
|
});
|
|
1097
1363
|
const mergedMessages = mergeConsecutiveRoles(normalized.messages);
|
|
1364
|
+
// ONE recount of the constructed wire artifacts, taken BEFORE the
|
|
1365
|
+
// tools/system fallback decision so the fallback and the float share a
|
|
1366
|
+
// single truth. Counted post-normalize, so phase-5.5 cache suppression is
|
|
1367
|
+
// already reflected. `request.system` is the caller's own system content:
|
|
1368
|
+
// it explicitly accepts pre-marked blocks, and those are real wire markers
|
|
1369
|
+
// that no running tally ever saw (three of them plus both fallbacks = 5 on
|
|
1370
|
+
// the wire = a 400 on every inference of that config).
|
|
1371
|
+
const upstreamWireMarkers = countWireCacheMarkers({
|
|
1372
|
+
messages: mergedMessages,
|
|
1373
|
+
system: request.system,
|
|
1374
|
+
});
|
|
1098
1375
|
// Convert tools to provider format.
|
|
1099
1376
|
// Native tool names must match ^[a-zA-Z0-9_-]{1,128}$ — sanitize colons
|
|
1100
1377
|
// from the module:tool namespace convention. Reversed in parseProviderContent.
|
|
@@ -1106,7 +1383,7 @@ export class Membrane {
|
|
|
1106
1383
|
};
|
|
1107
1384
|
// Cache the tool list (last tool) only as a fallback — a marked message
|
|
1108
1385
|
// breakpoint already caches the tools as part of its prefix.
|
|
1109
|
-
if (cacheControl &&
|
|
1386
|
+
if (cacheControl && upstreamWireMarkers === 0 && request.tools && idx === request.tools.length - 1) {
|
|
1110
1387
|
t.cache_control = cacheControl;
|
|
1111
1388
|
}
|
|
1112
1389
|
return t;
|
|
@@ -1114,11 +1391,11 @@ export class Membrane {
|
|
|
1114
1391
|
// Wrap system prompt with cache_control only as a fallback (no message
|
|
1115
1392
|
// breakpoint marked); otherwise a message breakpoint already caches
|
|
1116
1393
|
// tools+system as part of its prefix.
|
|
1117
|
-
let system = request.system;
|
|
1118
|
-
if (cacheControl &&
|
|
1394
|
+
let system = ownSystemBlocks(request.system);
|
|
1395
|
+
if (cacheControl && upstreamWireMarkers === 0 && typeof system === 'string' && system.length > 0) {
|
|
1119
1396
|
system = [{ type: 'text', text: system, cache_control: cacheControl }];
|
|
1120
1397
|
}
|
|
1121
|
-
else if (cacheControl &&
|
|
1398
|
+
else if (cacheControl && upstreamWireMarkers === 0 && Array.isArray(system) && system.length > 0) {
|
|
1122
1399
|
const blocks = system;
|
|
1123
1400
|
system = blocks.map((block, idx) => idx === blocks.length - 1 ? { ...block, cache_control: cacheControl } : block);
|
|
1124
1401
|
}
|
|
@@ -1153,44 +1430,13 @@ export class Membrane {
|
|
|
1153
1430
|
// normalizer's phase 5.5 cache suppression.
|
|
1154
1431
|
// ------------------------------------------------------------------
|
|
1155
1432
|
const floatingEnabled = request.floatingCacheMarker ?? this.config.defaultFloatingCacheMarker ?? true;
|
|
1156
|
-
if (toolLoopRebuild && floatingEnabled && cacheControl && !
|
|
1157
|
-
//
|
|
1158
|
-
//
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
// landing on a block already carrying stale cache_control is one
|
|
1162
|
-
// physical marker counted twice, and a pre-marked system block is a
|
|
1163
|
-
// real wire marker the tally never sees. Counted post-fallback and
|
|
1164
|
-
// post-normalize, so fallback spend and phase-5.5 suppression are
|
|
1165
|
-
// both reflected.
|
|
1166
|
-
let wireMarkers = 0;
|
|
1167
|
-
for (const m of mergedMessages) {
|
|
1168
|
-
if (!Array.isArray(m.content))
|
|
1169
|
-
continue;
|
|
1170
|
-
for (const b of m.content) {
|
|
1171
|
-
if (b.cache_control)
|
|
1172
|
-
wireMarkers++;
|
|
1173
|
-
}
|
|
1174
|
-
}
|
|
1175
|
-
if (tools)
|
|
1176
|
-
for (const t of tools) {
|
|
1177
|
-
if (t.cache_control)
|
|
1178
|
-
wireMarkers++;
|
|
1179
|
-
}
|
|
1180
|
-
if (Array.isArray(system)) {
|
|
1181
|
-
for (const b of system) {
|
|
1182
|
-
if (b.cache_control)
|
|
1183
|
-
wireMarkers++;
|
|
1184
|
-
}
|
|
1185
|
-
}
|
|
1186
|
-
let residuum = 4 - wireMarkers;
|
|
1433
|
+
if (toolLoopRebuild && floatingEnabled && cacheControl && !prefixRewritten) {
|
|
1434
|
+
// Same recount as the fallback gate, re-taken POST-fallback so the
|
|
1435
|
+
// fallback's own spend is inside the residuum.
|
|
1436
|
+
const wireMarkers = countWireCacheMarkers({ messages: mergedMessages, system, tools });
|
|
1437
|
+
let residuum = MAX_CACHE_BREAKPOINTS - wireMarkers;
|
|
1187
1438
|
if (residuum <= 0) {
|
|
1188
|
-
|
|
1189
|
-
this.floatBudgetWarned = true;
|
|
1190
|
-
console.warn(`[membrane] floating cache marker withheld: upstream markers already ` +
|
|
1191
|
-
`occupy all 4 cache_control slots (${wireMarkers} on the wire). ` +
|
|
1192
|
-
`Tool-round suffixes will not cache incrementally.`);
|
|
1193
|
-
}
|
|
1439
|
+
this.warnFloatBudgetExhausted(wireMarkers);
|
|
1194
1440
|
}
|
|
1195
1441
|
else {
|
|
1196
1442
|
// Newest message first; then the previous round's endpoint (two
|
|
@@ -1261,6 +1507,9 @@ export class Membrane {
|
|
|
1261
1507
|
id: item.id,
|
|
1262
1508
|
name: unsanitizeToolName(item.name),
|
|
1263
1509
|
input: item.input,
|
|
1510
|
+
// Arguments that never parsed: carry the marker through so a
|
|
1511
|
+
// consumer can refuse the block instead of trusting `input`.
|
|
1512
|
+
...(item.unparseableInput !== undefined ? { unparseableInput: item.unparseableInput } : {}),
|
|
1264
1513
|
...(item.rawItem ? { rawItem: item.rawItem } : {}),
|
|
1265
1514
|
});
|
|
1266
1515
|
}
|
|
@@ -1283,14 +1532,24 @@ export class Membrane {
|
|
|
1283
1532
|
mimeType: item.mimeType,
|
|
1284
1533
|
});
|
|
1285
1534
|
}
|
|
1286
|
-
else if (item.rawItem) {
|
|
1535
|
+
else if (item.rawItem || item.type) {
|
|
1287
1536
|
// Opaque Responses items such as encrypted compaction or custom
|
|
1288
1537
|
// tool records have no normalized ContentBlock equivalent. Retain a
|
|
1289
1538
|
// zero-width carrier so Chronicle and the Responses formatter can
|
|
1290
1539
|
// replay the raw item without surfacing synthetic prompt text.
|
|
1291
1540
|
// Anthropic-bound conversion paths filter these out (empty text
|
|
1292
1541
|
// blocks are a 400 there); the Responses formatter replays rawItem.
|
|
1293
|
-
|
|
1542
|
+
//
|
|
1543
|
+
// An item with a `type` this switch does not know (server_tool_use,
|
|
1544
|
+
// web_search_tool_result, search_result, mcp_tool_use, or whatever a
|
|
1545
|
+
// provider adds next) used to fall out of this chain and vanish. It
|
|
1546
|
+
// gets the same carrier treatment, holding the item itself, plus a
|
|
1547
|
+
// one-time warning so the gap surfaces instead of being inferred
|
|
1548
|
+
// later from missing content.
|
|
1549
|
+
const carriedRawItem = item.rawItem ?? item;
|
|
1550
|
+
if (!item.rawItem)
|
|
1551
|
+
warnUnconvertibleProviderItem(item.type);
|
|
1552
|
+
blocks.push({ type: 'text', text: '', rawItem: carriedRawItem });
|
|
1294
1553
|
}
|
|
1295
1554
|
}
|
|
1296
1555
|
return blocks;
|
|
@@ -1324,10 +1583,32 @@ export class Membrane {
|
|
|
1324
1583
|
}
|
|
1325
1584
|
/**
|
|
1326
1585
|
* Merge provider thinking signatures into parser-derived thinking blocks
|
|
1327
|
-
*
|
|
1328
|
-
*
|
|
1329
|
-
*
|
|
1330
|
-
*
|
|
1586
|
+
* and prepend any leftover provider blocks — signature-only thinking
|
|
1587
|
+
* (display:'omitted') never appears in the text stream, so the parser
|
|
1588
|
+
* produces no block for it. redacted_thinking blocks are always prepended
|
|
1589
|
+
* verbatim.
|
|
1590
|
+
*
|
|
1591
|
+
* Pairing is by CONTENT IDENTITY, never by index. The two lists are
|
|
1592
|
+
* differently shaped whenever the provider emits a block the parser cannot
|
|
1593
|
+
* see (signature-only), the parser emits a block the provider never
|
|
1594
|
+
* produced (the XML path's literal `Claude: <thinking>` prefill turns
|
|
1595
|
+
* VISIBLE text into a thinking block), or one provider block spans several
|
|
1596
|
+
* (auto-continuation: capture runs per round while the parser sees the
|
|
1597
|
+
* CONCATENATED accumulation). Index-zipping crosses the lists in all three
|
|
1598
|
+
* shapes and stamps a signature onto content that never produced it —
|
|
1599
|
+
* which round-trips into the consumer's stored history and fails Anthropic
|
|
1600
|
+
* signature validation on the next turn.
|
|
1601
|
+
*
|
|
1602
|
+
* The three rules, in order:
|
|
1603
|
+
* 1. identity — a provider block pairs with the parsed block whose
|
|
1604
|
+
* thinking text is the same; empty-thinking (signature-only) blocks
|
|
1605
|
+
* are never text-match candidates and are prepend-only.
|
|
1606
|
+
* 2. span — a parsed block that reconstructs as the concatenation of a
|
|
1607
|
+
* RUN of consecutive unpaired provider blocks is REPLACED in place by
|
|
1608
|
+
* those originals, so the spanning block never wears a fragment's
|
|
1609
|
+
* signature and no reasoning is sent twice.
|
|
1610
|
+
* 3. leftover — everything still unpaired is prepended, de-duplicated
|
|
1611
|
+
* against what `content` already carries (and against itself).
|
|
1331
1612
|
*
|
|
1332
1613
|
* Mutates `content` in place. Shared by the XML stream paths
|
|
1333
1614
|
* (streamWithXmlTools and runXmlToolsYielding).
|
|
@@ -1335,20 +1616,70 @@ export class Membrane {
|
|
|
1335
1616
|
mergeProviderThinkingBlocks(content, providerThinkingBlocks) {
|
|
1336
1617
|
if (providerThinkingBlocks.length === 0)
|
|
1337
1618
|
return;
|
|
1338
|
-
const parsedThinking = content.filter((b) => b.type === 'thinking');
|
|
1339
1619
|
const providerThinking = providerThinkingBlocks.filter((b) => b.type === 'thinking');
|
|
1340
1620
|
const redacted = providerThinkingBlocks.filter((b) => b.type === 'redacted_thinking');
|
|
1341
|
-
const
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
if (
|
|
1345
|
-
|
|
1346
|
-
|
|
1621
|
+
const pairedProviderBlocks = new Set();
|
|
1622
|
+
const claimedParsedIndices = new Set();
|
|
1623
|
+
const parsedThinkingIndices = () => content.reduce((acc, block, index) => {
|
|
1624
|
+
if (block.type === 'thinking')
|
|
1625
|
+
acc.push(index);
|
|
1626
|
+
return acc;
|
|
1627
|
+
}, []);
|
|
1628
|
+
for (let p = 0; p < providerThinking.length; p++) {
|
|
1629
|
+
const providerText = providerThinking[p].thinking ?? '';
|
|
1630
|
+
if (providerText === '')
|
|
1631
|
+
continue;
|
|
1632
|
+
const match = parsedThinkingIndices().find((index) => !claimedParsedIndices.has(index) &&
|
|
1633
|
+
sameThinkingText(content[index].thinking ?? '', providerText));
|
|
1634
|
+
if (match === undefined)
|
|
1635
|
+
continue;
|
|
1636
|
+
const signature = providerThinking[p].signature;
|
|
1637
|
+
if (signature)
|
|
1638
|
+
content[match].signature = signature;
|
|
1639
|
+
claimedParsedIndices.add(match);
|
|
1640
|
+
pairedProviderBlocks.add(p);
|
|
1347
1641
|
}
|
|
1348
|
-
const
|
|
1349
|
-
|
|
1350
|
-
|
|
1642
|
+
for (const parsedIndex of parsedThinkingIndices().reverse()) {
|
|
1643
|
+
if (claimedParsedIndices.has(parsedIndex))
|
|
1644
|
+
continue;
|
|
1645
|
+
const parsedText = content[parsedIndex].thinking ?? '';
|
|
1646
|
+
if (parsedText === '')
|
|
1647
|
+
continue;
|
|
1648
|
+
const run = findSpanningProviderRun(providerThinking, pairedProviderBlocks, parsedText);
|
|
1649
|
+
if (!run)
|
|
1650
|
+
continue;
|
|
1651
|
+
content.splice(parsedIndex, 1, ...run.map((p) => {
|
|
1652
|
+
pairedProviderBlocks.add(p);
|
|
1653
|
+
const block = providerThinking[p];
|
|
1654
|
+
return {
|
|
1655
|
+
type: 'thinking',
|
|
1656
|
+
thinking: block.thinking ?? '',
|
|
1657
|
+
...(block.signature ? { signature: block.signature } : {}),
|
|
1658
|
+
};
|
|
1659
|
+
}));
|
|
1660
|
+
claimedParsedIndices.add(parsedIndex);
|
|
1661
|
+
}
|
|
1662
|
+
const seen = new Set(content.map((block) => thinkingCarrierKey(block)));
|
|
1663
|
+
const leftover = [];
|
|
1664
|
+
for (let p = 0; p < providerThinking.length; p++) {
|
|
1665
|
+
if (pairedProviderBlocks.has(p))
|
|
1666
|
+
continue;
|
|
1667
|
+
const block = providerThinking[p];
|
|
1668
|
+
const key = thinkingCarrierKey(block);
|
|
1669
|
+
if (seen.has(key))
|
|
1670
|
+
continue;
|
|
1671
|
+
seen.add(key);
|
|
1672
|
+
leftover.push(block);
|
|
1673
|
+
}
|
|
1674
|
+
for (const block of redacted) {
|
|
1675
|
+
const key = thinkingCarrierKey(block);
|
|
1676
|
+
if (seen.has(key))
|
|
1677
|
+
continue;
|
|
1678
|
+
seen.add(key);
|
|
1679
|
+
leftover.push(block);
|
|
1351
1680
|
}
|
|
1681
|
+
if (leftover.length > 0)
|
|
1682
|
+
content.unshift(...leftover);
|
|
1352
1683
|
}
|
|
1353
1684
|
// ==========================================================================
|
|
1354
1685
|
// Internal Methods
|
|
@@ -1424,19 +1755,15 @@ export class Membrane {
|
|
|
1424
1755
|
return { type: 'enabled', budget_tokens: budget, ...(display ? { display } : {}) };
|
|
1425
1756
|
}
|
|
1426
1757
|
/**
|
|
1427
|
-
* Transform a normalized request into provider format using the formatter
|
|
1758
|
+
* Transform a normalized request into provider format using the formatter.
|
|
1759
|
+
*
|
|
1760
|
+
* `activeFormatter` is the instance the caller already selected via
|
|
1761
|
+
* resolveActiveFormatter — including that selection's Responses-transport
|
|
1762
|
+
* authority rule, which used to live inline here. It is a parameter and not
|
|
1763
|
+
* a re-derivation so that the formatter which BUILDS is the same one that
|
|
1764
|
+
* resolved the tool mode and drives the loop.
|
|
1428
1765
|
*/
|
|
1429
|
-
transformRequest(request, formatter) {
|
|
1430
|
-
// The Responses adapter's input is a provider-native item array. A generic
|
|
1431
|
-
// per-request formatter (for example Context Manager's NativeFormatter)
|
|
1432
|
-
// produces Anthropic-style `{ role, content: [{ type: 'text' }] }`
|
|
1433
|
-
// envelopes, which the Responses API rejects before inference. Keep the
|
|
1434
|
-
// configured Responses formatter authoritative at this transport boundary;
|
|
1435
|
-
// per-request formatter overrides remain available for adapters whose wire
|
|
1436
|
-
// format supports them.
|
|
1437
|
-
const activeFormatter = this.adapter.name === 'openai-responses-api' && this.formatter.name === 'openai-responses'
|
|
1438
|
-
? this.formatter
|
|
1439
|
-
: formatter ?? this.formatter;
|
|
1766
|
+
transformRequest(request, activeFormatter = this.formatter) {
|
|
1440
1767
|
// Extract user-provided stop sequences
|
|
1441
1768
|
const additionalStopSequences = Array.isArray(request.stopSequences)
|
|
1442
1769
|
? request.stopSequences
|
|
@@ -1450,9 +1777,14 @@ export class Membrane {
|
|
|
1450
1777
|
participantMode: 'multiuser',
|
|
1451
1778
|
assistantParticipant: request.assistantParticipant ?? this.config.assistantParticipant ?? 'Claude',
|
|
1452
1779
|
tools: request.tools,
|
|
1780
|
+
// One resolution for every entry point: complete() used to build from the
|
|
1781
|
+
// formatter's constructor-time mode alone, so request.toolMode was a
|
|
1782
|
+
// second, disconnected source of truth on this path.
|
|
1783
|
+
toolMode: this.resolveToolMode(request, activeFormatter),
|
|
1453
1784
|
thinking: request.config.thinking,
|
|
1454
1785
|
systemPrompt: request.system,
|
|
1455
1786
|
promptCaching: request.promptCaching ?? this.config.defaultPromptCaching ?? true, // Default true for backward compat
|
|
1787
|
+
cacheMarkers: request.cacheMarkers ?? 'membrane-system',
|
|
1456
1788
|
cacheTtl: request.cacheTtl,
|
|
1457
1789
|
additionalStopSequences,
|
|
1458
1790
|
maxParticipantsForStop,
|
|
@@ -1473,7 +1805,9 @@ export class Membrane {
|
|
|
1473
1805
|
const providerRequest = {
|
|
1474
1806
|
...this.getBaseProviderParams(request.config),
|
|
1475
1807
|
messages: buildResult.messages,
|
|
1476
|
-
|
|
1808
|
+
// Owned, not aliased: the wire clamp strips markers in place, and a
|
|
1809
|
+
// formatter may pass the caller's own system array straight through.
|
|
1810
|
+
system: ownSystemBlocks(buildResult.systemContent),
|
|
1477
1811
|
stopSequences: buildResult.stopSequences,
|
|
1478
1812
|
tools: buildResult.nativeTools,
|
|
1479
1813
|
extra: {
|
|
@@ -1486,8 +1820,8 @@ export class Membrane {
|
|
|
1486
1820
|
// literal `<thinking>` text prefix instead of the API feature — drop the
|
|
1487
1821
|
// API param when the built request actually ends in an assistant prefill.
|
|
1488
1822
|
// Chat-style builds (no prefill) keep it.
|
|
1489
|
-
if (buildResult.assistantPrefill
|
|
1490
|
-
|
|
1823
|
+
if (buildResult.assistantPrefill) {
|
|
1824
|
+
stripThinkingForPrefill(providerRequest);
|
|
1491
1825
|
}
|
|
1492
1826
|
return { providerRequest, prefillResult: buildResult };
|
|
1493
1827
|
}
|
|
@@ -1497,17 +1831,45 @@ export class Membrane {
|
|
|
1497
1831
|
// compatibility won't catch the excess field (checked only on object
|
|
1498
1832
|
// literals, not on variables). Leaving it in would silently leak the
|
|
1499
1833
|
// normalized form into every adapter's options.
|
|
1500
|
-
const { normalizedRequest, refusalRetries, onRetrying, ...adapterOptions } = options;
|
|
1834
|
+
const { normalizedRequest, refusalRetries, onRetrying, onWireCacheMarkers, ...adapterOptions } = options;
|
|
1501
1835
|
const finalRequest = (await this.applyBeforeRequestHook(normalizedRequest, request));
|
|
1836
|
+
// Last exit before the adapter: the only place that sees EVERY
|
|
1837
|
+
// contribution (builder, formatter, passthrough, float, hook). Every
|
|
1838
|
+
// streaming path — stream(), streamYielding(), both tool loops — funnels
|
|
1839
|
+
// through here, so this is the one clamp they all get, and its tally is
|
|
1840
|
+
// therefore the only count that describes the wire.
|
|
1841
|
+
const markerCount = normalizedRequest.cacheMarkers === 'cm-owned'
|
|
1842
|
+
? assertCacheMarkersWithinLimit(finalRequest, 'streamOnce')
|
|
1843
|
+
: clampCacheMarkers(finalRequest, 'streamOnce').total;
|
|
1844
|
+
normalizedRequest.onCacheWireReceipt?.(computeCacheWireReceipt(finalRequest));
|
|
1845
|
+
onWireCacheMarkers?.(markerCount);
|
|
1502
1846
|
// Retries are only safe when the caller can discard the abandoned
|
|
1503
1847
|
// attempt, so they require BOTH a budget and an onRetrying hook.
|
|
1504
1848
|
const maxAttempts = onRetrying ? Math.max(0, refusalRetries ?? 0) : 0;
|
|
1505
1849
|
let retried = 0;
|
|
1850
|
+
// Every re-issued attempt was a completed, billed provider call. The
|
|
1851
|
+
// caller's usage accumulator only ever sees the surviving result, so the
|
|
1852
|
+
// abandoned spend rides back out on the result itself.
|
|
1853
|
+
let discardedUsage;
|
|
1854
|
+
let providerCalls = 0;
|
|
1506
1855
|
while (true) {
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1856
|
+
providerCalls++;
|
|
1857
|
+
const rawResult = await this.adapter.stream(finalRequest, callbacks, adapterOptions);
|
|
1858
|
+
// Restate usage in the one convention before any accumulator, ratio or
|
|
1859
|
+
// price sees it — this is the only door streamed usage enters through.
|
|
1860
|
+
const result = {
|
|
1861
|
+
...rawResult,
|
|
1862
|
+
usage: normalizeUsageToCacheExcluded(rawResult.usage, this.adapter.name, this.adapter.usageCacheConvention),
|
|
1863
|
+
};
|
|
1864
|
+
if (result.stopReason !== 'refusal' || retried >= maxAttempts) {
|
|
1865
|
+
return {
|
|
1866
|
+
...result,
|
|
1867
|
+
providerCalls,
|
|
1868
|
+
...(discardedUsage ? { discardedUsage } : {}),
|
|
1869
|
+
};
|
|
1870
|
+
}
|
|
1510
1871
|
retried++;
|
|
1872
|
+
discardedUsage = this.mergeDiscardedAttempts(discardedUsage, this.discardedAttemptFrom(result.usage));
|
|
1511
1873
|
const category = result.raw
|
|
1512
1874
|
?.response?.stop_details?.category;
|
|
1513
1875
|
onRetrying({ attempt: retried, maxAttempts, category });
|
|
@@ -1516,6 +1878,12 @@ export class Membrane {
|
|
|
1516
1878
|
buildContinuationRequest(originalRequest, prefillResult, accumulated) {
|
|
1517
1879
|
// Anthropic quirk: assistant content cannot end with trailing whitespace
|
|
1518
1880
|
const trimmedAccumulated = accumulated.trimEnd();
|
|
1881
|
+
// Everything before the watermark already rides EARLIER messages (a
|
|
1882
|
+
// persisted split turn), so only the suffix belongs in the trailing
|
|
1883
|
+
// assistant prefill — replacing it with the whole document would
|
|
1884
|
+
// duplicate the pre-seam text and flatten the image user-turn away.
|
|
1885
|
+
const baseOffset = prefillResult.accumulatedBaseOffset ?? 0;
|
|
1886
|
+
const trailingContent = accumulated.slice(baseOffset).trimEnd();
|
|
1519
1887
|
// Build continuation messages: keep all messages up to last assistant,
|
|
1520
1888
|
// then replace/add the accumulated content
|
|
1521
1889
|
const messages = [...prefillResult.messages];
|
|
@@ -1523,32 +1891,32 @@ export class Membrane {
|
|
|
1523
1891
|
let foundAssistant = false;
|
|
1524
1892
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
1525
1893
|
if (messages[i]?.role === 'assistant') {
|
|
1526
|
-
messages[i] = { role: 'assistant', content:
|
|
1894
|
+
messages[i] = { role: 'assistant', content: trailingContent };
|
|
1527
1895
|
foundAssistant = true;
|
|
1528
1896
|
break;
|
|
1529
1897
|
}
|
|
1530
1898
|
}
|
|
1531
1899
|
if (!foundAssistant) {
|
|
1532
|
-
messages.push({ role: 'assistant', content:
|
|
1900
|
+
messages.push({ role: 'assistant', content: trailingContent });
|
|
1533
1901
|
}
|
|
1534
|
-
return {
|
|
1902
|
+
return stripThinkingForPrefill({
|
|
1535
1903
|
...this.getBaseProviderParams(originalRequest.config),
|
|
1536
1904
|
// Continuations always end in an assistant prefill — the API rejects
|
|
1537
1905
|
// extended thinking combined with prefill, so never send the param here
|
|
1538
1906
|
thinking: undefined,
|
|
1539
1907
|
messages,
|
|
1540
|
-
system: prefillResult.systemContent
|
|
1541
|
-
? (Array.isArray(prefillResult.systemContent) && prefillResult.systemContent.length > 0
|
|
1542
|
-
? prefillResult.systemContent
|
|
1543
|
-
: prefillResult.systemContent)
|
|
1544
|
-
: undefined,
|
|
1908
|
+
system: ownSystemBlocks(prefillResult.systemContent) ?? undefined,
|
|
1545
1909
|
stopSequences: prefillResult.stopSequences,
|
|
1546
1910
|
extra: {
|
|
1547
1911
|
...originalRequest.providerParams,
|
|
1912
|
+
// Same contract transformRequest sends: adapters that reason about
|
|
1913
|
+
// the normalized shape (or fall back to serializing it) must not see
|
|
1914
|
+
// a continuation as a request with no normalized form at all.
|
|
1915
|
+
normalizedMessages: originalRequest.messages,
|
|
1548
1916
|
// Pre-serialized prompt for completions adapters — skip re-serialization
|
|
1549
1917
|
prompt: trimmedAccumulated,
|
|
1550
1918
|
},
|
|
1551
|
-
};
|
|
1919
|
+
});
|
|
1552
1920
|
}
|
|
1553
1921
|
/**
|
|
1554
1922
|
* Build continuation request with split-turn image injection.
|
|
@@ -1570,6 +1938,11 @@ export class Membrane {
|
|
|
1570
1938
|
buildContinuationRequestWithImages(originalRequest, prefillResult, accumulated, images, afterImageXml) {
|
|
1571
1939
|
// Anthropic quirk: assistant content cannot end with trailing whitespace
|
|
1572
1940
|
const trimmedAccumulated = accumulated.trimEnd();
|
|
1941
|
+
// The split replaces only the CURRENT trailing assistant message, which
|
|
1942
|
+
// covers the accumulated text from the previous seam onward (0 on the
|
|
1943
|
+
// first split, the previous image seam on a later one).
|
|
1944
|
+
const baseOffset = prefillResult.accumulatedBaseOffset ?? 0;
|
|
1945
|
+
const trailingContent = accumulated.slice(baseOffset).trimEnd();
|
|
1573
1946
|
// Build messages: copy all, then replace only the last assistant with split-turn
|
|
1574
1947
|
const messages = prefillResult.messages.map(msg => ({ ...msg }));
|
|
1575
1948
|
// Find last assistant — replace in-place via splice to preserve history
|
|
@@ -1583,7 +1956,7 @@ export class Membrane {
|
|
|
1583
1956
|
// Anthropic quirk: assistant content cannot end with trailing whitespace
|
|
1584
1957
|
const trimmedAfterXml = afterImageXml.trimEnd();
|
|
1585
1958
|
const splitTurnMessages = [
|
|
1586
|
-
{ role: 'assistant', content:
|
|
1959
|
+
{ role: 'assistant', content: trailingContent },
|
|
1587
1960
|
{ role: 'user', content: images },
|
|
1588
1961
|
{ role: 'assistant', content: trimmedAfterXml },
|
|
1589
1962
|
];
|
|
@@ -1593,20 +1966,39 @@ export class Membrane {
|
|
|
1593
1966
|
else {
|
|
1594
1967
|
messages.push(...splitTurnMessages);
|
|
1595
1968
|
}
|
|
1596
|
-
|
|
1969
|
+
// PERSIST the split. Later rounds rebuild from prefillResult.messages;
|
|
1970
|
+
// without this the image user-turn exists on exactly one request and the
|
|
1971
|
+
// next continuation flattens the accumulated document back over it —
|
|
1972
|
+
// leaving <function_results> XML asserting a screenshot the model can no
|
|
1973
|
+
// longer see. Reassign (never mutate in place): the previous array is
|
|
1974
|
+
// still referenced by the request already on the wire. The watermark
|
|
1975
|
+
// moves to the seam — the point in `accumulated` where afterImageXml is
|
|
1976
|
+
// about to be appended — so the next builder replaces only the closing
|
|
1977
|
+
// assistant turn.
|
|
1978
|
+
prefillResult.messages = messages;
|
|
1979
|
+
prefillResult.accumulatedBaseOffset = accumulated.length;
|
|
1980
|
+
return stripThinkingForPrefill({
|
|
1597
1981
|
...this.getBaseProviderParams(originalRequest.config),
|
|
1598
1982
|
// Continuations always end in an assistant prefill — the API rejects
|
|
1599
1983
|
// extended thinking combined with prefill, so never send the param here
|
|
1600
1984
|
thinking: undefined,
|
|
1601
1985
|
messages,
|
|
1602
|
-
system: prefillResult.systemContent
|
|
1603
|
-
? (Array.isArray(prefillResult.systemContent) && prefillResult.systemContent.length > 0
|
|
1604
|
-
? prefillResult.systemContent
|
|
1605
|
-
: prefillResult.systemContent)
|
|
1606
|
-
: undefined,
|
|
1986
|
+
system: ownSystemBlocks(prefillResult.systemContent) ?? undefined,
|
|
1607
1987
|
stopSequences: prefillResult.stopSequences,
|
|
1608
|
-
|
|
1609
|
-
|
|
1988
|
+
// Copied, not aliased: the guard below deletes the smuggled thinking
|
|
1989
|
+
// config, and mutating the caller's own providerParams object would
|
|
1990
|
+
// silently disable thinking on their NEXT (non-prefill) request.
|
|
1991
|
+
extra: {
|
|
1992
|
+
...originalRequest.providerParams,
|
|
1993
|
+
// Same contract as transformRequest and the plain continuation
|
|
1994
|
+
// builder. Without these a completions-style adapter fell through to
|
|
1995
|
+
// serializing PROVIDER-shaped messages as if they were normalized
|
|
1996
|
+
// ones, re-adding participant stop sequences the continuation
|
|
1997
|
+
// deliberately suppresses.
|
|
1998
|
+
normalizedMessages: originalRequest.messages,
|
|
1999
|
+
prompt: trimmedAccumulated,
|
|
2000
|
+
},
|
|
2001
|
+
});
|
|
1610
2002
|
}
|
|
1611
2003
|
transformResponse(providerResponse, request, prefillResult, startTime, attempts, rawRequest) {
|
|
1612
2004
|
// Extract text from response
|
|
@@ -1674,6 +2066,7 @@ export class Membrane {
|
|
|
1674
2066
|
}
|
|
1675
2067
|
// Parse XML tool calls from text if no native tool_use blocks were found
|
|
1676
2068
|
// This handles prefill mode where tools are XML in the text
|
|
2069
|
+
let emptyToolBlocks = 0;
|
|
1677
2070
|
if (toolCalls.length === 0 && rawAssistantText.includes('<function_calls>')) {
|
|
1678
2071
|
const parsed = parseToolCalls(rawAssistantText);
|
|
1679
2072
|
if (parsed?.calls.length) {
|
|
@@ -1681,12 +2074,25 @@ export class Membrane {
|
|
|
1681
2074
|
toolCalls.push(tc);
|
|
1682
2075
|
}
|
|
1683
2076
|
}
|
|
2077
|
+
else if (parsed) {
|
|
2078
|
+
emptyToolBlocks = 1;
|
|
2079
|
+
}
|
|
1684
2080
|
}
|
|
2081
|
+
const unclosedToolBlock = endsWithPartialToolBlock(rawAssistantText);
|
|
1685
2082
|
const stopReason = this.mapStopReason(providerResponse.stopReason);
|
|
2083
|
+
this.reportToolParseDiagnostics({ unclosedToolBlock, emptyToolBlocks }, stopReason);
|
|
1686
2084
|
const durationMs = Date.now() - startTime;
|
|
2085
|
+
// `NormalizedResponse.usage` is typed DetailedUsage and the streaming paths
|
|
2086
|
+
// already return the whole thing; complete() used to narrow it to
|
|
2087
|
+
// input/output here, so a caller reading `response.usage.cacheReadTokens`
|
|
2088
|
+
// saw undefined on one path and a number on the other.
|
|
1687
2089
|
const usage = {
|
|
1688
2090
|
inputTokens: providerResponse.usage.inputTokens,
|
|
1689
2091
|
outputTokens: providerResponse.usage.outputTokens,
|
|
2092
|
+
cacheCreationTokens: providerResponse.usage.cacheCreationTokens,
|
|
2093
|
+
cacheReadTokens: providerResponse.usage.cacheReadTokens,
|
|
2094
|
+
thinkingTokens: providerResponse.usage.thinkingTokens,
|
|
2095
|
+
estimatedCost: this.estimateCost(providerResponse.usage, request.config.model, providerResponse.model),
|
|
1690
2096
|
};
|
|
1691
2097
|
return {
|
|
1692
2098
|
content,
|
|
@@ -1700,14 +2106,9 @@ export class Membrane {
|
|
|
1700
2106
|
reason: stopReason,
|
|
1701
2107
|
triggeredSequence: providerResponse.stopSequence,
|
|
1702
2108
|
wasTruncated: stopReason === 'max_tokens',
|
|
2109
|
+
unclosedToolBlock,
|
|
1703
2110
|
},
|
|
1704
|
-
usage
|
|
1705
|
-
inputTokens: providerResponse.usage.inputTokens,
|
|
1706
|
-
outputTokens: providerResponse.usage.outputTokens,
|
|
1707
|
-
cacheCreationTokens: providerResponse.usage.cacheCreationTokens,
|
|
1708
|
-
cacheReadTokens: providerResponse.usage.cacheReadTokens,
|
|
1709
|
-
estimatedCost: this.estimateCost(providerResponse.usage, request.config.model),
|
|
1710
|
-
},
|
|
2111
|
+
usage,
|
|
1711
2112
|
timing: {
|
|
1712
2113
|
totalDurationMs: durationMs,
|
|
1713
2114
|
attempts,
|
|
@@ -1730,12 +2131,47 @@ export class Membrane {
|
|
|
1730
2131
|
},
|
|
1731
2132
|
};
|
|
1732
2133
|
}
|
|
1733
|
-
|
|
2134
|
+
/**
|
|
2135
|
+
* The turn is over, and the two guards that detect a half-written tool block
|
|
2136
|
+
* finally have a call site. Both shapes are defects a consumer must not
|
|
2137
|
+
* persist blind: an unclosed block splices onto the NEXT round's closing tag
|
|
2138
|
+
* (the loop does not resume on a length stop, so max_tokens leaves exactly
|
|
2139
|
+
* this), and a block that parsed to nothing means the model believes it
|
|
2140
|
+
* called a tool that never ran.
|
|
2141
|
+
*/
|
|
2142
|
+
reportToolParseDiagnostics(diagnostics, stopReason) {
|
|
2143
|
+
const warnLog = this.config.logger ?? console;
|
|
2144
|
+
if (diagnostics.unclosedToolBlock) {
|
|
2145
|
+
warnLog.warn(`[membrane] turn ended (${stopReason}) with an unclosed tool block in the ` +
|
|
2146
|
+
`assistant text — the loop does not resume on a length stop. Persisting this ` +
|
|
2147
|
+
`turn verbatim lets the next round's closing tag splice onto the stale ` +
|
|
2148
|
+
`opener; see details.stop.unclosedToolBlock.`);
|
|
2149
|
+
}
|
|
2150
|
+
if (diagnostics.emptyToolBlocks > 0) {
|
|
2151
|
+
warnLog.warn(`[membrane] ${diagnostics.emptyToolBlocks} function_calls block(s) parsed to ` +
|
|
2152
|
+
`zero tool calls — always a defect, never a normal ending. The call was ` +
|
|
2153
|
+
`returned as assistant text and nothing executed.`);
|
|
2154
|
+
}
|
|
2155
|
+
if (diagnostics.splicedToolBlocks) {
|
|
2156
|
+
warnLog.warn(`[membrane] ${diagnostics.splicedToolBlocks} tool block(s) spanned a second ` +
|
|
2157
|
+
`<function_calls> opener and were re-anchored to the innermost one — an ` +
|
|
2158
|
+
`earlier truncated block is present in this conversation's assistant text.`);
|
|
2159
|
+
}
|
|
2160
|
+
if (diagnostics.unclosedInvokeHeads) {
|
|
2161
|
+
warnLog.warn(`[membrane] ${diagnostics.unclosedInvokeHeads} <invoke> head(s) were left ` +
|
|
2162
|
+
`unclosed and swallowed the invoke that followed — nothing was dispatched ` +
|
|
2163
|
+
`under an unclosed head's name, and the call it absorbed was re-anchored ` +
|
|
2164
|
+
`and ran with its own parameters.`);
|
|
2165
|
+
}
|
|
2166
|
+
}
|
|
2167
|
+
buildFinalResponse(accumulated, contentBlocks, stopReason, turnUsage, request, prefillResult, startTime, attempts, rawRequest, rawResponse, executedToolCalls = [], executedToolResults = [], startInsideBlock = null, triggeredSequence) {
|
|
2168
|
+
const usage = turnUsage.total;
|
|
1734
2169
|
// Parse accumulated text into structured content blocks
|
|
1735
2170
|
// This extracts thinking, tool_use, tool_result, and text blocks
|
|
1736
2171
|
let finalContent;
|
|
1737
2172
|
let toolCalls;
|
|
1738
2173
|
let toolResults;
|
|
2174
|
+
let unclosedToolBlock = false;
|
|
1739
2175
|
if (contentBlocks.length > 0) {
|
|
1740
2176
|
// Native mode - content blocks already structured
|
|
1741
2177
|
finalContent = contentBlocks;
|
|
@@ -1751,6 +2187,8 @@ export class Membrane {
|
|
|
1751
2187
|
finalContent = parsed.blocks;
|
|
1752
2188
|
toolCalls = parsed.toolCalls.length > 0 ? parsed.toolCalls : executedToolCalls;
|
|
1753
2189
|
toolResults = parsed.toolResults.length > 0 ? parsed.toolResults : executedToolResults;
|
|
2190
|
+
unclosedToolBlock = parsed.unclosedToolBlock;
|
|
2191
|
+
this.reportToolParseDiagnostics(parsed, stopReason);
|
|
1754
2192
|
}
|
|
1755
2193
|
const durationMs = Date.now() - startTime;
|
|
1756
2194
|
return {
|
|
@@ -1765,19 +2203,21 @@ export class Membrane {
|
|
|
1765
2203
|
reason: stopReason,
|
|
1766
2204
|
triggeredSequence,
|
|
1767
2205
|
wasTruncated: stopReason === 'max_tokens',
|
|
2206
|
+
unclosedToolBlock,
|
|
1768
2207
|
},
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
2208
|
+
// Priced per round by the accumulator and summed — NOT re-derived here
|
|
2209
|
+
// from the turn totals, which would re-bill every round at the last
|
|
2210
|
+
// model's rate.
|
|
2211
|
+
usage,
|
|
1773
2212
|
timing: {
|
|
1774
2213
|
totalDurationMs: durationMs,
|
|
1775
2214
|
attempts,
|
|
1776
2215
|
},
|
|
1777
2216
|
model: {
|
|
1778
2217
|
requested: request.config.model,
|
|
1779
|
-
actual: request.config.model,
|
|
2218
|
+
actual: turnUsage.lastServedModel || request.config.model,
|
|
1780
2219
|
provider: this.adapter.name,
|
|
2220
|
+
perRound: turnUsage.perRound,
|
|
1781
2221
|
},
|
|
1782
2222
|
cache: {
|
|
1783
2223
|
markersInRequest: prefillResult.cacheMarkersApplied ?? 0,
|
|
@@ -1792,6 +2232,45 @@ export class Membrane {
|
|
|
1792
2232
|
},
|
|
1793
2233
|
};
|
|
1794
2234
|
}
|
|
2235
|
+
/**
|
|
2236
|
+
* Fold one discarded (billed but abandoned) attempt's usage into a carry.
|
|
2237
|
+
* Returns a NEW object so a caller's earlier snapshot is never mutated.
|
|
2238
|
+
*/
|
|
2239
|
+
mergeDiscardedAttempts(carry, add) {
|
|
2240
|
+
if (!add)
|
|
2241
|
+
return carry;
|
|
2242
|
+
const next = carry
|
|
2243
|
+
? { ...carry }
|
|
2244
|
+
: { attempts: 0, inputTokens: 0, outputTokens: 0 };
|
|
2245
|
+
next.attempts += add.attempts;
|
|
2246
|
+
next.inputTokens += add.inputTokens;
|
|
2247
|
+
next.outputTokens += add.outputTokens;
|
|
2248
|
+
if (add.cacheCreationTokens) {
|
|
2249
|
+
next.cacheCreationTokens = (next.cacheCreationTokens ?? 0) + add.cacheCreationTokens;
|
|
2250
|
+
}
|
|
2251
|
+
if (add.cacheReadTokens) {
|
|
2252
|
+
next.cacheReadTokens = (next.cacheReadTokens ?? 0) + add.cacheReadTokens;
|
|
2253
|
+
}
|
|
2254
|
+
return next;
|
|
2255
|
+
}
|
|
2256
|
+
/** One provider call's usage as a single-attempt discard record. */
|
|
2257
|
+
discardedAttemptFrom(usage) {
|
|
2258
|
+
const detailed = (usage ?? { inputTokens: 0, outputTokens: 0 });
|
|
2259
|
+
return {
|
|
2260
|
+
attempts: 1,
|
|
2261
|
+
inputTokens: detailed.inputTokens ?? 0,
|
|
2262
|
+
outputTokens: detailed.outputTokens ?? 0,
|
|
2263
|
+
...(detailed.cacheCreationTokens ? { cacheCreationTokens: detailed.cacheCreationTokens } : {}),
|
|
2264
|
+
...(detailed.cacheReadTokens ? { cacheReadTokens: detailed.cacheReadTokens } : {}),
|
|
2265
|
+
};
|
|
2266
|
+
}
|
|
2267
|
+
/** Price the discarded spend so a caller can read it without re-deriving. */
|
|
2268
|
+
pricedDiscardedAttempts(discarded, model) {
|
|
2269
|
+
if (!discarded)
|
|
2270
|
+
return undefined;
|
|
2271
|
+
const estimatedCost = this.estimateCost(discarded, model);
|
|
2272
|
+
return estimatedCost ? { ...discarded, estimatedCost } : discarded;
|
|
2273
|
+
}
|
|
1795
2274
|
mapStopReason(providerReason) {
|
|
1796
2275
|
switch (providerReason) {
|
|
1797
2276
|
case 'end_turn':
|
|
@@ -1812,19 +2291,50 @@ export class Membrane {
|
|
|
1812
2291
|
}
|
|
1813
2292
|
}
|
|
1814
2293
|
calculateCacheHitRatio(usage) {
|
|
1815
|
-
|
|
1816
|
-
const total = usage.inputTokens ?? 0;
|
|
1817
|
-
if (total === 0)
|
|
1818
|
-
return 0;
|
|
1819
|
-
return cacheRead / total;
|
|
2294
|
+
return calculateCacheHitRatio(usage);
|
|
1820
2295
|
}
|
|
1821
|
-
|
|
1822
|
-
|
|
2296
|
+
/**
|
|
2297
|
+
* Pricing is resolved on TWO axes, and SOURCE outranks SPECIFICITY:
|
|
2298
|
+
*
|
|
2299
|
+
* registry[served] → registry[requested] → builtin[served] → builtin[requested]
|
|
2300
|
+
*
|
|
2301
|
+
* Specificity — preferring the model that ACTUALLY served over the id that
|
|
2302
|
+
* was requested — is real: an alias or an auto-routed request otherwise
|
|
2303
|
+
* prices against a string the provider already replaced, and a live
|
|
2304
|
+
* 2026-08-25 call asking for `gpt-4o-mini` was served by
|
|
2305
|
+
* `gpt-4o-mini-2024-07-18`. But it only breaks ties WITHIN one source.
|
|
2306
|
+
* A configured `ModelRegistry` is the caller stating their own rates —
|
|
2307
|
+
* account-specific, negotiated, authoritative; the built-in table is
|
|
2308
|
+
* membrane's shipped guess at public list prices. Merging the two per-model
|
|
2309
|
+
* (`registry[served] ?? builtin[served]`, return on the first hit) let the
|
|
2310
|
+
* guess for a snapshot outrank the caller's own entry for the alias they
|
|
2311
|
+
* asked for, so a caller who prices their alias and lets the provider pick
|
|
2312
|
+
* the snapshot was billed at membrane's number instead of theirs.
|
|
2313
|
+
*
|
|
2314
|
+
* Both fallbacks stay: the served model may be absent from a source, and the
|
|
2315
|
+
* provider may name none at all.
|
|
2316
|
+
*/
|
|
2317
|
+
resolvePricing(requestedModel, actualModel) {
|
|
2318
|
+
const servedModel = actualModel && actualModel !== requestedModel ? actualModel : undefined;
|
|
2319
|
+
const fromRegistry = (modelId) => modelId === undefined ? undefined : this.registry?.getPricing(modelId);
|
|
2320
|
+
const fromBuiltin = (modelId) => modelId === undefined ? undefined : getDefaultPricing(modelId);
|
|
2321
|
+
return fromRegistry(servedModel)
|
|
2322
|
+
?? fromRegistry(requestedModel)
|
|
2323
|
+
?? fromBuiltin(servedModel)
|
|
2324
|
+
?? fromBuiltin(requestedModel);
|
|
1823
2325
|
}
|
|
1824
2326
|
/** Resolve pricing + calculate cost in one call (for one-shot use outside loops). */
|
|
1825
|
-
estimateCost(usage,
|
|
1826
|
-
const pricing = this.resolvePricing(
|
|
1827
|
-
|
|
2327
|
+
estimateCost(usage, requestedModel, actualModel) {
|
|
2328
|
+
const pricing = this.resolvePricing(requestedModel, actualModel);
|
|
2329
|
+
if (!pricing) {
|
|
2330
|
+
// An absent cost and a zero cost are different claims. Returning
|
|
2331
|
+
// undefined says "membrane does not know what this costs"; saying it out
|
|
2332
|
+
// loud once per model keeps that from reading as "free" to a caller that
|
|
2333
|
+
// only ever sees the omission.
|
|
2334
|
+
warnUnpricedModel(actualModel || requestedModel);
|
|
2335
|
+
return undefined;
|
|
2336
|
+
}
|
|
2337
|
+
return calculateCost(usage, pricing);
|
|
1828
2338
|
}
|
|
1829
2339
|
calculateRetryDelay(attempt, overloaded = false) {
|
|
1830
2340
|
const { retryDelayMs, backoffMultiplier, maxRetryDelayMs } = overloaded
|
|
@@ -1858,6 +2368,11 @@ export class Membrane {
|
|
|
1858
2368
|
* Check if an error is an abort error
|
|
1859
2369
|
*/
|
|
1860
2370
|
isAbortError(error) {
|
|
2371
|
+
// An adapter's own deadline: a timeout by classification, still an abort
|
|
2372
|
+
// by provenance, so the streaming paths hand back the partial content
|
|
2373
|
+
// they collected instead of throwing.
|
|
2374
|
+
if (isTimeoutAbortError(error))
|
|
2375
|
+
return true;
|
|
1861
2376
|
if (error instanceof Error) {
|
|
1862
2377
|
// Standard AbortError
|
|
1863
2378
|
if (error.name === 'AbortError')
|
|
@@ -1872,6 +2387,21 @@ export class Membrane {
|
|
|
1872
2387
|
}
|
|
1873
2388
|
return false;
|
|
1874
2389
|
}
|
|
2390
|
+
/**
|
|
2391
|
+
* Why a caught abort happened. The caller's own signal is authoritative:
|
|
2392
|
+
* if it fired, the cancellation is theirs whatever the error text says.
|
|
2393
|
+
* Otherwise an adapter-side deadline classifies as a timeout — the adapters
|
|
2394
|
+
* mark the abort createCombinedSignal's timeoutMs raises and map it to a
|
|
2395
|
+
* TimeoutAbortError, so the identity survives their error handling — and
|
|
2396
|
+
* anything else that reached the abort catch is a failure, not a person.
|
|
2397
|
+
*/
|
|
2398
|
+
abortReason(error, signal) {
|
|
2399
|
+
if (signal?.aborted)
|
|
2400
|
+
return 'user';
|
|
2401
|
+
if (classifyError(error).type === 'timeout')
|
|
2402
|
+
return 'timeout';
|
|
2403
|
+
return 'error';
|
|
2404
|
+
}
|
|
1875
2405
|
/**
|
|
1876
2406
|
* Build an AbortedResponse from current execution state
|
|
1877
2407
|
*/
|
|
@@ -1921,7 +2451,12 @@ export class Membrane {
|
|
|
1921
2451
|
* ```
|
|
1922
2452
|
*/
|
|
1923
2453
|
streamYielding(request, options = {}) {
|
|
1924
|
-
|
|
2454
|
+
// YieldingStreamOptions carries no per-request formatter override, so the
|
|
2455
|
+
// selection here can only land on the instance formatter — it goes through
|
|
2456
|
+
// resolveActiveFormatter anyway so this path reads the same single source
|
|
2457
|
+
// as complete() and stream() if an override is ever added.
|
|
2458
|
+
const activeFormatter = this.resolveActiveFormatter();
|
|
2459
|
+
const toolMode = this.resolveToolMode(request, activeFormatter);
|
|
1925
2460
|
// refusalRetries is implemented on the native path only. The XML path
|
|
1926
2461
|
// accumulates into a streaming parser carrying prefill context and
|
|
1927
2462
|
// resumption depths; rolling that back mid-turn is a separate problem,
|
|
@@ -1933,14 +2468,14 @@ export class Membrane {
|
|
|
1933
2468
|
}
|
|
1934
2469
|
// Create the yielding stream with the appropriate inference runner
|
|
1935
2470
|
const runInference = toolMode === 'native'
|
|
1936
|
-
? (stream) => this.runNativeToolsYielding(request, options, stream)
|
|
1937
|
-
: (stream) => this.runXmlToolsYielding(request, options, stream);
|
|
2471
|
+
? (stream) => this.runNativeToolsYielding(request, options, stream, activeFormatter)
|
|
2472
|
+
: (stream) => this.runXmlToolsYielding(request, options, stream, activeFormatter);
|
|
1938
2473
|
return new YieldingStreamImpl(options, runInference);
|
|
1939
2474
|
}
|
|
1940
2475
|
/**
|
|
1941
2476
|
* Run XML-based tool execution with yielding stream.
|
|
1942
2477
|
*/
|
|
1943
|
-
async runXmlToolsYielding(request, options, stream) {
|
|
2478
|
+
async runXmlToolsYielding(request, options, stream, activeFormatter = this.resolveActiveFormatter()) {
|
|
1944
2479
|
const startTime = Date.now();
|
|
1945
2480
|
const { maxToolDepth: maxToolDepthOpt, emitTokens = true, emitBlocks = true, emitUsage = true, } = options;
|
|
1946
2481
|
// Yielding paths default to unlimited (the caller — typically an agent
|
|
@@ -1972,14 +2507,20 @@ export class Membrane {
|
|
|
1972
2507
|
let enteredViaResumption = false;
|
|
1973
2508
|
let prevRoundStopSequence;
|
|
1974
2509
|
const warnLog = this.config.logger ?? console;
|
|
1975
|
-
// Initialize parser from formatter
|
|
1976
|
-
|
|
2510
|
+
// Initialize parser from the formatter streamYielding selected, so the
|
|
2511
|
+
// parser and the build below read the same format.
|
|
2512
|
+
const formatter = activeFormatter;
|
|
1977
2513
|
const parser = formatter.createStreamParser();
|
|
1978
2514
|
let toolDepth = 0;
|
|
2515
|
+
// Honest turn telemetry: provider calls actually made (including refusal
|
|
2516
|
+
// re-issues inside streamOnce) and continuation rounds.
|
|
2517
|
+
let providerCalls = 0;
|
|
2518
|
+
let rounds = 0;
|
|
1979
2519
|
// Once-per-stream latch for the injectedMessages-unsupported warning.
|
|
1980
2520
|
let warnedInjectionUnsupported = false;
|
|
1981
|
-
|
|
1982
|
-
|
|
2521
|
+
// See streamWithXmlTools: one accumulator per turn, pricing each round
|
|
2522
|
+
// under the model that served it.
|
|
2523
|
+
const turnUsage = new TurnUsageAccumulator(request.config.model, (servedModel) => this.resolvePricing(request.config.model, servedModel));
|
|
1983
2524
|
const contentBlocks = [];
|
|
1984
2525
|
let lastStopReason = 'end_turn';
|
|
1985
2526
|
let lastStopSequence;
|
|
@@ -2021,13 +2562,13 @@ export class Membrane {
|
|
|
2021
2562
|
resumptionRounds++;
|
|
2022
2563
|
if (resumptionRounds === RESUMPTION_WARN_ROUNDS) {
|
|
2023
2564
|
warnLog.warn(`[membrane] automatic resumption at round ${resumptionRounds} ` +
|
|
2024
|
-
`(${
|
|
2565
|
+
`(${turnUsage.total.inputTokens} input tokens so far this turn) — ` +
|
|
2025
2566
|
`a spin shows up here before it shows up on the bill`);
|
|
2026
2567
|
}
|
|
2027
2568
|
if (resumptionRounds > maxResumptionRounds) {
|
|
2028
2569
|
warnLog.warn(`[membrane] automatic resumption cap (${maxResumptionRounds}) reached — ` +
|
|
2029
2570
|
`ending turn with stopReason 'round_limit'. ` +
|
|
2030
|
-
`${
|
|
2571
|
+
`${turnUsage.total.inputTokens} input tokens spent this turn.`);
|
|
2031
2572
|
return false;
|
|
2032
2573
|
}
|
|
2033
2574
|
return true;
|
|
@@ -2110,6 +2651,8 @@ export class Membrane {
|
|
|
2110
2651
|
wrapThinkingTags: true,
|
|
2111
2652
|
onRequest: (req) => { rawRequest = req; },
|
|
2112
2653
|
});
|
|
2654
|
+
rounds++;
|
|
2655
|
+
providerCalls += streamResult.providerCalls;
|
|
2113
2656
|
// If we detected stop sequence manually, fix up the parser and result
|
|
2114
2657
|
if (detectedStopSequence && truncatedAccumulated !== null) {
|
|
2115
2658
|
parser.reset();
|
|
@@ -2124,19 +2667,10 @@ export class Membrane {
|
|
|
2124
2667
|
rawResponse = streamResult.raw;
|
|
2125
2668
|
lastStopReason = this.mapStopReason(streamResult.stopReason);
|
|
2126
2669
|
lastStopSequence = streamResult.stopSequence ?? undefined;
|
|
2127
|
-
// Accumulate usage (including cache metrics)
|
|
2128
|
-
|
|
2129
|
-
totalUsage.outputTokens += streamResult.usage.outputTokens;
|
|
2130
|
-
if (streamResult.usage.cacheCreationTokens) {
|
|
2131
|
-
totalUsage.cacheCreationTokens = (totalUsage.cacheCreationTokens ?? 0) + streamResult.usage.cacheCreationTokens;
|
|
2132
|
-
}
|
|
2133
|
-
if (streamResult.usage.cacheReadTokens) {
|
|
2134
|
-
totalUsage.cacheReadTokens = (totalUsage.cacheReadTokens ?? 0) + streamResult.usage.cacheReadTokens;
|
|
2135
|
-
}
|
|
2136
|
-
if (pricing)
|
|
2137
|
-
totalUsage.estimatedCost = calculateCost(totalUsage, pricing);
|
|
2670
|
+
// Accumulate usage (including cache metrics), priced at this round's model
|
|
2671
|
+
const usageSoFar = turnUsage.addRound(streamResult.model, streamResult.usage);
|
|
2138
2672
|
if (emitUsage) {
|
|
2139
|
-
stream.emit({ type: 'usage', usage:
|
|
2673
|
+
stream.emit({ type: 'usage', usage: usageSoFar });
|
|
2140
2674
|
}
|
|
2141
2675
|
// Flush the parser
|
|
2142
2676
|
const flushResult = parser.flush();
|
|
@@ -2159,7 +2693,7 @@ export class Membrane {
|
|
|
2159
2693
|
warnLog.warn(`[membrane] ${consecutiveStalledResumptions} consecutive automatic resumptions ` +
|
|
2160
2694
|
`made no progress (${streamedThisRound} chars this round, stop ` +
|
|
2161
2695
|
`${JSON.stringify(lastStopSequence ?? null)} repeated) — ending turn with ` +
|
|
2162
|
-
`stopReason 'no_progress'. ${
|
|
2696
|
+
`stopReason 'no_progress'. ${turnUsage.total.inputTokens} input tokens spent this turn.`);
|
|
2163
2697
|
lastStopReason = 'no_progress';
|
|
2164
2698
|
break;
|
|
2165
2699
|
}
|
|
@@ -2407,9 +2941,10 @@ export class Membrane {
|
|
|
2407
2941
|
// Build final response
|
|
2408
2942
|
const fullAccumulated = parser.getAccumulated();
|
|
2409
2943
|
const newContent = fullAccumulated.slice(initialPrefillLength);
|
|
2410
|
-
const response = this.buildFinalResponse(newContent, contentBlocks, lastStopReason,
|
|
2944
|
+
const response = this.buildFinalResponse(newContent, contentBlocks, lastStopReason, turnUsage, request, prefillResult, startTime, providerCalls, rawRequest, rawResponse, executedToolCalls, executedToolResults, initialBlockType, lastStopSequence);
|
|
2411
2945
|
// Merge provider thinking signatures into parser-derived thinking blocks
|
|
2412
2946
|
this.mergeProviderThinkingBlocks(response.content, providerThinkingBlocks);
|
|
2947
|
+
response.details.timing.rounds = rounds;
|
|
2413
2948
|
stream.emit({ type: 'complete', response });
|
|
2414
2949
|
}
|
|
2415
2950
|
catch (error) {
|
|
@@ -2418,7 +2953,7 @@ export class Membrane {
|
|
|
2418
2953
|
const newContent = fullAccumulated.slice(initialPrefillLength);
|
|
2419
2954
|
stream.emit({
|
|
2420
2955
|
type: 'aborted',
|
|
2421
|
-
reason:
|
|
2956
|
+
reason: this.abortReason(error, stream.signal),
|
|
2422
2957
|
partialContent: parseAccumulatedIntoBlocks(newContent).blocks,
|
|
2423
2958
|
rawAssistantText: newContent,
|
|
2424
2959
|
toolCalls: executedToolCalls,
|
|
@@ -2433,7 +2968,7 @@ export class Membrane {
|
|
|
2433
2968
|
/**
|
|
2434
2969
|
* Run native tool execution with yielding stream.
|
|
2435
2970
|
*/
|
|
2436
|
-
async runNativeToolsYielding(request, options, stream) {
|
|
2971
|
+
async runNativeToolsYielding(request, options, stream, activeFormatter = this.resolveActiveFormatter()) {
|
|
2437
2972
|
const startTime = Date.now();
|
|
2438
2973
|
const { maxToolDepth: maxToolDepthOpt, emitTokens = true, emitBlocks = true, emitUsage = true, } = options;
|
|
2439
2974
|
// Yielding paths default to unlimited (the caller — typically an agent
|
|
@@ -2445,8 +2980,13 @@ export class Membrane {
|
|
|
2445
2980
|
? Infinity
|
|
2446
2981
|
: maxToolDepthOpt;
|
|
2447
2982
|
let toolDepth = 0;
|
|
2448
|
-
|
|
2449
|
-
|
|
2983
|
+
// See streamWithXmlTools: one accumulator per turn, pricing each round
|
|
2984
|
+
// under the model that served it.
|
|
2985
|
+
const turnUsage = new TurnUsageAccumulator(request.config.model, (servedModel) => this.resolvePricing(request.config.model, servedModel));
|
|
2986
|
+
// Honest turn telemetry: provider calls actually made (including refusal
|
|
2987
|
+
// re-issues inside streamOnce) and continuation rounds.
|
|
2988
|
+
let providerCalls = 0;
|
|
2989
|
+
let rounds = 0;
|
|
2450
2990
|
let lastStopReason = 'end_turn';
|
|
2451
2991
|
let lastStopSequence;
|
|
2452
2992
|
let rawRequest;
|
|
@@ -2454,8 +2994,11 @@ export class Membrane {
|
|
|
2454
2994
|
let allTextAccumulated = '';
|
|
2455
2995
|
const executedToolCalls = [];
|
|
2456
2996
|
const executedToolResults = [];
|
|
2997
|
+
// Spend on refusal attempts this turn threw away (see streamOnce).
|
|
2998
|
+
let discardedUsage;
|
|
2457
2999
|
let messages = [...request.messages];
|
|
2458
3000
|
let allContentBlocks = [];
|
|
3001
|
+
let markersInLastRequest = 0;
|
|
2459
3002
|
try {
|
|
2460
3003
|
// Tool execution loop
|
|
2461
3004
|
while (toolDepth <= maxToolDepth) {
|
|
@@ -2471,26 +3014,17 @@ export class Membrane {
|
|
|
2471
3014
|
return;
|
|
2472
3015
|
}
|
|
2473
3016
|
// Build provider request with native tools
|
|
2474
|
-
const providerRequest = this.buildNativeToolRequest(request, messages, toolDepth > 0);
|
|
3017
|
+
const providerRequest = this.buildNativeToolRequest(request, messages, toolDepth > 0, activeFormatter);
|
|
2475
3018
|
// Stream from provider
|
|
2476
3019
|
let textAccumulated = '';
|
|
2477
|
-
let blockIndex = 0;
|
|
2478
3020
|
// Where this attempt starts inside the tool-loop-spanning buffer, so
|
|
2479
3021
|
// a refusal retry can roll back exactly this attempt's contribution.
|
|
2480
3022
|
const allTextBefore = allTextAccumulated.length;
|
|
2481
|
-
// Track block-type from the provider's
|
|
3023
|
+
// Track block-type from the provider's content_block signals so
|
|
2482
3024
|
// every token chunk is tagged with the membrane block it belongs to.
|
|
2483
3025
|
// Without this, thinking_delta chunks get mislabelled as 'text' and
|
|
2484
3026
|
// downstream consumers (TUIs, WebUIs) can't render them distinctly.
|
|
2485
|
-
|
|
2486
|
-
const seenBlockIndices = new Set();
|
|
2487
|
-
const mapApiBlockType = (apiType) => {
|
|
2488
|
-
if (apiType === 'thinking')
|
|
2489
|
-
return 'thinking';
|
|
2490
|
-
if (apiType === 'tool_use')
|
|
2491
|
-
return 'tool_call';
|
|
2492
|
-
return 'text';
|
|
2493
|
-
};
|
|
3027
|
+
const tracker = new NativeBlockTracker(emitBlocks ? (event) => stream.emit({ type: 'block', event }) : undefined);
|
|
2494
3028
|
const streamResult = await this.streamOnce(providerRequest, {
|
|
2495
3029
|
onChunk: (chunk) => {
|
|
2496
3030
|
if (stream.isCancelled)
|
|
@@ -2499,9 +3033,9 @@ export class Membrane {
|
|
|
2499
3033
|
allTextAccumulated += chunk;
|
|
2500
3034
|
if (emitTokens) {
|
|
2501
3035
|
const meta = {
|
|
2502
|
-
type:
|
|
2503
|
-
visible:
|
|
2504
|
-
blockIndex,
|
|
3036
|
+
type: tracker.currentType,
|
|
3037
|
+
visible: tracker.currentType === 'text',
|
|
3038
|
+
blockIndex: tracker.blockIndex,
|
|
2505
3039
|
};
|
|
2506
3040
|
stream.emit({ type: 'tokens', content: chunk, meta });
|
|
2507
3041
|
}
|
|
@@ -2509,41 +3043,7 @@ export class Membrane {
|
|
|
2509
3043
|
onContentBlock: (index, block) => {
|
|
2510
3044
|
if (stream.isCancelled)
|
|
2511
3045
|
return;
|
|
2512
|
-
|
|
2513
|
-
const mbType = mapApiBlockType(apiType);
|
|
2514
|
-
const isStart = !seenBlockIndices.has(index);
|
|
2515
|
-
if (isStart) {
|
|
2516
|
-
seenBlockIndices.add(index);
|
|
2517
|
-
currentBlockType = mbType;
|
|
2518
|
-
blockIndex = index;
|
|
2519
|
-
if (emitBlocks) {
|
|
2520
|
-
stream.emit({
|
|
2521
|
-
type: 'block',
|
|
2522
|
-
event: { event: 'block_start', index, block: { type: mbType } },
|
|
2523
|
-
});
|
|
2524
|
-
}
|
|
2525
|
-
}
|
|
2526
|
-
else if (emitBlocks) {
|
|
2527
|
-
// Second call for the same index = content_block_stop. The
|
|
2528
|
-
// provider has filled the block with final content; surface
|
|
2529
|
-
// a block_complete with the relevant fields for consumers
|
|
2530
|
-
// that want full block payloads (e.g. context-manager).
|
|
2531
|
-
const apiBlock = block;
|
|
2532
|
-
const mb = { type: mbType };
|
|
2533
|
-
if (mbType === 'text')
|
|
2534
|
-
mb.content = apiBlock?.text;
|
|
2535
|
-
else if (mbType === 'thinking')
|
|
2536
|
-
mb.content = apiBlock?.thinking;
|
|
2537
|
-
else if (mbType === 'tool_call') {
|
|
2538
|
-
mb.toolId = apiBlock?.id;
|
|
2539
|
-
mb.toolName = apiBlock?.name;
|
|
2540
|
-
mb.input = apiBlock?.input;
|
|
2541
|
-
}
|
|
2542
|
-
stream.emit({
|
|
2543
|
-
type: 'block',
|
|
2544
|
-
event: { event: 'block_complete', index, block: mb },
|
|
2545
|
-
});
|
|
2546
|
-
}
|
|
3046
|
+
tracker.onProviderBlock(index, block);
|
|
2547
3047
|
},
|
|
2548
3048
|
}, {
|
|
2549
3049
|
signal: stream.signal,
|
|
@@ -2551,6 +3051,14 @@ export class Membrane {
|
|
|
2551
3051
|
idleTimeoutMs: options.idleTimeoutMs,
|
|
2552
3052
|
normalizedRequest: request,
|
|
2553
3053
|
onRequest: (req) => { rawRequest = req; },
|
|
3054
|
+
// Telemetry reports what this request actually SHIPPED with —
|
|
3055
|
+
// builder breakpoints, stale passthrough, fallback, float, plus
|
|
3056
|
+
// whatever the beforeRequest hook and the wire clamp did after
|
|
3057
|
+
// the build. Both native paths used to hardcode 0, and counting
|
|
3058
|
+
// at build time reported a number no request ever had.
|
|
3059
|
+
onWireCacheMarkers: (markerCount) => {
|
|
3060
|
+
markersInLastRequest = markerCount;
|
|
3061
|
+
},
|
|
2554
3062
|
refusalRetries: options.refusalRetries,
|
|
2555
3063
|
// Discard the refused attempt: roll the accumulators back to
|
|
2556
3064
|
// where this attempt began and tell the consumer to drop what it
|
|
@@ -2559,9 +3067,7 @@ export class Membrane {
|
|
|
2559
3067
|
onRetrying: (info) => {
|
|
2560
3068
|
allTextAccumulated = allTextAccumulated.slice(0, allTextBefore);
|
|
2561
3069
|
textAccumulated = '';
|
|
2562
|
-
|
|
2563
|
-
currentBlockType = 'text';
|
|
2564
|
-
seenBlockIndices.clear();
|
|
3070
|
+
tracker.reset();
|
|
2565
3071
|
stream.emit({
|
|
2566
3072
|
type: 'retrying',
|
|
2567
3073
|
attempt: info.attempt,
|
|
@@ -2571,22 +3077,21 @@ export class Membrane {
|
|
|
2571
3077
|
});
|
|
2572
3078
|
},
|
|
2573
3079
|
});
|
|
3080
|
+
// Single-callback adapters (OpenAI Responses) report each finalised
|
|
3081
|
+
// block once, after the stream: complete whatever never saw a stop.
|
|
3082
|
+
tracker.flush();
|
|
3083
|
+
rounds++;
|
|
3084
|
+
providerCalls += streamResult.providerCalls;
|
|
2574
3085
|
rawResponse = streamResult.raw;
|
|
2575
3086
|
lastStopReason = this.mapStopReason(streamResult.stopReason);
|
|
2576
3087
|
lastStopSequence = streamResult.stopSequence ?? undefined;
|
|
2577
|
-
//
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
}
|
|
2583
|
-
if (streamResult.usage.cacheReadTokens) {
|
|
2584
|
-
totalUsage.cacheReadTokens = (totalUsage.cacheReadTokens ?? 0) + streamResult.usage.cacheReadTokens;
|
|
2585
|
-
}
|
|
2586
|
-
if (pricing)
|
|
2587
|
-
totalUsage.estimatedCost = calculateCost(totalUsage, pricing);
|
|
3088
|
+
// Attempts this round re-issued past a refusal are billed calls whose
|
|
3089
|
+
// output was discarded — carry their spend to the final response.
|
|
3090
|
+
discardedUsage = this.mergeDiscardedAttempts(discardedUsage, streamResult.discardedUsage);
|
|
3091
|
+
// Accumulate usage (including cache metrics), priced at this round's model
|
|
3092
|
+
const usageSoFar = turnUsage.addRound(streamResult.model, streamResult.usage);
|
|
2588
3093
|
if (emitUsage) {
|
|
2589
|
-
stream.emit({ type: 'usage', usage:
|
|
3094
|
+
stream.emit({ type: 'usage', usage: usageSoFar });
|
|
2590
3095
|
}
|
|
2591
3096
|
// Parse content blocks from response
|
|
2592
3097
|
const responseBlocks = this.parseProviderContent(streamResult.content);
|
|
@@ -2672,6 +3177,7 @@ export class Membrane {
|
|
|
2672
3177
|
break;
|
|
2673
3178
|
}
|
|
2674
3179
|
const durationMs = Date.now() - startTime;
|
|
3180
|
+
const totalUsage = turnUsage.total;
|
|
2675
3181
|
const response = {
|
|
2676
3182
|
content: allContentBlocks,
|
|
2677
3183
|
rawAssistantText: allTextAccumulated,
|
|
@@ -2685,18 +3191,25 @@ export class Membrane {
|
|
|
2685
3191
|
triggeredSequence: lastStopSequence,
|
|
2686
3192
|
wasTruncated: lastStopReason === 'max_tokens',
|
|
2687
3193
|
},
|
|
2688
|
-
usage: {
|
|
3194
|
+
usage: {
|
|
3195
|
+
...totalUsage,
|
|
3196
|
+
...(discardedUsage
|
|
3197
|
+
? { discardedAttempts: this.pricedDiscardedAttempts(discardedUsage, request.config.model) }
|
|
3198
|
+
: {}),
|
|
3199
|
+
},
|
|
2689
3200
|
timing: {
|
|
2690
3201
|
totalDurationMs: durationMs,
|
|
2691
|
-
attempts:
|
|
3202
|
+
attempts: providerCalls,
|
|
3203
|
+
rounds,
|
|
2692
3204
|
},
|
|
2693
3205
|
model: {
|
|
2694
3206
|
requested: request.config.model,
|
|
2695
|
-
actual: request.config.model,
|
|
3207
|
+
actual: turnUsage.lastServedModel || request.config.model,
|
|
2696
3208
|
provider: this.adapter.name,
|
|
3209
|
+
perRound: turnUsage.perRound,
|
|
2697
3210
|
},
|
|
2698
3211
|
cache: {
|
|
2699
|
-
markersInRequest:
|
|
3212
|
+
markersInRequest: markersInLastRequest,
|
|
2700
3213
|
tokensCreated: totalUsage.cacheCreationTokens ?? 0,
|
|
2701
3214
|
tokensRead: totalUsage.cacheReadTokens ?? 0,
|
|
2702
3215
|
hitRatio: this.calculateCacheHitRatio(totalUsage),
|
|
@@ -2713,7 +3226,7 @@ export class Membrane {
|
|
|
2713
3226
|
if (this.isAbortError(error)) {
|
|
2714
3227
|
stream.emit({
|
|
2715
3228
|
type: 'aborted',
|
|
2716
|
-
reason:
|
|
3229
|
+
reason: this.abortReason(error, stream.signal),
|
|
2717
3230
|
rawAssistantText: allTextAccumulated,
|
|
2718
3231
|
toolCalls: executedToolCalls,
|
|
2719
3232
|
toolResults: executedToolResults,
|