@animalabs/membrane 0.5.79 → 0.5.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cache-keepalive.d.ts +115 -0
- package/dist/cache-keepalive.d.ts.map +1 -0
- package/dist/cache-keepalive.js +0 -0
- package/dist/cache-keepalive.js.map +1 -0
- package/dist/cache-keepalive.test.d.ts +2 -0
- package/dist/cache-keepalive.test.d.ts.map +1 -0
- package/dist/cache-keepalive.test.js +206 -0
- package/dist/cache-keepalive.test.js.map +1 -0
- package/dist/context/index.d.ts +1 -1
- package/dist/context/index.d.ts.map +1 -1
- package/dist/context/index.js +1 -1
- package/dist/context/index.js.map +1 -1
- package/dist/context/process.d.ts +3 -1
- package/dist/context/process.d.ts.map +1 -1
- package/dist/context/process.js +230 -49
- package/dist/context/process.js.map +1 -1
- package/dist/context/types.d.ts +30 -0
- package/dist/context/types.d.ts.map +1 -1
- package/dist/context/types.js +16 -0
- package/dist/context/types.js.map +1 -1
- package/dist/floating-cache-marker.test.d.ts +2 -0
- package/dist/floating-cache-marker.test.d.ts.map +1 -0
- package/dist/floating-cache-marker.test.js +263 -0
- package/dist/floating-cache-marker.test.js.map +1 -0
- package/dist/formatters/anthropic-xml.d.ts +2 -0
- package/dist/formatters/anthropic-xml.d.ts.map +1 -1
- package/dist/formatters/anthropic-xml.js +57 -19
- package/dist/formatters/anthropic-xml.js.map +1 -1
- package/dist/formatters/normalize-tool-pairs.d.ts +7 -0
- package/dist/formatters/normalize-tool-pairs.d.ts.map +1 -1
- package/dist/formatters/normalize-tool-pairs.js +45 -0
- package/dist/formatters/normalize-tool-pairs.js.map +1 -1
- package/dist/formatters/types.d.ts +29 -0
- package/dist/formatters/types.d.ts.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/membrane.d.ts +142 -7
- package/dist/membrane.d.ts.map +1 -1
- package/dist/membrane.js +881 -270
- package/dist/membrane.js.map +1 -1
- package/dist/providers/anthropic.d.ts +37 -0
- package/dist/providers/anthropic.d.ts.map +1 -1
- package/dist/providers/anthropic.js +163 -26
- package/dist/providers/anthropic.js.map +1 -1
- package/dist/providers/bedrock.d.ts +8 -0
- package/dist/providers/bedrock.d.ts.map +1 -1
- package/dist/providers/bedrock.js +24 -1
- package/dist/providers/bedrock.js.map +1 -1
- package/dist/providers/gemini.d.ts +11 -0
- package/dist/providers/gemini.d.ts.map +1 -1
- package/dist/providers/gemini.js +104 -82
- package/dist/providers/gemini.js.map +1 -1
- package/dist/providers/mock.d.ts +2 -0
- package/dist/providers/mock.d.ts.map +1 -1
- package/dist/providers/mock.js +2 -0
- package/dist/providers/mock.js.map +1 -1
- package/dist/providers/openai-compatible.d.ts +7 -0
- package/dist/providers/openai-compatible.d.ts.map +1 -1
- package/dist/providers/openai-compatible.js +99 -48
- package/dist/providers/openai-compatible.js.map +1 -1
- package/dist/providers/openai-completions.d.ts +5 -0
- package/dist/providers/openai-completions.d.ts.map +1 -1
- package/dist/providers/openai-completions.js +107 -44
- package/dist/providers/openai-completions.js.map +1 -1
- package/dist/providers/openai-responses-api.d.ts +16 -0
- package/dist/providers/openai-responses-api.d.ts.map +1 -1
- package/dist/providers/openai-responses-api.js +38 -10
- package/dist/providers/openai-responses-api.js.map +1 -1
- package/dist/providers/openai-responses.d.ts +2 -0
- package/dist/providers/openai-responses.d.ts.map +1 -1
- package/dist/providers/openai-responses.js +8 -1
- package/dist/providers/openai-responses.js.map +1 -1
- package/dist/providers/openai.d.ts +7 -0
- package/dist/providers/openai.d.ts.map +1 -1
- package/dist/providers/openai.js +92 -44
- package/dist/providers/openai.js.map +1 -1
- package/dist/providers/openrouter.d.ts +9 -0
- package/dist/providers/openrouter.d.ts.map +1 -1
- package/dist/providers/openrouter.js +118 -62
- package/dist/providers/openrouter.js.map +1 -1
- package/dist/providers/utils.d.ts +74 -2
- package/dist/providers/utils.d.ts.map +1 -1
- package/dist/providers/utils.js +184 -3
- package/dist/providers/utils.js.map +1 -1
- package/dist/registry/default-pricing.d.ts +22 -0
- package/dist/registry/default-pricing.d.ts.map +1 -1
- package/dist/registry/default-pricing.js +162 -28
- package/dist/registry/default-pricing.js.map +1 -1
- package/dist/types/config.d.ts +5 -0
- package/dist/types/config.d.ts.map +1 -1
- package/dist/types/config.js.map +1 -1
- package/dist/types/content.d.ts +13 -0
- package/dist/types/content.d.ts.map +1 -1
- package/dist/types/content.js.map +1 -1
- package/dist/types/errors.d.ts +15 -0
- package/dist/types/errors.d.ts.map +1 -1
- package/dist/types/errors.js +26 -0
- package/dist/types/errors.js.map +1 -1
- package/dist/types/index.d.ts +3 -3
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/index.js +1 -1
- package/dist/types/index.js.map +1 -1
- package/dist/types/provider.d.ts +50 -0
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/request.d.ts +13 -0
- package/dist/types/request.d.ts.map +1 -1
- package/dist/types/response.d.ts +85 -4
- package/dist/types/response.d.ts.map +1 -1
- package/dist/types/response.js.map +1 -1
- package/dist/types/streaming.d.ts +6 -1
- package/dist/types/streaming.d.ts.map +1 -1
- package/dist/utils/cache-marker-budget.d.ts +82 -0
- package/dist/utils/cache-marker-budget.d.ts.map +1 -0
- package/dist/utils/cache-marker-budget.js +189 -0
- package/dist/utils/cache-marker-budget.js.map +1 -0
- package/dist/utils/cost.d.ts +22 -0
- package/dist/utils/cost.d.ts.map +1 -1
- package/dist/utils/cost.js +55 -0
- package/dist/utils/cost.js.map +1 -1
- package/dist/utils/thinking-carriers.d.ts +80 -0
- package/dist/utils/thinking-carriers.d.ts.map +1 -0
- package/dist/utils/thinking-carriers.js +115 -0
- package/dist/utils/thinking-carriers.js.map +1 -0
- package/dist/utils/tool-parser.d.ts +24 -0
- package/dist/utils/tool-parser.d.ts.map +1 -1
- package/dist/utils/tool-parser.js +367 -161
- package/dist/utils/tool-parser.js.map +1 -1
- package/dist/utils/usage.d.ts +71 -0
- package/dist/utils/usage.d.ts.map +1 -0
- package/dist/utils/usage.js +161 -0
- package/dist/utils/usage.js.map +1 -0
- package/dist/yielding-stream.d.ts +8 -0
- package/dist/yielding-stream.d.ts.map +1 -1
- package/dist/yielding-stream.js +49 -3
- package/dist/yielding-stream.js.map +1 -1
- package/package.json +3 -2
- package/src/cache-keepalive.test.ts +244 -0
- package/src/cache-keepalive.ts +385 -0
- package/src/context/index.ts +1 -0
- package/src/context/process.ts +305 -59
- package/src/context/types.ts +38 -0
- package/src/floating-cache-marker.test.ts +285 -0
- package/src/formatters/anthropic-xml.ts +62 -20
- package/src/formatters/normalize-tool-pairs.ts +49 -0
- package/src/formatters/types.ts +32 -0
- package/src/index.ts +13 -0
- package/src/membrane.ts +1031 -279
- package/src/providers/anthropic.ts +205 -20
- package/src/providers/bedrock.ts +25 -1
- package/src/providers/gemini.ts +125 -87
- package/src/providers/mock.ts +3 -0
- package/src/providers/openai-compatible.ts +102 -45
- package/src/providers/openai-completions.ts +112 -46
- package/src/providers/openai-responses-api.ts +45 -15
- package/src/providers/openai-responses.ts +8 -1
- package/src/providers/openai.ts +95 -42
- package/src/providers/openrouter.ts +125 -63
- package/src/providers/utils.ts +227 -6
- package/src/registry/default-pricing.ts +165 -28
- package/src/types/config.ts +6 -0
- package/src/types/content.ts +14 -0
- package/src/types/errors.ts +28 -0
- package/src/types/index.ts +6 -0
- package/src/types/provider.ts +55 -0
- package/src/types/request.ts +14 -0
- package/src/types/response.ts +109 -8
- package/src/types/streaming.ts +8 -2
- package/src/utils/cache-marker-budget.ts +208 -0
- package/src/utils/cost.ts +64 -0
- package/src/utils/thinking-carriers.ts +125 -0
- package/src/utils/tool-parser.ts +473 -174
- package/src/utils/usage.ts +184 -0
- package/src/yielding-stream.ts +54 -3
package/dist/membrane.js
CHANGED
|
@@ -4,17 +4,104 @@
|
|
|
4
4
|
* A selective boundary that transforms what passes through.
|
|
5
5
|
*/
|
|
6
6
|
import { lastCacheableBlockIndex } from './formatters/native.js';
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
7
|
+
import { sameThinkingText, findSpanningProviderRun, thinkingCarrierKey, stripThinkingForPrefill, } from './utils/thinking-carriers.js';
|
|
8
|
+
import { countWireCacheMarkers, clampCacheMarkers, ownSystemBlocks, MAX_CACHE_BREAKPOINTS, } from './utils/cache-marker-budget.js';
|
|
9
|
+
import { DEFAULT_RETRY_CONFIG, MembraneError, classifyError, isOverloadedError, isTimeoutAbortError, unsupportedError, } from './types/index.js';
|
|
10
|
+
import { parseToolCalls, formatToolResults, parseAccumulatedIntoBlocks, endsWithPartialToolBlock, hasImageInToolResults, formatToolResultsForSplitTurn, } from './utils/tool-parser.js';
|
|
9
11
|
import { AnthropicXmlFormatter } from './formatters/anthropic-xml.js';
|
|
10
|
-
import { normalizeToolPairs, mergeConsecutiveRoles } from './formatters/normalize-tool-pairs.js';
|
|
12
|
+
import { normalizeToolPairs, mergeConsecutiveRoles, PREFIX_REWRITING_NORMALIZE_EVENT_KINDS, } from './formatters/normalize-tool-pairs.js';
|
|
11
13
|
import { YieldingStreamImpl } from './yielding-stream.js';
|
|
12
|
-
import { calculateCost } from './utils/cost.js';
|
|
14
|
+
import { calculateCost, warnUnpricedModel } from './utils/cost.js';
|
|
15
|
+
import { TurnUsageAccumulator, calculateCacheHitRatio, normalizeUsageToCacheExcluded, warnUnconvertibleProviderItem, } from './utils/usage.js';
|
|
13
16
|
import { isAcceptedImageMediaType, strippedImagePlaceholder, shedImagesToFitByteBudget, assertWithinByteBudget, } from './utils/image-media.js';
|
|
14
17
|
import { getDefaultPricing } from './registry/default-pricing.js';
|
|
15
18
|
// ============================================================================
|
|
16
19
|
// Membrane Class
|
|
17
20
|
// ============================================================================
|
|
21
|
+
/**
|
|
22
|
+
* Block-lifecycle tracking shared by the two native-tools streaming paths
|
|
23
|
+
* (`streamWithNativeTools` and `runNativeToolsYielding`).
|
|
24
|
+
*
|
|
25
|
+
* Providers signal blocks through `onContentBlock(index, block)`, but not all
|
|
26
|
+
* of them the same way: the Anthropic and Bedrock adapters fire it twice per
|
|
27
|
+
* index (content_block_start with an empty block, content_block_stop with the
|
|
28
|
+
* finalised one), while the OpenAI Responses adapter fires it ONCE per block,
|
|
29
|
+
* already finalised, after the stream has ended. Treating "second sighting"
|
|
30
|
+
* as the only completion signal therefore left single-callback adapters with
|
|
31
|
+
* `block_start` events that never completed (#63 review). The tracker keeps
|
|
32
|
+
* the paired semantics and adds `flush()`, which the caller runs once the
|
|
33
|
+
* provider stream has returned: every started block that never saw a second
|
|
34
|
+
* callback is completed from the last block payload seen for it.
|
|
35
|
+
*/
|
|
36
|
+
class NativeBlockTracker {
|
|
37
|
+
emit;
|
|
38
|
+
currentType = 'text';
|
|
39
|
+
blockIndex = 0;
|
|
40
|
+
started = new Map();
|
|
41
|
+
completed = new Set();
|
|
42
|
+
lastSeen = new Map();
|
|
43
|
+
constructor(emit) {
|
|
44
|
+
this.emit = emit;
|
|
45
|
+
}
|
|
46
|
+
static mapApiBlockType(apiType) {
|
|
47
|
+
if (apiType === 'thinking' || apiType === 'redacted_thinking' || apiType === 'reasoning')
|
|
48
|
+
return 'thinking';
|
|
49
|
+
if (apiType === 'tool_use' || apiType === 'function_call' || apiType === 'tool_call')
|
|
50
|
+
return 'tool_call';
|
|
51
|
+
return 'text';
|
|
52
|
+
}
|
|
53
|
+
/** Provider block callback: first sighting of an index starts it, a second completes it. */
|
|
54
|
+
onProviderBlock(index, block) {
|
|
55
|
+
this.lastSeen.set(index, block);
|
|
56
|
+
if (!this.started.has(index)) {
|
|
57
|
+
const mbType = NativeBlockTracker.mapApiBlockType(block?.type);
|
|
58
|
+
this.started.set(index, mbType);
|
|
59
|
+
this.currentType = mbType;
|
|
60
|
+
this.blockIndex = index;
|
|
61
|
+
this.emit?.({ event: 'block_start', index, block: { type: mbType } });
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
this.complete(index, block);
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Complete every started block that never received its second callback.
|
|
68
|
+
* Run after the provider stream has returned; idempotent, and a no-op for
|
|
69
|
+
* paired-callback adapters.
|
|
70
|
+
*/
|
|
71
|
+
flush() {
|
|
72
|
+
for (const index of this.started.keys()) {
|
|
73
|
+
if (!this.completed.has(index))
|
|
74
|
+
this.complete(index, this.lastSeen.get(index));
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
/** Discard tracking state (refusal retry rolled the attempt back). */
|
|
78
|
+
reset() {
|
|
79
|
+
this.currentType = 'text';
|
|
80
|
+
this.blockIndex = 0;
|
|
81
|
+
this.started.clear();
|
|
82
|
+
this.completed.clear();
|
|
83
|
+
this.lastSeen.clear();
|
|
84
|
+
}
|
|
85
|
+
complete(index, block) {
|
|
86
|
+
if (this.completed.has(index))
|
|
87
|
+
return;
|
|
88
|
+
this.completed.add(index);
|
|
89
|
+
const mbType = this.started.get(index)
|
|
90
|
+
?? NativeBlockTracker.mapApiBlockType(block?.type);
|
|
91
|
+
const apiBlock = block;
|
|
92
|
+
const mb = { type: mbType };
|
|
93
|
+
if (mbType === 'text')
|
|
94
|
+
mb.content = apiBlock?.text;
|
|
95
|
+
else if (mbType === 'thinking')
|
|
96
|
+
mb.content = apiBlock?.thinking;
|
|
97
|
+
else if (mbType === 'tool_call') {
|
|
98
|
+
mb.toolId = apiBlock?.id;
|
|
99
|
+
mb.toolName = apiBlock?.name;
|
|
100
|
+
mb.input = apiBlock?.input;
|
|
101
|
+
}
|
|
102
|
+
this.emit?.({ event: 'block_complete', index, block: mb });
|
|
103
|
+
}
|
|
104
|
+
}
|
|
18
105
|
export class Membrane {
|
|
19
106
|
adapter;
|
|
20
107
|
registry;
|
|
@@ -47,17 +134,27 @@ export class Membrane {
|
|
|
47
134
|
// refusal is a successful HTTP call with an unwanted verdict, and letting
|
|
48
135
|
// it consume error retries would couple two unrelated budgets.
|
|
49
136
|
let refusalRetriesUsed = 0;
|
|
137
|
+
// Spend on attempts we threw away. A refused attempt is a completed,
|
|
138
|
+
// billed HTTP call; reporting only the surviving attempt's usage
|
|
139
|
+
// under-reports the turn by one full call per retry.
|
|
140
|
+
let discardedUsage;
|
|
141
|
+
// One selection for the whole call: mode resolution and the build must
|
|
142
|
+
// name the same formatter instance (see resolveActiveFormatter).
|
|
143
|
+
const activeFormatter = this.resolveActiveFormatter(options.formatter);
|
|
50
144
|
while (true) {
|
|
51
145
|
attempts++;
|
|
52
146
|
try {
|
|
53
|
-
const { providerRequest, prefillResult } = this.transformRequest(request,
|
|
147
|
+
const { providerRequest, prefillResult } = this.transformRequest(request, activeFormatter);
|
|
54
148
|
// Route through the single canonical hook helper so any future
|
|
55
149
|
// change to hook semantics (logging, retry interaction, error
|
|
56
150
|
// handling) applies to both complete() and the streaming paths.
|
|
57
151
|
// Cast back to the local provider-request shape: the hook returns
|
|
58
152
|
// `unknown` deliberately, and we acknowledge the cast at the boundary.
|
|
59
153
|
const finalRequest = (await this.applyBeforeRequestHook(request, providerRequest));
|
|
60
|
-
|
|
154
|
+
// Last exit before the adapter: the only place that sees EVERY
|
|
155
|
+
// contribution (builder, formatter, passthrough, float, hook).
|
|
156
|
+
clampCacheMarkers(finalRequest, 'complete');
|
|
157
|
+
const rawProviderResponse = await this.adapter.complete(finalRequest, {
|
|
61
158
|
signal: options.signal,
|
|
62
159
|
timeoutMs: options.timeoutMs,
|
|
63
160
|
onRequest: (req) => {
|
|
@@ -65,6 +162,11 @@ export class Membrane {
|
|
|
65
162
|
options.onRequest?.(req);
|
|
66
163
|
},
|
|
67
164
|
});
|
|
165
|
+
// Restate usage in the one convention before any ratio or price sees it.
|
|
166
|
+
const providerResponse = {
|
|
167
|
+
...rawProviderResponse,
|
|
168
|
+
usage: normalizeUsageToCacheExcluded(rawProviderResponse.usage, this.adapter.name, this.adapter.usageCacheConvention),
|
|
169
|
+
};
|
|
68
170
|
// Call onResponse callback with raw response from API
|
|
69
171
|
options.onResponse?.(providerResponse.raw);
|
|
70
172
|
const response = this.transformResponse(providerResponse, request, prefillResult, startTime, attempts, rawRequest);
|
|
@@ -76,8 +178,16 @@ export class Membrane {
|
|
|
76
178
|
if (response.stopReason === 'refusal' &&
|
|
77
179
|
refusalRetriesUsed < Math.max(0, options.refusalRetries ?? 0)) {
|
|
78
180
|
refusalRetriesUsed++;
|
|
181
|
+
discardedUsage = this.mergeDiscardedAttempts(discardedUsage, this.discardedAttemptFrom(response.usage));
|
|
79
182
|
continue;
|
|
80
183
|
}
|
|
184
|
+
// Report what the discarded attempts cost. Set BEFORE afterResponse
|
|
185
|
+
// so a hook that logs spend sees the whole turn, not just the
|
|
186
|
+
// attempt that stands.
|
|
187
|
+
if (discardedUsage) {
|
|
188
|
+
response.details.usage.discardedAttempts =
|
|
189
|
+
this.pricedDiscardedAttempts(discardedUsage, request.config.model);
|
|
190
|
+
}
|
|
81
191
|
// Call afterResponse hook
|
|
82
192
|
if (this.config.hooks?.afterResponse) {
|
|
83
193
|
return await this.config.hooks.afterResponse(response, providerResponse.raw);
|
|
@@ -111,9 +221,17 @@ export class Membrane {
|
|
|
111
221
|
throw new MembraneError(errorInfo);
|
|
112
222
|
}
|
|
113
223
|
}
|
|
114
|
-
// Wait before retry (abort-aware)
|
|
224
|
+
// Wait before retry (abort-aware). An abort landing inside the
|
|
225
|
+
// sleep must fail like every other failure of this method — a
|
|
226
|
+
// MembraneError — rather than escaping the loop as a raw
|
|
227
|
+
// DOMException whose shape no caller of complete() expects.
|
|
115
228
|
const delay = this.calculateRetryDelay(attempts, isOverloaded);
|
|
116
|
-
|
|
229
|
+
try {
|
|
230
|
+
await this.sleep(delay, options.signal);
|
|
231
|
+
}
|
|
232
|
+
catch (sleepError) {
|
|
233
|
+
throw this.attachRawRequest(sleepError, rawRequest);
|
|
234
|
+
}
|
|
117
235
|
continue;
|
|
118
236
|
}
|
|
119
237
|
throw new MembraneError(errorInfo);
|
|
@@ -142,6 +260,18 @@ export class Membrane {
|
|
|
142
260
|
// If streaming is explicitly disabled on the request, fall back to complete()
|
|
143
261
|
// and synthesize the streaming callbacks from the full response
|
|
144
262
|
if (request.streaming === false) {
|
|
263
|
+
// complete() has no tool loop, and neither branch of this fallback can
|
|
264
|
+
// build one: honouring onToolCalls here would mean re-implementing the
|
|
265
|
+
// whole XML/native continuation machinery. Silently dropping it turned
|
|
266
|
+
// a working agent into one that narrates tool calls it never makes —
|
|
267
|
+
// the raw <function_calls> XML lands in the returned text and the turn
|
|
268
|
+
// ends. Refuse where the option is passed, before spending a call.
|
|
269
|
+
if (options.onToolCalls) {
|
|
270
|
+
throw unsupportedError('stream() cannot execute tools with streaming: false — the non-streaming ' +
|
|
271
|
+
'fallback routes to complete(), which has no tool loop, so onToolCalls ' +
|
|
272
|
+
'would never run. Leave streaming enabled (or drive the loop yourself ' +
|
|
273
|
+
'with complete() per round).');
|
|
274
|
+
}
|
|
145
275
|
const response = await this.complete(request, options);
|
|
146
276
|
// Synthesize onChunk callbacks so callers that depend on them still work
|
|
147
277
|
if (options.onChunk && 'content' in response) {
|
|
@@ -158,8 +288,9 @@ export class Membrane {
|
|
|
158
288
|
}
|
|
159
289
|
return response;
|
|
160
290
|
}
|
|
161
|
-
// Determine tool mode
|
|
162
|
-
const
|
|
291
|
+
// Determine tool mode against the formatter that will build the request
|
|
292
|
+
const activeFormatter = this.resolveActiveFormatter(options.formatter);
|
|
293
|
+
const toolMode = this.resolveToolMode(request, activeFormatter);
|
|
163
294
|
const useNative = toolMode === 'native' && !!request.tools && request.tools.length > 0;
|
|
164
295
|
// Overloaded (529) pre-emission retry. The streaming paths have no retry
|
|
165
296
|
// loop of their own, so a capacity error used to kill the turn outright —
|
|
@@ -186,14 +317,16 @@ export class Membrane {
|
|
|
186
317
|
};
|
|
187
318
|
try {
|
|
188
319
|
const result = useNative
|
|
189
|
-
? await this.streamWithNativeTools(request, tracked)
|
|
190
|
-
: await this.streamWithXmlTools(request, tracked);
|
|
191
|
-
// The inner paths
|
|
192
|
-
//
|
|
193
|
-
//
|
|
194
|
-
//
|
|
320
|
+
? await this.streamWithNativeTools(request, tracked, activeFormatter)
|
|
321
|
+
: await this.streamWithXmlTools(request, tracked, activeFormatter);
|
|
322
|
+
// The inner paths count their own provider calls but cannot see this
|
|
323
|
+
// wrapper's discarded attempts. Each failed attempt here died before
|
|
324
|
+
// emitting anything (that is the precondition for retrying), so it
|
|
325
|
+
// cost at least the one call it failed on — ADD those to the inner
|
|
326
|
+
// count rather than overwriting it, or a turn that retried twice and
|
|
327
|
+
// then ran three tool rounds would report 2 calls instead of 5.
|
|
195
328
|
if (attempts > 1 && 'details' in result) {
|
|
196
|
-
result.details.timing.attempts
|
|
329
|
+
result.details.timing.attempts += attempts - 1;
|
|
197
330
|
result.details.timing.retryDelaysMs = retryDelaysMs;
|
|
198
331
|
}
|
|
199
332
|
return result;
|
|
@@ -218,7 +351,21 @@ export class Membrane {
|
|
|
218
351
|
}
|
|
219
352
|
const delay = this.calculateRetryDelay(attempts, true);
|
|
220
353
|
retryDelaysMs.push(delay);
|
|
221
|
-
|
|
354
|
+
// An abort during the backoff window is still a cancellation of
|
|
355
|
+
// this stream, and stream() documents cancellation as an
|
|
356
|
+
// AbortedResponse. Letting the sleep's rejection escape made that
|
|
357
|
+
// contract depend on which millisecond the abort landed in.
|
|
358
|
+
// Nothing has been emitted on this path (that is the precondition
|
|
359
|
+
// for retrying at all), so there is no partial content to report.
|
|
360
|
+
try {
|
|
361
|
+
await this.sleep(delay, options.signal);
|
|
362
|
+
}
|
|
363
|
+
catch (sleepError) {
|
|
364
|
+
if (this.isAbortError(sleepError)) {
|
|
365
|
+
return this.buildAbortedResponse('', { inputTokens: 0, outputTokens: 0 }, [], [], this.abortReason(sleepError, options.signal));
|
|
366
|
+
}
|
|
367
|
+
throw sleepError;
|
|
368
|
+
}
|
|
222
369
|
continue;
|
|
223
370
|
}
|
|
224
371
|
throw error;
|
|
@@ -226,17 +373,62 @@ export class Membrane {
|
|
|
226
373
|
}
|
|
227
374
|
}
|
|
228
375
|
/**
|
|
229
|
-
*
|
|
376
|
+
* Select the ACTIVE formatter for a request: the one instance that resolves
|
|
377
|
+
* its tool mode, builds its provider request, and parses its stream.
|
|
378
|
+
*
|
|
379
|
+
* A per-request override (`CompleteOptions.formatter` /
|
|
380
|
+
* `StreamOptions.formatter`) wins over the instance formatter, with ONE
|
|
381
|
+
* transport exception: the Responses adapter's input is a provider-native
|
|
382
|
+
* item array, and a generic override (for example Context Manager's
|
|
383
|
+
* NativeFormatter) produces Anthropic-style `{ role, content: [{ type:
|
|
384
|
+
* 'text' }] }` envelopes the Responses API rejects before inference — so a
|
|
385
|
+
* configured Responses formatter stays authoritative there.
|
|
386
|
+
*
|
|
387
|
+
* The exception is why this selection is a method rather than a `??` at each
|
|
388
|
+
* call site: while it lived inside transformRequest alone, the BUILD honored
|
|
389
|
+
* it and every other formatter reader resolved against a different instance,
|
|
390
|
+
* which is the split resolveToolMode exists to prevent, one layer down.
|
|
391
|
+
* Every entry point selects once, here, and threads the result.
|
|
230
392
|
*/
|
|
231
|
-
|
|
393
|
+
resolveActiveFormatter(requestFormatter) {
|
|
394
|
+
if (this.adapter.name === 'openai-responses-api' && this.formatter.name === 'openai-responses') {
|
|
395
|
+
return this.formatter;
|
|
396
|
+
}
|
|
397
|
+
return requestFormatter ?? this.formatter;
|
|
398
|
+
}
|
|
399
|
+
/**
|
|
400
|
+
* Determine the effective tool mode.
|
|
401
|
+
*
|
|
402
|
+
* THE single source of truth for the mode: both complete() (via
|
|
403
|
+
* transformRequest → BuildOptions.toolMode) and the streaming paths (via
|
|
404
|
+
* their native-vs-XML path choice) resolve here, so a given request resolves
|
|
405
|
+
* to the same mode whichever entry point it arrives through.
|
|
406
|
+
*
|
|
407
|
+
* Precedence, strongest first:
|
|
408
|
+
* 1. an explicit non-'auto' `request.toolMode`
|
|
409
|
+
* 2. the mode the BUILDING formatter was explicitly constructed with
|
|
410
|
+
* (`AnthropicXmlFormatter({ toolMode: 'native' })`) — a caller's stated
|
|
411
|
+
* choice, not a derivation
|
|
412
|
+
* 3. formatter/provider derivation
|
|
413
|
+
*
|
|
414
|
+
* `formatter` is the formatter that will actually build the request — the
|
|
415
|
+
* instance `resolveActiveFormatter` selected for this call — because
|
|
416
|
+
* resolving against one formatter while building with another is exactly the
|
|
417
|
+
* split this method exists to prevent.
|
|
418
|
+
*/
|
|
419
|
+
resolveToolMode(request, formatter = this.formatter) {
|
|
232
420
|
// Explicit mode takes precedence
|
|
233
421
|
if (request.toolMode && request.toolMode !== 'auto') {
|
|
234
422
|
return request.toolMode;
|
|
235
423
|
}
|
|
424
|
+
// A formatter constructed with an explicit mode states its caller's choice
|
|
425
|
+
if (formatter.configuredToolMode) {
|
|
426
|
+
return formatter.configuredToolMode;
|
|
427
|
+
}
|
|
236
428
|
// Auto mode: choose based on formatter
|
|
237
429
|
// NativeFormatter → native tools via API
|
|
238
430
|
// AnthropicXmlFormatter (default) → XML tools in prefill
|
|
239
|
-
if (
|
|
431
|
+
if (formatter.name === 'native' || formatter.name === 'openai-responses') {
|
|
240
432
|
return 'native';
|
|
241
433
|
}
|
|
242
434
|
// Also handle known native-tool providers regardless of formatter
|
|
@@ -253,16 +445,25 @@ export class Membrane {
|
|
|
253
445
|
* - False-positive stop sequence detection (e.g., "\nUser:" inside tool results)
|
|
254
446
|
* - Structured block events for UI
|
|
255
447
|
*/
|
|
256
|
-
async streamWithXmlTools(request, options) {
|
|
448
|
+
async streamWithXmlTools(request, options, activeFormatter = this.resolveActiveFormatter(options.formatter)) {
|
|
257
449
|
const startTime = Date.now();
|
|
258
|
-
const { onChunk, onContentBlockUpdate, onToolCalls, onPreToolContent, onUsage, onBlock, onRequest, onResponse, maxToolDepth = 10, signal,
|
|
259
|
-
//
|
|
260
|
-
|
|
450
|
+
const { onChunk, onContentBlockUpdate, onToolCalls, onPreToolContent, onUsage, onBlock, onRequest, onResponse, maxToolDepth = 10, signal, timeoutMs, idleTimeoutMs, } = options;
|
|
451
|
+
// The formatter stream() selected: the same instance that resolved the
|
|
452
|
+
// mode and will build the request, so the parser can never be reading a
|
|
453
|
+
// different format than the one on the wire.
|
|
454
|
+
const formatter = activeFormatter;
|
|
261
455
|
// Initialize parser from formatter for format-specific tracking
|
|
262
456
|
const parser = formatter.createStreamParser();
|
|
263
457
|
let toolDepth = 0;
|
|
264
|
-
|
|
265
|
-
|
|
458
|
+
// Each round is priced under the model that served THAT round and the
|
|
459
|
+
// costs are summed: a routed turn can change models mid-turn, and pricing
|
|
460
|
+
// the whole accumulated usage at the latest rate re-bills every earlier
|
|
461
|
+
// round at a price it was never charged.
|
|
462
|
+
const turnUsage = new TurnUsageAccumulator(request.config.model, (servedModel) => this.resolvePricing(request.config.model, servedModel));
|
|
463
|
+
// Honest turn telemetry: provider calls actually made (including refusal
|
|
464
|
+
// re-issues inside streamOnce) and continuation rounds.
|
|
465
|
+
let providerCalls = 0;
|
|
466
|
+
let rounds = 0;
|
|
266
467
|
const contentBlocks = [];
|
|
267
468
|
let lastStopReason = 'end_turn';
|
|
268
469
|
let lastStopSequence;
|
|
@@ -337,13 +538,13 @@ export class Membrane {
|
|
|
337
538
|
resumptionRounds++;
|
|
338
539
|
if (resumptionRounds === RESUMPTION_WARN_ROUNDS) {
|
|
339
540
|
warnLog.warn(`[membrane] automatic resumption at round ${resumptionRounds} ` +
|
|
340
|
-
`(${
|
|
541
|
+
`(${turnUsage.total.inputTokens} input tokens so far this turn) — ` +
|
|
341
542
|
`a spin shows up here before it shows up on the bill`);
|
|
342
543
|
}
|
|
343
544
|
if (resumptionRounds > maxResumptionRounds) {
|
|
344
545
|
warnLog.warn(`[membrane] automatic resumption cap (${maxResumptionRounds}) reached — ` +
|
|
345
546
|
`ending turn with stopReason 'round_limit'. ` +
|
|
346
|
-
`${
|
|
547
|
+
`${turnUsage.total.inputTokens} input tokens spent this turn.`);
|
|
347
548
|
return false;
|
|
348
549
|
}
|
|
349
550
|
return true;
|
|
@@ -407,6 +608,8 @@ export class Membrane {
|
|
|
407
608
|
: undefined,
|
|
408
609
|
}, {
|
|
409
610
|
signal,
|
|
611
|
+
timeoutMs,
|
|
612
|
+
idleTimeoutMs,
|
|
410
613
|
normalizedRequest: request,
|
|
411
614
|
// The tag-based parser tracks thinking via <thinking> tags — ask the
|
|
412
615
|
// provider to wrap native thinking deltas so they don't stream as
|
|
@@ -417,6 +620,8 @@ export class Membrane {
|
|
|
417
620
|
onRequest?.(req);
|
|
418
621
|
},
|
|
419
622
|
});
|
|
623
|
+
rounds++;
|
|
624
|
+
providerCalls += streamResult.providerCalls;
|
|
420
625
|
// If we detected stop sequence manually, fix up the parser and result
|
|
421
626
|
if (detectedStopSequence && truncatedAccumulated !== null) {
|
|
422
627
|
parser.reset();
|
|
@@ -446,18 +651,11 @@ export class Membrane {
|
|
|
446
651
|
onResponse?.(rawResponse);
|
|
447
652
|
lastStopReason = this.mapStopReason(streamResult.stopReason);
|
|
448
653
|
lastStopSequence = streamResult.stopSequence ?? undefined;
|
|
449
|
-
// Accumulate usage (including cache metrics)
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
}
|
|
455
|
-
if (streamResult.usage.cacheReadTokens) {
|
|
456
|
-
totalUsage.cacheReadTokens = (totalUsage.cacheReadTokens ?? 0) + streamResult.usage.cacheReadTokens;
|
|
457
|
-
}
|
|
458
|
-
if (pricing)
|
|
459
|
-
totalUsage.estimatedCost = calculateCost(totalUsage, pricing);
|
|
460
|
-
onUsage?.(totalUsage);
|
|
654
|
+
// Accumulate usage (including cache metrics), priced at this round's
|
|
655
|
+
// model. NOT inlined into the optional call — `onUsage?.(addRound())`
|
|
656
|
+
// skips evaluating its argument entirely when no callback is set.
|
|
657
|
+
const usageSoFar = turnUsage.addRound(streamResult.model, streamResult.usage);
|
|
658
|
+
onUsage?.(usageSoFar);
|
|
461
659
|
// Flush the parser to complete any in-progress streaming block
|
|
462
660
|
const flushResult = parser.flush();
|
|
463
661
|
for (const emission of flushResult.emissions) {
|
|
@@ -485,7 +683,7 @@ export class Membrane {
|
|
|
485
683
|
warnLog.warn(`[membrane] ${consecutiveStalledResumptions} consecutive automatic resumptions ` +
|
|
486
684
|
`made no progress (${streamedThisRound} chars this round, stop ` +
|
|
487
685
|
`${JSON.stringify(lastStopSequence ?? null)} repeated) — ending turn with ` +
|
|
488
|
-
`stopReason 'no_progress'. ${
|
|
686
|
+
`stopReason 'no_progress'. ${turnUsage.total.inputTokens} input tokens spent this turn.`);
|
|
489
687
|
lastStopReason = 'no_progress';
|
|
490
688
|
break;
|
|
491
689
|
}
|
|
@@ -718,14 +916,14 @@ export class Membrane {
|
|
|
718
916
|
// The full accumulated text is still available in raw.response
|
|
719
917
|
const fullAccumulated = parser.getAccumulated();
|
|
720
918
|
const newContent = fullAccumulated.slice(initialPrefillLength);
|
|
721
|
-
const response = this.buildFinalResponse(newContent, contentBlocks, lastStopReason,
|
|
722
|
-
rawRequest, rawResponse, executedToolCalls, executedToolResults, initialBlockType, lastStopSequence);
|
|
919
|
+
const response = this.buildFinalResponse(newContent, contentBlocks, lastStopReason, turnUsage, request, prefillResult, startTime, providerCalls, rawRequest, rawResponse, executedToolCalls, executedToolResults, initialBlockType, lastStopSequence);
|
|
723
920
|
// Append non-text content blocks (e.g., generated_image) that the XML parser can't handle
|
|
724
921
|
if (extraContentBlocks.length > 0) {
|
|
725
922
|
response.content.push(...extraContentBlocks);
|
|
726
923
|
}
|
|
727
924
|
// Merge provider thinking signatures into parser-derived thinking blocks
|
|
728
925
|
this.mergeProviderThinkingBlocks(response.content, providerThinkingBlocks);
|
|
926
|
+
response.details.timing.rounds = rounds;
|
|
729
927
|
return response;
|
|
730
928
|
}
|
|
731
929
|
catch (error) {
|
|
@@ -734,7 +932,7 @@ export class Membrane {
|
|
|
734
932
|
// Only use NEW content (after initial prefill) for partial content
|
|
735
933
|
const fullAccumulated = parser.getAccumulated();
|
|
736
934
|
const newContent = fullAccumulated.slice(initialPrefillLength);
|
|
737
|
-
return this.buildAbortedResponse(newContent,
|
|
935
|
+
return this.buildAbortedResponse(newContent, turnUsage.total, executedToolCalls, executedToolResults, this.abortReason(error, signal), initialBlockType);
|
|
738
936
|
}
|
|
739
937
|
// Re-throw with rawRequest attached for logging
|
|
740
938
|
throw this.attachRawRequest(error, rawRequest);
|
|
@@ -743,12 +941,17 @@ export class Membrane {
|
|
|
743
941
|
/**
|
|
744
942
|
* Stream with native API tool execution
|
|
745
943
|
*/
|
|
746
|
-
async streamWithNativeTools(request, options) {
|
|
944
|
+
async streamWithNativeTools(request, options, activeFormatter = this.resolveActiveFormatter(options.formatter)) {
|
|
747
945
|
const startTime = Date.now();
|
|
748
|
-
const { onChunk, onContentBlockUpdate, onToolCalls, onPreToolContent, onUsage, onRequest, onResponse, maxToolDepth = 10, signal, } = options;
|
|
946
|
+
const { onChunk, onContentBlockUpdate, onBlock, onToolCalls, onPreToolContent, onUsage, onRequest, onResponse, maxToolDepth = 10, signal, timeoutMs, idleTimeoutMs, } = options;
|
|
749
947
|
let toolDepth = 0;
|
|
750
|
-
|
|
751
|
-
|
|
948
|
+
// See streamWithXmlTools: one accumulator per turn, pricing each round
|
|
949
|
+
// under the model that served it.
|
|
950
|
+
const turnUsage = new TurnUsageAccumulator(request.config.model, (servedModel) => this.resolvePricing(request.config.model, servedModel));
|
|
951
|
+
// Honest turn telemetry: provider calls actually made (including refusal
|
|
952
|
+
// re-issues inside streamOnce) and continuation rounds.
|
|
953
|
+
let providerCalls = 0;
|
|
954
|
+
let rounds = 0;
|
|
752
955
|
let lastStopReason = 'end_turn';
|
|
753
956
|
let lastStopSequence;
|
|
754
957
|
let rawRequest;
|
|
@@ -761,55 +964,69 @@ export class Membrane {
|
|
|
761
964
|
// Build messages array that we'll update with tool results
|
|
762
965
|
let messages = [...request.messages];
|
|
763
966
|
let allContentBlocks = [];
|
|
967
|
+
let markersInLastRequest = 0;
|
|
764
968
|
try {
|
|
765
969
|
// Tool execution loop
|
|
766
970
|
while (toolDepth <= maxToolDepth) {
|
|
767
971
|
// Build provider request with native tools
|
|
768
|
-
const providerRequest = this.buildNativeToolRequest(request, messages);
|
|
972
|
+
const providerRequest = this.buildNativeToolRequest(request, messages, toolDepth > 0, activeFormatter);
|
|
769
973
|
// Stream from provider
|
|
770
974
|
let textAccumulated = '';
|
|
771
|
-
|
|
975
|
+
// Tag every token chunk with the membrane block it belongs to and
|
|
976
|
+
// surface the block lifecycle through onBlock — the same shape
|
|
977
|
+
// runNativeToolsYielding uses (#19). Before this, meta.type was
|
|
978
|
+
// hardcoded to 'text' on every chunk and onBlock was never invoked
|
|
979
|
+
// from this path (#20).
|
|
980
|
+
const tracker = new NativeBlockTracker(onBlock ? (event) => onBlock(event) : undefined);
|
|
772
981
|
const streamResult = await this.streamOnce(providerRequest, {
|
|
773
982
|
onChunk: (chunk) => {
|
|
774
983
|
textAccumulated += chunk;
|
|
775
984
|
allTextAccumulated += chunk;
|
|
776
|
-
// For native mode, emit text chunks with basic metadata
|
|
777
|
-
// TODO: Use native API content_block events for richer metadata
|
|
778
985
|
const meta = {
|
|
779
|
-
type:
|
|
780
|
-
visible:
|
|
781
|
-
blockIndex,
|
|
986
|
+
type: tracker.currentType,
|
|
987
|
+
visible: tracker.currentType === 'text',
|
|
988
|
+
blockIndex: tracker.blockIndex,
|
|
782
989
|
};
|
|
783
990
|
onChunk?.(chunk, meta);
|
|
784
991
|
},
|
|
785
|
-
onContentBlock:
|
|
786
|
-
|
|
787
|
-
|
|
992
|
+
onContentBlock: (index, block) => {
|
|
993
|
+
tracker.onProviderBlock(index, block);
|
|
994
|
+
// Deprecated pass-through, kept for callers still on it.
|
|
995
|
+
onContentBlockUpdate?.(index, block);
|
|
996
|
+
},
|
|
788
997
|
}, {
|
|
789
998
|
signal,
|
|
999
|
+
timeoutMs,
|
|
1000
|
+
idleTimeoutMs,
|
|
790
1001
|
normalizedRequest: request,
|
|
791
1002
|
onRequest: (req) => {
|
|
792
1003
|
rawRequest = req;
|
|
793
1004
|
onRequest?.(req);
|
|
794
1005
|
},
|
|
1006
|
+
// Telemetry reports what this request actually SHIPPED with —
|
|
1007
|
+
// builder breakpoints, stale passthrough, fallback, float, plus
|
|
1008
|
+
// whatever the beforeRequest hook and the wire clamp did after
|
|
1009
|
+
// the build. Both native paths used to hardcode 0, and counting
|
|
1010
|
+
// at build time reported a number no request ever had.
|
|
1011
|
+
onWireCacheMarkers: (markerCount) => {
|
|
1012
|
+
markersInLastRequest = markerCount;
|
|
1013
|
+
},
|
|
795
1014
|
});
|
|
1015
|
+
// Single-callback adapters (OpenAI Responses) report each finalised
|
|
1016
|
+
// block once, after the stream: complete whatever never saw a stop.
|
|
1017
|
+
tracker.flush();
|
|
1018
|
+
rounds++;
|
|
1019
|
+
providerCalls += streamResult.providerCalls;
|
|
796
1020
|
rawResponse = streamResult.raw;
|
|
797
1021
|
// Call onResponse callback with raw response from API
|
|
798
1022
|
onResponse?.(rawResponse);
|
|
799
1023
|
lastStopReason = this.mapStopReason(streamResult.stopReason);
|
|
800
1024
|
lastStopSequence = streamResult.stopSequence ?? undefined;
|
|
801
|
-
// Accumulate usage (including cache metrics)
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
}
|
|
807
|
-
if (streamResult.usage.cacheReadTokens) {
|
|
808
|
-
totalUsage.cacheReadTokens = (totalUsage.cacheReadTokens ?? 0) + streamResult.usage.cacheReadTokens;
|
|
809
|
-
}
|
|
810
|
-
if (pricing)
|
|
811
|
-
totalUsage.estimatedCost = calculateCost(totalUsage, pricing);
|
|
812
|
-
onUsage?.(totalUsage);
|
|
1025
|
+
// Accumulate usage (including cache metrics), priced at this round's
|
|
1026
|
+
// model. NOT inlined into the optional call — `onUsage?.(addRound())`
|
|
1027
|
+
// skips evaluating its argument entirely when no callback is set.
|
|
1028
|
+
const usageSoFar = turnUsage.addRound(streamResult.model, streamResult.usage);
|
|
1029
|
+
onUsage?.(usageSoFar);
|
|
813
1030
|
// Parse content blocks from response
|
|
814
1031
|
const responseBlocks = this.parseProviderContent(streamResult.content);
|
|
815
1032
|
allContentBlocks.push(...responseBlocks);
|
|
@@ -879,6 +1096,7 @@ export class Membrane {
|
|
|
879
1096
|
break;
|
|
880
1097
|
}
|
|
881
1098
|
const durationMs = Date.now() - startTime;
|
|
1099
|
+
const totalUsage = turnUsage.total;
|
|
882
1100
|
return {
|
|
883
1101
|
content: allContentBlocks,
|
|
884
1102
|
rawAssistantText: allTextAccumulated,
|
|
@@ -895,15 +1113,17 @@ export class Membrane {
|
|
|
895
1113
|
usage: { ...totalUsage },
|
|
896
1114
|
timing: {
|
|
897
1115
|
totalDurationMs: durationMs,
|
|
898
|
-
attempts:
|
|
1116
|
+
attempts: providerCalls,
|
|
1117
|
+
rounds,
|
|
899
1118
|
},
|
|
900
1119
|
model: {
|
|
901
1120
|
requested: request.config.model,
|
|
902
|
-
actual: request.config.model,
|
|
1121
|
+
actual: turnUsage.lastServedModel || request.config.model,
|
|
903
1122
|
provider: this.adapter.name,
|
|
1123
|
+
perRound: turnUsage.perRound,
|
|
904
1124
|
},
|
|
905
1125
|
cache: {
|
|
906
|
-
markersInRequest:
|
|
1126
|
+
markersInRequest: markersInLastRequest,
|
|
907
1127
|
tokensCreated: totalUsage.cacheCreationTokens ?? 0,
|
|
908
1128
|
tokensRead: totalUsage.cacheReadTokens ?? 0,
|
|
909
1129
|
hitRatio: this.calculateCacheHitRatio(totalUsage),
|
|
@@ -918,22 +1138,60 @@ export class Membrane {
|
|
|
918
1138
|
catch (error) {
|
|
919
1139
|
// Check if this is an abort error
|
|
920
1140
|
if (this.isAbortError(error)) {
|
|
921
|
-
return this.buildAbortedResponse(allTextAccumulated,
|
|
1141
|
+
return this.buildAbortedResponse(allTextAccumulated, turnUsage.total, executedToolCalls, executedToolResults, this.abortReason(error, signal));
|
|
922
1142
|
}
|
|
923
1143
|
// Re-throw with rawRequest attached for logging
|
|
924
1144
|
throw this.attachRawRequest(error, rawRequest);
|
|
925
1145
|
}
|
|
926
1146
|
}
|
|
927
1147
|
/**
|
|
928
|
-
*
|
|
1148
|
+
* Rate-limit state for the float's budget warning. See the
|
|
1149
|
+
* floating-cache-marker block in buildNativeToolRequest.
|
|
1150
|
+
*
|
|
1151
|
+
* A once-per-instance latch made the ONLY observable of an over-budget wire
|
|
1152
|
+
* go quiet for the life of the process: a long-lived Membrane warns for the
|
|
1153
|
+
* first agent that trips it and never again, so the condition looks like it
|
|
1154
|
+
* healed. Warn on the first occurrence, then at most once per interval,
|
|
1155
|
+
* carrying the count of what was suppressed in between.
|
|
929
1156
|
*/
|
|
930
|
-
|
|
1157
|
+
floatBudgetWarnState = { lastWarnedAtMs: 0, suppressedSinceWarn: 0 };
|
|
1158
|
+
static FLOAT_BUDGET_WARN_INTERVAL_MS = 60_000;
|
|
1159
|
+
warnFloatBudgetExhausted(wireMarkers) {
|
|
1160
|
+
const now = Date.now();
|
|
1161
|
+
const state = this.floatBudgetWarnState;
|
|
1162
|
+
const elapsed = now - state.lastWarnedAtMs;
|
|
1163
|
+
if (state.lastWarnedAtMs !== 0 && elapsed < Membrane.FLOAT_BUDGET_WARN_INTERVAL_MS) {
|
|
1164
|
+
state.suppressedSinceWarn++;
|
|
1165
|
+
return;
|
|
1166
|
+
}
|
|
1167
|
+
const suppressed = state.suppressedSinceWarn;
|
|
1168
|
+
state.lastWarnedAtMs = now;
|
|
1169
|
+
state.suppressedSinceWarn = 0;
|
|
1170
|
+
console.warn(`[membrane] floating cache marker withheld: upstream markers already ` +
|
|
1171
|
+
`occupy all ${MAX_CACHE_BREAKPOINTS} cache_control slots (${wireMarkers} on the wire). ` +
|
|
1172
|
+
`Tool-round suffixes will not cache incrementally.` +
|
|
1173
|
+
(suppressed > 0 ? ` (${suppressed} further occurrences suppressed since the last warning.)` : ''));
|
|
1174
|
+
}
|
|
1175
|
+
/**
|
|
1176
|
+
* Build a provider request with native tool support.
|
|
1177
|
+
*
|
|
1178
|
+
* `toolLoopRebuild` is true when this build is a tool-loop continuation
|
|
1179
|
+
* (toolDepth > 0) rather than the turn's first request — the only case
|
|
1180
|
+
* where the floating cache marker applies.
|
|
1181
|
+
*
|
|
1182
|
+
* `activeFormatter` is the formatter the caller selected for the request
|
|
1183
|
+
* (see resolveActiveFormatter). Reading `this.formatter` here instead made
|
|
1184
|
+
* the native loop build through the instance formatter while the mode had
|
|
1185
|
+
* been resolved against a per-request override — the two disagreeing about
|
|
1186
|
+
* which formatter is active.
|
|
1187
|
+
*/
|
|
1188
|
+
buildNativeToolRequest(request, messages, toolLoopRebuild = false, activeFormatter = this.formatter) {
|
|
931
1189
|
// Provider-native formatters own their complete input-item shape. The
|
|
932
1190
|
// legacy implementation below is intentionally Anthropic-specific; using
|
|
933
1191
|
// it for Responses would normalize away item IDs, encrypted reasoning,
|
|
934
1192
|
// assistant phases, and compaction items.
|
|
935
|
-
if (
|
|
936
|
-
return this.transformRequest({ ...request, messages },
|
|
1193
|
+
if (activeFormatter.name === 'openai-responses') {
|
|
1194
|
+
return this.transformRequest({ ...request, messages }, activeFormatter).providerRequest;
|
|
937
1195
|
}
|
|
938
1196
|
// Convert messages to provider format
|
|
939
1197
|
const providerMessages = [];
|
|
@@ -944,10 +1202,13 @@ export class Membrane {
|
|
|
944
1202
|
// Anthropic allows at most 4 cache_control breakpoints per request. The
|
|
945
1203
|
// message breakpoints are the valuable ones (they cache the longest prefixes,
|
|
946
1204
|
// and every one already includes tools+system at the front of the request).
|
|
947
|
-
// So tools/system get a breakpoint only as a FALLBACK — when no
|
|
948
|
-
//
|
|
949
|
-
// total past 4, which the API hard-rejects (the agent goes
|
|
950
|
-
|
|
1205
|
+
// So tools/system get a breakpoint only as a FALLBACK — when no marker
|
|
1206
|
+
// exists anywhere on the wire — otherwise they're redundant and would push
|
|
1207
|
+
// the total past 4, which the API hard-rejects (the agent goes
|
|
1208
|
+
// unresponsive). The fallback gate reads a RECOUNT of the built artifacts
|
|
1209
|
+
// (see below), never a running tally: a running tally cannot see a
|
|
1210
|
+
// caller-marked system block, and double-counts a message breakpoint that
|
|
1211
|
+
// lands on a block already carrying stale cache_control.
|
|
951
1212
|
for (const msg of messages) {
|
|
952
1213
|
const isAssistant = msg.participant === assistantName;
|
|
953
1214
|
const role = isAssistant ? 'assistant' : 'user';
|
|
@@ -968,14 +1229,13 @@ export class Membrane {
|
|
|
968
1229
|
}
|
|
969
1230
|
const textBlock = { type: 'text', text };
|
|
970
1231
|
if (block.cache_control) {
|
|
971
|
-
textBlock.cache_control = block.cache_control;
|
|
972
1232
|
// A block-level passthrough occupies one of the 4 breakpoint slots
|
|
973
|
-
// exactly like a marked message
|
|
974
|
-
//
|
|
975
|
-
//
|
|
976
|
-
//
|
|
977
|
-
//
|
|
978
|
-
|
|
1233
|
+
// exactly like a marked message; the recount below sees it.
|
|
1234
|
+
// (Imported/seeded conversations carry stale request-time
|
|
1235
|
+
// cache_control on stored blocks — first seen wedging Sill
|
|
1236
|
+
// 2026-07-25: 3 cm markers + 2 stale Arc-export blocks = 5 → hard
|
|
1237
|
+
// 400 on every inference.)
|
|
1238
|
+
textBlock.cache_control = block.cache_control;
|
|
979
1239
|
}
|
|
980
1240
|
content.push(textBlock);
|
|
981
1241
|
}
|
|
@@ -1052,7 +1312,6 @@ export class Membrane {
|
|
|
1052
1312
|
const bpIdx = lastCacheableBlockIndex(content);
|
|
1053
1313
|
if (bpIdx >= 0) {
|
|
1054
1314
|
content[bpIdx].cache_control = cacheControl;
|
|
1055
|
-
messageBreakpoints++;
|
|
1056
1315
|
}
|
|
1057
1316
|
}
|
|
1058
1317
|
providerMessages.push({ role, content });
|
|
@@ -1076,8 +1335,36 @@ export class Membrane {
|
|
|
1076
1335
|
// tool_results to `messages`. Any unmatched tool_use that reaches
|
|
1077
1336
|
// this splice is upstream stranding (the bug class this fix exists
|
|
1078
1337
|
// to catch) — `[pending]` is exactly the right synthesis.
|
|
1079
|
-
|
|
1338
|
+
// A synthesized [pending] tool_result's bytes are rewritten when the
|
|
1339
|
+
// real result lands — the floating-marker block below must not cache
|
|
1340
|
+
// past one. `synthetic_pending_result` (not the downstream
|
|
1341
|
+
// cache_suppressed_for_synthetic, which only fires when a marker was
|
|
1342
|
+
// actually stripped) is the root condition.
|
|
1343
|
+
// Every repair that REWRITES prefix bytes stands the float down, not just
|
|
1344
|
+
// the synthetic [pending] result: a textified orphan tool_result is
|
|
1345
|
+
// rewritten the same way when its real pairing arrives, so caching at or
|
|
1346
|
+
// past one poisons the prefix identically. The kinds live in one exported
|
|
1347
|
+
// set so a normalizer that grows a new prefix-rewriting repair cannot
|
|
1348
|
+
// silently escape this guard.
|
|
1349
|
+
let prefixRewritten = false;
|
|
1350
|
+
const normalized = normalizeToolPairs(providerMessages, {
|
|
1351
|
+
onEvent: (e) => {
|
|
1352
|
+
if (PREFIX_REWRITING_NORMALIZE_EVENT_KINDS.has(e.kind))
|
|
1353
|
+
prefixRewritten = true;
|
|
1354
|
+
},
|
|
1355
|
+
});
|
|
1080
1356
|
const mergedMessages = mergeConsecutiveRoles(normalized.messages);
|
|
1357
|
+
// ONE recount of the constructed wire artifacts, taken BEFORE the
|
|
1358
|
+
// tools/system fallback decision so the fallback and the float share a
|
|
1359
|
+
// single truth. Counted post-normalize, so phase-5.5 cache suppression is
|
|
1360
|
+
// already reflected. `request.system` is the caller's own system content:
|
|
1361
|
+
// it explicitly accepts pre-marked blocks, and those are real wire markers
|
|
1362
|
+
// that no running tally ever saw (three of them plus both fallbacks = 5 on
|
|
1363
|
+
// the wire = a 400 on every inference of that config).
|
|
1364
|
+
const upstreamWireMarkers = countWireCacheMarkers({
|
|
1365
|
+
messages: mergedMessages,
|
|
1366
|
+
system: request.system,
|
|
1367
|
+
});
|
|
1081
1368
|
// Convert tools to provider format.
|
|
1082
1369
|
// Native tool names must match ^[a-zA-Z0-9_-]{1,128}$ — sanitize colons
|
|
1083
1370
|
// from the module:tool namespace convention. Reversed in parseProviderContent.
|
|
@@ -1089,7 +1376,7 @@ export class Membrane {
|
|
|
1089
1376
|
};
|
|
1090
1377
|
// Cache the tool list (last tool) only as a fallback — a marked message
|
|
1091
1378
|
// breakpoint already caches the tools as part of its prefix.
|
|
1092
|
-
if (cacheControl &&
|
|
1379
|
+
if (cacheControl && upstreamWireMarkers === 0 && request.tools && idx === request.tools.length - 1) {
|
|
1093
1380
|
t.cache_control = cacheControl;
|
|
1094
1381
|
}
|
|
1095
1382
|
return t;
|
|
@@ -1097,14 +1384,75 @@ export class Membrane {
|
|
|
1097
1384
|
// Wrap system prompt with cache_control only as a fallback (no message
|
|
1098
1385
|
// breakpoint marked); otherwise a message breakpoint already caches
|
|
1099
1386
|
// tools+system as part of its prefix.
|
|
1100
|
-
let system = request.system;
|
|
1101
|
-
if (cacheControl &&
|
|
1387
|
+
let system = ownSystemBlocks(request.system);
|
|
1388
|
+
if (cacheControl && upstreamWireMarkers === 0 && typeof system === 'string' && system.length > 0) {
|
|
1102
1389
|
system = [{ type: 'text', text: system, cache_control: cacheControl }];
|
|
1103
1390
|
}
|
|
1104
|
-
else if (cacheControl &&
|
|
1391
|
+
else if (cacheControl && upstreamWireMarkers === 0 && Array.isArray(system) && system.length > 0) {
|
|
1105
1392
|
const blocks = system;
|
|
1106
1393
|
system = blocks.map((block, idx) => idx === blocks.length - 1 ? { ...block, cache_control: cacheControl } : block);
|
|
1107
1394
|
}
|
|
1395
|
+
// ------------------------------------------------------------------
|
|
1396
|
+
// Floating cache marker: incremental prompt caching inside the native
|
|
1397
|
+
// tool loop. Message breakpoints are placed by the context strategy at
|
|
1398
|
+
// compile time — once per turn — but this builder re-runs on every
|
|
1399
|
+
// tool round with that round's messages appended, so the deepest
|
|
1400
|
+
// upstream marker stays glued to the turn-start snapshot and each
|
|
1401
|
+
// rebuild re-pays the entire appended suffix at full input price
|
|
1402
|
+
// (qa-ops incident, 2026-08-20: two subagents re-sent a suffix growing
|
|
1403
|
+
// to ~118k tokens ~30 times each — ~5.3M uncached tokens in 18 min —
|
|
1404
|
+
// with their one marker sitting on message 2 of 61).
|
|
1405
|
+
//
|
|
1406
|
+
// The tool loop only ever appends, so a marker riding the newest
|
|
1407
|
+
// message yields the intended incremental pattern: each round writes
|
|
1408
|
+
// its delta and cache-reads everything before it.
|
|
1409
|
+
//
|
|
1410
|
+
// Authority contract: the float spends only the RESIDUAL breakpoint
|
|
1411
|
+
// budget (Anthropic allows 4 cache_control including tools/system).
|
|
1412
|
+
// Upstream markers are never displaced or stripped — if they fill all
|
|
1413
|
+
// 4 slots the float is withheld (with a warning) and behavior is
|
|
1414
|
+
// exactly pre-float. With 2+ slots free, the previous round's
|
|
1415
|
+
// endpoint is marked too: a wide parallel-tool round can append more
|
|
1416
|
+
// blocks than the provider's ~20-block backward search covers, which
|
|
1417
|
+
// would orphan the previous round's cache entry behind an unmarked
|
|
1418
|
+
// boundary.
|
|
1419
|
+
//
|
|
1420
|
+
// Skipped when the normalizer synthesized a [pending] tool_result:
|
|
1421
|
+
// those bytes are rewritten when the real result lands, and caching
|
|
1422
|
+
// past them poisons the prefix — the same rationale as the
|
|
1423
|
+
// normalizer's phase 5.5 cache suppression.
|
|
1424
|
+
// ------------------------------------------------------------------
|
|
1425
|
+
const floatingEnabled = request.floatingCacheMarker ?? this.config.defaultFloatingCacheMarker ?? true;
|
|
1426
|
+
if (toolLoopRebuild && floatingEnabled && cacheControl && !prefixRewritten) {
|
|
1427
|
+
// Same recount as the fallback gate, re-taken POST-fallback so the
|
|
1428
|
+
// fallback's own spend is inside the residuum.
|
|
1429
|
+
const wireMarkers = countWireCacheMarkers({ messages: mergedMessages, system, tools });
|
|
1430
|
+
let residuum = MAX_CACHE_BREAKPOINTS - wireMarkers;
|
|
1431
|
+
if (residuum <= 0) {
|
|
1432
|
+
this.warnFloatBudgetExhausted(wireMarkers);
|
|
1433
|
+
}
|
|
1434
|
+
else {
|
|
1435
|
+
// Newest message first; then the previous round's endpoint (two
|
|
1436
|
+
// wire messages back: [..., prevResults, assistant, results]).
|
|
1437
|
+
const targets = [mergedMessages.length - 1, mergedMessages.length - 3];
|
|
1438
|
+
for (const mi of targets) {
|
|
1439
|
+
if (residuum <= 0 || mi < 0)
|
|
1440
|
+
continue;
|
|
1441
|
+
const content = mergedMessages[mi]?.content;
|
|
1442
|
+
if (!Array.isArray(content) || content.length === 0)
|
|
1443
|
+
continue;
|
|
1444
|
+
const bpIdx = lastCacheableBlockIndex(content);
|
|
1445
|
+
if (bpIdx < 0)
|
|
1446
|
+
continue;
|
|
1447
|
+
// Already a breakpoint here (e.g. the strategy's own end marker
|
|
1448
|
+
// on the turn's first rebuild) — nothing to add.
|
|
1449
|
+
if (content[bpIdx].cache_control)
|
|
1450
|
+
continue;
|
|
1451
|
+
content[bpIdx].cache_control = cacheControl;
|
|
1452
|
+
residuum--;
|
|
1453
|
+
}
|
|
1454
|
+
}
|
|
1455
|
+
}
|
|
1108
1456
|
// Build thinking config for native extended thinking (budget clamped to max_tokens)
|
|
1109
1457
|
// Fable/Mythos models: thinking is always on and unconfigurable; sampling params are removed.
|
|
1110
1458
|
// Sending thinking config or temperature returns a 400 — omit both entirely.
|
|
@@ -1152,6 +1500,9 @@ export class Membrane {
|
|
|
1152
1500
|
id: item.id,
|
|
1153
1501
|
name: unsanitizeToolName(item.name),
|
|
1154
1502
|
input: item.input,
|
|
1503
|
+
// Arguments that never parsed: carry the marker through so a
|
|
1504
|
+
// consumer can refuse the block instead of trusting `input`.
|
|
1505
|
+
...(item.unparseableInput !== undefined ? { unparseableInput: item.unparseableInput } : {}),
|
|
1155
1506
|
...(item.rawItem ? { rawItem: item.rawItem } : {}),
|
|
1156
1507
|
});
|
|
1157
1508
|
}
|
|
@@ -1174,14 +1525,24 @@ export class Membrane {
|
|
|
1174
1525
|
mimeType: item.mimeType,
|
|
1175
1526
|
});
|
|
1176
1527
|
}
|
|
1177
|
-
else if (item.rawItem) {
|
|
1528
|
+
else if (item.rawItem || item.type) {
|
|
1178
1529
|
// Opaque Responses items such as encrypted compaction or custom
|
|
1179
1530
|
// tool records have no normalized ContentBlock equivalent. Retain a
|
|
1180
1531
|
// zero-width carrier so Chronicle and the Responses formatter can
|
|
1181
1532
|
// replay the raw item without surfacing synthetic prompt text.
|
|
1182
1533
|
// Anthropic-bound conversion paths filter these out (empty text
|
|
1183
1534
|
// blocks are a 400 there); the Responses formatter replays rawItem.
|
|
1184
|
-
|
|
1535
|
+
//
|
|
1536
|
+
// An item with a `type` this switch does not know (server_tool_use,
|
|
1537
|
+
// web_search_tool_result, search_result, mcp_tool_use, or whatever a
|
|
1538
|
+
// provider adds next) used to fall out of this chain and vanish. It
|
|
1539
|
+
// gets the same carrier treatment, holding the item itself, plus a
|
|
1540
|
+
// one-time warning so the gap surfaces instead of being inferred
|
|
1541
|
+
// later from missing content.
|
|
1542
|
+
const carriedRawItem = item.rawItem ?? item;
|
|
1543
|
+
if (!item.rawItem)
|
|
1544
|
+
warnUnconvertibleProviderItem(item.type);
|
|
1545
|
+
blocks.push({ type: 'text', text: '', rawItem: carriedRawItem });
|
|
1185
1546
|
}
|
|
1186
1547
|
}
|
|
1187
1548
|
return blocks;
|
|
@@ -1215,10 +1576,32 @@ export class Membrane {
|
|
|
1215
1576
|
}
|
|
1216
1577
|
/**
|
|
1217
1578
|
* Merge provider thinking signatures into parser-derived thinking blocks
|
|
1218
|
-
*
|
|
1219
|
-
*
|
|
1220
|
-
*
|
|
1221
|
-
*
|
|
1579
|
+
* and prepend any leftover provider blocks — signature-only thinking
|
|
1580
|
+
* (display:'omitted') never appears in the text stream, so the parser
|
|
1581
|
+
* produces no block for it. redacted_thinking blocks are always prepended
|
|
1582
|
+
* verbatim.
|
|
1583
|
+
*
|
|
1584
|
+
* Pairing is by CONTENT IDENTITY, never by index. The two lists are
|
|
1585
|
+
* differently shaped whenever the provider emits a block the parser cannot
|
|
1586
|
+
* see (signature-only), the parser emits a block the provider never
|
|
1587
|
+
* produced (the XML path's literal `Claude: <thinking>` prefill turns
|
|
1588
|
+
* VISIBLE text into a thinking block), or one provider block spans several
|
|
1589
|
+
* (auto-continuation: capture runs per round while the parser sees the
|
|
1590
|
+
* CONCATENATED accumulation). Index-zipping crosses the lists in all three
|
|
1591
|
+
* shapes and stamps a signature onto content that never produced it —
|
|
1592
|
+
* which round-trips into the consumer's stored history and fails Anthropic
|
|
1593
|
+
* signature validation on the next turn.
|
|
1594
|
+
*
|
|
1595
|
+
* The three rules, in order:
|
|
1596
|
+
* 1. identity — a provider block pairs with the parsed block whose
|
|
1597
|
+
* thinking text is the same; empty-thinking (signature-only) blocks
|
|
1598
|
+
* are never text-match candidates and are prepend-only.
|
|
1599
|
+
* 2. span — a parsed block that reconstructs as the concatenation of a
|
|
1600
|
+
* RUN of consecutive unpaired provider blocks is REPLACED in place by
|
|
1601
|
+
* those originals, so the spanning block never wears a fragment's
|
|
1602
|
+
* signature and no reasoning is sent twice.
|
|
1603
|
+
* 3. leftover — everything still unpaired is prepended, de-duplicated
|
|
1604
|
+
* against what `content` already carries (and against itself).
|
|
1222
1605
|
*
|
|
1223
1606
|
* Mutates `content` in place. Shared by the XML stream paths
|
|
1224
1607
|
* (streamWithXmlTools and runXmlToolsYielding).
|
|
@@ -1226,20 +1609,70 @@ export class Membrane {
|
|
|
1226
1609
|
mergeProviderThinkingBlocks(content, providerThinkingBlocks) {
|
|
1227
1610
|
if (providerThinkingBlocks.length === 0)
|
|
1228
1611
|
return;
|
|
1229
|
-
const parsedThinking = content.filter((b) => b.type === 'thinking');
|
|
1230
1612
|
const providerThinking = providerThinkingBlocks.filter((b) => b.type === 'thinking');
|
|
1231
1613
|
const redacted = providerThinkingBlocks.filter((b) => b.type === 'redacted_thinking');
|
|
1232
|
-
const
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
if (
|
|
1236
|
-
|
|
1237
|
-
|
|
1614
|
+
const pairedProviderBlocks = new Set();
|
|
1615
|
+
const claimedParsedIndices = new Set();
|
|
1616
|
+
const parsedThinkingIndices = () => content.reduce((acc, block, index) => {
|
|
1617
|
+
if (block.type === 'thinking')
|
|
1618
|
+
acc.push(index);
|
|
1619
|
+
return acc;
|
|
1620
|
+
}, []);
|
|
1621
|
+
for (let p = 0; p < providerThinking.length; p++) {
|
|
1622
|
+
const providerText = providerThinking[p].thinking ?? '';
|
|
1623
|
+
if (providerText === '')
|
|
1624
|
+
continue;
|
|
1625
|
+
const match = parsedThinkingIndices().find((index) => !claimedParsedIndices.has(index) &&
|
|
1626
|
+
sameThinkingText(content[index].thinking ?? '', providerText));
|
|
1627
|
+
if (match === undefined)
|
|
1628
|
+
continue;
|
|
1629
|
+
const signature = providerThinking[p].signature;
|
|
1630
|
+
if (signature)
|
|
1631
|
+
content[match].signature = signature;
|
|
1632
|
+
claimedParsedIndices.add(match);
|
|
1633
|
+
pairedProviderBlocks.add(p);
|
|
1634
|
+
}
|
|
1635
|
+
for (const parsedIndex of parsedThinkingIndices().reverse()) {
|
|
1636
|
+
if (claimedParsedIndices.has(parsedIndex))
|
|
1637
|
+
continue;
|
|
1638
|
+
const parsedText = content[parsedIndex].thinking ?? '';
|
|
1639
|
+
if (parsedText === '')
|
|
1640
|
+
continue;
|
|
1641
|
+
const run = findSpanningProviderRun(providerThinking, pairedProviderBlocks, parsedText);
|
|
1642
|
+
if (!run)
|
|
1643
|
+
continue;
|
|
1644
|
+
content.splice(parsedIndex, 1, ...run.map((p) => {
|
|
1645
|
+
pairedProviderBlocks.add(p);
|
|
1646
|
+
const block = providerThinking[p];
|
|
1647
|
+
return {
|
|
1648
|
+
type: 'thinking',
|
|
1649
|
+
thinking: block.thinking ?? '',
|
|
1650
|
+
...(block.signature ? { signature: block.signature } : {}),
|
|
1651
|
+
};
|
|
1652
|
+
}));
|
|
1653
|
+
claimedParsedIndices.add(parsedIndex);
|
|
1654
|
+
}
|
|
1655
|
+
const seen = new Set(content.map((block) => thinkingCarrierKey(block)));
|
|
1656
|
+
const leftover = [];
|
|
1657
|
+
for (let p = 0; p < providerThinking.length; p++) {
|
|
1658
|
+
if (pairedProviderBlocks.has(p))
|
|
1659
|
+
continue;
|
|
1660
|
+
const block = providerThinking[p];
|
|
1661
|
+
const key = thinkingCarrierKey(block);
|
|
1662
|
+
if (seen.has(key))
|
|
1663
|
+
continue;
|
|
1664
|
+
seen.add(key);
|
|
1665
|
+
leftover.push(block);
|
|
1238
1666
|
}
|
|
1239
|
-
const
|
|
1240
|
-
|
|
1241
|
-
|
|
1667
|
+
for (const block of redacted) {
|
|
1668
|
+
const key = thinkingCarrierKey(block);
|
|
1669
|
+
if (seen.has(key))
|
|
1670
|
+
continue;
|
|
1671
|
+
seen.add(key);
|
|
1672
|
+
leftover.push(block);
|
|
1242
1673
|
}
|
|
1674
|
+
if (leftover.length > 0)
|
|
1675
|
+
content.unshift(...leftover);
|
|
1243
1676
|
}
|
|
1244
1677
|
// ==========================================================================
|
|
1245
1678
|
// Internal Methods
|
|
@@ -1315,19 +1748,15 @@ export class Membrane {
|
|
|
1315
1748
|
return { type: 'enabled', budget_tokens: budget, ...(display ? { display } : {}) };
|
|
1316
1749
|
}
|
|
1317
1750
|
/**
|
|
1318
|
-
* Transform a normalized request into provider format using the formatter
|
|
1751
|
+
* Transform a normalized request into provider format using the formatter.
|
|
1752
|
+
*
|
|
1753
|
+
* `activeFormatter` is the instance the caller already selected via
|
|
1754
|
+
* resolveActiveFormatter — including that selection's Responses-transport
|
|
1755
|
+
* authority rule, which used to live inline here. It is a parameter and not
|
|
1756
|
+
* a re-derivation so that the formatter which BUILDS is the same one that
|
|
1757
|
+
* resolved the tool mode and drives the loop.
|
|
1319
1758
|
*/
|
|
1320
|
-
transformRequest(request, formatter) {
|
|
1321
|
-
// The Responses adapter's input is a provider-native item array. A generic
|
|
1322
|
-
// per-request formatter (for example Context Manager's NativeFormatter)
|
|
1323
|
-
// produces Anthropic-style `{ role, content: [{ type: 'text' }] }`
|
|
1324
|
-
// envelopes, which the Responses API rejects before inference. Keep the
|
|
1325
|
-
// configured Responses formatter authoritative at this transport boundary;
|
|
1326
|
-
// per-request formatter overrides remain available for adapters whose wire
|
|
1327
|
-
// format supports them.
|
|
1328
|
-
const activeFormatter = this.adapter.name === 'openai-responses-api' && this.formatter.name === 'openai-responses'
|
|
1329
|
-
? this.formatter
|
|
1330
|
-
: formatter ?? this.formatter;
|
|
1759
|
+
transformRequest(request, activeFormatter = this.formatter) {
|
|
1331
1760
|
// Extract user-provided stop sequences
|
|
1332
1761
|
const additionalStopSequences = Array.isArray(request.stopSequences)
|
|
1333
1762
|
? request.stopSequences
|
|
@@ -1341,6 +1770,10 @@ export class Membrane {
|
|
|
1341
1770
|
participantMode: 'multiuser',
|
|
1342
1771
|
assistantParticipant: request.assistantParticipant ?? this.config.assistantParticipant ?? 'Claude',
|
|
1343
1772
|
tools: request.tools,
|
|
1773
|
+
// One resolution for every entry point: complete() used to build from the
|
|
1774
|
+
// formatter's constructor-time mode alone, so request.toolMode was a
|
|
1775
|
+
// second, disconnected source of truth on this path.
|
|
1776
|
+
toolMode: this.resolveToolMode(request, activeFormatter),
|
|
1344
1777
|
thinking: request.config.thinking,
|
|
1345
1778
|
systemPrompt: request.system,
|
|
1346
1779
|
promptCaching: request.promptCaching ?? this.config.defaultPromptCaching ?? true, // Default true for backward compat
|
|
@@ -1364,7 +1797,9 @@ export class Membrane {
|
|
|
1364
1797
|
const providerRequest = {
|
|
1365
1798
|
...this.getBaseProviderParams(request.config),
|
|
1366
1799
|
messages: buildResult.messages,
|
|
1367
|
-
|
|
1800
|
+
// Owned, not aliased: the wire clamp strips markers in place, and a
|
|
1801
|
+
// formatter may pass the caller's own system array straight through.
|
|
1802
|
+
system: ownSystemBlocks(buildResult.systemContent),
|
|
1368
1803
|
stopSequences: buildResult.stopSequences,
|
|
1369
1804
|
tools: buildResult.nativeTools,
|
|
1370
1805
|
extra: {
|
|
@@ -1377,8 +1812,8 @@ export class Membrane {
|
|
|
1377
1812
|
// literal `<thinking>` text prefix instead of the API feature — drop the
|
|
1378
1813
|
// API param when the built request actually ends in an assistant prefill.
|
|
1379
1814
|
// Chat-style builds (no prefill) keep it.
|
|
1380
|
-
if (buildResult.assistantPrefill
|
|
1381
|
-
|
|
1815
|
+
if (buildResult.assistantPrefill) {
|
|
1816
|
+
stripThinkingForPrefill(providerRequest);
|
|
1382
1817
|
}
|
|
1383
1818
|
return { providerRequest, prefillResult: buildResult };
|
|
1384
1819
|
}
|
|
@@ -1388,17 +1823,42 @@ export class Membrane {
|
|
|
1388
1823
|
// compatibility won't catch the excess field (checked only on object
|
|
1389
1824
|
// literals, not on variables). Leaving it in would silently leak the
|
|
1390
1825
|
// normalized form into every adapter's options.
|
|
1391
|
-
const { normalizedRequest, refusalRetries, onRetrying, ...adapterOptions } = options;
|
|
1826
|
+
const { normalizedRequest, refusalRetries, onRetrying, onWireCacheMarkers, ...adapterOptions } = options;
|
|
1392
1827
|
const finalRequest = (await this.applyBeforeRequestHook(normalizedRequest, request));
|
|
1828
|
+
// Last exit before the adapter: the only place that sees EVERY
|
|
1829
|
+
// contribution (builder, formatter, passthrough, float, hook). Every
|
|
1830
|
+
// streaming path — stream(), streamYielding(), both tool loops — funnels
|
|
1831
|
+
// through here, so this is the one clamp they all get, and its tally is
|
|
1832
|
+
// therefore the only count that describes the wire.
|
|
1833
|
+
const clampOutcome = clampCacheMarkers(finalRequest, 'streamOnce');
|
|
1834
|
+
onWireCacheMarkers?.(clampOutcome.total);
|
|
1393
1835
|
// Retries are only safe when the caller can discard the abandoned
|
|
1394
1836
|
// attempt, so they require BOTH a budget and an onRetrying hook.
|
|
1395
1837
|
const maxAttempts = onRetrying ? Math.max(0, refusalRetries ?? 0) : 0;
|
|
1396
1838
|
let retried = 0;
|
|
1839
|
+
// Every re-issued attempt was a completed, billed provider call. The
|
|
1840
|
+
// caller's usage accumulator only ever sees the surviving result, so the
|
|
1841
|
+
// abandoned spend rides back out on the result itself.
|
|
1842
|
+
let discardedUsage;
|
|
1843
|
+
let providerCalls = 0;
|
|
1397
1844
|
while (true) {
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1845
|
+
providerCalls++;
|
|
1846
|
+
const rawResult = await this.adapter.stream(finalRequest, callbacks, adapterOptions);
|
|
1847
|
+
// Restate usage in the one convention before any accumulator, ratio or
|
|
1848
|
+
// price sees it — this is the only door streamed usage enters through.
|
|
1849
|
+
const result = {
|
|
1850
|
+
...rawResult,
|
|
1851
|
+
usage: normalizeUsageToCacheExcluded(rawResult.usage, this.adapter.name, this.adapter.usageCacheConvention),
|
|
1852
|
+
};
|
|
1853
|
+
if (result.stopReason !== 'refusal' || retried >= maxAttempts) {
|
|
1854
|
+
return {
|
|
1855
|
+
...result,
|
|
1856
|
+
providerCalls,
|
|
1857
|
+
...(discardedUsage ? { discardedUsage } : {}),
|
|
1858
|
+
};
|
|
1859
|
+
}
|
|
1401
1860
|
retried++;
|
|
1861
|
+
discardedUsage = this.mergeDiscardedAttempts(discardedUsage, this.discardedAttemptFrom(result.usage));
|
|
1402
1862
|
const category = result.raw
|
|
1403
1863
|
?.response?.stop_details?.category;
|
|
1404
1864
|
onRetrying({ attempt: retried, maxAttempts, category });
|
|
@@ -1407,6 +1867,12 @@ export class Membrane {
|
|
|
1407
1867
|
buildContinuationRequest(originalRequest, prefillResult, accumulated) {
|
|
1408
1868
|
// Anthropic quirk: assistant content cannot end with trailing whitespace
|
|
1409
1869
|
const trimmedAccumulated = accumulated.trimEnd();
|
|
1870
|
+
// Everything before the watermark already rides EARLIER messages (a
|
|
1871
|
+
// persisted split turn), so only the suffix belongs in the trailing
|
|
1872
|
+
// assistant prefill — replacing it with the whole document would
|
|
1873
|
+
// duplicate the pre-seam text and flatten the image user-turn away.
|
|
1874
|
+
const baseOffset = prefillResult.accumulatedBaseOffset ?? 0;
|
|
1875
|
+
const trailingContent = accumulated.slice(baseOffset).trimEnd();
|
|
1410
1876
|
// Build continuation messages: keep all messages up to last assistant,
|
|
1411
1877
|
// then replace/add the accumulated content
|
|
1412
1878
|
const messages = [...prefillResult.messages];
|
|
@@ -1414,32 +1880,32 @@ export class Membrane {
|
|
|
1414
1880
|
let foundAssistant = false;
|
|
1415
1881
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
1416
1882
|
if (messages[i]?.role === 'assistant') {
|
|
1417
|
-
messages[i] = { role: 'assistant', content:
|
|
1883
|
+
messages[i] = { role: 'assistant', content: trailingContent };
|
|
1418
1884
|
foundAssistant = true;
|
|
1419
1885
|
break;
|
|
1420
1886
|
}
|
|
1421
1887
|
}
|
|
1422
1888
|
if (!foundAssistant) {
|
|
1423
|
-
messages.push({ role: 'assistant', content:
|
|
1889
|
+
messages.push({ role: 'assistant', content: trailingContent });
|
|
1424
1890
|
}
|
|
1425
|
-
return {
|
|
1891
|
+
return stripThinkingForPrefill({
|
|
1426
1892
|
...this.getBaseProviderParams(originalRequest.config),
|
|
1427
1893
|
// Continuations always end in an assistant prefill — the API rejects
|
|
1428
1894
|
// extended thinking combined with prefill, so never send the param here
|
|
1429
1895
|
thinking: undefined,
|
|
1430
1896
|
messages,
|
|
1431
|
-
system: prefillResult.systemContent
|
|
1432
|
-
? (Array.isArray(prefillResult.systemContent) && prefillResult.systemContent.length > 0
|
|
1433
|
-
? prefillResult.systemContent
|
|
1434
|
-
: prefillResult.systemContent)
|
|
1435
|
-
: undefined,
|
|
1897
|
+
system: ownSystemBlocks(prefillResult.systemContent) ?? undefined,
|
|
1436
1898
|
stopSequences: prefillResult.stopSequences,
|
|
1437
1899
|
extra: {
|
|
1438
1900
|
...originalRequest.providerParams,
|
|
1901
|
+
// Same contract transformRequest sends: adapters that reason about
|
|
1902
|
+
// the normalized shape (or fall back to serializing it) must not see
|
|
1903
|
+
// a continuation as a request with no normalized form at all.
|
|
1904
|
+
normalizedMessages: originalRequest.messages,
|
|
1439
1905
|
// Pre-serialized prompt for completions adapters — skip re-serialization
|
|
1440
1906
|
prompt: trimmedAccumulated,
|
|
1441
1907
|
},
|
|
1442
|
-
};
|
|
1908
|
+
});
|
|
1443
1909
|
}
|
|
1444
1910
|
/**
|
|
1445
1911
|
* Build continuation request with split-turn image injection.
|
|
@@ -1461,6 +1927,11 @@ export class Membrane {
|
|
|
1461
1927
|
buildContinuationRequestWithImages(originalRequest, prefillResult, accumulated, images, afterImageXml) {
|
|
1462
1928
|
// Anthropic quirk: assistant content cannot end with trailing whitespace
|
|
1463
1929
|
const trimmedAccumulated = accumulated.trimEnd();
|
|
1930
|
+
// The split replaces only the CURRENT trailing assistant message, which
|
|
1931
|
+
// covers the accumulated text from the previous seam onward (0 on the
|
|
1932
|
+
// first split, the previous image seam on a later one).
|
|
1933
|
+
const baseOffset = prefillResult.accumulatedBaseOffset ?? 0;
|
|
1934
|
+
const trailingContent = accumulated.slice(baseOffset).trimEnd();
|
|
1464
1935
|
// Build messages: copy all, then replace only the last assistant with split-turn
|
|
1465
1936
|
const messages = prefillResult.messages.map(msg => ({ ...msg }));
|
|
1466
1937
|
// Find last assistant — replace in-place via splice to preserve history
|
|
@@ -1474,7 +1945,7 @@ export class Membrane {
|
|
|
1474
1945
|
// Anthropic quirk: assistant content cannot end with trailing whitespace
|
|
1475
1946
|
const trimmedAfterXml = afterImageXml.trimEnd();
|
|
1476
1947
|
const splitTurnMessages = [
|
|
1477
|
-
{ role: 'assistant', content:
|
|
1948
|
+
{ role: 'assistant', content: trailingContent },
|
|
1478
1949
|
{ role: 'user', content: images },
|
|
1479
1950
|
{ role: 'assistant', content: trimmedAfterXml },
|
|
1480
1951
|
];
|
|
@@ -1484,20 +1955,39 @@ export class Membrane {
|
|
|
1484
1955
|
else {
|
|
1485
1956
|
messages.push(...splitTurnMessages);
|
|
1486
1957
|
}
|
|
1487
|
-
|
|
1958
|
+
// PERSIST the split. Later rounds rebuild from prefillResult.messages;
|
|
1959
|
+
// without this the image user-turn exists on exactly one request and the
|
|
1960
|
+
// next continuation flattens the accumulated document back over it —
|
|
1961
|
+
// leaving <function_results> XML asserting a screenshot the model can no
|
|
1962
|
+
// longer see. Reassign (never mutate in place): the previous array is
|
|
1963
|
+
// still referenced by the request already on the wire. The watermark
|
|
1964
|
+
// moves to the seam — the point in `accumulated` where afterImageXml is
|
|
1965
|
+
// about to be appended — so the next builder replaces only the closing
|
|
1966
|
+
// assistant turn.
|
|
1967
|
+
prefillResult.messages = messages;
|
|
1968
|
+
prefillResult.accumulatedBaseOffset = accumulated.length;
|
|
1969
|
+
return stripThinkingForPrefill({
|
|
1488
1970
|
...this.getBaseProviderParams(originalRequest.config),
|
|
1489
1971
|
// Continuations always end in an assistant prefill — the API rejects
|
|
1490
1972
|
// extended thinking combined with prefill, so never send the param here
|
|
1491
1973
|
thinking: undefined,
|
|
1492
1974
|
messages,
|
|
1493
|
-
system: prefillResult.systemContent
|
|
1494
|
-
? (Array.isArray(prefillResult.systemContent) && prefillResult.systemContent.length > 0
|
|
1495
|
-
? prefillResult.systemContent
|
|
1496
|
-
: prefillResult.systemContent)
|
|
1497
|
-
: undefined,
|
|
1975
|
+
system: ownSystemBlocks(prefillResult.systemContent) ?? undefined,
|
|
1498
1976
|
stopSequences: prefillResult.stopSequences,
|
|
1499
|
-
|
|
1500
|
-
|
|
1977
|
+
// Copied, not aliased: the guard below deletes the smuggled thinking
|
|
1978
|
+
// config, and mutating the caller's own providerParams object would
|
|
1979
|
+
// silently disable thinking on their NEXT (non-prefill) request.
|
|
1980
|
+
extra: {
|
|
1981
|
+
...originalRequest.providerParams,
|
|
1982
|
+
// Same contract as transformRequest and the plain continuation
|
|
1983
|
+
// builder. Without these a completions-style adapter fell through to
|
|
1984
|
+
// serializing PROVIDER-shaped messages as if they were normalized
|
|
1985
|
+
// ones, re-adding participant stop sequences the continuation
|
|
1986
|
+
// deliberately suppresses.
|
|
1987
|
+
normalizedMessages: originalRequest.messages,
|
|
1988
|
+
prompt: trimmedAccumulated,
|
|
1989
|
+
},
|
|
1990
|
+
});
|
|
1501
1991
|
}
|
|
1502
1992
|
transformResponse(providerResponse, request, prefillResult, startTime, attempts, rawRequest) {
|
|
1503
1993
|
// Extract text from response
|
|
@@ -1565,6 +2055,7 @@ export class Membrane {
|
|
|
1565
2055
|
}
|
|
1566
2056
|
// Parse XML tool calls from text if no native tool_use blocks were found
|
|
1567
2057
|
// This handles prefill mode where tools are XML in the text
|
|
2058
|
+
let emptyToolBlocks = 0;
|
|
1568
2059
|
if (toolCalls.length === 0 && rawAssistantText.includes('<function_calls>')) {
|
|
1569
2060
|
const parsed = parseToolCalls(rawAssistantText);
|
|
1570
2061
|
if (parsed?.calls.length) {
|
|
@@ -1572,12 +2063,25 @@ export class Membrane {
|
|
|
1572
2063
|
toolCalls.push(tc);
|
|
1573
2064
|
}
|
|
1574
2065
|
}
|
|
2066
|
+
else if (parsed) {
|
|
2067
|
+
emptyToolBlocks = 1;
|
|
2068
|
+
}
|
|
1575
2069
|
}
|
|
2070
|
+
const unclosedToolBlock = endsWithPartialToolBlock(rawAssistantText);
|
|
1576
2071
|
const stopReason = this.mapStopReason(providerResponse.stopReason);
|
|
2072
|
+
this.reportToolParseDiagnostics({ unclosedToolBlock, emptyToolBlocks }, stopReason);
|
|
1577
2073
|
const durationMs = Date.now() - startTime;
|
|
2074
|
+
// `NormalizedResponse.usage` is typed DetailedUsage and the streaming paths
|
|
2075
|
+
// already return the whole thing; complete() used to narrow it to
|
|
2076
|
+
// input/output here, so a caller reading `response.usage.cacheReadTokens`
|
|
2077
|
+
// saw undefined on one path and a number on the other.
|
|
1578
2078
|
const usage = {
|
|
1579
2079
|
inputTokens: providerResponse.usage.inputTokens,
|
|
1580
2080
|
outputTokens: providerResponse.usage.outputTokens,
|
|
2081
|
+
cacheCreationTokens: providerResponse.usage.cacheCreationTokens,
|
|
2082
|
+
cacheReadTokens: providerResponse.usage.cacheReadTokens,
|
|
2083
|
+
thinkingTokens: providerResponse.usage.thinkingTokens,
|
|
2084
|
+
estimatedCost: this.estimateCost(providerResponse.usage, request.config.model, providerResponse.model),
|
|
1581
2085
|
};
|
|
1582
2086
|
return {
|
|
1583
2087
|
content,
|
|
@@ -1591,14 +2095,9 @@ export class Membrane {
|
|
|
1591
2095
|
reason: stopReason,
|
|
1592
2096
|
triggeredSequence: providerResponse.stopSequence,
|
|
1593
2097
|
wasTruncated: stopReason === 'max_tokens',
|
|
2098
|
+
unclosedToolBlock,
|
|
1594
2099
|
},
|
|
1595
|
-
usage
|
|
1596
|
-
inputTokens: providerResponse.usage.inputTokens,
|
|
1597
|
-
outputTokens: providerResponse.usage.outputTokens,
|
|
1598
|
-
cacheCreationTokens: providerResponse.usage.cacheCreationTokens,
|
|
1599
|
-
cacheReadTokens: providerResponse.usage.cacheReadTokens,
|
|
1600
|
-
estimatedCost: this.estimateCost(providerResponse.usage, request.config.model),
|
|
1601
|
-
},
|
|
2100
|
+
usage,
|
|
1602
2101
|
timing: {
|
|
1603
2102
|
totalDurationMs: durationMs,
|
|
1604
2103
|
attempts,
|
|
@@ -1621,12 +2120,47 @@ export class Membrane {
|
|
|
1621
2120
|
},
|
|
1622
2121
|
};
|
|
1623
2122
|
}
|
|
1624
|
-
|
|
2123
|
+
/**
|
|
2124
|
+
* The turn is over, and the two guards that detect a half-written tool block
|
|
2125
|
+
* finally have a call site. Both shapes are defects a consumer must not
|
|
2126
|
+
* persist blind: an unclosed block splices onto the NEXT round's closing tag
|
|
2127
|
+
* (the loop does not resume on a length stop, so max_tokens leaves exactly
|
|
2128
|
+
* this), and a block that parsed to nothing means the model believes it
|
|
2129
|
+
* called a tool that never ran.
|
|
2130
|
+
*/
|
|
2131
|
+
reportToolParseDiagnostics(diagnostics, stopReason) {
|
|
2132
|
+
const warnLog = this.config.logger ?? console;
|
|
2133
|
+
if (diagnostics.unclosedToolBlock) {
|
|
2134
|
+
warnLog.warn(`[membrane] turn ended (${stopReason}) with an unclosed tool block in the ` +
|
|
2135
|
+
`assistant text — the loop does not resume on a length stop. Persisting this ` +
|
|
2136
|
+
`turn verbatim lets the next round's closing tag splice onto the stale ` +
|
|
2137
|
+
`opener; see details.stop.unclosedToolBlock.`);
|
|
2138
|
+
}
|
|
2139
|
+
if (diagnostics.emptyToolBlocks > 0) {
|
|
2140
|
+
warnLog.warn(`[membrane] ${diagnostics.emptyToolBlocks} function_calls block(s) parsed to ` +
|
|
2141
|
+
`zero tool calls — always a defect, never a normal ending. The call was ` +
|
|
2142
|
+
`returned as assistant text and nothing executed.`);
|
|
2143
|
+
}
|
|
2144
|
+
if (diagnostics.splicedToolBlocks) {
|
|
2145
|
+
warnLog.warn(`[membrane] ${diagnostics.splicedToolBlocks} tool block(s) spanned a second ` +
|
|
2146
|
+
`<function_calls> opener and were re-anchored to the innermost one — an ` +
|
|
2147
|
+
`earlier truncated block is present in this conversation's assistant text.`);
|
|
2148
|
+
}
|
|
2149
|
+
if (diagnostics.unclosedInvokeHeads) {
|
|
2150
|
+
warnLog.warn(`[membrane] ${diagnostics.unclosedInvokeHeads} <invoke> head(s) were left ` +
|
|
2151
|
+
`unclosed and swallowed the invoke that followed — nothing was dispatched ` +
|
|
2152
|
+
`under an unclosed head's name, and the call it absorbed was re-anchored ` +
|
|
2153
|
+
`and ran with its own parameters.`);
|
|
2154
|
+
}
|
|
2155
|
+
}
|
|
2156
|
+
buildFinalResponse(accumulated, contentBlocks, stopReason, turnUsage, request, prefillResult, startTime, attempts, rawRequest, rawResponse, executedToolCalls = [], executedToolResults = [], startInsideBlock = null, triggeredSequence) {
|
|
2157
|
+
const usage = turnUsage.total;
|
|
1625
2158
|
// Parse accumulated text into structured content blocks
|
|
1626
2159
|
// This extracts thinking, tool_use, tool_result, and text blocks
|
|
1627
2160
|
let finalContent;
|
|
1628
2161
|
let toolCalls;
|
|
1629
2162
|
let toolResults;
|
|
2163
|
+
let unclosedToolBlock = false;
|
|
1630
2164
|
if (contentBlocks.length > 0) {
|
|
1631
2165
|
// Native mode - content blocks already structured
|
|
1632
2166
|
finalContent = contentBlocks;
|
|
@@ -1642,6 +2176,8 @@ export class Membrane {
|
|
|
1642
2176
|
finalContent = parsed.blocks;
|
|
1643
2177
|
toolCalls = parsed.toolCalls.length > 0 ? parsed.toolCalls : executedToolCalls;
|
|
1644
2178
|
toolResults = parsed.toolResults.length > 0 ? parsed.toolResults : executedToolResults;
|
|
2179
|
+
unclosedToolBlock = parsed.unclosedToolBlock;
|
|
2180
|
+
this.reportToolParseDiagnostics(parsed, stopReason);
|
|
1645
2181
|
}
|
|
1646
2182
|
const durationMs = Date.now() - startTime;
|
|
1647
2183
|
return {
|
|
@@ -1656,19 +2192,21 @@ export class Membrane {
|
|
|
1656
2192
|
reason: stopReason,
|
|
1657
2193
|
triggeredSequence,
|
|
1658
2194
|
wasTruncated: stopReason === 'max_tokens',
|
|
2195
|
+
unclosedToolBlock,
|
|
1659
2196
|
},
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
2197
|
+
// Priced per round by the accumulator and summed — NOT re-derived here
|
|
2198
|
+
// from the turn totals, which would re-bill every round at the last
|
|
2199
|
+
// model's rate.
|
|
2200
|
+
usage,
|
|
1664
2201
|
timing: {
|
|
1665
2202
|
totalDurationMs: durationMs,
|
|
1666
2203
|
attempts,
|
|
1667
2204
|
},
|
|
1668
2205
|
model: {
|
|
1669
2206
|
requested: request.config.model,
|
|
1670
|
-
actual: request.config.model,
|
|
2207
|
+
actual: turnUsage.lastServedModel || request.config.model,
|
|
1671
2208
|
provider: this.adapter.name,
|
|
2209
|
+
perRound: turnUsage.perRound,
|
|
1672
2210
|
},
|
|
1673
2211
|
cache: {
|
|
1674
2212
|
markersInRequest: prefillResult.cacheMarkersApplied ?? 0,
|
|
@@ -1683,6 +2221,45 @@ export class Membrane {
|
|
|
1683
2221
|
},
|
|
1684
2222
|
};
|
|
1685
2223
|
}
|
|
2224
|
+
/**
|
|
2225
|
+
* Fold one discarded (billed but abandoned) attempt's usage into a carry.
|
|
2226
|
+
* Returns a NEW object so a caller's earlier snapshot is never mutated.
|
|
2227
|
+
*/
|
|
2228
|
+
mergeDiscardedAttempts(carry, add) {
|
|
2229
|
+
if (!add)
|
|
2230
|
+
return carry;
|
|
2231
|
+
const next = carry
|
|
2232
|
+
? { ...carry }
|
|
2233
|
+
: { attempts: 0, inputTokens: 0, outputTokens: 0 };
|
|
2234
|
+
next.attempts += add.attempts;
|
|
2235
|
+
next.inputTokens += add.inputTokens;
|
|
2236
|
+
next.outputTokens += add.outputTokens;
|
|
2237
|
+
if (add.cacheCreationTokens) {
|
|
2238
|
+
next.cacheCreationTokens = (next.cacheCreationTokens ?? 0) + add.cacheCreationTokens;
|
|
2239
|
+
}
|
|
2240
|
+
if (add.cacheReadTokens) {
|
|
2241
|
+
next.cacheReadTokens = (next.cacheReadTokens ?? 0) + add.cacheReadTokens;
|
|
2242
|
+
}
|
|
2243
|
+
return next;
|
|
2244
|
+
}
|
|
2245
|
+
/** One provider call's usage as a single-attempt discard record. */
|
|
2246
|
+
discardedAttemptFrom(usage) {
|
|
2247
|
+
const detailed = (usage ?? { inputTokens: 0, outputTokens: 0 });
|
|
2248
|
+
return {
|
|
2249
|
+
attempts: 1,
|
|
2250
|
+
inputTokens: detailed.inputTokens ?? 0,
|
|
2251
|
+
outputTokens: detailed.outputTokens ?? 0,
|
|
2252
|
+
...(detailed.cacheCreationTokens ? { cacheCreationTokens: detailed.cacheCreationTokens } : {}),
|
|
2253
|
+
...(detailed.cacheReadTokens ? { cacheReadTokens: detailed.cacheReadTokens } : {}),
|
|
2254
|
+
};
|
|
2255
|
+
}
|
|
2256
|
+
/** Price the discarded spend so a caller can read it without re-deriving. */
|
|
2257
|
+
pricedDiscardedAttempts(discarded, model) {
|
|
2258
|
+
if (!discarded)
|
|
2259
|
+
return undefined;
|
|
2260
|
+
const estimatedCost = this.estimateCost(discarded, model);
|
|
2261
|
+
return estimatedCost ? { ...discarded, estimatedCost } : discarded;
|
|
2262
|
+
}
|
|
1686
2263
|
mapStopReason(providerReason) {
|
|
1687
2264
|
switch (providerReason) {
|
|
1688
2265
|
case 'end_turn':
|
|
@@ -1703,19 +2280,50 @@ export class Membrane {
|
|
|
1703
2280
|
}
|
|
1704
2281
|
}
|
|
1705
2282
|
calculateCacheHitRatio(usage) {
|
|
1706
|
-
|
|
1707
|
-
const total = usage.inputTokens ?? 0;
|
|
1708
|
-
if (total === 0)
|
|
1709
|
-
return 0;
|
|
1710
|
-
return cacheRead / total;
|
|
2283
|
+
return calculateCacheHitRatio(usage);
|
|
1711
2284
|
}
|
|
1712
|
-
|
|
1713
|
-
|
|
2285
|
+
/**
|
|
2286
|
+
* Pricing is resolved on TWO axes, and SOURCE outranks SPECIFICITY:
|
|
2287
|
+
*
|
|
2288
|
+
* registry[served] → registry[requested] → builtin[served] → builtin[requested]
|
|
2289
|
+
*
|
|
2290
|
+
* Specificity — preferring the model that ACTUALLY served over the id that
|
|
2291
|
+
* was requested — is real: an alias or an auto-routed request otherwise
|
|
2292
|
+
* prices against a string the provider already replaced, and a live
|
|
2293
|
+
* 2026-08-25 call asking for `gpt-4o-mini` was served by
|
|
2294
|
+
* `gpt-4o-mini-2024-07-18`. But it only breaks ties WITHIN one source.
|
|
2295
|
+
* A configured `ModelRegistry` is the caller stating their own rates —
|
|
2296
|
+
* account-specific, negotiated, authoritative; the built-in table is
|
|
2297
|
+
* membrane's shipped guess at public list prices. Merging the two per-model
|
|
2298
|
+
* (`registry[served] ?? builtin[served]`, return on the first hit) let the
|
|
2299
|
+
* guess for a snapshot outrank the caller's own entry for the alias they
|
|
2300
|
+
* asked for, so a caller who prices their alias and lets the provider pick
|
|
2301
|
+
* the snapshot was billed at membrane's number instead of theirs.
|
|
2302
|
+
*
|
|
2303
|
+
* Both fallbacks stay: the served model may be absent from a source, and the
|
|
2304
|
+
* provider may name none at all.
|
|
2305
|
+
*/
|
|
2306
|
+
resolvePricing(requestedModel, actualModel) {
|
|
2307
|
+
const servedModel = actualModel && actualModel !== requestedModel ? actualModel : undefined;
|
|
2308
|
+
const fromRegistry = (modelId) => modelId === undefined ? undefined : this.registry?.getPricing(modelId);
|
|
2309
|
+
const fromBuiltin = (modelId) => modelId === undefined ? undefined : getDefaultPricing(modelId);
|
|
2310
|
+
return fromRegistry(servedModel)
|
|
2311
|
+
?? fromRegistry(requestedModel)
|
|
2312
|
+
?? fromBuiltin(servedModel)
|
|
2313
|
+
?? fromBuiltin(requestedModel);
|
|
1714
2314
|
}
|
|
1715
2315
|
/** Resolve pricing + calculate cost in one call (for one-shot use outside loops). */
|
|
1716
|
-
estimateCost(usage,
|
|
1717
|
-
const pricing = this.resolvePricing(
|
|
1718
|
-
|
|
2316
|
+
estimateCost(usage, requestedModel, actualModel) {
|
|
2317
|
+
const pricing = this.resolvePricing(requestedModel, actualModel);
|
|
2318
|
+
if (!pricing) {
|
|
2319
|
+
// An absent cost and a zero cost are different claims. Returning
|
|
2320
|
+
// undefined says "membrane does not know what this costs"; saying it out
|
|
2321
|
+
// loud once per model keeps that from reading as "free" to a caller that
|
|
2322
|
+
// only ever sees the omission.
|
|
2323
|
+
warnUnpricedModel(actualModel || requestedModel);
|
|
2324
|
+
return undefined;
|
|
2325
|
+
}
|
|
2326
|
+
return calculateCost(usage, pricing);
|
|
1719
2327
|
}
|
|
1720
2328
|
calculateRetryDelay(attempt, overloaded = false) {
|
|
1721
2329
|
const { retryDelayMs, backoffMultiplier, maxRetryDelayMs } = overloaded
|
|
@@ -1749,6 +2357,11 @@ export class Membrane {
|
|
|
1749
2357
|
* Check if an error is an abort error
|
|
1750
2358
|
*/
|
|
1751
2359
|
isAbortError(error) {
|
|
2360
|
+
// An adapter's own deadline: a timeout by classification, still an abort
|
|
2361
|
+
// by provenance, so the streaming paths hand back the partial content
|
|
2362
|
+
// they collected instead of throwing.
|
|
2363
|
+
if (isTimeoutAbortError(error))
|
|
2364
|
+
return true;
|
|
1752
2365
|
if (error instanceof Error) {
|
|
1753
2366
|
// Standard AbortError
|
|
1754
2367
|
if (error.name === 'AbortError')
|
|
@@ -1763,6 +2376,21 @@ export class Membrane {
|
|
|
1763
2376
|
}
|
|
1764
2377
|
return false;
|
|
1765
2378
|
}
|
|
2379
|
+
/**
|
|
2380
|
+
* Why a caught abort happened. The caller's own signal is authoritative:
|
|
2381
|
+
* if it fired, the cancellation is theirs whatever the error text says.
|
|
2382
|
+
* Otherwise an adapter-side deadline classifies as a timeout — the adapters
|
|
2383
|
+
* mark the abort createCombinedSignal's timeoutMs raises and map it to a
|
|
2384
|
+
* TimeoutAbortError, so the identity survives their error handling — and
|
|
2385
|
+
* anything else that reached the abort catch is a failure, not a person.
|
|
2386
|
+
*/
|
|
2387
|
+
abortReason(error, signal) {
|
|
2388
|
+
if (signal?.aborted)
|
|
2389
|
+
return 'user';
|
|
2390
|
+
if (classifyError(error).type === 'timeout')
|
|
2391
|
+
return 'timeout';
|
|
2392
|
+
return 'error';
|
|
2393
|
+
}
|
|
1766
2394
|
/**
|
|
1767
2395
|
* Build an AbortedResponse from current execution state
|
|
1768
2396
|
*/
|
|
@@ -1812,7 +2440,12 @@ export class Membrane {
|
|
|
1812
2440
|
* ```
|
|
1813
2441
|
*/
|
|
1814
2442
|
streamYielding(request, options = {}) {
|
|
1815
|
-
|
|
2443
|
+
// YieldingStreamOptions carries no per-request formatter override, so the
|
|
2444
|
+
// selection here can only land on the instance formatter — it goes through
|
|
2445
|
+
// resolveActiveFormatter anyway so this path reads the same single source
|
|
2446
|
+
// as complete() and stream() if an override is ever added.
|
|
2447
|
+
const activeFormatter = this.resolveActiveFormatter();
|
|
2448
|
+
const toolMode = this.resolveToolMode(request, activeFormatter);
|
|
1816
2449
|
// refusalRetries is implemented on the native path only. The XML path
|
|
1817
2450
|
// accumulates into a streaming parser carrying prefill context and
|
|
1818
2451
|
// resumption depths; rolling that back mid-turn is a separate problem,
|
|
@@ -1824,14 +2457,14 @@ export class Membrane {
|
|
|
1824
2457
|
}
|
|
1825
2458
|
// Create the yielding stream with the appropriate inference runner
|
|
1826
2459
|
const runInference = toolMode === 'native'
|
|
1827
|
-
? (stream) => this.runNativeToolsYielding(request, options, stream)
|
|
1828
|
-
: (stream) => this.runXmlToolsYielding(request, options, stream);
|
|
2460
|
+
? (stream) => this.runNativeToolsYielding(request, options, stream, activeFormatter)
|
|
2461
|
+
: (stream) => this.runXmlToolsYielding(request, options, stream, activeFormatter);
|
|
1829
2462
|
return new YieldingStreamImpl(options, runInference);
|
|
1830
2463
|
}
|
|
1831
2464
|
/**
|
|
1832
2465
|
* Run XML-based tool execution with yielding stream.
|
|
1833
2466
|
*/
|
|
1834
|
-
async runXmlToolsYielding(request, options, stream) {
|
|
2467
|
+
async runXmlToolsYielding(request, options, stream, activeFormatter = this.resolveActiveFormatter()) {
|
|
1835
2468
|
const startTime = Date.now();
|
|
1836
2469
|
const { maxToolDepth: maxToolDepthOpt, emitTokens = true, emitBlocks = true, emitUsage = true, } = options;
|
|
1837
2470
|
// Yielding paths default to unlimited (the caller — typically an agent
|
|
@@ -1863,14 +2496,20 @@ export class Membrane {
|
|
|
1863
2496
|
let enteredViaResumption = false;
|
|
1864
2497
|
let prevRoundStopSequence;
|
|
1865
2498
|
const warnLog = this.config.logger ?? console;
|
|
1866
|
-
// Initialize parser from formatter
|
|
1867
|
-
|
|
2499
|
+
// Initialize parser from the formatter streamYielding selected, so the
|
|
2500
|
+
// parser and the build below read the same format.
|
|
2501
|
+
const formatter = activeFormatter;
|
|
1868
2502
|
const parser = formatter.createStreamParser();
|
|
1869
2503
|
let toolDepth = 0;
|
|
2504
|
+
// Honest turn telemetry: provider calls actually made (including refusal
|
|
2505
|
+
// re-issues inside streamOnce) and continuation rounds.
|
|
2506
|
+
let providerCalls = 0;
|
|
2507
|
+
let rounds = 0;
|
|
1870
2508
|
// Once-per-stream latch for the injectedMessages-unsupported warning.
|
|
1871
2509
|
let warnedInjectionUnsupported = false;
|
|
1872
|
-
|
|
1873
|
-
|
|
2510
|
+
// See streamWithXmlTools: one accumulator per turn, pricing each round
|
|
2511
|
+
// under the model that served it.
|
|
2512
|
+
const turnUsage = new TurnUsageAccumulator(request.config.model, (servedModel) => this.resolvePricing(request.config.model, servedModel));
|
|
1874
2513
|
const contentBlocks = [];
|
|
1875
2514
|
let lastStopReason = 'end_turn';
|
|
1876
2515
|
let lastStopSequence;
|
|
@@ -1912,13 +2551,13 @@ export class Membrane {
|
|
|
1912
2551
|
resumptionRounds++;
|
|
1913
2552
|
if (resumptionRounds === RESUMPTION_WARN_ROUNDS) {
|
|
1914
2553
|
warnLog.warn(`[membrane] automatic resumption at round ${resumptionRounds} ` +
|
|
1915
|
-
`(${
|
|
2554
|
+
`(${turnUsage.total.inputTokens} input tokens so far this turn) — ` +
|
|
1916
2555
|
`a spin shows up here before it shows up on the bill`);
|
|
1917
2556
|
}
|
|
1918
2557
|
if (resumptionRounds > maxResumptionRounds) {
|
|
1919
2558
|
warnLog.warn(`[membrane] automatic resumption cap (${maxResumptionRounds}) reached — ` +
|
|
1920
2559
|
`ending turn with stopReason 'round_limit'. ` +
|
|
1921
|
-
`${
|
|
2560
|
+
`${turnUsage.total.inputTokens} input tokens spent this turn.`);
|
|
1922
2561
|
return false;
|
|
1923
2562
|
}
|
|
1924
2563
|
return true;
|
|
@@ -2001,6 +2640,8 @@ export class Membrane {
|
|
|
2001
2640
|
wrapThinkingTags: true,
|
|
2002
2641
|
onRequest: (req) => { rawRequest = req; },
|
|
2003
2642
|
});
|
|
2643
|
+
rounds++;
|
|
2644
|
+
providerCalls += streamResult.providerCalls;
|
|
2004
2645
|
// If we detected stop sequence manually, fix up the parser and result
|
|
2005
2646
|
if (detectedStopSequence && truncatedAccumulated !== null) {
|
|
2006
2647
|
parser.reset();
|
|
@@ -2015,19 +2656,10 @@ export class Membrane {
|
|
|
2015
2656
|
rawResponse = streamResult.raw;
|
|
2016
2657
|
lastStopReason = this.mapStopReason(streamResult.stopReason);
|
|
2017
2658
|
lastStopSequence = streamResult.stopSequence ?? undefined;
|
|
2018
|
-
// Accumulate usage (including cache metrics)
|
|
2019
|
-
|
|
2020
|
-
totalUsage.outputTokens += streamResult.usage.outputTokens;
|
|
2021
|
-
if (streamResult.usage.cacheCreationTokens) {
|
|
2022
|
-
totalUsage.cacheCreationTokens = (totalUsage.cacheCreationTokens ?? 0) + streamResult.usage.cacheCreationTokens;
|
|
2023
|
-
}
|
|
2024
|
-
if (streamResult.usage.cacheReadTokens) {
|
|
2025
|
-
totalUsage.cacheReadTokens = (totalUsage.cacheReadTokens ?? 0) + streamResult.usage.cacheReadTokens;
|
|
2026
|
-
}
|
|
2027
|
-
if (pricing)
|
|
2028
|
-
totalUsage.estimatedCost = calculateCost(totalUsage, pricing);
|
|
2659
|
+
// Accumulate usage (including cache metrics), priced at this round's model
|
|
2660
|
+
const usageSoFar = turnUsage.addRound(streamResult.model, streamResult.usage);
|
|
2029
2661
|
if (emitUsage) {
|
|
2030
|
-
stream.emit({ type: 'usage', usage:
|
|
2662
|
+
stream.emit({ type: 'usage', usage: usageSoFar });
|
|
2031
2663
|
}
|
|
2032
2664
|
// Flush the parser
|
|
2033
2665
|
const flushResult = parser.flush();
|
|
@@ -2050,7 +2682,7 @@ export class Membrane {
|
|
|
2050
2682
|
warnLog.warn(`[membrane] ${consecutiveStalledResumptions} consecutive automatic resumptions ` +
|
|
2051
2683
|
`made no progress (${streamedThisRound} chars this round, stop ` +
|
|
2052
2684
|
`${JSON.stringify(lastStopSequence ?? null)} repeated) — ending turn with ` +
|
|
2053
|
-
`stopReason 'no_progress'. ${
|
|
2685
|
+
`stopReason 'no_progress'. ${turnUsage.total.inputTokens} input tokens spent this turn.`);
|
|
2054
2686
|
lastStopReason = 'no_progress';
|
|
2055
2687
|
break;
|
|
2056
2688
|
}
|
|
@@ -2298,9 +2930,10 @@ export class Membrane {
|
|
|
2298
2930
|
// Build final response
|
|
2299
2931
|
const fullAccumulated = parser.getAccumulated();
|
|
2300
2932
|
const newContent = fullAccumulated.slice(initialPrefillLength);
|
|
2301
|
-
const response = this.buildFinalResponse(newContent, contentBlocks, lastStopReason,
|
|
2933
|
+
const response = this.buildFinalResponse(newContent, contentBlocks, lastStopReason, turnUsage, request, prefillResult, startTime, providerCalls, rawRequest, rawResponse, executedToolCalls, executedToolResults, initialBlockType, lastStopSequence);
|
|
2302
2934
|
// Merge provider thinking signatures into parser-derived thinking blocks
|
|
2303
2935
|
this.mergeProviderThinkingBlocks(response.content, providerThinkingBlocks);
|
|
2936
|
+
response.details.timing.rounds = rounds;
|
|
2304
2937
|
stream.emit({ type: 'complete', response });
|
|
2305
2938
|
}
|
|
2306
2939
|
catch (error) {
|
|
@@ -2309,7 +2942,7 @@ export class Membrane {
|
|
|
2309
2942
|
const newContent = fullAccumulated.slice(initialPrefillLength);
|
|
2310
2943
|
stream.emit({
|
|
2311
2944
|
type: 'aborted',
|
|
2312
|
-
reason:
|
|
2945
|
+
reason: this.abortReason(error, stream.signal),
|
|
2313
2946
|
partialContent: parseAccumulatedIntoBlocks(newContent).blocks,
|
|
2314
2947
|
rawAssistantText: newContent,
|
|
2315
2948
|
toolCalls: executedToolCalls,
|
|
@@ -2324,7 +2957,7 @@ export class Membrane {
|
|
|
2324
2957
|
/**
|
|
2325
2958
|
* Run native tool execution with yielding stream.
|
|
2326
2959
|
*/
|
|
2327
|
-
async runNativeToolsYielding(request, options, stream) {
|
|
2960
|
+
async runNativeToolsYielding(request, options, stream, activeFormatter = this.resolveActiveFormatter()) {
|
|
2328
2961
|
const startTime = Date.now();
|
|
2329
2962
|
const { maxToolDepth: maxToolDepthOpt, emitTokens = true, emitBlocks = true, emitUsage = true, } = options;
|
|
2330
2963
|
// Yielding paths default to unlimited (the caller — typically an agent
|
|
@@ -2336,8 +2969,13 @@ export class Membrane {
|
|
|
2336
2969
|
? Infinity
|
|
2337
2970
|
: maxToolDepthOpt;
|
|
2338
2971
|
let toolDepth = 0;
|
|
2339
|
-
|
|
2340
|
-
|
|
2972
|
+
// See streamWithXmlTools: one accumulator per turn, pricing each round
|
|
2973
|
+
// under the model that served it.
|
|
2974
|
+
const turnUsage = new TurnUsageAccumulator(request.config.model, (servedModel) => this.resolvePricing(request.config.model, servedModel));
|
|
2975
|
+
// Honest turn telemetry: provider calls actually made (including refusal
|
|
2976
|
+
// re-issues inside streamOnce) and continuation rounds.
|
|
2977
|
+
let providerCalls = 0;
|
|
2978
|
+
let rounds = 0;
|
|
2341
2979
|
let lastStopReason = 'end_turn';
|
|
2342
2980
|
let lastStopSequence;
|
|
2343
2981
|
let rawRequest;
|
|
@@ -2345,8 +2983,11 @@ export class Membrane {
|
|
|
2345
2983
|
let allTextAccumulated = '';
|
|
2346
2984
|
const executedToolCalls = [];
|
|
2347
2985
|
const executedToolResults = [];
|
|
2986
|
+
// Spend on refusal attempts this turn threw away (see streamOnce).
|
|
2987
|
+
let discardedUsage;
|
|
2348
2988
|
let messages = [...request.messages];
|
|
2349
2989
|
let allContentBlocks = [];
|
|
2990
|
+
let markersInLastRequest = 0;
|
|
2350
2991
|
try {
|
|
2351
2992
|
// Tool execution loop
|
|
2352
2993
|
while (toolDepth <= maxToolDepth) {
|
|
@@ -2362,26 +3003,17 @@ export class Membrane {
|
|
|
2362
3003
|
return;
|
|
2363
3004
|
}
|
|
2364
3005
|
// Build provider request with native tools
|
|
2365
|
-
const providerRequest = this.buildNativeToolRequest(request, messages);
|
|
3006
|
+
const providerRequest = this.buildNativeToolRequest(request, messages, toolDepth > 0, activeFormatter);
|
|
2366
3007
|
// Stream from provider
|
|
2367
3008
|
let textAccumulated = '';
|
|
2368
|
-
let blockIndex = 0;
|
|
2369
3009
|
// Where this attempt starts inside the tool-loop-spanning buffer, so
|
|
2370
3010
|
// a refusal retry can roll back exactly this attempt's contribution.
|
|
2371
3011
|
const allTextBefore = allTextAccumulated.length;
|
|
2372
|
-
// Track block-type from the provider's
|
|
3012
|
+
// Track block-type from the provider's content_block signals so
|
|
2373
3013
|
// every token chunk is tagged with the membrane block it belongs to.
|
|
2374
3014
|
// Without this, thinking_delta chunks get mislabelled as 'text' and
|
|
2375
3015
|
// downstream consumers (TUIs, WebUIs) can't render them distinctly.
|
|
2376
|
-
|
|
2377
|
-
const seenBlockIndices = new Set();
|
|
2378
|
-
const mapApiBlockType = (apiType) => {
|
|
2379
|
-
if (apiType === 'thinking')
|
|
2380
|
-
return 'thinking';
|
|
2381
|
-
if (apiType === 'tool_use')
|
|
2382
|
-
return 'tool_call';
|
|
2383
|
-
return 'text';
|
|
2384
|
-
};
|
|
3016
|
+
const tracker = new NativeBlockTracker(emitBlocks ? (event) => stream.emit({ type: 'block', event }) : undefined);
|
|
2385
3017
|
const streamResult = await this.streamOnce(providerRequest, {
|
|
2386
3018
|
onChunk: (chunk) => {
|
|
2387
3019
|
if (stream.isCancelled)
|
|
@@ -2390,9 +3022,9 @@ export class Membrane {
|
|
|
2390
3022
|
allTextAccumulated += chunk;
|
|
2391
3023
|
if (emitTokens) {
|
|
2392
3024
|
const meta = {
|
|
2393
|
-
type:
|
|
2394
|
-
visible:
|
|
2395
|
-
blockIndex,
|
|
3025
|
+
type: tracker.currentType,
|
|
3026
|
+
visible: tracker.currentType === 'text',
|
|
3027
|
+
blockIndex: tracker.blockIndex,
|
|
2396
3028
|
};
|
|
2397
3029
|
stream.emit({ type: 'tokens', content: chunk, meta });
|
|
2398
3030
|
}
|
|
@@ -2400,41 +3032,7 @@ export class Membrane {
|
|
|
2400
3032
|
onContentBlock: (index, block) => {
|
|
2401
3033
|
if (stream.isCancelled)
|
|
2402
3034
|
return;
|
|
2403
|
-
|
|
2404
|
-
const mbType = mapApiBlockType(apiType);
|
|
2405
|
-
const isStart = !seenBlockIndices.has(index);
|
|
2406
|
-
if (isStart) {
|
|
2407
|
-
seenBlockIndices.add(index);
|
|
2408
|
-
currentBlockType = mbType;
|
|
2409
|
-
blockIndex = index;
|
|
2410
|
-
if (emitBlocks) {
|
|
2411
|
-
stream.emit({
|
|
2412
|
-
type: 'block',
|
|
2413
|
-
event: { event: 'block_start', index, block: { type: mbType } },
|
|
2414
|
-
});
|
|
2415
|
-
}
|
|
2416
|
-
}
|
|
2417
|
-
else if (emitBlocks) {
|
|
2418
|
-
// Second call for the same index = content_block_stop. The
|
|
2419
|
-
// provider has filled the block with final content; surface
|
|
2420
|
-
// a block_complete with the relevant fields for consumers
|
|
2421
|
-
// that want full block payloads (e.g. context-manager).
|
|
2422
|
-
const apiBlock = block;
|
|
2423
|
-
const mb = { type: mbType };
|
|
2424
|
-
if (mbType === 'text')
|
|
2425
|
-
mb.content = apiBlock?.text;
|
|
2426
|
-
else if (mbType === 'thinking')
|
|
2427
|
-
mb.content = apiBlock?.thinking;
|
|
2428
|
-
else if (mbType === 'tool_call') {
|
|
2429
|
-
mb.toolId = apiBlock?.id;
|
|
2430
|
-
mb.toolName = apiBlock?.name;
|
|
2431
|
-
mb.input = apiBlock?.input;
|
|
2432
|
-
}
|
|
2433
|
-
stream.emit({
|
|
2434
|
-
type: 'block',
|
|
2435
|
-
event: { event: 'block_complete', index, block: mb },
|
|
2436
|
-
});
|
|
2437
|
-
}
|
|
3035
|
+
tracker.onProviderBlock(index, block);
|
|
2438
3036
|
},
|
|
2439
3037
|
}, {
|
|
2440
3038
|
signal: stream.signal,
|
|
@@ -2442,6 +3040,14 @@ export class Membrane {
|
|
|
2442
3040
|
idleTimeoutMs: options.idleTimeoutMs,
|
|
2443
3041
|
normalizedRequest: request,
|
|
2444
3042
|
onRequest: (req) => { rawRequest = req; },
|
|
3043
|
+
// Telemetry reports what this request actually SHIPPED with —
|
|
3044
|
+
// builder breakpoints, stale passthrough, fallback, float, plus
|
|
3045
|
+
// whatever the beforeRequest hook and the wire clamp did after
|
|
3046
|
+
// the build. Both native paths used to hardcode 0, and counting
|
|
3047
|
+
// at build time reported a number no request ever had.
|
|
3048
|
+
onWireCacheMarkers: (markerCount) => {
|
|
3049
|
+
markersInLastRequest = markerCount;
|
|
3050
|
+
},
|
|
2445
3051
|
refusalRetries: options.refusalRetries,
|
|
2446
3052
|
// Discard the refused attempt: roll the accumulators back to
|
|
2447
3053
|
// where this attempt began and tell the consumer to drop what it
|
|
@@ -2450,9 +3056,7 @@ export class Membrane {
|
|
|
2450
3056
|
onRetrying: (info) => {
|
|
2451
3057
|
allTextAccumulated = allTextAccumulated.slice(0, allTextBefore);
|
|
2452
3058
|
textAccumulated = '';
|
|
2453
|
-
|
|
2454
|
-
currentBlockType = 'text';
|
|
2455
|
-
seenBlockIndices.clear();
|
|
3059
|
+
tracker.reset();
|
|
2456
3060
|
stream.emit({
|
|
2457
3061
|
type: 'retrying',
|
|
2458
3062
|
attempt: info.attempt,
|
|
@@ -2462,22 +3066,21 @@ export class Membrane {
|
|
|
2462
3066
|
});
|
|
2463
3067
|
},
|
|
2464
3068
|
});
|
|
3069
|
+
// Single-callback adapters (OpenAI Responses) report each finalised
|
|
3070
|
+
// block once, after the stream: complete whatever never saw a stop.
|
|
3071
|
+
tracker.flush();
|
|
3072
|
+
rounds++;
|
|
3073
|
+
providerCalls += streamResult.providerCalls;
|
|
2465
3074
|
rawResponse = streamResult.raw;
|
|
2466
3075
|
lastStopReason = this.mapStopReason(streamResult.stopReason);
|
|
2467
3076
|
lastStopSequence = streamResult.stopSequence ?? undefined;
|
|
2468
|
-
//
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
}
|
|
2474
|
-
if (streamResult.usage.cacheReadTokens) {
|
|
2475
|
-
totalUsage.cacheReadTokens = (totalUsage.cacheReadTokens ?? 0) + streamResult.usage.cacheReadTokens;
|
|
2476
|
-
}
|
|
2477
|
-
if (pricing)
|
|
2478
|
-
totalUsage.estimatedCost = calculateCost(totalUsage, pricing);
|
|
3077
|
+
// Attempts this round re-issued past a refusal are billed calls whose
|
|
3078
|
+
// output was discarded — carry their spend to the final response.
|
|
3079
|
+
discardedUsage = this.mergeDiscardedAttempts(discardedUsage, streamResult.discardedUsage);
|
|
3080
|
+
// Accumulate usage (including cache metrics), priced at this round's model
|
|
3081
|
+
const usageSoFar = turnUsage.addRound(streamResult.model, streamResult.usage);
|
|
2479
3082
|
if (emitUsage) {
|
|
2480
|
-
stream.emit({ type: 'usage', usage:
|
|
3083
|
+
stream.emit({ type: 'usage', usage: usageSoFar });
|
|
2481
3084
|
}
|
|
2482
3085
|
// Parse content blocks from response
|
|
2483
3086
|
const responseBlocks = this.parseProviderContent(streamResult.content);
|
|
@@ -2563,6 +3166,7 @@ export class Membrane {
|
|
|
2563
3166
|
break;
|
|
2564
3167
|
}
|
|
2565
3168
|
const durationMs = Date.now() - startTime;
|
|
3169
|
+
const totalUsage = turnUsage.total;
|
|
2566
3170
|
const response = {
|
|
2567
3171
|
content: allContentBlocks,
|
|
2568
3172
|
rawAssistantText: allTextAccumulated,
|
|
@@ -2576,18 +3180,25 @@ export class Membrane {
|
|
|
2576
3180
|
triggeredSequence: lastStopSequence,
|
|
2577
3181
|
wasTruncated: lastStopReason === 'max_tokens',
|
|
2578
3182
|
},
|
|
2579
|
-
usage: {
|
|
3183
|
+
usage: {
|
|
3184
|
+
...totalUsage,
|
|
3185
|
+
...(discardedUsage
|
|
3186
|
+
? { discardedAttempts: this.pricedDiscardedAttempts(discardedUsage, request.config.model) }
|
|
3187
|
+
: {}),
|
|
3188
|
+
},
|
|
2580
3189
|
timing: {
|
|
2581
3190
|
totalDurationMs: durationMs,
|
|
2582
|
-
attempts:
|
|
3191
|
+
attempts: providerCalls,
|
|
3192
|
+
rounds,
|
|
2583
3193
|
},
|
|
2584
3194
|
model: {
|
|
2585
3195
|
requested: request.config.model,
|
|
2586
|
-
actual: request.config.model,
|
|
3196
|
+
actual: turnUsage.lastServedModel || request.config.model,
|
|
2587
3197
|
provider: this.adapter.name,
|
|
3198
|
+
perRound: turnUsage.perRound,
|
|
2588
3199
|
},
|
|
2589
3200
|
cache: {
|
|
2590
|
-
markersInRequest:
|
|
3201
|
+
markersInRequest: markersInLastRequest,
|
|
2591
3202
|
tokensCreated: totalUsage.cacheCreationTokens ?? 0,
|
|
2592
3203
|
tokensRead: totalUsage.cacheReadTokens ?? 0,
|
|
2593
3204
|
hitRatio: this.calculateCacheHitRatio(totalUsage),
|
|
@@ -2604,7 +3215,7 @@ export class Membrane {
|
|
|
2604
3215
|
if (this.isAbortError(error)) {
|
|
2605
3216
|
stream.emit({
|
|
2606
3217
|
type: 'aborted',
|
|
2607
|
-
reason:
|
|
3218
|
+
reason: this.abortReason(error, stream.signal),
|
|
2608
3219
|
rawAssistantText: allTextAccumulated,
|
|
2609
3220
|
toolCalls: executedToolCalls,
|
|
2610
3221
|
toolResults: executedToolResults,
|