@animalabs/membrane 0.5.80 → 0.5.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cache-wire-receipt.d.ts +13 -0
- package/dist/cache-wire-receipt.d.ts.map +1 -0
- package/dist/cache-wire-receipt.js +108 -0
- package/dist/cache-wire-receipt.js.map +1 -0
- package/dist/context/index.d.ts +1 -1
- package/dist/context/index.d.ts.map +1 -1
- package/dist/context/index.js +1 -1
- package/dist/context/index.js.map +1 -1
- package/dist/context/process.d.ts +3 -1
- package/dist/context/process.d.ts.map +1 -1
- package/dist/context/process.js +230 -49
- package/dist/context/process.js.map +1 -1
- package/dist/context/types.d.ts +30 -0
- package/dist/context/types.d.ts.map +1 -1
- package/dist/context/types.js +16 -0
- package/dist/context/types.js.map +1 -1
- package/dist/floating-cache-marker.test.js +21 -0
- package/dist/floating-cache-marker.test.js.map +1 -1
- package/dist/formatters/anthropic-xml.d.ts +2 -0
- package/dist/formatters/anthropic-xml.d.ts.map +1 -1
- package/dist/formatters/anthropic-xml.js +64 -23
- package/dist/formatters/anthropic-xml.js.map +1 -1
- package/dist/formatters/native.d.ts.map +1 -1
- package/dist/formatters/native.js +16 -3
- package/dist/formatters/native.js.map +1 -1
- package/dist/formatters/normalize-tool-pairs.d.ts +7 -0
- package/dist/formatters/normalize-tool-pairs.d.ts.map +1 -1
- package/dist/formatters/normalize-tool-pairs.js +45 -0
- package/dist/formatters/normalize-tool-pairs.js.map +1 -1
- package/dist/formatters/types.d.ts +31 -0
- package/dist/formatters/types.d.ts.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/membrane.d.ts +137 -8
- package/dist/membrane.d.ts.map +1 -1
- package/dist/membrane.js +823 -310
- package/dist/membrane.js.map +1 -1
- package/dist/providers/anthropic.d.ts +37 -0
- package/dist/providers/anthropic.d.ts.map +1 -1
- package/dist/providers/anthropic.js +142 -25
- package/dist/providers/anthropic.js.map +1 -1
- package/dist/providers/bedrock.d.ts +8 -0
- package/dist/providers/bedrock.d.ts.map +1 -1
- package/dist/providers/bedrock.js +24 -1
- package/dist/providers/bedrock.js.map +1 -1
- package/dist/providers/gemini.d.ts +11 -0
- package/dist/providers/gemini.d.ts.map +1 -1
- package/dist/providers/gemini.js +104 -82
- package/dist/providers/gemini.js.map +1 -1
- package/dist/providers/index.d.ts +1 -1
- package/dist/providers/index.d.ts.map +1 -1
- package/dist/providers/index.js.map +1 -1
- package/dist/providers/mock.d.ts +2 -0
- package/dist/providers/mock.d.ts.map +1 -1
- package/dist/providers/mock.js +2 -0
- package/dist/providers/mock.js.map +1 -1
- package/dist/providers/openai-compatible.d.ts +7 -0
- package/dist/providers/openai-compatible.d.ts.map +1 -1
- package/dist/providers/openai-compatible.js +99 -48
- package/dist/providers/openai-compatible.js.map +1 -1
- package/dist/providers/openai-completions.d.ts +5 -0
- package/dist/providers/openai-completions.d.ts.map +1 -1
- package/dist/providers/openai-completions.js +107 -44
- package/dist/providers/openai-completions.js.map +1 -1
- package/dist/providers/openai-responses-api.d.ts +16 -0
- package/dist/providers/openai-responses-api.d.ts.map +1 -1
- package/dist/providers/openai-responses-api.js +38 -10
- package/dist/providers/openai-responses-api.js.map +1 -1
- package/dist/providers/openai-responses.d.ts +2 -0
- package/dist/providers/openai-responses.d.ts.map +1 -1
- package/dist/providers/openai-responses.js +8 -1
- package/dist/providers/openai-responses.js.map +1 -1
- package/dist/providers/openai.d.ts +7 -0
- package/dist/providers/openai.d.ts.map +1 -1
- package/dist/providers/openai.js +92 -44
- package/dist/providers/openai.js.map +1 -1
- package/dist/providers/openrouter.d.ts +9 -0
- package/dist/providers/openrouter.d.ts.map +1 -1
- package/dist/providers/openrouter.js +118 -62
- package/dist/providers/openrouter.js.map +1 -1
- package/dist/providers/utils.d.ts +74 -2
- package/dist/providers/utils.d.ts.map +1 -1
- package/dist/providers/utils.js +184 -3
- package/dist/providers/utils.js.map +1 -1
- package/dist/registry/default-pricing.d.ts +22 -0
- package/dist/registry/default-pricing.d.ts.map +1 -1
- package/dist/registry/default-pricing.js +162 -28
- package/dist/registry/default-pricing.js.map +1 -1
- package/dist/types/content.d.ts +13 -0
- package/dist/types/content.d.ts.map +1 -1
- package/dist/types/content.js.map +1 -1
- package/dist/types/errors.d.ts +15 -0
- package/dist/types/errors.d.ts.map +1 -1
- package/dist/types/errors.js +26 -0
- package/dist/types/errors.js.map +1 -1
- package/dist/types/index.d.ts +3 -3
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/index.js +1 -1
- package/dist/types/index.js.map +1 -1
- package/dist/types/provider.d.ts +50 -0
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/request.d.ts +6 -0
- package/dist/types/request.d.ts.map +1 -1
- package/dist/types/response.d.ts +85 -4
- package/dist/types/response.d.ts.map +1 -1
- package/dist/types/response.js.map +1 -1
- package/dist/types/streaming.d.ts +6 -1
- package/dist/types/streaming.d.ts.map +1 -1
- package/dist/utils/cache-marker-budget.d.ts +91 -0
- package/dist/utils/cache-marker-budget.d.ts.map +1 -0
- package/dist/utils/cache-marker-budget.js +208 -0
- package/dist/utils/cache-marker-budget.js.map +1 -0
- package/dist/utils/cost.d.ts +22 -0
- package/dist/utils/cost.d.ts.map +1 -1
- package/dist/utils/cost.js +55 -0
- package/dist/utils/cost.js.map +1 -1
- package/dist/utils/thinking-carriers.d.ts +80 -0
- package/dist/utils/thinking-carriers.d.ts.map +1 -0
- package/dist/utils/thinking-carriers.js +115 -0
- package/dist/utils/thinking-carriers.js.map +1 -0
- package/dist/utils/tool-parser.d.ts +24 -0
- package/dist/utils/tool-parser.d.ts.map +1 -1
- package/dist/utils/tool-parser.js +367 -161
- package/dist/utils/tool-parser.js.map +1 -1
- package/dist/utils/usage.d.ts +71 -0
- package/dist/utils/usage.d.ts.map +1 -0
- package/dist/utils/usage.js +161 -0
- package/dist/utils/usage.js.map +1 -0
- package/dist/yielding-stream.d.ts +8 -0
- package/dist/yielding-stream.d.ts.map +1 -1
- package/dist/yielding-stream.js +49 -3
- package/dist/yielding-stream.js.map +1 -1
- package/package.json +1 -1
- package/src/cache-wire-receipt.ts +125 -0
- package/src/context/index.ts +1 -0
- package/src/context/process.ts +305 -59
- package/src/context/types.ts +38 -0
- package/src/floating-cache-marker.test.ts +24 -0
- package/src/formatters/anthropic-xml.ts +66 -23
- package/src/formatters/native.ts +23 -2
- package/src/formatters/normalize-tool-pairs.ts +49 -0
- package/src/formatters/types.ts +35 -0
- package/src/index.ts +6 -0
- package/src/membrane.ts +974 -313
- package/src/providers/anthropic.ts +173 -19
- package/src/providers/bedrock.ts +25 -1
- package/src/providers/gemini.ts +125 -87
- package/src/providers/index.ts +1 -0
- package/src/providers/mock.ts +3 -0
- package/src/providers/openai-compatible.ts +102 -45
- package/src/providers/openai-completions.ts +112 -46
- package/src/providers/openai-responses-api.ts +45 -15
- package/src/providers/openai-responses.ts +8 -1
- package/src/providers/openai.ts +95 -42
- package/src/providers/openrouter.ts +125 -63
- package/src/providers/utils.ts +227 -6
- package/src/registry/default-pricing.ts +165 -28
- package/src/types/content.ts +14 -0
- package/src/types/errors.ts +28 -0
- package/src/types/index.ts +6 -0
- package/src/types/provider.ts +55 -0
- package/src/types/request.ts +7 -0
- package/src/types/response.ts +109 -8
- package/src/types/streaming.ts +8 -2
- package/src/utils/cache-marker-budget.ts +237 -0
- package/src/utils/cost.ts +64 -0
- package/src/utils/thinking-carriers.ts +125 -0
- package/src/utils/tool-parser.ts +473 -174
- package/src/utils/usage.ts +184 -0
- package/src/yielding-stream.ts +54 -3
package/src/membrane.ts
CHANGED
|
@@ -10,12 +10,14 @@ import type {
|
|
|
10
10
|
AbortedResponse,
|
|
11
11
|
ContentBlock,
|
|
12
12
|
ProviderAdapter,
|
|
13
|
+
ProviderResponse,
|
|
13
14
|
ModelRegistry,
|
|
14
15
|
MembraneConfig,
|
|
15
16
|
StreamOptions,
|
|
16
17
|
CompleteOptions,
|
|
17
18
|
BasicUsage,
|
|
18
19
|
DetailedUsage,
|
|
20
|
+
DiscardedAttemptsUsage,
|
|
19
21
|
StopReason,
|
|
20
22
|
TimingInfo,
|
|
21
23
|
CacheInfo,
|
|
@@ -23,23 +25,39 @@ import type {
|
|
|
23
25
|
ToolResult,
|
|
24
26
|
ToolContext,
|
|
25
27
|
RetryConfig,
|
|
26
|
-
ToolMode,
|
|
27
28
|
ToolDefinition,
|
|
28
29
|
} from './types/index.js';
|
|
29
30
|
import { lastCacheableBlockIndex } from './formatters/native.js';
|
|
31
|
+
import {
|
|
32
|
+
sameThinkingText,
|
|
33
|
+
findSpanningProviderRun,
|
|
34
|
+
thinkingCarrierKey,
|
|
35
|
+
stripThinkingForPrefill,
|
|
36
|
+
} from './utils/thinking-carriers.js';
|
|
37
|
+
import {
|
|
38
|
+
assertCacheMarkersWithinLimit,
|
|
39
|
+
countWireCacheMarkers,
|
|
40
|
+
clampCacheMarkers,
|
|
41
|
+
ownSystemBlocks,
|
|
42
|
+
MAX_CACHE_BREAKPOINTS,
|
|
43
|
+
} from './utils/cache-marker-budget.js';
|
|
30
44
|
import {
|
|
31
45
|
DEFAULT_RETRY_CONFIG,
|
|
32
46
|
MembraneError,
|
|
33
47
|
classifyError,
|
|
34
48
|
isOverloadedError,
|
|
49
|
+
isTimeoutAbortError,
|
|
35
50
|
isTextContent,
|
|
36
51
|
isAbortedResponse,
|
|
52
|
+
unsupportedError,
|
|
37
53
|
} from './types/index.js';
|
|
38
54
|
import type { BuildResult } from './formatters/types.js';
|
|
55
|
+
import { computeCacheWireReceipt } from './cache-wire-receipt.js';
|
|
39
56
|
import {
|
|
40
57
|
parseToolCalls,
|
|
41
58
|
formatToolResults,
|
|
42
59
|
parseAccumulatedIntoBlocks,
|
|
60
|
+
endsWithPartialToolBlock,
|
|
43
61
|
hasImageInToolResults,
|
|
44
62
|
formatToolResultsForSplitTurn,
|
|
45
63
|
type ProviderImageBlock,
|
|
@@ -54,9 +72,19 @@ import type {
|
|
|
54
72
|
} from './types/yielding-stream.js';
|
|
55
73
|
import type { PrefillFormatter, StreamParser } from './formatters/types.js';
|
|
56
74
|
import { AnthropicXmlFormatter } from './formatters/anthropic-xml.js';
|
|
57
|
-
import {
|
|
75
|
+
import {
|
|
76
|
+
normalizeToolPairs,
|
|
77
|
+
mergeConsecutiveRoles,
|
|
78
|
+
PREFIX_REWRITING_NORMALIZE_EVENT_KINDS,
|
|
79
|
+
} from './formatters/normalize-tool-pairs.js';
|
|
58
80
|
import { YieldingStreamImpl } from './yielding-stream.js';
|
|
59
|
-
import { calculateCost } from './utils/cost.js';
|
|
81
|
+
import { calculateCost, warnUnpricedModel } from './utils/cost.js';
|
|
82
|
+
import {
|
|
83
|
+
TurnUsageAccumulator,
|
|
84
|
+
calculateCacheHitRatio,
|
|
85
|
+
normalizeUsageToCacheExcluded,
|
|
86
|
+
warnUnconvertibleProviderItem,
|
|
87
|
+
} from './utils/usage.js';
|
|
60
88
|
import {
|
|
61
89
|
isAcceptedImageMediaType,
|
|
62
90
|
strippedImagePlaceholder,
|
|
@@ -68,6 +96,94 @@ import { getDefaultPricing } from './registry/default-pricing.js';
|
|
|
68
96
|
// Membrane Class
|
|
69
97
|
// ============================================================================
|
|
70
98
|
|
|
99
|
+
/**
|
|
100
|
+
* Block-lifecycle tracking shared by the two native-tools streaming paths
|
|
101
|
+
* (`streamWithNativeTools` and `runNativeToolsYielding`).
|
|
102
|
+
*
|
|
103
|
+
* Providers signal blocks through `onContentBlock(index, block)`, but not all
|
|
104
|
+
* of them the same way: the Anthropic and Bedrock adapters fire it twice per
|
|
105
|
+
* index (content_block_start with an empty block, content_block_stop with the
|
|
106
|
+
* finalised one), while the OpenAI Responses adapter fires it ONCE per block,
|
|
107
|
+
* already finalised, after the stream has ended. Treating "second sighting"
|
|
108
|
+
* as the only completion signal therefore left single-callback adapters with
|
|
109
|
+
* `block_start` events that never completed (#63 review). The tracker keeps
|
|
110
|
+
* the paired semantics and adds `flush()`, which the caller runs once the
|
|
111
|
+
* provider stream has returned: every started block that never saw a second
|
|
112
|
+
* callback is completed from the last block payload seen for it.
|
|
113
|
+
*/
|
|
114
|
+
class NativeBlockTracker {
|
|
115
|
+
currentType: MembraneBlockType = 'text';
|
|
116
|
+
blockIndex = 0;
|
|
117
|
+
private readonly started = new Map<number, MembraneBlockType>();
|
|
118
|
+
private readonly completed = new Set<number>();
|
|
119
|
+
private readonly lastSeen = new Map<number, unknown>();
|
|
120
|
+
|
|
121
|
+
constructor(private readonly emit: ((event: BlockEvent) => void) | undefined) {}
|
|
122
|
+
|
|
123
|
+
static mapApiBlockType(apiType: string | undefined): MembraneBlockType {
|
|
124
|
+
if (apiType === 'thinking' || apiType === 'redacted_thinking' || apiType === 'reasoning') return 'thinking';
|
|
125
|
+
if (apiType === 'tool_use' || apiType === 'function_call' || apiType === 'tool_call') return 'tool_call';
|
|
126
|
+
return 'text';
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** Provider block callback: first sighting of an index starts it, a second completes it. */
|
|
130
|
+
onProviderBlock(index: number, block: unknown): void {
|
|
131
|
+
this.lastSeen.set(index, block);
|
|
132
|
+
if (!this.started.has(index)) {
|
|
133
|
+
const mbType = NativeBlockTracker.mapApiBlockType((block as { type?: string } | undefined)?.type);
|
|
134
|
+
this.started.set(index, mbType);
|
|
135
|
+
this.currentType = mbType;
|
|
136
|
+
this.blockIndex = index;
|
|
137
|
+
this.emit?.({ event: 'block_start', index, block: { type: mbType } });
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
this.complete(index, block);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Complete every started block that never received its second callback.
|
|
145
|
+
* Run after the provider stream has returned; idempotent, and a no-op for
|
|
146
|
+
* paired-callback adapters.
|
|
147
|
+
*/
|
|
148
|
+
flush(): void {
|
|
149
|
+
for (const index of this.started.keys()) {
|
|
150
|
+
if (!this.completed.has(index)) this.complete(index, this.lastSeen.get(index));
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** Discard tracking state (refusal retry rolled the attempt back). */
|
|
155
|
+
reset(): void {
|
|
156
|
+
this.currentType = 'text';
|
|
157
|
+
this.blockIndex = 0;
|
|
158
|
+
this.started.clear();
|
|
159
|
+
this.completed.clear();
|
|
160
|
+
this.lastSeen.clear();
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
private complete(index: number, block: unknown): void {
|
|
164
|
+
if (this.completed.has(index)) return;
|
|
165
|
+
this.completed.add(index);
|
|
166
|
+
const mbType = this.started.get(index)
|
|
167
|
+
?? NativeBlockTracker.mapApiBlockType((block as { type?: string } | undefined)?.type);
|
|
168
|
+
const apiBlock = block as {
|
|
169
|
+
text?: string;
|
|
170
|
+
thinking?: string;
|
|
171
|
+
id?: string;
|
|
172
|
+
name?: string;
|
|
173
|
+
input?: unknown;
|
|
174
|
+
} | undefined;
|
|
175
|
+
const mb: MembraneBlock = { type: mbType };
|
|
176
|
+
if (mbType === 'text') mb.content = apiBlock?.text;
|
|
177
|
+
else if (mbType === 'thinking') mb.content = apiBlock?.thinking;
|
|
178
|
+
else if (mbType === 'tool_call') {
|
|
179
|
+
mb.toolId = apiBlock?.id;
|
|
180
|
+
mb.toolName = apiBlock?.name;
|
|
181
|
+
mb.input = apiBlock?.input as Record<string, unknown> | undefined;
|
|
182
|
+
}
|
|
183
|
+
this.emit?.({ event: 'block_complete', index, block: mb });
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
71
187
|
export class Membrane {
|
|
72
188
|
private adapter: ProviderAdapter;
|
|
73
189
|
private registry?: ModelRegistry;
|
|
@@ -109,12 +225,20 @@ export class Membrane {
|
|
|
109
225
|
// refusal is a successful HTTP call with an unwanted verdict, and letting
|
|
110
226
|
// it consume error retries would couple two unrelated budgets.
|
|
111
227
|
let refusalRetriesUsed = 0;
|
|
228
|
+
// Spend on attempts we threw away. A refused attempt is a completed,
|
|
229
|
+
// billed HTTP call; reporting only the surviving attempt's usage
|
|
230
|
+
// under-reports the turn by one full call per retry.
|
|
231
|
+
let discardedUsage: DiscardedAttemptsUsage | undefined;
|
|
232
|
+
|
|
233
|
+
// One selection for the whole call: mode resolution and the build must
|
|
234
|
+
// name the same formatter instance (see resolveActiveFormatter).
|
|
235
|
+
const activeFormatter = this.resolveActiveFormatter(options.formatter);
|
|
112
236
|
|
|
113
237
|
while (true) {
|
|
114
238
|
attempts++;
|
|
115
239
|
|
|
116
240
|
try {
|
|
117
|
-
const { providerRequest, prefillResult } = this.transformRequest(request,
|
|
241
|
+
const { providerRequest, prefillResult } = this.transformRequest(request, activeFormatter);
|
|
118
242
|
|
|
119
243
|
// Route through the single canonical hook helper so any future
|
|
120
244
|
// change to hook semantics (logging, retry interaction, error
|
|
@@ -123,7 +247,16 @@ export class Membrane {
|
|
|
123
247
|
// `unknown` deliberately, and we acknowledge the cast at the boundary.
|
|
124
248
|
const finalRequest = (await this.applyBeforeRequestHook(request, providerRequest)) as typeof providerRequest;
|
|
125
249
|
|
|
126
|
-
|
|
250
|
+
// Last exit before the adapter: the only place that sees EVERY
|
|
251
|
+
// contribution (builder, formatter, passthrough, float, hook).
|
|
252
|
+
if (request.cacheMarkers === 'cm-owned') {
|
|
253
|
+
assertCacheMarkersWithinLimit(finalRequest, 'complete');
|
|
254
|
+
} else {
|
|
255
|
+
clampCacheMarkers(finalRequest, 'complete');
|
|
256
|
+
}
|
|
257
|
+
request.onCacheWireReceipt?.(computeCacheWireReceipt(finalRequest));
|
|
258
|
+
|
|
259
|
+
const rawProviderResponse = await this.adapter.complete(finalRequest, {
|
|
127
260
|
signal: options.signal,
|
|
128
261
|
timeoutMs: options.timeoutMs,
|
|
129
262
|
onRequest: (req) => {
|
|
@@ -131,6 +264,15 @@ export class Membrane {
|
|
|
131
264
|
options.onRequest?.(req);
|
|
132
265
|
},
|
|
133
266
|
});
|
|
267
|
+
// Restate usage in the one convention before any ratio or price sees it.
|
|
268
|
+
const providerResponse: ProviderResponse = {
|
|
269
|
+
...rawProviderResponse,
|
|
270
|
+
usage: normalizeUsageToCacheExcluded(
|
|
271
|
+
rawProviderResponse.usage,
|
|
272
|
+
this.adapter.name,
|
|
273
|
+
this.adapter.usageCacheConvention,
|
|
274
|
+
),
|
|
275
|
+
};
|
|
134
276
|
|
|
135
277
|
// Call onResponse callback with raw response from API
|
|
136
278
|
options.onResponse?.(providerResponse.raw);
|
|
@@ -154,9 +296,21 @@ export class Membrane {
|
|
|
154
296
|
refusalRetriesUsed < Math.max(0, options.refusalRetries ?? 0)
|
|
155
297
|
) {
|
|
156
298
|
refusalRetriesUsed++;
|
|
299
|
+
discardedUsage = this.mergeDiscardedAttempts(
|
|
300
|
+
discardedUsage,
|
|
301
|
+
this.discardedAttemptFrom(response.usage)
|
|
302
|
+
);
|
|
157
303
|
continue;
|
|
158
304
|
}
|
|
159
305
|
|
|
306
|
+
// Report what the discarded attempts cost. Set BEFORE afterResponse
|
|
307
|
+
// so a hook that logs spend sees the whole turn, not just the
|
|
308
|
+
// attempt that stands.
|
|
309
|
+
if (discardedUsage) {
|
|
310
|
+
response.details.usage.discardedAttempts =
|
|
311
|
+
this.pricedDiscardedAttempts(discardedUsage, request.config.model);
|
|
312
|
+
}
|
|
313
|
+
|
|
160
314
|
// Call afterResponse hook
|
|
161
315
|
if (this.config.hooks?.afterResponse) {
|
|
162
316
|
return await this.config.hooks.afterResponse(response, providerResponse.raw);
|
|
@@ -195,9 +349,16 @@ export class Membrane {
|
|
|
195
349
|
}
|
|
196
350
|
}
|
|
197
351
|
|
|
198
|
-
// Wait before retry (abort-aware)
|
|
352
|
+
// Wait before retry (abort-aware). An abort landing inside the
|
|
353
|
+
// sleep must fail like every other failure of this method — a
|
|
354
|
+
// MembraneError — rather than escaping the loop as a raw
|
|
355
|
+
// DOMException whose shape no caller of complete() expects.
|
|
199
356
|
const delay = this.calculateRetryDelay(attempts, isOverloaded);
|
|
200
|
-
|
|
357
|
+
try {
|
|
358
|
+
await this.sleep(delay, options.signal);
|
|
359
|
+
} catch (sleepError) {
|
|
360
|
+
throw this.attachRawRequest(sleepError, rawRequest);
|
|
361
|
+
}
|
|
201
362
|
continue;
|
|
202
363
|
}
|
|
203
364
|
|
|
@@ -231,6 +392,20 @@ export class Membrane {
|
|
|
231
392
|
// If streaming is explicitly disabled on the request, fall back to complete()
|
|
232
393
|
// and synthesize the streaming callbacks from the full response
|
|
233
394
|
if (request.streaming === false) {
|
|
395
|
+
// complete() has no tool loop, and neither branch of this fallback can
|
|
396
|
+
// build one: honouring onToolCalls here would mean re-implementing the
|
|
397
|
+
// whole XML/native continuation machinery. Silently dropping it turned
|
|
398
|
+
// a working agent into one that narrates tool calls it never makes —
|
|
399
|
+
// the raw <function_calls> XML lands in the returned text and the turn
|
|
400
|
+
// ends. Refuse where the option is passed, before spending a call.
|
|
401
|
+
if (options.onToolCalls) {
|
|
402
|
+
throw unsupportedError(
|
|
403
|
+
'stream() cannot execute tools with streaming: false — the non-streaming ' +
|
|
404
|
+
'fallback routes to complete(), which has no tool loop, so onToolCalls ' +
|
|
405
|
+
'would never run. Leave streaming enabled (or drive the loop yourself ' +
|
|
406
|
+
'with complete() per round).'
|
|
407
|
+
);
|
|
408
|
+
}
|
|
234
409
|
const response = await this.complete(request, options);
|
|
235
410
|
// Synthesize onChunk callbacks so callers that depend on them still work
|
|
236
411
|
if (options.onChunk && 'content' in response) {
|
|
@@ -248,8 +423,9 @@ export class Membrane {
|
|
|
248
423
|
return response;
|
|
249
424
|
}
|
|
250
425
|
|
|
251
|
-
// Determine tool mode
|
|
252
|
-
const
|
|
426
|
+
// Determine tool mode against the formatter that will build the request
|
|
427
|
+
const activeFormatter = this.resolveActiveFormatter(options.formatter);
|
|
428
|
+
const toolMode = this.resolveToolMode(request, activeFormatter);
|
|
253
429
|
const useNative = toolMode === 'native' && !!request.tools && request.tools.length > 0;
|
|
254
430
|
|
|
255
431
|
// Overloaded (529) pre-emission retry. The streaming paths have no retry
|
|
@@ -279,14 +455,16 @@ export class Membrane {
|
|
|
279
455
|
|
|
280
456
|
try {
|
|
281
457
|
const result = useNative
|
|
282
|
-
? await this.streamWithNativeTools(request, tracked)
|
|
283
|
-
: await this.streamWithXmlTools(request, tracked);
|
|
284
|
-
// The inner paths
|
|
285
|
-
//
|
|
286
|
-
//
|
|
287
|
-
//
|
|
458
|
+
? await this.streamWithNativeTools(request, tracked, activeFormatter)
|
|
459
|
+
: await this.streamWithXmlTools(request, tracked, activeFormatter);
|
|
460
|
+
// The inner paths count their own provider calls but cannot see this
|
|
461
|
+
// wrapper's discarded attempts. Each failed attempt here died before
|
|
462
|
+
// emitting anything (that is the precondition for retrying), so it
|
|
463
|
+
// cost at least the one call it failed on — ADD those to the inner
|
|
464
|
+
// count rather than overwriting it, or a turn that retried twice and
|
|
465
|
+
// then ran three tool rounds would report 2 calls instead of 5.
|
|
288
466
|
if (attempts > 1 && 'details' in result) {
|
|
289
|
-
result.details.timing.attempts
|
|
467
|
+
result.details.timing.attempts += attempts - 1;
|
|
290
468
|
result.details.timing.retryDelaysMs = retryDelaysMs;
|
|
291
469
|
}
|
|
292
470
|
return result;
|
|
@@ -313,7 +491,26 @@ export class Membrane {
|
|
|
313
491
|
}
|
|
314
492
|
const delay = this.calculateRetryDelay(attempts, true);
|
|
315
493
|
retryDelaysMs.push(delay);
|
|
316
|
-
|
|
494
|
+
// An abort during the backoff window is still a cancellation of
|
|
495
|
+
// this stream, and stream() documents cancellation as an
|
|
496
|
+
// AbortedResponse. Letting the sleep's rejection escape made that
|
|
497
|
+
// contract depend on which millisecond the abort landed in.
|
|
498
|
+
// Nothing has been emitted on this path (that is the precondition
|
|
499
|
+
// for retrying at all), so there is no partial content to report.
|
|
500
|
+
try {
|
|
501
|
+
await this.sleep(delay, options.signal);
|
|
502
|
+
} catch (sleepError) {
|
|
503
|
+
if (this.isAbortError(sleepError)) {
|
|
504
|
+
return this.buildAbortedResponse(
|
|
505
|
+
'',
|
|
506
|
+
{ inputTokens: 0, outputTokens: 0 },
|
|
507
|
+
[],
|
|
508
|
+
[],
|
|
509
|
+
this.abortReason(sleepError, options.signal)
|
|
510
|
+
);
|
|
511
|
+
}
|
|
512
|
+
throw sleepError;
|
|
513
|
+
}
|
|
317
514
|
continue;
|
|
318
515
|
}
|
|
319
516
|
throw error;
|
|
@@ -322,18 +519,68 @@ export class Membrane {
|
|
|
322
519
|
}
|
|
323
520
|
|
|
324
521
|
/**
|
|
325
|
-
*
|
|
522
|
+
* Select the ACTIVE formatter for a request: the one instance that resolves
|
|
523
|
+
* its tool mode, builds its provider request, and parses its stream.
|
|
524
|
+
*
|
|
525
|
+
* A per-request override (`CompleteOptions.formatter` /
|
|
526
|
+
* `StreamOptions.formatter`) wins over the instance formatter, with ONE
|
|
527
|
+
* transport exception: the Responses adapter's input is a provider-native
|
|
528
|
+
* item array, and a generic override (for example Context Manager's
|
|
529
|
+
* NativeFormatter) produces Anthropic-style `{ role, content: [{ type:
|
|
530
|
+
* 'text' }] }` envelopes the Responses API rejects before inference — so a
|
|
531
|
+
* configured Responses formatter stays authoritative there.
|
|
532
|
+
*
|
|
533
|
+
* The exception is why this selection is a method rather than a `??` at each
|
|
534
|
+
* call site: while it lived inside transformRequest alone, the BUILD honored
|
|
535
|
+
* it and every other formatter reader resolved against a different instance,
|
|
536
|
+
* which is the split resolveToolMode exists to prevent, one layer down.
|
|
537
|
+
* Every entry point selects once, here, and threads the result.
|
|
538
|
+
*/
|
|
539
|
+
private resolveActiveFormatter(requestFormatter?: PrefillFormatter): PrefillFormatter {
|
|
540
|
+
if (this.adapter.name === 'openai-responses-api' && this.formatter.name === 'openai-responses') {
|
|
541
|
+
return this.formatter;
|
|
542
|
+
}
|
|
543
|
+
return requestFormatter ?? this.formatter;
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
/**
|
|
547
|
+
* Determine the effective tool mode.
|
|
548
|
+
*
|
|
549
|
+
* THE single source of truth for the mode: both complete() (via
|
|
550
|
+
* transformRequest → BuildOptions.toolMode) and the streaming paths (via
|
|
551
|
+
* their native-vs-XML path choice) resolve here, so a given request resolves
|
|
552
|
+
* to the same mode whichever entry point it arrives through.
|
|
553
|
+
*
|
|
554
|
+
* Precedence, strongest first:
|
|
555
|
+
* 1. an explicit non-'auto' `request.toolMode`
|
|
556
|
+
* 2. the mode the BUILDING formatter was explicitly constructed with
|
|
557
|
+
* (`AnthropicXmlFormatter({ toolMode: 'native' })`) — a caller's stated
|
|
558
|
+
* choice, not a derivation
|
|
559
|
+
* 3. formatter/provider derivation
|
|
560
|
+
*
|
|
561
|
+
* `formatter` is the formatter that will actually build the request — the
|
|
562
|
+
* instance `resolveActiveFormatter` selected for this call — because
|
|
563
|
+
* resolving against one formatter while building with another is exactly the
|
|
564
|
+
* split this method exists to prevent.
|
|
326
565
|
*/
|
|
327
|
-
private resolveToolMode(
|
|
566
|
+
private resolveToolMode(
|
|
567
|
+
request: NormalizedRequest,
|
|
568
|
+
formatter: PrefillFormatter = this.formatter
|
|
569
|
+
): 'xml' | 'native' {
|
|
328
570
|
// Explicit mode takes precedence
|
|
329
571
|
if (request.toolMode && request.toolMode !== 'auto') {
|
|
330
572
|
return request.toolMode;
|
|
331
573
|
}
|
|
332
574
|
|
|
575
|
+
// A formatter constructed with an explicit mode states its caller's choice
|
|
576
|
+
if (formatter.configuredToolMode) {
|
|
577
|
+
return formatter.configuredToolMode;
|
|
578
|
+
}
|
|
579
|
+
|
|
333
580
|
// Auto mode: choose based on formatter
|
|
334
581
|
// NativeFormatter → native tools via API
|
|
335
582
|
// AnthropicXmlFormatter (default) → XML tools in prefill
|
|
336
|
-
if (
|
|
583
|
+
if (formatter.name === 'native' || formatter.name === 'openai-responses') {
|
|
337
584
|
return 'native';
|
|
338
585
|
}
|
|
339
586
|
|
|
@@ -355,7 +602,8 @@ export class Membrane {
|
|
|
355
602
|
*/
|
|
356
603
|
private async streamWithXmlTools(
|
|
357
604
|
request: NormalizedRequest,
|
|
358
|
-
options: StreamOptions
|
|
605
|
+
options: StreamOptions,
|
|
606
|
+
activeFormatter: PrefillFormatter = this.resolveActiveFormatter(options.formatter)
|
|
359
607
|
): Promise<NormalizedResponse | AbortedResponse> {
|
|
360
608
|
const startTime = Date.now();
|
|
361
609
|
const {
|
|
@@ -369,17 +617,30 @@ export class Membrane {
|
|
|
369
617
|
onResponse,
|
|
370
618
|
maxToolDepth = 10,
|
|
371
619
|
signal,
|
|
372
|
-
|
|
620
|
+
timeoutMs,
|
|
621
|
+
idleTimeoutMs,
|
|
373
622
|
} = options;
|
|
374
623
|
|
|
375
|
-
//
|
|
376
|
-
|
|
624
|
+
// The formatter stream() selected: the same instance that resolved the
|
|
625
|
+
// mode and will build the request, so the parser can never be reading a
|
|
626
|
+
// different format than the one on the wire.
|
|
627
|
+
const formatter = activeFormatter;
|
|
377
628
|
|
|
378
629
|
// Initialize parser from formatter for format-specific tracking
|
|
379
630
|
const parser = formatter.createStreamParser();
|
|
380
631
|
let toolDepth = 0;
|
|
381
|
-
|
|
382
|
-
|
|
632
|
+
// Each round is priced under the model that served THAT round and the
|
|
633
|
+
// costs are summed: a routed turn can change models mid-turn, and pricing
|
|
634
|
+
// the whole accumulated usage at the latest rate re-bills every earlier
|
|
635
|
+
// round at a price it was never charged.
|
|
636
|
+
const turnUsage = new TurnUsageAccumulator(
|
|
637
|
+
request.config.model,
|
|
638
|
+
(servedModel) => this.resolvePricing(request.config.model, servedModel),
|
|
639
|
+
);
|
|
640
|
+
// Honest turn telemetry: provider calls actually made (including refusal
|
|
641
|
+
// re-issues inside streamOnce) and continuation rounds.
|
|
642
|
+
let providerCalls = 0;
|
|
643
|
+
let rounds = 0;
|
|
383
644
|
const contentBlocks: ContentBlock[] = [];
|
|
384
645
|
let lastStopReason: StopReason = 'end_turn';
|
|
385
646
|
let lastStopSequence: string | undefined;
|
|
@@ -463,7 +724,7 @@ export class Membrane {
|
|
|
463
724
|
if (resumptionRounds === RESUMPTION_WARN_ROUNDS) {
|
|
464
725
|
warnLog.warn(
|
|
465
726
|
`[membrane] automatic resumption at round ${resumptionRounds} ` +
|
|
466
|
-
`(${
|
|
727
|
+
`(${turnUsage.total.inputTokens} input tokens so far this turn) — ` +
|
|
467
728
|
`a spin shows up here before it shows up on the bill`
|
|
468
729
|
);
|
|
469
730
|
}
|
|
@@ -471,7 +732,7 @@ export class Membrane {
|
|
|
471
732
|
warnLog.warn(
|
|
472
733
|
`[membrane] automatic resumption cap (${maxResumptionRounds}) reached — ` +
|
|
473
734
|
`ending turn with stopReason 'round_limit'. ` +
|
|
474
|
-
`${
|
|
735
|
+
`${turnUsage.total.inputTokens} input tokens spent this turn.`
|
|
475
736
|
);
|
|
476
737
|
return false;
|
|
477
738
|
}
|
|
@@ -547,6 +808,8 @@ export class Membrane {
|
|
|
547
808
|
},
|
|
548
809
|
{
|
|
549
810
|
signal,
|
|
811
|
+
timeoutMs,
|
|
812
|
+
idleTimeoutMs,
|
|
550
813
|
normalizedRequest: request,
|
|
551
814
|
// The tag-based parser tracks thinking via <thinking> tags — ask the
|
|
552
815
|
// provider to wrap native thinking deltas so they don't stream as
|
|
@@ -559,6 +822,9 @@ export class Membrane {
|
|
|
559
822
|
}
|
|
560
823
|
);
|
|
561
824
|
|
|
825
|
+
rounds++;
|
|
826
|
+
providerCalls += streamResult.providerCalls;
|
|
827
|
+
|
|
562
828
|
// If we detected stop sequence manually, fix up the parser and result
|
|
563
829
|
if (detectedStopSequence && truncatedAccumulated !== null) {
|
|
564
830
|
parser.reset();
|
|
@@ -593,17 +859,11 @@ export class Membrane {
|
|
|
593
859
|
lastStopReason = this.mapStopReason(streamResult.stopReason);
|
|
594
860
|
lastStopSequence = streamResult.stopSequence ?? undefined;
|
|
595
861
|
|
|
596
|
-
// Accumulate usage (including cache metrics)
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
}
|
|
602
|
-
if (streamResult.usage.cacheReadTokens) {
|
|
603
|
-
totalUsage.cacheReadTokens = (totalUsage.cacheReadTokens ?? 0) + streamResult.usage.cacheReadTokens;
|
|
604
|
-
}
|
|
605
|
-
if (pricing) totalUsage.estimatedCost = calculateCost(totalUsage, pricing);
|
|
606
|
-
onUsage?.(totalUsage);
|
|
862
|
+
// Accumulate usage (including cache metrics), priced at this round's
|
|
863
|
+
// model. NOT inlined into the optional call — `onUsage?.(addRound())`
|
|
864
|
+
// skips evaluating its argument entirely when no callback is set.
|
|
865
|
+
const usageSoFar = turnUsage.addRound(streamResult.model, streamResult.usage);
|
|
866
|
+
onUsage?.(usageSoFar);
|
|
607
867
|
|
|
608
868
|
// Flush the parser to complete any in-progress streaming block
|
|
609
869
|
const flushResult = parser.flush();
|
|
@@ -637,7 +897,7 @@ export class Membrane {
|
|
|
637
897
|
`[membrane] ${consecutiveStalledResumptions} consecutive automatic resumptions ` +
|
|
638
898
|
`made no progress (${streamedThisRound} chars this round, stop ` +
|
|
639
899
|
`${JSON.stringify(lastStopSequence ?? null)} repeated) — ending turn with ` +
|
|
640
|
-
`stopReason 'no_progress'. ${
|
|
900
|
+
`stopReason 'no_progress'. ${turnUsage.total.inputTokens} input tokens spent this turn.`
|
|
641
901
|
);
|
|
642
902
|
lastStopReason = 'no_progress';
|
|
643
903
|
break;
|
|
@@ -919,17 +1179,17 @@ export class Membrane {
|
|
|
919
1179
|
newContent,
|
|
920
1180
|
contentBlocks,
|
|
921
1181
|
lastStopReason,
|
|
922
|
-
|
|
1182
|
+
turnUsage,
|
|
923
1183
|
request,
|
|
924
1184
|
prefillResult,
|
|
925
1185
|
startTime,
|
|
926
|
-
|
|
1186
|
+
providerCalls,
|
|
927
1187
|
rawRequest,
|
|
928
1188
|
rawResponse,
|
|
929
1189
|
executedToolCalls,
|
|
930
1190
|
executedToolResults,
|
|
931
1191
|
initialBlockType,
|
|
932
|
-
lastStopSequence
|
|
1192
|
+
lastStopSequence,
|
|
933
1193
|
);
|
|
934
1194
|
|
|
935
1195
|
// Append non-text content blocks (e.g., generated_image) that the XML parser can't handle
|
|
@@ -940,6 +1200,8 @@ export class Membrane {
|
|
|
940
1200
|
// Merge provider thinking signatures into parser-derived thinking blocks
|
|
941
1201
|
this.mergeProviderThinkingBlocks(response.content, providerThinkingBlocks);
|
|
942
1202
|
|
|
1203
|
+
response.details.timing.rounds = rounds;
|
|
1204
|
+
|
|
943
1205
|
return response;
|
|
944
1206
|
} catch (error) {
|
|
945
1207
|
// Check if this is an abort error
|
|
@@ -950,10 +1212,10 @@ export class Membrane {
|
|
|
950
1212
|
|
|
951
1213
|
return this.buildAbortedResponse(
|
|
952
1214
|
newContent,
|
|
953
|
-
|
|
1215
|
+
turnUsage.total,
|
|
954
1216
|
executedToolCalls,
|
|
955
1217
|
executedToolResults,
|
|
956
|
-
|
|
1218
|
+
this.abortReason(error, signal),
|
|
957
1219
|
initialBlockType
|
|
958
1220
|
);
|
|
959
1221
|
}
|
|
@@ -967,12 +1229,14 @@ export class Membrane {
|
|
|
967
1229
|
*/
|
|
968
1230
|
private async streamWithNativeTools(
|
|
969
1231
|
request: NormalizedRequest,
|
|
970
|
-
options: StreamOptions
|
|
1232
|
+
options: StreamOptions,
|
|
1233
|
+
activeFormatter: PrefillFormatter = this.resolveActiveFormatter(options.formatter)
|
|
971
1234
|
): Promise<NormalizedResponse | AbortedResponse> {
|
|
972
1235
|
const startTime = Date.now();
|
|
973
1236
|
const {
|
|
974
1237
|
onChunk,
|
|
975
1238
|
onContentBlockUpdate,
|
|
1239
|
+
onBlock,
|
|
976
1240
|
onToolCalls,
|
|
977
1241
|
onPreToolContent,
|
|
978
1242
|
onUsage,
|
|
@@ -980,11 +1244,21 @@ export class Membrane {
|
|
|
980
1244
|
onResponse,
|
|
981
1245
|
maxToolDepth = 10,
|
|
982
1246
|
signal,
|
|
1247
|
+
timeoutMs,
|
|
1248
|
+
idleTimeoutMs,
|
|
983
1249
|
} = options;
|
|
984
1250
|
|
|
985
1251
|
let toolDepth = 0;
|
|
986
|
-
|
|
987
|
-
|
|
1252
|
+
// See streamWithXmlTools: one accumulator per turn, pricing each round
|
|
1253
|
+
// under the model that served it.
|
|
1254
|
+
const turnUsage = new TurnUsageAccumulator(
|
|
1255
|
+
request.config.model,
|
|
1256
|
+
(servedModel) => this.resolvePricing(request.config.model, servedModel),
|
|
1257
|
+
);
|
|
1258
|
+
// Honest turn telemetry: provider calls actually made (including refusal
|
|
1259
|
+
// re-issues inside streamOnce) and continuation rounds.
|
|
1260
|
+
let providerCalls = 0;
|
|
1261
|
+
let rounds = 0;
|
|
988
1262
|
let lastStopReason: StopReason = 'end_turn';
|
|
989
1263
|
let lastStopSequence: string | undefined;
|
|
990
1264
|
let rawRequest: unknown;
|
|
@@ -1000,45 +1274,67 @@ export class Membrane {
|
|
|
1000
1274
|
// Build messages array that we'll update with tool results
|
|
1001
1275
|
let messages = [...request.messages];
|
|
1002
1276
|
let allContentBlocks: ContentBlock[] = [];
|
|
1277
|
+
let markersInLastRequest = 0;
|
|
1003
1278
|
|
|
1004
1279
|
try {
|
|
1005
1280
|
// Tool execution loop
|
|
1006
1281
|
while (toolDepth <= maxToolDepth) {
|
|
1007
1282
|
// Build provider request with native tools
|
|
1008
|
-
const providerRequest = this.buildNativeToolRequest(request, messages, toolDepth > 0);
|
|
1283
|
+
const providerRequest = this.buildNativeToolRequest(request, messages, toolDepth > 0, activeFormatter);
|
|
1009
1284
|
|
|
1010
1285
|
// Stream from provider
|
|
1011
1286
|
let textAccumulated = '';
|
|
1012
|
-
|
|
1287
|
+
// Tag every token chunk with the membrane block it belongs to and
|
|
1288
|
+
// surface the block lifecycle through onBlock — the same shape
|
|
1289
|
+
// runNativeToolsYielding uses (#19). Before this, meta.type was
|
|
1290
|
+
// hardcoded to 'text' on every chunk and onBlock was never invoked
|
|
1291
|
+
// from this path (#20).
|
|
1292
|
+
const tracker = new NativeBlockTracker(onBlock ? (event) => onBlock(event) : undefined);
|
|
1013
1293
|
const streamResult = await this.streamOnce(
|
|
1014
1294
|
providerRequest,
|
|
1015
1295
|
{
|
|
1016
1296
|
onChunk: (chunk) => {
|
|
1017
1297
|
textAccumulated += chunk;
|
|
1018
1298
|
allTextAccumulated += chunk;
|
|
1019
|
-
// For native mode, emit text chunks with basic metadata
|
|
1020
|
-
// TODO: Use native API content_block events for richer metadata
|
|
1021
1299
|
const meta: ChunkMeta = {
|
|
1022
|
-
type:
|
|
1023
|
-
visible:
|
|
1024
|
-
blockIndex,
|
|
1300
|
+
type: tracker.currentType,
|
|
1301
|
+
visible: tracker.currentType === 'text',
|
|
1302
|
+
blockIndex: tracker.blockIndex,
|
|
1025
1303
|
};
|
|
1026
1304
|
onChunk?.(chunk, meta);
|
|
1027
1305
|
},
|
|
1028
|
-
onContentBlock:
|
|
1029
|
-
|
|
1030
|
-
|
|
1306
|
+
onContentBlock: (index: number, block: unknown) => {
|
|
1307
|
+
tracker.onProviderBlock(index, block);
|
|
1308
|
+
// Deprecated pass-through, kept for callers still on it.
|
|
1309
|
+
onContentBlockUpdate?.(index, block as ContentBlock);
|
|
1310
|
+
},
|
|
1031
1311
|
},
|
|
1032
1312
|
{
|
|
1033
1313
|
signal,
|
|
1314
|
+
timeoutMs,
|
|
1315
|
+
idleTimeoutMs,
|
|
1034
1316
|
normalizedRequest: request,
|
|
1035
1317
|
onRequest: (req) => {
|
|
1036
1318
|
rawRequest = req;
|
|
1037
1319
|
onRequest?.(req);
|
|
1038
1320
|
},
|
|
1321
|
+
// Telemetry reports what this request actually SHIPPED with —
|
|
1322
|
+
// builder breakpoints, stale passthrough, fallback, float, plus
|
|
1323
|
+
// whatever the beforeRequest hook and the wire clamp did after
|
|
1324
|
+
// the build. Both native paths used to hardcode 0, and counting
|
|
1325
|
+
// at build time reported a number no request ever had.
|
|
1326
|
+
onWireCacheMarkers: (markerCount) => {
|
|
1327
|
+
markersInLastRequest = markerCount;
|
|
1328
|
+
},
|
|
1039
1329
|
}
|
|
1040
1330
|
);
|
|
1041
1331
|
|
|
1332
|
+
// Single-callback adapters (OpenAI Responses) report each finalised
|
|
1333
|
+
// block once, after the stream: complete whatever never saw a stop.
|
|
1334
|
+
tracker.flush();
|
|
1335
|
+
rounds++;
|
|
1336
|
+
providerCalls += streamResult.providerCalls;
|
|
1337
|
+
|
|
1042
1338
|
rawResponse = streamResult.raw;
|
|
1043
1339
|
|
|
1044
1340
|
// Call onResponse callback with raw response from API
|
|
@@ -1047,17 +1343,11 @@ export class Membrane {
|
|
|
1047
1343
|
lastStopReason = this.mapStopReason(streamResult.stopReason);
|
|
1048
1344
|
lastStopSequence = streamResult.stopSequence ?? undefined;
|
|
1049
1345
|
|
|
1050
|
-
// Accumulate usage (including cache metrics)
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
}
|
|
1056
|
-
if (streamResult.usage.cacheReadTokens) {
|
|
1057
|
-
totalUsage.cacheReadTokens = (totalUsage.cacheReadTokens ?? 0) + streamResult.usage.cacheReadTokens;
|
|
1058
|
-
}
|
|
1059
|
-
if (pricing) totalUsage.estimatedCost = calculateCost(totalUsage, pricing);
|
|
1060
|
-
onUsage?.(totalUsage);
|
|
1346
|
+
// Accumulate usage (including cache metrics), priced at this round's
|
|
1347
|
+
// model. NOT inlined into the optional call — `onUsage?.(addRound())`
|
|
1348
|
+
// skips evaluating its argument entirely when no callback is set.
|
|
1349
|
+
const usageSoFar = turnUsage.addRound(streamResult.model, streamResult.usage);
|
|
1350
|
+
onUsage?.(usageSoFar);
|
|
1061
1351
|
|
|
1062
1352
|
// Parse content blocks from response
|
|
1063
1353
|
const responseBlocks = this.parseProviderContent(streamResult.content);
|
|
@@ -1145,6 +1435,7 @@ export class Membrane {
|
|
|
1145
1435
|
}
|
|
1146
1436
|
|
|
1147
1437
|
const durationMs = Date.now() - startTime;
|
|
1438
|
+
const totalUsage = turnUsage.total;
|
|
1148
1439
|
|
|
1149
1440
|
return {
|
|
1150
1441
|
content: allContentBlocks,
|
|
@@ -1162,15 +1453,17 @@ export class Membrane {
|
|
|
1162
1453
|
usage: { ...totalUsage },
|
|
1163
1454
|
timing: {
|
|
1164
1455
|
totalDurationMs: durationMs,
|
|
1165
|
-
attempts:
|
|
1456
|
+
attempts: providerCalls,
|
|
1457
|
+
rounds,
|
|
1166
1458
|
},
|
|
1167
1459
|
model: {
|
|
1168
1460
|
requested: request.config.model,
|
|
1169
|
-
actual: request.config.model,
|
|
1461
|
+
actual: turnUsage.lastServedModel || request.config.model,
|
|
1170
1462
|
provider: this.adapter.name,
|
|
1463
|
+
perRound: turnUsage.perRound,
|
|
1171
1464
|
},
|
|
1172
1465
|
cache: {
|
|
1173
|
-
markersInRequest:
|
|
1466
|
+
markersInRequest: markersInLastRequest,
|
|
1174
1467
|
tokensCreated: totalUsage.cacheCreationTokens ?? 0,
|
|
1175
1468
|
tokensRead: totalUsage.cacheReadTokens ?? 0,
|
|
1176
1469
|
hitRatio: this.calculateCacheHitRatio(totalUsage),
|
|
@@ -1186,10 +1479,10 @@ export class Membrane {
|
|
|
1186
1479
|
if (this.isAbortError(error)) {
|
|
1187
1480
|
return this.buildAbortedResponse(
|
|
1188
1481
|
allTextAccumulated,
|
|
1189
|
-
|
|
1482
|
+
turnUsage.total,
|
|
1190
1483
|
executedToolCalls,
|
|
1191
1484
|
executedToolResults,
|
|
1192
|
-
|
|
1485
|
+
this.abortReason(error, signal)
|
|
1193
1486
|
);
|
|
1194
1487
|
}
|
|
1195
1488
|
// Re-throw with rawRequest attached for logging
|
|
@@ -1197,8 +1490,37 @@ export class Membrane {
|
|
|
1197
1490
|
}
|
|
1198
1491
|
}
|
|
1199
1492
|
|
|
1200
|
-
/**
|
|
1201
|
-
|
|
1493
|
+
/**
|
|
1494
|
+
* Rate-limit state for the float's budget warning. See the
|
|
1495
|
+
* floating-cache-marker block in buildNativeToolRequest.
|
|
1496
|
+
*
|
|
1497
|
+
* A once-per-instance latch made the ONLY observable of an over-budget wire
|
|
1498
|
+
* go quiet for the life of the process: a long-lived Membrane warns for the
|
|
1499
|
+
* first agent that trips it and never again, so the condition looks like it
|
|
1500
|
+
* healed. Warn on the first occurrence, then at most once per interval,
|
|
1501
|
+
* carrying the count of what was suppressed in between.
|
|
1502
|
+
*/
|
|
1503
|
+
private floatBudgetWarnState = { lastWarnedAtMs: 0, suppressedSinceWarn: 0 };
|
|
1504
|
+
private static readonly FLOAT_BUDGET_WARN_INTERVAL_MS = 60_000;
|
|
1505
|
+
|
|
1506
|
+
private warnFloatBudgetExhausted(wireMarkers: number): void {
|
|
1507
|
+
const now = Date.now();
|
|
1508
|
+
const state = this.floatBudgetWarnState;
|
|
1509
|
+
const elapsed = now - state.lastWarnedAtMs;
|
|
1510
|
+
if (state.lastWarnedAtMs !== 0 && elapsed < Membrane.FLOAT_BUDGET_WARN_INTERVAL_MS) {
|
|
1511
|
+
state.suppressedSinceWarn++;
|
|
1512
|
+
return;
|
|
1513
|
+
}
|
|
1514
|
+
const suppressed = state.suppressedSinceWarn;
|
|
1515
|
+
state.lastWarnedAtMs = now;
|
|
1516
|
+
state.suppressedSinceWarn = 0;
|
|
1517
|
+
console.warn(
|
|
1518
|
+
`[membrane] floating cache marker withheld: upstream markers already ` +
|
|
1519
|
+
`occupy all ${MAX_CACHE_BREAKPOINTS} cache_control slots (${wireMarkers} on the wire). ` +
|
|
1520
|
+
`Tool-round suffixes will not cache incrementally.` +
|
|
1521
|
+
(suppressed > 0 ? ` (${suppressed} further occurrences suppressed since the last warning.)` : '')
|
|
1522
|
+
);
|
|
1523
|
+
}
|
|
1202
1524
|
|
|
1203
1525
|
/**
|
|
1204
1526
|
* Build a provider request with native tool support.
|
|
@@ -1206,18 +1528,25 @@ export class Membrane {
|
|
|
1206
1528
|
* `toolLoopRebuild` is true when this build is a tool-loop continuation
|
|
1207
1529
|
* (toolDepth > 0) rather than the turn's first request — the only case
|
|
1208
1530
|
* where the floating cache marker applies.
|
|
1531
|
+
*
|
|
1532
|
+
* `activeFormatter` is the formatter the caller selected for the request
|
|
1533
|
+
* (see resolveActiveFormatter). Reading `this.formatter` here instead made
|
|
1534
|
+
* the native loop build through the instance formatter while the mode had
|
|
1535
|
+
* been resolved against a per-request override — the two disagreeing about
|
|
1536
|
+
* which formatter is active.
|
|
1209
1537
|
*/
|
|
1210
1538
|
private buildNativeToolRequest(
|
|
1211
1539
|
request: NormalizedRequest,
|
|
1212
1540
|
messages: typeof request.messages,
|
|
1213
|
-
toolLoopRebuild = false
|
|
1541
|
+
toolLoopRebuild = false,
|
|
1542
|
+
activeFormatter: PrefillFormatter = this.formatter
|
|
1214
1543
|
): any {
|
|
1215
1544
|
// Provider-native formatters own their complete input-item shape. The
|
|
1216
1545
|
// legacy implementation below is intentionally Anthropic-specific; using
|
|
1217
1546
|
// it for Responses would normalize away item IDs, encrypted reasoning,
|
|
1218
1547
|
// assistant phases, and compaction items.
|
|
1219
|
-
if (
|
|
1220
|
-
return this.transformRequest({ ...request, messages },
|
|
1548
|
+
if (activeFormatter.name === 'openai-responses') {
|
|
1549
|
+
return this.transformRequest({ ...request, messages }, activeFormatter).providerRequest;
|
|
1221
1550
|
}
|
|
1222
1551
|
|
|
1223
1552
|
// Convert messages to provider format
|
|
@@ -1232,11 +1561,13 @@ export class Membrane {
|
|
|
1232
1561
|
// Anthropic allows at most 4 cache_control breakpoints per request. The
|
|
1233
1562
|
// message breakpoints are the valuable ones (they cache the longest prefixes,
|
|
1234
1563
|
// and every one already includes tools+system at the front of the request).
|
|
1235
|
-
// So tools/system get a breakpoint only as a FALLBACK — when no
|
|
1236
|
-
//
|
|
1237
|
-
// total past 4, which the API hard-rejects (the agent goes
|
|
1238
|
-
|
|
1239
|
-
|
|
1564
|
+
// So tools/system get a breakpoint only as a FALLBACK — when no marker
|
|
1565
|
+
// exists anywhere on the wire — otherwise they're redundant and would push
|
|
1566
|
+
// the total past 4, which the API hard-rejects (the agent goes
|
|
1567
|
+
// unresponsive). The fallback gate reads a RECOUNT of the built artifacts
|
|
1568
|
+
// (see below), never a running tally: a running tally cannot see a
|
|
1569
|
+
// caller-marked system block, and double-counts a message breakpoint that
|
|
1570
|
+
// lands on a block already carrying stale cache_control.
|
|
1240
1571
|
for (const msg of messages) {
|
|
1241
1572
|
const isAssistant = msg.participant === assistantName;
|
|
1242
1573
|
const role = isAssistant ? 'assistant' : 'user';
|
|
@@ -1257,14 +1588,13 @@ export class Membrane {
|
|
|
1257
1588
|
}
|
|
1258
1589
|
const textBlock: Record<string, unknown> = { type: 'text', text };
|
|
1259
1590
|
if ((block as any).cache_control) {
|
|
1260
|
-
textBlock.cache_control = (block as any).cache_control;
|
|
1261
1591
|
// A block-level passthrough occupies one of the 4 breakpoint slots
|
|
1262
|
-
// exactly like a marked message
|
|
1263
|
-
//
|
|
1264
|
-
//
|
|
1265
|
-
//
|
|
1266
|
-
//
|
|
1267
|
-
|
|
1592
|
+
// exactly like a marked message; the recount below sees it.
|
|
1593
|
+
// (Imported/seeded conversations carry stale request-time
|
|
1594
|
+
// cache_control on stored blocks — first seen wedging Sill
|
|
1595
|
+
// 2026-07-25: 3 cm markers + 2 stale Arc-export blocks = 5 → hard
|
|
1596
|
+
// 400 on every inference.)
|
|
1597
|
+
textBlock.cache_control = (block as any).cache_control;
|
|
1268
1598
|
}
|
|
1269
1599
|
content.push(textBlock);
|
|
1270
1600
|
} else if (block.type === 'tool_use') {
|
|
@@ -1336,7 +1666,6 @@ export class Membrane {
|
|
|
1336
1666
|
const bpIdx = lastCacheableBlockIndex(content as Array<Record<string, unknown>>);
|
|
1337
1667
|
if (bpIdx >= 0) {
|
|
1338
1668
|
content[bpIdx].cache_control = cacheControl;
|
|
1339
|
-
messageBreakpoints++;
|
|
1340
1669
|
}
|
|
1341
1670
|
}
|
|
1342
1671
|
|
|
@@ -1367,14 +1696,32 @@ export class Membrane {
|
|
|
1367
1696
|
// past one. `synthetic_pending_result` (not the downstream
|
|
1368
1697
|
// cache_suppressed_for_synthetic, which only fires when a marker was
|
|
1369
1698
|
// actually stripped) is the root condition.
|
|
1370
|
-
|
|
1699
|
+
// Every repair that REWRITES prefix bytes stands the float down, not just
|
|
1700
|
+
// the synthetic [pending] result: a textified orphan tool_result is
|
|
1701
|
+
// rewritten the same way when its real pairing arrives, so caching at or
|
|
1702
|
+
// past one poisons the prefix identically. The kinds live in one exported
|
|
1703
|
+
// set so a normalizer that grows a new prefix-rewriting repair cannot
|
|
1704
|
+
// silently escape this guard.
|
|
1705
|
+
let prefixRewritten = false;
|
|
1371
1706
|
const normalized = normalizeToolPairs(providerMessages, {
|
|
1372
1707
|
onEvent: (e) => {
|
|
1373
|
-
if (e.kind
|
|
1708
|
+
if (PREFIX_REWRITING_NORMALIZE_EVENT_KINDS.has(e.kind)) prefixRewritten = true;
|
|
1374
1709
|
},
|
|
1375
1710
|
});
|
|
1376
1711
|
const mergedMessages = mergeConsecutiveRoles(normalized.messages);
|
|
1377
1712
|
|
|
1713
|
+
// ONE recount of the constructed wire artifacts, taken BEFORE the
|
|
1714
|
+
// tools/system fallback decision so the fallback and the float share a
|
|
1715
|
+
// single truth. Counted post-normalize, so phase-5.5 cache suppression is
|
|
1716
|
+
// already reflected. `request.system` is the caller's own system content:
|
|
1717
|
+
// it explicitly accepts pre-marked blocks, and those are real wire markers
|
|
1718
|
+
// that no running tally ever saw (three of them plus both fallbacks = 5 on
|
|
1719
|
+
// the wire = a 400 on every inference of that config).
|
|
1720
|
+
const upstreamWireMarkers = countWireCacheMarkers({
|
|
1721
|
+
messages: mergedMessages,
|
|
1722
|
+
system: request.system,
|
|
1723
|
+
});
|
|
1724
|
+
|
|
1378
1725
|
// Convert tools to provider format.
|
|
1379
1726
|
// Native tool names must match ^[a-zA-Z0-9_-]{1,128}$ — sanitize colons
|
|
1380
1727
|
// from the module:tool namespace convention. Reversed in parseProviderContent.
|
|
@@ -1386,7 +1733,7 @@ export class Membrane {
|
|
|
1386
1733
|
};
|
|
1387
1734
|
// Cache the tool list (last tool) only as a fallback — a marked message
|
|
1388
1735
|
// breakpoint already caches the tools as part of its prefix.
|
|
1389
|
-
if (cacheControl &&
|
|
1736
|
+
if (cacheControl && upstreamWireMarkers === 0 && request.tools && idx === request.tools.length - 1) {
|
|
1390
1737
|
t.cache_control = cacheControl;
|
|
1391
1738
|
}
|
|
1392
1739
|
return t;
|
|
@@ -1395,10 +1742,10 @@ export class Membrane {
|
|
|
1395
1742
|
// Wrap system prompt with cache_control only as a fallback (no message
|
|
1396
1743
|
// breakpoint marked); otherwise a message breakpoint already caches
|
|
1397
1744
|
// tools+system as part of its prefix.
|
|
1398
|
-
let system: unknown = request.system;
|
|
1399
|
-
if (cacheControl &&
|
|
1745
|
+
let system: unknown = ownSystemBlocks(request.system);
|
|
1746
|
+
if (cacheControl && upstreamWireMarkers === 0 && typeof system === 'string' && system.length > 0) {
|
|
1400
1747
|
system = [{ type: 'text', text: system, cache_control: cacheControl }];
|
|
1401
|
-
} else if (cacheControl &&
|
|
1748
|
+
} else if (cacheControl && upstreamWireMarkers === 0 && Array.isArray(system) && system.length > 0) {
|
|
1402
1749
|
const blocks = system as Record<string, unknown>[];
|
|
1403
1750
|
system = blocks.map((block, idx) =>
|
|
1404
1751
|
idx === blocks.length - 1 ? { ...block, cache_control: cacheControl } : block
|
|
@@ -1437,39 +1784,13 @@ export class Membrane {
|
|
|
1437
1784
|
// ------------------------------------------------------------------
|
|
1438
1785
|
const floatingEnabled =
|
|
1439
1786
|
request.floatingCacheMarker ?? this.config.defaultFloatingCacheMarker ?? true;
|
|
1440
|
-
if (toolLoopRebuild && floatingEnabled && cacheControl && !
|
|
1441
|
-
//
|
|
1442
|
-
//
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
// landing on a block already carrying stale cache_control is one
|
|
1446
|
-
// physical marker counted twice, and a pre-marked system block is a
|
|
1447
|
-
// real wire marker the tally never sees. Counted post-fallback and
|
|
1448
|
-
// post-normalize, so fallback spend and phase-5.5 suppression are
|
|
1449
|
-
// both reflected.
|
|
1450
|
-
let wireMarkers = 0;
|
|
1451
|
-
for (const m of mergedMessages) {
|
|
1452
|
-
if (!Array.isArray(m.content)) continue;
|
|
1453
|
-
for (const b of m.content as Array<Record<string, unknown>>) {
|
|
1454
|
-
if (b.cache_control) wireMarkers++;
|
|
1455
|
-
}
|
|
1456
|
-
}
|
|
1457
|
-
if (tools) for (const t of tools) { if (t.cache_control) wireMarkers++; }
|
|
1458
|
-
if (Array.isArray(system)) {
|
|
1459
|
-
for (const b of system as Array<Record<string, unknown>>) {
|
|
1460
|
-
if (b.cache_control) wireMarkers++;
|
|
1461
|
-
}
|
|
1462
|
-
}
|
|
1463
|
-
let residuum = 4 - wireMarkers;
|
|
1787
|
+
if (toolLoopRebuild && floatingEnabled && cacheControl && !prefixRewritten) {
|
|
1788
|
+
// Same recount as the fallback gate, re-taken POST-fallback so the
|
|
1789
|
+
// fallback's own spend is inside the residuum.
|
|
1790
|
+
const wireMarkers = countWireCacheMarkers({ messages: mergedMessages, system, tools });
|
|
1791
|
+
let residuum = MAX_CACHE_BREAKPOINTS - wireMarkers;
|
|
1464
1792
|
if (residuum <= 0) {
|
|
1465
|
-
|
|
1466
|
-
this.floatBudgetWarned = true;
|
|
1467
|
-
console.warn(
|
|
1468
|
-
`[membrane] floating cache marker withheld: upstream markers already ` +
|
|
1469
|
-
`occupy all 4 cache_control slots (${wireMarkers} on the wire). ` +
|
|
1470
|
-
`Tool-round suffixes will not cache incrementally.`
|
|
1471
|
-
);
|
|
1472
|
-
}
|
|
1793
|
+
this.warnFloatBudgetExhausted(wireMarkers);
|
|
1473
1794
|
} else {
|
|
1474
1795
|
// Newest message first; then the previous round's endpoint (two
|
|
1475
1796
|
// wire messages back: [..., prevResults, assistant, results]).
|
|
@@ -1538,6 +1859,9 @@ export class Membrane {
|
|
|
1538
1859
|
id: item.id,
|
|
1539
1860
|
name: unsanitizeToolName(item.name),
|
|
1540
1861
|
input: item.input,
|
|
1862
|
+
// Arguments that never parsed: carry the marker through so a
|
|
1863
|
+
// consumer can refuse the block instead of trusting `input`.
|
|
1864
|
+
...(item.unparseableInput !== undefined ? { unparseableInput: item.unparseableInput } : {}),
|
|
1541
1865
|
...(item.rawItem ? { rawItem: item.rawItem } : {}),
|
|
1542
1866
|
});
|
|
1543
1867
|
} else if (item.type === 'thinking') {
|
|
@@ -1556,14 +1880,23 @@ export class Membrane {
|
|
|
1556
1880
|
data: item.data,
|
|
1557
1881
|
mimeType: item.mimeType,
|
|
1558
1882
|
});
|
|
1559
|
-
} else if (item.rawItem) {
|
|
1883
|
+
} else if (item.rawItem || item.type) {
|
|
1560
1884
|
// Opaque Responses items such as encrypted compaction or custom
|
|
1561
1885
|
// tool records have no normalized ContentBlock equivalent. Retain a
|
|
1562
1886
|
// zero-width carrier so Chronicle and the Responses formatter can
|
|
1563
1887
|
// replay the raw item without surfacing synthetic prompt text.
|
|
1564
1888
|
// Anthropic-bound conversion paths filter these out (empty text
|
|
1565
1889
|
// blocks are a 400 there); the Responses formatter replays rawItem.
|
|
1566
|
-
|
|
1890
|
+
//
|
|
1891
|
+
// An item with a `type` this switch does not know (server_tool_use,
|
|
1892
|
+
// web_search_tool_result, search_result, mcp_tool_use, or whatever a
|
|
1893
|
+
// provider adds next) used to fall out of this chain and vanish. It
|
|
1894
|
+
// gets the same carrier treatment, holding the item itself, plus a
|
|
1895
|
+
// one-time warning so the gap surfaces instead of being inferred
|
|
1896
|
+
// later from missing content.
|
|
1897
|
+
const carriedRawItem = item.rawItem ?? item;
|
|
1898
|
+
if (!item.rawItem) warnUnconvertibleProviderItem(item.type);
|
|
1899
|
+
blocks.push({ type: 'text', text: '', rawItem: carriedRawItem });
|
|
1567
1900
|
}
|
|
1568
1901
|
}
|
|
1569
1902
|
return blocks;
|
|
@@ -1602,10 +1935,32 @@ export class Membrane {
|
|
|
1602
1935
|
|
|
1603
1936
|
/**
|
|
1604
1937
|
* Merge provider thinking signatures into parser-derived thinking blocks
|
|
1605
|
-
*
|
|
1606
|
-
*
|
|
1607
|
-
*
|
|
1608
|
-
*
|
|
1938
|
+
* and prepend any leftover provider blocks — signature-only thinking
|
|
1939
|
+
* (display:'omitted') never appears in the text stream, so the parser
|
|
1940
|
+
* produces no block for it. redacted_thinking blocks are always prepended
|
|
1941
|
+
* verbatim.
|
|
1942
|
+
*
|
|
1943
|
+
* Pairing is by CONTENT IDENTITY, never by index. The two lists are
|
|
1944
|
+
* differently shaped whenever the provider emits a block the parser cannot
|
|
1945
|
+
* see (signature-only), the parser emits a block the provider never
|
|
1946
|
+
* produced (the XML path's literal `Claude: <thinking>` prefill turns
|
|
1947
|
+
* VISIBLE text into a thinking block), or one provider block spans several
|
|
1948
|
+
* (auto-continuation: capture runs per round while the parser sees the
|
|
1949
|
+
* CONCATENATED accumulation). Index-zipping crosses the lists in all three
|
|
1950
|
+
* shapes and stamps a signature onto content that never produced it —
|
|
1951
|
+
* which round-trips into the consumer's stored history and fails Anthropic
|
|
1952
|
+
* signature validation on the next turn.
|
|
1953
|
+
*
|
|
1954
|
+
* The three rules, in order:
|
|
1955
|
+
* 1. identity — a provider block pairs with the parsed block whose
|
|
1956
|
+
* thinking text is the same; empty-thinking (signature-only) blocks
|
|
1957
|
+
* are never text-match candidates and are prepend-only.
|
|
1958
|
+
* 2. span — a parsed block that reconstructs as the concatenation of a
|
|
1959
|
+
* RUN of consecutive unpaired provider blocks is REPLACED in place by
|
|
1960
|
+
* those originals, so the spanning block never wears a fragment's
|
|
1961
|
+
* signature and no reasoning is sent twice.
|
|
1962
|
+
* 3. leftover — everything still unpaired is prepended, de-duplicated
|
|
1963
|
+
* against what `content` already carries (and against itself).
|
|
1609
1964
|
*
|
|
1610
1965
|
* Mutates `content` in place. Shared by the XML stream paths
|
|
1611
1966
|
* (streamWithXmlTools and runXmlToolsYielding).
|
|
@@ -1616,25 +1971,74 @@ export class Membrane {
|
|
|
1616
1971
|
): void {
|
|
1617
1972
|
if (providerThinkingBlocks.length === 0) return;
|
|
1618
1973
|
|
|
1619
|
-
const
|
|
1974
|
+
const providerThinking = providerThinkingBlocks.filter(
|
|
1620
1975
|
(b) => b.type === 'thinking'
|
|
1621
|
-
) as Array<{ type: 'thinking'; thinking
|
|
1622
|
-
|
|
1623
|
-
const providerThinking = providerThinkingBlocks.filter((b) => b.type === 'thinking');
|
|
1976
|
+
) as Array<{ type: 'thinking'; thinking?: string; signature?: string }>;
|
|
1624
1977
|
const redacted = providerThinkingBlocks.filter((b) => b.type === 'redacted_thinking');
|
|
1625
1978
|
|
|
1626
|
-
const
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1979
|
+
const pairedProviderBlocks = new Set<number>();
|
|
1980
|
+
const claimedParsedIndices = new Set<number>();
|
|
1981
|
+
const parsedThinkingIndices = () =>
|
|
1982
|
+
content.reduce<number[]>((acc, block, index) => {
|
|
1983
|
+
if (block.type === 'thinking') acc.push(index);
|
|
1984
|
+
return acc;
|
|
1985
|
+
}, []);
|
|
1986
|
+
|
|
1987
|
+
for (let p = 0; p < providerThinking.length; p++) {
|
|
1988
|
+
const providerText = providerThinking[p]!.thinking ?? '';
|
|
1989
|
+
if (providerText === '') continue;
|
|
1990
|
+
const match = parsedThinkingIndices().find(
|
|
1991
|
+
(index) =>
|
|
1992
|
+
!claimedParsedIndices.has(index) &&
|
|
1993
|
+
sameThinkingText((content[index] as { thinking?: string }).thinking ?? '', providerText)
|
|
1994
|
+
);
|
|
1995
|
+
if (match === undefined) continue;
|
|
1996
|
+
const signature = providerThinking[p]!.signature;
|
|
1997
|
+
if (signature) (content[match] as { signature?: string }).signature = signature;
|
|
1998
|
+
claimedParsedIndices.add(match);
|
|
1999
|
+
pairedProviderBlocks.add(p);
|
|
2000
|
+
}
|
|
2001
|
+
|
|
2002
|
+
for (const parsedIndex of parsedThinkingIndices().reverse()) {
|
|
2003
|
+
if (claimedParsedIndices.has(parsedIndex)) continue;
|
|
2004
|
+
const parsedText = (content[parsedIndex] as { thinking?: string }).thinking ?? '';
|
|
2005
|
+
if (parsedText === '') continue;
|
|
2006
|
+
const run = findSpanningProviderRun(providerThinking, pairedProviderBlocks, parsedText);
|
|
2007
|
+
if (!run) continue;
|
|
2008
|
+
content.splice(
|
|
2009
|
+
parsedIndex,
|
|
2010
|
+
1,
|
|
2011
|
+
...run.map((p) => {
|
|
2012
|
+
pairedProviderBlocks.add(p);
|
|
2013
|
+
const block = providerThinking[p]!;
|
|
2014
|
+
return {
|
|
2015
|
+
type: 'thinking',
|
|
2016
|
+
thinking: block.thinking ?? '',
|
|
2017
|
+
...(block.signature ? { signature: block.signature } : {}),
|
|
2018
|
+
} as ContentBlock;
|
|
2019
|
+
})
|
|
2020
|
+
);
|
|
2021
|
+
claimedParsedIndices.add(parsedIndex);
|
|
1632
2022
|
}
|
|
1633
2023
|
|
|
1634
|
-
const
|
|
1635
|
-
|
|
1636
|
-
|
|
2024
|
+
const seen = new Set(content.map((block) => thinkingCarrierKey(block)));
|
|
2025
|
+
const leftover: ContentBlock[] = [];
|
|
2026
|
+
for (let p = 0; p < providerThinking.length; p++) {
|
|
2027
|
+
if (pairedProviderBlocks.has(p)) continue;
|
|
2028
|
+
const block = providerThinking[p]! as unknown as ContentBlock;
|
|
2029
|
+
const key = thinkingCarrierKey(block);
|
|
2030
|
+
if (seen.has(key)) continue;
|
|
2031
|
+
seen.add(key);
|
|
2032
|
+
leftover.push(block);
|
|
2033
|
+
}
|
|
2034
|
+
for (const block of redacted) {
|
|
2035
|
+
const key = thinkingCarrierKey(block);
|
|
2036
|
+
if (seen.has(key)) continue;
|
|
2037
|
+
seen.add(key);
|
|
2038
|
+
leftover.push(block);
|
|
1637
2039
|
}
|
|
2040
|
+
|
|
2041
|
+
if (leftover.length > 0) content.unshift(...leftover);
|
|
1638
2042
|
}
|
|
1639
2043
|
|
|
1640
2044
|
// ==========================================================================
|
|
@@ -1722,24 +2126,18 @@ export class Membrane {
|
|
|
1722
2126
|
}
|
|
1723
2127
|
|
|
1724
2128
|
/**
|
|
1725
|
-
* Transform a normalized request into provider format using the formatter
|
|
2129
|
+
* Transform a normalized request into provider format using the formatter.
|
|
2130
|
+
*
|
|
2131
|
+
* `activeFormatter` is the instance the caller already selected via
|
|
2132
|
+
* resolveActiveFormatter — including that selection's Responses-transport
|
|
2133
|
+
* authority rule, which used to live inline here. It is a parameter and not
|
|
2134
|
+
* a re-derivation so that the formatter which BUILDS is the same one that
|
|
2135
|
+
* resolved the tool mode and drives the loop.
|
|
1726
2136
|
*/
|
|
1727
|
-
private transformRequest(request: NormalizedRequest,
|
|
2137
|
+
private transformRequest(request: NormalizedRequest, activeFormatter: PrefillFormatter = this.formatter): {
|
|
1728
2138
|
providerRequest: any;
|
|
1729
2139
|
prefillResult: BuildResult;
|
|
1730
2140
|
} {
|
|
1731
|
-
// The Responses adapter's input is a provider-native item array. A generic
|
|
1732
|
-
// per-request formatter (for example Context Manager's NativeFormatter)
|
|
1733
|
-
// produces Anthropic-style `{ role, content: [{ type: 'text' }] }`
|
|
1734
|
-
// envelopes, which the Responses API rejects before inference. Keep the
|
|
1735
|
-
// configured Responses formatter authoritative at this transport boundary;
|
|
1736
|
-
// per-request formatter overrides remain available for adapters whose wire
|
|
1737
|
-
// format supports them.
|
|
1738
|
-
const activeFormatter =
|
|
1739
|
-
this.adapter.name === 'openai-responses-api' && this.formatter.name === 'openai-responses'
|
|
1740
|
-
? this.formatter
|
|
1741
|
-
: formatter ?? this.formatter;
|
|
1742
|
-
|
|
1743
2141
|
// Extract user-provided stop sequences
|
|
1744
2142
|
const additionalStopSequences = Array.isArray(request.stopSequences)
|
|
1745
2143
|
? request.stopSequences
|
|
@@ -1755,9 +2153,14 @@ export class Membrane {
|
|
|
1755
2153
|
participantMode: 'multiuser',
|
|
1756
2154
|
assistantParticipant: request.assistantParticipant ?? this.config.assistantParticipant ?? 'Claude',
|
|
1757
2155
|
tools: request.tools,
|
|
2156
|
+
// One resolution for every entry point: complete() used to build from the
|
|
2157
|
+
// formatter's constructor-time mode alone, so request.toolMode was a
|
|
2158
|
+
// second, disconnected source of truth on this path.
|
|
2159
|
+
toolMode: this.resolveToolMode(request, activeFormatter),
|
|
1758
2160
|
thinking: request.config.thinking,
|
|
1759
2161
|
systemPrompt: request.system,
|
|
1760
2162
|
promptCaching: request.promptCaching ?? this.config.defaultPromptCaching ?? true, // Default true for backward compat
|
|
2163
|
+
cacheMarkers: request.cacheMarkers ?? 'membrane-system',
|
|
1761
2164
|
cacheTtl: request.cacheTtl,
|
|
1762
2165
|
additionalStopSequences,
|
|
1763
2166
|
maxParticipantsForStop,
|
|
@@ -1779,7 +2182,9 @@ export class Membrane {
|
|
|
1779
2182
|
const providerRequest = {
|
|
1780
2183
|
...this.getBaseProviderParams(request.config),
|
|
1781
2184
|
messages: buildResult.messages,
|
|
1782
|
-
|
|
2185
|
+
// Owned, not aliased: the wire clamp strips markers in place, and a
|
|
2186
|
+
// formatter may pass the caller's own system array straight through.
|
|
2187
|
+
system: ownSystemBlocks(buildResult.systemContent),
|
|
1783
2188
|
stopSequences: buildResult.stopSequences,
|
|
1784
2189
|
tools: buildResult.nativeTools,
|
|
1785
2190
|
extra: {
|
|
@@ -1793,8 +2198,8 @@ export class Membrane {
|
|
|
1793
2198
|
// literal `<thinking>` text prefix instead of the API feature — drop the
|
|
1794
2199
|
// API param when the built request actually ends in an assistant prefill.
|
|
1795
2200
|
// Chat-style builds (no prefill) keep it.
|
|
1796
|
-
if (buildResult.assistantPrefill
|
|
1797
|
-
|
|
2201
|
+
if (buildResult.assistantPrefill) {
|
|
2202
|
+
stripThinkingForPrefill(providerRequest);
|
|
1798
2203
|
}
|
|
1799
2204
|
|
|
1800
2205
|
return { providerRequest, prefillResult: buildResult };
|
|
@@ -1836,24 +2241,75 @@ export class Membrane {
|
|
|
1836
2241
|
* somewhere upstream.
|
|
1837
2242
|
*/
|
|
1838
2243
|
onRetrying?: (info: { attempt: number; maxAttempts: number; category?: string }) => void;
|
|
2244
|
+
/**
|
|
2245
|
+
* Receives the number of cache_control markers the request ACTUALLY
|
|
2246
|
+
* ships with, taken from the clamp's own tally below — i.e. after the
|
|
2247
|
+
* `beforeRequest` hook has added or removed markers of its own and
|
|
2248
|
+
* after everything past the 4-breakpoint budget has been dropped.
|
|
2249
|
+
*
|
|
2250
|
+
* Telemetry that counts the request at BUILD time reports a number no
|
|
2251
|
+
* request ever had (a hook placing 7 markers on a wire that carries 4
|
|
2252
|
+
* was reported as the builder's 1), which defeats the audit the count
|
|
2253
|
+
* exists for. This is the only count that describes the wire.
|
|
2254
|
+
*/
|
|
2255
|
+
onWireCacheMarkers?: (markerCount: number) => void;
|
|
1839
2256
|
}
|
|
1840
|
-
)
|
|
2257
|
+
): Promise<
|
|
2258
|
+
import('./types/provider.js').ProviderResponse & {
|
|
2259
|
+
discardedUsage?: DiscardedAttemptsUsage;
|
|
2260
|
+
/** Provider calls this helper made, including refusal re-issues. */
|
|
2261
|
+
providerCalls: number;
|
|
2262
|
+
}
|
|
2263
|
+
> {
|
|
1841
2264
|
// Strip `normalizedRequest` before forwarding to the adapter — it's
|
|
1842
2265
|
// not part of `ProviderRequestOptions` and TypeScript's structural
|
|
1843
2266
|
// compatibility won't catch the excess field (checked only on object
|
|
1844
2267
|
// literals, not on variables). Leaving it in would silently leak the
|
|
1845
2268
|
// normalized form into every adapter's options.
|
|
1846
|
-
const { normalizedRequest, refusalRetries, onRetrying, ...adapterOptions } = options;
|
|
2269
|
+
const { normalizedRequest, refusalRetries, onRetrying, onWireCacheMarkers, ...adapterOptions } = options;
|
|
1847
2270
|
const finalRequest = (await this.applyBeforeRequestHook(normalizedRequest, request)) as typeof request;
|
|
1848
2271
|
|
|
2272
|
+
// Last exit before the adapter: the only place that sees EVERY
|
|
2273
|
+
// contribution (builder, formatter, passthrough, float, hook). Every
|
|
2274
|
+
// streaming path — stream(), streamYielding(), both tool loops — funnels
|
|
2275
|
+
// through here, so this is the one clamp they all get, and its tally is
|
|
2276
|
+
// therefore the only count that describes the wire.
|
|
2277
|
+
const markerCount = normalizedRequest.cacheMarkers === 'cm-owned'
|
|
2278
|
+
? assertCacheMarkersWithinLimit(finalRequest, 'streamOnce')
|
|
2279
|
+
: clampCacheMarkers(finalRequest, 'streamOnce').total;
|
|
2280
|
+
normalizedRequest.onCacheWireReceipt?.(computeCacheWireReceipt(finalRequest));
|
|
2281
|
+
onWireCacheMarkers?.(markerCount);
|
|
2282
|
+
|
|
1849
2283
|
// Retries are only safe when the caller can discard the abandoned
|
|
1850
2284
|
// attempt, so they require BOTH a budget and an onRetrying hook.
|
|
1851
2285
|
const maxAttempts = onRetrying ? Math.max(0, refusalRetries ?? 0) : 0;
|
|
1852
2286
|
let retried = 0;
|
|
2287
|
+
// Every re-issued attempt was a completed, billed provider call. The
|
|
2288
|
+
// caller's usage accumulator only ever sees the surviving result, so the
|
|
2289
|
+
// abandoned spend rides back out on the result itself.
|
|
2290
|
+
let discardedUsage: DiscardedAttemptsUsage | undefined;
|
|
2291
|
+
let providerCalls = 0;
|
|
1853
2292
|
while (true) {
|
|
1854
|
-
|
|
1855
|
-
|
|
2293
|
+
providerCalls++;
|
|
2294
|
+
const rawResult = await this.adapter.stream(finalRequest, callbacks, adapterOptions);
|
|
2295
|
+
// Restate usage in the one convention before any accumulator, ratio or
|
|
2296
|
+
// price sees it — this is the only door streamed usage enters through.
|
|
2297
|
+
const result: ProviderResponse = {
|
|
2298
|
+
...rawResult,
|
|
2299
|
+
usage: normalizeUsageToCacheExcluded(rawResult.usage, this.adapter.name, this.adapter.usageCacheConvention),
|
|
2300
|
+
};
|
|
2301
|
+
if (result.stopReason !== 'refusal' || retried >= maxAttempts) {
|
|
2302
|
+
return {
|
|
2303
|
+
...result,
|
|
2304
|
+
providerCalls,
|
|
2305
|
+
...(discardedUsage ? { discardedUsage } : {}),
|
|
2306
|
+
};
|
|
2307
|
+
}
|
|
1856
2308
|
retried++;
|
|
2309
|
+
discardedUsage = this.mergeDiscardedAttempts(
|
|
2310
|
+
discardedUsage,
|
|
2311
|
+
this.discardedAttemptFrom(result.usage)
|
|
2312
|
+
);
|
|
1857
2313
|
const category = (result.raw as { response?: { stop_details?: { category?: string } } } | undefined)
|
|
1858
2314
|
?.response?.stop_details?.category;
|
|
1859
2315
|
onRetrying!({ attempt: retried, maxAttempts, category });
|
|
@@ -1867,7 +2323,14 @@ export class Membrane {
|
|
|
1867
2323
|
): any {
|
|
1868
2324
|
// Anthropic quirk: assistant content cannot end with trailing whitespace
|
|
1869
2325
|
const trimmedAccumulated = accumulated.trimEnd();
|
|
1870
|
-
|
|
2326
|
+
|
|
2327
|
+
// Everything before the watermark already rides EARLIER messages (a
|
|
2328
|
+
// persisted split turn), so only the suffix belongs in the trailing
|
|
2329
|
+
// assistant prefill — replacing it with the whole document would
|
|
2330
|
+
// duplicate the pre-seam text and flatten the image user-turn away.
|
|
2331
|
+
const baseOffset = prefillResult.accumulatedBaseOffset ?? 0;
|
|
2332
|
+
const trailingContent = accumulated.slice(baseOffset).trimEnd();
|
|
2333
|
+
|
|
1871
2334
|
// Build continuation messages: keep all messages up to last assistant,
|
|
1872
2335
|
// then replace/add the accumulated content
|
|
1873
2336
|
const messages = [...prefillResult.messages];
|
|
@@ -1876,34 +2339,34 @@ export class Membrane {
|
|
|
1876
2339
|
let foundAssistant = false;
|
|
1877
2340
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
1878
2341
|
if (messages[i]?.role === 'assistant') {
|
|
1879
|
-
messages[i] = { role: 'assistant', content:
|
|
2342
|
+
messages[i] = { role: 'assistant', content: trailingContent };
|
|
1880
2343
|
foundAssistant = true;
|
|
1881
2344
|
break;
|
|
1882
2345
|
}
|
|
1883
2346
|
}
|
|
1884
2347
|
|
|
1885
2348
|
if (!foundAssistant) {
|
|
1886
|
-
messages.push({ role: 'assistant', content:
|
|
2349
|
+
messages.push({ role: 'assistant', content: trailingContent });
|
|
1887
2350
|
}
|
|
1888
2351
|
|
|
1889
|
-
return {
|
|
2352
|
+
return stripThinkingForPrefill({
|
|
1890
2353
|
...this.getBaseProviderParams(originalRequest.config),
|
|
1891
2354
|
// Continuations always end in an assistant prefill — the API rejects
|
|
1892
2355
|
// extended thinking combined with prefill, so never send the param here
|
|
1893
2356
|
thinking: undefined,
|
|
1894
2357
|
messages,
|
|
1895
|
-
system: prefillResult.systemContent
|
|
1896
|
-
? (Array.isArray(prefillResult.systemContent) && prefillResult.systemContent.length > 0
|
|
1897
|
-
? prefillResult.systemContent
|
|
1898
|
-
: prefillResult.systemContent)
|
|
1899
|
-
: undefined,
|
|
2358
|
+
system: ownSystemBlocks(prefillResult.systemContent) ?? undefined,
|
|
1900
2359
|
stopSequences: prefillResult.stopSequences,
|
|
1901
2360
|
extra: {
|
|
1902
2361
|
...originalRequest.providerParams,
|
|
2362
|
+
// Same contract transformRequest sends: adapters that reason about
|
|
2363
|
+
// the normalized shape (or fall back to serializing it) must not see
|
|
2364
|
+
// a continuation as a request with no normalized form at all.
|
|
2365
|
+
normalizedMessages: originalRequest.messages,
|
|
1903
2366
|
// Pre-serialized prompt for completions adapters — skip re-serialization
|
|
1904
2367
|
prompt: trimmedAccumulated,
|
|
1905
2368
|
},
|
|
1906
|
-
};
|
|
2369
|
+
});
|
|
1907
2370
|
}
|
|
1908
2371
|
|
|
1909
2372
|
/**
|
|
@@ -1933,6 +2396,12 @@ export class Membrane {
|
|
|
1933
2396
|
// Anthropic quirk: assistant content cannot end with trailing whitespace
|
|
1934
2397
|
const trimmedAccumulated = accumulated.trimEnd();
|
|
1935
2398
|
|
|
2399
|
+
// The split replaces only the CURRENT trailing assistant message, which
|
|
2400
|
+
// covers the accumulated text from the previous seam onward (0 on the
|
|
2401
|
+
// first split, the previous image seam on a later one).
|
|
2402
|
+
const baseOffset = prefillResult.accumulatedBaseOffset ?? 0;
|
|
2403
|
+
const trailingContent = accumulated.slice(baseOffset).trimEnd();
|
|
2404
|
+
|
|
1936
2405
|
// Build messages: copy all, then replace only the last assistant with split-turn
|
|
1937
2406
|
const messages: any[] = prefillResult.messages.map(msg => ({ ...msg }));
|
|
1938
2407
|
|
|
@@ -1948,7 +2417,7 @@ export class Membrane {
|
|
|
1948
2417
|
// Anthropic quirk: assistant content cannot end with trailing whitespace
|
|
1949
2418
|
const trimmedAfterXml = afterImageXml.trimEnd();
|
|
1950
2419
|
const splitTurnMessages = [
|
|
1951
|
-
{ role: 'assistant', content:
|
|
2420
|
+
{ role: 'assistant', content: trailingContent },
|
|
1952
2421
|
{ role: 'user', content: images },
|
|
1953
2422
|
{ role: 'assistant', content: trimmedAfterXml },
|
|
1954
2423
|
];
|
|
@@ -1959,20 +2428,40 @@ export class Membrane {
|
|
|
1959
2428
|
messages.push(...splitTurnMessages);
|
|
1960
2429
|
}
|
|
1961
2430
|
|
|
1962
|
-
|
|
2431
|
+
// PERSIST the split. Later rounds rebuild from prefillResult.messages;
|
|
2432
|
+
// without this the image user-turn exists on exactly one request and the
|
|
2433
|
+
// next continuation flattens the accumulated document back over it —
|
|
2434
|
+
// leaving <function_results> XML asserting a screenshot the model can no
|
|
2435
|
+
// longer see. Reassign (never mutate in place): the previous array is
|
|
2436
|
+
// still referenced by the request already on the wire. The watermark
|
|
2437
|
+
// moves to the seam — the point in `accumulated` where afterImageXml is
|
|
2438
|
+
// about to be appended — so the next builder replaces only the closing
|
|
2439
|
+
// assistant turn.
|
|
2440
|
+
prefillResult.messages = messages;
|
|
2441
|
+
prefillResult.accumulatedBaseOffset = accumulated.length;
|
|
2442
|
+
|
|
2443
|
+
return stripThinkingForPrefill({
|
|
1963
2444
|
...this.getBaseProviderParams(originalRequest.config),
|
|
1964
2445
|
// Continuations always end in an assistant prefill — the API rejects
|
|
1965
2446
|
// extended thinking combined with prefill, so never send the param here
|
|
1966
2447
|
thinking: undefined,
|
|
1967
2448
|
messages,
|
|
1968
|
-
system: prefillResult.systemContent
|
|
1969
|
-
? (Array.isArray(prefillResult.systemContent) && prefillResult.systemContent.length > 0
|
|
1970
|
-
? prefillResult.systemContent
|
|
1971
|
-
: prefillResult.systemContent)
|
|
1972
|
-
: undefined,
|
|
2449
|
+
system: ownSystemBlocks(prefillResult.systemContent) ?? undefined,
|
|
1973
2450
|
stopSequences: prefillResult.stopSequences,
|
|
1974
|
-
|
|
1975
|
-
|
|
2451
|
+
// Copied, not aliased: the guard below deletes the smuggled thinking
|
|
2452
|
+
// config, and mutating the caller's own providerParams object would
|
|
2453
|
+
// silently disable thinking on their NEXT (non-prefill) request.
|
|
2454
|
+
extra: {
|
|
2455
|
+
...originalRequest.providerParams,
|
|
2456
|
+
// Same contract as transformRequest and the plain continuation
|
|
2457
|
+
// builder. Without these a completions-style adapter fell through to
|
|
2458
|
+
// serializing PROVIDER-shaped messages as if they were normalized
|
|
2459
|
+
// ones, re-adding participant stop sequences the continuation
|
|
2460
|
+
// deliberately suppresses.
|
|
2461
|
+
normalizedMessages: originalRequest.messages,
|
|
2462
|
+
prompt: trimmedAccumulated,
|
|
2463
|
+
},
|
|
2464
|
+
});
|
|
1976
2465
|
}
|
|
1977
2466
|
|
|
1978
2467
|
private transformResponse(
|
|
@@ -2049,20 +2538,33 @@ export class Membrane {
|
|
|
2049
2538
|
|
|
2050
2539
|
// Parse XML tool calls from text if no native tool_use blocks were found
|
|
2051
2540
|
// This handles prefill mode where tools are XML in the text
|
|
2541
|
+
let emptyToolBlocks = 0;
|
|
2052
2542
|
if (toolCalls.length === 0 && rawAssistantText.includes('<function_calls>')) {
|
|
2053
2543
|
const parsed = parseToolCalls(rawAssistantText);
|
|
2054
2544
|
if (parsed?.calls.length) {
|
|
2055
2545
|
for (const tc of parsed.calls) {
|
|
2056
2546
|
toolCalls.push(tc);
|
|
2057
2547
|
}
|
|
2548
|
+
} else if (parsed) {
|
|
2549
|
+
emptyToolBlocks = 1;
|
|
2058
2550
|
}
|
|
2059
2551
|
}
|
|
2552
|
+
const unclosedToolBlock = endsWithPartialToolBlock(rawAssistantText);
|
|
2060
2553
|
|
|
2061
2554
|
const stopReason = this.mapStopReason(providerResponse.stopReason);
|
|
2555
|
+
this.reportToolParseDiagnostics({ unclosedToolBlock, emptyToolBlocks }, stopReason);
|
|
2062
2556
|
const durationMs = Date.now() - startTime;
|
|
2063
|
-
|
|
2557
|
+
// `NormalizedResponse.usage` is typed DetailedUsage and the streaming paths
|
|
2558
|
+
// already return the whole thing; complete() used to narrow it to
|
|
2559
|
+
// input/output here, so a caller reading `response.usage.cacheReadTokens`
|
|
2560
|
+
// saw undefined on one path and a number on the other.
|
|
2561
|
+
const usage: DetailedUsage = {
|
|
2064
2562
|
inputTokens: providerResponse.usage.inputTokens,
|
|
2065
2563
|
outputTokens: providerResponse.usage.outputTokens,
|
|
2564
|
+
cacheCreationTokens: providerResponse.usage.cacheCreationTokens,
|
|
2565
|
+
cacheReadTokens: providerResponse.usage.cacheReadTokens,
|
|
2566
|
+
thinkingTokens: providerResponse.usage.thinkingTokens,
|
|
2567
|
+
estimatedCost: this.estimateCost(providerResponse.usage, request.config.model, providerResponse.model),
|
|
2066
2568
|
};
|
|
2067
2569
|
|
|
2068
2570
|
return {
|
|
@@ -2077,14 +2579,9 @@ export class Membrane {
|
|
|
2077
2579
|
reason: stopReason,
|
|
2078
2580
|
triggeredSequence: providerResponse.stopSequence,
|
|
2079
2581
|
wasTruncated: stopReason === 'max_tokens',
|
|
2582
|
+
unclosedToolBlock,
|
|
2080
2583
|
},
|
|
2081
|
-
usage
|
|
2082
|
-
inputTokens: providerResponse.usage.inputTokens,
|
|
2083
|
-
outputTokens: providerResponse.usage.outputTokens,
|
|
2084
|
-
cacheCreationTokens: providerResponse.usage.cacheCreationTokens,
|
|
2085
|
-
cacheReadTokens: providerResponse.usage.cacheReadTokens,
|
|
2086
|
-
estimatedCost: this.estimateCost(providerResponse.usage, request.config.model),
|
|
2087
|
-
},
|
|
2584
|
+
usage,
|
|
2088
2585
|
timing: {
|
|
2089
2586
|
totalDurationMs: durationMs,
|
|
2090
2587
|
attempts,
|
|
@@ -2108,11 +2605,65 @@ export class Membrane {
|
|
|
2108
2605
|
};
|
|
2109
2606
|
}
|
|
2110
2607
|
|
|
2608
|
+
/**
|
|
2609
|
+
* The turn is over, and the two guards that detect a half-written tool block
|
|
2610
|
+
* finally have a call site. Both shapes are defects a consumer must not
|
|
2611
|
+
* persist blind: an unclosed block splices onto the NEXT round's closing tag
|
|
2612
|
+
* (the loop does not resume on a length stop, so max_tokens leaves exactly
|
|
2613
|
+
* this), and a block that parsed to nothing means the model believes it
|
|
2614
|
+
* called a tool that never ran.
|
|
2615
|
+
*/
|
|
2616
|
+
private reportToolParseDiagnostics(
|
|
2617
|
+
diagnostics: {
|
|
2618
|
+
unclosedToolBlock: boolean;
|
|
2619
|
+
emptyToolBlocks: number;
|
|
2620
|
+
splicedToolBlocks?: number;
|
|
2621
|
+
unclosedInvokeHeads?: number;
|
|
2622
|
+
},
|
|
2623
|
+
stopReason: StopReason
|
|
2624
|
+
): void {
|
|
2625
|
+
const warnLog = this.config.logger ?? console;
|
|
2626
|
+
|
|
2627
|
+
if (diagnostics.unclosedToolBlock) {
|
|
2628
|
+
warnLog.warn(
|
|
2629
|
+
`[membrane] turn ended (${stopReason}) with an unclosed tool block in the ` +
|
|
2630
|
+
`assistant text — the loop does not resume on a length stop. Persisting this ` +
|
|
2631
|
+
`turn verbatim lets the next round's closing tag splice onto the stale ` +
|
|
2632
|
+
`opener; see details.stop.unclosedToolBlock.`
|
|
2633
|
+
);
|
|
2634
|
+
}
|
|
2635
|
+
|
|
2636
|
+
if (diagnostics.emptyToolBlocks > 0) {
|
|
2637
|
+
warnLog.warn(
|
|
2638
|
+
`[membrane] ${diagnostics.emptyToolBlocks} function_calls block(s) parsed to ` +
|
|
2639
|
+
`zero tool calls — always a defect, never a normal ending. The call was ` +
|
|
2640
|
+
`returned as assistant text and nothing executed.`
|
|
2641
|
+
);
|
|
2642
|
+
}
|
|
2643
|
+
|
|
2644
|
+
if (diagnostics.splicedToolBlocks) {
|
|
2645
|
+
warnLog.warn(
|
|
2646
|
+
`[membrane] ${diagnostics.splicedToolBlocks} tool block(s) spanned a second ` +
|
|
2647
|
+
`<function_calls> opener and were re-anchored to the innermost one — an ` +
|
|
2648
|
+
`earlier truncated block is present in this conversation's assistant text.`
|
|
2649
|
+
);
|
|
2650
|
+
}
|
|
2651
|
+
|
|
2652
|
+
if (diagnostics.unclosedInvokeHeads) {
|
|
2653
|
+
warnLog.warn(
|
|
2654
|
+
`[membrane] ${diagnostics.unclosedInvokeHeads} <invoke> head(s) were left ` +
|
|
2655
|
+
`unclosed and swallowed the invoke that followed — nothing was dispatched ` +
|
|
2656
|
+
`under an unclosed head's name, and the call it absorbed was re-anchored ` +
|
|
2657
|
+
`and ran with its own parameters.`
|
|
2658
|
+
);
|
|
2659
|
+
}
|
|
2660
|
+
}
|
|
2661
|
+
|
|
2111
2662
|
private buildFinalResponse(
|
|
2112
2663
|
accumulated: string,
|
|
2113
2664
|
contentBlocks: ContentBlock[],
|
|
2114
2665
|
stopReason: StopReason,
|
|
2115
|
-
|
|
2666
|
+
turnUsage: TurnUsageAccumulator,
|
|
2116
2667
|
request: NormalizedRequest,
|
|
2117
2668
|
prefillResult: {
|
|
2118
2669
|
cacheMarkersApplied?: number;
|
|
@@ -2124,14 +2675,17 @@ export class Membrane {
|
|
|
2124
2675
|
executedToolCalls: ToolCall[] = [],
|
|
2125
2676
|
executedToolResults: ToolResult[] = [],
|
|
2126
2677
|
startInsideBlock: 'thinking' | 'tool_call' | 'tool_result' | null = null,
|
|
2127
|
-
triggeredSequence?: string
|
|
2678
|
+
triggeredSequence?: string,
|
|
2128
2679
|
): NormalizedResponse {
|
|
2680
|
+
const usage = turnUsage.total;
|
|
2129
2681
|
// Parse accumulated text into structured content blocks
|
|
2130
2682
|
// This extracts thinking, tool_use, tool_result, and text blocks
|
|
2131
2683
|
let finalContent: ContentBlock[];
|
|
2132
2684
|
let toolCalls: ToolCall[];
|
|
2133
2685
|
let toolResults: ToolResult[];
|
|
2134
2686
|
|
|
2687
|
+
let unclosedToolBlock = false;
|
|
2688
|
+
|
|
2135
2689
|
if (contentBlocks.length > 0) {
|
|
2136
2690
|
// Native mode - content blocks already structured
|
|
2137
2691
|
finalContent = contentBlocks;
|
|
@@ -2146,6 +2700,8 @@ export class Membrane {
|
|
|
2146
2700
|
finalContent = parsed.blocks;
|
|
2147
2701
|
toolCalls = parsed.toolCalls.length > 0 ? parsed.toolCalls : executedToolCalls;
|
|
2148
2702
|
toolResults = parsed.toolResults.length > 0 ? parsed.toolResults : executedToolResults;
|
|
2703
|
+
unclosedToolBlock = parsed.unclosedToolBlock;
|
|
2704
|
+
this.reportToolParseDiagnostics(parsed, stopReason);
|
|
2149
2705
|
}
|
|
2150
2706
|
|
|
2151
2707
|
const durationMs = Date.now() - startTime;
|
|
@@ -2162,19 +2718,21 @@ export class Membrane {
|
|
|
2162
2718
|
reason: stopReason,
|
|
2163
2719
|
triggeredSequence,
|
|
2164
2720
|
wasTruncated: stopReason === 'max_tokens',
|
|
2721
|
+
unclosedToolBlock,
|
|
2165
2722
|
},
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2723
|
+
// Priced per round by the accumulator and summed — NOT re-derived here
|
|
2724
|
+
// from the turn totals, which would re-bill every round at the last
|
|
2725
|
+
// model's rate.
|
|
2726
|
+
usage,
|
|
2170
2727
|
timing: {
|
|
2171
2728
|
totalDurationMs: durationMs,
|
|
2172
2729
|
attempts,
|
|
2173
2730
|
},
|
|
2174
2731
|
model: {
|
|
2175
2732
|
requested: request.config.model,
|
|
2176
|
-
actual: request.config.model,
|
|
2733
|
+
actual: turnUsage.lastServedModel || request.config.model,
|
|
2177
2734
|
provider: this.adapter.name,
|
|
2735
|
+
perRound: turnUsage.perRound,
|
|
2178
2736
|
},
|
|
2179
2737
|
cache: {
|
|
2180
2738
|
markersInRequest: prefillResult.cacheMarkersApplied ?? 0,
|
|
@@ -2190,6 +2748,52 @@ export class Membrane {
|
|
|
2190
2748
|
};
|
|
2191
2749
|
}
|
|
2192
2750
|
|
|
2751
|
+
/**
|
|
2752
|
+
* Fold one discarded (billed but abandoned) attempt's usage into a carry.
|
|
2753
|
+
* Returns a NEW object so a caller's earlier snapshot is never mutated.
|
|
2754
|
+
*/
|
|
2755
|
+
private mergeDiscardedAttempts(
|
|
2756
|
+
carry: DiscardedAttemptsUsage | undefined,
|
|
2757
|
+
add: DiscardedAttemptsUsage | undefined
|
|
2758
|
+
): DiscardedAttemptsUsage | undefined {
|
|
2759
|
+
if (!add) return carry;
|
|
2760
|
+
const next: DiscardedAttemptsUsage = carry
|
|
2761
|
+
? { ...carry }
|
|
2762
|
+
: { attempts: 0, inputTokens: 0, outputTokens: 0 };
|
|
2763
|
+
next.attempts += add.attempts;
|
|
2764
|
+
next.inputTokens += add.inputTokens;
|
|
2765
|
+
next.outputTokens += add.outputTokens;
|
|
2766
|
+
if (add.cacheCreationTokens) {
|
|
2767
|
+
next.cacheCreationTokens = (next.cacheCreationTokens ?? 0) + add.cacheCreationTokens;
|
|
2768
|
+
}
|
|
2769
|
+
if (add.cacheReadTokens) {
|
|
2770
|
+
next.cacheReadTokens = (next.cacheReadTokens ?? 0) + add.cacheReadTokens;
|
|
2771
|
+
}
|
|
2772
|
+
return next;
|
|
2773
|
+
}
|
|
2774
|
+
|
|
2775
|
+
/** One provider call's usage as a single-attempt discard record. */
|
|
2776
|
+
private discardedAttemptFrom(usage: DetailedUsage | BasicUsage | undefined): DiscardedAttemptsUsage {
|
|
2777
|
+
const detailed = (usage ?? { inputTokens: 0, outputTokens: 0 }) as DetailedUsage;
|
|
2778
|
+
return {
|
|
2779
|
+
attempts: 1,
|
|
2780
|
+
inputTokens: detailed.inputTokens ?? 0,
|
|
2781
|
+
outputTokens: detailed.outputTokens ?? 0,
|
|
2782
|
+
...(detailed.cacheCreationTokens ? { cacheCreationTokens: detailed.cacheCreationTokens } : {}),
|
|
2783
|
+
...(detailed.cacheReadTokens ? { cacheReadTokens: detailed.cacheReadTokens } : {}),
|
|
2784
|
+
};
|
|
2785
|
+
}
|
|
2786
|
+
|
|
2787
|
+
/** Price the discarded spend so a caller can read it without re-deriving. */
|
|
2788
|
+
private pricedDiscardedAttempts(
|
|
2789
|
+
discarded: DiscardedAttemptsUsage | undefined,
|
|
2790
|
+
model: string
|
|
2791
|
+
): DiscardedAttemptsUsage | undefined {
|
|
2792
|
+
if (!discarded) return undefined;
|
|
2793
|
+
const estimatedCost = this.estimateCost(discarded, model);
|
|
2794
|
+
return estimatedCost ? { ...discarded, estimatedCost } : discarded;
|
|
2795
|
+
}
|
|
2796
|
+
|
|
2193
2797
|
private mapStopReason(providerReason: string): StopReason {
|
|
2194
2798
|
switch (providerReason) {
|
|
2195
2799
|
case 'end_turn':
|
|
@@ -2211,20 +2815,62 @@ export class Membrane {
|
|
|
2211
2815
|
}
|
|
2212
2816
|
|
|
2213
2817
|
private calculateCacheHitRatio(usage: Pick<DetailedUsage, 'inputTokens' | 'cacheReadTokens'>): number {
|
|
2214
|
-
|
|
2215
|
-
const total = usage.inputTokens ?? 0;
|
|
2216
|
-
if (total === 0) return 0;
|
|
2217
|
-
return cacheRead / total;
|
|
2818
|
+
return calculateCacheHitRatio(usage);
|
|
2218
2819
|
}
|
|
2219
2820
|
|
|
2220
|
-
|
|
2221
|
-
|
|
2821
|
+
/**
|
|
2822
|
+
* Pricing is resolved on TWO axes, and SOURCE outranks SPECIFICITY:
|
|
2823
|
+
*
|
|
2824
|
+
* registry[served] → registry[requested] → builtin[served] → builtin[requested]
|
|
2825
|
+
*
|
|
2826
|
+
* Specificity — preferring the model that ACTUALLY served over the id that
|
|
2827
|
+
* was requested — is real: an alias or an auto-routed request otherwise
|
|
2828
|
+
* prices against a string the provider already replaced, and a live
|
|
2829
|
+
* 2026-08-25 call asking for `gpt-4o-mini` was served by
|
|
2830
|
+
* `gpt-4o-mini-2024-07-18`. But it only breaks ties WITHIN one source.
|
|
2831
|
+
* A configured `ModelRegistry` is the caller stating their own rates —
|
|
2832
|
+
* account-specific, negotiated, authoritative; the built-in table is
|
|
2833
|
+
* membrane's shipped guess at public list prices. Merging the two per-model
|
|
2834
|
+
* (`registry[served] ?? builtin[served]`, return on the first hit) let the
|
|
2835
|
+
* guess for a snapshot outrank the caller's own entry for the alias they
|
|
2836
|
+
* asked for, so a caller who prices their alias and lets the provider pick
|
|
2837
|
+
* the snapshot was billed at membrane's number instead of theirs.
|
|
2838
|
+
*
|
|
2839
|
+
* Both fallbacks stay: the served model may be absent from a source, and the
|
|
2840
|
+
* provider may name none at all.
|
|
2841
|
+
*/
|
|
2842
|
+
private resolvePricing(
|
|
2843
|
+
requestedModel: string,
|
|
2844
|
+
actualModel?: string
|
|
2845
|
+
): import('./types/provider.js').ModelPricing | undefined {
|
|
2846
|
+
const servedModel = actualModel && actualModel !== requestedModel ? actualModel : undefined;
|
|
2847
|
+
const fromRegistry = (modelId: string | undefined) =>
|
|
2848
|
+
modelId === undefined ? undefined : this.registry?.getPricing(modelId);
|
|
2849
|
+
const fromBuiltin = (modelId: string | undefined) =>
|
|
2850
|
+
modelId === undefined ? undefined : getDefaultPricing(modelId);
|
|
2851
|
+
|
|
2852
|
+
return fromRegistry(servedModel)
|
|
2853
|
+
?? fromRegistry(requestedModel)
|
|
2854
|
+
?? fromBuiltin(servedModel)
|
|
2855
|
+
?? fromBuiltin(requestedModel);
|
|
2222
2856
|
}
|
|
2223
2857
|
|
|
2224
2858
|
/** Resolve pricing + calculate cost in one call (for one-shot use outside loops). */
|
|
2225
|
-
private estimateCost(
|
|
2226
|
-
|
|
2227
|
-
|
|
2859
|
+
private estimateCost(
|
|
2860
|
+
usage: import('./utils/cost.js').CostableUsage,
|
|
2861
|
+
requestedModel: string,
|
|
2862
|
+
actualModel?: string
|
|
2863
|
+
): import('./types/response.js').CostBreakdown | undefined {
|
|
2864
|
+
const pricing = this.resolvePricing(requestedModel, actualModel);
|
|
2865
|
+
if (!pricing) {
|
|
2866
|
+
// An absent cost and a zero cost are different claims. Returning
|
|
2867
|
+
// undefined says "membrane does not know what this costs"; saying it out
|
|
2868
|
+
// loud once per model keeps that from reading as "free" to a caller that
|
|
2869
|
+
// only ever sees the omission.
|
|
2870
|
+
warnUnpricedModel(actualModel || requestedModel);
|
|
2871
|
+
return undefined;
|
|
2872
|
+
}
|
|
2873
|
+
return calculateCost(usage, pricing);
|
|
2228
2874
|
}
|
|
2229
2875
|
|
|
2230
2876
|
private calculateRetryDelay(attempt: number, overloaded = false): number {
|
|
@@ -2262,6 +2908,10 @@ export class Membrane {
|
|
|
2262
2908
|
* Check if an error is an abort error
|
|
2263
2909
|
*/
|
|
2264
2910
|
private isAbortError(error: unknown): boolean {
|
|
2911
|
+
// An adapter's own deadline: a timeout by classification, still an abort
|
|
2912
|
+
// by provenance, so the streaming paths hand back the partial content
|
|
2913
|
+
// they collected instead of throwing.
|
|
2914
|
+
if (isTimeoutAbortError(error)) return true;
|
|
2265
2915
|
if (error instanceof Error) {
|
|
2266
2916
|
// Standard AbortError
|
|
2267
2917
|
if (error.name === 'AbortError') return true;
|
|
@@ -2275,6 +2925,20 @@ export class Membrane {
|
|
|
2275
2925
|
return false;
|
|
2276
2926
|
}
|
|
2277
2927
|
|
|
2928
|
+
/**
|
|
2929
|
+
* Why a caught abort happened. The caller's own signal is authoritative:
|
|
2930
|
+
* if it fired, the cancellation is theirs whatever the error text says.
|
|
2931
|
+
* Otherwise an adapter-side deadline classifies as a timeout — the adapters
|
|
2932
|
+
* mark the abort createCombinedSignal's timeoutMs raises and map it to a
|
|
2933
|
+
* TimeoutAbortError, so the identity survives their error handling — and
|
|
2934
|
+
* anything else that reached the abort catch is a failure, not a person.
|
|
2935
|
+
*/
|
|
2936
|
+
private abortReason(error: unknown, signal?: AbortSignal): 'user' | 'timeout' | 'error' {
|
|
2937
|
+
if (signal?.aborted) return 'user';
|
|
2938
|
+
if (classifyError(error).type === 'timeout') return 'timeout';
|
|
2939
|
+
return 'error';
|
|
2940
|
+
}
|
|
2941
|
+
|
|
2278
2942
|
/**
|
|
2279
2943
|
* Build an AbortedResponse from current execution state
|
|
2280
2944
|
*/
|
|
@@ -2337,7 +3001,12 @@ export class Membrane {
|
|
|
2337
3001
|
request: NormalizedRequest,
|
|
2338
3002
|
options: YieldingStreamOptions = {}
|
|
2339
3003
|
): YieldingStream {
|
|
2340
|
-
|
|
3004
|
+
// YieldingStreamOptions carries no per-request formatter override, so the
|
|
3005
|
+
// selection here can only land on the instance formatter — it goes through
|
|
3006
|
+
// resolveActiveFormatter anyway so this path reads the same single source
|
|
3007
|
+
// as complete() and stream() if an override is ever added.
|
|
3008
|
+
const activeFormatter = this.resolveActiveFormatter();
|
|
3009
|
+
const toolMode = this.resolveToolMode(request, activeFormatter);
|
|
2341
3010
|
|
|
2342
3011
|
// refusalRetries is implemented on the native path only. The XML path
|
|
2343
3012
|
// accumulates into a streaming parser carrying prefill context and
|
|
@@ -2353,8 +3022,8 @@ export class Membrane {
|
|
|
2353
3022
|
|
|
2354
3023
|
// Create the yielding stream with the appropriate inference runner
|
|
2355
3024
|
const runInference = toolMode === 'native'
|
|
2356
|
-
? (stream: YieldingStreamImpl) => this.runNativeToolsYielding(request, options, stream)
|
|
2357
|
-
: (stream: YieldingStreamImpl) => this.runXmlToolsYielding(request, options, stream);
|
|
3025
|
+
? (stream: YieldingStreamImpl) => this.runNativeToolsYielding(request, options, stream, activeFormatter)
|
|
3026
|
+
: (stream: YieldingStreamImpl) => this.runXmlToolsYielding(request, options, stream, activeFormatter);
|
|
2358
3027
|
|
|
2359
3028
|
return new YieldingStreamImpl(options, runInference);
|
|
2360
3029
|
}
|
|
@@ -2365,7 +3034,8 @@ export class Membrane {
|
|
|
2365
3034
|
private async runXmlToolsYielding(
|
|
2366
3035
|
request: NormalizedRequest,
|
|
2367
3036
|
options: YieldingStreamOptions,
|
|
2368
|
-
stream: YieldingStreamImpl
|
|
3037
|
+
stream: YieldingStreamImpl,
|
|
3038
|
+
activeFormatter: PrefillFormatter = this.resolveActiveFormatter()
|
|
2369
3039
|
): Promise<void> {
|
|
2370
3040
|
const startTime = Date.now();
|
|
2371
3041
|
const {
|
|
@@ -2407,14 +3077,23 @@ export class Membrane {
|
|
|
2407
3077
|
let prevRoundStopSequence: string | undefined;
|
|
2408
3078
|
const warnLog = this.config.logger ?? console;
|
|
2409
3079
|
|
|
2410
|
-
// Initialize parser from formatter
|
|
2411
|
-
|
|
3080
|
+
// Initialize parser from the formatter streamYielding selected, so the
|
|
3081
|
+
// parser and the build below read the same format.
|
|
3082
|
+
const formatter = activeFormatter;
|
|
2412
3083
|
const parser = formatter.createStreamParser();
|
|
2413
3084
|
let toolDepth = 0;
|
|
3085
|
+
// Honest turn telemetry: provider calls actually made (including refusal
|
|
3086
|
+
// re-issues inside streamOnce) and continuation rounds.
|
|
3087
|
+
let providerCalls = 0;
|
|
3088
|
+
let rounds = 0;
|
|
2414
3089
|
// Once-per-stream latch for the injectedMessages-unsupported warning.
|
|
2415
3090
|
let warnedInjectionUnsupported = false;
|
|
2416
|
-
|
|
2417
|
-
|
|
3091
|
+
// See streamWithXmlTools: one accumulator per turn, pricing each round
|
|
3092
|
+
// under the model that served it.
|
|
3093
|
+
const turnUsage = new TurnUsageAccumulator(
|
|
3094
|
+
request.config.model,
|
|
3095
|
+
(servedModel) => this.resolvePricing(request.config.model, servedModel),
|
|
3096
|
+
);
|
|
2418
3097
|
const contentBlocks: ContentBlock[] = [];
|
|
2419
3098
|
let lastStopReason: StopReason = 'end_turn';
|
|
2420
3099
|
let lastStopSequence: string | undefined;
|
|
@@ -2463,7 +3142,7 @@ export class Membrane {
|
|
|
2463
3142
|
if (resumptionRounds === RESUMPTION_WARN_ROUNDS) {
|
|
2464
3143
|
warnLog.warn(
|
|
2465
3144
|
`[membrane] automatic resumption at round ${resumptionRounds} ` +
|
|
2466
|
-
`(${
|
|
3145
|
+
`(${turnUsage.total.inputTokens} input tokens so far this turn) — ` +
|
|
2467
3146
|
`a spin shows up here before it shows up on the bill`
|
|
2468
3147
|
);
|
|
2469
3148
|
}
|
|
@@ -2471,7 +3150,7 @@ export class Membrane {
|
|
|
2471
3150
|
warnLog.warn(
|
|
2472
3151
|
`[membrane] automatic resumption cap (${maxResumptionRounds}) reached — ` +
|
|
2473
3152
|
`ending turn with stopReason 'round_limit'. ` +
|
|
2474
|
-
`${
|
|
3153
|
+
`${turnUsage.total.inputTokens} input tokens spent this turn.`
|
|
2475
3154
|
);
|
|
2476
3155
|
return false;
|
|
2477
3156
|
}
|
|
@@ -2567,6 +3246,9 @@ export class Membrane {
|
|
|
2567
3246
|
}
|
|
2568
3247
|
);
|
|
2569
3248
|
|
|
3249
|
+
rounds++;
|
|
3250
|
+
providerCalls += streamResult.providerCalls;
|
|
3251
|
+
|
|
2570
3252
|
// If we detected stop sequence manually, fix up the parser and result
|
|
2571
3253
|
if (detectedStopSequence && truncatedAccumulated !== null) {
|
|
2572
3254
|
parser.reset();
|
|
@@ -2584,18 +3266,10 @@ export class Membrane {
|
|
|
2584
3266
|
lastStopReason = this.mapStopReason(streamResult.stopReason);
|
|
2585
3267
|
lastStopSequence = streamResult.stopSequence ?? undefined;
|
|
2586
3268
|
|
|
2587
|
-
// Accumulate usage (including cache metrics)
|
|
2588
|
-
|
|
2589
|
-
totalUsage.outputTokens += streamResult.usage.outputTokens;
|
|
2590
|
-
if (streamResult.usage.cacheCreationTokens) {
|
|
2591
|
-
totalUsage.cacheCreationTokens = (totalUsage.cacheCreationTokens ?? 0) + streamResult.usage.cacheCreationTokens;
|
|
2592
|
-
}
|
|
2593
|
-
if (streamResult.usage.cacheReadTokens) {
|
|
2594
|
-
totalUsage.cacheReadTokens = (totalUsage.cacheReadTokens ?? 0) + streamResult.usage.cacheReadTokens;
|
|
2595
|
-
}
|
|
2596
|
-
if (pricing) totalUsage.estimatedCost = calculateCost(totalUsage, pricing);
|
|
3269
|
+
// Accumulate usage (including cache metrics), priced at this round's model
|
|
3270
|
+
const usageSoFar = turnUsage.addRound(streamResult.model, streamResult.usage);
|
|
2597
3271
|
if (emitUsage) {
|
|
2598
|
-
stream.emit({ type: 'usage', usage:
|
|
3272
|
+
stream.emit({ type: 'usage', usage: usageSoFar });
|
|
2599
3273
|
}
|
|
2600
3274
|
|
|
2601
3275
|
// Flush the parser
|
|
@@ -2623,7 +3297,7 @@ export class Membrane {
|
|
|
2623
3297
|
`[membrane] ${consecutiveStalledResumptions} consecutive automatic resumptions ` +
|
|
2624
3298
|
`made no progress (${streamedThisRound} chars this round, stop ` +
|
|
2625
3299
|
`${JSON.stringify(lastStopSequence ?? null)} repeated) — ending turn with ` +
|
|
2626
|
-
`stopReason 'no_progress'. ${
|
|
3300
|
+
`stopReason 'no_progress'. ${turnUsage.total.inputTokens} input tokens spent this turn.`
|
|
2627
3301
|
);
|
|
2628
3302
|
lastStopReason = 'no_progress';
|
|
2629
3303
|
break;
|
|
@@ -2924,22 +3598,24 @@ export class Membrane {
|
|
|
2924
3598
|
newContent,
|
|
2925
3599
|
contentBlocks,
|
|
2926
3600
|
lastStopReason,
|
|
2927
|
-
|
|
3601
|
+
turnUsage,
|
|
2928
3602
|
request,
|
|
2929
3603
|
prefillResult,
|
|
2930
3604
|
startTime,
|
|
2931
|
-
|
|
3605
|
+
providerCalls,
|
|
2932
3606
|
rawRequest,
|
|
2933
3607
|
rawResponse,
|
|
2934
3608
|
executedToolCalls,
|
|
2935
3609
|
executedToolResults,
|
|
2936
3610
|
initialBlockType,
|
|
2937
|
-
lastStopSequence
|
|
3611
|
+
lastStopSequence,
|
|
2938
3612
|
);
|
|
2939
3613
|
|
|
2940
3614
|
// Merge provider thinking signatures into parser-derived thinking blocks
|
|
2941
3615
|
this.mergeProviderThinkingBlocks(response.content, providerThinkingBlocks);
|
|
2942
3616
|
|
|
3617
|
+
response.details.timing.rounds = rounds;
|
|
3618
|
+
|
|
2943
3619
|
stream.emit({ type: 'complete', response });
|
|
2944
3620
|
} catch (error) {
|
|
2945
3621
|
if (this.isAbortError(error)) {
|
|
@@ -2947,7 +3623,7 @@ export class Membrane {
|
|
|
2947
3623
|
const newContent = fullAccumulated.slice(initialPrefillLength);
|
|
2948
3624
|
stream.emit({
|
|
2949
3625
|
type: 'aborted',
|
|
2950
|
-
reason:
|
|
3626
|
+
reason: this.abortReason(error, stream.signal),
|
|
2951
3627
|
partialContent: parseAccumulatedIntoBlocks(newContent).blocks,
|
|
2952
3628
|
rawAssistantText: newContent,
|
|
2953
3629
|
toolCalls: executedToolCalls,
|
|
@@ -2965,7 +3641,8 @@ export class Membrane {
|
|
|
2965
3641
|
private async runNativeToolsYielding(
|
|
2966
3642
|
request: NormalizedRequest,
|
|
2967
3643
|
options: YieldingStreamOptions,
|
|
2968
|
-
stream: YieldingStreamImpl
|
|
3644
|
+
stream: YieldingStreamImpl,
|
|
3645
|
+
activeFormatter: PrefillFormatter = this.resolveActiveFormatter()
|
|
2969
3646
|
): Promise<void> {
|
|
2970
3647
|
const startTime = Date.now();
|
|
2971
3648
|
const {
|
|
@@ -2985,8 +3662,16 @@ export class Membrane {
|
|
|
2985
3662
|
: maxToolDepthOpt;
|
|
2986
3663
|
|
|
2987
3664
|
let toolDepth = 0;
|
|
2988
|
-
|
|
2989
|
-
|
|
3665
|
+
// See streamWithXmlTools: one accumulator per turn, pricing each round
|
|
3666
|
+
// under the model that served it.
|
|
3667
|
+
const turnUsage = new TurnUsageAccumulator(
|
|
3668
|
+
request.config.model,
|
|
3669
|
+
(servedModel) => this.resolvePricing(request.config.model, servedModel),
|
|
3670
|
+
);
|
|
3671
|
+
// Honest turn telemetry: provider calls actually made (including refusal
|
|
3672
|
+
// re-issues inside streamOnce) and continuation rounds.
|
|
3673
|
+
let providerCalls = 0;
|
|
3674
|
+
let rounds = 0;
|
|
2990
3675
|
let lastStopReason: StopReason = 'end_turn';
|
|
2991
3676
|
let lastStopSequence: string | undefined;
|
|
2992
3677
|
let rawRequest: unknown;
|
|
@@ -2995,9 +3680,12 @@ export class Membrane {
|
|
|
2995
3680
|
let allTextAccumulated = '';
|
|
2996
3681
|
const executedToolCalls: ToolCall[] = [];
|
|
2997
3682
|
const executedToolResults: ToolResult[] = [];
|
|
3683
|
+
// Spend on refusal attempts this turn threw away (see streamOnce).
|
|
3684
|
+
let discardedUsage: DiscardedAttemptsUsage | undefined;
|
|
2998
3685
|
|
|
2999
3686
|
let messages = [...request.messages];
|
|
3000
3687
|
let allContentBlocks: ContentBlock[] = [];
|
|
3688
|
+
let markersInLastRequest = 0;
|
|
3001
3689
|
|
|
3002
3690
|
try {
|
|
3003
3691
|
// Tool execution loop
|
|
@@ -3015,25 +3703,20 @@ export class Membrane {
|
|
|
3015
3703
|
}
|
|
3016
3704
|
|
|
3017
3705
|
// Build provider request with native tools
|
|
3018
|
-
const providerRequest = this.buildNativeToolRequest(request, messages, toolDepth > 0);
|
|
3706
|
+
const providerRequest = this.buildNativeToolRequest(request, messages, toolDepth > 0, activeFormatter);
|
|
3019
3707
|
|
|
3020
3708
|
// Stream from provider
|
|
3021
3709
|
let textAccumulated = '';
|
|
3022
|
-
let blockIndex = 0;
|
|
3023
3710
|
// Where this attempt starts inside the tool-loop-spanning buffer, so
|
|
3024
3711
|
// a refusal retry can roll back exactly this attempt's contribution.
|
|
3025
3712
|
const allTextBefore = allTextAccumulated.length;
|
|
3026
|
-
// Track block-type from the provider's
|
|
3713
|
+
// Track block-type from the provider's content_block signals so
|
|
3027
3714
|
// every token chunk is tagged with the membrane block it belongs to.
|
|
3028
3715
|
// Without this, thinking_delta chunks get mislabelled as 'text' and
|
|
3029
3716
|
// downstream consumers (TUIs, WebUIs) can't render them distinctly.
|
|
3030
|
-
|
|
3031
|
-
|
|
3032
|
-
|
|
3033
|
-
if (apiType === 'thinking') return 'thinking';
|
|
3034
|
-
if (apiType === 'tool_use') return 'tool_call';
|
|
3035
|
-
return 'text';
|
|
3036
|
-
};
|
|
3717
|
+
const tracker = new NativeBlockTracker(
|
|
3718
|
+
emitBlocks ? (event) => stream.emit({ type: 'block', event }) : undefined,
|
|
3719
|
+
);
|
|
3037
3720
|
const streamResult = await this.streamOnce(
|
|
3038
3721
|
providerRequest,
|
|
3039
3722
|
{
|
|
@@ -3045,54 +3728,16 @@ export class Membrane {
|
|
|
3045
3728
|
|
|
3046
3729
|
if (emitTokens) {
|
|
3047
3730
|
const meta: ChunkMeta = {
|
|
3048
|
-
type:
|
|
3049
|
-
visible:
|
|
3050
|
-
blockIndex,
|
|
3731
|
+
type: tracker.currentType,
|
|
3732
|
+
visible: tracker.currentType === 'text',
|
|
3733
|
+
blockIndex: tracker.blockIndex,
|
|
3051
3734
|
};
|
|
3052
3735
|
stream.emit({ type: 'tokens', content: chunk, meta });
|
|
3053
3736
|
}
|
|
3054
3737
|
},
|
|
3055
3738
|
onContentBlock: (index, block) => {
|
|
3056
3739
|
if (stream.isCancelled) return;
|
|
3057
|
-
|
|
3058
|
-
const mbType = mapApiBlockType(apiType);
|
|
3059
|
-
const isStart = !seenBlockIndices.has(index);
|
|
3060
|
-
if (isStart) {
|
|
3061
|
-
seenBlockIndices.add(index);
|
|
3062
|
-
currentBlockType = mbType;
|
|
3063
|
-
blockIndex = index;
|
|
3064
|
-
if (emitBlocks) {
|
|
3065
|
-
stream.emit({
|
|
3066
|
-
type: 'block',
|
|
3067
|
-
event: { event: 'block_start', index, block: { type: mbType } },
|
|
3068
|
-
});
|
|
3069
|
-
}
|
|
3070
|
-
} else if (emitBlocks) {
|
|
3071
|
-
// Second call for the same index = content_block_stop. The
|
|
3072
|
-
// provider has filled the block with final content; surface
|
|
3073
|
-
// a block_complete with the relevant fields for consumers
|
|
3074
|
-
// that want full block payloads (e.g. context-manager).
|
|
3075
|
-
const apiBlock = block as {
|
|
3076
|
-
type?: string;
|
|
3077
|
-
text?: string;
|
|
3078
|
-
thinking?: string;
|
|
3079
|
-
id?: string;
|
|
3080
|
-
name?: string;
|
|
3081
|
-
input?: unknown;
|
|
3082
|
-
} | undefined;
|
|
3083
|
-
const mb: MembraneBlock = { type: mbType };
|
|
3084
|
-
if (mbType === 'text') mb.content = apiBlock?.text;
|
|
3085
|
-
else if (mbType === 'thinking') mb.content = apiBlock?.thinking;
|
|
3086
|
-
else if (mbType === 'tool_call') {
|
|
3087
|
-
mb.toolId = apiBlock?.id;
|
|
3088
|
-
mb.toolName = apiBlock?.name;
|
|
3089
|
-
mb.input = apiBlock?.input as Record<string, unknown> | undefined;
|
|
3090
|
-
}
|
|
3091
|
-
stream.emit({
|
|
3092
|
-
type: 'block',
|
|
3093
|
-
event: { event: 'block_complete', index, block: mb },
|
|
3094
|
-
});
|
|
3095
|
-
}
|
|
3740
|
+
tracker.onProviderBlock(index, block);
|
|
3096
3741
|
},
|
|
3097
3742
|
},
|
|
3098
3743
|
{
|
|
@@ -3101,6 +3746,14 @@ export class Membrane {
|
|
|
3101
3746
|
idleTimeoutMs: options.idleTimeoutMs,
|
|
3102
3747
|
normalizedRequest: request,
|
|
3103
3748
|
onRequest: (req: unknown) => { rawRequest = req; },
|
|
3749
|
+
// Telemetry reports what this request actually SHIPPED with —
|
|
3750
|
+
// builder breakpoints, stale passthrough, fallback, float, plus
|
|
3751
|
+
// whatever the beforeRequest hook and the wire clamp did after
|
|
3752
|
+
// the build. Both native paths used to hardcode 0, and counting
|
|
3753
|
+
// at build time reported a number no request ever had.
|
|
3754
|
+
onWireCacheMarkers: (markerCount: number) => {
|
|
3755
|
+
markersInLastRequest = markerCount;
|
|
3756
|
+
},
|
|
3104
3757
|
refusalRetries: options.refusalRetries,
|
|
3105
3758
|
// Discard the refused attempt: roll the accumulators back to
|
|
3106
3759
|
// where this attempt began and tell the consumer to drop what it
|
|
@@ -3109,9 +3762,7 @@ export class Membrane {
|
|
|
3109
3762
|
onRetrying: (info) => {
|
|
3110
3763
|
allTextAccumulated = allTextAccumulated.slice(0, allTextBefore);
|
|
3111
3764
|
textAccumulated = '';
|
|
3112
|
-
|
|
3113
|
-
currentBlockType = 'text';
|
|
3114
|
-
seenBlockIndices.clear();
|
|
3765
|
+
tracker.reset();
|
|
3115
3766
|
stream.emit({
|
|
3116
3767
|
type: 'retrying',
|
|
3117
3768
|
attempt: info.attempt,
|
|
@@ -3123,22 +3774,24 @@ export class Membrane {
|
|
|
3123
3774
|
}
|
|
3124
3775
|
);
|
|
3125
3776
|
|
|
3777
|
+
// Single-callback adapters (OpenAI Responses) report each finalised
|
|
3778
|
+
// block once, after the stream: complete whatever never saw a stop.
|
|
3779
|
+
tracker.flush();
|
|
3780
|
+
rounds++;
|
|
3781
|
+
providerCalls += streamResult.providerCalls;
|
|
3782
|
+
|
|
3126
3783
|
rawResponse = streamResult.raw;
|
|
3127
3784
|
lastStopReason = this.mapStopReason(streamResult.stopReason);
|
|
3128
3785
|
lastStopSequence = streamResult.stopSequence ?? undefined;
|
|
3129
3786
|
|
|
3130
|
-
//
|
|
3131
|
-
|
|
3132
|
-
|
|
3133
|
-
|
|
3134
|
-
|
|
3135
|
-
|
|
3136
|
-
if (streamResult.usage.cacheReadTokens) {
|
|
3137
|
-
totalUsage.cacheReadTokens = (totalUsage.cacheReadTokens ?? 0) + streamResult.usage.cacheReadTokens;
|
|
3138
|
-
}
|
|
3139
|
-
if (pricing) totalUsage.estimatedCost = calculateCost(totalUsage, pricing);
|
|
3787
|
+
// Attempts this round re-issued past a refusal are billed calls whose
|
|
3788
|
+
// output was discarded — carry their spend to the final response.
|
|
3789
|
+
discardedUsage = this.mergeDiscardedAttempts(discardedUsage, streamResult.discardedUsage);
|
|
3790
|
+
|
|
3791
|
+
// Accumulate usage (including cache metrics), priced at this round's model
|
|
3792
|
+
const usageSoFar = turnUsage.addRound(streamResult.model, streamResult.usage);
|
|
3140
3793
|
if (emitUsage) {
|
|
3141
|
-
stream.emit({ type: 'usage', usage:
|
|
3794
|
+
stream.emit({ type: 'usage', usage: usageSoFar });
|
|
3142
3795
|
}
|
|
3143
3796
|
|
|
3144
3797
|
// Parse content blocks from response
|
|
@@ -3241,6 +3894,7 @@ export class Membrane {
|
|
|
3241
3894
|
}
|
|
3242
3895
|
|
|
3243
3896
|
const durationMs = Date.now() - startTime;
|
|
3897
|
+
const totalUsage = turnUsage.total;
|
|
3244
3898
|
|
|
3245
3899
|
const response: NormalizedResponse = {
|
|
3246
3900
|
content: allContentBlocks,
|
|
@@ -3255,18 +3909,25 @@ export class Membrane {
|
|
|
3255
3909
|
triggeredSequence: lastStopSequence,
|
|
3256
3910
|
wasTruncated: lastStopReason === 'max_tokens',
|
|
3257
3911
|
},
|
|
3258
|
-
usage: {
|
|
3912
|
+
usage: {
|
|
3913
|
+
...totalUsage,
|
|
3914
|
+
...(discardedUsage
|
|
3915
|
+
? { discardedAttempts: this.pricedDiscardedAttempts(discardedUsage, request.config.model) }
|
|
3916
|
+
: {}),
|
|
3917
|
+
},
|
|
3259
3918
|
timing: {
|
|
3260
3919
|
totalDurationMs: durationMs,
|
|
3261
|
-
attempts:
|
|
3920
|
+
attempts: providerCalls,
|
|
3921
|
+
rounds,
|
|
3262
3922
|
},
|
|
3263
3923
|
model: {
|
|
3264
3924
|
requested: request.config.model,
|
|
3265
|
-
actual: request.config.model,
|
|
3925
|
+
actual: turnUsage.lastServedModel || request.config.model,
|
|
3266
3926
|
provider: this.adapter.name,
|
|
3927
|
+
perRound: turnUsage.perRound,
|
|
3267
3928
|
},
|
|
3268
3929
|
cache: {
|
|
3269
|
-
markersInRequest:
|
|
3930
|
+
markersInRequest: markersInLastRequest,
|
|
3270
3931
|
tokensCreated: totalUsage.cacheCreationTokens ?? 0,
|
|
3271
3932
|
tokensRead: totalUsage.cacheReadTokens ?? 0,
|
|
3272
3933
|
hitRatio: this.calculateCacheHitRatio(totalUsage),
|
|
@@ -3283,7 +3944,7 @@ export class Membrane {
|
|
|
3283
3944
|
if (this.isAbortError(error)) {
|
|
3284
3945
|
stream.emit({
|
|
3285
3946
|
type: 'aborted',
|
|
3286
|
-
reason:
|
|
3947
|
+
reason: this.abortReason(error, stream.signal),
|
|
3287
3948
|
rawAssistantText: allTextAccumulated,
|
|
3288
3949
|
toolCalls: executedToolCalls,
|
|
3289
3950
|
toolResults: executedToolResults,
|