@animalabs/membrane 0.5.79 → 0.5.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cache-keepalive.d.ts +115 -0
- package/dist/cache-keepalive.d.ts.map +1 -0
- package/dist/cache-keepalive.js +0 -0
- package/dist/cache-keepalive.js.map +1 -0
- package/dist/cache-keepalive.test.d.ts +2 -0
- package/dist/cache-keepalive.test.d.ts.map +1 -0
- package/dist/cache-keepalive.test.js +206 -0
- package/dist/cache-keepalive.test.js.map +1 -0
- package/dist/context/index.d.ts +1 -1
- package/dist/context/index.d.ts.map +1 -1
- package/dist/context/index.js +1 -1
- package/dist/context/index.js.map +1 -1
- package/dist/context/process.d.ts +3 -1
- package/dist/context/process.d.ts.map +1 -1
- package/dist/context/process.js +230 -49
- package/dist/context/process.js.map +1 -1
- package/dist/context/types.d.ts +30 -0
- package/dist/context/types.d.ts.map +1 -1
- package/dist/context/types.js +16 -0
- package/dist/context/types.js.map +1 -1
- package/dist/floating-cache-marker.test.d.ts +2 -0
- package/dist/floating-cache-marker.test.d.ts.map +1 -0
- package/dist/floating-cache-marker.test.js +263 -0
- package/dist/floating-cache-marker.test.js.map +1 -0
- package/dist/formatters/anthropic-xml.d.ts +2 -0
- package/dist/formatters/anthropic-xml.d.ts.map +1 -1
- package/dist/formatters/anthropic-xml.js +57 -19
- package/dist/formatters/anthropic-xml.js.map +1 -1
- package/dist/formatters/normalize-tool-pairs.d.ts +7 -0
- package/dist/formatters/normalize-tool-pairs.d.ts.map +1 -1
- package/dist/formatters/normalize-tool-pairs.js +45 -0
- package/dist/formatters/normalize-tool-pairs.js.map +1 -1
- package/dist/formatters/types.d.ts +29 -0
- package/dist/formatters/types.d.ts.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/membrane.d.ts +142 -7
- package/dist/membrane.d.ts.map +1 -1
- package/dist/membrane.js +881 -270
- package/dist/membrane.js.map +1 -1
- package/dist/providers/anthropic.d.ts +37 -0
- package/dist/providers/anthropic.d.ts.map +1 -1
- package/dist/providers/anthropic.js +163 -26
- package/dist/providers/anthropic.js.map +1 -1
- package/dist/providers/bedrock.d.ts +8 -0
- package/dist/providers/bedrock.d.ts.map +1 -1
- package/dist/providers/bedrock.js +24 -1
- package/dist/providers/bedrock.js.map +1 -1
- package/dist/providers/gemini.d.ts +11 -0
- package/dist/providers/gemini.d.ts.map +1 -1
- package/dist/providers/gemini.js +104 -82
- package/dist/providers/gemini.js.map +1 -1
- package/dist/providers/mock.d.ts +2 -0
- package/dist/providers/mock.d.ts.map +1 -1
- package/dist/providers/mock.js +2 -0
- package/dist/providers/mock.js.map +1 -1
- package/dist/providers/openai-compatible.d.ts +7 -0
- package/dist/providers/openai-compatible.d.ts.map +1 -1
- package/dist/providers/openai-compatible.js +99 -48
- package/dist/providers/openai-compatible.js.map +1 -1
- package/dist/providers/openai-completions.d.ts +5 -0
- package/dist/providers/openai-completions.d.ts.map +1 -1
- package/dist/providers/openai-completions.js +107 -44
- package/dist/providers/openai-completions.js.map +1 -1
- package/dist/providers/openai-responses-api.d.ts +16 -0
- package/dist/providers/openai-responses-api.d.ts.map +1 -1
- package/dist/providers/openai-responses-api.js +38 -10
- package/dist/providers/openai-responses-api.js.map +1 -1
- package/dist/providers/openai-responses.d.ts +2 -0
- package/dist/providers/openai-responses.d.ts.map +1 -1
- package/dist/providers/openai-responses.js +8 -1
- package/dist/providers/openai-responses.js.map +1 -1
- package/dist/providers/openai.d.ts +7 -0
- package/dist/providers/openai.d.ts.map +1 -1
- package/dist/providers/openai.js +92 -44
- package/dist/providers/openai.js.map +1 -1
- package/dist/providers/openrouter.d.ts +9 -0
- package/dist/providers/openrouter.d.ts.map +1 -1
- package/dist/providers/openrouter.js +118 -62
- package/dist/providers/openrouter.js.map +1 -1
- package/dist/providers/utils.d.ts +74 -2
- package/dist/providers/utils.d.ts.map +1 -1
- package/dist/providers/utils.js +184 -3
- package/dist/providers/utils.js.map +1 -1
- package/dist/registry/default-pricing.d.ts +22 -0
- package/dist/registry/default-pricing.d.ts.map +1 -1
- package/dist/registry/default-pricing.js +162 -28
- package/dist/registry/default-pricing.js.map +1 -1
- package/dist/types/config.d.ts +5 -0
- package/dist/types/config.d.ts.map +1 -1
- package/dist/types/config.js.map +1 -1
- package/dist/types/content.d.ts +13 -0
- package/dist/types/content.d.ts.map +1 -1
- package/dist/types/content.js.map +1 -1
- package/dist/types/errors.d.ts +15 -0
- package/dist/types/errors.d.ts.map +1 -1
- package/dist/types/errors.js +26 -0
- package/dist/types/errors.js.map +1 -1
- package/dist/types/index.d.ts +3 -3
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/index.js +1 -1
- package/dist/types/index.js.map +1 -1
- package/dist/types/provider.d.ts +50 -0
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/request.d.ts +13 -0
- package/dist/types/request.d.ts.map +1 -1
- package/dist/types/response.d.ts +85 -4
- package/dist/types/response.d.ts.map +1 -1
- package/dist/types/response.js.map +1 -1
- package/dist/types/streaming.d.ts +6 -1
- package/dist/types/streaming.d.ts.map +1 -1
- package/dist/utils/cache-marker-budget.d.ts +82 -0
- package/dist/utils/cache-marker-budget.d.ts.map +1 -0
- package/dist/utils/cache-marker-budget.js +189 -0
- package/dist/utils/cache-marker-budget.js.map +1 -0
- package/dist/utils/cost.d.ts +22 -0
- package/dist/utils/cost.d.ts.map +1 -1
- package/dist/utils/cost.js +55 -0
- package/dist/utils/cost.js.map +1 -1
- package/dist/utils/thinking-carriers.d.ts +80 -0
- package/dist/utils/thinking-carriers.d.ts.map +1 -0
- package/dist/utils/thinking-carriers.js +115 -0
- package/dist/utils/thinking-carriers.js.map +1 -0
- package/dist/utils/tool-parser.d.ts +24 -0
- package/dist/utils/tool-parser.d.ts.map +1 -1
- package/dist/utils/tool-parser.js +367 -161
- package/dist/utils/tool-parser.js.map +1 -1
- package/dist/utils/usage.d.ts +71 -0
- package/dist/utils/usage.d.ts.map +1 -0
- package/dist/utils/usage.js +161 -0
- package/dist/utils/usage.js.map +1 -0
- package/dist/yielding-stream.d.ts +8 -0
- package/dist/yielding-stream.d.ts.map +1 -1
- package/dist/yielding-stream.js +49 -3
- package/dist/yielding-stream.js.map +1 -1
- package/package.json +3 -2
- package/src/cache-keepalive.test.ts +244 -0
- package/src/cache-keepalive.ts +385 -0
- package/src/context/index.ts +1 -0
- package/src/context/process.ts +305 -59
- package/src/context/types.ts +38 -0
- package/src/floating-cache-marker.test.ts +285 -0
- package/src/formatters/anthropic-xml.ts +62 -20
- package/src/formatters/normalize-tool-pairs.ts +49 -0
- package/src/formatters/types.ts +32 -0
- package/src/index.ts +13 -0
- package/src/membrane.ts +1031 -279
- package/src/providers/anthropic.ts +205 -20
- package/src/providers/bedrock.ts +25 -1
- package/src/providers/gemini.ts +125 -87
- package/src/providers/mock.ts +3 -0
- package/src/providers/openai-compatible.ts +102 -45
- package/src/providers/openai-completions.ts +112 -46
- package/src/providers/openai-responses-api.ts +45 -15
- package/src/providers/openai-responses.ts +8 -1
- package/src/providers/openai.ts +95 -42
- package/src/providers/openrouter.ts +125 -63
- package/src/providers/utils.ts +227 -6
- package/src/registry/default-pricing.ts +165 -28
- package/src/types/config.ts +6 -0
- package/src/types/content.ts +14 -0
- package/src/types/errors.ts +28 -0
- package/src/types/index.ts +6 -0
- package/src/types/provider.ts +55 -0
- package/src/types/request.ts +14 -0
- package/src/types/response.ts +109 -8
- package/src/types/streaming.ts +8 -2
- package/src/utils/cache-marker-budget.ts +208 -0
- package/src/utils/cost.ts +64 -0
- package/src/utils/thinking-carriers.ts +125 -0
- package/src/utils/tool-parser.ts +473 -174
- package/src/utils/usage.ts +184 -0
- package/src/yielding-stream.ts +54 -3
package/src/membrane.ts
CHANGED
|
@@ -10,12 +10,14 @@ import type {
|
|
|
10
10
|
AbortedResponse,
|
|
11
11
|
ContentBlock,
|
|
12
12
|
ProviderAdapter,
|
|
13
|
+
ProviderResponse,
|
|
13
14
|
ModelRegistry,
|
|
14
15
|
MembraneConfig,
|
|
15
16
|
StreamOptions,
|
|
16
17
|
CompleteOptions,
|
|
17
18
|
BasicUsage,
|
|
18
19
|
DetailedUsage,
|
|
20
|
+
DiscardedAttemptsUsage,
|
|
19
21
|
StopReason,
|
|
20
22
|
TimingInfo,
|
|
21
23
|
CacheInfo,
|
|
@@ -23,23 +25,37 @@ import type {
|
|
|
23
25
|
ToolResult,
|
|
24
26
|
ToolContext,
|
|
25
27
|
RetryConfig,
|
|
26
|
-
ToolMode,
|
|
27
28
|
ToolDefinition,
|
|
28
29
|
} from './types/index.js';
|
|
29
30
|
import { lastCacheableBlockIndex } from './formatters/native.js';
|
|
31
|
+
import {
|
|
32
|
+
sameThinkingText,
|
|
33
|
+
findSpanningProviderRun,
|
|
34
|
+
thinkingCarrierKey,
|
|
35
|
+
stripThinkingForPrefill,
|
|
36
|
+
} from './utils/thinking-carriers.js';
|
|
37
|
+
import {
|
|
38
|
+
countWireCacheMarkers,
|
|
39
|
+
clampCacheMarkers,
|
|
40
|
+
ownSystemBlocks,
|
|
41
|
+
MAX_CACHE_BREAKPOINTS,
|
|
42
|
+
} from './utils/cache-marker-budget.js';
|
|
30
43
|
import {
|
|
31
44
|
DEFAULT_RETRY_CONFIG,
|
|
32
45
|
MembraneError,
|
|
33
46
|
classifyError,
|
|
34
47
|
isOverloadedError,
|
|
48
|
+
isTimeoutAbortError,
|
|
35
49
|
isTextContent,
|
|
36
50
|
isAbortedResponse,
|
|
51
|
+
unsupportedError,
|
|
37
52
|
} from './types/index.js';
|
|
38
53
|
import type { BuildResult } from './formatters/types.js';
|
|
39
54
|
import {
|
|
40
55
|
parseToolCalls,
|
|
41
56
|
formatToolResults,
|
|
42
57
|
parseAccumulatedIntoBlocks,
|
|
58
|
+
endsWithPartialToolBlock,
|
|
43
59
|
hasImageInToolResults,
|
|
44
60
|
formatToolResultsForSplitTurn,
|
|
45
61
|
type ProviderImageBlock,
|
|
@@ -54,9 +70,19 @@ import type {
|
|
|
54
70
|
} from './types/yielding-stream.js';
|
|
55
71
|
import type { PrefillFormatter, StreamParser } from './formatters/types.js';
|
|
56
72
|
import { AnthropicXmlFormatter } from './formatters/anthropic-xml.js';
|
|
57
|
-
import {
|
|
73
|
+
import {
|
|
74
|
+
normalizeToolPairs,
|
|
75
|
+
mergeConsecutiveRoles,
|
|
76
|
+
PREFIX_REWRITING_NORMALIZE_EVENT_KINDS,
|
|
77
|
+
} from './formatters/normalize-tool-pairs.js';
|
|
58
78
|
import { YieldingStreamImpl } from './yielding-stream.js';
|
|
59
|
-
import { calculateCost } from './utils/cost.js';
|
|
79
|
+
import { calculateCost, warnUnpricedModel } from './utils/cost.js';
|
|
80
|
+
import {
|
|
81
|
+
TurnUsageAccumulator,
|
|
82
|
+
calculateCacheHitRatio,
|
|
83
|
+
normalizeUsageToCacheExcluded,
|
|
84
|
+
warnUnconvertibleProviderItem,
|
|
85
|
+
} from './utils/usage.js';
|
|
60
86
|
import {
|
|
61
87
|
isAcceptedImageMediaType,
|
|
62
88
|
strippedImagePlaceholder,
|
|
@@ -68,6 +94,94 @@ import { getDefaultPricing } from './registry/default-pricing.js';
|
|
|
68
94
|
// Membrane Class
|
|
69
95
|
// ============================================================================
|
|
70
96
|
|
|
97
|
+
/**
|
|
98
|
+
* Block-lifecycle tracking shared by the two native-tools streaming paths
|
|
99
|
+
* (`streamWithNativeTools` and `runNativeToolsYielding`).
|
|
100
|
+
*
|
|
101
|
+
* Providers signal blocks through `onContentBlock(index, block)`, but not all
|
|
102
|
+
* of them the same way: the Anthropic and Bedrock adapters fire it twice per
|
|
103
|
+
* index (content_block_start with an empty block, content_block_stop with the
|
|
104
|
+
* finalised one), while the OpenAI Responses adapter fires it ONCE per block,
|
|
105
|
+
* already finalised, after the stream has ended. Treating "second sighting"
|
|
106
|
+
* as the only completion signal therefore left single-callback adapters with
|
|
107
|
+
* `block_start` events that never completed (#63 review). The tracker keeps
|
|
108
|
+
* the paired semantics and adds `flush()`, which the caller runs once the
|
|
109
|
+
* provider stream has returned: every started block that never saw a second
|
|
110
|
+
* callback is completed from the last block payload seen for it.
|
|
111
|
+
*/
|
|
112
|
+
class NativeBlockTracker {
|
|
113
|
+
currentType: MembraneBlockType = 'text';
|
|
114
|
+
blockIndex = 0;
|
|
115
|
+
private readonly started = new Map<number, MembraneBlockType>();
|
|
116
|
+
private readonly completed = new Set<number>();
|
|
117
|
+
private readonly lastSeen = new Map<number, unknown>();
|
|
118
|
+
|
|
119
|
+
constructor(private readonly emit: ((event: BlockEvent) => void) | undefined) {}
|
|
120
|
+
|
|
121
|
+
static mapApiBlockType(apiType: string | undefined): MembraneBlockType {
|
|
122
|
+
if (apiType === 'thinking' || apiType === 'redacted_thinking' || apiType === 'reasoning') return 'thinking';
|
|
123
|
+
if (apiType === 'tool_use' || apiType === 'function_call' || apiType === 'tool_call') return 'tool_call';
|
|
124
|
+
return 'text';
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** Provider block callback: first sighting of an index starts it, a second completes it. */
|
|
128
|
+
onProviderBlock(index: number, block: unknown): void {
|
|
129
|
+
this.lastSeen.set(index, block);
|
|
130
|
+
if (!this.started.has(index)) {
|
|
131
|
+
const mbType = NativeBlockTracker.mapApiBlockType((block as { type?: string } | undefined)?.type);
|
|
132
|
+
this.started.set(index, mbType);
|
|
133
|
+
this.currentType = mbType;
|
|
134
|
+
this.blockIndex = index;
|
|
135
|
+
this.emit?.({ event: 'block_start', index, block: { type: mbType } });
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
this.complete(index, block);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Complete every started block that never received its second callback.
|
|
143
|
+
* Run after the provider stream has returned; idempotent, and a no-op for
|
|
144
|
+
* paired-callback adapters.
|
|
145
|
+
*/
|
|
146
|
+
flush(): void {
|
|
147
|
+
for (const index of this.started.keys()) {
|
|
148
|
+
if (!this.completed.has(index)) this.complete(index, this.lastSeen.get(index));
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** Discard tracking state (refusal retry rolled the attempt back). */
|
|
153
|
+
reset(): void {
|
|
154
|
+
this.currentType = 'text';
|
|
155
|
+
this.blockIndex = 0;
|
|
156
|
+
this.started.clear();
|
|
157
|
+
this.completed.clear();
|
|
158
|
+
this.lastSeen.clear();
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
private complete(index: number, block: unknown): void {
|
|
162
|
+
if (this.completed.has(index)) return;
|
|
163
|
+
this.completed.add(index);
|
|
164
|
+
const mbType = this.started.get(index)
|
|
165
|
+
?? NativeBlockTracker.mapApiBlockType((block as { type?: string } | undefined)?.type);
|
|
166
|
+
const apiBlock = block as {
|
|
167
|
+
text?: string;
|
|
168
|
+
thinking?: string;
|
|
169
|
+
id?: string;
|
|
170
|
+
name?: string;
|
|
171
|
+
input?: unknown;
|
|
172
|
+
} | undefined;
|
|
173
|
+
const mb: MembraneBlock = { type: mbType };
|
|
174
|
+
if (mbType === 'text') mb.content = apiBlock?.text;
|
|
175
|
+
else if (mbType === 'thinking') mb.content = apiBlock?.thinking;
|
|
176
|
+
else if (mbType === 'tool_call') {
|
|
177
|
+
mb.toolId = apiBlock?.id;
|
|
178
|
+
mb.toolName = apiBlock?.name;
|
|
179
|
+
mb.input = apiBlock?.input as Record<string, unknown> | undefined;
|
|
180
|
+
}
|
|
181
|
+
this.emit?.({ event: 'block_complete', index, block: mb });
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
71
185
|
export class Membrane {
|
|
72
186
|
private adapter: ProviderAdapter;
|
|
73
187
|
private registry?: ModelRegistry;
|
|
@@ -109,12 +223,20 @@ export class Membrane {
|
|
|
109
223
|
// refusal is a successful HTTP call with an unwanted verdict, and letting
|
|
110
224
|
// it consume error retries would couple two unrelated budgets.
|
|
111
225
|
let refusalRetriesUsed = 0;
|
|
226
|
+
// Spend on attempts we threw away. A refused attempt is a completed,
|
|
227
|
+
// billed HTTP call; reporting only the surviving attempt's usage
|
|
228
|
+
// under-reports the turn by one full call per retry.
|
|
229
|
+
let discardedUsage: DiscardedAttemptsUsage | undefined;
|
|
230
|
+
|
|
231
|
+
// One selection for the whole call: mode resolution and the build must
|
|
232
|
+
// name the same formatter instance (see resolveActiveFormatter).
|
|
233
|
+
const activeFormatter = this.resolveActiveFormatter(options.formatter);
|
|
112
234
|
|
|
113
235
|
while (true) {
|
|
114
236
|
attempts++;
|
|
115
237
|
|
|
116
238
|
try {
|
|
117
|
-
const { providerRequest, prefillResult } = this.transformRequest(request,
|
|
239
|
+
const { providerRequest, prefillResult } = this.transformRequest(request, activeFormatter);
|
|
118
240
|
|
|
119
241
|
// Route through the single canonical hook helper so any future
|
|
120
242
|
// change to hook semantics (logging, retry interaction, error
|
|
@@ -123,7 +245,11 @@ export class Membrane {
|
|
|
123
245
|
// `unknown` deliberately, and we acknowledge the cast at the boundary.
|
|
124
246
|
const finalRequest = (await this.applyBeforeRequestHook(request, providerRequest)) as typeof providerRequest;
|
|
125
247
|
|
|
126
|
-
|
|
248
|
+
// Last exit before the adapter: the only place that sees EVERY
|
|
249
|
+
// contribution (builder, formatter, passthrough, float, hook).
|
|
250
|
+
clampCacheMarkers(finalRequest, 'complete');
|
|
251
|
+
|
|
252
|
+
const rawProviderResponse = await this.adapter.complete(finalRequest, {
|
|
127
253
|
signal: options.signal,
|
|
128
254
|
timeoutMs: options.timeoutMs,
|
|
129
255
|
onRequest: (req) => {
|
|
@@ -131,6 +257,15 @@ export class Membrane {
|
|
|
131
257
|
options.onRequest?.(req);
|
|
132
258
|
},
|
|
133
259
|
});
|
|
260
|
+
// Restate usage in the one convention before any ratio or price sees it.
|
|
261
|
+
const providerResponse: ProviderResponse = {
|
|
262
|
+
...rawProviderResponse,
|
|
263
|
+
usage: normalizeUsageToCacheExcluded(
|
|
264
|
+
rawProviderResponse.usage,
|
|
265
|
+
this.adapter.name,
|
|
266
|
+
this.adapter.usageCacheConvention,
|
|
267
|
+
),
|
|
268
|
+
};
|
|
134
269
|
|
|
135
270
|
// Call onResponse callback with raw response from API
|
|
136
271
|
options.onResponse?.(providerResponse.raw);
|
|
@@ -154,9 +289,21 @@ export class Membrane {
|
|
|
154
289
|
refusalRetriesUsed < Math.max(0, options.refusalRetries ?? 0)
|
|
155
290
|
) {
|
|
156
291
|
refusalRetriesUsed++;
|
|
292
|
+
discardedUsage = this.mergeDiscardedAttempts(
|
|
293
|
+
discardedUsage,
|
|
294
|
+
this.discardedAttemptFrom(response.usage)
|
|
295
|
+
);
|
|
157
296
|
continue;
|
|
158
297
|
}
|
|
159
298
|
|
|
299
|
+
// Report what the discarded attempts cost. Set BEFORE afterResponse
|
|
300
|
+
// so a hook that logs spend sees the whole turn, not just the
|
|
301
|
+
// attempt that stands.
|
|
302
|
+
if (discardedUsage) {
|
|
303
|
+
response.details.usage.discardedAttempts =
|
|
304
|
+
this.pricedDiscardedAttempts(discardedUsage, request.config.model);
|
|
305
|
+
}
|
|
306
|
+
|
|
160
307
|
// Call afterResponse hook
|
|
161
308
|
if (this.config.hooks?.afterResponse) {
|
|
162
309
|
return await this.config.hooks.afterResponse(response, providerResponse.raw);
|
|
@@ -195,9 +342,16 @@ export class Membrane {
|
|
|
195
342
|
}
|
|
196
343
|
}
|
|
197
344
|
|
|
198
|
-
// Wait before retry (abort-aware)
|
|
345
|
+
// Wait before retry (abort-aware). An abort landing inside the
|
|
346
|
+
// sleep must fail like every other failure of this method — a
|
|
347
|
+
// MembraneError — rather than escaping the loop as a raw
|
|
348
|
+
// DOMException whose shape no caller of complete() expects.
|
|
199
349
|
const delay = this.calculateRetryDelay(attempts, isOverloaded);
|
|
200
|
-
|
|
350
|
+
try {
|
|
351
|
+
await this.sleep(delay, options.signal);
|
|
352
|
+
} catch (sleepError) {
|
|
353
|
+
throw this.attachRawRequest(sleepError, rawRequest);
|
|
354
|
+
}
|
|
201
355
|
continue;
|
|
202
356
|
}
|
|
203
357
|
|
|
@@ -231,6 +385,20 @@ export class Membrane {
|
|
|
231
385
|
// If streaming is explicitly disabled on the request, fall back to complete()
|
|
232
386
|
// and synthesize the streaming callbacks from the full response
|
|
233
387
|
if (request.streaming === false) {
|
|
388
|
+
// complete() has no tool loop, and neither branch of this fallback can
|
|
389
|
+
// build one: honouring onToolCalls here would mean re-implementing the
|
|
390
|
+
// whole XML/native continuation machinery. Silently dropping it turned
|
|
391
|
+
// a working agent into one that narrates tool calls it never makes —
|
|
392
|
+
// the raw <function_calls> XML lands in the returned text and the turn
|
|
393
|
+
// ends. Refuse where the option is passed, before spending a call.
|
|
394
|
+
if (options.onToolCalls) {
|
|
395
|
+
throw unsupportedError(
|
|
396
|
+
'stream() cannot execute tools with streaming: false — the non-streaming ' +
|
|
397
|
+
'fallback routes to complete(), which has no tool loop, so onToolCalls ' +
|
|
398
|
+
'would never run. Leave streaming enabled (or drive the loop yourself ' +
|
|
399
|
+
'with complete() per round).'
|
|
400
|
+
);
|
|
401
|
+
}
|
|
234
402
|
const response = await this.complete(request, options);
|
|
235
403
|
// Synthesize onChunk callbacks so callers that depend on them still work
|
|
236
404
|
if (options.onChunk && 'content' in response) {
|
|
@@ -248,8 +416,9 @@ export class Membrane {
|
|
|
248
416
|
return response;
|
|
249
417
|
}
|
|
250
418
|
|
|
251
|
-
// Determine tool mode
|
|
252
|
-
const
|
|
419
|
+
// Determine tool mode against the formatter that will build the request
|
|
420
|
+
const activeFormatter = this.resolveActiveFormatter(options.formatter);
|
|
421
|
+
const toolMode = this.resolveToolMode(request, activeFormatter);
|
|
253
422
|
const useNative = toolMode === 'native' && !!request.tools && request.tools.length > 0;
|
|
254
423
|
|
|
255
424
|
// Overloaded (529) pre-emission retry. The streaming paths have no retry
|
|
@@ -279,14 +448,16 @@ export class Membrane {
|
|
|
279
448
|
|
|
280
449
|
try {
|
|
281
450
|
const result = useNative
|
|
282
|
-
? await this.streamWithNativeTools(request, tracked)
|
|
283
|
-
: await this.streamWithXmlTools(request, tracked);
|
|
284
|
-
// The inner paths
|
|
285
|
-
//
|
|
286
|
-
//
|
|
287
|
-
//
|
|
451
|
+
? await this.streamWithNativeTools(request, tracked, activeFormatter)
|
|
452
|
+
: await this.streamWithXmlTools(request, tracked, activeFormatter);
|
|
453
|
+
// The inner paths count their own provider calls but cannot see this
|
|
454
|
+
// wrapper's discarded attempts. Each failed attempt here died before
|
|
455
|
+
// emitting anything (that is the precondition for retrying), so it
|
|
456
|
+
// cost at least the one call it failed on — ADD those to the inner
|
|
457
|
+
// count rather than overwriting it, or a turn that retried twice and
|
|
458
|
+
// then ran three tool rounds would report 2 calls instead of 5.
|
|
288
459
|
if (attempts > 1 && 'details' in result) {
|
|
289
|
-
result.details.timing.attempts
|
|
460
|
+
result.details.timing.attempts += attempts - 1;
|
|
290
461
|
result.details.timing.retryDelaysMs = retryDelaysMs;
|
|
291
462
|
}
|
|
292
463
|
return result;
|
|
@@ -313,7 +484,26 @@ export class Membrane {
|
|
|
313
484
|
}
|
|
314
485
|
const delay = this.calculateRetryDelay(attempts, true);
|
|
315
486
|
retryDelaysMs.push(delay);
|
|
316
|
-
|
|
487
|
+
// An abort during the backoff window is still a cancellation of
|
|
488
|
+
// this stream, and stream() documents cancellation as an
|
|
489
|
+
// AbortedResponse. Letting the sleep's rejection escape made that
|
|
490
|
+
// contract depend on which millisecond the abort landed in.
|
|
491
|
+
// Nothing has been emitted on this path (that is the precondition
|
|
492
|
+
// for retrying at all), so there is no partial content to report.
|
|
493
|
+
try {
|
|
494
|
+
await this.sleep(delay, options.signal);
|
|
495
|
+
} catch (sleepError) {
|
|
496
|
+
if (this.isAbortError(sleepError)) {
|
|
497
|
+
return this.buildAbortedResponse(
|
|
498
|
+
'',
|
|
499
|
+
{ inputTokens: 0, outputTokens: 0 },
|
|
500
|
+
[],
|
|
501
|
+
[],
|
|
502
|
+
this.abortReason(sleepError, options.signal)
|
|
503
|
+
);
|
|
504
|
+
}
|
|
505
|
+
throw sleepError;
|
|
506
|
+
}
|
|
317
507
|
continue;
|
|
318
508
|
}
|
|
319
509
|
throw error;
|
|
@@ -322,18 +512,68 @@ export class Membrane {
|
|
|
322
512
|
}
|
|
323
513
|
|
|
324
514
|
/**
|
|
325
|
-
*
|
|
515
|
+
* Select the ACTIVE formatter for a request: the one instance that resolves
|
|
516
|
+
* its tool mode, builds its provider request, and parses its stream.
|
|
517
|
+
*
|
|
518
|
+
* A per-request override (`CompleteOptions.formatter` /
|
|
519
|
+
* `StreamOptions.formatter`) wins over the instance formatter, with ONE
|
|
520
|
+
* transport exception: the Responses adapter's input is a provider-native
|
|
521
|
+
* item array, and a generic override (for example Context Manager's
|
|
522
|
+
* NativeFormatter) produces Anthropic-style `{ role, content: [{ type:
|
|
523
|
+
* 'text' }] }` envelopes the Responses API rejects before inference — so a
|
|
524
|
+
* configured Responses formatter stays authoritative there.
|
|
525
|
+
*
|
|
526
|
+
* The exception is why this selection is a method rather than a `??` at each
|
|
527
|
+
* call site: while it lived inside transformRequest alone, the BUILD honored
|
|
528
|
+
* it and every other formatter reader resolved against a different instance,
|
|
529
|
+
* which is the split resolveToolMode exists to prevent, one layer down.
|
|
530
|
+
* Every entry point selects once, here, and threads the result.
|
|
531
|
+
*/
|
|
532
|
+
private resolveActiveFormatter(requestFormatter?: PrefillFormatter): PrefillFormatter {
|
|
533
|
+
if (this.adapter.name === 'openai-responses-api' && this.formatter.name === 'openai-responses') {
|
|
534
|
+
return this.formatter;
|
|
535
|
+
}
|
|
536
|
+
return requestFormatter ?? this.formatter;
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
/**
|
|
540
|
+
* Determine the effective tool mode.
|
|
541
|
+
*
|
|
542
|
+
* THE single source of truth for the mode: both complete() (via
|
|
543
|
+
* transformRequest → BuildOptions.toolMode) and the streaming paths (via
|
|
544
|
+
* their native-vs-XML path choice) resolve here, so a given request resolves
|
|
545
|
+
* to the same mode whichever entry point it arrives through.
|
|
546
|
+
*
|
|
547
|
+
* Precedence, strongest first:
|
|
548
|
+
* 1. an explicit non-'auto' `request.toolMode`
|
|
549
|
+
* 2. the mode the BUILDING formatter was explicitly constructed with
|
|
550
|
+
* (`AnthropicXmlFormatter({ toolMode: 'native' })`) — a caller's stated
|
|
551
|
+
* choice, not a derivation
|
|
552
|
+
* 3. formatter/provider derivation
|
|
553
|
+
*
|
|
554
|
+
* `formatter` is the formatter that will actually build the request — the
|
|
555
|
+
* instance `resolveActiveFormatter` selected for this call — because
|
|
556
|
+
* resolving against one formatter while building with another is exactly the
|
|
557
|
+
* split this method exists to prevent.
|
|
326
558
|
*/
|
|
327
|
-
private resolveToolMode(
|
|
559
|
+
private resolveToolMode(
|
|
560
|
+
request: NormalizedRequest,
|
|
561
|
+
formatter: PrefillFormatter = this.formatter
|
|
562
|
+
): 'xml' | 'native' {
|
|
328
563
|
// Explicit mode takes precedence
|
|
329
564
|
if (request.toolMode && request.toolMode !== 'auto') {
|
|
330
565
|
return request.toolMode;
|
|
331
566
|
}
|
|
332
567
|
|
|
568
|
+
// A formatter constructed with an explicit mode states its caller's choice
|
|
569
|
+
if (formatter.configuredToolMode) {
|
|
570
|
+
return formatter.configuredToolMode;
|
|
571
|
+
}
|
|
572
|
+
|
|
333
573
|
// Auto mode: choose based on formatter
|
|
334
574
|
// NativeFormatter → native tools via API
|
|
335
575
|
// AnthropicXmlFormatter (default) → XML tools in prefill
|
|
336
|
-
if (
|
|
576
|
+
if (formatter.name === 'native' || formatter.name === 'openai-responses') {
|
|
337
577
|
return 'native';
|
|
338
578
|
}
|
|
339
579
|
|
|
@@ -355,7 +595,8 @@ export class Membrane {
|
|
|
355
595
|
*/
|
|
356
596
|
private async streamWithXmlTools(
|
|
357
597
|
request: NormalizedRequest,
|
|
358
|
-
options: StreamOptions
|
|
598
|
+
options: StreamOptions,
|
|
599
|
+
activeFormatter: PrefillFormatter = this.resolveActiveFormatter(options.formatter)
|
|
359
600
|
): Promise<NormalizedResponse | AbortedResponse> {
|
|
360
601
|
const startTime = Date.now();
|
|
361
602
|
const {
|
|
@@ -369,17 +610,30 @@ export class Membrane {
|
|
|
369
610
|
onResponse,
|
|
370
611
|
maxToolDepth = 10,
|
|
371
612
|
signal,
|
|
372
|
-
|
|
613
|
+
timeoutMs,
|
|
614
|
+
idleTimeoutMs,
|
|
373
615
|
} = options;
|
|
374
616
|
|
|
375
|
-
//
|
|
376
|
-
|
|
617
|
+
// The formatter stream() selected: the same instance that resolved the
|
|
618
|
+
// mode and will build the request, so the parser can never be reading a
|
|
619
|
+
// different format than the one on the wire.
|
|
620
|
+
const formatter = activeFormatter;
|
|
377
621
|
|
|
378
622
|
// Initialize parser from formatter for format-specific tracking
|
|
379
623
|
const parser = formatter.createStreamParser();
|
|
380
624
|
let toolDepth = 0;
|
|
381
|
-
|
|
382
|
-
|
|
625
|
+
// Each round is priced under the model that served THAT round and the
|
|
626
|
+
// costs are summed: a routed turn can change models mid-turn, and pricing
|
|
627
|
+
// the whole accumulated usage at the latest rate re-bills every earlier
|
|
628
|
+
// round at a price it was never charged.
|
|
629
|
+
const turnUsage = new TurnUsageAccumulator(
|
|
630
|
+
request.config.model,
|
|
631
|
+
(servedModel) => this.resolvePricing(request.config.model, servedModel),
|
|
632
|
+
);
|
|
633
|
+
// Honest turn telemetry: provider calls actually made (including refusal
|
|
634
|
+
// re-issues inside streamOnce) and continuation rounds.
|
|
635
|
+
let providerCalls = 0;
|
|
636
|
+
let rounds = 0;
|
|
383
637
|
const contentBlocks: ContentBlock[] = [];
|
|
384
638
|
let lastStopReason: StopReason = 'end_turn';
|
|
385
639
|
let lastStopSequence: string | undefined;
|
|
@@ -463,7 +717,7 @@ export class Membrane {
|
|
|
463
717
|
if (resumptionRounds === RESUMPTION_WARN_ROUNDS) {
|
|
464
718
|
warnLog.warn(
|
|
465
719
|
`[membrane] automatic resumption at round ${resumptionRounds} ` +
|
|
466
|
-
`(${
|
|
720
|
+
`(${turnUsage.total.inputTokens} input tokens so far this turn) — ` +
|
|
467
721
|
`a spin shows up here before it shows up on the bill`
|
|
468
722
|
);
|
|
469
723
|
}
|
|
@@ -471,7 +725,7 @@ export class Membrane {
|
|
|
471
725
|
warnLog.warn(
|
|
472
726
|
`[membrane] automatic resumption cap (${maxResumptionRounds}) reached — ` +
|
|
473
727
|
`ending turn with stopReason 'round_limit'. ` +
|
|
474
|
-
`${
|
|
728
|
+
`${turnUsage.total.inputTokens} input tokens spent this turn.`
|
|
475
729
|
);
|
|
476
730
|
return false;
|
|
477
731
|
}
|
|
@@ -547,6 +801,8 @@ export class Membrane {
|
|
|
547
801
|
},
|
|
548
802
|
{
|
|
549
803
|
signal,
|
|
804
|
+
timeoutMs,
|
|
805
|
+
idleTimeoutMs,
|
|
550
806
|
normalizedRequest: request,
|
|
551
807
|
// The tag-based parser tracks thinking via <thinking> tags — ask the
|
|
552
808
|
// provider to wrap native thinking deltas so they don't stream as
|
|
@@ -559,6 +815,9 @@ export class Membrane {
|
|
|
559
815
|
}
|
|
560
816
|
);
|
|
561
817
|
|
|
818
|
+
rounds++;
|
|
819
|
+
providerCalls += streamResult.providerCalls;
|
|
820
|
+
|
|
562
821
|
// If we detected stop sequence manually, fix up the parser and result
|
|
563
822
|
if (detectedStopSequence && truncatedAccumulated !== null) {
|
|
564
823
|
parser.reset();
|
|
@@ -593,17 +852,11 @@ export class Membrane {
|
|
|
593
852
|
lastStopReason = this.mapStopReason(streamResult.stopReason);
|
|
594
853
|
lastStopSequence = streamResult.stopSequence ?? undefined;
|
|
595
854
|
|
|
596
|
-
// Accumulate usage (including cache metrics)
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
}
|
|
602
|
-
if (streamResult.usage.cacheReadTokens) {
|
|
603
|
-
totalUsage.cacheReadTokens = (totalUsage.cacheReadTokens ?? 0) + streamResult.usage.cacheReadTokens;
|
|
604
|
-
}
|
|
605
|
-
if (pricing) totalUsage.estimatedCost = calculateCost(totalUsage, pricing);
|
|
606
|
-
onUsage?.(totalUsage);
|
|
855
|
+
// Accumulate usage (including cache metrics), priced at this round's
|
|
856
|
+
// model. NOT inlined into the optional call — `onUsage?.(addRound())`
|
|
857
|
+
// skips evaluating its argument entirely when no callback is set.
|
|
858
|
+
const usageSoFar = turnUsage.addRound(streamResult.model, streamResult.usage);
|
|
859
|
+
onUsage?.(usageSoFar);
|
|
607
860
|
|
|
608
861
|
// Flush the parser to complete any in-progress streaming block
|
|
609
862
|
const flushResult = parser.flush();
|
|
@@ -637,7 +890,7 @@ export class Membrane {
|
|
|
637
890
|
`[membrane] ${consecutiveStalledResumptions} consecutive automatic resumptions ` +
|
|
638
891
|
`made no progress (${streamedThisRound} chars this round, stop ` +
|
|
639
892
|
`${JSON.stringify(lastStopSequence ?? null)} repeated) — ending turn with ` +
|
|
640
|
-
`stopReason 'no_progress'. ${
|
|
893
|
+
`stopReason 'no_progress'. ${turnUsage.total.inputTokens} input tokens spent this turn.`
|
|
641
894
|
);
|
|
642
895
|
lastStopReason = 'no_progress';
|
|
643
896
|
break;
|
|
@@ -919,17 +1172,17 @@ export class Membrane {
|
|
|
919
1172
|
newContent,
|
|
920
1173
|
contentBlocks,
|
|
921
1174
|
lastStopReason,
|
|
922
|
-
|
|
1175
|
+
turnUsage,
|
|
923
1176
|
request,
|
|
924
1177
|
prefillResult,
|
|
925
1178
|
startTime,
|
|
926
|
-
|
|
1179
|
+
providerCalls,
|
|
927
1180
|
rawRequest,
|
|
928
1181
|
rawResponse,
|
|
929
1182
|
executedToolCalls,
|
|
930
1183
|
executedToolResults,
|
|
931
1184
|
initialBlockType,
|
|
932
|
-
lastStopSequence
|
|
1185
|
+
lastStopSequence,
|
|
933
1186
|
);
|
|
934
1187
|
|
|
935
1188
|
// Append non-text content blocks (e.g., generated_image) that the XML parser can't handle
|
|
@@ -940,6 +1193,8 @@ export class Membrane {
|
|
|
940
1193
|
// Merge provider thinking signatures into parser-derived thinking blocks
|
|
941
1194
|
this.mergeProviderThinkingBlocks(response.content, providerThinkingBlocks);
|
|
942
1195
|
|
|
1196
|
+
response.details.timing.rounds = rounds;
|
|
1197
|
+
|
|
943
1198
|
return response;
|
|
944
1199
|
} catch (error) {
|
|
945
1200
|
// Check if this is an abort error
|
|
@@ -950,10 +1205,10 @@ export class Membrane {
|
|
|
950
1205
|
|
|
951
1206
|
return this.buildAbortedResponse(
|
|
952
1207
|
newContent,
|
|
953
|
-
|
|
1208
|
+
turnUsage.total,
|
|
954
1209
|
executedToolCalls,
|
|
955
1210
|
executedToolResults,
|
|
956
|
-
|
|
1211
|
+
this.abortReason(error, signal),
|
|
957
1212
|
initialBlockType
|
|
958
1213
|
);
|
|
959
1214
|
}
|
|
@@ -967,12 +1222,14 @@ export class Membrane {
|
|
|
967
1222
|
*/
|
|
968
1223
|
private async streamWithNativeTools(
|
|
969
1224
|
request: NormalizedRequest,
|
|
970
|
-
options: StreamOptions
|
|
1225
|
+
options: StreamOptions,
|
|
1226
|
+
activeFormatter: PrefillFormatter = this.resolveActiveFormatter(options.formatter)
|
|
971
1227
|
): Promise<NormalizedResponse | AbortedResponse> {
|
|
972
1228
|
const startTime = Date.now();
|
|
973
1229
|
const {
|
|
974
1230
|
onChunk,
|
|
975
1231
|
onContentBlockUpdate,
|
|
1232
|
+
onBlock,
|
|
976
1233
|
onToolCalls,
|
|
977
1234
|
onPreToolContent,
|
|
978
1235
|
onUsage,
|
|
@@ -980,11 +1237,21 @@ export class Membrane {
|
|
|
980
1237
|
onResponse,
|
|
981
1238
|
maxToolDepth = 10,
|
|
982
1239
|
signal,
|
|
1240
|
+
timeoutMs,
|
|
1241
|
+
idleTimeoutMs,
|
|
983
1242
|
} = options;
|
|
984
1243
|
|
|
985
1244
|
let toolDepth = 0;
|
|
986
|
-
|
|
987
|
-
|
|
1245
|
+
// See streamWithXmlTools: one accumulator per turn, pricing each round
|
|
1246
|
+
// under the model that served it.
|
|
1247
|
+
const turnUsage = new TurnUsageAccumulator(
|
|
1248
|
+
request.config.model,
|
|
1249
|
+
(servedModel) => this.resolvePricing(request.config.model, servedModel),
|
|
1250
|
+
);
|
|
1251
|
+
// Honest turn telemetry: provider calls actually made (including refusal
|
|
1252
|
+
// re-issues inside streamOnce) and continuation rounds.
|
|
1253
|
+
let providerCalls = 0;
|
|
1254
|
+
let rounds = 0;
|
|
988
1255
|
let lastStopReason: StopReason = 'end_turn';
|
|
989
1256
|
let lastStopSequence: string | undefined;
|
|
990
1257
|
let rawRequest: unknown;
|
|
@@ -1000,45 +1267,67 @@ export class Membrane {
|
|
|
1000
1267
|
// Build messages array that we'll update with tool results
|
|
1001
1268
|
let messages = [...request.messages];
|
|
1002
1269
|
let allContentBlocks: ContentBlock[] = [];
|
|
1270
|
+
let markersInLastRequest = 0;
|
|
1003
1271
|
|
|
1004
1272
|
try {
|
|
1005
1273
|
// Tool execution loop
|
|
1006
1274
|
while (toolDepth <= maxToolDepth) {
|
|
1007
1275
|
// Build provider request with native tools
|
|
1008
|
-
const providerRequest = this.buildNativeToolRequest(request, messages);
|
|
1276
|
+
const providerRequest = this.buildNativeToolRequest(request, messages, toolDepth > 0, activeFormatter);
|
|
1009
1277
|
|
|
1010
1278
|
// Stream from provider
|
|
1011
1279
|
let textAccumulated = '';
|
|
1012
|
-
|
|
1280
|
+
// Tag every token chunk with the membrane block it belongs to and
|
|
1281
|
+
// surface the block lifecycle through onBlock — the same shape
|
|
1282
|
+
// runNativeToolsYielding uses (#19). Before this, meta.type was
|
|
1283
|
+
// hardcoded to 'text' on every chunk and onBlock was never invoked
|
|
1284
|
+
// from this path (#20).
|
|
1285
|
+
const tracker = new NativeBlockTracker(onBlock ? (event) => onBlock(event) : undefined);
|
|
1013
1286
|
const streamResult = await this.streamOnce(
|
|
1014
1287
|
providerRequest,
|
|
1015
1288
|
{
|
|
1016
1289
|
onChunk: (chunk) => {
|
|
1017
1290
|
textAccumulated += chunk;
|
|
1018
1291
|
allTextAccumulated += chunk;
|
|
1019
|
-
// For native mode, emit text chunks with basic metadata
|
|
1020
|
-
// TODO: Use native API content_block events for richer metadata
|
|
1021
1292
|
const meta: ChunkMeta = {
|
|
1022
|
-
type:
|
|
1023
|
-
visible:
|
|
1024
|
-
blockIndex,
|
|
1293
|
+
type: tracker.currentType,
|
|
1294
|
+
visible: tracker.currentType === 'text',
|
|
1295
|
+
blockIndex: tracker.blockIndex,
|
|
1025
1296
|
};
|
|
1026
1297
|
onChunk?.(chunk, meta);
|
|
1027
1298
|
},
|
|
1028
|
-
onContentBlock:
|
|
1029
|
-
|
|
1030
|
-
|
|
1299
|
+
onContentBlock: (index: number, block: unknown) => {
|
|
1300
|
+
tracker.onProviderBlock(index, block);
|
|
1301
|
+
// Deprecated pass-through, kept for callers still on it.
|
|
1302
|
+
onContentBlockUpdate?.(index, block as ContentBlock);
|
|
1303
|
+
},
|
|
1031
1304
|
},
|
|
1032
1305
|
{
|
|
1033
1306
|
signal,
|
|
1307
|
+
timeoutMs,
|
|
1308
|
+
idleTimeoutMs,
|
|
1034
1309
|
normalizedRequest: request,
|
|
1035
1310
|
onRequest: (req) => {
|
|
1036
1311
|
rawRequest = req;
|
|
1037
1312
|
onRequest?.(req);
|
|
1038
1313
|
},
|
|
1314
|
+
// Telemetry reports what this request actually SHIPPED with —
|
|
1315
|
+
// builder breakpoints, stale passthrough, fallback, float, plus
|
|
1316
|
+
// whatever the beforeRequest hook and the wire clamp did after
|
|
1317
|
+
// the build. Both native paths used to hardcode 0, and counting
|
|
1318
|
+
// at build time reported a number no request ever had.
|
|
1319
|
+
onWireCacheMarkers: (markerCount) => {
|
|
1320
|
+
markersInLastRequest = markerCount;
|
|
1321
|
+
},
|
|
1039
1322
|
}
|
|
1040
1323
|
);
|
|
1041
1324
|
|
|
1325
|
+
// Single-callback adapters (OpenAI Responses) report each finalised
|
|
1326
|
+
// block once, after the stream: complete whatever never saw a stop.
|
|
1327
|
+
tracker.flush();
|
|
1328
|
+
rounds++;
|
|
1329
|
+
providerCalls += streamResult.providerCalls;
|
|
1330
|
+
|
|
1042
1331
|
rawResponse = streamResult.raw;
|
|
1043
1332
|
|
|
1044
1333
|
// Call onResponse callback with raw response from API
|
|
@@ -1047,17 +1336,11 @@ export class Membrane {
|
|
|
1047
1336
|
lastStopReason = this.mapStopReason(streamResult.stopReason);
|
|
1048
1337
|
lastStopSequence = streamResult.stopSequence ?? undefined;
|
|
1049
1338
|
|
|
1050
|
-
// Accumulate usage (including cache metrics)
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
}
|
|
1056
|
-
if (streamResult.usage.cacheReadTokens) {
|
|
1057
|
-
totalUsage.cacheReadTokens = (totalUsage.cacheReadTokens ?? 0) + streamResult.usage.cacheReadTokens;
|
|
1058
|
-
}
|
|
1059
|
-
if (pricing) totalUsage.estimatedCost = calculateCost(totalUsage, pricing);
|
|
1060
|
-
onUsage?.(totalUsage);
|
|
1339
|
+
// Accumulate usage (including cache metrics), priced at this round's
|
|
1340
|
+
// model. NOT inlined into the optional call — `onUsage?.(addRound())`
|
|
1341
|
+
// skips evaluating its argument entirely when no callback is set.
|
|
1342
|
+
const usageSoFar = turnUsage.addRound(streamResult.model, streamResult.usage);
|
|
1343
|
+
onUsage?.(usageSoFar);
|
|
1061
1344
|
|
|
1062
1345
|
// Parse content blocks from response
|
|
1063
1346
|
const responseBlocks = this.parseProviderContent(streamResult.content);
|
|
@@ -1145,6 +1428,7 @@ export class Membrane {
|
|
|
1145
1428
|
}
|
|
1146
1429
|
|
|
1147
1430
|
const durationMs = Date.now() - startTime;
|
|
1431
|
+
const totalUsage = turnUsage.total;
|
|
1148
1432
|
|
|
1149
1433
|
return {
|
|
1150
1434
|
content: allContentBlocks,
|
|
@@ -1162,15 +1446,17 @@ export class Membrane {
|
|
|
1162
1446
|
usage: { ...totalUsage },
|
|
1163
1447
|
timing: {
|
|
1164
1448
|
totalDurationMs: durationMs,
|
|
1165
|
-
attempts:
|
|
1449
|
+
attempts: providerCalls,
|
|
1450
|
+
rounds,
|
|
1166
1451
|
},
|
|
1167
1452
|
model: {
|
|
1168
1453
|
requested: request.config.model,
|
|
1169
|
-
actual: request.config.model,
|
|
1454
|
+
actual: turnUsage.lastServedModel || request.config.model,
|
|
1170
1455
|
provider: this.adapter.name,
|
|
1456
|
+
perRound: turnUsage.perRound,
|
|
1171
1457
|
},
|
|
1172
1458
|
cache: {
|
|
1173
|
-
markersInRequest:
|
|
1459
|
+
markersInRequest: markersInLastRequest,
|
|
1174
1460
|
tokensCreated: totalUsage.cacheCreationTokens ?? 0,
|
|
1175
1461
|
tokensRead: totalUsage.cacheReadTokens ?? 0,
|
|
1176
1462
|
hitRatio: this.calculateCacheHitRatio(totalUsage),
|
|
@@ -1186,10 +1472,10 @@ export class Membrane {
|
|
|
1186
1472
|
if (this.isAbortError(error)) {
|
|
1187
1473
|
return this.buildAbortedResponse(
|
|
1188
1474
|
allTextAccumulated,
|
|
1189
|
-
|
|
1475
|
+
turnUsage.total,
|
|
1190
1476
|
executedToolCalls,
|
|
1191
1477
|
executedToolResults,
|
|
1192
|
-
|
|
1478
|
+
this.abortReason(error, signal)
|
|
1193
1479
|
);
|
|
1194
1480
|
}
|
|
1195
1481
|
// Re-throw with rawRequest attached for logging
|
|
@@ -1198,18 +1484,62 @@ export class Membrane {
|
|
|
1198
1484
|
}
|
|
1199
1485
|
|
|
1200
1486
|
/**
|
|
1201
|
-
*
|
|
1487
|
+
* Rate-limit state for the float's budget warning. See the
|
|
1488
|
+
* floating-cache-marker block in buildNativeToolRequest.
|
|
1489
|
+
*
|
|
1490
|
+
* A once-per-instance latch made the ONLY observable of an over-budget wire
|
|
1491
|
+
* go quiet for the life of the process: a long-lived Membrane warns for the
|
|
1492
|
+
* first agent that trips it and never again, so the condition looks like it
|
|
1493
|
+
* healed. Warn on the first occurrence, then at most once per interval,
|
|
1494
|
+
* carrying the count of what was suppressed in between.
|
|
1495
|
+
*/
|
|
1496
|
+
private floatBudgetWarnState = { lastWarnedAtMs: 0, suppressedSinceWarn: 0 };
|
|
1497
|
+
private static readonly FLOAT_BUDGET_WARN_INTERVAL_MS = 60_000;
|
|
1498
|
+
|
|
1499
|
+
private warnFloatBudgetExhausted(wireMarkers: number): void {
|
|
1500
|
+
const now = Date.now();
|
|
1501
|
+
const state = this.floatBudgetWarnState;
|
|
1502
|
+
const elapsed = now - state.lastWarnedAtMs;
|
|
1503
|
+
if (state.lastWarnedAtMs !== 0 && elapsed < Membrane.FLOAT_BUDGET_WARN_INTERVAL_MS) {
|
|
1504
|
+
state.suppressedSinceWarn++;
|
|
1505
|
+
return;
|
|
1506
|
+
}
|
|
1507
|
+
const suppressed = state.suppressedSinceWarn;
|
|
1508
|
+
state.lastWarnedAtMs = now;
|
|
1509
|
+
state.suppressedSinceWarn = 0;
|
|
1510
|
+
console.warn(
|
|
1511
|
+
`[membrane] floating cache marker withheld: upstream markers already ` +
|
|
1512
|
+
`occupy all ${MAX_CACHE_BREAKPOINTS} cache_control slots (${wireMarkers} on the wire). ` +
|
|
1513
|
+
`Tool-round suffixes will not cache incrementally.` +
|
|
1514
|
+
(suppressed > 0 ? ` (${suppressed} further occurrences suppressed since the last warning.)` : '')
|
|
1515
|
+
);
|
|
1516
|
+
}
|
|
1517
|
+
|
|
1518
|
+
/**
|
|
1519
|
+
* Build a provider request with native tool support.
|
|
1520
|
+
*
|
|
1521
|
+
* `toolLoopRebuild` is true when this build is a tool-loop continuation
|
|
1522
|
+
* (toolDepth > 0) rather than the turn's first request — the only case
|
|
1523
|
+
* where the floating cache marker applies.
|
|
1524
|
+
*
|
|
1525
|
+
* `activeFormatter` is the formatter the caller selected for the request
|
|
1526
|
+
* (see resolveActiveFormatter). Reading `this.formatter` here instead made
|
|
1527
|
+
* the native loop build through the instance formatter while the mode had
|
|
1528
|
+
* been resolved against a per-request override — the two disagreeing about
|
|
1529
|
+
* which formatter is active.
|
|
1202
1530
|
*/
|
|
1203
1531
|
private buildNativeToolRequest(
|
|
1204
1532
|
request: NormalizedRequest,
|
|
1205
|
-
messages: typeof request.messages
|
|
1533
|
+
messages: typeof request.messages,
|
|
1534
|
+
toolLoopRebuild = false,
|
|
1535
|
+
activeFormatter: PrefillFormatter = this.formatter
|
|
1206
1536
|
): any {
|
|
1207
1537
|
// Provider-native formatters own their complete input-item shape. The
|
|
1208
1538
|
// legacy implementation below is intentionally Anthropic-specific; using
|
|
1209
1539
|
// it for Responses would normalize away item IDs, encrypted reasoning,
|
|
1210
1540
|
// assistant phases, and compaction items.
|
|
1211
|
-
if (
|
|
1212
|
-
return this.transformRequest({ ...request, messages },
|
|
1541
|
+
if (activeFormatter.name === 'openai-responses') {
|
|
1542
|
+
return this.transformRequest({ ...request, messages }, activeFormatter).providerRequest;
|
|
1213
1543
|
}
|
|
1214
1544
|
|
|
1215
1545
|
// Convert messages to provider format
|
|
@@ -1224,11 +1554,13 @@ export class Membrane {
|
|
|
1224
1554
|
// Anthropic allows at most 4 cache_control breakpoints per request. The
|
|
1225
1555
|
// message breakpoints are the valuable ones (they cache the longest prefixes,
|
|
1226
1556
|
// and every one already includes tools+system at the front of the request).
|
|
1227
|
-
// So tools/system get a breakpoint only as a FALLBACK — when no
|
|
1228
|
-
//
|
|
1229
|
-
// total past 4, which the API hard-rejects (the agent goes
|
|
1230
|
-
|
|
1231
|
-
|
|
1557
|
+
// So tools/system get a breakpoint only as a FALLBACK — when no marker
|
|
1558
|
+
// exists anywhere on the wire — otherwise they're redundant and would push
|
|
1559
|
+
// the total past 4, which the API hard-rejects (the agent goes
|
|
1560
|
+
// unresponsive). The fallback gate reads a RECOUNT of the built artifacts
|
|
1561
|
+
// (see below), never a running tally: a running tally cannot see a
|
|
1562
|
+
// caller-marked system block, and double-counts a message breakpoint that
|
|
1563
|
+
// lands on a block already carrying stale cache_control.
|
|
1232
1564
|
for (const msg of messages) {
|
|
1233
1565
|
const isAssistant = msg.participant === assistantName;
|
|
1234
1566
|
const role = isAssistant ? 'assistant' : 'user';
|
|
@@ -1249,14 +1581,13 @@ export class Membrane {
|
|
|
1249
1581
|
}
|
|
1250
1582
|
const textBlock: Record<string, unknown> = { type: 'text', text };
|
|
1251
1583
|
if ((block as any).cache_control) {
|
|
1252
|
-
textBlock.cache_control = (block as any).cache_control;
|
|
1253
1584
|
// A block-level passthrough occupies one of the 4 breakpoint slots
|
|
1254
|
-
// exactly like a marked message
|
|
1255
|
-
//
|
|
1256
|
-
//
|
|
1257
|
-
//
|
|
1258
|
-
//
|
|
1259
|
-
|
|
1585
|
+
// exactly like a marked message; the recount below sees it.
|
|
1586
|
+
// (Imported/seeded conversations carry stale request-time
|
|
1587
|
+
// cache_control on stored blocks — first seen wedging Sill
|
|
1588
|
+
// 2026-07-25: 3 cm markers + 2 stale Arc-export blocks = 5 → hard
|
|
1589
|
+
// 400 on every inference.)
|
|
1590
|
+
textBlock.cache_control = (block as any).cache_control;
|
|
1260
1591
|
}
|
|
1261
1592
|
content.push(textBlock);
|
|
1262
1593
|
} else if (block.type === 'tool_use') {
|
|
@@ -1328,7 +1659,6 @@ export class Membrane {
|
|
|
1328
1659
|
const bpIdx = lastCacheableBlockIndex(content as Array<Record<string, unknown>>);
|
|
1329
1660
|
if (bpIdx >= 0) {
|
|
1330
1661
|
content[bpIdx].cache_control = cacheControl;
|
|
1331
|
-
messageBreakpoints++;
|
|
1332
1662
|
}
|
|
1333
1663
|
}
|
|
1334
1664
|
|
|
@@ -1354,9 +1684,37 @@ export class Membrane {
|
|
|
1354
1684
|
// tool_results to `messages`. Any unmatched tool_use that reaches
|
|
1355
1685
|
// this splice is upstream stranding (the bug class this fix exists
|
|
1356
1686
|
// to catch) — `[pending]` is exactly the right synthesis.
|
|
1357
|
-
|
|
1687
|
+
// A synthesized [pending] tool_result's bytes are rewritten when the
|
|
1688
|
+
// real result lands — the floating-marker block below must not cache
|
|
1689
|
+
// past one. `synthetic_pending_result` (not the downstream
|
|
1690
|
+
// cache_suppressed_for_synthetic, which only fires when a marker was
|
|
1691
|
+
// actually stripped) is the root condition.
|
|
1692
|
+
// Every repair that REWRITES prefix bytes stands the float down, not just
|
|
1693
|
+
// the synthetic [pending] result: a textified orphan tool_result is
|
|
1694
|
+
// rewritten the same way when its real pairing arrives, so caching at or
|
|
1695
|
+
// past one poisons the prefix identically. The kinds live in one exported
|
|
1696
|
+
// set so a normalizer that grows a new prefix-rewriting repair cannot
|
|
1697
|
+
// silently escape this guard.
|
|
1698
|
+
let prefixRewritten = false;
|
|
1699
|
+
const normalized = normalizeToolPairs(providerMessages, {
|
|
1700
|
+
onEvent: (e) => {
|
|
1701
|
+
if (PREFIX_REWRITING_NORMALIZE_EVENT_KINDS.has(e.kind)) prefixRewritten = true;
|
|
1702
|
+
},
|
|
1703
|
+
});
|
|
1358
1704
|
const mergedMessages = mergeConsecutiveRoles(normalized.messages);
|
|
1359
1705
|
|
|
1706
|
+
// ONE recount of the constructed wire artifacts, taken BEFORE the
|
|
1707
|
+
// tools/system fallback decision so the fallback and the float share a
|
|
1708
|
+
// single truth. Counted post-normalize, so phase-5.5 cache suppression is
|
|
1709
|
+
// already reflected. `request.system` is the caller's own system content:
|
|
1710
|
+
// it explicitly accepts pre-marked blocks, and those are real wire markers
|
|
1711
|
+
// that no running tally ever saw (three of them plus both fallbacks = 5 on
|
|
1712
|
+
// the wire = a 400 on every inference of that config).
|
|
1713
|
+
const upstreamWireMarkers = countWireCacheMarkers({
|
|
1714
|
+
messages: mergedMessages,
|
|
1715
|
+
system: request.system,
|
|
1716
|
+
});
|
|
1717
|
+
|
|
1360
1718
|
// Convert tools to provider format.
|
|
1361
1719
|
// Native tool names must match ^[a-zA-Z0-9_-]{1,128}$ — sanitize colons
|
|
1362
1720
|
// from the module:tool namespace convention. Reversed in parseProviderContent.
|
|
@@ -1368,7 +1726,7 @@ export class Membrane {
|
|
|
1368
1726
|
};
|
|
1369
1727
|
// Cache the tool list (last tool) only as a fallback — a marked message
|
|
1370
1728
|
// breakpoint already caches the tools as part of its prefix.
|
|
1371
|
-
if (cacheControl &&
|
|
1729
|
+
if (cacheControl && upstreamWireMarkers === 0 && request.tools && idx === request.tools.length - 1) {
|
|
1372
1730
|
t.cache_control = cacheControl;
|
|
1373
1731
|
}
|
|
1374
1732
|
return t;
|
|
@@ -1377,16 +1735,74 @@ export class Membrane {
|
|
|
1377
1735
|
// Wrap system prompt with cache_control only as a fallback (no message
|
|
1378
1736
|
// breakpoint marked); otherwise a message breakpoint already caches
|
|
1379
1737
|
// tools+system as part of its prefix.
|
|
1380
|
-
let system: unknown = request.system;
|
|
1381
|
-
if (cacheControl &&
|
|
1738
|
+
let system: unknown = ownSystemBlocks(request.system);
|
|
1739
|
+
if (cacheControl && upstreamWireMarkers === 0 && typeof system === 'string' && system.length > 0) {
|
|
1382
1740
|
system = [{ type: 'text', text: system, cache_control: cacheControl }];
|
|
1383
|
-
} else if (cacheControl &&
|
|
1741
|
+
} else if (cacheControl && upstreamWireMarkers === 0 && Array.isArray(system) && system.length > 0) {
|
|
1384
1742
|
const blocks = system as Record<string, unknown>[];
|
|
1385
1743
|
system = blocks.map((block, idx) =>
|
|
1386
1744
|
idx === blocks.length - 1 ? { ...block, cache_control: cacheControl } : block
|
|
1387
1745
|
);
|
|
1388
1746
|
}
|
|
1389
1747
|
|
|
1748
|
+
// ------------------------------------------------------------------
|
|
1749
|
+
// Floating cache marker: incremental prompt caching inside the native
|
|
1750
|
+
// tool loop. Message breakpoints are placed by the context strategy at
|
|
1751
|
+
// compile time — once per turn — but this builder re-runs on every
|
|
1752
|
+
// tool round with that round's messages appended, so the deepest
|
|
1753
|
+
// upstream marker stays glued to the turn-start snapshot and each
|
|
1754
|
+
// rebuild re-pays the entire appended suffix at full input price
|
|
1755
|
+
// (qa-ops incident, 2026-08-20: two subagents re-sent a suffix growing
|
|
1756
|
+
// to ~118k tokens ~30 times each — ~5.3M uncached tokens in 18 min —
|
|
1757
|
+
// with their one marker sitting on message 2 of 61).
|
|
1758
|
+
//
|
|
1759
|
+
// The tool loop only ever appends, so a marker riding the newest
|
|
1760
|
+
// message yields the intended incremental pattern: each round writes
|
|
1761
|
+
// its delta and cache-reads everything before it.
|
|
1762
|
+
//
|
|
1763
|
+
// Authority contract: the float spends only the RESIDUAL breakpoint
|
|
1764
|
+
// budget (Anthropic allows 4 cache_control including tools/system).
|
|
1765
|
+
// Upstream markers are never displaced or stripped — if they fill all
|
|
1766
|
+
// 4 slots the float is withheld (with a warning) and behavior is
|
|
1767
|
+
// exactly pre-float. With 2+ slots free, the previous round's
|
|
1768
|
+
// endpoint is marked too: a wide parallel-tool round can append more
|
|
1769
|
+
// blocks than the provider's ~20-block backward search covers, which
|
|
1770
|
+
// would orphan the previous round's cache entry behind an unmarked
|
|
1771
|
+
// boundary.
|
|
1772
|
+
//
|
|
1773
|
+
// Skipped when the normalizer synthesized a [pending] tool_result:
|
|
1774
|
+
// those bytes are rewritten when the real result lands, and caching
|
|
1775
|
+
// past them poisons the prefix — the same rationale as the
|
|
1776
|
+
// normalizer's phase 5.5 cache suppression.
|
|
1777
|
+
// ------------------------------------------------------------------
|
|
1778
|
+
const floatingEnabled =
|
|
1779
|
+
request.floatingCacheMarker ?? this.config.defaultFloatingCacheMarker ?? true;
|
|
1780
|
+
if (toolLoopRebuild && floatingEnabled && cacheControl && !prefixRewritten) {
|
|
1781
|
+
// Same recount as the fallback gate, re-taken POST-fallback so the
|
|
1782
|
+
// fallback's own spend is inside the residuum.
|
|
1783
|
+
const wireMarkers = countWireCacheMarkers({ messages: mergedMessages, system, tools });
|
|
1784
|
+
let residuum = MAX_CACHE_BREAKPOINTS - wireMarkers;
|
|
1785
|
+
if (residuum <= 0) {
|
|
1786
|
+
this.warnFloatBudgetExhausted(wireMarkers);
|
|
1787
|
+
} else {
|
|
1788
|
+
// Newest message first; then the previous round's endpoint (two
|
|
1789
|
+
// wire messages back: [..., prevResults, assistant, results]).
|
|
1790
|
+
const targets = [mergedMessages.length - 1, mergedMessages.length - 3];
|
|
1791
|
+
for (const mi of targets) {
|
|
1792
|
+
if (residuum <= 0 || mi < 0) continue;
|
|
1793
|
+
const content = mergedMessages[mi]?.content;
|
|
1794
|
+
if (!Array.isArray(content) || content.length === 0) continue;
|
|
1795
|
+
const bpIdx = lastCacheableBlockIndex(content as Array<Record<string, unknown>>);
|
|
1796
|
+
if (bpIdx < 0) continue;
|
|
1797
|
+
// Already a breakpoint here (e.g. the strategy's own end marker
|
|
1798
|
+
// on the turn's first rebuild) — nothing to add.
|
|
1799
|
+
if ((content[bpIdx] as Record<string, unknown>).cache_control) continue;
|
|
1800
|
+
(content[bpIdx] as Record<string, unknown>).cache_control = cacheControl;
|
|
1801
|
+
residuum--;
|
|
1802
|
+
}
|
|
1803
|
+
}
|
|
1804
|
+
}
|
|
1805
|
+
|
|
1390
1806
|
// Build thinking config for native extended thinking (budget clamped to max_tokens)
|
|
1391
1807
|
// Fable/Mythos models: thinking is always on and unconfigurable; sampling params are removed.
|
|
1392
1808
|
// Sending thinking config or temperature returns a 400 — omit both entirely.
|
|
@@ -1436,6 +1852,9 @@ export class Membrane {
|
|
|
1436
1852
|
id: item.id,
|
|
1437
1853
|
name: unsanitizeToolName(item.name),
|
|
1438
1854
|
input: item.input,
|
|
1855
|
+
// Arguments that never parsed: carry the marker through so a
|
|
1856
|
+
// consumer can refuse the block instead of trusting `input`.
|
|
1857
|
+
...(item.unparseableInput !== undefined ? { unparseableInput: item.unparseableInput } : {}),
|
|
1439
1858
|
...(item.rawItem ? { rawItem: item.rawItem } : {}),
|
|
1440
1859
|
});
|
|
1441
1860
|
} else if (item.type === 'thinking') {
|
|
@@ -1454,14 +1873,23 @@ export class Membrane {
|
|
|
1454
1873
|
data: item.data,
|
|
1455
1874
|
mimeType: item.mimeType,
|
|
1456
1875
|
});
|
|
1457
|
-
} else if (item.rawItem) {
|
|
1876
|
+
} else if (item.rawItem || item.type) {
|
|
1458
1877
|
// Opaque Responses items such as encrypted compaction or custom
|
|
1459
1878
|
// tool records have no normalized ContentBlock equivalent. Retain a
|
|
1460
1879
|
// zero-width carrier so Chronicle and the Responses formatter can
|
|
1461
1880
|
// replay the raw item without surfacing synthetic prompt text.
|
|
1462
1881
|
// Anthropic-bound conversion paths filter these out (empty text
|
|
1463
1882
|
// blocks are a 400 there); the Responses formatter replays rawItem.
|
|
1464
|
-
|
|
1883
|
+
//
|
|
1884
|
+
// An item with a `type` this switch does not know (server_tool_use,
|
|
1885
|
+
// web_search_tool_result, search_result, mcp_tool_use, or whatever a
|
|
1886
|
+
// provider adds next) used to fall out of this chain and vanish. It
|
|
1887
|
+
// gets the same carrier treatment, holding the item itself, plus a
|
|
1888
|
+
// one-time warning so the gap surfaces instead of being inferred
|
|
1889
|
+
// later from missing content.
|
|
1890
|
+
const carriedRawItem = item.rawItem ?? item;
|
|
1891
|
+
if (!item.rawItem) warnUnconvertibleProviderItem(item.type);
|
|
1892
|
+
blocks.push({ type: 'text', text: '', rawItem: carriedRawItem });
|
|
1465
1893
|
}
|
|
1466
1894
|
}
|
|
1467
1895
|
return blocks;
|
|
@@ -1500,10 +1928,32 @@ export class Membrane {
|
|
|
1500
1928
|
|
|
1501
1929
|
/**
|
|
1502
1930
|
* Merge provider thinking signatures into parser-derived thinking blocks
|
|
1503
|
-
*
|
|
1504
|
-
*
|
|
1505
|
-
*
|
|
1506
|
-
*
|
|
1931
|
+
* and prepend any leftover provider blocks — signature-only thinking
|
|
1932
|
+
* (display:'omitted') never appears in the text stream, so the parser
|
|
1933
|
+
* produces no block for it. redacted_thinking blocks are always prepended
|
|
1934
|
+
* verbatim.
|
|
1935
|
+
*
|
|
1936
|
+
* Pairing is by CONTENT IDENTITY, never by index. The two lists are
|
|
1937
|
+
* differently shaped whenever the provider emits a block the parser cannot
|
|
1938
|
+
* see (signature-only), the parser emits a block the provider never
|
|
1939
|
+
* produced (the XML path's literal `Claude: <thinking>` prefill turns
|
|
1940
|
+
* VISIBLE text into a thinking block), or one provider block spans several
|
|
1941
|
+
* (auto-continuation: capture runs per round while the parser sees the
|
|
1942
|
+
* CONCATENATED accumulation). Index-zipping crosses the lists in all three
|
|
1943
|
+
* shapes and stamps a signature onto content that never produced it —
|
|
1944
|
+
* which round-trips into the consumer's stored history and fails Anthropic
|
|
1945
|
+
* signature validation on the next turn.
|
|
1946
|
+
*
|
|
1947
|
+
* The three rules, in order:
|
|
1948
|
+
* 1. identity — a provider block pairs with the parsed block whose
|
|
1949
|
+
* thinking text is the same; empty-thinking (signature-only) blocks
|
|
1950
|
+
* are never text-match candidates and are prepend-only.
|
|
1951
|
+
* 2. span — a parsed block that reconstructs as the concatenation of a
|
|
1952
|
+
* RUN of consecutive unpaired provider blocks is REPLACED in place by
|
|
1953
|
+
* those originals, so the spanning block never wears a fragment's
|
|
1954
|
+
* signature and no reasoning is sent twice.
|
|
1955
|
+
* 3. leftover — everything still unpaired is prepended, de-duplicated
|
|
1956
|
+
* against what `content` already carries (and against itself).
|
|
1507
1957
|
*
|
|
1508
1958
|
* Mutates `content` in place. Shared by the XML stream paths
|
|
1509
1959
|
* (streamWithXmlTools and runXmlToolsYielding).
|
|
@@ -1514,25 +1964,74 @@ export class Membrane {
|
|
|
1514
1964
|
): void {
|
|
1515
1965
|
if (providerThinkingBlocks.length === 0) return;
|
|
1516
1966
|
|
|
1517
|
-
const
|
|
1967
|
+
const providerThinking = providerThinkingBlocks.filter(
|
|
1518
1968
|
(b) => b.type === 'thinking'
|
|
1519
|
-
) as Array<{ type: 'thinking'; thinking
|
|
1520
|
-
|
|
1521
|
-
const providerThinking = providerThinkingBlocks.filter((b) => b.type === 'thinking');
|
|
1969
|
+
) as Array<{ type: 'thinking'; thinking?: string; signature?: string }>;
|
|
1522
1970
|
const redacted = providerThinkingBlocks.filter((b) => b.type === 'redacted_thinking');
|
|
1523
1971
|
|
|
1524
|
-
const
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1972
|
+
const pairedProviderBlocks = new Set<number>();
|
|
1973
|
+
const claimedParsedIndices = new Set<number>();
|
|
1974
|
+
const parsedThinkingIndices = () =>
|
|
1975
|
+
content.reduce<number[]>((acc, block, index) => {
|
|
1976
|
+
if (block.type === 'thinking') acc.push(index);
|
|
1977
|
+
return acc;
|
|
1978
|
+
}, []);
|
|
1979
|
+
|
|
1980
|
+
for (let p = 0; p < providerThinking.length; p++) {
|
|
1981
|
+
const providerText = providerThinking[p]!.thinking ?? '';
|
|
1982
|
+
if (providerText === '') continue;
|
|
1983
|
+
const match = parsedThinkingIndices().find(
|
|
1984
|
+
(index) =>
|
|
1985
|
+
!claimedParsedIndices.has(index) &&
|
|
1986
|
+
sameThinkingText((content[index] as { thinking?: string }).thinking ?? '', providerText)
|
|
1987
|
+
);
|
|
1988
|
+
if (match === undefined) continue;
|
|
1989
|
+
const signature = providerThinking[p]!.signature;
|
|
1990
|
+
if (signature) (content[match] as { signature?: string }).signature = signature;
|
|
1991
|
+
claimedParsedIndices.add(match);
|
|
1992
|
+
pairedProviderBlocks.add(p);
|
|
1993
|
+
}
|
|
1994
|
+
|
|
1995
|
+
for (const parsedIndex of parsedThinkingIndices().reverse()) {
|
|
1996
|
+
if (claimedParsedIndices.has(parsedIndex)) continue;
|
|
1997
|
+
const parsedText = (content[parsedIndex] as { thinking?: string }).thinking ?? '';
|
|
1998
|
+
if (parsedText === '') continue;
|
|
1999
|
+
const run = findSpanningProviderRun(providerThinking, pairedProviderBlocks, parsedText);
|
|
2000
|
+
if (!run) continue;
|
|
2001
|
+
content.splice(
|
|
2002
|
+
parsedIndex,
|
|
2003
|
+
1,
|
|
2004
|
+
...run.map((p) => {
|
|
2005
|
+
pairedProviderBlocks.add(p);
|
|
2006
|
+
const block = providerThinking[p]!;
|
|
2007
|
+
return {
|
|
2008
|
+
type: 'thinking',
|
|
2009
|
+
thinking: block.thinking ?? '',
|
|
2010
|
+
...(block.signature ? { signature: block.signature } : {}),
|
|
2011
|
+
} as ContentBlock;
|
|
2012
|
+
})
|
|
2013
|
+
);
|
|
2014
|
+
claimedParsedIndices.add(parsedIndex);
|
|
1530
2015
|
}
|
|
1531
2016
|
|
|
1532
|
-
const
|
|
1533
|
-
|
|
1534
|
-
|
|
2017
|
+
const seen = new Set(content.map((block) => thinkingCarrierKey(block)));
|
|
2018
|
+
const leftover: ContentBlock[] = [];
|
|
2019
|
+
for (let p = 0; p < providerThinking.length; p++) {
|
|
2020
|
+
if (pairedProviderBlocks.has(p)) continue;
|
|
2021
|
+
const block = providerThinking[p]! as unknown as ContentBlock;
|
|
2022
|
+
const key = thinkingCarrierKey(block);
|
|
2023
|
+
if (seen.has(key)) continue;
|
|
2024
|
+
seen.add(key);
|
|
2025
|
+
leftover.push(block);
|
|
2026
|
+
}
|
|
2027
|
+
for (const block of redacted) {
|
|
2028
|
+
const key = thinkingCarrierKey(block);
|
|
2029
|
+
if (seen.has(key)) continue;
|
|
2030
|
+
seen.add(key);
|
|
2031
|
+
leftover.push(block);
|
|
1535
2032
|
}
|
|
2033
|
+
|
|
2034
|
+
if (leftover.length > 0) content.unshift(...leftover);
|
|
1536
2035
|
}
|
|
1537
2036
|
|
|
1538
2037
|
// ==========================================================================
|
|
@@ -1620,24 +2119,18 @@ export class Membrane {
|
|
|
1620
2119
|
}
|
|
1621
2120
|
|
|
1622
2121
|
/**
|
|
1623
|
-
* Transform a normalized request into provider format using the formatter
|
|
2122
|
+
* Transform a normalized request into provider format using the formatter.
|
|
2123
|
+
*
|
|
2124
|
+
* `activeFormatter` is the instance the caller already selected via
|
|
2125
|
+
* resolveActiveFormatter — including that selection's Responses-transport
|
|
2126
|
+
* authority rule, which used to live inline here. It is a parameter and not
|
|
2127
|
+
* a re-derivation so that the formatter which BUILDS is the same one that
|
|
2128
|
+
* resolved the tool mode and drives the loop.
|
|
1624
2129
|
*/
|
|
1625
|
-
private transformRequest(request: NormalizedRequest,
|
|
2130
|
+
private transformRequest(request: NormalizedRequest, activeFormatter: PrefillFormatter = this.formatter): {
|
|
1626
2131
|
providerRequest: any;
|
|
1627
2132
|
prefillResult: BuildResult;
|
|
1628
2133
|
} {
|
|
1629
|
-
// The Responses adapter's input is a provider-native item array. A generic
|
|
1630
|
-
// per-request formatter (for example Context Manager's NativeFormatter)
|
|
1631
|
-
// produces Anthropic-style `{ role, content: [{ type: 'text' }] }`
|
|
1632
|
-
// envelopes, which the Responses API rejects before inference. Keep the
|
|
1633
|
-
// configured Responses formatter authoritative at this transport boundary;
|
|
1634
|
-
// per-request formatter overrides remain available for adapters whose wire
|
|
1635
|
-
// format supports them.
|
|
1636
|
-
const activeFormatter =
|
|
1637
|
-
this.adapter.name === 'openai-responses-api' && this.formatter.name === 'openai-responses'
|
|
1638
|
-
? this.formatter
|
|
1639
|
-
: formatter ?? this.formatter;
|
|
1640
|
-
|
|
1641
2134
|
// Extract user-provided stop sequences
|
|
1642
2135
|
const additionalStopSequences = Array.isArray(request.stopSequences)
|
|
1643
2136
|
? request.stopSequences
|
|
@@ -1653,6 +2146,10 @@ export class Membrane {
|
|
|
1653
2146
|
participantMode: 'multiuser',
|
|
1654
2147
|
assistantParticipant: request.assistantParticipant ?? this.config.assistantParticipant ?? 'Claude',
|
|
1655
2148
|
tools: request.tools,
|
|
2149
|
+
// One resolution for every entry point: complete() used to build from the
|
|
2150
|
+
// formatter's constructor-time mode alone, so request.toolMode was a
|
|
2151
|
+
// second, disconnected source of truth on this path.
|
|
2152
|
+
toolMode: this.resolveToolMode(request, activeFormatter),
|
|
1656
2153
|
thinking: request.config.thinking,
|
|
1657
2154
|
systemPrompt: request.system,
|
|
1658
2155
|
promptCaching: request.promptCaching ?? this.config.defaultPromptCaching ?? true, // Default true for backward compat
|
|
@@ -1677,7 +2174,9 @@ export class Membrane {
|
|
|
1677
2174
|
const providerRequest = {
|
|
1678
2175
|
...this.getBaseProviderParams(request.config),
|
|
1679
2176
|
messages: buildResult.messages,
|
|
1680
|
-
|
|
2177
|
+
// Owned, not aliased: the wire clamp strips markers in place, and a
|
|
2178
|
+
// formatter may pass the caller's own system array straight through.
|
|
2179
|
+
system: ownSystemBlocks(buildResult.systemContent),
|
|
1681
2180
|
stopSequences: buildResult.stopSequences,
|
|
1682
2181
|
tools: buildResult.nativeTools,
|
|
1683
2182
|
extra: {
|
|
@@ -1691,8 +2190,8 @@ export class Membrane {
|
|
|
1691
2190
|
// literal `<thinking>` text prefix instead of the API feature — drop the
|
|
1692
2191
|
// API param when the built request actually ends in an assistant prefill.
|
|
1693
2192
|
// Chat-style builds (no prefill) keep it.
|
|
1694
|
-
if (buildResult.assistantPrefill
|
|
1695
|
-
|
|
2193
|
+
if (buildResult.assistantPrefill) {
|
|
2194
|
+
stripThinkingForPrefill(providerRequest);
|
|
1696
2195
|
}
|
|
1697
2196
|
|
|
1698
2197
|
return { providerRequest, prefillResult: buildResult };
|
|
@@ -1734,24 +2233,72 @@ export class Membrane {
|
|
|
1734
2233
|
* somewhere upstream.
|
|
1735
2234
|
*/
|
|
1736
2235
|
onRetrying?: (info: { attempt: number; maxAttempts: number; category?: string }) => void;
|
|
2236
|
+
/**
|
|
2237
|
+
* Receives the number of cache_control markers the request ACTUALLY
|
|
2238
|
+
* ships with, taken from the clamp's own tally below — i.e. after the
|
|
2239
|
+
* `beforeRequest` hook has added or removed markers of its own and
|
|
2240
|
+
* after everything past the 4-breakpoint budget has been dropped.
|
|
2241
|
+
*
|
|
2242
|
+
* Telemetry that counts the request at BUILD time reports a number no
|
|
2243
|
+
* request ever had (a hook placing 7 markers on a wire that carries 4
|
|
2244
|
+
* was reported as the builder's 1), which defeats the audit the count
|
|
2245
|
+
* exists for. This is the only count that describes the wire.
|
|
2246
|
+
*/
|
|
2247
|
+
onWireCacheMarkers?: (markerCount: number) => void;
|
|
1737
2248
|
}
|
|
1738
|
-
)
|
|
2249
|
+
): Promise<
|
|
2250
|
+
import('./types/provider.js').ProviderResponse & {
|
|
2251
|
+
discardedUsage?: DiscardedAttemptsUsage;
|
|
2252
|
+
/** Provider calls this helper made, including refusal re-issues. */
|
|
2253
|
+
providerCalls: number;
|
|
2254
|
+
}
|
|
2255
|
+
> {
|
|
1739
2256
|
// Strip `normalizedRequest` before forwarding to the adapter — it's
|
|
1740
2257
|
// not part of `ProviderRequestOptions` and TypeScript's structural
|
|
1741
2258
|
// compatibility won't catch the excess field (checked only on object
|
|
1742
2259
|
// literals, not on variables). Leaving it in would silently leak the
|
|
1743
2260
|
// normalized form into every adapter's options.
|
|
1744
|
-
const { normalizedRequest, refusalRetries, onRetrying, ...adapterOptions } = options;
|
|
2261
|
+
const { normalizedRequest, refusalRetries, onRetrying, onWireCacheMarkers, ...adapterOptions } = options;
|
|
1745
2262
|
const finalRequest = (await this.applyBeforeRequestHook(normalizedRequest, request)) as typeof request;
|
|
1746
2263
|
|
|
2264
|
+
// Last exit before the adapter: the only place that sees EVERY
|
|
2265
|
+
// contribution (builder, formatter, passthrough, float, hook). Every
|
|
2266
|
+
// streaming path — stream(), streamYielding(), both tool loops — funnels
|
|
2267
|
+
// through here, so this is the one clamp they all get, and its tally is
|
|
2268
|
+
// therefore the only count that describes the wire.
|
|
2269
|
+
const clampOutcome = clampCacheMarkers(finalRequest, 'streamOnce');
|
|
2270
|
+
onWireCacheMarkers?.(clampOutcome.total);
|
|
2271
|
+
|
|
1747
2272
|
// Retries are only safe when the caller can discard the abandoned
|
|
1748
2273
|
// attempt, so they require BOTH a budget and an onRetrying hook.
|
|
1749
2274
|
const maxAttempts = onRetrying ? Math.max(0, refusalRetries ?? 0) : 0;
|
|
1750
2275
|
let retried = 0;
|
|
2276
|
+
// Every re-issued attempt was a completed, billed provider call. The
|
|
2277
|
+
// caller's usage accumulator only ever sees the surviving result, so the
|
|
2278
|
+
// abandoned spend rides back out on the result itself.
|
|
2279
|
+
let discardedUsage: DiscardedAttemptsUsage | undefined;
|
|
2280
|
+
let providerCalls = 0;
|
|
1751
2281
|
while (true) {
|
|
1752
|
-
|
|
1753
|
-
|
|
2282
|
+
providerCalls++;
|
|
2283
|
+
const rawResult = await this.adapter.stream(finalRequest, callbacks, adapterOptions);
|
|
2284
|
+
// Restate usage in the one convention before any accumulator, ratio or
|
|
2285
|
+
// price sees it — this is the only door streamed usage enters through.
|
|
2286
|
+
const result: ProviderResponse = {
|
|
2287
|
+
...rawResult,
|
|
2288
|
+
usage: normalizeUsageToCacheExcluded(rawResult.usage, this.adapter.name, this.adapter.usageCacheConvention),
|
|
2289
|
+
};
|
|
2290
|
+
if (result.stopReason !== 'refusal' || retried >= maxAttempts) {
|
|
2291
|
+
return {
|
|
2292
|
+
...result,
|
|
2293
|
+
providerCalls,
|
|
2294
|
+
...(discardedUsage ? { discardedUsage } : {}),
|
|
2295
|
+
};
|
|
2296
|
+
}
|
|
1754
2297
|
retried++;
|
|
2298
|
+
discardedUsage = this.mergeDiscardedAttempts(
|
|
2299
|
+
discardedUsage,
|
|
2300
|
+
this.discardedAttemptFrom(result.usage)
|
|
2301
|
+
);
|
|
1755
2302
|
const category = (result.raw as { response?: { stop_details?: { category?: string } } } | undefined)
|
|
1756
2303
|
?.response?.stop_details?.category;
|
|
1757
2304
|
onRetrying!({ attempt: retried, maxAttempts, category });
|
|
@@ -1765,7 +2312,14 @@ export class Membrane {
|
|
|
1765
2312
|
): any {
|
|
1766
2313
|
// Anthropic quirk: assistant content cannot end with trailing whitespace
|
|
1767
2314
|
const trimmedAccumulated = accumulated.trimEnd();
|
|
1768
|
-
|
|
2315
|
+
|
|
2316
|
+
// Everything before the watermark already rides EARLIER messages (a
|
|
2317
|
+
// persisted split turn), so only the suffix belongs in the trailing
|
|
2318
|
+
// assistant prefill — replacing it with the whole document would
|
|
2319
|
+
// duplicate the pre-seam text and flatten the image user-turn away.
|
|
2320
|
+
const baseOffset = prefillResult.accumulatedBaseOffset ?? 0;
|
|
2321
|
+
const trailingContent = accumulated.slice(baseOffset).trimEnd();
|
|
2322
|
+
|
|
1769
2323
|
// Build continuation messages: keep all messages up to last assistant,
|
|
1770
2324
|
// then replace/add the accumulated content
|
|
1771
2325
|
const messages = [...prefillResult.messages];
|
|
@@ -1774,34 +2328,34 @@ export class Membrane {
|
|
|
1774
2328
|
let foundAssistant = false;
|
|
1775
2329
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
1776
2330
|
if (messages[i]?.role === 'assistant') {
|
|
1777
|
-
messages[i] = { role: 'assistant', content:
|
|
2331
|
+
messages[i] = { role: 'assistant', content: trailingContent };
|
|
1778
2332
|
foundAssistant = true;
|
|
1779
2333
|
break;
|
|
1780
2334
|
}
|
|
1781
2335
|
}
|
|
1782
2336
|
|
|
1783
2337
|
if (!foundAssistant) {
|
|
1784
|
-
messages.push({ role: 'assistant', content:
|
|
2338
|
+
messages.push({ role: 'assistant', content: trailingContent });
|
|
1785
2339
|
}
|
|
1786
2340
|
|
|
1787
|
-
return {
|
|
2341
|
+
return stripThinkingForPrefill({
|
|
1788
2342
|
...this.getBaseProviderParams(originalRequest.config),
|
|
1789
2343
|
// Continuations always end in an assistant prefill — the API rejects
|
|
1790
2344
|
// extended thinking combined with prefill, so never send the param here
|
|
1791
2345
|
thinking: undefined,
|
|
1792
2346
|
messages,
|
|
1793
|
-
system: prefillResult.systemContent
|
|
1794
|
-
? (Array.isArray(prefillResult.systemContent) && prefillResult.systemContent.length > 0
|
|
1795
|
-
? prefillResult.systemContent
|
|
1796
|
-
: prefillResult.systemContent)
|
|
1797
|
-
: undefined,
|
|
2347
|
+
system: ownSystemBlocks(prefillResult.systemContent) ?? undefined,
|
|
1798
2348
|
stopSequences: prefillResult.stopSequences,
|
|
1799
2349
|
extra: {
|
|
1800
2350
|
...originalRequest.providerParams,
|
|
2351
|
+
// Same contract transformRequest sends: adapters that reason about
|
|
2352
|
+
// the normalized shape (or fall back to serializing it) must not see
|
|
2353
|
+
// a continuation as a request with no normalized form at all.
|
|
2354
|
+
normalizedMessages: originalRequest.messages,
|
|
1801
2355
|
// Pre-serialized prompt for completions adapters — skip re-serialization
|
|
1802
2356
|
prompt: trimmedAccumulated,
|
|
1803
2357
|
},
|
|
1804
|
-
};
|
|
2358
|
+
});
|
|
1805
2359
|
}
|
|
1806
2360
|
|
|
1807
2361
|
/**
|
|
@@ -1831,6 +2385,12 @@ export class Membrane {
|
|
|
1831
2385
|
// Anthropic quirk: assistant content cannot end with trailing whitespace
|
|
1832
2386
|
const trimmedAccumulated = accumulated.trimEnd();
|
|
1833
2387
|
|
|
2388
|
+
// The split replaces only the CURRENT trailing assistant message, which
|
|
2389
|
+
// covers the accumulated text from the previous seam onward (0 on the
|
|
2390
|
+
// first split, the previous image seam on a later one).
|
|
2391
|
+
const baseOffset = prefillResult.accumulatedBaseOffset ?? 0;
|
|
2392
|
+
const trailingContent = accumulated.slice(baseOffset).trimEnd();
|
|
2393
|
+
|
|
1834
2394
|
// Build messages: copy all, then replace only the last assistant with split-turn
|
|
1835
2395
|
const messages: any[] = prefillResult.messages.map(msg => ({ ...msg }));
|
|
1836
2396
|
|
|
@@ -1846,7 +2406,7 @@ export class Membrane {
|
|
|
1846
2406
|
// Anthropic quirk: assistant content cannot end with trailing whitespace
|
|
1847
2407
|
const trimmedAfterXml = afterImageXml.trimEnd();
|
|
1848
2408
|
const splitTurnMessages = [
|
|
1849
|
-
{ role: 'assistant', content:
|
|
2409
|
+
{ role: 'assistant', content: trailingContent },
|
|
1850
2410
|
{ role: 'user', content: images },
|
|
1851
2411
|
{ role: 'assistant', content: trimmedAfterXml },
|
|
1852
2412
|
];
|
|
@@ -1857,20 +2417,40 @@ export class Membrane {
|
|
|
1857
2417
|
messages.push(...splitTurnMessages);
|
|
1858
2418
|
}
|
|
1859
2419
|
|
|
1860
|
-
|
|
2420
|
+
// PERSIST the split. Later rounds rebuild from prefillResult.messages;
|
|
2421
|
+
// without this the image user-turn exists on exactly one request and the
|
|
2422
|
+
// next continuation flattens the accumulated document back over it —
|
|
2423
|
+
// leaving <function_results> XML asserting a screenshot the model can no
|
|
2424
|
+
// longer see. Reassign (never mutate in place): the previous array is
|
|
2425
|
+
// still referenced by the request already on the wire. The watermark
|
|
2426
|
+
// moves to the seam — the point in `accumulated` where afterImageXml is
|
|
2427
|
+
// about to be appended — so the next builder replaces only the closing
|
|
2428
|
+
// assistant turn.
|
|
2429
|
+
prefillResult.messages = messages;
|
|
2430
|
+
prefillResult.accumulatedBaseOffset = accumulated.length;
|
|
2431
|
+
|
|
2432
|
+
return stripThinkingForPrefill({
|
|
1861
2433
|
...this.getBaseProviderParams(originalRequest.config),
|
|
1862
2434
|
// Continuations always end in an assistant prefill — the API rejects
|
|
1863
2435
|
// extended thinking combined with prefill, so never send the param here
|
|
1864
2436
|
thinking: undefined,
|
|
1865
2437
|
messages,
|
|
1866
|
-
system: prefillResult.systemContent
|
|
1867
|
-
? (Array.isArray(prefillResult.systemContent) && prefillResult.systemContent.length > 0
|
|
1868
|
-
? prefillResult.systemContent
|
|
1869
|
-
: prefillResult.systemContent)
|
|
1870
|
-
: undefined,
|
|
2438
|
+
system: ownSystemBlocks(prefillResult.systemContent) ?? undefined,
|
|
1871
2439
|
stopSequences: prefillResult.stopSequences,
|
|
1872
|
-
|
|
1873
|
-
|
|
2440
|
+
// Copied, not aliased: the guard below deletes the smuggled thinking
|
|
2441
|
+
// config, and mutating the caller's own providerParams object would
|
|
2442
|
+
// silently disable thinking on their NEXT (non-prefill) request.
|
|
2443
|
+
extra: {
|
|
2444
|
+
...originalRequest.providerParams,
|
|
2445
|
+
// Same contract as transformRequest and the plain continuation
|
|
2446
|
+
// builder. Without these a completions-style adapter fell through to
|
|
2447
|
+
// serializing PROVIDER-shaped messages as if they were normalized
|
|
2448
|
+
// ones, re-adding participant stop sequences the continuation
|
|
2449
|
+
// deliberately suppresses.
|
|
2450
|
+
normalizedMessages: originalRequest.messages,
|
|
2451
|
+
prompt: trimmedAccumulated,
|
|
2452
|
+
},
|
|
2453
|
+
});
|
|
1874
2454
|
}
|
|
1875
2455
|
|
|
1876
2456
|
private transformResponse(
|
|
@@ -1947,20 +2527,33 @@ export class Membrane {
|
|
|
1947
2527
|
|
|
1948
2528
|
// Parse XML tool calls from text if no native tool_use blocks were found
|
|
1949
2529
|
// This handles prefill mode where tools are XML in the text
|
|
2530
|
+
let emptyToolBlocks = 0;
|
|
1950
2531
|
if (toolCalls.length === 0 && rawAssistantText.includes('<function_calls>')) {
|
|
1951
2532
|
const parsed = parseToolCalls(rawAssistantText);
|
|
1952
2533
|
if (parsed?.calls.length) {
|
|
1953
2534
|
for (const tc of parsed.calls) {
|
|
1954
2535
|
toolCalls.push(tc);
|
|
1955
2536
|
}
|
|
2537
|
+
} else if (parsed) {
|
|
2538
|
+
emptyToolBlocks = 1;
|
|
1956
2539
|
}
|
|
1957
2540
|
}
|
|
2541
|
+
const unclosedToolBlock = endsWithPartialToolBlock(rawAssistantText);
|
|
1958
2542
|
|
|
1959
2543
|
const stopReason = this.mapStopReason(providerResponse.stopReason);
|
|
2544
|
+
this.reportToolParseDiagnostics({ unclosedToolBlock, emptyToolBlocks }, stopReason);
|
|
1960
2545
|
const durationMs = Date.now() - startTime;
|
|
1961
|
-
|
|
2546
|
+
// `NormalizedResponse.usage` is typed DetailedUsage and the streaming paths
|
|
2547
|
+
// already return the whole thing; complete() used to narrow it to
|
|
2548
|
+
// input/output here, so a caller reading `response.usage.cacheReadTokens`
|
|
2549
|
+
// saw undefined on one path and a number on the other.
|
|
2550
|
+
const usage: DetailedUsage = {
|
|
1962
2551
|
inputTokens: providerResponse.usage.inputTokens,
|
|
1963
2552
|
outputTokens: providerResponse.usage.outputTokens,
|
|
2553
|
+
cacheCreationTokens: providerResponse.usage.cacheCreationTokens,
|
|
2554
|
+
cacheReadTokens: providerResponse.usage.cacheReadTokens,
|
|
2555
|
+
thinkingTokens: providerResponse.usage.thinkingTokens,
|
|
2556
|
+
estimatedCost: this.estimateCost(providerResponse.usage, request.config.model, providerResponse.model),
|
|
1964
2557
|
};
|
|
1965
2558
|
|
|
1966
2559
|
return {
|
|
@@ -1975,14 +2568,9 @@ export class Membrane {
|
|
|
1975
2568
|
reason: stopReason,
|
|
1976
2569
|
triggeredSequence: providerResponse.stopSequence,
|
|
1977
2570
|
wasTruncated: stopReason === 'max_tokens',
|
|
2571
|
+
unclosedToolBlock,
|
|
1978
2572
|
},
|
|
1979
|
-
usage
|
|
1980
|
-
inputTokens: providerResponse.usage.inputTokens,
|
|
1981
|
-
outputTokens: providerResponse.usage.outputTokens,
|
|
1982
|
-
cacheCreationTokens: providerResponse.usage.cacheCreationTokens,
|
|
1983
|
-
cacheReadTokens: providerResponse.usage.cacheReadTokens,
|
|
1984
|
-
estimatedCost: this.estimateCost(providerResponse.usage, request.config.model),
|
|
1985
|
-
},
|
|
2573
|
+
usage,
|
|
1986
2574
|
timing: {
|
|
1987
2575
|
totalDurationMs: durationMs,
|
|
1988
2576
|
attempts,
|
|
@@ -2006,11 +2594,65 @@ export class Membrane {
|
|
|
2006
2594
|
};
|
|
2007
2595
|
}
|
|
2008
2596
|
|
|
2597
|
+
/**
|
|
2598
|
+
* The turn is over, and the two guards that detect a half-written tool block
|
|
2599
|
+
* finally have a call site. Both shapes are defects a consumer must not
|
|
2600
|
+
* persist blind: an unclosed block splices onto the NEXT round's closing tag
|
|
2601
|
+
* (the loop does not resume on a length stop, so max_tokens leaves exactly
|
|
2602
|
+
* this), and a block that parsed to nothing means the model believes it
|
|
2603
|
+
* called a tool that never ran.
|
|
2604
|
+
*/
|
|
2605
|
+
private reportToolParseDiagnostics(
|
|
2606
|
+
diagnostics: {
|
|
2607
|
+
unclosedToolBlock: boolean;
|
|
2608
|
+
emptyToolBlocks: number;
|
|
2609
|
+
splicedToolBlocks?: number;
|
|
2610
|
+
unclosedInvokeHeads?: number;
|
|
2611
|
+
},
|
|
2612
|
+
stopReason: StopReason
|
|
2613
|
+
): void {
|
|
2614
|
+
const warnLog = this.config.logger ?? console;
|
|
2615
|
+
|
|
2616
|
+
if (diagnostics.unclosedToolBlock) {
|
|
2617
|
+
warnLog.warn(
|
|
2618
|
+
`[membrane] turn ended (${stopReason}) with an unclosed tool block in the ` +
|
|
2619
|
+
`assistant text — the loop does not resume on a length stop. Persisting this ` +
|
|
2620
|
+
`turn verbatim lets the next round's closing tag splice onto the stale ` +
|
|
2621
|
+
`opener; see details.stop.unclosedToolBlock.`
|
|
2622
|
+
);
|
|
2623
|
+
}
|
|
2624
|
+
|
|
2625
|
+
if (diagnostics.emptyToolBlocks > 0) {
|
|
2626
|
+
warnLog.warn(
|
|
2627
|
+
`[membrane] ${diagnostics.emptyToolBlocks} function_calls block(s) parsed to ` +
|
|
2628
|
+
`zero tool calls — always a defect, never a normal ending. The call was ` +
|
|
2629
|
+
`returned as assistant text and nothing executed.`
|
|
2630
|
+
);
|
|
2631
|
+
}
|
|
2632
|
+
|
|
2633
|
+
if (diagnostics.splicedToolBlocks) {
|
|
2634
|
+
warnLog.warn(
|
|
2635
|
+
`[membrane] ${diagnostics.splicedToolBlocks} tool block(s) spanned a second ` +
|
|
2636
|
+
`<function_calls> opener and were re-anchored to the innermost one — an ` +
|
|
2637
|
+
`earlier truncated block is present in this conversation's assistant text.`
|
|
2638
|
+
);
|
|
2639
|
+
}
|
|
2640
|
+
|
|
2641
|
+
if (diagnostics.unclosedInvokeHeads) {
|
|
2642
|
+
warnLog.warn(
|
|
2643
|
+
`[membrane] ${diagnostics.unclosedInvokeHeads} <invoke> head(s) were left ` +
|
|
2644
|
+
`unclosed and swallowed the invoke that followed — nothing was dispatched ` +
|
|
2645
|
+
`under an unclosed head's name, and the call it absorbed was re-anchored ` +
|
|
2646
|
+
`and ran with its own parameters.`
|
|
2647
|
+
);
|
|
2648
|
+
}
|
|
2649
|
+
}
|
|
2650
|
+
|
|
2009
2651
|
private buildFinalResponse(
|
|
2010
2652
|
accumulated: string,
|
|
2011
2653
|
contentBlocks: ContentBlock[],
|
|
2012
2654
|
stopReason: StopReason,
|
|
2013
|
-
|
|
2655
|
+
turnUsage: TurnUsageAccumulator,
|
|
2014
2656
|
request: NormalizedRequest,
|
|
2015
2657
|
prefillResult: {
|
|
2016
2658
|
cacheMarkersApplied?: number;
|
|
@@ -2022,14 +2664,17 @@ export class Membrane {
|
|
|
2022
2664
|
executedToolCalls: ToolCall[] = [],
|
|
2023
2665
|
executedToolResults: ToolResult[] = [],
|
|
2024
2666
|
startInsideBlock: 'thinking' | 'tool_call' | 'tool_result' | null = null,
|
|
2025
|
-
triggeredSequence?: string
|
|
2667
|
+
triggeredSequence?: string,
|
|
2026
2668
|
): NormalizedResponse {
|
|
2669
|
+
const usage = turnUsage.total;
|
|
2027
2670
|
// Parse accumulated text into structured content blocks
|
|
2028
2671
|
// This extracts thinking, tool_use, tool_result, and text blocks
|
|
2029
2672
|
let finalContent: ContentBlock[];
|
|
2030
2673
|
let toolCalls: ToolCall[];
|
|
2031
2674
|
let toolResults: ToolResult[];
|
|
2032
2675
|
|
|
2676
|
+
let unclosedToolBlock = false;
|
|
2677
|
+
|
|
2033
2678
|
if (contentBlocks.length > 0) {
|
|
2034
2679
|
// Native mode - content blocks already structured
|
|
2035
2680
|
finalContent = contentBlocks;
|
|
@@ -2044,6 +2689,8 @@ export class Membrane {
|
|
|
2044
2689
|
finalContent = parsed.blocks;
|
|
2045
2690
|
toolCalls = parsed.toolCalls.length > 0 ? parsed.toolCalls : executedToolCalls;
|
|
2046
2691
|
toolResults = parsed.toolResults.length > 0 ? parsed.toolResults : executedToolResults;
|
|
2692
|
+
unclosedToolBlock = parsed.unclosedToolBlock;
|
|
2693
|
+
this.reportToolParseDiagnostics(parsed, stopReason);
|
|
2047
2694
|
}
|
|
2048
2695
|
|
|
2049
2696
|
const durationMs = Date.now() - startTime;
|
|
@@ -2060,19 +2707,21 @@ export class Membrane {
|
|
|
2060
2707
|
reason: stopReason,
|
|
2061
2708
|
triggeredSequence,
|
|
2062
2709
|
wasTruncated: stopReason === 'max_tokens',
|
|
2710
|
+
unclosedToolBlock,
|
|
2063
2711
|
},
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2712
|
+
// Priced per round by the accumulator and summed — NOT re-derived here
|
|
2713
|
+
// from the turn totals, which would re-bill every round at the last
|
|
2714
|
+
// model's rate.
|
|
2715
|
+
usage,
|
|
2068
2716
|
timing: {
|
|
2069
2717
|
totalDurationMs: durationMs,
|
|
2070
2718
|
attempts,
|
|
2071
2719
|
},
|
|
2072
2720
|
model: {
|
|
2073
2721
|
requested: request.config.model,
|
|
2074
|
-
actual: request.config.model,
|
|
2722
|
+
actual: turnUsage.lastServedModel || request.config.model,
|
|
2075
2723
|
provider: this.adapter.name,
|
|
2724
|
+
perRound: turnUsage.perRound,
|
|
2076
2725
|
},
|
|
2077
2726
|
cache: {
|
|
2078
2727
|
markersInRequest: prefillResult.cacheMarkersApplied ?? 0,
|
|
@@ -2088,6 +2737,52 @@ export class Membrane {
|
|
|
2088
2737
|
};
|
|
2089
2738
|
}
|
|
2090
2739
|
|
|
2740
|
+
/**
|
|
2741
|
+
* Fold one discarded (billed but abandoned) attempt's usage into a carry.
|
|
2742
|
+
* Returns a NEW object so a caller's earlier snapshot is never mutated.
|
|
2743
|
+
*/
|
|
2744
|
+
private mergeDiscardedAttempts(
|
|
2745
|
+
carry: DiscardedAttemptsUsage | undefined,
|
|
2746
|
+
add: DiscardedAttemptsUsage | undefined
|
|
2747
|
+
): DiscardedAttemptsUsage | undefined {
|
|
2748
|
+
if (!add) return carry;
|
|
2749
|
+
const next: DiscardedAttemptsUsage = carry
|
|
2750
|
+
? { ...carry }
|
|
2751
|
+
: { attempts: 0, inputTokens: 0, outputTokens: 0 };
|
|
2752
|
+
next.attempts += add.attempts;
|
|
2753
|
+
next.inputTokens += add.inputTokens;
|
|
2754
|
+
next.outputTokens += add.outputTokens;
|
|
2755
|
+
if (add.cacheCreationTokens) {
|
|
2756
|
+
next.cacheCreationTokens = (next.cacheCreationTokens ?? 0) + add.cacheCreationTokens;
|
|
2757
|
+
}
|
|
2758
|
+
if (add.cacheReadTokens) {
|
|
2759
|
+
next.cacheReadTokens = (next.cacheReadTokens ?? 0) + add.cacheReadTokens;
|
|
2760
|
+
}
|
|
2761
|
+
return next;
|
|
2762
|
+
}
|
|
2763
|
+
|
|
2764
|
+
/** One provider call's usage as a single-attempt discard record. */
|
|
2765
|
+
private discardedAttemptFrom(usage: DetailedUsage | BasicUsage | undefined): DiscardedAttemptsUsage {
|
|
2766
|
+
const detailed = (usage ?? { inputTokens: 0, outputTokens: 0 }) as DetailedUsage;
|
|
2767
|
+
return {
|
|
2768
|
+
attempts: 1,
|
|
2769
|
+
inputTokens: detailed.inputTokens ?? 0,
|
|
2770
|
+
outputTokens: detailed.outputTokens ?? 0,
|
|
2771
|
+
...(detailed.cacheCreationTokens ? { cacheCreationTokens: detailed.cacheCreationTokens } : {}),
|
|
2772
|
+
...(detailed.cacheReadTokens ? { cacheReadTokens: detailed.cacheReadTokens } : {}),
|
|
2773
|
+
};
|
|
2774
|
+
}
|
|
2775
|
+
|
|
2776
|
+
/** Price the discarded spend so a caller can read it without re-deriving. */
|
|
2777
|
+
private pricedDiscardedAttempts(
|
|
2778
|
+
discarded: DiscardedAttemptsUsage | undefined,
|
|
2779
|
+
model: string
|
|
2780
|
+
): DiscardedAttemptsUsage | undefined {
|
|
2781
|
+
if (!discarded) return undefined;
|
|
2782
|
+
const estimatedCost = this.estimateCost(discarded, model);
|
|
2783
|
+
return estimatedCost ? { ...discarded, estimatedCost } : discarded;
|
|
2784
|
+
}
|
|
2785
|
+
|
|
2091
2786
|
private mapStopReason(providerReason: string): StopReason {
|
|
2092
2787
|
switch (providerReason) {
|
|
2093
2788
|
case 'end_turn':
|
|
@@ -2109,20 +2804,62 @@ export class Membrane {
|
|
|
2109
2804
|
}
|
|
2110
2805
|
|
|
2111
2806
|
private calculateCacheHitRatio(usage: Pick<DetailedUsage, 'inputTokens' | 'cacheReadTokens'>): number {
|
|
2112
|
-
|
|
2113
|
-
const total = usage.inputTokens ?? 0;
|
|
2114
|
-
if (total === 0) return 0;
|
|
2115
|
-
return cacheRead / total;
|
|
2807
|
+
return calculateCacheHitRatio(usage);
|
|
2116
2808
|
}
|
|
2117
2809
|
|
|
2118
|
-
|
|
2119
|
-
|
|
2810
|
+
/**
|
|
2811
|
+
* Pricing is resolved on TWO axes, and SOURCE outranks SPECIFICITY:
|
|
2812
|
+
*
|
|
2813
|
+
* registry[served] → registry[requested] → builtin[served] → builtin[requested]
|
|
2814
|
+
*
|
|
2815
|
+
* Specificity — preferring the model that ACTUALLY served over the id that
|
|
2816
|
+
* was requested — is real: an alias or an auto-routed request otherwise
|
|
2817
|
+
* prices against a string the provider already replaced, and a live
|
|
2818
|
+
* 2026-08-25 call asking for `gpt-4o-mini` was served by
|
|
2819
|
+
* `gpt-4o-mini-2024-07-18`. But it only breaks ties WITHIN one source.
|
|
2820
|
+
* A configured `ModelRegistry` is the caller stating their own rates —
|
|
2821
|
+
* account-specific, negotiated, authoritative; the built-in table is
|
|
2822
|
+
* membrane's shipped guess at public list prices. Merging the two per-model
|
|
2823
|
+
* (`registry[served] ?? builtin[served]`, return on the first hit) let the
|
|
2824
|
+
* guess for a snapshot outrank the caller's own entry for the alias they
|
|
2825
|
+
* asked for, so a caller who prices their alias and lets the provider pick
|
|
2826
|
+
* the snapshot was billed at membrane's number instead of theirs.
|
|
2827
|
+
*
|
|
2828
|
+
* Both fallbacks stay: the served model may be absent from a source, and the
|
|
2829
|
+
* provider may name none at all.
|
|
2830
|
+
*/
|
|
2831
|
+
private resolvePricing(
|
|
2832
|
+
requestedModel: string,
|
|
2833
|
+
actualModel?: string
|
|
2834
|
+
): import('./types/provider.js').ModelPricing | undefined {
|
|
2835
|
+
const servedModel = actualModel && actualModel !== requestedModel ? actualModel : undefined;
|
|
2836
|
+
const fromRegistry = (modelId: string | undefined) =>
|
|
2837
|
+
modelId === undefined ? undefined : this.registry?.getPricing(modelId);
|
|
2838
|
+
const fromBuiltin = (modelId: string | undefined) =>
|
|
2839
|
+
modelId === undefined ? undefined : getDefaultPricing(modelId);
|
|
2840
|
+
|
|
2841
|
+
return fromRegistry(servedModel)
|
|
2842
|
+
?? fromRegistry(requestedModel)
|
|
2843
|
+
?? fromBuiltin(servedModel)
|
|
2844
|
+
?? fromBuiltin(requestedModel);
|
|
2120
2845
|
}
|
|
2121
2846
|
|
|
2122
2847
|
/** Resolve pricing + calculate cost in one call (for one-shot use outside loops). */
|
|
2123
|
-
private estimateCost(
|
|
2124
|
-
|
|
2125
|
-
|
|
2848
|
+
private estimateCost(
|
|
2849
|
+
usage: import('./utils/cost.js').CostableUsage,
|
|
2850
|
+
requestedModel: string,
|
|
2851
|
+
actualModel?: string
|
|
2852
|
+
): import('./types/response.js').CostBreakdown | undefined {
|
|
2853
|
+
const pricing = this.resolvePricing(requestedModel, actualModel);
|
|
2854
|
+
if (!pricing) {
|
|
2855
|
+
// An absent cost and a zero cost are different claims. Returning
|
|
2856
|
+
// undefined says "membrane does not know what this costs"; saying it out
|
|
2857
|
+
// loud once per model keeps that from reading as "free" to a caller that
|
|
2858
|
+
// only ever sees the omission.
|
|
2859
|
+
warnUnpricedModel(actualModel || requestedModel);
|
|
2860
|
+
return undefined;
|
|
2861
|
+
}
|
|
2862
|
+
return calculateCost(usage, pricing);
|
|
2126
2863
|
}
|
|
2127
2864
|
|
|
2128
2865
|
private calculateRetryDelay(attempt: number, overloaded = false): number {
|
|
@@ -2160,6 +2897,10 @@ export class Membrane {
|
|
|
2160
2897
|
* Check if an error is an abort error
|
|
2161
2898
|
*/
|
|
2162
2899
|
private isAbortError(error: unknown): boolean {
|
|
2900
|
+
// An adapter's own deadline: a timeout by classification, still an abort
|
|
2901
|
+
// by provenance, so the streaming paths hand back the partial content
|
|
2902
|
+
// they collected instead of throwing.
|
|
2903
|
+
if (isTimeoutAbortError(error)) return true;
|
|
2163
2904
|
if (error instanceof Error) {
|
|
2164
2905
|
// Standard AbortError
|
|
2165
2906
|
if (error.name === 'AbortError') return true;
|
|
@@ -2173,6 +2914,20 @@ export class Membrane {
|
|
|
2173
2914
|
return false;
|
|
2174
2915
|
}
|
|
2175
2916
|
|
|
2917
|
+
/**
|
|
2918
|
+
* Why a caught abort happened. The caller's own signal is authoritative:
|
|
2919
|
+
* if it fired, the cancellation is theirs whatever the error text says.
|
|
2920
|
+
* Otherwise an adapter-side deadline classifies as a timeout — the adapters
|
|
2921
|
+
* mark the abort createCombinedSignal's timeoutMs raises and map it to a
|
|
2922
|
+
* TimeoutAbortError, so the identity survives their error handling — and
|
|
2923
|
+
* anything else that reached the abort catch is a failure, not a person.
|
|
2924
|
+
*/
|
|
2925
|
+
private abortReason(error: unknown, signal?: AbortSignal): 'user' | 'timeout' | 'error' {
|
|
2926
|
+
if (signal?.aborted) return 'user';
|
|
2927
|
+
if (classifyError(error).type === 'timeout') return 'timeout';
|
|
2928
|
+
return 'error';
|
|
2929
|
+
}
|
|
2930
|
+
|
|
2176
2931
|
/**
|
|
2177
2932
|
* Build an AbortedResponse from current execution state
|
|
2178
2933
|
*/
|
|
@@ -2235,7 +2990,12 @@ export class Membrane {
|
|
|
2235
2990
|
request: NormalizedRequest,
|
|
2236
2991
|
options: YieldingStreamOptions = {}
|
|
2237
2992
|
): YieldingStream {
|
|
2238
|
-
|
|
2993
|
+
// YieldingStreamOptions carries no per-request formatter override, so the
|
|
2994
|
+
// selection here can only land on the instance formatter — it goes through
|
|
2995
|
+
// resolveActiveFormatter anyway so this path reads the same single source
|
|
2996
|
+
// as complete() and stream() if an override is ever added.
|
|
2997
|
+
const activeFormatter = this.resolveActiveFormatter();
|
|
2998
|
+
const toolMode = this.resolveToolMode(request, activeFormatter);
|
|
2239
2999
|
|
|
2240
3000
|
// refusalRetries is implemented on the native path only. The XML path
|
|
2241
3001
|
// accumulates into a streaming parser carrying prefill context and
|
|
@@ -2251,8 +3011,8 @@ export class Membrane {
|
|
|
2251
3011
|
|
|
2252
3012
|
// Create the yielding stream with the appropriate inference runner
|
|
2253
3013
|
const runInference = toolMode === 'native'
|
|
2254
|
-
? (stream: YieldingStreamImpl) => this.runNativeToolsYielding(request, options, stream)
|
|
2255
|
-
: (stream: YieldingStreamImpl) => this.runXmlToolsYielding(request, options, stream);
|
|
3014
|
+
? (stream: YieldingStreamImpl) => this.runNativeToolsYielding(request, options, stream, activeFormatter)
|
|
3015
|
+
: (stream: YieldingStreamImpl) => this.runXmlToolsYielding(request, options, stream, activeFormatter);
|
|
2256
3016
|
|
|
2257
3017
|
return new YieldingStreamImpl(options, runInference);
|
|
2258
3018
|
}
|
|
@@ -2263,7 +3023,8 @@ export class Membrane {
|
|
|
2263
3023
|
private async runXmlToolsYielding(
|
|
2264
3024
|
request: NormalizedRequest,
|
|
2265
3025
|
options: YieldingStreamOptions,
|
|
2266
|
-
stream: YieldingStreamImpl
|
|
3026
|
+
stream: YieldingStreamImpl,
|
|
3027
|
+
activeFormatter: PrefillFormatter = this.resolveActiveFormatter()
|
|
2267
3028
|
): Promise<void> {
|
|
2268
3029
|
const startTime = Date.now();
|
|
2269
3030
|
const {
|
|
@@ -2305,14 +3066,23 @@ export class Membrane {
|
|
|
2305
3066
|
let prevRoundStopSequence: string | undefined;
|
|
2306
3067
|
const warnLog = this.config.logger ?? console;
|
|
2307
3068
|
|
|
2308
|
-
// Initialize parser from formatter
|
|
2309
|
-
|
|
3069
|
+
// Initialize parser from the formatter streamYielding selected, so the
|
|
3070
|
+
// parser and the build below read the same format.
|
|
3071
|
+
const formatter = activeFormatter;
|
|
2310
3072
|
const parser = formatter.createStreamParser();
|
|
2311
3073
|
let toolDepth = 0;
|
|
3074
|
+
// Honest turn telemetry: provider calls actually made (including refusal
|
|
3075
|
+
// re-issues inside streamOnce) and continuation rounds.
|
|
3076
|
+
let providerCalls = 0;
|
|
3077
|
+
let rounds = 0;
|
|
2312
3078
|
// Once-per-stream latch for the injectedMessages-unsupported warning.
|
|
2313
3079
|
let warnedInjectionUnsupported = false;
|
|
2314
|
-
|
|
2315
|
-
|
|
3080
|
+
// See streamWithXmlTools: one accumulator per turn, pricing each round
|
|
3081
|
+
// under the model that served it.
|
|
3082
|
+
const turnUsage = new TurnUsageAccumulator(
|
|
3083
|
+
request.config.model,
|
|
3084
|
+
(servedModel) => this.resolvePricing(request.config.model, servedModel),
|
|
3085
|
+
);
|
|
2316
3086
|
const contentBlocks: ContentBlock[] = [];
|
|
2317
3087
|
let lastStopReason: StopReason = 'end_turn';
|
|
2318
3088
|
let lastStopSequence: string | undefined;
|
|
@@ -2361,7 +3131,7 @@ export class Membrane {
|
|
|
2361
3131
|
if (resumptionRounds === RESUMPTION_WARN_ROUNDS) {
|
|
2362
3132
|
warnLog.warn(
|
|
2363
3133
|
`[membrane] automatic resumption at round ${resumptionRounds} ` +
|
|
2364
|
-
`(${
|
|
3134
|
+
`(${turnUsage.total.inputTokens} input tokens so far this turn) — ` +
|
|
2365
3135
|
`a spin shows up here before it shows up on the bill`
|
|
2366
3136
|
);
|
|
2367
3137
|
}
|
|
@@ -2369,7 +3139,7 @@ export class Membrane {
|
|
|
2369
3139
|
warnLog.warn(
|
|
2370
3140
|
`[membrane] automatic resumption cap (${maxResumptionRounds}) reached — ` +
|
|
2371
3141
|
`ending turn with stopReason 'round_limit'. ` +
|
|
2372
|
-
`${
|
|
3142
|
+
`${turnUsage.total.inputTokens} input tokens spent this turn.`
|
|
2373
3143
|
);
|
|
2374
3144
|
return false;
|
|
2375
3145
|
}
|
|
@@ -2465,6 +3235,9 @@ export class Membrane {
|
|
|
2465
3235
|
}
|
|
2466
3236
|
);
|
|
2467
3237
|
|
|
3238
|
+
rounds++;
|
|
3239
|
+
providerCalls += streamResult.providerCalls;
|
|
3240
|
+
|
|
2468
3241
|
// If we detected stop sequence manually, fix up the parser and result
|
|
2469
3242
|
if (detectedStopSequence && truncatedAccumulated !== null) {
|
|
2470
3243
|
parser.reset();
|
|
@@ -2482,18 +3255,10 @@ export class Membrane {
|
|
|
2482
3255
|
lastStopReason = this.mapStopReason(streamResult.stopReason);
|
|
2483
3256
|
lastStopSequence = streamResult.stopSequence ?? undefined;
|
|
2484
3257
|
|
|
2485
|
-
// Accumulate usage (including cache metrics)
|
|
2486
|
-
|
|
2487
|
-
totalUsage.outputTokens += streamResult.usage.outputTokens;
|
|
2488
|
-
if (streamResult.usage.cacheCreationTokens) {
|
|
2489
|
-
totalUsage.cacheCreationTokens = (totalUsage.cacheCreationTokens ?? 0) + streamResult.usage.cacheCreationTokens;
|
|
2490
|
-
}
|
|
2491
|
-
if (streamResult.usage.cacheReadTokens) {
|
|
2492
|
-
totalUsage.cacheReadTokens = (totalUsage.cacheReadTokens ?? 0) + streamResult.usage.cacheReadTokens;
|
|
2493
|
-
}
|
|
2494
|
-
if (pricing) totalUsage.estimatedCost = calculateCost(totalUsage, pricing);
|
|
3258
|
+
// Accumulate usage (including cache metrics), priced at this round's model
|
|
3259
|
+
const usageSoFar = turnUsage.addRound(streamResult.model, streamResult.usage);
|
|
2495
3260
|
if (emitUsage) {
|
|
2496
|
-
stream.emit({ type: 'usage', usage:
|
|
3261
|
+
stream.emit({ type: 'usage', usage: usageSoFar });
|
|
2497
3262
|
}
|
|
2498
3263
|
|
|
2499
3264
|
// Flush the parser
|
|
@@ -2521,7 +3286,7 @@ export class Membrane {
|
|
|
2521
3286
|
`[membrane] ${consecutiveStalledResumptions} consecutive automatic resumptions ` +
|
|
2522
3287
|
`made no progress (${streamedThisRound} chars this round, stop ` +
|
|
2523
3288
|
`${JSON.stringify(lastStopSequence ?? null)} repeated) — ending turn with ` +
|
|
2524
|
-
`stopReason 'no_progress'. ${
|
|
3289
|
+
`stopReason 'no_progress'. ${turnUsage.total.inputTokens} input tokens spent this turn.`
|
|
2525
3290
|
);
|
|
2526
3291
|
lastStopReason = 'no_progress';
|
|
2527
3292
|
break;
|
|
@@ -2822,22 +3587,24 @@ export class Membrane {
|
|
|
2822
3587
|
newContent,
|
|
2823
3588
|
contentBlocks,
|
|
2824
3589
|
lastStopReason,
|
|
2825
|
-
|
|
3590
|
+
turnUsage,
|
|
2826
3591
|
request,
|
|
2827
3592
|
prefillResult,
|
|
2828
3593
|
startTime,
|
|
2829
|
-
|
|
3594
|
+
providerCalls,
|
|
2830
3595
|
rawRequest,
|
|
2831
3596
|
rawResponse,
|
|
2832
3597
|
executedToolCalls,
|
|
2833
3598
|
executedToolResults,
|
|
2834
3599
|
initialBlockType,
|
|
2835
|
-
lastStopSequence
|
|
3600
|
+
lastStopSequence,
|
|
2836
3601
|
);
|
|
2837
3602
|
|
|
2838
3603
|
// Merge provider thinking signatures into parser-derived thinking blocks
|
|
2839
3604
|
this.mergeProviderThinkingBlocks(response.content, providerThinkingBlocks);
|
|
2840
3605
|
|
|
3606
|
+
response.details.timing.rounds = rounds;
|
|
3607
|
+
|
|
2841
3608
|
stream.emit({ type: 'complete', response });
|
|
2842
3609
|
} catch (error) {
|
|
2843
3610
|
if (this.isAbortError(error)) {
|
|
@@ -2845,7 +3612,7 @@ export class Membrane {
|
|
|
2845
3612
|
const newContent = fullAccumulated.slice(initialPrefillLength);
|
|
2846
3613
|
stream.emit({
|
|
2847
3614
|
type: 'aborted',
|
|
2848
|
-
reason:
|
|
3615
|
+
reason: this.abortReason(error, stream.signal),
|
|
2849
3616
|
partialContent: parseAccumulatedIntoBlocks(newContent).blocks,
|
|
2850
3617
|
rawAssistantText: newContent,
|
|
2851
3618
|
toolCalls: executedToolCalls,
|
|
@@ -2863,7 +3630,8 @@ export class Membrane {
|
|
|
2863
3630
|
private async runNativeToolsYielding(
|
|
2864
3631
|
request: NormalizedRequest,
|
|
2865
3632
|
options: YieldingStreamOptions,
|
|
2866
|
-
stream: YieldingStreamImpl
|
|
3633
|
+
stream: YieldingStreamImpl,
|
|
3634
|
+
activeFormatter: PrefillFormatter = this.resolveActiveFormatter()
|
|
2867
3635
|
): Promise<void> {
|
|
2868
3636
|
const startTime = Date.now();
|
|
2869
3637
|
const {
|
|
@@ -2883,8 +3651,16 @@ export class Membrane {
|
|
|
2883
3651
|
: maxToolDepthOpt;
|
|
2884
3652
|
|
|
2885
3653
|
let toolDepth = 0;
|
|
2886
|
-
|
|
2887
|
-
|
|
3654
|
+
// See streamWithXmlTools: one accumulator per turn, pricing each round
|
|
3655
|
+
// under the model that served it.
|
|
3656
|
+
const turnUsage = new TurnUsageAccumulator(
|
|
3657
|
+
request.config.model,
|
|
3658
|
+
(servedModel) => this.resolvePricing(request.config.model, servedModel),
|
|
3659
|
+
);
|
|
3660
|
+
// Honest turn telemetry: provider calls actually made (including refusal
|
|
3661
|
+
// re-issues inside streamOnce) and continuation rounds.
|
|
3662
|
+
let providerCalls = 0;
|
|
3663
|
+
let rounds = 0;
|
|
2888
3664
|
let lastStopReason: StopReason = 'end_turn';
|
|
2889
3665
|
let lastStopSequence: string | undefined;
|
|
2890
3666
|
let rawRequest: unknown;
|
|
@@ -2893,9 +3669,12 @@ export class Membrane {
|
|
|
2893
3669
|
let allTextAccumulated = '';
|
|
2894
3670
|
const executedToolCalls: ToolCall[] = [];
|
|
2895
3671
|
const executedToolResults: ToolResult[] = [];
|
|
3672
|
+
// Spend on refusal attempts this turn threw away (see streamOnce).
|
|
3673
|
+
let discardedUsage: DiscardedAttemptsUsage | undefined;
|
|
2896
3674
|
|
|
2897
3675
|
let messages = [...request.messages];
|
|
2898
3676
|
let allContentBlocks: ContentBlock[] = [];
|
|
3677
|
+
let markersInLastRequest = 0;
|
|
2899
3678
|
|
|
2900
3679
|
try {
|
|
2901
3680
|
// Tool execution loop
|
|
@@ -2913,25 +3692,20 @@ export class Membrane {
|
|
|
2913
3692
|
}
|
|
2914
3693
|
|
|
2915
3694
|
// Build provider request with native tools
|
|
2916
|
-
const providerRequest = this.buildNativeToolRequest(request, messages);
|
|
3695
|
+
const providerRequest = this.buildNativeToolRequest(request, messages, toolDepth > 0, activeFormatter);
|
|
2917
3696
|
|
|
2918
3697
|
// Stream from provider
|
|
2919
3698
|
let textAccumulated = '';
|
|
2920
|
-
let blockIndex = 0;
|
|
2921
3699
|
// Where this attempt starts inside the tool-loop-spanning buffer, so
|
|
2922
3700
|
// a refusal retry can roll back exactly this attempt's contribution.
|
|
2923
3701
|
const allTextBefore = allTextAccumulated.length;
|
|
2924
|
-
// Track block-type from the provider's
|
|
3702
|
+
// Track block-type from the provider's content_block signals so
|
|
2925
3703
|
// every token chunk is tagged with the membrane block it belongs to.
|
|
2926
3704
|
// Without this, thinking_delta chunks get mislabelled as 'text' and
|
|
2927
3705
|
// downstream consumers (TUIs, WebUIs) can't render them distinctly.
|
|
2928
|
-
|
|
2929
|
-
|
|
2930
|
-
|
|
2931
|
-
if (apiType === 'thinking') return 'thinking';
|
|
2932
|
-
if (apiType === 'tool_use') return 'tool_call';
|
|
2933
|
-
return 'text';
|
|
2934
|
-
};
|
|
3706
|
+
const tracker = new NativeBlockTracker(
|
|
3707
|
+
emitBlocks ? (event) => stream.emit({ type: 'block', event }) : undefined,
|
|
3708
|
+
);
|
|
2935
3709
|
const streamResult = await this.streamOnce(
|
|
2936
3710
|
providerRequest,
|
|
2937
3711
|
{
|
|
@@ -2943,54 +3717,16 @@ export class Membrane {
|
|
|
2943
3717
|
|
|
2944
3718
|
if (emitTokens) {
|
|
2945
3719
|
const meta: ChunkMeta = {
|
|
2946
|
-
type:
|
|
2947
|
-
visible:
|
|
2948
|
-
blockIndex,
|
|
3720
|
+
type: tracker.currentType,
|
|
3721
|
+
visible: tracker.currentType === 'text',
|
|
3722
|
+
blockIndex: tracker.blockIndex,
|
|
2949
3723
|
};
|
|
2950
3724
|
stream.emit({ type: 'tokens', content: chunk, meta });
|
|
2951
3725
|
}
|
|
2952
3726
|
},
|
|
2953
3727
|
onContentBlock: (index, block) => {
|
|
2954
3728
|
if (stream.isCancelled) return;
|
|
2955
|
-
|
|
2956
|
-
const mbType = mapApiBlockType(apiType);
|
|
2957
|
-
const isStart = !seenBlockIndices.has(index);
|
|
2958
|
-
if (isStart) {
|
|
2959
|
-
seenBlockIndices.add(index);
|
|
2960
|
-
currentBlockType = mbType;
|
|
2961
|
-
blockIndex = index;
|
|
2962
|
-
if (emitBlocks) {
|
|
2963
|
-
stream.emit({
|
|
2964
|
-
type: 'block',
|
|
2965
|
-
event: { event: 'block_start', index, block: { type: mbType } },
|
|
2966
|
-
});
|
|
2967
|
-
}
|
|
2968
|
-
} else if (emitBlocks) {
|
|
2969
|
-
// Second call for the same index = content_block_stop. The
|
|
2970
|
-
// provider has filled the block with final content; surface
|
|
2971
|
-
// a block_complete with the relevant fields for consumers
|
|
2972
|
-
// that want full block payloads (e.g. context-manager).
|
|
2973
|
-
const apiBlock = block as {
|
|
2974
|
-
type?: string;
|
|
2975
|
-
text?: string;
|
|
2976
|
-
thinking?: string;
|
|
2977
|
-
id?: string;
|
|
2978
|
-
name?: string;
|
|
2979
|
-
input?: unknown;
|
|
2980
|
-
} | undefined;
|
|
2981
|
-
const mb: MembraneBlock = { type: mbType };
|
|
2982
|
-
if (mbType === 'text') mb.content = apiBlock?.text;
|
|
2983
|
-
else if (mbType === 'thinking') mb.content = apiBlock?.thinking;
|
|
2984
|
-
else if (mbType === 'tool_call') {
|
|
2985
|
-
mb.toolId = apiBlock?.id;
|
|
2986
|
-
mb.toolName = apiBlock?.name;
|
|
2987
|
-
mb.input = apiBlock?.input as Record<string, unknown> | undefined;
|
|
2988
|
-
}
|
|
2989
|
-
stream.emit({
|
|
2990
|
-
type: 'block',
|
|
2991
|
-
event: { event: 'block_complete', index, block: mb },
|
|
2992
|
-
});
|
|
2993
|
-
}
|
|
3729
|
+
tracker.onProviderBlock(index, block);
|
|
2994
3730
|
},
|
|
2995
3731
|
},
|
|
2996
3732
|
{
|
|
@@ -2999,6 +3735,14 @@ export class Membrane {
|
|
|
2999
3735
|
idleTimeoutMs: options.idleTimeoutMs,
|
|
3000
3736
|
normalizedRequest: request,
|
|
3001
3737
|
onRequest: (req: unknown) => { rawRequest = req; },
|
|
3738
|
+
// Telemetry reports what this request actually SHIPPED with —
|
|
3739
|
+
// builder breakpoints, stale passthrough, fallback, float, plus
|
|
3740
|
+
// whatever the beforeRequest hook and the wire clamp did after
|
|
3741
|
+
// the build. Both native paths used to hardcode 0, and counting
|
|
3742
|
+
// at build time reported a number no request ever had.
|
|
3743
|
+
onWireCacheMarkers: (markerCount: number) => {
|
|
3744
|
+
markersInLastRequest = markerCount;
|
|
3745
|
+
},
|
|
3002
3746
|
refusalRetries: options.refusalRetries,
|
|
3003
3747
|
// Discard the refused attempt: roll the accumulators back to
|
|
3004
3748
|
// where this attempt began and tell the consumer to drop what it
|
|
@@ -3007,9 +3751,7 @@ export class Membrane {
|
|
|
3007
3751
|
onRetrying: (info) => {
|
|
3008
3752
|
allTextAccumulated = allTextAccumulated.slice(0, allTextBefore);
|
|
3009
3753
|
textAccumulated = '';
|
|
3010
|
-
|
|
3011
|
-
currentBlockType = 'text';
|
|
3012
|
-
seenBlockIndices.clear();
|
|
3754
|
+
tracker.reset();
|
|
3013
3755
|
stream.emit({
|
|
3014
3756
|
type: 'retrying',
|
|
3015
3757
|
attempt: info.attempt,
|
|
@@ -3021,22 +3763,24 @@ export class Membrane {
|
|
|
3021
3763
|
}
|
|
3022
3764
|
);
|
|
3023
3765
|
|
|
3766
|
+
// Single-callback adapters (OpenAI Responses) report each finalised
|
|
3767
|
+
// block once, after the stream: complete whatever never saw a stop.
|
|
3768
|
+
tracker.flush();
|
|
3769
|
+
rounds++;
|
|
3770
|
+
providerCalls += streamResult.providerCalls;
|
|
3771
|
+
|
|
3024
3772
|
rawResponse = streamResult.raw;
|
|
3025
3773
|
lastStopReason = this.mapStopReason(streamResult.stopReason);
|
|
3026
3774
|
lastStopSequence = streamResult.stopSequence ?? undefined;
|
|
3027
3775
|
|
|
3028
|
-
//
|
|
3029
|
-
|
|
3030
|
-
|
|
3031
|
-
|
|
3032
|
-
|
|
3033
|
-
|
|
3034
|
-
if (streamResult.usage.cacheReadTokens) {
|
|
3035
|
-
totalUsage.cacheReadTokens = (totalUsage.cacheReadTokens ?? 0) + streamResult.usage.cacheReadTokens;
|
|
3036
|
-
}
|
|
3037
|
-
if (pricing) totalUsage.estimatedCost = calculateCost(totalUsage, pricing);
|
|
3776
|
+
// Attempts this round re-issued past a refusal are billed calls whose
|
|
3777
|
+
// output was discarded — carry their spend to the final response.
|
|
3778
|
+
discardedUsage = this.mergeDiscardedAttempts(discardedUsage, streamResult.discardedUsage);
|
|
3779
|
+
|
|
3780
|
+
// Accumulate usage (including cache metrics), priced at this round's model
|
|
3781
|
+
const usageSoFar = turnUsage.addRound(streamResult.model, streamResult.usage);
|
|
3038
3782
|
if (emitUsage) {
|
|
3039
|
-
stream.emit({ type: 'usage', usage:
|
|
3783
|
+
stream.emit({ type: 'usage', usage: usageSoFar });
|
|
3040
3784
|
}
|
|
3041
3785
|
|
|
3042
3786
|
// Parse content blocks from response
|
|
@@ -3139,6 +3883,7 @@ export class Membrane {
|
|
|
3139
3883
|
}
|
|
3140
3884
|
|
|
3141
3885
|
const durationMs = Date.now() - startTime;
|
|
3886
|
+
const totalUsage = turnUsage.total;
|
|
3142
3887
|
|
|
3143
3888
|
const response: NormalizedResponse = {
|
|
3144
3889
|
content: allContentBlocks,
|
|
@@ -3153,18 +3898,25 @@ export class Membrane {
|
|
|
3153
3898
|
triggeredSequence: lastStopSequence,
|
|
3154
3899
|
wasTruncated: lastStopReason === 'max_tokens',
|
|
3155
3900
|
},
|
|
3156
|
-
usage: {
|
|
3901
|
+
usage: {
|
|
3902
|
+
...totalUsage,
|
|
3903
|
+
...(discardedUsage
|
|
3904
|
+
? { discardedAttempts: this.pricedDiscardedAttempts(discardedUsage, request.config.model) }
|
|
3905
|
+
: {}),
|
|
3906
|
+
},
|
|
3157
3907
|
timing: {
|
|
3158
3908
|
totalDurationMs: durationMs,
|
|
3159
|
-
attempts:
|
|
3909
|
+
attempts: providerCalls,
|
|
3910
|
+
rounds,
|
|
3160
3911
|
},
|
|
3161
3912
|
model: {
|
|
3162
3913
|
requested: request.config.model,
|
|
3163
|
-
actual: request.config.model,
|
|
3914
|
+
actual: turnUsage.lastServedModel || request.config.model,
|
|
3164
3915
|
provider: this.adapter.name,
|
|
3916
|
+
perRound: turnUsage.perRound,
|
|
3165
3917
|
},
|
|
3166
3918
|
cache: {
|
|
3167
|
-
markersInRequest:
|
|
3919
|
+
markersInRequest: markersInLastRequest,
|
|
3168
3920
|
tokensCreated: totalUsage.cacheCreationTokens ?? 0,
|
|
3169
3921
|
tokensRead: totalUsage.cacheReadTokens ?? 0,
|
|
3170
3922
|
hitRatio: this.calculateCacheHitRatio(totalUsage),
|
|
@@ -3181,7 +3933,7 @@ export class Membrane {
|
|
|
3181
3933
|
if (this.isAbortError(error)) {
|
|
3182
3934
|
stream.emit({
|
|
3183
3935
|
type: 'aborted',
|
|
3184
|
-
reason:
|
|
3936
|
+
reason: this.abortReason(error, stream.signal),
|
|
3185
3937
|
rawAssistantText: allTextAccumulated,
|
|
3186
3938
|
toolCalls: executedToolCalls,
|
|
3187
3939
|
toolResults: executedToolResults,
|