@animalabs/membrane 0.5.76 → 0.5.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/membrane.d.ts.map +1 -1
- package/dist/membrane.js +301 -19
- package/dist/membrane.js.map +1 -1
- package/dist/providers/bedrock.d.ts +17 -0
- package/dist/providers/bedrock.d.ts.map +1 -1
- package/dist/providers/bedrock.js +86 -15
- package/dist/providers/bedrock.js.map +1 -1
- package/dist/types/config.d.ts +31 -1
- package/dist/types/config.d.ts.map +1 -1
- package/dist/types/config.js +8 -0
- package/dist/types/config.js.map +1 -1
- package/dist/types/errors.d.ts +12 -0
- package/dist/types/errors.d.ts.map +1 -1
- package/dist/types/errors.js +19 -0
- package/dist/types/errors.js.map +1 -1
- package/dist/types/index.d.ts +1 -1
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/index.js +1 -1
- package/dist/types/index.js.map +1 -1
- package/dist/types/response.d.ts +1 -1
- package/dist/types/response.d.ts.map +1 -1
- package/dist/types/response.js.map +1 -1
- package/dist/types/streaming.d.ts +24 -0
- package/dist/types/streaming.d.ts.map +1 -1
- package/dist/types/yielding-stream.d.ts +57 -1
- package/dist/types/yielding-stream.d.ts.map +1 -1
- package/dist/types/yielding-stream.js.map +1 -1
- package/package.json +1 -1
- package/src/membrane.ts +350 -17
- package/src/providers/bedrock.ts +100 -15
- package/src/types/config.ts +48 -4
- package/src/types/errors.ts +18 -0
- package/src/types/index.ts +1 -0
- package/src/types/response.ts +5 -1
- package/src/types/streaming.ts +26 -0
- package/src/types/yielding-stream.ts +60 -0
package/dist/membrane.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"membrane.d.ts","sourceRoot":"","sources":["../src/membrane.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EACV,iBAAiB,EACjB,kBAAkB,EAClB,eAAe,EAEf,eAAe,EAEf,cAAc,EACd,aAAa,EACb,eAAe,EAYhB,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"membrane.d.ts","sourceRoot":"","sources":["../src/membrane.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EACV,iBAAiB,EACjB,kBAAkB,EAClB,eAAe,EAEf,eAAe,EAEf,cAAc,EACd,aAAa,EACb,eAAe,EAYhB,MAAM,kBAAkB,CAAC;AAqB1B,OAAO,KAAK,EACV,cAAc,EACd,qBAAqB,EAGtB,MAAM,4BAA4B,CAAC;AAiBpC,qBAAa,QAAQ;IACnB,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,QAAQ,CAAC,CAAgB;IACjC,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,SAAS,CAAmB;gBAGlC,OAAO,EAAE,eAAe,EACxB,MAAM,GAAE,cAAmB;IAkB7B;;OAEG;IACG,QAAQ,CACZ,OAAO,EAAE,iBAAiB,EAC1B,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,kBAAkB,CAAC;IAyG9B;;;;;;;;;;;;;;;;;OAiBG;IACG,MAAM,CACV,OAAO,EAAE,iBAAiB,EAC1B,OAAO,GAAE,aAAkB,GAC1B,OAAO,CAAC,kBAAkB,GAAG,eAAe,CAAC;IA8FhD;;OAEG;IACH,OAAO,CAAC,eAAe;IAsBvB;;;;;;OAMG;YACW,kBAAkB;IA8kBhC;;OAEG;YACW,qBAAqB;IAwOnC;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAwN9B;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAuD5B;;;;;OAKG;IACH,OAAO,CAAC,6BAA6B;IAkBrC;;;;;;;;;OASG;IACH,OAAO,CAAC,2BAA2B;IA+BnC;;;;;;;OAOG;YACW,sBAAsB;IASpC;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAoB7B;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,qBAAqB;IAIpC;;;;;;;;OAQG;IACH,OAAO,CAAC,kBAAkB;IAqB1B;;OAEG;IACH,OAAO,CAAC,gBAAgB;YA4EV,UAAU;IA4DxB,OAAO,CAAC,wBAAwB;IA8ChC;;;;;;;;;;;;;;;;OAgBG;IACH,OAAO,CAAC,kCAAkC;IAoD1C,OAAO,CAAC,iBAAiB;IAqIzB,OAAO,CAAC,kBAAkB;IAkF1B,OAAO,CAAC,aAAa;IAoBrB,OAAO,CAAC,sBAAsB;IAO9B,OAAO,CAAC,cAAc;IAItB,qFAAqF;IACrF,OAAO,CAAC,YAAY;IAKpB,OAAO,CAAC,mBAAmB;IAW3B,OAAO,CAAC,gBAAgB;IAMxB,OAAO,CAAC,KAAK;IAcb;;OAEG;IACH,OAAO,CAAC,YAAY;IAcpB;;OAEG;IACH,OAAO,CAAC,oBAAoB;IA4B5B;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACH,cAAc,CACZ,OAAO,EAAE,iBAAiB,EAC1B,OAAO,GAAE,qBAA0B,GAClC,cAAc;IAuBjB;;OAEG;YACW,mBAAmB;IAokBjC;;OAEG;YACW,sBAAsB;CA2UrC"}
|
package/dist/membrane.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* A selective boundary that transforms what passes through.
|
|
5
5
|
*/
|
|
6
6
|
import { lastCacheableBlockIndex } from './formatters/native.js';
|
|
7
|
-
import { DEFAULT_RETRY_CONFIG, MembraneError, classifyError, } from './types/index.js';
|
|
7
|
+
import { DEFAULT_RETRY_CONFIG, MembraneError, classifyError, isOverloadedError, } from './types/index.js';
|
|
8
8
|
import { parseToolCalls, formatToolResults, parseAccumulatedIntoBlocks, hasImageInToolResults, formatToolResultsForSplitTurn, } from './utils/tool-parser.js';
|
|
9
9
|
import { AnthropicXmlFormatter } from './formatters/anthropic-xml.js';
|
|
10
10
|
import { normalizeToolPairs, mergeConsecutiveRoles } from './formatters/normalize-tool-pairs.js';
|
|
@@ -24,7 +24,11 @@ export class Membrane {
|
|
|
24
24
|
constructor(adapter, config = {}) {
|
|
25
25
|
this.adapter = adapter;
|
|
26
26
|
this.registry = config.registry;
|
|
27
|
-
this.retryConfig = {
|
|
27
|
+
this.retryConfig = {
|
|
28
|
+
...DEFAULT_RETRY_CONFIG,
|
|
29
|
+
...config.retry,
|
|
30
|
+
overloaded: { ...DEFAULT_RETRY_CONFIG.overloaded, ...config.retry?.overloaded },
|
|
31
|
+
};
|
|
28
32
|
this.config = config;
|
|
29
33
|
// Use provided formatter or default to AnthropicXmlFormatter
|
|
30
34
|
this.formatter = config.formatter ?? new AnthropicXmlFormatter();
|
|
@@ -39,6 +43,10 @@ export class Membrane {
|
|
|
39
43
|
const startTime = Date.now();
|
|
40
44
|
let attempts = 0;
|
|
41
45
|
let rawRequest;
|
|
46
|
+
// Counted separately from `attempts` (the transport-error budget): a
|
|
47
|
+
// refusal is a successful HTTP call with an unwanted verdict, and letting
|
|
48
|
+
// it consume error retries would couple two unrelated budgets.
|
|
49
|
+
let refusalRetriesUsed = 0;
|
|
42
50
|
while (true) {
|
|
43
51
|
attempts++;
|
|
44
52
|
try {
|
|
@@ -60,6 +68,16 @@ export class Membrane {
|
|
|
60
68
|
// Call onResponse callback with raw response from API
|
|
61
69
|
options.onResponse?.(providerResponse.raw);
|
|
62
70
|
const response = this.transformResponse(providerResponse, request, prefillResult, startTime, attempts, rawRequest);
|
|
71
|
+
// Re-issue a content-policy refusal (opt-in, default off). Safe here
|
|
72
|
+
// in a way the streaming paths are not: nothing has reached the
|
|
73
|
+
// caller yet, so the abandoned attempt leaves no trace to retract.
|
|
74
|
+
// Deliberately BEFORE afterResponse — a hook that logs or transforms
|
|
75
|
+
// should see the attempt that actually stands, not the discarded one.
|
|
76
|
+
if (response.stopReason === 'refusal' &&
|
|
77
|
+
refusalRetriesUsed < Math.max(0, options.refusalRetries ?? 0)) {
|
|
78
|
+
refusalRetriesUsed++;
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
63
81
|
// Call afterResponse hook
|
|
64
82
|
if (this.config.hooks?.afterResponse) {
|
|
65
83
|
return await this.config.hooks.afterResponse(response, providerResponse.raw);
|
|
@@ -69,12 +87,22 @@ export class Membrane {
|
|
|
69
87
|
catch (error) {
|
|
70
88
|
const errorInfo = classifyError(error);
|
|
71
89
|
errorInfo.rawRequest = rawRequest;
|
|
72
|
-
// Rate limits (429) always retry up to 5 attempts regardless of
|
|
73
|
-
//
|
|
90
|
+
// Rate limits (429) always retry up to 5 attempts regardless of
|
|
91
|
+
// config, and overloaded (529) always retries on its own longer
|
|
92
|
+
// schedule — both are transient by definition, and the default
|
|
93
|
+
// maxRetries of 0 would otherwise turn a capacity blip into a dead
|
|
94
|
+
// turn. Other retryable errors only retry when maxRetries > 0.
|
|
95
|
+
// overloaded.maxRetries: 0 disables the dedicated policy entirely;
|
|
96
|
+
// the 529 then follows the base config like any retryable server
|
|
97
|
+
// error (exactly the pre-policy behavior), rather than being
|
|
98
|
+
// silently re-promoted to the long schedule by a positive base limit.
|
|
74
99
|
const isRateLimit = errorInfo.type === 'rate_limit';
|
|
100
|
+
const isOverloaded = isOverloadedError(errorInfo) && this.retryConfig.overloaded.maxRetries > 0;
|
|
75
101
|
const effectiveMax = isRateLimit
|
|
76
102
|
? Math.max(this.retryConfig.maxRetries, 5)
|
|
77
|
-
:
|
|
103
|
+
: isOverloaded
|
|
104
|
+
? Math.max(this.retryConfig.maxRetries, this.retryConfig.overloaded.maxRetries)
|
|
105
|
+
: this.retryConfig.maxRetries;
|
|
78
106
|
if (errorInfo.retryable && attempts < effectiveMax) {
|
|
79
107
|
// Check hook for retry decision
|
|
80
108
|
if (this.config.hooks?.onError) {
|
|
@@ -84,7 +112,7 @@ export class Membrane {
|
|
|
84
112
|
}
|
|
85
113
|
}
|
|
86
114
|
// Wait before retry (abort-aware)
|
|
87
|
-
const delay = this.calculateRetryDelay(attempts);
|
|
115
|
+
const delay = this.calculateRetryDelay(attempts, isOverloaded);
|
|
88
116
|
await this.sleep(delay, options.signal);
|
|
89
117
|
continue;
|
|
90
118
|
}
|
|
@@ -132,11 +160,69 @@ export class Membrane {
|
|
|
132
160
|
}
|
|
133
161
|
// Determine tool mode
|
|
134
162
|
const toolMode = this.resolveToolMode(request);
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
163
|
+
const useNative = toolMode === 'native' && !!request.tools && request.tools.length > 0;
|
|
164
|
+
// Overloaded (529) pre-emission retry. The streaming paths have no retry
|
|
165
|
+
// loop of their own, so a capacity error used to kill the turn outright —
|
|
166
|
+
// and 529s most often arrive INSTEAD of a stream, before anything reaches
|
|
167
|
+
// the caller, where retrying is transparent. Once any callback has
|
|
168
|
+
// delivered output (tokens, blocks, usage), retrying would replay content
|
|
169
|
+
// the caller already consumed, so mid-stream errors still throw.
|
|
170
|
+
let attempts = 0;
|
|
171
|
+
const retryDelaysMs = [];
|
|
172
|
+
while (true) {
|
|
173
|
+
attempts++;
|
|
174
|
+
let emitted = false;
|
|
175
|
+
const mark = (fn) => fn && ((...args) => { emitted = true; return fn(...args); });
|
|
176
|
+
const tracked = {
|
|
177
|
+
...options,
|
|
178
|
+
onChunk: mark(options.onChunk),
|
|
179
|
+
onContentBlockUpdate: mark(options.onContentBlockUpdate),
|
|
180
|
+
onToolCalls: mark(options.onToolCalls),
|
|
181
|
+
onPreToolContent: mark(options.onPreToolContent),
|
|
182
|
+
onUsage: mark(options.onUsage),
|
|
183
|
+
onBlock: mark(options.onBlock),
|
|
184
|
+
onResponse: mark(options.onResponse),
|
|
185
|
+
// onRequest fires before the send — it is not an emission.
|
|
186
|
+
};
|
|
187
|
+
try {
|
|
188
|
+
const result = useNative
|
|
189
|
+
? await this.streamWithNativeTools(request, tracked)
|
|
190
|
+
: await this.streamWithXmlTools(request, tracked);
|
|
191
|
+
// The inner paths report attempts: 1 — they can't see this wrapper.
|
|
192
|
+
// A call that succeeded after N overloaded retries must not look like
|
|
193
|
+
// a first-attempt success in durable logs, so patch the real count
|
|
194
|
+
// (and the waits) into the response telemetry.
|
|
195
|
+
if (attempts > 1 && 'details' in result) {
|
|
196
|
+
result.details.timing.attempts = attempts;
|
|
197
|
+
result.details.timing.retryDelaysMs = retryDelaysMs;
|
|
198
|
+
}
|
|
199
|
+
return result;
|
|
200
|
+
}
|
|
201
|
+
catch (error) {
|
|
202
|
+
const errorInfo = classifyError(error);
|
|
203
|
+
// Same semantics as complete(): maxRetries bounds total attempts,
|
|
204
|
+
// the overloaded floor applies over the base config, and
|
|
205
|
+
// overloaded.maxRetries: 0 opts out of stream retries entirely
|
|
206
|
+
// (streaming had no retry before this policy existed).
|
|
207
|
+
const overloadedEnabled = this.retryConfig.overloaded.maxRetries > 0;
|
|
208
|
+
const maxOverloaded = Math.max(this.retryConfig.maxRetries, this.retryConfig.overloaded.maxRetries);
|
|
209
|
+
if (!emitted && overloadedEnabled && isOverloadedError(errorInfo) && attempts < maxOverloaded) {
|
|
210
|
+
// Honor the same pre-retry hook contract as complete(): hosts use
|
|
211
|
+
// onError for circuit-breaking, and its 'abort' decision must work
|
|
212
|
+
// on the streaming path too.
|
|
213
|
+
if (this.config.hooks?.onError) {
|
|
214
|
+
const decision = await this.config.hooks.onError(errorInfo, attempts);
|
|
215
|
+
if (decision === 'abort') {
|
|
216
|
+
throw error;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
const delay = this.calculateRetryDelay(attempts, true);
|
|
220
|
+
retryDelaysMs.push(delay);
|
|
221
|
+
await this.sleep(delay, options.signal);
|
|
222
|
+
continue;
|
|
223
|
+
}
|
|
224
|
+
throw error;
|
|
225
|
+
}
|
|
140
226
|
}
|
|
141
227
|
}
|
|
142
228
|
/**
|
|
@@ -216,13 +302,56 @@ export class Membrane {
|
|
|
216
302
|
// blocks inherited from prefill context (e.g., unclosed <thinking> from other bots)
|
|
217
303
|
// from blocks the model itself opened during generation
|
|
218
304
|
const prefillDepths = parser.getDepths();
|
|
305
|
+
// Resumption spin guards (issue #39). Observed live on Ash 2026-07-26:
|
|
306
|
+
// each automatic resumption re-sent ~172k input tokens, streamed ~6
|
|
307
|
+
// output tokens, and stopped on the same (dropped) stop sequence — 43
|
|
308
|
+
// rounds, ~7M input tokens, zero progress, found only because a human
|
|
309
|
+
// noticed. Two guards, both scoped to AUTOMATIC false-positive
|
|
310
|
+
// resumptions — tool rounds are real caller-governed work (maxToolDepth
|
|
311
|
+
// / the yielding API's uncapped contract) and are never counted here:
|
|
312
|
+
// - stall guard: several CONSECUTIVE resumptions that each stream
|
|
313
|
+
// almost nothing and stop identically end the turn ('no_progress').
|
|
314
|
+
// One short repeated round is low progress, not proof of none — a
|
|
315
|
+
// stop sequence inside legitimate tool-argument text can cause a
|
|
316
|
+
// couple of short resumptions on the way to completing.
|
|
317
|
+
// - round cap: a hard bound on resumptions per turn ('round_limit'),
|
|
318
|
+
// the backstop for a spin that keeps technically progressing.
|
|
319
|
+
const MIN_ROUND_PROGRESS_CHARS = 16;
|
|
320
|
+
const MAX_CONSECUTIVE_STALLED_RESUMPTIONS = 3;
|
|
321
|
+
const RESUMPTION_WARN_ROUNDS = 5;
|
|
322
|
+
const maxResumptionRounds = options.maxResumptionRounds ?? 24;
|
|
323
|
+
let resumptionRounds = 0;
|
|
324
|
+
let consecutiveStalledResumptions = 0;
|
|
325
|
+
let enteredViaResumption = false;
|
|
326
|
+
let prevRoundStopSequence;
|
|
327
|
+
const warnLog = this.config.logger ?? console;
|
|
328
|
+
/** Count an automatic resumption; emits the visibility warning at the
|
|
329
|
+
* threshold and returns false when the cap says the turn should end. */
|
|
330
|
+
const registerResumptionRound = () => {
|
|
331
|
+
resumptionRounds++;
|
|
332
|
+
if (resumptionRounds === RESUMPTION_WARN_ROUNDS) {
|
|
333
|
+
warnLog.warn(`[membrane] automatic resumption at round ${resumptionRounds} ` +
|
|
334
|
+
`(${totalUsage.inputTokens} input tokens so far this turn) — ` +
|
|
335
|
+
`a spin shows up here before it shows up on the bill`);
|
|
336
|
+
}
|
|
337
|
+
if (resumptionRounds > maxResumptionRounds) {
|
|
338
|
+
warnLog.warn(`[membrane] automatic resumption cap (${maxResumptionRounds}) reached — ` +
|
|
339
|
+
`ending turn with stopReason 'round_limit'. ` +
|
|
340
|
+
`${totalUsage.inputTokens} input tokens spent this turn.`);
|
|
341
|
+
return false;
|
|
342
|
+
}
|
|
343
|
+
return true;
|
|
344
|
+
};
|
|
219
345
|
try {
|
|
220
346
|
// Tool execution loop
|
|
221
347
|
while (toolDepth <= maxToolDepth) {
|
|
222
348
|
// Track if we manually detected a stop sequence (API doesn't always stop)
|
|
223
349
|
let detectedStopSequence = null;
|
|
224
350
|
let truncatedAccumulated = null;
|
|
225
|
-
// Track where to start checking for stop sequences (skip already-processed content)
|
|
351
|
+
// Track where to start checking for stop sequences (skip already-processed content).
|
|
352
|
+
// Also the round's progress baseline: XML we pushed ourselves at the
|
|
353
|
+
// end of the previous round (tool results, closing tags) sits below
|
|
354
|
+
// this index and doesn't count as model progress.
|
|
226
355
|
const checkFromIndex = parser.getAccumulated().length;
|
|
227
356
|
// Stream from provider
|
|
228
357
|
const streamResult = await this.streamOnce(providerRequest, {
|
|
@@ -332,6 +461,34 @@ export class Membrane {
|
|
|
332
461
|
}
|
|
333
462
|
// Get accumulated text from parser
|
|
334
463
|
const accumulated = parser.getAccumulated();
|
|
464
|
+
// Stall accounting (issue #39): only rounds ENTERED via automatic
|
|
465
|
+
// resumption can stall — tool rounds are caller-governed work and a
|
|
466
|
+
// real tool call is longer than the threshold anyway. A stall is a
|
|
467
|
+
// resumption that streamed almost nothing and stopped identically to
|
|
468
|
+
// the previous round; the turn ends only after several IN A ROW
|
|
469
|
+
// (one short repeated round is low progress, not proof of none —
|
|
470
|
+
// a stop sequence inside legitimate tool-argument text can cause a
|
|
471
|
+
// couple of short resumptions on the way to completing).
|
|
472
|
+
const streamedThisRound = accumulated.length - checkFromIndex;
|
|
473
|
+
if (enteredViaResumption &&
|
|
474
|
+
lastStopReason === 'stop_sequence' &&
|
|
475
|
+
streamedThisRound < MIN_ROUND_PROGRESS_CHARS &&
|
|
476
|
+
lastStopSequence === prevRoundStopSequence) {
|
|
477
|
+
consecutiveStalledResumptions++;
|
|
478
|
+
if (consecutiveStalledResumptions >= MAX_CONSECUTIVE_STALLED_RESUMPTIONS) {
|
|
479
|
+
warnLog.warn(`[membrane] ${consecutiveStalledResumptions} consecutive automatic resumptions ` +
|
|
480
|
+
`made no progress (${streamedThisRound} chars this round, stop ` +
|
|
481
|
+
`${JSON.stringify(lastStopSequence ?? null)} repeated) — ending turn with ` +
|
|
482
|
+
`stopReason 'no_progress'. ${totalUsage.inputTokens} input tokens spent this turn.`);
|
|
483
|
+
lastStopReason = 'no_progress';
|
|
484
|
+
break;
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
else {
|
|
488
|
+
consecutiveStalledResumptions = 0;
|
|
489
|
+
}
|
|
490
|
+
prevRoundStopSequence = lastStopSequence;
|
|
491
|
+
enteredViaResumption = false;
|
|
335
492
|
// Check for tool calls (if handler provided)
|
|
336
493
|
if (onToolCalls && streamResult.stopSequence === '</function_calls>') {
|
|
337
494
|
// Append the closing tag (we truncated before it, or API stopped before it)
|
|
@@ -490,7 +647,9 @@ export class Membrane {
|
|
|
490
647
|
prefillResult.assistantPrefill = parser.getAccumulated();
|
|
491
648
|
providerRequest = this.buildContinuationRequest(request, prefillResult, parser.getAccumulated());
|
|
492
649
|
}
|
|
493
|
-
// Reset parser state for new streaming iteration
|
|
650
|
+
// Reset parser state for new streaming iteration. Tool rounds
|
|
651
|
+
// are the caller's work — they count against maxToolDepth only,
|
|
652
|
+
// never against the resumption guards (issue #39 review).
|
|
494
653
|
parser.resetForNewIteration();
|
|
495
654
|
toolDepth++;
|
|
496
655
|
continue;
|
|
@@ -523,6 +682,11 @@ export class Membrane {
|
|
|
523
682
|
if (toolDepth > maxToolDepth) {
|
|
524
683
|
break;
|
|
525
684
|
}
|
|
685
|
+
if (!registerResumptionRound()) {
|
|
686
|
+
lastStopReason = 'round_limit';
|
|
687
|
+
break;
|
|
688
|
+
}
|
|
689
|
+
enteredViaResumption = true;
|
|
526
690
|
prefillResult.assistantPrefill = parser.getAccumulated();
|
|
527
691
|
providerRequest = this.buildContinuationRequest(request, prefillResult, parser.getAccumulated());
|
|
528
692
|
// Reset parser state for new streaming iteration
|
|
@@ -1206,9 +1370,21 @@ export class Membrane {
|
|
|
1206
1370
|
// compatibility won't catch the excess field (checked only on object
|
|
1207
1371
|
// literals, not on variables). Leaving it in would silently leak the
|
|
1208
1372
|
// normalized form into every adapter's options.
|
|
1209
|
-
const { normalizedRequest, ...adapterOptions } = options;
|
|
1373
|
+
const { normalizedRequest, refusalRetries, onRetrying, ...adapterOptions } = options;
|
|
1210
1374
|
const finalRequest = (await this.applyBeforeRequestHook(normalizedRequest, request));
|
|
1211
|
-
|
|
1375
|
+
// Retries are only safe when the caller can discard the abandoned
|
|
1376
|
+
// attempt, so they require BOTH a budget and an onRetrying hook.
|
|
1377
|
+
const maxAttempts = onRetrying ? Math.max(0, refusalRetries ?? 0) : 0;
|
|
1378
|
+
let retried = 0;
|
|
1379
|
+
while (true) {
|
|
1380
|
+
const result = await this.adapter.stream(finalRequest, callbacks, adapterOptions);
|
|
1381
|
+
if (result.stopReason !== 'refusal' || retried >= maxAttempts)
|
|
1382
|
+
return result;
|
|
1383
|
+
retried++;
|
|
1384
|
+
const category = result.raw
|
|
1385
|
+
?.response?.stop_details?.category;
|
|
1386
|
+
onRetrying({ attempt: retried, maxAttempts, category });
|
|
1387
|
+
}
|
|
1212
1388
|
}
|
|
1213
1389
|
buildContinuationRequest(originalRequest, prefillResult, accumulated) {
|
|
1214
1390
|
// Anthropic quirk: assistant content cannot end with trailing whitespace
|
|
@@ -1523,10 +1699,15 @@ export class Membrane {
|
|
|
1523
1699
|
const pricing = this.resolvePricing(model);
|
|
1524
1700
|
return pricing ? calculateCost(usage, pricing) : undefined;
|
|
1525
1701
|
}
|
|
1526
|
-
calculateRetryDelay(attempt) {
|
|
1527
|
-
const { retryDelayMs, backoffMultiplier, maxRetryDelayMs } =
|
|
1528
|
-
|
|
1529
|
-
|
|
1702
|
+
calculateRetryDelay(attempt, overloaded = false) {
|
|
1703
|
+
const { retryDelayMs, backoffMultiplier, maxRetryDelayMs } = overloaded
|
|
1704
|
+
? this.retryConfig.overloaded
|
|
1705
|
+
: this.retryConfig;
|
|
1706
|
+
const delay = Math.min(retryDelayMs * Math.pow(backoffMultiplier, attempt - 1), maxRetryDelayMs);
|
|
1707
|
+
// Equal jitter on the overloaded schedule only: a capacity storm is
|
|
1708
|
+
// exactly the case where a fleet retrying in sync re-creates the
|
|
1709
|
+
// stampede it's backing off from. [delay/2, delay) keeps the wait long.
|
|
1710
|
+
return overloaded ? Math.floor(delay / 2 + Math.random() * (delay / 2)) : delay;
|
|
1530
1711
|
}
|
|
1531
1712
|
attachRawRequest(error, rawRequest) {
|
|
1532
1713
|
const errorInfo = classifyError(error);
|
|
@@ -1614,6 +1795,15 @@ export class Membrane {
|
|
|
1614
1795
|
*/
|
|
1615
1796
|
streamYielding(request, options = {}) {
|
|
1616
1797
|
const toolMode = this.resolveToolMode(request);
|
|
1798
|
+
// refusalRetries is implemented on the native path only. The XML path
|
|
1799
|
+
// accumulates into a streaming parser carrying prefill context and
|
|
1800
|
+
// resumption depths; rolling that back mid-turn is a separate problem,
|
|
1801
|
+
// and a partial implementation would corrupt the turn instead of
|
|
1802
|
+
// retrying it. Fail LOUD and OFF rather than silently mis-retrying.
|
|
1803
|
+
if (toolMode !== 'native' && (options.refusalRetries ?? 0) > 0) {
|
|
1804
|
+
(this.config.logger ?? console).warn('[membrane] refusalRetries is ignored in XML tool mode ' +
|
|
1805
|
+
'(native-only for now) — the turn will surface the refusal as before.');
|
|
1806
|
+
}
|
|
1617
1807
|
// Create the yielding stream with the appropriate inference runner
|
|
1618
1808
|
const runInference = toolMode === 'native'
|
|
1619
1809
|
? (stream) => this.runNativeToolsYielding(request, options, stream)
|
|
@@ -1634,6 +1824,27 @@ export class Membrane {
|
|
|
1634
1824
|
const maxToolDepth = maxToolDepthOpt === undefined || maxToolDepthOpt === -1
|
|
1635
1825
|
? Infinity
|
|
1636
1826
|
: maxToolDepthOpt;
|
|
1827
|
+
// Resumption spin guards (issue #39). This is the path the Ash spin ran
|
|
1828
|
+
// on: tool depth here is unlimited BY DESIGN (the caller budgets its own
|
|
1829
|
+
// tool work — that contract stands untouched), and the false-positive
|
|
1830
|
+
// resumption path counted against that same unlimited bound — 43 rounds
|
|
1831
|
+
// × ~172k input tokens of zero progress. Only AUTOMATIC resumptions are
|
|
1832
|
+
// guarded: a stall guard (consecutive no-progress resumptions →
|
|
1833
|
+
// 'no_progress') and a hard resumption cap ('round_limit'). Tool rounds
|
|
1834
|
+
// are never counted. See streamWithXmlTools for the rationale details.
|
|
1835
|
+
const MIN_ROUND_PROGRESS_CHARS = 16;
|
|
1836
|
+
const MAX_CONSECUTIVE_STALLED_RESUMPTIONS = 3;
|
|
1837
|
+
const RESUMPTION_WARN_ROUNDS = 5;
|
|
1838
|
+
const maxResumptionRounds = options.maxResumptionRounds === undefined
|
|
1839
|
+
? 24
|
|
1840
|
+
: options.maxResumptionRounds === -1
|
|
1841
|
+
? Infinity
|
|
1842
|
+
: options.maxResumptionRounds;
|
|
1843
|
+
let resumptionRounds = 0;
|
|
1844
|
+
let consecutiveStalledResumptions = 0;
|
|
1845
|
+
let enteredViaResumption = false;
|
|
1846
|
+
let prevRoundStopSequence;
|
|
1847
|
+
const warnLog = this.config.logger ?? console;
|
|
1637
1848
|
// Initialize parser from formatter for format-specific tracking
|
|
1638
1849
|
const formatter = this.formatter;
|
|
1639
1850
|
const parser = formatter.createStreamParser();
|
|
@@ -1673,6 +1884,23 @@ export class Membrane {
|
|
|
1673
1884
|
// blocks inherited from prefill context (e.g., unclosed <thinking> from other bots)
|
|
1674
1885
|
// from blocks the model itself opened during generation
|
|
1675
1886
|
const prefillDepths = parser.getDepths();
|
|
1887
|
+
/** Count an automatic resumption; emits the visibility warning at the
|
|
1888
|
+
* threshold and returns false when the cap says the turn should end. */
|
|
1889
|
+
const registerResumptionRound = () => {
|
|
1890
|
+
resumptionRounds++;
|
|
1891
|
+
if (resumptionRounds === RESUMPTION_WARN_ROUNDS) {
|
|
1892
|
+
warnLog.warn(`[membrane] automatic resumption at round ${resumptionRounds} ` +
|
|
1893
|
+
`(${totalUsage.inputTokens} input tokens so far this turn) — ` +
|
|
1894
|
+
`a spin shows up here before it shows up on the bill`);
|
|
1895
|
+
}
|
|
1896
|
+
if (resumptionRounds > maxResumptionRounds) {
|
|
1897
|
+
warnLog.warn(`[membrane] automatic resumption cap (${maxResumptionRounds}) reached — ` +
|
|
1898
|
+
`ending turn with stopReason 'round_limit'. ` +
|
|
1899
|
+
`${totalUsage.inputTokens} input tokens spent this turn.`);
|
|
1900
|
+
return false;
|
|
1901
|
+
}
|
|
1902
|
+
return true;
|
|
1903
|
+
};
|
|
1676
1904
|
try {
|
|
1677
1905
|
// Tool execution loop
|
|
1678
1906
|
while (toolDepth <= maxToolDepth) {
|
|
@@ -1786,6 +2014,30 @@ export class Membrane {
|
|
|
1786
2014
|
stream.emit({ type: 'block', event: emission.event });
|
|
1787
2015
|
}
|
|
1788
2016
|
}
|
|
2017
|
+
// Stall accounting (issue #39): only rounds ENTERED via automatic
|
|
2018
|
+
// resumption can stall; the turn ends only after several consecutive
|
|
2019
|
+
// stalls. Tool rounds are never counted. Mirrors streamWithXmlTools —
|
|
2020
|
+
// see the detailed rationale there.
|
|
2021
|
+
const streamedThisRound = parser.getAccumulated().length - checkFromIndex;
|
|
2022
|
+
if (enteredViaResumption &&
|
|
2023
|
+
lastStopReason === 'stop_sequence' &&
|
|
2024
|
+
streamedThisRound < MIN_ROUND_PROGRESS_CHARS &&
|
|
2025
|
+
lastStopSequence === prevRoundStopSequence) {
|
|
2026
|
+
consecutiveStalledResumptions++;
|
|
2027
|
+
if (consecutiveStalledResumptions >= MAX_CONSECUTIVE_STALLED_RESUMPTIONS) {
|
|
2028
|
+
warnLog.warn(`[membrane] ${consecutiveStalledResumptions} consecutive automatic resumptions ` +
|
|
2029
|
+
`made no progress (${streamedThisRound} chars this round, stop ` +
|
|
2030
|
+
`${JSON.stringify(lastStopSequence ?? null)} repeated) — ending turn with ` +
|
|
2031
|
+
`stopReason 'no_progress'. ${totalUsage.inputTokens} input tokens spent this turn.`);
|
|
2032
|
+
lastStopReason = 'no_progress';
|
|
2033
|
+
break;
|
|
2034
|
+
}
|
|
2035
|
+
}
|
|
2036
|
+
else {
|
|
2037
|
+
consecutiveStalledResumptions = 0;
|
|
2038
|
+
}
|
|
2039
|
+
prevRoundStopSequence = lastStopSequence;
|
|
2040
|
+
enteredViaResumption = false;
|
|
1789
2041
|
// Check for tool calls
|
|
1790
2042
|
if (streamResult.stopSequence === '</function_calls>') {
|
|
1791
2043
|
const closeTag = '</function_calls>';
|
|
@@ -1966,6 +2218,9 @@ export class Membrane {
|
|
|
1966
2218
|
prefillResult.assistantPrefill = parser.getAccumulated();
|
|
1967
2219
|
providerRequest = this.buildContinuationRequest(request, prefillResult, parser.getAccumulated());
|
|
1968
2220
|
}
|
|
2221
|
+
// Tool rounds are the caller's work — they count against
|
|
2222
|
+
// maxToolDepth only, never against the resumption guards
|
|
2223
|
+
// (issue #39 review: the uncapped tool-loop contract stands).
|
|
1969
2224
|
parser.resetForNewIteration();
|
|
1970
2225
|
toolDepth++;
|
|
1971
2226
|
continue;
|
|
@@ -1993,6 +2248,11 @@ export class Membrane {
|
|
|
1993
2248
|
if (toolDepth > maxToolDepth) {
|
|
1994
2249
|
break;
|
|
1995
2250
|
}
|
|
2251
|
+
if (!registerResumptionRound()) {
|
|
2252
|
+
lastStopReason = 'round_limit';
|
|
2253
|
+
break;
|
|
2254
|
+
}
|
|
2255
|
+
enteredViaResumption = true;
|
|
1996
2256
|
prefillResult.assistantPrefill = parser.getAccumulated();
|
|
1997
2257
|
providerRequest = this.buildContinuationRequest(request, prefillResult, parser.getAccumulated());
|
|
1998
2258
|
parser.resetForNewIteration();
|
|
@@ -2072,6 +2332,9 @@ export class Membrane {
|
|
|
2072
2332
|
// Stream from provider
|
|
2073
2333
|
let textAccumulated = '';
|
|
2074
2334
|
let blockIndex = 0;
|
|
2335
|
+
// Where this attempt starts inside the tool-loop-spanning buffer, so
|
|
2336
|
+
// a refusal retry can roll back exactly this attempt's contribution.
|
|
2337
|
+
const allTextBefore = allTextAccumulated.length;
|
|
2075
2338
|
// Track block-type from the provider's content_block_start signal so
|
|
2076
2339
|
// every token chunk is tagged with the membrane block it belongs to.
|
|
2077
2340
|
// Without this, thinking_delta chunks get mislabelled as 'text' and
|
|
@@ -2145,6 +2408,25 @@ export class Membrane {
|
|
|
2145
2408
|
idleTimeoutMs: options.idleTimeoutMs,
|
|
2146
2409
|
normalizedRequest: request,
|
|
2147
2410
|
onRequest: (req) => { rawRequest = req; },
|
|
2411
|
+
refusalRetries: options.refusalRetries,
|
|
2412
|
+
// Discard the refused attempt: roll the accumulators back to
|
|
2413
|
+
// where this attempt began and tell the consumer to drop what it
|
|
2414
|
+
// already received. `allTextAccumulated` spans the whole tool
|
|
2415
|
+
// loop, so it is truncated rather than cleared.
|
|
2416
|
+
onRetrying: (info) => {
|
|
2417
|
+
allTextAccumulated = allTextAccumulated.slice(0, allTextBefore);
|
|
2418
|
+
textAccumulated = '';
|
|
2419
|
+
blockIndex = 0;
|
|
2420
|
+
currentBlockType = 'text';
|
|
2421
|
+
seenBlockIndices.clear();
|
|
2422
|
+
stream.emit({
|
|
2423
|
+
type: 'retrying',
|
|
2424
|
+
attempt: info.attempt,
|
|
2425
|
+
maxAttempts: info.maxAttempts,
|
|
2426
|
+
reason: 'refusal',
|
|
2427
|
+
...(info.category ? { category: info.category } : {}),
|
|
2428
|
+
});
|
|
2429
|
+
},
|
|
2148
2430
|
});
|
|
2149
2431
|
rawResponse = streamResult.raw;
|
|
2150
2432
|
lastStopReason = this.mapStopReason(streamResult.stopReason);
|