@beeos-ai/bma-dsh-adapter 0.3.1 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/adapter.d.ts +21 -1
- package/lib/adapter.js +228 -27
- package/lib/context-store.d.ts +39 -0
- package/lib/context-store.js +95 -0
- package/lib/index.d.ts +5 -2
- package/lib/index.js +4 -2
- package/lib/kernel.d.ts +7 -0
- package/lib/kernel.js +11 -1
- package/lib/model.d.ts +33 -0
- package/lib/model.js +143 -18
- package/lib/request-budget.d.ts +51 -0
- package/lib/request-budget.js +246 -0
- package/lib/spi.d.ts +39 -3
- package/lib/stop.js +6 -1
- package/lib/testing/tmp.js +3 -1
- package/package.json +1 -1
package/lib/model.d.ts
CHANGED
|
@@ -1,10 +1,19 @@
|
|
|
1
1
|
import { LlmAdapter, type GenerateOptions, type ImageBlock, type LlmResolvedModelInfo, type StreamChunk } from "@deepseek-ai/dsh-llm";
|
|
2
|
+
import { type OpenAiMessage } from "./request-budget.ts";
|
|
2
3
|
import type { ModelPort } from "./spi.ts";
|
|
3
4
|
export declare const PROVIDER = "bma";
|
|
4
5
|
/** Error code of a model stream that sent no data for the idle limit (typed stop model_stream_idle). */
|
|
5
6
|
export declare const MODEL_STREAM_IDLE = "MODEL_STREAM_IDLE";
|
|
6
7
|
/** Error code of a model request whose connection failed before any response (refused, reset, DNS). */
|
|
7
8
|
export declare const MODEL_CONNECTION = "MODEL_CONNECTION";
|
|
9
|
+
/**
|
|
10
|
+
* Error code of a model request whose body is over the endpoint's size limit, refused by the
|
|
11
|
+
* endpoint (413, or 400 naming the body size) or still over MAX_REQUEST_BODY_BYTES after every
|
|
12
|
+
* reduction. Its own code (not CONTEXT_WINDOW_EXCEEDED): a summary does not shrink images.
|
|
13
|
+
*/
|
|
14
|
+
export declare const REQUEST_TOO_LARGE = "REQUEST_TOO_LARGE";
|
|
15
|
+
/** Endpoint refusals of one step for size that are answered with a smaller request (each 3/4 of the refused one). */
|
|
16
|
+
export declare const MAX_SIZE_REJECTIONS = 3;
|
|
8
17
|
/** One image as the request carries it. */
|
|
9
18
|
export interface RequestImage {
|
|
10
19
|
mediaType: string;
|
|
@@ -81,10 +90,34 @@ export declare function transientBackoffMs(n: number, [first, cap]: readonly [nu
|
|
|
81
90
|
* bytes count, SSE keep-alive comments included.
|
|
82
91
|
*/
|
|
83
92
|
export declare const DEFAULT_STREAM_IDLE_MS = 90000;
|
|
93
|
+
/**
|
|
94
|
+
* Prompt cache breakpoints. Claude caches a request prefix only up to an explicit cache_control
|
|
95
|
+
* breakpoint (OpenAI-compatible routes to it accept the marker on a content part): without one every
|
|
96
|
+
* request, screenshots and all, is billed as uncached input. Other models cache prefixes by themselves
|
|
97
|
+
* and get no marker. At most 4: the first system message (it ends the tools + system prefix) and the last
|
|
98
|
+
* content part of the 3 most recent user messages (the latest screenshots). Same rule as the Go brain
|
|
99
|
+
* (server/gobrain/infrastructure/openaimodel/cache.go), shared vectors
|
|
100
|
+
* test/brain-conformance/prompt-cache-breakpoints.vectors.json. Basis: Anthropic, computer use tool,
|
|
101
|
+
* "Manage screenshot history" and "Prompt caching" (at most 4 breakpoints).
|
|
102
|
+
*/
|
|
103
|
+
export declare function markCacheBreakpoints(model: string, messages: OpenAiMessage[]): void;
|
|
84
104
|
export declare class ChatCompletionsAdapter extends LlmAdapter {
|
|
85
105
|
private readonly port;
|
|
86
106
|
private readonly options;
|
|
107
|
+
private readonly ladders;
|
|
87
108
|
constructor(port: ModelPort, options: ModelAdapterOptions);
|
|
109
|
+
/**
|
|
110
|
+
* Called when nothing else (compaction) took a size failure of the session: allows level 4
|
|
111
|
+
* and answers whether the request is retried. False when the failure was final.
|
|
112
|
+
*/
|
|
113
|
+
lastResort(sessionId: string, code: string): boolean;
|
|
114
|
+
/** A new turn starts from level 1 (learned budgets stay). */
|
|
115
|
+
newTurn(sessionId: string): void;
|
|
116
|
+
forget(sessionId: string): void;
|
|
117
|
+
private ladder;
|
|
118
|
+
/** An endpoint refusal for size: lower the budget to 3/4 of the refused request and wait for a retry, or end the step. */
|
|
119
|
+
private refused;
|
|
120
|
+
private report;
|
|
88
121
|
/** Window and output cap come from the host's model catalog, never from constants here. */
|
|
89
122
|
resolveModel(provider: string, model: string): Promise<LlmResolvedModelInfo>;
|
|
90
123
|
stream(request: GenerateOptions): AsyncIterable<StreamChunk>;
|
package/lib/model.js
CHANGED
|
@@ -5,11 +5,20 @@
|
|
|
5
5
|
// a session is never sent (fail closed). No provider key lives in the brain.
|
|
6
6
|
import { LlmAdapter, LlmError, CONTEXT_WINDOW_EXCEEDED_CODE, QUOTA_EXCEEDED_CODE, attributionHeaders, isContextWindowExceededError, offloadedImageText, projectOffloadedImages, } from "@deepseek-ai/dsh-llm";
|
|
7
7
|
import { requestMaxTokens } from "./model-catalog.js";
|
|
8
|
+
import { MAX_REQUEST_BODY_BYTES, TOOL_TEXT_FLOOR, fitRequest, toolImageMessage } from "./request-budget.js";
|
|
8
9
|
export const PROVIDER = "bma";
|
|
9
10
|
/** Error code of a model stream that sent no data for the idle limit (typed stop model_stream_idle). */
|
|
10
11
|
export const MODEL_STREAM_IDLE = "MODEL_STREAM_IDLE";
|
|
11
12
|
/** Error code of a model request whose connection failed before any response (refused, reset, DNS). */
|
|
12
13
|
export const MODEL_CONNECTION = "MODEL_CONNECTION";
|
|
14
|
+
/**
|
|
15
|
+
* Error code of a model request whose body is over the endpoint's size limit, refused by the
|
|
16
|
+
* endpoint (413, or 400 naming the body size) or still over MAX_REQUEST_BODY_BYTES after every
|
|
17
|
+
* reduction. Its own code (not CONTEXT_WINDOW_EXCEEDED): a summary does not shrink images.
|
|
18
|
+
*/
|
|
19
|
+
export const REQUEST_TOO_LARGE = "REQUEST_TOO_LARGE";
|
|
20
|
+
/** Endpoint refusals of one step for size that are answered with a smaller request (each 3/4 of the refused one). */
|
|
21
|
+
export const MAX_SIZE_REJECTIONS = 3;
|
|
13
22
|
/**
|
|
14
23
|
* Tool result images (computer_observe screenshots) are kept in memory per
|
|
15
24
|
* session and sent with their results; pruned ones, and all of them after a
|
|
@@ -141,7 +150,7 @@ async function toOpenAi(input, options, sessionId, system, signal) {
|
|
|
141
150
|
let toolImages = [];
|
|
142
151
|
const flushToolImages = () => {
|
|
143
152
|
if (toolImages.length)
|
|
144
|
-
out.push(
|
|
153
|
+
out.push(toolImageMessage(toolImages));
|
|
145
154
|
toolImages = [];
|
|
146
155
|
};
|
|
147
156
|
for (const m of messages) {
|
|
@@ -182,12 +191,46 @@ async function toOpenAi(input, options, sessionId, system, signal) {
|
|
|
182
191
|
flushToolImages();
|
|
183
192
|
return out;
|
|
184
193
|
}
|
|
194
|
+
/**
|
|
195
|
+
* Prompt cache breakpoints. Claude caches a request prefix only up to an explicit cache_control
|
|
196
|
+
* breakpoint (OpenAI-compatible routes to it accept the marker on a content part): without one every
|
|
197
|
+
* request, screenshots and all, is billed as uncached input. Other models cache prefixes by themselves
|
|
198
|
+
* and get no marker. At most 4: the first system message (it ends the tools + system prefix) and the last
|
|
199
|
+
* content part of the 3 most recent user messages (the latest screenshots). Same rule as the Go brain
|
|
200
|
+
* (server/gobrain/infrastructure/openaimodel/cache.go), shared vectors
|
|
201
|
+
* test/brain-conformance/prompt-cache-breakpoints.vectors.json. Basis: Anthropic, computer use tool,
|
|
202
|
+
* "Manage screenshot history" and "Prompt caching" (at most 4 breakpoints).
|
|
203
|
+
*/
|
|
204
|
+
export function markCacheBreakpoints(model, messages) {
|
|
205
|
+
if (!model.toLowerCase().includes("claude"))
|
|
206
|
+
return;
|
|
207
|
+
const mark = (m) => {
|
|
208
|
+
const parts = typeof m.content === "string" ? [{ type: "text", text: m.content }] : m.content;
|
|
209
|
+
if (!parts?.length)
|
|
210
|
+
return;
|
|
211
|
+
parts[parts.length - 1] = { ...parts[parts.length - 1], cache_control: { type: "ephemeral" } };
|
|
212
|
+
m.content = parts;
|
|
213
|
+
};
|
|
214
|
+
const system = messages.find((m) => m.role === "system");
|
|
215
|
+
if (system)
|
|
216
|
+
mark(system);
|
|
217
|
+
let marked = 0;
|
|
218
|
+
for (let i = messages.length - 1; i >= 0 && marked < 3; i--) {
|
|
219
|
+
if (messages[i].role !== "user")
|
|
220
|
+
continue;
|
|
221
|
+
mark(messages[i]);
|
|
222
|
+
marked++;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
/** An endpoint's answer to a body over its size limit (ARouter: 400 "request body too large or unreadable"). */
|
|
226
|
+
const BODY_TOO_LARGE = /request (body|entity) too large|payload too large|request_too_large/i;
|
|
185
227
|
function failure(status, body, headers) {
|
|
186
228
|
// The canonical overflow code lets dsh compact and retry (compaction-basic reacts to it).
|
|
187
229
|
// The others name the failure class, which stop.ts reports as the contract's model failure stop code.
|
|
188
|
-
const code =
|
|
189
|
-
:
|
|
190
|
-
: status
|
|
230
|
+
const code = status === 413 || (status === 400 && BODY_TOO_LARGE.test(body)) ? REQUEST_TOO_LARGE
|
|
231
|
+
: isContextWindowExceededError(body) ? CONTEXT_WINDOW_EXCEEDED_CODE
|
|
232
|
+
: status === 401 || status === 403 ? "AUTH" : status === 402 ? QUOTA_EXCEEDED_CODE : status === 429 ? "RATE_LIMIT"
|
|
233
|
+
: status >= 500 ? "SERVER" : "PROVIDER_HTTP";
|
|
191
234
|
const err = new LlmError(`model endpoint HTTP ${status}: ${body.slice(0, 300)}`, code, { status });
|
|
192
235
|
const json = parseBody(body);
|
|
193
236
|
if (status !== 502 && status !== 503 && status !== 504 && json?.error?.retryable !== true && json?.retryable !== true)
|
|
@@ -198,11 +241,61 @@ function failure(status, body, headers) {
|
|
|
198
241
|
export class ChatCompletionsAdapter extends LlmAdapter {
|
|
199
242
|
port;
|
|
200
243
|
options;
|
|
244
|
+
ladders = new Map();
|
|
201
245
|
constructor(port, options) {
|
|
202
246
|
super();
|
|
203
247
|
this.port = port;
|
|
204
248
|
this.options = options;
|
|
205
249
|
}
|
|
250
|
+
/**
|
|
251
|
+
* Called when nothing else (compaction) took a size failure of the session: allows level 4
|
|
252
|
+
* and answers whether the request is retried. False when the failure was final.
|
|
253
|
+
*/
|
|
254
|
+
lastResort(sessionId, code) {
|
|
255
|
+
const ladder = this.ladders.get(sessionId);
|
|
256
|
+
if ((code !== CONTEXT_WINDOW_EXCEEDED_CODE && code !== REQUEST_TOO_LARGE) || !ladder?.pending)
|
|
257
|
+
return false;
|
|
258
|
+
ladder.pending = false;
|
|
259
|
+
ladder.lastResort = true;
|
|
260
|
+
return true;
|
|
261
|
+
}
|
|
262
|
+
/** A new turn starts from level 1 (learned budgets stay). */
|
|
263
|
+
newTurn(sessionId) {
|
|
264
|
+
const ladder = this.ladders.get(sessionId);
|
|
265
|
+
if (ladder)
|
|
266
|
+
Object.assign(ladder, { rejections: 0, lastResort: false, pending: false });
|
|
267
|
+
}
|
|
268
|
+
forget(sessionId) {
|
|
269
|
+
this.ladders.delete(sessionId);
|
|
270
|
+
}
|
|
271
|
+
ladder(sessionId) {
|
|
272
|
+
let ladder = this.ladders.get(sessionId);
|
|
273
|
+
if (!ladder)
|
|
274
|
+
this.ladders.set(sessionId, (ladder = { rejections: 0, lastResort: false, pending: false }));
|
|
275
|
+
return ladder;
|
|
276
|
+
}
|
|
277
|
+
/** An endpoint refusal for size: lower the budget to 3/4 of the refused request and wait for a retry, or end the step. */
|
|
278
|
+
refused(sessionId, ladder, err, sent, budget) {
|
|
279
|
+
const status = err.failure.status;
|
|
280
|
+
this.report(sessionId, { step: "rejected", levels: [], bytes: sent.bytes, tokens: sent.tokens, byteBudget: budget.bytes, tokenBudget: budget.tokens, ...(status !== undefined ? { status } : {}) });
|
|
281
|
+
if (ladder.rejections >= MAX_SIZE_REJECTIONS) {
|
|
282
|
+
ladder.pending = false;
|
|
283
|
+
this.report(sessionId, { step: "exhausted", levels: [], bytes: sent.bytes, tokens: sent.tokens, byteBudget: budget.bytes, tokenBudget: budget.tokens });
|
|
284
|
+
return err.code === REQUEST_TOO_LARGE
|
|
285
|
+
? new LlmError(`request_too_large: model request body too large for the endpoint, refused ${ladder.rejections + 1} times (HTTP ${status}, last ${sent.bytes} bytes)`, REQUEST_TOO_LARGE, { cause: err })
|
|
286
|
+
: err;
|
|
287
|
+
}
|
|
288
|
+
ladder.rejections++;
|
|
289
|
+
if (err.code === REQUEST_TOO_LARGE)
|
|
290
|
+
ladder.bytes = Math.floor(Math.min(ladder.bytes ?? Infinity, sent.bytes) * 3 / 4);
|
|
291
|
+
else
|
|
292
|
+
ladder.tokens = Math.floor(Math.min(ladder.tokens ?? Infinity, sent.tokens) * 3 / 4);
|
|
293
|
+
ladder.pending = true;
|
|
294
|
+
return err;
|
|
295
|
+
}
|
|
296
|
+
report(sessionId, reduction) {
|
|
297
|
+
this.port.requestReduced?.(sessionId, reduction);
|
|
298
|
+
}
|
|
206
299
|
/** Window and output cap come from the host's model catalog, never from constants here. */
|
|
207
300
|
async resolveModel(provider, model) {
|
|
208
301
|
const limits = this.port.limits(model);
|
|
@@ -226,13 +319,16 @@ export class ChatCompletionsAdapter extends LlmAdapter {
|
|
|
226
319
|
const maxTokens = Math.min(request.maxTokens ?? requestMaxTokens(limits), limits.maxOutputTokens);
|
|
227
320
|
const [minPause, maxPause] = this.options.idleRetryBackoffMs ?? DEFAULT_IDLE_RETRY_BACKOFF_MS;
|
|
228
321
|
const transientBackoff = this.options.transientRetryBackoffMs ?? DEFAULT_TRANSIENT_RETRY_BACKOFF_MS;
|
|
322
|
+
// A summary request (dsh compaction) is trimmed (level 2) but never refused locally: it is how level 3 runs.
|
|
323
|
+
const ladder = request.purpose === "compaction" ? undefined : this.ladder(sessionId);
|
|
229
324
|
let idleRetried = false;
|
|
230
325
|
let transientRetries = 0;
|
|
231
326
|
for (;;) {
|
|
232
327
|
const idle = new IdleGuard(this.options.streamIdleMs, request.signal);
|
|
233
328
|
let content = false;
|
|
329
|
+
const attempt = {};
|
|
234
330
|
try {
|
|
235
|
-
for await (const chunk of this.run(request, sessionId, route, maxTokens, idle, started)) {
|
|
331
|
+
for await (const chunk of this.run(request, sessionId, route, maxTokens, idle, started, limits.contextWindow, ladder, attempt)) {
|
|
236
332
|
content = true; // the first chunk is a block-start, only ever produced by a content delta
|
|
237
333
|
yield chunk;
|
|
238
334
|
}
|
|
@@ -240,9 +336,15 @@ export class ChatCompletionsAdapter extends LlmAdapter {
|
|
|
240
336
|
this.port.idleRetry?.(sessionId, "recovered");
|
|
241
337
|
if (transientRetries > 0)
|
|
242
338
|
this.port.transientRetry?.(sessionId, "recovered");
|
|
339
|
+
if (ladder)
|
|
340
|
+
Object.assign(ladder, { rejections: 0, lastResort: false, pending: false });
|
|
243
341
|
return;
|
|
244
342
|
}
|
|
245
343
|
catch (err) {
|
|
344
|
+
if (ladder && !content && !request.signal?.aborted && err instanceof LlmError && attempt.sent && attempt.budget
|
|
345
|
+
&& (err.code === REQUEST_TOO_LARGE || err.code === CONTEXT_WINDOW_EXCEEDED_CODE)) {
|
|
346
|
+
throw this.refused(sessionId, ladder, err, attempt.sent, attempt.budget);
|
|
347
|
+
}
|
|
246
348
|
if (idle.fired) {
|
|
247
349
|
const stop = new LlmError(`model stream idle: no data for ${this.options.streamIdleMs} ms`, MODEL_STREAM_IDLE, { cause: err });
|
|
248
350
|
if (content || request.signal?.aborted)
|
|
@@ -277,23 +379,44 @@ export class ChatCompletionsAdapter extends LlmAdapter {
|
|
|
277
379
|
}
|
|
278
380
|
}
|
|
279
381
|
}
|
|
280
|
-
async *run(request, sessionId, route, maxTokens, idle, started) {
|
|
281
|
-
const
|
|
382
|
+
async *run(request, sessionId, route, maxTokens, idle, started, contextWindow, ladder, attempt) {
|
|
383
|
+
const rest = {
|
|
384
|
+
model: route.model,
|
|
385
|
+
stream: true,
|
|
386
|
+
stream_options: { include_usage: true },
|
|
387
|
+
...(!route.toolsDisabled && request.tools?.length ? { tools: request.tools.map((t) => ({ type: "function", function: { name: t.name, description: t.description, parameters: t.parameters } })) } : {}),
|
|
388
|
+
max_tokens: maxTokens,
|
|
389
|
+
...(route.reasoningEffort ? { reasoning_effort: route.reasoningEffort } : {}),
|
|
390
|
+
...(request.temperature !== undefined ? { temperature: request.temperature } : {}),
|
|
391
|
+
...(request.stop?.length ? { stop: request.stop } : {}),
|
|
392
|
+
};
|
|
393
|
+
// The window holds the reply too: the input budget is what max_tokens leaves.
|
|
394
|
+
const budget = { tokens: Math.min(ladder?.tokens ?? Infinity, contextWindow - maxTokens), bytes: Math.min(ladder?.bytes ?? Infinity, MAX_REQUEST_BODY_BYTES) };
|
|
395
|
+
const all = await toOpenAi(request.messages, this.options, sessionId, request.system, idle.signal);
|
|
396
|
+
const fit = fitRequest(all, rest, budget, Math.max(TOOL_TEXT_FLOOR, contextWindow), ladder?.lastResort ?? false);
|
|
397
|
+
const sizes = { bytes: fit.bytes, tokens: fit.tokens, byteBudget: budget.bytes, tokenBudget: budget.tokens };
|
|
398
|
+
if (ladder && fit.over) {
|
|
399
|
+
// Nothing is sent. Before the last resort the overflow code hands the step to dsh compaction
|
|
400
|
+
// (a body over the byte budget skips it: a summary does not shrink images); after it, the step fails.
|
|
401
|
+
const final = ladder.lastResort;
|
|
402
|
+
ladder.pending = !final;
|
|
403
|
+
this.report(sessionId, { step: final ? "exhausted" : "compact", levels: final ? fit.levels : [...fit.levels, 3], ...sizes });
|
|
404
|
+
const err = fit.over === "bytes"
|
|
405
|
+
? new LlmError(`request_too_large: model request body too large for the endpoint (${fit.bytes} bytes, budget ${budget.bytes})${final ? " after every reduction" : ""}`, REQUEST_TOO_LARGE)
|
|
406
|
+
: new LlmError(`model request estimated at ${fit.tokens} tokens, over the model window budget of ${budget.tokens} tokens${final ? " after every reduction" : ""}`, CONTEXT_WINDOW_EXCEEDED_CODE);
|
|
407
|
+
throw err;
|
|
408
|
+
}
|
|
409
|
+
if (ladder && fit.levels.length)
|
|
410
|
+
this.report(sessionId, { step: "reduced", levels: fit.levels, ...sizes });
|
|
411
|
+
const messages = fit.messages;
|
|
412
|
+
markCacheBreakpoints(route.model, messages);
|
|
413
|
+
attempt.sent = fit;
|
|
414
|
+
attempt.budget = budget;
|
|
282
415
|
const response = await connect(`${route.baseUrl.replace(/\/$/, "")}/chat/completions`, {
|
|
283
416
|
method: "POST",
|
|
284
417
|
signal: idle.signal,
|
|
285
418
|
headers: { ...attributionHeaders(), "content-type": "application/json", authorization: `Bearer ${route.token}` },
|
|
286
|
-
body: JSON.stringify({
|
|
287
|
-
model: route.model,
|
|
288
|
-
stream: true,
|
|
289
|
-
stream_options: { include_usage: true },
|
|
290
|
-
messages,
|
|
291
|
-
...(request.tools?.length ? { tools: request.tools.map((t) => ({ type: "function", function: { name: t.name, description: t.description, parameters: t.parameters } })) } : {}),
|
|
292
|
-
max_tokens: maxTokens,
|
|
293
|
-
...(route.reasoningEffort ? { reasoning_effort: route.reasoningEffort } : {}),
|
|
294
|
-
...(request.temperature !== undefined ? { temperature: request.temperature } : {}),
|
|
295
|
-
...(request.stop?.length ? { stop: request.stop } : {}),
|
|
296
|
-
}),
|
|
419
|
+
body: JSON.stringify({ ...rest, messages }),
|
|
297
420
|
});
|
|
298
421
|
if (!response.ok || !response.body)
|
|
299
422
|
throw failure(response.status, await response.text().catch(() => ""), response.headers);
|
|
@@ -369,6 +492,8 @@ export class ChatCompletionsAdapter extends LlmAdapter {
|
|
|
369
492
|
texts.set(textIdx, (texts.get(textIdx) ?? "") + delta.content);
|
|
370
493
|
yield { type: "text-delta", index: textIdx, text: delta.content };
|
|
371
494
|
}
|
|
495
|
+
if (route.toolsDisabled && delta.tool_calls?.length)
|
|
496
|
+
throw new LlmError("maintenance model attempted a tool call", "TOOLS_DISABLED");
|
|
372
497
|
for (const tc of delta.tool_calls ?? []) {
|
|
373
498
|
let call = calls.get(tc.index);
|
|
374
499
|
if (!call) {
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
export type OpenAiMessage = Record<string, unknown>;
|
|
2
|
+
type Part = Record<string, unknown>;
|
|
3
|
+
/**
|
|
4
|
+
* Body budget. The model gateway accepts 64 MiB (HTTP 413 request_too_large above it), but upstream
|
|
5
|
+
* providers are tighter: Bedrock Claude and Gemini refuse a request over 20 MB. 16 MiB stays under
|
|
6
|
+
* those with room for the gateway's format conversion and estimate drift. The one source is
|
|
7
|
+
* contracts/bma/model-request-budget.vectors.json (the Go brain and legacy are locked to it too).
|
|
8
|
+
*/
|
|
9
|
+
export declare const MAX_REQUEST_BODY_BYTES: number;
|
|
10
|
+
/**
|
|
11
|
+
* Images sent with a request, newest first, besides those of the newest user input (always sent).
|
|
12
|
+
* Older ones are a text placeholder. Basis: Anthropic computer use reference implementation
|
|
13
|
+
* (only_n_most_recent_images = 3). Anthropic also removes them in batches so the cached prefix
|
|
14
|
+
* moves less often; a batch would carry up to 3 + interval images, which is what overflowed the
|
|
15
|
+
* gateway, so the cap is exact: each new image changes the request from the 4th newest image on,
|
|
16
|
+
* the prefix before it stays cached.
|
|
17
|
+
*/
|
|
18
|
+
export declare const RECENT_IMAGES = 3;
|
|
19
|
+
/** Tool results never cleared by level 2 (Anthropic context editing clear_tool_uses: keep 3). */
|
|
20
|
+
export declare const RECENT_TOOL_RESULTS = 3;
|
|
21
|
+
/** Token estimate of one image: about 1.15 megapixels at 750 px per token (Anthropic vision guide). */
|
|
22
|
+
export declare const IMAGE_TOKENS = 1600;
|
|
23
|
+
/** Floor of the per-result tool text cap; the cap itself is the window in characters (about a quarter of it in tokens). */
|
|
24
|
+
export declare const TOOL_TEXT_FLOOR = 16384;
|
|
25
|
+
export declare const OLDER_IMAGE = "[image omitted: older image, not resent]";
|
|
26
|
+
/** Ladder levels that rewrite a request (3 is dsh compaction). */
|
|
27
|
+
export type FitLevel = 2 | 4;
|
|
28
|
+
export interface Budget {
|
|
29
|
+
readonly tokens: number;
|
|
30
|
+
readonly bytes: number;
|
|
31
|
+
}
|
|
32
|
+
export interface Fit {
|
|
33
|
+
readonly messages: OpenAiMessage[];
|
|
34
|
+
/** Body bytes and estimated input tokens of the request with these messages. */
|
|
35
|
+
readonly bytes: number;
|
|
36
|
+
readonly tokens: number;
|
|
37
|
+
/** The budget-driven levels applied, in order. */
|
|
38
|
+
readonly levels: readonly FitLevel[];
|
|
39
|
+
/** Which budget is still exceeded after every allowed level, if any. */
|
|
40
|
+
readonly over?: "tokens" | "bytes";
|
|
41
|
+
}
|
|
42
|
+
export declare function toolImageMessage(content: Part[]): OpenAiMessage;
|
|
43
|
+
/** "<head>\n[... N characters truncated ...]\n<tail>" keeping `keep` characters, or the text when it is short enough. */
|
|
44
|
+
export declare function headTail(text: string, keep: number): string;
|
|
45
|
+
/**
|
|
46
|
+
* Fits `messages` (system first, as toOpenAi made them) into `budget`. `rest` is the request body
|
|
47
|
+
* without its messages (tools and parameters count too). `lastResort` allows level 4: the
|
|
48
|
+
* caller sets it once dsh compaction (level 3) could not make the request fit.
|
|
49
|
+
*/
|
|
50
|
+
export declare function fitRequest(messages: readonly OpenAiMessage[], rest: Record<string, unknown>, budget: Budget, toolTextCap: number, lastResort: boolean): Fit;
|
|
51
|
+
export {};
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
// Fits one Chat Completions request into the model's window (tokens) and the model endpoint's body
|
|
2
|
+
// limit (bytes). Only the request is rewritten; dsh's durable history stays as it is. The levels of
|
|
3
|
+
// the ladder, each used only when the one before is not enough:
|
|
4
|
+
// 1 estimate tokens and body bytes before sending (fitRequest measures every candidate)
|
|
5
|
+
// 2 trim bulk: older images out, old tool results cleared, the largest tool texts cut head + tail
|
|
6
|
+
// 3 dsh summary compaction (not here: model.ts hands the step to dsh, see ChatCompletionsAdapter)
|
|
7
|
+
// 4 sliding window: the oldest whole messages go, tool calls together with their results
|
|
8
|
+
// The newest user input is never cut and keeps its images: a request still over budget is not sent
|
|
9
|
+
// and fails with REQUEST_TOO_LARGE or the context overflow code, naming the budget (model.ts).
|
|
10
|
+
// Every request, over budget or not, also keeps only the RECENT_IMAGES newest images and caps each
|
|
11
|
+
// tool result at toolTextCap characters.
|
|
12
|
+
/**
|
|
13
|
+
* Body budget. The model gateway accepts 64 MiB (HTTP 413 request_too_large above it), but upstream
|
|
14
|
+
* providers are tighter: Bedrock Claude and Gemini refuse a request over 20 MB. 16 MiB stays under
|
|
15
|
+
* those with room for the gateway's format conversion and estimate drift. The one source is
|
|
16
|
+
* contracts/bma/model-request-budget.vectors.json (the Go brain and legacy are locked to it too).
|
|
17
|
+
*/
|
|
18
|
+
export const MAX_REQUEST_BODY_BYTES = 16 << 20;
|
|
19
|
+
/**
|
|
20
|
+
* Images sent with a request, newest first, besides those of the newest user input (always sent).
|
|
21
|
+
* Older ones are a text placeholder. Basis: Anthropic computer use reference implementation
|
|
22
|
+
* (only_n_most_recent_images = 3). Anthropic also removes them in batches so the cached prefix
|
|
23
|
+
* moves less often; a batch would carry up to 3 + interval images, which is what overflowed the
|
|
24
|
+
* gateway, so the cap is exact: each new image changes the request from the 4th newest image on,
|
|
25
|
+
* the prefix before it stays cached.
|
|
26
|
+
*/
|
|
27
|
+
export const RECENT_IMAGES = 3;
|
|
28
|
+
/** Tool results never cleared by level 2 (Anthropic context editing clear_tool_uses: keep 3). */
|
|
29
|
+
export const RECENT_TOOL_RESULTS = 3;
|
|
30
|
+
/** Token estimate of one image: about 1.15 megapixels at 750 px per token (Anthropic vision guide). */
|
|
31
|
+
export const IMAGE_TOKENS = 1_600;
|
|
32
|
+
/** Floor of the per-result tool text cap; the cap itself is the window in characters (about a quarter of it in tokens). */
|
|
33
|
+
export const TOOL_TEXT_FLOOR = 16_384;
|
|
34
|
+
/** Level 2 never cuts a tool text below this many characters. */
|
|
35
|
+
const MIN_TOOL_TEXT = 1_024;
|
|
36
|
+
export const OLDER_IMAGE = "[image omitted: older image, not resent]";
|
|
37
|
+
const DROPPED_IMAGE = "[image omitted: not resent, to fit the model request]";
|
|
38
|
+
const CLEARED_TOOL_RESULT = "[tool result cleared to fit the model request; run the tool again if it is needed]";
|
|
39
|
+
const omitted = (n) => `[${n} earlier message(s) omitted to fit the model request]`;
|
|
40
|
+
/** Messages toOpenAi made to carry the images of tool results (they belong to the assistant step before them). */
|
|
41
|
+
const toolImageMessages = new WeakSet();
|
|
42
|
+
export function toolImageMessage(content) {
|
|
43
|
+
const m = { role: "user", content };
|
|
44
|
+
toolImageMessages.add(m);
|
|
45
|
+
return m;
|
|
46
|
+
}
|
|
47
|
+
/** "<head>\n[... N characters truncated ...]\n<tail>" keeping `keep` characters, or the text when it is short enough. */
|
|
48
|
+
export function headTail(text, keep) {
|
|
49
|
+
if (text.length <= keep)
|
|
50
|
+
return text;
|
|
51
|
+
const head = Math.ceil(keep / 2);
|
|
52
|
+
return `${text.slice(0, head)}\n[... ${text.length - keep} characters truncated ...]\n${text.slice(text.length - (keep - head))}`;
|
|
53
|
+
}
|
|
54
|
+
function measure(m) {
|
|
55
|
+
let text = 0;
|
|
56
|
+
let images = 0;
|
|
57
|
+
if (typeof m.content === "string")
|
|
58
|
+
text += Buffer.byteLength(m.content);
|
|
59
|
+
else if (Array.isArray(m.content)) {
|
|
60
|
+
for (const p of m.content) {
|
|
61
|
+
if (p.type === "image_url")
|
|
62
|
+
images++;
|
|
63
|
+
else if (typeof p.text === "string")
|
|
64
|
+
text += Buffer.byteLength(p.text);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
for (const c of m.tool_calls ?? [])
|
|
68
|
+
text += Buffer.byteLength(c.function.name) + Buffer.byteLength(c.function.arguments);
|
|
69
|
+
// UTF-8 bytes / 4: ASCII is about 4 characters a token; CJK (3 bytes a character) comes out near 1 token a character.
|
|
70
|
+
return { bytes: Buffer.byteLength(JSON.stringify(m)), tokens: Math.ceil(text / 4) + 4 + images * IMAGE_TOKENS };
|
|
71
|
+
}
|
|
72
|
+
const isImage = (p) => p.type === "image_url";
|
|
73
|
+
const isSummary = (m) => m.role === "user" && textOf(m).includes("<compacted-summary>");
|
|
74
|
+
function textOf(m) {
|
|
75
|
+
if (typeof m.content === "string")
|
|
76
|
+
return m.content;
|
|
77
|
+
return Array.isArray(m.content) ? m.content.map((p) => (typeof p.text === "string" ? p.text : "")).join("") : "";
|
|
78
|
+
}
|
|
79
|
+
/** A copy with new content; a tool image message stays one. */
|
|
80
|
+
function withContent(m, content) {
|
|
81
|
+
const copy = { ...m, content };
|
|
82
|
+
if (toolImageMessages.has(m))
|
|
83
|
+
toolImageMessages.add(copy);
|
|
84
|
+
return copy;
|
|
85
|
+
}
|
|
86
|
+
/** Index of the newest user input: the last user message that is neither a tool image message nor a summary. */
|
|
87
|
+
function latestInput(ms) {
|
|
88
|
+
for (let i = ms.length - 1; i >= 0; i--)
|
|
89
|
+
if (ms[i].role === "user" && !toolImageMessages.has(ms[i]) && !isSummary(ms[i]))
|
|
90
|
+
return i;
|
|
91
|
+
return -1;
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Fits `messages` (system first, as toOpenAi made them) into `budget`. `rest` is the request body
|
|
95
|
+
* without its messages (tools and parameters count too). `lastResort` allows level 4: the
|
|
96
|
+
* caller sets it once dsh compaction (level 3) could not make the request fit.
|
|
97
|
+
*/
|
|
98
|
+
export function fitRequest(messages, rest, budget, toolTextCap, lastResort) {
|
|
99
|
+
const sizes = new WeakMap();
|
|
100
|
+
const size = (m) => {
|
|
101
|
+
let s = sizes.get(m);
|
|
102
|
+
if (!s)
|
|
103
|
+
sizes.set(m, (s = measure(m)));
|
|
104
|
+
return s;
|
|
105
|
+
};
|
|
106
|
+
const base = { bytes: Buffer.byteLength(JSON.stringify({ ...rest, messages: [] })), tokens: Math.ceil(Buffer.byteLength(JSON.stringify(rest.tools ?? [])) / 4) };
|
|
107
|
+
const total = (ms) => {
|
|
108
|
+
const t = { bytes: base.bytes + Math.max(0, ms.length - 1), tokens: base.tokens };
|
|
109
|
+
for (const m of ms) {
|
|
110
|
+
const s = size(m);
|
|
111
|
+
t.bytes += s.bytes;
|
|
112
|
+
t.tokens += s.tokens;
|
|
113
|
+
}
|
|
114
|
+
return t;
|
|
115
|
+
};
|
|
116
|
+
const overOf = (ms) => {
|
|
117
|
+
const t = total(ms);
|
|
118
|
+
return t.bytes > budget.bytes ? "bytes" : t.tokens > budget.tokens ? "tokens" : undefined;
|
|
119
|
+
};
|
|
120
|
+
let ms = keepRecentImages(capToolText([...messages], toolTextCap));
|
|
121
|
+
const levels = [];
|
|
122
|
+
const level = (n) => { if (!levels.includes(n))
|
|
123
|
+
levels.push(n); };
|
|
124
|
+
// Level 2: images first (they are most of the bytes), then old tool results, then the largest tool texts.
|
|
125
|
+
for (const keep of [1, 0]) {
|
|
126
|
+
if (overOf(ms) !== "bytes")
|
|
127
|
+
break;
|
|
128
|
+
const next = dropImages(ms, keep, DROPPED_IMAGE);
|
|
129
|
+
if (next !== ms) {
|
|
130
|
+
ms = next;
|
|
131
|
+
level(2);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
if (overOf(ms)) {
|
|
135
|
+
const tools = ms.flatMap((m, i) => (m.role === "tool" ? [i] : [])).slice(0, -RECENT_TOOL_RESULTS || undefined);
|
|
136
|
+
for (const i of tools) {
|
|
137
|
+
if (!overOf(ms))
|
|
138
|
+
break;
|
|
139
|
+
if (ms[i].content === CLEARED_TOOL_RESULT)
|
|
140
|
+
continue;
|
|
141
|
+
ms[i] = withContent(ms[i], CLEARED_TOOL_RESULT);
|
|
142
|
+
level(2);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
if (overOf(ms)) {
|
|
146
|
+
const original = new Map();
|
|
147
|
+
for (;;) {
|
|
148
|
+
if (!overOf(ms))
|
|
149
|
+
break;
|
|
150
|
+
let largest = -1;
|
|
151
|
+
for (let i = 0; i < ms.length; i++)
|
|
152
|
+
if (ms[i].role === "tool" && typeof ms[i].content === "string" && (largest < 0 || ms[i].content.length > ms[largest].content.length))
|
|
153
|
+
largest = i;
|
|
154
|
+
if (largest < 0)
|
|
155
|
+
break;
|
|
156
|
+
const text = original.get(largest) ?? ms[largest].content;
|
|
157
|
+
const keep = Math.floor(ms[largest].content.length / 2);
|
|
158
|
+
if (keep < MIN_TOOL_TEXT)
|
|
159
|
+
break;
|
|
160
|
+
original.set(largest, text);
|
|
161
|
+
ms[largest] = withContent(ms[largest], headTail(text, keep));
|
|
162
|
+
level(2);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
if (lastResort && overOf(ms)) {
|
|
166
|
+
const windowed = slidingWindow(ms, (candidate) => !overOf(candidate));
|
|
167
|
+
if (windowed !== ms) {
|
|
168
|
+
ms = windowed;
|
|
169
|
+
level(4);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
const t = total(ms);
|
|
173
|
+
return { messages: ms, bytes: t.bytes, tokens: t.tokens, levels, ...(overOf(ms) ? { over: overOf(ms) } : {}) };
|
|
174
|
+
}
|
|
175
|
+
/** Every tool result at most `cap` characters, head + tail. */
|
|
176
|
+
function capToolText(ms, cap) {
|
|
177
|
+
return ms.map((m) => (m.role === "tool" && typeof m.content === "string" && m.content.length > cap ? withContent(m, headTail(m.content, cap)) : m));
|
|
178
|
+
}
|
|
179
|
+
/** RECENT_IMAGES newest images kept besides those of the newest user input; older ones are a placeholder. */
|
|
180
|
+
function keepRecentImages(ms) {
|
|
181
|
+
return dropImages(ms, RECENT_IMAGES, OLDER_IMAGE);
|
|
182
|
+
}
|
|
183
|
+
/** Keeps the `keep` newest images outside the newest user input; the others become `placeholder`. Same array when nothing changed. */
|
|
184
|
+
function dropImages(ms, keep, placeholder) {
|
|
185
|
+
const input = latestInput(ms);
|
|
186
|
+
let seen = 0;
|
|
187
|
+
let out = ms;
|
|
188
|
+
for (let i = ms.length - 1; i >= 0; i--) {
|
|
189
|
+
if (i === input || !Array.isArray(ms[i].content))
|
|
190
|
+
continue;
|
|
191
|
+
const parts = ms[i].content;
|
|
192
|
+
let changed;
|
|
193
|
+
for (let j = parts.length - 1; j >= 0; j--) {
|
|
194
|
+
if (!isImage(parts[j]) || ++seen <= keep)
|
|
195
|
+
continue;
|
|
196
|
+
changed ??= [...parts];
|
|
197
|
+
changed[j] = { type: "text", text: placeholder };
|
|
198
|
+
}
|
|
199
|
+
if (changed) {
|
|
200
|
+
if (out === ms)
|
|
201
|
+
out = [...ms];
|
|
202
|
+
out[i] = withContent(ms[i], changed);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
return out;
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Level 4. Drops the fewest oldest units so that `fits`, each unit whole: an assistant message with
|
|
209
|
+
* its tool results and their image message, or one user message. System messages, summaries, the
|
|
210
|
+
* newest user input and the last unit stay. Each run of dropped units becomes one user note, so no
|
|
211
|
+
* tool result is left without its call (or a call without its result) and the model knows.
|
|
212
|
+
*/
|
|
213
|
+
function slidingWindow(ms, fits) {
|
|
214
|
+
const units = [];
|
|
215
|
+
for (let i = 0; i < ms.length; i++) {
|
|
216
|
+
const m = ms[i];
|
|
217
|
+
const joins = (m.role === "tool" || toolImageMessages.has(m)) && units.length > 0 && ms[units.at(-1)[0]].role === "assistant";
|
|
218
|
+
if (joins)
|
|
219
|
+
units.at(-1).push(i);
|
|
220
|
+
else
|
|
221
|
+
units.push([i]);
|
|
222
|
+
}
|
|
223
|
+
const input = latestInput(ms);
|
|
224
|
+
const droppable = units.slice(0, -1).filter((u) => !u.includes(input) && ms[u[0]].role !== "system" && !isSummary(ms[u[0]]));
|
|
225
|
+
let candidate = ms;
|
|
226
|
+
for (let k = 1; k <= droppable.length; k++) {
|
|
227
|
+
const dropped = new Set(droppable.slice(0, k).flat());
|
|
228
|
+
candidate = [];
|
|
229
|
+
let run = 0;
|
|
230
|
+
for (let i = 0; i <= ms.length; i++) {
|
|
231
|
+
if (i < ms.length && dropped.has(i)) {
|
|
232
|
+
run++;
|
|
233
|
+
continue;
|
|
234
|
+
}
|
|
235
|
+
if (run) {
|
|
236
|
+
candidate.push({ role: "user", content: omitted(run) });
|
|
237
|
+
run = 0;
|
|
238
|
+
}
|
|
239
|
+
if (i < ms.length)
|
|
240
|
+
candidate.push(ms[i]);
|
|
241
|
+
}
|
|
242
|
+
if (fits(candidate))
|
|
243
|
+
return candidate;
|
|
244
|
+
}
|
|
245
|
+
return candidate;
|
|
246
|
+
}
|