@beeos-ai/bma-dsh-adapter 0.3.1 → 0.3.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/model.d.ts CHANGED
@@ -1,10 +1,19 @@
1
1
  import { LlmAdapter, type GenerateOptions, type ImageBlock, type LlmResolvedModelInfo, type StreamChunk } from "@deepseek-ai/dsh-llm";
2
+ import { type OpenAiMessage } from "./request-budget.ts";
2
3
  import type { ModelPort } from "./spi.ts";
3
4
  export declare const PROVIDER = "bma";
4
5
  /** Error code of a model stream that sent no data for the idle limit (typed stop model_stream_idle). */
5
6
  export declare const MODEL_STREAM_IDLE = "MODEL_STREAM_IDLE";
6
7
  /** Error code of a model request whose connection failed before any response (refused, reset, DNS). */
7
8
  export declare const MODEL_CONNECTION = "MODEL_CONNECTION";
9
+ /**
10
+ * Error code of a model request whose body is over the endpoint's size limit, refused by the
11
+ * endpoint (413, or 400 naming the body size) or still over MAX_REQUEST_BODY_BYTES after every
12
+ * reduction. Its own code (not CONTEXT_WINDOW_EXCEEDED): a summary does not shrink images.
13
+ */
14
+ export declare const REQUEST_TOO_LARGE = "REQUEST_TOO_LARGE";
15
+ /** Endpoint refusals of one step for size that are answered with a smaller request (each 3/4 of the refused one). */
16
+ export declare const MAX_SIZE_REJECTIONS = 3;
8
17
  /** One image as the request carries it. */
9
18
  export interface RequestImage {
10
19
  mediaType: string;
@@ -81,10 +90,34 @@ export declare function transientBackoffMs(n: number, [first, cap]: readonly [nu
81
90
  * bytes count, SSE keep-alive comments included.
82
91
  */
83
92
  export declare const DEFAULT_STREAM_IDLE_MS = 90000;
93
+ /**
94
+ * Prompt cache breakpoints. Claude caches a request prefix only up to an explicit cache_control
95
+ * breakpoint (OpenAI-compatible routes to it accept the marker on a content part): without one every
96
+ * request, screenshots and all, is billed as uncached input. Other models cache prefixes by themselves
97
+ * and get no marker. At most 4: the first system message (it ends the tools + system prefix) and the last
98
+ * content part of the 3 most recent user messages (the latest screenshots). Same rule as the Go brain
99
+ * (server/gobrain/infrastructure/openaimodel/cache.go), shared vectors
100
+ * test/brain-conformance/prompt-cache-breakpoints.vectors.json. Basis: Anthropic, computer use tool,
101
+ * "Manage screenshot history" and "Prompt caching" (at most 4 breakpoints).
102
+ */
103
+ export declare function markCacheBreakpoints(model: string, messages: OpenAiMessage[]): void;
84
104
  export declare class ChatCompletionsAdapter extends LlmAdapter {
85
105
  private readonly port;
86
106
  private readonly options;
107
+ private readonly ladders;
87
108
  constructor(port: ModelPort, options: ModelAdapterOptions);
109
+ /**
110
+ * Called when nothing else (compaction) took a size failure of the session: allows level 4
111
+ * and answers whether the request is retried. False when the failure was final.
112
+ */
113
+ lastResort(sessionId: string, code: string): boolean;
114
+ /** A new turn starts from level 1 (learned budgets stay). */
115
+ newTurn(sessionId: string): void;
116
+ forget(sessionId: string): void;
117
+ private ladder;
118
+ /** An endpoint refusal for size: lower the budget to 3/4 of the refused request and wait for a retry, or end the step. */
119
+ private refused;
120
+ private report;
88
121
  /** Window and output cap come from the host's model catalog, never from constants here. */
89
122
  resolveModel(provider: string, model: string): Promise<LlmResolvedModelInfo>;
90
123
  stream(request: GenerateOptions): AsyncIterable<StreamChunk>;
package/lib/model.js CHANGED
@@ -5,11 +5,20 @@
5
5
  // a session is never sent (fail closed). No provider key lives in the brain.
6
6
  import { LlmAdapter, LlmError, CONTEXT_WINDOW_EXCEEDED_CODE, QUOTA_EXCEEDED_CODE, attributionHeaders, isContextWindowExceededError, offloadedImageText, projectOffloadedImages, } from "@deepseek-ai/dsh-llm";
7
7
  import { requestMaxTokens } from "./model-catalog.js";
8
+ import { MAX_REQUEST_BODY_BYTES, TOOL_TEXT_FLOOR, fitRequest, toolImageMessage } from "./request-budget.js";
8
9
  export const PROVIDER = "bma";
9
10
  /** Error code of a model stream that sent no data for the idle limit (typed stop model_stream_idle). */
10
11
  export const MODEL_STREAM_IDLE = "MODEL_STREAM_IDLE";
11
12
  /** Error code of a model request whose connection failed before any response (refused, reset, DNS). */
12
13
  export const MODEL_CONNECTION = "MODEL_CONNECTION";
14
+ /**
15
+ * Error code of a model request whose body is over the endpoint's size limit, refused by the
16
+ * endpoint (413, or 400 naming the body size) or still over MAX_REQUEST_BODY_BYTES after every
17
+ * reduction. Its own code (not CONTEXT_WINDOW_EXCEEDED): a summary does not shrink images.
18
+ */
19
+ export const REQUEST_TOO_LARGE = "REQUEST_TOO_LARGE";
20
+ /** Endpoint refusals of one step for size that are answered with a smaller request (each 3/4 of the refused one). */
21
+ export const MAX_SIZE_REJECTIONS = 3;
13
22
  /**
14
23
  * Tool result images (computer_observe screenshots) are kept in memory per
15
24
  * session and sent with their results; pruned ones, and all of them after a
@@ -141,7 +150,7 @@ async function toOpenAi(input, options, sessionId, system, signal) {
141
150
  let toolImages = [];
142
151
  const flushToolImages = () => {
143
152
  if (toolImages.length)
144
- out.push({ role: "user", content: toolImages });
153
+ out.push(toolImageMessage(toolImages));
145
154
  toolImages = [];
146
155
  };
147
156
  for (const m of messages) {
@@ -182,12 +191,46 @@ async function toOpenAi(input, options, sessionId, system, signal) {
182
191
  flushToolImages();
183
192
  return out;
184
193
  }
194
+ /**
195
+ * Prompt cache breakpoints. Claude caches a request prefix only up to an explicit cache_control
196
+ * breakpoint (OpenAI-compatible routes to it accept the marker on a content part): without one every
197
+ * request, screenshots and all, is billed as uncached input. Other models cache prefixes by themselves
198
+ * and get no marker. At most 4: the first system message (it ends the tools + system prefix) and the last
199
+ * content part of the 3 most recent user messages (the latest screenshots). Same rule as the Go brain
200
+ * (server/gobrain/infrastructure/openaimodel/cache.go), shared vectors
201
+ * test/brain-conformance/prompt-cache-breakpoints.vectors.json. Basis: Anthropic, computer use tool,
202
+ * "Manage screenshot history" and "Prompt caching" (at most 4 breakpoints).
203
+ */
204
+ export function markCacheBreakpoints(model, messages) {
205
+ if (!model.toLowerCase().includes("claude"))
206
+ return;
207
+ const mark = (m) => {
208
+ const parts = typeof m.content === "string" ? [{ type: "text", text: m.content }] : m.content;
209
+ if (!parts?.length)
210
+ return;
211
+ parts[parts.length - 1] = { ...parts[parts.length - 1], cache_control: { type: "ephemeral" } };
212
+ m.content = parts;
213
+ };
214
+ const system = messages.find((m) => m.role === "system");
215
+ if (system)
216
+ mark(system);
217
+ let marked = 0;
218
+ for (let i = messages.length - 1; i >= 0 && marked < 3; i--) {
219
+ if (messages[i].role !== "user")
220
+ continue;
221
+ mark(messages[i]);
222
+ marked++;
223
+ }
224
+ }
225
+ /** An endpoint's answer to a body over its size limit (ARouter: 400 "request body too large or unreadable"). */
226
+ const BODY_TOO_LARGE = /request (body|entity) too large|payload too large|request_too_large/i;
185
227
  function failure(status, body, headers) {
186
228
  // The canonical overflow code lets dsh compact and retry (compaction-basic reacts to it).
187
229
  // The others name the failure class, which stop.ts reports as the contract's model failure stop code.
188
- const code = isContextWindowExceededError(body) ? CONTEXT_WINDOW_EXCEEDED_CODE
189
- : status === 401 || status === 403 ? "AUTH" : status === 402 ? QUOTA_EXCEEDED_CODE : status === 429 ? "RATE_LIMIT"
190
- : status >= 500 ? "SERVER" : "PROVIDER_HTTP";
230
+ const code = status === 413 || (status === 400 && BODY_TOO_LARGE.test(body)) ? REQUEST_TOO_LARGE
231
+ : isContextWindowExceededError(body) ? CONTEXT_WINDOW_EXCEEDED_CODE
232
+ : status === 401 || status === 403 ? "AUTH" : status === 402 ? QUOTA_EXCEEDED_CODE : status === 429 ? "RATE_LIMIT"
233
+ : status >= 500 ? "SERVER" : "PROVIDER_HTTP";
191
234
  const err = new LlmError(`model endpoint HTTP ${status}: ${body.slice(0, 300)}`, code, { status });
192
235
  const json = parseBody(body);
193
236
  if (status !== 502 && status !== 503 && status !== 504 && json?.error?.retryable !== true && json?.retryable !== true)
@@ -198,11 +241,61 @@ function failure(status, body, headers) {
198
241
  export class ChatCompletionsAdapter extends LlmAdapter {
199
242
  port;
200
243
  options;
244
+ ladders = new Map();
201
245
  constructor(port, options) {
202
246
  super();
203
247
  this.port = port;
204
248
  this.options = options;
205
249
  }
250
+ /**
251
+ * Called when nothing else (compaction) took a size failure of the session: allows level 4
252
+ * and answers whether the request is retried. False when the failure was final.
253
+ */
254
+ lastResort(sessionId, code) {
255
+ const ladder = this.ladders.get(sessionId);
256
+ if ((code !== CONTEXT_WINDOW_EXCEEDED_CODE && code !== REQUEST_TOO_LARGE) || !ladder?.pending)
257
+ return false;
258
+ ladder.pending = false;
259
+ ladder.lastResort = true;
260
+ return true;
261
+ }
262
+ /** A new turn starts from level 1 (learned budgets stay). */
263
+ newTurn(sessionId) {
264
+ const ladder = this.ladders.get(sessionId);
265
+ if (ladder)
266
+ Object.assign(ladder, { rejections: 0, lastResort: false, pending: false });
267
+ }
268
+ forget(sessionId) {
269
+ this.ladders.delete(sessionId);
270
+ }
271
+ ladder(sessionId) {
272
+ let ladder = this.ladders.get(sessionId);
273
+ if (!ladder)
274
+ this.ladders.set(sessionId, (ladder = { rejections: 0, lastResort: false, pending: false }));
275
+ return ladder;
276
+ }
277
+ /** An endpoint refusal for size: lower the budget to 3/4 of the refused request and wait for a retry, or end the step. */
278
+ refused(sessionId, ladder, err, sent, budget) {
279
+ const status = err.failure.status;
280
+ this.report(sessionId, { step: "rejected", levels: [], bytes: sent.bytes, tokens: sent.tokens, byteBudget: budget.bytes, tokenBudget: budget.tokens, ...(status !== undefined ? { status } : {}) });
281
+ if (ladder.rejections >= MAX_SIZE_REJECTIONS) {
282
+ ladder.pending = false;
283
+ this.report(sessionId, { step: "exhausted", levels: [], bytes: sent.bytes, tokens: sent.tokens, byteBudget: budget.bytes, tokenBudget: budget.tokens });
284
+ return err.code === REQUEST_TOO_LARGE
285
+ ? new LlmError(`request_too_large: model request body too large for the endpoint, refused ${ladder.rejections + 1} times (HTTP ${status}, last ${sent.bytes} bytes)`, REQUEST_TOO_LARGE, { cause: err })
286
+ : err;
287
+ }
288
+ ladder.rejections++;
289
+ if (err.code === REQUEST_TOO_LARGE)
290
+ ladder.bytes = Math.floor(Math.min(ladder.bytes ?? Infinity, sent.bytes) * 3 / 4);
291
+ else
292
+ ladder.tokens = Math.floor(Math.min(ladder.tokens ?? Infinity, sent.tokens) * 3 / 4);
293
+ ladder.pending = true;
294
+ return err;
295
+ }
296
+ report(sessionId, reduction) {
297
+ this.port.requestReduced?.(sessionId, reduction);
298
+ }
206
299
  /** Window and output cap come from the host's model catalog, never from constants here. */
207
300
  async resolveModel(provider, model) {
208
301
  const limits = this.port.limits(model);
@@ -226,13 +319,16 @@ export class ChatCompletionsAdapter extends LlmAdapter {
226
319
  const maxTokens = Math.min(request.maxTokens ?? requestMaxTokens(limits), limits.maxOutputTokens);
227
320
  const [minPause, maxPause] = this.options.idleRetryBackoffMs ?? DEFAULT_IDLE_RETRY_BACKOFF_MS;
228
321
  const transientBackoff = this.options.transientRetryBackoffMs ?? DEFAULT_TRANSIENT_RETRY_BACKOFF_MS;
322
+ // A summary request (dsh compaction) is trimmed (level 2) but never refused locally: it is how level 3 runs.
323
+ const ladder = request.purpose === "compaction" ? undefined : this.ladder(sessionId);
229
324
  let idleRetried = false;
230
325
  let transientRetries = 0;
231
326
  for (;;) {
232
327
  const idle = new IdleGuard(this.options.streamIdleMs, request.signal);
233
328
  let content = false;
329
+ const attempt = {};
234
330
  try {
235
- for await (const chunk of this.run(request, sessionId, route, maxTokens, idle, started)) {
331
+ for await (const chunk of this.run(request, sessionId, route, maxTokens, idle, started, limits.contextWindow, ladder, attempt)) {
236
332
  content = true; // the first chunk is a block-start, only ever produced by a content delta
237
333
  yield chunk;
238
334
  }
@@ -240,9 +336,15 @@ export class ChatCompletionsAdapter extends LlmAdapter {
240
336
  this.port.idleRetry?.(sessionId, "recovered");
241
337
  if (transientRetries > 0)
242
338
  this.port.transientRetry?.(sessionId, "recovered");
339
+ if (ladder)
340
+ Object.assign(ladder, { rejections: 0, lastResort: false, pending: false });
243
341
  return;
244
342
  }
245
343
  catch (err) {
344
+ if (ladder && !content && !request.signal?.aborted && err instanceof LlmError && attempt.sent && attempt.budget
345
+ && (err.code === REQUEST_TOO_LARGE || err.code === CONTEXT_WINDOW_EXCEEDED_CODE)) {
346
+ throw this.refused(sessionId, ladder, err, attempt.sent, attempt.budget);
347
+ }
246
348
  if (idle.fired) {
247
349
  const stop = new LlmError(`model stream idle: no data for ${this.options.streamIdleMs} ms`, MODEL_STREAM_IDLE, { cause: err });
248
350
  if (content || request.signal?.aborted)
@@ -277,23 +379,44 @@ export class ChatCompletionsAdapter extends LlmAdapter {
277
379
  }
278
380
  }
279
381
  }
280
- async *run(request, sessionId, route, maxTokens, idle, started) {
281
- const messages = await toOpenAi(request.messages, this.options, sessionId, request.system, idle.signal);
382
+ async *run(request, sessionId, route, maxTokens, idle, started, contextWindow, ladder, attempt) {
383
+ const rest = {
384
+ model: route.model,
385
+ stream: true,
386
+ stream_options: { include_usage: true },
387
+ ...(!route.toolsDisabled && request.tools?.length ? { tools: request.tools.map((t) => ({ type: "function", function: { name: t.name, description: t.description, parameters: t.parameters } })) } : {}),
388
+ max_tokens: maxTokens,
389
+ ...(route.reasoningEffort ? { reasoning_effort: route.reasoningEffort } : {}),
390
+ ...(request.temperature !== undefined ? { temperature: request.temperature } : {}),
391
+ ...(request.stop?.length ? { stop: request.stop } : {}),
392
+ };
393
+ // The window holds the reply too: the input budget is what max_tokens leaves.
394
+ const budget = { tokens: Math.min(ladder?.tokens ?? Infinity, contextWindow - maxTokens), bytes: Math.min(ladder?.bytes ?? Infinity, MAX_REQUEST_BODY_BYTES) };
395
+ const all = await toOpenAi(request.messages, this.options, sessionId, request.system, idle.signal);
396
+ const fit = fitRequest(all, rest, budget, Math.max(TOOL_TEXT_FLOOR, contextWindow), ladder?.lastResort ?? false);
397
+ const sizes = { bytes: fit.bytes, tokens: fit.tokens, byteBudget: budget.bytes, tokenBudget: budget.tokens };
398
+ if (ladder && fit.over) {
399
+ // Nothing is sent. Before the last resort the overflow code hands the step to dsh compaction
400
+ // (a body over the byte budget skips it: a summary does not shrink images); after it, the step fails.
401
+ const final = ladder.lastResort;
402
+ ladder.pending = !final;
403
+ this.report(sessionId, { step: final ? "exhausted" : "compact", levels: final ? fit.levels : [...fit.levels, 3], ...sizes });
404
+ const err = fit.over === "bytes"
405
+ ? new LlmError(`request_too_large: model request body too large for the endpoint (${fit.bytes} bytes, budget ${budget.bytes})${final ? " after every reduction" : ""}`, REQUEST_TOO_LARGE)
406
+ : new LlmError(`model request estimated at ${fit.tokens} tokens, over the model window budget of ${budget.tokens} tokens${final ? " after every reduction" : ""}`, CONTEXT_WINDOW_EXCEEDED_CODE);
407
+ throw err;
408
+ }
409
+ if (ladder && fit.levels.length)
410
+ this.report(sessionId, { step: "reduced", levels: fit.levels, ...sizes });
411
+ const messages = fit.messages;
412
+ markCacheBreakpoints(route.model, messages);
413
+ attempt.sent = fit;
414
+ attempt.budget = budget;
282
415
  const response = await connect(`${route.baseUrl.replace(/\/$/, "")}/chat/completions`, {
283
416
  method: "POST",
284
417
  signal: idle.signal,
285
418
  headers: { ...attributionHeaders(), "content-type": "application/json", authorization: `Bearer ${route.token}` },
286
- body: JSON.stringify({
287
- model: route.model,
288
- stream: true,
289
- stream_options: { include_usage: true },
290
- messages,
291
- ...(request.tools?.length ? { tools: request.tools.map((t) => ({ type: "function", function: { name: t.name, description: t.description, parameters: t.parameters } })) } : {}),
292
- max_tokens: maxTokens,
293
- ...(route.reasoningEffort ? { reasoning_effort: route.reasoningEffort } : {}),
294
- ...(request.temperature !== undefined ? { temperature: request.temperature } : {}),
295
- ...(request.stop?.length ? { stop: request.stop } : {}),
296
- }),
419
+ body: JSON.stringify({ ...rest, messages }),
297
420
  });
298
421
  if (!response.ok || !response.body)
299
422
  throw failure(response.status, await response.text().catch(() => ""), response.headers);
@@ -369,6 +492,8 @@ export class ChatCompletionsAdapter extends LlmAdapter {
369
492
  texts.set(textIdx, (texts.get(textIdx) ?? "") + delta.content);
370
493
  yield { type: "text-delta", index: textIdx, text: delta.content };
371
494
  }
495
+ if (route.toolsDisabled && delta.tool_calls?.length)
496
+ throw new LlmError("maintenance model attempted a tool call", "TOOLS_DISABLED");
372
497
  for (const tc of delta.tool_calls ?? []) {
373
498
  let call = calls.get(tc.index);
374
499
  if (!call) {
@@ -0,0 +1,51 @@
1
+ export type OpenAiMessage = Record<string, unknown>;
2
+ type Part = Record<string, unknown>;
3
+ /**
4
+ * Body budget. The model gateway accepts 64 MiB (HTTP 413 request_too_large above it), but upstream
5
+ * providers are tighter: Bedrock Claude and Gemini refuse a request over 20 MB. 16 MiB stays under
6
+ * those with room for the gateway's format conversion and estimate drift. The one source is
7
+ * contracts/bma/model-request-budget.vectors.json (the Go brain and legacy are locked to it too).
8
+ */
9
+ export declare const MAX_REQUEST_BODY_BYTES: number;
10
+ /**
11
+ * Images sent with a request, newest first, besides those of the newest user input (always sent).
12
+ * Older ones are a text placeholder. Basis: Anthropic computer use reference implementation
13
+ * (only_n_most_recent_images = 3). Anthropic also removes them in batches so the cached prefix
14
+ * moves less often; a batch would carry up to 3 + interval images, which is what overflowed the
15
+ * gateway, so the cap is exact: each new image changes the request from the 4th newest image on,
16
+ * the prefix before it stays cached.
17
+ */
18
+ export declare const RECENT_IMAGES = 3;
19
+ /** Tool results never cleared by level 2 (Anthropic context editing clear_tool_uses: keep 3). */
20
+ export declare const RECENT_TOOL_RESULTS = 3;
21
+ /** Token estimate of one image: about 1.15 megapixels at 750 px per token (Anthropic vision guide). */
22
+ export declare const IMAGE_TOKENS = 1600;
23
+ /** Floor of the per-result tool text cap; the cap itself is the window in characters (about a quarter of it in tokens). */
24
+ export declare const TOOL_TEXT_FLOOR = 16384;
25
+ export declare const OLDER_IMAGE = "[image omitted: older image, not resent]";
26
+ /** Ladder levels that rewrite a request (3 is dsh compaction). */
27
+ export type FitLevel = 2 | 4;
28
+ export interface Budget {
29
+ readonly tokens: number;
30
+ readonly bytes: number;
31
+ }
32
+ export interface Fit {
33
+ readonly messages: OpenAiMessage[];
34
+ /** Body bytes and estimated input tokens of the request with these messages. */
35
+ readonly bytes: number;
36
+ readonly tokens: number;
37
+ /** The budget-driven levels applied, in order. */
38
+ readonly levels: readonly FitLevel[];
39
+ /** Which budget is still exceeded after every allowed level, if any. */
40
+ readonly over?: "tokens" | "bytes";
41
+ }
42
+ export declare function toolImageMessage(content: Part[]): OpenAiMessage;
43
+ /** "<head>\n[... N characters truncated ...]\n<tail>" keeping `keep` characters, or the text when it is short enough. */
44
+ export declare function headTail(text: string, keep: number): string;
45
+ /**
46
+ * Fits `messages` (system first, as toOpenAi made them) into `budget`. `rest` is the request body
47
+ * without its messages (tools and parameters count too). `lastResort` allows level 4: the
48
+ * caller sets it once dsh compaction (level 3) could not make the request fit.
49
+ */
50
+ export declare function fitRequest(messages: readonly OpenAiMessage[], rest: Record<string, unknown>, budget: Budget, toolTextCap: number, lastResort: boolean): Fit;
51
+ export {};
@@ -0,0 +1,246 @@
1
+ // Fits one Chat Completions request into the model's window (tokens) and the model endpoint's body
2
+ // limit (bytes). Only the request is rewritten; dsh's durable history stays as it is. The levels of
3
+ // the ladder, each used only when the one before is not enough:
4
+ // 1 estimate tokens and body bytes before sending (fitRequest measures every candidate)
5
+ // 2 trim bulk: older images out, old tool results cleared, the largest tool texts cut head + tail
6
+ // 3 dsh summary compaction (not here: model.ts hands the step to dsh, see ChatCompletionsAdapter)
7
+ // 4 sliding window: the oldest whole messages go, tool calls together with their results
8
+ // The newest user input is never cut and keeps its images: a request still over budget is not sent
9
+ // and fails with REQUEST_TOO_LARGE or the context overflow code, naming the budget (model.ts).
10
+ // Every request, over budget or not, also keeps only the RECENT_IMAGES newest images and caps each
11
+ // tool result at toolTextCap characters.
12
+ /**
13
+ * Body budget. The model gateway accepts 64 MiB (HTTP 413 request_too_large above it), but upstream
14
+ * providers are tighter: Bedrock Claude and Gemini refuse a request over 20 MB. 16 MiB stays under
15
+ * those with room for the gateway's format conversion and estimate drift. The one source is
16
+ * contracts/bma/model-request-budget.vectors.json (the Go brain and legacy are locked to it too).
17
+ */
18
+ export const MAX_REQUEST_BODY_BYTES = 16 << 20;
19
+ /**
20
+ * Images sent with a request, newest first, besides those of the newest user input (always sent).
21
+ * Older ones are a text placeholder. Basis: Anthropic computer use reference implementation
22
+ * (only_n_most_recent_images = 3). Anthropic also removes them in batches so the cached prefix
23
+ * moves less often; a batch would carry up to 3 + interval images, which is what overflowed the
24
+ * gateway, so the cap is exact: each new image changes the request from the 4th newest image on,
25
+ * the prefix before it stays cached.
26
+ */
27
+ export const RECENT_IMAGES = 3;
28
+ /** Tool results never cleared by level 2 (Anthropic context editing clear_tool_uses: keep 3). */
29
+ export const RECENT_TOOL_RESULTS = 3;
30
+ /** Token estimate of one image: about 1.15 megapixels at 750 px per token (Anthropic vision guide). */
31
+ export const IMAGE_TOKENS = 1_600;
32
+ /** Floor of the per-result tool text cap; the cap itself is the window in characters (about a quarter of it in tokens). */
33
+ export const TOOL_TEXT_FLOOR = 16_384;
34
+ /** Level 2 never cuts a tool text below this many characters. */
35
+ const MIN_TOOL_TEXT = 1_024;
36
+ export const OLDER_IMAGE = "[image omitted: older image, not resent]";
37
+ const DROPPED_IMAGE = "[image omitted: not resent, to fit the model request]";
38
+ const CLEARED_TOOL_RESULT = "[tool result cleared to fit the model request; run the tool again if it is needed]";
39
+ const omitted = (n) => `[${n} earlier message(s) omitted to fit the model request]`;
40
+ /** Messages toOpenAi made to carry the images of tool results (they belong to the assistant step before them). */
41
+ const toolImageMessages = new WeakSet();
42
+ export function toolImageMessage(content) {
43
+ const m = { role: "user", content };
44
+ toolImageMessages.add(m);
45
+ return m;
46
+ }
47
+ /** "<head>\n[... N characters truncated ...]\n<tail>" keeping `keep` characters, or the text when it is short enough. */
48
+ export function headTail(text, keep) {
49
+ if (text.length <= keep)
50
+ return text;
51
+ const head = Math.ceil(keep / 2);
52
+ return `${text.slice(0, head)}\n[... ${text.length - keep} characters truncated ...]\n${text.slice(text.length - (keep - head))}`;
53
+ }
54
+ function measure(m) {
55
+ let text = 0;
56
+ let images = 0;
57
+ if (typeof m.content === "string")
58
+ text += Buffer.byteLength(m.content);
59
+ else if (Array.isArray(m.content)) {
60
+ for (const p of m.content) {
61
+ if (p.type === "image_url")
62
+ images++;
63
+ else if (typeof p.text === "string")
64
+ text += Buffer.byteLength(p.text);
65
+ }
66
+ }
67
+ for (const c of m.tool_calls ?? [])
68
+ text += Buffer.byteLength(c.function.name) + Buffer.byteLength(c.function.arguments);
69
+ // UTF-8 bytes / 4: ASCII is about 4 characters a token; CJK (3 bytes a character) comes out near 1 token a character.
70
+ return { bytes: Buffer.byteLength(JSON.stringify(m)), tokens: Math.ceil(text / 4) + 4 + images * IMAGE_TOKENS };
71
+ }
72
+ const isImage = (p) => p.type === "image_url";
73
+ const isSummary = (m) => m.role === "user" && textOf(m).includes("<compacted-summary>");
74
+ function textOf(m) {
75
+ if (typeof m.content === "string")
76
+ return m.content;
77
+ return Array.isArray(m.content) ? m.content.map((p) => (typeof p.text === "string" ? p.text : "")).join("") : "";
78
+ }
79
+ /** A copy with new content; a tool image message stays one. */
80
+ function withContent(m, content) {
81
+ const copy = { ...m, content };
82
+ if (toolImageMessages.has(m))
83
+ toolImageMessages.add(copy);
84
+ return copy;
85
+ }
86
+ /** Index of the newest user input: the last user message that is neither a tool image message nor a summary. */
87
+ function latestInput(ms) {
88
+ for (let i = ms.length - 1; i >= 0; i--)
89
+ if (ms[i].role === "user" && !toolImageMessages.has(ms[i]) && !isSummary(ms[i]))
90
+ return i;
91
+ return -1;
92
+ }
93
+ /**
94
+ * Fits `messages` (system first, as toOpenAi made them) into `budget`. `rest` is the request body
95
+ * without its messages (tools and parameters count too). `lastResort` allows level 4: the
96
+ * caller sets it once dsh compaction (level 3) could not make the request fit.
97
+ */
98
+ export function fitRequest(messages, rest, budget, toolTextCap, lastResort) {
99
+ const sizes = new WeakMap();
100
+ const size = (m) => {
101
+ let s = sizes.get(m);
102
+ if (!s)
103
+ sizes.set(m, (s = measure(m)));
104
+ return s;
105
+ };
106
+ const base = { bytes: Buffer.byteLength(JSON.stringify({ ...rest, messages: [] })), tokens: Math.ceil(Buffer.byteLength(JSON.stringify(rest.tools ?? [])) / 4) };
107
+ const total = (ms) => {
108
+ const t = { bytes: base.bytes + Math.max(0, ms.length - 1), tokens: base.tokens };
109
+ for (const m of ms) {
110
+ const s = size(m);
111
+ t.bytes += s.bytes;
112
+ t.tokens += s.tokens;
113
+ }
114
+ return t;
115
+ };
116
+ const overOf = (ms) => {
117
+ const t = total(ms);
118
+ return t.bytes > budget.bytes ? "bytes" : t.tokens > budget.tokens ? "tokens" : undefined;
119
+ };
120
+ let ms = keepRecentImages(capToolText([...messages], toolTextCap));
121
+ const levels = [];
122
+ const level = (n) => { if (!levels.includes(n))
123
+ levels.push(n); };
124
+ // Level 2: images first (they are most of the bytes), then old tool results, then the largest tool texts.
125
+ for (const keep of [1, 0]) {
126
+ if (overOf(ms) !== "bytes")
127
+ break;
128
+ const next = dropImages(ms, keep, DROPPED_IMAGE);
129
+ if (next !== ms) {
130
+ ms = next;
131
+ level(2);
132
+ }
133
+ }
134
+ if (overOf(ms)) {
135
+ const tools = ms.flatMap((m, i) => (m.role === "tool" ? [i] : [])).slice(0, -RECENT_TOOL_RESULTS || undefined);
136
+ for (const i of tools) {
137
+ if (!overOf(ms))
138
+ break;
139
+ if (ms[i].content === CLEARED_TOOL_RESULT)
140
+ continue;
141
+ ms[i] = withContent(ms[i], CLEARED_TOOL_RESULT);
142
+ level(2);
143
+ }
144
+ }
145
+ if (overOf(ms)) {
146
+ const original = new Map();
147
+ for (;;) {
148
+ if (!overOf(ms))
149
+ break;
150
+ let largest = -1;
151
+ for (let i = 0; i < ms.length; i++)
152
+ if (ms[i].role === "tool" && typeof ms[i].content === "string" && (largest < 0 || ms[i].content.length > ms[largest].content.length))
153
+ largest = i;
154
+ if (largest < 0)
155
+ break;
156
+ const text = original.get(largest) ?? ms[largest].content;
157
+ const keep = Math.floor(ms[largest].content.length / 2);
158
+ if (keep < MIN_TOOL_TEXT)
159
+ break;
160
+ original.set(largest, text);
161
+ ms[largest] = withContent(ms[largest], headTail(text, keep));
162
+ level(2);
163
+ }
164
+ }
165
+ if (lastResort && overOf(ms)) {
166
+ const windowed = slidingWindow(ms, (candidate) => !overOf(candidate));
167
+ if (windowed !== ms) {
168
+ ms = windowed;
169
+ level(4);
170
+ }
171
+ }
172
+ const t = total(ms);
173
+ return { messages: ms, bytes: t.bytes, tokens: t.tokens, levels, ...(overOf(ms) ? { over: overOf(ms) } : {}) };
174
+ }
175
+ /** Every tool result at most `cap` characters, head + tail. */
176
+ function capToolText(ms, cap) {
177
+ return ms.map((m) => (m.role === "tool" && typeof m.content === "string" && m.content.length > cap ? withContent(m, headTail(m.content, cap)) : m));
178
+ }
179
+ /** RECENT_IMAGES newest images kept besides those of the newest user input; older ones are a placeholder. */
180
+ function keepRecentImages(ms) {
181
+ return dropImages(ms, RECENT_IMAGES, OLDER_IMAGE);
182
+ }
183
+ /** Keeps the `keep` newest images outside the newest user input; the others become `placeholder`. Same array when nothing changed. */
184
+ function dropImages(ms, keep, placeholder) {
185
+ const input = latestInput(ms);
186
+ let seen = 0;
187
+ let out = ms;
188
+ for (let i = ms.length - 1; i >= 0; i--) {
189
+ if (i === input || !Array.isArray(ms[i].content))
190
+ continue;
191
+ const parts = ms[i].content;
192
+ let changed;
193
+ for (let j = parts.length - 1; j >= 0; j--) {
194
+ if (!isImage(parts[j]) || ++seen <= keep)
195
+ continue;
196
+ changed ??= [...parts];
197
+ changed[j] = { type: "text", text: placeholder };
198
+ }
199
+ if (changed) {
200
+ if (out === ms)
201
+ out = [...ms];
202
+ out[i] = withContent(ms[i], changed);
203
+ }
204
+ }
205
+ return out;
206
+ }
207
+ /**
208
+ * Level 4. Drops the fewest oldest units so that `fits`, each unit whole: an assistant message with
209
+ * its tool results and their image message, or one user message. System messages, summaries, the
210
+ * newest user input and the last unit stay. Each run of dropped units becomes one user note, so no
211
+ * tool result is left without its call (or a call without its result) and the model knows.
212
+ */
213
+ function slidingWindow(ms, fits) {
214
+ const units = [];
215
+ for (let i = 0; i < ms.length; i++) {
216
+ const m = ms[i];
217
+ const joins = (m.role === "tool" || toolImageMessages.has(m)) && units.length > 0 && ms[units.at(-1)[0]].role === "assistant";
218
+ if (joins)
219
+ units.at(-1).push(i);
220
+ else
221
+ units.push([i]);
222
+ }
223
+ const input = latestInput(ms);
224
+ const droppable = units.slice(0, -1).filter((u) => !u.includes(input) && ms[u[0]].role !== "system" && !isSummary(ms[u[0]]));
225
+ let candidate = ms;
226
+ for (let k = 1; k <= droppable.length; k++) {
227
+ const dropped = new Set(droppable.slice(0, k).flat());
228
+ candidate = [];
229
+ let run = 0;
230
+ for (let i = 0; i <= ms.length; i++) {
231
+ if (i < ms.length && dropped.has(i)) {
232
+ run++;
233
+ continue;
234
+ }
235
+ if (run) {
236
+ candidate.push({ role: "user", content: omitted(run) });
237
+ run = 0;
238
+ }
239
+ if (i < ms.length)
240
+ candidate.push(ms[i]);
241
+ }
242
+ if (fits(candidate))
243
+ return candidate;
244
+ }
245
+ return candidate;
246
+ }