@beeos-ai/bma-dsh-adapter 0.3.0 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/lib/adapter.d.ts +24 -1
- package/lib/adapter.js +231 -27
- package/lib/conformance/index.js +2 -2
- package/lib/context-store.d.ts +39 -0
- package/lib/context-store.js +95 -0
- package/lib/index.d.ts +5 -2
- package/lib/index.js +4 -2
- package/lib/kernel.d.ts +7 -0
- package/lib/kernel.js +11 -1
- package/lib/model.d.ts +70 -7
- package/lib/model.js +278 -42
- package/lib/request-budget.d.ts +51 -0
- package/lib/request-budget.js +246 -0
- package/lib/spi.d.ts +62 -7
- package/lib/stop.d.ts +1 -1
- package/lib/stop.js +26 -2
- package/lib/testing/scripted-model.d.ts +6 -0
- package/lib/testing/scripted-model.js +5 -1
- package/lib/testing/tmp.js +3 -1
- package/package.json +1 -1
package/lib/kernel.d.ts
CHANGED
|
@@ -15,4 +15,11 @@ export interface KernelOptions {
|
|
|
15
15
|
export declare const COMPACTION_HEADROOM_TOKENS = 8192;
|
|
16
16
|
/** Summary output cap (ADR 0009 preferredSummaryOutputTokens); the model adapter clamps it to the route's real output limit. */
|
|
17
17
|
export declare const COMPACTION_SUMMARY_MAX_TOKENS = 32768;
|
|
18
|
+
/**
|
|
19
|
+
* Summary compaction (level 3 of the request size ladder, model.ts) runs at most 3 times per
|
|
20
|
+
* trigger: 3 attempts at step pressure (dsh default 2), and 3 recoveries of an overflow (dsh
|
|
21
|
+
* default 1), each taking the previous summary and more history; an overflow recovery keeps no
|
|
22
|
+
* verbatim tail. When it still does not fit, the model adapter's sliding window takes over.
|
|
23
|
+
*/
|
|
24
|
+
export declare const COMPACTION_ATTEMPTS = 3;
|
|
18
25
|
export declare function mountKernel(options: KernelOptions): Promise<Context>;
|
package/lib/kernel.js
CHANGED
|
@@ -29,6 +29,13 @@ import LocalAttachmentStore from "@deepseek-ai/dsh-attachment-local";
|
|
|
29
29
|
export const COMPACTION_HEADROOM_TOKENS = 8192;
|
|
30
30
|
/** Summary output cap (ADR 0009 preferredSummaryOutputTokens); the model adapter clamps it to the route's real output limit. */
|
|
31
31
|
export const COMPACTION_SUMMARY_MAX_TOKENS = 32_768;
|
|
32
|
+
/**
|
|
33
|
+
* Summary compaction (level 3 of the request size ladder, model.ts) runs at most 3 times per
|
|
34
|
+
* trigger: 3 attempts at step pressure (dsh default 2), and 3 recoveries of an overflow (dsh
|
|
35
|
+
* default 1), each taking the previous summary and more history; an overflow recovery keeps no
|
|
36
|
+
* verbatim tail. When it still does not fit, the model adapter's sliding window takes over.
|
|
37
|
+
*/
|
|
38
|
+
export const COMPACTION_ATTEMPTS = 3;
|
|
32
39
|
export async function mountKernel(options) {
|
|
33
40
|
const ctx = new Context();
|
|
34
41
|
await ctx.plugin(LlmRuntime);
|
|
@@ -47,7 +54,10 @@ export async function mountKernel(options) {
|
|
|
47
54
|
// from a route's image budget.
|
|
48
55
|
await ctx.plugin(TokenMeter);
|
|
49
56
|
await ctx.plugin(ToolResultPruner);
|
|
50
|
-
await ctx.plugin(CompactionBasic, {
|
|
57
|
+
await ctx.plugin(CompactionBasic, {
|
|
58
|
+
headroomTokens: COMPACTION_HEADROOM_TOKENS, maxTokens: COMPACTION_SUMMARY_MAX_TOKENS,
|
|
59
|
+
compactionRetries: COMPACTION_ATTEMPTS - 1, maxOverflowRetries: COMPACTION_ATTEMPTS,
|
|
60
|
+
});
|
|
51
61
|
await ctx.plugin(ImageOffload, {});
|
|
52
62
|
await ctx.plugin(AgentLoop, { agents: [], ...(options.maxParallelToolCalls !== undefined ? { maxParallelToolCalls: options.maxParallelToolCalls } : {}) });
|
|
53
63
|
return ctx;
|
package/lib/model.d.ts
CHANGED
|
@@ -1,8 +1,19 @@
|
|
|
1
1
|
import { LlmAdapter, type GenerateOptions, type ImageBlock, type LlmResolvedModelInfo, type StreamChunk } from "@deepseek-ai/dsh-llm";
|
|
2
|
+
import { type OpenAiMessage } from "./request-budget.ts";
|
|
2
3
|
import type { ModelPort } from "./spi.ts";
|
|
3
4
|
export declare const PROVIDER = "bma";
|
|
4
5
|
/** Error code of a model stream that sent no data for the idle limit (typed stop model_stream_idle). */
|
|
5
6
|
export declare const MODEL_STREAM_IDLE = "MODEL_STREAM_IDLE";
|
|
7
|
+
/** Error code of a model request whose connection failed before any response (refused, reset, DNS). */
|
|
8
|
+
export declare const MODEL_CONNECTION = "MODEL_CONNECTION";
|
|
9
|
+
/**
|
|
10
|
+
* Error code of a model request whose body is over the endpoint's size limit, refused by the
|
|
11
|
+
* endpoint (413, or 400 naming the body size) or still over MAX_REQUEST_BODY_BYTES after every
|
|
12
|
+
* reduction. Its own code (not CONTEXT_WINDOW_EXCEEDED): a summary does not shrink images.
|
|
13
|
+
*/
|
|
14
|
+
export declare const REQUEST_TOO_LARGE = "REQUEST_TOO_LARGE";
|
|
15
|
+
/** Endpoint refusals of one step for size that are answered with a smaller request (each 3/4 of the refused one). */
|
|
16
|
+
export declare const MAX_SIZE_REJECTIONS = 3;
|
|
6
17
|
/** One image as the request carries it. */
|
|
7
18
|
export interface RequestImage {
|
|
8
19
|
mediaType: string;
|
|
@@ -16,18 +27,26 @@ export interface ModelAdapterOptions {
|
|
|
16
27
|
readImage: (block: ImageBlock, signal?: AbortSignal) => Promise<RequestImage>;
|
|
17
28
|
/** Pause before the one idle retry, uniform in [min, max] ms; default DEFAULT_IDLE_RETRY_BACKOFF_MS. */
|
|
18
29
|
idleRetryBackoffMs?: readonly [number, number];
|
|
30
|
+
/** [first, cap] of the transient retry backoff in ms; default DEFAULT_TRANSIENT_RETRY_BACKOFF_MS. */
|
|
31
|
+
transientRetryBackoffMs?: readonly [number, number];
|
|
19
32
|
/** The images of a tool result the model still sees (RecentToolImages), by session and call. */
|
|
20
33
|
toolImages?: (sessionId: string, callId: string) => readonly RequestImage[] | undefined;
|
|
21
34
|
}
|
|
22
35
|
/**
|
|
23
|
-
* Tool result images (computer_observe screenshots) are kept in memory
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
36
|
+
* Tool result images (computer_observe screenshots) are kept in memory per
|
|
37
|
+
* session and sent with their results; pruned ones, and all of them after a
|
|
38
|
+
* restart, are left out (the result text stays). Pruning is in batches: once
|
|
39
|
+
* more than TOOL_IMAGE_KEEP + TOOL_IMAGE_INTERVAL calls have images, the oldest
|
|
40
|
+
* TOOL_IMAGE_INTERVAL drop out together, so the request prefix stays
|
|
41
|
+
* byte-identical between two such steps and the provider's prompt cache keeps
|
|
42
|
+
* hitting (a sliding window changes the prefix on every request). Same rule as
|
|
43
|
+
* the Go brain (server/gobrain/application/toolimages.go), shared vectors
|
|
44
|
+
* test/brain-conformance/tool-image-pruning.vectors.json. Basis: Anthropic,
|
|
45
|
+
* "Best practices for computer and browser use with Claude" (rolling buffer,
|
|
46
|
+
* cache-aware: keep_n = 3, interval = 25).
|
|
29
47
|
*/
|
|
30
|
-
export declare const
|
|
48
|
+
export declare const TOOL_IMAGE_KEEP = 3;
|
|
49
|
+
export declare const TOOL_IMAGE_INTERVAL = 25;
|
|
31
50
|
export declare class RecentToolImages {
|
|
32
51
|
private readonly bySession;
|
|
33
52
|
remember(sessionId: string, callId: string, images: readonly {
|
|
@@ -45,16 +64,60 @@ export declare class RecentToolImages {
|
|
|
45
64
|
* we use 1 retry and 1-2s (Lead ruling). Once any content arrived nothing is retried.
|
|
46
65
|
*/
|
|
47
66
|
export declare const DEFAULT_IDLE_RETRY_BACKOFF_MS: readonly [number, number];
|
|
67
|
+
/**
|
|
68
|
+
* A transient failure before any content is retried up to TRANSIENT_RETRIES times on a
|
|
69
|
+
* fresh request: HTTP 502/503/504, an error body with `retryable: true`, or a connection
|
|
70
|
+
* that failed before any response. The pause doubles from the first value up to the cap,
|
|
71
|
+
* less up to 25% jitter. Basis: the official OpenAI and Anthropic SDKs (max_retries=2,
|
|
72
|
+
* 0.5s doubling to 8s, jitter). Example: a rolling release of the model gateway refuses
|
|
73
|
+
* connections for a moment (BeeWork tsk_4ec607d9, 2026-11-14: one 503 ended the turn).
|
|
74
|
+
* A failure after content arrived is never retried.
|
|
75
|
+
*
|
|
76
|
+
* A `retry-after-ms` or `Retry-After` header (seconds or an HTTP date) on such a failure
|
|
77
|
+
* lengthens the pause when it is longer than the backoff (same rule as the Go brain,
|
|
78
|
+
* server/gobrain/domain/retry.go). One longer than the backoff cap is not waited for: the
|
|
79
|
+
* failure ends the turn as before, so a pause never exceeds the cap.
|
|
80
|
+
*/
|
|
81
|
+
export declare const TRANSIENT_RETRIES = 2;
|
|
82
|
+
export declare const DEFAULT_TRANSIENT_RETRY_BACKOFF_MS: readonly [number, number];
|
|
83
|
+
/** retry-after-ms, then Retry-After as seconds or an HTTP date; undefined when absent, malformed or not in the future. */
|
|
84
|
+
export declare function retryAfterMs(headers: Headers, now?: number): number | undefined;
|
|
85
|
+
/** The pause before transient retry `n` (0-based): first * 2^n up to cap, less up to 25% jitter. */
|
|
86
|
+
export declare function transientBackoffMs(n: number, [first, cap]: readonly [number, number]): number;
|
|
48
87
|
/**
|
|
49
88
|
* Default stream idle limit. Same value as the Beework Desktop brain's model
|
|
50
89
|
* stream guard (behaviour description only); no industry basis. Any received
|
|
51
90
|
* bytes count, SSE keep-alive comments included.
|
|
52
91
|
*/
|
|
53
92
|
export declare const DEFAULT_STREAM_IDLE_MS = 90000;
|
|
93
|
+
/**
|
|
94
|
+
* Prompt cache breakpoints. Claude caches a request prefix only up to an explicit cache_control
|
|
95
|
+
* breakpoint (OpenAI-compatible routes to it accept the marker on a content part): without one every
|
|
96
|
+
* request, screenshots and all, is billed as uncached input. Other models cache prefixes by themselves
|
|
97
|
+
* and get no marker. At most 4: the first system message (it ends the tools + system prefix) and the last
|
|
98
|
+
* content part of the 3 most recent user messages (the latest screenshots). Same rule as the Go brain
|
|
99
|
+
* (server/gobrain/infrastructure/openaimodel/cache.go), shared vectors
|
|
100
|
+
* test/brain-conformance/prompt-cache-breakpoints.vectors.json. Basis: Anthropic, computer use tool,
|
|
101
|
+
* "Manage screenshot history" and "Prompt caching" (at most 4 breakpoints).
|
|
102
|
+
*/
|
|
103
|
+
export declare function markCacheBreakpoints(model: string, messages: OpenAiMessage[]): void;
|
|
54
104
|
export declare class ChatCompletionsAdapter extends LlmAdapter {
|
|
55
105
|
private readonly port;
|
|
56
106
|
private readonly options;
|
|
107
|
+
private readonly ladders;
|
|
57
108
|
constructor(port: ModelPort, options: ModelAdapterOptions);
|
|
109
|
+
/**
|
|
110
|
+
* Called when nothing else (compaction) took a size failure of the session: allows level 4
|
|
111
|
+
* and answers whether the request is retried. False when the failure was final.
|
|
112
|
+
*/
|
|
113
|
+
lastResort(sessionId: string, code: string): boolean;
|
|
114
|
+
/** A new turn starts from level 1 (learned budgets stay). */
|
|
115
|
+
newTurn(sessionId: string): void;
|
|
116
|
+
forget(sessionId: string): void;
|
|
117
|
+
private ladder;
|
|
118
|
+
/** An endpoint refusal for size: lower the budget to 3/4 of the refused request and wait for a retry, or end the step. */
|
|
119
|
+
private refused;
|
|
120
|
+
private report;
|
|
58
121
|
/** Window and output cap come from the host's model catalog, never from constants here. */
|
|
59
122
|
resolveModel(provider: string, model: string): Promise<LlmResolvedModelInfo>;
|
|
60
123
|
stream(request: GenerateOptions): AsyncIterable<StreamChunk>;
|
package/lib/model.js
CHANGED
|
@@ -3,26 +3,45 @@
|
|
|
3
3
|
// ModelPort: each session carries its own platform-issued token, so the
|
|
4
4
|
// platform attributes each request to exactly one session. A request without
|
|
5
5
|
// a session is never sent (fail closed). No provider key lives in the brain.
|
|
6
|
-
import { LlmAdapter, LlmError, CONTEXT_WINDOW_EXCEEDED_CODE, attributionHeaders, isContextWindowExceededError, offloadedImageText, projectOffloadedImages, } from "@deepseek-ai/dsh-llm";
|
|
6
|
+
import { LlmAdapter, LlmError, CONTEXT_WINDOW_EXCEEDED_CODE, QUOTA_EXCEEDED_CODE, attributionHeaders, isContextWindowExceededError, offloadedImageText, projectOffloadedImages, } from "@deepseek-ai/dsh-llm";
|
|
7
7
|
import { requestMaxTokens } from "./model-catalog.js";
|
|
8
|
+
import { MAX_REQUEST_BODY_BYTES, TOOL_TEXT_FLOOR, fitRequest, toolImageMessage } from "./request-budget.js";
|
|
8
9
|
export const PROVIDER = "bma";
|
|
9
10
|
/** Error code of a model stream that sent no data for the idle limit (typed stop model_stream_idle). */
|
|
10
11
|
export const MODEL_STREAM_IDLE = "MODEL_STREAM_IDLE";
|
|
12
|
+
/** Error code of a model request whose connection failed before any response (refused, reset, DNS). */
|
|
13
|
+
export const MODEL_CONNECTION = "MODEL_CONNECTION";
|
|
11
14
|
/**
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
* text stays). Basis: Anthropic's computer-use reference loop keeps only the N
|
|
16
|
-
* most recent screenshots in the request (only_n_most_recent_images, default
|
|
17
|
-
* 3) — screenshots are large and only the latest describe the screen.
|
|
15
|
+
* Error code of a model request whose body is over the endpoint's size limit, refused by the
|
|
16
|
+
* endpoint (413, or 400 naming the body size) or still over MAX_REQUEST_BODY_BYTES after every
|
|
17
|
+
* reduction. Its own code (not CONTEXT_WINDOW_EXCEEDED): a summary does not shrink images.
|
|
18
18
|
*/
|
|
19
|
-
export const
|
|
19
|
+
export const REQUEST_TOO_LARGE = "REQUEST_TOO_LARGE";
|
|
20
|
+
/** Endpoint refusals of one step for size that are answered with a smaller request (each 3/4 of the refused one). */
|
|
21
|
+
export const MAX_SIZE_REJECTIONS = 3;
|
|
22
|
+
/**
|
|
23
|
+
* Tool result images (computer_observe screenshots) are kept in memory per
|
|
24
|
+
* session and sent with their results; pruned ones, and all of them after a
|
|
25
|
+
* restart, are left out (the result text stays). Pruning is in batches: once
|
|
26
|
+
* more than TOOL_IMAGE_KEEP + TOOL_IMAGE_INTERVAL calls have images, the oldest
|
|
27
|
+
* TOOL_IMAGE_INTERVAL drop out together, so the request prefix stays
|
|
28
|
+
* byte-identical between two such steps and the provider's prompt cache keeps
|
|
29
|
+
* hitting (a sliding window changes the prefix on every request). Same rule as
|
|
30
|
+
* the Go brain (server/gobrain/application/toolimages.go), shared vectors
|
|
31
|
+
* test/brain-conformance/tool-image-pruning.vectors.json. Basis: Anthropic,
|
|
32
|
+
* "Best practices for computer and browser use with Claude" (rolling buffer,
|
|
33
|
+
* cache-aware: keep_n = 3, interval = 25).
|
|
34
|
+
*/
|
|
35
|
+
export const TOOL_IMAGE_KEEP = 3;
|
|
36
|
+
export const TOOL_IMAGE_INTERVAL = 25;
|
|
20
37
|
export class RecentToolImages {
|
|
21
38
|
bySession = new Map();
|
|
22
39
|
remember(sessionId, callId, images) {
|
|
40
|
+
if (!images.length)
|
|
41
|
+
return;
|
|
23
42
|
const list = this.bySession.get(sessionId) ?? [];
|
|
24
43
|
list.push({ callId, images: images.map((i) => ({ mediaType: i.mediaType, data: i.data })) });
|
|
25
|
-
this.bySession.set(sessionId, list.slice(
|
|
44
|
+
this.bySession.set(sessionId, list.length > TOOL_IMAGE_KEEP + TOOL_IMAGE_INTERVAL ? list.slice(TOOL_IMAGE_INTERVAL) : list);
|
|
26
45
|
}
|
|
27
46
|
get(sessionId, callId) {
|
|
28
47
|
return this.bySession.get(sessionId)?.find((e) => e.callId === callId)?.images;
|
|
@@ -39,6 +58,63 @@ export class RecentToolImages {
|
|
|
39
58
|
* we use 1 retry and 1-2s (Lead ruling). Once any content arrived nothing is retried.
|
|
40
59
|
*/
|
|
41
60
|
export const DEFAULT_IDLE_RETRY_BACKOFF_MS = [1_000, 2_000];
|
|
61
|
+
/**
|
|
62
|
+
* A transient failure before any content is retried up to TRANSIENT_RETRIES times on a
|
|
63
|
+
* fresh request: HTTP 502/503/504, an error body with `retryable: true`, or a connection
|
|
64
|
+
* that failed before any response. The pause doubles from the first value up to the cap,
|
|
65
|
+
* less up to 25% jitter. Basis: the official OpenAI and Anthropic SDKs (max_retries=2,
|
|
66
|
+
* 0.5s doubling to 8s, jitter). Example: a rolling release of the model gateway refuses
|
|
67
|
+
* connections for a moment (BeeWork tsk_4ec607d9, 2026-11-14: one 503 ended the turn).
|
|
68
|
+
* A failure after content arrived is never retried.
|
|
69
|
+
*
|
|
70
|
+
* A `retry-after-ms` or `Retry-After` header (seconds or an HTTP date) on such a failure
|
|
71
|
+
* lengthens the pause when it is longer than the backoff (same rule as the Go brain,
|
|
72
|
+
* server/gobrain/domain/retry.go). One longer than the backoff cap is not waited for: the
|
|
73
|
+
* failure ends the turn as before, so a pause never exceeds the cap.
|
|
74
|
+
*/
|
|
75
|
+
export const TRANSIENT_RETRIES = 2;
|
|
76
|
+
export const DEFAULT_TRANSIENT_RETRY_BACKOFF_MS = [500, 8_000];
|
|
77
|
+
const transientErrors = new WeakMap();
|
|
78
|
+
function transient(err, info) {
|
|
79
|
+
transientErrors.set(err, info);
|
|
80
|
+
return err;
|
|
81
|
+
}
|
|
82
|
+
function transientInfo(err) {
|
|
83
|
+
return err instanceof LlmError ? transientErrors.get(err) : undefined;
|
|
84
|
+
}
|
|
85
|
+
function parseBody(body) {
|
|
86
|
+
try {
|
|
87
|
+
const json = JSON.parse(body);
|
|
88
|
+
return json && typeof json === "object" ? json : undefined;
|
|
89
|
+
}
|
|
90
|
+
catch {
|
|
91
|
+
return undefined;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
/** An error code from the body, only when it looks like an identifier (it goes to logs). */
|
|
95
|
+
function errorCode(json) {
|
|
96
|
+
const code = json?.error?.code ?? json?.error?.type;
|
|
97
|
+
return typeof code === "string" && /^[\w.-]{1,64}$/.test(code) ? code : undefined;
|
|
98
|
+
}
|
|
99
|
+
/** retry-after-ms, then Retry-After as seconds or an HTTP date; undefined when absent, malformed or not in the future. */
|
|
100
|
+
export function retryAfterMs(headers, now = Date.now()) {
|
|
101
|
+
const ms = Number(headers.get("retry-after-ms") ?? NaN);
|
|
102
|
+
if (Number.isFinite(ms) && ms > 0)
|
|
103
|
+
return ms;
|
|
104
|
+
const value = headers.get("retry-after")?.trim();
|
|
105
|
+
if (!value)
|
|
106
|
+
return undefined;
|
|
107
|
+
if (/^\d+(\.\d+)?$/.test(value)) {
|
|
108
|
+
const seconds = Number(value);
|
|
109
|
+
return seconds > 0 ? seconds * 1_000 : undefined;
|
|
110
|
+
}
|
|
111
|
+
const at = Date.parse(value);
|
|
112
|
+
return Number.isFinite(at) && at > now ? at - now : undefined;
|
|
113
|
+
}
|
|
114
|
+
/** The pause before transient retry `n` (0-based): first * 2^n up to cap, less up to 25% jitter. */
|
|
115
|
+
export function transientBackoffMs(n, [first, cap]) {
|
|
116
|
+
return Math.min(first * 2 ** n, cap) * (1 - Math.random() * 0.25);
|
|
117
|
+
}
|
|
42
118
|
/**
|
|
43
119
|
* Default stream idle limit. Same value as the Beework Desktop brain's model
|
|
44
120
|
* stream guard (behaviour description only); no industry basis. Any received
|
|
@@ -74,7 +150,7 @@ async function toOpenAi(input, options, sessionId, system, signal) {
|
|
|
74
150
|
let toolImages = [];
|
|
75
151
|
const flushToolImages = () => {
|
|
76
152
|
if (toolImages.length)
|
|
77
|
-
out.push(
|
|
153
|
+
out.push(toolImageMessage(toolImages));
|
|
78
154
|
toolImages = [];
|
|
79
155
|
};
|
|
80
156
|
for (const m of messages) {
|
|
@@ -115,20 +191,111 @@ async function toOpenAi(input, options, sessionId, system, signal) {
|
|
|
115
191
|
flushToolImages();
|
|
116
192
|
return out;
|
|
117
193
|
}
|
|
118
|
-
|
|
194
|
+
/**
|
|
195
|
+
* Prompt cache breakpoints. Claude caches a request prefix only up to an explicit cache_control
|
|
196
|
+
* breakpoint (OpenAI-compatible routes to it accept the marker on a content part): without one every
|
|
197
|
+
* request, screenshots and all, is billed as uncached input. Other models cache prefixes by themselves
|
|
198
|
+
* and get no marker. At most 4: the first system message (it ends the tools + system prefix) and the last
|
|
199
|
+
* content part of the 3 most recent user messages (the latest screenshots). Same rule as the Go brain
|
|
200
|
+
* (server/gobrain/infrastructure/openaimodel/cache.go), shared vectors
|
|
201
|
+
* test/brain-conformance/prompt-cache-breakpoints.vectors.json. Basis: Anthropic, computer use tool,
|
|
202
|
+
* "Manage screenshot history" and "Prompt caching" (at most 4 breakpoints).
|
|
203
|
+
*/
|
|
204
|
+
export function markCacheBreakpoints(model, messages) {
|
|
205
|
+
if (!model.toLowerCase().includes("claude"))
|
|
206
|
+
return;
|
|
207
|
+
const mark = (m) => {
|
|
208
|
+
const parts = typeof m.content === "string" ? [{ type: "text", text: m.content }] : m.content;
|
|
209
|
+
if (!parts?.length)
|
|
210
|
+
return;
|
|
211
|
+
parts[parts.length - 1] = { ...parts[parts.length - 1], cache_control: { type: "ephemeral" } };
|
|
212
|
+
m.content = parts;
|
|
213
|
+
};
|
|
214
|
+
const system = messages.find((m) => m.role === "system");
|
|
215
|
+
if (system)
|
|
216
|
+
mark(system);
|
|
217
|
+
let marked = 0;
|
|
218
|
+
for (let i = messages.length - 1; i >= 0 && marked < 3; i--) {
|
|
219
|
+
if (messages[i].role !== "user")
|
|
220
|
+
continue;
|
|
221
|
+
mark(messages[i]);
|
|
222
|
+
marked++;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
/** An endpoint's answer to a body over its size limit (ARouter: 400 "request body too large or unreadable"). */
|
|
226
|
+
const BODY_TOO_LARGE = /request (body|entity) too large|payload too large|request_too_large/i;
|
|
227
|
+
function failure(status, body, headers) {
|
|
119
228
|
// The canonical overflow code lets dsh compact and retry (compaction-basic reacts to it).
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
229
|
+
// The others name the failure class, which stop.ts reports as the contract's model failure stop code.
|
|
230
|
+
const code = status === 413 || (status === 400 && BODY_TOO_LARGE.test(body)) ? REQUEST_TOO_LARGE
|
|
231
|
+
: isContextWindowExceededError(body) ? CONTEXT_WINDOW_EXCEEDED_CODE
|
|
232
|
+
: status === 401 || status === 403 ? "AUTH" : status === 402 ? QUOTA_EXCEEDED_CODE : status === 429 ? "RATE_LIMIT"
|
|
233
|
+
: status >= 500 ? "SERVER" : "PROVIDER_HTTP";
|
|
234
|
+
const err = new LlmError(`model endpoint HTTP ${status}: ${body.slice(0, 300)}`, code, { status });
|
|
235
|
+
const json = parseBody(body);
|
|
236
|
+
if (status !== 502 && status !== 503 && status !== 504 && json?.error?.retryable !== true && json?.retryable !== true)
|
|
237
|
+
return err;
|
|
238
|
+
const after = headers ? retryAfterMs(headers) : undefined;
|
|
239
|
+
return transient(err, { reason: `HTTP ${status}${errorCode(json) ? ` ${errorCode(json)}` : ""}`, ...(after !== undefined ? { retryAfterMs: after } : {}) });
|
|
123
240
|
}
|
|
124
241
|
export class ChatCompletionsAdapter extends LlmAdapter {
|
|
125
242
|
port;
|
|
126
243
|
options;
|
|
244
|
+
ladders = new Map();
|
|
127
245
|
constructor(port, options) {
|
|
128
246
|
super();
|
|
129
247
|
this.port = port;
|
|
130
248
|
this.options = options;
|
|
131
249
|
}
|
|
250
|
+
/**
|
|
251
|
+
* Called when nothing else (compaction) took a size failure of the session: allows level 4
|
|
252
|
+
* and answers whether the request is retried. False when the failure was final.
|
|
253
|
+
*/
|
|
254
|
+
lastResort(sessionId, code) {
|
|
255
|
+
const ladder = this.ladders.get(sessionId);
|
|
256
|
+
if ((code !== CONTEXT_WINDOW_EXCEEDED_CODE && code !== REQUEST_TOO_LARGE) || !ladder?.pending)
|
|
257
|
+
return false;
|
|
258
|
+
ladder.pending = false;
|
|
259
|
+
ladder.lastResort = true;
|
|
260
|
+
return true;
|
|
261
|
+
}
|
|
262
|
+
/** A new turn starts from level 1 (learned budgets stay). */
|
|
263
|
+
newTurn(sessionId) {
|
|
264
|
+
const ladder = this.ladders.get(sessionId);
|
|
265
|
+
if (ladder)
|
|
266
|
+
Object.assign(ladder, { rejections: 0, lastResort: false, pending: false });
|
|
267
|
+
}
|
|
268
|
+
forget(sessionId) {
|
|
269
|
+
this.ladders.delete(sessionId);
|
|
270
|
+
}
|
|
271
|
+
ladder(sessionId) {
|
|
272
|
+
let ladder = this.ladders.get(sessionId);
|
|
273
|
+
if (!ladder)
|
|
274
|
+
this.ladders.set(sessionId, (ladder = { rejections: 0, lastResort: false, pending: false }));
|
|
275
|
+
return ladder;
|
|
276
|
+
}
|
|
277
|
+
/** An endpoint refusal for size: lower the budget to 3/4 of the refused request and wait for a retry, or end the step. */
|
|
278
|
+
refused(sessionId, ladder, err, sent, budget) {
|
|
279
|
+
const status = err.failure.status;
|
|
280
|
+
this.report(sessionId, { step: "rejected", levels: [], bytes: sent.bytes, tokens: sent.tokens, byteBudget: budget.bytes, tokenBudget: budget.tokens, ...(status !== undefined ? { status } : {}) });
|
|
281
|
+
if (ladder.rejections >= MAX_SIZE_REJECTIONS) {
|
|
282
|
+
ladder.pending = false;
|
|
283
|
+
this.report(sessionId, { step: "exhausted", levels: [], bytes: sent.bytes, tokens: sent.tokens, byteBudget: budget.bytes, tokenBudget: budget.tokens });
|
|
284
|
+
return err.code === REQUEST_TOO_LARGE
|
|
285
|
+
? new LlmError(`request_too_large: model request body too large for the endpoint, refused ${ladder.rejections + 1} times (HTTP ${status}, last ${sent.bytes} bytes)`, REQUEST_TOO_LARGE, { cause: err })
|
|
286
|
+
: err;
|
|
287
|
+
}
|
|
288
|
+
ladder.rejections++;
|
|
289
|
+
if (err.code === REQUEST_TOO_LARGE)
|
|
290
|
+
ladder.bytes = Math.floor(Math.min(ladder.bytes ?? Infinity, sent.bytes) * 3 / 4);
|
|
291
|
+
else
|
|
292
|
+
ladder.tokens = Math.floor(Math.min(ladder.tokens ?? Infinity, sent.tokens) * 3 / 4);
|
|
293
|
+
ladder.pending = true;
|
|
294
|
+
return err;
|
|
295
|
+
}
|
|
296
|
+
report(sessionId, reduction) {
|
|
297
|
+
this.port.requestReduced?.(sessionId, reduction);
|
|
298
|
+
}
|
|
132
299
|
/** Window and output cap come from the host's model catalog, never from constants here. */
|
|
133
300
|
async resolveModel(provider, model) {
|
|
134
301
|
const limits = this.port.limits(model);
|
|
@@ -151,58 +318,108 @@ export class ChatCompletionsAdapter extends LlmAdapter {
|
|
|
151
318
|
// caller (loop or compaction) asked for is never above the ceiling, the model's real output limit.
|
|
152
319
|
const maxTokens = Math.min(request.maxTokens ?? requestMaxTokens(limits), limits.maxOutputTokens);
|
|
153
320
|
const [minPause, maxPause] = this.options.idleRetryBackoffMs ?? DEFAULT_IDLE_RETRY_BACKOFF_MS;
|
|
154
|
-
|
|
321
|
+
const transientBackoff = this.options.transientRetryBackoffMs ?? DEFAULT_TRANSIENT_RETRY_BACKOFF_MS;
|
|
322
|
+
// A summary request (dsh compaction) is trimmed (level 2) but never refused locally: it is how level 3 runs.
|
|
323
|
+
const ladder = request.purpose === "compaction" ? undefined : this.ladder(sessionId);
|
|
324
|
+
let idleRetried = false;
|
|
325
|
+
let transientRetries = 0;
|
|
326
|
+
for (;;) {
|
|
155
327
|
const idle = new IdleGuard(this.options.streamIdleMs, request.signal);
|
|
156
328
|
let content = false;
|
|
329
|
+
const attempt = {};
|
|
157
330
|
try {
|
|
158
|
-
for await (const chunk of this.run(request, sessionId, route, maxTokens, idle, started)) {
|
|
331
|
+
for await (const chunk of this.run(request, sessionId, route, maxTokens, idle, started, limits.contextWindow, ladder, attempt)) {
|
|
159
332
|
content = true; // the first chunk is a block-start, only ever produced by a content delta
|
|
160
333
|
yield chunk;
|
|
161
334
|
}
|
|
162
|
-
if (
|
|
335
|
+
if (idleRetried)
|
|
163
336
|
this.port.idleRetry?.(sessionId, "recovered");
|
|
337
|
+
if (transientRetries > 0)
|
|
338
|
+
this.port.transientRetry?.(sessionId, "recovered");
|
|
339
|
+
if (ladder)
|
|
340
|
+
Object.assign(ladder, { rejections: 0, lastResort: false, pending: false });
|
|
164
341
|
return;
|
|
165
342
|
}
|
|
166
343
|
catch (err) {
|
|
167
|
-
if (!
|
|
344
|
+
if (ladder && !content && !request.signal?.aborted && err instanceof LlmError && attempt.sent && attempt.budget
|
|
345
|
+
&& (err.code === REQUEST_TOO_LARGE || err.code === CONTEXT_WINDOW_EXCEEDED_CODE)) {
|
|
346
|
+
throw this.refused(sessionId, ladder, err, attempt.sent, attempt.budget);
|
|
347
|
+
}
|
|
348
|
+
if (idle.fired) {
|
|
349
|
+
const stop = new LlmError(`model stream idle: no data for ${this.options.streamIdleMs} ms`, MODEL_STREAM_IDLE, { cause: err });
|
|
350
|
+
if (content || request.signal?.aborted)
|
|
351
|
+
throw stop;
|
|
352
|
+
if (idleRetried) {
|
|
353
|
+
this.port.idleRetry?.(sessionId, "exhausted");
|
|
354
|
+
throw stop;
|
|
355
|
+
}
|
|
356
|
+
idleRetried = true;
|
|
357
|
+
this.port.idleRetry?.(sessionId, "retry");
|
|
358
|
+
await pause(minPause + Math.random() * (maxPause - minPause), request.signal);
|
|
359
|
+
if (request.signal?.aborted)
|
|
360
|
+
throw stop; // cancelled while waiting: give up, no new request
|
|
361
|
+
continue;
|
|
362
|
+
}
|
|
363
|
+
const info = transientInfo(err);
|
|
364
|
+
if (content || request.signal?.aborted || !info)
|
|
365
|
+
throw err;
|
|
366
|
+
const wait = Math.max(transientBackoffMs(transientRetries, transientBackoff), info.retryAfterMs ?? 0);
|
|
367
|
+
if (transientRetries >= TRANSIENT_RETRIES || wait > transientBackoff[1]) {
|
|
368
|
+
this.port.transientRetry?.(sessionId, "exhausted", info.reason);
|
|
168
369
|
throw err;
|
|
169
|
-
const stop = new LlmError(`model stream idle: no data for ${this.options.streamIdleMs} ms`, MODEL_STREAM_IDLE, { cause: err });
|
|
170
|
-
if (content || request.signal?.aborted)
|
|
171
|
-
throw stop;
|
|
172
|
-
if (attempt > 0) {
|
|
173
|
-
this.port.idleRetry?.(sessionId, "exhausted");
|
|
174
|
-
throw stop;
|
|
175
370
|
}
|
|
176
|
-
|
|
177
|
-
|
|
371
|
+
transientRetries++;
|
|
372
|
+
this.port.transientRetry?.(sessionId, "retry", info.reason);
|
|
373
|
+
await pause(wait, request.signal);
|
|
178
374
|
if (request.signal?.aborted)
|
|
179
|
-
throw
|
|
375
|
+
throw err; // cancelled while waiting: give up, no new request
|
|
180
376
|
}
|
|
181
377
|
finally {
|
|
182
378
|
idle.stop();
|
|
183
379
|
}
|
|
184
380
|
}
|
|
185
381
|
}
|
|
186
|
-
async *run(request, sessionId, route, maxTokens, idle, started) {
|
|
187
|
-
const
|
|
188
|
-
|
|
382
|
+
async *run(request, sessionId, route, maxTokens, idle, started, contextWindow, ladder, attempt) {
|
|
383
|
+
const rest = {
|
|
384
|
+
model: route.model,
|
|
385
|
+
stream: true,
|
|
386
|
+
stream_options: { include_usage: true },
|
|
387
|
+
...(!route.toolsDisabled && request.tools?.length ? { tools: request.tools.map((t) => ({ type: "function", function: { name: t.name, description: t.description, parameters: t.parameters } })) } : {}),
|
|
388
|
+
max_tokens: maxTokens,
|
|
389
|
+
...(route.reasoningEffort ? { reasoning_effort: route.reasoningEffort } : {}),
|
|
390
|
+
...(request.temperature !== undefined ? { temperature: request.temperature } : {}),
|
|
391
|
+
...(request.stop?.length ? { stop: request.stop } : {}),
|
|
392
|
+
};
|
|
393
|
+
// The window holds the reply too: the input budget is what max_tokens leaves.
|
|
394
|
+
const budget = { tokens: Math.min(ladder?.tokens ?? Infinity, contextWindow - maxTokens), bytes: Math.min(ladder?.bytes ?? Infinity, MAX_REQUEST_BODY_BYTES) };
|
|
395
|
+
const all = await toOpenAi(request.messages, this.options, sessionId, request.system, idle.signal);
|
|
396
|
+
const fit = fitRequest(all, rest, budget, Math.max(TOOL_TEXT_FLOOR, contextWindow), ladder?.lastResort ?? false);
|
|
397
|
+
const sizes = { bytes: fit.bytes, tokens: fit.tokens, byteBudget: budget.bytes, tokenBudget: budget.tokens };
|
|
398
|
+
if (ladder && fit.over) {
|
|
399
|
+
// Nothing is sent. Before the last resort the overflow code hands the step to dsh compaction
|
|
400
|
+
// (a body over the byte budget skips it: a summary does not shrink images); after it, the step fails.
|
|
401
|
+
const final = ladder.lastResort;
|
|
402
|
+
ladder.pending = !final;
|
|
403
|
+
this.report(sessionId, { step: final ? "exhausted" : "compact", levels: final ? fit.levels : [...fit.levels, 3], ...sizes });
|
|
404
|
+
const err = fit.over === "bytes"
|
|
405
|
+
? new LlmError(`request_too_large: model request body too large for the endpoint (${fit.bytes} bytes, budget ${budget.bytes})${final ? " after every reduction" : ""}`, REQUEST_TOO_LARGE)
|
|
406
|
+
: new LlmError(`model request estimated at ${fit.tokens} tokens, over the model window budget of ${budget.tokens} tokens${final ? " after every reduction" : ""}`, CONTEXT_WINDOW_EXCEEDED_CODE);
|
|
407
|
+
throw err;
|
|
408
|
+
}
|
|
409
|
+
if (ladder && fit.levels.length)
|
|
410
|
+
this.report(sessionId, { step: "reduced", levels: fit.levels, ...sizes });
|
|
411
|
+
const messages = fit.messages;
|
|
412
|
+
markCacheBreakpoints(route.model, messages);
|
|
413
|
+
attempt.sent = fit;
|
|
414
|
+
attempt.budget = budget;
|
|
415
|
+
const response = await connect(`${route.baseUrl.replace(/\/$/, "")}/chat/completions`, {
|
|
189
416
|
method: "POST",
|
|
190
417
|
signal: idle.signal,
|
|
191
418
|
headers: { ...attributionHeaders(), "content-type": "application/json", authorization: `Bearer ${route.token}` },
|
|
192
|
-
body: JSON.stringify({
|
|
193
|
-
model: route.model,
|
|
194
|
-
stream: true,
|
|
195
|
-
stream_options: { include_usage: true },
|
|
196
|
-
messages,
|
|
197
|
-
...(request.tools?.length ? { tools: request.tools.map((t) => ({ type: "function", function: { name: t.name, description: t.description, parameters: t.parameters } })) } : {}),
|
|
198
|
-
max_tokens: maxTokens,
|
|
199
|
-
...(route.reasoningEffort ? { reasoning_effort: route.reasoningEffort } : {}),
|
|
200
|
-
...(request.temperature !== undefined ? { temperature: request.temperature } : {}),
|
|
201
|
-
...(request.stop?.length ? { stop: request.stop } : {}),
|
|
202
|
-
}),
|
|
419
|
+
body: JSON.stringify({ ...rest, messages }),
|
|
203
420
|
});
|
|
204
421
|
if (!response.ok || !response.body)
|
|
205
|
-
throw failure(response.status, await response.text().catch(() => ""));
|
|
422
|
+
throw failure(response.status, await response.text().catch(() => ""), response.headers);
|
|
206
423
|
let next = 0;
|
|
207
424
|
let textIdx;
|
|
208
425
|
let reasoningIdx;
|
|
@@ -275,6 +492,8 @@ export class ChatCompletionsAdapter extends LlmAdapter {
|
|
|
275
492
|
texts.set(textIdx, (texts.get(textIdx) ?? "") + delta.content);
|
|
276
493
|
yield { type: "text-delta", index: textIdx, text: delta.content };
|
|
277
494
|
}
|
|
495
|
+
if (route.toolsDisabled && delta.tool_calls?.length)
|
|
496
|
+
throw new LlmError("maintenance model attempted a tool call", "TOOLS_DISABLED");
|
|
278
497
|
for (const tc of delta.tool_calls ?? []) {
|
|
279
498
|
let call = calls.get(tc.index);
|
|
280
499
|
if (!call) {
|
|
@@ -321,6 +540,23 @@ export class ChatCompletionsAdapter extends LlmAdapter {
|
|
|
321
540
|
yield { type: "finish", reason };
|
|
322
541
|
}
|
|
323
542
|
}
|
|
543
|
+
/**
|
|
544
|
+
* fetch, with a connection that failed before any response (refused, reset, DNS, TLS) as a
|
|
545
|
+
* transient MODEL_CONNECTION error. An abort (cancel or idle guard) stays as it is.
|
|
546
|
+
*/
|
|
547
|
+
async function connect(url, init) {
|
|
548
|
+
try {
|
|
549
|
+
return await fetch(url, init);
|
|
550
|
+
}
|
|
551
|
+
catch (err) {
|
|
552
|
+
if (init.signal?.aborted)
|
|
553
|
+
throw err;
|
|
554
|
+
const cause = err.cause;
|
|
555
|
+
const detail = cause?.code ?? cause?.message ?? err.message;
|
|
556
|
+
const code = typeof cause?.code === "string" && /^[\w.-]{1,64}$/.test(cause.code) ? cause.code : "failed";
|
|
557
|
+
throw transient(new LlmError(`model endpoint unreachable: ${detail}`, MODEL_CONNECTION, { cause: err }), { reason: `connection ${code}` });
|
|
558
|
+
}
|
|
559
|
+
}
|
|
324
560
|
/** Resolves after ms, or at once when the signal aborts. */
|
|
325
561
|
function pause(ms, signal) {
|
|
326
562
|
return new Promise((resolve) => {
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
export type OpenAiMessage = Record<string, unknown>;
|
|
2
|
+
type Part = Record<string, unknown>;
|
|
3
|
+
/**
|
|
4
|
+
* Body budget. The model gateway accepts 64 MiB (HTTP 413 request_too_large above it), but upstream
|
|
5
|
+
* providers are tighter: Bedrock Claude and Gemini refuse a request over 20 MB. 16 MiB stays under
|
|
6
|
+
* those with room for the gateway's format conversion and estimate drift. The one source is
|
|
7
|
+
* contracts/bma/model-request-budget.vectors.json (the Go brain and legacy are locked to it too).
|
|
8
|
+
*/
|
|
9
|
+
export declare const MAX_REQUEST_BODY_BYTES: number;
|
|
10
|
+
/**
|
|
11
|
+
* Images sent with a request, newest first, besides those of the newest user input (always sent).
|
|
12
|
+
* Older ones are a text placeholder. Basis: Anthropic computer use reference implementation
|
|
13
|
+
* (only_n_most_recent_images = 3). Anthropic also removes them in batches so the cached prefix
|
|
14
|
+
* moves less often; a batch would carry up to 3 + interval images, which is what overflowed the
|
|
15
|
+
* gateway, so the cap is exact: each new image changes the request from the 4th newest image on,
|
|
16
|
+
* the prefix before it stays cached.
|
|
17
|
+
*/
|
|
18
|
+
export declare const RECENT_IMAGES = 3;
|
|
19
|
+
/** Tool results never cleared by level 2 (Anthropic context editing clear_tool_uses: keep 3). */
|
|
20
|
+
export declare const RECENT_TOOL_RESULTS = 3;
|
|
21
|
+
/** Token estimate of one image: about 1.15 megapixels at 750 px per token (Anthropic vision guide). */
|
|
22
|
+
export declare const IMAGE_TOKENS = 1600;
|
|
23
|
+
/** Floor of the per-result tool text cap; the cap itself is the window in characters (about a quarter of it in tokens). */
|
|
24
|
+
export declare const TOOL_TEXT_FLOOR = 16384;
|
|
25
|
+
export declare const OLDER_IMAGE = "[image omitted: older image, not resent]";
|
|
26
|
+
/** Ladder levels that rewrite a request (3 is dsh compaction). */
|
|
27
|
+
export type FitLevel = 2 | 4;
|
|
28
|
+
export interface Budget {
|
|
29
|
+
readonly tokens: number;
|
|
30
|
+
readonly bytes: number;
|
|
31
|
+
}
|
|
32
|
+
export interface Fit {
|
|
33
|
+
readonly messages: OpenAiMessage[];
|
|
34
|
+
/** Body bytes and estimated input tokens of the request with these messages. */
|
|
35
|
+
readonly bytes: number;
|
|
36
|
+
readonly tokens: number;
|
|
37
|
+
/** The budget-driven levels applied, in order. */
|
|
38
|
+
readonly levels: readonly FitLevel[];
|
|
39
|
+
/** Which budget is still exceeded after every allowed level, if any. */
|
|
40
|
+
readonly over?: "tokens" | "bytes";
|
|
41
|
+
}
|
|
42
|
+
export declare function toolImageMessage(content: Part[]): OpenAiMessage;
|
|
43
|
+
/** "<head>\n[... N characters truncated ...]\n<tail>" keeping `keep` characters, or the text when it is short enough. */
|
|
44
|
+
export declare function headTail(text: string, keep: number): string;
|
|
45
|
+
/**
|
|
46
|
+
* Fits `messages` (system first, as toOpenAi made them) into `budget`. `rest` is the request body
|
|
47
|
+
* without its messages (tools and parameters count too). `lastResort` allows level 4: the
|
|
48
|
+
* caller sets it once dsh compaction (level 3) could not make the request fit.
|
|
49
|
+
*/
|
|
50
|
+
export declare function fitRequest(messages: readonly OpenAiMessage[], rest: Record<string, unknown>, budget: Budget, toolTextCap: number, lastResort: boolean): Fit;
|
|
51
|
+
export {};
|