@cubicecho/agent-core 2.24.0 → 2.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -7
- package/dist/agent-loop.d.ts +84 -205
- package/dist/agent-loop.js +288 -597
- package/dist/calibration.d.ts +14 -10
- package/dist/calibration.js +30 -23
- package/dist/capabilities.d.ts +111 -42
- package/dist/capabilities.js +281 -138
- package/dist/client.d.ts +104 -35
- package/dist/client.js +205 -74
- package/dist/compaction.d.ts +103 -60
- package/dist/compaction.js +160 -86
- package/dist/config.d.ts +48 -12
- package/dist/config.js +20 -1
- package/dist/continuation.d.ts +24 -16
- package/dist/continuation.js +70 -44
- package/dist/digest.d.ts +7 -0
- package/dist/digest.js +18 -0
- package/dist/errors.d.ts +24 -15
- package/dist/errors.js +23 -15
- package/dist/events.d.ts +100 -63
- package/dist/events.js +250 -120
- package/dist/guards.d.ts +35 -4
- package/dist/guards.js +38 -5
- package/dist/hook-events.d.ts +21 -5
- package/dist/hook-events.js +23 -9
- package/dist/hooks.d.ts +96 -73
- package/dist/hooks.js +202 -116
- package/dist/index.d.ts +30 -26
- package/dist/index.js +29 -24
- package/dist/ledger.d.ts +62 -30
- package/dist/ledger.js +163 -56
- package/dist/platform.d.ts +13 -0
- package/dist/platform.js +13 -0
- package/dist/preselect.d.ts +222 -0
- package/dist/preselect.js +290 -0
- package/dist/request-body.d.ts +32 -0
- package/dist/request-body.js +47 -0
- package/dist/reset.d.ts +1 -0
- package/dist/reset.js +7 -6
- package/dist/retry.d.ts +29 -138
- package/dist/retry.js +69 -282
- package/dist/run-calls.d.ts +123 -0
- package/dist/run-calls.js +351 -0
- package/dist/run-turn.d.ts +36 -15
- package/dist/run-turn.js +74 -37
- package/dist/runtime.d.ts +23 -17
- package/dist/runtime.js +34 -24
- package/dist/schema-compat.d.ts +35 -18
- package/dist/schema-compat.js +340 -175
- package/dist/scope.d.ts +28 -5
- package/dist/scope.js +37 -10
- package/dist/side-task.d.ts +123 -40
- package/dist/side-task.js +181 -63
- package/dist/snapshot.d.ts +10 -5
- package/dist/snapshot.js +73 -30
- package/dist/spec.d.ts +56 -25
- package/dist/spec.js +494 -215
- package/dist/stream.d.ts +66 -16
- package/dist/stream.js +220 -97
- package/dist/thinking.d.ts +23 -8
- package/dist/thinking.js +92 -44
- package/dist/tokens.d.ts +164 -6
- package/dist/tokens.js +319 -6
- package/dist/tool-calls.d.ts +82 -17
- package/dist/tool-calls.js +198 -83
- package/dist/tool-loading.d.ts +101 -189
- package/dist/tool-loading.js +277 -371
- package/dist/wire.d.ts +81 -0
- package/dist/wire.js +73 -0
- package/llms.txt +76 -39
- package/package.json +4 -4
package/dist/retry.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import OpenAI from
|
|
2
|
-
import {
|
|
1
|
+
import OpenAI from 'openai';
|
|
2
|
+
import { ABORT_EVENT } from './platform.js';
|
|
3
|
+
import { HttpStatus } from './wire.js';
|
|
3
4
|
/**
|
|
4
5
|
* Everything about a request failing that is not about what the request said.
|
|
5
6
|
*
|
|
@@ -10,270 +11,21 @@ import { CHARS_PER_TOKEN } from "./tokens.js";
|
|
|
10
11
|
*/
|
|
11
12
|
/** The endpoint stopped answering mid-request. Its own class so the retry can recognise it. */
|
|
12
13
|
export class EndpointSilent extends Error {
|
|
13
|
-
name =
|
|
14
|
+
name = 'EndpointSilent';
|
|
14
15
|
}
|
|
15
16
|
/**
|
|
16
17
|
* The request was bigger than the model will read. Its own class so nothing retries it: sending
|
|
17
18
|
* the same too-large request again is the same refusal, one round trip later.
|
|
18
19
|
*
|
|
20
|
+
* @remarks
|
|
19
21
|
* `runTurn` raises it from either side of the round trip — its own pre-flight guard, or the
|
|
20
22
|
* endpoint's refusal read back through `isOverflow` — so a caller has one thing to catch whether
|
|
21
23
|
* or not it gave a `contextLimit`. The second carries the endpoint's own message, and the error
|
|
22
24
|
* it was built from as `cause`.
|
|
23
25
|
*/
|
|
24
26
|
export class ContextOverflow extends Error {
|
|
25
|
-
name =
|
|
27
|
+
name = 'ContextOverflow';
|
|
26
28
|
}
|
|
27
|
-
/**
|
|
28
|
-
* 1234 → "1.2k". The numbers in an overflow message are large and nobody reads the units digit.
|
|
29
|
-
*
|
|
30
|
-
* @param tokens The count to render.
|
|
31
|
-
*/
|
|
32
|
-
export const compact = (tokens) => tokens >= 1000 ? `${(tokens / 1000).toFixed(1)}k` : String(tokens);
|
|
33
|
-
/** What `{"role":"","content":""},` costs around a message's own text, in characters. */
|
|
34
|
-
const ENVELOPE = 25;
|
|
35
|
-
/** The same for `{"id":"","type":"function","function":{"name":"","arguments":""}},` in a call. */
|
|
36
|
-
const CALL_ENVELOPE = 66;
|
|
37
|
-
// `ENVELOPE` is the two keys every message has. These are the three that only some do, and each
|
|
38
|
-
// is the key with its punctuation and the comma after the value it holds — the value's own
|
|
39
|
-
// length is counted where the value is read. Applying `ENVELOPE` alone to these shapes left the
|
|
40
|
-
// keys out, which cost 4.5 tokens on every tool result: the message a tool-using run has most
|
|
41
|
-
// of, and short in the direction that lets an overflow through the guard meant to catch it.
|
|
42
|
-
/** What `"name":"",` costs around a message's name. */
|
|
43
|
-
const NAME_KEY = 10;
|
|
44
|
-
/** The same for `"tool_call_id":"",` around a tool result's call id. */
|
|
45
|
-
const TOOL_CALL_ID_KEY = 18;
|
|
46
|
-
/** The same for `"tool_calls":[]` around the calls; each call's own comma is in `CALL_ENVELOPE`. */
|
|
47
|
-
const TOOL_CALLS_KEY = 15;
|
|
48
|
-
// A content part is an object in the body the same way a message is, and its two keys are in the
|
|
49
|
-
// request exactly as `tool_call_id` was. Charging only `part.text` left them out, which is 6.5
|
|
50
|
-
// tokens per part — short in proportion to how finely the content is split rather than to how
|
|
51
|
-
// much it says, so a transcript a client appends block by block is worst hit.
|
|
52
|
-
/** What `{"type":"text","text":""},` costs around a text part. */
|
|
53
|
-
const TEXT_PART = 26;
|
|
54
|
-
/** The same for `{"type":"refusal","refusal":""},` around a refusal part. */
|
|
55
|
-
const REFUSAL_PART = 32;
|
|
56
|
-
/**
|
|
57
|
-
* The same for `"reasoning_content":"",` around an assistant message's scratchpad.
|
|
58
|
-
*
|
|
59
|
-
* Not in the SDK's types, and passed back by the caller who keeps it: gpt-oss and DeepSeek in
|
|
60
|
-
* thinking mode want the analysis behind a tool call on the next request. Left uncounted, the
|
|
61
|
-
* guard came up short by the whole scratchpad on exactly the runs that follow that rule.
|
|
62
|
-
*/
|
|
63
|
-
const REASONING_KEY = 23;
|
|
64
|
-
/** The same for `"reasoning":"",`, OpenRouter's spelling of it. */
|
|
65
|
-
const REASONING_ALT_KEY = 15;
|
|
66
|
-
/** The divisor an option asked for, or the fallback when it asked for nothing usable. */
|
|
67
|
-
const divisor = (charsPerToken) => charsPerToken !== undefined && charsPerToken > 0 ? charsPerToken : CHARS_PER_TOKEN;
|
|
68
|
-
/**
|
|
69
|
-
* How many characters one message is worth: its keys, and its content in whichever shape.
|
|
70
|
-
*
|
|
71
|
-
* @param message The message as it will be sent.
|
|
72
|
-
*/
|
|
73
|
-
export function messageChars(message) {
|
|
74
|
-
let chars = message.role.length + ENVELOPE;
|
|
75
|
-
const { content } = message;
|
|
76
|
-
if (typeof content === "string")
|
|
77
|
-
chars += content.length;
|
|
78
|
-
else if (Array.isArray(content))
|
|
79
|
-
for (const part of content) {
|
|
80
|
-
// Text and refusal parts carry their own strings; an image or an audio part carries a URL
|
|
81
|
-
// or a blob, and neither is priced by its length anyway — a vision model does not charge
|
|
82
|
-
// an image by its base64 length, so counting the data URL would overshoot by more than
|
|
83
|
-
// leaving the part out undershoots.
|
|
84
|
-
if (part.type === "text")
|
|
85
|
-
chars += TEXT_PART + part.text.length;
|
|
86
|
-
else if (part.type === "refusal")
|
|
87
|
-
chars += REFUSAL_PART + part.refusal.length;
|
|
88
|
-
}
|
|
89
|
-
if ("name" in message && typeof message.name === "string")
|
|
90
|
-
chars += NAME_KEY + message.name.length;
|
|
91
|
-
if ("tool_call_id" in message && typeof message.tool_call_id === "string")
|
|
92
|
-
chars += TOOL_CALL_ID_KEY + message.tool_call_id.length;
|
|
93
|
-
const { reasoning_content: reasoning, reasoning: alternate } = message;
|
|
94
|
-
if (typeof reasoning === "string")
|
|
95
|
-
chars += REASONING_KEY + reasoning.length;
|
|
96
|
-
if (typeof alternate === "string")
|
|
97
|
-
chars += REASONING_ALT_KEY + alternate.length;
|
|
98
|
-
if ("tool_calls" in message && Array.isArray(message.tool_calls)) {
|
|
99
|
-
chars += TOOL_CALLS_KEY;
|
|
100
|
-
for (const call of message.tool_calls) {
|
|
101
|
-
chars += CALL_ENVELOPE + call.id.length;
|
|
102
|
-
if (call.type === "function")
|
|
103
|
-
chars += call.function.name.length + call.function.arguments.length;
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
return chars;
|
|
107
|
-
}
|
|
108
|
-
/**
|
|
109
|
-
* The tools half, cached against the array.
|
|
110
|
-
*
|
|
111
|
-
* Serialising two dozen JSON schemas to measure them, on every turn, to get the same number every
|
|
112
|
-
* time, was the more expensive half of this function.
|
|
113
|
-
*
|
|
114
|
-
* The array is the key, so this pays only a caller that hands the same one back. That is not what
|
|
115
|
-
* a turn built through `sanitizeTools` or `relaxTools` does — both are a `map`, so each build
|
|
116
|
-
* allocates a fresh array however stable the tools inside it are, and those builds miss here every
|
|
117
|
-
* time. It is not free to change: keying on the tools instead would hit for them, at the cost of
|
|
118
|
-
* the array-level memoisation `tests/retry.test.ts` pins, which deliberately holds a mutated array
|
|
119
|
-
* to its first reading. Sizing happens once per turn either way, so the miss costs one walk of the
|
|
120
|
-
* schemas rather than a walk per attempt.
|
|
121
|
-
*
|
|
122
|
-
* Characters rather than tokens, so a calibrated divisor applies to the schemas without walking
|
|
123
|
-
* them again.
|
|
124
|
-
*/
|
|
125
|
-
const toolLengths = new WeakMap();
|
|
126
|
-
/**
|
|
127
|
-
* How many characters a tool array is worth, measured once per array.
|
|
128
|
-
*
|
|
129
|
-
* @param tools The tool definitions as they will be sent. An empty array is worth nothing.
|
|
130
|
-
*/
|
|
131
|
-
export function toolsChars(tools) {
|
|
132
|
-
if (!tools.length)
|
|
133
|
-
return 0;
|
|
134
|
-
const hit = toolLengths.get(tools);
|
|
135
|
-
if (hit !== undefined)
|
|
136
|
-
return hit;
|
|
137
|
-
// Schemas are arbitrarily shaped, so this one really is a serialisation — but it happens once
|
|
138
|
-
// per tool array rather than once per turn.
|
|
139
|
-
const length = JSON.stringify(tools).length;
|
|
140
|
-
toolLengths.set(tools, length);
|
|
141
|
-
return length;
|
|
142
|
-
}
|
|
143
|
-
/**
|
|
144
|
-
* How many characters a request is worth: the walk `requestTokens` divides, without the division.
|
|
145
|
-
*
|
|
146
|
-
* What calibration reads a reported prompt count against, since a ratio is only as good as the
|
|
147
|
-
* character count it was taken over agreeing with the one it is later applied to.
|
|
148
|
-
*
|
|
149
|
-
* @param body The request as it was sent, tools included.
|
|
150
|
-
*/
|
|
151
|
-
export function requestChars(body) {
|
|
152
|
-
let chars = 0;
|
|
153
|
-
for (const message of body.messages)
|
|
154
|
-
chars += messageChars(message);
|
|
155
|
-
return chars;
|
|
156
|
-
}
|
|
157
|
-
/**
|
|
158
|
-
* What this request will cost the window, in tokens, near enough.
|
|
159
|
-
*
|
|
160
|
-
* See `estimateTokens` for why it is characters over four and which way it is wrong on purpose,
|
|
161
|
-
* and `charsPerTokenFor` for the divisor a model's own reported usage has measured instead.
|
|
162
|
-
*
|
|
163
|
-
* Summed by walking the body rather than by serialising it. `JSON.stringify` on the messages
|
|
164
|
-
* built the entire transcript into a string on every call and threw it away having read nothing
|
|
165
|
-
* but its `.length` — against a transcript that grows by a turn each turn, and one the SDK is
|
|
166
|
-
* about to serialise again to send. What the walk misses is JSON's own punctuation and the keys,
|
|
167
|
-
* which the envelope constants put back — one per key and one per content part, rather than one
|
|
168
|
-
* per message, since the keys only some shapes carry and the parts a client appends block by
|
|
169
|
-
* block are most of what a tool-using transcript is made of. What is left is a message's
|
|
170
|
-
* escaping, which is not a constant and is small against an estimate that is already characters
|
|
171
|
-
* over four.
|
|
172
|
-
*
|
|
173
|
-
* @param body The request as it will be sent, tools included.
|
|
174
|
-
* @param options The divisor, `CHARS_PER_TOKEN` when none is given.
|
|
175
|
-
*/
|
|
176
|
-
export const requestTokens = (body, { charsPerToken } = {}) => {
|
|
177
|
-
// Characters first and the division once at the end, rather than a rounded count per message:
|
|
178
|
-
// `Math.ceil` on every one of a few hundred messages is a few hundred tokens of pure rounding.
|
|
179
|
-
// The tools are divided on their own, as they were when their tokens were cached, so the
|
|
180
|
-
// uncalibrated count is the one it always was.
|
|
181
|
-
const per = divisor(charsPerToken);
|
|
182
|
-
return Math.ceil(requestChars(body) / per) + Math.ceil(toolsChars(body.tools ?? []) / per);
|
|
183
|
-
};
|
|
184
|
-
/** The parts, in the order a readout reads them. */
|
|
185
|
-
const PARTS = ["system", "tools", "history", "toolResults"];
|
|
186
|
-
/**
|
|
187
|
-
* What each part of a request is worth in characters, by the same walk `requestTokens` divides.
|
|
188
|
-
*
|
|
189
|
-
* Exact and additive: the parts sum to `total`, which is `requestChars` plus `toolsChars`. The
|
|
190
|
-
* conversion to tokens is `contextTokens`' business, because that is where an estimate and a
|
|
191
|
-
* reported count have to be told apart.
|
|
192
|
-
*
|
|
193
|
-
* @param body The request as it will be sent, tools included.
|
|
194
|
-
*/
|
|
195
|
-
export function contextChars(body) {
|
|
196
|
-
const out = {
|
|
197
|
-
system: 0,
|
|
198
|
-
tools: toolsChars(body.tools ?? []),
|
|
199
|
-
history: 0,
|
|
200
|
-
toolResults: 0,
|
|
201
|
-
total: 0,
|
|
202
|
-
};
|
|
203
|
-
for (const message of body.messages) {
|
|
204
|
-
const chars = messageChars(message);
|
|
205
|
-
// Every system message and not just the leading one: a host that appends guidance, or a
|
|
206
|
-
// hook that injects a preface, has put more of the window there and wants to be told so.
|
|
207
|
-
if (message.role === "system" || message.role === "developer")
|
|
208
|
-
out.system += chars;
|
|
209
|
-
else if (message.role === "tool")
|
|
210
|
-
out.toolResults += chars;
|
|
211
|
-
else
|
|
212
|
-
out.history += chars;
|
|
213
|
-
}
|
|
214
|
-
out.total = out.system + out.tools + out.history + out.toolResults;
|
|
215
|
-
return out;
|
|
216
|
-
}
|
|
217
|
-
/**
|
|
218
|
-
* Shares `total` out over these parts by their character counts, the largest absorbing the
|
|
219
|
-
* rounding so they add up to it exactly rather than to within a few tokens of it.
|
|
220
|
-
*/
|
|
221
|
-
function share(chars, over, total) {
|
|
222
|
-
const out = { system: 0, tools: 0, history: 0, toolResults: 0, total };
|
|
223
|
-
const measured = over.reduce((sum, part) => sum + chars[part], 0);
|
|
224
|
-
if (measured <= 0 || total <= 0)
|
|
225
|
-
return out;
|
|
226
|
-
const absorber = over.reduce((a, b) => (chars[b] > chars[a] ? b : a));
|
|
227
|
-
let assigned = 0;
|
|
228
|
-
for (const part of over) {
|
|
229
|
-
if (part === absorber)
|
|
230
|
-
continue;
|
|
231
|
-
out[part] = Math.round((chars[part] / measured) * total);
|
|
232
|
-
assigned += out[part];
|
|
233
|
-
}
|
|
234
|
-
out[absorber] = Math.max(0, total - assigned);
|
|
235
|
-
return out;
|
|
236
|
-
}
|
|
237
|
-
/**
|
|
238
|
-
* What each part of a request costs the window, in tokens, adding up to the whole.
|
|
239
|
-
*
|
|
240
|
-
* Shares rather than four independent estimates, because a readout whose parts do not add up to
|
|
241
|
-
* the total beside them is a readout nobody trusts. Nothing in the round trip reports anything
|
|
242
|
-
* finer than a prompt count — a completion says how many tokens it read and not a word about
|
|
243
|
-
* where they came from — so the proportions are an estimate whatever the total is.
|
|
244
|
-
*
|
|
245
|
-
* Without a reported count the total is `requestTokens`, and the tools are counted the way it and
|
|
246
|
-
* `TurnMetrics.toolSchemaTokens` count them rather than shared out, so the two agree by
|
|
247
|
-
* construction and an operator does not read one number for the tool block in the metrics and a
|
|
248
|
-
* different one here. With a reported count every part is a share of it, the tools included:
|
|
249
|
-
* that number is the server's, and the point of using it is that the parts sum to what was
|
|
250
|
-
* charged.
|
|
251
|
-
*
|
|
252
|
-
* @param body The request as it will be sent, tools included.
|
|
253
|
-
* @param options The divisor, and the reported prompt count when there is one.
|
|
254
|
-
*/
|
|
255
|
-
export function contextTokens(body, { charsPerToken, promptTokens } = {}) {
|
|
256
|
-
const chars = contextChars(body);
|
|
257
|
-
if (promptTokens !== undefined && promptTokens > 0)
|
|
258
|
-
return share(chars, PARTS, promptTokens);
|
|
259
|
-
const per = divisor(charsPerToken);
|
|
260
|
-
const tools = Math.ceil(chars.tools / per);
|
|
261
|
-
const rest = Math.ceil((chars.total - chars.tools) / per);
|
|
262
|
-
const out = share(chars, ["system", "history", "toolResults"], rest);
|
|
263
|
-
out.tools = tools;
|
|
264
|
-
out.total = rest + tools;
|
|
265
|
-
return out;
|
|
266
|
-
}
|
|
267
|
-
/**
|
|
268
|
-
* One message's estimated tokens, by the same count `requestTokens` sums for a whole request.
|
|
269
|
-
*
|
|
270
|
-
* For the arithmetic that weighs part of a transcript against a window — `planCompaction`'s kept
|
|
271
|
-
* tail — where `estimateTokens` on the text alone would leave out the calls and the envelope.
|
|
272
|
-
*
|
|
273
|
-
* @param message The message as it will be sent.
|
|
274
|
-
* @param options The divisor, `CHARS_PER_TOKEN` when none is given.
|
|
275
|
-
*/
|
|
276
|
-
export const messageTokens = (message, { charsPerToken } = {}) => Math.ceil(messageChars(message) / divisor(charsPerToken));
|
|
277
29
|
/**
|
|
278
30
|
* Servers refuse an over-long request in their own words; these are the ones worth reading as
|
|
279
31
|
* that rather than as a broken request. Matched loosely — every one of them is some
|
|
@@ -288,6 +40,7 @@ const OVERFLOW = [
|
|
|
288
40
|
/**
|
|
289
41
|
* A rate limit says the same words and means the opposite thing.
|
|
290
42
|
*
|
|
43
|
+
* @remarks
|
|
291
44
|
* OpenAI refuses a request over the per-minute token budget with "Request too large for gpt-4o
|
|
292
45
|
* ... on tokens per min (TPM)", which is "too large for" beside "tokens" — both halves of the
|
|
293
46
|
* test below. That is a 429, which `isTransient` accepts and which succeeds on the next attempt;
|
|
@@ -298,17 +51,19 @@ const RATE_LIMITED = /per (min|hour|day)|rate.?limit|\b[tr]pm\b|quota/i;
|
|
|
298
51
|
/**
|
|
299
52
|
* Whether a refusal means the request was too big, rather than merely refused.
|
|
300
53
|
*
|
|
54
|
+
* @param detail - The endpoint's own message.
|
|
55
|
+
* @returns `true` for an overflow: one of the over-long wordings, beside `token` or `context`.
|
|
56
|
+
* `false` for anything that reads as a rate limit, whatever else it says.
|
|
57
|
+
*
|
|
58
|
+
* @remarks
|
|
301
59
|
* Rate limits are ruled out first: they borrow the same words and mean the opposite, being worth
|
|
302
60
|
* another attempt where an overflow never is.
|
|
303
|
-
*
|
|
304
|
-
* @param detail The endpoint's own message.
|
|
305
61
|
*/
|
|
306
|
-
export const isOverflow = (detail) => !RATE_LIMITED.test(detail) &&
|
|
307
|
-
OVERFLOW.some((pattern) => pattern.test(detail)) &&
|
|
308
|
-
/token|context/i.test(detail);
|
|
62
|
+
export const isOverflow = (detail) => !RATE_LIMITED.test(detail) && OVERFLOW.some((pattern) => pattern.test(detail)) && /token|context/i.test(detail);
|
|
309
63
|
/**
|
|
310
64
|
* The smallest window worth believing in, and the floor under `runTurn`'s guard.
|
|
311
65
|
*
|
|
66
|
+
* @remarks
|
|
312
67
|
* `runTurn`'s `contextLimit` reads it as a sanity check on a number it was handed: below this,
|
|
313
68
|
* the limit is taken for a placeholder — an unset column, a listing that said nothing — rather
|
|
314
69
|
* than a window worth refusing a run over. That is the whole of what reads it; `contextLimitFor`
|
|
@@ -322,83 +77,115 @@ export const isOverflow = (detail) => !RATE_LIMITED.test(detail) &&
|
|
|
322
77
|
* guarded wrongly.
|
|
323
78
|
*/
|
|
324
79
|
export const SMALLEST_LIKELY_WINDOW = 8192;
|
|
80
|
+
/** The statuses below 500 that say "not now" rather than "not this": a timeout, a conflict, a rate limit. */
|
|
81
|
+
const TRANSIENT_STATUSES = new Set([
|
|
82
|
+
HttpStatus.RequestTimeout,
|
|
83
|
+
HttpStatus.Conflict,
|
|
84
|
+
HttpStatus.TooManyRequests,
|
|
85
|
+
]);
|
|
325
86
|
/**
|
|
326
87
|
* Whether a failed request is worth trying again.
|
|
327
88
|
*
|
|
89
|
+
* @param error - The rejection, as caught. What is not an SDK error is not transient.
|
|
90
|
+
* @returns `true` for an `EndpointSilent`, a connection error, a 408, 409 or 429, and any status
|
|
91
|
+
* of 500 or above; `false` for any other SDK error, one with no status included.
|
|
92
|
+
*
|
|
93
|
+
* @remarks
|
|
328
94
|
* The question is whether the request was *refused or lost*, rather than answered with a
|
|
329
95
|
* complaint about its contents: a connection that never landed, a server too busy or too broken
|
|
330
96
|
* to answer, an endpoint that went quiet. A 400 for a malformed tool schema would fail exactly
|
|
331
97
|
* the same way on every attempt, and the two capability cases below are negotiated rather than
|
|
332
98
|
* retried blindly.
|
|
333
|
-
*
|
|
334
|
-
* @param error The rejection, as caught. What is not an SDK error is not transient.
|
|
335
99
|
*/
|
|
336
100
|
export function isTransient(error) {
|
|
337
|
-
if (error instanceof EndpointSilent)
|
|
101
|
+
if (error instanceof EndpointSilent) {
|
|
338
102
|
return true;
|
|
339
|
-
|
|
103
|
+
}
|
|
104
|
+
if (error instanceof OpenAI.APIConnectionError) {
|
|
340
105
|
return true;
|
|
341
|
-
|
|
106
|
+
}
|
|
107
|
+
if (!(error instanceof OpenAI.APIError)) {
|
|
342
108
|
return false;
|
|
109
|
+
}
|
|
343
110
|
const { status } = error;
|
|
344
|
-
return status
|
|
111
|
+
return status !== undefined && (TRANSIENT_STATUSES.has(status) || status >= HttpStatus.InternalServerError);
|
|
345
112
|
}
|
|
346
113
|
/**
|
|
347
114
|
* Whether a failure is the endpoint refusing the request as written, rather than losing it.
|
|
348
115
|
*
|
|
116
|
+
* @param error - The rejection, as caught. What is not an SDK error refuses nothing.
|
|
117
|
+
* @returns `true` for an SDK error whose status is 400 or 422, and for nothing else.
|
|
118
|
+
*
|
|
119
|
+
* @remarks
|
|
349
120
|
* 400 and 422 are what a server says when it read the body and disliked it, which makes them the
|
|
350
121
|
* only statuses worth answering by sending something different. Any 4xx is too wide: 401, 404 and
|
|
351
122
|
* 429 are 4xx and none of them is about the fields, and a 429 is one `isTransient` accepts — a
|
|
352
123
|
* caller that re-sent on it at once doubled the rate against a server that had just asked for less.
|
|
353
124
|
* Catching everything is wider still, since an abort or a connection that never landed then
|
|
354
125
|
* latches off whatever the re-send left out.
|
|
355
|
-
*
|
|
356
|
-
* @param error The rejection, as caught. What is not an SDK error refuses nothing.
|
|
357
126
|
*/
|
|
358
|
-
export const refusesRequest = (error) => error instanceof OpenAI.APIError &&
|
|
127
|
+
export const refusesRequest = (error) => error instanceof OpenAI.APIError &&
|
|
128
|
+
(error.status === HttpStatus.BadRequest || error.status === HttpStatus.UnprocessableEntity);
|
|
129
|
+
/** The error type llama.cpp gives a request it cannot serve yet. */
|
|
130
|
+
const UNAVAILABLE_ERROR = 'unavailable_error';
|
|
359
131
|
/**
|
|
360
132
|
* Whether a failure is a local server still loading the model, rather than one failing to serve.
|
|
361
133
|
*
|
|
134
|
+
* @param error - The rejection, as caught.
|
|
135
|
+
* @returns `true` for a 503 whose body is of type `unavailable_error`, or whose message says the
|
|
136
|
+
* model is loading; `false` for any other status and for what is not an SDK error.
|
|
137
|
+
*
|
|
138
|
+
* @remarks
|
|
362
139
|
* llama.cpp answers 503 `Loading model` with type `unavailable_error` from the moment it starts
|
|
363
140
|
* until the weights are mapped, and a router build says the same while it swaps models. That is
|
|
364
141
|
* thirty to ninety seconds for a large model from a cold page cache, and `backoffMs` gives up
|
|
365
142
|
* inside fifteen: sized for a busy host, not for one reading a file. A plain 503 is not this.
|
|
366
|
-
*
|
|
367
|
-
* @param error The rejection, as caught.
|
|
368
143
|
*/
|
|
369
144
|
export function isModelLoading(error) {
|
|
370
|
-
if (!(error instanceof OpenAI.APIError) || error.status !==
|
|
145
|
+
if (!(error instanceof OpenAI.APIError) || error.status !== HttpStatus.ServiceUnavailable) {
|
|
371
146
|
return false;
|
|
147
|
+
}
|
|
372
148
|
const body = error.error;
|
|
373
|
-
return (body?.type ===
|
|
374
|
-
/loading model|model is loading|unavailable_error/i.test(`${error.message} ${body?.message ??
|
|
149
|
+
return (body?.type === UNAVAILABLE_ERROR ||
|
|
150
|
+
/loading model|model is loading|unavailable_error/i.test(`${error.message} ${body?.message ?? ''}`));
|
|
375
151
|
}
|
|
376
152
|
/** How long to wait between asking a loading server again. */
|
|
377
153
|
export const LOADING_POLL_MS = 3000;
|
|
378
154
|
/** How long `runTurn` waits for a model to load unless told otherwise. */
|
|
379
155
|
export const LOADING_TIMEOUT_MS = 120_000;
|
|
156
|
+
/** The first backoff, before jitter, which each attempt after it doubles. */
|
|
157
|
+
const BACKOFF_BASE_MS = 500;
|
|
158
|
+
/** The longest a backoff grows to, before jitter. */
|
|
159
|
+
const BACKOFF_CEILING_MS = 8000;
|
|
160
|
+
/** The least of a backoff that jitter leaves: a wait is somewhere between this share of it and all of it. */
|
|
161
|
+
const JITTER_FLOOR = 0.5;
|
|
380
162
|
/**
|
|
381
|
-
*
|
|
163
|
+
* How long to wait before a retry: exponential, with jitter so several tasks failing at once do
|
|
164
|
+
* not return in lockstep.
|
|
382
165
|
*
|
|
383
|
-
* @param attempt Zero-based. Doubles from 500ms to a ceiling of eight seconds, before jitter.
|
|
166
|
+
* @param attempt - Zero-based. Doubles from 500ms to a ceiling of eight seconds, before jitter.
|
|
167
|
+
* @returns Milliseconds, somewhere between half of that wait and all of it, and not a whole number.
|
|
384
168
|
*/
|
|
385
|
-
export const backoffMs = (attempt) => Math.min(
|
|
169
|
+
export const backoffMs = (attempt) => Math.min(BACKOFF_CEILING_MS, 2 ** attempt * BACKOFF_BASE_MS) * (JITTER_FLOOR + Math.random() * (1 - JITTER_FLOOR));
|
|
386
170
|
/**
|
|
387
171
|
* A delay an abort cuts short, rejecting rather than resolving early.
|
|
388
172
|
*
|
|
389
|
-
* @param ms How long to wait.
|
|
390
|
-
* @param signal Abandons the wait. One already aborted rejects without waiting at all.
|
|
173
|
+
* @param ms - How long to wait.
|
|
174
|
+
* @param [signal] - Abandons the wait. One already aborted rejects without waiting at all.
|
|
175
|
+
* @returns A promise that resolves once `ms` has passed, or rejects with the signal's reason — an
|
|
176
|
+
* `Error` of its own where the signal carries none.
|
|
391
177
|
*/
|
|
392
178
|
export const sleep = (ms, signal) => new Promise((resolve, reject) => {
|
|
393
179
|
const timer = setTimeout(() => {
|
|
394
|
-
signal?.removeEventListener(
|
|
180
|
+
signal?.removeEventListener(ABORT_EVENT, onAbort);
|
|
395
181
|
resolve();
|
|
396
182
|
}, ms);
|
|
397
183
|
const onAbort = () => {
|
|
398
184
|
clearTimeout(timer);
|
|
399
|
-
reject(signal?.reason ?? new Error(
|
|
185
|
+
reject(signal?.reason ?? new Error('aborted'));
|
|
400
186
|
};
|
|
401
|
-
if (signal?.aborted)
|
|
187
|
+
if (signal?.aborted) {
|
|
402
188
|
return onAbort();
|
|
403
|
-
|
|
189
|
+
}
|
|
190
|
+
signal?.addEventListener(ABORT_EVENT, onAbort, { once: true });
|
|
404
191
|
});
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import type OpenAI from 'openai';
|
|
2
|
+
import type { CatalogServer } from './catalog.ts';
|
|
3
|
+
import type { RunEventInput } from './events.ts';
|
|
4
|
+
import type { Turn } from './stream.ts';
|
|
5
|
+
import { type ToolCall, type ToolCallOutcome, type ToolCallRequest, type ToolCallResult } from './tool-calls.ts';
|
|
6
|
+
/**
|
|
7
|
+
* The tools between two turns: a step's calls read, run and written into the transcript.
|
|
8
|
+
*
|
|
9
|
+
* Apart from the loop because it is a different job with its own rules — a call is announced
|
|
10
|
+
* before it runs and answered whatever happens, `load_tools` and `call_tool` are answered here
|
|
11
|
+
* rather than dispatched, and a failure part-way has to leave a transcript an endpoint will take
|
|
12
|
+
* back — and none of them is about when to send the next request.
|
|
13
|
+
*/
|
|
14
|
+
/**
|
|
15
|
+
* A long tool argument or result cut to what a watcher needs, with the full length said.
|
|
16
|
+
*
|
|
17
|
+
* @param text - What to show.
|
|
18
|
+
* @param [limit] - Characters kept. Text at or under it comes back as it was.
|
|
19
|
+
* @returns The text to show. Cut, it ends in an ellipsis and the whole length, `… (N chars)`.
|
|
20
|
+
*
|
|
21
|
+
* @remarks
|
|
22
|
+
* For events, never for the transcript: the model reads the whole of what a tool returned.
|
|
23
|
+
*/
|
|
24
|
+
export declare const preview: (text: string, limit?: number) => string;
|
|
25
|
+
/** One of a turn's calls with its arguments read, or with the reason they could not be. */
|
|
26
|
+
export interface ReadCall {
|
|
27
|
+
call: ToolCall;
|
|
28
|
+
/** Absent where the arguments could not be read. */
|
|
29
|
+
args?: Record<string, unknown>;
|
|
30
|
+
/** Why they could not, which is then the call's answer. */
|
|
31
|
+
error?: unknown;
|
|
32
|
+
/** The arguments as every later request replays them: the repaired JSON, or `{}` for none. */
|
|
33
|
+
normal: string;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* A turn's calls with their arguments read.
|
|
37
|
+
*
|
|
38
|
+
* @param calls - The turn's calls, in the order the model made them.
|
|
39
|
+
* @param finishReason - Why the turn ended, which tells a call cut off from one written wrongly.
|
|
40
|
+
* @returns One entry per call, in the same order. Unreadable arguments do not throw: the entry
|
|
41
|
+
* carries the error in place of them.
|
|
42
|
+
*
|
|
43
|
+
* @remarks
|
|
44
|
+
* A server that parses replayed arguments refuses the almost-JSON a model wrote, so each is
|
|
45
|
+
* normalised here, and one that could not be read at all is replayed as no arguments.
|
|
46
|
+
*/
|
|
47
|
+
export declare const readCalls: (calls: ToolCall[], finishReason: Turn['finishReason']) => ReadCall[];
|
|
48
|
+
/**
|
|
49
|
+
* The assistant message a turn is written into the transcript as, its calls' arguments repaired.
|
|
50
|
+
*
|
|
51
|
+
* @param content - The turn's text. Empty is written as `null`.
|
|
52
|
+
* @param parsed - The turn's calls as `readCalls` read them. Not written to. None leaves
|
|
53
|
+
* `tool_calls` off the message rather than empty.
|
|
54
|
+
* @returns A new message, each call on it a copy with `normal` for its arguments.
|
|
55
|
+
*/
|
|
56
|
+
export declare const assistantMessage: (content: string, parsed: ReadCall[]) => OpenAI.ChatCompletionAssistantMessageParam;
|
|
57
|
+
/**
|
|
58
|
+
* The definitions the host gave for these names, in the order the names are given.
|
|
59
|
+
*
|
|
60
|
+
* @param definitions - The host's definitions by name.
|
|
61
|
+
* @param names - The names wanted. One the host gave no definition for is left out, and one given
|
|
62
|
+
* twice is answered twice.
|
|
63
|
+
* @returns A new array of the definitions themselves, not copies.
|
|
64
|
+
*/
|
|
65
|
+
export declare const definedAs: (definitions: ReadonlyMap<string, OpenAI.ChatCompletionTool>, names: Iterable<string>) => (OpenAI.ChatCompletionCustomTool | OpenAI.ChatCompletionFunctionTool)[];
|
|
66
|
+
/** Runs one tool call and resolves to what the model reads; what it throws, the model reads too. */
|
|
67
|
+
export type ToolDispatch = (call: ToolCallRequest, signal?: AbortSignal) => Promise<string>;
|
|
68
|
+
/**
|
|
69
|
+
* What running a step's calls reads of the run and writes back into it. One per run, shared by
|
|
70
|
+
* every step: the sets and counts here are the run's own, not copies.
|
|
71
|
+
*/
|
|
72
|
+
export interface Calling {
|
|
73
|
+
catalog: CatalogServer[];
|
|
74
|
+
/** Whether tools are loaded from the catalogue, behind a moving tool array or a fixed one. */
|
|
75
|
+
onDemand: boolean;
|
|
76
|
+
/** Whether the tool array is the fixed one, so a definition lives in the history. */
|
|
77
|
+
proxied: boolean;
|
|
78
|
+
/** Whether a `call_tool` is looked through, which it is unless the host has a tool by that name. */
|
|
79
|
+
proxies: boolean;
|
|
80
|
+
definitions: ReadonlyMap<string, OpenAI.ChatCompletionTool>;
|
|
81
|
+
loaded: Set<string>;
|
|
82
|
+
loads: {
|
|
83
|
+
toolsLoaded: number;
|
|
84
|
+
redundantLoads: number;
|
|
85
|
+
unknownToolNames: number;
|
|
86
|
+
};
|
|
87
|
+
used: Set<string>;
|
|
88
|
+
toolCalls: ToolCallOutcome[];
|
|
89
|
+
dedupable: (call: ToolCallRequest) => boolean;
|
|
90
|
+
dispatch: ToolDispatch;
|
|
91
|
+
parallel: boolean;
|
|
92
|
+
signal: AbortSignal | undefined;
|
|
93
|
+
onEvent: (input: RunEventInput) => void;
|
|
94
|
+
onToolCall: ((call: ToolCallRequest) => void) | undefined;
|
|
95
|
+
onToolResult: ((result: ToolCallResult) => void) | undefined;
|
|
96
|
+
/** Offers a message just written into the transcript to the host. */
|
|
97
|
+
announce: (message: OpenAI.ChatCompletionMessageParam, step: number) => Promise<void>;
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Runs a step's calls and writes each result into the transcript, in the order asked.
|
|
101
|
+
*
|
|
102
|
+
* @param run - The run the step belongs to.
|
|
103
|
+
* @param parsed - The step's calls, with their arguments read.
|
|
104
|
+
* @param messages - The transcript the results are written into.
|
|
105
|
+
* @param step - Which step this is, for the host told of each result.
|
|
106
|
+
*
|
|
107
|
+
* @remarks
|
|
108
|
+
* A failure part-way leaves the transcript well-formed before it is rethrown: a call with no
|
|
109
|
+
* result is a transcript no endpoint takes back, so every call of the step is answered — with what
|
|
110
|
+
* it returned where it had, and otherwise with a line saying it was stopped, or never run.
|
|
111
|
+
*/
|
|
112
|
+
export declare function runCalls(run: Calling, parsed: ReadCall[], messages: OpenAI.ChatCompletionMessageParam[], step: number): Promise<void>;
|
|
113
|
+
/**
|
|
114
|
+
* Writes a proxied preselection into the transcript as a `load_tools` exchange: the tool array is
|
|
115
|
+
* fixed, so a shortlist has nowhere else to go, and it is answered as though the model had loaded
|
|
116
|
+
* it. Told to the host as a call the model made would be, so one pairing calls with results by id
|
|
117
|
+
* shows this one like the rest.
|
|
118
|
+
*
|
|
119
|
+
* @param run - The run it opens.
|
|
120
|
+
* @param shortlist - The definitions preselected, each of them in the catalogue.
|
|
121
|
+
* @param messages - The transcript, which the exchange is appended to.
|
|
122
|
+
*/
|
|
123
|
+
export declare function loadShortlist(run: Calling, shortlist: OpenAI.ChatCompletionTool[], messages: OpenAI.ChatCompletionMessageParam[]): Promise<void>;
|