@tobi-or-not/quorum 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bench/fingerprint.d.ts +7 -5
- package/dist/bench/fingerprint.js +22 -9
- package/dist/bench/records.d.ts +70 -0
- package/dist/bench/records.js +30 -0
- package/dist/bench/runner.d.ts +18 -0
- package/dist/bench/runner.js +29 -3
- package/dist/cli/bench.d.ts +11 -4
- package/dist/cli/bench.js +34 -17
- package/dist/cli/bin.js +2 -2
- package/dist/cli/main.d.ts +34 -14
- package/dist/cli/main.js +99 -20
- package/dist/cli/print-config.d.ts +8 -6
- package/dist/cli/print-config.js +21 -8
- package/dist/cli/render.d.ts +2 -2
- package/dist/cli/render.js +21 -2
- package/dist/config/env.d.ts +32 -19
- package/dist/config/env.js +95 -32
- package/dist/config/from-config.js +15 -3
- package/dist/config/resolve.d.ts +9 -1
- package/dist/config/resolve.js +17 -1
- package/dist/config/schema.d.ts +141 -5
- package/dist/config/schema.js +125 -12
- package/dist/core/clock.d.ts +17 -0
- package/dist/core/clock.js +14 -0
- package/dist/core/coverage.d.ts +49 -0
- package/dist/core/coverage.js +144 -0
- package/dist/core/diff.d.ts +24 -0
- package/dist/core/diff.js +63 -18
- package/dist/core/errors.d.ts +12 -3
- package/dist/core/errors.js +21 -2
- package/dist/core/exclude.d.ts +42 -0
- package/dist/core/exclude.js +189 -0
- package/dist/core/limits.d.ts +82 -0
- package/dist/core/limits.js +163 -0
- package/dist/core/output.d.ts +3 -2
- package/dist/core/output.js +3 -2
- package/dist/core/ports.d.ts +57 -2
- package/dist/core/ports.js +36 -1
- package/dist/core/quorum.d.ts +24 -8
- package/dist/core/quorum.js +187 -56
- package/dist/core/retry.d.ts +36 -0
- package/dist/core/retry.js +46 -0
- package/dist/core/stages/context.d.ts +118 -23
- package/dist/core/stages/context.js +304 -46
- package/dist/core/stages/panel.d.ts +5 -4
- package/dist/core/stages/panel.js +30 -11
- package/dist/core/stages/research.d.ts +15 -6
- package/dist/core/stages/research.js +75 -18
- package/dist/core/tokens.d.ts +31 -0
- package/dist/core/tokens.js +81 -0
- package/dist/core/types.d.ts +77 -10
- package/dist/index.d.ts +1 -1
- package/dist/providers/anthropic.d.ts +2 -1
- package/dist/providers/anthropic.js +8 -3
- package/dist/providers/claude-code.d.ts +51 -11
- package/dist/providers/claude-code.js +60 -25
- package/dist/providers/gemini.d.ts +4 -1
- package/dist/providers/gemini.js +63 -11
- package/dist/providers/http.d.ts +49 -4
- package/dist/providers/http.js +155 -14
- package/dist/providers/ollama.d.ts +3 -1
- package/dist/providers/ollama.js +11 -5
- package/dist/providers/openai-compatible.d.ts +3 -1
- package/dist/providers/openai-compatible.js +11 -5
- package/dist/providers/options.d.ts +12 -3
- package/dist/providers/options.js +30 -10
- package/dist/providers/size.d.ts +10 -14
- package/dist/providers/size.js +15 -69
- package/package.json +1 -1
- package/schema/quorum.schema.json +34 -1
package/dist/providers/http.js
CHANGED
|
@@ -8,9 +8,20 @@ const MAX_ERROR_BODY = 500;
|
|
|
8
8
|
const MAX_ERROR_BYTES = 64 * 1024;
|
|
9
9
|
/** A reply body over this many bytes is refused: the core refuses reply text over 1,000,000 characters anyway. */
|
|
10
10
|
const MAX_REPLY_BYTES = 16 * 1024 * 1024;
|
|
11
|
+
/** The symbol under which Node keeps the dispatcher its `fetch` uses. */
|
|
12
|
+
const NODE_GLOBAL_DISPATCHER = Symbol.for('undici.globalDispatcher.1');
|
|
11
13
|
const REDACTED = '[redacted]';
|
|
12
14
|
const UNKNOWN_ENDPOINT = 'the endpoint';
|
|
13
15
|
const PLAIN_IDENTIFIER = /^[A-Z]\w{0,39}$/i;
|
|
16
|
+
/** A wait as a header writes it: decimal digits with an optional fraction. Never a sign, an exponent, a hexadecimal or a list. */
|
|
17
|
+
const DECIMAL = /^\d+(?:\.\d+)?$/;
|
|
18
|
+
/** The one date form read: an IMF-fixdate. Both engines read the others as local time, or `1` as a year. */
|
|
19
|
+
const IMF_FIXDATE = /^(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun), \d{2} (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{4} \d{2}:\d{2}:\d{2} GMT$/;
|
|
20
|
+
/** The codes of OpenAI that say a credit or a spend limit is used up: a retry cannot succeed, and would be billed. */
|
|
21
|
+
const SPEND_CODES = new Set(['credit_balance_exhausted', 'organization_usage_limit_exceeded', 'insufficient_quota']);
|
|
22
|
+
const SPEND_LIMIT_SUFFIX = '_spend_limit_exceeded';
|
|
23
|
+
const ANTHROPIC_SPEND_CODE = 'enforced_spend_limit_reached';
|
|
24
|
+
const OPENROUTER_IN_FLIGHT = 'openrouter_in_flight_budget';
|
|
14
25
|
/** A JSON unicode escape (a backslash, then u00 and two hex digits) for printable ASCII, U+0020 to U+007E. */
|
|
15
26
|
const ESCAPED_PRINTABLE_ASCII = /\\u00(2[0-9a-fA-F]|[3-6][0-9a-fA-F]|7[0-9a-eA-E])/g;
|
|
16
27
|
/** Every form in which a server may echo the secret. */
|
|
@@ -159,6 +170,29 @@ function codeSuffix(error) {
|
|
|
159
170
|
}
|
|
160
171
|
return '';
|
|
161
172
|
}
|
|
173
|
+
/**
|
|
174
|
+
* Node's global dispatcher handed each request with both of undici's timers off, the 300-second
|
|
175
|
+
* limits on the headers and on the body, or undefined when the runtime has none: Bun's `fetch`
|
|
176
|
+
* never reads a dispatcher, and its own timer goes off with `timeout: false`. The global is read at
|
|
177
|
+
* each request, never kept, so a proxy agent an operator set still carries it. The delegate calls
|
|
178
|
+
* the global's `dispatch` as its method, with the request's options and both timeouts at 0 (none),
|
|
179
|
+
* and the same handler.
|
|
180
|
+
*/
|
|
181
|
+
export function timersOff() {
|
|
182
|
+
// eslint-disable-next-line no-restricted-globals -- Node's fetch keeps its dispatcher on the global object, under this symbol alone.
|
|
183
|
+
const shared = globalThis[NODE_GLOBAL_DISPATCHER];
|
|
184
|
+
if (typeof shared !== 'object' || shared === null)
|
|
185
|
+
return undefined;
|
|
186
|
+
const { dispatch } = shared;
|
|
187
|
+
if (typeof dispatch !== 'function')
|
|
188
|
+
return undefined;
|
|
189
|
+
return {
|
|
190
|
+
dispatch: (options, handler) => Reflect.apply(dispatch, shared, [
|
|
191
|
+
{ ...options, headersTimeout: 0, bodyTimeout: 0 },
|
|
192
|
+
handler,
|
|
193
|
+
]),
|
|
194
|
+
};
|
|
195
|
+
}
|
|
162
196
|
/** At most `max` bytes of the body, decoded as UTF-8, and whether that was all of it. The rest is never read. */
|
|
163
197
|
async function readCapped(response, max) {
|
|
164
198
|
const reader = response.body?.getReader();
|
|
@@ -188,15 +222,119 @@ async function readErrorText(response) {
|
|
|
188
222
|
return '';
|
|
189
223
|
}
|
|
190
224
|
}
|
|
225
|
+
/** The number if it is positive and finite, else nothing: a hint of 0, a negative one or one that is no number is none. */
|
|
226
|
+
function positiveOrUndefined(value) {
|
|
227
|
+
return typeof value === 'number' && Number.isFinite(value) && value > 0
|
|
228
|
+
? value
|
|
229
|
+
: undefined;
|
|
230
|
+
}
|
|
231
|
+
/** `value` times `unit`, when it is written as decimal digits with an optional fraction. */
|
|
232
|
+
function scaled(value, unit) {
|
|
233
|
+
return value !== null && DECIMAL.test(value)
|
|
234
|
+
? positiveOrUndefined(Number(value) * unit)
|
|
235
|
+
: undefined;
|
|
236
|
+
}
|
|
237
|
+
/** The wait, in milliseconds, until an HTTP date written as an IMF-fixdate, given the wall clock `now`. */
|
|
238
|
+
function untilDate(value, now) {
|
|
239
|
+
return value !== null && IMF_FIXDATE.test(value)
|
|
240
|
+
? positiveOrUndefined(Date.parse(value) - now)
|
|
241
|
+
: undefined;
|
|
242
|
+
}
|
|
243
|
+
/**
|
|
244
|
+
* The wait a failed reply asks for, in milliseconds: `retry-after-ms`, then `retry-after` in seconds, then
|
|
245
|
+
* `retry-after` as an HTTP date less `now`, the wall clock in milliseconds. The first that is a positive
|
|
246
|
+
* finite number is the hint. A value is read as decimal digits with an optional fraction, or as an
|
|
247
|
+
* IMF-fixdate matched by its shape before `Date.parse` reads it, since the runtimes read a bare `1` or
|
|
248
|
+
* `120` as a year and the other date forms as local time. Anything else is no hint, and the header is
|
|
249
|
+
* never copied anywhere.
|
|
250
|
+
*/
|
|
251
|
+
export function retryAfterHint(headers, now) {
|
|
252
|
+
const after = headers.get('retry-after');
|
|
253
|
+
return scaled(headers.get('retry-after-ms'), 1) ?? scaled(after, 1000) ?? untilDate(after, now);
|
|
254
|
+
}
|
|
255
|
+
/** The own property `key` of `value` when it is an object, else nothing: the body is untrusted, and so is its prototype. */
|
|
256
|
+
function member(value, key) {
|
|
257
|
+
return typeof value === 'object' && value !== null && Object.hasOwn(value, key)
|
|
258
|
+
? value[key]
|
|
259
|
+
: undefined;
|
|
260
|
+
}
|
|
261
|
+
function isSpendCode(code) {
|
|
262
|
+
return typeof code === 'string' && (SPEND_CODES.has(code) || code.endsWith(SPEND_LIMIT_SUFFIX));
|
|
263
|
+
}
|
|
264
|
+
/**
|
|
265
|
+
* What the body of a failed reply says of a retry that the vendors' own codes settle: `false` for a used-up
|
|
266
|
+
* credit, spend limit or quota (OpenAI's `error.code`, or its `error.type` for `insufficient_quota`, and
|
|
267
|
+
* Anthropic's `error.details.error_code`), `true` for OpenRouter's 402 of its in-flight budget when the
|
|
268
|
+
* reply is `hinted` with a wait, else `undefined`, and the status decides. A code is only compared, never
|
|
269
|
+
* copied. This checks objects with a local test, since `options.ts` imports this module.
|
|
270
|
+
*/
|
|
271
|
+
export function vendorRetryable(body, status, hinted) {
|
|
272
|
+
const error = member(body, 'error');
|
|
273
|
+
if (isSpendCode(member(error, 'code')) || member(error, 'type') === 'insufficient_quota' || member(member(error, 'details'), 'error_code') === ANTHROPIC_SPEND_CODE)
|
|
274
|
+
return false;
|
|
275
|
+
if (status === 402 && hinted && member(member(error, 'metadata'), 'limit_source') === OPENROUTER_IN_FLIGHT)
|
|
276
|
+
return true;
|
|
277
|
+
return undefined;
|
|
278
|
+
}
|
|
279
|
+
/** The body as JSON, or nothing when it is none. */
|
|
280
|
+
function jsonOrUndefined(text) {
|
|
281
|
+
try {
|
|
282
|
+
return JSON.parse(text);
|
|
283
|
+
}
|
|
284
|
+
catch {
|
|
285
|
+
return undefined;
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
/** A failed reply's error: its message quotes the body, and its hint and verdict are numbers and booleans, never text. */
|
|
289
|
+
function httpFailure(options, response, text) {
|
|
290
|
+
const { providerId, redact } = options;
|
|
291
|
+
const { status } = response;
|
|
292
|
+
const body = jsonOrUndefined(text);
|
|
293
|
+
const extra = options.errorHints?.(body) ?? {};
|
|
294
|
+
const retryAfterMs = retryAfterHint(response.headers, Date.now()) ?? positiveOrUndefined(extra.retryAfterMs);
|
|
295
|
+
const said = response.headers.get('x-should-retry');
|
|
296
|
+
const verdicts = [
|
|
297
|
+
said === 'false'
|
|
298
|
+
? false
|
|
299
|
+
: said === 'true'
|
|
300
|
+
? true
|
|
301
|
+
: undefined,
|
|
302
|
+
vendorRetryable(body, status, retryAfterMs !== undefined),
|
|
303
|
+
extra.retryable,
|
|
304
|
+
];
|
|
305
|
+
const retryable = verdicts.includes(false)
|
|
306
|
+
? false
|
|
307
|
+
: verdicts.includes(true)
|
|
308
|
+
? true
|
|
309
|
+
: undefined;
|
|
310
|
+
const redirect = status >= 300 && status < 400
|
|
311
|
+
? ' (redirects are not followed)'
|
|
312
|
+
: '';
|
|
313
|
+
return new ProviderError({
|
|
314
|
+
providerId,
|
|
315
|
+
kind: 'http',
|
|
316
|
+
message: `HTTP ${status}${redirect}: ${snippet(text, MAX_ERROR_BODY, redact)}`,
|
|
317
|
+
status,
|
|
318
|
+
retryAfterMs,
|
|
319
|
+
retryable,
|
|
320
|
+
});
|
|
321
|
+
}
|
|
191
322
|
/**
|
|
192
323
|
* POST a JSON body and return the parsed JSON reply. A redirect is not followed: it is an `http` error,
|
|
193
|
-
* so a key sent in any header never reaches another endpoint.
|
|
194
|
-
*
|
|
195
|
-
*
|
|
196
|
-
* a
|
|
324
|
+
* so a key sent in any header never reaches another endpoint. The runtimes' own timers are off, so
|
|
325
|
+
* the core's deadline is the only one: a long reply of a thinking model is not cut at 300 seconds.
|
|
326
|
+
* A reply body over 16 MiB is refused unread. Every failure is a `ProviderError` without a `cause`:
|
|
327
|
+
* the runtime's own error text can hold the URL with its credentials or a header value, so it is
|
|
328
|
+
* never copied, and no header value appears in a message.
|
|
329
|
+
*
|
|
330
|
+
* A failed reply's error quotes the body in its message, as it always did, and carries what the reply
|
|
331
|
+
* says of a retry as a number and a boolean, never as text: `retryAfterMs` from `retry-after-ms` or
|
|
332
|
+
* `retry-after`, else from `errorHints`, and `retryable` when the reply settles it, which a "never" wins:
|
|
333
|
+
* `x-should-retry: false`, a used-up credit, spend limit or quota, or a hook's, else `x-should-retry: true`
|
|
334
|
+
* or OpenRouter's in-flight 402 with a hint, else the status decides.
|
|
197
335
|
*/
|
|
198
336
|
export async function postJson(options) {
|
|
199
|
-
const { providerId, url, headers, body, signal
|
|
337
|
+
const { providerId, url, headers, body, signal } = options;
|
|
200
338
|
const fail = (kind, message, status) => new ProviderError({ providerId, kind, message, status });
|
|
201
339
|
const lost = (what, error) => signal?.aborted
|
|
202
340
|
? fail('aborted', 'request aborted')
|
|
@@ -205,24 +343,27 @@ export async function postJson(options) {
|
|
|
205
343
|
const payload = JSON.stringify(body);
|
|
206
344
|
let response;
|
|
207
345
|
try {
|
|
208
|
-
|
|
346
|
+
// `timeout: false` turns Bun's idle timer off. Node reads `dispatcher` when its fetch reads the
|
|
347
|
+
// init, which is the first moment its global dispatcher is certain to exist, so it is a getter.
|
|
348
|
+
// undici types the option as its whole `Dispatcher` class, which the delegate is not.
|
|
349
|
+
const init = {
|
|
209
350
|
method: 'POST',
|
|
210
351
|
headers: { 'content-type': 'application/json', ...headers },
|
|
211
352
|
body: payload,
|
|
212
353
|
redirect: 'manual',
|
|
213
354
|
signal,
|
|
214
|
-
|
|
355
|
+
timeout: false,
|
|
356
|
+
get dispatcher() {
|
|
357
|
+
return timersOff();
|
|
358
|
+
},
|
|
359
|
+
};
|
|
360
|
+
response = await fetch(url, init);
|
|
215
361
|
}
|
|
216
362
|
catch (error) {
|
|
217
363
|
throw lost('request to', error);
|
|
218
364
|
}
|
|
219
|
-
if (!response.ok)
|
|
220
|
-
|
|
221
|
-
const redirect = status >= 300 && status < 400
|
|
222
|
-
? ' (redirects are not followed)'
|
|
223
|
-
: '';
|
|
224
|
-
throw fail('http', `HTTP ${status}${redirect}: ${snippet(await readErrorText(response), MAX_ERROR_BODY, redact)}`, status);
|
|
225
|
-
}
|
|
365
|
+
if (!response.ok)
|
|
366
|
+
throw httpFailure(options, response, await readErrorText(response));
|
|
226
367
|
let reply;
|
|
227
368
|
try {
|
|
228
369
|
reply = await readCapped(response, MAX_REPLY_BYTES);
|
|
@@ -10,7 +10,9 @@ export type OllamaOptions = HttpProviderOptions;
|
|
|
10
10
|
* stop as a truncation, instead of part of the prompt dropped so the reply can go on (Ollama 0.12.6 or
|
|
11
11
|
* later; an older server ignores both fields). It sends `contextTokens` as `options.num_ctx`, so the server
|
|
12
12
|
* loads the context the size guard counts with, and refuses, before any request, a request that may not
|
|
13
|
-
* fit it.
|
|
13
|
+
* fit it. It sends the reply's limit, `maxTokens` else the budget of the size guard, as `options.num_predict`,
|
|
14
|
+
* so the server bounds a reply's length. It states its worst case from the count its size guard keeps: the
|
|
15
|
+
* text it sends, the template reserve and that budget.
|
|
14
16
|
*
|
|
15
17
|
* @throws ConfigError with every problem of the options, before any request
|
|
16
18
|
*/
|
package/dist/providers/ollama.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { issuePath } from "../core/errors.js";
|
|
2
|
+
import { DEFAULT_RETRIES, DEFAULT_TIMEOUT_MS } from "../core/ports.js";
|
|
2
3
|
import { postJson, snippet } from "./http.js";
|
|
3
4
|
import { shapeMessages } from "./messages.js";
|
|
4
5
|
import { bodyKeyForm, isRecord, mergeBody, prepareHttp, spellingIssues } from "./options.js";
|
|
5
6
|
import { MAX_MODEL_TEXT, replyFailure, tokenUsage } from "./reply.js";
|
|
6
|
-
import { checkFits, contextBudget } from "./size.js";
|
|
7
|
+
import { checkFits, contextBudget, worstCaseTokens } from "./size.js";
|
|
7
8
|
/**
|
|
8
9
|
* The protocol's rules. The tool fields: `tools`, its only one, and `_debug_render_only`, which answers
|
|
9
10
|
* 200 with the rendered template instead of a reply.
|
|
@@ -98,21 +99,26 @@ function readReply(reply, { id, model }, redact) {
|
|
|
98
99
|
* stop as a truncation, instead of part of the prompt dropped so the reply can go on (Ollama 0.12.6 or
|
|
99
100
|
* later; an older server ignores both fields). It sends `contextTokens` as `options.num_ctx`, so the server
|
|
100
101
|
* loads the context the size guard counts with, and refuses, before any request, a request that may not
|
|
101
|
-
* fit it.
|
|
102
|
+
* fit it. It sends the reply's limit, `maxTokens` else the budget of the size guard, as `options.num_predict`,
|
|
103
|
+
* so the server bounds a reply's length. It states its worst case from the count its size guard keeps: the
|
|
104
|
+
* text it sends, the template reserve and that budget.
|
|
102
105
|
*
|
|
103
106
|
* @throws ConfigError with every problem of the options, before any request
|
|
104
107
|
*/
|
|
105
108
|
export function ollama(options) {
|
|
106
|
-
const { id, model, auth = 'bearer', temperature = 0, maxTokens, contextTokens, systemPrompt = 'native', structuredOutput = 'json' } = options;
|
|
109
|
+
const { id, model, auth = 'bearer', temperature = 0, maxTokens, contextTokens, systemPrompt = 'native', structuredOutput = 'json', timeoutMs = DEFAULT_TIMEOUT_MS, retries = DEFAULT_RETRIES } = options;
|
|
107
110
|
const { url, headers, redact, extraBody } = prepareHttp(options, { ...RULES, auth }, '/api/chat', numCtxIssues(id, options.extraBody));
|
|
108
111
|
const budget = contextBudget(contextTokens, maxTokens);
|
|
109
112
|
return {
|
|
110
113
|
id,
|
|
111
114
|
model,
|
|
115
|
+
timeoutMs,
|
|
116
|
+
retries,
|
|
117
|
+
worstCase: request => worstCaseTokens(shapeMessages(request, systemPrompt), request.output, budget),
|
|
112
118
|
async complete(request) {
|
|
113
119
|
const { system, messages } = shapeMessages(request, systemPrompt);
|
|
114
120
|
checkFits(id, { system, messages }, request.output, budget);
|
|
115
|
-
// JSON.stringify drops the undefined fields: no format for `prompt`, no temperature for null
|
|
121
|
+
// JSON.stringify drops the undefined fields: no format for `prompt`, no temperature for null.
|
|
116
122
|
const reply = await postJson({
|
|
117
123
|
providerId: id,
|
|
118
124
|
url,
|
|
@@ -128,7 +134,7 @@ export function ollama(options) {
|
|
|
128
134
|
stream: false,
|
|
129
135
|
truncate: false,
|
|
130
136
|
shift: false,
|
|
131
|
-
options: { temperature: temperature ?? undefined, num_predict:
|
|
137
|
+
options: { temperature: temperature ?? undefined, num_predict: budget.replyTokens, num_ctx: contextTokens },
|
|
132
138
|
format: format(structuredOutput, request.output),
|
|
133
139
|
}, extraBody),
|
|
134
140
|
signal: request.signal,
|
|
@@ -8,7 +8,9 @@ export interface OpenAICompatibleOptions extends HttpProviderOptions {
|
|
|
8
8
|
/**
|
|
9
9
|
* A provider for any OpenAI-compatible `/chat/completions` endpoint. The key, when there is one, travels
|
|
10
10
|
* only in the header `auth` names, `Authorization: Bearer` by default (invariant I1). It refuses, before
|
|
11
|
-
* any request, a request that may not fit `contextTokens`.
|
|
11
|
+
* any request, a request that may not fit `contextTokens`. It sends the reply's limit, `maxTokens` else the
|
|
12
|
+
* budget of the size guard, under `maxTokensField`, so the server bounds a reply's length. It states its
|
|
13
|
+
* worst case from the count its size guard keeps: the text it sends, the template reserve and that budget.
|
|
12
14
|
*
|
|
13
15
|
* @throws ConfigError with every problem of the options, before any request
|
|
14
16
|
*/
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
+
import { DEFAULT_RETRIES, DEFAULT_TIMEOUT_MS } from "../core/ports.js";
|
|
1
2
|
import { postJson, snippet } from "./http.js";
|
|
2
3
|
import { shapeMessages } from "./messages.js";
|
|
3
4
|
import { isRecord, mergeBody, prepareHttp } from "./options.js";
|
|
4
5
|
import { endedEarly, joinedText, MAX_MODEL_TEXT, replyFailure, tokenUsage } from "./reply.js";
|
|
5
|
-
import { checkFits, contextBudget } from "./size.js";
|
|
6
|
+
import { checkFits, contextBudget, worstCaseTokens } from "./size.js";
|
|
6
7
|
/**
|
|
7
8
|
* The protocol's rules. The tool fields are the union over the OpenAI-compatible vendors: OpenAI's and
|
|
8
9
|
* the others' function tools and web search, OpenRouter's plugins, server-tool loop, presets and model
|
|
@@ -145,12 +146,14 @@ function readReply(reply, { id, model }, redact) {
|
|
|
145
146
|
/**
|
|
146
147
|
* A provider for any OpenAI-compatible `/chat/completions` endpoint. The key, when there is one, travels
|
|
147
148
|
* only in the header `auth` names, `Authorization: Bearer` by default (invariant I1). It refuses, before
|
|
148
|
-
* any request, a request that may not fit `contextTokens`.
|
|
149
|
+
* any request, a request that may not fit `contextTokens`. It sends the reply's limit, `maxTokens` else the
|
|
150
|
+
* budget of the size guard, under `maxTokensField`, so the server bounds a reply's length. It states its
|
|
151
|
+
* worst case from the count its size guard keeps: the text it sends, the template reserve and that budget.
|
|
149
152
|
*
|
|
150
153
|
* @throws ConfigError with every problem of the options, before any request
|
|
151
154
|
*/
|
|
152
155
|
export function openaiCompatible(options) {
|
|
153
|
-
const { id, model, auth = 'bearer', temperature = 0, maxTokens, contextTokens, maxTokensField = 'max_tokens', systemPrompt = 'native', structuredOutput = 'json' } = options;
|
|
156
|
+
const { id, model, auth = 'bearer', temperature = 0, maxTokens, contextTokens, maxTokensField = 'max_tokens', systemPrompt = 'native', structuredOutput = 'json', timeoutMs = DEFAULT_TIMEOUT_MS, retries = DEFAULT_RETRIES } = options;
|
|
154
157
|
const fieldIssues = MAX_TOKENS_FIELDS.has(maxTokensField)
|
|
155
158
|
? []
|
|
156
159
|
: [
|
|
@@ -161,10 +164,13 @@ export function openaiCompatible(options) {
|
|
|
161
164
|
return {
|
|
162
165
|
id,
|
|
163
166
|
model,
|
|
167
|
+
timeoutMs,
|
|
168
|
+
retries,
|
|
169
|
+
worstCase: request => worstCaseTokens(shapeMessages(request, systemPrompt), request.output, budget),
|
|
164
170
|
async complete(request) {
|
|
165
171
|
const { system, messages } = shapeMessages(request, systemPrompt);
|
|
166
172
|
checkFits(id, { system, messages }, request.output, budget);
|
|
167
|
-
// JSON.stringify drops the undefined fields: no response_format for `prompt`, no temperature for null
|
|
173
|
+
// JSON.stringify drops the undefined fields: no response_format for `prompt`, no temperature for null.
|
|
168
174
|
const reply = await postJson({
|
|
169
175
|
providerId: id,
|
|
170
176
|
url,
|
|
@@ -179,7 +185,7 @@ export function openaiCompatible(options) {
|
|
|
179
185
|
],
|
|
180
186
|
response_format: responseFormat(structuredOutput, request.output),
|
|
181
187
|
temperature: temperature ?? undefined,
|
|
182
|
-
[maxTokensField]:
|
|
188
|
+
[maxTokensField]: budget.replyTokens,
|
|
183
189
|
}, extraBody),
|
|
184
190
|
signal: request.signal,
|
|
185
191
|
redact,
|
|
@@ -44,10 +44,14 @@ export interface HttpProviderOptions {
|
|
|
44
44
|
extraBody?: Readonly<Record<string, unknown>>;
|
|
45
45
|
/** `null` leaves the field out, for models that reject it. */
|
|
46
46
|
temperature?: number | null;
|
|
47
|
-
/** The most tokens the reply may hold. */
|
|
47
|
+
/** The most tokens the reply may hold, thinking included, sent as the server's limit. Default 16000. */
|
|
48
48
|
maxTokens?: number;
|
|
49
49
|
systemPrompt?: SystemPrompt;
|
|
50
50
|
structuredOutput?: StructuredOutput;
|
|
51
|
+
/** The deadline of one attempt, in milliseconds, from 1000 to 86400000. Default 600000. */
|
|
52
|
+
timeoutMs?: number;
|
|
53
|
+
/** The retries of a call that failed in passing, from 0 to 5. Default 2. */
|
|
54
|
+
retries?: number;
|
|
51
55
|
}
|
|
52
56
|
/** What an adapter declares of its protocol, which the shared checks hold the options to. */
|
|
53
57
|
export interface AdapterRules {
|
|
@@ -97,6 +101,8 @@ export declare function bodyKeyForm(key: string): string;
|
|
|
97
101
|
export declare function spellingIssues(id: string, key: string, canonical: string): ConfigIssue[];
|
|
98
102
|
/** The words of a list as a message names them: commas between them, and the conjunction before the last. */
|
|
99
103
|
export declare function wordList(words: readonly string[], conjunction?: 'or' | 'and'): string;
|
|
104
|
+
/** The problem of `maxTokens`, the reply's limit, when it is set and is not a whole number of at least 1. */
|
|
105
|
+
export declare function maxTokensIssues(id: string, maxTokens: unknown): ConfigIssue[];
|
|
100
106
|
/**
|
|
101
107
|
* The problems of `contextTokens`: not set, not a whole number of at least 1, or no more than the reply budget
|
|
102
108
|
* and the template reserve (`TEMPLATE_TOKENS` unless the adapter names one), which leaves no room for a
|
|
@@ -104,6 +110,11 @@ export declare function wordList(words: readonly string[], conjunction?: 'or' |
|
|
|
104
110
|
* one has its own issue.
|
|
105
111
|
*/
|
|
106
112
|
export declare function contextTokensIssues(id: string, contextTokens: number | undefined, maxTokens: number | undefined, templateTokens?: number): ConfigIssue[];
|
|
113
|
+
/**
|
|
114
|
+
* The problems of the deadline and the retries an adapter takes, each at `<id>.<field>` when it is set
|
|
115
|
+
* and out of range: the deadline first. No message repeats a value.
|
|
116
|
+
*/
|
|
117
|
+
export declare function attemptIssues(id: string, timeoutMs: unknown, retries: unknown): ConfigIssue[];
|
|
107
118
|
/**
|
|
108
119
|
* The problems of the options every HTTP adapter shares, under the rules the adapter declares. No
|
|
109
120
|
* message repeats a value: a key, a header value or a URL can be a secret.
|
|
@@ -121,7 +132,5 @@ export declare function prepareHttp(options: HttpProviderOptions, rules: Adapter
|
|
|
121
132
|
export declare function requestHeaders(auth: Auth, apiKey: string | undefined, headers?: Readonly<Record<string, string>>): Record<string, string>;
|
|
122
133
|
/** `body` with `extra` deep-merged into it: objects merge key by key, anything else, arrays included, replaces. */
|
|
123
134
|
export declare function mergeBody(body: Record<string, unknown>, extra: Readonly<Record<string, unknown>> | undefined): Record<string, unknown>;
|
|
124
|
-
/** `fields`, or undefined when every one of them is undefined, so the body leaves out an object that would be empty. */
|
|
125
|
-
export declare function unlessEmpty(fields: Record<string, unknown>): Record<string, unknown> | undefined;
|
|
126
135
|
/** Whether `value` is a non-null object that is not an array. */
|
|
127
136
|
export declare function isRecord(value: unknown): value is Record<string, unknown>;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { ConfigError, issuePath } from "../core/errors.js";
|
|
2
|
+
import { retriesProblem, timeoutProblem } from "../core/ports.js";
|
|
2
3
|
import { endpointUrl, redactionsOf } from "./http.js";
|
|
3
4
|
import { DEFAULT_REPLY_TOKENS, TEMPLATE_TOKENS } from "./size.js";
|
|
4
5
|
const HTTP_URL_WITH_HOST = /^https?:\/\/[^/\\?#\s]/i;
|
|
@@ -209,6 +210,14 @@ export function wordList(words, conjunction = 'or') {
|
|
|
209
210
|
function isTokenCount(value) {
|
|
210
211
|
return Number.isSafeInteger(value) && value >= 1;
|
|
211
212
|
}
|
|
213
|
+
/** The problem of `maxTokens`, the reply's limit, when it is set and is not a whole number of at least 1. */
|
|
214
|
+
export function maxTokensIssues(id, maxTokens) {
|
|
215
|
+
if (maxTokens === undefined || (typeof maxTokens === 'number' && isTokenCount(maxTokens)))
|
|
216
|
+
return [];
|
|
217
|
+
return [
|
|
218
|
+
{ path: `${id}.maxTokens`, message: 'must be a whole number of at least 1' },
|
|
219
|
+
];
|
|
220
|
+
}
|
|
212
221
|
/**
|
|
213
222
|
* The problems of `contextTokens`: not set, not a whole number of at least 1, or no more than the reply budget
|
|
214
223
|
* and the template reserve (`TEMPLATE_TOKENS` unless the adapter names one), which leaves no room for a
|
|
@@ -236,12 +245,30 @@ export function contextTokensIssues(id, contextTokens, maxTokens, templateTokens
|
|
|
236
245
|
{ path, message: `must be more than ${reply + templateTokens}, the ${reply} tokens of the reply budget and ${templateTokens} for the chat template` },
|
|
237
246
|
];
|
|
238
247
|
}
|
|
248
|
+
/**
|
|
249
|
+
* The problems of the deadline and the retries an adapter takes, each at `<id>.<field>` when it is set
|
|
250
|
+
* and out of range: the deadline first. No message repeats a value.
|
|
251
|
+
*/
|
|
252
|
+
export function attemptIssues(id, timeoutMs, retries) {
|
|
253
|
+
const issues = [];
|
|
254
|
+
const deadline = timeoutMs === undefined
|
|
255
|
+
? undefined
|
|
256
|
+
: timeoutProblem(timeoutMs);
|
|
257
|
+
if (deadline !== undefined)
|
|
258
|
+
issues.push({ path: `${id}.timeoutMs`, message: deadline });
|
|
259
|
+
const count = retries === undefined
|
|
260
|
+
? undefined
|
|
261
|
+
: retriesProblem(retries);
|
|
262
|
+
if (count !== undefined)
|
|
263
|
+
issues.push({ path: `${id}.retries`, message: count });
|
|
264
|
+
return issues;
|
|
265
|
+
}
|
|
239
266
|
/**
|
|
240
267
|
* The problems of the options every HTTP adapter shares, under the rules the adapter declares. No
|
|
241
268
|
* message repeats a value: a key, a header value or a URL can be a secret.
|
|
242
269
|
*/
|
|
243
270
|
export function httpOptionIssues(options, rules) {
|
|
244
|
-
const { id, baseUrl, apiKey, headers, query, extraBody, temperature, maxTokens, contextTokens, systemPrompt, structuredOutput } = options;
|
|
271
|
+
const { id, baseUrl, apiKey, headers, query, extraBody, temperature, maxTokens, contextTokens, systemPrompt, structuredOutput, timeoutMs, retries } = options;
|
|
245
272
|
const { auth, structuredOutputs } = rules;
|
|
246
273
|
const fixedHeaders = rules.fixedHeaders.map(name => name.toLowerCase());
|
|
247
274
|
const issues = [
|
|
@@ -275,15 +302,14 @@ export function httpOptionIssues(options, rules) {
|
|
|
275
302
|
if (temperature !== undefined && temperature !== null && !Number.isFinite(temperature)) {
|
|
276
303
|
issues.push({ path: `${id}.temperature`, message: 'must be a finite number, or null to leave it out' });
|
|
277
304
|
}
|
|
278
|
-
|
|
279
|
-
issues.push({ path: `${id}.maxTokens`, message: 'must be a whole number of at least 1' });
|
|
280
|
-
}
|
|
305
|
+
issues.push(...maxTokensIssues(id, maxTokens));
|
|
281
306
|
issues.push(...contextTokensIssues(id, contextTokens, maxTokens));
|
|
282
307
|
if (systemPrompt !== undefined && !SYSTEM_PROMPTS.has(systemPrompt))
|
|
283
308
|
issues.push({ path: `${id}.systemPrompt`, message: 'must be native or inline' });
|
|
284
309
|
if (structuredOutput !== undefined && !structuredOutputs.includes(structuredOutput)) {
|
|
285
310
|
issues.push({ path: `${id}.structuredOutput`, message: `must be ${wordList(structuredOutputs)}` });
|
|
286
311
|
}
|
|
312
|
+
issues.push(...attemptIssues(id, timeoutMs, retries));
|
|
287
313
|
return issues;
|
|
288
314
|
}
|
|
289
315
|
/**
|
|
@@ -339,12 +365,6 @@ export function mergeBody(body, extra) {
|
|
|
339
365
|
}
|
|
340
366
|
return merged;
|
|
341
367
|
}
|
|
342
|
-
/** `fields`, or undefined when every one of them is undefined, so the body leaves out an object that would be empty. */
|
|
343
|
-
export function unlessEmpty(fields) {
|
|
344
|
-
return Object.values(fields).every(value => value === undefined)
|
|
345
|
-
? undefined
|
|
346
|
-
: fields;
|
|
347
|
-
}
|
|
348
368
|
/** Whether `value` is a non-null object that is not an array. */
|
|
349
369
|
export function isRecord(value) {
|
|
350
370
|
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
package/dist/providers/size.d.ts
CHANGED
|
@@ -1,16 +1,6 @@
|
|
|
1
1
|
import type { CompletionRequest } from '../core/ports.ts';
|
|
2
2
|
import type { ShapedMessages } from './messages.ts';
|
|
3
|
-
|
|
4
|
-
* Tokens kept for what a server adds around the messages: the chat template's role markers, a thinking
|
|
5
|
-
* or Harmony preamble, a tokenizer's dummy prefix. Measured at 10 to 29 tokens for Llama 3 and Qwen 2.5;
|
|
6
|
-
* the margin is many times that, since a preamble can be longer than any measured.
|
|
7
|
-
*/
|
|
8
|
-
export declare const TEMPLATE_TOKENS = 512;
|
|
9
|
-
/**
|
|
10
|
-
* Tokens kept for the reply when `maxTokens` is not set: the limit `anthropic` sends then, and room for a
|
|
11
|
-
* thinking model's longest reply. Every other adapter sends no limit, so there it is room the guard keeps.
|
|
12
|
-
*/
|
|
13
|
-
export declare const DEFAULT_REPLY_TOKENS = 16000;
|
|
3
|
+
export { DEFAULT_REPLY_TOKENS, TEMPLATE_TOKENS, utf8Length } from '../core/tokens.ts';
|
|
14
4
|
/** The model's context window, the tokens a reply may take of it, and the tokens kept for what is added around the messages. */
|
|
15
5
|
export interface ContextBudget {
|
|
16
6
|
readonly contextTokens: number;
|
|
@@ -24,8 +14,6 @@ export interface ContextBudget {
|
|
|
24
14
|
* chat template raises.
|
|
25
15
|
*/
|
|
26
16
|
export declare function contextBudget(contextTokens: number, maxTokens: number | undefined, templateTokens?: number): ContextBudget;
|
|
27
|
-
/** The length of `text` in UTF-8 bytes. A lone surrogate counts the 3 bytes of U+FFFD, as the encoder writes it. */
|
|
28
|
-
export declare function utf8Length(text: string): number;
|
|
29
17
|
/**
|
|
30
18
|
* A bound of the tokens in the text of a request: one per UTF-8 byte of the system text, of every message,
|
|
31
19
|
* and of the reply schema's JSON, each counted at the sum of its code points' longest forms. One token per
|
|
@@ -35,9 +23,17 @@ export declare function utf8Length(text: string): number;
|
|
|
35
23
|
* the count is not the raw bytes. One that folds case is not covered; no chat model's tokenizer is known
|
|
36
24
|
* to. The price is that accented letters count one more byte per accent and Hangul syllables up to three
|
|
37
25
|
* times their size. The schema counts once more than the body carries it, since a server may add it to the
|
|
38
|
-
* prompt.
|
|
26
|
+
* prompt. The core counts a request's text with the same bound (`requestTokens`), so the worst case an adapter
|
|
27
|
+
* states and the core's own count never drift.
|
|
39
28
|
*/
|
|
40
29
|
export declare function textTokens(shaped: ShapedMessages, output: CompletionRequest['output']): number;
|
|
30
|
+
/**
|
|
31
|
+
* The most tokens, input and output, that one attempt of a request may spend as the provider is billed for
|
|
32
|
+
* it, from what the guard counts: each of `turns` model turns sends the text and the template reserve, and
|
|
33
|
+
* the reply of every turn but the last is the input of the next, so the replies count `2 * turns - 1` times.
|
|
34
|
+
* One turn is the text, the template reserve and the reply budget, which `checkFits` keeps within the context.
|
|
35
|
+
*/
|
|
36
|
+
export declare function worstCaseTokens(shaped: ShapedMessages, output: CompletionRequest['output'], budget: ContextBudget, turns?: number): number;
|
|
41
37
|
/**
|
|
42
38
|
* Refuse, before it is sent, a request that may not fit the model's context: some endpoints cut an oversized
|
|
43
39
|
* prompt in silence and answer 200, which would leave a review of text the model never read. The request
|
package/dist/providers/size.js
CHANGED
|
@@ -1,16 +1,6 @@
|
|
|
1
1
|
import { ProviderError } from "../core/errors.js";
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
* or Harmony preamble, a tokenizer's dummy prefix. Measured at 10 to 29 tokens for Llama 3 and Qwen 2.5;
|
|
5
|
-
* the margin is many times that, since a preamble can be longer than any measured.
|
|
6
|
-
*/
|
|
7
|
-
export const TEMPLATE_TOKENS = 512;
|
|
8
|
-
/**
|
|
9
|
-
* Tokens kept for the reply when `maxTokens` is not set: the limit `anthropic` sends then, and room for a
|
|
10
|
-
* thinking model's longest reply. Every other adapter sends no limit, so there it is room the guard keeps.
|
|
11
|
-
*/
|
|
12
|
-
export const DEFAULT_REPLY_TOKENS = 16_000;
|
|
13
|
-
const encoder = new TextEncoder();
|
|
2
|
+
import { DEFAULT_REPLY_TOKENS, TEMPLATE_TOKENS, tokenBound } from "../core/tokens.js";
|
|
3
|
+
export { DEFAULT_REPLY_TOKENS, TEMPLATE_TOKENS, utf8Length } from "../core/tokens.js";
|
|
14
4
|
/**
|
|
15
5
|
* The budget a request is checked against: the context size, the reply budget, `maxTokens` else
|
|
16
6
|
* `DEFAULT_REPLY_TOKENS`, and the template reserve, which an adapter whose server adds more context than a
|
|
@@ -19,61 +9,6 @@ const encoder = new TextEncoder();
|
|
|
19
9
|
export function contextBudget(contextTokens, maxTokens, templateTokens = TEMPLATE_TOKENS) {
|
|
20
10
|
return { contextTokens, replyTokens: maxTokens ?? DEFAULT_REPLY_TOKENS, templateTokens };
|
|
21
11
|
}
|
|
22
|
-
/** The length of `text` in UTF-8 bytes. A lone surrogate counts the 3 bytes of U+FFFD, as the encoder writes it. */
|
|
23
|
-
export function utf8Length(text) {
|
|
24
|
-
return encoder.encode(text).length;
|
|
25
|
-
}
|
|
26
|
-
/**
|
|
27
|
-
* The bytes of the longest form of each code point, 0 until it is first seen, so a repeated character is
|
|
28
|
-
* normalized once. A fixed table of 1.1 MB, where a map grew past 200 MB on a text holding every code
|
|
29
|
-
* point; the longest form, U+FDFA's, is 33 bytes, so a byte holds any of them.
|
|
30
|
-
*/
|
|
31
|
-
const longestFormBytes = new Uint8Array(0x110000);
|
|
32
|
-
/**
|
|
33
|
-
* The most UTF-8 bytes a code point takes as sent, decomposed (NFD) or compatibility-decomposed (NFKD): U+0F81
|
|
34
|
-
* is 3 bytes and 6 decomposed, U+00E9 is 2 and 3, a Hangul syllable up to 3 times its size, U+FDFA is 3 and 33.
|
|
35
|
-
* A lone surrogate is 3, the bytes of the U+FFFD the encoder writes for it.
|
|
36
|
-
*/
|
|
37
|
-
function longestFormLength(codePoint) {
|
|
38
|
-
if (codePoint >= 0xD800 && codePoint <= 0xDFFF)
|
|
39
|
-
return 3;
|
|
40
|
-
const cached = longestFormBytes[codePoint];
|
|
41
|
-
if (cached !== undefined && cached !== 0)
|
|
42
|
-
return cached;
|
|
43
|
-
const char = String.fromCodePoint(codePoint);
|
|
44
|
-
const bytes = Math.max(utf8Length(char), utf8Length(char.normalize('NFD')), utf8Length(char.normalize('NFKD')));
|
|
45
|
-
longestFormBytes[codePoint] = bytes;
|
|
46
|
-
return bytes;
|
|
47
|
-
}
|
|
48
|
-
/**
|
|
49
|
-
* A bound of the UTF-8 bytes of `text` in any normalization: the sum over its code points of each one's
|
|
50
|
-
* longest form. The text itself is never normalized, since that takes quadratic time on a long run of
|
|
51
|
-
* combining marks (canonical reordering) and a gigabyte on a run of U+FDFA. It is a bound because
|
|
52
|
-
* decomposition works on one code point at a time, reordering only permutes code points, and composing
|
|
53
|
-
* never lengthens UTF-8 (true of every code point, which `size.spec.ts` checks), so no NFC, NFKC, NFD or
|
|
54
|
-
* NFKD form of the whole text is longer. Normalizing each character as NFC, or the text in chunks, is not
|
|
55
|
-
* a bound: a mark that composes first, or one that crosses a chunk, can lengthen the whole.
|
|
56
|
-
*/
|
|
57
|
-
function normalizedLength(text) {
|
|
58
|
-
let bytes = 0;
|
|
59
|
-
for (let index = 0; index < text.length; index++) {
|
|
60
|
-
const unit = text.charCodeAt(index);
|
|
61
|
-
if (unit < 0x80) {
|
|
62
|
-
bytes += 1;
|
|
63
|
-
continue;
|
|
64
|
-
}
|
|
65
|
-
const low = index + 1 < text.length
|
|
66
|
-
? text.charCodeAt(index + 1)
|
|
67
|
-
: 0;
|
|
68
|
-
const pair = unit >= 0xD800 && unit <= 0xDBFF && low >= 0xDC00 && low <= 0xDFFF;
|
|
69
|
-
bytes += longestFormLength(pair
|
|
70
|
-
? 0x10000 + ((unit - 0xD800) << 10) + (low - 0xDC00)
|
|
71
|
-
: unit);
|
|
72
|
-
if (pair)
|
|
73
|
-
index++;
|
|
74
|
-
}
|
|
75
|
-
return bytes;
|
|
76
|
-
}
|
|
77
12
|
/**
|
|
78
13
|
* A bound of the tokens in the text of a request: one per UTF-8 byte of the system text, of every message,
|
|
79
14
|
* and of the reply schema's JSON, each counted at the sum of its code points' longest forms. One token per
|
|
@@ -83,7 +18,8 @@ function normalizedLength(text) {
|
|
|
83
18
|
* the count is not the raw bytes. One that folds case is not covered; no chat model's tokenizer is known
|
|
84
19
|
* to. The price is that accented letters count one more byte per accent and Hangul syllables up to three
|
|
85
20
|
* times their size. The schema counts once more than the body carries it, since a server may add it to the
|
|
86
|
-
* prompt.
|
|
21
|
+
* prompt. The core counts a request's text with the same bound (`requestTokens`), so the worst case an adapter
|
|
22
|
+
* states and the core's own count never drift.
|
|
87
23
|
*/
|
|
88
24
|
export function textTokens(shaped, output) {
|
|
89
25
|
const texts = [
|
|
@@ -91,7 +27,17 @@ export function textTokens(shaped, output) {
|
|
|
91
27
|
...shaped.messages.map(message => message.content),
|
|
92
28
|
JSON.stringify(output.schema),
|
|
93
29
|
];
|
|
94
|
-
return texts.reduce((sum, text) => sum +
|
|
30
|
+
return texts.reduce((sum, text) => sum + tokenBound(text), 0);
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* The most tokens, input and output, that one attempt of a request may spend as the provider is billed for
|
|
34
|
+
* it, from what the guard counts: each of `turns` model turns sends the text and the template reserve, and
|
|
35
|
+
* the reply of every turn but the last is the input of the next, so the replies count `2 * turns - 1` times.
|
|
36
|
+
* One turn is the text, the template reserve and the reply budget, which `checkFits` keeps within the context.
|
|
37
|
+
*/
|
|
38
|
+
export function worstCaseTokens(shaped, output, budget, turns = 1) {
|
|
39
|
+
const template = budget.templateTokens ?? TEMPLATE_TOKENS;
|
|
40
|
+
return turns * (textTokens(shaped, output) + template) + (2 * turns - 1) * budget.replyTokens;
|
|
95
41
|
}
|
|
96
42
|
/**
|
|
97
43
|
* Refuse, before it is sent, a request that may not fit the model's context: some endpoints cut an oversized
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tobi-or-not/quorum",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.5.0",
|
|
5
5
|
"packageManager": "bun@1.4.4",
|
|
6
6
|
"description": "Multi-agent code review for diffs: a model proposes candidate findings, then a panel of blind voters judges each before it's reported.",
|
|
7
7
|
"license": "Apache-2.0",
|