@cubicecho/agent-core 2.23.0 → 2.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -9
- package/dist/agent-loop.d.ts +84 -204
- package/dist/agent-loop.js +416 -643
- package/dist/calibration.d.ts +14 -10
- package/dist/calibration.js +30 -22
- package/dist/capabilities.d.ts +140 -35
- package/dist/capabilities.js +345 -152
- package/dist/client.d.ts +104 -35
- package/dist/client.js +210 -78
- package/dist/compaction.d.ts +103 -60
- package/dist/compaction.js +163 -93
- package/dist/config.d.ts +48 -12
- package/dist/config.js +20 -1
- package/dist/continuation.d.ts +25 -17
- package/dist/continuation.js +70 -47
- package/dist/digest.d.ts +7 -0
- package/dist/digest.js +18 -0
- package/dist/errors.d.ts +24 -15
- package/dist/errors.js +23 -15
- package/dist/events.d.ts +108 -58
- package/dist/events.js +265 -128
- package/dist/guards.d.ts +53 -0
- package/dist/guards.js +55 -0
- package/dist/hook-events.d.ts +38 -0
- package/dist/hook-events.js +36 -0
- package/dist/hooks.d.ts +105 -83
- package/dist/hooks.js +224 -139
- package/dist/index.d.ts +30 -25
- package/dist/index.js +29 -23
- package/dist/ledger.d.ts +62 -30
- package/dist/ledger.js +163 -56
- package/dist/platform.d.ts +13 -0
- package/dist/platform.js +13 -0
- package/dist/preselect.d.ts +222 -0
- package/dist/preselect.js +290 -0
- package/dist/request-body.d.ts +32 -0
- package/dist/request-body.js +47 -0
- package/dist/reset.d.ts +1 -0
- package/dist/reset.js +7 -6
- package/dist/retry.d.ts +40 -143
- package/dist/retry.js +79 -285
- package/dist/run-calls.d.ts +123 -0
- package/dist/run-calls.js +351 -0
- package/dist/run-turn.d.ts +44 -16
- package/dist/run-turn.js +74 -37
- package/dist/runtime.d.ts +23 -17
- package/dist/runtime.js +34 -24
- package/dist/schema-compat.d.ts +51 -11
- package/dist/schema-compat.js +375 -193
- package/dist/scope.d.ts +28 -5
- package/dist/scope.js +37 -10
- package/dist/side-task.d.ts +127 -39
- package/dist/side-task.js +189 -100
- package/dist/snapshot.d.ts +12 -7
- package/dist/snapshot.js +87 -65
- package/dist/spec.d.ts +66 -36
- package/dist/spec.js +514 -265
- package/dist/stream.d.ts +73 -16
- package/dist/stream.js +228 -98
- package/dist/thinking.d.ts +23 -8
- package/dist/thinking.js +96 -46
- package/dist/tokens.d.ts +171 -6
- package/dist/tokens.js +326 -6
- package/dist/tool-calls.d.ts +89 -15
- package/dist/tool-calls.js +205 -85
- package/dist/tool-loading.d.ts +109 -187
- package/dist/tool-loading.js +286 -372
- package/dist/wire.d.ts +81 -0
- package/dist/wire.js +73 -0
- package/llms.txt +87 -43
- package/package.json +4 -4
package/dist/client.js
CHANGED
|
@@ -1,24 +1,46 @@
|
|
|
1
|
-
import
|
|
2
|
-
import
|
|
3
|
-
import {
|
|
1
|
+
import OpenAI from 'openai';
|
|
2
|
+
import { digestOf } from './digest.js';
|
|
3
|
+
import { isPositive } from './guards.js';
|
|
4
|
+
import { MS_PER_SECOND } from './platform.js';
|
|
5
|
+
import { assignSettings, scoped } from './scope.js';
|
|
6
|
+
import { HttpStatus } from './wire.js';
|
|
4
7
|
/**
|
|
5
8
|
* The SDK insists on a non-empty key even where the server will not look at it. This is what it
|
|
6
9
|
* gets. It also reads as "this endpoint has no key" at a call site, which an empty string does
|
|
7
10
|
* not — a caller must not let a local endpoint silently borrow the key meant for a paid one.
|
|
8
11
|
*/
|
|
9
|
-
export const NO_KEY =
|
|
12
|
+
export const NO_KEY = 'agent-core';
|
|
10
13
|
/**
|
|
11
|
-
*
|
|
14
|
+
* A wait in the SDK's spelling: milliseconds, and `undefined` where zero or less means no limit.
|
|
12
15
|
*
|
|
13
|
-
* @param
|
|
16
|
+
* @param seconds - The wait as a config states it.
|
|
17
|
+
* @returns Milliseconds, or `undefined` for zero or less.
|
|
14
18
|
*/
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
+
const limitMs = (seconds) => (seconds > 0 ? seconds * MS_PER_SECOND : undefined);
|
|
20
|
+
/**
|
|
21
|
+
* An endpoint's request timeout in the SDK's spelling: milliseconds, and `undefined` for no limit.
|
|
22
|
+
*
|
|
23
|
+
* @param config - Read for `requestTimeoutSeconds` alone. Zero, less, or absent means no limit.
|
|
24
|
+
* @returns The request timeout in milliseconds, or `undefined` for no limit.
|
|
25
|
+
*/
|
|
26
|
+
export const timeoutMs = (config) => limitMs(config.requestTimeoutSeconds ?? 0);
|
|
27
|
+
/**
|
|
28
|
+
* How long to wait out a model that is still loading, in milliseconds — `undefined` where the
|
|
29
|
+
* config has no opinion, and zero where it says not to wait at all.
|
|
30
|
+
*
|
|
31
|
+
* @param config - Read for `loadingTimeoutSeconds` alone. Zero or less is no wait.
|
|
32
|
+
* @returns Milliseconds: `undefined` only where the field is absent, and zero where it is zero or
|
|
33
|
+
* less.
|
|
34
|
+
*
|
|
35
|
+
* @remarks
|
|
36
|
+
* Zero rather than the SDK's `undefined` for "no wait", because here absent already means
|
|
37
|
+
* something else: `runTurn`'s own default.
|
|
38
|
+
*/
|
|
39
|
+
export const loadingMs = (config) => config.loadingTimeoutSeconds === undefined ? undefined : (limitMs(config.loadingTimeoutSeconds) ?? 0);
|
|
19
40
|
/**
|
|
20
41
|
* A client per endpoint, made once and kept.
|
|
21
42
|
*
|
|
43
|
+
* @remarks
|
|
22
44
|
* The SDK holds its own connection pool, and a run makes a request per tool iteration on top of
|
|
23
45
|
* whatever side tasks it asks for — building a fresh client for each of them throws that pool
|
|
24
46
|
* away every time. A map rather than the single slot it would otherwise be, because agents each
|
|
@@ -36,6 +58,7 @@ const clients = scoped(() => new Map());
|
|
|
36
58
|
/**
|
|
37
59
|
* How many endpoints' clients are kept at once, until `configureClients` moves it.
|
|
38
60
|
*
|
|
61
|
+
* @remarks
|
|
39
62
|
* This is a backstop rather than a design. The key includes the API key, and every argument in
|
|
40
63
|
* this file for what bounds these caches is "a settings row's worth" — true of the deployments
|
|
41
64
|
* this was written for, and false the moment a consumer mints a key per *user*, where the map
|
|
@@ -65,8 +88,9 @@ const poolLimits = scoped(() => ({ ...CLIENT_DEFAULTS }));
|
|
|
65
88
|
*/
|
|
66
89
|
const evict = () => {
|
|
67
90
|
for (const oldest of clients().keys()) {
|
|
68
|
-
if (clients().size <= poolLimits().maxClients)
|
|
91
|
+
if (clients().size <= poolLimits().maxClients) {
|
|
69
92
|
break;
|
|
93
|
+
}
|
|
70
94
|
clients().delete(oldest);
|
|
71
95
|
}
|
|
72
96
|
};
|
|
@@ -74,6 +98,13 @@ const evict = () => {
|
|
|
74
98
|
* Changes what the client pool is held to, for a process whose endpoints are not shaped like the
|
|
75
99
|
* deployments these defaults were chosen for.
|
|
76
100
|
*
|
|
101
|
+
* @param [options] - The bounds to change. A field left out — or given anything that is not a number
|
|
102
|
+
* above zero — keeps what it has, so a half-built config narrows nothing. `Infinity` is a number
|
|
103
|
+
* above zero: as `maxClients` it lifts the bound, and as `listingMissMs` a miss is never asked
|
|
104
|
+
* about again until `resetClients`.
|
|
105
|
+
* @returns Everything in force afterwards, including what this call did not change.
|
|
106
|
+
*
|
|
107
|
+
* @remarks
|
|
77
108
|
* One setting per runtime for the same reason `configureEvents` is: the pool is one thing per
|
|
78
109
|
* runtime, and its size is a deployment's setting, said once at startup. A multi-tenant host
|
|
79
110
|
* keying on a key per user raises `maxClients` so its tenants stop evicting each other's
|
|
@@ -83,39 +114,36 @@ const evict = () => {
|
|
|
83
114
|
* A `maxClients` below the pool's current size evicts down to it at once, least recently asked
|
|
84
115
|
* for first, as the next `getClient` would have. A shorter `listingMissMs` applies to misses
|
|
85
116
|
* already remembered, since each is a timestamp compared against it when read.
|
|
86
|
-
*
|
|
87
|
-
* @param options The bounds to change. A field left out — or given anything that is not a number
|
|
88
|
-
* above zero — keeps what it has, so a half-built config narrows nothing. `Infinity` is a number
|
|
89
|
-
* above zero: as `maxClients` it lifts the bound, and as `listingMissMs` a miss is never asked
|
|
90
|
-
* about again until `resetClients`.
|
|
91
|
-
* @returns Everything in force afterwards, including what this call did not change.
|
|
92
117
|
*/
|
|
93
118
|
export function configureClients(options = {}) {
|
|
94
|
-
|
|
95
|
-
if (typeof value === "number" && value > 0) {
|
|
96
|
-
poolLimits()[name] = value;
|
|
97
|
-
}
|
|
98
|
-
}
|
|
119
|
+
const inForce = assignSettings(poolLimits(), options);
|
|
99
120
|
evict();
|
|
100
|
-
return
|
|
121
|
+
return inForce;
|
|
101
122
|
}
|
|
102
123
|
/** How many idle windows the first chunk gets when `firstTokenSeconds` is not given. */
|
|
103
124
|
export const FIRST_TOKEN_FACTOR = 5;
|
|
104
125
|
/**
|
|
105
126
|
* The wait for a streamed turn's first chunk, in the SDK's spelling: `undefined` is no limit.
|
|
106
127
|
*
|
|
107
|
-
* @param config Read for `firstTokenSeconds`, and `requestTimeoutSeconds` where that is absent.
|
|
128
|
+
* @param config - Read for `firstTokenSeconds`, and `requestTimeoutSeconds` where that is absent.
|
|
129
|
+
* @returns Milliseconds. `firstTokenSeconds` where it is given, zero or less being no limit;
|
|
130
|
+
* otherwise `FIRST_TOKEN_FACTOR` request timeouts, and no limit where the request has none.
|
|
108
131
|
*/
|
|
109
132
|
export const firstTokenMs = (config) => {
|
|
110
133
|
if (config.firstTokenSeconds === undefined) {
|
|
111
134
|
const idle = timeoutMs(config);
|
|
112
135
|
return idle === undefined ? undefined : idle * FIRST_TOKEN_FACTOR;
|
|
113
136
|
}
|
|
114
|
-
return config.firstTokenSeconds
|
|
137
|
+
return limitMs(config.firstTokenSeconds);
|
|
115
138
|
};
|
|
116
139
|
/**
|
|
117
140
|
* The client for an endpoint, built once and kept.
|
|
118
141
|
*
|
|
142
|
+
* @param config - Where to send requests and how long to wait. An absent `apiKey` becomes `NO_KEY`.
|
|
143
|
+
* @returns The pooled client: the same object for the same base URL, key and timeout, until it is
|
|
144
|
+
* evicted or `resetClients` runs. Asking for it makes it the last in line to be evicted.
|
|
145
|
+
*
|
|
146
|
+
* @remarks
|
|
119
147
|
* Pooled on the three fields that change how a request is sent, so agents sharing a server share
|
|
120
148
|
* a connection and agents on different servers never share a client.
|
|
121
149
|
*
|
|
@@ -125,8 +153,6 @@ export const firstTokenMs = (config) => {
|
|
|
125
153
|
* connection pools rather than sharing them and holding every one of them — `maxClients` keeps
|
|
126
154
|
* that from being unbounded, and it is the point at which a client of your own, built and held
|
|
127
155
|
* per tenant, is the better answer than this.
|
|
128
|
-
*
|
|
129
|
-
* @param config Where to send requests and how long to wait. An absent `apiKey` becomes `NO_KEY`.
|
|
130
156
|
*/
|
|
131
157
|
export function getClient(config) {
|
|
132
158
|
const apiKey = config.apiKey || NO_KEY;
|
|
@@ -150,23 +176,30 @@ export function getClient(config) {
|
|
|
150
176
|
/**
|
|
151
177
|
* The context window, spelled every way a server spells it at the top of a listing entry.
|
|
152
178
|
*
|
|
179
|
+
* @remarks
|
|
153
180
|
* None of these is in the OpenAI listing schema, so every server that says anything says it as
|
|
154
181
|
* an extra key of its own: `max_model_len` is vLLM, `context_length` OpenRouter, `n_ctx` the raw
|
|
155
182
|
* llama bindings. Whichever turns up first is taken — a server reporting two of them is
|
|
156
183
|
* reporting the same number twice. llama.cpp puts its number under `meta` instead, and LM Studio
|
|
157
184
|
* and Ollama put none on this route at all; see `servedWindow`.
|
|
158
185
|
*/
|
|
159
|
-
const CONTEXT_KEYS = [
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
186
|
+
const CONTEXT_KEYS = ['context_length', 'max_context_window', 'max_model_len', 'context_window', 'n_ctx'];
|
|
187
|
+
/**
|
|
188
|
+
* A number above zero as itself and anything else as zero, which here means "not stated".
|
|
189
|
+
*
|
|
190
|
+
* @param value - Whatever a server put under a key. Absent, a string and a negative all read as
|
|
191
|
+
* zero.
|
|
192
|
+
* @returns The number, or 0.
|
|
193
|
+
*/
|
|
194
|
+
const positive = (value) => (isPositive(value) ? value : 0);
|
|
167
195
|
/**
|
|
168
196
|
* A listing entry's window, and whether it is only the one the model was trained with.
|
|
169
197
|
*
|
|
198
|
+
* @param model - A listing entry as the server sent it, extra keys included. Not written to.
|
|
199
|
+
* @returns The window in tokens, and `trained` where it came from `meta.n_ctx_train`. An entry
|
|
200
|
+
* that states neither answers zero and `false`.
|
|
201
|
+
*
|
|
202
|
+
* @remarks
|
|
170
203
|
* The two are told apart because they are trusted differently: a top-level key is taken as the
|
|
171
204
|
* window being served and settles the lookup, and a trained one is a ceiling the server may be
|
|
172
205
|
* running well under, so `contextLimitFor` still asks the native routes before settling for it.
|
|
@@ -175,8 +208,9 @@ function contextLengthOf(model) {
|
|
|
175
208
|
const record = model;
|
|
176
209
|
for (const key of CONTEXT_KEYS) {
|
|
177
210
|
const value = positive(record[key]);
|
|
178
|
-
if (value)
|
|
211
|
+
if (value) {
|
|
179
212
|
return { contextLength: value, trained: false };
|
|
213
|
+
}
|
|
180
214
|
}
|
|
181
215
|
// llama.cpp's, and the window the model was trained with rather than the one it is served in:
|
|
182
216
|
// a 256k model started at `-c 16384` lists 262144 here. Better than nothing, and why the
|
|
@@ -188,6 +222,7 @@ function contextLengthOf(model) {
|
|
|
188
222
|
/**
|
|
189
223
|
* The last listing from each endpoint, so a run can size its window without a round trip.
|
|
190
224
|
*
|
|
225
|
+
* @remarks
|
|
191
226
|
* Keyed by `endpointKey`, because two endpoints are two different sets of models and one of
|
|
192
227
|
* them having answered says nothing about the other — and because a key can be the difference
|
|
193
228
|
* between what a router will show one caller and another.
|
|
@@ -203,6 +238,7 @@ const listings = scoped(() => new Map());
|
|
|
203
238
|
/**
|
|
204
239
|
* The models in each endpoint's last listing whose window is only the trained one, by id.
|
|
205
240
|
*
|
|
241
|
+
* @remarks
|
|
206
242
|
* Kept beside `listings` rather than on `ModelInfo`, so what `listModels` hands back is the shape
|
|
207
243
|
* it has always been. Replaced with each listing that lands, and left alone by one that fails.
|
|
208
244
|
*/
|
|
@@ -210,6 +246,7 @@ const trainedOnly = scoped(() => new Map());
|
|
|
210
246
|
/**
|
|
211
247
|
* When an endpoint was last asked about a model it did not name, keyed on the two together.
|
|
212
248
|
*
|
|
249
|
+
* @remarks
|
|
213
250
|
* `contextLimitFor` asks again whenever the listing does not hold the model, which is right for
|
|
214
251
|
* a model that arrives late and wrong for one that is never coming. The second is the ordinary
|
|
215
252
|
* case rather than the exotic one — a llama.cpp box served under a `-a` alias that does not
|
|
@@ -226,6 +263,13 @@ const misses = scoped(() => new Map());
|
|
|
226
263
|
* What counts as one endpoint, everywhere in this package that has to remember something about
|
|
227
264
|
* one — the model listings, `capabilities`, and the no-thinking hints in `side-task`.
|
|
228
265
|
*
|
|
266
|
+
* @param config - Read for `baseUrl` and `apiKey` alone, and the key is optional here where
|
|
267
|
+
* `Endpoint` requires it — `capabilitiesFor` is handed a URL and maybe a key rather than a whole
|
|
268
|
+
* config, and absent and empty already mean the same thing. The timeout is deliberately not in
|
|
269
|
+
* it; see `listings`.
|
|
270
|
+
* @returns The URL and the key as one JSON string, the key in the clear.
|
|
271
|
+
*
|
|
272
|
+
* @remarks
|
|
229
273
|
* The URL and the key together, because the key is part of what is on the other end rather than
|
|
230
274
|
* only how it is paid for: a router is free to send two keys to two different backends, and then
|
|
231
275
|
* what one of them refused is not a fact about the other. Absent reads as `NO_KEY`, so an
|
|
@@ -235,27 +279,36 @@ const misses = scoped(() => new Map());
|
|
|
235
279
|
* is impossible in a URL or a key, and two endpoints must never collide on one entry. A host
|
|
236
280
|
* keeping its own per-endpoint state keys it on this rather than on a copy of it, so the two
|
|
237
281
|
* cannot drift apart. It holds the key in the clear; `endpointId` is the one to write down.
|
|
238
|
-
*
|
|
239
|
-
* @param config Read for `baseUrl` and `apiKey` alone, and the key is optional here where
|
|
240
|
-
* `Endpoint` requires it — `capabilitiesFor` is handed a URL and maybe a key rather than a whole
|
|
241
|
-
* config, and absent and empty already mean the same thing. The timeout is deliberately not in
|
|
242
|
-
* it; see `listings`.
|
|
243
282
|
*/
|
|
244
283
|
export const endpointKey = (config) => JSON.stringify([config.baseUrl, config.apiKey || NO_KEY]);
|
|
284
|
+
/**
|
|
285
|
+
* One model on one endpoint, as every per-model cache keys it: stringified, so neither half runs
|
|
286
|
+
* into the other.
|
|
287
|
+
*
|
|
288
|
+
* @param endpoint - Whatever the cache names an endpoint by — `endpointKey` here, `endpointId`
|
|
289
|
+
* where the key may be written down.
|
|
290
|
+
* @param model - The model's name as the endpoint knows it.
|
|
291
|
+
* @returns The pair as a JSON array, which `JSON.parse` takes apart again.
|
|
292
|
+
*/
|
|
293
|
+
export const modelKey = (endpoint, model) => JSON.stringify([endpoint, model]);
|
|
245
294
|
/**
|
|
246
295
|
* `endpointKey` hashed, for the remembered facts that can leave the process.
|
|
247
296
|
*
|
|
297
|
+
* @param config - Read for `baseUrl` and `apiKey` alone, as `endpointKey` reads it.
|
|
298
|
+
* @returns The digest of that endpoint's `endpointKey`: the same for the same URL and key in any
|
|
299
|
+
* process, and no key to be read out of it.
|
|
300
|
+
*
|
|
301
|
+
* @remarks
|
|
248
302
|
* What an endpoint refused is exported by `exportCapabilities` to be written into a settings row
|
|
249
303
|
* or a file, and a key inside that blob is a credential copied somewhere nobody meant to keep one.
|
|
250
304
|
* A digest identifies the same endpoint on the next boot without saying what the key was, and it
|
|
251
305
|
* is how a host finds its own endpoint's entry in a `CapabilitySnapshot`.
|
|
252
|
-
*
|
|
253
|
-
* @param config Read for `baseUrl` and `apiKey` alone, as `endpointKey` reads it.
|
|
254
306
|
*/
|
|
255
|
-
export const endpointId = (config) =>
|
|
307
|
+
export const endpointId = (config) => digestOf(endpointKey(config));
|
|
256
308
|
/**
|
|
257
309
|
* Served windows found by asking a server's own API, keyed on endpoint and model together.
|
|
258
310
|
*
|
|
311
|
+
* @remarks
|
|
259
312
|
* A model's entry stays until `resetClients`, like a listing; a server that answered without one
|
|
260
313
|
* is asked again after `listingMissMs`, like a listing that did not name the model.
|
|
261
314
|
*/
|
|
@@ -265,6 +318,7 @@ const unserved = scoped(() => new Set());
|
|
|
265
318
|
/**
|
|
266
319
|
* Endpoints whose `/api/v0/models` answered with an error, and when.
|
|
267
320
|
*
|
|
321
|
+
* @remarks
|
|
268
322
|
* `gone` is a route the server does not have (`NOT_THERE`), which is a fact about the server and
|
|
269
323
|
* stands until `resetClients`. Any other error stands for `listingMissMs` only, because it may be
|
|
270
324
|
* one the server gets over, and LM Studio latched for good on a passing 5xx would have no window
|
|
@@ -274,21 +328,45 @@ const unserved = scoped(() => new Set());
|
|
|
274
328
|
* window is never `unserved`, and without this its missing route was asked again with every miss.
|
|
275
329
|
*/
|
|
276
330
|
const refusals = scoped(() => new Map());
|
|
277
|
-
/**
|
|
331
|
+
/**
|
|
332
|
+
* Whether `/api/v0/models` is to be left alone on this endpoint for now.
|
|
333
|
+
*
|
|
334
|
+
* @param endpoint - The endpoint, by `endpointKey`.
|
|
335
|
+
* @returns `true` where the route last answered as one the server does not have, or with any other
|
|
336
|
+
* error inside `listingMissMs`; `false` for an endpoint with no error on record.
|
|
337
|
+
*/
|
|
278
338
|
const refused = (endpoint) => {
|
|
279
339
|
const refusal = refusals().get(endpoint);
|
|
280
|
-
return
|
|
340
|
+
return refusal !== undefined && (refusal.gone || Date.now() - refusal.at < poolLimits().listingMissMs);
|
|
281
341
|
};
|
|
282
342
|
/** How long a probe may take before the window is taken from the listing instead. */
|
|
283
343
|
const PROBE_TIMEOUT_MS = 10_000;
|
|
284
|
-
/**
|
|
285
|
-
|
|
344
|
+
/**
|
|
345
|
+
* The server root behind an OpenAI-compatible base URL, which is where the native APIs live.
|
|
346
|
+
*
|
|
347
|
+
* @param baseUrl - The base URL as configured. Not trimmed of whitespace.
|
|
348
|
+
* @returns The URL less its trailing slashes and then one trailing `/v1`; one that ends in neither
|
|
349
|
+
* comes back as given.
|
|
350
|
+
*/
|
|
351
|
+
const rootOf = (baseUrl) => baseUrl.replace(/\/+$/, '').replace(/\/v1$/, '');
|
|
286
352
|
/** A route this server does not have, rather than one that failed to answer. */
|
|
287
|
-
const NOT_THERE = new Set([
|
|
353
|
+
const NOT_THERE = new Set([
|
|
354
|
+
HttpStatus.NotFound,
|
|
355
|
+
HttpStatus.MethodNotAllowed,
|
|
356
|
+
HttpStatus.NotImplemented,
|
|
357
|
+
]);
|
|
288
358
|
/**
|
|
289
|
-
* Asks one native endpoint.
|
|
290
|
-
*
|
|
291
|
-
*
|
|
359
|
+
* Asks one native endpoint.
|
|
360
|
+
*
|
|
361
|
+
* @param config - The endpoint. Its key goes as a bearer token where it has one, and its request
|
|
362
|
+
* timeout bounds the wait — `PROBE_TIMEOUT_MS` where it has none.
|
|
363
|
+
* @param path - The route and its query, from the server root rather than from the base URL.
|
|
364
|
+
* @returns How the server answered, with `body` only for a 2xx that parsed as JSON.
|
|
365
|
+
*
|
|
366
|
+
* @remarks
|
|
367
|
+
* `failed` is any answer that was not a 2xx and `missing` the ones among them that mean a server
|
|
368
|
+
* without the route; `body` is absent for both, and for a 2xx that was not JSON. A server that
|
|
369
|
+
* could not be reached throws.
|
|
292
370
|
*/
|
|
293
371
|
async function probe(config, path) {
|
|
294
372
|
const apiKey = config.apiKey || undefined;
|
|
@@ -296,8 +374,9 @@ async function probe(config, path) {
|
|
|
296
374
|
headers: apiKey ? { authorization: `Bearer ${apiKey}` } : {},
|
|
297
375
|
signal: AbortSignal.timeout(timeoutMs(config) ?? PROBE_TIMEOUT_MS),
|
|
298
376
|
});
|
|
299
|
-
if (!response.ok)
|
|
377
|
+
if (!response.ok) {
|
|
300
378
|
return { failed: true, missing: NOT_THERE.has(response.status) };
|
|
379
|
+
}
|
|
301
380
|
try {
|
|
302
381
|
return { failed: false, missing: false, body: await response.json() };
|
|
303
382
|
}
|
|
@@ -309,6 +388,11 @@ async function probe(config, path) {
|
|
|
309
388
|
* The window a local server's own API says it is serving a model in, for the servers whose
|
|
310
389
|
* listing does not say.
|
|
311
390
|
*
|
|
391
|
+
* @param config - The endpoint, plus the model whose window is wanted.
|
|
392
|
+
* @returns The window in tokens, or zero where none was found. Never rejects: a server that could
|
|
393
|
+
* not be reached answers zero too.
|
|
394
|
+
*
|
|
395
|
+
* @remarks
|
|
312
396
|
* llama.cpp reports it on `/props` as `default_generation_settings.n_ctx`, per slot, and LM
|
|
313
397
|
* Studio on `/api/v0/models` as `loaded_context_length` while the model is loaded. Both differ
|
|
314
398
|
* from the trained window in the case that matters, a model started in a smaller one than it was
|
|
@@ -321,31 +405,32 @@ async function probe(config, path) {
|
|
|
321
405
|
* other error. A server that could not be reached is not remembered at all.
|
|
322
406
|
*
|
|
323
407
|
* The listing is not read here; `contextLimitFor` reads it first and comes here after.
|
|
324
|
-
*
|
|
325
|
-
* @param config The endpoint, plus the model whose window is wanted.
|
|
326
408
|
*/
|
|
327
409
|
export async function servedWindow(config) {
|
|
328
410
|
const endpoint = endpointKey(config);
|
|
329
|
-
if (unserved().has(endpoint))
|
|
411
|
+
if (unserved().has(endpoint)) {
|
|
330
412
|
return 0;
|
|
331
|
-
|
|
413
|
+
}
|
|
414
|
+
const key = modelKey(endpoint, config.model);
|
|
332
415
|
const known = served().get(key);
|
|
333
|
-
if (known && (known.window > 0 || Date.now() - known.at < poolLimits().listingMissMs))
|
|
416
|
+
if (known && (known.window > 0 || Date.now() - known.at < poolLimits().listingMissMs)) {
|
|
334
417
|
return known.window;
|
|
418
|
+
}
|
|
335
419
|
let window = 0;
|
|
336
420
|
try {
|
|
337
421
|
// Named, for a llama.cpp router serving several models; a single-model server ignores it.
|
|
338
422
|
const props = await probe(config, `/props?model=${encodeURIComponent(config.model)}`);
|
|
339
|
-
const settings = props.body
|
|
340
|
-
?.default_generation_settings;
|
|
423
|
+
const settings = props.body?.default_generation_settings;
|
|
341
424
|
window = positive(settings?.n_ctx);
|
|
342
425
|
if (!window) {
|
|
343
426
|
if (!refused(endpoint)) {
|
|
344
|
-
const lmstudio = await probe(config,
|
|
345
|
-
if (lmstudio.failed)
|
|
427
|
+
const lmstudio = await probe(config, '/api/v0/models');
|
|
428
|
+
if (lmstudio.failed) {
|
|
346
429
|
refusals().set(endpoint, { at: Date.now(), gone: lmstudio.missing });
|
|
347
|
-
|
|
430
|
+
}
|
|
431
|
+
else {
|
|
348
432
|
refusals().delete(endpoint);
|
|
433
|
+
}
|
|
349
434
|
const { data } = (lmstudio.body ?? {});
|
|
350
435
|
const entry = Array.isArray(data)
|
|
351
436
|
? data.find((model) => model?.id === config.model)
|
|
@@ -367,19 +452,21 @@ export async function servedWindow(config) {
|
|
|
367
452
|
/**
|
|
368
453
|
* Asks an endpoint what it serves, and remembers the answer.
|
|
369
454
|
*
|
|
370
|
-
* @param config The endpoint to ask. Remembered per base URL and key, not per model.
|
|
455
|
+
* @param config - The endpoint to ask. Remembered per base URL and key, not per model.
|
|
456
|
+
* @returns The models in order of id, whatever order the endpoint listed them in.
|
|
371
457
|
*/
|
|
372
458
|
export async function listModels(config) {
|
|
373
459
|
const { data } = await getClient(config).models.list();
|
|
374
460
|
const trained = new Set();
|
|
375
|
-
const models =
|
|
376
|
-
|
|
461
|
+
const models = [];
|
|
462
|
+
for (const model of data) {
|
|
377
463
|
const window = contextLengthOf(model);
|
|
378
|
-
if (window.trained)
|
|
464
|
+
if (window.trained) {
|
|
379
465
|
trained.add(model.id);
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
466
|
+
}
|
|
467
|
+
models.push({ id: model.id, contextLength: window.contextLength });
|
|
468
|
+
}
|
|
469
|
+
models.sort((a, b) => a.id.localeCompare(b.id));
|
|
383
470
|
const key = endpointKey(config);
|
|
384
471
|
listings().set(key, models);
|
|
385
472
|
trainedOnly().set(key, trained);
|
|
@@ -388,6 +475,12 @@ export async function listModels(config) {
|
|
|
388
475
|
/**
|
|
389
476
|
* How much a model will read, in tokens. Zero means nobody knows.
|
|
390
477
|
*
|
|
478
|
+
* @param config - The endpoint, plus the model whose window is wanted.
|
|
479
|
+
* @param [declared] - The operator's own number. Above zero it wins and the endpoint is not asked.
|
|
480
|
+
* @returns The window in tokens, or zero. Never rejects: a listing or a native route that failed
|
|
481
|
+
* is an unknown window.
|
|
482
|
+
*
|
|
483
|
+
* @remarks
|
|
391
484
|
* `declared` is the operator's own number, and it wins outright: an endpoint can report the
|
|
392
485
|
* window a model was *built* with while serving it in a much smaller one — llama.cpp will
|
|
393
486
|
* happily load a 256k model at `-c 16384` and go on listing it as 256k — and a run refused on
|
|
@@ -405,13 +498,11 @@ export async function listModels(config) {
|
|
|
405
498
|
* are two more requests that a server stating its window in the listing never needed. Asked
|
|
406
499
|
* first, they were also asked again every `listingMissMs` for the life of the process on a server
|
|
407
500
|
* whose `/props` answers without a window, by any host that sizes the window per turn.
|
|
408
|
-
*
|
|
409
|
-
* @param config The endpoint, plus the model whose window is wanted.
|
|
410
|
-
* @param declared The operator's own number. Above zero it wins and the endpoint is not asked.
|
|
411
501
|
*/
|
|
412
502
|
export async function contextLimitFor(config, declared = 0) {
|
|
413
|
-
if (declared > 0)
|
|
503
|
+
if (declared > 0) {
|
|
414
504
|
return declared;
|
|
505
|
+
}
|
|
415
506
|
const key = endpointKey(config);
|
|
416
507
|
const listed = () => listings()
|
|
417
508
|
.get(key)
|
|
@@ -424,7 +515,7 @@ export async function contextLimitFor(config, declared = 0) {
|
|
|
424
515
|
// context meter and, in a consumer that compacts on it, compaction: the session then runs at
|
|
425
516
|
// the window instead of under it and fails against the endpoint's own refusal.
|
|
426
517
|
if (!listed()) {
|
|
427
|
-
const missKey =
|
|
518
|
+
const missKey = modelKey(key, config.model);
|
|
428
519
|
const asked = misses().get(missKey);
|
|
429
520
|
// Asked again, but not on every call. A model that is never coming answers the same zero
|
|
430
521
|
// however often the endpoint is asked, and a caller sizing a window per turn pays a round
|
|
@@ -438,10 +529,12 @@ export async function contextLimitFor(config, declared = 0) {
|
|
|
438
529
|
// than the interval, which is a memory that never expires rather than one that expires in
|
|
439
530
|
// half a minute.
|
|
440
531
|
if (answered) {
|
|
441
|
-
if (listed())
|
|
532
|
+
if (listed()) {
|
|
442
533
|
misses().delete(missKey);
|
|
443
|
-
|
|
534
|
+
}
|
|
535
|
+
else {
|
|
444
536
|
misses().set(missKey, Date.now());
|
|
537
|
+
}
|
|
445
538
|
}
|
|
446
539
|
}
|
|
447
540
|
}
|
|
@@ -449,8 +542,9 @@ export async function contextLimitFor(config, declared = 0) {
|
|
|
449
542
|
const fromListing = entry?.contextLength ?? 0;
|
|
450
543
|
// A top-level key settles it. A trained window does not: llama.cpp lists the one the model was
|
|
451
544
|
// built with, and taking it over what `/props` says lets an overflow through the guard.
|
|
452
|
-
if (fromListing > 0 && !trainedOnly().get(key)?.has(config.model))
|
|
545
|
+
if (fromListing > 0 && !trainedOnly().get(key)?.has(config.model)) {
|
|
453
546
|
return fromListing;
|
|
547
|
+
}
|
|
454
548
|
const window = await servedWindow(config);
|
|
455
549
|
return window > 0 ? window : fromListing;
|
|
456
550
|
}
|
|
@@ -468,3 +562,41 @@ export function resetClients() {
|
|
|
468
562
|
unserved().clear();
|
|
469
563
|
refusals().clear();
|
|
470
564
|
}
|
|
565
|
+
/**
|
|
566
|
+
* A base URL as two settings rows would agree on it: trimmed, without the trailing slash.
|
|
567
|
+
*
|
|
568
|
+
* @param a - The URL as one row has it.
|
|
569
|
+
* @param b - The URL as the other has it.
|
|
570
|
+
* @returns `true` where the two differ in nothing but surrounding whitespace and trailing slashes.
|
|
571
|
+
* Case counts, and so does a `/v1`.
|
|
572
|
+
*/
|
|
573
|
+
export const sameUrl = (a, b) => a.trim().replace(/\/+$/, '') === b.trim().replace(/\/+$/, '');
|
|
574
|
+
/**
|
|
575
|
+
* The key to send, where an endpoint may inherit one from the settings it overrides.
|
|
576
|
+
*
|
|
577
|
+
* @param own - The endpoint as the agent or profile states it. Its own key always wins. An empty or
|
|
578
|
+
* absent `baseUrl` is one that inherits the endpoint too.
|
|
579
|
+
* @param [inherited] - The settings it overrides. Absent treats `own` as the configured endpoint, so
|
|
580
|
+
* only its key and the environment's are in play.
|
|
581
|
+
* @param [env] - Where `OPENAI_API_KEY` is read from.
|
|
582
|
+
* @returns The key, never empty: `own`'s, else `NO_KEY` for an endpoint other than the inherited
|
|
583
|
+
* one, else the inherited key, then the environment's, then `NO_KEY`.
|
|
584
|
+
*
|
|
585
|
+
* @remarks
|
|
586
|
+
* A credential issued for one endpoint has no business being posted to another. A profile that
|
|
587
|
+
* names its own `baseUrl` and no key of its own is sent `NO_KEY` — not the operator's key, and
|
|
588
|
+
* not `$OPENAI_API_KEY` — because "I pointed an agent at a friend's server and it sent my OpenAI
|
|
589
|
+
* key" is not a mistake worth being able to make, and a local server wants no key anyway. One on
|
|
590
|
+
* the same endpoint inherits the key as it inherits everything else, and the environment is the
|
|
591
|
+
* last word on the endpoint that was configured rather than overridden.
|
|
592
|
+
*/
|
|
593
|
+
export function resolveApiKey(own, inherited, env = process.env) {
|
|
594
|
+
if (own.apiKey) {
|
|
595
|
+
return own.apiKey;
|
|
596
|
+
}
|
|
597
|
+
const baseUrl = own.baseUrl?.trim();
|
|
598
|
+
if (inherited && baseUrl && !sameUrl(baseUrl, inherited.baseUrl)) {
|
|
599
|
+
return NO_KEY;
|
|
600
|
+
}
|
|
601
|
+
return inherited?.apiKey || env.OPENAI_API_KEY || NO_KEY;
|
|
602
|
+
}
|