@stackstackstack/dsh-llm 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.i18n.yaml +6 -0
- package/README.md +101 -0
- package/README.zh.md +101 -0
- package/lib/index.js +1407 -0
- package/lib/invariant.js +84 -0
- package/lib/types/adapter-failure.d.ts +14 -0
- package/lib/types/adapter-failure.js +105 -0
- package/lib/types/api-key.d.ts +28 -0
- package/lib/types/api-key.js +34 -0
- package/lib/types/assembler.d.ts +56 -0
- package/lib/types/assembler.js +148 -0
- package/lib/types/attribution.d.ts +47 -0
- package/lib/types/attribution.js +46 -0
- package/lib/types/brand.d.ts +48 -0
- package/lib/types/brand.js +44 -0
- package/lib/types/call-config.d.ts +62 -0
- package/lib/types/call-config.js +86 -0
- package/lib/types/content.d.ts +12 -0
- package/lib/types/content.js +14 -0
- package/lib/types/error.d.ts +73 -0
- package/lib/types/error.js +145 -0
- package/lib/types/index.d.ts +341 -0
- package/lib/types/index.js +730 -0
- package/lib/types/invariant.d.ts +13 -0
- package/lib/types/invariant.js +100 -0
- package/lib/types/message.d.ts +206 -0
- package/lib/types/message.js +100 -0
- package/lib/types/never.d.ts +16 -0
- package/lib/types/never.js +21 -0
- package/lib/types/retry-policy.d.ts +66 -0
- package/lib/types/retry-policy.js +123 -0
- package/lib/types/types.d.ts +349 -0
- package/lib/types/types.js +7 -0
- package/package.json +64 -0
|
@@ -0,0 +1,730 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM service: adapter registry with a waterfall-interceptable streaming call
|
|
3
|
+
* API. Exports the `LlmRuntime` default, the abstract `LlmAdapter` for
|
|
4
|
+
* provider backends, and `BlockAssembler` for chunk assembly.
|
|
5
|
+
*
|
|
6
|
+
* @module @stackstackstack/dsh-llm
|
|
7
|
+
*/
|
|
8
|
+
import { Service } from '@deepseek-ai/cordis';
|
|
9
|
+
import { freezeMessage } from "./message.js";
|
|
10
|
+
import { resolveRetryPolicy } from "./retry-policy.js";
|
|
11
|
+
import { callConfigEquals, deepFreeze } from "./call-config.js";
|
|
12
|
+
import { HarnessError, INVALID_CREDENTIAL_CODE } from "./error.js";
|
|
13
|
+
import { normalizeLlmFailure } from "./adapter-failure.js";
|
|
14
|
+
import { normalizeApiKey } from "./api-key.js";
|
|
15
|
+
export * from "./attribution.js";
|
|
16
|
+
export * from "./brand.js";
|
|
17
|
+
export * from "./never.js";
|
|
18
|
+
export * from "./error.js";
|
|
19
|
+
export * from "./api-key.js";
|
|
20
|
+
export * from "./types.js";
|
|
21
|
+
export * from "./content.js";
|
|
22
|
+
export * from "./message.js";
|
|
23
|
+
export * from "./retry-policy.js";
|
|
24
|
+
export { BlockAssembler } from "./assembler.js";
|
|
25
|
+
export { callConfigEquals, deepFreeze, isAgentLoopRequest, markAgentLoopRequest } from "./call-config.js";
|
|
26
|
+
/**
|
|
27
|
+
* Typed error for LLM-related failures. Extends {@link HarnessError}, so the
|
|
28
|
+
* `code` string (e.g. `AUTH`, `RATE_LIMIT`, `NO_ADAPTER`) is shared taxonomy.
|
|
29
|
+
*/
|
|
30
|
+
export class LlmError extends HarnessError {
|
|
31
|
+
/** Serializable facts retained beside this live Error. */
|
|
32
|
+
failure;
|
|
33
|
+
/**
|
|
34
|
+
* @param message - non-empty human-readable failure summary.
|
|
35
|
+
* @param code - non-empty stable provider-neutral machine code.
|
|
36
|
+
* @param options - optional cause and validated serializable provider facts.
|
|
37
|
+
*/
|
|
38
|
+
constructor(message, code, options) {
|
|
39
|
+
if (typeof message !== 'string' || message.length === 0)
|
|
40
|
+
throw new Error('LlmError message must be a non-empty string');
|
|
41
|
+
if (typeof code !== 'string' || code.length === 0)
|
|
42
|
+
throw new Error('LlmError code must be a non-empty string');
|
|
43
|
+
if (options?.status !== undefined
|
|
44
|
+
&& (!Number.isInteger(options.status) || options.status < 100 || options.status > 599)) {
|
|
45
|
+
throw new Error('LlmError status must be an integer from 100 through 599');
|
|
46
|
+
}
|
|
47
|
+
if (options?.providerRetryAfterMs !== undefined
|
|
48
|
+
&& (!Number.isFinite(options.providerRetryAfterMs) || options.providerRetryAfterMs <= 0)) {
|
|
49
|
+
throw new Error('LlmError providerRetryAfterMs must be a positive finite number');
|
|
50
|
+
}
|
|
51
|
+
if (options?.requestId !== undefined
|
|
52
|
+
&& (typeof options.requestId !== 'string' || options.requestId.length === 0)) {
|
|
53
|
+
throw new Error('LlmError requestId must be a non-empty string');
|
|
54
|
+
}
|
|
55
|
+
super(message, code, options);
|
|
56
|
+
this.name = 'LlmError';
|
|
57
|
+
this.failure = Object.freeze({
|
|
58
|
+
message,
|
|
59
|
+
code,
|
|
60
|
+
...options?.status === undefined ? {} : { status: options.status },
|
|
61
|
+
...options?.providerRetryAfterMs === undefined ? {} : { providerRetryAfterMs: options.providerRetryAfterMs },
|
|
62
|
+
...options?.requestId === undefined ? {} : { requestId: options.requestId },
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Accept one supplied credential, or refuse it as unusable.
|
|
68
|
+
*
|
|
69
|
+
* A stored key arrives from the credentials seam, a `.env` line, or a shell
|
|
70
|
+
* export, all of which pick up surrounding whitespace, so trimming is silent.
|
|
71
|
+
* Anything else fails here rather than inside `fetch`, whose ByteString
|
|
72
|
+
* refusal names a UTF-16 code point instead of the setting to change. The key
|
|
73
|
+
* never enters the message: `ref` names where to fix it, and echoing any part
|
|
74
|
+
* of a secret into a log or a UI is the failure this diagnosis avoids.
|
|
75
|
+
*
|
|
76
|
+
* Lives beside {@link LlmError} rather than in `./api-key.ts` so the predicate
|
|
77
|
+
* module stays dependency-free; both adapters share this one diagnosis instead
|
|
78
|
+
* of keeping near-identical local copies.
|
|
79
|
+
* @param raw - the credential exactly as supplied.
|
|
80
|
+
* @param pkg - the refusing package name, prefixed to the diagnostic.
|
|
81
|
+
* @param ref - the credential reference the value resolved through.
|
|
82
|
+
* @returns the trimmed, usable key.
|
|
83
|
+
*/
|
|
84
|
+
export function assertUsableApiKey(raw, pkg, ref) {
|
|
85
|
+
const checked = normalizeApiKey(raw);
|
|
86
|
+
if (checked.ok)
|
|
87
|
+
return checked.value;
|
|
88
|
+
// The Models page is named as the writer it usually is, not as the only one:
|
|
89
|
+
// the same value can arrive from a hand-edited .env or a shell export in a
|
|
90
|
+
// composition that mounts no credentials seam at all, where directing the
|
|
91
|
+
// user to a page that deployment does not serve would be a dead end.
|
|
92
|
+
throw new LlmError(checked.reason === 'empty'
|
|
93
|
+
? `${pkg}: the API key resolved from ${ref} is blank; set ${ref} to the raw key`
|
|
94
|
+
+ ' (the web Models page writes it) or export it in the launching environment'
|
|
95
|
+
: `${pkg}: the API key resolved from ${ref} contains characters no HTTP header can carry;`
|
|
96
|
+
+ ` set ${ref} to the raw key alone (the web Models page writes it)`, INVALID_CREDENTIAL_CODE);
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Provider-wire adapter for the harness message and stream vocabulary. Register implementations
|
|
100
|
+
* with `ctx.llm.registerAdapter(providers, adapter)`. Every provider HTTP request must include
|
|
101
|
+
* `attributionHeaders()`; prove the headers are added in the wire request or library header hook. The direct-fetch
|
|
102
|
+
* DeepSeek and library-backed pi-ai adapters meet this contract through different internals.
|
|
103
|
+
*/
|
|
104
|
+
export class LlmAdapter {
|
|
105
|
+
/**
|
|
106
|
+
* Describe one provider route owned by this adapter.
|
|
107
|
+
* @param provider - a route passed to `registerAdapter()` for this instance.
|
|
108
|
+
* @returns detached display metadata whose id must equal `provider`.
|
|
109
|
+
*/
|
|
110
|
+
providerInfo(provider) {
|
|
111
|
+
return { id: provider, name: provider };
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Return the provider-owned retry policy captured with this route.
|
|
115
|
+
* @param _provider - a route passed to `registerAdapter()` for this instance.
|
|
116
|
+
* @returns a resolved policy, or `undefined` to use the normal defaults.
|
|
117
|
+
*/
|
|
118
|
+
providerRetryPolicy(_provider) {
|
|
119
|
+
return undefined;
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* List models this adapter can currently advertise for one owned provider.
|
|
123
|
+
* The result is advisory: an adapter may accept unlisted model ids, and
|
|
124
|
+
* consumers must not turn absence into request rejection.
|
|
125
|
+
* @param _provider - one provider route owned by this adapter.
|
|
126
|
+
* @returns discoverable models in adapter-preferred order.
|
|
127
|
+
*/
|
|
128
|
+
listModels(_provider) {
|
|
129
|
+
return Promise.resolve([]);
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Resolve all metadata available for one exact model. This query is
|
|
133
|
+
* independent of the advisory catalog and does not validate request routing.
|
|
134
|
+
* @param provider - one provider route owned by this adapter.
|
|
135
|
+
* @param model - exact model id passed to {@link GenerateOptions.model}.
|
|
136
|
+
* @param _signal - cancellation for this exact-model lookup; asynchronous
|
|
137
|
+
* implementations must settle promptly after it aborts.
|
|
138
|
+
* @returns provider/model identity plus any context, call-default, and reasoning metadata.
|
|
139
|
+
*/
|
|
140
|
+
resolveModel(provider, model, _signal) {
|
|
141
|
+
return Promise.resolve({ provider, id: model, name: model });
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* The abstract `llm` service: an adapter registry plus a streaming model-call
|
|
146
|
+
* API, interceptable via the `llm/stream` waterfall.
|
|
147
|
+
*/
|
|
148
|
+
export class LlmRuntime extends Service {
|
|
149
|
+
adapters = new Map();
|
|
150
|
+
directory = new Map();
|
|
151
|
+
discoveries = new Map();
|
|
152
|
+
constructor(ctx) {
|
|
153
|
+
super(ctx, 'llm');
|
|
154
|
+
}
|
|
155
|
+
/** Notify topology observers without letting one broken listener veto the commit. */
|
|
156
|
+
emitAdaptersUpdated() {
|
|
157
|
+
// Cordis emit uses Array.map: one synchronous throw starves later
|
|
158
|
+
// listeners. Registry notifications are non-vetoing, so contain each
|
|
159
|
+
// callback independently; INVARIANT-coded failures still surface.
|
|
160
|
+
let invariantFailure;
|
|
161
|
+
for (const listener of this.ctx.events.dispatch('emit', ['llm/adapters-updated'])) {
|
|
162
|
+
try {
|
|
163
|
+
const returned = listener();
|
|
164
|
+
if (returned != null && typeof returned.then === 'function') {
|
|
165
|
+
// An emit listener may still be an async function; its rejection
|
|
166
|
+
// cannot reach the synchronous INVARIANT rethrow below, so it is
|
|
167
|
+
// contained here instead of becoming an unhandled rejection.
|
|
168
|
+
void Promise.resolve(returned).then(undefined, (error) => {
|
|
169
|
+
this.warnAdaptersListenerFailure(error);
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
catch (error) {
|
|
174
|
+
if (error?.code === 'INVARIANT') {
|
|
175
|
+
invariantFailure ??= error;
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
this.warnAdaptersListenerFailure(error);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
if (invariantFailure !== undefined)
|
|
182
|
+
throw invariantFailure;
|
|
183
|
+
}
|
|
184
|
+
/** Contained-listener diagnostic shared by the sync and async failure paths. */
|
|
185
|
+
warnAdaptersListenerFailure(error) {
|
|
186
|
+
this.ctx.logger.warn('llm: an llm/adapters-updated listener failed');
|
|
187
|
+
this.ctx.logger.warn(error);
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Register an adapter for the given provider routes. Throws `LlmError` with code
|
|
191
|
+
* `DUPLICATE_ADAPTER` if any provider already has an adapter (all-or-nothing).
|
|
192
|
+
* Disposed with the fiber.
|
|
193
|
+
* @param providers - every provider route this adapter should serve.
|
|
194
|
+
* @param adapter - the adapter that streams calls for those providers.
|
|
195
|
+
* @returns the disposer, carrying {@link AdapterRegistrationHandle.replace}.
|
|
196
|
+
*/
|
|
197
|
+
registerAdapter(providers, adapter) {
|
|
198
|
+
// The routes this registration currently holds; `replace` rewrites it, and
|
|
199
|
+
// the disposer releases whatever it holds at disposal time.
|
|
200
|
+
const owned = new Set();
|
|
201
|
+
// The disposer has run: `owned` being empty cannot say so on its own,
|
|
202
|
+
// because `replace([])` legally leaves a live registration holding none.
|
|
203
|
+
let released = false;
|
|
204
|
+
const dispose = this.ctx.effect(function* () {
|
|
205
|
+
if (providers.length === 0)
|
|
206
|
+
throw new LlmError('an adapter must register at least one provider', 'INVALID_ADAPTER');
|
|
207
|
+
this.commitRoutes(owned, this.prepareRoutes(providers, adapter, owned));
|
|
208
|
+
yield () => {
|
|
209
|
+
released = true;
|
|
210
|
+
for (const provider of owned)
|
|
211
|
+
this.adapters.delete(provider);
|
|
212
|
+
owned.clear();
|
|
213
|
+
this.emitAdaptersUpdated();
|
|
214
|
+
};
|
|
215
|
+
}.bind(this), 'llm.registerAdapter()');
|
|
216
|
+
// ctx.effect's disposer returns Promise<void>; our disposer API is
|
|
217
|
+
// synchronous fire-and-forget — discard the (always-resolved) promise.
|
|
218
|
+
const handle = (() => void dispose());
|
|
219
|
+
handle.replace = (next) => {
|
|
220
|
+
// Registering here would leak: the effect's disposer already ran, so
|
|
221
|
+
// nothing remains to release whatever this call would put in the map.
|
|
222
|
+
if (released) {
|
|
223
|
+
throw new LlmError('a disposed adapter registration cannot replace its routes', 'REGISTRATION_DISPOSED');
|
|
224
|
+
}
|
|
225
|
+
this.commitRoutes(owned, this.prepareRoutes(next, adapter, owned));
|
|
226
|
+
};
|
|
227
|
+
return handle;
|
|
228
|
+
}
|
|
229
|
+
/**
|
|
230
|
+
* Validate one candidate route set for `adapter`, treating routes this
|
|
231
|
+
* registration already holds as available. Nothing is mutated: a rejected
|
|
232
|
+
* candidate leaves the registry exactly as it was.
|
|
233
|
+
*/
|
|
234
|
+
prepareRoutes(providers, adapter, owned) {
|
|
235
|
+
const unique = new Set();
|
|
236
|
+
const registrations = [];
|
|
237
|
+
for (const provider of providers) {
|
|
238
|
+
if (provider.length === 0)
|
|
239
|
+
throw new LlmError('adapter provider names must be non-empty', 'INVALID_ADAPTER');
|
|
240
|
+
if (unique.has(provider) || (this.adapters.has(provider) && !owned.has(provider))) {
|
|
241
|
+
throw new LlmError(`an adapter for provider "${provider}" is already registered`, 'DUPLICATE_ADAPTER');
|
|
242
|
+
}
|
|
243
|
+
const info = adapter.providerInfo(provider);
|
|
244
|
+
if (typeof info.id !== 'string' || info.id !== provider || typeof info.name !== 'string' || info.name.length === 0) {
|
|
245
|
+
throw new LlmError(`adapter metadata for provider "${provider}" must preserve its id and have a non-empty name`, 'INVALID_ADAPTER');
|
|
246
|
+
}
|
|
247
|
+
unique.add(provider);
|
|
248
|
+
const retryPolicy = adapter.providerRetryPolicy(provider)
|
|
249
|
+
?? resolveRetryPolicy(undefined, `llm: provider "${provider}" retryPolicy`);
|
|
250
|
+
registrations.push({
|
|
251
|
+
adapter,
|
|
252
|
+
provider: { id: info.id, name: info.name },
|
|
253
|
+
retryPolicy,
|
|
254
|
+
});
|
|
255
|
+
}
|
|
256
|
+
return registrations;
|
|
257
|
+
}
|
|
258
|
+
/**
|
|
259
|
+
* Swap this registration's routes for the prepared ones in one synchronous
|
|
260
|
+
* section, so no observer can see the registry between the release and the
|
|
261
|
+
* re-registration. The route set's one mutation point is also where
|
|
262
|
+
* `llm/adapters-updated` is published, so a `replace` announces itself
|
|
263
|
+
* exactly like a first registration.
|
|
264
|
+
*/
|
|
265
|
+
commitRoutes(owned, registrations) {
|
|
266
|
+
for (const provider of owned)
|
|
267
|
+
this.adapters.delete(provider);
|
|
268
|
+
owned.clear();
|
|
269
|
+
for (const registration of registrations) {
|
|
270
|
+
this.adapters.set(registration.provider.id, registration);
|
|
271
|
+
owned.add(registration.provider.id);
|
|
272
|
+
}
|
|
273
|
+
this.emitAdaptersUpdated();
|
|
274
|
+
}
|
|
275
|
+
/**
|
|
276
|
+
* Describe provider routes with a registered adapter.
|
|
277
|
+
* @returns detached provider metadata in registration order.
|
|
278
|
+
*/
|
|
279
|
+
listProviders() {
|
|
280
|
+
return [...this.adapters.values()].map(({ provider }) => ({ ...provider }));
|
|
281
|
+
}
|
|
282
|
+
/**
|
|
283
|
+
* Declare provider routes an adapter plugin can activate through
|
|
284
|
+
* configuration. Registration is all-or-nothing: an empty list, invalid
|
|
285
|
+
* entry, or a provider already declared by any registration throws
|
|
286
|
+
* `LlmError` without registering the rest. Disposed with the fiber.
|
|
287
|
+
* @param entries - every configurable provider this plugin owns.
|
|
288
|
+
* @returns a handle that withdraws all of them, and can atomically replace them.
|
|
289
|
+
*/
|
|
290
|
+
registerConfigurableProviders(entries) {
|
|
291
|
+
let held = [];
|
|
292
|
+
let disposed = false;
|
|
293
|
+
/**
|
|
294
|
+
* Validate a candidate set in full against everything this registration
|
|
295
|
+
* does not already hold, then publish it. Nothing is written until the
|
|
296
|
+
* whole set passes, so a refused candidate leaves the current entries in
|
|
297
|
+
* place — the property that makes `replace` a swap rather than a
|
|
298
|
+
* delete-then-add that can strand the directory empty.
|
|
299
|
+
*/
|
|
300
|
+
const commit = (candidates) => {
|
|
301
|
+
const detached = [];
|
|
302
|
+
const own = new Set(held.map(entry => entry.provider));
|
|
303
|
+
for (const entry of candidates) {
|
|
304
|
+
if (entry.provider.length === 0 || entry.displayName.length === 0 || entry.settingsNs.length === 0) {
|
|
305
|
+
throw new LlmError('configurable providers need a non-empty provider, displayName, and settingsNs', 'INVALID_DIRECTORY');
|
|
306
|
+
}
|
|
307
|
+
if (entry.settingsPath.some(segment => segment.length === 0)) {
|
|
308
|
+
throw new LlmError(`configurable provider "${entry.provider}" has an empty settingsPath segment`, 'INVALID_DIRECTORY');
|
|
309
|
+
}
|
|
310
|
+
if ((this.directory.has(entry.provider) && !own.has(entry.provider))
|
|
311
|
+
|| detached.some(seen => seen.provider === entry.provider)) {
|
|
312
|
+
throw new LlmError(`configurable provider "${entry.provider}" is already declared`, 'DUPLICATE_DIRECTORY');
|
|
313
|
+
}
|
|
314
|
+
detached.push({ ...entry, settingsPath: [...entry.settingsPath] });
|
|
315
|
+
}
|
|
316
|
+
for (const entry of held)
|
|
317
|
+
this.directory.delete(entry.provider);
|
|
318
|
+
for (const entry of detached)
|
|
319
|
+
this.directory.set(entry.provider, entry);
|
|
320
|
+
held = detached;
|
|
321
|
+
this.emitAdaptersUpdated();
|
|
322
|
+
};
|
|
323
|
+
const dispose = this.ctx.effect(function* () {
|
|
324
|
+
if (entries.length === 0) {
|
|
325
|
+
throw new LlmError('a configurable-provider registration must declare at least one provider', 'INVALID_DIRECTORY');
|
|
326
|
+
}
|
|
327
|
+
commit(entries);
|
|
328
|
+
yield () => {
|
|
329
|
+
disposed = true;
|
|
330
|
+
for (const entry of held)
|
|
331
|
+
this.directory.delete(entry.provider);
|
|
332
|
+
held = [];
|
|
333
|
+
this.emitAdaptersUpdated();
|
|
334
|
+
};
|
|
335
|
+
}.bind(this), 'llm.registerConfigurableProviders()');
|
|
336
|
+
const handle = (() => void dispose());
|
|
337
|
+
handle.replace = (next) => {
|
|
338
|
+
if (disposed) {
|
|
339
|
+
throw new LlmError('this configurable-provider registration was disposed', 'REGISTRATION_DISPOSED');
|
|
340
|
+
}
|
|
341
|
+
commit(next);
|
|
342
|
+
};
|
|
343
|
+
return handle;
|
|
344
|
+
}
|
|
345
|
+
/**
|
|
346
|
+
* List every declared configurable provider, registered or dormant.
|
|
347
|
+
* @returns detached directory entries in declaration order.
|
|
348
|
+
*/
|
|
349
|
+
listConfigurableProviders() {
|
|
350
|
+
return [...this.directory.values()].map(entry => ({ ...entry, settingsPath: [...entry.settingsPath] }));
|
|
351
|
+
}
|
|
352
|
+
/**
|
|
353
|
+
* Offer to interrogate provider endpoints on behalf of the settings
|
|
354
|
+
* namespace this plugin owns. The namespace is the key because that is what
|
|
355
|
+
* a configuration surface already holds from the configurable-provider
|
|
356
|
+
* directory, and because a provider being *added* has no route to name yet.
|
|
357
|
+
* Disposed with the fiber.
|
|
358
|
+
* @param settingsNs - the namespace whose profiles this discovery serves.
|
|
359
|
+
* @param discover - interrogates one endpoint; must honor `request.signal`.
|
|
360
|
+
* @returns the disposer that withdraws the offer.
|
|
361
|
+
*/
|
|
362
|
+
registerModelDiscovery(settingsNs, discover) {
|
|
363
|
+
const dispose = this.ctx.effect(function* () {
|
|
364
|
+
if (settingsNs.length === 0) {
|
|
365
|
+
throw new LlmError('model discovery needs a non-empty settings namespace', 'INVALID_DISCOVERY');
|
|
366
|
+
}
|
|
367
|
+
if (this.discoveries.has(settingsNs)) {
|
|
368
|
+
throw new LlmError(`model discovery for "${settingsNs}" is already registered`, 'DUPLICATE_DISCOVERY');
|
|
369
|
+
}
|
|
370
|
+
this.discoveries.set(settingsNs, discover);
|
|
371
|
+
yield () => {
|
|
372
|
+
this.discoveries.delete(settingsNs);
|
|
373
|
+
};
|
|
374
|
+
}.bind(this), 'llm.registerModelDiscovery()');
|
|
375
|
+
return () => void dispose();
|
|
376
|
+
}
|
|
377
|
+
/**
|
|
378
|
+
* Interrogate one provider endpoint for the models it advertises. The
|
|
379
|
+
* request describes a draft, not a stored route, so nothing here reads or
|
|
380
|
+
* writes settings or credentials — the caller owns both, and the reply is
|
|
381
|
+
* candidate metadata a surface may offer for adoption.
|
|
382
|
+
* @param settingsNs - namespace whose registered discovery serves this draft.
|
|
383
|
+
* @param request - the endpoint, protocol, and one-shot credential to use.
|
|
384
|
+
* @returns the advertised models, deduplicated in endpoint order.
|
|
385
|
+
*/
|
|
386
|
+
async discoverModels(settingsNs, request) {
|
|
387
|
+
const discover = this.discoveries.get(settingsNs);
|
|
388
|
+
if (discover === undefined) {
|
|
389
|
+
throw new LlmError(`no model discovery is registered for "${settingsNs}"`, 'NO_DISCOVERY');
|
|
390
|
+
}
|
|
391
|
+
// One of the two identifies what to describe: a route the adapter knows, or
|
|
392
|
+
// an endpoint to ask. Neither leaves nothing to answer about.
|
|
393
|
+
if ((request.provider ?? '').length === 0 && (request.baseURL ?? '').length === 0) {
|
|
394
|
+
throw new LlmError('model discovery needs a provider route or a baseURL', 'INVALID_DISCOVERY');
|
|
395
|
+
}
|
|
396
|
+
const discovered = await discover(request);
|
|
397
|
+
const seen = new Set();
|
|
398
|
+
const models = [];
|
|
399
|
+
for (const model of discovered) {
|
|
400
|
+
if (typeof model.id !== 'string' || model.id.length === 0 || seen.has(model.id))
|
|
401
|
+
continue;
|
|
402
|
+
seen.add(model.id);
|
|
403
|
+
models.push({
|
|
404
|
+
id: model.id,
|
|
405
|
+
...model.name === undefined ? {} : { name: model.name },
|
|
406
|
+
...model.contextWindow === undefined ? {} : { contextWindow: model.contextWindow },
|
|
407
|
+
...model.maxTokens === undefined ? {} : { maxTokens: model.maxTokens },
|
|
408
|
+
});
|
|
409
|
+
}
|
|
410
|
+
return models;
|
|
411
|
+
}
|
|
412
|
+
/**
|
|
413
|
+
* Resolve the retry policy captured when one provider route was registered.
|
|
414
|
+
* @param provider - registered provider route to inspect.
|
|
415
|
+
* @returns the provider-owned policy, with normal defaults already resolved.
|
|
416
|
+
*/
|
|
417
|
+
providerRetryPolicy(provider) {
|
|
418
|
+
return this.registration(provider).retryPolicy;
|
|
419
|
+
}
|
|
420
|
+
/** Detach typed adapter-owned modality metadata. */
|
|
421
|
+
detachedModalities(modalities) {
|
|
422
|
+
return modalities === undefined ? undefined : [...modalities];
|
|
423
|
+
}
|
|
424
|
+
/**
|
|
425
|
+
* Discover models advertised by one registered provider. Catalog membership
|
|
426
|
+
* is advisory and never changes routing or request validation.
|
|
427
|
+
* @param provider - registered provider route to inspect.
|
|
428
|
+
* @returns detached model metadata in adapter-preferred order.
|
|
429
|
+
*/
|
|
430
|
+
async listModels(provider) {
|
|
431
|
+
const adapter = this.registration(provider).adapter;
|
|
432
|
+
const models = await adapter.listModels(provider);
|
|
433
|
+
const seen = new Set();
|
|
434
|
+
return models.map((model) => {
|
|
435
|
+
if (typeof model.provider !== 'string'
|
|
436
|
+
|| model.provider !== provider
|
|
437
|
+
|| typeof model.id !== 'string'
|
|
438
|
+
|| model.id.length === 0
|
|
439
|
+
|| typeof model.name !== 'string'
|
|
440
|
+
|| model.name.length === 0
|
|
441
|
+
|| (model.description !== undefined && typeof model.description !== 'string')
|
|
442
|
+
|| seen.has(model.id)) {
|
|
443
|
+
throw new LlmError(`adapter returned invalid or duplicate model metadata for provider "${provider}"`, 'INVALID_CATALOG');
|
|
444
|
+
}
|
|
445
|
+
seen.add(model.id);
|
|
446
|
+
const inputModalities = this.detachedModalities(model.inputModalities);
|
|
447
|
+
return {
|
|
448
|
+
provider: model.provider,
|
|
449
|
+
id: model.id,
|
|
450
|
+
name: model.name,
|
|
451
|
+
...model.description === undefined ? {} : { description: model.description },
|
|
452
|
+
...inputModalities === undefined ? {} : { inputModalities },
|
|
453
|
+
};
|
|
454
|
+
});
|
|
455
|
+
}
|
|
456
|
+
/**
|
|
457
|
+
* Resolve and validate all metadata from the adapter that owns one exact
|
|
458
|
+
* route. The result is detached from adapter-owned objects; catalog
|
|
459
|
+
* membership remains advisory and does not control request routing.
|
|
460
|
+
* @param provider - registered provider route to inspect.
|
|
461
|
+
* @param model - exact model id passed to the adapter.
|
|
462
|
+
* @param signal - optional cancellation for adapter-owned asynchronous lookup.
|
|
463
|
+
* @returns exact model identity plus available context and reasoning metadata.
|
|
464
|
+
*/
|
|
465
|
+
async resolveModelInfo(provider, model, signal) {
|
|
466
|
+
return this.resolveModelInfoFor(this.registration(provider), model, signal);
|
|
467
|
+
}
|
|
468
|
+
async resolveModelInfoFor(registration, model, signal) {
|
|
469
|
+
const provider = registration.provider.id;
|
|
470
|
+
const resolved = await registration.adapter.resolveModel(provider, model, signal);
|
|
471
|
+
if (typeof resolved.provider !== 'string'
|
|
472
|
+
|| resolved.provider !== provider
|
|
473
|
+
|| typeof resolved.id !== 'string'
|
|
474
|
+
|| resolved.id !== model
|
|
475
|
+
|| typeof resolved.name !== 'string'
|
|
476
|
+
|| resolved.name.length === 0
|
|
477
|
+
|| (resolved.description !== undefined && typeof resolved.description !== 'string')) {
|
|
478
|
+
throw new LlmError(`adapter returned invalid exact model metadata for provider "${provider}" model "${model}"`, 'INVALID_MODEL_INFO');
|
|
479
|
+
}
|
|
480
|
+
const context = resolved.context;
|
|
481
|
+
if (context !== undefined && (!Number.isInteger(context.contextWindow) || context.contextWindow <= 0)) {
|
|
482
|
+
throw new LlmError(`adapter returned invalid context metadata for provider "${provider}" model "${model}"`, 'INVALID_MODEL_CONTEXT');
|
|
483
|
+
}
|
|
484
|
+
// Capability metadata rides through: an explicit modality omission is
|
|
485
|
+
// negative capability downstream preflights act on (image admission).
|
|
486
|
+
const inputModalities = this.detachedModalities(resolved.inputModalities);
|
|
487
|
+
const defaultMaxTokens = resolved.defaultMaxTokens;
|
|
488
|
+
if (defaultMaxTokens !== undefined
|
|
489
|
+
&& (!Number.isSafeInteger(defaultMaxTokens) || defaultMaxTokens <= 0)) {
|
|
490
|
+
throw new LlmError(`adapter returned invalid default maxTokens for provider "${provider}" model "${model}"`, 'INVALID_MODEL_MAX_TOKENS');
|
|
491
|
+
}
|
|
492
|
+
const info = {
|
|
493
|
+
provider,
|
|
494
|
+
id: model,
|
|
495
|
+
name: resolved.name,
|
|
496
|
+
...resolved.description === undefined ? {} : { description: resolved.description },
|
|
497
|
+
...inputModalities === undefined ? {} : { inputModalities },
|
|
498
|
+
...context === undefined ? {} : { context: { contextWindow: context.contextWindow } },
|
|
499
|
+
...defaultMaxTokens === undefined ? {} : { defaultMaxTokens },
|
|
500
|
+
};
|
|
501
|
+
const reasoning = resolved.reasoning;
|
|
502
|
+
if (reasoning === undefined)
|
|
503
|
+
return info;
|
|
504
|
+
if (reasoning.efforts.length === 0) {
|
|
505
|
+
throw new LlmError(`adapter returned invalid reasoning metadata for provider "${provider}" model "${model}"`, 'INVALID_MODEL_REASONING');
|
|
506
|
+
}
|
|
507
|
+
const seen = new Set();
|
|
508
|
+
const efforts = reasoning.efforts.map((effort) => {
|
|
509
|
+
if (typeof effort.id !== 'string'
|
|
510
|
+
|| effort.id.length === 0
|
|
511
|
+
|| typeof effort.name !== 'string'
|
|
512
|
+
|| effort.name.length === 0
|
|
513
|
+
|| (effort.description !== undefined && typeof effort.description !== 'string')
|
|
514
|
+
|| seen.has(effort.id)) {
|
|
515
|
+
throw new LlmError(`adapter returned invalid or duplicate reasoning effort metadata for provider "${provider}" model "${model}"`, 'INVALID_MODEL_REASONING');
|
|
516
|
+
}
|
|
517
|
+
seen.add(effort.id);
|
|
518
|
+
return {
|
|
519
|
+
id: effort.id,
|
|
520
|
+
name: effort.name,
|
|
521
|
+
...effort.description === undefined ? {} : { description: effort.description },
|
|
522
|
+
};
|
|
523
|
+
});
|
|
524
|
+
if (reasoning.defaultEffort !== undefined && !seen.has(reasoning.defaultEffort)) {
|
|
525
|
+
throw new LlmError(`adapter returned an unknown default reasoning effort for provider "${provider}" model "${model}"`, 'INVALID_MODEL_REASONING');
|
|
526
|
+
}
|
|
527
|
+
return {
|
|
528
|
+
...info,
|
|
529
|
+
reasoning: {
|
|
530
|
+
efforts,
|
|
531
|
+
...reasoning.defaultEffort === undefined ? {} : { defaultEffort: reasoning.defaultEffort },
|
|
532
|
+
},
|
|
533
|
+
};
|
|
534
|
+
}
|
|
535
|
+
/**
|
|
536
|
+
* Validate a conversation call config against its exact model capability and
|
|
537
|
+
* materialize adapter-configured defaults. Unsupported explicit efforts
|
|
538
|
+
* reject before provider I/O; no clamping or aliasing is performed. This
|
|
539
|
+
* standalone query does not bind a later dispatch; use {@link prepareCall}
|
|
540
|
+
* when logging and streaming must share one adapter registration.
|
|
541
|
+
* @param config - provider/model route and optional request controls.
|
|
542
|
+
* @param signal - optional cancellation for adapter-owned capability lookup.
|
|
543
|
+
* @returns a detached config only when a default must be materialized.
|
|
544
|
+
*/
|
|
545
|
+
async resolveCallConfig(config, signal) {
|
|
546
|
+
return (await this.resolveCallFor(this.registration(config.provider), config, signal)).config;
|
|
547
|
+
}
|
|
548
|
+
async resolveCallFor(registration, config, signal) {
|
|
549
|
+
const info = await this.resolveModelInfoFor(registration, config.model, signal);
|
|
550
|
+
const defaulted = config.maxTokens === undefined && info.defaultMaxTokens !== undefined
|
|
551
|
+
? { ...config, maxTokens: info.defaultMaxTokens }
|
|
552
|
+
: config;
|
|
553
|
+
const reasoning = info.reasoning;
|
|
554
|
+
const requested = defaulted.reasoningEffort;
|
|
555
|
+
let resolvedConfig = defaulted;
|
|
556
|
+
if (reasoning === undefined) {
|
|
557
|
+
if (requested !== undefined) {
|
|
558
|
+
throw new LlmError(`provider "${config.provider}" model "${config.model}" does not support reasoning effort "${requested}"`, 'UNSUPPORTED_REASONING_EFFORT');
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
else {
|
|
562
|
+
const effective = requested ?? reasoning.defaultEffort;
|
|
563
|
+
if (effective !== undefined) {
|
|
564
|
+
if (!reasoning.efforts.some(effort => effort.id === effective)) {
|
|
565
|
+
throw new LlmError(`provider "${config.provider}" model "${config.model}" does not support reasoning effort "${effective}"`, 'UNSUPPORTED_REASONING_EFFORT');
|
|
566
|
+
}
|
|
567
|
+
if (requested !== effective)
|
|
568
|
+
resolvedConfig = { ...defaulted, reasoningEffort: effective };
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
return {
|
|
572
|
+
config: resolvedConfig,
|
|
573
|
+
...info.context === undefined ? {} : { context: info.context },
|
|
574
|
+
};
|
|
575
|
+
}
|
|
576
|
+
/**
|
|
577
|
+
* Resolve one call under its current adapter registration. The returned
|
|
578
|
+
* one-shot handle keeps that registration across header logging and dispatch,
|
|
579
|
+
* so HMR cannot combine one adapter's capability result with another adapter.
|
|
580
|
+
* @param config - provider/model route and optional request controls.
|
|
581
|
+
* @param signal - optional cancellation for adapter-owned capability lookup.
|
|
582
|
+
* @returns a prepared config and its registration-bound stream entry point.
|
|
583
|
+
*/
|
|
584
|
+
async prepareCall(config, signal) {
|
|
585
|
+
const registration = this.registration(config.provider);
|
|
586
|
+
const resolved = await this.resolveCallFor(registration, config, signal);
|
|
587
|
+
const resolvedConfig = deepFreeze(structuredClone(resolved.config));
|
|
588
|
+
const context = resolved.context === undefined
|
|
589
|
+
? undefined
|
|
590
|
+
: deepFreeze(structuredClone(resolved.context));
|
|
591
|
+
const adapterDefaults = deepFreeze({
|
|
592
|
+
...config.reasoningEffort === undefined && resolvedConfig.reasoningEffort !== undefined
|
|
593
|
+
? { reasoningEffort: true }
|
|
594
|
+
: {},
|
|
595
|
+
...config.maxTokens === undefined && resolvedConfig.maxTokens !== undefined
|
|
596
|
+
? { maxTokens: true }
|
|
597
|
+
: {},
|
|
598
|
+
});
|
|
599
|
+
let dispatched = false;
|
|
600
|
+
return Object.freeze({
|
|
601
|
+
config: resolvedConfig,
|
|
602
|
+
retryPolicy: registration.retryPolicy,
|
|
603
|
+
adapterDefaults,
|
|
604
|
+
...context === undefined ? {} : { context },
|
|
605
|
+
stream: (options) => {
|
|
606
|
+
if (dispatched) {
|
|
607
|
+
throw new LlmError('a prepared LLM call can only be dispatched once', 'INVALID_PREPARED_CALL');
|
|
608
|
+
}
|
|
609
|
+
if (!callConfigEquals(options, resolvedConfig)) {
|
|
610
|
+
throw new LlmError('prepared LLM call config changed before adapter dispatch', 'INVALID_PREPARED_CALL');
|
|
611
|
+
}
|
|
612
|
+
dispatched = true;
|
|
613
|
+
return this.streamWithRegistration(options, { registration, config: resolvedConfig });
|
|
614
|
+
},
|
|
615
|
+
});
|
|
616
|
+
}
|
|
617
|
+
registration(provider) {
|
|
618
|
+
const registration = this.adapters.get(provider);
|
|
619
|
+
if (!registration)
|
|
620
|
+
throw new LlmError(`no adapter registered for provider "${provider}"`, 'NO_ADAPTER');
|
|
621
|
+
return registration;
|
|
622
|
+
}
|
|
623
|
+
/** Remove replay state whose historical route is owned by another adapter. */
|
|
624
|
+
forAdapter(options, adapter) {
|
|
625
|
+
const messages = options.messages.map((message) => {
|
|
626
|
+
const source = message.source;
|
|
627
|
+
if (message.role !== 'assistant' || source.kind !== 'model' || source.replayState === undefined)
|
|
628
|
+
return message;
|
|
629
|
+
if (this.adapters.get(source.provider)?.adapter === adapter)
|
|
630
|
+
return message;
|
|
631
|
+
return freezeMessage({
|
|
632
|
+
...message,
|
|
633
|
+
source: { kind: 'model', provider: source.provider, model: source.model },
|
|
634
|
+
});
|
|
635
|
+
});
|
|
636
|
+
if (messages.every((message, index) => message === options.messages[index]))
|
|
637
|
+
return options;
|
|
638
|
+
const filtered = { ...options, messages };
|
|
639
|
+
return Object.isFrozen(options) ? deepFreeze(filtered) : filtered;
|
|
640
|
+
}
|
|
641
|
+
/**
|
|
642
|
+
* Final adapter boundary. Adapter selection, dispatch, iterator construction,
|
|
643
|
+
* and iteration failures become one terminal failure chunk. Middleware and
|
|
644
|
+
* downstream consumer failures remain thrown plugin or consumer errors.
|
|
645
|
+
*/
|
|
646
|
+
async *adapterStream(options, prepared) {
|
|
647
|
+
let iterator;
|
|
648
|
+
try {
|
|
649
|
+
const registration = prepared?.registration ?? this.registration(options.provider);
|
|
650
|
+
const resolvedConfig = prepared === undefined
|
|
651
|
+
? (await this.resolveCallFor(registration, options, options.signal)).config
|
|
652
|
+
: prepared.config;
|
|
653
|
+
if (prepared !== undefined && !callConfigEquals(options, resolvedConfig)) {
|
|
654
|
+
throw new LlmError('prepared LLM call config changed before adapter dispatch', 'INVALID_PREPARED_CALL');
|
|
655
|
+
}
|
|
656
|
+
const resolvedOptions = callConfigEquals(options, resolvedConfig)
|
|
657
|
+
? options
|
|
658
|
+
: Object.isFrozen(options)
|
|
659
|
+
? deepFreeze({ ...options, ...resolvedConfig })
|
|
660
|
+
: { ...options, ...resolvedConfig };
|
|
661
|
+
const adapter = registration.adapter;
|
|
662
|
+
const stream = adapter.stream(this.forAdapter(resolvedOptions, adapter));
|
|
663
|
+
iterator = stream[Symbol.asyncIterator]();
|
|
664
|
+
}
|
|
665
|
+
catch (error) {
|
|
666
|
+
yield adapterFailureChunk(error, options.signal);
|
|
667
|
+
return;
|
|
668
|
+
}
|
|
669
|
+
let completed = false;
|
|
670
|
+
try {
|
|
671
|
+
while (true) {
|
|
672
|
+
let item;
|
|
673
|
+
try {
|
|
674
|
+
const next = await iterator.next();
|
|
675
|
+
item = next.done
|
|
676
|
+
? { done: true }
|
|
677
|
+
: { done: false, value: next.value };
|
|
678
|
+
}
|
|
679
|
+
catch (error) {
|
|
680
|
+
completed = true;
|
|
681
|
+
yield adapterFailureChunk(error, options.signal);
|
|
682
|
+
return;
|
|
683
|
+
}
|
|
684
|
+
if (item.done) {
|
|
685
|
+
completed = true;
|
|
686
|
+
return;
|
|
687
|
+
}
|
|
688
|
+
// End the adapter-owned try before yielding: consumer/middleware
|
|
689
|
+
// failures resumed into this generator must remain thrown.
|
|
690
|
+
yield item.value;
|
|
691
|
+
}
|
|
692
|
+
}
|
|
693
|
+
finally {
|
|
694
|
+
if (!completed) {
|
|
695
|
+
const close = iterator.return?.bind(iterator);
|
|
696
|
+
if (close)
|
|
697
|
+
await close();
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
/**
|
|
702
|
+
* Stream one model call as raw chunks (token-level deltas). Replay state is
|
|
703
|
+
* retained only when the same adapter instance owns its historical provider
|
|
704
|
+
* and the target provider. Final adapter selection remains fixed through
|
|
705
|
+
* asynchronous exact-model resolution and dispatch. Adapter selection,
|
|
706
|
+
* dispatch, and iteration failures become terminal `error` or `aborted`
|
|
707
|
+
* finish chunks; middleware, nested-call, cleanup, and consumer failures
|
|
708
|
+
* remain thrown.
|
|
709
|
+
* @param options - the full request; `options.provider` selects the adapter.
|
|
710
|
+
* @returns the chunk stream, possibly wrapped by `llm/stream` listeners.
|
|
711
|
+
*/
|
|
712
|
+
stream(options) {
|
|
713
|
+
return this.streamWithRegistration(options);
|
|
714
|
+
}
|
|
715
|
+
streamWithRegistration(options, prepared) {
|
|
716
|
+
return this.ctx.waterfall(this, 'llm/stream', options, () => this.adapterStream(options, prepared));
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
/** Convert one adapter throw into the stream protocol's terminal outcome. */
|
|
720
|
+
function adapterFailureChunk(error, signal) {
|
|
721
|
+
const failure = normalizeLlmFailure(error);
|
|
722
|
+
return {
|
|
723
|
+
type: 'finish',
|
|
724
|
+
reason: signal?.aborted || failure.code === 'ABORTED'
|
|
725
|
+
? { kind: 'aborted', failure }
|
|
726
|
+
: { kind: 'error', failure },
|
|
727
|
+
};
|
|
728
|
+
}
|
|
729
|
+
export default LlmRuntime;
|
|
730
|
+
//# sourceMappingURL=index.js.map
|