@framers/agentos 0.12.3 → 0.12.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/agent.d.ts +7 -5
- package/dist/api/agent.d.ts.map +1 -1
- package/dist/api/agent.js.map +1 -1
- package/dist/api/generateObject.d.ts +5 -5
- package/dist/api/generateText.d.ts +173 -50
- package/dist/api/generateText.d.ts.map +1 -1
- package/dist/api/generateText.js +272 -81
- package/dist/api/generateText.js.map +1 -1
- package/dist/api/index.d.ts +6 -1
- package/dist/api/index.d.ts.map +1 -1
- package/dist/api/index.js +5 -1
- package/dist/api/index.js.map +1 -1
- package/dist/api/runtime/GMIChunkTransformer.d.ts +1 -1
- package/dist/api/runtime/GMIChunkTransformer.d.ts.map +1 -1
- package/dist/api/runtime/GMIChunkTransformer.js +7 -2
- package/dist/api/runtime/GMIChunkTransformer.js.map +1 -1
- package/dist/api/runtime/completionGateway.d.ts +98 -0
- package/dist/api/runtime/completionGateway.d.ts.map +1 -0
- package/dist/api/runtime/completionGateway.js +347 -0
- package/dist/api/runtime/completionGateway.js.map +1 -0
- package/dist/api/runtime/contextWindowFit.d.ts +49 -0
- package/dist/api/runtime/contextWindowFit.d.ts.map +1 -0
- package/dist/api/runtime/contextWindowFit.js +109 -0
- package/dist/api/runtime/contextWindowFit.js.map +1 -0
- package/dist/api/runtime/gatewayProviderManager.d.ts +37 -0
- package/dist/api/runtime/gatewayProviderManager.d.ts.map +1 -0
- package/dist/api/runtime/gatewayProviderManager.js +49 -0
- package/dist/api/runtime/gatewayProviderManager.js.map +1 -0
- package/dist/api/streamText.d.ts.map +1 -1
- package/dist/api/streamText.js +90 -6
- package/dist/api/streamText.js.map +1 -1
- package/dist/cognition/emergent/ComposableToolBuilder.d.ts.map +1 -1
- package/dist/cognition/emergent/ComposableToolBuilder.js +16 -8
- package/dist/cognition/emergent/ComposableToolBuilder.js.map +1 -1
- package/dist/cognition/emergent/EffectsStore.d.ts +58 -0
- package/dist/cognition/emergent/EffectsStore.d.ts.map +1 -0
- package/dist/cognition/emergent/EffectsStore.js +96 -0
- package/dist/cognition/emergent/EffectsStore.js.map +1 -0
- package/dist/cognition/emergent/EmergentCapabilityEngine.d.ts +22 -2
- package/dist/cognition/emergent/EmergentCapabilityEngine.d.ts.map +1 -1
- package/dist/cognition/emergent/EmergentCapabilityEngine.js +128 -19
- package/dist/cognition/emergent/EmergentCapabilityEngine.js.map +1 -1
- package/dist/cognition/emergent/EmergentJudge.d.ts +1 -1
- package/dist/cognition/emergent/EmergentJudge.d.ts.map +1 -1
- package/dist/cognition/emergent/EmergentToolRegistry.d.ts +12 -3
- package/dist/cognition/emergent/EmergentToolRegistry.d.ts.map +1 -1
- package/dist/cognition/emergent/EmergentToolRegistry.js +51 -4
- package/dist/cognition/emergent/EmergentToolRegistry.js.map +1 -1
- package/dist/cognition/emergent/ForgeToolMetaTool.d.ts.map +1 -1
- package/dist/cognition/emergent/ForgeToolMetaTool.js +4 -1
- package/dist/cognition/emergent/ForgeToolMetaTool.js.map +1 -1
- package/dist/cognition/emergent/SandboxedToolForge.d.ts +55 -51
- package/dist/cognition/emergent/SandboxedToolForge.d.ts.map +1 -1
- package/dist/cognition/emergent/SandboxedToolForge.js +103 -109
- package/dist/cognition/emergent/SandboxedToolForge.js.map +1 -1
- package/dist/cognition/emergent/ToolPackage.d.ts +2 -2
- package/dist/cognition/emergent/ToolPackage.d.ts.map +1 -1
- package/dist/cognition/emergent/broker/CapabilityBroker.d.ts +58 -0
- package/dist/cognition/emergent/broker/CapabilityBroker.d.ts.map +1 -0
- package/dist/cognition/emergent/broker/CapabilityBroker.js +288 -0
- package/dist/cognition/emergent/broker/CapabilityBroker.js.map +1 -0
- package/dist/cognition/emergent/broker/fetch.d.ts +44 -0
- package/dist/cognition/emergent/broker/fetch.d.ts.map +1 -0
- package/dist/cognition/emergent/broker/fetch.js +197 -0
- package/dist/cognition/emergent/broker/fetch.js.map +1 -0
- package/dist/cognition/emergent/broker/fs-read.d.ts +63 -0
- package/dist/cognition/emergent/broker/fs-read.d.ts.map +1 -0
- package/dist/cognition/emergent/broker/fs-read.js +133 -0
- package/dist/cognition/emergent/broker/fs-read.js.map +1 -0
- package/dist/cognition/emergent/broker/refusal.d.ts +10 -0
- package/dist/cognition/emergent/broker/refusal.d.ts.map +1 -0
- package/dist/cognition/emergent/broker/refusal.js +13 -0
- package/dist/cognition/emergent/broker/refusal.js.map +1 -0
- package/dist/cognition/emergent/ceiling.d.ts +75 -0
- package/dist/cognition/emergent/ceiling.d.ts.map +1 -0
- package/dist/cognition/emergent/ceiling.js +166 -0
- package/dist/cognition/emergent/ceiling.js.map +1 -0
- package/dist/cognition/emergent/index.d.ts +2 -0
- package/dist/cognition/emergent/index.d.ts.map +1 -1
- package/dist/cognition/emergent/index.js +1 -0
- package/dist/cognition/emergent/index.js.map +1 -1
- package/dist/cognition/emergent/types.d.ts +75 -2
- package/dist/cognition/emergent/types.d.ts.map +1 -1
- package/dist/cognition/emergent/types.js.map +1 -1
- package/dist/cognition/substrate/GMI.d.ts +21 -0
- package/dist/cognition/substrate/GMI.d.ts.map +1 -1
- package/dist/cognition/substrate/GMI.js +286 -88
- package/dist/cognition/substrate/GMI.js.map +1 -1
- package/dist/cognition/substrate/IGMI.d.ts +48 -1
- package/dist/cognition/substrate/IGMI.d.ts.map +1 -1
- package/dist/cognition/substrate/IGMI.js +4 -0
- package/dist/cognition/substrate/IGMI.js.map +1 -1
- package/dist/core/llm/providers/errors/ContextWindowExceededError.d.ts +22 -0
- package/dist/core/llm/providers/errors/ContextWindowExceededError.d.ts.map +1 -0
- package/dist/core/llm/providers/errors/ContextWindowExceededError.js +21 -0
- package/dist/core/llm/providers/errors/ContextWindowExceededError.js.map +1 -0
- package/dist/core/llm/providers/implementations/OpenRouterProvider.d.ts.map +1 -1
- package/dist/core/llm/providers/implementations/OpenRouterProvider.js +34 -13
- package/dist/core/llm/providers/implementations/OpenRouterProvider.js.map +1 -1
- package/dist/core/llm/routing/IModelRouter.d.ts +6 -0
- package/dist/core/llm/routing/IModelRouter.d.ts.map +1 -1
- package/dist/core/llm/routing/PolicyAwareRouter.d.ts +7 -0
- package/dist/core/llm/routing/PolicyAwareRouter.d.ts.map +1 -1
- package/dist/core/llm/routing/PolicyAwareRouter.js +12 -0
- package/dist/core/llm/routing/PolicyAwareRouter.js.map +1 -1
- package/dist/core/llm/routing/UncensoredModelCatalog.d.ts +37 -0
- package/dist/core/llm/routing/UncensoredModelCatalog.d.ts.map +1 -1
- package/dist/core/llm/routing/UncensoredModelCatalog.js +64 -74
- package/dist/core/llm/routing/UncensoredModelCatalog.js.map +1 -1
- package/dist/core/tools/ITool.d.ts +45 -0
- package/dist/core/tools/ITool.d.ts.map +1 -1
- package/dist/core/tools/ToolOrchestrator.d.ts.map +1 -1
- package/dist/core/tools/ToolOrchestrator.js +4 -8
- package/dist/core/tools/ToolOrchestrator.js.map +1 -1
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -0
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,347 @@
|
|
|
1
|
+
import { globalLLMProviderHealth } from '../../core/safety/LLMProviderHealthRegistry.js';
|
|
2
|
+
import { lowerZodToJsonSchema } from '../../orchestration/compiler/SchemaLowering.js';
|
|
3
|
+
import { resolveModelOption, resolveProvider, createProviderManager } from '../model.js';
|
|
4
|
+
import { buildPolicyAwareFallbackChain, fallbackHopOverrides, isContentPolicyRefusal, isRetryableError, usageOfError, } from '../generateText.js';
|
|
5
|
+
import { hostPolicyToRouteParams, mergeRequiredCapabilities } from './hostPolicy.js';
|
|
6
|
+
import { buildResponseFormatForProvider } from './responseFormatForProvider.js';
|
|
7
|
+
/** The tool format a provider expects; the same mapping as GMI.determineToolFormat. */
|
|
8
|
+
export function toolFormatFor(providerId) {
|
|
9
|
+
const pid = providerId.toLowerCase();
|
|
10
|
+
if (pid.includes('anthropic'))
|
|
11
|
+
return 'anthropic_tools';
|
|
12
|
+
if (pid.includes('google') || pid.includes('gemini'))
|
|
13
|
+
return 'google_function_calling';
|
|
14
|
+
return 'openai_functions';
|
|
15
|
+
}
|
|
16
|
+
function definedOnly(value) {
|
|
17
|
+
return Object.fromEntries(Object.entries(value).filter(([, v]) => v !== undefined));
|
|
18
|
+
}
|
|
19
|
+
/** The text of the last user message, its text parts joined when the content is multimodal. */
|
|
20
|
+
function lastUserText(messages) {
|
|
21
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
22
|
+
const m = messages[i];
|
|
23
|
+
if (m.role !== 'user')
|
|
24
|
+
continue;
|
|
25
|
+
if (typeof m.content === 'string')
|
|
26
|
+
return m.content;
|
|
27
|
+
if (Array.isArray(m.content)) {
|
|
28
|
+
return m.content
|
|
29
|
+
.map((part) => {
|
|
30
|
+
const { type, text } = part;
|
|
31
|
+
return type === 'text' && typeof text === 'string' ? text : '';
|
|
32
|
+
})
|
|
33
|
+
.filter((text) => text.length > 0)
|
|
34
|
+
.join('\n');
|
|
35
|
+
}
|
|
36
|
+
return '';
|
|
37
|
+
}
|
|
38
|
+
return '';
|
|
39
|
+
}
|
|
40
|
+
function toolNamesOf(tools) {
|
|
41
|
+
return (tools ?? [])
|
|
42
|
+
.map((t) => t.name ?? t.function?.name)
|
|
43
|
+
.filter((n) => typeof n === 'string' && n.length > 0);
|
|
44
|
+
}
|
|
45
|
+
/** The error a hop with an open circuit fails with, worded as streamText words it. */
|
|
46
|
+
function circuitOpenError(providerId) {
|
|
47
|
+
const stats = globalLLMProviderHealth.getStats(providerId);
|
|
48
|
+
return Object.assign(new Error(`[503] Provider '${providerId}' circuit open; cooldown ${stats?.cooldownRemainingMs ?? 0}ms`), { name: 'LLMProviderCircuitOpenError', httpStatus: 503 });
|
|
49
|
+
}
|
|
50
|
+
/** Primary selection, the same as streamText's: explicit model and provider, then the router when one is set. */
|
|
51
|
+
async function selectPrimary(route) {
|
|
52
|
+
let { providerId, modelId } = resolveModelOption({ provider: route.providerId, model: route.modelId }, 'text');
|
|
53
|
+
if (route.router) {
|
|
54
|
+
try {
|
|
55
|
+
const toolNames = toolNamesOf(route.tools);
|
|
56
|
+
const hostParams = hostPolicyToRouteParams(route.hostPolicy);
|
|
57
|
+
const params = {
|
|
58
|
+
taskHint: route.routerParams?.taskHint ?? lastUserText(route.messages),
|
|
59
|
+
...hostParams,
|
|
60
|
+
...route.routerParams,
|
|
61
|
+
optimizationPreference: route.routerParams?.optimizationPreference ?? hostParams.optimizationPreference ?? 'balanced',
|
|
62
|
+
requiredCapabilities: mergeRequiredCapabilities(hostParams.requiredCapabilities, route.routerParams?.requiredCapabilities, toolNames.length > 0 ? ['function_calling'] : undefined),
|
|
63
|
+
preferredProviderIds: route.routerParams?.preferredProviderIds ?? hostParams.preferredProviderIds,
|
|
64
|
+
policyTier: route.routerParams?.policyTier ?? route.policyTier ?? hostParams.policyTier,
|
|
65
|
+
};
|
|
66
|
+
const routed = await route.router.selectModel(params, undefined);
|
|
67
|
+
if (routed) {
|
|
68
|
+
providerId = routed.modelInfo?.providerId ?? providerId;
|
|
69
|
+
modelId = routed.modelId;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
catch (routerErr) {
|
|
73
|
+
console.warn('[agentos] Model router error, falling back to standard resolution:', routerErr);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
return { provider: providerId, model: modelId };
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* The HTTP status an in-band error names in a numeric code (Gemini's stream
|
|
80
|
+
* errors carry `code: 500` beside `type: 'INTERNAL'`), read as streamText reads
|
|
81
|
+
* it: a 401 or 403 inside a stream describes an upstream attempt, not this
|
|
82
|
+
* key, so it is not taken as a status.
|
|
83
|
+
*/
|
|
84
|
+
function statusOfChunkCode(code) {
|
|
85
|
+
return typeof code === 'number' && Number.isInteger(code) && code >= 400 && code <= 599 && code !== 401 && code !== 403
|
|
86
|
+
? code
|
|
87
|
+
: undefined;
|
|
88
|
+
}
|
|
89
|
+
/** An in-band provider error as an Error that `isRetryableError` and the health registry can judge (type, code, HTTP status kept). */
|
|
90
|
+
function errorFromChunk(e) {
|
|
91
|
+
const status = statusOfChunkCode(e.code);
|
|
92
|
+
return Object.assign(new Error(e.message), {
|
|
93
|
+
name: 'ProviderStreamError',
|
|
94
|
+
...(e.type !== undefined ? { type: e.type } : {}),
|
|
95
|
+
...(e.code !== undefined ? { code: e.code } : {}),
|
|
96
|
+
...(status !== undefined ? { httpStatus: status } : {}),
|
|
97
|
+
...(e.details !== undefined ? { details: e.details } : {}),
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
function terminalErrorChunk(resolution, error) {
|
|
101
|
+
return {
|
|
102
|
+
id: `gateway-error-${resolution.providerId}-${resolution.hop}`,
|
|
103
|
+
object: 'chat.completion.chunk',
|
|
104
|
+
created: Math.floor(Date.now() / 1000),
|
|
105
|
+
modelId: resolution.modelId,
|
|
106
|
+
choices: [],
|
|
107
|
+
isFinal: true,
|
|
108
|
+
error: { message: error.message, type: error.type ?? 'provider_error', details: { terminal: true, providerId: resolution.providerId, hop: resolution.hop } },
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
/** Text, tool activity or a lifted schema answer: the first such chunk ends the buffering. */
|
|
112
|
+
function carriesContent(chunk) {
|
|
113
|
+
if (chunk.responseTextDelta)
|
|
114
|
+
return true;
|
|
115
|
+
if (chunk.toolCallsDeltas && chunk.toolCallsDeltas.length > 0)
|
|
116
|
+
return true;
|
|
117
|
+
if ((chunk.choices?.[0]?.message?.tool_calls?.length ?? 0) > 0)
|
|
118
|
+
return true;
|
|
119
|
+
return chunk.structuredOutput !== undefined;
|
|
120
|
+
}
|
|
121
|
+
/** A usage report as a provider gives it: an object with a numeric token count. */
|
|
122
|
+
function asUsageReport(value) {
|
|
123
|
+
if (!value || typeof value !== 'object')
|
|
124
|
+
return undefined;
|
|
125
|
+
const { promptTokens, completionTokens, totalTokens } = value;
|
|
126
|
+
return [promptTokens, completionTokens, totalTokens].some((n) => typeof n === 'number' && Number.isFinite(n))
|
|
127
|
+
? value
|
|
128
|
+
: undefined;
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* What the provider billed an attempt that failed before any content: the
|
|
132
|
+
* error chunk's own usage, else the usage the error carries (`details.usage`,
|
|
133
|
+
* as a refused Claude turn reports it), else the last usage among the chunks
|
|
134
|
+
* buffered before the failure. Providers report the request's running total,
|
|
135
|
+
* so the latest report is the attempt's whole bill.
|
|
136
|
+
*/
|
|
137
|
+
function billedUsage(buffered, error, chunkUsage) {
|
|
138
|
+
const reported = asUsageReport(chunkUsage) ?? asUsageReport(usageOfError(error));
|
|
139
|
+
if (reported)
|
|
140
|
+
return reported;
|
|
141
|
+
for (let i = buffered.length - 1; i >= 0; i--) {
|
|
142
|
+
const usage = asUsageReport(buffered[i].usage);
|
|
143
|
+
if (usage)
|
|
144
|
+
return usage;
|
|
145
|
+
}
|
|
146
|
+
return undefined;
|
|
147
|
+
}
|
|
148
|
+
/** The provider-native schema payload for this hop, built as session.send builds it. */
|
|
149
|
+
function lowerForHop(resolution, schema, schemaName) {
|
|
150
|
+
const responseFormat = buildResponseFormatForProvider({
|
|
151
|
+
providerId: resolution.providerId,
|
|
152
|
+
modelId: resolution.modelId,
|
|
153
|
+
jsonSchema: lowerZodToJsonSchema(schema),
|
|
154
|
+
effectiveSchema: schema,
|
|
155
|
+
schemaName,
|
|
156
|
+
});
|
|
157
|
+
const marker = responseFormat;
|
|
158
|
+
return { responseFormat, toolName: marker?._agentosUseToolForStructuredOutput ? marker.tool?.name : undefined };
|
|
159
|
+
}
|
|
160
|
+
/** A streamed forced schema tool call becomes `structuredOutput` on the chunk, never a tool call the GMI would dispatch. */
|
|
161
|
+
function liftSchemaToolCall(chunk, toolName) {
|
|
162
|
+
const choice = chunk.choices?.[0];
|
|
163
|
+
const calls = choice?.message?.tool_calls;
|
|
164
|
+
if (!choice || !calls?.length)
|
|
165
|
+
return chunk;
|
|
166
|
+
const schemaCall = calls.find((call) => call.function?.name === toolName);
|
|
167
|
+
if (!schemaCall)
|
|
168
|
+
return chunk;
|
|
169
|
+
let structuredOutput = schemaCall.function.arguments;
|
|
170
|
+
if (typeof structuredOutput === 'string') {
|
|
171
|
+
try {
|
|
172
|
+
structuredOutput = JSON.parse(structuredOutput);
|
|
173
|
+
}
|
|
174
|
+
catch {
|
|
175
|
+
// Kept as the raw string; the session reports the parse failure.
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
const rest = calls.filter((call) => call !== schemaCall);
|
|
179
|
+
return {
|
|
180
|
+
...chunk,
|
|
181
|
+
choices: [{ ...choice, message: { ...choice.message, tool_calls: rest.length > 0 ? rest : undefined } }, ...chunk.choices.slice(1)],
|
|
182
|
+
structuredOutput,
|
|
183
|
+
};
|
|
184
|
+
}
|
|
185
|
+
export function createCompletionGateway(defaults = {}) {
|
|
186
|
+
async function resolve(input, after) {
|
|
187
|
+
const route = { ...defaults, ...definedOnly(input) };
|
|
188
|
+
let chain;
|
|
189
|
+
if (after) {
|
|
190
|
+
chain = after.chain;
|
|
191
|
+
}
|
|
192
|
+
else {
|
|
193
|
+
const primary = await selectPrimary(route);
|
|
194
|
+
const fallbacks = route.fallbackProviders ?? buildPolicyAwareFallbackChain(route.policyTier, primary.provider);
|
|
195
|
+
chain = [{ provider: primary.provider, model: primary.model }, ...fallbacks.map((entry) => ({ provider: entry.provider, model: entry.model, entry }))];
|
|
196
|
+
}
|
|
197
|
+
let lastError;
|
|
198
|
+
for (let hop = after ? after.hop + 1 : 0; hop < chain.length; hop++) {
|
|
199
|
+
const h = chain[hop];
|
|
200
|
+
// Set once credentials resolve: the provider that serves the hop (resolveProvider
|
|
201
|
+
// may serve anthropic through openrouter), whose circuit and health are the hop's.
|
|
202
|
+
let servingProviderId;
|
|
203
|
+
try {
|
|
204
|
+
const parsed = hop === 0 && h.model ? { providerId: h.provider, modelId: h.model } : resolveModelOption({ provider: h.provider, model: h.model }, 'text');
|
|
205
|
+
const resolved = hop === 0
|
|
206
|
+
? resolveProvider(parsed.providerId, parsed.modelId, { apiKey: route.apiKey, baseUrl: route.baseUrl })
|
|
207
|
+
: resolveProvider(parsed.providerId, parsed.modelId);
|
|
208
|
+
servingProviderId = resolved.providerId;
|
|
209
|
+
if (globalLLMProviderHealth.isOpen(resolved.providerId))
|
|
210
|
+
throw circuitOpenError(resolved.providerId);
|
|
211
|
+
const providerManager = await createProviderManager(resolved);
|
|
212
|
+
const info = await providerManager.getModelInfo(resolved.modelId, resolved.providerId);
|
|
213
|
+
const overrides = h.entry ? fallbackHopOverrides(route.callOptions ?? {}, h.entry) : undefined;
|
|
214
|
+
const resolution = {
|
|
215
|
+
providerId: resolved.providerId,
|
|
216
|
+
modelId: resolved.modelId,
|
|
217
|
+
hop,
|
|
218
|
+
maxContextTokens: info?.contextWindowSize ?? 8192,
|
|
219
|
+
capabilities: [...(info?.capabilities ?? [])],
|
|
220
|
+
toolFormat: toolFormatFor(resolved.providerId),
|
|
221
|
+
providerManager,
|
|
222
|
+
optionOverrides: overrides
|
|
223
|
+
? definedOnly({ maxTokens: overrides.maxTokens, effort: overrides.effort, cache: overrides.cache })
|
|
224
|
+
: {},
|
|
225
|
+
chain,
|
|
226
|
+
};
|
|
227
|
+
if (hop > 0) {
|
|
228
|
+
route.onFallback?.({ from: after?.providerId ?? chain[0].provider, to: resolved.providerId, hop, error: lastError ?? new Error('the previous hop failed') });
|
|
229
|
+
}
|
|
230
|
+
return resolution;
|
|
231
|
+
}
|
|
232
|
+
catch (err) {
|
|
233
|
+
lastError = err instanceof Error ? err : new Error(String(err));
|
|
234
|
+
// As in streamText: a configuration error (no credentials, unknown model)
|
|
235
|
+
// says nothing about the provider's health, and an open circuit is the
|
|
236
|
+
// registry's own verdict; neither is recorded.
|
|
237
|
+
if (servingProviderId && lastError.name !== 'LLMProviderCircuitOpenError') {
|
|
238
|
+
globalLLMProviderHealth.recordFailure(servingProviderId, lastError);
|
|
239
|
+
}
|
|
240
|
+
route.onHopFailure?.({ providerId: servingProviderId ?? h.provider, hop, error: lastError });
|
|
241
|
+
// The primary's configuration error (no credentials, unknown provider) is the caller's to see.
|
|
242
|
+
// A fallback leg that cannot start is skipped.
|
|
243
|
+
if (hop === 0 && !isRetryableError(lastError))
|
|
244
|
+
throw lastError;
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
return null;
|
|
248
|
+
}
|
|
249
|
+
function stream(resolution, messages, options, responseSchema, schemaName = 'response') {
|
|
250
|
+
let settle;
|
|
251
|
+
const outcome = new Promise((resolveOutcome) => {
|
|
252
|
+
settle = resolveOutcome;
|
|
253
|
+
});
|
|
254
|
+
let settled = false;
|
|
255
|
+
const finish = (o) => {
|
|
256
|
+
if (!settled) {
|
|
257
|
+
settled = true;
|
|
258
|
+
settle(o);
|
|
259
|
+
}
|
|
260
|
+
};
|
|
261
|
+
const failed = (error, usage, retryable = isRetryableError(error) || isContentPolicyRefusal(error)) => ({ kind: 'hopFailed', error, retryable, ...(usage ? { usage } : {}) });
|
|
262
|
+
const structured = responseSchema ? lowerForHop(resolution, responseSchema, schemaName) : undefined;
|
|
263
|
+
const callOptions = {
|
|
264
|
+
...options,
|
|
265
|
+
...resolution.optionOverrides,
|
|
266
|
+
...(structured
|
|
267
|
+
? { responseFormat: structured.responseFormat, tools: undefined, toolChoice: undefined }
|
|
268
|
+
: {}),
|
|
269
|
+
stream: true,
|
|
270
|
+
};
|
|
271
|
+
async function* run() {
|
|
272
|
+
let delivered = false;
|
|
273
|
+
let completed = false;
|
|
274
|
+
const buffer = [];
|
|
275
|
+
try {
|
|
276
|
+
const provider = resolution.providerManager.getProvider(resolution.providerId);
|
|
277
|
+
if (!provider) {
|
|
278
|
+
const error = Object.assign(new Error(`Provider '${resolution.providerId}' is not available.`), { name: 'ProviderInitializationError' });
|
|
279
|
+
globalLLMProviderHealth.recordFailure(resolution.providerId, error);
|
|
280
|
+
finish(failed(error, undefined));
|
|
281
|
+
return;
|
|
282
|
+
}
|
|
283
|
+
for await (const raw of provider.generateCompletionStream(resolution.modelId, messages, callOptions)) {
|
|
284
|
+
const chunk = structured?.toolName ? liftSchemaToolCall(raw, structured.toolName) : raw;
|
|
285
|
+
if (chunk.error) {
|
|
286
|
+
// An abort is the caller's own stop: it is never walked to another
|
|
287
|
+
// hop and says nothing about the provider's health (as in streamText).
|
|
288
|
+
const aborted = chunk.error.type === 'abort';
|
|
289
|
+
const error = errorFromChunk(chunk.error);
|
|
290
|
+
if (!aborted)
|
|
291
|
+
globalLLMProviderHealth.recordFailure(resolution.providerId, error);
|
|
292
|
+
if (!delivered) {
|
|
293
|
+
const usage = billedUsage(buffer, error, chunk.usage);
|
|
294
|
+
finish(aborted ? failed(error, usage, false) : failed(error, usage));
|
|
295
|
+
return;
|
|
296
|
+
}
|
|
297
|
+
// After content the step ends with the provider's error chunk. The
|
|
298
|
+
// outcome settles first, so a consumer that stops reading at the
|
|
299
|
+
// error still finds the attempt delivered.
|
|
300
|
+
completed = true;
|
|
301
|
+
finish({ kind: 'delivered' });
|
|
302
|
+
yield chunk;
|
|
303
|
+
return;
|
|
304
|
+
}
|
|
305
|
+
if (!delivered) {
|
|
306
|
+
buffer.push(chunk);
|
|
307
|
+
if (carriesContent(chunk)) {
|
|
308
|
+
delivered = true;
|
|
309
|
+
yield* buffer.splice(0);
|
|
310
|
+
}
|
|
311
|
+
continue;
|
|
312
|
+
}
|
|
313
|
+
yield chunk;
|
|
314
|
+
}
|
|
315
|
+
// A normal end with no content chunk (an empty reply, a content-filter stop,
|
|
316
|
+
// a usage-only trailing chunk) still reaches the GMI.
|
|
317
|
+
if (!delivered) {
|
|
318
|
+
delivered = true;
|
|
319
|
+
yield* buffer.splice(0);
|
|
320
|
+
}
|
|
321
|
+
completed = true;
|
|
322
|
+
globalLLMProviderHealth.recordSuccess(resolution.providerId);
|
|
323
|
+
finish({ kind: 'delivered' });
|
|
324
|
+
}
|
|
325
|
+
catch (err) {
|
|
326
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
327
|
+
globalLLMProviderHealth.recordFailure(resolution.providerId, error);
|
|
328
|
+
if (!delivered) {
|
|
329
|
+
finish(failed(error, billedUsage(buffer, error)));
|
|
330
|
+
return;
|
|
331
|
+
}
|
|
332
|
+
completed = true;
|
|
333
|
+
finish({ kind: 'delivered' });
|
|
334
|
+
yield terminalErrorChunk(resolution, error);
|
|
335
|
+
}
|
|
336
|
+
finally {
|
|
337
|
+
// Only an attempt that neither delivered nor failed gets here unsettled:
|
|
338
|
+
// the consumer stopped reading. `finish` ignores every later call.
|
|
339
|
+
if (!completed)
|
|
340
|
+
finish({ kind: 'abandoned' });
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
return { [Symbol.asyncIterator]: () => run(), outcome };
|
|
344
|
+
}
|
|
345
|
+
return { resolve, stream };
|
|
346
|
+
}
|
|
347
|
+
//# sourceMappingURL=completionGateway.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"completionGateway.js","sourceRoot":"","sources":["../../../src/api/runtime/completionGateway.ts"],"names":[],"mappings":"AAgBA,OAAO,EAAE,uBAAuB,EAAE,MAAM,gDAAgD,CAAC;AACzF,OAAO,EAAE,oBAAoB,EAAE,MAAM,gDAAgD,CAAC;AACtF,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACzF,OAAO,EACL,6BAA6B,EAC7B,oBAAoB,EACpB,sBAAsB,EACtB,gBAAgB,EAChB,YAAY,GAGb,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,uBAAuB,EAAE,yBAAyB,EAAsB,MAAM,iBAAiB,CAAC;AACzG,OAAO,EAAE,8BAA8B,EAAE,MAAM,gCAAgC,CAAC;AA2EhF,uFAAuF;AACvF,MAAM,UAAU,aAAa,CAAC,UAAkB;IAC9C,MAAM,GAAG,GAAG,UAAU,CAAC,WAAW,EAAE,CAAC;IACrC,IAAI,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC;QAAE,OAAO,iBAAiB,CAAC;IACxD,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAAE,OAAO,yBAAyB,CAAC;IACvF,OAAO,kBAAkB,CAAC;AAC5B,CAAC;AAED,SAAS,WAAW,CAAoC,KAAQ;IAC9D,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAe,CAAC;AACpG,CAAC;AAED,+FAA+F;AAC/F,SAAS,YAAY,CAAC,QAAuB;IAC3C,KAAK,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,MAAM,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;YAAE,SAAS;QAChC,IAAI,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ;YAAE,OAAO,CAAC,CAAC,OAAO,CAAC;QACpD,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7B,OAAO,CAAC,CAAC,OAAO;iBACb,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;gBACZ,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,IAA0C,CAAC;gBAClE,OAAO,IAAI,KAAK,MAAM,IAAI,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,CAAC,CAAC;iBACD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;iBACjC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChB,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,WAAW,CAAC,KAA+B;IAClD,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;SACjB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAE,CAAuB,CAAC,IAAI,IAAK,CAAsC,CAAC,QAAQ,EAAE,IAAI,CAAC;SACnG,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACvE,CAAC;AAED,sFAAsF;AACtF,SAAS,gBAAgB,CAAC,UAAkB;IAC1C,MAAM,KAAK,GAAG,uBAAuB,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;IAC3D,OAAO,MAAM,CAAC,MAAM,CAClB,IAAI,KAAK,CAAC,mBAAmB,UAAU,4BAA4B,KAAK,EAAE,mBAAmB,IAAI,CAAC,IAAI,CAAC,EACvG,EAAE,IAAI,EAAE,6BAA6B,EAAE,UAAU,EAAE,GAAG,EAAE,CACzD,CAAC;AACJ,CAAC;AAED,iHAAiH;AACjH,KAAK,UAAU,aAAa,CAAC,KAAsB;IACjD,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,kBAAkB,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,EAAE,MAAM,CAAC,CAAC;IAC/G,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC3C,MAAM,UAAU,GAAG,uBAAuB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAC7D,MAAM,MAAM,GAAG;gBACb,QAAQ,EAAE,KAAK,CAAC,YAAY,EAAE,QAAQ,IAAI,YAAY,CAAC,KAAK,CAAC,QAAQ,CAAC;gBACtE,GAAG,UAAU;gBACb,GAAG,KAAK,CAAC,YAAY;gBACrB,sBAAsB,EAAE,KAAK,CAAC,YAAY,EAAE,sBAAsB,IAAI,UAAU,CAAC,sBAAsB,IAAI,UAAU;gBACrH,oBAAoB,EAAE,yBAAyB,CAC7C,UAAU,CAAC,oBAAoB,EAC/B,KAAK,CAAC,YAAY,EAAE,oBAAoB,EACxC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,SAAS,CACxD;gBACD,oBAAoB,EAAE,KAAK,CAAC,YAAY,EAAE,oBAAoB,IAAI,UAAU,CAAC,oBAAoB;gBACjG,UAAU,EAAE,KAAK,CAAC,YAAY,EAAE,UAAU,IAAI,KAAK,CAAC,UAAU,IAAI,UAAU,CAAC,UAAU;aACpE,CAAC;YACtB,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YACjE,IAAI,MAAM,EAAE,CAAC;gBACX,UAAU,GAAG,MAAM,CAAC,SAAS,EAAE,UAAU,IAAI,UAAU,CAAC;gBACxD,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;YAC3B,CAAC;QACH,CAAC;QAAC,OAAO,SAAS,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,oEAAoE,EAAE,SAAS,CAAC,CAAC;QAChG,CAAC;IACH,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AAClD,CAAC;AAID;;;;;GAKG;AACH,SAAS,iBAAiB,CAAC,IAAa;IACtC,OAAO,OAAO,IAAI,KAAK,QAAQ,IAAI,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,GAAG,IAAI,IAAI,IAAI,GAAG,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG;QACrH,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,SAAS,CAAC;AAChB,CAAC;AAED,sIAAsI;AACtI,SAAS,cAAc,CAAC,CAAc;IACpC,MAAM,MAAM,GAAG,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACzC,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE;QACzC,IAAI,EAAE,qBAAqB;QAC3B,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjD,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjD,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvD,GAAG,CAAC,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC3D,CAAC,CAAC;AACL,CAAC;AAED,SAAS,kBAAkB,CAAC,UAAgC,EAAE,KAAY;IACxE,OAAO;QACL,EAAE,EAAE,iBAAiB,UAAU,CAAC,UAAU,IAAI,UAAU,CAAC,GAAG,EAAE;QAC9D,MAAM,EAAE,uBAAuB;QAC/B,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;QACtC,OAAO,EAAE,UAAU,CAAC,OAAO;QAC3B,OAAO,EAAE,EAAE;QACX,OAAO,EAAE,IAAI;QACb,KAAK,EAAE,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,IAAI,EAAG,KAA2B,CAAC,IAAI,IAAI,gBAAgB,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,UAAU,CAAC,UAAU,EAAE,GAAG,EAAE,UAAU,CAAC,GAAG,EAAE,EAAE;KACpL,CAAC;AACJ,CAAC;AAED,8FAA8F;AAC9F,SAAS,cAAc,CAAC,KAA8B;IACpD,IAAI,KAAK,CAAC,iBAAiB;QAAE,OAAO,IAAI,CAAC;IACzC,IAAI,KAAK,CAAC,eAAe,IAAI,KAAK,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAC3E,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5E,OAAQ,KAAwC,CAAC,gBAAgB,KAAK,SAAS,CAAC;AAClF,CAAC;AAED,mFAAmF;AACnF,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC1D,MAAM,EAAE,YAAY,EAAE,gBAAgB,EAAE,WAAW,EAAE,GAAG,KAAgC,CAAC;IACzF,OAAO,CAAC,YAAY,EAAE,gBAAgB,EAAE,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC3G,CAAC,CAAE,KAAoB;QACvB,CAAC,CAAC,SAAS,CAAC;AAChB,CAAC;AAED;;;;;;GAMG;AACH,SAAS,WAAW,CAAC,QAAgD,EAAE,KAAY,EAAE,UAAuB;IAC1G,MAAM,QAAQ,GAAG,aAAa,CAAC,UAAU,CAAC,IAAI,aAAa,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC;IACjF,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAC9B,KAAK,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,MAAM,KAAK,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAC/C,IAAI,KAAK;YAAE,OAAO,KAAK,CAAC;IAC1B,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,wFAAwF;AACxF,SAAS,WAAW,CAAC,UAAgC,EAAE,MAAe,EAAE,UAAkB;IACxF,MAAM,cAAc,GAAG,8BAA8B,CAAC;QACpD,UAAU,EAAE,UAAU,CAAC,UAAU;QACjC,OAAO,EAAE,UAAU,CAAC,OAAO;QAC3B,UAAU,EAAE,oBAAoB,CAAC,MAAM,CAAC;QACxC,eAAe,EAAE,MAAM;QACvB,UAAU;KACX,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,cAAwG,CAAC;IACxH,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,EAAE,kCAAkC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;AAClH,CAAC;AAED,4HAA4H;AAC5H,SAAS,kBAAkB,CAAC,KAA8B,EAAE,QAAgB;IAC1E,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;IAClC,MAAM,KAAK,GAAG,MAAM,EAAE,OAAO,EAAE,UAAU,CAAC;IAC1C,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,EAAE,MAAM;QAAE,OAAO,KAAK,CAAC;IAC5C,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC;IAC1E,IAAI,CAAC,UAAU;QAAE,OAAO,KAAK,CAAC;IAC9B,IAAI,gBAAgB,GAAY,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC;IAC9D,IAAI,OAAO,gBAAgB,KAAK,QAAQ,EAAE,CAAC;QACzC,IAAI,CAAC;YACH,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QAClD,CAAC;QAAC,MAAM,CAAC;YACP,iEAAiE;QACnE,CAAC;IACH,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;IACzD,OAAO;QACL,GAAG,KAAK;QACR,OAAO,EAAE,CAAC,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,EAAE,GAAG,MAAM,CAAC,OAAO,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,EAAE,EAAE,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACnI,gBAAgB;KAC0C,CAAC;AAC/D,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,WAAqC,EAAE;IAC7E,KAAK,UAAU,OAAO,CAAC,KAAsB,EAAE,KAA4B;QACzE,MAAM,KAAK,GAAoB,EAAE,GAAG,QAAQ,EAAE,GAAG,WAAW,CAAC,KAA2C,CAAC,EAAqB,CAAC;QAC/H,IAAI,KAAmC,CAAC;QACxC,IAAI,KAAK,EAAE,CAAC;YACV,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;QACtB,CAAC;aAAM,CAAC;YACN,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,KAAK,CAAC,CAAC;YAC3C,MAAM,SAAS,GAAG,KAAK,CAAC,iBAAiB,IAAI,6BAA6B,CAAC,KAAK,CAAC,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC/G,KAAK,GAAG,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,EAAE,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;QACzJ,CAAC;QACD,IAAI,SAA4B,CAAC;QACjC,KAAK,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,GAAG,KAAK,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;YACpE,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;YACrB,kFAAkF;YAClF,mFAAmF;YACnF,IAAI,iBAAqC,CAAC;YAC1C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,kBAAkB,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE,MAAM,CAAC,CAAC;gBAC1J,MAAM,QAAQ,GAAG,GAAG,KAAK,CAAC;oBACxB,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;oBACtG,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;gBACvD,iBAAiB,GAAG,QAAQ,CAAC,UAAU,CAAC;gBACxC,IAAI,uBAAuB,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC;oBAAE,MAAM,gBAAgB,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;gBACrG,MAAM,eAAe,GAAG,MAAM,qBAAqB,CAAC,QAAQ,CAAC,CAAC;gBAC9D,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC,YAAY,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;gBACvF,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,oBAAoB,CAAC,KAAK,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBAC/F,MAAM,UAAU,GAAyB;oBACvC,UAAU,EAAE,QAAQ,CAAC,UAAU;oBAC/B,OAAO,EAAE,QAAQ,CAAC,OAAO;oBACzB,GAAG;oBACH,gBAAgB,EAAE,IAAI,EAAE,iBAAiB,IAAI,IAAI;oBACjD,YAAY,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,YAAY,IAAI,EAAE,CAAC,CAAC;oBAC7C,UAAU,EAAE,aAAa,CAAC,QAAQ,CAAC,UAAU,CAAC;oBAC9C,eAAe;oBACf,eAAe,EAAE,SAAS;wBACxB,CAAC,CAAE,WAAW,CAAC,EAAE,SAAS,EAAE,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,CAAqC;wBACxI,CAAC,CAAC,EAAE;oBACN,KAAK;iBACN,CAAC;gBACF,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;oBACZ,KAAK,CAAC,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,EAAE,QAAQ,CAAC,UAAU,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,IAAI,IAAI,KAAK,CAAC,yBAAyB,CAAC,EAAE,CAAC,CAAC;gBAC/J,CAAC;gBACD,OAAO,UAAU,CAAC;YACpB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,SAAS,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;gBAChE,0EAA0E;gBAC1E,uEAAuE;gBACvE,+CAA+C;gBAC/C,IAAI,iBAAiB,IAAI,SAAS,CAAC,IAAI,KAAK,6BAA6B,EAAE,CAAC;oBAC1E,uBAAuB,CAAC,aAAa,CAAC,iBAAiB,EAAE,SAAS,CAAC,CAAC;gBACtE,CAAC;gBACD,KAAK,CAAC,YAAY,EAAE,CAAC,EAAE,UAAU,EAAE,iBAAiB,IAAI,CAAC,CAAC,QAAQ,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;gBAC7F,+FAA+F;gBAC/F,+CAA+C;gBAC/C,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC;oBAAE,MAAM,SAAS,CAAC;YACjE,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,SAAS,MAAM,CACb,UAAgC,EAChC,QAAuB,EACvB,OAA+B,EAC/B,cAAwB,EACxB,UAAU,GAAG,UAAU;QAEvB,IAAI,MAA6C,CAAC;QAClD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAoB,CAAC,cAAc,EAAE,EAAE;YAChE,MAAM,GAAG,cAAc,CAAC;QAC1B,CAAC,CAAC,CAAC;QACH,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,MAAM,MAAM,GAAG,CAAC,CAAoB,EAAQ,EAAE;YAC5C,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO,GAAG,IAAI,CAAC;gBACf,MAAM,CAAC,CAAC,CAAC,CAAC;YACZ,CAAC;QACH,CAAC,CAAC;QACF,MAAM,MAAM,GAAG,CACb,KAAY,EACZ,KAA6B,EAC7B,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,IAAI,sBAAsB,CAAC,KAAK,CAAC,EACjD,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAE/F,MAAM,UAAU,GAAG,cAAc,CAAC,CAAC,CAAC,WAAW,CAAC,UAAU,EAAE,cAAc,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACpG,MAAM,WAAW,GAA2B;YAC1C,GAAG,OAAO;YACV,GAAG,UAAU,CAAC,eAAe;YAC7B,GAAG,CAAC,UAAU;gBACZ,CAAC,CAAC,EAAE,cAAc,EAAE,UAAU,CAAC,cAA0D,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE;gBACpI,CAAC,CAAC,EAAE,CAAC;YACP,MAAM,EAAE,IAAI;SACb,CAAC;QAEF,KAAK,SAAS,CAAC,CAAC,GAAG;YACjB,IAAI,SAAS,GAAG,KAAK,CAAC;YACtB,IAAI,SAAS,GAAG,KAAK,CAAC;YACtB,MAAM,MAAM,GAA8B,EAAE,CAAC;YAC7C,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,UAAU,CAAC,eAAe,CAAC,WAAW,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;gBAC/E,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,aAAa,UAAU,CAAC,UAAU,qBAAqB,CAAC,EAAE,EAAE,IAAI,EAAE,6BAA6B,EAAE,CAAC,CAAC;oBACzI,uBAAuB,CAAC,aAAa,CAAC,UAAU,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;oBACpE,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;oBACjC,OAAO;gBACT,CAAC;gBACD,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,QAAQ,CAAC,wBAAwB,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,EAAE,WAAW,CAAC,EAAE,CAAC;oBACrG,MAAM,KAAK,GAAG,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;oBACxF,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;wBAChB,mEAAmE;wBACnE,uEAAuE;wBACvE,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,OAAO,CAAC;wBAC7C,MAAM,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;wBAC1C,IAAI,CAAC,OAAO;4BAAE,uBAAuB,CAAC,aAAa,CAAC,UAAU,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;wBAClF,IAAI,CAAC,SAAS,EAAE,CAAC;4BACf,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;4BACtD,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;4BACrE,OAAO;wBACT,CAAC;wBACD,mEAAmE;wBACnE,iEAAiE;wBACjE,2CAA2C;wBAC3C,SAAS,GAAG,IAAI,CAAC;wBACjB,MAAM,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;wBAC9B,MAAM,KAAK,CAAC;wBACZ,OAAO;oBACT,CAAC;oBACD,IAAI,CAAC,SAAS,EAAE,CAAC;wBACf,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;wBACnB,IAAI,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC;4BAC1B,SAAS,GAAG,IAAI,CAAC;4BACjB,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;wBAC1B,CAAC;wBACD,SAAS;oBACX,CAAC;oBACD,MAAM,KAAK,CAAC;gBACd,CAAC;gBACD,6EAA6E;gBAC7E,sDAAsD;gBACtD,IAAI,CAAC,SAAS,EAAE,CAAC;oBACf,SAAS,GAAG,IAAI,CAAC;oBACjB,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAC1B,CAAC;gBACD,SAAS,GAAG,IAAI,CAAC;gBACjB,uBAAuB,CAAC,aAAa,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;gBAC7D,MAAM,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;YAChC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;gBAClE,uBAAuB,CAAC,aAAa,CAAC,UAAU,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;gBACpE,IAAI,CAAC,SAAS,EAAE,CAAC;oBACf,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;oBAClD,OAAO;gBACT,CAAC;gBACD,SAAS,GAAG,IAAI,CAAC;gBACjB,MAAM,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;gBAC9B,MAAM,kBAAkB,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YAC9C,CAAC;oBAAS,CAAC;gBACT,yEAAyE;gBACzE,mEAAmE;gBACnE,IAAI,CAAC,SAAS;oBAAE,MAAM,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;QAED,OAAO,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC;IAC1D,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AAC7B,CAAC"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Whether a request fits a model's context window, measured the
|
|
3
|
+
* way the fallback walkers check a model before they send it.
|
|
4
|
+
*
|
|
5
|
+
* The input estimate is characters / 4 plus a 10% margin, over the text the
|
|
6
|
+
* request sends: message content (system blocks included), tool-call fields,
|
|
7
|
+
* and tool schemas or the tool text a prompt shim renders; on OpenRouter a
|
|
8
|
+
* `customModelParams.messages` or `.tools` override replaces them, as it does
|
|
9
|
+
* in the payload. Image parts and
|
|
10
|
+
* replayed reasoning fields (Anthropic thinking blocks and signatures, Gemini
|
|
11
|
+
* thought signatures) are not counted. The output allowance is the one the
|
|
12
|
+
* provider will send. A call that enables OpenRouter's context-compression
|
|
13
|
+
* plugin is taken as fitting: OpenRouter trims the prompt to the window.
|
|
14
|
+
*
|
|
15
|
+
* @module api/runtime/contextWindowFit
|
|
16
|
+
*/
|
|
17
|
+
/** The request one model would be sent. */
|
|
18
|
+
export interface ContextFitRequest {
|
|
19
|
+
provider: string;
|
|
20
|
+
model: string;
|
|
21
|
+
/** The messages as sent, system messages included. */
|
|
22
|
+
messages?: ReadonlyArray<unknown>;
|
|
23
|
+
/** A system prompt kept apart from `messages`, when the caller has one. */
|
|
24
|
+
system?: unknown;
|
|
25
|
+
prompt?: string;
|
|
26
|
+
/** The tool schemas as sent. */
|
|
27
|
+
tools?: unknown;
|
|
28
|
+
/** The call's `maxTokens` (a fallback hop's, after its overrides). */
|
|
29
|
+
maxTokens?: number;
|
|
30
|
+
customModelParams?: Record<string, unknown>;
|
|
31
|
+
}
|
|
32
|
+
/** The check's verdict and the numbers behind it. */
|
|
33
|
+
export interface ContextFit {
|
|
34
|
+
/**
|
|
35
|
+
* False only when the catalog knows the model's window, the request exceeds
|
|
36
|
+
* it, and the call has not enabled OpenRouter's context compression.
|
|
37
|
+
*/
|
|
38
|
+
fits: boolean;
|
|
39
|
+
contextWindow?: number;
|
|
40
|
+
estimatedInputTokens: number;
|
|
41
|
+
outputTokens: number;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Whether `request` fits its model's context window. A model the catalog
|
|
45
|
+
* does not list (every direct-provider model) is taken as fitting, and so is
|
|
46
|
+
* a call that enables OpenRouter's context compression.
|
|
47
|
+
*/
|
|
48
|
+
export declare function checkContextFit(request: ContextFitRequest): ContextFit;
|
|
49
|
+
//# sourceMappingURL=contextWindowFit.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"contextWindowFit.d.ts","sourceRoot":"","sources":["../../../src/api/runtime/contextWindowFit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAoBH,2CAA2C;AAC3C,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,sDAAsD;IACtD,QAAQ,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,CAAC;IAClC,2EAA2E;IAC3E,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gCAAgC;IAChC,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,sEAAsE;IACtE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC7C;AAED,qDAAqD;AACrD,MAAM,WAAW,UAAU;IACzB;;;OAGG;IACH,IAAI,EAAE,OAAO,CAAC;IACd,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,YAAY,EAAE,MAAM,CAAC;CACtB;AAmDD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,iBAAiB,GAAG,UAAU,CAqBtE"}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Whether a request fits a model's context window, measured the
|
|
3
|
+
* way the fallback walkers check a model before they send it.
|
|
4
|
+
*
|
|
5
|
+
* The input estimate is characters / 4 plus a 10% margin, over the text the
|
|
6
|
+
* request sends: message content (system blocks included), tool-call fields,
|
|
7
|
+
* and tool schemas or the tool text a prompt shim renders; on OpenRouter a
|
|
8
|
+
* `customModelParams.messages` or `.tools` override replaces them, as it does
|
|
9
|
+
* in the payload. Image parts and
|
|
10
|
+
* replayed reasoning fields (Anthropic thinking blocks and signatures, Gemini
|
|
11
|
+
* thought signatures) are not counted. The output allowance is the one the
|
|
12
|
+
* provider will send. A call that enables OpenRouter's context-compression
|
|
13
|
+
* plugin is taken as fitting: OpenRouter trims the prompt to the window.
|
|
14
|
+
*
|
|
15
|
+
* @module api/runtime/contextWindowFit
|
|
16
|
+
*/
|
|
17
|
+
import { clampMaxOutputTokens } from '../../core/llm/providers/model-output-limits.js';
|
|
18
|
+
import { findCatalogTextModel } from '../../core/llm/routing/UncensoredModelCatalog.js';
|
|
19
|
+
const CHARS_PER_TOKEN = 4;
|
|
20
|
+
const ESTIMATE_MARGIN = 1.1;
|
|
21
|
+
/** OpenRouterProvider's `max_tokens` for a call that sets none. */
|
|
22
|
+
const OPENROUTER_DEFAULT_MAX_TOKENS = 4096;
|
|
23
|
+
/** Keys whose values are not sent to the model as text. */
|
|
24
|
+
const UNCOUNTED_KEYS = new Set([
|
|
25
|
+
'role',
|
|
26
|
+
'type',
|
|
27
|
+
'cache_control',
|
|
28
|
+
'thinking',
|
|
29
|
+
'thinkingBlocks',
|
|
30
|
+
'signature',
|
|
31
|
+
'thoughtSignature',
|
|
32
|
+
]);
|
|
33
|
+
function textChars(value) {
|
|
34
|
+
if (typeof value === 'string')
|
|
35
|
+
return value.length;
|
|
36
|
+
if (typeof value === 'number' || typeof value === 'boolean')
|
|
37
|
+
return String(value).length;
|
|
38
|
+
if (Array.isArray(value))
|
|
39
|
+
return value.reduce((sum, item) => sum + textChars(item), 0);
|
|
40
|
+
if (value && typeof value === 'object') {
|
|
41
|
+
const record = value;
|
|
42
|
+
if (record.type === 'image_url' || record.type === 'image')
|
|
43
|
+
return 0;
|
|
44
|
+
let sum = 0;
|
|
45
|
+
for (const [key, item] of Object.entries(record)) {
|
|
46
|
+
if (!UNCOUNTED_KEYS.has(key))
|
|
47
|
+
sum += textChars(item);
|
|
48
|
+
}
|
|
49
|
+
return sum;
|
|
50
|
+
}
|
|
51
|
+
return 0;
|
|
52
|
+
}
|
|
53
|
+
/** The output tokens a provider sends for a call. */
|
|
54
|
+
function outputAllowance(provider, model, maxTokens, customModelParams) {
|
|
55
|
+
if (provider === 'openrouter') {
|
|
56
|
+
// OpenRouterProvider sends `max_tokens: clamp(model, maxTokens) ?? 4096`
|
|
57
|
+
// and spreads customModelParams after it, so an override wins.
|
|
58
|
+
const override = customModelParams?.max_tokens;
|
|
59
|
+
if (typeof override === 'number' && Number.isFinite(override) && override > 0)
|
|
60
|
+
return override;
|
|
61
|
+
return clampMaxOutputTokens(model, maxTokens) ?? OPENROUTER_DEFAULT_MAX_TOKENS;
|
|
62
|
+
}
|
|
63
|
+
return maxTokens ?? 0;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Whether the call enables OpenRouter's context-compression plugin, which
|
|
67
|
+
* trims a prompt over the window from the middle until it fits
|
|
68
|
+
* (`plugins: [{ id: 'context-compression' }]`; `enabled: false` turns it off).
|
|
69
|
+
*/
|
|
70
|
+
function compressesToFit(provider, customModelParams) {
|
|
71
|
+
if (provider !== 'openrouter')
|
|
72
|
+
return false;
|
|
73
|
+
const plugins = customModelParams?.plugins;
|
|
74
|
+
if (!Array.isArray(plugins))
|
|
75
|
+
return false;
|
|
76
|
+
return plugins.some((plugin) => {
|
|
77
|
+
if (!plugin || typeof plugin !== 'object')
|
|
78
|
+
return false;
|
|
79
|
+
const entry = plugin;
|
|
80
|
+
return entry.id === 'context-compression' && entry.enabled !== false;
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Whether `request` fits its model's context window. A model the catalog
|
|
85
|
+
* does not list (every direct-provider model) is taken as fitting, and so is
|
|
86
|
+
* a call that enables OpenRouter's context compression.
|
|
87
|
+
*/
|
|
88
|
+
export function checkContextFit(request) {
|
|
89
|
+
// OpenRouterProvider spreads customModelParams over its payload last, so a
|
|
90
|
+
// `messages` or `tools` override there is what the model is sent.
|
|
91
|
+
const overrides = request.provider === 'openrouter' ? request.customModelParams : undefined;
|
|
92
|
+
const sentTools = overrides?.tools !== undefined ? overrides.tools : request.tools;
|
|
93
|
+
const messageChars = Array.isArray(overrides?.messages)
|
|
94
|
+
? textChars(overrides?.messages)
|
|
95
|
+
: textChars(request.messages) + textChars(request.system) + textChars(request.prompt);
|
|
96
|
+
const chars = messageChars + (sentTools === undefined ? 0 : (JSON.stringify(sentTools) ?? '').length);
|
|
97
|
+
const estimatedInputTokens = Math.ceil((chars / CHARS_PER_TOKEN) * ESTIMATE_MARGIN);
|
|
98
|
+
const outputTokens = outputAllowance(request.provider, request.model, request.maxTokens, request.customModelParams);
|
|
99
|
+
const contextWindow = findCatalogTextModel(request.model, request.provider)?.contextWindow;
|
|
100
|
+
return {
|
|
101
|
+
fits: contextWindow === undefined ||
|
|
102
|
+
estimatedInputTokens + outputTokens <= contextWindow ||
|
|
103
|
+
compressesToFit(request.provider, request.customModelParams),
|
|
104
|
+
...(contextWindow !== undefined ? { contextWindow } : {}),
|
|
105
|
+
estimatedInputTokens,
|
|
106
|
+
outputTokens,
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
//# sourceMappingURL=contextWindowFit.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"contextWindowFit.js","sourceRoot":"","sources":["../../../src/api/runtime/contextWindowFit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,oBAAoB,EAAE,MAAM,iDAAiD,CAAC;AACvF,OAAO,EAAE,oBAAoB,EAAE,MAAM,kDAAkD,CAAC;AAExF,MAAM,eAAe,GAAG,CAAC,CAAC;AAC1B,MAAM,eAAe,GAAG,GAAG,CAAC;AAC5B,mEAAmE;AACnE,MAAM,6BAA6B,GAAG,IAAI,CAAC;AAC3C,2DAA2D;AAC3D,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC;IAC7B,MAAM;IACN,MAAM;IACN,eAAe;IACf,UAAU;IACV,gBAAgB;IAChB,WAAW;IACX,kBAAkB;CACnB,CAAC,CAAC;AA8BH,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC,MAAM,CAAC;IACnD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;IACzF,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,MAAM,CAAS,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/F,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,KAAgC,CAAC;QAChD,IAAI,MAAM,CAAC,IAAI,KAAK,WAAW,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO;YAAE,OAAO,CAAC,CAAC;QACrE,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACjD,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC;QACvD,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,qDAAqD;AACrD,SAAS,eAAe,CACtB,QAAgB,EAChB,KAAa,EACb,SAA6B,EAC7B,iBAAsD;IAEtD,IAAI,QAAQ,KAAK,YAAY,EAAE,CAAC;QAC9B,yEAAyE;QACzE,+DAA+D;QAC/D,MAAM,QAAQ,GAAG,iBAAiB,EAAE,UAAU,CAAC;QAC/C,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC;YAAE,OAAO,QAAQ,CAAC;QAC/F,OAAO,oBAAoB,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,6BAA6B,CAAC;IACjF,CAAC;IACD,OAAO,SAAS,IAAI,CAAC,CAAC;AACxB,CAAC;AAED;;;;GAIG;AACH,SAAS,eAAe,CAAC,QAAgB,EAAE,iBAAsD;IAC/F,IAAI,QAAQ,KAAK,YAAY;QAAE,OAAO,KAAK,CAAC;IAC5C,MAAM,OAAO,GAAG,iBAAiB,EAAE,OAAO,CAAC;IAC3C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC1C,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,MAAe,EAAE,EAAE;QACtC,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QACxD,MAAM,KAAK,GAAG,MAA6C,CAAC;QAC5D,OAAO,KAAK,CAAC,EAAE,KAAK,qBAAqB,IAAI,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC;IACvE,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,OAA0B;IACxD,2EAA2E;IAC3E,kEAAkE;IAClE,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,KAAK,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,SAAS,CAAC;IAC5F,MAAM,SAAS,GAAG,SAAS,EAAE,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IACnF,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC;QACrD,CAAC,CAAC,SAAS,CAAC,SAAS,EAAE,QAAQ,CAAC;QAChC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACxF,MAAM,KAAK,GAAG,YAAY,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;IACtG,MAAM,oBAAoB,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,GAAG,eAAe,CAAC,GAAG,eAAe,CAAC,CAAC;IACpF,MAAM,YAAY,GAAG,eAAe,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACpH,MAAM,aAAa,GAAG,oBAAoB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,aAAa,CAAC;IAC3F,OAAO;QACL,IAAI,EACF,aAAa,KAAK,SAAS;YAC3B,oBAAoB,GAAG,YAAY,IAAI,aAAa;YACpD,eAAe,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,iBAAiB,CAAC;QAC9D,GAAG,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzD,oBAAoB;QACpB,YAAY;KACb,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file gatewayProviderManager.ts
|
|
3
|
+
* The provider manager of a GMI whose turns run through a completion gateway.
|
|
4
|
+
*/
|
|
5
|
+
import type { AIModelProviderManager } from '../../core/llm/providers/AIModelProviderManager.js';
|
|
6
|
+
import type { IProvider, ModelInfo } from '../../core/llm/providers/IProvider.js';
|
|
7
|
+
import type { CompletionResolution } from './completionGateway.js';
|
|
8
|
+
/** The provider-manager calls the GMI path makes. */
|
|
9
|
+
type GmiProviderManagerCalls = Pick<AIModelProviderManager, 'getProvider' | 'getDefaultProvider' | 'getProviderForModel' | 'getModelInfo'>;
|
|
10
|
+
/**
|
|
11
|
+
* The provider-manager calls the GMI, MetapromptExecutor and LLMUtilityAI make
|
|
12
|
+
* (getProvider, getDefaultProvider, getProviderForModel, getModelInfo), answered
|
|
13
|
+
* from the hop that serves the turn. The GMI sets the resolution before it
|
|
14
|
+
* builds each prompt. Metaprompt and utility calls have no fallback of their own
|
|
15
|
+
* in this increment: on a fallback hop a call that names the primary's provider
|
|
16
|
+
* or model finds nothing, which the caller reports.
|
|
17
|
+
*/
|
|
18
|
+
export declare class GatewayProviderManager implements GmiProviderManagerCalls {
|
|
19
|
+
private resolution;
|
|
20
|
+
/** Present for code that checks a manager's readiness. */
|
|
21
|
+
readonly isInitialized = true;
|
|
22
|
+
/** Makes `resolution` the hop that answers every later call. */
|
|
23
|
+
setResolution(resolution: CompletionResolution): void;
|
|
24
|
+
private current;
|
|
25
|
+
/** The serving hop's provider when `providerId` names it; otherwise undefined. */
|
|
26
|
+
getProvider(providerId: string): IProvider | undefined;
|
|
27
|
+
/** The serving hop's provider. */
|
|
28
|
+
getDefaultProvider(): IProvider | undefined;
|
|
29
|
+
/** The serving hop's provider when `modelId` is the hop's model; otherwise undefined. */
|
|
30
|
+
getProviderForModel(modelId: string): IProvider | undefined;
|
|
31
|
+
/** Model info from the serving hop's manager, for the hop's provider unless `providerId` names another. */
|
|
32
|
+
getModelInfo(modelId: string, providerId?: string): Promise<ModelInfo | undefined>;
|
|
33
|
+
/** This adapter typed as `GMIBaseConfig.llmProviderManager` expects; it implements every method the GMI path calls. */
|
|
34
|
+
asProviderManager(): AIModelProviderManager;
|
|
35
|
+
}
|
|
36
|
+
export {};
|
|
37
|
+
//# sourceMappingURL=gatewayProviderManager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gatewayProviderManager.d.ts","sourceRoot":"","sources":["../../../src/api/runtime/gatewayProviderManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,oDAAoD,CAAC;AACjG,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,uCAAuC,CAAC;AAClF,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAEnE,qDAAqD;AACrD,KAAK,uBAAuB,GAAG,IAAI,CAAC,sBAAsB,EAAE,aAAa,GAAG,oBAAoB,GAAG,qBAAqB,GAAG,cAAc,CAAC,CAAC;AAE3I;;;;;;;GAOG;AACH,qBAAa,sBAAuB,YAAW,uBAAuB;IACpE,OAAO,CAAC,UAAU,CAAmC;IAErD,0DAA0D;IAC1D,SAAgB,aAAa,QAAQ;IAErC,gEAAgE;IACzD,aAAa,CAAC,UAAU,EAAE,oBAAoB,GAAG,IAAI;IAI5D,OAAO,CAAC,OAAO;IAOf,kFAAkF;IAC3E,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,SAAS;IAK7D,kCAAkC;IAC3B,kBAAkB,IAAI,SAAS,GAAG,SAAS;IAKlD,yFAAyF;IAClF,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,GAAG,SAAS;IAKlE,2GAA2G;IAC9F,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;IAK/F,uHAAuH;IAChH,iBAAiB,IAAI,sBAAsB;CAGnD"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The provider-manager calls the GMI, MetapromptExecutor and LLMUtilityAI make
|
|
3
|
+
* (getProvider, getDefaultProvider, getProviderForModel, getModelInfo), answered
|
|
4
|
+
* from the hop that serves the turn. The GMI sets the resolution before it
|
|
5
|
+
* builds each prompt. Metaprompt and utility calls have no fallback of their own
|
|
6
|
+
* in this increment: on a fallback hop a call that names the primary's provider
|
|
7
|
+
* or model finds nothing, which the caller reports.
|
|
8
|
+
*/
|
|
9
|
+
export class GatewayProviderManager {
|
|
10
|
+
constructor() {
|
|
11
|
+
/** Present for code that checks a manager's readiness. */
|
|
12
|
+
this.isInitialized = true;
|
|
13
|
+
}
|
|
14
|
+
/** Makes `resolution` the hop that answers every later call. */
|
|
15
|
+
setResolution(resolution) {
|
|
16
|
+
this.resolution = resolution;
|
|
17
|
+
}
|
|
18
|
+
current() {
|
|
19
|
+
if (!this.resolution) {
|
|
20
|
+
throw new Error('GatewayProviderManager: no resolution yet. The GMI resolves a hop before it builds a prompt; call setResolution first.');
|
|
21
|
+
}
|
|
22
|
+
return this.resolution;
|
|
23
|
+
}
|
|
24
|
+
/** The serving hop's provider when `providerId` names it; otherwise undefined. */
|
|
25
|
+
getProvider(providerId) {
|
|
26
|
+
const r = this.current();
|
|
27
|
+
return providerId === r.providerId ? r.providerManager.getProvider(r.providerId) : undefined;
|
|
28
|
+
}
|
|
29
|
+
/** The serving hop's provider. */
|
|
30
|
+
getDefaultProvider() {
|
|
31
|
+
const r = this.current();
|
|
32
|
+
return r.providerManager.getProvider(r.providerId);
|
|
33
|
+
}
|
|
34
|
+
/** The serving hop's provider when `modelId` is the hop's model; otherwise undefined. */
|
|
35
|
+
getProviderForModel(modelId) {
|
|
36
|
+
const r = this.current();
|
|
37
|
+
return modelId === r.modelId ? r.providerManager.getProvider(r.providerId) : undefined;
|
|
38
|
+
}
|
|
39
|
+
/** Model info from the serving hop's manager, for the hop's provider unless `providerId` names another. */
|
|
40
|
+
async getModelInfo(modelId, providerId) {
|
|
41
|
+
const r = this.current();
|
|
42
|
+
return r.providerManager.getModelInfo(modelId, providerId ?? r.providerId);
|
|
43
|
+
}
|
|
44
|
+
/** This adapter typed as `GMIBaseConfig.llmProviderManager` expects; it implements every method the GMI path calls. */
|
|
45
|
+
asProviderManager() {
|
|
46
|
+
return this;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=gatewayProviderManager.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gatewayProviderManager.js","sourceRoot":"","sources":["../../../src/api/runtime/gatewayProviderManager.ts"],"names":[],"mappings":"AAWA;;;;;;;GAOG;AACH,MAAM,OAAO,sBAAsB;IAAnC;QAGE,0DAA0D;QAC1C,kBAAa,GAAG,IAAI,CAAC;IA0CvC,CAAC;IAxCC,gEAAgE;IACzD,aAAa,CAAC,UAAgC;QACnD,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAEO,OAAO;QACb,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,wHAAwH,CAAC,CAAC;QAC5I,CAAC;QACD,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAED,kFAAkF;IAC3E,WAAW,CAAC,UAAkB;QACnC,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QACzB,OAAO,UAAU,KAAK,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC/F,CAAC;IAED,kCAAkC;IAC3B,kBAAkB;QACvB,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QACzB,OAAO,CAAC,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IACrD,CAAC;IAED,yFAAyF;IAClF,mBAAmB,CAAC,OAAe;QACxC,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QACzB,OAAO,OAAO,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACzF,CAAC;IAED,2GAA2G;IACpG,KAAK,CAAC,YAAY,CAAC,OAAe,EAAE,UAAmB;QAC5D,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QACzB,OAAO,CAAC,CAAC,eAAe,CAAC,YAAY,CAAC,OAAO,EAAE,UAAU,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC;IAC7E,CAAC;IAED,uHAAuH;IAChH,iBAAiB;QACtB,OAAO,IAAyC,CAAC;IACnD,CAAC;CACF"}
|