genai-lite 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -1
- package/dist/index.d.ts +1 -1
- package/dist/llm/LLMService.d.ts +52 -1
- package/dist/llm/LLMService.js +464 -186
- package/dist/llm/clients/AnthropicClientAdapter.d.ts +6 -1
- package/dist/llm/clients/AnthropicClientAdapter.js +271 -87
- package/dist/llm/clients/GeminiClientAdapter.d.ts +8 -1
- package/dist/llm/clients/GeminiClientAdapter.js +208 -54
- package/dist/llm/clients/LlamaCppClientAdapter.d.ts +7 -1
- package/dist/llm/clients/LlamaCppClientAdapter.js +303 -125
- package/dist/llm/clients/MistralClientAdapter.d.ts +8 -1
- package/dist/llm/clients/MistralClientAdapter.js +216 -49
- package/dist/llm/clients/MockClientAdapter.d.ts +2 -1
- package/dist/llm/clients/MockClientAdapter.js +51 -0
- package/dist/llm/clients/OpenAIClientAdapter.d.ts +6 -1
- package/dist/llm/clients/OpenAIClientAdapter.js +200 -72
- package/dist/llm/clients/OpenRouterClientAdapter.d.ts +6 -1
- package/dist/llm/clients/OpenRouterClientAdapter.js +242 -109
- package/dist/llm/clients/types.d.ts +9 -1
- package/dist/llm/services/ModelResolver.d.ts +12 -1
- package/dist/llm/services/ModelResolver.js +4 -3
- package/dist/llm/types.d.ts +113 -0
- package/dist/llm/types.js +1 -0
- package/package.json +1 -1
package/dist/llm/LLMService.js
CHANGED
|
@@ -76,6 +76,93 @@ class LLMService {
|
|
|
76
76
|
this.logger.debug(`Found ${models.length} models for provider: ${providerId}`);
|
|
77
77
|
return [...models]; // Return a copy to prevent external modification
|
|
78
78
|
}
|
|
79
|
+
/**
|
|
80
|
+
* Gets static capability metadata for a provider/model pair without retrieving
|
|
81
|
+
* API keys, calling provider adapters, or performing network I/O.
|
|
82
|
+
*
|
|
83
|
+
* Unknown or unregistered models use the same fallback model resolution policy
|
|
84
|
+
* as sendMessage(), but their optional capabilities are reported as unknown.
|
|
85
|
+
*/
|
|
86
|
+
async getModelCapabilities(providerId, modelId) {
|
|
87
|
+
this.logger.debug(`LLMService.getModelCapabilities called for provider: ${providerId}, model: ${modelId}`);
|
|
88
|
+
const resolved = await this.modelResolver.resolve({ providerId, modelId }, { detectLocalCapabilities: false });
|
|
89
|
+
if (resolved.error) {
|
|
90
|
+
return resolved.error;
|
|
91
|
+
}
|
|
92
|
+
const resolvedProviderId = resolved.providerId;
|
|
93
|
+
const resolvedModelId = resolved.modelId;
|
|
94
|
+
const modelInfo = resolved.modelInfo;
|
|
95
|
+
const source = this.getCapabilitySource(resolvedProviderId, resolvedModelId);
|
|
96
|
+
return {
|
|
97
|
+
object: "model.capabilities",
|
|
98
|
+
provider: resolvedProviderId,
|
|
99
|
+
model: resolvedModelId,
|
|
100
|
+
modelInfo: { ...modelInfo },
|
|
101
|
+
structuredOutput: this.getStructuredOutputSupport(modelInfo, source),
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Convenience wrapper for checking structured-output support.
|
|
106
|
+
*
|
|
107
|
+
* Returns `unknown` when genai-lite has no explicit capability metadata. Callers
|
|
108
|
+
* should decide whether unknown is acceptable for their use case.
|
|
109
|
+
*/
|
|
110
|
+
async supportsStructuredOutput(providerId, modelId) {
|
|
111
|
+
const capabilities = await this.getModelCapabilities(providerId, modelId);
|
|
112
|
+
if (capabilities.object === "error") {
|
|
113
|
+
return capabilities;
|
|
114
|
+
}
|
|
115
|
+
return capabilities.structuredOutput;
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Preflights provider/model capability requirements for a request without
|
|
119
|
+
* retrieving API keys, calling provider adapters, or performing network I/O.
|
|
120
|
+
*
|
|
121
|
+
* This checks the effective settings after preset and model defaults are
|
|
122
|
+
* applied. Unknown/unspecified capabilities are reported as unknown and remain
|
|
123
|
+
* valid; explicitly unsupported capabilities return the same validation error
|
|
124
|
+
* shape as sendMessage() where possible.
|
|
125
|
+
*/
|
|
126
|
+
async validateRequestCapabilities(request) {
|
|
127
|
+
this.logger.debug(`LLMService.validateRequestCapabilities called with presetId: ${request.presetId}, provider: ${request.providerId}, model: ${request.modelId}`);
|
|
128
|
+
try {
|
|
129
|
+
const validation = await this.resolveAndValidateCapabilities(request, {
|
|
130
|
+
validateStructure: false,
|
|
131
|
+
detectLocalCapabilities: false,
|
|
132
|
+
});
|
|
133
|
+
if ("error" in validation) {
|
|
134
|
+
return {
|
|
135
|
+
...validation.error,
|
|
136
|
+
valid: false,
|
|
137
|
+
...(validation.capabilities && { capabilities: validation.capabilities }),
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
return {
|
|
141
|
+
object: "capability.validation",
|
|
142
|
+
valid: true,
|
|
143
|
+
provider: validation.context.providerId,
|
|
144
|
+
model: validation.context.modelId,
|
|
145
|
+
capabilities: validation.context.capabilities,
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
catch (error) {
|
|
149
|
+
this.logger.error("Error in LLMService.validateRequestCapabilities:", error);
|
|
150
|
+
return {
|
|
151
|
+
provider: request.providerId || request.presetId || 'unknown',
|
|
152
|
+
model: request.modelId || request.presetId || 'unknown',
|
|
153
|
+
error: {
|
|
154
|
+
message: error instanceof Error
|
|
155
|
+
? error.message
|
|
156
|
+
: "Unexpected error during capability preflight",
|
|
157
|
+
code: "UNKNOWN_ERROR",
|
|
158
|
+
type: "server_error",
|
|
159
|
+
providerError: error,
|
|
160
|
+
},
|
|
161
|
+
object: "error",
|
|
162
|
+
valid: false,
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
}
|
|
79
166
|
/**
|
|
80
167
|
* Sends a chat message to an LLM provider
|
|
81
168
|
*
|
|
@@ -85,116 +172,23 @@ class LLMService {
|
|
|
85
172
|
async sendMessage(request, callOptions) {
|
|
86
173
|
this.logger.info(`LLMService.sendMessage called with presetId: ${request.presetId}, provider: ${request.providerId}, model: ${request.modelId}`);
|
|
87
174
|
try {
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
return resolved.error;
|
|
92
|
-
}
|
|
93
|
-
const { providerId, modelId, modelInfo, settings: resolvedSettings } = resolved;
|
|
94
|
-
// Create a proper LLMChatRequest with resolved values
|
|
95
|
-
const resolvedRequest = {
|
|
96
|
-
...request,
|
|
97
|
-
providerId: providerId,
|
|
98
|
-
modelId: modelId,
|
|
99
|
-
};
|
|
100
|
-
// Validate basic request structure
|
|
101
|
-
const structureValidationResult = this.requestValidator.validateRequestStructure(resolvedRequest);
|
|
102
|
-
if (structureValidationResult) {
|
|
103
|
-
return structureValidationResult;
|
|
104
|
-
}
|
|
105
|
-
// Validate settings if provided
|
|
106
|
-
const combinedSettings = { ...resolvedSettings, ...request.settings };
|
|
107
|
-
if (combinedSettings) {
|
|
108
|
-
const settingsValidation = this.requestValidator.validateSettings(combinedSettings, providerId, modelId);
|
|
109
|
-
if (settingsValidation) {
|
|
110
|
-
return settingsValidation;
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
// Apply model-specific defaults and merge with user settings
|
|
114
|
-
const finalSettings = this.settingsManager.mergeSettingsForModel(modelId, providerId, combinedSettings, modelInfo);
|
|
115
|
-
// Validate reasoning settings for model capabilities
|
|
116
|
-
const reasoningValidation = this.requestValidator.validateReasoningSettings(modelInfo, finalSettings.reasoning, resolvedRequest);
|
|
117
|
-
if (reasoningValidation) {
|
|
118
|
-
return reasoningValidation;
|
|
175
|
+
const preparedResult = await this.prepareRequest(request, callOptions);
|
|
176
|
+
if ("error" in preparedResult) {
|
|
177
|
+
return preparedResult.error;
|
|
119
178
|
}
|
|
120
|
-
|
|
121
|
-
const structuredOutputValidation = this.requestValidator.validateStructuredOutputSettings(modelInfo, finalSettings.structuredOutput, resolvedRequest);
|
|
122
|
-
if (structuredOutputValidation) {
|
|
123
|
-
return structuredOutputValidation;
|
|
124
|
-
}
|
|
125
|
-
// Get provider info for parameter filtering
|
|
126
|
-
const providerInfo = (0, config_1.getProviderById)(providerId);
|
|
127
|
-
if (!providerInfo) {
|
|
128
|
-
return {
|
|
129
|
-
provider: providerId,
|
|
130
|
-
model: modelId,
|
|
131
|
-
error: {
|
|
132
|
-
message: `Provider information not found: ${providerId}`,
|
|
133
|
-
code: "PROVIDER_ERROR",
|
|
134
|
-
type: "server_error",
|
|
135
|
-
},
|
|
136
|
-
object: "error",
|
|
137
|
-
};
|
|
138
|
-
}
|
|
139
|
-
// Filter out unsupported parameters
|
|
140
|
-
const filteredSettings = this.settingsManager.filterUnsupportedParameters(finalSettings, modelInfo, providerInfo);
|
|
141
|
-
const internalRequest = {
|
|
142
|
-
...resolvedRequest,
|
|
143
|
-
settings: filteredSettings,
|
|
144
|
-
};
|
|
145
|
-
this.logger.debug(`Processing LLM request with (potentially filtered) settings:`, {
|
|
146
|
-
provider: providerId,
|
|
147
|
-
model: modelId,
|
|
148
|
-
settings: filteredSettings,
|
|
149
|
-
messageCount: resolvedRequest.messages.length,
|
|
150
|
-
});
|
|
151
|
-
// Get client adapter
|
|
152
|
-
const clientAdapter = this.adapterRegistry.getAdapter(providerId);
|
|
153
|
-
// Use ApiKeyProvider to get the API key and make the request
|
|
179
|
+
const prepared = preparedResult.prepared;
|
|
154
180
|
try {
|
|
155
|
-
|
|
156
|
-
if (!apiKey) {
|
|
157
|
-
return {
|
|
158
|
-
provider: providerId,
|
|
159
|
-
model: modelId,
|
|
160
|
-
error: {
|
|
161
|
-
message: `API key for provider '${providerId}' could not be retrieved. Ensure your ApiKeyProvider is configured correctly.`,
|
|
162
|
-
code: "API_KEY_ERROR",
|
|
163
|
-
type: "authentication_error",
|
|
164
|
-
},
|
|
165
|
-
object: "error",
|
|
166
|
-
};
|
|
167
|
-
}
|
|
168
|
-
// Validate API key format if adapter supports it
|
|
169
|
-
if (clientAdapter.validateApiKey && !clientAdapter.validateApiKey(apiKey)) {
|
|
170
|
-
return {
|
|
171
|
-
provider: providerId,
|
|
172
|
-
model: modelId,
|
|
173
|
-
error: {
|
|
174
|
-
message: `Invalid API key format for provider '${providerId}'. Please check your API key.`,
|
|
175
|
-
code: "INVALID_API_KEY",
|
|
176
|
-
type: "authentication_error",
|
|
177
|
-
},
|
|
178
|
-
object: "error",
|
|
179
|
-
};
|
|
180
|
-
}
|
|
181
|
-
this.logger.info(`Making LLM request with ${clientAdapter.constructor.name} for provider: ${providerId}`);
|
|
181
|
+
this.logger.info(`Making LLM request with ${prepared.clientAdapter.constructor.name} for provider: ${prepared.providerId}`);
|
|
182
182
|
// Unified retry layer: adapters never throw, so retry decisions are made on
|
|
183
183
|
// RETURNED failure responses. SDK-internal retries are disabled (maxRetries: 0
|
|
184
184
|
// at client construction) — this loop is the single owner of retry behavior.
|
|
185
|
-
const adapterOptions = {
|
|
186
|
-
...(callOptions?.signal && { signal: callOptions.signal }),
|
|
187
|
-
...((callOptions?.timeoutMs ?? this.defaultTimeoutMs) !== undefined && {
|
|
188
|
-
timeoutMs: callOptions?.timeoutMs ?? this.defaultTimeoutMs,
|
|
189
|
-
}),
|
|
190
|
-
};
|
|
191
185
|
const retryOnTimeout = this.retryOptions?.retryOnTimeout ?? true;
|
|
192
186
|
const retryableCodes = new Set([
|
|
193
187
|
types_1.ADAPTER_ERROR_CODES.RATE_LIMIT_EXCEEDED,
|
|
194
188
|
types_1.ADAPTER_ERROR_CODES.NETWORK_ERROR,
|
|
195
189
|
...(retryOnTimeout ? [types_1.ADAPTER_ERROR_CODES.REQUEST_TIMEOUT] : []),
|
|
196
190
|
]);
|
|
197
|
-
const result = await (0, withRetry_1.withRetry)(() => clientAdapter.sendMessage(internalRequest, apiKey, adapterOptions), (res) => {
|
|
191
|
+
const result = await (0, withRetry_1.withRetry)(() => prepared.clientAdapter.sendMessage(prepared.internalRequest, prepared.apiKey, prepared.adapterOptions), (res) => {
|
|
198
192
|
if (res.object !== 'error') {
|
|
199
193
|
return { retry: false };
|
|
200
194
|
}
|
|
@@ -212,99 +206,19 @@ class LLMService {
|
|
|
212
206
|
}),
|
|
213
207
|
signal: callOptions?.signal,
|
|
214
208
|
logger: this.logger,
|
|
215
|
-
label: `${providerId}/${modelId}`,
|
|
209
|
+
label: `${prepared.providerId}/${prepared.modelId}`,
|
|
216
210
|
});
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
const fallbackSettings = internalRequest.settings.thinkingTagFallback;
|
|
221
|
-
if (result.object === 'chat.completion' && fallbackSettings && fallbackSettings.enabled !== false) {
|
|
222
|
-
const tagName = fallbackSettings.tagName || 'thinking';
|
|
223
|
-
// Check if native reasoning is active for this request
|
|
224
|
-
const isNativeReasoningActive = modelInfo.reasoning?.supported === true &&
|
|
225
|
-
(internalRequest.settings.reasoning?.enabled === true ||
|
|
226
|
-
(modelInfo.reasoning?.enabledByDefault === true &&
|
|
227
|
-
internalRequest.settings.reasoning?.enabled !== false) ||
|
|
228
|
-
modelInfo.reasoning?.canDisable === false);
|
|
229
|
-
// Process the response - extract thinking tags if present
|
|
230
|
-
const choice = result.choices[0];
|
|
231
|
-
if (choice?.message?.content) {
|
|
232
|
-
const { extracted, remaining } = (0, parser_1.extractInitialTaggedContent)(choice.message.content, tagName);
|
|
233
|
-
if (extracted !== null) {
|
|
234
|
-
// Success: thinking tag found
|
|
235
|
-
this.logger.debug(`Extracted <${tagName}> block from response.`);
|
|
236
|
-
// Handle the edge case: append to existing reasoning if present (e.g., native reasoning + thinking tags)
|
|
237
|
-
const existingReasoning = choice.reasoning || '';
|
|
238
|
-
if (existingReasoning) {
|
|
239
|
-
// Use a neutral markdown header that works for any consumer (human or AI)
|
|
240
|
-
choice.reasoning = `${existingReasoning}\n\n#### Additional Reasoning\n\n${extracted}`;
|
|
241
|
-
}
|
|
242
|
-
else {
|
|
243
|
-
// No existing reasoning, just use the extracted content directly
|
|
244
|
-
choice.reasoning = extracted;
|
|
245
|
-
}
|
|
246
|
-
choice.message.content = remaining;
|
|
247
|
-
}
|
|
248
|
-
else {
|
|
249
|
-
// Tag was not found
|
|
250
|
-
// Enforce only if: (1) enforce: true AND (2) native reasoning is NOT active
|
|
251
|
-
if (fallbackSettings.enforce === true && !isNativeReasoningActive) {
|
|
252
|
-
const nativeReasoningCapable = modelInfo.reasoning?.supported === true;
|
|
253
|
-
return {
|
|
254
|
-
provider: providerId,
|
|
255
|
-
model: modelId,
|
|
256
|
-
error: {
|
|
257
|
-
message: `Model response missing required <${tagName}> tags.`,
|
|
258
|
-
code: "THINKING_TAGS_MISSING",
|
|
259
|
-
type: "validation_error",
|
|
260
|
-
param: nativeReasoningCapable && !isNativeReasoningActive
|
|
261
|
-
? `You disabled native reasoning for this model (${modelId}). ` +
|
|
262
|
-
`To see its reasoning, you must prompt it to use <${tagName}> tags. ` +
|
|
263
|
-
`Example: "Write your step-by-step reasoning in <${tagName}> tags before answering."`
|
|
264
|
-
: `This model (${modelId}) does not support native reasoning. ` +
|
|
265
|
-
`To get reasoning, you must prompt it to use <${tagName}> tags. ` +
|
|
266
|
-
`Example: "Write your step-by-step reasoning in <${tagName}> tags before answering."`,
|
|
267
|
-
},
|
|
268
|
-
object: "error",
|
|
269
|
-
partialResponse: {
|
|
270
|
-
id: result.id,
|
|
271
|
-
provider: result.provider,
|
|
272
|
-
model: result.model,
|
|
273
|
-
created: result.created,
|
|
274
|
-
choices: result.choices,
|
|
275
|
-
usage: result.usage
|
|
276
|
-
}
|
|
277
|
-
};
|
|
278
|
-
}
|
|
279
|
-
// If enforce: false or native reasoning is active, do nothing
|
|
280
|
-
}
|
|
281
|
-
}
|
|
211
|
+
const processedResult = this.postProcessResponse(result, prepared);
|
|
212
|
+
if (processedResult.object === "chat.completion") {
|
|
213
|
+
this.logger.info(`LLM request completed successfully for model: ${prepared.modelId}`);
|
|
282
214
|
}
|
|
283
|
-
|
|
284
|
-
// Parse JSON response when structuredOutput is enabled and autoParse is not disabled
|
|
285
|
-
const structuredOutputSettings = internalRequest.settings.structuredOutput;
|
|
286
|
-
if (result.object === 'chat.completion' && structuredOutputSettings &&
|
|
287
|
-
structuredOutputSettings.enabled !== false && structuredOutputSettings.autoParse !== false) {
|
|
288
|
-
for (const choice of result.choices) {
|
|
289
|
-
if (choice.message?.content) {
|
|
290
|
-
try {
|
|
291
|
-
choice.parsedContent = JSON.parse(choice.message.content);
|
|
292
|
-
}
|
|
293
|
-
catch (e) {
|
|
294
|
-
choice.parseError = `JSON parse failed: ${e instanceof Error ? e.message : String(e)}`;
|
|
295
|
-
this.logger.warn(`Failed to parse structured output for choice ${choice.index}: ${choice.parseError}`);
|
|
296
|
-
}
|
|
297
|
-
}
|
|
298
|
-
}
|
|
299
|
-
}
|
|
300
|
-
this.logger.info(`LLM request completed successfully for model: ${modelId}`);
|
|
301
|
-
return result;
|
|
215
|
+
return processedResult;
|
|
302
216
|
}
|
|
303
217
|
catch (error) {
|
|
304
218
|
this.logger.error("Error in LLMService.sendMessage:", error);
|
|
305
219
|
return {
|
|
306
|
-
provider: providerId,
|
|
307
|
-
model: modelId,
|
|
220
|
+
provider: prepared.providerId,
|
|
221
|
+
model: prepared.modelId,
|
|
308
222
|
error: {
|
|
309
223
|
message: error instanceof Error
|
|
310
224
|
? error.message
|
|
@@ -334,6 +248,370 @@ class LLMService {
|
|
|
334
248
|
};
|
|
335
249
|
}
|
|
336
250
|
}
|
|
251
|
+
/**
|
|
252
|
+
* Streams a chat message from an LLM provider.
|
|
253
|
+
*
|
|
254
|
+
* The final `complete` event contains the same normalized response shape as
|
|
255
|
+
* sendMessage(). Validation/API key failures and unsupported streaming adapters
|
|
256
|
+
* are yielded as a single `error` event.
|
|
257
|
+
*/
|
|
258
|
+
async *streamMessage(request, callOptions) {
|
|
259
|
+
this.logger.info(`LLMService.streamMessage called with presetId: ${request.presetId}, provider: ${request.providerId}, model: ${request.modelId}`);
|
|
260
|
+
try {
|
|
261
|
+
const preparedResult = await this.prepareRequest(request, callOptions);
|
|
262
|
+
if ("error" in preparedResult) {
|
|
263
|
+
yield { type: "error", error: preparedResult.error };
|
|
264
|
+
return;
|
|
265
|
+
}
|
|
266
|
+
const prepared = preparedResult.prepared;
|
|
267
|
+
if (!prepared.clientAdapter.streamMessage) {
|
|
268
|
+
yield {
|
|
269
|
+
type: "error",
|
|
270
|
+
error: {
|
|
271
|
+
provider: prepared.providerId,
|
|
272
|
+
model: prepared.modelId,
|
|
273
|
+
error: {
|
|
274
|
+
message: `Streaming is not supported for provider '${prepared.providerId}'.`,
|
|
275
|
+
code: "PROVIDER_ERROR",
|
|
276
|
+
type: "unsupported_feature",
|
|
277
|
+
},
|
|
278
|
+
object: "error",
|
|
279
|
+
},
|
|
280
|
+
};
|
|
281
|
+
return;
|
|
282
|
+
}
|
|
283
|
+
try {
|
|
284
|
+
for await (const event of prepared.clientAdapter.streamMessage(prepared.internalRequest, prepared.apiKey, prepared.adapterOptions)) {
|
|
285
|
+
if (event.type === "complete") {
|
|
286
|
+
const processedResult = this.postProcessResponse(event.response, prepared);
|
|
287
|
+
if (processedResult.object === "error") {
|
|
288
|
+
yield { type: "error", error: processedResult };
|
|
289
|
+
}
|
|
290
|
+
else {
|
|
291
|
+
yield { type: "complete", response: processedResult };
|
|
292
|
+
}
|
|
293
|
+
continue;
|
|
294
|
+
}
|
|
295
|
+
yield event;
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
catch (error) {
|
|
299
|
+
this.logger.error("Error in LLMService.streamMessage:", error);
|
|
300
|
+
yield {
|
|
301
|
+
type: "error",
|
|
302
|
+
error: {
|
|
303
|
+
provider: prepared.providerId,
|
|
304
|
+
model: prepared.modelId,
|
|
305
|
+
error: {
|
|
306
|
+
message: error instanceof Error
|
|
307
|
+
? error.message
|
|
308
|
+
: "An unknown error occurred during message streaming.",
|
|
309
|
+
code: "PROVIDER_ERROR",
|
|
310
|
+
type: "server_error",
|
|
311
|
+
providerError: error,
|
|
312
|
+
},
|
|
313
|
+
object: "error",
|
|
314
|
+
},
|
|
315
|
+
};
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
catch (error) {
|
|
319
|
+
this.logger.error("Error in LLMService.streamMessage (outer):", error);
|
|
320
|
+
yield {
|
|
321
|
+
type: "error",
|
|
322
|
+
error: {
|
|
323
|
+
provider: request.providerId || request.presetId || 'unknown',
|
|
324
|
+
model: request.modelId || request.presetId || 'unknown',
|
|
325
|
+
error: {
|
|
326
|
+
message: error instanceof Error
|
|
327
|
+
? error.message
|
|
328
|
+
: "An unknown error occurred.",
|
|
329
|
+
code: "UNEXPECTED_ERROR",
|
|
330
|
+
type: "server_error",
|
|
331
|
+
providerError: error,
|
|
332
|
+
},
|
|
333
|
+
object: "error",
|
|
334
|
+
},
|
|
335
|
+
};
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
async resolveAndValidateCapabilities(request, options) {
|
|
339
|
+
const resolved = await this.modelResolver.resolve(request, {
|
|
340
|
+
detectLocalCapabilities: options.detectLocalCapabilities,
|
|
341
|
+
});
|
|
342
|
+
if (resolved.error) {
|
|
343
|
+
return { error: resolved.error };
|
|
344
|
+
}
|
|
345
|
+
const providerId = resolved.providerId;
|
|
346
|
+
const modelId = resolved.modelId;
|
|
347
|
+
const modelInfo = resolved.modelInfo;
|
|
348
|
+
const source = this.getCapabilitySource(providerId, modelId);
|
|
349
|
+
const capabilities = this.buildModelCapabilities(modelInfo, source);
|
|
350
|
+
const resolvedRequest = {
|
|
351
|
+
...request,
|
|
352
|
+
providerId,
|
|
353
|
+
modelId,
|
|
354
|
+
messages: "messages" in request && Array.isArray(request.messages)
|
|
355
|
+
? request.messages
|
|
356
|
+
: [],
|
|
357
|
+
};
|
|
358
|
+
if (options.validateStructure) {
|
|
359
|
+
const structureValidationResult = this.requestValidator.validateRequestStructure(resolvedRequest);
|
|
360
|
+
if (structureValidationResult) {
|
|
361
|
+
return { error: structureValidationResult, capabilities };
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
const combinedSettings = {
|
|
365
|
+
...(resolved.settings || {}),
|
|
366
|
+
...(request.settings || {}),
|
|
367
|
+
};
|
|
368
|
+
const settingsValidation = this.requestValidator.validateSettings(combinedSettings, providerId, modelId);
|
|
369
|
+
if (settingsValidation) {
|
|
370
|
+
return { error: settingsValidation, capabilities };
|
|
371
|
+
}
|
|
372
|
+
const finalSettings = this.settingsManager.mergeSettingsForModel(modelId, providerId, combinedSettings, modelInfo);
|
|
373
|
+
const reasoningValidation = this.requestValidator.validateReasoningSettings(modelInfo, finalSettings.reasoning, resolvedRequest);
|
|
374
|
+
if (reasoningValidation) {
|
|
375
|
+
return { error: reasoningValidation, capabilities };
|
|
376
|
+
}
|
|
377
|
+
const structuredOutputValidation = this.requestValidator.validateStructuredOutputSettings(modelInfo, finalSettings.structuredOutput, resolvedRequest);
|
|
378
|
+
if (structuredOutputValidation) {
|
|
379
|
+
return { error: structuredOutputValidation, capabilities };
|
|
380
|
+
}
|
|
381
|
+
return {
|
|
382
|
+
context: {
|
|
383
|
+
providerId,
|
|
384
|
+
modelId,
|
|
385
|
+
modelInfo,
|
|
386
|
+
resolvedRequest,
|
|
387
|
+
finalSettings,
|
|
388
|
+
capabilities,
|
|
389
|
+
},
|
|
390
|
+
};
|
|
391
|
+
}
|
|
392
|
+
buildModelCapabilities(modelInfo, source) {
|
|
393
|
+
return {
|
|
394
|
+
structuredOutput: this.getStructuredOutputSupport(modelInfo, source),
|
|
395
|
+
};
|
|
396
|
+
}
|
|
397
|
+
getStructuredOutputSupport(modelInfo, source) {
|
|
398
|
+
if (modelInfo.structuredOutput === undefined) {
|
|
399
|
+
return {
|
|
400
|
+
status: "unknown",
|
|
401
|
+
source,
|
|
402
|
+
};
|
|
403
|
+
}
|
|
404
|
+
return {
|
|
405
|
+
status: modelInfo.structuredOutput.supported ? "supported" : "unsupported",
|
|
406
|
+
...(modelInfo.structuredOutput.strictMode !== undefined && {
|
|
407
|
+
strictMode: modelInfo.structuredOutput.strictMode,
|
|
408
|
+
}),
|
|
409
|
+
...(modelInfo.structuredOutput.notes && {
|
|
410
|
+
notes: modelInfo.structuredOutput.notes,
|
|
411
|
+
}),
|
|
412
|
+
source,
|
|
413
|
+
};
|
|
414
|
+
}
|
|
415
|
+
getCapabilitySource(providerId, modelId) {
|
|
416
|
+
return (0, config_1.getModelById)(modelId, providerId) ? "registry" : "fallback";
|
|
417
|
+
}
|
|
418
|
+
async prepareRequest(request, callOptions) {
|
|
419
|
+
const validation = await this.resolveAndValidateCapabilities(request, {
|
|
420
|
+
validateStructure: true,
|
|
421
|
+
detectLocalCapabilities: true,
|
|
422
|
+
});
|
|
423
|
+
if ("error" in validation) {
|
|
424
|
+
return { error: validation.error };
|
|
425
|
+
}
|
|
426
|
+
const { providerId, modelId, modelInfo, resolvedRequest, finalSettings, } = validation.context;
|
|
427
|
+
// Get provider info for parameter filtering
|
|
428
|
+
const providerInfo = (0, config_1.getProviderById)(providerId);
|
|
429
|
+
if (!providerInfo) {
|
|
430
|
+
return {
|
|
431
|
+
error: {
|
|
432
|
+
provider: providerId,
|
|
433
|
+
model: modelId,
|
|
434
|
+
error: {
|
|
435
|
+
message: `Provider information not found: ${providerId}`,
|
|
436
|
+
code: "PROVIDER_ERROR",
|
|
437
|
+
type: "server_error",
|
|
438
|
+
},
|
|
439
|
+
object: "error",
|
|
440
|
+
},
|
|
441
|
+
};
|
|
442
|
+
}
|
|
443
|
+
// Filter out unsupported parameters
|
|
444
|
+
const filteredSettings = this.settingsManager.filterUnsupportedParameters(finalSettings, modelInfo, providerInfo);
|
|
445
|
+
const internalRequest = {
|
|
446
|
+
...resolvedRequest,
|
|
447
|
+
settings: filteredSettings,
|
|
448
|
+
};
|
|
449
|
+
this.logger.debug(`Processing LLM request with (potentially filtered) settings:`, {
|
|
450
|
+
provider: providerId,
|
|
451
|
+
model: modelId,
|
|
452
|
+
settings: filteredSettings,
|
|
453
|
+
messageCount: resolvedRequest.messages.length,
|
|
454
|
+
});
|
|
455
|
+
// Get client adapter
|
|
456
|
+
const clientAdapter = this.adapterRegistry.getAdapter(providerId);
|
|
457
|
+
try {
|
|
458
|
+
const apiKey = await this.getApiKey(providerId);
|
|
459
|
+
if (!apiKey) {
|
|
460
|
+
return {
|
|
461
|
+
error: {
|
|
462
|
+
provider: providerId,
|
|
463
|
+
model: modelId,
|
|
464
|
+
error: {
|
|
465
|
+
message: `API key for provider '${providerId}' could not be retrieved. Ensure your ApiKeyProvider is configured correctly.`,
|
|
466
|
+
code: "API_KEY_ERROR",
|
|
467
|
+
type: "authentication_error",
|
|
468
|
+
},
|
|
469
|
+
object: "error",
|
|
470
|
+
},
|
|
471
|
+
};
|
|
472
|
+
}
|
|
473
|
+
// Validate API key format if adapter supports it
|
|
474
|
+
if (clientAdapter.validateApiKey && !clientAdapter.validateApiKey(apiKey)) {
|
|
475
|
+
return {
|
|
476
|
+
error: {
|
|
477
|
+
provider: providerId,
|
|
478
|
+
model: modelId,
|
|
479
|
+
error: {
|
|
480
|
+
message: `Invalid API key format for provider '${providerId}'. Please check your API key.`,
|
|
481
|
+
code: "INVALID_API_KEY",
|
|
482
|
+
type: "authentication_error",
|
|
483
|
+
},
|
|
484
|
+
object: "error",
|
|
485
|
+
},
|
|
486
|
+
};
|
|
487
|
+
}
|
|
488
|
+
const adapterOptions = {
|
|
489
|
+
...(callOptions?.signal && { signal: callOptions.signal }),
|
|
490
|
+
...((callOptions?.timeoutMs ?? this.defaultTimeoutMs) !== undefined && {
|
|
491
|
+
timeoutMs: callOptions?.timeoutMs ?? this.defaultTimeoutMs,
|
|
492
|
+
}),
|
|
493
|
+
};
|
|
494
|
+
return {
|
|
495
|
+
prepared: {
|
|
496
|
+
providerId,
|
|
497
|
+
modelId,
|
|
498
|
+
modelInfo,
|
|
499
|
+
resolvedRequest,
|
|
500
|
+
internalRequest,
|
|
501
|
+
clientAdapter,
|
|
502
|
+
apiKey,
|
|
503
|
+
adapterOptions,
|
|
504
|
+
},
|
|
505
|
+
};
|
|
506
|
+
}
|
|
507
|
+
catch (error) {
|
|
508
|
+
this.logger.error("Error preparing LLM request:", error);
|
|
509
|
+
return {
|
|
510
|
+
error: {
|
|
511
|
+
provider: providerId,
|
|
512
|
+
model: modelId,
|
|
513
|
+
error: {
|
|
514
|
+
message: error instanceof Error
|
|
515
|
+
? error.message
|
|
516
|
+
: "An unknown error occurred during request preparation.",
|
|
517
|
+
code: "PROVIDER_ERROR",
|
|
518
|
+
type: "server_error",
|
|
519
|
+
providerError: error,
|
|
520
|
+
},
|
|
521
|
+
object: "error",
|
|
522
|
+
},
|
|
523
|
+
};
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
postProcessResponse(result, prepared) {
|
|
527
|
+
if (result.object === "error") {
|
|
528
|
+
return result;
|
|
529
|
+
}
|
|
530
|
+
// Post-process for thinking tag fallback
|
|
531
|
+
// This feature extracts reasoning from XML tags when native reasoning is not active.
|
|
532
|
+
// It's a fallback mechanism for models without native reasoning or when native is disabled.
|
|
533
|
+
const fallbackSettings = prepared.internalRequest.settings.thinkingTagFallback;
|
|
534
|
+
if (fallbackSettings && fallbackSettings.enabled !== false) {
|
|
535
|
+
const tagName = fallbackSettings.tagName || 'thinking';
|
|
536
|
+
// Check if native reasoning is active for this request
|
|
537
|
+
const isNativeReasoningActive = prepared.modelInfo.reasoning?.supported === true &&
|
|
538
|
+
(prepared.internalRequest.settings.reasoning?.enabled === true ||
|
|
539
|
+
(prepared.modelInfo.reasoning?.enabledByDefault === true &&
|
|
540
|
+
prepared.internalRequest.settings.reasoning?.enabled !== false) ||
|
|
541
|
+
prepared.modelInfo.reasoning?.canDisable === false);
|
|
542
|
+
// Process the response - extract thinking tags if present
|
|
543
|
+
const choice = result.choices[0];
|
|
544
|
+
if (choice?.message?.content) {
|
|
545
|
+
const { extracted, remaining } = (0, parser_1.extractInitialTaggedContent)(choice.message.content, tagName);
|
|
546
|
+
if (extracted !== null) {
|
|
547
|
+
// Success: thinking tag found
|
|
548
|
+
this.logger.debug(`Extracted <${tagName}> block from response.`);
|
|
549
|
+
// Handle the edge case: append to existing reasoning if present (e.g., native reasoning + thinking tags)
|
|
550
|
+
const existingReasoning = choice.reasoning || '';
|
|
551
|
+
if (existingReasoning) {
|
|
552
|
+
// Use a neutral markdown header that works for any consumer (human or AI)
|
|
553
|
+
choice.reasoning = `${existingReasoning}\n\n#### Additional Reasoning\n\n${extracted}`;
|
|
554
|
+
}
|
|
555
|
+
else {
|
|
556
|
+
// No existing reasoning, just use the extracted content directly
|
|
557
|
+
choice.reasoning = extracted;
|
|
558
|
+
}
|
|
559
|
+
choice.message.content = remaining;
|
|
560
|
+
}
|
|
561
|
+
else {
|
|
562
|
+
// Tag was not found
|
|
563
|
+
// Enforce only if: (1) enforce: true AND (2) native reasoning is NOT active
|
|
564
|
+
if (fallbackSettings.enforce === true && !isNativeReasoningActive) {
|
|
565
|
+
const nativeReasoningCapable = prepared.modelInfo.reasoning?.supported === true;
|
|
566
|
+
return {
|
|
567
|
+
provider: prepared.providerId,
|
|
568
|
+
model: prepared.modelId,
|
|
569
|
+
error: {
|
|
570
|
+
message: `Model response missing required <${tagName}> tags.`,
|
|
571
|
+
code: "THINKING_TAGS_MISSING",
|
|
572
|
+
type: "validation_error",
|
|
573
|
+
param: nativeReasoningCapable && !isNativeReasoningActive
|
|
574
|
+
? `You disabled native reasoning for this model (${prepared.modelId}). ` +
|
|
575
|
+
`To see its reasoning, you must prompt it to use <${tagName}> tags. ` +
|
|
576
|
+
`Example: "Write your step-by-step reasoning in <${tagName}> tags before answering."`
|
|
577
|
+
: `This model (${prepared.modelId}) does not support native reasoning. ` +
|
|
578
|
+
`To get reasoning, you must prompt it to use <${tagName}> tags. ` +
|
|
579
|
+
`Example: "Write your step-by-step reasoning in <${tagName}> tags before answering."`,
|
|
580
|
+
},
|
|
581
|
+
object: "error",
|
|
582
|
+
partialResponse: {
|
|
583
|
+
id: result.id,
|
|
584
|
+
provider: result.provider,
|
|
585
|
+
model: result.model,
|
|
586
|
+
created: result.created,
|
|
587
|
+
choices: result.choices,
|
|
588
|
+
usage: result.usage
|
|
589
|
+
}
|
|
590
|
+
};
|
|
591
|
+
}
|
|
592
|
+
// If enforce: false or native reasoning is active, do nothing
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
// Post-process for structured output auto-parsing
|
|
597
|
+
// Parse JSON response when structuredOutput is enabled and autoParse is not disabled
|
|
598
|
+
const structuredOutputSettings = prepared.internalRequest.settings.structuredOutput;
|
|
599
|
+
if (structuredOutputSettings &&
|
|
600
|
+
structuredOutputSettings.enabled !== false && structuredOutputSettings.autoParse !== false) {
|
|
601
|
+
for (const choice of result.choices) {
|
|
602
|
+
if (choice.message?.content) {
|
|
603
|
+
try {
|
|
604
|
+
choice.parsedContent = JSON.parse(choice.message.content);
|
|
605
|
+
}
|
|
606
|
+
catch (e) {
|
|
607
|
+
choice.parseError = `JSON parse failed: ${e instanceof Error ? e.message : String(e)}`;
|
|
608
|
+
this.logger.warn(`Failed to parse structured output for choice ${choice.index}: ${choice.parseError}`);
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
return result;
|
|
614
|
+
}
|
|
337
615
|
/**
|
|
338
616
|
* Gets all configured model presets
|
|
339
617
|
*
|