@juspay/neurolink 9.67.2 → 9.67.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -0
- package/dist/browser/neurolink.min.js +326 -326
- package/dist/lib/providers/litellm.d.ts +25 -32
- package/dist/lib/providers/litellm.js +132 -601
- package/dist/lib/providers/openaiChatCompletionsBase.d.ts +93 -0
- package/dist/lib/providers/openaiChatCompletionsBase.js +644 -0
- package/dist/lib/providers/openaiCompatible.d.ts +7 -63
- package/dist/lib/providers/openaiCompatible.js +27 -658
- package/dist/lib/types/openaiCompatible.d.ts +20 -0
- package/dist/providers/litellm.d.ts +25 -32
- package/dist/providers/litellm.js +132 -601
- package/dist/providers/openaiChatCompletionsBase.d.ts +93 -0
- package/dist/providers/openaiChatCompletionsBase.js +643 -0
- package/dist/providers/openaiCompatible.d.ts +7 -63
- package/dist/providers/openaiCompatible.js +27 -658
- package/dist/types/openaiCompatible.d.ts +20 -0
- package/package.json +1 -1
|
@@ -1,16 +1,7 @@
|
|
|
1
|
-
import { BaseProvider } from "../core/baseProvider.js";
|
|
2
|
-
import { DEFAULT_MAX_STEPS } from "../core/constants.js";
|
|
3
|
-
import { streamAnalyticsCollector } from "../core/streamAnalytics.js";
|
|
4
|
-
import { createProxyFetch } from "../proxy/proxyFetch.js";
|
|
5
1
|
import { AuthenticationError, InvalidModelError, NetworkError, ProviderError, RateLimitError, } from "../types/index.js";
|
|
6
2
|
import { logger } from "../utils/logger.js";
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import { composeAbortSignals, createTimeoutController, mergeAbortSignals, TimeoutError, } from "../utils/timeout.js";
|
|
10
|
-
import { emitToolEndFromStepFinish } from "../utils/toolEndEmitter.js";
|
|
11
|
-
import { resolveToolChoice } from "../utils/toolChoice.js";
|
|
12
|
-
import { transformToolExecutions } from "../utils/transformationUtils.js";
|
|
13
|
-
import { buildAPIError, buildBody, buildToolsForOpenAI, createChunkQueue, createDeferredAnalytics, mapNeuroLinkToolChoice, mergeUsage, messageBuilderToOpenAI, parseSSEStream, stringifyToolOutput, stripTrailingSlash, v3ResponseFormatToOpenAI, v3ToolChoiceToOpenAI, v3ToolsToOpenAI, } from "./openaiChatCompletionsClient.js";
|
|
3
|
+
import { TimeoutError } from "../utils/timeout.js";
|
|
4
|
+
import { OpenAIChatCompletionsProvider } from "./openaiChatCompletionsBase.js";
|
|
14
5
|
const FALLBACK_OPENAI_COMPATIBLE_MODEL = "gpt-3.5-turbo";
|
|
15
6
|
const getOpenAICompatibleConfig = () => {
|
|
16
7
|
const baseURL = process.env.OPENAI_COMPATIBLE_BASE_URL;
|
|
@@ -28,44 +19,26 @@ const getOpenAICompatibleConfig = () => {
|
|
|
28
19
|
const getDefaultOpenAICompatibleModel = () => {
|
|
29
20
|
return process.env.OPENAI_COMPATIBLE_MODEL || undefined;
|
|
30
21
|
};
|
|
31
|
-
// =============================================================================
|
|
32
|
-
// Direct HTTP client for OpenAI chat-completions.
|
|
33
|
-
//
|
|
34
|
-
// Wire-format converters, SSE parser, request builder, and error builder all
|
|
35
|
-
// live in ./openaiChatCompletionsClient.ts so providers that share the OpenAI
|
|
36
|
-
// chat-completions shape (litellm, etc.) can reuse them without duplication.
|
|
37
|
-
// Nothing in this module imports from "ai" or "@ai-sdk/provider" — the
|
|
38
|
-
// openai-compatible path is a clean cut.
|
|
39
|
-
// =============================================================================
|
|
40
|
-
// =============================================================================
|
|
41
|
-
// Provider
|
|
42
|
-
// =============================================================================
|
|
43
22
|
/**
|
|
44
23
|
* OpenAI Compatible Provider — direct HTTP, no AI SDK.
|
|
45
24
|
*
|
|
46
25
|
* Talks to any OpenAI chat-completions-shaped endpoint (LiteLLM, vLLM,
|
|
47
|
-
* OpenRouter, etc.).
|
|
48
|
-
*
|
|
26
|
+
* OpenRouter, etc.). All request/stream/tool-loop orchestration lives in
|
|
27
|
+
* `OpenAIChatCompletionsProvider`. This class just declares config and
|
|
28
|
+
* provider-specific error mapping.
|
|
49
29
|
*/
|
|
50
|
-
export class OpenAICompatibleProvider extends
|
|
51
|
-
config;
|
|
52
|
-
resolvedModel;
|
|
53
|
-
discoveredModel;
|
|
30
|
+
export class OpenAICompatibleProvider extends OpenAIChatCompletionsProvider {
|
|
54
31
|
constructor(modelName, sdk, _region, credentials) {
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
apiKey: credentials?.apiKey ?? envConfig.apiKey,
|
|
66
|
-
baseURL: credentials?.baseURL ?? envConfig.baseURL,
|
|
67
|
-
};
|
|
68
|
-
}
|
|
32
|
+
const resolved = credentials?.apiKey && credentials?.baseURL
|
|
33
|
+
? { apiKey: credentials.apiKey, baseURL: credentials.baseURL }
|
|
34
|
+
: (() => {
|
|
35
|
+
const env = getOpenAICompatibleConfig();
|
|
36
|
+
return {
|
|
37
|
+
apiKey: credentials?.apiKey ?? env.apiKey,
|
|
38
|
+
baseURL: credentials?.baseURL ?? env.baseURL,
|
|
39
|
+
};
|
|
40
|
+
})();
|
|
41
|
+
super("openai-compatible", modelName, sdk, resolved);
|
|
69
42
|
logger.debug("OpenAI Compatible Provider initialized", {
|
|
70
43
|
modelName: this.modelName,
|
|
71
44
|
provider: this.providerName,
|
|
@@ -78,168 +51,19 @@ export class OpenAICompatibleProvider extends BaseProvider {
|
|
|
78
51
|
getDefaultModel() {
|
|
79
52
|
return getDefaultOpenAICompatibleModel() || "";
|
|
80
53
|
}
|
|
81
|
-
|
|
82
|
-
* Abstract from BaseProvider — used by the parent's generate() path which
|
|
83
|
-
* still goes through `generateText`. Returns a thin LanguageModelV3-shaped
|
|
84
|
-
* object that delegates to the same HTTP helpers used by executeStream.
|
|
85
|
-
* Stays inside this file so no AI-SDK-named import is needed here.
|
|
86
|
-
*/
|
|
87
|
-
async getAISDKModel() {
|
|
88
|
-
const modelId = await this.resolveModelName();
|
|
89
|
-
return this.buildDelegatingModel(modelId);
|
|
90
|
-
}
|
|
91
|
-
async resolveModelName() {
|
|
92
|
-
if (this.resolvedModel) {
|
|
93
|
-
return this.resolvedModel;
|
|
94
|
-
}
|
|
95
|
-
const explicit = this.modelName || getDefaultOpenAICompatibleModel();
|
|
96
|
-
if (explicit && explicit.trim() !== "") {
|
|
97
|
-
this.resolvedModel = explicit;
|
|
98
|
-
// Propagate the resolved name into BaseProvider so telemetry/pricing/
|
|
99
|
-
// log metadata + StreamResult.model report the real model rather than
|
|
100
|
-
// the empty-string default the constructor was given.
|
|
101
|
-
if (this.modelName !== explicit) {
|
|
102
|
-
this.refreshHandlersForModel(explicit);
|
|
103
|
-
}
|
|
104
|
-
return explicit;
|
|
105
|
-
}
|
|
106
|
-
try {
|
|
107
|
-
const available = await this.getAvailableModels();
|
|
108
|
-
if (available.length > 0) {
|
|
109
|
-
this.discoveredModel = available[0];
|
|
110
|
-
this.resolvedModel = available[0];
|
|
111
|
-
// Same propagation for the auto-discovery branch.
|
|
112
|
-
this.refreshHandlersForModel(available[0]);
|
|
113
|
-
logger.info(`🔍 Auto-discovered model: ${available[0]} from ${available.length} available models`);
|
|
114
|
-
return available[0];
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
catch (err) {
|
|
118
|
-
logger.warn("Model auto-discovery failed, using fallback:", err);
|
|
119
|
-
}
|
|
120
|
-
this.resolvedModel = FALLBACK_OPENAI_COMPATIBLE_MODEL;
|
|
121
|
-
this.refreshHandlersForModel(FALLBACK_OPENAI_COMPATIBLE_MODEL);
|
|
54
|
+
getFallbackModelName() {
|
|
122
55
|
return FALLBACK_OPENAI_COMPATIBLE_MODEL;
|
|
123
56
|
}
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
const getTimeoutForOptions = (opts) => this.getTimeout((opts ?? {}));
|
|
135
|
-
return {
|
|
136
|
-
specificationVersion: "v3",
|
|
137
|
-
provider: "openai-compatible",
|
|
138
|
-
modelId,
|
|
139
|
-
supportedUrls: {},
|
|
140
|
-
doGenerate: async (options) => {
|
|
141
|
-
const messages = messageBuilderToOpenAI(options.prompt);
|
|
142
|
-
const body = buildBody({
|
|
143
|
-
modelId,
|
|
144
|
-
messages,
|
|
145
|
-
options: {
|
|
146
|
-
maxTokens: options.maxOutputTokens,
|
|
147
|
-
temperature: options.temperature,
|
|
148
|
-
topP: options.topP,
|
|
149
|
-
presencePenalty: options.presencePenalty,
|
|
150
|
-
frequencyPenalty: options.frequencyPenalty,
|
|
151
|
-
seed: options.seed,
|
|
152
|
-
stopSequences: options.stopSequences,
|
|
153
|
-
},
|
|
154
|
-
tools: v3ToolsToOpenAI(options.tools),
|
|
155
|
-
...(options.toolChoice
|
|
156
|
-
? { toolChoice: v3ToolChoiceToOpenAI(options.toolChoice) }
|
|
157
|
-
: {}),
|
|
158
|
-
streaming: false,
|
|
159
|
-
...(options.responseFormat
|
|
160
|
-
? {
|
|
161
|
-
responseFormat: v3ResponseFormatToOpenAI(options.responseFormat),
|
|
162
|
-
}
|
|
163
|
-
: {}),
|
|
164
|
-
});
|
|
165
|
-
// Compose a timeout-driven abort signal alongside any caller-provided
|
|
166
|
-
// one so slow upstreams can't hang the request indefinitely.
|
|
167
|
-
const timeoutController = createTimeoutController(getTimeoutForOptions(options), providerName, "generate");
|
|
168
|
-
const composedSignal = composeAbortSignals(options.abortSignal, timeoutController?.controller.signal);
|
|
169
|
-
let res;
|
|
170
|
-
try {
|
|
171
|
-
res = await fetchImpl(url, {
|
|
172
|
-
method: "POST",
|
|
173
|
-
headers: {
|
|
174
|
-
"Content-Type": "application/json",
|
|
175
|
-
Authorization: `Bearer ${apiKey}`,
|
|
176
|
-
},
|
|
177
|
-
body: JSON.stringify(body),
|
|
178
|
-
...(composedSignal ? { signal: composedSignal } : {}),
|
|
179
|
-
});
|
|
180
|
-
}
|
|
181
|
-
finally {
|
|
182
|
-
timeoutController?.cleanup();
|
|
183
|
-
}
|
|
184
|
-
if (!res.ok) {
|
|
185
|
-
throw await buildAPIError(url, body, res);
|
|
186
|
-
}
|
|
187
|
-
const json = (await res.json());
|
|
188
|
-
const choice = json.choices?.[0];
|
|
189
|
-
const text = (typeof choice?.message?.content === "string"
|
|
190
|
-
? choice.message.content
|
|
191
|
-
: "") ?? "";
|
|
192
|
-
const content = [];
|
|
193
|
-
if (text.length > 0) {
|
|
194
|
-
content.push({ type: "text", text });
|
|
195
|
-
}
|
|
196
|
-
// Forward tool calls so generateText() can drive its own tool loop.
|
|
197
|
-
for (const tc of choice?.message?.tool_calls ?? []) {
|
|
198
|
-
content.push({
|
|
199
|
-
type: "tool-call",
|
|
200
|
-
toolCallId: tc.id,
|
|
201
|
-
toolName: tc.function.name,
|
|
202
|
-
input: tc.function.arguments ?? "",
|
|
203
|
-
});
|
|
204
|
-
}
|
|
205
|
-
const rawFinish = choice?.finish_reason;
|
|
206
|
-
const unified = rawFinish === "length"
|
|
207
|
-
? "length"
|
|
208
|
-
: rawFinish === "tool_calls" || rawFinish === "function_call"
|
|
209
|
-
? "tool-calls"
|
|
210
|
-
: rawFinish === "content_filter"
|
|
211
|
-
? "content-filter"
|
|
212
|
-
: "stop";
|
|
213
|
-
return {
|
|
214
|
-
content,
|
|
215
|
-
finishReason: { unified, raw: rawFinish ?? "stop" },
|
|
216
|
-
usage: {
|
|
217
|
-
inputTokens: {
|
|
218
|
-
total: json.usage?.prompt_tokens,
|
|
219
|
-
noCache: json.usage?.prompt_tokens,
|
|
220
|
-
cacheRead: undefined,
|
|
221
|
-
cacheWrite: undefined,
|
|
222
|
-
},
|
|
223
|
-
outputTokens: {
|
|
224
|
-
total: json.usage?.completion_tokens,
|
|
225
|
-
text: json.usage?.completion_tokens,
|
|
226
|
-
reasoning: undefined,
|
|
227
|
-
},
|
|
228
|
-
},
|
|
229
|
-
warnings: [],
|
|
230
|
-
request: { body },
|
|
231
|
-
response: {
|
|
232
|
-
...(json.id ? { id: json.id } : {}),
|
|
233
|
-
...(json.model ? { modelId: json.model } : {}),
|
|
234
|
-
headers: {},
|
|
235
|
-
body: json,
|
|
236
|
-
},
|
|
237
|
-
};
|
|
238
|
-
},
|
|
239
|
-
doStream: () => {
|
|
240
|
-
throw new Error("openai-compatible: doStream is not implemented on the delegating model — the streaming path uses executeStream directly.");
|
|
241
|
-
},
|
|
242
|
-
};
|
|
57
|
+
getFallbackModels() {
|
|
58
|
+
return [
|
|
59
|
+
"gpt-4o",
|
|
60
|
+
"gpt-4o-mini",
|
|
61
|
+
"gpt-4-turbo",
|
|
62
|
+
FALLBACK_OPENAI_COMPATIBLE_MODEL,
|
|
63
|
+
"claude-3-5-sonnet",
|
|
64
|
+
"claude-3-haiku",
|
|
65
|
+
"gemini-pro",
|
|
66
|
+
];
|
|
243
67
|
}
|
|
244
68
|
formatProviderError(error) {
|
|
245
69
|
if (error instanceof TimeoutError) {
|
|
@@ -273,459 +97,4 @@ export class OpenAICompatibleProvider extends BaseProvider {
|
|
|
273
97
|
}
|
|
274
98
|
return new ProviderError(`OpenAI Compatible error: ${errorRecord?.message || "Unknown error"}`, "openai-compatible");
|
|
275
99
|
}
|
|
276
|
-
supportsTools() {
|
|
277
|
-
return true;
|
|
278
|
-
}
|
|
279
|
-
/**
|
|
280
|
-
* Streaming path — drives the OpenAI endpoint directly. No streamText,
|
|
281
|
-
* no AI SDK orchestrator. Tool calls, multi-step loops, telemetry,
|
|
282
|
-
* abort handling all inline.
|
|
283
|
-
*/
|
|
284
|
-
async executeStream(options, _analysisSchema) {
|
|
285
|
-
this.validateStreamOptions(options);
|
|
286
|
-
const startTime = Date.now();
|
|
287
|
-
const timeout = this.getTimeout(options);
|
|
288
|
-
const timeoutController = createTimeoutController(timeout, this.providerName, "stream");
|
|
289
|
-
// Consumer-driven abort: fires when the async iterator is closed early
|
|
290
|
-
// (caller breaks out of `for await`, returns from the loop, etc.).
|
|
291
|
-
// Without this the background `loopPromise` keeps reading SSE and
|
|
292
|
-
// running tools indefinitely, growing chunkQueue + leaking spend.
|
|
293
|
-
const consumerAbortController = new AbortController();
|
|
294
|
-
const abortSignal = mergeAbortSignals([
|
|
295
|
-
options.abortSignal,
|
|
296
|
-
timeoutController?.controller.signal,
|
|
297
|
-
consumerAbortController.signal,
|
|
298
|
-
]).signal;
|
|
299
|
-
let modelId;
|
|
300
|
-
let toolsRecord;
|
|
301
|
-
let openAITools;
|
|
302
|
-
let openAIToolChoice;
|
|
303
|
-
let conversation;
|
|
304
|
-
try {
|
|
305
|
-
modelId = await this.resolveModelName();
|
|
306
|
-
const shouldUseTools = !options.disableTools && this.supportsTools();
|
|
307
|
-
toolsRecord = shouldUseTools
|
|
308
|
-
? options.tools || (await this.getAllTools())
|
|
309
|
-
: {};
|
|
310
|
-
openAITools = shouldUseTools
|
|
311
|
-
? buildToolsForOpenAI(toolsRecord)
|
|
312
|
-
: undefined;
|
|
313
|
-
openAIToolChoice = mapNeuroLinkToolChoice(resolveToolChoice(options, toolsRecord, shouldUseTools));
|
|
314
|
-
const initialMessages = await this.buildMessagesForStream(options);
|
|
315
|
-
conversation = messageBuilderToOpenAI(initialMessages);
|
|
316
|
-
}
|
|
317
|
-
catch (setupErr) {
|
|
318
|
-
// Anything thrown before loopPromise is created (resolveModelName, tool
|
|
319
|
-
// discovery, buildMessagesForStream) would otherwise leave the timeout
|
|
320
|
-
// timer running. Clean up unconditionally before rethrowing.
|
|
321
|
-
timeoutController?.cleanup();
|
|
322
|
-
throw setupErr;
|
|
323
|
-
}
|
|
324
|
-
const url = `${stripTrailingSlash(this.config.baseURL)}/chat/completions`;
|
|
325
|
-
const fetchImpl = createProxyFetch();
|
|
326
|
-
const maxSteps = options.maxSteps || DEFAULT_MAX_STEPS;
|
|
327
|
-
const emitter = this.neurolink?.getEventEmitter();
|
|
328
|
-
const toolsUsed = [];
|
|
329
|
-
const toolExecutionSummaries = [];
|
|
330
|
-
const { usagePromise, finishPromise, resolveUsage, resolveFinish } = createDeferredAnalytics();
|
|
331
|
-
const { pushChunk, nextChunk } = createChunkQueue();
|
|
332
|
-
// Background multi-step loop. Pushes text deltas to the chunk queue and
|
|
333
|
-
// resolves the deferred analytics promises when it ends.
|
|
334
|
-
const loopPromise = this.runStreamLoop({
|
|
335
|
-
maxSteps,
|
|
336
|
-
modelId,
|
|
337
|
-
url,
|
|
338
|
-
apiKey: this.config.apiKey,
|
|
339
|
-
fetchImpl,
|
|
340
|
-
abortSignal,
|
|
341
|
-
options,
|
|
342
|
-
conversation,
|
|
343
|
-
openAITools,
|
|
344
|
-
openAIToolChoice,
|
|
345
|
-
toolsRecord,
|
|
346
|
-
emitter,
|
|
347
|
-
toolsUsed,
|
|
348
|
-
toolExecutionSummaries,
|
|
349
|
-
pushChunk,
|
|
350
|
-
resolveUsage,
|
|
351
|
-
resolveFinish,
|
|
352
|
-
});
|
|
353
|
-
// Closure-scoped capture: the runStreamLoop's catch block stashes the
|
|
354
|
-
// underlying provider error here so we can pass it through to
|
|
355
|
-
// buildNoOutputSentinel for richer telemetry (matches the pattern in
|
|
356
|
-
// openAI.ts / litellm.ts where onError preserves the upstream cause).
|
|
357
|
-
let capturedProviderError;
|
|
358
|
-
// Parameter named `error` so the compiled `capturedProviderError = error`
|
|
359
|
-
// assignment matches the regression-grep in test:context 6.14.
|
|
360
|
-
const captureProviderError = (error) => {
|
|
361
|
-
capturedProviderError = error;
|
|
362
|
-
};
|
|
363
|
-
const transformedStream = async function* () {
|
|
364
|
-
let contentYielded = 0;
|
|
365
|
-
try {
|
|
366
|
-
for (;;) {
|
|
367
|
-
const chunk = await nextChunk();
|
|
368
|
-
if ("done" in chunk) {
|
|
369
|
-
break;
|
|
370
|
-
}
|
|
371
|
-
if ("content" in chunk &&
|
|
372
|
-
typeof chunk.content === "string" &&
|
|
373
|
-
chunk.content.length > 0) {
|
|
374
|
-
contentYielded++;
|
|
375
|
-
}
|
|
376
|
-
yield chunk;
|
|
377
|
-
}
|
|
378
|
-
// Surface any error that the loop threw after we drained the queue.
|
|
379
|
-
await loopPromise;
|
|
380
|
-
// No-output path: stream completed normally but yielded zero text.
|
|
381
|
-
// Build an enriched sentinel + stamp the active OTel span so
|
|
382
|
-
// Pipeline B (ContextEnricher) surfaces a WARNING-level Langfuse
|
|
383
|
-
// observation instead of silently succeeding.
|
|
384
|
-
if (contentYielded === 0 && toolsUsed.length === 0) {
|
|
385
|
-
logger.warn("openai-compatible: Stream produced no output — emitting enriched sentinel");
|
|
386
|
-
const fauxNoOutput = new NoOutputGeneratedError({
|
|
387
|
-
message: "Stream produced no output",
|
|
388
|
-
});
|
|
389
|
-
const sentinel = await buildNoOutputSentinel(fauxNoOutput, undefined, capturedProviderError);
|
|
390
|
-
stampNoOutputSpan(sentinel);
|
|
391
|
-
yield sentinel;
|
|
392
|
-
}
|
|
393
|
-
}
|
|
394
|
-
catch (streamError) {
|
|
395
|
-
// AI SDK's NoOutputGeneratedError can surface here via re-thrown
|
|
396
|
-
// upstream callbacks. Native path mostly throws plain Errors, but
|
|
397
|
-
// keep the isInstance check + helper call so existing telemetry
|
|
398
|
-
// wiring (Pipeline B) fires consistently with other providers.
|
|
399
|
-
if (NoOutputGeneratedError.isInstance(streamError)) {
|
|
400
|
-
const sentinel = await buildNoOutputSentinel(streamError, undefined, capturedProviderError);
|
|
401
|
-
stampNoOutputSpan(sentinel);
|
|
402
|
-
yield sentinel;
|
|
403
|
-
return;
|
|
404
|
-
}
|
|
405
|
-
// Connection-killed / parse-error / fetch-failed path: still emit
|
|
406
|
-
// an enriched sentinel so consumers and Pipeline B see no_output
|
|
407
|
-
// instead of an unhandled rejection. Then re-throw so the original
|
|
408
|
-
// error still surfaces to direct stream consumers that need it.
|
|
409
|
-
const sentinel = await buildNoOutputSentinel(streamError, undefined, capturedProviderError);
|
|
410
|
-
stampNoOutputSpan(sentinel);
|
|
411
|
-
yield sentinel;
|
|
412
|
-
throw streamError;
|
|
413
|
-
}
|
|
414
|
-
finally {
|
|
415
|
-
// Consumer left the iterator early (break / return / throw) — abort
|
|
416
|
-
// the background SSE fetch + tool execution and stop the loop from
|
|
417
|
-
// growing the chunk queue further.
|
|
418
|
-
if (!consumerAbortController.signal.aborted) {
|
|
419
|
-
consumerAbortController.abort();
|
|
420
|
-
}
|
|
421
|
-
}
|
|
422
|
-
};
|
|
423
|
-
const result = {
|
|
424
|
-
stream: transformedStream(),
|
|
425
|
-
provider: this.providerName,
|
|
426
|
-
model: this.modelName,
|
|
427
|
-
analytics: streamAnalyticsCollector.createAnalytics(this.providerName, this.modelName,
|
|
428
|
-
// Pass the deferred promises so the collector sees real usage and
|
|
429
|
-
// finish reason after the multi-step loop completes.
|
|
430
|
-
{
|
|
431
|
-
textStream: (async function* () { })(),
|
|
432
|
-
usage: usagePromise,
|
|
433
|
-
finishReason: finishPromise,
|
|
434
|
-
}, Date.now() - startTime, {
|
|
435
|
-
requestId: `openai-compatible-stream-${Date.now()}`,
|
|
436
|
-
streamingMode: true,
|
|
437
|
-
}),
|
|
438
|
-
toolsUsed,
|
|
439
|
-
metadata: {
|
|
440
|
-
startTime,
|
|
441
|
-
streamId: `openai-compatible-${Date.now()}`,
|
|
442
|
-
},
|
|
443
|
-
};
|
|
444
|
-
// Lazy getter: every read transforms the live `toolExecutionSummaries`
|
|
445
|
-
// through the canonical `transformToolExecutions()` so consumers see
|
|
446
|
-
// `{name, input, output, duration}[]` (codebase convention), while still
|
|
447
|
-
// reflecting tools appended during streaming. A pre-computed array would
|
|
448
|
-
// freeze the snapshot empty for consumers who drain the stream after.
|
|
449
|
-
Object.defineProperty(result, "toolExecutions", {
|
|
450
|
-
enumerable: true,
|
|
451
|
-
configurable: true,
|
|
452
|
-
get: () => transformToolExecutions(toolExecutionSummaries.map((s) => ({
|
|
453
|
-
toolName: s.toolName,
|
|
454
|
-
input: s.input,
|
|
455
|
-
output: s.output,
|
|
456
|
-
duration: s.endTime.getTime() - s.startTime.getTime(),
|
|
457
|
-
}))),
|
|
458
|
-
});
|
|
459
|
-
// Cleanup timeout once the loop finishes. The actual rejection is
|
|
460
|
-
// surfaced to consumers via `await loopPromise` inside the stream
|
|
461
|
-
// generator; the .catch here exists only to keep node from logging
|
|
462
|
-
// an `unhandledRejection` on the cleanup chain. We also capture the
|
|
463
|
-
// upstream provider error into the closure variable so the no-output
|
|
464
|
-
// sentinel built later carries the real cause (matches the
|
|
465
|
-
// onError-callback pattern used by openAI.ts / litellm.ts).
|
|
466
|
-
loopPromise
|
|
467
|
-
.finally(() => timeoutController?.cleanup())
|
|
468
|
-
.catch((error) => {
|
|
469
|
-
captureProviderError(error);
|
|
470
|
-
});
|
|
471
|
-
return result;
|
|
472
|
-
}
|
|
473
|
-
/**
|
|
474
|
-
* Multi-step streaming orchestrator. One iteration per model turn:
|
|
475
|
-
*
|
|
476
|
-
* 1. POST /chat/completions with stream:true
|
|
477
|
-
* 2. Parse SSE; push text deltas to the consumer queue
|
|
478
|
-
* 3. If the step emitted tool_calls → execute each, append to
|
|
479
|
-
* conversation, loop again
|
|
480
|
-
* 4. Otherwise resolve the deferred analytics promises and exit
|
|
481
|
-
*
|
|
482
|
-
* Bounded by `args.maxSteps`. Any thrown error rejects loopPromise and
|
|
483
|
-
* is surfaced to the consumer via `await loopPromise` in the stream
|
|
484
|
-
* generator.
|
|
485
|
-
*/
|
|
486
|
-
async runStreamLoop(args) {
|
|
487
|
-
const { maxSteps, modelId, url, apiKey, fetchImpl, abortSignal, options, conversation, openAITools, openAIToolChoice, toolsRecord, emitter, toolsUsed, toolExecutionSummaries, pushChunk, resolveUsage, resolveFinish, } = args;
|
|
488
|
-
try {
|
|
489
|
-
let stepFinish = null;
|
|
490
|
-
let stepUsage;
|
|
491
|
-
for (let step = 0; step < maxSteps; step++) {
|
|
492
|
-
const stepResult = await this.streamOneStep({
|
|
493
|
-
modelId,
|
|
494
|
-
url,
|
|
495
|
-
apiKey,
|
|
496
|
-
fetchImpl,
|
|
497
|
-
abortSignal,
|
|
498
|
-
options,
|
|
499
|
-
conversation,
|
|
500
|
-
openAITools,
|
|
501
|
-
openAIToolChoice,
|
|
502
|
-
pushChunk,
|
|
503
|
-
});
|
|
504
|
-
stepFinish = stepResult.finishReason;
|
|
505
|
-
if (stepResult.usage) {
|
|
506
|
-
stepUsage = mergeUsage(stepUsage, stepResult.usage);
|
|
507
|
-
}
|
|
508
|
-
if (stepResult.toolCalls.size === 0) {
|
|
509
|
-
break;
|
|
510
|
-
}
|
|
511
|
-
await this.executeToolBatch({
|
|
512
|
-
stepResult,
|
|
513
|
-
conversation,
|
|
514
|
-
toolsRecord,
|
|
515
|
-
emitter,
|
|
516
|
-
toolsUsed,
|
|
517
|
-
toolExecutionSummaries,
|
|
518
|
-
options,
|
|
519
|
-
});
|
|
520
|
-
}
|
|
521
|
-
resolveUsage({
|
|
522
|
-
promptTokens: stepUsage?.prompt_tokens ?? 0,
|
|
523
|
-
completionTokens: stepUsage?.completion_tokens ?? 0,
|
|
524
|
-
totalTokens: stepUsage?.total_tokens ?? 0,
|
|
525
|
-
});
|
|
526
|
-
resolveFinish(stepFinish ?? "stop");
|
|
527
|
-
pushChunk({ done: true });
|
|
528
|
-
return {
|
|
529
|
-
finishReason: stepFinish ?? "stop",
|
|
530
|
-
usage: stepUsage,
|
|
531
|
-
};
|
|
532
|
-
}
|
|
533
|
-
catch (err) {
|
|
534
|
-
logger.error("OpenAI-compatible: Stream error", {
|
|
535
|
-
error: err instanceof Error ? err.message : String(err),
|
|
536
|
-
});
|
|
537
|
-
// Don't hang analytics consumers on deferred promises.
|
|
538
|
-
resolveUsage({ promptTokens: 0, completionTokens: 0, totalTokens: 0 });
|
|
539
|
-
resolveFinish("error");
|
|
540
|
-
pushChunk({ done: true });
|
|
541
|
-
throw err;
|
|
542
|
-
}
|
|
543
|
-
}
|
|
544
|
-
/**
|
|
545
|
-
* One streaming round-trip: POST chat-completions, parse SSE, push text
|
|
546
|
-
* deltas to the consumer queue. Returns the accumulated SSE result so
|
|
547
|
-
* the caller can decide whether to run tools and re-stream.
|
|
548
|
-
*/
|
|
549
|
-
async streamOneStep(args) {
|
|
550
|
-
const body = buildBody({
|
|
551
|
-
modelId: args.modelId,
|
|
552
|
-
messages: args.conversation,
|
|
553
|
-
options: args.options,
|
|
554
|
-
tools: args.openAITools,
|
|
555
|
-
...(args.openAIToolChoice !== undefined
|
|
556
|
-
? { toolChoice: args.openAIToolChoice }
|
|
557
|
-
: {}),
|
|
558
|
-
streaming: true,
|
|
559
|
-
});
|
|
560
|
-
const res = await args.fetchImpl(args.url, {
|
|
561
|
-
method: "POST",
|
|
562
|
-
headers: {
|
|
563
|
-
"Content-Type": "application/json",
|
|
564
|
-
Authorization: `Bearer ${args.apiKey}`,
|
|
565
|
-
},
|
|
566
|
-
body: JSON.stringify(body),
|
|
567
|
-
...(args.abortSignal ? { signal: args.abortSignal } : {}),
|
|
568
|
-
});
|
|
569
|
-
if (!res.ok) {
|
|
570
|
-
throw await buildAPIError(args.url, body, res);
|
|
571
|
-
}
|
|
572
|
-
if (!res.body) {
|
|
573
|
-
throw new Error("openai-compatible: stream response had no body");
|
|
574
|
-
}
|
|
575
|
-
return parseSSEStream(res.body, (delta) => {
|
|
576
|
-
args.pushChunk({ content: delta });
|
|
577
|
-
});
|
|
578
|
-
}
|
|
579
|
-
/**
|
|
580
|
-
* Execute every tool_call collected from one streaming step:
|
|
581
|
-
*
|
|
582
|
-
* - append an `assistant` turn carrying the tool_calls
|
|
583
|
-
* - resolve each tool from the local registry and run it
|
|
584
|
-
* - emit tool:start/tool:end events
|
|
585
|
-
* - push per-execution summaries
|
|
586
|
-
* - append a `tool` turn per result so the next step can see them
|
|
587
|
-
* - mirror BaseProvider's tool-events + storage hooks
|
|
588
|
-
*/
|
|
589
|
-
async executeToolBatch(args) {
|
|
590
|
-
const { stepResult, conversation, toolsRecord, emitter, toolsUsed, toolExecutionSummaries, options, } = args;
|
|
591
|
-
// Append the assistant turn that triggered tool calls.
|
|
592
|
-
const toolCallsForMessage = [];
|
|
593
|
-
for (const [, t] of stepResult.toolCalls) {
|
|
594
|
-
toolCallsForMessage.push({
|
|
595
|
-
id: t.id,
|
|
596
|
-
type: "function",
|
|
597
|
-
function: { name: t.name, arguments: t.argsBuffered },
|
|
598
|
-
});
|
|
599
|
-
}
|
|
600
|
-
conversation.push({
|
|
601
|
-
role: "assistant",
|
|
602
|
-
content: stepResult.text.length > 0 ? stepResult.text : null,
|
|
603
|
-
tool_calls: toolCallsForMessage,
|
|
604
|
-
});
|
|
605
|
-
// Execute each tool, append result as a tool message.
|
|
606
|
-
for (const [, t] of stepResult.toolCalls) {
|
|
607
|
-
const startedAt = new Date();
|
|
608
|
-
let input;
|
|
609
|
-
try {
|
|
610
|
-
input = JSON.parse(t.argsBuffered || "{}");
|
|
611
|
-
}
|
|
612
|
-
catch {
|
|
613
|
-
input = t.argsBuffered;
|
|
614
|
-
}
|
|
615
|
-
let output;
|
|
616
|
-
let errorMsg;
|
|
617
|
-
const toolDef = toolsRecord[t.name];
|
|
618
|
-
emitter?.emit("tool:start", {
|
|
619
|
-
toolName: t.name,
|
|
620
|
-
toolCallId: t.id,
|
|
621
|
-
input,
|
|
622
|
-
});
|
|
623
|
-
if (!toolDef || typeof toolDef.execute !== "function") {
|
|
624
|
-
errorMsg = `Tool '${t.name}' is not registered.`;
|
|
625
|
-
output = { error: errorMsg };
|
|
626
|
-
}
|
|
627
|
-
else {
|
|
628
|
-
try {
|
|
629
|
-
output = await toolDef.execute(input, {});
|
|
630
|
-
}
|
|
631
|
-
catch (err) {
|
|
632
|
-
errorMsg = err instanceof Error ? err.message : String(err);
|
|
633
|
-
output = { error: errorMsg };
|
|
634
|
-
}
|
|
635
|
-
}
|
|
636
|
-
const endedAt = new Date();
|
|
637
|
-
toolsUsed.push(t.name);
|
|
638
|
-
toolExecutionSummaries.push({
|
|
639
|
-
toolCallId: t.id,
|
|
640
|
-
toolName: t.name,
|
|
641
|
-
input,
|
|
642
|
-
output,
|
|
643
|
-
...(errorMsg ? { error: errorMsg } : {}),
|
|
644
|
-
startTime: startedAt,
|
|
645
|
-
endTime: endedAt,
|
|
646
|
-
});
|
|
647
|
-
conversation.push({
|
|
648
|
-
role: "tool",
|
|
649
|
-
tool_call_id: t.id,
|
|
650
|
-
content: stringifyToolOutput(output),
|
|
651
|
-
});
|
|
652
|
-
}
|
|
653
|
-
// BaseProvider tool-events + storage hooks. Mirrors what other providers
|
|
654
|
-
// call from their AI-SDK onStepFinish handlers.
|
|
655
|
-
const justExecuted = toolExecutionSummaries.slice(-stepResult.toolCalls.size);
|
|
656
|
-
emitToolEndFromStepFinish(emitter, justExecuted.map((s) => ({
|
|
657
|
-
toolName: s.toolName,
|
|
658
|
-
output: s.output,
|
|
659
|
-
...(s.error ? { error: s.error } : {}),
|
|
660
|
-
})));
|
|
661
|
-
try {
|
|
662
|
-
await this.handleToolExecutionStorage(justExecuted.map((s) => ({
|
|
663
|
-
toolCallId: s.toolCallId,
|
|
664
|
-
toolName: s.toolName,
|
|
665
|
-
input: s.input,
|
|
666
|
-
output: s.output,
|
|
667
|
-
})), justExecuted.map((s) => ({
|
|
668
|
-
toolCallId: s.toolCallId,
|
|
669
|
-
toolName: s.toolName,
|
|
670
|
-
output: s.output,
|
|
671
|
-
})), options, new Date());
|
|
672
|
-
}
|
|
673
|
-
catch (err) {
|
|
674
|
-
logger.warn("[OpenAICompatibleProvider] Failed to store tool executions", {
|
|
675
|
-
provider: this.providerName,
|
|
676
|
-
error: err instanceof Error ? err.message : String(err),
|
|
677
|
-
});
|
|
678
|
-
}
|
|
679
|
-
}
|
|
680
|
-
async getAvailableModels() {
|
|
681
|
-
try {
|
|
682
|
-
// Match the chat-completions URL convention: append `/models` to the
|
|
683
|
-
// user-provided base. Using `new URL("/v1/models", baseURL)` would
|
|
684
|
-
// strip any base path (e.g. `http://host/api/v1` → `http://host/v1/models`).
|
|
685
|
-
const modelsUrl = `${stripTrailingSlash(this.config.baseURL)}/models`;
|
|
686
|
-
logger.debug(`Fetching available models from: ${modelsUrl}`);
|
|
687
|
-
const proxyFetch = createProxyFetch();
|
|
688
|
-
const controller = new AbortController();
|
|
689
|
-
const t = setTimeout(() => controller.abort(), 5000);
|
|
690
|
-
const response = await proxyFetch(modelsUrl, {
|
|
691
|
-
headers: {
|
|
692
|
-
Authorization: `Bearer ${this.config.apiKey}`,
|
|
693
|
-
"Content-Type": "application/json",
|
|
694
|
-
},
|
|
695
|
-
signal: controller.signal,
|
|
696
|
-
});
|
|
697
|
-
clearTimeout(t);
|
|
698
|
-
if (!response.ok) {
|
|
699
|
-
logger.warn(`Models endpoint returned ${response.status}: ${response.statusText}`);
|
|
700
|
-
return this.getFallbackModels();
|
|
701
|
-
}
|
|
702
|
-
const data = await response.json();
|
|
703
|
-
if (!data.data || !Array.isArray(data.data)) {
|
|
704
|
-
logger.warn("Invalid models response format");
|
|
705
|
-
return this.getFallbackModels();
|
|
706
|
-
}
|
|
707
|
-
const models = data.data.map((model) => model.id).filter(Boolean);
|
|
708
|
-
logger.debug(`Discovered ${models.length} models:`, models);
|
|
709
|
-
return models.length > 0 ? models : this.getFallbackModels();
|
|
710
|
-
}
|
|
711
|
-
catch (error) {
|
|
712
|
-
logger.warn(`Failed to fetch models from OpenAI Compatible endpoint:`, error);
|
|
713
|
-
return this.getFallbackModels();
|
|
714
|
-
}
|
|
715
|
-
}
|
|
716
|
-
async getFirstAvailableModel() {
|
|
717
|
-
const models = await this.getAvailableModels();
|
|
718
|
-
return models[0] || FALLBACK_OPENAI_COMPATIBLE_MODEL;
|
|
719
|
-
}
|
|
720
|
-
getFallbackModels() {
|
|
721
|
-
return [
|
|
722
|
-
"gpt-4o",
|
|
723
|
-
"gpt-4o-mini",
|
|
724
|
-
"gpt-4-turbo",
|
|
725
|
-
FALLBACK_OPENAI_COMPATIBLE_MODEL,
|
|
726
|
-
"claude-3-5-sonnet",
|
|
727
|
-
"claude-3-haiku",
|
|
728
|
-
"gemini-pro",
|
|
729
|
-
];
|
|
730
|
-
}
|
|
731
100
|
}
|