@juspay/neurolink 9.68.6 → 9.68.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,20 +1,9 @@
1
- import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
2
1
  import { NvidiaNimModels } from "../constants/enums.js";
3
- import { BaseProvider } from "../core/baseProvider.js";
4
- import { DEFAULT_MAX_STEPS } from "../core/constants.js";
5
- import { streamAnalyticsCollector } from "../core/streamAnalytics.js";
6
- import { isNeuroLink } from "../neurolink.js";
7
- import { createProxyFetch, maskProxyUrl } from "../proxy/proxyFetch.js";
8
- import { tracers, ATTR, withClientStreamSpan } from "../telemetry/index.js";
9
2
  import { AuthenticationError, InvalidModelError, NetworkError, ProviderError, RateLimitError, } from "../types/index.js";
10
3
  import { logger } from "../utils/logger.js";
11
4
  import { createNvidiaNimConfig, getProviderModel, validateApiKey, } from "../utils/providerConfig.js";
12
- import { composeAbortSignals, createTimeoutController, TimeoutError, } from "../utils/timeout.js";
13
- import { emitToolEndFromStepFinish } from "../utils/toolEndEmitter.js";
14
- import { resolveToolChoice } from "../utils/toolChoice.js";
15
- import { toAnalyticsStreamResult } from "./providerTypeUtils.js";
16
- import { stepCountIs } from "../utils/tool.js";
17
- import { streamText } from "../utils/generation.js";
5
+ import { TimeoutError } from "../utils/timeout.js";
6
+ import { OpenAIChatCompletionsProvider } from "./openaiChatCompletionsBase.js";
18
7
  /**
19
8
  * Decide whether a NIM 400 response body is a rejection of the named
20
9
  * field (as opposed to an unrelated 400 that happens to mention the
@@ -94,71 +83,6 @@ const stripFieldFromJsonBody = (body, field) => {
94
83
  return null;
95
84
  }
96
85
  };
97
- const makeLoggingFetch = (provider) => {
98
- const base = createProxyFetch();
99
- return (async (input, init) => {
100
- const url = typeof input === "string"
101
- ? input
102
- : input instanceof URL
103
- ? input.toString()
104
- : input.url;
105
- const reqSize = init?.body && typeof init.body === "string" ? init.body.length : 0;
106
- let response = await base(input, init);
107
- // Generic NIM 400 retry-strip: works for BOTH generate and stream paths.
108
- // NIM sometimes returns HTTP 400 when a model rejects `reasoning_budget`
109
- // or `chat_template`. The stream path already retries by reconstructing
110
- // its provider options; this fetch-level retry is the symmetric fix for
111
- // generate (and any other transport that lands here).
112
- //
113
- // We require BOTH (a) the offending field name AND (b) a rejection
114
- // keyword (unsupported / not supported / unknown / invalid /
115
- // unrecognized / does not support) within 80 chars of it. Without the
116
- // rejection-keyword guard, an unrelated 400 whose error body happened
117
- // to mention `chat_template` (e.g. the user prompt got echoed back)
118
- // would cause us to silently strip a field the user actually wanted
119
- // sent, and either succeed for the wrong reason or fail with a
120
- // misleading error.
121
- if (response.status === 400 &&
122
- typeof init?.body === "string" &&
123
- init.body.length > 0) {
124
- const cloned = response.clone();
125
- const body = await cloned.text().catch(() => "");
126
- let retryBody = null;
127
- let stripped = null;
128
- if (isNimFieldRejection(body, "reasoning_budget")) {
129
- retryBody = stripFieldFromJsonBody(init.body, "reasoning_budget");
130
- stripped = "reasoning_budget";
131
- }
132
- else if (isNimFieldRejection(body, "chat_template")) {
133
- retryBody = stripFieldFromJsonBody(init.body, "chat_template");
134
- stripped = "chat_template";
135
- }
136
- if (retryBody !== null && stripped !== null) {
137
- logger.warn(`[${provider}] NIM rejected ${stripped}; retrying with field stripped`);
138
- response = await base(input, { ...init, body: retryBody });
139
- }
140
- }
141
- if (!response.ok) {
142
- // If maskProxyUrl can't safely sanitize the URL (returns null), don't
143
- // log the raw URL — that defeats the redaction. Use a placeholder so
144
- // operators still get the warning without leaking credentials.
145
- const safeUrl = maskProxyUrl(url) ?? "<redacted>";
146
- if (process.env.NEUROLINK_DEBUG_HTTP === "1") {
147
- const clone = response.clone();
148
- const body = await clone.text().catch(() => "<unreadable>");
149
- logger.warn(`[${provider}] upstream ${response.status}`, {
150
- url: safeUrl,
151
- body: body.slice(0, 800),
152
- reqSize,
153
- });
154
- }
155
- else {
156
- logger.warn(`[${provider}] upstream ${response.status} url=${safeUrl} reqSize=${reqSize}`);
157
- }
158
- }
159
- return response;
160
- });
161
- };
162
86
  const NVIDIA_NIM_DEFAULT_BASE_URL = "https://integrate.api.nvidia.com/v1";
163
87
  const envInt = (k) => {
164
88
  const v = process.env[k];
@@ -207,21 +131,6 @@ const buildNvidiaNimExtraBody = (thinkingEnabled, maxTokens) => {
207
131
  }
208
132
  return extra;
209
133
  };
210
- const stripReasoningBudget = (body) => {
211
- const cloned = { ...body };
212
- if (cloned.chat_template_kwargs) {
213
- const { reasoning_budget: _ignored, ...rest } = cloned.chat_template_kwargs;
214
- cloned.chat_template_kwargs = rest;
215
- if (Object.keys(cloned.chat_template_kwargs).length === 0) {
216
- delete cloned.chat_template_kwargs;
217
- }
218
- }
219
- return cloned;
220
- };
221
- const stripChatTemplate = (body) => {
222
- const { chat_template: _ignored, ...rest } = body;
223
- return rest;
224
- };
225
134
  const getNimApiKey = () => {
226
135
  return validateApiKey(createNvidiaNimConfig());
227
136
  };
@@ -229,222 +138,61 @@ const getDefaultNimModel = () => {
229
138
  return getProviderModel("NVIDIA_NIM_MODEL", NvidiaNimModels.LLAMA_3_3_70B_INSTRUCT);
230
139
  };
231
140
  /**
232
- * NVIDIA NIM Provider
233
- * Wraps NVIDIA's hosted (or self-hosted) inference endpoints via OpenAI-compat.
141
+ * NVIDIA NIM Provider — native HTTP+SSE, no AI SDK.
142
+ *
143
+ * Wraps NVIDIA's hosted (or self-hosted) inference endpoints.
234
144
  * Passes NIM-specific extras (top_k, min_p, repetition_penalty,
235
- * chat_template_kwargs.reasoning_budget) via providerOptions.openai.body.
236
- * Implements one-retry-on-400 to drop unsupported extras gracefully.
145
+ * chat_template_kwargs.reasoning_budget) via adjustBuildBodyOptions.
146
+ * reasoning_content surfacing is a pending base-client follow-up (not emitted natively yet); all other behavior is preserved.
147
+ *
148
+ * @see https://docs.api.nvidia.com/nim/reference/
237
149
  */
238
- export class NvidiaNimProvider extends BaseProvider {
239
- model;
240
- apiKey;
241
- baseURL;
150
+ export class NvidiaNimProvider extends OpenAIChatCompletionsProvider {
242
151
  constructor(modelName, sdk, _region, credentials) {
243
- const validatedNeurolink = isNeuroLink(sdk) ? sdk : undefined;
244
- super(modelName, "nvidia-nim", validatedNeurolink);
245
152
  // Trim the override before applying precedence. A blank/whitespace
246
153
  // `credentials.apiKey` should NOT bypass `getNimApiKey()` — that would
247
154
  // build a client with an unusable bearer token and fail at request time
248
155
  // with a confusing 401 instead of at construction time.
249
156
  const overrideApiKey = credentials?.apiKey?.trim();
250
- this.apiKey =
251
- overrideApiKey && overrideApiKey.length > 0
252
- ? overrideApiKey
253
- : getNimApiKey();
254
- this.baseURL =
255
- credentials?.baseURL ??
256
- process.env.NVIDIA_NIM_BASE_URL ??
257
- NVIDIA_NIM_DEFAULT_BASE_URL;
258
- // We deliberately use `@ai-sdk/openai-compatible` rather than
259
- // `@ai-sdk/openai`. Two upstream behaviors of `@ai-sdk/openai` break us:
260
- // 1. It always sends `response_format: { type: "json_schema" }` when a
261
- // schema is provided. Most NIM-served chat models don't enforce
262
- // json_schema strictly — the schema goes through but `result.object`
263
- // stays empty because the SDK never gets the typed response back.
264
- // 2. It does not parse the `reasoning_content` field that NIM-hosted
265
- // reasoning models (deepseek-r1, qwq, llama-nemotron-ultra) emit,
266
- // so chain-of-thought is silently dropped.
267
- // `@ai-sdk/openai-compatible` honors `supportsStructuredOutputs: false`
268
- // (falls back to `{ type: "json_object" }` and injects the schema into
269
- // the prompt — works across the entire NIM model fleet) and parses both
270
- // `choice.message.reasoning_content` and `delta.reasoning_content` into
271
- // the SDK-standard `reasoning` part. NIM-specific extras (`min_tokens`,
272
- // `chat_template_kwargs.reasoning_budget`, `chat_template`) are still
273
- // injected via `providerOptions.openai.body` in `executeStreamInner`.
274
- const nim = createOpenAICompatible({
275
- name: "nvidia-nim",
276
- apiKey: this.apiKey,
277
- baseURL: this.baseURL,
278
- fetch: makeLoggingFetch("nvidia-nim"),
279
- supportsStructuredOutputs: false,
280
- includeUsage: true,
281
- });
282
- this.model = nim.chatModel(this.modelName);
157
+ const apiKey = overrideApiKey && overrideApiKey.length > 0
158
+ ? overrideApiKey
159
+ : getNimApiKey();
160
+ const baseURL = credentials?.baseURL ??
161
+ process.env.NVIDIA_NIM_BASE_URL ??
162
+ NVIDIA_NIM_DEFAULT_BASE_URL;
163
+ super("nvidia-nim", modelName, sdk, { baseURL, apiKey });
283
164
  logger.debug("NVIDIA NIM Provider initialized", {
284
165
  modelName: this.modelName,
285
166
  providerName: this.providerName,
286
- baseURL: this.baseURL,
167
+ baseURL: this.config.baseURL,
287
168
  });
288
169
  }
289
- async executeStream(options, _analysisSchema) {
290
- return withClientStreamSpan({
291
- name: "neurolink.provider.stream",
292
- tracer: tracers.provider,
293
- attributes: {
294
- [ATTR.GEN_AI_SYSTEM]: "nvidia-nim",
295
- [ATTR.GEN_AI_MODEL]: this.modelName,
296
- [ATTR.GEN_AI_OPERATION]: "stream",
297
- [ATTR.NL_STREAM_MODE]: true,
298
- },
299
- }, async () => this.executeStreamInner(options), (r) => r.stream, (r, wrapped) => ({ ...r, stream: wrapped }));
300
- }
301
- async executeStreamInner(options) {
302
- this.validateStreamOptions(options);
303
- const startTime = Date.now();
304
- const timeout = this.getTimeout(options);
305
- const timeoutController = createTimeoutController(timeout, this.providerName, "stream");
306
- try {
307
- const shouldUseTools = !options.disableTools && this.supportsTools();
308
- const tools = shouldUseTools
309
- ? options.tools || (await this.getAllTools())
310
- : {};
311
- const messages = await this.buildMessagesForStream(options);
312
- const model = await this.getAISDKModelWithMiddleware(options);
313
- // Callers pass `thinkingLevel` directly on generate/stream options
314
- // (matching Anthropic / Gemini 2.5+ / Gemini 3 conventions). Fall back
315
- // to the legacy `thinkingConfig.thinkingLevel` shape for compatibility.
316
- const tl = options.thinkingLevel ??
317
- options.thinkingConfig?.thinkingLevel;
318
- const thinkingEnabled = tl !== undefined && tl !== "minimal";
319
- let extraBody = buildNvidiaNimExtraBody(thinkingEnabled, options.maxTokens);
320
- // Inline the retry-strip union — CLAUDE.md rule 2 forbids type aliases
321
- // outside src/lib/types/. The two literals match the 400 error keys NIM
322
- // returns for the only two extras we know how to drop and retry.
323
- const callStream = (body, stripped = []) => streamText({
324
- model,
325
- messages,
326
- temperature: options.temperature,
327
- maxOutputTokens: options.maxTokens,
328
- tools,
329
- stopWhen: stepCountIs(options.maxSteps || DEFAULT_MAX_STEPS),
330
- toolChoice: resolveToolChoice(options, tools, shouldUseTools),
331
- abortSignal: composeAbortSignals(options.abortSignal, timeoutController?.controller.signal),
332
- providerOptions: (() => {
333
- // StreamOptions doesn't formally type providerOptions but the
334
- // upstream Vercel AI SDK accepts it. Read it via an indexed access
335
- // and merge with NIM extras instead of overwriting any per-call
336
- // openai.body.
337
- const callerBase = options
338
- .providerOptions ?? {};
339
- const callerOpenai = callerBase.openai ?? {};
340
- const callerBody = callerOpenai.body ?? {};
341
- // Per-call overrides win over env/NIM defaults — defaults first,
342
- // overrides last. chat_template_kwargs is merged shallowly too so
343
- // a request that only sets `reasoning_budget` doesn't drop the
344
- // env-driven `thinking: true` flag (and vice versa).
345
- const defaultsBody = body;
346
- const mergedBody = {
347
- ...defaultsBody,
348
- ...callerBody,
349
- };
350
- const mergedKwargs = {
351
- ...(defaultsBody.chat_template_kwargs ?? {}),
352
- ...(callerBody.chat_template_kwargs ?? {}),
353
- };
354
- // Apply retry-strip AFTER merging so caller-supplied copies of
355
- // the offending field are also dropped (otherwise the retry would
356
- // re-send the field that NIM just rejected).
357
- if (stripped.includes("chat_template")) {
358
- delete mergedBody.chat_template;
359
- }
360
- if (stripped.includes("reasoning_budget")) {
361
- delete mergedKwargs.reasoning_budget;
362
- }
363
- if (Object.keys(mergedKwargs).length > 0) {
364
- mergedBody.chat_template_kwargs = mergedKwargs;
365
- }
366
- else {
367
- delete mergedBody.chat_template_kwargs;
368
- }
369
- if (Object.keys(callerBase).length === 0 &&
370
- Object.keys(mergedBody).length === 0) {
371
- return undefined;
372
- }
373
- return {
374
- ...callerBase,
375
- openai: {
376
- ...callerOpenai,
377
- body: mergedBody,
378
- },
379
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
380
- };
381
- })(),
382
- experimental_telemetry: this.telemetryHandler.getTelemetryConfig(options),
383
- experimental_repairToolCall: this.getToolCallRepairFn(options),
384
- onStepFinish: ({ toolCalls, toolResults }) => {
385
- emitToolEndFromStepFinish(this.neurolink?.getEventEmitter(), toolResults);
386
- this.handleToolExecutionStorage(toolCalls, toolResults, options, new Date()).catch((error) => {
387
- logger.warn("[NvidiaNimProvider] Failed to store tool executions", {
388
- provider: this.providerName,
389
- error: error instanceof Error ? error.message : String(error),
390
- });
391
- });
392
- },
393
- });
394
- let result;
395
- try {
396
- result = await callStream(extraBody);
397
- }
398
- catch (error) {
399
- const errMsg = error instanceof Error ? error.message : String(error);
400
- const status = error?.statusCode;
401
- if (status === 400) {
402
- const lower = errMsg.toLowerCase();
403
- if (lower.includes("reasoning_budget")) {
404
- logger.warn("NIM rejected reasoning_budget; retrying without it");
405
- extraBody = stripReasoningBudget(extraBody);
406
- result = await callStream(extraBody, ["reasoning_budget"]);
407
- }
408
- else if (lower.includes("chat_template")) {
409
- logger.warn("NIM rejected chat_template; retrying without it");
410
- extraBody = stripChatTemplate(extraBody);
411
- result = await callStream(extraBody, ["chat_template"]);
412
- }
413
- else {
414
- throw error;
415
- }
416
- }
417
- else {
418
- throw error;
419
- }
420
- }
421
- timeoutController?.cleanup();
422
- const transformedStream = this.createTextStream(result);
423
- const analyticsPromise = streamAnalyticsCollector.createAnalytics(this.providerName, this.modelName, toAnalyticsStreamResult(result), Date.now() - startTime, {
424
- requestId: `nvidia-nim-stream-${Date.now()}`,
425
- streamingMode: true,
426
- });
427
- return {
428
- stream: transformedStream,
429
- provider: this.providerName,
430
- model: this.modelName,
431
- analytics: analyticsPromise,
432
- metadata: { startTime, streamId: `nvidia-nim-${Date.now()}` },
433
- };
434
- }
435
- catch (error) {
436
- timeoutController?.cleanup();
437
- throw this.handleProviderError(error);
438
- }
439
- }
440
170
  getProviderName() {
441
- return this.providerName;
171
+ return "nvidia-nim";
442
172
  }
443
173
  getDefaultModel() {
444
174
  return getDefaultNimModel();
445
175
  }
446
- getAISDKModel() {
447
- return this.model;
176
+ /**
177
+ * Inject NIM-specific body fields (top_k, min_p, repetition_penalty,
178
+ * min_tokens, chat_template, chat_template_kwargs) into the wire body.
179
+ *
180
+ * The base passes the full StreamOptions as `opts` at runtime (typed
181
+ * narrowly as OpenAICompatBuildBodyArgs["options"] for standard fields).
182
+ * We access thinkingLevel via an indexed-access cast since the runtime
183
+ * value is the full StreamOptions object.
184
+ */
185
+ adjustBuildBodyOptions(_modelId, opts) {
186
+ // The runtime value of opts is the full StreamOptions; TypeScript types
187
+ // it narrowly so we cast to access NIM-specific thinking fields.
188
+ const fullOpts = opts;
189
+ const thinkingConfigRaw = fullOpts.thinkingConfig;
190
+ const tl = fullOpts.thinkingLevel ??
191
+ thinkingConfigRaw?.thinkingLevel;
192
+ const thinkingEnabled = tl !== undefined && tl !== "minimal";
193
+ const maxTokens = typeof fullOpts.maxTokens === "number" ? fullOpts.maxTokens : undefined;
194
+ const extra = buildNvidiaNimExtraBody(thinkingEnabled, maxTokens);
195
+ return { ...opts, ...extra };
448
196
  }
449
197
  formatProviderError(error) {
450
198
  if (error instanceof TimeoutError) {
@@ -481,16 +229,18 @@ export class NvidiaNimProvider extends BaseProvider {
481
229
  return new ProviderError(`NVIDIA NIM error: ${message}`, "nvidia-nim");
482
230
  }
483
231
  async validateConfiguration() {
484
- return typeof this.apiKey === "string" && this.apiKey.trim().length > 0;
232
+ return (typeof this.config.apiKey === "string" &&
233
+ this.config.apiKey.trim().length > 0);
485
234
  }
486
235
  getConfiguration() {
487
236
  return {
488
237
  provider: this.providerName,
489
238
  model: this.modelName,
490
239
  defaultModel: getDefaultNimModel(),
491
- baseURL: this.baseURL,
240
+ baseURL: this.config.baseURL,
492
241
  };
493
242
  }
494
243
  }
495
- export default NvidiaNimProvider;
244
+ // Exported for test suites that probe NIM's 400-retry behaviour directly.
245
+ export { isNimFieldRejection, stripFieldFromJsonBody };
496
246
  //# sourceMappingURL=nvidiaNim.js.map
@@ -1,9 +1,8 @@
1
1
  import type { AIProviderName } from "../constants/enums.js";
2
- import { BaseProvider } from "../core/baseProvider.js";
3
- import type { NeurolinkCredentials, StreamOptions, StreamResult, ValidationSchema } from "../types/index.js";
4
- import type { LanguageModel } from "../types/index.js";
2
+ import type { NeurolinkCredentials } from "../types/index.js";
3
+ import { OpenAIChatCompletionsProvider } from "./openaiChatCompletionsBase.js";
5
4
  /**
6
- * Cloudflare Workers AI Provider
5
+ * Cloudflare Workers AI Provider — direct HTTP, no AI SDK.
7
6
  *
8
7
  * Edge-served open models (Llama / Mistral / Qwen / Gemma) at
9
8
  * `https://api.cloudflare.com/client/v4/accounts/{accountId}/ai/v1`
@@ -11,25 +10,17 @@ import type { LanguageModel } from "../types/index.js";
11
10
  *
12
11
  * Required env: `CLOUDFLARE_API_KEY` + `CLOUDFLARE_ACCOUNT_ID`.
13
12
  *
13
+ * All request/stream/tool-loop orchestration lives in
14
+ * `OpenAIChatCompletionsProvider`; this class only declares configuration
15
+ * and provider-specific error mapping.
16
+ *
14
17
  * @see https://developers.cloudflare.com/workers-ai/configuration/open-ai-compatibility/
15
18
  */
16
- export declare class CloudflareProvider extends BaseProvider {
17
- private model;
18
- private apiKey;
19
- private baseURL;
19
+ export declare class CloudflareProvider extends OpenAIChatCompletionsProvider {
20
20
  constructor(modelName?: string, sdk?: unknown, _region?: string, credentials?: NeurolinkCredentials["cloudflare"]);
21
- protected executeStream(options: StreamOptions, _analysisSchema?: ValidationSchema): Promise<StreamResult>;
22
- private executeStreamInner;
23
21
  protected getProviderName(): AIProviderName;
24
22
  protected getDefaultModel(): string;
25
- protected getAISDKModel(): LanguageModel;
23
+ protected getFallbackModelName(): string;
24
+ protected getFallbackModels(): string[];
26
25
  protected formatProviderError(error: unknown): Error;
27
- validateConfiguration(): Promise<boolean>;
28
- getConfiguration(): {
29
- provider: AIProviderName;
30
- model: string;
31
- defaultModel: string;
32
- baseURL: string;
33
- };
34
26
  }
35
- export default CloudflareProvider;
@@ -1,20 +1,9 @@
1
- import { createOpenAI } from "@ai-sdk/openai";
2
1
  import { CloudflareModels } from "../constants/enums.js";
3
- import { BaseProvider } from "../core/baseProvider.js";
4
- import { DEFAULT_MAX_STEPS } from "../core/constants.js";
5
- import { streamAnalyticsCollector } from "../core/streamAnalytics.js";
6
- import { isNeuroLink } from "../neurolink.js";
7
- import { createLoggingFetch } from "../utils/loggingFetch.js";
8
- import { tracers, ATTR, withClientStreamSpan } from "../telemetry/index.js";
9
2
  import { AuthenticationError, InvalidModelError, NetworkError, ProviderError, RateLimitError, } from "../types/index.js";
10
3
  import { logger } from "../utils/logger.js";
11
4
  import { createCloudflareConfig, getProviderModel, validateApiKey, } from "../utils/providerConfig.js";
12
- import { composeAbortSignals, createTimeoutController, TimeoutError, } from "../utils/timeout.js";
13
- import { emitToolEndFromStepFinish } from "../utils/toolEndEmitter.js";
14
- import { resolveToolChoice } from "../utils/toolChoice.js";
15
- import { toAnalyticsStreamResult } from "./providerTypeUtils.js";
16
- import { stepCountIs } from "../utils/tool.js";
17
- import { streamText } from "../utils/generation.js";
5
+ import { TimeoutError } from "../utils/timeout.js";
6
+ import { OpenAIChatCompletionsProvider } from "./openaiChatCompletionsBase.js";
18
7
  /**
19
8
  * Cloudflare Workers AI exposes an OpenAI-compatible endpoint scoped per
20
9
  * account: `https://api.cloudflare.com/client/v4/accounts/{ACCOUNT_ID}/ai/v1`.
@@ -24,7 +13,7 @@ const buildCloudflareBaseURL = (accountId) => `https://api.cloudflare.com/client
24
13
  const getCloudflareApiKey = () => validateApiKey(createCloudflareConfig());
25
14
  const getDefaultCloudflareModel = () => getProviderModel("CLOUDFLARE_MODEL", CloudflareModels.LLAMA_3_3_70B_FAST);
26
15
  /**
27
- * Cloudflare Workers AI Provider
16
+ * Cloudflare Workers AI Provider — direct HTTP, no AI SDK.
28
17
  *
29
18
  * Edge-served open models (Llama / Mistral / Qwen / Gemma) at
30
19
  * `https://api.cloudflare.com/client/v4/accounts/{accountId}/ai/v1`
@@ -32,111 +21,50 @@ const getDefaultCloudflareModel = () => getProviderModel("CLOUDFLARE_MODEL", Clo
32
21
  *
33
22
  * Required env: `CLOUDFLARE_API_KEY` + `CLOUDFLARE_ACCOUNT_ID`.
34
23
  *
24
+ * All request/stream/tool-loop orchestration lives in
25
+ * `OpenAIChatCompletionsProvider`; this class only declares configuration
26
+ * and provider-specific error mapping.
27
+ *
35
28
  * @see https://developers.cloudflare.com/workers-ai/configuration/open-ai-compatibility/
36
29
  */
37
- export class CloudflareProvider extends BaseProvider {
38
- model;
39
- apiKey;
40
- baseURL;
30
+ export class CloudflareProvider extends OpenAIChatCompletionsProvider {
41
31
  constructor(modelName, sdk, _region, credentials) {
42
- const validatedNeurolink = isNeuroLink(sdk) ? sdk : undefined;
43
- super(modelName, "cloudflare", validatedNeurolink);
44
32
  const overrideApiKey = credentials?.apiKey?.trim();
45
- this.apiKey =
46
- overrideApiKey && overrideApiKey.length > 0
47
- ? overrideApiKey
48
- : getCloudflareApiKey();
33
+ const apiKey = overrideApiKey && overrideApiKey.length > 0
34
+ ? overrideApiKey
35
+ : getCloudflareApiKey();
49
36
  const accountId = (credentials?.accountId ??
50
37
  process.env.CLOUDFLARE_ACCOUNT_ID ??
51
38
  "").trim();
52
39
  if (!accountId) {
53
40
  throw new Error("CLOUDFLARE_ACCOUNT_ID is required (or pass credentials.cloudflare.accountId). Get the account id from https://dash.cloudflare.com/");
54
41
  }
55
- this.baseURL = credentials?.baseURL ?? buildCloudflareBaseURL(accountId);
56
- const cloudflare = createOpenAI({
57
- apiKey: this.apiKey,
58
- baseURL: this.baseURL,
59
- fetch: createLoggingFetch("cloudflare"),
60
- });
61
- this.model = cloudflare.chat(this.modelName);
42
+ const baseURL = credentials?.baseURL ?? buildCloudflareBaseURL(accountId);
43
+ super("cloudflare", modelName, sdk, { baseURL, apiKey });
62
44
  logger.debug("Cloudflare Workers AI Provider initialized", {
63
45
  modelName: this.modelName,
64
46
  providerName: this.providerName,
65
- baseURL: this.baseURL,
47
+ baseURL: this.config.baseURL,
66
48
  });
67
49
  }
68
- async executeStream(options, _analysisSchema) {
69
- return withClientStreamSpan({
70
- name: "neurolink.provider.stream",
71
- tracer: tracers.provider,
72
- attributes: {
73
- [ATTR.GEN_AI_SYSTEM]: "cloudflare",
74
- [ATTR.GEN_AI_MODEL]: this.modelName,
75
- [ATTR.GEN_AI_OPERATION]: "stream",
76
- [ATTR.NL_STREAM_MODE]: true,
77
- },
78
- }, async () => this.executeStreamInner(options), (r) => r.stream, (r, wrapped) => ({ ...r, stream: wrapped }));
79
- }
80
- async executeStreamInner(options) {
81
- this.validateStreamOptions(options);
82
- const startTime = Date.now();
83
- const timeout = this.getTimeout(options);
84
- const timeoutController = createTimeoutController(timeout, this.providerName, "stream");
85
- try {
86
- const shouldUseTools = !options.disableTools && this.supportsTools();
87
- const tools = shouldUseTools
88
- ? options.tools || (await this.getAllTools())
89
- : {};
90
- const messages = await this.buildMessagesForStream(options);
91
- const model = await this.getAISDKModelWithMiddleware(options);
92
- const result = await streamText({
93
- model,
94
- messages,
95
- temperature: options.temperature,
96
- maxOutputTokens: options.maxTokens,
97
- tools,
98
- stopWhen: stepCountIs(options.maxSteps || DEFAULT_MAX_STEPS),
99
- toolChoice: resolveToolChoice(options, tools, shouldUseTools),
100
- abortSignal: composeAbortSignals(options.abortSignal, timeoutController?.controller.signal),
101
- experimental_telemetry: this.telemetryHandler.getTelemetryConfig(options),
102
- experimental_repairToolCall: this.getToolCallRepairFn(options),
103
- onStepFinish: ({ toolCalls, toolResults }) => {
104
- emitToolEndFromStepFinish(this.neurolink?.getEventEmitter(), toolResults);
105
- this.handleToolExecutionStorage(toolCalls, toolResults, options, new Date()).catch((error) => {
106
- logger.warn("[CloudflareProvider] Failed to store tool executions", {
107
- provider: this.providerName,
108
- error: error instanceof Error ? error.message : String(error),
109
- });
110
- });
111
- },
112
- });
113
- timeoutController?.cleanup();
114
- const transformedStream = this.createTextStream(result);
115
- const analyticsPromise = streamAnalyticsCollector.createAnalytics(this.providerName, this.modelName, toAnalyticsStreamResult(result), Date.now() - startTime, {
116
- requestId: `cloudflare-stream-${Date.now()}`,
117
- streamingMode: true,
118
- });
119
- return {
120
- stream: transformedStream,
121
- provider: this.providerName,
122
- model: this.modelName,
123
- analytics: analyticsPromise,
124
- metadata: { startTime, streamId: `cloudflare-${Date.now()}` },
125
- };
126
- }
127
- catch (error) {
128
- timeoutController?.cleanup();
129
- throw this.handleProviderError(error);
130
- }
131
- }
132
50
  getProviderName() {
133
- return this.providerName;
51
+ return "cloudflare";
134
52
  }
135
53
  getDefaultModel() {
136
54
  return getDefaultCloudflareModel();
137
55
  }
138
- getAISDKModel() {
139
- return this.model;
56
+ getFallbackModelName() {
57
+ return CloudflareModels.LLAMA_3_1_8B_FAST;
58
+ }
59
+ getFallbackModels() {
60
+ return [
61
+ CloudflareModels.LLAMA_3_3_70B_FAST,
62
+ CloudflareModels.LLAMA_3_1_70B_INSTRUCT,
63
+ CloudflareModels.LLAMA_3_1_8B_FAST,
64
+ CloudflareModels.LLAMA_3_2_11B_VISION,
65
+ CloudflareModels.MISTRAL_7B_INSTRUCT_V0_2,
66
+ CloudflareModels.QWEN_1P5_14B_CHAT_AWQ,
67
+ ];
140
68
  }
141
69
  formatProviderError(error) {
142
70
  if (error instanceof TimeoutError) {
@@ -159,16 +87,4 @@ export class CloudflareProvider extends BaseProvider {
159
87
  }
160
88
  return new ProviderError(`Cloudflare Workers AI error: ${message}`, "cloudflare");
161
89
  }
162
- async validateConfiguration() {
163
- return typeof this.apiKey === "string" && this.apiKey.trim().length > 0;
164
- }
165
- getConfiguration() {
166
- return {
167
- provider: this.providerName,
168
- model: this.modelName,
169
- defaultModel: getDefaultCloudflareModel(),
170
- baseURL: this.baseURL,
171
- };
172
- }
173
90
  }
174
- export default CloudflareProvider;