@juspay/neurolink 9.68.6 → 9.68.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -0
- package/dist/browser/neurolink.min.js +325 -325
- package/dist/lib/providers/nvidiaNim.d.ts +28 -15
- package/dist/lib/providers/nvidiaNim.js +44 -294
- package/dist/providers/nvidiaNim.d.ts +28 -15
- package/dist/providers/nvidiaNim.js +44 -294
- package/package.json +1 -1
|
@@ -1,24 +1,37 @@
|
|
|
1
1
|
import type { AIProviderName } from "../constants/enums.js";
|
|
2
|
-
import {
|
|
3
|
-
import
|
|
4
|
-
|
|
2
|
+
import type { NeurolinkCredentials, OpenAICompatBuildBodyArgs } from "../types/index.js";
|
|
3
|
+
import { OpenAIChatCompletionsProvider } from "./openaiChatCompletionsBase.js";
|
|
4
|
+
declare const isNimFieldRejection: (body: string, field: string) => boolean;
|
|
5
5
|
/**
|
|
6
|
-
*
|
|
7
|
-
*
|
|
6
|
+
* Strip an offending field from a JSON request body and return the rebuilt
|
|
7
|
+
* stringified body. Returns `null` if the body isn't JSON-parseable or the
|
|
8
|
+
* field isn't present (signal: nothing to retry).
|
|
9
|
+
*/
|
|
10
|
+
declare const stripFieldFromJsonBody: (body: string, field: "reasoning_budget" | "chat_template") => string | null;
|
|
11
|
+
/**
|
|
12
|
+
* NVIDIA NIM Provider — native HTTP+SSE, no AI SDK.
|
|
13
|
+
*
|
|
14
|
+
* Wraps NVIDIA's hosted (or self-hosted) inference endpoints.
|
|
8
15
|
* Passes NIM-specific extras (top_k, min_p, repetition_penalty,
|
|
9
|
-
* chat_template_kwargs.reasoning_budget) via
|
|
10
|
-
*
|
|
16
|
+
* chat_template_kwargs.reasoning_budget) via adjustBuildBodyOptions.
|
|
17
|
+
* reasoning_content surfacing is a pending base-client follow-up (not emitted natively yet); all other behavior is preserved.
|
|
18
|
+
*
|
|
19
|
+
* @see https://docs.api.nvidia.com/nim/reference/
|
|
11
20
|
*/
|
|
12
|
-
export declare class NvidiaNimProvider extends
|
|
13
|
-
private model;
|
|
14
|
-
private apiKey;
|
|
15
|
-
private baseURL;
|
|
21
|
+
export declare class NvidiaNimProvider extends OpenAIChatCompletionsProvider {
|
|
16
22
|
constructor(modelName?: string, sdk?: unknown, _region?: string, credentials?: NeurolinkCredentials["nvidiaNim"]);
|
|
17
|
-
protected executeStream(options: StreamOptions, _analysisSchema?: ValidationSchema): Promise<StreamResult>;
|
|
18
|
-
private executeStreamInner;
|
|
19
23
|
protected getProviderName(): AIProviderName;
|
|
20
24
|
protected getDefaultModel(): string;
|
|
21
|
-
|
|
25
|
+
/**
|
|
26
|
+
* Inject NIM-specific body fields (top_k, min_p, repetition_penalty,
|
|
27
|
+
* min_tokens, chat_template, chat_template_kwargs) into the wire body.
|
|
28
|
+
*
|
|
29
|
+
* The base passes the full StreamOptions as `opts` at runtime (typed
|
|
30
|
+
* narrowly as OpenAICompatBuildBodyArgs["options"] for standard fields).
|
|
31
|
+
* We access thinkingLevel via an indexed-access cast since the runtime
|
|
32
|
+
* value is the full StreamOptions object.
|
|
33
|
+
*/
|
|
34
|
+
protected adjustBuildBodyOptions(_modelId: string, opts: OpenAICompatBuildBodyArgs["options"]): OpenAICompatBuildBodyArgs["options"] & Record<string, unknown>;
|
|
22
35
|
protected formatProviderError(error: unknown): Error;
|
|
23
36
|
validateConfiguration(): Promise<boolean>;
|
|
24
37
|
getConfiguration(): {
|
|
@@ -28,4 +41,4 @@ export declare class NvidiaNimProvider extends BaseProvider {
|
|
|
28
41
|
baseURL: string;
|
|
29
42
|
};
|
|
30
43
|
}
|
|
31
|
-
export
|
|
44
|
+
export { isNimFieldRejection, stripFieldFromJsonBody };
|
|
@@ -1,20 +1,9 @@
|
|
|
1
|
-
import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
|
|
2
1
|
import { NvidiaNimModels } from "../constants/enums.js";
|
|
3
|
-
import { BaseProvider } from "../core/baseProvider.js";
|
|
4
|
-
import { DEFAULT_MAX_STEPS } from "../core/constants.js";
|
|
5
|
-
import { streamAnalyticsCollector } from "../core/streamAnalytics.js";
|
|
6
|
-
import { isNeuroLink } from "../neurolink.js";
|
|
7
|
-
import { createProxyFetch, maskProxyUrl } from "../proxy/proxyFetch.js";
|
|
8
|
-
import { tracers, ATTR, withClientStreamSpan } from "../telemetry/index.js";
|
|
9
2
|
import { AuthenticationError, InvalidModelError, NetworkError, ProviderError, RateLimitError, } from "../types/index.js";
|
|
10
3
|
import { logger } from "../utils/logger.js";
|
|
11
4
|
import { createNvidiaNimConfig, getProviderModel, validateApiKey, } from "../utils/providerConfig.js";
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import { resolveToolChoice } from "../utils/toolChoice.js";
|
|
15
|
-
import { toAnalyticsStreamResult } from "./providerTypeUtils.js";
|
|
16
|
-
import { stepCountIs } from "../utils/tool.js";
|
|
17
|
-
import { streamText } from "../utils/generation.js";
|
|
5
|
+
import { TimeoutError } from "../utils/timeout.js";
|
|
6
|
+
import { OpenAIChatCompletionsProvider } from "./openaiChatCompletionsBase.js";
|
|
18
7
|
/**
|
|
19
8
|
* Decide whether a NIM 400 response body is a rejection of the named
|
|
20
9
|
* field (as opposed to an unrelated 400 that happens to mention the
|
|
@@ -94,71 +83,6 @@ const stripFieldFromJsonBody = (body, field) => {
|
|
|
94
83
|
return null;
|
|
95
84
|
}
|
|
96
85
|
};
|
|
97
|
-
const makeLoggingFetch = (provider) => {
|
|
98
|
-
const base = createProxyFetch();
|
|
99
|
-
return (async (input, init) => {
|
|
100
|
-
const url = typeof input === "string"
|
|
101
|
-
? input
|
|
102
|
-
: input instanceof URL
|
|
103
|
-
? input.toString()
|
|
104
|
-
: input.url;
|
|
105
|
-
const reqSize = init?.body && typeof init.body === "string" ? init.body.length : 0;
|
|
106
|
-
let response = await base(input, init);
|
|
107
|
-
// Generic NIM 400 retry-strip: works for BOTH generate and stream paths.
|
|
108
|
-
// NIM sometimes returns HTTP 400 when a model rejects `reasoning_budget`
|
|
109
|
-
// or `chat_template`. The stream path already retries by reconstructing
|
|
110
|
-
// its provider options; this fetch-level retry is the symmetric fix for
|
|
111
|
-
// generate (and any other transport that lands here).
|
|
112
|
-
//
|
|
113
|
-
// We require BOTH (a) the offending field name AND (b) a rejection
|
|
114
|
-
// keyword (unsupported / not supported / unknown / invalid /
|
|
115
|
-
// unrecognized / does not support) within 80 chars of it. Without the
|
|
116
|
-
// rejection-keyword guard, an unrelated 400 whose error body happened
|
|
117
|
-
// to mention `chat_template` (e.g. the user prompt got echoed back)
|
|
118
|
-
// would cause us to silently strip a field the user actually wanted
|
|
119
|
-
// sent, and either succeed for the wrong reason or fail with a
|
|
120
|
-
// misleading error.
|
|
121
|
-
if (response.status === 400 &&
|
|
122
|
-
typeof init?.body === "string" &&
|
|
123
|
-
init.body.length > 0) {
|
|
124
|
-
const cloned = response.clone();
|
|
125
|
-
const body = await cloned.text().catch(() => "");
|
|
126
|
-
let retryBody = null;
|
|
127
|
-
let stripped = null;
|
|
128
|
-
if (isNimFieldRejection(body, "reasoning_budget")) {
|
|
129
|
-
retryBody = stripFieldFromJsonBody(init.body, "reasoning_budget");
|
|
130
|
-
stripped = "reasoning_budget";
|
|
131
|
-
}
|
|
132
|
-
else if (isNimFieldRejection(body, "chat_template")) {
|
|
133
|
-
retryBody = stripFieldFromJsonBody(init.body, "chat_template");
|
|
134
|
-
stripped = "chat_template";
|
|
135
|
-
}
|
|
136
|
-
if (retryBody !== null && stripped !== null) {
|
|
137
|
-
logger.warn(`[${provider}] NIM rejected ${stripped}; retrying with field stripped`);
|
|
138
|
-
response = await base(input, { ...init, body: retryBody });
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
if (!response.ok) {
|
|
142
|
-
// If maskProxyUrl can't safely sanitize the URL (returns null), don't
|
|
143
|
-
// log the raw URL — that defeats the redaction. Use a placeholder so
|
|
144
|
-
// operators still get the warning without leaking credentials.
|
|
145
|
-
const safeUrl = maskProxyUrl(url) ?? "<redacted>";
|
|
146
|
-
if (process.env.NEUROLINK_DEBUG_HTTP === "1") {
|
|
147
|
-
const clone = response.clone();
|
|
148
|
-
const body = await clone.text().catch(() => "<unreadable>");
|
|
149
|
-
logger.warn(`[${provider}] upstream ${response.status}`, {
|
|
150
|
-
url: safeUrl,
|
|
151
|
-
body: body.slice(0, 800),
|
|
152
|
-
reqSize,
|
|
153
|
-
});
|
|
154
|
-
}
|
|
155
|
-
else {
|
|
156
|
-
logger.warn(`[${provider}] upstream ${response.status} url=${safeUrl} reqSize=${reqSize}`);
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
|
-
return response;
|
|
160
|
-
});
|
|
161
|
-
};
|
|
162
86
|
const NVIDIA_NIM_DEFAULT_BASE_URL = "https://integrate.api.nvidia.com/v1";
|
|
163
87
|
const envInt = (k) => {
|
|
164
88
|
const v = process.env[k];
|
|
@@ -207,21 +131,6 @@ const buildNvidiaNimExtraBody = (thinkingEnabled, maxTokens) => {
|
|
|
207
131
|
}
|
|
208
132
|
return extra;
|
|
209
133
|
};
|
|
210
|
-
const stripReasoningBudget = (body) => {
|
|
211
|
-
const cloned = { ...body };
|
|
212
|
-
if (cloned.chat_template_kwargs) {
|
|
213
|
-
const { reasoning_budget: _ignored, ...rest } = cloned.chat_template_kwargs;
|
|
214
|
-
cloned.chat_template_kwargs = rest;
|
|
215
|
-
if (Object.keys(cloned.chat_template_kwargs).length === 0) {
|
|
216
|
-
delete cloned.chat_template_kwargs;
|
|
217
|
-
}
|
|
218
|
-
}
|
|
219
|
-
return cloned;
|
|
220
|
-
};
|
|
221
|
-
const stripChatTemplate = (body) => {
|
|
222
|
-
const { chat_template: _ignored, ...rest } = body;
|
|
223
|
-
return rest;
|
|
224
|
-
};
|
|
225
134
|
const getNimApiKey = () => {
|
|
226
135
|
return validateApiKey(createNvidiaNimConfig());
|
|
227
136
|
};
|
|
@@ -229,222 +138,61 @@ const getDefaultNimModel = () => {
|
|
|
229
138
|
return getProviderModel("NVIDIA_NIM_MODEL", NvidiaNimModels.LLAMA_3_3_70B_INSTRUCT);
|
|
230
139
|
};
|
|
231
140
|
/**
|
|
232
|
-
* NVIDIA NIM Provider
|
|
233
|
-
*
|
|
141
|
+
* NVIDIA NIM Provider — native HTTP+SSE, no AI SDK.
|
|
142
|
+
*
|
|
143
|
+
* Wraps NVIDIA's hosted (or self-hosted) inference endpoints.
|
|
234
144
|
* Passes NIM-specific extras (top_k, min_p, repetition_penalty,
|
|
235
|
-
* chat_template_kwargs.reasoning_budget) via
|
|
236
|
-
*
|
|
145
|
+
* chat_template_kwargs.reasoning_budget) via adjustBuildBodyOptions.
|
|
146
|
+
* reasoning_content surfacing is a pending base-client follow-up (not emitted natively yet); all other behavior is preserved.
|
|
147
|
+
*
|
|
148
|
+
* @see https://docs.api.nvidia.com/nim/reference/
|
|
237
149
|
*/
|
|
238
|
-
export class NvidiaNimProvider extends
|
|
239
|
-
model;
|
|
240
|
-
apiKey;
|
|
241
|
-
baseURL;
|
|
150
|
+
export class NvidiaNimProvider extends OpenAIChatCompletionsProvider {
|
|
242
151
|
constructor(modelName, sdk, _region, credentials) {
|
|
243
|
-
const validatedNeurolink = isNeuroLink(sdk) ? sdk : undefined;
|
|
244
|
-
super(modelName, "nvidia-nim", validatedNeurolink);
|
|
245
152
|
// Trim the override before applying precedence. A blank/whitespace
|
|
246
153
|
// `credentials.apiKey` should NOT bypass `getNimApiKey()` — that would
|
|
247
154
|
// build a client with an unusable bearer token and fail at request time
|
|
248
155
|
// with a confusing 401 instead of at construction time.
|
|
249
156
|
const overrideApiKey = credentials?.apiKey?.trim();
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
NVIDIA_NIM_DEFAULT_BASE_URL;
|
|
258
|
-
// We deliberately use `@ai-sdk/openai-compatible` rather than
|
|
259
|
-
// `@ai-sdk/openai`. Two upstream behaviors of `@ai-sdk/openai` break us:
|
|
260
|
-
// 1. It always sends `response_format: { type: "json_schema" }` when a
|
|
261
|
-
// schema is provided. Most NIM-served chat models don't enforce
|
|
262
|
-
// json_schema strictly — the schema goes through but `result.object`
|
|
263
|
-
// stays empty because the SDK never gets the typed response back.
|
|
264
|
-
// 2. It does not parse the `reasoning_content` field that NIM-hosted
|
|
265
|
-
// reasoning models (deepseek-r1, qwq, llama-nemotron-ultra) emit,
|
|
266
|
-
// so chain-of-thought is silently dropped.
|
|
267
|
-
// `@ai-sdk/openai-compatible` honors `supportsStructuredOutputs: false`
|
|
268
|
-
// (falls back to `{ type: "json_object" }` and injects the schema into
|
|
269
|
-
// the prompt — works across the entire NIM model fleet) and parses both
|
|
270
|
-
// `choice.message.reasoning_content` and `delta.reasoning_content` into
|
|
271
|
-
// the SDK-standard `reasoning` part. NIM-specific extras (`min_tokens`,
|
|
272
|
-
// `chat_template_kwargs.reasoning_budget`, `chat_template`) are still
|
|
273
|
-
// injected via `providerOptions.openai.body` in `executeStreamInner`.
|
|
274
|
-
const nim = createOpenAICompatible({
|
|
275
|
-
name: "nvidia-nim",
|
|
276
|
-
apiKey: this.apiKey,
|
|
277
|
-
baseURL: this.baseURL,
|
|
278
|
-
fetch: makeLoggingFetch("nvidia-nim"),
|
|
279
|
-
supportsStructuredOutputs: false,
|
|
280
|
-
includeUsage: true,
|
|
281
|
-
});
|
|
282
|
-
this.model = nim.chatModel(this.modelName);
|
|
157
|
+
const apiKey = overrideApiKey && overrideApiKey.length > 0
|
|
158
|
+
? overrideApiKey
|
|
159
|
+
: getNimApiKey();
|
|
160
|
+
const baseURL = credentials?.baseURL ??
|
|
161
|
+
process.env.NVIDIA_NIM_BASE_URL ??
|
|
162
|
+
NVIDIA_NIM_DEFAULT_BASE_URL;
|
|
163
|
+
super("nvidia-nim", modelName, sdk, { baseURL, apiKey });
|
|
283
164
|
logger.debug("NVIDIA NIM Provider initialized", {
|
|
284
165
|
modelName: this.modelName,
|
|
285
166
|
providerName: this.providerName,
|
|
286
|
-
baseURL: this.baseURL,
|
|
167
|
+
baseURL: this.config.baseURL,
|
|
287
168
|
});
|
|
288
169
|
}
|
|
289
|
-
async executeStream(options, _analysisSchema) {
|
|
290
|
-
return withClientStreamSpan({
|
|
291
|
-
name: "neurolink.provider.stream",
|
|
292
|
-
tracer: tracers.provider,
|
|
293
|
-
attributes: {
|
|
294
|
-
[ATTR.GEN_AI_SYSTEM]: "nvidia-nim",
|
|
295
|
-
[ATTR.GEN_AI_MODEL]: this.modelName,
|
|
296
|
-
[ATTR.GEN_AI_OPERATION]: "stream",
|
|
297
|
-
[ATTR.NL_STREAM_MODE]: true,
|
|
298
|
-
},
|
|
299
|
-
}, async () => this.executeStreamInner(options), (r) => r.stream, (r, wrapped) => ({ ...r, stream: wrapped }));
|
|
300
|
-
}
|
|
301
|
-
async executeStreamInner(options) {
|
|
302
|
-
this.validateStreamOptions(options);
|
|
303
|
-
const startTime = Date.now();
|
|
304
|
-
const timeout = this.getTimeout(options);
|
|
305
|
-
const timeoutController = createTimeoutController(timeout, this.providerName, "stream");
|
|
306
|
-
try {
|
|
307
|
-
const shouldUseTools = !options.disableTools && this.supportsTools();
|
|
308
|
-
const tools = shouldUseTools
|
|
309
|
-
? options.tools || (await this.getAllTools())
|
|
310
|
-
: {};
|
|
311
|
-
const messages = await this.buildMessagesForStream(options);
|
|
312
|
-
const model = await this.getAISDKModelWithMiddleware(options);
|
|
313
|
-
// Callers pass `thinkingLevel` directly on generate/stream options
|
|
314
|
-
// (matching Anthropic / Gemini 2.5+ / Gemini 3 conventions). Fall back
|
|
315
|
-
// to the legacy `thinkingConfig.thinkingLevel` shape for compatibility.
|
|
316
|
-
const tl = options.thinkingLevel ??
|
|
317
|
-
options.thinkingConfig?.thinkingLevel;
|
|
318
|
-
const thinkingEnabled = tl !== undefined && tl !== "minimal";
|
|
319
|
-
let extraBody = buildNvidiaNimExtraBody(thinkingEnabled, options.maxTokens);
|
|
320
|
-
// Inline the retry-strip union — CLAUDE.md rule 2 forbids type aliases
|
|
321
|
-
// outside src/lib/types/. The two literals match the 400 error keys NIM
|
|
322
|
-
// returns for the only two extras we know how to drop and retry.
|
|
323
|
-
const callStream = (body, stripped = []) => streamText({
|
|
324
|
-
model,
|
|
325
|
-
messages,
|
|
326
|
-
temperature: options.temperature,
|
|
327
|
-
maxOutputTokens: options.maxTokens,
|
|
328
|
-
tools,
|
|
329
|
-
stopWhen: stepCountIs(options.maxSteps || DEFAULT_MAX_STEPS),
|
|
330
|
-
toolChoice: resolveToolChoice(options, tools, shouldUseTools),
|
|
331
|
-
abortSignal: composeAbortSignals(options.abortSignal, timeoutController?.controller.signal),
|
|
332
|
-
providerOptions: (() => {
|
|
333
|
-
// StreamOptions doesn't formally type providerOptions but the
|
|
334
|
-
// upstream Vercel AI SDK accepts it. Read it via an indexed access
|
|
335
|
-
// and merge with NIM extras instead of overwriting any per-call
|
|
336
|
-
// openai.body.
|
|
337
|
-
const callerBase = options
|
|
338
|
-
.providerOptions ?? {};
|
|
339
|
-
const callerOpenai = callerBase.openai ?? {};
|
|
340
|
-
const callerBody = callerOpenai.body ?? {};
|
|
341
|
-
// Per-call overrides win over env/NIM defaults — defaults first,
|
|
342
|
-
// overrides last. chat_template_kwargs is merged shallowly too so
|
|
343
|
-
// a request that only sets `reasoning_budget` doesn't drop the
|
|
344
|
-
// env-driven `thinking: true` flag (and vice versa).
|
|
345
|
-
const defaultsBody = body;
|
|
346
|
-
const mergedBody = {
|
|
347
|
-
...defaultsBody,
|
|
348
|
-
...callerBody,
|
|
349
|
-
};
|
|
350
|
-
const mergedKwargs = {
|
|
351
|
-
...(defaultsBody.chat_template_kwargs ?? {}),
|
|
352
|
-
...(callerBody.chat_template_kwargs ?? {}),
|
|
353
|
-
};
|
|
354
|
-
// Apply retry-strip AFTER merging so caller-supplied copies of
|
|
355
|
-
// the offending field are also dropped (otherwise the retry would
|
|
356
|
-
// re-send the field that NIM just rejected).
|
|
357
|
-
if (stripped.includes("chat_template")) {
|
|
358
|
-
delete mergedBody.chat_template;
|
|
359
|
-
}
|
|
360
|
-
if (stripped.includes("reasoning_budget")) {
|
|
361
|
-
delete mergedKwargs.reasoning_budget;
|
|
362
|
-
}
|
|
363
|
-
if (Object.keys(mergedKwargs).length > 0) {
|
|
364
|
-
mergedBody.chat_template_kwargs = mergedKwargs;
|
|
365
|
-
}
|
|
366
|
-
else {
|
|
367
|
-
delete mergedBody.chat_template_kwargs;
|
|
368
|
-
}
|
|
369
|
-
if (Object.keys(callerBase).length === 0 &&
|
|
370
|
-
Object.keys(mergedBody).length === 0) {
|
|
371
|
-
return undefined;
|
|
372
|
-
}
|
|
373
|
-
return {
|
|
374
|
-
...callerBase,
|
|
375
|
-
openai: {
|
|
376
|
-
...callerOpenai,
|
|
377
|
-
body: mergedBody,
|
|
378
|
-
},
|
|
379
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
380
|
-
};
|
|
381
|
-
})(),
|
|
382
|
-
experimental_telemetry: this.telemetryHandler.getTelemetryConfig(options),
|
|
383
|
-
experimental_repairToolCall: this.getToolCallRepairFn(options),
|
|
384
|
-
onStepFinish: ({ toolCalls, toolResults }) => {
|
|
385
|
-
emitToolEndFromStepFinish(this.neurolink?.getEventEmitter(), toolResults);
|
|
386
|
-
this.handleToolExecutionStorage(toolCalls, toolResults, options, new Date()).catch((error) => {
|
|
387
|
-
logger.warn("[NvidiaNimProvider] Failed to store tool executions", {
|
|
388
|
-
provider: this.providerName,
|
|
389
|
-
error: error instanceof Error ? error.message : String(error),
|
|
390
|
-
});
|
|
391
|
-
});
|
|
392
|
-
},
|
|
393
|
-
});
|
|
394
|
-
let result;
|
|
395
|
-
try {
|
|
396
|
-
result = await callStream(extraBody);
|
|
397
|
-
}
|
|
398
|
-
catch (error) {
|
|
399
|
-
const errMsg = error instanceof Error ? error.message : String(error);
|
|
400
|
-
const status = error?.statusCode;
|
|
401
|
-
if (status === 400) {
|
|
402
|
-
const lower = errMsg.toLowerCase();
|
|
403
|
-
if (lower.includes("reasoning_budget")) {
|
|
404
|
-
logger.warn("NIM rejected reasoning_budget; retrying without it");
|
|
405
|
-
extraBody = stripReasoningBudget(extraBody);
|
|
406
|
-
result = await callStream(extraBody, ["reasoning_budget"]);
|
|
407
|
-
}
|
|
408
|
-
else if (lower.includes("chat_template")) {
|
|
409
|
-
logger.warn("NIM rejected chat_template; retrying without it");
|
|
410
|
-
extraBody = stripChatTemplate(extraBody);
|
|
411
|
-
result = await callStream(extraBody, ["chat_template"]);
|
|
412
|
-
}
|
|
413
|
-
else {
|
|
414
|
-
throw error;
|
|
415
|
-
}
|
|
416
|
-
}
|
|
417
|
-
else {
|
|
418
|
-
throw error;
|
|
419
|
-
}
|
|
420
|
-
}
|
|
421
|
-
timeoutController?.cleanup();
|
|
422
|
-
const transformedStream = this.createTextStream(result);
|
|
423
|
-
const analyticsPromise = streamAnalyticsCollector.createAnalytics(this.providerName, this.modelName, toAnalyticsStreamResult(result), Date.now() - startTime, {
|
|
424
|
-
requestId: `nvidia-nim-stream-${Date.now()}`,
|
|
425
|
-
streamingMode: true,
|
|
426
|
-
});
|
|
427
|
-
return {
|
|
428
|
-
stream: transformedStream,
|
|
429
|
-
provider: this.providerName,
|
|
430
|
-
model: this.modelName,
|
|
431
|
-
analytics: analyticsPromise,
|
|
432
|
-
metadata: { startTime, streamId: `nvidia-nim-${Date.now()}` },
|
|
433
|
-
};
|
|
434
|
-
}
|
|
435
|
-
catch (error) {
|
|
436
|
-
timeoutController?.cleanup();
|
|
437
|
-
throw this.handleProviderError(error);
|
|
438
|
-
}
|
|
439
|
-
}
|
|
440
170
|
getProviderName() {
|
|
441
|
-
return
|
|
171
|
+
return "nvidia-nim";
|
|
442
172
|
}
|
|
443
173
|
getDefaultModel() {
|
|
444
174
|
return getDefaultNimModel();
|
|
445
175
|
}
|
|
446
|
-
|
|
447
|
-
|
|
176
|
+
/**
|
|
177
|
+
* Inject NIM-specific body fields (top_k, min_p, repetition_penalty,
|
|
178
|
+
* min_tokens, chat_template, chat_template_kwargs) into the wire body.
|
|
179
|
+
*
|
|
180
|
+
* The base passes the full StreamOptions as `opts` at runtime (typed
|
|
181
|
+
* narrowly as OpenAICompatBuildBodyArgs["options"] for standard fields).
|
|
182
|
+
* We access thinkingLevel via an indexed-access cast since the runtime
|
|
183
|
+
* value is the full StreamOptions object.
|
|
184
|
+
*/
|
|
185
|
+
adjustBuildBodyOptions(_modelId, opts) {
|
|
186
|
+
// The runtime value of opts is the full StreamOptions; TypeScript types
|
|
187
|
+
// it narrowly so we cast to access NIM-specific thinking fields.
|
|
188
|
+
const fullOpts = opts;
|
|
189
|
+
const thinkingConfigRaw = fullOpts.thinkingConfig;
|
|
190
|
+
const tl = fullOpts.thinkingLevel ??
|
|
191
|
+
thinkingConfigRaw?.thinkingLevel;
|
|
192
|
+
const thinkingEnabled = tl !== undefined && tl !== "minimal";
|
|
193
|
+
const maxTokens = typeof fullOpts.maxTokens === "number" ? fullOpts.maxTokens : undefined;
|
|
194
|
+
const extra = buildNvidiaNimExtraBody(thinkingEnabled, maxTokens);
|
|
195
|
+
return { ...opts, ...extra };
|
|
448
196
|
}
|
|
449
197
|
formatProviderError(error) {
|
|
450
198
|
if (error instanceof TimeoutError) {
|
|
@@ -481,16 +229,18 @@ export class NvidiaNimProvider extends BaseProvider {
|
|
|
481
229
|
return new ProviderError(`NVIDIA NIM error: ${message}`, "nvidia-nim");
|
|
482
230
|
}
|
|
483
231
|
async validateConfiguration() {
|
|
484
|
-
return typeof this.apiKey === "string" &&
|
|
232
|
+
return (typeof this.config.apiKey === "string" &&
|
|
233
|
+
this.config.apiKey.trim().length > 0);
|
|
485
234
|
}
|
|
486
235
|
getConfiguration() {
|
|
487
236
|
return {
|
|
488
237
|
provider: this.providerName,
|
|
489
238
|
model: this.modelName,
|
|
490
239
|
defaultModel: getDefaultNimModel(),
|
|
491
|
-
baseURL: this.baseURL,
|
|
240
|
+
baseURL: this.config.baseURL,
|
|
492
241
|
};
|
|
493
242
|
}
|
|
494
243
|
}
|
|
495
|
-
|
|
244
|
+
// Exported for test suites that probe NIM's 400-retry behaviour directly.
|
|
245
|
+
export { isNimFieldRejection, stripFieldFromJsonBody };
|
|
496
246
|
//# sourceMappingURL=nvidiaNim.js.map
|
|
@@ -1,24 +1,37 @@
|
|
|
1
1
|
import type { AIProviderName } from "../constants/enums.js";
|
|
2
|
-
import {
|
|
3
|
-
import
|
|
4
|
-
|
|
2
|
+
import type { NeurolinkCredentials, OpenAICompatBuildBodyArgs } from "../types/index.js";
|
|
3
|
+
import { OpenAIChatCompletionsProvider } from "./openaiChatCompletionsBase.js";
|
|
4
|
+
declare const isNimFieldRejection: (body: string, field: string) => boolean;
|
|
5
5
|
/**
|
|
6
|
-
*
|
|
7
|
-
*
|
|
6
|
+
* Strip an offending field from a JSON request body and return the rebuilt
|
|
7
|
+
* stringified body. Returns `null` if the body isn't JSON-parseable or the
|
|
8
|
+
* field isn't present (signal: nothing to retry).
|
|
9
|
+
*/
|
|
10
|
+
declare const stripFieldFromJsonBody: (body: string, field: "reasoning_budget" | "chat_template") => string | null;
|
|
11
|
+
/**
|
|
12
|
+
* NVIDIA NIM Provider — native HTTP+SSE, no AI SDK.
|
|
13
|
+
*
|
|
14
|
+
* Wraps NVIDIA's hosted (or self-hosted) inference endpoints.
|
|
8
15
|
* Passes NIM-specific extras (top_k, min_p, repetition_penalty,
|
|
9
|
-
* chat_template_kwargs.reasoning_budget) via
|
|
10
|
-
*
|
|
16
|
+
* chat_template_kwargs.reasoning_budget) via adjustBuildBodyOptions.
|
|
17
|
+
* reasoning_content surfacing is a pending base-client follow-up (not emitted natively yet); all other behavior is preserved.
|
|
18
|
+
*
|
|
19
|
+
* @see https://docs.api.nvidia.com/nim/reference/
|
|
11
20
|
*/
|
|
12
|
-
export declare class NvidiaNimProvider extends
|
|
13
|
-
private model;
|
|
14
|
-
private apiKey;
|
|
15
|
-
private baseURL;
|
|
21
|
+
export declare class NvidiaNimProvider extends OpenAIChatCompletionsProvider {
|
|
16
22
|
constructor(modelName?: string, sdk?: unknown, _region?: string, credentials?: NeurolinkCredentials["nvidiaNim"]);
|
|
17
|
-
protected executeStream(options: StreamOptions, _analysisSchema?: ValidationSchema): Promise<StreamResult>;
|
|
18
|
-
private executeStreamInner;
|
|
19
23
|
protected getProviderName(): AIProviderName;
|
|
20
24
|
protected getDefaultModel(): string;
|
|
21
|
-
|
|
25
|
+
/**
|
|
26
|
+
* Inject NIM-specific body fields (top_k, min_p, repetition_penalty,
|
|
27
|
+
* min_tokens, chat_template, chat_template_kwargs) into the wire body.
|
|
28
|
+
*
|
|
29
|
+
* The base passes the full StreamOptions as `opts` at runtime (typed
|
|
30
|
+
* narrowly as OpenAICompatBuildBodyArgs["options"] for standard fields).
|
|
31
|
+
* We access thinkingLevel via an indexed-access cast since the runtime
|
|
32
|
+
* value is the full StreamOptions object.
|
|
33
|
+
*/
|
|
34
|
+
protected adjustBuildBodyOptions(_modelId: string, opts: OpenAICompatBuildBodyArgs["options"]): OpenAICompatBuildBodyArgs["options"] & Record<string, unknown>;
|
|
22
35
|
protected formatProviderError(error: unknown): Error;
|
|
23
36
|
validateConfiguration(): Promise<boolean>;
|
|
24
37
|
getConfiguration(): {
|
|
@@ -28,4 +41,4 @@ export declare class NvidiaNimProvider extends BaseProvider {
|
|
|
28
41
|
baseURL: string;
|
|
29
42
|
};
|
|
30
43
|
}
|
|
31
|
-
export
|
|
44
|
+
export { isNimFieldRejection, stripFieldFromJsonBody };
|