@juspay/neurolink 9.64.0 → 9.65.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (322) hide show
  1. package/CHANGELOG.md +6 -0
  2. package/README.md +18 -17
  3. package/dist/adapters/providerImageAdapter.js +29 -1
  4. package/dist/adapters/replicate/auth.d.ts +19 -0
  5. package/dist/adapters/replicate/auth.js +32 -0
  6. package/dist/adapters/replicate/predictionLifecycle.d.ts +46 -0
  7. package/dist/adapters/replicate/predictionLifecycle.js +283 -0
  8. package/dist/adapters/video/klingVideoHandler.d.ts +37 -0
  9. package/dist/adapters/video/klingVideoHandler.js +305 -0
  10. package/dist/adapters/video/replicateVideoHandler.d.ts +29 -0
  11. package/dist/adapters/video/replicateVideoHandler.js +157 -0
  12. package/dist/adapters/video/runwayVideoHandler.d.ts +32 -0
  13. package/dist/adapters/video/runwayVideoHandler.js +316 -0
  14. package/dist/adapters/video/vertexVideoHandler.d.ts +19 -1
  15. package/dist/adapters/video/vertexVideoHandler.js +33 -9
  16. package/dist/autoresearch/runner.js +8 -2
  17. package/dist/avatar/index.d.ts +13 -0
  18. package/dist/avatar/index.js +13 -0
  19. package/dist/avatar/providers/DIDAvatar.d.ts +49 -0
  20. package/dist/avatar/providers/DIDAvatar.js +501 -0
  21. package/dist/avatar/providers/HeyGenAvatar.d.ts +30 -0
  22. package/dist/avatar/providers/HeyGenAvatar.js +337 -0
  23. package/dist/avatar/providers/ReplicateAvatar.d.ts +36 -0
  24. package/dist/avatar/providers/ReplicateAvatar.js +267 -0
  25. package/dist/browser/neurolink.min.js +633 -610
  26. package/dist/cli/commands/mcp.js +29 -0
  27. package/dist/cli/commands/proxy.js +24 -5
  28. package/dist/cli/factories/commandFactory.d.ts +11 -1
  29. package/dist/cli/factories/commandFactory.js +291 -38
  30. package/dist/constants/contextWindows.js +101 -0
  31. package/dist/constants/enums.d.ts +273 -2
  32. package/dist/constants/enums.js +290 -1
  33. package/dist/constants/videoErrors.d.ts +4 -0
  34. package/dist/constants/videoErrors.js +4 -0
  35. package/dist/core/baseProvider.d.ts +22 -2
  36. package/dist/core/baseProvider.js +217 -11
  37. package/dist/core/constants.d.ts +11 -0
  38. package/dist/core/constants.js +69 -1
  39. package/dist/core/redisConversationMemoryManager.js +6 -0
  40. package/dist/evaluation/index.d.ts +2 -0
  41. package/dist/evaluation/index.js +4 -0
  42. package/dist/factories/providerFactory.js +7 -1
  43. package/dist/factories/providerRegistry.js +202 -5
  44. package/dist/features/ppt/contentPlanner.js +42 -14
  45. package/dist/index.d.ts +9 -1
  46. package/dist/index.js +16 -1
  47. package/dist/lib/adapters/providerImageAdapter.js +29 -1
  48. package/dist/lib/adapters/replicate/auth.d.ts +19 -0
  49. package/dist/lib/adapters/replicate/auth.js +33 -0
  50. package/dist/lib/adapters/replicate/predictionLifecycle.d.ts +46 -0
  51. package/dist/lib/adapters/replicate/predictionLifecycle.js +284 -0
  52. package/dist/lib/adapters/video/klingVideoHandler.d.ts +37 -0
  53. package/dist/lib/adapters/video/klingVideoHandler.js +306 -0
  54. package/dist/lib/adapters/video/replicateVideoHandler.d.ts +29 -0
  55. package/dist/lib/adapters/video/replicateVideoHandler.js +158 -0
  56. package/dist/lib/adapters/video/runwayVideoHandler.d.ts +32 -0
  57. package/dist/lib/adapters/video/runwayVideoHandler.js +317 -0
  58. package/dist/lib/adapters/video/vertexVideoHandler.d.ts +19 -1
  59. package/dist/lib/adapters/video/vertexVideoHandler.js +33 -9
  60. package/dist/lib/autoresearch/runner.js +8 -2
  61. package/dist/lib/avatar/index.d.ts +13 -0
  62. package/dist/lib/avatar/index.js +14 -0
  63. package/dist/lib/avatar/providers/DIDAvatar.d.ts +49 -0
  64. package/dist/lib/avatar/providers/DIDAvatar.js +502 -0
  65. package/dist/lib/avatar/providers/HeyGenAvatar.d.ts +30 -0
  66. package/dist/lib/avatar/providers/HeyGenAvatar.js +338 -0
  67. package/dist/lib/avatar/providers/ReplicateAvatar.d.ts +36 -0
  68. package/dist/lib/avatar/providers/ReplicateAvatar.js +268 -0
  69. package/dist/lib/constants/contextWindows.js +101 -0
  70. package/dist/lib/constants/enums.d.ts +273 -2
  71. package/dist/lib/constants/enums.js +290 -1
  72. package/dist/lib/constants/videoErrors.d.ts +4 -0
  73. package/dist/lib/constants/videoErrors.js +4 -0
  74. package/dist/lib/core/baseProvider.d.ts +22 -2
  75. package/dist/lib/core/baseProvider.js +217 -11
  76. package/dist/lib/core/constants.d.ts +11 -0
  77. package/dist/lib/core/constants.js +69 -1
  78. package/dist/lib/core/redisConversationMemoryManager.js +6 -0
  79. package/dist/lib/evaluation/index.d.ts +2 -0
  80. package/dist/lib/evaluation/index.js +4 -0
  81. package/dist/lib/factories/providerFactory.js +7 -1
  82. package/dist/lib/factories/providerRegistry.js +202 -5
  83. package/dist/lib/features/ppt/contentPlanner.js +42 -14
  84. package/dist/lib/index.d.ts +9 -1
  85. package/dist/lib/index.js +16 -1
  86. package/dist/lib/middleware/builtin/lifecycle.js +39 -9
  87. package/dist/lib/music/index.d.ts +13 -0
  88. package/dist/lib/music/index.js +14 -0
  89. package/dist/lib/music/providers/BeatovenMusic.d.ts +31 -0
  90. package/dist/lib/music/providers/BeatovenMusic.js +334 -0
  91. package/dist/lib/music/providers/ElevenLabsMusic.d.ts +30 -0
  92. package/dist/lib/music/providers/ElevenLabsMusic.js +169 -0
  93. package/dist/lib/music/providers/LyriaMusic.d.ts +29 -0
  94. package/dist/lib/music/providers/LyriaMusic.js +173 -0
  95. package/dist/lib/music/providers/ReplicateMusic.d.ts +31 -0
  96. package/dist/lib/music/providers/ReplicateMusic.js +262 -0
  97. package/dist/lib/neurolink.d.ts +30 -0
  98. package/dist/lib/neurolink.js +323 -77
  99. package/dist/lib/providers/amazonBedrock.d.ts +10 -0
  100. package/dist/lib/providers/amazonBedrock.js +94 -39
  101. package/dist/lib/providers/anthropic.js +55 -7
  102. package/dist/lib/providers/anthropicBaseProvider.js +1 -1
  103. package/dist/lib/providers/azureOpenai.js +66 -17
  104. package/dist/lib/providers/cloudflare.d.ts +35 -0
  105. package/dist/lib/providers/cloudflare.js +174 -0
  106. package/dist/lib/providers/cohere.d.ts +52 -0
  107. package/dist/lib/providers/cohere.js +253 -0
  108. package/dist/lib/providers/deepseek.js +72 -17
  109. package/dist/lib/providers/fireworks.d.ts +33 -0
  110. package/dist/lib/providers/fireworks.js +164 -0
  111. package/dist/lib/providers/googleAiStudio.js +45 -6
  112. package/dist/lib/providers/googleNativeGemini3.d.ts +24 -1
  113. package/dist/lib/providers/googleNativeGemini3.js +173 -21
  114. package/dist/lib/providers/googleVertex.js +173 -17
  115. package/dist/lib/providers/groq.d.ts +33 -0
  116. package/dist/lib/providers/groq.js +181 -0
  117. package/dist/lib/providers/huggingFace.js +9 -8
  118. package/dist/lib/providers/ideogram.d.ts +34 -0
  119. package/dist/lib/providers/ideogram.js +184 -0
  120. package/dist/lib/providers/index.d.ts +13 -0
  121. package/dist/lib/providers/index.js +13 -0
  122. package/dist/lib/providers/jina.d.ts +59 -0
  123. package/dist/lib/providers/jina.js +218 -0
  124. package/dist/lib/providers/llamaCpp.js +14 -46
  125. package/dist/lib/providers/lmStudio.js +14 -47
  126. package/dist/lib/providers/mistral.js +7 -7
  127. package/dist/lib/providers/nvidiaNim.js +160 -19
  128. package/dist/lib/providers/ollama.js +7 -7
  129. package/dist/lib/providers/openAI.d.ts +22 -1
  130. package/dist/lib/providers/openAI.js +181 -0
  131. package/dist/lib/providers/openRouter.js +35 -23
  132. package/dist/lib/providers/openaiCompatible.js +9 -8
  133. package/dist/lib/providers/perplexity.d.ts +33 -0
  134. package/dist/lib/providers/perplexity.js +179 -0
  135. package/dist/lib/providers/recraft.d.ts +34 -0
  136. package/dist/lib/providers/recraft.js +197 -0
  137. package/dist/lib/providers/replicate.d.ts +75 -0
  138. package/dist/lib/providers/replicate.js +403 -0
  139. package/dist/lib/providers/stability.d.ts +37 -0
  140. package/dist/lib/providers/stability.js +191 -0
  141. package/dist/lib/providers/togetherAi.d.ts +33 -0
  142. package/dist/lib/providers/togetherAi.js +176 -0
  143. package/dist/lib/providers/voyage.d.ts +47 -0
  144. package/dist/lib/providers/voyage.js +177 -0
  145. package/dist/lib/providers/xai.d.ts +33 -0
  146. package/dist/lib/providers/xai.js +172 -0
  147. package/dist/lib/telemetry/index.d.ts +1 -1
  148. package/dist/lib/telemetry/index.js +1 -1
  149. package/dist/lib/telemetry/tracers.d.ts +19 -0
  150. package/dist/lib/telemetry/tracers.js +19 -0
  151. package/dist/lib/telemetry/withSpan.d.ts +35 -0
  152. package/dist/lib/telemetry/withSpan.js +103 -0
  153. package/dist/lib/types/avatar.d.ts +143 -0
  154. package/dist/lib/types/avatar.js +20 -0
  155. package/dist/lib/types/cli.d.ts +6 -0
  156. package/dist/lib/types/generate.d.ts +62 -5
  157. package/dist/lib/types/index.d.ts +5 -0
  158. package/dist/lib/types/index.js +7 -0
  159. package/dist/lib/types/middleware.d.ts +27 -0
  160. package/dist/lib/types/multimodal.d.ts +35 -2
  161. package/dist/lib/types/music.d.ts +165 -0
  162. package/dist/lib/types/music.js +21 -0
  163. package/dist/lib/types/providers.d.ts +144 -1
  164. package/dist/lib/types/replicate.d.ts +67 -0
  165. package/dist/lib/types/replicate.js +10 -0
  166. package/dist/lib/types/safeFetch.d.ts +15 -0
  167. package/dist/lib/types/safeFetch.js +7 -0
  168. package/dist/lib/types/stream.d.ts +2 -1
  169. package/dist/lib/types/tools.d.ts +13 -0
  170. package/dist/lib/types/video.d.ts +89 -0
  171. package/dist/lib/types/video.js +15 -0
  172. package/dist/lib/utils/avatarProcessor.d.ts +68 -0
  173. package/dist/lib/utils/avatarProcessor.js +172 -0
  174. package/dist/lib/utils/cloneOptions.d.ts +36 -0
  175. package/dist/lib/utils/cloneOptions.js +62 -0
  176. package/dist/lib/utils/lifecycleCallbacks.d.ts +51 -8
  177. package/dist/lib/utils/lifecycleCallbacks.js +82 -26
  178. package/dist/lib/utils/lifecycleTimeout.d.ts +25 -0
  179. package/dist/lib/utils/lifecycleTimeout.js +39 -0
  180. package/dist/lib/utils/logSanitize.d.ts +49 -0
  181. package/dist/lib/utils/logSanitize.js +170 -0
  182. package/dist/lib/utils/loggingFetch.d.ts +29 -0
  183. package/dist/lib/utils/loggingFetch.js +60 -0
  184. package/dist/lib/utils/messageBuilder.js +43 -25
  185. package/dist/lib/utils/modelChoices.js +236 -3
  186. package/dist/lib/utils/musicProcessor.d.ts +67 -0
  187. package/dist/lib/utils/musicProcessor.js +189 -0
  188. package/dist/lib/utils/optionsConversion.js +3 -2
  189. package/dist/lib/utils/parameterValidation.js +14 -4
  190. package/dist/lib/utils/pricing.js +193 -0
  191. package/dist/lib/utils/providerConfig.d.ts +55 -0
  192. package/dist/lib/utils/providerConfig.js +224 -0
  193. package/dist/lib/utils/safeFetch.d.ts +26 -0
  194. package/dist/lib/utils/safeFetch.js +83 -0
  195. package/dist/lib/utils/sizeGuard.d.ts +34 -0
  196. package/dist/lib/utils/sizeGuard.js +45 -0
  197. package/dist/lib/utils/ssrfGuard.d.ts +52 -0
  198. package/dist/lib/utils/ssrfGuard.js +411 -0
  199. package/dist/lib/utils/videoProcessor.d.ts +60 -0
  200. package/dist/lib/utils/videoProcessor.js +201 -0
  201. package/dist/lib/voice/providers/FishAudioTTS.d.ts +27 -0
  202. package/dist/lib/voice/providers/FishAudioTTS.js +183 -0
  203. package/dist/lib/workflow/core/ensembleExecutor.js +26 -9
  204. package/dist/middleware/builtin/lifecycle.js +39 -9
  205. package/dist/music/index.d.ts +13 -0
  206. package/dist/music/index.js +13 -0
  207. package/dist/music/providers/BeatovenMusic.d.ts +31 -0
  208. package/dist/music/providers/BeatovenMusic.js +333 -0
  209. package/dist/music/providers/ElevenLabsMusic.d.ts +30 -0
  210. package/dist/music/providers/ElevenLabsMusic.js +168 -0
  211. package/dist/music/providers/LyriaMusic.d.ts +29 -0
  212. package/dist/music/providers/LyriaMusic.js +172 -0
  213. package/dist/music/providers/ReplicateMusic.d.ts +31 -0
  214. package/dist/music/providers/ReplicateMusic.js +261 -0
  215. package/dist/neurolink.d.ts +30 -0
  216. package/dist/neurolink.js +323 -77
  217. package/dist/providers/amazonBedrock.d.ts +10 -0
  218. package/dist/providers/amazonBedrock.js +94 -39
  219. package/dist/providers/anthropic.js +55 -7
  220. package/dist/providers/anthropicBaseProvider.js +1 -1
  221. package/dist/providers/azureOpenai.js +66 -17
  222. package/dist/providers/cloudflare.d.ts +35 -0
  223. package/dist/providers/cloudflare.js +173 -0
  224. package/dist/providers/cohere.d.ts +52 -0
  225. package/dist/providers/cohere.js +252 -0
  226. package/dist/providers/deepseek.js +72 -17
  227. package/dist/providers/fireworks.d.ts +33 -0
  228. package/dist/providers/fireworks.js +163 -0
  229. package/dist/providers/googleAiStudio.js +45 -6
  230. package/dist/providers/googleNativeGemini3.d.ts +24 -1
  231. package/dist/providers/googleNativeGemini3.js +173 -21
  232. package/dist/providers/googleVertex.js +173 -17
  233. package/dist/providers/groq.d.ts +33 -0
  234. package/dist/providers/groq.js +180 -0
  235. package/dist/providers/huggingFace.js +9 -8
  236. package/dist/providers/ideogram.d.ts +34 -0
  237. package/dist/providers/ideogram.js +183 -0
  238. package/dist/providers/index.d.ts +13 -0
  239. package/dist/providers/index.js +13 -0
  240. package/dist/providers/jina.d.ts +59 -0
  241. package/dist/providers/jina.js +217 -0
  242. package/dist/providers/llamaCpp.js +14 -46
  243. package/dist/providers/lmStudio.js +14 -47
  244. package/dist/providers/mistral.js +7 -7
  245. package/dist/providers/nvidiaNim.js +160 -19
  246. package/dist/providers/ollama.js +7 -7
  247. package/dist/providers/openAI.d.ts +22 -1
  248. package/dist/providers/openAI.js +181 -0
  249. package/dist/providers/openRouter.js +35 -23
  250. package/dist/providers/openaiCompatible.js +9 -8
  251. package/dist/providers/perplexity.d.ts +33 -0
  252. package/dist/providers/perplexity.js +178 -0
  253. package/dist/providers/recraft.d.ts +34 -0
  254. package/dist/providers/recraft.js +196 -0
  255. package/dist/providers/replicate.d.ts +75 -0
  256. package/dist/providers/replicate.js +402 -0
  257. package/dist/providers/stability.d.ts +37 -0
  258. package/dist/providers/stability.js +190 -0
  259. package/dist/providers/togetherAi.d.ts +33 -0
  260. package/dist/providers/togetherAi.js +175 -0
  261. package/dist/providers/voyage.d.ts +47 -0
  262. package/dist/providers/voyage.js +176 -0
  263. package/dist/providers/xai.d.ts +33 -0
  264. package/dist/providers/xai.js +171 -0
  265. package/dist/telemetry/index.d.ts +1 -1
  266. package/dist/telemetry/index.js +1 -1
  267. package/dist/telemetry/tracers.d.ts +19 -0
  268. package/dist/telemetry/tracers.js +19 -0
  269. package/dist/telemetry/withSpan.d.ts +35 -0
  270. package/dist/telemetry/withSpan.js +103 -0
  271. package/dist/types/avatar.d.ts +143 -0
  272. package/dist/types/avatar.js +19 -0
  273. package/dist/types/cli.d.ts +6 -0
  274. package/dist/types/generate.d.ts +62 -5
  275. package/dist/types/index.d.ts +5 -0
  276. package/dist/types/index.js +7 -0
  277. package/dist/types/middleware.d.ts +27 -0
  278. package/dist/types/multimodal.d.ts +35 -2
  279. package/dist/types/music.d.ts +165 -0
  280. package/dist/types/music.js +20 -0
  281. package/dist/types/providers.d.ts +144 -1
  282. package/dist/types/replicate.d.ts +67 -0
  283. package/dist/types/replicate.js +9 -0
  284. package/dist/types/safeFetch.d.ts +15 -0
  285. package/dist/types/safeFetch.js +6 -0
  286. package/dist/types/stream.d.ts +2 -1
  287. package/dist/types/tools.d.ts +13 -0
  288. package/dist/types/video.d.ts +89 -0
  289. package/dist/types/video.js +14 -0
  290. package/dist/utils/avatarProcessor.d.ts +68 -0
  291. package/dist/utils/avatarProcessor.js +171 -0
  292. package/dist/utils/cloneOptions.d.ts +36 -0
  293. package/dist/utils/cloneOptions.js +61 -0
  294. package/dist/utils/lifecycleCallbacks.d.ts +51 -8
  295. package/dist/utils/lifecycleCallbacks.js +82 -26
  296. package/dist/utils/lifecycleTimeout.d.ts +25 -0
  297. package/dist/utils/lifecycleTimeout.js +38 -0
  298. package/dist/utils/logSanitize.d.ts +49 -0
  299. package/dist/utils/logSanitize.js +169 -0
  300. package/dist/utils/loggingFetch.d.ts +29 -0
  301. package/dist/utils/loggingFetch.js +59 -0
  302. package/dist/utils/messageBuilder.js +43 -25
  303. package/dist/utils/modelChoices.js +236 -3
  304. package/dist/utils/musicProcessor.d.ts +67 -0
  305. package/dist/utils/musicProcessor.js +188 -0
  306. package/dist/utils/optionsConversion.js +3 -2
  307. package/dist/utils/parameterValidation.js +14 -4
  308. package/dist/utils/pricing.js +193 -0
  309. package/dist/utils/providerConfig.d.ts +55 -0
  310. package/dist/utils/providerConfig.js +224 -0
  311. package/dist/utils/safeFetch.d.ts +26 -0
  312. package/dist/utils/safeFetch.js +82 -0
  313. package/dist/utils/sizeGuard.d.ts +34 -0
  314. package/dist/utils/sizeGuard.js +44 -0
  315. package/dist/utils/ssrfGuard.d.ts +52 -0
  316. package/dist/utils/ssrfGuard.js +410 -0
  317. package/dist/utils/videoProcessor.d.ts +60 -0
  318. package/dist/utils/videoProcessor.js +200 -0
  319. package/dist/voice/providers/FishAudioTTS.d.ts +27 -0
  320. package/dist/voice/providers/FishAudioTTS.js +182 -0
  321. package/dist/workflow/core/ensembleExecutor.js +26 -9
  322. package/package.json +32 -5
@@ -1,15 +1,98 @@
1
- import { createOpenAI } from "@ai-sdk/openai";
1
+ import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
2
2
  import { stepCountIs, streamText } from "ai";
3
3
  import { NvidiaNimModels } from "../constants/enums.js";
4
4
  import { BaseProvider } from "../core/baseProvider.js";
5
5
  import { DEFAULT_MAX_STEPS } from "../core/constants.js";
6
6
  import { streamAnalyticsCollector } from "../core/streamAnalytics.js";
7
+ import { isNeuroLink } from "../neurolink.js";
7
8
  import { createProxyFetch, maskProxyUrl } from "../proxy/proxyFetch.js";
8
- import { tracers, ATTR, withClientSpan } from "../telemetry/index.js";
9
+ import { tracers, ATTR, withClientStreamSpan } from "../telemetry/index.js";
10
+ import { AuthenticationError, InvalidModelError, NetworkError, ProviderError, RateLimitError, } from "../types/index.js";
9
11
  import { logger } from "../utils/logger.js";
10
12
  import { createNvidiaNimConfig, getProviderModel, validateApiKey, } from "../utils/providerConfig.js";
11
13
  import { composeAbortSignals, createTimeoutController, TimeoutError, } from "../utils/timeout.js";
12
14
  import { emitToolEndFromStepFinish } from "../utils/toolEndEmitter.js";
15
+ import { resolveToolChoice } from "../utils/toolChoice.js";
16
+ import { toAnalyticsStreamResult } from "./providerTypeUtils.js";
17
+ /**
18
+ * Decide whether a NIM 400 response body is a rejection of the named
19
+ * field (as opposed to an unrelated 400 that happens to mention the
20
+ * field name — e.g. when the user's prompt is echoed back inside the
21
+ * error envelope).
22
+ *
23
+ * A rejection requires both:
24
+ * - the field name appears in the body, and
25
+ * - a rejection keyword (`unsupported`, `not supported`, `unknown`,
26
+ * `invalid`, `unrecognized`, `does not support`) appears within
27
+ * 80 characters of any occurrence.
28
+ *
29
+ * The 80-character window is loose enough to absorb NIM's "Unsupported
30
+ * argument: `chat_template`" framing and tight enough that a 1KB error
31
+ * body mentioning the field once in a code sample plus an unrelated
32
+ * "invalid" elsewhere won't trigger a strip.
33
+ */
34
+ const NIM_REJECTION_KEYWORDS = [
35
+ "unsupported",
36
+ "not supported",
37
+ "does not support",
38
+ "unrecognized",
39
+ "unknown field",
40
+ "unknown parameter",
41
+ "unknown argument",
42
+ "invalid field",
43
+ "invalid parameter",
44
+ "invalid argument",
45
+ ];
46
+ const isNimFieldRejection = (body, field) => {
47
+ if (!body) {
48
+ return false;
49
+ }
50
+ const lower = body.toLowerCase();
51
+ const fieldLower = field.toLowerCase();
52
+ let idx = lower.indexOf(fieldLower);
53
+ while (idx !== -1) {
54
+ const windowStart = Math.max(0, idx - 80);
55
+ const windowEnd = Math.min(lower.length, idx + fieldLower.length + 80);
56
+ const slice = lower.slice(windowStart, windowEnd);
57
+ if (NIM_REJECTION_KEYWORDS.some((kw) => slice.includes(kw))) {
58
+ return true;
59
+ }
60
+ idx = lower.indexOf(fieldLower, idx + fieldLower.length);
61
+ }
62
+ return false;
63
+ };
64
+ /**
65
+ * Strip an offending field from a JSON request body and return the rebuilt
66
+ * stringified body. Returns `null` if the body isn't JSON-parseable or the
67
+ * field isn't present (signal: nothing to retry).
68
+ */
69
+ const stripFieldFromJsonBody = (body, field) => {
70
+ try {
71
+ const parsed = JSON.parse(body);
72
+ let mutated = false;
73
+ if (field === "chat_template" && "chat_template" in parsed) {
74
+ delete parsed.chat_template;
75
+ mutated = true;
76
+ }
77
+ if (field === "reasoning_budget") {
78
+ const kw = parsed.chat_template_kwargs;
79
+ if (kw && "reasoning_budget" in kw) {
80
+ delete kw.reasoning_budget;
81
+ mutated = true;
82
+ if (Object.keys(kw).length === 0) {
83
+ delete parsed.chat_template_kwargs;
84
+ }
85
+ }
86
+ }
87
+ if (!mutated) {
88
+ return null;
89
+ }
90
+ return JSON.stringify(parsed);
91
+ }
92
+ catch {
93
+ return null;
94
+ }
95
+ };
13
96
  const makeLoggingFetch = (provider) => {
14
97
  const base = createProxyFetch();
15
98
  return (async (input, init) => {
@@ -19,7 +102,41 @@ const makeLoggingFetch = (provider) => {
19
102
  ? input.toString()
20
103
  : input.url;
21
104
  const reqSize = init?.body && typeof init.body === "string" ? init.body.length : 0;
22
- const response = await base(input, init);
105
+ let response = await base(input, init);
106
+ // Generic NIM 400 retry-strip: works for BOTH generate and stream paths.
107
+ // NIM sometimes returns HTTP 400 when a model rejects `reasoning_budget`
108
+ // or `chat_template`. The stream path already retries by reconstructing
109
+ // its provider options; this fetch-level retry is the symmetric fix for
110
+ // generate (and any other transport that lands here).
111
+ //
112
+ // We require BOTH (a) the offending field name AND (b) a rejection
113
+ // keyword (unsupported / not supported / unknown / invalid /
114
+ // unrecognized / does not support) within 80 chars of it. Without the
115
+ // rejection-keyword guard, an unrelated 400 whose error body happened
116
+ // to mention `chat_template` (e.g. the user prompt got echoed back)
117
+ // would cause us to silently strip a field the user actually wanted
118
+ // sent, and either succeed for the wrong reason or fail with a
119
+ // misleading error.
120
+ if (response.status === 400 &&
121
+ typeof init?.body === "string" &&
122
+ init.body.length > 0) {
123
+ const cloned = response.clone();
124
+ const body = await cloned.text().catch(() => "");
125
+ let retryBody = null;
126
+ let stripped = null;
127
+ if (isNimFieldRejection(body, "reasoning_budget")) {
128
+ retryBody = stripFieldFromJsonBody(init.body, "reasoning_budget");
129
+ stripped = "reasoning_budget";
130
+ }
131
+ else if (isNimFieldRejection(body, "chat_template")) {
132
+ retryBody = stripFieldFromJsonBody(init.body, "chat_template");
133
+ stripped = "chat_template";
134
+ }
135
+ if (retryBody !== null && stripped !== null) {
136
+ logger.warn(`[${provider}] NIM rejected ${stripped}; retrying with field stripped`);
137
+ response = await base(input, { ...init, body: retryBody });
138
+ }
139
+ }
23
140
  if (!response.ok) {
24
141
  // If maskProxyUrl can't safely sanitize the URL (returns null), don't
25
142
  // log the raw URL — that defeats the redaction. Use a placeholder so
@@ -41,8 +158,6 @@ const makeLoggingFetch = (provider) => {
41
158
  return response;
42
159
  });
43
160
  };
44
- import { resolveToolChoice } from "../utils/toolChoice.js";
45
- import { toAnalyticsStreamResult } from "./providerTypeUtils.js";
46
161
  const NVIDIA_NIM_DEFAULT_BASE_URL = "https://integrate.api.nvidia.com/v1";
47
162
  const envInt = (k) => {
48
163
  const v = process.env[k];
@@ -124,9 +239,7 @@ export class NvidiaNimProvider extends BaseProvider {
124
239
  apiKey;
125
240
  baseURL;
126
241
  constructor(modelName, sdk, _region, credentials) {
127
- const validatedNeurolink = sdk && typeof sdk === "object" && "getInMemoryServers" in sdk
128
- ? sdk
129
- : undefined;
242
+ const validatedNeurolink = isNeuroLink(sdk) ? sdk : undefined;
130
243
  super(modelName, "nvidia-nim", validatedNeurolink);
131
244
  // Trim the override before applying precedence. A blank/whitespace
132
245
  // `credentials.apiKey` should NOT bypass `getNimApiKey()` — that would
@@ -141,13 +254,31 @@ export class NvidiaNimProvider extends BaseProvider {
141
254
  credentials?.baseURL ??
142
255
  process.env.NVIDIA_NIM_BASE_URL ??
143
256
  NVIDIA_NIM_DEFAULT_BASE_URL;
144
- const nim = createOpenAI({
257
+ // We deliberately use `@ai-sdk/openai-compatible` rather than
258
+ // `@ai-sdk/openai`. Two upstream behaviors of `@ai-sdk/openai` break us:
259
+ // 1. It always sends `response_format: { type: "json_schema" }` when a
260
+ // schema is provided. Most NIM-served chat models don't enforce
261
+ // json_schema strictly — the schema goes through but `result.object`
262
+ // stays empty because the SDK never gets the typed response back.
263
+ // 2. It does not parse the `reasoning_content` field that NIM-hosted
264
+ // reasoning models (deepseek-r1, qwq, llama-nemotron-ultra) emit,
265
+ // so chain-of-thought is silently dropped.
266
+ // `@ai-sdk/openai-compatible` honors `supportsStructuredOutputs: false`
267
+ // (falls back to `{ type: "json_object" }` and injects the schema into
268
+ // the prompt — works across the entire NIM model fleet) and parses both
269
+ // `choice.message.reasoning_content` and `delta.reasoning_content` into
270
+ // the SDK-standard `reasoning` part. NIM-specific extras (`min_tokens`,
271
+ // `chat_template_kwargs.reasoning_budget`, `chat_template`) are still
272
+ // injected via `providerOptions.openai.body` in `executeStreamInner`.
273
+ const nim = createOpenAICompatible({
274
+ name: "nvidia-nim",
145
275
  apiKey: this.apiKey,
146
276
  baseURL: this.baseURL,
147
277
  fetch: makeLoggingFetch("nvidia-nim"),
278
+ supportsStructuredOutputs: false,
279
+ includeUsage: true,
148
280
  });
149
- // .chat() — NIM exposes /v1/chat/completions, not /v1/responses
150
- this.model = nim.chat(this.modelName);
281
+ this.model = nim.chatModel(this.modelName);
151
282
  logger.debug("NVIDIA NIM Provider initialized", {
152
283
  modelName: this.modelName,
153
284
  providerName: this.providerName,
@@ -155,7 +286,7 @@ export class NvidiaNimProvider extends BaseProvider {
155
286
  });
156
287
  }
157
288
  async executeStream(options, _analysisSchema) {
158
- return withClientSpan({
289
+ return withClientStreamSpan({
159
290
  name: "neurolink.provider.stream",
160
291
  tracer: tracers.provider,
161
292
  attributes: {
@@ -164,7 +295,7 @@ export class NvidiaNimProvider extends BaseProvider {
164
295
  [ATTR.GEN_AI_OPERATION]: "stream",
165
296
  [ATTR.NL_STREAM_MODE]: true,
166
297
  },
167
- }, async () => this.executeStreamInner(options));
298
+ }, async () => this.executeStreamInner(options), (r) => r.stream, (r, wrapped) => ({ ...r, stream: wrapped }));
168
299
  }
169
300
  async executeStreamInner(options) {
170
301
  this.validateStreamOptions(options);
@@ -316,27 +447,37 @@ export class NvidiaNimProvider extends BaseProvider {
316
447
  }
317
448
  formatProviderError(error) {
318
449
  if (error instanceof TimeoutError) {
319
- return new Error(`NVIDIA NIM request timed out: ${error.message}`);
450
+ return new NetworkError(`Request timed out: ${error.message}`, "nvidia-nim");
320
451
  }
321
452
  const errorRecord = error;
322
453
  const message = typeof errorRecord?.message === "string"
323
454
  ? errorRecord.message
324
455
  : "Unknown error";
456
+ // NIM canonically returns HTTP 401/Unauthorized for invalid API keys,
457
+ // but its OpenAI-compatible gateway sometimes surfaces a bare 400 +
458
+ // "Bad Request" with no body details for both malformed-credentials
459
+ // and bad-parameter cases. Because the two are indistinguishable from
460
+ // the message alone, we DON'T promote bare 400/Bad Request to "invalid
461
+ // key" here — that would mis-classify legitimate parameter errors
462
+ // (e.g. unsupported `reasoning_budget`, unsupported `chat_template`)
463
+ // as auth failures. Tests that probe the auth path (K1) detect
464
+ // "bad request" / "400" themselves; tests that probe parameter retry
465
+ // (K5) need the original "Bad Request" message to surface.
325
466
  if (message.includes("Invalid API key") ||
326
467
  message.includes("401") ||
327
468
  message.includes("Unauthorized")) {
328
- return new Error("Invalid NVIDIA NIM API key. Get one at https://build.nvidia.com/settings/api-keys");
469
+ return new AuthenticationError("Invalid NVIDIA NIM API key. Get one at https://build.nvidia.com/settings/api-keys", "nvidia-nim");
329
470
  }
330
471
  if (message.includes("rate limit") || message.includes("429")) {
331
- return new Error("NVIDIA NIM rate limit exceeded");
472
+ return new RateLimitError("NVIDIA NIM rate limit exceeded", "nvidia-nim");
332
473
  }
333
474
  if (message.includes("404") || message.includes("model_not_found")) {
334
- return new Error(`NVIDIA NIM model '${this.modelName}' not available. Browse the catalog at https://build.nvidia.com/models`);
475
+ return new InvalidModelError(`NVIDIA NIM model '${this.modelName}' not available. Browse the catalog at https://build.nvidia.com/models`, "nvidia-nim");
335
476
  }
336
477
  if (message.includes("quota") || message.includes("403")) {
337
- return new Error("NVIDIA NIM quota exceeded for your account");
478
+ return new ProviderError("NVIDIA NIM quota exceeded for your account", "nvidia-nim");
338
479
  }
339
- return new Error(`NVIDIA NIM error: ${message}`);
480
+ return new ProviderError(`NVIDIA NIM error: ${message}`, "nvidia-nim");
340
481
  }
341
482
  async validateConfiguration() {
342
483
  return typeof this.apiKey === "string" && this.apiKey.trim().length > 0;
@@ -8,7 +8,7 @@ import { buildMultimodalMessagesArray } from "../utils/messageBuilder.js";
8
8
  import { buildMultimodalOptions } from "../utils/multimodalOptionsBuilder.js";
9
9
  import { estimateTokens } from "../utils/tokenEstimation.js";
10
10
  import { InvalidModelError, NetworkError, ProviderError, } from "../types/index.js";
11
- import { tracers, ATTR, withClientSpan } from "../telemetry/index.js";
11
+ import { tracers, ATTR, withClientStreamSpan } from "../telemetry/index.js";
12
12
  import { emitToolEndFromStepFinish } from "../utils/toolEndEmitter.js";
13
13
  import { TimeoutError } from "../utils/timeout.js";
14
14
  // Model version constants (configurable via environment)
@@ -690,7 +690,7 @@ export class OllamaProvider extends BaseProvider {
690
690
  * Uses conversation loop to handle multi-step tool execution
691
691
  */
692
692
  async executeStreamWithTools(options, _analysisSchema) {
693
- return withClientSpan({
693
+ return withClientStreamSpan({
694
694
  name: "neurolink.provider.stream",
695
695
  tracer: tracers.provider,
696
696
  attributes: {
@@ -741,7 +741,7 @@ export class OllamaProvider extends BaseProvider {
741
741
  }
742
742
  conversationHistory.push({
743
743
  role: "user",
744
- content: options.input.text,
744
+ content: options.input.text ?? "",
745
745
  });
746
746
  }
747
747
  // Capture instance references before the stream for use in the finally block.
@@ -908,14 +908,14 @@ export class OllamaProvider extends BaseProvider {
908
908
  streamId: `ollama-${Date.now()}`,
909
909
  },
910
910
  };
911
- });
911
+ }, (r) => r.stream, (r, wrapped) => ({ ...r, stream: wrapped }));
912
912
  }
913
913
  /**
914
914
  * Execute streaming without tools using the generate API
915
915
  * Fallback for non-tool scenarios or when chat API is unavailable
916
916
  */
917
917
  async executeStreamWithoutTools(options, _analysisSchema) {
918
- return withClientSpan({
918
+ return withClientStreamSpan({
919
919
  name: "neurolink.provider.stream",
920
920
  tracer: tracers.provider,
921
921
  attributes: {
@@ -954,7 +954,7 @@ export class OllamaProvider extends BaseProvider {
954
954
  messages.push({ role: "user", content });
955
955
  }
956
956
  else {
957
- messages.push({ role: "user", content: options.input.text });
957
+ messages.push({ role: "user", content: options.input.text ?? "" });
958
958
  }
959
959
  const requestUrl = `${this.baseUrl}/v1/chat/completions`;
960
960
  const requestBody = {
@@ -1064,7 +1064,7 @@ export class OllamaProvider extends BaseProvider {
1064
1064
  model: this.modelName,
1065
1065
  };
1066
1066
  }
1067
- });
1067
+ }, (r) => r.stream, (r, wrapped) => ({ ...r, stream: wrapped }));
1068
1068
  }
1069
1069
  /**
1070
1070
  * Convert AI SDK tools format to Ollama's function calling format
@@ -2,7 +2,7 @@ import { type LanguageModel } from "ai";
2
2
  import { AIProviderName } from "../constants/enums.js";
3
3
  import { BaseProvider } from "../core/baseProvider.js";
4
4
  import type { NeuroLink } from "../neurolink.js";
5
- import type { ValidationSchema, StreamOptions, StreamResult } from "../types/index.js";
5
+ import type { EnhancedGenerateResult, TextGenerationOptions, ValidationSchema, StreamOptions, StreamResult } from "../types/index.js";
6
6
  /**
7
7
  * OpenAI Provider v2 - BaseProvider Implementation
8
8
  * Migrated to use factory pattern with exact Google AI provider pattern
@@ -73,5 +73,26 @@ export declare class OpenAIProvider extends BaseProvider {
73
73
  * @returns Promise resolving to an array of embedding vectors
74
74
  */
75
75
  embedMany(texts: string[], modelName?: string): Promise<number[][]>;
76
+ /**
77
+ * Image generation via the OpenAI Images API (`/v1/images/generations`).
78
+ *
79
+ * Supports `gpt-image-1`, `dall-e-3`, and `dall-e-2`. The three models
80
+ * differ in which body params they accept:
81
+ *
82
+ * - `gpt-image-1` returns base64 by default; does NOT accept `response_format`.
83
+ * - `dall-e-3` / `dall-e-2` accept `response_format: "b64_json"` to get base64.
84
+ * - `dall-e-2` does NOT accept `quality` / `style`.
85
+ *
86
+ * The model is taken from `options.model || this.modelName`.
87
+ *
88
+ * @see https://platform.openai.com/docs/api-reference/images/create
89
+ */
90
+ protected executeImageGeneration(options: TextGenerationOptions): Promise<EnhancedGenerateResult>;
91
+ /**
92
+ * Map a NeuroLink-style aspect ratio (e.g. "16:9") to the OpenAI
93
+ * `size` parameter accepted by the active image model. Falls back to
94
+ * the per-model square default when the ratio is unknown.
95
+ */
96
+ private aspectRatioToOpenAISize;
76
97
  }
77
98
  export default OpenAIProvider;
@@ -15,6 +15,8 @@ import { isZodSchema } from "../utils/schemaConversion.js";
15
15
  import { composeAbortSignals, createTimeoutController, TimeoutError, } from "../utils/timeout.js";
16
16
  import { resolveToolChoice } from "../utils/toolChoice.js";
17
17
  import { emitToolEndFromStepFinish } from "../utils/toolEndEmitter.js";
18
+ import { MAX_IMAGE_BYTES, readBoundedBuffer } from "../utils/sizeGuard.js";
19
+ import { assertSafeUrl } from "../utils/ssrfGuard.js";
18
20
  import { getModelId } from "./providerTypeUtils.js";
19
21
  /**
20
22
  * Retrieve a tool's schema, handling both AI SDK v6 (`inputSchema`) and
@@ -692,6 +694,185 @@ export class OpenAIProvider extends BaseProvider {
692
694
  throw this.handleProviderError(error);
693
695
  }
694
696
  }
697
+ /**
698
+ * Image generation via the OpenAI Images API (`/v1/images/generations`).
699
+ *
700
+ * Supports `gpt-image-1`, `dall-e-3`, and `dall-e-2`. The three models
701
+ * differ in which body params they accept:
702
+ *
703
+ * - `gpt-image-1` returns base64 by default; does NOT accept `response_format`.
704
+ * - `dall-e-3` / `dall-e-2` accept `response_format: "b64_json"` to get base64.
705
+ * - `dall-e-2` does NOT accept `quality` / `style`.
706
+ *
707
+ * The model is taken from `options.model || this.modelName`.
708
+ *
709
+ * @see https://platform.openai.com/docs/api-reference/images/create
710
+ */
711
+ async executeImageGeneration(options) {
712
+ const startTime = Date.now();
713
+ const prompt = options.prompt ?? options.input?.text ?? "";
714
+ if (!prompt.trim()) {
715
+ throw new Error("OpenAI image generation requires a prompt (input.text or prompt)");
716
+ }
717
+ const model = options.model ?? this.modelName;
718
+ const apiKey = this.credentials?.apiKey ?? getOpenAIApiKey();
719
+ const baseURL = (this.credentials?.baseURL ??
720
+ process.env.OPENAI_BASE_URL ??
721
+ "https://api.openai.com/v1").replace(/\/$/, "");
722
+ // Image-gen extras live on `options` but are not part of the strict
723
+ // TextGenerationOptions shape — cast to a permissive type to read them.
724
+ const extras = options;
725
+ // Map aspect ratio to OpenAI's `size` parameter. gpt-image-1 supports
726
+ // 1024x1024 / 1024x1536 / 1536x1024 / auto; dall-e-3 supports
727
+ // 1024x1024 / 1792x1024 / 1024x1792; dall-e-2 supports 256x256 /
728
+ // 512x512 / 1024x1024. We pick safe defaults and let users override
729
+ // via `extras.size` directly.
730
+ const size = extras.size ?? this.aspectRatioToOpenAISize(extras.aspectRatio, model);
731
+ // Clamp n per-model: gpt-image-1 and dall-e-3 only support n=1;
732
+ // dall-e-2 supports n=1..10; default to 1 for any future models.
733
+ const rawN = extras.numberOfImages ?? 1;
734
+ let clampedN;
735
+ if (model === "gpt-image-1" || model.startsWith("dall-e-3")) {
736
+ clampedN = 1;
737
+ }
738
+ else if (model.startsWith("dall-e-2")) {
739
+ clampedN = Math.min(Math.max(rawN, 1), 10);
740
+ }
741
+ else {
742
+ clampedN = 1;
743
+ }
744
+ const n = clampedN;
745
+ const body = {
746
+ model,
747
+ prompt,
748
+ n,
749
+ size,
750
+ };
751
+ if (model === "gpt-image-1") {
752
+ // gpt-image-1 always returns base64; rejects `response_format`.
753
+ if (extras.quality) {
754
+ body.quality = extras.quality;
755
+ }
756
+ }
757
+ else if (model.startsWith("dall-e-3")) {
758
+ body.response_format = "b64_json";
759
+ if (extras.quality) {
760
+ body.quality = extras.quality;
761
+ }
762
+ if (extras.style) {
763
+ body.style = extras.style;
764
+ }
765
+ }
766
+ else {
767
+ // dall-e-2 (and forward-compat default).
768
+ body.response_format = "b64_json";
769
+ }
770
+ const REQUEST_TIMEOUT_MS = 120_000;
771
+ const controller = new AbortController();
772
+ const timeoutId = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
773
+ let response;
774
+ try {
775
+ const proxyFetch = createProxyFetch();
776
+ response = await proxyFetch(`${baseURL}/images/generations`, {
777
+ method: "POST",
778
+ headers: {
779
+ Authorization: `Bearer ${apiKey}`,
780
+ "Content-Type": "application/json",
781
+ },
782
+ body: JSON.stringify(body),
783
+ signal: controller.signal,
784
+ });
785
+ }
786
+ catch (err) {
787
+ if (err instanceof Error && err.name === "AbortError") {
788
+ throw new Error(`OpenAI image generation timed out after ${REQUEST_TIMEOUT_MS / 1000}s`, { cause: err });
789
+ }
790
+ throw err;
791
+ }
792
+ finally {
793
+ clearTimeout(timeoutId);
794
+ }
795
+ if (!response.ok) {
796
+ const text = await response.text();
797
+ throw new Error(`OpenAI image generation failed: ${response.status} — ${text}`);
798
+ }
799
+ const data = (await response.json());
800
+ const first = data.data?.[0];
801
+ if (!first) {
802
+ throw new Error("OpenAI image generation returned no images");
803
+ }
804
+ let base64 = first.b64_json;
805
+ // dall-e-2 with `response_format: "b64_json"` should always include
806
+ // b64_json. If a hosted URL came back instead (e.g. older keys, or
807
+ // url-mode), download it inline so callers always get base64.
808
+ if (!base64 && first.url) {
809
+ // Guard the API-returned URL before fetching (provider-returned URLs
810
+ // carry the same SSRF risk as caller-supplied ones).
811
+ await assertSafeUrl(first.url);
812
+ const proxyFetch = createProxyFetch();
813
+ const dlController = new AbortController();
814
+ const dlTimeoutId = setTimeout(() => dlController.abort(), 60_000);
815
+ let imgResp;
816
+ try {
817
+ imgResp = await proxyFetch(first.url, { signal: dlController.signal });
818
+ }
819
+ catch (err) {
820
+ if (err instanceof Error && err.name === "AbortError") {
821
+ throw new Error("OpenAI image URL download timed out after 60s", {
822
+ cause: err,
823
+ });
824
+ }
825
+ throw err;
826
+ }
827
+ finally {
828
+ clearTimeout(dlTimeoutId);
829
+ }
830
+ if (!imgResp.ok) {
831
+ throw new Error(`OpenAI image generation: failed to fetch hosted URL ${first.url} (${imgResp.status})`);
832
+ }
833
+ const buf = await readBoundedBuffer(imgResp, MAX_IMAGE_BYTES, "OpenAI image fallback");
834
+ base64 = buf.toString("base64");
835
+ }
836
+ if (!base64) {
837
+ throw new Error("OpenAI image generation returned neither b64_json nor a URL");
838
+ }
839
+ const generationTimeMs = Date.now() - startTime;
840
+ logger.info(`[OpenAIProvider] Generated image (${base64.length} base64 chars) in ${generationTimeMs}ms — model ${model}`);
841
+ return {
842
+ content: first.revised_prompt ?? prompt,
843
+ provider: this.providerName,
844
+ model,
845
+ usage: { input: 0, output: 0, total: 0 },
846
+ imageOutput: { base64 },
847
+ };
848
+ }
849
+ /**
850
+ * Map a NeuroLink-style aspect ratio (e.g. "16:9") to the OpenAI
851
+ * `size` parameter accepted by the active image model. Falls back to
852
+ * the per-model square default when the ratio is unknown.
853
+ */
854
+ aspectRatioToOpenAISize(aspectRatio, model) {
855
+ if (model === "gpt-image-1") {
856
+ if (aspectRatio === "16:9" || aspectRatio === "3:2") {
857
+ return "1536x1024";
858
+ }
859
+ if (aspectRatio === "9:16" || aspectRatio === "2:3") {
860
+ return "1024x1536";
861
+ }
862
+ return "1024x1024";
863
+ }
864
+ if (model.startsWith("dall-e-3")) {
865
+ if (aspectRatio === "16:9" || aspectRatio === "3:2") {
866
+ return "1792x1024";
867
+ }
868
+ if (aspectRatio === "9:16" || aspectRatio === "2:3") {
869
+ return "1024x1792";
870
+ }
871
+ return "1024x1024";
872
+ }
873
+ // dall-e-2 — only square sizes supported.
874
+ return "1024x1024";
875
+ }
695
876
  }
696
877
  // Export for factory registration
697
878
  export default OpenAIProvider;