@juspay/neurolink 9.15.0 → 9.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (196) hide show
  1. package/CHANGELOG.md +12 -0
  2. package/README.md +22 -20
  3. package/dist/adapters/video/videoAnalyzer.d.ts +1 -1
  4. package/dist/adapters/video/videoAnalyzer.js +10 -8
  5. package/dist/cli/commands/setup-anthropic.js +1 -14
  6. package/dist/cli/commands/setup-azure.js +1 -12
  7. package/dist/cli/commands/setup-bedrock.js +1 -9
  8. package/dist/cli/commands/setup-google-ai.js +1 -12
  9. package/dist/cli/commands/setup-openai.js +1 -14
  10. package/dist/cli/commands/workflow.d.ts +27 -0
  11. package/dist/cli/commands/workflow.js +216 -0
  12. package/dist/cli/factories/commandFactory.js +79 -20
  13. package/dist/cli/index.js +0 -1
  14. package/dist/cli/parser.js +4 -1
  15. package/dist/cli/utils/maskCredential.d.ts +11 -0
  16. package/dist/cli/utils/maskCredential.js +23 -0
  17. package/dist/constants/contextWindows.js +107 -16
  18. package/dist/constants/enums.d.ts +99 -15
  19. package/dist/constants/enums.js +152 -22
  20. package/dist/context/budgetChecker.js +1 -1
  21. package/dist/context/contextCompactor.js +31 -4
  22. package/dist/context/emergencyTruncation.d.ts +21 -0
  23. package/dist/context/emergencyTruncation.js +88 -0
  24. package/dist/context/errorDetection.d.ts +16 -0
  25. package/dist/context/errorDetection.js +48 -1
  26. package/dist/context/errors.d.ts +19 -0
  27. package/dist/context/errors.js +21 -0
  28. package/dist/context/stages/slidingWindowTruncator.d.ts +6 -0
  29. package/dist/context/stages/slidingWindowTruncator.js +159 -24
  30. package/dist/context/stages/structuredSummarizer.js +2 -2
  31. package/dist/core/baseProvider.js +306 -200
  32. package/dist/core/conversationMemoryManager.js +104 -61
  33. package/dist/core/evaluationProviders.js +16 -33
  34. package/dist/core/factory.js +237 -164
  35. package/dist/core/modules/GenerationHandler.js +175 -116
  36. package/dist/core/modules/MessageBuilder.js +222 -170
  37. package/dist/core/modules/StreamHandler.d.ts +1 -0
  38. package/dist/core/modules/StreamHandler.js +95 -27
  39. package/dist/core/modules/TelemetryHandler.d.ts +10 -1
  40. package/dist/core/modules/TelemetryHandler.js +25 -7
  41. package/dist/core/modules/ToolsManager.js +115 -191
  42. package/dist/core/redisConversationMemoryManager.js +418 -282
  43. package/dist/factories/providerRegistry.d.ts +5 -0
  44. package/dist/factories/providerRegistry.js +20 -2
  45. package/dist/index.d.ts +2 -2
  46. package/dist/index.js +4 -2
  47. package/dist/lib/adapters/video/videoAnalyzer.d.ts +1 -1
  48. package/dist/lib/adapters/video/videoAnalyzer.js +10 -8
  49. package/dist/lib/constants/contextWindows.js +107 -16
  50. package/dist/lib/constants/enums.d.ts +99 -15
  51. package/dist/lib/constants/enums.js +152 -22
  52. package/dist/lib/context/budgetChecker.js +1 -1
  53. package/dist/lib/context/contextCompactor.js +31 -4
  54. package/dist/lib/context/emergencyTruncation.d.ts +21 -0
  55. package/dist/lib/context/emergencyTruncation.js +89 -0
  56. package/dist/lib/context/errorDetection.d.ts +16 -0
  57. package/dist/lib/context/errorDetection.js +48 -1
  58. package/dist/lib/context/errors.d.ts +19 -0
  59. package/dist/lib/context/errors.js +22 -0
  60. package/dist/lib/context/stages/slidingWindowTruncator.d.ts +6 -0
  61. package/dist/lib/context/stages/slidingWindowTruncator.js +159 -24
  62. package/dist/lib/context/stages/structuredSummarizer.js +2 -2
  63. package/dist/lib/core/baseProvider.js +306 -200
  64. package/dist/lib/core/conversationMemoryManager.js +104 -61
  65. package/dist/lib/core/evaluationProviders.js +16 -33
  66. package/dist/lib/core/factory.js +237 -164
  67. package/dist/lib/core/modules/GenerationHandler.js +175 -116
  68. package/dist/lib/core/modules/MessageBuilder.js +222 -170
  69. package/dist/lib/core/modules/StreamHandler.d.ts +1 -0
  70. package/dist/lib/core/modules/StreamHandler.js +95 -27
  71. package/dist/lib/core/modules/TelemetryHandler.d.ts +10 -1
  72. package/dist/lib/core/modules/TelemetryHandler.js +25 -7
  73. package/dist/lib/core/modules/ToolsManager.js +115 -191
  74. package/dist/lib/core/redisConversationMemoryManager.js +418 -282
  75. package/dist/lib/factories/providerRegistry.d.ts +5 -0
  76. package/dist/lib/factories/providerRegistry.js +20 -2
  77. package/dist/lib/index.d.ts +2 -2
  78. package/dist/lib/index.js +4 -2
  79. package/dist/lib/mcp/externalServerManager.js +66 -0
  80. package/dist/lib/mcp/mcpCircuitBreaker.js +24 -0
  81. package/dist/lib/mcp/mcpClientFactory.js +16 -0
  82. package/dist/lib/mcp/toolDiscoveryService.js +32 -6
  83. package/dist/lib/mcp/toolRegistry.js +193 -123
  84. package/dist/lib/neurolink.d.ts +6 -0
  85. package/dist/lib/neurolink.js +1162 -646
  86. package/dist/lib/providers/amazonBedrock.d.ts +1 -1
  87. package/dist/lib/providers/amazonBedrock.js +521 -319
  88. package/dist/lib/providers/anthropic.js +73 -17
  89. package/dist/lib/providers/anthropicBaseProvider.js +77 -17
  90. package/dist/lib/providers/googleAiStudio.d.ts +1 -1
  91. package/dist/lib/providers/googleAiStudio.js +292 -227
  92. package/dist/lib/providers/googleVertex.d.ts +36 -1
  93. package/dist/lib/providers/googleVertex.js +553 -260
  94. package/dist/lib/providers/ollama.js +329 -278
  95. package/dist/lib/providers/openAI.js +77 -19
  96. package/dist/lib/providers/sagemaker/parsers.js +3 -3
  97. package/dist/lib/providers/sagemaker/streaming.js +3 -3
  98. package/dist/lib/proxy/proxyFetch.js +81 -48
  99. package/dist/lib/rag/ChunkerFactory.js +1 -1
  100. package/dist/lib/rag/chunkers/MarkdownChunker.d.ts +22 -0
  101. package/dist/lib/rag/chunkers/MarkdownChunker.js +213 -9
  102. package/dist/lib/rag/chunking/markdownChunker.d.ts +16 -0
  103. package/dist/lib/rag/chunking/markdownChunker.js +174 -2
  104. package/dist/lib/rag/pipeline/contextAssembly.js +2 -1
  105. package/dist/lib/rag/ragIntegration.d.ts +18 -1
  106. package/dist/lib/rag/ragIntegration.js +94 -14
  107. package/dist/lib/rag/retrieval/vectorQueryTool.js +21 -4
  108. package/dist/lib/server/abstract/baseServerAdapter.js +4 -1
  109. package/dist/lib/server/adapters/fastifyAdapter.js +35 -30
  110. package/dist/lib/services/server/ai/observability/instrumentation.d.ts +32 -0
  111. package/dist/lib/services/server/ai/observability/instrumentation.js +39 -0
  112. package/dist/lib/telemetry/attributes.d.ts +52 -0
  113. package/dist/lib/telemetry/attributes.js +61 -0
  114. package/dist/lib/telemetry/index.d.ts +3 -0
  115. package/dist/lib/telemetry/index.js +3 -0
  116. package/dist/lib/telemetry/telemetryService.d.ts +6 -0
  117. package/dist/lib/telemetry/telemetryService.js +6 -0
  118. package/dist/lib/telemetry/tracers.d.ts +15 -0
  119. package/dist/lib/telemetry/tracers.js +17 -0
  120. package/dist/lib/telemetry/withSpan.d.ts +9 -0
  121. package/dist/lib/telemetry/withSpan.js +35 -0
  122. package/dist/lib/types/contextTypes.d.ts +10 -0
  123. package/dist/lib/types/streamTypes.d.ts +14 -0
  124. package/dist/lib/utils/conversationMemory.js +123 -84
  125. package/dist/lib/utils/logger.d.ts +5 -0
  126. package/dist/lib/utils/logger.js +50 -2
  127. package/dist/lib/utils/messageBuilder.js +22 -42
  128. package/dist/lib/utils/modelDetection.js +3 -3
  129. package/dist/lib/utils/providerRetry.d.ts +41 -0
  130. package/dist/lib/utils/providerRetry.js +114 -0
  131. package/dist/lib/utils/retryability.d.ts +14 -0
  132. package/dist/lib/utils/retryability.js +23 -0
  133. package/dist/lib/utils/sanitizers/svg.js +4 -5
  134. package/dist/lib/utils/tokenEstimation.d.ts +11 -1
  135. package/dist/lib/utils/tokenEstimation.js +19 -4
  136. package/dist/lib/utils/videoAnalysisProcessor.js +7 -3
  137. package/dist/mcp/externalServerManager.js +66 -0
  138. package/dist/mcp/mcpCircuitBreaker.js +24 -0
  139. package/dist/mcp/mcpClientFactory.js +16 -0
  140. package/dist/mcp/toolDiscoveryService.js +32 -6
  141. package/dist/mcp/toolRegistry.js +193 -123
  142. package/dist/neurolink.d.ts +6 -0
  143. package/dist/neurolink.js +1162 -646
  144. package/dist/providers/amazonBedrock.d.ts +1 -1
  145. package/dist/providers/amazonBedrock.js +521 -319
  146. package/dist/providers/anthropic.js +73 -17
  147. package/dist/providers/anthropicBaseProvider.js +77 -17
  148. package/dist/providers/googleAiStudio.d.ts +1 -1
  149. package/dist/providers/googleAiStudio.js +292 -227
  150. package/dist/providers/googleVertex.d.ts +36 -1
  151. package/dist/providers/googleVertex.js +553 -260
  152. package/dist/providers/ollama.js +329 -278
  153. package/dist/providers/openAI.js +77 -19
  154. package/dist/providers/sagemaker/parsers.js +3 -3
  155. package/dist/providers/sagemaker/streaming.js +3 -3
  156. package/dist/proxy/proxyFetch.js +81 -48
  157. package/dist/rag/ChunkerFactory.js +1 -1
  158. package/dist/rag/chunkers/MarkdownChunker.d.ts +22 -0
  159. package/dist/rag/chunkers/MarkdownChunker.js +213 -9
  160. package/dist/rag/chunking/markdownChunker.d.ts +16 -0
  161. package/dist/rag/chunking/markdownChunker.js +174 -2
  162. package/dist/rag/pipeline/contextAssembly.js +2 -1
  163. package/dist/rag/ragIntegration.d.ts +18 -1
  164. package/dist/rag/ragIntegration.js +94 -14
  165. package/dist/rag/retrieval/vectorQueryTool.js +21 -4
  166. package/dist/server/abstract/baseServerAdapter.js +4 -1
  167. package/dist/server/adapters/fastifyAdapter.js +35 -30
  168. package/dist/services/server/ai/observability/instrumentation.d.ts +32 -0
  169. package/dist/services/server/ai/observability/instrumentation.js +39 -0
  170. package/dist/telemetry/attributes.d.ts +52 -0
  171. package/dist/telemetry/attributes.js +60 -0
  172. package/dist/telemetry/index.d.ts +3 -0
  173. package/dist/telemetry/index.js +3 -0
  174. package/dist/telemetry/telemetryService.d.ts +6 -0
  175. package/dist/telemetry/telemetryService.js +6 -0
  176. package/dist/telemetry/tracers.d.ts +15 -0
  177. package/dist/telemetry/tracers.js +16 -0
  178. package/dist/telemetry/withSpan.d.ts +9 -0
  179. package/dist/telemetry/withSpan.js +34 -0
  180. package/dist/types/contextTypes.d.ts +10 -0
  181. package/dist/types/streamTypes.d.ts +14 -0
  182. package/dist/utils/conversationMemory.js +123 -84
  183. package/dist/utils/logger.d.ts +5 -0
  184. package/dist/utils/logger.js +50 -2
  185. package/dist/utils/messageBuilder.js +22 -42
  186. package/dist/utils/modelDetection.js +3 -3
  187. package/dist/utils/providerRetry.d.ts +41 -0
  188. package/dist/utils/providerRetry.js +113 -0
  189. package/dist/utils/retryability.d.ts +14 -0
  190. package/dist/utils/retryability.js +22 -0
  191. package/dist/utils/sanitizers/svg.js +4 -5
  192. package/dist/utils/tokenEstimation.d.ts +11 -1
  193. package/dist/utils/tokenEstimation.js +19 -4
  194. package/dist/utils/videoAnalysisProcessor.js +7 -3
  195. package/dist/workflow/config.d.ts +26 -26
  196. package/package.json +1 -1
@@ -1,8 +1,12 @@
1
+ import { generateText } from "ai";
2
+ import { SpanKind, SpanStatusCode } from "@opentelemetry/api";
3
+ import { tracers } from "../telemetry/tracers.js";
1
4
  import { directAgentTools } from "../agent/directTools.js";
2
5
  import { IMAGE_GENERATION_MODELS } from "../core/constants.js";
3
6
  import { MiddlewareFactory } from "../middleware/factory.js";
4
7
  import { isAbortError } from "../utils/errorHandling.js";
5
8
  import { logger } from "../utils/logger.js";
9
+ import { calculateCost } from "../utils/pricing.js";
6
10
  import { composeAbortSignals, createTimeoutController, TimeoutError, } from "../utils/timeout.js";
7
11
  import { shouldDisableBuiltinTools } from "../utils/toolUtils.js";
8
12
  import { getKeyCount, getKeysAsString } from "../utils/transformationUtils.js";
@@ -15,6 +19,7 @@ import { StreamHandler } from "./modules/StreamHandler.js";
15
19
  import { TelemetryHandler } from "./modules/TelemetryHandler.js";
16
20
  import { ToolsManager } from "./modules/ToolsManager.js";
17
21
  import { Utilities } from "./modules/Utilities.js";
22
+ const providerTracer = tracers.provider;
18
23
  /**
19
24
  * Abstract base class for all AI providers
20
25
  * Tools are integrated as first-class citizens - always available by default
@@ -49,8 +54,8 @@ export class BaseProvider {
49
54
  // Initialize composition modules
50
55
  this.messageBuilder = new MessageBuilder(this.providerName, this.modelName);
51
56
  this.streamHandler = new StreamHandler(this.providerName, this.modelName);
52
- this.generationHandler = new GenerationHandler(this.providerName, this.modelName, () => this.supportsTools(), (options, type) => this.telemetryHandler.getTelemetryConfig(options, type), (toolCalls, toolResults, options, timestamp) => this.handleToolExecutionStorage(toolCalls, toolResults, options, timestamp));
53
57
  this.telemetryHandler = new TelemetryHandler(this.providerName, this.modelName, this.neurolink);
58
+ this.generationHandler = new GenerationHandler(this.providerName, this.modelName, () => this.supportsTools(), (options, type) => this.telemetryHandler.getTelemetryConfig(options, type), (toolCalls, toolResults, options, timestamp) => this.handleToolExecutionStorage(toolCalls, toolResults, options, timestamp));
54
59
  this.utilities = new Utilities(this.providerName, this.modelName, this.defaultTimeout, this.middlewareOptions);
55
60
  this.toolsManager = new ToolsManager(this.providerName, this.directTools, this.neurolink, {
56
61
  isZodSchema: (schema) => this.isZodSchema(schema),
@@ -75,86 +80,107 @@ export class BaseProvider {
75
80
  * When tools are involved, falls back to generate() with synthetic streaming
76
81
  */
77
82
  async stream(optionsOrPrompt, analysisSchema) {
78
- let options = this.normalizeStreamOptions(optionsOrPrompt);
79
- logger.info(`Starting stream`, {
80
- provider: this.providerName,
81
- hasTools: !options.disableTools && this.supportsTools(),
82
- disableTools: !!options.disableTools,
83
- supportsTools: this.supportsTools(),
84
- inputLength: options.input?.text?.length || 0,
85
- maxTokens: options.maxTokens,
86
- temperature: options.temperature,
87
- timestamp: Date.now(),
88
- });
89
- // ===== EARLY MULTIMODAL DETECTION =====
90
- const hasFileInput = !!options.input?.files?.length || !!options.input?.videoFiles?.length;
91
- if (hasFileInput) {
92
- // ===== VIDEO ANALYSIS DETECTION =====
93
- // Check if video frames are present and handle with fake streaming
94
- const messages = await this.buildMessagesForStream(options);
95
- if (hasVideoFrames(messages)) {
96
- logger.info(`Video frames detected in stream, using fake streaming for video analysis`, {
97
- provider: this.providerName,
98
- model: this.modelName,
99
- });
100
- return await this.executeFakeStreaming(options, analysisSchema);
101
- }
102
- }
103
- // 🔧 CRITICAL: Image generation models don't support real streaming
104
- // Force fake streaming for image models to ensure image output is yielded
105
- const isImageModel = IMAGE_GENERATION_MODELS.some((m) => this.modelName.includes(m));
106
- if (isImageModel) {
107
- logger.info(`Image model detected, forcing fake streaming`, {
83
+ return providerTracer.startActiveSpan("neurolink.provider.stream", { kind: SpanKind.INTERNAL }, async (span) => {
84
+ let options = this.normalizeStreamOptions(optionsOrPrompt);
85
+ span.setAttribute("gen_ai.system", this.providerName || "unknown");
86
+ span.setAttribute("gen_ai.request.model", this.modelName || options.model || "unknown");
87
+ logger.info(`Starting stream`, {
108
88
  provider: this.providerName,
109
- model: this.modelName,
110
- reason: "Image generation requires fake streaming to yield image output",
111
- });
112
- // Skip real streaming, go directly to fake streaming
113
- return await this.executeFakeStreaming(options, analysisSchema);
114
- }
115
- // Central tool merge: Pre-merge base tools (MCP/built-in) with user-provided
116
- // tools (e.g. RAG tools) into options.tools. This way, every provider's
117
- // executeStream() can simply use options.tools (or getAllTools() + options.tools)
118
- // and get the complete tool set without needing per-provider merge logic.
119
- if (!options.disableTools && this.supportsTools()) {
120
- const mergedTools = await this.getToolsForStream(options);
121
- options = { ...options, tools: mergedTools };
122
- }
123
- else {
124
- options = { ...options, tools: {} };
125
- }
126
- // CRITICAL FIX: Always prefer real streaming over fake streaming
127
- // Try real streaming first, use fake streaming only as fallback
128
- try {
129
- logger.debug(`Attempting real streaming`, {
130
- provider: this.providerName,
131
- timestamp: Date.now(),
132
- });
133
- const realStreamResult = await this.executeStream(options, analysisSchema);
134
- logger.info(`Real streaming succeeded`, {
135
- provider: this.providerName,
136
- timestamp: Date.now(),
137
- });
138
- // If real streaming succeeds, return it (with tools support via Vercel AI SDK)
139
- return realStreamResult;
140
- }
141
- catch (realStreamError) {
142
- logger.warn(`Real streaming failed for ${this.providerName}, falling back to fake streaming:`, {
143
- error: realStreamError instanceof Error
144
- ? realStreamError.message
145
- : String(realStreamError),
89
+ hasTools: !options.disableTools && this.supportsTools(),
90
+ disableTools: !!options.disableTools,
91
+ supportsTools: this.supportsTools(),
92
+ inputLength: options.input?.text?.length || 0,
93
+ maxTokens: options.maxTokens,
94
+ temperature: options.temperature,
146
95
  timestamp: Date.now(),
147
96
  });
148
- // Fallback to fake streaming only if real streaming fails AND tools are enabled
149
- if (!options.disableTools && this.supportsTools()) {
150
- return await this.executeFakeStreaming(options, analysisSchema);
97
+ try {
98
+ // ===== EARLY MULTIMODAL DETECTION =====
99
+ const hasFileInput = !!options.input?.files?.length ||
100
+ !!options.input?.videoFiles?.length;
101
+ if (hasFileInput) {
102
+ // ===== VIDEO ANALYSIS DETECTION =====
103
+ // Check if video frames are present and handle with fake streaming
104
+ const messages = await this.buildMessagesForStream(options);
105
+ if (hasVideoFrames(messages)) {
106
+ logger.info(`Video frames detected in stream, using fake streaming for video analysis`, {
107
+ provider: this.providerName,
108
+ model: this.modelName,
109
+ });
110
+ span.setAttribute("neurolink.stream_mode", "fake");
111
+ return await this.executeFakeStreaming(options, analysisSchema);
112
+ }
113
+ }
114
+ // Image generation models don't support real streaming
115
+ // Force fake streaming for image models to ensure image output is yielded
116
+ const isImageModel = IMAGE_GENERATION_MODELS.some((m) => this.modelName.includes(m));
117
+ if (isImageModel) {
118
+ logger.info(`Image model detected, forcing fake streaming`, {
119
+ provider: this.providerName,
120
+ model: this.modelName,
121
+ reason: "Image generation requires fake streaming to yield image output",
122
+ });
123
+ // Skip real streaming, go directly to fake streaming
124
+ span.setAttribute("neurolink.stream_mode", "fake");
125
+ return await this.executeFakeStreaming(options, analysisSchema);
126
+ }
127
+ // Central tool merge: Pre-merge base tools (MCP/built-in) with user-provided
128
+ // tools (e.g. RAG tools) into options.tools. This way, every provider's
129
+ // executeStream() can simply use options.tools (or getAllTools() + options.tools)
130
+ // and get the complete tool set without needing per-provider merge logic.
131
+ if (!options.disableTools && this.supportsTools()) {
132
+ const mergedTools = await this.getToolsForStream(options);
133
+ options = { ...options, tools: mergedTools };
134
+ }
135
+ else {
136
+ options = { ...options, tools: {} };
137
+ }
138
+ // CRITICAL FIX: Always prefer real streaming over fake streaming
139
+ // Try real streaming first, use fake streaming only as fallback
140
+ try {
141
+ logger.debug(`Attempting real streaming`, {
142
+ provider: this.providerName,
143
+ timestamp: Date.now(),
144
+ });
145
+ const realStreamResult = await this.executeStream(options, analysisSchema);
146
+ logger.info(`Real streaming succeeded`, {
147
+ provider: this.providerName,
148
+ timestamp: Date.now(),
149
+ });
150
+ span.setAttribute("neurolink.stream_mode", "real");
151
+ // If real streaming succeeds, return it (with tools support via Vercel AI SDK)
152
+ return realStreamResult;
153
+ }
154
+ catch (realStreamError) {
155
+ logger.warn(`Real streaming failed for ${this.providerName}, falling back to fake streaming:`, {
156
+ error: realStreamError instanceof Error
157
+ ? realStreamError.message
158
+ : String(realStreamError),
159
+ timestamp: Date.now(),
160
+ });
161
+ // Fallback to fake streaming only if real streaming fails AND tools are enabled
162
+ if (!options.disableTools && this.supportsTools()) {
163
+ span.setAttribute("neurolink.stream_mode", "fake");
164
+ return await this.executeFakeStreaming(options, analysisSchema);
165
+ }
166
+ else {
167
+ // If real streaming failed and no tools are enabled, re-throw the original error
168
+ logger.error(`Real streaming failed for ${this.providerName}:`, realStreamError);
169
+ throw this.handleProviderError(realStreamError);
170
+ }
171
+ }
151
172
  }
152
- else {
153
- // If real streaming failed and no tools are enabled, re-throw the original error
154
- logger.error(`Real streaming failed for ${this.providerName}:`, realStreamError);
155
- throw this.handleProviderError(realStreamError);
173
+ catch (error) {
174
+ span.setStatus({
175
+ code: SpanStatusCode.ERROR,
176
+ message: error instanceof Error ? error.message : String(error),
177
+ });
178
+ throw error;
156
179
  }
157
- }
180
+ finally {
181
+ span.end();
182
+ }
183
+ });
158
184
  }
159
185
  /**
160
186
  * Execute fake streaming - extracted method for reusability
@@ -442,141 +468,223 @@ export class BaseProvider {
442
468
  * for consistency and better performance
443
469
  */
444
470
  async generate(optionsOrPrompt, _analysisSchema) {
445
- const options = this.normalizeTextOptions(optionsOrPrompt);
446
- this.validateOptions(options);
447
- const startTime = Date.now();
448
- try {
449
- // ===== VIDEO GENERATION MODE =====
450
- // Generate video from image + prompt using Veo 3.1
451
- if (options.output?.mode === "video") {
452
- return await this.handleVideoGeneration(options, startTime);
453
- }
454
- // ===== IMAGE GENERATION MODE =====
455
- // Route to executeImageGeneration for image generation models
456
- const isImageModel = IMAGE_GENERATION_MODELS.some((m) => this.modelName.includes(m));
457
- if (isImageModel) {
458
- logger.info(`Image generation model detected, routing to executeImageGeneration`, {
459
- provider: this.providerName,
460
- model: this.modelName,
461
- });
462
- const imageResult = await this.executeImageGeneration(options);
463
- return await this.enhanceResult(imageResult, options, startTime);
464
- }
465
- // ===== TTS MODE 1: Direct Input Synthesis (useAiResponse=false) =====
466
- // Synthesize input text directly without AI generation
467
- // This is optimal for simple read-aloud scenarios
468
- if (options.tts?.enabled && !options.tts?.useAiResponse) {
469
- const textToSynthesize = options.prompt ?? options.input?.text ?? "";
470
- // Build base result structure - common to both paths
471
- const baseResult = {
472
- content: textToSynthesize,
473
- provider: options.provider ?? this.providerName,
474
- model: this.modelName,
475
- usage: { input: 0, output: 0, total: 0 },
476
- };
477
- try {
478
- const ttsResult = await TTSProcessor.synthesize(textToSynthesize, options.provider ?? this.providerName, options.tts);
479
- baseResult.audio = ttsResult;
471
+ return providerTracer.startActiveSpan("neurolink.provider.generate", { kind: SpanKind.INTERNAL }, async (span) => {
472
+ const options = this.normalizeTextOptions(optionsOrPrompt);
473
+ this.validateOptions(options);
474
+ const startTime = Date.now();
475
+ span.setAttribute("gen_ai.system", this.providerName || "unknown");
476
+ span.setAttribute("gen_ai.request.model", this.modelName || options.model || "unknown");
477
+ try {
478
+ // ===== VIDEO GENERATION MODE =====
479
+ // Generate video from image + prompt using Veo 3.1
480
+ if (options.output?.mode === "video") {
481
+ return await this.handleVideoGeneration(options, startTime);
480
482
  }
481
- catch (ttsError) {
482
- logger.error(`TTS synthesis failed in Mode 1 (direct input synthesis):`, ttsError);
483
- // baseResult remains without audio - graceful degradation
483
+ // ===== IMAGE GENERATION MODE =====
484
+ // Route to executeImageGeneration for image generation models
485
+ const isImageModel = IMAGE_GENERATION_MODELS.some((m) => this.modelName.includes(m));
486
+ if (isImageModel) {
487
+ logger.info(`Image generation model detected, routing to executeImageGeneration`, {
488
+ provider: this.providerName,
489
+ model: this.modelName,
490
+ });
491
+ const imageResult = await this.executeImageGeneration(options);
492
+ return await this.enhanceResult(imageResult, options, startTime);
484
493
  }
485
- // Call enhanceResult for consistency - enables analytics/evaluation for TTS-only requests
486
- return await this.enhanceResult(baseResult, options, startTime);
487
- }
488
- // ===== Normal AI Generation Flow =====
489
- const { tools, model } = await this.prepareGenerationContext(options);
490
- const messages = await this.buildMessages(options);
491
- // ===== VIDEO ANALYSIS FROM MESSAGES CONTENT =====
492
- // Check if video files are present in messages content array
493
- // If video analysis is needed, perform it and return early to avoid running generation
494
- if (hasVideoFrames(messages)) {
495
- const videoAnalysisResult = await executeVideoAnalysis(messages, {
496
- provider: options.provider,
497
- providerName: this.providerName,
498
- region: options.region,
499
- model: options.model,
500
- });
501
- // Return video analysis result directly without running generation
502
- const videoResult = {
503
- content: videoAnalysisResult,
504
- provider: options.provider ?? this.providerName,
505
- model: this.modelName,
506
- usage: { input: 0, output: 0, total: 0 }, // Video analysis doesn't use standard token counting
507
- };
508
- return await this.enhanceResult(videoResult, options, startTime);
509
- }
510
- // Compose timeout signal with user-provided abort signal (mirrors stream path)
511
- const timeoutController = createTimeoutController(options.timeout, this.providerName, "generate");
512
- const composedSignal = composeAbortSignals(options.abortSignal, timeoutController?.controller.signal);
513
- const composedOptions = composedSignal
514
- ? { ...options, abortSignal: composedSignal }
515
- : options;
516
- let generateResult;
517
- try {
518
- generateResult = await this.executeGeneration(model, messages, tools, composedOptions);
519
- }
520
- finally {
521
- timeoutController?.cleanup();
522
- }
523
- this.analyzeAIResponse(generateResult);
524
- this.logGenerationComplete(generateResult);
525
- const responseTime = Date.now() - startTime;
526
- await this.recordPerformanceMetrics(generateResult.usage, responseTime);
527
- const { toolsUsed, toolExecutions } = this.extractToolInformation(generateResult);
528
- let enhancedResult = this.formatEnhancedResult(generateResult, tools, toolsUsed, toolExecutions, options);
529
- // ===== TTS MODE 2: AI Response Synthesis (useAiResponse=true) =====
530
- // Synthesize AI-generated response after generation completes
531
- if (options.tts?.enabled && options.tts?.useAiResponse) {
532
- const aiResponse = enhancedResult.content;
533
- const provider = options.provider ?? this.providerName;
534
- // Validate AI response and provider before synthesis
535
- if (aiResponse && provider) {
494
+ // ===== TTS MODE 1: Direct Input Synthesis (useAiResponse=false) =====
495
+ // Synthesize input text directly without AI generation
496
+ // This is optimal for simple read-aloud scenarios
497
+ if (options.tts?.enabled && !options.tts?.useAiResponse) {
498
+ const textToSynthesize = options.prompt ?? options.input?.text ?? "";
499
+ // Build base result structure - common to both paths
500
+ const baseResult = {
501
+ content: textToSynthesize,
502
+ provider: options.provider ?? this.providerName,
503
+ model: this.modelName,
504
+ usage: { input: 0, output: 0, total: 0 },
505
+ };
536
506
  try {
537
- const ttsResult = await TTSProcessor.synthesize(aiResponse, provider, options.tts);
538
- // Add audio to enhanced result (TTSProcessor already includes latency in metadata)
539
- enhancedResult = {
540
- ...enhancedResult,
541
- audio: ttsResult,
542
- };
507
+ const ttsResult = await TTSProcessor.synthesize(textToSynthesize, options.provider ?? this.providerName, options.tts);
508
+ baseResult.audio = ttsResult;
543
509
  }
544
510
  catch (ttsError) {
545
- // Log TTS error but continue with text-only result
546
- logger.error(`TTS synthesis failed in Mode 2 (AI response synthesis):`, ttsError);
547
- // enhancedResult remains unchanged (no audio field added)
511
+ logger.error(`TTS synthesis failed in Mode 1 (direct input synthesis):`, ttsError);
512
+ // baseResult remains without audio - graceful degradation
548
513
  }
514
+ // Call enhanceResult for consistency - enables analytics/evaluation for TTS-only requests
515
+ return await this.enhanceResult(baseResult, options, startTime);
549
516
  }
550
- else {
551
- logger.warn(`TTS synthesis skipped despite being enabled`, {
552
- provider: this.providerName,
553
- hasAiResponse: !!aiResponse,
554
- aiResponseLength: aiResponse?.length ?? 0,
555
- hasProvider: !!provider,
556
- ttsConfig: {
557
- enabled: options.tts?.enabled,
558
- useAiResponse: options.tts?.useAiResponse,
559
- },
560
- reason: !aiResponse
561
- ? "AI response is empty or undefined"
562
- : "Provider is missing",
517
+ // ===== Normal AI Generation Flow =====
518
+ const { tools, model } = await this.prepareGenerationContext(options);
519
+ const messages = await this.buildMessages(options);
520
+ // ===== VIDEO ANALYSIS FROM MESSAGES CONTENT =====
521
+ // Check if video files are present in messages content array
522
+ // If video analysis is needed, perform it via Gemini, then pass through Claude for formatting
523
+ if (hasVideoFrames(messages)) {
524
+ const videoAnalysisResult = await executeVideoAnalysis(messages, {
525
+ provider: options.provider,
526
+ providerName: this.providerName,
527
+ region: options.region,
528
+ // Don't pass the main conversation model — video analysis uses
529
+ // Google's Gemini API (generateContent) which only supports Gemini models.
530
+ // Let videoAnalysisProcessor use its own default (gemini-2.5-flash).
563
531
  });
532
+ // Extract user's original text from messages (excluding image parts)
533
+ const userTextParts = messages
534
+ .filter((m) => m.role === "user")
535
+ .flatMap((m) => Array.isArray(m.content)
536
+ ? m.content
537
+ .filter((p) => p.type === "text")
538
+ .map((p) => p.text)
539
+ : [typeof m.content === "string" ? m.content : ""])
540
+ .filter(Boolean);
541
+ const userText = userTextParts.join("\n").trim();
542
+ // Pass Gemini's analysis through Claude for structured JSON formatting
543
+ // The system prompt (from Curator) includes JSON_REPORT_PROMPT_SUFFIX
544
+ // which instructs Claude to output {"summary": "...", "details": "..."}
545
+ let formattedContent = videoAnalysisResult;
546
+ let usage = { input: 0, output: 0, total: 0 };
547
+ if (options.systemPrompt) {
548
+ try {
549
+ const formattingPrompt = userText
550
+ ? `The user asked: "${userText}"\n\nHere is the video/image analysis result from the visual analysis system:\n\n${videoAnalysisResult}\n\nBased on this analysis, provide your response.`
551
+ : `Here is a video/image analysis result from the visual analysis system:\n\n${videoAnalysisResult}\n\nBased on this analysis, provide your response.`;
552
+ logger.debug("[VideoAnalysis] Formatting via Claude", {
553
+ userTextLength: userText.length,
554
+ analysisLength: videoAnalysisResult.length,
555
+ });
556
+ const formattedResult = await generateText({
557
+ model,
558
+ system: options.systemPrompt,
559
+ messages: [
560
+ { role: "user", content: formattingPrompt },
561
+ ],
562
+ maxTokens: options.maxTokens || 8192,
563
+ temperature: 0.3,
564
+ abortSignal: options.abortSignal,
565
+ experimental_telemetry: this.telemetryHandler?.getTelemetryConfig(options, "generate"),
566
+ });
567
+ formattedContent = formattedResult.text;
568
+ usage = {
569
+ input: formattedResult.usage?.promptTokens || 0,
570
+ output: formattedResult.usage?.completionTokens || 0,
571
+ total: (formattedResult.usage?.promptTokens || 0) +
572
+ (formattedResult.usage?.completionTokens || 0),
573
+ };
574
+ logger.debug("[VideoAnalysis] Claude formatting complete", {
575
+ formattedLength: formattedContent.length,
576
+ usage,
577
+ });
578
+ }
579
+ catch (error) {
580
+ logger.warn("[VideoAnalysis] Claude formatting failed, using raw Gemini output", {
581
+ error: error instanceof Error ? error.message : String(error),
582
+ });
583
+ // formattedContent remains as raw videoAnalysisResult (graceful degradation)
584
+ }
585
+ }
586
+ const videoResult = {
587
+ content: formattedContent,
588
+ provider: options.provider ?? this.providerName,
589
+ model: this.modelName,
590
+ usage,
591
+ };
592
+ return await this.enhanceResult(videoResult, options, startTime);
564
593
  }
594
+ // Compose timeout signal with user-provided abort signal (mirrors stream path)
595
+ const timeoutController = createTimeoutController(options.timeout, this.providerName, "generate");
596
+ const composedSignal = composeAbortSignals(options.abortSignal, timeoutController?.controller.signal);
597
+ const composedOptions = composedSignal
598
+ ? { ...options, abortSignal: composedSignal }
599
+ : options;
600
+ let generateResult;
601
+ try {
602
+ generateResult = await this.executeGeneration(model, messages, tools, composedOptions);
603
+ }
604
+ finally {
605
+ timeoutController?.cleanup();
606
+ }
607
+ this.analyzeAIResponse(generateResult);
608
+ this.logGenerationComplete(generateResult);
609
+ const responseTime = Date.now() - startTime;
610
+ await this.recordPerformanceMetrics(generateResult.usage, responseTime);
611
+ const { toolsUsed, toolExecutions } = this.extractToolInformation(generateResult);
612
+ let enhancedResult = this.formatEnhancedResult(generateResult, tools, toolsUsed, toolExecutions, options);
613
+ // ===== TTS MODE 2: AI Response Synthesis (useAiResponse=true) =====
614
+ // Synthesize AI-generated response after generation completes
615
+ if (options.tts?.enabled && options.tts?.useAiResponse) {
616
+ const aiResponse = enhancedResult.content;
617
+ const provider = options.provider ?? this.providerName;
618
+ // Validate AI response and provider before synthesis
619
+ if (aiResponse && provider) {
620
+ try {
621
+ const ttsResult = await TTSProcessor.synthesize(aiResponse, provider, options.tts);
622
+ // Add audio to enhanced result (TTSProcessor already includes latency in metadata)
623
+ enhancedResult = {
624
+ ...enhancedResult,
625
+ audio: ttsResult,
626
+ };
627
+ }
628
+ catch (ttsError) {
629
+ // Log TTS error but continue with text-only result
630
+ logger.error(`TTS synthesis failed in Mode 2 (AI response synthesis):`, ttsError);
631
+ // enhancedResult remains unchanged (no audio field added)
632
+ }
633
+ }
634
+ else {
635
+ logger.warn(`TTS synthesis skipped despite being enabled`, {
636
+ provider: this.providerName,
637
+ hasAiResponse: !!aiResponse,
638
+ aiResponseLength: aiResponse?.length ?? 0,
639
+ hasProvider: !!provider,
640
+ ttsConfig: {
641
+ enabled: options.tts?.enabled,
642
+ useAiResponse: options.tts?.useAiResponse,
643
+ },
644
+ reason: !aiResponse
645
+ ? "AI response is empty or undefined"
646
+ : "Provider is missing",
647
+ });
648
+ }
649
+ }
650
+ // Set token usage on span from the result
651
+ if (enhancedResult?.usage) {
652
+ span.setAttribute("gen_ai.usage.input_tokens", enhancedResult.usage.input || 0);
653
+ span.setAttribute("gen_ai.usage.output_tokens", enhancedResult.usage.output || 0);
654
+ // Cost on span so users can query "what did this trace cost?"
655
+ const cost = calculateCost(this.providerName, this.modelName, {
656
+ input: enhancedResult.usage.input || 0,
657
+ output: enhancedResult.usage.output || 0,
658
+ total: enhancedResult.usage.total || 0,
659
+ });
660
+ span.setAttribute("neurolink.cost", cost ?? 0);
661
+ }
662
+ if (enhancedResult?.finishReason) {
663
+ span.setAttribute("gen_ai.response.finish_reason", enhancedResult.finishReason);
664
+ }
665
+ span.setStatus({ code: SpanStatusCode.OK });
666
+ return await this.enhanceResult(enhancedResult, options, startTime);
565
667
  }
566
- return await this.enhanceResult(enhancedResult, options, startTime);
567
- }
568
- catch (error) {
569
- // Abort errors are expected when a generation is cancelled — log at info, not error
570
- if (isAbortError(error)) {
571
- logger.info(`Generate aborted for ${this.providerName}`, {
572
- error: error instanceof Error ? error.message : String(error),
668
+ catch (error) {
669
+ span.setStatus({
670
+ code: SpanStatusCode.ERROR,
671
+ message: error instanceof Error ? error.message : String(error),
573
672
  });
673
+ // Abort errors are expected when a generation is cancelled — log at info, not error
674
+ if (isAbortError(error)) {
675
+ logger.info(`Generate aborted for ${this.providerName}`, {
676
+ error: error instanceof Error ? error.message : String(error),
677
+ });
678
+ }
679
+ else {
680
+ logger.error(`Generate failed for ${this.providerName}:`, error);
681
+ }
682
+ throw this.handleProviderError(error);
574
683
  }
575
- else {
576
- logger.error(`Generate failed for ${this.providerName}:`, error);
684
+ finally {
685
+ span.end();
577
686
  }
578
- throw this.handleProviderError(error);
579
- }
687
+ });
580
688
  }
581
689
  /**
582
690
  * Alias for generate method - implements AIProvider interface
@@ -857,8 +965,6 @@ export class BaseProvider {
857
965
  * @param functionTag - Function name for logging
858
966
  */
859
967
  setupToolExecutor(sdk, functionTag) {
860
- this.customTools = sdk.customTools;
861
- this.toolExecutor = sdk.executeTool;
862
968
  this.toolsManager.setupToolExecutor(sdk, functionTag);
863
969
  }
864
970
  // ===================