@juspay/neurolink 9.15.0 → 9.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +22 -20
- package/dist/adapters/video/videoAnalyzer.d.ts +1 -1
- package/dist/adapters/video/videoAnalyzer.js +10 -8
- package/dist/cli/commands/setup-anthropic.js +1 -14
- package/dist/cli/commands/setup-azure.js +1 -12
- package/dist/cli/commands/setup-bedrock.js +1 -9
- package/dist/cli/commands/setup-google-ai.js +1 -12
- package/dist/cli/commands/setup-openai.js +1 -14
- package/dist/cli/commands/workflow.d.ts +27 -0
- package/dist/cli/commands/workflow.js +216 -0
- package/dist/cli/factories/commandFactory.js +79 -20
- package/dist/cli/index.js +0 -1
- package/dist/cli/parser.js +4 -1
- package/dist/cli/utils/maskCredential.d.ts +11 -0
- package/dist/cli/utils/maskCredential.js +23 -0
- package/dist/constants/contextWindows.js +107 -16
- package/dist/constants/enums.d.ts +99 -15
- package/dist/constants/enums.js +152 -22
- package/dist/context/budgetChecker.js +1 -1
- package/dist/context/contextCompactor.js +31 -4
- package/dist/context/emergencyTruncation.d.ts +21 -0
- package/dist/context/emergencyTruncation.js +88 -0
- package/dist/context/errorDetection.d.ts +16 -0
- package/dist/context/errorDetection.js +48 -1
- package/dist/context/errors.d.ts +19 -0
- package/dist/context/errors.js +21 -0
- package/dist/context/stages/slidingWindowTruncator.d.ts +6 -0
- package/dist/context/stages/slidingWindowTruncator.js +159 -24
- package/dist/context/stages/structuredSummarizer.js +2 -2
- package/dist/core/baseProvider.js +306 -200
- package/dist/core/conversationMemoryManager.js +104 -61
- package/dist/core/evaluationProviders.js +16 -33
- package/dist/core/factory.js +237 -164
- package/dist/core/modules/GenerationHandler.js +175 -116
- package/dist/core/modules/MessageBuilder.js +222 -170
- package/dist/core/modules/StreamHandler.d.ts +1 -0
- package/dist/core/modules/StreamHandler.js +95 -27
- package/dist/core/modules/TelemetryHandler.d.ts +10 -1
- package/dist/core/modules/TelemetryHandler.js +25 -7
- package/dist/core/modules/ToolsManager.js +115 -191
- package/dist/core/redisConversationMemoryManager.js +418 -282
- package/dist/factories/providerRegistry.d.ts +5 -0
- package/dist/factories/providerRegistry.js +20 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +4 -2
- package/dist/lib/adapters/video/videoAnalyzer.d.ts +1 -1
- package/dist/lib/adapters/video/videoAnalyzer.js +10 -8
- package/dist/lib/constants/contextWindows.js +107 -16
- package/dist/lib/constants/enums.d.ts +99 -15
- package/dist/lib/constants/enums.js +152 -22
- package/dist/lib/context/budgetChecker.js +1 -1
- package/dist/lib/context/contextCompactor.js +31 -4
- package/dist/lib/context/emergencyTruncation.d.ts +21 -0
- package/dist/lib/context/emergencyTruncation.js +89 -0
- package/dist/lib/context/errorDetection.d.ts +16 -0
- package/dist/lib/context/errorDetection.js +48 -1
- package/dist/lib/context/errors.d.ts +19 -0
- package/dist/lib/context/errors.js +22 -0
- package/dist/lib/context/stages/slidingWindowTruncator.d.ts +6 -0
- package/dist/lib/context/stages/slidingWindowTruncator.js +159 -24
- package/dist/lib/context/stages/structuredSummarizer.js +2 -2
- package/dist/lib/core/baseProvider.js +306 -200
- package/dist/lib/core/conversationMemoryManager.js +104 -61
- package/dist/lib/core/evaluationProviders.js +16 -33
- package/dist/lib/core/factory.js +237 -164
- package/dist/lib/core/modules/GenerationHandler.js +175 -116
- package/dist/lib/core/modules/MessageBuilder.js +222 -170
- package/dist/lib/core/modules/StreamHandler.d.ts +1 -0
- package/dist/lib/core/modules/StreamHandler.js +95 -27
- package/dist/lib/core/modules/TelemetryHandler.d.ts +10 -1
- package/dist/lib/core/modules/TelemetryHandler.js +25 -7
- package/dist/lib/core/modules/ToolsManager.js +115 -191
- package/dist/lib/core/redisConversationMemoryManager.js +418 -282
- package/dist/lib/factories/providerRegistry.d.ts +5 -0
- package/dist/lib/factories/providerRegistry.js +20 -2
- package/dist/lib/index.d.ts +2 -2
- package/dist/lib/index.js +4 -2
- package/dist/lib/mcp/externalServerManager.js +66 -0
- package/dist/lib/mcp/mcpCircuitBreaker.js +24 -0
- package/dist/lib/mcp/mcpClientFactory.js +16 -0
- package/dist/lib/mcp/toolDiscoveryService.js +32 -6
- package/dist/lib/mcp/toolRegistry.js +193 -123
- package/dist/lib/neurolink.d.ts +6 -0
- package/dist/lib/neurolink.js +1162 -646
- package/dist/lib/providers/amazonBedrock.d.ts +1 -1
- package/dist/lib/providers/amazonBedrock.js +521 -319
- package/dist/lib/providers/anthropic.js +73 -17
- package/dist/lib/providers/anthropicBaseProvider.js +77 -17
- package/dist/lib/providers/googleAiStudio.d.ts +1 -1
- package/dist/lib/providers/googleAiStudio.js +292 -227
- package/dist/lib/providers/googleVertex.d.ts +36 -1
- package/dist/lib/providers/googleVertex.js +553 -260
- package/dist/lib/providers/ollama.js +329 -278
- package/dist/lib/providers/openAI.js +77 -19
- package/dist/lib/providers/sagemaker/parsers.js +3 -3
- package/dist/lib/providers/sagemaker/streaming.js +3 -3
- package/dist/lib/proxy/proxyFetch.js +81 -48
- package/dist/lib/rag/ChunkerFactory.js +1 -1
- package/dist/lib/rag/chunkers/MarkdownChunker.d.ts +22 -0
- package/dist/lib/rag/chunkers/MarkdownChunker.js +213 -9
- package/dist/lib/rag/chunking/markdownChunker.d.ts +16 -0
- package/dist/lib/rag/chunking/markdownChunker.js +174 -2
- package/dist/lib/rag/pipeline/contextAssembly.js +2 -1
- package/dist/lib/rag/ragIntegration.d.ts +18 -1
- package/dist/lib/rag/ragIntegration.js +94 -14
- package/dist/lib/rag/retrieval/vectorQueryTool.js +21 -4
- package/dist/lib/server/abstract/baseServerAdapter.js +4 -1
- package/dist/lib/server/adapters/fastifyAdapter.js +35 -30
- package/dist/lib/services/server/ai/observability/instrumentation.d.ts +32 -0
- package/dist/lib/services/server/ai/observability/instrumentation.js +39 -0
- package/dist/lib/telemetry/attributes.d.ts +52 -0
- package/dist/lib/telemetry/attributes.js +61 -0
- package/dist/lib/telemetry/index.d.ts +3 -0
- package/dist/lib/telemetry/index.js +3 -0
- package/dist/lib/telemetry/telemetryService.d.ts +6 -0
- package/dist/lib/telemetry/telemetryService.js +6 -0
- package/dist/lib/telemetry/tracers.d.ts +15 -0
- package/dist/lib/telemetry/tracers.js +17 -0
- package/dist/lib/telemetry/withSpan.d.ts +9 -0
- package/dist/lib/telemetry/withSpan.js +35 -0
- package/dist/lib/types/contextTypes.d.ts +10 -0
- package/dist/lib/types/streamTypes.d.ts +14 -0
- package/dist/lib/utils/conversationMemory.js +123 -84
- package/dist/lib/utils/logger.d.ts +5 -0
- package/dist/lib/utils/logger.js +50 -2
- package/dist/lib/utils/messageBuilder.js +22 -42
- package/dist/lib/utils/modelDetection.js +3 -3
- package/dist/lib/utils/providerRetry.d.ts +41 -0
- package/dist/lib/utils/providerRetry.js +114 -0
- package/dist/lib/utils/retryability.d.ts +14 -0
- package/dist/lib/utils/retryability.js +23 -0
- package/dist/lib/utils/sanitizers/svg.js +4 -5
- package/dist/lib/utils/tokenEstimation.d.ts +11 -1
- package/dist/lib/utils/tokenEstimation.js +19 -4
- package/dist/lib/utils/videoAnalysisProcessor.js +7 -3
- package/dist/mcp/externalServerManager.js +66 -0
- package/dist/mcp/mcpCircuitBreaker.js +24 -0
- package/dist/mcp/mcpClientFactory.js +16 -0
- package/dist/mcp/toolDiscoveryService.js +32 -6
- package/dist/mcp/toolRegistry.js +193 -123
- package/dist/neurolink.d.ts +6 -0
- package/dist/neurolink.js +1162 -646
- package/dist/providers/amazonBedrock.d.ts +1 -1
- package/dist/providers/amazonBedrock.js +521 -319
- package/dist/providers/anthropic.js +73 -17
- package/dist/providers/anthropicBaseProvider.js +77 -17
- package/dist/providers/googleAiStudio.d.ts +1 -1
- package/dist/providers/googleAiStudio.js +292 -227
- package/dist/providers/googleVertex.d.ts +36 -1
- package/dist/providers/googleVertex.js +553 -260
- package/dist/providers/ollama.js +329 -278
- package/dist/providers/openAI.js +77 -19
- package/dist/providers/sagemaker/parsers.js +3 -3
- package/dist/providers/sagemaker/streaming.js +3 -3
- package/dist/proxy/proxyFetch.js +81 -48
- package/dist/rag/ChunkerFactory.js +1 -1
- package/dist/rag/chunkers/MarkdownChunker.d.ts +22 -0
- package/dist/rag/chunkers/MarkdownChunker.js +213 -9
- package/dist/rag/chunking/markdownChunker.d.ts +16 -0
- package/dist/rag/chunking/markdownChunker.js +174 -2
- package/dist/rag/pipeline/contextAssembly.js +2 -1
- package/dist/rag/ragIntegration.d.ts +18 -1
- package/dist/rag/ragIntegration.js +94 -14
- package/dist/rag/retrieval/vectorQueryTool.js +21 -4
- package/dist/server/abstract/baseServerAdapter.js +4 -1
- package/dist/server/adapters/fastifyAdapter.js +35 -30
- package/dist/services/server/ai/observability/instrumentation.d.ts +32 -0
- package/dist/services/server/ai/observability/instrumentation.js +39 -0
- package/dist/telemetry/attributes.d.ts +52 -0
- package/dist/telemetry/attributes.js +60 -0
- package/dist/telemetry/index.d.ts +3 -0
- package/dist/telemetry/index.js +3 -0
- package/dist/telemetry/telemetryService.d.ts +6 -0
- package/dist/telemetry/telemetryService.js +6 -0
- package/dist/telemetry/tracers.d.ts +15 -0
- package/dist/telemetry/tracers.js +16 -0
- package/dist/telemetry/withSpan.d.ts +9 -0
- package/dist/telemetry/withSpan.js +34 -0
- package/dist/types/contextTypes.d.ts +10 -0
- package/dist/types/streamTypes.d.ts +14 -0
- package/dist/utils/conversationMemory.js +123 -84
- package/dist/utils/logger.d.ts +5 -0
- package/dist/utils/logger.js +50 -2
- package/dist/utils/messageBuilder.js +22 -42
- package/dist/utils/modelDetection.js +3 -3
- package/dist/utils/providerRetry.d.ts +41 -0
- package/dist/utils/providerRetry.js +113 -0
- package/dist/utils/retryability.d.ts +14 -0
- package/dist/utils/retryability.js +22 -0
- package/dist/utils/sanitizers/svg.js +4 -5
- package/dist/utils/tokenEstimation.d.ts +11 -1
- package/dist/utils/tokenEstimation.js +19 -4
- package/dist/utils/videoAnalysisProcessor.js +7 -3
- package/dist/workflow/config.d.ts +26 -26
- package/package.json +1 -1
|
@@ -1,8 +1,12 @@
|
|
|
1
|
+
import { generateText } from "ai";
|
|
2
|
+
import { SpanKind, SpanStatusCode } from "@opentelemetry/api";
|
|
3
|
+
import { tracers } from "../telemetry/tracers.js";
|
|
1
4
|
import { directAgentTools } from "../agent/directTools.js";
|
|
2
5
|
import { IMAGE_GENERATION_MODELS } from "../core/constants.js";
|
|
3
6
|
import { MiddlewareFactory } from "../middleware/factory.js";
|
|
4
7
|
import { isAbortError } from "../utils/errorHandling.js";
|
|
5
8
|
import { logger } from "../utils/logger.js";
|
|
9
|
+
import { calculateCost } from "../utils/pricing.js";
|
|
6
10
|
import { composeAbortSignals, createTimeoutController, TimeoutError, } from "../utils/timeout.js";
|
|
7
11
|
import { shouldDisableBuiltinTools } from "../utils/toolUtils.js";
|
|
8
12
|
import { getKeyCount, getKeysAsString } from "../utils/transformationUtils.js";
|
|
@@ -15,6 +19,7 @@ import { StreamHandler } from "./modules/StreamHandler.js";
|
|
|
15
19
|
import { TelemetryHandler } from "./modules/TelemetryHandler.js";
|
|
16
20
|
import { ToolsManager } from "./modules/ToolsManager.js";
|
|
17
21
|
import { Utilities } from "./modules/Utilities.js";
|
|
22
|
+
const providerTracer = tracers.provider;
|
|
18
23
|
/**
|
|
19
24
|
* Abstract base class for all AI providers
|
|
20
25
|
* Tools are integrated as first-class citizens - always available by default
|
|
@@ -49,8 +54,8 @@ export class BaseProvider {
|
|
|
49
54
|
// Initialize composition modules
|
|
50
55
|
this.messageBuilder = new MessageBuilder(this.providerName, this.modelName);
|
|
51
56
|
this.streamHandler = new StreamHandler(this.providerName, this.modelName);
|
|
52
|
-
this.generationHandler = new GenerationHandler(this.providerName, this.modelName, () => this.supportsTools(), (options, type) => this.telemetryHandler.getTelemetryConfig(options, type), (toolCalls, toolResults, options, timestamp) => this.handleToolExecutionStorage(toolCalls, toolResults, options, timestamp));
|
|
53
57
|
this.telemetryHandler = new TelemetryHandler(this.providerName, this.modelName, this.neurolink);
|
|
58
|
+
this.generationHandler = new GenerationHandler(this.providerName, this.modelName, () => this.supportsTools(), (options, type) => this.telemetryHandler.getTelemetryConfig(options, type), (toolCalls, toolResults, options, timestamp) => this.handleToolExecutionStorage(toolCalls, toolResults, options, timestamp));
|
|
54
59
|
this.utilities = new Utilities(this.providerName, this.modelName, this.defaultTimeout, this.middlewareOptions);
|
|
55
60
|
this.toolsManager = new ToolsManager(this.providerName, this.directTools, this.neurolink, {
|
|
56
61
|
isZodSchema: (schema) => this.isZodSchema(schema),
|
|
@@ -75,86 +80,107 @@ export class BaseProvider {
|
|
|
75
80
|
* When tools are involved, falls back to generate() with synthetic streaming
|
|
76
81
|
*/
|
|
77
82
|
async stream(optionsOrPrompt, analysisSchema) {
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
supportsTools: this.supportsTools(),
|
|
84
|
-
inputLength: options.input?.text?.length || 0,
|
|
85
|
-
maxTokens: options.maxTokens,
|
|
86
|
-
temperature: options.temperature,
|
|
87
|
-
timestamp: Date.now(),
|
|
88
|
-
});
|
|
89
|
-
// ===== EARLY MULTIMODAL DETECTION =====
|
|
90
|
-
const hasFileInput = !!options.input?.files?.length || !!options.input?.videoFiles?.length;
|
|
91
|
-
if (hasFileInput) {
|
|
92
|
-
// ===== VIDEO ANALYSIS DETECTION =====
|
|
93
|
-
// Check if video frames are present and handle with fake streaming
|
|
94
|
-
const messages = await this.buildMessagesForStream(options);
|
|
95
|
-
if (hasVideoFrames(messages)) {
|
|
96
|
-
logger.info(`Video frames detected in stream, using fake streaming for video analysis`, {
|
|
97
|
-
provider: this.providerName,
|
|
98
|
-
model: this.modelName,
|
|
99
|
-
});
|
|
100
|
-
return await this.executeFakeStreaming(options, analysisSchema);
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
// 🔧 CRITICAL: Image generation models don't support real streaming
|
|
104
|
-
// Force fake streaming for image models to ensure image output is yielded
|
|
105
|
-
const isImageModel = IMAGE_GENERATION_MODELS.some((m) => this.modelName.includes(m));
|
|
106
|
-
if (isImageModel) {
|
|
107
|
-
logger.info(`Image model detected, forcing fake streaming`, {
|
|
83
|
+
return providerTracer.startActiveSpan("neurolink.provider.stream", { kind: SpanKind.INTERNAL }, async (span) => {
|
|
84
|
+
let options = this.normalizeStreamOptions(optionsOrPrompt);
|
|
85
|
+
span.setAttribute("gen_ai.system", this.providerName || "unknown");
|
|
86
|
+
span.setAttribute("gen_ai.request.model", this.modelName || options.model || "unknown");
|
|
87
|
+
logger.info(`Starting stream`, {
|
|
108
88
|
provider: this.providerName,
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
// Central tool merge: Pre-merge base tools (MCP/built-in) with user-provided
|
|
116
|
-
// tools (e.g. RAG tools) into options.tools. This way, every provider's
|
|
117
|
-
// executeStream() can simply use options.tools (or getAllTools() + options.tools)
|
|
118
|
-
// and get the complete tool set without needing per-provider merge logic.
|
|
119
|
-
if (!options.disableTools && this.supportsTools()) {
|
|
120
|
-
const mergedTools = await this.getToolsForStream(options);
|
|
121
|
-
options = { ...options, tools: mergedTools };
|
|
122
|
-
}
|
|
123
|
-
else {
|
|
124
|
-
options = { ...options, tools: {} };
|
|
125
|
-
}
|
|
126
|
-
// CRITICAL FIX: Always prefer real streaming over fake streaming
|
|
127
|
-
// Try real streaming first, use fake streaming only as fallback
|
|
128
|
-
try {
|
|
129
|
-
logger.debug(`Attempting real streaming`, {
|
|
130
|
-
provider: this.providerName,
|
|
131
|
-
timestamp: Date.now(),
|
|
132
|
-
});
|
|
133
|
-
const realStreamResult = await this.executeStream(options, analysisSchema);
|
|
134
|
-
logger.info(`Real streaming succeeded`, {
|
|
135
|
-
provider: this.providerName,
|
|
136
|
-
timestamp: Date.now(),
|
|
137
|
-
});
|
|
138
|
-
// If real streaming succeeds, return it (with tools support via Vercel AI SDK)
|
|
139
|
-
return realStreamResult;
|
|
140
|
-
}
|
|
141
|
-
catch (realStreamError) {
|
|
142
|
-
logger.warn(`Real streaming failed for ${this.providerName}, falling back to fake streaming:`, {
|
|
143
|
-
error: realStreamError instanceof Error
|
|
144
|
-
? realStreamError.message
|
|
145
|
-
: String(realStreamError),
|
|
89
|
+
hasTools: !options.disableTools && this.supportsTools(),
|
|
90
|
+
disableTools: !!options.disableTools,
|
|
91
|
+
supportsTools: this.supportsTools(),
|
|
92
|
+
inputLength: options.input?.text?.length || 0,
|
|
93
|
+
maxTokens: options.maxTokens,
|
|
94
|
+
temperature: options.temperature,
|
|
146
95
|
timestamp: Date.now(),
|
|
147
96
|
});
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
97
|
+
try {
|
|
98
|
+
// ===== EARLY MULTIMODAL DETECTION =====
|
|
99
|
+
const hasFileInput = !!options.input?.files?.length ||
|
|
100
|
+
!!options.input?.videoFiles?.length;
|
|
101
|
+
if (hasFileInput) {
|
|
102
|
+
// ===== VIDEO ANALYSIS DETECTION =====
|
|
103
|
+
// Check if video frames are present and handle with fake streaming
|
|
104
|
+
const messages = await this.buildMessagesForStream(options);
|
|
105
|
+
if (hasVideoFrames(messages)) {
|
|
106
|
+
logger.info(`Video frames detected in stream, using fake streaming for video analysis`, {
|
|
107
|
+
provider: this.providerName,
|
|
108
|
+
model: this.modelName,
|
|
109
|
+
});
|
|
110
|
+
span.setAttribute("neurolink.stream_mode", "fake");
|
|
111
|
+
return await this.executeFakeStreaming(options, analysisSchema);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
// Image generation models don't support real streaming
|
|
115
|
+
// Force fake streaming for image models to ensure image output is yielded
|
|
116
|
+
const isImageModel = IMAGE_GENERATION_MODELS.some((m) => this.modelName.includes(m));
|
|
117
|
+
if (isImageModel) {
|
|
118
|
+
logger.info(`Image model detected, forcing fake streaming`, {
|
|
119
|
+
provider: this.providerName,
|
|
120
|
+
model: this.modelName,
|
|
121
|
+
reason: "Image generation requires fake streaming to yield image output",
|
|
122
|
+
});
|
|
123
|
+
// Skip real streaming, go directly to fake streaming
|
|
124
|
+
span.setAttribute("neurolink.stream_mode", "fake");
|
|
125
|
+
return await this.executeFakeStreaming(options, analysisSchema);
|
|
126
|
+
}
|
|
127
|
+
// Central tool merge: Pre-merge base tools (MCP/built-in) with user-provided
|
|
128
|
+
// tools (e.g. RAG tools) into options.tools. This way, every provider's
|
|
129
|
+
// executeStream() can simply use options.tools (or getAllTools() + options.tools)
|
|
130
|
+
// and get the complete tool set without needing per-provider merge logic.
|
|
131
|
+
if (!options.disableTools && this.supportsTools()) {
|
|
132
|
+
const mergedTools = await this.getToolsForStream(options);
|
|
133
|
+
options = { ...options, tools: mergedTools };
|
|
134
|
+
}
|
|
135
|
+
else {
|
|
136
|
+
options = { ...options, tools: {} };
|
|
137
|
+
}
|
|
138
|
+
// CRITICAL FIX: Always prefer real streaming over fake streaming
|
|
139
|
+
// Try real streaming first, use fake streaming only as fallback
|
|
140
|
+
try {
|
|
141
|
+
logger.debug(`Attempting real streaming`, {
|
|
142
|
+
provider: this.providerName,
|
|
143
|
+
timestamp: Date.now(),
|
|
144
|
+
});
|
|
145
|
+
const realStreamResult = await this.executeStream(options, analysisSchema);
|
|
146
|
+
logger.info(`Real streaming succeeded`, {
|
|
147
|
+
provider: this.providerName,
|
|
148
|
+
timestamp: Date.now(),
|
|
149
|
+
});
|
|
150
|
+
span.setAttribute("neurolink.stream_mode", "real");
|
|
151
|
+
// If real streaming succeeds, return it (with tools support via Vercel AI SDK)
|
|
152
|
+
return realStreamResult;
|
|
153
|
+
}
|
|
154
|
+
catch (realStreamError) {
|
|
155
|
+
logger.warn(`Real streaming failed for ${this.providerName}, falling back to fake streaming:`, {
|
|
156
|
+
error: realStreamError instanceof Error
|
|
157
|
+
? realStreamError.message
|
|
158
|
+
: String(realStreamError),
|
|
159
|
+
timestamp: Date.now(),
|
|
160
|
+
});
|
|
161
|
+
// Fallback to fake streaming only if real streaming fails AND tools are enabled
|
|
162
|
+
if (!options.disableTools && this.supportsTools()) {
|
|
163
|
+
span.setAttribute("neurolink.stream_mode", "fake");
|
|
164
|
+
return await this.executeFakeStreaming(options, analysisSchema);
|
|
165
|
+
}
|
|
166
|
+
else {
|
|
167
|
+
// If real streaming failed and no tools are enabled, re-throw the original error
|
|
168
|
+
logger.error(`Real streaming failed for ${this.providerName}:`, realStreamError);
|
|
169
|
+
throw this.handleProviderError(realStreamError);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
151
172
|
}
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
173
|
+
catch (error) {
|
|
174
|
+
span.setStatus({
|
|
175
|
+
code: SpanStatusCode.ERROR,
|
|
176
|
+
message: error instanceof Error ? error.message : String(error),
|
|
177
|
+
});
|
|
178
|
+
throw error;
|
|
156
179
|
}
|
|
157
|
-
|
|
180
|
+
finally {
|
|
181
|
+
span.end();
|
|
182
|
+
}
|
|
183
|
+
});
|
|
158
184
|
}
|
|
159
185
|
/**
|
|
160
186
|
* Execute fake streaming - extracted method for reusability
|
|
@@ -442,141 +468,223 @@ export class BaseProvider {
|
|
|
442
468
|
* for consistency and better performance
|
|
443
469
|
*/
|
|
444
470
|
async generate(optionsOrPrompt, _analysisSchema) {
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
const isImageModel = IMAGE_GENERATION_MODELS.some((m) => this.modelName.includes(m));
|
|
457
|
-
if (isImageModel) {
|
|
458
|
-
logger.info(`Image generation model detected, routing to executeImageGeneration`, {
|
|
459
|
-
provider: this.providerName,
|
|
460
|
-
model: this.modelName,
|
|
461
|
-
});
|
|
462
|
-
const imageResult = await this.executeImageGeneration(options);
|
|
463
|
-
return await this.enhanceResult(imageResult, options, startTime);
|
|
464
|
-
}
|
|
465
|
-
// ===== TTS MODE 1: Direct Input Synthesis (useAiResponse=false) =====
|
|
466
|
-
// Synthesize input text directly without AI generation
|
|
467
|
-
// This is optimal for simple read-aloud scenarios
|
|
468
|
-
if (options.tts?.enabled && !options.tts?.useAiResponse) {
|
|
469
|
-
const textToSynthesize = options.prompt ?? options.input?.text ?? "";
|
|
470
|
-
// Build base result structure - common to both paths
|
|
471
|
-
const baseResult = {
|
|
472
|
-
content: textToSynthesize,
|
|
473
|
-
provider: options.provider ?? this.providerName,
|
|
474
|
-
model: this.modelName,
|
|
475
|
-
usage: { input: 0, output: 0, total: 0 },
|
|
476
|
-
};
|
|
477
|
-
try {
|
|
478
|
-
const ttsResult = await TTSProcessor.synthesize(textToSynthesize, options.provider ?? this.providerName, options.tts);
|
|
479
|
-
baseResult.audio = ttsResult;
|
|
471
|
+
return providerTracer.startActiveSpan("neurolink.provider.generate", { kind: SpanKind.INTERNAL }, async (span) => {
|
|
472
|
+
const options = this.normalizeTextOptions(optionsOrPrompt);
|
|
473
|
+
this.validateOptions(options);
|
|
474
|
+
const startTime = Date.now();
|
|
475
|
+
span.setAttribute("gen_ai.system", this.providerName || "unknown");
|
|
476
|
+
span.setAttribute("gen_ai.request.model", this.modelName || options.model || "unknown");
|
|
477
|
+
try {
|
|
478
|
+
// ===== VIDEO GENERATION MODE =====
|
|
479
|
+
// Generate video from image + prompt using Veo 3.1
|
|
480
|
+
if (options.output?.mode === "video") {
|
|
481
|
+
return await this.handleVideoGeneration(options, startTime);
|
|
480
482
|
}
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
483
|
+
// ===== IMAGE GENERATION MODE =====
|
|
484
|
+
// Route to executeImageGeneration for image generation models
|
|
485
|
+
const isImageModel = IMAGE_GENERATION_MODELS.some((m) => this.modelName.includes(m));
|
|
486
|
+
if (isImageModel) {
|
|
487
|
+
logger.info(`Image generation model detected, routing to executeImageGeneration`, {
|
|
488
|
+
provider: this.providerName,
|
|
489
|
+
model: this.modelName,
|
|
490
|
+
});
|
|
491
|
+
const imageResult = await this.executeImageGeneration(options);
|
|
492
|
+
return await this.enhanceResult(imageResult, options, startTime);
|
|
484
493
|
}
|
|
485
|
-
//
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
providerName: this.providerName,
|
|
498
|
-
region: options.region,
|
|
499
|
-
model: options.model,
|
|
500
|
-
});
|
|
501
|
-
// Return video analysis result directly without running generation
|
|
502
|
-
const videoResult = {
|
|
503
|
-
content: videoAnalysisResult,
|
|
504
|
-
provider: options.provider ?? this.providerName,
|
|
505
|
-
model: this.modelName,
|
|
506
|
-
usage: { input: 0, output: 0, total: 0 }, // Video analysis doesn't use standard token counting
|
|
507
|
-
};
|
|
508
|
-
return await this.enhanceResult(videoResult, options, startTime);
|
|
509
|
-
}
|
|
510
|
-
// Compose timeout signal with user-provided abort signal (mirrors stream path)
|
|
511
|
-
const timeoutController = createTimeoutController(options.timeout, this.providerName, "generate");
|
|
512
|
-
const composedSignal = composeAbortSignals(options.abortSignal, timeoutController?.controller.signal);
|
|
513
|
-
const composedOptions = composedSignal
|
|
514
|
-
? { ...options, abortSignal: composedSignal }
|
|
515
|
-
: options;
|
|
516
|
-
let generateResult;
|
|
517
|
-
try {
|
|
518
|
-
generateResult = await this.executeGeneration(model, messages, tools, composedOptions);
|
|
519
|
-
}
|
|
520
|
-
finally {
|
|
521
|
-
timeoutController?.cleanup();
|
|
522
|
-
}
|
|
523
|
-
this.analyzeAIResponse(generateResult);
|
|
524
|
-
this.logGenerationComplete(generateResult);
|
|
525
|
-
const responseTime = Date.now() - startTime;
|
|
526
|
-
await this.recordPerformanceMetrics(generateResult.usage, responseTime);
|
|
527
|
-
const { toolsUsed, toolExecutions } = this.extractToolInformation(generateResult);
|
|
528
|
-
let enhancedResult = this.formatEnhancedResult(generateResult, tools, toolsUsed, toolExecutions, options);
|
|
529
|
-
// ===== TTS MODE 2: AI Response Synthesis (useAiResponse=true) =====
|
|
530
|
-
// Synthesize AI-generated response after generation completes
|
|
531
|
-
if (options.tts?.enabled && options.tts?.useAiResponse) {
|
|
532
|
-
const aiResponse = enhancedResult.content;
|
|
533
|
-
const provider = options.provider ?? this.providerName;
|
|
534
|
-
// Validate AI response and provider before synthesis
|
|
535
|
-
if (aiResponse && provider) {
|
|
494
|
+
// ===== TTS MODE 1: Direct Input Synthesis (useAiResponse=false) =====
|
|
495
|
+
// Synthesize input text directly without AI generation
|
|
496
|
+
// This is optimal for simple read-aloud scenarios
|
|
497
|
+
if (options.tts?.enabled && !options.tts?.useAiResponse) {
|
|
498
|
+
const textToSynthesize = options.prompt ?? options.input?.text ?? "";
|
|
499
|
+
// Build base result structure - common to both paths
|
|
500
|
+
const baseResult = {
|
|
501
|
+
content: textToSynthesize,
|
|
502
|
+
provider: options.provider ?? this.providerName,
|
|
503
|
+
model: this.modelName,
|
|
504
|
+
usage: { input: 0, output: 0, total: 0 },
|
|
505
|
+
};
|
|
536
506
|
try {
|
|
537
|
-
const ttsResult = await TTSProcessor.synthesize(
|
|
538
|
-
|
|
539
|
-
enhancedResult = {
|
|
540
|
-
...enhancedResult,
|
|
541
|
-
audio: ttsResult,
|
|
542
|
-
};
|
|
507
|
+
const ttsResult = await TTSProcessor.synthesize(textToSynthesize, options.provider ?? this.providerName, options.tts);
|
|
508
|
+
baseResult.audio = ttsResult;
|
|
543
509
|
}
|
|
544
510
|
catch (ttsError) {
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
// enhancedResult remains unchanged (no audio field added)
|
|
511
|
+
logger.error(`TTS synthesis failed in Mode 1 (direct input synthesis):`, ttsError);
|
|
512
|
+
// baseResult remains without audio - graceful degradation
|
|
548
513
|
}
|
|
514
|
+
// Call enhanceResult for consistency - enables analytics/evaluation for TTS-only requests
|
|
515
|
+
return await this.enhanceResult(baseResult, options, startTime);
|
|
549
516
|
}
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
517
|
+
// ===== Normal AI Generation Flow =====
|
|
518
|
+
const { tools, model } = await this.prepareGenerationContext(options);
|
|
519
|
+
const messages = await this.buildMessages(options);
|
|
520
|
+
// ===== VIDEO ANALYSIS FROM MESSAGES CONTENT =====
|
|
521
|
+
// Check if video files are present in messages content array
|
|
522
|
+
// If video analysis is needed, perform it via Gemini, then pass through Claude for formatting
|
|
523
|
+
if (hasVideoFrames(messages)) {
|
|
524
|
+
const videoAnalysisResult = await executeVideoAnalysis(messages, {
|
|
525
|
+
provider: options.provider,
|
|
526
|
+
providerName: this.providerName,
|
|
527
|
+
region: options.region,
|
|
528
|
+
// Don't pass the main conversation model — video analysis uses
|
|
529
|
+
// Google's Gemini API (generateContent) which only supports Gemini models.
|
|
530
|
+
// Let videoAnalysisProcessor use its own default (gemini-2.5-flash).
|
|
563
531
|
});
|
|
532
|
+
// Extract user's original text from messages (excluding image parts)
|
|
533
|
+
const userTextParts = messages
|
|
534
|
+
.filter((m) => m.role === "user")
|
|
535
|
+
.flatMap((m) => Array.isArray(m.content)
|
|
536
|
+
? m.content
|
|
537
|
+
.filter((p) => p.type === "text")
|
|
538
|
+
.map((p) => p.text)
|
|
539
|
+
: [typeof m.content === "string" ? m.content : ""])
|
|
540
|
+
.filter(Boolean);
|
|
541
|
+
const userText = userTextParts.join("\n").trim();
|
|
542
|
+
// Pass Gemini's analysis through Claude for structured JSON formatting
|
|
543
|
+
// The system prompt (from Curator) includes JSON_REPORT_PROMPT_SUFFIX
|
|
544
|
+
// which instructs Claude to output {"summary": "...", "details": "..."}
|
|
545
|
+
let formattedContent = videoAnalysisResult;
|
|
546
|
+
let usage = { input: 0, output: 0, total: 0 };
|
|
547
|
+
if (options.systemPrompt) {
|
|
548
|
+
try {
|
|
549
|
+
const formattingPrompt = userText
|
|
550
|
+
? `The user asked: "${userText}"\n\nHere is the video/image analysis result from the visual analysis system:\n\n${videoAnalysisResult}\n\nBased on this analysis, provide your response.`
|
|
551
|
+
: `Here is a video/image analysis result from the visual analysis system:\n\n${videoAnalysisResult}\n\nBased on this analysis, provide your response.`;
|
|
552
|
+
logger.debug("[VideoAnalysis] Formatting via Claude", {
|
|
553
|
+
userTextLength: userText.length,
|
|
554
|
+
analysisLength: videoAnalysisResult.length,
|
|
555
|
+
});
|
|
556
|
+
const formattedResult = await generateText({
|
|
557
|
+
model,
|
|
558
|
+
system: options.systemPrompt,
|
|
559
|
+
messages: [
|
|
560
|
+
{ role: "user", content: formattingPrompt },
|
|
561
|
+
],
|
|
562
|
+
maxTokens: options.maxTokens || 8192,
|
|
563
|
+
temperature: 0.3,
|
|
564
|
+
abortSignal: options.abortSignal,
|
|
565
|
+
experimental_telemetry: this.telemetryHandler?.getTelemetryConfig(options, "generate"),
|
|
566
|
+
});
|
|
567
|
+
formattedContent = formattedResult.text;
|
|
568
|
+
usage = {
|
|
569
|
+
input: formattedResult.usage?.promptTokens || 0,
|
|
570
|
+
output: formattedResult.usage?.completionTokens || 0,
|
|
571
|
+
total: (formattedResult.usage?.promptTokens || 0) +
|
|
572
|
+
(formattedResult.usage?.completionTokens || 0),
|
|
573
|
+
};
|
|
574
|
+
logger.debug("[VideoAnalysis] Claude formatting complete", {
|
|
575
|
+
formattedLength: formattedContent.length,
|
|
576
|
+
usage,
|
|
577
|
+
});
|
|
578
|
+
}
|
|
579
|
+
catch (error) {
|
|
580
|
+
logger.warn("[VideoAnalysis] Claude formatting failed, using raw Gemini output", {
|
|
581
|
+
error: error instanceof Error ? error.message : String(error),
|
|
582
|
+
});
|
|
583
|
+
// formattedContent remains as raw videoAnalysisResult (graceful degradation)
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
const videoResult = {
|
|
587
|
+
content: formattedContent,
|
|
588
|
+
provider: options.provider ?? this.providerName,
|
|
589
|
+
model: this.modelName,
|
|
590
|
+
usage,
|
|
591
|
+
};
|
|
592
|
+
return await this.enhanceResult(videoResult, options, startTime);
|
|
564
593
|
}
|
|
594
|
+
// Compose timeout signal with user-provided abort signal (mirrors stream path)
|
|
595
|
+
const timeoutController = createTimeoutController(options.timeout, this.providerName, "generate");
|
|
596
|
+
const composedSignal = composeAbortSignals(options.abortSignal, timeoutController?.controller.signal);
|
|
597
|
+
const composedOptions = composedSignal
|
|
598
|
+
? { ...options, abortSignal: composedSignal }
|
|
599
|
+
: options;
|
|
600
|
+
let generateResult;
|
|
601
|
+
try {
|
|
602
|
+
generateResult = await this.executeGeneration(model, messages, tools, composedOptions);
|
|
603
|
+
}
|
|
604
|
+
finally {
|
|
605
|
+
timeoutController?.cleanup();
|
|
606
|
+
}
|
|
607
|
+
this.analyzeAIResponse(generateResult);
|
|
608
|
+
this.logGenerationComplete(generateResult);
|
|
609
|
+
const responseTime = Date.now() - startTime;
|
|
610
|
+
await this.recordPerformanceMetrics(generateResult.usage, responseTime);
|
|
611
|
+
const { toolsUsed, toolExecutions } = this.extractToolInformation(generateResult);
|
|
612
|
+
let enhancedResult = this.formatEnhancedResult(generateResult, tools, toolsUsed, toolExecutions, options);
|
|
613
|
+
// ===== TTS MODE 2: AI Response Synthesis (useAiResponse=true) =====
|
|
614
|
+
// Synthesize AI-generated response after generation completes
|
|
615
|
+
if (options.tts?.enabled && options.tts?.useAiResponse) {
|
|
616
|
+
const aiResponse = enhancedResult.content;
|
|
617
|
+
const provider = options.provider ?? this.providerName;
|
|
618
|
+
// Validate AI response and provider before synthesis
|
|
619
|
+
if (aiResponse && provider) {
|
|
620
|
+
try {
|
|
621
|
+
const ttsResult = await TTSProcessor.synthesize(aiResponse, provider, options.tts);
|
|
622
|
+
// Add audio to enhanced result (TTSProcessor already includes latency in metadata)
|
|
623
|
+
enhancedResult = {
|
|
624
|
+
...enhancedResult,
|
|
625
|
+
audio: ttsResult,
|
|
626
|
+
};
|
|
627
|
+
}
|
|
628
|
+
catch (ttsError) {
|
|
629
|
+
// Log TTS error but continue with text-only result
|
|
630
|
+
logger.error(`TTS synthesis failed in Mode 2 (AI response synthesis):`, ttsError);
|
|
631
|
+
// enhancedResult remains unchanged (no audio field added)
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
else {
|
|
635
|
+
logger.warn(`TTS synthesis skipped despite being enabled`, {
|
|
636
|
+
provider: this.providerName,
|
|
637
|
+
hasAiResponse: !!aiResponse,
|
|
638
|
+
aiResponseLength: aiResponse?.length ?? 0,
|
|
639
|
+
hasProvider: !!provider,
|
|
640
|
+
ttsConfig: {
|
|
641
|
+
enabled: options.tts?.enabled,
|
|
642
|
+
useAiResponse: options.tts?.useAiResponse,
|
|
643
|
+
},
|
|
644
|
+
reason: !aiResponse
|
|
645
|
+
? "AI response is empty or undefined"
|
|
646
|
+
: "Provider is missing",
|
|
647
|
+
});
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
// Set token usage on span from the result
|
|
651
|
+
if (enhancedResult?.usage) {
|
|
652
|
+
span.setAttribute("gen_ai.usage.input_tokens", enhancedResult.usage.input || 0);
|
|
653
|
+
span.setAttribute("gen_ai.usage.output_tokens", enhancedResult.usage.output || 0);
|
|
654
|
+
// Cost on span so users can query "what did this trace cost?"
|
|
655
|
+
const cost = calculateCost(this.providerName, this.modelName, {
|
|
656
|
+
input: enhancedResult.usage.input || 0,
|
|
657
|
+
output: enhancedResult.usage.output || 0,
|
|
658
|
+
total: enhancedResult.usage.total || 0,
|
|
659
|
+
});
|
|
660
|
+
span.setAttribute("neurolink.cost", cost ?? 0);
|
|
661
|
+
}
|
|
662
|
+
if (enhancedResult?.finishReason) {
|
|
663
|
+
span.setAttribute("gen_ai.response.finish_reason", enhancedResult.finishReason);
|
|
664
|
+
}
|
|
665
|
+
span.setStatus({ code: SpanStatusCode.OK });
|
|
666
|
+
return await this.enhanceResult(enhancedResult, options, startTime);
|
|
565
667
|
}
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
if (isAbortError(error)) {
|
|
571
|
-
logger.info(`Generate aborted for ${this.providerName}`, {
|
|
572
|
-
error: error instanceof Error ? error.message : String(error),
|
|
668
|
+
catch (error) {
|
|
669
|
+
span.setStatus({
|
|
670
|
+
code: SpanStatusCode.ERROR,
|
|
671
|
+
message: error instanceof Error ? error.message : String(error),
|
|
573
672
|
});
|
|
673
|
+
// Abort errors are expected when a generation is cancelled — log at info, not error
|
|
674
|
+
if (isAbortError(error)) {
|
|
675
|
+
logger.info(`Generate aborted for ${this.providerName}`, {
|
|
676
|
+
error: error instanceof Error ? error.message : String(error),
|
|
677
|
+
});
|
|
678
|
+
}
|
|
679
|
+
else {
|
|
680
|
+
logger.error(`Generate failed for ${this.providerName}:`, error);
|
|
681
|
+
}
|
|
682
|
+
throw this.handleProviderError(error);
|
|
574
683
|
}
|
|
575
|
-
|
|
576
|
-
|
|
684
|
+
finally {
|
|
685
|
+
span.end();
|
|
577
686
|
}
|
|
578
|
-
|
|
579
|
-
}
|
|
687
|
+
});
|
|
580
688
|
}
|
|
581
689
|
/**
|
|
582
690
|
* Alias for generate method - implements AIProvider interface
|
|
@@ -857,8 +965,6 @@ export class BaseProvider {
|
|
|
857
965
|
* @param functionTag - Function name for logging
|
|
858
966
|
*/
|
|
859
967
|
setupToolExecutor(sdk, functionTag) {
|
|
860
|
-
this.customTools = sdk.customTools;
|
|
861
|
-
this.toolExecutor = sdk.executeTool;
|
|
862
968
|
this.toolsManager.setupToolExecutor(sdk, functionTag);
|
|
863
969
|
}
|
|
864
970
|
// ===================
|