@smythos/sre 1.7.18 → 1.7.40

package/src/subsystems/LLMManager/LLM.service/connectors/GoogleAI.class.ts

@@ -139,6 +139,7 @@ export class GoogleAIConnector extends LLMConnector {
                             ? toolCall.functionCall?.args
                             : JSON.stringify(toolCall.functionCall?.args ?? {}),
                     role: TLLMMessageRole.Assistant,
+                    thoughtSignature: (toolCall as any).thoughtSignature, // Preserve Google AI's reasoning context
                 }));
                 useTool = true;
             }
@@ -185,9 +186,11 @@ export class GoogleAIConnector extends LLMConnector {
             (async () => {
                 try {
                     for await (const chunk of stream) {
+                        emitter.emit(TLLMEvent.Data, chunk);
+
                         const chunkText = chunk.text ?? '';
                         if (chunkText) {
-                            emitter.emit('content', chunkText);
+                            emitter.emit(TLLMEvent.Content, chunkText);
                         }
 
                         const toolCalls = chunk.candidates?.[0]?.content?.parts?.filter((part) => part.functionCall);
@@ -202,6 +205,7 @@ export class GoogleAIConnector extends LLMConnector {
                                         ? toolCall.functionCall?.args
                                         : JSON.stringify(toolCall.functionCall?.args ?? {}),
                                 role: TLLMMessageRole.Assistant,
+                                thoughtSignature: (toolCall as any).thoughtSignature, // Preserve Google AI's reasoning context
                             }));
                             emitter.emit(TLLMEvent.ToolInfo, toolsData);
                         }
@@ -211,21 +215,28 @@ export class GoogleAIConnector extends LLMConnector {
                         }
                     }
 
+                    const finishReason = 'stop'; // GoogleAI doesn't provide finishReason in streaming
+                    const reportedUsage: any[] = [];
+
                     if (usage) {
-                        this.reportUsage(usage, {
+                        const reported = this.reportUsage(usage, {
                             modelEntryName: context.modelEntryName,
                             keySource: context.isUserKey ? APIKeySource.User : APIKeySource.Smyth,
                             agentId: context.agentId,
                             teamId: context.teamId,
                         });
+                        reportedUsage.push(reported);
                     }
 
+                    // Note: GoogleAI stream doesn't provide explicit finish reasons
+                    // If we had a non-stop finish reason, we would emit Interrupted here
+
                     setTimeout(() => {
-                        emitter.emit('end', toolsData);
+                        emitter.emit(TLLMEvent.End, toolsData, reportedUsage, finishReason);
                     }, 100);
                 } catch (error) {
                     logger.error(`streamRequest ${this.name}`, error, acRequest.candidate);
-                    emitter.emit('error', error);
+                    emitter.emit(TLLMEvent.Error, error);
                 }
             })();
 
@@ -426,6 +437,13 @@ export class GoogleAIConnector extends LLMConnector {
         if (params.stopSequences?.length) config.stopSequences = params.stopSequences;
         if (responseMimeType) config.responseMimeType = responseMimeType;
 
+        // #region Gemini 3 specific fields
+        const isGemini3Model = params.modelEntryName?.includes('gemini-3');
+
+        if (isGemini3Model) {
+            if (params?.reasoningEffort) config.thinkingConfig = { thinkingLevel: params.reasoningEffort };
+        }
+
         if (systemInstruction) body.systemInstruction = systemInstruction;
         if (Object.keys(config).length > 0) {
             body.generationConfig = config;
@@ -505,36 +523,76 @@ export class GoogleAIConnector extends LLMConnector {
     ) {
         // SmythOS (built-in) models have a prefix, so we need to remove it to get the model name
         const modelName = metadata.modelEntryName.replace(BUILT_IN_MODEL_PREFIX, '');
-        let tier = '';
+
+        // Initially, all input tokens – such as text, audio, image, video, document, etc. – were included in promptTokenCount.
+        let inputTokens = usage?.promptTokenCount || 0;
+
+        // The pricing is the same for output and thinking tokens, so we can add them together.
+        const outputTokens = (usage?.candidatesTokenCount || 0) + (usage?.thoughtsTokenCount || 0);
+
+        // If cached input tokens are available, we need to subtract them from the input tokens.
+        let cachedInputTokens = usage?.cachedContentTokenCount || 0;
+
+        if (cachedInputTokens) {
+            inputTokens = inputTokens - cachedInputTokens;
+        }
+
+        // #region Find matching model and set tier based on threshold
         const tierThresholds = {
             'gemini-1.5-pro': 128_000,
             'gemini-2.5-pro': 200_000,
+            'gemini-3-pro': 200_000,
         };
 
-        const textInputTokens =
-            usage?.['promptTokensDetails']?.find((detail) => detail.modality === 'TEXT')?.tokenCount || usage?.promptTokenCount || 0;
-        const audioInputTokens = usage?.['promptTokensDetails']?.find((detail) => detail.modality === 'AUDIO')?.tokenCount || 0;
+        let inTier = '';
+        let outTier = '';
+        let crTier = '';
 
-        // Find matching model and set tier based on threshold
         const modelWithTier = Object.keys(tierThresholds).find((model) => modelName.includes(model));
         if (modelWithTier) {
-            tier = textInputTokens < tierThresholds[modelWithTier] ? 'tier1' : 'tier2';
+            inTier = inputTokens <= tierThresholds[modelWithTier] ? 'tier1' : 'tier2';
+            outTier = outputTokens <= tierThresholds[modelWithTier] ? 'tier1' : 'tier2';
+            crTier = cachedInputTokens <= tierThresholds[modelWithTier] ? 'tier1' : 'tier2';
         }
+        // #endregion
+
+        // #region Calculate audio input tokens
+        // Since Gemini 2.5 Flash has a different pricing model for audio input tokens, we need to report audio input tokens separately.
+        let audioInputTokens = 0;
+        let cachedAudioInputTokens = 0;
+        const isFlashModel = ['gemini-2.5-flash'].includes(modelName);
+
+        if (isFlashModel) {
+            // There is no concept of different pricing for Flash models based on token tiers (e.g., less than or greater than 200k),
+            // so we don't need to provide tier information for audio input tokens.
+            audioInputTokens = usage?.promptTokensDetails?.find((detail) => detail.modality === 'AUDIO')?.tokenCount || 0;
+
+            // subtract the audio cached input tokens from the audio input tokens and total cached input tokens.
+            cachedAudioInputTokens = usage?.cacheTokensDetails?.find((detail) => detail.modality === 'AUDIO')?.tokenCount || 0;
+            if (cachedAudioInputTokens) {
+                audioInputTokens = audioInputTokens - cachedAudioInputTokens;
+                cachedInputTokens = cachedInputTokens - cachedAudioInputTokens;
+            }
 
+            inputTokens = inputTokens - audioInputTokens;
+        }
         // #endregion
 
         const usageData = {
             sourceId: `llm:${modelName}`,
-            input_tokens: textInputTokens,
-            output_tokens: usage?.candidatesTokenCount || 0,
+            input_tokens: inputTokens,
+            output_tokens: outputTokens,
             input_tokens_audio: audioInputTokens,
-            input_tokens_cache_read: usage?.cachedContentTokenCount || 0,
+            input_tokens_cache_read: cachedInputTokens,
+            input_tokens_cache_read_audio: cachedAudioInputTokens,
             input_tokens_cache_write: 0,
-            reasoning_tokens: usage?.thoughtsTokenCount,
+            // reasoning_tokens: usage?.thoughtsTokenCount, // * reasoning tokens are included in the output tokens.
             keySource: metadata.keySource,
             agentId: metadata.agentId,
             teamId: metadata.teamId,
-            tier,
+            inTier,
+            outTier,
+            crTier,
         };
         SystemEvents.emit('USAGE:LLM', usageData);
 
@@ -665,12 +723,17 @@ export class GoogleAIConnector extends LLMConnector {
                     }
 
                     if (part.functionCall) {
-                        content.push({
+                        const functionCallPart: any = {
                             functionCall: {
                                 name: part.functionCall.name,
                                 args: parseFunctionArgs(part.functionCall.args),
                             },
-                        });
+                        };
+                        // Preserve thoughtSignature if present for Google AI reasoning context
+                        if ((part as any).thoughtSignature) {
+                            functionCallPart.thoughtSignature = (part as any).thoughtSignature;
+                        }
+                        content.push(functionCallPart);
                         continue;
                     }
 
@@ -699,12 +762,17 @@ export class GoogleAIConnector extends LLMConnector {
             const hasFunctionCall = content.some((part) => part.functionCall);
             if (!hasFunctionCall && toolsData.length > 0) {
                 toolsData.forEach((toolCall) => {
-                    content.push({
+                    const functionCallPart: any = {
                         functionCall: {
                             name: toolCall.name,
                             args: parseFunctionArgs(toolCall.arguments),
                         },
-                    });
+                    };
+                    // Preserve thoughtSignature if present for Google AI reasoning context
+                    if (toolCall.thoughtSignature) {
+                        functionCallPart.thoughtSignature = toolCall.thoughtSignature;
+                    }
+                    content.push(functionCallPart);
                 });
             }
 
@@ -811,6 +879,10 @@ export class GoogleAIConnector extends LLMConnector {
                             name: part.functionCall.name,
                             args: parseFunctionArgs(part.functionCall.args),
                         };
+                        // Preserve thoughtSignature if present for Google AI reasoning context
+                        if ((part as any).thoughtSignature) {
+                            normalizedPart.thoughtSignature = (part as any).thoughtSignature;
+                        }
                     }
 
                     if (part.functionResponse) {
@@ -839,12 +911,17 @@ export class GoogleAIConnector extends LLMConnector {
                             pushTextPart(normalizedParts, contentPart.text);
                         } else if ('functionCall' in contentPart && (contentPart as any).functionCall) {
                             const functionCallPart = (contentPart as any).functionCall;
-                            normalizedParts.push({
+                            const normalizedFunctionCall: any = {
                                 functionCall: {
                                     name: functionCallPart.name,
                                     args: parseFunctionArgs(functionCallPart.args),
                                 },
-                            });
+                            };
+                            // Preserve thoughtSignature if present for Google AI reasoning context
+                            if ((contentPart as any).thoughtSignature) {
+                                normalizedFunctionCall.thoughtSignature = (contentPart as any).thoughtSignature;
+                            }
+                            normalizedParts.push(normalizedFunctionCall);
                         } else if ('functionResponse' in contentPart && (contentPart as any).functionResponse) {
                             const functionResponsePart = (contentPart as any).functionResponse;
                             normalizedParts.push({
@@ -882,12 +959,17 @@ export class GoogleAIConnector extends LLMConnector {
                 for (const toolCall of message.tool_calls) {
                     if (!toolCall?.function?.name) continue;
 
-                    normalizedParts.push({
+                    const normalizedFunctionCall: any = {
                         functionCall: {
                             name: toolCall.function.name,
                             args: parseFunctionArgs(toolCall.function.arguments),
                         },
-                    });
+                    };
+                    // Preserve thoughtSignature if present for Google AI reasoning context
+                    if ((toolCall as any).thoughtSignature) {
+                        normalizedFunctionCall.thoughtSignature = (toolCall as any).thoughtSignature;
+                    }
+                    normalizedParts.push(normalizedFunctionCall);
                 }
             }
 

package/src/subsystems/LLMManager/LLM.service/connectors/Groq.class.ts

@@ -108,6 +108,7 @@ export class GroqConnector extends LLMConnector {
             const stream = await groq.chat.completions.create({ ...body, stream: true, stream_options: { include_usage: true } });
 
             let toolsData: ToolData[] = [];
+            let finishReason = 'stop';
 
             (async () => {
                 for await (const chunk of stream as any) {
@@ -117,10 +118,10 @@ export class GroqConnector extends LLMConnector {
                     if (usage) {
                         usage_data.push(usage);
                     }
-                    emitter.emit('data', delta);
+                    emitter.emit(TLLMEvent.Data, delta);
 
                     if (delta?.content) {
-                        emitter.emit('content', delta.content);
+                        emitter.emit(TLLMEvent.Content, delta.content);
                     }
 
                     if (delta?.tool_calls) {
@@ -139,24 +140,35 @@ export class GroqConnector extends LLMConnector {
                             }
                         });
                     }
+
+                    // Capture finish reason
+                    if (chunk.choices[0]?.finish_reason) {
+                        finishReason = chunk.choices[0].finish_reason;
+                    }
                 }
 
                 if (toolsData.length > 0) {
                     emitter.emit(TLLMEvent.ToolInfo, toolsData);
                 }
 
+                const reportedUsage: any[] = [];
                 usage_data.forEach((usage) => {
-                    // probably we can acc them and send them as one event
-                    this.reportUsage(usage, {
+                    const reported = this.reportUsage(usage, {
                         modelEntryName: context.modelEntryName,
                         keySource: context.isUserKey ? APIKeySource.User : APIKeySource.Smyth,
                         agentId: context.agentId,
                         teamId: context.teamId,
                     });
+                    reportedUsage.push(reported);
                 });
 
+                // Emit interrupted event if finishReason is not 'stop'
+                if (finishReason !== 'stop') {
+                    emitter.emit(TLLMEvent.Interrupted, finishReason);
+                }
+
                 setTimeout(() => {
-                    emitter.emit('end', toolsData);
+                    emitter.emit(TLLMEvent.End, toolsData, reportedUsage, finishReason);
                 }, 100);
             })();
 

package/src/subsystems/LLMManager/LLM.service/connectors/Ollama.class.ts

@@ -138,14 +138,17 @@ export class OllamaConnector extends LLMConnector {
 
             let toolsData: ToolData[] = [];
             let fullContent = '';
+            let finishReason = 'stop';
 
             (async () => {
                 for await (const chunk of stream) {
+                    emitter.emit(TLLMEvent.Data, chunk);
+
                     // Emit content deltas
                     if (chunk.message?.content) {
                         const content = chunk.message.content;
                         fullContent += content;
-                        emitter.emit('content', content);
+                        emitter.emit(TLLMEvent.Content, content);
                     }
 
                     // Handle tool calls accumulation
@@ -181,6 +184,11 @@ export class OllamaConnector extends LLMConnector {
                         };
                         usage_data.push(usage);
                     }
+
+                    // Capture finish reason from Ollama's done_reason
+                    if (chunk.done_reason) {
+                        finishReason = chunk.done_reason;
+                    }
                 }
 
                 // Emit tool info if tools were requested
@@ -189,18 +197,25 @@ export class OllamaConnector extends LLMConnector {
                 }
 
                 // Report usage
+                const reportedUsage: any[] = [];
                 usage_data.forEach((usage) => {
-                    this.reportUsage(usage, {
+                    const reported = this.reportUsage(usage, {
                         modelEntryName: context.modelEntryName,
                         keySource: context.isUserKey ? APIKeySource.User : APIKeySource.Smyth,
                         agentId: context.agentId,
                         teamId: context.teamId,
                     });
+                    reportedUsage.push(reported);
                 });
 
+                // Emit interrupted event if finishReason is not 'stop'
+                if (finishReason !== 'stop') {
+                    emitter.emit(TLLMEvent.Interrupted, finishReason);
+                }
+
                 // Final end event
                 setTimeout(() => {
-                    emitter.emit('end', toolsData);
+                    emitter.emit(TLLMEvent.End, toolsData, reportedUsage, finishReason);
                 }, 100);
             })();
 

package/src/subsystems/LLMManager/LLM.service/connectors/Perplexity.class.ts

@@ -14,6 +14,7 @@ import {
     TLLMChatResponse,
     ILLMRequestContext,
     TLLMPreparedParams,
+    TLLMEvent,
 } from '@sre/types/LLM.types';
 import { LLMHelper } from '@sre/LLMManager/LLM.helper';
 
@@ -97,6 +98,8 @@ export class PerplexityConnector extends LLMConnector {
         //fallback to chatRequest
         const emitter = new EventEmitter();
 
+        // TODO: need to implement proper streaming for Perplexity
+
         setTimeout(() => {
             try {
                 logger.debug(`streamRequest ${this.name}`, acRequest.candidate);
@@ -105,17 +108,23 @@ export class PerplexityConnector extends LLMConnector {
                         const finishReason = respose.finishReason;
                         const usage = respose.usage;
 
-                        emitter.emit('interrupted', finishReason);
-                        emitter.emit('content', respose.content);
-                        emitter.emit('end', undefined, usage, finishReason);
+                        emitter.emit(TLLMEvent.Data, respose);
+                        emitter.emit(TLLMEvent.Content, respose.content);
+
+                        // Only emit Interrupted if finishReason is not 'stop'
+                        if (finishReason !== 'stop') {
+                            emitter.emit(TLLMEvent.Interrupted, finishReason);
+                        }
+
+                        emitter.emit(TLLMEvent.End, [], [usage], finishReason);
                     })
                     .catch((error) => {
-                        emitter.emit('error', error.message || error.toString());
+                        emitter.emit(TLLMEvent.Error, error.message || error.toString());
                     });
                 //emitter.emit('finishReason', respose.finishReason);
             } catch (error) {
                 logger.error(`streamRequest ${this.name}`, error, acRequest.candidate);
-                emitter.emit('error', error.message || error.toString());
+                emitter.emit(TLLMEvent.Error, error.message || error.toString());
             }
         }, 100);
 

package/src/subsystems/LLMManager/LLM.service/connectors/VertexAI.class.ts

@@ -138,12 +138,14 @@ export class VertexAIConnector extends LLMConnector {
                 for await (const chunk of streamResult.stream) {
                     const chunkText = chunk.candidates?.[0]?.content?.parts?.[0]?.text || '';
                     if (chunkText) {
-                        emitter.emit('content', chunkText);
+                        emitter.emit(TLLMEvent.Content, chunkText);
                     }
                 }
 
                 const aggregatedResponse = await streamResult.response;
 
+                emitter.emit(TLLMEvent.Data, aggregatedResponse);
+
                 // Check for function calls in the final response (like Anthropic does)
                 const functionCalls = aggregatedResponse.candidates?.[0]?.content?.parts?.filter((part) => part.functionCall);
                 if (functionCalls && functionCalls.length > 0) {
@@ -176,15 +178,15 @@ export class VertexAIConnector extends LLMConnector {
                 const finishReason = (aggregatedResponse.candidates?.[0]?.finishReason || 'stop').toLowerCase();
 
                 if (finishReason !== 'stop') {
-                    emitter.emit('interrupted', finishReason);
+                    emitter.emit(TLLMEvent.Interrupted, finishReason);
                 }
 
                 setTimeout(() => {
-                    emitter.emit('end', toolsData, usageData, finishReason);
+                    emitter.emit(TLLMEvent.End, toolsData, usageData, finishReason);
                 }, 100);
             } catch (error) {
                 logger.error(`streamRequest ${this.name}`, error, acRequest.candidate);
-                emitter.emit('error', error);
+                emitter.emit(TLLMEvent.Error, error);
             }
         }, 100);
 

package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/ChatCompletionsApiInterface.ts

@@ -73,7 +73,7 @@ export class ChatCompletionsApiInterface extends OpenAIApiInterface {
                 // Step 3: Emit final events
                 this.emitFinalEvents(emitter, finalToolsData, reportedUsage, finishReason);
             } catch (error) {
-                emitter.emit('error', error);
+                emitter.emit(TLLMEvent.Error, error);
             }
         })();
 
@@ -267,11 +267,11 @@ export class ChatCompletionsApiInterface extends OpenAIApiInterface {
             }
 
             // Emit data event for delta
-            emitter.emit('data', delta);
+            emitter.emit(TLLMEvent.Data, delta);
 
             // Handle content deltas
             if (!delta?.tool_calls && delta?.content) {
-                emitter.emit('content', delta?.content, delta?.role);
+                emitter.emit(TLLMEvent.Content, delta?.content, delta?.role);
             }
 
             // Handle tool calls
@@ -350,12 +350,12 @@ export class ChatCompletionsApiInterface extends OpenAIApiInterface {
 
         // Emit interrupted event if finishReason is not 'stop'
         if (finishReason !== 'stop') {
-            emitter.emit('interrupted', finishReason);
+            emitter.emit(TLLMEvent.Interrupted, finishReason);
         }
 
         // Emit end event with setImmediate to ensure proper event ordering
         setImmediate(() => {
-            emitter.emit('end', toolsData, reportedUsage, finishReason);
+            emitter.emit(TLLMEvent.End, toolsData, reportedUsage, finishReason);
         });
     }
 

package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/ResponsesApiInterface.ts

@@ -119,7 +119,7 @@ export class ResponsesApiInterface extends OpenAIApiInterface {
                 // Step 3: Emit final events
                 this.emitFinalEvents(emitter, finalToolsData, reportedUsage, finishReason);
             } catch (error) {
-                emitter.emit('error', error);
+                emitter.emit(TLLMEvent.Error, error);
             }
         })();
 
@@ -362,8 +362,10 @@ export class ResponsesApiInterface extends OpenAIApiInterface {
                     role: 'assistant',
                     content: part.delta,
                 };
-                emitter.emit('data', deltaMsg);
-                emitter.emit('content', part.delta, 'assistant');
+
+                // TODO: we have inconsistency for data event with chat completions API, we need to check and fix it
+                emitter.emit(TLLMEvent.Data, deltaMsg);
+                emitter.emit(TLLMEvent.Content, part.delta, 'assistant');
             }
         } catch (error) {
             console.warn('Error handling output text delta:', error);
@@ -412,6 +414,7 @@ export class ResponsesApiInterface extends OpenAIApiInterface {
                     }
 
                     if (addingNew) {
+                        // TODO: Check whether this event is being used.
                         emitter.emit('tool_call_started', {
                             id: callId,
                             name: functionName || '',
@@ -458,6 +461,7 @@ export class ResponsesApiInterface extends OpenAIApiInterface {
                 }
 
                 const entry = existingIndex === -1 ? updated[finalIndex] : updated[finalIndex];
+                // TODO: Check whether this event is being used.
                 emitter.emit('tool_call_progress', {
                     id: entry.callId || itemId,
                     name: entry.name,
@@ -489,6 +493,7 @@ export class ResponsesApiInterface extends OpenAIApiInterface {
                     const updated = toolsData.map((t, idx) => (idx === toolIndex ? { ...t, arguments: finalArguments } : t));
 
                     const updatedEntry = updated[toolIndex];
+                    // TODO: Check whether this event is being used.
                     emitter.emit('tool_call_completed', {
                         id: updatedEntry.callId || itemId,
                         name: updatedEntry.name,

package/src/subsystems/LLMManager/LLM.service/connectors/xAI.class.ts

@@ -14,6 +14,7 @@ import {
     ILLMRequestFuncParams,
     TLLMChatResponse,
     ILLMRequestContext,
+    TLLMEvent,
 } from '@sre/types/LLM.types';
 import { LLMHelper } from '@sre/LLMManager/LLM.helper';
 
@@ -202,10 +203,10 @@ export class xAIConnector extends LLMConnector {
                             }
 
                             if (delta) {
-                                emitter.emit('data', delta);
+                                emitter.emit(TLLMEvent.Data, delta);
 
                                 if (delta.content) {
-                                    emitter.emit('content', delta.content, delta.role);
+                                    emitter.emit(TLLMEvent.Content, delta.content, delta.role);
                                 }
 
                                 if (delta.tool_calls) {
@@ -238,11 +239,11 @@ export class xAIConnector extends LLMConnector {
                 if (citations && citations.length > 0) {
                     const citationsText = '\n\n**Sources:**\n' + citations.map((url, index) => `${index + 1}. ${url}`).join('\n');
 
-                    emitter.emit('content', citationsText, 'assistant');
+                    emitter.emit(TLLMEvent.Content, citationsText, 'assistant');
                 }
 
                 if (toolsData.length > 0) {
-                    emitter.emit('toolInfo', toolsData);
+                    emitter.emit(TLLMEvent.ToolInfo, toolsData);
                 }
 
                 // Report usage if available
@@ -257,20 +258,20 @@ export class xAIConnector extends LLMConnector {
                 }
 
                 if (finishReason !== 'stop') {
-                    emitter.emit('interrupted', finishReason);
+                    emitter.emit(TLLMEvent.Interrupted, finishReason);
                 }
 
                 setTimeout(() => {
-                    emitter.emit('end', toolsData, reportedUsage, finishReason);
+                    emitter.emit(TLLMEvent.End, toolsData, reportedUsage, finishReason);
                 }, 100);
             });
 
             response.data.on('error', (error) => {
-                emitter.emit('error', error);
+                emitter.emit(TLLMEvent.Error, error);
             });
         } catch (error) {
             logger.error(`streamRequest ${this.name}`, error, acRequest.candidate);
-            emitter.emit('error', error);
+            emitter.emit(TLLMEvent.Error, error);
         }
 
         return emitter;