@smythos/sre 1.5.0 → 1.5.2

package/src/subsystems/LLMManager/LLM.service/connectors/Perplexity.class.ts

@@ -0,0 +1,255 @@
+import EventEmitter from 'events';
+import axios, { AxiosInstance } from 'axios';
+
+import { JSON_RESPONSE_INSTRUCTION, BUILT_IN_MODEL_PREFIX } from '@sre/constants';
+
+import {
+    TLLMParams,
+    ToolData,
+    TLLMMessageBlock,
+    TLLMToolResultMessageBlock,
+    TLLMMessageRole,
+    APIKeySource,
+    BasicCredentials,
+    ILLMRequestFuncParams,
+    TLLMChatResponse,
+    TLLMConnectorParams,
+    ILLMRequestContext,
+} from '@sre/types/LLM.types';
+import { LLMHelper } from '@sre/LLMManager/LLM.helper';
+
+import { LLMConnector } from '../LLMConnector';
+import { SystemEvents } from '@sre/Core/SystemEvents';
+
+type ChatCompletionParams = {
+    model: string;
+    messages: any[];
+    max_tokens?: number;
+    temperature?: number;
+    top_p?: number;
+    top_k?: number;
+    frequency_penalty?: number;
+    presence_penalty?: number;
+    response_format?: { type: string };
+};
+type TUsage = {
+    prompt_tokens: number;
+    completion_tokens: number;
+    total_tokens: number;
+    prompt_tokens_details?: { cached_tokens?: number };
+    reasoning_tokens?: number;
+};
+
+// TODO [Forhad]: Need to adjust some type definitions
+
+export class PerplexityConnector extends LLMConnector {
+    public name = 'LLM:Perplexity';
+
+    private async getClient(params: ILLMRequestContext): Promise<AxiosInstance> {
+        const apiKey = (params.credentials as BasicCredentials)?.apiKey;
+
+        if (!apiKey) throw new Error('Please provide an API key for Perplexity');
+
+        return axios.create({
+            baseURL: params?.modelInfo?.baseURL,
+            headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' },
+        });
+    }
+
+    protected async request({ acRequest, body, context }: ILLMRequestFuncParams): Promise<TLLMChatResponse> {
+        try {
+            const perplexity = await this.getClient(context);
+            const response = await perplexity.post('/chat/completions', body);
+
+            const content = response?.data?.choices?.[0]?.message.content;
+            const finishReason = response?.data?.choices?.[0]?.finish_reason;
+            const usage = response?.data?.usage as any;
+
+            this.reportUsage(usage, {
+                modelEntryName: context.modelEntryName,
+                keySource: context.isUserKey ? APIKeySource.User : APIKeySource.Smyth,
+                agentId: context.agentId,
+                teamId: context.teamId,
+            });
+
+            return {
+                content,
+                finishReason,
+                useTool: false,
+                toolsData: [],
+                message: { content, role: 'assistant' },
+                usage,
+            };
+        } catch (error) {
+            throw error;
+        }
+    }
+
+    protected async streamRequest({ acRequest, body, context }: ILLMRequestFuncParams): Promise<EventEmitter> {
+        //throw new Error('Multimodal request is not supported for Perplexity.');
+        //fallback to chatRequest
+        const emitter = new EventEmitter();
+
+        setTimeout(() => {
+            try {
+                this.request({ acRequest, body, context })
+                    .then((respose) => {
+                        const finishReason = respose.finishReason;
+                        const usage = respose.usage;
+
+                        emitter.emit('interrupted', finishReason);
+                        emitter.emit('content', respose.content);
+                        emitter.emit('end', undefined, usage, finishReason);
+                    })
+                    .catch((error) => {
+                        emitter.emit('error', error.message || error.toString());
+                    });
+                //emitter.emit('finishReason', respose.finishReason);
+            } catch (error) {
+                emitter.emit('error', error.message || error.toString());
+            }
+        }, 100);
+
+        return emitter;
+    }
+
+    protected async webSearchRequest({ acRequest, body, context }: ILLMRequestFuncParams): Promise<EventEmitter> {
+        throw new Error('Web search is not supported for Perplexity');
+    }
+
+    protected async reqBodyAdapter(params: TLLMParams): Promise<ChatCompletionParams> {
+        const messages = params?.messages || [];
+
+        //#region Handle JSON response format
+        // TODO: For now attach JSON response instruction, Perplexity has option to provide response_format as parameter. We'll check it later
+        const responseFormat = params?.responseFormat || '';
+        if (responseFormat === 'json') {
+            // We assume that the system message is first item in messages array
+            if (messages?.[0]?.role === TLLMMessageRole.System) {
+                messages[0].content += JSON_RESPONSE_INSTRUCTION;
+            } else {
+                messages.unshift({ role: TLLMMessageRole.System, content: JSON_RESPONSE_INSTRUCTION });
+            }
+
+            delete params.responseFormat;
+        }
+        //#endregion Handle JSON response format
+
+        const body: ChatCompletionParams = {
+            model: params.model as string,
+            messages,
+        };
+
+        if (params?.maxTokens !== undefined) body.max_tokens = params.maxTokens;
+        if (params?.temperature !== undefined) body.temperature = params.temperature;
+        if (params?.topP !== undefined) body.top_p = params.topP;
+        if (params?.topK !== undefined) body.top_k = params.topK;
+        if (params?.frequencyPenalty) body.frequency_penalty = params.frequencyPenalty;
+        if (params?.presencePenalty !== undefined) body.presence_penalty = params.presencePenalty;
+
+        if (params.responseFormat) {
+            body.response_format = params.responseFormat;
+        }
+
+        return body;
+    }
+
+    protected reportUsage(usage: TUsage, metadata: { modelEntryName: string; keySource: APIKeySource; agentId: string; teamId: string }) {
+        // SmythOS (built-in) models have a prefix, so we need to remove it to get the model name
+        const modelName = metadata.modelEntryName.replace(BUILT_IN_MODEL_PREFIX, '');
+
+        const usageData = {
+            sourceId: `llm:${modelName}`,
+            input_tokens: usage?.prompt_tokens - (usage?.prompt_tokens_details?.cached_tokens || 0),
+            output_tokens: usage?.completion_tokens,
+            input_tokens_cache_write: 0,
+            input_tokens_cache_read: usage?.prompt_tokens_details?.cached_tokens || 0,
+            reasoning_tokens: usage?.reasoning_tokens || 0,
+            keySource: metadata.keySource,
+            agentId: metadata.agentId,
+            teamId: metadata.teamId,
+        };
+        SystemEvents.emit('USAGE:LLM', usageData);
+
+        return usageData;
+    }
+
+    public formatToolsConfig({ type = 'function', toolDefinitions, toolChoice = 'auto' }) {
+        let tools: any[] = [];
+
+        if (type === 'function') {
+            tools = toolDefinitions.map((tool) => {
+                const { name, description, properties, requiredFields } = tool;
+
+                return {
+                    type: 'function',
+                    function: {
+                        name,
+                        description,
+                        parameters: {
+                            type: 'object',
+                            properties,
+                            required: requiredFields,
+                        },
+                    },
+                };
+            });
+        }
+
+        return tools?.length > 0 ? { tools, tool_choice: toolChoice || 'auto' } : {};
+    }
+
+    public transformToolMessageBlocks({
+        messageBlock,
+        toolsData,
+    }: {
+        messageBlock: TLLMMessageBlock;
+        toolsData: ToolData[];
+    }): TLLMToolResultMessageBlock[] {
+        const messageBlocks: TLLMToolResultMessageBlock[] = [];
+
+        if (messageBlock) {
+            const transformedMessageBlock = {
+                ...messageBlock,
+                content: typeof messageBlock.content === 'object' ? JSON.stringify(messageBlock.content) : messageBlock.content,
+            };
+            if (transformedMessageBlock.tool_calls) {
+                for (let toolCall of transformedMessageBlock.tool_calls) {
+                    toolCall.function.arguments =
+                        typeof toolCall.function.arguments === 'object' ? JSON.stringify(toolCall.function.arguments) : toolCall.function.arguments;
+                }
+            }
+            messageBlocks.push(transformedMessageBlock);
+        }
+
+        const transformedToolsData = toolsData.map((toolData) => ({
+            tool_call_id: toolData.id,
+            role: TLLMMessageRole.Tool, // toolData.role as TLLMMessageRole, //should always be 'tool' for OpenAI
+            name: toolData.name,
+            content: typeof toolData.result === 'string' ? toolData.result : JSON.stringify(toolData.result), // Ensure content is a string
+        }));
+
+        return [...messageBlocks, ...transformedToolsData];
+    }
+
+    public getConsistentMessages(messages) {
+        const _messages = LLMHelper.removeDuplicateUserMessages(messages);
+
+        return _messages.map((message) => {
+            const _message = { ...message };
+            let textContent = '';
+
+            if (message?.parts) {
+                textContent = message.parts.map((textBlock) => textBlock?.text || '').join(' ');
+            } else if (Array.isArray(message?.content)) {
+                textContent = message.content.map((textBlock) => textBlock?.text || '').join(' ');
+            } else if (message?.content) {
+                textContent = message.content;
+            }
+
+            _message.content = textContent;
+
+            return _message;
+        });
+    }
+}

package/src/subsystems/LLMManager/LLM.service/connectors/VertexAI.class.ts

@@ -0,0 +1,193 @@
+import { VertexAI, type GenerationConfig, type UsageMetadata } from '@google-cloud/vertexai';
+import EventEmitter from 'events';
+
+import { JSON_RESPONSE_INSTRUCTION, BUILT_IN_MODEL_PREFIX } from '@sre/constants';
+import {
+    TLLMParams,
+    TCustomLLMModel,
+    APIKeySource,
+    TVertexAISettings,
+    ILLMRequestFuncParams,
+    TGoogleAIRequestBody,
+    TLLMConnectorParams,
+    ILLMRequestContext,
+} from '@sre/types/LLM.types';
+import { LLMHelper } from '@sre/LLMManager/LLM.helper';
+
+import { LLMConnector } from '../LLMConnector';
+import { SystemEvents } from '@sre/Core/SystemEvents';
+
+//TODO: [AHMED/FORHAD]: test the usage reporting for VertexAI because by the time we were implementing the feature of usage reporting
+// we had no access to VertexAI so we assumed it is working (potential bug)
+
+export class VertexAIConnector extends LLMConnector {
+    public name = 'LLM:VertexAI';
+
+    private async getClient(params: ILLMRequestContext): Promise<VertexAI> {
+        const credentials = params.credentials as any;
+        const modelInfo = params.modelInfo as TCustomLLMModel;
+        const projectId = (modelInfo?.settings as TVertexAISettings)?.projectId;
+        const region = modelInfo?.settings?.region;
+
+        return new VertexAI({
+            project: projectId,
+            location: region,
+            googleAuthOptions: {
+                credentials: credentials as any,
+            },
+        });
+    }
+
+    protected async request({ acRequest, body, context }: ILLMRequestFuncParams): Promise<any> {
+        const messages = body.messages;
+        delete body.messages;
+
+        try {
+            const vertexAI = await this.getClient(context);
+            const generativeModel = vertexAI.getGenerativeModel(body);
+
+            const result = await generativeModel.generateContent({ contents: messages });
+            const content = result?.response?.candidates?.[0]?.content?.parts?.[0]?.text;
+            const usage = result?.response?.usageMetadata;
+
+            this.reportUsage(usage, {
+                modelEntryName: context.modelEntryName,
+                keySource: context.isUserKey ? APIKeySource.User : APIKeySource.Smyth,
+                agentId: context.agentId,
+                teamId: context.teamId,
+            });
+
+            return {
+                content,
+                finishReason: 'stop',
+                useTool: false,
+                toolsData: [],
+                message: { content, role: 'assistant' },
+                usage,
+            };
+        } catch (error) {
+            throw error;
+        }
+    }
+
+    protected async streamRequest({ acRequest, body, context }: ILLMRequestFuncParams): Promise<EventEmitter> {
+        // Simulate streaming similar to Perplexity's approach - fallback to regular request
+        const emitter = new EventEmitter();
+
+        setTimeout(() => {
+            try {
+                this.request({ acRequest, body, context })
+                    .then((response) => {
+                        const finishReason = response.finishReason;
+                        const usage = response.usage;
+
+                        // Emit the content as a stream-like response
+                        emitter.emit('interrupted', finishReason);
+                        emitter.emit('content', response.content);
+                        emitter.emit('end', undefined, usage, finishReason);
+                    })
+                    .catch((error) => {
+                        emitter.emit('error', error.message || error.toString());
+                    });
+            } catch (error) {
+                emitter.emit('error', error.message || error.toString());
+            }
+        }, 100);
+
+        return emitter;
+    }
+
+    protected async webSearchRequest({ acRequest, body, context }: ILLMRequestFuncParams): Promise<EventEmitter> {
+        throw new Error('Web search is not supported for Vertex AI');
+    }
+
+    protected async reqBodyAdapter(params: TLLMParams): Promise<TGoogleAIRequestBody> {
+        let messages = params?.messages || [];
+
+        //#region Separate system message and add JSON response instruction if needed
+        let systemInstruction;
+        const { systemMessage, otherMessages } = LLMHelper.separateSystemMessages(messages);
+
+        if ('content' in systemMessage) {
+            systemInstruction = systemMessage.content;
+        }
+
+        messages = otherMessages;
+
+        const responseFormat = params?.responseFormat || '';
+        if (responseFormat === 'json') {
+            systemInstruction = JSON_RESPONSE_INSTRUCTION;
+        }
+        //#endregion Separate system message and add JSON response instruction if needed
+
+        const modelInfo = params.modelInfo as TCustomLLMModel;
+
+        let body: TGoogleAIRequestBody = {
+            model: modelInfo?.settings?.customModel || modelInfo?.settings?.foundationModel,
+            messages,
+        };
+
+        const config: GenerationConfig = {};
+
+        if (params?.maxTokens !== undefined) config.maxOutputTokens = params.maxTokens;
+        if (params?.temperature !== undefined) config.temperature = params.temperature;
+        if (params?.topP !== undefined) config.topP = params.topP;
+        if (params?.topK !== undefined) config.topK = params.topK;
+        if (params?.stopSequences?.length) config.stopSequences = params.stopSequences;
+
+        if (systemInstruction) {
+            body.systemInstruction = systemInstruction;
+        }
+
+        if (Object.keys(config).length > 0) {
+            body.generationConfig = config as any;
+        }
+
+        return body;
+    }
+
+    protected reportUsage(
+        usage: UsageMetadata & { cachedContentTokenCount?: number },
+        metadata: { modelEntryName: string; keySource: APIKeySource; agentId: string; teamId: string }
+    ) {
+        // SmythOS (built-in) models have a prefix, so we need to remove it to get the model name
+        const modelName = metadata.modelEntryName.replace(BUILT_IN_MODEL_PREFIX, '');
+
+        const usageData = {
+            sourceId: `llm:${modelName}`,
+            input_tokens: usage.promptTokenCount || 0,
+            output_tokens: usage.candidatesTokenCount || 0,
+            input_tokens_cache_read: usage.cachedContentTokenCount || 0,
+            input_tokens_cache_write: 0,
+            keySource: metadata.keySource,
+            agentId: metadata.agentId,
+            teamId: metadata.teamId,
+        };
+        SystemEvents.emit('USAGE:LLM', usageData);
+
+        return usageData;
+    }
+
+    public formatToolsConfig({ type = 'function', toolDefinitions, toolChoice = 'auto' }) {
+        throw new Error('Tool configuration is not currently implemented for Vertex AI');
+    }
+
+    public getConsistentMessages(messages) {
+        const _messages = LLMHelper.removeDuplicateUserMessages(messages);
+
+        return _messages.map((message) => {
+            let textBlock = [];
+
+            if (message?.parts) {
+                textBlock = message.parts;
+            } else if (message?.content) {
+                textBlock = Array.isArray(message.content) ? message.content : [{ text: message.content as string }];
+            }
+
+            return {
+                role: message.role,
+                parts: textBlock,
+            };
+        });
+    }
+}

package/src/subsystems/LLMManager/LLM.service/index.ts

@@ -0,0 +1,43 @@
+//==[ SRE: LLM ]======================
+
+import { ConnectorService, ConnectorServiceProvider } from '@sre/Core/ConnectorsService';
+import { TConnectorService } from '@sre/types/SRE.types';
+import { EchoConnector } from './connectors/Echo.class';
+import { OpenAIConnector } from './connectors/OpenAI.class';
+import { GoogleAIConnector } from './connectors/GoogleAI.class';
+import { AnthropicConnector } from './connectors/Anthropic.class';
+import { GroqConnector } from './connectors/Groq.class';
+import { BedrockConnector } from './connectors/Bedrock.class';
+import { VertexAIConnector } from './connectors/VertexAI.class';
+import { PerplexityConnector } from './connectors/Perplexity.class';
+
+export class LLMService extends ConnectorServiceProvider {
+    public register() {
+        ConnectorService.register(TConnectorService.LLM, 'Echo', EchoConnector);
+        ConnectorService.register(TConnectorService.LLM, 'OpenAI', OpenAIConnector);
+        ConnectorService.register(TConnectorService.LLM, 'DeepSeek', OpenAIConnector);
+        ConnectorService.register(TConnectorService.LLM, 'GoogleAI', GoogleAIConnector);
+        ConnectorService.register(TConnectorService.LLM, 'Anthropic', AnthropicConnector);
+        ConnectorService.register(TConnectorService.LLM, 'Groq', GroqConnector);
+        ConnectorService.register(TConnectorService.LLM, 'TogetherAI', OpenAIConnector);
+        ConnectorService.register(TConnectorService.LLM, 'Bedrock', BedrockConnector);
+        ConnectorService.register(TConnectorService.LLM, 'VertexAI', VertexAIConnector);
+        ConnectorService.register(TConnectorService.LLM, 'xAI', OpenAIConnector);
+        ConnectorService.register(TConnectorService.LLM, 'Perplexity', PerplexityConnector);
+    }
+
+    public init() {
+        //auto initialize built-in models
+        ConnectorService.init(TConnectorService.LLM, 'Echo');
+        ConnectorService.init(TConnectorService.LLM, 'OpenAI');
+        ConnectorService.init(TConnectorService.LLM, 'DeepSeek');
+        ConnectorService.init(TConnectorService.LLM, 'GoogleAI');
+        ConnectorService.init(TConnectorService.LLM, 'Anthropic');
+        ConnectorService.init(TConnectorService.LLM, 'Groq');
+        ConnectorService.init(TConnectorService.LLM, 'TogetherAI');
+        ConnectorService.init(TConnectorService.LLM, 'Bedrock');
+        ConnectorService.init(TConnectorService.LLM, 'VertexAI');
+        ConnectorService.init(TConnectorService.LLM, 'xAI');
+        ConnectorService.init(TConnectorService.LLM, 'Perplexity');
+    }
+}