@smythos/sre 1.5.0 → 1.5.2

package/src/subsystems/LLMManager/LLM.helper.ts

@@ -0,0 +1,221 @@
+import { type TLLMMessageBlock, TLLMMessageRole } from '@sre/types/LLM.types';
+
+import axios from 'axios';
+import imageSize from 'image-size';
+import { encode } from 'gpt-tokenizer';
+import { isBase64FileUrl, isUrl } from '@sre/utils';
+
+export class LLMHelper {
+    /**
+     * Checks if the given array of messages contains a system message.
+     *
+     * @param {any} messages - The array of messages to check.
+     * @returns {boolean} True if a system message is found, false otherwise.
+     *
+     * @example
+     * const messages = [
+     *   { role: 'user', content: 'Hello' },
+     *   { role: 'system', content: 'You are a helpful assistant' }
+     * ];
+     * const hasSystem = LLMHelper.hasSystemMessage(messages);
+     * console.log(hasSystem); // true
+     */
+    public static hasSystemMessage(messages: any): boolean {
+        if (!Array.isArray(messages)) return false;
+        return messages?.some((message) => message.role === 'system');
+    }
+
+    /**
+     * Separates system messages from other messages in an array of LLM message blocks.
+     *
+     * @param {TLLMMessageBlock[]} messages - The array of message blocks to process.
+     * @returns {Object} An object containing the system message (if any) and an array of other messages.
+     * @property {TLLMMessageBlock | {}} systemMessage - The first system message found, or an empty object if none.
+     * @property {TLLMMessageBlock[]} otherMessages - An array of all non-system messages.
+     *
+     * @example
+     * const messages = [
+     *   { role: 'system', content: 'You are a helpful assistant' },
+     *   { role: 'user', content: 'Hello' },
+     *   { role: 'assistant', content: 'Hi there!' }
+     * ];
+     * const { systemMessage, otherMessages } = LLMHelper.separateSystemMessages(messages);
+     * console.log(systemMessage); // { role: 'system', content: 'You are a helpful assistant' }
+     * console.log(otherMessages); // [{ role: 'user', content: 'Hello' }, { role: 'assistant', content: 'Hi there!' }]
+     */
+    public static separateSystemMessages(messages: TLLMMessageBlock[]): {
+        systemMessage: TLLMMessageBlock | {};
+        otherMessages: TLLMMessageBlock[];
+    } {
+        const systemMessage = messages.find((message) => message.role === 'system') || {};
+        const otherMessages = messages.filter((message) => message.role !== 'system');
+
+        return { systemMessage, otherMessages };
+    }
+
+    /**
+     * Counts the total number of tokens in a vision prompt, including both text and image tokens.
+     *
+     * @param {any} prompt - The vision prompt object containing text and image items.
+     * @returns {Promise<number>} A promise that resolves to the total number of tokens in the prompt.
+     *
+     * @description
+     * This method processes a vision prompt by:
+     * 1. Counting tokens in the text portion of the prompt.
+     * 2. Calculating tokens for each image in the prompt based on its dimensions.
+     * 3. Summing up text and image tokens to get the total token count.
+     *
+     * @example
+     * const prompt = [
+     *   { type: 'text', text: 'Describe this image:' },
+     *   { type: 'image_url', image_url: { url: 'https://example.com/image.jpg' } }
+     * ];
+     * const tokenCount = await countVisionPromptTokens(prompt);
+     * console.log(tokenCount); // e.g., 150
+     */
+    public static async countVisionPromptTokens(prompt: any): Promise<number> {
+        let tokens = 0;
+
+        const textObj = prompt?.filter((item) => item.type === 'text');
+        const textTokens = encode(textObj?.[0]?.text).length;
+
+        const images = prompt?.filter((item) => item.type === 'image_url');
+        let imageTokens = 0;
+
+        for (const image of images) {
+            const imageUrl = image?.image_url?.url;
+            const { width, height } = await this.getImageDimensions(imageUrl);
+            const tokens = this.countImageTokens(width, height);
+            imageTokens += tokens;
+        }
+
+        tokens = textTokens + imageTokens;
+        return tokens;
+    }
+
+    /**
+     * Retrieves the dimensions (width and height) of an image from a given URL or base64 encoded string.
+     *
+     * @param {string} imageUrl - The URL or base64 encoded string of the image.
+     * @returns {Promise<{ width: number; height: number }>} A promise that resolves to an object containing the width and height of the image.
+     * @throws {Error} If the provided imageUrl is invalid or if there's an error retrieving the image dimensions.
+     *
+     * @example
+     * // Using a URL
+     * const dimensions = await getImageDimensions('https://example.com/image.jpg');
+     * console.log(dimensions); // { width: 800, height: 600 }
+     *
+     * @example
+     * // Using a base64 encoded string
+     * const dimensions = await getImageDimensions('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACklEQVR4nGMAAQAABQABDQottAAAAABJRU5ErkJggg==');
+     * console.log(dimensions); // { width: 1, height: 1 }
+     */
+    public static async getImageDimensions(imageUrl: string): Promise<{ width: number; height: number }> {
+        try {
+            let buffer: Buffer;
+
+            if (isBase64FileUrl(imageUrl)) {
+                const base64Data = imageUrl.replace(/^data:image\/\w+;base64,/, '');
+                buffer = Buffer.from(base64Data, 'base64');
+            } else if (isUrl(imageUrl)) {
+                const response = await axios.get(imageUrl, { responseType: 'arraybuffer' });
+                buffer = Buffer.from(response.data);
+            } else {
+                throw new Error('Please provide a valid image url!');
+            }
+
+            const dimensions = imageSize(buffer);
+
+            return {
+                width: dimensions?.width || 0,
+                height: dimensions?.height || 0,
+            };
+        } catch (error) {
+            console.error('Error getting image dimensions', error);
+            throw new Error('Please provide a valid image url!');
+        }
+    }
+
+    /**
+     * Calculates the number of tokens required to process an image based on its dimensions and detail mode.
+     *
+     * @param {number} width - The width of the image in pixels.
+     * @param {number} height - The height of the image in pixels.
+     * @param {string} detailMode - The detail mode for processing the image. Defaults to 'auto'.
+     * @returns {number} The number of tokens required to process the image.
+     *
+     * @description
+     * This method estimates the token count for image processing based on the image dimensions and detail mode.
+     * It uses a tiling approach to calculate the token count, scaling the image if necessary.
+     *
+     * - If detailMode is 'low', it returns a fixed token count of 85.
+     * - For other modes, it calculates based on the image dimensions:
+     *   - Scales down images larger than 2048 pixels in any dimension.
+     *   - Adjusts the scaled dimension to fit within a 768x1024 aspect ratio.
+     *   - Calculates the number of 512x512 tiles needed to cover the image.
+     *   - Returns the total token count based on the number of tiles.
+     *
+     * @example
+     * const tokenCount = countImageTokens(1024, 768);
+     * console.log(tokenCount); // Outputs the calculated token count
+     */
+    public static countImageTokens(width: number, height: number, detailMode: string = 'auto'): number {
+        if (detailMode === 'low') return 85;
+
+        const maxDimension = Math.max(width, height);
+        const minDimension = Math.min(width, height);
+        let scaledMinDimension = minDimension;
+
+        if (maxDimension > 2048) {
+            scaledMinDimension = (2048 / maxDimension) * minDimension;
+        }
+        scaledMinDimension = Math.floor((768 / 1024) * scaledMinDimension);
+
+        let tileSize = 512;
+        let tiles = Math.ceil(scaledMinDimension / tileSize);
+
+        if (minDimension !== scaledMinDimension) {
+            tiles *= Math.ceil((scaledMinDimension * (maxDimension / minDimension)) / tileSize);
+        }
+
+        return tiles * 170 + 85;
+    }
+
+    /**
+     * Removes duplicate user messages from the beginning and end of the messages array.
+     *
+     * This method checks if there are two consecutive user messages at the start or end of the array
+     *
+     * @param {Array<{ role: string; content: string }>} messages - The array of message objects to process.
+     *
+     * @example
+     * const messages = [
+     *   { role: 'user', content: 'Hello' },
+     *   { role: 'user', content: 'Hello' },
+     *   { role: 'assistant', content: 'Hi there!' }
+     * ];
+     * LLMHelper.removeDuplicateUserMessages(messages);
+     * console.log(messages); // [{ role: 'user', content: 'Hello' }, { role: 'assistant', content: 'Hi there!' }]
+     *
+     * @returns {TLLMMessageBlock[]} The modified array of message objects.
+     */
+    public static removeDuplicateUserMessages(messages: TLLMMessageBlock[]): TLLMMessageBlock[] {
+        const _messages = JSON.parse(JSON.stringify(messages));
+
+        // Check for two user messages at the beginning
+        if (_messages.length > 1 && _messages[0].role === TLLMMessageRole.User && _messages[1].role === TLLMMessageRole.User) {
+            _messages.shift(); // Remove the first user message
+        }
+
+        // Check for two user messages at the end
+        if (
+            _messages.length > 1 &&
+            _messages[_messages.length - 1].role === TLLMMessageRole.User &&
+            _messages[_messages.length - 2].role === TLLMMessageRole.User
+        ) {
+            _messages.pop(); // Remove the last user message
+        }
+
+        return _messages;
+    }
+}

package/src/subsystems/LLMManager/LLM.inference.ts

@@ -0,0 +1,335 @@
+import _ from 'lodash';
+import { type OpenAI } from 'openai';
+import { encodeChat } from 'gpt-tokenizer';
+import { ChatMessage } from 'gpt-tokenizer/esm/GptEncoding';
+import { ConnectorService } from '@sre/Core/ConnectorsService';
+import { BinaryInput } from '@sre/helpers/BinaryInput.helper';
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+import { LLMConnector } from './LLM.service/LLMConnector';
+import { EventEmitter } from 'events';
+import { GenerateImageConfig, TLLMMessageRole, TLLMModel, TLLMChatResponse } from '@sre/types/LLM.types';
+import { IModelsProviderRequest, ModelsProviderConnector } from './ModelsProvider.service/ModelsProviderConnector';
+import { Logger } from '@sre/helpers/Log.helper';
+import { IAgent } from '@sre/types/Agent.types';
+import { isAgent } from '@sre/AgentManager/Agent.helper';
+import { TLLMParams } from '@sre/types/LLM.types';
+
+const console = Logger('LLMInference');
+
+type TPromptParams = { query?: string; contextWindow?: any[]; files?: any[]; params: TLLMParams };
+
+export class LLMInference {
+    private model: string | TLLMModel;
+    private llmConnector: LLMConnector;
+    private modelProviderReq: IModelsProviderRequest;
+    public teamId?: string;
+
+    public static async getInstance(model: string | TLLMModel, candidate: AccessCandidate) {
+        const modelsProvider: ModelsProviderConnector = ConnectorService.getModelsProviderConnector();
+        if (!modelsProvider.valid) {
+            throw new Error(`Model provider Not available, cannot create LLM instance`);
+        }
+        const accountConnector = ConnectorService.getAccountConnector();
+        const teamId = await accountConnector.requester(candidate).getTeam();
+
+        const llmInference = new LLMInference();
+        llmInference.teamId = teamId;
+
+        llmInference.modelProviderReq = modelsProvider.requester(candidate);
+
+        const llmProvider = await llmInference.modelProviderReq.getProvider(model);
+        if (llmProvider) {
+            llmInference.llmConnector = ConnectorService.getLLMConnector(llmProvider);
+        }
+
+        if (!llmInference.llmConnector) {
+            console.error(`Model ${model} unavailable for team ${teamId}`);
+        }
+
+        llmInference.model = model;
+
+        return llmInference;
+    }
+
+    public static user(candidate: AccessCandidate): any {}
+
+    public get connector(): LLMConnector {
+        return this.llmConnector;
+    }
+
+    public async prompt({ query, contextWindow, files, params }: TPromptParams) {
+        let messages = contextWindow || [];
+
+        if (query) {
+            const content = this.llmConnector.enhancePrompt(query, params);
+            messages.push({ role: TLLMMessageRole.User, content });
+        }
+
+        if (!params.model) params.model = this.model;
+        params.messages = messages;
+        params.files = files;
+
+        try {
+            let response: TLLMChatResponse = await this.llmConnector.requester(AccessCandidate.agent(params.agentId)).request(params);
+
+            const result = this.llmConnector.postProcess(response?.content);
+            if (result.error) {
+                // If the model stopped before completing the response, this is usually due to output token limit reached.
+                if (response.finishReason !== 'stop') {
+                    throw new Error('The model stopped before completing the response, this is usually due to output token limit reached.');
+                }
+
+                // If the model stopped due to other reasons, throw the error
+                throw new Error(result.error);
+            }
+            return result;
+        } catch (error: any) {
+            console.error('Error in chatRequest: ', error);
+
+            throw error;
+        }
+    }
+
+    public async promptStream({ query, contextWindow, files, params }: TPromptParams) {
+        let messages = contextWindow || [];
+
+        if (query) {
+            const content = this.llmConnector.enhancePrompt(query, params);
+            messages.push({ role: TLLMMessageRole.User, content });
+        }
+
+        if (!params.model) params.model = this.model;
+        params.messages = messages;
+        params.files = files;
+
+        try {
+            return await this.llmConnector.user(AccessCandidate.agent(params.agentId)).streamRequest(params);
+        } catch (error) {
+            console.error('Error in streamRequest:', error);
+
+            const dummyEmitter = new EventEmitter();
+            process.nextTick(() => {
+                dummyEmitter.emit('error', error);
+                dummyEmitter.emit('end');
+            });
+            return dummyEmitter;
+        }
+    }
+
+    public async imageGenRequest({ query, files, params }: TPromptParams) {
+        params.prompt = query;
+        return this.llmConnector.user(AccessCandidate.agent(params.agentId)).imageGenRequest(params);
+    }
+
+    public async imageEditRequest({ query, files, params }: TPromptParams) {
+        params.prompt = query;
+        params.files = files;
+        return this.llmConnector.user(AccessCandidate.agent(params.agentId)).imageEditRequest(params);
+    }
+
+    public async streamRequest(params: any, agent: string | IAgent) {
+        const agentId = isAgent(agent) ? (agent as IAgent).id : agent;
+        try {
+            if (!params.messages || !params.messages?.length) {
+                throw new Error('Input messages are required.');
+            }
+
+            const model = params.model || this.model;
+
+            return await this.llmConnector.user(AccessCandidate.agent(agentId)).streamRequest({ ...params, model });
+        } catch (error) {
+            console.error('Error in streamRequest:', error);
+
+            const dummyEmitter = new EventEmitter();
+            process.nextTick(() => {
+                dummyEmitter.emit('error', error);
+                dummyEmitter.emit('end');
+            });
+            return dummyEmitter;
+        }
+    }
+
+    public async multimodalStreamRequest(params: any, fileSources, agent: string | IAgent) {
+        const agentId = isAgent(agent) ? (agent as IAgent).id : agent;
+
+        const promises = [];
+        const _fileSources = [];
+
+        // TODO [Forhad]: For models from Google AI, we currently store files twice — once here and once in the GoogleAIConnector. We need to optimize this process.
+        for (let file of fileSources) {
+            const binaryInput = BinaryInput.from(file);
+            _fileSources.push(binaryInput);
+            promises.push(binaryInput.upload(AccessCandidate.agent(agentId)));
+        }
+
+        await Promise.all(promises);
+
+        params.fileSources = _fileSources;
+
+        try {
+            //FIXME we need to update the connector multimediaStreamRequest in order to ignore prompt param if not provided
+            const userMessage = Array.isArray(params.messages) ? params.messages.pop() : {};
+            const prompt = userMessage?.content || '';
+            const model = params.model || this.model;
+
+            return await this.llmConnector.user(AccessCandidate.agent(agentId)).multimodalStreamRequest(prompt, { ...params, model });
+        } catch (error: any) {
+            console.error('Error in multimodalRequest: ', error);
+
+            throw error;
+        }
+    }
+
+    public async multimodalStreamRequestLegacy(prompt, files: string[], config: any = {}, agent: string | IAgent) {
+        const agentId = isAgent(agent) ? (agent as IAgent).id : agent;
+
+        const promises = [];
+        const _files = [];
+
+        // TODO [Forhad]: For models from Google AI, we currently store files twice — once here and once in the GoogleAIConnector. We need to optimize this process.
+        for (let file of files) {
+            const binaryInput = BinaryInput.from(file);
+            _files.push(binaryInput);
+            promises.push(binaryInput.upload(AccessCandidate.agent(agentId)));
+        }
+
+        await Promise.all(promises);
+
+        const params = config.data;
+
+        params.files = _files;
+
+        try {
+            prompt = this.llmConnector.enhancePrompt(prompt, config);
+            const model = params.model || this.model;
+
+            return await this.llmConnector.user(AccessCandidate.agent(agentId)).multimodalStreamRequest(prompt, { ...params, model });
+        } catch (error: any) {
+            console.error('Error in multimodalRequest: ', error);
+
+            throw error;
+        }
+    }
+
+    //Not needed
+    // public getConsistentMessages(messages: TLLMMessageBlock[]) {
+    //     if (!messages?.length) {
+    //         throw new Error('Input messages are required.');
+    //     }
+
+    //     try {
+    //         return this.llmConnector.getConsistentMessages(messages);
+    //     } catch (error) {
+    //         console.warn('Something went wrong in getConsistentMessages: ', error);
+
+    //         return messages; // if something went wrong then we return the original messages
+    //     }
+    // }
+
+    /**
+     * Get the context window for the given messages
+     * @param _messages - The messages to get the context window for (the messages are in smythos generic format)
+     * @param maxTokens - The maximum number of tokens to use for the context window
+     * @param maxOutputTokens - The maximum number of tokens to use for the output
+     * @returns The context window for the given messages
+     */
+    public async getContextWindow(systemPrompt: string, _messages: any[], maxTokens: number, maxOutputTokens: number = 1024): Promise<any[]> {
+        //TODO: handle non key accounts (limit tokens)
+        // const maxModelContext = this._llmHelper?.modelInfo?.keyOptions?.tokens || this._llmHelper?.modelInfo?.tokens || 256;
+
+        //#region get max model context
+
+        const modelInfo = await this.modelProviderReq.getModelInfo(this.model, true);
+        let maxModelContext = modelInfo?.tokens;
+        let maxModelOutputTokens = modelInfo?.completionTokens || modelInfo?.tokens;
+        // const isStandardLLM = LLMRegistry.isStandardLLM(this.model);
+
+        // if (isStandardLLM) {
+        //     maxModelContext = LLMRegistry.getMaxContextTokens(this.model, true); // we just provide true for hasAPIKey to get the original max context
+        // } else {
+        //     const team = AccessCandidate.team(this.teamId);
+        //     const customLLMRegistry = await CustomLLMRegistry.getInstance(team);
+        //     maxModelContext = customLLMRegistry.getMaxContextTokens(this.model);
+        //     maxModelOutputTokens = customLLMRegistry.getMaxCompletionTokens(this.model);
+        // }
+        //#endregion get max model context
+
+        let maxInputContext = Math.min(maxTokens, maxModelContext);
+        let maxOutputContext = Math.min(maxOutputTokens, maxModelOutputTokens || 0);
+
+        if (maxInputContext + maxOutputContext > maxModelContext) {
+            maxInputContext -= maxInputContext + maxOutputContext - maxModelContext;
+        }
+
+        const systemMessage = { role: 'system', content: systemPrompt };
+
+        let smythContextWindow = [];
+
+        //loop through messages from last to first and use encodeChat to calculate token lengths
+        //we will use fake chatMessages to calculate the token lengths, these are not used by the LLM, but just for token counting
+        let tokensCount = encodeChat([systemMessage as ChatMessage], 'gpt-4o').length;
+        for (let i = _messages?.length - 1; i >= 0; i--) {
+            const curMessage = _messages[i];
+            if (curMessage.role === 'system') continue;
+
+            tokensCount = 0;
+            if (curMessage?.content) {
+                // tokensCount += encodeChat([{ role: 'user', content: curMessage.content } as ChatMessage], 'gpt-4o').length;
+                tokensCount += countTokens(curMessage.content);
+            }
+
+            if (curMessage?.messageBlock?.content) {
+                // tokensCount += encodeChat([{ role: 'user', content: curMessage.messageBlock.content } as ChatMessage], 'gpt-4o').length;
+                tokensCount += countTokens(curMessage.messageBlock.content);
+            }
+            if (curMessage.toolsData) {
+                for (let tool of curMessage.toolsData) {
+                    // tokensCount += encodeChat([{ role: 'user', content: tool.result } as ChatMessage], 'gpt-4o').length;
+                    tokensCount += countTokens(tool.result);
+                }
+            }
+
+            //did the last message exceed the context window ?
+            if (tokensCount > maxInputContext) {
+                break;
+            }
+
+            smythContextWindow.unshift(curMessage);
+        }
+        smythContextWindow.unshift(systemMessage);
+
+        let modelContextWindow = [];
+        //now transform the messages to the model format
+        for (let message of smythContextWindow) {
+            if (message.role && message.content) {
+                modelContextWindow.push({ role: message.role, content: message.content });
+            }
+
+            if (message.messageBlock && message.toolsData) {
+                const internal_message = this.connector.transformToolMessageBlocks({
+                    messageBlock: message?.messageBlock,
+                    toolsData: message?.toolsData,
+                });
+
+                modelContextWindow.push(...internal_message);
+            }
+        }
+
+        modelContextWindow = this.connector.getConsistentMessages(modelContextWindow);
+
+        return modelContextWindow;
+    }
+}
+
+function countTokens(content: any, model: 'gpt-4o' | 'gpt-4o-mini' = 'gpt-4o') {
+    try {
+        // Content must be stringified since some providers like Anthropic use object content
+        const _stringifiedContent = typeof content === 'string' ? content : JSON.stringify(content);
+
+        const tokens = encodeChat([{ role: 'user', content: _stringifiedContent } as ChatMessage], model);
+        return tokens.length;
+    } catch (error) {
+        console.warn('Error in countTokens: ', error);
+        return 0;
+    }
+}