@smythos/sre 1.5.0

package/dist/types/subsystems/IO/Storage.service/connectors/LocalStorage.class.d.ts

@@ -0,0 +1,58 @@
+import { StorageConnector } from '@sre/IO/Storage.service/StorageConnector';
+import { ACL } from '@sre/Security/AccessControl/ACL.class';
+import { IAccessCandidate, IACL } from '@sre/types/ACL.types';
+import { StorageData, StorageMetadata } from '@sre/types/Storage.types';
+import { AccessRequest } from '@sre/Security/AccessControl/AccessRequest.class';
+export type LocalStorageConfig = {
+    /**
+     * The folder to use for the local storage.
+     *
+     * If not provided, the default folder will be used.
+     * The default folder is ~/.smyth/storage
+     */
+    folder?: string;
+};
+export declare class LocalStorage extends StorageConnector {
+    protected _settings?: LocalStorageConfig;
+    name: string;
+    private folder;
+    private storagePrefix;
+    private metadataPrefix;
+    private isInitialized;
+    constructor(_settings?: LocalStorageConfig);
+    /**
+     * Reads an object from the local storage.
+     *
+     * @param {string} resourceId - The key of the object to be read.
+     * @returns {Promise<any>} - A promise that resolves with the object data.
+     */
+    read(acRequest: AccessRequest, resourceId: string): Promise<string>;
+    getMetadata(acRequest: AccessRequest, resourceId: string): Promise<StorageMetadata | undefined>;
+    setMetadata(acRequest: AccessRequest, resourceId: string, metadata: StorageMetadata): Promise<void>;
+    /**
+     * Writes an object to the local storage.
+     *
+     * @param {string} resourceId - The key of the object to be written.
+     * @param {any} value - The value of the object to be written.
+     * @param {Metadata} metadata - Optional metadata to be associated with the object.
+     * @returns {Promise<void>} - A promise that resolves when the object has been written.
+     */
+    write(acRequest: AccessRequest, resourceId: string, value: StorageData, acl?: IACL, metadata?: StorageMetadata): Promise<void>;
+    /**
+     * Deletes an object from the local storage.
+     *
+     * @param {string} resourceId - The key of the object to be deleted.
+     * @returns {Promise<void>} - A promise that resolves when the object has been deleted.
+     */
+    delete(acRequest: AccessRequest, resourceId: string): Promise<void>;
+    exists(acRequest: AccessRequest, resourceId: string): Promise<boolean>;
+    getResourceACL(resourceId: string, candidate: IAccessCandidate): Promise<ACL>;
+    getACL(acRequest: AccessRequest, resourceId: string): Promise<ACL | undefined>;
+    setACL(acRequest: AccessRequest, resourceId: string, acl: IACL): Promise<void>;
+    expire(acRequest: AccessRequest, resourceId: string, ttl: number): Promise<void>;
+    private initialize;
+    private getStorageFilePath;
+    private getMetadataFilePath;
+    private serializeMetadata;
+    private deserializeMetadata;
+}

package/dist/types/subsystems/IO/Storage.service/connectors/S3Storage.class.d.ts

@@ -0,0 +1,53 @@
+import { StorageConnector } from '@sre/IO/Storage.service/StorageConnector';
+import { ACL } from '@sre/Security/AccessControl/ACL.class';
+import { IAccessCandidate, IACL } from '@sre/types/ACL.types';
+import { AWSRegionConfig, AWSCredentials } from '@sre/types/AWS.types';
+import { StorageData, StorageMetadata } from '@sre/types/Storage.types';
+import { AccessRequest } from '@sre/Security/AccessControl/AccessRequest.class';
+export type S3Config = AWSCredentials & AWSRegionConfig & {
+    bucket: string;
+};
+export declare class S3Storage extends StorageConnector {
+    protected _settings: S3Config;
+    name: string;
+    private client;
+    private bucket;
+    private isInitialized;
+    constructor(_settings: S3Config);
+    private initialize;
+    /**
+     * Reads an object from the S3 bucket.
+     *
+     * @param {string} resourceId - The key of the object to be read.
+     * @returns {Promise<any>} - A promise that resolves with the object data.
+     */
+    read(acRequest: AccessRequest, resourceId: string): Promise<Buffer<ArrayBufferLike>>;
+    getMetadata(acRequest: AccessRequest, resourceId: string): Promise<StorageMetadata | undefined>;
+    setMetadata(acRequest: AccessRequest, resourceId: string, metadata: StorageMetadata): Promise<void>;
+    /**
+     * Writes an object to the S3 bucket.
+     *
+     * @param {string} resourceId - The key of the object to be written.
+     * @param {any} value - The value of the object to be written.
+     * @param {Metadata} metadata - Optional metadata to be associated with the object.
+     * @returns {Promise<void>} - A promise that resolves when the object has been written.
+     */
+    write(acRequest: AccessRequest, resourceId: string, value: StorageData, acl?: IACL, metadata?: StorageMetadata): Promise<void>;
+    /**
+     * Deletes an object from the S3 bucket.
+     *
+     * @param {string} resourceId - The key of the object to be deleted.
+     * @returns {Promise<void>} - A promise that resolves when the object has been deleted.
+     */
+    delete(acRequest: AccessRequest, resourceId: string): Promise<void>;
+    exists(acRequest: AccessRequest, resourceId: string): Promise<boolean>;
+    getResourceACL(resourceId: string, candidate: IAccessCandidate): Promise<ACL>;
+    getACL(acRequest: AccessRequest, resourceId: string): Promise<ACL | undefined>;
+    setACL(acRequest: AccessRequest, resourceId: string, acl: IACL): Promise<void>;
+    expire(acRequest: AccessRequest, resourceId: string, ttl: number): Promise<void>;
+    private migrateMetadata;
+    private serializeS3Metadata;
+    private deserializeS3Metadata;
+    private getS3Metadata;
+    private setS3Metadata;
+}

package/dist/types/subsystems/IO/Storage.service/index.d.ts

@@ -0,0 +1,4 @@
+import { ConnectorServiceProvider } from '@sre/Core/ConnectorsService';
+export declare class StorageService extends ConnectorServiceProvider {
+    register(): void;
+}

package/dist/types/subsystems/IO/VectorDB.service/VectorDBConnector.d.ts

@@ -0,0 +1,41 @@
+import { ACL } from '@sre/Security/AccessControl/ACL.class';
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+import { AccessRequest } from '@sre/Security/AccessControl/AccessRequest.class';
+import { SecureConnector } from '@sre/Security/SecureConnector.class';
+import { IAccessCandidate } from '@sre/types/ACL.types';
+import { DatasourceDto, IStorageVectorDataSource, IVectorDataSourceDto, QueryOptions, VectorsResultData } from '@sre/types/VectorDB.types';
+export type DeleteFilterOptions = {
+    datasourceId?: string;
+};
+export type DeleteTarget = string | string[] | DeleteFilterOptions;
+export interface IVectorDBRequest {
+    search(namespace: string, query: string | number[], options?: QueryOptions): Promise<VectorsResultData>;
+    createDatasource(namespace: string, datasource: DatasourceDto): Promise<IStorageVectorDataSource>;
+    deleteDatasource(namespace: string, datasourceId: string): Promise<void>;
+    listDatasources(namespace: string): Promise<IStorageVectorDataSource[]>;
+    getDatasource(namespace: string, datasourceId: string): Promise<IStorageVectorDataSource>;
+    createNamespace(namespace: string, metadata?: {
+        [key: string]: any;
+    }): Promise<void>;
+    deleteNamespace(namespace: string): Promise<void>;
+    namespaceExists(namespace: string): Promise<boolean>;
+}
+export declare abstract class VectorDBConnector extends SecureConnector<IVectorDBRequest> {
+    protected readonly USER_METADATA_KEY = "user_metadata";
+    abstract id: string;
+    abstract getResourceACL(resourceId: string, candidate: IAccessCandidate): Promise<ACL>;
+    requester(candidate: AccessCandidate): IVectorDBRequest;
+    protected abstract search(acRequest: AccessRequest, namespace: string, query: string | number[], options: QueryOptions): Promise<VectorsResultData>;
+    protected abstract insert(acRequest: AccessRequest, namespace: string, source: IVectorDataSourceDto | IVectorDataSourceDto[]): Promise<string[]>;
+    protected abstract delete(acRequest: AccessRequest, namespace: string, deleteTarget: DeleteTarget): Promise<void>;
+    protected abstract createDatasource(acRequest: AccessRequest, namespace: string, datasource: DatasourceDto): Promise<IStorageVectorDataSource>;
+    protected abstract deleteDatasource(acRequest: AccessRequest, namespace: string, datasourceId: string): Promise<void>;
+    protected abstract listDatasources(acRequest: AccessRequest, namespace: string): Promise<IStorageVectorDataSource[]>;
+    protected abstract getDatasource(acRequest: AccessRequest, namespace: string, datasourceId: string): Promise<IStorageVectorDataSource | undefined>;
+    protected abstract createNamespace(acRequest: AccessRequest, namespace: string, metadata?: {
+        [key: string]: any;
+    }): Promise<void>;
+    protected abstract deleteNamespace(acRequest: AccessRequest, namespace: string): Promise<void>;
+    protected abstract namespaceExists(acRequest: AccessRequest, namespace: string): Promise<boolean>;
+    constructNsName(candidate: AccessCandidate, name: string): string;
+}

package/dist/types/subsystems/IO/VectorDB.service/connectors/MilvusVectorDB.class.d.ts

@@ -0,0 +1,49 @@
+import { ACL } from '@sre/Security/AccessControl/ACL.class';
+import { IAccessCandidate } from '@sre/types/ACL.types';
+import { AccessRequest } from '@sre/Security/AccessControl/AccessRequest.class';
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+import { DeleteTarget, VectorDBConnector } from '../VectorDBConnector';
+import { DatasourceDto, IStorageVectorDataSource, IVectorDataSourceDto, QueryOptions, VectorsResultData } from '@sre/types/VectorDB.types';
+import { BaseEmbedding, TEmbeddings } from '../embed/BaseEmbedding';
+export type IMilvusCredentials = {
+    address: string;
+    token: string;
+} | {
+    address: string;
+    user: string;
+    password: string;
+    token?: string;
+};
+export type MilvusConfig = {
+    credentials: IMilvusCredentials;
+    embeddings: TEmbeddings;
+};
+export declare class MilvusVectorDB extends VectorDBConnector {
+    protected _settings: MilvusConfig;
+    name: string;
+    id: string;
+    private client;
+    private cache;
+    private accountConnector;
+    embedder: BaseEmbedding;
+    private SCHEMA_DEFINITION;
+    private INDEX_PARAMS;
+    constructor(_settings: MilvusConfig);
+    protected createNamespace(acRequest: AccessRequest, namespace: string, metadata?: {
+        [key: string]: any;
+    }): Promise<void>;
+    protected namespaceExists(acRequest: AccessRequest, namespace: string): Promise<boolean>;
+    protected deleteNamespace(acRequest: AccessRequest, namespace: string): Promise<void>;
+    protected search(acRequest: AccessRequest, namespace: string, query: string | number[], options?: QueryOptions): Promise<VectorsResultData>;
+    protected insert(acRequest: AccessRequest, namespace: string, sourceWrapper: IVectorDataSourceDto | IVectorDataSourceDto[]): Promise<string[]>;
+    protected delete(acRequest: AccessRequest, namespace: string, deleteTarget: DeleteTarget): Promise<void>;
+    protected createDatasource(acRequest: AccessRequest, namespace: string, datasource: DatasourceDto): Promise<IStorageVectorDataSource>;
+    protected deleteDatasource(acRequest: AccessRequest, namespace: string, datasourceId: string): Promise<void>;
+    protected listDatasources(acRequest: AccessRequest, namespace: string): Promise<IStorageVectorDataSource[]>;
+    protected getDatasource(acRequest: AccessRequest, namespace: string, datasourceId: string): Promise<IStorageVectorDataSource | undefined>;
+    private setACL;
+    private getACL;
+    getResourceACL(resourceId: string, candidate: IAccessCandidate): Promise<ACL>;
+    private deleteACL;
+    constructNsName(candidate: AccessCandidate, name: string): string;
+}

package/dist/types/subsystems/IO/VectorDB.service/connectors/PineconeVectorDB.class.d.ts

@@ -0,0 +1,50 @@
+import { ACL } from '@sre/Security/AccessControl/ACL.class';
+import { IAccessCandidate } from '@sre/types/ACL.types';
+import { AccessRequest } from '@sre/Security/AccessControl/AccessRequest.class';
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+import { VectorDBConnector } from '../VectorDBConnector';
+import { DatasourceDto, IStorageVectorDataSource, IVectorDataSourceDto, QueryOptions, VectorsResultData } from '@sre/types/VectorDB.types';
+import { BaseEmbedding, TEmbeddings } from '../embed/BaseEmbedding';
+export type PineconeConfig = {
+    /**
+     * The Pinecone API key
+     */
+    apiKey: string;
+    /**
+     * The Pinecone index name
+     */
+    indexName: string;
+    /**
+     * The embeddings model to use
+     */
+    embeddings: TEmbeddings;
+};
+export declare class PineconeVectorDB extends VectorDBConnector {
+    protected _settings: PineconeConfig;
+    name: string;
+    id: string;
+    private client;
+    private indexName;
+    private cache;
+    private accountConnector;
+    private nkvConnector;
+    embedder: BaseEmbedding;
+    constructor(_settings: PineconeConfig);
+    getResourceACL(resourceId: string, candidate: IAccessCandidate): Promise<ACL>;
+    protected createNamespace(acRequest: AccessRequest, namespace: string, metadata?: {
+        [key: string]: any;
+    }): Promise<void>;
+    protected namespaceExists(acRequest: AccessRequest, namespace: string): Promise<boolean>;
+    protected deleteNamespace(acRequest: AccessRequest, namespace: string): Promise<void>;
+    protected search(acRequest: AccessRequest, namespace: string, query: string | number[], options?: QueryOptions): Promise<VectorsResultData>;
+    protected insert(acRequest: AccessRequest, namespace: string, sourceWrapper: IVectorDataSourceDto | IVectorDataSourceDto[]): Promise<string[]>;
+    protected delete(acRequest: AccessRequest, namespace: string, id: string | string[]): Promise<void>;
+    protected createDatasource(acRequest: AccessRequest, namespace: string, datasource: DatasourceDto): Promise<IStorageVectorDataSource>;
+    protected deleteDatasource(acRequest: AccessRequest, namespace: string, datasourceId: string): Promise<void>;
+    protected listDatasources(acRequest: AccessRequest, namespace: string): Promise<IStorageVectorDataSource[]>;
+    protected getDatasource(acRequest: AccessRequest, namespace: string, datasourceId: string): Promise<IStorageVectorDataSource>;
+    private setACL;
+    private getACL;
+    private deleteACL;
+    constructNsName(candidate: AccessCandidate, name: string): string;
+}

package/dist/types/subsystems/IO/VectorDB.service/connectors/RAMVecrtorDB.class.d.ts

@@ -0,0 +1,52 @@
+import { ACL } from '@sre/Security/AccessControl/ACL.class';
+import { IAccessCandidate } from '@sre/types/ACL.types';
+import { AccessRequest } from '@sre/Security/AccessControl/AccessRequest.class';
+import { VectorDBConnector } from '../VectorDBConnector';
+import { DatasourceDto, IStorageVectorDataSource, IStorageVectorNamespace, IVectorDataSourceDto, QueryOptions, VectorsResultData } from '@sre/types/VectorDB.types';
+import { BaseEmbedding, TEmbeddings } from '../embed/BaseEmbedding';
+export type RAMVectorDBConfig = {
+    embeddings: TEmbeddings;
+};
+/**
+ * RAM Vector Database - stores everything in memory
+ * Data structure:
+ * - vectors: namespace -> VectorData[]
+ * - namespaces: namespace -> IStorageVectorNamespace
+ * - datasources: namespace -> datasourceId -> IStorageVectorDataSource
+ * - acls: resourceId -> IACL
+ */
+export declare class RAMVectorDB extends VectorDBConnector {
+    protected _settings: RAMVectorDBConfig;
+    name: string;
+    id: string;
+    private accountConnector;
+    private vectors;
+    private namespaces;
+    private datasources;
+    private acls;
+    embedder: BaseEmbedding;
+    constructor(_settings: RAMVectorDBConfig);
+    getResourceACL(resourceId: string, candidate: IAccessCandidate): Promise<ACL>;
+    protected createNamespace(acRequest: AccessRequest, namespace: string, metadata?: {
+        [key: string]: any;
+    }): Promise<void>;
+    protected namespaceExists(acRequest: AccessRequest, namespace: string): Promise<boolean>;
+    protected getNamespace(acRequest: AccessRequest, namespace: string): Promise<IStorageVectorNamespace>;
+    protected listNamespaces(acRequest: AccessRequest): Promise<IStorageVectorNamespace[]>;
+    protected deleteNamespace(acRequest: AccessRequest, namespace: string): Promise<void>;
+    protected search(acRequest: AccessRequest, namespace: string, query: string | number[], options?: QueryOptions): Promise<VectorsResultData>;
+    protected insert(acRequest: AccessRequest, namespace: string, sourceWrapper: IVectorDataSourceDto | IVectorDataSourceDto[]): Promise<string[]>;
+    protected delete(acRequest: AccessRequest, namespace: string, id: string | string[]): Promise<void>;
+    protected createDatasource(acRequest: AccessRequest, namespace: string, datasource: DatasourceDto): Promise<IStorageVectorDataSource>;
+    protected deleteDatasource(acRequest: AccessRequest, namespace: string, datasourceId: string): Promise<void>;
+    protected listDatasources(acRequest: AccessRequest, namespace: string): Promise<IStorageVectorDataSource[]>;
+    protected getDatasource(acRequest: AccessRequest, namespace: string, datasourceId: string): Promise<IStorageVectorDataSource>;
+    /**
+     * Calculate cosine similarity between two vectors
+     */
+    private cosineSimilarity;
+    /**
+     * Split text into chunks with overlap
+     */
+    private splitTextIntoChunks;
+}

package/dist/types/subsystems/IO/VectorDB.service/embed/BaseEmbedding.d.ts

@@ -0,0 +1,53 @@
+import { IVectorDataSourceDto, Source } from '@sre/types/VectorDB.types';
+import { SupportedProviders, SupportedModels } from './index';
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+export type TEmbeddings = {
+    provider: SupportedProviders;
+    model: SupportedModels[SupportedProviders];
+    dimensions?: number;
+    timeout?: number;
+    chunkSize?: number;
+    stripNewLines?: boolean;
+    maxConcurrency?: number;
+    credentials?: {
+        apiKey: string;
+    };
+};
+type SupportedSources = 'text' | 'vector' | 'url';
+export declare abstract class BaseEmbedding {
+    model: string;
+    modelName: string;
+    chunkSize: number;
+    stripNewLines: boolean;
+    dimensions?: number;
+    timeout?: number;
+    maxConcurrency?: number;
+    constructor(fields?: Partial<TEmbeddings>);
+    /**
+     * Embed multiple texts and return their vector representations
+     */
+    abstract embedTexts(texts: string[], candidate: AccessCandidate): Promise<number[][]>;
+    /**
+     * Embed a single text and return its vector representation
+     */
+    abstract embedText(text: string, candidate: AccessCandidate): Promise<number[]>;
+    /**
+     * Utility method to chunk arrays into smaller batches
+     */
+    protected chunkArr<T>(arr: T[], sizePerChunk: number): T[][];
+    /**
+     * Utility method to process multiple texts based on stripNewLines setting
+     */
+    protected processTexts(texts: string[]): string[];
+    detectSourceType(source: Source): SupportedSources | 'unknown';
+    transformSource(source: IVectorDataSourceDto[], sourceType: SupportedSources, candidate: AccessCandidate): IVectorDataSourceDto[] | Promise<{
+        source: number[];
+        metadata: {
+            text: string;
+            user?: string;
+        };
+        id: string;
+    }[]>;
+    get dummyVector(): number[];
+}
+export {};

package/dist/types/subsystems/IO/VectorDB.service/embed/OpenAIEmbedding.d.ts

@@ -0,0 +1,16 @@
+import OpenAI, { type ClientOptions, OpenAI as OpenAIClient } from 'openai';
+import { BaseEmbedding, TEmbeddings } from './BaseEmbedding';
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+export declare class OpenAIEmbeds extends BaseEmbedding {
+    private settings?;
+    protected client: OpenAIClient;
+    protected clientConfig: ClientOptions;
+    static models: string[];
+    canSpecifyDimensions: boolean;
+    constructor(settings?: Partial<TEmbeddings>);
+    embedTexts(texts: string[], candidate: AccessCandidate): Promise<number[][]>;
+    embedText(text: string, candidate: AccessCandidate): Promise<number[]>;
+    protected embed(request: OpenAIClient.EmbeddingCreateParams, candidate: AccessCandidate): Promise<OpenAI.Embeddings.CreateEmbeddingResponse & {
+        _request_id?: string | null;
+    }>;
+}

package/dist/types/subsystems/IO/VectorDB.service/embed/index.d.ts

@@ -0,0 +1,16 @@
+import { OpenAIEmbeds } from './OpenAIEmbedding';
+import { TEmbeddings } from './BaseEmbedding';
+declare const supportedProviders: {
+    readonly OpenAI: {
+        readonly embedder: typeof OpenAIEmbeds;
+        readonly models: string[];
+    };
+};
+export type SupportedProviders = keyof typeof supportedProviders;
+export type SupportedModels = {
+    [K in SupportedProviders]: (typeof supportedProviders)[K]['models'][number];
+};
+export declare class EmbeddingsFactory {
+    static create(provider: SupportedProviders, config: TEmbeddings): OpenAIEmbeds;
+}
+export {};

package/dist/types/subsystems/IO/VectorDB.service/index.d.ts

@@ -0,0 +1,4 @@
+import { ConnectorServiceProvider } from '@sre/Core/ConnectorsService';
+export declare class VectorDBService extends ConnectorServiceProvider {
+    register(): void;
+}

package/dist/types/subsystems/LLMManager/LLM.helper.d.ts

@@ -0,0 +1,125 @@
+import { type TLLMMessageBlock } from '@sre/types/LLM.types';
+export declare class LLMHelper {
+    /**
+     * Checks if the given array of messages contains a system message.
+     *
+     * @param {any} messages - The array of messages to check.
+     * @returns {boolean} True if a system message is found, false otherwise.
+     *
+     * @example
+     * const messages = [
+     *   { role: 'user', content: 'Hello' },
+     *   { role: 'system', content: 'You are a helpful assistant' }
+     * ];
+     * const hasSystem = LLMHelper.hasSystemMessage(messages);
+     * console.log(hasSystem); // true
+     */
+    static hasSystemMessage(messages: any): boolean;
+    /**
+     * Separates system messages from other messages in an array of LLM message blocks.
+     *
+     * @param {TLLMMessageBlock[]} messages - The array of message blocks to process.
+     * @returns {Object} An object containing the system message (if any) and an array of other messages.
+     * @property {TLLMMessageBlock | {}} systemMessage - The first system message found, or an empty object if none.
+     * @property {TLLMMessageBlock[]} otherMessages - An array of all non-system messages.
+     *
+     * @example
+     * const messages = [
+     *   { role: 'system', content: 'You are a helpful assistant' },
+     *   { role: 'user', content: 'Hello' },
+     *   { role: 'assistant', content: 'Hi there!' }
+     * ];
+     * const { systemMessage, otherMessages } = LLMHelper.separateSystemMessages(messages);
+     * console.log(systemMessage); // { role: 'system', content: 'You are a helpful assistant' }
+     * console.log(otherMessages); // [{ role: 'user', content: 'Hello' }, { role: 'assistant', content: 'Hi there!' }]
+     */
+    static separateSystemMessages(messages: TLLMMessageBlock[]): {
+        systemMessage: TLLMMessageBlock | {};
+        otherMessages: TLLMMessageBlock[];
+    };
+    /**
+     * Counts the total number of tokens in a vision prompt, including both text and image tokens.
+     *
+     * @param {any} prompt - The vision prompt object containing text and image items.
+     * @returns {Promise<number>} A promise that resolves to the total number of tokens in the prompt.
+     *
+     * @description
+     * This method processes a vision prompt by:
+     * 1. Counting tokens in the text portion of the prompt.
+     * 2. Calculating tokens for each image in the prompt based on its dimensions.
+     * 3. Summing up text and image tokens to get the total token count.
+     *
+     * @example
+     * const prompt = [
+     *   { type: 'text', text: 'Describe this image:' },
+     *   { type: 'image_url', image_url: { url: 'https://example.com/image.jpg' } }
+     * ];
+     * const tokenCount = await countVisionPromptTokens(prompt);
+     * console.log(tokenCount); // e.g., 150
+     */
+    static countVisionPromptTokens(prompt: any): Promise<number>;
+    /**
+     * Retrieves the dimensions (width and height) of an image from a given URL or base64 encoded string.
+     *
+     * @param {string} imageUrl - The URL or base64 encoded string of the image.
+     * @returns {Promise<{ width: number; height: number }>} A promise that resolves to an object containing the width and height of the image.
+     * @throws {Error} If the provided imageUrl is invalid or if there's an error retrieving the image dimensions.
+     *
+     * @example
+     * // Using a URL
+     * const dimensions = await getImageDimensions('https://example.com/image.jpg');
+     * console.log(dimensions); // { width: 800, height: 600 }
+     *
+     * @example
+     * // Using a base64 encoded string
+     * const dimensions = await getImageDimensions('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACklEQVR4nGMAAQAABQABDQottAAAAABJRU5ErkJggg==');
+     * console.log(dimensions); // { width: 1, height: 1 }
+     */
+    static getImageDimensions(imageUrl: string): Promise<{
+        width: number;
+        height: number;
+    }>;
+    /**
+     * Calculates the number of tokens required to process an image based on its dimensions and detail mode.
+     *
+     * @param {number} width - The width of the image in pixels.
+     * @param {number} height - The height of the image in pixels.
+     * @param {string} detailMode - The detail mode for processing the image. Defaults to 'auto'.
+     * @returns {number} The number of tokens required to process the image.
+     *
+     * @description
+     * This method estimates the token count for image processing based on the image dimensions and detail mode.
+     * It uses a tiling approach to calculate the token count, scaling the image if necessary.
+     *
+     * - If detailMode is 'low', it returns a fixed token count of 85.
+     * - For other modes, it calculates based on the image dimensions:
+     *   - Scales down images larger than 2048 pixels in any dimension.
+     *   - Adjusts the scaled dimension to fit within a 768x1024 aspect ratio.
+     *   - Calculates the number of 512x512 tiles needed to cover the image.
+     *   - Returns the total token count based on the number of tiles.
+     *
+     * @example
+     * const tokenCount = countImageTokens(1024, 768);
+     * console.log(tokenCount); // Outputs the calculated token count
+     */
+    static countImageTokens(width: number, height: number, detailMode?: string): number;
+    /**
+     * Removes duplicate user messages from the beginning and end of the messages array.
+     *
+     * This method checks if there are two consecutive user messages at the start or end of the array
+     *
+     * @param {Array<{ role: string; content: string }>} messages - The array of message objects to process.
+     *
+     * @example
+     * const messages = [
+     *   { role: 'user', content: 'Hello' },
+     *   { role: 'user', content: 'Hello' },
+     *   { role: 'assistant', content: 'Hi there!' }
+     * ];
+     * LLMHelper.removeDuplicateUserMessages(messages);
+     * console.log(messages); // [{ role: 'user', content: 'Hello' }, { role: 'assistant', content: 'Hi there!' }]
+     *
+     * @returns {TLLMMessageBlock[]} The modified array of message objects.
+     */
+    static removeDuplicateUserMessages(messages: TLLMMessageBlock[]): TLLMMessageBlock[];
+}

package/dist/types/subsystems/LLMManager/LLM.inference.d.ts

@@ -0,0 +1,36 @@
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+import { LLMConnector } from './LLM.service/LLMConnector';
+import { TLLMModel } from '@sre/types/LLM.types';
+import { IAgent } from '@sre/types/Agent.types';
+import { TLLMParams } from '@sre/types/LLM.types';
+type TPromptParams = {
+    query?: string;
+    contextWindow?: any[];
+    files?: any[];
+    params: TLLMParams;
+};
+export declare class LLMInference {
+    private model;
+    private llmConnector;
+    private modelProviderReq;
+    teamId?: string;
+    static getInstance(model: string | TLLMModel, candidate: AccessCandidate): Promise<LLMInference>;
+    static user(candidate: AccessCandidate): any;
+    get connector(): LLMConnector;
+    prompt({ query, contextWindow, files, params }: TPromptParams): Promise<any>;
+    promptStream({ query, contextWindow, files, params }: TPromptParams): Promise<any>;
+    imageGenRequest({ query, files, params }: TPromptParams): Promise<any>;
+    imageEditRequest({ query, files, params }: TPromptParams): Promise<any>;
+    streamRequest(params: any, agent: string | IAgent): Promise<any>;
+    multimodalStreamRequest(params: any, fileSources: any, agent: string | IAgent): Promise<any>;
+    multimodalStreamRequestLegacy(prompt: any, files: string[], config: any, agent: string | IAgent): Promise<any>;
+    /**
+     * Get the context window for the given messages
+     * @param _messages - The messages to get the context window for (the messages are in smythos generic format)
+     * @param maxTokens - The maximum number of tokens to use for the context window
+     * @param maxOutputTokens - The maximum number of tokens to use for the output
+     * @returns The context window for the given messages
+     */
+    getContextWindow(systemPrompt: string, _messages: any[], maxTokens: number, maxOutputTokens?: number): Promise<any[]>;
+}
+export {};

package/dist/types/subsystems/LLMManager/LLM.service/LLMConnector.d.ts

@@ -0,0 +1,62 @@
+import { Connector } from '@sre/Core/Connector.class';
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+import { TLLMConnectorParams, TLLMMessageBlock, TLLMToolResultMessageBlock, ToolData, APIKeySource, ILLMRequestFuncParams, TLLMChatResponse, TLLMRequestBody } from '@sre/types/LLM.types';
+import EventEmitter from 'events';
+import { Readable } from 'stream';
+export interface ILLMConnectorRequest {
+    request(params: TLLMConnectorParams): Promise<TLLMChatResponse>;
+    streamRequest(params: TLLMConnectorParams): Promise<EventEmitter>;
+    imageGenRequest({ acRequest, body, context }: ILLMRequestFuncParams): Promise<any>;
+    imageEditRequest({ acRequest, body, context }: ILLMRequestFuncParams): Promise<any>;
+}
+export declare class LLMStream extends Readable {
+    private dataQueue;
+    private toolsData;
+    private hasData;
+    isReading: boolean;
+    constructor(options?: any);
+    _read(size: any): void;
+    enqueueData(data: any): void;
+    endStream(): void;
+}
+export declare abstract class LLMConnector extends Connector {
+    abstract name: string;
+    protected abstract request({ acRequest, body, context }: ILLMRequestFuncParams): Promise<TLLMChatResponse>;
+    protected abstract streamRequest({ acRequest, body, context }: ILLMRequestFuncParams): Promise<EventEmitter>;
+    protected abstract webSearchRequest({ acRequest, body, context }: ILLMRequestFuncParams): Promise<EventEmitter>;
+    protected abstract reqBodyAdapter(params: TLLMConnectorParams): Promise<TLLMRequestBody>;
+    protected abstract reportUsage(usage: any, metadata: {
+        modelEntryName: string;
+        keySource: APIKeySource;
+        agentId: string;
+        teamId: string;
+    }): any;
+    protected imageGenRequest({ acRequest, body, context }: ILLMRequestFuncParams): Promise<any>;
+    protected imageEditRequest({ acRequest, body, context }: ILLMRequestFuncParams): Promise<any>;
+    private vaultConnector;
+    requester(candidate: AccessCandidate): ILLMConnectorRequest;
+    enhancePrompt(prompt: string, config: any): string;
+    postProcess(response: string): any;
+    formatToolsConfig({ type, toolDefinitions, toolChoice }: {
+        type?: string;
+        toolDefinitions: any;
+        toolChoice?: string;
+    }): void;
+    transformToolMessageBlocks({ messageBlock, toolsData, }: {
+        messageBlock: TLLMMessageBlock;
+        toolsData: ToolData[];
+    }): TLLMToolResultMessageBlock[];
+    getConsistentMessages(messages: TLLMMessageBlock[]): TLLMMessageBlock[];
+    private prepareParams;
+    private formatParamValues;
+    /**
+     * Retrieves the team ID associated with the given access candidate
+     * @param candidate - The access candidate whose team ID needs to be retrieved
+     * @returns Promise<string> - The unique identifier of the team associated with the candidate
+     * @throws {Error} If the Account Connector service is unavailable or cannot be accessed
+     * @throws {Error} If the candidate's team cannot be retrieved
+     * @private
+     * @remarks This method is used internally to determine the team context for custom LLM operations
+     */
+    private getTeamId;
+}

package/dist/types/subsystems/LLMManager/LLM.service/LLMCredentials.helper.d.ts

@@ -0,0 +1,3 @@
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+import { TCustomLLMModel, TLLMModel } from '@sre/types/LLM.types';
+export declare function getLLMCredentials(candidate: AccessCandidate, modelInfo: TLLMModel | TCustomLLMModel): Promise<any>;