@smythos/sre 1.5.1 → 1.5.4

package/src/subsystems/LLMManager/ModelsProvider.service/ModelsProviderConnector.ts

@@ -0,0 +1,281 @@
+import { ConnectorService } from '@sre/Core/ConnectorsService';
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+import { AccessRequest } from '@sre/Security/AccessControl/AccessRequest.class';
+import { ACL } from '@sre/Security/AccessControl/ACL.class';
+import { SecureConnector } from '@sre/Security/SecureConnector.class';
+import { IAccessCandidate } from '@sre/types/ACL.types';
+import { TCustomLLMModel, TLLMCredentials, TLLMModel, TLLMModelsList, TLLMProvider } from '@sre/types/LLM.types';
+import { customModels } from '../custom-models';
+import { LocalCache } from '@sre/helpers/LocalCache.helper';
+
+export interface IModelsProviderRequest {
+    getModels(): Promise<any>;
+    getMaxContextTokens(model: string, hasAPIKey?: boolean): Promise<number>;
+    addModels(models: TLLMModelsList): Promise<void>;
+    getModelInfo(model: string | TLLMModel | TCustomLLMModel, hasAPIKey?: boolean): Promise<TLLMModel>;
+    getModelId(model: string | TLLMModel | TCustomLLMModel): Promise<string>;
+    getProvider(model: string | TLLMModel | TCustomLLMModel): Promise<string>;
+    isStandardLLM(model: string | TLLMModel | TCustomLLMModel): Promise<boolean>;
+    adjustMaxCompletionTokens(model: string | TLLMModel | TCustomLLMModel, maxCompletionTokens: number, hasAPIKey?: boolean): Promise<number>;
+    getMaxContextTokens(model: string | TLLMModel | TCustomLLMModel, hasAPIKey?: boolean): Promise<number>;
+    getMaxCompletionTokens(model: string | TLLMModel | TCustomLLMModel, hasAPIKey?: boolean): Promise<number>;
+    validateTokensLimit({
+        model,
+        promptTokens,
+        completionTokens,
+        hasAPIKey,
+    }: {
+        model: string | TLLMModel | TCustomLLMModel;
+        promptTokens: number;
+        completionTokens: number;
+        hasAPIKey?: boolean;
+    }): Promise<void>;
+}
+
+export abstract class ModelsProviderConnector extends SecureConnector {
+    protected static localCache = new LocalCache();
+    public abstract getResourceACL(resourceId: string, candidate: IAccessCandidate): Promise<ACL>;
+    public abstract getModels(acRequest: AccessRequest): Promise<TLLMModelsList>;
+    public abstract addModels(acRequest: AccessRequest, models: TLLMModelsList): Promise<void>;
+
+    public requester(candidate: AccessCandidate): IModelsProviderRequest {
+        const cacheKey = `ModelsProviderConnector:${candidate.toString()}`;
+        if (ModelsProviderConnector.localCache.has(cacheKey)) {
+            //update the TTL every time the requester is called
+            return ModelsProviderConnector.localCache.get(cacheKey, 60 * 60 * 1000) as IModelsProviderRequest;
+        }
+
+        let teamModels = null;
+        let customModels = null;
+
+        const loadTeamModels = async () => {
+            if (!teamModels) {
+                try {
+                    const builtinmodels = await this.getModels(candidate.readRequest);
+                    customModels = await this.getCustomModels(candidate);
+                    teamModels = { ...builtinmodels, ...customModels };
+                } catch (error) {
+                    return null;
+                }
+            }
+            return teamModels;
+        };
+        loadTeamModels();
+
+        const instance: IModelsProviderRequest = {
+            getModels: async () => {
+                return await loadTeamModels();
+            },
+            addModels: async (models: TLLMModelsList) => {
+                return await this.addModels(candidate.readRequest, models);
+            },
+            getModelInfo: async (model: string | TLLMModel | TCustomLLMModel, hasAPIKey: boolean = false) => {
+                const teamModels = typeof model === 'string' ? await loadTeamModels() : {};
+                const modelInfo = await this.getModelInfo(candidate.readRequest, teamModels, model, hasAPIKey);
+                return modelInfo;
+            },
+
+            getModelId: async (model: string | TLLMModel | TCustomLLMModel) => {
+                const teamModels = typeof model === 'string' ? await loadTeamModels() : {};
+                return this.getModelId(candidate.readRequest, teamModels, model);
+            },
+            getProvider: async (model: string | TLLMModel | TCustomLLMModel) => {
+                const teamModels = typeof model === 'string' ? await loadTeamModels() : {};
+                return this.getProvider(candidate.readRequest, teamModels, model);
+            },
+            isStandardLLM: async (model: string | TLLMModel | TCustomLLMModel) => {
+                const teamModels = typeof model === 'string' ? await loadTeamModels() : {};
+                const modelInfo = await this.getModelInfo(candidate.readRequest, teamModels, model);
+                return !modelInfo.isCustomLLM;
+            },
+            adjustMaxCompletionTokens: async (
+                model: string | TLLMModel | TCustomLLMModel,
+                maxCompletionTokens: number,
+                hasAPIKey: boolean = false,
+            ) => {
+                const teamModels = typeof model === 'string' ? await loadTeamModels() : {};
+                const modelInfo = await this.getModelInfo(candidate.readRequest, teamModels, model, hasAPIKey);
+                return Math.min(maxCompletionTokens || 512, modelInfo?.completionTokens || modelInfo?.tokens || maxCompletionTokens || 512);
+            },
+            getMaxContextTokens: async (model: string | TLLMModel | TCustomLLMModel, hasAPIKey: boolean = false) => {
+                const teamModels = typeof model === 'string' ? await loadTeamModels() : {};
+                const modelInfo = await this.getModelInfo(candidate.readRequest, teamModels, model, hasAPIKey);
+                return modelInfo?.tokens || 1024;
+            },
+            getMaxCompletionTokens: async (model: string | TLLMModel | TCustomLLMModel, hasAPIKey: boolean = false) => {
+                const teamModels = typeof model === 'string' ? await loadTeamModels() : {};
+                const modelInfo = await this.getModelInfo(candidate.readRequest, teamModels, model, hasAPIKey);
+                return modelInfo?.completionTokens || modelInfo?.tokens || 512;
+            },
+            validateTokensLimit: async ({
+                model,
+                promptTokens,
+                completionTokens,
+                hasAPIKey,
+            }: {
+                model: string | TLLMModel | TCustomLLMModel;
+                promptTokens: number;
+                completionTokens: number;
+                hasAPIKey: boolean;
+            }) => {
+                const teamModels = typeof model === 'string' ? await loadTeamModels() : {};
+                const modelInfo = await this.getModelInfo(candidate.readRequest, teamModels, model, hasAPIKey);
+                const allowedContextTokens = modelInfo?.tokens;
+                const totalTokens = promptTokens + completionTokens;
+
+                const teamAPIKeyExceededMessage = `This models' maximum content length is ${allowedContextTokens} tokens. (This is the sum of your prompt with all variables and the maximum output tokens you've set in Advanced Settings) However, you requested approx ${totalTokens} tokens (${promptTokens} in the prompt, ${completionTokens} in the output). Please reduce the length of either the input prompt or the Maximum output tokens.`;
+                const noAPIKeyExceededMessage = `Input exceeds max tokens limit of ${allowedContextTokens}. Please add your API key and select Personal tagged models to unlock full length.`;
+
+                if (totalTokens > allowedContextTokens) {
+                    throw new Error(hasAPIKey ? teamAPIKeyExceededMessage : noAPIKeyExceededMessage);
+                }
+            },
+        };
+        ModelsProviderConnector.localCache.set(cacheKey, instance, 60 * 60 * 1000); // cache for 1 hour
+        return instance;
+    }
+
+    protected async getModelInfo(
+        acRequest: AccessRequest,
+        models: TLLMModelsList,
+        model: string | TLLMModel | TCustomLLMModel,
+        hasAPIKey: boolean = false,
+    ): Promise<TLLMModel> {
+        //model can be passed directly, in which case we do not need to look it up in the models list
+        if (typeof model === 'object' && model.modelId) {
+            return model;
+        }
+
+        //model can be passed as a string, in which case we need to look it up in the models list
+
+        const modelId = await this.getModelId(acRequest, models, model);
+        const alias = models?.[model as string]?.alias;
+        const aliasModelInfo = models?.[alias];
+
+        const modelInfo = models?.[model as string];
+
+        const aliasKeyOptions = aliasModelInfo && hasAPIKey ? aliasModelInfo?.keyOptions : null;
+
+        const modelKeyOptions = modelInfo?.keyOptions || aliasKeyOptions;
+
+        return { ...aliasModelInfo, ...modelInfo, ...aliasKeyOptions, ...modelKeyOptions, modelId };
+    }
+
+    protected async getModelId(acRequest: AccessRequest, models: TLLMModelsList, model: string | TLLMModel | TCustomLLMModel): Promise<string> {
+        //model can be passed directly, in which case we do not need to look it up in the models list
+        if (typeof model === 'object' && model.modelId) {
+            return model.modelId;
+        }
+
+        //model can be passed as a string, in which case we need to look it up in the models list
+        const modelId = models?.[model as string]?.modelId || (model as string);
+        const alias = models?.[model as string]?.alias;
+        if (alias) {
+            const aliasModelId = models?.[alias]?.modelId || alias || (model as string);
+            return aliasModelId;
+        }
+
+        return modelId;
+    }
+
+    // public static async validateTokensLimit({
+    //     model,
+    //     promptTokens,
+    //     completionTokens,
+    //     hasAPIKey = false,
+    // }: {
+    //     model: string;
+    //     promptTokens: number;
+    //     completionTokens: number;
+    //     hasAPIKey?: boolean;
+    // }): Promise<void> {
+
+    //     const allowedContextTokens = this.getMaxContextTokens(model, hasAPIKey);
+    //     const totalTokens = promptTokens + completionTokens;
+
+    //     const teamAPIKeyExceededMessage = `This models' maximum content length is ${allowedContextTokens} tokens. (This is the sum of your prompt with all variables and the maximum output tokens you've set in Advanced Settings) However, you requested approx ${totalTokens} tokens (${promptTokens} in the prompt, ${completionTokens} in the output). Please reduce the length of either the input prompt or the Maximum output tokens.`;
+    //     const noAPIKeyExceededMessage = `Input exceeds max tokens limit of ${allowedContextTokens}. Please add your API key to unlock full length.`;
+
+    //     if (totalTokens > allowedContextTokens) {
+    //         throw new Error(hasAPIKey ? teamAPIKeyExceededMessage : noAPIKeyExceededMessage);
+    //     }
+    // }
+
+    protected async getProvider(acRequest: AccessRequest, models: TLLMModelsList, model: string | TLLMModel | TCustomLLMModel): Promise<string> {
+        //model can be passed directly, in which case we do not need to look it up in the models list
+        if (typeof model === 'object' && model.provider) {
+            return model.provider;
+        }
+
+        //model can be passed as a string, in which case we need to look it up in the models list
+
+        const modelId = await this.getModelId(acRequest, models, model);
+
+        return models?.[modelId]?.provider || models?.[model as string]?.provider || models?.[modelId]?.llm || models?.[model as string]?.llm;
+    }
+    protected async getCustomModels(candidate: IAccessCandidate): Promise<Record<string, any>> {
+        const models = {};
+        const settingsKey = 'custom-llm';
+
+        try {
+            const accountConnector = ConnectorService.getAccountConnector();
+            const team = await accountConnector.requester(candidate as AccessCandidate).getTeam();
+
+            const teamSettings = await accountConnector.team(team).getTeamSetting(settingsKey);
+            const savedCustomModelsData = JSON.parse(teamSettings || '{}') as Record<string, any>;
+
+            for (const [entryId, entry] of Object.entries(savedCustomModelsData)) {
+                const foundationModel = entry.settings.foundationModel;
+                const customModel = entry.settings.customModel;
+                const supportsSystemPrompt = customModels[foundationModel]?.supportsSystemPrompt || entry.settings.supportsSystemPrompt;
+                const customModelData = customModels[foundationModel] || {};
+
+                let credentials = null;
+                switch (entry.provider) {
+                    case TLLMProvider.Bedrock:
+                        credentials = TLLMCredentials.BedrockVault;
+                        break;
+                    case TLLMProvider.VertexAI:
+                        credentials = TLLMCredentials.VertexAIVault;
+                        break;
+                    default:
+                        credentials = TLLMCredentials.Internal;
+                        break;
+                }
+                models[entry.name] = {
+                    label: entry.name,
+                    modelId: customModel || foundationModel,
+                    provider: entry.provider,
+                    features: entry.features?.map((feature) => {
+                        switch (feature) {
+                            case 'text-completion':
+                                return 'text';
+                            case 'tool-use':
+                                return 'tools';
+                            default:
+                                return feature;
+                        }
+                    }),
+                    tags: Array.isArray(entry?.tags) ? ['Enterprise', ...entry?.tags] : ['Enterprise'],
+                    tokens: customModelData?.tokens ?? 100000,
+                    completionTokens: customModelData?.completionTokens ?? 4096,
+                    enabled: true,
+
+                    id: entryId,
+                    name: entry.name,
+                    alias: foundationModel,
+                    llm: entry.provider,
+                    components: customModelData?.components ?? [],
+                    isCustomLLM: true,
+                    supportsSystemPrompt,
+                    settings: entry.settings,
+                    credentials,
+                };
+            }
+
+            return models;
+        } catch (error) {
+            return {};
+        }
+    }
+}

package/src/subsystems/LLMManager/ModelsProvider.service/connectors/SmythModelsProvider.class.ts

@@ -0,0 +1,229 @@
+import { AccessRequest } from '@sre/Security/AccessControl/AccessRequest.class';
+import { models } from '@sre/LLMManager/models';
+import { ModelsProviderConnector } from '../ModelsProviderConnector';
+import { IAccessCandidate } from '@sre/types/ACL.types';
+import { TAccessLevel } from '@sre/types/ACL.types';
+import { TAccessRole } from '@sre/types/ACL.types';
+import { ConnectorService } from '@sre/Core/ConnectorsService';
+import { ACL } from '@sre/Security/AccessControl/ACL.class';
+import { SecureConnector } from '@sre/Security/SecureConnector.class';
+import { TLLMModel, TLLMModelsList } from '@sre/types/LLM.types';
+import { Logger } from '@sre/helpers/Log.helper';
+import { debounce } from '@sre/utils/general.utils';
+import chokidar from 'chokidar';
+import fs from 'fs/promises';
+import path from 'path';
+
+const console = Logger('SmythModelsProvider');
+
+type SmythModelsProviderConfig = {
+    /**
+     * The models to be used.
+     *
+     * If a string is provided, it will be used as the directory name to load the models from.
+     * If a TLLMModelsList is provided, it will be used as the models to be used.
+     *
+     */
+    models?: string | TLLMModelsList;
+
+    /**
+     * The mode to be used.
+     *
+     * If 'append' is used, the models will be appended to the existing models.
+     * If 'replace' is used, the existing models will be replaced with the new models.
+     */
+    mode?: 'merge' | 'replace';
+};
+
+export class SmythModelsProvider extends ModelsProviderConnector {
+    public name = 'SmythModelsProvider';
+
+    private models: TLLMModelsList;
+
+    constructor(protected _settings?: SmythModelsProviderConfig) {
+        super(_settings);
+
+        this.models = JSON.parse(JSON.stringify(models));
+        if (typeof this._settings.models === 'string') {
+            this.initDirWatcher(this._settings.models);
+        } else if (typeof this._settings.models === 'object') {
+            if (this._settings.mode === 'merge') this.models = { ...this.models, ...(this._settings.models as TLLMModelsList) };
+            else this.models = this._settings.models as TLLMModelsList;
+            this.started = true;
+        } else {
+            this.started = true;
+        }
+    }
+    public async start() {
+        super.start();
+    }
+
+    @SecureConnector.AccessControl
+    public async addModels(acRequest: AccessRequest, models: TLLMModelsList): Promise<void> {
+        await this.ready();
+        const validModels = (await this.getValidModels(models)) || {};
+        if (Object.keys(validModels).length > 0) {
+            this.models = { ...this.models, ...validModels };
+        }
+    }
+
+    @SecureConnector.AccessControl
+    public async getModels(acRequest: AccessRequest): Promise<any> {
+        await this.ready();
+
+        return this.models;
+    }
+
+    public async getResourceACL(resourceId: string, candidate: IAccessCandidate) {
+        const accountConnector = ConnectorService.getAccountConnector();
+
+        const acl = new ACL();
+        //give read access to the candidate by default
+        acl.addAccess(candidate.role, candidate.id, TAccessLevel.Read);
+
+        return acl;
+    }
+
+    private async reindexModels(dir: string) {
+        try {
+            console.debug(`Reindexing models from directory: ${dir}`);
+
+            // Scan directory for models and get them as an object
+            const scannedModels = await this.scanDirectoryForModels(dir);
+
+            // Apply models based on settings mode or default behavior
+            if (this._settings?.mode === 'merge') {
+                this.models = { ...this.models, ...scannedModels };
+            } else {
+                // Default behavior: reset to base models first, then add scanned models
+                this.models = scannedModels;
+            }
+
+            SmythModelsProvider.localCache.clear();
+
+            console.debug(`Successfully reindexed models. Total models: ${Object.keys(this.models).length}`);
+        } catch (error) {
+            console.error(`Error reindexing models from directory "${dir}":`, error);
+        }
+    }
+
+    private async scanDirectoryForModels(dir: string): Promise<TLLMModelsList> {
+        const scannedModels: TLLMModelsList = {};
+
+        try {
+            const entries = await fs.readdir(dir, { withFileTypes: true });
+
+            for (const entry of entries) {
+                const fullPath = path.join(dir, entry.name);
+
+                if (entry.isDirectory()) {
+                    // Recursively scan subdirectories and merge results
+                    const subDirModels = await this.scanDirectoryForModels(fullPath);
+                    Object.assign(scannedModels, subDirModels);
+                } else if (entry.isFile() && entry.name.endsWith('.json')) {
+                    // Process JSON files and merge results
+                    const fileContent = await fs.readFile(fullPath, 'utf-8');
+                    const modelData = JSON.parse(fileContent);
+                    const validModels = await this.getValidModels(modelData);
+                    Object.assign(scannedModels, validModels);
+                }
+            }
+        } catch (error) {
+            console.warn(`Error scanning directory "${dir}":`, error);
+        }
+
+        return scannedModels;
+    }
+
+    private async getValidModels(modelData: any): Promise<TLLMModelsList> {
+        const validModels: TLLMModelsList = {};
+
+        try {
+            // Check if it's a single model or an object of models
+            if (modelData.modelId) {
+                // Single model case
+                if (this.isValidSingleModel(modelData)) {
+                    validModels[modelData.modelId] = modelData as TLLMModel;
+                    console.debug(`Loaded model: ${modelData.modelId}`);
+                } else {
+                    console.warn(`Invalid model format`, modelData);
+                }
+            } else if (typeof modelData === 'object' && !Array.isArray(modelData)) {
+                // Object of models case
+                for (const [modelId, model] of Object.entries(modelData)) {
+                    try {
+                        if (this.isValidSingleModel(model)) {
+                            validModels[modelId] = model as TLLMModel;
+                            console.debug(`Loaded model: ${modelId}`);
+                        } else {
+                            console.warn(`Invalid model format for model "${modelId}"`);
+                        }
+                    } catch (error) {
+                        console.warn(`Error processing model "${modelId}":`, error);
+                        // Continue processing other models instead of failing the whole file
+                    }
+                }
+            } else {
+                console.warn(`Invalid format (not a model or object of models)`);
+            }
+        } catch (error) {
+            console.warn(`Error loading model:`, error);
+        }
+
+        return validModels;
+    }
+
+    private isValidSingleModel(data: any): boolean {
+        // Basic validation for single model structure
+        return (
+            data && typeof data === 'object' && typeof data.modelId === 'string' && (data.provider === undefined || typeof data.provider === 'string')
+        );
+    }
+
+    private isValidModel(data: any): boolean {
+        // Keep for backward compatibility, but delegate to simpler function
+        return this.isValidSingleModel(data);
+    }
+
+    private async initDirWatcher(dir) {
+        if (!(await fs.stat(dir)).isDirectory()) {
+            console.warn(`Directory "${dir}" does not exist`);
+            return;
+        }
+
+        const debouncedReindex = debounce(this.reindexModels.bind(this, dir), 1000, {
+            leading: false,
+            trailing: true,
+            maxWait: 5000,
+        });
+
+        const watcher = chokidar.watch(dir, {
+            ignored: /(^|[\/\\])\../, // ignore dotfiles
+            persistent: true,
+            ignoreInitial: true, // Don't fire events for files that already exist
+            awaitWriteFinish: {
+                stabilityThreshold: 2000,
+                pollInterval: 100,
+            },
+        });
+
+        watcher
+            .on('add', (path) => {
+                console.debug(`File ${path} has been added`);
+                debouncedReindex();
+            })
+            .on('change', (path) => {
+                console.debug(`File ${path} has been changed`);
+                debouncedReindex();
+            })
+            .on('unlink', (path) => {
+                console.debug(`File ${path} has been removed`);
+                debouncedReindex();
+            })
+            .on('ready', () => {
+                console.debug(`Watcher ready. Performing initial scan of ${dir}`);
+                // Do initial scan once when watcher is ready
+                this.reindexModels(dir);
+            });
+    }
+}

package/src/subsystems/LLMManager/ModelsProvider.service/index.ts

@@ -0,0 +1,11 @@
+//==[ SRE: ModelsProvider ]======================
+
+import { ConnectorService, ConnectorServiceProvider } from '@sre/Core/ConnectorsService';
+import { TConnectorService } from '@sre/types/SRE.types';
+import { SmythModelsProvider } from './connectors/SmythModelsProvider.class';
+
+export class ModelsProviderService extends ConnectorServiceProvider {
+    public register() {
+        ConnectorService.register(TConnectorService.ModelsProvider, 'SmythModelsProvider', SmythModelsProvider);
+    }
+}