@smythos/sre 1.5.0 → 1.5.2

package/src/Components/Await.class.ts

@@ -0,0 +1,90 @@
+import { IAgent as Agent } from '@sre/types/Agent.types';
+import { Component } from './Component.class';
+import axios from 'axios';
+import Joi from 'joi';
+import { Async } from './Async.class';
+
+export class Await extends Component {
+    static WAITS = {};
+
+    protected configSchema = Joi.object({
+        jobs_count: Joi.number().min(1).max(100).default(1).label('Jobs Count'),
+        max_time: Joi.number().min(1).max(21600).default(1).label('Max time'),
+    });
+    constructor() {
+        super();
+    }
+    init() {}
+    async process(input, config, agent: Agent) {
+        await super.process(input, config, agent);
+
+        const logger = this.createComponentLogger(agent, config);
+        try {
+            let Results: any = {};
+            const _error = null;
+            let jobs_count = parseInt(config.data.jobs_count || 1);
+            let max_time = parseInt(config.data.max_time || 1);
+
+            const jobs = Array.isArray(input.Jobs) ? input.Jobs : [input.Jobs];
+
+            if (!Await.WAITS[agent.id]) Await.WAITS[agent.id] = {};
+            if (!Await.WAITS[agent.id][config.id]) Await.WAITS[agent.id][config.id] = {};
+            if (!Await.WAITS[agent.id][config.id][agent.agentRuntime.workflowReqId])
+                Await.WAITS[agent.id][config.id][agent.agentRuntime.workflowReqId] = [];
+
+            //add jobs to the list
+            for (let jobID of jobs) Await.WAITS[agent.id][config.id][agent.agentRuntime.workflowReqId].push(jobID);
+
+            logger.debug('Jobs', jobs);
+            logger.debug('Waiting for jobs to finish');
+
+            let promise = new Promise((resolve, reject) => {
+                let interval = setInterval(() => {
+                    if (max_time < 0) {
+                        clearInterval(interval);
+                        return resolve(true);
+                    }
+                    let done = true;
+                    let completed = 0;
+                    for (let jobID of jobs) {
+                        if (Async.JOBS?.[agent.id]?.[jobID]?.status == 'pending') {
+                            done = false;
+                            break;
+                        } else {
+                            completed++;
+                        }
+                    }
+                    if (completed >= jobs_count) {
+                        done = true;
+                    }
+
+                    if (done) {
+                        clearInterval(interval);
+                        return resolve(true);
+                    }
+
+                    max_time -= 1;
+                }, 1000);
+            });
+
+            await promise;
+            logger.debug('Jobs finished, collecting results');
+            for (let jobID of jobs) {
+                Results[jobID] = {
+                    output: Async.JOBS?.[agent.id]?.[jobID]?.result,
+                    status: Async.JOBS?.[agent.id]?.[jobID]?.status || 'unknown_job',
+                };
+            }
+            delete Await.WAITS[agent.id][config.id][agent.agentRuntime.workflowReqId];
+
+            logger.debug('Results', Results);
+            return { Results, _error, _debug: logger.output, _debug_time: logger.elapsedTime };
+        } catch (err: any) {
+            const _error = err?.response?.data || err?.message || err.toString();
+            logger.error(` Error running code \n${_error}\n`);
+            delete Await.WAITS[agent.id][config.id][agent.agentRuntime.workflowReqId];
+
+            return { Output: undefined, _error, _debug: logger.output, _debug_time: logger.elapsedTime };
+        }
+    }
+}

package/src/Components/Classifier.class.ts

@@ -0,0 +1,158 @@
+import Joi from 'joi';
+
+import { JSONContentHelper } from '@sre/helpers/JsonContent.helper';
+import { Component } from './Component.class';
+import { IAgent as Agent } from '@sre/types/Agent.types';
+import { TemplateString } from '@sre/helpers/TemplateString.helper';
+import { LLMInference } from '@sre/LLMManager/LLM.inference';
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+
+export class Classifier extends Component {
+    protected schema = {
+        name: 'Classifier',
+        settings: {
+            model: {
+                type: 'string',
+                max: 200,
+                required: true,
+            },
+            prompt: {
+                type: 'string',
+                max: 30000,
+                allow: '',
+                label: 'Prompt',
+            },
+        },
+
+        inputs: {
+            Input: {
+                type: 'Any',
+                default: true,
+            },
+        },
+    };
+    protected configSchema = Joi.object({
+        model: Joi.string().max(200).required(),
+        prompt: Joi.string().max(30000).allow('').label('Prompt'),
+    });
+    constructor() {
+        super();
+    }
+    init() {}
+    escapeJSONString(str: string) {
+        return str.replace(/\{/g, '<[<(').replace(/\}/g, ')>]>').replace(/"/g, '`');
+    }
+    unescapeJSONString(str: string) {
+        return str
+            .replace(/<\[<\(/g, '{')
+            .replace(/\)>]>/g, '}')
+            .replace(/`/g, '"');
+    }
+    async process(input, config, agent: Agent) {
+        await super.process(input, config, agent);
+        //let debugLog = agent.agentRuntime?.debug ? [] : undefined;
+        const logger = this.createComponentLogger(agent, config);
+
+        const inputCopy = JSON.parse(JSON.stringify(input));
+        for (let key in inputCopy) {
+            if (typeof inputCopy[key] === 'string') {
+                inputCopy[key] = this.escapeJSONString(inputCopy[key]);
+            } else if (typeof inputCopy[key] === 'object') {
+                inputCopy[key] = JSON.stringify(inputCopy[key]);
+                inputCopy[key] = this.escapeJSONString(inputCopy[key]);
+            }
+        }
+
+        const _input = typeof input === 'string' ? input : JSON.stringify(inputCopy, null, 2);
+        //const categories = config.outputs.map((output) => (output.name[0] != '[' ? output.name : null)).filter((e) => e);
+        const categories = {};
+        for (let con of config.outputs) categories[con.name] = con.description || '';
+
+        const outputs = {};
+        for (let con of config.outputs) {
+            outputs[con.name] = '<Boolean|String>';
+        }
+
+        const model: string = config.data.model;
+        const modelId = await agent.modelsProvider.getModelId(model);
+        //const isStandardLLM = await agent.modelsProvider.isStandardLLM(model);
+
+        logger.log(` Selected model : ${modelId || model}`);
+
+        let prompt = '';
+        const excludedKeys = ['_debug', '_error'];
+        const outputKeys = Object.keys(outputs).filter((key) => !excludedKeys.includes(key));
+
+        if (outputKeys.length > 0) {
+            const outputFormat = {};
+            outputKeys.forEach((key) => (outputFormat[key] = outputs[key]));
+
+            prompt = `${config.data.prompt}
+${_input}
+
+---
+Categories: 
+${JSON.stringify(categories, null, 2)}`;
+
+            prompt = TemplateString(prompt).parse(input).result;
+        }
+
+        logger.log(` Enhanced prompt \n${prompt}\n`);
+
+        if (!prompt) {
+            logger.error(` Missing information, Cannot run classifier`);
+
+            return { _error: 'Missing information, Cannot run classifier', _debug: logger.output };
+        }
+
+        const llmInference: LLMInference = await LLMInference.getInstance(model || 'echo', AccessCandidate.agent(agent.id));
+        if (!llmInference.connector) {
+            return {
+                _error: `The model '${model}' is not available. Please try a different one.`,
+                _debug: logger.output,
+            };
+        }
+
+        try {
+            let response = await llmInference
+                .prompt({ query: prompt, params: { ...config, agentId: agent.id } })
+                .catch((error) => ({ error: error }));
+
+            if (response?.error) {
+                const error = response?.error + ' ' + (response?.details || '');
+                logger.error(` LLM Error=`, error);
+
+                return { _error: error, _debug: logger.output };
+            }
+
+            // let parsed = parseJson(response);
+            let parsed = typeof response === 'string' ? JSONContentHelper.create(response).tryParse() : response;
+
+            for (let entry in parsed) {
+                if (!parsed[entry]) delete parsed[entry];
+                else {
+                    if (typeof parsed[entry] === 'string') {
+                        parsed[entry] = this.unescapeJSONString(parsed[entry]);
+                        // const parsedValue = parseJson(parsed[entry]);
+                        const parsedValue = JSONContentHelper.create(parsed[entry]).tryParse();
+                        if (typeof parsedValue === 'object' && !parsedValue.error) parsed[entry] = parsedValue;
+                    }
+                }
+            }
+
+            if (parsed.error) {
+                parsed._error = parsed.error;
+                logger.warn(` Post process error=${parsed.error}`);
+                delete parsed.error;
+            }
+
+            logger.log(' Classifier result\n', parsed);
+
+            parsed['_debug'] = logger.output;
+
+            return parsed;
+        } catch (error) {
+            return { _error: error.message, _debug: logger.output };
+        }
+    }
+}

package/src/Components/Component.class.ts

@@ -0,0 +1,94 @@
+import Joi from 'joi';
+import { IAgent as Agent } from '@sre/types/Agent.types';
+import { Logger } from '@sre/helpers/Log.helper';
+import { performTypeInference } from '@sre/helpers/TypeChecker.helper';
+import { hookAsync } from '@sre/Core/HookService';
+
+export type ComponentSchema = {
+    name: string;
+    settings?: Record<string, any>;
+    inputs?: Record<string, any>;
+    outputs?: Record<string, any>;
+};
+
+export class Component {
+    public hasReadOutput = false;
+    public hasPostProcess = true;
+    public alwaysActive = false; //for components like readable memories
+    public exclusive = false; //for components like writable memories : when exclusive components are active, they are processed in a run cycle bofore other components
+    protected schema: ComponentSchema = {
+        name: 'Component',
+        settings: {},
+        inputs: {},
+        //outputs: {},
+    };
+    protected configSchema;
+    constructor() {}
+    init() {}
+
+    createComponentLogger(agent: Agent, configuration: any) {
+        const logger = Logger(configuration.name || this.constructor.name, agent?.agentRuntime?.debug);
+
+        logger.on('logged', (info: { level: string; message: string }) => {
+            if (agent.sse && configuration.eventId) {
+                agent.sse.send('component', {
+                    eventId: configuration.eventId,
+                    action: 'log',
+                    name: configuration.name || this.constructor.name,
+                    title: configuration.title,
+                    logs: [{ level: info.level, message: info.message }],
+                });
+            }
+        });
+        return logger;
+    }
+
+    async validateConfig(config) {
+        if (!this.configSchema) return {};
+        if (config.data._templateVars) {
+            //Accept dynamically added template data
+            for (let tplVar in config.data._templateVars) {
+                this.configSchema = this.configSchema.append({ [tplVar]: Joi.any() });
+            }
+        }
+        const valid = await this.configSchema.validate(config.data);
+        if (valid.error) {
+            return {
+                id: config.id,
+                name: config.name,
+                _error: `Schema Validation error: ${valid?.error?.message} on component ${config.displayName}:${config.title}`,
+                _debug: `Schema Validation error: ${valid?.error?.message} on component ${config.displayName}:${config.title}`,
+            };
+        }
+
+        return {};
+    }
+
+    @hookAsync('Component.process')
+    async process(input, config, agent: Agent): Promise<any> {
+        if (agent.isKilled()) {
+            throw new Error('Agent killed');
+        }
+        const _input = await performTypeInference(input, config?.inputs, agent);
+
+        // modify the input object for component's process method
+        for (const [key, value] of Object.entries(_input)) {
+            input[key] = value;
+        }
+    }
+    async postProcess(output, config, agent: Agent): Promise<any> {
+        if (output?.result) {
+            delete output?.result?._debug;
+            if (!output?.result?._error) delete output?.result?._error;
+        }
+        return output;
+    }
+    async enable(config, agent: Agent): Promise<any> {}
+    async disable(config, agent: Agent): Promise<any> {}
+    readOutput(id, config, agent: Agent): any {
+        return null;
+    }
+    hasOutput(id, config, agent: Agent): any {
+        return false;
+    }
+}

package/src/Components/ComponentHost.class.ts

@@ -0,0 +1,38 @@
+import { Component } from './Component.class';
+import { IAgent as Agent } from '@sre/types/Agent.types';
+import { ConnectorService } from '@sre/Core/ConnectorsService';
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+
+/**
+ * This component allows running components that are not natively shiped with SRE
+ * it can be used to extend SRE components by registering custom component using ComponentService
+ */
+export class ComponentHost extends Component {
+    protected configSchema = null;
+    constructor() {
+        super();
+    }
+    init() {}
+
+    async process(input, config, agent: Agent) {
+        await super.process(input, config, agent);
+        //let debugLog = agent.agentRuntime?.debug ? [] : undefined;
+        const logger = this.createComponentLogger(agent, config);
+
+        try {
+            const componentName = config.data._component;
+            const componentConnector = ConnectorService.getComponentConnector();
+
+            const component = await componentConnector.requester(AccessCandidate.agent(agent.id)).get(componentName);
+
+            if (!component) {
+                logger.debug(`Component ${componentName} not found`);
+                return { _error: `Component ${componentName} not found`, _debug: logger.output };
+            }
+
+            return await component.process(input, config, agent);
+        } catch (error) {
+            return { _error: error.message, _debug: logger.output };
+        }
+    }
+}

package/src/Components/DataSourceCleaner.class.ts

@@ -0,0 +1,92 @@
+import { IAgent as Agent } from '@sre/types/Agent.types';
+import { Component } from './Component.class';
+import { TemplateString } from '@sre/helpers/TemplateString.helper';
+import Joi from 'joi';
+import { validateCharacterSet } from '../utils';
+import { ConnectorService } from '@sre/Core/ConnectorsService';
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+
+import { DataSourceIndexer } from './DataSourceIndexer.class';
+
+export class DataSourceCleaner extends Component {
+    protected configSchema = Joi.object({
+        namespaceId: Joi.string().max(50).allow('').label('namespace'),
+        id: Joi.string().custom(validateCharacterSet, 'custom validation characterSet').allow('').label('source identifier'),
+    });
+    constructor() {
+        super();
+    }
+    init() {}
+    async process(input, config, agent: Agent) {
+        await super.process(input, config, agent);
+        const teamId = agent.teamId;
+        const agentId = agent.id;
+        let debugOutput = agent.agentRuntime?.debug ? '== Source Indexer Log ==\n' : null;
+
+        try {
+            const configSchema = this.validateConfigData(config.data);
+            if (configSchema.error) {
+                throw new Error(`Config data validation error: ${configSchema.error}\n EXITING...`);
+            }
+
+            const outputs = {};
+            for (let con of config.outputs) {
+                if (con.default) continue;
+                outputs[con.name] = con?.description ? `<${con?.description}>` : '';
+            }
+
+            const inputSchema = this.validateInput(input);
+            if (inputSchema.error) {
+                throw new Error(`Input validation error: ${inputSchema.error}\n EXITING...`);
+            }
+
+            const namespaceId = configSchema.value.namespaceId.split('_')?.slice(1).join('_') || configSchema.value.namespaceId;
+
+            let vectorDbConnector = ConnectorService.getVectorDBConnector();
+
+            let existingnamespace = await vectorDbConnector.requester(AccessCandidate.team(teamId)).namespaceExists(namespaceId);
+            if (!existingnamespace) {
+                throw new Error(`Namespace ${namespaceId} does not exist`);
+            }
+
+            const providedId = TemplateString(config.data.id).parse(input).result;
+            const idRegex = /^[a-zA-Z0-9\-\_\.]+$/;
+            if (!idRegex.test(providedId)) {
+                throw new Error(`Invalid id. Accepted characters: 'a-z', 'A-Z', '0-9', '-', '_', '.'`);
+            }
+            debugOutput += `Searching for data source with id: ${providedId}\n`;
+
+            const dsId = DataSourceIndexer.genDsId(providedId, teamId, namespaceId);
+
+            await vectorDbConnector.requester(AccessCandidate.team(teamId)).deleteDatasource(namespaceId, dsId);
+
+            debugOutput += `Deleted data source with id: ${providedId}\n`;
+
+            return {
+                _debug: debugOutput,
+                Success: true,
+                // _error,
+            };
+        } catch (err: any) {
+            debugOutput += `Failed to delete data source: \n Error: ${err?.message}\n`;
+
+            return {
+                _debug: debugOutput,
+                _error: err?.message || "Couldn't delete data source",
+            };
+        }
+    }
+
+    validateInput(input: any) {
+        return Joi.object({}).unknown(true).validate(input);
+    }
+
+    validateConfigData(data: any) {
+        return Joi.object({
+            namespaceId: Joi.string().required(),
+            id: Joi.string().optional().allow('').allow(null),
+        })
+            .unknown(true)
+            .validate(data);
+    }
+}

package/src/Components/DataSourceIndexer.class.ts

@@ -0,0 +1,181 @@
+import { IAgent as Agent } from '@sre/types/Agent.types';
+import { Component } from './Component.class';
+import Joi from 'joi';
+import { validateCharacterSet } from '@sre/utils/validation.utils';
+import { TemplateString } from '@sre/helpers/TemplateString.helper';
+import { isUrl, detectURLSourceType } from '../utils';
+import { SmythFS } from '@sre/IO/Storage.service/SmythFS.class';
+import { ConnectorService } from '@sre/Core/ConnectorsService';
+
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+
+export class DataSourceIndexer extends Component {
+    private MAX_ALLOWED_URLS_PER_INPUT = 20;
+    protected configSchema = Joi.object({
+        namespace: Joi.string().max(50).allow(''),
+        id: Joi.string().custom(validateCharacterSet, 'id custom validation').allow('').label('source identifier'),
+        name: Joi.string().max(50).allow('').label('label'),
+        metadata: Joi.string().allow(null).allow('').max(10000).label('metadata'),
+    });
+    constructor() {
+        super();
+    }
+    init() {}
+    async process(input, config, agent: Agent) {
+        await super.process(input, config, agent);
+        const teamId = agent.teamId;
+        const agentId = agent.id;
+        let debugOutput = agent.agentRuntime?.debug ? '== Source Indexer Log ==\n' : null;
+
+        try {
+            const _config = {
+                ...config.data,
+                name: TemplateString(config.data.name).parse(input).result,
+                id: TemplateString(config.data.id).parse(input).result,
+                metadata: TemplateString(config.data.metadata).parse(input).result,
+            };
+
+            const outputs = {};
+            for (let con of config.outputs) {
+                if (con.default) continue;
+                outputs[con.name] = con?.description ? `<${con?.description}>` : '';
+            }
+
+            const namespaceId = _config.namespace.split('_').slice(1).join('_') || _config.namespace;
+            debugOutput += `[Selected namespace id] \n${namespaceId}\n\n`;
+
+            const vectorDbConnector =
+                // (await vectorDBHelper.getTeamConnector(teamId)) ||
+                ConnectorService.getVectorDBConnector();
+            const nsExists = await vectorDbConnector.requester(AccessCandidate.team(teamId)).namespaceExists(namespaceId);
+
+            if (!nsExists) {
+                const newNs = await vectorDbConnector.requester(AccessCandidate.team(teamId)).createNamespace(namespaceId);
+                debugOutput += `[Created namespace] \n${newNs}\n\n`;
+            }
+
+            const inputSchema = this.validateInput(input);
+            if (inputSchema.error) {
+                throw new Error(`Input validation error: ${inputSchema.error}\n EXITING...`);
+            }
+
+            const providedId = _config.id;
+            // const isAutoId = _config.isAutoId;
+            const idRegex = /^[a-zA-Z0-9\-\_\.]+$/;
+
+            if (!providedId) {
+                // Assign a new ID if it's set to auto-generate or not provided
+                // _config.id = crypto.randomBytes(16).toString('hex');
+                throw new Error(`Id is required`);
+            } else if (!idRegex.test(providedId)) {
+                // Validate the provided ID if it's not auto-generated
+                throw new Error(`Invalid id. Accepted characters: 'a-z', 'A-Z', '0-9', '-', '_', '.'`);
+            }
+
+            // check if the datasource already exists with the same id
+            // await this.checkForRecordDuplicate(dsId, token);
+
+            let indexRes: any = null;
+            let parsedUrlArray: string[] | null = null;
+
+            //! DISABLE URL ARRAY PARSING FOR NOW UNTIL WE HAVE A GOOD WAY TO HANDLE BULK INDEXING
+            // if ((parsedUrlArray = parseUrlArray(inputSchema.value.Source))) {
+            //     debugOutput += `STEP: Parsing input as url array\n\n`;
+            //     if (parsedUrlArray.length > this.MAX_ALLOWED_URLS_PER_INPUT) {
+            //         throw new Error(`Too many urls in input. Max allowed: ${this.MAX_ALLOWED_URLS_PER_INPUT}`);
+            //     }
+
+            //     for (let url of parsedUrlArray) {
+            //         indexRes = await this.addDSFromUrl({
+            //             teamId,
+            //             namespaceId,
+            //             dsId, // WILL OVERRIDE EACH OTHER
+            //             type: detectURLSourceType(url),
+            //             url,
+            //             name: _config.name || 'Untitled',
+            //         });
+
+            //         debugOutput += `STEP: Created datasource for url: ${url}\n\n`;
+            //     }
+            // } else
+
+            const dsId = DataSourceIndexer.genDsId(providedId, teamId, namespaceId);
+
+            if (isUrl(inputSchema.value.Source)) {
+                debugOutput += `STEP: Parsing input as url\n\n`;
+                throw new Error('URLs are not supported yet');
+                // indexRes = await this.addDSFromUrl({
+                //     teamId,
+                //     namespaceId,
+                //     dsId,
+                //     type: detectURLSourceType(inputSchema.value.Source),
+                //     url: inputSchema.value.Source,
+                //     name: _config.name || 'Untitled',
+                //     metadata: _config.metadata || null,
+                // });
+            } else {
+                debugOutput += `STEP: Parsing input as text\n\n`;
+                indexRes = await this.addDSFromText({
+                    teamId,
+                    namespaceId: namespaceId,
+                    text: inputSchema.value.Source,
+                    name: _config.name || 'Untitled',
+                    metadata: _config.metadata || null,
+                    sourceId: dsId,
+                });
+            }
+
+            debugOutput += `Created datasource successfully\n\n`;
+
+            return {
+                _debug: debugOutput,
+                Success: {
+                    result: indexRes?.data?.dataSource || true,
+                    id: _config.id,
+                },
+                // _error,
+            };
+        } catch (err: any) {
+            debugOutput += `Error: ${err?.message || "Couldn't index data source"}\n\n`;
+            return {
+                _debug: debugOutput,
+                _error: err?.message || "Couldn't index data source",
+            };
+        }
+    }
+
+    validateInput(input: any) {
+        return Joi.object({
+            Source: Joi.any().required(),
+        })
+            .unknown(true)
+            .validate(input);
+    }
+
+    private async addDSFromText({ teamId, sourceId, namespaceId, text, name, metadata }) {
+        let vectorDbConnector = ConnectorService.getVectorDBConnector();
+        // const isOnCustomStorage = await vectorDBHelper.isNamespaceOnCustomStorage(teamId, namespaceId);
+        // if (isOnCustomStorage) {
+        // const customTeamConnector = await vectorDBHelper.getTeamConnector(teamId);
+        // if (customTeamConnector) {
+        // vectorDbConnector = customTeamConnector;
+        // }
+        // }
+        const id = await vectorDbConnector.requester(AccessCandidate.team(teamId)).createDatasource(namespaceId, {
+            text,
+            metadata,
+            id: sourceId,
+            label: name,
+        });
+
+        return id;
+    }
+
+    public static genDsId(providedId: string, teamId: string, namespaceId: string) {
+        return `${teamId}::${namespaceId}::${providedId}`;
+    }
+
+    private async addDSFromUrl({ teamId, namespaceId, dsId, type, url, name, metadata }) {
+        throw new Error('URLs are not supported yet');
+    }
+}