@smythos/sre 1.5.0 → 1.5.2

package/src/Components/DataSourceLookup.class.ts

@@ -0,0 +1,141 @@
+import Joi from 'joi';
+import { validateInteger } from '../utils';
+import { jsonrepair } from 'jsonrepair';
+import { TemplateString } from '@sre/helpers/TemplateString.helper';
+import { JSONContent } from '@sre/helpers/JsonContent.helper';
+import { ConnectorService } from '@sre/Core/ConnectorsService';
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+import { IAgent as Agent } from '@sre/types/Agent.types';
+import { Component } from './Component.class';
+
+// Note: LLMHelper renamed to LLMInference
+class LLMInference {
+    static async getInstance(model: string) {
+        throw new Error('Method not implemented.');
+    }
+}
+
+export class DataSourceLookup extends Component {
+    protected configSchema = Joi.object({
+        topK: Joi.string()
+            .custom(validateInteger({ min: 0 }), 'custom range validation')
+            .label('Result Count'),
+        model: Joi.string().valid('gpt-4o-mini', 'gpt-4', 'gpt-3.5-turbo', 'gpt-4', 'gpt-3.5-turbo-16k').optional(),
+        prompt: Joi.string().max(30000).allow('').label('Prompt').optional(),
+        postprocess: Joi.boolean().strict().optional(),
+        includeMetadata: Joi.boolean().strict().optional(),
+        namespace: Joi.string().allow('').max(80).messages({
+            // Need to reserve 30 characters for the prefixed unique id
+            'string.max': `The length of the 'namespace' name must be 50 characters or fewer.`,
+        }),
+    });
+    constructor() {
+        super();
+    }
+    init() {}
+    async process(input, config, agent: Agent) {
+        await super.process(input, config, agent);
+        const componentId = config.id;
+        const component = agent.components[componentId];
+        const teamId = agent.teamId;
+        let debugOutput = agent.agentRuntime?.debug ? '== Data Source Lookup Log ==\n' : null;
+
+        const outputs = {};
+        for (let con of config.outputs) {
+            if (con.default) continue;
+            outputs[con.name] = '';
+        }
+
+        const namespace = config.data.namespace.split('_').slice(1).join('_') || config.data.namespace;
+        const model = config.data?.model || 'gpt-4o-mini';
+        const prompt = config.data?.prompt?.trim?.() || '';
+        const postprocess = config.data?.postprocess || false;
+        const includeMetadata = config.data?.includeMetadata || false;
+
+        const _input = typeof input.Query === 'string' ? input.Query : JSON.stringify(input.Query);
+
+        const topK = Math.max(config.data?.topK || 50, 50);
+
+        let vectorDbConnector = ConnectorService.getVectorDBConnector();
+        let existingNs = await vectorDbConnector.requester(AccessCandidate.team(teamId)).namespaceExists(namespace);
+
+        if (!existingNs) {
+            throw new Error(`Namespace ${namespace} does not exist`);
+        }
+
+        let results: string[] | { content: string; metadata: any }[];
+        let _error;
+        try {
+            const response = await vectorDbConnector
+                .requester(AccessCandidate.team(teamId))
+                .search(namespace, _input, { topK, includeMetadata: true });
+            results = response.slice(0, config.data.topK).map((result) => ({
+                content: result.metadata?.text,
+                metadata: result.metadata,
+            }));
+
+            if (includeMetadata) {
+                // only show user-level metadata
+                results = results.map((result) => ({
+                    content: result.content,
+                    //* legacy user-specific metadata key [result.metadata?.metadata]),
+                    metadata: this.parseMetadata(result.metadata || result.metadata?.metadata),
+                }));
+            } else {
+                results = results.map((result) => result.content);
+            }
+            debugOutput += `[Results] \nLoaded ${results.length} results from namespace: ${namespace}\n\n`;
+        } catch (error) {
+            _error = error.toString();
+        }
+
+        //is there a post processing LLM?
+
+        //TODO : better handling of context window exceeding max length
+        if (postprocess && prompt) {
+            const promises: any = [];
+            for (let result of results) {
+                const _prompt = TemplateString(prompt.replace(/{{result}}/g, JSON.stringify(result))).parse(input).result;
+                const llmInference = await LLMInference.getInstance(model);
+                // const req = llmInference.prompt({ query: _prompt, params: { ...config, agentId: agent.id } }).catch((error) => ({ error: error }));
+                // promises.push(req);
+            }
+            results = await Promise.all(promises);
+            for (let i = 0; i < results.length; i++) {
+                if (typeof results[i] === 'string') {
+                    // results[i] = parseJson(results[i]);
+                    results[i] = JSONContent(results[i] as string).tryParse();
+                }
+            }
+        }
+
+        const totalLength = JSON.stringify(results).length;
+        debugOutput += `[Total Length] \n${totalLength}\n\n`;
+        return {
+            Results: results,
+            _error,
+            _debug: debugOutput,
+            //_debug: `Query: ${_input}. \nTotal Length = ${totalLength} \nResults: ${JSON.stringify(results)}`,
+        };
+    }
+
+    // private async checkIfTeamOwnsNamespace(teamId: string, namespaceId: string, token: string) {
+    //     try {
+    //         const res = await SmythAPIHelper.fromAuth({ token }).mwSysAPI.get(`/vectors/namespaces/${namespaceId}`);
+    //         if (res.data?.namespace?.teamId !== teamId) {
+    //             throw new Error(`Namespace does not exist`);
+    //         }
+    //         return true;
+    //     } catch (err) {
+    //         throw new Error(`Namespace does not exist`);
+    //     }
+    // }
+
+    private parseMetadata(metadata: any) {
+        try {
+            return JSON.parse(jsonrepair(metadata));
+        } catch (err) {
+            return metadata;
+        }
+    }
+}

package/src/Components/FEncDec.class.ts

@@ -0,0 +1,29 @@
+import { Component } from './Component.class';
+import { IAgent as Agent } from '@sre/types/Agent.types';
+
+export class FEncDec extends Component {
+    constructor() {
+        super();
+    }
+    init() {}
+    async process(input, config, agent: Agent) {
+        await super.process(input, config, agent);
+        const logger = this.createComponentLogger(agent, config);
+        try {
+            const _error = undefined;
+
+            const data = input.Data;
+            const action = config.data.action || 'Encode';
+            const encoding = config.data.encoding;
+            logger.debug(`${encoding} ${action} data`);
+
+            const Output = action == 'Encode' ? Buffer.from(data).toString(encoding) : Buffer.from(data, encoding).toString('utf8');
+
+            return { Output, _error, _debug: logger.output };
+        } catch (err: any) {
+            const _error = err?.response?.data || err?.message || err.toString();
+            logger.error(` Error processing data \n${_error}\n`);
+            return { hash: undefined, _error, _debug: logger.output };
+        }
+    }
+}

package/src/Components/FHash.class.ts

@@ -0,0 +1,33 @@
+import { Component } from './Component.class';
+import { IAgent as Agent } from '@sre/types/Agent.types';
+import crypto from 'crypto';
+
+export class FHash extends Component {
+    constructor() {
+        super();
+    }
+    init() {}
+    async process(input, config, agent: Agent) {
+        await super.process(input, config, agent);
+        const logger = this.createComponentLogger(agent, config);
+        try {
+            const _error = undefined;
+
+            const data = input.Data;
+            const algorithm = config.data.algorithm;
+            const encoding = config.data.encoding;
+            logger.debug(` Generating hash using ${algorithm} algorithm and ${encoding} encoding`);
+
+            const hashAlgo = crypto.createHash(algorithm);
+            hashAlgo.update(data);
+
+            const Hash = hashAlgo.digest(encoding);
+            logger.debug(` Generated hash: ${Hash}`);
+            return { Hash, _error, _debug: logger.output };
+        } catch (err: any) {
+            const _error = err?.response?.data || err?.message || err.toString();
+            logger.error(` Error generating hash \n${_error}\n`);
+            return { hash: undefined, _error, _debug: logger.output };
+        }
+    }
+}

package/src/Components/FSign.class.ts

@@ -0,0 +1,80 @@
+import { Component } from './Component.class';
+import { IAgent as Agent } from '@sre/types/Agent.types';
+import { TemplateString } from '@sre/helpers/TemplateString.helper';
+import crypto from 'crypto';
+import querystring from 'querystring';
+
+export class FSign extends Component {
+    constructor() {
+        super();
+    }
+    init() {}
+    async process(input, config, agent: Agent) {
+        await super.process(input, config, agent);
+        const logger = this.createComponentLogger(agent, config);
+        try {
+            const _error = undefined;
+            const teamId = agent ? agent.teamId : null;
+
+            let data = input.Data;
+            //if (typeof data === 'object') data = JSON.stringify(data);
+            let signingKey = input.Key || config.data.key;
+            // signingKey = parseTemplate(signingKey, input, { processUnmatched: false });
+            // signingKey = await parseKey(signingKey, teamId);
+            signingKey = await TemplateString(signingKey).parse(input).parseTeamKeysAsync(teamId).asyncResult;
+
+            const signMethod = config.data.signMethod || 'HMAC';
+            const dataTransform = config.data.dataTransform || 'None';
+            const hashType = config.data.hashType || 'md5';
+            const RSA_padding = config.data.RSA_padding;
+            const RSA_saltLength = config.data.RSA_saltLength;
+            const encoding = config.data.encoding || 'hex';
+
+            if (typeof data != 'string') {
+                switch (dataTransform) {
+                    case 'Stringify':
+                        data = JSON.stringify(data);
+                        break;
+                    case 'Querystring':
+                        data = querystring.stringify(data);
+                        break;
+                }
+            }
+            logger.debug(' Data to sign = ', data);
+            logger.debug(` Signing data using ${signMethod} algorithm and ${encoding} encoding`);
+            const Signature = this.signData(data, signingKey, signMethod, encoding, { hashType, RSA_padding, RSA_saltLength });
+
+            logger.debug(` Signature generated: ${Signature}`);
+            return { Signature, _error, _debug: logger.output };
+        } catch (err: any) {
+            const _error = err?.response?.data || err?.message || err.toString();
+            logger.error(` Error generating hash \n${_error}\n`);
+            return { hash: undefined, _error, _debug: logger.output };
+        }
+    }
+
+    private signData(data, key, signMethod, encoding = 'hex', options: any = {}) {
+        // Determine if the algorithm is for HMAC or RSA/DSA/ECDSA
+        switch (signMethod) {
+            case 'RSA':
+                const algo = `${signMethod}-${options.hashType || 'md5'}`.toUpperCase();
+                const sign = crypto.createSign(algo);
+                sign.update(data);
+
+                const sign_options = {
+                    key,
+                    padding: options.RSA_padding ? crypto.constants[options.RSA_padding] : undefined,
+                    saltLength: options.RSA_saltLength ? crypto.constants[options.RSA_saltLength] : undefined,
+                };
+                // For RSA/DSA/ECDSA, options may include padding and saltLength
+                return sign.sign(sign_options, encoding.toLowerCase() as crypto.BinaryToTextEncoding);
+
+            case 'HMAC':
+                const hmac = crypto.createHmac(options.hashType, key);
+                hmac.update(data);
+                return hmac.digest(encoding as crypto.BinaryToTextEncoding);
+        }
+
+        return null;
+    }
+}

package/src/Components/FSleep.class.ts

@@ -0,0 +1,25 @@
+import { Component } from './Component.class';
+import { IAgent as Agent } from '@sre/types/Agent.types';
+
+export class FSleep extends Component {
+    constructor() {
+        super();
+    }
+    init() {}
+    async process(input, config, agent: Agent) {
+        await super.process(input, config, agent);
+        const logger = this.createComponentLogger(agent, config);
+        try {
+            const _error = undefined;
+            const delay = parseInt(config.data.delay || 1);
+            const Output = input.Input;
+            logger.debug(`Sleeping for ${delay} seconds`);
+            await new Promise((resolve) => setTimeout(resolve, delay * 1000));
+            return { Output, _error, _debug: logger.output, _debug_time: logger.elapsedTime };
+        } catch (err: any) {
+            const _error = err?.response?.data || err?.message || err.toString();
+            logger.error(` Error processing data \n${_error}\n`);
+            return { hash: undefined, _error, _debug: logger.output, _debug_time: logger.elapsedTime };
+        }
+    }
+}

package/src/Components/FTimestamp.class.ts

@@ -0,0 +1,25 @@
+import { Component } from './Component.class';
+import { IAgent as Agent } from '@sre/types/Agent.types';
+
+export class FTimestamp extends Component {
+    constructor() {
+        super();
+    }
+    init() {}
+    async process(input, config, agent: Agent) {
+        await super.process(input, config, agent);
+        const logger = this.createComponentLogger(agent, config);
+        try {
+            const _error = undefined;
+            const format = config.data.format; //TODO set timestamp format
+            const Timestamp = Date.now();
+            logger.debug(`Timestamp : ${Timestamp}`);
+
+            return { Timestamp, _error, _debug: logger.output, _debug_time: logger.elapsedTime };
+        } catch (err: any) {
+            const _error = err?.response?.data || err?.message || err.toString();
+            logger.error(` Error processing data \n${_error}\n`);
+            return { hash: undefined, _error, _debug: logger.output, _debug_time: logger.elapsedTime };
+        }
+    }
+}

package/src/Components/FileStore.class.ts

@@ -0,0 +1,75 @@
+import { IAgent as Agent } from '@sre/types/Agent.types';
+import { Component } from './Component.class';
+import Joi from 'joi';
+import _config from '@sre/config';
+import { S3Storage } from '@sre/IO/Storage.service/connectors/S3Storage.class';
+import { ConnectorService } from '@sre/Core/ConnectorsService';
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+import { BinaryInput } from '@sre/helpers/BinaryInput.helper';
+import { TemplateString } from '@sre/helpers/TemplateString.helper';
+import { SmythFS } from '@sre/IO/Storage.service/SmythFS.class';
+
+export class FileStore extends Component {
+    protected configSchema = Joi.object({
+        name: Joi.string().max(1000).allow('').label('Name'),
+        ttl: Joi.number().integer().label('TTL'),
+    });
+    constructor() {
+        super();
+    }
+    init() {}
+    async process(input, config, agent: Agent) {
+        await super.process(input, config, agent);
+
+        const logger = this.createComponentLogger(agent, config);
+        try {
+            logger.debug(`=== File Store Log ===`);
+            let Output: any = {};
+            let _error = undefined;
+            let data = input.Data;
+            const accessCandidate = AccessCandidate.agent(agent.id);
+            const binaryData = await BinaryInput.from(data, null, null, accessCandidate);
+            const fileData = await binaryData.getJsonData(accessCandidate);
+            const buffer = await binaryData.getBuffer();
+            const customFileName = TemplateString(config.data.name).parse(input).result;
+            const metadata = {
+                ContentType: fileData.mimetype,
+            };
+            const ttl = config.data.ttl || 86400;
+            const extension = fileData.url?.split('.').pop();
+            const s3StorageConnector = ConnectorService.getStorageConnector('S3') as S3Storage;
+            const fileName = this.getFileName(customFileName, extension);
+            try {
+                const s3Key = `teams/${agent.teamId}/components_data/${fileName}`;
+
+                await s3StorageConnector.requester(AccessCandidate.agent(agent.teamId)).write(s3Key, buffer, null, metadata);
+                await s3StorageConnector.requester(AccessCandidate.agent(agent.teamId)).expire(s3Key, +ttl);
+                const smythFSUrl = `smythfs://${agent.teamId}.team/components_data/${fileName}`;
+                const url = await SmythFS.Instance.genResourceUrl(smythFSUrl, AccessCandidate.agent(agent.teamId));
+                Output = {
+                    Url: url,
+                };
+            } catch (error: any) {
+                logger.error(`Error saving file \n${error}\n`);
+                _error = error?.response?.data || error?.message || error.toString();
+                Output = undefined; //prevents running next component if the code execution failed
+            }
+
+            return { ...Output, _error, _debug: logger.output };
+        } catch (err: any) {
+            const _error = err?.response?.data || err?.message || err.toString();
+            logger.error(` Error saving file \n${_error}\n`);
+            return { Output: undefined, _error, _debug: logger.output };
+        }
+    }
+
+    getExtension(mimeType: string) {
+        const extension = mimeType.split('/')[1];
+        return extension;
+    }
+
+    getFileName(customName: string, extension: string) {
+        const uniqueId = (() => btoa(String.fromCharCode(...crypto.getRandomValues(new Uint8Array(9)))).replace(/[+/=]/g, ''))();
+        return `${uniqueId}${customName ? `.${customName}` : ''}.${extension}`;
+    }
+}

package/src/Components/ForEach.class.ts

@@ -0,0 +1,97 @@
+import { IAgent as Agent } from '@sre/types/Agent.types';
+import { Component } from './Component.class';
+import Joi from 'joi';
+import { JSONContent } from '@sre/helpers/JsonContent.helper';
+
+export class ForEach extends Component {
+    protected configSchema = null;
+    constructor() {
+        super();
+    }
+    init() {}
+    async process(input, config, agent: Agent) {
+        await super.process(input, config, agent);
+        let Loop = {};
+        let Result;
+        let _temp_result;
+        let _error = null;
+        let _in_progress = true;
+        const logger = this.createComponentLogger(agent, config);
+        try {
+            const inputObject = input.Input;
+            let inputArray;
+
+            //Try to parse multiple arrays formats since forEach always expects and array
+            if (Array.isArray(inputObject)) inputArray = inputObject;
+            else {
+                if (typeof inputObject === 'string') {
+                    inputArray = inputObject.trim().startsWith('[') ? JSONContent(inputObject).tryParse() : inputObject.split(',');
+                } else {
+                    inputArray = [inputObject];
+                }
+            }
+
+            if (!Array.isArray(inputArray) && typeof inputArray === 'object')
+                //if json object, use the values
+                inputArray = Object.values(inputArray);
+
+            const runtimeData = agent.agentRuntime.getRuntimeData(config.id);
+            const _ForEachData = runtimeData._LoopData || { parentId: config.id, loopIndex: 0, loopLength: inputArray.length };
+
+            logger.debug(`Loop: ${_ForEachData.loopIndex} / ${_ForEachData.loopLength}`);
+            delete _ForEachData.branches; //reset branches (the number of branches is calculated in CallComponent@Agent.class.ts )
+
+            if (_ForEachData.result) {
+                _temp_result = _ForEachData.result;
+                logger.debug(`  => Loop Result : ${JSON.stringify(Loop, null, 2)}`);
+                logger.debug(`---------------------------------------------------`);
+            }
+
+            Loop = inputArray[_ForEachData.loopIndex];
+
+            logger.debug(`  => Loop Data : ${JSON.stringify(Loop, null, 2)}`);
+
+            _in_progress = Loop !== undefined;
+            if (_in_progress) {
+                _ForEachData.loopIndex++;
+            }
+            _ForEachData._in_progress = _in_progress;
+
+            agent.agentRuntime.updateRuntimeData(config.id, { _LoopData: _ForEachData });
+        } catch (error: any) {
+            _error = error;
+            logger.error(error);
+        }
+        if (!_in_progress) {
+            Result = _temp_result || [];
+
+            switch (config?.data?.format) {
+                case 'minimal':
+                    Result = Result.map((item) => cleanupResult(item.result));
+                    break;
+                case 'results-array':
+                    Result = Result.map((item) => Object.values(cleanupResult(item.result))).flat(Infinity);
+                    break;
+            }
+        }
+
+        return { Loop, Result, _temp_result, _error, _in_progress, _debug: logger.output };
+    }
+    async postProcess(output, config, agent: Agent): Promise<any> {
+        output = await super.postProcess(output, config, agent);
+        if (output?.result) {
+            delete output.result._temp_result;
+            delete output.result._in_progress;
+            delete output.result.Loop;
+        }
+        return output;
+    }
+}
+function cleanupResult(result) {
+    if (typeof result !== 'object') return result;
+    if (result._debug) delete result._debug;
+    if (result._error) delete result._error;
+    if (result._temp_result) delete result._temp_result;
+    if (result._in_progress) delete result._in_progress;
+    return result;
+}

package/src/Components/GPTPlugin.class.ts

@@ -0,0 +1,70 @@
+import Joi from 'joi';
+
+import { Agent } from '@sre/AgentManager/Agent.class';
+import { Conversation } from '@sre/helpers/Conversation.helper';
+import { TemplateString } from '@sre/helpers/TemplateString.helper';
+
+import { Component } from './Component.class';
+
+export class GPTPlugin extends Component {
+    protected configSchema = Joi.object({
+        model: Joi.string().optional(),
+        openAiModel: Joi.string().optional(), // for backward compatibility
+        specUrl: Joi.string().max(2048).uri().required().description('URL of the OpenAPI specification'),
+        descForModel: Joi.string().max(5000).required().allow('').label('Description for Model'),
+        name: Joi.string().max(500).required().allow(''),
+        desc: Joi.string().max(5000).required().allow('').label('Description'),
+        logoUrl: Joi.string().max(8192).allow(''),
+        id: Joi.string().max(200),
+        version: Joi.string().max(100).allow(''),
+        domain: Joi.string().max(253).allow(''),
+    });
+
+    constructor() {
+        super();
+    }
+
+    init() {}
+
+    async process(input, config, agent: Agent) {
+        await super.process(input, config, agent);
+        const logger = this.createComponentLogger(agent, config);
+
+        logger.debug(`=== Open API Log ===`);
+
+        try {
+            const specUrl = config?.data?.specUrl;
+
+            if (!specUrl) {
+                return { _error: 'Please provide a Open API Specification URL!', _debug: logger.output };
+            }
+
+            const model = config?.data?.model || config?.data?.openAiModel;
+            const descForModel = TemplateString(config?.data?.descForModel).parse(input).result;
+            let prompt = '';
+
+            if (input?.Prompt) {
+                prompt = typeof input?.Prompt === 'string' ? input?.Prompt : JSON.stringify(input?.Prompt);
+            } else if (input?.Query) {
+                prompt = typeof input?.Query === 'string' ? input?.Query : JSON.stringify(input?.Query);
+            }
+
+            if (!prompt) {
+                return { _error: 'Please provide a prompt', _debug: logger.output };
+            }
+
+            // TODO [Forhad]: Need to check and validate input prompt token
+
+            const conv = new Conversation(model, specUrl, { systemPrompt: descForModel, agentId: agent?.id });
+
+            const result = await conv.prompt(prompt);
+
+            logger.debug(`Response:\n`, result, '\n');
+
+            return { Output: result, _debug: logger.output };
+        } catch (error: any) {
+            console.error('Error on running Open API: ', error);
+            return { _error: `Error on running Open API!\n${error?.message || JSON.stringify(error)}`, _debug: logger.output };
+        }
+    }
+}