@smythos/sre 1.5.1 → 1.5.2

package/src/Components/LogicAND.class.ts

@@ -0,0 +1,28 @@
+import { IAgent as Agent } from '@sre/types/Agent.types';
+import { Component } from './Component.class';
+
+export class LogicAND extends Component {
+    constructor() {
+        super();
+    }
+    init() {}
+    async process(input, config, agent: Agent) {
+        await super.process(input, config, agent);
+        const logger = this.createComponentLogger(agent, config);
+        const result: any = { Output: true };
+
+        for (let cfgInput of config.inputs) {
+            // check if all inputs are set (expected inputs are in "config.inputs" actual inputs are in "input")
+            if (!input[cfgInput.name]) {
+                result.Output = undefined;
+                break;
+            }
+        }
+
+        result.Verified = result.Output !== undefined;
+        result.Unverified = !result.Verified;
+        if (!result.Verified) delete result.Verified;
+        if (!result.Unverified) delete result.Unverified;
+        return result;
+    }
+}

package/src/Components/LogicAtLeast.class.ts

@@ -0,0 +1,85 @@
+import Joi from 'joi';
+
+import { IAgent as Agent } from '@sre/types/Agent.types';
+import { Component } from './Component.class';
+
+export class LogicAtLeast extends Component {
+    protected configSchema = Joi.object({
+        // TODO (Forhad): Need to check if min and max work instead of the custom validateInteger
+        minSetInputs: Joi.string()
+            .custom(validateInteger({ min: 0, max: 9 }), 'custom range validation')
+            .label('Minimum Inputs'),
+    });
+    constructor() {
+        super();
+    }
+
+    init() {}
+
+    async process(input, config, agent: Agent) {
+        await super.process(input, config, agent);
+        const logger = this.createComponentLogger(agent, config);
+        const result: any = { Output: undefined };
+
+        if (typeof config.data.minSetInputs !== 'string' || config.data.minSetInputs.trim() === '' || isNaN(Number(config.data.minSetInputs))) {
+            return result;
+        }
+
+        const minSetInputs = Number(config.data.minSetInputs);
+        if (config.inputs.length < minSetInputs) {
+            return result;
+        }
+
+        let trueCount = 0;
+        for (let cfgInput of config.inputs) {
+            if (input[cfgInput.name]) {
+                trueCount++;
+            }
+        }
+
+        if (trueCount >= minSetInputs) {
+            result.Output = true;
+        }
+
+        result.Verified = result.Output !== undefined;
+        result.Unverified = !result.Verified;
+        if (!result.Verified) delete result.Verified;
+        if (!result.Unverified) delete result.Unverified;
+
+        return result;
+    }
+}
+
+interface RangeValidationArgs {
+    min?: number;
+    max?: number;
+}
+
+function validateInteger(args: RangeValidationArgs) {
+    return (value: string, helpers: any) => {
+        const numValue = Number(value);
+        const fieldName = helpers.schema._flags.label || helpers.state.path[helpers.state.path.length - 1];
+
+        // Check if the value is a number
+        if (isNaN(numValue)) {
+            throw new Error(`The value for '${fieldName}' must be a number`);
+        }
+
+        // Range validations
+        if (args.min !== undefined && args.max !== undefined) {
+            if (numValue < args.min || numValue > args.max) {
+                throw new Error(`The value for '${fieldName}' must be from ${args.min} to ${args.max}`);
+            }
+        } else if (args.min !== undefined) {
+            if (numValue < args.min) {
+                throw new Error(`The value for '${fieldName}' must be greater or equal to ${args.min}`);
+            }
+        } else if (args.max !== undefined) {
+            if (numValue > args.max) {
+                throw new Error(`The value for '${fieldName}' must be less or equal to ${args.max}`);
+            }
+        }
+
+        return value; // Value is valid
+    };
+}

package/src/Components/LogicAtMost.class.ts

@@ -0,0 +1,86 @@
+import Joi from 'joi';
+
+import { IAgent as Agent } from '@sre/types/Agent.types';
+import { Component } from './Component.class';
+
+export class LogicAtMost extends Component {
+    protected configSchema = Joi.object({
+        // TODO (Forhad): Need to check if min and max work instead of the custom validateInteger
+        maxSetInputs: Joi.string()
+            .custom(validateInteger({ min: 0, max: 9 }), 'custom range validation')
+            .label('Maximum Inputs'),
+    });
+    constructor() {
+        super();
+    }
+
+    init() {}
+
+    async process(input, config, agent: Agent) {
+        await super.process(input, config, agent);
+        const result: any = { Output: undefined };
+
+        if (typeof config.data.maxSetInputs !== 'string' || config.data.maxSetInputs.trim() === '' || isNaN(Number(config.data.maxSetInputs))) {
+            return result;
+        }
+
+        const maxSetInputs = Number(config.data.maxSetInputs);
+        if (config.inputs.length < maxSetInputs) {
+            return result;
+        }
+
+        let trueCount = 0;
+        for (let cfgInput of config.inputs) {
+            if (input[cfgInput.name]) {
+                trueCount++;
+                if (trueCount > maxSetInputs) {
+                    break;
+                }
+            }
+        }
+
+        if (trueCount <= maxSetInputs) {
+            result.Output = true;
+        }
+
+        result.Verified = result.Output !== undefined;
+        result.Unverified = !result.Verified;
+        if (!result.Verified) delete result.Verified;
+        if (!result.Unverified) delete result.Unverified;
+
+        return result;
+    }
+}
+interface RangeValidationArgs {
+    min?: number;
+    max?: number;
+}
+
+function validateInteger(args: RangeValidationArgs) {
+    return (value: string, helpers: any) => {
+        const numValue = Number(value);
+        const fieldName = helpers.schema._flags.label || helpers.state.path[helpers.state.path.length - 1];
+
+        // Check if the value is a number
+        if (isNaN(numValue)) {
+            throw new Error(`The value for '${fieldName}' must be a number`);
+        }
+
+        // Range validations
+        if (args.min !== undefined && args.max !== undefined) {
+            if (numValue < args.min || numValue > args.max) {
+                throw new Error(`The value for '${fieldName}' must be from ${args.min} to ${args.max}`);
+            }
+        } else if (args.min !== undefined) {
+            if (numValue < args.min) {
+                throw new Error(`The value for '${fieldName}' must be greater or equal to ${args.min}`);
+            }
+        } else if (args.max !== undefined) {
+            if (numValue > args.max) {
+                throw new Error(`The value for '${fieldName}' must be less or equal to ${args.max}`);
+            }
+        }
+
+        return value; // Value is valid
+    };
+}

package/src/Components/LogicOR.class.ts

@@ -0,0 +1,29 @@
+import { IAgent as Agent } from '@sre/types/Agent.types';
+import { Component } from './Component.class';
+
+export class LogicOR extends Component {
+    constructor() {
+        super();
+    }
+    init() {}
+    async process(input, config, agent: Agent) {
+        await super.process(input, config, agent);
+        const result: any = { Output: undefined };
+        console.log(input);
+        console.log(config);
+        for (let cfgInput of config.inputs) {
+            // check if one of the inputs are set (expected inputs are in "config.inputs" actual inputs are in "input")
+            if (input[cfgInput.name]) {
+                result.Output = true;
+                break;
+            }
+        }
+
+        result.Verified = result.Output !== undefined;
+        result.Unverified = !result.Verified;
+        if (!result.Verified) delete result.Verified;
+        if (!result.Unverified) delete result.Unverified;
+
+        return result;
+    }
+}

package/src/Components/LogicXOR.class.ts

@@ -0,0 +1,34 @@
+import { IAgent as Agent } from '@sre/types/Agent.types';
+import { Component } from './Component.class';
+
+export class LogicXOR extends Component {
+    constructor() {
+        super();
+    }
+
+    init() {}
+
+    async process(input, config, agent: Agent) {
+        await super.process(input, config, agent);
+        const result: any = { Output: undefined };
+        let trueCount = 0;
+
+        for (let cfgInput of config.inputs) {
+            // counts the number of set inputs
+            if (input[cfgInput.name]) {
+                trueCount++;
+            }
+        }
+        // checks if only one input is set, to trigger output
+        if (trueCount === 1) {
+            result.Output = true;
+        }
+
+        result.Verified = result.Output !== undefined;
+        result.Unverified = !result.Verified;
+        if (!result.Verified) delete result.Verified;
+        if (!result.Unverified) delete result.Unverified;
+
+        return result;
+    }
+}

package/src/Components/MCPClient.class.ts

@@ -0,0 +1,112 @@
+import Joi from 'joi';
+
+import { IAgent as Agent } from '@sre/types/Agent.types';
+import { Conversation } from '@sre/helpers/Conversation.helper';
+import { TemplateString } from '@sre/helpers/TemplateString.helper';
+
+import { Component } from './Component.class';
+import { Client } from '@modelcontextprotocol/sdk/client/index.js';
+import { SSEClientTransport } from '@modelcontextprotocol/sdk/client/sse.js';
+
+export class MCPClient extends Component {
+    protected configSchema = Joi.object({
+        model: Joi.string().optional(),
+        openAiModel: Joi.string().optional(), // for backward compatibility
+        mcpUrl: Joi.string().max(2048).uri().required().description('URL of the MCP'),
+        descForModel: Joi.string().max(5000).required().allow('').label('Description for Model'),
+        name: Joi.string().max(500).required().allow(''),
+        desc: Joi.string().max(5000).required().allow('').label('Description'),
+        logoUrl: Joi.string().max(8192).allow(''),
+        id: Joi.string().max(200),
+        version: Joi.string().max(100).allow(''),
+        domain: Joi.string().max(253).allow(''),
+        prompt: Joi.string().max(5000).optional().allow('').label('Prompt'),
+    });
+
+    constructor() {
+        super();
+    }
+
+    init() {}
+
+    async process(input, config, agent: Agent) {
+        await super.process(input, config, agent);
+        const logger = this.createComponentLogger(agent, config);
+
+        logger.debug(`=== MCP Client Log ===`);
+
+        try {
+            const mcpUrl = config?.data?.mcpUrl;
+            if (!mcpUrl) {
+                return { _error: 'Please provide a MCP URL!', _debug: logger.output };
+            }
+
+            const model = config?.data?.model || config?.data?.openAiModel;
+            const descForModel = TemplateString(config?.data?.descForModel).parse(input).result;
+            let prompt = TemplateString(config?.data?.prompt).parse(input).result;
+
+            if (!prompt) {
+                return { _error: 'Please provide a prompt', _debug: logger.output };
+            }
+
+            // TODO [Forhad]: Need to check and validate input prompt token
+            const baseUrl = new URL(mcpUrl);
+            const transport = new SSEClientTransport(baseUrl);
+            const client = new Client({
+                name: 'sse-client',
+                version: '1.0.0',
+            });
+            await client.connect(transport);
+            const toolsData = await client.listTools();
+            const conv = new Conversation(
+                model,
+                {
+                    openapi: '3.0.1',
+                    info: {
+                        title: `${agent?.name}`,
+                        version: `${agent?.version}`,
+                        description: descForModel,
+                    },
+                    servers: [
+                        {
+                            url: agent?.domain,
+                        },
+                    ],
+                    paths: {},
+                },
+                { agentId: agent?.id },
+            );
+
+            for (const tool of toolsData.tools) {
+                let toolArgs = {};
+                Object.entries(tool.inputSchema.properties).forEach(([propName, propDetails]) => {
+                    toolArgs[propName] = {
+                        description: '',
+                        required: ((tool.inputSchema.required as string[]) || []).includes(propName) || false,
+                        type: (propDetails as any).type,
+                        ...((propDetails as any).type === 'array' ? { items: { type: 'string' } } : {}),
+                    };
+                });
+                await conv.addTool({
+                    name: tool.name,
+                    description: tool.description,
+                    arguments: toolArgs,
+                    handler: async (input) => {
+                        const result = await client.callTool({
+                            name: tool.name,
+                            arguments: input,
+                        });
+                        return result;
+                    },
+                });
+            }
+            const result = await conv.prompt(prompt);
+
+            logger.debug(`Response:\n`, result, '\n');
+
+            return { Output: result, _debug: logger.output };
+        } catch (error: any) {
+            return { _error: `Error on running MCP Client!\n${error?.message || JSON.stringify(error)}`, _debug: logger.output };
+        }
+    }
+}

package/src/Components/PromptGenerator.class.ts

@@ -0,0 +1,122 @@
+import Joi from 'joi';
+import { IAgent as Agent } from '@sre/types/Agent.types';
+import { LLMInference } from '@sre/LLMManager/LLM.inference';
+import { TemplateString } from '@sre/helpers/TemplateString.helper';
+import { Component } from './Component.class';
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+
+//TODO : better handling of context window exceeding max length
+
+export class PromptGenerator extends Component {
+    protected configSchema = Joi.object({
+        model: Joi.string().max(200).required(),
+        prompt: Joi.string().required().max(8_000_000).label('Prompt'), // 2M tokens is around 8M characters
+        temperature: Joi.number().min(0).max(5).label('Temperature'), // max temperature is 2 for OpenAI and togetherAI but 5 for cohere
+        maxTokens: Joi.number().min(1).label('Maximum Tokens'),
+        stopSequences: Joi.string().allow('').max(400).label('Stop Sequences'),
+        topP: Joi.number().min(0).max(1).label('Top P'),
+        topK: Joi.number().min(0).max(500).label('Top K'), // max top_k is 100 for togetherAI but 500 for cohere
+        frequencyPenalty: Joi.number().min(0).max(2).label('Frequency Penalty'),
+        presencePenalty: Joi.number().min(0).max(2).label('Presence Penalty'),
+        responseFormat: Joi.string().valid('json', 'text').optional().label('Response Format'),
+        passthrough: Joi.boolean().optional().label('Passthrough'),
+    });
+    constructor() {
+        super();
+    }
+    init() {}
+    async process(input, config, agent: Agent) {
+        await super.process(input, config, agent);
+
+        //let debugLog = agent.agentRuntime?.debug ? [] : undefined;
+        const logger = this.createComponentLogger(agent, config);
+
+        try {
+            logger.debug(`=== LLM Prompt Log ===`);
+            let teamId = agent?.teamId;
+
+            const passThrough: boolean = config.data.passthrough || false;
+            const model: string = config.data.model || 'echo';
+            const llmInference: LLMInference = await LLMInference.getInstance(model, AccessCandidate.agent(agent.id));
+
+            // if the llm is undefined, then it means we removed the model from our system
+            if (!llmInference.connector) {
+                return {
+                    _error: `The model '${model}' is not available. Please try a different one.`,
+                    _debug: logger.output,
+                };
+            }
+
+            const modelId = await agent.modelsProvider.getModelId(model);
+            logger.debug(` Model : ${modelId || model}`);
+
+            let prompt: any = TemplateString(config.data.prompt).parse(input).result;
+
+            logger.debug(` Prompt\n`, prompt, '\n');
+
+            // default to json response format
+            config.data.responseFormat = config.data?.responseFormat || 'json';
+
+            // request to LLM
+            let response: any;
+            if (passThrough) {
+                const contentPromise = new Promise(async (resolve, reject) => {
+                    let _content = '';
+                    const eventEmitter: any = await llmInference
+                        .promptStream({
+                            query: prompt,
+                            params: { ...config, model, agentId: agent.id },
+                        })
+                        .catch((error) => {
+                            console.error('Error on promptStream: ', error);
+                            reject(error);
+                        });
+                    eventEmitter.on('content', (content) => {
+                        if (typeof agent.callback === 'function') {
+                            agent.callback({ content });
+                        }
+                        agent.sse.send('llm/passthrough/content', content);
+                        _content += content;
+                    });
+                    eventEmitter.on('thinking', (thinking) => {
+                        if (typeof agent.callback === 'function') {
+                            agent.callback({ thinking });
+                        }
+                        agent.sse.send('llm/passthrough/thinking', thinking);
+                    });
+                    eventEmitter.on('end', () => {
+                        console.log('end');
+                        resolve(_content);
+                    });
+                });
+                response = await contentPromise;
+            } else {
+                response = await llmInference
+                    .prompt({ query: prompt, params: { ...config, agentId: agent.id } })
+                    .catch((error) => ({ error: error }));
+            }
+
+            // in case we have the response but it's empty string, undefined or null
+            if (!response) {
+                return { _error: ' LLM Error = Empty Response!', _debug: logger.output };
+            }
+
+            if (response?.error) {
+                const error = response?.error + ' ' + (response?.details || '');
+                logger.error(` LLM Error=`, error);
+
+                return { Reply: response?.data, _error: error, _debug: logger.output };
+            }
+
+            logger.debug(' Response \n', response);
+
+            const result = { Reply: response };
+
+            result['_debug'] = logger.output;
+
+            return result;
+        } catch (error) {
+            return { _error: error.message, _debug: logger.output };
+        }
+    }
+}

package/src/Components/ScrapflyWebScrape.class.ts

@@ -0,0 +1,159 @@
+import { IAgent as Agent } from '@sre/types/Agent.types';
+import { Component } from './Component.class';
+import Joi from 'joi';
+import SREConfig from '@sre/config';
+import axios from 'axios';
+import { SystemEvents } from '@sre/Core/SystemEvents';
+import { AccessCandidate } from '../subsystems/Security/AccessControl/AccessCandidate.class';
+import { getCredentials } from '../subsystems/Security/Credentials.helper';
+// const CREDITS_PER_URL = 0.2;
+
+export class ScrapflyWebScrape extends Component {
+    protected configSchema = Joi.object({
+        // includeImages: Joi.boolean().default(false).label('Include Image Results'),
+        antiScrapingProtection: Joi.boolean().default(false).label('Enable Anti-Scraping Protection'),
+        javascriptRendering: Joi.boolean().default(false).label('Enable JavaScript Rendering'),
+        autoScroll: Joi.boolean().default(false).label('Enable Auto Scroll'),
+        format: Joi.string().default('markdown').label('Format').optional(),
+    });
+
+    constructor() {
+        super();
+    }
+    init() {}
+    async process(input, config, agent: Agent) {
+        await super.process(input, config, agent);
+
+        const logger = this.createComponentLogger(agent, config);
+        try {
+            logger.debug(`=== Web Scrape Log ===`);
+            let Output: any = {};
+            let _error = undefined;
+            const scrapeUrls = this.extractUrls(input);
+            logger.debug('Payload:', JSON.stringify(config.data));
+            logger.debug(`Vaild URLs: ${JSON.stringify(scrapeUrls)}`);
+            const teamId = agent.teamId;
+            const key = await getCredentials(AccessCandidate.team(teamId), 'scrapfly');
+
+            const scrapeResults = await Promise.all(scrapeUrls.map((url) => this.scrapeURL(url, config.data, key)));
+            const results = scrapeResults
+                .filter((result) => result.success)
+                .map((result) => {
+                    return { url: result.url, content: result.content };
+                });
+            const failedResults = scrapeResults
+                .filter((result) => !result.success)
+                .map((result) => {
+                    return { url: result.url, error: result.error };
+                });
+
+            Output = { Results: results, FailedURLs: failedResults };
+            const totalCredits = scrapeResults.reduce((acc, result) => acc + (result.cost || 0), 0);
+            this.reportUsage({
+                urlsScraped: results?.length,
+                agentId: agent.id,
+                teamId: agent.teamId,
+                totalCredits,
+            });
+            return { ...Output, _error, _debug: logger.output };
+        } catch (err: any) {
+            const _error = err?.message || err?.response?.data || err.toString();
+            logger.error(` Error scraping web \n${_error}\n`);
+            return { Output: undefined, _error, _debug: logger.output };
+        }
+    }
+
+    async scrapeURL(url, data, key) {
+        try {
+            const response = await axios({
+                method: 'get',
+                url: 'https://api.scrapfly.io/scrape',
+                params: {
+                    url: encodeURIComponent(url),
+                    key,
+                    cost_budget: 80,
+                    ...(data.format ? { format: data.format } : { format: 'markdown' }),
+                    ...(data.antiScrapingProtection && { asp: true }),
+                    ...(data.javascriptRendering && { render_js: true }),
+                    ...(data.autoScroll && { auto_scroll: true, render_js: true }),
+                },
+            });
+            return {
+                content: response.data?.result?.content,
+                success: true,
+                url,
+                cost: response.data?.context?.cost?.total || 0,
+            };
+        } catch (error) {
+            return {
+                content: undefined,
+                success: false,
+                error: error?.response?.data?.result?.error?.message || 'Failed to scrape URL',
+                url,
+                cost: 0,
+            };
+        }
+    }
+
+    extractUrls(input: any) {
+        const scrapeUrls = [];
+        for (const key in input) {
+            if (Object.prototype.hasOwnProperty.call(input, key)) {
+                const inputItem = input[key];
+                if (typeof inputItem === 'string') {
+                    try {
+                        let urls = JSON.parse(inputItem);
+                        for (const url of urls) {
+                            if (this.isValidUrl(url)) {
+                                scrapeUrls.push(url.trim());
+                            }
+                        }
+                    } catch (error) {
+                        const commaSeparatedUrls = inputItem.split(',');
+                        for (const url of commaSeparatedUrls) {
+                            if (this.isValidUrl(url)) {
+                                scrapeUrls.push(url.trim());
+                            }
+                        }
+                    }
+                } else if (typeof inputItem === 'object') {
+                    for (const url of inputItem) {
+                        if (this.isValidUrl(url)) {
+                            scrapeUrls.push(url.trim());
+                        }
+                    }
+                }
+            }
+        }
+        return scrapeUrls;
+    }
+
+    isValidUrl(urlString: string) {
+        try {
+            const urlToCheck = urlString;
+            new URL(urlToCheck);
+            return true;
+        } catch (error) {
+            return false;
+        }
+    }
+
+    protected reportUsage({
+        urlsScraped,
+        agentId,
+        teamId,
+        totalCredits,
+    }: {
+        urlsScraped: number;
+        agentId: string;
+        teamId: string;
+        totalCredits: number;
+    }) {
+        SystemEvents.emit('USAGE:API', {
+            sourceId: 'api:webscrape.smyth',
+            credits: totalCredits,
+            agentId,
+            teamId,
+        });
+    }
+}