@smythos/sre 1.5.0 → 1.5.2

package/src/Components/TavilyWebSearch.class.ts

@@ -0,0 +1,98 @@
+import { IAgent as Agent } from '@sre/types/Agent.types';
+import { Component } from './Component.class';
+import Joi from 'joi';
+import SREConfig from '@sre/config';
+import axios from 'axios';
+import { SystemEvents } from '@sre/Core/SystemEvents';
+import { getCredentials } from '../subsystems/Security/Credentials.helper';
+import { AccessCandidate } from '../subsystems/Security/AccessControl/AccessCandidate.class';
+
+export class TavilyWebSearch extends Component {
+    protected schema = {
+        name: 'TavilyWebSearch',
+        description: 'Use this component to generate a responses from an LLM',
+
+        inputs: {
+            SearchQuery: {
+                type: 'Text',
+                description: 'The search query to get the web search results of',
+                default: true,
+            },
+        },
+        outputs: {
+            Results: {
+                type: 'Array',
+                description: 'The web search results',
+                default: true,
+            },
+        },
+    };
+
+    protected configSchema = Joi.object({
+        includeImages: Joi.boolean().default(false).label('Include Image Results'),
+        sourcesLimit: Joi.number().integer().default(10).label('Sources Limit'),
+        searchTopic: Joi.string().valid('general', 'news').label('Search Topic'),
+        includeQAs: Joi.boolean().default(false).label('Include QAs'),
+        timeRange: Joi.string().valid('None', 'day', 'week', 'month', 'year').label('Time Range'),
+        includeRawContent: Joi.boolean().default(false).label('Include Raw Content'),
+        excludeDomains: Joi.string().allow('').label('Exclude Domains'),
+    });
+    constructor() {
+        super();
+    }
+    init() {}
+    async process(input, config, agent: Agent) {
+        await super.process(input, config, agent);
+
+        const logger = this.createComponentLogger(agent, config);
+        try {
+            logger.debug(`=== Web Scrape Log ===`);
+            let Output: any = {};
+            let _error = undefined;
+            let searchQuery = input['SearchQuery'];
+
+            const teamId = agent.teamId;
+            const api_key = await getCredentials(AccessCandidate.team(teamId), 'tavily');
+
+            logger.debug('Payload:', JSON.stringify(config.data));
+            const response = await axios({
+                method: 'post',
+                url: 'https://api.tavily.com/search',
+                data: {
+                    api_key,
+                    query: searchQuery,
+                    topic: config.data.searchTopic,
+                    exclude_domains: config.data.excludeDomains?.length ? config.data.excludeDomains.split(',') : [],
+                    max_results: config.data.sourcesLimit,
+                    ...(config.data.timeRange !== 'None' ? { time_range: config.data.timeRange } : {}),
+                    ...(config.data.includeImages ? { include_images: true } : {}),
+                    ...(config.data.includeQAs ? { include_answer: true } : {}),
+                    ...(config.data.includeRawContent ? { include_raw_content: true } : {}),
+                },
+            });
+            Output = {
+                Results: response.data.results,
+                ...(config.data.includeImages ? { Images: response.data.images } : {}),
+                ...(config.data.includeQAs ? { Answer: response.data.answer } : {}),
+            };
+            this.reportUsage({
+                agentId: agent.id,
+                teamId: agent.teamId,
+            });
+            return { ...Output, _error, _debug: logger.output };
+        } catch (err: any) {
+            const _error = err?.message || err?.response?.data || err.toString();
+            logger.error(` Error scraping web \n${JSON.stringify(_error)}\n`);
+            return { Output: undefined, _error, _debug: logger.output };
+        }
+    }
+
+    protected reportUsage({ agentId, teamId }: { agentId: string; teamId: string }) {
+        SystemEvents.emit('USAGE:API', {
+            sourceId: 'api:websearch.smyth',
+            credits: 1,
+            agentId,
+            teamId,
+        });
+    }
+}

package/src/Components/index.ts

@@ -0,0 +1,77 @@
+import { Component } from './Component.class';
+import { APIEndpoint } from './APIEndpoint.class';
+import { APIOutput } from './APIOutput.class';
+import { PromptGenerator } from './PromptGenerator.class';
+import { APICall } from './APICall/APICall.class';
+import { FSleep } from './FSleep.class';
+import { FHash } from './FHash.class';
+import { FEncDec } from './FEncDec.class';
+import { FTimestamp } from './FTimestamp.class';
+import { DataSourceLookup } from './DataSourceLookup.class';
+import { DataSourceIndexer } from './DataSourceIndexer.class';
+import { DataSourceCleaner } from './DataSourceCleaner.class';
+import { JSONFilter } from './JSONFilter.class';
+import { LogicAND } from './LogicAND.class';
+import { LogicOR } from './LogicOR.class';
+import { LogicXOR } from './LogicXOR.class';
+import { LogicAtLeast } from './LogicAtLeast.class';
+import { LogicAtMost } from './LogicAtMost.class';
+import { AgentPlugin } from './AgentPlugin.class';
+import { LLMAssistant } from './LLMAssistant.class';
+import { Async } from './Async.class';
+import { Await } from './Await.class';
+import { ForEach } from './ForEach.class';
+import { HuggingFace } from './HuggingFace.class';
+import { GPTPlugin } from './GPTPlugin.class';
+import { Classifier } from './Classifier.class';
+import { FSign } from './FSign.class';
+import { GenAILLM } from './GenAILLM.class';
+import { FileStore } from './FileStore.class';
+import { ScrapflyWebScrape } from './ScrapflyWebScrape.class';
+import { TavilyWebSearch } from './TavilyWebSearch.class';
+import { ComponentHost } from './ComponentHost.class';
+import { ImageGenerator } from './ImageGenerator.class'; // Legacy
+import { MCPClient } from './MCPClient.class';
+
+const components = {
+    Component: new Component(),
+    Note: new Component(), //this is a fake component
+    APIEndpoint: new APIEndpoint(),
+    APIOutput: new APIOutput(),
+    PromptGenerator: new PromptGenerator(),
+    LLMPrompt: new PromptGenerator(),
+    APICall: new APICall(),
+    FSleep: new FSleep(),
+    FHash: new FHash(),
+    FEncDec: new FEncDec(),
+    FSign: new FSign(),
+    FTimestamp: new FTimestamp(),
+    DataSourceLookup: new DataSourceLookup(),
+    DataSourceIndexer: new DataSourceIndexer(),
+    DataSourceCleaner: new DataSourceCleaner(),
+    JSONFilter: new JSONFilter(),
+    LogicAND: new LogicAND(),
+    LogicOR: new LogicOR(),
+    LogicXOR: new LogicXOR(),
+    LogicAtLeast: new LogicAtLeast(),
+    LogicAtMost: new LogicAtMost(),
+    AgentPlugin: new AgentPlugin(),
+    LLMAssistant: new LLMAssistant(),
+    Async: new Async(),
+    Await: new Await(),
+    ForEach: new ForEach(),
+    HuggingFace: new HuggingFace(),
+    GPTPlugin: new GPTPlugin(),
+    Classifier: new Classifier(),
+    GenAILLM: new GenAILLM(),
+    FileStore: new FileStore(),
+    WebSearch: new TavilyWebSearch(),
+    WebScrape: new ScrapflyWebScrape(),
+    TavilyWebSearch: new TavilyWebSearch(),
+    ScrapflyWebScrape: new ScrapflyWebScrape(),
+    ComponentHost: new ComponentHost(),
+    ImageGenerator: new ImageGenerator(),
+    MCPClient: new MCPClient(),
+};
+
+export const ComponentInstances = components;

package/src/Core/AgentProcess.helper.ts

@@ -0,0 +1,240 @@
+//import { SmythRuntime } from '@sre/Core/SmythRuntime.class';
+import { Agent } from '@sre/AgentManager/Agent.class';
+import { AgentRequest } from '@sre/AgentManager/AgentRequest.class';
+import { AgentSettings } from '@sre/AgentManager/AgentSettings.class';
+import { TAgentProcessParams } from '@sre/types/Agent.types';
+import { uid } from '../utils';
+
+import { CLIConnector } from '@sre/IO/CLI.service/CLIConnector';
+import * as FileType from 'file-type';
+import fs from 'fs';
+import mime from 'mime';
+import path from 'path';
+import { ConnectorService } from './ConnectorsService';
+
+export class AgentProcess {
+    public agent: Agent;
+
+    private _loadPromise: Promise<any>;
+
+    private constructor(
+        private agentData: any,
+        private agentVersion?: string,
+    ) {
+        this.initAgent(agentData, agentVersion);
+    }
+    private async initAgent(agentData: any, agentVersion?: string) {
+        //await SmythRuntime.Instance.ready();
+        let data;
+        let agentId;
+
+        if (typeof agentData === 'object') {
+            data = agentData;
+            if (data.components && data.connections) {
+                data = { data, version: '1.0' };
+            }
+
+            agentId = data.data.id || 'tmp-' + uid();
+        } else {
+            const jsonRegex = /^{.*}$/g;
+            const jsonData = agentData.match(jsonRegex)?.[0];
+
+            const idRegex = /^[a-zA-Z0-9\-]+$/g;
+            agentId = agentData.match(idRegex)?.[0];
+
+            //We are loading from an agentId
+            if (agentId) {
+                const agentDataConnector = ConnectorService.getAgentDataConnector();
+
+                data = await agentDataConnector.getAgentData(agentId, agentVersion);
+            }
+
+            //we are loading an agent from provided data
+            if (!data && jsonData) {
+                data = JSON.parse(jsonData);
+                //generate an agentId if not provided
+                agentId = data.id || 'tmp-' + uid();
+
+                if (data.components && data.connections) {
+                    data = { data, version: '1.0' };
+                }
+            }
+        }
+
+        const agentSettings = new AgentSettings(agentId);
+        this.agent = new Agent(agentId, data, agentSettings);
+    }
+
+    public async ready() {
+        if (this._loadPromise) {
+            return this._loadPromise;
+        }
+
+        return (this._loadPromise = new Promise((resolve) => {
+            let maxWait = 10000;
+            //wait for agent to be set
+            const itv = setInterval(() => {
+                if (this.agent) {
+                    clearInterval(itv);
+                    resolve(true);
+                }
+                maxWait -= 100;
+                if (maxWait <= 0) {
+                    clearInterval(itv);
+                    resolve(false);
+                }
+            }, 100);
+        }));
+    }
+
+    public static load(agentData: any, agentVersion?: string) {
+        const agentProcess = new AgentProcess(agentData, agentVersion);
+        return agentProcess;
+    }
+
+    /**
+     * Run the agent process
+     * @param reqConfig - The request configuration
+     * @param callback - The callback function is used if we want to send status data, meta information, or stream response progressively.
+     * Note: even if the response is streamed through the callback, the response is still returned as a single object in the response.data field.
+     * @returns The result of the agent process
+     */
+    public async run(
+        reqConfig: TAgentProcessParams | Array<string> | AgentRequest,
+        callback?: (data: any) => void,
+    ): Promise<{
+        status?: number;
+        data: any;
+    }> {
+        await this.ready();
+        if (!this.agent) throw new Error('Failed to load agent');
+        let request: AgentRequest = this.parseReqConfig(reqConfig);
+
+        this.agent.setRequest(request);
+
+        if (typeof callback === 'function') {
+            this.agent.setCallback(callback);
+        }
+
+        const pathMatches = request.path.match(/(^\/v[0-9]+\.[0-9]+?)?(\/api\/(.+)?)/);
+        if (!pathMatches || !pathMatches[2]) {
+            return { status: 404, data: { error: 'Endpoint not found' } };
+        }
+        const endpointPath = pathMatches[2];
+        const input = request.method == 'GET' ? request.query : request.body;
+        const result: any = await this.agent.process(endpointPath, input).catch((error) => ({ error: error.message }));
+
+        return { data: result };
+    }
+
+    public reset() {
+        //the current version of agent cannot be used to run multiple requests
+        //as a workaround we provide this function to reset AgentProcess state by generating a new Agent
+        //TODO: refactor Agent.class in order to allow multiple consecutive requests running
+        this.initAgent(this.agentData);
+    }
+
+    private parseReqConfig(reqConfig: TAgentProcessParams | Array<string> | AgentRequest): AgentRequest {
+        if (reqConfig instanceof AgentRequest) return reqConfig;
+        if (Array.isArray(reqConfig)) return this.parseCLI(reqConfig);
+        return new AgentRequest(reqConfig);
+    }
+
+    private parseCLI(argList: Array<string>): AgentRequest {
+        const cliConnector: CLIConnector = ConnectorService.getCLIConnector();
+        const methods = ['get', 'post', 'put', 'delete', 'patch', 'head', 'options'];
+        const cli: any = cliConnector.parse(argList, ['endpoint', 'post', 'get', 'put', 'delete', 'patch', 'head', 'options', 'headers', 'session']);
+
+        const usedMethod = methods.find((method) => cli[method]);
+
+        const req: AgentRequest = new AgentRequest();
+
+        req.method = usedMethod?.toUpperCase() || 'GET';
+        req.body = {};
+        req.query = {};
+
+        switch (usedMethod) {
+            case 'get':
+            case 'delete':
+            case 'head':
+            case 'options':
+                req.query = cli[usedMethod];
+                break;
+            case 'post':
+            case 'put':
+            case 'patch':
+                req.body = cli[usedMethod];
+                break;
+        }
+
+        req.path = `/api/${cli.endpoint}`;
+        req.params = cli.endpoint?.split('/');
+
+        req.headers = cli.headers || {};
+        //convert all keys to lowercase
+        for (let key in req.headers) {
+            req.headers[key.toLowerCase()] = req.headers[key];
+            delete req.headers[key];
+        }
+
+        req.sessionID = cli.session || uid();
+
+        req.files = [];
+        if (req.body) {
+            for (let entry in req.body) {
+                let value = req.body[entry];
+                const filePath = path.join(process.cwd(), value);
+                const fileName = path.basename(filePath);
+                if (!fs.existsSync(filePath)) continue;
+
+                //read the file and create a file object
+
+                try {
+                    // Read the file content
+                    const fileBuffer = fs.readFileSync(filePath);
+                    const ext: any = fileName.split('.').pop();
+
+                    const fileObj = {
+                        fieldname: entry,
+                        originalname: fileName,
+                        buffer: fileBuffer,
+                        mimetype: mime.getType(ext) || 'application/octet-stream',
+                    };
+
+                    delete req.body[entry];
+                    req.files.push(fileObj);
+
+                    // Try to determine the MIME type from the file content
+                    FileType.fileTypeFromBuffer(fileBuffer).then((fileType) => {
+                        if (fileType) {
+                            fileObj.mimetype = fileType.mime;
+                        }
+                    });
+                } catch (error) {
+                    console.warn('Coud not read file', filePath, error.message);
+                }
+            }
+        }
+
+        return req;
+    }
+
+    public async post(path: string, input?: any, headers?: any) {
+        return this.run({ method: 'POST', path, body: input || {}, headers });
+    }
+
+    public async get(path: string, query?: any, headers?: any) {
+        return this.run({ method: 'GET', path, query, headers });
+    }
+
+    public async readDebugState(stateId: string, reqConfig: TAgentProcessParams | Array<string> | AgentRequest) {
+        await this.ready();
+        if (!this.agent) throw new Error('Failed to load agent');
+        let request: AgentRequest = this.parseReqConfig(reqConfig);
+
+        this.agent.setRequest(request);
+        await this.agent.agentRuntime.ready();
+
+        return this.agent.agentRuntime.readState(stateId, true);
+    }
+}

package/src/Core/Connector.class.ts

@@ -0,0 +1,123 @@
+import { LocalCache } from '@sre/helpers/LocalCache.helper';
+import { Logger } from '../helpers/Log.helper';
+import { createHash } from 'crypto';
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+
+const console = Logger('Connector');
+//const lCache = new LocalCache();
+
+export class Connector<TRequest = any> {
+    public name: string;
+    public started = false;
+    private _interactionHandler: () => void;
+    private _readyPromise: Promise<boolean>;
+    private static lCache = new LocalCache();
+
+    public get settings() {
+        return this._settings;
+    }
+
+    public get interactionHandler() {
+        return this._interactionHandler;
+    }
+
+    //this flag is used to check if the connector is valid
+    //when a connector fails to load, it is replaced by a DummyConnector instance which returns false for this flag
+    public get valid() {
+        return true;
+    }
+
+    constructor(protected _settings?: any) {
+        //TODO : check if smyth runtime is initialized and throw an error if it is not
+    }
+
+    /**
+     * If the connector is interactive, The connector initializer will wait for start() method to complete before loading the next connector
+     */
+    protected setInteraction(handler: () => void) {
+        this._interactionHandler = handler;
+    }
+    /**
+     * Creates a new instance of the current class using the provided settings.
+     * This method can be called on both Connector instances and its subclasses.
+     * This is used when we need to create a connector instance with a specific configuration (for example with user provided keys)
+     *
+     * @param settings - Configuration settings for the new instance.
+     * @returns A new instance of the current class.
+     */
+    public instance(settings: any): this {
+        const configHash = createHash('sha256')
+            .update(JSON.stringify(settings || {}))
+            .digest('hex');
+        const key = `${this.name}-${configHash}`;
+
+        if (Connector.lCache.has(key)) {
+            return Connector.lCache.get(key) as this;
+        }
+
+        // if not in cache, create a new instance from the concrete class
+        const constructor = this.constructor as { new (config: any): any };
+        const instance = new constructor(settings);
+        Connector.lCache.set(key, instance, 60 * 60 * 1000); // cache for 1 hour
+
+        return instance;
+    }
+
+    static isValid(connector: Connector): boolean {
+        return connector.name !== undefined && connector.name !== null && connector.name !== '';
+    }
+
+    public async start() {
+        console.info(`Starting ${this.name} connector ...`);
+        this.started = true;
+    }
+
+    public async stop() {
+        console.info(`Stopping ${this.name} connector ...`);
+    }
+
+    public ready() {
+        if (!this._readyPromise) {
+            this._readyPromise = new Promise((resolve) => {
+                let maxWait = 60000;
+                const tick = 100;
+                if (this.started) {
+                    resolve(true);
+                } else {
+                    const interval = setInterval(() => {
+                        if (this.started) {
+                            clearInterval(interval);
+                            resolve(true);
+                        }
+
+                        maxWait -= tick;
+                        if (maxWait <= 0) {
+                            clearInterval(interval);
+                            resolve(false);
+                        }
+                    }, tick);
+                }
+            });
+        }
+        return this._readyPromise;
+    }
+
+    public requester(candidate: AccessCandidate): TRequest {
+        return null as TRequest;
+    }
+
+    public user(candidate: AccessCandidate | string): TRequest {
+        if (typeof candidate === 'string') {
+            return this.requester(AccessCandidate.user(candidate));
+        }
+        return this.requester(candidate);
+    }
+
+    public team(teamId: string): TRequest {
+        return this.requester(AccessCandidate.team(teamId));
+    }
+
+    public agent(agentId: string): TRequest {
+        return this.requester(AccessCandidate.agent(agentId));
+    }
+}