@smythos/sre 1.5.1 → 1.5.4

package/src/subsystems/AgentManager/Agent.helper.ts

@@ -0,0 +1,3 @@
+export function isAgent(agent: any): boolean {
+    return typeof agent === 'object' && agent.id && typeof agent.callComponent === 'function';
+}

package/src/subsystems/AgentManager/AgentData.service/AgentDataConnector.ts

@@ -0,0 +1,230 @@
+import { Connector } from '@sre/Core/Connector.class';
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+import { IAccessCandidate } from '@sre/types/ACL.types';
+import config from '@sre/config';
+import { escapeString, TemplateString } from '@sre/helpers/TemplateString.helper';
+import { JSONContent, JSONContentHelper } from '@sre/helpers/JsonContent.helper';
+
+export interface IAgentDataConnector {
+    getAgentData(agentId: string, version?: string): Promise<any>;
+    getAgentIdByDomain(domain: string): Promise<string>;
+    getAgentSettings(agentId: string, version?: string): Promise<any>;
+    listTeamAgents(teamId: string, deployedOnly?: boolean, includeData?: boolean): Promise<any[]>;
+    getAgentConfig(agentId: string): { [key: string]: any };
+}
+
+const openapiTemplate = JSON.stringify({
+    openapi: '3.0.1',
+    info: {
+        title: '{{model_name}}',
+        description: '{{model_description}}',
+        version: '{{version}}',
+    },
+    servers: [
+        {
+            url: '{{server_url}}',
+        },
+    ],
+    paths: {},
+    components: {
+        schemas: {},
+    },
+});
+
+const openapiEndpointTemplate = JSON.stringify({
+    summary: '{{summary}}',
+    operationId: '{{operationId}}',
+    'x-openai-isConsequential': false,
+    requestBody: {
+        required: true,
+        content: {},
+    },
+    responses: {
+        '200': {
+            description: 'response',
+            content: {
+                'text/plain': {
+                    schema: {
+                        type: 'string',
+                    },
+                },
+            },
+        },
+    },
+});
+export abstract class AgentDataConnector extends Connector implements IAgentDataConnector {
+    public name = 'AgentDataConnector';
+    public abstract getAgentData(agentId: string, version?: string): Promise<any>;
+    public abstract getAgentIdByDomain(domain: string): Promise<string>;
+    public abstract getAgentSettings(agentId: string, version?: string): Promise<{ [key: string]: any }>;
+    public abstract getAgentEmbodiments(agentId: string): Promise<any>;
+    public abstract isDeployed(agentId: string): Promise<boolean>;
+    public abstract listTeamAgents(teamId: string, deployedOnly?: boolean, includeData?: boolean): Promise<any[]>;
+    public abstract getAgentConfig(agentId: string): { [key: string]: any };
+    /**
+     * Loads openAPI JSON for the agent
+     * @param source this represents either the agentId or the agent data
+     * @param domain
+     * @param version
+     * @param aiOnly
+     * @returns
+     */
+    public async getOpenAPIJSON(source: string | Record<string, any>, server_url, version, aiOnly: boolean = false) {
+        if (!source) {
+            throw new Error('Agent not found');
+        }
+
+        const apiBasePath = version && version != 'latest' ? `/v${version}/api` : '/api';
+
+        const agentData: any = typeof source === 'object' ? source : await this.getAgentData(source, version);
+        const name = agentData.name;
+
+        let description = aiOnly ? agentData.data.behavior : agentData.data.shortDescription;
+        if (!description) description = agentData.data.description; //data.description is deprecated, we just use it as a fallback for now
+
+        const _version = agentData.data.version || '1.0.0';
+
+        const openAPITpl = TemplateString(openapiTemplate)
+            .parse({
+                model_name: escapeString(name),
+                model_description: escapeString(description),
+                server_url,
+                version: _version,
+            })
+            .clean().result;
+        const openAPIObj = JSON.parse(openAPITpl);
+
+        const components = agentData.data.components.filter((component: any) => component.name === 'APIEndpoint');
+        for (let component of components) {
+            const ai_exposed = component.data.ai_exposed || typeof component.data.ai_exposed === 'undefined';
+            if (aiOnly && !ai_exposed) continue;
+            let method = (component.data.method || 'post').toLowerCase();
+            let summary = aiOnly ? component.data.description || component.data.doc : component.data.doc || component.data.description;
+
+            const openAPIEntry = JSONContent(
+                TemplateString(openapiEndpointTemplate)
+                    .parse({
+                        summary: summary?.replace(/"/g, '\\"'),
+                        operationId: component?.data?.endpoint,
+                    })
+                    .clean().result,
+            ).tryParse();
+
+            if (typeof openAPIEntry !== 'object') {
+                console.warn('Error on openAPIEntry: ', openAPIEntry);
+                continue;
+            }
+
+            if (!openAPIObj.paths[apiBasePath + '/' + component.data.endpoint]) openAPIObj.paths[apiBasePath + '/' + component.data.endpoint] = {};
+            //const componentsSchemas = openAPIObj.components.schemas;
+
+            openAPIObj.paths[apiBasePath + '/' + component.data.endpoint][method] = openAPIEntry;
+
+            if (component.inputs.length > 0) {
+                if (method === 'get') {
+                    delete openAPIEntry.requestBody;
+
+                    openAPIEntry.parameters = [];
+
+                    for (let input of component.inputs) {
+                        const parameter: {
+                            name: string;
+                            in: string;
+                            description: string;
+                            required: boolean;
+                            schema: { type: string };
+                            style?: string;
+                            explode?: boolean;
+                        } = {
+                            name: input.name,
+                            in: 'query',
+                            description: input.description,
+                            required: !input.optional,
+                            schema: getOpenAPIInputSchema(input.type),
+                        };
+
+                        // for array and object types
+                        const { style, explode } = getOpenAPIParameterStyle(input.type);
+                        if (style) {
+                            parameter.style = style;
+                            parameter.explode = explode;
+                        }
+
+                        openAPIEntry.parameters.push(parameter);
+                    }
+                } else {
+                    const requiredProps: any = [];
+
+                    const hasBinaryType = !aiOnly && component.inputs.some((input) => input.type.toLowerCase().trim() === 'binary');
+                    //if it's an AI format, we force application/json format, becauwe we want to receive binary data as a url
+                    const mimetype = hasBinaryType ? 'multipart/form-data' : 'application/json';
+                    openAPIEntry.requestBody.content[mimetype] = {};
+                    for (let input of component.inputs) {
+                        if (!input.optional) requiredProps.push(input.name);
+
+                        if (!openAPIEntry.requestBody.content[mimetype].schema)
+                            openAPIEntry.requestBody.content[mimetype].schema = { type: 'object' };
+
+                        const schema: any = openAPIEntry.requestBody.content[mimetype].schema || {
+                            type: 'object',
+                        };
+
+                        if (!schema.properties) schema.properties = {};
+                        schema.properties[input.name] = {
+                            ...getOpenAPIInputSchema(input.type),
+                            format: !aiOnly && input.type.toLowerCase().trim() === 'binary' ? 'binary' : undefined,
+                            description: input.description,
+                            default: input.defaultVal,
+                        };
+                        schema.required = requiredProps;
+
+                        if (!openAPIEntry.requestBody.content[mimetype].schema) openAPIEntry.requestBody.content['application/json'].schema = schema;
+                    }
+                }
+            } else {
+                delete openAPIEntry.requestBody;
+            }
+        }
+
+        return openAPIObj;
+    }
+}
+
+function getOpenAPIInputSchema(input_type) {
+    switch (input_type?.toLowerCase()) {
+        case 'binary':
+        case 'string':
+        case 'any':
+            return { type: 'string' };
+        case 'number':
+        case 'float':
+            return { type: 'number' };
+        case 'integer':
+            return { type: 'integer' };
+        case 'boolean':
+            return { type: 'boolean' };
+        case 'array':
+            return { type: 'array', items: {} };
+        case 'object':
+            return { type: 'object', additionalProperties: {} };
+        default:
+            return { type: 'string' };
+    }
+}
+
+function getOpenAPIParameterStyle(input_type) {
+    switch (input_type.toLowerCase()) {
+        case 'array':
+            return {
+                style: 'form',
+                explode: false, // results in `ids=1,2,3`
+            };
+        case 'object':
+            return {
+                style: 'deepObject',
+                explode: true, // results in `lat=value1&long=value2`
+            };
+        default:
+            return { style: '', explode: false };
+    }
+}

package/src/subsystems/AgentManager/AgentData.service/connectors/CLIAgentDataConnector.class.ts

@@ -0,0 +1,66 @@
+import { ConnectorService } from '@sre/Core/ConnectorsService';
+import { CLIConnector } from '@sre/IO/CLI.service/CLIConnector';
+import fs from 'fs';
+import path from 'path';
+import { AgentDataConnector } from '../AgentDataConnector';
+
+type TArgs = { args: Record<string, any> };
+export class CLIAgentDataConnector extends AgentDataConnector {
+    public name: string = 'CLIAgentDataConnector';
+    private argv;
+    constructor(settings: TArgs) {
+        super();
+        this.argv = settings.args || process.argv;
+    }
+
+    public getAgentConfig(agentId: string): Partial<TArgs> {
+        return {};
+    }
+
+    public async getAgentData(agentId: string, version?: string) {
+        const cliConnector: CLIConnector = ConnectorService.getCLIConnector();
+
+        const params: any = cliConnector.get('agent');
+
+        //get current directory
+        const __dirname = fs.realpathSync(process.cwd());
+        const filePath = path.join(__dirname, params.agent);
+
+        if (fs.existsSync(filePath)) {
+            const data = fs.readFileSync(filePath, 'utf8');
+
+            return { data: JSON.parse(data), version: version || '1.0' };
+        }
+    }
+
+    public getAgentIdByDomain(domain: string): Promise<string> {
+        return Promise.resolve('');
+    }
+    public async getAgentSettings(agentId: string, version?: string) {
+        const cliConnector: CLIConnector = ConnectorService.getCLIConnector();
+
+        const params: any = cliConnector.get('settings');
+        let settings: any;
+
+        if (typeof params.settings === 'string') {
+            if (fs.existsSync(params.settings)) {
+                settings = JSON.parse(fs.readFileSync(params.settings, 'utf8'));
+            }
+        } else {
+            settings = params.settings;
+        }
+        return settings;
+    }
+
+    public async getAgentEmbodiments(agentId: string): Promise<any> {
+        return [];
+    }
+
+    public async listTeamAgents(teamId: string, deployedOnly?: boolean): Promise<any[]> {
+        console.warn(`listTeamAgents is not implemented for CLIAgentDataConnector`);
+        return [];
+    }
+    public async isDeployed(agentId: string): Promise<boolean> {
+        return true;
+    }
+}

package/src/subsystems/AgentManager/AgentData.service/connectors/LocalAgentDataConnector.class.ts

@@ -0,0 +1,142 @@
+import fs from 'fs';
+import path from 'path';
+import { AgentDataConnector } from '../AgentDataConnector';
+import { uid } from '@sre/utils/general.utils';
+
+export type LocalAgentDataSettings = { devDir: string; prodDir: string };
+
+/**
+ * This connector loads Agent data and settings from a provided directory, it then indexes the loaded agents and settings by agent IDs.
+ * Agent data files should be in JSON format and contain an 'id' field with the agent ID and at least a 'components' field.
+ * Settings files should be in JSON format and contain an 'id' field with the agent ID and the settings in a 'settings' field.
+ *     'settings' field is a key-value object with the Agent settings.
+ */
+export class LocalAgentDataConnector extends AgentDataConnector {
+    public name: string = 'LocalAgentDataConnector';
+    private devDir;
+    private prodDir;
+    private agentsData = { dev: {}, prod: {} };
+    private agentSettings = { dev: {}, prod: {} };
+
+    constructor(settings: LocalAgentDataSettings) {
+        super();
+        this.devDir = settings.devDir;
+        this.prodDir = settings.prodDir;
+    }
+
+    public getAgentConfig(agentId: string): Partial<LocalAgentDataSettings> {
+        return {};
+    }
+
+    private indexDir(dir: string) {
+        const agents = fs.readdirSync(dir);
+
+        const agentsData = {};
+        const agentSettings = {};
+        for (const agent of agents) {
+            if (!agent.endsWith('.smyth')) continue;
+
+            const agentData = fs.readFileSync(path.join(dir, agent), 'utf8');
+            let jsonData;
+            try {
+                jsonData = JSON.parse(agentData);
+
+                if (!jsonData.components) {
+                    console.warn(`File ${agent} is not a valid agent data file, skipping...`);
+                    continue;
+                }
+
+                if (!jsonData.id) {
+                    console.warn(`Agent data for ${agent} does not contain an id, generating one...`);
+                    jsonData.id = 'tmp-' + uid();
+                }
+            } catch (e) {
+                console.warn(`Error parsing agent data for ${agent}: ${e.message}`);
+            }
+
+            //is this an agent data file?
+            if (jsonData.components) agentsData[jsonData.id] = jsonData;
+
+            //does this file contain settings?
+            if (jsonData.settings) agentSettings[jsonData.id] = jsonData.settings;
+        }
+
+        return { agentsData, agentSettings };
+    }
+    private indexAgentsData() {
+        const { agentsData: devAgentsData, agentSettings: devAgentSettings } = this.indexDir(this.devDir);
+        const { agentsData: prodAgentsData, agentSettings: prodAgentSettings } = this.indexDir(this.prodDir);
+        this.agentsData = { dev: devAgentsData, prod: prodAgentsData };
+        this.agentSettings = { dev: devAgentSettings, prod: prodAgentSettings };
+    }
+
+    public async start() {
+        super.start();
+        this.started = false;
+        this.indexAgentsData();
+        this.started = true;
+    }
+
+    /**
+     * returns the agent data for the provided agent ID
+     * if the version is not provided, it defaults to the dev version
+     * otherwise it loads the corresponding prod version
+     * @param agentId
+     * @param version
+     * @returns
+     */
+    public async getAgentData(agentId: string, version?: string) {
+        const ready = await this.ready();
+        if (!ready) {
+            throw new Error('Connector not ready');
+        }
+
+        const data = version ? this.agentsData.prod[agentId] : this.agentsData.dev[agentId];
+
+        if (data) {
+            return { data, version: version || '1.0' };
+        } else {
+            throw new Error(`Agent with id ${agentId} not found`);
+        }
+    }
+
+    public getAgentIdByDomain(domain: string): Promise<string> {
+        return Promise.resolve('');
+    }
+
+    /**
+     * returns the agent settings for the provided agent ID
+     * if the version is not provided, it defaults to the dev version
+     * otherwise it loads the corresponding prod version
+     * @param agentId
+     * @param version
+     * @returns
+     */
+    public async getAgentSettings(agentId: string, version?: string) {
+        const ready = await this.ready();
+        if (!ready) {
+            throw new Error('Connector not ready');
+        }
+
+        const settings = version ? this.agentSettings.prod[agentId] : this.agentSettings.dev[agentId];
+
+        if (settings) {
+            return settings;
+        } else {
+            throw new Error(`Settings for agent with id ${agentId} not found`);
+        }
+    }
+
+    public async getAgentEmbodiments(agentId: string): Promise<any> {
+        return [];
+    }
+
+    public async listTeamAgents(teamId: string, deployedOnly?: boolean): Promise<any[]> {
+        console.warn(`listTeamAgents is not implemented for LocalAgentDataConnector`);
+        return [];
+    }
+
+    public async isDeployed(agentId: string): Promise<boolean> {
+        return !!this.agentsData.prod[agentId];
+    }
+}

package/src/subsystems/AgentManager/AgentData.service/connectors/NullAgentData.class.ts

@@ -0,0 +1,39 @@
+import { ConnectorService } from '@sre/Core/ConnectorsService';
+import { CLIConnector } from '@sre/IO/CLI.service/CLIConnector';
+import fs from 'fs';
+import path from 'path';
+import { AgentDataConnector } from '../AgentDataConnector';
+
+type TArgs = { args: Record<string, any> };
+export class NullAgentData extends AgentDataConnector {
+    public name: string = 'NullAgentData';
+    constructor(settings: TArgs) {
+        super();
+    }
+
+    public getAgentConfig(agentId: string): Partial<TArgs> {
+        return {};
+    }
+
+    public async getAgentData(agentId: string, version?: string) {
+        return { data: {}, version: '1.0' };
+    }
+
+    public getAgentIdByDomain(domain: string): Promise<string> {
+        return Promise.resolve('');
+    }
+    public async getAgentSettings(agentId: string, version?: string) {
+        return {};
+    }
+
+    public async getAgentEmbodiments(agentId: string): Promise<any> {
+        return [];
+    }
+
+    public async listTeamAgents(teamId: string, deployedOnly?: boolean): Promise<any[]> {
+        return [];
+    }
+    public async isDeployed(agentId: string): Promise<boolean> {
+        return true;
+    }
+}

package/src/subsystems/AgentManager/AgentData.service/index.ts

@@ -0,0 +1,18 @@
+//==[ SRE: LLM ]======================
+
+import { ConnectorService, ConnectorServiceProvider } from '@sre/Core/ConnectorsService';
+import { TConnectorService } from '@sre/types/SRE.types';
+import { CLIAgentDataConnector } from './connectors/CLIAgentDataConnector.class';
+import { AgentDataConnector } from './AgentDataConnector';
+import { LocalAgentDataConnector } from './connectors/LocalAgentDataConnector.class';
+import { NullAgentData } from './connectors/NullAgentData.class';
+export class AgentDataService extends ConnectorServiceProvider {
+    public register() {
+        //FIXME : register an actual account connector, not the abstract one
+        ConnectorService.register(TConnectorService.AgentData, 'AgentData', AgentDataConnector);
+        ConnectorService.register(TConnectorService.AgentData, 'CLI', CLIAgentDataConnector);
+        ConnectorService.register(TConnectorService.AgentData, 'Local', LocalAgentDataConnector);
+
+        ConnectorService.register(TConnectorService.AgentData, 'NullAgentData', NullAgentData);
+    }
+}