@smythos/sre 1.6.14 → 1.7.5

package/src/subsystems/AgentManager/Agent.class.ts

@@ -12,6 +12,7 @@ import { Logger } from '@sre/helpers/Log.helper';
 import { TemplateString } from '@sre/helpers/TemplateString.helper';
 import { IModelsProviderRequest, ModelsProviderConnector } from '@sre/LLMManager/ModelsProvider.service/ModelsProviderConnector';
 import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+import { Trigger } from '@sre/Components/Triggers/Trigger.class';
 import { IAgent } from '@sre/types/Agent.types';
 import { AgentSSE } from './AgentSSE.class';
 
@@ -26,10 +27,12 @@ export class Agent implements IAgent {
     public components: any;
     public connections: any;
     public endpoints: any = {};
+    public triggers: any = {};
     public sessionId;
     public sessionTag = '';
     public callerSessionId;
     public apiBasePath = '/api';
+    public triggerBasePath = '/trigger';
     public agentRuntime: AgentRuntime | any;
 
     public usingTestDomain = false;
@@ -40,7 +43,7 @@ export class Agent implements IAgent {
 
     //public baseUrl = '';
     public agentVariables: any = {};
-    private _kill = false;
+    private _killReason = '';
     //public agentRequest: Request | AgentRequest | any;
     public async = false;
     public jobID = '';
@@ -89,6 +92,11 @@ export class Agent implements IAgent {
             this.endpoints[`${this.apiBasePath}/${endpoint.data.endpoint}`][method] = endpoint;
         }
 
+        const triggers = this.data.components.filter((c) => typeof c.data?.triggerEndpoint == 'string');
+        for (let trigger of triggers) {
+            this.triggers[`${this.triggerBasePath}/${trigger.data.triggerEndpoint}`] = trigger;
+        }
+
         this.components = {};
         for (let component of this.data.components) {
             //FIXME : this does not persist in debug mode, it breaks key value mem logic
@@ -117,14 +125,27 @@ export class Agent implements IAgent {
             const output = sourceComponent.outputs[sourceIndex];
             output.index = sourceIndex; // legacy ids (numbers)
 
-            const input = targetComponent.inputs[targetIndex];
-            input.index = targetIndex;
-
             if (!output.next) output.next = [];
             output.next.push(targetComponent.id);
 
-            if (!input.prev) input.prev = [];
-            input.prev.push(sourceComponent.id);
+            //when a trigger is connected to an APIEndpoint it does not use the standard inputs
+            //the targetIndex is then -1 and the input does not really exist
+            //in that case, we should consider the trigger as an ancestor for all APIEndpoint inputs
+
+            if (targetIndex >= 0) {
+                const input = targetComponent.inputs[targetIndex];
+                if (input) {
+                    input.index = targetIndex;
+                    if (!input.prev) input.prev = [];
+                    input.prev.push(sourceComponent.id);
+                }
+            } else {
+                //if the targetIndex is -1, we should consider the trigger as an ancestor for all APIEndpoint inputs
+                for (let input of targetComponent.inputs) {
+                    if (!input.prev) input.prev = [];
+                    input.prev.push(sourceComponent.id);
+                }
+            }
         }
 
         this.tagAsyncComponents();
@@ -183,8 +204,12 @@ export class Agent implements IAgent {
         const sessionTags = this?.agentRequest?.headers['x-session-tag'];
         if (sessionTags) this.sessionTag += this.sessionTag ? `,${sessionTags}` : sessionTags;
 
-        var regex = new RegExp(`^\/v[0-9]+(\.[0-9]+)?${this.apiBasePath}\/(.*)`);
-        if (this.agentRequest?.path?.startsWith(`${this.apiBasePath}/`) || this.agentRequest?.path?.match(regex)) {
+        var regex = new RegExp(`^\/v[0-9]+(\.[0-9]+)?(${this.apiBasePath}|${this.triggerBasePath})\/(.*)`);
+        if (
+            this.agentRequest?.path?.startsWith(`${this.apiBasePath}/`) ||
+            this.agentRequest?.path?.startsWith(`${this.triggerBasePath}/`) ||
+            this.agentRequest?.path?.match(regex)
+        ) {
             //we only need runtime context for API calls
             this.agentRuntime = new AgentRuntime(this);
             this.callerSessionId =
@@ -197,11 +222,11 @@ export class Agent implements IAgent {
         this.callback = callback;
     }
 
-    public kill() {
-        this._kill = true;
+    public kill(reason: string = 'kill') {
+        this._killReason = reason;
     }
     public isKilled() {
-        return this._kill;
+        return !!this._killReason;
     }
     private async parseVariables() {
         //parse vault agent variables
@@ -216,7 +241,7 @@ export class Agent implements IAgent {
         }
     }
 
-    @hookAsync('Agent.process')
+    @hookAsync('SREAgent.process')
     async process(endpointPath, input) {
         await this.agentRuntime.ready();
 
@@ -252,7 +277,7 @@ export class Agent implements IAgent {
         });
 
         const method = this.agentRequest.method.toUpperCase();
-        const endpoint = this.endpoints[endpointPath]?.[method];
+        const endpoint = this.endpoints[endpointPath]?.[method] || this.triggers[endpointPath];
 
         //first check if this is a debug session, and return debug result if it's the case
         if (this.agentRuntime.debug) {
@@ -307,9 +332,9 @@ export class Agent implements IAgent {
             const qosLatency = Math.floor(OSResourceMonitor.cpu.load * MAX_LATENCY || 0);
 
             await delay(10 + qosLatency);
-        } while (!step?.finalResult && !this._kill);
+        } while (!step?.finalResult && !this._killReason);
 
-        if (this._kill) {
+        if (this._killReason) {
             const endTime = Date.now();
             const duration = endTime - startTime;
             this.sse.send('agent', {
@@ -325,7 +350,7 @@ export class Agent implements IAgent {
                 error: 'Agent killed',
             });
             console.warn(`Agent ${this.id} was killed`, AccessCandidate.agent(this.id));
-            return { error: 'Agent killed' };
+            return { error: 'AGENT_KILLED', reason: this._killReason };
         }
         result = await this.postProcess(step?.finalResult).catch((error) => ({ error }));
 
@@ -379,7 +404,7 @@ export class Agent implements IAgent {
         //tasks count update logic
     }
 
-    @hookAsync('Agent.postProcess')
+    @hookAsync('SREAgent.postProcess')
     public async postProcess(result) {
         if (Array.isArray(result)) result = result.flat(Infinity);
         if (!Array.isArray(result)) result = [result];
@@ -392,6 +417,8 @@ export class Agent implements IAgent {
             if (!_result) continue;
             if (_result._debug) delete _result._debug;
             if (_result._debug_time) delete _result._debug_time;
+            if (_result.result?._debug) delete _result.result._debug;
+            if (_result.result?._debug_time) delete _result.result._debug_time;
             const _componentData = this.components[_result.id];
             if (!_componentData) continue;
             const _component: Component = this._componentInstance[_componentData.name];
@@ -425,16 +452,31 @@ export class Agent implements IAgent {
     //     this.agentRuntime.resetComponent(componentId);
     // }
 
+    // private hasLoopAncestor(inputEntry) {
+    //     if (!inputEntry.prev) return false;
+    //     for (let prevId of inputEntry.prev) {
+    //         const prevComponentData = this.components[prevId];
+    //         if (prevComponentData.name == 'ForEach') return true;
+
+    //         for (let inputEntry of prevComponentData.inputs) {
+    //             if (this.hasLoopAncestor(inputEntry)) return true;
+    //         }
+    //     }
+    // }
     private hasLoopAncestor(inputEntry) {
         if (!inputEntry.prev) return false;
         for (let prevId of inputEntry.prev) {
             const prevComponentData = this.components[prevId];
-            if (prevComponentData.name == 'ForEach') return true;
+            const prevRuntimeData = this.agentRuntime.getRuntimeData(prevId);
+
+            // Consider any component with _LoopData as a loop ancestor
+            if (prevRuntimeData?._LoopData) return true;
 
             for (let inputEntry of prevComponentData.inputs) {
                 if (this.hasLoopAncestor(inputEntry)) return true;
             }
         }
+        return false;
     }
 
     private clearChildLoopRuntimeComponentData(componentId) {
@@ -527,7 +569,7 @@ export class Agent implements IAgent {
         agentRuntime.updateComponent(componentId, { step });
     }
 
-    @hookAsync('Agent.callComponent')
+    @hookAsync('SREAgent.callComponent')
     async callComponent(sourceId, componentId, input?) {
         const startTime = Date.now();
         const agentRuntime = this.agentRuntime;
@@ -547,7 +589,7 @@ export class Agent implements IAgent {
             input,
         });
 
-        if (this._kill) {
+        if (this._killReason) {
             console.warn(`Agent ${this.id} was killed, skipping component ${componentData.name}`, AccessCandidate.agent(this.id));
 
             const output = { id: componentData.id, name: componentData.displayName, result: null, error: 'Agent killed' };
@@ -850,7 +892,7 @@ export class Agent implements IAgent {
         return currentProperty;
     }
 
-    @hookAsync('Agent.callNextComponents')
+    @hookAsync('SREAgent.callNextComponents')
     async callNextComponents(componentId, output) {
         const agentRuntime = this.agentRuntime;
         //agentRuntime.incStep();
@@ -914,6 +956,16 @@ export class Agent implements IAgent {
             if (Array.isArray(connections) && connections.length > 0) {
                 const nextInput = {};
                 for (let connection of connections) {
+                    // if (component instanceof Trigger) {
+                    //     //handle trigger connections
+                    //     console.log('Trigger', connection);
+                    //     for (let input of targetComponentData.inputs) {
+                    //         if (connection.output?.Payload) {
+                    //             nextInput[input.name] = TemplateString(input.defaultVal).parse(connection.output?.Payload).clean().result;
+                    //         }
+                    //     }
+                    //     continue;
+                    // }
                     const output = connection.output;
                     const componentData = connection.componentData;
                     const outputEndpoint = componentData.outputs[connection.sourceIndex]; //source

package/src/subsystems/AgentManager/AgentData.service/AgentDataConnector.ts

@@ -52,7 +52,13 @@ const openapiEndpointTemplate = JSON.stringify({
         },
     },
 });
+
+//TODO: use SecureConnector instead of Connector to harden agent data security
+//When implemented, the getEphemeralAgentData becomes part of the default requester
 export abstract class AgentDataConnector extends Connector implements IAgentDataConnector {
+    //Ephemeral agent data is used to store data that is not persistent and is only available for the current session
+    //this is usually the case of programmatic agents implemented with the SDK
+    static ephemeralAgentData: Map<string, any> = new Map();
     public name = 'AgentDataConnector';
     public abstract getAgentData(agentId: string, version?: string): Promise<any>;
     public abstract getAgentIdByDomain(domain: string): Promise<string>;
@@ -76,13 +82,14 @@ export abstract class AgentDataConnector extends Connector implements IAgentData
 
         const apiBasePath = version && version != 'latest' ? `/v${version}/api` : '/api';
 
-        const agentData: any = typeof source === 'object' ? source : await this.getAgentData(source, version);
+        let agentData: any = typeof source === 'object' ? source : await this.getAgentData(source, version);
+        if (agentData.data) agentData = agentData.data;
         const name = agentData.name;
 
-        let description = aiOnly ? agentData.data.behavior : agentData.data.shortDescription;
-        if (!description) description = agentData.data.description; //data.description is deprecated, we just use it as a fallback for now
+        let description = aiOnly ? agentData.behavior : agentData.shortDescription;
+        if (!description) description = agentData.description; //data.description is deprecated, we just use it as a fallback for now
 
-        const _version = agentData.data.version || '1.0.0';
+        const _version = agentData.version || '1.0.0';
 
         const openAPITpl = TemplateString(openapiTemplate)
             .parse({
@@ -94,7 +101,7 @@ export abstract class AgentDataConnector extends Connector implements IAgentData
             .clean().result;
         const openAPIObj = JSON.parse(openAPITpl);
 
-        const components = agentData.data.components.filter((component: any) => component.name === 'APIEndpoint');
+        const components = agentData.components.filter((component: any) => component.name === 'APIEndpoint');
         for (let component of components) {
             const ai_exposed = component.data.ai_exposed || typeof component.data.ai_exposed === 'undefined';
             if (aiOnly && !ai_exposed) continue;
@@ -107,7 +114,7 @@ export abstract class AgentDataConnector extends Connector implements IAgentData
                         summary: summary?.replace(/"/g, '\\"'),
                         operationId: component?.data?.endpoint,
                     })
-                    .clean().result,
+                    .clean().result
             ).tryParse();
 
             if (typeof openAPIEntry !== 'object') {
@@ -188,6 +195,23 @@ export abstract class AgentDataConnector extends Connector implements IAgentData
 
         return openAPIObj;
     }
+
+    /**
+     * Sets ephemeral agent data for the given agent ID
+     * @param agentId
+     * @param data
+     */
+    public async setEphemeralAgentData(agentId: string, data: any) {
+        AgentDataConnector.ephemeralAgentData.set(agentId, data);
+    }
+    /**
+     * Gets ephemeral agent data for the given agent ID
+     * @param agentId
+     * @returns
+     */
+    public async getEphemeralAgentData(agentId: string) {
+        return AgentDataConnector.ephemeralAgentData.get(agentId);
+    }
 }
 
 function getOpenAPIInputSchema(input_type) {

package/src/subsystems/AgentManager/AgentData.service/connectors/NullAgentData.class.ts

@@ -12,11 +12,11 @@ export class NullAgentData extends AgentDataConnector {
     }
 
     public getAgentConfig(agentId: string): Partial<TArgs> {
-        return {};
+        return null; //{};
     }
 
     public async getAgentData(agentId: string, version?: string) {
-        return { data: {}, version: '1.0' };
+        return null; // { data: {}, version: '1.0' };
     }
 
     public getAgentIdByDomain(domain: string): Promise<string> {

package/src/subsystems/AgentManager/AgentLogger.class.ts

@@ -261,7 +261,7 @@ export class AgentLogger {
     public static log(agent, trId, logData: AgentCallLog) {
         const logConnector = ConnectorService.getLogConnector();
         if (!logConnector.valid) return;
-        if (agent.agentRuntime.debug) logData.tags = 'DEBUG ';
+        if (agent?.agentRuntime?.debug || agent?.debugSessionEnabled) logData.tags = 'DEBUG ';
         if (!trId) trId = 'log-' + uid();
         if (!this.transactions[trId]) {
             this.transactions[trId] = new LogTransaction(agent, trId);

package/src/subsystems/AgentManager/AgentRuntime.class.ts

@@ -140,6 +140,11 @@ export class AgentRuntime {
 
         this.agentContext = new RuntimeContext(this);
         this.agentContext.on('ready', () => {
+            let method = (agent.agentRequest.method || 'POST').toUpperCase();
+            const endpoint = agent.endpoints?.[agent.agentRequest.path]?.[method];
+            const trigger = agent.triggers?.[agent.agentRequest.path];
+            const endpointDBGCall = this.xDebugId?.startsWith('dbg-');
+
             this.alwaysActiveComponents = {};
             this.exclusiveComponents = {};
             for (let component of this.agent.data.components) {
@@ -149,6 +154,15 @@ export class AgentRuntime {
                     continue;
                 }
 
+                if (trigger && trigger.id != undefined && component.id == trigger.id && this.xDebugId?.startsWith('dbg-')) {
+                    this.updateComponent(component.id, { active: true });
+                }
+
+                //if this debug session was initiated from an endpoint, we mark the endpoint component as active
+                if (endpoint && endpoint.id != undefined && component.id == endpoint.id && endpointDBGCall) {
+                    this.updateComponent(component.id, { active: true });
+                }
+
                 if (cpt.alwaysActive) {
                     this.alwaysActiveComponents[component.id] = cpt;
                     this.updateComponent(component.id, { active: true, alwaysActive: true });
@@ -380,7 +394,20 @@ export class AgentRuntime {
                 ctxData.sessionResult = true;
             }
 
-            //capture results
+            //capture triggers results
+
+            const triggers = Object.values(agent.triggers);
+            let triggersResults = dbgResults.flat().filter(
+                (e) =>
+                    e.id &&
+                    e.result &&
+                    !e.result._missing_inputs &&
+                    !e.result._in_progress &&
+                    //check if the current result is a trigger
+                    triggers.find((t: any) => t.id == e.id)
+            );
+
+            //capture workflow results
             let sessionResults = dbgResults.flat().filter(
                 (e) =>
                     e.id &&
@@ -390,6 +417,8 @@ export class AgentRuntime {
                     !agent.connections.find((c) => c.sourceId == e.id)
             );
 
+            sessionResults = sessionResults.concat(triggersResults);
+
             let errorResults = dbgResults.flat().filter((e) => e.id && (e.error || e.result?._error));
 
             //also filter out erroneous components that are children of a loop
@@ -452,13 +481,13 @@ export class AgentRuntime {
             const circularLimitData = this.circularLimitReached;
             result = { error: `Circular Calls Limit Reached on ${circularLimitData}. Current circular limit is ${this.agent.circularLimit}` };
         } else {
-            let state = [sessionResults, errorResults].flat(Infinity);
-            if (!state || state.length == 0) state = errorResults.flat(Infinity);
+            let _state = [sessionResults, errorResults].flat(Infinity);
+            if (!_state || _state.length == 0) _state = errorResults.flat(Infinity);
 
             //post process run cycle results
             //deduplicating redundant entries
 
-            const data = state
+            const _data = _state
                 .reduce(
                     (acc, current) => {
                         if (!acc.seen[current.id]) {
@@ -473,7 +502,7 @@ export class AgentRuntime {
 
             //data.forEach((d: any) => delete d?.result?._debug);
 
-            result = data;
+            result = _data;
             /////////////
         }