@smythos/sre 1.6.14 → 1.7.5

package/src/Core/ExternalEventsReceiver.ts

@@ -0,0 +1,317 @@
+import { createServer, IncomingMessage, ServerResponse } from 'http';
+import { WebSocket, WebSocketServer } from 'ws';
+import { SystemEvents } from './SystemEvents';
+import { Logger } from '../helpers/Log.helper';
+import { ConnectorService } from './ConnectorsService';
+import { Connector } from './Connector.class';
+import { TConnectorService } from '@sre/types/SRE.types';
+import { validateServiceKey } from '@sre/helpers/Crypto.helper';
+
+const logger = Logger('ExternalEventsReceiver');
+
+export interface ExternalEventsReceiverConfig {
+    port: number;
+    authTokens: string[]; // List of valid authentication tokens
+    enableHttp?: boolean; // Enable HTTP endpoint (default: true)
+    enableWebSocket?: boolean; // Enable WebSocket endpoint (default: true)
+    path?: string; // WebSocket path (default: '/ws')
+}
+
+export class ExternalEventsReceiver {
+    private server: ReturnType<typeof createServer> | null = null;
+    private wss: WebSocketServer | null = null;
+    private config: Required<ExternalEventsReceiverConfig>;
+    private isRunning = false;
+
+    constructor(config: ExternalEventsReceiverConfig) {
+        this.config = {
+            enableHttp: true,
+            enableWebSocket: true,
+            path: '/ws',
+            ...config,
+        };
+
+        if (!this.config.authTokens || this.config.authTokens.length === 0) {
+            logger.warn('At least one authentication token must be provided');
+            throw new Error('At least one authentication token must be provided');
+        }
+    }
+
+    /**
+     * Start the HTTP and WebSocket server
+     */
+    public async start(): Promise<void> {
+        if (this.isRunning) {
+            logger.warn('Server is already running');
+            return;
+        }
+
+        // Create HTTP server
+        this.server = createServer((req, res) => this.handleHttpRequest(req, res));
+
+        // Create WebSocket server if enabled
+        if (this.config.enableWebSocket) {
+            this.wss = new WebSocketServer({
+                server: this.server,
+                path: this.config.path,
+            });
+
+            this.wss.on('connection', (ws: WebSocket, req: IncomingMessage) => {
+                this.handleWebSocketConnection(ws, req);
+            });
+        }
+
+        return new Promise((resolve, reject) => {
+            this.server!.listen(this.config.port, '127.0.0.1', () => {
+                this.isRunning = true;
+                logger.debug(
+                    `Server started on port ${this.config.port}` +
+                        (this.config.enableHttp ? ' [HTTP]' : '') +
+                        (this.config.enableWebSocket ? ` [WebSocket: ${this.config.path}]` : '')
+                );
+                resolve();
+            });
+
+            this.server!.on('error', (error) => {
+                logger.error('Server error', error);
+                reject(error);
+            });
+        });
+    }
+
+    /**
+     * Stop the server
+     */
+    public async stop(): Promise<void> {
+        if (!this.isRunning) {
+            return;
+        }
+
+        return new Promise((resolve, reject) => {
+            // Close WebSocket server first
+            if (this.wss) {
+                this.wss.close((err) => {
+                    if (err) {
+                        logger.error('Error closing WebSocket server', err);
+                    }
+                });
+            }
+
+            // Close HTTP server
+            if (this.server) {
+                this.server.close((err) => {
+                    if (err) {
+                        logger.error('Error closing HTTP server', err);
+                        reject(err);
+                    } else {
+                        this.isRunning = false;
+                        logger.debug('Server stopped');
+                        resolve();
+                    }
+                });
+            } else {
+                resolve();
+            }
+        });
+    }
+
+    /**
+     * Handle HTTP requests
+     */
+    private handleHttpRequest(req: IncomingMessage, res: ServerResponse): void {
+        if (!this.config.enableHttp) {
+            this.sendResponse(res, 404, { error: 'HTTP endpoint is disabled' });
+            return;
+        }
+
+        // Only accept POST requests
+        if (req.method !== 'POST') {
+            this.sendResponse(res, 405, { error: 'Method not allowed. Only POST requests are accepted.' });
+            return;
+        }
+
+        // Validate authentication
+        const authResult = this.validateAuth(req.headers);
+        if (!authResult.valid) {
+            this.sendResponse(res, 401, { error: authResult.error });
+            return;
+        }
+
+        // Get connector name from header
+        const connectorName = this.getConnectorName(req.headers);
+        if (!connectorName) {
+            this.sendResponse(res, 400, { error: 'Missing x-connector-name header' });
+            return;
+        }
+
+        // Read request body
+        let body = '';
+        req.on('data', (chunk) => {
+            body += chunk.toString();
+        });
+
+        req.on('end', () => {
+            try {
+                const data = body ? JSON.parse(body) : {};
+
+                // Emit event
+                this.emitExternalEvent(connectorName, data);
+
+                this.sendResponse(res, 200, {
+                    success: true,
+                    message: `Event EXT:${connectorName} emitted successfully`,
+                });
+            } catch (error) {
+                logger.error('Error processing HTTP request', error);
+                this.sendResponse(res, 400, {
+                    error: 'Invalid JSON payload',
+                });
+            }
+        });
+
+        req.on('error', (error) => {
+            logger.error('HTTP request error', error);
+            this.sendResponse(res, 500, { error: 'Internal server error' });
+        });
+    }
+
+    /**
+     * Handle WebSocket connections
+     */
+    private handleWebSocketConnection(ws: WebSocket, req: IncomingMessage): void {
+        logger.debug('New WebSocket connection');
+
+        // Validate authentication from headers
+        const authResult = this.validateAuth(req.headers);
+        if (!authResult.valid) {
+            ws.close(1008, authResult.error);
+            return;
+        }
+
+        // Get connector name from headers
+        let connectorName = this.getConnectorName(req.headers);
+
+        ws.on('message', (message: Buffer) => {
+            try {
+                const data = JSON.parse(message.toString());
+
+                // If connector name not in headers, check message
+                if (!connectorName) {
+                    connectorName = data.connectorName || data['x-connector-name'];
+
+                    if (!connectorName) {
+                        ws.send(
+                            JSON.stringify({
+                                error: 'Missing connector name in headers or message',
+                            })
+                        );
+                        return;
+                    }
+                }
+
+                // Emit event
+                this.emitExternalEvent(connectorName, data);
+
+                // Send acknowledgment
+                ws.send(
+                    JSON.stringify({
+                        success: true,
+                        message: `Event EXT:${connectorName} emitted successfully`,
+                    })
+                );
+            } catch (error) {
+                logger.error('Error processing WebSocket message', error);
+                ws.send(
+                    JSON.stringify({
+                        error: 'Invalid JSON message',
+                    })
+                );
+            }
+        });
+
+        ws.on('error', (error) => {
+            logger.error('WebSocket error', error);
+        });
+
+        ws.on('close', () => {
+            logger.debug('WebSocket connection closed');
+        });
+    }
+
+    /**
+     * Validate authentication token
+     */
+    private validateAuth(headers: IncomingMessage['headers']): {
+        valid: boolean;
+        serviceName?: string;
+        error?: string;
+    } {
+        const authHeader = headers['authorization'] || headers['Authorization'];
+        const serviceName = headers['x-service-name'] || headers['X-Service-Name'];
+
+        if (!authHeader) {
+            logger.warn('Missing Authorization header');
+            return { valid: false, error: 'Missing Authorization header' };
+        }
+
+        if (!serviceName || typeof serviceName !== 'string') {
+            logger.warn('Missing or invalid X-Service-Name header');
+            return { valid: false, error: 'Missing X-Service-Name header' };
+        }
+
+        // Extract token
+        const token = (authHeader as string).replace(/^Bearer\s+/i, '');
+
+        if (!token) {
+            return { valid: false, error: 'Invalid Authorization header format' };
+        }
+
+        // Validate using service key derivation
+        if (!validateServiceKey(serviceName, token)) {
+            logger.warn(`Invalid key for service: ${serviceName}`);
+            return { valid: false, error: 'Invalid authentication' };
+        }
+
+        logger.debug(`Authenticated service: ${serviceName}`);
+        return { valid: true, serviceName };
+    }
+
+    /**
+     * Extract connector name from headers
+     */
+    private getConnectorName(headers: IncomingMessage['headers']): string | null {
+        const connectorName = headers['x-connector-name'] || headers['X-Connector-Name'];
+        return typeof connectorName === 'string' ? connectorName : connectorName?.[0] || null;
+    }
+
+    /**
+     * Emit external connector event
+     */
+    private emitExternalEvent(connectorName: string, data: any): void {
+        const eventName = `EXT:${connectorName}` as const;
+        const connector = ConnectorService.getInstance<Connector>(TConnectorService[connectorName]);
+        if (connector) {
+            connector.handleEvent(eventName, data);
+        }
+    }
+
+    /**
+     * Send HTTP response
+     */
+    private sendResponse(res: ServerResponse, statusCode: number, body: any): void {
+        res.statusCode = statusCode;
+        res.setHeader('Content-Type', 'application/json');
+        res.end(JSON.stringify(body));
+    }
+
+    /**
+     * Get server status
+     */
+    public getStatus(): { running: boolean; port: number; config: Required<ExternalEventsReceiverConfig> } {
+        return {
+            running: this.isRunning,
+            port: this.config.port,
+            config: this.config,
+        };
+    }
+}

package/src/Core/HookService.ts

@@ -118,14 +118,26 @@ export function hook(hookName: string) {
  * Decorator function that executes registered hooks asynchronously before and after the decorated method
  * @param hookName The name of the hook to trigger
  */
-export function hookAsync(hookName: string) {
+export function hookAsync(hookName: string, customContext?: Record<string, any> | ((instance: any) => Record<string, any>)) {
     return function (target: any, propertyKey: string, descriptor: PropertyDescriptor) {
         const originalMethod = descriptor.value;
 
         descriptor.value = async function (...args: any[]) {
+            // Get additional context if contextFn is provided
+            let _context;
+            if (typeof customContext === 'function') {
+                _context = await customContext(this);
+            } else if (typeof customContext === 'object') {
+                _context = customContext;
+            } else {
+                _context = this;
+            }
+
             // Execute non-blocking pre-hooks first (fire and forget)
             if (nonBlockingHooks[hookName]) {
-                void Promise.allSettled(nonBlockingHooks[hookName].map((callback) => Promise.resolve().then(() => callback.apply(this, args))));
+                void Promise.allSettled(nonBlockingHooks[hookName].map((callback) => Promise.resolve(callback.apply(_context, args)))).catch((err) =>
+                    console.error(`Non-blocking hook ${hookName} error:`, err)
+                );
             }
 
             let result: any;
@@ -134,7 +146,7 @@ export function hookAsync(hookName: string) {
             try {
                 // Execute blocking pre-hooks and wait for them
                 if (blockingHooks[hookName]) {
-                    await Promise.all(blockingHooks[hookName].map((callback) => Promise.resolve(callback.apply(this, args))));
+                    await Promise.all(blockingHooks[hookName].map((callback) => Promise.resolve(callback.apply(_context, args))));
                 }
 
                 // Call the original method
@@ -148,13 +160,15 @@ export function hookAsync(hookName: string) {
                 // Execute non-blocking after-hooks first (fire and forget)
                 if (nonBlockingAfterHooks[hookName]) {
                     void Promise.allSettled(
-                        nonBlockingAfterHooks[hookName].map((callback) => Promise.resolve().then(() => callback({ result, args, error })))
-                    );
+                        nonBlockingAfterHooks[hookName].map((callback) => Promise.resolve(callback.apply(_context, [{ result, args, error }])))
+                    ).catch((err) => console.error(`Non-blocking after-hook ${hookName} error:`, err));
                 }
 
                 // Execute blocking after-hooks and wait for them
                 if (blockingAfterHooks[hookName]) {
-                    await Promise.all(blockingAfterHooks[hookName].map((callback) => Promise.resolve(callback({ result, args, error }))));
+                    await Promise.all(
+                        blockingAfterHooks[hookName].map((callback) => Promise.resolve(callback.apply(_context, [{ result, args, error }])))
+                    );
                 }
             } catch (afterHookError) {
                 // Log after-hook errors but don't let them override the original error

package/src/Core/SmythRuntime.class.ts

@@ -9,6 +9,8 @@ import pkg from '../../package.json';
 
 const logger = Logger('SRE');
 
+const SRE_GLOBAL_KEY = Symbol.for('SRE@singleton');
+
 export class SmythRuntime {
     public started = false;
 
@@ -67,6 +69,13 @@ export class SmythRuntime {
         Router: {
             Connector: 'NullRouter',
         },
+        Scheduler: {
+            Connector: 'LocalScheduler',
+            Settings: {
+                runJobs: true,
+                persistExecutionHistory: true,
+            },
+        },
     };
 
     protected constructor() {
@@ -77,10 +86,19 @@ export class SmythRuntime {
     }
 
     protected static instance?: SmythRuntime;
+    // public static get Instance(): SmythRuntime {
+    //     if (!SmythRuntime.instance) {
+    //         SmythRuntime.instance = new SmythRuntime();
+    //     }
+    //     return SmythRuntime.instance;
+    // }
+
     public static get Instance(): SmythRuntime {
-        if (!SmythRuntime.instance) {
-            SmythRuntime.instance = new SmythRuntime();
+        if (global[SRE_GLOBAL_KEY]) {
+            return global[SRE_GLOBAL_KEY];
         }
+        SmythRuntime.instance = new SmythRuntime();
+        global[SRE_GLOBAL_KEY] = SmythRuntime.instance;
         return SmythRuntime.instance;
     }
 

package/src/Core/SystemEvents.ts

@@ -1,6 +1,11 @@
 import { SmythLLMUsage, SmythTaskUsage } from '@sre/types/LLM.types';
 import { TServiceRegistry } from '@sre/types/SRE.types';
 import { EventEmitter } from 'events';
+import { ExternalEventsReceiver } from './ExternalEventsReceiver';
+import { createHash } from 'crypto';
+import { Logger } from '../helpers/Log.helper';
+
+const logger = Logger('SystemEvents');
 
 export type SystemEventMap = {
     'SRE:BootStart': [];
@@ -13,4 +18,16 @@ export type SystemEventMap = {
 
 const SystemEvents = new EventEmitter<SystemEventMap>();
 
+// /!\ incomplete implementation, do not enable yet
+// if (process.env?.SRE_SECRET?.trim()) {
+//     const secretHash = createHash('sha256').update(process.env.SRE_SECRET).digest('hex');
+//     // Create server instance
+//     new ExternalEventsReceiver({
+//         port: process.env.SRE_PORT ? parseInt(process.env.SRE_PORT) : 55555,
+//         authTokens: [secretHash],
+//     });
+// } else {
+//     logger.warn('SRE_SECRET is not set, external events receiver will not be started');
+// }
+
 export { SystemEvents };

package/src/Core/boot.ts

@@ -16,6 +16,7 @@ import { LogService } from '@sre/IO/Log.service';
 import { ComponentService } from '@sre/AgentManager/Component.service';
 import { ModelsProviderService } from '@sre/LLMManager/ModelsProvider.service';
 import { CodeService } from '@sre/ComputeManager/Code.service';
+import { SchedulerService } from '@sre/AgentManager/Scheduler.service';
 const console = Logger('Boot');
 let _booted = false;
 export function boot() {
@@ -42,6 +43,7 @@ export function boot() {
     service.Log = new LogService();
     service.Component = new ComponentService();
     service.Code = new CodeService();
+    service.Scheduler = new SchedulerService();
 
     SystemEvents.on('SRE:Initialized', () => {
         console.debug('SRE Initialized');

package/src/helpers/BinaryInput.helper.ts

@@ -23,7 +23,7 @@ export class BinaryInput {
         data: BinaryInput | Buffer | ArrayBuffer | Blob | string | Record<string, any>,
         private _name?: string,
         public mimetype?: string,
-        private candidate?: IAccessCandidate,
+        private candidate?: IAccessCandidate
     ) {
         if (!_name) _name = uid();
         this._name = _name;
@@ -97,6 +97,9 @@ export class BinaryInput {
                     this._ready = true;
                 }
             } else {
+                //try to guess the mimetype from the url extension
+                const ext = this.url.split('.').pop();
+                this.mimetype = mime.getType(ext) || '';
                 this._ready = true;
             }
             return;
@@ -205,8 +208,6 @@ export class BinaryInput {
             this._ready = true;
             return;
         }
-
-        
     }
 
     private async getUrlInfo(url) {
@@ -221,9 +222,9 @@ export class BinaryInput {
         }
     }
 
-    public static from(data, name?: string, mimetype?: string, candidate?: IAccessCandidate) {
-        if (data instanceof BinaryInput) return data;
-        return new BinaryInput(data, name, mimetype, candidate);
+    public static from(source, name?: string, mimetype?: string, candidate?: IAccessCandidate) {
+        if (source instanceof BinaryInput) return source;
+        return new BinaryInput(source, name, mimetype, candidate);
     }
 
     public async upload(candidate: IAccessCandidate, ttl?: number) {
@@ -239,8 +240,7 @@ export class BinaryInput {
                 this.url = `smythfs://${teamId}.team/${candidate.id}/_temp/${this._name}`;
                 await SmythFS.Instance.write(this.url, this._source, candidate, undefined, ttl);
                 this._uploading = false;
-            }
-            else {
+            } else {
                 this._uploading = false;
             }
         } catch (error) {

package/src/helpers/Conversation.helper.ts

@@ -80,6 +80,7 @@ export class Conversation extends EventEmitter {
     public set spec(specSource) {
         this.ready.then(() => {
             this._status = '';
+            this._currentWaitPromise = undefined;
             this.loadSpecFromSource(specSource).then(async (spec) => {
                 if (!spec) {
                     this._status = 'error';
@@ -223,6 +224,7 @@ export class Conversation extends EventEmitter {
             teamId: instance._teamId,
             agentId: instance._agentId,
             model: instance._model,
+            agentData: instance.agentData,
         };
     })
     public async prompt(message?: string | any, toolHeaders = {}, concurrentToolCalls = 4, abortSignal?: AbortSignal) {
@@ -249,6 +251,7 @@ export class Conversation extends EventEmitter {
             teamId: instance._teamId,
             agentId: instance._agentId,
             model: instance._model,
+            agentData: instance.agentData,
         };
     })
     public async streamPrompt(message?: string | any, toolHeaders = {}, concurrentToolCalls = 4, abortSignal?: AbortSignal) {
@@ -401,6 +404,11 @@ export class Conversation extends EventEmitter {
                     llmMessage.thinkingBlocks = thinkingBlocks;
                 }
 
+                //add tool status for every tool entry
+                toolsData.forEach((tool) => {
+                    tool.status = tool.name ? this._toolStatusMap?.[tool.name] : undefined;
+                });
+
                 llmMessage.tool_calls = toolsData.map((tool) => {
                     return {
                         id: tool.id,
@@ -677,7 +685,10 @@ export class Conversation extends EventEmitter {
                 const requiresRemoteCall =
                     reqConfig.headers['X-DEBUG'] !== undefined ||
                     reqConfig.headers['X-MONITOR-ID'] !== undefined ||
-                    reqConfig.headers['X-AGENT-REMOTE-CALL'] !== undefined;
+                    //This is used for cases that requre to inject a default attachment handler
+                    //TODO : this need to be removed from the conversation helper in the future since default attachment handler is a specific server feature
+                    //reqConfig.headers['X-AGENT-HAS-ATTACHMENTS'] !== undefined;
+                    reqConfig.headers['x-conversation-id'] !== undefined;
 
                 if (canRunLocally && !requiresRemoteCall) {
                     console.log('RUNNING AGENT LOCALLY');
@@ -783,19 +794,24 @@ export class Conversation extends EventEmitter {
             const _arguments: any = {};
             for (let arg of openApiArgs) {
                 _arguments[arg.name] = arg.schema;
-                if (tool.inputs && arg.schema.properties) {
-                    const required = [];
-                    for (let prop in arg.schema.properties) {
-                        const input = tool.inputs?.find((i) => i.name === prop);
-                        if (!arg.schema.properties[prop].description) {
-                            arg.schema.properties[prop].description = input?.description;
+                if (tool.inputs) {
+                    if (arg.schema.properties) {
+                        const required = [];
+                        for (let prop in arg.schema.properties) {
+                            const input = tool.inputs?.find((i) => i.name === prop);
+                            if (!arg.schema.properties[prop].description) {
+                                arg.schema.properties[prop].description = input?.description;
+                            }
+                            if (!input?.optional) {
+                                required.push(prop);
+                            }
                         }
-                        if (!input?.optional) {
-                            required.push(prop);
+                        if (required.length) {
+                            arg.schema.required = required;
                         }
-                    }
-                    if (required.length) {
-                        arg.schema.required = required;
+                    } else {
+                        const input = tool.inputs?.find((i) => i.name === arg.name);
+                        arg.schema.description = input?.description;
                     }
                 }
             }
@@ -847,6 +863,23 @@ export class Conversation extends EventEmitter {
 
         this._toolsConfig = toolsConfig;
     }
+
+    async removeTool(toolName: string) {
+        this._customToolsDeclarations = this._customToolsDeclarations.filter((tool) => tool.name !== toolName);
+        delete this._customToolsHandlers[toolName];
+        const llmInference: LLMInference = await LLMInference.getInstance(this.model, AccessCandidate.team(this._teamId));
+
+        const toolsConfig: any = llmInference.connector.formatToolsConfig({
+            type: 'function',
+            toolDefinitions: this._customToolsDeclarations,
+            toolChoice: this.toolChoice,
+        });
+        this._toolsConfig = toolsConfig;
+    }
+
+    public get toolNames() {
+        return this._customToolsDeclarations.map((tool) => tool.name);
+    }
     /**
      * updates LLM model, if spec is available, it will update the tools config
      * @param model
@@ -928,6 +961,7 @@ export class Conversation extends EventEmitter {
             //is this a valid agent data?
             if (typeof specSource?.behavior === 'string' && specSource?.components && specSource?.connections) {
                 this.agentData = specSource; //agent loaded from data directly
+                this._specSource = specSource;
                 this._agentId = specSource.id;
                 return await this.loadSpecFromAgent(specSource);
             }

package/src/helpers/Crypto.helper.ts

@@ -0,0 +1,28 @@
+import { createHmac, timingSafeEqual } from 'crypto';
+
+/**
+ * Generate a service-specific authentication key
+ */
+export function generateServiceKey(serviceName: string): string {
+    const secret = process.env.SRE_SECRET;
+
+    if (!secret || secret.trim().length < 32) {
+        throw new Error('SRE_SECRET must be at least 32 characters');
+    }
+
+    return createHmac('sha256', secret).update(`sre-service:${serviceName}`).digest('hex');
+}
+
+/**
+ * Validate a service key
+ */
+export function validateServiceKey(serviceName: string, providedKey: string): boolean {
+    try {
+        const expectedKey = generateServiceKey(serviceName);
+
+        // Constant-time comparison to prevent timing attacks
+        return timingSafeEqual(Buffer.from(expectedKey), Buffer.from(providedKey));
+    } catch {
+        return false;
+    }
+}