@smythos/sre 1.5.1 → 1.5.4

package/src/Core/ConnectorsService.ts

@@ -0,0 +1,192 @@
+import { SREConnectorConfig, TConnectorService, TServiceRegistry } from '@sre/types/SRE.types';
+import { DummyConnector } from './DummyConnector';
+import { Logger } from '../helpers/Log.helper';
+import { Connector } from './Connector.class';
+import { getFormattedStackTrace, isSubclassOf, printStackTrace } from '@sre/utils';
+import { SystemEvents } from './SystemEvents';
+import { StorageConnector } from '@sre/IO/Storage.service/StorageConnector';
+import { CacheConnector } from '@sre/MemoryManager/Cache.service/CacheConnector';
+import { LLMConnector } from '@sre/LLMManager/LLM.service/LLMConnector';
+import { VaultConnector } from '@sre/Security/Vault.service/VaultConnector';
+import { AccountConnector } from '@sre/Security/Account.service/AccountConnector';
+import { AgentDataConnector } from '@sre/AgentManager/AgentData.service/AgentDataConnector';
+import { VectorDBConnector } from '@sre/IO/VectorDB.service/VectorDBConnector';
+import { CLIConnector } from '@sre/IO/CLI.service/CLIConnector';
+import { NKVConnector } from '@sre/IO/NKV.service/NKVConnector';
+import { RouterConnector } from '@sre/IO/Router.service/RouterConnector';
+import { ManagedVaultConnector } from '@sre/Security/ManagedVault.service/ManagedVaultConnector';
+import { LogConnector } from '@sre/IO/Log.service/LogConnector';
+import { ComponentConnector } from '@sre/AgentManager/Component.service/ComponentConnector';
+import { ModelsProviderConnector } from '@sre/LLMManager/ModelsProvider.service/ModelsProviderConnector';
+const console = Logger('ConnectorService');
+
+let ServiceRegistry: TServiceRegistry = {};
+let _ready = false;
+SystemEvents.on('SRE:Booted', (services) => {
+    ServiceRegistry = services;
+    _ready = true;
+});
+export class ConnectorService {
+    public static Connectors = {};
+
+    public static ConnectorInstances: any = {};
+    public static get ready() {
+        return _ready;
+    }
+
+    public static get service(): TServiceRegistry {
+        return ServiceRegistry;
+    }
+    /**
+     * Allows SRE services to register their connectors, a registered conector can then be initialized and used by SRE or its services
+     * @param connectorType
+     * @param connectorName
+     * @param connectorConstructor
+     * @returns
+     */
+    static register(connectorType: TConnectorService, connectorName: string, connectorConstructor: any) {
+        if (typeof connectorConstructor !== 'function' || !isSubclassOf(connectorConstructor, Connector)) {
+            console.error(`Invalid Connector ${connectorType}:${connectorName}`);
+            return;
+        }
+        if (!ConnectorService.Connectors[connectorType]) {
+            ConnectorService.Connectors[connectorType] = {};
+        }
+        ConnectorService.Connectors[connectorType][connectorName] = connectorConstructor;
+    }
+
+    /**
+     * The init method instantiates a connector and starts it, a connector cannot be used before it is initialized
+     * Usually the initialization phase happens during the SRE startup, but some connectors can be initialized later if they are not mandatory for the SRE to start
+     *
+     *
+     * @param connectorType
+     * @param connectorName
+     * @param settings
+     * @param isDefault
+     * @returns
+     */
+    static init(connectorType: TConnectorService, connectorName: string, connectorId?: string, settings: any = {}, isDefault = false) {
+        if (ConnectorService.ConnectorInstances[connectorType]?.[connectorName]) {
+            throw new Error(`Connector ${connectorType}:${connectorName} already initialized`);
+        }
+
+        const entry = ConnectorService.Connectors[connectorType];
+        if (!entry) return;
+        const connectorConstructor = entry[connectorName];
+
+        if (connectorConstructor) {
+            const connector: Connector = new connectorConstructor(settings);
+            if (connector.interactionHandler) {
+                connector.interactionHandler();
+            }
+
+            connector.start();
+            if (!ConnectorService.ConnectorInstances[connectorType]) ConnectorService.ConnectorInstances[connectorType] = {};
+            const id = connectorId || connectorName;
+            ConnectorService.ConnectorInstances[connectorType][id] = connector;
+
+            if (!ConnectorService.ConnectorInstances[connectorType].default && isDefault) {
+                ConnectorService.ConnectorInstances[connectorType].default = connector;
+            }
+
+            return connector;
+        }
+    }
+    static async _stop() {
+        for (let connectorName in ConnectorService.ConnectorInstances) {
+            let allConnectors: Connector[] = Object.values(ConnectorService.ConnectorInstances[connectorName]);
+            //deduplicate
+            allConnectors = allConnectors.filter((value, index, self) => self.indexOf(value) === index);
+            for (let connector of allConnectors) {
+                connector.stop();
+            }
+        }
+    }
+    static getInstance<T>(connectorType: TConnectorService, connectorName: string = 'default'): T {
+        const instance = ConnectorService.ConnectorInstances[connectorType]?.[connectorName || 'default'] as T;
+        if (!instance) {
+            //TODO only apply the fallback below if resilient mode is enabled
+            // if (ConnectorService.ConnectorInstances[connectorType] && Object.keys(ConnectorService.ConnectorInstances[connectorType]).length > 0) {
+            //     //return the first instance
+            //     return ConnectorService.ConnectorInstances[connectorType][Object.keys(ConnectorService.ConnectorInstances[connectorType])[0]] as T;
+            // }
+            console.warn(`Connector ${connectorType} not initialized returning DummyConnector`);
+            //print stack trace
+
+            printStackTrace(console, 5);
+
+            return DummyConnector(connectorType) as T;
+        }
+        return instance;
+    }
+
+    static getStorageConnector(name?: string): StorageConnector {
+        return ConnectorService.getInstance<StorageConnector>(TConnectorService.Storage, name);
+    }
+
+    static getCacheConnector(name?: string): CacheConnector {
+        return ConnectorService.getInstance<any>(TConnectorService.Cache, name);
+    }
+
+    static getVectorDBConnector(name?: string): VectorDBConnector {
+        return ConnectorService.getInstance<VectorDBConnector>(TConnectorService.VectorDB, name);
+    }
+
+    static getNKVConnector(name?: string): NKVConnector {
+        return ConnectorService.getInstance<NKVConnector>(TConnectorService.NKV, name);
+    }
+
+    static getLLMConnector(name?: string): LLMConnector {
+        return ConnectorService.getInstance<LLMConnector>(TConnectorService.LLM, name);
+    }
+
+    static getVaultConnector(name?: string): VaultConnector {
+        return ConnectorService.getInstance<VaultConnector>(TConnectorService.Vault, name);
+    }
+
+    static getManagedVaultConnector(name?: string): ManagedVaultConnector {
+        return ConnectorService.getInstance<ManagedVaultConnector>(TConnectorService.ManagedVault, name);
+    }
+
+    static getAccountConnector(name?: string): AccountConnector {
+        return ConnectorService.getInstance<AccountConnector>(TConnectorService.Account, name);
+    }
+
+    static getAgentDataConnector(name?: string): AgentDataConnector {
+        return ConnectorService.getInstance<AgentDataConnector>(TConnectorService.AgentData, name);
+    }
+
+    static getCLIConnector(name?: string): CLIConnector {
+        return ConnectorService.getInstance<CLIConnector>(TConnectorService.CLI, name);
+    }
+
+    static getLogConnector(name?: string): LogConnector {
+        return ConnectorService.getInstance<LogConnector>(TConnectorService.Log, name);
+    }
+
+    static getComponentConnector(name?: string): ComponentConnector {
+        return ConnectorService.getInstance<ComponentConnector>(TConnectorService.Component, name);
+    }
+
+    static getModelsProviderConnector(name?: string): ModelsProviderConnector {
+        return ConnectorService.getInstance<ModelsProviderConnector>(TConnectorService.ModelsProvider, name);
+    }
+
+    static hasInstance(connectorType: TConnectorService, connectorName: string = 'default') {
+        const instance = ConnectorService.ConnectorInstances[connectorType]?.[connectorName];
+        return instance && instance.valid;
+    }
+
+    static getRouterConnector(name?: string): RouterConnector {
+        return ConnectorService.getInstance<RouterConnector>(TConnectorService.Router, name);
+    }
+}
+
+export abstract class ConnectorServiceProvider {
+    public abstract register();
+    public init() {}
+    public constructor() {
+        this.register();
+    }
+}

package/src/Core/DummyConnector.ts

@@ -0,0 +1,49 @@
+import { Logger } from '../helpers/Log.helper';
+import { getFormattedStackTrace, printStackTrace } from '../utils';
+
+/**
+ * DummyConnector is a placeholder for unimplemented connectors, it logs a warning when a method is called in order to help developers identify missing connectors
+ */
+export const DummyConnector: any = (name: string) => {
+    const logger = Logger(`DummyConnector<${name}>`);
+    return new Proxy(
+        {},
+        {
+            get: function (target, prop, receiver) {
+                //check if we are accessing the valid property
+                if (prop === 'valid') {
+                    return false; //when DummyConnector is used it means that the main connector failed to load
+                }
+
+                // Check if the property being accessed is a function
+                if (typeof target[prop] === 'function') {
+                    return target[prop];
+                } else {
+                    // Return a function that logs "unavailable"
+                    return function (...args: any[]) {
+                        const argsString =
+                            args.length > 0
+                                ? args
+                                      .map((arg) => {
+                                          if (typeof arg === 'object') return JSON.stringify(arg, null, 0).slice(0, 50) + '...';
+                                          if (typeof arg === 'string') return `"${arg.slice(0, 50)}..."`;
+                                          if (typeof arg === 'number') return arg.toString();
+                                          if (typeof arg === 'boolean') return arg.toString();
+                                          if (typeof arg === 'function') return arg.toString();
+                                          if (typeof arg === 'symbol') return arg.toString();
+                                          if (typeof arg === 'undefined') return 'undefined';
+
+                                          return String(arg);
+                                      })
+                                      .join(', ')
+                                : '(no arguments)';
+
+                        logger.warn(`[!!] Unimplemented Connector tried to call: ${name}.${prop.toString()}(${argsString})`);
+
+                        printStackTrace(logger, 3, 1); //the argument 1 means that we skip one more strack element because we are calling printStackTrace from an anonymous function
+                    };
+                }
+            },
+        },
+    );
+};

package/src/Core/HookService.ts

@@ -0,0 +1,105 @@
+// Type definition for hook callbacks
+type HookCallback = (...args: any[]) => void;
+
+// Store hooks in a map where each hook name can have multiple callbacks
+const hooks: { [key: string]: HookCallback[] } = {};
+
+export class HookService {
+    /**
+     * Register a new hook callback for a given hook name
+     * @param hookName The name of the hook to register
+     * @param callback The callback function to execute when the hook is triggered
+     */
+    static register(hookName: string, callback: HookCallback): void {
+        if (typeof callback !== 'function') {
+            throw new Error('Hook callback must be a function');
+        }
+
+        if (!hooks[hookName]) {
+            hooks[hookName] = [];
+        }
+
+        hooks[hookName].push(callback);
+    }
+
+    static trigger(hookName: string, ...args: any[]) {
+        if (hooks[hookName]) {
+            hooks[hookName].forEach((callback) => callback(...args));
+        }
+    }
+}
+
+/**
+ * Decorator function that executes registered hooks before the decorated method
+ * @param hookName The name of the hook to trigger
+ */
+export function hook(hookName: string) {
+    return function (target: any, propertyKey: string, descriptor: PropertyDescriptor) {
+        const originalMethod = descriptor.value;
+
+        descriptor.value = function (...args: any[]) {
+            // Execute all registered hooks for this hook name
+            if (hooks[hookName]) {
+                hooks[hookName].forEach((callback) => {
+                    callback.apply(this, args);
+                });
+            }
+
+            // Call the original method
+            return originalMethod.apply(this, args);
+        };
+
+        return descriptor;
+    };
+}
+
+/**
+ * Decorator function that executes registered hooks asynchronously before the decorated method
+ * @param hookName The name of the hook to trigger
+ */
+export function hookAsync(hookName: string) {
+    return function (target: any, propertyKey: string, descriptor: PropertyDescriptor) {
+        const originalMethod = descriptor.value;
+
+        descriptor.value = async function (...args: any[]) {
+            // Execute all registered hooks for this hook name
+            if (hooks[hookName]) {
+                // Wait for all hooks to complete before proceeding
+                await Promise.all(hooks[hookName].map((callback) => callback.apply(this, args)));
+            }
+
+            // Call the original method
+            return originalMethod.apply(this, args);
+        };
+
+        return descriptor;
+    };
+}
+
+/**
+ * Decorator function that executes registered hooks asynchronously before the decorated method
+ * @param hookName The name of the hook to trigger
+ * @param contextFn Optional function to extract additional context from the class instance
+ */
+export function hookAsyncWithContext(hookName: string, contextFn?: (instance: any) => Record<string, any>) {
+    return function (target: any, propertyKey: string, descriptor: PropertyDescriptor) {
+        const originalMethod = descriptor.value;
+
+        descriptor.value = async function (...args: any[]) {
+            // Execute all registered hooks for this hook name
+            if (hooks[hookName]) {
+                // Get additional context if contextFn is provided
+                const additionalContext = typeof contextFn === 'function' ? await contextFn(this) : {};
+
+                // Wait for all hooks to complete before proceeding
+                // We need to prepend the additional context before the args; otherwise, optional parameters may cause incorrect argument mapping.
+                await Promise.all(hooks[hookName].map((callback) => callback.apply(this, [additionalContext, ...args])));
+            }
+
+            // Call the original method
+            return originalMethod.apply(this, args);
+        };
+
+        return descriptor;
+    };
+}

package/src/Core/SmythRuntime.class.ts

@@ -0,0 +1,292 @@
+import { boot } from './boot';
+
+import { SREConfig, TConnectorService } from '@sre/types/SRE.types';
+import { ConnectorService } from './ConnectorsService';
+import { SystemEvents } from './SystemEvents';
+import { Logger } from '../helpers/Log.helper';
+import path from 'path';
+import fs from 'fs';
+import os from 'os';
+import { fileURLToPath } from 'url';
+import { dirname } from 'path';
+
+const logger = Logger('SRE');
+
+export class SmythRuntime {
+    public started = false;
+
+    private _smythDir: string;
+    public get smythDir() {
+        return this._smythDir;
+    }
+    private _readyPromise: Promise<boolean>;
+    private _readyResolve: (value: boolean) => void;
+
+    private defaultConfig: SREConfig = {
+        Vault: {
+            Connector: 'JSONFileVault',
+        },
+        Account: {
+            Connector: 'DummyAccount',
+        },
+        Cache: {
+            Connector: 'RAM',
+        },
+        Storage: {
+            Connector: 'LocalStorage',
+        },
+        Code: {
+            Connector: 'DummyConnector',
+        },
+        //NKV should be loaded before VectorDB
+        NKV: {
+            Connector: 'RAM',
+        },
+        VectorDB: {
+            Connector: 'RAMVec',
+        },
+        ModelsProvider: {
+            Connector: 'SmythModelsProvider',
+        },
+        AgentData: {
+            Connector: 'NullAgentData',
+        },
+        Component: {
+            Connector: 'LocalComponent',
+        },
+        ManagedVault: {
+            Connector: 'NullManagedVault',
+        },
+        Log: {
+            Connector: 'ConsoleLog',
+        },
+        Router: {
+            Connector: 'NullRouter',
+        },
+    };
+
+    protected constructor() {
+        this.started = true;
+        this._readyPromise = new Promise((resolve) => {
+            this._readyResolve = resolve;
+        });
+    }
+
+    protected static instance?: SmythRuntime;
+    public static get Instance(): SmythRuntime {
+        if (!SmythRuntime.instance) {
+            SmythRuntime.instance = new SmythRuntime();
+        }
+        return SmythRuntime.instance;
+    }
+
+    private _initializing = false;
+
+    public get initializing() {
+        return this._initializing;
+    }
+
+    private _initialized = false;
+
+    public init(_config?: SREConfig): SmythRuntime {
+        if (!_config || JSON.stringify(_config) === '{}') {
+            this._smythDir = findSmythDir();
+            logger.info('.smyth directory found in:', this._smythDir);
+        }
+
+        if (this._initializing) {
+            console.warn('You tried to initialize SRE while it is already initializing ... skipping');
+            return;
+        }
+        if (this._initialized) {
+            throw new Error('SRE already initialized');
+        }
+        this._initializing = true;
+        SystemEvents.on('SRE:Booted', () => {
+            this._readyResolve(true);
+        });
+        boot();
+
+        const config = this.autoConf(_config);
+
+        for (let connectorType in config) {
+            for (let configEntry of config[connectorType]) {
+                ConnectorService.init(
+                    connectorType as TConnectorService,
+                    configEntry.Connector,
+                    configEntry.Id,
+                    configEntry.Settings,
+                    configEntry.Default
+                );
+            }
+        }
+
+        this._initialized = true;
+        SystemEvents.emit('SRE:Initialized');
+
+        return SmythRuntime.Instance as SmythRuntime;
+    }
+
+    /**
+     * This function tries to auto configure, or fixes the provided configuration
+     *
+     * FIXME: The current version does not actually auto configure SRE, it just fixes the provided configuration for now
+     * TODO: Implement auto configuration based on present environment variables and auto-detected configs
+     * @param config
+     */
+    private autoConf(config: SREConfig = {}) {
+        // default config for missing connectors
+        const defaultConfig = JSON.parse(JSON.stringify(this.defaultConfig));
+
+        // for (let connectorType in defaultConfig) {
+        //     if (!config[connectorType]) {
+        //         config[connectorType] = defaultConfig[connectorType];
+        //     }
+        // }
+
+        const keys = Object.keys({ ...defaultConfig, ...config });
+
+        const newConfig: SREConfig = {};
+        for (let connectorType of keys) {
+            newConfig[connectorType] = [];
+
+            let entry = config[connectorType] || defaultConfig[connectorType];
+            if (!Array.isArray(entry)) {
+                entry = [entry];
+            }
+
+            let hasDefault = false;
+            for (let connector of entry) {
+                if (!connector.Connector) {
+                    logger.warn(`Missing Connector Name in ${connectorType} entry ... it will be ignored`);
+                    continue;
+                }
+                if (connector.Default) {
+                    if (hasDefault) {
+                        logger.warn(`Entry ${connectorType} has more than one default Connector ... only the first one will be used`);
+                    }
+                    hasDefault = true;
+                }
+                newConfig[connectorType].push(connector);
+            }
+
+            if (!hasDefault && newConfig[connectorType].length > 0) {
+                newConfig[connectorType][0].Default = true;
+            }
+        }
+
+        return newConfig;
+    }
+
+    public ready(): Promise<boolean> {
+        return this._readyPromise;
+    }
+
+    private _stopping = false;
+    public async _stop() {
+        if (this._stopping) {
+            return;
+        }
+        this._stopping = true;
+        logger.info('Sending Shutdown Signals To All Subsystems...');
+        await ConnectorService._stop();
+        SmythRuntime.instance = undefined;
+        this.started = false;
+    }
+}
+
+export const SRE = SmythRuntime.Instance;
+let shuttingDown = false;
+
+function findSmythDir() {
+    // 1. Try to find in local directory (the directory from which the program was run)
+    const localDir = path.resolve(process.cwd(), '.smyth');
+    if (fs.existsSync(localDir)) {
+        return localDir;
+    }
+
+    // 2. Try to find from main script's directory (entry point)
+    const mainScriptPath = process.argv[1];
+    const mainScriptDir = path.dirname(path.resolve(mainScriptPath));
+    if (mainScriptPath) {
+        const mainScriptSmythDir = path.resolve(mainScriptDir, '.smyth');
+        if (fs.existsSync(mainScriptSmythDir)) {
+            return mainScriptSmythDir;
+        }
+    }
+
+    // 3. Try to find it in current package root directory (where package.json is)
+    let packageRootDir = findPackageRoot();
+    if (packageRootDir) {
+        const packageSmythDir = path.resolve(packageRootDir, '.smyth');
+        if (fs.existsSync(packageSmythDir)) {
+            return packageSmythDir;
+        }
+    }
+
+    // 4. Try to find it in current package root directory from main script's directory
+    packageRootDir = findPackageRoot(mainScriptDir);
+    if (packageRootDir) {
+        const packageSmythDir = path.resolve(packageRootDir, '.smyth');
+        if (fs.existsSync(packageSmythDir)) {
+            return packageSmythDir;
+        }
+    }
+
+    // 5. Try to find it in user home directory
+    const homeDir = path.resolve(os.homedir(), '.smyth');
+    if (fs.existsSync(homeDir)) {
+        return homeDir;
+    }
+
+    // Default: return path in current working directory (will be created if needed)
+    return localDir;
+}
+
+function findPackageRoot(startDir = process.cwd()) {
+    let currentDir = startDir;
+
+    while (currentDir !== path.dirname(currentDir)) {
+        const packageJsonPath = path.resolve(currentDir, 'package.json');
+        if (fs.existsSync(packageJsonPath)) {
+            return currentDir;
+        }
+        currentDir = path.dirname(currentDir);
+    }
+
+    return null;
+}
+
+async function shutdown(reason) {
+    if (!SmythRuntime.Instance.started) return;
+    if (shuttingDown) return;
+    shuttingDown = true;
+
+    logger.info(`Caught ${reason} ... Attempting graceful shutdown`);
+    if (SmythRuntime.Instance) {
+        try {
+            await SmythRuntime.Instance._stop();
+        } catch (err) {
+            logger.error('Shutdown error:', err);
+        }
+    }
+}
+
+['SIGINT', 'SIGTERM'].forEach((signal) => {
+    process.on(signal, async () => {
+        await shutdown(signal);
+        process.exit(0); // Required after async
+    });
+});
+
+process.on('beforeExit', (code) => {
+    shutdown('beforeExit');
+});
+
+process.on('exit', (code) => {
+    logger.info(`Goodbye!`);
+});
+
+// process.on('uncaughtException', (err) => {
+
+// });

package/src/Core/SystemEvents.ts

@@ -0,0 +1,15 @@
+import { SmythLLMUsage, SmythTaskUsage } from '@sre/types/LLM.types';
+import { TServiceRegistry } from '@sre/types/SRE.types';
+import { EventEmitter } from 'events';
+
+export type SystemEventMap = {
+    'SRE:Booted': [TServiceRegistry];
+    'SRE:Initialized': [];
+    'USAGE:LLM': [SmythLLMUsage];
+    'USAGE:API': any;
+    'USAGE:TASK': [SmythTaskUsage];
+};
+
+const SystemEvents = new EventEmitter<SystemEventMap>();
+
+export { SystemEvents };