@smythos/sre 1.7.18 → 1.7.40

package/src/subsystems/LLMManager/ModelsProvider.service/connectors/JSONModelsProvider.class.ts

@@ -16,7 +16,7 @@ import fsSync from 'fs';
 import path from 'path';
 import { findSmythPath } from '@sre/helpers/Sysconfig.helper';
 
-const console = Logger('SmythModelsProvider');
+const logger = Logger('SmythModelsProvider');
 
 type SmythModelsProviderConfig = {
     /**
@@ -58,7 +58,7 @@ export class JSONModelsProvider extends ModelsProviderConnector {
                 this._settings.mode = 'merge'; //Force merge mode if using models from .smyth folder
                 this.initDirWatcher(modelsFolder); //this.started will be set to true when the watcher is ready
             } else {
-                console.warn('No models folder found ... falling back to built-in models only');
+                logger.warn('No models folder found ... falling back to built-in models only');
                 this.started = true;
             }
         }
@@ -71,7 +71,7 @@ export class JSONModelsProvider extends ModelsProviderConnector {
         const _modelsFolder = findSmythPath('models');
 
         if (fsSync.existsSync(_modelsFolder)) {
-            console.warn('Using default models folder  : ', _modelsFolder);
+            logger.warn('Using default models folder  : ', _modelsFolder);
             return _modelsFolder;
         }
 
@@ -106,7 +106,7 @@ export class JSONModelsProvider extends ModelsProviderConnector {
 
     private async reindexModels(dir: string) {
         try {
-            console.debug(`Reindexing models from directory: ${dir}`);
+            logger.debug(`Reindexing models from directory: ${dir}`);
 
             // Scan directory for models and get them as an object
             const scannedModels = await this.scanDirectoryForModels(dir);
@@ -121,9 +121,9 @@ export class JSONModelsProvider extends ModelsProviderConnector {
 
             JSONModelsProvider.localCache.clear();
 
-            console.debug(`Successfully reindexed models. Total models: ${Object.keys(this.models).length}`);
+            logger.debug(`Successfully reindexed models. Total models: ${Object.keys(this.models).length}`);
         } catch (error) {
-            console.error(`Error reindexing models from directory "${dir}":`, error);
+            logger.error(`Error reindexing models from directory "${dir}":`, error);
         }
     }
 
@@ -141,15 +141,21 @@ export class JSONModelsProvider extends ModelsProviderConnector {
                     const subDirModels = await this.scanDirectoryForModels(fullPath);
                     Object.assign(scannedModels, subDirModels);
                 } else if (entry.isFile() && entry.name.endsWith('.json')) {
-                    // Process JSON files and merge results
-                    const fileContent = await fs.readFile(fullPath, 'utf-8');
-                    const modelData = JSON.parse(fileContent);
-                    const validModels = await this.getValidModels(modelData);
-                    Object.assign(scannedModels, validModels);
+                    try {
+                        // Process JSON files and merge results
+
+                        const fileContent = await fs.readFile(fullPath, 'utf-8');
+                        const modelData = JSON.parse(fileContent);
+                        const validModels = await this.getValidModels(modelData);
+                        Object.assign(scannedModels, validModels);
+                    } catch (error) {
+                        console.error(`Error parsing model data from file "${fullPath}"`);
+                        logger.warn(`Error parsing model data from file "${fullPath}":`, error.message);
+                    }
                 }
             }
         } catch (error) {
-            console.warn(`Error scanning directory "${dir}":`, error);
+            logger.warn(`Error scanning directory "${dir}":`, error);
         }
 
         return scannedModels;
@@ -164,9 +170,9 @@ export class JSONModelsProvider extends ModelsProviderConnector {
                 // Single model case
                 if (this.isValidSingleModel(modelData)) {
                     validModels[modelData.modelId] = modelData as TLLMModel;
-                    console.debug(`Loaded model: ${modelData.modelId}`);
+                    logger.debug(`Loaded model: ${modelData.modelId}`);
                 } else {
-                    console.warn(`Invalid model format`, modelData);
+                    logger.warn(`Invalid model format`, modelData);
                 }
             } else if (typeof modelData === 'object' && !Array.isArray(modelData)) {
                 // Object of models case
@@ -178,19 +184,19 @@ export class JSONModelsProvider extends ModelsProviderConnector {
                             //console.debug(`Loaded model: ${modelId}`);
                             models += `${modelId} `;
                         } else {
-                            console.warn(`Invalid model format for model "${modelId}"`);
+                            logger.warn(`Invalid model format for model "${modelId}"`);
                         }
                     } catch (error) {
-                        console.warn(`Error processing model "${modelId}":`, error);
+                        logger.warn(`Error processing model "${modelId}":`, error);
                         // Continue processing other models instead of failing the whole file
                     }
                 }
-                console.debug(`Loaded models: ${models}`);
+                logger.debug(`Loaded models: ${models}`);
             } else {
-                console.warn(`Invalid format (not a model or object of models)`);
+                logger.warn(`Invalid format (not a model or object of models)`);
             }
         } catch (error) {
-            console.warn(`Error loading model:`, error);
+            logger.warn(`Error loading model:`, error);
         }
 
         return validModels;
@@ -208,11 +214,98 @@ export class JSONModelsProvider extends ModelsProviderConnector {
         return this.isValidSingleModel(data);
     }
 
+    /**
+     * Determines whether a file path should be ignored by the directory watcher.
+     *
+     * This method implements a sophisticated filtering strategy for dot-segment paths
+     * (paths containing directories that start with a dot, like .git, .env, .cache).
+     *
+     * **Filtering Strategy:**
+     * 1. Paths WITHOUT dot segments: Never ignored
+     * 2. Paths WITH dot segments:
+     *    - If SMYTH_PATH is not configured: All ignored
+     *    - If SMYTH_PATH is configured:
+     *      - Allow the watched directory even if SMYTH_PATH contains dot-segments
+     *        (e.g., /home/user/.smyth/models/OpenAI/default.json is allowed)
+     *      - Ignore dot-segments INSIDE the models directory
+     *        (e.g., /home/user/.smyth/models/.hidden/model.json is ignored)
+     *      - Paths outside watched directory: Ignored
+     *
+     * @param filePath - The file path to check
+     * @param watchedDir - The absolute path of the directory being watched (models folder)
+     * @param smythPath - The resolved SMYTH_PATH, or null if not configured
+     * @returns true if the path should be ignored, false if it should be watched
+     *
+     * @example
+     * ```typescript
+     * // Path without dot-segment (allowed)
+     * shouldIgnorePath('/models/OpenAI/default.json', '/models', '/home/.smyth') // => false
+     *
+     * // Dot-segment inside models directory (ignored)
+     * shouldIgnorePath('/models/.git/config', '/models', '/home/.smyth') // => true
+     *
+     * // Dot-segment in parent path only (allowed)
+     * shouldIgnorePath('/home/.smyth/models/OpenAI/default.json', '/home/.smyth/models', '/home/.smyth') // => false
+     * ```
+     */
+    private shouldIgnorePath(filePath: string, watchedDir: string, smythPath: string | null): boolean {
+        // Check if the file path contains a dot-segment (e.g., /.git/, /.env/, /.cache/)
+        // Regex explanation: [\\/]\. matches a path separator (/ or \) followed by a dot
+        const hasDotSegment = /[\\/]\./.test(filePath);
+
+        // CASE 1: If there is NO dot-segment at all, we never ignore this path
+        // Examples: /models/OpenAI/default.json, /models/Anthropic/claude.json
+        if (!hasDotSegment) {
+            return false;
+        }
+
+        // CASE 2: If there IS a dot-segment and SMYTH_PATH is not configured,
+        // we ignore all such paths to prevent watching system/hidden files
+        // Examples: /.git/config, /node_modules/.cache/file.json
+        if (hasDotSegment && !smythPath) {
+            return true;
+        }
+
+        // Resolve the file path to an absolute path for accurate comparison
+        // This ensures we can reliably compare against the watched directory path
+        const resolvedPath = path.resolve(filePath);
+
+        // Check if the resolved path is inside the watched directory (models folder)
+        // This handles two cases:
+        // 1. The path exactly matches the watched directory
+        // 2. The path is a child of the watched directory (starts with watchedDir + separator)
+        const isInsideWatchedDir = resolvedPath === watchedDir || resolvedPath.startsWith(watchedDir + path.sep);
+
+        // CASE 3: If the path is outside the watched directory, ignore it
+        // This prevents watching unrelated files that happen to have dot-segments
+        if (!isInsideWatchedDir) {
+            return true;
+        }
+
+        // CASE 4: Path is inside the watched directory
+        // Now we need to determine if the dot-segment is in the models directory itself
+        // or if it is part of the parent path (e.g., SMYTH_PATH containing .smyth)
+
+        // Get the relative path from the watched directory to determine where the dot-segment is
+        const relativePath = path.relative(watchedDir, resolvedPath);
+
+        // Check if the dot-segment appears in the relative portion (inside models directory)
+        // Regex explanation: (^|[\\/])\. matches a dot at the start OR after a path separator
+        // Examples that match: '.git/config', 'subdir/.hidden/file.json'
+        const hasDotSegmentInsideWatchedDir = /(^|[\\/])\./.test(relativePath);
+
+        // FINAL DECISION:
+        // - If dot-segment is INSIDE the models directory (e.g., models/.git/config), IGNORE it (return true)
+        // - If dot-segment is OUTSIDE the models directory (e.g., /home/user/.smyth/models/OpenAI/default.json), ALLOW it (return false)
+        // This allows SMYTH_PATH to contain dot-segments while preventing dot-segments within the models folder
+        return hasDotSegmentInsideWatchedDir;
+    }
+
     private initDirWatcher(dir) {
         const stats = fsSync.statSync(dir);
 
         if (!stats.isDirectory() && !stats.isFile()) {
-            console.warn(`Path "${dir}" is neither a file nor a directory ... skipping models watcher and falling back to built-in models only`);
+            logger.warn(`Path "${dir}" is neither a file nor a directory ... skipping models watcher and falling back to built-in models only`);
             this.started = true;
             return;
         }
@@ -230,17 +323,18 @@ export class JSONModelsProvider extends ModelsProviderConnector {
                         if (this._settings?.mode === 'merge') this.models = { ...this.models, ...modelData };
                         else this.models = modelData;
                     } catch (error) {
-                        console.error(`Error parsing model data from file "${dir}":`, error);
+                        console.error(`Error parsing model data from file "${dir}":`);
+                        logger.warn(`Error parsing model data from file "${dir}":`, error.message);
                     }
                     this.started = true;
                     return;
                 }
 
-                console.warn(`Path "${dir}" is neither a file nor a directory`);
+                logger.warn(`Path "${dir}" is neither a file nor a directory`);
                 return;
             }
         } catch (error) {
-            console.warn(`Path "${dir}" does not exist or cannot be accessed:`, error.message);
+            logger.warn(`Path "${dir}" does not exist or cannot be accessed:`, error.message);
             return;
         }
 
@@ -250,8 +344,12 @@ export class JSONModelsProvider extends ModelsProviderConnector {
             maxWait: 5000,
         });
 
+        const smythPath = process.env.SMYTH_PATH ? path.resolve(process.env.SMYTH_PATH) : null;
+        const watchedDir = path.resolve(dir);
+
         const watcher = chokidar.watch(dir, {
-            ignored: /(^|[\/\\])\../, // ignore dotfiles
+            // Use the extracted method for path filtering
+            ignored: (filePath: string) => this.shouldIgnorePath(filePath, watchedDir, smythPath),
             persistent: true,
             ignoreInitial: true, // Don't fire events for files that already exist
             awaitWriteFinish: {
@@ -262,19 +360,19 @@ export class JSONModelsProvider extends ModelsProviderConnector {
 
         watcher
             .on('add', (path) => {
-                console.debug(`File ${path} has been added`);
+                logger.debug(`File ${path} has been added`);
                 debouncedReindex();
             })
             .on('change', (path) => {
-                console.debug(`File ${path} has been changed`);
+                logger.debug(`File ${path} has been changed`);
                 debouncedReindex();
             })
             .on('unlink', (path) => {
-                console.debug(`File ${path} has been removed`);
+                logger.debug(`File ${path} has been removed`);
                 debouncedReindex();
             })
             .on('ready', async () => {
-                console.debug(`Watcher ready. Performing initial scan of ${dir}`);
+                logger.debug(`Watcher ready. Performing initial scan of ${dir}`);
                 // Do initial scan once when watcher is ready
                 await this.reindexModels(dir);
                 this.started = true;

package/src/subsystems/ObservabilityManager/Telemetry.service/connectors/OTel/OTel.class.ts

@@ -20,8 +20,9 @@ import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http';
 import { IAgent } from '@sre/types/Agent.types';
 import { Conversation } from '@sre/helpers/Conversation.helper';
 import { TLLMEvent } from '@sre/types/LLM.types';
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
 
-const outputLogger = Logger('OTelLog');
+const outputLogger = Logger('OTel');
 
 export type OTelLogConfig = {
     endpoint: string;
@@ -50,7 +51,7 @@ export type OTelLogConfig = {
      */
     redactFields?: string[];
 };
-
+const OTEL_DEBUG_LOGS = true;
 export class OTel extends TelemetryConnector {
     public name: string = 'OTel';
     public id: string;
@@ -208,6 +209,8 @@ export class OTel extends TelemetryConnector {
 
         const createToolInfoHandler = function (hookContext) {
             return function (toolInfo: any) {
+                const accessCandidate = AccessCandidate.agent(hookContext?.agentId);
+                if (OTEL_DEBUG_LOGS) outputLogger.debug('createToolInfoHandler started', accessCandidate);
                 if (!hookContext.curLLMGenSpan || !hookContext.convSpan) return;
 
                 const modelId = toolInfo.model;
@@ -238,6 +241,7 @@ export class OTel extends TelemetryConnector {
 
                 hookContext.curLLMGenSpan.end();
                 delete hookContext.curLLMGenSpan;
+                if (OTEL_DEBUG_LOGS) outputLogger.debug('createToolInfoHandler completed', accessCandidate);
             };
         };
 
@@ -245,6 +249,8 @@ export class OTel extends TelemetryConnector {
             return function (data: any, reqInfo: any) {
                 if (!hookContext.convSpan) return;
                 if (hookContext.curLLMGenSpan) return;
+                const accessCandidate = AccessCandidate.agent(hookContext?.agentId);
+                if (OTEL_DEBUG_LOGS) outputLogger.debug('createDataHandler started', reqInfo?.requestId, accessCandidate);
 
                 const modelId = reqInfo.model;
                 const contextWindow = reqInfo.contextWindow;
@@ -285,13 +291,15 @@ export class OTel extends TelemetryConnector {
                     'context.preview': JSON.stringify(lastContext).substring(0, 200),
                 });
                 hookContext.curLLMGenSpan = llmGenSpan;
+                if (OTEL_DEBUG_LOGS) outputLogger.debug('createDataHandler completed', reqInfo?.requestId, accessCandidate);
             };
         };
 
         const createRequestedHandler = function (hookContext) {
             return function (reqInfo: any) {
                 if (!hookContext.convSpan) return;
-
+                const accessCandidate = AccessCandidate.agent(hookContext?.agentId);
+                if (OTEL_DEBUG_LOGS) outputLogger.debug('createRequestedHandler started', reqInfo?.requestId, accessCandidate);
                 if (!hookContext.latencySpans) hookContext.latencySpans = {};
                 const contextWindow = reqInfo.contextWindow;
 
@@ -317,6 +325,7 @@ export class OTel extends TelemetryConnector {
                     'context.preview': JSON.stringify(lastContext).substring(0, 200),
                 });
                 hookContext.latencySpans[reqInfo.requestId] = llmGenLatencySpan;
+                if (OTEL_DEBUG_LOGS) outputLogger.debug('createRequestedHandler completed', reqInfo?.requestId, accessCandidate);
             };
         };
         HookService.register(
@@ -332,26 +341,35 @@ export class OTel extends TelemetryConnector {
 
                     return;
                 }
+                const accessCandidate = AccessCandidate.agent(agentId);
+                if (OTEL_DEBUG_LOGS) outputLogger.debug('Conversation.streamPrompt started', { processId, message }, accessCandidate);
 
                 const modelId = typeof conversation?.model === 'string' ? conversation?.model : conversation?.model?.modelId;
 
                 const convSpan = tracer.startSpan('Agent.Conv', {
                     attributes: {
+                        // OTel standard attributes
+                        'gen_ai.operation.name': 'chat',
+                        'gen_ai.provider.name': conversation?.llmInference?.llmProviderName || 'unknown',
+                        'gen_ai.conversation.id': processId,
+                        'gen_ai.request.model': modelId || 'unknown',
+                        ////////////////////////////////
                         'agent.id': agentId,
                         'conv.id': processId,
                         'llm.model': modelId || 'unknown',
                     },
                 });
                 hookContext.convSpan = convSpan;
+                hookContext.agentId = agentId;
+                hookContext.processId = processId;
 
                 hookContext.dataHandler = createDataHandler(hookContext);
                 conversation.on(TLLMEvent.Data, hookContext.dataHandler);
+
                 hookContext.requestedHandler = createRequestedHandler(hookContext);
                 conversation.on(TLLMEvent.Requested, hookContext.requestedHandler);
-                hookContext.agentId = agentId;
-                hookContext.processId = processId;
-                hookContext.toolInfoHandler = createToolInfoHandler(hookContext);
 
+                hookContext.toolInfoHandler = createToolInfoHandler(hookContext);
                 conversation.on(TLLMEvent.ToolInfo, hookContext.toolInfoHandler);
 
                 // Add start event
@@ -403,6 +421,9 @@ export class OTel extends TelemetryConnector {
                 const ctx = OTelContextRegistry.get(agentId, processId);
                 if (!ctx) return;
 
+                const accessCandidate = AccessCandidate.agent(agentId);
+                if (OTEL_DEBUG_LOGS) outputLogger.debug('Conversation.streamPrompt completed', { processId }, accessCandidate);
+
                 if (hookContext.curLLMGenSpan) {
                     hookContext.curLLMGenSpan.addEvent('llm.gen.content', {
                         'content.size': JSON.stringify(result || {}).length,
@@ -455,6 +476,9 @@ export class OTel extends TelemetryConnector {
                 const teamId = agent.teamId;
                 const _hookContext: any = this.context;
 
+                const accessCandidate = AccessCandidate.agent(agentId);
+                if (OTEL_DEBUG_LOGS) outputLogger.debug('SREAgent.process started', { processId, agentProcessId, endpointPath }, accessCandidate);
+
                 const body = oTelInstance.prepareComponentData(agentRequest.body || {});
                 const query = oTelInstance.prepareComponentData(agentRequest.query || {});
                 const headers = oTelInstance.prepareComponentData(agentRequest.headers || {});
@@ -531,6 +555,9 @@ export class OTel extends TelemetryConnector {
 
                 if (!agentSpan) return;
 
+                const accessCandidate = AccessCandidate.agent(agentId);
+                if (OTEL_DEBUG_LOGS) outputLogger.debug('SREAgent.process completed', { agentProcessId }, accessCandidate);
+
                 if (error) {
                     agentSpan.recordException(error);
                     agentSpan.setStatus({ code: SpanStatusCode.ERROR, message: error.message });
@@ -597,6 +624,8 @@ export class OTel extends TelemetryConnector {
                 const componentType = settings.name;
                 const componentName = settings.displayName || settings.name;
                 const eventId = settings.eventId; // specific event id attached to this component execution
+                const accessCandidate = AccessCandidate.agent(agentId);
+                if (OTEL_DEBUG_LOGS) outputLogger.debug('Component.process started', { componentId }, accessCandidate);
 
                 const ctx = OTelContextRegistry.get(agentId, processId);
                 const parentSpan = ctx?.rootSpan;
@@ -671,6 +700,9 @@ export class OTel extends TelemetryConnector {
                 const componentType = settings.name;
                 const componentName = settings.displayName || settings.name;
 
+                const accessCandidate = AccessCandidate.agent(agentId);
+                if (OTEL_DEBUG_LOGS) outputLogger.debug('Component.process completed', { componentId }, accessCandidate);
+
                 if (error) {
                     // Capture error details
                     span.recordException(error);

package/src/subsystems/Security/Account.service/AccountConnector.ts

@@ -10,7 +10,7 @@ export interface ISmythAccountRequest {
     getCandidateTeam(): Promise<string | undefined>;
     getAllTeamSettings(): Promise<KeyValueObject>;
     getAllUserSettings(): Promise<KeyValueObject>;
-    getTeamSetting(settingKey: string): Promise<string>;
+    getTeamSetting(settingKey: string, group?: string): Promise<string>;
     getUserSetting(settingKey: string): Promise<string>;
     getAgentSetting(settingKey: string): Promise<string>;
     getTeam(): Promise<string>;
@@ -25,7 +25,7 @@ export abstract class AccountConnector extends Connector {
             getAllUserSettings: async () => this.getAllUserSettings(candidate.readRequest, candidate.id),
             getUserSetting: async (settingKey: string) => this.getUserSetting(candidate.readRequest, candidate.id, settingKey),
             getAllTeamSettings: async () => this.getAllTeamSettings(candidate.readRequest, candidate.id),
-            getTeamSetting: async (settingKey: string) => this.getTeamSetting(candidate.readRequest, candidate.id, settingKey),
+            getTeamSetting: async (settingKey: string, group?: string) => this.getTeamSetting(candidate.readRequest, candidate.id, settingKey, group),
             isTeamMember: async (teamId: string) => this.isTeamMember(teamId, candidate),
             getCandidateTeam: async () => this.getCandidateTeam(candidate),
             getTeam: async () => this.getCandidateTeam(candidate),
@@ -38,7 +38,7 @@ export abstract class AccountConnector extends Connector {
     public abstract getCandidateTeam(candidate: IAccessCandidate): Promise<string | undefined>;
     public abstract getAllTeamSettings(acRequest: AccessRequest, teamId: string): Promise<KeyValueObject>;
     public abstract getAllUserSettings(acRequest: AccessRequest, accountId: string): Promise<KeyValueObject>;
-    public abstract getTeamSetting(acRequest: AccessRequest, teamId: string, settingKey: string): Promise<string>;
+    public abstract getTeamSetting(acRequest: AccessRequest, teamId: string, settingKey: string, group?: string): Promise<string>;
     public abstract getUserSetting(acRequest: AccessRequest, accountId: string, settingKey: string): Promise<string>;
     public abstract getAgentSetting(acRequest: AccessRequest, agentId: string, settingKey: string): Promise<string>;
 }