@smythos/sre 1.7.40 → 1.7.42

package/src/Components/LLMAssistant.class.ts

@@ -10,7 +10,8 @@ import { encode } from 'gpt-tokenizer';
 import { Component } from './Component.class';
 import { JSONContent } from '@sre/helpers/JsonContent.helper';
 import { LLMInference } from '@sre/LLMManager/LLM.inference';
-import { TLLMMessageRole } from '@sre/types/LLM.types';
+import { LLMHelper } from '@sre/LLMManager/LLM.helper';
+import { TLLMMessageRole, TLLMEvent } from '@sre/types/LLM.types';
 import { VaultHelper } from '@sre/Security/Vault.service/Vault.helper';
 import path from 'path';
 import config from '@sre/config';
@@ -113,7 +114,7 @@ export class LLMAssistant extends Component {
             const conversationId = input.ConversationId;
 
             let behavior = TemplateString(config.data.behavior).parse(input).result;
-            logger.debug(`[Parsed Behavior] \n${behavior}`);
+            logger.debug(` [Parsed Behavior] \n${behavior}`);
 
             //#region get max tokens
             let maxTokens = 2048;
@@ -147,7 +148,14 @@ export class LLMAssistant extends Component {
             };
 
             let response: any;
-            if (passThrough) {
+            const isClaude4 = LLMHelper.isClaude4Family(modelId || model);
+
+            // Use streaming for passthrough OR Claude 4 family models
+            if (passThrough || isClaude4) {
+                if (isClaude4) {
+                    logger.debug(`\n ⚡ Using streaming for Claude 4 family model`);
+                }
+
                 const contentPromise = new Promise(async (resolve, reject) => {
                     let _content = '';
                     const eventEmitter: any = await llmInference
@@ -162,25 +170,53 @@ export class LLMAssistant extends Component {
                             console.error('Error on promptStream: ', error);
                             reject(error);
                         });
-                    eventEmitter.on('content', (content) => {
-                        if (typeof agent.callback === 'function') {
-                            agent.callback({ content });
+
+                    eventEmitter.on(TLLMEvent.Content, (content) => {
+                        if (passThrough) {
+                            if (typeof agent.callback === 'function') {
+                                agent.callback({ content });
+                            }
+                            agent.sse.send('llm/passthrough/content', content);
                         }
-                        agent.sse.send('llm/passthrough/content', content);
                         _content += content;
                     });
-                    eventEmitter.on('thinking', (thinking) => {
-                        if (typeof agent.callback === 'function') {
-                            agent.callback({ thinking });
+
+                    eventEmitter.on(TLLMEvent.Thinking, (thinking) => {
+                        if (passThrough) {
+                            if (typeof agent.callback === 'function') {
+                                agent.callback({ thinking });
+                            }
+                            agent.sse.send('llm/passthrough/thinking', thinking);
                         }
-                        agent.sse.send('llm/passthrough/thinking', thinking);
                     });
-                    eventEmitter.on('end', () => {
-                        console.log('end');
+
+                    eventEmitter.on(TLLMEvent.End, () => {
                         resolve(_content);
                     });
+
+                    eventEmitter.on(TLLMEvent.Error, (error) => {
+                        reject(error);
+                    });
                 });
-                response = await contentPromise;
+
+                response = await contentPromise.catch((error) => {
+                    return { error: error.message || error };
+                });
+
+                // Handle streaming errors
+                if (response?.error) {
+                    const error = response?.error + ' ' + (response?.details || '');
+                    logger.error(` LLM Error=`, error);
+                    return { Response: response?.data, _error: error, _debug: logger.output };
+                }
+
+                // Post-process the streaming response
+                const postProcessed = llmInference.connector.postProcess(response);
+                if (postProcessed.error) {
+                    logger.error(` LLM Post-process Error=`, postProcessed.error);
+                    return { _error: postProcessed.error, _debug: logger.output };
+                }
+                response = postProcessed;
             } else {
                 response = await llmInference
                     .prompt({
@@ -191,6 +227,12 @@ export class LLMAssistant extends Component {
                         },
                     })
                     .catch((error) => ({ error: error }));
+
+                if (response?.error) {
+                    const error = response?.error + ' ' + (response?.details || '');
+                    logger.error(` LLM Error=`, error);
+                    return { Response: response?.data, _error: error, _debug: logger.output };
+                }
             }
 
             // in case we have the response but it's empty string, undefined or null
@@ -198,13 +240,6 @@ export class LLMAssistant extends Component {
                 return { _error: ' LLM Error = Empty Response!', _debug: logger.output };
             }
 
-            if (response?.error) {
-                const error = response?.error + ' ' + (response?.details || '');
-                logger.error(` LLM Error=`, error);
-
-                return { Response: response?.data, _error: error, _debug: logger.output };
-            }
-
             messages.push({ role: 'assistant', content: response });
             saveMessagesToSession(agent.id, userId, conversationId, messages, ttl);
 

package/src/Components/RAG/DataSourceCleaner.class.ts

@@ -7,8 +7,9 @@ import { ConnectorService } from '@sre/Core/ConnectorsService';
 import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
 
 import { DataSourceIndexer } from './DataSourceIndexer.class';
-import { DataSourceComponent } from './DataSourceComponent.class';
+import { DataSourceComponent, NsRecord } from './DataSourceComponent.class';
 import { VectorDBConnector } from '@sre/IO/VectorDB.service/VectorDBConnector';
+import envConfig from '@sre/config';
 
 export class DataSourceCleaner extends DataSourceComponent {
     protected configSchema = Joi.object({
@@ -23,10 +24,10 @@ export class DataSourceCleaner extends DataSourceComponent {
 
     async process(input, config, agent: Agent) {
         await super.process(input, config, agent);
-        if (!config.data.version || config.data.version === 'v1') {
-            return await this.processV1(input, config, agent);
-        } else if (config.data.version === 'v2') {
+        if (envConfig.env.ROLLOUT_RAG_V2) {
             return await this.processV2(input, config, agent);
+        } else {
+            return await this.processV1(input, config, agent);
         }
     }
     async processV1(input, config, agent: Agent) {
@@ -114,14 +115,15 @@ export class DataSourceCleaner extends DataSourceComponent {
             }
 
             // const namespaceId = configSchema.value.namespaceId.split('_')?.slice(1).join('_') || configSchema.value.namespaceId;
-            const namespaceLabel = /^c[a-z0-9]{24}.+$/.test(configSchema.value.namespaceId)
-                ? configSchema.value.namespaceId.split('_').slice(1).join('_')
-                : configSchema.value.namespaceId;
-            const namespaceId = configSchema.value.namespaceId;
+            const namespaceLabelorId = configSchema.value.namespaceId;
+            // const namespaceId = configSchema.value.namespaceId;
 
             let vecDbConnector: VectorDBConnector = null;
+            let namespaceRecord: NsRecord = null;
             try {
-                vecDbConnector = await this.resolveVectorDbConnector(namespaceId, teamId);
+                const resolved = await this.resolveVectorDbConnector(namespaceLabelorId, teamId);
+                vecDbConnector = resolved.vecDbConnector;
+                namespaceRecord = resolved.namespaceRecord;
             } catch (err: any) {
                 debugOutput += `Error: ${err?.message || "Couldn't get vector database connector"}\n\n`;
                 return {
@@ -142,9 +144,9 @@ export class DataSourceCleaner extends DataSourceComponent {
             }
             debugOutput += `Searching for data source with id: ${providedId}\n`;
 
-            const dsId = DataSourceComponent.normalizeDsId(providedId, teamId, namespaceLabel);
+            const dsId = DataSourceComponent.normalizeDsId(providedId, teamId, namespaceRecord.label);
 
-            await vecDbConnector.requester(AccessCandidate.team(teamId)).deleteDatasource(namespaceLabel, dsId);
+            await vecDbConnector.requester(AccessCandidate.team(teamId)).deleteDatasource(namespaceRecord.label, dsId);
 
             debugOutput += `Deleted data source with id: ${providedId}\n`;
 

package/src/Components/RAG/DataSourceComponent.class.ts

@@ -34,23 +34,16 @@ export class DataSourceComponent extends Component {
         super();
     }
 
-    public async resolveVectorDbConnector(namespace: string | NsRecord, teamId: string): Promise<VectorDBConnector> {
+    public async resolveVectorDbConnector(
+        namespace: string | NsRecord,
+        teamId: string
+    ): Promise<{ vecDbConnector: VectorDBConnector; namespaceRecord: NsRecord }> {
         // resolve the ns record, if not exist, throw an error (new in v2)
         // then we also need to resolve the credentials
         let namespaceRecord = namespace as NsRecord;
 
         if (typeof namespace === 'string') {
-            // if it's a string, we need to get the namespace record from the NKV
-            const nkvConnector = ConnectorService.getNKVConnector();
-            const nkvClient = nkvConnector.requester(AccessCandidate.team(teamId));
-            const rawNsRecord = await nkvClient.get(`vectorDB:namespaces`, namespace);
-
-            if (!rawNsRecord) {
-                throw new DataSourceCompError(`Namespace ${namespace} does not exist`, TDataSourceCompErrorCodes.NAMESPACE_NOT_FOUND);
-            }
-
-            // const { credentialId, embeddings: embeddingsOptions } = JSON.parse(rawNsRecord.toString());
-            namespaceRecord = JSON.parse(rawNsRecord.toString()) as NsRecord;
+            namespaceRecord = await this.resolveNamespaceRecord(namespace, teamId);
         }
 
         const accountConnector = ConnectorService.getAccountConnector();
@@ -75,7 +68,40 @@ export class DataSourceComponent extends Component {
             embeddings: await this.buildEmbeddingConfig(namespaceRecord.embeddings, teamId),
         });
 
-        return vecDbConnector;
+        return { vecDbConnector, namespaceRecord };
+    }
+
+    private async resolveNamespaceRecord(namespace: string, teamId: string): Promise<NsRecord> {
+        // if it's a string, we need to get the namespace record from the NKV
+        // TRY 1) try to get namespace using the label provided from teamSettings
+        const nkvConnector = ConnectorService.getNKVConnector();
+        const nkvClient = nkvConnector.requester(AccessCandidate.team(teamId));
+        const rawNsRecord = await nkvClient.get(`vectorDB:namespaces`, namespace);
+
+        if (rawNsRecord) {
+            return JSON.parse(rawNsRecord.toString()) as NsRecord;
+        } else {
+            // throw new DataSourceCompError(`Namespace ${namespace} does not exist`, TDataSourceCompErrorCodes.NAMESPACE_NOT_FOUND);
+            console.warn(`Namespace ${namespace} does not exist using the label ${namespace}. Trying to get namespace by mapping legacy id to V2 id`);
+        }
+
+        // TRY 2) try to get namespace using the legacy id. list all namespaces and find the one that matches the legacy id
+        const namespaces = (await nkvClient.list(`vectorDB:namespaces`))
+            .map((namespace) => {
+                try {
+                    return JSON.parse(namespace.data.toString());
+                } catch (error) {
+                    return null;
+                }
+            })
+            .filter((namespace) => namespace !== null);
+
+        const matchingNamespace = namespaces.find((ns) => ns.__legacy_id === namespace);
+        if (!matchingNamespace) {
+            console.warn(`Namespace ${namespace} does not exist using the legacy id ${namespace}.`);
+            throw new DataSourceCompError(`Namespace ${namespace} does not exist`, TDataSourceCompErrorCodes.NAMESPACE_NOT_FOUND);
+        }
+        return matchingNamespace;
     }
 
     public async buildEmbeddingConfig(embedding: { dimensions: string; modelId: string }, teamId: string): Promise<TEmbeddings> {

package/src/Components/RAG/DataSourceIndexer.class.ts

@@ -1,5 +1,5 @@
 import { IAgent as Agent } from '@sre/types/Agent.types';
-import { DataSourceComponent } from './DataSourceComponent.class';
+import { DataSourceComponent, NsRecord } from './DataSourceComponent.class';
 import Joi from 'joi';
 import { validateCharacterSet } from '@sre/utils/validation.utils';
 import { TemplateString } from '@sre/helpers/TemplateString.helper';
@@ -11,6 +11,7 @@ import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.cla
 import { TEmbeddings } from '@sre/IO/VectorDB.service/embed/BaseEmbedding';
 import { VectorDBConnector } from '@sre/IO/VectorDB.service/VectorDBConnector';
 import { JSONContentHelper } from '@sre/helpers/JsonContent.helper';
+import envConfig from '@sre/config';
 
 export class DataSourceIndexer extends DataSourceComponent {
     private MAX_ALLOWED_URLS_PER_INPUT = 20;
@@ -31,10 +32,10 @@ export class DataSourceIndexer extends DataSourceComponent {
         await super.process(input, config, agent);
 
         let response: any = null;
-        if (!config.data.version || config.data.version === 'v1') {
-            response = await this.processV1(input, config, agent);
-        } else if (config.data.version === 'v2') {
+        if (envConfig.env.ROLLOUT_RAG_V2) {
             response = await this.processV2(input, config, agent);
+        } else {
+            response = await this.processV1(input, config, agent);
         }
 
         return response;
@@ -60,7 +61,7 @@ export class DataSourceIndexer extends DataSourceComponent {
             }
 
             const namespaceId = _config.namespace.split('_').slice(1).join('_') || _config.namespace;
-            debugOutput += `[Selected namespace id] \n${namespaceId}\n\n`;
+            debugOutput += `[Selected data space id] \n${namespaceId}\n\n`;
 
             const vectorDbConnector =
                 // (await vectorDBHelper.getTeamConnector(teamId)) ||
@@ -149,13 +150,15 @@ export class DataSourceIndexer extends DataSourceComponent {
 
             // we try to get the namespace without the prefix teamId, if not exist, we use the full namespace id
             // const namespaceLabel = _config.namespace.split('_').slice(1).join('_') || _config.namespace;
-            const namespaceLabel = /^c[a-z0-9]{24}.+$/.test(_config.namespace) ? _config.namespace.split('_').slice(1).join('_') : _config.namespace;
-            const namespaceId = _config.namespace;
-            debugOutput += `[Selected namespace] \n${namespaceLabel}\n\n`;
+            const namespaceLabelorId = _config.namespace;
+            // const namespaceId = _config.namespace;
 
             let vecDbConnector: VectorDBConnector = null;
+            let namespaceRecord: NsRecord = null;
             try {
-                vecDbConnector = await this.resolveVectorDbConnector(namespaceId, teamId);
+                const resolved = await this.resolveVectorDbConnector(namespaceLabelorId, teamId);
+                vecDbConnector = resolved.vecDbConnector;
+                namespaceRecord = resolved.namespaceRecord;
             } catch (err: any) {
                 debugOutput += `Error: ${err?.message || "Couldn't get vector database connector"}\n\n`;
                 return {
@@ -163,6 +166,9 @@ export class DataSourceIndexer extends DataSourceComponent {
                     _error: err?.message || "Couldn't get vector database connector",
                 };
             }
+
+            debugOutput += `[Selected data space] \n${namespaceRecord.label}\n\n`;
+
             const vecDbClient = vecDbConnector.requester(AccessCandidate.team(teamId));
 
             const inputSchema = this.validateInput(input);
@@ -183,10 +189,10 @@ export class DataSourceIndexer extends DataSourceComponent {
                 throw new Error(`Invalid id. Accepted characters: 'a-z', 'A-Z', '0-9', '-', '_', '.'`);
             }
 
-            const dsId = DataSourceIndexer.normalizeDsId(providedId, teamId, namespaceLabel);
+            const dsId = DataSourceIndexer.normalizeDsId(providedId, teamId, namespaceRecord.label);
 
             // check if the datasource already exists
-            const dsExists = await vecDbClient.getDatasource(namespaceLabel, dsId);
+            const dsExists = await vecDbClient.getDatasource(namespaceRecord.label, dsId);
             if (dsExists) {
                 debugOutput += `Datasource already exists\n\n`;
                 return {
@@ -197,7 +203,7 @@ export class DataSourceIndexer extends DataSourceComponent {
 
             debugOutput += `STEP: Parsing input as text\n\n`;
 
-            const response = await vecDbClient.createDatasource(namespaceLabel, {
+            const response = await vecDbClient.createDatasource(namespaceRecord.label, {
                 text: inputSchema.value.Source,
                 metadata: JSONContentHelper.create(_config.metadata).tryParse() || null,
                 id: dsId,

package/src/Components/RAG/DataSourceLookup.class.ts

@@ -7,6 +7,7 @@ import { ConnectorService } from '@sre/Core/ConnectorsService';
 import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
 import { IAgent as Agent } from '@sre/types/Agent.types';
 import { DataSourceComponent } from './DataSourceComponent.class';
+import envConfig from '@sre/config';
 
 // Note: LLMHelper renamed to LLMInference
 class LLMInference {
@@ -40,10 +41,15 @@ export class DataSourceLookup extends DataSourceComponent {
 
     async process(input, config, agent: Agent) {
         await super.process(input, config, agent);
-        if (!config.data.version || config.data.version === 'v1') {
-            return await this.processV1(input, config, agent);
-        } else if (config.data.version === 'v2') {
+        // if (!config.data.version || config.data.version === 'v1') {
+        //     return await this.processV1(input, config, agent);
+        // } else if (config.data.version === 'v2') {
+        //     return await this.processV2(input, config, agent);
+        // }
+        if (envConfig.env.ROLLOUT_RAG_V2) {
             return await this.processV2(input, config, agent);
+        } else {
+            return await this.processV1(input, config, agent);
         }
     }
     async processV1(input, config, agent: Agent) {
@@ -158,10 +164,8 @@ export class DataSourceLookup extends DataSourceComponent {
             outputs[con.name] = '';
         }
 
-        const namespaceLabel = /^c[a-z0-9]{24}.+$/.test(config.data.namespace)
-            ? config.data.namespace.split('_').slice(1).join('_')
-            : config.data.namespace;
-        const namespaceId = config.data.namespace;
+        const namespaceLabelorId = config.data.namespace;
+        // const namespaceId = config.data.namespace;
         const model = config.data?.model || 'gpt-4o-mini';
         const includeMetadata = config.data?.includeMetadata || false;
 
@@ -175,14 +179,14 @@ export class DataSourceLookup extends DataSourceComponent {
         // let vectorDbConnector = ConnectorService.getVectorDBConnector();
         // let existingNs = await vectorDbConnector.requester(AccessCandidate.team(teamId)).namespaceExists(namespaceLabel);
 
-        const vecDbConnector = await this.resolveVectorDbConnector(namespaceId, teamId);
+        const { vecDbConnector, namespaceRecord } = await this.resolveVectorDbConnector(namespaceLabelorId, teamId);
 
         let results: string[] | { content: string; metadata: any; score?: number }[];
         let _error;
         try {
             const response = await vecDbConnector
                 .requester(AccessCandidate.team(teamId))
-                .search(namespaceLabel, _input, { topK, includeMetadata: true });
+                .search(namespaceRecord.label, _input, { topK, includeMetadata: true });
 
             results = response.slice(0, config.data.topK).map((result) => ({
                 content: result.text,
@@ -211,7 +215,7 @@ export class DataSourceLookup extends DataSourceComponent {
                 return includeMetadata || includeScore ? transformedResult : result.content;
             });
 
-            debugOutput += `[Results] \nLoaded ${results.length} results from namespace: ${namespaceLabel}\n\n`;
+            debugOutput += `[Results] \nLoaded ${results.length} results from namespace: ${namespaceRecord.label}\n\n`;
         } catch (error) {
             debugOutput += `Error: ${error instanceof Error ? error.message : error.toString()}\n\n`;
             _error = error instanceof Error ? error.message : error.toString();

package/src/Components/ScrapflyWebScrape.class.ts

@@ -39,6 +39,7 @@ export class ScrapflyWebScrape extends Component {
         javascriptRendering: Joi.boolean().default(false).label('Enable JavaScript Rendering'),
         autoScroll: Joi.boolean().default(false).label('Enable Auto Scroll'),
         format: Joi.string().default('markdown').label('Format').optional(),
+        countries: Joi.array().items(Joi.string()).label('Countries').optional().allow(''),
     });
 
     constructor() {
@@ -96,6 +97,10 @@ export class ScrapflyWebScrape extends Component {
 
     async scrapeURL(url, data, key) {
         try {
+            // Handle countries - convert array to comma-separated string (e.g., "us,ca,mx")
+            const countryValue =
+                data.countries && Array.isArray(data.countries) && data.countries.length > 0 ? data.countries.join(',').toLowerCase() : undefined;
+
             const response = await axios({
                 method: 'get',
                 url: 'https://api.scrapfly.io/scrape',
@@ -107,8 +112,10 @@ export class ScrapflyWebScrape extends Component {
                     ...(data.antiScrapingProtection && { asp: true }),
                     ...(data.javascriptRendering && { render_js: true }),
                     ...(data.autoScroll && { auto_scroll: true, render_js: true }),
+                    ...(countryValue && { country: countryValue }),
                 },
             });
+
             return {
                 content: response.data?.result?.content,
                 success: true,

package/src/config.ts

@@ -6,6 +6,7 @@ const config = {
         LOG_LEVEL: process.env.LOG_LEVEL || 'none',
         LOG_FILTER: process.env.LOG_FILTER || '',
         NODE_ENV: process.env?.NODE_ENV,
+        ROLLOUT_RAG_V2: process.env.ROLLOUT_RAG_V2,
     },
     agent: {
         ENDPOINT_PREFIX: '/api',