@smythos/sre 1.7.20 → 1.7.41

package/src/Components/RAG/DataSourceCleaner.class.ts

@@ -0,0 +1,180 @@
+import { IAgent as Agent } from '@sre/types/Agent.types';
+import { Component } from '../Component.class';
+import { TemplateString } from '@sre/helpers/TemplateString.helper';
+import Joi from 'joi';
+import { validateCharacterSet } from '../../utils';
+import { ConnectorService } from '@sre/Core/ConnectorsService';
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+
+import { DataSourceIndexer } from './DataSourceIndexer.class';
+import { DataSourceComponent, NsRecord } from './DataSourceComponent.class';
+import { VectorDBConnector } from '@sre/IO/VectorDB.service/VectorDBConnector';
+import envConfig from '@sre/config';
+
+export class DataSourceCleaner extends DataSourceComponent {
+    protected configSchema = Joi.object({
+        namespaceId: Joi.string().max(50).allow('').label('namespace'),
+        id: Joi.string().custom(validateCharacterSet, 'custom validation characterSet').allow('').label('source identifier'),
+        version: Joi.string().valid('v1', 'v2').default('v1'),
+    });
+    constructor() {
+        super();
+    }
+    init() {}
+
+    async process(input, config, agent: Agent) {
+        await super.process(input, config, agent);
+        if (envConfig.env.ROLLOUT_RAG_V2) {
+            return await this.processV2(input, config, agent);
+        } else {
+            return await this.processV1(input, config, agent);
+        }
+    }
+    async processV1(input, config, agent: Agent) {
+        const teamId = agent.teamId;
+        const agentId = agent.id;
+        let debugOutput = agent.agentRuntime?.debug ? '== Source Indexer Log ==\n' : null;
+
+        try {
+            const configSchema = this.validateConfigData(config.data);
+            if (configSchema.error) {
+                throw new Error(`Config data validation error: ${configSchema.error}\n EXITING...`);
+            }
+
+            const outputs = {};
+            for (let con of config.outputs) {
+                if (con.default) continue;
+                outputs[con.name] = con?.description ? `<${con?.description}>` : '';
+            }
+
+            const inputSchema = this.validateInput(input);
+            if (inputSchema.error) {
+                throw new Error(`Input validation error: ${inputSchema.error}\n EXITING...`);
+            }
+
+            // const namespaceId = configSchema.value.namespaceId.split('_')?.slice(1).join('_') || configSchema.value.namespaceId;
+            const namespaceId = /^c[a-z0-9]{24}.+$/.test(configSchema.value.namespaceId)
+                ? configSchema.value.namespaceId.split('_').slice(1).join('_')
+                : configSchema.value.namespaceId;
+
+            let vectorDbConnector = ConnectorService.getVectorDBConnector();
+
+            let existingnamespace = await vectorDbConnector.requester(AccessCandidate.team(teamId)).namespaceExists(namespaceId);
+            if (!existingnamespace) {
+                throw new Error(`Namespace ${namespaceId} does not exist`);
+            }
+
+            const providedId = TemplateString(config.data.id).parse(input).result;
+            const idRegex = /^[a-zA-Z0-9\-\_\.]+$/;
+            if (!idRegex.test(providedId)) {
+                throw new Error(`Invalid id. Accepted characters: 'a-z', 'A-Z', '0-9', '-', '_', '.'`);
+            }
+            debugOutput += `Searching for data source with id: ${providedId}\n`;
+
+            const dsId = DataSourceIndexer.normalizeDsId(providedId, teamId, namespaceId);
+
+            await vectorDbConnector.requester(AccessCandidate.team(teamId)).deleteDatasource(namespaceId, dsId);
+
+            debugOutput += `Deleted data source with id: ${providedId}\n`;
+
+            return {
+                _debug: debugOutput,
+                Success: true,
+                // _error,
+            };
+        } catch (err: any) {
+            debugOutput += `Failed to delete data source: \n Error: ${err?.message}\n`;
+
+            return {
+                _debug: debugOutput,
+                _error: err?.message || "Couldn't delete data source",
+            };
+        }
+    }
+
+    async processV2(input, config, agent: Agent) {
+        const teamId = agent.teamId;
+        const agentId = agent.id;
+        let debugOutput = agent.agentRuntime?.debug ? '== Source Indexer Log ==\n' : null;
+
+        try {
+            const configSchema = this.validateConfigData(config.data);
+            if (configSchema.error) {
+                throw new Error(`Config data validation error: ${configSchema.error}\n EXITING...`);
+            }
+
+            const outputs = {};
+            for (let con of config.outputs) {
+                if (con.default) continue;
+                outputs[con.name] = con?.description ? `<${con?.description}>` : '';
+            }
+
+            const inputSchema = this.validateInput(input);
+            if (inputSchema.error) {
+                throw new Error(`Input validation error: ${inputSchema.error}\n EXITING...`);
+            }
+
+            // const namespaceId = configSchema.value.namespaceId.split('_')?.slice(1).join('_') || configSchema.value.namespaceId;
+            const namespaceLabelorId = configSchema.value.namespaceId;
+            // const namespaceId = configSchema.value.namespaceId;
+
+            let vecDbConnector: VectorDBConnector = null;
+            let namespaceRecord: NsRecord = null;
+            try {
+                const resolved = await this.resolveVectorDbConnector(namespaceLabelorId, teamId);
+                vecDbConnector = resolved.vecDbConnector;
+                namespaceRecord = resolved.namespaceRecord;
+            } catch (err: any) {
+                debugOutput += `Error: ${err?.message || "Couldn't get vector database connector"}\n\n`;
+                return {
+                    _debug: debugOutput,
+                    _error: err?.message || "Couldn't get vector database connector",
+                };
+            }
+
+            // let existingnamespace = await vecDbConnector.requester(AccessCandidate.team(teamId)).namespaceExists(namespaceId);
+            // if (!existingnamespace) {
+            //     throw new Error(`Namespace ${namespaceId} does not exist`);
+            // }
+
+            const providedId = TemplateString(config.data.id).parse(input).result;
+            const idRegex = /^[a-zA-Z0-9\-\_\.]+$/;
+            if (!idRegex.test(providedId)) {
+                throw new Error(`Invalid id. Accepted characters: 'a-z', 'A-Z', '0-9', '-', '_', '.'`);
+            }
+            debugOutput += `Searching for data source with id: ${providedId}\n`;
+
+            const dsId = DataSourceComponent.normalizeDsId(providedId, teamId, namespaceRecord.label);
+
+            await vecDbConnector.requester(AccessCandidate.team(teamId)).deleteDatasource(namespaceRecord.label, dsId);
+
+            debugOutput += `Deleted data source with id: ${providedId}\n`;
+
+            return {
+                _debug: debugOutput,
+                Success: true,
+                // _error,
+            };
+        } catch (err: any) {
+            debugOutput += `Failed to delete data source: \n Error: ${err?.message}\n`;
+
+            return {
+                _debug: debugOutput,
+                _error: err?.message || "Couldn't delete data source",
+            };
+        }
+    }
+
+    validateInput(input: any) {
+        return Joi.object({}).unknown(true).validate(input);
+    }
+
+    validateConfigData(data: any) {
+        return Joi.object({
+            namespaceId: Joi.string().required(),
+            id: Joi.string().optional().allow('').allow(null),
+        })
+            .unknown(true)
+            .validate(data);
+    }
+}

package/src/Components/RAG/DataSourceComponent.class.ts

@@ -0,0 +1,137 @@
+// Base class for the RAG components that handle the shared logic for the RAG components
+import { EmbeddingsFactory } from '@sre/IO/VectorDB.service/embed';
+import { Component } from '../Component.class';
+import { ConnectorService } from '@sre/Core/ConnectorsService';
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+import { getLLMCredentials } from '@sre/LLMManager/LLM.service/LLMCredentials.helper';
+import { TEmbeddings } from '@sre/IO/VectorDB.service/embed/BaseEmbedding';
+import { TemplateString } from '@sre/helpers/TemplateString.helper';
+import { VectorDBConnector } from '@sre/IO/VectorDB.service/VectorDBConnector';
+import { TLLMCredentials } from '@sre/types/LLM.types';
+
+export type NsRecord = {
+    credentialId: string;
+    embeddings: { dimensions: string; modelId: string };
+    label: string;
+    createdAt: string;
+};
+
+export enum TDataSourceCompErrorCodes {
+    NAMESPACE_NOT_FOUND = 1,
+    CREDENTIAL_NOT_FOUND = 2,
+    EMBEDDING_CONFIG_ERROR = 3,
+}
+export class DataSourceCompError extends Error {
+    public code: TDataSourceCompErrorCodes;
+    constructor(message: string, code: TDataSourceCompErrorCodes) {
+        super(message);
+        this.name = 'DataSourceCompError';
+        this.code = code;
+    }
+}
+export class DataSourceComponent extends Component {
+    constructor() {
+        super();
+    }
+
+    public async resolveVectorDbConnector(
+        namespace: string | NsRecord,
+        teamId: string
+    ): Promise<{ vecDbConnector: VectorDBConnector; namespaceRecord: NsRecord }> {
+        // resolve the ns record, if not exist, throw an error (new in v2)
+        // then we also need to resolve the credentials
+        let namespaceRecord = namespace as NsRecord;
+
+        if (typeof namespace === 'string') {
+            namespaceRecord = await this.resolveNamespaceRecord(namespace, teamId);
+        }
+
+        const accountConnector = ConnectorService.getAccountConnector();
+        const accountClient = accountConnector.requester(AccessCandidate.team(teamId));
+        const rawCredRecord = await accountClient.getTeamSetting(namespaceRecord.credentialId, 'vector_db_creds');
+        if (!rawCredRecord) {
+            throw new DataSourceCompError(
+                `Credential ${namespaceRecord.credentialId} does not exist`,
+                TDataSourceCompErrorCodes.CREDENTIAL_NOT_FOUND
+            );
+        }
+        const credRecord = JSON.parse(rawCredRecord);
+        await Promise.all(
+            Object.keys(credRecord.credentials).map(async (key) => {
+                if (typeof credRecord.credentials[key] !== 'string') return;
+                credRecord.credentials[key] = await TemplateString(credRecord.credentials[key]).parseTeamKeysAsync(teamId).asyncResult;
+            })
+        );
+
+        const vecDbConnector = ConnectorService.getVectorDBConnector(credRecord.provider).instance({
+            credentials: credRecord.credentials,
+            embeddings: await this.buildEmbeddingConfig(namespaceRecord.embeddings, teamId),
+        });
+
+        return { vecDbConnector, namespaceRecord };
+    }
+
+    private async resolveNamespaceRecord(namespace: string, teamId: string): Promise<NsRecord> {
+        // if it's a string, we need to get the namespace record from the NKV
+        // TRY 1) try to get namespace using the label provided from teamSettings
+        const nkvConnector = ConnectorService.getNKVConnector();
+        const nkvClient = nkvConnector.requester(AccessCandidate.team(teamId));
+        const rawNsRecord = await nkvClient.get(`vectorDB:namespaces`, namespace);
+
+        if (rawNsRecord) {
+            return JSON.parse(rawNsRecord.toString()) as NsRecord;
+        } else {
+            // throw new DataSourceCompError(`Namespace ${namespace} does not exist`, TDataSourceCompErrorCodes.NAMESPACE_NOT_FOUND);
+            console.warn(`Namespace ${namespace} does not exist using the label ${namespace}. Trying to get namespace by mapping legacy id to V2 id`);
+        }
+
+        // TRY 2) try to get namespace using the legacy id. list all namespaces and find the one that matches the legacy id
+        const namespaces = (await nkvClient.list(`vectorDB:namespaces`))
+            .map((namespace) => {
+                try {
+                    return JSON.parse(namespace.data.toString());
+                } catch (error) {
+                    return null;
+                }
+            })
+            .filter((namespace) => namespace !== null);
+
+        const matchingNamespace = namespaces.find((ns) => ns.__legacy_id === namespace);
+        if (!matchingNamespace) {
+            console.warn(`Namespace ${namespace} does not exist using the legacy id ${namespace}.`);
+            throw new DataSourceCompError(`Namespace ${namespace} does not exist`, TDataSourceCompErrorCodes.NAMESPACE_NOT_FOUND);
+        }
+        return matchingNamespace;
+    }
+
+    public async buildEmbeddingConfig(embedding: { dimensions: string; modelId: string }, teamId: string): Promise<TEmbeddings> {
+        // we need to take this and return a proper TEmbeddings object
+
+        const provider = EmbeddingsFactory.getProviderByModel(embedding.modelId as any);
+
+        // based on the provider, we should be able to retreive the correct credentials
+        const modelsProvider = ConnectorService.getModelsProviderConnector();
+        const modelProviderCandidate = modelsProvider.requester(AccessCandidate.team(teamId));
+        // const modelInfo = await modelProviderCandidate.getModelInfo(embedding.modelId);
+
+        const llmCreds = await getLLMCredentials(AccessCandidate.team(teamId), {
+            provider,
+            modelId: embedding.modelId,
+            credentials: [TLLMCredentials.Vault, TLLMCredentials.Internal],
+        });
+
+        return {
+            provider,
+            model: embedding.modelId,
+            credentials: llmCreds,
+            dimensions: parseInt(embedding.dimensions), // pass both for backwards compatibility
+            params: {
+                dimensions: parseInt(embedding.dimensions), // pass both for backwards compatibility
+            },
+        };
+    }
+
+    public static normalizeDsId(providedId: string, teamId: string, namespaceId: string) {
+        return `${teamId}::${namespaceId}::${providedId}`;
+    }
+}

package/src/Components/RAG/DataSourceIndexer.class.ts

@@ -0,0 +1,260 @@
+import { IAgent as Agent } from '@sre/types/Agent.types';
+import { DataSourceComponent, NsRecord } from './DataSourceComponent.class';
+import Joi from 'joi';
+import { validateCharacterSet } from '@sre/utils/validation.utils';
+import { TemplateString } from '@sre/helpers/TemplateString.helper';
+import { isUrl, detectURLSourceType } from '../../utils';
+import { SmythFS } from '@sre/IO/Storage.service/SmythFS.class';
+import { ConnectorService } from '@sre/Core/ConnectorsService';
+
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+import { TEmbeddings } from '@sre/IO/VectorDB.service/embed/BaseEmbedding';
+import { VectorDBConnector } from '@sre/IO/VectorDB.service/VectorDBConnector';
+import { JSONContentHelper } from '@sre/helpers/JsonContent.helper';
+import envConfig from '@sre/config';
+
+export class DataSourceIndexer extends DataSourceComponent {
+    private MAX_ALLOWED_URLS_PER_INPUT = 20;
+    protected configSchema = Joi.object({
+        namespace: Joi.string().max(50).allow(''),
+        id: Joi.string().custom(validateCharacterSet, 'id custom validation').allow('').label('source identifier'),
+        name: Joi.string().max(50).allow('').label('label'),
+        metadata: Joi.string().allow(null).allow('').max(10000).label('metadata'),
+        chunkSize: Joi.number().optional(),
+        chunkOverlap: Joi.number().optional(),
+        version: Joi.string().valid('v1', 'v2').default('v1'),
+    });
+    constructor() {
+        super();
+    }
+    init() {}
+    async process(input, config, agent: Agent) {
+        await super.process(input, config, agent);
+
+        let response: any = null;
+        if (envConfig.env.ROLLOUT_RAG_V2) {
+            response = await this.processV2(input, config, agent);
+        } else {
+            response = await this.processV1(input, config, agent);
+        }
+
+        return response;
+    }
+
+    private async processV1(input, config, agent: Agent) {
+        const teamId = agent.teamId;
+        const agentId = agent.id;
+        let debugOutput = agent.agentRuntime?.debug ? '== Source Indexer Log ==\n' : null;
+
+        try {
+            const _config = {
+                ...config.data,
+                name: TemplateString(config.data.name).parse(input).result,
+                id: TemplateString(config.data.id).parse(input).result,
+                metadata: TemplateString(config.data.metadata).parse(input).result,
+            };
+
+            const outputs = {};
+            for (let con of config.outputs) {
+                if (con.default) continue;
+                outputs[con.name] = con?.description ? `<${con?.description}>` : '';
+            }
+
+            const namespaceId = _config.namespace.split('_').slice(1).join('_') || _config.namespace;
+            debugOutput += `[Selected data space id] \n${namespaceId}\n\n`;
+
+            const vectorDbConnector =
+                // (await vectorDBHelper.getTeamConnector(teamId)) ||
+                ConnectorService.getVectorDBConnector();
+            const nsExists = await vectorDbConnector.requester(AccessCandidate.team(teamId)).namespaceExists(namespaceId);
+
+            if (!nsExists) {
+                const newNs = await vectorDbConnector.requester(AccessCandidate.team(teamId)).createNamespace(namespaceId);
+                debugOutput += `[Created namespace] \n${newNs}\n\n`;
+            }
+
+            const inputSchema = this.validateInput(input);
+            if (inputSchema.error) {
+                throw new Error(`Input validation error: ${inputSchema.error}\n EXITING...`);
+            }
+
+            const providedId = _config.id;
+            // const isAutoId = _config.isAutoId;
+            const idRegex = /^[a-zA-Z0-9\-\_\.]+$/;
+
+            if (!providedId) {
+                // Assign a new ID if it's set to auto-generate or not provided
+                // _config.id = crypto.randomBytes(16).toString('hex');
+                throw new Error(`Id is required`);
+            } else if (!idRegex.test(providedId)) {
+                // Validate the provided ID if it's not auto-generated
+                throw new Error(`Invalid id. Accepted characters: 'a-z', 'A-Z', '0-9', '-', '_', '.'`);
+            }
+
+            let indexRes: any = null;
+            let parsedUrlArray: string[] | null = null;
+            const dsId = DataSourceIndexer.normalizeDsId(providedId, teamId, namespaceId);
+
+            if (isUrl(inputSchema.value.Source)) {
+                debugOutput += `STEP: Parsing input as url\n\n`;
+                throw new Error('URLs are not supported yet');
+            } else {
+                debugOutput += `STEP: Parsing input as text\n\n`;
+                indexRes = await this.addDSFromText({
+                    teamId,
+                    namespaceId: namespaceId,
+                    text: inputSchema.value.Source,
+                    name: _config.name || 'Untitled',
+                    metadata: _config.metadata || null,
+                    sourceId: dsId,
+                });
+            }
+
+            debugOutput += `Created datasource successfully\n\n`;
+
+            return {
+                _debug: debugOutput,
+                Success: {
+                    result: indexRes?.data?.dataSource || true,
+                    id: _config.id,
+                },
+                // _error,
+            };
+        } catch (err: any) {
+            debugOutput += `Error: ${err?.message || "Couldn't index data source"}\n\n`;
+            return {
+                _debug: debugOutput,
+                _error: err?.message || "Couldn't index data source",
+            };
+        }
+    }
+
+    private async processV2(input, config, agent: Agent) {
+        const teamId = agent.teamId;
+        const agentId = agent.id;
+        let debugOutput = agent.agentRuntime?.debug ? '== Source Indexer Log ==\n' : null;
+
+        try {
+            const _config = {
+                ...config.data,
+                name: TemplateString(config.data.name).parse(input).result,
+                id: TemplateString(config.data.id).parse(input).result,
+                metadata: TemplateString(config.data.metadata).parse(input).result,
+            };
+
+            const outputs = {};
+            for (let con of config.outputs) {
+                if (con.default) continue;
+                outputs[con.name] = con?.description ? `<${con?.description}>` : '';
+            }
+
+            // we try to get the namespace without the prefix teamId, if not exist, we use the full namespace id
+            // const namespaceLabel = _config.namespace.split('_').slice(1).join('_') || _config.namespace;
+            const namespaceLabelorId = _config.namespace;
+            // const namespaceId = _config.namespace;
+
+            let vecDbConnector: VectorDBConnector = null;
+            let namespaceRecord: NsRecord = null;
+            try {
+                const resolved = await this.resolveVectorDbConnector(namespaceLabelorId, teamId);
+                vecDbConnector = resolved.vecDbConnector;
+                namespaceRecord = resolved.namespaceRecord;
+            } catch (err: any) {
+                debugOutput += `Error: ${err?.message || "Couldn't get vector database connector"}\n\n`;
+                return {
+                    _debug: debugOutput,
+                    _error: err?.message || "Couldn't get vector database connector",
+                };
+            }
+
+            debugOutput += `[Selected data space] \n${namespaceRecord.label}\n\n`;
+
+            const vecDbClient = vecDbConnector.requester(AccessCandidate.team(teamId));
+
+            const inputSchema = this.validateInput(input);
+            if (inputSchema.error) {
+                throw new Error(`Input validation error: ${inputSchema.error}\n EXITING...`);
+            }
+
+            const providedId = _config.id;
+            // const isAutoId = _config.isAutoId;
+            const idRegex = /^[a-zA-Z0-9\-\_\.]+$/;
+
+            if (!providedId) {
+                // Assign a new ID if it's set to auto-generate or not provided
+                // _config.id = crypto.randomBytes(16).toString('hex');
+                throw new Error(`Id is required`);
+            } else if (!idRegex.test(providedId)) {
+                // Validate the provided ID if it's not auto-generated
+                throw new Error(`Invalid id. Accepted characters: 'a-z', 'A-Z', '0-9', '-', '_', '.'`);
+            }
+
+            const dsId = DataSourceIndexer.normalizeDsId(providedId, teamId, namespaceRecord.label);
+
+            // check if the datasource already exists
+            const dsExists = await vecDbClient.getDatasource(namespaceRecord.label, dsId);
+            if (dsExists) {
+                debugOutput += `Datasource already exists\n\n`;
+                return {
+                    _debug: debugOutput,
+                    _error: `Datasource already exists`,
+                };
+            }
+
+            debugOutput += `STEP: Parsing input as text\n\n`;
+
+            const response = await vecDbClient.createDatasource(namespaceRecord.label, {
+                text: inputSchema.value.Source,
+                metadata: JSONContentHelper.create(_config.metadata).tryParse() || null,
+                id: dsId,
+                label: _config.name || 'Untitled',
+                chunkSize: _config.chunkSize ? parseInt(_config.chunkSize) : undefined,
+                chunkOverlap: _config.chunkOverlap ? parseInt(_config.chunkOverlap) : undefined,
+            });
+
+            debugOutput += `Created datasource successfully\n\n`;
+
+            return {
+                _debug: debugOutput,
+                Success: {
+                    result: response || true,
+                    id: _config.id,
+                },
+                // _error,
+            };
+        } catch (err: any) {
+            debugOutput += `Error: ${err?.message || "Couldn't index data source"}\n\n`;
+            return {
+                _debug: debugOutput,
+                _error: err?.message || "Couldn't index data source",
+            };
+        }
+    }
+
+    validateInput(input: any) {
+        return Joi.object({
+            Source: Joi.any().required(),
+        })
+            .unknown(true)
+            .validate(input);
+    }
+
+    private async addDSFromText({ teamId, sourceId, namespaceId, text, name, metadata }) {
+        let vectorDbConnector = ConnectorService.getVectorDBConnector();
+        // const isOnCustomStorage = await vectorDBHelper.isNamespaceOnCustomStorage(teamId, namespaceId);
+        // if (isOnCustomStorage) {
+        // const customTeamConnector = await vectorDBHelper.getTeamConnector(teamId);
+        // if (customTeamConnector) {
+        // vectorDbConnector = customTeamConnector;
+        // }
+        // }
+        const id = await vectorDbConnector.requester(AccessCandidate.team(teamId)).createDatasource(namespaceId, {
+            text,
+            metadata,
+            id: sourceId,
+            label: name,
+        });
+
+        return id;
+    }
+}