@smythos/sre 1.5.70 → 1.5.72

package/src/helpers/Sysconfig.helper.ts

@@ -13,6 +13,24 @@ export function findSmythPath(_path: string = '', callback?: (smythDir: string,
 
     const searchDirectories = [];
 
+    if (process.env.SMYTH_PATH) {
+        if (!fs.existsSync(process.env.SMYTH_PATH)) {
+            console.error('CRITICAL : SMYTH_PATH environment variable is not a valid directory');
+            process.exit(1);
+        }
+
+        const envDir = path.resolve(process.env.SMYTH_PATH, _path);
+        //check if the directory exists
+        if (!fs.existsSync(envDir)) {
+            callback?.(envDir, false, null);
+            console.error(`CRITICAL : missing directory (${envDir}) under SMYTH_PATH `);
+            //process.exit(1);
+        } else {
+            callback?.(envDir, true, null);
+        }
+        return envDir;
+    }
+
     // 1. Try to find in local directory (the directory from which the program was run)
     const localDir = path.resolve(process.cwd(), '.smyth', _path);
     searchDirectories.push(localDir);

package/src/subsystems/AgentManager/AgentData.service/connectors/LocalAgentDataConnector.class.ts

@@ -2,6 +2,9 @@ import fs from 'fs';
 import path from 'path';
 import { AgentDataConnector } from '../AgentDataConnector';
 import { uid } from '@sre/utils/general.utils';
+import { Logger } from '@sre/helpers/Log.helper';
+
+const console = Logger('LocalAgentDataConnector');
 
 export type LocalAgentDataSettings = { devDir: string; prodDir: string };
 

package/src/subsystems/IO/VectorDB.service/connectors/RAMVecrtorDB.class.ts

@@ -5,7 +5,7 @@ import { IAccessCandidate, IACL, TAccessLevel } from '@sre/types/ACL.types';
 import { AccessRequest } from '@sre/Security/AccessControl/AccessRequest.class';
 import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
 import { SecureConnector } from '@sre/Security/SecureConnector.class';
-import { VectorDBConnector } from '../VectorDBConnector';
+import { VectorDBConnector, DeleteTarget } from '../VectorDBConnector';
 import {
     DatasourceDto,
     IStorageVectorDataSource,
@@ -21,6 +21,8 @@ import { OpenAIEmbeds } from '@sre/IO/VectorDB.service/embed/OpenAIEmbedding';
 import crypto from 'crypto';
 import { BaseEmbedding, TEmbeddings } from '../embed/BaseEmbedding';
 import { EmbeddingsFactory } from '../embed';
+import { chunkText } from '@sre/utils/string.utils';
+import { jsonrepair } from 'jsonrepair';
 
 const console = Logger('RAM VectorDB');
 
@@ -66,6 +68,7 @@ export class RAMVectorDB extends VectorDBConnector {
         if (!_settings.embeddings) {
             _settings.embeddings = { provider: 'OpenAI', model: 'text-embedding-3-large', params: { dimensions: 1024 } };
         }
+        if (!_settings.embeddings.params) _settings.embeddings.params = { dimensions: 1024 };
         if (!_settings.embeddings.params?.dimensions) _settings.embeddings.params.dimensions = 1024;
 
         this.embedder = EmbeddingsFactory.create(_settings.embeddings.provider, _settings.embeddings);
@@ -182,12 +185,26 @@ export class RAMVectorDB extends VectorDBConnector {
 
         for (const vector of namespaceData) {
             const similarity = this.cosineSimilarity(queryVector as number[], vector.values);
+
+            let userMetadata = undefined;
+            if (options.includeMetadata) {
+                if (vector.metadata?.[this.USER_METADATA_KEY]) {
+                    try {
+                        userMetadata = JSON.parse(vector.metadata[this.USER_METADATA_KEY]);
+                    } catch {
+                        userMetadata = vector.metadata[this.USER_METADATA_KEY];
+                    }
+                } else {
+                    userMetadata = {}; // Return empty object when no metadata exists, like Milvus
+                }
+            }
+
             results.push({
                 id: vector.id,
                 score: similarity,
                 values: vector.values,
-                metadata: options.includeMetadata ? vector.metadata : undefined,
-                text: vector.metadata?.text,
+                text: vector.metadata?.text as string | undefined,
+                metadata: options.includeMetadata ? userMetadata : undefined,
             });
         }
 
@@ -207,28 +224,28 @@ export class RAMVectorDB extends VectorDBConnector {
         //const teamId = await this.accountConnector.getCandidateTeam(acRequest.candidate);
         const preparedNs = this.constructNsName(acRequest.candidate as AccessCandidate, namespace);
 
-        const sources = Array.isArray(sourceWrapper) ? sourceWrapper : [sourceWrapper];
-        const insertedIds: string[] = [];
+        sourceWrapper = Array.isArray(sourceWrapper) ? sourceWrapper : [sourceWrapper];
 
-        if (!this.vectors[preparedNs]) {
-            this.vectors[preparedNs] = [];
+        // make sure that all sources are of the same type (source.source)
+        if (sourceWrapper.some((s) => this.embedder.detectSourceType(s.source) !== this.embedder.detectSourceType(sourceWrapper[0].source))) {
+            throw new Error('All sources must be of the same type');
         }
 
-        for (const source of sources) {
-            let vector: number[] = [];
+        const sourceType = this.embedder.detectSourceType(sourceWrapper[0].source);
+        if (sourceType === 'unknown' || sourceType === 'url') throw new Error('Invalid source type');
 
-            if (typeof source.source === 'string') {
-                // Text embedding
+        const transformedSource = await this.embedder.transformSource(sourceWrapper, sourceType, acRequest.candidate as AccessCandidate);
 
-                vector = await this.embedder.embedText(source.source, acRequest.candidate as AccessCandidate);
-            } else {
-                // Direct vector
-                vector = source.source;
-            }
+        if (!this.vectors[preparedNs]) {
+            this.vectors[preparedNs] = [];
+        }
+
+        const insertedIds: string[] = [];
 
+        for (const source of transformedSource) {
             const vectorData: VectorData = {
                 id: source.id,
-                values: vector,
+                values: source.source as number[],
                 datasource: source.metadata?.datasourceId || 'unknown',
                 metadata: source.metadata,
             };
@@ -248,107 +265,96 @@ export class RAMVectorDB extends VectorDBConnector {
     }
 
     @SecureConnector.AccessControl
-    protected async delete(acRequest: AccessRequest, namespace: string, id: string | string[]): Promise<void> {
+    protected async delete(acRequest: AccessRequest, namespace: string, deleteTarget: DeleteTarget): Promise<void> {
         //const teamId = await this.accountConnector.getCandidateTeam(acRequest.candidate);
         const preparedNs = this.constructNsName(acRequest.candidate as AccessCandidate, namespace);
 
-        const ids = Array.isArray(id) ? id : [id];
+        const isDeleteByFilter = typeof deleteTarget === 'object' && !Array.isArray(deleteTarget);
 
-        if (this.vectors[preparedNs]) {
-            this.vectors[preparedNs] = this.vectors[preparedNs].filter((vector) => !ids.includes(vector.id));
+        if (isDeleteByFilter) {
+            // Handle delete by filter (e.g., by datasourceId)
+            if ('datasourceId' in deleteTarget && deleteTarget.datasourceId) {
+                if (this.vectors[preparedNs]) {
+                    this.vectors[preparedNs] = this.vectors[preparedNs].filter((vector) => vector.datasource !== deleteTarget.datasourceId);
+                }
+            } else {
+                throw new Error('Unsupported delete filter');
+            }
+        } else {
+            // Handle delete by ID(s)
+            const ids = Array.isArray(deleteTarget) ? deleteTarget : [deleteTarget];
+            if (this.vectors[preparedNs]) {
+                this.vectors[preparedNs] = this.vectors[preparedNs].filter((vector) => !ids.includes(vector.id));
+            }
         }
     }
 
     @SecureConnector.AccessControl
     protected async createDatasource(acRequest: AccessRequest, namespace: string, datasource: DatasourceDto): Promise<IStorageVectorDataSource> {
         //const teamId = await this.accountConnector.getCandidateTeam(acRequest.candidate);
-        const preparedNs = this.constructNsName(acRequest.candidate as AccessCandidate, namespace);
-        const datasourceId = datasource.id || crypto.randomUUID();
-
-        // Ensure namespace exists
-        if (!this.namespaces[preparedNs]) {
-            await this.createNamespace(acRequest, namespace);
-        }
-
-        // Process text and create vectors
-        const vectorIds: string[] = [];
-
-        // Split text into chunks if needed
-        const chunkSize = datasource.chunkSize || 1000;
-        const chunkOverlap = datasource.chunkOverlap || 200;
-        const chunks = this.splitTextIntoChunks(datasource.text, chunkSize, chunkOverlap);
-
-        // Initialize namespace vectors if not exists (should already exist if namespace was created properly)
-        if (!this.vectors[preparedNs]) {
-            this.vectors[preparedNs] = [];
-        }
-
-        for (let i = 0; i < chunks.length; i++) {
-            const chunkId = `${datasourceId}_chunk_${i}`;
-            const vector = await this.embedder.embedText(chunks[i], acRequest.candidate as AccessCandidate);
-
-            const vectorData: VectorData = {
-                id: chunkId,
-                values: vector,
-                datasource: datasourceId,
+        const acl = new ACL().addAccess(acRequest.candidate.role, acRequest.candidate.id, TAccessLevel.Owner);
+        const dsId = datasource.id || crypto.randomUUID();
+
+        const formattedNs = this.constructNsName(acRequest.candidate as AccessCandidate, namespace);
+        const chunkedText = chunkText(datasource.text, {
+            chunkSize: datasource.chunkSize,
+            chunkOverlap: datasource.chunkOverlap,
+        });
+        const label = datasource.label || 'Untitled';
+        const ids = Array.from({ length: chunkedText.length }, (_, i) => `${dsId}_${crypto.randomUUID()}`);
+        const source: IVectorDataSourceDto[] = chunkedText.map((doc, i) => {
+            return {
+                id: ids[i],
+                source: doc,
                 metadata: {
-                    ...datasource.metadata,
-                    text: chunks[i],
-                    chunkIndex: i,
-                    totalChunks: chunks.length,
+                    acl: acl.serializedACL,
+                    namespaceId: formattedNs,
+                    datasourceId: dsId,
+                    datasourceLabel: label,
+                    [this.USER_METADATA_KEY]: datasource.metadata ? jsonrepair(JSON.stringify(datasource.metadata)) : undefined,
                 },
             };
+        });
 
-            this.vectors[preparedNs].push(vectorData);
-            vectorIds.push(chunkId);
-        }
+        const _vIds = await this.insert(acRequest, namespace, source);
 
-        const storageDataSource: IStorageVectorDataSource = {
-            namespaceId: preparedNs,
+        const dsData: IStorageVectorDataSource = {
+            namespaceId: formattedNs,
             candidateId: acRequest.candidate.id,
             candidateRole: acRequest.candidate.role,
-            name: datasource.label || `Datasource ${datasourceId}`,
-            metadata: JSON.stringify(datasource.metadata || {}),
+            name: datasource.label || 'Untitled',
+            metadata: datasource.metadata ? jsonrepair(JSON.stringify(datasource.metadata)) : undefined,
             text: datasource.text,
-            vectorIds,
-            id: datasourceId,
+            vectorIds: _vIds,
+            id: dsId,
         };
 
         // Store datasource metadata in memory
-        if (!this.datasources[preparedNs]) {
-            this.datasources[preparedNs] = {};
-        }
-        if (!this.datasources[preparedNs][datasourceId]) {
-            this.datasources[preparedNs][datasourceId] = storageDataSource;
-        } else {
-            this.datasources[preparedNs][datasourceId].vectorIds.push(...vectorIds);
+        if (!this.datasources[formattedNs]) {
+            this.datasources[formattedNs] = {};
         }
+        this.datasources[formattedNs][dsId] = dsData;
 
-        return storageDataSource;
+        return dsData;
     }
 
     @SecureConnector.AccessControl
     protected async deleteDatasource(acRequest: AccessRequest, namespace: string, datasourceId: string): Promise<void> {
         //const teamId = await this.accountConnector.getCandidateTeam(acRequest.candidate);
-        const preparedNs = this.constructNsName(acRequest.candidate as AccessCandidate, namespace);
+        const formattedNs = this.constructNsName(acRequest.candidate as AccessCandidate, namespace);
 
-        // Ensure namespace exists
-        if (!this.namespaces[preparedNs]) {
-            throw new Error('Namespace does not exist');
+        // Get datasource info to get vector IDs
+        const ds = this.datasources[formattedNs]?.[datasourceId];
+        if (!ds) {
+            throw new Error(`Data source not found with id: ${datasourceId}`);
         }
 
-        // Get datasource info
-        const datasource = this.datasources[preparedNs]?.[datasourceId];
-        if (datasource) {
-            // Delete all vectors belonging to this datasource
-            if (this.vectors[preparedNs]) {
-                this.vectors[preparedNs] = this.vectors[preparedNs].filter((vector) => vector.datasource !== datasourceId);
-            }
-        }
+        // Delete all vectors belonging to this datasource using the delete method
+        await this.delete(acRequest, namespace, ds.vectorIds || []);
 
         // Delete datasource metadata
-        if (this.datasources[preparedNs]) {
-            delete this.datasources[preparedNs][datasourceId];
+        if (this.datasources[formattedNs]) {
+            delete this.datasources[formattedNs][datasourceId];
         }
     }
 
@@ -362,15 +368,12 @@ export class RAMVectorDB extends VectorDBConnector {
     }
 
     @SecureConnector.AccessControl
-    protected async getDatasource(acRequest: AccessRequest, namespace: string, datasourceId: string): Promise<IStorageVectorDataSource> {
+    protected async getDatasource(acRequest: AccessRequest, namespace: string, datasourceId: string): Promise<IStorageVectorDataSource | undefined> {
         //const teamId = await this.accountConnector.getCandidateTeam(acRequest.candidate);
         const preparedNs = this.constructNsName(acRequest.candidate as AccessCandidate, namespace);
 
         const datasource = this.datasources[preparedNs]?.[datasourceId];
-        if (!datasource) {
-            throw new Error(`Datasource ${datasourceId} not found`);
-        }
-        return datasource;
+        return datasource; // Return undefined if not found, like MilvusVectorDB
     }
 
     /**
@@ -400,22 +403,4 @@ export class RAMVectorDB extends VectorDBConnector {
 
         return dotProduct / (normA * normB);
     }
-
-    /**
-     * Split text into chunks with overlap
-     */
-    private splitTextIntoChunks(text: string, chunkSize: number, overlap: number): string[] {
-        const chunks: string[] = [];
-        let start = 0;
-
-        while (start < text.length) {
-            const end = Math.min(start + chunkSize, text.length);
-            chunks.push(text.slice(start, end));
-
-            if (end === text.length) break;
-            start = end - overlap;
-        }
-
-        return chunks;
-    }
 }

package/src/subsystems/IO/VectorDB.service/embed/GoogleEmbedding.ts

@@ -50,7 +50,7 @@ export class GoogleEmbeds extends BaseEmbedding {
             const modelInfo: TLLMModel = {
                 provider: 'GoogleAI',
                 modelId: this.model,
-                credentials: this.settings?.credentials as unknown as TLLMCredentials,
+                credentials: (this.settings?.credentials as unknown as TLLMCredentials) || [TLLMCredentials.Internal, TLLMCredentials.Vault],
             };
             const credentials = await getLLMCredentials(candidate, modelInfo);
             apiKey = (credentials as BasicCredentials)?.apiKey;

package/src/subsystems/IO/VectorDB.service/embed/OpenAIEmbedding.ts

@@ -82,7 +82,7 @@ export class OpenAIEmbeds extends BaseEmbedding {
         const modelInfo: TLLMModel = {
             provider: 'OpenAI',
             modelId: this.model,
-            credentials: this.settings?.credentials as unknown as TLLMCredentials,
+            credentials: (this.settings?.credentials as unknown as TLLMCredentials) || [TLLMCredentials.Internal, TLLMCredentials.Vault],
         };
         const credentials = await getLLMCredentials(candidate, modelInfo);
 

package/src/subsystems/LLMManager/LLM.inference.ts

@@ -127,6 +127,7 @@ export class LLMInference {
         return this.llmConnector.user(AccessCandidate.agent(params.agentId)).imageEditRequest(params);
     }
 
+    //@deprecated
     public async streamRequest(params: any, agent: string | IAgent) {
         const agentId = isAgent(agent) ? (agent as IAgent).id : agent;
         try {
@@ -149,6 +150,7 @@ export class LLMInference {
         }
     }
 
+    //@deprecated
     public async multimodalStreamRequest(params: any, fileSources, agent: string | IAgent) {
         const agentId = isAgent(agent) ? (agent as IAgent).id : agent;
 
@@ -180,6 +182,7 @@ export class LLMInference {
         }
     }
 
+    //@deprecated
     public async multimodalStreamRequestLegacy(prompt, files: string[], config: any = {}, agent: string | IAgent) {
         const agentId = isAgent(agent) ? (agent as IAgent).id : agent;
 

package/src/subsystems/LLMManager/ModelsProvider.service/connectors/JSONModelsProvider.class.ts

@@ -14,6 +14,7 @@ import chokidar from 'chokidar';
 import fs from 'fs/promises';
 import fsSync from 'fs';
 import path from 'path';
+import { findSmythPath } from '@sre/helpers/Sysconfig.helper';
 
 const console = Logger('SmythModelsProvider');
 
@@ -52,6 +53,10 @@ export class JSONModelsProvider extends ModelsProviderConnector {
             else this.models = this._settings.models as TLLMModelsList;
             this.started = true;
         } else {
+            const modelsFolder = this.findModelsFolder();
+            if (modelsFolder) {
+                this.initDirWatcher(modelsFolder);
+            }
             this.started = true;
         }
     }
@@ -59,6 +64,17 @@ export class JSONModelsProvider extends ModelsProviderConnector {
         super.start();
     }
 
+    private findModelsFolder() {
+        const _modelsFolder = findSmythPath('models');
+
+        if (fsSync.existsSync(_modelsFolder)) {
+            console.warn('Using default models folder  : ', _modelsFolder);
+            return _modelsFolder;
+        }
+
+        return null;
+    }
+
     @SecureConnector.AccessControl
     public async addModels(acRequest: AccessRequest, models: TLLMModelsList): Promise<void> {
         await this.ready();

package/src/subsystems/LLMManager/models.ts

@@ -68,11 +68,11 @@ export const models = {
     // keep the gpt-4o-mini as default model for now
     'gpt-4o-mini': {
         llm: 'OpenAI',
-        alias: 'gpt-4o-mini-2024-07-18',
+
         components: ['PromptGenerator', 'LLMAssistant', 'Classifier', 'VisionLLM', 'AgentPlugin', 'Chatbot', 'GPTPlugin', 'GenAILLM'],
 
         label: 'GPT 4o Mini',
-        modelId: 'gpt-4o-mini-2024-07-18',
+        modelId: 'gpt-4o-mini',
         provider: 'OpenAI',
         features: ['text', 'tools', 'image', 'search'],
         tags: ['Personal'],

package/src/subsystems/Security/Account.service/connectors/JSONFileAccount.class.ts

@@ -74,8 +74,14 @@ export class JSONFileAccount extends AccountConnector {
     }
 
     public async isTeamMember(team: string, candidate: IAccessCandidate): Promise<boolean> {
+        if (team === DEFAULT_TEAM_ID) return true; //everyone is a member of the default team
+
         if (!this.data[team]) return false;
 
+        if (candidate.role === TAccessRole.Team && team === candidate.id) {
+            return true;
+        }
+
         if (candidate.role === TAccessRole.User) {
             return !!this.data[team].users?.[candidate.id];
         } else if (candidate.role === TAccessRole.Agent) {

package/src/types/LLM.types.ts

@@ -221,7 +221,7 @@ export type TLLMModel = {
         tokens: number;
         completionTokens: number;
     };
-    credentials?: TLLMCredentials;
+    credentials?: TLLMCredentials | TLLMCredentials[];
 
     //models can come with predefined params
     //this can also be used to pass a preconfigured model object