@smythos/sre 1.5.2 → 1.5.5

package/src/subsystems/IO/NKV.service/index.ts

@@ -4,9 +4,11 @@ import { ConnectorService, ConnectorServiceProvider } from '@sre/Core/Connectors
 import { TConnectorService } from '@sre/types/SRE.types';
 import { NKVRedis } from './connectors/NKVRedis.class';
 import { NKVRAM } from './connectors/NKVRAM.class';
+import { NKVLocalStorage } from './connectors/NKVLocalStorage.class';
 export class NKVService extends ConnectorServiceProvider {
     public register() {
         ConnectorService.register(TConnectorService.NKV, 'Redis', NKVRedis);
         ConnectorService.register(TConnectorService.NKV, 'RAM', NKVRAM);
+        ConnectorService.register(TConnectorService.NKV, 'LocalStorage', NKVLocalStorage);
     }
 }

package/src/subsystems/IO/Storage.service/connectors/LocalStorage.class.ts

@@ -11,6 +11,7 @@ import { SecureConnector } from '@sre/Security/SecureConnector.class';
 import fs, { existsSync } from 'fs';
 import os from 'os';
 import path from 'path';
+import { findSmythPath } from '../../../..';
 
 const console = Logger('LocalStorage');
 
@@ -35,7 +36,7 @@ export class LocalStorage extends StorageConnector {
         super(_settings);
         //if (!SmythRuntime.Instance) throw new Error('SRE not initialized');
 
-        this.folder = _settings?.folder || path.join(os.homedir(), '.smyth/storage');
+        this.folder = this.findStorageFolder(_settings?.folder);
         this.initialize();
         if (!fs.existsSync(this.folder)) {
             //throw new Error('Invalid folder provided');
@@ -43,6 +44,25 @@ export class LocalStorage extends StorageConnector {
         }
     }
 
+    private findStorageFolder(folder) {
+        let _storageFolder = folder;
+
+        if (fs.existsSync(_storageFolder)) {
+            return _storageFolder;
+        }
+
+        _storageFolder = findSmythPath('storage');
+
+        if (fs.existsSync(_storageFolder)) {
+            console.warn('Using alternative storage folder found in : ', _storageFolder);
+            return _storageFolder;
+        }
+
+        console.warn('!!! All attempts to find an existing storage folder failed !!!');
+        console.warn('!!! I will use this folder: ', _storageFolder);
+        return _storageFolder;
+    }
+
     /**
      * Reads an object from the local storage.
      *

package/src/subsystems/IO/VectorDB.service/connectors/MilvusVectorDB.class.ts

@@ -45,10 +45,13 @@ export class MilvusVectorDB extends VectorDBConnector {
 
     constructor(protected _settings: MilvusConfig) {
         super(_settings);
+        if (!_settings.credentials) {
+            return;
+        }
 
         // Create client config based on credential type
         const clientConfig = {
-            address: _settings.credentials.address,
+            address: _settings.credentials?.address,
             token: 'token' in _settings.credentials ? _settings.credentials.token : undefined,
             user: 'user' in _settings.credentials ? _settings.credentials.user : undefined,
             password: 'password' in _settings.credentials ? _settings.credentials.password : undefined,
@@ -60,7 +63,8 @@ export class MilvusVectorDB extends VectorDBConnector {
         console.info('Milvus client initialized');
         this.accountConnector = ConnectorService.getAccountConnector();
         this.cache = ConnectorService.getCacheConnector();
-        if (!_settings.embeddings.dimensions) _settings.embeddings.dimensions = 1024;
+        if (!_settings.embeddings.params) _settings.embeddings.params = { dimensions: 1024 };
+        if (!_settings.embeddings.params?.dimensions) _settings.embeddings.params.dimensions = 1024;
 
         this.embedder = EmbeddingsFactory.create(_settings.embeddings.provider, _settings.embeddings);
 

package/src/subsystems/IO/VectorDB.service/connectors/PineconeVectorDB.class.ts

@@ -4,7 +4,7 @@ import { IAccessCandidate, IACL, TAccessLevel, TAccessRole } from '@sre/types/AC
 import { AccessRequest } from '@sre/Security/AccessControl/AccessRequest.class';
 import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
 import { SecureConnector } from '@sre/Security/SecureConnector.class';
-import { VectorDBConnector } from '../VectorDBConnector';
+import { VectorDBConnector, DeleteTarget } from '../VectorDBConnector';
 import {
     DatasourceDto,
     IStorageVectorDataSource,
@@ -72,7 +72,8 @@ export class PineconeVectorDB extends VectorDBConnector {
         this.accountConnector = ConnectorService.getAccountConnector();
         this.cache = ConnectorService.getCacheConnector();
         this.nkvConnector = ConnectorService.getNKVConnector();
-        if (!_settings.embeddings.dimensions) _settings.embeddings.dimensions = 1024;
+        if (!_settings.embeddings.params) _settings.embeddings.params = { dimensions: 1024 };
+        if (!_settings.embeddings.params?.dimensions) _settings.embeddings.params.dimensions = 1024;
 
         this.embedder = EmbeddingsFactory.create(_settings.embeddings.provider, _settings.embeddings);
     }
@@ -176,18 +177,21 @@ export class PineconeVectorDB extends VectorDBConnector {
             includeValues: true,
         });
 
-        let matches: VectorsResultData = [];
+        let matches = [];
 
         for (const match of results.matches) {
             if (match.metadata?.isSkeletonVector) continue;
 
-            if (match.metadata?.user) {
-                match.metadata.user = JSONContentHelper.create(match.metadata.user.toString()).tryParse();
+            if (match.metadata?.[this.USER_METADATA_KEY]) {
+                match.metadata[this.USER_METADATA_KEY] = JSONContentHelper.create(match.metadata[this.USER_METADATA_KEY].toString()).tryParse();
             }
+
             matches.push({
                 id: match.id,
                 values: match.values,
-                metadata: match.metadata,
+                text: match.metadata?.text as string | undefined,
+                metadata: match.metadata?.[this.USER_METADATA_KEY] as Record<string, any> | undefined,
+                score: match.score,
             });
         }
 
@@ -236,14 +240,19 @@ export class PineconeVectorDB extends VectorDBConnector {
     }
 
     @SecureConnector.AccessControl
-    protected async delete(acRequest: AccessRequest, namespace: string, id: string | string[]): Promise<void> {
-        const _ids = Array.isArray(id) ? id : [id];
-        //const teamId = await this.accountConnector.getCandidateTeam(acRequest.candidate);
-
-        const res = await this.client
-            .Index(this.indexName)
-            .namespace(this.constructNsName(acRequest.candidate as AccessCandidate, namespace))
-            .deleteMany(_ids);
+    protected async delete(acRequest: AccessRequest, namespace: string, deleteTarget: DeleteTarget): Promise<void> {
+        const isDeleteByFilter = typeof deleteTarget === 'object';
+
+        if (isDeleteByFilter) {
+            // TODO: handle delete by filter logic
+        } else {
+            const _ids = Array.isArray(deleteTarget) ? deleteTarget : [deleteTarget];
+
+            const res = await this.client
+                .Index(this.indexName)
+                .namespace(this.constructNsName(acRequest.candidate as AccessCandidate, namespace))
+                .deleteMany(_ids);
+        }
     }
 
     @SecureConnector.AccessControl
@@ -257,6 +266,7 @@ export class PineconeVectorDB extends VectorDBConnector {
             chunkSize: datasource.chunkSize,
             chunkOverlap: datasource.chunkOverlap,
         });
+        const label = datasource.label || 'Untitled';
         const ids = Array.from({ length: chunkedText.length }, (_, i) => `${dsId}_${crypto.randomUUID()}`);
         const source: IVectorDataSourceDto[] = chunkedText.map((doc, i) => {
             return {
@@ -266,7 +276,8 @@ export class PineconeVectorDB extends VectorDBConnector {
                     acl: acl.serializedACL,
                     namespaceId: formattedNs,
                     datasourceId: dsId,
-                    user: datasource.metadata ? jsonrepair(JSON.stringify(datasource.metadata)) : undefined,
+                    datasourceLabel: label,
+                    user_metadata: datasource.metadata ? jsonrepair(JSON.stringify(datasource.metadata)) : undefined,
                 },
             };
         });

package/src/subsystems/IO/VectorDB.service/connectors/RAMVecrtorDB.class.ts

@@ -64,9 +64,9 @@ export class RAMVectorDB extends VectorDBConnector {
         this.accountConnector = ConnectorService.getAccountConnector();
 
         if (!_settings.embeddings) {
-            _settings.embeddings = { provider: 'OpenAI', model: 'text-embedding-3-large' };
+            _settings.embeddings = { provider: 'OpenAI', model: 'text-embedding-3-large', params: { dimensions: 1024 } };
         }
-        if (!_settings.embeddings.dimensions) _settings.embeddings.dimensions = 1024;
+        if (!_settings.embeddings.params?.dimensions) _settings.embeddings.params.dimensions = 1024;
 
         this.embedder = EmbeddingsFactory.create(_settings.embeddings.provider, _settings.embeddings);
     }
@@ -90,7 +90,7 @@ export class RAMVectorDB extends VectorDBConnector {
         const preparedNs = this.constructNsName(acRequest.candidate as AccessCandidate, namespace);
 
         if (!this.namespaces[preparedNs]) {
-            const nsData: IStorageVectorNamespace = {
+            const nsData = {
                 namespace: preparedNs,
                 displayName: namespace,
                 candidateId: acRequest.candidate.id,
@@ -178,7 +178,7 @@ export class RAMVectorDB extends VectorDBConnector {
 
         // Search in namespace data
         const namespaceData = this.vectors[preparedNs] || [];
-        const results: Array<{ id: string; score: number; values: number[]; metadata?: any }> = [];
+        const results: Array<{ id: string; score: number; values: number[]; metadata?: any; text: string }> = [];
 
         for (const vector of namespaceData) {
             const similarity = this.cosineSimilarity(queryVector as number[], vector.values);
@@ -187,6 +187,7 @@ export class RAMVectorDB extends VectorDBConnector {
                 score: similarity,
                 values: vector.values,
                 metadata: options.includeMetadata ? vector.metadata : undefined,
+                text: vector.metadata?.text,
             });
         }
 
@@ -228,7 +229,7 @@ export class RAMVectorDB extends VectorDBConnector {
             const vectorData: VectorData = {
                 id: source.id,
                 values: vector,
-                datasource: source.metadata?.datasource || 'unknown',
+                datasource: source.metadata?.datasourceId || 'unknown',
                 metadata: source.metadata,
             };
 

package/src/subsystems/IO/VectorDB.service/embed/BaseEmbedding.ts

@@ -6,14 +6,16 @@ import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.cla
 export type TEmbeddings = {
     provider: SupportedProviders;
     model: SupportedModels[SupportedProviders];
-    dimensions?: number;
-    timeout?: number;
-    chunkSize?: number;
-    stripNewLines?: boolean;
-    maxConcurrency?: number;
+
     credentials?: {
         apiKey: string;
     };
+    params?: {
+        dimensions?: number;
+        timeout?: number;
+        chunkSize?: number;
+        stripNewLines?: boolean;
+    };
 };
 
 type SupportedSources = 'text' | 'vector' | 'url';
@@ -25,14 +27,13 @@ export abstract class BaseEmbedding {
     stripNewLines = true;
     dimensions?: number;
     timeout?: number;
-    maxConcurrency?: number;
 
     constructor(fields?: Partial<TEmbeddings>) {
         this.model = fields?.model ?? this.model;
-        this.chunkSize = fields?.chunkSize ?? this.chunkSize;
-        this.stripNewLines = fields?.stripNewLines ?? this.stripNewLines;
-        this.timeout = fields?.timeout;
-        this.dimensions = fields?.dimensions;
+        this.chunkSize = fields?.params?.chunkSize ?? this.chunkSize;
+        this.stripNewLines = fields?.params?.stripNewLines ?? this.stripNewLines;
+        this.timeout = fields?.params?.timeout;
+        this.dimensions = fields?.params?.dimensions;
     }
 
     /**

package/src/subsystems/IO/VectorDB.service/embed/OpenAIEmbedding.ts

@@ -28,7 +28,7 @@ export class OpenAIEmbeds extends BaseEmbedding {
     public canSpecifyDimensions = true;
 
     constructor(private settings?: Partial<TEmbeddings>) {
-        super({ maxConcurrency: 2, model: settings?.model ?? DEFAULT_MODEL, ...settings });
+        super({ model: settings?.model ?? DEFAULT_MODEL, ...settings });
 
         this.clientConfig = {
             //apiKey: fields?.credentials?.apiKey || process.env.OPENAI_API_KEY || '',

package/src/subsystems/LLMManager/LLM.service/LLMConnector.ts

@@ -296,7 +296,8 @@ export abstract class LLMConnector extends Connector {
         //_params.model = (await modelProviderCandidate.getModelId(model)) || model;
 
         _params.baseURL = modelInfo?.baseURL;
-        if (!isStandardLLM) _params.modelInfo = modelInfo as TCustomLLMModel; //only if custom LLM ?
+        // if (!isStandardLLM) _params.modelInfo = modelInfo as TCustomLLMModel; //only if custom LLM ?
+        _params.modelInfo = modelInfo as TCustomLLMModel; // We need model info for both standard and custom LLMs
 
         if (_params.maxTokens) {
             _params.maxTokens = await modelProviderCandidate.adjustMaxCompletionTokens(model, _params.maxTokens, _params?.isUserKey as boolean);

package/src/subsystems/Security/Vault.service/connectors/JSONFileVault.class.ts

@@ -13,6 +13,7 @@ import crypto from 'crypto';
 import fs from 'fs';
 import * as readlineSync from 'readline-sync';
 import path from 'path';
+import { findSmythPath } from '../../../../helpers/Sysconfig.helper';
 
 const console = Logger('JSONFileVault');
 
@@ -57,6 +58,8 @@ export class JSONFileVault extends VaultConnector {
                     this.vaultData = JSON.parse(fs.readFileSync(vaultFile).toString());
                 }
             } catch (e) {
+                console.error('Error parsing vault file:', e);
+                console.error('!!! Vault features might not work properly !!!');
                 this.vaultData = {};
             }
 
@@ -82,24 +85,21 @@ export class JSONFileVault extends VaultConnector {
         if (fs.existsSync(_vaultFile)) {
             return _vaultFile;
         }
-        console.warn('Vault file not found in:', _vaultFile, 'trying to find in local .smyth directory');
+        console.warn('Vault file not found in:', _vaultFile);
 
-        //try local directory
-        _vaultFile = path.join(process.cwd(), '.smyth', '.sre', 'vault.json');
-        if (fs.existsSync(_vaultFile)) {
-            console.warn('Using alternative vault file found in : ', _vaultFile);
-            return _vaultFile;
-        }
+        //try to find the .smyth directory and check if it contains a valid vault
+
+        _vaultFile = findSmythPath('.sre/vault.json', (dir, success, nextDir) => {
+            if (!success) {
+                console.warn('Vault file not found in:', nextDir);
+            }
+        });
 
-        console.warn('Vault file not found in:', _vaultFile, 'trying to find in user home directory');
-        //try to find the vault file in the .smyth directory
-        _vaultFile = path.join(os.homedir(), '.smyth', '.sre', 'vault.json');
         if (fs.existsSync(_vaultFile)) {
             console.warn('Using alternative vault file found in : ', _vaultFile);
             return _vaultFile;
         }
 
-        console.warn('Vault file not found in:', _vaultFile);
         console.warn('!!! All attempts to find the vault file failed !!!');
         console.warn('!!! Will continue without vault !!!');
         console.warn('!!! Many features might not work !!!');

package/src/types/VectorDB.types.ts

@@ -1,10 +1,18 @@
-export type VectorDBMetadata = ({ text?: string; user?: string } & Record<string, string>) | undefined;
+export type VectorDBMetadata = {
+    namespaceId: string;
+    datasourceId: string;
+    datasourceLabel: string;
+    acl: string;
+    user_metadata?: string;
+    text?: string;
+};
 
 export type VectorsResultData = {
     id: string;
     score?: number;
     values: number[];
-    metadata?: ({ text?: string; user?: Record<string, string> } & Record<string, any>) | undefined;
+    text: string;
+    metadata?: Record<string, any>;
 }[];
 
 export interface NsKnownMetadata {
@@ -46,21 +54,21 @@ export interface IVectorDataSourceDto {
 export interface IStorageVectorDataSource {
     namespaceId: string;
     // indexName: string;
-    candidateId: string;
-    candidateRole: string;
     name: string;
     metadata: string;
-    text: string;
+    text?: string;
     vectorIds: string[];
     id: string;
+    candidateId: string;
+    candidateRole: string;
 }
 
 export interface IStorageVectorNamespace {
     namespace: string;
     displayName: string;
+    metadata?: StorageVectorNamespaceMetadata;
     candidateId: string;
     candidateRole: string;
-    metadata?: StorageVectorNamespaceMetadata;
 }
 
 export type StorageVectorNamespaceMetadata = Partial<PineconeNamespaceMetadata> & { isOnCustomStorage?: boolean } & { [key: string]: any };