@smythos/sre 1.7.1 → 1.7.5

package/src/subsystems/IO/VectorDB.service/connectors/MilvusVectorDB.class.ts

@@ -9,8 +9,14 @@ import { ACL } from '@sre/Security/AccessControl/ACL.class';
 import { AccountConnector } from '@sre/Security/Account.service/AccountConnector';
 import { SecureConnector } from '@sre/Security/SecureConnector.class';
 import { IAccessCandidate, IACL, TAccessLevel } from '@sre/types/ACL.types';
-import { DatasourceDto, IStorageVectorDataSource, IVectorDataSourceDto, QueryOptions, VectorsResultData } from '@sre/types/VectorDB.types';
-import { chunkText } from '@sre/utils/string.utils';
+import {
+    DatasourceDto,
+    IStorageVectorDataSource,
+    IVectorDataSourceDto,
+    QueryOptions,
+    VectorDBResult,
+    VectorsResultData,
+} from '@sre/types/VectorDB.types';
 import { CreateIndexSimpleReq, DataType, ErrorCode, FieldType, MilvusClient } from '@zilliz/milvus2-sdk-node';
 import crypto from 'crypto';
 import { jsonrepair } from 'jsonrepair';
@@ -72,10 +78,10 @@ export class MilvusVectorDB extends VectorDBConnector {
         this.cache = ConnectorService.getCacheConnector();
 
         if (!_settings.embeddings) {
-            _settings.embeddings = { provider: 'OpenAI', model: 'text-embedding-3-large', params: { dimensions: 1024 } };
+            _settings.embeddings = { provider: 'OpenAI', model: 'text-embedding-3-large', dimensions: 1024 };
         }
-        if (!_settings.embeddings.params) _settings.embeddings.params = { dimensions: 1024 };
-        if (!_settings.embeddings.params?.dimensions) _settings.embeddings.params.dimensions = 1024;
+
+        if (!_settings.embeddings.dimensions) _settings.embeddings.dimensions = 1024;
 
         this.embedder = EmbeddingsFactory.create(_settings.embeddings.provider, _settings.embeddings);
 
@@ -231,7 +237,7 @@ export class MilvusVectorDB extends VectorDBConnector {
         acRequest: AccessRequest,
         namespace: string,
         sourceWrapper: IVectorDataSourceDto | IVectorDataSourceDto[]
-    ): Promise<string[]> {
+    ): Promise<VectorDBResult[]> {
         //const teamId = await this.accountConnector.getCandidateTeam(acRequest.candidate);
         sourceWrapper = Array.isArray(sourceWrapper) ? sourceWrapper : [sourceWrapper];
         const preparedNs = this.constructNsName(acRequest.candidate as AccessCandidate, namespace);
@@ -264,7 +270,18 @@ export class MilvusVectorDB extends VectorDBConnector {
             throw new Error(`Error inserting data: ${res?.status?.error_code}`);
         }
 
-        return preparedSource.map((s) => s.id);
+        return preparedSource.map((s) => {
+            const { text, acl, user_metadata, ...restMetadata } = s || {};
+            return {
+                id: s.id,
+                values: s.vector as number[],
+                text: text as string,
+                metadata: {
+                    ...restMetadata,
+                    ...((typeof user_metadata === 'string' ? JSON.parse(user_metadata) : user_metadata) as Record<string, any>),
+                },
+            };
+        });
     }
 
     @SecureConnector.AccessControl
@@ -306,7 +323,7 @@ export class MilvusVectorDB extends VectorDBConnector {
         const dsId = datasource.id || crypto.randomUUID();
 
         const formattedNs = this.constructNsName(acRequest.candidate as AccessCandidate, namespace);
-        const chunkedText = chunkText(datasource.text, {
+        const chunkedText = this.embedder.chunkText(datasource.text, {
             chunkSize: datasource.chunkSize,
             chunkOverlap: datasource.chunkOverlap,
         });
@@ -328,16 +345,20 @@ export class MilvusVectorDB extends VectorDBConnector {
 
         const _vIds = await this.insert(acRequest, namespace, source);
 
-        return {
+        const dsData: IStorageVectorDataSource = {
             namespaceId: formattedNs,
             candidateId: acRequest.candidate.id,
             candidateRole: acRequest.candidate.role,
             name: label,
             metadata: datasource.metadata ? jsonrepair(JSON.stringify(datasource.metadata)) : undefined,
             text: datasource.text,
-            vectorIds: _vIds,
+            vectorIds: _vIds.map((v) => v.id),
             id: dsId,
         };
+        if (datasource.returnFullVectorInfo) {
+            dsData.vectorInfo = _vIds;
+        }
+        return dsData;
     }
 
     @SecureConnector.AccessControl

package/src/subsystems/IO/VectorDB.service/connectors/PineconeVectorDB.class.ts

@@ -11,6 +11,7 @@ import {
     IStorageVectorNamespace,
     IVectorDataSourceDto,
     QueryOptions,
+    VectorDBResult,
     VectorsResultData,
 } from '@sre/types/VectorDB.types';
 import { Pinecone } from '@pinecone-database/pinecone';
@@ -23,7 +24,7 @@ import { CacheConnector } from '@sre/MemoryManager/Cache.service/CacheConnector'
 import crypto from 'crypto';
 import { BaseEmbedding, TEmbeddings } from '../embed/BaseEmbedding';
 import { EmbeddingsFactory, SupportedProviders, SupportedModels } from '../embed';
-import { chunkText } from '@sre/utils/string.utils';
+
 import { jsonrepair } from 'jsonrepair';
 
 const console = Logger('Pinecone VectorDB');
@@ -73,10 +74,9 @@ export class PineconeVectorDB extends VectorDBConnector {
         this.cache = ConnectorService.getCacheConnector();
         this.nkvConnector = ConnectorService.getNKVConnector();
         if (!_settings.embeddings) {
-            _settings.embeddings = { provider: 'OpenAI', model: 'text-embedding-3-large', params: { dimensions: 1024 } };
+            _settings.embeddings = { provider: 'OpenAI', model: 'text-embedding-3-large', dimensions: 1024 };
         }
-        if (!_settings.embeddings.params) _settings.embeddings.params = { dimensions: 1024 };
-        if (!_settings.embeddings.params?.dimensions) _settings.embeddings.params.dimensions = 1024;
+        if (!_settings.embeddings.dimensions) _settings.embeddings.dimensions = 1024;
 
         this.embedder = EmbeddingsFactory.create(_settings.embeddings.provider, _settings.embeddings);
     }
@@ -189,11 +189,14 @@ export class PineconeVectorDB extends VectorDBConnector {
                 match.metadata[this.USER_METADATA_KEY] = JSONContentHelper.create(match.metadata[this.USER_METADATA_KEY].toString()).tryParse();
             }
 
+            const text = match.metadata?.text as string | undefined;
+            delete match.metadata?.text; // delete the text metadata to avoid duplication in case we returned the default raw metadata
+
             matches.push({
                 id: match.id,
                 values: match.values,
-                text: match.metadata?.text as string | undefined,
-                metadata: match.metadata?.[this.USER_METADATA_KEY] as Record<string, any> | undefined,
+                text: text,
+                metadata: match.metadata?.[this.USER_METADATA_KEY] || match.metadata, // fallback to the default metadata if the user metadata is not present, this is for backward compatibility
                 score: match.score,
             });
         }
@@ -207,7 +210,7 @@ export class PineconeVectorDB extends VectorDBConnector {
         acRequest: AccessRequest,
         namespace: string,
         sourceWrapper: IVectorDataSourceDto | IVectorDataSourceDto[]
-    ): Promise<string[]> {
+    ): Promise<VectorDBResult[]> {
         //const teamId = await this.accountConnector.getCandidateTeam(acRequest.candidate);
         sourceWrapper = Array.isArray(sourceWrapper) ? sourceWrapper : [sourceWrapper];
 
@@ -239,7 +242,18 @@ export class PineconeVectorDB extends VectorDBConnector {
             await this.setACL(acRequest, namespace, acl);
         }
 
-        return preparedSource.map((s) => s.id);
+        return preparedSource.map((s) => {
+            const { text, acl, user_metadata, ...restMetadata } = s.metadata || {};
+            return {
+                id: s.id,
+                values: s.values as number[],
+                text: text as string,
+                metadata: {
+                    ...restMetadata,
+                    ...((typeof user_metadata === 'string' ? JSON.parse(user_metadata) : user_metadata) as Record<string, any>),
+                },
+            };
+        });
     }
 
     @SecureConnector.AccessControl
@@ -265,7 +279,7 @@ export class PineconeVectorDB extends VectorDBConnector {
         const dsId = datasource.id || crypto.randomUUID();
 
         const formattedNs = this.constructNsName(acRequest.candidate as AccessCandidate, namespace);
-        const chunkedText = chunkText(datasource.text, {
+        const chunkedText = this.embedder.chunkText(datasource.text, {
             chunkSize: datasource.chunkSize,
             chunkOverlap: datasource.chunkOverlap,
         });
@@ -294,9 +308,12 @@ export class PineconeVectorDB extends VectorDBConnector {
             name: datasource.label || 'Untitled',
             metadata: datasource.metadata ? jsonrepair(JSON.stringify(datasource.metadata)) : undefined,
             text: datasource.text,
-            vectorIds: _vIds,
+            vectorIds: _vIds.map((v) => v.id),
             id: dsId,
         };
+        if (datasource.returnFullVectorInfo) {
+            dsData.vectorInfo = _vIds;
+        }
         // const url = `smythfs://${teamId}.team/_datasources/${dsId}.json`;
         // await SmythFS.Instance.write(url, JSON.stringify(dsData), AccessCandidate.team(teamId));
         await this.nkvConnector

package/src/subsystems/IO/VectorDB.service/connectors/RAMVecrtorDB.class.ts

@@ -12,6 +12,7 @@ import {
     IStorageVectorNamespace,
     IVectorDataSourceDto,
     QueryOptions,
+    VectorDBResult,
     VectorsResultData,
 } from '@sre/types/VectorDB.types';
 import { ConnectorService } from '@sre/Core/ConnectorsService';
@@ -21,7 +22,7 @@ import { OpenAIEmbeds } from '@sre/IO/VectorDB.service/embed/OpenAIEmbedding';
 import crypto from 'crypto';
 import { BaseEmbedding, TEmbeddings } from '../embed/BaseEmbedding';
 import { EmbeddingsFactory } from '../embed';
-import { chunkText } from '@sre/utils/string.utils';
+
 import { jsonrepair } from 'jsonrepair';
 
 const console = Logger('RAM VectorDB');
@@ -71,10 +72,10 @@ export class RAMVectorDB extends VectorDBConnector {
         this.accountConnector = ConnectorService.getAccountConnector();
 
         if (!_settings.embeddings) {
-            _settings.embeddings = { provider: 'OpenAI', model: 'text-embedding-3-large', params: { dimensions: 1024 } };
+            _settings.embeddings = { provider: 'OpenAI', model: 'text-embedding-3-large', dimensions: 1024 };
         }
-        if (!_settings.embeddings.params) _settings.embeddings.params = { dimensions: 1024 };
-        if (!_settings.embeddings.params?.dimensions) _settings.embeddings.params.dimensions = 1024;
+
+        if (!_settings.embeddings.dimensions) _settings.embeddings.dimensions = 1024;
 
         this.embedder = EmbeddingsFactory.create(_settings.embeddings.provider, _settings.embeddings);
     }
@@ -225,7 +226,7 @@ export class RAMVectorDB extends VectorDBConnector {
         acRequest: AccessRequest,
         namespace: string,
         sourceWrapper: IVectorDataSourceDto | IVectorDataSourceDto[]
-    ): Promise<string[]> {
+    ): Promise<VectorDBResult[]> {
         //const teamId = await this.accountConnector.getCandidateTeam(acRequest.candidate);
         const preparedNs = this.constructNsName(acRequest.candidate as AccessCandidate, namespace);
 
@@ -245,7 +246,7 @@ export class RAMVectorDB extends VectorDBConnector {
             RAMVectorDB.vectors[preparedNs] = [];
         }
 
-        const insertedIds: string[] = [];
+        const insertedIds: VectorDBResult[] = [];
 
         for (const source of transformedSource) {
             const vectorData: VectorData = {
@@ -263,7 +264,18 @@ export class RAMVectorDB extends VectorDBConnector {
                 RAMVectorDB.vectors[preparedNs].push(vectorData);
             }
 
-            insertedIds.push(source.id);
+            const { text, acl, user_metadata, ...restMetadata } = source.metadata || {};
+
+            (insertedIds as VectorDBResult[]).push({
+                id: source.id,
+                values: source.source as number[],
+                text: text as string,
+                metadata: {
+                    ...restMetadata,
+                    ...((typeof user_metadata === 'string' ? JSON.parse(user_metadata) : user_metadata) as Record<string, any>),
+                },
+            });
+            //insertedIds.push(source.id);
         }
 
         return insertedIds;
@@ -303,7 +315,7 @@ export class RAMVectorDB extends VectorDBConnector {
         const dsId = datasource.id || crypto.randomUUID();
 
         const formattedNs = this.constructNsName(acRequest.candidate as AccessCandidate, namespace);
-        const chunkedText = chunkText(datasource.text, {
+        const chunkedText = this.embedder.chunkText(datasource.text, {
             chunkSize: datasource.chunkSize,
             chunkOverlap: datasource.chunkOverlap,
         });
@@ -332,10 +344,14 @@ export class RAMVectorDB extends VectorDBConnector {
             name: datasource.label || 'Untitled',
             metadata: datasource.metadata ? jsonrepair(JSON.stringify(datasource.metadata)) : undefined,
             text: datasource.text,
-            vectorIds: _vIds,
+            vectorIds: _vIds.map((v) => v.id),
             id: dsId,
         };
 
+        if (datasource.returnFullVectorInfo) {
+            dsData.vectorInfo = _vIds;
+        }
+
         // Store datasource metadata in memory
         if (!RAMVectorDB.datasources[formattedNs]) {
             RAMVectorDB.datasources[formattedNs] = {};

package/src/subsystems/IO/VectorDB.service/embed/BaseEmbedding.ts

@@ -4,18 +4,21 @@ import { SupportedProviders, SupportedModels } from './index';
 import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
 
 export type TEmbeddings = {
-    provider: SupportedProviders;
-    model: SupportedModels[SupportedProviders];
+    provider?: SupportedProviders;
+    model?: SupportedModels[SupportedProviders];
 
     credentials?: {
         apiKey: string;
     };
-    params?: {
-        dimensions?: number;
-        timeout?: number;
-        chunkSize?: number;
-        stripNewLines?: boolean;
-    };
+
+    dimensions?: number;
+    timeout?: number;
+    chunkSize?: number;
+    chunkOverlap?: number;
+    batchSize?: number;
+    stripNewLines?: boolean;
+
+    params?: any;
 };
 
 type SupportedSources = 'text' | 'vector' | 'url';
@@ -24,16 +27,19 @@ export abstract class BaseEmbedding {
     model: string;
     modelName: string;
     chunkSize = 512;
+    chunkOverlap = 100;
     stripNewLines = true;
     dimensions?: number;
     timeout?: number;
+    batchSize = 10;
 
     constructor(fields?: Partial<TEmbeddings>) {
         this.model = fields?.model ?? this.model;
-        this.chunkSize = fields?.params?.chunkSize ?? this.chunkSize;
-        this.stripNewLines = fields?.params?.stripNewLines ?? this.stripNewLines;
-        this.timeout = fields?.params?.timeout;
-        this.dimensions = fields?.params?.dimensions;
+        this.chunkSize = fields?.chunkSize || fields?.params?.chunkSize || this.chunkSize;
+        this.chunkOverlap = fields?.chunkOverlap || fields?.params?.chunkOverlap || this.chunkOverlap;
+        this.stripNewLines = fields?.stripNewLines || fields?.params?.stripNewLines || this.stripNewLines;
+        this.timeout = fields?.timeout || fields?.params?.timeout;
+        this.dimensions = fields?.dimensions || fields?.params?.dimensions;
     }
 
     /**
@@ -58,6 +64,13 @@ export abstract class BaseEmbedding {
         }, [] as T[][]);
     }
 
+    public chunkText(text: string, { chunkSize, chunkOverlap }: { chunkSize?: number; chunkOverlap?: number }): string[] {
+        const textSplitter = new TextSplitter({
+            chunkSize: chunkSize || this.chunkSize,
+            chunkOverlap: chunkOverlap || this.chunkOverlap,
+        });
+        return textSplitter.splitText(text);
+    }
     /**
      * Utility method to process multiple texts based on stripNewLines setting
      */
@@ -105,3 +118,160 @@ export abstract class BaseEmbedding {
             .concat([1]);
     }
 }
+
+class TextSplitter {
+    private chunkSize: number;
+    private chunkOverlap: number;
+    private separators: string[] = ['\n\n', '\n', ' ', ''];
+    private keepSeparator: boolean = true;
+
+    constructor({
+        chunkSize = 1000,
+        chunkOverlap = 200,
+        separators,
+        keepSeparator,
+    }: {
+        chunkSize?: number;
+        chunkOverlap?: number;
+        separators?: string[];
+        keepSeparator?: boolean;
+    } = {}) {
+        this.chunkSize = chunkSize;
+        this.chunkOverlap = chunkOverlap;
+
+        if (separators) {
+            this.separators = separators;
+        }
+
+        if (keepSeparator !== undefined) {
+            this.keepSeparator = keepSeparator;
+        }
+
+        if (this.chunkOverlap >= this.chunkSize) {
+            throw new Error('Cannot have chunkOverlap >= chunkSize');
+        }
+    }
+
+    public splitText(text: string): string[] {
+        return this._splitText(text, this.separators);
+    }
+
+    private _splitText(text: string, separators: string[]): string[] {
+        const finalChunks: string[] = [];
+
+        // Get appropriate separator to use
+        let separator: string = separators[separators.length - 1];
+        let newSeparators: string[] | undefined;
+
+        for (let i = 0; i < separators.length; i += 1) {
+            const s = separators[i];
+            if (s === '') {
+                separator = s;
+                break;
+            }
+            if (text.includes(s)) {
+                separator = s;
+                newSeparators = separators.slice(i + 1);
+                break;
+            }
+        }
+
+        // Split the text using the identified separator
+        const splits = this.splitOnSeparator(text, separator);
+
+        // Process splits, recursively splitting longer texts
+        let goodSplits: string[] = [];
+        const _separator = this.keepSeparator ? '' : separator;
+
+        for (const s of splits) {
+            if (this.lengthFunction(s) < this.chunkSize) {
+                goodSplits.push(s);
+            } else {
+                if (goodSplits.length) {
+                    const mergedText = this.mergeSplits(goodSplits, _separator);
+                    finalChunks.push(...mergedText);
+                    goodSplits = [];
+                }
+
+                if (!newSeparators) {
+                    finalChunks.push(s);
+                } else {
+                    const otherInfo = this._splitText(s, newSeparators);
+                    finalChunks.push(...otherInfo);
+                }
+            }
+        }
+
+        if (goodSplits.length) {
+            const mergedText = this.mergeSplits(goodSplits, _separator);
+            finalChunks.push(...mergedText);
+        }
+
+        return finalChunks;
+    }
+
+    private splitOnSeparator(text: string, separator: string): string[] {
+        let splits: string[];
+
+        if (separator) {
+            if (this.keepSeparator) {
+                const regexEscapedSeparator = separator.replace(/[/\-\\^$*+?.()|[\]{}]/g, '\\$&');
+                splits = text.split(new RegExp(`(?=${regexEscapedSeparator})`));
+            } else {
+                splits = text.split(separator);
+            }
+        } else {
+            splits = text.split('');
+        }
+
+        return splits.filter((s) => s !== '');
+    }
+
+    private lengthFunction(text: string): number {
+        return text.length;
+    }
+
+    private joinDocs(docs: string[], separator: string): string | null {
+        const text = docs.join(separator).trim();
+        return text === '' ? null : text;
+    }
+
+    private mergeSplits(splits: string[], separator: string): string[] {
+        const docs: string[] = [];
+        const currentDoc: string[] = [];
+        let total = 0;
+
+        for (const d of splits) {
+            const _len = this.lengthFunction(d);
+
+            if (total + _len + currentDoc.length * separator.length > this.chunkSize) {
+                if (total > this.chunkSize) {
+                    console.warn(`Created a chunk of size ${total}, which is longer than the specified ${this.chunkSize}`);
+                }
+
+                if (currentDoc.length > 0) {
+                    const doc = this.joinDocs(currentDoc, separator);
+                    if (doc !== null) {
+                        docs.push(doc);
+                    }
+
+                    // Keep popping if conditions are met
+                    while (total > this.chunkOverlap || (total + _len + currentDoc.length * separator.length > this.chunkSize && total > 0)) {
+                        total -= this.lengthFunction(currentDoc[0]);
+                        currentDoc.shift();
+                    }
+                }
+            }
+
+            currentDoc.push(d);
+            total += _len;
+        }
+
+        const doc = this.joinDocs(currentDoc, separator);
+        if (doc !== null) {
+            docs.push(doc);
+        }
+
+        return docs;
+    }
+}

package/src/subsystems/IO/VectorDB.service/embed/GoogleEmbedding.ts

@@ -18,7 +18,7 @@ export class GoogleEmbeds extends BaseEmbedding {
     }
 
     async embedTexts(texts: string[], candidate: AccessCandidate): Promise<number[][]> {
-        const batches = this.chunkArr(this.processTexts(texts), this.chunkSize);
+        const batches = this.chunkArr(this.processTexts(texts), this.batchSize);
 
         const batchRequests = batches.map((batch) => {
             return this.embed(batch, candidate);

package/src/subsystems/IO/VectorDB.service/embed/OpenAIEmbedding.ts

@@ -41,7 +41,7 @@ export class OpenAIEmbeds extends BaseEmbedding {
     }
 
     async embedTexts(texts: string[], candidate: AccessCandidate): Promise<number[][]> {
-        const batches = this.chunkArr(this.processTexts(texts), this.chunkSize);
+        const batches = this.chunkArr(this.processTexts(texts), this.batchSize);
 
         const batchRequests = batches.map((batch) => {
             const params: OpenAIClient.EmbeddingCreateParams = {

package/src/subsystems/IO/VectorDB.service/embed/index.ts

@@ -1,6 +1,7 @@
 import { OpenAIEmbeds } from './OpenAIEmbedding';
 import { GoogleEmbeds } from './GoogleEmbedding';
 import { TEmbeddings } from './BaseEmbedding';
+import { TLLMModel } from '@sre/types/LLM.types';
 
 // a factory to get the correct embedding provider based on the provider name
 const supportedProviders = {
@@ -20,7 +21,16 @@ export type SupportedModels = {
 };
 
 export class EmbeddingsFactory {
-    public static create(provider: SupportedProviders, config: TEmbeddings) {
-        return new supportedProviders[provider].embedder(config);
+    public static create(provider?: SupportedProviders, config?: TEmbeddings & { model?: SupportedModels[SupportedProviders] | TLLMModel }) {
+        if (!provider) provider = 'OpenAI';
+        if (!config) config = { provider: 'OpenAI', model: 'text-embedding-3-large', dimensions: 1024 };
+
+        //if the model is a TLLMModel, we need to convert it to a SupportedModels[SupportedProviders]
+        if (config.model && typeof config.model === 'object') {
+            provider = (config.model as TLLMModel).provider as SupportedProviders;
+            config.model = (config.model as TLLMModel).modelId;
+        }
+
+        return new supportedProviders[provider as SupportedProviders].embedder(config);
     }
 }