@smythos/sre 1.5.44 → 1.5.45

package/src/subsystems/IO/VectorDB.service/embed/BaseEmbedding.ts

@@ -1,107 +1,107 @@
-import { IVectorDataSourceDto, Source } from '@sre/types/VectorDB.types';
-import { isUrl } from '@sre/utils/index';
-import { SupportedProviders, SupportedModels } from './index';
-import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
-
-export type TEmbeddings = {
-    provider: SupportedProviders;
-    model: SupportedModels[SupportedProviders];
-
-    credentials?: {
-        apiKey: string;
-    };
-    params?: {
-        dimensions?: number;
-        timeout?: number;
-        chunkSize?: number;
-        stripNewLines?: boolean;
-    };
-};
-
-type SupportedSources = 'text' | 'vector' | 'url';
-
-export abstract class BaseEmbedding {
-    model: string;
-    modelName: string;
-    chunkSize = 512;
-    stripNewLines = true;
-    dimensions?: number;
-    timeout?: number;
-
-    constructor(fields?: Partial<TEmbeddings>) {
-        this.model = fields?.model ?? this.model;
-        this.chunkSize = fields?.params?.chunkSize ?? this.chunkSize;
-        this.stripNewLines = fields?.params?.stripNewLines ?? this.stripNewLines;
-        this.timeout = fields?.params?.timeout;
-        this.dimensions = fields?.params?.dimensions;
-    }
-
-    /**
-     * Embed multiple texts and return their vector representations
-     */
-    abstract embedTexts(texts: string[], candidate: AccessCandidate): Promise<number[][]>;
-
-    /**
-     * Embed a single text and return its vector representation
-     */
-    abstract embedText(text: string, candidate: AccessCandidate): Promise<number[]>;
-
-    /**
-     * Utility method to chunk arrays into smaller batches
-     */
-    protected chunkArr<T>(arr: T[], sizePerChunk: number): T[][] {
-        return arr.reduce((chunks, elem, index) => {
-            const chunkIndex = Math.floor(index / sizePerChunk);
-            const chunk = chunks[chunkIndex] || [];
-            chunks[chunkIndex] = chunk.concat([elem]);
-            return chunks;
-        }, [] as T[][]);
-    }
-
-    /**
-     * Utility method to process multiple texts based on stripNewLines setting
-     */
-    protected processTexts(texts: string[]): string[] {
-        return this.stripNewLines ? texts.map((t) => t.replace(/\n/g, ' ')) : texts;
-    }
-
-    public detectSourceType(source: Source): SupportedSources | 'unknown' {
-        if (typeof source === 'string') {
-            return isUrl(source) ? 'url' : 'text';
-        } else if (Array.isArray(source) && source.every((v) => typeof v === 'number')) {
-            return 'vector';
-        } else {
-            return 'unknown';
-        }
-    }
-
-    public transformSource(source: IVectorDataSourceDto[], sourceType: SupportedSources, candidate: AccessCandidate) {
-        //* as the accepted sources increases, you will need to implement the strategy pattern instead of a switch case
-        switch (sourceType) {
-            case 'text': {
-                const texts = source.map((s) => s.source as string);
-
-                return this.embedTexts(texts, candidate).then((vectors) => {
-                    return source.map((s, i) => ({
-                        ...s,
-                        source: vectors[i],
-                        metadata: { ...s.metadata, text: texts[i] },
-                    }));
-                });
-            }
-            case 'vector': {
-                return source;
-            }
-        }
-    }
-
-    // this is a dummy vector used to maximize search distance
-    // we use it to create reserved vectors that store namespace data
-    public get dummyVector(): number[] {
-        //This is good for cosine similarity, but not for euclidean distance
-        //TODO: detect current similarity metric and use the appropriate vector (e.g for euclidean distance, use a vector with all large values like [1e6, 1e6, ..., 1e6])
-        return Array(this.dimensions - 1)
-            .fill(0)
-            .concat([1]);
-    }
-}
+import { IVectorDataSourceDto, Source } from '@sre/types/VectorDB.types';
+import { isUrl } from '@sre/utils/index';
+import { SupportedProviders, SupportedModels } from './index';
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+
+export type TEmbeddings = {
+    provider: SupportedProviders;
+    model: SupportedModels[SupportedProviders];
+
+    credentials?: {
+        apiKey: string;
+    };
+    params?: {
+        dimensions?: number;
+        timeout?: number;
+        chunkSize?: number;
+        stripNewLines?: boolean;
+    };
+};
+
+type SupportedSources = 'text' | 'vector' | 'url';
+
+export abstract class BaseEmbedding {
+    model: string;
+    modelName: string;
+    chunkSize = 512;
+    stripNewLines = true;
+    dimensions?: number;
+    timeout?: number;
+
+    constructor(fields?: Partial<TEmbeddings>) {
+        this.model = fields?.model ?? this.model;
+        this.chunkSize = fields?.params?.chunkSize ?? this.chunkSize;
+        this.stripNewLines = fields?.params?.stripNewLines ?? this.stripNewLines;
+        this.timeout = fields?.params?.timeout;
+        this.dimensions = fields?.params?.dimensions;
+    }
+
+    /**
+     * Embed multiple texts and return their vector representations
+     */
+    abstract embedTexts(texts: string[], candidate: AccessCandidate): Promise<number[][]>;
+
+    /**
+     * Embed a single text and return its vector representation
+     */
+    abstract embedText(text: string, candidate: AccessCandidate): Promise<number[]>;
+
+    /**
+     * Utility method to chunk arrays into smaller batches
+     */
+    protected chunkArr<T>(arr: T[], sizePerChunk: number): T[][] {
+        return arr.reduce((chunks, elem, index) => {
+            const chunkIndex = Math.floor(index / sizePerChunk);
+            const chunk = chunks[chunkIndex] || [];
+            chunks[chunkIndex] = chunk.concat([elem]);
+            return chunks;
+        }, [] as T[][]);
+    }
+
+    /**
+     * Utility method to process multiple texts based on stripNewLines setting
+     */
+    protected processTexts(texts: string[]): string[] {
+        return this.stripNewLines ? texts.map((t) => t.replace(/\n/g, ' ')) : texts;
+    }
+
+    public detectSourceType(source: Source): SupportedSources | 'unknown' {
+        if (typeof source === 'string') {
+            return isUrl(source) ? 'url' : 'text';
+        } else if (Array.isArray(source) && source.every((v) => typeof v === 'number')) {
+            return 'vector';
+        } else {
+            return 'unknown';
+        }
+    }
+
+    public transformSource(source: IVectorDataSourceDto[], sourceType: SupportedSources, candidate: AccessCandidate) {
+        //* as the accepted sources increases, you will need to implement the strategy pattern instead of a switch case
+        switch (sourceType) {
+            case 'text': {
+                const texts = source.map((s) => s.source as string);
+
+                return this.embedTexts(texts, candidate).then((vectors) => {
+                    return source.map((s, i) => ({
+                        ...s,
+                        source: vectors[i],
+                        metadata: { ...s.metadata, text: texts[i] },
+                    }));
+                });
+            }
+            case 'vector': {
+                return source;
+            }
+        }
+    }
+
+    // this is a dummy vector used to maximize search distance
+    // we use it to create reserved vectors that store namespace data
+    public get dummyVector(): number[] {
+        //This is good for cosine similarity, but not for euclidean distance
+        //TODO: detect current similarity metric and use the appropriate vector (e.g for euclidean distance, use a vector with all large values like [1e6, 1e6, ..., 1e6])
+        return Array(this.dimensions - 1)
+            .fill(0)
+            .concat([1]);
+    }
+}

package/src/subsystems/IO/VectorDB.service/embed/OpenAIEmbedding.ts

@@ -1,109 +1,109 @@
-import OpenAI, { type ClientOptions, OpenAI as OpenAIClient } from 'openai';
-import { BaseEmbedding, TEmbeddings } from './BaseEmbedding';
-import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
-import { getLLMCredentials } from '@sre/LLMManager/LLM.service/LLMCredentials.helper';
-import { TLLMCredentials, TLLMModel } from '@sre/types/LLM.types';
-
-// Helper function to create OpenAI API errors
-const createOpenAIError = (statusCode: number, error: any) => {
-    return new OpenAI.APIError(
-        statusCode,
-        {
-            code: error?.errKey || error?.code,
-            message: error?.message,
-            type: error?.name,
-        },
-        error?.message,
-        null
-    );
-};
-
-const DEFAULT_MODEL = 'text-embedding-ada-002';
-
-export class OpenAIEmbeds extends BaseEmbedding {
-    protected client: OpenAIClient;
-    protected clientConfig: ClientOptions;
-
-    public static models = ['text-embedding-ada-002', 'text-embedding-3-large'];
-    public canSpecifyDimensions = true;
-
-    constructor(private settings?: Partial<TEmbeddings>) {
-        super({ model: settings?.model ?? DEFAULT_MODEL, ...settings });
-
-        this.clientConfig = {
-            //apiKey: fields?.credentials?.apiKey || process.env.OPENAI_API_KEY || '',
-            dangerouslyAllowBrowser: true,
-        };
-
-        if (this.model === 'text-embedding-ada-002') {
-            this.canSpecifyDimensions = false; // special case for ada-002, it doesn't support dimensions passing
-        }
-    }
-
-    async embedTexts(texts: string[], candidate: AccessCandidate): Promise<number[][]> {
-        const batches = this.chunkArr(this.processTexts(texts), this.chunkSize);
-
-        const batchRequests = batches.map((batch) => {
-            const params: OpenAIClient.EmbeddingCreateParams = {
-                model: this.model,
-                input: batch,
-            };
-            if (this.dimensions && this.canSpecifyDimensions) {
-                params.dimensions = this.dimensions;
-            }
-            return this.embed(params, candidate);
-        });
-        const batchResponses = await Promise.all(batchRequests);
-
-        const embeddings: number[][] = [];
-        for (let i = 0; i < batchResponses.length; i += 1) {
-            const batch = batches[i];
-            const { data: batchResponse } = batchResponses[i];
-            for (let j = 0; j < batch.length; j += 1) {
-                embeddings.push(batchResponse[j].embedding);
-            }
-        }
-        return embeddings;
-    }
-
-    async embedText(text: string, candidate: AccessCandidate): Promise<number[]> {
-        const params: OpenAIClient.EmbeddingCreateParams = {
-            model: this.model,
-            input: this.processTexts([text])[0],
-        };
-        if (this.dimensions && this.canSpecifyDimensions) {
-            params.dimensions = this.dimensions;
-        }
-        const { data } = await this.embed(params, candidate);
-        return data[0].embedding;
-    }
-
-    protected async embed(request: OpenAIClient.EmbeddingCreateParams, candidate: AccessCandidate) {
-        const modelInfo: TLLMModel = {
-            provider: 'OpenAI',
-            modelId: this.model,
-            credentials: this.settings?.credentials as unknown as TLLMCredentials,
-        };
-        const credentials = await getLLMCredentials(candidate, modelInfo);
-
-        if (!this.client) {
-            const params: ClientOptions = {
-                ...this.clientConfig,
-                apiKey: credentials.apiKey,
-                timeout: this.timeout,
-                maxRetries: 0,
-            };
-
-            this.client = new OpenAIClient(params);
-        }
-        try {
-            const res = await this.client.embeddings.create(request);
-            return res;
-        } catch (e) {
-            // const error = wrapOpenAIClientError(e);
-            // import openapi error from openai and throw it
-            const error = createOpenAIError(e.statusCode, e);
-            throw error;
-        }
-    }
-}
+import OpenAI, { type ClientOptions, OpenAI as OpenAIClient } from 'openai';
+import { BaseEmbedding, TEmbeddings } from './BaseEmbedding';
+import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
+import { getLLMCredentials } from '@sre/LLMManager/LLM.service/LLMCredentials.helper';
+import { TLLMCredentials, TLLMModel } from '@sre/types/LLM.types';
+
+// Helper function to create OpenAI API errors
+const createOpenAIError = (statusCode: number, error: any) => {
+    return new OpenAI.APIError(
+        statusCode,
+        {
+            code: error?.errKey || error?.code,
+            message: error?.message,
+            type: error?.name,
+        },
+        error?.message,
+        null
+    );
+};
+
+const DEFAULT_MODEL = 'text-embedding-ada-002';
+
+export class OpenAIEmbeds extends BaseEmbedding {
+    protected client: OpenAIClient;
+    protected clientConfig: ClientOptions;
+
+    public static models = ['text-embedding-ada-002', 'text-embedding-3-large'];
+    public canSpecifyDimensions = true;
+
+    constructor(private settings?: Partial<TEmbeddings>) {
+        super({ model: settings?.model ?? DEFAULT_MODEL, ...settings });
+
+        this.clientConfig = {
+            //apiKey: fields?.credentials?.apiKey || process.env.OPENAI_API_KEY || '',
+            dangerouslyAllowBrowser: true,
+        };
+
+        if (this.model === 'text-embedding-ada-002') {
+            this.canSpecifyDimensions = false; // special case for ada-002, it doesn't support dimensions passing
+        }
+    }
+
+    async embedTexts(texts: string[], candidate: AccessCandidate): Promise<number[][]> {
+        const batches = this.chunkArr(this.processTexts(texts), this.chunkSize);
+
+        const batchRequests = batches.map((batch) => {
+            const params: OpenAIClient.EmbeddingCreateParams = {
+                model: this.model,
+                input: batch,
+            };
+            if (this.dimensions && this.canSpecifyDimensions) {
+                params.dimensions = this.dimensions;
+            }
+            return this.embed(params, candidate);
+        });
+        const batchResponses = await Promise.all(batchRequests);
+
+        const embeddings: number[][] = [];
+        for (let i = 0; i < batchResponses.length; i += 1) {
+            const batch = batches[i];
+            const { data: batchResponse } = batchResponses[i];
+            for (let j = 0; j < batch.length; j += 1) {
+                embeddings.push(batchResponse[j].embedding);
+            }
+        }
+        return embeddings;
+    }
+
+    async embedText(text: string, candidate: AccessCandidate): Promise<number[]> {
+        const params: OpenAIClient.EmbeddingCreateParams = {
+            model: this.model,
+            input: this.processTexts([text])[0],
+        };
+        if (this.dimensions && this.canSpecifyDimensions) {
+            params.dimensions = this.dimensions;
+        }
+        const { data } = await this.embed(params, candidate);
+        return data[0].embedding;
+    }
+
+    protected async embed(request: OpenAIClient.EmbeddingCreateParams, candidate: AccessCandidate) {
+        const modelInfo: TLLMModel = {
+            provider: 'OpenAI',
+            modelId: this.model,
+            credentials: this.settings?.credentials as unknown as TLLMCredentials,
+        };
+        const credentials = await getLLMCredentials(candidate, modelInfo);
+
+        if (!this.client) {
+            const params: ClientOptions = {
+                ...this.clientConfig,
+                apiKey: credentials.apiKey,
+                timeout: this.timeout,
+                maxRetries: 0,
+            };
+
+            this.client = new OpenAIClient(params);
+        }
+        try {
+            const res = await this.client.embeddings.create(request);
+            return res;
+        } catch (e) {
+            // const error = wrapOpenAIClientError(e);
+            // import openapi error from openai and throw it
+            const error = createOpenAIError(e.statusCode, e);
+            throw error;
+        }
+    }
+}

package/src/subsystems/IO/VectorDB.service/embed/index.ts

@@ -1,21 +1,21 @@
-import { OpenAIEmbeds } from './OpenAIEmbedding';
-import { TEmbeddings } from './BaseEmbedding';
-
-// a factory to get the correct embedding provider based on the provider name
-const supportedProviders = {
-    OpenAI: {
-        embedder: OpenAIEmbeds,
-        models: OpenAIEmbeds.models,
-    },
-} as const;
-
-export type SupportedProviders = keyof typeof supportedProviders;
-export type SupportedModels = {
-    [K in SupportedProviders]: (typeof supportedProviders)[K]['models'][number];
-};
-
-export class EmbeddingsFactory {
-    public static create(provider: SupportedProviders, config: TEmbeddings) {
-        return new supportedProviders[provider].embedder(config);
-    }
-}
+import { OpenAIEmbeds } from './OpenAIEmbedding';
+import { TEmbeddings } from './BaseEmbedding';
+
+// a factory to get the correct embedding provider based on the provider name
+const supportedProviders = {
+    OpenAI: {
+        embedder: OpenAIEmbeds,
+        models: OpenAIEmbeds.models,
+    },
+} as const;
+
+export type SupportedProviders = keyof typeof supportedProviders;
+export type SupportedModels = {
+    [K in SupportedProviders]: (typeof supportedProviders)[K]['models'][number];
+};
+
+export class EmbeddingsFactory {
+    public static create(provider: SupportedProviders, config: TEmbeddings) {
+        return new supportedProviders[provider].embedder(config);
+    }
+}

package/src/subsystems/IO/VectorDB.service/index.ts

@@ -1,14 +1,14 @@
-//==[ SRE: Storage ]======================
-
-import { ConnectorService, ConnectorServiceProvider } from '@sre/Core/ConnectorsService';
-import { TConnectorService } from '@sre/types/SRE.types';
-import { PineconeVectorDB } from './connectors/PineconeVectorDB.class';
-import { MilvusVectorDB } from './connectors/MilvusVectorDB.class';
-import { RAMVectorDB } from './connectors/RAMVecrtorDB.class';
-export class VectorDBService extends ConnectorServiceProvider {
-    public register() {
-        ConnectorService.register(TConnectorService.VectorDB, 'Pinecone', PineconeVectorDB);
-        ConnectorService.register(TConnectorService.VectorDB, 'RAMVec', RAMVectorDB);
-        ConnectorService.register(TConnectorService.VectorDB, 'Milvus', MilvusVectorDB);
-    }
-}
+//==[ SRE: Storage ]======================
+
+import { ConnectorService, ConnectorServiceProvider } from '@sre/Core/ConnectorsService';
+import { TConnectorService } from '@sre/types/SRE.types';
+import { PineconeVectorDB } from './connectors/PineconeVectorDB.class';
+import { MilvusVectorDB } from './connectors/MilvusVectorDB.class';
+import { RAMVectorDB } from './connectors/RAMVecrtorDB.class';
+export class VectorDBService extends ConnectorServiceProvider {
+    public register() {
+        ConnectorService.register(TConnectorService.VectorDB, 'Pinecone', PineconeVectorDB);
+        ConnectorService.register(TConnectorService.VectorDB, 'RAMVec', RAMVectorDB);
+        ConnectorService.register(TConnectorService.VectorDB, 'Milvus', MilvusVectorDB);
+    }
+}