@meaningfully/core 0.1.0

package/dist/services/csvLoader.test.js

@@ -0,0 +1,75 @@
+//@ts-nocheck
+import { describe, it, expect, vi } from 'vitest';
+import { readFileSync } from 'fs';
+import { loadDocumentsFromCsv } from './csvLoader';
+import { Document } from 'llamaindex';
+import Papa from 'papaparse';
+vi.mock('fs');
+vi.mock('papaparse');
+describe('csvLoader.ts', () => {
+    describe('loadDocumentsFromCsv', () => {
+        it('should load documents from CSV and return Document instances', async () => {
+            const mockFileContent = 'text,metadata1,metadata2\ncontent1,meta1,meta2\ncontent2,meta3,meta4';
+            const mockParsedData = {
+                data: [
+                    { text: 'content1', metadata1: 'meta1', metadata2: 'meta2' },
+                    { text: 'content2', metadata1: 'meta3', metadata2: 'meta4' }
+                ]
+            };
+            readFileSync.mockReturnValue(mockFileContent);
+            Papa.parse.mockReturnValue(mockParsedData);
+            const result = await loadDocumentsFromCsv('path/to/csv', 'text');
+            expect(remove_id(result)).toEqual(remove_id([
+                new Document({ text: 'content1', metadata: { metadata1: 'meta1', metadata2: 'meta2' } }),
+                new Document({ text: 'content2', metadata: { metadata1: 'meta3', metadata2: 'meta4' } })
+            ]));
+        });
+        it('should handle empty CSV file', async () => {
+            const mockFileContent = '';
+            const mockParsedData = { data: [] };
+            readFileSync.mockReturnValue(mockFileContent);
+            Papa.parse.mockReturnValue(mockParsedData);
+            const result = await loadDocumentsFromCsv('path/to/csv', 'text');
+            expect(result).toEqual([]);
+        });
+        it('should handle missing text column', async () => {
+            const mockFileContent = 'metadata1,metadata2\nmeta1,meta2\nmeta3,meta4';
+            const mockParsedData = {
+                data: [
+                    { metadata1: 'meta1', metadata2: 'meta2' },
+                    { metadata1: 'meta3', metadata2: 'meta4' }
+                ]
+            };
+            readFileSync.mockReturnValue(mockFileContent);
+            Papa.parse.mockReturnValue(mockParsedData);
+            const result = await loadDocumentsFromCsv('path/to/csv', 'text');
+            expect(remove_id(result)).toEqual(remove_id([
+                new Document({ text: undefined, metadata: { metadata1: 'meta1', metadata2: 'meta2' } }),
+                new Document({ text: undefined, metadata: { metadata1: 'meta3', metadata2: 'meta4' } })
+            ]));
+        });
+        it('should handle null values in metadata', async () => {
+            const mockFileContent = 'text,metadata1,metadata2\ncontent1,,meta2\ncontent2,meta3,';
+            const mockParsedData = {
+                data: [
+                    { text: 'content1', metadata1: null, metadata2: 'meta2' },
+                    { text: 'content2', metadata1: 'meta3', metadata2: null }
+                ]
+            };
+            readFileSync.mockReturnValue(mockFileContent);
+            Papa.parse.mockReturnValue(mockParsedData);
+            const result = await loadDocumentsFromCsv('path/to/csv', 'text');
+            expect(remove_id(result)).toEqual(remove_id([
+                new Document({ text: 'content1', metadata: { metadata1: '', metadata2: 'meta2' } }),
+                new Document({ text: 'content2', metadata: { metadata1: 'meta3', metadata2: '' } })
+            ]));
+        });
+    });
+});
+function remove_id(list_of_documents) {
+    return list_of_documents.map((doc) => {
+        const { id_, ...doc_without_id } = doc;
+        return doc_without_id;
+    });
+}
+//# sourceMappingURL=csvLoader.test.js.map

package/dist/services/csvLoader.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"csvLoader.test.js","sourceRoot":"","sources":["../../src/services/csvLoader.test.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACd,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;AAErB,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC5B,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;QACpC,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;YAC5E,MAAM,eAAe,GAAG,sEAAsE,CAAC;YAC/F,MAAM,cAAc,GAAG;gBACrB,IAAI,EAAE;oBACJ,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE;oBAC5D,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE;iBAC7D;aACF,CAAC;YACF,YAAY,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;YAC9C,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;YAE3C,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;YAEjE,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;gBAC1C,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC;gBACxF,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC;aACzF,CAAC,CAAC,CAAC;QACN,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC5C,MAAM,eAAe,GAAG,EAAE,CAAC;YAC3B,MAAM,cAAc,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;YACpC,YAAY,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;YAC9C,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;YAE3C,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;YAEjE,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC7B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;YACjD,MAAM,eAAe,GAAG,+CAA+C,CAAC;YACxE,MAAM,cAAc,GAAG;gBACrB,IAAI,EAAE;oBACJ,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE;oBAC1C,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE;iBAC3C;aACF,CAAC;YACF,YAAY,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;YAC9C,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;YAE3C,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;YAEjE,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;gBAC1C,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC;gBACvF,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC;aACxF,CAAC,CAAC,CAAC;QACN,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACnD,MAAM,eAAe,GAAG,4DAA4D,CAAC;YACrF,MAAM,cAAc,GAAG;gBACnB,IAAI,EAAE;oBACN,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE;oBACzD,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE;iBACxD;aACJ,CAAC;YACF,YAAY,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;YAC9C,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;YAE3C,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;YACjE,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;gBACxC,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC;gBACnF,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,EAAE,CAAC;aACtF,CAAC,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC;AAEH,SAAS,SAAS,CAAC,iBAAiB;IAClC,OAAO,iBAAiB,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACnC,MAAM,EAAE,GAAG,EAAE,GAAG,cAAc,EAAE,GAAG,GAAG,CAAC;QACvC,OAAO,cAAc,CAAC;IAC1B,CAAC,CAAC,CAAC;AACH,CAAC"}

package/dist/services/embeddings.d.ts

@@ -0,0 +1,22 @@
+import { Document, VectorStoreIndex, TextNode, type StorageContext } from "llamaindex";
+import { OllamaEmbedding } from '@llamaindex/ollama';
+import { MistralAIEmbedding } from '@llamaindex/mistral';
+import { GeminiEmbedding } from '@llamaindex/google';
+import { MockEmbedding } from "./mockEmbedding.js";
+import type { EmbeddingConfig, Settings, MetadataFilter, Clients } from "../types/index.js";
+import { OpenAIEmbedding } from "@llamaindex/openai";
+import { ProgressVectorStoreIndex } from "./progressVectorStoreIndex.js";
+export declare function estimateCost(nodes: TextNode[], modelName: string): {
+    estimatedPrice: number;
+    tokenCount: number;
+    pricePer1M: number;
+};
+export declare function getExistingVectorStoreIndex(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<VectorStoreIndex>;
+export declare function getExistingDocStore(config: EmbeddingConfig): Promise<import("llamaindex").BaseDocumentStore>;
+export declare function transformDocumentsToNodes(documents: Document[], config: EmbeddingConfig): Promise<TextNode<import("llamaindex").Metadata>[]>;
+export declare function getEmbedModel(config: EmbeddingConfig, settings: Settings): OpenAIEmbedding | OllamaEmbedding | MistralAIEmbedding | GeminiEmbedding | MockEmbedding;
+export declare function getStorageContext(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<StorageContext>;
+export declare function persistDocuments(documents: Document[], config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<void>;
+export declare function persistNodes(nodes: TextNode[], config: EmbeddingConfig, settings: Settings, clients: Clients, progressCallback?: (progress: number, total: number) => void): Promise<ProgressVectorStoreIndex>;
+export declare function searchDocuments(index: VectorStoreIndex, query: string, numResults?: number, filters?: MetadataFilter[]): Promise<import("llamaindex").NodeWithScore<import("llamaindex").Metadata>[]>;
+//# sourceMappingURL=embeddings.d.ts.map

package/dist/services/embeddings.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../../src/services/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,gBAAgB,EAIhB,QAAQ,EAKR,KAAK,cAAc,EAGpB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,eAAe,EAAC,MAAM,oBAAoB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAA+B,MAAM,qBAAqB,CAAA;AACrF,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAA;AAKpD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAGnD,OAAO,KAAK,EAAE,eAAe,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAG,MAAM,mBAAmB,CAAC;AAG7F,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAErD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAuCzE,wBAAgB,YAAY,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG;IAClE,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB,CAuBA;AAED,wBAAsB,2BAA2B,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,6BAqD9G;AAED,wBAAsB,mBAAmB,CAAC,MAAM,EAAE,eAAe,mDAehE;AAID,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,QAAQ,EAAE,EACrB,MAAM,EAAE,eAAe,sDA0BxB;AAED,wBAAgB,aAAa,CAC3B,MAAM,EAAE,eAAe,EACvB,QAAQ,EAAE,QAAQ,4FA2CnB;AAED,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,cAAc,CAAC,CAiB9H;AAED,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAW1I;AAED,wBAAsB,YAAY,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAqCpN;AAyCD,wBAAsB,eAAe,CACnC,KAAK,EAAE,gBAAgB,EACvB,KAAK,EAAE,MAAM,EACb,UAAU,GAAE,MAAW,EACvB,OAAO,CAAC,EAAE,cAAc,EAAE,gFAU3B"}

package/dist/services/embeddings.js

@@ -0,0 +1,314 @@
+import { VectorStoreIndex, 
+// OpenAIEmbedding,
+IngestionPipeline, ModalityType, storageContextFromDefaults, SimpleVectorStore, Settings as LlamaindexSettings, SimpleDocumentStore } from "llamaindex";
+import { OllamaEmbedding } from '@llamaindex/ollama';
+import { MistralAIEmbedding, MistralAIEmbeddingModelType } from '@llamaindex/mistral';
+import { GeminiEmbedding } from '@llamaindex/google';
+import { PGVectorStore } from '@llamaindex/postgres';
+import { AzureOpenAIEmbedding } from "@llamaindex/azure";
+import { Sploder } from "./sploder.js";
+import { CustomSentenceSplitter } from "./sentenceSplitter.js";
+import { MockEmbedding } from "./mockEmbedding.js";
+import { encodingForModel } from "js-tiktoken";
+import { join } from "path";
+import { sanitizeProjectName, capitalizeFirstLetter } from "../utils.js";
+import * as fs from 'fs';
+import { OpenAIEmbedding } from "@llamaindex/openai";
+import { BatchingWeaviateVectorStore } from "./batchingWeaviateVectorStore.js";
+import { ProgressVectorStoreIndex } from "./progressVectorStoreIndex.js";
+// unused, but probalby eventually will be used.
+// to be used by postgres store, which it' slooking increasingly like I have to enable again
+const MODEL_DIMENSIONS = {
+    "text-embedding-3-small": 1536,
+    "text-embedding-3-large": 3072,
+    "mxbai-embed-large": 1024,
+    "mistral-embed": 1024,
+    "gemini-embedding-001": 768, // Gemini embedding model
+};
+const PRICE_PER_1M = {
+    "text-embedding-3-small": 0.02,
+    "text-embedding-3-large": 0.13,
+    "mistral-embed": 0.1,
+    "mxbai-embed-large": 0, // local model, free
+    "nomic-embed-text": 0, // local model, free
+    "gemini-embedding-001": 0.0, // Gemini embedding is currently free (unless you're on the paid tier, in which case it is $0.15/million tokens)
+};
+/* all transformations except the embedding step (which is handled by VectorStoreIndex.init) */
+function getBaseTransformations(config) {
+    const transformations = [
+        new CustomSentenceSplitter({ chunkSize: config.chunkSize, chunkOverlap: config.chunkOverlap }),
+    ];
+    if (config.combineSentencesIntoChunks) {
+        transformations.push(new Sploder({
+            maxStringTokenCount: config.sploderMaxSize
+        }));
+    }
+    return transformations;
+}
+export function estimateCost(nodes, modelName) {
+    const pricePer1M = PRICE_PER_1M[modelName] || 0; // default to 0 if model not found or free
+    let tokenizer;
+    try {
+        tokenizer = encodingForModel(modelName); // This doesn't work for ollama
+    }
+    catch (error) {
+        // If the tokenizer is not found, it means the model is likely not supported by tiktoken
+        // or is a local model (like Ollama). In this case, we can't estimate the cost.
+        tokenizer = encodingForModel("text-embedding-3-small"); // fallback to a known tokenizer
+        console.warn(`Tokenizer for model ${modelName} not found. Using fallback tokenizer.`);
+    }
+    const tokenCount = nodes.reduce((sum, node) => {
+        return sum + tokenizer.encode(node.text).length;
+    }, 0);
+    const estimatedPrice = tokenCount * (pricePer1M / 1_000_000);
+    return {
+        estimatedPrice,
+        tokenCount,
+        pricePer1M
+    };
+}
+export async function getExistingVectorStoreIndex(config, settings, clients) {
+    const embedModel = getEmbedModel(config, settings);
+    switch (config.vectorStoreType) {
+        case "simple":
+            const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName));
+            const storageContext = await storageContextFromDefaults({
+                persistDir: persistDir,
+            });
+            let vsi = await VectorStoreIndex.init({
+                storageContext: storageContext,
+            });
+            vsi.embedModel = embedModel;
+            return vsi;
+        case "postgres":
+            if (!clients.postgresClient) {
+                throw new Error("Postgres client required but not provided");
+            }
+            const pgStore = new PGVectorStore({
+                clientConfig: { connectionString: process.env.POSTGRES_CONNECTION_STRING },
+                tableName: sanitizeProjectName(config.projectName),
+                dimensions: MODEL_DIMENSIONS[config.modelName] || 1536, // default to 1536 if model not found
+                embeddingModel: embedModel
+            });
+            const pgStorageContext = await storageContextFromDefaults({
+                vectorStores: { [ModalityType.TEXT]: pgStore },
+            });
+            return await VectorStoreIndex.init({
+                storageContext: pgStorageContext,
+            });
+        case "weaviate":
+            if (!clients.weaviateClient) {
+                throw new Error("Weaviate client required but not provided");
+            }
+            const weaviateStore = new BatchingWeaviateVectorStore({
+                indexName: capitalizeFirstLetter(sanitizeProjectName(config.projectName)),
+                weaviateClient: clients.weaviateClient,
+                embeddingModel: embedModel
+            });
+            // WeaviateVectorStore's getNodeSimilarity method looks for distance, but current weaviate provides score
+            // (WeaviateVectorStore would get `score` if we were doing hybrid search)
+            // Overwrite the private getNodeSimilarity method to use 'score' from metadata
+            // @ts-ignore
+            weaviateStore.getNodeSimilarity = (entry, _similarityKey = "score") => {
+                return entry.metadata.score;
+            };
+            return await VectorStoreIndex.fromVectorStore(weaviateStore);
+        default:
+            throw new Error(`Unsupported vector store type: ${config.vectorStoreType}`);
+    }
+}
+export async function getExistingDocStore(config) {
+    // switch (config.vectorStoreType) {
+    //   case "simple":
+    const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName));
+    const storageContext = await storageContextFromDefaults({
+        persistDir: persistDir,
+    });
+    return storageContext.docStore;
+    //   case "postgres":
+    //     throw new Error(`Not yet implemented vector store type: ${config.vectorStoreType}`);
+    //     // return await createVectorStore(config);
+    //   default:
+    //     throw new Error(`Unsupported vector store type: ${config.vectorStoreType}`);
+    // }
+}
+export async function transformDocumentsToNodes(documents, config) {
+    console.time("transformDocumentsToNodes Run Time");
+    const transformations = getBaseTransformations(config);
+    // llama-index stupidly includes all the metadata in the embedding, which is a waste of tokens
+    // so we exclude everything except the text column from the embedding
+    for (const document of documents) {
+        document.excludedEmbedMetadataKeys = Object.keys(document.metadata);
+    }
+    console.time("transformDocumentsToNodes transformDocuments Run Time");
+    // remove empty documents. we can't meaningfully embed these, so we're just gonna ignore 'em.
+    // that might not ultimately be the right solution. 
+    documents = documents.filter((document_) => document_.text && document_.text.length > 0);
+    // Create nodes with sentence splitting and optional sploder
+    const pipeline = new IngestionPipeline({
+        transformations
+    });
+    const nodes = (await pipeline.run({ documents: documents }));
+    console.timeEnd("transformDocumentsToNodes transformDocuments Run Time");
+    console.timeEnd("transformDocumentsToNodes Run Time");
+    return nodes;
+}
+export function getEmbedModel(config, settings) {
+    let embedModel;
+    if (config.modelProvider === "openai") {
+        embedModel = new OpenAIEmbedding({ model: config.modelName, apiKey: settings.openAIKey ? settings.openAIKey : undefined });
+        embedModel.embedBatchSize = 50; // all embedding models enforce a maximum of 300,000 tokens summed across all inputs in a single request
+    }
+    else if (config.modelProvider === "ollama") {
+        embedModel = new OllamaEmbedding({ model: config.modelName, config: {
+                host: settings.oLlamaBaseURL ? settings.oLlamaBaseURL : undefined
+            }, });
+    }
+    else if (config.modelProvider === "azure") {
+        if (!settings.azureOpenAIKey || !settings.azureOpenAIEndpoint) {
+            throw new Error("Azure OpenAI API key and endpoint are required for Azure embedding models");
+        }
+        embedModel = new AzureOpenAIEmbedding({
+            model: config.modelName,
+            apiKey: settings.azureOpenAIKey,
+            endpoint: settings.azureOpenAIEndpoint,
+            apiVersion: settings.azureOpenAIApiVersion ?? undefined
+        });
+    }
+    else if (config.modelProvider === "mistral") {
+        if (!settings.mistralApiKey) {
+            throw new Error("Mistral API key is required for Mistral embedding models");
+        }
+        embedModel = new MistralAIEmbedding({
+            model: MistralAIEmbeddingModelType.MISTRAL_EMBED, // only one choice!
+            apiKey: settings.mistralApiKey
+        });
+    }
+    else if (config.modelProvider === "gemini") {
+        if (!settings.geminiApiKey) {
+            throw new Error("Gemini API key is required for Gemini embedding models");
+        }
+        embedModel = new GeminiEmbedding({
+            apiKey: settings.geminiApiKey,
+        });
+        embedModel.embedBatchSize = 50;
+    }
+    else if (config.modelProvider === "mock") {
+        embedModel = new MockEmbedding();
+    }
+    else {
+        throw new Error(`Unsupported embedding model provider: ${config.modelProvider}`);
+    }
+    LlamaindexSettings.embedModel = embedModel;
+    return embedModel;
+}
+export async function getStorageContext(config, settings, clients) {
+    const vectorStore = await createVectorStore(config, settings, clients);
+    fs.mkdirSync(config.storagePath, { recursive: true });
+    const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName));
+    return await storageContextFromDefaults({
+        persistDir: persistDir,
+        vectorStores: { [ModalityType.TEXT]: vectorStore },
+        docStore: new SimpleDocumentStore()
+        /*
+          if docStore is created with a persist path (as it is by default in storageContextFromDefaults)
+          then it will write to disk after every put(), which happens 2+ times per document.
+  
+          so we create it without a persist path, and then explicitly persist it when we're done adding documents.
+  
+          see https://github.com/jeremybmerrill/meaningfully/issues/52
+        */
+    });
+}
+export async function persistDocuments(documents, config, settings, clients) {
+    console.time("persistDocuments Run Time");
+    const storageContext = await getStorageContext(config, settings, clients);
+    await storageContext.docStore.addDocuments(documents, true);
+    // see comments in getStorageContext
+    const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName));
+    // @ts-ignore
+    await storageContext.docStore.kvStore.persist(join(persistDir, "doc_store.json"));
+    console.timeEnd("persistDocuments Run Time");
+}
+export async function persistNodes(nodes, config, settings, clients, progressCallback) {
+    // Create and configure vector store based on type
+    console.time("persistNodes Run Time");
+    const storageContext = await getStorageContext(config, settings, clients);
+    const vectorStore = storageContext.vectorStores[ModalityType.TEXT];
+    if (!vectorStore) {
+        throw new Error("Vector store is undefined");
+    }
+    // Create index and embed documents
+    // this is what actaully embeds the nodes
+    // (even if they already have embeddings, stupidly)
+    const index = await ProgressVectorStoreIndex.init({
+        nodes,
+        storageContext,
+        logProgress: true,
+        progressCallback,
+    });
+    // I'm not sure why this explicit call to persist is necessary. 
+    // storageContext should handle this, but it doesn't.
+    // all the if statements are just type-checking boilerplate.
+    // N.B. WeaviateVectorStore does not need to be explicitly persisted, so we don't include it in the OR conditional here..
+    if (vectorStore) {
+        if (vectorStore instanceof PGVectorStore || vectorStore instanceof SimpleVectorStore) {
+            await vectorStore.persist(join(config.storagePath, sanitizeProjectName(config.projectName), "vector_store.json"));
+        }
+        else if (vectorStore instanceof BatchingWeaviateVectorStore) {
+            // WeaviateVectorStore does not have a persist method, it persists automatically
+            console.log("Pretending to persist Weaviate vector store, but it actually persists automatically.");
+        }
+        else {
+            throw new Error("Vector store does not support persist method");
+        }
+    }
+    else {
+        throw new Error("Vector store is undefined");
+    }
+    console.timeEnd("persistNodes Run Time");
+    return index;
+}
+async function createVectorStore(config, settings, clients) {
+    const embeddingModel = getEmbedModel(config, settings);
+    switch (config.vectorStoreType) {
+        // for some reason the embedding model has to be specified here TOO
+        // otherwise it defaults to Ada.
+        case "postgres":
+            return new PGVectorStore({
+                clientConfig: { connectionString: process.env.POSTGRES_CONNECTION_STRING },
+                tableName: sanitizeProjectName(config.projectName),
+                dimensions: MODEL_DIMENSIONS[config.modelName] || 1536, // default to 1536 if model not found
+                embeddingModel: embeddingModel
+            });
+        case "simple":
+            const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName));
+            return SimpleVectorStore.fromPersistDir(persistDir, embeddingModel);
+        case "weaviate":
+            const vectorStore = new BatchingWeaviateVectorStore({
+                indexName: capitalizeFirstLetter(sanitizeProjectName(config.projectName)),
+                weaviateClient: clients.weaviateClient,
+                embeddingModel: embeddingModel
+            });
+            // WeaviateVectorStore's getNodeSimilarity method looks for distance, but current weaviate provides score
+            // (WeaviateVectorStore would get `score` if we were doing hybrid search)
+            // Overwrite the private getNodeSimilarity method to use 'score' from metadata
+            // @ts-ignore
+            vectorStore.getNodeSimilarity = (entry, _similarityKey = "score") => {
+                return entry.metadata.score;
+            };
+            return vectorStore;
+        default:
+            throw new Error(`Unsupported vector store type: ${config.vectorStoreType}`);
+    }
+}
+export async function searchDocuments(index, query, numResults = 10, filters) {
+    // const metadataFilters: MetadataFilters | undefined = filters ? {filters: filters} : undefined;
+    const metadataFilters = {
+        filters: filters ? filters : [],
+    };
+    const retriever = index.asRetriever({ similarityTopK: numResults, filters: metadataFilters });
+    const results = await retriever.retrieve(query);
+    return results;
+}
+//# sourceMappingURL=embeddings.js.map

package/dist/services/embeddings.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../../src/services/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,gBAAgB;AAChB,mBAAmB;AACnB,iBAAiB,EAGjB,YAAY,EAEZ,0BAA0B,EAC1B,iBAAiB,EAEjB,QAAQ,IAAI,kBAAkB,EAC9B,mBAAmB,EACpB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,eAAe,EAAC,MAAM,oBAAoB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAAE,2BAA2B,EAAE,MAAM,qBAAqB,CAAA;AACrF,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAA;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAsB,MAAM,aAAa,CAAC;AACnE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACzE,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,2BAA2B,EAAE,MAAM,kCAAkC,CAAC;AAC/E,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,gDAAgD;AAChD,4FAA4F;AAC5F,MAAM,gBAAgB,GAA2B;IAC/C,wBAAwB,EAAE,IAAI;IAC9B,wBAAwB,EAAE,IAAI;IAC9B,mBAAmB,EAAE,IAAI;IACzB,eAAe,EAAE,IAAI;IACrB,sBAAsB,EAAE,GAAG,EAAE,yBAAyB;CACvD,CAAC;AAEF,MAAM,YAAY,GAA2B;IAC3C,wBAAwB,EAAE,IAAI;IAC9B,wBAAwB,EAAE,IAAI;IAC9B,eAAe,EAAE,GAAG;IACpB,mBAAmB,EAAE,CAAC,EAAE,oBAAoB;IAC5C,kBAAkB,EAAE,CAAC,EAAE,oBAAoB;IAC3C,sBAAsB,EAAE,GAAG,EAAE,gHAAgH;CAC9I,CAAC;AAGF,+FAA+F;AAC/F,SAAS,sBAAsB,CAAC,MAAuB;IACrD,MAAM,eAAe,GAAyB;QAC5C,IAAI,sBAAsB,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC;KAC/F,CAAC;IAEF,IAAI,MAAM,CAAC,0BAA0B,EAAE,CAAC;QACtC,eAAe,CAAC,IAAI,CAClB,IAAI,OAAO,CAAC;YACV,mBAAmB,EAAE,MAAM,CAAC,cAAc;SAC3C,CAAC,CACH,CAAC;IACJ,CAAC;IAED,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAiB,EAAE,SAAiB;IAK/D,MAAM,UAAU,GAAG,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,0CAA0C;IAE3F,IAAI,SAAS,CAAC;IACd,IAAG,CAAC;QACF,SAAS,GAAG,gBAAgB,CAAC,SAA0B,CAAC,CAAC,CAAC,+BAA+B;IAC3F,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,wFAAwF;QACxF,+EAA+E;QAC/E,SAAS,GAAG,gBAAgB,CAAC,wBAAwB,CAAC,CAAC,CAAC,gCAAgC;QACxF,OAAO,CAAC,IAAI,CAAC,uBAAuB,SAAS,uCAAuC,CAAC,CAAC;IACxF,CAAC;IACD,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;QAC5C,OAAO,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAClD,CAAC,EAAE,CAAC,CAAC,CAAC;IAEN,MAAM,cAAc,GAAG,UAAU,GAAG,CAAC,UAAU,GAAG,SAAS,CAAC,CAAC;IAE7D,OAAO;QACL,cAAc;QACd,UAAU;QACV,UAAU;KACX,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC7G,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACnD,QAAQ,MAAM,CAAC,eAAe,EAAE,CAAC;QAC/B,KAAK,QAAQ;YACX,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;YACrF,MAAM,cAAc,GAAG,MAAM,0BAA0B,CAAC;gBACtD,UAAU,EAAE,UAAU;aACvB,CAAC,CAAC;YACH,IAAI,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC;gBACpC,cAAc,EAAE,cAAc;aAC/B,CAAC,CAAC;YACH,GAAG,CAAC,UAAU,GAAG,UAAU,CAAC;YAC5B,OAAO,GAAG,CAAC;QAEb,KAAK,UAAU;YACb,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YAC/D,CAAC;YACD,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC;gBAChC,YAAY,EAAE,EAAE,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,0BAA0B,EAAE;gBAC1E,SAAS,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC;gBAClD,UAAU,EAAE,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,IAAI,EAAE,qCAAqC;gBAC7F,cAAc,EAAE,UAAU;aAC3B,CAAC,CAAC;YACH,MAAM,gBAAgB,GAAG,MAAM,0BAA0B,CAAC;gBACxD,YAAY,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE;aAC/C,CAAC,CAAC;YACH,OAAO,MAAM,gBAAgB,CAAC,IAAI,CAAC;gBACjC,cAAc,EAAE,gBAAgB;aACjC,CAAC,CAAC;QACL,KAAK,UAAU;YACb,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YAC/D,CAAC;YACD,MAAM,aAAa,GAAG,IAAI,2BAA2B,CAAC;gBACpD,SAAS,EAAE,qBAAqB,CAAC,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;gBACzE,cAAc,EAAE,OAAO,CAAC,cAAc;gBACtC,cAAc,EAAE,UAAU;aAC3B,CAAC,CAAC;YAEH,yGAAyG;YACzG,yEAAyE;YACzE,8EAA8E;YAC9E,aAAa;YACb,aAAa,CAAC,iBAAiB,GAAG,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,EAAE,EAAE;gBACpE,OAAQ,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC;YAC/B,CAAC,CAAA;YAED,OAAO,MAAM,gBAAgB,CAAC,eAAe,CAAC,aAAa,CAAC,CAAA;QAE9D;YACE,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,MAAuB;IAC/D,oCAAoC;IACpC,mBAAmB;IACf,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAE,CAAC;IACtF,MAAM,cAAc,GAAG,MAAM,0BAA0B,CAAC;QACtD,UAAU,EAAE,UAAU;KACvB,CAAC,CAAC;IACH,OAAO,cAAc,CAAC,QAAQ,CAAC;IAEnC,qBAAqB;IACrB,2FAA2F;IAC3F,iDAAiD;IACjD,aAAa;IACb,mFAAmF;IACnF,IAAI;AACN,CAAC;AAID,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,SAAqB,EACrB,MAAuB;IAEvB,OAAO,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;IAEnD,MAAM,eAAe,GAAG,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAEvD,8FAA8F;IAC9F,qEAAqE;IACrE,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,QAAQ,CAAC,yBAAyB,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACtE,CAAC;IACD,OAAO,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;IACtE,6FAA6F;IAC7F,oDAAoD;IACpD,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAEzF,4DAA4D;IAC5D,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC;QACrC,eAAe;KAChB,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAAC,GAAG,CAAC,EAAC,SAAS,EAAE,SAAS,EAAC,CAAC,CAAe,CAAC;IAEzE,OAAO,CAAC,OAAO,CAAC,uDAAuD,CAAC,CAAC;IACzE,OAAO,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC;IACtD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,MAAuB,EACvB,QAAkB;IAElB,IAAI,UAAU,CAAC;IACf,IAAI,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QACtC,UAAU,GAAG,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,EAAC,CAAE,CAAC;QAC3H,UAAU,CAAC,cAAc,GAAG,EAAE,CAAC,CAAC,wGAAwG;IAC1I,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QAC7C,UAAU,GAAG,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE;gBAClE,IAAI,EAAE,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;aAClE,GAAG,CAAC,CAAC;IACR,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,OAAO,EAAE,CAAC;QAC5C,IAAI,CAAC,QAAQ,CAAC,cAAc,IAAI,CAAC,QAAQ,CAAC,mBAAmB,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;QAC/F,CAAC;QACD,UAAU,GAAG,IAAI,oBAAoB,CAAC;YACpC,KAAK,EAAE,MAAM,CAAC,SAAS;YACvB,MAAM,EAAE,QAAQ,CAAC,cAAc;YAC/B,QAAQ,EAAE,QAAQ,CAAC,mBAAmB;YACtC,UAAU,EAAE,QAAQ,CAAC,qBAAqB,IAAI,SAAS;SACxD,CAAC,CAAC;IACL,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;QAC9C,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;QAC9E,CAAC;QACD,UAAU,GAAG,IAAI,kBAAkB,CAAC;YAClC,KAAK,EAAE,2BAA2B,CAAC,aAAa,EAAE,mBAAmB;YACrE,MAAM,EAAE,QAAQ,CAAC,aAAa;SAC/B,CAAC,CAAC;IACL,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QAC7C,IAAI,CAAC,QAAQ,CAAC,YAAY,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC5E,CAAC;QACD,UAAU,GAAG,IAAI,eAAe,CAAC;YAC/B,MAAM,EAAE,QAAQ,CAAC,YAAY;SAC9B,CAAC,CAAC;QACH,UAAU,CAAC,cAAc,GAAG,EAAE,CAAC;IACjC,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,MAAM,EAAE,CAAC;QAC3C,UAAU,GAAG,IAAI,aAAa,EAAE,CAAC;IACnC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,yCAAyC,MAAM,CAAC,aAAa,EAAE,CAAC,CAAC;IACnF,CAAC;IACD,kBAAkB,CAAC,UAAU,GAAG,UAAU,CAAC;IAC3C,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IACnG,MAAM,WAAW,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IACvE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAE,CAAC;IACtF,OAAO,MAAM,0BAA0B,CAAC;QACtC,UAAU,EAAE,UAAU;QACtB,YAAY,EAAE,EAAC,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,WAAW,EAAC;QAChD,QAAQ,EAAE,IAAI,mBAAmB,EAAE;QACjC;;;;;;;UAOE;KACL,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,SAAqB,EAAE,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IACzH,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IAC1C,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC1E,MAAM,cAAc,CAAC,QAAQ,CAAC,YAAY,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAE5D,oCAAoC;IACpC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAE,CAAC;IACtF,aAAa;IACb,MAAO,cAAc,CAAC,QAAgC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAE3G,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAiB,EAAE,MAAuB,EAAE,QAAkB,EAAE,OAAgB,EAAE,gBAA4D;IAC/K,kDAAkD;IAClD,OAAO,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAEtC,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC1E,MAAM,WAAW,GAAG,cAAc,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IACnE,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IACD,mCAAmC;IACnC,yCAAyC;IACzC,mDAAmD;IACnD,MAAM,KAAK,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC;QAChD,KAAK;QACL,cAAc;QACd,WAAW,EAAE,IAAI;QACjB,gBAAgB;KACjB,CAAC,CAAC;IAEH,gEAAgE;IAChE,qDAAqD;IACrD,4DAA4D;IAC5D,yHAAyH;IACzH,IAAI,WAAW,EAAE,CAAC;QAChB,IAAI,WAAW,YAAY,aAAa,IAAI,WAAW,YAAY,iBAAiB,EAAE,CAAC;YACrF,MAAM,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,mBAAmB,CAAC,CAAC,CAAC;QACpH,CAAC;aAAM,IAAI,WAAW,YAAY,2BAA2B,EAAE,CAAC;YAC9D,gFAAgF;YAChF,OAAO,CAAC,GAAG,CAAC,sFAAsF,CAAC,CAAC;QACtG,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACzC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC5F,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACvD,QAAQ,MAAM,CAAC,eAAe,EAAE,CAAC;QAE/B,mEAAmE;QACnE,gCAAgC;QAChC,KAAK,UAAU;YACb,OAAO,IAAI,aAAa,CAAC;gBACvB,YAAY,EAAE,EAAC,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,0BAA0B,EAAC;gBACxE,SAAS,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC;gBAClD,UAAU,EAAE,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,IAAI,EAAE,qCAAqC;gBAC7F,cAAc,EAAE,cAAc;aAC/B,CAAC,CAAC;QAEL,KAAK,QAAQ;YACX,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;YACrF,OAAO,iBAAiB,CAAC,cAAc,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;QAEtE,KAAK,UAAU;YACb,MAAM,WAAW,GAAG,IAAI,2BAA2B,CAAC;gBAClD,SAAS,EAAE,qBAAqB,CAAC,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;gBACzE,cAAc,EAAE,OAAO,CAAC,cAAc;gBACtC,cAAc,EAAE,cAAc;aAC/B,CAAC,CAAC;YAEH,yGAAyG;YACzG,yEAAyE;YACzE,8EAA8E;YAC9E,aAAa;YACb,WAAW,CAAC,iBAAiB,GAAG,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,EAAE,EAAE;gBAClE,OAAQ,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC;YAC/B,CAAC,CAAA;YAED,OAAO,WAAW,CAAC;QACrB;YACE,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAuB,EACvB,KAAa,EACb,aAAqB,EAAE,EACvB,OAA0B;IAE1B,iGAAiG;IACjG,MAAM,eAAe,GAAoB;QACvC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;KAChC,CAAC;IACF,MAAM,SAAS,GAAG,KAAK,CAAC,WAAW,CAAC,EAAE,cAAc,EAAE,UAAU,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAC;IAE9F,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAE,CAAC;IACjD,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/services/embeddings.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=embeddings.test.d.ts.map

package/dist/services/embeddings.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embeddings.test.d.ts","sourceRoot":"","sources":["../../src/services/embeddings.test.ts"],"names":[],"mappings":""}

package/dist/services/embeddings.test.js

@@ -0,0 +1,115 @@
+//@ts-nocheck
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { Document, TextNode } from 'llamaindex';
+// First, set up the mock before importing the module
+vi.mock(import("./embeddings"), async (importOriginal) => {
+    const actual = await importOriginal();
+    return {
+        ...actual,
+        // your mocked methods
+        estimateCost: vi.fn(),
+        getExistingVectorStoreIndex: vi.fn(),
+        persistNodes: vi.fn(),
+        persistDocuments: vi.fn(),
+        getExistingDocStore: vi.fn(),
+        searchDocuments: vi.fn()
+    };
+});
+// Now import the mocked functions
+import { transformDocumentsToNodes, getEmbedModel } from './embeddings';
+describe('transformDocumentsToNodes', () => {
+    beforeEach(() => {
+        vi.clearAllMocks();
+    });
+    const mockConfig = {
+        chunkSize: 100,
+        chunkOverlap: 10,
+        combineSentencesIntoChunks: true,
+        sploderMaxSize: 500,
+        modelProvider: 'mock',
+        modelName: 'text-embedding-3-small',
+        vectorStoreType: "simple",
+        storagePath: './storage',
+        projectName: 'test_project',
+        splitIntoSentences: true,
+    };
+    const mockSettings = {
+        openAIKey: 'mock-api-key',
+        oLlamaBaseURL: 'http://localhost',
+        azureOpenAIKey: null,
+        azureOpenAIEndpoint: null,
+        azureOpenAIApiVersion: null,
+        mistralApiKey: null,
+        geminiApiKey: null,
+    };
+    it('should process documents and return nodes', async () => {
+        const mockDocuments = [
+            new Document({ text: 'Document 1', metadata: { key1: 'value1' } }),
+            new Document({ text: 'Document 2', metadata: { key2: 'value2' } }),
+        ];
+        const mockNodes = [
+            new TextNode({ text: 'Document 1' }),
+            new TextNode({ text: 'Document 2' }),
+        ];
+        const result = await transformDocumentsToNodes(mockDocuments, mockConfig, mockSettings);
+        expect(result.map((node) => node.text)).toEqual(mockNodes.map((node) => node.text));
+    });
+    it('should filter out documents with null, undefined, or zero-length text', async () => {
+        const mockDocuments = [
+            new Document({ text: 'Valid Document', metadata: { key1: 'value1' } }),
+            new Document({ text: undefined, metadata: { key3: 'value3' } }),
+            new Document({ text: '', metadata: { key4: 'value4' } }),
+        ];
+        const filteredDocuments = [mockDocuments[0]];
+        const mockNodes = [new TextNode({ text: 'Valid Document' })];
+        // (transformDocumentsToNodes as vi.Mock).mockResolvedValue(mockNodes);
+        const result = await transformDocumentsToNodes(mockDocuments, mockConfig, mockSettings);
+        expect(result.map((n) => n.text)).toEqual(mockNodes.map((n) => n.text));
+        // TODO: I can't get these to work. Apparently you can't spyOn a function that is imported from the same file.
+        // all well and good but ... why did CoPilot generate a test that can't work?
+        // expect(transformDocumentsToNodes).toHaveBeenCalledWith(filteredDocuments, expect.any(Array));
+    });
+    it('should exclude all metadata keys from embedding', async () => {
+        const mockDocuments = [
+            new Document({ text: 'Document 1', metadata: { key1: 'value1', key2: 'value2' } }),
+        ];
+        const nodes = await transformDocumentsToNodes(mockDocuments, mockConfig, mockSettings);
+        expect(nodes[0].excludedEmbedMetadataKeys).toEqual(['key1', 'key2']);
+    });
+});
+describe('getEmbedModel', () => {
+    const mockConfig = {
+        chunkSize: 100,
+        chunkOverlap: 10,
+        combineSentencesIntoChunks: true,
+        sploderMaxSize: 500,
+        modelProvider: 'openai',
+        modelName: 'text-embedding-3-small',
+        vectorStoreType: "simple",
+        storagePath: './storage',
+        projectName: 'test_project',
+        splitIntoSentences: true,
+    };
+    const mockSettings = {
+        openAIKey: 'mock-api-key',
+        oLlamaBaseURL: 'http://localhost',
+        azureOpenAIKey: null,
+        azureOpenAIEndpoint: null,
+        azureOpenAIApiVersion: null,
+        mistralApiKey: null,
+        geminiApiKey: null,
+    };
+    it('should handle different model providers correctly', () => {
+        // Test with 'ollama' provider
+        const ollamaModel = getEmbedModel({ ...mockConfig, modelProvider: 'ollama' }, mockSettings);
+        expect(ollamaModel).toBeDefined();
+        // Test with 'mock' provider
+        const mockModel = getEmbedModel({ ...mockConfig, modelProvider: 'mock' }, mockSettings);
+        expect(mockModel).toBeDefined();
+        // Test with invalid provider
+        expect(() => {
+            getEmbedModel({ ...mockConfig, modelProvider: 'invalid' }, mockSettings);
+        }).toThrow('Unsupported embedding model provider: invalid');
+    });
+});
+//# sourceMappingURL=embeddings.test.js.map

package/dist/services/embeddings.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embeddings.test.js","sourceRoot":"","sources":["../../src/services/embeddings.test.ts"],"names":[],"mappings":"AAAA,aAAa;AAEb,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC9D,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEhD,qDAAqD;AACrD,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,EAAE;IACvD,MAAM,MAAM,GAAG,MAAM,cAAc,EAAE,CAAA;IACrC,OAAO;QACL,GAAG,MAAM;QACT,sBAAsB;QACtB,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE;QACrB,2BAA2B,EAAE,EAAE,CAAC,EAAE,EAAE;QACpC,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE;QACrB,gBAAgB,EAAE,EAAE,CAAC,EAAE,EAAE;QACzB,mBAAmB,EAAE,EAAE,CAAC,EAAE,EAAE;QAC5B,eAAe,EAAE,EAAE,CAAC,EAAE,EAAE;KACzB,CAAA;AACH,CAAC,CAAC,CAAA;AAEF,kCAAkC;AAClC,OAAO,EAAE,yBAAyB,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAExE,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;IACzC,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;IACrB,CAAC,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG;QACjB,SAAS,EAAE,GAAG;QACd,YAAY,EAAE,EAAE;QAChB,0BAA0B,EAAE,IAAI;QAChC,cAAc,EAAE,GAAG;QACnB,aAAa,EAAE,MAAM;QACrB,SAAS,EAAE,wBAAwB;QACnC,eAAe,EAAE,QAAoB;QACrC,WAAW,EAAE,WAAW;QACxB,WAAW,EAAE,cAAc;QAC3B,kBAAkB,EAAE,IAAI;KACzB,CAAC;IAEF,MAAM,YAAY,GAAG;QACnB,SAAS,EAAE,cAAc;QACzB,aAAa,EAAE,kBAAkB;QACjC,cAAc,EAAE,IAAI;QACpB,mBAAmB,EAAE,IAAI;QACzB,qBAAqB,EAAE,IAAI;QAC3B,aAAa,EAAE,IAAI;QACnB,YAAY,EAAE,IAAI;KACnB,CAAC;IAEF,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,aAAa,GAAG;YACpB,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC;YAClE,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC;SACnE,CAAC;QACF,MAAM,SAAS,GAAG;YAChB,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC;YACpC,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC;SACrC,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,yBAAyB,CAAC,aAAa,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;QAExF,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACtF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uEAAuE,EAAE,KAAK,IAAI,EAAE;QACrF,MAAM,aAAa,GAAG;YACpB,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC;YACtE,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC;YAC/D,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC;SACzD,CAAC;QACF,MAAM,iBAAiB,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,SAAS,GAAG,CAAC,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC;QAE7D,uEAAuE;QAEvE,MAAM,MAAM,GAAG,MAAM,yBAAyB,CAAC,aAAa,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;QACxF,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAExE,8GAA8G;QAC9G,6EAA6E;QAC7E,gGAAgG;IAClG,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;QAC/D,MAAM,aAAa,GAAG;YACpB,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC;SACnF,CAAC;QAEF,MAAM,KAAK,GAAG,MAAM,yBAAyB,CAAC,aAAa,EAAE,UAAU,EAAE,YAAY,CAAC,CAAA;QACtF,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACvE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,MAAM,UAAU,GAAG;QACjB,SAAS,EAAE,GAAG;QACd,YAAY,EAAE,EAAE;QAChB,0BAA0B,EAAE,IAAI;QAChC,cAAc,EAAE,GAAG;QACnB,aAAa,EAAE,QAAQ;QACvB,SAAS,EAAE,wBAAwB;QACnC,eAAe,EAAE,QAAoB;QACrC,WAAW,EAAE,WAAW;QACxB,WAAW,EAAE,cAAc;QAC3B,kBAAkB,EAAE,IAAI;KACzB,CAAC;IAEF,MAAM,YAAY,GAAG;QACnB,SAAS,EAAE,cAAc;QACzB,aAAa,EAAE,kBAAkB;QACjC,cAAc,EAAE,IAAI;QACpB,mBAAmB,EAAE,IAAI;QACzB,qBAAqB,EAAE,IAAI;QAC3B,aAAa,EAAE,IAAI;QACnB,YAAY,EAAE,IAAI;KACnB,CAAC;IAGF,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,8BAA8B;QAC9B,MAAM,WAAW,GAAG,aAAa,CAC/B,EAAE,GAAG,UAAU,EAAE,aAAa,EAAE,QAAQ,EAAE,EAC1C,YAAY,CACb,CAAC;QACF,MAAM,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,CAAC;QAElC,4BAA4B;QAC5B,MAAM,SAAS,GAAG,aAAa,CAC7B,EAAE,GAAG,UAAU,EAAE,aAAa,EAAE,MAAM,EAAE,EACxC,YAAY,CACb,CAAC;QACF,MAAM,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;QAEhC,6BAA6B;QAC7B,MAAM,CAAC,GAAG,EAAE;YACV,aAAa,CACX,EAAE,GAAG,UAAU,EAAE,aAAa,EAAE,SAAgB,EAAE,EAClD,YAAY,CACb,CAAC;QACJ,CAAC,CAAC,CAAC,OAAO,CAAC,+CAA+C,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/services/loggingOpenAIEmbedding.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=loggingOpenAIEmbedding.d.ts.map

package/dist/services/loggingOpenAIEmbedding.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loggingOpenAIEmbedding.d.ts","sourceRoot":"","sources":["../../src/services/loggingOpenAIEmbedding.ts"],"names":[],"mappings":""}

package/dist/services/loggingOpenAIEmbedding.js

@@ -0,0 +1,41 @@
+// // temporary
+// // this is a wrapper around OpenAIEmbedding that logs the input of the embedding
+// // it's used to debug the embedding process (to make sure random metadata isn't wrongfully included)
+// // it's not used in the production code
+export {};
+// import { OpenAIEmbedding } from "@llamaindex/openai";
+// import type {
+//   OpenAI as OpenAILLM,
+// } from "openai";
+// type LLMInstance = Pick<OpenAILLM, "embeddings" | "apiKey" | "baseURL">;
+// export class LoggingOpenAIEmbedding extends OpenAIEmbedding {
+//   constructor(
+//     init?: Omit<Partial<OpenAIEmbedding>, "session"> & {
+//       session?: LLMInstance;
+//     },
+//   ) {
+//     super(init);
+//     // overwrite private member "getMessage" 🙀
+//     (this as any).getOpenAIEmbedding = async function(input: string[]): Promise<number[][]> {
+//       // TODO: ensure this for every sub class by calling it in the base class
+//       input = this.truncateMaxTokens(input);
+//       console.log("LoggingOpenAIEmbedding input", input);
+//       const { data } = await (
+//         await this.session
+//       ).embeddings.create(
+//         this.dimensions
+//           ? {
+//               model: this.model,
+//               dimensions: this.dimensions, // only sent to OpenAI if set by user
+//               input,
+//             }
+//           : {
+//               model: this.model,
+//               input,
+//             },
+//       );
+//       return data.map((d) => d.embedding);
+//     }
+//   }
+// }
+//# sourceMappingURL=loggingOpenAIEmbedding.js.map

package/dist/services/loggingOpenAIEmbedding.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"loggingOpenAIEmbedding.js","sourceRoot":"","sources":["../../src/services/loggingOpenAIEmbedding.ts"],"names":[],"mappings":"AACA,eAAe;AACf,mFAAmF;AACnF,uGAAuG;AACvG,0CAA0C;;AAE1C,wDAAwD;AACxD,gBAAgB;AAChB,yBAAyB;AACzB,mBAAmB;AACnB,2EAA2E;AAG3E,gEAAgE;AAChE,iBAAiB;AACjB,2DAA2D;AAC3D,+BAA+B;AAC/B,SAAS;AACT,QAAQ;AACR,mBAAmB;AACnB,kDAAkD;AAClD,gGAAgG;AAChG,iFAAiF;AACjF,+CAA+C;AAE/C,4DAA4D;AAE5D,iCAAiC;AACjC,6BAA6B;AAC7B,6BAA6B;AAC7B,0BAA0B;AAC1B,gBAAgB;AAChB,mCAAmC;AACnC,mFAAmF;AACnF,uBAAuB;AACvB,gBAAgB;AAChB,gBAAgB;AAChB,mCAAmC;AACnC,uBAAuB;AACvB,iBAAiB;AACjB,WAAW;AAEX,6CAA6C;AAC7C,QAAQ;AACR,MAAM;AACN,IAAI"}

package/dist/services/mockEmbedding.d.ts

@@ -0,0 +1,6 @@
+import { BaseEmbedding } from "llamaindex";
+export declare class MockEmbedding extends BaseEmbedding {
+    constructor();
+    getTextEmbedding(text: string): Promise<number[]>;
+}
+//# sourceMappingURL=mockEmbedding.d.ts.map