@meaningfully/core 0.1.0

package/dist/api/embedding.js

@@ -0,0 +1,94 @@
+import { transformDocumentsToNodes, estimateCost, searchDocuments, getExistingVectorStoreIndex, persistNodes, persistDocuments, getExistingDocStore } from "../services/embeddings.js";
+import { loadDocumentsFromCsv } from "../services/csvLoader.js";
+import { MetadataMode } from "llamaindex";
+import { ProgressManager } from "../services/progressManager.js";
+export async function createEmbeddings(csvPath, textColumnName, config, settings, clients) {
+    try {
+        console.time("createEmbeddings Run Time");
+        const operationId = `embed-${Date.now()}`;
+        const progressManager = ProgressManager.getInstance();
+        progressManager.startOperation(operationId, 100);
+        const documents = await loadDocumentsFromCsv(csvPath, textColumnName);
+        if (documents.length === 0) {
+            progressManager.clearOperation(operationId);
+            console.timeEnd("createEmbeddings Run Time");
+            return {
+                success: false,
+                error: "That CSV does not appear to contain any documents. Please check the file and try again.",
+            };
+        }
+        progressManager.updateProgress(operationId, 5);
+        const nodes = await transformDocumentsToNodes(documents, config);
+        const [index] = await Promise.all([
+            persistNodes(nodes, config, settings, clients, (progress, total) => {
+                const percentage = Math.floor((progress / total) * 90) + 5; // Map to 5-95% of total progress
+                progressManager.updateProgress(operationId, percentage);
+            }),
+            persistDocuments(documents, config, settings, clients)
+        ]);
+        progressManager.completeOperation(operationId);
+        console.timeEnd("createEmbeddings Run Time");
+        return {
+            success: true,
+            index,
+        };
+    }
+    catch (error) {
+        return {
+            success: false,
+            error: error instanceof Error ? error.message : "Unknown error occurred",
+        };
+    }
+}
+// TODO: rename this to be parallel to createEmbeddings
+export async function previewResults(csvPath, textColumnName, config) {
+    try {
+        const documents = await loadDocumentsFromCsv(csvPath, textColumnName);
+        if (documents.length === 0) {
+            return {
+                success: false,
+                error: "That CSV does not appear to contain any documents. Please check the file and try again.",
+            };
+        }
+        // Take 10 rows from the middle of the dataset for preview
+        // we take a consistent 10 so that the results of the preview are consistent (i.e. with a larger chunk size, you have fewer, longer results, but more shorter ones if you adjust it)
+        // and we take from the middle because the initial rows may be idiosyncratic.
+        const previewDocumentsSubset = documents.slice(Math.floor(documents.length / 2), Math.floor(documents.length / 2) + 10);
+        const previewNodes = await transformDocumentsToNodes(documents, config);
+        const previewSubsetNodes = await transformDocumentsToNodes(previewDocumentsSubset, config);
+        const { estimatedPrice, tokenCount, pricePer1M } = estimateCost(previewNodes, config.modelName);
+        return {
+            success: true,
+            nodes: previewSubsetNodes.map((node) => ({
+                text: node.text,
+                metadata: node.metadata
+            })),
+            estimatedPrice,
+            tokenCount,
+            pricePer1M
+        };
+    }
+    catch (error) {
+        return {
+            success: false,
+            error: error instanceof Error ? error.message : "Unknown error occurred"
+        };
+    }
+}
+export async function getDocStore(config) {
+    return await getExistingDocStore(config);
+}
+export async function getIndex(config, settings, clients) {
+    return await getExistingVectorStoreIndex(config, settings, clients);
+}
+export async function search(index, query, numResults = 10, filters) {
+    const results = await searchDocuments(index, query, numResults, filters);
+    return results.map((result) => ({
+        text: result.node.getContent(MetadataMode.NONE),
+        score: result.score ?? 0,
+        metadata: result.node.metadata,
+        //  @ts-ignore
+        sourceNodeId: result.node.relationships?.SOURCE?.nodeId
+    }));
+}
+//# sourceMappingURL=embedding.js.map

package/dist/api/embedding.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding.js","sourceRoot":"","sources":["../../src/api/embedding.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,YAAY,EAAE,eAAe,EAAE,2BAA2B,EAAE,YAAY,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAEvL,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAEjE,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,OAAe,EACf,cAAsB,EACtB,MAAuB,EACvB,QAAkB,EAClB,OAAgB;IAEhB,IAAI,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,SAAS,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;QAC1C,MAAM,eAAe,GAAG,eAAe,CAAC,WAAW,EAAE,CAAC;QACtD,eAAe,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;QAEjD,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QACtE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,eAAe,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;YAC5C,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;YAC7C,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,yFAAyF;aACjG,CAAC;QACJ,CAAC;QAED,eAAe,CAAC,cAAc,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAE/C,MAAM,KAAK,GAAG,MAAM,yBAAyB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAEjE,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAChC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE;gBACjE,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,iCAAiC;gBAC7F,eAAe,CAAC,cAAc,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;YAC1D,CAAC,CAAC;YACF,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC;SACvD,CAAC,CAAC;QAEH,eAAe,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAC/C,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;QAC7C,OAAO;YACL,OAAO,EAAE,IAAI;YACb,KAAK;SACN,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;SACzE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,uDAAuD;AACvD,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAAe,EACf,cAAsB,EACtB,MAAuB;IAEvB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QACtE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,yFAAyF;aACjG,CAAC;QACJ,CAAC;QACD,0DAA0D;QAC1D,oLAAoL;QACpL,6EAA6E;QAC7E,MAAM,sBAAsB,GAAG,SAAS,CAAC,KAAK,CAC5C,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,EAChC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,EAAE,CACtC,CAAC;QAEF,MAAM,YAAY,GAAG,MAAM,yBAAyB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACxE,MAAM,kBAAkB,GAAG,MAAM,yBAAyB,CAAC,sBAAsB,EAAE,MAAM,CAAC,CAAC;QAC3F,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;QAEhG,OAAO;YACL,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,kBAAkB,CAAC,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE,CAAC,CAAC;gBAC5C,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;aACxB,CAAC,CAAC;YACH,cAAc;YACd,UAAU;YACV,UAAU;SACX,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;SACzE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAAuB;IACvD,OAAO,MAAM,mBAAmB,CAAC,MAAM,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC1F,OAAO,MAAM,2BAA2B,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;AACtE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,KAAU,EACV,KAAa,EACb,aAAqB,EAAE,EACvB,OAA0B;IAE1B,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IACzE,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAW,EAAE,EAAE,CAAC,CAAC;QACnC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,IAAI,CAAC;QAC/C,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,CAAC;QACxB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;QAC9B,cAAc;QACd,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,EAAE,MAAM;KACxD,CAAC,CAAC,CAAC;AACN,CAAC"}

package/dist/api/embedding.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=embedding.test.d.ts.map

package/dist/api/embedding.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding.test.d.ts","sourceRoot":"","sources":["../../src/api/embedding.test.ts"],"names":[],"mappings":""}

package/dist/api/embedding.test.js

@@ -0,0 +1,340 @@
+//@ts-nocheck
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { createEmbeddings, previewResults, getDocStore, getIndex, search } from './embedding';
+import { loadDocumentsFromCsv } from '../services/csvLoader';
+import { transformDocumentsToNodes, estimateCost, searchDocuments, getExistingVectorStoreIndex, persistNodes, getExistingDocStore } from '../services/embeddings';
+// filepath: /Users/jeremybmerrill/code/meaningfully/src/main/api/embedding.test.ts
+vi.mock('../services/csvLoader');
+vi.mock('../services/embeddings');
+describe('embedding.ts', () => {
+    describe('createEmbeddings', () => {
+        it('should create embeddings and return success', async () => {
+            const mockDocuments = [{ text: 'doc1' }, { text: 'doc2' }];
+            const mockNodes = [{ node: 'node1' }, { node: 'node2' }];
+            const mockIndex = 'index1';
+            loadDocumentsFromCsv.mockResolvedValue(mockDocuments);
+            transformDocumentsToNodes.mockResolvedValue(mockNodes);
+            persistNodes.mockResolvedValue(mockIndex);
+            const result = await createEmbeddings('path/to/csv', 'text', {}, {});
+            expect(result).toEqual({ success: true, index: mockIndex });
+        });
+        it('should return error on failure', async () => {
+            loadDocumentsFromCsv.mockRejectedValue(new Error('Failed to load documents'));
+            const result = await createEmbeddings('path/to/csv', 'text', {}, {});
+            expect(result).toEqual({ success: false, error: 'Failed to load documents' });
+        });
+        it('should handle empty documents', async () => {
+            loadDocumentsFromCsv.mockResolvedValue([]);
+            const result = await createEmbeddings('path/to/csv', 'text', {}, {});
+            expect(result).toEqual({ success: false, error: 'That CSV does not appear to contain any documents. Please check the file and try again.' });
+        });
+    });
+    describe('previewResults', () => {
+        it('should return preview results and estimated cost', async () => {
+            const mockDocuments = Array(20).fill({ text: 'doc' });
+            const mockNodes = [{ text: 'node1', metadata: {} }, { text: 'node2', metadata: {} }];
+            const mockPreviewNodes = [{ text: 'node1', metadata: {} }, { text: 'node2', metadata: {} }];
+            const mockEstimate = { estimatedPrice: 10, tokenCount: 100, pricePer1M: 0.01 };
+            loadDocumentsFromCsv.mockResolvedValue(mockDocuments);
+            transformDocumentsToNodes.mockResolvedValue(mockNodes);
+            estimateCost.mockReturnValue(mockEstimate);
+            const result = await previewResults('path/to/csv', 'text', {});
+            expect(result).toEqual({
+                success: true,
+                nodes: mockPreviewNodes,
+                ...mockEstimate
+            });
+        });
+        it('should return error on failure', async () => {
+            loadDocumentsFromCsv.mockRejectedValue(new Error('Failed to load documents'));
+            const result = await previewResults('path/to/csv', 'text', {});
+            expect(result).toEqual({ success: false, error: 'Failed to load documents' });
+        });
+        it('should handle empty documents', async () => {
+            loadDocumentsFromCsv.mockResolvedValue([]);
+            const result = await previewResults('path/to/csv', 'text', {});
+            expect(result).toEqual({ success: false, error: 'That CSV does not appear to contain any documents. Please check the file and try again.' });
+        });
+    });
+    describe('getDocStore', () => {
+        it('should return existing doc store', async () => {
+            const mockDocStore = 'docStore';
+            getExistingDocStore.mockResolvedValue(mockDocStore);
+            const result = await getDocStore({});
+            expect(result).toBe(mockDocStore);
+        });
+    });
+    describe('getIndex', () => {
+        it('should return existing vector store index', async () => {
+            const mockIndex = 'index';
+            getExistingVectorStoreIndex.mockResolvedValue(mockIndex);
+            const result = await getIndex({}, {});
+            expect(result).toBe(mockIndex);
+        });
+    });
+    describe('search', () => {
+        it('should return search results', async () => {
+            const mockResults = [
+                { node: { getContent: () => 'content1', metadata: {} }, score: 1 },
+                { node: { getContent: () => 'content2', metadata: {} }, score: 2 }
+            ];
+            searchDocuments.mockResolvedValue(mockResults);
+            const result = await search('index', 'query');
+            expect(result).toEqual([
+                { text: 'content1', score: 1, metadata: {} },
+                { text: 'content2', score: 2, metadata: {} }
+            ]);
+        });
+        it('should handle no search results', async () => {
+            searchDocuments.mockResolvedValue([]);
+            const result = await search('index', 'query');
+            expect(result).toEqual([]);
+        });
+        it('should handle search results with null scores', async () => {
+            const mockResults = [
+                { node: { getContent: () => 'content1', metadata: {} }, score: null },
+                { node: { getContent: () => 'content2', metadata: {} }, score: null }
+            ];
+            searchDocuments.mockResolvedValue(mockResults);
+            const result = await search('index', 'query');
+            expect(result).toEqual([
+                { text: 'content1', score: 0, metadata: {} },
+                { text: 'content2', score: 0, metadata: {} }
+            ]);
+        });
+    });
+});
+describe('previewResults', () => {
+    it('should return preview results and estimated cost', async () => {
+        const mockDocuments = Array(20).fill({ text: 'doc' });
+        const mockNodes = [{ text: 'node1', metadata: {} }, { text: 'node2', metadata: {} }];
+        const mockPreviewNodes = [{ text: 'node1', metadata: {} }, { text: 'node2', metadata: {} }];
+        const mockEstimate = { estimatedPrice: 10, tokenCount: 100, pricePer1M: 0.01 };
+        loadDocumentsFromCsv.mockResolvedValue(mockDocuments);
+        transformDocumentsToNodes.mockResolvedValue(mockNodes);
+        estimateCost.mockReturnValue(mockEstimate);
+        const result = await previewResults('path/to/csv', 'text', {});
+        expect(result).toEqual({
+            success: true,
+            nodes: mockPreviewNodes,
+            ...mockEstimate
+        });
+    });
+    it('should return error on failure', async () => {
+        loadDocumentsFromCsv.mockRejectedValue(new Error('Failed to load documents'));
+        const result = await previewResults('path/to/csv', 'text', {});
+        expect(result).toEqual({ success: false, error: 'Failed to load documents' });
+    });
+});
+describe('getDocStore', () => {
+    it('should return existing doc store', async () => {
+        const mockDocStore = 'docStore';
+        getExistingDocStore.mockResolvedValue(mockDocStore);
+        const result = await getDocStore({});
+        expect(result).toBe(mockDocStore);
+    });
+});
+describe('getIndex', () => {
+    it('should return existing vector store index', async () => {
+        const mockIndex = 'index';
+        getExistingVectorStoreIndex.mockResolvedValue(mockIndex);
+        const result = await getIndex({}, {});
+        expect(result).toBe(mockIndex);
+    });
+});
+describe('search', () => {
+    it('should return search results', async () => {
+        const mockResults = [
+            { node: { getContent: () => 'content1', metadata: {} }, score: 1 },
+            { node: { getContent: () => 'content2', metadata: {} }, score: 2 }
+        ];
+        searchDocuments.mockResolvedValue(mockResults);
+        const result = await search('index', 'query');
+        expect(result).toEqual([
+            { text: 'content1', score: 1, metadata: {} },
+            { text: 'content2', score: 2, metadata: {} }
+        ]);
+    });
+    it('should handle no search results', async () => {
+        searchDocuments.mockResolvedValue([]);
+        const result = await search('index', 'query');
+        expect(result).toEqual([]);
+    });
+    it('should handle search results with null scores', async () => {
+        const mockResults = [
+            { node: { getContent: () => 'content1', metadata: {} }, score: null },
+            { node: { getContent: () => 'content2', metadata: {} }, score: null }
+        ];
+        searchDocuments.mockResolvedValue(mockResults);
+        const result = await search('index', 'query');
+        expect(result).toEqual([
+            { text: 'content1', score: 0, metadata: {} },
+            { text: 'content2', score: 0, metadata: {} }
+        ]);
+    });
+});
+describe('createEmbeddings with progress tracking', () => {
+    beforeEach(() => {
+        vi.clearAllMocks();
+    });
+    it('should pass progress callback to persistNodes', async () => {
+        // Setup mocks
+        const mockDocuments = [{ text: 'doc1' }, { text: 'doc2' }];
+        const mockNodes = [{ text: 'node1', metadata: {} }, { text: 'node2', metadata: {} }];
+        const mockIndex = 'index1';
+        loadDocumentsFromCsv.mockResolvedValue(mockDocuments);
+        transformDocumentsToNodes.mockResolvedValue(mockNodes);
+        persistNodes.mockImplementation((nodes, config, settings, clients, callback) => {
+            // Call the callback with sample progress
+            if (callback) {
+                callback(1, 2); // 50% progress
+                callback(2, 2); // 100% progress
+            }
+            return Promise.resolve(mockIndex);
+        });
+        // Execute
+        const result = await createEmbeddings('path/to/csv', 'text', { modelName: 'test-model' }, {}, {});
+        // Verify
+        expect(persistNodes).toHaveBeenCalledTimes(1);
+        expect(persistNodes.mock.calls[0][4]).toBeInstanceOf(Function); // Verify callback was passed
+        expect(result).toEqual({ success: true, index: mockIndex });
+    });
+    it('should correctly track progress through ProgressManager', async () => {
+        // Setup
+        vi.mock('../services/progressManager', () => {
+            const mockInstance = {
+                startOperation: vi.fn(),
+                updateProgress: vi.fn(),
+                completeOperation: vi.fn(),
+                clearOperation: vi.fn()
+            };
+            return {
+                ProgressManager: {
+                    getInstance: () => mockInstance
+                }
+            };
+        });
+        // Re-import to use mocked version
+        const { createEmbeddings } = await import('./embedding');
+        const { ProgressManager } = await import('../services/progressManager');
+        const mockDocuments = [{ text: 'doc1' }, { text: 'doc2' }];
+        const mockNodes = [{ text: 'node1', metadata: {} }];
+        const mockIndex = 'testIndex';
+        loadDocumentsFromCsv.mockResolvedValue(mockDocuments);
+        transformDocumentsToNodes.mockResolvedValue(mockNodes);
+        persistNodes.mockImplementation((nodes, config, settings, clients, callback) => {
+            if (callback)
+                callback(1, 2); // Call with 50% progress
+            return Promise.resolve(mockIndex);
+        });
+        // Execute
+        await createEmbeddings('path/to/csv', 'text', {}, {}, {});
+        // Verify
+        const progressManager = ProgressManager.getInstance();
+        expect(progressManager.startOperation).toHaveBeenCalledWith(expect.stringMatching(/^embed-\d+$/), 100);
+        expect(progressManager.updateProgress).toHaveBeenCalledWith(expect.any(String), 5);
+        expect(progressManager.updateProgress).toHaveBeenCalledWith(expect.any(String), expect.any(Number));
+        expect(progressManager.completeOperation).toHaveBeenCalledWith(expect.any(String));
+    });
+    it('should properly calculate percentage in progress callback', async () => {
+        // Setup mocks with spy on updateProgress
+        vi.mock('../services/progressManager', () => {
+            const mockInstance = {
+                startOperation: vi.fn(),
+                updateProgress: vi.fn(),
+                completeOperation: vi.fn(),
+                clearOperation: vi.fn()
+            };
+            return {
+                ProgressManager: {
+                    getInstance: () => mockInstance
+                }
+            };
+        });
+        // Re-import to use mocked version
+        const { createEmbeddings } = await import('./embedding');
+        const { ProgressManager } = await import('../services/progressManager');
+        const mockDocuments = [{ text: 'doc1' }];
+        const mockNodes = [{ text: 'node1', metadata: {} }];
+        loadDocumentsFromCsv.mockResolvedValue(mockDocuments);
+        transformDocumentsToNodes.mockResolvedValue(mockNodes);
+        // Simulate persistNodes calling the callback with various progress values
+        persistNodes.mockImplementation((nodes, config, settings, clients, callback) => {
+            if (callback) {
+                callback(0, 10); // 0% progress
+                callback(5, 10); // 50% progress
+                callback(10, 10); // 100% progress
+            }
+            return Promise.resolve('mockIndex');
+        });
+        // Execute
+        await createEmbeddings('path/to/csv', 'text', {}, {}, {});
+        // Verify percentage calculations
+        // Initial update at 5%
+        const progressManager = ProgressManager.getInstance();
+        expect(progressManager.updateProgress).toHaveBeenCalledWith(expect.any(String), 5);
+        // Progress updates: 0%, 50%, 100% mapped to 5-95% range
+        expect(progressManager.updateProgress).toHaveBeenCalledWith(expect.any(String), 5); // 0% -> 5%
+        expect(progressManager.updateProgress).toHaveBeenCalledWith(expect.any(String), 50); // 50% -> 50%
+        expect(progressManager.updateProgress).toHaveBeenCalledWith(expect.any(String), 95); // 100% -> 95%
+    });
+    it('should clear operation on empty documents', async () => {
+        // Setup
+        vi.mock('../services/progressManager', () => {
+            const mockInstance = {
+                startOperation: vi.fn(),
+                updateProgress: vi.fn(),
+                completeOperation: vi.fn(),
+                clearOperation: vi.fn()
+            };
+            return {
+                ProgressManager: {
+                    getInstance: () => mockInstance
+                }
+            };
+        });
+        // Re-import to use mocked version
+        const { createEmbeddings } = await import('./embedding');
+        const { ProgressManager } = await import('../services/progressManager');
+        loadDocumentsFromCsv.mockResolvedValue([]);
+        // Execute
+        const result = await createEmbeddings('path/to/csv', 'text', {}, {}, {});
+        // Verify
+        const progressManager = ProgressManager.getInstance();
+        expect(progressManager.clearOperation).toHaveBeenCalled();
+        expect(result).toEqual({
+            success: false,
+            error: "That CSV does not appear to contain any documents. Please check the file and try again."
+        });
+    });
+    it('shoulde complete operation on successful embedding', async () => {
+        // Setup
+        vi.mock('../services/progressManager', () => {
+            const mockInstance = {
+                startOperation: vi.fn(),
+                updateProgress: vi.fn(),
+                completeOperation: vi.fn(),
+                clearOperation: vi.fn()
+            };
+            return {
+                ProgressManager: {
+                    getInstance: () => mockInstance
+                }
+            };
+        });
+        // Re-import to use mocked version
+        const { createEmbeddings } = await import('./embedding');
+        const { ProgressManager } = await import('../services/progressManager');
+        const mockDocuments = [{ text: 'doc1' }];
+        const mockNodes = [{ text: 'node1', metadata: {} }];
+        loadDocumentsFromCsv.mockResolvedValue(mockDocuments);
+        transformDocumentsToNodes.mockResolvedValue(mockNodes);
+        persistNodes.mockResolvedValue('mockIndex');
+        // Execute
+        await createEmbeddings('path/to/csv', 'text', {}, {}, {});
+        // Verify
+        const progressManager = ProgressManager.getInstance();
+        expect(progressManager.completeOperation).toHaveBeenCalled();
+    });
+});
+//# sourceMappingURL=embedding.test.js.map

package/dist/api/embedding.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding.test.js","sourceRoot":"","sources":["../../src/api/embedding.test.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC9D,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC9F,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,yBAAyB,EAAE,YAAY,EAAE,eAAe,EAAE,2BAA2B,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAGlK,mFAAmF;AAGnF,EAAE,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;AACjC,EAAE,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;AAElC,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC1B,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,aAAa,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;YAC3D,MAAM,SAAS,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;YACzD,MAAM,SAAS,GAAG,QAAQ,CAAC;YAC3B,oBAAoB,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;YACtD,yBAAyB,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YACvD,YAAY,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAE1C,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;YAErE,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC5C,oBAAoB,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC,CAAC;YAE9E,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;YAErE,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,0BAA0B,EAAE,CAAC,CAAC;QAClF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;YAC3C,oBAAoB,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;YAE3C,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;YAErE,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,yFAAyF,EAAE,CAAC,CAAC;QACjJ,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAC9D,MAAM,aAAa,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YACtD,MAAM,SAAS,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YACrF,MAAM,gBAAgB,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YAC5F,MAAM,YAAY,GAAG,EAAE,cAAc,EAAE,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;YAC/E,oBAAoB,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;YACtD,yBAAyB,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YACvD,YAAY,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;YAE3C,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;YAE/D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACnB,OAAO,EAAE,IAAI;gBACb,KAAK,EAAE,gBAAgB;gBACvB,GAAG,YAAY;aAClB,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC5C,oBAAoB,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC,CAAC;YAE9E,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;YAE/D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,0BAA0B,EAAE,CAAC,CAAC;QAClF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;YAC3C,oBAAoB,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;YAE3C,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;YAE/D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,yFAAyF,EAAE,CAAC,CAAC;QACjJ,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QACzB,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,YAAY,GAAG,UAAU,CAAC;YAChC,mBAAmB,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAEpD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,EAAE,CAAC,CAAC;YAErC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,UAAU,EAAE,GAAG,EAAE;QACtB,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,SAAS,GAAG,OAAO,CAAC;YAC1B,2BAA2B,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAEzD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;YAEtC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE;QACpB,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC1C,MAAM,WAAW,GAAG;gBAChB,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE;gBAClE,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE;aACrE,CAAC;YACF,eAAe,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;YAE/C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAE9C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACnB,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;gBAC5C,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;aAC/C,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;YAC7C,eAAe,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;YAEtC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAE9C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC/B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,WAAW,GAAG;gBAChB,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;gBACrE,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;aACxE,CAAC;YACF,eAAe,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;YAE/C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAE9C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACnB,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;gBAC5C,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;aAC/C,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC;AAED,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;QAChE,MAAM,aAAa,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACtD,MAAM,SAAS,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QACrF,MAAM,gBAAgB,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QAC5F,MAAM,YAAY,GAAG,EAAE,cAAc,EAAE,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;QAC/E,oBAAoB,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;QACtD,yBAAyB,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QACvD,YAAY,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;QAE/D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,gBAAgB;YACvB,GAAG,YAAY;SAChB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;QAC9C,oBAAoB,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC,CAAC;QAE9E,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;QAE/D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,0BAA0B,EAAE,CAAC,CAAC;IAChF,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;QAChD,MAAM,YAAY,GAAG,UAAU,CAAC;QAChC,mBAAmB,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;QAEpD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,EAAE,CAAC,CAAC;QAErC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,UAAU,EAAE,GAAG,EAAE;IACxB,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,SAAS,GAAG,OAAO,CAAC;QAC1B,2BAA2B,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QAEzD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QAEtC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AACD,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE;IACpB,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;QAC9C,MAAM,WAAW,GAAG;YAChB,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE;YAClE,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE;SACrE,CAAC;QACF,eAAe,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAE/C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAE9C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACnB,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;YAC5C,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;SAC/C,CAAC,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;QACjD,eAAe,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;QAEtC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAE9C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;QAC/D,MAAM,WAAW,GAAG;YAChB,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;YACrE,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;SACxE,CAAC;QACF,eAAe,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAE/C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAE9C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACnB,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;YAC5C,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;SAC/C,CAAC,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC;AACH,QAAQ,CAAC,yCAAyC,EAAE,GAAG,EAAE;IACvD,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;IACrB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;QAC7D,cAAc;QACd,MAAM,aAAa,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QAC3D,MAAM,SAAS,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QACrF,MAAM,SAAS,GAAG,QAAQ,CAAC;QAC3B,oBAAoB,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;QACtD,yBAAyB,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QACvD,YAAY,CAAC,kBAAkB,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;YAC7E,yCAAyC;YACzC,IAAI,QAAQ,EAAE,CAAC;gBACb,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,eAAe;gBAChC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,gBAAgB;YACnC,CAAC;YACD,OAAO,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,UAAU;QACV,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,YAAY,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QAElG,SAAS;QACT,MAAM,CAAC,YAAY,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC9C,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC,6BAA6B;QAC7F,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;QACvE,QAAQ;QACR,EAAE,CAAC,IAAI,CAAC,6BAA6B,EAAE,GAAG,EAAE;YAC1C,MAAM,YAAY,GAAG;gBACnB,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;gBACvB,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;gBACvB,iBAAiB,EAAE,EAAE,CAAC,EAAE,EAAE;gBAC1B,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;aACxB,CAAC;YAEF,OAAO;gBACL,eAAe,EAAE;oBACf,WAAW,EAAE,GAAG,EAAE,CAAC,YAAY;iBAChC;aACF,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,kCAAkC;QAClC,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QACzD,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,6BAA6B,CAAC,CAAC;QAExE,MAAM,aAAa,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QAC3D,MAAM,SAAS,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QACpD,MAAM,SAAS,GAAG,WAAW,CAAC;QAE9B,oBAAoB,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;QACtD,yBAAyB,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QACvD,YAAY,CAAC,kBAAkB,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;YAC7E,IAAI,QAAQ;gBAAE,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,yBAAyB;YACvD,OAAO,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,UAAU;QACV,MAAM,gBAAgB,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QAE1D,SAAS;QACT,MAAM,eAAe,GAAG,eAAe,CAAC,WAAW,EAAE,CAAC;QACtD,MAAM,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,cAAc,CAAC,aAAa,CAAC,EAAE,GAAG,CAAC,CAAC;QACvG,MAAM,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QACnF,MAAM,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QACpG,MAAM,CAAC,eAAe,CAAC,iBAAiB,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IACrF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;QACzE,yCAAyC;QACzC,EAAE,CAAC,IAAI,CAAC,6BAA6B,EAAE,GAAG,EAAE;YAC1C,MAAM,YAAY,GAAG;gBACnB,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;gBACvB,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;gBACvB,iBAAiB,EAAE,EAAE,CAAC,EAAE,EAAE;gBAC1B,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;aACxB,CAAC;YAEF,OAAO;gBACL,eAAe,EAAE;oBACf,WAAW,EAAE,GAAG,EAAE,CAAC,YAAY;iBAChC;aACF,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,kCAAkC;QAClC,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QACzD,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,6BAA6B,CAAC,CAAC;QAExE,MAAM,aAAa,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QACzC,MAAM,SAAS,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QACpD,oBAAoB,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;QACtD,yBAAyB,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QAEvD,0EAA0E;QAC1E,YAAY,CAAC,kBAAkB,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;YAC7E,IAAI,QAAQ,EAAE,CAAC;gBACb,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAE,cAAc;gBAChC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAE,eAAe;gBACjC,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB;YACpC,CAAC;YACD,OAAO,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,UAAU;QACV,MAAM,gBAAgB,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QAE1D,iCAAiC;QACjC,uBAAuB;QACvB,MAAM,eAAe,GAAG,eAAe,CAAC,WAAW,EAAE,CAAC;QACtD,MAAM,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAEnF,wDAAwD;QACxD,MAAM,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAG,WAAW;QACjG,MAAM,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,CAAE,aAAa;QACnG,MAAM,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,CAAE,cAAc;IACtG,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,QAAQ;QACR,EAAE,CAAC,IAAI,CAAC,6BAA6B,EAAE,GAAG,EAAE;YAC1C,MAAM,YAAY,GAAG;gBACnB,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;gBACvB,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;gBACvB,iBAAiB,EAAE,EAAE,CAAC,EAAE,EAAE;gBAC1B,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;aACxB,CAAC;YAEF,OAAO;gBACL,eAAe,EAAE;oBACf,WAAW,EAAE,GAAG,EAAE,CAAC,YAAY;iBAChC;aACF,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,kCAAkC;QAClC,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QACzD,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,6BAA6B,CAAC,CAAC;QAExE,oBAAoB,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;QAE3C,UAAU;QACV,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QAEzE,SAAS;QACT,MAAM,eAAe,GAAG,eAAe,CAAC,WAAW,EAAE,CAAC;QACtD,MAAM,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,gBAAgB,EAAE,CAAC;QAC1D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,yFAAyF;SACjG,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;QAClE,QAAQ;QACR,EAAE,CAAC,IAAI,CAAC,6BAA6B,EAAE,GAAG,EAAE;YAC1C,MAAM,YAAY,GAAG;gBACnB,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;gBACvB,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;gBACvB,iBAAiB,EAAE,EAAE,CAAC,EAAE,EAAE;gBAC1B,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;aACxB,CAAC;YAEF,OAAO;gBACL,eAAe,EAAE;oBACf,WAAW,EAAE,GAAG,EAAE,CAAC,YAAY;iBAChC;aACF,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,kCAAkC;QAClC,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QACzD,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,6BAA6B,CAAC,CAAC;QAExE,MAAM,aAAa,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QACzC,MAAM,SAAS,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QACpD,oBAAoB,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;QACtD,yBAAyB,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QACvD,YAAY,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAE5C,UAAU;QACV,MAAM,gBAAgB,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QAE1D,SAAS;QACT,MAAM,eAAe,GAAG,eAAe,CAAC,WAAW,EAAE,CAAC;QACtD,MAAM,CAAC,eAAe,CAAC,iBAAiB,CAAC,CAAC,gBAAgB,EAAE,CAAC;IAC/D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,5 @@
+export * from './Meaningfully.js';
+export * from './MetadataManager.js';
+export * from './services/progressManager.js';
+export * from './types/index.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,sBAAsB,CAAC;AACrC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,kBAAkB,CAAC"}

package/dist/index.js

@@ -0,0 +1,6 @@
+export * from './Meaningfully.js';
+export * from './MetadataManager.js';
+export * from './services/progressManager.js';
+export * from './types/index.js';
+// src/index.ts
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,sBAAsB,CAAC;AACrC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,kBAAkB,CAAC;AACjC,eAAe"}

package/dist/services/batchingWeaviateVectorStore.d.ts

@@ -0,0 +1,6 @@
+import { WeaviateVectorStore } from '@llamaindex/weaviate';
+import { BaseNode } from 'llamaindex';
+export declare class BatchingWeaviateVectorStore extends WeaviateVectorStore {
+    add(nodes: BaseNode[]): Promise<string[]>;
+}
+//# sourceMappingURL=batchingWeaviateVectorStore.d.ts.map

package/dist/services/batchingWeaviateVectorStore.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"batchingWeaviateVectorStore.d.ts","sourceRoot":"","sources":["../../src/services/batchingWeaviateVectorStore.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAUtC,qBAAa,2BAA4B,SAAQ,mBAAmB;IAC5D,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;CAUhD"}

package/dist/services/batchingWeaviateVectorStore.js

@@ -0,0 +1,21 @@
+import { WeaviateVectorStore } from '@llamaindex/weaviate';
+/*
+
+Patched version of WeaviateVectorStore to handle large batches by splitting into smaller chunks.
+
+When I loaded a large-ish (5.4MB) spreadsheet, I got a Weaviate error about trying to load too much data at once.
+
+*/
+export class BatchingWeaviateVectorStore extends WeaviateVectorStore {
+    async add(nodes) {
+        const batchSize = 100; // Define the batch size
+        const results = []; // Collect results from each batch
+        for (let i = 0; i < nodes.length; i += batchSize) {
+            const batch = nodes.slice(i, i + batchSize);
+            const batchResults = await super.add(batch); // Call the parent class's add method for each batch
+            results.push(...batchResults); // Aggregate results
+        }
+        return results; // Return aggregated results
+    }
+}
+//# sourceMappingURL=batchingWeaviateVectorStore.js.map

package/dist/services/batchingWeaviateVectorStore.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"batchingWeaviateVectorStore.js","sourceRoot":"","sources":["../../src/services/batchingWeaviateVectorStore.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAG3D;;;;;;EAME;AAEF,MAAM,OAAO,2BAA4B,SAAQ,mBAAmB;IAClE,KAAK,CAAC,GAAG,CAAC,KAAiB;QACzB,MAAM,SAAS,GAAG,GAAG,CAAC,CAAC,wBAAwB;QAC/C,MAAM,OAAO,GAAa,EAAE,CAAC,CAAC,kCAAkC;QAChE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAC5C,MAAM,YAAY,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,oDAAoD;YACjG,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC,CAAC,oBAAoB;QACrD,CAAC;QACD,OAAO,OAAO,CAAC,CAAC,4BAA4B;IAC9C,CAAC;CACF"}

package/dist/services/csvLoader.d.ts

@@ -0,0 +1,3 @@
+import { Document } from "llamaindex";
+export declare function loadDocumentsFromCsv(filePath: string, textColumnName: string): Promise<Document[]>;
+//# sourceMappingURL=csvLoader.d.ts.map

package/dist/services/csvLoader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"csvLoader.d.ts","sourceRoot":"","sources":["../../src/services/csvLoader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAItC,wBAAsB,oBAAoB,CACxC,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAgBrB"}

package/dist/services/csvLoader.js

@@ -0,0 +1,18 @@
+import { Document } from "llamaindex";
+import { readFileSync } from "fs";
+import Papa from "papaparse";
+export async function loadDocumentsFromCsv(filePath, textColumnName) {
+    const fileContent = readFileSync(filePath, "utf-8");
+    const { data: records } = Papa.parse(fileContent, {
+        header: true,
+        skipEmptyLines: true,
+    });
+    return records.map((record) => {
+        const { [textColumnName]: text, ...metadata } = record;
+        return new Document({
+            text,
+            metadata: Object.fromEntries(Object.entries(metadata).map(([k, v]) => [k, v ?? ""])),
+        });
+    });
+}
+//# sourceMappingURL=csvLoader.js.map

package/dist/services/csvLoader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"csvLoader.js","sourceRoot":"","sources":["../../src/services/csvLoader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,QAAgB,EAChB,cAAsB;IAEtB,MAAM,WAAW,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACpD,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE;QAChD,MAAM,EAAE,IAAI;QACZ,cAAc,EAAE,IAAI;KACrB,CAAC,CAAC;IAEH,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAW,EAAE,EAAE;QACjC,MAAM,EAAE,CAAC,cAAc,CAAC,EAAE,IAAI,EAAE,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC;QACvD,OAAO,IAAI,QAAQ,CAAC;YAClB,IAAI;YACJ,QAAQ,EAAE,MAAM,CAAC,WAAW,CAC1B,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CACvD;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/services/csvLoader.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=csvLoader.test.d.ts.map

package/dist/services/csvLoader.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"csvLoader.test.d.ts","sourceRoot":"","sources":["../../src/services/csvLoader.test.ts"],"names":[],"mappings":""}