@meaningfully/core 0.1.2 → 0.1.4

package/README.md

@@ -1,3 +1,12 @@
 # meaningfully-core
 
-This is a library of core/API components for [meaningfully](https://www.github.com/jeremybmerrill/meaningfully), which is an app (or collection of apps!) for semantic search over spreadsheets. See the main repo URL for more discussion.
+This is a library of core/API components for [meaningfully](https://www.github.com/jeremybmerrill/meaningfully), which is an app (or collection of apps!) for semantic search over spreadsheets. See the main repo URL for more discussion.
+
+## How to build:
+
+`npm run build`
+
+### How to build on any file modification
+
+`npm run watch`
+

package/dist/Meaningfully.d.ts

@@ -5,7 +5,7 @@ type HasFilePath = {
 };
 type DocumentSetParamsFilePath = DocumentSetParams & HasFilePath;
 export declare class MeaningfullyAPI {
-    private manager;
+    private metadataManager;
     private storagePath;
     private clients;
     constructor({ storagePath, weaviateClient, postgresClient, metadataManager }: {
@@ -48,6 +48,13 @@ export declare class MeaningfullyAPI {
     setMaskedSettings(newSettings: Settings): Promise<{
         success: boolean;
     }>;
+    deletePostgresVectorStore(projectName: string): Promise<void>;
+    deletePostgresIndexStore(projectName: string): Promise<void>;
+    deletePostgresDocStore(projectName: string): Promise<void>;
+    deleteWeaviateVectorStore(projectName: string): Promise<void>;
+    deleteSimpleVectorStore(projectName: string): Promise<void>;
+    deleteSimpleDocStore(projectName: string): Promise<void>;
+    deleteSimpleIndexStore(projectName: string): Promise<void>;
 }
 export {};
 //# sourceMappingURL=Meaningfully.d.ts.map

package/dist/Meaningfully.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Meaningfully.d.ts","sourceRoot":"","sources":["../src/Meaningfully.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAKvD,OAAO,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAG7F,KAAK,WAAW,GAAG;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAC,CAAC;AACtC,KAAK,yBAAyB,GAAG,iBAAiB,GAAG,WAAW,CAAC;AAUjE,qBAAa,eAAe;IAC1B,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,OAAO,CAAU;gBAEb,EAAE,WAAW,EAAE,cAAc,EAAE,cAAc,EAAE,eAAe,EAAE,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,GAAG,CAAC;QAAC,cAAc,CAAC,EAAE,GAAG,CAAC;QAAC,eAAe,EAAE,eAAe,CAAA;KAAE;IASnL,UAAU,CAAC,OAAO,EAAE,OAAO;IAG3B,UAAU;IAIJ,gBAAgB,CAAC,IAAI,GAAE,MAAU,EAAE,QAAQ,GAAE,MAAW;;;;IAIxD,cAAc,CAAC,aAAa,EAAE,MAAM;IAGpC,iBAAiB,CAAC,aAAa,EAAE,MAAM;;;IAgB7C,kBAAkB;IAIZ,mBAAmB,CAAC,IAAI,EAAE,yBAAyB;IAuBnD,SAAS,CAAC,IAAI,EAAE,yBAAyB;;;;IA8DzC,iBAAiB,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,GAAE,MAAW,EAAI,OAAO,CAAC,EAAE,cAAc,EAAE;IAsB5G,WAAW,CAAC,aAAa,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM;IA0BzD,WAAW;IAGX,WAAW,CAAC,QAAQ,EAAE,QAAQ;;;IAI9B,iBAAiB;;;;;;;;;IAYjB,iBAAiB,CAAC,WAAW,EAAE,QAAQ;;;CAY9C"}
+{"version":3,"file":"Meaningfully.d.ts","sourceRoot":"","sources":["../src/Meaningfully.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAKvD,OAAO,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAG7F,KAAK,WAAW,GAAG;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAC,CAAC;AACtC,KAAK,yBAAyB,GAAG,iBAAiB,GAAG,WAAW,CAAC;AAUjE,qBAAa,eAAe;IAC1B,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,OAAO,CAAU;gBAEb,EAAE,WAAW,EAAE,cAAc,EAAE,cAAc,EAAE,eAAe,EAAE,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,GAAG,CAAC;QAAC,cAAc,CAAC,EAAE,GAAG,CAAC;QAAC,eAAe,EAAE,eAAe,CAAA;KAAE;IASnL,UAAU,CAAC,OAAO,EAAE,OAAO;IAG3B,UAAU;IAIJ,gBAAgB,CAAC,IAAI,GAAE,MAAU,EAAE,QAAQ,GAAE,MAAW;;;;IAIxD,cAAc,CAAC,aAAa,EAAE,MAAM;IAKpC,iBAAiB,CAAC,aAAa,EAAE,MAAM;;;IA2C7C,kBAAkB;IAIZ,mBAAmB,CAAC,IAAI,EAAE,yBAAyB;IAwBnD,SAAS,CAAC,IAAI,EAAE,yBAAyB;;;;IA8DzC,iBAAiB,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,GAAE,MAAW,EAAI,OAAO,CAAC,EAAE,cAAc,EAAE;IAsB5G,WAAW,CAAC,aAAa,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM;IA0BzD,WAAW;IAGX,WAAW,CAAC,QAAQ,EAAE,QAAQ;;;IAI9B,iBAAiB;;;;;;;;;IAYjB,iBAAiB,CAAC,WAAW,EAAE,QAAQ;;;IAcvC,yBAAyB,CAAC,WAAW,EAAE,MAAM;IAS7C,wBAAwB,CAAC,WAAW,EAAE,MAAM;IAS5C,sBAAsB,CAAC,WAAW,EAAE,MAAM;IAU1C,yBAAyB,CAAC,WAAW,EAAE,MAAM;IAM7C,uBAAuB,CAAC,WAAW,EAAE,MAAM;IAI3C,oBAAoB,CAAC,WAAW,EAAE,MAAM;IAIxC,sBAAsB,CAAC,WAAW,EAAE,MAAM;CAGjD"}

package/dist/Meaningfully.js

@@ -1,6 +1,6 @@
 import { loadDocumentsFromCsv } from './services/csvLoader.js';
 import { createEmbeddings, getIndex, search, previewResults, getDocStore } from './api/embedding.js';
-import { capitalizeFirstLetter } from './utils.js';
+import { sanitizeProjectName, capitalizeFirstLetter } from "./utils.js";
 import { join } from 'path';
 import fs from 'fs';
 const MASKING_PREFIX_LENGTH = 8; // how many characters to show at the start and end of an API key when masking it for display
@@ -11,12 +11,12 @@ const maskKey = (key, n = MASKING_PREFIX_LENGTH) => {
     return (key.length > (n * 2)) ? key.slice(0, n) + "*******" + key.slice(key.length - n) : key;
 };
 export class MeaningfullyAPI {
-    manager;
+    metadataManager;
     storagePath;
     clients;
     constructor({ storagePath, weaviateClient, postgresClient, metadataManager }) {
         this.storagePath = storagePath;
-        this.manager = metadataManager;
+        this.metadataManager = metadataManager;
         this.clients = {
             weaviateClient: weaviateClient,
             postgresClient: postgresClient
@@ -29,20 +29,51 @@ export class MeaningfullyAPI {
         return this.clients;
     }
     async listDocumentSets(page = 1, pageSize = 10) {
-        return await this.manager.getDocumentSets(page, pageSize);
+        return await this.metadataManager.getDocumentSets(page, pageSize);
     }
     async getDocumentSet(documentSetId) {
-        return await this.manager.getDocumentSet(documentSetId);
+        return await this.metadataManager.getDocumentSet(documentSetId);
     }
     async deleteDocumentSet(documentSetId) {
         // Delete the document set from the database
-        const result = await this.manager.getDocumentSet(documentSetId);
+        const result = await this.metadataManager.getDocumentSet(documentSetId);
         if (result) {
             // Delete the document set from the database
-            await this.manager.deleteDocumentSet(documentSetId);
+            await this.metadataManager.deleteDocumentSet(documentSetId);
             // Delete the associated files from the filesystem
-            fs.rmSync(join(this.storagePath, result.name), { recursive: true, force: true });
-            fs.rmSync(join(this.storagePath, 'weaviate_data', capitalizeFirstLetter(result.name)), { recursive: true, force: true });
+            if (result.parameters.vectorStoreType === 'postgres') {
+                if (this.clients.postgresClient) {
+                    try {
+                        await this.deletePostgresVectorStore(result.name);
+                        await this.deletePostgresIndexStore(result.name);
+                        await this.deletePostgresDocStore(result.name);
+                    }
+                    catch (error) {
+                        console.error(`Error deleting Postgres tables for ${sanitizeProjectName(result.name)}`, error);
+                    }
+                }
+            }
+            else if (result.parameters.vectorStoreType === 'weaviate') {
+                if (this.clients.weaviateClient) {
+                    try {
+                        await this.deleteWeaviateVectorStore(result.name);
+                    }
+                    catch (error) {
+                        console.error("Error deleting Weaviate class:", error);
+                    }
+                }
+                else {
+                    this.deleteSimpleVectorStore(result.name);
+                }
+                // Remove the directory and its contents
+                this.deleteSimpleDocStore(result.name);
+                this.deleteSimpleIndexStore(result.name);
+            }
+            else {
+                this.deleteSimpleDocStore(result.name);
+                this.deleteSimpleIndexStore(result.name);
+                this.deleteSimpleVectorStore(result.name);
+            }
         }
         return { success: true };
     }
@@ -58,6 +89,7 @@ export class MeaningfullyAPI {
             if (!data.textColumns[0]) {
                 throw new Error("No text column specified for preview.");
             }
+            console.log(data);
             return await previewResults(data.filePath, data.textColumns[0], {
                 modelName: data.modelName, // needed to tokenize, estimate costs
                 modelProvider: data.modelProvider,
@@ -79,7 +111,7 @@ export class MeaningfullyAPI {
         // figure out if weaviate is available
         const vectorStoreType = this.getVectorStoreType();
         // First create the document set record
-        const documentSetId = await this.manager.addDocumentSet({
+        const documentSetId = await this.metadataManager.addDocumentSet({
             name: data.datasetName,
             uploadDate: new Date(),
             parameters: {
@@ -97,14 +129,14 @@ export class MeaningfullyAPI {
             },
             totalDocuments: 0 // We'll update this after processing
         });
-        const embedSettings = await this.manager.getSettings();
+        const embedSettings = await this.metadataManager.getSettings();
         // Load and process the documents
         try {
             // Process each text column
             for (const textColumn of data.textColumns) {
                 const documents = await loadDocumentsFromCsv(data.filePath, textColumn);
                 // Update total documents count
-                await this.manager.updateDocumentCount(documentSetId, documents.length);
+                await this.metadataManager.updateDocumentCount(documentSetId, documents.length);
                 // Create embeddings for this column
                 let ret = await createEmbeddings(data.filePath, textColumn, {
                     modelName: data.modelName,
@@ -127,14 +159,14 @@ export class MeaningfullyAPI {
         }
         catch (error) {
             // If something fails, we should probably delete the document set
-            await this.manager.deleteDocumentSet(documentSetId);
+            await this.metadataManager.deleteDocumentSet(documentSetId);
             console.error("deleting document set due to failure ", documentSetId, error);
             throw error;
         }
     }
     async searchDocumentSet(documentSetId, query, n_results = 10, filters) {
-        const documentSet = await this.manager.getDocumentSet(documentSetId);
-        const settings = await this.manager.getSettings();
+        const documentSet = await this.metadataManager.getDocumentSet(documentSetId);
+        const settings = await this.metadataManager.getSettings();
         if (!documentSet) {
             throw new Error('Document set not found');
         }
@@ -154,8 +186,8 @@ export class MeaningfullyAPI {
         return results;
     }
     async getDocument(documentSetId, documentNodeId) {
-        const documentSet = await this.manager.getDocumentSet(documentSetId);
-        const settings = await this.manager.getSettings();
+        const documentSet = await this.metadataManager.getDocumentSet(documentSetId);
+        const settings = await this.metadataManager.getSettings();
         if (!documentSet) {
             throw new Error('Document set not found');
         }
@@ -178,13 +210,13 @@ export class MeaningfullyAPI {
         return document;
     }
     async getSettings() {
-        return this.manager.getSettings();
+        return this.metadataManager.getSettings();
     }
     async setSettings(settings) {
-        return this.manager.setSettings(settings);
+        return this.metadataManager.setSettings(settings);
     }
     async getMaskedSettings() {
-        const settings = await this.manager.getSettings();
+        const settings = await this.metadataManager.getSettings();
         return {
             openAIKey: maskKey(settings.openAIKey),
             oLlamaBaseURL: settings.oLlamaBaseURL,
@@ -196,7 +228,7 @@ export class MeaningfullyAPI {
         };
     }
     async setMaskedSettings(newSettings) {
-        const oldSettings = await this.manager.getSettings();
+        const oldSettings = await this.metadataManager.getSettings();
         const settings = {
             ...newSettings,
             openAIKey: newSettings.openAIKey == maskKey(oldSettings.openAIKey) ? oldSettings.openAIKey : newSettings.openAIKey,
@@ -204,7 +236,53 @@ export class MeaningfullyAPI {
             mistralApiKey: newSettings.mistralApiKey == maskKey(oldSettings.mistralApiKey) ? oldSettings.mistralApiKey : newSettings.mistralApiKey,
             geminiApiKey: newSettings.geminiApiKey == maskKey(oldSettings.geminiApiKey) ? oldSettings.geminiApiKey : newSettings.geminiApiKey
         };
-        return this.manager.setSettings(settings);
+        return this.metadataManager.setSettings(settings);
+    }
+    // these should be moved to another file, just because they're too low-level for this one.
+    async deletePostgresVectorStore(projectName) {
+        if (this.clients.postgresClient) {
+            try {
+                await this.clients.postgresClient.query('DROP TABLE IF EXISTS vecs_' + sanitizeProjectName(projectName));
+            }
+            catch (error) {
+                console.error(`Error deleting Postgres tables for ${sanitizeProjectName(projectName)}`, error);
+            }
+        }
+    }
+    async deletePostgresIndexStore(projectName) {
+        if (this.clients.postgresClient) {
+            try {
+                await this.clients.postgresClient.query('DROP TABLE IF EXISTS idx_' + sanitizeProjectName(projectName));
+            }
+            catch (error) {
+                console.error(`Error deleting Postgres tables for ${sanitizeProjectName(projectName)}`, error);
+            }
+        }
+    }
+    async deletePostgresDocStore(projectName) {
+        if (this.clients.postgresClient) {
+            try {
+                await this.clients.postgresClient.query('DROP TABLE IF EXISTS docs_' + sanitizeProjectName(projectName));
+            }
+            catch (error) {
+                console.error(`Error deleting Postgres tables for ${sanitizeProjectName(projectName)}`, error);
+            }
+        }
+    }
+    async deleteWeaviateVectorStore(projectName) {
+        if (this.clients.weaviateClient) {
+            await this.clients.weaviateClient.collections.delete(capitalizeFirstLetter(projectName));
+        }
+        // fs.rmSync(join(this.storagePath, 'weaviate_data', capitalizeFirstLetter(result.name)), { recursive: true, force: true });
+    }
+    async deleteSimpleVectorStore(projectName) {
+        fs.rmSync(join(this.storagePath, projectName), { recursive: true, force: true });
+    }
+    async deleteSimpleDocStore(projectName) {
+        fs.rmSync(join(this.storagePath, projectName), { recursive: true, force: true });
+    }
+    async deleteSimpleIndexStore(projectName) {
+        fs.rmSync(join(this.storagePath, projectName), { recursive: true, force: true });
     }
 }
 //# sourceMappingURL=Meaningfully.js.map

package/dist/Meaningfully.js.map

@@ -1 +1 @@
-{"version":3,"file":"Meaningfully.js","sourceRoot":"","sources":["../src/Meaningfully.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACrG,OAAO,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAE,MAAM,IAAI,CAAC;AAKpB,MAAM,qBAAqB,GAAG,CAAC,CAAC,CAAC,6FAA6F;AAC5F,gHAAgH;AAClJ,MAAM,OAAO,GAAG,CAAC,GAAkB,EAAE,IAAY,qBAAqB,EAAiB,EAAE;IACvF,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,OAAO,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AAC9F,CAAC,CAAC;AAGF,MAAM,OAAO,eAAe;IAClB,OAAO,CAAkB;IACzB,WAAW,CAAS;IACpB,OAAO,CAAU;IAEzB,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,cAAc,EAAE,eAAe,EAAyG;QACjL,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,eAAe,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG;YACb,cAAc,EAAE,cAAc;YAC9B,cAAc,EAAE,cAAc;SAC/B,CAAC;IACJ,CAAC;IAED,UAAU,CAAC,OAAgB;QACzB,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,EAAE,CAAC;IACjD,CAAC;IACD,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,OAAe,CAAC,EAAE,WAAmB,EAAE;QAC5D,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC5D,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,aAAqB;QACxC,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAC1D,CAAC;IACD,KAAK,CAAC,iBAAiB,CAAC,aAAqB;QAC3C,4CAA4C;QAC5C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QAChE,IAAI,MAAM,EAAC,CAAC;YACV,4CAA4C;YAC5C,MAAM,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;YACpD,kDAAkD;YAClD,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACjF,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,eAAe,EAAE,qBAAqB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3H,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;IAED,+BAA+B;IAC/B,mEAAmE;IACnE,uDAAuD;IACvD,kBAAkB;QAChB,OAAO,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC1G,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,IAA+B;QACvD,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAClD,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAC3D,CAAC;YACD,OAAO,MAAM,cAAc,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAW,EAAE;gBACxE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,qCAAqC;gBAChE,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;gBAC3C,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;gBAC3D,cAAc,EAAE,GAAG;gBACnB,eAAe,EAAE,eAAe;gBAChC,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;aAChC,CAAC,CAAC;QACP,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEC,KAAK,CAAC,SAAS,CAAC,IAA+B;QAC7C,sCAAsC;QACtC,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAClD,uCAAuC;QACvC,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC;YACtD,IAAI,EAAE,IAAI,CAAC,WAAW;YACtB,UAAU,EAAE,IAAI,IAAI,EAAE;YACtB,UAAU,EAAE;gBACV,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;gBAC3C,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;gBAC3D,cAAc,EAAE,IAAI,CAAC,cAAc;gBACnC,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,eAAe,EAAE,eAAe;aACjC;YACD,cAAc,EAAE,CAAC,CAAC,qCAAqC;SACxD,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAA;QAEtD,iCAAiC;QACjC,IAAI,CAAC;YACH,2BAA2B;YAC3B,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;gBAC1C,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;gBAExE,+BAA+B;gBAC/B,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,aAAa,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;gBAExE,oCAAoC;gBACpC,IAAI,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE;oBAC1D,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;oBAC3C,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;oBAC3D,cAAc,EAAE,GAAG,EAAE,0BAA0B;oBAC/C,eAAe,EAAE,eAAe;oBAChC,WAAW,EAAE,IAAI,CAAC,WAAW;oBACf,uFAAuF;oBACrG,WAAW,EAAG,IAAI,CAAC,WAAW;oBAC9B,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;iBAChC,EAAE,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;gBAChC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;oBACjB,MAAM,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC7B,CAAC;YACH,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC;QAC1C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,iEAAiE;YACjE,MAAM,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;YACpD,OAAO,CAAC,KAAK,CAAC,uCAAuC,EAAE,aAAa,EAAE,KAAK,CAAC,CAAC;YAC7E,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAGD,KAAK,CAAC,iBAAiB,CAAC,aAAqB,EAAE,KAAa,EAAE,YAAoB,EAAE,EAAI,OAA0B;QAChH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QACrE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAClD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC;YAC3B,SAAS,EAAE,WAAW,CAAC,UAAU,CAAC,SAAmB;YACrD,aAAa,EAAE,WAAW,CAAC,UAAU,CAAC,aAAuB;YAC7D,kBAAkB,EAAE,WAAW,CAAC,UAAU,CAAC,kBAA6B;YACxE,0BAA0B,EAAE,WAAW,CAAC,UAAU,CAAC,0BAAqC;YACxF,cAAc,EAAE,GAAG;YACnB,eAAe,EAAE,WAAW,CAAC,UAAU,CAAC,eAAwC;YAChF,WAAW,EAAE,WAAW,CAAC,IAAI;YAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,SAAS,EAAE,IAAI,EAAE,yEAAyE;YAC1F,YAAY,EAAE,EAAE,EAAE,yEAAyE;SAC5F,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3B,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAC/D,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,aAAqB,EAAE,cAAsB;QAC7D,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QACrE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAClD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC;YACjC,SAAS,EAAE,WAAW,CAAC,UAAU,CAAC,SAAmB;YACrD,aAAa,EAAE,WAAW,CAAC,UAAU,CAAC,aAAuB;YAC7D,kBAAkB,EAAE,WAAW,CAAC,UAAU,CAAC,kBAA6B;YACxE,0BAA0B,EAAE,WAAW,CAAC,UAAU,CAAC,0BAAqC;YACxF,cAAc,EAAE,GAAG;YACnB,eAAe,EAAE,WAAW,CAAC,UAAU,CAAC,eAAwC;YAChF,WAAW,EAAE,WAAW,CAAC,IAAI;YAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,SAAS,EAAE,IAAI,EAAE,yEAAyE;YAC1F,YAAY,EAAE,EAAE,EAAE,yEAAyE;SAC5F,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3B,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QACxD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACxC,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAGD,KAAK,CAAC,WAAW;QACf,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IACpC,CAAC;IACD,KAAK,CAAC,WAAW,CAAC,QAAkB;QAClC,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,iBAAiB;QACrB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAClD,OAAO;YACL,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;YACtC,aAAa,EAAE,QAAQ,CAAC,aAAa;YACrC,cAAc,EAAE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;YAChD,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;YACjD,qBAAqB,EAAE,QAAQ,CAAC,qBAAqB;YACrD,aAAa,EAAE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;YAC9C,YAAY,EAAE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;SAC7C,CAAC;IACJ,CAAC;IACD,KAAK,CAAC,iBAAiB,CAAC,WAAqB;QAC3C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QACrD,MAAM,QAAQ,GAAG;YACf,GAAG,WAAW;YACd,SAAS,EAAE,WAAW,CAAC,SAAS,IAAI,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS;YAClH,cAAc,EAAE,WAAW,CAAC,cAAc,IAAI,OAAO,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc;YAC3I,aAAa,EAAE,WAAW,CAAC,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa;YACtI,YAAY,EAAE,WAAW,CAAC,YAAY,IAAI,OAAO,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,YAAY;SAClI,CAAC;QACF,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC5C,CAAC;CAEF"}
+{"version":3,"file":"Meaningfully.js","sourceRoot":"","sources":["../src/Meaningfully.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACrG,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AACxE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAE,MAAM,IAAI,CAAC;AAKpB,MAAM,qBAAqB,GAAG,CAAC,CAAC,CAAC,6FAA6F;AAC5F,gHAAgH;AAClJ,MAAM,OAAO,GAAG,CAAC,GAAkB,EAAE,IAAY,qBAAqB,EAAiB,EAAE;IACvF,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,OAAO,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AAC9F,CAAC,CAAC;AAGF,MAAM,OAAO,eAAe;IAClB,eAAe,CAAkB;IACjC,WAAW,CAAS;IACpB,OAAO,CAAU;IAEzB,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,cAAc,EAAE,eAAe,EAAyG;QACjL,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,OAAO,GAAG;YACb,cAAc,EAAE,cAAc;YAC9B,cAAc,EAAE,cAAc;SAC/B,CAAC;IACJ,CAAC;IAED,UAAU,CAAC,OAAgB;QACzB,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,EAAE,CAAC;IACjD,CAAC;IACD,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,OAAe,CAAC,EAAE,WAAmB,EAAE;QAC5D,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,eAAe,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACpE,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,aAAqB;QACxC,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAClE,CAAC;IAGD,KAAK,CAAC,iBAAiB,CAAC,aAAqB;QAC3C,4CAA4C;QAC5C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QAExE,IAAI,MAAM,EAAC,CAAC;YACV,4CAA4C;YAC5C,MAAM,IAAI,CAAC,eAAe,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;YAC5D,kDAAkD;YAClD,IAAI,MAAM,CAAC,UAAU,CAAC,eAAe,KAAK,UAAU,EAAC,CAAC;gBACpD,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;oBAChC,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,yBAAyB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;wBACjD,MAAM,IAAI,CAAC,wBAAwB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;wBAChD,MAAM,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;oBAChD,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,OAAO,CAAC,KAAK,CAAC,sCAAsC,mBAAmB,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;oBACjG,CAAC;gBACH,CAAC;YACH,CAAC;iBAAM,IAAI,MAAM,CAAC,UAAU,CAAC,eAAe,KAAK,UAAU,EAAC,CAAC;gBAC3D,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;oBAChC,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,yBAAyB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;oBACpD,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;oBACzD,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC5C,CAAC;gBACD,wCAAwC;gBACxC,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACzC,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC5C,CAAC;QACH,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;IAED,+BAA+B;IAC/B,mEAAmE;IACnE,uDAAuD;IACvD,kBAAkB;QAChB,OAAO,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC1G,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,IAA+B;QACvD,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAClD,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAC3D,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAClB,OAAO,MAAM,cAAc,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAW,EAAE;gBACxE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,qCAAqC;gBAChE,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;gBAC3C,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;gBAC3D,cAAc,EAAE,GAAG;gBACnB,eAAe,EAAE,eAAe;gBAChC,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;aAChC,CAAC,CAAC;QACP,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEC,KAAK,CAAC,SAAS,CAAC,IAA+B;QAC7C,sCAAsC;QACtC,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAClD,uCAAuC;QACvC,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC;YAC9D,IAAI,EAAE,IAAI,CAAC,WAAW;YACtB,UAAU,EAAE,IAAI,IAAI,EAAE;YACtB,UAAU,EAAE;gBACV,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;gBAC3C,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;gBAC3D,cAAc,EAAE,IAAI,CAAC,cAAc;gBACnC,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,eAAe,EAAE,eAAe;aACjC;YACD,cAAc,EAAE,CAAC,CAAC,qCAAqC;SACxD,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAA;QAE9D,iCAAiC;QACjC,IAAI,CAAC;YACH,2BAA2B;YAC3B,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;gBAC1C,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;gBAExE,+BAA+B;gBAC/B,MAAM,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC,aAAa,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;gBAEhF,oCAAoC;gBACpC,IAAI,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE;oBAC1D,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;oBAC3C,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;oBAC3D,cAAc,EAAE,GAAG,EAAE,0BAA0B;oBAC/C,eAAe,EAAE,eAAe;oBAChC,WAAW,EAAE,IAAI,CAAC,WAAW;oBACf,uFAAuF;oBACrG,WAAW,EAAG,IAAI,CAAC,WAAW;oBAC9B,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;iBAChC,EAAE,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;gBAChC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;oBACjB,MAAM,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC7B,CAAC;YACH,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC;QAC1C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,iEAAiE;YACjE,MAAM,IAAI,CAAC,eAAe,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;YAC5D,OAAO,CAAC,KAAK,CAAC,uCAAuC,EAAE,aAAa,EAAE,KAAK,CAAC,CAAC;YAC7E,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAGD,KAAK,CAAC,iBAAiB,CAAC,aAAqB,EAAE,KAAa,EAAE,YAAoB,EAAE,EAAI,OAA0B;QAChH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QAC7E,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;QAC1D,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC;YAC3B,SAAS,EAAE,WAAW,CAAC,UAAU,CAAC,SAAmB;YACrD,aAAa,EAAE,WAAW,CAAC,UAAU,CAAC,aAAuB;YAC7D,kBAAkB,EAAE,WAAW,CAAC,UAAU,CAAC,kBAA6B;YACxE,0BAA0B,EAAE,WAAW,CAAC,UAAU,CAAC,0BAAqC;YACxF,cAAc,EAAE,GAAG;YACnB,eAAe,EAAE,WAAW,CAAC,UAAU,CAAC,eAAwC;YAChF,WAAW,EAAE,WAAW,CAAC,IAAI;YAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,SAAS,EAAE,IAAI,EAAE,yEAAyE;YAC1F,YAAY,EAAE,EAAE,EAAE,yEAAyE;SAC5F,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3B,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAC/D,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,aAAqB,EAAE,cAAsB;QAC7D,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QAC7E,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;QAC1D,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC;YACjC,SAAS,EAAE,WAAW,CAAC,UAAU,CAAC,SAAmB;YACrD,aAAa,EAAE,WAAW,CAAC,UAAU,CAAC,aAAuB;YAC7D,kBAAkB,EAAE,WAAW,CAAC,UAAU,CAAC,kBAA6B;YACxE,0BAA0B,EAAE,WAAW,CAAC,UAAU,CAAC,0BAAqC;YACxF,cAAc,EAAE,GAAG;YACnB,eAAe,EAAE,WAAW,CAAC,UAAU,CAAC,eAAwC;YAChF,WAAW,EAAE,WAAW,CAAC,IAAI;YAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,SAAS,EAAE,IAAI,EAAE,yEAAyE;YAC1F,YAAY,EAAE,EAAE,EAAE,yEAAyE;SAC5F,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3B,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QACxD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACxC,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAGD,KAAK,CAAC,WAAW;QACf,OAAO,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;IAC5C,CAAC;IACD,KAAK,CAAC,WAAW,CAAC,QAAkB;QAClC,OAAO,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,iBAAiB;QACrB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;QAC1D,OAAO;YACL,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;YACtC,aAAa,EAAE,QAAQ,CAAC,aAAa;YACrC,cAAc,EAAE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;YAChD,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;YACjD,qBAAqB,EAAE,QAAQ,CAAC,qBAAqB;YACrD,aAAa,EAAE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;YAC9C,YAAY,EAAE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;SAC7C,CAAC;IACJ,CAAC;IACD,KAAK,CAAC,iBAAiB,CAAC,WAAqB;QAC3C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;QAC7D,MAAM,QAAQ,GAAG;YACf,GAAG,WAAW;YACd,SAAS,EAAE,WAAW,CAAC,SAAS,IAAI,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS;YAClH,cAAc,EAAE,WAAW,CAAC,cAAc,IAAI,OAAO,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc;YAC3I,aAAa,EAAE,WAAW,CAAC,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa;YACtI,YAAY,EAAE,WAAW,CAAC,YAAY,IAAI,OAAO,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,YAAY;SAClI,CAAC;QACF,OAAO,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IACpD,CAAC;IAGD,0FAA0F;IAC1F,KAAK,CAAC,yBAAyB,CAAC,WAAmB;QACjD,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YAChC,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,4BAA4B,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAC,CAAC;YAC3G,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,sCAAsC,mBAAmB,CAAC,WAAW,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;YACjG,CAAC;QACH,CAAC;IACH,CAAC;IACD,KAAK,CAAC,wBAAwB,CAAC,WAAmB;QAChD,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YAChC,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,2BAA2B,GAAG,mBAAmB,CAAC,WAAW,CAAE,CAAC,CAAC;YAC3G,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,sCAAsC,mBAAmB,CAAC,WAAW,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;YACjG,CAAC;QACH,CAAC;IACH,CAAC;IACD,KAAK,CAAC,sBAAsB,CAAC,WAAmB;QAC9C,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YAChC,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,4BAA4B,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAC,CAAC;YAC3G,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,sCAAsC,mBAAmB,CAAC,WAAW,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;YACjG,CAAC;QACH,CAAC;IAEH,CAAC;IACD,KAAK,CAAC,yBAAyB,CAAC,WAAmB;QACjD,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YAChC,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,WAAW,CAAC,MAAM,CAAC,qBAAqB,CAAC,WAAW,CAAC,CAAC,CAAC;QAC3F,CAAC;QACG,4HAA4H;IAClI,CAAC;IACD,KAAK,CAAC,uBAAuB,CAAC,WAAmB;QAC/C,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAEnF,CAAC;IACD,KAAK,CAAC,oBAAoB,CAAC,WAAmB;QAC5C,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAEnF,CAAC;IACD,KAAK,CAAC,sBAAsB,CAAC,WAAmB;QAC9C,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACnF,CAAC;CACF"}

package/dist/api/embedding.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"embedding.d.ts","sourceRoot":"","sources":["../../src/api/embedding.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAK1I,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,MAAM,EACf,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,eAAe,EACvB,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,eAAe,CAAC,CAyC1B;AAGD,wBAAsB,cAAc,CAClC,OAAO,EAAE,MAAM,EACf,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,eAAe,GACtB,OAAO,CAAC,aAAa,CAAC,CAqCxB;AAED,wBAAsB,WAAW,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,mDAE9F;AAED,wBAAsB,QAAQ,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,kDAE3F;AAED,wBAAsB,MAAM,CAC1B,KAAK,EAAE,GAAG,EACV,KAAK,EAAE,MAAM,EACb,UAAU,GAAE,MAAW,EACvB,OAAO,CAAC,EAAE,cAAc,EAAE,GACzB,OAAO,CAAC,YAAY,EAAE,CAAC,CASzB"}
+{"version":3,"file":"embedding.d.ts","sourceRoot":"","sources":["../../src/api/embedding.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAK1I,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,MAAM,EACf,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,eAAe,EACvB,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,eAAe,CAAC,CA0C1B;AAGD,wBAAsB,cAAc,CAClC,OAAO,EAAE,MAAM,EACf,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,eAAe,GACtB,OAAO,CAAC,aAAa,CAAC,CAqCxB;AAED,wBAAsB,WAAW,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,mDAE9F;AAED,wBAAsB,QAAQ,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,kDAE3F;AAED,wBAAsB,MAAM,CAC1B,KAAK,EAAE,GAAG,EACV,KAAK,EAAE,MAAM,EACb,UAAU,GAAE,MAAW,EACvB,OAAO,CAAC,EAAE,cAAc,EAAE,GACzB,OAAO,CAAC,YAAY,EAAE,CAAC,CASzB"}

package/dist/api/embedding.js

@@ -19,11 +19,12 @@ export async function createEmbeddings(csvPath, textColumnName, config, settings
         }
         progressManager.updateProgress(operationId, 5);
         const nodes = await transformDocumentsToNodes(documents, config);
+        const progressCallback = (progress, total) => {
+            const percentage = Math.floor((progress / total) * 90) + 5; // Map to 5-95% of total progress
+            progressManager.updateProgress(operationId, percentage);
+        };
         const [index] = await Promise.all([
-            persistNodes(nodes, config, settings, clients, (progress, total) => {
-                const percentage = Math.floor((progress / total) * 90) + 5; // Map to 5-95% of total progress
-                progressManager.updateProgress(operationId, percentage);
-            }),
+            persistNodes(nodes, config, settings, clients, progressCallback),
             persistDocuments(documents, config, settings, clients)
         ]);
         progressManager.completeOperation(operationId);

package/dist/api/embedding.js.map

@@ -1 +1 @@
-{"version":3,"file":"embedding.js","sourceRoot":"","sources":["../../src/api/embedding.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,YAAY,EAAE,eAAe,EAAE,2BAA2B,EAAE,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAErL,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAEjE,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,OAAe,EACf,cAAsB,EACtB,MAAuB,EACvB,QAAkB,EAClB,OAAgB;IAEhB,IAAI,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,SAAS,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;QAC1C,MAAM,eAAe,GAAG,eAAe,CAAC,WAAW,EAAE,CAAC;QACtD,eAAe,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;QAEjD,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QACtE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,eAAe,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;YAC5C,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;YAC7C,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,yFAAyF;aACjG,CAAC;QACJ,CAAC;QAED,eAAe,CAAC,cAAc,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAE/C,MAAM,KAAK,GAAG,MAAM,yBAAyB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAEjE,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAChC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE;gBACjE,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,iCAAiC;gBAC7F,eAAe,CAAC,cAAc,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;YAC1D,CAAC,CAAC;YACF,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC;SACvD,CAAC,CAAC;QAEH,eAAe,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAC/C,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;QAC7C,OAAO;YACL,OAAO,EAAE,IAAI;YACb,KAAK;SACN,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;SACzE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,uDAAuD;AACvD,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAAe,EACf,cAAsB,EACtB,MAAuB;IAEvB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QACtE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,yFAAyF;aACjG,CAAC;QACJ,CAAC;QACD,0DAA0D;QAC1D,oLAAoL;QACpL,6EAA6E;QAC7E,MAAM,sBAAsB,GAAG,SAAS,CAAC,KAAK,CAC5C,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,EAChC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,EAAE,CACtC,CAAC;QAEF,MAAM,YAAY,GAAG,MAAM,yBAAyB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACxE,MAAM,kBAAkB,GAAG,MAAM,yBAAyB,CAAC,sBAAsB,EAAE,MAAM,CAAC,CAAC;QAC3F,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;QAEhG,OAAO;YACL,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,kBAAkB,CAAC,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE,CAAC,CAAC;gBAC5C,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;aACxB,CAAC,CAAC;YACH,cAAc;YACd,UAAU;YACV,UAAU;SACX,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;SACzE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC7F,OAAO,CAAC,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;AACvE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC1F,OAAO,MAAM,2BAA2B,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;AACtE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,KAAU,EACV,KAAa,EACb,aAAqB,EAAE,EACvB,OAA0B;IAE1B,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IACzE,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAW,EAAE,EAAE,CAAC,CAAC;QACnC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,IAAI,CAAC;QAC/C,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,CAAC;QACxB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;QAC9B,cAAc;QACd,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,EAAE,MAAM;KACxD,CAAC,CAAC,CAAC;AACN,CAAC"}
+{"version":3,"file":"embedding.js","sourceRoot":"","sources":["../../src/api/embedding.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,YAAY,EAAE,eAAe,EAAE,2BAA2B,EAAE,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAErL,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAEjE,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,OAAe,EACf,cAAsB,EACtB,MAAuB,EACvB,QAAkB,EAClB,OAAgB;IAEhB,IAAI,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,SAAS,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;QAC1C,MAAM,eAAe,GAAG,eAAe,CAAC,WAAW,EAAE,CAAC;QACtD,eAAe,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;QAEjD,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QACtE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,eAAe,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;YAC5C,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;YAC7C,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,yFAAyF;aACjG,CAAC;QACJ,CAAC;QAED,eAAe,CAAC,cAAc,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAE/C,MAAM,KAAK,GAAG,MAAM,yBAAyB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAEjE,MAAM,gBAAgB,GAAG,CAAC,QAAgB,EAAE,KAAa,EAAE,EAAE;YACzD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,iCAAiC;YAC7F,eAAe,CAAC,cAAc,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;QAC1D,CAAC,CAAC;QACJ,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAChC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,gBAAgB,CAAC;YAChE,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC;SACvD,CAAC,CAAC;QAEH,eAAe,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAC/C,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;QAC7C,OAAO;YACL,OAAO,EAAE,IAAI;YACb,KAAK;SACN,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;SACzE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,uDAAuD;AACvD,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAAe,EACf,cAAsB,EACtB,MAAuB;IAEvB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QACtE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,yFAAyF;aACjG,CAAC;QACJ,CAAC;QACD,0DAA0D;QAC1D,oLAAoL;QACpL,6EAA6E;QAC7E,MAAM,sBAAsB,GAAG,SAAS,CAAC,KAAK,CAC5C,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,EAChC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,EAAE,CACtC,CAAC;QAEF,MAAM,YAAY,GAAG,MAAM,yBAAyB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACxE,MAAM,kBAAkB,GAAG,MAAM,yBAAyB,CAAC,sBAAsB,EAAE,MAAM,CAAC,CAAC;QAC3F,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;QAEhG,OAAO;YACL,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,kBAAkB,CAAC,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE,CAAC,CAAC;gBAC5C,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;aACxB,CAAC,CAAC;YACH,cAAc;YACd,UAAU;YACV,UAAU;SACX,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;SACzE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC7F,OAAO,CAAC,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;AACvE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC1F,OAAO,MAAM,2BAA2B,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;AACtE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,KAAU,EACV,KAAa,EACb,aAAqB,EAAE,EACvB,OAA0B;IAE1B,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IACzE,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAW,EAAE,EAAE,CAAC,CAAC;QACnC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,IAAI,CAAC;QAC/C,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,CAAC;QACxB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;QAC9B,cAAc;QACd,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,EAAE,MAAM;KACxD,CAAC,CAAC,CAAC;AACN,CAAC"}

package/dist/services/embeddings.d.ts

@@ -1,10 +1,12 @@
-import { Document, VectorStoreIndex, TextNode, type StorageContext } from "llamaindex";
+import { Document, VectorStoreIndex, TextNode, SimpleVectorStore, type StorageContext, BaseDocumentStore, BaseIndexStore } from "llamaindex";
 import { OllamaEmbedding } from '@llamaindex/ollama';
 import { MistralAIEmbedding } from '@llamaindex/mistral';
 import { GeminiEmbedding } from '@llamaindex/google';
+import { PGVectorStore } from '@llamaindex/postgres';
 import { MockEmbedding } from "./mockEmbedding.js";
 import type { EmbeddingConfig, Settings, MetadataFilter, Clients } from "../types/index.js";
 import { OpenAIEmbedding } from "@llamaindex/openai";
+import { BatchingWeaviateVectorStore } from "./batchingWeaviateVectorStore.js";
 import { ProgressVectorStoreIndex } from "./progressVectorStoreIndex.js";
 export declare function estimateCost(nodes: TextNode[], modelName: string): {
     estimatedPrice: number;
@@ -17,5 +19,8 @@ export declare function getEmbedModel(config: EmbeddingConfig, settings: Setting
 export declare function getStorageContext(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<StorageContext>;
 export declare function persistDocuments(documents: Document[], config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<void>;
 export declare function persistNodes(nodes: TextNode[], config: EmbeddingConfig, settings: Settings, clients: Clients, progressCallback?: (progress: number, total: number) => void): Promise<ProgressVectorStoreIndex>;
+export declare function createVectorStore(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<PGVectorStore | SimpleVectorStore | BatchingWeaviateVectorStore>;
+export declare function createDocumentStore(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<BaseDocumentStore>;
+export declare function createIndexStore(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<BaseIndexStore>;
 export declare function searchDocuments(index: VectorStoreIndex, query: string, numResults?: number, filters?: MetadataFilter[]): Promise<import("llamaindex").NodeWithScore<import("llamaindex").Metadata>[]>;
 //# sourceMappingURL=embeddings.d.ts.map

package/dist/services/embeddings.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../../src/services/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,gBAAgB,EAIhB,QAAQ,EAKR,KAAK,cAAc,EAMpB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,eAAe,EAAC,MAAM,oBAAoB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAA+B,MAAM,qBAAqB,CAAA;AACrF,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAA;AAKpD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAGnD,OAAO,KAAK,EAAE,eAAe,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAG,MAAM,mBAAmB,CAAC;AAG7F,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAErD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAuCzE,wBAAgB,YAAY,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG;IAClE,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB,CAuBA;AAED,wBAAsB,2BAA2B,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,6BAK9G;AAED,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,QAAQ,EAAE,EACrB,MAAM,EAAE,eAAe,sDA0BxB;AAED,wBAAgB,aAAa,CAC3B,MAAM,EAAE,eAAe,EACvB,QAAQ,EAAE,QAAQ,4FA2CnB;AAED,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,cAAc,CAAC,CAoB9H;AAED,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAgB1I;AAED,wBAAsB,YAAY,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAqCpN;AAyED,wBAAsB,eAAe,CACnC,KAAK,EAAE,gBAAgB,EACvB,KAAK,EAAE,MAAM,EACb,UAAU,GAAE,MAAW,EACvB,OAAO,CAAC,EAAE,cAAc,EAAE,gFAU3B"}
+{"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../../src/services/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,gBAAgB,EAIhB,QAAQ,EAIR,iBAAiB,EACjB,KAAK,cAAc,EAGnB,iBAAiB,EACjB,cAAc,EAEf,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,eAAe,EAAC,MAAM,oBAAoB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAA+B,MAAM,qBAAqB,CAAA;AACrF,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAA;AACpD,OAAO,EAAE,aAAa,EAA6C,MAAM,sBAAsB,CAAC;AAIhG,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAGnD,OAAO,KAAK,EAAE,eAAe,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAG,MAAM,mBAAmB,CAAC;AAG7F,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,2BAA2B,EAAE,MAAM,kCAAkC,CAAC;AAC/E,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAuCzE,wBAAgB,YAAY,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG;IAClE,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB,CAuBA;AAED,wBAAsB,2BAA2B,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,6BAO9G;AAED,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,QAAQ,EAAE,EACrB,MAAM,EAAE,eAAe,sDA0BxB;AAED,wBAAgB,aAAa,CAC3B,MAAM,EAAE,eAAe,EACvB,QAAQ,EAAE,QAAQ,4FA2CnB;AAED,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,cAAc,CAAC,CAoB9H;AAED,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAgB1I;AAED,wBAAsB,YAAY,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAiDpN;AAGD,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,aAAa,GAAG,iBAAiB,GAAG,2BAA2B,CAAC,CAqC/K;AAGD,wBAAsB,mBAAmB,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAcnI;AAGD,wBAAsB,gBAAgB,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,cAAc,CAAC,CAc7H;AAED,wBAAsB,eAAe,CACnC,KAAK,EAAE,gBAAgB,EACvB,KAAK,EAAE,MAAM,EACb,UAAU,GAAE,MAAW,EACvB,OAAO,CAAC,EAAE,cAAc,EAAE,gFAU3B"}

package/dist/services/embeddings.js

@@ -69,9 +69,11 @@ export function estimateCost(nodes, modelName) {
 }
 export async function getExistingVectorStoreIndex(config, settings, clients) {
     const storageContext = await getStorageContext(config, settings, clients);
-    return await VectorStoreIndex.init({
-        storageContext: storageContext,
-    });
+    const vectorStore = storageContext.vectorStores[ModalityType.TEXT];
+    if (!vectorStore) {
+        throw new Error("Vector store for ModalityType.TEXT is undefined");
+    }
+    return await VectorStoreIndex.fromVectorStore(vectorStore);
 }
 export async function transformDocumentsToNodes(documents, config) {
     console.time("transformDocumentsToNodes Run Time");
@@ -191,12 +193,24 @@ export async function persistNodes(nodes, config, settings, clients, progressCal
     // Create index and embed documents
     // this is what actaully embeds the nodes
     // (even if they already have embeddings, stupidly)
+    const NODE_CHUNK_SIZE = 10000;
+    const modifiedProgressCallback = (progress, total) => {
+        if (progressCallback) {
+            progressCallback(progress, nodes.length);
+        }
+        console.log('progress total nodes.length', progress, nodes.length);
+    };
     const index = await ProgressVectorStoreIndex.init({
-        nodes,
+        nodes: nodes.slice(0, NODE_CHUNK_SIZE),
         storageContext,
-        logProgress: true,
-        progressCallback,
+        progressCallback: modifiedProgressCallback
     });
+    if (nodes.length > NODE_CHUNK_SIZE) {
+        for (let i = NODE_CHUNK_SIZE; i < nodes.length; i += NODE_CHUNK_SIZE) {
+            const chunk = nodes.slice(i, i + NODE_CHUNK_SIZE);
+            await index.insertNodes(chunk, { progressCallback: modifiedProgressCallback });
+        }
+    }
     // I'm not sure why this explicit call to persist is necessary. 
     // storageContext should handle this, but it doesn't.
     // all the if statements are just type-checking boilerplate.
@@ -219,7 +233,8 @@ export async function persistNodes(nodes, config, settings, clients, progressCal
     console.timeEnd("persistNodes Run Time");
     return index;
 }
-async function createVectorStore(config, settings, clients) {
+// exported only for tests
+export async function createVectorStore(config, settings, clients) {
     const embeddingModel = getEmbedModel(config, settings);
     switch (config.vectorStoreType) {
         // for some reason the embedding model has to be specified here TOO
@@ -252,7 +267,8 @@ async function createVectorStore(config, settings, clients) {
             throw new Error(`Unsupported vector store type: ${config.vectorStoreType}`);
     }
 }
-async function createDocumentStore(config, settings, clients) {
+// exported only for tests
+export async function createDocumentStore(config, settings, clients) {
     // we create the doc store without a persist path, so it doesn't write to disk after every put()
     switch (config.documentStoreType || config.vectorStoreType) {
         case "postgres":
@@ -267,7 +283,8 @@ async function createDocumentStore(config, settings, clients) {
             throw new Error(`Unsupported vector store type: ${config.vectorStoreType}`);
     }
 }
-async function createIndexStore(config, settings, clients) {
+// exported only for tests
+export async function createIndexStore(config, settings, clients) {
     switch (config.documentStoreType || config.vectorStoreType) {
         case "postgres":
             return new PostgresIndexStore({

package/dist/services/embeddings.js.map

@@ -1 +1 @@
-{"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../../src/services/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,gBAAgB;AAChB,mBAAmB;AACnB,iBAAiB,EAGjB,YAAY,EAEZ,0BAA0B,EAC1B,iBAAiB,EAEjB,QAAQ,IAAI,kBAAkB,EAC9B,mBAAmB,EAGnB,gBAAgB,EACjB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,eAAe,EAAC,MAAM,oBAAoB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAAE,2BAA2B,EAAE,MAAM,qBAAqB,CAAA;AACrF,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAA;AACpD,OAAO,EAAE,aAAa,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAChG,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAsB,MAAM,aAAa,CAAC;AACnE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACzE,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,2BAA2B,EAAE,MAAM,kCAAkC,CAAC;AAC/E,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,gDAAgD;AAChD,4FAA4F;AAC5F,MAAM,gBAAgB,GAA2B;IAC/C,wBAAwB,EAAE,IAAI;IAC9B,wBAAwB,EAAE,IAAI;IAC9B,mBAAmB,EAAE,IAAI;IACzB,eAAe,EAAE,IAAI;IACrB,sBAAsB,EAAE,GAAG,EAAE,yBAAyB;CACvD,CAAC;AAEF,MAAM,YAAY,GAA2B;IAC3C,wBAAwB,EAAE,IAAI;IAC9B,wBAAwB,EAAE,IAAI;IAC9B,eAAe,EAAE,GAAG;IACpB,mBAAmB,EAAE,CAAC,EAAE,oBAAoB;IAC5C,kBAAkB,EAAE,CAAC,EAAE,oBAAoB;IAC3C,sBAAsB,EAAE,GAAG,EAAE,gHAAgH;CAC9I,CAAC;AAGF,+FAA+F;AAC/F,SAAS,sBAAsB,CAAC,MAAuB;IACrD,MAAM,eAAe,GAAyB;QAC5C,IAAI,sBAAsB,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC;KAC/F,CAAC;IAEF,IAAI,MAAM,CAAC,0BAA0B,EAAE,CAAC;QACtC,eAAe,CAAC,IAAI,CAClB,IAAI,OAAO,CAAC;YACV,mBAAmB,EAAE,MAAM,CAAC,cAAc;SAC3C,CAAC,CACH,CAAC;IACJ,CAAC;IAED,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAiB,EAAE,SAAiB;IAK/D,MAAM,UAAU,GAAG,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,0CAA0C;IAE3F,IAAI,SAAS,CAAC;IACd,IAAG,CAAC;QACF,SAAS,GAAG,gBAAgB,CAAC,SAA0B,CAAC,CAAC,CAAC,+BAA+B;IAC3F,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,wFAAwF;QACxF,+EAA+E;QAC/E,SAAS,GAAG,gBAAgB,CAAC,wBAAwB,CAAC,CAAC,CAAC,gCAAgC;QACxF,OAAO,CAAC,IAAI,CAAC,uBAAuB,SAAS,uCAAuC,CAAC,CAAC;IACxF,CAAC;IACD,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;QAC5C,OAAO,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAClD,CAAC,EAAE,CAAC,CAAC,CAAC;IAEN,MAAM,cAAc,GAAG,UAAU,GAAG,CAAC,UAAU,GAAG,SAAS,CAAC,CAAC;IAE7D,OAAO;QACL,cAAc;QACd,UAAU;QACV,UAAU;KACX,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC7G,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC1E,OAAO,MAAM,gBAAgB,CAAC,IAAI,CAAC;QACjC,cAAc,EAAE,cAAc;KAC/B,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,SAAqB,EACrB,MAAuB;IAEvB,OAAO,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;IAEnD,MAAM,eAAe,GAAG,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAEvD,8FAA8F;IAC9F,qEAAqE;IACrE,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,QAAQ,CAAC,yBAAyB,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACtE,CAAC;IACD,OAAO,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;IACtE,6FAA6F;IAC7F,oDAAoD;IACpD,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAEzF,4DAA4D;IAC5D,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC;QACrC,eAAe;KAChB,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAAC,GAAG,CAAC,EAAC,SAAS,EAAE,SAAS,EAAC,CAAC,CAAe,CAAC;IAEzE,OAAO,CAAC,OAAO,CAAC,uDAAuD,CAAC,CAAC;IACzE,OAAO,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC;IACtD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,MAAuB,EACvB,QAAkB;IAElB,IAAI,UAAU,CAAC;IACf,IAAI,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QACtC,UAAU,GAAG,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,EAAC,CAAE,CAAC;QAC3H,UAAU,CAAC,cAAc,GAAG,EAAE,CAAC,CAAC,wGAAwG;IAC1I,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QAC7C,UAAU,GAAG,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE;gBAClE,IAAI,EAAE,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;aAClE,GAAG,CAAC,CAAC;IACR,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,OAAO,EAAE,CAAC;QAC5C,IAAI,CAAC,QAAQ,CAAC,cAAc,IAAI,CAAC,QAAQ,CAAC,mBAAmB,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;QAC/F,CAAC;QACD,UAAU,GAAG,IAAI,oBAAoB,CAAC;YACpC,KAAK,EAAE,MAAM,CAAC,SAAS;YACvB,MAAM,EAAE,QAAQ,CAAC,cAAc;YAC/B,QAAQ,EAAE,QAAQ,CAAC,mBAAmB;YACtC,UAAU,EAAE,QAAQ,CAAC,qBAAqB,IAAI,SAAS;SACxD,CAAC,CAAC;IACL,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;QAC9C,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;QAC9E,CAAC;QACD,UAAU,GAAG,IAAI,kBAAkB,CAAC;YAClC,KAAK,EAAE,2BAA2B,CAAC,aAAa,EAAE,mBAAmB;YACrE,MAAM,EAAE,QAAQ,CAAC,aAAa;SAC/B,CAAC,CAAC;IACL,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QAC7C,IAAI,CAAC,QAAQ,CAAC,YAAY,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC5E,CAAC;QACD,UAAU,GAAG,IAAI,eAAe,CAAC;YAC/B,MAAM,EAAE,QAAQ,CAAC,YAAY;SAC9B,CAAC,CAAC;QACH,UAAU,CAAC,cAAc,GAAG,EAAE,CAAC;IACjC,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,MAAM,EAAE,CAAC;QAC3C,UAAU,GAAG,IAAI,aAAa,EAAE,CAAC;IACnC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,yCAAyC,MAAM,CAAC,aAAa,EAAE,CAAC,CAAC;IACnF,CAAC;IACD,kBAAkB,CAAC,UAAU,GAAG,UAAU,CAAC;IAC3C,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IACnG,MAAM,WAAW,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IACvE,MAAM,QAAQ,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,4BAA4B;IACnG,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IACrE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAE,CAAC;IACtF,OAAO,MAAM,0BAA0B,CAAC;QACtC,UAAU,EAAE,UAAU;QACtB,YAAY,EAAE,EAAC,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,WAAW,EAAC;QAChD,QAAQ,EAAE,QAAQ;QAClB,UAAU,EAAE,UAAU;QACpB;;;;;;;UAOE;KACL,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,SAAqB,EAAE,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IACzH,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IAC1C,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC1E,MAAM,cAAc,CAAC,QAAQ,CAAC,YAAY,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAE5D,oCAAoC;IACpC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAE,CAAC;IACtF,IAAI,cAAc,CAAC,QAAQ,YAAY,mBAAmB,EAAE,CAAC;QAC3D,aAAa;QACb,MAAO,cAAc,CAAC,QAAgC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAC7G,CAAC;SAAK,IAAI,cAAc,CAAC,QAAQ,YAAY,qBAAqB,EAAE,CAAC;QACnE,sHAAsH;QACtH,OAAO,CAAC,GAAG,CAAC,wFAAwF,CAAC,CAAC;IACxG,CAAC;IAED,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAiB,EAAE,MAAuB,EAAE,QAAkB,EAAE,OAAgB,EAAE,gBAA4D;IAC/K,kDAAkD;IAClD,OAAO,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAEtC,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC1E,MAAM,WAAW,GAAG,cAAc,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IACnE,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IACD,mCAAmC;IACnC,yCAAyC;IACzC,mDAAmD;IACnD,MAAM,KAAK,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC;QAChD,KAAK;QACL,cAAc;QACd,WAAW,EAAE,IAAI;QACjB,gBAAgB;KACjB,CAAC,CAAC;IAEH,gEAAgE;IAChE,qDAAqD;IACrD,4DAA4D;IAC5D,yHAAyH;IACzH,IAAI,WAAW,EAAE,CAAC;QAChB,IAAI,WAAW,YAAY,iBAAiB,EAAE,CAAC;YAC7C,MAAM,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,mBAAmB,CAAC,CAAC,CAAC;QACpH,CAAC;aAAM,IAAI,WAAW,YAAY,aAAa,IAAI,WAAW,YAAY,2BAA2B,EAAE,CAAC;YACtG,gFAAgF;YAChF,OAAO,CAAC,GAAG,CAAC,kGAAkG,CAAC,CAAC;QAClH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACzC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC5F,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACvD,QAAQ,MAAM,CAAC,eAAe,EAAE,CAAC;QAE/B,mEAAmE;QACnE,gCAAgC;QAChC,KAAK,UAAU;YACb,OAAO,IAAI,aAAa,CAAC;gBACvB,MAAM,EAAE,OAAO,CAAC,cAAc;gBAC9B,SAAS,EAAE,OAAO,GAAG,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC;gBAC5D,UAAU,EAAE,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,IAAI,EAAE,qCAAqC;gBAC7F,cAAc,EAAE,cAAc;aAC/B,CAAC,CAAC;QAEL,KAAK,QAAQ;YACX,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;YACrF,OAAO,iBAAiB,CAAC,cAAc,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;QAEtE,KAAK,UAAU;YACb,MAAM,WAAW,GAAG,IAAI,2BAA2B,CAAC;gBAClD,SAAS,EAAE,qBAAqB,CAAC,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;gBACzE,cAAc,EAAE,OAAO,CAAC,cAAc;gBACtC,cAAc,EAAE,cAAc;aAC/B,CAAC,CAAC;YAEH,yGAAyG;YACzG,yEAAyE;YACzE,8EAA8E;YAC9E,aAAa;YACb,WAAW,CAAC,iBAAiB,GAAG,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,EAAE,EAAE;gBAClE,OAAQ,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC;YAC/B,CAAC,CAAA;YAED,OAAO,WAAW,CAAC;QACrB;YACE,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC9F,gGAAgG;IAChG,QAAQ,MAAM,CAAC,iBAAiB,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;QAC3D,KAAK,UAAU;YACb,OAAO,IAAI,qBAAqB,CAAC;gBAC/B,MAAM,EAAE,OAAO,CAAC,cAAc;gBAC9B,SAAS,EAAE,OAAO,GAAG,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC;aAC7D,CAAC,CAAC;QACL,KAAK,QAAQ,CAAC;QACd,KAAK,UAAU;YACb,OAAO,IAAI,mBAAmB,EAAE,CAAC;QACnC;YACE,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC3F,QAAQ,MAAM,CAAC,iBAAiB,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;QAC3D,KAAK,UAAU;YACb,OAAO,IAAI,kBAAkB,CAAC;gBAC5B,MAAM,EAAE,OAAO,CAAC,cAAc;gBAC9B,SAAS,EAAE,MAAM,GAAG,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC;aAC5D,CAAC,CAAC;QACL,KAAK,QAAQ,CAAC;QACd,KAAK,UAAU;YACb,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAChC;YACE,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAChF,CAAC;AAEH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAuB,EACvB,KAAa,EACb,aAAqB,EAAE,EACvB,OAA0B;IAE1B,iGAAiG;IACjG,MAAM,eAAe,GAAoB;QACvC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;KAChC,CAAC;IACF,MAAM,SAAS,GAAG,KAAK,CAAC,WAAW,CAAC,EAAE,cAAc,EAAE,UAAU,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAC;IAE9F,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAE,CAAC;IACjD,OAAO,OAAO,CAAC;AACjB,CAAC"}
+{"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../../src/services/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,gBAAgB;AAChB,mBAAmB;AACnB,iBAAiB,EAGjB,YAAY,EAEZ,0BAA0B,EAC1B,iBAAiB,EAEjB,QAAQ,IAAI,kBAAkB,EAC9B,mBAAmB,EAGnB,gBAAgB,EACjB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,eAAe,EAAC,MAAM,oBAAoB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAAE,2BAA2B,EAAE,MAAM,qBAAqB,CAAA;AACrF,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAA;AACpD,OAAO,EAAE,aAAa,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAChG,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAsB,MAAM,aAAa,CAAC;AACnE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACzE,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,2BAA2B,EAAE,MAAM,kCAAkC,CAAC;AAC/E,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,gDAAgD;AAChD,4FAA4F;AAC5F,MAAM,gBAAgB,GAA2B;IAC/C,wBAAwB,EAAE,IAAI;IAC9B,wBAAwB,EAAE,IAAI;IAC9B,mBAAmB,EAAE,IAAI;IACzB,eAAe,EAAE,IAAI;IACrB,sBAAsB,EAAE,GAAG,EAAE,yBAAyB;CACvD,CAAC;AAEF,MAAM,YAAY,GAA2B;IAC3C,wBAAwB,EAAE,IAAI;IAC9B,wBAAwB,EAAE,IAAI;IAC9B,eAAe,EAAE,GAAG;IACpB,mBAAmB,EAAE,CAAC,EAAE,oBAAoB;IAC5C,kBAAkB,EAAE,CAAC,EAAE,oBAAoB;IAC3C,sBAAsB,EAAE,GAAG,EAAE,gHAAgH;CAC9I,CAAC;AAGF,+FAA+F;AAC/F,SAAS,sBAAsB,CAAC,MAAuB;IACrD,MAAM,eAAe,GAAyB;QAC5C,IAAI,sBAAsB,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC;KAC/F,CAAC;IAEF,IAAI,MAAM,CAAC,0BAA0B,EAAE,CAAC;QACtC,eAAe,CAAC,IAAI,CAClB,IAAI,OAAO,CAAC;YACV,mBAAmB,EAAE,MAAM,CAAC,cAAc;SAC3C,CAAC,CACH,CAAC;IACJ,CAAC;IAED,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAiB,EAAE,SAAiB;IAK/D,MAAM,UAAU,GAAG,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,0CAA0C;IAE3F,IAAI,SAAS,CAAC;IACd,IAAG,CAAC;QACF,SAAS,GAAG,gBAAgB,CAAC,SAA0B,CAAC,CAAC,CAAC,+BAA+B;IAC3F,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,wFAAwF;QACxF,+EAA+E;QAC/E,SAAS,GAAG,gBAAgB,CAAC,wBAAwB,CAAC,CAAC,CAAC,gCAAgC;QACxF,OAAO,CAAC,IAAI,CAAC,uBAAuB,SAAS,uCAAuC,CAAC,CAAC;IACxF,CAAC;IACD,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;QAC5C,OAAO,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAClD,CAAC,EAAE,CAAC,CAAC,CAAC;IAEN,MAAM,cAAc,GAAG,UAAU,GAAG,CAAC,UAAU,GAAG,SAAS,CAAC,CAAC;IAE7D,OAAO;QACL,cAAc;QACd,UAAU;QACV,UAAU;KACX,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC7G,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC1E,MAAM,WAAW,GAAG,cAAc,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IACnE,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACrE,CAAC;IACD,OAAO,MAAM,gBAAgB,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;AAC7D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,SAAqB,EACrB,MAAuB;IAEvB,OAAO,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;IAEnD,MAAM,eAAe,GAAG,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAEvD,8FAA8F;IAC9F,qEAAqE;IACrE,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,QAAQ,CAAC,yBAAyB,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACtE,CAAC;IACD,OAAO,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;IACtE,6FAA6F;IAC7F,oDAAoD;IACpD,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAEzF,4DAA4D;IAC5D,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC;QACrC,eAAe;KAChB,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAAC,GAAG,CAAC,EAAC,SAAS,EAAE,SAAS,EAAC,CAAC,CAAe,CAAC;IAEzE,OAAO,CAAC,OAAO,CAAC,uDAAuD,CAAC,CAAC;IACzE,OAAO,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC;IACtD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,MAAuB,EACvB,QAAkB;IAElB,IAAI,UAAU,CAAC;IACf,IAAI,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QACtC,UAAU,GAAG,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,EAAC,CAAE,CAAC;QAC3H,UAAU,CAAC,cAAc,GAAG,EAAE,CAAC,CAAC,wGAAwG;IAC1I,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QAC7C,UAAU,GAAG,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE;gBAClE,IAAI,EAAE,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;aAClE,GAAG,CAAC,CAAC;IACR,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,OAAO,EAAE,CAAC;QAC5C,IAAI,CAAC,QAAQ,CAAC,cAAc,IAAI,CAAC,QAAQ,CAAC,mBAAmB,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;QAC/F,CAAC;QACD,UAAU,GAAG,IAAI,oBAAoB,CAAC;YACpC,KAAK,EAAE,MAAM,CAAC,SAAS;YACvB,MAAM,EAAE,QAAQ,CAAC,cAAc;YAC/B,QAAQ,EAAE,QAAQ,CAAC,mBAAmB;YACtC,UAAU,EAAE,QAAQ,CAAC,qBAAqB,IAAI,SAAS;SACxD,CAAC,CAAC;IACL,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;QAC9C,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;QAC9E,CAAC;QACD,UAAU,GAAG,IAAI,kBAAkB,CAAC;YAClC,KAAK,EAAE,2BAA2B,CAAC,aAAa,EAAE,mBAAmB;YACrE,MAAM,EAAE,QAAQ,CAAC,aAAa;SAC/B,CAAC,CAAC;IACL,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QAC7C,IAAI,CAAC,QAAQ,CAAC,YAAY,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC5E,CAAC;QACD,UAAU,GAAG,IAAI,eAAe,CAAC;YAC/B,MAAM,EAAE,QAAQ,CAAC,YAAY;SAC9B,CAAC,CAAC;QACH,UAAU,CAAC,cAAc,GAAG,EAAE,CAAC;IACjC,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,MAAM,EAAE,CAAC;QAC3C,UAAU,GAAG,IAAI,aAAa,EAAE,CAAC;IACnC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,yCAAyC,MAAM,CAAC,aAAa,EAAE,CAAC,CAAC;IACnF,CAAC;IACD,kBAAkB,CAAC,UAAU,GAAG,UAAU,CAAC;IAC3C,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IACnG,MAAM,WAAW,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IACvE,MAAM,QAAQ,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,4BAA4B;IACnG,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IACrE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAE,CAAC;IACtF,OAAO,MAAM,0BAA0B,CAAC;QACtC,UAAU,EAAE,UAAU;QACtB,YAAY,EAAE,EAAC,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,WAAW,EAAC;QAChD,QAAQ,EAAE,QAAQ;QAClB,UAAU,EAAE,UAAU;QACpB;;;;;;;UAOE;KACL,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,SAAqB,EAAE,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IACzH,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IAC1C,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC1E,MAAM,cAAc,CAAC,QAAQ,CAAC,YAAY,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAE5D,oCAAoC;IACpC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAE,CAAC;IACtF,IAAI,cAAc,CAAC,QAAQ,YAAY,mBAAmB,EAAE,CAAC;QAC3D,aAAa;QACb,MAAO,cAAc,CAAC,QAAgC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAC7G,CAAC;SAAK,IAAI,cAAc,CAAC,QAAQ,YAAY,qBAAqB,EAAE,CAAC;QACnE,sHAAsH;QACtH,OAAO,CAAC,GAAG,CAAC,wFAAwF,CAAC,CAAC;IACxG,CAAC;IAED,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAiB,EAAE,MAAuB,EAAE,QAAkB,EAAE,OAAgB,EAAE,gBAA4D;IAC/K,kDAAkD;IAClD,OAAO,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAEtC,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC1E,MAAM,WAAW,GAAG,cAAc,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IACnE,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IACD,mCAAmC;IACnC,yCAAyC;IACzC,mDAAmD;IACnD,MAAM,eAAe,GAAG,KAAK,CAAA;IAC7B,MAAM,wBAAwB,GAAG,CAAC,QAAgB,EAAE,KAAa,EAAE,EAAE;QACnE,IAAI,gBAAgB,EAAE,CAAC;YACrB,gBAAgB,CAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;QAC1C,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,6BAA6B,EAAE,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACrE,CAAC,CAAA;IACD,MAAM,KAAK,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC;QAChD,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC;QACtC,cAAc;QACd,gBAAgB,EAAE,wBAAwB;KAC3C,CAAC,CAAC;IACH,IAAI,KAAK,CAAC,MAAM,GAAG,eAAe,EAAE,CAAC;QACnC,KAAK,IAAI,CAAC,GAAG,eAAe,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,eAAe,EAAE,CAAC;YACrE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,eAAe,CAAC,CAAC;YAClD,MAAM,KAAK,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,gBAAgB,EAAE,wBAAwB,EAAE,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IAED,gEAAgE;IAChE,qDAAqD;IACrD,4DAA4D;IAC5D,yHAAyH;IACzH,IAAI,WAAW,EAAE,CAAC;QAChB,IAAI,WAAW,YAAY,iBAAiB,EAAE,CAAC;YAC7C,MAAM,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,mBAAmB,CAAC,CAAC,CAAC;QACpH,CAAC;aAAM,IAAI,WAAW,YAAY,aAAa,IAAI,WAAW,YAAY,2BAA2B,EAAE,CAAC;YACtG,gFAAgF;YAChF,OAAO,CAAC,GAAG,CAAC,kGAAkG,CAAC,CAAC;QAClH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACzC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,0BAA0B;AAC1B,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IACnG,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACvD,QAAQ,MAAM,CAAC,eAAe,EAAE,CAAC;QAE/B,mEAAmE;QACnE,gCAAgC;QAChC,KAAK,UAAU;YACb,OAAO,IAAI,aAAa,CAAC;gBACvB,MAAM,EAAE,OAAO,CAAC,cAAc;gBAC9B,SAAS,EAAE,OAAO,GAAG,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC;gBAC5D,UAAU,EAAE,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,IAAI,EAAE,qCAAqC;gBAC7F,cAAc,EAAE,cAAc;aAC/B,CAAC,CAAC;QAEL,KAAK,QAAQ;YACX,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;YACrF,OAAO,iBAAiB,CAAC,cAAc,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;QAEtE,KAAK,UAAU;YACb,MAAM,WAAW,GAAG,IAAI,2BAA2B,CAAC;gBAClD,SAAS,EAAE,qBAAqB,CAAC,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;gBACzE,cAAc,EAAE,OAAO,CAAC,cAAc;gBACtC,cAAc,EAAE,cAAc;aAC/B,CAAC,CAAC;YAEH,yGAAyG;YACzG,yEAAyE;YACzE,8EAA8E;YAC9E,aAAa;YACb,WAAW,CAAC,iBAAiB,GAAG,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,EAAE,EAAE;gBAClE,OAAQ,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC;YAC/B,CAAC,CAAA;YAED,OAAO,WAAW,CAAC;QACrB;YACE,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,0BAA0B;AAC1B,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IACrG,gGAAgG;IAChG,QAAQ,MAAM,CAAC,iBAAiB,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;QAC3D,KAAK,UAAU;YACb,OAAO,IAAI,qBAAqB,CAAC;gBAC/B,MAAM,EAAE,OAAO,CAAC,cAAc;gBAC9B,SAAS,EAAE,OAAO,GAAG,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC;aAC7D,CAAC,CAAC;QACL,KAAK,QAAQ,CAAC;QACd,KAAK,UAAU;YACb,OAAO,IAAI,mBAAmB,EAAE,CAAC;QACnC;YACE,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,0BAA0B;AAC1B,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAClG,QAAQ,MAAM,CAAC,iBAAiB,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;QAC3D,KAAK,UAAU;YACb,OAAO,IAAI,kBAAkB,CAAC;gBAC5B,MAAM,EAAE,OAAO,CAAC,cAAc;gBAC9B,SAAS,EAAE,MAAM,GAAG,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC;aAC5D,CAAC,CAAC;QACL,KAAK,QAAQ,CAAC;QACd,KAAK,UAAU;YACb,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAChC;YACE,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAChF,CAAC;AAEH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAuB,EACvB,KAAa,EACb,aAAqB,EAAE,EACvB,OAA0B;IAE1B,iGAAiG;IACjG,MAAM,eAAe,GAAoB;QACvC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;KAChC,CAAC;IACF,MAAM,SAAS,GAAG,KAAK,CAAC,WAAW,CAAC,EAAE,cAAc,EAAE,UAAU,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAC;IAE9F,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAE,CAAC;IACjD,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@meaningfully/core",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "description": "Core functionality for meaningfully semantic search",
   "repo": "https://github.com/jeremybmerrill/meaningfully-core",
   "license": "MIT",

package/src/Meaningfully.ts

@@ -1,7 +1,7 @@
 import { MetadataManager } from './MetadataManager.js';
 import { loadDocumentsFromCsv } from './services/csvLoader.js';
 import { createEmbeddings, getIndex, search, previewResults, getDocStore } from './api/embedding.js';
-import { capitalizeFirstLetter } from './utils.js';
+import { sanitizeProjectName, capitalizeFirstLetter } from "./utils.js";
 import { join } from 'path';
 import type { DocumentSetParams, Settings, MetadataFilter, Clients } from './types/index.js';
 import fs from 'fs';
@@ -18,13 +18,13 @@ const maskKey = (key: string | null, n: number = MASKING_PREFIX_LENGTH): string
 
 
 export class MeaningfullyAPI {
-  private manager: MetadataManager;
+  private metadataManager: MetadataManager;
   private storagePath: string;
   private clients: Clients;
 
   constructor({ storagePath, weaviateClient, postgresClient, metadataManager }: { storagePath: string, weaviateClient?: any, postgresClient?: any, metadataManager: MetadataManager }) {
     this.storagePath = storagePath;
-    this.manager = metadataManager;
+    this.metadataManager = metadataManager;
     this.clients = {
       weaviateClient: weaviateClient,
       postgresClient: postgresClient
@@ -39,21 +39,50 @@ export class MeaningfullyAPI {
   }
 
   async listDocumentSets(page: number = 1, pageSize: number = 10) {
-    return await this.manager.getDocumentSets(page, pageSize);
+    return await this.metadataManager.getDocumentSets(page, pageSize);
   }
 
   async getDocumentSet(documentSetId: number) {
-    return await this.manager.getDocumentSet(documentSetId);
+    return await this.metadataManager.getDocumentSet(documentSetId);
   }
+
+
   async deleteDocumentSet(documentSetId: number) {
     // Delete the document set from the database
-    const result = await this.manager.getDocumentSet(documentSetId);
+    const result = await this.metadataManager.getDocumentSet(documentSetId);
+
     if (result){
       // Delete the document set from the database
-      await this.manager.deleteDocumentSet(documentSetId);
+      await this.metadataManager.deleteDocumentSet(documentSetId);
       // Delete the associated files from the filesystem
-      fs.rmSync(join(this.storagePath, result.name), { recursive: true, force: true });
-      fs.rmSync(join(this.storagePath, 'weaviate_data', capitalizeFirstLetter(result.name)), { recursive: true, force: true });
+      if (result.parameters.vectorStoreType === 'postgres'){
+        if (this.clients.postgresClient) {
+          try {
+            await this.deletePostgresVectorStore(result.name)
+            await this.deletePostgresIndexStore(result.name)
+            await this.deletePostgresDocStore(result.name)
+          } catch (error) {
+            console.error(`Error deleting Postgres tables for ${sanitizeProjectName(result.name)}`, error);
+          }
+        }
+      } else if (result.parameters.vectorStoreType === 'weaviate'){
+        if (this.clients.weaviateClient) {
+          try {
+            await this.deleteWeaviateVectorStore(result.name);
+          } catch (error) {
+            console.error("Error deleting Weaviate class:", error);
+          }
+        } else {
+          this.deleteSimpleVectorStore(result.name);
+        }
+        // Remove the directory and its contents
+        this.deleteSimpleDocStore(result.name);
+        this.deleteSimpleIndexStore(result.name);
+      } else {
+        this.deleteSimpleDocStore(result.name);
+        this.deleteSimpleIndexStore(result.name);
+        this.deleteSimpleVectorStore(result.name);
+      }
     }
     return { success: true };
   }
@@ -71,6 +100,7 @@ export class MeaningfullyAPI {
       if (!data.textColumns[0]) {
         throw new Error("No text column specified for preview.");
       }
+      console.log(data);
       return await previewResults(data.filePath, data.textColumns[0] as string, {
         modelName: data.modelName, // needed to tokenize, estimate costs
         modelProvider: data.modelProvider,
@@ -92,7 +122,7 @@ export class MeaningfullyAPI {
     // figure out if weaviate is available
     const vectorStoreType = this.getVectorStoreType();
     // First create the document set record
-    const documentSetId = await this.manager.addDocumentSet({
+    const documentSetId = await this.metadataManager.addDocumentSet({
       name: data.datasetName,
       uploadDate: new Date(),
       parameters: {
@@ -111,7 +141,7 @@ export class MeaningfullyAPI {
       totalDocuments: 0 // We'll update this after processing
     });
 
-    const embedSettings = await this.manager.getSettings()
+    const embedSettings = await this.metadataManager.getSettings()
 
     // Load and process the documents
     try {
@@ -120,7 +150,7 @@ export class MeaningfullyAPI {
         const documents = await loadDocumentsFromCsv(data.filePath, textColumn);
         
         // Update total documents count
-        await this.manager.updateDocumentCount(documentSetId, documents.length);
+        await this.metadataManager.updateDocumentCount(documentSetId, documents.length);
 
         // Create embeddings for this column
         let ret = await createEmbeddings(data.filePath, textColumn, {
@@ -143,7 +173,7 @@ export class MeaningfullyAPI {
       return { success: true, documentSetId };
     } catch (error) {
       // If something fails, we should probably delete the document set
-      await this.manager.deleteDocumentSet(documentSetId);
+      await this.metadataManager.deleteDocumentSet(documentSetId);
       console.error("deleting document set due to failure ", documentSetId, error);
       throw error;
     }
@@ -151,8 +181,8 @@ export class MeaningfullyAPI {
 
 
   async searchDocumentSet(documentSetId: number, query: string, n_results: number = 10,   filters?: MetadataFilter[]  ) {
-    const documentSet = await this.manager.getDocumentSet(documentSetId);
-    const settings = await this.manager.getSettings();
+    const documentSet = await this.metadataManager.getDocumentSet(documentSetId);
+    const settings = await this.metadataManager.getSettings();
     if (!documentSet) {
       throw new Error('Document set not found');
     } 
@@ -173,8 +203,8 @@ export class MeaningfullyAPI {
   }   
 
   async getDocument(documentSetId: number, documentNodeId: string){
-    const documentSet = await this.manager.getDocumentSet(documentSetId);
-    const settings = await this.manager.getSettings();
+    const documentSet = await this.metadataManager.getDocumentSet(documentSetId);
+    const settings = await this.metadataManager.getSettings();
     if (!documentSet) {
       throw new Error('Document set not found');
     } 
@@ -199,14 +229,14 @@ export class MeaningfullyAPI {
 
 
   async getSettings() {
-    return this.manager.getSettings();
+    return this.metadataManager.getSettings();
   }
   async setSettings(settings: Settings) {
-    return this.manager.setSettings(settings);
+    return this.metadataManager.setSettings(settings);
   } 
 
   async getMaskedSettings() {
-    const settings = await this.manager.getSettings();
+    const settings = await this.metadataManager.getSettings();
     return {
       openAIKey: maskKey(settings.openAIKey),
       oLlamaBaseURL: settings.oLlamaBaseURL,
@@ -218,7 +248,7 @@ export class MeaningfullyAPI {
     };
   }
   async setMaskedSettings(newSettings: Settings) { 
-    const oldSettings = await this.manager.getSettings();
+    const oldSettings = await this.metadataManager.getSettings();
     const settings = {
       ...newSettings,
       openAIKey: newSettings.openAIKey == maskKey(oldSettings.openAIKey) ? oldSettings.openAIKey : newSettings.openAIKey,
@@ -226,7 +256,54 @@ export class MeaningfullyAPI {
       mistralApiKey: newSettings.mistralApiKey == maskKey(oldSettings.mistralApiKey) ? oldSettings.mistralApiKey : newSettings.mistralApiKey,
       geminiApiKey: newSettings.geminiApiKey == maskKey(oldSettings.geminiApiKey) ? oldSettings.geminiApiKey : newSettings.geminiApiKey
     };
-    return this.manager.setSettings(settings);
+    return this.metadataManager.setSettings(settings);
+  }
+
+
+  // these should be moved to another file, just because they're too low-level for this one.
+  async deletePostgresVectorStore(projectName: string) {
+    if (this.clients.postgresClient) {
+      try {
+        await this.clients.postgresClient.query('DROP TABLE IF EXISTS vecs_' + sanitizeProjectName(projectName));
+      } catch (error) {
+        console.error(`Error deleting Postgres tables for ${sanitizeProjectName(projectName)}`, error);
+      }
+    }
+  }
+  async deletePostgresIndexStore(projectName: string) {
+    if (this.clients.postgresClient) {
+      try {
+        await this.clients.postgresClient.query('DROP TABLE IF EXISTS idx_' + sanitizeProjectName(projectName ));
+      } catch (error) {
+        console.error(`Error deleting Postgres tables for ${sanitizeProjectName(projectName)}`, error);
+      }
+    }
+  }
+  async deletePostgresDocStore(projectName: string) {
+    if (this.clients.postgresClient) {
+      try {
+        await this.clients.postgresClient.query('DROP TABLE IF EXISTS docs_' + sanitizeProjectName(projectName));
+      } catch (error) {
+        console.error(`Error deleting Postgres tables for ${sanitizeProjectName(projectName)}`, error);
+      }
+    }
+
   }
+  async deleteWeaviateVectorStore(projectName: string) {
+    if (this.clients.weaviateClient) {
+      await this.clients.weaviateClient.collections.delete(capitalizeFirstLetter(projectName));
+    }
+        // fs.rmSync(join(this.storagePath, 'weaviate_data', capitalizeFirstLetter(result.name)), { recursive: true, force: true });
+  }
+  async deleteSimpleVectorStore(projectName: string) {
+    fs.rmSync(join(this.storagePath, projectName), { recursive: true, force: true });
+
+  }
+  async deleteSimpleDocStore(projectName: string) {
+    fs.rmSync(join(this.storagePath, projectName), { recursive: true, force: true });
 
+  }
+  async deleteSimpleIndexStore(projectName: string) {
+    fs.rmSync(join(this.storagePath, projectName), { recursive: true, force: true });
+  }
 }

package/src/__tests__/Meaningfully.test.ts

@@ -0,0 +1,327 @@
+import { describe, it, beforeEach, expect, vi } from 'vitest';
+import { MetadataManager } from '../MetadataManager';
+import fs from 'fs';
+import path from 'path';
+import { sanitizeProjectName } from '../utils.js';
+import { createVectorStore, createDocumentStore, createIndexStore} from '../services/embeddings.js';
+import { IndexStruct } from 'llamaindex';
+import { Client } from 'pg'; // Import the real Postgres client
+
+
+vi.mock('../MetadataManager');
+vi.mock('fs');
+vi.mock('path');
+
+// Mock the embedding module before importing MeaningfullyAPI
+vi.doMock('../api/embedding.js', () => ({
+  getIndex: vi.fn(),
+  search: vi.fn().mockResolvedValue([{ id: 1, text: 'result' }]),
+  createEmbeddings: vi.fn().mockResolvedValue({ success: true, error: null }),
+}));
+vi.doMock('../services/csvLoader.js', () => ({
+  loadDocumentsFromCsv: vi.fn().mockResolvedValue([]),
+}));
+import { BaseNode } from 'llamaindex';
+
+// Mock BaseNode so that getEmbeddings returns made up numbers
+vi.mock('llamaindex', async (importOriginal) => {
+  const actual = await importOriginal();
+  return {
+    ...actual,
+    BaseNode: class extends actual.BaseNode {
+      async getEmbeddings() {
+        return [0.1, 0.2, 0.3, 0.4];
+      }
+      generateHash() {
+        return 'hash';
+      }
+      getContent(){
+        return "content";
+      }
+    }
+  };
+});
+
+// Import MeaningfullyAPI after mocking
+const { MeaningfullyAPI } = await import('../Meaningfully');
+
+const FAKE_SETTINGS = {
+        openAIKey: 'sk-proj-testtesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttest',
+        azureOpenAIKey: 'testtesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttest',
+        mistralApiKey: 'testtesttesttesttesttesttesttest',
+        geminiApiKey:  'testtesttesttesttesttesttesttesttesttes',
+        azureOpenAIApiVersion: "2024-02-01",
+        azureOpenAIEndpoint: "https://test.openai.azure.com",
+        oLlamaBaseURL: "http://localhost:11434",
+      }
+
+describe('MeaningfullyAPI', () => {
+  let api: MeaningfullyAPI;
+  let mockMetadataManager: MetadataManager;
+
+  beforeEach(() => {
+    // @ts-ignore
+    mockMetadataManager = new MetadataManager() as MetadataManager;
+    vi.spyOn(mockMetadataManager, 'addDocumentSet').mockResolvedValue(1);
+    vi.spyOn(mockMetadataManager, 'getSettings').mockResolvedValue(FAKE_SETTINGS);
+    vi.spyOn(mockMetadataManager, 'deleteDocumentSet').mockResolvedValue();
+    api = new MeaningfullyAPI({
+      storagePath: 'mock_storage_path',
+      metadataManager: mockMetadataManager,
+    });
+  });
+
+  describe('uploadCsv', () => {
+    it('should upload a CSV and create embeddings successfully', async () => {
+      const mockData = {
+        filePath: '/mock/file.csv',
+        datasetName: 'testDataset',
+        textColumns: ['text'],
+        metadataColumns: [],
+        splitIntoSentences: true,
+        combineSentencesIntoChunks: false,
+        sploderMaxSize: 100,
+        chunkSize: 512,
+        chunkOverlap: 0,
+        modelName: 'testModel',
+        modelProvider: 'openai',
+        description: 'Test dataset',
+      };
+
+      // Mock createEmbeddings for this test
+      const createEmbeddingsMock = vi.spyOn(await import('../api/embedding.js'), 'createEmbeddings');
+      createEmbeddingsMock.mockResolvedValue({ success: true });
+
+      const result = await api.uploadCsv(mockData);
+
+      expect(createEmbeddingsMock).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.any(String),
+        expect.objectContaining({ modelName: 'testModel' }),
+        expect.any(Object),
+        expect.any(Object)
+      );
+      expect(result).toEqual({ success: true, documentSetId: 1 });
+
+      createEmbeddingsMock.mockRestore(); // Restore the original implementation after the test
+    });
+
+    it('should handle errors during embeddings creation', async () => {
+      const mockData = {
+        filePath: '/mock/file.csv',
+        datasetName: 'testDataset',
+        textColumns: ['text'],
+        metadataColumns: [],
+        splitIntoSentences: true,
+        combineSentencesIntoChunks: false,
+        sploderMaxSize: 100,
+        chunkSize: 512,
+        chunkOverlap: 0,
+        modelName: 'testModel',
+        modelProvider: 'openai',
+        description: 'Test dataset',
+      };
+
+      // Mock createEmbeddings to simulate an error
+      const createEmbeddingsMock = vi.spyOn(await import('../api/embedding.js'), 'createEmbeddings');
+      createEmbeddingsMock.mockResolvedValue({ success: false, error: 'Embedding error' });
+
+      await expect(api.uploadCsv(mockData)).rejects.toThrow('Embedding error');
+      expect(mockMetadataManager.deleteDocumentSet).toHaveBeenCalledWith(1);
+
+      createEmbeddingsMock.mockRestore(); // Restore the original implementation after the test
+    });
+  });
+
+  describe('searchDocumentSet', () => {
+    it('should search a document set and return results', async () => {
+      vi.spyOn(mockMetadataManager, 'getDocumentSet').mockResolvedValue({
+        parameters: { modelName: 'testModel', modelProvider: 'openai', vectorStoreType: 'simple' },
+        name: 'testDataset',
+        documentSetId: 5,
+        uploadDate: new Date(),
+        totalDocuments: 420
+      });
+
+      const results = await api.searchDocumentSet(1, 'query', 10);
+
+      expect(results).toEqual([{ id: 1, text: 'result' }]);
+      expect(mockMetadataManager.getDocumentSet).toHaveBeenCalledWith(1);
+    });
+  });
+
+  describe('deleteDocumentSet', () => {
+    it('should delete a document set and associated files', async () => {
+      vi.spyOn(mockMetadataManager, 'getDocumentSet').mockResolvedValue({
+        parameters: { vectorStoreType: 'simple' },
+        name: 'testDataset',
+        documentSetId: 1,
+        uploadDate: new Date(),
+        totalDocuments: 100
+      });
+
+      vi.spyOn(fs, 'rmSync').mockImplementation(() => {});
+
+      const result = await api.deleteDocumentSet(1);
+
+      expect(mockMetadataManager.deleteDocumentSet).toHaveBeenCalledWith(1);
+      expect(fs.rmSync).toHaveBeenCalledWith(
+        path.join('mock_storage_path', 'testDataset'),
+        { recursive: true, force: true }
+      );
+      expect(result).toEqual({ success: true });
+    });
+  });
+
+  describe('getMaskedSettings', () => {
+    it('should return masked settings', async () => {
+      vi.spyOn(mockMetadataManager, 'getSettings').mockResolvedValue(FAKE_SETTINGS);
+
+      const settings = await api.getMaskedSettings();
+
+      expect(settings).toEqual({
+        openAIKey: 'sk-proj-*******testtest',
+        azureOpenAIKey: 'testtest*******testtest',
+        mistralApiKey: 'testtest*******testtest',
+        geminiApiKey: 'testtest*******ttesttes',
+        azureOpenAIApiVersion: "2024-02-01",
+        azureOpenAIEndpoint: "https://test.openai.azure.com",
+        oLlamaBaseURL:  "http://localhost:11434",
+      });
+    });
+  });
+});
+
+
+describe('MeaningfullyAPI - Store Deletion with Real Implementation', () => {
+  let api: MeaningfullyAPI;
+  let mockMetadataManager: MetadataManager;
+  let realPostgresClient: Client;
+
+  beforeEach(async () => {
+    // @ts-ignore
+    mockMetadataManager = new MetadataManager() as MetadataManager;
+
+    // Initialize a real Postgres client
+    realPostgresClient = new Client({
+      connectionString: process.env.POSTGRES_CONNECTION_STRING,
+    });
+    await realPostgresClient.connect();
+
+    api = new MeaningfullyAPI({
+      storagePath: 'mock_storage_path',
+      metadataManager: mockMetadataManager,
+      postgresClient: realPostgresClient, // Use the real client
+    });
+    vi.unmock('fs')
+    if (!fs.existsSync("mock_storage_path")){
+      fs.mkdirSync("mock_storage_path");
+    }
+  });
+
+  it('should create and delete a Postgres vector store using real implementation', async () => {
+    const projectName = 'test_project';
+    const sanitizedProjectName = sanitizeProjectName(projectName);
+    const tableName = `vecs_${sanitizedProjectName}`;
+
+    // Create the vector store using the real implementation
+    const vectorStore = await createVectorStore(
+      { vectorStoreType: 'postgres', projectName, storagePath: 'mock_storage_path', modelProvider: 'openai', modelName: 'text-embedding-ada-002' },
+      FAKE_SETTINGS,
+      api.getClients()
+    );
+    await vectorStore.add([new BaseNode({ id: '1', text: 'test document', embedding: Array(1536).fill(0.01) })]);
+
+    // Verify the table exists
+    const tableExistsQuery = `
+      SELECT EXISTS (
+        SELECT FROM information_schema.tables 
+        WHERE table_name = $1
+      );
+    `;
+    const tableExistsResult = await realPostgresClient.query(tableExistsQuery, [tableName]);
+    expect(tableExistsResult.rows[0].exists).toBe(true);
+
+    // Call the delete method
+    await api.deletePostgresVectorStore(projectName);
+
+    // Verify the table no longer exists
+    const tableDeletedResult = await realPostgresClient.query(tableExistsQuery, [tableName]);
+    expect(tableDeletedResult.rows[0].exists).toBe(false);
+  });
+
+  // it('should delete Simple vector store using real implementation', async () => {
+  //   const projectName = 'test_project';
+  //   const sanitizedProjectName = sanitizeProjectName(projectName);
+  //   const storagePath = 'mock_storage_path';
+  //   const persistDir = path.join(storagePath, sanitizedProjectName);
+
+  //   // Create the vector store using the real implementation
+  //   const vectorStore = await createVectorStore(
+  //     { vectorStoreType: 'simple', projectName, storagePath, modelProvider: "openai", modelName: 'text-embedding-3-small' },
+  //     FAKE_SETTINGS,
+  //     api.getClients()
+  //   );
+  //   await vectorStore.add([new BaseNode({ id: '1', text: 'test document', embedding: [1,2,3] })]);
+  //   await vectorStore.persist(path.join(persistDir, 'vector_store.json'));
+
+  //   // Verify the vector store exists
+  //   expect(fs.existsSync(path.join(persistDir, 'vector_store.json'))).toBe(true);
+
+  //   // Call the delete method
+  //   await api.deleteSimpleVectorStore(projectName);
+
+  //   // Verify the vector store no longer exists
+  //   expect(fs.existsSync(path.join(persistDir, 'vector_store.json'))).toBe(false);
+  // });
+
+  // it('should delete Simple document store using real implementation', async () => {
+  //   const projectName = 'test_project';
+  //   const sanitizedProjectName = sanitizeProjectName(projectName);
+  //   const storagePath = 'mock_storage_path';
+  //   const persistDir = path.join(storagePath, sanitizedProjectName);
+
+  //   // Create the document store using the real implementation
+  //   const docStore = await createDocumentStore(
+  //     { vectorStoreType: 'simple', projectName, storagePath },
+  //     FAKE_SETTINGS,
+  //     api.getClients()
+  //   );
+  //   await docStore.addDocuments([new BaseNode({ id: '1', text: 'test document' })], true);
+  //   await docStore.persist(path.join(persistDir, 'doc_store.json'));
+
+  //   // Verify the document store exists
+  //   expect(fs.existsSync(path.join(persistDir, 'doc_store.json'))).toBe(true);
+
+  //   // Call the delete method
+  //   await api.deleteSimpleDocStore(projectName);
+
+  //   // Verify the document store no longer exists
+  //   expect(fs.existsSync(path.join(persistDir, 'doc_store.json'))).toBe(false);
+  // });
+
+  // it('should delete Simple index store using real implementation', async () => {
+  //   const projectName = 'test_project';
+  //   const sanitizedProjectName = sanitizeProjectName(projectName);
+  //   const storagePath = 'mock_storage_path';
+  //   const persistDir = path.join(storagePath, sanitizedProjectName);
+
+  //   // Create the index store using the real implementation
+  //   const indexStore = await createIndexStore(
+  //     { vectorStoreType: 'simple', projectName, storagePath },
+  //     FAKE_SETTINGS,
+  //     api.getClients()
+  //   );
+  //   indexStore.addIndexStruct(new IndexStruct({ summary: 'test document' }));
+  //   await indexStore.persist(path.join(persistDir, 'index_store.json'));
+
+  //   // Verify the index store exists
+  //   expect(fs.existsSync(path.join(persistDir, 'index_store.json'))).toBe(true);
+
+  //   // Call the delete method
+  //   await api.deleteSimpleIndexStore(projectName);
+
+  //   // Verify the index store no longer exists
+  //   expect(fs.existsSync(path.join(persistDir, 'index_store.json'))).toBe(false);
+  // });
+});

package/src/api/embedding.ts

@@ -30,12 +30,13 @@ export async function createEmbeddings(
     progressManager.updateProgress(operationId, 5);
     
     const nodes = await transformDocumentsToNodes(documents, config);
-        
-    const [index] = await Promise.all([
-      persistNodes(nodes, config, settings, clients, (progress, total) => {
+      
+    const progressCallback = (progress: number, total: number) => {
         const percentage = Math.floor((progress / total) * 90) + 5; // Map to 5-95% of total progress
         progressManager.updateProgress(operationId, percentage);
-      }),
+      };
+    const [index] = await Promise.all([
+      persistNodes(nodes, config, settings, clients, progressCallback),
       persistDocuments(documents, config, settings, clients)
     ]);
     

package/src/services/embeddings.ts

@@ -101,9 +101,11 @@ export function estimateCost(nodes: TextNode[], modelName: string): {
 
 export async function getExistingVectorStoreIndex(config: EmbeddingConfig, settings: Settings, clients: Clients) {
   const storageContext = await getStorageContext(config, settings, clients);
-  return await VectorStoreIndex.init({
-    storageContext: storageContext,
-  });
+  const vectorStore = storageContext.vectorStores[ModalityType.TEXT];
+  if (!vectorStore) {
+    throw new Error("Vector store for ModalityType.TEXT is undefined");
+  }
+  return await VectorStoreIndex.fromVectorStore(vectorStore);
 }
 
 export async function transformDocumentsToNodes(
@@ -235,12 +237,24 @@ export async function persistNodes(nodes: TextNode[], config: EmbeddingConfig, s
   // Create index and embed documents
   // this is what actaully embeds the nodes
   // (even if they already have embeddings, stupidly)
+  const NODE_CHUNK_SIZE = 10000
+  const modifiedProgressCallback = (progress: number, total: number) => {
+    if (progressCallback) {
+      progressCallback(progress, nodes.length)
+    }
+    console.log('progress total nodes.length', progress, nodes.length);
+  }
   const index = await ProgressVectorStoreIndex.init({
-    nodes, 
-    storageContext, 
-    logProgress: true,
-    progressCallback,
+    nodes: nodes.slice(0, NODE_CHUNK_SIZE), 
+    storageContext,
+    progressCallback: modifiedProgressCallback
   });
+  if (nodes.length > NODE_CHUNK_SIZE) {
+    for (let i = NODE_CHUNK_SIZE; i < nodes.length; i += NODE_CHUNK_SIZE) {
+      const chunk = nodes.slice(i, i + NODE_CHUNK_SIZE);
+      await index.insertNodes(chunk, { progressCallback: modifiedProgressCallback });
+    }
+  }
 
   // I'm not sure why this explicit call to persist is necessary. 
   // storageContext should handle this, but it doesn't.
@@ -262,7 +276,8 @@ export async function persistNodes(nodes: TextNode[], config: EmbeddingConfig, s
   return index;
 }
 
-async function createVectorStore(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<PGVectorStore | SimpleVectorStore | BatchingWeaviateVectorStore> {
+// exported only for tests
+export async function createVectorStore(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<PGVectorStore | SimpleVectorStore | BatchingWeaviateVectorStore> {
   const embeddingModel = getEmbedModel(config, settings);
   switch (config.vectorStoreType) {
 
@@ -301,7 +316,8 @@ async function createVectorStore(config: EmbeddingConfig, settings: Settings, cl
   }
 }
 
-async function createDocumentStore(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<BaseDocumentStore> {
+// exported only for tests
+export async function createDocumentStore(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<BaseDocumentStore> {
   // we create the doc store without a persist path, so it doesn't write to disk after every put()
   switch (config.documentStoreType || config.vectorStoreType) {
     case "postgres":
@@ -317,7 +333,8 @@ async function createDocumentStore(config: EmbeddingConfig, settings: Settings,
   }
 }
 
-async function createIndexStore(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<BaseIndexStore> {
+// exported only for tests
+export async function createIndexStore(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<BaseIndexStore> {
   switch (config.documentStoreType || config.vectorStoreType) {
     case "postgres":
       return new PostgresIndexStore({