@meaningfully/core 0.1.0

package/.nvmrc

@@ -0,0 +1 @@
+24

package/LICENSE

@@ -0,0 +1,7 @@
+Copyright 2025 Jeremy B. F. Merrill
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -0,0 +1,3 @@
+# meaningfully-core
+
+This is a library of core/API components for [meaningfully](https://www.github.com/jeremybmerrill/meaningfully), which is an app (or collection of apps!) for semantic search over spreadsheets. See the main repo URL for more discussion.

package/dist/DocumentSetManager.d.ts

@@ -0,0 +1,28 @@
+import type { DocumentSetMetadata, Settings } from './types/index.js';
+export declare class DocumentSetManager {
+    private sqliteDb;
+    constructor(storagePath: string);
+    private initializeDatabase;
+    addDocumentSet(metadata: Omit<DocumentSetMetadata, 'documentSetId'>): Promise<number>;
+    getDocumentSet(documentSetId: number): Promise<DocumentSetMetadata | null>;
+    getDocumentSets(page?: number, pageSize?: number): Promise<{
+        documents: DocumentSetMetadata[];
+        total: number;
+    }>;
+    updateDocumentCount(documentSetId: number, count: number): Promise<void>;
+    deleteDocumentSet(documentSetId: number): Promise<void>;
+    getSettings(): Promise<{
+        openAIKey: null;
+        oLlamaBaseURL: null;
+        azureOpenAIKey: null;
+        azureOpenAIEndpoint: null;
+        azureOpenAIApiVersion: string;
+        mistralApiKey: null;
+        geminiApiKey: null;
+    } & Settings>;
+    setSettings(settings: Settings): Promise<Settings & {
+        success: boolean;
+    }>;
+    close(): void;
+}
+//# sourceMappingURL=DocumentSetManager.d.ts.map

package/dist/DocumentSetManager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"DocumentSetManager.d.ts","sourceRoot":"","sources":["../src/DocumentSetManager.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,mBAAmB,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAGtE,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,QAAQ,CAAiB;gBAErB,WAAW,EAAE,MAAM;YAQjB,kBAAkB;IAmB1B,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,mBAAmB,EAAE,eAAe,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAgBrF,cAAc,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC;IAuB1E,eAAe,CAAC,IAAI,GAAE,MAAU,EAAE,QAAQ,GAAE,MAAW,GAAG,OAAO,CAAC;QAAC,SAAS,EAAE,mBAAmB,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAC,CAAC;IAoCpH,mBAAmB,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;IAUxD,iBAAiB,CAAC,aAAa,EAAE,MAAM;IASvC,WAAW;;;;;;;;;IA8BX,WAAW,CAAC,QAAQ,EAAE,QAAQ;;;IAUpC,KAAK;CAGN"}

package/dist/DocumentSetManager.js

@@ -0,0 +1,134 @@
+import Database from 'better-sqlite3';
+import { join } from 'path';
+export class DocumentSetManager {
+    sqliteDb;
+    constructor(storagePath) {
+        // Initialize SQLite database for document set metadata
+        this.sqliteDb = new Database(join(storagePath, 'metadata.db'));
+        // Initialize the collection
+        this.initializeDatabase();
+    }
+    async initializeDatabase() {
+        // Create SQLite table for document sets
+        this.sqliteDb.exec(`
+      CREATE TABLE IF NOT EXISTS document_sets (
+        set_id INTEGER PRIMARY KEY AUTOINCREMENT,
+        name TEXT NOT NULL UNIQUE,
+        upload_date TEXT NOT NULL,
+        parameters TEXT NOT NULL,
+        total_documents INTEGER NOT NULL DEFAULT 0
+      );
+    `);
+        this.sqliteDb.exec(`
+      CREATE TABLE IF NOT EXISTS meaningfully_settings (
+        settings_id INTEGER PRIMARY KEY AUTOINCREMENT,
+        settings TEXT NOT NULL
+      );
+    `);
+    }
+    async addDocumentSet(metadata) {
+        const stmt = this.sqliteDb.prepare(`
+      INSERT INTO document_sets (name, upload_date, parameters, total_documents)
+      VALUES (?, ?, ?, ?)
+    `);
+        const result = stmt.run(metadata.name, metadata.uploadDate.toISOString(), JSON.stringify(metadata.parameters), metadata.totalDocuments);
+        return result.lastInsertRowid;
+    }
+    async getDocumentSet(documentSetId) {
+        const stmt = this.sqliteDb.prepare(`
+      SELECT * FROM document_sets WHERE set_id = ?
+    `);
+        const row = stmt.get(documentSetId);
+        if (!row)
+            return null;
+        return {
+            documentSetId: row.set_id,
+            name: row.name,
+            uploadDate: new Date(row.upload_date),
+            parameters: JSON.parse(row.parameters),
+            totalDocuments: row.total_documents
+        };
+    }
+    async getDocumentSets(page = 1, pageSize = 10) {
+        const offset = (page - 1) * pageSize;
+        // Get total count
+        const countStmt = this.sqliteDb.prepare('SELECT COUNT(*) as count FROM document_sets');
+        const totalCountRow = countStmt.get();
+        const totalCount = totalCountRow.count;
+        // Get paginated results
+        const stmt = this.sqliteDb.prepare(`
+        SELECT * FROM document_sets 
+        ORDER BY upload_date DESC 
+        LIMIT ? OFFSET ?
+    `);
+        const rows = stmt.all(pageSize, offset);
+        const documents = rows.map((row) => ({
+            documentSetId: row.set_id,
+            name: row.name,
+            uploadDate: new Date(row.upload_date),
+            parameters: JSON.parse(row.parameters),
+            totalDocuments: row.total_documents
+        }));
+        return {
+            documents,
+            total: totalCount
+        };
+    }
+    async updateDocumentCount(documentSetId, count) {
+        const stmt = this.sqliteDb.prepare(`
+      UPDATE document_sets 
+      SET total_documents = total_documents + ?
+      WHERE set_id = ?
+    `);
+        stmt.run(count, documentSetId);
+    }
+    async deleteDocumentSet(documentSetId) {
+        const stmt = this.sqliteDb.prepare(`
+      DELETE FROM document_sets
+      WHERE set_id = ?
+    `);
+        stmt.run(documentSetId);
+    }
+    async getSettings() {
+        const DEFAULT_SETTINGS = {
+            "openAIKey": null,
+            "oLlamaBaseURL": null,
+            "azureOpenAIKey": null,
+            "azureOpenAIEndpoint": null,
+            "azureOpenAIApiVersion": "2024-02-01",
+            "mistralApiKey": null,
+            "geminiApiKey": null,
+        };
+        const stmt = this.sqliteDb.prepare(`
+      SELECT * FROM meaningfully_settings WHERE settings_id = 1
+    `);
+        const row = stmt.get();
+        let settings;
+        if (row) {
+            try {
+                settings = JSON.parse(row.settings);
+            }
+            catch (error) {
+                console.error("Error parsing settings JSON:", error);
+                settings = DEFAULT_SETTINGS;
+            }
+        }
+        else {
+            settings = DEFAULT_SETTINGS;
+        }
+        settings = Object.assign({}, DEFAULT_SETTINGS, settings);
+        return settings;
+    }
+    async setSettings(settings) {
+        const stmt = this.sqliteDb.prepare(`
+      INSERT OR REPLACE INTO meaningfully_settings (settings_id, settings)
+      VALUES (1, ?)
+    `);
+        stmt.run(JSON.stringify(settings));
+        return Object.assign(settings, { "success": true });
+    }
+    close() {
+        this.sqliteDb.close();
+    }
+}
+//# sourceMappingURL=DocumentSetManager.js.map

package/dist/DocumentSetManager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"DocumentSetManager.js","sourceRoot":"","sources":["../src/DocumentSetManager.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AAGtC,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,MAAM,OAAO,kBAAkB;IACrB,QAAQ,CAAiB;IAEjC,YAAY,WAAmB;QAC7B,uDAAuD;QACvD,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC,CAAC;QAE/D,4BAA4B;QAC5B,IAAI,CAAC,kBAAkB,EAAE,CAAC;IAC5B,CAAC;IAEO,KAAK,CAAC,kBAAkB;QAC9B,wCAAwC;QACxC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;;;;;;;;KAQlB,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;;;;;KAKlB,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,QAAoD;QACvE,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;;;KAGlC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CACrB,QAAQ,CAAC,IAAI,EACb,QAAQ,CAAC,UAAU,CAAC,WAAW,EAAE,EACjC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,EACnC,QAAQ,CAAC,cAAc,CACxB,CAAC;QAEF,OAAO,MAAM,CAAC,eAAyB,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,aAAqB;QACxC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;;KAElC,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,CAMrB,CAAC;QACd,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QAEtB,OAAO;YACL,aAAa,EAAE,GAAG,CAAC,MAAM;YACzB,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,UAAU,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC;YACrC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC;YACtC,cAAc,EAAE,GAAG,CAAC,eAAe;SACpC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAe,CAAC,EAAE,WAAmB,EAAE;QAC3D,MAAM,MAAM,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC;QAErC,kBAAkB;QAClB,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,6CAA6C,CAAC,CAAC;QACvF,MAAM,aAAa,GAAG,SAAS,CAAC,GAAG,EAAuB,CAAC;QAC3D,MAAM,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC;QAEvC,wBAAwB;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;;;;KAIlC,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAMpC,CAAE;QAEJ,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACjC,aAAa,EAAE,GAAG,CAAC,MAAM;YACzB,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,UAAU,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC;YACrC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC;YACtC,cAAc,EAAE,GAAG,CAAC,eAAe;SACtC,CAAC,CAAC,CAAC;QAEJ,OAAO;YACH,SAAS;YACT,KAAK,EAAE,UAAU;SACpB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,aAAqB,EAAE,KAAa;QAC5D,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;;;;KAIlC,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,aAAqB;QAC3C,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;;;KAGlC,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,WAAW;QACf,MAAM,gBAAgB,GAAG;YACvB,WAAW,EAAE,IAAI;YACjB,eAAe,EAAE,IAAI;YACrB,gBAAgB,EAAE,IAAI;YACtB,qBAAqB,EAAE,IAAI;YAC3B,uBAAuB,EAAE,YAAY;YACrC,eAAe,EAAE,IAAI;YACrB,cAAc,EAAE,IAAI;SACrB,CAAA;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;;KAElC,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAsC,CAAC;QAC3D,IAAI,QAAQ,CAAC;QACb,IAAI,GAAG,EAAC,CAAC;YACP,IAAI,CAAC;gBACH,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAa,CAAC;YAClD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,8BAA8B,EAAE,KAAK,CAAC,CAAC;gBACrD,QAAQ,GAAG,gBAAgB,CAAC;YAC9B,CAAC;QACH,CAAC;aAAI,CAAC;YACJ,QAAQ,GAAG,gBAAgB,CAAC;QAC9B,CAAC;QACD,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,EAAE,QAAQ,CAAC,CAAA;QACxD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,QAAkB;QAClC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;;;KAGlC,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;QACnC,OAAO,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAC,SAAS,EAAE,IAAI,EAAC,CAAC,CAAC;IACpD,CAAC;IAED,KAAK;QACH,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;CACF"}

package/dist/Meaningfully.d.ts

@@ -0,0 +1,52 @@
+import { MetadataManager } from './MetadataManager.js';
+import type { DocumentSetParams, Settings, MetadataFilter, Clients } from './types/index.js';
+type HasFilePath = {
+    filePath: string;
+};
+type DocumentSetParamsFilePath = DocumentSetParams & HasFilePath;
+export declare class MeaningfullyAPI {
+    private manager;
+    private storagePath;
+    private clients;
+    constructor({ storagePath, weaviateClient, metadataManager }: {
+        storagePath: string;
+        weaviateClient?: any;
+        metadataManager: MetadataManager;
+    });
+    setClients(clients: Clients): void;
+    getClients(): Clients;
+    listDocumentSets(page?: number, pageSize?: number): Promise<{
+        documents: import("./types/index.js").DocumentSetMetadata[];
+        total: number;
+    }>;
+    getDocumentSet(documentSetId: number): Promise<import("./types/index.js").DocumentSetMetadata | null>;
+    deleteDocumentSet(documentSetId: number): Promise<{
+        success: boolean;
+    }>;
+    getVectorStoreType(): "weaviate" | "simple";
+    generatePreviewData(data: DocumentSetParamsFilePath): Promise<import("./types/index.js").PreviewResult>;
+    uploadCsv(data: DocumentSetParamsFilePath): Promise<{
+        success: boolean;
+        documentSetId: number;
+    }>;
+    searchDocumentSet(documentSetId: number, query: string, n_results?: number, filters?: MetadataFilter[]): Promise<import("./types/index.js").SearchResult[]>;
+    getDocument(documentSetId: number, documentNodeId: string): Promise<import("llamaindex").BaseNode<import("llamaindex").Metadata>>;
+    getSettings(): Promise<Settings>;
+    setSettings(settings: Settings): Promise<{
+        success: boolean;
+    }>;
+    getMaskedSettings(): Promise<{
+        openAIKey: string | null;
+        oLlamaBaseURL: string | null;
+        azureOpenAIKey: string | null;
+        azureOpenAIEndpoint: string | null;
+        azureOpenAIApiVersion: string | null;
+        mistralApiKey: string | null;
+        geminiApiKey: string | null;
+    }>;
+    setMaskedSettings(newSettings: Settings): Promise<{
+        success: boolean;
+    }>;
+}
+export {};
+//# sourceMappingURL=Meaningfully.d.ts.map

package/dist/Meaningfully.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Meaningfully.d.ts","sourceRoot":"","sources":["../src/Meaningfully.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAKvD,OAAO,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAG7F,KAAK,WAAW,GAAG;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAC,CAAC;AACtC,KAAK,yBAAyB,GAAG,iBAAiB,GAAG,WAAW,CAAC;AAUjE,qBAAa,eAAe;IAC1B,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,OAAO,CAAU;gBAEb,EAAE,WAAW,EAAE,cAAc,EAAE,eAAe,EAAE,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,GAAG,CAAC;QAAC,eAAe,EAAE,eAAe,CAAA;KAAE;IAS7I,UAAU,CAAC,OAAO,EAAE,OAAO;IAG3B,UAAU;IAIJ,gBAAgB,CAAC,IAAI,GAAE,MAAU,EAAE,QAAQ,GAAE,MAAW;;;;IAIxD,cAAc,CAAC,aAAa,EAAE,MAAM;IAGpC,iBAAiB,CAAC,aAAa,EAAE,MAAM;;;IAa7C,kBAAkB;IAIZ,mBAAmB,CAAC,IAAI,EAAE,yBAAyB;IAuBnD,SAAS,CAAC,IAAI,EAAE,yBAAyB;;;;IA8DzC,iBAAiB,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,GAAE,MAAW,EAAI,OAAO,CAAC,EAAE,cAAc,EAAE;IAsB5G,WAAW,CAAC,aAAa,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM;IAyBzD,WAAW;IAGX,WAAW,CAAC,QAAQ,EAAE,QAAQ;;;IAI9B,iBAAiB;;;;;;;;;IAYjB,iBAAiB,CAAC,WAAW,EAAE,QAAQ;;;CAY9C"}

package/dist/Meaningfully.js

@@ -0,0 +1,206 @@
+import { loadDocumentsFromCsv } from './services/csvLoader.js';
+import { createEmbeddings, getIndex, search, previewResults, getDocStore } from './api/embedding.js';
+import { capitalizeFirstLetter } from './utils.js';
+import { join } from 'path';
+import fs from 'fs';
+const MASKING_PREFIX_LENGTH = 8; // how many characters to show at the start and end of an API key when masking it for display
+// Gemini API keys are 39 chars; Mistral is 32, so MASKING_PREFIX_LENGTH must be < 16 for ANYTHING to be masked.
+const maskKey = (key, n = MASKING_PREFIX_LENGTH) => {
+    if (!key)
+        return null;
+    return (key.length > (n * 2)) ? key.slice(0, n) + "*******" + key.slice(key.length - n) : key;
+};
+export class MeaningfullyAPI {
+    manager;
+    storagePath;
+    clients;
+    constructor({ storagePath, weaviateClient, metadataManager }) {
+        this.storagePath = storagePath;
+        this.manager = metadataManager;
+        this.clients = {
+            weaviateClient: weaviateClient,
+            postgresClient: null
+        };
+    }
+    setClients(clients) {
+        this.clients = { ...this.clients, ...clients };
+    }
+    getClients() {
+        return this.clients;
+    }
+    async listDocumentSets(page = 1, pageSize = 10) {
+        return await this.manager.getDocumentSets(page, pageSize);
+    }
+    async getDocumentSet(documentSetId) {
+        return await this.manager.getDocumentSet(documentSetId);
+    }
+    async deleteDocumentSet(documentSetId) {
+        // Delete the document set from the database
+        const result = await this.manager.getDocumentSet(documentSetId);
+        if (result) {
+            // Delete the document set from the database
+            await this.manager.deleteDocumentSet(documentSetId);
+            // Delete the associated files from the filesystem
+            fs.rmSync(join(this.storagePath, result.name), { recursive: true, force: true });
+            fs.rmSync(join(this.storagePath, 'weaviate_data', capitalizeFirstLetter(result.name)), { recursive: true, force: true });
+        }
+        return { success: true };
+    }
+    getVectorStoreType() {
+        return this.clients.weaviateClient ? 'weaviate' : 'simple';
+    }
+    async generatePreviewData(data) {
+        const vectorStoreType = this.getVectorStoreType();
+        try {
+            if (!data.textColumns[0]) {
+                throw new Error("No text column specified for preview.");
+            }
+            return await previewResults(data.filePath, data.textColumns[0], {
+                modelName: data.modelName, // needed to tokenize, estimate costs
+                modelProvider: data.modelProvider,
+                splitIntoSentences: data.splitIntoSentences,
+                combineSentencesIntoChunks: data.combineSentencesIntoChunks,
+                sploderMaxSize: 100,
+                vectorStoreType: vectorStoreType,
+                projectName: data.datasetName,
+                storagePath: this.storagePath,
+                chunkSize: data.chunkSize,
+                chunkOverlap: data.chunkOverlap
+            });
+        }
+        catch (error) {
+            throw error;
+        }
+    }
+    async uploadCsv(data) {
+        // figure out if weaviate is available
+        const vectorStoreType = this.getVectorStoreType();
+        // First create the document set record
+        const documentSetId = await this.manager.addDocumentSet({
+            name: data.datasetName,
+            uploadDate: new Date(),
+            parameters: {
+                description: data.description,
+                textColumns: data.textColumns,
+                metadataColumns: data.metadataColumns,
+                splitIntoSentences: data.splitIntoSentences,
+                combineSentencesIntoChunks: data.combineSentencesIntoChunks,
+                sploderMaxSize: data.sploderMaxSize,
+                chunkSize: data.chunkSize,
+                chunkOverlap: data.chunkOverlap,
+                modelName: data.modelName,
+                modelProvider: data.modelProvider,
+                vectorStoreType: vectorStoreType,
+            },
+            totalDocuments: 0 // We'll update this after processing
+        });
+        const embedSettings = await this.manager.getSettings();
+        // Load and process the documents
+        try {
+            // Process each text column
+            for (const textColumn of data.textColumns) {
+                const documents = await loadDocumentsFromCsv(data.filePath, textColumn);
+                // Update total documents count
+                await this.manager.updateDocumentCount(documentSetId, documents.length);
+                // Create embeddings for this column
+                let ret = await createEmbeddings(data.filePath, textColumn, {
+                    modelName: data.modelName,
+                    modelProvider: data.modelProvider,
+                    splitIntoSentences: data.splitIntoSentences,
+                    combineSentencesIntoChunks: data.combineSentencesIntoChunks,
+                    sploderMaxSize: 100, // TODO: make configurable
+                    vectorStoreType: vectorStoreType,
+                    projectName: data.datasetName,
+                    // via https://medium.com/cameron-nokes/how-to-store-user-data-in-electron-3ba6bf66bc1e
+                    storagePath: this.storagePath,
+                    chunkSize: data.chunkSize,
+                    chunkOverlap: data.chunkOverlap,
+                }, embedSettings, this.clients);
+                if (!ret.success) {
+                    throw new Error(ret.error);
+                }
+            }
+            return { success: true, documentSetId };
+        }
+        catch (error) {
+            // If something fails, we should probably delete the document set
+            await this.manager.deleteDocumentSet(documentSetId);
+            console.error("deleting document set due to failure ", documentSetId, error);
+            throw error;
+        }
+    }
+    async searchDocumentSet(documentSetId, query, n_results = 10, filters) {
+        const documentSet = await this.manager.getDocumentSet(documentSetId);
+        const settings = await this.manager.getSettings();
+        if (!documentSet) {
+            throw new Error('Document set not found');
+        }
+        const index = await getIndex({
+            modelName: documentSet.parameters.modelName,
+            modelProvider: documentSet.parameters.modelProvider,
+            splitIntoSentences: documentSet.parameters.splitIntoSentences,
+            combineSentencesIntoChunks: documentSet.parameters.combineSentencesIntoChunks,
+            sploderMaxSize: 100,
+            vectorStoreType: documentSet.parameters.vectorStoreType,
+            projectName: documentSet.name,
+            storagePath: this.storagePath,
+            chunkSize: 1024, // not actually used, we just re-use a config object that has this option
+            chunkOverlap: 20, // not actually used, we just re-use a config object that has this option
+        }, settings, this.clients);
+        const results = await search(index, query, n_results, filters);
+        return results;
+    }
+    async getDocument(documentSetId, documentNodeId) {
+        const documentSet = await this.manager.getDocumentSet(documentSetId);
+        if (!documentSet) {
+            throw new Error('Document set not found');
+        }
+        const docStore = await getDocStore({
+            modelName: documentSet.parameters.modelName,
+            modelProvider: documentSet.parameters.modelProvider,
+            splitIntoSentences: documentSet.parameters.splitIntoSentences,
+            combineSentencesIntoChunks: documentSet.parameters.combineSentencesIntoChunks,
+            sploderMaxSize: 100,
+            vectorStoreType: documentSet.parameters.vectorStoreType,
+            projectName: documentSet.name,
+            storagePath: this.storagePath,
+            chunkSize: 1024, // not actually used, we just re-use a config object that has this option
+            chunkOverlap: 20, // not actually used, we just re-use a config object that has this option
+        });
+        const document = await docStore.getNode(documentNodeId);
+        if (!document) {
+            throw new Error('Document not found');
+        }
+        return document;
+    }
+    async getSettings() {
+        return this.manager.getSettings();
+    }
+    async setSettings(settings) {
+        return this.manager.setSettings(settings);
+    }
+    async getMaskedSettings() {
+        const settings = await this.manager.getSettings();
+        return {
+            openAIKey: maskKey(settings.openAIKey),
+            oLlamaBaseURL: settings.oLlamaBaseURL,
+            azureOpenAIKey: maskKey(settings.azureOpenAIKey),
+            azureOpenAIEndpoint: settings.azureOpenAIEndpoint,
+            azureOpenAIApiVersion: settings.azureOpenAIApiVersion,
+            mistralApiKey: maskKey(settings.mistralApiKey),
+            geminiApiKey: maskKey(settings.geminiApiKey)
+        };
+    }
+    async setMaskedSettings(newSettings) {
+        const oldSettings = await this.manager.getSettings();
+        const settings = {
+            ...newSettings,
+            openAIKey: newSettings.openAIKey == maskKey(oldSettings.openAIKey) ? oldSettings.openAIKey : newSettings.openAIKey,
+            azureOpenAIKey: newSettings.azureOpenAIKey == maskKey(oldSettings.azureOpenAIKey) ? oldSettings.azureOpenAIKey : newSettings.azureOpenAIKey,
+            mistralApiKey: newSettings.mistralApiKey == maskKey(oldSettings.mistralApiKey) ? oldSettings.mistralApiKey : newSettings.mistralApiKey,
+            geminiApiKey: newSettings.geminiApiKey == maskKey(oldSettings.geminiApiKey) ? oldSettings.geminiApiKey : newSettings.geminiApiKey
+        };
+        return this.manager.setSettings(settings);
+    }
+}
+//# sourceMappingURL=Meaningfully.js.map

package/dist/Meaningfully.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Meaningfully.js","sourceRoot":"","sources":["../src/Meaningfully.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACrG,OAAO,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAE,MAAM,IAAI,CAAC;AAKpB,MAAM,qBAAqB,GAAG,CAAC,CAAC,CAAC,6FAA6F;AAC5F,gHAAgH;AAClJ,MAAM,OAAO,GAAG,CAAC,GAAkB,EAAE,IAAY,qBAAqB,EAAiB,EAAE;IACvF,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,OAAO,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AAC9F,CAAC,CAAC;AAGF,MAAM,OAAO,eAAe;IAClB,OAAO,CAAkB;IACzB,WAAW,CAAS;IACpB,OAAO,CAAU;IAEzB,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,eAAe,EAAmF;QAC3I,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,eAAe,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG;YACb,cAAc,EAAE,cAAc;YAC9B,cAAc,EAAE,IAAI;SACrB,CAAC;IACJ,CAAC;IAED,UAAU,CAAC,OAAgB;QACzB,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,EAAE,CAAC;IACjD,CAAC;IACD,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,OAAe,CAAC,EAAE,WAAmB,EAAE;QAC5D,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC5D,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,aAAqB;QACxC,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAC1D,CAAC;IACD,KAAK,CAAC,iBAAiB,CAAC,aAAqB;QAC3C,4CAA4C;QAC5C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QAChE,IAAI,MAAM,EAAC,CAAC;YACV,4CAA4C;YAC5C,MAAM,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;YACpD,kDAAkD;YAClD,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACjF,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,eAAe,EAAE,qBAAqB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3H,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;IAED,kBAAkB;QAChB,OAAO,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,IAA+B;QACvD,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAClD,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAC3D,CAAC;YACD,OAAO,MAAM,cAAc,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAW,EAAE;gBACxE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,qCAAqC;gBAChE,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;gBAC3C,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;gBAC3D,cAAc,EAAE,GAAG;gBACnB,eAAe,EAAE,eAAe;gBAChC,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;aAClC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEC,KAAK,CAAC,SAAS,CAAC,IAA+B;QAC7C,sCAAsC;QACtC,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAClD,uCAAuC;QACvC,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC;YACtD,IAAI,EAAE,IAAI,CAAC,WAAW;YACtB,UAAU,EAAE,IAAI,IAAI,EAAE;YACtB,UAAU,EAAE;gBACV,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;gBAC3C,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;gBAC3D,cAAc,EAAE,IAAI,CAAC,cAAc;gBACnC,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,eAAe,EAAE,eAAe;aACjC;YACD,cAAc,EAAE,CAAC,CAAC,qCAAqC;SACxD,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAA;QAEtD,iCAAiC;QACjC,IAAI,CAAC;YACH,2BAA2B;YAC3B,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;gBAC1C,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;gBAExE,+BAA+B;gBAC/B,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,aAAa,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;gBAExE,oCAAoC;gBACpC,IAAI,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE;oBAC1D,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;oBAC3C,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;oBAC3D,cAAc,EAAE,GAAG,EAAE,0BAA0B;oBAC/C,eAAe,EAAE,eAAe;oBAChC,WAAW,EAAE,IAAI,CAAC,WAAW;oBACf,uFAAuF;oBACrG,WAAW,EAAG,IAAI,CAAC,WAAW;oBAC9B,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;iBAChC,EAAE,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;gBAChC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;oBACjB,MAAM,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC7B,CAAC;YACH,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC;QAC1C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,iEAAiE;YACjE,MAAM,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;YACpD,OAAO,CAAC,KAAK,CAAC,uCAAuC,EAAE,aAAa,EAAE,KAAK,CAAC,CAAC;YAC7E,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAGD,KAAK,CAAC,iBAAiB,CAAC,aAAqB,EAAE,KAAa,EAAE,YAAoB,EAAE,EAAI,OAA0B;QAChH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QACrE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAClD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC;YAC3B,SAAS,EAAE,WAAW,CAAC,UAAU,CAAC,SAAmB;YACrD,aAAa,EAAE,WAAW,CAAC,UAAU,CAAC,aAAuB;YAC7D,kBAAkB,EAAE,WAAW,CAAC,UAAU,CAAC,kBAA6B;YACxE,0BAA0B,EAAE,WAAW,CAAC,UAAU,CAAC,0BAAqC;YACxF,cAAc,EAAE,GAAG;YACnB,eAAe,EAAE,WAAW,CAAC,UAAU,CAAC,eAAwC;YAChF,WAAW,EAAE,WAAW,CAAC,IAAI;YAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,SAAS,EAAE,IAAI,EAAE,yEAAyE;YAC1F,YAAY,EAAE,EAAE,EAAE,yEAAyE;SAC5F,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3B,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAC/D,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,aAAqB,EAAE,cAAsB;QAC7D,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QACrE,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC;YACjC,SAAS,EAAE,WAAW,CAAC,UAAU,CAAC,SAAmB;YACrD,aAAa,EAAE,WAAW,CAAC,UAAU,CAAC,aAAuB;YAC7D,kBAAkB,EAAE,WAAW,CAAC,UAAU,CAAC,kBAA6B;YACxE,0BAA0B,EAAE,WAAW,CAAC,UAAU,CAAC,0BAAqC;YACxF,cAAc,EAAE,GAAG;YACnB,eAAe,EAAE,WAAW,CAAC,UAAU,CAAC,eAAwC;YAChF,WAAW,EAAE,WAAW,CAAC,IAAI;YAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,SAAS,EAAE,IAAI,EAAE,yEAAyE;YAC1F,YAAY,EAAE,EAAE,EAAE,yEAAyE;SAC5F,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QACxD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACxC,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAGD,KAAK,CAAC,WAAW;QACf,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IACpC,CAAC;IACD,KAAK,CAAC,WAAW,CAAC,QAAkB;QAClC,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,iBAAiB;QACrB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAClD,OAAO;YACL,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;YACtC,aAAa,EAAE,QAAQ,CAAC,aAAa;YACrC,cAAc,EAAE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;YAChD,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;YACjD,qBAAqB,EAAE,QAAQ,CAAC,qBAAqB;YACrD,aAAa,EAAE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;YAC9C,YAAY,EAAE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;SAC7C,CAAC;IACJ,CAAC;IACD,KAAK,CAAC,iBAAiB,CAAC,WAAqB;QAC3C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QACrD,MAAM,QAAQ,GAAG;YACf,GAAG,WAAW;YACd,SAAS,EAAE,WAAW,CAAC,SAAS,IAAI,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS;YAClH,cAAc,EAAE,WAAW,CAAC,cAAc,IAAI,OAAO,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc;YAC3I,aAAa,EAAE,WAAW,CAAC,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa;YACtI,YAAY,EAAE,WAAW,CAAC,YAAY,IAAI,OAAO,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,YAAY;SAClI,CAAC;QACF,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC5C,CAAC;CAEF"}

package/dist/MetadataManager.d.ts

@@ -0,0 +1,32 @@
+import type { DocumentSetMetadata, Settings } from './types/index.js';
+export declare abstract class MetadataManager {
+    protected queries: {
+        createDocumentSetsTable: string;
+        createSettingsTable: string;
+        insertDocumentSet: string;
+        selectDocumentSet: string;
+        selectDocumentSets: string;
+        countDocumentSets: string;
+        updateDocumentCount: string;
+        deleteDocumentSet: string;
+        selectSettings: string;
+        upsertSettings: string;
+    };
+    protected abstract runQuery<T>(query: string, params?: any[]): Promise<T[]>;
+    protected abstract runQuerySingle<T>(query: string, params?: any[]): Promise<T | null>;
+    protected abstract initializeDatabase(): Promise<void>;
+    protected abstract close(): void;
+    addDocumentSet(metadata: Omit<DocumentSetMetadata, 'documentSetId'>): Promise<number>;
+    getDocumentSet(documentSetId: number): Promise<DocumentSetMetadata | null>;
+    getDocumentSets(page?: number, pageSize?: number): Promise<{
+        documents: DocumentSetMetadata[];
+        total: number;
+    }>;
+    updateDocumentCount(documentSetId: number, count: number): Promise<void>;
+    deleteDocumentSet(documentSetId: number): Promise<void>;
+    getSettings(): Promise<Settings>;
+    setSettings(settings: Settings): Promise<{
+        success: boolean;
+    }>;
+}
+//# sourceMappingURL=MetadataManager.d.ts.map

package/dist/MetadataManager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"MetadataManager.d.ts","sourceRoot":"","sources":["../src/MetadataManager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAEtE,8BAAsB,eAAe;IACnC,SAAS,CAAC,OAAO;;;;;;;;;;;MAmDf;IAEF,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;IAC3E,SAAS,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IACtF,SAAS,CAAC,QAAQ,CAAC,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC;IACtD,SAAS,CAAC,QAAQ,CAAC,KAAK,IAAI,IAAI;IAE1B,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,mBAAmB,EAAE,eAAe,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAUrF,cAAc,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC;IAoB1E,eAAe,CAAC,IAAI,GAAE,MAAU,EAAE,QAAQ,GAAE,MAAW,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,mBAAmB,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAwBtH,mBAAmB,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIxE,iBAAiB,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIvD,WAAW,IAAI,OAAO,CAAC,QAAQ,CAAC;IAehC,WAAW,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC;CAMrE"}

package/dist/MetadataManager.js

@@ -0,0 +1,115 @@
+export class MetadataManager {
+    queries = {
+        /*
+        Note: RETURNING on non-select/non-create statements is important for compatibility between SQLite and PostgreSQL.
+        (Without it, better-sqlite would demand to use run() instead of all() or get(), which would break the abstraction.)
+        */
+        createDocumentSetsTable: `
+      CREATE TABLE IF NOT EXISTS document_sets (
+        set_id SERIAL PRIMARY KEY,
+        name TEXT NOT NULL UNIQUE,
+        upload_date TIMESTAMP NOT NULL,
+        parameters TEXT NOT NULL,
+        total_documents INTEGER NOT NULL DEFAULT 0
+      );
+    `,
+        createSettingsTable: `
+      CREATE TABLE IF NOT EXISTS meaningfully_settings (
+        settings_id SERIAL PRIMARY KEY,
+        settings TEXT NOT NULL
+      );
+    `,
+        insertDocumentSet: `
+      INSERT INTO document_sets (name, upload_date, parameters, total_documents)
+      VALUES ($1, $2, $3, $4) RETURNING set_id
+    `,
+        selectDocumentSet: `
+      SELECT * FROM document_sets WHERE set_id = $1
+    `,
+        selectDocumentSets: `
+      SELECT * FROM document_sets ORDER BY upload_date DESC LIMIT $1 OFFSET $2
+    `,
+        countDocumentSets: `
+      SELECT COUNT(*) as count FROM document_sets
+    `,
+        updateDocumentCount: `
+      UPDATE document_sets SET total_documents = total_documents + $1 WHERE set_id = $2 RETURNING *
+    `,
+        deleteDocumentSet: `
+      DELETE FROM document_sets WHERE set_id = $1 RETURNING *
+    `,
+        selectSettings: `
+      SELECT * FROM meaningfully_settings WHERE settings_id = 1
+    `,
+        upsertSettings: `
+      INSERT INTO meaningfully_settings (settings_id, settings)
+      VALUES (1, $1)
+      ON CONFLICT (settings_id) DO UPDATE SET settings = $2
+      RETURNING *
+    `
+        // the two arguments $1 and $2 are identical, but, to work around a cross-compatibility bug in SQLite versus Postgresql,
+        // where PG can accept the same argument twice (specified as $1 in two places), but SQLITE cannot (it just has ? placeholders)
+        // they are specified separately.
+    };
+    async addDocumentSet(metadata) {
+        const result = await this.runQuerySingle(this.queries.insertDocumentSet, [
+            metadata.name,
+            metadata.uploadDate.toISOString(),
+            JSON.stringify(metadata.parameters),
+            metadata.totalDocuments
+        ]);
+        return result?.set_id || 0;
+    }
+    async getDocumentSet(documentSetId) {
+        const row = await this.runQuerySingle(this.queries.selectDocumentSet, [documentSetId]);
+        if (!row)
+            return null;
+        return {
+            documentSetId: row.set_id,
+            name: row.name,
+            uploadDate: new Date(row.upload_date),
+            parameters: JSON.parse(row.parameters),
+            totalDocuments: row.total_documents
+        };
+    }
+    async getDocumentSets(page = 1, pageSize = 10) {
+        const offset = (page - 1) * pageSize;
+        const totalCountRow = await this.runQuerySingle(this.queries.countDocumentSets);
+        const totalCount = totalCountRow?.count || 0;
+        const rows = await this.runQuery(this.queries.selectDocumentSets, [pageSize, offset]);
+        const documents = rows.map((row) => ({
+            documentSetId: row.set_id,
+            name: row.name,
+            uploadDate: new Date(row.upload_date),
+            parameters: JSON.parse(row.parameters),
+            totalDocuments: row.total_documents
+        }));
+        return { documents, total: totalCount };
+    }
+    async updateDocumentCount(documentSetId, count) {
+        await this.runQuery(this.queries.updateDocumentCount, [count, documentSetId]);
+    }
+    async deleteDocumentSet(documentSetId) {
+        await this.runQuery(this.queries.deleteDocumentSet, [documentSetId]);
+    }
+    async getSettings() {
+        const DEFAULT_SETTINGS = {
+            openAIKey: null,
+            oLlamaBaseURL: null,
+            azureOpenAIKey: null,
+            azureOpenAIEndpoint: null,
+            azureOpenAIApiVersion: "2024-02-01",
+            mistralApiKey: null,
+            geminiApiKey: null,
+        };
+        const row = await this.runQuerySingle(this.queries.selectSettings);
+        return row ? { ...DEFAULT_SETTINGS, ...JSON.parse(row.settings) } : DEFAULT_SETTINGS;
+    }
+    async setSettings(settings) {
+        // the JSON.stringify(settings) is repeated to work around a cross-compatibility bug in SQLite versus Postgresql
+        // where PG can accept the same argument twice (specified as $1 in two places), but SQLITE cannot (it just has ? placeholders)
+        await this.runQuery(this.queries.upsertSettings, [JSON.stringify(settings), JSON.stringify(settings)]);
+        return { success: true };
+    }
+}
+//# sourceMappingURL=MetadataManager.js.map

package/dist/MetadataManager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MetadataManager.js","sourceRoot":"","sources":["../src/MetadataManager.ts"],"names":[],"mappings":"AAEA,MAAM,OAAgB,eAAe;IACzB,OAAO,GAAG;QAClB;;;UAGE;QACF,uBAAuB,EAAE;;;;;;;;KAQxB;QACD,mBAAmB,EAAE;;;;;KAKpB;QACD,iBAAiB,EAAE;;;KAGlB;QACD,iBAAiB,EAAE;;KAElB;QACD,kBAAkB,EAAE;;KAEnB;QACD,iBAAiB,EAAE;;KAElB;QACD,mBAAmB,EAAE;;KAEpB;QACD,iBAAiB,EAAE;;KAElB;QACD,cAAc,EAAE;;KAEf;QACD,cAAc,EAAE;;;;;KAKf;QACD,wHAAwH;QACxH,8HAA8H;QAC9H,iCAAiC;KAClC,CAAC;IAOF,KAAK,CAAC,cAAc,CAAC,QAAoD;QACvE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAqB,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE;YAC3F,QAAQ,CAAC,IAAI;YACb,QAAQ,CAAC,UAAU,CAAC,WAAW,EAAE;YACjC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC;YACnC,QAAQ,CAAC,cAAc;SACxB,CAAC,CAAC;QACH,OAAO,MAAM,EAAE,MAAM,IAAI,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,aAAqB;QACxC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,cAAc,CAMlC,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC;QAEpD,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QAEtB,OAAO;YACL,aAAa,EAAE,GAAG,CAAC,MAAM;YACzB,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,UAAU,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC;YACrC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC;YACtC,cAAc,EAAE,GAAG,CAAC,eAAe;SACpC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAe,CAAC,EAAE,WAAmB,EAAE;QAC3D,MAAM,MAAM,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC;QACrC,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,cAAc,CAAoB,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;QACnG,MAAM,UAAU,GAAG,aAAa,EAAE,KAAK,IAAI,CAAC,CAAC;QAE7C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAM7B,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;QAExD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACnC,aAAa,EAAE,GAAG,CAAC,MAAM;YACzB,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,UAAU,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC;YACrC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC;YACtC,cAAc,EAAE,GAAG,CAAC,eAAe;SACpC,CAAC,CAAC,CAAC;QAEJ,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,aAAqB,EAAE,KAAa;QAC5D,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC;IAChF,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,aAAqB;QAC3C,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,WAAW;QACf,MAAM,gBAAgB,GAAa;YACjC,SAAS,EAAE,IAAI;YACf,aAAa,EAAE,IAAI;YACnB,cAAc,EAAE,IAAI;YACpB,mBAAmB,EAAE,IAAI;YACzB,qBAAqB,EAAE,YAAY;YACnC,aAAa,EAAE,IAAI;YACnB,YAAY,EAAE,IAAI;SACnB,CAAC;QAEF,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,cAAc,CAAuB,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QACzF,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,gBAAgB,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,gBAAgB,CAAC;IACvF,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,QAAkB;QAClC,gHAAgH;QAChH,8HAA8H;QAC9H,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QACvG,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;CACF"}

package/dist/api/embedding.d.ts

@@ -0,0 +1,7 @@
+import type { EmbeddingConfig, EmbeddingResult, SearchResult, PreviewResult, Settings, MetadataFilter, Clients } from "../types/index.js";
+export declare function createEmbeddings(csvPath: string, textColumnName: string, config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<EmbeddingResult>;
+export declare function previewResults(csvPath: string, textColumnName: string, config: EmbeddingConfig): Promise<PreviewResult>;
+export declare function getDocStore(config: EmbeddingConfig): Promise<import("llamaindex").BaseDocumentStore>;
+export declare function getIndex(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<import("llamaindex").VectorStoreIndex>;
+export declare function search(index: any, query: string, numResults?: number, filters?: MetadataFilter[]): Promise<SearchResult[]>;
+//# sourceMappingURL=embedding.d.ts.map

package/dist/api/embedding.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding.d.ts","sourceRoot":"","sources":["../../src/api/embedding.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAK1I,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,MAAM,EACf,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,eAAe,EACvB,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,eAAe,CAAC,CAyC1B;AAGD,wBAAsB,cAAc,CAClC,OAAO,EAAE,MAAM,EACf,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,eAAe,GACtB,OAAO,CAAC,aAAa,CAAC,CAqCxB;AAED,wBAAsB,WAAW,CAAC,MAAM,EAAE,eAAe,mDAExD;AAED,wBAAsB,QAAQ,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,kDAE3F;AAED,wBAAsB,MAAM,CAC1B,KAAK,EAAE,GAAG,EACV,KAAK,EAAE,MAAM,EACb,UAAU,GAAE,MAAW,EACvB,OAAO,CAAC,EAAE,cAAc,EAAE,GACzB,OAAO,CAAC,YAAY,EAAE,CAAC,CASzB"}