@meaningfully/core 0.1.0

package/dist/types/index.d.ts

@@ -0,0 +1,77 @@
+export interface SearchResult {
+    text: string;
+    score: number;
+    metadata: Record<string, any>;
+}
+export interface EmbeddingResult {
+    success: boolean;
+    error?: string;
+    index?: any;
+}
+export interface PreviewResult {
+    success: boolean;
+    error?: string;
+    nodes?: Array<{
+        text: string;
+        metadata: Record<string, any>;
+    }>;
+    estimatedPrice?: number;
+    tokenCount?: number;
+    pricePer1M?: number;
+}
+export interface SearchConfig {
+    modelProvider: string;
+    modelName: string;
+    projectName: string;
+}
+export interface DocumentSetMetadata {
+    documentSetId: number;
+    name: string;
+    uploadDate: Date;
+    parameters: Record<string, unknown>;
+    totalDocuments: number;
+}
+export interface DocumentSetParams {
+    datasetName: string;
+    description: string;
+    textColumns: string[];
+    metadataColumns: string[];
+    splitIntoSentences: boolean;
+    combineSentencesIntoChunks: boolean;
+    sploderMaxSize: number;
+    chunkSize: number;
+    chunkOverlap: number;
+    modelName: string;
+    modelProvider: string;
+}
+export interface EmbeddingConfig {
+    modelName: string;
+    modelProvider: string;
+    vectorStoreType: "simple" | "postgres" | "weaviate";
+    projectName: string;
+    storagePath: string;
+    splitIntoSentences: boolean;
+    combineSentencesIntoChunks: boolean;
+    sploderMaxSize: number;
+    chunkSize: number;
+    chunkOverlap: number;
+}
+export interface Settings {
+    openAIKey: string | null;
+    oLlamaBaseURL: string | null;
+    azureOpenAIKey: string | null;
+    azureOpenAIEndpoint: string | null;
+    azureOpenAIApiVersion: string | null;
+    mistralApiKey: string | null;
+    geminiApiKey: string | null;
+}
+export interface MetadataFilter {
+    key: string;
+    operator: "==" | "in" | ">" | "<" | "!=" | ">=" | "<=" | "nin" | "any" | "all" | "text_match" | "contains" | "is_empty";
+    value: any;
+}
+export interface Clients {
+    weaviateClient: any;
+    postgresClient: any;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/types/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,GAAG,CAAC;CACb;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,KAAK,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KAC/B,CAAC,CAAC;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAGD,MAAM,WAAW,YAAY;IAC3B,aAAa,EAAE,MAAM,CAAA;IACrB,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;CACpB;AAGD,MAAM,WAAW,mBAAmB;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,IAAI,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACpC,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,kBAAkB,EAAE,OAAO,CAAC;IAC5B,0BAA0B,EAAE,OAAO,CAAC;IACpC,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAA;CACtB;AAGD,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAA;IACrB,eAAe,EAAE,QAAQ,GAAG,UAAU,GAAG,UAAU,CAAC;IACpD,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,0BAA0B,EAAE,OAAO,CAAC;IACpC,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;CACtB;AAGD,MAAM,WAAW,QAAQ;IACvB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,IAAI,GAAG,IAAI,GAAG,GAAG,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,YAAY,GAAG,UAAU,GAAG,UAAU,CAAC;IACxH,KAAK,EAAE,GAAG,CAAA;CACX;AAED,MAAM,WAAW,OAAO;IACtB,cAAc,EAAE,GAAG,CAAC;IACpB,cAAc,EAAE,GAAG,CAAC;CACrB"}

package/dist/types/index.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=index.js.map

package/dist/types/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":""}

package/dist/utils.d.ts

@@ -0,0 +1,3 @@
+export declare function sanitizeProjectName(projectName: string): string;
+export declare function capitalizeFirstLetter(val: string): string;
+//# sourceMappingURL=utils.d.ts.map

package/dist/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,wBAAgB,mBAAmB,CAAC,WAAW,EAAE,MAAM,UAEtD;AACD,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,MAAM,UAEhD"}

package/dist/utils.js

@@ -0,0 +1,7 @@
+export function sanitizeProjectName(projectName) {
+    return projectName.replace(/[^a-zA-Z0-9]/g, "_");
+}
+export function capitalizeFirstLetter(val) {
+    return String(val).charAt(0).toUpperCase() + String(val).slice(1);
+}
+//# sourceMappingURL=utils.js.map

package/dist/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,mBAAmB,CAAC,WAAmB;IACrD,OAAO,WAAW,CAAC,OAAO,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;AACnD,CAAC;AACD,MAAM,UAAU,qBAAqB,CAAC,GAAW;IAC/C,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACpE,CAAC"}

package/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "@meaningfully/core",
+  "version": "0.1.0",
+  "description": "Core functionality for meaningfully semantic search",
+  "license": "MIT",
+  "type": "module",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "scripts": {
+    "build": "tsc",
+    "watch": "tsc --watch",
+    "test": "vitest --silent=false --disable-console-intercept"
+  },
+  "dependencies": {
+    "@llamaindex/azure": "^0.1.31",
+    "@llamaindex/google": "^0.3.18",
+    "@llamaindex/mistral": "^0.1.14",
+    "@llamaindex/ollama": "^0.1.14",
+    "@llamaindex/openai": "^0.4.14",
+    "@llamaindex/postgres": "^0.0.64",
+    "@llamaindex/weaviate": "^0.0.36",
+    "js-tiktoken": "^1.0.8",
+    "llamaindex": "^0.11.14",
+    "lodash": "^4.17.21",
+    "natural": "^8.1.0",
+    "openai": "^5.9.0",
+    "papaparse": "^5.5.3"
+  },
+  "devDependencies": {
+    "@types/node": "^20.5.6",
+    "@types/papaparse": "^5.3.16",
+    "typescript": "^5.7.3",
+    "vite": "^5.4.4",
+    "vitest": "^3.0.8"
+  }
+}

package/src/Meaningfully.d.ts

@@ -0,0 +1,57 @@
+import { DocumentSetParams, Settings, MetadataFilter, Clients } from './types';
+type HasFilePath = {
+    filePath: string;
+};
+type DocumentSetParamsFilePath = DocumentSetParams & HasFilePath;
+export declare class MeaningfullyAPI {
+    private manager;
+    private storagePath;
+    private clients;
+    constructor({ storagePath, weaviateClient }: {
+        storagePath: string;
+        weaviateClient?: any;
+    });
+    setClients(clients: Clients): void;
+    getClients(): Clients;
+    listDocumentSets(page?: number, pageSize?: number): Promise<{
+        documents: import("./types").DocumentSetMetadata[];
+        total: number;
+    }>;
+    getDocumentSet(documentSetId: number): Promise<import("./types").DocumentSetMetadata | null>;
+    deleteDocumentSet(documentSetId: number): Promise<{
+        success: boolean;
+    }>;
+    getVectorStoreType(): "simple" | "weaviate";
+    generatePreviewData(data: DocumentSetParamsFilePath): Promise<import("./types").PreviewResult>;
+    uploadCsv(data: DocumentSetParamsFilePath): Promise<{
+        success: boolean;
+        documentSetId: number;
+    }>;
+    searchDocumentSet(documentSetId: number, query: string, n_results?: number, filters?: MetadataFilter[]): Promise<import("./types").SearchResult[]>;
+    getDocument(documentSetId: number, documentNodeId: string): Promise<import("llamaindex").BaseNode<import("llamaindex").Metadata>>;
+    getSettings(): Promise<{
+        openAIKey: null;
+        oLlamaBaseURL: null;
+        azureOpenAIKey: null;
+        azureOpenAIEndpoint: null;
+        azureOpenAIApiVersion: string;
+        mistralApiKey: null;
+        geminiApiKey: null;
+    } & Settings>;
+    setSettings(settings: Settings): Promise<Settings & {
+        success: boolean;
+    }>;
+    getMaskedSettings(): Promise<{
+        openAIKey: string | null;
+        oLlamaBaseURL: null;
+        azureOpenAIKey: string | null;
+        azureOpenAIEndpoint: null;
+        azureOpenAIApiVersion: string;
+        mistralApiKey: string | null;
+        geminiApiKey: string | null;
+    }>;
+    setMaskedSettings(newSettings: Settings): Promise<Settings & {
+        success: boolean;
+    }>;
+}
+export {};

package/src/Meaningfully.ts

@@ -0,0 +1,228 @@
+import { MetadataManager } from './MetadataManager.js';
+import { loadDocumentsFromCsv } from './services/csvLoader.js';
+import { createEmbeddings, getIndex, search, previewResults, getDocStore } from './api/embedding.js';
+import { capitalizeFirstLetter } from './utils.js';
+import { join } from 'path';
+import type { DocumentSetParams, Settings, MetadataFilter, Clients } from './types/index.js';
+import fs from 'fs';
+
+type HasFilePath = {filePath: string};
+type DocumentSetParamsFilePath = DocumentSetParams & HasFilePath;
+
+const MASKING_PREFIX_LENGTH = 8; // how many characters to show at the start and end of an API key when masking it for display
+                                  // Gemini API keys are 39 chars; Mistral is 32, so MASKING_PREFIX_LENGTH must be < 16 for ANYTHING to be masked.
+const maskKey = (key: string | null, n: number = MASKING_PREFIX_LENGTH): string | null => {
+  if (!key) return null;
+  return (key.length > (n*2)) ? key.slice(0, n) + "*******" + key.slice(key.length - n) : key;
+};
+
+
+export class MeaningfullyAPI {
+  private manager: MetadataManager;
+  private storagePath: string;
+  private clients: Clients;
+
+  constructor({ storagePath, weaviateClient, metadataManager }: { storagePath: string, weaviateClient?: any, metadataManager: MetadataManager }) {
+    this.storagePath = storagePath;
+    this.manager = metadataManager;
+    this.clients = {
+      weaviateClient: weaviateClient,
+      postgresClient: null
+    };
+  }
+
+  setClients(clients: Clients) {
+    this.clients = { ...this.clients, ...clients };
+  }
+  getClients() {
+    return this.clients;
+  }
+
+  async listDocumentSets(page: number = 1, pageSize: number = 10) {
+    return await this.manager.getDocumentSets(page, pageSize);
+  }
+
+  async getDocumentSet(documentSetId: number) {
+    return await this.manager.getDocumentSet(documentSetId);
+  }
+  async deleteDocumentSet(documentSetId: number) {
+    // Delete the document set from the database
+    const result = await this.manager.getDocumentSet(documentSetId);
+    if (result){
+      // Delete the document set from the database
+      await this.manager.deleteDocumentSet(documentSetId);
+      // Delete the associated files from the filesystem
+      fs.rmSync(join(this.storagePath, result.name), { recursive: true, force: true });
+      fs.rmSync(join(this.storagePath, 'weaviate_data', capitalizeFirstLetter(result.name)), { recursive: true, force: true });
+    }
+    return { success: true };
+  }
+
+  getVectorStoreType() {
+    return this.clients.weaviateClient ? 'weaviate' : 'simple';
+  }
+
+  async generatePreviewData(data: DocumentSetParamsFilePath) {
+    const vectorStoreType = this.getVectorStoreType();
+    try {
+      if (!data.textColumns[0]) {
+        throw new Error("No text column specified for preview.");
+      }
+      return await previewResults(data.filePath, data.textColumns[0] as string, {
+        modelName: data.modelName, // needed to tokenize, estimate costs
+        modelProvider: data.modelProvider,
+        splitIntoSentences: data.splitIntoSentences,
+        combineSentencesIntoChunks: data.combineSentencesIntoChunks,
+        sploderMaxSize: 100,
+        vectorStoreType: vectorStoreType,
+        projectName: data.datasetName,
+        storagePath: this.storagePath,
+        chunkSize: data.chunkSize,
+        chunkOverlap: data.chunkOverlap
+    });
+  } catch (error) {
+    throw error;
+  }
+}
+
+  async uploadCsv(data: DocumentSetParamsFilePath) {
+    // figure out if weaviate is available
+    const vectorStoreType = this.getVectorStoreType();
+    // First create the document set record
+    const documentSetId = await this.manager.addDocumentSet({
+      name: data.datasetName,
+      uploadDate: new Date(),
+      parameters: {
+        description: data.description,
+        textColumns: data.textColumns,
+        metadataColumns: data.metadataColumns,
+        splitIntoSentences: data.splitIntoSentences,
+        combineSentencesIntoChunks: data.combineSentencesIntoChunks,
+        sploderMaxSize: data.sploderMaxSize,
+        chunkSize: data.chunkSize,
+        chunkOverlap: data.chunkOverlap,
+        modelName: data.modelName,
+        modelProvider: data.modelProvider,
+        vectorStoreType: vectorStoreType,
+      },
+      totalDocuments: 0 // We'll update this after processing
+    });
+
+    const embedSettings = await this.manager.getSettings()
+
+    // Load and process the documents
+    try {
+      // Process each text column
+      for (const textColumn of data.textColumns) {
+        const documents = await loadDocumentsFromCsv(data.filePath, textColumn);
+        
+        // Update total documents count
+        await this.manager.updateDocumentCount(documentSetId, documents.length);
+
+        // Create embeddings for this column
+        let ret = await createEmbeddings(data.filePath, textColumn, {
+          modelName: data.modelName,
+          modelProvider: data.modelProvider,
+          splitIntoSentences: data.splitIntoSentences,
+          combineSentencesIntoChunks: data.combineSentencesIntoChunks,
+          sploderMaxSize: 100, // TODO: make configurable
+          vectorStoreType: vectorStoreType,
+          projectName: data.datasetName,
+                        // via https://medium.com/cameron-nokes/how-to-store-user-data-in-electron-3ba6bf66bc1e
+          storagePath:  this.storagePath,
+          chunkSize: data.chunkSize,
+          chunkOverlap: data.chunkOverlap,
+        }, embedSettings, this.clients);
+        if (!ret.success) {
+          throw new Error(ret.error);
+        }
+      }
+      return { success: true, documentSetId };
+    } catch (error) {
+      // If something fails, we should probably delete the document set
+      await this.manager.deleteDocumentSet(documentSetId);
+      console.error("deleting document set due to failure ", documentSetId, error);
+      throw error;
+    }
+  }
+
+
+  async searchDocumentSet(documentSetId: number, query: string, n_results: number = 10,   filters?: MetadataFilter[]  ) {
+    const documentSet = await this.manager.getDocumentSet(documentSetId);
+    const settings = await this.manager.getSettings();
+    if (!documentSet) {
+      throw new Error('Document set not found');
+    } 
+    const index = await getIndex({
+      modelName: documentSet.parameters.modelName as string,
+      modelProvider: documentSet.parameters.modelProvider as string,
+      splitIntoSentences: documentSet.parameters.splitIntoSentences as boolean,
+      combineSentencesIntoChunks: documentSet.parameters.combineSentencesIntoChunks as boolean,
+      sploderMaxSize: 100,
+      vectorStoreType: documentSet.parameters.vectorStoreType as 'simple' | 'weaviate',
+      projectName: documentSet.name,
+      storagePath: this.storagePath,
+      chunkSize: 1024, // not actually used, we just re-use a config object that has this option
+      chunkOverlap: 20, // not actually used, we just re-use a config object that has this option
+    }, settings, this.clients);
+    const results = await search(index, query, n_results, filters);
+    return results;
+  }   
+
+  async getDocument(documentSetId: number, documentNodeId: string){
+    const documentSet = await this.manager.getDocumentSet(documentSetId);
+    if (!documentSet) {
+      throw new Error('Document set not found');
+    } 
+    const docStore = await getDocStore({
+      modelName: documentSet.parameters.modelName as string,
+      modelProvider: documentSet.parameters.modelProvider as string,
+      splitIntoSentences: documentSet.parameters.splitIntoSentences as boolean,
+      combineSentencesIntoChunks: documentSet.parameters.combineSentencesIntoChunks as boolean,
+      sploderMaxSize: 100,
+      vectorStoreType: documentSet.parameters.vectorStoreType as 'simple' | 'weaviate',
+      projectName: documentSet.name,
+      storagePath: this.storagePath,
+      chunkSize: 1024, // not actually used, we just re-use a config object that has this option
+      chunkOverlap: 20, // not actually used, we just re-use a config object that has this option
+    });
+    const document = await docStore.getNode(documentNodeId);
+    if (!document) {
+      throw new Error('Document not found');
+    }
+    return document;
+  }
+
+
+  async getSettings() {
+    return this.manager.getSettings();
+  }
+  async setSettings(settings: Settings) {
+    return this.manager.setSettings(settings);
+  } 
+
+  async getMaskedSettings() {
+    const settings = await this.manager.getSettings();
+    return {
+      openAIKey: maskKey(settings.openAIKey),
+      oLlamaBaseURL: settings.oLlamaBaseURL,
+      azureOpenAIKey: maskKey(settings.azureOpenAIKey),
+      azureOpenAIEndpoint: settings.azureOpenAIEndpoint,
+      azureOpenAIApiVersion: settings.azureOpenAIApiVersion,
+      mistralApiKey: maskKey(settings.mistralApiKey),
+      geminiApiKey: maskKey(settings.geminiApiKey)
+    };
+  }
+  async setMaskedSettings(newSettings: Settings) { 
+    const oldSettings = await this.manager.getSettings();
+    const settings = {
+      ...newSettings,
+      openAIKey: newSettings.openAIKey == maskKey(oldSettings.openAIKey) ? oldSettings.openAIKey : newSettings.openAIKey,
+      azureOpenAIKey: newSettings.azureOpenAIKey == maskKey(oldSettings.azureOpenAIKey) ? oldSettings.azureOpenAIKey : newSettings.azureOpenAIKey,
+      mistralApiKey: newSettings.mistralApiKey == maskKey(oldSettings.mistralApiKey) ? oldSettings.mistralApiKey : newSettings.mistralApiKey,
+      geminiApiKey: newSettings.geminiApiKey == maskKey(oldSettings.geminiApiKey) ? oldSettings.geminiApiKey : newSettings.geminiApiKey
+    };
+    return this.manager.setSettings(settings);
+  }
+
+}

package/src/MetadataManager.d.ts

@@ -0,0 +1,27 @@
+import type { DocumentSetMetadata, Settings } from './types';
+export declare class MetadataManager {
+    private sqliteDb;
+    constructor(storagePath: string);
+    private initializeDatabase;
+    addDocumentSet(metadata: Omit<DocumentSetMetadata, 'documentSetId'>): Promise<number>;
+    getDocumentSet(documentSetId: number): Promise<DocumentSetMetadata | null>;
+    getDocumentSets(page?: number, pageSize?: number): Promise<{
+        documents: DocumentSetMetadata[];
+        total: number;
+    }>;
+    updateDocumentCount(documentSetId: number, count: number): Promise<void>;
+    deleteDocumentSet(documentSetId: number): Promise<void>;
+    getSettings(): Promise<{
+        openAIKey: null;
+        oLlamaBaseURL: null;
+        azureOpenAIKey: null;
+        azureOpenAIEndpoint: null;
+        azureOpenAIApiVersion: string;
+        mistralApiKey: null;
+        geminiApiKey: null;
+    } & Settings>;
+    setSettings(settings: Settings): Promise<Settings & {
+        success: boolean;
+    }>;
+    close(): void;
+}

package/src/MetadataManager.ts

@@ -0,0 +1,145 @@
+import type { DocumentSetMetadata, Settings } from './types/index.js';
+
+export abstract class MetadataManager {
+  protected queries = {
+    /* 
+    Note: RETURNING on non-select/non-create statements is important for compatibility between SQLite and PostgreSQL.
+    (Without it, better-sqlite would demand to use run() instead of all() or get(), which would break the abstraction.)
+    */
+    createDocumentSetsTable: `
+      CREATE TABLE IF NOT EXISTS document_sets (
+        set_id SERIAL PRIMARY KEY,
+        name TEXT NOT NULL UNIQUE,
+        upload_date TIMESTAMP NOT NULL,
+        parameters TEXT NOT NULL,
+        total_documents INTEGER NOT NULL DEFAULT 0
+      );
+    `,
+    createSettingsTable: `
+      CREATE TABLE IF NOT EXISTS meaningfully_settings (
+        settings_id SERIAL PRIMARY KEY,
+        settings TEXT NOT NULL
+      );
+    `,
+    insertDocumentSet: `
+      INSERT INTO document_sets (name, upload_date, parameters, total_documents)
+      VALUES ($1, $2, $3, $4) RETURNING set_id
+    `,
+    selectDocumentSet: `
+      SELECT * FROM document_sets WHERE set_id = $1
+    `,
+    selectDocumentSets: `
+      SELECT * FROM document_sets ORDER BY upload_date DESC LIMIT $1 OFFSET $2
+    `,
+    countDocumentSets: `
+      SELECT COUNT(*) as count FROM document_sets
+    `,
+    updateDocumentCount: `
+      UPDATE document_sets SET total_documents = total_documents + $1 WHERE set_id = $2 RETURNING *
+    `,
+    deleteDocumentSet: `
+      DELETE FROM document_sets WHERE set_id = $1 RETURNING *
+    `,
+    selectSettings: `
+      SELECT * FROM meaningfully_settings WHERE settings_id = 1
+    `,
+    upsertSettings: `
+      INSERT INTO meaningfully_settings (settings_id, settings)
+      VALUES (1, $1)
+      ON CONFLICT (settings_id) DO UPDATE SET settings = $2
+      RETURNING *
+    `     
+    // the two arguments $1 and $2 are identical, but, to work around a cross-compatibility bug in SQLite versus Postgresql,
+    // where PG can accept the same argument twice (specified as $1 in two places), but SQLITE cannot (it just has ? placeholders)
+    // they are specified separately.
+  };
+
+  protected abstract runQuery<T>(query: string, params?: any[]): Promise<T[]>;
+  protected abstract runQuerySingle<T>(query: string, params?: any[]): Promise<T | null>;
+  protected abstract initializeDatabase(): Promise<void>;
+  protected abstract close(): void;
+
+  async addDocumentSet(metadata: Omit<DocumentSetMetadata, 'documentSetId'>): Promise<number> {
+    const result = await this.runQuerySingle<{ set_id: number }>(this.queries.insertDocumentSet, [
+      metadata.name,
+      metadata.uploadDate.toISOString(),
+      JSON.stringify(metadata.parameters),
+      metadata.totalDocuments
+    ]);
+    return result?.set_id || 0;
+  }
+
+  async getDocumentSet(documentSetId: number): Promise<DocumentSetMetadata | null> {
+    const row = await this.runQuerySingle<{
+      set_id: number;
+      name: string;
+      upload_date: string;
+      parameters: string;
+      total_documents: number;
+    }>(this.queries.selectDocumentSet, [documentSetId]);
+
+    if (!row) return null;
+
+    return {
+      documentSetId: row.set_id,
+      name: row.name,
+      uploadDate: new Date(row.upload_date),
+      parameters: JSON.parse(row.parameters),
+      totalDocuments: row.total_documents
+    };
+  }
+
+  async getDocumentSets(page: number = 1, pageSize: number = 10): Promise<{ documents: DocumentSetMetadata[]; total: number }> {
+    const offset = (page - 1) * pageSize;
+    const totalCountRow = await this.runQuerySingle<{ count: number }>(this.queries.countDocumentSets);
+    const totalCount = totalCountRow?.count || 0;
+
+    const rows = await this.runQuery<{
+      set_id: number;
+      name: string;
+      upload_date: string;
+      parameters: string;
+      total_documents: number;
+    }>(this.queries.selectDocumentSets, [pageSize, offset]);
+
+    const documents = rows.map((row) => ({
+      documentSetId: row.set_id,
+      name: row.name,
+      uploadDate: new Date(row.upload_date),
+      parameters: JSON.parse(row.parameters),
+      totalDocuments: row.total_documents
+    }));
+
+    return { documents, total: totalCount };
+  }
+
+  async updateDocumentCount(documentSetId: number, count: number): Promise<void> {
+    await this.runQuery(this.queries.updateDocumentCount, [count, documentSetId]);
+  }
+
+  async deleteDocumentSet(documentSetId: number): Promise<void> {
+    await this.runQuery(this.queries.deleteDocumentSet, [documentSetId]);
+  }
+
+  async getSettings(): Promise<Settings> {
+    const DEFAULT_SETTINGS: Settings = {
+      openAIKey: null,
+      oLlamaBaseURL: null,
+      azureOpenAIKey: null,
+      azureOpenAIEndpoint: null,
+      azureOpenAIApiVersion: "2024-02-01",
+      mistralApiKey: null,
+      geminiApiKey: null,
+    };
+
+    const row = await this.runQuerySingle<{ settings: string }>(this.queries.selectSettings);
+    return row ? { ...DEFAULT_SETTINGS, ...JSON.parse(row.settings) } : DEFAULT_SETTINGS;
+  }
+
+  async setSettings(settings: Settings): Promise<{ success: boolean }> {
+    // the JSON.stringify(settings) is repeated to work around a cross-compatibility bug in SQLite versus Postgresql
+    // where PG can accept the same argument twice (specified as $1 in two places), but SQLITE cannot (it just has ? placeholders)
+    await this.runQuery(this.queries.upsertSettings, [JSON.stringify(settings), JSON.stringify(settings)]);
+    return { success: true };
+  }
+}

package/src/api/embedding.d.ts

@@ -0,0 +1,6 @@
+import type { EmbeddingConfig, EmbeddingResult, SearchResult, PreviewResult, Settings, MetadataFilter, Clients } from "../types";
+export declare function createEmbeddings(csvPath: string, textColumnName: string, config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<EmbeddingResult>;
+export declare function previewResults(csvPath: string, textColumnName: string, config: EmbeddingConfig): Promise<PreviewResult>;
+export declare function getDocStore(config: EmbeddingConfig): Promise<import("llamaindex").BaseDocumentStore>;
+export declare function getIndex(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<import("llamaindex").VectorStoreIndex>;
+export declare function search(index: any, query: string, numResults?: number, filters?: MetadataFilter[]): Promise<SearchResult[]>;