@meaningfully/core 0.1.0

package/src/api/embedding.ts

@@ -0,0 +1,122 @@
+import { transformDocumentsToNodes, estimateCost, searchDocuments, getExistingVectorStoreIndex, persistNodes, persistDocuments, getExistingDocStore } from "../services/embeddings.js";
+import type { EmbeddingConfig, EmbeddingResult, SearchResult, PreviewResult, Settings, MetadataFilter, Clients } from "../types/index.js";
+import { loadDocumentsFromCsv } from "../services/csvLoader.js";
+import { MetadataMode } from "llamaindex";
+import { ProgressManager } from "../services/progressManager.js";
+
+export async function createEmbeddings(
+  csvPath: string,
+  textColumnName: string,
+  config: EmbeddingConfig,
+  settings: Settings,
+  clients: Clients
+): Promise<EmbeddingResult> {
+  try {
+    console.time("createEmbeddings Run Time");
+    const operationId = `embed-${Date.now()}`;
+    const progressManager = ProgressManager.getInstance();
+    progressManager.startOperation(operationId, 100);
+
+    const documents = await loadDocumentsFromCsv(csvPath, textColumnName);
+    if (documents.length === 0) {
+      progressManager.clearOperation(operationId);
+      console.timeEnd("createEmbeddings Run Time");
+      return {
+        success: false,
+        error: "That CSV does not appear to contain any documents. Please check the file and try again.",
+      };
+    }
+    
+    progressManager.updateProgress(operationId, 5);
+    
+    const nodes = await transformDocumentsToNodes(documents, config);
+        
+    const [index] = await Promise.all([
+      persistNodes(nodes, config, settings, clients, (progress, total) => {
+        const percentage = Math.floor((progress / total) * 90) + 5; // Map to 5-95% of total progress
+        progressManager.updateProgress(operationId, percentage);
+      }),
+      persistDocuments(documents, config, settings, clients)
+    ]);
+    
+    progressManager.completeOperation(operationId);
+    console.timeEnd("createEmbeddings Run Time");
+    return {
+      success: true,
+      index,
+    };
+  } catch (error) {
+    return {
+      success: false,
+      error: error instanceof Error ? error.message : "Unknown error occurred",
+    };
+  }
+}
+
+// TODO: rename this to be parallel to createEmbeddings
+export async function previewResults(
+  csvPath: string,
+  textColumnName: string,
+  config: EmbeddingConfig
+): Promise<PreviewResult> {
+  try {
+    const documents = await loadDocumentsFromCsv(csvPath, textColumnName);
+    if (documents.length === 0) {
+      return {
+        success: false,
+        error: "That CSV does not appear to contain any documents. Please check the file and try again.",
+      };
+    }
+    // Take 10 rows from the middle of the dataset for preview
+    // we take a consistent 10 so that the results of the preview are consistent (i.e. with a larger chunk size, you have fewer, longer results, but more shorter ones if you adjust it)
+    // and we take from the middle because the initial rows may be idiosyncratic.
+    const previewDocumentsSubset = documents.slice(
+      Math.floor(documents.length / 2),
+      Math.floor(documents.length / 2) + 10
+    );
+
+    const previewNodes = await transformDocumentsToNodes(documents, config);
+    const previewSubsetNodes = await transformDocumentsToNodes(previewDocumentsSubset, config);
+    const { estimatedPrice, tokenCount, pricePer1M } = estimateCost(previewNodes, config.modelName);
+
+    return {
+      success: true,
+      nodes: previewSubsetNodes.map((node: any) => ({
+        text: node.text,
+        metadata: node.metadata
+      })),
+      estimatedPrice,
+      tokenCount,
+      pricePer1M
+    };
+  } catch (error) {
+    return {
+      success: false,
+      error: error instanceof Error ? error.message : "Unknown error occurred"
+    };
+  }
+} 
+
+export async function getDocStore(config: EmbeddingConfig) {
+  return await getExistingDocStore(config);
+}
+
+export async function getIndex(config: EmbeddingConfig, settings: Settings, clients: Clients) {
+  return await getExistingVectorStoreIndex(config, settings, clients);
+}
+
+export async function search(
+  index: any,
+  query: string,
+  numResults: number = 10,
+  filters?: MetadataFilter[]
+): Promise<SearchResult[]> {
+  const results = await searchDocuments(index, query, numResults, filters);
+  return results.map((result: any) => ({
+    text: result.node.getContent(MetadataMode.NONE),
+    score: result.score ?? 0,
+    metadata: result.node.metadata,
+    //  @ts-ignore
+    sourceNodeId: result.node.relationships?.SOURCE?.nodeId
+  }));
+}

package/src/index.ts

@@ -0,0 +1,5 @@
+export * from './Meaningfully.js';
+export * from './MetadataManager.js';
+export * from './services/progressManager.js';
+export * from './types/index.js';
+// src/index.ts

package/src/services/batchingWeaviateVectorStore.d.ts

@@ -0,0 +1,5 @@
+import { WeaviateVectorStore } from '@llamaindex/weaviate';
+import { BaseNode } from 'llamaindex';
+export declare class BatchingWeaviateVectorStore extends WeaviateVectorStore {
+    add(nodes: BaseNode[]): Promise<string[]>;
+}

package/src/services/batchingWeaviateVectorStore.ts

@@ -0,0 +1,23 @@
+import { WeaviateVectorStore } from '@llamaindex/weaviate';
+import { BaseNode } from 'llamaindex';
+
+/*
+
+Patched version of WeaviateVectorStore to handle large batches by splitting into smaller chunks.
+
+When I loaded a large-ish (5.4MB) spreadsheet, I got a Weaviate error about trying to load too much data at once.
+
+*/
+
+export class BatchingWeaviateVectorStore extends WeaviateVectorStore {
+  async add(nodes: BaseNode[]): Promise<string[]> {
+    const batchSize = 100; // Define the batch size
+    const results: string[] = []; // Collect results from each batch
+    for (let i = 0; i < nodes.length; i += batchSize) {
+      const batch = nodes.slice(i, i + batchSize);
+      const batchResults = await super.add(batch); // Call the parent class's add method for each batch
+      results.push(...batchResults); // Aggregate results
+    }
+    return results; // Return aggregated results
+  }
+}

package/src/services/csvLoader.d.ts

@@ -0,0 +1,2 @@
+import { Document } from "llamaindex";
+export declare function loadDocumentsFromCsv(filePath: string, textColumnName: string): Promise<Document[]>;

package/src/services/csvLoader.ts

@@ -0,0 +1,24 @@
+import { Document } from "llamaindex";
+import { readFileSync } from "fs";
+import Papa from "papaparse";
+
+export async function loadDocumentsFromCsv(
+  filePath: string,
+  textColumnName: string
+): Promise<Document[]> {
+  const fileContent = readFileSync(filePath, "utf-8");
+  const { data: records } = Papa.parse(fileContent, {
+    header: true,
+    skipEmptyLines: true,
+  });
+
+  return records.map((record: any) => {
+    const { [textColumnName]: text, ...metadata } = record;
+    return new Document({
+      text,
+      metadata: Object.fromEntries(
+        Object.entries(metadata).map(([k, v]) => [k, v ?? ""])
+      ),
+    });
+  });
+}

package/src/services/embeddings.d.ts

@@ -0,0 +1,21 @@
+import { Document, VectorStoreIndex, TextNode, StorageContext } from "llamaindex";
+import { OllamaEmbedding } from '@llamaindex/ollama';
+import { MistralAIEmbedding } from '@llamaindex/mistral';
+import { GeminiEmbedding } from '@llamaindex/google';
+import { MockEmbedding } from "./mockEmbedding";
+import { EmbeddingConfig, Settings, MetadataFilter, Clients } from "../types";
+import { OpenAIEmbedding } from "@llamaindex/openai";
+import { ProgressVectorStoreIndex } from "./progressVectorStoreIndex";
+export declare function estimateCost(nodes: TextNode[], modelName: string): {
+    estimatedPrice: number;
+    tokenCount: number;
+    pricePer1M: number;
+};
+export declare function getExistingVectorStoreIndex(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<VectorStoreIndex>;
+export declare function getExistingDocStore(config: EmbeddingConfig): Promise<import("llamaindex").BaseDocumentStore>;
+export declare function transformDocumentsToNodes(documents: Document[], config: EmbeddingConfig): Promise<TextNode<import("llamaindex").Metadata>[]>;
+export declare function getEmbedModel(config: EmbeddingConfig, settings: Settings): MockEmbedding | OpenAIEmbedding | OllamaEmbedding | MistralAIEmbedding | GeminiEmbedding;
+export declare function getStorageContext(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<StorageContext>;
+export declare function persistDocuments(documents: Document[], config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<void>;
+export declare function persistNodes(nodes: TextNode[], config: EmbeddingConfig, settings: Settings, clients: Clients, progressCallback?: (progress: number, total: number) => void): Promise<ProgressVectorStoreIndex>;
+export declare function searchDocuments(index: VectorStoreIndex, query: string, numResults?: number, filters?: MetadataFilter[]): Promise<import("llamaindex").NodeWithScore<import("llamaindex").Metadata>[]>;

package/src/services/embeddings.ts

@@ -0,0 +1,374 @@
+import { 
+  Document, 
+  VectorStoreIndex, 
+  // OpenAIEmbedding,
+  IngestionPipeline,
+  TransformComponent,
+  TextNode,
+  ModalityType,
+  type MetadataFilters,
+  storageContextFromDefaults,
+  SimpleVectorStore,
+  type StorageContext,
+  Settings as LlamaindexSettings,
+  SimpleDocumentStore
+} from "llamaindex";
+import { OllamaEmbedding} from '@llamaindex/ollama'
+import { MistralAIEmbedding, MistralAIEmbeddingModelType } from '@llamaindex/mistral'
+import { GeminiEmbedding } from '@llamaindex/google'
+import { PGVectorStore } from '@llamaindex/postgres';
+import { AzureOpenAIEmbedding } from "@llamaindex/azure";
+import { Sploder } from "./sploder.js";
+import { CustomSentenceSplitter } from "./sentenceSplitter.js";
+import { MockEmbedding } from "./mockEmbedding.js";
+import { encodingForModel, type TiktokenModel } from "js-tiktoken";
+import { join } from "path";
+import type { EmbeddingConfig, Settings, MetadataFilter, Clients  } from "../types/index.js";
+import { sanitizeProjectName, capitalizeFirstLetter } from "../utils.js";
+import * as fs from 'fs';
+import { OpenAIEmbedding } from "@llamaindex/openai";
+import { BatchingWeaviateVectorStore } from "./batchingWeaviateVectorStore.js";
+import { ProgressVectorStoreIndex } from "./progressVectorStoreIndex.js";
+
+// unused, but probalby eventually will be used.
+// to be used by postgres store, which it' slooking increasingly like I have to enable again
+const MODEL_DIMENSIONS: Record<string, number> = {
+  "text-embedding-3-small": 1536,
+  "text-embedding-3-large": 3072,
+  "mxbai-embed-large": 1024,
+  "mistral-embed": 1024,
+  "gemini-embedding-001": 768, // Gemini embedding model
+};
+
+const PRICE_PER_1M: Record<string, number> = {
+  "text-embedding-3-small": 0.02,
+  "text-embedding-3-large": 0.13,
+  "mistral-embed": 0.1, 
+  "mxbai-embed-large": 0, // local model, free
+  "nomic-embed-text": 0, // local model, free
+  "gemini-embedding-001": 0.0, // Gemini embedding is currently free (unless you're on the paid tier, in which case it is $0.15/million tokens)
+};
+
+
+/* all transformations except the embedding step (which is handled by VectorStoreIndex.init) */
+function getBaseTransformations(config: EmbeddingConfig){
+  const transformations: TransformComponent[] = [
+    new CustomSentenceSplitter({ chunkSize: config.chunkSize, chunkOverlap: config.chunkOverlap }),
+  ];
+
+  if (config.combineSentencesIntoChunks) {
+    transformations.push(
+      new Sploder({
+        maxStringTokenCount: config.sploderMaxSize
+      })
+    );
+  }
+
+  return transformations;
+}
+
+export function estimateCost(nodes: TextNode[], modelName: string): {
+  estimatedPrice: number;
+  tokenCount: number;
+  pricePer1M: number;
+} {
+  const pricePer1M = PRICE_PER_1M[modelName] || 0; // default to 0 if model not found or free
+
+  let tokenizer; 
+  try{
+    tokenizer = encodingForModel(modelName as TiktokenModel); // This doesn't work for ollama
+  } catch (error) {
+    // If the tokenizer is not found, it means the model is likely not supported by tiktoken
+    // or is a local model (like Ollama). In this case, we can't estimate the cost.
+    tokenizer = encodingForModel("text-embedding-3-small"); // fallback to a known tokenizer
+    console.warn(`Tokenizer for model ${modelName} not found. Using fallback tokenizer.`);
+  }
+  const tokenCount = nodes.reduce((sum, node) => {
+    return sum + tokenizer.encode(node.text).length;
+  }, 0);
+
+  const estimatedPrice = tokenCount * (pricePer1M / 1_000_000);
+
+  return {
+    estimatedPrice,
+    tokenCount,
+    pricePer1M
+  };
+}
+
+export async function getExistingVectorStoreIndex(config: EmbeddingConfig, settings: Settings, clients: Clients) {
+  const embedModel = getEmbedModel(config, settings);
+  switch (config.vectorStoreType) {
+    case "simple":
+      const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName));
+      const storageContext = await storageContextFromDefaults({
+        persistDir: persistDir,
+      });
+      let vsi = await VectorStoreIndex.init({
+        storageContext: storageContext,
+      });
+      vsi.embedModel = embedModel;
+      return vsi;
+
+    case "postgres":
+      if (!clients.postgresClient) {
+        throw new Error("Postgres client required but not provided");
+      }
+      const pgStore = new PGVectorStore({
+        clientConfig: { connectionString: process.env.POSTGRES_CONNECTION_STRING }, 
+        tableName: sanitizeProjectName(config.projectName),
+        dimensions: MODEL_DIMENSIONS[config.modelName] || 1536, // default to 1536 if model not found
+        embeddingModel: embedModel
+      });
+      const pgStorageContext = await storageContextFromDefaults({
+        vectorStores: { [ModalityType.TEXT]: pgStore },
+      });
+      return await VectorStoreIndex.init({
+        storageContext: pgStorageContext,
+      });
+    case "weaviate":
+      if (!clients.weaviateClient) {
+        throw new Error("Weaviate client required but not provided");
+      }
+      const weaviateStore = new BatchingWeaviateVectorStore({
+        indexName: capitalizeFirstLetter(sanitizeProjectName(config.projectName)),
+        weaviateClient: clients.weaviateClient,
+        embeddingModel: embedModel
+      });
+
+      // WeaviateVectorStore's getNodeSimilarity method looks for distance, but current weaviate provides score
+      // (WeaviateVectorStore would get `score` if we were doing hybrid search)
+      // Overwrite the private getNodeSimilarity method to use 'score' from metadata
+      // @ts-ignore
+      weaviateStore.getNodeSimilarity = (entry, _similarityKey = "score") => {
+        return  entry.metadata.score;
+      }
+
+      return await VectorStoreIndex.fromVectorStore(weaviateStore)
+
+    default:
+      throw new Error(`Unsupported vector store type: ${config.vectorStoreType}`);
+  }
+}
+
+export async function getExistingDocStore(config: EmbeddingConfig) {
+  // switch (config.vectorStoreType) {
+  //   case "simple":
+      const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName) );
+      const storageContext = await storageContextFromDefaults({
+        persistDir: persistDir,
+      });
+      return storageContext.docStore;
+
+  //   case "postgres":
+  //     throw new Error(`Not yet implemented vector store type: ${config.vectorStoreType}`);
+  //     // return await createVectorStore(config);
+  //   default:
+  //     throw new Error(`Unsupported vector store type: ${config.vectorStoreType}`);
+  // }
+}
+
+
+
+export async function transformDocumentsToNodes(
+  documents: Document[],
+  config: EmbeddingConfig,
+) {
+  console.time("transformDocumentsToNodes Run Time");
+
+  const transformations = getBaseTransformations(config);
+
+  // llama-index stupidly includes all the metadata in the embedding, which is a waste of tokens
+  // so we exclude everything except the text column from the embedding
+  for (const document of documents) {
+    document.excludedEmbedMetadataKeys = Object.keys(document.metadata);
+  }
+  console.time("transformDocumentsToNodes transformDocuments Run Time");
+  // remove empty documents. we can't meaningfully embed these, so we're just gonna ignore 'em.
+  // that might not ultimately be the right solution. 
+  documents = documents.filter((document_) => document_.text && document_.text.length > 0);
+
+  // Create nodes with sentence splitting and optional sploder
+  const pipeline = new IngestionPipeline({
+    transformations
+  });
+
+  const nodes = (await pipeline.run({documents: documents})) as TextNode[];
+
+  console.timeEnd("transformDocumentsToNodes transformDocuments Run Time");
+  console.timeEnd("transformDocumentsToNodes Run Time");  
+  return nodes;
+}
+
+export function getEmbedModel(
+  config: EmbeddingConfig, 
+  settings: Settings,
+) {
+  let embedModel; 
+  if (config.modelProvider === "openai" ){
+    embedModel = new OpenAIEmbedding({ model: config.modelName, apiKey: settings.openAIKey ? settings.openAIKey : undefined} );
+    embedModel.embedBatchSize = 50; // all embedding models enforce a maximum of 300,000 tokens summed across all inputs in a single request
+  } else if (config.modelProvider === "ollama") {
+    embedModel = new OllamaEmbedding({ model: config.modelName, config: {
+      host: settings.oLlamaBaseURL ? settings.oLlamaBaseURL : undefined
+    }, }); 
+  } else if (config.modelProvider === "azure") {
+    if (!settings.azureOpenAIKey || !settings.azureOpenAIEndpoint) {
+      throw new Error("Azure OpenAI API key and endpoint are required for Azure embedding models");
+    }
+    embedModel = new AzureOpenAIEmbedding({ 
+      model: config.modelName, 
+      apiKey: settings.azureOpenAIKey,
+      endpoint: settings.azureOpenAIEndpoint,
+      apiVersion: settings.azureOpenAIApiVersion ?? undefined
+    });
+  } else if (config.modelProvider === "mistral") {
+    if (!settings.mistralApiKey) {
+      throw new Error("Mistral API key is required for Mistral embedding models");
+    }
+    embedModel = new MistralAIEmbedding({ 
+      model: MistralAIEmbeddingModelType.MISTRAL_EMBED, // only one choice!
+      apiKey: settings.mistralApiKey
+    });
+  } else if (config.modelProvider === "gemini") {
+    if (!settings.geminiApiKey) {
+      throw new Error("Gemini API key is required for Gemini embedding models");
+    }
+    embedModel = new GeminiEmbedding({ 
+      apiKey: settings.geminiApiKey,
+    });
+    embedModel.embedBatchSize = 50;
+  } else if (config.modelProvider === "mock") {
+    embedModel = new MockEmbedding();
+  } else {
+    throw new Error(`Unsupported embedding model provider: ${config.modelProvider}`);
+  }
+  LlamaindexSettings.embedModel = embedModel;
+  return embedModel;
+}
+
+export async function getStorageContext(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<StorageContext> {
+  const vectorStore = await createVectorStore(config, settings, clients);
+  fs.mkdirSync(config.storagePath, { recursive: true }); 
+  const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName) );
+  return await storageContextFromDefaults({
+    persistDir: persistDir,
+    vectorStores: {[ModalityType.TEXT]: vectorStore},
+    docStore: new SimpleDocumentStore()
+      /*
+        if docStore is created with a persist path (as it is by default in storageContextFromDefaults)
+        then it will write to disk after every put(), which happens 2+ times per document.
+
+        so we create it without a persist path, and then explicitly persist it when we're done adding documents.
+
+        see https://github.com/jeremybmerrill/meaningfully/issues/52
+      */
+  });
+}
+
+export async function persistDocuments(documents: Document[], config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<void> {
+  console.time("persistDocuments Run Time");
+  const storageContext = await getStorageContext(config, settings, clients);
+  await storageContext.docStore.addDocuments(documents, true);
+
+  // see comments in getStorageContext
+  const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName) );
+  // @ts-ignore
+  await (storageContext.docStore as SimpleDocumentStore).kvStore.persist(join(persistDir, "doc_store.json"));
+
+  console.timeEnd("persistDocuments Run Time");
+}
+
+export async function persistNodes(nodes: TextNode[], config: EmbeddingConfig, settings: Settings, clients: Clients, progressCallback?: (progress: number, total: number) => void): Promise<ProgressVectorStoreIndex> {
+  // Create and configure vector store based on type
+  console.time("persistNodes Run Time");
+
+  const storageContext = await getStorageContext(config, settings, clients);
+  const vectorStore = storageContext.vectorStores[ModalityType.TEXT];
+  if (!vectorStore) {
+    throw new Error("Vector store is undefined");
+  }
+  // Create index and embed documents
+  // this is what actaully embeds the nodes
+  // (even if they already have embeddings, stupidly)
+  const index = await ProgressVectorStoreIndex.init({
+    nodes, 
+    storageContext, 
+    logProgress: true,
+    progressCallback,
+  });
+
+  // I'm not sure why this explicit call to persist is necessary. 
+  // storageContext should handle this, but it doesn't.
+  // all the if statements are just type-checking boilerplate.
+  // N.B. WeaviateVectorStore does not need to be explicitly persisted, so we don't include it in the OR conditional here..
+  if (vectorStore) {
+    if (vectorStore instanceof PGVectorStore || vectorStore instanceof SimpleVectorStore) {
+      await vectorStore.persist(join(config.storagePath, sanitizeProjectName(config.projectName), "vector_store.json"));
+    } else if (vectorStore instanceof BatchingWeaviateVectorStore) {
+      // WeaviateVectorStore does not have a persist method, it persists automatically
+      console.log("Pretending to persist Weaviate vector store, but it actually persists automatically.");
+    } else {
+      throw new Error("Vector store does not support persist method");
+    }
+  } else {
+    throw new Error("Vector store is undefined");
+  }
+  console.timeEnd("persistNodes Run Time");
+  return index;
+}
+
+async function createVectorStore(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<PGVectorStore | SimpleVectorStore | BatchingWeaviateVectorStore> {
+  const embeddingModel = getEmbedModel(config, settings);
+  switch (config.vectorStoreType) {
+
+    // for some reason the embedding model has to be specified here TOO
+    // otherwise it defaults to Ada.
+    case "postgres":
+      return new PGVectorStore({
+        clientConfig: {connectionString: process.env.POSTGRES_CONNECTION_STRING},
+        tableName: sanitizeProjectName(config.projectName),
+        dimensions: MODEL_DIMENSIONS[config.modelName] || 1536, // default to 1536 if model not found
+        embeddingModel: embeddingModel
+      });
+
+    case "simple":
+      const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName));
+      return SimpleVectorStore.fromPersistDir(persistDir, embeddingModel);
+
+    case "weaviate": 
+      const vectorStore = new BatchingWeaviateVectorStore({
+        indexName: capitalizeFirstLetter(sanitizeProjectName(config.projectName)), 
+        weaviateClient: clients.weaviateClient, 
+        embeddingModel: embeddingModel 
+      });
+
+      // WeaviateVectorStore's getNodeSimilarity method looks for distance, but current weaviate provides score
+      // (WeaviateVectorStore would get `score` if we were doing hybrid search)
+      // Overwrite the private getNodeSimilarity method to use 'score' from metadata
+      // @ts-ignore
+      vectorStore.getNodeSimilarity = (entry, _similarityKey = "score") => {
+        return  entry.metadata.score;
+      }
+
+      return vectorStore;
+    default:
+      throw new Error(`Unsupported vector store type: ${config.vectorStoreType}`);
+  }
+}
+
+export async function searchDocuments(
+  index: VectorStoreIndex,
+  query: string,
+  numResults: number = 10,
+  filters?: MetadataFilter[]
+) {
+  // const metadataFilters: MetadataFilters | undefined = filters ? {filters: filters} : undefined;
+  const metadataFilters: MetadataFilters = {
+    filters: filters ? filters : [],
+  };
+  const retriever = index.asRetriever({ similarityTopK: numResults, filters: metadataFilters });
+
+  const results = await retriever.retrieve(query );
+  return results;
+}

package/src/services/loggingOpenAIEmbedding.ts

@@ -0,0 +1,46 @@
+
+// // temporary
+// // this is a wrapper around OpenAIEmbedding that logs the input of the embedding
+// // it's used to debug the embedding process (to make sure random metadata isn't wrongfully included)
+// // it's not used in the production code
+
+// import { OpenAIEmbedding } from "@llamaindex/openai";
+// import type {
+//   OpenAI as OpenAILLM,
+// } from "openai";
+// type LLMInstance = Pick<OpenAILLM, "embeddings" | "apiKey" | "baseURL">;
+
+
+// export class LoggingOpenAIEmbedding extends OpenAIEmbedding {
+//   constructor(
+//     init?: Omit<Partial<OpenAIEmbedding>, "session"> & {
+//       session?: LLMInstance;
+//     },
+//   ) {
+//     super(init);
+//     // overwrite private member "getMessage" 🙀
+//     (this as any).getOpenAIEmbedding = async function(input: string[]): Promise<number[][]> {
+//       // TODO: ensure this for every sub class by calling it in the base class
+//       input = this.truncateMaxTokens(input);
+
+//       console.log("LoggingOpenAIEmbedding input", input);
+    
+//       const { data } = await (
+//         await this.session
+//       ).embeddings.create(
+//         this.dimensions
+//           ? {
+//               model: this.model,
+//               dimensions: this.dimensions, // only sent to OpenAI if set by user
+//               input,
+//             }
+//           : {
+//               model: this.model,
+//               input,
+//             },
+//       );
+    
+//       return data.map((d) => d.embedding);
+//     }
+//   }
+// }

package/src/services/mockEmbedding.d.ts

@@ -0,0 +1,5 @@
+import { BaseEmbedding } from "llamaindex";
+export declare class MockEmbedding extends BaseEmbedding {
+    constructor();
+    getTextEmbedding(text: string): Promise<number[]>;
+}

package/src/services/mockEmbedding.ts

@@ -0,0 +1,13 @@
+//@ts-nocheck
+import { BaseEmbedding } from "llamaindex";
+
+export class MockEmbedding extends BaseEmbedding {
+    constructor() {
+        super();
+    }   
+    async getTextEmbedding(text: string): Promise<number[]> {
+        return new Promise((resolve) => {
+            resolve([1, 0, 0, 0, 0, 0]);
+        });
+    }
+};

package/src/services/progressManager.d.ts

@@ -0,0 +1,20 @@
+/**
+ * A simple manager to track progress of various operations
+ */
+export declare class ProgressManager {
+    private static instance;
+    private progressMap;
+    private currentOperation;
+    private constructor();
+    static getInstance(): ProgressManager;
+    startOperation(operationId: string, total?: number): void;
+    updateProgress(operationId: string, progress: number): void;
+    completeOperation(operationId: string): void;
+    getCurrentProgress(): {
+        progress: number;
+        total: number;
+        elapsedTimeMs: number;
+        estimatedTimeRemainingMs: number | null;
+    };
+    clearOperation(operationId: string): void;
+}