@meaningfully/core 0.1.0

package/src/services/progressManager.ts

@@ -0,0 +1,88 @@
+/**
+ * A simple manager to track progress of various operations
+ */
+
+const FUDGE_FACTOR = 1.2; // seat of my pants guess
+
+export class ProgressManager {
+  private static instance: ProgressManager;
+  private progressMap: Map<string, { progress: number; total: number; startTime: number; lastUpdateTime: number }> = new Map();
+  private currentOperation: string | null = null;
+
+
+  private constructor() {}
+
+  public static getInstance(): ProgressManager {
+    if (!ProgressManager.instance) {
+      ProgressManager.instance = new ProgressManager();
+    }
+    return ProgressManager.instance;
+  }
+
+  public startOperation(operationId: string, total: number = 100): void {
+    const now = Date.now();
+    this.progressMap.set(operationId, { progress: 0, total, startTime: now, lastUpdateTime: now });
+    this.currentOperation = operationId;
+  }
+
+  public updateProgress(operationId: string, progress: number): void {
+    const currentProgress = this.progressMap.get(operationId);
+    if (currentProgress) {
+      this.progressMap.set(operationId, { 
+        progress, 
+        total: currentProgress.total,
+        startTime: currentProgress.startTime,
+        lastUpdateTime: Date.now()
+      });
+    }
+  }
+
+  public completeOperation(operationId: string): void {
+    const currentProgress = this.progressMap.get(operationId);
+    if (currentProgress) {
+      this.progressMap.set(operationId, { 
+        progress: currentProgress.total, 
+        total: currentProgress.total,
+        startTime: currentProgress.startTime,
+        lastUpdateTime: Date.now()
+      });
+    }
+    
+    if (this.currentOperation === operationId) {
+      this.currentOperation = null;
+    }
+  }
+
+  public getCurrentProgress(): { progress: number; total: number; elapsedTimeMs: number; estimatedTimeRemainingMs: number | null } {
+    if (this.currentOperation) {
+      const operationData = this.progressMap.get(this.currentOperation);
+      if (operationData) {
+        const now = Date.now();
+        const elapsedTimeMs = now - operationData.startTime;
+        let estimatedTimeRemainingMs: number | null = null;
+        
+        // Only estimate if we have meaningful progress (at least 5% complete and some time elapsed)
+        if (operationData.progress > 0.05 * operationData.total && elapsedTimeMs > 1000) {
+          const progressPercentage = operationData.progress / operationData.total;
+          const estimatedTotalTime = (elapsedTimeMs / progressPercentage) * FUDGE_FACTOR;
+          estimatedTimeRemainingMs = Math.max(0, estimatedTotalTime - elapsedTimeMs);
+        }
+        
+        return {
+          progress: operationData.progress,
+          total: operationData.total,
+          elapsedTimeMs,
+          estimatedTimeRemainingMs
+        };
+      }
+    }
+    return { progress: 0, total: 100, elapsedTimeMs: 0, estimatedTimeRemainingMs: null };
+  }
+
+  public clearOperation(operationId: string): void {
+    this.progressMap.delete(operationId);
+    if (this.currentOperation === operationId) {
+      this.currentOperation = null;
+    }
+  }
+}

package/src/services/progressVectorStoreIndex.d.ts

@@ -0,0 +1,20 @@
+import { VectorStoreIndex, VectorIndexOptions as BaseVectorIndexOptions } from "llamaindex";
+import { BaseNode } from "llamaindex";
+export interface VectorIndexOptions extends BaseVectorIndexOptions {
+    progressCallback?: (progress: number, total: number) => void;
+}
+export declare class ProgressVectorStoreIndex extends VectorStoreIndex {
+    static init(options: VectorIndexOptions): Promise<VectorStoreIndex>;
+    buildIndexFromNodes(nodes: BaseNode[], options?: {
+        logProgress?: boolean;
+        progressCallback?: (progress: number, total: number) => void;
+    }): Promise<void>;
+    insertNodes(nodes: BaseNode[], options?: {
+        logProgress?: boolean;
+        progressCallback?: (progress: number, total: number) => void;
+    }): Promise<void>;
+    getNodeEmbeddingResults(nodes: BaseNode[], options?: {
+        logProgress?: boolean;
+        progressCallback?: (progress: number, total: number) => void;
+    }): Promise<BaseNode[]>;
+}

package/src/services/progressVectorStoreIndex.ts

@@ -0,0 +1,95 @@
+import { VectorStoreIndex, type VectorIndexOptions as BaseVectorIndexOptions, storageContextFromDefaults, IndexDict } from "llamaindex";
+import { BaseNode, ModalityType, splitNodesByType } from "llamaindex";
+import { addNodesToVectorStores } from "llamaindex";
+
+// Extend the VectorIndexOptions interface to include progressCallback
+export interface VectorIndexOptions extends BaseVectorIndexOptions {
+  progressCallback?: (progress: number, total: number) => void;
+}
+
+// Subclass VectorStoreIndex to handle progressCallback
+// @ts-ignore
+export class ProgressVectorStoreIndex extends VectorStoreIndex {
+  public static async init(
+    options: VectorIndexOptions,
+  ): Promise<VectorStoreIndex> {
+    const storageContext =
+      options.storageContext ?? (await storageContextFromDefaults({}));
+    const indexStore = storageContext.indexStore;
+    const docStore = storageContext.docStore;
+
+    // @ts-ignore 
+    let indexStruct = await VectorStoreIndex.setupIndexStructFromStorage(
+      indexStore,
+      options,
+    );
+
+    if (!options.nodes && !indexStruct) {
+      throw new Error(
+        "Cannot initialize VectorStoreIndex without nodes or indexStruct",
+      );
+    }
+
+    indexStruct = indexStruct ?? new IndexDict();
+
+    // @ts-ignore
+    const index = new this({
+      storageContext,
+      docStore,
+      indexStruct,
+      indexStore,
+      vectorStores: options.vectorStores,
+    });
+
+    if (options.nodes) {
+      // If nodes are passed in, then we need to update the index
+      await index.buildIndexFromNodes(options.nodes, {
+        logProgress: options.logProgress,
+        progressCallback: options.progressCallback,
+      });
+    }
+    return index;
+  }
+
+  async buildIndexFromNodes(
+    nodes: BaseNode[],
+    options?: { logProgress?: boolean; progressCallback?: (progress: number, total: number) => void }
+  ) {
+    await this.insertNodes(nodes, options);
+  }
+
+  async insertNodes(
+    nodes: BaseNode[],
+    options?: { logProgress?: boolean; progressCallback?: (progress: number, total: number) => void }
+  ) {
+    if (!nodes || nodes.length === 0) {
+      return;
+    }
+
+    nodes = await this.getNodeEmbeddingResults(nodes, options);
+    await addNodesToVectorStores(
+      nodes,
+      this.vectorStores,
+      this.insertNodesToStore.bind(this),
+    );
+    await this.indexStore.addIndexStruct(this.indexStruct);
+  }
+
+  async getNodeEmbeddingResults(
+    nodes: BaseNode[],
+    options?: { logProgress?: boolean; progressCallback?: (progress: number, total: number) => void }
+  ): Promise<BaseNode[]> {
+    const nodeMap = splitNodesByType(nodes);
+    for (const type in nodeMap) {
+      const nodes = nodeMap[type as ModalityType];
+      const embedModel = this.vectorStores[type as ModalityType]?.embedModel ?? this.embedModel;
+      if (embedModel && nodes) {
+        await embedModel(nodes, {
+          logProgress: options?.logProgress,
+          progressCallback: options?.progressCallback, // Pass progressCallback to embedModel
+        });
+      }
+    }
+    return nodes;
+  }
+}

package/src/services/sentenceSplitter.d.ts

@@ -0,0 +1,16 @@
+import { SentenceSplitter } from "llamaindex";
+import natural from "natural";
+type TextSplitterFn = (text: string) => string[];
+export declare class CustomSentenceSplitter extends SentenceSplitter {
+    #private;
+    chunkingTokenizerFn: () => TextSplitterFn;
+    abbreviations: string[];
+    tokenizer: natural.SentenceTokenizer;
+    constructor(params?: {
+        chunkSize?: number;
+        chunkOverlap?: number;
+        abbreviations?: string[];
+    });
+    _splitText(text: string, chunkSize: number): string[];
+}
+export {};

package/src/services/sentenceSplitter.ts

@@ -0,0 +1,243 @@
+import { SentenceSplitter, splitBySep, splitByRegex, splitByChar, Settings } from "llamaindex";
+import natural from "natural"
+
+/*
+LlamaIndex's includes the length of the metadata as part of the size of the chunk when splitting by sentences. 
+This produces very unintuitive behavior: e.g. when the user specifies a chunk-size of 50 and nodes have metadata of length 40, 
+the resulting split sentences are about 10 tokens long -- as opposed to the specified 50.
+
+This modified SentenceSplitter adds a `include_metadata_in_chunksize` flag that disables the above behavior,
+ignoring metadata when calculating chunksize (i.e. only including the size of the text datga when calculating chunksize.)
+
+Additionally, splitTextMetadataAware does some bizarre stuff where it will split sentences at abbreviations -- even if the 
+underlying tokenizer knows about the abbreviations, I think due to some weird sub-sentence splitting. It also sews sentence
+chunks back together in a way that eliminates spaces, e.g. `JPMorgan Chase & Co.elected Mark Weinberger` and  `Mr.Weinberger was Global Chairman`.
+
+I also tried making SentenceSplitter just split on sentences (with Natural) but this misbehaved by splitting TOO much. I do need short sentences grouped
+together (whether they are true short sentences, or false-positives like "USA v. one 12 ft. I.B.M. mainframe").
+
+
+*/
+// TODO: make this configurable
+const INCLUDE_METADATA_IN_CHUNKSIZE = false;
+SentenceSplitter.prototype.splitTextMetadataAware = function(text: string, metadata: string): string[] {
+  const metadataLength = this.tokenSize(metadata);
+  const effectiveChunkSize = INCLUDE_METADATA_IN_CHUNKSIZE ? this.chunkSize - metadataLength : this.chunkSize;
+  if (effectiveChunkSize <= 0) {
+    throw new Error(
+      `Metadata length (${metadataLength}) is longer than chunk size (${this.chunkSize}). Consider increasing the chunk size or decreasing the size of your metadata to avoid this.`,
+    );
+  } else if (effectiveChunkSize < 50) {
+    console.log(
+      `Metadata length (${metadataLength}) is close to chunk size (${this.chunkSize}). Resulting chunks are less than 50 tokens. Consider increasing the chunk size or decreasing the size of your metadata to avoid this.`,
+    );
+  }
+  return this._splitText(text, effectiveChunkSize);
+}
+
+const default_abbreviations= ['dr.', 'vs.', 'mr.', 'ms.', 'mx.', 'mrs.', 'prof.', 'inc.', 'corp.', 'co.', 'llc.', 'ltd.', 'etc.', "i.e.",
+  "etc.",
+  "vs.",
+  "A.S.A.P.",
+];
+
+// verbatim copies
+type TextSplitterFn = (text: string) => string[];
+type _Split = {
+  text: string;
+  isSentence: boolean;
+  tokenSize: number;
+};
+
+
+// This varies from SentenceSplitter in two ways:
+// 1. it uses abbreviations set here.
+// 2. it uses a custom SentenceTokenizer with a second trimSentences arguemnt that controls
+//    whether or not leading/trailing whitespace is preserved.
+//    We want to preserve it, so that when sentences are merged back again, we don't end up with 
+//    sentences that are not separated by spaces.
+// Because JavaScript is stupid, we have to copy over almost the whole SentenceSplitter just to make those few small changes.
+export class CustomSentenceSplitter extends SentenceSplitter {
+
+  // this function is new.
+  chunkingTokenizerFn = (): TextSplitterFn => {
+    return (text: string) => {
+      try {
+        return this.tokenizer.tokenize(text);
+      } catch {
+        return [text];
+      }
+    };
+  };
+  #splitFns: Set<TextSplitterFn> = new Set();
+  #subSentenceSplitFns: Set<TextSplitterFn> = new Set();
+  abbreviations: string[];
+  tokenizer: natural.SentenceTokenizer;
+
+  constructor(params: { chunkSize?: number; chunkOverlap?: number; abbreviations?: string[] } = {}) {
+    super(params);
+    // Create custom tokenizer with abbreviations
+    this.abbreviations = params.abbreviations || default_abbreviations;
+
+    // I modified my local node_modules/natural/lib/natural/tokenizers/index.d.ts to add the second argument to the natural.SentenceTokenizer constructor.
+    // once that gets fixed in the next version of the library, remove the ts-ignore.
+    // @ts-ignore
+    this.tokenizer = new natural.SentenceTokenizer(this.abbreviations, false); // false is don't trim sentences
+
+    // copied from the superclass.
+    this.#splitFns.add(splitBySep(this.paragraphSeparator));
+
+    this.#splitFns.add(this.chunkingTokenizerFn()); // the ONLY change here in the constructor.
+
+    // copied from the superclass.
+    this.#subSentenceSplitFns.add(splitByRegex(this.secondaryChunkingRegex));
+    this.#subSentenceSplitFns.add(splitBySep(this.separator));
+    this.#subSentenceSplitFns.add(splitByChar());
+
+    // left over from a failed attempt to JUST use natural.SentenceTokenizer
+    // but I DO in fact need the merge stuff.
+    // const tokenizer = 
+    // Override the default splitText method
+    // this.splitText = (text: string): string[] => {
+    //   return tokenizer.tokenize(text);
+    // };
+    // /* tslint:disable:no-unused-variable */
+    // this.splitTextMetadataAware = (text: string, metadata: string): string[] => {
+    //   return tokenizer.tokenize(text);
+    // }
+  }
+
+
+  //just verbatim copies of the parent class
+
+  _splitText(text: string, chunkSize: number): string[] {
+    if (text === "") return [text];
+
+    const callbackManager = Settings.callbackManager;
+
+    callbackManager.dispatchEvent("chunking-start", {
+      text: [text],
+    });
+    const splits = this.#split(text, chunkSize);
+    const chunks = this.#merge(splits, chunkSize);
+
+    callbackManager.dispatchEvent("chunking-end", {
+      chunks,
+    });
+    return chunks;
+  }
+
+  #split(text: string, chunkSize: number): _Split[] {
+    const tokenSize = this.tokenSize(text);
+    if (tokenSize <= chunkSize) {
+      return [
+        {
+          text,
+          isSentence: true,
+          tokenSize,
+        },
+      ];
+    }
+    const [textSplitsByFns, isSentence] = this.#getSplitsByFns(text);
+    const textSplits: _Split[] = [];
+
+    for (const textSplit of textSplitsByFns) {
+      const tokenSize = this.tokenSize(textSplit);
+      if (tokenSize <= chunkSize) {
+        textSplits.push({
+          text: textSplit,
+          isSentence,
+          tokenSize,
+        });
+      } else {
+        const recursiveTextSplits = this.#split(textSplit, chunkSize);
+        textSplits.push(...recursiveTextSplits);
+      }
+    }
+    return textSplits;
+  }
+
+  #getSplitsByFns(text: string): [splits: string[], isSentence: boolean] {
+    for (const splitFn of this.#splitFns) {
+      const splits = splitFn(text);
+      if (splits.length > 1) {
+        return [splits, true];
+      }
+    }
+    for (const splitFn of this.#subSentenceSplitFns) {
+      const splits = splitFn(text);
+      if (splits.length > 1) {
+        return [splits, false];
+      }
+    }
+    return [[text], true];
+  }
+
+  #merge(splits: _Split[], chunkSize: number): string[] {
+    const chunks: string[] = [];
+    let currentChunk: [string, number][] = [];
+    let lastChunk: [string, number][] = [];
+    let currentChunkLength = 0;
+    let newChunk = true;
+
+    const closeChunk = (): void => {
+      chunks.push(currentChunk.map(([text]) => text).join(""));
+      lastChunk = currentChunk;
+      currentChunk = [];
+      currentChunkLength = 0;
+      newChunk = true;
+
+      let lastIndex = lastChunk.length - 1;
+      while (
+        lastIndex >= 0 &&
+        currentChunkLength + lastChunk[lastIndex]![1] <= this.chunkOverlap
+      ) {
+        const [text, length] = lastChunk[lastIndex]!;
+        currentChunkLength += length;
+        currentChunk.unshift([text, length]);
+        lastIndex -= 1;
+      }
+    };
+
+    while (splits.length > 0) {
+      const curSplit = splits[0]!;
+      if (curSplit.tokenSize > chunkSize) {
+        throw new Error("Single token exceeded chunk size");
+      }
+      if (currentChunkLength + curSplit.tokenSize > chunkSize && !newChunk) {
+        closeChunk();
+      } else {
+        if (
+          curSplit.isSentence ||
+          currentChunkLength + curSplit.tokenSize <= chunkSize ||
+          newChunk
+        ) {
+          currentChunkLength += curSplit.tokenSize;
+          currentChunk.push([curSplit.text, curSplit.tokenSize]);
+          splits.shift();
+          newChunk = false;
+        } else {
+          closeChunk();
+        }
+      }
+    }
+
+    // Handle the last chunk
+    if (!newChunk) {
+      chunks.push(currentChunk.map(([text]) => text).join(""));
+    }
+
+    return this.#postprocessChunks(chunks);
+  }
+
+  #postprocessChunks(chunks: string[]): string[] {
+    const newChunks: string[] = [];
+    for (const chunk of chunks) {
+      const trimmedChunk = chunk.trim();
+      if (trimmedChunk !== "") {
+        newChunks.push(trimmedChunk);
+      }
+    }
+    return newChunks;
+  }
+} 

package/src/services/sploder.d.ts

@@ -0,0 +1,12 @@
+import { TextNode, TransformComponent } from "llamaindex";
+interface SploderConfig {
+    maxStringTokenCount: number;
+}
+export declare class Sploder extends TransformComponent {
+    private maxTokenCount;
+    private tokenizer;
+    constructor(config: SploderConfig);
+    private getTokenCount;
+    transform(nodes: TextNode[]): Promise<TextNode[]>;
+}
+export {};

package/src/services/sploder.ts

@@ -0,0 +1,62 @@
+import { TextNode, BaseNode, TransformComponent } from "llamaindex";
+import { encodingForModel } from "js-tiktoken";
+
+interface SploderConfig {
+  maxStringTokenCount: number;
+}
+
+export class Sploder extends TransformComponent {
+  private maxTokenCount: number;
+  private tokenizer: any; // js-tiktoken encoder
+
+  // TODO: this is a hack to get the tokenizer for the embedding model
+  // TODO: this should be a singleton
+  constructor(config: SploderConfig) {
+    super(async (nodes: BaseNode[]) => nodes); // no-op, to be replaced later
+    this.maxTokenCount = config.maxStringTokenCount;
+    this.tokenizer = encodingForModel("text-embedding-3-small");
+  }
+
+  private getTokenCount(text: string): number {
+    return this.tokenizer.encode(text).length;
+  }
+
+  async transform(nodes: TextNode[]): Promise<TextNode[]> {
+    const newNodes: TextNode[] = [];
+
+    nodes.forEach((node, index) => {
+      // Keep original node
+      newNodes.push(node);
+
+      // Skip if text is too long
+      if (this.getTokenCount(node.text) > this.maxTokenCount) {
+        return;
+      }
+
+      const prevNode = index > 0 ? nodes[index - 1] : null;
+      const nextNode = index < nodes.length - 1 ? nodes[index + 1] : null;
+
+      // Create node with current + next if available
+      if (nextNode) {
+        newNodes.push(
+          new TextNode({
+            text: node.text + " " + nextNode.text,
+            metadata: { ...node.metadata, isExpanded: true }
+          })
+        );
+      }
+
+      // Create node with prev + current + next if both available
+      if (prevNode && nextNode) {
+        newNodes.push(
+          new TextNode({
+            text: prevNode.text + " " + node.text + " " + nextNode.text,
+            metadata: { ...node.metadata, isExpanded: true }
+          })
+        );
+      }
+    });
+
+    return newNodes;
+  }
+} 

package/src/types/index.d.ts

@@ -0,0 +1,71 @@
+export interface SearchResult {
+    text: string;
+    score: number;
+    metadata: Record<string, any>;
+}
+export interface EmbeddingResult {
+    success: boolean;
+    error?: string;
+    index?: any;
+}
+export interface PreviewResult {
+    success: boolean;
+    error?: string;
+    nodes?: Array<{
+        text: string;
+        metadata: Record<string, any>;
+    }>;
+    estimatedPrice?: number;
+    tokenCount?: number;
+    pricePer1M?: number;
+}
+export interface DocumentSetMetadata {
+    documentSetId: number;
+    name: string;
+    uploadDate: Date;
+    parameters: Record<string, unknown>;
+    totalDocuments: number;
+}
+export interface DocumentSetParams {
+    datasetName: string;
+    description: string;
+    textColumns: string[];
+    metadataColumns: string[];
+    splitIntoSentences: boolean;
+    combineSentencesIntoChunks: boolean;
+    sploderMaxSize: number;
+    chunkSize: number;
+    chunkOverlap: number;
+    modelName: string;
+    modelProvider: string;
+}
+export interface EmbeddingConfig {
+    modelName: string;
+    modelProvider: string;
+    vectorStoreType: "simple" | "postgres" | "weaviate";
+    projectName: string;
+    storagePath: string;
+    splitIntoSentences: boolean;
+    combineSentencesIntoChunks: boolean;
+    sploderMaxSize: number;
+    chunkSize: number;
+    chunkOverlap: number;
+}
+export interface Settings {
+    openAIKey: string | null;
+    oLlamaBaseURL: string | null;
+    azureOpenAIKey: string | null;
+    azureOpenAIEndpoint: string | null;
+    azureOpenAIApiVersion: string | null;
+    mistralApiKey: string | null;
+    geminiApiKey: string | null;
+}
+export interface MetadataFilter {
+    key: string;
+    operator: "==" | "in" | ">" | "<" | "!=" | ">=" | "<=" | "nin" | "any" | "all" | "text_match" | "contains" | "is_empty";
+    value: any;
+}
+export interface Clients {
+    weaviateClient: any;
+    postgresClient: any;
+}

package/src/types/index.ts

@@ -0,0 +1,89 @@
+export interface SearchResult {
+  text: string;
+  score: number;
+  metadata: Record<string, any>;
+}
+
+export interface EmbeddingResult {
+  success: boolean;
+  error?: string;
+  index?: any;
+}
+
+export interface PreviewResult {
+  success: boolean;
+  error?: string;
+  nodes?: Array<{
+    text: string;
+    metadata: Record<string, any>;
+  }>;
+  estimatedPrice?: number;
+  tokenCount?: number;
+  pricePer1M?: number;
+} 
+
+// Type definitions for meaningfully core
+export interface SearchConfig {
+  modelProvider: string
+  modelName: string
+  projectName: string
+}
+
+// Define types for our document set metadata
+export interface DocumentSetMetadata {
+  documentSetId: number;
+  name: string;
+  uploadDate: Date;
+  parameters: Record<string, unknown>;
+  totalDocuments: number;
+}
+
+export interface DocumentSetParams {
+  datasetName: string,
+  description: string,
+  textColumns: string[],
+  metadataColumns: string[],
+  splitIntoSentences: boolean,
+  combineSentencesIntoChunks: boolean,
+  sploderMaxSize: number,
+  chunkSize: number,
+  chunkOverlap: number,
+  modelName: string,
+  modelProvider: string
+}
+
+
+export interface EmbeddingConfig {
+  modelName: string;
+  modelProvider: string
+  vectorStoreType: "simple" | "postgres" | "weaviate";
+  projectName: string;
+  storagePath: string;
+  splitIntoSentences: boolean;
+  combineSentencesIntoChunks: boolean;
+  sploderMaxSize: number;
+  chunkSize: number;
+  chunkOverlap: number;
+}
+
+
+export interface Settings {
+  openAIKey: string | null;
+  oLlamaBaseURL: string | null;
+  azureOpenAIKey: string | null;
+  azureOpenAIEndpoint: string | null;
+  azureOpenAIApiVersion: string | null;
+  mistralApiKey: string | null;
+  geminiApiKey: string | null;
+}
+
+export interface MetadataFilter{ 
+  key: string, 
+  operator: "==" | "in" | ">" | "<" | "!=" | ">=" | "<=" | "nin" | "any" | "all" | "text_match" | "contains" | "is_empty", 
+  value: any 
+}
+
+export interface Clients {
+  weaviateClient: any;
+  postgresClient: any;
+}