@mastra/rag 1.2.3-alpha.0 → 1.2.3-alpha.2

package/src/document/transformers/sentence.ts

@@ -1,314 +0,0 @@
-import type { SentenceChunkOptions } from '../types';
-import { TextTransformer } from './text';
-
-export class SentenceTransformer extends TextTransformer {
-  protected minSize: number;
-  protected maxSize: number;
-  protected targetSize: number;
-  protected sentenceEnders: string[];
-  protected fallbackToWords: boolean;
-  protected fallbackToCharacters: boolean;
-  protected keepSeparator: boolean | 'start' | 'end';
-
-  constructor(options: SentenceChunkOptions) {
-    // Ensure overlap doesn't exceed maxSize for parent validation
-    const parentOverlap = Math.min(options.overlap ?? 0, options.maxSize - 1);
-
-    const baseOptions = {
-      ...options,
-      overlap: parentOverlap, // Use adjusted overlap for parent
-    };
-
-    super(baseOptions);
-
-    this.maxSize = options.maxSize;
-    this.minSize = options.minSize ?? 50;
-    this.targetSize = options.targetSize ?? Math.floor(options.maxSize * 0.8);
-    this.sentenceEnders = options.sentenceEnders ?? ['.', '!', '?'];
-    this.fallbackToWords = options.fallbackToWords ?? true;
-    this.fallbackToCharacters = options.fallbackToCharacters ?? true;
-    this.keepSeparator = options.keepSeparator ?? false;
-
-    // Override with original overlap for our sentence logic
-    this.overlap = options.overlap ?? 0;
-  }
-
-  private detectSentenceBoundaries(text: string): string[] {
-    if (!text) return [];
-
-    const sentences: string[] = [];
-    let currentSentence = '';
-    let i = 0;
-
-    while (i < text.length) {
-      const char = text[i];
-      if (!char) break; // Safety check
-
-      currentSentence += char;
-
-      if (this.sentenceEnders.includes(char)) {
-        const remainingText = text.slice(i + 1);
-
-        if (this.isRealSentenceBoundary(currentSentence, remainingText)) {
-          sentences.push(currentSentence.trim());
-          currentSentence = '';
-        }
-      }
-      i++;
-    }
-
-    if (currentSentence.trim()) {
-      sentences.push(currentSentence.trim());
-    }
-
-    return sentences.filter(s => s.length > 0);
-  }
-
-  private isRealSentenceBoundary(currentSentence: string, remainingText: string): boolean {
-    if (!remainingText.trim()) {
-      return true;
-    }
-
-    if (!/^\s+[A-Z]/.test(remainingText)) {
-      return false;
-    }
-
-    const words = currentSentence.trim().split(/\s+/);
-    const lastWord = words[words.length - 1] || '';
-
-    const baseWord = lastWord.slice(0, -1);
-
-    if (this.isCommonAbbreviation(baseWord)) {
-      return false;
-    }
-
-    return true;
-  }
-
-  private isCommonAbbreviation(word: string): boolean {
-    // Common titles
-    const titles = ['Dr', 'Mr', 'Mrs', 'Ms', 'Prof', 'Sr', 'Jr'];
-    if (titles.includes(word)) {
-      return true;
-    }
-
-    // Multi-character abbreviations with periods (U.S.A., a.m., p.m., etc.)
-    if (/^[A-Z](\.[A-Z])*$/.test(word) || /^[a-z](\.[a-z])*$/.test(word)) {
-      return true;
-    }
-
-    // Single capital letters (initials)
-    if (/^[A-Z]$/.test(word)) {
-      return true;
-    }
-
-    // Numbers (versions, decimals)
-    if (/^\d+$/.test(word)) {
-      return true;
-    }
-
-    // Time abbreviations
-    if (/^[ap]\.?m$/i.test(word)) {
-      return true;
-    }
-
-    return false;
-  }
-
-  /**
-   * Group sentences into chunks with integrated overlap processing
-   */
-  private groupSentencesIntoChunks(sentences: string[]): string[] {
-    const chunks: string[] = [];
-    let currentChunk: string[] = [];
-    let currentSize = 0;
-
-    const separator = ' ';
-
-    for (const sentence of sentences) {
-      const sentenceLength = this.lengthFunction(sentence);
-      const separatorLength = currentChunk.length > 0 ? this.lengthFunction(separator) : 0;
-      const totalLength = currentSize + sentenceLength + separatorLength;
-
-      // Handle oversized sentences with fallback strategies
-      if (sentenceLength > this.maxSize) {
-        if (currentChunk.length > 0) {
-          chunks.push(currentChunk.join(separator));
-          currentChunk = [];
-          currentSize = 0;
-        }
-
-        const fallbackChunks = this.handleOversizedSentence(sentence);
-        chunks.push(...fallbackChunks);
-        continue;
-      }
-
-      // If adding this sentence would exceed maxSize, finalize current chunk
-      if (currentChunk.length > 0 && totalLength > this.maxSize) {
-        chunks.push(currentChunk.join(separator));
-
-        const overlapSentences = this.calculateSentenceOverlap(currentChunk);
-        currentChunk = overlapSentences;
-        currentSize = this.calculateChunkSize(currentChunk);
-      }
-
-      currentChunk.push(sentence);
-      currentSize += sentenceLength + separatorLength;
-
-      // If we've reached our target size, consider finalizing the chunk
-      if (currentSize >= this.targetSize) {
-        chunks.push(currentChunk.join(separator));
-
-        const overlapSentences = this.calculateSentenceOverlap(currentChunk);
-        currentChunk = overlapSentences;
-        currentSize = this.calculateChunkSize(currentChunk);
-      }
-    }
-
-    if (currentChunk.length > 0) {
-      chunks.push(currentChunk.join(separator));
-    }
-
-    return chunks;
-  }
-
-  /**
-   * Handle oversized sentences with fallback strategies
-   */
-  private handleOversizedSentence(sentence: string): string[] {
-    // First fallback
-    if (this.fallbackToWords) {
-      const wordChunks = this.splitSentenceIntoWords(sentence);
-      if (wordChunks.length > 1) {
-        return wordChunks;
-      }
-    }
-
-    // Second fallback
-    if (this.fallbackToCharacters) {
-      return this.splitSentenceIntoCharacters(sentence);
-    }
-
-    // Last resort
-    console.warn(
-      `Sentence exceeds maxSize (${this.maxSize}) and fallbacks are disabled: "${sentence.substring(0, 50)}..."`,
-    );
-    return [sentence];
-  }
-
-  private splitSentenceIntoWords(sentence: string): string[] {
-    const words = sentence.split(/\s+/);
-    const chunks: string[] = [];
-    let currentChunk = '';
-
-    for (const word of words) {
-      const testChunk = currentChunk ? currentChunk + ' ' + word : word;
-
-      if (this.lengthFunction(testChunk) <= this.maxSize) {
-        currentChunk = testChunk;
-      } else {
-        if (currentChunk) {
-          chunks.push(currentChunk);
-        }
-
-        if (this.lengthFunction(word) > this.maxSize) {
-          if (this.fallbackToCharacters) {
-            chunks.push(...this.splitSentenceIntoCharacters(word));
-          } else {
-            chunks.push(word);
-          }
-          currentChunk = '';
-        } else {
-          currentChunk = word;
-        }
-      }
-    }
-
-    if (currentChunk) {
-      chunks.push(currentChunk);
-    }
-
-    return chunks;
-  }
-
-  private splitSentenceIntoCharacters(text: string): string[] {
-    const chunks: string[] = [];
-    let currentChunk = '';
-
-    for (const char of text) {
-      if (this.lengthFunction(currentChunk + char) <= this.maxSize) {
-        currentChunk += char;
-      } else {
-        if (currentChunk) {
-          chunks.push(currentChunk);
-        }
-        currentChunk = char;
-      }
-    }
-
-    if (currentChunk) {
-      chunks.push(currentChunk);
-    }
-
-    return chunks;
-  }
-
-  private calculateSentenceOverlap(currentChunk: string[]): string[] {
-    if (this.overlap === 0 || currentChunk.length === 0) {
-      return [];
-    }
-
-    const overlapSentences: string[] = [];
-    let overlapSize = 0;
-    const separator = ' ';
-
-    // Work backwards through sentences to build overlap
-    for (let i = currentChunk.length - 1; i >= 0; i--) {
-      const sentence = currentChunk[i];
-      if (!sentence) continue;
-
-      const sentenceLength = this.lengthFunction(sentence);
-      const separatorLength = overlapSentences.length > 0 ? this.lengthFunction(separator) : 0;
-
-      if (overlapSize + sentenceLength + separatorLength > this.overlap) {
-        break;
-      }
-
-      overlapSentences.unshift(sentence);
-      overlapSize += sentenceLength + separatorLength;
-    }
-
-    return overlapSentences;
-  }
-
-  private calculateChunkSize(sentences: string[]): number {
-    if (!sentences || sentences.length === 0) {
-      return 0;
-    }
-
-    let totalSize = 0;
-    const separator = ' ';
-
-    for (let i = 0; i < sentences.length; i++) {
-      const sentence = sentences[i]!;
-      totalSize += this.lengthFunction(sentence);
-
-      // Add separator length for all but the last sentence
-      if (i < sentences.length - 1) {
-        totalSize += this.lengthFunction(separator);
-      }
-    }
-
-    return totalSize;
-  }
-
-  splitText({ text }: { text: string }): string[] {
-    if (!text) return [];
-
-    const sentences = this.detectSentenceBoundaries(text);
-
-    const chunks = this.groupSentencesIntoChunks(sentences);
-
-    return chunks.filter(chunk => chunk.trim().length > 0);
-  }
-}

package/src/document/transformers/text.ts

@@ -1,158 +0,0 @@
-import { Document } from '../schema';
-
-import type { BaseChunkOptions } from '../types';
-
-import type { Transformer } from './transformer';
-
-export abstract class TextTransformer implements Transformer {
-  protected maxSize: number;
-  protected overlap: number;
-  protected lengthFunction: (text: string) => number;
-  protected keepSeparator: boolean | 'start' | 'end';
-  protected addStartIndex: boolean;
-  protected stripWhitespace: boolean;
-
-  constructor({
-    maxSize = 4000,
-    overlap = 200,
-    lengthFunction = (text: string) => text.length,
-    keepSeparator = false,
-    addStartIndex = false,
-    stripWhitespace = true,
-  }: BaseChunkOptions) {
-    if (overlap > maxSize) {
-      throw new Error(`Got a larger chunk overlap (${overlap}) than chunk size ` + `(${maxSize}), should be smaller.`);
-    }
-    this.maxSize = maxSize;
-    this.overlap = overlap;
-    this.lengthFunction = lengthFunction;
-    this.keepSeparator = keepSeparator;
-    this.addStartIndex = addStartIndex;
-    this.stripWhitespace = stripWhitespace;
-  }
-
-  setAddStartIndex(value: boolean): void {
-    this.addStartIndex = value;
-  }
-
-  abstract splitText({ text }: { text: string }): string[];
-
-  createDocuments(texts: string[], metadatas?: Record<string, any>[]): Document[] {
-    const _metadatas = metadatas || Array(texts.length).fill({});
-    const documents: Document[] = [];
-
-    texts.forEach((text, i) => {
-      let index = 0;
-      let previousChunkLen = 0;
-
-      this.splitText({ text }).forEach(chunk => {
-        const metadata = { ..._metadatas[i] };
-        if (this.addStartIndex) {
-          const offset = index + previousChunkLen - this.overlap;
-          index = text.indexOf(chunk, Math.max(0, offset));
-          metadata.startIndex = index;
-          previousChunkLen = chunk.length;
-        }
-        documents.push(
-          new Document({
-            text: chunk,
-            metadata,
-          }),
-        );
-      });
-    });
-
-    return documents;
-  }
-
-  splitDocuments(documents: Document[]): Document[] {
-    const texts: string[] = [];
-    const metadatas: Record<string, any>[] = [];
-    for (const doc of documents) {
-      texts.push(doc.text);
-      metadatas.push(doc.metadata);
-    }
-    return this.createDocuments(texts, metadatas);
-  }
-
-  transformDocuments(documents: Document[]): Document[] {
-    const texts: string[] = [];
-    const metadatas: Record<string, any>[] = [];
-
-    for (const doc of documents) {
-      texts.push(doc.text);
-      metadatas.push(doc.metadata);
-    }
-
-    return this.createDocuments(texts, metadatas);
-  }
-
-  protected joinDocs(docs: string[], separator: string): string | null {
-    let text = docs.join(separator);
-    if (this.stripWhitespace) {
-      text = text.trim();
-    }
-    return text === '' ? null : text;
-  }
-
-  protected mergeSplits(splits: string[], separator: string): string[] {
-    const docs: string[] = [];
-    let currentDoc: string[] = [];
-    let total = 0;
-
-    for (const d of splits) {
-      const len = this.lengthFunction(d);
-      const separatorLen = separator ? this.lengthFunction(separator) : 0;
-
-      if (total + len + (currentDoc.length > 0 ? separatorLen : 0) > this.maxSize) {
-        if (total > this.maxSize) {
-          console.warn(`Created a chunk of size ${total}, which is longer than the specified ${this.maxSize}`);
-        }
-
-        if (currentDoc.length > 0) {
-          const doc = this.joinDocs(currentDoc, separator);
-          if (doc !== null) {
-            docs.push(doc);
-          }
-
-          // Handle overlap: keep enough content from the end of current chunk
-          if (this.overlap > 0) {
-            let overlapContent: string[] = [];
-            let overlapSize = 0;
-
-            // Work backwards through currentDoc until we have enough overlap
-            for (let i = currentDoc.length - 1; i >= 0; i--) {
-              const piece = currentDoc[i]!;
-              const pieceLen = this.lengthFunction(piece);
-
-              if (overlapSize + pieceLen > this.overlap) {
-                break;
-              }
-
-              overlapContent.unshift(piece);
-              overlapSize += pieceLen + (overlapContent.length > 1 ? separatorLen : 0);
-            }
-
-            currentDoc = overlapContent;
-            total = overlapSize;
-          } else {
-            currentDoc = [];
-            total = 0;
-          }
-        }
-      }
-
-      currentDoc.push(d);
-      total += len + (currentDoc.length > 1 ? separatorLen : 0);
-    }
-
-    if (currentDoc.length > 0) {
-      const doc = this.joinDocs(currentDoc, separator);
-      if (doc !== null) {
-        docs.push(doc);
-      }
-    }
-
-    return docs;
-  }
-}

package/src/document/transformers/token.ts

@@ -1,137 +0,0 @@
-import type { TiktokenModel, TiktokenEncoding, Tiktoken } from 'js-tiktoken';
-import { encodingForModel, getEncoding } from 'js-tiktoken';
-import type { TokenChunkOptions } from '../types';
-
-import { TextTransformer } from './text';
-
-interface Tokenizer {
-  overlap: number;
-  tokensPerChunk: number;
-  decode: (tokens: number[]) => string;
-  encode: (text: string) => number[];
-}
-
-export function splitTextOnTokens({ text, tokenizer }: { text: string; tokenizer: Tokenizer }): string[] {
-  const splits: string[] = [];
-  const inputIds = tokenizer.encode(text);
-  let startIdx = 0;
-  let curIdx = Math.min(startIdx + tokenizer.tokensPerChunk, inputIds.length);
-  let chunkIds = inputIds.slice(startIdx, curIdx);
-
-  while (startIdx < inputIds.length) {
-    splits.push(tokenizer.decode(chunkIds));
-    if (curIdx === inputIds.length) {
-      break;
-    }
-    startIdx += tokenizer.tokensPerChunk - tokenizer.overlap;
-    curIdx = Math.min(startIdx + tokenizer.tokensPerChunk, inputIds.length);
-    chunkIds = inputIds.slice(startIdx, curIdx);
-  }
-
-  return splits;
-}
-
-export class TokenTransformer extends TextTransformer {
-  private tokenizer: Tiktoken;
-  private allowedSpecial: Set<string> | 'all';
-  private disallowedSpecial: Set<string> | 'all';
-
-  constructor({
-    encodingName = 'cl100k_base',
-    modelName,
-    allowedSpecial = new Set(),
-    disallowedSpecial = 'all',
-    options = {},
-  }: {
-    encodingName?: TiktokenEncoding;
-    modelName?: TiktokenModel;
-    allowedSpecial?: Set<string> | 'all';
-    disallowedSpecial?: Set<string> | 'all';
-    options: TokenChunkOptions;
-  }) {
-    super(options);
-
-    try {
-      this.tokenizer = modelName ? encodingForModel(modelName) : getEncoding(encodingName);
-    } catch {
-      throw new Error('Could not load tiktoken encoding. ' + 'Please install it with `npm install js-tiktoken`.');
-    }
-
-    this.allowedSpecial = allowedSpecial;
-    this.disallowedSpecial = disallowedSpecial;
-  }
-
-  splitText({ text }: { text: string }): string[] {
-    const encode = (text: string): number[] => {
-      const allowed = this.allowedSpecial === 'all' ? 'all' : Array.from(this.allowedSpecial);
-
-      const disallowed = this.disallowedSpecial === 'all' ? 'all' : Array.from(this.disallowedSpecial);
-
-      // If stripWhitespace is enabled, trim the text before encoding
-      const processedText = this.stripWhitespace ? text.trim() : text;
-      return Array.from(this.tokenizer.encode(processedText, allowed, disallowed));
-    };
-
-    const decode = (tokens: number[]): string => {
-      const text = this.tokenizer.decode(tokens);
-      return this.stripWhitespace ? text.trim() : text;
-    };
-
-    const tokenizer: Tokenizer = {
-      overlap: this.overlap,
-      tokensPerChunk: this.maxSize,
-      decode,
-      encode,
-    };
-
-    return splitTextOnTokens({ text, tokenizer });
-  }
-
-  static fromTikToken({
-    encodingName = 'cl100k_base',
-    modelName,
-    options = {},
-  }: {
-    encodingName?: TiktokenEncoding;
-    modelName?: TiktokenModel;
-    options?: TokenChunkOptions;
-  }): TokenTransformer {
-    let tokenizer: Tiktoken;
-
-    try {
-      if (modelName) {
-        tokenizer = encodingForModel(modelName);
-      } else {
-        tokenizer = getEncoding(encodingName);
-      }
-    } catch {
-      throw new Error('Could not load tiktoken encoding. ' + 'Please install it with `npm install js-tiktoken`.');
-    }
-
-    const tikTokenEncoder = (text: string): number => {
-      const allowed =
-        options.allowedSpecial === 'all' ? 'all' : options.allowedSpecial ? Array.from(options.allowedSpecial) : [];
-
-      const disallowed =
-        options.disallowedSpecial === 'all'
-          ? 'all'
-          : options.disallowedSpecial
-            ? Array.from(options.disallowedSpecial)
-            : [];
-
-      return tokenizer.encode(text, allowed, disallowed).length;
-    };
-
-    return new TokenTransformer({
-      encodingName,
-      modelName,
-      allowedSpecial: options.allowedSpecial,
-      disallowedSpecial: options.disallowedSpecial,
-      options: {
-        maxSize: options.maxSize,
-        overlap: options.overlap,
-        lengthFunction: tikTokenEncoder,
-      },
-    });
-  }
-}

package/src/document/transformers/transformer.ts

@@ -1,5 +0,0 @@
-import type { Document } from '../schema';
-
-export interface Transformer {
-  transformDocuments(documents: Document[]): Document[];
-}