@mastra/rag 1.2.2 → 1.2.3-alpha.1

package/src/document/transformers/character.ts

@@ -1,267 +0,0 @@
-import { Language } from '../types';
-import type { BaseChunkOptions, CharacterChunkOptions, RecursiveChunkOptions } from '../types';
-
-import { TextTransformer } from './text';
-
-function splitTextWithRegex(text: string, separator: string, keepSeparator: boolean | 'start' | 'end'): string[] {
-  if (!separator) {
-    return text.split('');
-  }
-
-  if (!keepSeparator) {
-    return text.split(new RegExp(separator)).filter(s => s !== '');
-  }
-
-  if (!text) {
-    return [];
-  }
-
-  // Split with capturing group to keep separators
-  const splits = text.split(new RegExp(`(${separator})`));
-  const result: string[] = [];
-
-  if (keepSeparator === 'end') {
-    // Process all complete pairs
-    for (let i = 0; i < splits.length - 1; i += 2) {
-      if (i + 1 < splits.length) {
-        // Current text + separator
-        const chunk = splits[i] + (splits[i + 1] || '');
-        if (chunk) result.push(chunk);
-      }
-    }
-    // Handle the last element if it exists and isn't a separator
-    if (splits.length % 2 === 1 && splits[splits.length - 1]) {
-      result.push(splits?.[splits.length - 1]!);
-    }
-  } else {
-    if (splits[0]) result.push(splits[0]);
-
-    for (let i = 1; i < splits.length - 1; i += 2) {
-      const separator = splits[i];
-      const text = splits[i + 1];
-      if (separator && text) {
-        result.push(separator + text);
-      }
-    }
-  }
-
-  return result.filter(s => s !== '');
-}
-
-export class CharacterTransformer extends TextTransformer {
-  protected separator: string;
-  protected isSeparatorRegex: boolean;
-
-  constructor({ separator = '\n\n', isSeparatorRegex = false, ...baseOptions }: CharacterChunkOptions = {}) {
-    super(baseOptions);
-    this.separator = separator;
-    this.isSeparatorRegex = isSeparatorRegex;
-  }
-
-  splitText({ text }: { text: string }): string[] {
-    // First, split the text into initial chunks
-    const separator = this.isSeparatorRegex ? this.separator : this.separator.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
-
-    const initialSplits = splitTextWithRegex(text, separator, this.keepSeparator);
-
-    // If length of any split is greater than chunk size, perform additional splitting
-    const chunks: string[] = [];
-    for (const split of initialSplits) {
-      if (this.lengthFunction(split) <= this.maxSize) {
-        chunks.push(split);
-      } else {
-        // If a single split is too large, split it further with overlap
-        const subChunks = this.__splitChunk(split);
-        chunks.push(...subChunks);
-      }
-    }
-
-    return chunks;
-  }
-
-  private __splitChunk(text: string): string[] {
-    const chunks: string[] = [];
-    let currentPosition = 0;
-
-    while (currentPosition < text.length) {
-      let chunkEnd = currentPosition;
-
-      // Build chunk up to max size
-      while (chunkEnd < text.length && this.lengthFunction(text.slice(currentPosition, chunkEnd + 1)) <= this.maxSize) {
-        chunkEnd++;
-      }
-
-      const currentChunk = text.slice(currentPosition, chunkEnd);
-      const chunkLength = this.lengthFunction(currentChunk);
-      chunks.push(currentChunk);
-
-      // If we're at the end, break to avoid tiny chunks
-      if (chunkEnd >= text.length) break;
-
-      // Move position forward by chunk size minus overlap
-      currentPosition += Math.max(1, chunkLength - this.overlap);
-    }
-
-    return chunks;
-  }
-}
-
-export class RecursiveCharacterTransformer extends TextTransformer {
-  protected separators: string[];
-  protected isSeparatorRegex: boolean;
-
-  constructor({ separators, isSeparatorRegex = false, language, ...baseOptions }: RecursiveChunkOptions = {}) {
-    super(baseOptions);
-    this.separators = separators || ['\n\n', '\n', ' ', ''];
-    this.isSeparatorRegex = isSeparatorRegex;
-  }
-
-  private _splitText(text: string, separators: string[]): string[] {
-    const finalChunks: string[] = [];
-
-    let separator = separators?.[separators.length - 1]!;
-    let newSeparators: string[] = [];
-
-    for (let i = 0; i < separators.length; i++) {
-      const s = separators[i]!;
-      const _separator = this.isSeparatorRegex ? s : s?.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
-
-      if (s === '') {
-        separator = s;
-        break;
-      }
-
-      if (new RegExp(_separator).test(text)) {
-        separator = s;
-        newSeparators = separators.slice(i + 1);
-        break;
-      }
-    }
-
-    const _separator = this.isSeparatorRegex ? separator : separator?.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
-
-    const splits = splitTextWithRegex(text, _separator, this.keepSeparator);
-
-    const goodSplits: string[] = [];
-    const mergeSeparator = this.keepSeparator ? '' : separator;
-
-    for (const s of splits) {
-      if (this.lengthFunction(s) < this.maxSize) {
-        goodSplits.push(s);
-      } else {
-        if (goodSplits.length > 0) {
-          const mergedText = this.mergeSplits(goodSplits, mergeSeparator);
-          finalChunks.push(...mergedText);
-          goodSplits.length = 0;
-        }
-        if (newSeparators.length === 0) {
-          finalChunks.push(s);
-        } else {
-          const otherInfo = this._splitText(s, newSeparators);
-          finalChunks.push(...otherInfo);
-        }
-      }
-    }
-
-    if (goodSplits.length > 0) {
-      const mergedText = this.mergeSplits(goodSplits, mergeSeparator);
-      finalChunks.push(...mergedText);
-    }
-
-    return finalChunks;
-  }
-
-  splitText({ text }: { text: string }): string[] {
-    return this._splitText(text, this.separators);
-  }
-
-  static fromLanguage(language: Language, options: BaseChunkOptions = {}): RecursiveCharacterTransformer {
-    const separators = RecursiveCharacterTransformer.getSeparatorsForLanguage(language);
-    return new RecursiveCharacterTransformer({
-      ...options,
-      separators,
-      isSeparatorRegex: true,
-      language,
-    });
-  }
-
-  static getSeparatorsForLanguage(language: Language): string[] {
-    switch (language) {
-      case Language.MARKDOWN:
-        return [
-          // First, try to split along Markdown headings (starting with level 2)
-          '\n#{1,6} ',
-          // End of code block
-          '```\n',
-          // Horizontal lines
-          '\n\\*\\*\\*+\n',
-          '\n---+\n',
-          '\n___+\n',
-          // Note that this splitter doesn't handle horizontal lines defined
-          // by *three or more* of ***, ---, or ___, but this is not handled
-          '\n\n',
-          '\n',
-          ' ',
-          '',
-        ];
-      case Language.CPP:
-      case Language.C:
-        return [
-          '\nclass ',
-          '\nvoid ',
-          '\nint ',
-          '\nfloat ',
-          '\ndouble ',
-          '\nif ',
-          '\nfor ',
-          '\nwhile ',
-          '\nswitch ',
-          '\ncase ',
-          '\n\n',
-          '\n',
-          ' ',
-          '',
-        ];
-      case Language.TS:
-        return [
-          '\nenum ',
-          '\ninterface ',
-          '\nnamespace ',
-          '\ntype ',
-          '\nclass ',
-          '\nfunction ',
-          '\nconst ',
-          '\nlet ',
-          '\nvar ',
-          '\nif ',
-          '\nfor ',
-          '\nwhile ',
-          '\nswitch ',
-          '\ncase ',
-          '\ndefault ',
-          '\n\n',
-          '\n',
-          ' ',
-          '',
-        ];
-      case Language.LATEX:
-        return [
-          '\\\\part\\*?\\{',
-          '\\\\chapter\\*?\\{',
-          '\\\\section\\*?\\{',
-          '\\\\subsection\\*?\\{',
-          '\\\\subsubsection\\*?\\{',
-          '\\\\begin\\{.*?\\}',
-          '\\\\end\\{.*?\\}',
-          '\\\\[a-zA-Z]+\\{.*?\\}',
-          '\n\n',
-          '\n',
-          ' ',
-          '',
-        ];
-      // ... (add other language cases following the same pattern)
-      default:
-        throw new Error(`Language ${language} is not supported! Please choose from ${Object.values(Language)}`);
-    }
-  }
-}

package/src/document/transformers/html.ts

@@ -1,346 +0,0 @@
-import { parse } from 'node-html-better-parser';
-import { Document } from '../schema';
-import type { HTMLChunkOptions } from '../types';
-
-import { RecursiveCharacterTransformer } from './character';
-
-interface ElementType {
-  url: string;
-  xpath: string;
-  content: string;
-  metadata: Record<string, string>;
-}
-
-export class HTMLHeaderTransformer {
-  private headersToSplitOn: [string, string][];
-  private returnEachElement: boolean;
-
-  constructor(options: HTMLChunkOptions & { headers: [string, string][] }) {
-    this.returnEachElement = options.returnEachLine ?? false;
-    this.headersToSplitOn = [...options.headers].sort();
-  }
-
-  splitText({ text }: { text: string }): Document[] {
-    const root = parse(text);
-
-    const headerFilter = this.headersToSplitOn.map(([header]) => header);
-    const headerMapping = Object.fromEntries(this.headersToSplitOn);
-
-    const elements: ElementType[] = [];
-    const headers = root.querySelectorAll(headerFilter.join(','));
-
-    headers.forEach(header => {
-      let content = '';
-      const parentNode = header.parentNode;
-
-      if (parentNode && parentNode.childNodes) {
-        let foundHeader = false;
-        for (const node of parentNode.childNodes) {
-          // Start collecting content after we find our header
-          if (node === header) {
-            foundHeader = true;
-            continue;
-          }
-
-          // If we found our header and hit another header, stop
-          // @ts-expect-error - node.tagName is not defined on type Node
-          if (foundHeader && node.tagName && headerFilter.includes(node.tagName.toLowerCase())) {
-            break;
-          }
-
-          // Collect content between headers
-          if (foundHeader) {
-            content += this.getTextContent(node) + ' ';
-          }
-        }
-      }
-
-      elements.push({
-        url: text,
-        xpath: this.getXPath(header),
-        content: content.trim(),
-        metadata: {
-          [headerMapping?.[header.tagName.toLowerCase()]!]: header.text || '',
-        },
-      });
-    });
-
-    return this.returnEachElement
-      ? elements.map(
-          el =>
-            new Document({
-              text: el.content,
-              metadata: { ...el.metadata, xpath: el.xpath },
-            }),
-        )
-      : this.aggregateElementsToChunks(elements);
-  }
-
-  private getXPath(element: any): string {
-    if (!element) return '';
-
-    const parts: string[] = [];
-    let current = element;
-
-    while (current && current.tagName) {
-      let index = 1;
-      const parent = current.parentNode;
-
-      if (parent && parent.childNodes) {
-        // Count preceding siblings with same tag
-        for (const sibling of parent.childNodes) {
-          if (sibling === current) break;
-          if (sibling.tagName === current.tagName) {
-            index++;
-          }
-        }
-      }
-
-      parts.unshift(`${current.tagName.toLowerCase()}[${index}]`);
-      current = current.parentNode;
-    }
-
-    return '/' + parts.join('/');
-  }
-
-  private getTextContent(element: any): string {
-    if (!element) return '';
-
-    // For text nodes, return their content
-    if (!element.tagName) {
-      return element.text || '';
-    }
-
-    // For element nodes, combine their text with children's text
-    let content = element.text || '';
-
-    if (element.childNodes) {
-      for (const child of element.childNodes) {
-        const childText = this.getTextContent(child);
-        if (childText) {
-          content += ' ' + childText;
-        }
-      }
-    }
-
-    return content.trim();
-  }
-
-  private aggregateElementsToChunks(elements: ElementType[]): Document[] {
-    const aggregatedChunks: ElementType[] = [];
-
-    for (const element of elements) {
-      if (
-        aggregatedChunks.length > 0 &&
-        JSON.stringify(aggregatedChunks[aggregatedChunks.length - 1]!.metadata) === JSON.stringify(element.metadata)
-      ) {
-        // If the last element has the same metadata, append content
-        aggregatedChunks[aggregatedChunks.length - 1]!.content += '  \n' + element.content;
-      } else {
-        // Otherwise, add as new element
-        aggregatedChunks.push({ ...element });
-      }
-    }
-
-    return aggregatedChunks.map(
-      chunk =>
-        new Document({
-          text: chunk.content,
-          metadata: { ...chunk.metadata, xpath: chunk.xpath },
-        }),
-    );
-  }
-
-  createDocuments(texts: string[], metadatas?: Record<string, any>[]): Document[] {
-    const _metadatas = metadatas || Array(texts.length).fill({});
-    const documents: Document[] = [];
-
-    for (let i = 0; i < texts.length; i++) {
-      const chunks = this.splitText({ text: texts[i]! });
-      for (const chunk of chunks) {
-        const metadata = { ...(_metadatas[i] || {}) };
-        const chunkMetadata = chunk.metadata;
-
-        if (chunkMetadata) {
-          for (const [key, value] of Object.entries(chunkMetadata || {})) {
-            if (value === '#TITLE#') {
-              chunkMetadata[key] = metadata['Title'];
-            }
-          }
-        }
-
-        documents.push(
-          new Document({
-            text: chunk.text!,
-            metadata: { ...metadata, ...chunkMetadata },
-          }),
-        );
-      }
-    }
-
-    return documents;
-  }
-
-  transformDocuments(documents: Document[]): Document[] {
-    const texts: string[] = [];
-    const metadatas: Record<string, any>[] = [];
-
-    for (const doc of documents) {
-      texts.push(doc.text);
-      metadatas.push(doc.metadata);
-    }
-
-    return this.createDocuments(texts, metadatas);
-  }
-}
-
-export class HTMLSectionTransformer {
-  private headersToSplitOn: Record<string, string>;
-  private textSplitter: RecursiveCharacterTransformer;
-
-  constructor(options: HTMLChunkOptions & { sections: [string, string][] }) {
-    this.headersToSplitOn = Object.fromEntries(options.sections.map(([tag, name]) => [tag.toLowerCase(), name]));
-    this.textSplitter = new RecursiveCharacterTransformer(options);
-  }
-
-  splitText(text: string): Document[] {
-    const sections = this.splitHtmlByHeaders(text);
-
-    return sections.map(
-      section =>
-        new Document({
-          text: section.content,
-          metadata: {
-            [this.headersToSplitOn[section.tagName.toLowerCase()]!]: section.header,
-            xpath: section.xpath,
-          },
-        }),
-    );
-  }
-
-  private getXPath(element: any): string {
-    const parts: string[] = [];
-    let current = element;
-
-    while (current && current.nodeType === 1) {
-      let index = 1;
-      let sibling = current.previousSibling;
-
-      while (sibling) {
-        if (sibling.nodeType === 1 && sibling.tagName === current.tagName) {
-          index++;
-        }
-        sibling = sibling.previousSibling;
-      }
-
-      if (current.tagName) {
-        parts.unshift(`${current.tagName.toLowerCase()}[${index}]`);
-      }
-      current = current.parentNode;
-    }
-
-    return '/' + parts.join('/');
-  }
-
-  private splitHtmlByHeaders(htmlDoc: string): Array<{
-    header: string;
-    content: string;
-    tagName: string;
-    xpath: string;
-  }> {
-    const sections: Array<{
-      header: string;
-      content: string;
-      tagName: string;
-      xpath: string;
-    }> = [];
-
-    const root = parse(htmlDoc);
-    const headers = Object.keys(this.headersToSplitOn);
-    const headerElements = root.querySelectorAll(headers.join(','));
-
-    headerElements.forEach((headerElement, index) => {
-      const header = headerElement.text?.trim() || '';
-      const tagName = headerElement.tagName;
-      const xpath = this.getXPath(headerElement);
-      let content = '';
-
-      // @ts-expect-error - nextElementSibling is not defined on type Element
-      let currentElement = headerElement.nextElementSibling;
-      const nextHeader = headerElements[index + 1];
-
-      while (currentElement && (!nextHeader || currentElement !== nextHeader)) {
-        if (currentElement.text) {
-          content += currentElement.text.trim() + ' ';
-        }
-        currentElement = currentElement.nextElementSibling;
-      }
-
-      content = content.trim();
-      sections.push({
-        header,
-        content,
-        tagName,
-        xpath,
-      });
-    });
-
-    return sections;
-  }
-
-  async splitDocuments(documents: Document[]): Promise<Document[]> {
-    const texts: string[] = [];
-    const metadatas: Record<string, any>[] = [];
-
-    for (const doc of documents) {
-      texts.push(doc.text);
-      metadatas.push(doc.metadata);
-    }
-    const results = await this.createDocuments(texts, metadatas);
-
-    return this.textSplitter.splitDocuments(results);
-  }
-
-  createDocuments(texts: string[], metadatas?: Record<string, any>[]): Document[] {
-    const _metadatas = metadatas || Array(texts.length).fill({});
-    const documents: Document[] = [];
-
-    for (let i = 0; i < texts.length; i++) {
-      const chunks = this.splitText(texts[i]!);
-      for (const chunk of chunks) {
-        const metadata = { ...(_metadatas[i] || {}) };
-
-        const chunkMetadata = chunk.metadata;
-
-        if (chunkMetadata) {
-          for (const [key, value] of Object.entries(chunkMetadata || {})) {
-            if (value === '#TITLE#') {
-              chunkMetadata[key] = metadata['Title'];
-            }
-          }
-        }
-
-        documents.push(
-          new Document({
-            text: chunk.text!,
-            metadata: { ...metadata, ...chunkMetadata },
-          }),
-        );
-      }
-    }
-
-    return documents;
-  }
-
-  transformDocuments(documents: Document[]): Document[] {
-    const texts: string[] = [];
-    const metadatas: Record<string, any>[] = [];
-
-    for (const doc of documents) {
-      texts.push(doc.text);
-      metadatas.push(doc.metadata);
-    }
-
-    return this.createDocuments(texts, metadatas);
-  }
-}