@sixthsummit/ship-api 0.0.0-stage → 4.1.1

package/lib/libraries/chunking/strategies/SemanticChunkingStrategy.ts

@@ -0,0 +1,389 @@
+import { encoding_for_model, get_encoding, type Tiktoken } from "tiktoken";
+import type { SemanticChunkingOptions, TextChunk } from "../types/index.js";
+import type { ChunkingStrategy } from "./ChunkingStrategy.js";
+
+/**
+ * Semantic chunking strategy that creates chunks based on document structure
+ *
+ * Industry Best Practices Implemented:
+ * - Content-aware chunking using document structure
+ * - Preserves semantic units (paragraphs, sections, lists)
+ * - Handles code blocks, headers, and lists specially
+ * - Merges small chunks to avoid fragmentation
+ * - Maintains context through structural boundaries
+ *
+ * This approach is superior for RAG and embedding applications as it:
+ * - Preserves meaning and context better than fixed-size chunking
+ * - Avoids splitting related content
+ * - Improves retrieval accuracy
+ * - Maintains document hierarchy
+ */
+export class SemanticChunkingStrategy implements ChunkingStrategy {
+  private encoder: Tiktoken;
+  private maxTokens: number;
+  private minTokens: number;
+  private mergeSmallChunks: boolean;
+
+  // Document structure patterns
+  private static readonly HEADER_PATTERN = /^#{1,6}\s+.+$|^.+\n[=-]{3,}$/gm;
+  private static readonly PARAGRAPH_BREAK = /\n\s*\n/g;
+  private static readonly CODE_BLOCK = /```[\s\S]*?```/g;
+  private static readonly LIST_PATTERN =
+    /(?:^|\n)(?:[\s]*[-*+•]\s+.+(?:\n(?![\s]*[-*+•]\s+).+)*)+/gm;
+  private static readonly NUMBERED_LIST =
+    /(?:^|\n)(?:[\s]*\d+\.\s+.+(?:\n(?![\s]*\d+\.\s+).+)*)+/gm;
+
+  constructor(options: SemanticChunkingOptions) {
+    this.maxTokens = options.maxTokens;
+    this.minTokens = options.minTokens ?? 100;
+    this.mergeSmallChunks = options.mergeSmallChunks ?? true;
+
+    // Validate options
+    if (this.minTokens >= this.maxTokens) {
+      throw new Error("minTokens must be less than maxTokens");
+    }
+
+    // Initialize the tokenizer
+    const encodingName = options.encoding || "cl100k_base";
+    try {
+      this.encoder = get_encoding(encodingName as any);
+    } catch {
+      this.encoder = encoding_for_model("gpt-3.5-turbo");
+    }
+  }
+
+  /**
+   * Chunks the text based on semantic/structural boundaries
+   * @param text - The text to chunk
+   * @returns An array of semantically coherent chunks
+   */
+  async chunk(text: string): Promise<TextChunk[]> {
+    // Parse the document into semantic units
+    const semanticUnits = this.parseSemanticUnits(text);
+
+    // Group semantic units into chunks
+    const chunks = this.groupIntoChunks(semanticUnits, text);
+
+    return chunks;
+  }
+
+  /**
+   * Parses the document into semantic units (paragraphs, headers, code blocks, lists)
+   * @param text - The text to parse
+   * @returns Array of semantic units with their positions
+   */
+  private parseSemanticUnits(
+    text: string,
+  ): Array<{ content: string; startIndex: number; endIndex: number; type: string }> {
+    const units: Array<{ content: string; startIndex: number; endIndex: number; type: string }> =
+      [];
+
+    // Track which parts of text have been processed
+    const processed = new Array(text.length).fill(false);
+
+    // Extract code blocks first (highest priority to avoid splitting)
+    const codeBlocks = Array.from(text.matchAll(SemanticChunkingStrategy.CODE_BLOCK));
+    for (const match of codeBlocks) {
+      const startIndex = match.index!;
+      const content = match[0];
+      const endIndex = startIndex + content.length;
+
+      units.push({
+        content,
+        startIndex,
+        endIndex,
+        type: "code",
+      });
+
+      // Mark as processed
+      for (let i = startIndex; i < endIndex; i++) {
+        processed[i] = true;
+      }
+    }
+
+    // Extract numbered lists
+    const numberedLists = Array.from(text.matchAll(SemanticChunkingStrategy.NUMBERED_LIST));
+    for (const match of numberedLists) {
+      const startIndex = match.index!;
+      const content = match[0];
+      const endIndex = startIndex + content.length;
+
+      // Check if already processed (e.g., in code block)
+      if (!processed[startIndex]) {
+        units.push({
+          content: content.trim(),
+          startIndex,
+          endIndex,
+          type: "numbered_list",
+        });
+
+        for (let i = startIndex; i < endIndex; i++) {
+          processed[i] = true;
+        }
+      }
+    }
+
+    // Extract bullet lists
+    const lists = Array.from(text.matchAll(SemanticChunkingStrategy.LIST_PATTERN));
+    for (const match of lists) {
+      const startIndex = match.index!;
+      const content = match[0];
+      const endIndex = startIndex + content.length;
+
+      if (!processed[startIndex]) {
+        units.push({
+          content: content.trim(),
+          startIndex,
+          endIndex,
+          type: "list",
+        });
+
+        for (let i = startIndex; i < endIndex; i++) {
+          processed[i] = true;
+        }
+      }
+    }
+
+    // Split remaining text by paragraphs
+    let currentIndex = 0;
+    const paragraphs = text.split(SemanticChunkingStrategy.PARAGRAPH_BREAK);
+
+    for (const paragraph of paragraphs) {
+      const startIndex = text.indexOf(paragraph, currentIndex);
+      if (startIndex === -1) continue;
+
+      const endIndex = startIndex + paragraph.length;
+
+      // Check if this paragraph is already processed
+      const alreadyProcessed = processed.slice(startIndex, endIndex).some((p) => p);
+
+      if (!alreadyProcessed && paragraph.trim().length > 0) {
+        // Check if it's a header
+        const isHeader = SemanticChunkingStrategy.HEADER_PATTERN.test(paragraph);
+
+        units.push({
+          content: paragraph.trim(),
+          startIndex,
+          endIndex,
+          type: isHeader ? "header" : "paragraph",
+        });
+
+        for (let i = startIndex; i < endIndex; i++) {
+          processed[i] = true;
+        }
+      }
+
+      currentIndex = endIndex;
+    }
+
+    // Sort by start index
+    return units.sort((a, b) => a.startIndex - b.startIndex);
+  }
+
+  /**
+   * Groups semantic units into chunks that respect token limits
+   * @param units - The semantic units to group
+   * @param fullText - The full document text
+   * @returns Array of text chunks
+   */
+  private groupIntoChunks(
+    units: Array<{ content: string; startIndex: number; endIndex: number; type: string }>,
+    fullText: string,
+  ): TextChunk[] {
+    const chunks: TextChunk[] = [];
+    let currentChunk: string[] = [];
+    let currentTokenCount = 0;
+    let chunkStartIndex = 0;
+    let chunkIndex = 0;
+
+    for (let i = 0; i < units.length; i++) {
+      const unit = units[i];
+      const unitTokens = this.encoder.encode(unit.content).length;
+
+      // If this unit alone exceeds maxTokens, split it
+      if (unitTokens > this.maxTokens) {
+        // Save current chunk if not empty
+        if (currentChunk.length > 0) {
+          const chunkContent = currentChunk.join("\n\n");
+          chunks.push({
+            content: chunkContent,
+            startIndex: chunkStartIndex,
+            endIndex: chunkStartIndex + chunkContent.length,
+            tokenCount: currentTokenCount,
+            chunkIndex: chunkIndex++,
+          });
+          currentChunk = [];
+          currentTokenCount = 0;
+        }
+
+        // Split the large unit
+        const subChunks = this.splitLargeUnit(unit, fullText);
+        subChunks.forEach((subChunk) => {
+          chunks.push({
+            ...subChunk,
+            chunkIndex: chunkIndex++,
+          });
+        });
+
+        chunkStartIndex = unit.endIndex;
+        continue;
+      }
+
+      // Check if adding this unit would exceed maxTokens
+      if (currentTokenCount + unitTokens > this.maxTokens && currentChunk.length > 0) {
+        // Save current chunk
+        const chunkContent = currentChunk.join("\n\n");
+        chunks.push({
+          content: chunkContent,
+          startIndex: chunkStartIndex,
+          endIndex: chunkStartIndex + chunkContent.length,
+          tokenCount: currentTokenCount,
+          chunkIndex: chunkIndex++,
+        });
+
+        // Start new chunk
+        currentChunk = [unit.content];
+        currentTokenCount = unitTokens;
+        chunkStartIndex = unit.startIndex;
+      } else {
+        // Add to current chunk
+        if (currentChunk.length === 0) {
+          chunkStartIndex = unit.startIndex;
+        }
+        currentChunk.push(unit.content);
+        currentTokenCount += unitTokens;
+      }
+    }
+
+    // Add final chunk
+    if (currentChunk.length > 0) {
+      const chunkContent = currentChunk.join("\n\n");
+      chunks.push({
+        content: chunkContent,
+        startIndex: chunkStartIndex,
+        endIndex: chunkStartIndex + chunkContent.length,
+        tokenCount: currentTokenCount,
+        chunkIndex: chunkIndex++,
+      });
+    }
+
+    // Merge small chunks if enabled
+    if (this.mergeSmallChunks) {
+      return this.mergeSmallChunksInternal(chunks);
+    }
+
+    return chunks;
+  }
+
+  /**
+   * Splits a large semantic unit into smaller chunks
+   * @param unit - The unit to split
+   * @param fullText - The full document text
+   * @returns Array of chunks
+   */
+  private splitLargeUnit(
+    unit: { content: string; startIndex: number; endIndex: number; type: string },
+    _fullText: string,
+  ): Omit<TextChunk, "chunkIndex">[] {
+    const chunks: Omit<TextChunk, "chunkIndex">[] = [];
+
+    // Split by sentences for large units
+    const sentences = unit.content.split(/(?<=[.!?])\s+/);
+    let currentChunk: string[] = [];
+    let currentTokenCount = 0;
+    let sentenceStartIndex = unit.startIndex;
+
+    for (const sentence of sentences) {
+      const sentenceTokens = this.encoder.encode(sentence).length;
+
+      if (currentTokenCount + sentenceTokens > this.maxTokens && currentChunk.length > 0) {
+        const chunkContent = currentChunk.join(" ");
+        chunks.push({
+          content: chunkContent,
+          startIndex: sentenceStartIndex,
+          endIndex: sentenceStartIndex + chunkContent.length,
+          tokenCount: currentTokenCount,
+        });
+
+        currentChunk = [sentence];
+        currentTokenCount = sentenceTokens;
+        sentenceStartIndex += chunkContent.length + 1;
+      } else {
+        currentChunk.push(sentence);
+        currentTokenCount += sentenceTokens;
+      }
+    }
+
+    if (currentChunk.length > 0) {
+      const chunkContent = currentChunk.join(" ");
+      chunks.push({
+        content: chunkContent,
+        startIndex: sentenceStartIndex,
+        endIndex: sentenceStartIndex + chunkContent.length,
+        tokenCount: currentTokenCount,
+      });
+    }
+
+    return chunks;
+  }
+
+  /**
+   * Merges small chunks with adjacent chunks to avoid fragmentation
+   * Industry Best Practice: Avoid chunks smaller than minTokens
+   *
+   * @param chunks - The chunks to merge
+   * @returns Merged chunks
+   */
+  private mergeSmallChunksInternal(chunks: TextChunk[]): TextChunk[] {
+    if (chunks.length <= 1) return chunks;
+
+    const merged: TextChunk[] = [];
+    let i = 0;
+
+    while (i < chunks.length) {
+      const chunk = chunks[i];
+
+      // If chunk is too small and not the last one, try to merge with next
+      if (chunk.tokenCount < this.minTokens && i < chunks.length - 1) {
+        const nextChunk = chunks[i + 1];
+        const combinedTokenCount = chunk.tokenCount + nextChunk.tokenCount;
+
+        // Merge if combined size is acceptable
+        if (combinedTokenCount <= this.maxTokens) {
+          const mergedContent = `${chunk.content}\n\n${nextChunk.content}`;
+          merged.push({
+            content: mergedContent,
+            startIndex: chunk.startIndex,
+            endIndex: nextChunk.endIndex,
+            tokenCount: combinedTokenCount,
+            chunkIndex: merged.length,
+          });
+          i += 2; // Skip both chunks
+          continue;
+        }
+      }
+
+      // Add chunk as is
+      merged.push({
+        ...chunk,
+        chunkIndex: merged.length,
+      });
+      i++;
+    }
+
+    return merged;
+  }
+
+  /**
+   * Cleanup method to free the encoder
+   */
+  dispose(): void {
+    try {
+      if (this.encoder) {
+        this.encoder.free();
+      }
+    } catch {
+      // Encoder may have already been freed
+    }
+  }
+}

package/lib/libraries/chunking/strategies/SlidingWindowChunkingStrategy.ts

@@ -0,0 +1,258 @@
+import { encoding_for_model, get_encoding, type Tiktoken } from "tiktoken";
+import type { SlidingWindowChunkingOptions, TextChunk } from "../types/index.js";
+import type { ChunkingStrategy } from "./ChunkingStrategy.js";
+
+/**
+ * Sliding window chunking strategy that creates overlapping chunks and respects semantic boundaries
+ *
+ * Industry Best Practices:
+ * - Respects paragraph boundaries (primary)
+ * - Respects sentence boundaries (fallback)
+ * - Improved sentence detection handling abbreviations, URLs, numbers
+ * - Configurable minimum chunk size
+ * - Avoids splitting semantic units
+ */
+export class SlidingWindowChunkingStrategy implements ChunkingStrategy {
+  private encoder: Tiktoken;
+
+  // Paragraph boundary patterns (double newline or more)
+  private static readonly PARAGRAPH_ENDINGS = /\n\s*\n/g;
+
+  // Improved sentence boundary patterns that handle edge cases
+  // Handles: abbreviations, numbers, URLs, quotes, ellipsis
+  private static readonly SENTENCE_ENDINGS =
+    /(?<=[.!?])\s+(?=[A-Z])|(?<=[.!?]["'])\s+(?=[A-Z])|(?<=[.!?])\s*$/gm;
+
+  // Patterns for content that should not be split
+  private static readonly CODE_BLOCK = /```[\s\S]*?```|`[^`]+`/g;
+
+  private minChunkRatio: number;
+  private preferParagraphBoundaries: boolean;
+
+  constructor(private options: SlidingWindowChunkingOptions) {
+    // Validate options
+    if (options.overlapTokens >= options.maxTokens) {
+      throw new Error("overlapTokens must be less than maxTokens");
+    }
+
+    // Set configurable options with defaults
+    this.minChunkRatio = options.minChunkRatio ?? 0.5;
+    this.preferParagraphBoundaries = options.preferParagraphBoundaries ?? true;
+
+    // Validate minChunkRatio
+    if (this.minChunkRatio < 0.1 || this.minChunkRatio > 0.9) {
+      throw new Error("minChunkRatio must be between 0.1 and 0.9");
+    }
+
+    // Initialize the tokenizer with the specified encoding
+    const encodingName = options.encoding || "cl100k_base";
+    try {
+      this.encoder = get_encoding(encodingName as any);
+    } catch {
+      // Fallback to GPT-3.5/4 encoding
+      this.encoder = encoding_for_model("gpt-3.5-turbo");
+    }
+  }
+
+  /**
+   * Chunks the text with sliding window approach and sentence boundary respect
+   * @param text - The text to chunk
+   * @returns An array of text chunks
+   */
+  async chunk(text: string): Promise<TextChunk[]> {
+    const chunks: TextChunk[] = [];
+
+    // Tokenize the entire text
+    const tokens = this.encoder.encode(text);
+    const maxTokens = this.options.maxTokens;
+    const overlapTokens = this.options.overlapTokens;
+    const stepSize = maxTokens - overlapTokens;
+
+    let chunkIndex = 0;
+    let currentPosition = 0;
+
+    while (currentPosition < tokens.length) {
+      // Get the chunk tokens
+      const chunkEndPosition = Math.min(currentPosition + maxTokens, tokens.length);
+      let chunkTokens: Uint32Array = tokens.slice(currentPosition, chunkEndPosition);
+
+      // Decode to text
+      const decoded = this.encoder.decode(chunkTokens);
+      let chunkText = typeof decoded === "string" ? decoded : new TextDecoder().decode(decoded);
+
+      // If this is not the last chunk, adjust to end at sentence boundary
+      if (chunkEndPosition < tokens.length) {
+        const adjustedChunk = this.adjustToSentenceBoundary(
+          chunkText,
+          chunkTokens,
+          text,
+          currentPosition,
+        );
+
+        if (adjustedChunk) {
+          chunkText = adjustedChunk.text;
+          chunkTokens = adjustedChunk.tokens as Uint32Array;
+        }
+      }
+
+      // Calculate character indices
+      const startIndex = this.getCharacterIndex(text, tokens, currentPosition);
+      const endIndex = startIndex + chunkText.length;
+
+      chunks.push({
+        content: chunkText,
+        startIndex,
+        endIndex,
+        tokenCount: chunkTokens.length,
+        chunkIndex: chunkIndex++,
+      });
+
+      // Move to next position with overlap
+      if (chunkEndPosition >= tokens.length) {
+        break; // We've reached the end
+      }
+
+      currentPosition += stepSize;
+    }
+
+    return chunks;
+  }
+
+  /**
+   * Adjusts the chunk to end at a semantic boundary (paragraph or sentence)
+   * Industry Best Practice: Prefer paragraph > sentence > word boundaries
+   *
+   * @param chunkText - The current chunk text
+   * @param chunkTokens - The current chunk tokens
+   * @param fullText - The full document text
+   * @param tokenStartPosition - The starting token position
+   * @returns Adjusted chunk or null if no adjustment needed
+   */
+  private adjustToSentenceBoundary(
+    chunkText: string,
+    _chunkTokens: Uint32Array,
+    _fullText: string,
+    _tokenStartPosition: number,
+  ): { text: string; tokens: Uint32Array } | null {
+    let boundaryPosition: number | null = null;
+
+    // Try to find a paragraph boundary first (best practice for semantic coherence)
+    if (this.preferParagraphBoundaries) {
+      boundaryPosition = this.findLastParagraphBoundary(chunkText);
+      if (boundaryPosition !== null && this.isValidBoundaryPosition(chunkText, boundaryPosition)) {
+        const adjustedText = chunkText.substring(0, boundaryPosition).trim();
+        const adjustedTokens = this.encoder.encode(adjustedText);
+        return {
+          text: adjustedText,
+          tokens: adjustedTokens,
+        };
+      }
+    }
+
+    // Fall back to sentence boundary if no paragraph boundary found
+    boundaryPosition = this.findLastSentenceBoundary(chunkText);
+    if (boundaryPosition !== null && this.isValidBoundaryPosition(chunkText, boundaryPosition)) {
+      const adjustedText = chunkText.substring(0, boundaryPosition).trim();
+      const adjustedTokens = this.encoder.encode(adjustedText);
+      return {
+        text: adjustedText,
+        tokens: adjustedTokens,
+      };
+    }
+
+    // No suitable boundary found, return original
+    return null;
+  }
+
+  /**
+   * Finds the last paragraph boundary in the text
+   * @param text - The text to search
+   * @returns The position of the last paragraph boundary, or null if none found
+   */
+  private findLastParagraphBoundary(text: string): number | null {
+    const matches = Array.from(text.matchAll(SlidingWindowChunkingStrategy.PARAGRAPH_ENDINGS));
+    if (matches.length === 0) return null;
+
+    const lastMatch = matches[matches.length - 1];
+    return lastMatch.index! + lastMatch[0].length;
+  }
+
+  /**
+   * Finds the last sentence boundary in the text
+   * Uses improved regex that handles abbreviations, URLs, and other edge cases
+   *
+   * @param text - The text to search
+   * @returns The position of the last sentence boundary, or null if none found
+   */
+  private findLastSentenceBoundary(text: string): number | null {
+    // Check if we're in the middle of a code block (should not split)
+    const codeBlocks = Array.from(text.matchAll(SlidingWindowChunkingStrategy.CODE_BLOCK));
+    const inCodeBlock = codeBlocks.some((match) => {
+      const matchEnd = match.index! + match[0].length;
+      return matchEnd > text.length * 0.8; // Last 20% is in code block
+    });
+
+    if (inCodeBlock) return null;
+
+    // Find sentence boundaries
+    const matches = Array.from(text.matchAll(SlidingWindowChunkingStrategy.SENTENCE_ENDINGS));
+    if (matches.length === 0) return null;
+
+    // Get the last match
+    const lastMatch = matches[matches.length - 1];
+    return lastMatch.index! + lastMatch[0].length;
+  }
+
+  /**
+   * Validates if a boundary position results in a chunk of acceptable size
+   * Industry Best Practice: Chunks should be at least minChunkRatio of target size
+   *
+   * @param text - The full chunk text
+   * @param boundaryPosition - The proposed boundary position
+   * @returns True if the boundary position is valid
+   */
+  private isValidBoundaryPosition(text: string, boundaryPosition: number): boolean {
+    // Ensure the resulting chunk is not too small
+    if (boundaryPosition < text.length * this.minChunkRatio) {
+      return false;
+    }
+
+    // Ensure we're not cutting off a tiny amount (less than 10 characters)
+    if (text.length - boundaryPosition < 10) {
+      return true; // Close enough to the end
+    }
+
+    return true;
+  }
+
+  /**
+   * Helper method to get the character index from a token index
+   * @param text - The original text
+   * @param tokens - All tokens
+   * @param tokenIndex - The token index to convert
+   * @returns The character index
+   */
+  private getCharacterIndex(text: string, tokens: Uint32Array, tokenIndex: number): number {
+    if (tokenIndex === 0) return 0;
+
+    // Decode tokens up to the target index to find character position
+    const tokensUpToIndex = tokens.slice(0, tokenIndex);
+    const decoded = this.encoder.decode(tokensUpToIndex);
+    const decodedUpToIndex =
+      typeof decoded === "string" ? decoded : new TextDecoder().decode(decoded);
+    return decodedUpToIndex.length;
+  }
+
+  /**
+   * Cleanup method to free the encoder
+   */
+  dispose(): void {
+    try {
+      if (this.encoder) {
+        this.encoder.free();
+      }
+    } catch {
+      // Encoder may have already been freed
+    }
+  }
+}

package/lib/libraries/chunking/types/index.ts

@@ -0,0 +1,63 @@
+/**
+ * Represents a chunk of text with metadata
+ */
+export interface TextChunk {
+  /** The text content of the chunk */
+  content: string;
+  /** The starting character index in the original document */
+  startIndex: number;
+  /** The ending character index in the original document */
+  endIndex: number;
+  /** The token count of the chunk */
+  tokenCount: number;
+  /** The chunk number (0-indexed) */
+  chunkIndex: number;
+}
+
+/**
+ * Configuration options for fixed length chunking
+ */
+export interface FixedLengthChunkingOptions {
+  /** The maximum number of tokens per chunk */
+  maxTokens: number;
+  /** The encoding to use for token counting (default: 'cl100k_base' for GPT-3.5/4) */
+  encoding?: string;
+}
+
+/**
+ * Configuration options for sliding window chunking
+ */
+export interface SlidingWindowChunkingOptions {
+  /** The maximum number of tokens per chunk */
+  maxTokens: number;
+  /** The number of overlapping tokens between chunks */
+  overlapTokens: number;
+  /** The encoding to use for token counting (default: 'cl100k_base' for GPT-3.5/4) */
+  encoding?: string;
+  /** Minimum chunk size as a ratio of maxTokens (default: 0.5 = 50%) */
+  minChunkRatio?: number;
+  /** Prefer paragraph boundaries over sentence boundaries (default: true) */
+  preferParagraphBoundaries?: boolean;
+}
+
+/**
+ * Configuration options for semantic chunking
+ */
+export interface SemanticChunkingOptions {
+  /** The maximum number of tokens per chunk */
+  maxTokens: number;
+  /** The minimum number of tokens per chunk (default: 100) */
+  minTokens?: number;
+  /** The encoding to use for token counting (default: 'cl100k_base') */
+  encoding?: string;
+  /** Whether to merge small chunks with adjacent chunks (default: true) */
+  mergeSmallChunks?: boolean;
+}
+
+/**
+ * Union type for all chunking options
+ */
+export type ChunkingOptions =
+  | FixedLengthChunkingOptions
+  | SlidingWindowChunkingOptions
+  | SemanticChunkingOptions;