@lov3kaizen/agentsea-embeddings 0.5.1

package/dist/chunking/index.js

@@ -0,0 +1,1408 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/chunking/index.ts
+var chunking_exports = {};
+__export(chunking_exports, {
+  BaseChunker: () => BaseChunker,
+  CodeChunker: () => CodeChunker,
+  FixedChunker: () => FixedChunker,
+  MarkdownChunker: () => MarkdownChunker,
+  RecursiveChunker: () => RecursiveChunker,
+  SemanticChunker: () => SemanticChunker,
+  chunk: () => chunk,
+  createChunker: () => createChunker,
+  createCodeChunker: () => createCodeChunker,
+  createFixedChunker: () => createFixedChunker,
+  createMarkdownChunker: () => createMarkdownChunker,
+  createRecursiveChunker: () => createRecursiveChunker,
+  createSemanticChunker: () => createSemanticChunker,
+  defaultTokenCounter: () => defaultTokenCounter,
+  mergeSmallChunks: () => mergeSmallChunks,
+  splitLargeChunks: () => splitLargeChunks
+});
+module.exports = __toCommonJS(chunking_exports);
+
+// src/chunking/BaseChunker.ts
+var import_nanoid = require("nanoid");
+var defaultTokenCounter = (text) => {
+  return Math.ceil(text.length / 4);
+};
+var BaseChunker = class {
+  /** Default options */
+  defaultOptions = {
+    chunkSize: 512,
+    chunkOverlap: 50,
+    minChunkSize: 100,
+    maxChunkSize: 2e3,
+    tokenCounter: defaultTokenCounter
+  };
+  /**
+   * Get merged options with defaults
+   */
+  getOptions(options) {
+    return {
+      chunkSize: options?.chunkSize ?? this.defaultOptions.chunkSize,
+      chunkOverlap: options?.chunkOverlap ?? this.defaultOptions.chunkOverlap,
+      minChunkSize: options?.minChunkSize ?? this.defaultOptions.minChunkSize,
+      maxChunkSize: options?.maxChunkSize ?? this.defaultOptions.maxChunkSize,
+      tokenCounter: options?.tokenCounter ?? this.defaultOptions.tokenCounter,
+      documentId: options?.documentId,
+      source: options?.source,
+      type: options?.type,
+      metadata: options?.metadata ?? {}
+    };
+  }
+  /**
+   * Create a chunk object
+   */
+  createChunk(text, index, startPosition, options, additionalMetadata) {
+    const tokenCounter = options.tokenCounter ?? defaultTokenCounter;
+    const metadata = {
+      ...options.metadata,
+      ...additionalMetadata
+    };
+    if (options.documentId) metadata.documentId = options.documentId;
+    if (options.source) metadata.source = options.source;
+    if (options.type) metadata.type = options.type;
+    return {
+      id: (0, import_nanoid.nanoid)(),
+      text,
+      index,
+      startPosition,
+      endPosition: startPosition + text.length,
+      tokenCount: tokenCounter(text),
+      charCount: text.length,
+      overlapPrev: 0,
+      overlapNext: 0,
+      metadata
+    };
+  }
+  /**
+   * Process chunks and set overlap information
+   */
+  setOverlapInfo(chunks, overlapChars) {
+    for (let i = 1; i < chunks.length; i++) {
+      chunks[i].overlapPrev = overlapChars;
+      chunks[i - 1].overlapNext = overlapChars;
+    }
+  }
+  /**
+   * Split text with overlap
+   */
+  splitWithOverlap(text, chunkSize, overlap, tokenCounter) {
+    const chunks = [];
+    let start = 0;
+    while (start < text.length) {
+      let end = start;
+      let tokens = 0;
+      while (end < text.length && tokens < chunkSize) {
+        end++;
+        tokens = tokenCounter(text.slice(start, end));
+      }
+      if (end < text.length) {
+        const lastSpace = text.lastIndexOf(" ", end);
+        if (lastSpace > start) {
+          end = lastSpace + 1;
+        }
+      }
+      chunks.push(text.slice(start, end).trim());
+      const overlapChars = Math.floor(overlap * 4);
+      start = Math.max(start + 1, end - overlapChars);
+      if (start >= text.length) break;
+    }
+    return chunks.filter((c) => c.length > 0);
+  }
+  /**
+   * Chunk text and return a result object
+   */
+  async chunkWithResult(text, options) {
+    const startTime = performance.now();
+    const chunks = await this.chunk(text, options);
+    const processingTimeMs = performance.now() - startTime;
+    const totalTokens = chunks.reduce((sum, c) => sum + c.tokenCount, 0);
+    return {
+      chunks,
+      totalChunks: chunks.length,
+      totalTokens,
+      avgChunkSize: chunks.length > 0 ? totalTokens / chunks.length : 0,
+      processingTimeMs,
+      strategy: this.strategyType,
+      originalLength: text.length
+    };
+  }
+};
+function mergeSmallChunks(chunks, minTokens, tokenCounter) {
+  if (chunks.length <= 1) return chunks;
+  const merged = [];
+  let current = null;
+  for (const chunk2 of chunks) {
+    if (!current) {
+      current = { ...chunk2 };
+      continue;
+    }
+    const combinedText = current.text + "\n" + chunk2.text;
+    const combinedTokens = tokenCounter(combinedText);
+    if (current.tokenCount < minTokens) {
+      current.text = combinedText;
+      current.tokenCount = combinedTokens;
+      current.charCount = combinedText.length;
+      current.endPosition = chunk2.endPosition;
+    } else {
+      merged.push(current);
+      current = { ...chunk2 };
+    }
+  }
+  if (current) {
+    merged.push(current);
+  }
+  return merged.map((c, i) => ({ ...c, index: i }));
+}
+function splitLargeChunks(chunks, maxTokens, tokenCounter) {
+  const result = [];
+  for (const chunk2 of chunks) {
+    if (chunk2.tokenCount <= maxTokens) {
+      result.push(chunk2);
+      continue;
+    }
+    const sentences = chunk2.text.split(/(?<=[.!?])\s+/);
+    let currentText = "";
+    let currentStart = chunk2.startPosition;
+    for (const sentence of sentences) {
+      const testText = currentText ? currentText + " " + sentence : sentence;
+      const testTokens = tokenCounter(testText);
+      if (testTokens > maxTokens && currentText) {
+        result.push({
+          ...chunk2,
+          id: (0, import_nanoid.nanoid)(),
+          text: currentText,
+          startPosition: currentStart,
+          endPosition: currentStart + currentText.length,
+          tokenCount: tokenCounter(currentText),
+          charCount: currentText.length
+        });
+        currentText = sentence;
+        currentStart = currentStart + currentText.length + 1;
+      } else {
+        currentText = testText;
+      }
+    }
+    if (currentText) {
+      result.push({
+        ...chunk2,
+        id: (0, import_nanoid.nanoid)(),
+        text: currentText,
+        startPosition: currentStart,
+        endPosition: currentStart + currentText.length,
+        tokenCount: tokenCounter(currentText),
+        charCount: currentText.length
+      });
+    }
+  }
+  return result.map((c, i) => ({ ...c, index: i }));
+}
+
+// src/chunking/FixedChunker.ts
+var FixedChunker = class extends BaseChunker {
+  strategyType = "fixed";
+  async chunk(text, options) {
+    const opts = this.getOptions(options);
+    const splitByChars = options?.splitByChars ?? false;
+    const separator = options?.separator ?? "\n";
+    const keepSeparator = options?.keepSeparator ?? false;
+    const tokenCounter = opts.tokenCounter ?? defaultTokenCounter;
+    const chunks = [];
+    let position = 0;
+    if (splitByChars) {
+      const chunkSize = opts.chunkSize * 4;
+      const overlap = opts.chunkOverlap * 4;
+      let start = 0;
+      while (start < text.length) {
+        const end = Math.min(start + chunkSize, text.length);
+        const chunkText = text.slice(start, end).trim();
+        if (chunkText.length > 0) {
+          chunks.push(this.createChunk(chunkText, chunks.length, start, opts));
+        }
+        start = end - overlap;
+        if (start >= text.length) break;
+      }
+    } else {
+      const parts = text.split(separator);
+      let currentChunk = "";
+      let chunkStart = 0;
+      for (let i = 0; i < parts.length; i++) {
+        const part = parts[i];
+        const partWithSep = keepSeparator && i < parts.length - 1 ? part + separator : part;
+        const testChunk = currentChunk ? currentChunk + (keepSeparator ? "" : separator) + partWithSep : partWithSep;
+        const testTokens = tokenCounter(testChunk);
+        if (testTokens > opts.chunkSize && currentChunk) {
+          chunks.push(
+            this.createChunk(
+              currentChunk.trim(),
+              chunks.length,
+              chunkStart,
+              opts
+            )
+          );
+          const overlapText = this.getOverlapText(
+            currentChunk,
+            opts.chunkOverlap,
+            tokenCounter
+          );
+          currentChunk = overlapText + (overlapText ? separator : "") + partWithSep;
+          chunkStart = position - (overlapText?.length ?? 0);
+        } else {
+          currentChunk = testChunk;
+        }
+        position += part.length + separator.length;
+      }
+      if (currentChunk.trim()) {
+        chunks.push(
+          this.createChunk(
+            currentChunk.trim(),
+            chunks.length,
+            chunkStart,
+            opts
+          )
+        );
+      }
+    }
+    this.setOverlapInfo(chunks, opts.chunkOverlap * 4);
+    return Promise.resolve(chunks);
+  }
+  /**
+   * Get text for overlap from the end of a chunk
+   */
+  getOverlapText(text, overlapTokens, tokenCounter) {
+    if (overlapTokens <= 0) return "";
+    const sentences = text.split(/(?<=[.!?])\s+/);
+    let overlapText = "";
+    for (let i = sentences.length - 1; i >= 0; i--) {
+      const testText = sentences[i] + (overlapText ? " " + overlapText : "");
+      const testTokens = tokenCounter(testText);
+      if (testTokens > overlapTokens && overlapText) {
+        break;
+      }
+      overlapText = testText;
+    }
+    return overlapText;
+  }
+};
+function createFixedChunker() {
+  return new FixedChunker();
+}
+
+// src/chunking/RecursiveChunker.ts
+var DEFAULT_SEPARATORS = [
+  "\n\n",
+  // Paragraphs
+  "\n",
+  // Lines
+  ". ",
+  // Sentences
+  ", ",
+  // Clauses
+  " ",
+  // Words
+  ""
+  // Characters
+];
+var RecursiveChunker = class extends BaseChunker {
+  strategyType = "recursive";
+  async chunk(text, options) {
+    const opts = this.getOptions(options);
+    const separators = options?.separators ?? DEFAULT_SEPARATORS;
+    const keepSeparator = options?.keepSeparator ?? true;
+    const mergeSmall = options?.mergeSmallChunks ?? true;
+    const tokenCounter = opts.tokenCounter ?? defaultTokenCounter;
+    const texts = this.splitRecursively(
+      text,
+      separators,
+      opts.chunkSize,
+      keepSeparator,
+      tokenCounter
+    );
+    let position = 0;
+    let chunks = [];
+    for (let i = 0; i < texts.length; i++) {
+      const chunkText = texts[i].trim();
+      if (chunkText) {
+        chunks.push(this.createChunk(chunkText, i, position, opts));
+        position += texts[i].length;
+      }
+    }
+    if (mergeSmall) {
+      chunks = mergeSmallChunks(chunks, opts.minChunkSize, tokenCounter);
+    }
+    chunks = this.addOverlap(chunks, opts.chunkOverlap, tokenCounter);
+    return Promise.resolve(chunks);
+  }
+  /**
+   * Recursively split text
+   */
+  splitRecursively(text, separators, chunkSize, keepSeparator, tokenCounter) {
+    if (tokenCounter(text) <= chunkSize) {
+      return [text];
+    }
+    for (let i = 0; i < separators.length; i++) {
+      const separator = separators[i];
+      if (separator === "") {
+        return this.splitByChars(text, chunkSize, tokenCounter);
+      }
+      if (!text.includes(separator)) {
+        continue;
+      }
+      const splits = this.splitBySeparator(text, separator, keepSeparator);
+      const result = [];
+      for (const split of splits) {
+        if (tokenCounter(split) <= chunkSize) {
+          result.push(split);
+        } else {
+          const subSplits = this.splitRecursively(
+            split,
+            separators.slice(i + 1),
+            chunkSize,
+            keepSeparator,
+            tokenCounter
+          );
+          result.push(...subSplits);
+        }
+      }
+      return result;
+    }
+    return this.splitByChars(text, chunkSize, tokenCounter);
+  }
+  /**
+   * Split by separator
+   */
+  splitBySeparator(text, separator, keepSeparator) {
+    if (keepSeparator) {
+      const parts = text.split(separator);
+      return parts.map((part, i) => i < parts.length - 1 ? part + separator : part).filter((p) => p.trim());
+    } else {
+      return text.split(separator).filter((p) => p.trim());
+    }
+  }
+  /**
+   * Split by characters (last resort)
+   */
+  splitByChars(text, chunkSize, tokenCounter) {
+    const chunks = [];
+    let start = 0;
+    while (start < text.length) {
+      let end = start;
+      while (end < text.length && tokenCounter(text.slice(start, end)) < chunkSize) {
+        end++;
+      }
+      if (end < text.length) {
+        const lastSpace = text.lastIndexOf(" ", end);
+        if (lastSpace > start) {
+          end = lastSpace;
+        }
+      }
+      chunks.push(text.slice(start, end));
+      start = end;
+    }
+    return chunks.filter((c) => c.trim());
+  }
+  /**
+   * Add overlap between chunks
+   */
+  addOverlap(chunks, overlapTokens, tokenCounter) {
+    if (overlapTokens <= 0 || chunks.length <= 1) {
+      return chunks;
+    }
+    const result = [];
+    for (let i = 0; i < chunks.length; i++) {
+      let chunkText = chunks[i].text;
+      let startOffset = 0;
+      if (i > 0) {
+        const prevText = chunks[i - 1].text;
+        const overlapText = this.getEndOverlap(
+          prevText,
+          overlapTokens,
+          tokenCounter
+        );
+        if (overlapText) {
+          chunkText = overlapText + " " + chunkText;
+          startOffset = -overlapText.length - 1;
+        }
+      }
+      result.push({
+        ...chunks[i],
+        text: chunkText,
+        startPosition: chunks[i].startPosition + startOffset,
+        tokenCount: tokenCounter(chunkText),
+        charCount: chunkText.length,
+        overlapPrev: i > 0 ? overlapTokens : 0,
+        overlapNext: i < chunks.length - 1 ? overlapTokens : 0
+      });
+    }
+    return result;
+  }
+  /**
+   * Get overlap text from end of string
+   */
+  getEndOverlap(text, overlapTokens, tokenCounter) {
+    const words = text.split(/\s+/);
+    let overlap = "";
+    let tokens = 0;
+    for (let i = words.length - 1; i >= 0; i--) {
+      const testOverlap = words[i] + (overlap ? " " + overlap : "");
+      tokens = tokenCounter(testOverlap);
+      if (tokens > overlapTokens) {
+        break;
+      }
+      overlap = testOverlap;
+    }
+    return overlap;
+  }
+};
+function createRecursiveChunker() {
+  return new RecursiveChunker();
+}
+
+// src/chunking/MarkdownChunker.ts
+var MarkdownChunker = class extends BaseChunker {
+  strategyType = "markdown";
+  async chunk(text, options) {
+    const opts = this.getOptions(options);
+    const preserveHeaders = options?.preserveHeaders ?? true;
+    const includeHeaderHierarchy = options?.includeHeaderHierarchy ?? true;
+    const headingLevels = options?.headingLevels ?? [1, 2, 3, 4, 5, 6];
+    const splitCodeBlocks = options?.splitCodeBlocks ?? false;
+    const tokenCounter = opts.tokenCounter ?? defaultTokenCounter;
+    const sections = this.parseMarkdown(text, headingLevels);
+    let chunks = [];
+    for (const section of sections) {
+      const sectionChunks = await this.chunkSection(
+        section,
+        opts,
+        preserveHeaders,
+        includeHeaderHierarchy,
+        splitCodeBlocks,
+        tokenCounter
+      );
+      chunks.push(...sectionChunks);
+    }
+    chunks = mergeSmallChunks(chunks, opts.minChunkSize, tokenCounter);
+    return chunks.map((c, i) => ({ ...c, index: i }));
+  }
+  /**
+   * Parse markdown into sections
+   */
+  parseMarkdown(text, headingLevels) {
+    const sections = [];
+    const lines = text.split("\n");
+    const headingRegex = /^(#{1,6})\s+(.+)$/;
+    let currentSection = {
+      headingLevel: 0,
+      content: "",
+      startPosition: 0,
+      path: []
+    };
+    const headingStack = [];
+    let position = 0;
+    for (const line of lines) {
+      const headingMatch = line.match(headingRegex);
+      if (headingMatch) {
+        const level = headingMatch[1].length;
+        const headingText = headingMatch[2];
+        if (headingLevels.includes(level)) {
+          if (currentSection.content.trim()) {
+            sections.push({ ...currentSection });
+          }
+          while (headingStack.length > 0 && headingStack[headingStack.length - 1].level >= level) {
+            headingStack.pop();
+          }
+          headingStack.push({ level, text: headingText });
+          currentSection = {
+            heading: headingText,
+            headingLevel: level,
+            content: "",
+            startPosition: position,
+            path: headingStack.map((h) => h.text)
+          };
+        } else {
+          currentSection.content += line + "\n";
+        }
+      } else {
+        currentSection.content += line + "\n";
+      }
+      position += line.length + 1;
+    }
+    if (currentSection.content.trim() || currentSection.heading) {
+      sections.push(currentSection);
+    }
+    return sections;
+  }
+  /**
+   * Chunk a markdown section
+   */
+  async chunkSection(section, options, preserveHeaders, includeHeaderHierarchy, splitCodeBlocks, tokenCounter) {
+    const chunks = [];
+    let content = section.content;
+    let headerPrefix = "";
+    if (preserveHeaders && section.heading) {
+      if (includeHeaderHierarchy && section.path.length > 1) {
+        headerPrefix = section.path.map((h, i) => "#".repeat(i + 1) + " " + h).join("\n") + "\n\n";
+      } else {
+        headerPrefix = "#".repeat(section.headingLevel) + " " + section.heading + "\n\n";
+      }
+    }
+    const codeBlocks = [];
+    if (!splitCodeBlocks) {
+      const codeBlockRegex = /```[\s\S]*?```/g;
+      let match;
+      let blockIndex = 0;
+      while ((match = codeBlockRegex.exec(content)) !== null) {
+        const placeholder = `__CODE_BLOCK_${blockIndex}__`;
+        codeBlocks.push({ placeholder, content: match[0] });
+        content = content.replace(match[0], placeholder);
+        blockIndex++;
+      }
+    }
+    const fullContent = headerPrefix + content;
+    if (tokenCounter(fullContent) <= options.chunkSize) {
+      let finalContent = fullContent;
+      for (const block of codeBlocks) {
+        finalContent = finalContent.replace(block.placeholder, block.content);
+      }
+      chunks.push(
+        this.createChunk(
+          finalContent.trim(),
+          0,
+          section.startPosition,
+          options,
+          {
+            section: section.heading,
+            headingLevel: section.headingLevel,
+            path: section.path
+          }
+        )
+      );
+    } else {
+      const paragraphs = content.split(/\n\n+/);
+      let currentContent = headerPrefix;
+      let chunkStart = section.startPosition;
+      for (const paragraph of paragraphs) {
+        let para = paragraph;
+        for (const block of codeBlocks) {
+          para = para.replace(block.placeholder, block.content);
+        }
+        const testContent = currentContent + para + "\n\n";
+        if (tokenCounter(testContent) > options.chunkSize && currentContent !== headerPrefix) {
+          chunks.push(
+            this.createChunk(
+              currentContent.trim(),
+              chunks.length,
+              chunkStart,
+              options,
+              {
+                section: section.heading,
+                headingLevel: section.headingLevel,
+                path: section.path
+              }
+            )
+          );
+          currentContent = headerPrefix + para + "\n\n";
+          chunkStart = section.startPosition + content.indexOf(paragraph);
+        } else {
+          currentContent = testContent;
+        }
+      }
+      if (currentContent.trim() && currentContent !== headerPrefix.trim()) {
+        chunks.push(
+          this.createChunk(
+            currentContent.trim(),
+            chunks.length,
+            chunkStart,
+            options,
+            {
+              section: section.heading,
+              headingLevel: section.headingLevel,
+              path: section.path
+            }
+          )
+        );
+      }
+    }
+    return Promise.resolve(chunks);
+  }
+};
+function createMarkdownChunker() {
+  return new MarkdownChunker();
+}
+
+// src/chunking/CodeChunker.ts
+var LANGUAGE_PATTERNS = {
+  typescript: {
+    functionStart: /^(?:export\s+)?(?:async\s+)?function\s+(\w+)|^(?:export\s+)?const\s+(\w+)\s*=\s*(?:async\s+)?(?:\([^)]*\)|[^=])\s*=>/m,
+    classStart: /^(?:export\s+)?(?:abstract\s+)?class\s+(\w+)/m,
+    importPattern: /^import\s+.*?(?:from\s+['"][^'"]+['"]|['"][^'"]+['"])/gm,
+    commentPattern: /\/\*[\s\S]*?\*\/|\/\/.*/g,
+    blockEnd: /^}/m
+  },
+  javascript: {
+    functionStart: /^(?:export\s+)?(?:async\s+)?function\s+(\w+)|^(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?(?:\([^)]*\)|[^=])\s*=>/m,
+    classStart: /^(?:export\s+)?class\s+(\w+)/m,
+    importPattern: /^(?:import|require)\s*\(?\s*['"][^'"]+['"]\)?/gm,
+    commentPattern: /\/\*[\s\S]*?\*\/|\/\/.*/g,
+    blockEnd: /^}/m
+  },
+  python: {
+    functionStart: /^(?:async\s+)?def\s+(\w+)/m,
+    classStart: /^class\s+(\w+)/m,
+    importPattern: /^(?:from\s+\S+\s+)?import\s+.+$/gm,
+    commentPattern: /'''[\s\S]*?'''|"""[\s\S]*?"""|#.*/g,
+    blockEnd: /^(?=\S)/m
+    // Python uses indentation
+  },
+  go: {
+    functionStart: /^func\s+(?:\([^)]+\)\s+)?(\w+)/m,
+    classStart: /^type\s+(\w+)\s+struct/m,
+    importPattern: /^import\s+(?:\([\s\S]*?\)|"[^"]+")/gm,
+    commentPattern: /\/\*[\s\S]*?\*\/|\/\/.*/g,
+    blockEnd: /^}/m
+  },
+  rust: {
+    functionStart: /^(?:pub\s+)?(?:async\s+)?fn\s+(\w+)/m,
+    classStart: /^(?:pub\s+)?(?:struct|impl|trait)\s+(\w+)/m,
+    importPattern: /^use\s+.+;$/gm,
+    commentPattern: /\/\*[\s\S]*?\*\/|\/\/.*/g,
+    blockEnd: /^}/m
+  }
+};
+var CodeChunker = class extends BaseChunker {
+  strategyType = "code";
+  async chunk(text, options) {
+    const opts = this.getOptions(options);
+    const language = options?.language ?? this.detectLanguage(text);
+    const splitBy = options?.splitBy ?? "auto";
+    const includeComments = options?.includeComments ?? true;
+    const includeImports = options?.includeImports ?? true;
+    const tokenCounter = opts.tokenCounter ?? defaultTokenCounter;
+    const patterns = LANGUAGE_PATTERNS[language] ?? LANGUAGE_PATTERNS.typescript;
+    const blocks = this.parseCode(text, patterns, splitBy, includeComments);
+    let importBlock = "";
+    if (includeImports) {
+      const imports = text.match(patterns.importPattern);
+      if (imports) {
+        importBlock = imports.join("\n") + "\n\n";
+      }
+    }
+    let chunks = [];
+    for (const block of blocks) {
+      if (block.type === "import") continue;
+      const blockContent = includeImports && block.type !== "comment" ? importBlock + block.content : block.content;
+      if (tokenCounter(blockContent) <= opts.chunkSize) {
+        chunks.push(
+          this.createChunk(
+            blockContent.trim(),
+            chunks.length,
+            block.startPosition,
+            opts,
+            {
+              language,
+              blockType: block.type,
+              blockName: block.name
+            }
+          )
+        );
+      } else {
+        const subChunks = this.splitLargeBlock(
+          block,
+          importBlock,
+          opts,
+          tokenCounter,
+          language
+        );
+        chunks.push(...subChunks);
+      }
+    }
+    chunks = mergeSmallChunks(chunks, opts.minChunkSize, tokenCounter);
+    return Promise.resolve(chunks.map((c, i) => ({ ...c, index: i })));
+  }
+  /**
+   * Detect programming language
+   */
+  detectLanguage(text) {
+    if (text.includes("import type") || text.includes(": string") || text.includes("interface ")) {
+      return "typescript";
+    }
+    if (text.includes("def ") && text.includes(":")) {
+      return "python";
+    }
+    if (text.includes("func ") && text.includes("package ")) {
+      return "go";
+    }
+    if (text.includes("fn ") && (text.includes("let mut") || text.includes("pub fn"))) {
+      return "rust";
+    }
+    if (text.includes("const ") || text.includes("function ") || text.includes("require(")) {
+      return "javascript";
+    }
+    return "typescript";
+  }
+  /**
+   * Parse code into blocks
+   */
+  parseCode(text, patterns, splitBy, includeComments) {
+    const blocks = [];
+    const lines = text.split("\n");
+    let currentBlock = null;
+    let braceCount = 0;
+    let position = 0;
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i];
+      const lineStart = position;
+      position += line.length + 1;
+      if (splitBy === "function" || splitBy === "auto") {
+        const funcMatch = line.match(patterns.functionStart);
+        if (funcMatch) {
+          if (currentBlock) {
+            blocks.push(currentBlock);
+          }
+          currentBlock = {
+            type: "function",
+            name: funcMatch[1] || funcMatch[2],
+            content: line + "\n",
+            startPosition: lineStart
+          };
+          braceCount = (line.match(/{/g) || []).length - (line.match(/}/g) || []).length;
+          continue;
+        }
+      }
+      if (splitBy === "class" || splitBy === "auto") {
+        const classMatch = line.match(patterns.classStart);
+        if (classMatch) {
+          if (currentBlock) {
+            blocks.push(currentBlock);
+          }
+          currentBlock = {
+            type: "class",
+            name: classMatch[1],
+            content: line + "\n",
+            startPosition: lineStart
+          };
+          braceCount = (line.match(/{/g) || []).length - (line.match(/}/g) || []).length;
+          continue;
+        }
+      }
+      if (currentBlock) {
+        currentBlock.content += line + "\n";
+        braceCount += (line.match(/{/g) || []).length - (line.match(/}/g) || []).length;
+        if (braceCount <= 0) {
+          blocks.push(currentBlock);
+          currentBlock = null;
+          braceCount = 0;
+        }
+      } else {
+        if (line.trim()) {
+          currentBlock = {
+            type: "other",
+            content: line + "\n",
+            startPosition: lineStart
+          };
+        }
+      }
+    }
+    if (currentBlock) {
+      blocks.push(currentBlock);
+    }
+    if (!includeComments) {
+      return blocks.map((block) => ({
+        ...block,
+        content: block.content.replace(patterns.commentPattern, "")
+      }));
+    }
+    return blocks;
+  }
+  /**
+   * Split a large code block
+   */
+  splitLargeBlock(block, importBlock, options, tokenCounter, language) {
+    const chunks = [];
+    const lines = block.content.split("\n");
+    let currentContent = importBlock;
+    let chunkStart = block.startPosition;
+    for (const line of lines) {
+      const testContent = currentContent + line + "\n";
+      if (tokenCounter(testContent) > options.chunkSize && currentContent !== importBlock) {
+        chunks.push(
+          this.createChunk(
+            currentContent.trim(),
+            chunks.length,
+            chunkStart,
+            options,
+            {
+              language,
+              blockType: block.type,
+              blockName: block.name,
+              partial: true
+            }
+          )
+        );
+        currentContent = importBlock + line + "\n";
+        chunkStart = block.startPosition + block.content.indexOf(line);
+      } else {
+        currentContent = testContent;
+      }
+    }
+    if (currentContent.trim() && currentContent !== importBlock.trim()) {
+      chunks.push(
+        this.createChunk(
+          currentContent.trim(),
+          chunks.length,
+          chunkStart,
+          options,
+          {
+            language,
+            blockType: block.type,
+            blockName: block.name,
+            partial: chunks.length > 0
+          }
+        )
+      );
+    }
+    return chunks;
+  }
+};
+function createCodeChunker() {
+  return new CodeChunker();
+}
+
+// src/core/EmbeddingModel.ts
+var EmbeddingModel = class {
+  /**
+   * Get model dimensions
+   */
+  get dimensions() {
+    return this.info.dimensions;
+  }
+  /**
+   * Get max tokens
+   */
+  get maxTokens() {
+    return this.info.maxTokens;
+  }
+  /**
+   * Get max batch size
+   */
+  get maxBatchSize() {
+    return this.info.maxBatchSize;
+  }
+  /**
+   * Get model name
+   */
+  get name() {
+    return this.info.name;
+  }
+  /**
+   * Get provider name
+   */
+  get provider() {
+    return this.info.provider;
+  }
+  /**
+   * Count tokens in text (default implementation)
+   * Subclasses should override for accurate counting
+   */
+  countTokens(text) {
+    return Math.ceil(text.length / 4);
+  }
+  /**
+   * Check if text exceeds max tokens
+   */
+  exceedsMaxTokens(text) {
+    return this.countTokens(text) > this.maxTokens;
+  }
+  /**
+   * Truncate text to max tokens
+   */
+  truncateToMaxTokens(text) {
+    const tokens = this.countTokens(text);
+    if (tokens <= this.maxTokens) {
+      return text;
+    }
+    const ratio = this.maxTokens / tokens;
+    const targetLength = Math.floor(text.length * ratio * 0.95);
+    return text.slice(0, targetLength);
+  }
+  /**
+   * Calculate similarity between two vectors
+   */
+  static cosineSimilarity(a, b) {
+    if (a.length !== b.length) {
+      throw new Error(`Vector dimensions mismatch: ${a.length} vs ${b.length}`);
+    }
+    let dotProduct = 0;
+    let normA = 0;
+    let normB = 0;
+    for (let i = 0; i < a.length; i++) {
+      dotProduct += a[i] * b[i];
+      normA += a[i] * a[i];
+      normB += b[i] * b[i];
+    }
+    const magnitude = Math.sqrt(normA) * Math.sqrt(normB);
+    if (magnitude === 0) {
+      return 0;
+    }
+    return dotProduct / magnitude;
+  }
+  /**
+   * Calculate Euclidean distance between two vectors
+   */
+  static euclideanDistance(a, b) {
+    if (a.length !== b.length) {
+      throw new Error(`Vector dimensions mismatch: ${a.length} vs ${b.length}`);
+    }
+    let sum = 0;
+    for (let i = 0; i < a.length; i++) {
+      const diff = a[i] - b[i];
+      sum += diff * diff;
+    }
+    return Math.sqrt(sum);
+  }
+  /**
+   * Calculate dot product of two vectors
+   */
+  static dotProduct(a, b) {
+    if (a.length !== b.length) {
+      throw new Error(`Vector dimensions mismatch: ${a.length} vs ${b.length}`);
+    }
+    let result = 0;
+    for (let i = 0; i < a.length; i++) {
+      result += a[i] * b[i];
+    }
+    return result;
+  }
+  /**
+   * Normalize a vector to unit length
+   */
+  static normalize(vector) {
+    let norm = 0;
+    for (let i = 0; i < vector.length; i++) {
+      norm += vector[i] * vector[i];
+    }
+    norm = Math.sqrt(norm);
+    if (norm === 0) {
+      return vector.slice();
+    }
+    return vector.map((v) => v / norm);
+  }
+  /**
+   * Average multiple vectors
+   */
+  static average(vectors) {
+    if (vectors.length === 0) {
+      throw new Error("Cannot average empty array of vectors");
+    }
+    const dimensions = vectors[0].length;
+    const result = new Array(dimensions).fill(0);
+    for (const vector of vectors) {
+      if (vector.length !== dimensions) {
+        throw new Error(
+          `Vector dimensions mismatch: expected ${dimensions}, got ${vector.length}`
+        );
+      }
+      for (let i = 0; i < dimensions; i++) {
+        result[i] += vector[i];
+      }
+    }
+    for (let i = 0; i < dimensions; i++) {
+      result[i] /= vectors.length;
+    }
+    return result;
+  }
+  /**
+   * Weighted average of vectors
+   */
+  static weightedAverage(vectors, weights) {
+    if (vectors.length === 0) {
+      throw new Error("Cannot average empty array of vectors");
+    }
+    if (vectors.length !== weights.length) {
+      throw new Error("Vectors and weights arrays must have same length");
+    }
+    const dimensions = vectors[0].length;
+    const result = new Array(dimensions).fill(0);
+    let totalWeight = 0;
+    for (let j = 0; j < vectors.length; j++) {
+      const vector = vectors[j];
+      const weight = weights[j];
+      totalWeight += weight;
+      if (vector.length !== dimensions) {
+        throw new Error(
+          `Vector dimensions mismatch: expected ${dimensions}, got ${vector.length}`
+        );
+      }
+      for (let i = 0; i < dimensions; i++) {
+        result[i] += vector[i] * weight;
+      }
+    }
+    if (totalWeight === 0) {
+      throw new Error("Total weight cannot be zero");
+    }
+    for (let i = 0; i < dimensions; i++) {
+      result[i] /= totalWeight;
+    }
+    return result;
+  }
+};
+var ModelRegistry = class {
+  models = /* @__PURE__ */ new Map();
+  defaultModel = null;
+  /**
+   * Register a model
+   */
+  register(model, isDefault = false) {
+    const key = `${model.provider}:${model.name}`;
+    this.models.set(key, model);
+    if (isDefault || this.defaultModel === null) {
+      this.defaultModel = key;
+    }
+  }
+  /**
+   * Get a model by provider and name
+   */
+  get(provider, name) {
+    return this.models.get(`${provider}:${name}`);
+  }
+  /**
+   * Get model by key
+   */
+  getByKey(key) {
+    return this.models.get(key);
+  }
+  /**
+   * Get the default model
+   */
+  getDefault() {
+    if (this.defaultModel === null) {
+      return void 0;
+    }
+    return this.models.get(this.defaultModel);
+  }
+  /**
+   * Set default model
+   */
+  setDefault(provider, name) {
+    const key = `${provider}:${name}`;
+    if (!this.models.has(key)) {
+      throw new Error(`Model ${key} not found in registry`);
+    }
+    this.defaultModel = key;
+  }
+  /**
+   * List all registered models
+   */
+  list() {
+    return Array.from(this.models.values()).map((m) => m.info);
+  }
+  /**
+   * Check if a model is registered
+   */
+  has(provider, name) {
+    return this.models.has(`${provider}:${name}`);
+  }
+  /**
+   * Remove a model
+   */
+  remove(provider, name) {
+    const key = `${provider}:${name}`;
+    if (this.defaultModel === key) {
+      this.defaultModel = null;
+    }
+    return this.models.delete(key);
+  }
+  /**
+   * Clear all models
+   */
+  clear() {
+    this.models.clear();
+    this.defaultModel = null;
+  }
+};
+var modelRegistry = new ModelRegistry();
+
+// src/chunking/SemanticChunker.ts
+var SemanticChunker = class extends BaseChunker {
+  strategyType = "semantic";
+  async chunk(text, options) {
+    const opts = this.getOptions(options);
+    const similarityThreshold = options?.similarityThreshold ?? 0.5;
+    const breakpointPercentile = options?.breakpointPercentileThreshold ?? 95;
+    const bufferSize = options?.bufferSize ?? 1;
+    const embeddingFn = options?.embeddingFn;
+    const tokenCounter = opts.tokenCounter ?? defaultTokenCounter;
+    const sentences = this.splitSentences(text);
+    if (sentences.length === 0) {
+      return [];
+    }
+    if (!embeddingFn) {
+      return this.fallbackChunk(sentences, opts, tokenCounter);
+    }
+    const sentenceTexts = sentences.map((s) => s.text);
+    const embeddings = await embeddingFn(sentenceTexts);
+    const sentencesWithEmbeddings = sentences.map(
+      (s, i) => ({
+        ...s,
+        embedding: embeddings[i]
+      })
+    );
+    const distances = this.calculateDistances(
+      sentencesWithEmbeddings,
+      bufferSize
+    );
+    const breakpoints = this.findBreakpoints(
+      distances,
+      breakpointPercentile,
+      similarityThreshold
+    );
+    let chunks = [];
+    let chunkStart = 0;
+    let chunkText = "";
+    let chunkPosition = sentences[0]?.position ?? 0;
+    for (let i = 0; i < sentences.length; i++) {
+      chunkText += (chunkText ? " " : "") + sentences[i].text;
+      if (breakpoints.includes(i) || i === sentences.length - 1) {
+        if (chunkText.trim()) {
+          chunks.push(
+            this.createChunk(
+              chunkText.trim(),
+              chunks.length,
+              chunkPosition,
+              opts,
+              {
+                boundaryType: "semantic",
+                sentenceCount: i - chunkStart + 1
+              }
+            )
+          );
+        }
+        if (i < sentences.length - 1) {
+          chunkStart = i + 1;
+          chunkText = "";
+          chunkPosition = sentences[i + 1].position;
+        }
+      }
+    }
+    chunks = mergeSmallChunks(chunks, opts.minChunkSize, tokenCounter);
+    chunks = this.splitLargeChunks(chunks, opts.maxChunkSize, tokenCounter);
+    return chunks;
+  }
+  /**
+   * Split text into sentences
+   */
+  splitSentences(text) {
+    const sentenceRegex = /[^.!?]+[.!?]+/g;
+    const sentences = [];
+    let match;
+    while ((match = sentenceRegex.exec(text)) !== null) {
+      const sentence = match[0].trim();
+      if (sentence) {
+        sentences.push({
+          text: sentence,
+          position: match.index
+        });
+      }
+    }
+    if (sentences.length === 0 && text.trim()) {
+      sentences.push({
+        text: text.trim(),
+        position: 0
+      });
+    }
+    return sentences;
+  }
+  /**
+   * Calculate distances between adjacent sentences
+   */
+  calculateDistances(sentences, bufferSize) {
+    const distances = [];
+    for (let i = 0; i < sentences.length - 1; i++) {
+      const leftStart = Math.max(0, i - bufferSize + 1);
+      const rightEnd = Math.min(sentences.length, i + bufferSize + 1);
+      const leftEmbeddings = sentences.slice(leftStart, i + 1).map((s) => s.embedding).filter((e) => e !== void 0);
+      const rightEmbeddings = sentences.slice(i + 1, rightEnd).map((s) => s.embedding).filter((e) => e !== void 0);
+      if (leftEmbeddings.length > 0 && rightEmbeddings.length > 0) {
+        const leftAvg = EmbeddingModel.average(leftEmbeddings);
+        const rightAvg = EmbeddingModel.average(rightEmbeddings);
+        const similarity = EmbeddingModel.cosineSimilarity(leftAvg, rightAvg);
+        distances.push(1 - similarity);
+      } else {
+        distances.push(0);
+      }
+    }
+    return distances;
+  }
+  /**
+   * Find breakpoints based on distance threshold
+   */
+  findBreakpoints(distances, percentile, minThreshold) {
+    if (distances.length === 0) return [];
+    const sortedDistances = [...distances].sort((a, b) => a - b);
+    const percentileIndex = Math.floor(
+      percentile / 100 * sortedDistances.length
+    );
+    const percentileThreshold = sortedDistances[percentileIndex] ?? sortedDistances[sortedDistances.length - 1];
+    const threshold = Math.max(percentileThreshold, 1 - minThreshold);
+    const breakpoints = [];
+    for (let i = 0; i < distances.length; i++) {
+      if (distances[i] >= threshold) {
+        breakpoints.push(i);
+      }
+    }
+    return breakpoints;
+  }
+  /**
+   * Fallback chunking when no embedding function available
+   */
+  fallbackChunk(sentences, options, tokenCounter) {
+    const chunks = [];
+    let currentText = "";
+    let chunkPosition = sentences[0]?.position ?? 0;
+    for (const sentence of sentences) {
+      const testText = currentText ? currentText + " " + sentence.text : sentence.text;
+      if (tokenCounter(testText) > options.chunkSize && currentText) {
+        chunks.push(
+          this.createChunk(
+            currentText.trim(),
+            chunks.length,
+            chunkPosition,
+            options,
+            { boundaryType: "sentence" }
+          )
+        );
+        currentText = sentence.text;
+        chunkPosition = sentence.position;
+      } else {
+        currentText = testText;
+      }
+    }
+    if (currentText.trim()) {
+      chunks.push(
+        this.createChunk(
+          currentText.trim(),
+          chunks.length,
+          chunkPosition,
+          options,
+          { boundaryType: "sentence" }
+        )
+      );
+    }
+    return chunks;
+  }
+  /**
+   * Split chunks that are too large
+   */
+  splitLargeChunks(chunks, maxTokens, tokenCounter) {
+    const result = [];
+    for (const chunk2 of chunks) {
+      if (chunk2.tokenCount <= maxTokens) {
+        result.push(chunk2);
+        continue;
+      }
+      const sentences = this.splitSentences(chunk2.text);
+      let currentText = "";
+      let currentStart = chunk2.startPosition;
+      for (const sentence of sentences) {
+        const testText = currentText ? currentText + " " + sentence.text : sentence.text;
+        if (tokenCounter(testText) > maxTokens && currentText) {
+          result.push({
+            ...chunk2,
+            id: chunk2.id + "_" + result.length,
+            text: currentText.trim(),
+            startPosition: currentStart,
+            endPosition: currentStart + currentText.length,
+            tokenCount: tokenCounter(currentText),
+            charCount: currentText.length,
+            index: result.length
+          });
+          currentText = sentence.text;
+          currentStart = chunk2.startPosition + sentence.position;
+        } else {
+          currentText = testText;
+        }
+      }
+      if (currentText.trim()) {
+        result.push({
+          ...chunk2,
+          id: chunk2.id + "_" + result.length,
+          text: currentText.trim(),
+          startPosition: currentStart,
+          endPosition: currentStart + currentText.length,
+          tokenCount: tokenCounter(currentText),
+          charCount: currentText.length,
+          index: result.length
+        });
+      }
+    }
+    return result;
+  }
+};
+function createSemanticChunker() {
+  return new SemanticChunker();
+}
+
+// src/chunking/index.ts
+function createChunker(strategy) {
+  switch (strategy) {
+    case "fixed":
+      return new FixedChunker();
+    case "recursive":
+      return new RecursiveChunker();
+    case "markdown":
+      return new MarkdownChunker();
+    case "code":
+      return new CodeChunker();
+    case "semantic":
+      return new SemanticChunker();
+    case "sentence":
+      return new FixedChunker();
+    case "paragraph":
+      return new FixedChunker();
+    default:
+      return new RecursiveChunker();
+  }
+}
+async function chunk(text, strategy = "recursive", options) {
+  const chunker = createChunker(strategy);
+  return chunker.chunk(text, options);
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  BaseChunker,
+  CodeChunker,
+  FixedChunker,
+  MarkdownChunker,
+  RecursiveChunker,
+  SemanticChunker,
+  chunk,
+  createChunker,
+  createCodeChunker,
+  createFixedChunker,
+  createMarkdownChunker,
+  createRecursiveChunker,
+  createSemanticChunker,
+  defaultTokenCounter,
+  mergeSmallChunks,
+  splitLargeChunks
+});