@chatbot-packages/rag 0.1.0

package/dist/chunking/index.d.ts

@@ -0,0 +1,51 @@
+import { e as ChunkingOptions, j as ExtractedDocument, d as ChunkResult } from '../types-CjnplPJD.js';
+
+/**
+ * Header-Aware Chunker
+ *
+ * Splits documents into chunks while respecting heading boundaries.
+ * Ported from the Python implementation in Gen21AIHelpAndQNA.
+ */
+
+interface HeaderAwareChunkerOptions extends ChunkingOptions {
+    /** Document ID to associate chunks with */
+    documentId?: string;
+}
+declare class HeaderAwareChunker {
+    private options;
+    constructor(options?: ChunkingOptions);
+    /**
+     * Chunk a document into smaller pieces
+     */
+    chunk(document: ExtractedDocument, documentId?: string): ChunkResult;
+    /**
+     * Split content by heading boundaries
+     */
+    private splitBySections;
+    /**
+     * Parse a heading line
+     */
+    private parseHeading;
+    /**
+     * Chunk a single section
+     */
+    private chunkSection;
+    /**
+     * Split a large paragraph into sentence-based chunks
+     */
+    private splitLargeParagraph;
+    /**
+     * Get overlap text from previous chunk
+     */
+    private getOverlapText;
+    /**
+     * Create a chunk object
+     */
+    private createChunk;
+    /**
+     * Estimate token count (rough approximation: ~4 chars per token)
+     */
+    private estimateTokens;
+}
+
+export { HeaderAwareChunker, type HeaderAwareChunkerOptions };

package/dist/chunking/index.js

@@ -0,0 +1,248 @@
+// src/chunking/header-aware-chunker.ts
+import { generateId } from "@chatbot-packages/utils";
+var HeaderAwareChunker = class {
+  options;
+  constructor(options) {
+    this.options = {
+      chunkSize: 512,
+      chunkOverlap: 50,
+      minChunkSize: 100,
+      maxChunkSize: 800,
+      respectHeadings: true,
+      splitOnHeadings: [1, 2],
+      ...options
+    };
+  }
+  /**
+   * Chunk a document into smaller pieces
+   */
+  chunk(document, documentId) {
+    const docId = documentId || generateId("doc");
+    const { content } = document;
+    if (!content.trim()) {
+      return {
+        chunks: [],
+        stats: { totalChunks: 0, avgChunkSize: 0, minChunkSize: 0, maxChunkSize: 0 }
+      };
+    }
+    const chunks = [];
+    const sections = this.splitBySections(content);
+    for (const section of sections) {
+      const sectionChunks = this.chunkSection(section, docId, document.path);
+      chunks.push(...sectionChunks);
+    }
+    const sizes = chunks.map((c) => c.text.length);
+    const stats = {
+      totalChunks: chunks.length,
+      avgChunkSize: sizes.length > 0 ? Math.round(sizes.reduce((a, b) => a + b, 0) / sizes.length) : 0,
+      minChunkSize: sizes.length > 0 ? Math.min(...sizes) : 0,
+      maxChunkSize: sizes.length > 0 ? Math.max(...sizes) : 0
+    };
+    return { chunks, stats };
+  }
+  /**
+   * Split content by heading boundaries
+   */
+  splitBySections(content) {
+    const lines = content.split("\n");
+    const sections = [];
+    let currentContext = { sectionPath: "" };
+    let currentText = [];
+    for (const line of lines) {
+      const heading = this.parseHeading(line);
+      if (heading && this.options.splitOnHeadings.includes(heading.level)) {
+        if (currentText.length > 0) {
+          sections.push({
+            text: currentText.join("\n").trim(),
+            context: { ...currentContext }
+          });
+          currentText = [];
+        }
+        if (heading.level === 1) {
+          currentContext = {
+            h1: heading.text,
+            sectionPath: heading.text
+          };
+        } else if (heading.level === 2) {
+          currentContext = {
+            ...currentContext,
+            h2: heading.text,
+            h3: void 0,
+            sectionPath: currentContext.h1 ? `${currentContext.h1} > ${heading.text}` : heading.text
+          };
+        } else if (heading.level === 3) {
+          currentContext = {
+            ...currentContext,
+            h3: heading.text,
+            sectionPath: currentContext.h2 ? `${currentContext.sectionPath} > ${heading.text}` : heading.text
+          };
+        }
+        currentText.push(line);
+      } else {
+        currentText.push(line);
+      }
+    }
+    if (currentText.length > 0) {
+      sections.push({
+        text: currentText.join("\n").trim(),
+        context: { ...currentContext }
+      });
+    }
+    return sections;
+  }
+  /**
+   * Parse a heading line
+   */
+  parseHeading(line) {
+    const match = line.match(/^(#{1,6})\s+(.+)$/);
+    if (match) {
+      return {
+        level: match[1].length,
+        text: match[2].trim()
+      };
+    }
+    return null;
+  }
+  /**
+   * Chunk a single section
+   */
+  chunkSection(section, documentId, sourcePath) {
+    const { text, context } = section;
+    const chunks = [];
+    if (this.estimateTokens(text) <= this.options.maxChunkSize) {
+      if (this.estimateTokens(text) >= this.options.minChunkSize) {
+        chunks.push(this.createChunk(text, context, documentId, sourcePath, 0));
+      }
+      return chunks;
+    }
+    const paragraphs = text.split(/\n\n+/);
+    let currentChunk = [];
+    let currentTokens = 0;
+    for (const para of paragraphs) {
+      const paraTokens = this.estimateTokens(para);
+      if (paraTokens > this.options.maxChunkSize) {
+        if (currentChunk.length > 0) {
+          chunks.push(
+            this.createChunk(
+              currentChunk.join("\n\n"),
+              context,
+              documentId,
+              sourcePath,
+              chunks.length
+            )
+          );
+          currentChunk = [];
+          currentTokens = 0;
+        }
+        const sentenceChunks = this.splitLargeParagraph(para);
+        for (const sentenceChunk of sentenceChunks) {
+          chunks.push(
+            this.createChunk(sentenceChunk, context, documentId, sourcePath, chunks.length)
+          );
+        }
+        continue;
+      }
+      if (currentTokens + paraTokens > this.options.chunkSize) {
+        if (currentChunk.length > 0) {
+          chunks.push(
+            this.createChunk(
+              currentChunk.join("\n\n"),
+              context,
+              documentId,
+              sourcePath,
+              chunks.length
+            )
+          );
+          const overlapText = this.getOverlapText(currentChunk);
+          currentChunk = overlapText ? [overlapText, para] : [para];
+          currentTokens = this.estimateTokens(currentChunk.join("\n\n"));
+        } else {
+          currentChunk = [para];
+          currentTokens = paraTokens;
+        }
+      } else {
+        currentChunk.push(para);
+        currentTokens += paraTokens;
+      }
+    }
+    if (currentChunk.length > 0 && currentTokens >= this.options.minChunkSize) {
+      chunks.push(
+        this.createChunk(currentChunk.join("\n\n"), context, documentId, sourcePath, chunks.length)
+      );
+    }
+    return chunks;
+  }
+  /**
+   * Split a large paragraph into sentence-based chunks
+   */
+  splitLargeParagraph(paragraph) {
+    const sentences = paragraph.match(/[^.!?]+[.!?]+/g) || [paragraph];
+    const chunks = [];
+    let currentChunk = [];
+    let currentTokens = 0;
+    for (const sentence of sentences) {
+      const sentenceTokens = this.estimateTokens(sentence);
+      if (currentTokens + sentenceTokens > this.options.chunkSize && currentChunk.length > 0) {
+        chunks.push(currentChunk.join(" ").trim());
+        currentChunk = [];
+        currentTokens = 0;
+      }
+      currentChunk.push(sentence.trim());
+      currentTokens += sentenceTokens;
+    }
+    if (currentChunk.length > 0) {
+      chunks.push(currentChunk.join(" ").trim());
+    }
+    return chunks;
+  }
+  /**
+   * Get overlap text from previous chunk
+   */
+  getOverlapText(previousChunk) {
+    if (!this.options.chunkOverlap || previousChunk.length === 0) {
+      return null;
+    }
+    const reversed = [...previousChunk].reverse();
+    const overlapParts = [];
+    let tokens = 0;
+    for (const part of reversed) {
+      const partTokens = this.estimateTokens(part);
+      if (tokens + partTokens > this.options.chunkOverlap) {
+        break;
+      }
+      overlapParts.unshift(part);
+      tokens += partTokens;
+    }
+    return overlapParts.length > 0 ? overlapParts.join("\n\n") : null;
+  }
+  /**
+   * Create a chunk object
+   */
+  createChunk(text, context, documentId, sourcePath, index) {
+    const metadata = {
+      sectionPath: context.sectionPath,
+      headingH1: context.h1,
+      headingH2: context.h2,
+      headingH3: context.h3,
+      sourcePath,
+      chunkIndex: index
+    };
+    return {
+      id: generateId("chunk"),
+      documentId,
+      text: text.trim(),
+      metadata,
+      createdAt: /* @__PURE__ */ new Date()
+    };
+  }
+  /**
+   * Estimate token count (rough approximation: ~4 chars per token)
+   */
+  estimateTokens(text) {
+    return Math.ceil(text.length / 4);
+  }
+};
+export {
+  HeaderAwareChunker
+};
+//# sourceMappingURL=index.js.map

package/dist/chunking/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/chunking/header-aware-chunker.ts"],"sourcesContent":["/**\n * Header-Aware Chunker\n *\n * Splits documents into chunks while respecting heading boundaries.\n * Ported from the Python implementation in Gen21AIHelpAndQNA.\n */\n\nimport { generateId } from '@chatbot-packages/utils';\nimport type {\n  ChunkingOptions,\n  ChunkResult,\n  DocumentChunk,\n  ChunkMetadata,\n  ExtractedDocument,\n} from '../types.js';\n\nexport interface HeaderAwareChunkerOptions extends ChunkingOptions {\n  /** Document ID to associate chunks with */\n  documentId?: string;\n}\n\ninterface HeadingContext {\n  h1?: string;\n  h2?: string;\n  h3?: string;\n  sectionPath: string;\n}\n\nexport class HeaderAwareChunker {\n  private options: Required<Omit<HeaderAwareChunkerOptions, 'documentId'>>;\n\n  constructor(options?: ChunkingOptions) {\n    this.options = {\n      chunkSize: 512,\n      chunkOverlap: 50,\n      minChunkSize: 100,\n      maxChunkSize: 800,\n      respectHeadings: true,\n      splitOnHeadings: [1, 2],\n      ...options,\n    };\n  }\n\n  /**\n   * Chunk a document into smaller pieces\n   */\n  chunk(document: ExtractedDocument, documentId?: string): ChunkResult {\n    const docId = documentId || generateId('doc');\n    const { content } = document;\n\n    if (!content.trim()) {\n      return {\n        chunks: [],\n        stats: { totalChunks: 0, avgChunkSize: 0, minChunkSize: 0, maxChunkSize: 0 },\n      };\n    }\n\n    const chunks: DocumentChunk[] = [];\n    const sections = this.splitBySections(content);\n\n    for (const section of sections) {\n      const sectionChunks = this.chunkSection(section, docId, document.path);\n      chunks.push(...sectionChunks);\n    }\n\n    // Calculate stats\n    const sizes = chunks.map((c) => c.text.length);\n    const stats = {\n      totalChunks: chunks.length,\n      avgChunkSize: sizes.length > 0 ? Math.round(sizes.reduce((a, b) => a + b, 0) / sizes.length) : 0,\n      minChunkSize: sizes.length > 0 ? Math.min(...sizes) : 0,\n      maxChunkSize: sizes.length > 0 ? Math.max(...sizes) : 0,\n    };\n\n    return { chunks, stats };\n  }\n\n  /**\n   * Split content by heading boundaries\n   */\n  private splitBySections(content: string): Array<{ text: string; context: HeadingContext }> {\n    const lines = content.split('\\n');\n    const sections: Array<{ text: string; context: HeadingContext }> = [];\n\n    let currentContext: HeadingContext = { sectionPath: '' };\n    let currentText: string[] = [];\n\n    for (const line of lines) {\n      const heading = this.parseHeading(line);\n\n      if (heading && this.options.splitOnHeadings.includes(heading.level)) {\n        // Save current section\n        if (currentText.length > 0) {\n          sections.push({\n            text: currentText.join('\\n').trim(),\n            context: { ...currentContext },\n          });\n          currentText = [];\n        }\n\n        // Update context based on heading level\n        if (heading.level === 1) {\n          currentContext = {\n            h1: heading.text,\n            sectionPath: heading.text,\n          };\n        } else if (heading.level === 2) {\n          currentContext = {\n            ...currentContext,\n            h2: heading.text,\n            h3: undefined,\n            sectionPath: currentContext.h1\n              ? `${currentContext.h1} > ${heading.text}`\n              : heading.text,\n          };\n        } else if (heading.level === 3) {\n          currentContext = {\n            ...currentContext,\n            h3: heading.text,\n            sectionPath: currentContext.h2\n              ? `${currentContext.sectionPath} > ${heading.text}`\n              : heading.text,\n          };\n        }\n\n        currentText.push(line);\n      } else {\n        currentText.push(line);\n      }\n    }\n\n    // Don't forget the last section\n    if (currentText.length > 0) {\n      sections.push({\n        text: currentText.join('\\n').trim(),\n        context: { ...currentContext },\n      });\n    }\n\n    return sections;\n  }\n\n  /**\n   * Parse a heading line\n   */\n  private parseHeading(line: string): { level: number; text: string } | null {\n    // Markdown-style headings: # H1, ## H2, ### H3\n    const match = line.match(/^(#{1,6})\\s+(.+)$/);\n    if (match) {\n      return {\n        level: match[1].length,\n        text: match[2].trim(),\n      };\n    }\n    return null;\n  }\n\n  /**\n   * Chunk a single section\n   */\n  private chunkSection(\n    section: { text: string; context: HeadingContext },\n    documentId: string,\n    sourcePath: string\n  ): DocumentChunk[] {\n    const { text, context } = section;\n    const chunks: DocumentChunk[] = [];\n\n    // If section is small enough, keep as single chunk\n    if (this.estimateTokens(text) <= this.options.maxChunkSize) {\n      if (this.estimateTokens(text) >= this.options.minChunkSize) {\n        chunks.push(this.createChunk(text, context, documentId, sourcePath, 0));\n      }\n      return chunks;\n    }\n\n    // Split into paragraphs\n    const paragraphs = text.split(/\\n\\n+/);\n    let currentChunk: string[] = [];\n    let currentTokens = 0;\n\n    for (const para of paragraphs) {\n      const paraTokens = this.estimateTokens(para);\n\n      // If single paragraph is too large, split by sentences\n      if (paraTokens > this.options.maxChunkSize) {\n        // Save current chunk first\n        if (currentChunk.length > 0) {\n          chunks.push(\n            this.createChunk(\n              currentChunk.join('\\n\\n'),\n              context,\n              documentId,\n              sourcePath,\n              chunks.length\n            )\n          );\n          currentChunk = [];\n          currentTokens = 0;\n        }\n\n        // Split large paragraph by sentences\n        const sentenceChunks = this.splitLargeParagraph(para);\n        for (const sentenceChunk of sentenceChunks) {\n          chunks.push(\n            this.createChunk(sentenceChunk, context, documentId, sourcePath, chunks.length)\n          );\n        }\n        continue;\n      }\n\n      // Check if adding paragraph exceeds chunk size\n      if (currentTokens + paraTokens > this.options.chunkSize) {\n        // Save current chunk\n        if (currentChunk.length > 0) {\n          chunks.push(\n            this.createChunk(\n              currentChunk.join('\\n\\n'),\n              context,\n              documentId,\n              sourcePath,\n              chunks.length\n            )\n          );\n\n          // Start new chunk with overlap\n          const overlapText = this.getOverlapText(currentChunk);\n          currentChunk = overlapText ? [overlapText, para] : [para];\n          currentTokens = this.estimateTokens(currentChunk.join('\\n\\n'));\n        } else {\n          currentChunk = [para];\n          currentTokens = paraTokens;\n        }\n      } else {\n        currentChunk.push(para);\n        currentTokens += paraTokens;\n      }\n    }\n\n    // Don't forget the last chunk\n    if (currentChunk.length > 0 && currentTokens >= this.options.minChunkSize) {\n      chunks.push(\n        this.createChunk(currentChunk.join('\\n\\n'), context, documentId, sourcePath, chunks.length)\n      );\n    }\n\n    return chunks;\n  }\n\n  /**\n   * Split a large paragraph into sentence-based chunks\n   */\n  private splitLargeParagraph(paragraph: string): string[] {\n    const sentences = paragraph.match(/[^.!?]+[.!?]+/g) || [paragraph];\n    const chunks: string[] = [];\n    let currentChunk: string[] = [];\n    let currentTokens = 0;\n\n    for (const sentence of sentences) {\n      const sentenceTokens = this.estimateTokens(sentence);\n\n      if (currentTokens + sentenceTokens > this.options.chunkSize && currentChunk.length > 0) {\n        chunks.push(currentChunk.join(' ').trim());\n        currentChunk = [];\n        currentTokens = 0;\n      }\n\n      currentChunk.push(sentence.trim());\n      currentTokens += sentenceTokens;\n    }\n\n    if (currentChunk.length > 0) {\n      chunks.push(currentChunk.join(' ').trim());\n    }\n\n    return chunks;\n  }\n\n  /**\n   * Get overlap text from previous chunk\n   */\n  private getOverlapText(previousChunk: string[]): string | null {\n    if (!this.options.chunkOverlap || previousChunk.length === 0) {\n      return null;\n    }\n\n    // Take last paragraph(s) that fit within overlap size\n    const reversed = [...previousChunk].reverse();\n    const overlapParts: string[] = [];\n    let tokens = 0;\n\n    for (const part of reversed) {\n      const partTokens = this.estimateTokens(part);\n      if (tokens + partTokens > this.options.chunkOverlap) {\n        break;\n      }\n      overlapParts.unshift(part);\n      tokens += partTokens;\n    }\n\n    return overlapParts.length > 0 ? overlapParts.join('\\n\\n') : null;\n  }\n\n  /**\n   * Create a chunk object\n   */\n  private createChunk(\n    text: string,\n    context: HeadingContext,\n    documentId: string,\n    sourcePath: string,\n    index: number\n  ): DocumentChunk {\n    const metadata: ChunkMetadata = {\n      sectionPath: context.sectionPath,\n      headingH1: context.h1,\n      headingH2: context.h2,\n      headingH3: context.h3,\n      sourcePath,\n      chunkIndex: index,\n    };\n\n    return {\n      id: generateId('chunk'),\n      documentId,\n      text: text.trim(),\n      metadata,\n      createdAt: new Date(),\n    };\n  }\n\n  /**\n   * Estimate token count (rough approximation: ~4 chars per token)\n   */\n  private estimateTokens(text: string): number {\n    return Math.ceil(text.length / 4);\n  }\n}\n"],"mappings":";AAOA,SAAS,kBAAkB;AAqBpB,IAAM,qBAAN,MAAyB;AAAA,EACtB;AAAA,EAER,YAAY,SAA2B;AACrC,SAAK,UAAU;AAAA,MACb,WAAW;AAAA,MACX,cAAc;AAAA,MACd,cAAc;AAAA,MACd,cAAc;AAAA,MACd,iBAAiB;AAAA,MACjB,iBAAiB,CAAC,GAAG,CAAC;AAAA,MACtB,GAAG;AAAA,IACL;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,UAA6B,YAAkC;AACnE,UAAM,QAAQ,cAAc,WAAW,KAAK;AAC5C,UAAM,EAAE,QAAQ,IAAI;AAEpB,QAAI,CAAC,QAAQ,KAAK,GAAG;AACnB,aAAO;AAAA,QACL,QAAQ,CAAC;AAAA,QACT,OAAO,EAAE,aAAa,GAAG,cAAc,GAAG,cAAc,GAAG,cAAc,EAAE;AAAA,MAC7E;AAAA,IACF;AAEA,UAAM,SAA0B,CAAC;AACjC,UAAM,WAAW,KAAK,gBAAgB,OAAO;AAE7C,eAAW,WAAW,UAAU;AAC9B,YAAM,gBAAgB,KAAK,aAAa,SAAS,OAAO,SAAS,IAAI;AACrE,aAAO,KAAK,GAAG,aAAa;AAAA,IAC9B;AAGA,UAAM,QAAQ,OAAO,IAAI,CAAC,MAAM,EAAE,KAAK,MAAM;AAC7C,UAAM,QAAQ;AAAA,MACZ,aAAa,OAAO;AAAA,MACpB,cAAc,MAAM,SAAS,IAAI,KAAK,MAAM,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,MAAM,MAAM,IAAI;AAAA,MAC/F,cAAc,MAAM,SAAS,IAAI,KAAK,IAAI,GAAG,KAAK,IAAI;AAAA,MACtD,cAAc,MAAM,SAAS,IAAI,KAAK,IAAI,GAAG,KAAK,IAAI;AAAA,IACxD;AAEA,WAAO,EAAE,QAAQ,MAAM;AAAA,EACzB;AAAA;AAAA;AAAA;AAAA,EAKQ,gBAAgB,SAAmE;AACzF,UAAM,QAAQ,QAAQ,MAAM,IAAI;AAChC,UAAM,WAA6D,CAAC;AAEpE,QAAI,iBAAiC,EAAE,aAAa,GAAG;AACvD,QAAI,cAAwB,CAAC;AAE7B,eAAW,QAAQ,OAAO;AACxB,YAAM,UAAU,KAAK,aAAa,IAAI;AAEtC,UAAI,WAAW,KAAK,QAAQ,gBAAgB,SAAS,QAAQ,KAAK,GAAG;AAEnE,YAAI,YAAY,SAAS,GAAG;AAC1B,mBAAS,KAAK;AAAA,YACZ,MAAM,YAAY,KAAK,IAAI,EAAE,KAAK;AAAA,YAClC,SAAS,EAAE,GAAG,eAAe;AAAA,UAC/B,CAAC;AACD,wBAAc,CAAC;AAAA,QACjB;AAGA,YAAI,QAAQ,UAAU,GAAG;AACvB,2BAAiB;AAAA,YACf,IAAI,QAAQ;AAAA,YACZ,aAAa,QAAQ;AAAA,UACvB;AAAA,QACF,WAAW,QAAQ,UAAU,GAAG;AAC9B,2BAAiB;AAAA,YACf,GAAG;AAAA,YACH,IAAI,QAAQ;AAAA,YACZ,IAAI;AAAA,YACJ,aAAa,eAAe,KACxB,GAAG,eAAe,EAAE,MAAM,QAAQ,IAAI,KACtC,QAAQ;AAAA,UACd;AAAA,QACF,WAAW,QAAQ,UAAU,GAAG;AAC9B,2BAAiB;AAAA,YACf,GAAG;AAAA,YACH,IAAI,QAAQ;AAAA,YACZ,aAAa,eAAe,KACxB,GAAG,eAAe,WAAW,MAAM,QAAQ,IAAI,KAC/C,QAAQ;AAAA,UACd;AAAA,QACF;AAEA,oBAAY,KAAK,IAAI;AAAA,MACvB,OAAO;AACL,oBAAY,KAAK,IAAI;AAAA,MACvB;AAAA,IACF;AAGA,QAAI,YAAY,SAAS,GAAG;AAC1B,eAAS,KAAK;AAAA,QACZ,MAAM,YAAY,KAAK,IAAI,EAAE,KAAK;AAAA,QAClC,SAAS,EAAE,GAAG,eAAe;AAAA,MAC/B,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKQ,aAAa,MAAsD;AAEzE,UAAM,QAAQ,KAAK,MAAM,mBAAmB;AAC5C,QAAI,OAAO;AACT,aAAO;AAAA,QACL,OAAO,MAAM,CAAC,EAAE;AAAA,QAChB,MAAM,MAAM,CAAC,EAAE,KAAK;AAAA,MACtB;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKQ,aACN,SACA,YACA,YACiB;AACjB,UAAM,EAAE,MAAM,QAAQ,IAAI;AAC1B,UAAM,SAA0B,CAAC;AAGjC,QAAI,KAAK,eAAe,IAAI,KAAK,KAAK,QAAQ,cAAc;AAC1D,UAAI,KAAK,eAAe,IAAI,KAAK,KAAK,QAAQ,cAAc;AAC1D,eAAO,KAAK,KAAK,YAAY,MAAM,SAAS,YAAY,YAAY,CAAC,CAAC;AAAA,MACxE;AACA,aAAO;AAAA,IACT;AAGA,UAAM,aAAa,KAAK,MAAM,OAAO;AACrC,QAAI,eAAyB,CAAC;AAC9B,QAAI,gBAAgB;AAEpB,eAAW,QAAQ,YAAY;AAC7B,YAAM,aAAa,KAAK,eAAe,IAAI;AAG3C,UAAI,aAAa,KAAK,QAAQ,cAAc;AAE1C,YAAI,aAAa,SAAS,GAAG;AAC3B,iBAAO;AAAA,YACL,KAAK;AAAA,cACH,aAAa,KAAK,MAAM;AAAA,cACxB;AAAA,cACA;AAAA,cACA;AAAA,cACA,OAAO;AAAA,YACT;AAAA,UACF;AACA,yBAAe,CAAC;AAChB,0BAAgB;AAAA,QAClB;AAGA,cAAM,iBAAiB,KAAK,oBAAoB,IAAI;AACpD,mBAAW,iBAAiB,gBAAgB;AAC1C,iBAAO;AAAA,YACL,KAAK,YAAY,eAAe,SAAS,YAAY,YAAY,OAAO,MAAM;AAAA,UAChF;AAAA,QACF;AACA;AAAA,MACF;AAGA,UAAI,gBAAgB,aAAa,KAAK,QAAQ,WAAW;AAEvD,YAAI,aAAa,SAAS,GAAG;AAC3B,iBAAO;AAAA,YACL,KAAK;AAAA,cACH,aAAa,KAAK,MAAM;AAAA,cACxB;AAAA,cACA;AAAA,cACA;AAAA,cACA,OAAO;AAAA,YACT;AAAA,UACF;AAGA,gBAAM,cAAc,KAAK,eAAe,YAAY;AACpD,yBAAe,cAAc,CAAC,aAAa,IAAI,IAAI,CAAC,IAAI;AACxD,0BAAgB,KAAK,eAAe,aAAa,KAAK,MAAM,CAAC;AAAA,QAC/D,OAAO;AACL,yBAAe,CAAC,IAAI;AACpB,0BAAgB;AAAA,QAClB;AAAA,MACF,OAAO;AACL,qBAAa,KAAK,IAAI;AACtB,yBAAiB;AAAA,MACnB;AAAA,IACF;AAGA,QAAI,aAAa,SAAS,KAAK,iBAAiB,KAAK,QAAQ,cAAc;AACzE,aAAO;AAAA,QACL,KAAK,YAAY,aAAa,KAAK,MAAM,GAAG,SAAS,YAAY,YAAY,OAAO,MAAM;AAAA,MAC5F;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKQ,oBAAoB,WAA6B;AACvD,UAAM,YAAY,UAAU,MAAM,gBAAgB,KAAK,CAAC,SAAS;AACjE,UAAM,SAAmB,CAAC;AAC1B,QAAI,eAAyB,CAAC;AAC9B,QAAI,gBAAgB;AAEpB,eAAW,YAAY,WAAW;AAChC,YAAM,iBAAiB,KAAK,eAAe,QAAQ;AAEnD,UAAI,gBAAgB,iBAAiB,KAAK,QAAQ,aAAa,aAAa,SAAS,GAAG;AACtF,eAAO,KAAK,aAAa,KAAK,GAAG,EAAE,KAAK,CAAC;AACzC,uBAAe,CAAC;AAChB,wBAAgB;AAAA,MAClB;AAEA,mBAAa,KAAK,SAAS,KAAK,CAAC;AACjC,uBAAiB;AAAA,IACnB;AAEA,QAAI,aAAa,SAAS,GAAG;AAC3B,aAAO,KAAK,aAAa,KAAK,GAAG,EAAE,KAAK,CAAC;AAAA,IAC3C;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKQ,eAAe,eAAwC;AAC7D,QAAI,CAAC,KAAK,QAAQ,gBAAgB,cAAc,WAAW,GAAG;AAC5D,aAAO;AAAA,IACT;AAGA,UAAM,WAAW,CAAC,GAAG,aAAa,EAAE,QAAQ;AAC5C,UAAM,eAAyB,CAAC;AAChC,QAAI,SAAS;AAEb,eAAW,QAAQ,UAAU;AAC3B,YAAM,aAAa,KAAK,eAAe,IAAI;AAC3C,UAAI,SAAS,aAAa,KAAK,QAAQ,cAAc;AACnD;AAAA,MACF;AACA,mBAAa,QAAQ,IAAI;AACzB,gBAAU;AAAA,IACZ;AAEA,WAAO,aAAa,SAAS,IAAI,aAAa,KAAK,MAAM,IAAI;AAAA,EAC/D;AAAA;AAAA;AAAA;AAAA,EAKQ,YACN,MACA,SACA,YACA,YACA,OACe;AACf,UAAM,WAA0B;AAAA,MAC9B,aAAa,QAAQ;AAAA,MACrB,WAAW,QAAQ;AAAA,MACnB,WAAW,QAAQ;AAAA,MACnB,WAAW,QAAQ;AAAA,MACnB;AAAA,MACA,YAAY;AAAA,IACd;AAEA,WAAO;AAAA,MACL,IAAI,WAAW,OAAO;AAAA,MACtB;AAAA,MACA,MAAM,KAAK,KAAK;AAAA,MAChB;AAAA,MACA,WAAW,oBAAI,KAAK;AAAA,IACtB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKQ,eAAe,MAAsB;AAC3C,WAAO,KAAK,KAAK,KAAK,SAAS,CAAC;AAAA,EAClC;AACF;","names":[]}

package/dist/embeddings/index.d.ts

@@ -0,0 +1,103 @@
+import { E as EmbeddingBackend, i as EmbeddingResult, h as EmbeddingOptions } from '../types-CjnplPJD.js';
+
+/**
+ * Local Embeddings using @xenova/transformers
+ *
+ * Runs embedding models locally without API calls.
+ * Supports BGE, all-MiniLM, and other sentence-transformer models.
+ */
+
+interface LocalEmbeddingOptions {
+    /** Model name (default: 'Xenova/bge-base-en-v1.5') */
+    model?: string;
+    /** Batch size for processing (default: 32) */
+    batchSize?: number;
+    /** Whether to normalize embeddings (default: true) */
+    normalize?: boolean;
+}
+declare class LocalEmbeddingBackend implements EmbeddingBackend {
+    private model;
+    private batchSize;
+    private normalize;
+    private dimensions;
+    private static MODEL_DIMENSIONS;
+    constructor(options?: LocalEmbeddingOptions);
+    embed(text: string): Promise<EmbeddingResult>;
+    embedBatch(texts: string[]): Promise<EmbeddingResult[]>;
+    getDimensions(): number;
+    getModel(): string;
+}
+/**
+ * Available local models
+ */
+declare const LOCAL_MODELS: {
+    /** BGE Large - Best quality, slower (1024 dims) */
+    BGE_LARGE: string;
+    /** BGE Base - Good balance (768 dims) */
+    BGE_BASE: string;
+    /** BGE Small - Fastest (384 dims) */
+    BGE_SMALL: string;
+    /** MiniLM L6 - Very fast (384 dims) */
+    MINILM_L6: string;
+    /** MiniLM L12 - Good quality (384 dims) */
+    MINILM_L12: string;
+    /** MPNet - High quality (768 dims) */
+    MPNET: string;
+};
+
+/**
+ * OpenAI Embeddings
+ *
+ * Uses OpenAI's embedding API for high-quality embeddings.
+ */
+
+interface OpenAIEmbeddingOptions {
+    /** OpenAI API key */
+    apiKey: string;
+    /** Model name (default: 'text-embedding-3-small') */
+    model?: string;
+    /** Embedding dimensions (for models that support it) */
+    dimensions?: number;
+    /** Batch size for processing (default: 100) */
+    batchSize?: number;
+    /** Base URL for API (for compatible APIs) */
+    baseUrl?: string;
+}
+declare class OpenAIEmbeddingBackend implements EmbeddingBackend {
+    private apiKey;
+    private model;
+    private dimensions;
+    private batchSize;
+    private baseUrl;
+    private static MODEL_DIMENSIONS;
+    constructor(options: OpenAIEmbeddingOptions);
+    embed(text: string): Promise<EmbeddingResult>;
+    embedBatch(texts: string[]): Promise<EmbeddingResult[]>;
+    private callAPI;
+    getDimensions(): number;
+    getModel(): string;
+}
+/**
+ * Available OpenAI embedding models
+ */
+declare const OPENAI_MODELS: {
+    /** text-embedding-3-large - Highest quality (3072 dims, can reduce) */
+    EMBEDDING_3_LARGE: string;
+    /** text-embedding-3-small - Good balance (1536 dims, can reduce) */
+    EMBEDDING_3_SMALL: string;
+    /** text-embedding-ada-002 - Legacy model (1536 dims) */
+    ADA_002: string;
+};
+
+/**
+ * Embedding Backends
+ *
+ * Generate vector embeddings from text using local or cloud models.
+ */
+
+/**
+ * Create an embedding backend based on options
+ */
+declare function createEmbeddingBackend(options: EmbeddingOptions): EmbeddingBackend;
+
+export { LOCAL_MODELS, LocalEmbeddingBackend, type LocalEmbeddingOptions, OPENAI_MODELS, OpenAIEmbeddingBackend, type OpenAIEmbeddingOptions, createEmbeddingBackend };

package/dist/embeddings/index.js

@@ -0,0 +1,195 @@
+// src/embeddings/local.ts
+var pipeline = null;
+var embedder = null;
+async function loadPipeline(model) {
+  if (!pipeline) {
+    const transformers = await import("@xenova/transformers");
+    pipeline = transformers.pipeline;
+  }
+  if (!embedder) {
+    console.log(`[LocalEmbeddings] Loading model: ${model}...`);
+    embedder = await pipeline("feature-extraction", model, {
+      quantized: true
+      // Use quantized model for faster inference
+    });
+    console.log(`[LocalEmbeddings] Model loaded successfully`);
+  }
+  return embedder;
+}
+var LocalEmbeddingBackend = class _LocalEmbeddingBackend {
+  model;
+  batchSize;
+  normalize;
+  dimensions;
+  // Model dimension map
+  static MODEL_DIMENSIONS = {
+    "Xenova/bge-large-en-v1.5": 1024,
+    "Xenova/bge-base-en-v1.5": 768,
+    "Xenova/bge-small-en-v1.5": 384,
+    "Xenova/all-MiniLM-L6-v2": 384,
+    "Xenova/all-MiniLM-L12-v2": 384,
+    "Xenova/all-mpnet-base-v2": 768
+  };
+  constructor(options) {
+    this.model = options?.model || "Xenova/bge-base-en-v1.5";
+    this.batchSize = options?.batchSize || 32;
+    this.normalize = options?.normalize ?? true;
+    this.dimensions = _LocalEmbeddingBackend.MODEL_DIMENSIONS[this.model] || 768;
+  }
+  async embed(text) {
+    const embedder2 = await loadPipeline(this.model);
+    const processedText = this.model.includes("bge") ? `Represent this sentence for searching relevant passages: ${text}` : text;
+    const output = await embedder2(processedText, {
+      pooling: "mean",
+      normalize: this.normalize
+    });
+    const embedding = Array.from(output.data);
+    return {
+      embedding,
+      tokens: Math.ceil(text.length / 4)
+      // Rough estimate
+    };
+  }
+  async embedBatch(texts) {
+    const results = [];
+    for (let i = 0; i < texts.length; i += this.batchSize) {
+      const batch = texts.slice(i, i + this.batchSize);
+      const batchResults = await Promise.all(batch.map((text) => this.embed(text)));
+      results.push(...batchResults);
+    }
+    return results;
+  }
+  getDimensions() {
+    return this.dimensions;
+  }
+  getModel() {
+    return this.model;
+  }
+};
+var LOCAL_MODELS = {
+  /** BGE Large - Best quality, slower (1024 dims) */
+  BGE_LARGE: "Xenova/bge-large-en-v1.5",
+  /** BGE Base - Good balance (768 dims) */
+  BGE_BASE: "Xenova/bge-base-en-v1.5",
+  /** BGE Small - Fastest (384 dims) */
+  BGE_SMALL: "Xenova/bge-small-en-v1.5",
+  /** MiniLM L6 - Very fast (384 dims) */
+  MINILM_L6: "Xenova/all-MiniLM-L6-v2",
+  /** MiniLM L12 - Good quality (384 dims) */
+  MINILM_L12: "Xenova/all-MiniLM-L12-v2",
+  /** MPNet - High quality (768 dims) */
+  MPNET: "Xenova/all-mpnet-base-v2"
+};
+
+// src/embeddings/openai.ts
+var OpenAIEmbeddingBackend = class _OpenAIEmbeddingBackend {
+  apiKey;
+  model;
+  dimensions;
+  batchSize;
+  baseUrl;
+  // Model dimension defaults
+  static MODEL_DIMENSIONS = {
+    "text-embedding-3-large": 3072,
+    "text-embedding-3-small": 1536,
+    "text-embedding-ada-002": 1536
+  };
+  constructor(options) {
+    this.apiKey = options.apiKey;
+    this.model = options.model || "text-embedding-3-small";
+    this.dimensions = options.dimensions || _OpenAIEmbeddingBackend.MODEL_DIMENSIONS[this.model] || 1536;
+    this.batchSize = options.batchSize || 100;
+    this.baseUrl = options.baseUrl || "https://api.openai.com/v1";
+  }
+  async embed(text) {
+    const results = await this.embedBatch([text]);
+    return results[0];
+  }
+  async embedBatch(texts) {
+    const allResults = [];
+    for (let i = 0; i < texts.length; i += this.batchSize) {
+      const batch = texts.slice(i, i + this.batchSize);
+      const batchResults = await this.callAPI(batch);
+      allResults.push(...batchResults);
+    }
+    return allResults;
+  }
+  async callAPI(texts) {
+    const body = {
+      model: this.model,
+      input: texts
+    };
+    if (this.model.startsWith("text-embedding-3-") && this.dimensions) {
+      body.dimensions = this.dimensions;
+    }
+    const response = await fetch(`${this.baseUrl}/embeddings`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${this.apiKey}`
+      },
+      body: JSON.stringify(body)
+    });
+    if (!response.ok) {
+      const error = await response.text();
+      throw new Error(`OpenAI API error: ${response.status} - ${error}`);
+    }
+    const data = await response.json();
+    const sorted = data.data.sort((a, b) => a.index - b.index);
+    return sorted.map((item, i) => ({
+      embedding: item.embedding,
+      tokens: Math.ceil(texts[i].length / 4)
+      // Rough estimate
+    }));
+  }
+  getDimensions() {
+    return this.dimensions;
+  }
+  getModel() {
+    return this.model;
+  }
+};
+var OPENAI_MODELS = {
+  /** text-embedding-3-large - Highest quality (3072 dims, can reduce) */
+  EMBEDDING_3_LARGE: "text-embedding-3-large",
+  /** text-embedding-3-small - Good balance (1536 dims, can reduce) */
+  EMBEDDING_3_SMALL: "text-embedding-3-small",
+  /** text-embedding-ada-002 - Legacy model (1536 dims) */
+  ADA_002: "text-embedding-ada-002"
+};
+
+// src/embeddings/index.ts
+function createEmbeddingBackend(options) {
+  switch (options.provider) {
+    case "local":
+      return new LocalEmbeddingBackend({
+        model: options.model,
+        batchSize: options.batchSize
+      });
+    case "openai":
+      if (!options.apiKey) {
+        throw new Error("OpenAI embedding requires an API key");
+      }
+      return new OpenAIEmbeddingBackend({
+        apiKey: options.apiKey,
+        model: options.model,
+        dimensions: options.dimensions,
+        batchSize: options.batchSize
+      });
+    case "huggingface":
+      return new LocalEmbeddingBackend({
+        model: options.model || "Xenova/all-MiniLM-L6-v2",
+        batchSize: options.batchSize
+      });
+    default:
+      throw new Error(`Unknown embedding provider: ${options.provider}`);
+  }
+}
+export {
+  LOCAL_MODELS,
+  LocalEmbeddingBackend,
+  OPENAI_MODELS,
+  OpenAIEmbeddingBackend,
+  createEmbeddingBackend
+};
+//# sourceMappingURL=index.js.map

package/dist/embeddings/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/embeddings/local.ts","../../src/embeddings/openai.ts","../../src/embeddings/index.ts"],"sourcesContent":["/**\n * Local Embeddings using @xenova/transformers\n *\n * Runs embedding models locally without API calls.\n * Supports BGE, all-MiniLM, and other sentence-transformer models.\n */\n\nimport type { EmbeddingBackend, EmbeddingResult } from '../types.js';\n\n// Dynamic import for transformers (it's a heavy module)\nlet pipeline: any = null;\nlet embedder: any = null;\n\nasync function loadPipeline(model: string) {\n  if (!pipeline) {\n    const transformers = await import('@xenova/transformers');\n    pipeline = transformers.pipeline;\n  }\n\n  if (!embedder) {\n    console.log(`[LocalEmbeddings] Loading model: ${model}...`);\n    embedder = await pipeline('feature-extraction', model, {\n      quantized: true, // Use quantized model for faster inference\n    });\n    console.log(`[LocalEmbeddings] Model loaded successfully`);\n  }\n\n  return embedder;\n}\n\nexport interface LocalEmbeddingOptions {\n  /** Model name (default: 'Xenova/bge-base-en-v1.5') */\n  model?: string;\n  /** Batch size for processing (default: 32) */\n  batchSize?: number;\n  /** Whether to normalize embeddings (default: true) */\n  normalize?: boolean;\n}\n\nexport class LocalEmbeddingBackend implements EmbeddingBackend {\n  private model: string;\n  private batchSize: number;\n  private normalize: boolean;\n  private dimensions: number;\n\n  // Model dimension map\n  private static MODEL_DIMENSIONS: Record<string, number> = {\n    'Xenova/bge-large-en-v1.5': 1024,\n    'Xenova/bge-base-en-v1.5': 768,\n    'Xenova/bge-small-en-v1.5': 384,\n    'Xenova/all-MiniLM-L6-v2': 384,\n    'Xenova/all-MiniLM-L12-v2': 384,\n    'Xenova/all-mpnet-base-v2': 768,\n  };\n\n  constructor(options?: LocalEmbeddingOptions) {\n    // Use bge-base for balance of quality and speed\n    this.model = options?.model || 'Xenova/bge-base-en-v1.5';\n    this.batchSize = options?.batchSize || 32;\n    this.normalize = options?.normalize ?? true;\n    this.dimensions = LocalEmbeddingBackend.MODEL_DIMENSIONS[this.model] || 768;\n  }\n\n  async embed(text: string): Promise<EmbeddingResult> {\n    const embedder = await loadPipeline(this.model);\n\n    // For BGE models, add instruction prefix for better retrieval\n    const processedText = this.model.includes('bge')\n      ? `Represent this sentence for searching relevant passages: ${text}`\n      : text;\n\n    const output = await embedder(processedText, {\n      pooling: 'mean',\n      normalize: this.normalize,\n    });\n\n    // Convert to regular array\n    const embedding = Array.from(output.data as Float32Array);\n\n    return {\n      embedding,\n      tokens: Math.ceil(text.length / 4), // Rough estimate\n    };\n  }\n\n  async embedBatch(texts: string[]): Promise<EmbeddingResult[]> {\n    const results: EmbeddingResult[] = [];\n\n    // Process in batches\n    for (let i = 0; i < texts.length; i += this.batchSize) {\n      const batch = texts.slice(i, i + this.batchSize);\n      const batchResults = await Promise.all(batch.map((text) => this.embed(text)));\n      results.push(...batchResults);\n    }\n\n    return results;\n  }\n\n  getDimensions(): number {\n    return this.dimensions;\n  }\n\n  getModel(): string {\n    return this.model;\n  }\n}\n\n/**\n * Available local models\n */\nexport const LOCAL_MODELS = {\n  /** BGE Large - Best quality, slower (1024 dims) */\n  BGE_LARGE: 'Xenova/bge-large-en-v1.5',\n  /** BGE Base - Good balance (768 dims) */\n  BGE_BASE: 'Xenova/bge-base-en-v1.5',\n  /** BGE Small - Fastest (384 dims) */\n  BGE_SMALL: 'Xenova/bge-small-en-v1.5',\n  /** MiniLM L6 - Very fast (384 dims) */\n  MINILM_L6: 'Xenova/all-MiniLM-L6-v2',\n  /** MiniLM L12 - Good quality (384 dims) */\n  MINILM_L12: 'Xenova/all-MiniLM-L12-v2',\n  /** MPNet - High quality (768 dims) */\n  MPNET: 'Xenova/all-mpnet-base-v2',\n};\n","/**\n * OpenAI Embeddings\n *\n * Uses OpenAI's embedding API for high-quality embeddings.\n */\n\nimport type { EmbeddingBackend, EmbeddingResult } from '../types.js';\n\nexport interface OpenAIEmbeddingOptions {\n  /** OpenAI API key */\n  apiKey: string;\n  /** Model name (default: 'text-embedding-3-small') */\n  model?: string;\n  /** Embedding dimensions (for models that support it) */\n  dimensions?: number;\n  /** Batch size for processing (default: 100) */\n  batchSize?: number;\n  /** Base URL for API (for compatible APIs) */\n  baseUrl?: string;\n}\n\nexport class OpenAIEmbeddingBackend implements EmbeddingBackend {\n  private apiKey: string;\n  private model: string;\n  private dimensions: number;\n  private batchSize: number;\n  private baseUrl: string;\n\n  // Model dimension defaults\n  private static MODEL_DIMENSIONS: Record<string, number> = {\n    'text-embedding-3-large': 3072,\n    'text-embedding-3-small': 1536,\n    'text-embedding-ada-002': 1536,\n  };\n\n  constructor(options: OpenAIEmbeddingOptions) {\n    this.apiKey = options.apiKey;\n    this.model = options.model || 'text-embedding-3-small';\n    this.dimensions =\n      options.dimensions || OpenAIEmbeddingBackend.MODEL_DIMENSIONS[this.model] || 1536;\n    this.batchSize = options.batchSize || 100;\n    this.baseUrl = options.baseUrl || 'https://api.openai.com/v1';\n  }\n\n  async embed(text: string): Promise<EmbeddingResult> {\n    const results = await this.embedBatch([text]);\n    return results[0];\n  }\n\n  async embedBatch(texts: string[]): Promise<EmbeddingResult[]> {\n    const allResults: EmbeddingResult[] = [];\n\n    // Process in batches (OpenAI has limits)\n    for (let i = 0; i < texts.length; i += this.batchSize) {\n      const batch = texts.slice(i, i + this.batchSize);\n      const batchResults = await this.callAPI(batch);\n      allResults.push(...batchResults);\n    }\n\n    return allResults;\n  }\n\n  private async callAPI(texts: string[]): Promise<EmbeddingResult[]> {\n    const body: Record<string, unknown> = {\n      model: this.model,\n      input: texts,\n    };\n\n    // Add dimensions param for text-embedding-3-* models\n    if (this.model.startsWith('text-embedding-3-') && this.dimensions) {\n      body.dimensions = this.dimensions;\n    }\n\n    const response = await fetch(`${this.baseUrl}/embeddings`, {\n      method: 'POST',\n      headers: {\n        'Content-Type': 'application/json',\n        Authorization: `Bearer ${this.apiKey}`,\n      },\n      body: JSON.stringify(body),\n    });\n\n    if (!response.ok) {\n      const error = await response.text();\n      throw new Error(`OpenAI API error: ${response.status} - ${error}`);\n    }\n\n    const data = (await response.json()) as {\n      data: Array<{ embedding: number[]; index: number }>;\n      usage: { prompt_tokens: number; total_tokens: number };\n    };\n\n    // Sort by index to maintain order\n    const sorted = data.data.sort((a, b) => a.index - b.index);\n\n    return sorted.map((item, i) => ({\n      embedding: item.embedding,\n      tokens: Math.ceil(texts[i].length / 4), // Rough estimate\n    }));\n  }\n\n  getDimensions(): number {\n    return this.dimensions;\n  }\n\n  getModel(): string {\n    return this.model;\n  }\n}\n\n/**\n * Available OpenAI embedding models\n */\nexport const OPENAI_MODELS = {\n  /** text-embedding-3-large - Highest quality (3072 dims, can reduce) */\n  EMBEDDING_3_LARGE: 'text-embedding-3-large',\n  /** text-embedding-3-small - Good balance (1536 dims, can reduce) */\n  EMBEDDING_3_SMALL: 'text-embedding-3-small',\n  /** text-embedding-ada-002 - Legacy model (1536 dims) */\n  ADA_002: 'text-embedding-ada-002',\n};\n","/**\n * Embedding Backends\n *\n * Generate vector embeddings from text using local or cloud models.\n */\n\nexport {\n  LocalEmbeddingBackend,\n  LOCAL_MODELS,\n  type LocalEmbeddingOptions,\n} from './local.js';\n\nexport {\n  OpenAIEmbeddingBackend,\n  OPENAI_MODELS,\n  type OpenAIEmbeddingOptions,\n} from './openai.js';\n\nimport type { EmbeddingBackend, EmbeddingOptions } from '../types.js';\nimport { LocalEmbeddingBackend } from './local.js';\nimport { OpenAIEmbeddingBackend } from './openai.js';\n\n/**\n * Create an embedding backend based on options\n */\nexport function createEmbeddingBackend(options: EmbeddingOptions): EmbeddingBackend {\n  switch (options.provider) {\n    case 'local':\n      return new LocalEmbeddingBackend({\n        model: options.model,\n        batchSize: options.batchSize,\n      });\n\n    case 'openai':\n      if (!options.apiKey) {\n        throw new Error('OpenAI embedding requires an API key');\n      }\n      return new OpenAIEmbeddingBackend({\n        apiKey: options.apiKey,\n        model: options.model,\n        dimensions: options.dimensions,\n        batchSize: options.batchSize,\n      });\n\n    case 'huggingface':\n      // For now, use local with HuggingFace models\n      return new LocalEmbeddingBackend({\n        model: options.model || 'Xenova/all-MiniLM-L6-v2',\n        batchSize: options.batchSize,\n      });\n\n    default:\n      throw new Error(`Unknown embedding provider: ${options.provider}`);\n  }\n}\n"],"mappings":";AAUA,IAAI,WAAgB;AACpB,IAAI,WAAgB;AAEpB,eAAe,aAAa,OAAe;AACzC,MAAI,CAAC,UAAU;AACb,UAAM,eAAe,MAAM,OAAO,sBAAsB;AACxD,eAAW,aAAa;AAAA,EAC1B;AAEA,MAAI,CAAC,UAAU;AACb,YAAQ,IAAI,oCAAoC,KAAK,KAAK;AAC1D,eAAW,MAAM,SAAS,sBAAsB,OAAO;AAAA,MACrD,WAAW;AAAA;AAAA,IACb,CAAC;AACD,YAAQ,IAAI,6CAA6C;AAAA,EAC3D;AAEA,SAAO;AACT;AAWO,IAAM,wBAAN,MAAM,uBAAkD;AAAA,EACrD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGR,OAAe,mBAA2C;AAAA,IACxD,4BAA4B;AAAA,IAC5B,2BAA2B;AAAA,IAC3B,4BAA4B;AAAA,IAC5B,2BAA2B;AAAA,IAC3B,4BAA4B;AAAA,IAC5B,4BAA4B;AAAA,EAC9B;AAAA,EAEA,YAAY,SAAiC;AAE3C,SAAK,QAAQ,SAAS,SAAS;AAC/B,SAAK,YAAY,SAAS,aAAa;AACvC,SAAK,YAAY,SAAS,aAAa;AACvC,SAAK,aAAa,uBAAsB,iBAAiB,KAAK,KAAK,KAAK;AAAA,EAC1E;AAAA,EAEA,MAAM,MAAM,MAAwC;AAClD,UAAMA,YAAW,MAAM,aAAa,KAAK,KAAK;AAG9C,UAAM,gBAAgB,KAAK,MAAM,SAAS,KAAK,IAC3C,4DAA4D,IAAI,KAChE;AAEJ,UAAM,SAAS,MAAMA,UAAS,eAAe;AAAA,MAC3C,SAAS;AAAA,MACT,WAAW,KAAK;AAAA,IAClB,CAAC;AAGD,UAAM,YAAY,MAAM,KAAK,OAAO,IAAoB;AAExD,WAAO;AAAA,MACL;AAAA,MACA,QAAQ,KAAK,KAAK,KAAK,SAAS,CAAC;AAAA;AAAA,IACnC;AAAA,EACF;AAAA,EAEA,MAAM,WAAW,OAA6C;AAC5D,UAAM,UAA6B,CAAC;AAGpC,aAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,KAAK,WAAW;AACrD,YAAM,QAAQ,MAAM,MAAM,GAAG,IAAI,KAAK,SAAS;AAC/C,YAAM,eAAe,MAAM,QAAQ,IAAI,MAAM,IAAI,CAAC,SAAS,KAAK,MAAM,IAAI,CAAC,CAAC;AAC5E,cAAQ,KAAK,GAAG,YAAY;AAAA,IAC9B;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,gBAAwB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,WAAmB;AACjB,WAAO,KAAK;AAAA,EACd;AACF;AAKO,IAAM,eAAe;AAAA;AAAA,EAE1B,WAAW;AAAA;AAAA,EAEX,UAAU;AAAA;AAAA,EAEV,WAAW;AAAA;AAAA,EAEX,WAAW;AAAA;AAAA,EAEX,YAAY;AAAA;AAAA,EAEZ,OAAO;AACT;;;ACtGO,IAAM,yBAAN,MAAM,wBAAmD;AAAA,EACtD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGR,OAAe,mBAA2C;AAAA,IACxD,0BAA0B;AAAA,IAC1B,0BAA0B;AAAA,IAC1B,0BAA0B;AAAA,EAC5B;AAAA,EAEA,YAAY,SAAiC;AAC3C,SAAK,SAAS,QAAQ;AACtB,SAAK,QAAQ,QAAQ,SAAS;AAC9B,SAAK,aACH,QAAQ,cAAc,wBAAuB,iBAAiB,KAAK,KAAK,KAAK;AAC/E,SAAK,YAAY,QAAQ,aAAa;AACtC,SAAK,UAAU,QAAQ,WAAW;AAAA,EACpC;AAAA,EAEA,MAAM,MAAM,MAAwC;AAClD,UAAM,UAAU,MAAM,KAAK,WAAW,CAAC,IAAI,CAAC;AAC5C,WAAO,QAAQ,CAAC;AAAA,EAClB;AAAA,EAEA,MAAM,WAAW,OAA6C;AAC5D,UAAM,aAAgC,CAAC;AAGvC,aAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,KAAK,WAAW;AACrD,YAAM,QAAQ,MAAM,MAAM,GAAG,IAAI,KAAK,SAAS;AAC/C,YAAM,eAAe,MAAM,KAAK,QAAQ,KAAK;AAC7C,iBAAW,KAAK,GAAG,YAAY;AAAA,IACjC;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,MAAc,QAAQ,OAA6C;AACjE,UAAM,OAAgC;AAAA,MACpC,OAAO,KAAK;AAAA,MACZ,OAAO;AAAA,IACT;AAGA,QAAI,KAAK,MAAM,WAAW,mBAAmB,KAAK,KAAK,YAAY;AACjE,WAAK,aAAa,KAAK;AAAA,IACzB;AAEA,UAAM,WAAW,MAAM,MAAM,GAAG,KAAK,OAAO,eAAe;AAAA,MACzD,QAAQ;AAAA,MACR,SAAS;AAAA,QACP,gBAAgB;AAAA,QAChB,eAAe,UAAU,KAAK,MAAM;AAAA,MACtC;AAAA,MACA,MAAM,KAAK,UAAU,IAAI;AAAA,IAC3B,CAAC;AAED,QAAI,CAAC,SAAS,IAAI;AAChB,YAAM,QAAQ,MAAM,SAAS,KAAK;AAClC,YAAM,IAAI,MAAM,qBAAqB,SAAS,MAAM,MAAM,KAAK,EAAE;AAAA,IACnE;AAEA,UAAM,OAAQ,MAAM,SAAS,KAAK;AAMlC,UAAM,SAAS,KAAK,KAAK,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AAEzD,WAAO,OAAO,IAAI,CAAC,MAAM,OAAO;AAAA,MAC9B,WAAW,KAAK;AAAA,MAChB,QAAQ,KAAK,KAAK,MAAM,CAAC,EAAE,SAAS,CAAC;AAAA;AAAA,IACvC,EAAE;AAAA,EACJ;AAAA,EAEA,gBAAwB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,WAAmB;AACjB,WAAO,KAAK;AAAA,EACd;AACF;AAKO,IAAM,gBAAgB;AAAA;AAAA,EAE3B,mBAAmB;AAAA;AAAA,EAEnB,mBAAmB;AAAA;AAAA,EAEnB,SAAS;AACX;;;AC/FO,SAAS,uBAAuB,SAA6C;AAClF,UAAQ,QAAQ,UAAU;AAAA,IACxB,KAAK;AACH,aAAO,IAAI,sBAAsB;AAAA,QAC/B,OAAO,QAAQ;AAAA,QACf,WAAW,QAAQ;AAAA,MACrB,CAAC;AAAA,IAEH,KAAK;AACH,UAAI,CAAC,QAAQ,QAAQ;AACnB,cAAM,IAAI,MAAM,sCAAsC;AAAA,MACxD;AACA,aAAO,IAAI,uBAAuB;AAAA,QAChC,QAAQ,QAAQ;AAAA,QAChB,OAAO,QAAQ;AAAA,QACf,YAAY,QAAQ;AAAA,QACpB,WAAW,QAAQ;AAAA,MACrB,CAAC;AAAA,IAEH,KAAK;AAEH,aAAO,IAAI,sBAAsB;AAAA,QAC/B,OAAO,QAAQ,SAAS;AAAA,QACxB,WAAW,QAAQ;AAAA,MACrB,CAAC;AAAA,IAEH;AACE,YAAM,IAAI,MAAM,+BAA+B,QAAQ,QAAQ,EAAE;AAAA,EACrE;AACF;","names":["embedder"]}