@chatbot-packages/rag 0.1.0

package/dist/index.js

@@ -0,0 +1,1576 @@
+// src/rag-service.ts
+import { generateId as generateId2 } from "@chatbot-packages/utils";
+import { OpenAIProvider, AnthropicProvider } from "@chatbot-packages/ai";
+
+// src/extractors/base.ts
+var BaseExtractor = class {
+  options;
+  constructor(options) {
+    this.options = options;
+  }
+  /** Get file extension */
+  getExtension(path) {
+    const parts = path.split(".");
+    return parts.length > 1 ? parts[parts.length - 1].toLowerCase() : "";
+  }
+  /** Normalize line endings */
+  normalizeLineEndings(text) {
+    return text.replace(/\r\n/g, "\n").replace(/\r/g, "\n");
+  }
+  /** Clean excessive whitespace */
+  cleanWhitespace(text) {
+    return text.replace(/\n{3,}/g, "\n\n").replace(/[ \t]+/g, " ").trim();
+  }
+};
+
+// src/extractors/chm.ts
+import { exec } from "child_process";
+import { promisify } from "util";
+import { readdir as readdir2, readFile as readFile2, mkdir, rm } from "fs/promises";
+import { existsSync } from "fs";
+import { join as join2, basename as basename2, dirname } from "path";
+
+// src/extractors/html.ts
+import { readFile, readdir, stat } from "fs/promises";
+import { join, basename } from "path";
+import * as cheerio from "cheerio";
+import { convert } from "html-to-text";
+var HTMLExtractor = class extends BaseExtractor {
+  htmlOptions;
+  constructor(options) {
+    super(options);
+    this.htmlOptions = {
+      preserveHeadings: true,
+      includeLinks: false,
+      removeTags: ["script", "style", "nav", "footer", "header", "aside", "meta", "link"],
+      ...options
+    };
+  }
+  getType() {
+    return "html";
+  }
+  canHandle(path) {
+    const ext = this.getExtension(path);
+    return ext === "html" || ext === "htm";
+  }
+  async extract() {
+    const { sourcePath } = this.options;
+    const stats = await stat(sourcePath);
+    if (stats.isFile()) {
+      const content = await readFile(sourcePath, "utf-8");
+      return [await this.extractFromString(content, sourcePath)];
+    }
+    if (stats.isDirectory()) {
+      return this.extractFromDirectory(sourcePath);
+    }
+    throw new Error(`Invalid path: ${sourcePath}`);
+  }
+  async extractFromString(html, filePath) {
+    const $ = cheerio.load(html);
+    for (const tag of this.htmlOptions.removeTags || []) {
+      $(tag).remove();
+    }
+    let title = $("title").text().trim();
+    if (!title) {
+      title = $("h1").first().text().trim();
+    }
+    if (!title) {
+      title = basename(filePath, ".html").replace(/-|_/g, " ");
+    }
+    const text = convert($.html(), {
+      wordwrap: false,
+      preserveNewlines: true,
+      selectors: [
+        { selector: "h1", options: { uppercase: false, prefix: "\n# " } },
+        { selector: "h2", options: { uppercase: false, prefix: "\n## " } },
+        { selector: "h3", options: { uppercase: false, prefix: "\n### " } },
+        { selector: "h4", options: { uppercase: false, prefix: "\n#### " } },
+        { selector: "h5", options: { uppercase: false, prefix: "\n##### " } },
+        { selector: "h6", options: { uppercase: false, prefix: "\n###### " } },
+        { selector: "ul", options: { itemPrefix: "  - " } },
+        { selector: "ol", options: { itemPrefix: "  1. " } },
+        { selector: "table", format: "dataTable" },
+        { selector: "a", options: { ignoreHref: !this.htmlOptions.includeLinks } },
+        { selector: "img", format: "skip" }
+      ]
+    });
+    const content = this.cleanWhitespace(this.normalizeLineEndings(text));
+    return {
+      path: filePath,
+      title,
+      content,
+      format: "html",
+      metadata: {
+        originalLength: html.length,
+        extractedLength: content.length
+      }
+    };
+  }
+  async extractFromDirectory(dir) {
+    const documents = [];
+    const scanDir = async (currentDir) => {
+      const entries = await readdir(currentDir, { withFileTypes: true });
+      for (const entry of entries) {
+        const fullPath = join(currentDir, entry.name);
+        if (entry.isDirectory()) {
+          await scanDir(fullPath);
+        } else if (entry.isFile() && this.canHandle(entry.name)) {
+          try {
+            const content = await readFile(fullPath, "utf-8");
+            const doc = await this.extractFromString(content, fullPath);
+            if (doc.content.trim()) {
+              documents.push(doc);
+            }
+          } catch (error) {
+            console.warn(`Failed to extract ${fullPath}:`, error);
+          }
+        }
+      }
+    };
+    await scanDir(dir);
+    return documents;
+  }
+};
+
+// src/extractors/chm.ts
+var execAsync = promisify(exec);
+var CHMExtractor = class extends BaseExtractor {
+  tempDir;
+  constructor(options) {
+    super(options);
+    this.tempDir = options.outputDir || join2(dirname(options.sourcePath), ".chm-extract-temp");
+  }
+  getType() {
+    return "chm";
+  }
+  canHandle(path) {
+    return this.getExtension(path) === "chm";
+  }
+  async extract() {
+    const { sourcePath } = this.options;
+    if (!existsSync(sourcePath)) {
+      throw new Error(`CHM file not found: ${sourcePath}`);
+    }
+    await mkdir(this.tempDir, { recursive: true });
+    try {
+      await this.extractWithSevenZip(sourcePath, this.tempDir);
+      const htmlFiles = await this.findHTMLFiles(this.tempDir);
+      const documents = [];
+      for (const htmlFile of htmlFiles) {
+        try {
+          const content = await readFile2(htmlFile, "utf-8");
+          const htmlExtractor = new HTMLExtractor({ sourcePath: htmlFile });
+          const extracted = await htmlExtractor.extractFromString(content, htmlFile);
+          if (extracted.content.trim()) {
+            documents.push({
+              ...extracted,
+              format: "chm",
+              metadata: {
+                ...extracted.metadata,
+                sourceChm: basename2(sourcePath),
+                originalPath: htmlFile.replace(this.tempDir, "")
+              }
+            });
+          }
+        } catch (error) {
+          console.warn(`Failed to parse HTML file: ${htmlFile}`, error);
+        }
+      }
+      return documents;
+    } finally {
+      if (existsSync(this.tempDir)) {
+        await rm(this.tempDir, { recursive: true, force: true });
+      }
+    }
+  }
+  async extractWithSevenZip(chmPath, outputDir) {
+    try {
+      await execAsync(`7z x "${chmPath}" -o"${outputDir}" -y`, {
+        maxBuffer: 50 * 1024 * 1024
+        // 50MB buffer
+      });
+    } catch (error) {
+      try {
+        await execAsync(`7za x "${chmPath}" -o"${outputDir}" -y`, {
+          maxBuffer: 50 * 1024 * 1024
+        });
+      } catch {
+        throw new Error(
+          `Failed to extract CHM file. Please ensure 7z is installed.
+  - On Ubuntu/Debian: sudo apt-get install p7zip-full
+  - On macOS: brew install p7zip
+  - On Windows: Install 7-Zip from https://7-zip.org/
+Original error: ${error}`
+        );
+      }
+    }
+  }
+  async findHTMLFiles(dir) {
+    const htmlFiles = [];
+    const scanDir = async (currentDir) => {
+      const entries = await readdir2(currentDir, { withFileTypes: true });
+      for (const entry of entries) {
+        const fullPath = join2(currentDir, entry.name);
+        if (entry.isDirectory()) {
+          if (!entry.name.startsWith("$") && !entry.name.startsWith("#")) {
+            await scanDir(fullPath);
+          }
+        } else if (entry.isFile()) {
+          const ext = this.getExtension(entry.name);
+          if (ext === "html" || ext === "htm") {
+            htmlFiles.push(fullPath);
+          }
+        }
+      }
+    };
+    await scanDir(dir);
+    return htmlFiles.sort();
+  }
+};
+
+// src/extractors/markdown.ts
+import { readFile as readFile3, readdir as readdir3, stat as stat2 } from "fs/promises";
+import { join as join3, basename as basename3 } from "path";
+var MarkdownExtractor = class extends BaseExtractor {
+  getType() {
+    return "markdown";
+  }
+  canHandle(path) {
+    const ext = this.getExtension(path);
+    return ext === "md" || ext === "markdown";
+  }
+  async extract() {
+    const { sourcePath } = this.options;
+    const stats = await stat2(sourcePath);
+    if (stats.isFile()) {
+      return [await this.extractFromFile(sourcePath)];
+    }
+    if (stats.isDirectory()) {
+      return this.extractFromDirectory(sourcePath);
+    }
+    throw new Error(`Invalid path: ${sourcePath}`);
+  }
+  async extractFromFile(filePath) {
+    const content = await readFile3(filePath, "utf-8");
+    const normalizedContent = this.normalizeLineEndings(content);
+    let title = this.extractTitle(normalizedContent);
+    if (!title) {
+      title = basename3(filePath, ".md").replace(/-|_/g, " ");
+    }
+    return {
+      path: filePath,
+      title,
+      content: this.cleanWhitespace(normalizedContent),
+      format: "markdown",
+      metadata: {
+        originalLength: content.length
+      }
+    };
+  }
+  extractTitle(content) {
+    const h1Match = content.match(/^#\s+(.+)$/m);
+    if (h1Match) {
+      return h1Match[1].trim();
+    }
+    const underlineMatch = content.match(/^(.+)\n=+\s*$/m);
+    if (underlineMatch) {
+      return underlineMatch[1].trim();
+    }
+    return void 0;
+  }
+  async extractFromDirectory(dir) {
+    const documents = [];
+    const scanDir = async (currentDir) => {
+      const entries = await readdir3(currentDir, { withFileTypes: true });
+      for (const entry of entries) {
+        const fullPath = join3(currentDir, entry.name);
+        if (entry.isDirectory()) {
+          if (!entry.name.startsWith(".") && entry.name !== "node_modules") {
+            await scanDir(fullPath);
+          }
+        } else if (entry.isFile() && this.canHandle(entry.name)) {
+          try {
+            const doc = await this.extractFromFile(fullPath);
+            if (doc.content.trim()) {
+              documents.push(doc);
+            }
+          } catch (error) {
+            console.warn(`Failed to extract ${fullPath}:`, error);
+          }
+        }
+      }
+    };
+    await scanDir(dir);
+    return documents;
+  }
+};
+
+// src/extractors/index.ts
+import { stat as stat3 } from "fs/promises";
+async function extractDocuments(sourcePath) {
+  const stats = await stat3(sourcePath);
+  const ext = sourcePath.split(".").pop()?.toLowerCase() || "";
+  if (stats.isFile()) {
+    if (ext === "chm") {
+      const extractor = new CHMExtractor({ sourcePath });
+      return extractor.extract();
+    }
+    if (ext === "html" || ext === "htm") {
+      const extractor = new HTMLExtractor({ sourcePath });
+      return extractor.extract();
+    }
+    if (ext === "md" || ext === "markdown") {
+      const extractor = new MarkdownExtractor({ sourcePath });
+      return extractor.extract();
+    }
+    throw new Error(`Unsupported file format: ${ext}`);
+  }
+  if (stats.isDirectory()) {
+    const allDocuments = [];
+    const htmlExtractor = new HTMLExtractor({ sourcePath });
+    const htmlDocs = await htmlExtractor.extract();
+    allDocuments.push(...htmlDocs);
+    const mdExtractor = new MarkdownExtractor({ sourcePath });
+    const mdDocs = await mdExtractor.extract();
+    allDocuments.push(...mdDocs);
+    return allDocuments;
+  }
+  throw new Error(`Invalid path: ${sourcePath}`);
+}
+
+// src/chunking/header-aware-chunker.ts
+import { generateId } from "@chatbot-packages/utils";
+var HeaderAwareChunker = class {
+  options;
+  constructor(options) {
+    this.options = {
+      chunkSize: 512,
+      chunkOverlap: 50,
+      minChunkSize: 100,
+      maxChunkSize: 800,
+      respectHeadings: true,
+      splitOnHeadings: [1, 2],
+      ...options
+    };
+  }
+  /**
+   * Chunk a document into smaller pieces
+   */
+  chunk(document, documentId) {
+    const docId = documentId || generateId("doc");
+    const { content } = document;
+    if (!content.trim()) {
+      return {
+        chunks: [],
+        stats: { totalChunks: 0, avgChunkSize: 0, minChunkSize: 0, maxChunkSize: 0 }
+      };
+    }
+    const chunks = [];
+    const sections = this.splitBySections(content);
+    for (const section of sections) {
+      const sectionChunks = this.chunkSection(section, docId, document.path);
+      chunks.push(...sectionChunks);
+    }
+    const sizes = chunks.map((c) => c.text.length);
+    const stats = {
+      totalChunks: chunks.length,
+      avgChunkSize: sizes.length > 0 ? Math.round(sizes.reduce((a, b) => a + b, 0) / sizes.length) : 0,
+      minChunkSize: sizes.length > 0 ? Math.min(...sizes) : 0,
+      maxChunkSize: sizes.length > 0 ? Math.max(...sizes) : 0
+    };
+    return { chunks, stats };
+  }
+  /**
+   * Split content by heading boundaries
+   */
+  splitBySections(content) {
+    const lines = content.split("\n");
+    const sections = [];
+    let currentContext = { sectionPath: "" };
+    let currentText = [];
+    for (const line of lines) {
+      const heading = this.parseHeading(line);
+      if (heading && this.options.splitOnHeadings.includes(heading.level)) {
+        if (currentText.length > 0) {
+          sections.push({
+            text: currentText.join("\n").trim(),
+            context: { ...currentContext }
+          });
+          currentText = [];
+        }
+        if (heading.level === 1) {
+          currentContext = {
+            h1: heading.text,
+            sectionPath: heading.text
+          };
+        } else if (heading.level === 2) {
+          currentContext = {
+            ...currentContext,
+            h2: heading.text,
+            h3: void 0,
+            sectionPath: currentContext.h1 ? `${currentContext.h1} > ${heading.text}` : heading.text
+          };
+        } else if (heading.level === 3) {
+          currentContext = {
+            ...currentContext,
+            h3: heading.text,
+            sectionPath: currentContext.h2 ? `${currentContext.sectionPath} > ${heading.text}` : heading.text
+          };
+        }
+        currentText.push(line);
+      } else {
+        currentText.push(line);
+      }
+    }
+    if (currentText.length > 0) {
+      sections.push({
+        text: currentText.join("\n").trim(),
+        context: { ...currentContext }
+      });
+    }
+    return sections;
+  }
+  /**
+   * Parse a heading line
+   */
+  parseHeading(line) {
+    const match = line.match(/^(#{1,6})\s+(.+)$/);
+    if (match) {
+      return {
+        level: match[1].length,
+        text: match[2].trim()
+      };
+    }
+    return null;
+  }
+  /**
+   * Chunk a single section
+   */
+  chunkSection(section, documentId, sourcePath) {
+    const { text, context } = section;
+    const chunks = [];
+    if (this.estimateTokens(text) <= this.options.maxChunkSize) {
+      if (this.estimateTokens(text) >= this.options.minChunkSize) {
+        chunks.push(this.createChunk(text, context, documentId, sourcePath, 0));
+      }
+      return chunks;
+    }
+    const paragraphs = text.split(/\n\n+/);
+    let currentChunk = [];
+    let currentTokens = 0;
+    for (const para of paragraphs) {
+      const paraTokens = this.estimateTokens(para);
+      if (paraTokens > this.options.maxChunkSize) {
+        if (currentChunk.length > 0) {
+          chunks.push(
+            this.createChunk(
+              currentChunk.join("\n\n"),
+              context,
+              documentId,
+              sourcePath,
+              chunks.length
+            )
+          );
+          currentChunk = [];
+          currentTokens = 0;
+        }
+        const sentenceChunks = this.splitLargeParagraph(para);
+        for (const sentenceChunk of sentenceChunks) {
+          chunks.push(
+            this.createChunk(sentenceChunk, context, documentId, sourcePath, chunks.length)
+          );
+        }
+        continue;
+      }
+      if (currentTokens + paraTokens > this.options.chunkSize) {
+        if (currentChunk.length > 0) {
+          chunks.push(
+            this.createChunk(
+              currentChunk.join("\n\n"),
+              context,
+              documentId,
+              sourcePath,
+              chunks.length
+            )
+          );
+          const overlapText = this.getOverlapText(currentChunk);
+          currentChunk = overlapText ? [overlapText, para] : [para];
+          currentTokens = this.estimateTokens(currentChunk.join("\n\n"));
+        } else {
+          currentChunk = [para];
+          currentTokens = paraTokens;
+        }
+      } else {
+        currentChunk.push(para);
+        currentTokens += paraTokens;
+      }
+    }
+    if (currentChunk.length > 0 && currentTokens >= this.options.minChunkSize) {
+      chunks.push(
+        this.createChunk(currentChunk.join("\n\n"), context, documentId, sourcePath, chunks.length)
+      );
+    }
+    return chunks;
+  }
+  /**
+   * Split a large paragraph into sentence-based chunks
+   */
+  splitLargeParagraph(paragraph) {
+    const sentences = paragraph.match(/[^.!?]+[.!?]+/g) || [paragraph];
+    const chunks = [];
+    let currentChunk = [];
+    let currentTokens = 0;
+    for (const sentence of sentences) {
+      const sentenceTokens = this.estimateTokens(sentence);
+      if (currentTokens + sentenceTokens > this.options.chunkSize && currentChunk.length > 0) {
+        chunks.push(currentChunk.join(" ").trim());
+        currentChunk = [];
+        currentTokens = 0;
+      }
+      currentChunk.push(sentence.trim());
+      currentTokens += sentenceTokens;
+    }
+    if (currentChunk.length > 0) {
+      chunks.push(currentChunk.join(" ").trim());
+    }
+    return chunks;
+  }
+  /**
+   * Get overlap text from previous chunk
+   */
+  getOverlapText(previousChunk) {
+    if (!this.options.chunkOverlap || previousChunk.length === 0) {
+      return null;
+    }
+    const reversed = [...previousChunk].reverse();
+    const overlapParts = [];
+    let tokens = 0;
+    for (const part of reversed) {
+      const partTokens = this.estimateTokens(part);
+      if (tokens + partTokens > this.options.chunkOverlap) {
+        break;
+      }
+      overlapParts.unshift(part);
+      tokens += partTokens;
+    }
+    return overlapParts.length > 0 ? overlapParts.join("\n\n") : null;
+  }
+  /**
+   * Create a chunk object
+   */
+  createChunk(text, context, documentId, sourcePath, index) {
+    const metadata = {
+      sectionPath: context.sectionPath,
+      headingH1: context.h1,
+      headingH2: context.h2,
+      headingH3: context.h3,
+      sourcePath,
+      chunkIndex: index
+    };
+    return {
+      id: generateId("chunk"),
+      documentId,
+      text: text.trim(),
+      metadata,
+      createdAt: /* @__PURE__ */ new Date()
+    };
+  }
+  /**
+   * Estimate token count (rough approximation: ~4 chars per token)
+   */
+  estimateTokens(text) {
+    return Math.ceil(text.length / 4);
+  }
+};
+
+// src/embeddings/local.ts
+var pipeline = null;
+var embedder = null;
+async function loadPipeline(model) {
+  if (!pipeline) {
+    const transformers = await import("@xenova/transformers");
+    pipeline = transformers.pipeline;
+  }
+  if (!embedder) {
+    console.log(`[LocalEmbeddings] Loading model: ${model}...`);
+    embedder = await pipeline("feature-extraction", model, {
+      quantized: true
+      // Use quantized model for faster inference
+    });
+    console.log(`[LocalEmbeddings] Model loaded successfully`);
+  }
+  return embedder;
+}
+var LocalEmbeddingBackend = class _LocalEmbeddingBackend {
+  model;
+  batchSize;
+  normalize;
+  dimensions;
+  // Model dimension map
+  static MODEL_DIMENSIONS = {
+    "Xenova/bge-large-en-v1.5": 1024,
+    "Xenova/bge-base-en-v1.5": 768,
+    "Xenova/bge-small-en-v1.5": 384,
+    "Xenova/all-MiniLM-L6-v2": 384,
+    "Xenova/all-MiniLM-L12-v2": 384,
+    "Xenova/all-mpnet-base-v2": 768
+  };
+  constructor(options) {
+    this.model = options?.model || "Xenova/bge-base-en-v1.5";
+    this.batchSize = options?.batchSize || 32;
+    this.normalize = options?.normalize ?? true;
+    this.dimensions = _LocalEmbeddingBackend.MODEL_DIMENSIONS[this.model] || 768;
+  }
+  async embed(text) {
+    const embedder2 = await loadPipeline(this.model);
+    const processedText = this.model.includes("bge") ? `Represent this sentence for searching relevant passages: ${text}` : text;
+    const output = await embedder2(processedText, {
+      pooling: "mean",
+      normalize: this.normalize
+    });
+    const embedding = Array.from(output.data);
+    return {
+      embedding,
+      tokens: Math.ceil(text.length / 4)
+      // Rough estimate
+    };
+  }
+  async embedBatch(texts) {
+    const results = [];
+    for (let i = 0; i < texts.length; i += this.batchSize) {
+      const batch = texts.slice(i, i + this.batchSize);
+      const batchResults = await Promise.all(batch.map((text) => this.embed(text)));
+      results.push(...batchResults);
+    }
+    return results;
+  }
+  getDimensions() {
+    return this.dimensions;
+  }
+  getModel() {
+    return this.model;
+  }
+};
+var LOCAL_MODELS = {
+  /** BGE Large - Best quality, slower (1024 dims) */
+  BGE_LARGE: "Xenova/bge-large-en-v1.5",
+  /** BGE Base - Good balance (768 dims) */
+  BGE_BASE: "Xenova/bge-base-en-v1.5",
+  /** BGE Small - Fastest (384 dims) */
+  BGE_SMALL: "Xenova/bge-small-en-v1.5",
+  /** MiniLM L6 - Very fast (384 dims) */
+  MINILM_L6: "Xenova/all-MiniLM-L6-v2",
+  /** MiniLM L12 - Good quality (384 dims) */
+  MINILM_L12: "Xenova/all-MiniLM-L12-v2",
+  /** MPNet - High quality (768 dims) */
+  MPNET: "Xenova/all-mpnet-base-v2"
+};
+
+// src/embeddings/openai.ts
+var OpenAIEmbeddingBackend = class _OpenAIEmbeddingBackend {
+  apiKey;
+  model;
+  dimensions;
+  batchSize;
+  baseUrl;
+  // Model dimension defaults
+  static MODEL_DIMENSIONS = {
+    "text-embedding-3-large": 3072,
+    "text-embedding-3-small": 1536,
+    "text-embedding-ada-002": 1536
+  };
+  constructor(options) {
+    this.apiKey = options.apiKey;
+    this.model = options.model || "text-embedding-3-small";
+    this.dimensions = options.dimensions || _OpenAIEmbeddingBackend.MODEL_DIMENSIONS[this.model] || 1536;
+    this.batchSize = options.batchSize || 100;
+    this.baseUrl = options.baseUrl || "https://api.openai.com/v1";
+  }
+  async embed(text) {
+    const results = await this.embedBatch([text]);
+    return results[0];
+  }
+  async embedBatch(texts) {
+    const allResults = [];
+    for (let i = 0; i < texts.length; i += this.batchSize) {
+      const batch = texts.slice(i, i + this.batchSize);
+      const batchResults = await this.callAPI(batch);
+      allResults.push(...batchResults);
+    }
+    return allResults;
+  }
+  async callAPI(texts) {
+    const body = {
+      model: this.model,
+      input: texts
+    };
+    if (this.model.startsWith("text-embedding-3-") && this.dimensions) {
+      body.dimensions = this.dimensions;
+    }
+    const response = await fetch(`${this.baseUrl}/embeddings`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${this.apiKey}`
+      },
+      body: JSON.stringify(body)
+    });
+    if (!response.ok) {
+      const error = await response.text();
+      throw new Error(`OpenAI API error: ${response.status} - ${error}`);
+    }
+    const data = await response.json();
+    const sorted = data.data.sort((a, b) => a.index - b.index);
+    return sorted.map((item, i) => ({
+      embedding: item.embedding,
+      tokens: Math.ceil(texts[i].length / 4)
+      // Rough estimate
+    }));
+  }
+  getDimensions() {
+    return this.dimensions;
+  }
+  getModel() {
+    return this.model;
+  }
+};
+var OPENAI_MODELS = {
+  /** text-embedding-3-large - Highest quality (3072 dims, can reduce) */
+  EMBEDDING_3_LARGE: "text-embedding-3-large",
+  /** text-embedding-3-small - Good balance (1536 dims, can reduce) */
+  EMBEDDING_3_SMALL: "text-embedding-3-small",
+  /** text-embedding-ada-002 - Legacy model (1536 dims) */
+  ADA_002: "text-embedding-ada-002"
+};
+
+// src/embeddings/index.ts
+function createEmbeddingBackend(options) {
+  switch (options.provider) {
+    case "local":
+      return new LocalEmbeddingBackend({
+        model: options.model,
+        batchSize: options.batchSize
+      });
+    case "openai":
+      if (!options.apiKey) {
+        throw new Error("OpenAI embedding requires an API key");
+      }
+      return new OpenAIEmbeddingBackend({
+        apiKey: options.apiKey,
+        model: options.model,
+        dimensions: options.dimensions,
+        batchSize: options.batchSize
+      });
+    case "huggingface":
+      return new LocalEmbeddingBackend({
+        model: options.model || "Xenova/all-MiniLM-L6-v2",
+        batchSize: options.batchSize
+      });
+    default:
+      throw new Error(`Unknown embedding provider: ${options.provider}`);
+  }
+}
+
+// src/vectorstore/sqlite.ts
+import Database from "better-sqlite3";
+var SQLiteVectorStore = class {
+  db;
+  tableName;
+  dimensions;
+  constructor(options) {
+    this.db = new Database(options.path || ":memory:");
+    this.tableName = options.tableName || "chunks";
+    this.dimensions = options.dimensions;
+    this.db.pragma("journal_mode = WAL");
+  }
+  async initialize() {
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS ${this.tableName} (
+        id TEXT PRIMARY KEY,
+        document_id TEXT NOT NULL,
+        text TEXT NOT NULL,
+        embedding TEXT,
+        metadata TEXT,
+        created_at TEXT DEFAULT CURRENT_TIMESTAMP
+      )
+    `);
+    this.db.exec(`
+      CREATE INDEX IF NOT EXISTS idx_${this.tableName}_document_id
+      ON ${this.tableName}(document_id)
+    `);
+    this.db.exec(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS ${this.tableName}_fts
+      USING fts5(id, text, content='${this.tableName}', content_rowid='rowid')
+    `);
+    this.db.exec(`
+      CREATE TRIGGER IF NOT EXISTS ${this.tableName}_ai AFTER INSERT ON ${this.tableName} BEGIN
+        INSERT INTO ${this.tableName}_fts(rowid, id, text) VALUES (new.rowid, new.id, new.text);
+      END
+    `);
+    this.db.exec(`
+      CREATE TRIGGER IF NOT EXISTS ${this.tableName}_ad AFTER DELETE ON ${this.tableName} BEGIN
+        INSERT INTO ${this.tableName}_fts(${this.tableName}_fts, rowid, id, text)
+        VALUES('delete', old.rowid, old.id, old.text);
+      END
+    `);
+  }
+  async insert(chunks) {
+    const stmt = this.db.prepare(`
+      INSERT OR REPLACE INTO ${this.tableName} (id, document_id, text, embedding, metadata, created_at)
+      VALUES (?, ?, ?, ?, ?, ?)
+    `);
+    const insertMany = this.db.transaction((items) => {
+      for (const chunk of items) {
+        stmt.run(
+          chunk.id,
+          chunk.documentId,
+          chunk.text,
+          chunk.embedding ? JSON.stringify(chunk.embedding) : null,
+          JSON.stringify(chunk.metadata),
+          chunk.createdAt.toISOString()
+        );
+      }
+    });
+    insertMany(chunks);
+  }
+  async denseSearch(embedding, topK, filter) {
+    let query = `SELECT * FROM ${this.tableName} WHERE embedding IS NOT NULL`;
+    const params = [];
+    if (filter?.documentId) {
+      query += ` AND document_id = ?`;
+      params.push(filter.documentId);
+    }
+    const rows = this.db.prepare(query).all(...params);
+    const results = [];
+    for (const row of rows) {
+      const chunkEmbedding = JSON.parse(row.embedding);
+      const score = this.cosineSimilarity(embedding, chunkEmbedding);
+      results.push({
+        chunk: this.rowToChunk(row),
+        score
+      });
+    }
+    results.sort((a, b) => b.score - a.score);
+    return results.slice(0, topK).map((r) => ({
+      ...r,
+      searchType: "dense"
+    }));
+  }
+  async sparseSearch(query, topK, filter) {
+    let ftsQuery = `
+      SELECT c.*, bm25(${this.tableName}_fts) as score
+      FROM ${this.tableName}_fts fts
+      JOIN ${this.tableName} c ON fts.id = c.id
+      WHERE ${this.tableName}_fts MATCH ?
+    `;
+    const params = [this.escapeFTSQuery(query)];
+    if (filter?.documentId) {
+      ftsQuery += ` AND c.document_id = ?`;
+      params.push(filter.documentId);
+    }
+    ftsQuery += ` ORDER BY score LIMIT ?`;
+    params.push(topK);
+    try {
+      const rows = this.db.prepare(ftsQuery).all(...params);
+      return rows.map((row) => ({
+        chunk: this.rowToChunk(row),
+        score: Math.abs(row.score),
+        // BM25 returns negative scores
+        searchType: "sparse"
+      }));
+    } catch {
+      return [];
+    }
+  }
+  async deleteByDocumentId(documentId) {
+    const result = this.db.prepare(`DELETE FROM ${this.tableName} WHERE document_id = ?`).run(documentId);
+    return result.changes;
+  }
+  async getById(id) {
+    const row = this.db.prepare(`SELECT * FROM ${this.tableName} WHERE id = ?`).get(id);
+    return row ? this.rowToChunk(row) : null;
+  }
+  async close() {
+    this.db.close();
+  }
+  /**
+   * Calculate cosine similarity between two vectors
+   */
+  cosineSimilarity(a, b) {
+    if (a.length !== b.length) {
+      throw new Error("Vectors must have same length");
+    }
+    let dotProduct = 0;
+    let normA = 0;
+    let normB = 0;
+    for (let i = 0; i < a.length; i++) {
+      dotProduct += a[i] * b[i];
+      normA += a[i] * a[i];
+      normB += b[i] * b[i];
+    }
+    const denominator = Math.sqrt(normA) * Math.sqrt(normB);
+    return denominator === 0 ? 0 : dotProduct / denominator;
+  }
+  /**
+   * Convert database row to DocumentChunk
+   */
+  rowToChunk(row) {
+    return {
+      id: row.id,
+      documentId: row.document_id,
+      text: row.text,
+      embedding: row.embedding ? JSON.parse(row.embedding) : void 0,
+      metadata: JSON.parse(row.metadata),
+      createdAt: new Date(row.created_at)
+    };
+  }
+  /**
+   * Escape FTS query to prevent syntax errors
+   */
+  escapeFTSQuery(query) {
+    const cleaned = query.replace(/['"(){}[\]^~*?:\\]/g, " ").trim();
+    return cleaned.split(/\s+/).filter((w) => w.length > 0).map((w) => `"${w}"`).join(" OR ");
+  }
+};
+
+// src/vectorstore/postgres.ts
+import { Pool } from "pg";
+var PostgresVectorStore = class {
+  pool;
+  tableName;
+  schema;
+  dimensions;
+  fullTableName;
+  constructor(options) {
+    const poolConfig = options.poolConfig || {
+      connectionString: options.connectionString,
+      max: 10,
+      idleTimeoutMillis: 3e4
+    };
+    this.pool = new Pool(poolConfig);
+    this.tableName = options.tableName || "chunks";
+    this.schema = options.schema || "public";
+    this.dimensions = options.dimensions;
+    this.fullTableName = `${this.schema}.${this.tableName}`;
+  }
+  async initialize() {
+    const client = await this.pool.connect();
+    try {
+      await client.query("CREATE EXTENSION IF NOT EXISTS vector");
+      await client.query("CREATE EXTENSION IF NOT EXISTS pg_trgm");
+      await client.query(`
+        CREATE TABLE IF NOT EXISTS ${this.schema}.documents (
+          id TEXT PRIMARY KEY,
+          source_id TEXT NOT NULL,
+          path TEXT NOT NULL,
+          title TEXT NOT NULL,
+          metadata JSONB DEFAULT '{}',
+          created_at TIMESTAMPTZ DEFAULT NOW(),
+          updated_at TIMESTAMPTZ DEFAULT NOW()
+        )
+      `);
+      await client.query(`
+        CREATE TABLE IF NOT EXISTS ${this.fullTableName} (
+          id TEXT PRIMARY KEY,
+          document_id TEXT NOT NULL REFERENCES ${this.schema}.documents(id) ON DELETE CASCADE,
+          text TEXT NOT NULL,
+          embedding vector(${this.dimensions}),
+          metadata JSONB DEFAULT '{}',
+          created_at TIMESTAMPTZ DEFAULT NOW()
+        )
+      `);
+      await client.query(`
+        CREATE INDEX IF NOT EXISTS idx_${this.tableName}_document_id
+        ON ${this.fullTableName}(document_id)
+      `);
+      await client.query(`
+        CREATE INDEX IF NOT EXISTS idx_${this.tableName}_embedding_hnsw
+        ON ${this.fullTableName}
+        USING hnsw (embedding vector_cosine_ops)
+        WITH (m = 16, ef_construction = 64)
+      `);
+      await client.query(`
+        CREATE INDEX IF NOT EXISTS idx_${this.tableName}_text_trgm
+        ON ${this.fullTableName}
+        USING gin (text gin_trgm_ops)
+      `);
+      await client.query(`
+        CREATE INDEX IF NOT EXISTS idx_${this.tableName}_metadata
+        ON ${this.fullTableName}
+        USING gin (metadata)
+      `);
+    } finally {
+      client.release();
+    }
+  }
+  async insert(chunks) {
+    if (chunks.length === 0) return;
+    const client = await this.pool.connect();
+    try {
+      await client.query("BEGIN");
+      for (const chunk of chunks) {
+        const embedding = chunk.embedding ? `[${chunk.embedding.join(",")}]` : null;
+        await client.query(
+          `
+          INSERT INTO ${this.fullTableName} (id, document_id, text, embedding, metadata, created_at)
+          VALUES ($1, $2, $3, $4::vector, $5, $6)
+          ON CONFLICT (id) DO UPDATE SET
+            text = EXCLUDED.text,
+            embedding = EXCLUDED.embedding,
+            metadata = EXCLUDED.metadata
+        `,
+          [
+            chunk.id,
+            chunk.documentId,
+            chunk.text,
+            embedding,
+            JSON.stringify(chunk.metadata),
+            chunk.createdAt
+          ]
+        );
+      }
+      await client.query("COMMIT");
+    } catch (error) {
+      await client.query("ROLLBACK");
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+  async denseSearch(embedding, topK, filter) {
+    const embeddingStr = `[${embedding.join(",")}]`;
+    let query = `
+      SELECT
+        id,
+        document_id,
+        text,
+        metadata,
+        created_at,
+        1 - (embedding <=> $1::vector) as score
+      FROM ${this.fullTableName}
+      WHERE embedding IS NOT NULL
+    `;
+    const params = [embeddingStr];
+    let paramIndex = 2;
+    if (filter?.documentId) {
+      query += ` AND document_id = $${paramIndex}`;
+      params.push(filter.documentId);
+      paramIndex++;
+    }
+    if (filter?.metadata) {
+      query += ` AND metadata @> $${paramIndex}`;
+      params.push(JSON.stringify(filter.metadata));
+      paramIndex++;
+    }
+    query += ` ORDER BY embedding <=> $1::vector LIMIT $${paramIndex}`;
+    params.push(topK);
+    const result = await this.pool.query(query, params);
+    return result.rows.map((row) => ({
+      chunk: this.rowToChunk(row),
+      score: row.score,
+      searchType: "dense"
+    }));
+  }
+  async sparseSearch(query, topK, filter) {
+    let sql = `
+      SELECT
+        id,
+        document_id,
+        text,
+        metadata,
+        created_at,
+        similarity(text, $1) as score
+      FROM ${this.fullTableName}
+      WHERE text % $1
+    `;
+    const params = [query];
+    let paramIndex = 2;
+    if (filter?.documentId) {
+      sql += ` AND document_id = $${paramIndex}`;
+      params.push(filter.documentId);
+      paramIndex++;
+    }
+    sql += ` ORDER BY score DESC LIMIT $${paramIndex}`;
+    params.push(topK);
+    const result = await this.pool.query(sql, params);
+    return result.rows.map((row) => ({
+      chunk: this.rowToChunk(row),
+      score: row.score,
+      searchType: "sparse"
+    }));
+  }
+  async deleteByDocumentId(documentId) {
+    const result = await this.pool.query(
+      `DELETE FROM ${this.fullTableName} WHERE document_id = $1`,
+      [documentId]
+    );
+    return result.rowCount || 0;
+  }
+  async getById(id) {
+    const result = await this.pool.query(
+      `SELECT * FROM ${this.fullTableName} WHERE id = $1`,
+      [id]
+    );
+    return result.rows.length > 0 ? this.rowToChunk(result.rows[0]) : null;
+  }
+  async close() {
+    await this.pool.end();
+  }
+  /**
+   * Get chunk and document counts
+   */
+  async getStats() {
+    const chunksResult = await this.pool.query(
+      `SELECT COUNT(*) FROM ${this.fullTableName}`
+    );
+    const docsResult = await this.pool.query(
+      `SELECT COUNT(*) FROM ${this.schema}.documents`
+    );
+    return {
+      chunks: parseInt(chunksResult.rows[0].count, 10),
+      documents: parseInt(docsResult.rows[0].count, 10)
+    };
+  }
+  /**
+   * Insert or update a document
+   */
+  async upsertDocument(doc) {
+    await this.pool.query(
+      `
+      INSERT INTO ${this.schema}.documents (id, source_id, path, title, metadata)
+      VALUES ($1, $2, $3, $4, $5)
+      ON CONFLICT (id) DO UPDATE SET
+        path = EXCLUDED.path,
+        title = EXCLUDED.title,
+        metadata = EXCLUDED.metadata,
+        updated_at = NOW()
+    `,
+      [doc.id, doc.sourceId, doc.path, doc.title, JSON.stringify(doc.metadata || {})]
+    );
+  }
+  rowToChunk(row) {
+    return {
+      id: row.id,
+      documentId: row.document_id,
+      text: row.text,
+      metadata: row.metadata,
+      createdAt: new Date(row.created_at)
+    };
+  }
+};
+
+// src/vectorstore/index.ts
+function createVectorStore(options) {
+  switch (options.type) {
+    case "sqlite":
+      return new SQLiteVectorStore({
+        path: options.connectionString || ":memory:",
+        tableName: options.tableName,
+        dimensions: options.dimensions
+      });
+    case "postgres":
+      if (!options.connectionString) {
+        throw new Error("PostgreSQL requires a connection string");
+      }
+      return new PostgresVectorStore({
+        connectionString: options.connectionString,
+        tableName: options.tableName,
+        dimensions: options.dimensions
+      });
+    case "memory":
+      return new SQLiteVectorStore({
+        path: ":memory:",
+        tableName: options.tableName,
+        dimensions: options.dimensions
+      });
+    default:
+      throw new Error(`Unknown vector store type: ${options.type}`);
+  }
+}
+
+// src/retrieval/hybrid.ts
+var HybridRetriever = class {
+  vectorStore;
+  embeddings;
+  defaultTopK;
+  denseWeight;
+  sparseWeight;
+  rrfK;
+  constructor(options) {
+    this.vectorStore = options.vectorStore;
+    this.embeddings = options.embeddings;
+    this.defaultTopK = options.defaultTopK || 10;
+    this.denseWeight = options.denseWeight ?? 0.7;
+    this.sparseWeight = options.sparseWeight ?? 0.3;
+    this.rrfK = options.rrfK || 60;
+  }
+  /**
+   * Perform hybrid search combining dense and sparse retrieval
+   */
+  async search(query, options) {
+    const startTime = Date.now();
+    const topK = options?.topK || this.defaultTopK;
+    const filter = options?.filter;
+    const denseWeight = options?.denseWeight ?? this.denseWeight;
+    const sparseWeight = options?.sparseWeight ?? this.sparseWeight;
+    const candidateK = Math.min(topK * 3, 100);
+    const [denseResults, sparseResults] = await Promise.all([
+      this.denseSearch(query, candidateK, filter),
+      this.sparseSearch(query, candidateK, filter)
+    ]);
+    const fusedResults = this.rrfFusion(
+      denseResults,
+      sparseResults,
+      denseWeight,
+      sparseWeight
+    );
+    const finalResults = fusedResults.slice(0, topK);
+    return {
+      results: finalResults,
+      stats: {
+        denseCount: denseResults.length,
+        sparseCount: sparseResults.length,
+        rerankingApplied: false,
+        totalTime: Date.now() - startTime
+      }
+    };
+  }
+  /**
+   * Dense (vector) search
+   */
+  async denseSearch(query, topK, filter) {
+    try {
+      const embeddingResult = await this.embeddings.embed(query);
+      return await this.vectorStore.denseSearch(embeddingResult.embedding, topK, filter);
+    } catch (error) {
+      console.warn("Dense search failed:", error);
+      return [];
+    }
+  }
+  /**
+   * Sparse (BM25/text) search
+   */
+  async sparseSearch(query, topK, filter) {
+    try {
+      return await this.vectorStore.sparseSearch(query, topK, filter);
+    } catch (error) {
+      console.warn("Sparse search failed:", error);
+      return [];
+    }
+  }
+  /**
+   * Reciprocal Rank Fusion
+   *
+   * Combines multiple ranked lists into a single ranking.
+   * Formula: score = sum(1 / (k + rank_i)) for each list i
+   */
+  rrfFusion(denseResults, sparseResults, denseWeight, sparseWeight) {
+    const k = this.rrfK;
+    const scores = /* @__PURE__ */ new Map();
+    denseResults.forEach((result, rank) => {
+      const rrfScore = denseWeight * 1 / (k + rank + 1);
+      const existing = scores.get(result.chunk.id);
+      if (existing) {
+        existing.score += rrfScore;
+      } else {
+        scores.set(result.chunk.id, {
+          score: rrfScore,
+          result: { ...result, searchType: "hybrid" }
+        });
+      }
+    });
+    sparseResults.forEach((result, rank) => {
+      const rrfScore = sparseWeight * 1 / (k + rank + 1);
+      const existing = scores.get(result.chunk.id);
+      if (existing) {
+        existing.score += rrfScore;
+      } else {
+        scores.set(result.chunk.id, {
+          score: rrfScore,
+          result: { ...result, searchType: "hybrid" }
+        });
+      }
+    });
+    const combined = Array.from(scores.values()).sort((a, b) => b.score - a.score).map((entry) => ({
+      ...entry.result,
+      score: entry.score
+    }));
+    return combined;
+  }
+};
+var DenseRetriever = class {
+  vectorStore;
+  embeddings;
+  defaultTopK;
+  constructor(options) {
+    this.vectorStore = options.vectorStore;
+    this.embeddings = options.embeddings;
+    this.defaultTopK = options.defaultTopK || 10;
+  }
+  async search(query, options) {
+    const startTime = Date.now();
+    const topK = options?.topK || this.defaultTopK;
+    const embeddingResult = await this.embeddings.embed(query);
+    const results = await this.vectorStore.denseSearch(
+      embeddingResult.embedding,
+      topK,
+      options?.filter
+    );
+    return {
+      results,
+      stats: {
+        denseCount: results.length,
+        sparseCount: 0,
+        rerankingApplied: false,
+        totalTime: Date.now() - startTime
+      }
+    };
+  }
+};
+
+// src/rag-service.ts
+var RAG = class {
+  constructor(options) {
+    this.options = options;
+    this.embeddings = createEmbeddingBackend(options.embeddings);
+    this.vectorStore = createVectorStore({
+      ...options.vectorStore,
+      dimensions: this.embeddings.getDimensions()
+    });
+    this.retriever = new HybridRetriever({
+      vectorStore: this.vectorStore,
+      embeddings: this.embeddings,
+      defaultTopK: options.retrieval?.topK || 8,
+      denseWeight: options.retrieval?.denseWeight,
+      sparseWeight: options.retrieval?.sparseWeight,
+      rrfK: options.retrieval?.rrfK
+    });
+    this.chunker = new HeaderAwareChunker(options.chunking);
+    this.llm = this.createLLM(options.llm);
+    this.systemPrompt = options.systemPrompt || `You are a helpful documentation assistant. Answer questions based on the provided context.
+Always cite your sources using [1], [2], etc. format when referencing specific information.
+If the context doesn't contain enough information to answer, say so clearly.
+Be concise and accurate.`;
+  }
+  vectorStore;
+  embeddings;
+  retriever;
+  llm;
+  chunker;
+  systemPrompt;
+  initialized = false;
+  createLLM(config) {
+    switch (config.provider) {
+      case "openai":
+      case "cerebras":
+      case "groq":
+        return new OpenAIProvider({
+          apiKey: config.apiKey || process.env.OPENAI_API_KEY || "",
+          baseUrl: config.baseUrl,
+          model: config.model || "gpt-4o-mini"
+        });
+      case "anthropic":
+        return new AnthropicProvider({
+          apiKey: config.apiKey || process.env.ANTHROPIC_API_KEY || "",
+          model: config.model || "claude-sonnet-4-20250514"
+        });
+      default:
+        throw new Error(`Unknown LLM provider: ${config.provider}`);
+    }
+  }
+  /**
+   * Initialize the RAG service
+   */
+  async initialize() {
+    if (this.initialized) return;
+    await this.vectorStore.initialize();
+    this.initialized = true;
+  }
+  /**
+   * Index documents from a path
+   */
+  async index(path, options) {
+    await this.initialize();
+    const sourceId = options?.sourceId || generateId2("source");
+    console.log(`[RAG] Indexing documents from: ${path}`);
+    const documents = await extractDocuments(path);
+    console.log(`[RAG] Extracted ${documents.length} documents`);
+    let totalChunks = 0;
+    for (const doc of documents) {
+      const documentId = generateId2("doc");
+      const { chunks, stats } = this.chunker.chunk(doc, documentId);
+      console.log(
+        `[RAG] Chunked "${doc.title}": ${stats.totalChunks} chunks (avg: ${stats.avgChunkSize} chars)`
+      );
+      if (chunks.length === 0) continue;
+      console.log(`[RAG] Generating embeddings for ${chunks.length} chunks...`);
+      const texts = chunks.map((c) => c.text);
+      const embeddingResults = await this.embeddings.embedBatch(texts);
+      for (let i = 0; i < chunks.length; i++) {
+        chunks[i].embedding = embeddingResults[i].embedding;
+      }
+      await this.vectorStore.insert(chunks);
+      totalChunks += chunks.length;
+    }
+    console.log(`[RAG] Indexing complete: ${documents.length} docs, ${totalChunks} chunks`);
+    return {
+      documentsIndexed: documents.length,
+      chunksCreated: totalChunks
+    };
+  }
+  /**
+   * Ask a question and get an answer with citations
+   */
+  async ask(question, options) {
+    await this.initialize();
+    const startTime = Date.now();
+    const retrievalStart = Date.now();
+    const { results, stats } = await this.retriever.search(question, options);
+    const retrievalTime = Date.now() - retrievalStart;
+    if (results.length === 0) {
+      return {
+        question,
+        answer: "I couldn't find any relevant information in the documentation to answer your question.",
+        citations: [],
+        context: [],
+        metadata: {
+          totalTime: Date.now() - startTime,
+          retrievalTime,
+          generationTime: 0,
+          cached: false,
+          model: this.llm.name || "unknown"
+        }
+      };
+    }
+    const context = results.map((r) => r.chunk);
+    const contextText = this.buildContextText(context);
+    const generationStart = Date.now();
+    const answer = await this.generateAnswer(question, contextText);
+    const generationTime = Date.now() - generationStart;
+    const citations = this.extractCitations(answer, context);
+    return {
+      question,
+      answer,
+      citations,
+      context,
+      metadata: {
+        totalTime: Date.now() - startTime,
+        retrievalTime,
+        generationTime,
+        cached: false,
+        model: this.llm.name || "unknown"
+      }
+    };
+  }
+  /**
+   * Build context text from chunks
+   */
+  buildContextText(chunks) {
+    return chunks.map((chunk, i) => {
+      const header = chunk.metadata.sectionPath || chunk.metadata.headingH1 || "Document";
+      return `[${i + 1}] ${header}
+${chunk.text}`;
+    }).join("\n\n---\n\n");
+  }
+  /**
+   * Generate answer using LLM
+   */
+  async generateAnswer(question, context) {
+    const prompt = `Based on the following documentation context, answer the user's question.
+Cite sources using [1], [2], etc. format when referencing specific information.
+
+CONTEXT:
+${context}
+
+QUESTION: ${question}
+
+ANSWER:`;
+    const response = await this.llm.complete({
+      messages: [
+        {
+          id: generateId2("msg"),
+          role: "user",
+          content: [{ type: "text", text: prompt }],
+          timestamp: /* @__PURE__ */ new Date()
+        }
+      ],
+      systemPrompt: this.systemPrompt,
+      maxTokens: 1e3
+    });
+    return response.content;
+  }
+  /**
+   * Extract citations from answer text
+   */
+  extractCitations(answer, context) {
+    const citations = [];
+    const citationRegex = /\[(\d+)\]/g;
+    const matches = answer.matchAll(citationRegex);
+    const seenIndexes = /* @__PURE__ */ new Set();
+    for (const match of matches) {
+      const index = parseInt(match[1], 10);
+      if (!seenIndexes.has(index) && index > 0 && index <= context.length) {
+        seenIndexes.add(index);
+        const chunk = context[index - 1];
+        citations.push({
+          index,
+          chunkId: chunk.id,
+          sectionPath: chunk.metadata.sectionPath,
+          heading: chunk.metadata.headingH3 || chunk.metadata.headingH2 || chunk.metadata.headingH1,
+          snippet: chunk.text.substring(0, 200) + (chunk.text.length > 200 ? "..." : "")
+        });
+      }
+    }
+    return citations.sort((a, b) => a.index - b.index);
+  }
+  /**
+   * Get document count
+   */
+  async getDocumentCount() {
+    return 0;
+  }
+  /**
+   * Get chunk count
+   */
+  async getChunkCount() {
+    return 0;
+  }
+  /**
+   * Clear all indexed data
+   */
+  async clear() {
+    console.log("[RAG] Clear not implemented yet");
+  }
+  /**
+   * Close connections
+   */
+  async close() {
+    await this.vectorStore.close();
+  }
+};
+async function createRAG(options) {
+  const rag = new RAG(options);
+  await rag.initialize();
+  return rag;
+}
+export {
+  BaseExtractor,
+  CHMExtractor,
+  DenseRetriever,
+  HTMLExtractor,
+  HeaderAwareChunker,
+  HybridRetriever,
+  LOCAL_MODELS,
+  LocalEmbeddingBackend,
+  MarkdownExtractor,
+  OPENAI_MODELS,
+  OpenAIEmbeddingBackend,
+  PostgresVectorStore,
+  RAG,
+  SQLiteVectorStore,
+  createEmbeddingBackend,
+  createRAG,
+  createVectorStore,
+  extractDocuments
+};
+//# sourceMappingURL=index.js.map