@voltx/rag 0.1.0 → 0.3.0

package/dist/index.js

@@ -21,20 +21,29 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 var index_exports = {};
 __export(index_exports, {
   CharacterSplitter: () => CharacterSplitter,
+  JSONLoader: () => JSONLoader,
+  MDocument: () => MDocument,
+  MarkdownLoader: () => MarkdownLoader,
+  MarkdownSplitter: () => MarkdownSplitter,
   RAGPipeline: () => RAGPipeline,
+  RecursiveTextSplitter: () => RecursiveTextSplitter,
+  TextLoader: () => TextLoader,
   VERSION: () => VERSION,
+  WebLoader: () => WebLoader,
+  cosineSimilarity: () => cosineSimilarity,
+  createEmbedder: () => createEmbedder,
   createRAGPipeline: () => createRAGPipeline
 });
 module.exports = __toCommonJS(index_exports);
+
+// src/splitters.ts
 var CharacterSplitter = class {
-  constructor(chunkSize = 1e3, overlap = 200) {
-    this.chunkSize = chunkSize;
-    this.overlap = overlap;
+  chunkSize;
+  overlap;
+  constructor(options = {}) {
+    this.chunkSize = options.chunkSize ?? 1e3;
+    this.overlap = options.overlap ?? 200;
   }
-  /**
-   * Find the best split point near `pos` by looking for sentence/paragraph
-   * boundaries first, then word boundaries, falling back to exact position.
-   */
   findBreakPoint(text, pos) {
     if (pos >= text.length) return text.length;
     const searchStart = Math.max(0, pos - Math.floor(this.chunkSize * 0.2));
@@ -48,9 +57,7 @@ var CharacterSplitter = class {
       }
     }
     for (let i = pos; i >= searchStart; i--) {
-      if (/\s/.test(text[i])) {
-        return i + 1;
-      }
+      if (/\s/.test(text[i])) return i + 1;
     }
     return pos;
   }
@@ -61,58 +68,503 @@ var CharacterSplitter = class {
     while (start < text.length) {
       const rawEnd = Math.min(start + this.chunkSize, text.length);
       const end = rawEnd >= text.length ? rawEnd : this.findBreakPoint(text, rawEnd);
-      chunks.push({
-        id: `chunk-${index++}`,
-        content: text.slice(start, end).trim(),
-        metadata: { start, end }
-      });
+      const content = text.slice(start, end).trim();
+      if (content.length > 0) {
+        chunks.push({
+          id: `chunk-${index++}`,
+          content,
+          metadata: { start, end, splitter: "character" }
+        });
+      }
       const step = end - start - this.overlap;
       start += step > 0 ? step : end - start;
     }
+    return chunks;
+  }
+};
+var RecursiveTextSplitter = class {
+  chunkSize;
+  overlap;
+  separators;
+  constructor(options = {}) {
+    this.chunkSize = options.chunkSize ?? 1e3;
+    this.overlap = options.overlap ?? 200;
+    this.separators = options.separators ?? ["\n\n", "\n", ". ", " ", ""];
+  }
+  split(text) {
+    const rawChunks = this.splitText(text, this.separators);
+    const merged = this.mergeWithOverlap(rawChunks);
+    return merged.map((content, i) => ({
+      id: `chunk-${i}`,
+      content,
+      metadata: { index: i, splitter: "recursive" }
+    }));
+  }
+  /**
+   * Recursively split text. Try the first separator; if any resulting piece
+   * is still too large, recurse with the next separator in the list.
+   */
+  splitText(text, separators) {
+    const results = [];
+    let bestSep = "";
+    let bestIdx = 0;
+    for (let i = 0; i < separators.length; i++) {
+      const sep = separators[i];
+      if (sep === "") {
+        bestSep = sep;
+        bestIdx = i;
+        break;
+      }
+      if (text.includes(sep)) {
+        bestSep = sep;
+        bestIdx = i;
+        break;
+      }
+    }
+    const pieces = bestSep === "" ? [...text] : text.split(bestSep);
+    const remainingSeps = separators.slice(bestIdx + 1);
+    let current = "";
+    for (const piece of pieces) {
+      const candidate = current ? current + bestSep + piece : piece;
+      if (candidate.length <= this.chunkSize) {
+        current = candidate;
+      } else {
+        if (current.trim()) {
+          results.push(current.trim());
+        }
+        if (piece.length > this.chunkSize && remainingSeps.length > 0) {
+          const subChunks = this.splitText(piece, remainingSeps);
+          results.push(...subChunks);
+          current = "";
+        } else {
+          current = piece;
+        }
+      }
+    }
+    if (current.trim()) {
+      results.push(current.trim());
+    }
+    return results;
+  }
+  /**
+   * Merge chunks with overlap to maintain context between adjacent chunks.
+   */
+  mergeWithOverlap(chunks) {
+    if (this.overlap <= 0 || chunks.length <= 1) return chunks;
+    const result = [];
+    for (let i = 0; i < chunks.length; i++) {
+      if (i === 0) {
+        result.push(chunks[i]);
+      } else {
+        const prev = chunks[i - 1];
+        const overlapText = prev.slice(-this.overlap);
+        const spaceIdx = overlapText.indexOf(" ");
+        const cleanOverlap = spaceIdx >= 0 ? overlapText.slice(spaceIdx + 1) : overlapText;
+        result.push((cleanOverlap + " " + chunks[i]).trim());
+      }
+    }
+    return result;
+  }
+};
+var MarkdownSplitter = class {
+  chunkSize;
+  overlap;
+  includeHeaders;
+  constructor(options = {}) {
+    this.chunkSize = options.chunkSize ?? 1500;
+    this.overlap = options.overlap ?? 100;
+    this.includeHeaders = options.includeHeaders ?? true;
+  }
+  split(text) {
+    const sections = this.splitByHeadings(text);
+    const chunks = [];
+    let index = 0;
+    const fallback = new RecursiveTextSplitter({
+      chunkSize: this.chunkSize,
+      overlap: this.overlap
+    });
+    for (const section of sections) {
+      if (section.content.length <= this.chunkSize) {
+        chunks.push({
+          id: `chunk-${index++}`,
+          content: section.content.trim(),
+          metadata: {
+            ...section.headers,
+            splitter: "markdown"
+          }
+        });
+      } else {
+        const subChunks = fallback.split(section.content);
+        for (const sub of subChunks) {
+          chunks.push({
+            id: `chunk-${index++}`,
+            content: sub.content.trim(),
+            metadata: {
+              ...section.headers,
+              ...sub.metadata,
+              splitter: "markdown"
+            }
+          });
+        }
+      }
+    }
     return chunks.filter((c) => c.content.length > 0);
   }
+  splitByHeadings(text) {
+    const lines = text.split("\n");
+    const sections = [];
+    const headerStack = {};
+    let currentContent = "";
+    for (const line of lines) {
+      const headerMatch = line.match(/^(#{1,6})\s+(.+)$/);
+      if (headerMatch) {
+        if (currentContent.trim()) {
+          sections.push({
+            content: this.includeHeaders ? this.buildHeaderPrefix(headerStack) + currentContent.trim() : currentContent.trim(),
+            headers: { ...headerStack }
+          });
+        }
+        const level = headerMatch[1].length;
+        const title = headerMatch[2].trim();
+        for (let i = level; i <= 6; i++) {
+          delete headerStack[`h${i}`];
+        }
+        headerStack[`h${level}`] = title;
+        currentContent = "";
+      } else {
+        currentContent += line + "\n";
+      }
+    }
+    if (currentContent.trim()) {
+      sections.push({
+        content: this.includeHeaders ? this.buildHeaderPrefix(headerStack) + currentContent.trim() : currentContent.trim(),
+        headers: { ...headerStack }
+      });
+    }
+    return sections;
+  }
+  buildHeaderPrefix(headers) {
+    const parts = [];
+    for (let i = 1; i <= 6; i++) {
+      const h = headers[`h${i}`];
+      if (h) parts.push(h);
+    }
+    return parts.length > 0 ? parts.join(" > ") + "\n\n" : "";
+  }
+};
+
+// src/loaders.ts
+var import_promises = require("fs/promises");
+var import_node_fs = require("fs");
+var TextLoader = class {
+  name = "text";
+  async load(source) {
+    if ((0, import_node_fs.existsSync)(source)) {
+      return (0, import_promises.readFile)(source, "utf-8");
+    }
+    return source;
+  }
+};
+var MarkdownLoader = class {
+  name = "markdown";
+  async load(source) {
+    let text;
+    if ((0, import_node_fs.existsSync)(source)) {
+      text = await (0, import_promises.readFile)(source, "utf-8");
+    } else {
+      text = source;
+    }
+    const frontMatterRegex = /^---\s*\n[\s\S]*?\n---\s*\n/;
+    return text.replace(frontMatterRegex, "").trim();
+  }
+};
+var JSONLoader = class {
+  name = "json";
+  textKeys;
+  separator;
+  constructor(options = {}) {
+    this.textKeys = options.textKeys ?? ["content", "text", "body", "description"];
+    this.separator = options.separator ?? "\n\n";
+  }
+  async load(source) {
+    let raw;
+    if ((0, import_node_fs.existsSync)(source)) {
+      raw = await (0, import_promises.readFile)(source, "utf-8");
+    } else {
+      raw = source;
+    }
+    const data = JSON.parse(raw);
+    const texts = this.extractTexts(data);
+    return texts.join(this.separator);
+  }
+  extractTexts(data) {
+    const results = [];
+    if (Array.isArray(data)) {
+      for (const item of data) {
+        results.push(...this.extractTexts(item));
+      }
+    } else if (data !== null && typeof data === "object") {
+      const obj = data;
+      for (const key of this.textKeys) {
+        if (key in obj && typeof obj[key] === "string") {
+          results.push(obj[key]);
+        }
+      }
+      if (results.length === 0) {
+        for (const value of Object.values(obj)) {
+          if (typeof value === "string" && value.length > 20) {
+            results.push(value);
+          } else if (typeof value === "object" && value !== null) {
+            results.push(...this.extractTexts(value));
+          }
+        }
+      }
+    } else if (typeof data === "string") {
+      results.push(data);
+    }
+    return results;
+  }
+};
+var WebLoader = class {
+  name = "web";
+  async load(source) {
+    const response = await fetch(source);
+    if (!response.ok) {
+      throw new Error(`[voltx/rag] WebLoader failed to fetch ${source}: ${response.status}`);
+    }
+    const contentType = response.headers.get("content-type") ?? "";
+    const text = await response.text();
+    if (contentType.includes("text/html")) {
+      return this.stripHTML(text);
+    }
+    return text;
+  }
+  stripHTML(html) {
+    return html.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&nbsp;/g, " ").replace(/\s+/g, " ").trim();
+  }
+};
+
+// src/embedder.ts
+var import_ai = require("@voltx/ai");
+function createEmbedder(config) {
+  const { model } = config;
+  return {
+    name: `voltx-embedder:${model}`,
+    async embed(text) {
+      const result = await (0, import_ai.embed)({ model, value: text });
+      return result.embedding;
+    },
+    async embedBatch(texts) {
+      if (texts.length === 0) return [];
+      if (texts.length === 1) {
+        const result2 = await (0, import_ai.embed)({ model, value: texts[0] });
+        return [result2.embedding];
+      }
+      const result = await (0, import_ai.embedMany)({ model, values: texts });
+      return result.embeddings;
+    }
+  };
+}
+
+// src/document.ts
+var MDocument = class _MDocument {
+  content;
+  format;
+  chunks = null;
+  constructor(content, format) {
+    this.content = content;
+    this.format = format;
+  }
+  /** Create from plain text */
+  static fromText(content) {
+    return new _MDocument(content, "text");
+  }
+  /** Create from markdown */
+  static fromMarkdown(content) {
+    return new _MDocument(content, "markdown");
+  }
+  /** Create from JSON string */
+  static fromJSON(content) {
+    JSON.parse(content);
+    return new _MDocument(content, "json");
+  }
+  /** Create from HTML (strips tags) */
+  static fromHTML(html) {
+    const text = html.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&nbsp;/g, " ").replace(/\s+/g, " ").trim();
+    return new _MDocument(text, "html");
+  }
+  /** Get the raw content */
+  getContent() {
+    return this.content;
+  }
+  /** Get the document format */
+  getFormat() {
+    return this.format;
+  }
+  /**
+   * Split the document into chunks using the specified strategy.
+   * Returns the chunks and caches them for subsequent embed() calls.
+   */
+  chunk(options = {}) {
+    const strategy = options.strategy ?? this.defaultStrategy();
+    const splitter = this.createSplitter(strategy, options);
+    this.chunks = splitter.split(this.content);
+    return this.chunks;
+  }
+  /**
+   * Embed the chunks using the provided embedder.
+   * Must call chunk() first.
+   */
+  async embed(embedder) {
+    if (!this.chunks) {
+      throw new Error("[voltx/rag] Call chunk() before embed()");
+    }
+    const texts = this.chunks.map((c) => c.content);
+    const embeddings = await embedder.embedBatch(texts);
+    for (let i = 0; i < this.chunks.length; i++) {
+      this.chunks[i].embedding = embeddings[i];
+    }
+    return this.chunks;
+  }
+  /** Get cached chunks (null if chunk() hasn't been called) */
+  getChunks() {
+    return this.chunks;
+  }
+  defaultStrategy() {
+    if (this.format === "markdown") return "markdown";
+    return "recursive";
+  }
+  createSplitter(strategy, options) {
+    switch (strategy) {
+      case "markdown":
+        return new MarkdownSplitter({
+          chunkSize: options.chunkSize,
+          overlap: options.overlap,
+          includeHeaders: options.includeHeaders
+        });
+      case "character":
+        return new CharacterSplitter({
+          chunkSize: options.chunkSize,
+          overlap: options.overlap
+        });
+      case "recursive":
+      default:
+        return new RecursiveTextSplitter({
+          chunkSize: options.chunkSize,
+          overlap: options.overlap,
+          separators: options.separators
+        });
+    }
+  }
 };
+
+// src/utils.ts
+function cosineSimilarity(a, b) {
+  if (a.length !== b.length) {
+    throw new Error(
+      `[voltx/rag] Vector dimension mismatch: ${a.length} vs ${b.length}`
+    );
+  }
+  let dotProduct = 0;
+  let normA = 0;
+  let normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dotProduct += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  const magnitude = Math.sqrt(normA) * Math.sqrt(normB);
+  if (magnitude === 0) return 0;
+  return dotProduct / magnitude;
+}
+
+// src/index.ts
 var RAGPipeline = class {
-  config;
+  loader;
+  splitter;
+  embedder;
+  vectorStore;
   constructor(config) {
-    this.config = config;
-  }
-  /** Ingest a document: load → split → embed → store */
-  async ingest(source) {
-    const { loader, splitter = new CharacterSplitter(), embedder, vectorStore } = this.config;
-    const text = loader ? await loader.load(source) : source;
-    const chunks = splitter.split(text);
-    const docs = [];
-    for (const chunk of chunks) {
-      const embedding = await embedder.embed(chunk.content);
-      docs.push({
-        id: chunk.id,
-        content: chunk.content,
-        embedding,
-        metadata: chunk.metadata
-      });
-    }
-    await vectorStore.upsert(docs);
-    return docs.length;
+    this.loader = config.loader;
+    this.splitter = config.splitter ?? new RecursiveTextSplitter();
+    this.embedder = config.embedder;
+    this.vectorStore = config.vectorStore;
   }
-  /** Query: embed question → search vector store → return sources */
-  async query(question, topK = 5) {
-    const { embedder, vectorStore } = this.config;
-    const embedding = await embedder.embed(question);
-    const results = await vectorStore.search(embedding, topK);
+  /**
+   * Ingest a document: load → split → embed (batch) → store in vector DB.
+   *
+   * @param source - File path, URL, or raw text (depends on loader)
+   * @param idPrefix - Optional prefix for chunk IDs (default: "doc")
+   * @returns Number of chunks ingested and their IDs
+   */
+  async ingest(source, idPrefix = "doc") {
+    const text = this.loader ? await this.loader.load(source) : source;
+    const chunks = this.splitter.split(text);
+    const texts = chunks.map((c) => c.content);
+    const embeddings = await this.embedder.embedBatch(texts);
+    const docs = chunks.map((chunk, i) => ({
+      id: `${idPrefix}-${chunk.id}`,
+      content: chunk.content,
+      embedding: embeddings[i],
+      metadata: chunk.metadata
+    }));
+    await this.vectorStore.upsert(docs);
     return {
-      sources: results.map((r) => r.document)
+      chunks: docs.length,
+      ids: docs.map((d) => d.id)
     };
   }
+  /**
+   * Query: embed question → search vector store → return ranked sources.
+   *
+   * @param question - The user's question
+   * @param options - Query options (topK, minScore)
+   */
+  async query(question, options = {}) {
+    const { topK = 5, minScore = 0 } = options;
+    const queryEmbedding = await this.embedder.embed(question);
+    const results = await this.vectorStore.search(queryEmbedding, topK);
+    const filtered = minScore > 0 ? results.filter((r) => r.score >= minScore) : results;
+    return {
+      sources: filtered.map((r) => r.document),
+      queryEmbedding
+    };
+  }
+  /**
+   * Convenience: query + format sources into a context string for LLM prompts.
+   */
+  async getContext(question, options = {}) {
+    const { sources } = await this.query(question, options);
+    if (sources.length === 0) {
+      return "No relevant context found.";
+    }
+    return sources.map((s, i) => `[Source ${i + 1}]
+${s.content}`).join("\n\n---\n\n");
+  }
+  /**
+   * Delete documents from the vector store by IDs.
+   */
+  async delete(ids) {
+    await this.vectorStore.delete(ids);
+  }
 };
 function createRAGPipeline(config) {
   return new RAGPipeline(config);
 }
-var VERSION = "0.1.0";
+var VERSION = "0.3.0";
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   CharacterSplitter,
+  JSONLoader,
+  MDocument,
+  MarkdownLoader,
+  MarkdownSplitter,
   RAGPipeline,
+  RecursiveTextSplitter,
+  TextLoader,
   VERSION,
+  WebLoader,
+  cosineSimilarity,
+  createEmbedder,
   createRAGPipeline
 });