@voltx/rag 0.1.0 → 0.3.0

package/README.md

@@ -0,0 +1,109 @@
+<p align="center">
+  <strong>@voltx/rag</strong><br/>
+  <em>RAG pipeline — document loading, chunking, embedding, and retrieval</em>
+</p>
+
+<p align="center">
+  <a href="https://www.npmjs.com/package/@voltx/rag"><img src="https://img.shields.io/npm/v/@voltx/rag?color=blue" alt="npm" /></a>
+  <a href="https://www.npmjs.com/package/@voltx/rag"><img src="https://img.shields.io/npm/dm/@voltx/rag" alt="downloads" /></a>
+  <a href="https://github.com/codewithshail/voltx/blob/main/LICENSE"><img src="https://img.shields.io/npm/l/@voltx/rag" alt="license" /></a>
+</p>
+
+---
+
+Production-ready Retrieval-Augmented Generation pipeline for the [VoltX](https://github.com/codewithshail/voltx) framework. Load documents, split into chunks, generate embeddings, store in a vector database, and retrieve relevant context for LLM prompts.
+
+## Installation
+
+```bash
+npm install @voltx/rag
+```
+
+## Quick Start
+
+```ts
+import { createRAGPipeline, createEmbedder } from "@voltx/rag";
+import { createVectorStore } from "@voltx/db";
+
+const pipeline = createRAGPipeline({
+  embedder: createEmbedder({ model: "openai:text-embedding-3-small" }),
+  vectorStore: createVectorStore(),
+});
+
+// Ingest documents
+await pipeline.ingest("Your long document text here...");
+
+// Query with natural language
+const { sources } = await pipeline.query("What is TypeScript?");
+
+// Or get formatted context for an LLM prompt
+const context = await pipeline.getContext("What is TypeScript?");
+```
+
+## Features
+
+### Document Loaders
+
+| Loader | Description |
+|--------|-------------|
+| `TextLoader` | Plain text files or raw strings |
+| `MarkdownLoader` | Markdown files (strips front-matter) |
+| `JSONLoader` | JSON files (extracts text from configurable keys) |
+| `WebLoader` | Fetches and extracts text from URLs |
+
+### Text Splitters
+
+| Splitter | Description |
+|----------|-------------|
+| `RecursiveTextSplitter` | Smart splitting with separator hierarchy (recommended) |
+| `MarkdownSplitter` | Heading-aware splitting, preserves header hierarchy |
+| `CharacterSplitter` | Simple character-based splitting with overlap |
+
+### Fluent Document API
+
+Inspired by [Mastra](https://mastra.ai)'s MDocument pattern:
+
+```ts
+import { MDocument, createEmbedder } from "@voltx/rag";
+
+const doc = MDocument.fromMarkdown("# Title\n\nContent here...");
+const chunks = doc.chunk({ strategy: "markdown", chunkSize: 500 });
+const embedded = await doc.embed(createEmbedder({ model: "openai:text-embedding-3-small" }));
+```
+
+### Embedder
+
+Wraps `@voltx/ai` embedding functions into a simple interface:
+
+```ts
+import { createEmbedder } from "@voltx/rag";
+
+const embedder = createEmbedder({ model: "openai:text-embedding-3-small" });
+const vector = await embedder.embed("Hello world");
+const vectors = await embedder.embedBatch(["Hello", "World"]);
+```
+
+## Pipeline Options
+
+```ts
+const pipeline = createRAGPipeline({
+  loader: new WebLoader(),                    // optional document loader
+  splitter: new RecursiveTextSplitter({       // text splitter (default: recursive)
+    chunkSize: 1000,
+    overlap: 200,
+  }),
+  embedder: createEmbedder({ model: "openai:text-embedding-3-small" }),
+  vectorStore: createVectorStore("pinecone", { indexName: "docs" }),
+});
+
+// Query with options
+const results = await pipeline.query("question", { topK: 5, minScore: 0.7 });
+```
+
+## Part of VoltX
+
+This package is part of the [VoltX](https://github.com/codewithshail/voltx) framework. See the [monorepo](https://github.com/codewithshail/voltx) for full documentation.
+
+## License
+
+[MIT](https://github.com/codewithshail/voltx/blob/main/LICENSE) — Made by the [Promptly AI Team](https://buymeacoffee.com/promptlyai)

package/dist/index.d.mts

@@ -4,53 +4,314 @@ interface DocumentChunk {
     id: string;
     content: string;
     metadata?: Record<string, unknown>;
+    embedding?: number[];
 }
 interface DocumentLoader {
     name: string;
-    /** Load and return raw text from a source */
+    /** Load and return raw text from a source (file path, URL, or raw content) */
     load(source: string): Promise<string>;
 }
 interface TextSplitter {
     /** Split text into chunks */
     split(text: string): DocumentChunk[];
 }
+interface CharacterSplitterOptions {
+    /** Maximum characters per chunk (default: 1000) */
+    chunkSize?: number;
+    /** Overlap between chunks in characters (default: 200) */
+    overlap?: number;
+}
+interface RecursiveSplitterOptions {
+    /** Maximum characters per chunk (default: 1000) */
+    chunkSize?: number;
+    /** Overlap between chunks in characters (default: 200) */
+    overlap?: number;
+    /**
+     * Separators to try in order of preference.
+     * Default: ["\n\n", "\n", ". ", " ", ""]
+     */
+    separators?: string[];
+}
+interface MarkdownSplitterOptions {
+    /** Maximum characters per chunk (default: 1500) */
+    chunkSize?: number;
+    /** Overlap between chunks in characters (default: 100) */
+    overlap?: number;
+    /** Whether to include header hierarchy in chunk metadata (default: true) */
+    includeHeaders?: boolean;
+}
 interface Embedder {
     name: string;
-    /** Generate embedding vector for text */
+    /** Generate embedding vector for a single text */
     embed(text: string): Promise<number[]>;
     /** Batch embed multiple texts */
     embedBatch(texts: string[]): Promise<number[][]>;
 }
+interface EmbedderConfig {
+    /** Model string in "provider:model" format, e.g. "openai:text-embedding-3-small" */
+    model: string;
+}
 interface RAGPipelineConfig {
+    /** Document loader (optional — if omitted, source is treated as raw text) */
     loader?: DocumentLoader;
+    /** Text splitter (default: RecursiveTextSplitter) */
     splitter?: TextSplitter;
+    /** Embedder — wraps @voltx/ai embed/embedMany */
     embedder: Embedder;
+    /** Vector store from @voltx/db */
     vectorStore: VectorStore;
 }
+interface RAGQueryOptions {
+    /** Number of results to return (default: 5) */
+    topK?: number;
+    /** Minimum similarity score threshold (0-1, default: 0) */
+    minScore?: number;
+    /** Metadata filter */
+    filter?: Record<string, unknown>;
+}
 interface RAGQueryResult {
-    answer?: string;
+    /** Retrieved source documents */
     sources: VectorDocument[];
+    /** The query embedding (useful for debugging) */
+    queryEmbedding?: number[];
 }
+interface RAGIngestResult {
+    /** Number of chunks ingested */
+    chunks: number;
+    /** Chunk IDs that were stored */
+    ids: string[];
+}
+
+/**
+ * Simple character-based text splitter with smart boundary detection.
+ * Tries to split at sentence/paragraph boundaries near the chunk size.
+ */
 declare class CharacterSplitter implements TextSplitter {
     private chunkSize;
     private overlap;
-    constructor(chunkSize?: number, overlap?: number);
+    constructor(options?: CharacterSplitterOptions);
+    private findBreakPoint;
+    split(text: string): DocumentChunk[];
+}
+/**
+ * Recursively splits text using a hierarchy of separators.
+ * Tries to keep semantically related text together by splitting on
+ * paragraph breaks first, then newlines, then sentences, then words.
+ *
+ * This is the recommended splitter for generic text (same approach as
+ * LangChain's RecursiveCharacterTextSplitter).
+ */
+declare class RecursiveTextSplitter implements TextSplitter {
+    private chunkSize;
+    private overlap;
+    private separators;
+    constructor(options?: RecursiveSplitterOptions);
+    split(text: string): DocumentChunk[];
     /**
-     * Find the best split point near `pos` by looking for sentence/paragraph
-     * boundaries first, then word boundaries, falling back to exact position.
+     * Recursively split text. Try the first separator; if any resulting piece
+     * is still too large, recurse with the next separator in the list.
      */
-    private findBreakPoint;
+    private splitText;
+    /**
+     * Merge chunks with overlap to maintain context between adjacent chunks.
+     */
+    private mergeWithOverlap;
+}
+/**
+ * Markdown-aware text splitter that respects heading hierarchy.
+ * Splits on headings first, then falls back to paragraph/sentence boundaries
+ * within sections that exceed the chunk size.
+ */
+declare class MarkdownSplitter implements TextSplitter {
+    private chunkSize;
+    private overlap;
+    private includeHeaders;
+    constructor(options?: MarkdownSplitterOptions);
     split(text: string): DocumentChunk[];
+    private splitByHeadings;
+    private buildHeaderPrefix;
 }
+
+/**
+ * Loads plain text from a file path or treats the source as raw text.
+ */
+declare class TextLoader implements DocumentLoader {
+    name: string;
+    load(source: string): Promise<string>;
+}
+/**
+ * Loads markdown files. Strips front-matter (YAML between --- delimiters)
+ * and returns the markdown body.
+ */
+declare class MarkdownLoader implements DocumentLoader {
+    name: string;
+    load(source: string): Promise<string>;
+}
+interface JSONLoaderOptions {
+    /** JSON path keys to extract text from (e.g. ["content", "text", "body"]) */
+    textKeys?: string[];
+    /** Separator between extracted values (default: "\n\n") */
+    separator?: string;
+}
+/**
+ * Loads JSON files and extracts text content from specified keys.
+ * Handles both single objects and arrays of objects.
+ */
+declare class JSONLoader implements DocumentLoader {
+    name: string;
+    private textKeys;
+    private separator;
+    constructor(options?: JSONLoaderOptions);
+    load(source: string): Promise<string>;
+    private extractTexts;
+}
+/**
+ * Fetches text content from a URL. Strips HTML tags for basic text extraction.
+ */
+declare class WebLoader implements DocumentLoader {
+    name: string;
+    load(source: string): Promise<string>;
+    private stripHTML;
+}
+
+/**
+ * Creates an embedder that uses @voltx/ai under the hood.
+ *
+ * @example
+ * ```ts
+ * const embedder = createEmbedder({ model: "openai:text-embedding-3-small" });
+ * const vector = await embedder.embed("Hello world");
+ * const vectors = await embedder.embedBatch(["Hello", "World"]);
+ * ```
+ */
+declare function createEmbedder(config: EmbedderConfig): Embedder;
+
+type ChunkStrategy = "recursive" | "character" | "markdown";
+interface ChunkOptions {
+    /** Chunking strategy (default: "recursive") */
+    strategy?: ChunkStrategy;
+    /** Maximum chunk size in characters */
+    chunkSize?: number;
+    /** Overlap between chunks in characters */
+    overlap?: number;
+    /** Custom separators (recursive strategy only) */
+    separators?: string[];
+    /** Include header hierarchy in metadata (markdown strategy only) */
+    includeHeaders?: boolean;
+}
+/**
+ * Fluent document processing class.
+ * Create from various formats, chunk, and embed in a pipeline.
+ *
+ * @example
+ * ```ts
+ * const doc = MDocument.fromText("Your long document...");
+ * const chunks = doc.chunk({ strategy: "recursive", chunkSize: 500 });
+ *
+ * // Or from markdown
+ * const mdDoc = MDocument.fromMarkdown("# Title\n\nContent...");
+ * const mdChunks = mdDoc.chunk({ strategy: "markdown" });
+ * ```
+ */
+declare class MDocument {
+    private content;
+    private format;
+    private chunks;
+    private constructor();
+    /** Create from plain text */
+    static fromText(content: string): MDocument;
+    /** Create from markdown */
+    static fromMarkdown(content: string): MDocument;
+    /** Create from JSON string */
+    static fromJSON(content: string): MDocument;
+    /** Create from HTML (strips tags) */
+    static fromHTML(html: string): MDocument;
+    /** Get the raw content */
+    getContent(): string;
+    /** Get the document format */
+    getFormat(): string;
+    /**
+     * Split the document into chunks using the specified strategy.
+     * Returns the chunks and caches them for subsequent embed() calls.
+     */
+    chunk(options?: ChunkOptions): DocumentChunk[];
+    /**
+     * Embed the chunks using the provided embedder.
+     * Must call chunk() first.
+     */
+    embed(embedder: Embedder): Promise<DocumentChunk[]>;
+    /** Get cached chunks (null if chunk() hasn't been called) */
+    getChunks(): DocumentChunk[] | null;
+    private defaultStrategy;
+    private createSplitter;
+}
+
+/**
+ * Calculate cosine similarity between two embedding vectors.
+ * Returns a value between -1 and 1, where 1 means identical direction.
+ *
+ * @example
+ * ```ts
+ * const score = cosineSimilarity(embeddingA, embeddingB);
+ * // score ≈ 0.95 means very similar
+ * ```
+ */
+declare function cosineSimilarity(a: number[], b: number[]): number;
+
 declare class RAGPipeline {
-    private config;
+    private loader;
+    private splitter;
+    private embedder;
+    private vectorStore;
     constructor(config: RAGPipelineConfig);
-    /** Ingest a document: load → split → embed → store */
-    ingest(source: string): Promise<number>;
-    /** Query: embed question → search vector store → return sources */
-    query(question: string, topK?: number): Promise<RAGQueryResult>;
+    /**
+     * Ingest a document: load → split → embed (batch) → store in vector DB.
+     *
+     * @param source - File path, URL, or raw text (depends on loader)
+     * @param idPrefix - Optional prefix for chunk IDs (default: "doc")
+     * @returns Number of chunks ingested and their IDs
+     */
+    ingest(source: string, idPrefix?: string): Promise<RAGIngestResult>;
+    /**
+     * Query: embed question → search vector store → return ranked sources.
+     *
+     * @param question - The user's question
+     * @param options - Query options (topK, minScore)
+     */
+    query(question: string, options?: RAGQueryOptions): Promise<RAGQueryResult>;
+    /**
+     * Convenience: query + format sources into a context string for LLM prompts.
+     */
+    getContext(question: string, options?: RAGQueryOptions): Promise<string>;
+    /**
+     * Delete documents from the vector store by IDs.
+     */
+    delete(ids: string[]): Promise<void>;
 }
+/**
+ * Create a RAG pipeline.
+ *
+ * @example
+ * ```ts
+ * import { createRAGPipeline, createEmbedder } from "@voltx/rag";
+ * import { createVectorStore } from "@voltx/db";
+ *
+ * const pipeline = createRAGPipeline({
+ *   embedder: createEmbedder({ model: "openai:text-embedding-3-small" }),
+ *   vectorStore: createVectorStore("pinecone", { indexName: "my-index" }),
+ * });
+ *
+ * // Ingest documents
+ * await pipeline.ingest("Your long document text here...");
+ *
+ * // Query
+ * const { sources } = await pipeline.query("What is TypeScript?");
+ *
+ * // Or get formatted context for LLM
+ * const context = await pipeline.getContext("What is TypeScript?");
+ * ```
+ */
 declare function createRAGPipeline(config: RAGPipelineConfig): RAGPipeline;
-declare const VERSION = "0.1.0";
+declare const VERSION = "0.3.0";
 
-export { CharacterSplitter, type DocumentChunk, type DocumentLoader, type Embedder, RAGPipeline, type RAGPipelineConfig, type RAGQueryResult, type TextSplitter, VERSION, createRAGPipeline };
+export { CharacterSplitter, type CharacterSplitterOptions, type ChunkOptions, type ChunkStrategy, type DocumentChunk, type DocumentLoader, type Embedder, type EmbedderConfig, JSONLoader, type JSONLoaderOptions, MDocument, MarkdownLoader, MarkdownSplitter, type MarkdownSplitterOptions, type RAGIngestResult, RAGPipeline, type RAGPipelineConfig, type RAGQueryOptions, type RAGQueryResult, type RecursiveSplitterOptions, RecursiveTextSplitter, TextLoader, type TextSplitter, VERSION, WebLoader, cosineSimilarity, createEmbedder, createRAGPipeline };