@retrivora-ai/rag-engine 0.1.4 → 0.1.5

package/dist/DocumentChunker-cfaMidtA.d.mts

@@ -0,0 +1,93 @@
+/**
+ * Generic LLM Provider interface.
+ * Covers both chat completion and embedding generation so a single
+ * provider can handle both responsibilities when appropriate.
+ */
+interface ChatMessage {
+    role: 'user' | 'assistant' | 'system';
+    content: string;
+}
+interface ChatOptions {
+    /** Override max tokens for this specific call */
+    maxTokens?: number;
+    /** Override temperature for this specific call */
+    temperature?: number;
+    /** Stop sequences */
+    stop?: string[];
+}
+interface EmbedOptions {
+    /** Override model for this specific embed call */
+    model?: string;
+}
+interface ILLMProvider {
+    /**
+     * Send a chat completion request.
+     * @param messages – the full conversation history
+     * @param context  – retrieved RAG context to inject
+     * @param options  – optional per-call overrides
+     * @returns        – the assistant's reply text
+     */
+    chat(messages: ChatMessage[], context: string, options?: ChatOptions): Promise<string>;
+    /**
+     * Generate an embedding vector for the given text.
+     * @param text    – text to embed
+     * @param options – optional overrides
+     * @returns       – float array (the embedding)
+     */
+    embed(text: string, options?: EmbedOptions): Promise<number[]>;
+    /**
+     * Generate embedding vectors for multiple texts in a single batch.
+     * @param texts   – array of texts to embed
+     * @param options – optional overrides
+     * @returns       – array of float arrays
+     */
+    batchEmbed(texts: string[], options?: EmbedOptions): Promise<number[][]>;
+    /**
+     * Check if the provider endpoint is reachable.
+     */
+    ping(): Promise<boolean>;
+}
+
+/**
+ * DocumentChunker — splits long text into overlapping chunks
+ * suitable for vector embedding and retrieval.
+ *
+ * Strategy: sentence-aware sliding window
+ *   1. Split on sentence boundaries
+ *   2. Accumulate sentences into chunks up to `chunkSize` characters
+ *   3. Carry over `chunkOverlap` characters from the previous chunk
+ */
+interface Chunk {
+    id: string;
+    content: string;
+    metadata?: Record<string, unknown>;
+}
+interface ChunkOptions {
+    /** Target chunk size in characters (default 1000) */
+    chunkSize?: number;
+    /** Overlap between consecutive chunks in characters (default 200) */
+    chunkOverlap?: number;
+    /** Source document identifier used as ID prefix */
+    docId?: string;
+    /** Extra metadata to attach to every chunk */
+    metadata?: Record<string, unknown>;
+}
+declare class DocumentChunker {
+    private readonly chunkSize;
+    private readonly chunkOverlap;
+    constructor(chunkSize?: number, chunkOverlap?: number);
+    /**
+     * Split a single text string into overlapping chunks.
+     */
+    chunk(text: string, options?: ChunkOptions): Chunk[];
+    /**
+     * Chunk multiple documents at once.
+     */
+    chunkMany(documents: Array<{
+        content: string;
+        docId?: string;
+        metadata?: Record<string, unknown>;
+    }>): Chunk[];
+}
+
+export { type ChatMessage as C, DocumentChunker as D, type EmbedOptions as E, type ILLMProvider as I, type ChatOptions as a, type Chunk as b, type ChunkOptions as c };

package/dist/DocumentChunker-cfaMidtA.d.ts

@@ -0,0 +1,93 @@
+/**
+ * Generic LLM Provider interface.
+ * Covers both chat completion and embedding generation so a single
+ * provider can handle both responsibilities when appropriate.
+ */
+interface ChatMessage {
+    role: 'user' | 'assistant' | 'system';
+    content: string;
+}
+interface ChatOptions {
+    /** Override max tokens for this specific call */
+    maxTokens?: number;
+    /** Override temperature for this specific call */
+    temperature?: number;
+    /** Stop sequences */
+    stop?: string[];
+}
+interface EmbedOptions {
+    /** Override model for this specific embed call */
+    model?: string;
+}
+interface ILLMProvider {
+    /**
+     * Send a chat completion request.
+     * @param messages – the full conversation history
+     * @param context  – retrieved RAG context to inject
+     * @param options  – optional per-call overrides
+     * @returns        – the assistant's reply text
+     */
+    chat(messages: ChatMessage[], context: string, options?: ChatOptions): Promise<string>;
+    /**
+     * Generate an embedding vector for the given text.
+     * @param text    – text to embed
+     * @param options – optional overrides
+     * @returns       – float array (the embedding)
+     */
+    embed(text: string, options?: EmbedOptions): Promise<number[]>;
+    /**
+     * Generate embedding vectors for multiple texts in a single batch.
+     * @param texts   – array of texts to embed
+     * @param options – optional overrides
+     * @returns       – array of float arrays
+     */
+    batchEmbed(texts: string[], options?: EmbedOptions): Promise<number[][]>;
+    /**
+     * Check if the provider endpoint is reachable.
+     */
+    ping(): Promise<boolean>;
+}
+
+/**
+ * DocumentChunker — splits long text into overlapping chunks
+ * suitable for vector embedding and retrieval.
+ *
+ * Strategy: sentence-aware sliding window
+ *   1. Split on sentence boundaries
+ *   2. Accumulate sentences into chunks up to `chunkSize` characters
+ *   3. Carry over `chunkOverlap` characters from the previous chunk
+ */
+interface Chunk {
+    id: string;
+    content: string;
+    metadata?: Record<string, unknown>;
+}
+interface ChunkOptions {
+    /** Target chunk size in characters (default 1000) */
+    chunkSize?: number;
+    /** Overlap between consecutive chunks in characters (default 200) */
+    chunkOverlap?: number;
+    /** Source document identifier used as ID prefix */
+    docId?: string;
+    /** Extra metadata to attach to every chunk */
+    metadata?: Record<string, unknown>;
+}
+declare class DocumentChunker {
+    private readonly chunkSize;
+    private readonly chunkOverlap;
+    constructor(chunkSize?: number, chunkOverlap?: number);
+    /**
+     * Split a single text string into overlapping chunks.
+     */
+    chunk(text: string, options?: ChunkOptions): Chunk[];
+    /**
+     * Chunk multiple documents at once.
+     */
+    chunkMany(documents: Array<{
+        content: string;
+        docId?: string;
+        metadata?: Record<string, unknown>;
+    }>): Chunk[];
+}
+
+export { type ChatMessage as C, DocumentChunker as D, type EmbedOptions as E, type ILLMProvider as I, type ChatOptions as a, type Chunk as b, type ChunkOptions as c };

package/dist/{LLMFactory-XC55FTD2.mjs → LLMFactory-JFOY2V4X.mjs}

@@ -1,6 +1,7 @@
 import {
   LLMFactory
-} from "./chunk-GD3QJFNN.mjs";
+} from "./chunk-JI6VD5TJ.mjs";
+import "./chunk-UKDXCXW7.mjs";
 import "./chunk-I4E63NIC.mjs";
 export {
   LLMFactory

package/dist/{Pipeline-Bo6CUCox.d.ts → RagConfig-DG_0f8ka.d.mts}

@@ -1,3 +1,25 @@
+interface VectorMatch {
+    id: string;
+    score: number;
+    content: string;
+    metadata?: Record<string, unknown>;
+}
+interface UpsertDocument {
+    id: string;
+    vector: number[];
+    content: string;
+    metadata?: Record<string, unknown>;
+}
+interface IngestDocument {
+    docId: string;
+    content: string;
+    metadata?: Record<string, unknown>;
+}
+interface ChatResponse {
+    reply: string;
+    sources: VectorMatch[];
+}
+
 /**
  * Master configuration interface for Retrivora AI.
  * Each consuming project provides one RagConfig object to drive
@@ -120,96 +142,4 @@ interface RagConfig {
     rag?: RAGConfig;
 }
 
-/**
- * Generic LLM Provider interface.
- * Covers both chat completion and embedding generation so a single
- * provider can handle both responsibilities when appropriate.
- */
-interface ChatMessage {
-    role: 'user' | 'assistant' | 'system';
-    content: string;
-}
-interface ChatOptions {
-    /** Override max tokens for this specific call */
-    maxTokens?: number;
-    /** Override temperature for this specific call */
-    temperature?: number;
-    /** Stop sequences */
-    stop?: string[];
-}
-interface EmbedOptions {
-    /** Override model for this specific embed call */
-    model?: string;
-}
-interface ILLMProvider {
-    /**
-     * Send a chat completion request.
-     * @param messages – the full conversation history
-     * @param context  – retrieved RAG context to inject
-     * @param options  – optional per-call overrides
-     * @returns        – the assistant's reply text
-     */
-    chat(messages: ChatMessage[], context: string, options?: ChatOptions): Promise<string>;
-    /**
-     * Generate an embedding vector for the given text.
-     * @param text    – text to embed
-     * @param options – optional overrides
-     * @returns       – float array (the embedding)
-     */
-    embed(text: string, options?: EmbedOptions): Promise<number[]>;
-    /**
-     * Generate embedding vectors for multiple texts in a single batch.
-     * @param texts   – array of texts to embed
-     * @param options – optional overrides
-     * @returns       – array of float arrays
-     */
-    batchEmbed(texts: string[], options?: EmbedOptions): Promise<number[][]>;
-    /**
-     * Check if the provider endpoint is reachable.
-     */
-    ping(): Promise<boolean>;
-}
-
-interface VectorMatch {
-    id: string;
-    score: number;
-    content: string;
-    metadata?: Record<string, unknown>;
-}
-interface UpsertDocument {
-    id: string;
-    vector: number[];
-    content: string;
-    metadata?: Record<string, unknown>;
-}
-
-interface IngestDocument {
-    docId: string;
-    content: string;
-    metadata?: Record<string, unknown>;
-}
-interface ChatResponse {
-    reply: string;
-    sources: VectorMatch[];
-}
-/**
- * Pipeline — orchestrates the RAG flow: Embed → Search → Augment → Generate.
- */
-declare class Pipeline {
-    private vectorDB;
-    private llmProvider;
-    private embeddingProvider;
-    private chunker;
-    private config;
-    private initialised;
-    constructor(config: RagConfig);
-    initialize(): Promise<void>;
-    ingest(documents: IngestDocument[], namespace?: string): Promise<Array<{
-        docId: string;
-        chunksIngested: number;
-    }>>;
-    ask(question: string, history?: ChatMessage[], namespace?: string): Promise<ChatResponse>;
-    health(): Promise<Record<string, boolean>>;
-}
-
-export { type ChatMessage as C, type EmbedOptions as E, type ILLMProvider as I, type LLMConfig as L, Pipeline as P, type RAGConfig as R, type UIConfig as U, type VectorMatch as V, type ChatOptions as a, type ChatResponse as b, type EmbeddingConfig as c, type IngestDocument as d, type RagConfig as e, type UpsertDocument as f, type VectorDBConfig as g };
+export type { ChatResponse as C, EmbeddingConfig as E, IngestDocument as I, LLMConfig as L, RAGConfig as R, UIConfig as U, VectorMatch as V, RagConfig as a, UpsertDocument as b, VectorDBConfig as c };

package/dist/{Pipeline-Bo6CUCox.d.mts → RagConfig-DG_0f8ka.d.ts}

@@ -1,3 +1,25 @@
+interface VectorMatch {
+    id: string;
+    score: number;
+    content: string;
+    metadata?: Record<string, unknown>;
+}
+interface UpsertDocument {
+    id: string;
+    vector: number[];
+    content: string;
+    metadata?: Record<string, unknown>;
+}
+interface IngestDocument {
+    docId: string;
+    content: string;
+    metadata?: Record<string, unknown>;
+}
+interface ChatResponse {
+    reply: string;
+    sources: VectorMatch[];
+}
+
 /**
  * Master configuration interface for Retrivora AI.
  * Each consuming project provides one RagConfig object to drive
@@ -120,96 +142,4 @@ interface RagConfig {
     rag?: RAGConfig;
 }
 
-/**
- * Generic LLM Provider interface.
- * Covers both chat completion and embedding generation so a single
- * provider can handle both responsibilities when appropriate.
- */
-interface ChatMessage {
-    role: 'user' | 'assistant' | 'system';
-    content: string;
-}
-interface ChatOptions {
-    /** Override max tokens for this specific call */
-    maxTokens?: number;
-    /** Override temperature for this specific call */
-    temperature?: number;
-    /** Stop sequences */
-    stop?: string[];
-}
-interface EmbedOptions {
-    /** Override model for this specific embed call */
-    model?: string;
-}
-interface ILLMProvider {
-    /**
-     * Send a chat completion request.
-     * @param messages – the full conversation history
-     * @param context  – retrieved RAG context to inject
-     * @param options  – optional per-call overrides
-     * @returns        – the assistant's reply text
-     */
-    chat(messages: ChatMessage[], context: string, options?: ChatOptions): Promise<string>;
-    /**
-     * Generate an embedding vector for the given text.
-     * @param text    – text to embed
-     * @param options – optional overrides
-     * @returns       – float array (the embedding)
-     */
-    embed(text: string, options?: EmbedOptions): Promise<number[]>;
-    /**
-     * Generate embedding vectors for multiple texts in a single batch.
-     * @param texts   – array of texts to embed
-     * @param options – optional overrides
-     * @returns       – array of float arrays
-     */
-    batchEmbed(texts: string[], options?: EmbedOptions): Promise<number[][]>;
-    /**
-     * Check if the provider endpoint is reachable.
-     */
-    ping(): Promise<boolean>;
-}
-
-interface VectorMatch {
-    id: string;
-    score: number;
-    content: string;
-    metadata?: Record<string, unknown>;
-}
-interface UpsertDocument {
-    id: string;
-    vector: number[];
-    content: string;
-    metadata?: Record<string, unknown>;
-}
-
-interface IngestDocument {
-    docId: string;
-    content: string;
-    metadata?: Record<string, unknown>;
-}
-interface ChatResponse {
-    reply: string;
-    sources: VectorMatch[];
-}
-/**
- * Pipeline — orchestrates the RAG flow: Embed → Search → Augment → Generate.
- */
-declare class Pipeline {
-    private vectorDB;
-    private llmProvider;
-    private embeddingProvider;
-    private chunker;
-    private config;
-    private initialised;
-    constructor(config: RagConfig);
-    initialize(): Promise<void>;
-    ingest(documents: IngestDocument[], namespace?: string): Promise<Array<{
-        docId: string;
-        chunksIngested: number;
-    }>>;
-    ask(question: string, history?: ChatMessage[], namespace?: string): Promise<ChatResponse>;
-    health(): Promise<Record<string, boolean>>;
-}
-
-export { type ChatMessage as C, type EmbedOptions as E, type ILLMProvider as I, type LLMConfig as L, Pipeline as P, type RAGConfig as R, type UIConfig as U, type VectorMatch as V, type ChatOptions as a, type ChatResponse as b, type EmbeddingConfig as c, type IngestDocument as d, type RagConfig as e, type UpsertDocument as f, type VectorDBConfig as g };
+export type { ChatResponse as C, EmbeddingConfig as E, IngestDocument as I, LLMConfig as L, RAGConfig as R, UIConfig as U, VectorMatch as V, RagConfig as a, UpsertDocument as b, VectorDBConfig as c };

package/dist/{chunk-AIAB2IEE.mjs → chunk-BP4U4TT5.mjs}

@@ -1,12 +1,17 @@
 import {
-  LLMFactory,
+  LLMFactory
+} from "./chunk-JI6VD5TJ.mjs";
+import {
   mergeDefined
-} from "./chunk-GD3QJFNN.mjs";
+} from "./chunk-UKDXCXW7.mjs";
 import {
   __spreadProps,
   __spreadValues
 } from "./chunk-I4E63NIC.mjs";
 
+// src/handlers/index.ts
+import { NextResponse } from "next/server";
+
 // src/config/serverConfig.ts
 var VECTOR_DB_PROVIDERS = ["pinecone", "pgvector", "postgresql", "mongodb", "milvus", "qdrant", "chromadb", "redis", "weaviate", "rest", "universal_rest", "custom"];
 var LLM_PROVIDERS = ["openai", "anthropic", "ollama", "rest", "universal_rest", "custom"];
@@ -293,7 +298,7 @@ var Pipeline = class {
     this.vectorDB = await ProviderRegistry.createVectorProvider(this.config.vectorDb);
     this.llmProvider = ProviderRegistry.createLLMProvider(this.config.llm, this.config.embedding);
     if (this.config.llm.provider === "anthropic") {
-      const { LLMFactory: LLMFactory2 } = await import("./LLMFactory-XC55FTD2.mjs");
+      const { LLMFactory: LLMFactory2 } = await import("./LLMFactory-JFOY2V4X.mjs");
       this.embeddingProvider = LLMFactory2.createEmbeddingProvider(this.config.embedding);
     } else {
       this.embeddingProvider = this.llmProvider;
@@ -384,11 +389,160 @@ var VectorPlugin = class {
   }
 };
 
+// src/utils/DocumentParser.ts
+var DocumentParser = class {
+  /**
+   * Extract text from a File or Buffer based on its type.
+   */
+  static async parse(file, fileName, mimeType) {
+    var _a;
+    const extension = (_a = fileName.split(".").pop()) == null ? void 0 : _a.toLowerCase();
+    if (extension === "txt" || extension === "md" || mimeType === "text/plain" || mimeType === "text/markdown") {
+      return this.readAsText(file);
+    }
+    if (extension === "json" || mimeType === "application/json") {
+      const text = await this.readAsText(file);
+      try {
+        const obj = JSON.parse(text);
+        return JSON.stringify(obj, null, 2);
+      } catch (e) {
+        return text;
+      }
+    }
+    if (extension === "csv" || mimeType === "text/csv") {
+      return this.readAsText(file);
+    }
+    if (extension === "pdf" || mimeType === "application/pdf") {
+      try {
+        const pdf = await import("pdf-parse/lib/pdf-parse.js");
+        const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
+        const data = await pdf.default(buffer);
+        return data.text;
+      } catch (err) {
+        console.warn('[DocumentParser] PDF parsing failed. Make sure "pdf-parse" is installed.', err);
+        return `[PDF Parsing Error for ${fileName}]`;
+      }
+    }
+    if (extension === "docx" || mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document") {
+      try {
+        const mammoth = await import("mammoth");
+        const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
+        const result = await mammoth.extractRawText({ buffer });
+        return result.value;
+      } catch (err) {
+        console.warn('[DocumentParser] DOCX parsing failed. Make sure "mammoth" is installed.', err);
+        return `[DOCX Parsing Error for ${fileName}]`;
+      }
+    }
+    try {
+      return await this.readAsText(file);
+    } catch (e) {
+      throw new Error(`[DocumentParser] Unsupported file format: ${fileName} (${mimeType})`);
+    }
+  }
+  static async readAsText(file) {
+    if (Buffer.isBuffer(file)) {
+      return file.toString("utf-8");
+    }
+    return await file.text();
+  }
+};
+
+// src/handlers/index.ts
+function createChatHandler(config) {
+  const plugin = new VectorPlugin(config);
+  return async function POST(req) {
+    try {
+      const body = await req.json();
+      const { message, history = [], namespace } = body;
+      if (!(message == null ? void 0 : message.trim())) {
+        return NextResponse.json({ error: "message is required" }, { status: 400 });
+      }
+      const result = await plugin.chat(message, history, namespace);
+      return NextResponse.json(result);
+    } catch (err) {
+      const message = err instanceof Error ? err.message : "Internal server error";
+      return NextResponse.json({ error: message }, { status: 500 });
+    }
+  };
+}
+function createIngestHandler(config) {
+  const plugin = new VectorPlugin(config);
+  return async function POST(req) {
+    try {
+      const body = await req.json();
+      const { documents, namespace } = body;
+      if (!Array.isArray(documents) || documents.length === 0) {
+        return NextResponse.json({ error: "documents array is required" }, { status: 400 });
+      }
+      const results = await plugin.ingest(documents, namespace);
+      return NextResponse.json({ results });
+    } catch (err) {
+      const message = err instanceof Error ? err.message : "Internal server error";
+      return NextResponse.json({ error: message }, { status: 500 });
+    }
+  };
+}
+function createHealthHandler(config) {
+  const plugin = new VectorPlugin(config);
+  return async function GET() {
+    try {
+      const health = await plugin.health();
+      const status = health.vectorDB && health.llm ? "ok" : "degraded";
+      return NextResponse.json(__spreadProps(__spreadValues({
+        status
+      }, health), {
+        timestamp: (/* @__PURE__ */ new Date()).toISOString()
+      }));
+    } catch (err) {
+      const message = err instanceof Error ? err.message : "Unknown error";
+      return NextResponse.json({ status: "error", error: message }, { status: 500 });
+    }
+  };
+}
+function createUploadHandler(config) {
+  const plugin = new VectorPlugin(config);
+  return async function POST(req) {
+    try {
+      const formData = await req.formData();
+      const files = formData.getAll("files");
+      const namespace = formData.get("namespace") || void 0;
+      if (!files || files.length === 0) {
+        return NextResponse.json({ error: "No files provided" }, { status: 400 });
+      }
+      const documents = await Promise.all(
+        files.map(async (file) => {
+          const content = await DocumentParser.parse(file, file.name, file.type);
+          return {
+            docId: file.name,
+            content,
+            metadata: {
+              fileName: file.name,
+              fileSize: file.size,
+              fileType: file.type,
+              uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
+            }
+          };
+        })
+      );
+      const results = await plugin.ingest(documents, namespace);
+      return NextResponse.json({ message: "Upload successful", results });
+    } catch (err) {
+      const message = err instanceof Error ? err.message : "Upload failed";
+      return NextResponse.json({ error: message }, { status: 500 });
+    }
+  };
+}
+
 export {
   getRagConfig,
   ConfigResolver,
   DocumentChunker,
   ProviderRegistry,
   Pipeline,
-  VectorPlugin
+  VectorPlugin,
+  createChatHandler,
+  createIngestHandler,
+  createHealthHandler,
+  createUploadHandler
 };

package/dist/{chunk-GD3QJFNN.mjs → chunk-JI6VD5TJ.mjs}

@@ -1,3 +1,7 @@
+import {
+  buildPayload,
+  resolvePath
+} from "./chunk-UKDXCXW7.mjs";
 import {
   __spreadProps,
   __spreadValues
@@ -192,46 +196,6 @@ ${context}`;
 // src/llm/providers/UniversalLLMAdapter.ts
 import axios2 from "axios";
 
-// src/utils/templateUtils.ts
-function isRecord(value) {
-  return typeof value === "object" && value !== null;
-}
-function resolvePath(obj, path) {
-  if (!path || !obj) return void 0;
-  return path.replace(/\[(\w+)\]/g, ".$1").replace(/^\./, "").split(".").reduce((current, segment) => {
-    if (!isRecord(current) && !Array.isArray(current)) {
-      return void 0;
-    }
-    return current[segment];
-  }, obj);
-}
-function buildPayload(template, vars) {
-  let raw = template;
-  for (const [key, val] of Object.entries(vars)) {
-    const stringifiedVal = val === void 0 ? "null" : JSON.stringify(val);
-    raw = raw.replace(new RegExp(`"\\{\\{${key}\\}\\}"`, "g"), stringifiedVal);
-    raw = raw.replace(new RegExp(`\\{\\{${key}\\}\\}`, "g"), stringifiedVal);
-  }
-  try {
-    return JSON.parse(raw);
-  } catch (err) {
-    throw new Error(`Failed to parse payload template: ${err.message}
-Raw payload: ${raw}`);
-  }
-}
-function mergeDefined(base, override) {
-  const merged = __spreadValues({}, base || {});
-  if (!override) {
-    return merged;
-  }
-  for (const [key, value] of Object.entries(override)) {
-    if (value !== void 0 && value !== null && value !== "") {
-      merged[key] = value;
-    }
-  }
-  return merged;
-}
-
 // src/config/UniversalProfiles.ts
 var OPENAI_BASE = {
   chatPath: "/chat/completions",
@@ -416,7 +380,6 @@ var LLMFactory = class _LLMFactory {
 };
 
 export {
-  mergeDefined,
   OpenAIProvider,
   AnthropicProvider,
   OllamaProvider,