@devilsdev/rag-pipeline-utils 1.0.0

package/src/loader/html-loader.js

@@ -0,0 +1,57 @@
+/**
+ * Version: 0.1.0
+ * Path: /src/loader/html-loader.js
+ * Description: Loader for HTML (.html) documents
+ * Author: Ali Kahwaji
+ */
+
+import fs from 'fs/promises';
+import path from 'path';
+import { JSDOM } from 'jsdom';
+
+/**
+ * HTMLLoader reads HTML files and extracts visible text.
+ * Implements the Loader interface.
+ */
+export class HTMLLoader {
+  /**
+   * Load and extract text from an HTML file
+   * @param {string} filePath - Path to .html file
+   * @returns {Promise<Array<{ chunk(): string[] }>>}
+   */
+  async load(filePath) {
+    const absPath = path.resolve(filePath);
+    const raw = await fs.readFile(absPath, 'utf-8');
+    const dom = new JSDOM(raw);
+    const text = dom.window.document.body.textContent || '';
+
+    return [
+      {
+        chunk: () => this._chunkText(text)
+      }
+    ];
+  }
+
+  /**
+   * Chunk HTML text content by sentence
+   * @param {string} input
+   * @param {number} maxLen
+   * @returns {string[]}
+   */
+  _chunkText(input, maxLen = 500) {
+    const sentences = input.split(/(?<=[.!?])\s+/);
+    const chunks = [];
+    let buffer = '';
+
+    for (const sentence of sentences) {
+      if ((buffer + sentence).length <= maxLen) {
+        buffer += sentence + ' ';
+      } else {
+        chunks.push(buffer.trim());
+        buffer = sentence + ' ';
+      }
+    }
+    if (buffer.trim()) chunks.push(buffer.trim());
+    return chunks;
+  }
+}

package/src/loader/markdown-loader.js

@@ -0,0 +1,58 @@
+/**
+ * Version: 0.1.0
+ * Path: /src/loader/markdown-loader.js
+ * Description: Loader for Markdown (.md) documents
+ * Author: Ali Kahwaji
+ */
+
+import fs from 'fs/promises';
+import path from 'path';
+import { marked } from 'marked';
+
+/**
+ * MarkdownLoader reads and parses .md files into plain text chunks.
+ * Implements the Loader interface.
+ */
+export class MarkdownLoader {
+  /**
+   * Load and parse Markdown file
+   * @param {string} filePath - Path to .md file
+   * @returns {Promise<Array<{ chunk(): string[] }>>}
+   */
+  async load(filePath) {
+    const absPath = path.resolve(filePath);
+    const raw = await fs.readFile(absPath, 'utf-8');
+    const html = marked(raw);
+    const text = html.replace(/<[^>]+>/g, ''); // strip tags
+
+    return [
+      {
+        chunk: () => this._chunkText(text)
+      }
+    ];
+  }
+
+  /**
+   * Simple sentence-based chunking strategy
+   * @param {string} input
+   * @param {number} maxLen
+   * @returns {string[]}
+   */
+  _chunkText(input, maxLen = 500) {
+    const sentences = input.split(/(?<=[.!?])\s+/);
+    const chunks = [];
+    let buffer = '';
+
+    for (const sentence of sentences) {
+      if ((buffer + sentence).length <= maxLen) {
+        buffer += sentence + ' ';
+      } else {
+        chunks.push(buffer.trim());
+        buffer = sentence + ' ';
+      }
+    }
+    if (buffer.trim()) chunks.push(buffer.trim());
+    return chunks;
+  }
+}
+

package/src/reranker/llm-reranker.js

@@ -0,0 +1,49 @@
+/**
+ * Version: 0.1.0
+ * Path: /src/reranker/llm-reranker.js
+ * Description: LLM-based reranker that reorders retrieved context for relevance
+ * Author: Ali Kahwaji
+ */
+
+/**
+ * LLMReranker ranks context chunks by relevance to a given prompt.
+ * @interface
+ */
+export class LLMReranker {
+    constructor({ llm }) {
+      if (!llm || typeof llm.generate !== 'function') {
+        throw new Error('LLM instance must implement generate(prompt, context)');
+      }
+      this.llm = llm;
+    }
+  
+    /**
+     * Reranks retrieved context chunks using the LLM.
+     * @param {string} prompt - The user's question or query
+     * @param {Array<{ text: string, metadata?: object }>} documents - Retrieved context chunks
+     * @param {number} [topK=5] - Number of top-ranked documents to return
+     * @returns {Promise<Array>} - Reranked top K chunks
+     */
+    async rerank(prompt, documents, topK = 5) {
+      const contextText = documents.map((d, i) => `Chunk ${i + 1}: ${d.text}`).join('\n');
+  
+      const instruction = `Rank the following document chunks by how well they answer the prompt.\n
+  Prompt: ${prompt}\n
+  Chunks:\n${contextText}\n
+  Respond with a JSON array of chunk indexes in descending order of relevance.`;
+  
+      const result = await this.llm.generate(instruction, []);
+  
+      let indexOrder = [];
+      try {
+        indexOrder = JSON.parse(result.match(/\[.*\]/s)?.[0] || '[]');
+      } catch (err) {
+        console.warn('Failed to parse reranker output:', result);
+        return documents.slice(0, topK); // fallback
+      }
+  
+      const ranked = indexOrder.map(i => documents[i]).filter(Boolean);
+      return ranked.slice(0, topK);
+    }
+  }
+  

package/src/utils/logger.js

@@ -0,0 +1,29 @@
+/**
+ * Version: 0.1.0
+ * Path: /src/utils/logger.js
+ * Description: Centralized structured logger using Pino
+ * Author: Ali Kahwaji
+ */
+
+import pino from 'pino';
+
+/**
+ * Create a structured JSON logger instance with sensible defaults.
+ * Output can be redirected or formatted in dev/prod contexts.
+ */
+const logger = pino({
+  name: 'rag-pipeline-utils',
+  level: process.env.LOG_LEVEL || 'info',
+  transport: process.env.NODE_ENV === 'development' ? {
+    target: 'pino-pretty',
+    options: {
+      colorize: true,
+      translateTime: 'SYS:standard',
+      ignore: 'pid,hostname'
+    }
+  } : undefined
+});
+
+export { logger };
+
+

package/src/utils/retry.js

@@ -0,0 +1,42 @@
+/**
+ * Version: 0.1.0
+ * Path: /src/utils/retry.js
+ * Description: Generic retry wrapper with exponential backoff for async operations
+ * Author: Ali Kahwaji
+ */
+
+import { logger } from './logger.js';
+
+/**
+ * Retry wrapper for async functions with exponential backoff
+ *
+ * @param {Function} fn - The async function to retry
+ * @param {Object} options
+ * @param {number} [options.retries=3] - Number of retry attempts
+ * @param {number} [options.initialDelay=300] - Initial delay in ms
+ * @param {string} [options.label] - Optional label for logging context
+ * @returns {Promise<*>} - The resolved result of the function
+ * @throws {Error} - The final error if all retries fail
+ */
+export async function withRetry(fn, { retries = 3, initialDelay = 300, label = 'operation' } = {}) {
+  let attempt = 0;
+  let delay = initialDelay;
+
+  while (attempt <= retries) {
+    try {
+      if (attempt > 0) {
+        logger.warn({ attempt, label }, `Retrying ${label} (attempt ${attempt})`);
+      }
+      return await fn();
+    } catch (err) {
+      attempt++;
+      if (attempt > retries) {
+        logger.error({ label, error: err.message }, `Failed ${label} after ${retries} retries`);
+        throw err;
+      }
+      await new Promise(resolve => setTimeout(resolve, delay));
+      delay *= 2; // Exponential backoff
+    }
+  }
+}
+