@devilsdev/rag-pipeline-utils 2.1.11 → 2.2.0

package/src/evaluate/evaluator.js

@@ -0,0 +1,52 @@
+/**
+ * Evaluation runner with scoring metrics for batch RAG QA
+ */
+
+const { createRagPipeline } = require('../core/create-pipeline.js');
+const { logger } = require('../utils/logger.js');
+const fs = require('fs/promises');
+const path = require('path');
+const { scoreAnswer } = require('./scoring.js');
+
+/**
+ * Evaluate a list of prompt/answer pairs using the RAG pipeline and compute scores
+ * @param {string} datasetPath - Path to JSON file with { prompt, expected }[]
+ * @param {object} _config - RAG plugin _config (loader optional)
+ * @returns {Promise<Array<{ prompt: string, expected: string, actual: string, success: boolean, scores: object }>>}
+ */
+async function evaluateRagDataset(_datasetPath, _config) {
+  const file = await fs.readFile(path.resolve(_datasetPath), 'utf-8');
+  const cases = JSON.parse(file);
+  const pipeline = createRagPipeline(_config);
+
+  const results = [];
+  for (const { prompt, expected } of cases) {
+    try {
+      logger.info({ prompt }, 'Evaluating case');
+      const actual = await pipeline.query(prompt);
+      const success = normalizeText(actual) === normalizeText(expected);
+      const scores = scoreAnswer(actual, expected);
+      results.push({ prompt, expected, actual, success, scores });
+    } catch (err) {
+      logger.error({ prompt, error: err.message }, 'Evaluation failed');
+      results.push({
+        prompt,
+        expected,
+        actual: '',
+        success: false,
+        scores: { bleu: 0, rouge: 0 },
+      });
+    }
+  }
+  return results;
+}
+
+function normalizeText(txt) {
+  return txt
+    .trim()
+    .toLowerCase()
+    .replace(/[^a-z0-9 ]/g, '')
+    .replace(/\s+/g, ' ');
+}
+
+module.exports = { evaluateRagDataset, normalizeText };

package/src/evaluate/scoring.js

@@ -0,0 +1,91 @@
+/**
+ * Version: 0.1.0
+ * Path: /src/evaluate/scoring.js
+ * Description: Basic scoring metrics (BLEU, ROUGE-L) for RAG output evaluation
+ * Author: Ali Kahwaji
+ */
+
+/**
+ * Tokenize a string by words
+ * @param {string} text
+ * @returns {string[]}
+ */
+function tokenize(text) {
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9 ]/g, '')
+    .split(/\s+/)
+    .filter(Boolean);
+}
+
+/**
+ * Compute BLEU-1 score: unigram precision
+ * @param {string[]} candidate - Generated tokens
+ * @param {string[]} reference - Reference tokens
+ * @returns {number}
+ */
+function computeBLEU(_candidate, _reference) {
+  const candCounts = _candidate.reduce(
+    (acc, t) => ((acc[t] = (acc[t] || 0) + 1), acc),
+    {},
+  );
+  const refCounts = _reference.reduce(
+    (acc, t) => ((acc[t] = (acc[t] || 0) + 1), acc),
+    {},
+  );
+
+  let match = 0;
+  for (const token of Object.keys(candCounts)) {
+    match += Math.min(candCounts[token], refCounts[token] || 0);
+  }
+  return _candidate.length ? match / _candidate.length : 0;
+}
+
+/**
+ * Compute ROUGE-L (Longest Common Subsequence) recall
+ * @param {string[]} candidate
+ * @param {string[]} reference
+ * @returns {number}
+ */
+function computeROUGE(_candidate, _reference) {
+  const m = _candidate.length;
+  const n = _reference.length;
+  const dp = Array(m + 1)
+    .fill(null)
+    .map(() => Array(n + 1).fill(0));
+
+  for (let i = 1; i <= m; i++) {
+    for (let j = 1; j <= n; j++) {
+      if (_candidate[i - 1] === _reference[j - 1]) {
+        dp[i][j] = dp[i - 1][j - 1] + 1;
+      } else {
+        dp[i][j] = Math.max(dp[i - 1][j], dp[i][j - 1]);
+      }
+    }
+  }
+
+  const lcs = dp[m][n];
+  return _reference.length ? lcs / _reference.length : 0;
+}
+
+/**
+ * Score a prompt-answer pair using multiple metrics
+ * @param {string} actual - LLM response
+ * @param {string} expected - Reference answer
+ * @returns {{ bleu: number, rouge: number }}
+ */
+function scoreAnswer(_actual, _expected) {
+  const candTokens = tokenize(_actual);
+  const refTokens = tokenize(_expected);
+  return {
+    bleu: computeBLEU(candTokens, refTokens),
+    rouge: computeROUGE(candTokens, refTokens),
+  };
+}
+
+module.exports = {
+  scoreAnswer,
+  tokenize,
+  computeBLEU,
+  computeROUGE,
+};

package/src/index.js

@@ -0,0 +1,67 @@
+/**
+ * RAG Pipeline Utils - Public API
+ * Main entry point for consumers of the library
+ */
+
+// Core pipeline functionality
+const { createRagPipeline } = require('./core/create-pipeline');
+
+// Configuration utilities
+const { loadConfig } = require('./config/load-config');
+const { validateRagrc } = require('./config/enhanced-ragrc-schema');
+const { normalizeConfig } = require('./config/normalize-config');
+
+// Plugin system
+const pluginRegistry = require('./core/plugin-registry');
+
+// Utilities
+const logger = require('./utils/logger');
+
+// AI/ML capabilities
+const MultiModalProcessor = require('./ai/multimodal');
+const AdaptiveRetrievalEngine = require('./ai/retrieval-engine');
+
+// DAG engine for complex workflows
+const { DAGEngine } = require('./dag/dag-engine');
+
+// Performance and observability
+const ParallelProcessor = require('./core/performance/parallel-processor');
+const eventLogger = require('./core/observability/event-logger');
+const metrics = require('./core/observability/metrics');
+
+// Enterprise features
+const { AuditLogger } = require('./enterprise/audit-logging');
+const { DataGovernance } = require('./enterprise/data-governance');
+
+module.exports = {
+  // Core API
+  createRagPipeline,
+  createPipeline: createRagPipeline, // Alias for backward compatibility
+
+  // Configuration
+  loadConfig,
+  validateRagrc,
+  normalizeConfig,
+
+  // Plugin system
+  pluginRegistry,
+
+  // Utilities
+  logger,
+
+  // AI/ML
+  MultiModalProcessor,
+  AdaptiveRetrievalEngine,
+
+  // Workflow engine
+  DAGEngine,
+
+  // Performance & Observability
+  ParallelProcessor,
+  eventLogger,
+  metrics,
+
+  // Enterprise
+  AuditLogger,
+  DataGovernance,
+};

package/src/ingest.js

@@ -0,0 +1,59 @@
+'use strict';
+/**
+ * Minimal ingest surface; wire real backend later.
+ */
+const path = require('path');
+const { logger } = require('./utils/logger.js');
+
+/**
+ * Ingest a single document by path.
+ * @param {string} filepath - Document path
+ * @param {object} config - Configuration object
+ * @returns {boolean} - Result (synchronous for now, can be made async later)
+ */
+function ingestDocument(filepath, config) {
+  // Validate document path - always throw synchronously for validation errors
+  if (!filepath || typeof filepath !== 'string' || filepath.trim() === '') {
+    throw new Error('Document path is required');
+  }
+
+  // Validate configuration - always throw synchronously for validation errors
+  if (!config || typeof config !== 'object') {
+    throw new Error('Configuration is required');
+  }
+
+  // Process document with safer error handling
+  try {
+    const abs = path.resolve(filepath);
+    logger.info({ file: abs }, 'ingest:start');
+
+    // TODO: real ingestion (mime sniff, loader selection, vector upsert)
+    // For now, return success
+    logger.info({ file: abs }, 'ingest:complete');
+    return true;
+  } catch (error) {
+    // Log processing errors but don't crash
+    logger.error({ file: filepath, error: error.message }, 'ingest:error');
+
+    // Re-throw with more context for better debugging
+    const enhancedError = new Error(
+      `Failed to ingest document '${filepath}': ${error.message}`,
+    );
+    enhancedError.originalError = error;
+    enhancedError.filepath = filepath;
+    throw enhancedError;
+  }
+}
+
+/**
+ * Async version of ingestDocument for future use
+ * @param {string} filepath - Document path
+ * @param {object} config - Configuration object
+ * @returns {Promise<boolean>} - Result promise
+ */
+async function ingestDocumentAsync(filepath, config) {
+  // Reuse sync validation and processing
+  return ingestDocument(filepath, config);
+}
+
+module.exports = { ingestDocument, ingestDocumentAsync };

package/src/loader/csv-loader.js

@@ -0,0 +1,38 @@
+/**
+ * CSV document loader with row-wise chunking
+ */
+
+const fs = require('fs/promises');
+const path = require('path');
+const { parse } = require('csv-parse/sync');
+
+class CSVLoader {
+  /**
+   * Load and parse a CSV file into chunks
+   * @param {string} filePath
+   * @returns {Promise<Array<{ chunk(): string[] }>>}
+   */
+  async load(filePath) {
+    const absPath = path.resolve(filePath);
+
+    try {
+      const raw = await fs.readFile(absPath, 'utf8');
+      const src = raw.replace(/^\uFEFF/, ''); // Remove BOM
+      const normalizedSrc = src.replace(/\r\n|\r/g, '\n'); // Normalize line endings
+
+      const records = parse(normalizedSrc, { columns: true });
+
+      return [
+        {
+          chunk: () => records.map((r) => Object.values(r).join(' ')),
+        },
+      ];
+    } catch (error) {
+      throw new Error(`CSV parsing failed for ${filePath}: ${error.message}`);
+    }
+  }
+}
+
+module.exports = {
+  CSVLoader,
+};

package/src/loader/directory-loader.js

@@ -0,0 +1,48 @@
+/**
+ * Version: 0.1.0
+ * Path: /src/loader/directory-loader.js
+ * Description: Recursive directory loader with file _type routing
+ * Author: Ali Kahwaji
+ */
+
+const fg = require('fast-glob');
+// eslint-disable-line global-require
+const { MarkdownLoader } = require('./markdown-loader.js');
+// eslint-disable-line global-require
+const { HTMLLoader } = require('./html-loader.js');
+// eslint-disable-line global-require
+const { CSVLoader } = require('./csv-loader.js');
+// eslint-disable-line global-require
+
+const loaders = {
+  '.md': new MarkdownLoader(),
+  '.html': new HTMLLoader(),
+  '.csv': new CSVLoader(),
+};
+
+class DirectoryLoader {
+  /**
+   * Load and route supported files from a directory
+   * @param {string} dirPath
+   * @returns {Promise<Array<{ chunk(): string[] }>>}
+   */
+  async load(dirPath) {
+    const matches = await fg([`${dirPath}/**/*.{md,html,csv}`]);
+    const all = [];
+    for (const file of matches) {
+      const ext = file.slice(file.lastIndexOf('.'));
+      const loader = loaders[ext];
+      if (loader) {
+        const docs = await loader.load(file);
+        all.push(...docs);
+      }
+    }
+    return all;
+  }
+}
+
+// Default export
+
+module.exports = {
+  DirectoryLoader,
+};

package/src/loader/html-loader.js

@@ -0,0 +1,62 @@
+/**
+ * Loader for HTML (.html) documents
+ */
+
+const fs = require('fs/promises');
+const path = require('path');
+const { JSDOM } = require('jsdom');
+
+/**
+ * HTMLLoader reads HTML files and extracts visible text.
+ * Implements the Loader interface.
+ */
+class HTMLLoader {
+  /**
+   * Load and extract text from an HTML file
+   * @param {string} filePath - Path to .html file
+   * @returns {Promise<Array<{ chunk(): string[] }>>}
+   */
+  async load(filePath) {
+    const absPath = path.resolve(filePath);
+    const raw = await fs.readFile(absPath, 'utf-8');
+    const dom = new JSDOM(raw);
+    let text = dom.window.document.body.textContent || '';
+
+    const MAX_LEN = 1_000_000;
+    text = String(text).replace(/\s+/g, ' ').trim();
+    if (text.length > MAX_LEN) text = text.slice(0, MAX_LEN);
+
+    return [
+      {
+        chunk: () => this._chunkText(text),
+      },
+    ];
+  }
+
+  /**
+   * Chunk HTML text content by sentence
+   * @param {string} input
+   * @param {number} maxLen
+   * @returns {string[]}
+   */
+  _chunkText(input, maxLen = 500) {
+    const sentences = input.split(/(?<=[.!?])\s+/);
+    const chunks = [];
+    let buffer = '';
+
+    for (const sentence of sentences) {
+      if ((buffer + sentence).length <= maxLen) {
+        buffer += sentence + ' ';
+      } else {
+        chunks.push(buffer.trim());
+        buffer = sentence + ' ';
+      }
+    }
+    if (buffer.trim()) chunks.push(buffer.trim());
+    return chunks;
+  }
+}
+
+module.exports = {
+  HTMLLoader,
+};

package/src/loader/markdown-loader.js

@@ -0,0 +1,67 @@
+/**
+const fs = require('fs');
+ * Version: 0.1.0
+ * Path: /src/loader/markdown-loader.js
+ * Description: Loader for Markdown (.md) documents
+ * Author: Ali Kahwaji
+ */
+
+const fs = require('fs/promises');
+// eslint-disable-line global-require
+const path = require('path');
+// eslint-disable-line global-require
+const { marked } = require('marked');
+// eslint-disable-line global-require
+
+/**
+ * MarkdownLoader reads and parses .md files into plain text chunks.
+ * Implements the Loader interface.
+ */
+class MarkdownLoader {
+  /**
+   * Load and parse Markdown file
+   * @param {string} _filePath - Path to .md file
+   * @returns {Promise<Array<{ chunk(): string[] }>>}
+   */
+  async load(_filePath) {
+    const absPath = path.resolve(_filePath);
+    const raw = await fs.readFile(absPath, 'utf-8');
+    const html = marked(raw);
+    const text = html.replace(/<[^>]+>/g, ''); // strip tags
+
+    return [
+      {
+        chunk: () => this._chunkText(text),
+      },
+    ];
+  }
+
+  /**
+   * Simple sentence-based chunking strategy
+   * @param {string} input
+   * @param {number} maxLen
+   * @returns {string[]}
+   */
+  _chunkText(input, maxLen = 500) {
+    const sentences = input.split(/(?<=[.!?])\s+/);
+    const chunks = [];
+    let buffer = '';
+
+    for (const sentence of sentences) {
+      if ((buffer + sentence).length <= maxLen) {
+        buffer += sentence + ' ';
+      } else {
+        chunks.push(buffer.trim());
+        buffer = sentence + ' ';
+      }
+    }
+    if (buffer.trim()) chunks.push(buffer.trim());
+    return chunks;
+  }
+}
+
+// Default export
+
+module.exports = {
+  MarkdownLoader,
+};

package/src/mocks/openai-embedder.js

@@ -0,0 +1,119 @@
+/**
+ * Version: 2.0.0
+ * Description: Mock implementation of OpenAI embedder for testing and development
+ * Author: Ali Kahwaji
+ */
+
+class OpenAIEmbedder {
+  constructor(_options = {}) {
+    this.apiKey = _options.apiKey || "mock-api-key";
+    this.model = _options.model || "text-embedding-ada-002";
+    this.dimensions = _options.dimensions || 1536;
+    this.options = _options;
+
+    // Configurable batch size from environment or options
+    this.batchSize =
+      parseInt(process.env.RAG_EMBEDDER_BATCH_SIZE) ||
+      _options.batchSize ||
+      100; // Default batch size for OpenAI API
+  }
+
+  /**
+   * Generate embeddings for an array of text chunks
+   * @param {string[]} chunks - Array of text chunks to embed
+   * @returns {Promise<number[][]>} Array of embedding vectors
+   */
+  async embed(chunks) {
+    if (!Array.isArray(chunks)) {
+      throw new Error("embed() expects an array of strings");
+    }
+
+    // Process in batches for better memory management and API rate limiting
+    const results = [];
+    for (let i = 0; i < chunks.length; i += this.batchSize) {
+      const batch = chunks.slice(i, i + this.batchSize);
+      const batchResults = await this._processBatch(batch);
+      results.push(...batchResults);
+
+      // Optional progress callback
+      if (this.options.onProgress) {
+        this.options.onProgress({
+          processed: Math.min(i + this.batchSize, chunks.length),
+          total: chunks.length,
+          stage: "embedding",
+        });
+      }
+
+      // Simulate API rate limiting delay
+      if (chunks.length > this.batchSize) {
+        await new Promise((resolve) => setTimeout(resolve, 50));
+      }
+    }
+
+    return results;
+  }
+
+  /**
+   * Process a batch of chunks
+   * @private
+   * @param {string[]} batch - Batch of text chunks
+   * @returns {Promise<number[][]>} Array of embedding vectors
+   */
+  async _processBatch(batch) {
+    // Return mock embeddings - deterministic based on content for testing
+    return batch.map((chunk) => this.#generateMockEmbedding(chunk));
+  }
+
+  /**
+   * Generate embedding for a single query string
+   * @param {string} query - Query string to embed
+   * @returns {Promise<number[]>} Embedding vector
+   */
+  async embedQuery(query) {
+    if (typeof query !== "string") {
+      throw new Error("embedQuery() expects a string");
+    }
+
+    return this.#generateMockEmbedding(query);
+  }
+
+  /**
+   * Generate a deterministic mock embedding based on input text
+   * @private
+   * @param {string} text - Input text
+   * @returns {number[]} Mock embedding vector
+   */
+  #generateMockEmbedding(text) {
+    // Create deterministic "embedding" based on text content
+    const hash = this.#simpleHash(text);
+    const embedding = [];
+
+    for (let i = 0; i < this.dimensions; i++) {
+      // Generate pseudo-random values based on hash and index
+      const value = Math.sin(hash + i) * 0.5;
+      embedding.push(value);
+    }
+
+    return embedding;
+  }
+
+  /**
+   * Simple hash function for deterministic mock data
+   * @private
+   * @param {string} str - Input string
+   * @returns {number} Hash value
+   */
+  #simpleHash(str) {
+    let hash = 0;
+    for (let i = 0; i < str.length; i++) {
+      const char = str.charCodeAt(i);
+      hash = (hash << 5) - hash + char;
+      hash = hash & hash; // Convert to 32-bit integer
+    }
+    return hash;
+  }
+}
+
+module.exports = {
+  OpenAIEmbedder,
+};

package/src/mocks/openai-llm.js

@@ -0,0 +1,77 @@
+/**
+ * Version: 2.0.0
+ * File: /src/mocks/openai-llm.js
+ * Description: Mock implementation of an OpenAI LLM with streaming support
+ * Author: Ali Kahwaji
+ */
+
+class OpenAILLM {
+  constructor(_options = {}) {
+    this.streamDelay = _options.streamDelay || 50; // ms between tokens
+    this.model = _options.model || "gpt-3.5-turbo";
+  }
+
+  /**
+   * Generates a complete response from input prompt and context
+   * @param {string} prompt
+   * @param {Array<{ text: string }>} context
+   * @returns {Promise<string>}
+   */
+  async generate(prompt, context) {
+    const contextText = Array.isArray(context)
+      ? context.map((d) => d.text || d).join(", ")
+      : "";
+    return `Generated answer using context: ${contextText}. Query: "${prompt}". This is a comprehensive response that addresses the user's question based on the retrieved information.`;
+  }
+
+  /**
+   * Generates a streaming response token-by-token
+   * @param {string} prompt
+   * @param {Array<{ text: string }>} context
+   * @returns {AsyncIterable<string>} Stream of tokens
+   */
+  async *generateStream(prompt, context) {
+    const fullResponse = await this.generate(prompt, context);
+    const tokens = this.#tokenize(fullResponse);
+
+    for (const token of tokens) {
+      // Simulate network delay between tokens
+      await this.#delay(this.streamDelay);
+      yield token;
+    }
+  }
+
+  /**
+   * Simple tokenization for mock streaming
+   * @private
+   * @param {string} text - Text to tokenize
+   * @returns {string[]} Array of tokens
+   */
+  #tokenize(text) {
+    // Simple word-based tokenization for demo purposes
+    // In real implementation, this would use proper tokenization
+    return text.split(/\s+/).map((word) => word + " ");
+  }
+
+  /**
+   * Utility delay function for simulating streaming
+   * @private
+   * @param {number} ms - Milliseconds to delay
+   * @returns {Promise<void>}
+   */
+  #delay(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+
+  /**
+   * Check if this LLM _instance supports streaming
+   * @returns {boolean}
+   */
+  supportsStreaming() {
+    return true;
+  }
+}
+
+// Default export
+
+module.exports = OpenAILLM;