@devilsdev/rag-pipeline-utils 2.3.0 → 2.4.0

package/src/observability/event-logger.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const { SecureLogger } = require('../utils/secure-logger');
+const { logger } = require('../utils/logger');
 
 const EventTypes = {
   PLUGIN_START: 'plugin.start',
@@ -88,7 +89,7 @@ class PipelineEventLogger {
         try {
           fn(evt);
         } catch (e) {
-          console.warn('Event listener error:', e);
+          logger.warn('Event listener error:', e);
         }
       }
     }

package/src/observability/security-monitor.js

@@ -826,5 +826,3 @@ module.exports = {
   getSecurityMonitor,
   initializeSecurityMonitoring,
 };
-
-module.exports.default = module.exports;

package/src/observability/telemetry.js

@@ -682,5 +682,3 @@ module.exports = {
   initializeTelemetry,
   traceFunction,
 };
-
-module.exports.default = module.exports;

package/src/reranker/cascade-reranker.js

@@ -0,0 +1,79 @@
+'use strict';
+
+/**
+ * Version: 1.0.0
+ * Path: /src/reranker/cascade-reranker.js
+ * Description: Chains multiple rerankers in sequence for multi-stage reranking
+ * Author: Ali Kahwaji
+ */
+
+const { EventEmitter } = require('events');
+
+/** @type {object} */
+const DEFAULT_CONFIG = {};
+
+/**
+ * Chains multiple rerankers in sequence, passing the output of each
+ * stage as input to the next. Implements the reranker contract:
+ * rerank(query, documents, options).
+ *
+ * @extends EventEmitter
+ */
+class CascadeReranker extends EventEmitter {
+  /**
+   * @param {object[]} [rerankers=[]] - Array of objects with .rerank() method
+   * @param {object} [options]
+   */
+  constructor(rerankers = [], options = {}) {
+    super();
+    this.rerankers = rerankers;
+    this.config = { ...DEFAULT_CONFIG, ...options };
+  }
+
+  /**
+   * Run documents through each reranker in sequence.
+   *
+   * @param {string} query - The search query
+   * @param {Array<object|string>} documents - Documents to rerank
+   * @param {object} [options]
+   * @param {number} [options.topK] - Maximum number of results to return
+   * @returns {Promise<object[]>} Reranked documents from the final stage
+   * @throws {Error} If no rerankers are configured
+   */
+  async rerank(query, documents, options = {}) {
+    if (this.rerankers.length === 0) {
+      throw new Error('CascadeReranker requires at least one reranker');
+    }
+
+    if (!Array.isArray(documents) || documents.length === 0) {
+      return [];
+    }
+
+    let currentDocs = documents;
+
+    for (let i = 0; i < this.rerankers.length; i++) {
+      const reranker = this.rerankers[i];
+      currentDocs = await reranker.rerank(query, currentDocs, options);
+
+      this.emit('stage', {
+        rerankerIndex: i,
+        resultCount: currentDocs.length,
+      });
+    }
+
+    this.emit('reranked', { count: currentDocs.length });
+
+    return currentDocs;
+  }
+
+  /**
+   * Add a reranker to the end of the cascade.
+   *
+   * @param {object} reranker - Must have a .rerank() method
+   */
+  addReranker(reranker) {
+    this.rerankers.push(reranker);
+  }
+}
+
+module.exports = { CascadeReranker, DEFAULT_CONFIG };

package/src/reranker/embedding-reranker.js

@@ -0,0 +1,124 @@
+'use strict';
+
+/**
+ * Version: 1.0.0
+ * Path: /src/reranker/embedding-reranker.js
+ * Description: Cosine similarity reranker using any embedder
+ * Author: Ali Kahwaji
+ */
+
+const { EventEmitter } = require('events');
+
+/** @type {object} */
+const DEFAULT_CONFIG = {};
+
+/**
+ * Reranks documents by computing cosine similarity between query and
+ * document embeddings produced by an external embedder.
+ * Implements the reranker contract: rerank(query, documents, options).
+ *
+ * @extends EventEmitter
+ */
+class EmbeddingReranker extends EventEmitter {
+  /**
+   * @param {object} embedder - Must expose .embedQuery(text) or .embed(text)
+   * @param {object} [options]
+   */
+  constructor(embedder, options = {}) {
+    super();
+    if (!embedder) {
+      throw new Error('EmbeddingReranker requires an embedder instance');
+    }
+    this.embedder = embedder;
+    this.config = { ...DEFAULT_CONFIG, ...options };
+  }
+
+  /**
+   * Rerank documents by cosine similarity to the query embedding.
+   *
+   * @param {string} query - The search query
+   * @param {Array<object|string>} documents - Documents to rerank
+   * @param {object} [options]
+   * @param {number} [options.topK] - Maximum number of results to return
+   * @returns {Promise<object[]>} Reranked documents with .relevanceScore
+   */
+  async rerank(query, documents, options = {}) {
+    if (!Array.isArray(documents) || documents.length === 0) {
+      return [];
+    }
+
+    const { topK = documents.length } = options;
+
+    // Get query embedding
+    const queryEmbedding = this.embedder.embedQuery
+      ? await this.embedder.embedQuery(query)
+      : await this.embedder.embed(query);
+
+    // Get document embeddings and compute similarities
+    const scored = await Promise.all(
+      documents.map(async (doc, originalIndex) => {
+        const text = doc.content || doc.text || String(doc);
+
+        const docEmbedding = this.embedder.embedQuery
+          ? await this.embedder.embedQuery(text)
+          : await this.embedder.embed(text);
+
+        const similarity = this._cosineSimilarity(queryEmbedding, docEmbedding);
+
+        return { doc, similarity, originalIndex };
+      }),
+    );
+
+    // Sort by similarity descending, then by original index for stability
+    scored.sort((a, b) => {
+      if (b.similarity !== a.similarity) return b.similarity - a.similarity;
+      return a.originalIndex - b.originalIndex;
+    });
+
+    const results = scored.slice(0, topK).map(({ doc, similarity }) => {
+      const result =
+        typeof doc === 'object' ? { ...doc } : { text: String(doc) };
+      result.relevanceScore = similarity;
+      return result;
+    });
+
+    this.emit('reranked', { count: results.length, topK });
+
+    return results;
+  }
+
+  /**
+   * Compute cosine similarity between two vectors.
+   *
+   * @param {number[]} a - First vector
+   * @param {number[]} b - Second vector
+   * @returns {number} Cosine similarity in [-1, 1], or 0 for zero vectors
+   */
+  _cosineSimilarity(a, b) {
+    if (!a || !b || a.length !== b.length || a.length === 0) {
+      return 0;
+    }
+
+    let dot = 0;
+    let magA = 0;
+    let magB = 0;
+
+    for (let i = 0; i < a.length; i++) {
+      dot += a[i] * b[i];
+      magA += a[i] * a[i];
+      magB += b[i] * b[i];
+    }
+
+    magA = Math.sqrt(magA);
+    magB = Math.sqrt(magB);
+
+    // Handle zero vectors
+    if (magA === 0 || magB === 0) {
+      return 0;
+    }
+
+    return dot / (magA * magB);
+  }
+}
+
+module.exports = { EmbeddingReranker, DEFAULT_CONFIG };

package/src/reranker/llm-reranker.js

@@ -30,12 +30,16 @@ class LLMReranker {
         }
 
         // Security validation: ensure all elements are valid numbers
-        if (!parsed.every(item => typeof item === 'number' && Number.isInteger(item))) {
+        if (
+          !parsed.every(
+            (item) => typeof item === 'number' && Number.isInteger(item),
+          )
+        ) {
           throw new Error('LLM response contains non-integer values');
         }
 
         // Security validation: ensure all indices are within valid bounds
-        const validIndices = parsed.filter(i => i >= 0 && i < docs.length);
+        const validIndices = parsed.filter((i) => i >= 0 && i < docs.length);
 
         // Security validation: remove duplicates to prevent manipulation
         rankingOrder = [...new Set(validIndices)];
@@ -108,4 +112,3 @@ const defaultInstance = new LLMReranker();
 // CJS+ESM interop pattern
 module.exports = defaultInstance;
 module.exports.LLMReranker = LLMReranker;
-module.exports.default = module.exports;

package/src/reranker/scoring-reranker.js

@@ -0,0 +1,123 @@
+'use strict';
+
+/**
+ * Version: 1.0.0
+ * Path: /src/reranker/scoring-reranker.js
+ * Description: BM25-based reranker with IDF-weighted scoring
+ * Author: Ali Kahwaji
+ */
+
+const { EventEmitter } = require('events');
+const { tokenize } = require('../evaluate/scoring');
+
+/** @type {{ k1: number, b: number }} */
+const DEFAULT_CONFIG = { k1: 1.2, b: 0.75 };
+
+/**
+ * Reranks documents using BM25-like scoring with IDF weighting.
+ * Implements the reranker contract: rerank(query, documents, options).
+ *
+ * @extends EventEmitter
+ */
+class ScoringReranker extends EventEmitter {
+  /**
+   * @param {object} [options]
+   * @param {number} [options.k1=1.2] - Term frequency saturation parameter
+   * @param {number} [options.b=0.75] - Length normalisation parameter
+   */
+  constructor(options = {}) {
+    super();
+    this.config = { ...DEFAULT_CONFIG, ...options };
+  }
+
+  /**
+   * Rerank documents against a query using BM25 scoring.
+   *
+   * @param {string} query - The search query
+   * @param {Array<object|string>} documents - Documents to rerank
+   * @param {object} [options]
+   * @param {number} [options.topK] - Maximum number of results to return
+   * @returns {Promise<object[]>} Reranked documents with .relevanceScore
+   */
+  async rerank(query, documents, options = {}) {
+    if (!Array.isArray(documents) || documents.length === 0) {
+      return [];
+    }
+
+    const { topK = documents.length } = options;
+    const { k1, b } = this.config;
+
+    const queryTokens = tokenize(query);
+    if (queryTokens.length === 0) {
+      return documents.slice(0, topK);
+    }
+
+    // Tokenize all documents and compute average document length
+    const docData = documents.map((doc) => {
+      const text = doc.content || doc.text || String(doc);
+      const tokens = tokenize(text);
+      return { doc, tokens };
+    });
+
+    const avgDocLen =
+      docData.reduce((sum, d) => sum + d.tokens.length, 0) / docData.length;
+
+    // Build document-frequency counts for IDF
+    const df = {};
+    for (const { tokens } of docData) {
+      const unique = new Set(tokens);
+      for (const token of unique) {
+        df[token] = (df[token] || 0) + 1;
+      }
+    }
+
+    const N = documents.length;
+
+    // Score each document
+    const scored = docData.map(({ doc, tokens }, originalIndex) => {
+      const docLen = tokens.length;
+
+      // Build term-frequency map for this document
+      const tf = {};
+      for (const t of tokens) {
+        tf[t] = (tf[t] || 0) + 1;
+      }
+
+      let score = 0;
+      for (const qToken of queryTokens) {
+        const termFreq = tf[qToken] || 0;
+        if (termFreq === 0) continue;
+
+        // IDF: log((N - df + 0.5) / (df + 0.5) + 1)
+        const docFreq = df[qToken] || 0;
+        const idf = Math.log((N - docFreq + 0.5) / (docFreq + 0.5) + 1);
+
+        // BM25 term score
+        const numerator = termFreq * (k1 + 1);
+        const denominator = termFreq + k1 * (1 - b + b * (docLen / avgDocLen));
+        score += idf * (numerator / denominator);
+      }
+
+      return { doc, score, originalIndex };
+    });
+
+    // Sort by score descending, then by original index for stability
+    scored.sort((a, bb) => {
+      if (bb.score !== a.score) return bb.score - a.score;
+      return a.originalIndex - bb.originalIndex;
+    });
+
+    const results = scored.slice(0, topK).map(({ doc, score }) => {
+      const result =
+        typeof doc === 'object' ? { ...doc } : { text: String(doc) };
+      result.relevanceScore = score;
+      return result;
+    });
+
+    this.emit('reranked', { count: results.length, topK });
+
+    return results;
+  }
+}
+
+module.exports = { ScoringReranker, DEFAULT_CONFIG };

package/src/retrieval/bm25-search.js

@@ -0,0 +1,157 @@
+'use strict';
+
+const { EventEmitter } = require('events');
+const { tokenize } = require('../evaluate/scoring');
+
+const DEFAULT_CONFIG = { k1: 1.2, b: 0.75 };
+
+class BM25Search extends EventEmitter {
+  constructor(options = {}) {
+    super();
+    this.config = { ...DEFAULT_CONFIG, ...options };
+    this.invertedIndex = new Map(); // token -> [{docIndex, tf}]
+    this.documents = [];
+    this.avgDocLength = 0;
+    this.docCount = 0;
+  }
+
+  /**
+   * Index documents for BM25 search
+   * @param {Array} documents - [{id, content|text, metadata?}]
+   */
+  index(documents) {
+    if (!Array.isArray(documents) || documents.length === 0) {
+      return;
+    }
+
+    this.documents = documents;
+    this.docCount = documents.length;
+    this.invertedIndex.clear();
+
+    let totalLength = 0;
+
+    for (let docIndex = 0; docIndex < documents.length; docIndex++) {
+      const doc = documents[docIndex];
+      const text = doc.content || doc.text || '';
+      const tokens = tokenize(text);
+
+      totalLength += tokens.length;
+
+      // Count term frequencies for this document
+      const tfMap = new Map();
+      for (const token of tokens) {
+        tfMap.set(token, (tfMap.get(token) || 0) + 1);
+      }
+
+      // Add to inverted index
+      for (const [token, tf] of tfMap) {
+        if (!this.invertedIndex.has(token)) {
+          this.invertedIndex.set(token, []);
+        }
+        this.invertedIndex.get(token).push({ docIndex, tf });
+      }
+    }
+
+    this.avgDocLength = this.docCount > 0 ? totalLength / this.docCount : 0;
+
+    this.emit('indexed', {
+      docCount: this.docCount,
+      uniqueTerms: this.invertedIndex.size,
+    });
+  }
+
+  /**
+   * Search using BM25 scoring
+   * @param {string} query
+   * @param {object} options - {k: 10}
+   * @returns {Array} [{id, score, content, metadata}] sorted by BM25 score
+   */
+  search(query, options = {}) {
+    const { k = 10 } = options;
+
+    if (!query || this.docCount === 0) {
+      return [];
+    }
+
+    const queryTokens = tokenize(query);
+    if (queryTokens.length === 0) {
+      return [];
+    }
+
+    const { k1, b } = this.config;
+    const N = this.docCount;
+
+    /** @type {Map<number, number>} docIndex -> score */
+    const scores = new Map();
+
+    for (const token of queryTokens) {
+      const postings = this.invertedIndex.get(token);
+      if (!postings) {
+        continue;
+      }
+
+      const df = postings.length;
+      const idf = Math.log((N - df + 0.5) / (df + 0.5) + 1);
+
+      for (const { docIndex, tf } of postings) {
+        const doc = this.documents[docIndex];
+        const docText = doc.content || doc.text || '';
+        const docLen = tokenize(docText).length;
+
+        const numerator = tf * (k1 + 1);
+        const denominator =
+          tf + k1 * (1 - b + b * (docLen / this.avgDocLength));
+        const termScore = idf * (numerator / denominator);
+
+        scores.set(docIndex, (scores.get(docIndex) || 0) + termScore);
+      }
+    }
+
+    // Build result array and sort
+    const results = [];
+    for (const [docIndex, score] of scores) {
+      const doc = this.documents[docIndex];
+      results.push({
+        id: doc.id,
+        score,
+        content: doc.content || doc.text || '',
+        metadata: doc.metadata || {},
+      });
+    }
+
+    results.sort((a, b) => b.score - a.score);
+    const topResults = results.slice(0, k);
+
+    this.emit('searched', {
+      query,
+      resultCount: topResults.length,
+      totalMatches: results.length,
+    });
+
+    return topResults;
+  }
+
+  /**
+   * Clear the index
+   */
+  clear() {
+    this.invertedIndex.clear();
+    this.documents = [];
+    this.avgDocLength = 0;
+    this.docCount = 0;
+  }
+
+  /**
+   * Get index stats
+   * @returns {{docCount: number, uniqueTerms: number, avgDocLength: number}}
+   */
+  getStats() {
+    return {
+      docCount: this.docCount,
+      uniqueTerms: this.invertedIndex.size,
+      avgDocLength: this.avgDocLength,
+    };
+  }
+}
+
+module.exports = { BM25Search, DEFAULT_CONFIG };

package/src/retrieval/hybrid-retriever.js

@@ -0,0 +1,111 @@
+'use strict';
+
+const { EventEmitter } = require('events');
+const { reciprocalRankFusion } = require('./rank-fusion');
+
+const DEFAULT_CONFIG = {
+  topK: 10,
+  fusionK: 60,
+};
+
+class HybridRetriever extends EventEmitter {
+  /**
+   * @param {Array<{retriever: object, weight: number, name: string}>} retrievers
+   * @param {object} options
+   */
+  constructor(retrievers = [], options = {}) {
+    super();
+    this.retrievers = retrievers.map((r, i) => ({
+      retriever: r.retriever,
+      weight: r.weight != null ? r.weight : 1.0,
+      name: r.name || `retriever-${i}`,
+    }));
+    this.config = { ...DEFAULT_CONFIG, ...options };
+  }
+
+  /**
+   * Run all retrievers in parallel, fuse results
+   * @param {object} query - {query: string, queryVector?: number[], topK?: number}
+   * @returns {Promise<Array>} Fused results
+   */
+  async retrieve(query) {
+    if (this.retrievers.length === 0) {
+      return [];
+    }
+
+    const topK = (query && query.topK) || this.config.topK;
+
+    // Run all retrievers in parallel
+    const promises = this.retrievers.map(({ retriever, name }) => {
+      const fn =
+        typeof retriever.retrieve === 'function'
+          ? retriever.retrieve.bind(retriever)
+          : typeof retriever.search === 'function'
+            ? retriever.search.bind(retriever)
+            : null;
+
+      if (!fn) {
+        return Promise.reject(
+          new Error(`Retriever "${name}" has no retrieve() or search() method`),
+        );
+      }
+
+      // Pass query string or the full query object depending on method
+      const arg = typeof query === 'string' ? query : query.query || query;
+      return fn(arg);
+    });
+
+    const settled = await Promise.allSettled(promises);
+
+    const resultSets = [];
+    const weights = [];
+    let successCount = 0;
+
+    for (let i = 0; i < settled.length; i++) {
+      const outcome = settled[i];
+      if (outcome.status === 'fulfilled') {
+        const results = Array.isArray(outcome.value) ? outcome.value : [];
+        resultSets.push(results);
+        weights.push(this.retrievers[i].weight);
+        successCount++;
+      } else {
+        // Graceful degradation: log warning and continue
+        this.emit('retrieverError', {
+          name: this.retrievers[i].name,
+          error: outcome.reason,
+        });
+      }
+    }
+
+    const fused = reciprocalRankFusion(resultSets, {
+      k: this.config.fusionK,
+      weights,
+    });
+
+    const topResults = fused.slice(0, topK);
+
+    this.emit('retrieved', {
+      totalRetrievers: this.retrievers.length,
+      successfulRetrievers: successCount,
+      resultCount: topResults.length,
+    });
+
+    return topResults;
+  }
+
+  /**
+   * Add a retriever
+   * @param {object} retriever - Object with .retrieve() or .search() method
+   * @param {number} weight - Weight for rank fusion
+   * @param {string} name - Name for this retriever
+   */
+  addRetriever(retriever, weight = 1.0, name = '') {
+    this.retrievers.push({
+      retriever,
+      weight,
+      name: name || `retriever-${this.retrievers.length}`,
+    });
+  }
+}
+
+module.exports = { HybridRetriever, DEFAULT_CONFIG };