@devilsdev/rag-pipeline-utils 2.3.1 → 2.4.0

package/src/retrieval/rank-fusion.js

@@ -0,0 +1,62 @@
+'use strict';
+
+const DEFAULT_CONFIG = { k: 60 };
+
+/**
+ * Reciprocal Rank Fusion - merges multiple ranked result lists
+ * @param {Array<Array>} resultSets - Array of result arrays, each result has {id, score, ...}
+ * @param {object} options - {k: 60, weights: number[] (per result set)}
+ * @returns {Array} Fused, deduplicated, sorted results
+ */
+function reciprocalRankFusion(resultSets, options = {}) {
+  const { k = DEFAULT_CONFIG.k, weights } = options;
+
+  if (!Array.isArray(resultSets) || resultSets.length === 0) {
+    return [];
+  }
+
+  /** @type {Map<string, {score: number, fields: object}>} */
+  const scoreMap = new Map();
+
+  for (let setIndex = 0; setIndex < resultSets.length; setIndex++) {
+    const results = resultSets[setIndex];
+    if (!Array.isArray(results)) {
+      continue;
+    }
+
+    const weight =
+      Array.isArray(weights) && weights[setIndex] != null
+        ? weights[setIndex]
+        : 1.0;
+
+    for (let rank = 0; rank < results.length; rank++) {
+      const doc = results[rank];
+      if (!doc || doc.id == null) {
+        continue;
+      }
+
+      const id = String(doc.id);
+      const rrfScore = weight / (k + rank);
+
+      if (scoreMap.has(id)) {
+        const entry = scoreMap.get(id);
+        entry.score += rrfScore;
+      } else {
+        // Copy all original fields except score
+        const { score: _origScore, ...rest } = doc;
+        scoreMap.set(id, { score: rrfScore, fields: rest });
+      }
+    }
+  }
+
+  const fused = [];
+  for (const [id, entry] of scoreMap) {
+    fused.push({ ...entry.fields, id, score: entry.score });
+  }
+
+  fused.sort((a, b) => b.score - a.score);
+
+  return fused;
+}
+
+module.exports = { reciprocalRankFusion, DEFAULT_CONFIG };

package/src/security/jwt-config.js

@@ -13,6 +13,7 @@
 const crypto = require('crypto');
 const fs = require('fs');
 const path = require('path');
+const { logger } = require('../utils/logger');
 
 /**
  * Load JWT configuration from environment or defaults
@@ -59,7 +60,7 @@ function loadJWTConfig() {
         // Generate a random secret for development/test/undefined environments
         secret = crypto.randomBytes(64).toString('hex');
         if (!isUndefined) {
-          console.warn(
+          logger.warn(
             '[JWT_CONFIG] Warning: Using generated JWT_SECRET for development. ' +
               'Set JWT_SECRET environment variable for consistent tokens.',
           );
@@ -97,7 +98,7 @@ function loadJWTConfig() {
           'JWT_PUBLIC_KEY_PATH environment variable is required for asymmetric algorithms in production',
         );
       } else {
-        console.warn(
+        logger.warn(
           '[JWT_CONFIG] Warning: No JWT_PUBLIC_KEY_PATH set. Token verification will not be available.',
         );
       }
@@ -112,7 +113,7 @@ function loadJWTConfig() {
     }
 
     if (!privateKeyPath) {
-      console.warn(
+      logger.warn(
         '[JWT_CONFIG] Warning: No JWT_PRIVATE_KEY_PATH set. Token signing will not be available.',
       );
     } else {

package/src/security/jwt-validator.js

@@ -18,6 +18,7 @@
 const jwt = require('jsonwebtoken');
 const crypto = require('crypto');
 const { EventEmitter } = require('events');
+const { logger } = require('../utils/logger');
 
 /**
  * Secure JWT signing and validation algorithms
@@ -198,7 +199,7 @@ class JWTValidator extends EventEmitter {
         throw new Error('publicKey is required for asymmetric algorithms');
       }
       if (!privateKey) {
-        console.warn(
+        logger.warn(
           '[JWT_VALIDATOR] Warning: privateKey not provided. Token signing will not be available.',
         );
       }

package/src/security/plugin-sandbox.js

@@ -581,6 +581,3 @@ module.exports = {
   OutputSanitizer,
   run,
 };
-
-// CommonJS default export compatibility
-module.exports.default = module.exports;

package/src/security/plugin-signature-verifier.js

@@ -10,6 +10,7 @@
 const crypto = require('crypto');
 const fs = require('fs');
 const path = require('path');
+const { logger } = require('../utils/logger');
 
 /**
  * Plugin signature verification with Ed25519
@@ -285,9 +286,9 @@ class PluginSignatureVerifier {
 
     // In production, this should write to secure audit log
     if (process.env.NODE_ENV === 'production') {
-      console.warn('[AUDIT]', JSON.stringify(auditEntry));
+      logger.warn('[AUDIT]', JSON.stringify(auditEntry));
     } else {
-      console.log('[AUDIT]', auditEntry);
+      logger.info('[AUDIT]', auditEntry);
     }
   }
 
@@ -301,7 +302,7 @@ class PluginSignatureVerifier {
       this.auditLogger(result, manifest);
     } catch (error) {
       // Audit logging failure should not break verification
-      console.error('Audit logging failed:', error.message);
+      logger.error('Audit logging failed:', error.message);
     }
   }
 
@@ -409,6 +410,3 @@ module.exports = {
   PluginSignatureVerifier,
   createPluginVerifier,
 };
-
-// CommonJS/ESM interop
-module.exports.default = module.exports;

package/src/security/validator.js

@@ -31,6 +31,4 @@ function validateInput(input, opts = {}) {
   };
 }
 
-// CJS+ESM interop pattern
 module.exports = { validateInput };
-module.exports.default = module.exports;

package/src/streaming/embedding-stream.js

@@ -0,0 +1,197 @@
+'use strict';
+
+/**
+ * Streaming Embeddings Generator
+ *
+ * Async generator that yields embeddings chunk by chunk with backpressure
+ * support. Processes documents in configurable batches while monitoring
+ * memory usage to prevent OOM conditions in large-scale pipelines.
+ *
+ * Features:
+ * - Batch-based document embedding with configurable size
+ * - Memory-aware backpressure (pauses when heap exceeds threshold)
+ * - Per-batch error isolation (failures don't abort the stream)
+ * - Progress tracking via yielded metadata
+ * - Support for both `.embed()` and `.embedQuery()` embedder APIs
+ *
+ * @module streaming/embedding-stream
+ * @since 2.5.0
+ */
+
+/**
+ * Default configuration for the embedding stream
+ * @type {Object}
+ */
+const DEFAULT_CONFIG = {
+  /** @type {number} Number of documents per embedding batch */
+  batchSize: 50,
+  /** @type {number} Maximum concurrent batch operations */
+  maxConcurrency: 3,
+  /** @type {number} Maximum heap memory budget in megabytes */
+  maxMemoryMB: 512,
+  /** @type {number} Fraction of maxMemoryMB that triggers backpressure (0-1) */
+  backpressureThreshold: 0.8,
+};
+
+/**
+ * Delay helper that resolves after the given milliseconds.
+ *
+ * @param {number} ms - Milliseconds to wait
+ * @returns {Promise<void>}
+ * @private
+ */
+function delay(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Returns current heap usage in megabytes.
+ *
+ * @returns {number} Heap used in MB
+ * @private
+ */
+function heapUsedMB() {
+  return process.memoryUsage().heapUsed / (1024 * 1024);
+}
+
+/**
+ * Validates that the provided embedder exposes a supported embedding method.
+ *
+ * @param {Object} embedder - The embedder instance to validate
+ * @throws {TypeError} If the embedder lacks both `.embed()` and `.embedQuery()`
+ * @private
+ */
+function validateEmbedder(embedder) {
+  if (
+    !embedder ||
+    (typeof embedder.embed !== 'function' &&
+      typeof embedder.embedQuery !== 'function')
+  ) {
+    throw new TypeError(
+      'Embedder must expose an .embed() or .embedQuery() method',
+    );
+  }
+}
+
+/**
+ * Embeds a single document using the best available method on the embedder.
+ *
+ * Prefers `.embed()` (which typically accepts an array and returns an array of
+ * vectors) but falls back to `.embedQuery()` for single-document embedders.
+ *
+ * @param {Object} embedder - Embedder instance
+ * @param {string} content - Text content to embed
+ * @returns {Promise<number[]>} The embedding vector
+ * @private
+ */
+async function embedOne(embedder, content) {
+  if (typeof embedder.embed === 'function') {
+    const result = await embedder.embed([content]);
+    return Array.isArray(result) ? result[0] : result;
+  }
+  return embedder.embedQuery(content);
+}
+
+/**
+ * Async generator that streams embedding results for a collection of documents.
+ *
+ * Documents are processed in batches of `batchSize`. Before each batch the
+ * generator checks heap memory; when usage exceeds
+ * `backpressureThreshold * maxMemoryMB` it pauses briefly and retries,
+ * giving the GC time to reclaim memory.
+ *
+ * Each yielded value is an object containing the document id, the computed
+ * vector, the document index, and a progress fraction (0-1).
+ *
+ * Errors within a batch are caught and yielded as objects with an `error`
+ * property so that consumers can decide how to handle partial failures
+ * without losing the rest of the stream.
+ *
+ * @param {Array<{id: string, content: string}>} documents - Documents to embed
+ * @param {Object} embedder - Embedder with `.embed()` or `.embedQuery()`
+ * @param {Object} [options] - Override any key from {@link DEFAULT_CONFIG}
+ * @yields {{ id: string, vector: number[], index: number, progress: number }
+ *          | { id: string, error: Error, index: number, progress: number }}
+ *
+ * @example
+ *   const docs = [{ id: '1', content: 'hello' }, { id: '2', content: 'world' }];
+ *   for await (const emb of createEmbeddingStream(docs, myEmbedder)) {
+ *     console.log(emb.id, emb.vector.length);
+ *   }
+ */
+async function* createEmbeddingStream(documents, embedder, options = {}) {
+  // --- Validate inputs ---
+  if (!Array.isArray(documents)) {
+    throw new TypeError('documents must be an array');
+  }
+  validateEmbedder(embedder);
+
+  const config = { ...DEFAULT_CONFIG, ...options };
+  const total = documents.length;
+
+  if (total === 0) {
+    return;
+  }
+
+  const memoryLimit = config.backpressureThreshold * config.maxMemoryMB;
+
+  // --- Process batches ---
+  for (let batchStart = 0; batchStart < total; batchStart += config.batchSize) {
+    const batchEnd = Math.min(batchStart + config.batchSize, total);
+    const batch = documents.slice(batchStart, batchEnd);
+
+    // --- Backpressure: wait while memory is above threshold ---
+    let backpressureApplied = false;
+    while (heapUsedMB() > memoryLimit) {
+      backpressureApplied = true;
+      await delay(100);
+    }
+
+    // --- Embed the batch (isolate errors per batch) ---
+    try {
+      // Attempt bulk embed if the embedder supports arrays natively
+      if (typeof embedder.embed === 'function') {
+        const contents = batch.map((doc) => doc.content || '');
+        const vectors = await embedder.embed(contents);
+        const vectorArray = Array.isArray(vectors) ? vectors : [vectors];
+
+        for (let j = 0; j < batch.length; j++) {
+          const globalIndex = batchStart + j;
+          yield {
+            id: batch[j].id,
+            vector: vectorArray[j] || [],
+            index: globalIndex,
+            progress: (globalIndex + 1) / total,
+            ...(backpressureApplied ? { backpressure: true } : {}),
+          };
+        }
+      } else {
+        // Fall back to embedQuery one at a time
+        for (let j = 0; j < batch.length; j++) {
+          const globalIndex = batchStart + j;
+          const vector = await embedder.embedQuery(batch[j].content || '');
+          yield {
+            id: batch[j].id,
+            vector,
+            index: globalIndex,
+            progress: (globalIndex + 1) / total,
+            ...(backpressureApplied ? { backpressure: true } : {}),
+          };
+        }
+      }
+    } catch (err) {
+      // Yield error entries for each document in the failed batch
+      for (let j = 0; j < batch.length; j++) {
+        const globalIndex = batchStart + j;
+        yield {
+          id: batch[j].id,
+          error: err,
+          index: globalIndex,
+          progress: (globalIndex + 1) / total,
+        };
+      }
+    }
+  }
+}
+
+module.exports = { createEmbeddingStream, DEFAULT_CONFIG };

package/src/streaming/sse-adapter.js

@@ -0,0 +1,168 @@
+'use strict';
+
+/**
+ * SSE (Server-Sent Events) Adapter
+ *
+ * Converts AsyncGenerator tokens to Server-Sent Events format
+ * for real-time streaming of RAG pipeline responses over HTTP.
+ *
+ * @module streaming/sse-adapter
+ * @since 2.4.0
+ */
+
+const { EventEmitter } = require('events');
+
+/**
+ * Default configuration for SSE streaming
+ * @type {Object}
+ */
+const DEFAULT_CONFIG = {
+  /** @type {string} SSE event name for token events */
+  eventName: 'token',
+  /** @type {string} Sentinel value indicating stream completion */
+  doneEvent: '[DONE]',
+  /** @type {number} Interval in ms between heartbeat comments */
+  heartbeatIntervalMs: 15000,
+};
+
+/**
+ * Adapts an AsyncGenerator to the Server-Sent Events protocol.
+ *
+ * Writes well-formed SSE frames to an HTTP response, manages heartbeat
+ * keep-alive comments, and handles client disconnects gracefully.
+ *
+ * @extends EventEmitter
+ *
+ * @example
+ *   const adapter = new SSEAdapter({ heartbeatIntervalMs: 10000 });
+ *   adapter.on('token', (t) => console.log('sent', t));
+ *   await adapter.stream(asyncTokenGenerator, res);
+ */
+class SSEAdapter extends EventEmitter {
+  /**
+   * @param {Object} [options] - Configuration overrides
+   * @param {string} [options.eventName='token'] - SSE event name for token frames
+   * @param {string} [options.doneEvent='[DONE]'] - Sentinel value for the final data frame
+   * @param {number} [options.heartbeatIntervalMs=15000] - Heartbeat interval in milliseconds
+   */
+  constructor(options = {}) {
+    super();
+    this.config = { ...DEFAULT_CONFIG, ...options };
+  }
+
+  /**
+   * Stream tokens from an AsyncGenerator to an HTTP response as SSE.
+   *
+   * Sets the required SSE headers, starts a heartbeat timer, and iterates
+   * over the generator. Each yielded value should be an object with at least
+   * a `token` property and an optional `done` boolean.
+   *
+   * @param {AsyncGenerator<{token: string, done?: boolean}>} asyncGenerator - Token source
+   * @param {import('http').ServerResponse} httpResponse - Writable HTTP response
+   * @returns {Promise<void>} Resolves when the stream ends or the client disconnects
+   */
+  async stream(asyncGenerator, httpResponse) {
+    let heartbeatTimer = null;
+    let clientDisconnected = false;
+
+    // --- helpers --------------------------------------------------------
+    const cleanup = () => {
+      if (heartbeatTimer) {
+        clearInterval(heartbeatTimer);
+        heartbeatTimer = null;
+      }
+    };
+
+    // --- 1. SSE headers -------------------------------------------------
+    httpResponse.writeHead(200, {
+      'Content-Type': 'text/event-stream',
+      'Cache-Control': 'no-cache',
+      Connection: 'keep-alive',
+    });
+
+    // --- 2. Heartbeat keep-alive ----------------------------------------
+    heartbeatTimer = setInterval(() => {
+      if (!clientDisconnected) {
+        httpResponse.write(': heartbeat\n\n');
+      }
+    }, this.config.heartbeatIntervalMs);
+
+    // --- 3. Client disconnect handler -----------------------------------
+    const onClose = () => {
+      clientDisconnected = true;
+      cleanup();
+      this.emit('disconnect');
+    };
+    httpResponse.on('close', onClose);
+
+    // --- 4. Iterate over the generator ----------------------------------
+    try {
+      this.emit('started');
+
+      for await (const value of asyncGenerator) {
+        if (clientDisconnected) {
+          break;
+        }
+
+        if (value && value.done) {
+          // Final frame
+          httpResponse.write(`data: ${this.config.doneEvent}\n\n`);
+          this.emit('done');
+          break;
+        }
+
+        const token = value && value.token !== undefined ? value.token : value;
+        const frame = this.formatEvent(this.config.eventName, { token });
+        httpResponse.write(frame);
+        this.emit('token', token);
+      }
+
+      // If the generator is exhausted without an explicit done flag, send DONE
+      if (!clientDisconnected && !httpResponse.writableEnded) {
+        httpResponse.write(`data: ${this.config.doneEvent}\n\n`);
+        this.emit('done');
+      }
+    } catch (err) {
+      this.emit('error', err);
+
+      if (!clientDisconnected && !httpResponse.writableEnded) {
+        const errorFrame = this.formatEvent('error', {
+          message: err.message || 'Internal streaming error',
+        });
+        httpResponse.write(errorFrame);
+      }
+    } finally {
+      cleanup();
+      httpResponse.removeListener('close', onClose);
+
+      if (!httpResponse.writableEnded) {
+        httpResponse.end();
+      }
+    }
+  }
+
+  /**
+   * Format a single SSE event string.
+   *
+   * Multi-line data is split so each line is prefixed with `data: ` as
+   * required by the SSE specification.
+   *
+   * @param {string} eventName - The SSE event type
+   * @param {*} data - Payload (will be JSON-stringified)
+   * @returns {string} Fully formatted SSE frame including trailing blank line
+   */
+  formatEvent(eventName, data) {
+    const json = JSON.stringify(data);
+    const lines = json.split('\n');
+
+    let frame = `event: ${eventName}\n`;
+    for (const line of lines) {
+      frame += `data: ${line}\n`;
+    }
+    frame += '\n';
+
+    return frame;
+  }
+}
+
+module.exports = { SSEAdapter, DEFAULT_CONFIG };