@devilsdev/rag-pipeline-utils 2.4.5 → 2.6.0

package/src/retrieval/file-vector-store.js

@@ -0,0 +1,241 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { randomBytes } = require('crypto');
+const { digest } = require('../cache/component-keys');
+
+const FORMAT_VERSION = 1;
+
+/**
+ * Vector store persisted to a single JSON file, so separate processes (e.g.
+ * `rag-pipeline ingest` then `rag-pipeline query`) share one index. Intended
+ * for local use and small corpora: every query is a linear scan.
+ *
+ * The index records which embedder produced the vectors (name, dimensions and
+ * optional serialized state) and refuses queries whose vector dimension does
+ * not match, instead of silently comparing incompatible vectors.
+ */
+class FileVectorStore {
+  /**
+   * @param {object} options
+   * @param {string} options.path - Index file location
+   */
+  constructor(options = {}) {
+    if (!options.path) {
+      throw new Error('FileVectorStore requires a `path` option');
+    }
+    this.path = path.resolve(options.path);
+    this._index = null;
+    this._loadedDigest = null;
+    this._cacheId = randomBytes(8).toString('hex');
+    this._revision = 0;
+  }
+
+  /**
+   * Cache identity for createRagPipeline({ cache }): a digest of the index
+   * as loaded, so processes reading the same unchanged index share cached
+   * results; after this instance changes the index the key also carries the
+   * instance and a revision.
+   * @param {'retriever'} role
+   * @returns {string|null}
+   */
+  getCacheKey(role) {
+    if (role !== 'retriever') return null;
+    this.load();
+    return this._revision === 0
+      ? `file|${this._loadedDigest}`
+      : `file|${this._loadedDigest}|${this._cacheId}|${this._revision}`;
+  }
+
+  _emptyIndex() {
+    return { version: FORMAT_VERSION, embedder: null, documents: {} };
+  }
+
+  /**
+   * Read the index from disk (cached after the first call).
+   * @returns {{version: number, embedder: object|null, documents: object}}
+   */
+  load() {
+    if (this._index) return this._index;
+    if (!fs.existsSync(this.path)) {
+      this._index = this._emptyIndex();
+      this._loadedDigest = 'empty';
+      return this._index;
+    }
+    const raw = fs.readFileSync(this.path, 'utf8');
+    const parsed = JSON.parse(raw);
+    if (parsed.version !== FORMAT_VERSION || !parsed.documents) {
+      throw new Error(
+        `Unsupported index format in ${this.path} (version ${parsed.version})`,
+      );
+    }
+    this._index = parsed;
+    this._loadedDigest = digest(raw);
+    return this._index;
+  }
+
+  /** Write the index atomically: temp file in the same directory, then rename. */
+  save() {
+    const index = this.load();
+    fs.mkdirSync(path.dirname(this.path), { recursive: true });
+    const tmp = `${this.path}.${process.pid}.tmp`;
+    fs.writeFileSync(tmp, JSON.stringify(index));
+    fs.renameSync(tmp, this.path);
+  }
+
+  /** @returns {object|null} Embedder metadata recorded with the vectors */
+  getEmbedderInfo() {
+    return this.load().embedder;
+  }
+
+  /** @param {{name: string, dimensions: number, state?: object}} info */
+  setEmbedderInfo(info) {
+    this.load().embedder = info;
+    this._revision += 1;
+  }
+
+  /**
+   * Add or replace documents. Ids are caller-chosen; re-storing an id
+   * replaces the previous entry.
+   *
+   * @param {Array<{id: string, vector: number[], content: string, metadata?: object}>} docs
+   */
+  async store(docs) {
+    const { documents } = this.load();
+    this._revision += 1;
+    for (const doc of docs) {
+      if (!doc.id || !Array.isArray(doc.vector)) {
+        throw new Error(
+          'FileVectorStore.store(): each doc needs id and vector',
+        );
+      }
+      documents[doc.id] = {
+        vector: encodeVector(doc.vector),
+        content: doc.content || '',
+        metadata: doc.metadata || {},
+      };
+    }
+  }
+
+  /**
+   * Remove every document whose metadata.source equals `source`.
+   * @param {string} source
+   * @returns {number} Number of documents removed
+   */
+  removeBySource(source) {
+    const { documents } = this.load();
+    this._revision += 1;
+    let removed = 0;
+    for (const [id, doc] of Object.entries(documents)) {
+      if (doc.metadata?.source === source) {
+        delete documents[id];
+        removed++;
+      }
+    }
+    return removed;
+  }
+
+  /** Remove every document and the recorded embedder info. */
+  clear() {
+    this.load();
+    this._index = this._emptyIndex();
+    this._revision += 1;
+  }
+
+  /** @returns {Array<{id: string, content: string, metadata: object}>} */
+  list() {
+    return Object.entries(this.load().documents).map(([id, doc]) => ({
+      id,
+      content: doc.content,
+      metadata: doc.metadata,
+    }));
+  }
+
+  /** @returns {number} */
+  size() {
+    return Object.keys(this.load().documents).length;
+  }
+
+  /**
+   * Cosine-similarity search.
+   *
+   * @param {{queryVector: number[], topK?: number}} request
+   * @returns {Promise<Array<{id: string, score: number, content: string, metadata: object}>>}
+   */
+  async retrieve(request = {}) {
+    const { queryVector, topK = 5 } = request;
+    if (!Array.isArray(queryVector)) {
+      throw new Error(
+        'FileVectorStore.retrieve(): queryVector is required. Configure an embedder stage.',
+      );
+    }
+    const info = this.getEmbedderInfo();
+    if (info && info.dimensions && info.dimensions !== queryVector.length) {
+      throw new Error(
+        `Query vector has ${queryVector.length} dimensions but the index was built ` +
+          `with ${info.dimensions} (embedder "${info.name}"). Re-run ingest with the ` +
+          'current embedder.',
+      );
+    }
+
+    const queryMagnitude = Math.sqrt(
+      queryVector.reduce((sum, x) => sum + x * x, 0),
+    );
+    const results = Object.entries(this.load().documents).map(([id, doc]) => ({
+      id,
+      score: cosine(queryVector, queryMagnitude, doc.vector),
+      content: doc.content,
+      metadata: doc.metadata,
+    }));
+    results.sort((a, b) => b.score - a.score);
+    return results.slice(0, topK);
+  }
+}
+
+// Stored vectors keep ~7 significant digits; more only inflates the file.
+const round = (x) => Number(x.toPrecision(7));
+
+/**
+ * Mostly-zero vectors (e.g. LocalEmbedder's TF-IDF over a large vocabulary)
+ * are stored sparsely as { n, i, v }; dense ones as plain arrays.
+ */
+function encodeVector(vector) {
+  const indices = [];
+  const values = [];
+  for (let i = 0; i < vector.length; i++) {
+    if (vector[i] !== 0) {
+      indices.push(i);
+      values.push(round(vector[i]));
+    }
+  }
+  if (indices.length < vector.length / 4) {
+    return { n: vector.length, i: indices, v: values };
+  }
+  return vector.map(round);
+}
+
+/** Cosine similarity between a dense query and a stored (dense or sparse) vector. */
+function cosine(query, magQuery, stored) {
+  const sparse = !Array.isArray(stored);
+  const length = sparse ? stored.n : stored.length;
+  if (query.length !== length) return 0;
+
+  let dot = 0;
+  let magStored = 0;
+  if (sparse) {
+    for (let k = 0; k < stored.i.length; k++) {
+      dot += query[stored.i[k]] * stored.v[k];
+      magStored += stored.v[k] * stored.v[k];
+    }
+  } else {
+    for (let k = 0; k < length; k++) {
+      dot += query[k] * stored[k];
+      magStored += stored[k] * stored[k];
+    }
+  }
+  const magnitude = Math.sqrt(magQuery) * Math.sqrt(magStored);
+  return magnitude === 0 ? 0 : dot / magnitude;
+}
+
+module.exports = { FileVectorStore };

package/src/retrieval/hybrid-retriever.js

@@ -33,26 +33,23 @@ class HybridRetriever extends EventEmitter {
       return [];
     }
 
-    const topK = (query && query.topK) || this.config.topK;
-
-    // Run all retrievers in parallel
-    const promises = this.retrievers.map(({ retriever, name }) => {
-      const fn =
-        typeof retriever.retrieve === 'function'
-          ? retriever.retrieve.bind(retriever)
-          : typeof retriever.search === 'function'
-            ? retriever.search.bind(retriever)
-            : null;
-
-      if (!fn) {
-        return Promise.reject(
-          new Error(`Retriever "${name}" has no retrieve() or search() method`),
-        );
+    const request = typeof query === 'string' ? { query } : { ...query };
+    const topK = request.topK || this.config.topK;
+    request.topK = topK;
+
+    // Run all retrievers in parallel. retrieve() follows the retriever
+    // contract and gets the full request (so vector retrievers receive
+    // queryVector); search() is the BM25-style (text, { k }) signature.
+    const promises = this.retrievers.map(async ({ retriever, name }) => {
+      if (typeof retriever.retrieve === 'function') {
+        return retriever.retrieve(request);
       }
-
-      // Pass query string or the full query object depending on method
-      const arg = typeof query === 'string' ? query : query.query || query;
-      return fn(arg);
+      if (typeof retriever.search === 'function') {
+        return retriever.search(request.query, { k: topK });
+      }
+      throw new Error(
+        `Retriever "${name}" has no retrieve() or search() method`,
+      );
     });
 
     const settled = await Promise.allSettled(promises);
@@ -77,6 +74,11 @@ class HybridRetriever extends EventEmitter {
       }
     }
 
+    if (successCount === 0) {
+      const reasons = settled.map((o) => o.reason?.message).join('; ');
+      throw new Error(`HybridRetriever: all retrievers failed (${reasons})`);
+    }
+
     const fused = reciprocalRankFusion(resultSets, {
       k: this.config.fusionK,
       weights,
@@ -87,6 +89,7 @@ class HybridRetriever extends EventEmitter {
     this.emit('retrieved', {
       totalRetrievers: this.retrievers.length,
       successfulRetrievers: successCount,
+      failedRetrievers: this.retrievers.length - successCount,
       resultCount: topResults.length,
     });
 

package/src/security/jwt-config.js

@@ -15,6 +15,16 @@ const fs = require('fs');
 const path = require('path');
 const { logger } = require('../utils/logger');
 
+/**
+ * Parse an integer env value; unset or invalid values use the fallback but
+ * an explicit 0 is kept (unlike `parseInt(v) || fallback`).
+ * @private
+ */
+function intOrDefault(value, fallback) {
+  const parsed = parseInt(value, 10);
+  return Number.isNaN(parsed) ? fallback : parsed;
+}
+
 /**
  * Load JWT configuration from environment or defaults
  *
@@ -145,9 +155,10 @@ function loadJWTConfig() {
     audience: process.env.JWT_AUDIENCE || 'rag-pipeline-api',
 
     // Timing configuration
-    clockTolerance: parseInt(process.env.JWT_CLOCK_TOLERANCE, 10) || 30,
+    // An explicit 0 is honoured (only unset/invalid values fall back to 30)
+    clockTolerance: intOrDefault(process.env.JWT_CLOCK_TOLERANCE, 30),
     maxTokenAge: parseInt(process.env.JWT_EXPIRY, 10) || 3600, // 1 hour
-    notBeforeTolerance: parseInt(process.env.JWT_NBF_TOLERANCE, 10) || 30,
+    notBeforeTolerance: intOrDefault(process.env.JWT_NBF_TOLERANCE, 30),
 
     // Security options
     requireJti: process.env.JWT_REQUIRE_JTI !== 'false',

package/src/security/jwt-validator.js

@@ -11,11 +11,27 @@
  * - Replay attacks (with JTI tracking)
  * - Token expiration and timing issues
  *
+ * Replay protection (default): when `enableJtiTracking` is on (the default)
+ * and `singleUseTokens` is on (the default), every externally issued token
+ * (one this validator did not sign) is single-use: a second verify() of the
+ * same JTI throws "Token replay detected". Tokens signed by this validator
+ * instance may be verified repeatedly. To accept ordinary bearer-token reuse
+ * (the same access token presented on many requests until it expires), pass
+ * `singleUseTokens: false`; JTIs are then not recorded on verify().
+ *
+ * The JTI store is bounded: at most `maxTrackedJtis` entries (default
+ * 100000) are kept, each entry expires at the token's `exp` (plus the clock
+ * tolerance), expired entries are dropped first and, if the store is still
+ * full, the least-recently-used entry is evicted (a `jti_evicted` event is
+ * emitted). An evicted external JTI could be replayed until it expires, so
+ * size `maxTrackedJtis` for your peak number of live tokens.
+ *
  * @module security/jwt-validator
  * @since 2.3.0
  */
 
-const jwt = require('jsonwebtoken');
+// jsonwebtoken loads on first use (about 90 ms otherwise, on every import).
+const loadJwt = () => require('jsonwebtoken'); // eslint-disable-line global-require
 const crypto = require('crypto');
 const { EventEmitter } = require('events');
 const { logger } = require('../utils/logger');
@@ -91,9 +107,10 @@ class JWTValidator extends EventEmitter {
       audience: options.audience,
 
       // Timing configuration
-      clockTolerance: options.clockTolerance || 30, // seconds
+      // `??` so an explicit 0 is honoured (`||` silently turned 0 into 30).
+      clockTolerance: options.clockTolerance ?? 30, // seconds
       maxTokenAge: options.maxTokenAge || 3600, // 1 hour in seconds
-      notBeforeTolerance: options.notBeforeTolerance || 30, // seconds
+      notBeforeTolerance: options.notBeforeTolerance ?? 30, // seconds
 
       // Security options
       requireJti: options.requireJti !== false, // Require JWT ID by default
@@ -104,6 +121,11 @@ class JWTValidator extends EventEmitter {
       // JTI tracking for replay protection
       enableJtiTracking: options.enableJtiTracking !== false,
       jtiExpirationMs: options.jtiExpirationMs || 7 * 24 * 60 * 60 * 1000, // 7 days
+      // Treat external tokens as single-use (default). Set to false to allow
+      // normal bearer-token reuse until the token expires.
+      singleUseTokens: options.singleUseTokens !== false,
+      // Upper bound on tracked JTIs (LRU eviction beyond this).
+      maxTrackedJtis: options.maxTrackedJtis ?? 100000,
 
       // Additional validation
       validateClaims: options.validateClaims || {},
@@ -115,6 +137,13 @@ class JWTValidator extends EventEmitter {
     // Validate configuration
     this._validateConfig();
 
+    if (
+      !Number.isInteger(this.config.maxTrackedJtis) ||
+      this.config.maxTrackedJtis < 1
+    ) {
+      throw new Error('maxTrackedJtis must be a positive integer');
+    }
+
     // Initialize JTI tracking
     if (this.config.enableJtiTracking) {
       this.usedJtis = new Map(); // jti -> expiration timestamp (for external tokens)
@@ -287,7 +316,7 @@ class JWTValidator extends EventEmitter {
       }
 
       // Sign token with explicit algorithm
-      const token = jwt.sign(completePayload, signingKey, {
+      const token = loadJwt().sign(completePayload, signingKey, {
         algorithm: this.config.algorithm,
         // Do not allow algorithm to be overridden
         header: {
@@ -300,10 +329,9 @@ class JWTValidator extends EventEmitter {
       // Self-signed tokens are tracked to prevent JTI collisions with external tokens
       // but they can be verified multiple times (marked as reusable)
       if (this.config.enableJtiTracking && jti) {
-        this.selfSignedJtis.add(jti);
         // Also track in usedJtis with reusable flag to prevent external tokens from using same JTI
         const expirationTime = (now + expiresIn) * 1000;
-        this.usedJtis.set(jti, { expirationTime, reusable: true });
+        this._trackJti(jti, { expirationTime, reusable: true });
       }
 
       // Update statistics
@@ -406,7 +434,7 @@ class JWTValidator extends EventEmitter {
         verifyOptions.maxAge = options.maxAge || `${this.config.maxTokenAge}s`;
       }
 
-      const payload = jwt.verify(token, verificationKey, verifyOptions);
+      const payload = loadJwt().verify(token, verificationKey, verifyOptions);
 
       // Additional claim validation
       this._validateRequiredClaims(payload);
@@ -417,22 +445,27 @@ class JWTValidator extends EventEmitter {
       // - External tokens (non-reusable): if race condition occurs, both requests succeed on first use
       //   (acceptable) but subsequent replays are blocked (secure)
       if (this.config.enableJtiTracking && payload.jti) {
-        // Check if this is a self-signed (reusable) token
-        const isSelfSigned = this.selfSignedJtis.has(payload.jti);
-
-        if (!this.usedJtis.has(payload.jti)) {
-          // First time seeing this JTI - atomically track it
-          const expirationTime = payload.exp * 1000;
-          this.usedJtis.set(payload.jti, {
-            expirationTime,
-            reusable: isSelfSigned,
-          });
+        const jtiEntry = this._getJtiEntry(payload.jti);
+        // Self-signed (reusable) tokens are recorded at sign() time
+        const isSelfSigned = Boolean(
+          jtiEntry && jtiEntry.reusable && this.selfSignedJtis.has(payload.jti),
+        );
+
+        if (!jtiEntry) {
+          // First time seeing this JTI. Record it only when single-use
+          // checking is on; with singleUseTokens: false bearer tokens may be
+          // presented repeatedly until they expire.
+          if (this.config.singleUseTokens) {
+            this._trackJti(payload.jti, {
+              expirationTime: payload.exp * 1000,
+              reusable: false,
+            });
+          }
         } else {
           // JTI has been seen before
-          const jtiEntry = this.usedJtis.get(payload.jti);
 
           // Self-signed tokens are reusable - allow multiple verifications
-          if (jtiEntry.reusable && isSelfSigned) {
+          if (isSelfSigned) {
             // This is valid for refresh flows and load balancer retries
           } else {
             // External token being replayed - this is a security violation
@@ -583,6 +616,91 @@ class JWTValidator extends EventEmitter {
     }
   }
 
+  /**
+   * Expiry time (ms) of a JTI store entry, including clock tolerance.
+   * @private
+   */
+  _jtiExpiry(jtiEntry) {
+    // Handle both old format (number) and new format (object)
+    const expirationTime =
+      typeof jtiEntry === 'number' ? jtiEntry : jtiEntry.expirationTime;
+    return expirationTime + this.config.clockTolerance * 1000;
+  }
+
+  /**
+   * Remove a JTI from both tracking structures.
+   * @private
+   */
+  _forgetJti(jti) {
+    this.usedJtis.delete(jti);
+    this.selfSignedJtis.delete(jti);
+  }
+
+  /**
+   * Look up a JTI, dropping it if it has expired and refreshing its LRU
+   * position otherwise.
+   * @private
+   */
+  _getJtiEntry(jti) {
+    const entry = this.usedJtis.get(jti);
+    if (entry === undefined) return undefined;
+    if (this._jtiExpiry(entry) < Date.now()) {
+      this._forgetJti(jti);
+      return undefined;
+    }
+    // Map preserves insertion order: delete + set moves it to most-recent.
+    this.usedJtis.delete(jti);
+    this.usedJtis.set(jti, entry);
+    return entry;
+  }
+
+  /**
+   * Record a JTI in the bounded store. When the store is full, expired
+   * entries are purged first; if it is still full the least-recently-used
+   * entry is evicted.
+   * @private
+   */
+  _trackJti(jti, entry) {
+    this._forgetJti(jti);
+
+    if (this.usedJtis.size >= this.config.maxTrackedJtis) {
+      this._purgeExpiredJtis();
+    }
+
+    while (this.usedJtis.size >= this.config.maxTrackedJtis) {
+      const oldest = this.usedJtis.keys().next().value;
+      this._forgetJti(oldest);
+      this.emit('jti_evicted', {
+        jti: oldest,
+        reason: 'capacity',
+        maxTrackedJtis: this.config.maxTrackedJtis,
+        timestamp: new Date().toISOString(),
+      });
+    }
+
+    this.usedJtis.set(jti, entry);
+    if (entry.reusable) {
+      this.selfSignedJtis.add(jti);
+    }
+  }
+
+  /**
+   * Drop every expired JTI from the store.
+   * @private
+   * @returns {number} Number of entries removed
+   */
+  _purgeExpiredJtis() {
+    const now = Date.now();
+    let cleanedCount = 0;
+    for (const [jti, jtiEntry] of this.usedJtis.entries()) {
+      if (this._jtiExpiry(jtiEntry) < now) {
+        this._forgetJti(jti);
+        cleanedCount++;
+      }
+    }
+    return cleanedCount;
+  }
+
   /**
    * Start periodic cleanup of expired JTIs
    * @private
@@ -590,20 +708,7 @@ class JWTValidator extends EventEmitter {
   _startJtiCleanup() {
     this.jtiCleanupInterval = setInterval(
       () => {
-        const now = Date.now();
-        let cleanedCount = 0;
-
-        for (const [jti, jtiEntry] of this.usedJtis.entries()) {
-          // Handle both old format (number) and new format (object)
-          const expirationTime =
-            typeof jtiEntry === 'number' ? jtiEntry : jtiEntry.expirationTime;
-          if (expirationTime < now) {
-            this.usedJtis.delete(jti);
-            // Also remove from selfSignedJtis if it's there
-            this.selfSignedJtis.delete(jti);
-            cleanedCount++;
-          }
-        }
+        const cleanedCount = this._purgeExpiredJtis();
 
         if (cleanedCount > 0 && this.config.enableAuditLog) {
           this.emit('jti_cleanup', {
@@ -630,7 +735,7 @@ class JWTValidator extends EventEmitter {
    */
   decode(token) {
     try {
-      return jwt.decode(token, { complete: false });
+      return loadJwt().decode(token, { complete: false });
     } catch (error) {
       throw new Error(`Failed to decode token: ${error.message}`);
     }

package/src/security/plugin-sandbox.js

@@ -7,7 +7,23 @@
  * @version 2.1.1
  */
 
-const ivm = require('isolated-vm');
+// isolated-vm is a native addon and an optional peer dependency: load it only
+// when a sandbox is actually used so importing the package never requires it.
+let ivmModule = null;
+function loadIsolatedVm() {
+  if (!ivmModule) {
+    try {
+      ivmModule = require('isolated-vm');
+    } catch (error) {
+      throw new Error(
+        'PluginSandbox requires the optional peer dependency isolated-vm. ' +
+          'Install it with `npm install isolated-vm`. ' +
+          `(${error.message})`,
+      );
+    }
+  }
+  return ivmModule;
+}
 const fs = require('fs');
 const path = require('path');
 const crypto = require('crypto');
@@ -295,6 +311,7 @@ class PluginSandbox extends EventEmitter {
     let context;
 
     try {
+      const ivm = loadIsolatedVm();
       // Create new isolate with memory limit
       isolate = new ivm.Isolate({
         memoryLimit: Math.floor(this.options.memoryLimit / (1024 * 1024)), // Convert to MB