@devilsdev/rag-pipeline-utils 2.4.5 → 2.6.0

package/src/utils/deprecate.js

@@ -0,0 +1,69 @@
+'use strict';
+
+/**
+ * Deprecation wrappers for public exports (see SEMVER.md, "Deprecation
+ * process"): the export keeps working and warns once per process when used.
+ */
+
+const wrapped = new WeakMap();
+const warned = new Set();
+
+function warnOnce(code, message) {
+  if (warned.has(code)) return;
+  warned.add(code);
+  process.emitWarning(message, { type: 'DeprecationWarning', code });
+}
+
+/**
+ * Wrap a class so constructing it (or calling a static method) emits a
+ * one-time DeprecationWarning. A Proxy keeps the original's identity for
+ * prototype, name, statics, `extends` and `instanceof`, so instances created
+ * inside the library still satisfy `instanceof Wrapped`. The same class
+ * always gets the same wrapper, so aliases stay identical.
+ *
+ * @param {Function} Class - Class to wrap
+ * @param {object} info
+ * @param {string} info.name - Export name, used in the message and code
+ * @param {string} info.message - Full warning text
+ * @returns {Function} The wrapped class
+ */
+function deprecateClass(Class, { name, message }) {
+  if (wrapped.has(Class)) return wrapped.get(Class);
+  // SSOManager -> RAG_DEP_SSO_MANAGER, OAuth2Provider -> RAG_DEP_O_AUTH2_PROVIDER
+  const snake = name
+    .replace(/([a-z0-9])([A-Z])/g, '$1_$2')
+    .replace(/([A-Z]+)([A-Z][a-z])/g, '$1_$2')
+    .toUpperCase();
+  const code = `RAG_DEP_${snake}`;
+  const proxy = new Proxy(Class, {
+    construct(target, args, newTarget) {
+      warnOnce(code, message);
+      return Reflect.construct(
+        target,
+        args,
+        newTarget === proxy ? target : newTarget,
+      );
+    },
+    get(target, property, receiver) {
+      const value = Reflect.get(target, property, receiver);
+      // Only the class's own static methods; inherited Function.prototype
+      // members (call, bind, toString, Symbol.hasInstance for instanceof)
+      // must not warn.
+      const ownStatic =
+        typeof value === 'function' &&
+        property !== 'prototype' &&
+        Object.prototype.hasOwnProperty.call(target, property);
+      if (ownStatic) {
+        return function deprecatedStatic(...args) {
+          warnOnce(code, message);
+          return value.apply(this === proxy ? target : this, args);
+        };
+      }
+      return value;
+    },
+  });
+  wrapped.set(Class, proxy);
+  return proxy;
+}
+
+module.exports = { deprecateClass };

package/src/utils/embed-query.js

@@ -0,0 +1,33 @@
+'use strict';
+
+/**
+ * Embed a single query string with any supported embedder shape.
+ *
+ * Prefers embedQuery(); falls back to embed(). Accepts the common return
+ * shapes: a vector, a list of vectors, or [{ values | vector | embedding }].
+ *
+ * @param {object} embedder - Object with embedQuery() or embed()
+ * @param {string} query - Text to embed
+ * @param {{ signal?: AbortSignal }} [options] - Passed to the embedder as its
+ *   second argument (only when given, so embedders keep their usual arity)
+ * @returns {Promise<number[]|undefined>} The query vector
+ */
+async function embedQuery(embedder, query, options) {
+  const args = options ? [query, options] : [query];
+  const raw = embedder.embedQuery
+    ? await embedder.embedQuery(...args)
+    : await embedder.embed(...args);
+  const first = Array.isArray(raw) && typeof raw[0] !== 'number' ? raw[0] : raw;
+  if (Array.isArray(first)) return first;
+  return first?.values || first?.vector || first?.embedding;
+}
+
+/**
+ * @param {object} [embedder]
+ * @returns {boolean} Whether embedQuery() can use this embedder
+ */
+function canEmbed(embedder) {
+  return Boolean(embedder && (embedder.embedQuery || embedder.embed));
+}
+
+module.exports = { embedQuery, canEmbed };

package/src/utils/input-sanitizer.js

@@ -65,6 +65,21 @@ const ValidationPatterns = {
   hex: /^[0-9a-fA-F]+$/,
 };
 
+/**
+ * Test a shared pattern from start of string. The DangerousPatterns regexes
+ * carry the /g flag (needed by replace()), and RegExp#test() on a /g regex
+ * resumes from lastIndex left by the previous call, which made a match
+ * against one input skip part of the next and let payloads through.
+ *
+ * @param {RegExp} pattern
+ * @param {string} str
+ * @returns {boolean}
+ */
+function matchesPattern(pattern, str) {
+  pattern.lastIndex = 0;
+  return pattern.test(str);
+}
+
 /**
  * Dangerous patterns that indicate attacks (compiled for performance)
  */
@@ -476,7 +491,7 @@ class InputSanitizer {
 
     // Apply all XSS patterns
     for (const pattern of DangerousPatterns.xss) {
-      if (pattern.test(sanitized)) {
+      if (matchesPattern(pattern, sanitized)) {
         this.stats.blocked++;
         sanitized = sanitized.replace(pattern, '');
       }
@@ -497,7 +512,7 @@ class InputSanitizer {
 
     // Check for SQL injection patterns
     for (const pattern of DangerousPatterns.sql) {
-      if (pattern.test(sanitized)) {
+      if (matchesPattern(pattern, sanitized)) {
         this.stats.blocked++;
         if (this.options.throwOnInvalid) {
           throw new Error('Potential SQL injection detected');
@@ -520,7 +535,7 @@ class InputSanitizer {
   _sanitizeCommand(str) {
     // Check for command injection patterns
     for (const pattern of DangerousPatterns.command) {
-      if (pattern.test(str)) {
+      if (matchesPattern(pattern, str)) {
         this.stats.blocked++;
         if (this.options.throwOnInvalid) {
           throw new Error('Potential command injection detected');
@@ -560,7 +575,7 @@ class InputSanitizer {
 
     // Check for path traversal patterns in both original and fully decoded versions
     for (const pattern of DangerousPatterns.pathTraversal) {
-      if (pattern.test(str) || pattern.test(current)) {
+      if (matchesPattern(pattern, str) || matchesPattern(pattern, current)) {
         this.stats.blocked++;
         // SECURITY: Always throw on path traversal - it should never be silently sanitized
         // Empty paths could resolve to application root, creating security vulnerabilities

package/src/utils/logger.js

@@ -31,9 +31,20 @@ const getLogLevel = () => {
 // Determine if pretty printing should be enabled
 const shouldUsePrettyPrint = () => {
   // Enable pretty print in development or when explicitly requested
-  return (
-    process.env.NODE_ENV === 'development' || process.env.LOG_PRETTY === 'true'
-  );
+  const requested =
+    process.env.NODE_ENV === 'development' || process.env.LOG_PRETTY === 'true';
+  return requested && isPinoPrettyAvailable();
+};
+
+// pino-pretty is an optional peer dependency; pino throws at startup if the
+// transport target cannot be resolved, so fall back to JSON output instead.
+const isPinoPrettyAvailable = () => {
+  try {
+    require.resolve('pino-pretty');
+    return true;
+  } catch {
+    return false;
+  }
 };
 
 /**
@@ -68,6 +79,25 @@ const baseLogger = pino({
   }),
 });
 
+/**
+ * Coerce the meta argument into an object. Callers often pass
+ * console-style extras such as `logger.error("Failed:", error.message)`;
+ * spreading a string would log one key per character.
+ *
+ * @param {*} meta
+ * @returns {object}
+ */
+function normalizeMeta(meta) {
+  if (meta == null) return {};
+  if (meta instanceof Error) {
+    return { error: meta.message, stack: meta.stack };
+  }
+  if (typeof meta !== 'object' || Array.isArray(meta)) {
+    return { detail: meta };
+  }
+  return meta;
+}
+
 /**
  * Enhanced logger with correlation ID support and structured context
  */
@@ -118,7 +148,7 @@ class StructuredLogger {
       ...this.context,
       correlationId: this.correlationId || crypto.randomUUID(),
       timestamp: new Date().toISOString(),
-      ...meta,
+      ...normalizeMeta(meta),
     };
 
     this.logger[level](logData, message);

package/src/utils/run-abort.js

@@ -0,0 +1,102 @@
+'use strict';
+
+/** Message used for a run that exceeded its timeout. */
+const TIMEOUT_MESSAGE = 'Execution timeout';
+/** Message used for a run cancelled through the caller's AbortSignal. */
+const ABORTED_MESSAGE = 'Execution aborted';
+
+function timeoutError() {
+  const err = new Error(TIMEOUT_MESSAGE);
+  err.name = 'TimeoutError';
+  err.code = 'ETIMEDOUT';
+  return err;
+}
+
+function callerAbortError(signal) {
+  const err = new Error(ABORTED_MESSAGE);
+  err.name = 'AbortError';
+  err.code = 'ABORT_ERR';
+  if (signal && signal.reason !== undefined) err.cause = signal.reason;
+  return err;
+}
+
+/**
+ * Per-run cancellation: one AbortController that is aborted when `timeout`
+ * elapses or when the caller's `signal` aborts. Its `signal` is handed to
+ * every component call; `race()` makes the run settle as soon as it is
+ * aborted even if a component ignores the signal; `throwIfAborted()` stops
+ * work between stages.
+ *
+ * @param {object} [options]
+ * @param {number} [options.timeout] - Milliseconds before the run is aborted
+ * @param {AbortSignal} [options.signal] - Caller's signal to follow
+ * @returns {{
+ *   signal: AbortSignal,
+ *   race: (promise: Promise<any>) => Promise<any>,
+ *   throwIfAborted: () => void,
+ *   clearTimer: () => void,
+ *   detach: () => void,
+ *   cleanup: () => void
+ * }}
+ */
+function createRunAbort({ timeout, signal } = {}) {
+  const controller = new AbortController();
+  let timer = null;
+  let onCallerAbort = null;
+  let rejectAborted;
+  const aborted = new Promise((_, reject) => {
+    rejectAborted = reject;
+  });
+  // The rejection is observed through race(); never report it as unhandled.
+  aborted.catch(() => {});
+
+  const abort = (error) => {
+    if (controller.signal.aborted) return;
+    controller.abort(error);
+    rejectAborted(error);
+  };
+
+  if (typeof timeout === 'number' && timeout > 0) {
+    timer = setTimeout(() => abort(timeoutError()), timeout);
+  }
+  if (signal && typeof signal.addEventListener === 'function') {
+    if (signal.aborted) {
+      abort(callerAbortError(signal));
+    } else {
+      onCallerAbort = () => abort(callerAbortError(signal));
+      signal.addEventListener('abort', onCallerAbort, { once: true });
+    }
+  }
+
+  const clearTimer = () => {
+    if (timer) clearTimeout(timer);
+    timer = null;
+  };
+  const detach = () => {
+    if (onCallerAbort) signal.removeEventListener('abort', onCallerAbort);
+    onCallerAbort = null;
+  };
+
+  return {
+    signal: controller.signal,
+    race(promise) {
+      if (controller.signal.aborted) {
+        // Swallow the late outcome of work that is no longer awaited.
+        Promise.resolve(promise).catch(() => {});
+        return Promise.reject(controller.signal.reason);
+      }
+      return Promise.race([promise, aborted]);
+    },
+    throwIfAborted() {
+      if (controller.signal.aborted) throw controller.signal.reason;
+    },
+    clearTimer,
+    detach,
+    cleanup() {
+      clearTimer();
+      detach();
+    },
+  };
+}
+
+module.exports = { createRunAbort, TIMEOUT_MESSAGE, ABORTED_MESSAGE };

package/src/ai/federated-learning.js

@@ -1,211 +0,0 @@
-/**
- * Federated Learning Coordinator
- * Event-driven federated learning with deterministic aggregation
- */
-
-"use strict";
-
-const { EventEmitter } = require("events");
-
-// In-memory storage for federations
-const federations = new Map();
-let federationCounter = 0;
-
-// Create EventEmitter instance
-const eventEmitter = new EventEmitter();
-
-/**
- * Create a new federation
- * @param {string} tenantId - Tenant identifier
- * @param {object} modelConfig - Model configuration
- * @param {object} options - Federation options
- * @returns {string} Federation ID
- */
-function createFederation(tenantId, modelConfig, options = {}) {
-  const federationId = `fed-${++federationCounter}`;
-  const federation = {
-    tenantId,
-    modelConfig,
-    options,
-    rounds: 0,
-    participants: [],
-  };
-
-  federations.set(federationId, federation);
-  return federationId;
-}
-
-/**
- * Register multiple participants in a federation
- * @param {string} federationId - Federation ID
- * @param {array} participantsArray - Participants to register
- * @returns {Promise<string|array>} Single string ID if one participant, array of IDs if multiple
- */
-async function registerParticipants(federationId, participantsArray) {
-  const federation = federations.get(federationId);
-  if (!federation) {
-    throw new Error(`Federation ${federationId} not found`);
-  }
-
-  const participantIds = [];
-  for (const participant of participantsArray) {
-    // Validate participant - throw error if invalid
-    if (!participant || typeof participant !== "object") {
-      throw new Error("Participant not eligible");
-    }
-
-    // Check data size requirement
-    if (
-      typeof participant.dataSize === "number" &&
-      participant.dataSize < 100
-    ) {
-      throw new Error("Participant not eligible");
-    }
-
-    const participantId =
-      participant.id || `p-${federation.participants.length + 1}`;
-    federation.participants.push({
-      id: participantId,
-      status: "active",
-      weight: participant.weight || 1,
-      ...participant,
-    });
-    participantIds.push(participantId);
-  }
-
-  // Return single string if only one participant, array if multiple (for test compatibility)
-  return participantsArray.length === 1 ? participantIds[0] : participantIds;
-}
-
-/**
- * Conduct a federated learning round
- * @param {string} federationId - Federation ID
- * @returns {object} Round result
- */
-function conductRound(federationId) {
-  const federation = federations.get(federationId);
-  if (!federation) {
-    throw new Error(`Federation ${federationId} not found`);
-  }
-
-  const round = ++federation.rounds;
-  const roundId = `r-${Date.now()}`;
-
-  // Emit round started event
-  eventEmitter.emit("federated_round_started", {
-    federationId,
-    round,
-  });
-
-  // Compute deterministic convergence metrics
-  const participants = federation.participants;
-  const globalAccuracy = 0.9 + participants.length * 0.01; // Slight improvement with more participants
-  const loss = Math.max(0.1 - participants.length * 0.005, 0.01); // Lower loss with more participants
-
-  // Emit round completed event
-  eventEmitter.emit("federated_round_completed", {
-    federationId,
-    metrics: { globalAccuracy, loss },
-  });
-
-  return {
-    roundId,
-    round,
-    participants: participants.length,
-    convergence: {
-      globalAccuracy: Math.round(globalAccuracy * 1000) / 1000,
-      loss: Math.round(loss * 1000) / 1000,
-    },
-  };
-}
-
-/**
- * Get federation statistics
- * @param {string} federationId - Federation ID
- * @returns {object} Federation stats
- */
-function getStats(federationId) {
-  const federation = federations.get(federationId);
-  if (!federation) {
-    throw new Error(`Federation ${federationId} not found`);
-  }
-
-  return {
-    federation: {
-      id: federationId,
-      rounds: federation.rounds,
-    },
-    participants: federation.participants.map((p) => ({
-      id: p.id,
-      status: p.status || "active",
-      weight: p.weight || 1,
-    })),
-    performance: {
-      averageAccuracy: 0.92,
-      convergenceRate: 0.85,
-      communicationRounds: federation.rounds,
-    },
-    privacy: {
-      differentialPrivacy: federation.options.differentialPrivacy || false,
-      encryptionEnabled: federation.options.encryptionEnabled || true,
-      dataLocality: "preserved",
-    },
-  };
-}
-
-/**
- * Validate participant eligibility
- * @param {object} participant - Participant to validate
- * @returns {boolean} Whether participant is valid
- */
-function validateParticipant(participant) {
-  if (!participant || typeof participant !== "object") {
-    return false;
-  }
-
-  // Check if participant has non-empty string id
-  if (
-    !participant.id ||
-    typeof participant.id !== "string" ||
-    participant.id.trim() === ""
-  ) {
-    return false;
-  }
-
-  // Check if participant is explicitly marked as ineligible
-  if (participant.eligible === false) {
-    return false;
-  }
-
-  return true;
-}
-
-/**
- * Clear all stored data (for testing)
- */
-function clear() {
-  federations.clear();
-  federationCounter = 0;
-  eventEmitter.removeAllListeners();
-}
-
-// Singleton object with required methods and EventEmitter functionality
-const federatedLearningCoordinator = {
-  // Core federated learning methods
-  createFederation,
-  registerParticipants,
-  conductRound,
-  getStats,
-  validateParticipant,
-
-  // EventEmitter methods
-  on: eventEmitter.on.bind(eventEmitter),
-  off: eventEmitter.removeListener.bind(eventEmitter),
-  emit: eventEmitter.emit.bind(eventEmitter),
-
-  // Additional utility methods
-  clear,
-};
-
-// Export singleton as default
-module.exports = federatedLearningCoordinator;

package/src/ai/federation.js

@@ -1,71 +0,0 @@
-/**
- * Federated Learning Coordinator
- * Manages distributed learning across multiple participants
- * Extracted from ai/index.js per CLAUDE.md decomposition requirements
- */
-
-"use strict";
-
-// Import the existing federated learning implementation
-const federatedLearning = require("./federated-learning");
-
-/**
- * FederatedLearningCoordinator class
- * Coordinates federated learning processes across multiple nodes
- */
-class FederatedLearningCoordinator {
-  constructor() {
-    this.participants = new Map();
-    this.rounds = new Map();
-    this.modelVersions = new Map();
-    this.aggregationStrategies = new Set(["fedavg", "fedprox", "scaffold"]);
-  }
-
-  /**
-   * Start federated learning round
-   * @param {string} tenantId - Tenant identifier
-   * @param {object} roundConfig - Round configuration
-   * @returns {Promise<string>} Round ID
-   */
-  async startRound(tenantId, roundConfig) {
-    return await federatedLearning.startRound(tenantId, roundConfig);
-  }
-
-  /**
-   * Register participant in federated learning
-   * @param {string} tenantId - Tenant identifier
-   * @param {string} participantId - Participant identifier
-   * @param {object} capabilities - Participant capabilities
-   * @returns {Promise<boolean>} Registration success
-   */
-  async registerParticipant(tenantId, participantId, capabilities) {
-    return await federatedLearning.registerParticipant(
-      tenantId,
-      participantId,
-      capabilities,
-    );
-  }
-
-  /**
-   * Aggregate model updates from participants
-   * @param {string} tenantId - Tenant identifier
-   * @param {string} roundId - Round identifier
-   * @param {Array} updates - Model updates from participants
-   * @returns {Promise<object>} Aggregated model
-   */
-  async aggregateUpdates(tenantId, roundId, updates) {
-    return await federatedLearning.aggregateUpdates(tenantId, roundId, updates);
-  }
-
-  /**
-   * Get federated learning progress
-   * @param {string} tenantId - Tenant identifier
-   * @param {string} roundId - Round identifier
-   * @returns {Promise<object>} Progress information
-   */
-  async getRoundProgress(tenantId, roundId) {
-    return await federatedLearning.getRoundProgress(tenantId, roundId);
-  }
-}
-
-module.exports = { FederatedLearningCoordinator };

package/src/ai/index.js

@@ -1,22 +0,0 @@
-/**
- * AI/ML Module Index - Decomposed Architecture
- * Provides unified interface to 4 separate AI modules
- *
- * Decomposed modules:
- * - orchestrator.js - ModelTrainingOrchestrator
- * - retrieval-engine.js - AdaptiveRetrievalEngine
- * - multimodal.js - MultiModalProcessor
- * - federation.js - FederatedLearningCoordinator
- */
-
-const { ModelTrainingOrchestrator } = require("./orchestrator");
-const { AdaptiveRetrievalEngine } = require("./retrieval-engine");
-const { MultiModalProcessor } = require("./multimodal");
-const { FederatedLearningCoordinator } = require("./federation");
-
-module.exports = {
-  ModelTrainingOrchestrator,
-  AdaptiveRetrievalEngine,
-  MultiModalProcessor,
-  FederatedLearningCoordinator,
-};