@devilsdev/rag-pipeline-utils 2.4.5 → 2.5.0

package/src/utils/run-abort.js

@@ -0,0 +1,102 @@
+'use strict';
+
+/** Message used for a run that exceeded its timeout. */
+const TIMEOUT_MESSAGE = 'Execution timeout';
+/** Message used for a run cancelled through the caller's AbortSignal. */
+const ABORTED_MESSAGE = 'Execution aborted';
+
+function timeoutError() {
+  const err = new Error(TIMEOUT_MESSAGE);
+  err.name = 'TimeoutError';
+  err.code = 'ETIMEDOUT';
+  return err;
+}
+
+function callerAbortError(signal) {
+  const err = new Error(ABORTED_MESSAGE);
+  err.name = 'AbortError';
+  err.code = 'ABORT_ERR';
+  if (signal && signal.reason !== undefined) err.cause = signal.reason;
+  return err;
+}
+
+/**
+ * Per-run cancellation: one AbortController that is aborted when `timeout`
+ * elapses or when the caller's `signal` aborts. Its `signal` is handed to
+ * every component call; `race()` makes the run settle as soon as it is
+ * aborted even if a component ignores the signal; `throwIfAborted()` stops
+ * work between stages.
+ *
+ * @param {object} [options]
+ * @param {number} [options.timeout] - Milliseconds before the run is aborted
+ * @param {AbortSignal} [options.signal] - Caller's signal to follow
+ * @returns {{
+ *   signal: AbortSignal,
+ *   race: (promise: Promise<any>) => Promise<any>,
+ *   throwIfAborted: () => void,
+ *   clearTimer: () => void,
+ *   detach: () => void,
+ *   cleanup: () => void
+ * }}
+ */
+function createRunAbort({ timeout, signal } = {}) {
+  const controller = new AbortController();
+  let timer = null;
+  let onCallerAbort = null;
+  let rejectAborted;
+  const aborted = new Promise((_, reject) => {
+    rejectAborted = reject;
+  });
+  // The rejection is observed through race(); never report it as unhandled.
+  aborted.catch(() => {});
+
+  const abort = (error) => {
+    if (controller.signal.aborted) return;
+    controller.abort(error);
+    rejectAborted(error);
+  };
+
+  if (typeof timeout === 'number' && timeout > 0) {
+    timer = setTimeout(() => abort(timeoutError()), timeout);
+  }
+  if (signal && typeof signal.addEventListener === 'function') {
+    if (signal.aborted) {
+      abort(callerAbortError(signal));
+    } else {
+      onCallerAbort = () => abort(callerAbortError(signal));
+      signal.addEventListener('abort', onCallerAbort, { once: true });
+    }
+  }
+
+  const clearTimer = () => {
+    if (timer) clearTimeout(timer);
+    timer = null;
+  };
+  const detach = () => {
+    if (onCallerAbort) signal.removeEventListener('abort', onCallerAbort);
+    onCallerAbort = null;
+  };
+
+  return {
+    signal: controller.signal,
+    race(promise) {
+      if (controller.signal.aborted) {
+        // Swallow the late outcome of work that is no longer awaited.
+        Promise.resolve(promise).catch(() => {});
+        return Promise.reject(controller.signal.reason);
+      }
+      return Promise.race([promise, aborted]);
+    },
+    throwIfAborted() {
+      if (controller.signal.aborted) throw controller.signal.reason;
+    },
+    clearTimer,
+    detach,
+    cleanup() {
+      clearTimer();
+      detach();
+    },
+  };
+}
+
+module.exports = { createRunAbort, TIMEOUT_MESSAGE, ABORTED_MESSAGE };

package/src/ai/federated-learning.js

@@ -1,211 +0,0 @@
-/**
- * Federated Learning Coordinator
- * Event-driven federated learning with deterministic aggregation
- */
-
-"use strict";
-
-const { EventEmitter } = require("events");
-
-// In-memory storage for federations
-const federations = new Map();
-let federationCounter = 0;
-
-// Create EventEmitter instance
-const eventEmitter = new EventEmitter();
-
-/**
- * Create a new federation
- * @param {string} tenantId - Tenant identifier
- * @param {object} modelConfig - Model configuration
- * @param {object} options - Federation options
- * @returns {string} Federation ID
- */
-function createFederation(tenantId, modelConfig, options = {}) {
-  const federationId = `fed-${++federationCounter}`;
-  const federation = {
-    tenantId,
-    modelConfig,
-    options,
-    rounds: 0,
-    participants: [],
-  };
-
-  federations.set(federationId, federation);
-  return federationId;
-}
-
-/**
- * Register multiple participants in a federation
- * @param {string} federationId - Federation ID
- * @param {array} participantsArray - Participants to register
- * @returns {Promise<string|array>} Single string ID if one participant, array of IDs if multiple
- */
-async function registerParticipants(federationId, participantsArray) {
-  const federation = federations.get(federationId);
-  if (!federation) {
-    throw new Error(`Federation ${federationId} not found`);
-  }
-
-  const participantIds = [];
-  for (const participant of participantsArray) {
-    // Validate participant - throw error if invalid
-    if (!participant || typeof participant !== "object") {
-      throw new Error("Participant not eligible");
-    }
-
-    // Check data size requirement
-    if (
-      typeof participant.dataSize === "number" &&
-      participant.dataSize < 100
-    ) {
-      throw new Error("Participant not eligible");
-    }
-
-    const participantId =
-      participant.id || `p-${federation.participants.length + 1}`;
-    federation.participants.push({
-      id: participantId,
-      status: "active",
-      weight: participant.weight || 1,
-      ...participant,
-    });
-    participantIds.push(participantId);
-  }
-
-  // Return single string if only one participant, array if multiple (for test compatibility)
-  return participantsArray.length === 1 ? participantIds[0] : participantIds;
-}
-
-/**
- * Conduct a federated learning round
- * @param {string} federationId - Federation ID
- * @returns {object} Round result
- */
-function conductRound(federationId) {
-  const federation = federations.get(federationId);
-  if (!federation) {
-    throw new Error(`Federation ${federationId} not found`);
-  }
-
-  const round = ++federation.rounds;
-  const roundId = `r-${Date.now()}`;
-
-  // Emit round started event
-  eventEmitter.emit("federated_round_started", {
-    federationId,
-    round,
-  });
-
-  // Compute deterministic convergence metrics
-  const participants = federation.participants;
-  const globalAccuracy = 0.9 + participants.length * 0.01; // Slight improvement with more participants
-  const loss = Math.max(0.1 - participants.length * 0.005, 0.01); // Lower loss with more participants
-
-  // Emit round completed event
-  eventEmitter.emit("federated_round_completed", {
-    federationId,
-    metrics: { globalAccuracy, loss },
-  });
-
-  return {
-    roundId,
-    round,
-    participants: participants.length,
-    convergence: {
-      globalAccuracy: Math.round(globalAccuracy * 1000) / 1000,
-      loss: Math.round(loss * 1000) / 1000,
-    },
-  };
-}
-
-/**
- * Get federation statistics
- * @param {string} federationId - Federation ID
- * @returns {object} Federation stats
- */
-function getStats(federationId) {
-  const federation = federations.get(federationId);
-  if (!federation) {
-    throw new Error(`Federation ${federationId} not found`);
-  }
-
-  return {
-    federation: {
-      id: federationId,
-      rounds: federation.rounds,
-    },
-    participants: federation.participants.map((p) => ({
-      id: p.id,
-      status: p.status || "active",
-      weight: p.weight || 1,
-    })),
-    performance: {
-      averageAccuracy: 0.92,
-      convergenceRate: 0.85,
-      communicationRounds: federation.rounds,
-    },
-    privacy: {
-      differentialPrivacy: federation.options.differentialPrivacy || false,
-      encryptionEnabled: federation.options.encryptionEnabled || true,
-      dataLocality: "preserved",
-    },
-  };
-}
-
-/**
- * Validate participant eligibility
- * @param {object} participant - Participant to validate
- * @returns {boolean} Whether participant is valid
- */
-function validateParticipant(participant) {
-  if (!participant || typeof participant !== "object") {
-    return false;
-  }
-
-  // Check if participant has non-empty string id
-  if (
-    !participant.id ||
-    typeof participant.id !== "string" ||
-    participant.id.trim() === ""
-  ) {
-    return false;
-  }
-
-  // Check if participant is explicitly marked as ineligible
-  if (participant.eligible === false) {
-    return false;
-  }
-
-  return true;
-}
-
-/**
- * Clear all stored data (for testing)
- */
-function clear() {
-  federations.clear();
-  federationCounter = 0;
-  eventEmitter.removeAllListeners();
-}
-
-// Singleton object with required methods and EventEmitter functionality
-const federatedLearningCoordinator = {
-  // Core federated learning methods
-  createFederation,
-  registerParticipants,
-  conductRound,
-  getStats,
-  validateParticipant,
-
-  // EventEmitter methods
-  on: eventEmitter.on.bind(eventEmitter),
-  off: eventEmitter.removeListener.bind(eventEmitter),
-  emit: eventEmitter.emit.bind(eventEmitter),
-
-  // Additional utility methods
-  clear,
-};
-
-// Export singleton as default
-module.exports = federatedLearningCoordinator;

package/src/ai/federation.js

@@ -1,71 +0,0 @@
-/**
- * Federated Learning Coordinator
- * Manages distributed learning across multiple participants
- * Extracted from ai/index.js per CLAUDE.md decomposition requirements
- */
-
-"use strict";
-
-// Import the existing federated learning implementation
-const federatedLearning = require("./federated-learning");
-
-/**
- * FederatedLearningCoordinator class
- * Coordinates federated learning processes across multiple nodes
- */
-class FederatedLearningCoordinator {
-  constructor() {
-    this.participants = new Map();
-    this.rounds = new Map();
-    this.modelVersions = new Map();
-    this.aggregationStrategies = new Set(["fedavg", "fedprox", "scaffold"]);
-  }
-
-  /**
-   * Start federated learning round
-   * @param {string} tenantId - Tenant identifier
-   * @param {object} roundConfig - Round configuration
-   * @returns {Promise<string>} Round ID
-   */
-  async startRound(tenantId, roundConfig) {
-    return await federatedLearning.startRound(tenantId, roundConfig);
-  }
-
-  /**
-   * Register participant in federated learning
-   * @param {string} tenantId - Tenant identifier
-   * @param {string} participantId - Participant identifier
-   * @param {object} capabilities - Participant capabilities
-   * @returns {Promise<boolean>} Registration success
-   */
-  async registerParticipant(tenantId, participantId, capabilities) {
-    return await federatedLearning.registerParticipant(
-      tenantId,
-      participantId,
-      capabilities,
-    );
-  }
-
-  /**
-   * Aggregate model updates from participants
-   * @param {string} tenantId - Tenant identifier
-   * @param {string} roundId - Round identifier
-   * @param {Array} updates - Model updates from participants
-   * @returns {Promise<object>} Aggregated model
-   */
-  async aggregateUpdates(tenantId, roundId, updates) {
-    return await federatedLearning.aggregateUpdates(tenantId, roundId, updates);
-  }
-
-  /**
-   * Get federated learning progress
-   * @param {string} tenantId - Tenant identifier
-   * @param {string} roundId - Round identifier
-   * @returns {Promise<object>} Progress information
-   */
-  async getRoundProgress(tenantId, roundId) {
-    return await federatedLearning.getRoundProgress(tenantId, roundId);
-  }
-}
-
-module.exports = { FederatedLearningCoordinator };

package/src/ai/index.js

@@ -1,22 +0,0 @@
-/**
- * AI/ML Module Index - Decomposed Architecture
- * Provides unified interface to 4 separate AI modules
- *
- * Decomposed modules:
- * - orchestrator.js - ModelTrainingOrchestrator
- * - retrieval-engine.js - AdaptiveRetrievalEngine
- * - multimodal.js - MultiModalProcessor
- * - federation.js - FederatedLearningCoordinator
- */
-
-const { ModelTrainingOrchestrator } = require("./orchestrator");
-const { AdaptiveRetrievalEngine } = require("./retrieval-engine");
-const { MultiModalProcessor } = require("./multimodal");
-const { FederatedLearningCoordinator } = require("./federation");
-
-module.exports = {
-  ModelTrainingOrchestrator,
-  AdaptiveRetrievalEngine,
-  MultiModalProcessor,
-  FederatedLearningCoordinator,
-};

package/src/ai/model-trainer.js

@@ -1,217 +0,0 @@
-/**
- * Model Trainer - In-memory deterministic training orchestration
- *
- * Google-style implementation with dependency injection for clocks/RNG.
- * Provides training job management, hyperparameter optimization, and model deployment.
- *
- * @module ModelTrainer
- * @exports {ModelTrainer, modelTrainer}
- */
-
-const EventEmitter = require("events");
-
-class ModelTrainer extends EventEmitter {
-  constructor(options = {}) {
-    super();
-    this.now = options.now || (() => Date.now());
-    this.sleep =
-      options.sleep ||
-      ((ms) => new Promise((resolve) => setTimeout(resolve, Math.min(ms, 5))));
-    this.jobs = new Map();
-    this.optimizations = new Map();
-    this.deployments = new Map();
-    this.jobCounter = 0;
-    this.optCounter = 0;
-    this.deployCounter = 0;
-  }
-
-  /**
-   * Create a training job
-   * @param {string} tenantId - Tenant identifier
-   * @param {object} config - Training configuration
-   * @returns {string} Job ID
-   */
-  createTrainingJob(tenantId, config) {
-    if (!tenantId || !config) {
-      const error = new Error("Tenant ID and config are required");
-      error.code = "INVALID_ARGUMENT";
-      throw error;
-    }
-
-    const jobId = `job-${++this.jobCounter}`;
-    const job = {
-      id: jobId,
-      tenantId,
-      config,
-      status: "created",
-      createdAt: this.now(),
-      progress: 0,
-    };
-
-    this.jobs.set(jobId, job);
-    return jobId;
-  }
-
-  /**
-   * Start training for a job
-   * @param {string} jobId - Job identifier
-   * @returns {Promise<void>}
-   */
-  async startTraining(jobId) {
-    const job = this.jobs.get(jobId);
-    if (!job) {
-      const error = new Error(`Training job not found: ${jobId}`);
-      error.code = "NOT_FOUND";
-      throw error;
-    }
-
-    job.status = "running";
-    job.startedAt = this.now();
-    this.emit("training_started", { jobId, tenantId: job.tenantId });
-
-    // Simulate training with tiny delay
-    await this.sleep(2);
-
-    job.status = "completed";
-    job.completedAt = this.now();
-    job.progress = 100;
-    this.emit("training_completed", {
-      jobId,
-      tenantId: job.tenantId,
-      status: "completed",
-    });
-  }
-
-  /**
-   * Optimize hyperparameters
-   * @param {string} tenantId - Tenant identifier
-   * @param {object} config - Optimization configuration
-   * @returns {string} Optimization ID
-   */
-  optimizeHyperparameters(tenantId, config) {
-    if (!tenantId || !config) {
-      const error = new Error("Tenant ID and config are required");
-      error.code = "INVALID_ARGUMENT";
-      throw error;
-    }
-
-    const optId = `opt-${++this.optCounter}`;
-    const optimization = {
-      id: optId,
-      tenantId,
-      config,
-      status: "running",
-      createdAt: this.now(),
-      trials: [
-        { params: { lr: 0.001, batch_size: 32 }, score: 0.85 },
-        { params: { lr: 0.01, batch_size: 64 }, score: 0.92 },
-        { params: { lr: 0.005, batch_size: 32 }, score: 0.89 },
-      ],
-    };
-
-    this.optimizations.set(optId, optimization);
-
-    // Emit completion immediately for deterministic tests
-    setImmediate(() => {
-      optimization.status = "completed";
-      optimization.completedAt = this.now();
-      this.emit("hpo_completed", { optId, tenantId, bestScore: 0.92 });
-    });
-
-    return optId;
-  }
-
-  /**
-   * Get optimization results
-   * @param {string} optId - Optimization identifier
-   * @returns {Array} Array of trials
-   */
-  getOptimizationResults(optId) {
-    const optimization = this.optimizations.get(optId);
-    if (!optimization) {
-      const error = new Error(`Optimization not found: ${optId}`);
-      error.code = "NOT_FOUND";
-      throw error;
-    }
-
-    return optimization.trials;
-  }
-
-  /**
-   * Deploy a trained model
-   * @param {string} jobId - Training job identifier
-   * @param {object} options - Deployment options
-   * @returns {string} Deployment ID
-   */
-  deployModel(jobId, options = {}) {
-    const job = this.jobs.get(jobId);
-    if (!job) {
-      const error = new Error(`Training job not found: ${jobId}`);
-      error.code = "NOT_FOUND";
-      throw error;
-    }
-
-    if (job.status !== "completed") {
-      const error = new Error("Job must be completed before deployment");
-      error.code = "FAILED_PRECONDITION";
-      throw error;
-    }
-
-    const deploymentId = `deploy-${++this.deployCounter}`;
-    const deployment = {
-      id: deploymentId,
-      jobId,
-      tenantId: job.tenantId,
-      environment: options.environment || "staging",
-      status: "deployed",
-      deployedAt: this.now(),
-    };
-
-    this.deployments.set(deploymentId, deployment);
-    this.emit("model_deployed", {
-      deploymentId,
-      jobId,
-      environment: deployment.environment,
-    });
-
-    return deploymentId;
-  }
-
-  /**
-   * Get deployment status
-   * @param {string} deploymentId - Deployment identifier
-   * @returns {object} Deployment status
-   */
-  getDeploymentStatus(deploymentId) {
-    const deployment = this.deployments.get(deploymentId);
-    if (!deployment) {
-      const error = new Error(`Deployment not found: ${deploymentId}`);
-      error.code = "NOT_FOUND";
-      throw error;
-    }
-
-    return {
-      id: deployment.id,
-      status: deployment.status,
-      environment: deployment.environment,
-      deployedAt: deployment.deployedAt,
-    };
-  }
-
-  /**
-   * Clear all data (for testing)
-   */
-  clear() {
-    this.jobs.clear();
-    this.optimizations.clear();
-    this.deployments.clear();
-    this.jobCounter = 0;
-    this.optCounter = 0;
-    this.deployCounter = 0;
-  }
-}
-
-// Default instance
-const modelTrainer = new ModelTrainer();
-
-module.exports = { ModelTrainer, modelTrainer };