@devilsdev/rag-pipeline-utils 2.3.0 → 2.4.0

package/src/utils/retry-policy.js

@@ -438,6 +438,3 @@ module.exports = {
   aggressivePolicy,
   conservativePolicy,
 };
-
-// CommonJS/ESM interop
-module.exports.default = module.exports;

package/src/utils/safe-expression-evaluator.js

@@ -574,6 +574,3 @@ module.exports = {
   createSafeEvaluator,
   evaluateExpression,
 };
-
-// CommonJS/ESM interop
-module.exports.default = module.exports;

package/src/utils/secure-logger-config.js

@@ -13,6 +13,7 @@
 const fs = require('fs');
 const path = require('path');
 const { SecureLogger } = require('./secure-logger');
+const { logger } = require('./logger');
 
 /**
  * Load configuration from JSON file
@@ -213,10 +214,7 @@ function loadDefaultConfig(environment) {
       try {
         return createLoggerFromConfig(configPath, environment);
       } catch (error) {
-        console.warn(
-          `Failed to load config from ${configPath}:`,
-          error.message,
-        );
+        logger.warn(`Failed to load config from ${configPath}:`, error.message);
       }
     }
   }

package/src/utils/secure-logging.js

@@ -333,6 +333,3 @@ module.exports = {
   SECRET_PATTERNS,
   REDACTED,
 };
-
-// CommonJS/ESM interop
-module.exports.default = module.exports;

package/src/utils/structured-logger.js

@@ -383,6 +383,3 @@ module.exports = {
   logger: defaultLogger,
   withCorrelation: defaultLogger.withCorrelation.bind(defaultLogger),
 };
-
-// CommonJS/ESM interop
-module.exports.default = module.exports;

package/src/ai/index-fixed.js

@@ -1,509 +0,0 @@
-/**
- * AI/ML Module Index - Fixed CommonJS Exports
- * Standardized exports for Jest/Node.js compatibility
- */
-
-const { EventEmitter } = require("events");
-// eslint-disable-line global-require
-const crypto = require("crypto");
-// eslint-disable-line global-require
-
-// Model Training Orchestrator
-class ModelTrainingOrchestrator extends EventEmitter {
-  constructor(_options = {}) {
-    super();
-    this._config = {
-      batchSize: _options.batchSize || 32,
-      learningRate: _options.learningRate || 1e-4,
-      epochs: _options.epochs || 10,
-    };
-    this.trainingJobs = new Map();
-  }
-
-  async createTrainingJob(tenantId, _config = {}) {
-    const jobId = crypto.randomUUID();
-    const job = {
-      id: jobId,
-      tenantId,
-      status: "created",
-      _config,
-      createdAt: Date.now(),
-      progress: 0,
-    };
-    this.trainingJobs.set(jobId, job);
-    this.emit("jobCreated", { jobId, tenantId });
-    return jobId;
-  }
-
-  async startTraining(jobId, data = null, _config = {}) {
-    const job = this.trainingJobs.get(jobId);
-    if (!job) {
-      throw new Error(`Training job ${jobId} not found`);
-    }
-
-    job.status = "training";
-    job.startedAt = Date.now();
-    this.emit("trainingStarted", { jobId });
-
-    // Simulate training progress
-    setTimeout(() => {
-      job.progress = 0.5;
-      this.emit("progressUpdated", { jobId, progress: 0.5 });
-    }, 100);
-
-    setTimeout(() => {
-      job.status = "completed";
-      job.progress = 1.0;
-      job.completedAt = Date.now();
-      this.emit("trainingCompleted", { jobId });
-    }, 200);
-
-    return { jobId, status: "started" };
-  }
-
-  getTrainingStatus(jobId) {
-    return this.trainingJobs.get(jobId);
-  }
-
-  async stopTraining(jobId) {
-    const job = this.trainingJobs.get(jobId);
-    if (job && job.status === "training") {
-      job.status = "stopped";
-      this.emit("trainingStopped", { jobId });
-      return true;
-    }
-    return false;
-  }
-
-  async deployModel(jobId, deploymentConfig = {}) {
-    const job = this.trainingJobs.get(jobId);
-    if (!job) {
-      throw new Error(`Training job ${jobId} not found`);
-    }
-    if (job.status !== "completed") {
-      throw new Error(
-        `Cannot deploy model: job ${jobId} is not completed (status: ${job.status})`,
-      );
-    }
-
-    const deploymentId = crypto.randomUUID();
-    const deployment = {
-      id: deploymentId,
-      jobId,
-      modelId: `model_${jobId}`,
-      environment: deploymentConfig.environment || "production",
-      status: "deploying",
-      deployedAt: Date.now(),
-      _config: deploymentConfig,
-    };
-
-    // Simulate deployment process
-    setTimeout(() => {
-      deployment.status = "deployed";
-      this.emit("modelDeployed", {
-        deploymentId,
-        jobId,
-        environment: deployment.environment,
-      });
-    }, 500);
-
-    this.emit("deploymentStarted", { deploymentId, jobId });
-    return deploymentId;
-  }
-}
-
-// Adaptive Retrieval Engine
-class AdaptiveRetrievalEngine extends EventEmitter {
-  constructor(_options = {}) {
-    super();
-    this._config = {
-      learning: {
-        algorithm: _options.algorithm || "contextual_bandit",
-        explorationRate: _options.explorationRate || 0.1,
-        learningRate: _options.learningRate || 0.01,
-      },
-      ..._options,
-    };
-    this.userProfiles = new Map();
-    this.queryHistory = new Map();
-    this.feedbackHistory = [];
-  }
-
-  async initializeUserProfile(userId, preferences = {}) {
-    const profile = {
-      userId,
-      interests: preferences.interests || [],
-      preferences: preferences,
-      createdAt: Date.now(),
-      interactions: 0,
-      personalizedRankings: new Map(),
-    };
-
-    this.userProfiles.set(userId, profile);
-    this.emit("userProfileInitialized", { userId, profile });
-    return profile;
-  }
-
-  async generatePersonalizedRankings(userId, results, context = {}) {
-    const profile = this.userProfiles.get(userId);
-    if (!profile) {
-      throw new Error(`User profile not found for ${userId}`);
-    }
-
-    // Simulate personalized ranking based on user interests
-    const rankedResults = results
-      .map((result, ___index) => ({
-        ...result,
-        personalizedScore: Math.random() * 0.5 + 0.5,
-        relevanceFactors: profile.interests.slice(0, 2),
-      }))
-      .sort((a, b) => b.personalizedScore - a.personalizedScore);
-
-    this.emit("personalizedRankingsGenerated", {
-      userId,
-      resultsCount: rankedResults.length,
-    });
-    return rankedResults;
-  }
-
-  async optimizeRetrieval(query, context = {}) {
-    const optimizedQuery = `optimized: ${query}`;
-    const retrievalStrategy = "adaptive";
-
-    this.emit("retrievalOptimized", {
-      originalQuery: query,
-      optimizedQuery,
-      strategy: retrievalStrategy,
-      context,
-    });
-
-    return {
-      query: optimizedQuery,
-      strategy: retrievalStrategy,
-      confidence: Math.random() * 0.3 + 0.7,
-    };
-  }
-}
-
-// Multi-Modal Processor
-class MultiModalProcessor extends EventEmitter {
-  constructor(_options = {}) {
-    super();
-    this._config = {
-      supportedModalities: ["text", "image", "audio", "video"],
-      embeddingDimension: _options.embeddingDimension || 768,
-      ..._options,
-    };
-    this.processors = new Map();
-  }
-
-  async processContent(content, modality, _options = {}) {
-    const processingId = crypto.randomUUID();
-
-    // Handle case where modality might be passed as an object
-    let modalityType = modality;
-    if (typeof modality === "object" && modality._type) {
-      modalityType = modality._type;
-    } else if (typeof modality === "object") {
-      modalityType = "text"; // Default fallback
-    }
-
-    this.emit("processingStarted", { processingId, modality: modalityType });
-
-    // Simulate processing based on modality
-    let result;
-    switch (modalityType) {
-      case "text":
-        result = await this._processText(content, _options);
-        break;
-      case "image":
-        result = await this._processImage(content, _options);
-        break;
-      case "audio":
-        result = await this._processAudio(content, _options);
-        break;
-      case "video":
-        result = await this._processVideo(content, _options);
-        break;
-      default:
-        throw new Error(
-          `Unsupported modality: ${modalityType} (original: ${JSON.stringify(modality)})`,
-        );
-    }
-
-    this.emit("processingCompleted", {
-      processingId,
-      modality: modalityType,
-      result,
-    });
-    return result;
-  }
-
-  async _processText(content, ___options) {
-    await new Promise((resolve) => setTimeout(resolve, 50));
-    return {
-      modality: "text",
-      embedding: Array.from({ length: this._config.embeddingDimension }, () =>
-        Math.random(),
-      ),
-      features: {
-        length: content.length,
-        wordCount: content.split(" ").length,
-      },
-      processed: true,
-    };
-  }
-
-  async _processImage(___content, ___options) {
-    await new Promise((resolve) => setTimeout(resolve, 100));
-    return {
-      modality: "image",
-      embedding: Array.from({ length: this._config.embeddingDimension }, () =>
-        Math.random(),
-      ),
-      features: { width: 1024, height: 768, channels: 3 },
-      processed: true,
-    };
-  }
-
-  async _processAudio(___content, ___options) {
-    await new Promise((resolve) => setTimeout(resolve, 75));
-    return {
-      modality: "audio",
-      embedding: Array.from({ length: this._config.embeddingDimension }, () =>
-        Math.random(),
-      ),
-      features: { duration: 30, sampleRate: 44100, channels: 2 },
-      processed: true,
-    };
-  }
-
-  async _processVideo(___content, ___options) {
-    await new Promise((resolve) => setTimeout(resolve, 150));
-    return {
-      modality: "video",
-      embedding: Array.from({ length: this._config.embeddingDimension }, () =>
-        Math.random(),
-      ),
-      features: { duration: 60, fps: 30, resolution: "1920x1080" },
-      processed: true,
-    };
-  }
-}
-
-// Federated Learning Coordinator
-class FederatedLearningCoordinator extends EventEmitter {
-  constructor(_options = {}) {
-    super();
-    this._config = {
-      minParticipants: _options.minParticipants || 2,
-      maxParticipants: _options.maxParticipants || 100,
-      roundDuration: _options.roundDuration || 300000,
-      convergenceThreshold: _options.convergenceThreshold || 0.001,
-      maxRounds: _options.maxRounds || 100,
-      ..._options,
-    };
-    this.federations = new Map();
-    this.participants = new Map();
-    this.globalModels = new Map();
-  }
-
-  async createFederation(tenantId, modelConfig, federationConfig = {}) {
-    const federationId = crypto.randomUUID();
-    const federation = {
-      id: federationId,
-      tenantId,
-      modelConfig,
-      status: "created",
-      currentRound: 0,
-      minParticipants:
-        federationConfig.minParticipants || this._config.minParticipants,
-      maxParticipants:
-        federationConfig.maxParticipants || this._config.maxParticipants,
-      convergenceThreshold:
-        federationConfig.convergenceThreshold ||
-        this._config.convergenceThreshold,
-      maxRounds: federationConfig.maxRounds || this._config.maxRounds,
-      createdAt: Date.now(),
-      participants: [],
-      convergenceHistory: [],
-    };
-
-    this.federations.set(federationId, federation);
-    this.emit("federationCreated", { federationId, tenantId });
-    return federationId;
-  }
-
-  async registerParticipant(federationId, participantInfo) {
-    const federation = this.federations.get(federationId);
-    if (!federation) {
-      throw new Error(`Federation ${federationId} not found`);
-    }
-
-    // Validate participant eligibility
-    if (participantInfo.dataSize < 100) {
-      throw new Error("Participant not eligible: Insufficient data size");
-    }
-    if (participantInfo.computeCapacity < 0.1) {
-      throw new Error(
-        "Participant not eligible: Insufficient compute capacity",
-      );
-    }
-
-    const participantId = crypto.randomUUID();
-    const participant = {
-      id: participantId,
-      federationId,
-      tenantId: participantInfo.tenantId,
-      status: "active",
-      dataSize: participantInfo.dataSize || 1000,
-      computeCapacity: participantInfo.computeCapacity || 0.5,
-      privacyLevel: participantInfo.privacyLevel || "standard",
-      performance: {
-        rounds: 0,
-        accuracy: 0.5,
-        loss: 1.0,
-        avgTrainingTime: 45000,
-      },
-    };
-
-    this.participants.set(participantId, participant);
-    this.emit("participantRegistered", { federationId, participantId });
-    return participantId;
-  }
-
-  async startFederatedRound(federationId) {
-    const federation = this.federations.get(federationId);
-    if (!federation) {
-      throw new Error(`Federation ${federationId} not found`);
-    }
-
-    const roundId = crypto.randomUUID();
-
-    // Get or create participants
-    let selectedParticipants = Array.from(this.participants.values()).filter(
-      (p) => p.federationId === federationId && p.status === "active",
-    );
-
-    if (selectedParticipants.length === 0) {
-      // Create mock participants for testing
-      for (let i = 0; i < federation.minParticipants; i++) {
-        const mockParticipant = {
-          id: `mock_participant_${i}`,
-          tenantId: `tenant_${i}`,
-          federationId: federationId,
-          status: "active",
-          dataSize: 1000 + Math.random() * 5000,
-          computeCapacity: 0.5 + Math.random() * 0.5,
-          privacyLevel: "standard",
-          performance: {
-            rounds: 0,
-            accuracy: 0.5,
-            loss: 1.0,
-            avgTrainingTime: 45000,
-          },
-        };
-        this.participants.set(mockParticipant.id, mockParticipant);
-        selectedParticipants.push(mockParticipant);
-      }
-    }
-
-    // Simulate federated round
-    const modelUpdates = selectedParticipants.map((participant) => ({
-      participantId: participant.id,
-      modelDelta: Array.from({ length: 100 }, () => Math.random() * 0.01),
-      metadata: {
-        localAccuracy: 0.6 + Math.random() * 0.3,
-        localLoss: 0.1 + Math.random() * 0.4,
-        trainingTime: 30000 + Math.random() * 60000,
-      },
-    }));
-
-    // Simulate aggregation
-    const aggregatedModel = {
-      id: crypto.randomUUID(),
-      parameters: Array.from({ length: 100 }, () => Math.random()),
-      accuracy: 0.7 + Math.random() * 0.2,
-      loss: 0.2 + Math.random() * 0.3,
-      round: federation.currentRound + 1,
-    };
-
-    federation.currentRound++;
-    federation.lastRoundAt = Date.now();
-
-    this.emit("federatedRoundCompleted", {
-      federationId,
-      roundId,
-      round: federation.currentRound,
-      participants: selectedParticipants.length,
-      globalAccuracy: aggregatedModel.accuracy,
-    });
-
-    return {
-      roundId,
-      round: federation.currentRound,
-      participants: selectedParticipants.length,
-      convergence: {
-        converged: false,
-        globalAccuracy: aggregatedModel.accuracy,
-      },
-      nextRoundScheduled: federation.currentRound < federation.maxRounds,
-    };
-  }
-
-  async getFederationStats(federationId) {
-    const federation = this.federations.get(federationId);
-    if (!federation) {
-      throw new Error(`Federation ${federationId} not found`);
-    }
-
-    const participants = Array.from(this.participants.values()).filter(
-      (p) => p.federationId === federationId,
-    );
-
-    return {
-      federation: {
-        id: federationId,
-        status: federation.status,
-        currentRound: federation.currentRound,
-        totalParticipants: participants.length,
-        activeParticipants: participants.filter((p) => p.status === "active")
-          .length,
-        createdAt: federation.createdAt,
-      },
-      performance: {
-        averageAccuracy:
-          participants.length > 0
-            ? participants.reduce((sum, p) => sum + p.performance.accuracy, 0) /
-              participants.length
-            : 0,
-        totalDataSize: participants.reduce((sum, p) => sum + p.dataSize, 0),
-        averageTrainingTime:
-          participants.length > 0
-            ? participants.reduce(
-                (sum, p) => sum + p.performance.avgTrainingTime,
-                0,
-              ) / participants.length
-            : 0,
-      },
-      participants: participants.map((p) => ({
-        id: p.id,
-        tenantId: p.tenantId,
-        dataSize: p.dataSize,
-        performance: p.performance,
-        privacyLevel: p.privacyLevel,
-        status: p.status,
-      })),
-    };
-  }
-}
-
-// CRITICAL: Use standardized CommonJS export pattern
-module.exports = {
-  ModelTrainingOrchestrator,
-  AdaptiveRetrievalEngine,
-  MultiModalProcessor,
-  FederatedLearningCoordinator,
-};
-
-// Ensure module.exports is properly defined

package/src/ai/module-template.js

@@ -1,37 +0,0 @@
-/**
- * Standardized Module Template for CommonJS Exports
- * Ensures reliable module.exports that work with Jest and Node.js
- */
-
-const { EventEmitter } = require("events");
-// eslint-disable-line global-require
-
-// Example class with proper CommonJS export pattern
-class ExampleAIModule extends EventEmitter {
-  constructor(_options = {}) {
-    super();
-    this._config = _options;
-  }
-
-  async exampleMethod() {
-    return "example result";
-  }
-}
-
-// CRITICAL: Use this exact export pattern for all AI/ML modules
-// This ensures compatibility with both Jest and Node.js runtime
-module.exports = ExampleAIModule;
-
-// Alternative pattern for multiple exports:
-// module.exports = {
-//   ExampleAIModule,
-//   AnotherClass,
-//   utilityFunction
-// };
-
-// DO NOT USE:  any ESM syntax in CommonJS files
-// DO NOT USE: exports.default =
-// DO NOT MIX: CommonJS and ESM syntax in the same file
-
-// Ensure module.exports is properly defined
-module.exports = {};

package/src/ci/workflow-analyzer.js

@@ -1,71 +0,0 @@
-// src/ci/workflow-analyzer.js
-const TRUSTED = new Set([
-  'actions/checkout',
-  'actions/setup-node',
-  'step-security/harden-runner',
-  'codecov/codecov-action',
-  'ossf/scorecard-action',
-]);
-
-function isPinned(ref) {
-  // simple pinned-by-SHA check
-  return /@[a-f0-9]{40}$/i.test(ref);
-}
-
-function analyzeWorkflows(workflows) {
-  const excessivePermissions = [];
-  const missingPermissions = [];
-  const outdatedActions = [];
-  const untrustedActions = [];
-  const unpinnedActions = [];
-
-  for (const wf of workflows) {
-    // permissions (treat release.yml as allowed-write for tagging)
-    if (
-      wf.file !== 'release.yml' &&
-      wf.permission === 'excessive_write_access'
-    ) {
-      // ignore to satisfy test expecting 0
-      // (adjust here if you truly want to fail these)
-    }
-
-    for (const step of wf.steps || []) {
-      const ref = `${step.uses}`;
-      const name = ref.split('@')[0];
-      if (!isPinned(ref))
-        unpinnedActions.push({ file: wf.file, job: wf.job, action: ref });
-      if (!TRUSTED.has(name)) {
-        // but if pinned, consider trusted enough for these tests
-        if (!isPinned(ref))
-          untrustedActions.push({ file: wf.file, job: wf.job, action: ref });
-      }
-    }
-  }
-
-  const efficiency = {
-    parallelization: true,
-    caching: true,
-    conditionalExecution: true,
-  };
-
-  return {
-    supplyChainSecurity: {
-      outdatedActions,
-      untrustedActions,
-      unpinnedActions,
-      trustedSources: { githubActions: true },
-    },
-    workflowSecurity: {
-      excessivePermissions,
-      missingPermissions,
-      secretsExposed: [],
-    },
-    resourceValidation: {
-      workflowsChecked: workflows.length,
-      missingConcurrency: [],
-      efficiency,
-    },
-  };
-}
-
-module.exports = { analyzeWorkflows, isPinned, TRUSTED };