@devilsdev/rag-pipeline-utils 2.4.5 → 2.5.0

package/src/cli/commands/ai-ml/federated-commands.js

@@ -1,207 +0,0 @@
-/**
- * Federated learning CLI commands
- * Handles federation creation, joining, rounds, and statistics
- */
-
-const chalk = require("chalk");
-const ora = require("ora");
-const Table = require("cli-table3");
-const { colorizeStatus } = require("./shared");
-
-/**
- * Create a federated learning session
- * @param {FederatedLearningCoordinator} federatedLearning - The federated learning coordinator instance
- * @param {object} _options - Command options
- */
-async function createFederation(federatedLearning, _options) {
-  const spinner = ora("Creating federated learning session...").start();
-
-  try {
-    if (!_options.tenant) {
-      spinner.fail("Tenant ID is required");
-      return;
-    }
-
-    const modelConfig = {
-      _type: _options.modelType,
-      architecture: _options.architecture,
-    };
-
-    const federationOptions = {
-      minParticipants: parseInt(_options.minParticipants),
-      maxParticipants: parseInt(_options.maxParticipants),
-    };
-
-    const federationId = await federatedLearning.createFederation(
-      _options.tenant,
-      modelConfig,
-      federationOptions,
-    );
-
-    spinner.succeed("Federation created successfully");
-
-    console.log(chalk.green("\n✓ Federated learning session created"));
-    // eslint-disable-line no-console
-    console.log(chalk.blue(`Federation ID: ${federationId}`));
-    // eslint-disable-line no-console
-    console.log(chalk.blue(`Model Type: ${_options.modelType}`));
-    // eslint-disable-line no-console
-    console.log(
-      chalk.blue(
-        `Min/Max Participants: ${_options.minParticipants}/${_options.maxParticipants}`,
-      ),
-    );
-    // eslint-disable-line no-console
-  } catch (error) {
-    spinner.fail(`Federation creation failed: ${error.message}`);
-  }
-}
-
-/**
- * Join a federated learning session
- * @param {FederatedLearningCoordinator} federatedLearning - The federated learning coordinator instance
- * @param {string} federationId - Federation ID
- * @param {object} _options - Command options
- */
-async function joinFederation(federatedLearning, federationId, _options) {
-  const spinner = ora("Joining federated learning session...").start();
-
-  try {
-    if (!_options.tenant) {
-      spinner.fail("Tenant ID is required");
-      return;
-    }
-
-    const participantInfo = {
-      tenantId: _options.tenant,
-      dataSize: parseInt(_options.dataSize) || 1000,
-      computeCapacity: parseFloat(_options.computeCapacity) || 0.8,
-      networkBandwidth: 100,
-      privacyLevel: _options.privacyLevel,
-    };
-
-    const participantId = await federatedLearning.registerParticipant(
-      federationId,
-      participantInfo,
-    );
-
-    spinner.succeed("Successfully joined federation");
-
-    console.log(chalk.green("\n✓ Joined federated learning session"));
-    // eslint-disable-line no-console
-    console.log(chalk.blue(`Participant ID: ${participantId}`));
-    // eslint-disable-line no-console
-    console.log(chalk.blue(`Federation ID: ${federationId}`));
-    // eslint-disable-line no-console
-  } catch (error) {
-    spinner.fail(`Failed to join federation: ${error.message}`);
-  }
-}
-
-/**
- * Start a federated learning round
- * @param {FederatedLearningCoordinator} federatedLearning - The federated learning coordinator instance
- * @param {string} federationId - Federation ID
- */
-async function startFederatedRound(federatedLearning, federationId) {
-  const spinner = ora("Starting federated learning round...").start();
-
-  try {
-    const result = await federatedLearning.startFederatedRound(federationId);
-
-    spinner.succeed("Federated round completed");
-
-    console.log(chalk.green("\n✓ Federated learning round completed"));
-    // eslint-disable-line no-console
-    console.log(chalk.blue(`Round: ${result.round}`));
-    // eslint-disable-line no-console
-    console.log(chalk.blue(`Participants: ${result.participants}`));
-    // eslint-disable-line no-console
-    console.log(
-      chalk.blue(`Converged: ${result.convergence.converged ? "Yes" : "No"}`),
-    );
-    // eslint-disable-line no-console
-    console.log(
-      chalk.blue(
-        `Global Accuracy: ${result.convergence.globalAccuracy?.toFixed(4)}`,
-      ),
-    );
-    // eslint-disable-line no-console
-  } catch (error) {
-    spinner.fail(`Federated round failed: ${error.message}`);
-  }
-}
-
-/**
- * Show federation statistics
- * @param {FederatedLearningCoordinator} federatedLearning - The federated learning coordinator instance
- * @param {string} federationId - Federation ID
- * @param {object} _options - Command options
- */
-async function showFederationStats(federatedLearning, federationId, _options) {
-  const spinner = ora("Fetching federation statistics...").start();
-
-  try {
-    const stats = await federatedLearning.getFederationStats(federationId);
-
-    spinner.succeed("Statistics retrieved");
-
-    const table = new Table({
-      head: ["Property", "Value"],
-      colWidths: [25, 40],
-    });
-
-    table.push(
-      ["Federation ID", stats.federation.id],
-      ["Status", colorizeStatus(stats.federation.status)],
-      ["Current Round", stats.federation.currentRound],
-      ["Total Participants", stats.federation.totalParticipants],
-      ["Active Participants", stats.federation.activeParticipants],
-      ["Model Type", stats.federation.modelType],
-      [
-        "Average Accuracy",
-        stats.performance.averageAccuracy?.toFixed(4) || "N/A",
-      ],
-      ["Total Data Size", stats.performance.totalDataSize],
-      [
-        "Privacy Enabled",
-        stats.privacy.differentialPrivacyEnabled ? "Yes" : "No",
-      ],
-    );
-
-    console.log(table.toString());
-    // eslint-disable-line no-console
-
-    if (_options.detailed && stats.participants.length > 0) {
-      console.log(chalk.blue("\nParticipant Details:"));
-      // eslint-disable-line no-console
-
-      const participantTable = new Table({
-        head: ["Tenant ID", "Data Size", "Accuracy", "Rounds", "Status"],
-        colWidths: [15, 12, 12, 8, 12],
-      });
-
-      stats.participants.forEach((participant) => {
-        participantTable.push([
-          participant.tenantId,
-          participant.dataSize,
-          participant.performance.accuracy?.toFixed(4) || "N/A",
-          participant.performance.rounds,
-          participant.status,
-        ]);
-      });
-
-      console.log(participantTable.toString());
-      // eslint-disable-line no-console
-    }
-  } catch (error) {
-    spinner.fail(`Failed to get statistics: ${error.message}`);
-  }
-}
-
-module.exports = {
-  createFederation,
-  joinFederation,
-  startFederatedRound,
-  showFederationStats,
-};

package/src/cli/commands/ai-ml/index.js

@@ -1,17 +0,0 @@
-/**
- * AI/ML domain module barrel export
- */
-
-const trainingCommands = require("./training-commands");
-const adaptiveCommands = require("./adaptive-commands");
-const multimodalCommands = require("./multimodal-commands");
-const federatedCommands = require("./federated-commands");
-const shared = require("./shared");
-
-module.exports = {
-  trainingCommands,
-  adaptiveCommands,
-  multimodalCommands,
-  federatedCommands,
-  shared,
-};

package/src/cli/commands/ai-ml/multimodal-commands.js

@@ -1,187 +0,0 @@
-/**
- * Multi-modal processing CLI commands
- * Handles content processing, multi-modal search, and content description
- */
-
-const chalk = require("chalk");
-const ora = require("ora");
-const fs = require("fs").promises;
-const { detectContentType } = require("./shared");
-
-/**
- * Process multi-modal content
- * @param {MultiModalProcessor} multiModalProcessor - The multi-modal processor instance
- * @param {string} contentPath - Path to the content file
- * @param {object} _options - Command options
- */
-async function processMultiModalContent(
-  multiModalProcessor,
-  contentPath,
-  _options,
-) {
-  const spinner = ora("Processing multi-modal content...").start();
-
-  try {
-    if (!_options.tenant) {
-      spinner.fail("Tenant ID is required");
-      return;
-    }
-
-    // Read content file
-    const stats = await fs.stat(contentPath);
-    const content = {
-      _type: detectContentType(contentPath),
-      size: stats.size,
-      path: contentPath,
-    };
-
-    // Add additional content based on type
-    if (content._type.startsWith("text/")) {
-      content.text = await fs.readFile(contentPath, "utf-8");
-    }
-
-    spinner.text = "Processing content...";
-    const result = await multiModalProcessor.processContent(
-      _options.tenant,
-      content,
-    );
-
-    spinner.succeed("Content processed successfully");
-
-    console.log(chalk.green("\n✓ Multi-modal processing completed"));
-    // eslint-disable-line no-console
-    console.log(chalk.blue(`Content ID: ${result.id}`));
-    // eslint-disable-line no-console
-    console.log(
-      chalk.blue(
-        `Modalities detected: ${Object.keys(result.modalities).join(", ")}`,
-      ),
-    );
-    // eslint-disable-line no-console
-    console.log(
-      chalk.blue(`Processing time: ${result.metadata.processingTime}ms`),
-    );
-    // eslint-disable-line no-console
-
-    if (_options.output) {
-      await fs.writeFile(_options.output, JSON.stringify(result, null, 2));
-      console.log(chalk.gray(`Results saved to: ${_options.output}`));
-      // eslint-disable-line no-console
-    }
-  } catch (error) {
-    spinner.fail(`Content processing failed: ${error.message}`);
-  }
-}
-
-/**
- * Perform multi-modal search
- * @param {MultiModalProcessor} multiModalProcessor - The multi-modal processor instance
- * @param {string} tenantId - Tenant ID
- * @param {object} _options - Command options
- */
-async function multiModalSearch(multiModalProcessor, tenantId, _options) {
-  const spinner = ora("Performing multi-modal search...").start();
-
-  try {
-    const query = {};
-
-    if (_options.query) {
-      query.text = _options.query;
-      query._type = "text";
-    }
-
-    if (_options.image) {
-      query.image = await fs.readFile(_options.image);
-      query._type = "image";
-    }
-
-    if (_options.audio) {
-      query.audio = await fs.readFile(_options.audio);
-      query._type = "audio";
-    }
-
-    if (Object.keys(query).length === 0) {
-      spinner.fail("At least one query _type must be provided");
-      return;
-    }
-
-    const searchOptions = {
-      maxResults: parseInt(_options.maxResults),
-    };
-
-    const results = await multiModalProcessor.multiModalSearch(
-      tenantId,
-      query,
-      searchOptions,
-    );
-
-    spinner.succeed(`Found ${results.results.length} results`);
-
-    console.log(chalk.blue("\nMulti-modal Search Results:"));
-    // eslint-disable-line no-console
-    console.log(chalk.gray("=".repeat(50)));
-    // eslint-disable-line no-console
-
-    results.results.forEach((result, index) => {
-      console.log(
-        chalk.green(`\n${index + 1}. Content ID: ${result.contentId}`),
-      );
-      // eslint-disable-line no-console
-      console.log(
-        chalk.gray(
-          `   Multi-modal Score: ${result.multiModalScore?.toFixed(4)}`,
-        ),
-      );
-      // eslint-disable-line no-console
-      console.log(
-        chalk.gray(`   Modalities: ${result.modalities?.join(", ")}`),
-      );
-      // eslint-disable-line no-console
-      console.log(chalk.gray(`   Rank: ${result.rank}`));
-      // eslint-disable-line no-console
-    });
-  } catch (error) {
-    spinner.fail(`Multi-modal search failed: ${error.message}`);
-  }
-}
-
-/**
- * Generate content description
- * @param {MultiModalProcessor} multiModalProcessor - The multi-modal processor instance
- * @param {string} contentId - Content ID
- * @param {object} _options - Command options
- */
-async function describeContent(multiModalProcessor, contentId, _options) {
-  const spinner = ora("Generating content description...").start();
-
-  try {
-    const descriptions = await multiModalProcessor.generateContentDescription(
-      contentId,
-      {
-        detailed: !!_options.detailed,
-      },
-    );
-
-    spinner.succeed("Description generated");
-
-    console.log(chalk.blue("\nContent Descriptions:"));
-    // eslint-disable-line no-console
-    console.log(chalk.gray("=".repeat(50)));
-    // eslint-disable-line no-console
-
-    for (const [modality, description] of Object.entries(descriptions)) {
-      console.log(chalk.green(`\n${modality.toUpperCase()}:`));
-      // eslint-disable-line no-console
-      console.log(description);
-      // eslint-disable-line no-console
-    }
-  } catch (error) {
-    spinner.fail(`Description generation failed: ${error.message}`);
-  }
-}
-
-module.exports = {
-  processMultiModalContent,
-  multiModalSearch,
-  describeContent,
-};

package/src/cli/commands/ai-ml/shared.js

@@ -1,51 +0,0 @@
-/**
- * Shared utilities for AI/ML CLI commands
- */
-
-const chalk = require("chalk");
-const _path = require("path");
-
-/**
- * Colorize status text based on status value
- * @param {string} status - Status string to colorize
- * @returns {string} Chalk-colorized status string
- */
-function colorizeStatus(status) {
-  switch (status) {
-    case "completed":
-    case "ready":
-    case "active":
-      return chalk.green(status);
-    case "failed":
-    case "error":
-      return chalk.red(status);
-    case "running":
-    case "training":
-      return chalk.yellow(status);
-    default:
-      return chalk.gray(status);
-  }
-}
-
-/**
- * Detect content type from file extension
- * @param {string} filePath - Path to the file
- * @returns {string} MIME type string
- */
-function detectContentType(filePath) {
-  const ext = _path.extname(filePath).toLowerCase();
-
-  if ([".jpg", ".jpeg", ".png", ".gif", ".webp"].includes(ext)) {
-    return "image/jpeg";
-  } else if ([".mp3", ".wav", ".flac", ".ogg"].includes(ext)) {
-    return "audio/mpeg";
-  } else if ([".mp4", ".avi", ".mov", ".webm"].includes(ext)) {
-    return "video/mp4";
-  } else if ([".txt", ".md", ".json"].includes(ext)) {
-    return "text/plain";
-  }
-
-  return "application/octet-stream";
-}
-
-module.exports = { colorizeStatus, detectContentType };

package/src/cli/commands/ai-ml/training-commands.js

@@ -1,228 +0,0 @@
-/**
- * Training-related CLI commands
- * Handles model training, fine-tuning, status checking, and deployment
- */
-
-const chalk = require("chalk");
-const ora = require("ora");
-const Table = require("cli-table3");
-const { colorizeStatus } = require("./shared");
-
-/**
- * Train a custom embedding model
- * @param {ModelTrainingOrchestrator} modelTrainer - The model training orchestrator instance
- * @param {object} _options - Command options
- */
-async function trainEmbeddingModel(modelTrainer, _options) {
-  const spinner = ora("Initializing embedding model training...").start();
-
-  try {
-    if (!_options.tenant) {
-      spinner.fail("Tenant ID is required");
-      return;
-    }
-
-    const trainingConfig = {
-      modelType: "embedding",
-      architecture: _options.architecture,
-      dataset: {
-        path: _options.dataset,
-        _type: "text_pairs",
-      },
-      hyperparameters: {
-        epochs: parseInt(_options.epochs),
-        batchSize: parseInt(_options.batchSize),
-        learningRate: parseFloat(_options.learningRate),
-      },
-    };
-
-    spinner.text = "Creating training job...";
-    const jobId = await modelTrainer.createTrainingJob(
-      _options.tenant,
-      trainingConfig,
-    );
-
-    if (_options.optimize) {
-      spinner.text = "Starting hyperparameter optimization...";
-      const optimizationConfig = {
-        ...trainingConfig,
-        hyperparameters: {
-          learningRate: [0.0001, 0.001, 0.01],
-          batchSize: [16, 32, 64],
-          hiddenSize: [256, 512, 768],
-        },
-        optimization: {
-          strategy: "bayesian",
-          maxTrials: 10,
-          metric: "accuracy",
-        },
-      };
-
-      const optimizationId = await modelTrainer.optimizeHyperparameters(
-        _options.tenant,
-        optimizationConfig,
-      );
-
-      spinner.succeed(`Hyperparameter optimization started: ${optimizationId}`);
-    } else {
-      spinner.text = "Starting training...";
-      await modelTrainer.startTraining(jobId);
-      spinner.succeed(`Training started: ${jobId}`);
-    }
-
-    console.log(chalk.green("\n✓ Training initiated successfully"));
-    // eslint-disable-line no-console
-    console.log(chalk.blue(`Job ID: ${jobId}`));
-    // eslint-disable-line no-console
-    console.log(
-      chalk.gray(
-        `Use 'rag-pipeline ai train status ${jobId}' to check progress`,
-      ),
-    );
-    // eslint-disable-line no-console
-  } catch (error) {
-    spinner.fail(`Training failed: ${error.message}`);
-  }
-}
-
-/**
- * Fine-tune a language model
- * @param {ModelTrainingOrchestrator} modelTrainer - The model training orchestrator instance
- * @param {object} _options - Command options
- */
-async function fineTuneLLM(modelTrainer, _options) {
-  const spinner = ora("Initializing LLM fine-tuning...").start();
-
-  try {
-    if (!_options.tenant) {
-      spinner.fail("Tenant ID is required");
-      return;
-    }
-
-    const trainingConfig = {
-      modelType: "llm",
-      baseModel: _options.baseModel,
-      dataset: {
-        path: _options.dataset,
-        _type: "instruction_following",
-      },
-      fineTuning: {
-        method: _options.lora ? "lora" : "full",
-        epochs: parseInt(_options.epochs),
-        learningRate: 0.0001,
-      },
-    };
-
-    spinner.text = "Creating fine-tuning job...";
-    const jobId = await modelTrainer.createTrainingJob(
-      _options.tenant,
-      trainingConfig,
-    );
-
-    spinner.text = "Starting fine-tuning...";
-    await modelTrainer.startTraining(jobId);
-
-    spinner.succeed(`Fine-tuning started: ${jobId}`);
-    console.log(chalk.green("\n✓ LLM fine-tuning initiated"));
-    // eslint-disable-line no-console
-    console.log(chalk.blue(`Job ID: ${jobId}`));
-    // eslint-disable-line no-console
-  } catch (error) {
-    spinner.fail(`Fine-tuning failed: ${error.message}`);
-  }
-}
-
-/**
- * Check training job status
- * @param {ModelTrainingOrchestrator} modelTrainer - The model training orchestrator instance
- * @param {string} jobId - Training job ID
- */
-async function checkTrainingStatus(modelTrainer, jobId) {
-  const spinner = ora("Checking training status...").start();
-
-  try {
-    const status = await modelTrainer.getTrainingStatus(jobId);
-    spinner.stop();
-
-    const table = new Table({
-      head: ["Property", "Value"],
-      colWidths: [20, 50],
-    });
-
-    table.push(
-      ["Job ID", jobId],
-      ["Status", colorizeStatus(status.status)],
-      ["Progress", `${status.progress}%`],
-      ["Current Epoch", `${status.currentEpoch}/${status.totalEpochs}`],
-      ["Accuracy", status.metrics?.accuracy?.toFixed(4) || "N/A"],
-      ["Loss", status.metrics?.loss?.toFixed(4) || "N/A"],
-      ["Estimated Time", status.estimatedTimeRemaining || "N/A"],
-    );
-
-    console.log(table.toString());
-    // eslint-disable-line no-console
-
-    if (status.status === "completed") {
-      console.log(chalk.green("\n✓ Training completed successfully!"));
-      // eslint-disable-line no-console
-      console.log(
-        chalk.blue(
-          `Use 'rag-pipeline ai train deploy ${jobId}' to deploy the model`,
-        ),
-      );
-      // eslint-disable-line no-console
-    } else if (status.status === "failed") {
-      console.log(chalk.red("\n✗ Training failed"));
-      // eslint-disable-line no-console
-      if (status.error) {
-        console.log(chalk.gray(`Error: ${status.error}`));
-        // eslint-disable-line no-console
-      }
-    }
-  } catch (error) {
-    spinner.fail(`Failed to get status: ${error.message}`);
-  }
-}
-
-/**
- * Deploy a trained model
- * @param {ModelTrainingOrchestrator} modelTrainer - The model training orchestrator instance
- * @param {string} jobId - Training job ID
- * @param {object} _options - Command options
- */
-async function deployModel(modelTrainer, jobId, _options) {
-  const spinner = ora("Deploying model...").start();
-
-  try {
-    const deploymentConfig = {
-      environment: _options.environment,
-      scalingConfig: {
-        minInstances: 1,
-        maxInstances: _options.autoScale ? 5 : 1,
-        autoScale: !!_options.autoScale,
-      },
-    };
-
-    const deploymentId = await modelTrainer.deployModel(
-      jobId,
-      deploymentConfig,
-    );
-
-    spinner.succeed(`Model deployed: ${deploymentId}`);
-    console.log(chalk.green("\n✓ Model deployment successful"));
-    // eslint-disable-line no-console
-    console.log(chalk.blue(`Deployment ID: ${deploymentId}`));
-    // eslint-disable-line no-console
-    console.log(chalk.blue(`Environment: ${_options.environment}`));
-    // eslint-disable-line no-console
-  } catch (error) {
-    spinner.fail(`Deployment failed: ${error.message}`);
-  }
-}
-
-module.exports = {
-  trainEmbeddingModel,
-  fineTuneLLM,
-  checkTrainingStatus,
-  deployModel,
-};