@devilsdev/rag-pipeline-utils 2.3.0 → 2.4.0

package/src/cli/handlers/config-handler.js

@@ -0,0 +1,126 @@
+/**
+ * Config command handlers
+ * Handles config show, set, and get commands
+ */
+
+const fs = require("fs/promises");
+const { logger } = require("../../utils/logger.js");
+
+/**
+ * Handle config show command
+ * @param {object} globalOptions - Global CLI options from program.opts()
+ * @param {object} options - Command-specific options
+ */
+async function handleConfigShow(globalOptions, options) {
+  try {
+    const config = JSON.parse(await fs.readFile(globalOptions.config, "utf-8"));
+
+    let output = config;
+    if (options.section) {
+      const sections = options.section.split(".");
+      for (const section of sections) {
+        output = output[section];
+        if (output === undefined) {
+          console.log(`❌ Section not found: ${options.section}`);
+          return;
+        }
+      }
+    }
+
+    console.log(JSON.stringify(output, null, 2));
+  } catch (error) {
+    logger.error("❌ Failed to show configuration:", error.message);
+    process.exit(1);
+  }
+}
+
+/**
+ * Handle config set command
+ * @param {object} globalOptions - Global CLI options from program.opts()
+ * @param {string} key - Configuration key (dot notation)
+ * @param {string} value - Configuration value
+ */
+async function handleConfigSet(globalOptions, key, value) {
+  try {
+    const config = JSON.parse(await fs.readFile(globalOptions.config, "utf-8"));
+
+    // Parse value
+    let parsedValue;
+    try {
+      parsedValue = JSON.parse(value);
+    } catch (error) {
+      parsedValue = value; // Keep as string
+    }
+
+    // Validate key segments against prototype pollution
+    const keySegments = key.split(".");
+    const forbidden = new Set(["__proto__", "constructor", "prototype"]);
+    if (keySegments.some((k) => forbidden.has(k))) {
+      logger.error("Invalid configuration key: contains restricted property");
+      process.exit(1);
+    }
+
+    // Build a patch object from the key path, then merge via JSON round-trip
+    // This avoids dynamic property assignment entirely (no prototype pollution risk)
+    const patch = JSON.parse(
+      keySegments.reduceRight(
+        (val, seg) => JSON.stringify({ [seg]: JSON.parse(val) }),
+        JSON.stringify(parsedValue),
+      ),
+    );
+
+    // Shallow-merge at each level
+    function safeMerge(target, source) {
+      for (const k of Object.keys(source)) {
+        if (
+          source[k] &&
+          typeof source[k] === "object" &&
+          !Array.isArray(source[k]) &&
+          target[k] &&
+          typeof target[k] === "object"
+        ) {
+          safeMerge(target[k], source[k]);
+        } else {
+          target[k] = source[k];
+        }
+      }
+    }
+    safeMerge(config, patch);
+
+    // Save configuration
+    await fs.writeFile(globalOptions.config, JSON.stringify(config, null, 2));
+    console.log("✅ Configuration updated");
+  } catch (error) {
+    logger.error("❌ Failed to set configuration:", error.message);
+    process.exit(1);
+  }
+}
+
+/**
+ * Handle config get command
+ * @param {object} globalOptions - Global CLI options from program.opts()
+ * @param {string} key - Configuration key (dot notation)
+ */
+async function handleConfigGet(globalOptions, key) {
+  try {
+    const config = JSON.parse(await fs.readFile(globalOptions.config, "utf-8"));
+
+    const keys = key.split(".");
+    let value = config;
+
+    for (const k of keys) {
+      value = value[k];
+      if (value === undefined) {
+        console.log("❌ Key not found");
+        return;
+      }
+    }
+
+    console.log(JSON.stringify(value, null, 2));
+  } catch (error) {
+    logger.error("❌ Failed to get configuration:", error.message);
+    process.exit(1);
+  }
+}
+
+module.exports = { handleConfigShow, handleConfigSet, handleConfigGet };

package/src/cli/handlers/diagnostics-handler.js

@@ -0,0 +1,113 @@
+/**
+ * Diagnostics command handlers
+ * Handles doctor, validate, and info commands
+ */
+
+const fs = require("fs/promises");
+const { logger } = require("../../utils/logger.js");
+
+/**
+ * Handle doctor command
+ * @param {object} globalOptions - Global CLI options from program.opts()
+ * @param {object} options - Command-specific options
+ * @param {Function} pipelineDoctorFn - Pipeline doctor function
+ */
+async function handleDoctor(globalOptions, options, pipelineDoctorFn) {
+  try {
+    const doctorOptions = {
+      configPath: globalOptions.config,
+      verbose: globalOptions.verbose,
+      autoFix: options.autoFix,
+      categories: options.category,
+    };
+
+    const report = await pipelineDoctorFn(doctorOptions);
+
+    if (options.report) {
+      await fs.writeFile(options.report, JSON.stringify(report, null, 2));
+      console.log(`📋 Diagnostic report saved to: ${options.report}`);
+    }
+  } catch (error) {
+    logger.error("❌ Doctor command failed:", error.message);
+    process.exit(1);
+  }
+}
+
+/**
+ * Handle validate command
+ * @param {object} globalOptions - Global CLI options from program.opts()
+ * @param {object} options - Command-specific options
+ * @param {Function} validateSchemaFn - Schema validation function
+ */
+async function handleValidate(globalOptions, options, validateSchemaFn) {
+  try {
+    const configPath = globalOptions.config;
+
+    console.log(`🔍 Validating configuration: ${configPath}`);
+
+    // Load and validate configuration
+    const configContent = await fs.readFile(configPath, "utf-8");
+    const config = JSON.parse(configContent);
+
+    const validation = validateSchemaFn(config);
+
+    if (validation.valid) {
+      console.log("✅ Configuration is valid");
+
+      if (validation.legacy) {
+        console.log("⚠️  Using legacy format - consider upgrading");
+      }
+    } else {
+      console.log("❌ Configuration validation failed:");
+      validation.errors?.forEach((error) => {
+        console.log(`  ${error.instancePath || "root"}: ${error.message}`);
+      });
+    }
+  } catch (error) {
+    logger.error("❌ Validation failed:", error.message);
+    process.exit(1);
+  }
+}
+
+/**
+ * Handle info command
+ * @param {object} globalOptions - Global CLI options from program.opts()
+ * @param {object} options - Command-specific options
+ */
+async function handleInfo(globalOptions, options) {
+  try {
+    console.log("📊 RAG Pipeline Information\n");
+
+    // System information
+    if ((!options.plugins && !options.config) || options.system) {
+      console.log("System:");
+      console.log(`  Node.js: ${process.version}`);
+      console.log(`  Platform: ${process.platform} ${process.arch}`);
+      console.log(
+        `  Memory: ${Math.round(process.memoryUsage().heapUsed / 1024 / 1024)}MB used`,
+      );
+      console.log("");
+    }
+
+    // Configuration summary
+    if ((!options.system && !options.plugins) || options.config) {
+      try {
+        const config = JSON.parse(
+          await fs.readFile(globalOptions.config, "utf-8"),
+        );
+        console.log("Configuration:");
+        console.log(`  File: ${globalOptions.config}`);
+        console.log(`  Format: ${config.plugins ? "Enhanced" : "Legacy"}`);
+        console.log("");
+      } catch (error) {
+        console.log("Configuration: Not found or invalid");
+        console.log("");
+      }
+    }
+  } catch (error) {
+    logger.error("❌ Failed to get system information:", error.message);
+    process.exit(1);
+  }
+}
+
+module.exports = { handleDoctor, handleValidate, handleInfo };

package/src/cli/handlers/ingest-handler.js

@@ -0,0 +1,42 @@
+/**
+ * Ingest command handler
+ * Handles document ingestion into the pipeline
+ */
+
+const { logger } = require("../../utils/logger.js");
+
+/**
+ * Handle ingest command
+ * @param {object} globalOptions - Global CLI options from program.opts()
+ * @param {string} file - Document file path
+ * @param {object} options - Command-specific options
+ */
+async function handleIngest(globalOptions, file, options) {
+  try {
+    if (globalOptions.dryRun) {
+      console.log("🧪 Dry run: Would ingest document");
+      console.log(`File: ${file}`);
+      console.log(
+        `Parallel processing: ${options.parallel ? "enabled" : "disabled"}`,
+      );
+      console.log(`Streaming: ${options.streaming ? "enabled" : "disabled"}`);
+      console.log(`Tracing: ${options.trace ? "enabled" : "disabled"}`);
+
+      if (options.preview) {
+        console.log("📄 Document Preview:");
+      }
+      if (options.validate) {
+        console.log("File validation passed");
+      }
+      return;
+    }
+
+    console.log(`📄 Ingesting document: ${file}`);
+    console.log("✅ Document ingested successfully!");
+  } catch (error) {
+    logger.error("❌ Document ingestion failed:", error.message);
+    process.exit(1);
+  }
+}
+
+module.exports = { handleIngest };

package/src/cli/handlers/init-handler.js

@@ -0,0 +1,51 @@
+/**
+ * Init command handler
+ * Handles RAG pipeline configuration initialization
+ */
+
+const fs = require("fs/promises");
+const { logger } = require("../../utils/logger.js");
+
+/**
+ * Handle init command
+ * @param {object} globalOptions - Global CLI options from program.opts()
+ * @param {object} options - Command-specific options
+ * @param {Function} createBasicConfigFn - Function to create basic config
+ */
+async function handleInit(globalOptions, options, createBasicConfigFn) {
+  try {
+    if (globalOptions.dryRun) {
+      console.log("🧪 Dry run: Would initialize RAG pipeline configuration");
+      console.log(`Output file: ${options.output}`);
+      console.log(`Interactive mode: ${!options.noInteractive}`);
+      return;
+    }
+
+    // Check if config already exists
+    if (!options.force) {
+      try {
+        await fs.access(options.output);
+        console.log(`❌ Configuration file already exists: ${options.output}`);
+        console.log(
+          "Use --force to overwrite or choose a different output path.",
+        );
+        return;
+      } catch (error) {
+        // File doesn't exist, which is good
+      }
+    }
+
+    if (!options.noInteractive) {
+      console.log("🧙‍♂️ Starting interactive configuration wizard...\n");
+      // Mock interactive wizard
+      await createBasicConfigFn(options.output);
+    } else {
+      await createBasicConfigFn(options.output);
+    }
+  } catch (error) {
+    logger.error("❌ Initialization failed:", error.message);
+    process.exit(1);
+  }
+}
+
+module.exports = { handleInit };

package/src/cli/handlers/query-handler.js

@@ -0,0 +1,45 @@
+/**
+ * Query command handler
+ * Handles RAG pipeline queries
+ */
+
+const { logger } = require("../../utils/logger.js");
+
+/**
+ * Handle query command
+ * @param {object} globalOptions - Global CLI options from program.opts()
+ * @param {string} prompt - Query prompt
+ * @param {object} options - Command-specific options
+ */
+async function handleQuery(globalOptions, prompt, options) {
+  try {
+    if (globalOptions.dryRun) {
+      console.log("🧪 Dry run: Would execute query");
+      console.log(
+        `Prompt: ${prompt.substring(0, 100)}${prompt.length > 100 ? "..." : ""}`,
+      );
+      console.log(`Streaming: ${options.stream ? "enabled" : "disabled"}`);
+
+      if (options.explain) {
+        console.log("🔍 Query Explanation:");
+        console.log("Processing steps:");
+        console.log("  1. Text embedding generation");
+        console.log("  2. Vector similarity search");
+        console.log("  3. Context retrieval");
+        console.log("  4. Language model generation");
+      }
+      return;
+    }
+
+    console.log(
+      `🤔 Processing query: ${prompt.substring(0, 100)}${prompt.length > 100 ? "..." : ""}`,
+    );
+    console.log("\n📝 Response:");
+    console.log("Mock response to query");
+  } catch (error) {
+    logger.error("❌ Query processing failed:", error.message);
+    process.exit(1);
+  }
+}
+
+module.exports = { handleQuery };

package/src/cli/handlers/run-handler.js

@@ -0,0 +1,181 @@
+/**
+ * Run command handler
+ * Handles pipeline execution with streaming, abort, and progress support
+ */
+
+const { logger } = require("../../utils/logger.js");
+const { createProgressBar } = require("../shared/cli-utils.js");
+
+/**
+ * Handle run command
+ * @param {object} globalOptions - Global CLI options from program.opts()
+ * @param {object} options - Command-specific options
+ */
+async function handleRun(globalOptions, options) {
+  if (globalOptions.dryRun) {
+    logger.info("🧪 Dry run: Would execute pipeline operation");
+    if (options.ingest) logger.info(`Would ingest: ${options.ingest}`);
+    if (options.query)
+      logger.info(`Would query: ${options.query.substring(0, 100)}...`);
+    logger.info(`Streaming: ${options.stream ? "enabled" : "disabled"}`);
+    logger.info(`Parallel: ${options.parallel ? "enabled" : "disabled"}`);
+    if (options.abortTimeout)
+      logger.info(`Abort timeout: ${options.abortTimeout}ms`);
+    return;
+  }
+
+  try {
+    // Load configuration
+    const { loadRagConfig } = require("../../config/load-config.js");
+    const config = await loadRagConfig(globalOptions.config);
+
+    // Create AbortController for timeout handling
+    const abortController = new AbortController();
+    let timeoutId;
+
+    if (options.abortTimeout) {
+      timeoutId = setTimeout(() => {
+        logger.warn(
+          `Operation aborted after ${options.abortTimeout}ms timeout`,
+        );
+        abortController.abort();
+      }, options.abortTimeout);
+    }
+
+    // Setup SIGINT handling for clean shutdown
+    const sigintHandler = () => {
+      logger.info("Received SIGINT, aborting operation...");
+      abortController.abort();
+      if (timeoutId) clearTimeout(timeoutId);
+    };
+    process.once("SIGINT", sigintHandler);
+
+    try {
+      // Create pipeline with performance options
+      const { createRagPipeline } = require("../../core/create-pipeline.js");
+      const pipeline = createRagPipeline(config.pipeline[0], {
+        useParallelProcessing: options.parallel,
+        useStreamingSafeguards: options.stream,
+        useLogging: true,
+        useRetry: true,
+      });
+
+      // Execute operation based on options
+      if (options.ingest) {
+        logger.info(`Starting ingestion of: ${options.ingest}`);
+
+        if (options.stream) {
+          let lastProgressTime = Date.now();
+          for await (const progress of pipeline.ingestStream(options.ingest)) {
+            if (abortController.signal.aborted) {
+              logger.info("Ingestion aborted");
+              break;
+            }
+
+            const now = Date.now();
+            if (progress.type === "chunk_processed") {
+              // Show progress ticks every 500ms or every 10 chunks
+              if (
+                now - lastProgressTime > 500 ||
+                progress.progress.processed % 10 === 0
+              ) {
+                const percentage = Math.round(
+                  (progress.progress.processed / progress.progress.total) * 100,
+                );
+                const progressBar = createProgressBar(percentage, 20);
+                process.stdout.write(
+                  `\r${progressBar} ${percentage}% (${progress.progress.processed}/${progress.progress.total}) `,
+                );
+                lastProgressTime = now;
+              }
+            } else if (progress.type === "ingest_complete") {
+              process.stdout.write("\n");
+              logger.info(
+                `✅ Ingestion complete: ${progress.summary.processedChunks}/${progress.summary.totalChunks} chunks processed`,
+              );
+              if (progress.summary.failedChunks > 0) {
+                logger.warn(
+                  `⚠️  ${progress.summary.failedChunks} chunks failed`,
+                );
+              }
+            } else if (progress.type === "chunk_failed") {
+              process.stdout.write("✗");
+            }
+          }
+        } else {
+          // Non-streaming ingestion with progress callback
+          let currentStage = "";
+          const progressCallback = (progress) => {
+            if (progress.stage !== currentStage) {
+              if (currentStage) process.stdout.write("\n");
+              currentStage = progress.stage;
+              logger.info(
+                `${progress.stage}: ${progress.message || "Processing..."}`,
+              );
+            }
+            if (
+              progress.completed !== undefined &&
+              progress.total !== undefined
+            ) {
+              const percentage = Math.round(
+                (progress.completed / progress.total) * 100,
+              );
+              const progressBar = createProgressBar(percentage, 20);
+              process.stdout.write(
+                `\r${progressBar} ${percentage}% (${progress.completed}/${progress.total}) `,
+              );
+            }
+          };
+
+          const pipelineWithProgress = createRagPipeline(config.pipeline[0], {
+            useParallelProcessing: options.parallel,
+            useStreamingSafeguards: false,
+            useLogging: true,
+            useRetry: true,
+            onProgress: progressCallback,
+          });
+
+          await pipelineWithProgress.ingest(options.ingest);
+          process.stdout.write("\n");
+          logger.info("✅ Ingestion completed successfully");
+        }
+      }
+
+      if (options.query) {
+        logger.info(`Executing query: ${options.query.substring(0, 100)}...`);
+
+        if (options.stream) {
+          process.stdout.write("\nResponse: ");
+          for await (const token of pipeline.queryStream(options.query)) {
+            if (abortController.signal.aborted) {
+              logger.info("\nQuery aborted");
+              break;
+            }
+            process.stdout.write(token);
+          }
+          process.stdout.write("\n\n");
+        } else {
+          const response = await pipeline.query(options.query);
+          logger.info("Query completed successfully");
+          console.log("\nResponse:", response);
+        }
+      }
+
+      if (!options.ingest && !options.query) {
+        logger.error(
+          "No operation specified. Use --ingest <file> or --query <prompt>",
+        );
+        process.exit(1);
+      }
+    } finally {
+      // Cleanup
+      process.removeListener("SIGINT", sigintHandler);
+      if (timeoutId) clearTimeout(timeoutId);
+    }
+  } catch (error) {
+    logger.error("Pipeline operation failed", { error: error.message });
+    process.exit(1);
+  }
+}
+
+module.exports = { handleRun };