@devilsdev/rag-pipeline-utils 2.3.0 → 2.4.0

package/src/cli/shared/cli-utils.js

@@ -0,0 +1,89 @@
+/**
+ * Shared CLI utility functions
+ */
+
+const pkg = require("../../../package.json");
+
+/**
+ * Create a simple progress bar for CLI output
+ * @param {number} percentage - Progress percentage (0-100)
+ * @param {number} width - Width of progress bar in characters
+ * @returns {string} Progress bar string
+ */
+function createProgressBar(percentage, width = 20) {
+  const filled = Math.round((percentage / 100) * width);
+  const empty = width - filled;
+  const bar = "\u2588".repeat(filled) + "\u2591".repeat(empty);
+  return `[${bar}]`;
+}
+
+/**
+ * Custom parser for positive integers
+ */
+function parsePositiveInteger(value) {
+  const parsed = parseInt(value, 10);
+  if (isNaN(parsed) || parsed <= 0) {
+    throw new Error(`Invalid value '${value}'. Expected positive integer.`);
+  }
+  return parsed;
+}
+
+/**
+ * Generate shell completion script
+ */
+function generateCompletionScript(shell) {
+  switch (shell) {
+    case "bash":
+      return `# RAG Pipeline bash completion
+_rag_pipeline_completions() {
+  COMPREPLY=($(compgen -W "init doctor ingest query plugin config info validate completion" -- "\${COMP_WORDS[COMP_CWORD]}"))
+}
+complete -F _rag_pipeline_completions rag-pipeline`;
+
+    case "zsh":
+      return `# RAG Pipeline zsh completion
+#compdef rag-pipeline
+_rag_pipeline() {
+  _arguments \\
+    '1:command:(init doctor ingest query plugin config info validate completion)'
+}
+_rag_pipeline "$@"`;
+
+    case "fish":
+      return `# RAG Pipeline fish completion
+complete -c rag-pipeline -n '__fish_use_subcommand' -a 'init doctor ingest query plugin config info validate completion'`;
+
+    default:
+      return `# Completion not available for ${shell}`;
+  }
+}
+
+/**
+ * Get version information from package.json
+ */
+function getVersion() {
+  return pkg.version;
+}
+
+/**
+ * Get extended help text
+ */
+function getExtendedHelp() {
+  return `
+Examples:
+  rag-pipeline init --interactive          Initialize with wizard
+  rag-pipeline doctor --auto-fix           Diagnose and fix issues
+  rag-pipeline plugin search openai        Search for plugins
+  rag-pipeline config show plugins         Show plugin configuration
+
+For more information, visit: https://github.com/DevilsDev/rag-pipeline-utils
+`;
+}
+
+module.exports = {
+  createProgressBar,
+  parsePositiveInteger,
+  generateCompletionScript,
+  getVersion,
+  getExtendedHelp,
+};

package/src/cli/wizard/configuration.js

@@ -0,0 +1,148 @@
+/**
+ * Configuration Step
+ * Handles general settings configuration (caching, timeout, pipeline stages)
+ */
+
+/**
+ * Configure general settings
+ * @param {object} config - Current wizard configuration state
+ * @param {object} inquirer - Inquirer instance for prompting
+ * @returns {Promise<object>} Updated configuration state
+ */
+async function configureSettings(config, inquirer) {
+  console.log("⚙️  General Settings\n");
+  // eslint-disable-line no-console
+
+  const answers = await inquirer.prompt([
+    {
+      _type: "confirm",
+      name: "enableCaching",
+      message: "Enable result caching?",
+      default: false,
+    },
+    {
+      _type: "number",
+      name: "cacheSize",
+      message: "Cache size (number of entries):",
+      default: 1000,
+      when: (answers) => answers.enableCaching,
+    },
+    {
+      _type: "number",
+      name: "cacheTtl",
+      message: "Cache TTL (seconds):",
+      default: 3600,
+      when: (answers) => answers.enableCaching,
+    },
+    {
+      _type: "number",
+      name: "timeout",
+      message: "Pipeline timeout (milliseconds):",
+      default: 30000,
+    },
+  ]);
+
+  if (answers.enableCaching) {
+    config.performance = config.performance || {};
+    config.performance.caching = {
+      enabled: true,
+      maxSize: answers.cacheSize,
+      ttl: answers.cacheTtl,
+    };
+  }
+
+  if (answers.timeout !== 30000) {
+    config.pipeline = config.pipeline || {};
+    config.pipeline.timeout = answers.timeout;
+  }
+
+  console.log("✅ General settings configured\n");
+  // eslint-disable-line no-console
+
+  return config;
+}
+
+/**
+ * Configure pipeline stages
+ * @param {object} config - Current wizard configuration state
+ * @param {object} inquirer - Inquirer instance for prompting
+ * @returns {Promise<object>} Updated configuration state
+ */
+async function configurePipeline(config, inquirer) {
+  console.log("🔄 Pipeline Configuration\n");
+  // eslint-disable-line no-console
+
+  const availableStages = Object.keys(config.plugins).filter(
+    (_type) => Object.keys(config.plugins[_type]).length > 0,
+  );
+
+  const answers = await inquirer.prompt([
+    {
+      _type: "checkbox",
+      name: "stages",
+      message: "Select pipeline stages (in order):",
+      choices: [
+        {
+          name: "Document Loading",
+          value: "loader",
+          checked: availableStages.includes("loader"),
+        },
+        {
+          name: "Text Embedding",
+          value: "embedder",
+          checked: availableStages.includes("embedder"),
+        },
+        {
+          name: "Vector Retrieval",
+          value: "retriever",
+          checked: availableStages.includes("retriever"),
+        },
+        {
+          name: "Language Model",
+          value: "llm",
+          checked: availableStages.includes("llm"),
+        },
+        {
+          name: "Result Reranking",
+          value: "reranker",
+          checked: availableStages.includes("reranker"),
+        },
+      ],
+      validate: (input) =>
+        input.length > 0 || "At least one stage must be selected",
+    },
+    {
+      _type: "confirm",
+      name: "enableRetries",
+      message: "Enable automatic retries on failures?",
+      default: true,
+    },
+    {
+      _type: "number",
+      name: "maxRetries",
+      message: "Maximum retry attempts:",
+      default: 3,
+      when: (answers) => answers.enableRetries,
+    },
+  ]);
+
+  config.pipeline = {
+    stages: answers.stages,
+    ...config.pipeline,
+  };
+
+  if (answers.enableRetries) {
+    config.pipeline.retries = {
+      enabled: true,
+      maxAttempts: answers.maxRetries,
+      backoff: "exponential",
+    };
+  }
+
+  console.log("✅ Pipeline configuration complete\n");
+  // eslint-disable-line no-console
+
+  return config;
+}
+
+module.exports = { configureSettings, configurePipeline };

package/src/cli/wizard/observability.js

@@ -0,0 +1,79 @@
+/**
+ * Observability Step
+ * Handles observability settings configuration (logging, tracing, metrics)
+ */
+
+/**
+ * Configure observability settings
+ * @param {object} config - Current wizard configuration state
+ * @param {object} inquirer - Inquirer instance for prompting
+ * @returns {Promise<object>} Updated configuration state
+ */
+async function configureObservability(config, inquirer) {
+  console.log("📊 Observability Settings\n");
+  // eslint-disable-line no-console
+
+  const answers = await inquirer.prompt([
+    {
+      _type: "list",
+      name: "logLevel",
+      message: "Log level:",
+      choices: [
+        { name: "Debug (verbose)", value: "debug" },
+        { name: "Info (default)", value: "info" },
+        { name: "Warning (minimal)", value: "warn" },
+        { name: "Error (errors only)", value: "error" },
+      ],
+      default: "info",
+    },
+    {
+      _type: "confirm",
+      name: "enableTracing",
+      message: "Enable distributed tracing?",
+      default: false,
+    },
+    {
+      _type: "confirm",
+      name: "enableMetrics",
+      message: "Enable metrics collection?",
+      default: false,
+    },
+    {
+      _type: "input",
+      name: "exportUrl",
+      message: "Metrics/tracing export URL (optional):",
+      when: (answers) => answers.enableTracing || answers.enableMetrics,
+    },
+  ]);
+
+  config.observability = {
+    logging: {
+      level: answers.logLevel,
+      structured: true,
+      events: answers.enableTracing,
+    },
+  };
+
+  if (answers.enableTracing) {
+    config.observability.tracing = {
+      enabled: true,
+      exportUrl: answers.exportUrl || undefined,
+      sampleRate: 1,
+    };
+  }
+
+  if (answers.enableMetrics) {
+    config.observability.metrics = {
+      enabled: true,
+      exportUrl: answers.exportUrl || undefined,
+      interval: 60000,
+    };
+  }
+
+  console.log("✅ Observability settings configured\n");
+  // eslint-disable-line no-console
+
+  return config;
+}
+
+module.exports = { configureObservability };

package/src/cli/wizard/performance.js

@@ -0,0 +1,76 @@
+/**
+ * Performance Step
+ * Handles performance settings configuration (parallel processing, streaming)
+ */
+
+/**
+ * Configure performance settings
+ * @param {object} config - Current wizard configuration state
+ * @param {object} inquirer - Inquirer instance for prompting
+ * @returns {Promise<object>} Updated configuration state
+ */
+async function configurePerformance(config, inquirer) {
+  console.log("⚡ Performance Settings\n");
+  // eslint-disable-line no-console
+
+  const answers = await inquirer.prompt([
+    {
+      _type: "confirm",
+      name: "enableParallel",
+      message: "Enable parallel processing?",
+      default: false,
+    },
+    {
+      _type: "number",
+      name: "maxConcurrency",
+      message: "Maximum concurrent operations:",
+      default: 3,
+      when: (answers) => answers.enableParallel,
+    },
+    {
+      _type: "number",
+      name: "batchSize",
+      message: "Batch size for parallel processing:",
+      default: 10,
+      when: (answers) => answers.enableParallel,
+    },
+    {
+      _type: "confirm",
+      name: "enableStreaming",
+      message: "Enable streaming for large documents?",
+      default: false,
+    },
+    {
+      _type: "number",
+      name: "maxMemoryMB",
+      message: "Maximum memory usage (MB):",
+      default: 512,
+      when: (answers) => answers.enableStreaming,
+    },
+  ]);
+
+  config.performance = config.performance || {};
+
+  if (answers.enableParallel) {
+    config.performance.parallel = {
+      enabled: true,
+      maxConcurrency: answers.maxConcurrency,
+      batchSize: answers.batchSize,
+    };
+  }
+
+  if (answers.enableStreaming) {
+    config.performance.streaming = {
+      enabled: true,
+      maxMemoryMB: answers.maxMemoryMB,
+      bufferSize: 100,
+    };
+  }
+
+  console.log("✅ Performance settings configured\n");
+  // eslint-disable-line no-console
+
+  return config;
+}
+
+module.exports = { configurePerformance };

package/src/cli/wizard/plugin-selection.js

@@ -0,0 +1,297 @@
+/**
+ * Plugin Selection Step
+ * Handles plugin selection, custom plugin configuration, and plugin settings
+ */
+
+/**
+ * Select plugins for each type
+ * @param {object} config - Current wizard configuration state
+ * @param {object} inquirer - Inquirer instance for prompting
+ * @param {object} context - Wizard context with registry and helper methods
+ * @param {object|null} context.registry - Plugin registry data
+ * @param {Function} context.getAvailablePlugins - Function to get available plugins for a type
+ * @param {Function} context.getPluginInfo - Function to get plugin info
+ * @param {Function} context.loadRegistry - Function to load plugin registry
+ * @returns {Promise<object>} Updated configuration state
+ */
+async function selectPlugins(config, inquirer, context) {
+  console.log("🔌 Plugin Selection\n");
+  // eslint-disable-line no-console
+
+  // Load registry for plugin suggestions
+  try {
+    context.registry = await context.loadRegistry();
+  } catch (error) {
+    console.warn(
+      "⚠️  Could not load plugin registry. Using built-in _options.",
+    );
+    // eslint-disable-line no-console
+  }
+
+  const pluginTypes = [
+    { key: "loader", name: "Document Loader", required: true },
+    { key: "embedder", name: "Text Embedder", required: true },
+    { key: "retriever", name: "Vector Retriever", required: true },
+    { key: "llm", name: "Language Model", required: true },
+    { key: "reranker", name: "Result Reranker", required: false },
+  ];
+
+  for (const pluginType of pluginTypes) {
+    await selectPluginForType(config, inquirer, context, pluginType);
+  }
+
+  console.log("✅ Plugin selection complete\n");
+  // eslint-disable-line no-console
+
+  return config;
+}
+
+/**
+ * Select plugin for a specific type
+ * @param {object} config - Current wizard configuration state
+ * @param {object} inquirer - Inquirer instance for prompting
+ * @param {object} context - Wizard context with helper methods
+ * @param {object} pluginType - Plugin type configuration
+ * @returns {Promise<void>}
+ */
+async function selectPluginForType(config, inquirer, context, pluginType) {
+  const availablePlugins = context.getAvailablePlugins(pluginType.key);
+
+  if (availablePlugins.length === 0 && pluginType.required) {
+    console.log(
+      `⚠️  No ${pluginType.name} plugins available. You'll need to configure this manually.`,
+    );
+    // eslint-disable-line no-console
+    return;
+  }
+
+  const choices = [
+    ...availablePlugins.map((plugin) => ({
+      name: `${plugin.name} - ${plugin.description}`,
+      value: plugin.name,
+      short: plugin.name,
+    })),
+    { name: "Skip (configure later)", value: null },
+    { name: "Custom plugin", value: "custom" },
+  ];
+
+  const answer = await inquirer.prompt([
+    {
+      _type: "list",
+      name: "plugin",
+      message: `Select ${pluginType.name}:`,
+      choices,
+      when: () => !pluginType.required || availablePlugins.length > 0,
+    },
+  ]);
+
+  if (!answer.plugin) {
+    return; // Skip
+  }
+
+  if (answer.plugin === "custom") {
+    await configureCustomPlugin(config, inquirer, pluginType.key);
+  } else {
+    await configureSelectedPlugin(
+      config,
+      inquirer,
+      context,
+      pluginType.key,
+      answer.plugin,
+    );
+  }
+}
+
+/**
+ * Configure a custom plugin
+ * @param {object} config - Current wizard configuration state
+ * @param {object} inquirer - Inquirer instance for prompting
+ * @param {string} pluginType - Plugin type key
+ * @returns {Promise<void>}
+ */
+async function configureCustomPlugin(config, inquirer, pluginType) {
+  const answers = await inquirer.prompt([
+    {
+      _type: "input",
+      name: "name",
+      message: "Plugin name:",
+      validate: (input) => input.length > 0 || "Plugin name is required",
+    },
+    {
+      _type: "list",
+      name: "source",
+      message: "Plugin source:",
+      choices: [
+        { name: "Registry", value: "registry" },
+        { name: "Local file", value: "local" },
+        { name: "Git repository", value: "git" },
+        { name: "NPM package", value: "npm" },
+      ],
+      default: "registry",
+    },
+    {
+      _type: "input",
+      name: "version",
+      message: "Version:",
+      default: "latest",
+      when: (answers) =>
+        answers.source === "registry" || answers.source === "npm",
+    },
+    {
+      _type: "input",
+      name: "path",
+      message: "Local path:",
+      when: (answers) => answers.source === "local",
+    },
+    {
+      _type: "input",
+      name: "url",
+      message: "Repository URL:",
+      when: (answers) => answers.source === "git",
+    },
+  ]);
+
+  const pluginSpec = {
+    name: answers.name,
+    source: answers.source,
+  };
+
+  if (answers.version) pluginSpec.version = answers.version;
+  if (answers.path) pluginSpec.path = answers.path;
+  if (answers.url) pluginSpec.url = answers.url;
+
+  config.plugins[pluginType][answers.name] = pluginSpec;
+}
+
+/**
+ * Configure a selected plugin from registry
+ * @param {object} config - Current wizard configuration state
+ * @param {object} inquirer - Inquirer instance for prompting
+ * @param {object} context - Wizard context with helper methods
+ * @param {string} pluginType - Plugin type key
+ * @param {string} pluginName - Plugin name
+ * @returns {Promise<void>}
+ */
+async function configureSelectedPlugin(
+  config,
+  inquirer,
+  context,
+  pluginType,
+  pluginName,
+) {
+  const plugin = context.getPluginInfo(pluginType, pluginName);
+
+  if (!plugin) {
+    config.plugins[pluginType][pluginName] = "latest";
+    return;
+  }
+
+  const versions = Object.keys(plugin.versions || {}).slice(0, 5);
+  const versionChoices = [
+    { name: "Latest stable", value: "latest" },
+    { name: "Latest beta", value: "beta" },
+    ...versions.map((v) => ({ name: v, value: v })),
+    { name: "Custom version", value: "custom" },
+  ];
+
+  const versionAnswer = await inquirer.prompt([
+    {
+      _type: "list",
+      name: "version",
+      message: `Select version for ${pluginName}:`,
+      choices: versionChoices,
+      default: "latest",
+    },
+    {
+      _type: "input",
+      name: "customVersion",
+      message: "Enter version:",
+      when: (answers) => answers.version === "custom",
+    },
+  ]);
+
+  const version = versionAnswer.customVersion || versionAnswer.version;
+
+  // Check if plugin needs configuration
+  const needsConfig =
+    plugin.metadata?._config &&
+    Object.keys(plugin.metadata._config.properties || {}).length > 0;
+
+  if (needsConfig) {
+    const configAnswer = await inquirer.prompt([
+      {
+        _type: "confirm",
+        name: "configure",
+        message: `Configure ${pluginName} settings?`,
+        default: false,
+      },
+    ]);
+
+    if (configAnswer.configure) {
+      const pluginConfig = await configurePluginSettings(plugin, inquirer);
+      config.plugins[pluginType][pluginName] = {
+        name: pluginName,
+        version,
+        _config: pluginConfig,
+      };
+    } else {
+      config.plugins[pluginType][pluginName] = {
+        name: pluginName,
+        version,
+      };
+    }
+  } else {
+    config.plugins[pluginType][pluginName] = version;
+  }
+}
+
+/**
+ * Configure plugin-specific settings
+ * @param {object} plugin - Plugin information
+ * @param {object} inquirer - Inquirer instance for prompting
+ * @returns {Promise<object>} Plugin configuration
+ */
+async function configurePluginSettings(plugin, inquirer) {
+  const _config = {};
+  const schema = plugin.metadata?._config?.properties || {};
+
+  console.log(`\n⚙️  Configuring ${plugin.metadata.name}...\n`);
+  // eslint-disable-line no-console
+
+  for (const [key, property] of Object.entries(schema)) {
+    const question = {
+      name: key,
+      message: `${key}${property.description ? ` (${property.description})` : ""}:`,
+    };
+
+    if (property._type === "boolean") {
+      question._type = "confirm";
+      question.default = property.default || false;
+    } else if (property.enum) {
+      question._type = "list";
+      question.choices = property.enum;
+      question.default = property.default;
+    } else if (property._type === "number") {
+      question._type = "number";
+      question.default = property.default;
+    } else {
+      question._type = "input";
+      question.default = property.default || "";
+    }
+
+    const answer = await inquirer.prompt([question]);
+    if (answer[key] !== "" && answer[key] !== undefined) {
+      _config[key] = answer[key];
+    }
+  }
+
+  return _config;
+}
+
+module.exports = {
+  selectPlugins,
+  selectPluginForType,
+  configureCustomPlugin,
+  configureSelectedPlugin,
+  configurePluginSettings,
+};