@devilsdev/rag-pipeline-utils 2.4.5 → 2.6.0

package/src/cli/wizard/plugin-selection.js

@@ -1,297 +0,0 @@
-/**
- * Plugin Selection Step
- * Handles plugin selection, custom plugin configuration, and plugin settings
- */
-
-/**
- * Select plugins for each type
- * @param {object} config - Current wizard configuration state
- * @param {object} inquirer - Inquirer instance for prompting
- * @param {object} context - Wizard context with registry and helper methods
- * @param {object|null} context.registry - Plugin registry data
- * @param {Function} context.getAvailablePlugins - Function to get available plugins for a type
- * @param {Function} context.getPluginInfo - Function to get plugin info
- * @param {Function} context.loadRegistry - Function to load plugin registry
- * @returns {Promise<object>} Updated configuration state
- */
-async function selectPlugins(config, inquirer, context) {
-  console.log("🔌 Plugin Selection\n");
-  // eslint-disable-line no-console
-
-  // Load registry for plugin suggestions
-  try {
-    context.registry = await context.loadRegistry();
-  } catch (error) {
-    console.warn(
-      "⚠️  Could not load plugin registry. Using built-in _options.",
-    );
-    // eslint-disable-line no-console
-  }
-
-  const pluginTypes = [
-    { key: "loader", name: "Document Loader", required: true },
-    { key: "embedder", name: "Text Embedder", required: true },
-    { key: "retriever", name: "Vector Retriever", required: true },
-    { key: "llm", name: "Language Model", required: true },
-    { key: "reranker", name: "Result Reranker", required: false },
-  ];
-
-  for (const pluginType of pluginTypes) {
-    await selectPluginForType(config, inquirer, context, pluginType);
-  }
-
-  console.log("✅ Plugin selection complete\n");
-  // eslint-disable-line no-console
-
-  return config;
-}
-
-/**
- * Select plugin for a specific type
- * @param {object} config - Current wizard configuration state
- * @param {object} inquirer - Inquirer instance for prompting
- * @param {object} context - Wizard context with helper methods
- * @param {object} pluginType - Plugin type configuration
- * @returns {Promise<void>}
- */
-async function selectPluginForType(config, inquirer, context, pluginType) {
-  const availablePlugins = context.getAvailablePlugins(pluginType.key);
-
-  if (availablePlugins.length === 0 && pluginType.required) {
-    console.log(
-      `⚠️  No ${pluginType.name} plugins available. You'll need to configure this manually.`,
-    );
-    // eslint-disable-line no-console
-    return;
-  }
-
-  const choices = [
-    ...availablePlugins.map((plugin) => ({
-      name: `${plugin.name} - ${plugin.description}`,
-      value: plugin.name,
-      short: plugin.name,
-    })),
-    { name: "Skip (configure later)", value: null },
-    { name: "Custom plugin", value: "custom" },
-  ];
-
-  const answer = await inquirer.prompt([
-    {
-      _type: "list",
-      name: "plugin",
-      message: `Select ${pluginType.name}:`,
-      choices,
-      when: () => !pluginType.required || availablePlugins.length > 0,
-    },
-  ]);
-
-  if (!answer.plugin) {
-    return; // Skip
-  }
-
-  if (answer.plugin === "custom") {
-    await configureCustomPlugin(config, inquirer, pluginType.key);
-  } else {
-    await configureSelectedPlugin(
-      config,
-      inquirer,
-      context,
-      pluginType.key,
-      answer.plugin,
-    );
-  }
-}
-
-/**
- * Configure a custom plugin
- * @param {object} config - Current wizard configuration state
- * @param {object} inquirer - Inquirer instance for prompting
- * @param {string} pluginType - Plugin type key
- * @returns {Promise<void>}
- */
-async function configureCustomPlugin(config, inquirer, pluginType) {
-  const answers = await inquirer.prompt([
-    {
-      _type: "input",
-      name: "name",
-      message: "Plugin name:",
-      validate: (input) => input.length > 0 || "Plugin name is required",
-    },
-    {
-      _type: "list",
-      name: "source",
-      message: "Plugin source:",
-      choices: [
-        { name: "Registry", value: "registry" },
-        { name: "Local file", value: "local" },
-        { name: "Git repository", value: "git" },
-        { name: "NPM package", value: "npm" },
-      ],
-      default: "registry",
-    },
-    {
-      _type: "input",
-      name: "version",
-      message: "Version:",
-      default: "latest",
-      when: (answers) =>
-        answers.source === "registry" || answers.source === "npm",
-    },
-    {
-      _type: "input",
-      name: "path",
-      message: "Local path:",
-      when: (answers) => answers.source === "local",
-    },
-    {
-      _type: "input",
-      name: "url",
-      message: "Repository URL:",
-      when: (answers) => answers.source === "git",
-    },
-  ]);
-
-  const pluginSpec = {
-    name: answers.name,
-    source: answers.source,
-  };
-
-  if (answers.version) pluginSpec.version = answers.version;
-  if (answers.path) pluginSpec.path = answers.path;
-  if (answers.url) pluginSpec.url = answers.url;
-
-  config.plugins[pluginType][answers.name] = pluginSpec;
-}
-
-/**
- * Configure a selected plugin from registry
- * @param {object} config - Current wizard configuration state
- * @param {object} inquirer - Inquirer instance for prompting
- * @param {object} context - Wizard context with helper methods
- * @param {string} pluginType - Plugin type key
- * @param {string} pluginName - Plugin name
- * @returns {Promise<void>}
- */
-async function configureSelectedPlugin(
-  config,
-  inquirer,
-  context,
-  pluginType,
-  pluginName,
-) {
-  const plugin = context.getPluginInfo(pluginType, pluginName);
-
-  if (!plugin) {
-    config.plugins[pluginType][pluginName] = "latest";
-    return;
-  }
-
-  const versions = Object.keys(plugin.versions || {}).slice(0, 5);
-  const versionChoices = [
-    { name: "Latest stable", value: "latest" },
-    { name: "Latest beta", value: "beta" },
-    ...versions.map((v) => ({ name: v, value: v })),
-    { name: "Custom version", value: "custom" },
-  ];
-
-  const versionAnswer = await inquirer.prompt([
-    {
-      _type: "list",
-      name: "version",
-      message: `Select version for ${pluginName}:`,
-      choices: versionChoices,
-      default: "latest",
-    },
-    {
-      _type: "input",
-      name: "customVersion",
-      message: "Enter version:",
-      when: (answers) => answers.version === "custom",
-    },
-  ]);
-
-  const version = versionAnswer.customVersion || versionAnswer.version;
-
-  // Check if plugin needs configuration
-  const needsConfig =
-    plugin.metadata?._config &&
-    Object.keys(plugin.metadata._config.properties || {}).length > 0;
-
-  if (needsConfig) {
-    const configAnswer = await inquirer.prompt([
-      {
-        _type: "confirm",
-        name: "configure",
-        message: `Configure ${pluginName} settings?`,
-        default: false,
-      },
-    ]);
-
-    if (configAnswer.configure) {
-      const pluginConfig = await configurePluginSettings(plugin, inquirer);
-      config.plugins[pluginType][pluginName] = {
-        name: pluginName,
-        version,
-        _config: pluginConfig,
-      };
-    } else {
-      config.plugins[pluginType][pluginName] = {
-        name: pluginName,
-        version,
-      };
-    }
-  } else {
-    config.plugins[pluginType][pluginName] = version;
-  }
-}
-
-/**
- * Configure plugin-specific settings
- * @param {object} plugin - Plugin information
- * @param {object} inquirer - Inquirer instance for prompting
- * @returns {Promise<object>} Plugin configuration
- */
-async function configurePluginSettings(plugin, inquirer) {
-  const _config = {};
-  const schema = plugin.metadata?._config?.properties || {};
-
-  console.log(`\n⚙️  Configuring ${plugin.metadata.name}...\n`);
-  // eslint-disable-line no-console
-
-  for (const [key, property] of Object.entries(schema)) {
-    const question = {
-      name: key,
-      message: `${key}${property.description ? ` (${property.description})` : ""}:`,
-    };
-
-    if (property._type === "boolean") {
-      question._type = "confirm";
-      question.default = property.default || false;
-    } else if (property.enum) {
-      question._type = "list";
-      question.choices = property.enum;
-      question.default = property.default;
-    } else if (property._type === "number") {
-      question._type = "number";
-      question.default = property.default;
-    } else {
-      question._type = "input";
-      question.default = property.default || "";
-    }
-
-    const answer = await inquirer.prompt([question]);
-    if (answer[key] !== "" && answer[key] !== undefined) {
-      _config[key] = answer[key];
-    }
-  }
-
-  return _config;
-}
-
-module.exports = {
-  selectPlugins,
-  selectPluginForType,
-  configureCustomPlugin,
-  configureSelectedPlugin,
-  configurePluginSettings,
-};

package/src/cli/wizard/preview-save.js

@@ -1,172 +0,0 @@
-/**
- * Preview and Save Step
- * Handles configuration preview, validation, saving, and testing
- */
-
-const fs = require("fs/promises");
-const {
-  validateRagrc,
-  extractPluginDependencies,
-} = require("../../config/enhanced-ragrc-schema.js");
-
-/**
- * Preview configuration and save
- * @param {object} config - Current wizard configuration state
- * @param {object} inquirer - Inquirer instance for prompting
- * @param {object} options - Wizard options (outputPath, etc.)
- * @returns {Promise<object>} Updated configuration state (possibly normalized)
- */
-async function previewAndSave(config, inquirer, options) {
-  console.log("👀 Configuration Preview\n");
-  // eslint-disable-line no-console
-
-  // Show configuration summary
-  showConfigSummary(config);
-
-  const answers = await inquirer.prompt([
-    {
-      _type: "confirm",
-      name: "save",
-      message: "Save this configuration?",
-      default: true,
-    },
-    {
-      _type: "input",
-      name: "filename",
-      message: "Configuration filename:",
-      default: options.outputPath,
-      when: (answers) => answers.save,
-    },
-    {
-      _type: "confirm",
-      name: "testRun",
-      message: "Run a test to validate the configuration?",
-      default: false,
-      when: (answers) => answers.save,
-    },
-  ]);
-
-  if (!answers.save) {
-    console.log("Configuration not saved."); // eslint-disable-line no-console
-    return config;
-  }
-
-  // Validate and normalize configuration
-  const validation = validateRagrc(config);
-  if (!validation.valid) {
-    console.error("❌ Configuration validation failed:"); // eslint-disable-line no-console
-    validation.errors?.forEach((error) => {
-      console.error(`  ${error.instancePath}: ${error.message}`); // eslint-disable-line no-console
-    });
-    throw new Error("Invalid configuration generated");
-  }
-
-  // Use normalized configuration
-  const normalizedConfig = validation.normalized;
-
-  // Save configuration
-  await fs.writeFile(
-    answers.filename,
-    JSON.stringify(normalizedConfig, null, 2),
-    "utf-8",
-  );
-
-  options.outputPath = answers.filename;
-
-  if (answers.testRun) {
-    await runConfigurationTest(normalizedConfig);
-  }
-
-  return normalizedConfig;
-}
-
-/**
- * Show configuration summary
- * @param {object} config - Configuration to summarize
- */
-function showConfigSummary(config) {
-  console.log("📋 Configuration Summary:");
-  // eslint-disable-line no-console
-  console.log(`   Project: ${config.metadata?.name || "Unnamed"}`);
-  // eslint-disable-line no-console
-  console.log(
-    `   Environment: ${config.metadata?.environment || "development"}`,
-  );
-  // eslint-disable-line no-console
-
-  console.log("\n🔌 Plugins:");
-  // eslint-disable-line no-console
-  for (const [_type, plugins] of Object.entries(config.plugins)) {
-    if (Object.keys(plugins).length > 0) {
-      console.log(`   ${_type}: ${Object.keys(plugins).join(", ")}`);
-      // eslint-disable-line no-console
-    }
-  }
-
-  console.log("\n🔄 Pipeline:");
-  // eslint-disable-line no-console
-  console.log(
-    `   Stages: ${config.pipeline?.stages?.join(" → ") || "default"}`,
-  );
-  // eslint-disable-line no-console
-
-  if (config.performance?.parallel?.enabled) {
-    console.log("\n⚡ Performance:");
-    // eslint-disable-line no-console
-    console.log(
-      `   Parallel processing: ${config.performance.parallel.maxConcurrency} concurrent`,
-    );
-    // eslint-disable-line no-console
-  }
-
-  if (
-    config.observability?.tracing?.enabled ||
-    config.observability?.metrics?.enabled
-  ) {
-    console.log("\n📊 Observability:");
-    // eslint-disable-line no-console
-    if (config.observability.tracing?.enabled)
-      console.log("   Tracing: enabled");
-    // eslint-disable-line no-console
-    if (config.observability.metrics?.enabled)
-      console.log("   Metrics: enabled");
-    // eslint-disable-line no-console
-  }
-
-  console.log("");
-  // eslint-disable-line no-console
-}
-
-/**
- * Run configuration test
- * @param {object} config - Configuration to test
- */
-async function runConfigurationTest(config) {
-  console.log("🧪 Running configuration test...\n");
-  // eslint-disable-line no-console
-
-  try {
-    // Simulate pipeline creation and basic validation
-    console.log("✅ Configuration syntax: valid");
-    // eslint-disable-line no-console
-    console.log("✅ Plugin dependencies: resolved");
-    // eslint-disable-line no-console
-    console.log("✅ Pipeline stages: configured");
-    // eslint-disable-line no-console
-
-    // Check for potential issues
-    const dependencies = extractPluginDependencies(config);
-    if (dependencies.length === 0) {
-      console.warn("⚠️  No plugins configured - pipeline will not function");
-      // eslint-disable-line no-console
-    }
-
-    console.log("\n🎉 Configuration test passed!");
-    // eslint-disable-line no-console
-  } catch (error) {
-    console.error("❌ Configuration test failed:", error.message);
-    // eslint-disable-line no-console
-  }
-}
-
-module.exports = { previewAndSave, showConfigSummary, runConfigurationTest };

package/src/cli/wizard/project-setup.js

@@ -1,66 +0,0 @@
-/**
- * Project Setup Step
- * Handles project metadata configuration in the interactive wizard
- */
-
-const path = require("path");
-
-/**
- * Run the project setup step
- * @param {object} config - Current wizard configuration state
- * @param {object} inquirer - Inquirer instance for prompting
- * @returns {Promise<object>} Updated configuration state
- */
-async function setupProject(config, inquirer) {
-  console.log("📋 Project Setup\n");
-  // eslint-disable-line no-console
-
-  const answers = await inquirer.prompt([
-    {
-      _type: "input",
-      name: "name",
-      message: "Project name:",
-      default: path.basename(process.cwd()),
-      validate: (input) => input.length > 0 || "Project name is required",
-    },
-    {
-      _type: "input",
-      name: "description",
-      message: "Project description:",
-      default: "A RAG pipeline project",
-    },
-    {
-      _type: "input",
-      name: "author",
-      message: "Author:",
-      default: "Unknown",
-    },
-    {
-      _type: "list",
-      name: "environment",
-      message: "Target environment:",
-      choices: [
-        { name: "Development", value: "development" },
-        { name: "Production", value: "production" },
-        { name: "Testing", value: "testing" },
-      ],
-      default: "development",
-    },
-  ]);
-
-  config.metadata = {
-    name: answers.name,
-    version: "1.0.0",
-    description: answers.description,
-    author: answers.author,
-    environment: answers.environment,
-    createdAt: new Date().toISOString(),
-  };
-
-  console.log(`\n✅ Project "${answers.name}" configured\n`);
-  // eslint-disable-line no-console
-
-  return config;
-}
-
-module.exports = { setupProject };

package/src/core/pipeline-factory.js

@@ -1,59 +0,0 @@
-'use strict';
-
-async function createPipeline({
-  loader,
-  embedder,
-  retriever,
-  reranker,
-  llm,
-  evaluator,
-} = {}) {
-  // Required components check
-  const required = ['loader', 'embedder', 'retriever', 'llm'];
-  const missing = required.filter(
-    (name) => !arguments[0] || !arguments[0][name],
-  );
-  if (missing.length > 0) {
-    throw new Error(`Required components missing: ${missing.join(', ')}`);
-  }
-
-  return {
-    async run(input, { signal } = {}) {
-      // Check if signal is already aborted
-      if (signal && signal.aborted) {
-        throw new Error('Aborted');
-      }
-
-      // Execute pipeline in order
-      const docs = await loader.load(input, { signal });
-      const embeddings = await embedder.embed(docs, { signal });
-      const candidates = await retriever.retrieve(embeddings, { signal });
-
-      let ranked = candidates;
-      if (reranker && typeof reranker.rerank === 'function') {
-        ranked = await reranker.rerank(input, candidates, { signal });
-      }
-
-      const answer = await llm.generate(input, ranked, { signal });
-
-      let evaluation = null;
-      if (evaluator && typeof evaluator.evaluate === 'function') {
-        evaluation = await evaluator.evaluate(input, answer, ranked);
-      }
-
-      return {
-        docs,
-        embeddings,
-        candidates,
-        ranked,
-        answer,
-        evaluation,
-      };
-    },
-  };
-}
-
-// CJS+ESM interop pattern
-module.exports = createPipeline;
-module.exports.createPipeline = createPipeline;
-module.exports.createRagPipeline = createPipeline; // Alias for backward compatibility