@devilsdev/rag-pipeline-utils 2.4.5 → 2.5.0

package/src/cli/commands/ai.js

@@ -0,0 +1,101 @@
+"use strict";
+
+/**
+ * `rag-pipeline ai ...`: training, benchmarking, relevance feedback and file
+ * inspection built on the real pipeline components (.ragrc.json + index).
+ */
+
+const { Command, Option } = require("commander");
+const { parsePositiveInteger } = require("../shared/cli-utils.js");
+const {
+  handleTrainEmbedding,
+  handleBenchmark,
+  handleFeedback,
+  handleSearch,
+  handleMultimodalProcess,
+} = require("../handlers/ai-handler.js");
+
+/**
+ * @param {() => object} getGlobalOptions - Returns program.opts()
+ * @returns {Command}
+ */
+function createAiCommand(getGlobalOptions) {
+  const ai = new Command("ai").description(
+    "Train the local embedder, benchmark queries, relevance feedback, file inspection",
+  );
+
+  const train = ai
+    .command("train")
+    .description("Fit corpus-trained components");
+  train
+    .command("embedding")
+    .description(
+      "Refit the local TF-IDF embedder (on the index, or on [path]) and re-embed the index",
+    )
+    .argument(
+      "[path]",
+      "Train on documents from this file or directory instead",
+    )
+    .action(async (inputPath) => {
+      await handleTrainEmbedding(getGlobalOptions(), inputPath);
+    });
+
+  ai.command("benchmark")
+    .description(
+      "Measure query latency against the configured pipeline and index",
+    )
+    .option(
+      "--queries <file>",
+      "Queries file: JSON array of strings or one per line (default: sampled from the index)",
+    )
+    .option("--runs <n>", "Passes over the query set", parsePositiveInteger, 3)
+    .option("--top-k <n>", "Passages to retrieve", parsePositiveInteger, 3)
+    .option("--json", "Print the measurements as JSON")
+    .action(async (options) => {
+      await handleBenchmark(getGlobalOptions(), options);
+    });
+
+  ai.command("feedback")
+    .description(
+      "Record whether an indexed chunk was a good result for a question",
+    )
+    .argument("<question>", "Question the chunk was returned for")
+    .argument("<docId>", "Chunk id as shown by `ai search` (e.g. docs/a.md#0)")
+    .addOption(
+      new Option("--rating <rating>", "Relevance vote")
+        .choices(["up", "down"])
+        .makeOptionMandatory(),
+    )
+    .action(async (question, docId, options) => {
+      await handleFeedback(getGlobalOptions(), question, docId, options);
+    });
+
+  ai.command("search")
+    .description(
+      "Retrieve passages re-scored with stored feedback (retrieval only)",
+    )
+    .argument("<question>", "Question text")
+    .option("--top-k <n>", "Passages to return", parsePositiveInteger, 3)
+    .option("--json", "Print the results as JSON")
+    .action(async (question, options) => {
+      await handleSearch(getGlobalOptions(), question, options);
+    });
+
+  const multimodal = ai
+    .command("multimodal")
+    .description("Inspect files before ingesting");
+  multimodal
+    .command("process")
+    .description(
+      "Report type, characters and chunks for text files (images need a vision model)",
+    )
+    .argument("<path>", "File or directory")
+    .option("--json", "Print the report as JSON")
+    .action(async (inputPath, options) => {
+      await handleMultimodalProcess(getGlobalOptions(), inputPath, options);
+    });
+
+  return ai;
+}
+
+module.exports = { createAiCommand };

package/src/cli/commands/plugin-hub/event-handlers.js

@@ -6,6 +6,12 @@
 const chalk = require("chalk");
 
 function setupEventListeners(hub, certification) {
+  // The hub emits 'error' and then throws a descriptive error that each
+  // command reports. Without a listener, EventEmitter would throw its own
+  // "Unhandled error" first and hide that message.
+  hub.on("error", () => {});
+  certification.on("error", () => {});
+
   // Hub events
   hub.on("install_start", (data) => {
     console.log(

package/src/cli/commands/plugin-hub/hub-commands.js

@@ -69,34 +69,6 @@ async function searchPlugins(hub, query, _options) {
   }
 }
 
-async function installPlugin(hub, plugin, _options) {
-  try {
-    const installOptions = {
-      securityScan: _options.securityScan,
-      requireCertified: _options.requireCertified,
-      sandboxTimeout: _options.sandboxTimeout,
-    };
-
-    const result = await hub.installPlugin(
-      plugin,
-      _options.version,
-      installOptions,
-    );
-
-    console.log(chalk.green("\n✅ Installation completed successfully!")); // eslint-disable-line no-console
-    console.log(chalk.blue(`Plugin: ${result.pluginInfo.name}`)); // eslint-disable-line no-console
-    console.log(chalk.blue(`Version: ${result.version}`)); // eslint-disable-line no-console
-    console.log(chalk.blue(`Install Path: ${result.installPath}`)); // eslint-disable-line no-console
-
-    if (result.pluginInfo.certified) {
-      console.log(chalk.green("🏆 This plugin is certified!")); // eslint-disable-line no-console
-    }
-  } catch (error) {
-    console.error(chalk.red(`Installation failed: ${error.message}`)); // eslint-disable-line no-console
-    process.exit(1);
-  }
-}
-
 async function getPluginInfo(hub, plugin) {
   const spinner = ora("Fetching plugin information...").start();
 
@@ -170,50 +142,6 @@ async function getPluginInfo(hub, plugin) {
   }
 }
 
-async function listInstalledPlugins(hub, _options) {
-  const spinner = ora("Loading installed plugins...").start();
-
-  try {
-    const installed = await hub.getInstalledPlugins();
-    spinner.stop();
-
-    if (installed.length === 0) {
-      console.log(chalk.yellow("No plugins installed.")); // eslint-disable-line no-console
-      return;
-    }
-
-    if (_options.format === "json") {
-      console.log(JSON.stringify(installed, null, 2)); // eslint-disable-line no-console
-      return;
-    }
-
-    console.log(chalk.green(`${installed.length} plugins installed:\n`)); // eslint-disable-line no-console
-
-    const table = new Table({
-      head: ["Name", "Version", "Type", "Last Used", "Status"],
-      colWidths: [25, 12, 15, 15, 10],
-    });
-
-    for (const plugin of installed) {
-      const lastUsed = plugin.lastUsed
-        ? new Date(plugin.lastUsed).toLocaleDateString()
-        : chalk.gray("Never");
-
-      const status = plugin.certified
-        ? chalk.green("Certified")
-        : chalk.gray("Standard");
-
-      table.push([plugin.name, plugin.version, plugin._type, lastUsed, status]);
-    }
-
-    console.log(table.toString()); // eslint-disable-line no-console
-  } catch (error) {
-    spinner.stop();
-    console.error(chalk.red(`Failed to list plugins: ${error.message}`)); // eslint-disable-line no-console
-    process.exit(1);
-  }
-}
-
 async function publishPlugin(hub, pluginPath, _options) {
   if (_options.dryRun) {
     console.log(
@@ -282,9 +210,7 @@ async function getTrendingPlugins(hub, _options) {
 
 module.exports = {
   searchPlugins,
-  installPlugin,
   getPluginInfo,
-  listInstalledPlugins,
   publishPlugin,
   getTrendingPlugins,
 };

package/src/cli/commands/plugin-hub/local-commands.js

@@ -0,0 +1,143 @@
+/**
+ * `hub install` and `hub certify submit|check`: real plugin installation and
+ * automated certification checks (see src/ecosystem/plugin-installer.js and
+ * plugin-checks.js).
+ */
+
+const fs = require("fs/promises");
+const { resolveConfigPath } = require("../../../config/load-config.js");
+
+const ICONS = { high: "❌", medium: "⚠️ ", low: "ℹ️ " };
+
+function printFindings(findings) {
+  for (const f of findings) {
+    const where = f.line ? `${f.file}:${f.line}` : f.file;
+    console.log(
+      `  ${ICONS[f.severity] || "-"} ${f.severity}: ${f.message} (${where})`,
+    );
+  }
+}
+
+/**
+ * @param {string} source - Registry id, path, .tgz, .js or npm:<spec>
+ * @param {object} options - { stage, name, force, allowRisky, version }
+ * @param {object} globals - Includes the root --config option
+ * @param {Function} getHub - Returns a PluginHub, or null when no registry is set
+ */
+async function installFromSource(source, options, globals, getHub) {
+  try {
+    const { installPlugin } = require("../../../ecosystem/plugin-installer.js");
+    const result = await installPlugin({
+      source,
+      configPath: resolveConfigPath(globals.config || ".ragrc.json"),
+      stage: options.stage,
+      name: options.name,
+      force: options.force,
+      allowRisky: options.allowRisky,
+      version: options.version,
+      hub: getHub(),
+      onProgress: (step) => console.log(`… ${step}`),
+    });
+
+    if (result.scan.findings.length > 0) {
+      console.log("Security scan findings:");
+      printFindings(result.scan.findings);
+    }
+    console.log(
+      `✅ Installed ${result.stage} plugin "${result.name}"` +
+        `${result.version ? ` ${result.version}` : ""} (${result.source}) at ${result.entry}`,
+    );
+    console.log(`   Methods: ${result.methods.join(", ")}`);
+    if (!result.inPipeline) {
+      console.log(
+        `   To use it, add { "stage": "${result.stage}", "name": "${result.name}" } to "pipeline" in your config.`,
+      );
+    }
+  } catch (error) {
+    console.error(`❌ Install failed: ${error.message}`);
+    if (error.scan) printFindings(error.scan.findings);
+    process.exitCode = 1;
+  }
+}
+
+/**
+ * @param {string} pluginPath - Local plugin directory or .js file
+ * @param {object} options - { stage, json, output }
+ */
+async function certifyLocal(pluginPath, options) {
+  try {
+    const { runPluginChecks } = require("../../../ecosystem/plugin-checks.js");
+    const report = await runPluginChecks(pluginPath, { stage: options.stage });
+
+    if (options.output) {
+      await fs.writeFile(options.output, JSON.stringify(report, null, 2));
+    }
+    if (options.json) {
+      console.log(JSON.stringify(report, null, 2));
+    } else {
+      console.log(`Certification checks: ${report.plugin} (${report.stage})\n`);
+      for (const check of report.checks) {
+        console.log(
+          `${check.label}: ${check.score}/100 (weight ${check.weight})`,
+        );
+        check.details.forEach((d) => console.log(`  - ${d}`));
+      }
+      console.log(`\nScore: ${report.score}/100`);
+      if (report.level) {
+        console.log(
+          `Meets the automated threshold for ${report.level}` +
+            (report.pendingReview
+              ? ` (also requires ${report.pendingReview})`
+              : ""),
+        );
+      } else {
+        console.log(
+          "Does not meet the BASIC threshold (60, with a passing contract).",
+        );
+      }
+      if (report.nextLevel) {
+        console.log(
+          `Next level: ${report.nextLevel.level} at ${report.nextLevel.score}`,
+        );
+      }
+      if (options.output) console.log(`Report saved to ${options.output}`);
+    }
+    if (!report.level) process.exitCode = 1;
+  } catch (error) {
+    console.error(`❌ Certification checks failed: ${error.message}`);
+    process.exitCode = 1;
+  }
+}
+
+/**
+ * @param {object} globals - Includes the root --config option
+ * @param {object} options - { json }
+ */
+function listPlugins(globals, options) {
+  const {
+    listInstalledPlugins,
+  } = require("../../../ecosystem/plugin-installer.js");
+  const plugins = listInstalledPlugins(
+    resolveConfigPath(globals.config || ".ragrc.json"),
+  );
+  if (options.json) {
+    console.log(JSON.stringify(plugins, null, 2));
+    return;
+  }
+  if (plugins.length === 0) {
+    console.log('No plugins registered under "plugins" in the config.');
+    return;
+  }
+  for (const p of plugins) {
+    const flags = [
+      p.installed
+        ? `installed${p.version ? ` ${p.version}` : ""}`
+        : "local path",
+      p.inPipeline ? "in pipeline" : "not in pipeline",
+      p.exists ? null : "FILE MISSING",
+    ].filter(Boolean);
+    console.log(`${p.stage}/${p.name}  ${p.entry}  (${flags.join(", ")})`);
+  }
+}
+
+module.exports = { installFromSource, certifyLocal, listPlugins };

package/src/cli/commands/plugin-hub.js

@@ -11,17 +11,80 @@ const hubCommands = require("./plugin-hub/hub-commands");
 const certificationCommands = require("./plugin-hub/certification-commands");
 const reviewCommands = require("./plugin-hub/review-commands");
 const { setupEventListeners } = require("./plugin-hub/event-handlers");
+const {
+  installFromSource,
+  certifyLocal,
+  listPlugins,
+} = require("./plugin-hub/local-commands");
 
 class PluginHubCLI {
-  constructor() {
-    this.hub = new PluginHub();
-    this.certification = new PluginCertification();
-    setupEventListeners(this.hub, this.certification);
+  /**
+   * @param {object} [options]
+   * @param {string} [options.registryUrl] - Overrides --registry / RAG_PLUGIN_REGISTRY
+   */
+  constructor(options = {}) {
+    this.registryUrl = options.registryUrl;
+  }
+
+  /**
+   * There is no public registry: the old default (registry.rag-pipeline.dev)
+   * does not resolve. Commands that talk to a registry need one configured.
+   */
+  _resolveRegistryUrl() {
+    const url = this.registryUrl || process.env.RAG_PLUGIN_REGISTRY;
+    if (!url) {
+      throw new Error(
+        "No plugin registry configured. Pass --registry <url> or set RAG_PLUGIN_REGISTRY.",
+      );
+    }
+    return url;
+  }
+
+  _init() {
+    if (!this._hub) {
+      const registryUrl = this._resolveRegistryUrl();
+      this._hub = new PluginHub({ registryUrl });
+      this._certification = new PluginCertification({ registryUrl });
+      setupEventListeners(this._hub, this._certification);
+    }
+  }
+
+  get hub() {
+    this._init();
+    return this._hub;
+  }
+
+  get certification() {
+    this._init();
+    return this._certification;
   }
 
   createCommands() {
     const program = new Command("hub");
-    program.description("Community plugin hub operations");
+    program
+      .alias("plugin")
+      .description("Plugin registry operations (requires --registry <url>)")
+      .option(
+        "--registry <url>",
+        "Plugin registry URL (or RAG_PLUGIN_REGISTRY)",
+      )
+      .hook("preAction", (_hubCommand, actionCommand) => {
+        // Local commands: requirements and submit never need a registry; install
+        // needs one only for registry ids and checks that itself.
+        if (
+          ["requirements", "install", "submit", "list"].includes(
+            actionCommand.name(),
+          )
+        )
+          return;
+        this.registryUrl = this.registryUrl || program.opts().registry;
+        try {
+          this._resolveRegistryUrl();
+        } catch (error) {
+          console.error(`❌ ${error.message}`);
+          process.exit(1);
+        }
+      });
 
     // Search command
     program
@@ -42,21 +105,22 @@ class PluginHubCLI {
         await hubCommands.searchPlugins(this.hub, query, _options);
       });
 
-    // Install command
     program
-      .command("install <plugin>")
-      .description("Install a plugin from the hub")
-      .option("-v, --version <version>", "Plugin version", "latest")
-      .option("--no-security-scan", "Skip security scan")
-      .option("--require-certified", "Only install certified plugins")
-      .option(
-        "--sandbox-timeout <timeout>",
-        "Sandbox timeout in ms",
-        parseInt,
-        30000,
+      .command("install <source>")
+      .description(
+        "Install a plugin from a local path, .tgz, .js, npm:<package> or the registry",
       )
-      .action(async (plugin, _options) => {
-        await hubCommands.installPlugin(this.hub, plugin, _options);
+      .option("--stage <stage>", "loader, embedder, retriever, llm or reranker")
+      .option("--name <name>", "Name to register the plugin under")
+      .option("-v, --version <version>", "Registry version", "latest")
+      .option("--force", "Replace an installed plugin with the same stage/name")
+      .option("--allow-risky", "Install despite high-risk security findings")
+      .action(async (source, _options, command) => {
+        const globals = command.optsWithGlobals();
+        this.registryUrl = this.registryUrl || globals.registry;
+        const getHub = () =>
+          this.registryUrl || process.env.RAG_PLUGIN_REGISTRY ? this.hub : null;
+        await installFromSource(source, _options, globals, getHub);
       });
 
     // Info command
@@ -71,10 +135,10 @@ class PluginHubCLI {
     program
       .command("list")
       .alias("ls")
-      .description("List installed plugins")
-      .option("--format <format>", "Output format: table, json", "table")
-      .action(async (_options) => {
-        await hubCommands.listInstalledPlugins(this.hub, _options);
+      .description("List plugins registered in the config")
+      .option("--json", "Print as JSON")
+      .action(async (_options, command) => {
+        listPlugins(command.optsWithGlobals(), _options);
       });
 
     // Publish command
@@ -132,19 +196,14 @@ class PluginHubCLI {
       .description("Plugin certification operations");
 
     certifyCmd
-      .command("submit <plugin>")
-      .description("Submit plugin for certification")
-      .option(
-        "-l, --level <level>",
-        "Certification level: BASIC, VERIFIED, ENTERPRISE",
-        "BASIC",
-      )
-      .action(async (plugin, _options) => {
-        await certificationCommands.submitForCertification(
-          this.certification,
-          plugin,
-          _options,
-        );
+      .command("submit <path>")
+      .alias("check")
+      .description("Run automated certification checks on a local plugin")
+      .option("--stage <stage>", "loader, embedder, retriever, llm or reranker")
+      .option("--json", "Print the report as JSON")
+      .option("--output <file>", "Save the report as JSON")
+      .action(async (pluginPath, _options) => {
+        await certifyLocal(pluginPath, _options);
       });
 
     certifyCmd
@@ -162,8 +221,9 @@ class PluginHubCLI {
       .command("requirements [level]")
       .description("Show certification requirements")
       .action(async (level) => {
+        // Requirements are a static list; no registry is contacted.
         await certificationCommands.showCertificationRequirements(
-          this.certification,
+          new PluginCertification(),
           level,
         );
       });

package/src/cli/diagnostics/config-checker.js

@@ -1,78 +1,91 @@
 /**
- * Configuration checker for RAG Pipeline Diagnostics
- * Checks configuration file existence, validity, and schema compliance
+ * Configuration diagnostics: the file exists, parses, validates, and every
+ * pipeline entry resolves to a component.
  */
 
-const fs = require("fs/promises");
+const fs = require("fs");
+const path = require("path");
+
+function issue(severity, code, message, fix) {
+  return {
+    category: "configuration",
+    severity,
+    code,
+    message,
+    fix,
+    autoFixable: false,
+  };
+}
 
-/**
- * Check configuration issues
- * @param {object} options - Doctor options (must include configPath)
- * @returns {Promise<Array>} Array of configuration issues
- */
 async function checkConfiguration(options) {
-  const issues = [];
+  const { loadRagConfig } = require("../../config/load-config.js");
+  const { buildComponents } = require("../../config/component-factory.js");
+  const configPath = path.resolve(options.configPath);
 
-  try {
-    await fs.access(options.configPath);
-  } catch (error) {
-    issues.push({
-      category: "configuration",
-      severity: "error",
-      code: "CONFIG_MISSING",
-      message: `Configuration file not found: ${options.configPath}`,
-      fix: 'Run "rag-pipeline init" to create a configuration file',
-      autoFixable: false,
-    });
-    return issues;
+  if (!fs.existsSync(configPath)) {
+    return [
+      issue(
+        "error",
+        "CONFIG_MISSING",
+        `Configuration file not found: ${options.configPath}`,
+        'Run "rag-pipeline init" to create a configuration file',
+      ),
+    ];
   }
 
+  let config;
   try {
-    const configContent = await fs.readFile(options.configPath, "utf8");
-    const config = JSON.parse(configContent);
+    config = loadRagConfig(configPath);
+  } catch (error) {
+    const invalidJson = /Invalid JSON/.test(error.message);
+    return [
+      issue(
+        "error",
+        invalidJson ? "CONFIG_INVALID_JSON" : "CONFIG_SCHEMA_ERROR",
+        error.message,
+        invalidJson
+          ? `Fix JSON syntax errors in ${options.configPath}`
+          : "Fix the reported configuration errors",
+      ),
+    ];
+  }
 
-    // Check if config is empty
-    if (Object.keys(config).length === 0 || !config.plugins) {
-      issues.push({
-        category: "configuration",
-        severity: "warning",
-        code: "CONFIG_EMPTY",
-        message: "Configuration file is empty or missing required sections",
-        fix: "Add plugin configurations and pipeline settings",
-        autoFixable: false,
-      });
+  const issues = [];
+  const stages = new Set(config.pipeline.map((entry) => entry.stage));
+  for (const stage of ["embedder", "retriever"]) {
+    if (!stages.has(stage)) {
+      issues.push(
+        issue(
+          "error",
+          "CONFIG_STAGE_MISSING",
+          `Pipeline has no ${stage} stage`,
+          `Add { "stage": "${stage}", "name": ... } to the pipeline (see rag-pipeline init)`,
+        ),
+      );
     }
+  }
+  if (!stages.has("llm")) {
+    issues.push(
+      issue(
+        "info",
+        "CONFIG_NO_LLM",
+        "Pipeline has no llm stage; query prints retrieved passages only",
+        'Add { "stage": "llm", "name": "openai" } (or anthropic, ollama) to generate answers',
+      ),
+    );
+  }
 
-    // Validate schema if available
-    try {
-      const {
-        validateEnhancedRagrcSchema,
-      } = require("../../config/enhanced-ragrc-schema.js");
-      const validation = validateEnhancedRagrcSchema(config);
-      if (!validation.valid && validation.errors) {
-        validation.errors.forEach((error) => {
-          issues.push({
-            category: "configuration",
-            severity: "error",
-            code: "CONFIG_SCHEMA_ERROR",
-            message: `Schema validation failed: ${error.instancePath} ${error.message}`,
-            fix: "Fix configuration schema errors",
-            autoFixable: false,
-          });
-        });
-      }
-    } catch (schemaError) {
-      // Schema validation not available, skip
-    }
-  } catch (parseError) {
-    issues.push({
-      category: "configuration",
-      severity: "error",
-      code: "CONFIG_INVALID_JSON",
-      message: "Configuration file contains invalid JSON syntax",
-      fix: `Fix JSON syntax errors in ${options.configPath}`,
-      autoFixable: false,
-    });
+  try {
+    buildComponents(config, { configDir: path.dirname(configPath) });
+  } catch (error) {
+    issues.push(
+      issue(
+        "error",
+        "CONFIG_COMPONENT_UNKNOWN",
+        error.message,
+        'Use a built-in name or register the plugin under "plugins"',
+      ),
+    );
   }
 
   return issues;