@devilsdev/rag-pipeline-utils 2.4.5 → 2.5.0

package/src/cli/handlers/ai-handler.js

@@ -0,0 +1,364 @@
+"use strict";
+
+/**
+ * Handlers for `rag-pipeline ai ...`. Everything printed here is computed from
+ * the configured pipeline, the index or the given files; failures set
+ * process.exitCode = 1 and explain why on stderr.
+ */
+
+const fs = require("fs");
+const path = require("path");
+const fg = require("fast-glob");
+
+const { loadCliConfig } = require("../shared/config-context.js");
+const {
+  retrain,
+  retrieve,
+  loadComponents,
+} = require("../services/rag-service.js");
+const { runBenchmark } = require("../services/benchmark.js");
+const {
+  feedbackPath,
+  loadFeedback,
+  addFeedback,
+  applyFeedback,
+} = require("../services/feedback.js");
+const { readDocuments } = require("../../loader/document-reader.js");
+
+function fail(prefix, error) {
+  console.error(`${prefix}: ${error.message}`);
+  process.exitCode = 1;
+}
+
+const fmt = (ms) => `${ms.toFixed(2)} ms`;
+const signed = (x) => `${x >= 0 ? "+" : "-"}${Math.abs(x).toFixed(3)}`;
+
+/**
+ * `ai train embedding [path]`
+ */
+async function handleTrainEmbedding(globalOptions, inputPath) {
+  try {
+    const { config, configDir } = loadCliConfig(globalOptions);
+    if (globalOptions.dryRun) {
+      console.log(
+        `Dry run: would fit the local embedder on ${inputPath || "the indexed chunks"} ` +
+          "and re-embed the index.",
+      );
+      return;
+    }
+    const r = await retrain({ config, configDir, inputPath });
+    const source =
+      r.trainedOn === "index"
+        ? `${r.trainingDocuments} indexed chunk(s)`
+        : `${r.trainingDocuments} chunk(s) read from ${r.trainedOn}`;
+    console.log(`Trained the local TF-IDF embedder on ${source}.`);
+    console.log(
+      `Vocabulary: ${r.vocabularySize} term(s)` +
+        (r.previousVocabularySize !== null
+          ? ` (previously ${r.previousVocabularySize})`
+          : ""),
+    );
+    console.log(
+      `Re-embedded ${r.reembedded} indexed chunk(s) with the new model.`,
+    );
+    if (r.zeroVectors > 0) {
+      console.log(
+        `Warning: ${r.zeroVectors} indexed chunk(s) share no term with the new ` +
+          "vocabulary and can no longer be retrieved.",
+      );
+    }
+    if (r.trainedOn !== "index") {
+      console.log(
+        "Note: the next `ingest` refits the embedder on the indexed chunks.",
+      );
+    }
+  } catch (error) {
+    fail("Training failed", error);
+  }
+}
+
+/**
+ * `ai benchmark`
+ */
+async function handleBenchmark(globalOptions, options = {}) {
+  try {
+    const { config, configDir } = loadCliConfig(globalOptions);
+    const report = await runBenchmark({
+      config,
+      configDir,
+      queriesFile: options.queries,
+      runs: options.runs || 3,
+      topK: options.topK || 3,
+    });
+
+    if (options.json) {
+      console.log(JSON.stringify(report, null, 2));
+      return;
+    }
+
+    const env = report.environment;
+    const n = report.queries.length;
+    console.log(
+      `Benchmark: ${n} quer${n === 1 ? "y" : "ies"} (${report.queriesSource}) ` +
+        `x ${report.runs} run(s), top-k ${report.topK}`,
+    );
+    console.log(
+      `Environment: Node ${env.node} (${env.platform}), ${env.chunks} indexed chunk(s), ` +
+        `embedder ${env.embedder}` +
+        (env.dimensions ? ` (${env.dimensions} dims)` : "") +
+        `, retriever ${env.retriever}, reranker ${env.reranker || "none"}, ` +
+        `llm ${env.llm || "none"}`,
+    );
+    console.log(`Setup (load config and index): ${fmt(report.setupMs)}`);
+    console.log(`Warm-up query (excluded): ${fmt(report.warmupMs)}`);
+    console.log("");
+    const labels = {
+      embed: "embed query",
+      retrieve: "retrieve",
+      rerank: "rerank",
+      endToEnd: report.includesGeneration
+        ? "end-to-end (incl. llm)"
+        : "end-to-end",
+    };
+    const header = ["stage", "samples", "p50", "p95", "mean", "max"];
+    const rows = Object.entries(report.stages).map(([stage, s]) => [
+      labels[stage],
+      String(s.count),
+      fmt(s.p50),
+      fmt(s.p95),
+      fmt(s.mean),
+      fmt(s.max),
+    ]);
+    const widths = header.map((h, i) =>
+      Math.max(h.length, ...rows.map((r) => r[i].length)),
+    );
+    const line = (cells) =>
+      cells
+        .map((c, i) => c.padEnd(widths[i]))
+        .join("  ")
+        .trimEnd();
+    console.log(line(header));
+    rows.forEach((r) => console.log(line(r)));
+    console.log("");
+    console.log(
+      `Results per query: ${report.results.min === report.results.max ? report.results.min : `${report.results.min}-${report.results.max}`}`,
+    );
+    console.log("Percentiles are nearest-rank over the measured samples.");
+  } catch (error) {
+    fail("Benchmark failed", error);
+  }
+}
+
+/**
+ * `ai feedback <question> <docId> --rating up|down`
+ */
+async function handleFeedback(globalOptions, question, docId, options = {}) {
+  try {
+    const { config, configDir } = loadCliConfig(globalOptions);
+    const components = loadComponents({ config, configDir });
+    const { retriever } = components;
+    if (typeof retriever.list !== "function") {
+      throw new Error(
+        `Retriever "${components.names.retriever}" cannot list documents. Use "file-store".`,
+      );
+    }
+    if (!retriever.list().some((doc) => doc.id === docId)) {
+      throw new Error(
+        `No indexed chunk with id "${docId}". Ids are shown by \`rag-pipeline ai search\`.`,
+      );
+    }
+    const file = feedbackPath(retriever, configDir);
+    if (globalOptions.dryRun) {
+      console.log(
+        `Dry run: would record a "${options.rating}" vote for ${docId} in ${file}`,
+      );
+      return;
+    }
+    const { total } = addFeedback(file, {
+      question,
+      docId,
+      rating: options.rating,
+    });
+    console.log(
+      `Recorded "${options.rating}" for ${docId} on "${question}" ` +
+        `(${total} vote(s) in ${path.relative(process.cwd(), file) || file}).`,
+    );
+  } catch (error) {
+    fail("Feedback failed", error);
+  }
+}
+
+/**
+ * `ai search <question>`: retrieval re-scored with stored feedback.
+ */
+async function handleSearch(globalOptions, question, options = {}) {
+  try {
+    const { config, configDir } = loadCliConfig(globalOptions);
+    const topK = options.topK || 3;
+    const components = loadComponents({ config, configDir });
+    const { retriever } = components;
+    const size = retriever.size ? retriever.size() : topK;
+    // Re-score a larger pool so a boosted chunk can move into the top-k.
+    const pool = Math.min(size, Math.max(topK * 4, topK + 10));
+    const results = await retrieve({ components, question, topK: pool });
+    const file = feedbackPath(retriever, configDir);
+    const { entries } = loadFeedback(file);
+    const ranked = applyFeedback(results, entries, question)
+      .filter((r) => r.baseScore > 0)
+      .slice(0, topK);
+
+    if (options.json) {
+      console.log(JSON.stringify({ question, results: ranked }, null, 2));
+      return;
+    }
+    console.log(
+      `${entries.length} stored feedback vote(s); retrieval only (no llm).`,
+    );
+    if (ranked.length === 0) {
+      console.log("No matching passages found.");
+      return;
+    }
+    ranked.forEach((r, i) => {
+      console.log(
+        `\n[${i + 1}] ${r.id}  score ${r.finalScore.toFixed(3)} = ` +
+          `${r.baseScore.toFixed(3)} retrieval ${signed(r.adjustment)} feedback`,
+      );
+      for (const reason of r.reasons) {
+        console.log(
+          `    ${reason.rating} vote on "${reason.question}" ` +
+            `(term overlap ${reason.similarity.toFixed(2)}): ${signed(reason.contribution)}`,
+        );
+      }
+      console.log(r.content);
+    });
+  } catch (error) {
+    fail("Search failed", error);
+  }
+}
+
+const TEXT_TYPES = {
+  ".md": "markdown",
+  ".markdown": "markdown",
+  ".html": "html",
+  ".htm": "html",
+  ".csv": "csv",
+  ".txt": "text",
+};
+const IMAGE_EXTENSIONS = new Set([
+  ".png",
+  ".jpg",
+  ".jpeg",
+  ".gif",
+  ".webp",
+  ".bmp",
+  ".tif",
+  ".tiff",
+]);
+
+function configuredLlm(globalOptions) {
+  try {
+    const { config } = loadCliConfig(globalOptions);
+    const entry = (config.pipeline || []).find((e) => e.stage === "llm");
+    return entry ? entry.name : "none";
+  } catch {
+    return "none (no readable config)";
+  }
+}
+
+function imageUnsupportedMessage(globalOptions) {
+  return (
+    "Image processing needs a vision-capable model, and none is available: the " +
+    "built-in llm connectors (openai, anthropic, ollama) only send text prompts. " +
+    `Configured llm: ${configuredLlm(globalOptions)}.`
+  );
+}
+
+async function describeTextFile(file, baseDir) {
+  const [doc] = await readDocuments(file, { baseDir });
+  return {
+    source: doc.source,
+    type: TEXT_TYPES[path.extname(file).toLowerCase()],
+    characters: fs.readFileSync(file, "utf8").length,
+    extractedCharacters: doc.chunks.reduce((n, c) => n + c.length, 0),
+    chunks: doc.chunks.length,
+  };
+}
+
+/**
+ * `ai multimodal process <path>`: report what the loaders extract. Does not
+ * modify the index.
+ */
+async function handleMultimodalProcess(globalOptions, inputPath, options = {}) {
+  try {
+    const absolute = path.resolve(inputPath);
+    if (!fs.existsSync(absolute))
+      throw new Error(`Path not found: ${inputPath}`);
+
+    const files = fs.statSync(absolute).isDirectory()
+      ? (
+          await fg("**/*", {
+            cwd: absolute,
+            absolute: true,
+            ignore: ["**/node_modules/**", "**/.git/**"],
+          })
+        ).sort()
+      : [absolute];
+
+    const processed = [];
+    const skipped = [];
+    for (const file of files) {
+      const ext = path.extname(file).toLowerCase();
+      const rel = path.relative(process.cwd(), file).split(path.sep).join("/");
+      if (TEXT_TYPES[ext]) {
+        processed.push(await describeTextFile(file, process.cwd()));
+      } else if (IMAGE_EXTENSIONS.has(ext)) {
+        if (files.length === 1)
+          throw new Error(imageUnsupportedMessage(globalOptions));
+        skipped.push({
+          source: rel,
+          reason: "image (needs a vision-capable model)",
+        });
+      } else {
+        if (files.length === 1) {
+          throw new Error(
+            `Unsupported file type "${ext}". Supported text types: ${Object.keys(TEXT_TYPES).join(", ")}`,
+          );
+        }
+        skipped.push({ source: rel, reason: `unsupported type "${ext}"` });
+      }
+    }
+
+    if (options.json) {
+      console.log(JSON.stringify({ processed, skipped }, null, 2));
+    } else {
+      for (const p of processed) {
+        console.log(
+          `${p.source}: ${p.type}, ${p.characters} characters, ` +
+            `${p.chunks} chunk(s) (${p.extractedCharacters} characters extracted)`,
+        );
+      }
+      for (const s of skipped) console.log(`skipped ${s.source}: ${s.reason}`);
+      console.log(
+        `Processed ${processed.length} file(s), skipped ${skipped.length}. ` +
+          "The index was not modified; use `rag-pipeline ingest` to add files.",
+      );
+    }
+    if (processed.length === 0) {
+      throw new Error(
+        skipped.some((s) => s.reason.startsWith("image"))
+          ? imageUnsupportedMessage(globalOptions)
+          : "No supported text files found.",
+      );
+    }
+  } catch (error) {
+    fail("Processing failed", error);
+  }
+}
+
+module.exports = {
+  handleTrainEmbedding,
+  handleBenchmark,
+  handleFeedback,
+  handleSearch,
+  handleMultimodalProcess,
+};

package/src/cli/handlers/diagnostics-handler.js

@@ -6,66 +6,82 @@
 const fs = require("fs/promises");
 const { logger } = require("../../utils/logger.js");
 
+const SEVERITY_ICONS = { error: "❌", warning: "⚠️ ", info: "ℹ️ " };
+
+function printIssues(issues) {
+  for (const issue of issues) {
+    console.log(
+      `${SEVERITY_ICONS[issue.severity] || "-"} [${issue.code}] ${issue.message}`,
+    );
+    if (issue.fix) console.log(`     Fix: ${issue.fix}`);
+  }
+}
+
 /**
- * Handle doctor command
+ * Handle doctor command: run the diagnostic checkers and print the issues.
+ * Exits 1 when any error-level issue is found.
+ *
  * @param {object} globalOptions - Global CLI options from program.opts()
  * @param {object} options - Command-specific options
- * @param {Function} pipelineDoctorFn - Pipeline doctor function
+ * @param {Function} [pipelineDoctorFn] - Injectable for tests
  */
 async function handleDoctor(globalOptions, options, pipelineDoctorFn) {
   try {
-    const doctorOptions = {
-      configPath: globalOptions.config,
+    const runDoctor =
+      pipelineDoctorFn || require("../doctor-command.js").runPipelineDoctor;
+    const report = await runDoctor({
+      configPath: globalOptions.config || ".ragrc.json",
       verbose: globalOptions.verbose,
       autoFix: options.autoFix,
-      categories: options.category,
-    };
+      categories: options.category || ["all"],
+    });
 
-    const report = await pipelineDoctorFn(doctorOptions);
+    console.log("🏥 Pipeline diagnostics\n");
+    if (report.issues.length === 0) {
+      console.log("✅ No issues found");
+    } else {
+      printIssues(report.issues);
+    }
+    const { errors, warnings, info } = report.summary;
+    console.log(`\n${errors} error(s), ${warnings} warning(s), ${info} info`);
 
     if (options.report) {
       await fs.writeFile(options.report, JSON.stringify(report, null, 2));
       console.log(`📋 Diagnostic report saved to: ${options.report}`);
     }
+    if (errors > 0) process.exitCode = 1;
   } catch (error) {
-    logger.error("❌ Doctor command failed:", error.message);
-    process.exit(1);
+    console.error(`❌ Doctor command failed: ${error.message}`);
+    process.exitCode = 1;
   }
 }
 
 /**
- * Handle validate command
+ * Handle validate command: schema validation plus resolving every pipeline
+ * component. Exits 1 when the configuration has errors.
+ *
  * @param {object} globalOptions - Global CLI options from program.opts()
- * @param {object} options - Command-specific options
- * @param {Function} validateSchemaFn - Schema validation function
  */
-async function handleValidate(globalOptions, options, validateSchemaFn) {
+async function handleValidate(globalOptions) {
   try {
-    const configPath = globalOptions.config;
-
+    const configPath = globalOptions.config || ".ragrc.json";
     console.log(`🔍 Validating configuration: ${configPath}`);
 
-    // Load and validate configuration
-    const configContent = await fs.readFile(configPath, "utf-8");
-    const config = JSON.parse(configContent);
+    const { checkConfiguration } = require("../diagnostics/config-checker.js");
+    const issues = await checkConfiguration({ configPath });
+    const errors = issues.filter((issue) => issue.severity === "error");
 
-    const validation = validateSchemaFn(config);
-
-    if (validation.valid) {
+    if (errors.length === 0) {
       console.log("✅ Configuration is valid");
-
-      if (validation.legacy) {
-        console.log("⚠️  Using legacy format - consider upgrading");
-      }
+      printIssues(issues);
     } else {
       console.log("❌ Configuration validation failed:");
-      validation.errors?.forEach((error) => {
-        console.log(`  ${error.instancePath || "root"}: ${error.message}`);
-      });
+      printIssues(issues);
+      process.exitCode = 1;
     }
   } catch (error) {
-    logger.error("❌ Validation failed:", error.message);
-    process.exit(1);
+    console.error(`❌ Validation failed: ${error.message}`);
+    process.exitCode = 1;
   }
 }
 
@@ -97,13 +113,44 @@ async function handleInfo(globalOptions, options) {
         );
         console.log("Configuration:");
         console.log(`  File: ${globalOptions.config}`);
-        console.log(`  Format: ${config.plugins ? "Enhanced" : "Legacy"}`);
+        for (const entry of Array.isArray(config.pipeline)
+          ? config.pipeline
+          : []) {
+          console.log(`  ${entry.stage}: ${entry.name}`);
+        }
         console.log("");
       } catch (error) {
         console.log("Configuration: Not found or invalid");
         console.log("");
       }
     }
+
+    // Available components: built-ins plus custom plugins from the config
+    if ((!options.system && !options.config) || options.plugins) {
+      const { BUILT_INS } = require("../../config/component-factory.js");
+      console.log("Built-in components:");
+      for (const [stage, names] of Object.entries(BUILT_INS)) {
+        console.log(`  ${stage}: ${Object.keys(names).join(", ")}`);
+      }
+      let custom = {};
+      try {
+        custom =
+          JSON.parse(await fs.readFile(globalOptions.config, "utf-8"))
+            .plugins || {};
+      } catch {
+        // no or invalid config: built-ins only
+      }
+      const entries = Object.entries(custom).flatMap(([stage, plugins]) =>
+        Object.entries(plugins || {}).map(
+          ([name, modulePath]) => `  ${stage}/${name}: ${modulePath}`,
+        ),
+      );
+      if (entries.length > 0) {
+        console.log("Custom plugins:");
+        entries.forEach((line) => console.log(line));
+      }
+      console.log("");
+    }
   } catch (error) {
     logger.error("❌ Failed to get system information:", error.message);
     process.exit(1);

package/src/cli/handlers/ingest-handler.js

@@ -1,41 +1,47 @@
 /**
  * Ingest command handler
- * Handles document ingestion into the pipeline
+ * Loads documents, embeds their chunks and stores them in the configured
+ * retriever (file-store persists them for later `query` runs).
  */
 
-const { logger } = require("../../utils/logger.js");
+const { loadCliConfig } = require("../shared/config-context.js");
+const { ingest } = require("../services/rag-service.js");
 
 /**
- * Handle ingest command
  * @param {object} globalOptions - Global CLI options from program.opts()
- * @param {string} file - Document file path
+ * @param {string} inputPath - File or directory to ingest
  * @param {object} options - Command-specific options
  */
-async function handleIngest(globalOptions, file, options) {
+async function handleIngest(globalOptions, inputPath, options = {}) {
   try {
-    if (globalOptions.dryRun) {
-      console.log("🧪 Dry run: Would ingest document");
-      console.log(`File: ${file}`);
-      console.log(
-        `Parallel processing: ${options.parallel ? "enabled" : "disabled"}`,
-      );
-      console.log(`Streaming: ${options.streaming ? "enabled" : "disabled"}`);
-      console.log(`Tracing: ${options.trace ? "enabled" : "disabled"}`);
+    const { config, configDir } = loadCliConfig(globalOptions);
 
-      if (options.preview) {
-        console.log("📄 Document Preview:");
-      }
-      if (options.validate) {
-        console.log("File validation passed");
+    if (globalOptions.dryRun) {
+      const { readDocuments } = require("../../loader/document-reader.js");
+      const documents = await readDocuments(inputPath, { baseDir: configDir });
+      console.log(`🧪 Dry run: would ingest ${documents.length} file(s)`);
+      for (const doc of documents) {
+        console.log(`  ${doc.source} (${doc.chunks.length} chunks)`);
       }
       return;
     }
 
-    console.log(`📄 Ingesting document: ${file}`);
-    console.log("✅ Document ingested successfully!");
+    const summary = await ingest({
+      config,
+      configDir,
+      inputPath,
+      rebuild: Boolean(options.rebuild),
+    });
+    console.log(
+      `✅ Ingested ${summary.files} file(s), ${summary.chunks} chunk(s). ` +
+        `Index now holds ${summary.totalChunks} chunk(s).`,
+    );
+    if (globalOptions.verbose) {
+      for (const source of summary.sources) console.log(`  ${source}`);
+    }
   } catch (error) {
-    logger.error("❌ Document ingestion failed:", error.message);
-    process.exit(1);
+    console.error(`❌ Ingestion failed: ${error.message}`);
+    process.exitCode = 1;
   }
 }