@devilsdev/rag-pipeline-utils 2.6.1 → 2.7.0

package/src/cli/handlers/eval-handler.js

@@ -0,0 +1,201 @@
+/**
+ * Eval command handler
+ * Runs an evaluation dataset through the configured pipeline, prints the
+ * scores and exits 1 when the run fails: a case errors, a metric is below
+ * its --min, or the run regressed from the --baseline report.
+ *
+ * Options: --baseline <report.json>, --output <report.json>, --report
+ * <file.html|file.md>, --min <metric=value> (repeatable), --tolerance,
+ * --case-tolerance, --concurrency, --timeout, --top-k, --json.
+ */
+
+const fs = require("fs");
+const path = require("path");
+const { loadCliConfig } = require("../shared/config-context.js");
+const { createPipeline } = require("../services/rag-service.js");
+const { validateDataset } = require("../../evaluate/dataset.js");
+const {
+  runEvaluation,
+  compareConfigurations,
+  EVALUATION_METRICS,
+} = require("../../evaluate/compare.js");
+const {
+  evaluationVerdict,
+  renderEvaluationReport,
+} = require("../../evaluate/report.js");
+
+/** Reads a dataset from .json (array or { cases }) or .jsonl (a case per line). */
+function readDataset(file) {
+  const text = fs.readFileSync(file, "utf8");
+  if (/\.jsonl$/i.test(file)) {
+    return text
+      .split(/\r?\n/)
+      .map((line, i) => ({ line: line.trim(), n: i + 1 }))
+      .filter(({ line }) => line.length > 0)
+      .map(({ line, n }) => {
+        try {
+          return JSON.parse(line);
+        } catch (err) {
+          throw new Error(`${file}:${n}: not valid JSON (${err.message})`);
+        }
+      });
+  }
+  try {
+    return JSON.parse(text);
+  } catch (err) {
+    throw new Error(`${file}: not valid JSON (${err.message})`);
+  }
+}
+
+function printText({ report, comparison, verdict, quiet }) {
+  const { summary } = report;
+  if (!quiet) {
+    console.log(
+      `\n${summary.cases} case${summary.cases === 1 ? "" : "s"}, ${summary.failed} failed`,
+    );
+    const names = EVALUATION_METRICS.filter((m) => summary.metrics[m]);
+    if (names.length > 0) {
+      console.log("");
+      for (const name of names) {
+        const mean = summary.metrics[name].mean.toFixed(3);
+        const cmp = comparison && comparison.metrics[name];
+        const change = cmp
+          ? `  (baseline ${cmp.baseline.toFixed(3)}, ${cmp.delta >= 0 ? "+" : ""}${cmp.delta.toFixed(3)}${cmp.regressed ? ", regressed" : ""})`
+          : "";
+        console.log(`  ${name.padEnd(17)} ${mean}${change}`);
+      }
+    }
+    const failed = report.cases.filter((c) => !c.success);
+    for (const c of failed.slice(0, 10)) {
+      console.log(`\n  ✗ ${c.id}: ${c.error}`);
+    }
+    if (comparison) {
+      const worse = comparison.regressions.filter((r) => r.metric);
+      if (worse.length > 0) {
+        console.log("\nCases that got worse:");
+        for (const r of worse.slice(0, 10)) {
+          console.log(
+            `  ${r.id}: ${r.metric} ${r.baseline.toFixed(3)} -> ${r.candidate.toFixed(3)}`,
+          );
+        }
+        if (worse.length > 10)
+          console.log(`  ...and ${worse.length - 10} more`);
+      }
+      for (const w of comparison.warnings) console.log(`  ! ${w}`);
+    }
+  }
+  if (verdict.passed) {
+    console.log("\n✅ Evaluation passed");
+  } else {
+    console.log("\n❌ Evaluation failed");
+    for (const reason of verdict.reasons) console.log(`  - ${reason}`);
+  }
+}
+
+async function handleEval(globalOptions, datasetPath, options = {}) {
+  try {
+    const datasetFile = path.resolve(datasetPath);
+    const dataset = readDataset(datasetFile);
+    const checked = validateDataset(dataset);
+    for (const w of checked.warnings) console.warn(`! ${w.message}`);
+    if (!checked.valid) {
+      console.error(`❌ Invalid dataset ${datasetPath}:`);
+      for (const e of checked.errors) console.error(`  - ${e.message}`);
+      process.exitCode = 1;
+      return;
+    }
+
+    let baseline;
+    if (options.baseline) {
+      baseline = JSON.parse(
+        fs.readFileSync(path.resolve(options.baseline), "utf8"),
+      );
+    }
+
+    if (globalOptions.dryRun) {
+      console.log(
+        `🧪 Dry run: would evaluate ${checked.cases.length} cases from ${datasetPath}${baseline ? ` against ${options.baseline}` : ""}`,
+      );
+      return;
+    }
+
+    const { config, configDir } = loadCliConfig(globalOptions);
+    const { pipeline } = createPipeline({ config, configDir });
+    const quiet = Boolean(globalOptions.quiet) || Boolean(options.json);
+    let done = 0;
+    const report = await runEvaluation(checked.cases, pipeline, {
+      concurrency: options.concurrency || 1,
+      ...(options.timeout && { timeout: options.timeout }),
+      runOptions: { topK: options.topK || 3 },
+      onCase: () => {
+        done += 1;
+        if (!quiet && process.stderr.isTTY) {
+          process.stderr.write(`\rEvaluating ${done}/${checked.cases.length}`);
+        }
+      },
+    });
+    if (!quiet && process.stderr.isTTY) process.stderr.write("\n");
+
+    const comparison = baseline
+      ? await compareConfigurations(null, {
+          baseline,
+          candidate: report,
+          ...(options.tolerance !== undefined && {
+            tolerance: options.tolerance,
+          }),
+          ...(options.caseTolerance !== undefined && {
+            caseTolerance: options.caseTolerance,
+          }),
+        })
+      : undefined;
+    const min = options.min || {};
+    const verdict = evaluationVerdict({ report, comparison, min });
+
+    if (options.output) {
+      fs.writeFileSync(
+        path.resolve(options.output),
+        `${JSON.stringify(report, null, 2)}\n`,
+      );
+    }
+    if (options.report) {
+      const file = path.resolve(options.report);
+      const format = /\.html?$/i.test(file) ? "html" : "markdown";
+      fs.writeFileSync(
+        file,
+        renderEvaluationReport({ report, comparison, min }, { format }),
+      );
+    }
+
+    if (options.json) {
+      const { baseline: _b, candidate: _c, ...rest } = comparison || {};
+      console.log(
+        JSON.stringify(
+          {
+            passed: verdict.passed,
+            reasons: verdict.reasons,
+            thresholds: verdict.thresholds,
+            summary: report.summary,
+            ...(comparison && { comparison: rest }),
+          },
+          null,
+          2,
+        ),
+      );
+    } else {
+      printText({
+        report,
+        comparison,
+        verdict,
+        quiet: Boolean(globalOptions.quiet),
+      });
+      if (options.output) console.log(`Report saved to ${options.output}`);
+      if (options.report) console.log(`Summary written to ${options.report}`);
+    }
+    if (!verdict.passed) process.exitCode = 1;
+  } catch (error) {
+    console.error(`❌ Evaluation failed: ${error.message}`);
+    process.exitCode = 1;
+  }
+}
+
+module.exports = { handleEval, readDataset };

package/src/cli/handlers/generate-handler.js

@@ -0,0 +1,103 @@
+/**
+ * generate-dataset command handler
+ * Writes an evaluation test set from the ingested index: the configured
+ * llm writes question and answer pairs about each chunk, and every case
+ * records its source file, so `rag-pipeline eval` can check retrieval too.
+ *
+ * Options: --output <file> (required), --count, --per-document,
+ * --min-faithfulness, --concurrency, --force (overwrite --output).
+ */
+
+const fs = require("fs");
+const path = require("path");
+const { loadCliConfig } = require("../shared/config-context.js");
+const { loadComponents } = require("../services/rag-service.js");
+const { generateSyntheticDataset } = require("../../evaluate/synthetic.js");
+
+async function handleGenerateDataset(globalOptions, options = {}) {
+  try {
+    const output = path.resolve(options.output);
+    if (fs.existsSync(output) && !options.force) {
+      console.error(
+        `❌ ${options.output} already exists. Pass --force to replace it (reviewed cases would be lost).`,
+      );
+      process.exitCode = 1;
+      return;
+    }
+
+    const { config, configDir } = loadCliConfig(globalOptions);
+    const components = loadComponents({ config, configDir });
+    if (!components.llm) {
+      console.error(
+        "❌ generate-dataset needs an llm stage in the config to write the questions.",
+      );
+      process.exitCode = 1;
+      return;
+    }
+    if (typeof components.retriever.list !== "function") {
+      console.error(
+        "❌ The retriever can't list its documents; generate-dataset reads them from the CLI's file-store index.",
+      );
+      process.exitCode = 1;
+      return;
+    }
+    const documents = await components.retriever.list();
+    const count = options.count || 20;
+
+    if (globalOptions.dryRun) {
+      console.log(
+        `🧪 Dry run: would write up to ${count} cases from ${documents.length} indexed chunks to ${options.output}`,
+      );
+      return;
+    }
+
+    const quiet = Boolean(globalOptions.quiet);
+    let written = 0;
+    const dataset = await generateSyntheticDataset(documents, {
+      llm: components.llm,
+      count,
+      perDocument: options.perDocument || 1,
+      ...(options.minFaithfulness !== undefined && {
+        minFaithfulness: options.minFaithfulness,
+      }),
+      concurrency: options.concurrency || 1,
+      onCase: () => {
+        written += 1;
+        if (!quiet && process.stderr.isTTY) {
+          process.stderr.write(`\rGenerated ${written}/${count}`);
+        }
+      },
+    });
+    if (!quiet && process.stderr.isTTY) process.stderr.write("\n");
+
+    fs.writeFileSync(output, `${JSON.stringify(dataset.cases, null, 2)}\n`);
+
+    if (!quiet && dataset.skipped.length > 0) {
+      const reasons = {};
+      for (const s of dataset.skipped) {
+        const reason = s.reason.replace(/ \(faithfulness [\d.]+\)$/, "");
+        reasons[reason] = (reasons[reason] || 0) + 1;
+      }
+      console.log("Skipped:");
+      for (const [reason, n] of Object.entries(reasons)) {
+        console.log(`  ${n} × ${reason}`);
+      }
+    }
+    if (dataset.cases.length === 0) {
+      console.error(
+        `❌ No cases were generated from ${documents.length} chunks; see the reasons above.`,
+      );
+      process.exitCode = 1;
+      return;
+    }
+    console.log(
+      `✅ Wrote ${dataset.cases.length} case${dataset.cases.length === 1 ? "" : "s"} to ${options.output}. ` +
+        'Review them: fix or delete wrong ones and set "reviewed": true.',
+    );
+  } catch (error) {
+    console.error(`❌ Generating the dataset failed: ${error.message}`);
+    process.exitCode = 1;
+  }
+}
+
+module.exports = { handleGenerateDataset };

package/src/cli/services/benchmark.js

@@ -87,7 +87,7 @@ async function runBenchmark({
 
   const queries = queriesFile
     ? readQueryFile(queriesFile)
-    : sampleQueries(retriever.list ? retriever.list() : []);
+    : sampleQueries(retriever.list ? await retriever.list() : []);
   if (queries.length === 0) {
     throw new Error(
       "No queries: pass --queries <file> (the index has no text to sample from).",
@@ -148,7 +148,7 @@ async function runBenchmark({
     environment: {
       node: process.version,
       platform: `${process.platform} ${process.arch}`,
-      chunks: retriever.size ? retriever.size() : null,
+      chunks: retriever.size ? await retriever.size() : null,
       embedder: components.names.embedder,
       dimensions: retriever.getEmbedderInfo?.()?.dimensions ?? null,
       retriever: components.names.retriever,

package/src/cli/services/rag-service.js

@@ -28,6 +28,30 @@ function embedDocument(embedder, text) {
   return embedQuery({ embed: (t) => embedder.embed(t) }, text);
 }
 
+/**
+ * Corpus-fitted embedders (the built-in `local` TF-IDF embedder) keep a
+ * vocabulary that must be saved with the index and restored before every
+ * query. A store that can't save it (pgvector, Qdrant, custom stores) would
+ * embed queries with an empty vocabulary and return wrong results, so stop
+ * with a clear message instead.
+ */
+function assertEmbedderFits(components) {
+  const { embedder, retriever, names } = components;
+  const fitted =
+    typeof embedder.train === "function" ||
+    typeof embedder.setState === "function";
+  const persists =
+    typeof retriever.getEmbedderInfo === "function" &&
+    typeof retriever.setEmbedderInfo === "function";
+  if (fitted && !persists) {
+    throw new Error(
+      `The "${names.embedder}" embedder is fitted to your documents and needs a retriever ` +
+        `that saves its state, such as "file-store". With "${names.retriever}", use an ` +
+        "embedder like openai, ollama or cohere.",
+    );
+  }
+}
+
 /**
  * Restore embedder state saved with the index (e.g. LocalEmbedder's
  * vocabulary), so queries embed into the same vector space as ingestion.
@@ -65,14 +89,14 @@ async function embedAndStore({
   }
 
   await store.store(vectors);
-  store.setEmbedderInfo?.({
+  await store.setEmbedderInfo?.({
     name: embedderName,
     dimensions,
     ...(typeof embedder.getState === "function" && {
       state: embedder.getState(),
     }),
   });
-  store.save?.();
+  await store.save?.();
   return vectors;
 }
 
@@ -97,10 +121,12 @@ async function ingest({ config, configDir, inputPath, rebuild = false }) {
     );
   }
 
+  assertEmbedderFits(components);
   const embedderName = components.names.embedder;
-  if (rebuild) store.clear?.();
-  const previous = store.getEmbedderInfo?.();
-  if (previous && previous.name !== embedderName && store.size() > 0) {
+  // Stores may be remote (pgvector, Qdrant), so await every call.
+  if (rebuild) await store.clear?.();
+  const previous = await store.getEmbedderInfo?.();
+  if (previous && previous.name !== embedderName && (await store.size()) > 0) {
     throw new Error(
       `The index was built with embedder "${previous.name}" but the config now uses ` +
         `"${embedderName}". Re-run with --rebuild to re-index everything.`,
@@ -108,7 +134,7 @@ async function ingest({ config, configDir, inputPath, rebuild = false }) {
   }
 
   const documents = await readDocuments(inputPath, { baseDir: configDir });
-  for (const { source } of documents) store.removeBySource?.(source);
+  for (const { source } of documents) await store.removeBySource?.(source);
 
   let entries = documents.flatMap(({ source, chunks }) =>
     chunks.map((content, i) => ({
@@ -121,7 +147,7 @@ async function ingest({ config, configDir, inputPath, rebuild = false }) {
   // Corpus-fitted embedders (LocalEmbedder) must be retrained on everything
   // stored, and every stored chunk re-embedded into the new vector space.
   if (typeof embedder.train === "function") {
-    entries = [...(store.list?.() || []), ...entries];
+    entries = [...((await store.list?.()) || []), ...entries];
     embedder.train(entries.map((e) => e.content));
   } else {
     restoreEmbedder(embedder, store);
@@ -138,7 +164,7 @@ async function ingest({ config, configDir, inputPath, rebuild = false }) {
   return {
     files: documents.length,
     chunks: documents.reduce((n, d) => n + d.chunks.length, 0),
-    totalChunks: store.size?.() ?? vectors.length,
+    totalChunks: (await store.size?.()) ?? vectors.length,
     sources: documents.map((d) => d.source),
   };
 }
@@ -186,13 +212,13 @@ async function retrain({ config, configDir, inputPath }) {
     );
   }
 
-  const stored = store.list();
+  const stored = await store.list();
   if (stored.length === 0) {
     throw new Error(
       "The index is empty. Run `rag-pipeline ingest <path>` first.",
     );
   }
-  const previous = store.getEmbedderInfo?.();
+  const previous = await store.getEmbedderInfo?.();
   if (previous && previous.name !== embedderName) {
     throw new Error(
       `The index was built with embedder "${previous.name}". ` +
@@ -241,6 +267,8 @@ function loadComponents({ config, configDir }) {
   const components = buildComponents(config, { configDir });
   const embedder = requireStage(components, "embedder");
   const retriever = requireStage(components, "retriever");
+  assertEmbedderFits(components);
+  // A remote store's size() is a Promise; its callers check emptiness.
   if (retriever.size && retriever.size() === 0) {
     throw new Error(
       "The index is empty. Run `rag-pipeline ingest <path>` first.",
@@ -351,10 +379,31 @@ async function query({
   };
 }
 
+/**
+ * A createRagPipeline() pipeline over the configured components and index,
+ * for running many questions (rag-pipeline eval).
+ *
+ * @param {object} params
+ * @param {object} params.config
+ * @param {string} params.configDir
+ * @returns {{ pipeline: object, hasLlm: boolean }}
+ */
+function createPipeline({ config, configDir }) {
+  const c = loadComponents({ config, configDir });
+  const pipeline = createRagPipeline({
+    embedder: c.embedder,
+    retriever: c.retriever,
+    llm: c.llm,
+    reranker: c.reranker,
+  });
+  return { pipeline, hasLlm: Boolean(c.llm) };
+}
+
 module.exports = {
   ingest,
   retrain,
   query,
+  createPipeline,
   retrieve,
   loadComponents,
   restoreEmbedder,

package/src/cli/shared/cli-utils.js

@@ -2,6 +2,7 @@
  * Shared CLI utility functions
  */
 
+const { InvalidArgumentError } = require("commander");
 const pkg = require("../../../package.json");
 
 /**
@@ -23,16 +24,52 @@ function createProgressBar(percentage, width = 20) {
 function parsePositiveInteger(value) {
   const parsed = parseInt(value, 10);
   if (isNaN(parsed) || parsed <= 0) {
-    throw new Error(`Invalid value '${value}'. Expected positive integer.`);
+    throw new InvalidArgumentError("Expected a positive integer.");
   }
   return parsed;
 }
 
+/**
+ * Collector for repeatable --min metric=value (rag-pipeline eval). The
+ * metric list loads only when the option is used.
+ */
+function collectMin(value, previous = {}) {
+  const match = /^([A-Za-z]+)=(\d*\.?\d+)$/.exec(String(value).trim());
+  if (!match) {
+    throw new InvalidArgumentError("Use metric=value, e.g. faithfulness=0.8.");
+  }
+  const [, metric, number] = match;
+  // eslint-disable-next-line global-require
+  const { EVALUATION_METRICS } = require("../../evaluate/compare.js");
+  if (!EVALUATION_METRICS.includes(metric)) {
+    throw new InvalidArgumentError(
+      `Unknown metric "${metric}"; use one of ${EVALUATION_METRICS.join(", ")}.`,
+    );
+  }
+  const min = Number(number);
+  if (min > 1)
+    throw new InvalidArgumentError("The value must be between 0 and 1.");
+  return { ...previous, [metric]: min };
+}
+
+/**
+ * Parser for a non-negative number (eval --tolerance, --case-tolerance)
+ */
+function parseTolerance(value) {
+  const n = Number(value);
+  if (value === "" || !Number.isFinite(n) || n < 0) {
+    throw new InvalidArgumentError("Expected a number of 0 or more.");
+  }
+  return n;
+}
+
 // Top-level commands offered by shell completion
 const COMMANDS = [
   "init",
   "ingest",
   "query",
+  "eval",
+  "generate-dataset",
   "run",
   "serve",
   "doctor",
@@ -100,6 +137,8 @@ For more information, visit: https://github.com/DevilsDev/rag-pipeline-utils
 module.exports = {
   createProgressBar,
   parsePositiveInteger,
+  collectMin,
+  parseTolerance,
   generateCompletionScript,
   getVersion,
   getExtendedHelp,

package/src/config/component-factory.js

@@ -30,6 +30,38 @@ const BUILT_INS = {
       }),
     memory: (cfg) =>
       new (require("../connectors/memory-retriever").MemoryRetriever)(cfg),
+    // The connection string comes from the environment, never the config
+    // file: `connectionStringEnv` names the variable (default DATABASE_URL).
+    pgvector: (cfg) => {
+      const variable = cfg.connectionStringEnv || "DATABASE_URL";
+      const connectionString = process.env[variable];
+      if (!connectionString) {
+        throw new Error(
+          `The pgvector retriever reads its connection string from ${variable}, which is not set`,
+        );
+      }
+      const { PgVectorRetriever } = require("../retrieval/pgvector-retriever");
+      return new PgVectorRetriever({
+        connectionString,
+        table: cfg.table,
+        dimensions: cfg.dimensions,
+        createTable: cfg.createTable,
+        index: cfg.index,
+        cacheKey: cfg.cacheKey,
+      });
+    },
+    // The API key comes only from QDRANT_API_KEY, never the config file.
+    qdrant: (cfg) => {
+      const { QdrantRetriever } = require("../retrieval/qdrant-retriever");
+      return new QdrantRetriever({
+        url: cfg.url || process.env.QDRANT_URL || "http://localhost:6333",
+        apiKey: process.env.QDRANT_API_KEY,
+        collection: cfg.collection,
+        dimensions: cfg.dimensions,
+        createCollection: cfg.createCollection,
+        cacheKey: cfg.cacheKey,
+      });
+    },
   },
   llm: {
     openai: (cfg) =>

package/src/connectors/local-embedder.js

@@ -1,9 +1,9 @@
-'use strict';
-const { digest } = require('../cache/component-keys');
+"use strict";
+const { digest } = require("../cache/component-keys");
 
-const { BaseConnector } = require('./base-connector');
-const { tokenize } = require('../evaluate/scoring');
-const { throwIfAborted } = require('./connector-utils');
+const { BaseConnector } = require("./base-connector");
+const { tokenize } = require("../evaluate/scoring");
+const { throwIfAborted } = require("./connector-utils");
 
 /**
  * Default configuration for the local TF-IDF embedder.
@@ -27,7 +27,7 @@ class LocalEmbedder extends BaseConnector {
    * @param {number} [options.minDocFrequency=1] - Minimum document frequency for a token to be included
    */
   constructor(options = {}) {
-    super({ name: 'local-embedder', ...options });
+    super({ name: "local-embedder", ...options });
     this.config = { ...DEFAULT_CONFIG, ...options };
     /** @type {Map<string, number>} token -> vocabulary index */
     this.vocabulary = new Map();
@@ -45,7 +45,7 @@ class LocalEmbedder extends BaseConnector {
   train(documents) {
     this._fingerprint = null; // the cache key changes with the training
     const docs = documents.map((doc) => {
-      if (typeof doc === 'string') return doc;
+      if (typeof doc === "string") return doc;
       return doc.content || doc.text || String(doc);
     });
 
@@ -70,8 +70,15 @@ class LocalEmbedder extends BaseConnector {
       idfEntries.push([token, idfValue]);
     }
 
-    // Sort by IDF descending and take top maxVocabularySize tokens
-    idfEntries.sort((a, b) => b[1] - a[1]);
+    // Keep the maxVocabularySize tokens found in the most documents (ties
+    // in token order, so training is deterministic). Keeping the highest
+    // IDF instead kept the rarest tokens: on a large corpus, words that
+    // appear in one document only, so most queries embedded to all zeros.
+    idfEntries.sort(
+      (a, b) =>
+        docFreq.get(b[0]) - docFreq.get(a[0]) ||
+        (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0),
+    );
     const topEntries = idfEntries.slice(0, this.config.maxVocabularySize);
 
     // Build vocabulary map and IDF map
@@ -83,7 +90,7 @@ class LocalEmbedder extends BaseConnector {
       this.idf.set(token, idfValue);
     }
 
-    this.emit('trained', {
+    this.emit("trained", {
       vocabSize: this.vocabulary.size,
       docCount: this.docCount,
     });
@@ -104,8 +111,8 @@ class LocalEmbedder extends BaseConnector {
     const vocabSize = this.vocabulary.size;
 
     if (vocabSize === 0) {
-      this.emit('warning', {
-        message: 'LocalEmbedder has not been trained yet',
+      this.emit("warning", {
+        message: "LocalEmbedder has not been trained yet",
       });
       return new Array(1).fill(0);
     }
@@ -187,7 +194,7 @@ class LocalEmbedder extends BaseConnector {
    * @returns {string|null}
    */
   getCacheKey(role) {
-    if (role !== 'embedder') return null;
+    if (role !== "embedder") return null;
     if (!this._fingerprint) {
       this._fingerprint = digest(
         JSON.stringify([this.docCount, [...this.vocabulary], [...this.idf]]),