@devilsdev/rag-pipeline-utils 2.4.5 → 2.5.0

package/src/cli/services/benchmark.js

@@ -0,0 +1,161 @@
+"use strict";
+
+/**
+ * `rag-pipeline ai benchmark`: time real queries against the configured
+ * pipeline and index. Every number reported is measured here.
+ */
+
+const fs = require("fs");
+const { embedQuery } = require("../../utils/embed-query");
+const { loadComponents, query } = require("./rag-service");
+const { summarize } = require("./bench-stats");
+
+const SAMPLE_QUERIES = 5;
+const SAMPLE_WORDS = 8;
+
+/**
+ * Read queries from a file: a JSON array of strings, or one query per line.
+ * @param {string} file
+ * @returns {string[]}
+ */
+function readQueryFile(file) {
+  if (!fs.existsSync(file)) throw new Error(`Queries file not found: ${file}`);
+  const text = fs.readFileSync(file, "utf8");
+  const trimmed = text.trim();
+  let queries;
+  if (trimmed.startsWith("[")) {
+    let parsed;
+    try {
+      parsed = JSON.parse(trimmed);
+    } catch (error) {
+      throw new Error(
+        `Queries file ${file} is not valid JSON: ${error.message}`,
+      );
+    }
+    if (!parsed.every((q) => typeof q === "string")) {
+      throw new Error(`Queries file ${file} must be a JSON array of strings`);
+    }
+    queries = parsed;
+  } else {
+    queries = text.split(/\r?\n/);
+  }
+  queries = queries.map((q) => q.trim()).filter(Boolean);
+  if (queries.length === 0) throw new Error(`No queries found in ${file}`);
+  return queries;
+}
+
+/**
+ * Deterministic sample: the first words of the first few indexed chunks.
+ * @param {Array<{content: string}>} chunks
+ * @returns {string[]}
+ */
+function sampleQueries(chunks, count = SAMPLE_QUERIES) {
+  return chunks
+    .map((c) => String(c.content).split(/\s+/).filter(Boolean))
+    .filter((words) => words.length > 0)
+    .slice(0, count)
+    .map((words) => words.slice(0, SAMPLE_WORDS).join(" "));
+}
+
+const elapsedMs = (start) => Number(process.hrtime.bigint() - start) / 1e6;
+
+async function timed(fn) {
+  const start = process.hrtime.bigint();
+  const value = await fn();
+  return { value, ms: elapsedMs(start) };
+}
+
+/**
+ * @param {object} params
+ * @param {object} params.config
+ * @param {string} params.configDir
+ * @param {string} [params.queriesFile]
+ * @param {number} [params.runs=3] - Passes over the query set
+ * @param {number} [params.topK=3]
+ * @returns {Promise<object>} Measured report
+ */
+async function runBenchmark({
+  config,
+  configDir,
+  queriesFile,
+  runs = 3,
+  topK = 3,
+}) {
+  const setup = await timed(() => loadComponents({ config, configDir }));
+  const components = setup.value;
+  const { embedder, retriever, reranker, llm } = components;
+
+  const queries = queriesFile
+    ? readQueryFile(queriesFile)
+    : sampleQueries(retriever.list ? retriever.list() : []);
+  if (queries.length === 0) {
+    throw new Error(
+      "No queries: pass --queries <file> (the index has no text to sample from).",
+    );
+  }
+
+  // First query pays one-time costs (lazy init, JIT); reported, not counted.
+  const warmup = await timed(() =>
+    query({ components, question: queries[0], topK }),
+  );
+
+  const samples = { embed: [], retrieve: [], rerank: [], endToEnd: [] };
+  const resultCounts = [];
+  for (let run = 0; run < runs; run++) {
+    for (const question of queries) {
+      const embedded = await timed(() => embedQuery(embedder, question));
+      samples.embed.push(embedded.ms);
+      const retrieved = await timed(() =>
+        retriever.retrieve({
+          query: question,
+          queryVector: embedded.value,
+          topK,
+        }),
+      );
+      samples.retrieve.push(retrieved.ms);
+      if (reranker) {
+        const reranked = await timed(() =>
+          reranker.rerank(question, retrieved.value, { topK }),
+        );
+        samples.rerank.push(reranked.ms);
+      }
+      const e2e = await timed(() => query({ components, question, topK }));
+      samples.endToEnd.push(e2e.ms);
+      resultCounts.push(e2e.value.results.length);
+    }
+  }
+
+  const stages = {
+    embed: summarize(samples.embed),
+    retrieve: summarize(samples.retrieve),
+    ...(reranker && { rerank: summarize(samples.rerank) }),
+    endToEnd: summarize(samples.endToEnd),
+  };
+
+  return {
+    queries,
+    queriesSource: queriesFile || "sampled from indexed chunks",
+    runs,
+    topK,
+    setupMs: setup.ms,
+    warmupMs: warmup.ms,
+    stages,
+    includesGeneration: Boolean(llm),
+    results: {
+      min: Math.min(...resultCounts),
+      max: Math.max(...resultCounts),
+    },
+    environment: {
+      node: process.version,
+      platform: `${process.platform} ${process.arch}`,
+      chunks: retriever.size ? retriever.size() : null,
+      embedder: components.names.embedder,
+      dimensions: retriever.getEmbedderInfo?.()?.dimensions ?? null,
+      retriever: components.names.retriever,
+      reranker: components.names.reranker || null,
+      llm: components.names.llm || null,
+    },
+  };
+}
+
+module.exports = { runBenchmark, readQueryFile, sampleQueries };

package/src/cli/services/feedback.js

@@ -0,0 +1,155 @@
+"use strict";
+
+/**
+ * Relevance feedback for `rag-pipeline ai feedback` / `ai search`.
+ *
+ * Feedback is stored as JSON next to the index. Scoring is deterministic and
+ * explainable: every stored vote for a result's docId contributes
+ *
+ *   sign(vote) * VOTE_WEIGHT * jaccard(terms(question), terms(votedQuestion))
+ *
+ * and the summed adjustment is capped to +/- MAX_ADJUSTMENT before being added
+ * to the retrieval score. Votes on unrelated questions (no shared terms)
+ * contribute nothing.
+ */
+
+const fs = require("fs");
+const path = require("path");
+const { tokenize } = require("../../evaluate/scoring");
+
+const FORMAT_VERSION = 1;
+const VOTE_WEIGHT = 0.1;
+const MAX_ADJUSTMENT = 0.3;
+const RATINGS = ["up", "down"];
+
+/**
+ * Where feedback lives: next to a file-store index, else <configDir>/.rag.
+ *
+ * @param {object} retriever - Built retriever (FileVectorStore has `path`)
+ * @param {string} configDir
+ * @returns {string}
+ */
+function feedbackPath(retriever, configDir) {
+  const dir =
+    retriever && typeof retriever.path === "string"
+      ? path.dirname(retriever.path)
+      : path.join(configDir, ".rag");
+  return path.join(dir, "feedback.json");
+}
+
+/**
+ * @param {string} file
+ * @returns {{version: number, entries: Array<{question: string, docId: string, rating: string, at: string}>}}
+ */
+function loadFeedback(file) {
+  if (!fs.existsSync(file)) return { version: FORMAT_VERSION, entries: [] };
+  let parsed;
+  try {
+    parsed = JSON.parse(fs.readFileSync(file, "utf8"));
+  } catch (error) {
+    throw new Error(
+      `Feedback file ${file} is not valid JSON: ${error.message}`,
+    );
+  }
+  if (parsed.version !== FORMAT_VERSION || !Array.isArray(parsed.entries)) {
+    throw new Error(`Unsupported feedback file format in ${file}`);
+  }
+  return parsed;
+}
+
+/**
+ * Append one vote and write the file atomically (temp file, then rename).
+ *
+ * @param {string} file
+ * @param {{question: string, docId: string, rating: string}} vote
+ * @returns {{entry: object, total: number}}
+ */
+function addFeedback(file, { question, docId, rating }) {
+  if (!RATINGS.includes(rating)) {
+    throw new Error(`Rating must be one of: ${RATINGS.join(", ")}`);
+  }
+  if (!String(question || "").trim()) throw new Error("Question is empty");
+  const data = loadFeedback(file);
+  const entry = { question, docId, rating, at: new Date().toISOString() };
+  data.entries.push(entry);
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  const tmp = `${file}.${process.pid}.tmp`;
+  fs.writeFileSync(tmp, JSON.stringify(data, null, 2));
+  fs.renameSync(tmp, file);
+  return { entry, total: data.entries.length };
+}
+
+/**
+ * Jaccard similarity of the two texts' token sets (0..1).
+ * @param {string} a
+ * @param {string} b
+ * @returns {number}
+ */
+function termSimilarity(a, b) {
+  const sa = new Set(tokenize(a));
+  const sb = new Set(tokenize(b));
+  if (sa.size === 0 || sb.size === 0) return 0;
+  let shared = 0;
+  for (const t of sa) if (sb.has(t)) shared++;
+  return shared / (sa.size + sb.size - shared);
+}
+
+/**
+ * Re-score retrieval results with stored feedback.
+ *
+ * @param {Array<{id: string, score: number}>} results - Retrieval results
+ * @param {Array<{question: string, docId: string, rating: string}>} entries - Stored votes
+ * @param {string} question - Current question
+ * @param {object} [options]
+ * @param {number} [options.voteWeight=VOTE_WEIGHT]
+ * @param {number} [options.maxAdjustment=MAX_ADJUSTMENT]
+ * @returns {Array<object>} Results sorted by finalScore (ties keep retrieval
+ *   order), each with baseScore, adjustment, finalScore and `reasons`.
+ */
+function applyFeedback(results, entries, question, options = {}) {
+  const voteWeight = options.voteWeight ?? VOTE_WEIGHT;
+  const maxAdjustment = options.maxAdjustment ?? MAX_ADJUSTMENT;
+
+  const scored = results.map((result, rank) => {
+    const reasons = [];
+    let raw = 0;
+    for (const vote of entries) {
+      if (vote.docId !== result.id) continue;
+      const similarity = termSimilarity(question, vote.question);
+      if (similarity === 0) continue;
+      const contribution =
+        (vote.rating === "up" ? 1 : -1) * voteWeight * similarity;
+      raw += contribution;
+      reasons.push({
+        rating: vote.rating,
+        question: vote.question,
+        similarity,
+        contribution,
+      });
+    }
+    const adjustment = Math.max(-maxAdjustment, Math.min(maxAdjustment, raw));
+    const baseScore = typeof result.score === "number" ? result.score : 0;
+    return {
+      ...result,
+      baseScore,
+      adjustment,
+      finalScore: baseScore + adjustment,
+      reasons,
+      rank,
+    };
+  });
+
+  scored.sort((a, b) => b.finalScore - a.finalScore || a.rank - b.rank);
+  return scored.map(({ rank, ...rest }) => rest);
+}
+
+module.exports = {
+  feedbackPath,
+  loadFeedback,
+  addFeedback,
+  termSimilarity,
+  applyFeedback,
+  RATINGS,
+  VOTE_WEIGHT,
+  MAX_ADJUSTMENT,
+};

package/src/cli/services/rag-service.js

@@ -0,0 +1,322 @@
+"use strict";
+
+/**
+ * Ingest and query operations shared by the `ingest`, `query` and `run` CLI
+ * commands (and the dev server). Components come from the .ragrc.json
+ * pipeline via buildComponents(); the retriever must persist between
+ * processes for ingest and query to be separate commands (file-store does).
+ */
+
+const { buildComponents } = require("../../config/component-factory");
+const { readDocuments } = require("../../loader/document-reader");
+const { embedQuery } = require("../../utils/embed-query");
+const { createRagPipeline } = require("../../core/create-pipeline");
+
+function requireStage(components, stage) {
+  if (!components[stage]) {
+    throw new Error(
+      `No ${stage} configured. Add a { "stage": "${stage}", ... } entry to the pipeline in .ragrc.json.`,
+    );
+  }
+  return components[stage];
+}
+
+/** Embed document text with embed() (not embedQuery), normalizing the shape. */
+function embedDocument(embedder, text) {
+  return embedQuery({ embed: (t) => embedder.embed(t) }, text);
+}
+
+/**
+ * Restore embedder state saved with the index (e.g. LocalEmbedder's
+ * vocabulary), so queries embed into the same vector space as ingestion.
+ */
+function restoreEmbedder(embedder, retriever) {
+  const info = retriever.getEmbedderInfo?.();
+  if (info?.state && typeof embedder.setState === "function") {
+    embedder.setState(info.state);
+  }
+}
+
+/**
+ * Embed entries with the (already trained/restored) embedder, store them and
+ * record the embedder metadata and state with the index.
+ *
+ * @returns {Promise<Array<{id: string, vector: number[]}>>} The stored vectors
+ */
+async function embedAndStore({
+  embedder,
+  embedderName,
+  store,
+  entries,
+  previous,
+}) {
+  const vectors = [];
+  for (const entry of entries) {
+    vectors.push({
+      ...entry,
+      vector: await embedDocument(embedder, entry.content),
+    });
+  }
+  const dimensions = vectors[0]?.vector?.length ?? previous?.dimensions ?? 0;
+  if (vectors.some((v) => v.vector.length !== dimensions)) {
+    throw new Error("Embedder returned vectors of inconsistent dimensions");
+  }
+
+  await store.store(vectors);
+  store.setEmbedderInfo?.({
+    name: embedderName,
+    dimensions,
+    ...(typeof embedder.getState === "function" && {
+      state: embedder.getState(),
+    }),
+  });
+  store.save?.();
+  return vectors;
+}
+
+/**
+ * Load documents from a file or directory, embed their chunks and store them.
+ * Re-ingesting a source replaces its previous chunks.
+ *
+ * @param {object} params
+ * @param {object} params.config - Normalized config
+ * @param {string} params.configDir - Directory of the config file
+ * @param {string} params.inputPath - File or directory to ingest
+ * @param {boolean} [params.rebuild=false] - Discard the existing index first
+ * @returns {Promise<{files: number, chunks: number, totalChunks: number, sources: string[]}>}
+ */
+async function ingest({ config, configDir, inputPath, rebuild = false }) {
+  const components = buildComponents(config, { configDir });
+  const embedder = requireStage(components, "embedder");
+  const store = requireStage(components, "retriever");
+  if (typeof store.store !== "function") {
+    throw new Error(
+      `Retriever "${components.names.retriever}" cannot store documents. Use "file-store".`,
+    );
+  }
+
+  const embedderName = components.names.embedder;
+  if (rebuild) store.clear?.();
+  const previous = store.getEmbedderInfo?.();
+  if (previous && previous.name !== embedderName && store.size() > 0) {
+    throw new Error(
+      `The index was built with embedder "${previous.name}" but the config now uses ` +
+        `"${embedderName}". Re-run with --rebuild to re-index everything.`,
+    );
+  }
+
+  const documents = await readDocuments(inputPath, { baseDir: configDir });
+  for (const { source } of documents) store.removeBySource?.(source);
+
+  let entries = documents.flatMap(({ source, chunks }) =>
+    chunks.map((content, i) => ({
+      id: `${source}#${i}`,
+      content,
+      metadata: { source, chunk: i },
+    })),
+  );
+
+  // Corpus-fitted embedders (LocalEmbedder) must be retrained on everything
+  // stored, and every stored chunk re-embedded into the new vector space.
+  if (typeof embedder.train === "function") {
+    entries = [...(store.list?.() || []), ...entries];
+    embedder.train(entries.map((e) => e.content));
+  } else {
+    restoreEmbedder(embedder, store);
+  }
+
+  const vectors = await embedAndStore({
+    embedder,
+    embedderName,
+    store,
+    entries,
+    previous,
+  });
+
+  return {
+    files: documents.length,
+    chunks: documents.reduce((n, d) => n + d.chunks.length, 0),
+    totalChunks: store.size?.() ?? vectors.length,
+    sources: documents.map((d) => d.source),
+  };
+}
+
+/**
+ * Refit a corpus-trained embedder (only the built-in `local` TF-IDF embedder)
+ * and re-embed every stored chunk with the new model.
+ *
+ * Without `inputPath` the embedder is fitted on the indexed chunks. With it,
+ * the embedder is fitted on the documents read from that path only (they are
+ * not added to the index); indexed chunks that share no term with the new
+ * vocabulary embed to all-zero vectors and are counted in `zeroVectors`.
+ *
+ * @param {object} params
+ * @param {object} params.config
+ * @param {string} params.configDir
+ * @param {string} [params.inputPath] - Optional training corpus
+ * @returns {Promise<{trainedOn: string, trainingDocuments: number, vocabularySize: number,
+ *   previousVocabularySize: number|null, reembedded: number, zeroVectors: number}>}
+ */
+async function retrain({ config, configDir, inputPath }) {
+  // Check the configured name before building components, so a hosted
+  // embedder without credentials still gets this explanation.
+  const entry = (config.pipeline || []).find((e) => e.stage === "embedder");
+  if (!entry) {
+    throw new Error(
+      'No embedder configured. Add { "stage": "embedder", "name": "local" } to the pipeline in .ragrc.json.',
+    );
+  }
+  if (entry.name !== "local") {
+    throw new Error(
+      `Training is only supported for the built-in "local" (TF-IDF) embedder; ` +
+        `this pipeline uses "${entry.name}". Hosted embedders (openai, cohere, ollama) ` +
+        "use fixed pre-trained models and cannot be fitted to your corpus.",
+    );
+  }
+
+  const components = buildComponents(config, { configDir });
+  const embedderName = components.names.embedder;
+  const embedder = requireStage(components, "embedder");
+  const store = requireStage(components, "retriever");
+  if (typeof store.list !== "function" || typeof store.save !== "function") {
+    throw new Error(
+      `Retriever "${components.names.retriever}" does not persist an index. Use "file-store".`,
+    );
+  }
+
+  const stored = store.list();
+  if (stored.length === 0) {
+    throw new Error(
+      "The index is empty. Run `rag-pipeline ingest <path>` first.",
+    );
+  }
+  const previous = store.getEmbedderInfo?.();
+  if (previous && previous.name !== embedderName) {
+    throw new Error(
+      `The index was built with embedder "${previous.name}". ` +
+        "Re-run `rag-pipeline ingest <path> --rebuild` with the local embedder first.",
+    );
+  }
+
+  let corpus = stored.map((e) => e.content);
+  let trainedOn = "index";
+  if (inputPath) {
+    const documents = await readDocuments(inputPath, { baseDir: configDir });
+    corpus = documents.flatMap((d) => d.chunks);
+    if (corpus.length === 0) {
+      throw new Error(`No text found to train on in ${inputPath}`);
+    }
+    trainedOn = inputPath;
+  }
+
+  embedder.train(corpus);
+  const vectors = await embedAndStore({
+    embedder,
+    embedderName,
+    store,
+    entries: stored,
+    previous,
+  });
+
+  return {
+    trainedOn,
+    trainingDocuments: corpus.length,
+    vocabularySize: embedder.getVocabularySize?.() ?? embedder.dimensions,
+    previousVocabularySize: previous?.dimensions ?? null,
+    reembedded: vectors.length,
+    zeroVectors: vectors.filter((v) => v.vector.every((x) => x === 0)).length,
+  };
+}
+
+/**
+ * Build the configured components once and restore the embedder state saved
+ * with the index, so they can be reused across several retrieve()/query()
+ * calls (e.g. by `ai benchmark`).
+ *
+ * @returns {object} Components as returned by buildComponents()
+ */
+function loadComponents({ config, configDir }) {
+  const components = buildComponents(config, { configDir });
+  const embedder = requireStage(components, "embedder");
+  const retriever = requireStage(components, "retriever");
+  if (retriever.size && retriever.size() === 0) {
+    throw new Error(
+      "The index is empty. Run `rag-pipeline ingest <path>` first.",
+    );
+  }
+  restoreEmbedder(embedder, retriever);
+  return components;
+}
+
+/**
+ * Retrieval only (no llm): embed the question, retrieve and, if configured,
+ * rerank.
+ *
+ * @param {object} params
+ * @param {object} [params.components] - Prebuilt components (loadComponents)
+ * @param {object} [params.config]
+ * @param {string} [params.configDir]
+ * @param {string} params.question
+ * @param {number} [params.topK=3]
+ * @returns {Promise<Array<{id: string, score: number, content: string, metadata: object}>>}
+ */
+async function retrieve({ components, config, configDir, question, topK = 3 }) {
+  const c = components || loadComponents({ config, configDir });
+  const queryVector = await embedQuery(c.embedder, question);
+  let results = await c.retriever.retrieve({
+    query: question,
+    queryVector,
+    topK,
+  });
+  if (c.reranker) {
+    results = await c.reranker.rerank(question, results, { topK });
+  }
+  return results;
+}
+
+/**
+ * Answer a question from the index. Without an llm stage this returns the
+ * retrieved passages only (answer: null) instead of a placeholder answer.
+ *
+ * @param {object} params
+ * @param {object} params.config
+ * @param {string} params.configDir
+ * @param {string} params.question
+ * @param {number} [params.topK=3]
+ * @param {object} [params.components] - Prebuilt components (loadComponents)
+ * @returns {Promise<{answer: string|null, results: Array, citations?: object}>}
+ */
+async function query({ config, configDir, question, topK = 3, components }) {
+  const c = components || loadComponents({ config, configDir });
+
+  if (!c.llm) {
+    const results = await retrieve({ components: c, question, topK });
+    return { answer: null, results };
+  }
+
+  const pipeline = createRagPipeline({
+    embedder: c.embedder,
+    retriever: c.retriever,
+    llm: c.llm,
+    reranker: c.reranker,
+  });
+  const result = await pipeline.run({
+    query: question,
+    options: { topK, citations: true },
+  });
+  if (!result.success) throw new Error(result.error);
+  return {
+    answer: result.answer,
+    results: result.results,
+    citations: result.citations,
+  };
+}
+
+module.exports = {
+  ingest,
+  retrain,
+  query,
+  retrieve,
+  loadComponents,
+  restoreEmbedder,
+};

package/src/cli/shared/cli-utils.js

@@ -28,6 +28,23 @@ function parsePositiveInteger(value) {
   return parsed;
 }
 
+// Top-level commands offered by shell completion
+const COMMANDS = [
+  "init",
+  "ingest",
+  "query",
+  "run",
+  "serve",
+  "doctor",
+  "validate",
+  "hub",
+  "ai",
+  "plugin",
+  "config",
+  "info",
+  "completion",
+].join(" ");
+
 /**
  * Generate shell completion script
  */
@@ -36,7 +53,7 @@ function generateCompletionScript(shell) {
     case "bash":
       return `# RAG Pipeline bash completion
 _rag_pipeline_completions() {
-  COMPREPLY=($(compgen -W "init doctor ingest query plugin config info validate completion" -- "\${COMP_WORDS[COMP_CWORD]}"))
+  COMPREPLY=($(compgen -W "${COMMANDS}" -- "\${COMP_WORDS[COMP_CWORD]}"))
 }
 complete -F _rag_pipeline_completions rag-pipeline`;
 
@@ -45,13 +62,13 @@ complete -F _rag_pipeline_completions rag-pipeline`;
 #compdef rag-pipeline
 _rag_pipeline() {
   _arguments \\
-    '1:command:(init doctor ingest query plugin config info validate completion)'
+    '1:command:(${COMMANDS})'
 }
 _rag_pipeline "$@"`;
 
     case "fish":
       return `# RAG Pipeline fish completion
-complete -c rag-pipeline -n '__fish_use_subcommand' -a 'init doctor ingest query plugin config info validate completion'`;
+complete -c rag-pipeline -n '__fish_use_subcommand' -a '${COMMANDS}'`;
 
     default:
       return `# Completion not available for ${shell}`;
@@ -71,10 +88,10 @@ function getVersion() {
 function getExtendedHelp() {
   return `
 Examples:
-  rag-pipeline init --interactive          Initialize with wizard
-  rag-pipeline doctor --auto-fix           Diagnose and fix issues
-  rag-pipeline plugin search openai        Search for plugins
-  rag-pipeline config show plugins         Show plugin configuration
+  rag-pipeline init                        Create .ragrc.json (prompts in a terminal)
+  rag-pipeline ingest ./docs               Index a file or directory
+  rag-pipeline query "What is X?"          Ask a question about the indexed documents
+  rag-pipeline config show                 Show the current configuration
 
 For more information, visit: https://github.com/DevilsDev/rag-pipeline-utils
 `;