@devilsdev/rag-pipeline-utils 2.4.5 → 2.6.0

package/src/cli/handlers/serve-handler.js

@@ -0,0 +1,60 @@
+/**
+ * Serve command handler
+ * Runs DevServer: a local HTTP API (GET /health, POST /query) over the
+ * pipeline in .ragrc.json, reloading custom plugin modules on change.
+ */
+
+const path = require("path");
+const { loadCliConfig } = require("../shared/config-context.js");
+
+/**
+ * @param {object} globalOptions - Global CLI options from program.opts()
+ * @param {object} options - { port, host }
+ * @returns {Promise<object|undefined>} The started server (for tests)
+ */
+async function handleServe(globalOptions, options = {}) {
+  try {
+    // Validate the config up front so a broken file fails at startup.
+    const { config, configPath, configDir } = loadCliConfig(globalOptions);
+    const pluginFiles = Object.values(config.plugins || {})
+      .flatMap((group) => Object.values(group || {}))
+      .filter((p) => typeof p === "string")
+      .map((p) => path.resolve(configDir, p));
+
+    const { DevServer } = require("../../dev/dev-server.js");
+    const server = new DevServer({
+      port: options.port,
+      host: options.host,
+      configPath,
+      watchPaths: pluginFiles,
+      verbose: false,
+    });
+    await server.start();
+
+    console.log(`🚀 Serving ${path.basename(configPath)} at ${server.url}`);
+    console.log(`   GET  ${server.url}/health`);
+    console.log(`   POST ${server.url}/query  {"question": "...", "topK": 3}`);
+    if (
+      !["127.0.0.1", "localhost", "::1"].includes(options.host || "127.0.0.1")
+    ) {
+      console.warn(
+        `⚠️  Listening on ${options.host}: the API has no authentication and anyone who can reach this address can query your index.`,
+      );
+    }
+    console.log("Press Ctrl+C to stop.");
+
+    const shutdown = async () => {
+      await server.stop();
+      process.exit(0);
+    };
+    process.once("SIGINT", shutdown);
+    process.once("SIGTERM", shutdown);
+    return server;
+  } catch (error) {
+    console.error(`❌ Server failed to start: ${error.message}`);
+    process.exitCode = 1;
+    return undefined;
+  }
+}
+
+module.exports = { handleServe };

package/src/cli/services/bench-stats.js

@@ -0,0 +1,54 @@
+"use strict";
+
+/**
+ * Summary statistics for latency samples measured by `rag-pipeline ai
+ * benchmark`. Pure functions so the math can be unit-tested.
+ */
+
+/**
+ * Nearest-rank percentile: the smallest sample such that at least p% of the
+ * samples are less than or equal to it. Always returns an observed value.
+ *
+ * @param {number[]} samples - Unsorted samples (not modified)
+ * @param {number} p - Percentile in (0, 100]
+ * @returns {number}
+ */
+function percentile(samples, p) {
+  if (!Array.isArray(samples) || samples.length === 0) {
+    throw new Error("percentile() needs at least one sample");
+  }
+  if (!(p > 0 && p <= 100)) {
+    throw new Error(`Percentile must be in (0, 100], got ${p}`);
+  }
+  const sorted = [...samples].sort((a, b) => a - b);
+  const rank = Math.ceil((p / 100) * sorted.length);
+  return sorted[rank - 1];
+}
+
+/**
+ * @param {number[]} samples
+ * @returns {number} Arithmetic mean
+ */
+function mean(samples) {
+  if (!Array.isArray(samples) || samples.length === 0) {
+    throw new Error("mean() needs at least one sample");
+  }
+  return samples.reduce((sum, x) => sum + x, 0) / samples.length;
+}
+
+/**
+ * @param {number[]} samples - Latencies in milliseconds
+ * @returns {{count: number, mean: number, p50: number, p95: number, min: number, max: number}}
+ */
+function summarize(samples) {
+  return {
+    count: samples.length,
+    mean: mean(samples),
+    p50: percentile(samples, 50),
+    p95: percentile(samples, 95),
+    min: Math.min(...samples),
+    max: Math.max(...samples),
+  };
+}
+
+module.exports = { percentile, mean, summarize };

package/src/cli/services/benchmark.js

@@ -0,0 +1,161 @@
+"use strict";
+
+/**
+ * `rag-pipeline ai benchmark`: time real queries against the configured
+ * pipeline and index. Every number reported is measured here.
+ */
+
+const fs = require("fs");
+const { embedQuery } = require("../../utils/embed-query");
+const { loadComponents, query } = require("./rag-service");
+const { summarize } = require("./bench-stats");
+
+const SAMPLE_QUERIES = 5;
+const SAMPLE_WORDS = 8;
+
+/**
+ * Read queries from a file: a JSON array of strings, or one query per line.
+ * @param {string} file
+ * @returns {string[]}
+ */
+function readQueryFile(file) {
+  if (!fs.existsSync(file)) throw new Error(`Queries file not found: ${file}`);
+  const text = fs.readFileSync(file, "utf8");
+  const trimmed = text.trim();
+  let queries;
+  if (trimmed.startsWith("[")) {
+    let parsed;
+    try {
+      parsed = JSON.parse(trimmed);
+    } catch (error) {
+      throw new Error(
+        `Queries file ${file} is not valid JSON: ${error.message}`,
+      );
+    }
+    if (!parsed.every((q) => typeof q === "string")) {
+      throw new Error(`Queries file ${file} must be a JSON array of strings`);
+    }
+    queries = parsed;
+  } else {
+    queries = text.split(/\r?\n/);
+  }
+  queries = queries.map((q) => q.trim()).filter(Boolean);
+  if (queries.length === 0) throw new Error(`No queries found in ${file}`);
+  return queries;
+}
+
+/**
+ * Deterministic sample: the first words of the first few indexed chunks.
+ * @param {Array<{content: string}>} chunks
+ * @returns {string[]}
+ */
+function sampleQueries(chunks, count = SAMPLE_QUERIES) {
+  return chunks
+    .map((c) => String(c.content).split(/\s+/).filter(Boolean))
+    .filter((words) => words.length > 0)
+    .slice(0, count)
+    .map((words) => words.slice(0, SAMPLE_WORDS).join(" "));
+}
+
+const elapsedMs = (start) => Number(process.hrtime.bigint() - start) / 1e6;
+
+async function timed(fn) {
+  const start = process.hrtime.bigint();
+  const value = await fn();
+  return { value, ms: elapsedMs(start) };
+}
+
+/**
+ * @param {object} params
+ * @param {object} params.config
+ * @param {string} params.configDir
+ * @param {string} [params.queriesFile]
+ * @param {number} [params.runs=3] - Passes over the query set
+ * @param {number} [params.topK=3]
+ * @returns {Promise<object>} Measured report
+ */
+async function runBenchmark({
+  config,
+  configDir,
+  queriesFile,
+  runs = 3,
+  topK = 3,
+}) {
+  const setup = await timed(() => loadComponents({ config, configDir }));
+  const components = setup.value;
+  const { embedder, retriever, reranker, llm } = components;
+
+  const queries = queriesFile
+    ? readQueryFile(queriesFile)
+    : sampleQueries(retriever.list ? retriever.list() : []);
+  if (queries.length === 0) {
+    throw new Error(
+      "No queries: pass --queries <file> (the index has no text to sample from).",
+    );
+  }
+
+  // First query pays one-time costs (lazy init, JIT); reported, not counted.
+  const warmup = await timed(() =>
+    query({ components, question: queries[0], topK }),
+  );
+
+  const samples = { embed: [], retrieve: [], rerank: [], endToEnd: [] };
+  const resultCounts = [];
+  for (let run = 0; run < runs; run++) {
+    for (const question of queries) {
+      const embedded = await timed(() => embedQuery(embedder, question));
+      samples.embed.push(embedded.ms);
+      const retrieved = await timed(() =>
+        retriever.retrieve({
+          query: question,
+          queryVector: embedded.value,
+          topK,
+        }),
+      );
+      samples.retrieve.push(retrieved.ms);
+      if (reranker) {
+        const reranked = await timed(() =>
+          reranker.rerank(question, retrieved.value, { topK }),
+        );
+        samples.rerank.push(reranked.ms);
+      }
+      const e2e = await timed(() => query({ components, question, topK }));
+      samples.endToEnd.push(e2e.ms);
+      resultCounts.push(e2e.value.results.length);
+    }
+  }
+
+  const stages = {
+    embed: summarize(samples.embed),
+    retrieve: summarize(samples.retrieve),
+    ...(reranker && { rerank: summarize(samples.rerank) }),
+    endToEnd: summarize(samples.endToEnd),
+  };
+
+  return {
+    queries,
+    queriesSource: queriesFile || "sampled from indexed chunks",
+    runs,
+    topK,
+    setupMs: setup.ms,
+    warmupMs: warmup.ms,
+    stages,
+    includesGeneration: Boolean(llm),
+    results: {
+      min: Math.min(...resultCounts),
+      max: Math.max(...resultCounts),
+    },
+    environment: {
+      node: process.version,
+      platform: `${process.platform} ${process.arch}`,
+      chunks: retriever.size ? retriever.size() : null,
+      embedder: components.names.embedder,
+      dimensions: retriever.getEmbedderInfo?.()?.dimensions ?? null,
+      retriever: components.names.retriever,
+      reranker: components.names.reranker || null,
+      llm: components.names.llm || null,
+    },
+  };
+}
+
+module.exports = { runBenchmark, readQueryFile, sampleQueries };

package/src/cli/services/feedback.js

@@ -0,0 +1,155 @@
+"use strict";
+
+/**
+ * Relevance feedback for `rag-pipeline ai feedback` / `ai search`.
+ *
+ * Feedback is stored as JSON next to the index. Scoring is deterministic and
+ * explainable: every stored vote for a result's docId contributes
+ *
+ *   sign(vote) * VOTE_WEIGHT * jaccard(terms(question), terms(votedQuestion))
+ *
+ * and the summed adjustment is capped to +/- MAX_ADJUSTMENT before being added
+ * to the retrieval score. Votes on unrelated questions (no shared terms)
+ * contribute nothing.
+ */
+
+const fs = require("fs");
+const path = require("path");
+const { tokenize } = require("../../evaluate/scoring");
+
+const FORMAT_VERSION = 1;
+const VOTE_WEIGHT = 0.1;
+const MAX_ADJUSTMENT = 0.3;
+const RATINGS = ["up", "down"];
+
+/**
+ * Where feedback lives: next to a file-store index, else <configDir>/.rag.
+ *
+ * @param {object} retriever - Built retriever (FileVectorStore has `path`)
+ * @param {string} configDir
+ * @returns {string}
+ */
+function feedbackPath(retriever, configDir) {
+  const dir =
+    retriever && typeof retriever.path === "string"
+      ? path.dirname(retriever.path)
+      : path.join(configDir, ".rag");
+  return path.join(dir, "feedback.json");
+}
+
+/**
+ * @param {string} file
+ * @returns {{version: number, entries: Array<{question: string, docId: string, rating: string, at: string}>}}
+ */
+function loadFeedback(file) {
+  if (!fs.existsSync(file)) return { version: FORMAT_VERSION, entries: [] };
+  let parsed;
+  try {
+    parsed = JSON.parse(fs.readFileSync(file, "utf8"));
+  } catch (error) {
+    throw new Error(
+      `Feedback file ${file} is not valid JSON: ${error.message}`,
+    );
+  }
+  if (parsed.version !== FORMAT_VERSION || !Array.isArray(parsed.entries)) {
+    throw new Error(`Unsupported feedback file format in ${file}`);
+  }
+  return parsed;
+}
+
+/**
+ * Append one vote and write the file atomically (temp file, then rename).
+ *
+ * @param {string} file
+ * @param {{question: string, docId: string, rating: string}} vote
+ * @returns {{entry: object, total: number}}
+ */
+function addFeedback(file, { question, docId, rating }) {
+  if (!RATINGS.includes(rating)) {
+    throw new Error(`Rating must be one of: ${RATINGS.join(", ")}`);
+  }
+  if (!String(question || "").trim()) throw new Error("Question is empty");
+  const data = loadFeedback(file);
+  const entry = { question, docId, rating, at: new Date().toISOString() };
+  data.entries.push(entry);
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  const tmp = `${file}.${process.pid}.tmp`;
+  fs.writeFileSync(tmp, JSON.stringify(data, null, 2));
+  fs.renameSync(tmp, file);
+  return { entry, total: data.entries.length };
+}
+
+/**
+ * Jaccard similarity of the two texts' token sets (0..1).
+ * @param {string} a
+ * @param {string} b
+ * @returns {number}
+ */
+function termSimilarity(a, b) {
+  const sa = new Set(tokenize(a));
+  const sb = new Set(tokenize(b));
+  if (sa.size === 0 || sb.size === 0) return 0;
+  let shared = 0;
+  for (const t of sa) if (sb.has(t)) shared++;
+  return shared / (sa.size + sb.size - shared);
+}
+
+/**
+ * Re-score retrieval results with stored feedback.
+ *
+ * @param {Array<{id: string, score: number}>} results - Retrieval results
+ * @param {Array<{question: string, docId: string, rating: string}>} entries - Stored votes
+ * @param {string} question - Current question
+ * @param {object} [options]
+ * @param {number} [options.voteWeight=VOTE_WEIGHT]
+ * @param {number} [options.maxAdjustment=MAX_ADJUSTMENT]
+ * @returns {Array<object>} Results sorted by finalScore (ties keep retrieval
+ *   order), each with baseScore, adjustment, finalScore and `reasons`.
+ */
+function applyFeedback(results, entries, question, options = {}) {
+  const voteWeight = options.voteWeight ?? VOTE_WEIGHT;
+  const maxAdjustment = options.maxAdjustment ?? MAX_ADJUSTMENT;
+
+  const scored = results.map((result, rank) => {
+    const reasons = [];
+    let raw = 0;
+    for (const vote of entries) {
+      if (vote.docId !== result.id) continue;
+      const similarity = termSimilarity(question, vote.question);
+      if (similarity === 0) continue;
+      const contribution =
+        (vote.rating === "up" ? 1 : -1) * voteWeight * similarity;
+      raw += contribution;
+      reasons.push({
+        rating: vote.rating,
+        question: vote.question,
+        similarity,
+        contribution,
+      });
+    }
+    const adjustment = Math.max(-maxAdjustment, Math.min(maxAdjustment, raw));
+    const baseScore = typeof result.score === "number" ? result.score : 0;
+    return {
+      ...result,
+      baseScore,
+      adjustment,
+      finalScore: baseScore + adjustment,
+      reasons,
+      rank,
+    };
+  });
+
+  scored.sort((a, b) => b.finalScore - a.finalScore || a.rank - b.rank);
+  return scored.map(({ rank, ...rest }) => rest);
+}
+
+module.exports = {
+  feedbackPath,
+  loadFeedback,
+  addFeedback,
+  termSimilarity,
+  applyFeedback,
+  RATINGS,
+  VOTE_WEIGHT,
+  MAX_ADJUSTMENT,
+};

package/src/cli/services/instrument.js

@@ -0,0 +1,83 @@
+"use strict";
+
+/**
+ * Wraps pipeline components so the CLI can report per-stage timings
+ * (`query --trace`) and see the retrieved passages when streaming (the
+ * stream's final chunk carries citations, not results).
+ *
+ * Each wrapper forwards to the real component with `this` bound to it; only
+ * the stage methods are timed.
+ */
+
+const { performance } = require("perf_hooks");
+
+const STAGE_METHODS = {
+  embedder: { embedQuery: "embed", embed: "embed" },
+  retriever: { retrieve: "retrieve" },
+  reranker: { rerank: "rerank" },
+  llm: { generate: "generate", generateStream: "generate" },
+};
+
+async function* timedStream(source, record) {
+  const start = performance.now();
+  try {
+    yield* await source;
+  } finally {
+    record(performance.now() - start);
+  }
+}
+
+/**
+ * @param {Object} components - { embedder, retriever, reranker, llm }
+ * @returns {{ components: Object, timings: Array<{stage: string, ms: number}>, captured: { results: Array|null } }}
+ */
+function instrumentComponents(components) {
+  const timings = [];
+  const captured = { results: null };
+
+  const wrap = (kind, component) => {
+    if (!component) return component;
+    const methods = STAGE_METHODS[kind];
+    return new Proxy(component, {
+      get(target, prop, receiver) {
+        const value = Reflect.get(target, prop, receiver);
+        const stage = methods[prop];
+        if (typeof value !== "function" || !stage) return value;
+
+        if (prop === "generateStream") {
+          return (...args) =>
+            timedStream(value.apply(target, args), (ms) =>
+              timings.push({ stage, ms }),
+            );
+        }
+        return async (...args) => {
+          const start = performance.now();
+          try {
+            const out = await value.apply(target, args);
+            if (kind === "retriever" || kind === "reranker") {
+              // The reranker runs after the retriever, so its order wins.
+              captured.results = out;
+            }
+            return out;
+          } finally {
+            timings.push({ stage, ms: performance.now() - start });
+          }
+        };
+      },
+    });
+  };
+
+  return {
+    components: {
+      ...components,
+      embedder: wrap("embedder", components.embedder),
+      retriever: wrap("retriever", components.retriever),
+      reranker: wrap("reranker", components.reranker),
+      llm: wrap("llm", components.llm),
+    },
+    timings,
+    captured,
+  };
+}
+
+module.exports = { instrumentComponents };