@devilsdev/rag-pipeline-utils 2.5.0 → 2.6.0

package/src/cli/enhanced-cli-commands.js

@@ -17,24 +17,19 @@ const {
   getExtendedHelp,
 } = require("./shared/cli-utils.js");
 
-// Handler modules
-const { handleRun } = require("./handlers/run-handler.js");
-const {
-  handleInit,
-  exitAfterConnectionTest,
-} = require("./handlers/init-handler.js");
-const {
-  handleDoctor,
-  handleValidate,
-  handleInfo,
-} = require("./handlers/diagnostics-handler.js");
-const {
-  handleConfigShow,
-  handleConfigSet,
-  handleConfigGet,
-} = require("./handlers/config-handler.js");
-const { handleIngest } = require("./handlers/ingest-handler.js");
-const { handleQuery } = require("./handlers/query-handler.js");
+// Handler modules load when their command runs, so `--help` or one command
+// does not pay for every other command's dependencies (inquirer, jsdom, ora,
+// marked, csv-parse...).
+/* eslint-disable global-require */
+const handlers = {
+  run: () => require("./handlers/run-handler.js"),
+  init: () => require("./handlers/init-handler.js"),
+  diagnostics: () => require("./handlers/diagnostics-handler.js"),
+  config: () => require("./handlers/config-handler.js"),
+  ingest: () => require("./handlers/ingest-handler.js"),
+  query: () => require("./handlers/query-handler.js"),
+};
+/* eslint-enable global-require */
 
 function parseNonNegativeInt(value) {
   const n = Number(value);
@@ -64,6 +59,7 @@ class EnhancedCLI {
       .version(getVersion(), "-v, --version", "Display version information")
       .option("--config <path>", "Configuration file path", ".ragrc.json")
       .option("--verbose", "Enable verbose output")
+      .option("--quiet", "Print only results, no status messages")
       .option("--no-color", "Disable colored output")
       .option("--dry-run", "Show what would be done without executing")
       .helpOption("-h, --help", "Display help information")
@@ -87,10 +83,12 @@ class EnhancedCLI {
       .option("--output <path>", "Output configuration file", ".ragrc.json")
       .option("--force", "Overwrite existing configuration")
       .action(async (options) => {
-        const result = await handleInit(this.program.opts(), options);
+        const result = await handlers
+          .init()
+          .handleInit(this.program.opts(), options);
         // After a connection test, exit even if a provider call left a
         // socket or timer open (see exitAfterConnectionTest).
-        exitAfterConnectionTest(result);
+        handlers.init().exitAfterConnectionTest(result);
       });
 
     this.program
@@ -102,7 +100,7 @@ class EnhancedCLI {
       .option("--top-k <n>", "Passages to retrieve", parsePositiveInteger, 3)
       .option("--json", "Print the query result as JSON")
       .action(async (options) => {
-        await handleRun(this.program.opts(), options);
+        await handlers.run().handleRun(this.program.opts(), options);
       });
 
     this.program
@@ -118,7 +116,7 @@ class EnhancedCLI {
       )
       .option("--report <path>", "Save the diagnostic report as JSON")
       .action(async (options) => {
-        await handleDoctor(this.program.opts(), options);
+        await handlers.diagnostics().handleDoctor(this.program.opts(), options);
       });
 
     this.program
@@ -127,7 +125,9 @@ class EnhancedCLI {
       .argument("<path>", "File or directory (.md, .html, .csv, .txt)")
       .option("--rebuild", "Discard the existing index before ingesting")
       .action(async (inputPath, options) => {
-        await handleIngest(this.program.opts(), inputPath, options);
+        await handlers
+          .ingest()
+          .handleIngest(this.program.opts(), inputPath, options);
       });
 
     this.program
@@ -136,15 +136,29 @@ class EnhancedCLI {
       .argument("<question>", "Question text")
       .option("--top-k <n>", "Passages to retrieve", parsePositiveInteger, 3)
       .option("--json", "Print the result as JSON")
+      .option("--streaming", "Print the answer as it is generated")
+      .option(
+        "--timeout <ms>",
+        "Abort the query after this many milliseconds",
+        parsePositiveInteger,
+      )
+      .option("--trace", "Show time spent in each stage")
+      .option(
+        "--explain",
+        "Show passage scores and which passages support each answer sentence",
+      )
+      .option("--stats", "Show latency and estimated tokens and cost")
       .action(async (question, options) => {
-        await handleQuery(this.program.opts(), question, options);
+        await handlers
+          .query()
+          .handleQuery(this.program.opts(), question, options);
       });
 
     this.program
       .command("validate")
       .description("Validate the configuration and resolve its components")
       .action(async () => {
-        await handleValidate(this.program.opts());
+        await handlers.diagnostics().handleValidate(this.program.opts());
       });
 
     this.program
@@ -177,7 +191,7 @@ class EnhancedCLI {
       .description("Show current configuration")
       .option("--section <section>", "Show specific configuration section")
       .action(async (options) => {
-        await handleConfigShow(this.program.opts(), options);
+        await handlers.config().handleConfigShow(this.program.opts(), options);
       });
 
     configCmd
@@ -186,7 +200,9 @@ class EnhancedCLI {
       .argument("<key>", "Configuration key (dot notation supported)")
       .argument("<value>", "Configuration value")
       .action(async (key, value, options) => {
-        await handleConfigSet(this.program.opts(), key, value);
+        await handlers
+          .config()
+          .handleConfigSet(this.program.opts(), key, value);
       });
 
     configCmd
@@ -194,7 +210,7 @@ class EnhancedCLI {
       .description("Get configuration value")
       .argument("<key>", "Configuration key (dot notation supported)")
       .action(async (key, options) => {
-        await handleConfigGet(this.program.opts(), key);
+        await handlers.config().handleConfigGet(this.program.opts(), key);
       });
 
     this.program.addCommand(configCmd);
@@ -207,7 +223,7 @@ class EnhancedCLI {
       .option("--system", "Show system information")
       .option("--config", "Show configuration summary")
       .action(async (options) => {
-        await handleInfo(this.program.opts(), options);
+        await handlers.diagnostics().handleInfo(this.program.opts(), options);
       });
 
     // Completion command for shell autocompletion

package/src/cli/handlers/query-handler.js

@@ -2,26 +2,146 @@
  * Query command handler
  * Answers a question from the ingested index. Without an llm stage it prints
  * the retrieved passages instead of an answer.
+ *
+ * Options: --top-k, --json, --streaming, --timeout, --trace (per-stage
+ * timings), --explain (passage scores and the passages supporting each
+ * answer sentence), --stats (latency, estimated tokens and cost). The global
+ * --quiet prints only the answer or the passages.
  */
 
 const { loadCliConfig } = require("../shared/config-context.js");
 const { query } = require("../services/rag-service.js");
+const { CostCalculator } = require("../../cost/cost-calculator.js");
 
-function printPassages(results) {
+const STAGES = ["embed", "retrieve", "rerank", "generate"];
+
+function sourceLabel(doc) {
+  return doc.metadata?.source || doc.id;
+}
+
+function printPassages(results, { quiet } = {}) {
   // A score of 0 means no overlap with the question at all.
   const matches = results.filter((doc) => !(doc.score <= 0));
   if (matches.length === 0) {
-    console.log("No matching passages found.");
+    if (!quiet) console.log("No matching passages found.");
     return;
   }
   matches.forEach((doc, i) => {
-    const source = doc.metadata?.source || doc.id;
     const score = typeof doc.score === "number" ? doc.score.toFixed(3) : "-";
-    console.log(`\n[${i + 1}] ${source} (score ${score})`);
+    console.log(`\n[${i + 1}] ${sourceLabel(doc)} (score ${score})`);
     console.log(doc.content);
   });
 }
 
+/** Sum the recorded timings per stage. */
+function stageTimings(timings) {
+  const totals = {};
+  for (const { stage, ms } of timings)
+    totals[stage] = (totals[stage] || 0) + ms;
+  return totals;
+}
+
+function printTrace(result) {
+  const totals = stageTimings(result.timings);
+  console.log("\nTrace");
+  for (const stage of STAGES) {
+    const ms = totals[stage];
+    console.log(
+      `  ${stage.padEnd(9)} ${ms === undefined ? "-" : `${ms.toFixed(1)} ms`}`,
+    );
+  }
+  console.log(`  ${"total".padEnd(9)} ${result.durationMs.toFixed(1)} ms`);
+}
+
+/** Rough token count: about 4 characters per token for English text. */
+const estimateTokens = (text) => Math.ceil(String(text || "").length / 4);
+
+function computeStats(result, question, llmModel) {
+  const context = result.results.map((d) => d.content || "").join("\n");
+  const stats = {
+    durationMs: Math.round(result.durationMs),
+    passages: result.results.length,
+    answerChars: result.answer ? result.answer.length : 0,
+  };
+  if (result.answer !== null) {
+    stats.estimatedInputTokens = estimateTokens(`${question}\n${context}`);
+    stats.estimatedOutputTokens = estimateTokens(result.answer);
+    const calculator = new CostCalculator();
+    // Only price models in the table; unknown models would print a warning.
+    if (llmModel && calculator.getModelPricing(llmModel)) {
+      const cost = calculator.estimate(
+        llmModel,
+        stats.estimatedInputTokens,
+        stats.estimatedOutputTokens,
+      );
+      stats.estimatedCostUsd = Number(cost.totalCost.toFixed(6));
+      stats.model = cost.model;
+    }
+  }
+  return stats;
+}
+
+function printStats(stats) {
+  console.log("\nStats");
+  console.log(`  latency   ${stats.durationMs} ms`);
+  console.log(`  passages  ${stats.passages}`);
+  if (stats.estimatedInputTokens !== undefined) {
+    console.log(
+      `  tokens    ~${stats.estimatedInputTokens} in, ~${stats.estimatedOutputTokens} out (estimated)`,
+    );
+  }
+  if (stats.estimatedCostUsd !== undefined) {
+    console.log(
+      `  cost      ~$${stats.estimatedCostUsd} (${stats.model}, estimated)`,
+    );
+  }
+}
+
+/** Passage rank and score, and the passages that support each sentence. */
+function explanation(result) {
+  const passages = result.results.map((doc, i) => ({
+    rank: i + 1,
+    source: sourceLabel(doc),
+    score: typeof doc.score === "number" ? doc.score : null,
+  }));
+  if (!result.citations) return { passages };
+  const rankOf = new Map(result.results.map((d, i) => [String(d.id), i + 1]));
+  const sentences = (result.citations.citations || []).map((c) => ({
+    sentence: c.sentence,
+    supportedBy: (c.sources || []).map((s) => ({
+      rank: rankOf.get(String(s.docId)) ?? s.docIndex + 1,
+      source: sourceLabel(result.results[s.docIndex] || { id: s.docId }),
+      support: s.score,
+    })),
+  }));
+  return {
+    passages,
+    sentences,
+    groundedness: result.citations.groundednessScore,
+  };
+}
+
+function printExplanation(explained) {
+  console.log("\nExplanation");
+  console.log("  Passages, in ranked order:");
+  for (const p of explained.passages) {
+    const score = p.score === null ? "-" : p.score.toFixed(3);
+    console.log(`    [${p.rank}] ${p.source}  score ${score}`);
+  }
+  if (!explained.sentences) return;
+  console.log("  Answer sentences:");
+  for (const s of explained.sentences) {
+    const support = s.supportedBy.length
+      ? s.supportedBy
+          .map((p) => `[${p.rank}] ${p.source} (${p.support.toFixed(2)})`)
+          .join(", ")
+      : "not supported by any passage";
+    console.log(`    "${s.sentence}"`);
+    console.log(`      -> ${support}`);
+  }
+  console.log(`  Groundedness: ${explained.groundedness.toFixed(2)}`);
+}
+
 /**
  * @param {object} globalOptions - Global CLI options from program.opts()
  * @param {string} question - Query text
@@ -31,30 +151,73 @@ async function handleQuery(globalOptions, question, options = {}) {
   try {
     const { config, configDir } = loadCliConfig(globalOptions);
     const topK = options.topK || 3;
+    const quiet = Boolean(globalOptions.quiet);
 
     if (globalOptions.dryRun) {
       console.log(`🧪 Dry run: would query the index for: ${question}`);
       return;
     }
 
-    const result = await query({ config, configDir, question, topK });
+    const hasLlm = (config.pipeline || []).some((s) => s.stage === "llm");
+    const streaming = Boolean(options.streaming && hasLlm && !options.json);
+    if (options.streaming && !hasLlm && !quiet && !options.json) {
+      console.log("--streaming needs an llm stage; showing passages instead.");
+    }
+
+    const result = await query({
+      config,
+      configDir,
+      question,
+      topK,
+      timeout: options.timeout,
+      ...(streaming && { onToken: (t) => process.stdout.write(t) }),
+    });
+
+    const llmStage = (config.pipeline || []).find((s) => s.stage === "llm");
+    const stats = options.stats
+      ? computeStats(result, question, llmStage?.config?.model)
+      : undefined;
+    const explained = options.explain ? explanation(result) : undefined;
 
     if (options.json) {
-      console.log(JSON.stringify(result, null, 2));
+      const { timings, durationMs, ...rest } = result;
+      console.log(
+        JSON.stringify(
+          {
+            ...rest,
+            ...(options.trace && {
+              trace: { stages: stageTimings(timings), totalMs: durationMs },
+            }),
+            ...(stats && { stats }),
+            ...(explained && { explanation: explained }),
+          },
+          null,
+          2,
+        ),
+      );
       return;
     }
 
     if (result.answer === null) {
-      console.log("No llm stage configured; showing retrieved passages.");
+      if (!quiet)
+        console.log("No llm stage configured; showing retrieved passages.");
+      printPassages(result.results, { quiet });
     } else {
-      console.log(result.answer);
-      console.log("\nSources:");
+      if (streaming) process.stdout.write("\n");
+      else console.log(result.answer);
+      if (!quiet) {
+        console.log("\nSources:");
+        printPassages(result.results);
+      }
     }
-    printPassages(result.results);
+
+    if (options.explain) printExplanation(explained);
+    if (options.trace) printTrace(result);
+    if (options.stats) printStats(stats);
   } catch (error) {
     console.error(`❌ Query failed: ${error.message}`);
     process.exitCode = 1;
   }
 }
 
-module.exports = { handleQuery };
+module.exports = { handleQuery, explanation, computeStats };

package/src/cli/services/instrument.js

@@ -0,0 +1,83 @@
+"use strict";
+
+/**
+ * Wraps pipeline components so the CLI can report per-stage timings
+ * (`query --trace`) and see the retrieved passages when streaming (the
+ * stream's final chunk carries citations, not results).
+ *
+ * Each wrapper forwards to the real component with `this` bound to it; only
+ * the stage methods are timed.
+ */
+
+const { performance } = require("perf_hooks");
+
+const STAGE_METHODS = {
+  embedder: { embedQuery: "embed", embed: "embed" },
+  retriever: { retrieve: "retrieve" },
+  reranker: { rerank: "rerank" },
+  llm: { generate: "generate", generateStream: "generate" },
+};
+
+async function* timedStream(source, record) {
+  const start = performance.now();
+  try {
+    yield* await source;
+  } finally {
+    record(performance.now() - start);
+  }
+}
+
+/**
+ * @param {Object} components - { embedder, retriever, reranker, llm }
+ * @returns {{ components: Object, timings: Array<{stage: string, ms: number}>, captured: { results: Array|null } }}
+ */
+function instrumentComponents(components) {
+  const timings = [];
+  const captured = { results: null };
+
+  const wrap = (kind, component) => {
+    if (!component) return component;
+    const methods = STAGE_METHODS[kind];
+    return new Proxy(component, {
+      get(target, prop, receiver) {
+        const value = Reflect.get(target, prop, receiver);
+        const stage = methods[prop];
+        if (typeof value !== "function" || !stage) return value;
+
+        if (prop === "generateStream") {
+          return (...args) =>
+            timedStream(value.apply(target, args), (ms) =>
+              timings.push({ stage, ms }),
+            );
+        }
+        return async (...args) => {
+          const start = performance.now();
+          try {
+            const out = await value.apply(target, args);
+            if (kind === "retriever" || kind === "reranker") {
+              // The reranker runs after the retriever, so its order wins.
+              captured.results = out;
+            }
+            return out;
+          } finally {
+            timings.push({ stage, ms: performance.now() - start });
+          }
+        };
+      },
+    });
+  };
+
+  return {
+    components: {
+      ...components,
+      embedder: wrap("embedder", components.embedder),
+      retriever: wrap("retriever", components.retriever),
+      reranker: wrap("reranker", components.reranker),
+      llm: wrap("llm", components.llm),
+    },
+    timings,
+    captured,
+  };
+}
+
+module.exports = { instrumentComponents };

package/src/cli/services/rag-service.js

@@ -11,6 +11,8 @@ const { buildComponents } = require("../../config/component-factory");
 const { readDocuments } = require("../../loader/document-reader");
 const { embedQuery } = require("../../utils/embed-query");
 const { createRagPipeline } = require("../../core/create-pipeline");
+const { performance } = require("perf_hooks");
+const { instrumentComponents } = require("./instrument");
 
 function requireStage(components, stage) {
   if (!components[stage]) {
@@ -284,15 +286,24 @@ async function retrieve({ components, config, configDir, question, topK = 3 }) {
  * @param {string} params.question
  * @param {number} [params.topK=3]
  * @param {object} [params.components] - Prebuilt components (loadComponents)
- * @returns {Promise<{answer: string|null, results: Array, citations?: object}>}
+ * @param {number} [params.timeout] - Abort the run after this many ms
+ * @param {function(string): void} [params.onToken] - Stream the answer: called
+ *   with each token (only with an llm stage)
+ * @returns {Promise<{answer: string|null, results: Array, citations?: object,
+ *   timings: Array<{stage: string, ms: number}>, durationMs: number}>}
  */
-async function query({ config, configDir, question, topK = 3, components }) {
-  const c = components || loadComponents({ config, configDir });
-
-  if (!c.llm) {
-    const results = await retrieve({ components: c, question, topK });
-    return { answer: null, results };
-  }
+async function query({
+  config,
+  configDir,
+  question,
+  topK = 3,
+  components,
+  timeout,
+  onToken,
+}) {
+  const loaded = components || loadComponents({ config, configDir });
+  const { components: c, timings, captured } = instrumentComponents(loaded);
+  const started = performance.now();
 
   const pipeline = createRagPipeline({
     embedder: c.embedder,
@@ -300,15 +311,43 @@ async function query({ config, configDir, question, topK = 3, components }) {
     llm: c.llm,
     reranker: c.reranker,
   });
+  const stream = Boolean(onToken && c.llm);
   const result = await pipeline.run({
     query: question,
-    options: { topK, citations: true },
+    options: {
+      topK,
+      citations: Boolean(c.llm),
+      ...(timeout && { timeout }),
+      ...(stream && { stream: true }),
+    },
   });
-  if (!result.success) throw new Error(result.error);
+  if (result && result.success === false) throw new Error(result.error);
+
+  if (stream) {
+    let answer = "";
+    let citations;
+    for await (const chunk of result) {
+      if (chunk.token) {
+        answer += chunk.token;
+        onToken(chunk.token);
+      }
+      if (chunk.done) citations = chunk.citations;
+    }
+    return {
+      answer,
+      results: captured.results || [],
+      citations,
+      timings,
+      durationMs: performance.now() - started,
+    };
+  }
+
   return {
-    answer: result.answer,
+    answer: c.llm ? result.answer : null,
     results: result.results,
-    citations: result.citations,
+    ...(result.citations && { citations: result.citations }),
+    timings,
+    durationMs: performance.now() - started,
   };
 }
 

package/src/cli/wizard/init-wizard.js

@@ -19,6 +19,8 @@ const {
 } = require("../../config/component-factory");
 const { embedQuery } = require("../../utils/embed-query");
 const { buildInitConfig, DEFAULT_INDEX_PATH } = require("./config-builder");
+// Methods a custom plugin must expose; shared with validatePlugin().
+const { STAGE_CONTRACTS } = require("../../core/stage-contracts");
 
 const CUSTOM = "custom";
 const NONE = "none";
@@ -58,23 +60,6 @@ const OPTION_LABELS = {
   baseURL: "server URL",
 };
 
-/**
- * Methods a custom plugin must expose (`anyOf`), and methods the CLI needs
- * for some commands (`recommended`, a warning when missing).
- */
-const STAGE_CONTRACTS = {
-  embedder: {
-    anyOf: ["embed", "embedQuery"],
-    recommended: { embed: "`ingest` calls embed() to embed documents" },
-  },
-  llm: { anyOf: ["generate"], recommended: {} },
-  retriever: {
-    anyOf: ["retrieve"],
-    recommended: { store: "`ingest` calls store() to save vectors" },
-  },
-  reranker: { anyOf: ["rerank"], recommended: {} },
-};
-
 const CUSTOM_CHOICE = {
   name: "custom plugin (path to a .js module)",
   value: CUSTOM,

package/src/config/enhanced-ragrc-schema.js

@@ -3,8 +3,6 @@
  * Single source of truth for configuration validation
  */
 
-const Ajv = require("ajv"); // eslint-disable-line global-require
-const addFormats = require("ajv-formats"); // eslint-disable-line global-require
 const { normalizeConfig } = require("./normalize-config");
 
 /**

package/src/config/validate-schema.js

@@ -5,7 +5,8 @@
  * File: /src/_config/validate-schema.js
  */
 
-const Ajv = require("ajv");
+// ajv loads when a schema is validated, not on import.
+const loadAjv = () => require("ajv"); // eslint-disable-line global-require
 // eslint-disable-line global-require
 
 /**
@@ -89,6 +90,7 @@ const pluginSchema = {
  * @returns {{ valid: boolean, errors?: any[] }}
  */
 function validateRagrcSchema(config) {
+  const Ajv = loadAjv();
   const ajv = new Ajv({ allErrors: true, strict: false });
   const validate = ajv.compile(ragrcSchema);
   const valid = validate(config);
@@ -101,6 +103,7 @@ function validateRagrcSchema(config) {
  * @returns {{ valid: boolean, errors?: any[] }}
  */
 function validatePluginSchema(config) {
+  const Ajv = loadAjv();
   const ajv = new Ajv({ allErrors: true, strict: false });
   const validate = ajv.compile(pluginSchema);
   const valid = validate(config);

package/src/connectors/anthropic-connector.js

@@ -1,6 +1,8 @@
 'use strict';
 
-const axios = require('axios');
+// axios loads on first use, so importing the package does not pay for it
+// (about 125 ms) unless this connector is constructed.
+const loadAxios = () => require('axios'); // eslint-disable-line global-require
 const { BaseConnector } = require('./base-connector');
 const {
   missingApiKeyError,
@@ -10,6 +12,7 @@ const {
   throwIfAborted,
 } = require('./connector-utils');
 const { buildRagPrompt } = require('./rag-prompt');
+const { componentCacheKey } = require('../cache/component-keys');
 
 const API_KEY_ENV = 'ANTHROPIC_API_KEY';
 
@@ -39,6 +42,23 @@ const DEFAULT_CONFIG = {
  * @extends BaseConnector
  */
 class AnthropicConnector extends BaseConnector {
+  /**
+   * Cache identity for createRagPipeline({ cache }).
+   * @param {'llm'} role
+   * @returns {string|null}
+   */
+  getCacheKey(role) {
+    if (role !== 'llm') return null;
+    const c = this.config;
+    return componentCacheKey('anthropic', {
+      baseURL: c.baseURL,
+      model: c.model,
+      temperature: c.temperature,
+      maxTokens: c.maxTokens,
+      systemPrompt: c.systemPrompt,
+    });
+  }
+
   /**
    * @param {Object} [options]
    * @param {string} [options.model='claude-3-5-sonnet-20241022'] - Claude model to use
@@ -58,7 +78,7 @@ class AnthropicConnector extends BaseConnector {
       ...options,
       apiKey: resolveApiKey(options, API_KEY_ENV),
     };
-    this.client = axios.create({
+    this.client = loadAxios().create({
       baseURL: this.config.baseURL,
       headers: {
         'x-api-key': this.config.apiKey,