@devilsdev/rag-pipeline-utils 2.4.5 → 2.6.0

package/src/cache/pipeline-cache.js

@@ -0,0 +1,260 @@
+'use strict';
+
+/**
+ * Caching for createRagPipeline({ cache }): three opt-in layers.
+ *
+ * - embeddings: the query vector, keyed by the embedder's identity and the
+ *   query text.
+ * - retrieval: the retriever's raw results, keyed by the retriever's state,
+ *   the query and topK. Cached before `filterResults` (guardrail ACLs) and
+ *   the reranker, so those still run on every request and one user's
+ *   permissions can never shape another user's results.
+ * - generation: the answer, keyed by the llm's identity, the query, the
+ *   exact documents it was given and the generation options. Skipped when
+ *   the temperature is above 0.
+ *
+ * A layer caches only when its component identifies itself through
+ * `getCacheKey(role)` or a `cacheKey` string (the built-ins do), so a cache
+ * can never return results for a different model, index or configuration.
+ * Only successful, completed work is stored: nothing from an aborted or
+ * timed-out run.
+ */
+
+const crypto = require('crypto');
+const { MemoryCache } = require('./memory-cache');
+
+const LAYERS = ['embeddings', 'retrieval', 'generation'];
+const DEFAULT_TTL_SECONDS = {
+  embeddings: 30 * 24 * 3600,
+  retrieval: 3600,
+  generation: 24 * 3600,
+};
+const ROLE = {
+  embeddings: 'embedder',
+  retrieval: 'retriever',
+  generation: 'llm',
+};
+
+/** Generation options that change the answer and so belong in its key. */
+const GENERATION_OPTIONS = [
+  'temperature',
+  'maxTokens',
+  'maxOutputTokens',
+  'topP',
+  'model',
+  'systemPrompt',
+  'seed',
+  'stopSequences',
+];
+
+function hash(value) {
+  return crypto
+    .createHash('sha256')
+    .update(JSON.stringify(value))
+    .digest('hex');
+}
+
+function isBackend(value) {
+  return (
+    value && typeof value.get === 'function' && typeof value.set === 'function'
+  );
+}
+
+/** A component's identity for a role, or null when it has none. */
+function componentKey(component, role) {
+  if (!component) return null;
+  if (typeof component.getCacheKey === 'function') {
+    const key = component.getCacheKey(role);
+    return typeof key === 'string' && key ? key : null;
+  }
+  return typeof component.cacheKey === 'string' && component.cacheKey
+    ? component.cacheKey
+    : null;
+}
+
+const warned = new Set();
+function warnOnce(code, message) {
+  if (warned.has(code)) return;
+  warned.add(code);
+  process.emitWarning(message, { type: 'RagCacheWarning', code });
+}
+
+/**
+ * Normalise the `cache` option of createRagPipeline.
+ *
+ * - `true`: all three layers in one in-memory cache
+ * - a CacheBackend: all three layers in it
+ * - `{ embeddings, retrieval, generation, ttlSeconds }`: per layer, each
+ *   `true` (in-memory), `false`/absent (off) or a CacheBackend;
+ *   `ttlSeconds: { embeddings, retrieval, generation }` overrides the
+ *   defaults (30 days, 1 hour, 24 hours)
+ *
+ * @returns {Object|null} Layer config, or null when caching is off
+ */
+function normalizeCacheOption(option) {
+  if (!option) return null;
+  const shared =
+    option === true ? new MemoryCache() : isBackend(option) ? option : null;
+  const layers = {};
+  for (const layer of LAYERS) {
+    let backend = null;
+    if (shared) {
+      backend = shared;
+    } else if (option[layer] === true) {
+      backend = new MemoryCache();
+    } else if (isBackend(option[layer])) {
+      backend = option[layer];
+    }
+    if (backend) {
+      layers[layer] = {
+        backend,
+        ttlSeconds:
+          (option.ttlSeconds && option.ttlSeconds[layer]) ??
+          DEFAULT_TTL_SECONDS[layer],
+        hits: 0,
+        misses: 0,
+        skipped: 0,
+      };
+    }
+  }
+  return Object.keys(layers).length ? layers : null;
+}
+
+/**
+ * Per-pipeline cache. Every method returns undefined and does nothing for a
+ * layer that is off or whose component has no cache key.
+ */
+class PipelineCache {
+  /**
+   * @param {Object} layers - normalizeCacheOption() output
+   * @param {{ embedder?: Object, retriever?: Object, llm?: Object }} components
+   */
+  constructor(layers, components) {
+    this.layers = layers;
+    this.components = components;
+  }
+
+  /** @private */
+  _key(layer, parts) {
+    const config = this.layers[layer];
+    if (!config) return null;
+    const component = this.components[ROLE[layer]];
+    const id = componentKey(component, ROLE[layer]);
+    if (!id) {
+      config.skipped += 1;
+      warnOnce(
+        `RAG_CACHE_NO_KEY_${layer.toUpperCase()}`,
+        `The ${layer} cache is off for this ${ROLE[layer]}: it has no getCacheKey(role) method or cacheKey string, so cached results could not be told apart from another configuration's. Add one to enable it.`,
+      );
+      return null;
+    }
+    return `${layer}:${hash([id, parts])}`;
+  }
+
+  /** @private */
+  async _get(layer, key, status) {
+    if (!key) {
+      status[layer] = 'skip';
+      return undefined;
+    }
+    const config = this.layers[layer];
+    let value;
+    try {
+      value = await config.backend.get(key);
+    } catch (error) {
+      // A cache outage must not fail the run.
+      warnOnce('RAG_CACHE_READ_FAILED', `Cache read failed: ${error.message}`);
+      value = undefined;
+    }
+    if (value === undefined) {
+      config.misses += 1;
+      status[layer] = 'miss';
+    } else {
+      config.hits += 1;
+      status[layer] = 'hit';
+    }
+    return value;
+  }
+
+  /** @private */
+  async _set(layer, key, value) {
+    if (!key || value === undefined) return;
+    const config = this.layers[layer];
+    try {
+      await config.backend.set(key, value, { ttlSeconds: config.ttlSeconds });
+    } catch (error) {
+      warnOnce(
+        'RAG_CACHE_WRITE_FAILED',
+        `Cache write failed: ${error.message}`,
+      );
+    }
+  }
+
+  embeddingKey(query) {
+    return this._key('embeddings', { query });
+  }
+
+  retrievalKey({ query, queryVector, topK }) {
+    // Keyed on the query vector, not just the text, so a different embedder
+    // (or a retrained one) never gets another embedder's results; the text
+    // stays in the key for keyword and hybrid retrievers.
+    return this._key('retrieval', {
+      query: query ?? null,
+      vector: queryVector ? hash(queryVector) : null,
+      topK,
+    });
+  }
+
+  /**
+   * @returns {string|null} The key, or null to skip (temperature above 0,
+   *   or no llm cache key)
+   */
+  generationKey({ query, docs, options }) {
+    if (!this.layers.generation) return null;
+    const llm = this.components.llm;
+    const temperature =
+      options.temperature ?? (llm && llm.config && llm.config.temperature);
+    if (typeof temperature === 'number' && temperature > 0) {
+      this.layers.generation.skipped += 1;
+      return null;
+    }
+    const generationOptions = {};
+    for (const name of GENERATION_OPTIONS) {
+      if (options[name] !== undefined) generationOptions[name] = options[name];
+    }
+    const context = (docs || []).map((d) =>
+      typeof d === 'string' ? d : [d.id ?? null, d.content ?? d.text ?? ''],
+    );
+    return this._key('generation', {
+      query,
+      context: hash(context),
+      options: generationOptions,
+    });
+  }
+
+  get(layer, key, status) {
+    if (!this.layers[layer]) return Promise.resolve(undefined);
+    return this._get(layer, key, status);
+  }
+
+  set(layer, key, value) {
+    if (!this.layers[layer]) return Promise.resolve();
+    return this._set(layer, key, value);
+  }
+
+  /** Hits, misses and skips per layer since the pipeline was created. */
+  stats() {
+    const out = {};
+    for (const [layer, c] of Object.entries(this.layers)) {
+      out[layer] = { hits: c.hits, misses: c.misses, skipped: c.skipped };
+    }
+    return out;
+  }
+}
+
+module.exports = {
+  normalizeCacheOption,
+  PipelineCache,
+  componentKey,
+  DEFAULT_TTL_SECONDS,
+};

package/src/cache/redis-cache.js

@@ -0,0 +1,83 @@
+'use strict';
+
+/**
+ * CacheBackend on Redis. Pass a connected client from `redis` (node-redis 4
+ * and later) or `ioredis`; neither is a dependency of this package.
+ *
+ * Values are stored as JSON under `prefix + key`, with a millisecond expiry
+ * when a time to live applies.
+ */
+class RedisCache {
+  /**
+   * @param {Object} options
+   * @param {Object} options.client - Connected node-redis or ioredis client
+   * @param {string} [options.prefix='rag:'] - Key prefix
+   * @param {number} [options.ttlSeconds] - Default time to live
+   */
+  constructor({ client, prefix = 'rag:', ttlSeconds } = {}) {
+    if (
+      !client ||
+      typeof client.get !== 'function' ||
+      typeof client.set !== 'function'
+    ) {
+      throw new Error(
+        'RedisCache needs a connected client from "redis" or "ioredis"',
+      );
+    }
+    this.client = client;
+    this.prefix = prefix;
+    this.ttlSeconds = ttlSeconds;
+    // node-redis 4+ exposes isOpen and takes set(key, value, { PX }); ioredis
+    // takes set(key, value, 'PX', ms).
+    this._nodeRedis = typeof client.isOpen === 'boolean';
+    this._hits = 0;
+    this._misses = 0;
+  }
+
+  /**
+   * @param {string} key
+   * @returns {Promise<unknown|undefined>}
+   */
+  async get(key) {
+    const raw = await this.client.get(this.prefix + key);
+    if (raw === null || raw === undefined) {
+      this._misses += 1;
+      return undefined;
+    }
+    this._hits += 1;
+    return JSON.parse(raw);
+  }
+
+  /**
+   * @param {string} key
+   * @param {unknown} value - JSON-serialisable value
+   * @param {{ ttlSeconds?: number }} [options]
+   */
+  async set(key, value, { ttlSeconds = this.ttlSeconds } = {}) {
+    const json = JSON.stringify(value);
+    if (json === undefined) return;
+    const k = this.prefix + key;
+    const ms = ttlSeconds > 0 ? Math.round(ttlSeconds * 1000) : 0;
+    if (!ms) {
+      await this.client.set(k, json);
+    } else if (this._nodeRedis) {
+      await this.client.set(k, json, { PX: ms });
+    } else {
+      await this.client.set(k, json, 'PX', ms);
+    }
+  }
+
+  /** @param {string} key */
+  async delete(key) {
+    const del = this.client.del || this.client.unlink;
+    if (typeof del === 'function')
+      await del.call(this.client, this.prefix + key);
+  }
+
+  /** Hits and misses seen by this instance (size is not tracked). */
+  async stats() {
+    return { hits: this._hits, misses: this._misses };
+  }
+}
+
+module.exports = { RedisCache };

package/src/citation/citation-tracker.js

@@ -10,7 +10,7 @@ const { detectHallucinations } = require('./hallucination-detector');
  * @type {object}
  */
 const DEFAULT_CONFIG = {
-  similarityThreshold: 0.3,
+  similarityThreshold: 0.5,
   maxCitationsPerSentence: 3,
   useIDFWeighting: true,
 };
@@ -24,7 +24,7 @@ const DEFAULT_CONFIG = {
 class CitationTracker extends EventEmitter {
   /**
    * @param {object} [options] - Override default configuration
-   * @param {number} [options.similarityThreshold=0.3] - Minimum similarity for a citation match
+   * @param {number} [options.similarityThreshold=0.5] - Minimum similarity for a citation match
    * @param {number} [options.maxCitationsPerSentence=3] - Max citations per sentence
    * @param {boolean} [options.useIDFWeighting=true] - Whether to apply IDF weighting
    */
@@ -76,9 +76,11 @@ class CitationTracker extends EventEmitter {
         'CitationTracker.track(): answer must be a non-empty string',
       );
     }
-    if (!Array.isArray(retrievedDocs) || retrievedDocs.length === 0) {
+    // An empty array is valid (retrieval found nothing): every sentence is
+    // then reported as unsupported rather than failing the whole run.
+    if (!Array.isArray(retrievedDocs)) {
       throw new Error(
-        'CitationTracker.track(): retrievedDocs must be a non-empty array',
+        'CitationTracker.track(): retrievedDocs must be an array',
       );
     }
 

package/src/citation/hallucination-detector.js

@@ -10,7 +10,7 @@
  *
  * @param {Array<{sentence: string, sources: Array<{docId: string|undefined, docIndex: number, score: number}>}>} citationResults
  * @param {object} [options]
- * @param {number} [options.threshold=0.3] - Score threshold for grounded classification
+ * @param {number} [options.threshold=0.5] - Score threshold for grounded classification
  * @returns {{
  *   hallucinationRate: number,
  *   sentences: Array<{text: string, classification: string, bestScore: number, bestSource: object|null}>,
@@ -18,7 +18,7 @@
  * }}
  */
 function detectHallucinations(citationResults, options = {}) {
-  const { threshold = 0.3 } = options;
+  const { threshold = 0.5 } = options;
 
   if (!Array.isArray(citationResults) || citationResults.length === 0) {
     return {

package/src/citation/source-mapper.js

@@ -1,10 +1,12 @@
 'use strict';
 
-const { tokenize, computeJaccardSimilarity } = require('../evaluate/scoring');
+const { tokenize, computeSupportScore } = require('../evaluate/scoring');
 
 /**
  * Build IDF (Inverse Document Frequency) weights from a collection of documents.
- * IDF = log(totalDocs / docsContainingToken)
+ * IDF = log(1 + totalDocs / docsContainingToken). The +1 keeps weights
+ * positive, so tokens found in every document (and every token when only one
+ * document was retrieved) still count instead of weighing zero.
  * @param {Array<{content?: string, text?: string}>} docs - Array of document objects
  * @returns {Map<string, number>} Map of token to IDF weight
  */
@@ -24,24 +26,25 @@ function buildIDFWeights(docs) {
 
   const idfWeights = new Map();
   for (const [token, df] of docFrequency) {
-    idfWeights.set(token, Math.log(totalDocs / df));
+    idfWeights.set(token, Math.log(1 + totalDocs / df));
   }
 
   return idfWeights;
 }
 
 /**
- * Map a sentence to its most relevant source documents using token similarity.
+ * Map a sentence to its most relevant source documents by how much of the
+ * sentence each document supports (see computeSupportScore).
  * @param {string} sentence - The sentence to map
  * @param {Array<{content?: string, text?: string, id?: string}>} docs - Source documents
  * @param {object} [options] - Configuration options
  * @param {number} [options.maxCitations=3] - Maximum number of citations to return
- * @param {number} [options.threshold=0.3] - Minimum similarity threshold
+ * @param {number} [options.threshold=0.5] - Minimum similarity threshold
  * @param {Map<string, number>|null} [options.idfWeights=null] - Optional IDF weights for weighted similarity
  * @returns {Array<{docIndex: number, docId: string|undefined, score: number, matchedTokens: string[]}>}
  */
 function mapSentenceToSources(sentence, docs, options = {}) {
-  const { maxCitations = 3, threshold = 0.3, idfWeights = null } = options;
+  const { maxCitations = 3, threshold = 0.5, idfWeights = null } = options;
 
   const sentenceTokens = tokenize(sentence);
   if (!sentenceTokens.length) return [];
@@ -54,29 +57,13 @@ function mapSentenceToSources(sentence, docs, options = {}) {
     const docTokens = tokenize(docText);
     if (!docTokens.length) continue;
 
-    let score;
-    const sentenceSet = new Set(sentenceTokens);
+    // Share of the sentence's content tokens found in this document.
     const docSet = new Set(docTokens);
-    const matched = [...sentenceSet].filter((t) => docSet.has(t));
-
-    if (idfWeights) {
-      // IDF-weighted Jaccard: sum IDF weights of intersection / sum IDF weights of union
-      const union = new Set([...sentenceSet, ...docSet]);
-      let intersectionWeight = 0;
-      let unionWeight = 0;
-
-      for (const token of union) {
-        const weight = idfWeights.get(token) || 1;
-        if (sentenceSet.has(token) && docSet.has(token)) {
-          intersectionWeight += weight;
-        }
-        unionWeight += weight;
-      }
-
-      score = unionWeight > 0 ? intersectionWeight / unionWeight : 0;
-    } else {
-      score = computeJaccardSimilarity(sentenceTokens, docTokens);
-    }
+    const score = computeSupportScore(sentenceTokens, docSet, idfWeights, {
+      claimText: sentence,
+      docText,
+    });
+    const matched = [...new Set(sentenceTokens)].filter((t) => docSet.has(t));
 
     if (score >= threshold) {
       results.push({

package/src/cli/commands/ai.js

@@ -0,0 +1,106 @@
+"use strict";
+
+/**
+ * `rag-pipeline ai ...`: training, benchmarking, relevance feedback and file
+ * inspection built on the real pipeline components (.ragrc.json + index).
+ */
+
+const { Command, Option } = require("commander");
+const { parsePositiveInteger } = require("../shared/cli-utils.js");
+// The handler (document loaders: jsdom, marked, csv-parse) loads when an
+// ai command runs, not when the CLI starts.
+const aiHandler = () => require("../handlers/ai-handler.js"); // eslint-disable-line global-require
+
+/**
+ * @param {() => object} getGlobalOptions - Returns program.opts()
+ * @returns {Command}
+ */
+function createAiCommand(getGlobalOptions) {
+  const ai = new Command("ai").description(
+    "Train the local embedder, benchmark queries, relevance feedback, file inspection",
+  );
+
+  const train = ai
+    .command("train")
+    .description("Fit corpus-trained components");
+  train
+    .command("embedding")
+    .description(
+      "Refit the local TF-IDF embedder (on the index, or on [path]) and re-embed the index",
+    )
+    .argument(
+      "[path]",
+      "Train on documents from this file or directory instead",
+    )
+    .action(async (inputPath) => {
+      await aiHandler().handleTrainEmbedding(getGlobalOptions(), inputPath);
+    });
+
+  ai.command("benchmark")
+    .description(
+      "Measure query latency against the configured pipeline and index",
+    )
+    .option(
+      "--queries <file>",
+      "Queries file: JSON array of strings or one per line (default: sampled from the index)",
+    )
+    .option("--runs <n>", "Passes over the query set", parsePositiveInteger, 3)
+    .option("--top-k <n>", "Passages to retrieve", parsePositiveInteger, 3)
+    .option("--json", "Print the measurements as JSON")
+    .action(async (options) => {
+      await aiHandler().handleBenchmark(getGlobalOptions(), options);
+    });
+
+  ai.command("feedback")
+    .description(
+      "Record whether an indexed chunk was a good result for a question",
+    )
+    .argument("<question>", "Question the chunk was returned for")
+    .argument("<docId>", "Chunk id as shown by `ai search` (e.g. docs/a.md#0)")
+    .addOption(
+      new Option("--rating <rating>", "Relevance vote")
+        .choices(["up", "down"])
+        .makeOptionMandatory(),
+    )
+    .action(async (question, docId, options) => {
+      await aiHandler().handleFeedback(
+        getGlobalOptions(),
+        question,
+        docId,
+        options,
+      );
+    });
+
+  ai.command("search")
+    .description(
+      "Retrieve passages re-scored with stored feedback (retrieval only)",
+    )
+    .argument("<question>", "Question text")
+    .option("--top-k <n>", "Passages to return", parsePositiveInteger, 3)
+    .option("--json", "Print the results as JSON")
+    .action(async (question, options) => {
+      await aiHandler().handleSearch(getGlobalOptions(), question, options);
+    });
+
+  const multimodal = ai
+    .command("multimodal")
+    .description("Inspect files before ingesting");
+  multimodal
+    .command("process")
+    .description(
+      "Report type, characters and chunks for text files (images need a vision model)",
+    )
+    .argument("<path>", "File or directory")
+    .option("--json", "Print the report as JSON")
+    .action(async (inputPath, options) => {
+      await aiHandler().handleMultimodalProcess(
+        getGlobalOptions(),
+        inputPath,
+        options,
+      );
+    });
+
+  return ai;
+}
+
+module.exports = { createAiCommand };

package/src/cli/commands/plugin-hub/event-handlers.js

@@ -6,6 +6,12 @@
 const chalk = require("chalk");
 
 function setupEventListeners(hub, certification) {
+  // The hub emits 'error' and then throws a descriptive error that each
+  // command reports. Without a listener, EventEmitter would throw its own
+  // "Unhandled error" first and hide that message.
+  hub.on("error", () => {});
+  certification.on("error", () => {});
+
   // Hub events
   hub.on("install_start", (data) => {
     console.log(

package/src/cli/commands/plugin-hub/hub-commands.js

@@ -69,34 +69,6 @@ async function searchPlugins(hub, query, _options) {
   }
 }
 
-async function installPlugin(hub, plugin, _options) {
-  try {
-    const installOptions = {
-      securityScan: _options.securityScan,
-      requireCertified: _options.requireCertified,
-      sandboxTimeout: _options.sandboxTimeout,
-    };
-
-    const result = await hub.installPlugin(
-      plugin,
-      _options.version,
-      installOptions,
-    );
-
-    console.log(chalk.green("\n✅ Installation completed successfully!")); // eslint-disable-line no-console
-    console.log(chalk.blue(`Plugin: ${result.pluginInfo.name}`)); // eslint-disable-line no-console
-    console.log(chalk.blue(`Version: ${result.version}`)); // eslint-disable-line no-console
-    console.log(chalk.blue(`Install Path: ${result.installPath}`)); // eslint-disable-line no-console
-
-    if (result.pluginInfo.certified) {
-      console.log(chalk.green("🏆 This plugin is certified!")); // eslint-disable-line no-console
-    }
-  } catch (error) {
-    console.error(chalk.red(`Installation failed: ${error.message}`)); // eslint-disable-line no-console
-    process.exit(1);
-  }
-}
-
 async function getPluginInfo(hub, plugin) {
   const spinner = ora("Fetching plugin information...").start();
 
@@ -170,50 +142,6 @@ async function getPluginInfo(hub, plugin) {
   }
 }
 
-async function listInstalledPlugins(hub, _options) {
-  const spinner = ora("Loading installed plugins...").start();
-
-  try {
-    const installed = await hub.getInstalledPlugins();
-    spinner.stop();
-
-    if (installed.length === 0) {
-      console.log(chalk.yellow("No plugins installed.")); // eslint-disable-line no-console
-      return;
-    }
-
-    if (_options.format === "json") {
-      console.log(JSON.stringify(installed, null, 2)); // eslint-disable-line no-console
-      return;
-    }
-
-    console.log(chalk.green(`${installed.length} plugins installed:\n`)); // eslint-disable-line no-console
-
-    const table = new Table({
-      head: ["Name", "Version", "Type", "Last Used", "Status"],
-      colWidths: [25, 12, 15, 15, 10],
-    });
-
-    for (const plugin of installed) {
-      const lastUsed = plugin.lastUsed
-        ? new Date(plugin.lastUsed).toLocaleDateString()
-        : chalk.gray("Never");
-
-      const status = plugin.certified
-        ? chalk.green("Certified")
-        : chalk.gray("Standard");
-
-      table.push([plugin.name, plugin.version, plugin._type, lastUsed, status]);
-    }
-
-    console.log(table.toString()); // eslint-disable-line no-console
-  } catch (error) {
-    spinner.stop();
-    console.error(chalk.red(`Failed to list plugins: ${error.message}`)); // eslint-disable-line no-console
-    process.exit(1);
-  }
-}
-
 async function publishPlugin(hub, pluginPath, _options) {
   if (_options.dryRun) {
     console.log(
@@ -282,9 +210,7 @@ async function getTrendingPlugins(hub, _options) {
 
 module.exports = {
   searchPlugins,
-  installPlugin,
   getPluginInfo,
-  listInstalledPlugins,
   publishPlugin,
   getTrendingPlugins,
 };