@devilsdev/rag-pipeline-utils 2.5.0 → 2.6.0

package/src/core/stage-contracts.js

@@ -0,0 +1,36 @@
+'use strict';
+
+/**
+ * What each pipeline stage must provide, as createRagPipeline and the CLI
+ * use it. A plugin needs at least one of `anyOf`; a missing `recommended`
+ * method only matters for the command named in its note.
+ *
+ * Shared by validatePlugin() and the `rag-pipeline init` wizard. The
+ * stricter rules pluginRegistry.register() applies (metadata, and the
+ * methods in contracts/*.json) are checked by validatePlugin(..., {
+ * registry: true }).
+ */
+const STAGE_CONTRACTS = Object.freeze({
+  loader: {
+    anyOf: ['load'],
+    recommended: {},
+  },
+  embedder: {
+    anyOf: ['embed', 'embedQuery'],
+    recommended: { embed: '`ingest` calls embed() to embed documents' },
+  },
+  retriever: {
+    anyOf: ['retrieve'],
+    recommended: { store: '`ingest` calls store() to save vectors' },
+  },
+  reranker: {
+    anyOf: ['rerank'],
+    recommended: {},
+  },
+  llm: {
+    anyOf: ['generate'],
+    recommended: {},
+  },
+});
+
+module.exports = { STAGE_CONTRACTS };

package/src/core/validate-plugin.js

@@ -0,0 +1,85 @@
+'use strict';
+
+const { STAGE_CONTRACTS } = require('./stage-contracts');
+
+/**
+ * @typedef {Object} PluginValidationResult
+ * @property {boolean} valid - True when there are no errors
+ * @property {string[]} errors - Problems that stop the plugin from working
+ * @property {string[]} warnings - Methods some commands need, e.g. `store`
+ *   for `rag-pipeline ingest`
+ */
+
+/**
+ * Checks that an object can be used as a pipeline stage, without throwing,
+ * so plugin authors can assert on it in their own tests.
+ *
+ * By default it checks what createRagPipeline() calls. With
+ * `{ registry: true }` it also applies the rules pluginRegistry.register()
+ * enforces: `metadata` with a string `name` and `version` and a `type`
+ * equal to the stage, plus the methods in the stage's contract.
+ *
+ * @example
+ * const { validatePlugin } = require('@devilsdev/rag-pipeline-utils');
+ * const result = validatePlugin('llm', myLlm, { registry: true });
+ * expect(result.errors).toEqual([]);
+ *
+ * @param {'loader'|'embedder'|'retriever'|'reranker'|'llm'} stage
+ * @param {object} plugin - Plugin instance or object
+ * @param {Object} [options]
+ * @param {boolean} [options.registry=false] - Also check registry rules
+ * @returns {PluginValidationResult}
+ */
+function validatePlugin(stage, plugin, options = {}) {
+  const contract = STAGE_CONTRACTS[stage];
+  if (!contract) {
+    return {
+      valid: false,
+      errors: [
+        `Unknown stage "${stage}". Expected one of: ${Object.keys(STAGE_CONTRACTS).join(', ')}`,
+      ],
+      warnings: [],
+    };
+  }
+
+  if (
+    plugin === null ||
+    (typeof plugin !== 'object' && typeof plugin !== 'function')
+  ) {
+    return {
+      valid: false,
+      errors: [`A ${stage} plugin must be an object or class instance`],
+      warnings: [],
+    };
+  }
+
+  const has = (method) => typeof plugin[method] === 'function';
+  const errors = [];
+  const warnings = [];
+
+  if (!contract.anyOf.some(has)) {
+    errors.push(
+      `A ${stage} needs ${contract.anyOf.map((m) => `${m}()`).join(' or ')}`,
+    );
+  }
+  for (const [method, why] of Object.entries(contract.recommended)) {
+    if (!has(method)) warnings.push(`No ${method}() method: ${why}`);
+  }
+
+  if (options.registry) {
+    // Reuse the registry's own check so the two cannot drift. It stops at
+    // the first problem, so this reports one registry error at a time.
+    // eslint-disable-next-line global-require
+    const { PluginRegistry } = require('./plugin-registry');
+    try {
+      new PluginRegistry()._validatePluginContract(stage, plugin);
+    } catch (error) {
+      const message = `pluginRegistry.register(): ${error.message}`;
+      if (!errors.includes(message)) errors.push(message);
+    }
+  }
+
+  return { valid: errors.length === 0, errors, warnings };
+}
+
+module.exports = { validatePlugin };

package/src/debug/execution-tracer.js

@@ -28,10 +28,17 @@ function generateTraceId() {
 }
 
 /**
- * Truncate a value if its JSON representation exceeds maxSize bytes
- * @param {*} value - Value to truncate
+ * Snapshot a value for a trace: a JSON copy, or a truncated string when its
+ * JSON exceeds maxSize characters.
+ *
+ * Always a copy, never the live object: the traced run attaches the trace to
+ * its result (`result.trace`), so storing the result itself made the trace
+ * circular and exportTraces('json') threw. A copy also keeps later changes to
+ * the result out of the recorded trace.
+ *
+ * @param {*} value - Value to snapshot
  * @param {number} maxSize - Maximum serialized size in characters
- * @returns {*} Original value or truncated string
+ * @returns {*} JSON copy of the value, or a truncated string
  */
 function truncatePayload(value, maxSize) {
   if (value === undefined || value === null) {
@@ -43,8 +50,11 @@ function truncatePayload(value, maxSize) {
   } catch {
     return '[unserializable]';
   }
+  if (serialized === undefined) {
+    return undefined; // functions and symbols have no JSON form
+  }
   if (serialized.length <= maxSize) {
-    return value;
+    return JSON.parse(serialized);
   }
   return serialized.slice(0, maxSize) + '...[truncated]';
 }

package/src/dev/dev-server.js

@@ -356,7 +356,14 @@ ${this.options.watchPaths.map((p) => `║   - ${p.padEnd(56)}║`).join('\n')}
     const { config, configDir } = loadCliConfig({
       config: this.options.configPath,
     });
-    return query({ config, configDir, question, topK });
+    // The HTTP response keeps its shape: answer, results, citations.
+    const { timings, durationMs, ...result } = await query({
+      config,
+      configDir,
+      question,
+      topK,
+    });
+    return result;
   }
 
   /** @private */

package/src/dev/hot-reload.js

@@ -18,7 +18,8 @@
  */
 
 const { EventEmitter } = require('events');
-const chokidar = require('chokidar');
+// chokidar loads when watching starts, not on import.
+const loadChokidar = () => require('chokidar'); // eslint-disable-line global-require
 const path = require('path');
 const fs = require('fs');
 const {
@@ -170,7 +171,7 @@ class HotReloadManager extends EventEmitter {
    * @returns {Promise<void>}
    */
   async _watchPlugins() {
-    const watcher = chokidar.watch(this.options.watchPaths, {
+    const watcher = loadChokidar().watch(this.options.watchPaths, {
       ignored: toIgnoreMatcher(this.options.ignorePatterns),
       persistent: true,
       ignoreInitial: true,
@@ -214,7 +215,7 @@ class HotReloadManager extends EventEmitter {
       return;
     }
 
-    const watcher = chokidar.watch(this.options.configPath, {
+    const watcher = loadChokidar().watch(this.options.configPath, {
       persistent: true,
       ignoreInitial: true,
       awaitWriteFinish: {

package/src/enterprise/sso-integration.js

@@ -17,7 +17,8 @@
 
 const crypto = require('crypto');
 const { EventEmitter } = require('events');
-const jwt = require('jsonwebtoken');
+// jsonwebtoken loads on first use (about 90 ms otherwise, on every import).
+const loadJwt = () => require('jsonwebtoken'); // eslint-disable-line global-require
 const RateLimiter = require('../utils/rate-limiter');
 const { logger } = require('../utils/logger');
 const { JWTValidator } = require('../security/jwt-validator');
@@ -1386,7 +1387,7 @@ class OIDCProvider {
       if (!this._config.clientId) {
         throw new Error('clientId must be configured');
       }
-      const decoded = jwt.decode(idToken, { complete: true });
+      const decoded = loadJwt().decode(idToken, { complete: true });
       if (!decoded || !decoded.header || !decoded.payload) {
         throw new Error('malformed id_token');
       }
@@ -1410,7 +1411,7 @@ class OIDCProvider {
         throw new Error('id_token has no sub claim');
       }
 
-      const claims = jwt.verify(idToken, key, {
+      const claims = loadJwt().verify(idToken, key, {
         algorithms: [header.alg],
         issuer: this._config.issuer,
         audience: this._config.clientId,

package/src/evaluate/semantic-similarity.js

@@ -0,0 +1,79 @@
+'use strict';
+
+/**
+ * Semantic similarity between an answer and a reference answer, from the
+ * cosine similarity of their embeddings. Unlike BLEU and ROUGE it does not
+ * penalise a correct answer that is worded differently.
+ *
+ * Embeddings capture topic and meaning, not facts: "twenty days" and
+ * "thirty days" score as very similar. Use it next to the grounding checks
+ * (computeFaithfulness, checkAnswer), not instead of them.
+ */
+
+function cosine(a, b) {
+  if (
+    !Array.isArray(a) ||
+    !Array.isArray(b) ||
+    a.length !== b.length ||
+    a.length === 0
+  ) {
+    throw new Error(
+      `Embeddings must be non-empty arrays of the same length (got ${a && a.length} and ${b && b.length})`,
+    );
+  }
+  let dot = 0;
+  let na = 0;
+  let nb = 0;
+  for (let i = 0; i < a.length; i += 1) {
+    dot += a[i] * b[i];
+    na += a[i] * a[i];
+    nb += b[i] * b[i];
+  }
+  if (na === 0 || nb === 0) return 0;
+  return dot / (Math.sqrt(na) * Math.sqrt(nb));
+}
+
+async function embedText(embedder, text, signal) {
+  const options = signal ? { signal } : {};
+  if (typeof embedder.embedQuery === 'function') {
+    return embedder.embedQuery(text, options);
+  }
+  return embedder.embed(text, options);
+}
+
+/**
+ * @param {string} answer - The generated answer
+ * @param {string} reference - The reference (ground-truth) answer
+ * @param {Object} options
+ * @param {Object} options.embedder - Any embedder with embed() or embedQuery()
+ *   (a built-in connector, LocalEmbedder fitted to your corpus, an AI SDK
+ *   adapter, HashEmbedder from /testing)
+ * @param {AbortSignal} [options.signal]
+ * @returns {Promise<number>} Cosine similarity clamped to 0..1
+ */
+async function computeSemanticSimilarity(
+  answer,
+  reference,
+  { embedder, signal } = {},
+) {
+  if (typeof answer !== 'string' || typeof reference !== 'string') {
+    throw new TypeError('answer and reference must be strings');
+  }
+  if (
+    !embedder ||
+    (typeof embedder.embed !== 'function' &&
+      typeof embedder.embedQuery !== 'function')
+  ) {
+    throw new TypeError(
+      'computeSemanticSimilarity needs an embedder with embed() or embedQuery()',
+    );
+  }
+  if (answer.trim() === '' || reference.trim() === '') return 0;
+  const [a, b] = await Promise.all([
+    embedText(embedder, answer, signal),
+    embedText(embedder, reference, signal),
+  ]);
+  return Math.min(1, Math.max(0, cosine(a, b)));
+}
+
+module.exports = { computeSemanticSimilarity };

package/src/grounding/check-answer.js

@@ -0,0 +1,173 @@
+'use strict';
+
+/**
+ * Check an answer that any framework produced (LangChain, LlamaIndex, Mastra,
+ * the AI SDK, your own code) against the sources it was given: citations
+ * per sentence, groundedness, unsupported sentences and evaluation scores,
+ * without using createRagPipeline().
+ */
+
+const { CitationTracker } = require('../citation/citation-tracker');
+const { PipelineEvaluator } = require('../evaluate/pipeline-evaluator');
+const {
+  computeSemanticSimilarity,
+} = require('../evaluate/semantic-similarity');
+
+/**
+ * Text of one source in the shapes common frameworks use.
+ * @private
+ */
+function sourceText(source) {
+  if (source == null) return '';
+  if (typeof source === 'string') return source;
+  if (typeof source.content === 'string') return source.content; // this library
+  if (typeof source.pageContent === 'string') return source.pageContent; // LangChain Document
+  if (typeof source.text === 'string') return source.text; // LlamaIndex node, plain
+  if (source.node) return sourceText(source.node); // LlamaIndex NodeWithScore
+  if (typeof source.getContent === 'function')
+    return String(source.getContent());
+  return '';
+}
+
+/**
+ * An id for a source: its own id when it has one, otherwise its position.
+ * @private
+ */
+function sourceId(source, index) {
+  const candidates = [
+    source && source.id,
+    source && source.id_,
+    source && source.node && (source.node.id_ || source.node.id),
+    source && source.metadata && (source.metadata.id || source.metadata.source),
+  ];
+  const id = candidates.find(
+    (c) => typeof c === 'string' || typeof c === 'number',
+  );
+  return id !== undefined ? String(id) : String(index + 1);
+}
+
+/**
+ * Normalize sources to `{ id, content, metadata }`.
+ *
+ * Accepts strings, `{ id, content }` (this library), LangChain Documents
+ * (`{ pageContent, metadata }`), LlamaIndex nodes (`{ text }`) and
+ * `NodeWithScore` (`{ node, score }`).
+ *
+ * @param {Array<string|Object>} sources
+ * @returns {Array<{id: string, content: string, metadata: Object}>}
+ */
+function normalizeSources(sources) {
+  if (!Array.isArray(sources)) {
+    throw new TypeError('sources must be an array');
+  }
+  return sources
+    .map((source, index) => ({
+      id: sourceId(source, index),
+      content: sourceText(source),
+      metadata:
+        (source &&
+          (source.metadata || (source.node && source.node.metadata))) ||
+        {},
+    }))
+    .filter((s) => s.content.trim() !== '');
+}
+
+/**
+ * @typedef {Object} AnswerCheck
+ * @property {boolean} grounded - Every sentence is supported by a source
+ * @property {number} groundedness - Share of supported sentences, 0..1
+ * @property {string[]} unsupported - Sentences no source supports (this
+ *   includes a sentence whose numbers do not match its source)
+ * @property {Array<{sentence: string, sources: Array}>} citations - Sources
+ *   supporting each sentence
+ * @property {{scores: Object, details: Object, metadata: Object}|undefined} evaluation
+ *   - Faithfulness, relevance, context precision (and context recall when a
+ *   referenceAnswer is given); only when `evaluate` is not false
+ * @property {Array<{id: string, content: string, metadata: Object}>} sources
+ *   - The sources as they were checked
+ */
+
+/**
+ * Check an answer against the sources it was generated from.
+ *
+ * @example
+ * // A LangChain chain's output
+ * const check = await checkAnswer({
+ *   question: input,
+ *   answer: result.answer,
+ *   sources: result.context, // LangChain Documents
+ * });
+ * if (!check.grounded) console.warn(check.unsupported);
+ *
+ * @param {Object} params
+ * @param {string} params.answer - The generated answer
+ * @param {Array<string|Object>} params.sources - Documents the answer was
+ *   generated from (see normalizeSources for accepted shapes)
+ * @param {string} [params.question] - The question; needed for relevance
+ * @param {string} [params.referenceAnswer] - Ground truth, for context recall
+ *   (and semantic similarity when an embedder is given)
+ * @param {Object} [params.embedder] - With a referenceAnswer, adds
+ *   `evaluation.scores.semanticSimilarity` (computeSemanticSimilarity)
+ * @param {boolean} [params.evaluate=true] - Also compute evaluation scores
+ * @returns {Promise<AnswerCheck>}
+ */
+async function checkAnswer({
+  answer,
+  sources,
+  question,
+  referenceAnswer,
+  embedder,
+  evaluate = true,
+} = {}) {
+  if (typeof answer !== 'string') {
+    throw new TypeError('answer must be a string');
+  }
+  const docs = normalizeSources(sources);
+
+  // Models sometimes return nothing; that is an ungrounded answer, not an
+  // error. (CitationTracker rejects empty text.)
+  if (answer.trim() === '') {
+    return {
+      grounded: false,
+      groundedness: 0,
+      unsupported: [],
+      citations: [],
+      evaluation: undefined,
+      sources: docs,
+    };
+  }
+
+  const tracked = new CitationTracker().track(answer, docs);
+  const sentences =
+    (tracked.hallucinationReport && tracked.hallucinationReport.sentences) ||
+    [];
+  const unsupported = sentences
+    .filter((s) => s.classification !== 'grounded')
+    .map((s) => s.text);
+
+  let evaluation;
+  if (evaluate) {
+    evaluation = await new PipelineEvaluator().evaluate(
+      { query: question || '', answer, results: docs },
+      referenceAnswer ? { referenceAnswer } : {},
+    );
+    if (referenceAnswer && embedder) {
+      evaluation.scores.semanticSimilarity = await computeSemanticSimilarity(
+        answer,
+        referenceAnswer,
+        { embedder },
+      );
+    }
+  }
+
+  return {
+    grounded: sentences.length > 0 && unsupported.length === 0,
+    groundedness: tracked.groundednessScore,
+    unsupported,
+    citations: tracked.citations,
+    evaluation,
+    sources: docs,
+  };
+}
+
+module.exports = { checkAnswer, normalizeSources };

package/src/index.js

@@ -13,6 +13,7 @@ const { normalizeConfig } = require('./config/normalize-config');
 
 // Plugin system
 const pluginRegistry = require('./core/plugin-registry');
+const { validatePlugin } = require('./core/validate-plugin');
 
 // Utilities
 const logger = require('./utils/logger');
@@ -59,6 +60,7 @@ const {
   computeContextRecall,
 } = require('./evaluate/context-metrics');
 const { computeGroundedness } = require('./evaluate/groundedness');
+const { computeSemanticSimilarity } = require('./evaluate/semantic-similarity');
 const { PipelineEvaluator } = require('./evaluate/pipeline-evaluator');
 
 // Chunking
@@ -66,6 +68,10 @@ const { ChunkingEngine } = require('./chunking/chunking-engine');
 
 // Citation & Grounding
 const { CitationTracker } = require('./citation/citation-tracker');
+const { checkAnswer, normalizeSources } = require('./grounding/check-answer');
+const { createAiSdkAdapter } = require('./adapters/ai-sdk');
+const { MemoryCache } = require('./cache/memory-cache');
+const { RedisCache } = require('./cache/redis-cache');
 const {
   mapSentenceToSources,
   buildIDFWeights,
@@ -188,6 +194,7 @@ module.exports = {
 
   // Plugin system
   pluginRegistry,
+  validatePlugin,
 
   // Utilities
   logger,
@@ -225,6 +232,7 @@ module.exports = {
   checkNumericConsistency,
   extractNumbers,
   computeFaithfulness,
+  computeSemanticSimilarity,
   computeAnswerRelevance,
   computeContextPrecision,
   computeContextRecall,
@@ -236,6 +244,11 @@ module.exports = {
 
   // Citation & Grounding
   CitationTracker,
+  checkAnswer,
+  normalizeSources,
+  createAiSdkAdapter,
+  MemoryCache,
+  RedisCache,
   mapSentenceToSources,
   buildIDFWeights,
   detectHallucinations,
@@ -358,6 +371,15 @@ for (const name of Object.keys(experimental)) {
   });
 }
 
+// The ./testing build entry reads its module from here, so it shares the
+// main bundle's classes (MemoryRetriever, createRagPipeline). A getter, so
+// importing the package does not load the testing helpers.
+Object.defineProperty(
+  module.exports,
+  Symbol.for('@devilsdev/rag-pipeline-utils/testing'),
+  { get: () => require('./testing'), enumerable: false }, // eslint-disable-line global-require
+);
+
 // The ./experimental build entry reads the unwrapped classes from here, so
 // both entries share one copy of each class (non-enumerable: not a named export).
 Object.defineProperty(

package/src/loader/csv-loader.js

@@ -4,7 +4,8 @@
 
 const fs = require('fs/promises');
 const path = require('path');
-const { parse } = require('csv-parse/sync');
+// csv-parse loads when a CSV file is read, not on import.
+const loadCsvParse = () => require('csv-parse/sync'); // eslint-disable-line global-require
 
 class CSVLoader {
   /**
@@ -20,7 +21,7 @@ class CSVLoader {
       const src = raw.replace(/^\uFEFF/, ''); // Remove BOM
       const normalizedSrc = src.replace(/\r\n|\r/g, '\n'); // Normalize line endings
 
-      const records = parse(normalizedSrc, { columns: true });
+      const records = loadCsvParse().parse(normalizedSrc, { columns: true });
 
       return [
         {

package/src/loader/html-loader.js

@@ -4,7 +4,8 @@
 
 const fs = require('fs/promises');
 const path = require('path');
-const { JSDOM } = require('jsdom');
+// jsdom (about 100 packages) loads when an HTML file is read, not on import.
+const loadJsdom = () => require('jsdom'); // eslint-disable-line global-require
 
 /**
  * HTMLLoader reads HTML files and extracts visible text.
@@ -19,6 +20,7 @@ class HTMLLoader {
   async load(filePath) {
     const absPath = path.resolve(filePath);
     const raw = await fs.readFile(absPath, 'utf-8');
+    const { JSDOM } = loadJsdom();
     const dom = new JSDOM(raw);
     let text = dom.window.document.body.textContent || '';
 

package/src/loader/markdown-loader.js

@@ -10,7 +10,8 @@ const fs = require('fs/promises');
 // eslint-disable-line global-require
 const path = require('path');
 // eslint-disable-line global-require
-const { marked } = require('marked');
+// marked loads when a Markdown file is read, not on import.
+const loadMarked = () => require('marked'); // eslint-disable-line global-require
 // eslint-disable-line global-require
 
 /**
@@ -26,7 +27,7 @@ class MarkdownLoader {
   async load(_filePath) {
     const absPath = path.resolve(_filePath);
     const raw = await fs.readFile(absPath, 'utf-8');
-    const html = marked(raw);
+    const html = loadMarked().marked(raw);
     const text = html.replace(/<[^>]+>/g, ''); // strip tags
 
     return [