@devilsdev/rag-pipeline-utils 2.4.5 → 2.6.0

package/src/evaluate/context-metrics.js

@@ -1,10 +1,6 @@
 'use strict';
 
-const {
-  tokenize,
-  splitSentences,
-  computeJaccardSimilarity,
-} = require('./scoring');
+const { tokenize, splitSentences, computeSupportScore } = require('./scoring');
 
 /**
  * Compute context precision: the fraction of retrieved documents that are relevant to the query.
@@ -12,7 +8,7 @@ const {
  * @param {string} query - The user query
  * @param {Array<{content: string}>} retrievedDocs - Array of retrieved documents
  * @param {object} [options={}] - Configuration options
- * @param {number} [options.threshold=0.2] - Minimum Jaccard similarity for a doc to be considered relevant
+ * @param {number} [options.threshold=0.2] - Minimum share of query content terms a doc must contain to be considered relevant
  * @returns {{ score: number, relevantDocs: number, totalDocs: number, perDocScores: Array<{ index: number, score: number, isRelevant: boolean }> }}
  */
 function computeContextPrecision(query, retrievedDocs, options = {}) {
@@ -39,7 +35,8 @@ function computeContextPrecision(query, retrievedDocs, options = {}) {
     }
 
     const docTokens = tokenize(docContent);
-    const score = computeJaccardSimilarity(queryTokens, docTokens);
+    // Share of the query's content terms that the document contains.
+    const score = computeSupportScore(queryTokens, docTokens);
     perDocScores.push({ index: i, score, isRelevant: score >= threshold });
   }
 
@@ -51,22 +48,34 @@ function computeContextPrecision(query, retrievedDocs, options = {}) {
 }
 
 /**
- * Compute context recall: the fraction of answer claims that are supported by retrieved documents.
+ * Compute context recall: the fraction of the claims in a REFERENCE
+ * (ground-truth) answer that the retrieved documents support. It measures
+ * whether retrieval found everything needed to answer correctly; pass the
+ * expected answer, not the generated one (scoring the generated answer
+ * against the documents is faithfulness).
  *
- * @param {string} answer - The generated answer text
+ * A claim is one sentence of the reference answer. It is supported when some
+ * document scores at least `threshold` with computeSupportScore, including
+ * the numeric consistency check.
+ *
+ * @param {string} referenceAnswer - The reference (ground-truth) answer text
  * @param {Array<{content: string}>} retrievedDocs - Array of retrieved documents
  * @param {object} [options={}] - Configuration options
- * @param {number} [options.threshold=0.3] - Minimum Jaccard similarity for a claim to be considered supported
+ * @param {number} [options.threshold=0.5] - Minimum share of a claim's content terms a doc must contain to support it
  * @returns {{ score: number, supportedClaims: number, totalClaims: number, details: Array<{ claim: string, score: number, isSupported: boolean, bestDocIndex: number }> }}
  */
-function computeContextRecall(answer, retrievedDocs, options = {}) {
-  const { threshold = 0.3 } = options;
-
-  if (!answer || typeof answer !== 'string' || answer.trim().length === 0) {
+function computeContextRecall(referenceAnswer, retrievedDocs, options = {}) {
+  const { threshold = 0.5 } = options;
+
+  if (
+    !referenceAnswer ||
+    typeof referenceAnswer !== 'string' ||
+    referenceAnswer.trim().length === 0
+  ) {
     return { score: 0, supportedClaims: 0, totalClaims: 0, details: [] };
   }
 
-  const claims = splitSentences(answer);
+  const claims = splitSentences(referenceAnswer);
 
   if (claims.length === 0) {
     return { score: 0, supportedClaims: 0, totalClaims: 0, details: [] };
@@ -101,7 +110,10 @@ function computeContextRecall(answer, retrievedDocs, options = {}) {
       }
 
       const docTokens = tokenize(docContent);
-      const similarity = computeJaccardSimilarity(claimTokens, docTokens);
+      const similarity = computeSupportScore(claimTokens, docTokens, null, {
+        claimText: claim,
+        docText: docContent,
+      });
 
       if (similarity > bestScore) {
         bestScore = similarity;

package/src/evaluate/evaluator.js

@@ -22,15 +22,15 @@ async function evaluateRagDataset(_datasetPath, _config) {
   const results = [];
   for (const { prompt, expected } of cases) {
     try {
-      logger.info({ prompt }, 'Evaluating case');
+      logger.info('Evaluating case', { prompt });
       const result = await pipeline.run({ query: prompt });
-      const actual = result.results || '';
+      const actual = typeof result.answer === 'string' ? result.answer : '';
       const success =
         result.success && normalizeText(actual) === normalizeText(expected);
       const scores = scoreAnswer(actual, expected);
       results.push({ prompt, expected, actual, success, scores });
     } catch (err) {
-      logger.error({ prompt, error: err.message }, 'Evaluation failed');
+      logger.error('Evaluation failed', { prompt, error: err.message });
       results.push({
         prompt,
         expected,

package/src/evaluate/faithfulness.js

@@ -1,10 +1,6 @@
 'use strict';
 
-const {
-  splitSentences,
-  tokenize,
-  computeJaccardSimilarity,
-} = require('./scoring');
+const { splitSentences, tokenize, computeSupportScore } = require('./scoring');
 const { mapSentenceToSources } = require('../citation/source-mapper');
 
 /**
@@ -14,11 +10,11 @@ const { mapSentenceToSources } = require('../citation/source-mapper');
  * @param {string} answer - The generated answer text
  * @param {Array<{content: string}>} retrievedDocs - Array of retrieved documents with content
  * @param {object} [options={}] - Configuration options
- * @param {number} [options.threshold=0.3] - Minimum similarity score to consider a sentence faithful
+ * @param {number} [options.threshold=0.5] - Minimum similarity score to consider a sentence faithful
  * @returns {{ score: number, faithfulSentences: number, totalSentences: number, details: Array<{ sentence: string, score: number, isFaithful: boolean, bestSource: number }> }}
  */
 function computeFaithfulness(answer, retrievedDocs, options = {}) {
-  const { threshold = 0.3 } = options;
+  const { threshold = 0.5 } = options;
 
   if (!answer || typeof answer !== 'string' || answer.trim().length === 0) {
     return { score: 0, faithfulSentences: 0, totalSentences: 0, details: [] };
@@ -60,7 +56,10 @@ function computeFaithfulness(answer, retrievedDocs, options = {}) {
       }
 
       const docTokens = tokenize(docContent);
-      const similarity = computeJaccardSimilarity(sentenceTokens, docTokens);
+      const similarity = computeSupportScore(sentenceTokens, docTokens, null, {
+        claimText: sentence,
+        docText: docContent,
+      });
 
       if (similarity > bestScore) {
         bestScore = similarity;
@@ -102,13 +101,18 @@ function computeFaithfulnessFromCitations(citationResult) {
       ? citationResult.groundednessScore
       : 0;
 
+  // CitationTracker only lists sources at or above its similarity threshold,
+  // so a sentence is faithful exactly when it has at least one source.
   const details = Array.isArray(citationResult.citations)
-    ? citationResult.citations.map((c) => ({
-        sentence: c.sentence || c.text || '',
-        score: c.score || c.similarity || 0,
-        isFaithful: (c.score || c.similarity || 0) > 0,
-        bestSource: c.sourceIndex != null ? c.sourceIndex : -1,
-      }))
+    ? citationResult.citations.map((c) => {
+        const best = Array.isArray(c.sources) ? c.sources[0] : undefined;
+        return {
+          sentence: c.sentence || c.text || '',
+          score: best ? best.score : 0,
+          isFaithful: Boolean(best),
+          bestSource: best ? best.docIndex : -1,
+        };
+      })
     : [];
 
   return { score, details };

package/src/evaluate/groundedness.js

@@ -16,20 +16,40 @@ const DEFAULT_WEIGHTS = {
   contextRecall: 0.3,
 };
 
+/** Reason reported when context recall cannot be computed. */
+const NO_REFERENCE_REASON =
+  'No reference answer provided: context recall measures how much of a ' +
+  'reference (ground-truth) answer the retrieved documents support.';
+
 /**
- * Compute a composite groundedness score combining faithfulness, context precision, and context recall.
- * Uses a weighted average of the three component metrics.
+ * Compute a composite groundedness score from faithfulness, context precision
+ * and (when a reference answer is given) context recall.
+ *
+ * Formula: a weighted mean over the components that are available,
+ *
+ *   score = sum(w_i * m_i) / sum(w_i)
+ *
+ * With a reference answer and the default weights this is
+ *   0.4 * faithfulness + 0.3 * contextPrecision + 0.3 * contextRecall.
+ * Without one, context recall is left out and the remaining weights are
+ * renormalised:
+ *   (0.4 * faithfulness + 0.3 * contextPrecision) / 0.7.
+ * If every available weight is 0 the score is 0.
  *
  * @param {string} query - The user query
  * @param {string} answer - The generated answer text
  * @param {Array<{content: string}>} retrievedDocs - Array of retrieved documents
  * @param {object} [options={}] - Configuration options
  * @param {{ faithfulness: number, contextPrecision: number, contextRecall: number }} [options.weights] - Custom weights for each component
- * @param {number} [options.threshold=0.3] - Similarity threshold passed to component metrics
+ * @param {number} [options.threshold=0.5] - Similarity threshold passed to component metrics
+ * @param {string} [options.referenceAnswer] - Reference (ground-truth) answer used for context recall
  * @returns {{ score: number, components: { faithfulness: object, contextPrecision: object, contextRecall: object }, weights: { faithfulness: number, contextPrecision: number, contextRecall: number } }}
+ *   `components.contextRecall` is `{ score: null, skipped: true, reason }`
+ *   when no reference answer was given.
  */
 function computeGroundedness(query, answer, retrievedDocs, options = {}) {
-  const { weights = DEFAULT_WEIGHTS, threshold = 0.3 } = options;
+  const { threshold = 0.5, referenceAnswer } = options;
+  const weights = { ...DEFAULT_WEIGHTS, ...(options.weights || {}) };
 
   const faithfulness = computeFaithfulness(answer, retrievedDocs, {
     threshold,
@@ -37,14 +57,25 @@ function computeGroundedness(query, answer, retrievedDocs, options = {}) {
   const contextPrecision = computeContextPrecision(query, retrievedDocs, {
     threshold: threshold * 0.67,
   });
-  const contextRecall = computeContextRecall(answer, retrievedDocs, {
-    threshold,
-  });
+  const hasReference =
+    typeof referenceAnswer === 'string' && referenceAnswer.trim().length > 0;
+  const contextRecall = hasReference
+    ? computeContextRecall(referenceAnswer, retrievedDocs, { threshold })
+    : { score: null, skipped: true, reason: NO_REFERENCE_REASON };
+
+  const parts = [
+    [weights.faithfulness, faithfulness.score],
+    [weights.contextPrecision, contextPrecision.score],
+  ];
+  if (hasReference) parts.push([weights.contextRecall, contextRecall.score]);
 
-  const score =
-    weights.faithfulness * faithfulness.score +
-    weights.contextPrecision * contextPrecision.score +
-    weights.contextRecall * contextRecall.score;
+  let weighted = 0;
+  let totalWeight = 0;
+  for (const [w, m] of parts) {
+    weighted += w * m;
+    totalWeight += w;
+  }
+  const score = totalWeight > 0 ? weighted / totalWeight : 0;
 
   return {
     score,
@@ -57,4 +88,4 @@ function computeGroundedness(query, answer, retrievedDocs, options = {}) {
   };
 }
 
-module.exports = { computeGroundedness, DEFAULT_WEIGHTS };
+module.exports = { computeGroundedness, DEFAULT_WEIGHTS, NO_REFERENCE_REASON };

package/src/evaluate/pipeline-evaluator.js

@@ -10,7 +10,7 @@ const {
   computeContextPrecision,
   computeContextRecall,
 } = require('./context-metrics');
-const { computeGroundedness } = require('./groundedness');
+const { computeGroundedness, NO_REFERENCE_REASON } = require('./groundedness');
 const { scoreAnswer } = require('./scoring');
 
 /**
@@ -25,7 +25,7 @@ const DEFAULT_CONFIG = {
     'contextRecall',
     'groundedness',
   ],
-  threshold: 0.3,
+  threshold: 0.5,
   includeBLEU: false,
   includeROUGE: false,
 };
@@ -64,8 +64,14 @@ class PipelineEvaluator extends EventEmitter {
    * @param {Array<{content: string}>} pipelineResult.results - Retrieved documents
    * @param {object} [options={}] - Additional evaluation options
    * @param {object} [options.citationResult] - Pre-computed citation result for faithfulness shortcut
-   * @param {string} [options.referenceAnswer] - Reference answer for BLEU/ROUGE scoring
-   * @returns {{ scores: Object<string, number>, details: Object<string, object>, metadata: { metricsComputed: string[], threshold: number, timestamp: string } }}
+   * @param {string} [options.referenceAnswer] - Reference (ground-truth)
+   *   answer. Required for context recall (how much of it the retrieved
+   *   documents support) and used for BLEU/ROUGE. Without it, contextRecall is
+   *   left out of `scores`, `details.contextRecall` is
+   *   `{ score: null, skipped: true, reason }`, it is listed in
+   *   `metadata.metricsSkipped`, and groundedness renormalises over
+   *   faithfulness and context precision.
+   * @returns {{ scores: Object<string, number>, details: Object<string, object>, metadata: { metricsComputed: string[], metricsSkipped: Array<{metric: string, reason: string}>, threshold: number, timestamp: string } }}
    */
   evaluate(pipelineResult, options = {}) {
     const { citationResult, referenceAnswer } = options;
@@ -76,6 +82,7 @@ class PipelineEvaluator extends EventEmitter {
     const scores = {};
     const details = {};
     const metricsComputed = [];
+    const metricsSkipped = [];
     const { metrics, threshold } = this.config;
 
     if (metrics.includes('faithfulness')) {
@@ -106,16 +113,35 @@ class PipelineEvaluator extends EventEmitter {
       metricsComputed.push('contextPrecision');
     }
 
+    const hasReference =
+      typeof referenceAnswer === 'string' && referenceAnswer.trim().length > 0;
+
     if (metrics.includes('contextRecall')) {
-      const recallResult = computeContextRecall(answer, results, { threshold });
-      scores.contextRecall = recallResult.score;
-      details.contextRecall = recallResult;
-      metricsComputed.push('contextRecall');
+      if (hasReference) {
+        const recallResult = computeContextRecall(referenceAnswer, results, {
+          threshold,
+        });
+        scores.contextRecall = recallResult.score;
+        details.contextRecall = recallResult;
+        metricsComputed.push('contextRecall');
+      } else {
+        // No ground truth: report why instead of a made-up score.
+        details.contextRecall = {
+          score: null,
+          skipped: true,
+          reason: NO_REFERENCE_REASON,
+        };
+        metricsSkipped.push({
+          metric: 'contextRecall',
+          reason: NO_REFERENCE_REASON,
+        });
+      }
     }
 
     if (metrics.includes('groundedness')) {
       const groundednessResult = computeGroundedness(query, answer, results, {
         threshold,
+        referenceAnswer: hasReference ? referenceAnswer : undefined,
       });
       scores.groundedness = groundednessResult.score;
       details.groundedness = groundednessResult;
@@ -142,6 +168,7 @@ class PipelineEvaluator extends EventEmitter {
       details,
       metadata: {
         metricsComputed,
+        metricsSkipped,
         threshold,
         timestamp: new Date().toISOString(),
       },

package/src/evaluate/scoring.js

@@ -11,10 +11,14 @@
  * @returns {string[]}
  */
 function tokenize(text) {
-  return text
+  // Unicode-aware: keep letters/digits in any script, drop apostrophes so
+  // "don't" stays one token, and treat every other character (including
+  // newlines and hyphens) as a separator.
+  return String(text)
     .toLowerCase()
-    .replace(/[^a-z0-9 ]/g, '')
-    .split(/\s+/)
+    .replace(/['’]/g, '')
+    .replace(/[^\p{L}\p{N}]+/gu, ' ')
+    .split(' ')
     .filter(Boolean);
 }
 
@@ -108,7 +112,251 @@ function computeJaccardSimilarity(tokensA, tokensB) {
   return intersection.size / union.size;
 }
 
+/**
+ * Common English function words. Ignored when measuring how much of a
+ * sentence a document supports, so "the" and "is" cannot ground a claim.
+ */
+const STOPWORDS = new Set(
+  (
+    'a an and are as at be been but by can could did do does for from had has ' +
+    'have he her his how i if in into is it its may me might must my no not ' +
+    'of on or our she should so than that the their them then there these ' +
+    'they this those to was we were what when where which while who why will ' +
+    'with would you your'
+  ).split(' '),
+);
+
+/**
+ * Content tokens of a sentence: tokens minus stopwords. Falls back to all
+ * tokens when nothing is left (e.g. "Yes."), so short sentences are still
+ * scored rather than skipped.
+ *
+ * @param {string[]} tokens
+ * @returns {string[]}
+ */
+function contentTokens(tokens) {
+  const content = tokens.filter((t) => !STOPWORDS.has(t));
+  return content.length > 0 ? content : tokens;
+}
+
+/** Values of English number words. "one" is left out on purpose: it is far
+ * more often a pronoun ("no one", "one of") than a quantity. */
+const NUMBER_WORDS = {
+  zero: 0,
+  two: 2,
+  three: 3,
+  four: 4,
+  five: 5,
+  six: 6,
+  seven: 7,
+  eight: 8,
+  nine: 9,
+  ten: 10,
+  eleven: 11,
+  twelve: 12,
+  thirteen: 13,
+  fourteen: 14,
+  fifteen: 15,
+  sixteen: 16,
+  seventeen: 17,
+  eighteen: 18,
+  nineteen: 19,
+  twenty: 20,
+  thirty: 30,
+  forty: 40,
+  fifty: 50,
+  sixty: 60,
+  seventy: 70,
+  eighty: 80,
+  ninety: 90,
+};
+/** Multipliers; "one" counts as 1 only inside a run ("one hundred"). */
+const NUMBER_SCALES = {
+  hundred: 100,
+  thousand: 1000,
+  million: 1e6,
+  billion: 1e9,
+  trillion: 1e12,
+};
+
+/**
+ * Digit numbers, optionally with thousands separators or a decimal part (a
+ * trailing "%" is ignored). The lookbehind skips digits glued to letters (BM25).
+ */
+const DIGIT_NUMBER_RE = /(?<![\p{L}\p{N}])\d+(?:[.,]\d+)*/gu;
+
+/**
+ * Turn one run of number words into its value: "twenty five" -> 25,
+ * "two hundred" -> 200, "three thousand four hundred" -> 3400.
+ *
+ * @param {string[]} words
+ * @returns {number}
+ */
+function parseNumberWords(words) {
+  let total = 0;
+  let current = 0;
+  for (const w of words) {
+    if (w === 'one') current += 1;
+    else if (w in NUMBER_WORDS) current += NUMBER_WORDS[w];
+    else if (w === 'hundred') current = (current || 1) * 100;
+    else {
+      total += (current || 1) * NUMBER_SCALES[w];
+      current = 0;
+    }
+  }
+  return total + current;
+}
+
+/**
+ * Extract the numbers stated in a text, as normalised value strings, so
+ * "20", "20%", "twenty" and "Twenty" all yield "20", "1,000" yields "1000"
+ * and "2.5" yields "2.5".
+ *
+ * @param {string} text
+ * @returns {{ values: string[], parts: string[] }} values: one entry per
+ *   number as written; parts: values plus each number word on its own
+ *   (used on the document side so "twenty-five" also supports "20" and "5"
+ *   is not required)
+ */
+function extractNumbers(text) {
+  const values = [];
+  const parts = [];
+  if (!text || typeof text !== 'string') return { values, parts };
+
+  for (const match of text.matchAll(DIGIT_NUMBER_RE)) {
+    const raw = match[0];
+    const pieces = /^\d{1,3}(,\d{3})+(\.\d+)?$/.test(raw)
+      ? [raw.replace(/,/g, '')]
+      : raw.split(',');
+    for (const piece of pieces) {
+      const n = Number(piece);
+      if (Number.isFinite(n)) values.push(String(n));
+    }
+  }
+
+  const words = text.toLowerCase().split(/[^\p{L}]+/u);
+  let run = [];
+  const flush = () => {
+    // "and" only joins a run ("two hundred and five"); never ends one.
+    while (run.length && run[run.length - 1] === 'and') run.pop();
+    // A lone "one" is not treated as a number (see NUMBER_WORDS).
+    if (run.length && !(run.length === 1 && run[0] === 'one')) {
+      const numberWords = run.filter((w) => w !== 'and');
+      values.push(String(parseNumberWords(numberWords)));
+      for (const w of numberWords) {
+        if (w in NUMBER_WORDS) parts.push(String(NUMBER_WORDS[w]));
+      }
+    }
+    run = [];
+  };
+  for (const w of words) {
+    const last = run[run.length - 1];
+    const isNumberWord =
+      w in NUMBER_WORDS ||
+      w in NUMBER_SCALES ||
+      w === 'one' ||
+      (w === 'and' && last in NUMBER_SCALES);
+    if (isNumberWord) {
+      run.push(w);
+    } else {
+      flush();
+    }
+  }
+  flush();
+
+  return { values, parts: [...values, ...parts] };
+}
+
+/**
+ * Numeric consistency between a claim and a document: every number the claim
+ * states (digits, decimals, percentages or number words) must also appear in
+ * the document. Lexical overlap alone would ground "thirty days" against a
+ * source that says "twenty days".
+ *
+ * @param {string} claimText
+ * @param {string} docText
+ * @returns {{ consistent: boolean, claimNumbers: string[], missing: string[] }}
+ */
+function checkNumericConsistency(claimText, docText) {
+  const claimNumbers = [...new Set(extractNumbers(claimText).values)];
+  if (claimNumbers.length === 0) {
+    return { consistent: true, claimNumbers, missing: [] };
+  }
+  const docNumbers = new Set(extractNumbers(docText).parts);
+  const missing = claimNumbers.filter((n) => !docNumbers.has(n));
+  return { consistent: missing.length === 0, claimNumbers, missing };
+}
+
+/**
+ * Multiplier applied to the support score when a claim states numbers the
+ * document does not contain. Together with the share of matching numbers it
+ * keeps the score below 0.4, i.e. under the default 0.5 support threshold.
+ */
+const NUMERIC_MISMATCH_FACTOR = 0.4;
+
+/**
+ * How much of a claim is supported by a document: the (optionally
+ * IDF-weighted) fraction of the claim's unique content tokens that appear in
+ * the document. Unlike Jaccard against the whole document, this does not
+ * shrink as the document gets longer, so a sentence copied verbatim from a
+ * long chunk scores 1.
+ *
+ * Numeric consistency: when `options.claimText` and `options.docText` are
+ * given and the claim states a number the document does not contain (see
+ * checkNumericConsistency), the score becomes
+ *   lexicalScore * NUMERIC_MISMATCH_FACTOR (0.4) * matchedNumbers / claimNumbers
+ * which is always below 0.4, so the claim fails the default 0.5 threshold.
+ *
+ * @param {string[]} claimTokens - Tokens of the sentence/claim/query
+ * @param {string[]|Set<string>} docTokens - Tokens of the document
+ * @param {Map<string, number>|null} [idfWeights] - Optional token weights
+ * @param {{ claimText?: string, docText?: string }} [options] - Raw texts
+ *   enabling the numeric consistency check
+ * @returns {number} Support score between 0 and 1
+ */
+function computeSupportScore(
+  claimTokens,
+  docTokens,
+  idfWeights = null,
+  options = {},
+) {
+  const claim = [...new Set(contentTokens(claimTokens))];
+  if (claim.length === 0) return 0;
+  const docSet = docTokens instanceof Set ? docTokens : new Set(docTokens);
+
+  let matched = 0;
+  let total = 0;
+  for (const token of claim) {
+    const weight = idfWeights ? (idfWeights.get(token) ?? 1) : 1;
+    total += weight;
+    if (docSet.has(token)) matched += weight;
+  }
+  let score = total > 0 ? matched / total : 0;
+
+  const { claimText, docText } = options || {};
+  if (
+    score > 0 &&
+    typeof claimText === 'string' &&
+    typeof docText === 'string'
+  ) {
+    const numeric = checkNumericConsistency(claimText, docText);
+    if (!numeric.consistent) {
+      const share =
+        (numeric.claimNumbers.length - numeric.missing.length) /
+        numeric.claimNumbers.length;
+      score *= NUMERIC_MISMATCH_FACTOR * share;
+    }
+  }
+  return score;
+}
+
 module.exports = {
+  STOPWORDS,
+  contentTokens,
+  computeSupportScore,
+  checkNumericConsistency,
+  extractNumbers,
+  NUMERIC_MISMATCH_FACTOR,
   scoreAnswer,
   tokenize,
   computeBLEU,