@devilsdev/rag-pipeline-utils 2.3.0 → 2.4.0

package/src/evaluate/faithfulness.js

@@ -0,0 +1,117 @@
+'use strict';
+
+const {
+  splitSentences,
+  tokenize,
+  computeJaccardSimilarity,
+} = require('./scoring');
+const { mapSentenceToSources } = require('../citation/source-mapper');
+
+/**
+ * Compute faithfulness score for an answer against retrieved documents.
+ * Measures how well each sentence in the answer is supported by the retrieved context.
+ *
+ * @param {string} answer - The generated answer text
+ * @param {Array<{content: string}>} retrievedDocs - Array of retrieved documents with content
+ * @param {object} [options={}] - Configuration options
+ * @param {number} [options.threshold=0.3] - Minimum similarity score to consider a sentence faithful
+ * @returns {{ score: number, faithfulSentences: number, totalSentences: number, details: Array<{ sentence: string, score: number, isFaithful: boolean, bestSource: number }> }}
+ */
+function computeFaithfulness(answer, retrievedDocs, options = {}) {
+  const { threshold = 0.3 } = options;
+
+  if (!answer || typeof answer !== 'string' || answer.trim().length === 0) {
+    return { score: 0, faithfulSentences: 0, totalSentences: 0, details: [] };
+  }
+
+  if (!Array.isArray(retrievedDocs) || retrievedDocs.length === 0) {
+    const sentences = splitSentences(answer);
+    const details = sentences.map((sentence) => ({
+      sentence,
+      score: 0,
+      isFaithful: false,
+      bestSource: -1,
+    }));
+    return {
+      score: 0,
+      faithfulSentences: 0,
+      totalSentences: sentences.length,
+      details,
+    };
+  }
+
+  const sentences = splitSentences(answer);
+
+  if (sentences.length === 0) {
+    return { score: 0, faithfulSentences: 0, totalSentences: 0, details: [] };
+  }
+
+  const details = sentences.map((sentence) => {
+    const sentenceTokens = tokenize(sentence);
+    let bestScore = 0;
+    let bestSource = -1;
+
+    for (let i = 0; i < retrievedDocs.length; i++) {
+      const doc = retrievedDocs[i];
+      const docContent = doc && (doc.content || doc.text || '');
+
+      if (!docContent) {
+        continue;
+      }
+
+      const docTokens = tokenize(docContent);
+      const similarity = computeJaccardSimilarity(sentenceTokens, docTokens);
+
+      if (similarity > bestScore) {
+        bestScore = similarity;
+        bestSource = i;
+      }
+    }
+
+    return {
+      sentence,
+      score: bestScore,
+      isFaithful: bestScore >= threshold,
+      bestSource,
+    };
+  });
+
+  const faithfulSentences = details.filter((d) => d.isFaithful).length;
+  const totalSentences = details.length;
+  const score = totalSentences > 0 ? faithfulSentences / totalSentences : 0;
+
+  return { score, faithfulSentences, totalSentences, details };
+}
+
+/**
+ * Derive faithfulness score from an existing citation result.
+ * Shortcut that reuses citation analysis data rather than recomputing.
+ *
+ * @param {object} citationResult - Result from citation analysis
+ * @param {number} citationResult.groundednessScore - Pre-computed groundedness score
+ * @param {Array} [citationResult.citations] - Array of citation details
+ * @returns {{ score: number, details: Array }}
+ */
+function computeFaithfulnessFromCitations(citationResult) {
+  if (!citationResult || typeof citationResult !== 'object') {
+    return { score: 0, details: [] };
+  }
+
+  const score =
+    typeof citationResult.groundednessScore === 'number'
+      ? citationResult.groundednessScore
+      : 0;
+
+  const details = Array.isArray(citationResult.citations)
+    ? citationResult.citations.map((c) => ({
+        sentence: c.sentence || c.text || '',
+        score: c.score || c.similarity || 0,
+        isFaithful: (c.score || c.similarity || 0) > 0,
+        bestSource: c.sourceIndex != null ? c.sourceIndex : -1,
+      }))
+    : [];
+
+  return { score, details };
+}
+
+module.exports = { computeFaithfulness, computeFaithfulnessFromCitations };

package/src/evaluate/groundedness.js

@@ -0,0 +1,60 @@
+'use strict';
+
+const { computeFaithfulness } = require('./faithfulness');
+const {
+  computeContextPrecision,
+  computeContextRecall,
+} = require('./context-metrics');
+
+/**
+ * Default weights for the groundedness composite score.
+ * @type {{ faithfulness: number, contextPrecision: number, contextRecall: number }}
+ */
+const DEFAULT_WEIGHTS = {
+  faithfulness: 0.4,
+  contextPrecision: 0.3,
+  contextRecall: 0.3,
+};
+
+/**
+ * Compute a composite groundedness score combining faithfulness, context precision, and context recall.
+ * Uses a weighted average of the three component metrics.
+ *
+ * @param {string} query - The user query
+ * @param {string} answer - The generated answer text
+ * @param {Array<{content: string}>} retrievedDocs - Array of retrieved documents
+ * @param {object} [options={}] - Configuration options
+ * @param {{ faithfulness: number, contextPrecision: number, contextRecall: number }} [options.weights] - Custom weights for each component
+ * @param {number} [options.threshold=0.3] - Similarity threshold passed to component metrics
+ * @returns {{ score: number, components: { faithfulness: object, contextPrecision: object, contextRecall: object }, weights: { faithfulness: number, contextPrecision: number, contextRecall: number } }}
+ */
+function computeGroundedness(query, answer, retrievedDocs, options = {}) {
+  const { weights = DEFAULT_WEIGHTS, threshold = 0.3 } = options;
+
+  const faithfulness = computeFaithfulness(answer, retrievedDocs, {
+    threshold,
+  });
+  const contextPrecision = computeContextPrecision(query, retrievedDocs, {
+    threshold: threshold * 0.67,
+  });
+  const contextRecall = computeContextRecall(answer, retrievedDocs, {
+    threshold,
+  });
+
+  const score =
+    weights.faithfulness * faithfulness.score +
+    weights.contextPrecision * contextPrecision.score +
+    weights.contextRecall * contextRecall.score;
+
+  return {
+    score,
+    components: {
+      faithfulness,
+      contextPrecision,
+      contextRecall,
+    },
+    weights,
+  };
+}
+
+module.exports = { computeGroundedness, DEFAULT_WEIGHTS };

package/src/evaluate/pipeline-evaluator.js

@@ -0,0 +1,156 @@
+'use strict';
+
+const { EventEmitter } = require('events');
+const {
+  computeFaithfulness,
+  computeFaithfulnessFromCitations,
+} = require('./faithfulness');
+const { computeAnswerRelevance } = require('./relevance');
+const {
+  computeContextPrecision,
+  computeContextRecall,
+} = require('./context-metrics');
+const { computeGroundedness } = require('./groundedness');
+const { scoreAnswer } = require('./scoring');
+
+/**
+ * Default configuration for PipelineEvaluator.
+ * @type {{ metrics: string[], threshold: number, includeBLEU: boolean, includeROUGE: boolean }}
+ */
+const DEFAULT_CONFIG = {
+  metrics: [
+    'faithfulness',
+    'relevance',
+    'contextPrecision',
+    'contextRecall',
+    'groundedness',
+  ],
+  threshold: 0.3,
+  includeBLEU: false,
+  includeROUGE: false,
+};
+
+/**
+ * Evaluates RAG pipeline outputs using multiple quality metrics.
+ * Emits an 'evaluated' event after each evaluation with the full result.
+ *
+ * @extends EventEmitter
+ *
+ * @example
+ * const evaluator = new PipelineEvaluator({ metrics: ['faithfulness', 'relevance'] });
+ * evaluator.on('evaluated', (result) => console.log(result.scores));
+ * const result = evaluator.evaluate({ query: 'What is RAG?', answer: '...', results: [...] });
+ */
+class PipelineEvaluator extends EventEmitter {
+  /**
+   * Create a new PipelineEvaluator.
+   * @param {object} [options={}] - Configuration options
+   * @param {string[]} [options.metrics] - List of metrics to compute
+   * @param {number} [options.threshold] - Similarity threshold for metrics
+   * @param {boolean} [options.includeBLEU] - Whether to include BLEU score when reference answer is provided
+   * @param {boolean} [options.includeROUGE] - Whether to include ROUGE score when reference answer is provided
+   */
+  constructor(options = {}) {
+    super();
+    this.config = { ...DEFAULT_CONFIG, ...options };
+  }
+
+  /**
+   * Evaluate a pipeline result across all configured metrics.
+   *
+   * @param {object} pipelineResult - The pipeline output to evaluate
+   * @param {string} pipelineResult.query - The original user query
+   * @param {string} pipelineResult.answer - The generated answer
+   * @param {Array<{content: string}>} pipelineResult.results - Retrieved documents
+   * @param {object} [options={}] - Additional evaluation options
+   * @param {object} [options.citationResult] - Pre-computed citation result for faithfulness shortcut
+   * @param {string} [options.referenceAnswer] - Reference answer for BLEU/ROUGE scoring
+   * @returns {{ scores: Object<string, number>, details: Object<string, object>, metadata: { metricsComputed: string[], threshold: number, timestamp: string } }}
+   */
+  evaluate(pipelineResult, options = {}) {
+    const { citationResult, referenceAnswer } = options;
+    const query = (pipelineResult && pipelineResult.query) || '';
+    const answer = (pipelineResult && pipelineResult.answer) || '';
+    const results = (pipelineResult && pipelineResult.results) || [];
+
+    const scores = {};
+    const details = {};
+    const metricsComputed = [];
+    const { metrics, threshold } = this.config;
+
+    if (metrics.includes('faithfulness')) {
+      let faithResult;
+      if (citationResult) {
+        faithResult = computeFaithfulnessFromCitations(citationResult);
+      } else {
+        faithResult = computeFaithfulness(answer, results, { threshold });
+      }
+      scores.faithfulness = faithResult.score;
+      details.faithfulness = faithResult;
+      metricsComputed.push('faithfulness');
+    }
+
+    if (metrics.includes('relevance')) {
+      const relevanceResult = computeAnswerRelevance(query, answer);
+      scores.relevance = relevanceResult.score;
+      details.relevance = relevanceResult;
+      metricsComputed.push('relevance');
+    }
+
+    if (metrics.includes('contextPrecision')) {
+      const precisionResult = computeContextPrecision(query, results, {
+        threshold: threshold * 0.67,
+      });
+      scores.contextPrecision = precisionResult.score;
+      details.contextPrecision = precisionResult;
+      metricsComputed.push('contextPrecision');
+    }
+
+    if (metrics.includes('contextRecall')) {
+      const recallResult = computeContextRecall(answer, results, { threshold });
+      scores.contextRecall = recallResult.score;
+      details.contextRecall = recallResult;
+      metricsComputed.push('contextRecall');
+    }
+
+    if (metrics.includes('groundedness')) {
+      const groundednessResult = computeGroundedness(query, answer, results, {
+        threshold,
+      });
+      scores.groundedness = groundednessResult.score;
+      details.groundedness = groundednessResult;
+      metricsComputed.push('groundedness');
+    }
+
+    // Optional BLEU/ROUGE when reference answer is provided
+    if (referenceAnswer && typeof referenceAnswer === 'string') {
+      if (this.config.includeBLEU || this.config.includeROUGE) {
+        const refScores = scoreAnswer(answer, referenceAnswer);
+        if (this.config.includeBLEU) {
+          scores.bleu = refScores.bleu;
+          metricsComputed.push('bleu');
+        }
+        if (this.config.includeROUGE) {
+          scores.rouge = refScores.rouge;
+          metricsComputed.push('rouge');
+        }
+      }
+    }
+
+    const result = {
+      scores,
+      details,
+      metadata: {
+        metricsComputed,
+        threshold,
+        timestamp: new Date().toISOString(),
+      },
+    };
+
+    this.emit('evaluated', result);
+
+    return result;
+  }
+}
+
+module.exports = { PipelineEvaluator, DEFAULT_CONFIG };

package/src/evaluate/relevance.js

@@ -0,0 +1,129 @@
+'use strict';
+
+const { tokenize, computeJaccardSimilarity } = require('./scoring');
+
+/**
+ * Common English stopwords to skip when measuring query token coverage.
+ * Only tokens longer than 3 characters are considered, but these are
+ * additionally excluded to avoid inflating coverage scores.
+ * @type {Set<string>}
+ */
+const STOPWORDS = new Set([
+  'the',
+  'that',
+  'this',
+  'with',
+  'from',
+  'have',
+  'been',
+  'will',
+  'would',
+  'could',
+  'should',
+  'does',
+  'what',
+  'when',
+  'where',
+  'which',
+  'their',
+  'there',
+  'they',
+  'them',
+  'then',
+  'than',
+  'these',
+  'those',
+  'your',
+  'about',
+  'into',
+  'each',
+  'also',
+  'more',
+  'some',
+  'such',
+  'only',
+  'over',
+  'very',
+  'just',
+  'being',
+  'were',
+  'here',
+]);
+
+/**
+ * Compute answer relevance score measuring how well the answer addresses the query.
+ * Combines Jaccard similarity with query token coverage for a balanced metric.
+ *
+ * @param {string} query - The user query
+ * @param {string} answer - The generated answer text
+ * @param {object} [options={}] - Configuration options
+ * @param {number} [options.jaccardWeight=0.5] - Weight for Jaccard similarity component
+ * @param {number} [options.coverageWeight=0.5] - Weight for query token coverage component
+ * @returns {{ score: number, jaccardSimilarity: number, queryTokensCovered: number, totalQueryTokens: number, missingQueryTerms: string[] }}
+ */
+function computeAnswerRelevance(query, answer, options = {}) {
+  const { jaccardWeight = 0.5, coverageWeight = 0.5 } = options;
+
+  if (!query || typeof query !== 'string' || query.trim().length === 0) {
+    return {
+      score: 0,
+      jaccardSimilarity: 0,
+      queryTokensCovered: 0,
+      totalQueryTokens: 0,
+      missingQueryTerms: [],
+    };
+  }
+
+  if (!answer || typeof answer !== 'string' || answer.trim().length === 0) {
+    const significantQueryTokens = tokenize(query).filter(
+      (t) => t.length > 3 && !STOPWORDS.has(t),
+    );
+    return {
+      score: 0,
+      jaccardSimilarity: 0,
+      queryTokensCovered: 0,
+      totalQueryTokens: significantQueryTokens.length,
+      missingQueryTerms: [...new Set(significantQueryTokens)],
+    };
+  }
+
+  const queryTokens = tokenize(query);
+  const answerTokens = tokenize(answer);
+
+  // Jaccard similarity between full token sets
+  const jaccardSimilarity = computeJaccardSimilarity(queryTokens, answerTokens);
+
+  // Query token coverage: filter to significant query tokens (length > 3, not stopwords)
+  const significantQueryTokens = queryTokens.filter(
+    (t) => t.length > 3 && !STOPWORDS.has(t),
+  );
+
+  const answerTokenSet = new Set(answerTokens);
+  const coveredTokens = significantQueryTokens.filter((t) =>
+    answerTokenSet.has(t),
+  );
+  const totalQueryTokens = significantQueryTokens.length;
+  const queryTokensCovered = coveredTokens.length;
+
+  const coverage =
+    totalQueryTokens > 0 ? queryTokensCovered / totalQueryTokens : 0;
+
+  // Combined score
+  const score = jaccardWeight * jaccardSimilarity + coverageWeight * coverage;
+
+  // Missing query terms (unique)
+  const coveredSet = new Set(coveredTokens);
+  const missingQueryTerms = [
+    ...new Set(significantQueryTokens.filter((t) => !coveredSet.has(t))),
+  ];
+
+  return {
+    score,
+    jaccardSimilarity,
+    queryTokensCovered,
+    totalQueryTokens,
+    missingQueryTerms,
+  };
+}
+
+module.exports = { computeAnswerRelevance };

package/src/evaluate/scoring.js

@@ -83,9 +83,36 @@ function scoreAnswer(_actual, _expected) {
   };
 }
 
+/**
+ * Split text into sentences on sentence-ending punctuation
+ * @param {string} text
+ * @returns {string[]}
+ */
+function splitSentences(text) {
+  if (!text || typeof text !== 'string') return [];
+  return text.split(/(?<=[.!?])\s+/).filter((s) => s.trim().length > 0);
+}
+
+/**
+ * Compute Jaccard similarity between two token arrays
+ * @param {string[]} tokensA
+ * @param {string[]} tokensB
+ * @returns {number} Similarity score between 0 and 1
+ */
+function computeJaccardSimilarity(tokensA, tokensB) {
+  if (!tokensA.length || !tokensB.length) return 0;
+  const setA = new Set(tokensA);
+  const setB = new Set(tokensB);
+  const intersection = new Set([...setA].filter((x) => setB.has(x)));
+  const union = new Set([...setA, ...setB]);
+  return intersection.size / union.size;
+}
+
 module.exports = {
   scoreAnswer,
   tokenize,
   computeBLEU,
   computeROUGE,
+  splitSentences,
+  computeJaccardSimilarity,
 };

package/src/graph/entity-extractor.js

@@ -0,0 +1,183 @@
+'use strict';
+
+const { EventEmitter } = require('events');
+const { tokenize } = require('../evaluate/scoring');
+
+/**
+ * Default regex patterns for heuristic entity extraction.
+ * @type {Object<string, RegExp>}
+ */
+const DEFAULT_PATTERNS = {
+  /** Capitalized multi-word sequences (proper nouns / names) */
+  properNoun: /\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+)+)\b/g,
+  /** Dates in YYYY-MM-DD or YYYY/MM/DD format */
+  date: /\b\d{4}[-/]\d{1,2}[-/]\d{1,2}\b/g,
+  /** Numbers with units */
+  measurement:
+    /\b\d+(?:\.\d+)?\s*(?:GB|MB|KB|ms|seconds?|minutes?|hours?|days?|%)\b/gi,
+  /** URLs */
+  url: /https?:\/\/[^\s]+/g,
+  /** Email addresses */
+  email: /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g,
+  /** Technical terms (CamelCase or ALLCAPS) */
+  technical: /\b[A-Z][a-z]+[A-Z]\w+\b/g,
+};
+
+/**
+ * Heuristic entity extraction from text — no LLM required.
+ *
+ * Uses configurable regex patterns to identify named entities,
+ * dates, measurements, URLs, emails, and technical terms.
+ */
+class EntityExtractor extends EventEmitter {
+  /**
+   * @param {Object} [options={}] - Extractor options
+   * @param {Object<string, RegExp>} [options.patterns] - Custom patterns to merge with defaults
+   */
+  constructor(options = {}) {
+    super();
+    this.patterns = { ...DEFAULT_PATTERNS, ...(options.patterns || {}) };
+    this.config = options;
+  }
+
+  /**
+   * Extract entities from a single text string.
+   *
+   * For each configured pattern type, finds all matches, deduplicates
+   * by normalized name, and returns an array of entity descriptors.
+   *
+   * @param {string} text - Input text to extract entities from
+   * @returns {Array<{name: string, type: string, positions: Array<{start: number, end: number}>, frequency: number}>}
+   */
+  extract(text) {
+    if (!text || typeof text !== 'string') {
+      return [];
+    }
+
+    /** @type {Map<string, {name: string, type: string, positions: Array<{start: number, end: number}>, frequency: number}>} */
+    const entityMap = new Map();
+
+    for (const [type, pattern] of Object.entries(this.patterns)) {
+      const regex = new RegExp(pattern.source, pattern.flags);
+      let match;
+
+      while ((match = regex.exec(text)) !== null) {
+        const name = match[1] || match[0];
+        const normalized = name.toLowerCase().trim();
+        const position = {
+          start: match.index,
+          end: match.index + match[0].length,
+        };
+
+        if (entityMap.has(normalized)) {
+          const existing = entityMap.get(normalized);
+          existing.positions.push(position);
+          existing.frequency += 1;
+        } else {
+          entityMap.set(normalized, {
+            name,
+            type,
+            positions: [position],
+            frequency: 1,
+          });
+        }
+      }
+    }
+
+    const entities = [...entityMap.values()];
+    this.emit('extracted', { count: entities.length, entities });
+    return entities;
+  }
+
+  /**
+   * Extract relationships between entities that co-occur in the same sentence.
+   *
+   * @param {string} text - Input text
+   * @param {Array<{name: string, type: string}>} entities - Previously extracted entities
+   * @returns {Array<{from: string, to: string, type: string, context: string}>}
+   */
+  extractRelationships(text, entities) {
+    if (!text || !entities || entities.length === 0) {
+      return [];
+    }
+
+    const sentences = text
+      .split(/[.!?]+/)
+      .map((s) => s.trim())
+      .filter(Boolean);
+    const relationships = [];
+
+    for (const sentence of sentences) {
+      const sentenceLower = sentence.toLowerCase();
+      const present = entities.filter((e) =>
+        sentenceLower.includes(e.name.toLowerCase()),
+      );
+
+      for (let i = 0; i < present.length; i++) {
+        for (let j = i + 1; j < present.length; j++) {
+          relationships.push({
+            from: present[i].name,
+            to: present[j].name,
+            type: 'co-occurs',
+            context: sentence,
+          });
+        }
+      }
+    }
+
+    return relationships;
+  }
+
+  /**
+   * Extract entities and relationships from multiple documents.
+   *
+   * Combines entities across documents and tracks which source
+   * document(s) each entity originates from.
+   *
+   * @param {Array<{id: string, content: string, metadata?: Object}>} documents
+   * @returns {{entities: Array, relationships: Array}}
+   */
+  extractFromDocuments(documents) {
+    if (!Array.isArray(documents)) {
+      return { entities: [], relationships: [] };
+    }
+
+    /** @type {Map<string, {name: string, type: string, positions: Array, frequency: number, sources: string[]}>} */
+    const combinedEntities = new Map();
+    const allRelationships = [];
+
+    for (const doc of documents) {
+      const text = doc.content || '';
+      const entities = this.extract(text);
+
+      for (const entity of entities) {
+        const normalized = entity.name.toLowerCase().trim();
+        if (combinedEntities.has(normalized)) {
+          const existing = combinedEntities.get(normalized);
+          existing.frequency += entity.frequency;
+          existing.positions.push(...entity.positions);
+          if (!existing.sources.includes(doc.id)) {
+            existing.sources.push(doc.id);
+          }
+        } else {
+          combinedEntities.set(normalized, {
+            ...entity,
+            sources: [doc.id],
+          });
+        }
+      }
+
+      const relationships = this.extractRelationships(text, entities);
+      for (const rel of relationships) {
+        allRelationships.push({ ...rel, source: doc.id });
+      }
+    }
+
+    return {
+      entities: [...combinedEntities.values()],
+      relationships: allRelationships,
+    };
+  }
+}
+
+module.exports = { EntityExtractor, DEFAULT_PATTERNS };