@devilsdev/rag-pipeline-utils 2.4.5 → 2.5.0

package/src/agentic/query-planner.js

@@ -14,12 +14,9 @@ const DEFAULT_CONFIG = {
 /** Question words that indicate query complexity */
 const QUESTION_WORDS = ['who', 'what', 'when', 'where', 'why', 'how', 'which'];
 
-/** Conjunctions used for splitting compound queries */
+/** Conjunctions used for scoring compound queries */
 const CONJUNCTIONS = ['and', 'or', 'also', 'as well as'];
 
-/** Splitter patterns for decomposing compound queries */
-const SPLIT_PATTERNS = [/ and /i, / or /i, /; ?/];
-
 /** Keywords that indicate a comparative strategy */
 const COMPARATIVE_KEYWORDS = [
   'compare',
@@ -27,19 +24,111 @@ const COMPARATIVE_KEYWORDS = [
   'difference',
   'versus',
   'vs',
+  'compared',
+  'comparing',
+  'comparison',
+  'contrasting',
+  'differ',
+];
+
+/**
+ * Words that start a new question or instruction. A conjunction ("and",
+ * "or", "as well as") splits a query only when one of these follows it, so
+ * "What is RAG and how does BM25 work" splits while "salt and pepper" and
+ * "Should I use A or B" do not.
+ */
+const CLAUSE_STARTERS = [
+  // question words
+  'who',
+  'whom',
+  'whose',
+  'what',
+  'when',
+  'where',
+  'why',
+  'how',
+  'which',
+  // auxiliaries opening a yes/no question
+  'is',
+  'are',
+  'was',
+  'were',
+  'do',
+  'does',
+  'did',
+  'can',
+  'could',
+  'should',
+  'would',
+  'will',
+  'has',
+  'have',
+  // imperatives opening an instruction
+  'explain',
+  'describe',
+  'list',
+  'compare',
+  'contrast',
+  'summarize',
+  'summarise',
+  'tell',
+  'show',
+  'give',
+  'define',
+  'find',
 ];
 
+const STARTER_ALT = CLAUSE_STARTERS.join('|');
+
+/** ", and how ...", " and what ...", " or is ...", " as well as why ..." */
+const CONJUNCTION_BOUNDARY = new RegExp(
+  `\\s*,?\\s+(?:and|or|as well as|and also|but)\\s+(?=(?:${STARTER_ALT})\\b)`,
+  'i',
+);
+
 /**
- * Heuristic query decomposition planner -- no LLM needed.
- * Splits compound queries into sub-queries with assigned strategies
- * based on detected question types.
+ * Explicit boundaries: "?" / "!" followed by more text, "." followed by a
+ * capitalised word (so "e.g. salt" stays whole), and ";".
+ */
+const PUNCTUATION_BOUNDARY = /(?<=[?!])\s+(?=\S)|(?<=\.)\s+(?=[A-Z])|\s*;\s*/;
+/** Abbreviations whose period does not end a sentence. */
+const ABBREVIATION_END =
+  /\b(?:e\.g|i\.e|etc|vs|cf|approx|mr|mrs|ms|dr|st|no)\.$/i;
+
+/** "compare|contrast X and|with|to|vs Y [for|in terms of|... aspect]" */
+const COMPARE_RE =
+  /\b(?:compare|contrast)\s+(.+?)\s+(?:and|with|to|against|vs\.?|versus)\s+(.+?)(?:\s+(?:for|in terms of|regarding|when|on)\s+(.+?))?\s*[?.!]?$/i;
+/** "difference(s) between X and Y [for|in|... aspect]" */
+const DIFFERENCE_RE =
+  /\bdifferences?\s+between\s+(.+?)\s+and\s+(.+?)(?:\s+(?:for|in terms of|regarding|when|in|on)\s+(.+?))?\s*[?.!]?$/i;
+/** "X vs Y [for|... aspect]" */
+const VERSUS_RE =
+  /^(.*?)\s+(?:vs\.?|versus)\s+(.+?)(?:\s+(?:for|in terms of|regarding|when|on)\s+(.+?))?\s*[?.!]?$/i;
+
+/** Upper bound on one LLM-proposed sub-query, in characters. */
+const MAX_LLM_SUBQUERY_LENGTH = 500;
+
+/**
+ * Query decomposition planner. The default heuristic needs no LLM:
+ *
+ * - splits only at clause/question boundaries: "?" / "." followed by more
+ *   text, ";", and "and" / "or" / "as well as" (optionally after a comma)
+ *   when a question word, auxiliary verb or imperative follows;
+ * - keeps noun-phrase conjunctions intact ("salt and pepper", "A or B");
+ * - expands comparisons ("X vs Y", "compare X and Y", "difference between X
+ *   and Y") into one sub-query per entity plus the comparison itself.
+ *
+ * `plan(query, { llm })` can instead ask an LLM for a JSON array of
+ * sub-queries (strictly validated, heuristic fallback).
  *
  * @extends EventEmitter
  */
 class QueryPlanner extends EventEmitter {
   /**
    * @param {object} [options] - Override default configuration
-   * @param {number} [options.minComplexityToSplit=2] - Minimum complexity score before splitting
+   * @param {number} [options.minComplexityToSplit=2] - Minimum complexity
+   *   score before splitting at conjunctions; punctuation boundaries and
+   *   comparisons are always split
    * @param {number} [options.maxSubQueries=5] - Maximum number of sub-queries to produce
    */
   constructor(options = {}) {
@@ -56,6 +145,8 @@ class QueryPlanner extends EventEmitter {
    *   +1 if length > 100 characters
    *   +1 per semicolon
    *   +1 per comma-separated clause (commas - 1 if > 0 commas)
+   *   +1 per conjunction followed by a question word, auxiliary or
+   *      imperative ("... and how ...", "... and explain ...")
    *
    * @param {string} query
    * @returns {number}
@@ -66,12 +157,10 @@ class QueryPlanner extends EventEmitter {
     const words = lower.split(/\s+/);
     let score = 0;
 
-    // +1 per question word
     for (const w of QUESTION_WORDS) {
       if (words.includes(w)) score += 1;
     }
 
-    // +1 per conjunction
     for (const c of CONJUNCTIONS) {
       // "as well as" is multi-word -- use indexOf on the full string
       if (c.includes(' ')) {
@@ -81,17 +170,17 @@ class QueryPlanner extends EventEmitter {
       }
     }
 
-    // +1 if length > 100 chars
     if (query.length > 100) score += 1;
 
-    // +1 per semicolon
     const semicolons = (query.match(/;/g) || []).length;
     score += semicolons;
 
-    // +1 per comma-separated clause (number of commas)
     const commas = (query.match(/,/g) || []).length;
     if (commas > 0) score += commas;
 
+    // +1 per conjunction that opens a new question or instruction
+    score += query.split(CONJUNCTION_BOUNDARY).length - 1;
+
     return score;
   }
 
@@ -104,14 +193,12 @@ class QueryPlanner extends EventEmitter {
    */
   _detectStrategy(text) {
     const lower = text.toLowerCase();
-    const words = lower.split(/\s+/);
+    const words = lower.split(/[^a-z]+/);
 
-    // Check comparative keywords first (more specific)
     for (const kw of COMPARATIVE_KEYWORDS) {
-      if (words.includes(kw) || lower.includes(kw)) return 'comparative';
+      if (words.includes(kw) || words.includes(`${kw}s`)) return 'comparative';
     }
 
-    // Check question words
     for (const w of words) {
       if (['who', 'what', 'where', 'when'].includes(w)) return 'factual';
       if (['why', 'how'].includes(w)) return 'analytical';
@@ -121,85 +208,249 @@ class QueryPlanner extends EventEmitter {
   }
 
   /**
-   * Plan query decomposition.
+   * Split a clause describing a comparison into its entities.
    *
-   * @param {string} query - The user query to decompose
-   * @returns {{
-   *   subQueries: Array<{subQuery: string, strategy: string, priority: number}>,
-   *   complexity: number,
-   *   originalQuery: string
-   * }}
-   * @throws {Error} If query is not a non-empty string
-   * @fires QueryPlanner#planned
+   * @param {string} clause
+   * @returns {{ entities: string[], aspect: string|null }|null}
+   * @private
    */
-  plan(query) {
-    // 1. Validate
-    if (!query || typeof query !== 'string' || !query.trim()) {
-      throw new Error('QueryPlanner.plan(): query must be a non-empty string');
+  _parseComparison(clause) {
+    const text = clause.trim();
+    let match = text.match(COMPARE_RE) || text.match(DIFFERENCE_RE);
+    let left;
+    if (match) {
+      left = match[1];
+    } else {
+      match = text.match(VERSUS_RE);
+      if (!match) return null;
+      // "Which is faster, Postgres vs MySQL" -> "Postgres"
+      left = match[1].split(/[,:]/).pop();
+      left = left.replace(
+        /^\s*(?:(?:what|which|who)\s+(?:is|are)\s+\w+(?:\s+\w+)?\s*[:,-]?\s*)/i,
+        '',
+      );
     }
+    const clean = (s) =>
+      String(s || '')
+        .trim()
+        .replace(/^(?:the|a|an)\s+/i, '')
+        .replace(/[?.!,;:]+$/, '')
+        .trim();
+    const entities = [...left.split(/\s*,\s*(?:and\s+)?|\s+and\s+/i), match[2]]
+      .map(clean)
+      .filter(Boolean);
+    if (entities.length < 2) return null;
+    const aspect = match[3] ? clean(match[3]) : null;
+    return { entities, aspect };
+  }
 
-    const trimmed = query.trim();
-
-    // 2. Compute complexity
-    const complexity = this._computeComplexity(trimmed);
+  /**
+   * Heuristic decomposition into sub-query strings.
+   *
+   * @param {string} query - Trimmed query
+   * @param {number} complexity
+   * @returns {Array<{subQuery: string, strategy: string}>}
+   * @private
+   */
+  _decompose(query, complexity) {
+    const tidy = (s) =>
+      s
+        .trim()
+        .replace(/^[,;]+\s*|[,;]+$/g, '')
+        .trim();
 
-    // 3. If complexity below threshold, return single sub-query
-    if (complexity < this.config.minComplexityToSplit) {
-      const result = {
-        subQueries: [
-          {
-            subQuery: trimmed,
-            strategy: this._detectStrategy(trimmed),
-            priority: 0,
-          },
-        ],
-        complexity,
-        originalQuery: trimmed,
-      };
-      this.emit('planned', result);
-      return result;
-    }
-
-    // 4. Split into sub-queries
-    let parts = [trimmed];
-    for (const pattern of SPLIT_PATTERNS) {
-      const newParts = [];
-      for (const part of parts) {
-        const split = part
-          .split(pattern)
-          .map((s) => s.trim())
-          .filter(Boolean);
-        newParts.push(...split);
+    let clauses = [];
+    for (const piece of query.split(PUNCTUATION_BOUNDARY)) {
+      const prev = clauses[clauses.length - 1];
+      // "e.g. Google" / "Dr. Smith": an abbreviation, not a sentence end.
+      if (prev && ABBREVIATION_END.test(prev)) {
+        clauses[clauses.length - 1] = `${prev} ${piece}`;
+      } else {
+        clauses.push(piece);
       }
-      parts = newParts;
+    }
+    clauses = clauses.map(tidy).filter(Boolean);
+    if (complexity >= this.config.minComplexityToSplit) {
+      clauses = clauses.flatMap((c) =>
+        c.split(CONJUNCTION_BOUNDARY).map(tidy).filter(Boolean),
+      );
     }
 
-    // Build sub-queries with strategy and priority
-    let subQueries = parts.map((part, i) => ({
-      subQuery: part,
-      strategy: this._detectStrategy(part),
-      priority: i,
-    }));
+    const out = [];
+    for (const clause of clauses) {
+      const comparison = this._parseComparison(clause);
+      if (comparison) {
+        for (const entity of comparison.entities) {
+          out.push({
+            subQuery: comparison.aspect
+              ? `${entity} ${comparison.aspect}`
+              : entity,
+            strategy: 'factual',
+          });
+        }
+        out.push({ subQuery: clause, strategy: 'comparative' });
+      } else {
+        out.push({ subQuery: clause, strategy: this._detectStrategy(clause) });
+      }
+    }
+    return out;
+  }
 
-    // 5. Deduplicate by subQuery text (case-insensitive)
+  /**
+   * Turn sub-query texts into the plan result (dedupe, limit, priorities).
+   *
+   * @param {Array<{subQuery: string, strategy: string}>} items
+   * @param {number} complexity
+   * @param {string} originalQuery
+   * @param {'heuristic'|'llm'} source
+   * @returns {object}
+   * @private
+   */
+  _finalize(items, complexity, originalQuery, source) {
     const seen = new Set();
-    subQueries = subQueries.filter((sq) => {
+    const unique = items.filter((sq) => {
       const key = sq.subQuery.toLowerCase();
       if (seen.has(key)) return false;
       seen.add(key);
       return true;
     });
+    const subQueries = unique
+      .slice(0, this.config.maxSubQueries)
+      .map((sq, i) => ({
+        subQuery: sq.subQuery,
+        strategy: sq.strategy,
+        priority: i,
+      }));
+    return { subQueries, complexity, originalQuery, source };
+  }
 
-    // Limit to maxSubQueries
-    subQueries = subQueries.slice(0, this.config.maxSubQueries);
+  /**
+   * Heuristic plan for a trimmed query.
+   * @private
+   */
+  _heuristicPlan(trimmed) {
+    const complexity = this._computeComplexity(trimmed);
+    const items = this._decompose(trimmed, complexity);
+    if (items.length === 0) {
+      items.push({
+        subQuery: trimmed,
+        strategy: this._detectStrategy(trimmed),
+      });
+    }
+    return this._finalize(items, complexity, trimmed, 'heuristic');
+  }
+
+  /**
+   * Strictly validate an LLM's sub-query proposal: a JSON array (optionally
+   * in a ```json fence) of 1..maxSubQueries non-empty, single-line strings of
+   * at most 500 characters. Anything else is rejected.
+   *
+   * @param {*} raw - Value returned by llm.generate()
+   * @returns {string[]|null}
+   * @private
+   */
+  _parseLlmSubQueries(raw) {
+    let text = raw;
+    if (text && typeof text === 'object') {
+      text = text.text ?? text.content ?? text.answer;
+    }
+    if (typeof text !== 'string') return null;
+    text = text.trim();
+    const fenced = text.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i);
+    if (fenced) text = fenced[1].trim();
+    if (!text.startsWith('[') || !text.endsWith(']')) return null;
+
+    let parsed;
+    try {
+      parsed = JSON.parse(text);
+    } catch (_err) {
+      return null;
+    }
+    if (!Array.isArray(parsed)) return null;
+    if (parsed.length === 0 || parsed.length > this.config.maxSubQueries) {
+      return null;
+    }
+    const out = [];
+    for (const item of parsed) {
+      if (typeof item !== 'string') return null;
+      const s = item.trim();
+      if (!s || s.length > MAX_LLM_SUBQUERY_LENGTH || /[\r\n]/.test(s)) {
+        return null;
+      }
+      out.push(s);
+    }
+    return out;
+  }
+
+  /**
+   * Ask the LLM for sub-queries; fall back to the heuristic plan.
+   * @private
+   */
+  async _planWithLlm(trimmed, { llm, signal }) {
+    const heuristic = this._heuristicPlan(trimmed);
+    let result = heuristic;
+    try {
+      const prompt =
+        'Split the user question into independent search queries for a ' +
+        'retrieval system. Keep phrases such as "salt and pepper" intact; ' +
+        'for a comparison, give one query per compared item plus the ' +
+        `comparison. Reply with ONLY a JSON array of 1 to ${this.config.maxSubQueries} ` +
+        `strings.\n\nQuestion: ${trimmed}`;
+      const raw = await llm.generate(prompt, [], signal ? { signal } : {});
+      const proposed = this._parseLlmSubQueries(raw);
+      if (proposed) {
+        result = this._finalize(
+          proposed.map((s) => ({
+            subQuery: s,
+            strategy: this._detectStrategy(s),
+          })),
+          heuristic.complexity,
+          trimmed,
+          'llm',
+        );
+      }
+    } catch (err) {
+      // A cancelled run must not continue with the heuristic plan.
+      if (signal && signal.aborted) throw err;
+    }
+    this.emit('planned', result);
+    return result;
+  }
+
+  /**
+   * Plan query decomposition.
+   *
+   * Without `options.llm` this is synchronous and returns the plan. With
+   * `options.llm` (an object with `generate()`), it returns a Promise of the
+   * plan: the LLM is asked for a JSON array of sub-queries, which is strictly
+   * validated (see _parseLlmSubQueries); on any failure the heuristic plan is
+   * used. `result.source` says which one was used.
+   *
+   * @param {string} query - The user query to decompose
+   * @param {object} [options]
+   * @param {{ generate: Function }} [options.llm] - Optional LLM planner
+   * @param {AbortSignal} [options.signal] - Passed to llm.generate()
+   * @returns {{
+   *   subQueries: Array<{subQuery: string, strategy: string, priority: number}>,
+   *   complexity: number,
+   *   originalQuery: string,
+   *   source: 'heuristic'|'llm'
+   * }|Promise<object>}
+   * @throws {Error} If query is not a non-empty string
+   * @fires QueryPlanner#planned
+   */
+  plan(query, options = {}) {
+    if (!query || typeof query !== 'string' || !query.trim()) {
+      throw new Error('QueryPlanner.plan(): query must be a non-empty string');
+    }
+    const trimmed = query.trim();
 
-    // 6. Emit 'planned' event
-    const result = {
-      subQueries,
-      complexity,
-      originalQuery: trimmed,
-    };
+    const llm = options && options.llm;
+    if (llm && typeof llm.generate === 'function') {
+      return this._planWithLlm(trimmed, options);
+    }
 
+    const result = this._heuristicPlan(trimmed);
     this.emit('planned', result);
     return result;
   }

package/src/citation/citation-tracker.js

@@ -10,7 +10,7 @@ const { detectHallucinations } = require('./hallucination-detector');
  * @type {object}
  */
 const DEFAULT_CONFIG = {
-  similarityThreshold: 0.3,
+  similarityThreshold: 0.5,
   maxCitationsPerSentence: 3,
   useIDFWeighting: true,
 };
@@ -24,7 +24,7 @@ const DEFAULT_CONFIG = {
 class CitationTracker extends EventEmitter {
   /**
    * @param {object} [options] - Override default configuration
-   * @param {number} [options.similarityThreshold=0.3] - Minimum similarity for a citation match
+   * @param {number} [options.similarityThreshold=0.5] - Minimum similarity for a citation match
    * @param {number} [options.maxCitationsPerSentence=3] - Max citations per sentence
    * @param {boolean} [options.useIDFWeighting=true] - Whether to apply IDF weighting
    */
@@ -76,9 +76,11 @@ class CitationTracker extends EventEmitter {
         'CitationTracker.track(): answer must be a non-empty string',
       );
     }
-    if (!Array.isArray(retrievedDocs) || retrievedDocs.length === 0) {
+    // An empty array is valid (retrieval found nothing): every sentence is
+    // then reported as unsupported rather than failing the whole run.
+    if (!Array.isArray(retrievedDocs)) {
       throw new Error(
-        'CitationTracker.track(): retrievedDocs must be a non-empty array',
+        'CitationTracker.track(): retrievedDocs must be an array',
       );
     }
 

package/src/citation/hallucination-detector.js

@@ -10,7 +10,7 @@
  *
  * @param {Array<{sentence: string, sources: Array<{docId: string|undefined, docIndex: number, score: number}>}>} citationResults
  * @param {object} [options]
- * @param {number} [options.threshold=0.3] - Score threshold for grounded classification
+ * @param {number} [options.threshold=0.5] - Score threshold for grounded classification
  * @returns {{
  *   hallucinationRate: number,
  *   sentences: Array<{text: string, classification: string, bestScore: number, bestSource: object|null}>,
@@ -18,7 +18,7 @@
  * }}
  */
 function detectHallucinations(citationResults, options = {}) {
-  const { threshold = 0.3 } = options;
+  const { threshold = 0.5 } = options;
 
   if (!Array.isArray(citationResults) || citationResults.length === 0) {
     return {

package/src/citation/source-mapper.js

@@ -1,10 +1,12 @@
 'use strict';
 
-const { tokenize, computeJaccardSimilarity } = require('../evaluate/scoring');
+const { tokenize, computeSupportScore } = require('../evaluate/scoring');
 
 /**
  * Build IDF (Inverse Document Frequency) weights from a collection of documents.
- * IDF = log(totalDocs / docsContainingToken)
+ * IDF = log(1 + totalDocs / docsContainingToken). The +1 keeps weights
+ * positive, so tokens found in every document (and every token when only one
+ * document was retrieved) still count instead of weighing zero.
  * @param {Array<{content?: string, text?: string}>} docs - Array of document objects
  * @returns {Map<string, number>} Map of token to IDF weight
  */
@@ -24,24 +26,25 @@ function buildIDFWeights(docs) {
 
   const idfWeights = new Map();
   for (const [token, df] of docFrequency) {
-    idfWeights.set(token, Math.log(totalDocs / df));
+    idfWeights.set(token, Math.log(1 + totalDocs / df));
   }
 
   return idfWeights;
 }
 
 /**
- * Map a sentence to its most relevant source documents using token similarity.
+ * Map a sentence to its most relevant source documents by how much of the
+ * sentence each document supports (see computeSupportScore).
  * @param {string} sentence - The sentence to map
  * @param {Array<{content?: string, text?: string, id?: string}>} docs - Source documents
  * @param {object} [options] - Configuration options
  * @param {number} [options.maxCitations=3] - Maximum number of citations to return
- * @param {number} [options.threshold=0.3] - Minimum similarity threshold
+ * @param {number} [options.threshold=0.5] - Minimum similarity threshold
  * @param {Map<string, number>|null} [options.idfWeights=null] - Optional IDF weights for weighted similarity
  * @returns {Array<{docIndex: number, docId: string|undefined, score: number, matchedTokens: string[]}>}
  */
 function mapSentenceToSources(sentence, docs, options = {}) {
-  const { maxCitations = 3, threshold = 0.3, idfWeights = null } = options;
+  const { maxCitations = 3, threshold = 0.5, idfWeights = null } = options;
 
   const sentenceTokens = tokenize(sentence);
   if (!sentenceTokens.length) return [];
@@ -54,29 +57,13 @@ function mapSentenceToSources(sentence, docs, options = {}) {
     const docTokens = tokenize(docText);
     if (!docTokens.length) continue;
 
-    let score;
-    const sentenceSet = new Set(sentenceTokens);
+    // Share of the sentence's content tokens found in this document.
     const docSet = new Set(docTokens);
-    const matched = [...sentenceSet].filter((t) => docSet.has(t));
-
-    if (idfWeights) {
-      // IDF-weighted Jaccard: sum IDF weights of intersection / sum IDF weights of union
-      const union = new Set([...sentenceSet, ...docSet]);
-      let intersectionWeight = 0;
-      let unionWeight = 0;
-
-      for (const token of union) {
-        const weight = idfWeights.get(token) || 1;
-        if (sentenceSet.has(token) && docSet.has(token)) {
-          intersectionWeight += weight;
-        }
-        unionWeight += weight;
-      }
-
-      score = unionWeight > 0 ? intersectionWeight / unionWeight : 0;
-    } else {
-      score = computeJaccardSimilarity(sentenceTokens, docTokens);
-    }
+    const score = computeSupportScore(sentenceTokens, docSet, idfWeights, {
+      claimText: sentence,
+      docText,
+    });
+    const matched = [...new Set(sentenceTokens)].filter((t) => docSet.has(t));
 
     if (score >= threshold) {
       results.push({