@devilsdev/rag-pipeline-utils 2.3.1 → 2.4.1

package/src/chunking/strategies/sentence.js

@@ -0,0 +1,32 @@
+'use strict';
+
+/**
+ * Sentence-based chunking strategy.
+ * Extracted from MarkdownLoader._chunkText for backward compatibility.
+ * Splits text on sentence boundaries and merges into chunks up to maxLen.
+ *
+ * @param {string} text - The input text to chunk.
+ * @param {object} [options={}] - Chunking options.
+ * @param {number} [options.maxLen=500] - Maximum character length per chunk.
+ * @returns {string[]} Array of text chunks.
+ */
+function sentenceChunk(text, options = {}) {
+  const { maxLen = 500 } = options;
+
+  const sentences = text.split(/(?<=[.!?])\s+/);
+  const chunks = [];
+  let buffer = '';
+
+  for (const sentence of sentences) {
+    if ((buffer + sentence).length <= maxLen) {
+      buffer += sentence + ' ';
+    } else {
+      chunks.push(buffer.trim());
+      buffer = sentence + ' ';
+    }
+  }
+  if (buffer.trim()) chunks.push(buffer.trim());
+  return chunks;
+}
+
+module.exports = { sentenceChunk };

package/src/chunking/strategies/structure-aware.js

@@ -0,0 +1,138 @@
+'use strict';
+
+const { recursiveChunk } = require('./recursive');
+
+/**
+ * Detect document format from content.
+ *
+ * @param {string} text - The text to sniff.
+ * @returns {'markdown'|'html'|'plain'} Detected format.
+ * @private
+ */
+function _detectFormat(text) {
+  // Check for Markdown headers
+  if (/^#{1,6}\s/m.test(text)) {
+    return 'markdown';
+  }
+  // Check for HTML heading or section tags
+  if (/<h[1-6][^>]*>/i.test(text) || /<section[^>]*>/i.test(text)) {
+    return 'html';
+  }
+  return 'plain';
+}
+
+/**
+ * Split Markdown text into sections by header boundaries.
+ * Keeps headers as chunk prefixes and preserves code blocks as atomic units.
+ *
+ * @param {string} text - Markdown text.
+ * @returns {string[]} Array of sections with their header prefixes.
+ * @private
+ */
+function _splitMarkdown(text) {
+  // Protect code blocks by replacing them with placeholders
+  const codeBlocks = [];
+  const withPlaceholders = text.replace(/```[\s\S]*?```/g, (match) => {
+    const index = codeBlocks.length;
+    codeBlocks.push(match);
+    return `__CODE_BLOCK_${index}__`;
+  });
+
+  // Split on header lines
+  const sections = [];
+  const lines = withPlaceholders.split('\n');
+  let currentSection = '';
+
+  for (const line of lines) {
+    if (/^#{1,6}\s/.test(line) && currentSection.trim()) {
+      sections.push(currentSection.trim());
+      currentSection = line + '\n';
+    } else {
+      currentSection += line + '\n';
+    }
+  }
+
+  if (currentSection.trim()) {
+    sections.push(currentSection.trim());
+  }
+
+  // Restore code blocks
+  return sections.map((section) =>
+    section.replace(
+      /__CODE_BLOCK_(\d+)__/g,
+      (_, idx) => codeBlocks[Number(idx)],
+    ),
+  );
+}
+
+/**
+ * Split HTML text into sections by heading and section tag boundaries.
+ *
+ * @param {string} text - HTML text.
+ * @returns {string[]} Array of sections.
+ * @private
+ */
+function _splitHtml(text) {
+  // Split on heading tags or section tags
+  const parts = text.split(/(?=<(?:h[1-6]|section)[^>]*>)/i);
+  return parts.map((p) => p.trim()).filter((p) => p.length > 0);
+}
+
+/**
+ * Structure-aware chunking strategy.
+ * Detects document format (Markdown, HTML, or plain) and respects
+ * structural boundaries. For Markdown, splits on headers and preserves
+ * code blocks. For HTML, splits on heading/section tags. Oversized
+ * sections are further split via the recursive strategy.
+ *
+ * @param {string} text - The input text to chunk.
+ * @param {object} [options={}] - Chunking options.
+ * @param {number} [options.chunkSize=512] - Maximum chunk size in characters.
+ * @param {number} [options.chunkOverlap=50] - Number of overlapping characters between chunks.
+ * @param {string} [options.format] - Document format: 'markdown', 'html', 'plain', or auto-detect.
+ * @returns {string[]} Array of text chunks.
+ */
+function structureAwareChunk(text, options = {}) {
+  const { chunkSize = 512, chunkOverlap = 50, format } = options;
+
+  if (!text || typeof text !== 'string') {
+    return [];
+  }
+
+  const detectedFormat = format || _detectFormat(text);
+
+  let sections;
+
+  switch (detectedFormat) {
+    case 'markdown':
+      sections = _splitMarkdown(text);
+      break;
+    case 'html':
+      sections = _splitHtml(text);
+      break;
+    case 'plain':
+    default:
+      sections = [text];
+      break;
+  }
+
+  // Apply chunkSize limit: oversized sections get further split via recursive strategy
+  const chunks = [];
+
+  for (const section of sections) {
+    const trimmed = section.trim();
+    if (!trimmed) {
+      continue;
+    }
+    if (trimmed.length <= chunkSize) {
+      chunks.push(trimmed);
+    } else {
+      const subChunks = recursiveChunk(trimmed, { chunkSize, chunkOverlap });
+      chunks.push(...subChunks);
+    }
+  }
+
+  return chunks.filter((c) => c.length > 0);
+}
+
+module.exports = { structureAwareChunk };

package/src/citation/citation-tracker.js

@@ -0,0 +1,161 @@
+'use strict';
+
+const { EventEmitter } = require('events');
+const { splitSentences, tokenize } = require('../evaluate/scoring');
+const { mapSentenceToSources, buildIDFWeights } = require('./source-mapper');
+const { detectHallucinations } = require('./hallucination-detector');
+
+/**
+ * Default configuration for the CitationTracker.
+ * @type {object}
+ */
+const DEFAULT_CONFIG = {
+  similarityThreshold: 0.3,
+  maxCitationsPerSentence: 3,
+  useIDFWeighting: true,
+};
+
+/**
+ * Tracks citations between an LLM-generated answer and retrieved source documents.
+ * Emits a 'tracked' event when citation analysis completes.
+ *
+ * @extends EventEmitter
+ */
+class CitationTracker extends EventEmitter {
+  /**
+   * @param {object} [options] - Override default configuration
+   * @param {number} [options.similarityThreshold=0.3] - Minimum similarity for a citation match
+   * @param {number} [options.maxCitationsPerSentence=3] - Max citations per sentence
+   * @param {boolean} [options.useIDFWeighting=true] - Whether to apply IDF weighting
+   */
+  constructor(options = {}) {
+    super();
+    this.config = { ...DEFAULT_CONFIG, ...options };
+  }
+
+  /**
+   * Normalize a document object to ensure it has a usable text content string.
+   * Looks for .content, .text, or .chunk fields. If .chunk is a function, calls it and joins the result.
+   * @param {object} doc
+   * @returns {object} Normalized doc with .content field
+   * @private
+   */
+  _normalizeDoc(doc) {
+    if (doc.content && typeof doc.content === 'string') return doc;
+
+    let content = '';
+    if (typeof doc.text === 'string') {
+      content = doc.text;
+    } else if (typeof doc.chunk === 'function') {
+      const chunks = doc.chunk();
+      content = Array.isArray(chunks) ? chunks.join(' ') : String(chunks);
+    } else if (typeof doc.chunk === 'string') {
+      content = doc.chunk;
+    }
+
+    return { ...doc, content };
+  }
+
+  /**
+   * Track citations for an answer against retrieved documents.
+   *
+   * @param {string} answer - The LLM-generated answer text
+   * @param {Array<object>} retrievedDocs - Array of source documents (each should have .content, .text, or .chunk)
+   * @returns {{
+   *   citations: Array<{sentence: string, sources: Array<{docId: string|undefined, docIndex: number, score: number}>}>,
+   *   groundednessScore: number,
+   *   hallucinationReport: object,
+   *   metadata: {sentenceCount: number, docCount: number, config: object}
+   * }}
+   * @fires CitationTracker#tracked
+   */
+  track(answer, retrievedDocs) {
+    // 1. Validate inputs
+    if (!answer || typeof answer !== 'string') {
+      throw new Error(
+        'CitationTracker.track(): answer must be a non-empty string',
+      );
+    }
+    if (!Array.isArray(retrievedDocs) || retrievedDocs.length === 0) {
+      throw new Error(
+        'CitationTracker.track(): retrievedDocs must be a non-empty array',
+      );
+    }
+
+    // 2. Normalize docs
+    const docs = retrievedDocs.map((doc) => this._normalizeDoc(doc));
+
+    // 3. Split answer into sentences
+    const sentences = splitSentences(answer);
+    if (sentences.length === 0) {
+      const result = {
+        citations: [],
+        groundednessScore: 0,
+        hallucinationReport: detectHallucinations([]),
+        metadata: {
+          sentenceCount: 0,
+          docCount: docs.length,
+          config: this.config,
+        },
+      };
+      this.emit('tracked', result);
+      return result;
+    }
+
+    // 4. Build IDF weights if enabled
+    const idfWeights = this.config.useIDFWeighting
+      ? buildIDFWeights(docs)
+      : null;
+
+    // 5 & 6. Map each sentence to sources and build citations array
+    const citations = sentences.map((sentence) => {
+      const sources = mapSentenceToSources(sentence, docs, {
+        maxCitations: this.config.maxCitationsPerSentence,
+        threshold: this.config.similarityThreshold,
+        idfWeights,
+      });
+
+      return {
+        sentence,
+        sources: sources.map((s) => ({
+          docId: s.docId,
+          docIndex: s.docIndex,
+          score: s.score,
+        })),
+      };
+    });
+
+    // 7. Detect hallucinations
+    const hallucinationReport = detectHallucinations(citations, {
+      threshold: this.config.similarityThreshold,
+    });
+
+    // 8. Compute groundedness score
+    const groundedCount = hallucinationReport.summary.grounded;
+    const groundednessScore =
+      sentences.length > 0 ? groundedCount / sentences.length : 0;
+
+    // 9. Emit 'tracked' event
+    const result = {
+      citations,
+      groundednessScore,
+      hallucinationReport,
+      metadata: {
+        sentenceCount: sentences.length,
+        docCount: docs.length,
+        config: this.config,
+      },
+    };
+
+    /**
+     * @event CitationTracker#tracked
+     * @type {object}
+     */
+    this.emit('tracked', result);
+
+    // 10. Return result
+    return result;
+  }
+}
+
+module.exports = { CitationTracker, DEFAULT_CONFIG };

package/src/citation/hallucination-detector.js

@@ -0,0 +1,74 @@
+'use strict';
+
+/**
+ * Detect hallucinations by analyzing citation results and classifying each sentence.
+ *
+ * Classification:
+ * - 'definite_hallucination': bestScore === 0 (no overlap with any source)
+ * - 'likely_hallucination': bestScore > 0 but below threshold
+ * - 'grounded': bestScore >= threshold
+ *
+ * @param {Array<{sentence: string, sources: Array<{docId: string|undefined, docIndex: number, score: number}>}>} citationResults
+ * @param {object} [options]
+ * @param {number} [options.threshold=0.3] - Score threshold for grounded classification
+ * @returns {{
+ *   hallucinationRate: number,
+ *   sentences: Array<{text: string, classification: string, bestScore: number, bestSource: object|null}>,
+ *   summary: {grounded: number, likely_hallucination: number, definite_hallucination: number}
+ * }}
+ */
+function detectHallucinations(citationResults, options = {}) {
+  const { threshold = 0.3 } = options;
+
+  if (!Array.isArray(citationResults) || citationResults.length === 0) {
+    return {
+      hallucinationRate: 0,
+      sentences: [],
+      summary: {
+        grounded: 0,
+        likely_hallucination: 0,
+        definite_hallucination: 0,
+      },
+    };
+  }
+
+  const summary = {
+    grounded: 0,
+    likely_hallucination: 0,
+    definite_hallucination: 0,
+  };
+  const sentences = [];
+
+  for (const entry of citationResults) {
+    const sources = entry.sources || [];
+    const bestSource = sources.length > 0 ? sources[0] : null;
+    const bestScore = bestSource ? bestSource.score : 0;
+
+    let classification;
+    if (bestScore === 0) {
+      classification = 'definite_hallucination';
+    } else if (bestScore < threshold) {
+      classification = 'likely_hallucination';
+    } else {
+      classification = 'grounded';
+    }
+
+    summary[classification]++;
+    sentences.push({
+      text: entry.sentence,
+      classification,
+      bestScore,
+      bestSource,
+    });
+  }
+
+  const total = sentences.length;
+  const hallucinationRate =
+    total > 0
+      ? (summary.definite_hallucination + summary.likely_hallucination) / total
+      : 0;
+
+  return { hallucinationRate, sentences, summary };
+}
+
+module.exports = { detectHallucinations };

package/src/citation/source-mapper.js

@@ -0,0 +1,95 @@
+'use strict';
+
+const { tokenize, computeJaccardSimilarity } = require('../evaluate/scoring');
+
+/**
+ * Build IDF (Inverse Document Frequency) weights from a collection of documents.
+ * IDF = log(totalDocs / docsContainingToken)
+ * @param {Array<{content?: string, text?: string}>} docs - Array of document objects
+ * @returns {Map<string, number>} Map of token to IDF weight
+ */
+function buildIDFWeights(docs) {
+  const totalDocs = docs.length;
+  if (totalDocs === 0) return new Map();
+
+  const docFrequency = new Map();
+
+  for (const doc of docs) {
+    const text = doc.content || doc.text || '';
+    const tokens = new Set(tokenize(text));
+    for (const token of tokens) {
+      docFrequency.set(token, (docFrequency.get(token) || 0) + 1);
+    }
+  }
+
+  const idfWeights = new Map();
+  for (const [token, df] of docFrequency) {
+    idfWeights.set(token, Math.log(totalDocs / df));
+  }
+
+  return idfWeights;
+}
+
+/**
+ * Map a sentence to its most relevant source documents using token similarity.
+ * @param {string} sentence - The sentence to map
+ * @param {Array<{content?: string, text?: string, id?: string}>} docs - Source documents
+ * @param {object} [options] - Configuration options
+ * @param {number} [options.maxCitations=3] - Maximum number of citations to return
+ * @param {number} [options.threshold=0.3] - Minimum similarity threshold
+ * @param {Map<string, number>|null} [options.idfWeights=null] - Optional IDF weights for weighted similarity
+ * @returns {Array<{docIndex: number, docId: string|undefined, score: number, matchedTokens: string[]}>}
+ */
+function mapSentenceToSources(sentence, docs, options = {}) {
+  const { maxCitations = 3, threshold = 0.3, idfWeights = null } = options;
+
+  const sentenceTokens = tokenize(sentence);
+  if (!sentenceTokens.length) return [];
+
+  const results = [];
+
+  for (let i = 0; i < docs.length; i++) {
+    const doc = docs[i];
+    const docText = doc.content || doc.text || '';
+    const docTokens = tokenize(docText);
+    if (!docTokens.length) continue;
+
+    let score;
+    const sentenceSet = new Set(sentenceTokens);
+    const docSet = new Set(docTokens);
+    const matched = [...sentenceSet].filter((t) => docSet.has(t));
+
+    if (idfWeights) {
+      // IDF-weighted Jaccard: sum IDF weights of intersection / sum IDF weights of union
+      const union = new Set([...sentenceSet, ...docSet]);
+      let intersectionWeight = 0;
+      let unionWeight = 0;
+
+      for (const token of union) {
+        const weight = idfWeights.get(token) || 1;
+        if (sentenceSet.has(token) && docSet.has(token)) {
+          intersectionWeight += weight;
+        }
+        unionWeight += weight;
+      }
+
+      score = unionWeight > 0 ? intersectionWeight / unionWeight : 0;
+    } else {
+      score = computeJaccardSimilarity(sentenceTokens, docTokens);
+    }
+
+    if (score >= threshold) {
+      results.push({
+        docIndex: i,
+        docId: doc.id,
+        score,
+        matchedTokens: matched,
+      });
+    }
+  }
+
+  results.sort((a, b) => b.score - a.score);
+  return results.slice(0, maxCitations);
+}
+
+module.exports = { mapSentenceToSources, buildIDFWeights };

package/src/cli/commands/ai-ml/adaptive-commands.js

@@ -0,0 +1,178 @@
+/**
+ * Adaptive retrieval CLI commands
+ * Handles user profiles, adaptive search, and feedback processing
+ */
+
+const chalk = require("chalk");
+const ora = require("ora");
+const inquirer = require("inquirer");
+const Table = require("cli-table3");
+
+/**
+ * Manage user profile (view or create)
+ * @param {AdaptiveRetrievalEngine} adaptiveRetrieval - The adaptive retrieval engine instance
+ * @param {string} userId - User ID
+ * @param {object} _options - Command options
+ */
+async function manageUserProfile(adaptiveRetrieval, userId, _options) {
+  const spinner = ora("Managing user profile...").start();
+
+  try {
+    let profile;
+
+    try {
+      profile = await adaptiveRetrieval.getUserProfile(userId);
+      spinner.succeed("User profile found");
+    } catch {
+      // Profile doesn't exist, create new one
+      const profileData = {};
+
+      if (_options.interests) {
+        profileData.interests = _options.interests
+          .split(",")
+          .map((i) => i.trim());
+      }
+
+      if (_options.expertise) {
+        profileData.expertise = _options.expertise;
+      }
+
+      profile = await adaptiveRetrieval.initializeUserProfile(
+        userId,
+        profileData,
+      );
+      spinner.succeed("User profile created");
+    }
+
+    const table = new Table({
+      head: ["Property", "Value"],
+      colWidths: [20, 50],
+    });
+
+    table.push(
+      ["User ID", profile.userId],
+      ["Interests", profile.preferences?.interests?.join(", ") || "None"],
+      ["Expertise", profile.preferences?.expertise || "Not set"],
+      [
+        "Learning History",
+        `${profile.learningHistory?.length || 0} interactions`,
+      ],
+      ["Created", profile.createdAt || "Unknown"],
+    );
+
+    console.log(table.toString());
+    // eslint-disable-line no-console
+  } catch (error) {
+    spinner.fail(`Profile management failed: ${error.message}`);
+  }
+}
+
+/**
+ * Perform adaptive search
+ * @param {AdaptiveRetrievalEngine} adaptiveRetrieval - The adaptive retrieval engine instance
+ * @param {string} userId - User ID
+ * @param {string} query - Search query
+ * @param {object} _options - Command options
+ */
+async function adaptiveSearch(adaptiveRetrieval, userId, query, _options) {
+  const spinner = ora("Performing adaptive search...").start();
+
+  try {
+    const searchOptions = {
+      maxResults: parseInt(_options.maxResults),
+      explain: !!_options.explain,
+    };
+
+    const results = await adaptiveRetrieval.adaptiveRetrieve(
+      userId,
+      query,
+      searchOptions,
+    );
+
+    spinner.succeed(`Found ${results.documents.length} results`);
+
+    // Display results
+    console.log(chalk.blue(`\nSearch Results for: "${query}"`));
+    // eslint-disable-line no-console
+    console.log(chalk.gray("=".repeat(50)));
+    // eslint-disable-line no-console
+
+    results.documents.forEach((doc, index) => {
+      console.log(chalk.green(`\n${index + 1}. ${doc.title || doc.id}`));
+      // eslint-disable-line no-console
+      console.log(chalk.gray(`   Score: ${doc.score?.toFixed(4) || "N/A"}`));
+      // eslint-disable-line no-console
+      console.log(
+        chalk.gray(
+          `   Personalized Score: ${doc.personalizedScore?.toFixed(4) || "N/A"}`,
+        ),
+      );
+      // eslint-disable-line no-console
+      console.log(`   ${doc.content?.substring(0, 200)}...`);
+      // eslint-disable-line no-console
+    });
+
+    if (_options.explain && results.adaptationMetadata) {
+      console.log(chalk.blue("\nAdaptation Explanation:"));
+      // eslint-disable-line no-console
+      console.log(
+        chalk.gray(JSON.stringify(results.adaptationMetadata, null, 2)),
+      );
+      // eslint-disable-line no-console
+    }
+  } catch (error) {
+    spinner.fail(`Adaptive search failed: ${error.message}`);
+  }
+}
+
+/**
+ * Provide feedback for learning
+ * @param {AdaptiveRetrievalEngine} adaptiveRetrieval - The adaptive retrieval engine instance
+ * @param {string} userId - User ID
+ * @param {object} _options - Command options
+ */
+async function provideFeedback(adaptiveRetrieval, userId, _options) {
+  if (!_options.query) {
+    const answers = await inquirer.prompt([
+      {
+        _type: "input",
+        name: "query",
+        message: "Enter the original query:",
+      },
+      {
+        _type: "input",
+        name: "ratings",
+        message: "Enter ratings for results (comma-separated, 1-5):",
+      },
+    ]);
+    _options.query = answers.query;
+    _options.ratings = answers.ratings;
+  }
+
+  const spinner = ora("Processing feedback...").start();
+
+  try {
+    const ratings = _options.ratings.split(",").map((r) => parseInt(r.trim()));
+
+    const feedback = {
+      query: _options.query,
+      ratings,
+      clickedResults: [0], // Assume first result was clicked
+      dwellTime: [120], // Assume 2 minutes dwell time
+    };
+
+    await adaptiveRetrieval.processFeedback(userId, feedback);
+
+    spinner.succeed("Feedback processed successfully");
+    console.log(chalk.green("\n✓ User profile updated with feedback"));
+    // eslint-disable-line no-console
+  } catch (error) {
+    spinner.fail(`Feedback processing failed: ${error.message}`);
+  }
+}
+
+module.exports = {
+  manageUserProfile,
+  adaptiveSearch,
+  provideFeedback,
+};