@devilsdev/rag-pipeline-utils 2.3.0 → 2.4.0

package/src/agentic/self-critique.js

@@ -0,0 +1,102 @@
+'use strict';
+
+const { EventEmitter } = require('events');
+
+/**
+ * Default configuration for the SelfCritiqueChecker.
+ * @type {object}
+ */
+const DEFAULT_CONFIG = {
+  approvalThreshold: 0.5,
+};
+
+/**
+ * Answer verification against retrieved source documents.
+ * Uses CitationTracker (lazy-loaded) to compute groundedness and detect hallucinations.
+ *
+ * @extends EventEmitter
+ */
+class SelfCritiqueChecker extends EventEmitter {
+  /**
+   * @param {object} [options] - Override default configuration
+   * @param {number} [options.approvalThreshold=0.5] - Minimum groundedness score to approve an answer
+   */
+  constructor(options = {}) {
+    super();
+    this.config = { ...DEFAULT_CONFIG, ...options };
+  }
+
+  /**
+   * Check an answer against retrieved documents for groundedness.
+   *
+   * @param {string} answer - The LLM-generated answer to verify
+   * @param {Array<object>} retrievedDocs - Source documents to check against
+   * @returns {{
+   *   approved: boolean,
+   *   score: number,
+   *   issues: Array<{sentence: string, classification: string}>,
+   *   citationResult: object
+   * }}
+   * @throws {Error} If answer or retrievedDocs are invalid
+   * @fires SelfCritiqueChecker#checked
+   */
+  check(answer, retrievedDocs) {
+    // 1. Validate inputs
+    if (!answer || typeof answer !== 'string' || !answer.trim()) {
+      throw new Error(
+        'SelfCritiqueChecker.check(): answer must be a non-empty string',
+      );
+    }
+    if (!Array.isArray(retrievedDocs) || retrievedDocs.length === 0) {
+      throw new Error(
+        'SelfCritiqueChecker.check(): retrievedDocs must be a non-empty array',
+      );
+    }
+
+    // 2. Lazy-require CitationTracker (not loaded at module level)
+    const { CitationTracker } = require('../citation/citation-tracker');
+
+    // 3. Create tracker instance
+    const tracker = new CitationTracker();
+
+    // 4. Track citations
+    const citationResult = tracker.track(answer, retrievedDocs);
+
+    // 5. Extract groundedness and hallucination data
+    const { groundednessScore, hallucinationReport } = citationResult;
+
+    // 6. Determine approval
+    const approved = groundednessScore >= this.config.approvalThreshold;
+
+    // 7. Build issues array from hallucination report
+    const issues = [];
+    if (hallucinationReport && Array.isArray(hallucinationReport.sentences)) {
+      for (const s of hallucinationReport.sentences) {
+        if (
+          s.classification === 'definite_hallucination' ||
+          s.classification === 'likely_hallucination'
+        ) {
+          issues.push({
+            sentence: s.sentence || s.text || '',
+            classification: s.classification,
+          });
+        }
+      }
+    }
+
+    // 8. Emit 'checked' event
+    const result = {
+      approved,
+      score: groundednessScore,
+      issues,
+      citationResult,
+    };
+
+    this.emit('checked', result);
+
+    // 9. Return
+    return result;
+  }
+}
+
+module.exports = { SelfCritiqueChecker, DEFAULT_CONFIG };

package/src/ai/adaptive-retrieval.js

@@ -186,6 +186,3 @@ const adaptiveRetrievalEngine = {
 };
 
 module.exports = adaptiveRetrievalEngine;
-
-// CJS+ESM interop pattern
-module.exports.default = module.exports;

package/src/ai/federated-learning.js

@@ -209,6 +209,3 @@ const federatedLearningCoordinator = {
 
 // Export singleton as default
 module.exports = federatedLearningCoordinator;
-
-// CJS+ESM interop pattern
-module.exports.default = module.exports;

package/src/ai/federation.js

@@ -68,9 +68,4 @@ class FederatedLearningCoordinator {
   }
 }
 
-// Create and export singleton instance
-const coordinator = new FederatedLearningCoordinator();
-
-module.exports = coordinator;
-module.exports.FederatedLearningCoordinator = FederatedLearningCoordinator;
-module.exports.default = module.exports;
+module.exports = { FederatedLearningCoordinator };

package/src/ai/index.js

@@ -1,6 +1,6 @@
 /**
  * AI/ML Module Index - Decomposed Architecture
- * Provides unified interface to 4 separate AI modules as required by CLAUDE.md
+ * Provides unified interface to 4 separate AI modules
  *
  * Decomposed modules:
  * - orchestrator.js - ModelTrainingOrchestrator
@@ -9,25 +9,14 @@
  * - federation.js - FederatedLearningCoordinator
  */
 
-const orchestrator = require("./orchestrator");
-const retrievalEngine = require("./retrieval-engine");
-const multimodal = require("./multimodal");
-const federation = require("./federation");
+const { ModelTrainingOrchestrator } = require("./orchestrator");
+const { AdaptiveRetrievalEngine } = require("./retrieval-engine");
+const { MultiModalProcessor } = require("./multimodal");
+const { FederatedLearningCoordinator } = require("./federation");
 
-// Export the decomposed modules
 module.exports = {
-  // Primary exports with new decomposed architecture
-  ModelTrainingOrchestrator: orchestrator,
-  AdaptiveRetrievalEngine: retrievalEngine,
-  MultiModalProcessor: multimodal,
-  FederatedLearningCoordinator: federation,
-
-  // Legacy names for backward compatibility
-  modelTrainer: orchestrator,
-  adaptiveRetrieval: retrievalEngine,
-  multiModalProcessor: multimodal,
-  federatedLearning: federation,
+  ModelTrainingOrchestrator,
+  AdaptiveRetrievalEngine,
+  MultiModalProcessor,
+  FederatedLearningCoordinator,
 };
-
-// CJS+ESM interop pattern
-module.exports.default = module.exports;

package/src/ai/model-training.js

@@ -200,6 +200,3 @@ const modelTrainingOrchestrator = {
 
 // Export singleton as default
 module.exports = modelTrainingOrchestrator;
-
-// CJS+ESM interop pattern
-module.exports.default = module.exports;

package/src/ai/multimodal-processing.js

@@ -263,6 +263,3 @@ const multiModalProcessor = {
 
 // Export singleton as default
 module.exports = multiModalProcessor;
-
-// CJS+ESM interop pattern
-module.exports.default = module.exports;

package/src/ai/multimodal.js

@@ -63,9 +63,4 @@ class MultiModalProcessor {
   }
 }
 
-// Create and export singleton instance
-const processor = new MultiModalProcessor();
-
-module.exports = processor;
-module.exports.MultiModalProcessor = MultiModalProcessor;
-module.exports.default = module.exports;
+module.exports = { MultiModalProcessor };

package/src/ai/orchestrator.js

@@ -62,9 +62,4 @@ class ModelTrainingOrchestrator {
   }
 }
 
-// Create and export singleton instance
-const orchestrator = new ModelTrainingOrchestrator();
-
-module.exports = orchestrator;
-module.exports.ModelTrainingOrchestrator = ModelTrainingOrchestrator;
-module.exports.default = module.exports;
+module.exports = { ModelTrainingOrchestrator };

package/src/ai/retrieval-engine.js

@@ -61,9 +61,4 @@ class AdaptiveRetrievalEngine {
   }
 }
 
-// Create and export singleton instance
-const engine = new AdaptiveRetrievalEngine();
-
-module.exports = engine;
-module.exports.AdaptiveRetrievalEngine = AdaptiveRetrievalEngine;
-module.exports.default = module.exports;
+module.exports = { AdaptiveRetrievalEngine };

package/src/chunking/chunking-engine.js

@@ -0,0 +1,142 @@
+'use strict';
+
+const { EventEmitter } = require('events');
+const { sentenceChunk } = require('./strategies/sentence');
+const { fixedSizeChunk } = require('./strategies/fixed-size');
+const { recursiveChunk } = require('./strategies/recursive');
+const { semanticChunk } = require('./strategies/semantic');
+const { structureAwareChunk } = require('./strategies/structure-aware');
+
+/**
+ * Default configuration for the chunking engine.
+ * @type {object}
+ */
+const DEFAULT_CONFIG = {
+  strategy: 'recursive',
+  chunkSize: 512,
+  chunkOverlap: 50,
+  minChunkSize: 50,
+  separators: ['\n\n', '\n', '. ', ' '],
+  similarityThreshold: 0.3,
+};
+
+/**
+ * ChunkingEngine orchestrates text chunking using pluggable strategies.
+ * Extends EventEmitter to emit 'chunk' events after each chunking operation.
+ *
+ * @extends EventEmitter
+ */
+class ChunkingEngine extends EventEmitter {
+  /**
+   * Create a ChunkingEngine instance.
+   *
+   * @param {object} [options={}] - Configuration options merged with DEFAULT_CONFIG.
+   * @param {string} [options.strategy='recursive'] - Default strategy name.
+   * @param {number} [options.chunkSize=512] - Default chunk size.
+   * @param {number} [options.chunkOverlap=50] - Default chunk overlap.
+   * @param {number} [options.minChunkSize=50] - Default minimum chunk size.
+   * @param {string[]} [options.separators] - Default separators for recursive strategy.
+   * @param {number} [options.similarityThreshold=0.3] - Default similarity threshold for semantic strategy.
+   */
+  constructor(options = {}) {
+    super();
+    this._config = { ...DEFAULT_CONFIG, ...options };
+    this._strategies = new Map();
+
+    // Register built-in strategies
+    this._strategies.set('sentence', sentenceChunk);
+    this._strategies.set('fixed-size', fixedSizeChunk);
+    this._strategies.set('recursive', recursiveChunk);
+    this._strategies.set('semantic', semanticChunk);
+    this._strategies.set('structure-aware', structureAwareChunk);
+  }
+
+  /**
+   * Chunk text using the configured or specified strategy.
+   *
+   * @param {string} text - The input text to chunk. Must be a non-empty string.
+   * @param {object} [options={}] - Per-call options that override the engine config.
+   * @param {string} [options.strategy] - Strategy name to use for this call.
+   * @returns {string[]} Array of text chunks.
+   * @throws {Error} If text is not a non-empty string.
+   * @throws {Error} If the specified strategy is not registered.
+   * @fires ChunkingEngine#chunk
+   */
+  chunk(text, options = {}) {
+    if (!text || typeof text !== 'string' || text.trim().length === 0) {
+      throw new Error('text must be a non-empty string');
+    }
+
+    const mergedOptions = { ...this._config, ...options };
+    const strategyName = mergedOptions.strategy;
+
+    const strategyFn = this._strategies.get(strategyName);
+    if (!strategyFn) {
+      throw new Error(
+        `Unknown chunking strategy "${strategyName}". Available strategies: ${this.listStrategies().join(', ')}`,
+      );
+    }
+
+    const results = strategyFn(text, mergedOptions);
+
+    // Filter out empty chunks
+    const chunks = results.filter((chunk) => chunk && chunk.trim().length > 0);
+
+    /**
+     * Chunk event emitted after a successful chunking operation.
+     *
+     * @event ChunkingEngine#chunk
+     * @type {object}
+     * @property {string} strategy - The strategy name used.
+     * @property {number} inputLength - Character length of the input text.
+     * @property {number} chunkCount - Number of resulting chunks.
+     */
+    this.emit('chunk', {
+      strategy: strategyName,
+      inputLength: text.length,
+      chunkCount: chunks.length,
+    });
+
+    return chunks;
+  }
+
+  /**
+   * Register a custom chunking strategy.
+   *
+   * @param {string} name - Strategy name (used to select it via options.strategy).
+   * @param {function} fn - Strategy function with signature (text, options) => string[].
+   * @throws {Error} If name is not a non-empty string.
+   * @throws {Error} If fn is not a function.
+   */
+  registerStrategy(name, fn) {
+    if (!name || typeof name !== 'string') {
+      throw new Error('Strategy name must be a non-empty string');
+    }
+    if (typeof fn !== 'function') {
+      throw new Error(
+        'Strategy must be a function with signature (text, options) => string[]',
+      );
+    }
+    this._strategies.set(name, fn);
+  }
+
+  /**
+   * List all registered strategy names.
+   *
+   * @returns {string[]} Array of strategy names.
+   */
+  listStrategies() {
+    return Array.from(this._strategies.keys());
+  }
+
+  /**
+   * Get the current engine configuration.
+   *
+   * @returns {object} A copy of the current configuration.
+   */
+  getConfig() {
+    return { ...this._config };
+  }
+}
+
+module.exports = { ChunkingEngine, DEFAULT_CONFIG };

package/src/chunking/strategies/fixed-size.js

@@ -0,0 +1,54 @@
+'use strict';
+
+/**
+ * Fixed-size chunking strategy.
+ * Splits text by character count with configurable overlap.
+ * Attempts to break at word boundaries by looking back up to 20 characters.
+ *
+ * @param {string} text - The input text to chunk.
+ * @param {object} [options={}] - Chunking options.
+ * @param {number} [options.chunkSize=512] - Target chunk size in characters.
+ * @param {number} [options.chunkOverlap=50] - Number of overlapping characters between chunks.
+ * @param {number} [options.minChunkSize=50] - Minimum chunk size; smaller chunks are filtered out.
+ * @returns {string[]} Array of text chunks.
+ */
+function fixedSizeChunk(text, options = {}) {
+  const { chunkSize = 512, chunkOverlap = 50, minChunkSize = 50 } = options;
+
+  if (!text || typeof text !== 'string') {
+    return [];
+  }
+
+  const step = chunkSize - chunkOverlap;
+  if (step <= 0) {
+    throw new Error('chunkSize must be greater than chunkOverlap');
+  }
+
+  const chunks = [];
+  let pos = 0;
+
+  while (pos < text.length) {
+    let end = Math.min(pos + chunkSize, text.length);
+
+    // Try to break at a word boundary by looking back up to 20 chars
+    if (end < text.length) {
+      const lookbackStart = Math.max(end - 20, pos);
+      const segment = text.slice(lookbackStart, end);
+      const lastSpace = segment.lastIndexOf(' ');
+      if (lastSpace !== -1) {
+        end = lookbackStart + lastSpace;
+      }
+    }
+
+    const chunk = text.slice(pos, end).trim();
+    if (chunk.length >= minChunkSize) {
+      chunks.push(chunk);
+    }
+
+    pos += step;
+  }
+
+  return chunks;
+}
+
+module.exports = { fixedSizeChunk };

package/src/chunking/strategies/recursive.js

@@ -0,0 +1,127 @@
+'use strict';
+
+const DEFAULT_SEPARATORS = ['\n\n', '\n', '. ', ' '];
+
+/**
+ * Recursive text chunking strategy.
+ * Splits text using a hierarchy of separators, merging consecutive pieces
+ * into chunks up to chunkSize. Pieces exceeding chunkSize are recursively
+ * split with the next separator in the hierarchy.
+ *
+ * @param {string} text - The input text to chunk.
+ * @param {object} [options={}] - Chunking options.
+ * @param {number} [options.chunkSize=512] - Maximum chunk size in characters.
+ * @param {number} [options.chunkOverlap=50] - Number of overlapping characters between chunks.
+ * @param {string[]} [options.separators] - Separator hierarchy for splitting.
+ * @param {number} [options.minChunkSize=50] - Minimum chunk size; smaller chunks are filtered out.
+ * @returns {string[]} Array of text chunks.
+ */
+function recursiveChunk(text, options = {}) {
+  const {
+    chunkSize = 512,
+    chunkOverlap = 50,
+    separators = DEFAULT_SEPARATORS,
+    minChunkSize = 50,
+  } = options;
+
+  if (!text || typeof text !== 'string') {
+    return [];
+  }
+
+  const results = _splitRecursive(text, separators, 0, chunkSize, chunkOverlap);
+
+  return results.filter((chunk) => chunk.length >= minChunkSize);
+}
+
+/**
+ * Internal recursive splitting function.
+ *
+ * @param {string} text - Text to split.
+ * @param {string[]} separators - Separator hierarchy.
+ * @param {number} sepIndex - Current separator index.
+ * @param {number} chunkSize - Maximum chunk size.
+ * @param {number} chunkOverlap - Overlap size.
+ * @returns {string[]} Array of text chunks.
+ * @private
+ */
+function _splitRecursive(text, separators, sepIndex, chunkSize, chunkOverlap) {
+  // Base case: no more separators or text fits in one chunk
+  if (text.length <= chunkSize) {
+    return [text.trim()];
+  }
+
+  if (sepIndex >= separators.length) {
+    // No more separators; hard-cut the text
+    return [text.slice(0, chunkSize).trim()];
+  }
+
+  const separator = separators[sepIndex];
+  const pieces = text.split(separator);
+
+  // If splitting produced only one piece, try next separator
+  if (pieces.length <= 1) {
+    return _splitRecursive(
+      text,
+      separators,
+      sepIndex + 1,
+      chunkSize,
+      chunkOverlap,
+    );
+  }
+
+  const chunks = [];
+  let buffer = '';
+
+  for (const piece of pieces) {
+    const candidate = buffer ? buffer + separator + piece : piece;
+
+    if (candidate.length <= chunkSize) {
+      buffer = candidate;
+    } else {
+      // Flush the current buffer
+      if (buffer) {
+        chunks.push(buffer.trim());
+      }
+
+      // If the piece itself exceeds chunkSize, recurse with next separator
+      if (piece.length > chunkSize) {
+        const subChunks = _splitRecursive(
+          piece,
+          separators,
+          sepIndex + 1,
+          chunkSize,
+          chunkOverlap,
+        );
+        chunks.push(...subChunks);
+        buffer = '';
+      } else {
+        buffer = piece;
+      }
+    }
+  }
+
+  if (buffer.trim()) {
+    chunks.push(buffer.trim());
+  }
+
+  // Apply overlap: prefix each chunk (except the first) with trailing text from previous chunk
+  if (chunkOverlap > 0 && chunks.length > 1) {
+    const overlapped = [chunks[0]];
+    for (let i = 1; i < chunks.length; i++) {
+      const prev = chunks[i - 1];
+      const overlapText = prev.slice(-chunkOverlap);
+      const combined = overlapText + chunks[i];
+      // Only add overlap if it doesn't push us way over chunkSize
+      if (combined.length <= chunkSize + chunkOverlap) {
+        overlapped.push(combined.trim());
+      } else {
+        overlapped.push(chunks[i]);
+      }
+    }
+    return overlapped;
+  }
+
+  return chunks;
+}
+
+module.exports = { recursiveChunk };

package/src/chunking/strategies/semantic.js

@@ -0,0 +1,128 @@
+'use strict';
+
+const { recursiveChunk } = require('./recursive');
+
+/**
+ * Tokenize text for Jaccard similarity computation.
+ * Lowercases, splits on whitespace, filters tokens with length > 2.
+ *
+ * @param {string} text - Text to tokenize.
+ * @returns {Set<string>} Set of tokens.
+ * @private
+ */
+function _tokenize(text) {
+  const tokens = text
+    .toLowerCase()
+    .split(/\s+/)
+    .filter((t) => t.length > 2);
+  return new Set(tokens);
+}
+
+/**
+ * Compute Jaccard similarity between two token sets.
+ *
+ * @param {Set<string>} setA - First token set.
+ * @param {Set<string>} setB - Second token set.
+ * @returns {number} Jaccard similarity coefficient (0-1).
+ * @private
+ */
+function _jaccardSimilarity(setA, setB) {
+  if (setA.size === 0 && setB.size === 0) {
+    return 1.0;
+  }
+
+  let intersection = 0;
+  for (const token of setA) {
+    if (setB.has(token)) {
+      intersection++;
+    }
+  }
+
+  const union = setA.size + setB.size - intersection;
+  return union === 0 ? 0 : intersection / union;
+}
+
+/**
+ * Semantic chunking strategy using topic-boundary detection.
+ * Splits text into sentences, computes Jaccard token overlap between
+ * consecutive sentence pairs, and inserts chunk boundaries where
+ * similarity drops below the threshold. Oversized segments fall back
+ * to recursive splitting.
+ *
+ * @param {string} text - The input text to chunk.
+ * @param {object} [options={}] - Chunking options.
+ * @param {number} [options.chunkSize=512] - Maximum chunk size in characters.
+ * @param {number} [options.similarityThreshold=0.3] - Minimum similarity to keep sentences in the same chunk.
+ * @param {number} [options.minChunkSize=50] - Minimum chunk size; smaller chunks are filtered out.
+ * @returns {string[]} Array of text chunks.
+ */
+function semanticChunk(text, options = {}) {
+  const {
+    chunkSize = 512,
+    similarityThreshold = 0.3,
+    minChunkSize = 50,
+  } = options;
+
+  if (!text || typeof text !== 'string') {
+    return [];
+  }
+
+  const sentences = text
+    .split(/(?<=[.!?])\s+/)
+    .filter((s) => s.trim().length > 0);
+
+  if (sentences.length === 0) {
+    return [];
+  }
+
+  if (sentences.length === 1) {
+    const trimmed = sentences[0].trim();
+    if (trimmed.length > chunkSize) {
+      return recursiveChunk(trimmed, { chunkSize, minChunkSize });
+    }
+    return trimmed.length >= minChunkSize ? [trimmed] : [];
+  }
+
+  // Compute token sets for each sentence
+  const tokenSets = sentences.map(_tokenize);
+
+  // Group sentences into segments based on similarity boundaries
+  const segments = [];
+  let currentSegment = [sentences[0]];
+
+  for (let i = 1; i < sentences.length; i++) {
+    const similarity = _jaccardSimilarity(tokenSets[i - 1], tokenSets[i]);
+
+    if (similarity < similarityThreshold) {
+      // Topic boundary detected; flush current segment
+      segments.push(currentSegment.join(' '));
+      currentSegment = [sentences[i]];
+    } else {
+      currentSegment.push(sentences[i]);
+    }
+  }
+
+  // Flush the last segment
+  if (currentSegment.length > 0) {
+    segments.push(currentSegment.join(' '));
+  }
+
+  // Apply chunkSize limit: oversized segments fall back to recursive splitting
+  const chunks = [];
+  for (const segment of segments) {
+    const trimmed = segment.trim();
+    if (trimmed.length === 0) {
+      continue;
+    }
+    if (trimmed.length > chunkSize) {
+      const subChunks = recursiveChunk(trimmed, { chunkSize, minChunkSize });
+      chunks.push(...subChunks);
+    } else if (trimmed.length >= minChunkSize) {
+      chunks.push(trimmed);
+    }
+  }
+
+  return chunks;
+}
+
+module.exports = { semanticChunk };