@devilsdev/rag-pipeline-utils 2.3.1 → 2.4.1

package/src/index.js

@@ -7,7 +7,7 @@
 const { createRagPipeline } = require('./core/create-pipeline');
 
 // Configuration utilities
-const { loadConfig } = require('./config/load-config');
+const { loadRagConfig } = require('./config/load-config');
 const { validateRagrc } = require('./config/enhanced-ragrc-schema');
 const { normalizeConfig } = require('./config/normalize-config');
 
@@ -17,13 +17,125 @@ const pluginRegistry = require('./core/plugin-registry');
 // Utilities
 const logger = require('./utils/logger');
 const errorFormatter = require('./utils/error-formatter');
+const { BatchProcessor } = require('./utils/batch-processor');
+const RateLimiter = require('./utils/rate-limiter');
+const { retry, withRetry } = require('./utils/retry');
+const { ConnectionPoolManager } = require('./utils/connection-pool');
+
+// Security
+const { JWTValidator } = require('./security/jwt-validator');
+const { PluginSandbox } = require('./security/plugin-sandbox');
+const {
+  PluginSignatureVerifier,
+} = require('./security/plugin-signature-verifier');
+const { validateInput } = require('./security/validator');
 
 // AI/ML capabilities
-const MultiModalProcessor = require('./ai/multimodal');
-const AdaptiveRetrievalEngine = require('./ai/retrieval-engine');
+const { MultiModalProcessor } = require('./ai/multimodal');
+const { AdaptiveRetrievalEngine } = require('./ai/retrieval-engine');
+const {
+  FederatedLearningCoordinator,
+} = require('./ai/federation/federated-learning-coordinator');
+const {
+  ModelTrainingOrchestrator,
+} = require('./ai/training/model-training-orchestrator');
 
 // DAG engine for complex workflows
-const { DAGEngine } = require('./dag/dag-engine');
+const { DAG } = require('./dag/dag-engine');
+
+// Evaluation
+const { evaluateRagDataset } = require('./evaluate/evaluator');
+const {
+  scoreAnswer,
+  computeBLEU,
+  computeROUGE,
+} = require('./evaluate/scoring');
+const { computeFaithfulness } = require('./evaluate/faithfulness');
+const { computeAnswerRelevance } = require('./evaluate/relevance');
+const {
+  computeContextPrecision,
+  computeContextRecall,
+} = require('./evaluate/context-metrics');
+const { computeGroundedness } = require('./evaluate/groundedness');
+const { PipelineEvaluator } = require('./evaluate/pipeline-evaluator');
+
+// Chunking
+const { ChunkingEngine } = require('./chunking/chunking-engine');
+
+// Citation & Grounding
+const { CitationTracker } = require('./citation/citation-tracker');
+const {
+  mapSentenceToSources,
+  buildIDFWeights,
+} = require('./citation/source-mapper');
+const { detectHallucinations } = require('./citation/hallucination-detector');
+
+// Hybrid Retrieval
+const { BM25Search } = require('./retrieval/bm25-search');
+const { reciprocalRankFusion } = require('./retrieval/rank-fusion');
+const { HybridRetriever } = require('./retrieval/hybrid-retriever');
+
+// Guardrails
+const { PreRetrievalGuard } = require('./guardrails/pre-retrieval-guard');
+const { RetrievalGuard } = require('./guardrails/retrieval-guard');
+const { PostGenerationGuard } = require('./guardrails/post-generation-guard');
+const { GuardrailsPipeline } = require('./guardrails/guardrails-pipeline');
+
+// Agentic RAG
+const { QueryPlanner } = require('./agentic/query-planner');
+const { IterativeRetriever } = require('./agentic/iterative-retriever');
+const { SelfCritiqueChecker } = require('./agentic/self-critique');
+const { AgenticPipeline } = require('./agentic/agentic-pipeline');
+
+// Cost Management
+const { CostCalculator } = require('./cost/cost-calculator');
+const { CostTracker } = require('./cost/cost-tracker');
+const { TokenBudget } = require('./cost/token-budget');
+
+// Pipeline Debugger
+const { ExecutionTracer } = require('./debug/execution-tracer');
+const { TraceInspector } = require('./debug/trace-inspector');
+
+// MCP Integration
+const { MCPServer } = require('./mcp/mcp-server');
+const { MCPToolBuilder } = require('./mcp/mcp-tool-builder');
+
+// Quick Start Templates
+const { ProjectScaffolder } = require('./templates/scaffold');
+const { TemplateRegistry } = require('./templates/template-registry');
+
+// Streaming Adapters
+const { SSEAdapter } = require('./streaming/sse-adapter');
+const { WebSocketAdapter } = require('./streaming/websocket-adapter');
+const { StreamRouter } = require('./streaming/stream-router');
+
+// Connectors
+const { BaseConnector } = require('./connectors/base-connector');
+const { OllamaConnector } = require('./connectors/ollama-connector');
+const { LocalEmbedder } = require('./connectors/local-embedder');
+const { MemoryRetriever } = require('./connectors/memory-retriever');
+const { OpenAIConnector } = require('./connectors/openai-connector');
+const { AnthropicConnector } = require('./connectors/anthropic-connector');
+const { CohereConnector } = require('./connectors/cohere-connector');
+
+// Streaming Embeddings
+const { StreamingEmbedder } = require('./streaming/streaming-embedder');
+const { createEmbeddingStream } = require('./streaming/embedding-stream');
+
+// Advanced Reranking
+const { ScoringReranker } = require('./reranker/scoring-reranker');
+const { EmbeddingReranker } = require('./reranker/embedding-reranker');
+const { CascadeReranker } = require('./reranker/cascade-reranker');
+
+// GraphRAG
+const { KnowledgeGraph } = require('./graph/knowledge-graph');
+const { EntityExtractor } = require('./graph/entity-extractor');
+const { GraphRetriever } = require('./graph/graph-retriever');
+const { GraphIndex } = require('./graph/graph-index');
+
+// Performance Dashboard
+const { DashboardGenerator } = require('./dashboard/dashboard-generator');
+const { MetricsAggregator } = require('./dashboard/metrics-aggregator');
 
 // Performance and observability
 const ParallelProcessor = require('./core/performance/parallel-processor');
@@ -32,7 +144,25 @@ const metrics = require('./core/observability/metrics');
 
 // Enterprise features
 const { AuditLogger } = require('./enterprise/audit-logging');
-const { DataGovernance } = require('./enterprise/data-governance');
+const { DataGovernanceManager } = require('./enterprise/data-governance');
+const {
+  TenantManager,
+  ResourceMonitor,
+} = require('./enterprise/multi-tenancy');
+const {
+  SSOManager,
+  SAMLProvider,
+  OAuth2Provider,
+  ActiveDirectoryProvider,
+  OIDCProvider,
+} = require('./enterprise/sso-integration');
+
+// Ecosystem
+const { PluginHub, PluginAnalytics } = require('./ecosystem/plugin-hub');
+const { PluginCertification } = require('./ecosystem/plugin-certification');
+const {
+  PluginAnalyticsDashboard,
+} = require('./ecosystem/plugin-analytics-dashboard');
 
 // Development tools
 const {
@@ -47,7 +177,8 @@ module.exports = {
   createPipeline: createRagPipeline, // Alias for backward compatibility
 
   // Configuration
-  loadConfig,
+  loadConfig: loadRagConfig,
+  loadRagConfig,
   validateRagrc,
   normalizeConfig,
 
@@ -57,17 +188,118 @@ module.exports = {
   // Utilities
   logger,
   errorFormatter,
-  // Error handling utilities
   createError: errorFormatter.createError,
   wrapError: errorFormatter.wrapError,
   ERROR_CODES: errorFormatter.ERROR_CODES,
+  BatchProcessor,
+  RateLimiter,
+  retry,
+  withRetry,
+  ConnectionPoolManager,
+
+  // Security
+  JWTValidator,
+  PluginSandbox,
+  PluginSignatureVerifier,
+  validateInput,
 
   // AI/ML
   MultiModalProcessor,
   AdaptiveRetrievalEngine,
+  FederatedLearningCoordinator,
+  ModelTrainingOrchestrator,
 
   // Workflow engine
-  DAGEngine,
+  DAGEngine: DAG,
+  DAG,
+
+  // Evaluation
+  evaluateRagDataset,
+  scoreAnswer,
+  computeBLEU,
+  computeROUGE,
+  computeFaithfulness,
+  computeAnswerRelevance,
+  computeContextPrecision,
+  computeContextRecall,
+  computeGroundedness,
+  PipelineEvaluator,
+
+  // Chunking
+  ChunkingEngine,
+
+  // Citation & Grounding
+  CitationTracker,
+  mapSentenceToSources,
+  buildIDFWeights,
+  detectHallucinations,
+
+  // Hybrid Retrieval
+  BM25Search,
+  reciprocalRankFusion,
+  HybridRetriever,
+
+  // Guardrails
+  PreRetrievalGuard,
+  RetrievalGuard,
+  PostGenerationGuard,
+  GuardrailsPipeline,
+
+  // Agentic RAG
+  QueryPlanner,
+  IterativeRetriever,
+  SelfCritiqueChecker,
+  AgenticPipeline,
+
+  // Cost Management
+  CostCalculator,
+  CostTracker,
+  TokenBudget,
+
+  // Pipeline Debugger
+  ExecutionTracer,
+  TraceInspector,
+
+  // MCP Integration
+  MCPServer,
+  MCPToolBuilder,
+
+  // Quick Start Templates
+  ProjectScaffolder,
+  TemplateRegistry,
+
+  // Streaming Adapters
+  SSEAdapter,
+  WebSocketAdapter,
+  StreamRouter,
+
+  // Connectors
+  BaseConnector,
+  OllamaConnector,
+  LocalEmbedder,
+  MemoryRetriever,
+  OpenAIConnector,
+  AnthropicConnector,
+  CohereConnector,
+
+  // Streaming Embeddings
+  StreamingEmbedder,
+  createEmbeddingStream,
+
+  // Advanced Reranking
+  ScoringReranker,
+  EmbeddingReranker,
+  CascadeReranker,
+
+  // GraphRAG
+  KnowledgeGraph,
+  EntityExtractor,
+  GraphRetriever,
+  GraphIndex,
+
+  // Performance Dashboard
+  DashboardGenerator,
+  MetricsAggregator,
 
   // Performance & Observability
   ParallelProcessor,
@@ -76,7 +308,21 @@ module.exports = {
 
   // Enterprise
   AuditLogger,
-  DataGovernance,
+  DataGovernance: DataGovernanceManager,
+  DataGovernanceManager,
+  TenantManager,
+  ResourceMonitor,
+  SSOManager,
+  SAMLProvider,
+  OAuth2Provider,
+  ActiveDirectoryProvider,
+  OIDCProvider,
+
+  // Ecosystem
+  PluginHub,
+  PluginAnalytics,
+  PluginCertification,
+  PluginAnalyticsDashboard,
 
   // Development tools
   HotReloadManager,

package/src/loader/html-loader.js

@@ -34,26 +34,15 @@ class HTMLLoader {
   }
 
   /**
-   * Chunk HTML text content by sentence
+   * Chunk HTML text content using the ChunkingEngine
    * @param {string} input
    * @param {number} maxLen
    * @returns {string[]}
    */
   _chunkText(input, maxLen = 500) {
-    const sentences = input.split(/(?<=[.!?])\s+/);
-    const chunks = [];
-    let buffer = '';
-
-    for (const sentence of sentences) {
-      if ((buffer + sentence).length <= maxLen) {
-        buffer += sentence + ' ';
-      } else {
-        chunks.push(buffer.trim());
-        buffer = sentence + ' ';
-      }
-    }
-    if (buffer.trim()) chunks.push(buffer.trim());
-    return chunks;
+    const { ChunkingEngine } = require('../chunking/chunking-engine');
+    const engine = new ChunkingEngine({ strategy: 'sentence', maxLen });
+    return engine.chunk(input);
   }
 }
 

package/src/loader/markdown-loader.js

@@ -37,26 +37,15 @@ class MarkdownLoader {
   }
 
   /**
-   * Simple sentence-based chunking strategy
+   * Chunk text using the ChunkingEngine
    * @param {string} input
    * @param {number} maxLen
    * @returns {string[]}
    */
   _chunkText(input, maxLen = 500) {
-    const sentences = input.split(/(?<=[.!?])\s+/);
-    const chunks = [];
-    let buffer = '';
-
-    for (const sentence of sentences) {
-      if ((buffer + sentence).length <= maxLen) {
-        buffer += sentence + ' ';
-      } else {
-        chunks.push(buffer.trim());
-        buffer = sentence + ' ';
-      }
-    }
-    if (buffer.trim()) chunks.push(buffer.trim());
-    return chunks;
+    const { ChunkingEngine } = require('../chunking/chunking-engine');
+    const engine = new ChunkingEngine({ strategy: 'sentence', maxLen });
+    return engine.chunk(input);
   }
 }
 

package/src/mcp/mcp-server.js

@@ -0,0 +1,199 @@
+'use strict';
+
+/**
+ * @module mcp/mcp-server
+ * @description Wraps a RAG pipeline as an MCP-compatible tool handler.
+ */
+
+const { EventEmitter } = require('events');
+const { MCPToolBuilder } = require('./mcp-tool-builder');
+
+/** @type {Object} Default configuration for the MCP server */
+const DEFAULT_CONFIG = {
+  includeMetadata: true,
+  maxResponseLength: 50000,
+};
+
+/**
+ * MCP server that registers RAG pipelines as callable MCP tools.
+ *
+ * @extends EventEmitter
+ * @fires MCPServer#tool_use
+ *
+ * @example
+ * const server = MCPServer.fromPipeline(myPipeline, { name: 'my_tool' });
+ * const tools = server.getToolDefinitions();
+ * const result = await server.handleToolUse('my_tool', { query: 'What is X?' });
+ */
+class MCPServer extends EventEmitter {
+  /**
+   * @param {Object} [options={}] - Server configuration
+   * @param {boolean} [options.includeMetadata] - Include metadata in responses
+   * @param {number} [options.maxResponseLength] - Maximum response text length
+   * @param {Object} [options.toolOptions] - Options forwarded to MCPToolBuilder
+   */
+  constructor(options = {}) {
+    super();
+    this.config = { ...DEFAULT_CONFIG, ...options };
+    this.toolBuilder = new MCPToolBuilder(options.toolOptions);
+    /** @type {Map<string, {pipeline: Object, config: Object, toolDefinition: Object}>} */
+    this.pipelines = new Map();
+  }
+
+  /**
+   * Register a pipeline that can be called as an MCP tool.
+   *
+   * @param {string} name - Tool name used to invoke this pipeline
+   * @param {Object} pipeline - Pipeline instance with a `run` method
+   * @param {Object} [config={}] - Pipeline-specific configuration
+   * @returns {void}
+   */
+  registerPipeline(name, pipeline, config = {}) {
+    if (!name || typeof name !== 'string') {
+      throw new TypeError('Pipeline name must be a non-empty string');
+    }
+    if (!pipeline || typeof pipeline.run !== 'function') {
+      throw new TypeError('Pipeline must have a run() method');
+    }
+
+    const builder = new MCPToolBuilder({
+      toolName: name,
+      toolDescription:
+        config.description || this.toolBuilder.config.toolDescription,
+    });
+
+    const toolDefinition = builder.buildToolDefinition(config);
+
+    this.pipelines.set(name, { pipeline, config, toolDefinition });
+  }
+
+  /**
+   * Return array of all registered tool definitions.
+   * Each is a valid MCP tool schema.
+   *
+   * @returns {Array<Object>} Array of MCP tool definition objects
+   */
+  getToolDefinitions() {
+    return Array.from(this.pipelines.values()).map(
+      (entry) => entry.toolDefinition,
+    );
+  }
+
+  /**
+   * Handle an MCP tool_use request by running the matching pipeline.
+   *
+   * @param {string} toolName - Name of the tool to invoke
+   * @param {Object} toolInput - Tool input parameters
+   * @param {string} toolInput.query - The search query (required)
+   * @param {number} [toolInput.topK] - Number of results to retrieve
+   * @param {boolean} [toolInput.stream] - Enable streaming response
+   * @param {boolean} [toolInput.citations] - Include source citations
+   * @param {boolean} [toolInput.evaluate] - Include quality evaluation metrics
+   * @returns {Promise<Object>} MCP-formatted response with type and text fields
+   * @throws {Error} If tool name is not registered or query is missing
+   */
+  async handleToolUse(toolName, toolInput) {
+    const entry = this.pipelines.get(toolName);
+    if (!entry) {
+      throw new Error(`Unknown tool: ${toolName}`);
+    }
+
+    if (
+      !toolInput ||
+      typeof toolInput.query !== 'string' ||
+      !toolInput.query.trim()
+    ) {
+      throw new Error(
+        'toolInput.query is required and must be a non-empty string',
+      );
+    }
+
+    const { pipeline } = entry;
+    const options = {};
+
+    if (typeof toolInput.topK === 'number') {
+      options.topK = toolInput.topK;
+    }
+    if (typeof toolInput.stream === 'boolean') {
+      options.stream = toolInput.stream;
+    }
+    if (typeof toolInput.citations === 'boolean') {
+      options.citations = toolInput.citations;
+    }
+    if (typeof toolInput.evaluate === 'boolean') {
+      options.evaluate = toolInput.evaluate;
+    }
+
+    const result = await pipeline.run({
+      query: toolInput.query,
+      options,
+    });
+
+    const hasExtras = result.citations || result.evaluation;
+    let responseText;
+
+    if (hasExtras && this.config.includeMetadata) {
+      const payload = { answer: result.answer || result.results };
+      if (result.citations) {
+        payload.citations = result.citations;
+      }
+      if (result.evaluation) {
+        payload.evaluation = result.evaluation;
+      }
+      responseText = JSON.stringify(payload);
+    } else {
+      responseText =
+        typeof result.answer === 'string'
+          ? result.answer
+          : JSON.stringify(result);
+    }
+
+    if (
+      this.config.maxResponseLength &&
+      responseText.length > this.config.maxResponseLength
+    ) {
+      responseText = responseText.slice(0, this.config.maxResponseLength);
+    }
+
+    const response = {
+      type: 'text',
+      text: responseText,
+    };
+
+    /**
+     * @event MCPServer#tool_use
+     * @type {Object}
+     * @property {string} toolName - Name of the tool that was invoked
+     * @property {Object} toolInput - Original input parameters
+     * @property {number} responseLength - Length of the response text
+     */
+    this.emit('tool_use', {
+      toolName,
+      toolInput,
+      responseLength: responseText.length,
+    });
+
+    return response;
+  }
+
+  /**
+   * Convenience factory: create an MCPServer with a single pipeline registered.
+   *
+   * @param {Object} pipeline - Pipeline instance with a `run` method
+   * @param {Object} [options={}] - Server and registration options
+   * @param {string} [options.name='rag_query'] - Tool name for the pipeline
+   * @param {Object} [options.config] - Pipeline-specific configuration
+   * @returns {MCPServer} Configured server instance
+   */
+  static fromPipeline(pipeline, options = {}) {
+    const server = new MCPServer(options);
+    server.registerPipeline(
+      options.name || 'rag_query',
+      pipeline,
+      options.config || {},
+    );
+    return server;
+  }
+}
+
+module.exports = { MCPServer, DEFAULT_CONFIG };

package/src/mcp/mcp-tool-builder.js

@@ -0,0 +1,121 @@
+'use strict';
+
+/**
+ * @module mcp/mcp-tool-builder
+ * @description Builds MCP tool definitions from pipeline configuration.
+ */
+
+/** @type {Object} Default configuration for the MCP tool builder */
+const DEFAULT_CONFIG = {
+  toolName: 'rag_pipeline_query',
+  toolDescription: 'Query a RAG pipeline with natural language',
+};
+
+/**
+ * Builds MCP-compatible tool definitions from RAG pipeline configuration.
+ *
+ * @example
+ * const builder = new MCPToolBuilder({ toolName: 'my_rag_tool' });
+ * const definition = builder.buildToolDefinition({ capabilities: { stream: true } });
+ */
+class MCPToolBuilder {
+  /**
+   * @param {Object} [options={}] - Builder configuration
+   * @param {string} [options.toolName] - Name for the generated MCP tool
+   * @param {string} [options.toolDescription] - Description for the generated MCP tool
+   */
+  constructor(options = {}) {
+    this.config = { ...DEFAULT_CONFIG, ...options };
+  }
+
+  /**
+   * Generate an MCP-compatible tool definition from pipeline configuration.
+   *
+   * @param {Object} [pipelineConfig={}] - Pipeline-specific configuration
+   * @param {Object} [pipelineConfig.capabilities] - Pipeline capabilities to reflect in schema
+   * @param {boolean} [pipelineConfig.capabilities.stream] - Whether streaming is supported
+   * @param {boolean} [pipelineConfig.capabilities.citations] - Whether citations are supported
+   * @param {boolean} [pipelineConfig.capabilities.evaluate] - Whether evaluation is supported
+   * @param {number} [pipelineConfig.capabilities.maxTopK] - Maximum allowed topK value
+   * @returns {Object} MCP tool definition with name, description, and input_schema
+   */
+  buildToolDefinition(pipelineConfig = {}) {
+    const capabilities = pipelineConfig.capabilities || {};
+
+    const properties = {
+      query: {
+        type: 'string',
+        description: 'The search query or question',
+      },
+      topK: {
+        type: 'number',
+        description: 'Number of results to retrieve',
+        default: 5,
+      },
+    };
+
+    if (typeof capabilities.maxTopK === 'number') {
+      properties.topK.maximum = capabilities.maxTopK;
+    }
+
+    if (capabilities.stream !== false) {
+      properties.stream = {
+        type: 'boolean',
+        description: 'Enable streaming response',
+        default: false,
+      };
+    }
+
+    if (capabilities.citations !== false) {
+      properties.citations = {
+        type: 'boolean',
+        description: 'Include source citations',
+        default: false,
+      };
+    }
+
+    if (capabilities.evaluate !== false) {
+      properties.evaluate = {
+        type: 'boolean',
+        description: 'Include quality evaluation metrics',
+        default: false,
+      };
+    }
+
+    return {
+      name: this.config.toolName,
+      description: this.config.toolDescription,
+      input_schema: {
+        type: 'object',
+        properties,
+        required: ['query'],
+      },
+    };
+  }
+
+  /**
+   * Build tool definitions for multiple named pipelines.
+   *
+   * @param {Array<Object>} pipelines - Array of pipeline descriptors
+   * @param {string} pipelines[].name - Tool name for this pipeline
+   * @param {string} [pipelines[].description] - Tool description
+   * @param {Object} [pipelines[].pipeline] - Pipeline instance (unused during definition building)
+   * @param {Object} [pipelines[].config] - Pipeline configuration passed to buildToolDefinition
+   * @returns {Array<Object>} Array of MCP tool definitions
+   */
+  buildMultipleTools(pipelines) {
+    if (!Array.isArray(pipelines)) {
+      throw new TypeError('pipelines must be an array');
+    }
+
+    return pipelines.map((entry) => {
+      const builder = new MCPToolBuilder({
+        toolName: entry.name,
+        toolDescription: entry.description || this.config.toolDescription,
+      });
+      return builder.buildToolDefinition(entry.config || {});
+    });
+  }
+}
+
+module.exports = { MCPToolBuilder, DEFAULT_CONFIG };

package/src/observability/audit-logger.js

@@ -754,5 +754,3 @@ module.exports = {
   initializeAuditLogging,
   logAuditEvent,
 };
-
-module.exports.default = module.exports;