@devilsdev/rag-pipeline-utils 2.3.0 → 2.4.0

package/src/debug/trace-inspector.js

@@ -0,0 +1,281 @@
+'use strict';
+
+/**
+ * Trace Inspector
+ *
+ * Analyzes execution traces for bottlenecks and optimization opportunities.
+ * Provides summary reports, comparisons, and anomaly detection across traces.
+ */
+
+const { EventEmitter } = require('events');
+
+/**
+ * Trace inspector that analyzes execution traces produced by ExecutionTracer
+ * @extends EventEmitter
+ */
+class TraceInspector extends EventEmitter {
+  /**
+   * Create a new TraceInspector
+   * @param {object} [options] - Inspector configuration
+   */
+  constructor(options = {}) {
+    super();
+    this.config = options;
+  }
+
+  /**
+   * Analyze a single trace for performance and quality insights
+   * @param {object} trace - Trace record from ExecutionTracer
+   * @returns {object} Analysis result with summary, breakdown, bottleneck, quality, and recommendations
+   */
+  analyze(trace) {
+    const totalDuration = trace.totalDurationMs || 0;
+    const steps = trace.steps || [];
+
+    // Time breakdown per step
+    const timeBreakdown = {};
+    for (const step of steps) {
+      timeBreakdown[step.step] = step.durationMs || 0;
+    }
+
+    const stepDurationSum = steps.reduce(
+      (sum, s) => sum + (s.durationMs || 0),
+      0,
+    );
+    if (steps.length > 0) {
+      timeBreakdown['overhead'] = Math.max(0, totalDuration - stepDurationSum);
+    }
+
+    // Identify bottleneck
+    let bottleneck = null;
+    if (steps.length > 0) {
+      const slowest = steps.reduce((a, b) =>
+        (a.durationMs || 0) >= (b.durationMs || 0) ? a : b,
+      );
+      bottleneck = {
+        step: slowest.step,
+        durationMs: slowest.durationMs || 0,
+      };
+    }
+
+    // Result count
+    const resultCount = (trace.metadata && trace.metadata.resultCount) || 0;
+
+    // Quality metrics
+    const qualityMetrics = {};
+    const metadata = trace.metadata || {};
+
+    if (metadata.citations) {
+      const citations = metadata.citations;
+      if (citations.groundedness !== undefined) {
+        qualityMetrics.groundedness = citations.groundedness;
+      }
+      if (
+        Array.isArray(citations) &&
+        citations.length > 0 &&
+        citations[0].groundedness !== undefined
+      ) {
+        qualityMetrics.groundedness = citations[0].groundedness;
+      }
+    }
+
+    if (metadata.evaluation) {
+      const evaluation = metadata.evaluation;
+      if (evaluation.groundedness !== undefined) {
+        qualityMetrics.groundedness = evaluation.groundedness;
+      }
+      if (evaluation.faithfulness !== undefined) {
+        qualityMetrics.faithfulness = evaluation.faithfulness;
+      }
+      if (evaluation.relevance !== undefined) {
+        qualityMetrics.relevance = evaluation.relevance;
+      }
+    }
+
+    // Recommendations
+    const recommendations = [];
+
+    if (totalDuration > 5000) {
+      recommendations.push('Consider caching frequent queries');
+    }
+    if (resultCount === 0) {
+      recommendations.push(
+        'No results retrieved - check retriever configuration',
+      );
+    }
+    if (
+      qualityMetrics.groundedness !== undefined &&
+      qualityMetrics.groundedness < 0.5
+    ) {
+      recommendations.push(
+        'Low groundedness - consider increasing retrieval topK',
+      );
+    }
+    if (!metadata.hasCitations) {
+      recommendations.push('Enable citations for source tracking');
+    }
+    if (!metadata.hasEvaluation) {
+      recommendations.push('Enable evaluation for quality metrics');
+    }
+
+    const analysis = {
+      summary: {
+        totalDuration,
+        stepCount: steps.length,
+        resultCount,
+      },
+      timeBreakdown,
+      bottleneck,
+      qualityMetrics,
+      recommendations,
+    };
+
+    this.emit('analyzed', analysis);
+    return analysis;
+  }
+
+  /**
+   * Compare multiple traces for consistency and trends
+   * @param {Array<object>} traces - Array of trace records
+   * @returns {object} Comparison result with averages, ranges, and quality trend
+   */
+  compare(traces) {
+    if (!traces || traces.length === 0) {
+      return {
+        avgDuration: 0,
+        minDuration: 0,
+        maxDuration: 0,
+        avgResultCount: 0,
+        qualityTrend: {},
+      };
+    }
+
+    const durations = traces.map((t) => t.totalDurationMs || 0);
+    const resultCounts = traces.map(
+      (t) => (t.metadata && t.metadata.resultCount) || 0,
+    );
+
+    const sum = (arr) => arr.reduce((a, b) => a + b, 0);
+
+    const avgDuration = sum(durations) / durations.length;
+    const minDuration = Math.min(...durations);
+    const maxDuration = Math.max(...durations);
+    const avgResultCount = sum(resultCounts) / resultCounts.length;
+
+    // Quality trend: collect quality metrics across traces
+    const qualityTrend = {};
+    const metricAccumulators = {};
+
+    for (const trace of traces) {
+      const metadata = trace.metadata || {};
+      const evaluation = metadata.evaluation || {};
+      for (const key of ['groundedness', 'faithfulness', 'relevance']) {
+        if (evaluation[key] !== undefined) {
+          if (!metricAccumulators[key]) {
+            metricAccumulators[key] = [];
+          }
+          metricAccumulators[key].push(evaluation[key]);
+        }
+      }
+    }
+
+    for (const [key, values] of Object.entries(metricAccumulators)) {
+      qualityTrend[key] = sum(values) / values.length;
+    }
+
+    const comparison = {
+      avgDuration,
+      minDuration,
+      maxDuration,
+      avgResultCount,
+      qualityTrend,
+    };
+
+    this.emit('compared', comparison);
+    return comparison;
+  }
+
+  /**
+   * Find anomalous traces that deviate significantly from the norm
+   * @param {Array<object>} traces - Array of trace records
+   * @returns {Array<object>} Array of anomaly descriptors with traceId, anomalyType, details
+   */
+  findAnomalies(traces) {
+    if (!traces || traces.length === 0) {
+      return [];
+    }
+
+    const anomalies = [];
+
+    const durations = traces.map((t) => t.totalDurationMs || 0);
+    const avgDuration = durations.reduce((a, b) => a + b, 0) / durations.length;
+
+    const resultCounts = traces.map(
+      (t) => (t.metadata && t.metadata.resultCount) || 0,
+    );
+    const hasResults = resultCounts.some((c) => c > 0);
+
+    // Collect quality scores for average calculation
+    const qualityScores = {};
+    for (const trace of traces) {
+      const evaluation = (trace.metadata && trace.metadata.evaluation) || {};
+      for (const key of ['groundedness', 'faithfulness', 'relevance']) {
+        if (evaluation[key] !== undefined) {
+          if (!qualityScores[key]) {
+            qualityScores[key] = [];
+          }
+          qualityScores[key].push({
+            traceId: trace.id,
+            score: evaluation[key],
+          });
+        }
+      }
+    }
+
+    const qualityAverages = {};
+    for (const [key, entries] of Object.entries(qualityScores)) {
+      qualityAverages[key] =
+        entries.reduce((sum, e) => sum + e.score, 0) / entries.length;
+    }
+
+    for (const trace of traces) {
+      const duration = trace.totalDurationMs || 0;
+      const resultCount = (trace.metadata && trace.metadata.resultCount) || 0;
+
+      // Duration > 2x average
+      if (avgDuration > 0 && duration > 2 * avgDuration) {
+        anomalies.push({
+          traceId: trace.id,
+          anomalyType: 'slow_execution',
+          details: `Duration ${duration}ms is more than 2x the average ${Math.round(avgDuration)}ms`,
+        });
+      }
+
+      // Zero results when others have results
+      if (hasResults && resultCount === 0) {
+        anomalies.push({
+          traceId: trace.id,
+          anomalyType: 'no_results',
+          details: 'No results retrieved while other traces returned results',
+        });
+      }
+
+      // Quality score significantly lower than average (> 0.3 below)
+      const evaluation = (trace.metadata && trace.metadata.evaluation) || {};
+      for (const [key, avg] of Object.entries(qualityAverages)) {
+        if (evaluation[key] !== undefined && evaluation[key] < avg - 0.3) {
+          anomalies.push({
+            traceId: trace.id,
+            anomalyType: 'low_quality',
+            details: `${key} score ${evaluation[key].toFixed(2)} is significantly below average ${avg.toFixed(2)}`,
+          });
+        }
+      }
+    }
+
+    this.emit('anomaliesFound', anomalies);
+    return anomalies;
+  }
+}
+
+module.exports = { TraceInspector };

package/src/dev/dev-server.js

@@ -19,7 +19,7 @@
 
 const { EventEmitter } = require('events');
 const { createHotReloadManager } = require('./hot-reload');
-const logger = require('../utils/logger');
+const { logger } = require('../utils/logger');
 
 /**
  * Development Server
@@ -290,17 +290,17 @@ ${this.options.watchPaths.map((p) => `║   - ${p.padEnd(56)}║`).join('\n')}
 
     switch (level) {
       case 'success':
-        console.log(`${prefix} ✓ ${message}`);
+        logger.info(`${prefix} ${message}`);
         break;
       case 'error':
-        console.error(`${prefix} ✗ ${message}`);
+        logger.error(`${prefix} ${message}`);
         break;
       case 'warn':
-        console.warn(`${prefix} ⚠ ${message}`);
+        logger.warn(`${prefix} ${message}`);
         break;
       case 'info':
       default:
-        console.log(`${prefix} ${message}`);
+        logger.info(`${prefix} ${message}`);
         break;
     }
   }

package/src/dev/hot-reload.js

@@ -26,6 +26,7 @@ const {
   wrapError,
   ERROR_CODES,
 } = require('../utils/error-formatter');
+const { logger } = require('../utils/logger');
 
 /**
  * Hot Reload Manager
@@ -701,9 +702,9 @@ class HotReloadManager extends EventEmitter {
     if (this.options.verbose || isError) {
       const prefix = '[HotReload]';
       if (isError) {
-        console.error(`${prefix} ${message}`);
+        logger.error(`${prefix} ${message}`);
       } else {
-        console.log(`${prefix} ${message}`);
+        logger.info(`${prefix} ${message}`);
       }
     }
   }

package/src/dx/integration-templates.js

@@ -660,6 +660,3 @@ module.exports._config = _config;
 module.exports.generateIntegration = generateIntegration;
 module.exports.getTemplateStats = getTemplateStats;
 module.exports.addTemplate = addTemplate;
-
-// CJS+ESM interop pattern
-module.exports.default = module.exports;

package/src/ecosystem/plugin-analytics-dashboard.js

@@ -9,6 +9,7 @@ const path = require('path');
 // eslint-disable-line global-require
 const { EventEmitter } = require('events');
 // eslint-disable-line global-require
+const { logger } = require('../utils/logger');
 
 class PluginAnalyticsDashboard extends EventEmitter {
   constructor(_options = {}) {
@@ -749,8 +750,7 @@ class PluginAnalyticsDashboard extends EventEmitter {
       try {
         callback(data);
       } catch (error) {
-        console.error('Error notifying subscriber:', error);
-        // eslint-disable-line no-console
+        logger.error('Error notifying subscriber:', error);
       }
     });
   }

package/src/ecosystem/plugin-hub.js

@@ -7,6 +7,7 @@ const fs = require('fs').promises; // eslint-disable-line global-require
 const path = require('path'); // eslint-disable-line global-require
 const crypto = require('crypto'); // eslint-disable-line global-require
 const { EventEmitter } = require('events'); // eslint-disable-line global-require
+const { logger } = require('../utils/logger');
 
 function pickFetch(pref) {
   // Prefer a provided fetch if it is a function; otherwise fall back to globalThis.fetch if available.
@@ -709,8 +710,7 @@ class PluginAnalytics {
     } catch (error) {
       // Silence warnings during tests to avoid "Cannot log after tests are done"
       if (process.env.NODE_ENV !== 'test') {
-        // eslint-disable-next-line no-console
-        console.warn('Analytics flush failed:', error.message);
+        logger.warn('Analytics flush failed:', error.message);
       }
     }
   }

package/src/enterprise/audit-logging.js

@@ -11,6 +11,7 @@ const crypto = require('crypto');
 // eslint-disable-line global-require
 const { EventEmitter } = require('events');
 // eslint-disable-line global-require
+const { logger } = require('../utils/logger');
 
 class AuditLogger extends EventEmitter {
   constructor(_options = {}) {
@@ -916,8 +917,7 @@ class AuditLogger extends EventEmitter {
 
     try {
       // Mock webhook sending - would use actual HTTP client
-      console.log(`Sending ${logs.length} audit logs to webhook`);
-      // eslint-disable-line no-console
+      logger.info(`Sending ${logs.length} audit logs to webhook`);
     } catch (error) {
       this.emit('webhook_failed', { error: error.message, count: logs.length });
     }

package/src/enterprise/sso-integration.js

@@ -15,7 +15,26 @@ class SSOManager extends EventEmitter {
   constructor(_options = {}) {
     super();
 
-    this._config = {
+    this._config = this._buildConfig(_options);
+
+    this.sessions = new Map();
+    this.providers = new Map();
+    this.userCache = new Map();
+
+    this._initializeRateLimiter();
+    this._initializeJWTValidator(_options);
+    this._wireJWTEvents();
+    this._initializeProviders();
+  }
+
+  /**
+   * Build the full SSO configuration from user-supplied options.
+   * @param {Object} _options - User-supplied configuration options
+   * @returns {Object} Complete SSO configuration
+   * @private
+   */
+  _buildConfig(_options) {
+    return {
       providers: {
         saml: {
           enabled: false,
@@ -69,12 +88,13 @@ class SSOManager extends EventEmitter {
       },
       ..._options,
     };
+  }
 
-    this.sessions = new Map();
-    this.providers = new Map();
-    this.userCache = new Map();
-
-    // Initialize rate limiter for login attempts
+  /**
+   * Create and configure the rate limiter for login attempt throttling.
+   * @private
+   */
+  _initializeRateLimiter() {
     if (this._config.rateLimit.enabled) {
       this.rateLimiter = new RateLimiter({
         maxAttempts: this._config.rateLimit.maxLoginAttempts,
@@ -82,8 +102,14 @@ class SSOManager extends EventEmitter {
         blockDurationMs: this._config.rateLimit.blockDurationMs,
       });
     }
+  }
 
-    // Initialize hardened JWT validator
+  /**
+   * Create and configure the hardened JWT validator for token operations.
+   * @param {Object} _options - Original user-supplied options for JWT settings
+   * @private
+   */
+  _initializeJWTValidator(_options) {
     this.jwtValidator = new JWTValidator(
       createJWTConfig({
         secret: this._config.security.jwtSecret,
@@ -99,29 +125,26 @@ class SSOManager extends EventEmitter {
         enableAuditLog: _options.jwtEnableAuditLog !== false,
       }),
     );
+  }
 
-    // Subscribe to JWT validator events for audit logging
-    this.jwtValidator.on('token_generated', (event) => {
-      this.emit('jwt_token_generated', event);
-    });
-
-    this.jwtValidator.on('token_validated', (event) => {
-      this.emit('jwt_token_validated', event);
-    });
-
-    this.jwtValidator.on('validation_failed', (event) => {
-      this.emit('jwt_validation_failed', event);
-    });
-
-    this.jwtValidator.on('algorithm_mismatch', (event) => {
-      this.emit('jwt_algorithm_mismatch', event);
-    });
-
-    this.jwtValidator.on('replay_detected', (event) => {
-      this.emit('jwt_replay_detected', event);
-    });
-
-    this._initializeProviders();
+  /**
+   * Forward JWT validator events to the SSOManager for audit logging.
+   * @private
+   */
+  _wireJWTEvents() {
+    const eventMappings = [
+      ['token_generated', 'jwt_token_generated'],
+      ['token_validated', 'jwt_token_validated'],
+      ['validation_failed', 'jwt_validation_failed'],
+      ['algorithm_mismatch', 'jwt_algorithm_mismatch'],
+      ['replay_detected', 'jwt_replay_detected'],
+    ];
+
+    for (const [sourceEvent, targetEvent] of eventMappings) {
+      this.jwtValidator.on(sourceEvent, (event) => {
+        this.emit(targetEvent, event);
+      });
+    }
   }
 
   /**
@@ -1067,6 +1090,3 @@ module.exports = {
   ActiveDirectoryProvider,
   OIDCProvider,
 };
-
-// Ensure module.exports is properly defined
-module.exports.default = module.exports;

package/src/evaluate/context-metrics.js

@@ -0,0 +1,127 @@
+'use strict';
+
+const {
+  tokenize,
+  splitSentences,
+  computeJaccardSimilarity,
+} = require('./scoring');
+
+/**
+ * Compute context precision: the fraction of retrieved documents that are relevant to the query.
+ *
+ * @param {string} query - The user query
+ * @param {Array<{content: string}>} retrievedDocs - Array of retrieved documents
+ * @param {object} [options={}] - Configuration options
+ * @param {number} [options.threshold=0.2] - Minimum Jaccard similarity for a doc to be considered relevant
+ * @returns {{ score: number, relevantDocs: number, totalDocs: number, perDocScores: Array<{ index: number, score: number, isRelevant: boolean }> }}
+ */
+function computeContextPrecision(query, retrievedDocs, options = {}) {
+  const { threshold = 0.2 } = options;
+
+  if (!query || typeof query !== 'string' || query.trim().length === 0) {
+    return { score: 0, relevantDocs: 0, totalDocs: 0, perDocScores: [] };
+  }
+
+  if (!Array.isArray(retrievedDocs) || retrievedDocs.length === 0) {
+    return { score: 0, relevantDocs: 0, totalDocs: 0, perDocScores: [] };
+  }
+
+  const queryTokens = tokenize(query);
+  const perDocScores = [];
+
+  for (let i = 0; i < retrievedDocs.length; i++) {
+    const doc = retrievedDocs[i];
+    const docContent = doc && (doc.content || doc.text || '');
+
+    if (!docContent) {
+      perDocScores.push({ index: i, score: 0, isRelevant: false });
+      continue;
+    }
+
+    const docTokens = tokenize(docContent);
+    const score = computeJaccardSimilarity(queryTokens, docTokens);
+    perDocScores.push({ index: i, score, isRelevant: score >= threshold });
+  }
+
+  const relevantDocs = perDocScores.filter((d) => d.isRelevant).length;
+  const totalDocs = perDocScores.length;
+  const score = totalDocs > 0 ? relevantDocs / totalDocs : 0;
+
+  return { score, relevantDocs, totalDocs, perDocScores };
+}
+
+/**
+ * Compute context recall: the fraction of answer claims that are supported by retrieved documents.
+ *
+ * @param {string} answer - The generated answer text
+ * @param {Array<{content: string}>} retrievedDocs - Array of retrieved documents
+ * @param {object} [options={}] - Configuration options
+ * @param {number} [options.threshold=0.3] - Minimum Jaccard similarity for a claim to be considered supported
+ * @returns {{ score: number, supportedClaims: number, totalClaims: number, details: Array<{ claim: string, score: number, isSupported: boolean, bestDocIndex: number }> }}
+ */
+function computeContextRecall(answer, retrievedDocs, options = {}) {
+  const { threshold = 0.3 } = options;
+
+  if (!answer || typeof answer !== 'string' || answer.trim().length === 0) {
+    return { score: 0, supportedClaims: 0, totalClaims: 0, details: [] };
+  }
+
+  const claims = splitSentences(answer);
+
+  if (claims.length === 0) {
+    return { score: 0, supportedClaims: 0, totalClaims: 0, details: [] };
+  }
+
+  if (!Array.isArray(retrievedDocs) || retrievedDocs.length === 0) {
+    const details = claims.map((claim) => ({
+      claim,
+      score: 0,
+      isSupported: false,
+      bestDocIndex: -1,
+    }));
+    return {
+      score: 0,
+      supportedClaims: 0,
+      totalClaims: claims.length,
+      details,
+    };
+  }
+
+  const details = claims.map((claim) => {
+    const claimTokens = tokenize(claim);
+    let bestScore = 0;
+    let bestDocIndex = -1;
+
+    for (let i = 0; i < retrievedDocs.length; i++) {
+      const doc = retrievedDocs[i];
+      const docContent = doc && (doc.content || doc.text || '');
+
+      if (!docContent) {
+        continue;
+      }
+
+      const docTokens = tokenize(docContent);
+      const similarity = computeJaccardSimilarity(claimTokens, docTokens);
+
+      if (similarity > bestScore) {
+        bestScore = similarity;
+        bestDocIndex = i;
+      }
+    }
+
+    return {
+      claim,
+      score: bestScore,
+      isSupported: bestScore >= threshold,
+      bestDocIndex,
+    };
+  });
+
+  const supportedClaims = details.filter((d) => d.isSupported).length;
+  const totalClaims = details.length;
+  const score = totalClaims > 0 ? supportedClaims / totalClaims : 0;
+
+  return { score, supportedClaims, totalClaims, details };
+}
+
+module.exports = { computeContextPrecision, computeContextRecall };

package/src/evaluate/evaluator.js

@@ -23,8 +23,10 @@ async function evaluateRagDataset(_datasetPath, _config) {
   for (const { prompt, expected } of cases) {
     try {
       logger.info({ prompt }, 'Evaluating case');
-      const actual = await pipeline.query(prompt);
-      const success = normalizeText(actual) === normalizeText(expected);
+      const result = await pipeline.run({ query: prompt });
+      const actual = result.results || '';
+      const success =
+        result.success && normalizeText(actual) === normalizeText(expected);
       const scores = scoreAnswer(actual, expected);
       results.push({ prompt, expected, actual, success, scores });
     } catch (err) {