@devilsdev/rag-pipeline-utils 2.3.0 → 2.4.0

package/src/core/create-pipeline.js

@@ -85,7 +85,27 @@ module.exports.createRagPipeline = function createRagPipeline({
       const answer = (await llmI.generate?.(query, results, options)) ?? '';
       // tiny delay so durations > 0 in timing tests
       await new Promise((r) => setTimeout(r, 5));
-      return { success: true, query, results };
+
+      const result = { success: true, query, results };
+
+      if (options.citations) {
+        const { CitationTracker } = require('../citation/citation-tracker');
+        const tracker = new CitationTracker(options.citationOptions || {});
+        result.answer = String(answer);
+        result.citations = tracker.track(String(answer), results);
+      }
+
+      if (options.evaluate) {
+        const { PipelineEvaluator } = require('../evaluate/pipeline-evaluator');
+        const evaluator = new PipelineEvaluator(options.evaluateOptions || {});
+        result.answer = result.answer || String(answer);
+        result.evaluation = evaluator.evaluate(
+          { query, answer: String(answer), results },
+          { citationResult: result.citations },
+        );
+      }
+
+      return result;
     })();
 
     try {

package/src/core/observability/tracing.js

@@ -10,6 +10,7 @@ const {
   EventSeverity,
   PipelineEventLogger,
 } = require('../../observability/event-logger.js');
+const { logger } = require('../../utils/logger');
 
 const _logger = global.eventLogger || eventLogger || new PipelineEventLogger();
 global.eventLogger = _logger;
@@ -233,7 +234,7 @@ class Span {
    */
   end(endTime = new Date()) {
     if (this.endTime !== null) {
-      console.warn(`Span ${this.name} already ended`); // eslint-disable-line no-console
+      logger.warn(`Span ${this.name} already ended`);
       return;
     }
 

package/src/core/pipeline-factory.js

@@ -57,4 +57,3 @@ async function createPipeline({
 module.exports = createPipeline;
 module.exports.createPipeline = createPipeline;
 module.exports.createRagPipeline = createPipeline; // Alias for backward compatibility
-module.exports.default = module.exports;

package/src/core/plugin-registry.js

@@ -6,6 +6,7 @@ const Ajv = require('ajv');
 const {
   createPluginVerifier,
 } = require('../security/plugin-signature-verifier.js');
+const { logger } = require('../utils/logger');
 
 class PluginRegistry {
   constructor(options = {}) {
@@ -46,9 +47,9 @@ class PluginRegistry {
           process.env.NODE_ENV !== 'production' &&
           !this._disableContractWarnings
         ) {
-          console.warn(
+          logger.warn(
             `[PLUGIN_REGISTRY] Warning: Failed to compile contract schema: ${error.message}`,
-          ); // eslint-disable-line no-console
+          );
         }
         this._contractSchema = null;
       }
@@ -93,9 +94,9 @@ class PluginRegistry {
         process.env.NODE_ENV !== 'production' &&
         !this._disableContractWarnings
       ) {
-        console.warn(
+        logger.warn(
           `[PLUGIN_REGISTRY] Warning: Failed to load contract schema: ${error.message}`,
-        ); // eslint-disable-line no-console
+        );
       }
     }
     return null;
@@ -218,7 +219,7 @@ class PluginRegistry {
 
     this._contractWarningsShown.add(warningKey);
 
-    console.warn(
+    logger.warn(
       `
 ╔════════════════════════════════════════════════════════════════════════════╗
 ║ [PLUGIN_REGISTRY] Missing Contract for '${type}' plugins                       ║
@@ -251,7 +252,7 @@ class PluginRegistry {
 ║  new PluginRegistry({ disableContractWarnings: true })                    ║
 ╚════════════════════════════════════════════════════════════════════════════╝
     `.trim(),
-    ); // eslint-disable-line no-console
+    );
   }
 
   _loadContracts() {
@@ -289,7 +290,7 @@ class PluginRegistry {
 
               if (process.env.NODE_ENV === 'production') {
                 // In production, log error but continue (fail open)
-                console.error(`[PLUGIN_REGISTRY] ERROR: ${error.message}`); // eslint-disable-line no-console
+                logger.error(`[PLUGIN_REGISTRY] ERROR: ${error.message}`);
               } else {
                 // In development, throw to fail fast
                 throw error;
@@ -312,9 +313,9 @@ class PluginRegistry {
             process.env.NODE_ENV !== 'production' &&
             !this._disableContractWarnings
           ) {
-            console.warn(
+            logger.warn(
               `[PLUGIN_REGISTRY] Warning: Failed to load contract for '${type}': ${error.message}`,
-            ); // eslint-disable-line no-console
+            );
           }
         }
       }
@@ -329,9 +330,9 @@ class PluginRegistry {
         process.env.NODE_ENV !== 'production' &&
         !this._disableContractWarnings
       ) {
-        console.warn(
+        logger.warn(
           `[PLUGIN_REGISTRY] Warning: Contracts directory not accessible: ${error.message}`,
-        ); // eslint-disable-line no-console
+        );
       }
     }
   }
@@ -487,9 +488,9 @@ class PluginRegistry {
 
     // In production, this should go to a secure audit log
     if (process.env.NODE_ENV === 'production') {
-      console.warn('[AUDIT]', JSON.stringify(auditEntry)); // eslint-disable-line no-console
+      logger.warn('[AUDIT]', JSON.stringify(auditEntry));
     } else {
-      console.log('[AUDIT]', auditEntry); // eslint-disable-line no-console
+      logger.info('[AUDIT]', auditEntry);
     }
   }
 
@@ -537,4 +538,3 @@ const registry = new PluginRegistry({
 // CJS+ESM interop pattern
 module.exports = registry;
 module.exports.PluginRegistry = PluginRegistry;
-module.exports.default = module.exports;

package/src/cost/cost-calculator.js

@@ -0,0 +1,146 @@
+'use strict';
+
+const { EventEmitter } = require('events');
+
+/**
+ * Built-in pricing table for common LLM and embedding models.
+ * Prices are in USD per `unit` tokens.
+ * @type {Record<string, {input: number, output: number, unit: number}>}
+ */
+const PRICING = {
+  'gpt-4': { input: 0.03, output: 0.06, unit: 1000 },
+  'gpt-4-turbo': { input: 0.01, output: 0.03, unit: 1000 },
+  'gpt-4o': { input: 0.005, output: 0.015, unit: 1000 },
+  'gpt-4o-mini': { input: 0.00015, output: 0.0006, unit: 1000 },
+  'gpt-3.5-turbo': { input: 0.0005, output: 0.0015, unit: 1000 },
+  'claude-3-opus': { input: 0.015, output: 0.075, unit: 1000 },
+  'claude-3-sonnet': { input: 0.003, output: 0.015, unit: 1000 },
+  'claude-3-haiku': { input: 0.00025, output: 0.00125, unit: 1000 },
+  'claude-3.5-sonnet': { input: 0.003, output: 0.015, unit: 1000 },
+  'cohere-command': { input: 0.001, output: 0.002, unit: 1000 },
+  'text-embedding-3-small': { input: 0.00002, output: 0, unit: 1000 },
+  'text-embedding-3-large': { input: 0.00013, output: 0, unit: 1000 },
+};
+
+/**
+ * Maps token counts to cost estimates by provider and model.
+ *
+ * @example
+ * const calc = new CostCalculator();
+ * const cost = calc.estimate('gpt-4o', 1500, 500);
+ * console.log(cost.totalCost); // 0.015
+ */
+class CostCalculator extends EventEmitter {
+  /**
+   * @param {object} [options]
+   * @param {Record<string, {input: number, output: number, unit: number}>} [options.customPricing]
+   *   Additional or override model pricing entries.
+   */
+  constructor(options = {}) {
+    super();
+    this.pricing = { ...PRICING, ...(options.customPricing || {}) };
+  }
+
+  /**
+   * Resolve a model key from the pricing table.
+   * Tries exact match (case-insensitive), then partial match.
+   * @param {string} model
+   * @returns {string|null} The matched pricing key or null.
+   * @private
+   */
+  _resolveModel(model) {
+    const lower = model.toLowerCase();
+    const keys = Object.keys(this.pricing);
+
+    // Exact case-insensitive match
+    const exact = keys.find((k) => k.toLowerCase() === lower);
+    if (exact) return exact;
+
+    // Partial match — model string contains a known key or vice-versa
+    const partial = keys.find(
+      (k) => lower.includes(k.toLowerCase()) || k.toLowerCase().includes(lower),
+    );
+    return partial || null;
+  }
+
+  /**
+   * Estimate the cost for a given token count.
+   *
+   * @param {string} model - Model identifier (case-insensitive, partial match supported).
+   * @param {number} inputTokens - Number of input / prompt tokens.
+   * @param {number} [outputTokens=0] - Number of output / completion tokens.
+   * @returns {{inputCost: number, outputCost: number, totalCost: number, model: string,
+   *            inputTokens: number, outputTokens: number, currency: string, warning?: string}}
+   */
+  estimate(model, inputTokens, outputTokens = 0) {
+    const resolvedKey = this._resolveModel(model);
+
+    if (!resolvedKey) {
+      return {
+        inputCost: 0,
+        outputCost: 0,
+        totalCost: 0,
+        model,
+        inputTokens,
+        outputTokens,
+        currency: 'USD',
+        warning: 'Unknown model',
+      };
+    }
+
+    const price = this.pricing[resolvedKey];
+    const inputCost = (inputTokens / price.unit) * price.input;
+    const outputCost = (outputTokens / price.unit) * price.output;
+    const totalCost = inputCost + outputCost;
+
+    return {
+      inputCost,
+      outputCost,
+      totalCost,
+      model: resolvedKey,
+      inputTokens,
+      outputTokens,
+      currency: 'USD',
+    };
+  }
+
+  /**
+   * Register custom model pricing.
+   *
+   * @param {string} model - Model identifier.
+   * @param {{input: number, output: number, unit: number}} pricing - Pricing definition.
+   */
+  addModel(model, pricing) {
+    if (
+      !pricing ||
+      typeof pricing.input !== 'number' ||
+      typeof pricing.output !== 'number'
+    ) {
+      throw new Error(
+        'Pricing must include numeric "input" and "output" fields',
+      );
+    }
+    this.pricing[model] = { unit: 1000, ...pricing };
+  }
+
+  /**
+   * List all known model names.
+   * @returns {string[]}
+   */
+  listModels() {
+    return Object.keys(this.pricing);
+  }
+
+  /**
+   * Retrieve pricing for a specific model.
+   *
+   * @param {string} model - Model identifier (supports fuzzy resolution).
+   * @returns {{input: number, output: number, unit: number}|null}
+   */
+  getModelPricing(model) {
+    const key = this._resolveModel(model);
+    return key ? { ...this.pricing[key] } : null;
+  }
+}
+
+module.exports = { CostCalculator, PRICING };

package/src/cost/cost-tracker.js

@@ -0,0 +1,260 @@
+'use strict';
+
+const { EventEmitter } = require('events');
+const { CostCalculator } = require('./cost-calculator');
+const { TokenBudget } = require('./token-budget');
+
+/**
+ * Default tracker configuration.
+ * @type {{model: string, embeddingModel: string, enableBudget: boolean, budgetOptions: object}}
+ */
+const DEFAULT_CONFIG = {
+  model: 'gpt-3.5-turbo',
+  embeddingModel: 'text-embedding-3-small',
+  enableBudget: false,
+  budgetOptions: {},
+};
+
+/**
+ * Estimated tokens per character when actual token counts are unavailable.
+ * @private
+ */
+const CHARS_PER_TOKEN = 4;
+
+/**
+ * Wraps pipeline execution to automatically track token usage and costs.
+ *
+ * @example
+ * const tracker = new CostTracker({ model: 'gpt-4o', enableBudget: true, budgetOptions: { maxCost: 10 } });
+ * const wrapped = tracker.wrap(pipeline);
+ * const result = await wrapped.run(input);
+ * console.log(result.cost); // { inputTokens, outputTokens, estimatedCost, model }
+ * console.log(tracker.getSummary());
+ */
+class CostTracker extends EventEmitter {
+  /**
+   * @param {object} [options]
+   * @param {string} [options.model='gpt-3.5-turbo'] - Default LLM model for cost estimation.
+   * @param {string} [options.embeddingModel='text-embedding-3-small'] - Default embedding model.
+   * @param {boolean} [options.enableBudget=false] - Whether to enforce a token/cost budget.
+   * @param {object} [options.budgetOptions] - Options forwarded to {@link TokenBudget}.
+   * @param {object} [options.calculatorOptions] - Options forwarded to {@link CostCalculator}.
+   */
+  constructor(options = {}) {
+    super();
+    this.config = { ...DEFAULT_CONFIG, ...options };
+    this.calculator = new CostCalculator(options.calculatorOptions);
+    this.budget = this.config.enableBudget
+      ? new TokenBudget(this.config.budgetOptions)
+      : null;
+    this.sessions = [];
+    this.currentSession = null;
+  }
+
+  /**
+   * Extract token usage from a pipeline result.
+   * Checks common locations: result.usage, result.evaluation, or estimates from content length.
+   * @param {object} result
+   * @returns {{inputTokens: number, outputTokens: number}}
+   * @private
+   */
+  _extractUsage(result) {
+    // Direct usage field (OpenAI-style)
+    if (result.usage) {
+      return {
+        inputTokens:
+          result.usage.prompt_tokens || result.usage.inputTokens || 0,
+        outputTokens:
+          result.usage.completion_tokens || result.usage.outputTokens || 0,
+      };
+    }
+
+    // Evaluation metadata
+    if (result.evaluation && result.evaluation.tokens) {
+      return {
+        inputTokens: result.evaluation.tokens.input || 0,
+        outputTokens: result.evaluation.tokens.output || 0,
+      };
+    }
+
+    // Estimate from content length
+    const inputText = result.input || result.query || result.prompt || '';
+    const outputText = result.output || result.answer || result.response || '';
+    return {
+      inputTokens: Math.ceil(String(inputText).length / CHARS_PER_TOKEN),
+      outputTokens: Math.ceil(String(outputText).length / CHARS_PER_TOKEN),
+    };
+  }
+
+  /**
+   * Wrap a pipeline so that every `.run()` call is cost-tracked.
+   *
+   * @param {object} pipeline - Object with a `.run()` method.
+   * @returns {object} A proxy pipeline whose `.run()` enriches results with cost data.
+   */
+  wrap(pipeline) {
+    const tracker = this;
+
+    return {
+      ...pipeline,
+
+      /**
+       * Execute the wrapped pipeline and attach cost metadata to the result.
+       * @param {...*} args - Arguments forwarded to the original pipeline.run().
+       * @returns {Promise<object>} The pipeline result enriched with a `.cost` property.
+       */
+      run: async (...args) => {
+        if (!tracker.currentSession) {
+          tracker.startSession({ autoStarted: true });
+        }
+
+        const result = await pipeline.run(...args);
+
+        const { inputTokens, outputTokens } = tracker._extractUsage(result);
+        const model = result.model || tracker.config.model;
+        const costEstimate = tracker.calculator.estimate(
+          model,
+          inputTokens,
+          outputTokens,
+        );
+
+        // Budget enforcement
+        if (tracker.budget) {
+          const budgetCheck = tracker.budget.check(
+            inputTokens + outputTokens,
+            costEstimate.totalCost,
+          );
+          if (!budgetCheck.allowed) {
+            const err = new Error(`Budget exceeded: ${budgetCheck.reason}`);
+            err.code = 'BUDGET_EXCEEDED';
+            err.budgetCheck = budgetCheck;
+            throw err;
+          }
+          tracker.budget.record(
+            inputTokens + outputTokens,
+            costEstimate.totalCost,
+          );
+        }
+
+        // Attach cost data to result
+        result.cost = {
+          inputTokens,
+          outputTokens,
+          estimatedCost: costEstimate.totalCost,
+          model: costEstimate.model,
+          currency: 'USD',
+        };
+
+        // Record in session
+        const entry = {
+          timestamp: Date.now(),
+          inputTokens,
+          outputTokens,
+          cost: costEstimate.totalCost,
+          model: costEstimate.model,
+        };
+
+        if (tracker.currentSession) {
+          tracker.currentSession.entries.push(entry);
+          tracker.currentSession.totalTokens += inputTokens + outputTokens;
+          tracker.currentSession.totalCost += costEstimate.totalCost;
+        }
+
+        tracker.emit('tracked', entry);
+
+        return result;
+      },
+    };
+  }
+
+  /**
+   * Start a new tracking session.
+   *
+   * @param {object} [metadata={}] - Arbitrary metadata to attach to the session.
+   * @returns {{id: string, startTime: number, metadata: object, entries: Array,
+   *            totalTokens: number, totalCost: number}}
+   */
+  startSession(metadata = {}) {
+    const session = {
+      id: `session_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
+      startTime: Date.now(),
+      endTime: null,
+      metadata,
+      entries: [],
+      totalTokens: 0,
+      totalCost: 0,
+    };
+
+    this.currentSession = session;
+    this.sessions.push(session);
+    return session;
+  }
+
+  /**
+   * End the current tracking session and compute totals.
+   *
+   * @returns {object|null} The closed session, or null if no session was active.
+   */
+  endSession() {
+    if (!this.currentSession) return null;
+
+    this.currentSession.endTime = Date.now();
+    const closed = this.currentSession;
+    this.currentSession = null;
+    return closed;
+  }
+
+  /**
+   * Return an aggregate summary across all sessions.
+   *
+   * @returns {{totalTokens: number, totalCost: number, sessionCount: number,
+   *            avgCostPerQuery: number, breakdown: Record<string, {tokens: number, cost: number, count: number}>}}
+   */
+  getSummary() {
+    let totalTokens = 0;
+    let totalCost = 0;
+    let totalOps = 0;
+    const breakdown = {};
+
+    for (const session of this.sessions) {
+      for (const entry of session.entries) {
+        totalTokens += entry.inputTokens + entry.outputTokens;
+        totalCost += entry.cost;
+        totalOps += 1;
+
+        if (!breakdown[entry.model]) {
+          breakdown[entry.model] = { tokens: 0, cost: 0, count: 0 };
+        }
+        breakdown[entry.model].tokens += entry.inputTokens + entry.outputTokens;
+        breakdown[entry.model].cost += entry.cost;
+        breakdown[entry.model].count += 1;
+      }
+    }
+
+    return {
+      totalTokens,
+      totalCost,
+      sessionCount: this.sessions.length,
+      avgCostPerQuery: totalOps > 0 ? totalCost / totalOps : 0,
+      breakdown,
+    };
+  }
+
+  /**
+   * Return the full history of tracked operations across all sessions.
+   *
+   * @returns {Array<{timestamp: number, inputTokens: number, outputTokens: number,
+   *                   cost: number, model: string, sessionId: string}>}
+   */
+  getHistory() {
+    const history = [];
+    for (const session of this.sessions) {
+      for (const entry of session.entries) {
+        history.push({ ...entry, sessionId: session.id });
+      }
+    }
+    return history;
+  }
+}
+
+module.exports = { CostTracker, DEFAULT_CONFIG };