@devilsdev/rag-pipeline-utils 2.4.5 → 2.6.0

package/src/core/plugin-registry.js

@@ -2,12 +2,28 @@
 
 const fs = require('fs');
 const path = require('path');
-const Ajv = require('ajv');
+// ajv loads with the contracts; see _ensureContracts().
+const loadAjv = () => require('ajv'); // eslint-disable-line global-require
 const {
   createPluginVerifier,
 } = require('../security/plugin-signature-verifier.js');
 const { logger } = require('../utils/logger');
 
+/**
+ * Locate the package's contracts/ directory. This file lives in src/core/ when
+ * loaded from source but is inlined into dist/index.cjs when bundled, so
+ * __dirname differs between the two layouts.
+ *
+ * @returns {string} Absolute path to the contracts directory
+ */
+function resolveContractsDir() {
+  const candidates = [
+    path.join(__dirname, '../../contracts'),
+    path.join(__dirname, '../contracts'),
+  ];
+  return candidates.find((dir) => fs.existsSync(dir)) || candidates[0];
+}
+
 class PluginRegistry {
   constructor(options = {}) {
     this._plugins = new Map();
@@ -27,7 +43,43 @@ class PluginRegistry {
         ? options.validateContractSchema
         : true; // Enable schema validation by default
 
-    // Initialize Ajv for contract schema validation
+    // Environment-aware signature verification defaults
+    const isDevelopment = process.env.NODE_ENV === 'development';
+    const isProduction = process.env.NODE_ENV === 'production';
+
+    this._signatureVerifier = createPluginVerifier({
+      enabled:
+        options.verifySignatures !== undefined
+          ? options.verifySignatures
+          : isProduction, // Enabled by default in production, disabled in development
+      failClosed:
+        options.failClosed !== undefined ? options.failClosed : isProduction, // Fail closed in production, fail open in development
+      trustedKeysPath: options.trustedKeysPath,
+      requireContentDigest: options.requireContentDigest === true,
+    });
+    // Opt-in: when verification is enabled, refuse plugins registered
+    // without a signed manifest (default false for backward compatibility).
+    this._requireSignedPlugins = options.requireSignedPlugins === true;
+
+    // Contracts (and ajv, which validates them) load in the constructor, or
+    // on first use with `lazyContracts: true`. The shared registry exported
+    // by this module is lazy, so importing the package does not pay for
+    // them unless a plugin is registered or validated.
+    this._contractsLoaded = false;
+    if (!options.lazyContracts) {
+      this._ensureContracts();
+    }
+  }
+
+  /**
+   * Load the contract schema and the stage contracts once.
+   * @private
+   */
+  _ensureContracts() {
+    if (this._contractsLoaded) return;
+    this._contractsLoaded = true;
+
+    const Ajv = loadAjv();
     this._ajv = new Ajv({
       allErrors: true,
       verbose: true,
@@ -55,19 +107,6 @@ class PluginRegistry {
       }
     }
 
-    // Environment-aware signature verification defaults
-    const isDevelopment = process.env.NODE_ENV === 'development';
-    const isProduction = process.env.NODE_ENV === 'production';
-
-    this._signatureVerifier = createPluginVerifier({
-      enabled:
-        options.verifySignatures !== undefined
-          ? options.verifySignatures
-          : isProduction, // Enabled by default in production, disabled in development
-      failClosed:
-        options.failClosed !== undefined ? options.failClosed : isProduction, // Fail closed in production, fail open in development
-      trustedKeysPath: options.trustedKeysPath,
-    });
     this._loadContracts();
   }
 
@@ -82,8 +121,8 @@ class PluginRegistry {
   _loadContractSchema() {
     try {
       const schemaPath = path.join(
-        __dirname,
-        '../../contracts/contract-schema.json',
+        resolveContractsDir(),
+        'contract-schema.json',
       );
       if (fs.existsSync(schemaPath)) {
         const schemaContent = fs.readFileSync(schemaPath, 'utf8');
@@ -186,6 +225,7 @@ class PluginRegistry {
    * @since 2.2.4
    */
   checkContractExists(type) {
+    this._ensureContracts();
     return this._contracts.has(type);
   }
 
@@ -256,7 +296,7 @@ class PluginRegistry {
   }
 
   _loadContracts() {
-    const contractsDir = path.join(__dirname, '../../contracts');
+    const contractsDir = resolveContractsDir();
 
     try {
       const contractFiles = {
@@ -338,6 +378,7 @@ class PluginRegistry {
   }
 
   _validatePluginContract(category, impl) {
+    this._ensureContracts();
     const contract = this._contracts.get(category);
     if (!contract) {
       // Use new warning method when registering a plugin without a contract
@@ -396,7 +437,20 @@ class PluginRegistry {
     return methods;
   }
 
-  async register(category, name, impl, manifest = null) {
+  /**
+   * Register a plugin.
+   *
+   * @param {string} category - Plugin type
+   * @param {string} name - Plugin name
+   * @param {object} impl - Plugin implementation
+   * @param {object|null} [manifest] - Signed manifest; when given, its
+   *   signature is verified before registration
+   * @param {object} [options]
+   * @param {object} [options.content] - Plugin content checked against the
+   *   manifest's contentDigest: `{ files, baseDir, contents }`
+   * @returns {Promise<PluginRegistry>}
+   */
+  async register(category, name, impl, manifest = null, options = {}) {
     if (!category || typeof category !== 'string') {
       throw new Error('Category must be a non-empty string');
     }
@@ -412,7 +466,22 @@ class PluginRegistry {
 
     // Verify plugin signature if manifest provided
     if (manifest) {
-      await this._verifyPluginSignature(manifest, name);
+      await this._verifyPluginSignature(manifest, name, options.content);
+    } else if (
+      this._requireSignedPlugins &&
+      this.isSignatureVerificationEnabled()
+    ) {
+      this._emitAuditEntry('plugin_verification_failed', {
+        pluginName: name,
+        signerId: 'unknown',
+        version: 'unknown',
+        verified: false,
+        error: 'missing signed manifest',
+        timestamp: new Date().toISOString(),
+      });
+      throw new Error(
+        `Plugin ${name} has no signed manifest and requireSignedPlugins is enabled`,
+      );
     }
 
     // Validate plugin against contract
@@ -428,7 +497,7 @@ class PluginRegistry {
    * @param {object} manifest - Plugin manifest
    * @param {string} pluginName - Plugin name
    */
-  async _verifyPluginSignature(manifest, pluginName) {
+  async _verifyPluginSignature(manifest, pluginName, content = undefined) {
     try {
       // Extract signature information from manifest
       if (!manifest.signature || !manifest.signerId) {
@@ -437,12 +506,12 @@ class PluginRegistry {
         );
       }
 
+      const args = [manifest, manifest.signature, manifest.signerId];
+      if (content) {
+        args.push(content);
+      }
       const verificationResult =
-        await this._signatureVerifier.verifyPluginSignature(
-          manifest,
-          manifest.signature,
-          manifest.signerId,
-        );
+        await this._signatureVerifier.verifyPluginSignature(...args);
 
       if (!verificationResult.verified) {
         throw new Error(
@@ -473,6 +542,35 @@ class PluginRegistry {
     }
   }
 
+  /**
+   * Whether plugin signature verification is enabled for this registry.
+   * @returns {boolean}
+   */
+  isSignatureVerificationEnabled() {
+    return Boolean(this._signatureVerifier && this._signatureVerifier.enabled);
+  }
+
+  /**
+   * Whether plugins without a signed manifest are rejected.
+   * @returns {boolean}
+   */
+  requiresSignedPlugins() {
+    return this._requireSignedPlugins && this.isSignatureVerificationEnabled();
+  }
+
+  /**
+   * Verify a plugin's signed manifest and code before it is loaded (use this
+   * before importing plugin files, since importing executes them).
+   *
+   * @param {string} pluginName - Plugin name (for audit entries)
+   * @param {object} manifest - Signed manifest
+   * @param {object} content - `{ files, baseDir, contents }`
+   * @returns {Promise<void>} Rejects if verification fails
+   */
+  async verifyPluginContent(pluginName, manifest, content) {
+    await this._verifyPluginSignature(manifest, pluginName, content);
+  }
+
   /**
    * Emit audit entry
    * @param {string} action - Audit action
@@ -533,6 +631,7 @@ class PluginRegistry {
 const registry = new PluginRegistry({
   verifySignatures: process.env.NODE_ENV === 'production',
   failClosed: process.env.NODE_ENV === 'production',
+  lazyContracts: true,
 });
 
 // CJS+ESM interop pattern

package/src/core/stage-contracts.js

@@ -0,0 +1,36 @@
+'use strict';
+
+/**
+ * What each pipeline stage must provide, as createRagPipeline and the CLI
+ * use it. A plugin needs at least one of `anyOf`; a missing `recommended`
+ * method only matters for the command named in its note.
+ *
+ * Shared by validatePlugin() and the `rag-pipeline init` wizard. The
+ * stricter rules pluginRegistry.register() applies (metadata, and the
+ * methods in contracts/*.json) are checked by validatePlugin(..., {
+ * registry: true }).
+ */
+const STAGE_CONTRACTS = Object.freeze({
+  loader: {
+    anyOf: ['load'],
+    recommended: {},
+  },
+  embedder: {
+    anyOf: ['embed', 'embedQuery'],
+    recommended: { embed: '`ingest` calls embed() to embed documents' },
+  },
+  retriever: {
+    anyOf: ['retrieve'],
+    recommended: { store: '`ingest` calls store() to save vectors' },
+  },
+  reranker: {
+    anyOf: ['rerank'],
+    recommended: {},
+  },
+  llm: {
+    anyOf: ['generate'],
+    recommended: {},
+  },
+});
+
+module.exports = { STAGE_CONTRACTS };

package/src/core/validate-plugin.js

@@ -0,0 +1,85 @@
+'use strict';
+
+const { STAGE_CONTRACTS } = require('./stage-contracts');
+
+/**
+ * @typedef {Object} PluginValidationResult
+ * @property {boolean} valid - True when there are no errors
+ * @property {string[]} errors - Problems that stop the plugin from working
+ * @property {string[]} warnings - Methods some commands need, e.g. `store`
+ *   for `rag-pipeline ingest`
+ */
+
+/**
+ * Checks that an object can be used as a pipeline stage, without throwing,
+ * so plugin authors can assert on it in their own tests.
+ *
+ * By default it checks what createRagPipeline() calls. With
+ * `{ registry: true }` it also applies the rules pluginRegistry.register()
+ * enforces: `metadata` with a string `name` and `version` and a `type`
+ * equal to the stage, plus the methods in the stage's contract.
+ *
+ * @example
+ * const { validatePlugin } = require('@devilsdev/rag-pipeline-utils');
+ * const result = validatePlugin('llm', myLlm, { registry: true });
+ * expect(result.errors).toEqual([]);
+ *
+ * @param {'loader'|'embedder'|'retriever'|'reranker'|'llm'} stage
+ * @param {object} plugin - Plugin instance or object
+ * @param {Object} [options]
+ * @param {boolean} [options.registry=false] - Also check registry rules
+ * @returns {PluginValidationResult}
+ */
+function validatePlugin(stage, plugin, options = {}) {
+  const contract = STAGE_CONTRACTS[stage];
+  if (!contract) {
+    return {
+      valid: false,
+      errors: [
+        `Unknown stage "${stage}". Expected one of: ${Object.keys(STAGE_CONTRACTS).join(', ')}`,
+      ],
+      warnings: [],
+    };
+  }
+
+  if (
+    plugin === null ||
+    (typeof plugin !== 'object' && typeof plugin !== 'function')
+  ) {
+    return {
+      valid: false,
+      errors: [`A ${stage} plugin must be an object or class instance`],
+      warnings: [],
+    };
+  }
+
+  const has = (method) => typeof plugin[method] === 'function';
+  const errors = [];
+  const warnings = [];
+
+  if (!contract.anyOf.some(has)) {
+    errors.push(
+      `A ${stage} needs ${contract.anyOf.map((m) => `${m}()`).join(' or ')}`,
+    );
+  }
+  for (const [method, why] of Object.entries(contract.recommended)) {
+    if (!has(method)) warnings.push(`No ${method}() method: ${why}`);
+  }
+
+  if (options.registry) {
+    // Reuse the registry's own check so the two cannot drift. It stops at
+    // the first problem, so this reports one registry error at a time.
+    // eslint-disable-next-line global-require
+    const { PluginRegistry } = require('./plugin-registry');
+    try {
+      new PluginRegistry()._validatePluginContract(stage, plugin);
+    } catch (error) {
+      const message = `pluginRegistry.register(): ${error.message}`;
+      if (!errors.includes(message)) errors.push(message);
+    }
+  }
+
+  return { valid: errors.length === 0, errors, warnings };
+}
+
+module.exports = { validatePlugin };

package/src/cost/cost-calculator.js

@@ -2,26 +2,134 @@
 
 const { EventEmitter } = require('events');
 
+/**
+ * Where the built-in prices come from and when they were recorded.
+ *
+ * The values are the providers' published list prices as copied into this
+ * library on `asOf`; they have not been re-verified since and providers
+ * change prices often. Pass `pricing` or `customPricing` to CostCalculator
+ * to use current or negotiated prices.
+ */
+const PRICING_METADATA = Object.freeze({
+  asOf: '2026-04-07',
+  verified: false,
+  currency: 'USD',
+  sources: Object.freeze({
+    openai: 'https://openai.com/api/pricing/',
+    anthropic: 'https://www.anthropic.com/pricing#api',
+    cohere: 'https://cohere.com/pricing',
+  }),
+  note: 'Built-in list prices, not re-verified since asOf. Override with the `pricing` or `customPricing` option.',
+});
+
+const OPENAI = PRICING_METADATA.sources.openai;
+const ANTHROPIC = PRICING_METADATA.sources.anthropic;
+const COHERE = PRICING_METADATA.sources.cohere;
+const AS_OF = PRICING_METADATA.asOf;
+
 /**
  * Built-in pricing table for common LLM and embedding models.
- * Prices are in USD per `unit` tokens.
- * @type {Record<string, {input: number, output: number, unit: number}>}
+ * Prices are in USD per `unit` tokens. Each entry records its `source`
+ * (provider pricing page) and `asOf` date.
+ * @type {Record<string, {input: number, output: number, unit: number, source: string, asOf: string}>}
  */
 const PRICING = {
-  'gpt-4': { input: 0.03, output: 0.06, unit: 1000 },
-  'gpt-4-turbo': { input: 0.01, output: 0.03, unit: 1000 },
-  'gpt-4o': { input: 0.005, output: 0.015, unit: 1000 },
-  'gpt-4o-mini': { input: 0.00015, output: 0.0006, unit: 1000 },
-  'gpt-3.5-turbo': { input: 0.0005, output: 0.0015, unit: 1000 },
-  'claude-3-opus': { input: 0.015, output: 0.075, unit: 1000 },
-  'claude-3-sonnet': { input: 0.003, output: 0.015, unit: 1000 },
-  'claude-3-haiku': { input: 0.00025, output: 0.00125, unit: 1000 },
-  'claude-3.5-sonnet': { input: 0.003, output: 0.015, unit: 1000 },
-  'cohere-command': { input: 0.001, output: 0.002, unit: 1000 },
-  'text-embedding-3-small': { input: 0.00002, output: 0, unit: 1000 },
-  'text-embedding-3-large': { input: 0.00013, output: 0, unit: 1000 },
+  'gpt-4': {
+    input: 0.03,
+    output: 0.06,
+    unit: 1000,
+    source: OPENAI,
+    asOf: AS_OF,
+  },
+  'gpt-4-turbo': {
+    input: 0.01,
+    output: 0.03,
+    unit: 1000,
+    source: OPENAI,
+    asOf: AS_OF,
+  },
+  'gpt-4o': {
+    input: 0.005,
+    output: 0.015,
+    unit: 1000,
+    source: OPENAI,
+    asOf: AS_OF,
+  },
+  'gpt-4o-mini': {
+    input: 0.00015,
+    output: 0.0006,
+    unit: 1000,
+    source: OPENAI,
+    asOf: AS_OF,
+  },
+  'gpt-3.5-turbo': {
+    input: 0.0005,
+    output: 0.0015,
+    unit: 1000,
+    source: OPENAI,
+    asOf: AS_OF,
+  },
+  'claude-3-opus': {
+    input: 0.015,
+    output: 0.075,
+    unit: 1000,
+    source: ANTHROPIC,
+    asOf: AS_OF,
+  },
+  'claude-3-sonnet': {
+    input: 0.003,
+    output: 0.015,
+    unit: 1000,
+    source: ANTHROPIC,
+    asOf: AS_OF,
+  },
+  'claude-3-haiku': {
+    input: 0.00025,
+    output: 0.00125,
+    unit: 1000,
+    source: ANTHROPIC,
+    asOf: AS_OF,
+  },
+  'claude-3.5-sonnet': {
+    input: 0.003,
+    output: 0.015,
+    unit: 1000,
+    source: ANTHROPIC,
+    asOf: AS_OF,
+  },
+  'cohere-command': {
+    input: 0.001,
+    output: 0.002,
+    unit: 1000,
+    source: COHERE,
+    asOf: AS_OF,
+  },
+  'text-embedding-3-small': {
+    input: 0.00002,
+    output: 0,
+    unit: 1000,
+    source: OPENAI,
+    asOf: AS_OF,
+  },
+  'text-embedding-3-large': {
+    input: 0.00013,
+    output: 0,
+    unit: 1000,
+    source: OPENAI,
+    asOf: AS_OF,
+  },
 };
 
+/**
+ * Canonical form used for model-name matching: lower case, with '.' and '_'
+ * treated like '-' so "claude-3-5-sonnet-20241022" matches "claude-3.5-sonnet".
+ * @param {string} name
+ * @returns {string}
+ */
+function canonical(name) {
+  return String(name).toLowerCase().replace(/[._]/g, '-');
+}
+
 /**
  * Maps token counts to cost estimates by provider and model.
  *
@@ -33,34 +141,80 @@ const PRICING = {
 class CostCalculator extends EventEmitter {
   /**
    * @param {object} [options]
+   * @param {Record<string, {input: number, output: number, unit?: number, source?: string, asOf?: string}>} [options.pricing]
+   *   Replaces the built-in pricing table entirely (only these models are known).
    * @param {Record<string, {input: number, output: number, unit: number}>} [options.customPricing]
-   *   Additional or override model pricing entries.
+   *   Additional or override model pricing entries, merged over the base table.
    */
   constructor(options = {}) {
     super();
-    this.pricing = { ...PRICING, ...(options.customPricing || {}) };
+    const base = options.pricing ? {} : PRICING;
+    const overrides = {
+      ...(options.pricing || {}),
+      ...(options.customPricing || {}),
+    };
+    this.pricing = { ...base };
+    for (const [model, entry] of Object.entries(overrides)) {
+      this.pricing[model] = {
+        unit: 1000,
+        source: 'custom',
+        asOf: null,
+        ...entry,
+      };
+    }
+    /** @private Models already warned about as unknown */
+    this._warnedModels = new Set();
   }
 
   /**
    * Resolve a model key from the pricing table.
-   * Tries exact match (case-insensitive), then partial match.
+   * Tries an exact match (case-insensitive, '.' and '-' equivalent), then the
+   * longest known key contained in the model name (so dated snapshots such
+   * as "gpt-4o-mini-2024-07-18" resolve to "gpt-4o-mini", not "gpt-4"), then
+   * a known key that contains the model name.
    * @param {string} model
    * @returns {string|null} The matched pricing key or null.
    * @private
    */
   _resolveModel(model) {
-    const lower = model.toLowerCase();
+    if (model == null) return null;
+    const wanted = canonical(model);
     const keys = Object.keys(this.pricing);
 
-    // Exact case-insensitive match
-    const exact = keys.find((k) => k.toLowerCase() === lower);
+    const exact = keys.find((k) => canonical(k) === wanted);
     if (exact) return exact;
 
-    // Partial match — model string contains a known key or vice-versa
-    const partial = keys.find(
-      (k) => lower.includes(k.toLowerCase()) || k.toLowerCase().includes(lower),
-    );
-    return partial || null;
+    let best = null;
+    for (const k of keys) {
+      if (wanted.includes(canonical(k)) && (!best || k.length > best.length)) {
+        best = k;
+      }
+    }
+    if (best) return best;
+
+    const containing = keys.find((k) => canonical(k).includes(wanted));
+    return containing || null;
+  }
+
+  /**
+   * Emit a one-time warning for a model with no known price.
+   * Listeners on 'warning' receive `{ type: 'unknown-model', model, message }`;
+   * without a listener the warning goes to process.emitWarning.
+   * @param {string} model
+   * @private
+   */
+  _warnUnknownModel(model) {
+    const key = String(model);
+    if (this._warnedModels.has(key)) return;
+    this._warnedModels.add(key);
+    const message =
+      `No pricing known for model "${key}"; cost reported as 0. ` +
+      'Pass `pricing` or `customPricing` to CostCalculator to set a price.';
+    if (this.listenerCount('warning') > 0) {
+      this.emit('warning', { type: 'unknown-model', model: key, message });
+    } else {
+      process.emitWarning(message, { code: 'RAG_UNKNOWN_MODEL_PRICING' });
+    }
   }
 
   /**
@@ -70,12 +224,14 @@ class CostCalculator extends EventEmitter {
    * @param {number} inputTokens - Number of input / prompt tokens.
    * @param {number} [outputTokens=0] - Number of output / completion tokens.
    * @returns {{inputCost: number, outputCost: number, totalCost: number, model: string,
-   *            inputTokens: number, outputTokens: number, currency: string, warning?: string}}
+   *            inputTokens: number, outputTokens: number, currency: string,
+   *            pricingSource?: string, pricingAsOf?: string|null, warning?: string}}
    */
   estimate(model, inputTokens, outputTokens = 0) {
     const resolvedKey = this._resolveModel(model);
 
     if (!resolvedKey) {
+      this._warnUnknownModel(model);
       return {
         inputCost: 0,
         outputCost: 0,
@@ -89,8 +245,9 @@ class CostCalculator extends EventEmitter {
     }
 
     const price = this.pricing[resolvedKey];
-    const inputCost = (inputTokens / price.unit) * price.input;
-    const outputCost = (outputTokens / price.unit) * price.output;
+    const unit = price.unit || 1000;
+    const inputCost = (inputTokens / unit) * price.input;
+    const outputCost = (outputTokens / unit) * price.output;
     const totalCost = inputCost + outputCost;
 
     return {
@@ -101,6 +258,8 @@ class CostCalculator extends EventEmitter {
       inputTokens,
       outputTokens,
       currency: 'USD',
+      pricingSource: price.source || 'custom',
+      pricingAsOf: price.asOf ?? null,
     };
   }
 
@@ -108,7 +267,7 @@ class CostCalculator extends EventEmitter {
    * Register custom model pricing.
    *
    * @param {string} model - Model identifier.
-   * @param {{input: number, output: number, unit: number}} pricing - Pricing definition.
+   * @param {{input: number, output: number, unit?: number, source?: string, asOf?: string}} pricing - Pricing definition.
    */
   addModel(model, pricing) {
     if (
@@ -120,7 +279,13 @@ class CostCalculator extends EventEmitter {
         'Pricing must include numeric "input" and "output" fields',
       );
     }
-    this.pricing[model] = { unit: 1000, ...pricing };
+    this.pricing[model] = {
+      unit: 1000,
+      source: 'custom',
+      asOf: null,
+      ...pricing,
+    };
+    this._warnedModels.clear();
   }
 
   /**
@@ -135,12 +300,20 @@ class CostCalculator extends EventEmitter {
    * Retrieve pricing for a specific model.
    *
    * @param {string} model - Model identifier (supports fuzzy resolution).
-   * @returns {{input: number, output: number, unit: number}|null}
+   * @returns {{input: number, output: number, unit: number, source?: string, asOf?: string|null}|null}
    */
   getModelPricing(model) {
     const key = this._resolveModel(model);
     return key ? { ...this.pricing[key] } : null;
   }
+
+  /**
+   * Provenance of the built-in pricing table.
+   * @returns {typeof PRICING_METADATA}
+   */
+  static getPricingMetadata() {
+    return PRICING_METADATA;
+  }
 }
 
-module.exports = { CostCalculator, PRICING };
+module.exports = { CostCalculator, PRICING, PRICING_METADATA };