@devilsdev/rag-pipeline-utils 2.4.5 → 2.5.0

package/src/security/plugin-signature-verifier.js

@@ -3,6 +3,22 @@
  * Verifies Ed25519 digital signatures for plugin manifests to ensure authenticity
  * Enforces fail-closed verification for production security
  *
+ * What is signed: the canonical JSON of the manifest (keys sorted
+ * recursively, without `signature`, `signerId` and `signedAt`). To bind the
+ * plugin's code to the signature, put a `contentDigest` in the manifest
+ * before signing (see computeContentDigest). Because `contentDigest` is part
+ * of the signed manifest, changing any covered file invalidates it.
+ *
+ * Defaults (2.5.0):
+ * - The trust store starts EMPTY. Nothing verifies until a signer is added
+ *   with addTrustedSigner() or a trustedKeysPath file.
+ * - If a manifest declares `contentDigest`, the plugin content (files or
+ *   contents) MUST be supplied to verifyPluginSignature() and must match;
+ *   otherwise verification fails.
+ * - A manifest without `contentDigest` still verifies (manifest-only
+ *   signatures keep working); pass `requireContentDigest: true` to reject
+ *   manifests that do not bind the code.
+ *
  * @author DevilsDev Team
  * @since 2.1.8
  */
@@ -12,6 +28,132 @@ const fs = require('fs');
 const path = require('path');
 const { logger } = require('../utils/logger');
 
+const DIGEST_PREFIX = 'sha256-';
+const HEX_KEY_RE = /^[0-9a-fA-F]{64}$/;
+const INVALID_KEY_MESSAGE =
+  'Public key must be 64 hex characters (32 bytes), or a PEM/DER-encoded Ed25519 public key';
+
+/**
+ * Convert supported public key encodings to an Ed25519 KeyObject.
+ *
+ * Accepted: a KeyObject; a 64-character hex string or 32-byte Buffer (raw
+ * key); a PEM string ("-----BEGIN PUBLIC KEY-----"); a DER (SPKI) Buffer or
+ * base64 string.
+ *
+ * @param {crypto.KeyObject|Buffer|string} publicKey
+ * @returns {crypto.KeyObject}
+ */
+function toEd25519PublicKey(publicKey) {
+  let keyObject;
+  try {
+    if (publicKey instanceof crypto.KeyObject) {
+      keyObject =
+        publicKey.type === 'private'
+          ? crypto.createPublicKey(publicKey)
+          : publicKey;
+    } else if (Buffer.isBuffer(publicKey) && publicKey.length === 32) {
+      keyObject = rawToKeyObject(publicKey);
+    } else if (Buffer.isBuffer(publicKey)) {
+      keyObject = crypto.createPublicKey({
+        key: publicKey,
+        format: 'der',
+        type: 'spki',
+      });
+    } else if (typeof publicKey === 'string' && HEX_KEY_RE.test(publicKey)) {
+      keyObject = rawToKeyObject(Buffer.from(publicKey, 'hex'));
+    } else if (
+      typeof publicKey === 'string' &&
+      publicKey.includes('-----BEGIN')
+    ) {
+      keyObject = crypto.createPublicKey({ key: publicKey, format: 'pem' });
+    } else if (typeof publicKey === 'string' && publicKey.length > 0) {
+      keyObject = crypto.createPublicKey({
+        key: Buffer.from(publicKey, 'base64'),
+        format: 'der',
+        type: 'spki',
+      });
+    }
+  } catch {
+    keyObject = undefined;
+  }
+
+  if (
+    !keyObject ||
+    keyObject.type !== 'public' ||
+    keyObject.asymmetricKeyType !== 'ed25519'
+  ) {
+    throw new Error(INVALID_KEY_MESSAGE);
+  }
+  return keyObject;
+}
+
+function rawToKeyObject(raw) {
+  return crypto.createPublicKey({
+    key: { kty: 'OKP', crv: 'Ed25519', x: raw.toString('base64url') },
+    format: 'jwk',
+  });
+}
+
+/**
+ * Compute the content digest of a plugin.
+ *
+ * The digest is `sha256-<hex>` of the lines `<relativePath>\0<sha256 of
+ * file>\n`, sorted by relative path. Paths are relative to `baseDir` and use
+ * forward slashes, so the digest is identical on Windows and POSIX.
+ *
+ * @param {Object} options
+ * @param {string[]} [options.files] - File paths (absolute or relative to baseDir)
+ * @param {string} [options.baseDir] - Plugin root; files must be inside it
+ * @param {Object<string, Buffer|string>} [options.contents] - In-memory
+ *   files keyed by relative path (alternative or addition to `files`)
+ * @returns {string} `sha256-<hex>`
+ */
+function computeContentDigest({ files = [], baseDir, contents = {} } = {}) {
+  const root = path.resolve(baseDir || process.cwd());
+  const entries = new Map();
+
+  for (const file of files) {
+    const abs = path.resolve(root, file);
+    const rel = path.relative(root, abs);
+    if (
+      !rel ||
+      rel === '..' ||
+      rel.startsWith(`..${path.sep}`) ||
+      path.isAbsolute(rel)
+    ) {
+      throw new Error(`Plugin file is outside the plugin directory: ${file}`);
+    }
+    entries.set(rel.split(path.sep).join('/'), fs.readFileSync(abs));
+  }
+
+  for (const [name, value] of Object.entries(contents)) {
+    const rel = path.posix.normalize(String(name).split('\\').join('/'));
+    if (
+      rel === '..' ||
+      rel.startsWith('../') ||
+      rel === '.' ||
+      path.posix.isAbsolute(rel)
+    ) {
+      throw new Error(`Plugin content path is not relative: ${name}`);
+    }
+    entries.set(rel, Buffer.isBuffer(value) ? value : Buffer.from(value));
+  }
+
+  if (entries.size === 0) {
+    throw new Error('No plugin content supplied for digest');
+  }
+
+  const hash = crypto.createHash('sha256');
+  for (const rel of [...entries.keys()].sort()) {
+    const fileHash = crypto
+      .createHash('sha256')
+      .update(entries.get(rel))
+      .digest('hex');
+    hash.update(`${rel}\0${fileHash}\n`, 'utf8');
+  }
+  return DIGEST_PREFIX + hash.digest('hex');
+}
+
 /**
  * Plugin signature verification with Ed25519
  */
@@ -22,23 +164,20 @@ class PluginSignatureVerifier {
     this.auditLogger =
       options.auditLogger || this._defaultAuditLogger.bind(this);
     this.failClosed = options.failClosed !== false; // Default to fail-closed
+    // Opt-in: reject manifests that do not bind the plugin code
+    this.requireContentDigest = options.requireContentDigest === true;
 
     // Load trusted public keys
     this._loadTrustedKeys(options.trustedKeysPath);
   }
 
   /**
-   * Load trusted public keys from configuration
+   * Load trusted public keys from configuration.
+   * Without a keys file the trust store stays empty (no built-in keys).
    * @param {string} keysPath - Path to trusted keys file
    */
   _loadTrustedKeys(keysPath) {
     if (!keysPath) {
-      // Default trusted keys for the DevilsDev organization
-      this.trustStore.set('devilsdev-official', {
-        publicKey: 'your-ed25519-public-key-here',
-        name: 'DevilsDev Official Plugins',
-        verified: true,
-      });
       return;
     }
 
@@ -49,6 +188,8 @@ class PluginSignatureVerifier {
 
         for (const [keyId, keyInfo] of Object.entries(trustedKeys)) {
           if (keyInfo.publicKey && keyInfo.name) {
+            // Reject unusable keys up front instead of at verification time
+            toEd25519PublicKey(keyInfo.publicKey);
             this.trustStore.set(keyId, keyInfo);
           }
         }
@@ -57,6 +198,9 @@ class PluginSignatureVerifier {
       if (this.failClosed) {
         throw new Error(`Failed to load trusted keys: ${error.message}`);
       }
+      logger.warn(
+        `[PLUGIN_SIGNATURE] Failed to load trusted keys: ${error.message}`,
+      );
     }
   }
 
@@ -65,11 +209,15 @@ class PluginSignatureVerifier {
    * @param {object} manifest - Plugin manifest
    * @param {string} signature - Base64 encoded signature
    * @param {string} signerId - Signer identifier
+   * @param {object} [content] - Plugin content to check against
+   *   manifest.contentDigest: `{ files, baseDir, contents }` (see
+   *   computeContentDigest)
    * @returns {Promise<object>} Verification result
    */
-  async verifyPluginSignature(manifest, signature, signerId) {
+  async verifyPluginSignature(manifest, signature, signerId, content = null) {
     const verificationResult = {
       verified: false,
+      contentVerified: false,
       signerId,
       timestamp: new Date().toISOString(),
       error: null,
@@ -115,15 +263,8 @@ class PluginSignatureVerifier {
         'utf8',
       );
 
-      // Decode signature
-      let signatureBuffer;
-      try {
-        signatureBuffer = Buffer.from(signature, 'base64');
-      } catch (error) {
-        throw new Error(`Invalid signature encoding: ${error.message}`);
-      }
+      const signatureBuffer = Buffer.from(signature, 'base64');
 
-      // Verify signature using Web Crypto API or Node.js crypto
       const isValid = await this._verifyEd25519Signature(
         manifestData,
         signatureBuffer,
@@ -134,14 +275,22 @@ class PluginSignatureVerifier {
         throw new Error('Signature verification failed');
       }
 
+      // The signature covers contentDigest; now check the code matches it.
+      verificationResult.contentVerified = this._verifyContentDigest(
+        canonicalManifest,
+        content,
+      );
+
       verificationResult.verified = true;
       verificationResult.auditTrail.push({
         action: 'signature_verified',
         signerId,
         timestamp: new Date().toISOString(),
         manifestHash: this._hashManifest(canonicalManifest),
+        contentVerified: verificationResult.contentVerified,
       });
     } catch (error) {
+      verificationResult.verified = false;
       verificationResult.error = error.message;
       verificationResult.auditTrail.push({
         action: 'verification_failed',
@@ -163,58 +312,66 @@ class PluginSignatureVerifier {
   }
 
   /**
-   * Verify Ed25519 signature
-   * @param {Buffer} data - Data that was signed
-   * @param {Buffer} signature - Signature bytes
-   * @param {string} publicKeyHex - Public key in hex format
-   * @returns {Promise<boolean>} Verification result
+   * Check the supplied plugin content against the signed contentDigest.
+   * @param {object} canonicalManifest - Signed manifest
+   * @param {object|null} content - `{ files, baseDir, contents }`
+   * @returns {boolean} true when the content digest was checked and matched
+   * @private
    */
-  async _verifyEd25519Signature(data, signature, publicKeyHex) {
-    try {
-      // Convert hex public key to buffer
-      const publicKeyBuffer = Buffer.from(publicKeyHex, 'hex');
-
-      // Use Node.js crypto for Ed25519 verification
-      const verify = crypto.createVerify('Ed25519');
-      verify.update(data);
+  _verifyContentDigest(canonicalManifest, content) {
+    const declared = canonicalManifest.contentDigest;
+    const hasContent =
+      content &&
+      ((Array.isArray(content.files) && content.files.length > 0) ||
+        (content.contents && Object.keys(content.contents).length > 0));
+
+    if (declared === undefined || declared === null) {
+      if (this.requireContentDigest) {
+        throw new Error(
+          'Manifest has no contentDigest; the signature does not cover the plugin code',
+        );
+      }
+      return false;
+    }
 
-      // Create public key object
-      const publicKeyObject = crypto.createPublicKey({
-        key: publicKeyBuffer,
-        format: 'der',
-        type: 'spki',
-      });
+    if (typeof declared !== 'string' || !declared.startsWith(DIGEST_PREFIX)) {
+      throw new Error('Invalid contentDigest: expected "sha256-<hex>"');
+    }
 
-      return verify.verify(publicKeyObject, signature);
-    } catch (error) {
-      // Fallback to Web Crypto API if available
-      if (typeof crypto.subtle !== 'undefined') {
-        try {
-          const publicKeyBuffer = Buffer.from(publicKeyHex, 'hex');
-          const publicKey = await crypto.subtle.importKey(
-            'raw',
-            publicKeyBuffer,
-            {
-              name: 'Ed25519',
-              namedCurve: 'Ed25519',
-            },
-            false,
-            ['verify'],
-          );
+    if (!hasContent) {
+      throw new Error(
+        'Manifest declares a contentDigest but no plugin content was supplied to verify it',
+      );
+    }
 
-          return await crypto.subtle.verify(
-            'Ed25519',
-            publicKey,
-            signature,
-            data,
-          );
-        } catch (webCryptoError) {
-          throw new Error(`Ed25519 verification failed: ${error.message}`);
-        }
-      }
+    const actual = Buffer.from(computeContentDigest(content), 'utf8');
+    const expected = Buffer.from(declared.toLowerCase(), 'utf8');
+    if (
+      actual.length !== expected.length ||
+      !crypto.timingSafeEqual(actual, expected)
+    ) {
+      throw new Error(
+        'Plugin content digest mismatch: plugin code does not match the signed manifest',
+      );
+    }
+    return true;
+  }
 
-      throw new Error(`Ed25519 verification failed: ${error.message}`);
+  /**
+   * Verify Ed25519 signature
+   * @param {Buffer} data - Data that was signed
+   * @param {Buffer} signature - Signature bytes
+   * @param {crypto.KeyObject|Buffer|string} publicKey - Ed25519 public key
+   *   (raw hex, PEM, DER or KeyObject)
+   * @returns {Promise<boolean>} Verification result
+   */
+  async _verifyEd25519Signature(data, signature, publicKey) {
+    const keyObject = toEd25519PublicKey(publicKey);
+    if (!Buffer.isBuffer(signature) || signature.length !== 64) {
+      return false;
     }
+    // Ed25519 takes no digest algorithm: pass null.
+    return crypto.verify(null, data, keyObject, signature);
   }
 
   /**
@@ -233,6 +390,18 @@ class PluginSignatureVerifier {
     return this._sortObjectKeys(canonical);
   }
 
+  /**
+   * Bytes that a signer must sign for a manifest (canonical JSON).
+   * @param {object} manifest - Plugin manifest
+   * @returns {Buffer}
+   */
+  getSigningPayload(manifest) {
+    return Buffer.from(
+      JSON.stringify(this._canonicalizeManifest(manifest)),
+      'utf8',
+    );
+  }
+
   /**
    * Sort object keys recursively
    * @param {any} obj - Object to sort
@@ -276,10 +445,11 @@ class PluginSignatureVerifier {
     const auditEntry = {
       timestamp: result.timestamp,
       action: 'plugin_signature_verification',
-      pluginId: manifest.name || 'unknown',
-      pluginVersion: manifest.version || 'unknown',
+      pluginId: (manifest && manifest.name) || 'unknown',
+      pluginVersion: (manifest && manifest.version) || 'unknown',
       signerId: result.signerId,
       verified: result.verified,
+      contentVerified: result.contentVerified,
       error: result.error,
       auditTrail: result.auditTrail,
     };
@@ -309,7 +479,8 @@ class PluginSignatureVerifier {
   /**
    * Add trusted signer
    * @param {string} signerId - Signer identifier
-   * @param {string} publicKey - Ed25519 public key in hex
+   * @param {string|Buffer|crypto.KeyObject} publicKey - Ed25519 public key:
+   *   64 hex characters (raw 32 bytes), PEM, DER (SPKI) or a KeyObject
    * @param {string} name - Human-readable name
    * @param {object} metadata - Additional metadata
    */
@@ -318,13 +489,15 @@ class PluginSignatureVerifier {
       throw new Error('signerId, publicKey, and name are required');
     }
 
-    // Validate public key format
-    if (!/^[0-9a-fA-F]{64}$/.test(publicKey)) {
-      throw new Error('Public key must be 64 hex characters (32 bytes)');
-    }
+    // Validate the key is a usable Ed25519 public key
+    const keyObject = toEd25519PublicKey(publicKey);
+    const storedKey =
+      typeof publicKey === 'string' && HEX_KEY_RE.test(publicKey)
+        ? publicKey.toLowerCase()
+        : keyObject.export({ format: 'pem', type: 'spki' });
 
     this.trustStore.set(signerId, {
-      publicKey: publicKey.toLowerCase(),
+      publicKey: storedKey,
       name,
       verified: metadata.verified || false,
       addedAt: new Date().toISOString(),
@@ -409,4 +582,6 @@ function createPluginVerifier(options = {}) {
 module.exports = {
   PluginSignatureVerifier,
   createPluginVerifier,
+  computeContentDigest,
+  toEd25519PublicKey,
 };

package/src/utils/deprecate.js

@@ -0,0 +1,69 @@
+'use strict';
+
+/**
+ * Deprecation wrappers for public exports (see SEMVER.md, "Deprecation
+ * process"): the export keeps working and warns once per process when used.
+ */
+
+const wrapped = new WeakMap();
+const warned = new Set();
+
+function warnOnce(code, message) {
+  if (warned.has(code)) return;
+  warned.add(code);
+  process.emitWarning(message, { type: 'DeprecationWarning', code });
+}
+
+/**
+ * Wrap a class so constructing it (or calling a static method) emits a
+ * one-time DeprecationWarning. A Proxy keeps the original's identity for
+ * prototype, name, statics, `extends` and `instanceof`, so instances created
+ * inside the library still satisfy `instanceof Wrapped`. The same class
+ * always gets the same wrapper, so aliases stay identical.
+ *
+ * @param {Function} Class - Class to wrap
+ * @param {object} info
+ * @param {string} info.name - Export name, used in the message and code
+ * @param {string} info.message - Full warning text
+ * @returns {Function} The wrapped class
+ */
+function deprecateClass(Class, { name, message }) {
+  if (wrapped.has(Class)) return wrapped.get(Class);
+  // SSOManager -> RAG_DEP_SSO_MANAGER, OAuth2Provider -> RAG_DEP_O_AUTH2_PROVIDER
+  const snake = name
+    .replace(/([a-z0-9])([A-Z])/g, '$1_$2')
+    .replace(/([A-Z]+)([A-Z][a-z])/g, '$1_$2')
+    .toUpperCase();
+  const code = `RAG_DEP_${snake}`;
+  const proxy = new Proxy(Class, {
+    construct(target, args, newTarget) {
+      warnOnce(code, message);
+      return Reflect.construct(
+        target,
+        args,
+        newTarget === proxy ? target : newTarget,
+      );
+    },
+    get(target, property, receiver) {
+      const value = Reflect.get(target, property, receiver);
+      // Only the class's own static methods; inherited Function.prototype
+      // members (call, bind, toString, Symbol.hasInstance for instanceof)
+      // must not warn.
+      const ownStatic =
+        typeof value === 'function' &&
+        property !== 'prototype' &&
+        Object.prototype.hasOwnProperty.call(target, property);
+      if (ownStatic) {
+        return function deprecatedStatic(...args) {
+          warnOnce(code, message);
+          return value.apply(this === proxy ? target : this, args);
+        };
+      }
+      return value;
+    },
+  });
+  wrapped.set(Class, proxy);
+  return proxy;
+}
+
+module.exports = { deprecateClass };

package/src/utils/embed-query.js

@@ -0,0 +1,33 @@
+'use strict';
+
+/**
+ * Embed a single query string with any supported embedder shape.
+ *
+ * Prefers embedQuery(); falls back to embed(). Accepts the common return
+ * shapes: a vector, a list of vectors, or [{ values | vector | embedding }].
+ *
+ * @param {object} embedder - Object with embedQuery() or embed()
+ * @param {string} query - Text to embed
+ * @param {{ signal?: AbortSignal }} [options] - Passed to the embedder as its
+ *   second argument (only when given, so embedders keep their usual arity)
+ * @returns {Promise<number[]|undefined>} The query vector
+ */
+async function embedQuery(embedder, query, options) {
+  const args = options ? [query, options] : [query];
+  const raw = embedder.embedQuery
+    ? await embedder.embedQuery(...args)
+    : await embedder.embed(...args);
+  const first = Array.isArray(raw) && typeof raw[0] !== 'number' ? raw[0] : raw;
+  if (Array.isArray(first)) return first;
+  return first?.values || first?.vector || first?.embedding;
+}
+
+/**
+ * @param {object} [embedder]
+ * @returns {boolean} Whether embedQuery() can use this embedder
+ */
+function canEmbed(embedder) {
+  return Boolean(embedder && (embedder.embedQuery || embedder.embed));
+}
+
+module.exports = { embedQuery, canEmbed };

package/src/utils/input-sanitizer.js

@@ -65,6 +65,21 @@ const ValidationPatterns = {
   hex: /^[0-9a-fA-F]+$/,
 };
 
+/**
+ * Test a shared pattern from start of string. The DangerousPatterns regexes
+ * carry the /g flag (needed by replace()), and RegExp#test() on a /g regex
+ * resumes from lastIndex left by the previous call, which made a match
+ * against one input skip part of the next and let payloads through.
+ *
+ * @param {RegExp} pattern
+ * @param {string} str
+ * @returns {boolean}
+ */
+function matchesPattern(pattern, str) {
+  pattern.lastIndex = 0;
+  return pattern.test(str);
+}
+
 /**
  * Dangerous patterns that indicate attacks (compiled for performance)
  */
@@ -476,7 +491,7 @@ class InputSanitizer {
 
     // Apply all XSS patterns
     for (const pattern of DangerousPatterns.xss) {
-      if (pattern.test(sanitized)) {
+      if (matchesPattern(pattern, sanitized)) {
         this.stats.blocked++;
         sanitized = sanitized.replace(pattern, '');
       }
@@ -497,7 +512,7 @@ class InputSanitizer {
 
     // Check for SQL injection patterns
     for (const pattern of DangerousPatterns.sql) {
-      if (pattern.test(sanitized)) {
+      if (matchesPattern(pattern, sanitized)) {
         this.stats.blocked++;
         if (this.options.throwOnInvalid) {
           throw new Error('Potential SQL injection detected');
@@ -520,7 +535,7 @@ class InputSanitizer {
   _sanitizeCommand(str) {
     // Check for command injection patterns
     for (const pattern of DangerousPatterns.command) {
-      if (pattern.test(str)) {
+      if (matchesPattern(pattern, str)) {
         this.stats.blocked++;
         if (this.options.throwOnInvalid) {
           throw new Error('Potential command injection detected');
@@ -560,7 +575,7 @@ class InputSanitizer {
 
     // Check for path traversal patterns in both original and fully decoded versions
     for (const pattern of DangerousPatterns.pathTraversal) {
-      if (pattern.test(str) || pattern.test(current)) {
+      if (matchesPattern(pattern, str) || matchesPattern(pattern, current)) {
         this.stats.blocked++;
         // SECURITY: Always throw on path traversal - it should never be silently sanitized
         // Empty paths could resolve to application root, creating security vulnerabilities

package/src/utils/logger.js

@@ -31,9 +31,20 @@ const getLogLevel = () => {
 // Determine if pretty printing should be enabled
 const shouldUsePrettyPrint = () => {
   // Enable pretty print in development or when explicitly requested
-  return (
-    process.env.NODE_ENV === 'development' || process.env.LOG_PRETTY === 'true'
-  );
+  const requested =
+    process.env.NODE_ENV === 'development' || process.env.LOG_PRETTY === 'true';
+  return requested && isPinoPrettyAvailable();
+};
+
+// pino-pretty is an optional peer dependency; pino throws at startup if the
+// transport target cannot be resolved, so fall back to JSON output instead.
+const isPinoPrettyAvailable = () => {
+  try {
+    require.resolve('pino-pretty');
+    return true;
+  } catch {
+    return false;
+  }
 };
 
 /**
@@ -68,6 +79,25 @@ const baseLogger = pino({
   }),
 });
 
+/**
+ * Coerce the meta argument into an object. Callers often pass
+ * console-style extras such as `logger.error("Failed:", error.message)`;
+ * spreading a string would log one key per character.
+ *
+ * @param {*} meta
+ * @returns {object}
+ */
+function normalizeMeta(meta) {
+  if (meta == null) return {};
+  if (meta instanceof Error) {
+    return { error: meta.message, stack: meta.stack };
+  }
+  if (typeof meta !== 'object' || Array.isArray(meta)) {
+    return { detail: meta };
+  }
+  return meta;
+}
+
 /**
  * Enhanced logger with correlation ID support and structured context
  */
@@ -118,7 +148,7 @@ class StructuredLogger {
       ...this.context,
       correlationId: this.correlationId || crypto.randomUUID(),
       timestamp: new Date().toISOString(),
-      ...meta,
+      ...normalizeMeta(meta),
     };
 
     this.logger[level](logData, message);