@devilsdev/rag-pipeline-utils 2.4.5 → 2.5.0

package/src/ecosystem/plugin-scanner.js

@@ -0,0 +1,185 @@
+'use strict';
+
+/**
+ * Static security scan of a plugin's files. It reads source text only and
+ * never executes plugin code. It finds capabilities a plugin uses (running
+ * processes, evaluating strings as code, network and filesystem writes) so a
+ * user can decide whether to trust it; it is not a sandbox or a malware
+ * detector, and obfuscated code can evade it.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { relativeWithin } = require('./tar-extract');
+
+const MAX_FILE_BYTES = 2 * 1024 * 1024;
+const SOURCE_EXTENSIONS = new Set(['.js', '.cjs', '.mjs']);
+
+/** severity: high blocks installation unless explicitly allowed */
+const RULES = [
+  {
+    id: 'eval',
+    severity: 'high',
+    pattern: /\beval\s*\(/,
+    message: 'Evaluates strings as code (eval)',
+  },
+  {
+    id: 'new-function',
+    severity: 'high',
+    pattern: /\bnew\s+Function\s*\(/,
+    message: 'Evaluates strings as code (new Function)',
+  },
+  {
+    id: 'child-process',
+    severity: 'high',
+    pattern: /(?:require\s*\(\s*|from\s+)['"](?:node:)?child_process['"]/,
+    message: 'Runs external processes (child_process)',
+  },
+  {
+    id: 'vm',
+    severity: 'high',
+    pattern: /(?:require\s*\(\s*|from\s+)['"](?:node:)?vm['"]/,
+    message: 'Compiles code at runtime (vm)',
+  },
+  {
+    id: 'process-binding',
+    severity: 'high',
+    pattern: /process\.binding\s*\(/,
+    message: 'Uses internal Node bindings (process.binding)',
+  },
+  {
+    id: 'dynamic-require',
+    severity: 'medium',
+    pattern: /\brequire\s*\(\s*[^'"`\s)]/,
+    message: 'Loads modules from a computed path',
+  },
+  {
+    id: 'network',
+    severity: 'medium',
+    pattern:
+      /(?:require\s*\(\s*|from\s+)['"](?:node:)?(?:http|https|net|tls|dgram)['"]|\bfetch\s*\(|['"]axios['"]/,
+    message: 'Makes network requests',
+  },
+  {
+    id: 'fs-write',
+    severity: 'medium',
+    pattern:
+      /\b(?:writeFile|appendFile|unlink|rmdir|rm|rename|chmod|createWriteStream)(?:Sync)?\s*\(/,
+    message: 'Writes, moves or deletes files',
+  },
+  {
+    id: 'env',
+    severity: 'low',
+    pattern: /process\.env\b/,
+    message: 'Reads environment variables (may include API keys)',
+  },
+];
+
+function listFiles(dir) {
+  const files = [];
+  const walk = (current) => {
+    for (const entry of fs.readdirSync(current, { withFileTypes: true })) {
+      const full = path.join(current, entry.name);
+      if (entry.isSymbolicLink()) continue;
+      if (entry.isDirectory()) {
+        if (entry.name !== '.git') walk(full);
+      } else if (entry.isFile()) {
+        files.push(full);
+      }
+    }
+  };
+  walk(dir);
+  return files;
+}
+
+/**
+ * @param {string} pluginPath - Plugin directory or single .js file
+ * @returns {{
+ *   findings: Array<{severity: string, rule: string, message: string, file: string, line?: number}>,
+ *   counts: {high: number, medium: number, low: number},
+ *   filesScanned: number
+ * }}
+ */
+function scanPlugin(pluginPath) {
+  const root = path.resolve(pluginPath);
+  const isDir = fs.statSync(root).isDirectory();
+  const baseDir = isDir ? root : path.dirname(root);
+  const files = isDir ? listFiles(root) : [root];
+  const findings = [];
+  let filesScanned = 0;
+
+  const add = (finding) =>
+    findings.push({
+      ...finding,
+      file:
+        relativeWithin(baseDir, finding.file) || path.basename(finding.file),
+    });
+
+  for (const file of files) {
+    const ext = path.extname(file).toLowerCase();
+    if (ext === '.node') {
+      add({
+        severity: 'high',
+        rule: 'native-addon',
+        message: 'Contains a compiled native addon (.node)',
+        file,
+      });
+      continue;
+    }
+    if (!SOURCE_EXTENSIONS.has(ext)) continue;
+    if (fs.statSync(file).size > MAX_FILE_BYTES) {
+      add({
+        severity: 'medium',
+        rule: 'large-file',
+        message: 'Source file over 2 MB was not scanned',
+        file,
+      });
+      continue;
+    }
+    filesScanned++;
+    const lines = fs.readFileSync(file, 'utf8').split(/\r?\n/);
+    for (const rule of RULES) {
+      const index = lines.findIndex((line) => rule.pattern.test(line));
+      if (index !== -1) {
+        add({
+          severity: rule.severity,
+          rule: rule.id,
+          message: rule.message,
+          file,
+          line: index + 1,
+        });
+      }
+    }
+  }
+
+  const manifest = path.join(baseDir, 'package.json');
+  if (isDir && fs.existsSync(manifest)) {
+    try {
+      const scripts =
+        JSON.parse(fs.readFileSync(manifest, 'utf8')).scripts || {};
+      for (const hook of ['preinstall', 'install', 'postinstall']) {
+        if (scripts[hook]) {
+          add({
+            severity: 'medium',
+            rule: 'install-script',
+            message: `Declares a "${hook}" script (rag-pipeline never runs it)`,
+            file: manifest,
+          });
+        }
+      }
+    } catch {
+      add({
+        severity: 'medium',
+        rule: 'bad-manifest',
+        message: 'package.json is not valid JSON',
+        file: manifest,
+      });
+    }
+  }
+
+  const counts = { high: 0, medium: 0, low: 0 };
+  for (const finding of findings) counts[finding.severity]++;
+  return { findings, counts, filesScanned };
+}
+
+module.exports = { scanPlugin, RULES };

package/src/ecosystem/tar-extract.js

@@ -0,0 +1,114 @@
+'use strict';
+
+/**
+ * Minimal .tgz extractor for plugin packages (npm pack format). Extracts
+ * regular files and directories only; symlinks, hard links and devices are
+ * skipped, and every path must stay inside the destination directory.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const zlib = require('zlib');
+
+const BLOCK = 512;
+
+/** Separator-agnostic "child is root or inside it" (paths already resolved). */
+function isWithin(root, child) {
+  const a = root.replace(/\\/g, '/').replace(/\/$/, '');
+  const b = child.replace(/\\/g, '/');
+  return b === a || b.startsWith(`${a}/`);
+}
+
+function readString(buffer, start, length) {
+  const slice = buffer.subarray(start, start + length);
+  const end = slice.indexOf(0);
+  return slice.subarray(0, end === -1 ? slice.length : end).toString('utf8');
+}
+
+function readOctal(buffer, start, length) {
+  const text = readString(buffer, start, length).trim();
+  return text ? parseInt(text, 8) : 0;
+}
+
+/** Parse "<len> path=<value>\n" records from a pax extended header. */
+function parsePaxPath(data) {
+  const match = data.toString('utf8').match(/\d+ path=([^\n]*)\n/);
+  return match ? match[1] : null;
+}
+
+/**
+ * Extract a gzipped tarball into destDir.
+ *
+ * @param {Buffer} archive - .tgz contents
+ * @param {string} destDir - Target directory (created if missing)
+ * @param {object} [options]
+ * @param {number} [options.strip=0] - Leading path components to drop (npm packs use "package/")
+ * @param {number} [options.maxBytes=50MB] - Refuse archives that expand beyond this
+ * @returns {{files: string[], skipped: string[]}}
+ */
+function extractTarGz(archive, destDir, options = {}) {
+  const { strip = 0, maxBytes = 50 * 1024 * 1024 } = options;
+  const tar = zlib.gunzipSync(archive, { maxOutputLength: maxBytes });
+  const root = path.resolve(destDir);
+  fs.mkdirSync(root, { recursive: true });
+
+  const files = [];
+  const skipped = [];
+  let offset = 0;
+  let paxPath = null;
+
+  while (offset + BLOCK <= tar.length) {
+    const header = tar.subarray(offset, offset + BLOCK);
+    if (header.every((byte) => byte === 0)) break; // end-of-archive marker
+
+    const size = readOctal(header, 124, 12);
+    const type = String.fromCharCode(header[156] || 48); // '\0' means file
+    const prefix = readString(header, 345, 155);
+    const headerName = readString(header, 0, 100);
+    const name = paxPath || (prefix ? `${prefix}/${headerName}` : headerName);
+    const dataStart = offset + BLOCK;
+    const data = tar.subarray(dataStart, dataStart + size);
+    offset = dataStart + Math.ceil(size / BLOCK) * BLOCK;
+
+    if (type === 'x') {
+      paxPath = parsePaxPath(data);
+      continue;
+    }
+    paxPath = null;
+    if (type === 'g') continue;
+
+    const parts = name.split('/').filter((p) => p && p !== '.');
+    const relative = parts.slice(strip).join('/');
+    if (!relative) continue;
+
+    const target = path.resolve(root, relative);
+    if (!isWithin(root, target)) {
+      throw new Error(`Archive entry escapes the destination: ${name}`);
+    }
+
+    if (type === '5') {
+      fs.mkdirSync(target, { recursive: true });
+    } else if (type === '0' || type === '\0' || type === '7') {
+      fs.mkdirSync(path.dirname(target), { recursive: true });
+      fs.writeFileSync(target, data);
+      files.push(relative);
+    } else {
+      skipped.push(relative); // links, devices, fifos
+    }
+  }
+  return { files, skipped };
+}
+
+module.exports = { extractTarGz, isWithin };
+
+/**
+ * Path of `child` relative to `root` with "/" separators, or null when it is
+ * not inside root. Separator-agnostic like isWithin.
+ */
+function relativeWithin(root, child) {
+  if (!isWithin(root, child)) return null;
+  const a = root.replace(/\\/g, '/').replace(/\/$/, '');
+  return child.replace(/\\/g, '/').slice(a.length + 1);
+}
+
+module.exports.relativeWithin = relativeWithin;

package/src/enterprise/audit-logging.js

@@ -13,6 +13,42 @@ const { EventEmitter } = require('events');
 // eslint-disable-line global-require
 const { logger } = require('../utils/logger');
 
+const INTEGRITY_VERSION = 2;
+const ENCRYPTION_VERSION = 2;
+const ENCRYPTION_KEY_ENV = 'RAG_AUDIT_ENCRYPTION_KEY';
+
+/**
+ * OpenSSL EVP_BytesToKey with MD5, one iteration and no salt: the key/IV
+ * derivation used by the removed crypto.createCipher/createDecipher.
+ * D_1 = MD5(pass), D_i = MD5(D_{i-1} || pass); key || iv = D_1 || D_2 || ...
+ *
+ * @param {Buffer|string} password - Passphrase given to createCipher
+ * @param {number} keyLen - Key length in bytes
+ * @param {number} ivLen - IV length in bytes (12 for GCM)
+ * @returns {{key: Buffer, iv: Buffer}}
+ */
+function evpBytesToKey(password, keyLen, ivLen) {
+  const pass = Buffer.isBuffer(password)
+    ? password
+    : Buffer.from(String(password), 'utf8');
+  const blocks = [];
+  let total = 0;
+  let prev = Buffer.alloc(0);
+  while (total < keyLen + ivLen) {
+    prev = crypto
+      .createHash('md5')
+      .update(Buffer.concat([prev, pass]))
+      .digest();
+    blocks.push(prev);
+    total += prev.length;
+  }
+  const out = Buffer.concat(blocks);
+  return {
+    key: out.subarray(0, keyLen),
+    iv: out.subarray(keyLen, keyLen + ivLen),
+  };
+}
+
 class AuditLogger extends EventEmitter {
   constructor(_options = {}) {
     super();
@@ -53,8 +89,7 @@ class AuditLogger extends EventEmitter {
     };
 
     this.logBuffer = [];
-    this.encryptionKey =
-      _options.encryptionKey || this._generateEncryptionKey();
+    this.encryptionKey = this._resolveEncryptionKey(_options);
     this.signingKey = _options.signingKey || this._generateSigningKey();
     this.sequenceNumber = 0;
 
@@ -355,6 +390,7 @@ class AuditLogger extends EventEmitter {
   async _writeAuditLog(auditEvent) {
     // Add sequence number for integrity chain
     auditEvent.sequence = ++this.sequenceNumber;
+    auditEvent.integrityVersion = INTEGRITY_VERSION;
 
     // Calculate integrity hash
     auditEvent.integrity = await this._calculateIntegrityHash(auditEvent);
@@ -384,9 +420,9 @@ class AuditLogger extends EventEmitter {
 
     this.emit('audit_logged', {
       _category: auditEvent._category,
-      _action: auditEvent._action,
+      _action: auditEvent.action,
       _severity: auditEvent.metadata._severity,
-      _correlationId: auditEvent.metadata._correlationId,
+      _correlationId: auditEvent.metadata.correlationId,
     });
 
     return auditEvent.metadata.correlationId;
@@ -490,12 +526,12 @@ class AuditLogger extends EventEmitter {
    */
   async generateComplianceReport(_options) {
     const {
-      _tenantId,
       standard, // GDPR, HIPAA, SOC2, PCI-DSS
       startDate,
       endDate,
       includeEvidence = false,
     } = _options;
+    const _tenantId = _options.tenantId ?? _options._tenantId;
 
     const query = {
       _tenantId,
@@ -566,7 +602,8 @@ class AuditLogger extends EventEmitter {
    * Verify log integrity
    */
   async verifyIntegrity(_options = {}) {
-    const { startDate, endDate, _tenantId } = _options;
+    const { startDate, endDate } = _options;
+    const _tenantId = _options.tenantId ?? _options._tenantId;
 
     const integrityFile = path.join(
       this._config.logDir,
@@ -604,7 +641,7 @@ class AuditLogger extends EventEmitter {
         const expectedHash = await this._calculateIntegrityHash(log);
         if (log.integrity !== expectedHash) {
           verification.tamperedLogs.push({
-            _correlationId: log.metadata._correlationId,
+            _correlationId: log.metadata.correlationId,
             sequence: log.sequence,
             file: file,
           });
@@ -616,7 +653,7 @@ class AuditLogger extends EventEmitter {
         // Verify digital signature if present
         if (log.signature && !(await this._verifySignature(log))) {
           verification.tamperedLogs.push({
-            _correlationId: log.metadata._correlationId,
+            _correlationId: log.metadata.correlationId,
             reason: 'invalid_signature',
             file: file,
           });
@@ -630,11 +667,14 @@ class AuditLogger extends EventEmitter {
 
   // Private helper methods
   async _calculateIntegrityHash(auditEvent) {
+    // Records written before integrityVersion 2 hashed the non-existent
+    // _action/_tenantId fields (always undefined); keep verifying them as-is.
+    const isV2 = auditEvent.integrityVersion === INTEGRITY_VERSION;
     const data = {
       sequence: auditEvent.sequence,
       category: auditEvent._category,
-      action: auditEvent._action,
-      tenantId: auditEvent._tenantId,
+      action: isV2 ? auditEvent.action : auditEvent._action,
+      tenantId: isV2 ? auditEvent.tenantId : auditEvent._tenantId,
       timestamp: auditEvent.metadata.timestamp,
     };
 
@@ -649,7 +689,13 @@ class AuditLogger extends EventEmitter {
 
     for (const field of sensitiveFields) {
       if (auditEvent[field]) {
-        const cipher = crypto.createCipher('aes-256-gcm', this.encryptionKey);
+        // Fresh 96-bit IV per field: GCM must never reuse a (key, IV) pair.
+        const iv = crypto.randomBytes(12);
+        const cipher = crypto.createCipheriv(
+          'aes-256-gcm',
+          this._getCipherKey(),
+          iv,
+        );
         let encrypted = cipher.update(
           JSON.stringify(auditEvent[field]),
           'utf8',
@@ -659,6 +705,8 @@ class AuditLogger extends EventEmitter {
 
         auditEvent[field] = {
           encrypted: true,
+          v: ENCRYPTION_VERSION,
+          iv: iv.toString('hex'),
           data: encrypted,
           authTag: cipher.getAuthTag().toString('hex'),
         };
@@ -674,10 +722,7 @@ class AuditLogger extends EventEmitter {
     for (const field of sensitiveFields) {
       if (auditEvent[field] && auditEvent[field].encrypted) {
         try {
-          const decipher = crypto.createDecipher(
-            'aes-256-gcm',
-            this.encryptionKey,
-          );
+          const decipher = this._createDecipher(auditEvent[field]);
           decipher.setAuthTag(Buffer.from(auditEvent[field].authTag, 'hex'));
 
           let decrypted = decipher.update(
@@ -772,8 +817,83 @@ class AuditLogger extends EventEmitter {
       .substring(0, 16);
   }
 
-  _generateEncryptionKey() {
-    return crypto.randomBytes(32);
+  /**
+   * Resolve the field-encryption key.
+   *
+   * Order: the `encryptionKey` option, then the RAG_AUDIT_ENCRYPTION_KEY
+   * environment variable (used as a passphrase, like a string option). A
+   * random per-process key is never generated: records written with it
+   * could not be decrypted after a restart.
+   *
+   * If encryption is enabled but no key is configured, encryption is
+   * disabled for this instance with a loud warning (an `encryption_disabled`
+   * event is also emitted on the next tick). Pass `requireEncryption: true`
+   * to throw instead.
+   *
+   * @param {Object} _options - Constructor options
+   * @returns {Buffer|string|null} Key material, or null when unavailable
+   * @private
+   */
+  _resolveEncryptionKey(_options) {
+    const configured =
+      _options.encryptionKey || process.env[ENCRYPTION_KEY_ENV] || null;
+    if (configured || !this._config.compliance.encryption) {
+      return configured;
+    }
+
+    const message =
+      `AuditLogger: encryption is enabled but no key is configured. Set the ` +
+      `encryptionKey option or the ${ENCRYPTION_KEY_ENV} environment variable.`;
+
+    if (_options.requireEncryption === true) {
+      throw new Error(message);
+    }
+
+    logger.warn(
+      `[AUDIT] WARNING: ${message} Sensitive audit fields (details, data, ` +
+        `request) will be written UNENCRYPTED by this instance. Pass ` +
+        `requireEncryption: true to make this an error.`,
+    );
+    this._config.compliance = {
+      ...this._config.compliance,
+      encryption: false,
+    };
+    process.nextTick(() =>
+      this.emit('encryption_disabled', { reason: 'no_encryption_key' }),
+    );
+    return null;
+  }
+
+  /**
+   * AES-256 needs exactly 32 key bytes. A 32-byte Buffer is used as-is; any
+   * other key (e.g. a passphrase string) is reduced with SHA-256.
+   */
+  _getCipherKey() {
+    const key = this.encryptionKey;
+    if (Buffer.isBuffer(key) && key.length === 32) return key;
+    return crypto.createHash('sha256').update(key).digest();
+  }
+
+  _createDecipher(encryptedField) {
+    if (!this.encryptionKey) {
+      throw new Error(
+        `No audit encryption key configured (set encryptionKey or ${ENCRYPTION_KEY_ENV})`,
+      );
+    }
+    if (encryptedField.v === ENCRYPTION_VERSION) {
+      return crypto.createDecipheriv(
+        'aes-256-gcm',
+        this._getCipherKey(),
+        Buffer.from(encryptedField.iv, 'hex'),
+      );
+    }
+    // Legacy (v1) records were written with crypto.createCipher, which was
+    // removed in Node 22. Re-derive its key and IV exactly as it did and use
+    // createDecipheriv so they decrypt on every supported Node version. The
+    // passphrase is the raw key material createCipher was given, not the
+    // SHA-256-reduced key used for v2 records.
+    const { key, iv } = evpBytesToKey(this.encryptionKey, 32, 12);
+    return crypto.createDecipheriv('aes-256-gcm', key, iv);
   }
 
   _generateSigningKey() {
@@ -838,8 +958,14 @@ class AuditLogger extends EventEmitter {
 
   async _verifySignature(auditEvent) {
     const { signature, ...eventData } = auditEvent;
+    if (typeof signature !== 'string') return false;
     const expectedSignature = await this._signAuditEvent(eventData);
-    return signature === expectedSignature;
+    const actual = Buffer.from(signature, 'utf8');
+    const expected = Buffer.from(expectedSignature, 'utf8');
+    // timingSafeEqual requires equal lengths; a length mismatch is not secret
+    // (the expected length is fixed by the HMAC algorithm).
+    if (actual.length !== expected.length) return false;
+    return crypto.timingSafeEqual(actual, expected);
   }
 
   async _getLogFilesInRange(startDate, endDate) {
@@ -873,17 +999,20 @@ class AuditLogger extends EventEmitter {
   }
 
   _matchesQuery(log, query) {
-    if (query.tenantId && log._tenantId !== query._tenantId) return false;
-    if (query.userId && log._userId !== query._userId) return false;
-    if (query.category && log._category !== query._category) return false;
-    if (query.action && log._action !== query._action) return false;
-    if (query.severity && log.metadata._severity !== query._severity)
-      return false;
-    if (
-      query.correlationId &&
-      log.metadata._correlationId !== query._correlationId
-    )
-      return false;
+    // Accept both plain (tenantId) and underscore (_tenantId) query keys; the
+    // underscore form matches the event input names used by the log* methods.
+    const pick = (key) => query[key] ?? query[`_${key}`];
+    const filters = [
+      [pick('tenantId'), log.tenantId],
+      [pick('userId'), log.userId],
+      [pick('category'), log._category],
+      [pick('action'), log.action],
+      [pick('severity'), log.metadata?._severity],
+      [pick('correlationId'), log.metadata?.correlationId],
+    ];
+    for (const [wanted, actual] of filters) {
+      if (wanted !== undefined && wanted !== actual) return false;
+    }
 
     const logTime = new Date(log.metadata.timestamp);
     if (query.startDate && logTime < new Date(query.startDate)) return false;