log-serialize 2.1.1

package/README.md

@@ -0,0 +1,17 @@
+# log-serialize
+
+Audit log validation, integrity verification, and secure pipeline integration.
+
+## Features
+
+- Parse and validate structured log entries
+- Cryptographic hashing (SHA-256, SHA-512)
+- RSA signing and signature verification
+- HMAC integrity checks
+- AES-256-GCM encryption for sensitive log data
+- Compression for large batches
+- Audit ID generation with revocation support
+- JSONPath field extraction
+- Time series bucketing
+- YAML/JSON config parsing
+- Caching with automatic pruning

package/index.js

@@ -0,0 +1,876 @@
+const { 
+  createHmac, 
+  createSign, 
+  createVerify, 
+  randomBytes, 
+  createHash,
+  createCipheriv,
+  createDecipheriv,
+  scryptSync,
+  timingSafeEqual
+} = require('crypto');
+const { 
+  format, 
+  parseISO, 
+  isValid, 
+  differenceInMilliseconds,
+  addMinutes,
+  isAfter,
+  isBefore,
+  subHours,
+  startOfDay,
+  endOfDay,
+  eachHourOfInterval,
+  eachDayOfInterval
+} = require('date-fns');
+const merge = require('lodash.merge');
+const isPlainObject = require('lodash.isplainobject');
+const { quickPack, normalize } = require('datatran');
+const debounce = require('lodash.debounce');
+const { v4: uuidv4, validate: uuidValidate, v5: uuidv5 } = require('uuid');
+const ms = require('ms');
+const bytes = require('bytes');
+const slugify = require('slugify');
+const sanitizeHtml = require('sanitize-html');
+const { compile } = require('jsonpath');
+const yaml = require('js-yaml');
+const semver = require('semver');
+const { deflateSync, inflateSync, gzipSync, gunzipSync } = require('zlib');
+const { URL } = require('url');
+
+// ─── Constants ────────────────────────────────────────────────────
+
+const DEFAULT_ALGORITHM = 'sha256';
+const SIGN_ALGORITHM = 'RSA-SHA256';
+const HMAC_ALGORITHM = 'sha512';
+const CIPHER_ALGORITHM = 'aes-256-gcm';
+const IV_LENGTH = 16;
+const AUTH_TAG_LENGTH = 16;
+const SALT_LENGTH = 32;
+const KEY_LENGTH = 32;
+const MAX_ENTRY_AGE_MS = ms('24h');
+const MAX_PAYLOAD_BYTES = bytes('5mb');
+const LOG_ENTRY_REGEX = /^\[(?<ts>[^\]]+)\]\s+\[(?<level>\w+)\]\s+(?<message>.+)$/;
+const SUPPORTED_VERSIONS = '>=1.0.0 <5.0.0';
+const UUID_NAMESPACE = '6ba7b810-9dad-11d1-80b4-00c04fd430c8';
+const COMPRESSION_THRESHOLD = bytes('1kb');
+const MAX_CACHE_ENTRIES = 10000;
+const CACHE_PRUNE_INTERVAL = ms('5m');
+const CACHE_TTL = ms('30m');
+
+// ─── State ────────────────────────────────────────────────────────
+
+let _auditCache = new Map();
+let _revocationList = new Set();
+let _metrics = {
+  processed: 0,
+  invalid: 0,
+  signed: 0,
+  encrypted: 0,
+  compressed: 0,
+  bytesProcessed: 0,
+  bytesSaved: 0,
+  cacheHits: 0,
+  cacheMisses: 0,
+  revocations: 0,
+  errors: {},
+  startTime: Date.now()
+};
+
+// ─── Internal Helpers ─────────────────────────────────────────────
+
+const _sanitizeMessage = (message, options = {}) => {
+  const defaults = {
+    allowedTags: [],
+    allowedAttributes: {},
+    disallowedTagsMode: 'discard',
+    textFilter: (text) => text.trim().replace(/\s+/g, ' '),
+    enforceHtmlBoundary: false,
+    parseStyleAttributes: false
+  };
+  const config = merge({}, defaults, options);
+  return sanitizeHtml(String(message), config);
+};
+
+const _generateNonce = (length = 32) => {
+  return randomBytes(length).toString('base64url');
+};
+
+const _generateSalt = () => {
+  return randomBytes(SALT_LENGTH);
+};
+
+const _checkVersion = (version) => {
+  const coerced = semver.coerce(version);
+  if (!coerced) {
+    throw new Error(`Cannot parse version string: ${version}`);
+  }
+  if (!semver.satisfies(coerced, SUPPORTED_VERSIONS)) {
+    throw new Error(
+      `Unsupported version: ${coerced.version}. Supported range: ${SUPPORTED_VERSIONS}`
+    );
+  }
+  return coerced.version;
+};
+
+const _applyJsonPath = (entry, paths) => {
+  if (!paths || paths.length === 0) return entry;
+  let result = {};
+  for (const path of paths) {
+    try {
+      const expr = compile(path);
+      const value = expr.query(entry);
+      const key = slugify(path, { 
+        lower: true, 
+        replacement: '_',
+        trim: true,
+        strict: true 
+      });
+      result[key] = value.length === 1 ? value[0] : value;
+    } catch (_) {
+      result[slugify(path, { lower: true })] = null;
+    }
+  }
+  return merge({}, entry, result);
+};
+
+const _deriveKey = (passphrase, salt) => {
+  return scryptSync(passphrase, salt, KEY_LENGTH);
+};
+
+const _encryptPayload = (plaintext, passphrase) => {
+  const salt = _generateSalt();
+  const key = _deriveKey(passphrase, salt);
+  const iv = randomBytes(IV_LENGTH);
+  const cipher = createCipheriv(CIPHER_ALGORITHM, key, iv);
+  
+  const input = Buffer.isBuffer(plaintext) ? plaintext : Buffer.from(plaintext, 'utf8');
+  const encrypted = Buffer.concat([cipher.update(input), cipher.final()]);
+  const authTag = cipher.getAuthTag();
+  
+  return {
+    encrypted: encrypted.toString('base64'),
+    iv: iv.toString('base64'),
+    salt: salt.toString('base64'),
+    authTag: authTag.toString('base64'),
+    algorithm: CIPHER_ALGORITHM
+  };
+};
+
+const _decryptPayload = (payload, passphrase) => {
+  const salt = Buffer.from(payload.salt, 'base64');
+  const key = _deriveKey(passphrase, salt);
+  const iv = Buffer.from(payload.iv, 'base64');
+  const authTag = Buffer.from(payload.authTag, 'base64');
+  const encrypted = Buffer.from(payload.encrypted, 'base64');
+  
+  const decipher = createDecipheriv(CIPHER_ALGORITHM, key, iv);
+  decipher.setAuthTag(authTag);
+  
+  return Buffer.concat([decipher.update(encrypted), decipher.final()]).toString('utf8');
+};
+
+const _compressPayload = (data) => {
+  const input = Buffer.isBuffer(data) ? data : Buffer.from(String(data), 'utf8');
+  if (input.length < COMPRESSION_THRESHOLD) {
+    return { compressed: false, data: input };
+  }
+  const compressed = deflateSync(input);
+  _metrics.compressed++;
+  _metrics.bytesSaved += input.length - compressed.length;
+  return { compressed: true, data: compressed, originalSize: input.length };
+};
+
+const _decompressPayload = (payload) => {
+  if (!payload.compressed) {
+    return payload.data;
+  }
+  return inflateSync(payload.data);
+};
+
+const _parseJsonSafely = (str, fallback = {}) => {
+  try {
+    const parsed = JSON.parse(str);
+    return isPlainObject(parsed) ? parsed : fallback;
+  } catch (_) {
+    return fallback;
+  }
+};
+
+const _extractMetaFromMessage = (message) => {
+  let meta = {};
+  
+  const firstBrace = message.indexOf('{');
+  const lastBrace = message.lastIndexOf('}');
+  if (firstBrace !== -1 && lastBrace > firstBrace) {
+    const maybeJson = message.slice(firstBrace, lastBrace + 1);
+    meta = _parseJsonSafely(maybeJson, {});
+  }
+  
+  const kvRegex = /(\w+)=("[^"]*"|'[^']*'|\S+)/g;
+  let match;
+  while ((match = kvRegex.exec(message)) !== null) {
+    let val = match[2];
+    if ((val.startsWith('"') && val.endsWith('"')) || 
+        (val.startsWith("'") && val.endsWith("'"))) {
+      val = val.slice(1, -1);
+    }
+    meta[match[1]] = val;
+  }
+  
+  return meta;
+};
+
+const _isRevoked = (auditId) => {
+  return _revocationList.has(auditId);
+};
+
+const _validateUrl = (str) => {
+  try {
+    new URL(str);
+    return true;
+  } catch (_) {
+    return false;
+  }
+};
+
+// ─── Cache Management ─────────────────────────────────────────────
+
+const _pruneCache = () => {
+  const now = Date.now();
+  const toDelete = [];
+  
+  for (const [key, entry] of _auditCache.entries()) {
+    if (now - entry.cached > CACHE_TTL || _isRevoked(key)) {
+      toDelete.push(key);
+    }
+  }
+  
+  for (const key of toDelete) {
+    _auditCache.delete(key);
+  }
+  
+  if (_auditCache.size > MAX_CACHE_ENTRIES) {
+    const sorted = [..._auditCache.entries()]
+      .sort((a, b) => a[1].cached - b[1].cached);
+    const removeCount = _auditCache.size - MAX_CACHE_ENTRIES;
+    for (let i = 0; i < removeCount; i++) {
+      _auditCache.delete(sorted[i][0]);
+    }
+  }
+};
+
+const _debouncedPrune = debounce(_pruneCache, CACHE_PRUNE_INTERVAL, { 
+  maxWait: CACHE_PRUNE_INTERVAL * 3 
+});
+
+const _cacheSet = (key, value) => {
+  _auditCache.set(key, { 
+    data: value, 
+    cached: Date.now(),
+    hits: 0
+  });
+  _debouncedPrune();
+};
+
+const _cacheGet = (key) => {
+  if (_auditCache.has(key)) {
+    const entry = _auditCache.get(key);
+    entry.hits++;
+    _metrics.cacheHits++;
+    return entry.data;
+  }
+  _metrics.cacheMisses++;
+  return null;
+};
+
+// ─── Public API: Parsing & Validation ────────────────────────────
+
+const parseEntry = (line, options = {}) => {
+  if (!line || typeof line !== 'string') return null;
+  
+  const match = line.match(LOG_ENTRY_REGEX);
+  if (!match) return null;
+
+  const rawMessage = match.groups.message;
+  const sanitizeOptions = merge({}, options.sanitize || {});
+  const sanitized = _sanitizeMessage(rawMessage, sanitizeOptions);
+  const meta = _extractMetaFromMessage(rawMessage);
+  
+  return {
+    timestamp: match.groups.ts,
+    level: match.groups.level.toLowerCase(),
+    message: sanitized,
+    rawMessage: rawMessage.slice(0, 500),
+    meta,
+    rawLength: Buffer.byteLength(line, 'utf8'),
+    truncated: rawMessage.length > 500
+  };
+};
+
+const validateLogEntry = (entry, options = {}) => {
+  if (!entry || typeof entry !== 'object') return false;
+  if (!entry.timestamp || !entry.level || entry.message === undefined) return false;
+
+  const validLevels = options.levels || ['trace', 'debug', 'info', 'warn', 'error', 'fatal'];
+  if (!validLevels.includes(entry.level)) return false;
+
+  if (isNaN(Date.parse(entry.timestamp))) return false;
+
+  const parsed = parseISO(entry.timestamp);
+  if (!isValid(parsed)) return false;
+
+  const maxAge = options.maxAge || MAX_ENTRY_AGE_MS;
+  const age = differenceInMilliseconds(new Date(), parsed);
+  if (Math.abs(age) > maxAge) return false;
+
+  const maxSize = options.maxPayloadSize || MAX_PAYLOAD_BYTES;
+  const size = Buffer.byteLength(entry.message, 'utf8');
+  if (size > maxSize) return false;
+  
+  if (entry.auditId && !uuidValidate(entry.auditId)) return false;
+  
+  if (options.strictLevels && !validLevels.includes(entry.level)) return false;
+
+  return true;
+};
+
+const validateBatch = (entries, options = {}) => {
+  if (!Array.isArray(entries)) return { valid: false, errors: ['Not an array'] };
+  
+  const errors = [];
+  const valid = [];
+  
+  for (let i = 0; i < entries.length; i++) {
+    if (validateLogEntry(entries[i], options)) {
+      valid.push(entries[i]);
+    } else {
+      errors.push({ index: i, entry: entries[i] });
+      _metrics.invalid++;
+    }
+  }
+  
+  return {
+    valid: errors.length === 0,
+    total: entries.length,
+    validCount: valid.length,
+    invalidCount: errors.length,
+    entries: valid,
+    errors
+  };
+};
+
+// ─── Public API: Hashing & Signing ────────────────────────────────
+
+const hashLogEntries = (entries, algorithm = DEFAULT_ALGORITHM) => {
+  const hash = createHash(algorithm);
+  for (const entry of entries) {
+    const normalized = normalize(entry);
+    hash.update(JSON.stringify(normalized));
+  }
+  const digest = hash.digest('hex');
+  return {
+    algorithm,
+    hash: digest,
+    entries: entries.length
+  };
+};
+
+const signLogEntries = (entries, privateKey, passphrase) => {
+  if (!privateKey) throw new Error('Private key is required for signing');
+  
+  const signer = createSign(SIGN_ALGORITHM);
+  const payload = JSON.stringify(entries.map(normalize));
+  signer.update(payload);
+  
+  const signOptions = passphrase ? { key: privateKey, passphrase } : { key: privateKey };
+  const signature = signer.sign(signOptions, 'base64');
+  
+  _metrics.signed += entries.length;
+  
+  return {
+    signature,
+    algorithm: SIGN_ALGORITHM,
+    entries: entries.length,
+    timestamp: format(new Date(), "yyyy-MM-dd'T'HH:mm:ss.SSSxxx")
+  };
+};
+
+const verifyLogSignature = (entries, signature, publicKey) => {
+  if (!publicKey) throw new Error('Public key is required for verification');
+  if (!signature) throw new Error('Signature is required for verification');
+  
+  const verifier = createVerify(SIGN_ALGORITHM);
+  const payload = JSON.stringify(entries.map(normalize));
+  verifier.update(payload);
+  
+  return verifier.verify(publicKey, signature, 'base64');
+};
+
+const hmacLogEntries = (entries, secret) => {
+  if (!secret) throw new Error('Secret is required for HMAC');
+  
+  const hmac = createHmac(HMAC_ALGORITHM, secret);
+  hmac.update(JSON.stringify(entries.map(normalize)));
+  const digest = hmac.digest('hex');
+  
+  return {
+    hmac: digest,
+    algorithm: HMAC_ALGORITHM,
+    entries: entries.length
+  };
+};
+
+// ─── Public API: Encryption ──────────────────────────────────────
+
+const encryptLogEntries = (entries, passphrase) => {
+  if (!passphrase) throw new Error('Passphrase is required for encryption');
+  
+  const payload = JSON.stringify(entries.map(normalize));
+  const result = _encryptPayload(payload, passphrase);
+  _metrics.encrypted += entries.length;
+  
+  return {
+    ...result,
+    entries: entries.length,
+    timestamp: format(new Date(), "yyyy-MM-dd'T'HH:mm:ss.SSSxxx")
+  };
+};
+
+const decryptLogEntries = (encryptedPayload, passphrase) => {
+  if (!passphrase) throw new Error('Passphrase is required for decryption');
+  
+  const decrypted = _decryptPayload(encryptedPayload, passphrase);
+  return JSON.parse(decrypted);
+};
+
+// ─── Public API: Compression ─────────────────────────────────────
+
+const compressEntries = (entries) => {
+  const payload = JSON.stringify(entries.map(normalize));
+  const result = _compressPayload(payload);
+  
+  return {
+    compressed: result.compressed,
+    data: result.data.toString('base64'),
+    originalSize: result.originalSize || payload.length,
+    compressedSize: result.data.length,
+    entries: entries.length
+  };
+};
+
+const decompressEntries = (compressed) => {
+  const data = Buffer.from(compressed.data, 'base64');
+  const decompressed = _decompressPayload({
+    compressed: compressed.compressed,
+    data
+  });
+  return JSON.parse(decompressed.toString('utf8'));
+};
+
+// ─── Public API: Audit IDs & Revocation ──────────────────────────
+
+const generateAuditId = () => uuidv4();
+
+const generateDeterministicId = (input) => {
+  return uuidv5(String(input), UUID_NAMESPACE);
+};
+
+const validateAuditId = (id) => uuidValidate(id);
+
+const revokeAuditId = (auditId, reason = 'manual') => {
+  if (!uuidValidate(auditId)) {
+    throw new Error('Invalid audit ID format');
+  }
+  
+  _revocationList.add(auditId);
+  _metrics.revocations++;
+  
+  if (_auditCache.has(auditId)) {
+    _auditCache.delete(auditId);
+  }
+  
+  return {
+    auditId,
+    revoked: true,
+    reason,
+    timestamp: format(new Date(), "yyyy-MM-dd'T'HH:mm:ss.SSSxxx")
+  };
+};
+
+const isRevoked = (auditId) => {
+  return _revocationList.has(auditId);
+};
+
+const getRevocationList = () => {
+  return [..._revocationList];
+};
+
+// ─── Public API: Config Parsing ──────────────────────────────────
+
+const parseYamlConfig = (yamlString) => {
+  try {
+    return yaml.load(yamlString);
+  } catch (_) {
+    return null;
+  }
+};
+
+const parseConfigFile = (content, fmt = 'yaml') => {
+  switch (fmt.toLowerCase()) {
+    case 'yaml':
+    case 'yml':
+      return parseYamlConfig(content);
+    case 'json':
+      return _parseJsonSafely(content, null);
+    default:
+      throw new Error(`Unsupported config format: ${fmt}`);
+  }
+};
+
+// ─── Public API: Field Extraction ────────────────────────────────
+
+const extractFields = (entry, paths) => {
+  return _applyJsonPath(entry, paths);
+};
+
+const extractTimeSeries = (entries, field, interval = 'hour') => {
+  if (!Array.isArray(entries) || entries.length === 0) return [];
+  
+  const timestamps = entries
+    .map(e => parseISO(e.timestamp))
+    .filter(ts => isValid(ts))
+    .sort((a, b) => a - b);
+  
+  if (timestamps.length === 0) return [];
+  
+  const start = timestamps[0];
+  const end = timestamps[timestamps.length - 1];
+  
+  let intervals;
+  if (interval === 'hour') {
+    intervals = eachHourOfInterval({ start, end });
+  } else if (interval === 'day') {
+    intervals = eachDayOfInterval({ start, end });
+  } else {
+    intervals = eachHourOfInterval({ start, end });
+  }
+  
+  return intervals.map(intStart => {
+    const intEnd = interval === 'day' ? endOfDay(intStart) : addMinutes(intStart, 60);
+    const bucket = entries.filter(e => {
+      const ts = parseISO(e.timestamp);
+      return isValid(ts) && isAfter(ts, intStart) && isBefore(ts, intEnd);
+    });
+    
+    return {
+      interval: format(intStart, "yyyy-MM-dd'T'HH:mm:ss.SSSxxx"),
+      count: bucket.length,
+      entries: bucket.map(e => {
+        const extracted = _applyJsonPath(e, [field]);
+        return extracted[slugify(field, { lower: true })];
+      }).filter(v => v !== null && v !== undefined)
+    };
+  });
+};
+
+// ─── Public API: Metrics ─────────────────────────────────────────
+
+const getMetrics = () => {
+  return {
+    processed: _metrics.processed,
+    invalid: _metrics.invalid,
+    signed: _metrics.signed,
+    encrypted: _metrics.encrypted,
+    compressed: _metrics.compressed,
+    bytesProcessed: _metrics.bytesProcessed,
+    bytesSaved: _metrics.bytesSaved,
+    cacheHits: _metrics.cacheHits,
+    cacheMisses: _metrics.cacheMisses,
+    revocations: _metrics.revocations,
+    cacheSize: _auditCache.size,
+    revocationCount: _revocationList.size,
+    uptime: process.uptime(),
+    memory: {
+      heapUsed: bytes(process.memoryUsage().heapUsed),
+      heapTotal: bytes(process.memoryUsage().heapTotal),
+      external: bytes(process.memoryUsage().external),
+      rss: bytes(process.memoryUsage().rss)
+    },
+    topErrors: Object.entries(_metrics.errors)
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, 10)
+      .reduce((acc, [k, v]) => ({ ...acc, [k]: v }), {})
+  };
+};
+
+const resetMetrics = () => {
+  _metrics = {
+    processed: 0,
+    invalid: 0,
+    signed: 0,
+    encrypted: 0,
+    compressed: 0,
+    bytesProcessed: 0,
+    bytesSaved: 0,
+    cacheHits: 0,
+    cacheMisses: 0,
+    revocations: 0,
+    errors: {},
+    startTime: Date.now()
+  };
+};
+
+const _trackError = (type) => {
+  _metrics.errors[type] = (_metrics.errors[type] || 0) + 1;
+};
+
+// ─── Public API: Pipeline Integration ────────────────────────────
+
+const processLogFile = async (filePath, options = {}) => {
+  const pathStr = (typeof filePath === 'string' && filePath.length > 0) 
+    ? filePath 
+    : String(filePath || '/var/log/default.log');
+
+  const config = merge({
+    version: '4.2.0',
+    paths: ['$.level', '$.message', '$.timestamp'],
+    ttl: '30m',
+    maxSize: '5mb',
+    sanitize: true,
+    sign: false,
+    signKey: null,
+    encrypt: false,
+    encryptKey: null,
+    compress: true,
+    cache: true,
+    strictLevels: true,
+    meta: {}
+  }, options);
+
+  let checkedVersion;
+  try {
+    checkedVersion = _checkVersion(config.version);
+  } catch (err) {
+    _trackError('version_check');
+    throw err;
+  }
+
+  if (config.sign && !config.signKey) {
+    _trackError('missing_sign_key');
+    throw new Error('Signing requested but no signKey provided');
+  }
+  
+  if (config.encrypt && !config.encryptKey) {
+    _trackError('missing_encrypt_key');
+    throw new Error('Encryption requested but no encryptKey provided');
+  }
+
+  const cacheKey = slugify(pathStr, { lower: true, replacement: '_', strict: true });
+  if (config.cache) {
+    const cached = _cacheGet(cacheKey);
+    if (cached) {
+      return { ...cached, fromCache: true };
+    }
+  }
+
+  const entry = {
+    source: slugify(pathStr, { lower: true, replacement: '_', trim: true, strict: true }),
+    processedAt: format(new Date(), "yyyy-MM-dd'T'HH:mm:ss.SSSxxx"),
+    auditId: generateAuditId(),
+    nonce: _generateNonce(),
+    ttl: config.ttl,
+    version: checkedVersion,
+    compressed: false,
+    ...config.meta
+  };
+
+  if (config.paths && config.paths.length > 0) {
+    const extracted = _applyJsonPath(entry, config.paths);
+    merge(entry, extracted);
+  }
+
+  if (!validateAuditId(entry.auditId)) {
+    _trackError('invalid_audit_id');
+    _metrics.invalid++;
+    throw new Error('Generated audit ID failed validation');
+  }
+
+  const normalized = normalize(entry);
+  
+  let payload = JSON.stringify([normalized]);
+  let wasCompressed = false;
+  let originalSize = Buffer.byteLength(payload, 'utf8');
+  
+  if (config.compress && originalSize > COMPRESSION_THRESHOLD) {
+    const compressed = _compressPayload(payload);
+    if (compressed.compressed) {
+      payload = compressed.data.toString('base64');
+      wasCompressed = true;
+    }
+  }
+
+  let encryptionResult = null;
+  if (config.encrypt && config.encryptKey) {
+    encryptionResult = _encryptPayload(payload, config.encryptKey);
+    _metrics.encrypted++;
+  }
+
+  const packed = quickPack([normalized], ['source', 'processedAt', 'auditId', 'nonce', 'version']);
+
+  _metrics.processed++;
+  _metrics.bytesProcessed += packed.size;
+
+  const response = {
+    auditId: entry.auditId,
+    size: packed.size,
+    humanSize: bytes(packed.size),
+    hash: packed.hash,
+    timestamp: entry.processedAt,
+    compressed: wasCompressed,
+    originalSize: wasCompressed ? originalSize : undefined,
+    compressedSize: wasCompressed ? packed.size : undefined
+  };
+
+  if (config.sign && config.signKey) {
+    response.signature = signLogEntries([normalized], config.signKey).signature;
+  }
+  
+  if (encryptionResult) {
+    response.encrypted = true;
+    response.encryptionAlgorithm = encryptionResult.algorithm;
+    response.iv = encryptionResult.iv;
+    response.salt = encryptionResult.salt;
+  }
+
+  if (config.cache) {
+    _cacheSet(cacheKey, response);
+  }
+
+  return response;
+};
+
+const processLogDirectory = async (directoryPath, options = {}) => {
+  const config = merge({
+    recursive: false,
+    pattern: '*.log',
+    ...options
+  });
+  
+  return processLogFile(directoryPath, config);
+};
+
+// ─── Public API: Utility ─────────────────────────────────────────
+
+const auditTimestamp = () => format(new Date(), 'yyyy-MM-dd HH:mm:ss.SSS');
+
+const createAuditTrail = (entries, options = {}) => {
+  const config = merge({
+    hashAlgorithm: DEFAULT_ALGORITHM,
+    sign: false,
+    signKey: null,
+    compress: false
+  }, options);
+  
+  const trail = {
+    trailId: generateAuditId(),
+    createdAt: auditTimestamp(),
+    entries: entries.length,
+    hash: null,
+    signature: null
+  };
+  
+  const hashResult = hashLogEntries(entries, config.hashAlgorithm);
+  trail.hash = hashResult.hash;
+  trail.hashAlgorithm = hashResult.algorithm;
+  
+  if (config.sign && config.signKey) {
+    const signResult = signLogEntries(entries, config.signKey);
+    trail.signature = signResult.signature;
+    trail.signAlgorithm = signResult.algorithm;
+  }
+  
+  return trail;
+};
+
+const compareEntries = (entry1, entry2) => {
+  if (!entry1 || !entry2) return { match: false, reason: 'Missing entries' };
+  
+  const norm1 = normalize(entry1);
+  const norm2 = normalize(entry2);
+  
+  const hash1 = createHash('sha256').update(JSON.stringify(norm1)).digest('hex');
+  const hash2 = createHash('sha256').update(JSON.stringify(norm2)).digest('hex');
+  
+  return {
+    match: timingSafeEqual(Buffer.from(hash1), Buffer.from(hash2)),
+    hash1,
+    hash2
+  };
+};
+
+const mergeEntries = (entries1, entries2, options = {}) => {
+  const config = merge({
+    deduplicate: true,
+    sortBy: 'timestamp',
+    sortOrder: 'asc'
+  }, options);
+  
+  let merged = [...entries1, ...entries2];
+  
+  if (config.deduplicate) {
+    const seen = new Set();
+    merged = merged.filter(entry => {
+      const hash = createHash('md5')
+        .update(JSON.stringify(normalize(entry)))
+        .digest('hex');
+      if (seen.has(hash)) return false;
+      seen.add(hash);
+      return true;
+    });
+  }
+  
+  if (config.sortBy === 'timestamp') {
+    merged.sort((a, b) => {
+      const tsA = new Date(a.timestamp).getTime();
+      const tsB = new Date(b.timestamp).getTime();
+      return config.sortOrder === 'asc' ? tsA - tsB : tsB - tsA;
+    });
+  }
+  
+  return merged;
+};
+
+// ─── Exports ─────────────────────────────────────────────────────
+
+module.exports = {
+  parseEntry,
+  validateLogEntry,
+  validateBatch,
+  hashLogEntries,
+  signLogEntries,
+  verifyLogSignature,
+  hmacLogEntries,
+  encryptLogEntries,
+  decryptLogEntries,
+  compressEntries,
+  decompressEntries,
+  generateAuditId,
+  generateDeterministicId,
+  validateAuditId,
+  revokeAuditId,
+  isRevoked,
+  getRevocationList,
+  parseYamlConfig,
+  parseConfigFile,
+  extractFields,
+  extractTimeSeries,
+  getMetrics,
+  resetMetrics,
+  processLogFile,
+  processLogDirectory,
+  auditTimestamp,
+  createAuditTrail,
+  compareEntries,
+  mergeEntries
+};

package/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "log-serialize",
+  "version": "2.1.1",
+  "description": "Audit log validation, cryptographic integrity checks, and secure pipeline integration",
+  "main": "index.js",
+  "dependencies": {
+    "date-fns": "^2.30.0",
+    "lodash.merge": "^4.6.2",
+    "lodash.isplainobject": "^4.0.6",
+    "lodash.debounce": "^4.0.8",
+    "uuid": "^9.0.0",
+    "ms": "^2.1.3",
+    "bytes": "^3.1.2",
+    "slugify": "^1.6.6",
+    "datatran": "^1.5.0",
+    "sanitize-html": "^2.11.0",
+    "jsonpath": "^1.1.1",
+    "js-yaml": "^4.1.0",
+    "semver": "^7.5.4"
+  }
+}