@blamejs/core 0.4.1

package/lib/queue.js

@@ -0,0 +1,430 @@
+"use strict";
+/**
+ * Queue dispatcher — pluggable job queue with retry, lease semantics, and
+ * graceful shutdown.
+ *
+ * Same dispatcher pattern as object-store: backends are configured per-name,
+ * each with a protocol + protocol-specific options. The built-in 'local'
+ * protocol is SQLite-backed (baked into the framework's main DB).
+ * External protocols (redis, sqs, amqp, nats) are listed as deferred and
+ * surface a clear error when selected.
+ *
+ * Public API:
+ *   queue.init({ backends: { name: { protocol: 'local' } }, defaultBackend? })
+ *   queue.enqueue(queueName, payload, opts?)
+ *                                       → { jobId, queueName, ... }
+ *   queue.consume(queueName, handler, opts?)
+ *                                       → consumer handle (with cancel())
+ *   queue.size(queueName, opts?)        → number (pending + inflight)
+ *   queue.purge(queueName, opts?)       → number deleted
+ *   queue.shutdown(opts?)               → drain handlers gracefully
+ *   queue.listBackends()                → [{ name, protocol }]
+ *
+ * Job lifecycle:
+ *   enqueued (status='pending')
+ *     ↓ availableAt reached + consumer leases
+ *   inflight (status='inflight', lease expires after leaseDurationMs)
+ *     ↓ handler returns                 ↓ handler throws
+ *   done (status='done')              if attempts < maxAttempts:
+ *                                       pending (with backoff)
+ *                                     else:
+ *                                       failed (status='failed')
+ */
+var C = require("./constants");
+var localProto = require("./queue-local");
+var retryHelper = require("./retry");
+var safeAsync = require("./safe-async");
+var lazyRequire = require("./lazy-require");
+var observability = require("./observability");
+var protocolDispatcher = require("./protocol-dispatcher");
+var { QueueError } = require("./framework-error");
+
+var dispatcher = protocolDispatcher.create({
+  name:       "queue",
+  errorClass: QueueError,
+  protocols:  { "local": localProto },
+  deferred:   {
+    "redis":  { description: "Redis Streams (XADD/XREADGROUP/XACK/XCLAIM)" },
+    "sqs":    { description: "AWS SQS (and S3-compatible queue endpoints) via SigV4" },
+    "amqp":   { description: "AMQP 0-9-1 (RabbitMQ etc.)" },
+    "nats":   { description: "NATS JetStream" },
+  },
+  fallbackProtocol: "local",
+});
+
+var _err = QueueError.factory;
+
+var audit = lazyRequire(function () { return require("./audit"); });
+
+var initialized = false;
+var backends = {};
+var defaultBackend = null;
+var consumers = [];   // [{ queueName, backendName, cancel(), running, inFlight: Set }]
+var sweepTimer = null;
+
+function init(opts) {
+  if (initialized) return;
+  if (!opts || !opts.backends) {
+    throw _err("INVALID_CONFIG", "queue.init({ backends }) is required", true);
+  }
+
+  backends = {};
+  for (var name in opts.backends) {
+    var cfg = opts.backends[name];
+    var proto = dispatcher.resolve(cfg.protocol);
+    var raw = proto.create(cfg);
+    var breaker = new retryHelper.CircuitBreaker(
+      "queue:" + name,
+      cfg.breaker
+    );
+
+    // Wrap mutating ops with retry + breaker (idempotent only — enqueue and
+    // complete are safe to retry; lease isn't because partial-double-lease
+    // is dangerous, so it goes through the breaker but not retry).
+    function wrapWithRetry(fn) {
+      return function () {
+        var args = Array.prototype.slice.call(arguments);
+        return retryHelper.withRetry(function () {
+          return breaker.wrap(function () { return fn.apply(raw, args); });
+        }, cfg.retry);
+      };
+    }
+    function wrapBreakerOnly(fn) {
+      return function () {
+        var args = Array.prototype.slice.call(arguments);
+        return breaker.wrap(function () { return fn.apply(raw, args); });
+      };
+    }
+
+    backends[name] = {
+      name:          name,
+      protocol:      cfg.protocol,
+      breaker:       breaker,
+      raw:           raw,
+      enqueue:       wrapWithRetry(raw.enqueue),
+      lease:         wrapBreakerOnly(raw.lease),
+      extendLease:   raw.extendLease ? wrapWithRetry(raw.extendLease) : null,
+      complete:      wrapWithRetry(raw.complete),
+      fail:          wrapWithRetry(raw.fail),
+      sweepExpired:  raw.sweepExpired ? wrapBreakerOnly(raw.sweepExpired) : null,
+      size:          wrapWithRetry(raw.size),
+      purge:         wrapWithRetry(raw.purge),
+      dlqList:       raw.dlqList ? wrapWithRetry(raw.dlqList) : null,
+      dlqRetry:      raw.dlqRetry ? wrapWithRetry(raw.dlqRetry) : null,
+      dlqSize:       raw.dlqSize ? wrapWithRetry(raw.dlqSize) : null,
+    };
+  }
+
+  defaultBackend = opts.defaultBackend || Object.keys(backends)[0];
+
+  // Sweep expired leases periodically (every 30s) so crashed-handler jobs
+  // get re-pended.
+  sweepTimer = setInterval(function () {
+    Object.keys(backends).forEach(function (n) {
+      if (backends[n].sweepExpired) {
+        backends[n].sweepExpired().catch(function () { /* best effort */ });
+      }
+    });
+  }, 30000);
+  sweepTimer.unref();
+
+  initialized = true;
+}
+
+function _backendFor(opts) {
+  opts = opts || {};
+  var name = opts.backend || defaultBackend;
+  var b = backends[name];
+  if (!b) throw _err("UNKNOWN_BACKEND", "no backend named '" + name + "'", true);
+  return b;
+}
+
+// ---- Public API ----
+
+function enqueue(queueName, payload, opts) {
+  _requireInit();
+  if (!queueName) throw _err("MISSING_QUEUE", "enqueue requires queueName", true);
+  opts = opts || {};
+  var b = _backendFor(opts);
+  return observability.tap("queue.enqueue",
+    { queueName: queueName, backend: b.name },
+    function () {
+      return b.enqueue(queueName, payload, opts).then(function (result) {
+        _emit("system.queue.enqueue", {
+          metadata: {
+            queue:          queueName,
+            backend:        b.name,
+            jobId:          result.jobId,
+            classification: result.classification,
+            traceId:        opts.traceId,
+            delaySeconds:   opts.delaySeconds || 0,
+          },
+        });
+        return result;
+      });
+    }
+  );
+}
+
+function consume(queueName, handler, opts) {
+  _requireInit();
+  if (!queueName) throw _err("MISSING_QUEUE", "consume requires queueName", true);
+  if (typeof handler !== "function") throw _err("INVALID_HANDLER", "handler must be a function", true);
+  opts = opts || {};
+  var b = _backendFor(opts);
+  var concurrency      = opts.concurrency      || 1;
+  var leaseDurationMs  = opts.leaseDurationMs  || 30000;
+  var pollIntervalMs   = opts.pollIntervalMs   || 1000;
+  var fastPollMs       = opts.fastPollMs       || 50;
+
+  // Each consumer has its own AbortController so cancel() unblocks any
+  // in-flight poll-sleep immediately rather than waiting up to
+  // pollIntervalMs (default 1s) for the next while-loop iteration.
+  var abortCtrl = new AbortController();
+  var state = {
+    queueName:    queueName,
+    backendName:  b.name,
+    cancelled:    false,
+    inFlight:     new Set(),
+    abortCtrl:    abortCtrl,
+    cancel:       function () {
+      state.cancelled = true;
+      try { abortCtrl.abort(); } catch (_e) {}
+    },
+  };
+  consumers.push(state);
+
+  (async function loop() {
+    // Helper — sleep with cancellation. On abort, returns instead of
+    // rejecting so the next while-iteration sees `state.cancelled` and
+    // exits cleanly.
+    async function _pollSleep(ms) {
+      try { await safeAsync.sleep(ms, { signal: abortCtrl.signal }); }
+      catch (_e) { /* aborted — loop condition will catch it */ }
+    }
+    while (!state.cancelled) {
+      // Don't lease more than (concurrency - inFlight) at a time
+      var slots = concurrency - state.inFlight.size;
+      if (slots <= 0) {
+        await _pollSleep(fastPollMs);
+        continue;
+      }
+      var jobs;
+      try { jobs = await b.lease(queueName, leaseDurationMs, slots); }
+      catch {
+        // Backend down (breaker open, etc.) — back off
+        await _pollSleep(pollIntervalMs);
+        continue;
+      }
+      if (!jobs || jobs.length === 0) {
+        await _pollSleep(pollIntervalMs);
+        continue;
+      }
+      for (var i = 0; i < jobs.length; i++) {
+        observability.event("queue.lease", 1, { queueName: queueName });
+        (function (job) {
+          state.inFlight.add(job.jobId);
+          _emit("system.queue.consume.start", {
+            metadata: { queue: queueName, backend: b.name, jobId: job.jobId, attempt: job.attempts, traceId: job.traceId },
+          });
+          // Handler context — second arg to handler. Carries
+          // ctx.extendLease(ms) for long-running handlers that need to
+          // bump their own lease before the sweeper reclaims the job.
+          var ctx = {
+            extendLease: function (additionalMs) {
+              if (typeof b.extendLease !== "function") {
+                throw _err("EXTEND_LEASE_UNSUPPORTED",
+                  "queue backend '" + b.name + "' does not support extendLease",
+                  true);
+              }
+              return b.extendLease(job.jobId, additionalMs).then(function (ok) {
+                if (ok) {
+                  _emit("system.queue.lease.extended", {
+                    metadata: { queue: queueName, backend: b.name, jobId: job.jobId, additionalMs: additionalMs },
+                  });
+                }
+                return ok;
+              });
+            },
+          };
+          observability.tap("queue.consume",
+            { queueName: queueName, backend: b.name, jobId: job.jobId, attempt: job.attempts },
+            function () {
+              return Promise.resolve()
+                .then(function () { return handler(job, ctx); })
+                .then(function () {
+                  return b.complete(job.jobId).then(function () {
+                    _emit("system.queue.consume.success", {
+                      metadata: { queue: queueName, backend: b.name, jobId: job.jobId, attempt: job.attempts, traceId: job.traceId },
+                    });
+                    observability.event("queue.complete", 1, { queueName: queueName });
+                  });
+                }, function (err) {
+              var msg = (err && err.message) || String(err);
+              var willRetry = job.attempts < job.maxAttempts;
+              return b.fail(job.jobId, msg, { retryDelayMs: _backoffDelay(job.attempts) })
+                .then(function () {
+                  observability.event("queue.fail", 1, { queueName: queueName, willRetry: willRetry });
+                  _emit("system.queue.consume.failure", {
+                    metadata: {
+                      queue:    queueName, backend: b.name, jobId: job.jobId,
+                      attempt:  job.attempts, traceId: job.traceId,
+                      maxAttempts: job.maxAttempts, willRetry: willRetry,
+                    },
+                    reason:   msg,
+                    outcome:  "failure",
+                  });
+                  // DLQ-write event when the job has exhausted its retries.
+                  // Operators wire this to their alerting / dashboards
+                  // — failed-after-retries is "needs human review" not
+                  // "in the normal flow." Audit chain captures the
+                  // final state for forensics.
+                  if (!willRetry) {
+                    _emit("system.queue.dlq.write", {
+                      metadata: {
+                        queue: queueName, backend: b.name, jobId: job.jobId,
+                        attempts: job.attempts, traceId: job.traceId,
+                      },
+                      reason:  msg,
+                      outcome: "failure",
+                    });
+                  }
+                });
+            })
+            .catch(function (_e) { /* lifecycle errors swallowed — operator sees via audit */ })
+            .then(function () { state.inFlight.delete(job.jobId); });
+            }
+          );
+        })(jobs[i]);
+      }
+      await _pollSleep(fastPollMs);
+    }
+  })();
+
+  return state;
+}
+
+var _QUEUE_BACKOFF_OPTS = {
+  baseDelayMs:  C.TIME.seconds(1),
+  maxDelayMs:   C.TIME.minutes(5),
+  jitterFactor: 0,                    // queue uses deterministic backoff (no jitter) for predictability
+};
+function _backoffDelay(attempt) {
+  return retryHelper.backoffDelay(attempt, _QUEUE_BACKOFF_OPTS);
+}
+
+function size(queueName, opts) {
+  _requireInit();
+  return _backendFor(opts).size(queueName);
+}
+
+function purge(queueName, opts) {
+  _requireInit();
+  var b = _backendFor(opts);
+  return b.purge(queueName).then(function (n) {
+    _emit("system.queue.purge", {
+      metadata: { queue: queueName, backend: b.name, deleted: n },
+    });
+    return n;
+  });
+}
+
+// ---- Dead-letter queue ----
+//
+// `dlqList` returns failed-after-retries jobs for operator review;
+// `dlqRetry` resets a single job back to 'pending' so it gets picked
+// up by consumers again (operator-driven — never automatic).
+
+function dlqList(queueName, opts) {
+  _requireInit();
+  var b = _backendFor(opts);
+  if (typeof b.dlqList !== "function") {
+    return Promise.reject(_err("DLQ_UNSUPPORTED",
+      "queue backend '" + b.name + "' does not support dlqList", true));
+  }
+  return b.dlqList(queueName, opts);
+}
+
+function dlqRetry(jobId, opts) {
+  _requireInit();
+  var b = _backendFor(opts);
+  if (typeof b.dlqRetry !== "function") {
+    return Promise.reject(_err("DLQ_UNSUPPORTED",
+      "queue backend '" + b.name + "' does not support dlqRetry", true));
+  }
+  return b.dlqRetry(jobId).then(function (ok) {
+    if (ok) {
+      _emit("system.queue.dlq.retry", {
+        metadata: { jobId: jobId, backend: b.name },
+      });
+    }
+    return ok;
+  });
+}
+
+function dlqSize(queueName, opts) {
+  _requireInit();
+  var b = _backendFor(opts);
+  if (typeof b.dlqSize !== "function") {
+    return Promise.reject(_err("DLQ_UNSUPPORTED",
+      "queue backend '" + b.name + "' does not support dlqSize", true));
+  }
+  return b.dlqSize(queueName);
+}
+
+async function shutdown(opts) {
+  if (!initialized) return;
+  opts = opts || {};
+  var timeoutMs = opts.timeoutMs != null ? opts.timeoutMs : C.TIME.seconds(30);
+  // Signal all consumers to stop
+  consumers.forEach(function (c) { c.cancel(); });
+  // Wait for in-flight handlers to complete
+  var deadline = Date.now() + timeoutMs;
+  while (consumers.some(function (c) { return c.inFlight.size > 0; })) {
+    if (Date.now() > deadline) break;
+    await safeAsync.sleep(50);
+  }
+  consumers = [];
+  if (sweepTimer) { clearInterval(sweepTimer); sweepTimer = null; }
+}
+
+function listBackends() {
+  _requireInit();
+  return Object.keys(backends).map(function (name) {
+    return { name: name, protocol: backends[name].protocol, breakerState: backends[name].breaker.getState() };
+  });
+}
+
+function _emit(action, info) {
+  audit().safeEmit({ action: action, ...(info || {}) });
+}
+
+function _requireInit() {
+  if (!initialized) throw _err("NOT_INITIALIZED", "queue.init() must be called first", true);
+}
+
+function _resetForTest() {
+  if (sweepTimer) { clearInterval(sweepTimer); sweepTimer = null; }
+  consumers.forEach(function (c) { c.cancel(); });
+  consumers = [];
+  backends = {};
+  defaultBackend = null;
+  initialized = false;
+  audit.reset();
+}
+
+module.exports = {
+  init:               init,
+  enqueue:            enqueue,
+  consume:            consume,
+  size:               size,
+  purge:              purge,
+  shutdown:           shutdown,
+  listBackends:       listBackends,
+  dlqList:            dlqList,
+  dlqRetry:           dlqRetry,
+  dlqSize:            dlqSize,
+  PROTOCOLS:          dispatcher.protocols,
+  DEFERRED_PROTOCOLS: dispatcher.deferred,
+  _resetForTest:      _resetForTest,
+};

package/lib/redact.js

@@ -0,0 +1,192 @@
+"use strict";
+/**
+ * Operational-log redaction layer.
+ *
+ * Wraps every operational log emit so PHI / PCI / personal data never
+ * reaches a debug sink, file, or external SIEM. Two complementary signals:
+ *
+ *   1. Field-name allow/deny: known sensitive fields are scrubbed by name
+ *      regardless of the value's contents.
+ *   2. Pattern detection: even if a field name slips past, common-shape
+ *      detectors (credit-card via Luhn, JWT, PEM blocks, AWS-key-shape)
+ *      catch the value and replace with a marker.
+ *
+ * The redactor returns a NEW object — the original log payload is never
+ * mutated. This matters because the same object may be passed to the
+ * audit-log path (which DOES seal sensitive content via vault) AND the
+ * operational path (which redacts) in the same call.
+ *
+ * Public API:
+ *   redact.redact(value, opts?)              → redacted value
+ *   redact.registerFieldRule(pattern, replacement)
+ *   redact.registerValueDetector(name, fn, replacement)
+ *   redact.MARKER                            → '[REDACTED]'
+ */
+
+var DEFAULT_MARKER = "[REDACTED]";
+
+// Field names that are always redacted, regardless of value contents.
+// Match is case-insensitive and includes substring (so 'userPassword' matches).
+var SENSITIVE_FIELDS = [
+  "password", "passwd", "secret", "token", "apikey", "api_key",
+  "authorization", "auth", "ssn", "creditcard", "credit_card",
+  "card_number", "cardnumber", "cvc", "cvv", "pin",
+  "privatekey", "private_key", "passphrase", "session", "sid",
+  "_authtoken", "auth_token", "bearer", "cookie",
+  // Vault-sealed values (don't log even though they're encrypted —
+  // operational logs aren't a place to leak ciphertext shape either)
+  // matched separately by value detector below
+];
+
+// Value-shape detectors. Each takes a string, returns true if the value
+// matches the pattern. Used as a fallback when field-name redaction misses.
+function _luhnCheck(num) {
+  var digits = num.replace(/\D/g, "");
+  if (digits.length < 13 || digits.length > 19) return false;
+  var sum = 0;
+  var alt = false;
+  for (var i = digits.length - 1; i >= 0; i--) {
+    var d = parseInt(digits.charAt(i), 10);
+    if (alt) { d *= 2; if (d > 9) d -= 9; }
+    sum += d;
+    alt = !alt;
+  }
+  return sum % 10 === 0;
+}
+
+var VALUE_DETECTORS = [
+  {
+    name:        "credit-card",
+    test:        function (v) {
+      if (typeof v !== "string") return false;
+      var digits = v.replace(/\s|-/g, "");
+      if (!/^\d{13,19}$/.test(digits)) return false;
+      return _luhnCheck(digits);
+    },
+    replacement: "[REDACTED-CC]",
+  },
+  {
+    name:        "jwt",
+    test:        function (v) {
+      return typeof v === "string" && /^eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(v);
+    },
+    replacement: "[REDACTED-JWT]",
+  },
+  {
+    name:        "pem",
+    test:        function (v) { return typeof v === "string" && /-----BEGIN [A-Z ]+-----/.test(v); },
+    replacement: "[REDACTED-PEM]",
+  },
+  {
+    name:        "ssh-private",
+    test:        function (v) { return typeof v === "string" && /-----BEGIN OPENSSH PRIVATE KEY-----/.test(v); },
+    replacement: "[REDACTED-SSH-KEY]",
+  },
+  {
+    name:        "aws-access-key",
+    test:        function (v) { return typeof v === "string" && /^(AKIA|ASIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASCA)[A-Z0-9]{16}$/.test(v); },
+    replacement: "[REDACTED-AWS-KEY]",
+  },
+  {
+    name:        "vault-sealed",
+    test:        function (v) { return typeof v === "string" && v.startsWith("vault:"); },
+    replacement: "[REDACTED-SEALED]",
+  },
+  {
+    name:        "ssn",
+    test:        function (v) { return typeof v === "string" && /^\d{3}-?\d{2}-?\d{4}$/.test(v); },
+    replacement: "[REDACTED-SSN]",
+  },
+];
+
+var sensitiveFieldsSet = new Set(SENSITIVE_FIELDS);
+var customDetectors = [];
+
+function registerFieldRule(name, replacement) {
+  void replacement; // marker not used here; redact replaces with marker
+  if (typeof name === "string") {
+    sensitiveFieldsSet.add(name.toLowerCase());
+    return;
+  }
+  throw new Error("registerFieldRule expects a string field name");
+}
+
+function registerValueDetector(name, testFn, replacement) {
+  if (typeof testFn !== "function") {
+    throw new Error("registerValueDetector requires a test function");
+  }
+  customDetectors.push({ name: name, test: testFn, replacement: replacement || DEFAULT_MARKER });
+}
+
+function _isSensitiveFieldName(key) {
+  if (typeof key !== "string") return false;
+  var lk = key.toLowerCase();
+  if (sensitiveFieldsSet.has(lk)) return true;
+  // Substring match: 'userPassword' contains 'password'
+  for (var s of sensitiveFieldsSet) {
+    if (lk.indexOf(s) !== -1) return true;
+  }
+  return false;
+}
+
+function _redactValue(value) {
+  if (typeof value !== "string") return value;
+  var allDetectors = VALUE_DETECTORS.concat(customDetectors);
+  for (var i = 0; i < allDetectors.length; i++) {
+    if (allDetectors[i].test(value)) return allDetectors[i].replacement;
+  }
+  return value;
+}
+
+function redact(value, opts) {
+  opts = opts || {};
+  var marker = opts.marker || DEFAULT_MARKER;
+  var maxDepth = opts.maxDepth || 50;
+  return _redact(value, 0, maxDepth, marker, opts.parentKey || null);
+}
+
+function _redact(value, depth, maxDepth, marker, parentKey) {
+  if (depth > maxDepth) return marker;
+  if (value === null || value === undefined) return value;
+  if (typeof value === "string") {
+    if (parentKey && _isSensitiveFieldName(parentKey)) return marker;
+    return _redactValue(value);
+  }
+  if (typeof value === "number" || typeof value === "boolean") {
+    if (parentKey && _isSensitiveFieldName(parentKey)) return marker;
+    return value;
+  }
+  if (Buffer.isBuffer(value) || value instanceof Uint8Array) {
+    return marker;  // never log raw binary
+  }
+  if (Array.isArray(value)) {
+    return value.map(function (v) { return _redact(v, depth + 1, maxDepth, marker, null); });
+  }
+  if (typeof value === "object") {
+    var out = {};
+    for (var k in value) {
+      if (!Object.prototype.hasOwnProperty.call(value, k)) continue;
+      if (_isSensitiveFieldName(k)) {
+        out[k] = marker;
+      } else {
+        out[k] = _redact(value[k], depth + 1, maxDepth, marker, k);
+      }
+    }
+    return out;
+  }
+  return value;
+}
+
+function _resetForTest() {
+  sensitiveFieldsSet = new Set(SENSITIVE_FIELDS);
+  customDetectors = [];
+}
+
+module.exports = {
+  redact:                redact,
+  registerFieldRule:     registerFieldRule,
+  registerValueDetector: registerValueDetector,
+  MARKER:                DEFAULT_MARKER,
+  SENSITIVE_FIELDS:      SENSITIVE_FIELDS,
+  _resetForTest:         _resetForTest,
+};