@blamejs/core 0.4.1

package/lib/jobs.js

@@ -0,0 +1,181 @@
+"use strict";
+/**
+ * jobs — name → handler registry on top of lib/queue.
+ *
+ * lib/queue handles dispatch (queue backend + lease + retry + breaker
+ * + audit emit). lib/jobs is the application-level pattern most apps
+ * actually use: register a handler ahead of time, enqueue by name,
+ * the framework runs the right code without per-call wiring.
+ *
+ *   var jobs = b.jobs.create();
+ *   jobs.define("send-welcome", async function ({ userId }) { … });
+ *   jobs.define("rebuild-index", async function () { … });
+ *
+ *   await jobs.enqueue("send-welcome", { userId: "u-1" });
+ *
+ *   await jobs.start();      // consumers running
+ *   …
+ *   await jobs.shutdown();   // drain in-flight + stop consuming
+ *
+ * Each defined name maps to one queue (queue named after the job
+ * name) with one consumer per jobs instance. Operators wanting
+ * higher concurrency for a specific job pass concurrency in the
+ * define options.
+ *
+ * Built on b.queue, which must be initialized first:
+ *
+ *   b.queue.init({ backends: { primary: { protocol: "local" } } });
+ *   var jobs = b.jobs.create();
+ *   jobs.define(...);
+ *   await jobs.start();
+ *
+ * createApp wires this for you when opts.jobs is a function — the
+ * factory boots queue with the default 'local' backend, instantiates
+ * a jobs registry, calls opts.jobs(jobs) so the operator defines
+ * handlers, and starts consumption before listen().
+ *
+ * Public API:
+ *
+ *   jobs.create(opts?)            → instance
+ *     opts.queueBackend           — backend name to dispatch through
+ *                                   (default 'primary' — the convention
+ *                                    queue.init uses when given a
+ *                                    single-backend config)
+ *     opts.consumerDefaults       — defaults forwarded to queue.consume
+ *                                   ({ concurrency, leaseDurationMs,
+ *                                      pollIntervalMs, fastPollMs })
+ *     opts.allowUnregisteredEnqueue  default false. When true, jobs.enqueue
+ *                                   accepts names not yet defined
+ *                                   (useful in test fixtures or for
+ *                                    apps that expect lazy handler
+ *                                    registration).
+ *
+ *   jobs.define(name, handler, defineOpts?)
+ *     handler signature: async function (job) { … } where job =
+ *     { jobId, queueName, payload, attempts, maxAttempts, traceId,
+ *       classification, enqueuedAt, leaseExpiresAt }.
+ *     defineOpts is forwarded to queue.consume — concurrency etc.
+ *     Throws if name is already defined.
+ *
+ *   await jobs.enqueue(name, payload, enqueueOpts?)
+ *     enqueueOpts is forwarded to queue.enqueue — delaySeconds,
+ *     maxAttempts, traceId, classification.
+ *     Returns { jobId, queueName, enqueuedAt, ... } from queue.enqueue.
+ *
+ *   await jobs.start()
+ *     Begins consuming each defined queue. Idempotent — a second
+ *     start() is a no-op once consumers are already running.
+ *
+ *   await jobs.shutdown(opts?)
+ *     Stops consumers, drains in-flight via queue.shutdown.
+ *     opts.timeoutMs forwards to queue.shutdown.
+ *
+ *   jobs.stats()
+ *     → { defined: [string], started: boolean }
+ */
+var queue = require("./queue");
+var validateOpts = require("./validate-opts");
+var { JobsError } = require("./framework-error");
+
+function create(opts) {
+  opts = opts || {};
+  validateOpts(opts, [
+    "queueBackend", "consumerDefaults", "allowUnregisteredEnqueue",
+  ], "b.jobs");
+  var queueBackend     = opts.queueBackend || "primary";
+  var consumerDefaults = opts.consumerDefaults || {};
+  var allowUnregistered = !!opts.allowUnregisteredEnqueue;
+
+  // name → { handler, defineOpts, consumerHandle (after start) }
+  var registry = new Map();
+  var started = false;
+
+  var _err = JobsError.factory;
+
+  function define(name, handler, defineOpts) {
+    if (typeof name !== "string" || name.length === 0) {
+      throw _err("INVALID_NAME", "jobs.define: name must be a non-empty string", true);
+    }
+    if (typeof handler !== "function") {
+      throw _err("INVALID_HANDLER", "jobs.define: handler must be a function", true);
+    }
+    if (registry.has(name)) {
+      throw _err("DUPLICATE_NAME",
+        "jobs.define: '" + name + "' is already defined", true);
+    }
+    if (started) {
+      // Defining after start would mean the new handler doesn't run
+      // until the next start cycle. Reject loudly so operators don't
+      // wonder why their newly-defined handler is silent.
+      throw _err("ALREADY_STARTED",
+        "jobs.define: cannot register '" + name + "' after start() — " +
+        "define all handlers before calling start()", true);
+    }
+    registry.set(name, {
+      handler:    handler,
+      defineOpts: defineOpts || {},
+    });
+  }
+
+  async function enqueue(name, payload, enqueueOpts) {
+    if (typeof name !== "string" || name.length === 0) {
+      throw _err("INVALID_NAME", "jobs.enqueue: name must be a non-empty string", true);
+    }
+    if (!allowUnregistered && !registry.has(name)) {
+      throw _err("UNDEFINED_NAME",
+        "jobs.enqueue: '" + name + "' has no registered handler. " +
+        "Either define(name, handler) first, or pass " +
+        "{ allowUnregisteredEnqueue: true } to jobs.create.", true);
+    }
+    return await queue.enqueue(name, payload, Object.assign(
+      { backend: queueBackend },
+      enqueueOpts || {}
+    ));
+  }
+
+  async function start() {
+    if (started) return;
+    var consumerOpts = Object.assign({ backend: queueBackend }, consumerDefaults);
+    registry.forEach(function (entry, name) {
+      var perJobOpts = Object.assign({}, consumerOpts, entry.defineOpts);
+      entry.consumerHandle = queue.consume(name, entry.handler, perJobOpts);
+    });
+    started = true;
+  }
+
+  async function shutdown(shutdownOpts) {
+    if (!started) {
+      // Even when not started, queue.shutdown handles its own state.
+      try { await queue.shutdown(shutdownOpts); } catch (_e) {}
+      return;
+    }
+    started = false;
+    await queue.shutdown(shutdownOpts);
+    // Don't clear the registry — operators inspecting stats() after
+    // shutdown should still see what was defined; only the running
+    // state changes.
+  }
+
+  function stats() {
+    return {
+      defined:  Array.from(registry.keys()),
+      started:  started,
+    };
+  }
+
+  function _resetForTest() {
+    registry.clear();
+    started = false;
+  }
+
+  return {
+    define:        define,
+    enqueue:       enqueue,
+    start:         start,
+    shutdown:      shutdown,
+    stats:         stats,
+    _resetForTest: _resetForTest,
+  };
+}
+
+module.exports = { create: create };

package/lib/lazy-require.js

@@ -0,0 +1,48 @@
+"use strict";
+/**
+ * Lazy-require — cached deferred-load helper.
+ *
+ * Centralizes the pattern used to break circular-load chains between
+ * modules that depend on each other through different code paths
+ * (audit ↔ db, vault ↔ db, middleware ↔ audit). Every dependent module
+ * was carrying its own copy of:
+ *
+ *   var _db = null;
+ *   function db() { if (!_db) _db = require("./db"); return _db; }
+ *   // and `_db = null;` in _resetForTest
+ *
+ * `lazyRequire(loader)` returns a callable getter `db()` that does the
+ * cache-on-first-call dance once, plus a `db.reset()` for test
+ * teardown. The `loader` is a function (NOT a path string) so the
+ * inner `require()` resolves relative to the CALLER's __filename, not
+ * lib/lazy-require.js — passing a string here would break relative
+ * paths from any module not co-located with lazy-require.js.
+ *
+ * Usage:
+ *
+ *   var lazyRequire = require("./lazy-require");
+ *   var db = lazyRequire(function () { return require("./db"); });
+ *   // ... later ...
+ *   db().findOne(...);   // first call resolves + caches
+ *   // in _resetForTest:
+ *   db.reset();
+ */
+
+function lazyRequire(loader) {
+  if (typeof loader !== "function") {
+    throw new Error("lazyRequire(loader): loader must be a function returning the require() result");
+  }
+  // Separate `loaded` flag from `cached` so a loader that legitimately
+  // returns null/undefined/0/false caches that value instead of re-running
+  // on every subsequent call.
+  var loaded = false;
+  var cached;
+  function get() {
+    if (!loaded) { cached = loader(); loaded = true; }
+    return cached;
+  }
+  get.reset = function () { loaded = false; cached = undefined; };
+  return get;
+}
+
+module.exports = lazyRequire;

package/lib/log-stream-local.js

@@ -0,0 +1,137 @@
+"use strict";
+/**
+ * Local file-based log sink with append-only journaling + rotation.
+ *
+ * Each event is written as one JSON line (jsonl format — one row per
+ * event). Files rotate by size (default 100 MiB) and/or age (default
+ * 7 days). Old rotations are gzip-compressed automatically and capped at
+ * a configured count (default 30) — older rotations are deleted.
+ *
+ * The active log file is opened in append mode and never updated in place.
+ * Operators can apply OS-level immutability (Linux chattr +a) to the
+ * directory if they want stronger tamper-resistance — the framework
+ * doesn't fight that; appends still work.
+ *
+ * Config:
+ *   {
+ *     dir:                './logs/operational'
+ *     maxFileBytes:       C.BYTES.mib(100)
+ *     maxFileAgeMs:       C.TIME.days(7)
+ *     keepRotations:      30
+ *     compressRotations:  true
+ *     fileMode:           0o600
+ *     fileNamePrefix:     'blamejs'
+ *   }
+ */
+var fs = require("fs");
+var path = require("path");
+var zlib = require("zlib");
+var atomicFile = require("./atomic-file");
+var C = require("./constants");
+var { LogStreamError } = require("./framework-error");
+
+var DEFAULTS = {
+  maxFileBytes:      C.BYTES.mib(100),
+  maxFileAgeMs:      C.TIME.days(7),
+  keepRotations:     30,
+  compressRotations: true,
+  fileMode:          0o600,
+  fileNamePrefix:    "blamejs",
+};
+
+var _err = LogStreamError.factory;
+
+function create(config) {
+  if (!config || !config.dir) throw new Error("log-stream local requires { dir }");
+  var cfg = Object.assign({}, DEFAULTS, config);
+  var dir = path.resolve(cfg.dir);
+  if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
+
+  var activePath = path.join(dir, cfg.fileNamePrefix + ".log");
+  var fd = null;
+  var openedAt = 0;
+  var bytesWritten = 0;
+
+  function _open() {
+    fd = fs.openSync(activePath, "a", cfg.fileMode);
+    openedAt = Date.now();
+    try {
+      var stat = fs.fstatSync(fd);
+      bytesWritten = stat.size;
+    } catch (_e) {
+      bytesWritten = 0;
+    }
+  }
+  _open();
+
+  function _shouldRotate() {
+    if (cfg.maxFileBytes && bytesWritten >= cfg.maxFileBytes) return true;
+    if (cfg.maxFileAgeMs && (Date.now() - openedAt) >= cfg.maxFileAgeMs) return true;
+    return false;
+  }
+
+  function _rotate() {
+    try {
+      if (fd != null) { try { fs.closeSync(fd); } catch (_e) {} fd = null; }
+      // Build rotated filename: blamejs-YYYYMMDDTHHMMSSZ.log
+      var stamp = new Date().toISOString().replace(/[-:]/g, "").replace(/\.\d{3}Z$/, "Z");
+      var rotated = path.join(dir, cfg.fileNamePrefix + "-" + stamp + ".log");
+      if (fs.existsSync(activePath)) {
+        fs.renameSync(activePath, rotated);
+        if (cfg.compressRotations) {
+          var data = fs.readFileSync(rotated);
+          var gz = zlib.gzipSync(data);
+          atomicFile.writeSync(rotated + ".gz", gz, { fileMode: cfg.fileMode });
+          fs.unlinkSync(rotated);
+        }
+      }
+      _pruneOld();
+    } finally {
+      _open();
+    }
+  }
+
+  function _pruneOld() {
+    if (!cfg.keepRotations || cfg.keepRotations <= 0) return;
+    var entries = atomicFile.listDir(dir, {
+      filter: function (f) {
+        return f.startsWith(cfg.fileNamePrefix + "-") &&
+          (f.endsWith(".log") || f.endsWith(".log.gz"));
+      },
+      includeStat: true,
+    }).sort(function (a, b) { return b.mtimeMs - a.mtimeMs; });   // newest first
+    for (var i = cfg.keepRotations; i < entries.length; i++) {
+      try { fs.unlinkSync(entries[i].fullPath); } catch (_e) { /* best effort */ }
+    }
+  }
+
+  function emit(record) {
+    if (_shouldRotate()) _rotate();
+    var line = JSON.stringify(record) + "\n";
+    var buf = Buffer.from(line, "utf8");
+    fs.writeSync(fd, buf, 0, buf.length, null);
+    bytesWritten += buf.length;
+    return Promise.resolve({ bytes: buf.length });
+  }
+
+  function close() {
+    if (fd != null) {
+      try { fs.fsyncSync(fd); } catch (_e) { /* best effort */ }
+      try { fs.closeSync(fd); } catch (_e) {}
+      fd = null;
+    }
+    return Promise.resolve();
+  }
+
+  function getActivePath() { return activePath; }
+
+  return {
+    protocol:      "local",
+    emit:          emit,
+    close:         close,
+    rotate:        function () { _rotate(); return Promise.resolve(); },
+    getActivePath: getActivePath,
+  };
+}
+
+module.exports = { create: create };

package/lib/log-stream-webhook.js

@@ -0,0 +1,170 @@
+"use strict";
+/**
+ * Generic webhook log sink — HTTP POST one event (or a batch) at a time.
+ *
+ * Covers most SIEM ingestion endpoints with simple HTTP POST + JSON body:
+ *   Splunk HEC                  — { auth: 'header', headers: { Authorization: 'Splunk <token>' } }
+ *   Datadog Logs                — { auth: 'header', headers: { 'DD-API-KEY': '<key>' } }
+ *   Sumo Logic HTTP source      — no auth (URL is the secret)
+ *   Grafana Loki push API       — { auth: 'basic' }
+ *   Generic OpenTelemetry HTTP  — { headers: { 'Content-Type': 'application/x-protobuf' } } — caller controls body shape
+ *
+ * Streaming model: events accumulate in a per-sink queue; a worker drains
+ * it in batches (default size 100, max age 5s) to balance throughput
+ * against latency. On webhook 5xx / network errors the batch retries with
+ * exponential backoff (via the framework's retry module). On permanent 4xx
+ * the batch is dropped and an audit event is recorded.
+ *
+ * Config:
+ *   {
+ *     url:                    'https://siem.example.com/ingest'
+ *     auth:                   'none'|'bearer'|'basic'|'header'
+ *     token / username+password / headers
+ *     batchSize:              100
+ *     maxBatchAgeMs:          C.TIME.seconds(5)
+ *     contentType:            'application/json'
+ *     bodyShape:              'array' | 'ndjson' | 'singleEnvelope'
+ *     timeoutMs:              C.TIME.seconds(30)
+ *     retry:                  { maxAttempts, baseDelayMs, ... }
+ *     bufferLimit:            10000   // ring-buffer cap; drops oldest on overflow
+ *   }
+ */
+var C = require("./constants");
+var retryHelper = require("./retry");
+var { LogStreamError } = require("./framework-error");
+var httpClient = require("./http-client");
+var safeUrl = require("./safe-url");
+var authHeader = require("./auth-header");
+
+// Webhook responses are ack-only (status + small body). 1 MiB cap is
+// generous; misbehaving log-aggregator endpoints don't get to OOM us.
+var MAX_RESPONSE_BYTES = C.BYTES.mib(1);
+
+var DEFAULTS = {
+  batchSize:     100,
+  maxBatchAgeMs: C.TIME.seconds(5),
+  contentType:   "application/json",
+  bodyShape:     "array",
+  timeoutMs:     C.TIME.seconds(30),
+  bufferLimit:   10000,
+};
+
+var _err = LogStreamError.factory;
+
+// Auth-header construction is delegated to lib/auth-header for the
+// none/bearer/basic triple. The "header" mode (pass-through arbitrary
+// headers) is handled here — it's not an auth scheme, just header
+// merging that's traditionally been bundled in the same config knob.
+function _authHeaders(config) {
+  if (config.auth === "header") return Object.assign({}, config.headers || {});
+  return authHeader.fromConfig(config);
+}
+
+function _post(url, body, headers, timeoutMs, allowedProtocols) {
+  return httpClient.request({
+    method:           "POST",
+    url:              url,
+    headers:          headers,
+    body:             body,
+    idleTimeoutMs:    timeoutMs,
+    maxResponseBytes: MAX_RESPONSE_BYTES,
+    errorClass:       LogStreamError,
+    allowedProtocols: allowedProtocols,
+  });
+}
+
+function _serializeBatch(records, shape) {
+  if (shape === "ndjson") {
+    return Buffer.from(records.map(function (r) { return JSON.stringify(r); }).join("\n") + "\n", "utf8");
+  }
+  if (shape === "singleEnvelope") {
+    return Buffer.from(JSON.stringify({ events: records }), "utf8");
+  }
+  // default: array
+  return Buffer.from(JSON.stringify(records), "utf8");
+}
+
+function create(config) {
+  if (!config || !config.url) throw new Error("log-stream webhook requires { url }");
+  var cfg = Object.assign({}, DEFAULTS, config);
+  // Fail fast on misconfig — validate URL shape + scheme at create time
+  // rather than at first emit. Default is HTTPS-only; operators with an
+  // internal cleartext aggregator pass cfg.allowedProtocols
+  // (safeUrl.ALLOW_HTTP_ALL).
+  safeUrl.parse(cfg.url, {
+    allowedProtocols: cfg.allowedProtocols || safeUrl.ALLOW_HTTP_TLS,
+    errorClass:       LogStreamError,
+  });
+  var headers = Object.assign({ "Content-Type": cfg.contentType }, _authHeaders(cfg));
+  var buffer = [];
+  var dropCount = 0;
+  var flushTimer = null;
+  var inFlight = false;
+  var closed = false;
+
+  function _scheduleFlush() {
+    if (flushTimer) return;
+    flushTimer = setTimeout(function () { flushTimer = null; _flush(); }, cfg.maxBatchAgeMs);
+    flushTimer.unref();
+  }
+
+  async function _flush() {
+    if (inFlight) return;
+    if (buffer.length === 0) return;
+    inFlight = true;
+    try {
+      while (buffer.length > 0 && !closed) {
+        var batch = buffer.splice(0, cfg.batchSize);
+        var body = _serializeBatch(batch, cfg.bodyShape);
+        try {
+          await retryHelper.withRetry(function () {
+            return _post(cfg.url, body, headers, cfg.timeoutMs, cfg.allowedProtocols);
+          }, cfg.retry);
+        } catch {
+          // Batch permanently rejected — surface via the dropped counter.
+          // Caller's audit hook recorded the drop already at the dispatcher.
+          break;
+        }
+      }
+    } finally {
+      inFlight = false;
+      if (buffer.length > 0) _scheduleFlush();
+    }
+  }
+
+  function emit(record) {
+    if (closed) return Promise.resolve({ accepted: false, reason: "sink closed" });
+    if (buffer.length >= cfg.bufferLimit) {
+      buffer.shift();   // drop oldest
+      dropCount += 1;
+    }
+    buffer.push(record);
+    if (buffer.length >= cfg.batchSize) {
+      // Don't await — non-blocking flush. Caller's emit returns immediately.
+      _flush().catch(function () {});
+    } else {
+      _scheduleFlush();
+    }
+    return Promise.resolve({ accepted: true, queued: buffer.length });
+  }
+
+  async function close() {
+    closed = true;
+    if (flushTimer) { clearTimeout(flushTimer); flushTimer = null; }
+    await _flush();
+  }
+
+  function stats() {
+    return { queued: buffer.length, dropped: dropCount, inFlight: inFlight };
+  }
+
+  return {
+    protocol:  "webhook",
+    emit:      emit,
+    close:     close,
+    stats:     stats,
+    flush:     _flush,
+  };
+}
+
+module.exports = { create: create };