@blamejs/core 0.6.33 → 0.6.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1 -0
- package/README.md +1 -1
- package/index.js +2 -0
- package/lib/cache.js +61 -0
- package/lib/db.js +9 -8
- package/lib/framework-schema.js +11 -10
- package/lib/pubsub-cluster.js +154 -0
- package/lib/pubsub-redis.js +160 -0
- package/lib/pubsub.js +357 -0
- package/lib/redis-client.js +46 -1
- package/lib/websocket-channels.js +131 -217
- package/package.json +1 -1
- package/sbom.cyclonedx.json +6 -6
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,7 @@ upgrading across more than a few patches at a time.
|
|
|
8
8
|
|
|
9
9
|
## v0.6.x
|
|
10
10
|
|
|
11
|
+
- **0.6.34** (2026-05-02) — distributed pub/sub primitive. **`b.pubsub`** — single API for cross-node fan-out across the framework, with three backends. `local` dispatches in-process to registered handlers (zero coordination overhead for single-node deploys); `cluster` polls a shared `_blamejs_pubsub_messages` table at `pollIntervalMs` (default 100ms) and dispatches new rows past `lastSeenId` from other nodes (publishedBy=self filter prevents loopback); `redis` opens a SUBSCRIBE-mode connection on `lib/redis-client.js` (now with new push-message demultiplexing — server-pushed `["message", channel, payload]` and `["pmessage", pattern, channel, payload]` arrays route through `setOnPushMessage` instead of consuming a pending request slot, while SUBSCRIBE / UNSUBSCRIBE acks still flow through the normal command pipeline). Per-instance nonce stamped on outgoing redis payloads so the SUBSCRIBE socket recognizes its own publishes and skips the loopback (without it every same-instance publish would fire local handlers twice). Operator API: `ps.subscribe(channel, handler) → token`, `ps.subscribePattern(pattern, handler) → token` (glob-style on local + cluster, native PSUBSCRIBE on redis), `ps.unsubscribe(token)`, `await ps.publish(channel, payload) → { local, remote }`, `await ps.close()`. `topicPrefix` opt scopes channel names so independent pubsub instances sharing a backend (cache invalidation + websocket channels + custom) don't collide. Handler errors are caught + logged via the framework's boot logger; they never abort dispatch to other handlers on the same channel. **`lib/websocket-channels.js`** — replaces the inline cluster-poll-and-fan-out logic with `b.pubsub` consumption. The hub now owns one pubsub instance per primitive; cross-node delivery is `pubsub.subscribe` per channel the hub joins, with the hub's `_localDispatch` as the handler. Per-channel pubsub subscription is refcounted by local conn count (subscribe on first conn, unsubscribe on last). The `_blamejs_ws_messages` table is renamed to `_blamejs_pubsub_messages` (column `channel` → `topic`) reflecting the generalization; pre-v1, no compat shim — operators upgrading wipe the previous table (which carried only ephemeral fan-out rows with default 60s retention). **`b.cache.create({ invalidationPubsub })`** — passing a `b.pubsub.create()` instance auto-publishes on every successful `del` / `clear` / `invalidateTag`, and subscribes for the same events so other cache instances on other nodes (or processes sharing the pubsub backend) react locally — primarily useful for the memory backend so stale per-node entries don't survive a global tag wipe. Re-entrancy guard prevents inbound invalidation events from re-publishing (no fan-out loops). Tests: 29 layer-0 assertions covering local + cluster behavior, topicPrefix isolation, pattern subscribe, handler error isolation, post-close error path, end-to-end cache invalidationPubsub fan-out via local pubsub. 10 live integration assertions against the docker redis container covering single-instance round-trip, PSUBSCRIBE pattern, cross-instance fan-out, and cache invalidation through redis PUB/SUB. Smoke 6817 / wiki e2e 178 / per-primitive integration 15 files / wiki integration 32 / shellcheck clean / eslint clean.
|
|
11
12
|
- **0.6.33** (2026-05-02) — log-stream sinks parity sweep. **`b.logStream` syslog sink (RFC 5424)** — first-class `protocol: "syslog"` for `b.logStream.init({ sinks: { ... } })`; new module `lib/log-stream-syslog.js`. URL-driven transport selection: `udp://host:514` / `tcp://host:514` / `tls://host:6514`. UDP is best-effort one-datagram-per-record; TCP / TLS use RFC 6587 octet-counting framing (`<length> <message>`) so collectors that prefer it over the older non-transparent newline framing parse cleanly. TLS is TLS 1.3 minimum; operators with private CAs pass `ca` (PEM string or array) for trust pinning, `servername` for SNI override (auto-suppressed on IP literals per the v0.6.28 redis-client convention). Outgoing records are formatted with PRI = `(facility << 3) | severity`, default facility `local0` (16), severity mapped from the framework's level field (debug=7 / info=6 / warn=4 / error=3); operators override via `facility`, `appName` (default `blamejs`), `procId` (default `process.pid`), `hostname` (default `os.hostname()`), `structuredData` (default `-`). Meta is JSON-encoded into the MSG body so the structured payload survives the wire as a single token. TCP / TLS reconnect with exponential backoff (default 250ms→30s); records buffer during the down window with `bufferLimit`-bounded oldest-drop semantics, replay on reconnect. `close()` waits up to 3s for an in-flight (re)connect to drain the buffer before tearing down — without this the slower TLS handshake raced fire-and-forget shutdown emits and silently dropped records. Operator-supplied `onDrop({reason, batch, error})` surfaces every drop class (`overflow` / `udp-send-error` / `write-error` / `sink-closed`). Removed `syslog` from `DEFERRED_PROTOCOLS`. **`b.logStream` cloudwatch sink: `autoCreate`** — pass `{ autoCreate: true }` to have the framework issue `CreateLogGroup` + `CreateLogStream` on first emit so operators provisioning collectors via env vars / runbooks don't need a separate aws-cli step. Idempotent: AWS's `ResourceAlreadyExistsException` is treated as success on both calls. Hard failures (5xx, AccessDenied, etc.) drop the batch with `onDrop` reason `autocreate-failed`. Default remains `autoCreate: false` so the AWS posture (operator pre-creates via aws / CDK / Terraform) stays the recommended path; `autoCreate` exists for ephemeral / dynamic-stream deployments where pre-provisioning is impractical. **Test suite**: 19 new framework-level cloudwatch assertions cover autoCreate fires both Create calls in order before PutLogEvents, ResourceAlreadyExists doesn't abort the post-create PutLogEvents, hard 5xx during CreateLogGroup drops the batch via `onDrop`, autoCreate=false skips Create calls entirely. Live syslog integration covers UDP / TCP wire delivery against the docker syslog-ng container (`/var/log/blamejs-test.log`) plus an in-process `tls.createServer` receiver that asserts the on-the-wire RFC 6587 octet-counting framing + RFC 5424 PRI / timestamp / structured-data slot the framework emits over TLS. **Test infrastructure fix**: `docker/init/generate-certs.sh` was overwriting an existing CA when re-run with `.complete` removed, invalidating every previously-issued leaf cert; now reuses an existing `(ca.crt, ca.key)` pair across `.complete`-only resets and only generates a fresh CA when neither file is present. Smoke 6790 / wiki e2e 178 / per-primitive integration 14 files (log-stream up from 16→18 checks, cloudwatch suite up from 39→58 assertions) / wiki integration 32 / shellcheck clean / eslint clean.
|
|
12
13
|
- **0.6.32** (2026-05-02) — E1 from the v0.6.x scope plan. **`b.cache` Redis backend** — first-class `backend: "redis"` for `b.cache.create({...})`, no operator-supplied glue needed. New module `lib/cache-redis.js` consumes `lib/redis-client.js` directly; storage layout: `<namespace>:e:<key>` (STRING, JSON-encoded value, PEXPIREAT-bounded), `<namespace>:t:<tag>` (SET, cacheKeys carrying that tag — powers `invalidateTag` fan-out), `<namespace>:k:<key>:tags` (SET, tags this key carries — powers per-key tag cleanup on `del`/`set`-overwrite, expires alongside the entry). TTL is enforced by Redis itself (PEXPIREAT) so the framework's sweeper is a no-op for this backend; sliding TTL bumps the entry's expiry on every read when `slidingTtl: true` AND `ttlMs` is finite. New opts on `cache.create`: `redisUrl` (required when `backend: "redis"`), `redisPassword`, `redisUsername`, `redisTls`, `redisCa` (private-CA trust pinning per v0.6.28), `redisServername`, `redisConnectTimeoutMs`, `redisCommandTimeoutMs`, `redisMaxReconnectAttempts`. Concurrency: invalidateTag filters out ghost entries (key already PEXPIRE'd from the keyspace but lingering in a tag SET) by EXISTS-checking before del. Lazy connect — `cache.create({backend:"redis"})` stays sync-safe; first op opens the socket. Tests: 10 new live integration assertions on top of the existing memory + custom-backend coverage (set+get round-trip, has, del, complex JSON value round-trip, short-TTL Redis-side expiry, multi-tag invalidateTag fan-out + preservation of untagged entries, wrap() single-flight memoization through Redis). Smoke 6780 / wiki e2e 178 / per-primitive integration 14 files (cache up from 15→25 checks) / wiki integration green.
|
|
13
14
|
- **0.6.31** (2026-05-02) — lazy-deferral cleanup wave from v0.6.27. **A1: queue-redis priority ordering** — `b.queue.enqueue({ priority })` on the Redis backend now matches queue-local's `ORDER BY priority DESC, availableAt ASC, enqueuedAt ASC` semantics. The Redis ZSET stays scored by availableAt only (so ZRANGEBYSCORE 0..nowMs cleanly filters ready vs not-yet-ready jobs), but `LEASE_LUA` now over-fetches `maxRows*5` candidates by score, HMGETs priority + availableAt + enqueuedAt for each, sorts server-side in Lua by the same triple queue-local sorts on, and leases the top `maxRows`. Closes the v0.6.27 deferral that punted priority ordering as "ZSET-incompatible". **A2: queue-redis flow `dependsOn` cascade** — `b.queue.enqueue({ flowId, flowChildName, dependsOn })` on the Redis backend now releases dependent jobs when their parents complete, mirroring queue-local's `_maybeReleaseFlowChildren`. A per-flow `<prefix>:flow:<flowId>` Redis SET tracks every job in the flow; `complete()` walks the set, looks up siblings whose `dependsOn` includes the just-completed job (by id OR `flowChildName`) AND every other dep is also satisfied, and HSET-bumps their `availableAt` to now + ZADDs them into the ready zset. The flow set persists across `complete()` calls so a later sibling's deps check can still find earlier-done parents by name; `purge()` cleans the set on queue teardown. Closes the v0.6.27 deferral. **A4: WebSocket per-message-deflate (RFC 7692)** — `b.websocket.handleUpgrade` negotiates the `permessage-deflate` extension when the client offers it, accepting `client_max_window_bits` / `server_max_window_bits` constraints (8-15 range, default 15) and always asserting `client_no_context_takeover` + `server_no_context_takeover` so every message uses a fresh zlib state. The connection's send path compresses TEXT/BINARY frames via `zlib.deflateRawSync`, strips the 4-byte `0x00 0x00 0xff 0xff` trailer per RFC 7692 §7.2.1, and sets RSV1 on the first frame of each compressed message. The receive path appends the trailer back and inflates via `zlib.inflateRawSync`. RSV1 on a continuation frame, RSV1 without negotiated extension, RSV2/RSV3 set, and decompressed payload exceeding `maxMessageBytes` all close with the appropriate RFC 6455 status (`PROTOCOL_ERROR`, `INVALID_PAYLOAD`, `MESSAGE_TOO_BIG`). Operator opt-out: pass `permessageDeflate: false` to `handleUpgrade` to refuse the extension even when offered. Tests cover the 101-handshake echo of negotiated params, real compression on a redundant payload (4000 bytes → ~30 bytes on the wire), uncompressed-frame round-trip on the same connection (mixed-mode per RFC 7692 §6), and unknown-extension graceful ignore. v0.6.27's third lazy deferral A3 (TLS-redis live test) was already covered by `test/integration/redis-client-tls.test.js` shipped in v0.6.28. Smoke 6780 / wiki e2e 178 / per-primitive integration 14 files green / wiki integration green.
|
package/README.md
CHANGED
|
@@ -46,7 +46,7 @@ The framework bundles the surface a typical Node app reaches for. Every primitiv
|
|
|
46
46
|
- **Crypto** — envelope-versioned PQC at rest (ML-KEM-1024 + P-384 hybrid, XChaCha20-Poly1305, SHAKE256), vault sealing, field-level crypto + cryptographic erasure (`b.cryptoField.eraseRow`), signed webhooks (SLH-DSA-SHAKE-256f), ECIES API encryption (`b.crypto`, `b.vault`, `b.webhook`); pure-JS mTLS CA that issues clientAuth / serverAuth / dual-EKU certs with SAN entries and auto-detects the highest-PQC signature algorithm the vendored x509 library accepts (today: ECDSA-P384-SHA384 bridge; self-upgrades to SLH-DSA / ML-DSA when the X.509 ecosystem catches up), PQC TLS gates inbound + outbound (`b.mtlsCa`, `b.pqcGate`, `b.pqcAgent`).
|
|
47
47
|
- **HTTP** — router with schema-validated routes + OpenAPI publication; full middleware stack (CSRF, CORS, rate-limit, security headers, CSP nonce, body parser, compression, SSE, request log, request-time DB role binding via `b.middleware.dbRoleFor`, in-process CIDR fence via `b.middleware.networkAllowlist`) wired by `createApp`; HTTP/1.1 + HTTP/2 outbound client with SSRF gate (cloud-metadata IPs hard-denied unconditionally; private / loopback / link-local overridable per call), scheme + userinfo + per-host (wildcard / per-method) destination allowlist, redirects, multipart, interceptors, progress, encrypted cookie jar (`b.httpClient`, `b.ssrfGuard`, `b.safeUrl`); operator-tunable network configurability — env-driven NTP / NTS (RFC 8915 authenticated time), IPv4-or-IPv6 NTP servers, DNS with IPv6 / DoH / DoT (private-CA trust pinning via `opts.ca`) / cache / lookup timeout, outbound HTTP proxy (`HTTP_PROXY` / `HTTPS_PROXY` / `NO_PROXY`), runtime DPI trust-store CA additions, application-level heartbeats, TCP socket defaults (`b.network`).
|
|
48
48
|
- **Defensive parsers** — `b.safeJson`, `b.safeBuffer`, `b.safeSql`, `b.safeSchema`, `b.parsers` (XML / TOML / YAML / .env), `b.config` (schema-validated env), `b.fileType` magic-byte content classification with deny-on-upload categories (image / document / archive / executable / etc.).
|
|
49
|
-
- **Communication** — WebSockets with channel/room fan-out across cluster replicas (`b.websocket`, `b.websocketChannels`); mail with multipart + attachments + DKIM + calendar invites + bounce intake (`b.mail`, `b.mailBounce`); generic notification dispatcher with operator-supplied transports (`b.notify`).
|
|
49
|
+
- **Communication** — WebSockets with channel/room fan-out across cluster replicas (`b.websocket`, `b.websocketChannels`); generic distributed pub/sub with cluster-table / Redis PUB/SUB / custom backends (`b.pubsub`); mail with multipart + attachments + DKIM + calendar invites + bounce intake (`b.mail`, `b.mailBounce`); generic notification dispatcher with operator-supplied transports (`b.notify`).
|
|
50
50
|
- **Observability** — tamper-evident audit chain with SLH-DSA-signed checkpoints, metrics, tracing (OTel pass-through when wired), PII redaction, log-stream sinks (local file rotation, generic webhook, OTLP/HTTP-JSON to an OTel collector, AWS CloudWatch Logs via SigV4 with optional autoCreate, RFC 5424 syslog over UDP/TCP/TLS), OTLP/HTTP-JSON exporter for traces + metrics (`b.audit`, `b.metrics`, `b.tracing`, `b.redact`, `b.logStream`, `b.otelExport`); operator-callable boot-time security policy assertions (`b.security.assertProduction`) and tamper-evident config-baseline drift detection signed with the audit-signing key (`b.configDrift`).
|
|
51
51
|
- **i18n** — CLDR plural rules, Accept-Language negotiation, Intl formatters, RTL (`b.i18n`).
|
|
52
52
|
- **Format helpers** — RFC 4180 CSV with Excel formula-injection prevention (`b.csv`), RFC 9562 UUID v4 + v7 (`b.uuid`), URL-safe slugs (`b.slug`), TZ-aware datetime (`b.time`), ZIP creation (`b.archive`), HMAC-signed cursor pagination (`b.pagination`), HTML form rendering + validation + CSRF (`b.forms`).
|
package/index.js
CHANGED
|
@@ -113,6 +113,7 @@ var time = require("./lib/time");
|
|
|
113
113
|
var uuid = require("./lib/uuid");
|
|
114
114
|
var mail = require("./lib/mail");
|
|
115
115
|
var mailBounce = require("./lib/mail-bounce");
|
|
116
|
+
var pubsub = require("./lib/pubsub");
|
|
116
117
|
var websocketChannels = require("./lib/websocket-channels");
|
|
117
118
|
var nonceStore = require("./lib/nonce-store");
|
|
118
119
|
var scheduler = require("./lib/scheduler");
|
|
@@ -227,6 +228,7 @@ module.exports = {
|
|
|
227
228
|
uuid: uuid,
|
|
228
229
|
mail: mail,
|
|
229
230
|
mailBounce: mailBounce,
|
|
231
|
+
pubsub: pubsub,
|
|
230
232
|
websocketChannels: websocketChannels,
|
|
231
233
|
nonceStore: nonceStore,
|
|
232
234
|
scheduler: scheduler,
|
package/lib/cache.js
CHANGED
|
@@ -739,6 +739,15 @@ function create(opts) {
|
|
|
739
739
|
"redisUrl", "redisPassword", "redisUsername", "redisTls", "redisCa",
|
|
740
740
|
"redisServername", "redisConnectTimeoutMs", "redisCommandTimeoutMs",
|
|
741
741
|
"redisMaxReconnectAttempts",
|
|
742
|
+
// Cross-node invalidation: when set, every successful
|
|
743
|
+
// del/clear/invalidateTag publishes an event on the supplied
|
|
744
|
+
// pubsub instance. Other cache instances on other nodes (or in
|
|
745
|
+
// other processes sharing the pubsub backend) react locally —
|
|
746
|
+
// mostly useful for the memory backend so stale per-node entries
|
|
747
|
+
// don't survive a global tag wipe. The cluster + redis backends
|
|
748
|
+
// are coherent by virtue of their shared store, but a hot
|
|
749
|
+
// memory-tier on top of either still benefits.
|
|
750
|
+
"invalidationPubsub",
|
|
742
751
|
], "cache");
|
|
743
752
|
_validateCreateOpts(opts);
|
|
744
753
|
|
|
@@ -756,6 +765,19 @@ function create(opts) {
|
|
|
756
765
|
var audit = opts.audit || null;
|
|
757
766
|
var operatorObs = opts.observability || null;
|
|
758
767
|
var clock = opts.clock || function () { return Date.now(); };
|
|
768
|
+
var invalidationPubsub = opts.invalidationPubsub || null;
|
|
769
|
+
if (invalidationPubsub && (
|
|
770
|
+
typeof invalidationPubsub.publish !== "function" ||
|
|
771
|
+
typeof invalidationPubsub.subscribe !== "function" ||
|
|
772
|
+
typeof invalidationPubsub.unsubscribe !== "function")) {
|
|
773
|
+
throw _err("BAD_OPT",
|
|
774
|
+
"cache.create: invalidationPubsub must implement { publish, subscribe, unsubscribe } (b.pubsub.create instance)");
|
|
775
|
+
}
|
|
776
|
+
var invalidationChannel = "cache:" + namespace + ":invalidate";
|
|
777
|
+
var invalidationToken = null;
|
|
778
|
+
// Re-entrancy guard — when we receive an invalidation event from
|
|
779
|
+
// another node we MUST NOT re-publish it (infinite fan-out loop).
|
|
780
|
+
var inboundInvalidation = false;
|
|
759
781
|
|
|
760
782
|
function emitObs(name, labels) {
|
|
761
783
|
try {
|
|
@@ -908,6 +930,7 @@ function create(opts) {
|
|
|
908
930
|
}
|
|
909
931
|
if (existed) emitObs("cache.del", { namespace: namespace });
|
|
910
932
|
softExpiry.delete(key);
|
|
933
|
+
_publishInvalidation({ kind: "del", key: key });
|
|
911
934
|
return existed;
|
|
912
935
|
}
|
|
913
936
|
|
|
@@ -946,6 +969,7 @@ function create(opts) {
|
|
|
946
969
|
inflight.clear();
|
|
947
970
|
swrInflight.clear();
|
|
948
971
|
softExpiry.clear();
|
|
972
|
+
_publishInvalidation({ kind: "clear" });
|
|
949
973
|
return purged;
|
|
950
974
|
}
|
|
951
975
|
|
|
@@ -1003,6 +1027,7 @@ function create(opts) {
|
|
|
1003
1027
|
// drop matches clear()'s safer-than-stale posture.
|
|
1004
1028
|
inflight.clear();
|
|
1005
1029
|
swrInflight.clear();
|
|
1030
|
+
_publishInvalidation({ kind: "tag", tag: tag });
|
|
1006
1031
|
return purged;
|
|
1007
1032
|
}
|
|
1008
1033
|
|
|
@@ -1134,9 +1159,45 @@ function create(opts) {
|
|
|
1134
1159
|
return promise;
|
|
1135
1160
|
}
|
|
1136
1161
|
|
|
1162
|
+
function _publishInvalidation(ev) {
|
|
1163
|
+
if (!invalidationPubsub || inboundInvalidation) return;
|
|
1164
|
+
try { invalidationPubsub.publish(invalidationChannel, ev); }
|
|
1165
|
+
catch (_e) { /* publish best-effort — local invalidation already happened */ }
|
|
1166
|
+
}
|
|
1167
|
+
|
|
1168
|
+
async function _onInboundInvalidation(ev /*, meta */) {
|
|
1169
|
+
if (!ev || closed) return;
|
|
1170
|
+
inboundInvalidation = true;
|
|
1171
|
+
try {
|
|
1172
|
+
if (ev.kind === "tag" && typeof ev.tag === "string" &&
|
|
1173
|
+
typeof backend.invalidateTag === "function") {
|
|
1174
|
+
try { await backend.invalidateTag(ev.tag); } catch (_e) {}
|
|
1175
|
+
} else if (ev.kind === "del" && typeof ev.key === "string") {
|
|
1176
|
+
try { await backend.del(ev.key); } catch (_e) {}
|
|
1177
|
+
} else if (ev.kind === "clear") {
|
|
1178
|
+
try { await backend.clear(); } catch (_e) {}
|
|
1179
|
+
}
|
|
1180
|
+
// Wipe local in-flight memoization so a freshly-invalidated key
|
|
1181
|
+
// can't resolve from a still-pending fetch on this node.
|
|
1182
|
+
inflight.clear();
|
|
1183
|
+
swrInflight.clear();
|
|
1184
|
+
softExpiry.clear();
|
|
1185
|
+
} finally {
|
|
1186
|
+
inboundInvalidation = false;
|
|
1187
|
+
}
|
|
1188
|
+
}
|
|
1189
|
+
|
|
1190
|
+
if (invalidationPubsub) {
|
|
1191
|
+
invalidationToken = invalidationPubsub.subscribe(invalidationChannel, _onInboundInvalidation);
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1137
1194
|
async function close() {
|
|
1138
1195
|
if (closed) return;
|
|
1139
1196
|
closed = true;
|
|
1197
|
+
if (invalidationPubsub && invalidationToken) {
|
|
1198
|
+
try { invalidationPubsub.unsubscribe(invalidationToken); } catch (_e) {}
|
|
1199
|
+
invalidationToken = null;
|
|
1200
|
+
}
|
|
1140
1201
|
inflight.clear();
|
|
1141
1202
|
swrInflight.clear();
|
|
1142
1203
|
softExpiry.clear();
|
package/lib/db.js
CHANGED
|
@@ -110,7 +110,7 @@ var RESERVED_TABLE_NAMES = new Set([
|
|
|
110
110
|
"_blamejs_audit_purge_anchor",
|
|
111
111
|
"_blamejs_scheduler_ticks",
|
|
112
112
|
"_blamejs_rate_limit_counters",
|
|
113
|
-
"
|
|
113
|
+
"_blamejs_pubsub_messages",
|
|
114
114
|
"_blamejs_api_encrypt_nonces",
|
|
115
115
|
"_blamejs_api_keys",
|
|
116
116
|
"_blamejs_cache",
|
|
@@ -260,15 +260,16 @@ var FRAMEWORK_SCHEMA = [
|
|
|
260
260
|
sealedFields: [],
|
|
261
261
|
},
|
|
262
262
|
{
|
|
263
|
-
//
|
|
264
|
-
//
|
|
265
|
-
//
|
|
266
|
-
//
|
|
267
|
-
//
|
|
268
|
-
|
|
263
|
+
// _blamejs_pubsub_messages — cluster fan-out for `b.pubsub` (the
|
|
264
|
+
// generalization of the previous WebSocket-specific table). Any
|
|
265
|
+
// pubsub instance using the `cluster` backend writes a row on
|
|
266
|
+
// publish; other nodes poll for new ids and dispatch to their
|
|
267
|
+
// local subscribers. Rows older than the configured retention
|
|
268
|
+
// window are pruned by the backend on a rate-limited basis.
|
|
269
|
+
name: "_blamejs_pubsub_messages",
|
|
269
270
|
columns: {
|
|
270
271
|
id: "INTEGER PRIMARY KEY AUTOINCREMENT",
|
|
271
|
-
|
|
272
|
+
topic: "TEXT NOT NULL",
|
|
272
273
|
payload: "TEXT NOT NULL",
|
|
273
274
|
publishedAt: "INTEGER NOT NULL",
|
|
274
275
|
publishedBy: "TEXT NOT NULL",
|
package/lib/framework-schema.js
CHANGED
|
@@ -94,7 +94,7 @@ var LOCAL_TO_EXTERNAL = Object.freeze({
|
|
|
94
94
|
// WebSocket channel-hub cluster fan-out — publish() writes a row,
|
|
95
95
|
// other nodes poll for new ids and dispatch to their local
|
|
96
96
|
// subscribers. Same dual-storage shape as sessions / jobs / etc.
|
|
97
|
-
|
|
97
|
+
_blamejs_pubsub_messages: "_blamejs_pubsub_messages",
|
|
98
98
|
_blamejs_api_encrypt_nonces: "_blamejs_api_encrypt_nonces",
|
|
99
99
|
// _blamejs_api_keys — operator-facing API-key registry table for the
|
|
100
100
|
// b.apiKey primitive. PRIMARY KEY is namespace-scoped id (so multiple
|
|
@@ -368,14 +368,15 @@ function _rateLimitCountersDDL(dialect) {
|
|
|
368
368
|
};
|
|
369
369
|
}
|
|
370
370
|
|
|
371
|
-
//
|
|
372
|
-
//
|
|
373
|
-
//
|
|
374
|
-
//
|
|
375
|
-
//
|
|
376
|
-
|
|
371
|
+
// _blamejs_pubsub_messages — cluster fan-out for `b.pubsub` (the
|
|
372
|
+
// generalization of the previous WebSocket-specific table). publish()
|
|
373
|
+
// on any node writes a row; other nodes poll for new ids past their
|
|
374
|
+
// last seen and dispatch to local subscribers. Auto-incrementing id
|
|
375
|
+
// is essential — postgres needs BIGSERIAL, sqlite gets INTEGER
|
|
376
|
+
// PRIMARY KEY (which auto-increments implicitly).
|
|
377
|
+
function _pubsubMessagesDDL(dialect) {
|
|
377
378
|
var t = _types(dialect);
|
|
378
|
-
var name = LOCAL_TO_EXTERNAL.
|
|
379
|
+
var name = LOCAL_TO_EXTERNAL._blamejs_pubsub_messages;
|
|
379
380
|
var idCol = dialect === "postgres"
|
|
380
381
|
? "id BIGSERIAL PRIMARY KEY"
|
|
381
382
|
: "id INTEGER PRIMARY KEY AUTOINCREMENT";
|
|
@@ -383,7 +384,7 @@ function _wsMessagesDDL(dialect) {
|
|
|
383
384
|
create:
|
|
384
385
|
"CREATE TABLE IF NOT EXISTS " + name + " (" +
|
|
385
386
|
" " + idCol + "," +
|
|
386
|
-
"
|
|
387
|
+
" topic TEXT NOT NULL," +
|
|
387
388
|
" payload TEXT NOT NULL," +
|
|
388
389
|
" publishedAt " + t.INT + " NOT NULL," +
|
|
389
390
|
" publishedBy TEXT NOT NULL" +
|
|
@@ -683,7 +684,7 @@ async function ensureSchema(opts) {
|
|
|
683
684
|
_auditPurgeAnchorDDL(dialect),
|
|
684
685
|
_schedulerTicksDDL(dialect),
|
|
685
686
|
_rateLimitCountersDDL(dialect),
|
|
686
|
-
|
|
687
|
+
_pubsubMessagesDDL(dialect),
|
|
687
688
|
_apiEncryptNoncesDDL(dialect),
|
|
688
689
|
_apiKeysDDL(dialect),
|
|
689
690
|
_sessionsDDL(dialect),
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* pubsub-cluster — table-polling backend for `lib/pubsub.js`.
|
|
4
|
+
*
|
|
5
|
+
* Generalizes the polling pattern previously inlined in
|
|
6
|
+
* `lib/websocket-channels.js`. One shared table
|
|
7
|
+
* `_blamejs_pubsub_messages` carries every cross-node fan-out event;
|
|
8
|
+
* subscribers on each node poll the table at `pollIntervalMs` and
|
|
9
|
+
* dispatch new rows past their last-seen id. Independent pubsub
|
|
10
|
+
* instances on the same database isolate via `topicPrefix` (see
|
|
11
|
+
* lib/pubsub.js).
|
|
12
|
+
*
|
|
13
|
+
* Trade-offs vs. the redis backend:
|
|
14
|
+
* - No external dependency — re-uses the cluster DB the framework
|
|
15
|
+
* already requires for leader election, queues, sessions, etc.
|
|
16
|
+
* - Latency floor is the polling interval (default 100ms). Operators
|
|
17
|
+
* wanting <10ms cross-node latency switch to the redis backend.
|
|
18
|
+
* - Survives transient network blips between app nodes; missed rows
|
|
19
|
+
* are picked up on the next poll until retentionMs elapses.
|
|
20
|
+
*
|
|
21
|
+
* Schema: `_blamejs_pubsub_messages (id, topic, payload, publishedAt,
|
|
22
|
+
* publishedBy)`. Created by `lib/cluster-storage.js` migrations.
|
|
23
|
+
*/
|
|
24
|
+
var clusterStorage = require("./cluster-storage");
|
|
25
|
+
var C = require("./constants");
|
|
26
|
+
var lazyRequire = require("./lazy-require");
|
|
27
|
+
|
|
28
|
+
var logger = lazyRequire(function () { return require("./log").boot("pubsub-cluster"); });
|
|
29
|
+
|
|
30
|
+
var DEFAULT_POLL_INTERVAL_MS = 100;
|
|
31
|
+
var DEFAULT_RETENTION_MS = C.TIME.minutes(1);
|
|
32
|
+
var DEFAULT_PRUNE_EVERY_MS = C.TIME.minutes(5);
|
|
33
|
+
|
|
34
|
+
function create(opts) {
|
|
35
|
+
var clusterInstance = opts.cluster;
|
|
36
|
+
var pollIntervalMs = Number(opts.pollIntervalMs) || DEFAULT_POLL_INTERVAL_MS;
|
|
37
|
+
var retentionMs = Number(opts.retentionMs) || DEFAULT_RETENTION_MS;
|
|
38
|
+
var pruneEveryMs = Number(opts.pruneEveryMs) || DEFAULT_PRUNE_EVERY_MS;
|
|
39
|
+
|
|
40
|
+
var lastSeenId = 0;
|
|
41
|
+
var primed = false;
|
|
42
|
+
var lastPruneAt = 0;
|
|
43
|
+
var pollTimer = null;
|
|
44
|
+
var stopped = false;
|
|
45
|
+
|
|
46
|
+
function _nodeId() {
|
|
47
|
+
if (clusterInstance && typeof clusterInstance.currentNodeId === "function") {
|
|
48
|
+
return clusterInstance.currentNodeId();
|
|
49
|
+
}
|
|
50
|
+
return "single-node-local";
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
async function publishRemote(scopedChannel, payload) {
|
|
54
|
+
var serialized = JSON.stringify(payload);
|
|
55
|
+
await clusterStorage.execute(
|
|
56
|
+
"INSERT INTO _blamejs_pubsub_messages " +
|
|
57
|
+
"(topic, payload, publishedAt, publishedBy) VALUES (?, ?, ?, ?)",
|
|
58
|
+
[scopedChannel, serialized, Date.now(), _nodeId()]
|
|
59
|
+
);
|
|
60
|
+
return { remote: 1 };
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async function _poll(onRemoteMessage) {
|
|
64
|
+
if (stopped) return;
|
|
65
|
+
var nodeId = _nodeId();
|
|
66
|
+
try {
|
|
67
|
+
// First poll: prime lastSeenId to the current MAX so we don't
|
|
68
|
+
// re-dispatch every historical row on startup.
|
|
69
|
+
if (!primed) {
|
|
70
|
+
var primer = await clusterStorage.execute(
|
|
71
|
+
"SELECT COALESCE(MAX(id), 0) AS maxId FROM _blamejs_pubsub_messages",
|
|
72
|
+
[]
|
|
73
|
+
);
|
|
74
|
+
if (primer.rows && primer.rows[0]) {
|
|
75
|
+
lastSeenId = Number(primer.rows[0].maxId) || 0;
|
|
76
|
+
}
|
|
77
|
+
primed = true;
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
var result = await clusterStorage.execute(
|
|
81
|
+
"SELECT id, topic, payload, publishedAt, publishedBy " +
|
|
82
|
+
"FROM _blamejs_pubsub_messages " +
|
|
83
|
+
"WHERE id > ? AND publishedBy <> ? ORDER BY id ASC",
|
|
84
|
+
[lastSeenId, nodeId]
|
|
85
|
+
);
|
|
86
|
+
var rows = result.rows || [];
|
|
87
|
+
for (var i = 0; i < rows.length; i++) {
|
|
88
|
+
var row = rows[i];
|
|
89
|
+
try {
|
|
90
|
+
onRemoteMessage(row.topic, row.payload, {
|
|
91
|
+
publishedBy: row.publishedBy,
|
|
92
|
+
publishedAt: Number(row.publishedAt) || null,
|
|
93
|
+
});
|
|
94
|
+
} catch (e) {
|
|
95
|
+
try { logger().warn("malformed pubsub fan-out row id=" + row.id +
|
|
96
|
+
": " + ((e && e.message) || String(e))); }
|
|
97
|
+
catch (_e) { /* logger best-effort */ }
|
|
98
|
+
}
|
|
99
|
+
if (Number(row.id) > lastSeenId) lastSeenId = Number(row.id);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// Rate-limited prune of expired rows.
|
|
103
|
+
var now = Date.now();
|
|
104
|
+
if (now - lastPruneAt >= pruneEveryMs) {
|
|
105
|
+
lastPruneAt = now;
|
|
106
|
+
await clusterStorage.execute(
|
|
107
|
+
"DELETE FROM _blamejs_pubsub_messages WHERE publishedAt < ?",
|
|
108
|
+
[now - retentionMs]
|
|
109
|
+
);
|
|
110
|
+
}
|
|
111
|
+
} catch (e) {
|
|
112
|
+
try { logger().warn("pubsub-cluster poll failed: " +
|
|
113
|
+
((e && e.message) || String(e))); }
|
|
114
|
+
catch (_e) { /* */ }
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function start(onRemoteMessage) {
|
|
119
|
+
if (pollTimer) return;
|
|
120
|
+
stopped = false;
|
|
121
|
+
var tick = function () {
|
|
122
|
+
_poll(onRemoteMessage).then(function () {
|
|
123
|
+
if (stopped) return;
|
|
124
|
+
pollTimer = setTimeout(tick, pollIntervalMs);
|
|
125
|
+
if (typeof pollTimer.unref === "function") pollTimer.unref();
|
|
126
|
+
}, function () {
|
|
127
|
+
if (stopped) return;
|
|
128
|
+
pollTimer = setTimeout(tick, pollIntervalMs);
|
|
129
|
+
if (typeof pollTimer.unref === "function") pollTimer.unref();
|
|
130
|
+
});
|
|
131
|
+
};
|
|
132
|
+
pollTimer = setTimeout(tick, 0);
|
|
133
|
+
if (typeof pollTimer.unref === "function") pollTimer.unref();
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function stop() {
|
|
137
|
+
stopped = true;
|
|
138
|
+
if (pollTimer) { clearTimeout(pollTimer); pollTimer = null; }
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
return {
|
|
142
|
+
name: "cluster",
|
|
143
|
+
publishRemote: publishRemote,
|
|
144
|
+
start: start,
|
|
145
|
+
stop: stop,
|
|
146
|
+
// Cluster backend doesn't need explicit subscribeRemote — every
|
|
147
|
+
// node sees every row. The pubsub.js wrapper still tracks the
|
|
148
|
+
// remoteSubCount for parity with backends that DO need it.
|
|
149
|
+
subscribeRemote: null,
|
|
150
|
+
unsubscribeRemote: null,
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
module.exports = { create: create };
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* pubsub-redis — Redis PUB/SUB backend for `lib/pubsub.js`.
|
|
4
|
+
*
|
|
5
|
+
* Two connections per pubsub instance:
|
|
6
|
+
*
|
|
7
|
+
* subscriberConn — placed in subscribe mode via SUBSCRIBE /
|
|
8
|
+
* PSUBSCRIBE. The `lib/redis-client.js` push hook
|
|
9
|
+
* (`setOnPushMessage`) demultiplexes server-pushed
|
|
10
|
+
* "message" / "pmessage" frames from
|
|
11
|
+
* SUBSCRIBE/UNSUBSCRIBE acks. Subscribe-mode
|
|
12
|
+
* connections can't issue arbitrary commands;
|
|
13
|
+
* splitting publisher off is mandatory.
|
|
14
|
+
* publisherConn — issues PUBLISH commands against the same Redis
|
|
15
|
+
* instance. Uses normal command pipelining.
|
|
16
|
+
*
|
|
17
|
+
* The framework's `lib/redis-client.js` is single-connection-per-create
|
|
18
|
+
* — both connections use the same options (URL / password / TLS / CA).
|
|
19
|
+
*
|
|
20
|
+
* Channels are passed through to Redis with the topicPrefix already
|
|
21
|
+
* applied by `lib/pubsub.js`; this backend doesn't add any naming
|
|
22
|
+
* conventions of its own.
|
|
23
|
+
*/
|
|
24
|
+
var crypto = require("node:crypto");
|
|
25
|
+
var redisClient = require("./redis-client");
|
|
26
|
+
var lazyRequire = require("./lazy-require");
|
|
27
|
+
|
|
28
|
+
var logger = lazyRequire(function () { return require("./log").boot("pubsub-redis"); });
|
|
29
|
+
|
|
30
|
+
function create(opts) {
|
|
31
|
+
if (typeof opts.redisUrl !== "string" || opts.redisUrl.length === 0) {
|
|
32
|
+
throw new Error("pubsub-redis: redisUrl is required");
|
|
33
|
+
}
|
|
34
|
+
// Per-instance nonce stamped on every outgoing payload so the
|
|
35
|
+
// SUBSCRIBE socket can recognize its own publishes and skip
|
|
36
|
+
// dispatching them (the framework's pubsub.publish() already did
|
|
37
|
+
// the local dispatch synchronously before awaiting the remote
|
|
38
|
+
// write — without this filter every same-instance publish would
|
|
39
|
+
// double-fire local handlers).
|
|
40
|
+
var instanceNonce = crypto.randomBytes(8).toString("hex");
|
|
41
|
+
|
|
42
|
+
var clientOpts = {
|
|
43
|
+
url: opts.redisUrl,
|
|
44
|
+
password: opts.redisPassword,
|
|
45
|
+
username: opts.redisUsername,
|
|
46
|
+
tls: opts.redisTls,
|
|
47
|
+
ca: opts.redisCa,
|
|
48
|
+
servername: opts.redisServername,
|
|
49
|
+
connectTimeoutMs: opts.redisConnectTimeoutMs,
|
|
50
|
+
commandTimeoutMs: opts.redisCommandTimeoutMs,
|
|
51
|
+
maxReconnectAttempts: opts.redisMaxReconnectAttempts,
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
var subscriberConn = null;
|
|
55
|
+
var publisherConn = null;
|
|
56
|
+
var connectPromise = null;
|
|
57
|
+
var stopped = false;
|
|
58
|
+
var savedOnRemoteMessage = null;
|
|
59
|
+
|
|
60
|
+
// Inbound demultiplex — the redis-client routes "message" /
|
|
61
|
+
// "pmessage" frames here. Strip the {_psnode, p} envelope; if the
|
|
62
|
+
// nonce matches this instance, the message is our own publish
|
|
63
|
+
// looping back through Redis — skip dispatch (pubsub.js already
|
|
64
|
+
// dispatched locally in publish()). Otherwise forward to the
|
|
65
|
+
// dispatcher with the unwrapped payload string.
|
|
66
|
+
function _onPush(ev) {
|
|
67
|
+
if (!savedOnRemoteMessage) return;
|
|
68
|
+
var rawPayload = ev.payload;
|
|
69
|
+
var payloadStr = Buffer.isBuffer(rawPayload)
|
|
70
|
+
? rawPayload.toString("utf8") : String(rawPayload);
|
|
71
|
+
var inner = payloadStr;
|
|
72
|
+
try {
|
|
73
|
+
var envelope = JSON.parse(payloadStr);
|
|
74
|
+
if (envelope && typeof envelope === "object" &&
|
|
75
|
+
typeof envelope._psnode === "string") {
|
|
76
|
+
if (envelope._psnode === instanceNonce) return; // own publish
|
|
77
|
+
inner = JSON.stringify(envelope.p);
|
|
78
|
+
}
|
|
79
|
+
} catch (_e) {
|
|
80
|
+
// Not an envelope — forward as-is for operators publishing raw
|
|
81
|
+
// strings via redis CLI etc.
|
|
82
|
+
}
|
|
83
|
+
try {
|
|
84
|
+
savedOnRemoteMessage(ev.channel, inner, {
|
|
85
|
+
pattern: ev.pattern || null,
|
|
86
|
+
});
|
|
87
|
+
} catch (e) {
|
|
88
|
+
try { logger().warn("pubsub-redis push dispatch failed: " +
|
|
89
|
+
((e && e.message) || String(e))); }
|
|
90
|
+
catch (_e) { /* */ }
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
async function _ensureConnected() {
|
|
95
|
+
if (stopped) throw new Error("pubsub-redis: backend stopped");
|
|
96
|
+
if (subscriberConn && publisherConn) return;
|
|
97
|
+
if (connectPromise) return connectPromise;
|
|
98
|
+
connectPromise = (async function () {
|
|
99
|
+
subscriberConn = redisClient.create(Object.assign({}, clientOpts, {
|
|
100
|
+
onPushMessage: _onPush,
|
|
101
|
+
}));
|
|
102
|
+
publisherConn = redisClient.create(clientOpts);
|
|
103
|
+
await Promise.all([subscriberConn.connect(), publisherConn.connect()]);
|
|
104
|
+
})();
|
|
105
|
+
try { await connectPromise; }
|
|
106
|
+
finally { connectPromise = null; }
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
async function publishRemote(scopedChannel, payload) {
|
|
110
|
+
await _ensureConnected();
|
|
111
|
+
var serialized = JSON.stringify({ _psnode: instanceNonce, p: payload });
|
|
112
|
+
var n = await publisherConn.command("PUBLISH", scopedChannel, serialized);
|
|
113
|
+
return { remote: Number(n) || 0 };
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
async function subscribeRemote(scopedChannel, isPattern) {
|
|
117
|
+
await _ensureConnected();
|
|
118
|
+
var cmd = isPattern ? "PSUBSCRIBE" : "SUBSCRIBE";
|
|
119
|
+
await subscriberConn.command(cmd, scopedChannel);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
async function unsubscribeRemote(scopedChannel, isPattern) {
|
|
123
|
+
if (!subscriberConn || !subscriberConn.isOpen()) return;
|
|
124
|
+
var cmd = isPattern ? "PUNSUBSCRIBE" : "UNSUBSCRIBE";
|
|
125
|
+
try { await subscriberConn.command(cmd, scopedChannel); }
|
|
126
|
+
catch (_e) { /* unsubscribe failure is informational */ }
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function start(onRemoteMessage) {
|
|
130
|
+
savedOnRemoteMessage = onRemoteMessage;
|
|
131
|
+
// Lazy connect — first subscribe / publish opens the sockets. This
|
|
132
|
+
// keeps `b.pubsub.create()` synchronous-safe on a misconfigured
|
|
133
|
+
// redis URL: the validation throw lands on the first publish/subscribe
|
|
134
|
+
// call, where the operator can catch it.
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
async function stop() {
|
|
138
|
+
stopped = true;
|
|
139
|
+
savedOnRemoteMessage = null;
|
|
140
|
+
if (subscriberConn) {
|
|
141
|
+
try { await subscriberConn.close(); } catch (_e) {}
|
|
142
|
+
subscriberConn = null;
|
|
143
|
+
}
|
|
144
|
+
if (publisherConn) {
|
|
145
|
+
try { await publisherConn.close(); } catch (_e) {}
|
|
146
|
+
publisherConn = null;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
return {
|
|
151
|
+
name: "redis",
|
|
152
|
+
publishRemote: publishRemote,
|
|
153
|
+
subscribeRemote: subscribeRemote,
|
|
154
|
+
unsubscribeRemote: unsubscribeRemote,
|
|
155
|
+
start: start,
|
|
156
|
+
stop: stop,
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
module.exports = { create: create };
|