@blamejs/core 0.6.32 → 0.6.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -0
- package/README.md +2 -2
- package/index.js +2 -0
- package/lib/cache.js +61 -0
- package/lib/db.js +9 -8
- package/lib/framework-schema.js +11 -10
- package/lib/log-stream-cloudwatch.js +64 -5
- package/lib/log-stream-syslog.js +297 -0
- package/lib/log-stream.js +12 -12
- package/lib/protocol-dispatcher.js +1 -1
- package/lib/pubsub-cluster.js +154 -0
- package/lib/pubsub-redis.js +160 -0
- package/lib/pubsub.js +357 -0
- package/lib/redis-client.js +46 -1
- package/lib/websocket-channels.js +131 -217
- package/package.json +1 -1
- package/sbom.cyclonedx.json +6 -6
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Syslog log-stream sink — RFC 5424 framing over UDP / TCP / TLS.
|
|
4
|
+
*
|
|
5
|
+
* Wire format (RFC 5424 §6):
|
|
6
|
+
* <PRI>VERSION SP TIMESTAMP SP HOSTNAME SP APP-NAME SP PROCID SP MSGID SP STRUCTURED-DATA SP MSG
|
|
7
|
+
*
|
|
8
|
+
* PRI = (facility * 8) + severity
|
|
9
|
+
* facility (default 16 = local0; 1 = user — operators commonly pick
|
|
10
|
+
* local0..local7 for app-emitted records)
|
|
11
|
+
* severity is mapped from the framework's level field:
|
|
12
|
+
* debug → 7, info → 6, warn → 4, error → 3
|
|
13
|
+
*
|
|
14
|
+
* Transport:
|
|
15
|
+
* udp — single datagram per record (no framing)
|
|
16
|
+
* tcp — octet-counting framing (RFC 6587 §3.4.1):
|
|
17
|
+
* "<length> <message>" with a SPACE between
|
|
18
|
+
* length and the rfc5424 message bytes
|
|
19
|
+
* tls — same octet-counting framing on a TLS socket
|
|
20
|
+
* (RFC 5425). Standard port 6514.
|
|
21
|
+
*
|
|
22
|
+
* Defaults match RFC 3164/5424 conventions: appName = "blamejs",
|
|
23
|
+
* facility = local0 (16), hostname = os.hostname(), structuredData = "-".
|
|
24
|
+
*
|
|
25
|
+
* Flow control:
|
|
26
|
+
* The TCP / TLS variants buffer pending writes during socket
|
|
27
|
+
* reconnect and replay them on the new connection. UDP is best-effort
|
|
28
|
+
* (datagrams that race a closed socket are dropped to onDrop).
|
|
29
|
+
*/
|
|
30
|
+
var dgram = require("dgram");
|
|
31
|
+
var net = require("net");
|
|
32
|
+
var os = require("os");
|
|
33
|
+
var tls = require("tls");
|
|
34
|
+
var C = require("./constants");
|
|
35
|
+
var { LogStreamError } = require("./framework-error");
|
|
36
|
+
|
|
37
|
+
var _err = LogStreamError.factory;
|
|
38
|
+
|
|
39
|
+
var DEFAULT_FACILITY = 16; // local0
|
|
40
|
+
var DEFAULT_APP_NAME = "blamejs";
|
|
41
|
+
var DEFAULT_PROC_ID = String(process.pid);
|
|
42
|
+
var DEFAULT_MSG_ID = "-";
|
|
43
|
+
var DEFAULT_STRUCT_DATA = "-";
|
|
44
|
+
var TCP_DEFAULT_PORT = 514;
|
|
45
|
+
var TLS_DEFAULT_PORT = 6514;
|
|
46
|
+
var UDP_DEFAULT_PORT = 514;
|
|
47
|
+
var DEFAULT_TIMEOUT_MS = C.TIME.seconds(10);
|
|
48
|
+
var DEFAULT_RECONNECT_BASE_MS = 250;
|
|
49
|
+
var DEFAULT_RECONNECT_MAX_MS = 30000;
|
|
50
|
+
var DEFAULT_BUFFER_LIMIT = 10000;
|
|
51
|
+
|
|
52
|
+
// RFC 5424 severity codes from the framework's level names.
|
|
53
|
+
var LEVEL_TO_SEVERITY = {
|
|
54
|
+
debug: 7,
|
|
55
|
+
info: 6,
|
|
56
|
+
warn: 4,
|
|
57
|
+
error: 3,
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
function _toRfc3339(tsMs) {
|
|
61
|
+
return new Date(tsMs).toISOString();
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function _formatRfc5424(record, cfg) {
|
|
65
|
+
var severity = LEVEL_TO_SEVERITY[record.level] != null
|
|
66
|
+
? LEVEL_TO_SEVERITY[record.level] : 6;
|
|
67
|
+
var pri = (cfg.facility * 8) + severity;
|
|
68
|
+
var ts = _toRfc3339(record.ts || Date.now());
|
|
69
|
+
// Body — JSON-encode meta + message together so the structured
|
|
70
|
+
// payload survives the wire as a single MSG token. Operators with
|
|
71
|
+
// their own RFC 5424 STRUCTURED-DATA producers pass cfg.structuredData
|
|
72
|
+
// (string) to override the default "-".
|
|
73
|
+
var body = record.message || "";
|
|
74
|
+
if (record.meta && Object.keys(record.meta).length > 0) {
|
|
75
|
+
try { body += " " + JSON.stringify(record.meta); }
|
|
76
|
+
catch (_e) { /* best-effort */ }
|
|
77
|
+
}
|
|
78
|
+
return "<" + pri + ">1 " + ts + " " + cfg.hostname + " " +
|
|
79
|
+
cfg.appName + " " + cfg.procId + " " + cfg.msgId + " " +
|
|
80
|
+
(cfg.structuredData || "-") + " " + body;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function create(config) {
|
|
84
|
+
if (!config) throw _err("BAD_OPT", "log-stream syslog requires { url } (e.g. udp://host:514, tcp://host:514, tls://host:6514)");
|
|
85
|
+
var url = config.url;
|
|
86
|
+
if (typeof url !== "string" || url.length === 0) {
|
|
87
|
+
throw _err("BAD_OPT", "log-stream syslog requires { url } (string)");
|
|
88
|
+
}
|
|
89
|
+
// Parse URL — accept udp://, tcp://, tls://. safeUrl rejects http://
|
|
90
|
+
// by default; allow these explicit transports.
|
|
91
|
+
var parsed;
|
|
92
|
+
try {
|
|
93
|
+
parsed = new URL(url);
|
|
94
|
+
} catch (e) {
|
|
95
|
+
throw _err("BAD_URL", "log-stream syslog: bad url '" + url + "': " +
|
|
96
|
+
((e && e.message) || String(e)));
|
|
97
|
+
}
|
|
98
|
+
var transport = parsed.protocol.replace(/:$/, "").toLowerCase();
|
|
99
|
+
if (transport !== "udp" && transport !== "tcp" && transport !== "tls") {
|
|
100
|
+
throw _err("BAD_URL",
|
|
101
|
+
"log-stream syslog: protocol must be udp / tcp / tls, got " +
|
|
102
|
+
JSON.stringify(parsed.protocol));
|
|
103
|
+
}
|
|
104
|
+
var defaultPort = transport === "tls" ? TLS_DEFAULT_PORT
|
|
105
|
+
: transport === "tcp" ? TCP_DEFAULT_PORT
|
|
106
|
+
: UDP_DEFAULT_PORT;
|
|
107
|
+
var host = parsed.hostname;
|
|
108
|
+
var port = parsed.port ? parseInt(parsed.port, 10) : defaultPort;
|
|
109
|
+
|
|
110
|
+
var cfg = {
|
|
111
|
+
transport: transport,
|
|
112
|
+
host: host,
|
|
113
|
+
port: port,
|
|
114
|
+
facility: (typeof config.facility === "number" && config.facility >= 0 && config.facility <= 23)
|
|
115
|
+
? Math.floor(config.facility) : DEFAULT_FACILITY,
|
|
116
|
+
appName: config.appName || DEFAULT_APP_NAME,
|
|
117
|
+
procId: config.procId || DEFAULT_PROC_ID,
|
|
118
|
+
msgId: config.msgId || DEFAULT_MSG_ID,
|
|
119
|
+
hostname: config.hostname || os.hostname(),
|
|
120
|
+
structuredData: config.structuredData || DEFAULT_STRUCT_DATA,
|
|
121
|
+
timeoutMs: config.timeoutMs || DEFAULT_TIMEOUT_MS,
|
|
122
|
+
bufferLimit: config.bufferLimit || DEFAULT_BUFFER_LIMIT,
|
|
123
|
+
reconnectBaseMs: config.reconnectBaseMs || DEFAULT_RECONNECT_BASE_MS,
|
|
124
|
+
reconnectMaxMs: config.reconnectMaxMs || DEFAULT_RECONNECT_MAX_MS,
|
|
125
|
+
ca: config.ca || null,
|
|
126
|
+
rejectUnauthorized: config.rejectUnauthorized !== false,
|
|
127
|
+
servername: config.servername || null,
|
|
128
|
+
};
|
|
129
|
+
// safeUrl-style guard for the URL — reject userinfo (no auth in
|
|
130
|
+
// syslog wire) so a stray "syslog://user:pw@host" doesn't silently
|
|
131
|
+
// get through.
|
|
132
|
+
if (parsed.username || parsed.password) {
|
|
133
|
+
throw _err("BAD_URL",
|
|
134
|
+
"log-stream syslog: url must not contain userinfo");
|
|
135
|
+
}
|
|
136
|
+
// Track the operator's onDrop so dropped events surface.
|
|
137
|
+
var onDrop = typeof config.onDrop === "function" ? config.onDrop : null;
|
|
138
|
+
function _emitDrop(reason, batch, err) {
|
|
139
|
+
if (!onDrop) return;
|
|
140
|
+
try { onDrop({ reason: reason, batch: batch, error: err || null }); }
|
|
141
|
+
catch (_e) {}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// ---- UDP transport ----
|
|
145
|
+
if (transport === "udp") {
|
|
146
|
+
var udpFamily = host.indexOf(":") !== -1 ? "udp6" : "udp4";
|
|
147
|
+
var udpSock = dgram.createSocket(udpFamily);
|
|
148
|
+
udpSock.unref && udpSock.unref();
|
|
149
|
+
var udpClosed = false;
|
|
150
|
+
udpSock.on("error", function () { /* non-fatal — datagrams race */ });
|
|
151
|
+
|
|
152
|
+
return {
|
|
153
|
+
protocol: "syslog-udp",
|
|
154
|
+
emit: function (record) {
|
|
155
|
+
if (udpClosed) {
|
|
156
|
+
_emitDrop("sink-closed", [record], null);
|
|
157
|
+
return Promise.resolve({ accepted: false, reason: "closed" });
|
|
158
|
+
}
|
|
159
|
+
var msg = _formatRfc5424(record, cfg);
|
|
160
|
+
var buf = Buffer.from(msg, "utf8");
|
|
161
|
+
return new Promise(function (resolve) {
|
|
162
|
+
udpSock.send(buf, 0, buf.length, cfg.port, cfg.host, function (err) {
|
|
163
|
+
if (err) _emitDrop("udp-send-error", [record], err);
|
|
164
|
+
resolve({ accepted: !err, queued: 0 });
|
|
165
|
+
});
|
|
166
|
+
});
|
|
167
|
+
},
|
|
168
|
+
close: function () {
|
|
169
|
+
udpClosed = true;
|
|
170
|
+
try { udpSock.close(); } catch (_e) {}
|
|
171
|
+
return Promise.resolve();
|
|
172
|
+
},
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// ---- TCP / TLS transport — octet-counting framing (RFC 6587) ----
|
|
177
|
+
// Buffer outgoing records during socket-down windows; replay on
|
|
178
|
+
// reconnect. Operator opts: bufferLimit caps the queue; oldest
|
|
179
|
+
// dropped first with the onDrop "overflow" reason.
|
|
180
|
+
var sock = null;
|
|
181
|
+
var sockReady = false;
|
|
182
|
+
var connecting = false;
|
|
183
|
+
var queue = [];
|
|
184
|
+
var closed = false;
|
|
185
|
+
var reconnectAttempt = 0;
|
|
186
|
+
|
|
187
|
+
function _writeFramed(record) {
|
|
188
|
+
var msg = _formatRfc5424(record, cfg);
|
|
189
|
+
var msgBuf = Buffer.from(msg, "utf8");
|
|
190
|
+
var prefix = Buffer.from(msgBuf.length + " ", "utf8");
|
|
191
|
+
sock.write(Buffer.concat([prefix, msgBuf]));
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function _connect() {
|
|
195
|
+
if (closed || connecting) return;
|
|
196
|
+
connecting = true;
|
|
197
|
+
sockReady = false;
|
|
198
|
+
var connectOpts = { host: cfg.host, port: cfg.port };
|
|
199
|
+
var onConnect = function () {
|
|
200
|
+
connecting = false;
|
|
201
|
+
sockReady = true;
|
|
202
|
+
reconnectAttempt = 0;
|
|
203
|
+
// Drain queue in arrival order on (re)connect.
|
|
204
|
+
while (queue.length > 0 && sockReady) {
|
|
205
|
+
try { _writeFramed(queue.shift()); }
|
|
206
|
+
catch (e) { _emitDrop("write-error", [/* drained */], e); break; }
|
|
207
|
+
}
|
|
208
|
+
};
|
|
209
|
+
if (transport === "tls") {
|
|
210
|
+
var tlsOpts = Object.assign({}, connectOpts, {
|
|
211
|
+
rejectUnauthorized: cfg.rejectUnauthorized,
|
|
212
|
+
minVersion: "TLSv1.3",
|
|
213
|
+
});
|
|
214
|
+
if (cfg.ca) tlsOpts.ca = cfg.ca;
|
|
215
|
+
if (cfg.servername) tlsOpts.servername = cfg.servername;
|
|
216
|
+
sock = tls.connect(tlsOpts, onConnect);
|
|
217
|
+
} else {
|
|
218
|
+
sock = net.connect(connectOpts, onConnect);
|
|
219
|
+
}
|
|
220
|
+
sock.unref && sock.unref();
|
|
221
|
+
sock.on("error", function () { /* defer to 'close' for reconnect */ });
|
|
222
|
+
sock.on("close", function () {
|
|
223
|
+
sockReady = false;
|
|
224
|
+
connecting = false;
|
|
225
|
+
try { sock.destroy(); } catch (_e) {}
|
|
226
|
+
sock = null;
|
|
227
|
+
if (closed) return;
|
|
228
|
+
reconnectAttempt += 1;
|
|
229
|
+
var delay = Math.min(cfg.reconnectMaxMs,
|
|
230
|
+
cfg.reconnectBaseMs * Math.pow(2, reconnectAttempt - 1));
|
|
231
|
+
var t = setTimeout(_connect, delay);
|
|
232
|
+
t.unref && t.unref();
|
|
233
|
+
});
|
|
234
|
+
}
|
|
235
|
+
_connect();
|
|
236
|
+
|
|
237
|
+
return {
|
|
238
|
+
protocol: "syslog-" + transport,
|
|
239
|
+
emit: function (record) {
|
|
240
|
+
if (closed) {
|
|
241
|
+
_emitDrop("sink-closed", [record], null);
|
|
242
|
+
return Promise.resolve({ accepted: false, reason: "closed" });
|
|
243
|
+
}
|
|
244
|
+
if (sockReady) {
|
|
245
|
+
try { _writeFramed(record); }
|
|
246
|
+
catch (e) {
|
|
247
|
+
_emitDrop("write-error", [record], e);
|
|
248
|
+
return Promise.resolve({ accepted: false, reason: "write-error" });
|
|
249
|
+
}
|
|
250
|
+
return Promise.resolve({ accepted: true, queued: 0 });
|
|
251
|
+
}
|
|
252
|
+
// Socket not yet up — buffer with overflow-by-oldest semantics.
|
|
253
|
+
if (queue.length >= cfg.bufferLimit) {
|
|
254
|
+
var dropped = queue.shift();
|
|
255
|
+
_emitDrop("overflow", [dropped], null);
|
|
256
|
+
}
|
|
257
|
+
queue.push(record);
|
|
258
|
+
return Promise.resolve({ accepted: true, queued: queue.length });
|
|
259
|
+
},
|
|
260
|
+
close: function () {
|
|
261
|
+
// Give an in-flight (re)connect a brief window to complete and
|
|
262
|
+
// drain the buffer. Without this, records emitted just before
|
|
263
|
+
// shutdown race the slower TLS handshake and surface as
|
|
264
|
+
// "sink-closed" drops even though the framework had a viable
|
|
265
|
+
// connection in progress.
|
|
266
|
+
var DRAIN_TIMEOUT_MS = 3000;
|
|
267
|
+
var started = Date.now();
|
|
268
|
+
return new Promise(function (resolve) {
|
|
269
|
+
function _finish() {
|
|
270
|
+
closed = true;
|
|
271
|
+
var pending = queue.splice(0, queue.length);
|
|
272
|
+
if (pending.length > 0) _emitDrop("sink-closed", pending, null);
|
|
273
|
+
try { if (sock) sock.end(); } catch (_e) {}
|
|
274
|
+
try { if (sock) sock.destroy(); } catch (_e) {}
|
|
275
|
+
sock = null;
|
|
276
|
+
resolve();
|
|
277
|
+
}
|
|
278
|
+
function _tick() {
|
|
279
|
+
if (queue.length === 0 || Date.now() - started > DRAIN_TIMEOUT_MS) {
|
|
280
|
+
return _finish();
|
|
281
|
+
}
|
|
282
|
+
// Keep the timer ref'd — close() is being awaited; if we
|
|
283
|
+
// unref the drain-tick timer the event loop can exit between
|
|
284
|
+
// ticks (UDP/TCP socket are also unref'd) and the close
|
|
285
|
+
// promise pends forever silently.
|
|
286
|
+
setTimeout(_tick, 25);
|
|
287
|
+
}
|
|
288
|
+
_tick();
|
|
289
|
+
});
|
|
290
|
+
},
|
|
291
|
+
// Test-only: returns the in-flight queue size for assertions.
|
|
292
|
+
_queueSizeForTest: function () { return queue.length; },
|
|
293
|
+
_formatRfc5424ForTest: function (rec) { return _formatRfc5424(rec, cfg); },
|
|
294
|
+
};
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
module.exports = { create: create };
|
package/lib/log-stream.js
CHANGED
|
@@ -15,15 +15,15 @@
|
|
|
15
15
|
* (k8s, cloud) get standard log forwarding without a
|
|
16
16
|
* vendor-specific adapter.
|
|
17
17
|
* cloudwatch — AWS CloudWatch Logs (PutLogEvents) over HTTPS with
|
|
18
|
-
* SigV4.
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
* IAM role + STS session tokens.
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
18
|
+
* SigV4. Pass { autoCreate: true } to have the framework
|
|
19
|
+
* issue CreateLogGroup + CreateLogStream on first emit
|
|
20
|
+
* (idempotent — ResourceAlreadyExistsException treated as
|
|
21
|
+
* success). Honors IAM role + STS session tokens. Respects
|
|
22
|
+
* the 10K-event / 1 MiB / 256 KiB-per-event AWS caps.
|
|
23
|
+
* syslog — RFC 5424 with octet-counting framing over UDP / TCP /
|
|
24
|
+
* TLS. UDP is best-effort; TCP/TLS buffer during socket
|
|
25
|
+
* reconnect and replay on connect. Default ports 514
|
|
26
|
+
* (UDP/TCP) and 6514 (TLS).
|
|
27
27
|
*
|
|
28
28
|
* Every emit goes through lib/redact.js BEFORE any sink sees it. PHI/PCI
|
|
29
29
|
* never reaches operational logs even on a misconfigured field name —
|
|
@@ -50,6 +50,7 @@ var localProto = require("./log-stream-local");
|
|
|
50
50
|
var webhookProto = require("./log-stream-webhook");
|
|
51
51
|
var otlpProto = require("./log-stream-otlp");
|
|
52
52
|
var cloudwatchProto = require("./log-stream-cloudwatch");
|
|
53
|
+
var syslogProto = require("./log-stream-syslog");
|
|
53
54
|
var redactor = require("./redact");
|
|
54
55
|
var lazyRequire = require("./lazy-require");
|
|
55
56
|
var protocolDispatcher = require("./protocol-dispatcher");
|
|
@@ -63,10 +64,9 @@ var dispatcher = protocolDispatcher.create({
|
|
|
63
64
|
"webhook": webhookProto,
|
|
64
65
|
"otlp": otlpProto,
|
|
65
66
|
"cloudwatch": cloudwatchProto,
|
|
67
|
+
"syslog": syslogProto,
|
|
66
68
|
},
|
|
67
|
-
deferred: {
|
|
68
|
-
"syslog": { description: "RFC 5424 syslog over TLS" },
|
|
69
|
-
},
|
|
69
|
+
deferred: {},
|
|
70
70
|
fallbackProtocol: "local",
|
|
71
71
|
});
|
|
72
72
|
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
*
|
|
6
6
|
* Three framework modules ship a "pick a backend by protocol name" surface:
|
|
7
7
|
* - lib/queue.js (local, deferred: redis/sqs/amqp/nats)
|
|
8
|
-
* - lib/log-stream.js (local + webhook
|
|
8
|
+
* - lib/log-stream.js (local + webhook + otlp + cloudwatch + syslog)
|
|
9
9
|
* - lib/object-store/index.js (local + http-put + sigv4 + gcs + azure-blob)
|
|
10
10
|
*
|
|
11
11
|
* Each previously copied a ~30-line dispatch block: validate config has a
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* pubsub-cluster — table-polling backend for `lib/pubsub.js`.
|
|
4
|
+
*
|
|
5
|
+
* Generalizes the polling pattern previously inlined in
|
|
6
|
+
* `lib/websocket-channels.js`. One shared table
|
|
7
|
+
* `_blamejs_pubsub_messages` carries every cross-node fan-out event;
|
|
8
|
+
* subscribers on each node poll the table at `pollIntervalMs` and
|
|
9
|
+
* dispatch new rows past their last-seen id. Independent pubsub
|
|
10
|
+
* instances on the same database isolate via `topicPrefix` (see
|
|
11
|
+
* lib/pubsub.js).
|
|
12
|
+
*
|
|
13
|
+
* Trade-offs vs. the redis backend:
|
|
14
|
+
* - No external dependency — re-uses the cluster DB the framework
|
|
15
|
+
* already requires for leader election, queues, sessions, etc.
|
|
16
|
+
* - Latency floor is the polling interval (default 100ms). Operators
|
|
17
|
+
* wanting <10ms cross-node latency switch to the redis backend.
|
|
18
|
+
* - Survives transient network blips between app nodes; missed rows
|
|
19
|
+
* are picked up on the next poll until retentionMs elapses.
|
|
20
|
+
*
|
|
21
|
+
* Schema: `_blamejs_pubsub_messages (id, topic, payload, publishedAt,
|
|
22
|
+
* publishedBy)`. Created by `lib/cluster-storage.js` migrations.
|
|
23
|
+
*/
|
|
24
|
+
var clusterStorage = require("./cluster-storage");
|
|
25
|
+
var C = require("./constants");
|
|
26
|
+
var lazyRequire = require("./lazy-require");
|
|
27
|
+
|
|
28
|
+
var logger = lazyRequire(function () { return require("./log").boot("pubsub-cluster"); });
|
|
29
|
+
|
|
30
|
+
var DEFAULT_POLL_INTERVAL_MS = 100;
|
|
31
|
+
var DEFAULT_RETENTION_MS = C.TIME.minutes(1);
|
|
32
|
+
var DEFAULT_PRUNE_EVERY_MS = C.TIME.minutes(5);
|
|
33
|
+
|
|
34
|
+
function create(opts) {
|
|
35
|
+
var clusterInstance = opts.cluster;
|
|
36
|
+
var pollIntervalMs = Number(opts.pollIntervalMs) || DEFAULT_POLL_INTERVAL_MS;
|
|
37
|
+
var retentionMs = Number(opts.retentionMs) || DEFAULT_RETENTION_MS;
|
|
38
|
+
var pruneEveryMs = Number(opts.pruneEveryMs) || DEFAULT_PRUNE_EVERY_MS;
|
|
39
|
+
|
|
40
|
+
var lastSeenId = 0;
|
|
41
|
+
var primed = false;
|
|
42
|
+
var lastPruneAt = 0;
|
|
43
|
+
var pollTimer = null;
|
|
44
|
+
var stopped = false;
|
|
45
|
+
|
|
46
|
+
function _nodeId() {
|
|
47
|
+
if (clusterInstance && typeof clusterInstance.currentNodeId === "function") {
|
|
48
|
+
return clusterInstance.currentNodeId();
|
|
49
|
+
}
|
|
50
|
+
return "single-node-local";
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
async function publishRemote(scopedChannel, payload) {
|
|
54
|
+
var serialized = JSON.stringify(payload);
|
|
55
|
+
await clusterStorage.execute(
|
|
56
|
+
"INSERT INTO _blamejs_pubsub_messages " +
|
|
57
|
+
"(topic, payload, publishedAt, publishedBy) VALUES (?, ?, ?, ?)",
|
|
58
|
+
[scopedChannel, serialized, Date.now(), _nodeId()]
|
|
59
|
+
);
|
|
60
|
+
return { remote: 1 };
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async function _poll(onRemoteMessage) {
|
|
64
|
+
if (stopped) return;
|
|
65
|
+
var nodeId = _nodeId();
|
|
66
|
+
try {
|
|
67
|
+
// First poll: prime lastSeenId to the current MAX so we don't
|
|
68
|
+
// re-dispatch every historical row on startup.
|
|
69
|
+
if (!primed) {
|
|
70
|
+
var primer = await clusterStorage.execute(
|
|
71
|
+
"SELECT COALESCE(MAX(id), 0) AS maxId FROM _blamejs_pubsub_messages",
|
|
72
|
+
[]
|
|
73
|
+
);
|
|
74
|
+
if (primer.rows && primer.rows[0]) {
|
|
75
|
+
lastSeenId = Number(primer.rows[0].maxId) || 0;
|
|
76
|
+
}
|
|
77
|
+
primed = true;
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
var result = await clusterStorage.execute(
|
|
81
|
+
"SELECT id, topic, payload, publishedAt, publishedBy " +
|
|
82
|
+
"FROM _blamejs_pubsub_messages " +
|
|
83
|
+
"WHERE id > ? AND publishedBy <> ? ORDER BY id ASC",
|
|
84
|
+
[lastSeenId, nodeId]
|
|
85
|
+
);
|
|
86
|
+
var rows = result.rows || [];
|
|
87
|
+
for (var i = 0; i < rows.length; i++) {
|
|
88
|
+
var row = rows[i];
|
|
89
|
+
try {
|
|
90
|
+
onRemoteMessage(row.topic, row.payload, {
|
|
91
|
+
publishedBy: row.publishedBy,
|
|
92
|
+
publishedAt: Number(row.publishedAt) || null,
|
|
93
|
+
});
|
|
94
|
+
} catch (e) {
|
|
95
|
+
try { logger().warn("malformed pubsub fan-out row id=" + row.id +
|
|
96
|
+
": " + ((e && e.message) || String(e))); }
|
|
97
|
+
catch (_e) { /* logger best-effort */ }
|
|
98
|
+
}
|
|
99
|
+
if (Number(row.id) > lastSeenId) lastSeenId = Number(row.id);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// Rate-limited prune of expired rows.
|
|
103
|
+
var now = Date.now();
|
|
104
|
+
if (now - lastPruneAt >= pruneEveryMs) {
|
|
105
|
+
lastPruneAt = now;
|
|
106
|
+
await clusterStorage.execute(
|
|
107
|
+
"DELETE FROM _blamejs_pubsub_messages WHERE publishedAt < ?",
|
|
108
|
+
[now - retentionMs]
|
|
109
|
+
);
|
|
110
|
+
}
|
|
111
|
+
} catch (e) {
|
|
112
|
+
try { logger().warn("pubsub-cluster poll failed: " +
|
|
113
|
+
((e && e.message) || String(e))); }
|
|
114
|
+
catch (_e) { /* */ }
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function start(onRemoteMessage) {
|
|
119
|
+
if (pollTimer) return;
|
|
120
|
+
stopped = false;
|
|
121
|
+
var tick = function () {
|
|
122
|
+
_poll(onRemoteMessage).then(function () {
|
|
123
|
+
if (stopped) return;
|
|
124
|
+
pollTimer = setTimeout(tick, pollIntervalMs);
|
|
125
|
+
if (typeof pollTimer.unref === "function") pollTimer.unref();
|
|
126
|
+
}, function () {
|
|
127
|
+
if (stopped) return;
|
|
128
|
+
pollTimer = setTimeout(tick, pollIntervalMs);
|
|
129
|
+
if (typeof pollTimer.unref === "function") pollTimer.unref();
|
|
130
|
+
});
|
|
131
|
+
};
|
|
132
|
+
pollTimer = setTimeout(tick, 0);
|
|
133
|
+
if (typeof pollTimer.unref === "function") pollTimer.unref();
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function stop() {
|
|
137
|
+
stopped = true;
|
|
138
|
+
if (pollTimer) { clearTimeout(pollTimer); pollTimer = null; }
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
return {
|
|
142
|
+
name: "cluster",
|
|
143
|
+
publishRemote: publishRemote,
|
|
144
|
+
start: start,
|
|
145
|
+
stop: stop,
|
|
146
|
+
// Cluster backend doesn't need explicit subscribeRemote — every
|
|
147
|
+
// node sees every row. The pubsub.js wrapper still tracks the
|
|
148
|
+
// remoteSubCount for parity with backends that DO need it.
|
|
149
|
+
subscribeRemote: null,
|
|
150
|
+
unsubscribeRemote: null,
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
module.exports = { create: create };
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* pubsub-redis — Redis PUB/SUB backend for `lib/pubsub.js`.
|
|
4
|
+
*
|
|
5
|
+
* Two connections per pubsub instance:
|
|
6
|
+
*
|
|
7
|
+
* subscriberConn — placed in subscribe mode via SUBSCRIBE /
|
|
8
|
+
* PSUBSCRIBE. The `lib/redis-client.js` push hook
|
|
9
|
+
* (`setOnPushMessage`) demultiplexes server-pushed
|
|
10
|
+
* "message" / "pmessage" frames from
|
|
11
|
+
* SUBSCRIBE/UNSUBSCRIBE acks. Subscribe-mode
|
|
12
|
+
* connections can't issue arbitrary commands;
|
|
13
|
+
* splitting publisher off is mandatory.
|
|
14
|
+
* publisherConn — issues PUBLISH commands against the same Redis
|
|
15
|
+
* instance. Uses normal command pipelining.
|
|
16
|
+
*
|
|
17
|
+
* The framework's `lib/redis-client.js` is single-connection-per-create
|
|
18
|
+
* — both connections use the same options (URL / password / TLS / CA).
|
|
19
|
+
*
|
|
20
|
+
* Channels are passed through to Redis with the topicPrefix already
|
|
21
|
+
* applied by `lib/pubsub.js`; this backend doesn't add any naming
|
|
22
|
+
* conventions of its own.
|
|
23
|
+
*/
|
|
24
|
+
var crypto = require("node:crypto");
|
|
25
|
+
var redisClient = require("./redis-client");
|
|
26
|
+
var lazyRequire = require("./lazy-require");
|
|
27
|
+
|
|
28
|
+
var logger = lazyRequire(function () { return require("./log").boot("pubsub-redis"); });
|
|
29
|
+
|
|
30
|
+
function create(opts) {
|
|
31
|
+
if (typeof opts.redisUrl !== "string" || opts.redisUrl.length === 0) {
|
|
32
|
+
throw new Error("pubsub-redis: redisUrl is required");
|
|
33
|
+
}
|
|
34
|
+
// Per-instance nonce stamped on every outgoing payload so the
|
|
35
|
+
// SUBSCRIBE socket can recognize its own publishes and skip
|
|
36
|
+
// dispatching them (the framework's pubsub.publish() already did
|
|
37
|
+
// the local dispatch synchronously before awaiting the remote
|
|
38
|
+
// write — without this filter every same-instance publish would
|
|
39
|
+
// double-fire local handlers).
|
|
40
|
+
var instanceNonce = crypto.randomBytes(8).toString("hex");
|
|
41
|
+
|
|
42
|
+
var clientOpts = {
|
|
43
|
+
url: opts.redisUrl,
|
|
44
|
+
password: opts.redisPassword,
|
|
45
|
+
username: opts.redisUsername,
|
|
46
|
+
tls: opts.redisTls,
|
|
47
|
+
ca: opts.redisCa,
|
|
48
|
+
servername: opts.redisServername,
|
|
49
|
+
connectTimeoutMs: opts.redisConnectTimeoutMs,
|
|
50
|
+
commandTimeoutMs: opts.redisCommandTimeoutMs,
|
|
51
|
+
maxReconnectAttempts: opts.redisMaxReconnectAttempts,
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
var subscriberConn = null;
|
|
55
|
+
var publisherConn = null;
|
|
56
|
+
var connectPromise = null;
|
|
57
|
+
var stopped = false;
|
|
58
|
+
var savedOnRemoteMessage = null;
|
|
59
|
+
|
|
60
|
+
// Inbound demultiplex — the redis-client routes "message" /
|
|
61
|
+
// "pmessage" frames here. Strip the {_psnode, p} envelope; if the
|
|
62
|
+
// nonce matches this instance, the message is our own publish
|
|
63
|
+
// looping back through Redis — skip dispatch (pubsub.js already
|
|
64
|
+
// dispatched locally in publish()). Otherwise forward to the
|
|
65
|
+
// dispatcher with the unwrapped payload string.
|
|
66
|
+
function _onPush(ev) {
|
|
67
|
+
if (!savedOnRemoteMessage) return;
|
|
68
|
+
var rawPayload = ev.payload;
|
|
69
|
+
var payloadStr = Buffer.isBuffer(rawPayload)
|
|
70
|
+
? rawPayload.toString("utf8") : String(rawPayload);
|
|
71
|
+
var inner = payloadStr;
|
|
72
|
+
try {
|
|
73
|
+
var envelope = JSON.parse(payloadStr);
|
|
74
|
+
if (envelope && typeof envelope === "object" &&
|
|
75
|
+
typeof envelope._psnode === "string") {
|
|
76
|
+
if (envelope._psnode === instanceNonce) return; // own publish
|
|
77
|
+
inner = JSON.stringify(envelope.p);
|
|
78
|
+
}
|
|
79
|
+
} catch (_e) {
|
|
80
|
+
// Not an envelope — forward as-is for operators publishing raw
|
|
81
|
+
// strings via redis CLI etc.
|
|
82
|
+
}
|
|
83
|
+
try {
|
|
84
|
+
savedOnRemoteMessage(ev.channel, inner, {
|
|
85
|
+
pattern: ev.pattern || null,
|
|
86
|
+
});
|
|
87
|
+
} catch (e) {
|
|
88
|
+
try { logger().warn("pubsub-redis push dispatch failed: " +
|
|
89
|
+
((e && e.message) || String(e))); }
|
|
90
|
+
catch (_e) { /* */ }
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
async function _ensureConnected() {
|
|
95
|
+
if (stopped) throw new Error("pubsub-redis: backend stopped");
|
|
96
|
+
if (subscriberConn && publisherConn) return;
|
|
97
|
+
if (connectPromise) return connectPromise;
|
|
98
|
+
connectPromise = (async function () {
|
|
99
|
+
subscriberConn = redisClient.create(Object.assign({}, clientOpts, {
|
|
100
|
+
onPushMessage: _onPush,
|
|
101
|
+
}));
|
|
102
|
+
publisherConn = redisClient.create(clientOpts);
|
|
103
|
+
await Promise.all([subscriberConn.connect(), publisherConn.connect()]);
|
|
104
|
+
})();
|
|
105
|
+
try { await connectPromise; }
|
|
106
|
+
finally { connectPromise = null; }
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
async function publishRemote(scopedChannel, payload) {
|
|
110
|
+
await _ensureConnected();
|
|
111
|
+
var serialized = JSON.stringify({ _psnode: instanceNonce, p: payload });
|
|
112
|
+
var n = await publisherConn.command("PUBLISH", scopedChannel, serialized);
|
|
113
|
+
return { remote: Number(n) || 0 };
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
async function subscribeRemote(scopedChannel, isPattern) {
|
|
117
|
+
await _ensureConnected();
|
|
118
|
+
var cmd = isPattern ? "PSUBSCRIBE" : "SUBSCRIBE";
|
|
119
|
+
await subscriberConn.command(cmd, scopedChannel);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
async function unsubscribeRemote(scopedChannel, isPattern) {
|
|
123
|
+
if (!subscriberConn || !subscriberConn.isOpen()) return;
|
|
124
|
+
var cmd = isPattern ? "PUNSUBSCRIBE" : "UNSUBSCRIBE";
|
|
125
|
+
try { await subscriberConn.command(cmd, scopedChannel); }
|
|
126
|
+
catch (_e) { /* unsubscribe failure is informational */ }
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function start(onRemoteMessage) {
|
|
130
|
+
savedOnRemoteMessage = onRemoteMessage;
|
|
131
|
+
// Lazy connect — first subscribe / publish opens the sockets. This
|
|
132
|
+
// keeps `b.pubsub.create()` synchronous-safe on a misconfigured
|
|
133
|
+
// redis URL: the validation throw lands on the first publish/subscribe
|
|
134
|
+
// call, where the operator can catch it.
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
async function stop() {
|
|
138
|
+
stopped = true;
|
|
139
|
+
savedOnRemoteMessage = null;
|
|
140
|
+
if (subscriberConn) {
|
|
141
|
+
try { await subscriberConn.close(); } catch (_e) {}
|
|
142
|
+
subscriberConn = null;
|
|
143
|
+
}
|
|
144
|
+
if (publisherConn) {
|
|
145
|
+
try { await publisherConn.close(); } catch (_e) {}
|
|
146
|
+
publisherConn = null;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
return {
|
|
151
|
+
name: "redis",
|
|
152
|
+
publishRemote: publishRemote,
|
|
153
|
+
subscribeRemote: subscribeRemote,
|
|
154
|
+
unsubscribeRemote: unsubscribeRemote,
|
|
155
|
+
start: start,
|
|
156
|
+
stop: stop,
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
module.exports = { create: create };
|