@bymax-one/nest-realtime 1.2.18 → 1.2.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -1
- package/README.md +90 -9
- package/dist/internal/index.cjs +479 -93
- package/dist/internal/index.d.cts +127 -6
- package/dist/internal/index.d.ts +127 -6
- package/dist/internal/index.mjs +476 -95
- package/dist/observability/index.cjs +104 -0
- package/dist/observability/index.d.cts +47 -0
- package/dist/observability/index.d.ts +47 -0
- package/dist/observability/index.mjs +104 -0
- package/dist/websocket/index.cjs +117 -19
- package/dist/websocket/index.d.cts +14 -4
- package/dist/websocket/index.d.ts +14 -4
- package/dist/websocket/index.mjs +118 -20
- package/package.json +31 -3
package/dist/internal/index.cjs
CHANGED
|
@@ -43,6 +43,16 @@ var REALTIME_ERROR_CODES = {
|
|
|
43
43
|
REPLAY_BUFFER_MISS: "REALTIME_REPLAY_BUFFER_MISS"
|
|
44
44
|
};
|
|
45
45
|
|
|
46
|
+
// src/server/utils/is-plain-object.ts
|
|
47
|
+
function isPlainObject(value) {
|
|
48
|
+
if (typeof value !== "object") return false;
|
|
49
|
+
try {
|
|
50
|
+
return Object.prototype.toString.call(value) === "[object Object]";
|
|
51
|
+
} catch {
|
|
52
|
+
return false;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
46
56
|
// src/server/config/drop-unset.ts
|
|
47
57
|
var SSE_KEYS = {
|
|
48
58
|
endpoint: true,
|
|
@@ -107,8 +117,7 @@ function dropUnset(options) {
|
|
|
107
117
|
const kept = readSet(options, TOP_LEVEL_KEYS);
|
|
108
118
|
for (const [group, keys] of Object.entries(GROUPS)) {
|
|
109
119
|
const value = kept[group];
|
|
110
|
-
if (
|
|
111
|
-
kept[group] = readSet(value, keys);
|
|
120
|
+
if (isPlainObject(value)) kept[group] = readSet(value, keys);
|
|
112
121
|
}
|
|
113
122
|
return kept;
|
|
114
123
|
}
|
|
@@ -162,7 +171,9 @@ function describeReceived(value) {
|
|
|
162
171
|
return Number.isFinite(value) ? String(value) : "a number that is not finite";
|
|
163
172
|
}
|
|
164
173
|
if (value === null) return "null";
|
|
165
|
-
|
|
174
|
+
if (Array.isArray(value)) return "an array";
|
|
175
|
+
if (typeof value === "object" && !isPlainObject(value)) return "a built-in object";
|
|
176
|
+
return KIND_OF_TYPE[typeof value] ?? "an object";
|
|
166
177
|
}
|
|
167
178
|
|
|
168
179
|
// src/server/utils/realtime-error.ts
|
|
@@ -173,6 +184,15 @@ function realtimeError(code, detail, options = {}) {
|
|
|
173
184
|
return Object.assign(error, { code });
|
|
174
185
|
}
|
|
175
186
|
|
|
187
|
+
// src/server/utils/refuse-object.ts
|
|
188
|
+
function refuseObject(who, path, value) {
|
|
189
|
+
throw realtimeError(
|
|
190
|
+
REALTIME_ERROR_CODES.INVALID_OPTIONS,
|
|
191
|
+
`${path} must be an object, received ${describeReceived(value)}`,
|
|
192
|
+
{ who }
|
|
193
|
+
);
|
|
194
|
+
}
|
|
195
|
+
|
|
176
196
|
// src/server/config/validate-shapes.ts
|
|
177
197
|
var REQUIRED_FUNCTIONS = {
|
|
178
198
|
pubsub: ["publish", "subscribe"],
|
|
@@ -194,11 +214,8 @@ function refuse(path, expected, received) {
|
|
|
194
214
|
{ who: "BymaxRealtimeModule" }
|
|
195
215
|
);
|
|
196
216
|
}
|
|
197
|
-
function isObject(value) {
|
|
198
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
199
|
-
}
|
|
200
217
|
function assertObject(path, value) {
|
|
201
|
-
if (value !== void 0 && !
|
|
218
|
+
if (value !== void 0 && !isPlainObject(value)) refuse(path, "an object", value);
|
|
202
219
|
}
|
|
203
220
|
function assertFunction(path, value) {
|
|
204
221
|
if (typeof value !== "function") refuse(path, "a function", value);
|
|
@@ -209,10 +226,15 @@ function assertOptionalFunction(path, value) {
|
|
|
209
226
|
function assertBoolean(path, value) {
|
|
210
227
|
if (value !== void 0 && typeof value !== "boolean") refuse(path, "a boolean", value);
|
|
211
228
|
}
|
|
212
|
-
function
|
|
213
|
-
if (value
|
|
229
|
+
function assertSseEndpoint(path, value) {
|
|
230
|
+
if (value === void 0 || value === null) return;
|
|
231
|
+
if (typeof value !== "string" || value.trim().length === 0) {
|
|
214
232
|
refuse(path, "a non-blank string", value);
|
|
215
233
|
}
|
|
234
|
+
if (/\s/.test(value)) refuse(path, "a string without whitespace", value);
|
|
235
|
+
}
|
|
236
|
+
function assertOptionsObject(value, who = "BymaxRealtimeModule") {
|
|
237
|
+
if (!isPlainObject(value)) refuseObject(who, "options", value);
|
|
216
238
|
}
|
|
217
239
|
function validatePieces(options) {
|
|
218
240
|
for (const [name, functions] of Object.entries(REQUIRED_FUNCTIONS)) {
|
|
@@ -237,12 +259,13 @@ function validateShapes(options) {
|
|
|
237
259
|
assertOptionalFunction("tenantResolver", options.tenantResolver);
|
|
238
260
|
assertBoolean("sse.emitConnectionEvent", options.sse?.emitConnectionEvent);
|
|
239
261
|
assertBoolean("websocket.emitConnectionEvent", options.websocket?.emitConnectionEvent);
|
|
240
|
-
|
|
262
|
+
assertSseEndpoint("sse.endpoint", options.sse?.endpoint);
|
|
241
263
|
}
|
|
242
264
|
|
|
243
265
|
// src/server/config/validate-options.ts
|
|
244
266
|
var VALID_TRANSPORTS = /* @__PURE__ */ new Set(["sse", "websocket", "both"]);
|
|
245
267
|
function validateOptions(raw) {
|
|
268
|
+
assertOptionsObject(raw);
|
|
246
269
|
const options = dropUnset(raw);
|
|
247
270
|
if (!options.transport || !VALID_TRANSPORTS.has(options.transport)) {
|
|
248
271
|
throw realtimeError(
|
|
@@ -333,12 +356,6 @@ function validateReauthPolicy(options) {
|
|
|
333
356
|
{ who: "BymaxRealtimeModule" }
|
|
334
357
|
);
|
|
335
358
|
}
|
|
336
|
-
if (policy?.cacheTtlMs !== void 0 && policy.intervalSeconds !== void 0 && policy.cacheTtlMs > policy.intervalSeconds * 1e3) {
|
|
337
|
-
common.Logger.warn(
|
|
338
|
-
`[BymaxRealtimeModule] reauthenticationPolicy.cacheTtlMs (${policy.cacheTtlMs} ms) exceeds intervalSeconds*1000 (${policy.intervalSeconds * 1e3} ms): the positive reauth cache outlives the interval, so revalidate is skipped on ticks within the cache window \u2014 reauthentication is effectively weakened`,
|
|
339
|
-
"BymaxRealtimeModule"
|
|
340
|
-
);
|
|
341
|
-
}
|
|
342
359
|
}
|
|
343
360
|
function validateSseOptions(options) {
|
|
344
361
|
const sse = options.sse;
|
|
@@ -370,6 +387,138 @@ function validateSseOptions(options) {
|
|
|
370
387
|
MAX_PER_USER_CONNECTIONS
|
|
371
388
|
);
|
|
372
389
|
}
|
|
390
|
+
var REFUSAL_REASONS = ["limit", "unauthorized", "unavailable", "error"];
|
|
391
|
+
var EVICTION_REASONS = ["per_user_cap", "slow_consumer", "revoked"];
|
|
392
|
+
var TALLY_TRANSPORTS = ["sse", "websocket"];
|
|
393
|
+
function zeroed(keys) {
|
|
394
|
+
return Object.fromEntries(keys.map((key) => [key, 0]));
|
|
395
|
+
}
|
|
396
|
+
var _refused, _evicted, _emitted, _slowConsumerDrops;
|
|
397
|
+
exports.RealtimeMetricsTally = class RealtimeMetricsTally {
|
|
398
|
+
constructor() {
|
|
399
|
+
__privateAdd(this, _refused, zeroed(REFUSAL_REASONS));
|
|
400
|
+
__privateAdd(this, _evicted, zeroed(EVICTION_REASONS));
|
|
401
|
+
__privateAdd(this, _emitted, zeroed(TALLY_TRANSPORTS));
|
|
402
|
+
__privateAdd(this, _slowConsumerDrops, 0);
|
|
403
|
+
}
|
|
404
|
+
refused(reason) {
|
|
405
|
+
__privateGet(this, _refused)[reason] += 1;
|
|
406
|
+
}
|
|
407
|
+
evicted(reason) {
|
|
408
|
+
__privateGet(this, _evicted)[reason] += 1;
|
|
409
|
+
}
|
|
410
|
+
emitted(transport) {
|
|
411
|
+
__privateGet(this, _emitted)[transport] += 1;
|
|
412
|
+
}
|
|
413
|
+
slowConsumerDropped() {
|
|
414
|
+
__privateSet(this, _slowConsumerDrops, __privateGet(this, _slowConsumerDrops) + 1);
|
|
415
|
+
}
|
|
416
|
+
snapshot() {
|
|
417
|
+
return {
|
|
418
|
+
refused: { ...__privateGet(this, _refused) },
|
|
419
|
+
evicted: { ...__privateGet(this, _evicted) },
|
|
420
|
+
emitted: { ...__privateGet(this, _emitted) },
|
|
421
|
+
slowConsumerDrops: __privateGet(this, _slowConsumerDrops)
|
|
422
|
+
};
|
|
423
|
+
}
|
|
424
|
+
};
|
|
425
|
+
_refused = new WeakMap();
|
|
426
|
+
_evicted = new WeakMap();
|
|
427
|
+
_emitted = new WeakMap();
|
|
428
|
+
_slowConsumerDrops = new WeakMap();
|
|
429
|
+
exports.RealtimeMetricsTally = __decorateClass([
|
|
430
|
+
common.Injectable()
|
|
431
|
+
], exports.RealtimeMetricsTally);
|
|
432
|
+
var STATUS_ERROR = 2;
|
|
433
|
+
var TRACER_NAME = "@bymax-one/nest-realtime";
|
|
434
|
+
var LIBRARY_CODE = /^REALTIME_[A-Z_]+$/;
|
|
435
|
+
function codeOf(error) {
|
|
436
|
+
if (typeof error !== "object" || error === null) return "REALTIME_ERROR";
|
|
437
|
+
const { code, message, data } = error;
|
|
438
|
+
for (const candidate of [code, data?.code, message]) {
|
|
439
|
+
if (typeof candidate === "string" && LIBRARY_CODE.test(candidate)) return candidate;
|
|
440
|
+
}
|
|
441
|
+
return "REALTIME_ERROR";
|
|
442
|
+
}
|
|
443
|
+
var _api;
|
|
444
|
+
exports.RealtimeTracing = class RealtimeTracing {
|
|
445
|
+
constructor() {
|
|
446
|
+
__privateAdd(this, _api);
|
|
447
|
+
}
|
|
448
|
+
async onModuleInit() {
|
|
449
|
+
try {
|
|
450
|
+
__privateSet(this, _api, await import('@opentelemetry/api'));
|
|
451
|
+
} catch {
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
async traced(span, run) {
|
|
455
|
+
let result;
|
|
456
|
+
try {
|
|
457
|
+
result = await run();
|
|
458
|
+
} catch (error) {
|
|
459
|
+
attempt(() => span.setStatus({ code: STATUS_ERROR, message: codeOf(error) }));
|
|
460
|
+
attempt(() => span.end());
|
|
461
|
+
throw error;
|
|
462
|
+
}
|
|
463
|
+
attempt(() => span.end());
|
|
464
|
+
return result;
|
|
465
|
+
}
|
|
466
|
+
span(name, attributes, run) {
|
|
467
|
+
const api = __privateGet(this, _api);
|
|
468
|
+
if (!api) return run();
|
|
469
|
+
const latch = { claimed: false };
|
|
470
|
+
attempt(
|
|
471
|
+
() => api.trace.getTracer(TRACER_NAME).startActiveSpan(name, { attributes }, (span) => {
|
|
472
|
+
if (!latch.claimed) {
|
|
473
|
+
latch.claimed = true;
|
|
474
|
+
latch.started = this.traced(span, run);
|
|
475
|
+
}
|
|
476
|
+
return latch.started;
|
|
477
|
+
})
|
|
478
|
+
);
|
|
479
|
+
if (latch.started) return latch.started;
|
|
480
|
+
latch.claimed = true;
|
|
481
|
+
return run();
|
|
482
|
+
}
|
|
483
|
+
};
|
|
484
|
+
_api = new WeakMap();
|
|
485
|
+
exports.RealtimeTracing = __decorateClass([
|
|
486
|
+
common.Injectable()
|
|
487
|
+
], exports.RealtimeTracing);
|
|
488
|
+
function attempt(call) {
|
|
489
|
+
try {
|
|
490
|
+
Promise.resolve(call()).catch(() => void 0);
|
|
491
|
+
} catch {
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
// src/server/utils/log-event.ts
|
|
496
|
+
var REALTIME_LOG_EVENTS = [
|
|
497
|
+
"REALTIME_REAUTH_CACHE_WEAKENS_POLICY",
|
|
498
|
+
"REALTIME_HANDSHAKE_FAILED",
|
|
499
|
+
"REALTIME_HOOK_FAILED",
|
|
500
|
+
"REALTIME_DELIVERY_FAILED",
|
|
501
|
+
"REALTIME_SLOW_CONSUMER",
|
|
502
|
+
"REALTIME_DISCONNECT_FAILED",
|
|
503
|
+
"REALTIME_RECONNECTS_GIVEN_UP",
|
|
504
|
+
"REALTIME_OFFLINE_QUEUE_LIMIT",
|
|
505
|
+
"REALTIME_OFFLINE_QUEUE_ACK_FAILED",
|
|
506
|
+
"REALTIME_PUBSUB_NOT_PROVIDED",
|
|
507
|
+
"REALTIME_PUBSUB_UNSUBSCRIBE_FAILED",
|
|
508
|
+
"REALTIME_PUBSUB_DROPPED",
|
|
509
|
+
"REALTIME_PUBSUB_UNAVAILABLE",
|
|
510
|
+
"REALTIME_CONNECTIONS_REFUSED",
|
|
511
|
+
"REALTIME_HANDSHAKES_REFUSED",
|
|
512
|
+
"REALTIME_REAUTH_FAILED",
|
|
513
|
+
"REALTIME_REVOCATION_FAILED",
|
|
514
|
+
"REALTIME_ADAPTER_MISSING",
|
|
515
|
+
"REALTIME_ADAPTER_COMMAND_FAILED",
|
|
516
|
+
"REALTIME_COMPOSITE_PARTIAL_FAILURE",
|
|
517
|
+
"REALTIME_SERVER_NOT_ATTACHED"
|
|
518
|
+
];
|
|
519
|
+
function logEvent(logger, level, event, message, fields = {}, error) {
|
|
520
|
+
logger[level]({ event, message, ...fields, ...error === void 0 ? {} : { err: error } });
|
|
521
|
+
}
|
|
373
522
|
|
|
374
523
|
// src/server/utils/describe-error.ts
|
|
375
524
|
function describeError(reason) {
|
|
@@ -383,13 +532,32 @@ function describeError(reason) {
|
|
|
383
532
|
|
|
384
533
|
// src/server/utils/report-delivery-failure.ts
|
|
385
534
|
function reportDeliveryFailure(logger, hooks, failure, what) {
|
|
386
|
-
|
|
535
|
+
logEvent(
|
|
536
|
+
logger,
|
|
537
|
+
"warn",
|
|
538
|
+
"REALTIME_DELIVERY_FAILED",
|
|
539
|
+
`${what}`,
|
|
540
|
+
{
|
|
541
|
+
code: failure.code,
|
|
542
|
+
op: failure.op
|
|
543
|
+
},
|
|
544
|
+
failure.error
|
|
545
|
+
);
|
|
387
546
|
const reported = {
|
|
388
547
|
...failure,
|
|
389
548
|
error: failure.error instanceof Error ? failure.error : new Error(describeError(failure.error))
|
|
390
549
|
};
|
|
391
550
|
void Promise.resolve().then(() => hooks?.onDeliveryFailure?.(reported)).catch((err) => {
|
|
392
|
-
|
|
551
|
+
logEvent(
|
|
552
|
+
logger,
|
|
553
|
+
"warn",
|
|
554
|
+
"REALTIME_HOOK_FAILED",
|
|
555
|
+
`onDeliveryFailure hook failed`,
|
|
556
|
+
{
|
|
557
|
+
hook: "onDeliveryFailure"
|
|
558
|
+
},
|
|
559
|
+
err
|
|
560
|
+
);
|
|
393
561
|
});
|
|
394
562
|
}
|
|
395
563
|
|
|
@@ -434,8 +602,12 @@ exports.OfflineQueueDeliveryService = class OfflineQueueDeliveryService {
|
|
|
434
602
|
timeouts = windowedReport(
|
|
435
603
|
() => 0,
|
|
436
604
|
(count) => {
|
|
437
|
-
|
|
438
|
-
|
|
605
|
+
logEvent(
|
|
606
|
+
this.logger,
|
|
607
|
+
"warn",
|
|
608
|
+
"REALTIME_RECONNECTS_GIVEN_UP",
|
|
609
|
+
`reconnects given up on since the last report: ${count}, because the offline queue did not answer in ${this.timeoutMs} ms or already had ${this.maxPending} reads waiting`,
|
|
610
|
+
{ count, timeoutMs: this.timeoutMs, maxPending: this.maxPending }
|
|
439
611
|
);
|
|
440
612
|
}
|
|
441
613
|
);
|
|
@@ -464,8 +636,12 @@ exports.OfflineQueueDeliveryService = class OfflineQueueDeliveryService {
|
|
|
464
636
|
return { events: [], degraded: true };
|
|
465
637
|
}
|
|
466
638
|
if (events.length === RETRIEVE_LIMIT) {
|
|
467
|
-
|
|
468
|
-
|
|
639
|
+
logEvent(
|
|
640
|
+
this.logger,
|
|
641
|
+
"warn",
|
|
642
|
+
"REALTIME_OFFLINE_QUEUE_LIMIT",
|
|
643
|
+
`Offline queue retrieve hit the ${RETRIEVE_LIMIT}-event limit for user; older gap events may be dropped.`,
|
|
644
|
+
{ limit: RETRIEVE_LIMIT }
|
|
469
645
|
);
|
|
470
646
|
}
|
|
471
647
|
return { events, degraded: false };
|
|
@@ -476,7 +652,14 @@ exports.OfflineQueueDeliveryService = class OfflineQueueDeliveryService {
|
|
|
476
652
|
try {
|
|
477
653
|
await this.storage.acknowledge(userId, lastId);
|
|
478
654
|
} catch (err) {
|
|
479
|
-
|
|
655
|
+
logEvent(
|
|
656
|
+
this.logger,
|
|
657
|
+
"warn",
|
|
658
|
+
"REALTIME_OFFLINE_QUEUE_ACK_FAILED",
|
|
659
|
+
`Offline queue acknowledge failed`,
|
|
660
|
+
{},
|
|
661
|
+
err
|
|
662
|
+
);
|
|
480
663
|
}
|
|
481
664
|
}
|
|
482
665
|
async retrieveAppended(userId, lastEventId, read) {
|
|
@@ -682,6 +865,7 @@ var AuthenticationUnavailableError = class extends Error {
|
|
|
682
865
|
this.reason = reason;
|
|
683
866
|
}
|
|
684
867
|
reason;
|
|
868
|
+
code = REALTIME_ERROR_CODES.TRANSPORT_UNAVAILABLE;
|
|
685
869
|
};
|
|
686
870
|
exports.AuthenticationGate = class AuthenticationGate {
|
|
687
871
|
constructor(options, authenticator) {
|
|
@@ -697,8 +881,17 @@ exports.AuthenticationGate = class AuthenticationGate {
|
|
|
697
881
|
refusals = windowedReport(
|
|
698
882
|
() => ({ saturated: 0, timeout: 0 }),
|
|
699
883
|
({ saturated, timeout }) => {
|
|
700
|
-
|
|
701
|
-
|
|
884
|
+
logEvent(
|
|
885
|
+
this.logger,
|
|
886
|
+
"warn",
|
|
887
|
+
"REALTIME_HANDSHAKES_REFUSED",
|
|
888
|
+
`handshakes refused since the last report: ${saturated} because ${this.maxPending} were already waiting for the authenticator, ${timeout} because it did not answer in ${this.timeoutMs} ms`,
|
|
889
|
+
{
|
|
890
|
+
saturated,
|
|
891
|
+
timedOut: timeout,
|
|
892
|
+
maxPendingHandshakes: this.maxPending,
|
|
893
|
+
authenticateTimeoutMs: this.timeoutMs
|
|
894
|
+
}
|
|
702
895
|
);
|
|
703
896
|
}
|
|
704
897
|
);
|
|
@@ -814,6 +1007,7 @@ var ConnectionLimitError = class extends Error {
|
|
|
814
1007
|
this.limit = limit;
|
|
815
1008
|
}
|
|
816
1009
|
limit;
|
|
1010
|
+
code = REALTIME_ERROR_CODES.TRANSPORT_UNAVAILABLE;
|
|
817
1011
|
};
|
|
818
1012
|
var UNBOUNDED_TICKET = {
|
|
819
1013
|
release: () => void 0,
|
|
@@ -833,8 +1027,12 @@ exports.ConnectionAdmission = class ConnectionAdmission {
|
|
|
833
1027
|
refusals = windowedReport(
|
|
834
1028
|
() => 0,
|
|
835
1029
|
(count) => {
|
|
836
|
-
|
|
837
|
-
|
|
1030
|
+
logEvent(
|
|
1031
|
+
this.logger,
|
|
1032
|
+
"warn",
|
|
1033
|
+
"REALTIME_CONNECTIONS_REFUSED",
|
|
1034
|
+
`connections refused since the last report: ${count}, because the instance already holds ${this.max} (maxConnections)`,
|
|
1035
|
+
{ count, maxConnections: this.max }
|
|
838
1036
|
);
|
|
839
1037
|
}
|
|
840
1038
|
);
|
|
@@ -1218,7 +1416,7 @@ var StallWatch = class {
|
|
|
1218
1416
|
// src/server/transports/sse/sse.transport.ts
|
|
1219
1417
|
var _shuttingDown;
|
|
1220
1418
|
exports.SseTransport = class SseTransport {
|
|
1221
|
-
constructor(connections, rooms, replayBuffer, idGen, heartbeat, gate, pubsub, hooks, options, instanceId, offlineQueue, admission) {
|
|
1419
|
+
constructor(connections, rooms, replayBuffer, idGen, heartbeat, gate, pubsub, hooks, options, instanceId, offlineQueue, admission, tally) {
|
|
1222
1420
|
this.connections = connections;
|
|
1223
1421
|
this.rooms = rooms;
|
|
1224
1422
|
this.replayBuffer = replayBuffer;
|
|
@@ -1231,6 +1429,7 @@ exports.SseTransport = class SseTransport {
|
|
|
1231
1429
|
this.instanceId = instanceId;
|
|
1232
1430
|
this.offlineQueue = offlineQueue;
|
|
1233
1431
|
this.admission = admission;
|
|
1432
|
+
this.tally = tally;
|
|
1234
1433
|
__publicField(this, "kind", "sse");
|
|
1235
1434
|
__publicField(this, "logger", new common.Logger(exports.SseTransport.name));
|
|
1236
1435
|
__privateAdd(this, _shuttingDown, false);
|
|
@@ -1248,6 +1447,7 @@ exports.SseTransport = class SseTransport {
|
|
|
1248
1447
|
instanceId;
|
|
1249
1448
|
offlineQueue;
|
|
1250
1449
|
admission;
|
|
1450
|
+
tally;
|
|
1251
1451
|
onModuleDestroy() {
|
|
1252
1452
|
__privateSet(this, _shuttingDown, true);
|
|
1253
1453
|
}
|
|
@@ -1334,7 +1534,18 @@ exports.SseTransport = class SseTransport {
|
|
|
1334
1534
|
}),
|
|
1335
1535
|
this.options.hookTimeoutMs ?? DEFAULT_HOOK_TIMEOUT_MS
|
|
1336
1536
|
);
|
|
1337
|
-
if (failed)
|
|
1537
|
+
if (failed) {
|
|
1538
|
+
logEvent(
|
|
1539
|
+
this.logger,
|
|
1540
|
+
"error",
|
|
1541
|
+
"REALTIME_HOOK_FAILED",
|
|
1542
|
+
`onDisconnect hook failed`,
|
|
1543
|
+
{
|
|
1544
|
+
hook: "onDisconnect"
|
|
1545
|
+
},
|
|
1546
|
+
failed.error
|
|
1547
|
+
);
|
|
1548
|
+
}
|
|
1338
1549
|
}
|
|
1339
1550
|
getReplayEvents(userId, lastEventId) {
|
|
1340
1551
|
return this.replayBuffer.since(userId, lastEventId);
|
|
@@ -1436,15 +1647,18 @@ exports.SseTransport = class SseTransport {
|
|
|
1436
1647
|
}
|
|
1437
1648
|
emitToUserLocal(userId, event, data, id) {
|
|
1438
1649
|
const message = this.buildMessage(id, event, data);
|
|
1650
|
+
this.tally?.emitted("sse");
|
|
1439
1651
|
this.replayBuffer.append(userId, message);
|
|
1440
1652
|
for (const conn of this.connections.byUser(userId, "sse")) this.deliver(conn, message);
|
|
1441
1653
|
}
|
|
1442
1654
|
emitToTenantLocal(tenantId, event, data, id) {
|
|
1443
1655
|
const message = this.buildMessage(id, event, data);
|
|
1656
|
+
this.tally?.emitted("sse");
|
|
1444
1657
|
for (const conn of this.connections.byTenant(tenantId, "sse")) this.deliver(conn, message);
|
|
1445
1658
|
}
|
|
1446
1659
|
emitToRoomLocal(roomId, event, data, id) {
|
|
1447
1660
|
const message = this.buildMessage(id, event, data);
|
|
1661
|
+
this.tally?.emitted("sse");
|
|
1448
1662
|
for (const connectionId of this.rooms.members(roomId)) {
|
|
1449
1663
|
const conn = this.connections.get(connectionId);
|
|
1450
1664
|
if (conn && conn.transport === "sse") this.deliver(conn, message);
|
|
@@ -1452,6 +1666,7 @@ exports.SseTransport = class SseTransport {
|
|
|
1452
1666
|
}
|
|
1453
1667
|
broadcastLocal(event, data, id) {
|
|
1454
1668
|
const message = this.buildMessage(id, event, data);
|
|
1669
|
+
this.tally?.emitted("sse");
|
|
1455
1670
|
for (const conn of this.connections.allByTransport("sse")) this.deliver(conn, message);
|
|
1456
1671
|
}
|
|
1457
1672
|
async disconnectLocal(connectionId, reason) {
|
|
@@ -1474,7 +1689,16 @@ exports.SseTransport = class SseTransport {
|
|
|
1474
1689
|
}
|
|
1475
1690
|
conn.subject?.next(message);
|
|
1476
1691
|
} catch (error) {
|
|
1477
|
-
|
|
1692
|
+
logEvent(
|
|
1693
|
+
this.logger,
|
|
1694
|
+
"warn",
|
|
1695
|
+
"REALTIME_DELIVERY_FAILED",
|
|
1696
|
+
`SSE delivery failed`,
|
|
1697
|
+
{
|
|
1698
|
+
connectionId: conn.connectionId
|
|
1699
|
+
},
|
|
1700
|
+
error
|
|
1701
|
+
);
|
|
1478
1702
|
}
|
|
1479
1703
|
}
|
|
1480
1704
|
track(params) {
|
|
@@ -1501,13 +1725,23 @@ exports.SseTransport = class SseTransport {
|
|
|
1501
1725
|
}
|
|
1502
1726
|
}
|
|
1503
1727
|
cutSlowConsumer(connectionId, why, tracked) {
|
|
1504
|
-
this.
|
|
1505
|
-
|
|
1506
|
-
|
|
1728
|
+
this.tally?.evicted("slow_consumer");
|
|
1729
|
+
this.tally?.slowConsumerDropped();
|
|
1730
|
+
logEvent(this.logger, "warn", "REALTIME_SLOW_CONSUMER", `closing an SSE connection, ${why}`, {
|
|
1731
|
+
connectionId,
|
|
1732
|
+
queuedEvents: tracked.backlog.queuedEvents,
|
|
1733
|
+
queuedBytes: tracked.backlog.queuedBytes,
|
|
1734
|
+
heldBytes: tracked.probe.buffered()
|
|
1735
|
+
});
|
|
1507
1736
|
this.disconnectLocal(connectionId, REALTIME_ERROR_CODES.SLOW_CONSUMER).catch(
|
|
1508
1737
|
(error) => {
|
|
1509
|
-
|
|
1510
|
-
|
|
1738
|
+
logEvent(
|
|
1739
|
+
this.logger,
|
|
1740
|
+
"warn",
|
|
1741
|
+
"REALTIME_DISCONNECT_FAILED",
|
|
1742
|
+
`Closing a slow SSE connection failed`,
|
|
1743
|
+
{ connectionId },
|
|
1744
|
+
error
|
|
1511
1745
|
);
|
|
1512
1746
|
}
|
|
1513
1747
|
);
|
|
@@ -1538,6 +1772,7 @@ exports.SseTransport = class SseTransport {
|
|
|
1538
1772
|
let userConnections = this.connections.byUser(userId, "sse");
|
|
1539
1773
|
while (userConnections.length > max) {
|
|
1540
1774
|
const oldest = userConnections.reduce((a, b) => a.connectedAt <= b.connectedAt ? a : b);
|
|
1775
|
+
this.tally?.evicted("per_user_cap");
|
|
1541
1776
|
await this.disconnectLocal(oldest.connectionId, REALTIME_ERROR_CODES.TOO_MANY_CONNECTIONS);
|
|
1542
1777
|
userConnections = this.connections.byUser(userId, "sse");
|
|
1543
1778
|
}
|
|
@@ -1559,7 +1794,9 @@ exports.SseTransport = __decorateClass([
|
|
|
1559
1794
|
__decorateParam(10, common.Optional()),
|
|
1560
1795
|
__decorateParam(10, common.Inject(REALTIME_OFFLINE_QUEUE_TOKEN)),
|
|
1561
1796
|
__decorateParam(11, common.Optional()),
|
|
1562
|
-
__decorateParam(11, common.Inject(exports.ConnectionAdmission))
|
|
1797
|
+
__decorateParam(11, common.Inject(exports.ConnectionAdmission)),
|
|
1798
|
+
__decorateParam(12, common.Optional()),
|
|
1799
|
+
__decorateParam(12, common.Inject(exports.RealtimeMetricsTally))
|
|
1563
1800
|
], exports.SseTransport);
|
|
1564
1801
|
|
|
1565
1802
|
// src/server/transports/sse/response-close.ts
|
|
@@ -1633,18 +1870,22 @@ function buildMeta(record) {
|
|
|
1633
1870
|
};
|
|
1634
1871
|
}
|
|
1635
1872
|
exports.SseSubscriptionHandler = class SseSubscriptionHandler {
|
|
1636
|
-
constructor(transport, heartbeat, options, hooks, offlineDelivery) {
|
|
1873
|
+
constructor(transport, heartbeat, options, hooks, offlineDelivery, tally, tracing) {
|
|
1637
1874
|
this.transport = transport;
|
|
1638
1875
|
this.heartbeat = heartbeat;
|
|
1639
1876
|
this.options = options;
|
|
1640
1877
|
this.hooks = hooks;
|
|
1641
1878
|
this.offlineDelivery = offlineDelivery;
|
|
1879
|
+
this.tally = tally;
|
|
1880
|
+
this.tracing = tracing;
|
|
1642
1881
|
}
|
|
1643
1882
|
transport;
|
|
1644
1883
|
heartbeat;
|
|
1645
1884
|
options;
|
|
1646
1885
|
hooks;
|
|
1647
1886
|
offlineDelivery;
|
|
1887
|
+
tally;
|
|
1888
|
+
tracing;
|
|
1648
1889
|
logger = new common.Logger(exports.SseSubscriptionHandler.name);
|
|
1649
1890
|
authenticated = /* @__PURE__ */ new WeakMap();
|
|
1650
1891
|
resolveAndValidateIdentity(auth) {
|
|
@@ -1653,7 +1894,15 @@ exports.SseSubscriptionHandler = class SseSubscriptionHandler {
|
|
|
1653
1894
|
assertIdentityIsNotBlank(resolvedAuth);
|
|
1654
1895
|
resolvedAuth = admitRooms(resolvedAuth);
|
|
1655
1896
|
} catch (err) {
|
|
1656
|
-
this.
|
|
1897
|
+
this.tally?.refused("error");
|
|
1898
|
+
logEvent(
|
|
1899
|
+
this.logger,
|
|
1900
|
+
"error",
|
|
1901
|
+
"REALTIME_HANDSHAKE_FAILED",
|
|
1902
|
+
`SSE connection refused \u2014 responding 500`,
|
|
1903
|
+
{},
|
|
1904
|
+
err
|
|
1905
|
+
);
|
|
1657
1906
|
throw new common.InternalServerErrorException();
|
|
1658
1907
|
}
|
|
1659
1908
|
return resolvedAuth;
|
|
@@ -1688,6 +1937,7 @@ exports.SseSubscriptionHandler = class SseSubscriptionHandler {
|
|
|
1688
1937
|
}
|
|
1689
1938
|
refuseDuringShutdown() {
|
|
1690
1939
|
if (this.transport.shuttingDown) {
|
|
1940
|
+
this.tally?.refused("unavailable");
|
|
1691
1941
|
throw new common.ServiceUnavailableException(REALTIME_ERROR_CODES.TRANSPORT_UNAVAILABLE);
|
|
1692
1942
|
}
|
|
1693
1943
|
}
|
|
@@ -1725,16 +1975,29 @@ exports.SseSubscriptionHandler = class SseSubscriptionHandler {
|
|
|
1725
1975
|
}
|
|
1726
1976
|
}
|
|
1727
1977
|
toClientError(err, what) {
|
|
1728
|
-
if (err instanceof
|
|
1978
|
+
if (err instanceof ConnectionLimitError) {
|
|
1979
|
+
this.tally?.refused("limit");
|
|
1980
|
+
return new common.ServiceUnavailableException();
|
|
1981
|
+
}
|
|
1982
|
+
if (err instanceof AuthenticationUnavailableError) {
|
|
1983
|
+
this.tally?.refused("unavailable");
|
|
1729
1984
|
return new common.ServiceUnavailableException();
|
|
1730
1985
|
}
|
|
1986
|
+
if (err instanceof common.HttpException && err.getStatus() === common.HttpStatus.UNAUTHORIZED) {
|
|
1987
|
+
this.tally?.refused("unauthorized");
|
|
1988
|
+
}
|
|
1731
1989
|
return err instanceof common.HttpException ? err : this.serverError(err, what);
|
|
1732
1990
|
}
|
|
1733
1991
|
serverError(err, what) {
|
|
1734
|
-
this.
|
|
1992
|
+
this.tally?.refused("error");
|
|
1993
|
+
logEvent(this.logger, "error", "REALTIME_HANDSHAKE_FAILED", `${what} \u2014 responding 500`, {}, err);
|
|
1735
1994
|
return new common.InternalServerErrorException();
|
|
1736
1995
|
}
|
|
1737
|
-
|
|
1996
|
+
authenticateRequest(req, res) {
|
|
1997
|
+
const run = () => this.admitAndAuthenticate(req, res);
|
|
1998
|
+
return this.tracing ? this.tracing.span("realtime.handshake", { "bymax.realtime.transport": "sse" }, run) : run();
|
|
1999
|
+
}
|
|
2000
|
+
async admitAndAuthenticate(req, res) {
|
|
1738
2001
|
const held = this.transport.admit();
|
|
1739
2002
|
const detach = onResponseClose(res, () => ticket.release());
|
|
1740
2003
|
const ticket = {
|
|
@@ -1868,7 +2131,14 @@ exports.SseSubscriptionHandler = class SseSubscriptionHandler {
|
|
|
1868
2131
|
fireHook(invoke, onFailure) {
|
|
1869
2132
|
void runHook(invoke).then((failed) => {
|
|
1870
2133
|
if (!failed) return;
|
|
1871
|
-
|
|
2134
|
+
logEvent(
|
|
2135
|
+
this.logger,
|
|
2136
|
+
"warn",
|
|
2137
|
+
"REALTIME_HOOK_FAILED",
|
|
2138
|
+
`Lifecycle hook failed`,
|
|
2139
|
+
{},
|
|
2140
|
+
failed.error
|
|
2141
|
+
);
|
|
1872
2142
|
onFailure?.();
|
|
1873
2143
|
});
|
|
1874
2144
|
}
|
|
@@ -1881,7 +2151,11 @@ exports.SseSubscriptionHandler = __decorateClass([
|
|
|
1881
2151
|
__decorateParam(3, common.Optional()),
|
|
1882
2152
|
__decorateParam(3, common.Inject(REALTIME_HOOKS_TOKEN)),
|
|
1883
2153
|
__decorateParam(4, common.Optional()),
|
|
1884
|
-
__decorateParam(4, common.Inject(exports.OfflineQueueDeliveryService))
|
|
2154
|
+
__decorateParam(4, common.Inject(exports.OfflineQueueDeliveryService)),
|
|
2155
|
+
__decorateParam(5, common.Optional()),
|
|
2156
|
+
__decorateParam(5, common.Inject(exports.RealtimeMetricsTally)),
|
|
2157
|
+
__decorateParam(6, common.Optional()),
|
|
2158
|
+
__decorateParam(6, common.Inject(exports.RealtimeTracing))
|
|
1885
2159
|
], exports.SseSubscriptionHandler);
|
|
1886
2160
|
|
|
1887
2161
|
// src/server/utils/normalize-endpoint-path.ts
|
|
@@ -1932,12 +2206,14 @@ function createSseController(endpoint) {
|
|
|
1932
2206
|
return DynamicSseController;
|
|
1933
2207
|
}
|
|
1934
2208
|
exports.RealtimeService = class RealtimeService {
|
|
1935
|
-
constructor(transport, pubsub) {
|
|
2209
|
+
constructor(transport, pubsub, tally) {
|
|
1936
2210
|
this.transport = transport;
|
|
1937
2211
|
this.pubsub = pubsub;
|
|
2212
|
+
this.tally = tally;
|
|
1938
2213
|
}
|
|
1939
2214
|
transport;
|
|
1940
2215
|
pubsub;
|
|
2216
|
+
tally;
|
|
1941
2217
|
isPubSubReady() {
|
|
1942
2218
|
if (this.transport.kind === "websocket") return true;
|
|
1943
2219
|
return this.pubsub?.isSubscribed?.() ?? true;
|
|
@@ -1963,15 +2239,18 @@ exports.RealtimeService = class RealtimeService {
|
|
|
1963
2239
|
refuseReservedRoom("leaveRoom", roomId);
|
|
1964
2240
|
return this.transport.leaveRoom(connectionId, roomId);
|
|
1965
2241
|
}
|
|
1966
|
-
disconnect(connectionId, reason) {
|
|
1967
|
-
|
|
2242
|
+
async disconnect(connectionId, reason) {
|
|
2243
|
+
await this.transport.disconnect(connectionId, reason);
|
|
2244
|
+
this.tally?.evicted("revoked");
|
|
1968
2245
|
}
|
|
1969
2246
|
};
|
|
1970
2247
|
exports.RealtimeService = __decorateClass([
|
|
1971
2248
|
common.Injectable(),
|
|
1972
2249
|
__decorateParam(0, common.Inject(REALTIME_TRANSPORT_TOKEN)),
|
|
1973
2250
|
__decorateParam(1, common.Optional()),
|
|
1974
|
-
__decorateParam(1, common.Inject(REALTIME_PUBSUB_TOKEN))
|
|
2251
|
+
__decorateParam(1, common.Inject(REALTIME_PUBSUB_TOKEN)),
|
|
2252
|
+
__decorateParam(2, common.Optional()),
|
|
2253
|
+
__decorateParam(2, common.Inject(exports.RealtimeMetricsTally))
|
|
1975
2254
|
], exports.RealtimeService);
|
|
1976
2255
|
|
|
1977
2256
|
// src/server/utils/semaphore.ts
|
|
@@ -2005,31 +2284,55 @@ function resolvePolicy(raw) {
|
|
|
2005
2284
|
};
|
|
2006
2285
|
}
|
|
2007
2286
|
exports.ReauthenticationService = class ReauthenticationService {
|
|
2008
|
-
constructor(connections, realtime, auth, options, hooks) {
|
|
2287
|
+
constructor(connections, realtime, auth, options, hooks, tracing) {
|
|
2009
2288
|
this.connections = connections;
|
|
2010
2289
|
this.realtime = realtime;
|
|
2011
2290
|
this.auth = auth;
|
|
2012
2291
|
this.hooks = hooks;
|
|
2292
|
+
this.tracing = tracing;
|
|
2013
2293
|
this.policy = resolvePolicy(options.reauthenticationPolicy);
|
|
2294
|
+
this.declared = options.reauthenticationPolicy;
|
|
2014
2295
|
this.slots = new Semaphore(this.policy.concurrency);
|
|
2015
2296
|
}
|
|
2016
2297
|
connections;
|
|
2017
2298
|
realtime;
|
|
2018
2299
|
auth;
|
|
2019
2300
|
hooks;
|
|
2301
|
+
tracing;
|
|
2020
2302
|
logger = new common.Logger(exports.ReauthenticationService.name);
|
|
2021
2303
|
timer = null;
|
|
2022
2304
|
policy;
|
|
2305
|
+
declared;
|
|
2023
2306
|
inFlight = /* @__PURE__ */ new Set();
|
|
2024
2307
|
slots;
|
|
2308
|
+
warnIfCacheOutlivesInterval() {
|
|
2309
|
+
const { cacheTtlMs, intervalSeconds } = this.declared ?? {};
|
|
2310
|
+
if (cacheTtlMs === void 0 || intervalSeconds === void 0) return;
|
|
2311
|
+
if (cacheTtlMs <= intervalSeconds * 1e3) return;
|
|
2312
|
+
logEvent(
|
|
2313
|
+
this.logger,
|
|
2314
|
+
"warn",
|
|
2315
|
+
"REALTIME_REAUTH_CACHE_WEAKENS_POLICY",
|
|
2316
|
+
`reauthenticationPolicy.cacheTtlMs (${cacheTtlMs} ms) exceeds intervalSeconds*1000 (${intervalSeconds * 1e3} ms): the positive reauth cache outlives the interval, so revalidate is skipped on ticks within the cache window \u2014 reauthentication is effectively weakened`,
|
|
2317
|
+
{ cacheTtlMs, intervalMs: intervalSeconds * 1e3 }
|
|
2318
|
+
);
|
|
2319
|
+
}
|
|
2025
2320
|
onModuleInit() {
|
|
2026
2321
|
if (!this.auth.revalidate) {
|
|
2027
2322
|
this.logger.log("Authenticator does not implement revalidate() \u2014 reauthentication disabled");
|
|
2028
2323
|
return;
|
|
2029
2324
|
}
|
|
2325
|
+
this.warnIfCacheOutlivesInterval();
|
|
2030
2326
|
this.timer = setInterval(() => {
|
|
2031
2327
|
this.runCycle().catch((err) => {
|
|
2032
|
-
|
|
2328
|
+
logEvent(
|
|
2329
|
+
this.logger,
|
|
2330
|
+
"warn",
|
|
2331
|
+
"REALTIME_REAUTH_FAILED",
|
|
2332
|
+
`Reauthentication cycle failed`,
|
|
2333
|
+
{},
|
|
2334
|
+
err
|
|
2335
|
+
);
|
|
2033
2336
|
});
|
|
2034
2337
|
}, this.policy.intervalSeconds * 1e3);
|
|
2035
2338
|
this.timer.unref();
|
|
@@ -2047,7 +2350,10 @@ exports.ReauthenticationService = class ReauthenticationService {
|
|
|
2047
2350
|
...this.connections.allByTransport("sse"),
|
|
2048
2351
|
...this.connections.allByTransport("websocket")
|
|
2049
2352
|
];
|
|
2050
|
-
|
|
2353
|
+
const run = async () => {
|
|
2354
|
+
await Promise.all(connections.map((conn) => this.check(conn, now)));
|
|
2355
|
+
};
|
|
2356
|
+
await (this.tracing ? this.tracing.span("realtime.reauthenticate", {}, run) : run());
|
|
2051
2357
|
}
|
|
2052
2358
|
async check(conn, now) {
|
|
2053
2359
|
if (this.inFlight.has(conn.connectionId)) return;
|
|
@@ -2084,7 +2390,14 @@ exports.ReauthenticationService = class ReauthenticationService {
|
|
|
2084
2390
|
}
|
|
2085
2391
|
await this.revoke(conn);
|
|
2086
2392
|
} catch (err) {
|
|
2087
|
-
|
|
2393
|
+
logEvent(
|
|
2394
|
+
this.logger,
|
|
2395
|
+
"warn",
|
|
2396
|
+
"REALTIME_REAUTH_FAILED",
|
|
2397
|
+
`Reauthentication errored`,
|
|
2398
|
+
{ connectionId: conn.connectionId },
|
|
2399
|
+
err
|
|
2400
|
+
);
|
|
2088
2401
|
await this.revoke(conn);
|
|
2089
2402
|
}
|
|
2090
2403
|
}
|
|
@@ -2108,8 +2421,13 @@ exports.ReauthenticationService = class ReauthenticationService {
|
|
|
2108
2421
|
try {
|
|
2109
2422
|
await this.bounded("revocation", () => this.handleFailure(conn));
|
|
2110
2423
|
} catch (err) {
|
|
2111
|
-
|
|
2112
|
-
|
|
2424
|
+
logEvent(
|
|
2425
|
+
this.logger,
|
|
2426
|
+
"warn",
|
|
2427
|
+
"REALTIME_REVOCATION_FAILED",
|
|
2428
|
+
`Revocation failed, retrying next cycle`,
|
|
2429
|
+
{ connectionId: conn.connectionId },
|
|
2430
|
+
err
|
|
2113
2431
|
);
|
|
2114
2432
|
}
|
|
2115
2433
|
}
|
|
@@ -2130,15 +2448,27 @@ exports.ReauthenticationService = class ReauthenticationService {
|
|
|
2130
2448
|
})
|
|
2131
2449
|
).then((failed) => {
|
|
2132
2450
|
if (failed) {
|
|
2133
|
-
|
|
2451
|
+
logEvent(
|
|
2452
|
+
this.logger,
|
|
2453
|
+
"warn",
|
|
2454
|
+
"REALTIME_HOOK_FAILED",
|
|
2455
|
+
`onReauthenticationFailed hook failed`,
|
|
2456
|
+
{ hook: "onReauthenticationFailed" },
|
|
2457
|
+
failed.error
|
|
2458
|
+
);
|
|
2134
2459
|
}
|
|
2135
2460
|
});
|
|
2136
2461
|
await this.realtime.disconnect(conn.connectionId, REALTIME_ERROR_CODES.REAUTHENTICATION_FAILED);
|
|
2137
2462
|
}
|
|
2138
2463
|
notify(conn) {
|
|
2139
2464
|
const failed = (err) => {
|
|
2140
|
-
|
|
2141
|
-
|
|
2465
|
+
logEvent(
|
|
2466
|
+
this.logger,
|
|
2467
|
+
"warn",
|
|
2468
|
+
"REALTIME_REAUTH_FAILED",
|
|
2469
|
+
`Reauthentication notification failed, disconnecting anyway`,
|
|
2470
|
+
{ connectionId: conn.connectionId },
|
|
2471
|
+
err
|
|
2142
2472
|
);
|
|
2143
2473
|
};
|
|
2144
2474
|
try {
|
|
@@ -2157,7 +2487,9 @@ exports.ReauthenticationService = __decorateClass([
|
|
|
2157
2487
|
__decorateParam(2, common.Inject(REALTIME_AUTHENTICATOR_TOKEN)),
|
|
2158
2488
|
__decorateParam(3, common.Inject(REALTIME_OPTIONS_TOKEN)),
|
|
2159
2489
|
__decorateParam(4, common.Optional()),
|
|
2160
|
-
__decorateParam(4, common.Inject(REALTIME_HOOKS_TOKEN))
|
|
2490
|
+
__decorateParam(4, common.Inject(REALTIME_HOOKS_TOKEN)),
|
|
2491
|
+
__decorateParam(5, common.Optional()),
|
|
2492
|
+
__decorateParam(5, common.Inject(exports.RealtimeTracing))
|
|
2161
2493
|
], exports.ReauthenticationService);
|
|
2162
2494
|
|
|
2163
2495
|
// src/server/composition/realtime-module.factory.ts
|
|
@@ -2165,6 +2497,7 @@ var DEFAULT_ASYNC_SSE_ENDPOINT = "/events";
|
|
|
2165
2497
|
var EXPORTED_TOKENS = [
|
|
2166
2498
|
exports.RealtimeService,
|
|
2167
2499
|
exports.ConnectionRegistry,
|
|
2500
|
+
exports.RealtimeMetricsTally,
|
|
2168
2501
|
REALTIME_OPTIONS_TOKEN,
|
|
2169
2502
|
REALTIME_INSTANCE_ID_TOKEN,
|
|
2170
2503
|
REALTIME_AUTHENTICATOR_TOKEN,
|
|
@@ -2184,7 +2517,9 @@ var SHARED_SERVICES = [
|
|
|
2184
2517
|
exports.ReauthenticationService,
|
|
2185
2518
|
exports.OfflineQueueDeliveryService,
|
|
2186
2519
|
exports.AuthenticationGate,
|
|
2187
|
-
exports.ConnectionAdmission
|
|
2520
|
+
exports.ConnectionAdmission,
|
|
2521
|
+
exports.RealtimeMetricsTally,
|
|
2522
|
+
exports.RealtimeTracing
|
|
2188
2523
|
];
|
|
2189
2524
|
function assertModeSupported(wiring, mode, moduleName) {
|
|
2190
2525
|
if (wiring.modes.includes(mode)) return;
|
|
@@ -2198,7 +2533,10 @@ function assertModeSupported(wiring, mode, moduleName) {
|
|
|
2198
2533
|
function warnIfSingleInstance(pubsub, transport, logger) {
|
|
2199
2534
|
if (transport === "websocket") return;
|
|
2200
2535
|
if (!pubsub && process.env["NODE_ENV"] === "production") {
|
|
2201
|
-
|
|
2536
|
+
logEvent(
|
|
2537
|
+
logger,
|
|
2538
|
+
"warn",
|
|
2539
|
+
"REALTIME_PUBSUB_NOT_PROVIDED",
|
|
2202
2540
|
"No IRealtimePubSub provided in production \u2014 single-instance only. Provide a Redis-backed IRealtimePubSub for horizontal scaling."
|
|
2203
2541
|
);
|
|
2204
2542
|
}
|
|
@@ -2216,6 +2554,7 @@ function buildCommonProviders(resolved, instanceId) {
|
|
|
2216
2554
|
];
|
|
2217
2555
|
}
|
|
2218
2556
|
function composeForRoot(moduleClass, wiring, logger, raw) {
|
|
2557
|
+
assertOptionsObject(raw, moduleClass.name);
|
|
2219
2558
|
const options = dropUnset(raw);
|
|
2220
2559
|
validateOptions(options);
|
|
2221
2560
|
assertModeSupported(wiring, options.transport, moduleClass.name);
|
|
@@ -2250,9 +2589,10 @@ function bindSseEndpoint(resolved, declared, bound, moduleName) {
|
|
|
2250
2589
|
return Object.freeze({ ...resolved, sse: { ...resolved.sse, endpoint: bound } });
|
|
2251
2590
|
}
|
|
2252
2591
|
function resolveAsyncOptions(raw, declaredMode, instanceId, source, logger, moduleName) {
|
|
2253
|
-
if (
|
|
2592
|
+
if (raw === void 0 || raw === null) {
|
|
2254
2593
|
throw realtimeError(REALTIME_ERROR_CODES.INVALID_OPTIONS, `${source} returned nothing`);
|
|
2255
2594
|
}
|
|
2595
|
+
assertOptionsObject(raw, moduleName);
|
|
2256
2596
|
const options = dropUnset(raw);
|
|
2257
2597
|
validateOptions(options);
|
|
2258
2598
|
if (options.transport !== declaredMode) {
|
|
@@ -2266,12 +2606,15 @@ function resolveAsyncOptions(raw, declaredMode, instanceId, source, logger, modu
|
|
|
2266
2606
|
logger.log(`Bootstrapped (transport=${resolved.transport}, instanceId=${instanceId})`);
|
|
2267
2607
|
return { resolved, declaredEndpoint: options.sse?.endpoint };
|
|
2268
2608
|
}
|
|
2609
|
+
function isSet(pattern) {
|
|
2610
|
+
return pattern !== void 0 && pattern !== null;
|
|
2611
|
+
}
|
|
2269
2612
|
function assertOneFactoryPattern(asyncOptions, moduleName) {
|
|
2270
2613
|
const patterns = [
|
|
2271
2614
|
asyncOptions.useFactory,
|
|
2272
2615
|
asyncOptions.useClass,
|
|
2273
2616
|
asyncOptions.useExisting
|
|
2274
|
-
].filter(
|
|
2617
|
+
].filter(isSet);
|
|
2275
2618
|
if (patterns.length === 1) return;
|
|
2276
2619
|
throw realtimeError(
|
|
2277
2620
|
REALTIME_ERROR_CODES.INVALID_OPTIONS,
|
|
@@ -2280,7 +2623,7 @@ function assertOneFactoryPattern(asyncOptions, moduleName) {
|
|
|
2280
2623
|
);
|
|
2281
2624
|
}
|
|
2282
2625
|
function buildOptionsProvider(asyncOptions, resolve, factoryToken) {
|
|
2283
|
-
if (asyncOptions.useFactory) {
|
|
2626
|
+
if (isSet(asyncOptions.useFactory)) {
|
|
2284
2627
|
return {
|
|
2285
2628
|
provide: REALTIME_OPTIONS_TOKEN,
|
|
2286
2629
|
useFactory: async (...args) => resolve(await asyncOptions.useFactory(...args), "useFactory"),
|
|
@@ -2289,10 +2632,18 @@ function buildOptionsProvider(asyncOptions, resolve, factoryToken) {
|
|
|
2289
2632
|
}
|
|
2290
2633
|
return {
|
|
2291
2634
|
provide: REALTIME_OPTIONS_TOKEN,
|
|
2292
|
-
useFactory: async (factory) =>
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
|
|
2635
|
+
useFactory: async (factory) => {
|
|
2636
|
+
if (typeof factory?.createRealtimeOptions !== "function") {
|
|
2637
|
+
throw realtimeError(
|
|
2638
|
+
REALTIME_ERROR_CODES.INVALID_OPTIONS,
|
|
2639
|
+
"the provider named by useExisting or useClass must implement createRealtimeOptions"
|
|
2640
|
+
);
|
|
2641
|
+
}
|
|
2642
|
+
return resolve(
|
|
2643
|
+
await factory.createRealtimeOptions(),
|
|
2644
|
+
"createRealtimeOptions"
|
|
2645
|
+
);
|
|
2646
|
+
},
|
|
2296
2647
|
inject: [asyncOptions.useExisting ?? factoryToken]
|
|
2297
2648
|
};
|
|
2298
2649
|
}
|
|
@@ -2328,14 +2679,16 @@ function buildResolver(asyncOptions, instanceId, boundSseEndpoint, logger, modul
|
|
|
2328
2679
|
}
|
|
2329
2680
|
function composeForRootAsync(moduleClass, wiring, logger, asyncOptions) {
|
|
2330
2681
|
const moduleName = moduleClass.name;
|
|
2682
|
+
assertOptionsObject(asyncOptions, moduleName);
|
|
2331
2683
|
assertOneFactoryPattern(asyncOptions, moduleName);
|
|
2332
2684
|
assertModeSupported(wiring, asyncOptions.transport, moduleName);
|
|
2685
|
+
assertSseEndpoint("sseEndpoint", asyncOptions.sseEndpoint);
|
|
2333
2686
|
assertSseEndpointApplies(asyncOptions, moduleName);
|
|
2334
2687
|
const FACTORY_TOKEN = /* @__PURE__ */ Symbol("REALTIME_OPTIONS_FACTORY");
|
|
2335
2688
|
const instanceId = crypto.randomUUID();
|
|
2336
2689
|
const boundSseEndpoint = asyncOptions.sseEndpoint ?? DEFAULT_ASYNC_SSE_ENDPOINT;
|
|
2337
2690
|
const resolve = buildResolver(asyncOptions, instanceId, boundSseEndpoint, logger, moduleName);
|
|
2338
|
-
const factoryClassProvider = asyncOptions.useClass ? [{ provide: FACTORY_TOKEN, useClass: asyncOptions.useClass }] : [];
|
|
2691
|
+
const factoryClassProvider = isSet(asyncOptions.useClass) ? [{ provide: FACTORY_TOKEN, useClass: asyncOptions.useClass }] : [];
|
|
2339
2692
|
const { providers: transportProviders, controllers } = wiring.buildAsync({
|
|
2340
2693
|
mode: asyncOptions.transport,
|
|
2341
2694
|
sseEndpoint: boundSseEndpoint
|
|
@@ -2374,7 +2727,14 @@ exports.RealtimePubSubSubscriber = class RealtimePubSubSubscriber {
|
|
|
2374
2727
|
try {
|
|
2375
2728
|
await this.unsubscribe();
|
|
2376
2729
|
} catch (err) {
|
|
2377
|
-
|
|
2730
|
+
logEvent(
|
|
2731
|
+
this.logger,
|
|
2732
|
+
"warn",
|
|
2733
|
+
"REALTIME_PUBSUB_UNSUBSCRIBE_FAILED",
|
|
2734
|
+
`Pub/sub unsubscribe failed`,
|
|
2735
|
+
{},
|
|
2736
|
+
err
|
|
2737
|
+
);
|
|
2378
2738
|
}
|
|
2379
2739
|
this.unsubscribe = null;
|
|
2380
2740
|
}
|
|
@@ -2382,8 +2742,12 @@ exports.RealtimePubSubSubscriber = class RealtimePubSubSubscriber {
|
|
|
2382
2742
|
dropped = windowedReport(
|
|
2383
2743
|
() => ({ count: 0, last: "none" }),
|
|
2384
2744
|
({ count, last }) => {
|
|
2385
|
-
|
|
2386
|
-
|
|
2745
|
+
logEvent(
|
|
2746
|
+
this.logger,
|
|
2747
|
+
"warn",
|
|
2748
|
+
"REALTIME_PUBSUB_DROPPED",
|
|
2749
|
+
`Pub/sub dropped ${count} message(s) it could not act on since the last report (last: ${last})`,
|
|
2750
|
+
{ count, last }
|
|
2387
2751
|
);
|
|
2388
2752
|
}
|
|
2389
2753
|
);
|
|
@@ -2411,17 +2775,9 @@ exports.RealtimePubSubSubscriber = class RealtimePubSubSubscriber {
|
|
|
2411
2775
|
this.sse.broadcastLocal(a.event, a.data, a.id);
|
|
2412
2776
|
break;
|
|
2413
2777
|
}
|
|
2414
|
-
case "disconnect":
|
|
2415
|
-
|
|
2416
|
-
Promise.resolve(this.sse.disconnectLocal(a.connectionId, a.reason)).catch(
|
|
2417
|
-
(err) => {
|
|
2418
|
-
this.logger.warn(
|
|
2419
|
-
`Remote disconnect of ${a.connectionId} failed: ${describeError(err)}`
|
|
2420
|
-
);
|
|
2421
|
-
}
|
|
2422
|
-
);
|
|
2778
|
+
case "disconnect":
|
|
2779
|
+
this.closeLocal(msg.args);
|
|
2423
2780
|
break;
|
|
2424
|
-
}
|
|
2425
2781
|
default: {
|
|
2426
2782
|
msg.op;
|
|
2427
2783
|
this.drop("unknown op");
|
|
@@ -2431,6 +2787,18 @@ exports.RealtimePubSubSubscriber = class RealtimePubSubSubscriber {
|
|
|
2431
2787
|
this.drop(String(Object(err).constructor?.name ?? "Error"));
|
|
2432
2788
|
}
|
|
2433
2789
|
}
|
|
2790
|
+
closeLocal(a) {
|
|
2791
|
+
Promise.resolve(this.sse.disconnectLocal(a.connectionId, a.reason)).catch((err) => {
|
|
2792
|
+
logEvent(
|
|
2793
|
+
this.logger,
|
|
2794
|
+
"warn",
|
|
2795
|
+
"REALTIME_DISCONNECT_FAILED",
|
|
2796
|
+
"Remote disconnect failed",
|
|
2797
|
+
{ connectionId: a.connectionId },
|
|
2798
|
+
err
|
|
2799
|
+
);
|
|
2800
|
+
});
|
|
2801
|
+
}
|
|
2434
2802
|
drop(last) {
|
|
2435
2803
|
this.dropped(({ count }) => ({ count: count + 1, last }));
|
|
2436
2804
|
}
|
|
@@ -2524,11 +2892,6 @@ var CANNOT_PUBLISH = /* @__PURE__ */ new Set([
|
|
|
2524
2892
|
var PUBLISHER_WAIT_MS = 2e3;
|
|
2525
2893
|
var PUBLISH_WAIT_MS = 2e3;
|
|
2526
2894
|
var CONNECTION_OVER = ["ready", "close", "reconnecting", "end"];
|
|
2527
|
-
function connectionReason(err) {
|
|
2528
|
-
const code = err?.code;
|
|
2529
|
-
const text = describeError(err);
|
|
2530
|
-
return [typeof code === "string" ? code : "", text].filter(Boolean).join(" ") || String(err);
|
|
2531
|
-
}
|
|
2532
2895
|
var RedisRealtimePubSub = class _RedisRealtimePubSub {
|
|
2533
2896
|
#pub;
|
|
2534
2897
|
channel;
|
|
@@ -2666,7 +3029,7 @@ var RedisRealtimePubSub = class _RedisRealtimePubSub {
|
|
|
2666
3029
|
if (outcome === "subscribed") {
|
|
2667
3030
|
this.markUp();
|
|
2668
3031
|
} else {
|
|
2669
|
-
this.markDown(
|
|
3032
|
+
this.markDown(void 0, outcome.unreachable);
|
|
2670
3033
|
first.catch((err) => this.refused(err));
|
|
2671
3034
|
}
|
|
2672
3035
|
} catch (err) {
|
|
@@ -2692,8 +3055,13 @@ var RedisRealtimePubSub = class _RedisRealtimePubSub {
|
|
|
2692
3055
|
const reason = describeError(err);
|
|
2693
3056
|
if (reason === this.#lastRefusal) return;
|
|
2694
3057
|
this.#lastRefusal = reason;
|
|
2695
|
-
|
|
2696
|
-
|
|
3058
|
+
logEvent(
|
|
3059
|
+
this.logger,
|
|
3060
|
+
"warn",
|
|
3061
|
+
"REALTIME_PUBSUB_UNAVAILABLE",
|
|
3062
|
+
"the server refused the subscription",
|
|
3063
|
+
{},
|
|
3064
|
+
err
|
|
2697
3065
|
);
|
|
2698
3066
|
}
|
|
2699
3067
|
markUp() {
|
|
@@ -2705,20 +3073,29 @@ var RedisRealtimePubSub = class _RedisRealtimePubSub {
|
|
|
2705
3073
|
`Subscribed to "${this.channel}"; remote events and revocations reach this instance`
|
|
2706
3074
|
);
|
|
2707
3075
|
}
|
|
2708
|
-
markDown(
|
|
3076
|
+
markDown(wording, error) {
|
|
2709
3077
|
this.#subscribed = false;
|
|
2710
3078
|
if (this.#outageLogged) return;
|
|
2711
3079
|
this.#outageLogged = true;
|
|
2712
|
-
|
|
2713
|
-
|
|
3080
|
+
logEvent(
|
|
3081
|
+
this.logger,
|
|
3082
|
+
"warn",
|
|
3083
|
+
"REALTIME_PUBSUB_UNAVAILABLE",
|
|
3084
|
+
`not subscribed${wording === void 0 ? "" : `: ${wording}`}. Remote events and revocations will not reach this instance until it subscribes again`,
|
|
3085
|
+
{},
|
|
3086
|
+
error
|
|
2714
3087
|
);
|
|
2715
3088
|
}
|
|
2716
3089
|
reportMalformed() {
|
|
2717
3090
|
this.malformedDropped++;
|
|
2718
3091
|
const now = Date.now();
|
|
2719
3092
|
if (now - this.malformedReportedAt < MALFORMED_REPORT_INTERVAL_MS) return;
|
|
2720
|
-
|
|
2721
|
-
|
|
3093
|
+
logEvent(
|
|
3094
|
+
this.logger,
|
|
3095
|
+
"warn",
|
|
3096
|
+
"REALTIME_PUBSUB_DROPPED",
|
|
3097
|
+
`dropped ${this.malformedDropped} malformed payload(s) since the last report`,
|
|
3098
|
+
{ count: this.malformedDropped, last: "malformed" }
|
|
2722
3099
|
);
|
|
2723
3100
|
this.malformedDropped = 0;
|
|
2724
3101
|
this.malformedReportedAt = now;
|
|
@@ -2765,6 +3142,10 @@ var RedisOfflineQueue = class _RedisOfflineQueue {
|
|
|
2765
3142
|
static MS_PAD = 16;
|
|
2766
3143
|
static COUNTER_PAD = 6;
|
|
2767
3144
|
constructor(options) {
|
|
3145
|
+
if (!isPlainObject(options)) refuseObject("RedisOfflineQueue", "options", options);
|
|
3146
|
+
if (typeof options.client !== "object" || options.client === null) {
|
|
3147
|
+
refuseObject("RedisOfflineQueue", "options.client", options.client);
|
|
3148
|
+
}
|
|
2768
3149
|
this.#client = options.client;
|
|
2769
3150
|
this.maxPerUser = requireInteger(
|
|
2770
3151
|
"maxPerUser",
|
|
@@ -2834,19 +3215,23 @@ exports.ADMISSION_ROOM_LIMITS = ADMISSION_ROOM_LIMITS;
|
|
|
2834
3215
|
exports.AuthenticationUnavailableError = AuthenticationUnavailableError;
|
|
2835
3216
|
exports.ConnectionLimitError = ConnectionLimitError;
|
|
2836
3217
|
exports.DEFAULT_HOOK_TIMEOUT_MS = DEFAULT_HOOK_TIMEOUT_MS;
|
|
3218
|
+
exports.EVICTION_REASONS = EVICTION_REASONS;
|
|
2837
3219
|
exports.REALTIME_AUTHENTICATOR_TOKEN = REALTIME_AUTHENTICATOR_TOKEN;
|
|
2838
3220
|
exports.REALTIME_ERROR_CODES = REALTIME_ERROR_CODES;
|
|
2839
3221
|
exports.REALTIME_HOOKS_TOKEN = REALTIME_HOOKS_TOKEN;
|
|
2840
3222
|
exports.REALTIME_INSTANCE_ID_TOKEN = REALTIME_INSTANCE_ID_TOKEN;
|
|
3223
|
+
exports.REALTIME_LOG_EVENTS = REALTIME_LOG_EVENTS;
|
|
2841
3224
|
exports.REALTIME_OFFLINE_QUEUE_TOKEN = REALTIME_OFFLINE_QUEUE_TOKEN;
|
|
2842
3225
|
exports.REALTIME_OPTIONS_TOKEN = REALTIME_OPTIONS_TOKEN;
|
|
2843
3226
|
exports.REALTIME_PRESENCE_TOKEN = REALTIME_PRESENCE_TOKEN;
|
|
2844
3227
|
exports.REALTIME_PUBSUB_TOKEN = REALTIME_PUBSUB_TOKEN;
|
|
2845
3228
|
exports.REALTIME_TRANSPORT_TOKEN = REALTIME_TRANSPORT_TOKEN;
|
|
3229
|
+
exports.REFUSAL_REASONS = REFUSAL_REASONS;
|
|
2846
3230
|
exports.RESERVED_EVENT_NAMES = RESERVED_EVENT_NAMES;
|
|
2847
3231
|
exports.ROOM_PREFIXES = ROOM_PREFIXES;
|
|
2848
3232
|
exports.RedisOfflineQueue = RedisOfflineQueue;
|
|
2849
3233
|
exports.RedisRealtimePubSub = RedisRealtimePubSub;
|
|
3234
|
+
exports.TALLY_TRANSPORTS = TALLY_TRANSPORTS;
|
|
2850
3235
|
exports.UNBOUNDED_TICKET = UNBOUNDED_TICKET;
|
|
2851
3236
|
exports.admitRooms = admitRooms;
|
|
2852
3237
|
exports.assertIdentityIsNotBlank = assertIdentityIsNotBlank;
|
|
@@ -2855,6 +3240,7 @@ exports.composeForRootAsync = composeForRootAsync;
|
|
|
2855
3240
|
exports.composeRoomId = composeRoomId;
|
|
2856
3241
|
exports.createSseController = createSseController;
|
|
2857
3242
|
exports.describeError = describeError;
|
|
3243
|
+
exports.logEvent = logEvent;
|
|
2858
3244
|
exports.parseCookieHeader = parseCookieHeader;
|
|
2859
3245
|
exports.quitWithin = quitWithin;
|
|
2860
3246
|
exports.realtimeError = realtimeError;
|