@dokebi/websocket-hub 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AI.md +21 -16
- package/README.md +17 -10
- package/dist/delivery.d.ts +13 -47
- package/dist/delivery.js +19 -177
- package/dist/event-queue.d.ts +6 -0
- package/dist/event-queue.js +21 -3
- package/dist/eviction.d.ts +9 -0
- package/dist/eviction.js +57 -0
- package/dist/index.js +14 -3
- package/dist/leader.d.ts +3 -1
- package/dist/leader.js +203 -67
- package/dist/local.d.ts +7 -7
- package/dist/local.js +77 -96
- package/dist/log.d.ts +24 -0
- package/dist/log.js +101 -0
- package/dist/node.js +78 -16
- package/dist/options.d.ts +8 -5
- package/dist/options.js +13 -8
- package/dist/peer-client.d.ts +13 -4
- package/dist/peer-client.js +288 -170
- package/dist/protocol.d.ts +6 -0
- package/dist/protocol.js +6 -0
- package/dist/status-cli.js +7 -2
- package/dist/store.js +11 -6
- package/dist/types.d.ts +18 -5
- package/dist/types.js +2 -2
- package/dist/wire.d.ts +8 -1
- package/dist/wire.js +14 -3
- package/options.schema.json +8 -12
- package/package.json +3 -3
package/dist/local.js
CHANGED
|
@@ -1,34 +1,40 @@
|
|
|
1
1
|
import cluster from 'node:cluster';
|
|
2
2
|
import { randomUUID } from 'node:crypto';
|
|
3
|
-
import {
|
|
3
|
+
import { ConsoleLogger } from '@dokebi/core';
|
|
4
|
+
import { recipientKey, select } from './delivery.js';
|
|
4
5
|
import { EventQueue } from './event-queue.js';
|
|
5
|
-
import {
|
|
6
|
+
import { errorText, LogThrottle } from './log.js';
|
|
7
|
+
import { identifier, matches, metadata, query, record } from './protocol.js';
|
|
6
8
|
import { hubStatus, LocalNonceStore } from './status.js';
|
|
7
9
|
import { HubError, } from './types.js';
|
|
10
|
+
/**
|
|
11
|
+
* Share of the event queue (maxPending events, queueBytes) that broadcasts may fill before they
|
|
12
|
+
* are refused with BUSY; the rest stays for presence and kick events, which cannot be refused.
|
|
13
|
+
*/
|
|
14
|
+
const BROADCAST_SHARE = 0.5;
|
|
8
15
|
/**
|
|
9
16
|
* The hub of one process (D-0029, `mode: 'local'`): the same HubApi as the cluster hub, kept in
|
|
10
|
-
* memory.
|
|
11
|
-
*
|
|
12
|
-
* missed. Undelivered messages are lost with the process, and there is no failover.
|
|
17
|
+
* memory. A broadcast is handed to the event queue the websocket gateway reads and resolves at
|
|
18
|
+
* once, at most once (D-0057): no ACK, no resend. There is no failover.
|
|
13
19
|
*/
|
|
14
20
|
export class LocalHub {
|
|
15
21
|
options;
|
|
16
|
-
|
|
22
|
+
logger;
|
|
17
23
|
api;
|
|
18
24
|
ownerEpoch = randomUUID();
|
|
19
25
|
#authority = randomUUID();
|
|
20
26
|
#records = new Map();
|
|
21
27
|
#events;
|
|
22
|
-
#deliveries = new Map();
|
|
23
|
-
#deliveryBytes = 0;
|
|
24
28
|
#revision = 0;
|
|
25
29
|
#closed = false;
|
|
26
|
-
|
|
30
|
+
/** Decisions on per-message paths (refusals), once per interval with a count. */
|
|
31
|
+
#throttle;
|
|
27
32
|
constructor(options,
|
|
28
|
-
/**
|
|
29
|
-
|
|
33
|
+
/** The plugin logger; the console default keeps failures visible. */
|
|
34
|
+
logger = new ConsoleLogger()) {
|
|
30
35
|
this.options = options;
|
|
31
|
-
this.
|
|
36
|
+
this.logger = logger;
|
|
37
|
+
this.#throttle = new LogThrottle(logger);
|
|
32
38
|
this.#events = new EventQueue(options.maxPending, options.queueBytes);
|
|
33
39
|
const self = this;
|
|
34
40
|
const nonces = new LocalNonceStore();
|
|
@@ -45,7 +51,6 @@ export class LocalHub {
|
|
|
45
51
|
register: (value) => this.#register(value),
|
|
46
52
|
list: (filter) => this.#list(filter),
|
|
47
53
|
broadcast: (filter, data) => this.#broadcast(filter, data),
|
|
48
|
-
acknowledge: (id, connections) => this.#acknowledge(id, connections),
|
|
49
54
|
kick: (filter, code, reason) => this.#kick(filter, code, reason),
|
|
50
55
|
events: () => this.#events,
|
|
51
56
|
// Always ready until closed: there is no leader to reconnect to.
|
|
@@ -56,7 +61,7 @@ export class LocalHub {
|
|
|
56
61
|
// replayed to another worker passes. Say it once, where the defence is asked for.
|
|
57
62
|
if (cluster.isWorker && !warned) {
|
|
58
63
|
warned = true;
|
|
59
|
-
this.
|
|
64
|
+
this.logger.log('warn', 'Replay nonces are per worker with a local hub', {
|
|
60
65
|
code: 'HUB_NONCES_NOT_SHARED',
|
|
61
66
|
fix: 'Use mode: cluster (Redis) so every worker shares one nonce store',
|
|
62
67
|
});
|
|
@@ -64,15 +69,19 @@ export class LocalHub {
|
|
|
64
69
|
return nonces;
|
|
65
70
|
},
|
|
66
71
|
});
|
|
67
|
-
this.#timer = setInterval(() => this.#tick(), Math.min(options.retryMs, 250));
|
|
68
|
-
this.#timer.unref();
|
|
69
72
|
}
|
|
70
73
|
#push(event) {
|
|
71
74
|
try {
|
|
72
75
|
this.#events.push(event);
|
|
73
76
|
}
|
|
74
77
|
catch (error) {
|
|
75
|
-
// The consumer fell too far behind: end the stream like the cluster client does.
|
|
78
|
+
// The consumer fell too far behind: end the stream like the cluster client does. Its
|
|
79
|
+
// reader (the websocket plugin) sees the error; the hub is closed from here on.
|
|
80
|
+
this.logger.log('error', 'Local hub event queue overflowed; hub closed', {
|
|
81
|
+
code: 'EVENT_OVERFLOW',
|
|
82
|
+
error: errorText(error),
|
|
83
|
+
maxPending: this.options.maxPending,
|
|
84
|
+
});
|
|
76
85
|
this.#events.finish(error);
|
|
77
86
|
this.#closed = true;
|
|
78
87
|
}
|
|
@@ -104,17 +113,24 @@ export class LocalHub {
|
|
|
104
113
|
throw new HubError('CONNECTION_LIMIT');
|
|
105
114
|
this.#records.set(value.connection, value);
|
|
106
115
|
this.#changed();
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
116
|
+
if (this.logger.isEnabled('debug'))
|
|
117
|
+
this.logger.log('debug', 'Hub connection registered', {
|
|
118
|
+
connection: value.connection,
|
|
119
|
+
connections: this.#records.size,
|
|
120
|
+
});
|
|
111
121
|
let closed = false;
|
|
112
122
|
const change = async (next) => {
|
|
113
123
|
const current = this.#records.get(value.connection);
|
|
114
124
|
if (closed || !current)
|
|
115
125
|
throw new HubError('UNKNOWN_CONNECTION');
|
|
116
|
-
|
|
126
|
+
const updated = next(current);
|
|
127
|
+
this.#records.set(value.connection, updated);
|
|
117
128
|
this.#changed();
|
|
129
|
+
if (this.logger.isEnabled('debug'))
|
|
130
|
+
this.logger.log('debug', 'Hub connection changed', {
|
|
131
|
+
connection: value.connection,
|
|
132
|
+
rooms: updated.rooms.length,
|
|
133
|
+
});
|
|
118
134
|
};
|
|
119
135
|
return Object.freeze({
|
|
120
136
|
identity: Object.freeze({ ownerEpoch: this.ownerEpoch, connection: value.connection }),
|
|
@@ -125,8 +141,14 @@ export class LocalHub {
|
|
|
125
141
|
if (closed)
|
|
126
142
|
return;
|
|
127
143
|
closed = true;
|
|
128
|
-
if (this.#records.delete(value.connection) && !this.#closed)
|
|
144
|
+
if (this.#records.delete(value.connection) && !this.#closed) {
|
|
129
145
|
this.#changed();
|
|
146
|
+
if (this.logger.isEnabled('debug'))
|
|
147
|
+
this.logger.log('debug', 'Hub connection removed', {
|
|
148
|
+
connection: value.connection,
|
|
149
|
+
connections: this.#records.size,
|
|
150
|
+
});
|
|
151
|
+
}
|
|
130
152
|
},
|
|
131
153
|
});
|
|
132
154
|
}
|
|
@@ -144,69 +166,39 @@ export class LocalHub {
|
|
|
144
166
|
return Promise.reject(new HubError('CLOSED'));
|
|
145
167
|
if (!(data instanceof Uint8Array) || data.byteLength > this.options.maxPayloadBytes)
|
|
146
168
|
return Promise.reject(new HubError('INVALID_ARGUMENT'));
|
|
147
|
-
const
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
169
|
+
const started = performance.now();
|
|
170
|
+
let filter;
|
|
171
|
+
try {
|
|
172
|
+
filter = query(input);
|
|
173
|
+
}
|
|
174
|
+
catch (error) {
|
|
175
|
+
return Promise.reject(error);
|
|
176
|
+
}
|
|
151
177
|
const id = randomUUID();
|
|
152
|
-
const recipients =
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
if (
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
return new Promise((resolve, reject) => {
|
|
159
|
-
const delivery = {
|
|
178
|
+
const { recipients, owners } = select(this.#records.values(), filter);
|
|
179
|
+
const connections = owners.get(this.ownerEpoch) ?? [];
|
|
180
|
+
// The caller may reuse its buffer once this resolves; the gateway reads the event later.
|
|
181
|
+
if (connections.length &&
|
|
182
|
+
!this.#events.offer({
|
|
183
|
+
kind: 'message',
|
|
160
184
|
id,
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
if (ok)
|
|
170
|
-
resolve(Object.freeze({ authorityEpoch: this.#authority, id, recipients }));
|
|
171
|
-
else
|
|
172
|
-
reject(new HubError(this.#closed ? 'CLOSED' : 'DELIVERY_TIMEOUT'));
|
|
173
|
-
},
|
|
174
|
-
};
|
|
175
|
-
this.#deliveries.set(id, delivery);
|
|
176
|
-
this.#deliveryBytes += copy.byteLength;
|
|
177
|
-
this.#transmit(delivery);
|
|
178
|
-
});
|
|
179
|
-
}
|
|
180
|
-
/** Sends the message to the connections that still owe an ACK (at most once per retryMs). */
|
|
181
|
-
#transmit(delivery) {
|
|
182
|
-
const now = Date.now();
|
|
183
|
-
if (now - delivery.lastSent < this.options.retryMs)
|
|
184
|
-
return;
|
|
185
|
-
const connections = [...this.#records.values()]
|
|
186
|
-
.filter((r) => delivery.remaining.has(recipientKey(r)))
|
|
187
|
-
.map((r) => r.connection);
|
|
188
|
-
if (!connections.length)
|
|
189
|
-
return;
|
|
190
|
-
delivery.lastSent = now;
|
|
191
|
-
this.#push({
|
|
192
|
-
kind: 'message',
|
|
193
|
-
id: delivery.id,
|
|
194
|
-
connections: Object.freeze(connections),
|
|
195
|
-
data: delivery.data,
|
|
196
|
-
expiresAt: delivery.expiresAt,
|
|
197
|
-
});
|
|
198
|
-
}
|
|
199
|
-
#acknowledge(id, connections) {
|
|
200
|
-
const delivery = this.#deliveries.get(id);
|
|
201
|
-
if (!delivery)
|
|
202
|
-
return;
|
|
203
|
-
for (const connection of array(connections, this.options.maxConnections)) {
|
|
204
|
-
const value = this.#records.get(String(connection));
|
|
205
|
-
if (value)
|
|
206
|
-
delivery.remaining.delete(recipientKey(value));
|
|
185
|
+
connections: Object.freeze(connections),
|
|
186
|
+
data: Uint8Array.from(data),
|
|
187
|
+
}, BROADCAST_SHARE)) {
|
|
188
|
+
// The gateway has not taken the earlier messages yet: the caller backs off.
|
|
189
|
+
this.#throttle.report('debug', 'Hub broadcast refused; event queue busy', 'BUSY', undefined, {
|
|
190
|
+
maxPending: this.options.maxPending,
|
|
191
|
+
});
|
|
192
|
+
return Promise.reject(new HubError('BUSY'));
|
|
207
193
|
}
|
|
208
|
-
if (
|
|
209
|
-
|
|
194
|
+
if (this.logger.isEnabled('debug'))
|
|
195
|
+
this.logger.log('debug', 'Hub broadcast sent', {
|
|
196
|
+
id,
|
|
197
|
+
recipients: recipients.length,
|
|
198
|
+
owners: connections.length ? 1 : 0,
|
|
199
|
+
ms: Math.round((performance.now() - started) * 1000) / 1000,
|
|
200
|
+
});
|
|
201
|
+
return Promise.resolve(Object.freeze({ id, recipients: Object.freeze(recipients) }));
|
|
210
202
|
}
|
|
211
203
|
async #kick(input, code = 4403, reason = 'Disconnected by hub') {
|
|
212
204
|
this.#ensureOpen();
|
|
@@ -221,23 +213,12 @@ export class LocalHub {
|
|
|
221
213
|
.map((r) => r.connection);
|
|
222
214
|
if (connections.length)
|
|
223
215
|
this.#push({ kind: 'kick', id, connections: Object.freeze(connections), code, reason });
|
|
216
|
+
this.logger.log('debug', 'Hub kick sent', { id, connections: connections.length, code });
|
|
224
217
|
return Object.freeze({ authorityEpoch: this.#authority, id });
|
|
225
218
|
}
|
|
226
|
-
#tick() {
|
|
227
|
-
const now = Date.now();
|
|
228
|
-
for (const delivery of [...this.#deliveries.values()]) {
|
|
229
|
-
if (now >= delivery.expiresAt)
|
|
230
|
-
delivery.done(false);
|
|
231
|
-
else
|
|
232
|
-
this.#transmit(delivery);
|
|
233
|
-
}
|
|
234
|
-
}
|
|
235
219
|
async close() {
|
|
236
|
-
if (this.#timer)
|
|
237
|
-
clearInterval(this.#timer);
|
|
238
220
|
this.#closed = true;
|
|
239
|
-
for (const delivery of [...this.#deliveries.values()])
|
|
240
|
-
delivery.done(false);
|
|
241
221
|
this.#events.finish();
|
|
222
|
+
this.#throttle.flush();
|
|
242
223
|
}
|
|
243
224
|
}
|
package/dist/log.d.ts
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { LogFields, Logger, LogLevel } from '@dokebi/core';
|
|
2
|
+
/**
|
|
3
|
+
* The full text of a caught value for the `error` log field (D-0047): name, whole message and
|
|
4
|
+
* stack, then each `cause` the same way. Only credentials in URLs (`//user:password@`) are
|
|
5
|
+
* removed; a text over 16 KB is cut and marked.
|
|
6
|
+
*/
|
|
7
|
+
export declare function errorText(error: unknown): string;
|
|
8
|
+
/** A stable code of an error (hub, Redis, NATS or errno code), or `fallback`. */
|
|
9
|
+
export declare function errorCode(error: unknown, fallback: string): string;
|
|
10
|
+
/**
|
|
11
|
+
* Logs a repeating event once per interval (D-0047): the first occurrence at once, the following
|
|
12
|
+
* ones of the same message and code are counted and reported together (`repeated`) when the
|
|
13
|
+
* interval ends. Hot paths (per message, per frame) can report every failure without a burst
|
|
14
|
+
* flooding the log. Field and error text are built only for lines that are written.
|
|
15
|
+
*/
|
|
16
|
+
export declare class LogThrottle {
|
|
17
|
+
#private;
|
|
18
|
+
readonly logger: Logger;
|
|
19
|
+
readonly intervalMs: number;
|
|
20
|
+
constructor(logger: Logger, intervalMs?: number);
|
|
21
|
+
report(level: LogLevel, message: string, code: string, error?: unknown, fields?: LogFields): void;
|
|
22
|
+
/** Writes the counts still pending (on close), so a burst at shutdown is not lost. */
|
|
23
|
+
flush(): void;
|
|
24
|
+
}
|
package/dist/log.js
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
const CODE = /^[A-Z][A-Z0-9_]{0,63}$/;
|
|
2
|
+
/**
|
|
3
|
+
* The full text of a caught value for the `error` log field (D-0047): name, whole message and
|
|
4
|
+
* stack, then each `cause` the same way. Only credentials in URLs (`//user:password@`) are
|
|
5
|
+
* removed; a text over 16 KB is cut and marked.
|
|
6
|
+
*/
|
|
7
|
+
export function errorText(error) {
|
|
8
|
+
const parts = [];
|
|
9
|
+
let current = error;
|
|
10
|
+
for (let depth = 0; current !== undefined && depth < 5; depth++) {
|
|
11
|
+
let text;
|
|
12
|
+
if (current instanceof Error)
|
|
13
|
+
text = current.stack ?? `${current.name}: ${current.message}`;
|
|
14
|
+
else if (typeof current === 'string')
|
|
15
|
+
text = current;
|
|
16
|
+
else {
|
|
17
|
+
try {
|
|
18
|
+
text = `NonError: ${JSON.stringify(current)}`;
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
text = `NonError: ${String(current)}`; // not serialisable (cycle, BigInt): its string form
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
parts.push(depth ? `Caused by: ${text}` : text);
|
|
25
|
+
current = current instanceof Error ? current.cause : undefined;
|
|
26
|
+
}
|
|
27
|
+
const text = parts.join('\n').replace(/\/\/[^\s/@]*@/g, '//***@');
|
|
28
|
+
return text.length > 16_384
|
|
29
|
+
? `${text.slice(0, 16_384)}… [truncated ${text.length - 16_384} characters]`
|
|
30
|
+
: text;
|
|
31
|
+
}
|
|
32
|
+
/** A stable code of an error (hub, Redis, NATS or errno code), or `fallback`. */
|
|
33
|
+
export function errorCode(error, fallback) {
|
|
34
|
+
const code = error?.code;
|
|
35
|
+
return typeof code === 'string' && CODE.test(code) ? code : fallback;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Logs a repeating event once per interval (D-0047): the first occurrence at once, the following
|
|
39
|
+
* ones of the same message and code are counted and reported together (`repeated`) when the
|
|
40
|
+
* interval ends. Hot paths (per message, per frame) can report every failure without a burst
|
|
41
|
+
* flooding the log. Field and error text are built only for lines that are written.
|
|
42
|
+
*/
|
|
43
|
+
export class LogThrottle {
|
|
44
|
+
logger;
|
|
45
|
+
intervalMs;
|
|
46
|
+
#entries = new Map();
|
|
47
|
+
constructor(logger, intervalMs = 10_000) {
|
|
48
|
+
this.logger = logger;
|
|
49
|
+
this.intervalMs = intervalMs;
|
|
50
|
+
}
|
|
51
|
+
report(level, message, code, error, fields) {
|
|
52
|
+
if (!this.logger.isEnabled(level))
|
|
53
|
+
return;
|
|
54
|
+
const key = `${level}\u0000${code}\u0000${message}`;
|
|
55
|
+
const entry = this.#entries.get(key);
|
|
56
|
+
if (entry) {
|
|
57
|
+
entry.count++;
|
|
58
|
+
entry.error = error;
|
|
59
|
+
entry.fields = fields;
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
this.#write(level, message, code, error, fields, 0);
|
|
63
|
+
const value = {
|
|
64
|
+
count: 0,
|
|
65
|
+
error: undefined,
|
|
66
|
+
fields: undefined,
|
|
67
|
+
timer: setTimeout(() => this.#elapsed(key, value, level, message, code), this.intervalMs),
|
|
68
|
+
};
|
|
69
|
+
value.timer.unref();
|
|
70
|
+
this.#entries.set(key, value);
|
|
71
|
+
}
|
|
72
|
+
#elapsed(key, entry, level, message, code) {
|
|
73
|
+
if (!entry.count) {
|
|
74
|
+
this.#entries.delete(key);
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
this.#write(level, message, code, entry.error, entry.fields, entry.count);
|
|
78
|
+
entry.count = 0;
|
|
79
|
+
entry.timer = setTimeout(() => this.#elapsed(key, entry, level, message, code), this.intervalMs);
|
|
80
|
+
entry.timer.unref();
|
|
81
|
+
}
|
|
82
|
+
#write(level, message, code, error, fields, repeated) {
|
|
83
|
+
this.logger.log(level, message, {
|
|
84
|
+
...fields,
|
|
85
|
+
code,
|
|
86
|
+
...(error === undefined ? {} : { error: errorText(error) }),
|
|
87
|
+
...(repeated ? { repeated, intervalMs: this.intervalMs } : {}),
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
/** Writes the counts still pending (on close), so a burst at shutdown is not lost. */
|
|
91
|
+
flush() {
|
|
92
|
+
for (const [key, entry] of this.#entries) {
|
|
93
|
+
clearTimeout(entry.timer);
|
|
94
|
+
if (entry.count) {
|
|
95
|
+
const [level, code, message] = key.split('\u0000');
|
|
96
|
+
this.#write(level, message, code, entry.error, entry.fields, entry.count);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
this.#entries.clear();
|
|
100
|
+
}
|
|
101
|
+
}
|
package/dist/node.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
2
|
import { createServer } from 'node:net';
|
|
3
3
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
4
|
+
import { checkEviction } from './eviction.js';
|
|
4
5
|
import { Leader } from './leader.js';
|
|
6
|
+
import { errorText, LogThrottle } from './log.js';
|
|
5
7
|
import { PeerClient } from './peer-client.js';
|
|
6
8
|
import { hubStatus } from './status.js';
|
|
7
9
|
import { HubStore } from './store.js';
|
|
@@ -28,6 +30,7 @@ export class HubNode {
|
|
|
28
30
|
#configMismatch;
|
|
29
31
|
#started = false;
|
|
30
32
|
#startupCode;
|
|
33
|
+
#throttle;
|
|
31
34
|
constructor(context, options, redis,
|
|
32
35
|
/** Optional (D-0029): without NATS, peers notice a new leader by Redis polling (pollMs). */
|
|
33
36
|
nats) {
|
|
@@ -36,9 +39,10 @@ export class HubNode {
|
|
|
36
39
|
this.redis = redis;
|
|
37
40
|
this.nats = nats;
|
|
38
41
|
this.store = new HubStore(redis, options);
|
|
39
|
-
|
|
40
|
-
//
|
|
41
|
-
|
|
42
|
+
this.#throttle = new LogThrottle(context.logger);
|
|
43
|
+
// The client logs why a ready connection dropped (revision gap, stale leader, socket closed);
|
|
44
|
+
// it is when upgrades see the hub recovering.
|
|
45
|
+
this.client = new PeerClient(options, context.logger);
|
|
42
46
|
const self = this;
|
|
43
47
|
// Shared by every server of the namespace: the first to see a nonce wins (D-0038).
|
|
44
48
|
const nonces = Object.freeze({
|
|
@@ -69,7 +73,6 @@ export class HubNode {
|
|
|
69
73
|
register: (value) => this.client.register(value),
|
|
70
74
|
list: (query) => this.client.list(query),
|
|
71
75
|
broadcast: (query, data) => this.client.broadcast(query, data),
|
|
72
|
-
acknowledge: (id, connections) => this.client.acknowledge(id, connections),
|
|
73
76
|
kick: (query, code, reason) => this.client.kick(query, code, reason),
|
|
74
77
|
events: () => this.client.events,
|
|
75
78
|
whenReady: (timeoutMs) => this.client.whenReady(timeoutMs),
|
|
@@ -78,7 +81,9 @@ export class HubNode {
|
|
|
78
81
|
try {
|
|
79
82
|
lease = await this.store.inspect();
|
|
80
83
|
}
|
|
81
|
-
catch {
|
|
84
|
+
catch (error) {
|
|
85
|
+
// Reported as leaseState 'unreachable'; the cause is only in this log.
|
|
86
|
+
this.#throttle.report('error', 'Hub status could not read the lease from Redis', 'STATUS_UNREACHABLE', error);
|
|
82
87
|
lease = 'unreachable';
|
|
83
88
|
}
|
|
84
89
|
return hubStatus({
|
|
@@ -93,6 +98,9 @@ export class HubNode {
|
|
|
93
98
|
});
|
|
94
99
|
}
|
|
95
100
|
async start() {
|
|
101
|
+
const started = performance.now();
|
|
102
|
+
// D-0049: warn when this Redis can evict the leader key; never refuses to start.
|
|
103
|
+
await checkEviction(this.redis, this.context.logger);
|
|
96
104
|
const server = createServer((socket) => {
|
|
97
105
|
if (!this.#closing && this.#leader?.active)
|
|
98
106
|
this.#leader.accept(socket);
|
|
@@ -107,7 +115,11 @@ export class HubNode {
|
|
|
107
115
|
resolve();
|
|
108
116
|
});
|
|
109
117
|
});
|
|
110
|
-
server.on('error', () => {
|
|
118
|
+
server.on('error', (error) => {
|
|
119
|
+
this.context.logger.log('error', 'Hub TCP server failed; reconnecting', {
|
|
120
|
+
code: 'HUB_SERVER_ERROR',
|
|
121
|
+
error: errorText(error),
|
|
122
|
+
});
|
|
111
123
|
this.client.disconnect();
|
|
112
124
|
});
|
|
113
125
|
const address = server.address();
|
|
@@ -141,6 +153,13 @@ export class HubNode {
|
|
|
141
153
|
await delay(this.options.pollMs);
|
|
142
154
|
}
|
|
143
155
|
this.#started = true;
|
|
156
|
+
this.context.logger.log('debug', 'Hub started', {
|
|
157
|
+
serverId: this.options.serverId,
|
|
158
|
+
role: this.api.role,
|
|
159
|
+
leader: this.client.leader?.serverId ?? null,
|
|
160
|
+
nats: this.nats !== undefined,
|
|
161
|
+
ms: Math.round(performance.now() - started),
|
|
162
|
+
});
|
|
144
163
|
this.#timer = setInterval(() => {
|
|
145
164
|
void this.#poll();
|
|
146
165
|
}, this.options.pollMs);
|
|
@@ -154,9 +173,9 @@ export class HubNode {
|
|
|
154
173
|
await this.#poll();
|
|
155
174
|
}
|
|
156
175
|
}
|
|
157
|
-
catch {
|
|
176
|
+
catch (error) {
|
|
158
177
|
if (!this.#closing)
|
|
159
|
-
this.context.logger.log('
|
|
178
|
+
this.context.logger.log('error', 'Hub control notifications unavailable; Redis discovery polling remains active', { code: 'NATS_SUBSCRIPTION_FAILED', error: errorText(error) });
|
|
160
179
|
}
|
|
161
180
|
}
|
|
162
181
|
#poll() {
|
|
@@ -175,19 +194,26 @@ export class HubNode {
|
|
|
175
194
|
}
|
|
176
195
|
else if (!this.#started) {
|
|
177
196
|
// Startup retries are normal (peers race to reach a leader that is still opening);
|
|
178
|
-
//
|
|
197
|
+
// the last code is reported if startup times out, each attempt at debug.
|
|
179
198
|
this.#startupCode = hubCode(error);
|
|
199
|
+
this.context.logger.log('debug', 'Hub startup attempt failed; retrying', {
|
|
200
|
+
code: this.#startupCode,
|
|
201
|
+
error: errorText(error),
|
|
202
|
+
});
|
|
180
203
|
}
|
|
181
204
|
else if (!this.#lastFailure) {
|
|
182
205
|
this.#failingSince = performance.now();
|
|
183
|
-
this.context.logger.log('
|
|
206
|
+
this.context.logger.log('error', 'Hub transport or store unavailable; recovering', {
|
|
184
207
|
code: hubCode(error),
|
|
208
|
+
error: errorText(error),
|
|
185
209
|
});
|
|
186
210
|
}
|
|
187
|
-
// Every further failure of the same outage, so a long recovery shows
|
|
211
|
+
// Every further failure of the same outage (once per pollMs), so a long recovery shows
|
|
212
|
+
// its causes.
|
|
188
213
|
else
|
|
189
214
|
this.context.logger.log('debug', 'Hub still recovering', {
|
|
190
215
|
code: hubCode(error),
|
|
216
|
+
error: errorText(error),
|
|
191
217
|
ms: Math.round(performance.now() - this.#failingSince),
|
|
192
218
|
leader: this.client.leader?.serverId ?? null,
|
|
193
219
|
});
|
|
@@ -212,9 +238,14 @@ export class HubNode {
|
|
|
212
238
|
if (!discovered) {
|
|
213
239
|
this.#identity = { ...this.#identity, epoch: randomUUID() };
|
|
214
240
|
discovered = await this.store.elect(this.#identity);
|
|
241
|
+
this.context.logger.log('debug', 'Hub leader election', {
|
|
242
|
+
won: discovered.epoch === this.#identity.epoch,
|
|
243
|
+
leader: discovered.serverId,
|
|
244
|
+
epoch: discovered.epoch,
|
|
245
|
+
});
|
|
215
246
|
}
|
|
216
247
|
if (discovered.epoch === this.#identity.epoch && !this.#leader) {
|
|
217
|
-
const leader = new Leader(discovered, this.store, this.options);
|
|
248
|
+
const leader = new Leader(discovered, this.store, this.options, this.context.logger);
|
|
218
249
|
try {
|
|
219
250
|
await leader.start();
|
|
220
251
|
}
|
|
@@ -227,15 +258,25 @@ export class HubNode {
|
|
|
227
258
|
try {
|
|
228
259
|
await this.nats.publish(`dokebi.websocket-hub.${this.options.namespace}.control`, Buffer.from(JSON.stringify({ kind: 'leader-changed', epoch: discovered.epoch })));
|
|
229
260
|
}
|
|
230
|
-
catch {
|
|
231
|
-
this.context.logger.log('
|
|
261
|
+
catch (error) {
|
|
262
|
+
this.context.logger.log('error', 'Hub change notification failed; peers use Redis discovery', { code: 'NATS_PUBLISH_FAILED', error: errorText(error) });
|
|
232
263
|
}
|
|
233
264
|
}
|
|
234
265
|
if (this.#leader && this.#identity.epoch !== discovered.epoch) {
|
|
266
|
+
this.context.logger.log('debug', 'Hub leadership lost', {
|
|
267
|
+
epoch: this.#identity.epoch,
|
|
268
|
+
leader: discovered.serverId,
|
|
269
|
+
});
|
|
235
270
|
const previous = this.#leader;
|
|
236
271
|
this.#leader = undefined;
|
|
237
272
|
await previous.stop();
|
|
238
273
|
}
|
|
274
|
+
if (this.client.leader?.epoch !== discovered.epoch)
|
|
275
|
+
this.context.logger.log('debug', 'Hub connecting to leader', {
|
|
276
|
+
leader: discovered.serverId,
|
|
277
|
+
epoch: discovered.epoch,
|
|
278
|
+
previous: this.client.leader?.serverId ?? null,
|
|
279
|
+
});
|
|
239
280
|
await this.client.connect(discovered);
|
|
240
281
|
if (this.#lastFailure && this.#started)
|
|
241
282
|
this.context.logger.log('info', 'Hub recovered', {
|
|
@@ -247,27 +288,41 @@ export class HubNode {
|
|
|
247
288
|
async close() {
|
|
248
289
|
if (this.#closed)
|
|
249
290
|
return;
|
|
291
|
+
const started = performance.now();
|
|
250
292
|
this.#closing = true;
|
|
251
293
|
if (this.#timer)
|
|
252
294
|
clearInterval(this.#timer);
|
|
253
295
|
this.#subscription?.unsubscribe();
|
|
254
296
|
await this.#tick;
|
|
255
297
|
const errors = [];
|
|
298
|
+
// If the leader went away first (whole cluster stopping), `bye` cannot reach it; `leave`
|
|
299
|
+
// below removes our record through Redis instead, so that is not a failure of its own.
|
|
300
|
+
let goodbye;
|
|
256
301
|
try {
|
|
257
302
|
await this.client.close();
|
|
258
303
|
}
|
|
259
304
|
catch (error) {
|
|
260
|
-
|
|
305
|
+
goodbye = error;
|
|
261
306
|
}
|
|
262
|
-
// If the leader went away first (whole cluster stopping), `bye` could not remove our record.
|
|
263
307
|
const session = this.client.session;
|
|
308
|
+
let left = false;
|
|
264
309
|
if (session)
|
|
265
310
|
try {
|
|
266
311
|
await this.store.run('leave', '', '', session, this.client.ownerEpoch);
|
|
312
|
+
left = true;
|
|
267
313
|
}
|
|
268
314
|
catch (error) {
|
|
269
315
|
errors.push(error);
|
|
270
316
|
}
|
|
317
|
+
if (goodbye !== undefined) {
|
|
318
|
+
if (left && GONE.has(hubCode(goodbye)))
|
|
319
|
+
this.context.logger.log('debug', 'Hub goodbye not delivered; left through Redis', {
|
|
320
|
+
code: hubCode(goodbye),
|
|
321
|
+
error: errorText(goodbye),
|
|
322
|
+
});
|
|
323
|
+
else
|
|
324
|
+
errors.unshift(goodbye);
|
|
325
|
+
}
|
|
271
326
|
const leader = this.#leader;
|
|
272
327
|
this.#leader = undefined;
|
|
273
328
|
await leader?.stop();
|
|
@@ -283,10 +338,17 @@ export class HubNode {
|
|
|
283
338
|
await new Promise((resolve) => this.#server?.close(() => resolve()));
|
|
284
339
|
await this.#notifications;
|
|
285
340
|
this.#closed = true;
|
|
341
|
+
this.#throttle.flush();
|
|
342
|
+
this.context.logger.log('debug', 'Hub closed', {
|
|
343
|
+
ms: Math.round(performance.now() - started),
|
|
344
|
+
failures: errors.length,
|
|
345
|
+
});
|
|
286
346
|
if (errors.length)
|
|
287
347
|
throw new AggregateError(errors, 'Hub cleanup failed');
|
|
288
348
|
}
|
|
289
349
|
}
|
|
350
|
+
/** The leader is gone or going: `bye` cannot be answered, `leave` takes its place. */
|
|
351
|
+
const GONE = new Set(['DISCONNECTED', 'STALE_LEADER', 'CLOSED', 'RECOVERING']);
|
|
290
352
|
/** A short code for logs: hub/redis/nats error codes or system errno codes, never messages. */
|
|
291
353
|
function hubCode(error) {
|
|
292
354
|
const code = error?.code;
|
package/dist/options.d.ts
CHANGED
|
@@ -1,8 +1,13 @@
|
|
|
1
1
|
export declare function object(value: unknown): Record<string, unknown>;
|
|
2
2
|
export declare function keys(value: Record<string, unknown>, allowed: readonly string[]): void;
|
|
3
3
|
export declare function integer(v: unknown, min: number, max: number): number;
|
|
4
|
+
/**
|
|
5
|
+
* Options of 0.2 that D-0057 removed with client ACKs. A dokebi.yaml that still has them starts:
|
|
6
|
+
* the hub logs warn `OPTION_REMOVED` once per key and ignores the value.
|
|
7
|
+
*/
|
|
8
|
+
export declare const REMOVED_OPTIONS: readonly ["deliveryTimeoutMs", "retryMs", "maxDeliveries", "deliveryBytes"];
|
|
4
9
|
export interface HubOptions {
|
|
5
|
-
/** `local` (default): one process, in memory. `cluster`: Redis leader
|
|
10
|
+
/** `local` (default): one process, in memory. `cluster`: Redis leader and TCP (D-0029). */
|
|
6
11
|
mode: 'local' | 'cluster';
|
|
7
12
|
/** Lets `local` run in a node:cluster worker anyway (each worker then has its own hub). */
|
|
8
13
|
allowInCluster: boolean;
|
|
@@ -27,9 +32,7 @@ export interface HubOptions {
|
|
|
27
32
|
maxTotalConnections: number;
|
|
28
33
|
maxPending: number;
|
|
29
34
|
queueBytes: number;
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
maxDeliveries: number;
|
|
33
|
-
deliveryBytes: number;
|
|
35
|
+
/** Removed options found in the input (REMOVED_OPTIONS), reported at setup and ignored. */
|
|
36
|
+
removedOptions: readonly string[];
|
|
34
37
|
}
|
|
35
38
|
export declare function parseOptions(input: unknown): HubOptions;
|