@dokebi/websocket-hub 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AI.md +96 -49
- package/README.md +71 -43
- package/dist/delivery.d.ts +13 -47
- package/dist/delivery.js +19 -177
- package/dist/dependency.d.ts +4 -0
- package/dist/dependency.js +14 -0
- package/dist/event-queue.d.ts +6 -0
- package/dist/event-queue.js +21 -3
- package/dist/eviction.d.ts +9 -0
- package/dist/eviction.js +57 -0
- package/dist/hub-index.d.ts +29 -0
- package/dist/hub-index.js +146 -0
- package/dist/index.d.ts +5 -3
- package/dist/index.js +25 -18
- package/dist/leader.d.ts +3 -1
- package/dist/leader.js +215 -76
- package/dist/local.d.ts +11 -7
- package/dist/local.js +94 -107
- package/dist/node.d.ts +2 -2
- package/dist/node.js +83 -21
- package/dist/options.d.ts +32 -34
- package/dist/options.js +115 -120
- package/dist/peer-client.d.ts +13 -4
- package/dist/peer-client.js +290 -172
- package/dist/protocol.d.ts +6 -0
- package/dist/protocol.js +7 -1
- package/dist/status-cli.js +7 -2
- package/dist/store.js +12 -7
- package/dist/types.d.ts +25 -8
- package/dist/types.js +22 -5
- package/dist/values.d.ts +4 -0
- package/dist/values.js +32 -0
- package/dist/wire.d.ts +8 -1
- package/dist/wire.js +14 -3
- package/options.schema.json +113 -36
- package/package.json +35 -11
package/dist/local.js
CHANGED
|
@@ -1,35 +1,46 @@
|
|
|
1
|
-
import cluster from 'node:cluster';
|
|
2
1
|
import { randomUUID } from 'node:crypto';
|
|
3
|
-
import {
|
|
2
|
+
import { ConsoleLogger } from '@dokebi/core';
|
|
3
|
+
import { diagnostics } from '@dokebi/core/libraries';
|
|
4
4
|
import { EventQueue } from './event-queue.js';
|
|
5
|
-
import {
|
|
5
|
+
import { HubIndex } from './hub-index.js';
|
|
6
|
+
import { identifier, metadata, query, record } from './protocol.js';
|
|
6
7
|
import { hubStatus, LocalNonceStore } from './status.js';
|
|
7
8
|
import { HubError, } from './types.js';
|
|
9
|
+
/**
|
|
10
|
+
* Share of the event queue (maxPending events, maxPendingBytes) that broadcasts may fill before they
|
|
11
|
+
* are refused with BUSY; the rest stays for presence and kick events, which cannot be refused.
|
|
12
|
+
*/
|
|
13
|
+
const BROADCAST_SHARE = 0.5;
|
|
8
14
|
/**
|
|
9
15
|
* The hub of one process (D-0029, `mode: 'local'`): the same HubApi as the cluster hub, kept in
|
|
10
|
-
* memory.
|
|
11
|
-
*
|
|
12
|
-
* missed. Undelivered messages are lost with the process, and there is no failover.
|
|
16
|
+
* memory. A broadcast is handed to the event queue the websocket gateway reads and resolves at
|
|
17
|
+
* once, at most once (D-0057): no ACK, no resend. There is no failover.
|
|
13
18
|
*/
|
|
14
19
|
export class LocalHub {
|
|
15
20
|
options;
|
|
16
|
-
|
|
21
|
+
logger;
|
|
22
|
+
worker;
|
|
17
23
|
api;
|
|
18
24
|
ownerEpoch = randomUUID();
|
|
19
25
|
#authority = randomUUID();
|
|
20
26
|
#records = new Map();
|
|
27
|
+
/** The same records by tenant, member and room: selection without a scan (D-0068). */
|
|
28
|
+
#index = new HubIndex();
|
|
21
29
|
#events;
|
|
22
|
-
#deliveries = new Map();
|
|
23
|
-
#deliveryBytes = 0;
|
|
24
30
|
#revision = 0;
|
|
25
31
|
#closed = false;
|
|
26
|
-
|
|
32
|
+
/** Decisions on per-message paths (refusals), once per interval with a count. */
|
|
33
|
+
#throttle;
|
|
27
34
|
constructor(options,
|
|
28
|
-
/**
|
|
29
|
-
|
|
35
|
+
/** The plugin logger; the console default keeps failures visible. */
|
|
36
|
+
logger = new ConsoleLogger(),
|
|
37
|
+
/** This process is a node:cluster worker (`context.runtime.role`, D-0063 §4). */
|
|
38
|
+
worker = false) {
|
|
30
39
|
this.options = options;
|
|
31
|
-
this.
|
|
32
|
-
this
|
|
40
|
+
this.logger = logger;
|
|
41
|
+
this.worker = worker;
|
|
42
|
+
this.#throttle = new diagnostics.Repeats({ logger });
|
|
43
|
+
this.#events = new EventQueue(options.maxPending, options.maxPendingBytes);
|
|
33
44
|
const self = this;
|
|
34
45
|
const nonces = new LocalNonceStore();
|
|
35
46
|
let warned = false;
|
|
@@ -45,7 +56,6 @@ export class LocalHub {
|
|
|
45
56
|
register: (value) => this.#register(value),
|
|
46
57
|
list: (filter) => this.#list(filter),
|
|
47
58
|
broadcast: (filter, data) => this.#broadcast(filter, data),
|
|
48
|
-
acknowledge: (id, connections) => this.#acknowledge(id, connections),
|
|
49
59
|
kick: (filter, code, reason) => this.#kick(filter, code, reason),
|
|
50
60
|
events: () => this.#events,
|
|
51
61
|
// Always ready until closed: there is no leader to reconnect to.
|
|
@@ -54,9 +64,9 @@ export class LocalHub {
|
|
|
54
64
|
nonces: () => {
|
|
55
65
|
// A local hub in a node:cluster worker (allowInCluster) keeps nonces per worker: a frame
|
|
56
66
|
// replayed to another worker passes. Say it once, where the defence is asked for.
|
|
57
|
-
if (
|
|
67
|
+
if (this.worker && !warned) {
|
|
58
68
|
warned = true;
|
|
59
|
-
this.
|
|
69
|
+
this.logger.log('warn', 'Replay nonces are per worker with a local hub', {
|
|
60
70
|
code: 'HUB_NONCES_NOT_SHARED',
|
|
61
71
|
fix: 'Use mode: cluster (Redis) so every worker shares one nonce store',
|
|
62
72
|
});
|
|
@@ -64,15 +74,19 @@ export class LocalHub {
|
|
|
64
74
|
return nonces;
|
|
65
75
|
},
|
|
66
76
|
});
|
|
67
|
-
this.#timer = setInterval(() => this.#tick(), Math.min(options.retryMs, 250));
|
|
68
|
-
this.#timer.unref();
|
|
69
77
|
}
|
|
70
78
|
#push(event) {
|
|
71
79
|
try {
|
|
72
80
|
this.#events.push(event);
|
|
73
81
|
}
|
|
74
82
|
catch (error) {
|
|
75
|
-
// The consumer fell too far behind: end the stream like the cluster client does.
|
|
83
|
+
// The consumer fell too far behind: end the stream like the cluster client does. Its
|
|
84
|
+
// reader (the websocket plugin) sees the error; the hub is closed from here on.
|
|
85
|
+
this.logger.log('error', 'Local hub event queue overflowed; hub closed', {
|
|
86
|
+
code: 'EVENT_OVERFLOW',
|
|
87
|
+
error: diagnostics.describeError(error),
|
|
88
|
+
maxPending: this.options.maxPending,
|
|
89
|
+
});
|
|
76
90
|
this.#events.finish(error);
|
|
77
91
|
this.#closed = true;
|
|
78
92
|
}
|
|
@@ -95,26 +109,33 @@ export class LocalHub {
|
|
|
95
109
|
const value = record(input, this.ownerEpoch);
|
|
96
110
|
if (this.#records.has(value.connection))
|
|
97
111
|
throw new HubError('DUPLICATE_CONNECTION');
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
if (recipientKey(existing) === key)
|
|
101
|
-
throw new HubError('DUPLICATE_CLIENT');
|
|
112
|
+
if (this.#index.client(value))
|
|
113
|
+
throw new HubError('DUPLICATE_CLIENT');
|
|
102
114
|
if (this.#records.size >= this.options.maxConnections ||
|
|
103
115
|
this.#records.size >= this.options.maxTotalConnections)
|
|
104
116
|
throw new HubError('CONNECTION_LIMIT');
|
|
105
117
|
this.#records.set(value.connection, value);
|
|
118
|
+
this.#index.add(value);
|
|
106
119
|
this.#changed();
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
120
|
+
if (this.logger.isEnabled('debug'))
|
|
121
|
+
this.logger.log('debug', 'Hub connection registered', {
|
|
122
|
+
connection: value.connection,
|
|
123
|
+
connections: this.#records.size,
|
|
124
|
+
});
|
|
111
125
|
let closed = false;
|
|
112
126
|
const change = async (next) => {
|
|
113
127
|
const current = this.#records.get(value.connection);
|
|
114
128
|
if (closed || !current)
|
|
115
129
|
throw new HubError('UNKNOWN_CONNECTION');
|
|
116
|
-
|
|
130
|
+
const updated = next(current);
|
|
131
|
+
this.#records.set(value.connection, updated);
|
|
132
|
+
this.#index.add(updated);
|
|
117
133
|
this.#changed();
|
|
134
|
+
if (this.logger.isEnabled('debug'))
|
|
135
|
+
this.logger.log('debug', 'Hub connection changed', {
|
|
136
|
+
connection: value.connection,
|
|
137
|
+
rooms: updated.rooms.length,
|
|
138
|
+
});
|
|
118
139
|
};
|
|
119
140
|
return Object.freeze({
|
|
120
141
|
identity: Object.freeze({ ownerEpoch: this.ownerEpoch, connection: value.connection }),
|
|
@@ -125,8 +146,17 @@ export class LocalHub {
|
|
|
125
146
|
if (closed)
|
|
126
147
|
return;
|
|
127
148
|
closed = true;
|
|
128
|
-
|
|
149
|
+
const current = this.#records.get(value.connection);
|
|
150
|
+
if (current)
|
|
151
|
+
this.#index.remove(current);
|
|
152
|
+
if (this.#records.delete(value.connection) && !this.#closed) {
|
|
129
153
|
this.#changed();
|
|
154
|
+
if (this.logger.isEnabled('debug'))
|
|
155
|
+
this.logger.log('debug', 'Hub connection removed', {
|
|
156
|
+
connection: value.connection,
|
|
157
|
+
connections: this.#records.size,
|
|
158
|
+
});
|
|
159
|
+
}
|
|
130
160
|
},
|
|
131
161
|
});
|
|
132
162
|
}
|
|
@@ -136,7 +166,7 @@ export class LocalHub {
|
|
|
136
166
|
return Object.freeze({
|
|
137
167
|
authorityEpoch: this.#authority,
|
|
138
168
|
revision: this.#revision,
|
|
139
|
-
connections: Object.freeze([...this.#
|
|
169
|
+
connections: Object.freeze([...this.#index.matching(filter)]),
|
|
140
170
|
});
|
|
141
171
|
}
|
|
142
172
|
#broadcast(input, data) {
|
|
@@ -144,69 +174,39 @@ export class LocalHub {
|
|
|
144
174
|
return Promise.reject(new HubError('CLOSED'));
|
|
145
175
|
if (!(data instanceof Uint8Array) || data.byteLength > this.options.maxPayloadBytes)
|
|
146
176
|
return Promise.reject(new HubError('INVALID_ARGUMENT'));
|
|
147
|
-
const
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
177
|
+
const started = performance.now();
|
|
178
|
+
let filter;
|
|
179
|
+
try {
|
|
180
|
+
filter = query(input);
|
|
181
|
+
}
|
|
182
|
+
catch (error) {
|
|
183
|
+
return Promise.reject(error);
|
|
184
|
+
}
|
|
151
185
|
const id = randomUUID();
|
|
152
|
-
const recipients =
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
if (
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
return new Promise((resolve, reject) => {
|
|
159
|
-
const delivery = {
|
|
186
|
+
const { recipients, owners } = this.#index.select(filter);
|
|
187
|
+
const connections = owners.get(this.ownerEpoch) ?? [];
|
|
188
|
+
// The caller may reuse its buffer once this resolves; the gateway reads the event later.
|
|
189
|
+
if (connections.length &&
|
|
190
|
+
!this.#events.offer({
|
|
191
|
+
kind: 'message',
|
|
160
192
|
id,
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
if (ok)
|
|
170
|
-
resolve(Object.freeze({ authorityEpoch: this.#authority, id, recipients }));
|
|
171
|
-
else
|
|
172
|
-
reject(new HubError(this.#closed ? 'CLOSED' : 'DELIVERY_TIMEOUT'));
|
|
173
|
-
},
|
|
174
|
-
};
|
|
175
|
-
this.#deliveries.set(id, delivery);
|
|
176
|
-
this.#deliveryBytes += copy.byteLength;
|
|
177
|
-
this.#transmit(delivery);
|
|
178
|
-
});
|
|
179
|
-
}
|
|
180
|
-
/** Sends the message to the connections that still owe an ACK (at most once per retryMs). */
|
|
181
|
-
#transmit(delivery) {
|
|
182
|
-
const now = Date.now();
|
|
183
|
-
if (now - delivery.lastSent < this.options.retryMs)
|
|
184
|
-
return;
|
|
185
|
-
const connections = [...this.#records.values()]
|
|
186
|
-
.filter((r) => delivery.remaining.has(recipientKey(r)))
|
|
187
|
-
.map((r) => r.connection);
|
|
188
|
-
if (!connections.length)
|
|
189
|
-
return;
|
|
190
|
-
delivery.lastSent = now;
|
|
191
|
-
this.#push({
|
|
192
|
-
kind: 'message',
|
|
193
|
-
id: delivery.id,
|
|
194
|
-
connections: Object.freeze(connections),
|
|
195
|
-
data: delivery.data,
|
|
196
|
-
expiresAt: delivery.expiresAt,
|
|
197
|
-
});
|
|
198
|
-
}
|
|
199
|
-
#acknowledge(id, connections) {
|
|
200
|
-
const delivery = this.#deliveries.get(id);
|
|
201
|
-
if (!delivery)
|
|
202
|
-
return;
|
|
203
|
-
for (const connection of array(connections, this.options.maxConnections)) {
|
|
204
|
-
const value = this.#records.get(String(connection));
|
|
205
|
-
if (value)
|
|
206
|
-
delivery.remaining.delete(recipientKey(value));
|
|
193
|
+
connections: Object.freeze(connections),
|
|
194
|
+
data: Uint8Array.from(data),
|
|
195
|
+
}, BROADCAST_SHARE)) {
|
|
196
|
+
// The gateway has not taken the earlier messages yet: the caller backs off.
|
|
197
|
+
this.#throttle.log('debug', 'Hub broadcast refused; event queue busy', 'BUSY', undefined, {
|
|
198
|
+
maxPending: this.options.maxPending,
|
|
199
|
+
});
|
|
200
|
+
return Promise.reject(new HubError('BUSY'));
|
|
207
201
|
}
|
|
208
|
-
if (
|
|
209
|
-
|
|
202
|
+
if (this.logger.isEnabled('debug'))
|
|
203
|
+
this.logger.log('debug', 'Hub broadcast sent', {
|
|
204
|
+
id,
|
|
205
|
+
recipients: recipients.length,
|
|
206
|
+
owners: connections.length ? 1 : 0,
|
|
207
|
+
ms: Math.round((performance.now() - started) * 1000) / 1000,
|
|
208
|
+
});
|
|
209
|
+
return Promise.resolve(Object.freeze({ id, recipients: Object.freeze(recipients) }));
|
|
210
210
|
}
|
|
211
211
|
async #kick(input, code = 4403, reason = 'Disconnected by hub') {
|
|
212
212
|
this.#ensureOpen();
|
|
@@ -216,28 +216,15 @@ export class LocalHub {
|
|
|
216
216
|
if (Buffer.byteLength(identifier(reason)) > 123)
|
|
217
217
|
throw new HubError('INVALID_ARGUMENT');
|
|
218
218
|
const id = randomUUID();
|
|
219
|
-
const connections =
|
|
220
|
-
.filter((r) => matches(r, filter))
|
|
221
|
-
.map((r) => r.connection);
|
|
219
|
+
const connections = Array.from(this.#index.matching(filter), (r) => r.connection);
|
|
222
220
|
if (connections.length)
|
|
223
221
|
this.#push({ kind: 'kick', id, connections: Object.freeze(connections), code, reason });
|
|
222
|
+
this.logger.log('debug', 'Hub kick sent', { id, connections: connections.length, code });
|
|
224
223
|
return Object.freeze({ authorityEpoch: this.#authority, id });
|
|
225
224
|
}
|
|
226
|
-
#tick() {
|
|
227
|
-
const now = Date.now();
|
|
228
|
-
for (const delivery of [...this.#deliveries.values()]) {
|
|
229
|
-
if (now >= delivery.expiresAt)
|
|
230
|
-
delivery.done(false);
|
|
231
|
-
else
|
|
232
|
-
this.#transmit(delivery);
|
|
233
|
-
}
|
|
234
|
-
}
|
|
235
225
|
async close() {
|
|
236
|
-
if (this.#timer)
|
|
237
|
-
clearInterval(this.#timer);
|
|
238
226
|
this.#closed = true;
|
|
239
|
-
for (const delivery of [...this.#deliveries.values()])
|
|
240
|
-
delivery.done(false);
|
|
241
227
|
this.#events.finish();
|
|
228
|
+
this.#throttle.flush();
|
|
242
229
|
}
|
|
243
230
|
}
|
package/dist/node.d.ts
CHANGED
|
@@ -10,13 +10,13 @@ export declare class HubNode {
|
|
|
10
10
|
readonly context: PluginContext;
|
|
11
11
|
readonly options: HubOptions;
|
|
12
12
|
readonly redis: RedisConnection;
|
|
13
|
-
/** Optional (D-0029): without NATS, peers notice a new leader by Redis polling (
|
|
13
|
+
/** Optional (D-0029): without NATS, peers notice a new leader by Redis polling (pollIntervalMs). */
|
|
14
14
|
readonly nats: NatsConnectionApi | undefined;
|
|
15
15
|
readonly api: HubApi;
|
|
16
16
|
readonly store: HubStore;
|
|
17
17
|
readonly client: PeerClient;
|
|
18
18
|
constructor(context: PluginContext, options: HubOptions, redis: RedisConnection,
|
|
19
|
-
/** Optional (D-0029): without NATS, peers notice a new leader by Redis polling (
|
|
19
|
+
/** Optional (D-0029): without NATS, peers notice a new leader by Redis polling (pollIntervalMs). */
|
|
20
20
|
nats: NatsConnectionApi | undefined);
|
|
21
21
|
start(): Promise<void>;
|
|
22
22
|
close(): Promise<void>;
|
package/dist/node.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
2
|
import { createServer } from 'node:net';
|
|
3
3
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
4
|
+
import { diagnostics } from '@dokebi/core/libraries';
|
|
5
|
+
import { checkEviction } from './eviction.js';
|
|
4
6
|
import { Leader } from './leader.js';
|
|
5
7
|
import { PeerClient } from './peer-client.js';
|
|
6
8
|
import { hubStatus } from './status.js';
|
|
@@ -28,17 +30,19 @@ export class HubNode {
|
|
|
28
30
|
#configMismatch;
|
|
29
31
|
#started = false;
|
|
30
32
|
#startupCode;
|
|
33
|
+
#throttle;
|
|
31
34
|
constructor(context, options, redis,
|
|
32
|
-
/** Optional (D-0029): without NATS, peers notice a new leader by Redis polling (
|
|
35
|
+
/** Optional (D-0029): without NATS, peers notice a new leader by Redis polling (pollIntervalMs). */
|
|
33
36
|
nats) {
|
|
34
37
|
this.context = context;
|
|
35
38
|
this.options = options;
|
|
36
39
|
this.redis = redis;
|
|
37
40
|
this.nats = nats;
|
|
38
41
|
this.store = new HubStore(redis, options);
|
|
39
|
-
|
|
40
|
-
//
|
|
41
|
-
|
|
42
|
+
this.#throttle = new diagnostics.Repeats({ logger: context.logger });
|
|
43
|
+
// The client logs why a ready connection dropped (revision gap, stale leader, socket closed);
|
|
44
|
+
// it is when upgrades see the hub recovering.
|
|
45
|
+
this.client = new PeerClient(options, context.logger);
|
|
42
46
|
const self = this;
|
|
43
47
|
// Shared by every server of the namespace: the first to see a nonce wins (D-0038).
|
|
44
48
|
const nonces = Object.freeze({
|
|
@@ -69,7 +73,6 @@ export class HubNode {
|
|
|
69
73
|
register: (value) => this.client.register(value),
|
|
70
74
|
list: (query) => this.client.list(query),
|
|
71
75
|
broadcast: (query, data) => this.client.broadcast(query, data),
|
|
72
|
-
acknowledge: (id, connections) => this.client.acknowledge(id, connections),
|
|
73
76
|
kick: (query, code, reason) => this.client.kick(query, code, reason),
|
|
74
77
|
events: () => this.client.events,
|
|
75
78
|
whenReady: (timeoutMs) => this.client.whenReady(timeoutMs),
|
|
@@ -78,7 +81,9 @@ export class HubNode {
|
|
|
78
81
|
try {
|
|
79
82
|
lease = await this.store.inspect();
|
|
80
83
|
}
|
|
81
|
-
catch {
|
|
84
|
+
catch (error) {
|
|
85
|
+
// Reported as leaseState 'unreachable'; the cause is only in this log.
|
|
86
|
+
this.#throttle.log('error', 'Hub status could not read the lease from Redis', 'STATUS_UNREACHABLE', error);
|
|
82
87
|
lease = 'unreachable';
|
|
83
88
|
}
|
|
84
89
|
return hubStatus({
|
|
@@ -93,6 +98,9 @@ export class HubNode {
|
|
|
93
98
|
});
|
|
94
99
|
}
|
|
95
100
|
async start() {
|
|
101
|
+
const started = performance.now();
|
|
102
|
+
// D-0049: warn when this Redis can evict the leader key; never refuses to start.
|
|
103
|
+
await checkEviction(this.redis, this.context.logger);
|
|
96
104
|
const server = createServer((socket) => {
|
|
97
105
|
if (!this.#closing && this.#leader?.active)
|
|
98
106
|
this.#leader.accept(socket);
|
|
@@ -107,7 +115,11 @@ export class HubNode {
|
|
|
107
115
|
resolve();
|
|
108
116
|
});
|
|
109
117
|
});
|
|
110
|
-
server.on('error', () => {
|
|
118
|
+
server.on('error', (error) => {
|
|
119
|
+
this.context.logger.log('error', 'Hub TCP server failed; reconnecting', {
|
|
120
|
+
code: 'HUB_SERVER_ERROR',
|
|
121
|
+
error: diagnostics.describeError(error),
|
|
122
|
+
});
|
|
111
123
|
this.client.disconnect();
|
|
112
124
|
});
|
|
113
125
|
const address = server.address();
|
|
@@ -124,26 +136,33 @@ export class HubNode {
|
|
|
124
136
|
await this.#subscription.ready;
|
|
125
137
|
this.#notifications = this.#readNotifications(this.#subscription);
|
|
126
138
|
}
|
|
127
|
-
const deadline = performance.now() + this.options.
|
|
139
|
+
const deadline = performance.now() + this.options.startupTimeoutMs;
|
|
128
140
|
while (!this.client.ready) {
|
|
129
141
|
this.context.signal.throwIfAborted();
|
|
130
142
|
await this.#poll();
|
|
131
143
|
if (this.#configMismatch)
|
|
132
144
|
throw this.#configMismatch;
|
|
133
145
|
if (performance.now() > deadline) {
|
|
134
|
-
this.context.logger.log('error', 'Hub did not become ready before
|
|
146
|
+
this.context.logger.log('error', 'Hub did not become ready before startupTimeoutMs', {
|
|
135
147
|
code: 'STARTUP_TIMEOUT',
|
|
136
148
|
lastError: this.#startupCode ?? 'NONE',
|
|
137
149
|
});
|
|
138
150
|
throw new HubError('STARTUP_TIMEOUT');
|
|
139
151
|
}
|
|
140
152
|
if (!this.client.ready)
|
|
141
|
-
await delay(this.options.
|
|
153
|
+
await delay(this.options.pollIntervalMs);
|
|
142
154
|
}
|
|
143
155
|
this.#started = true;
|
|
156
|
+
this.context.logger.log('debug', 'Hub started', {
|
|
157
|
+
serverId: this.options.serverId,
|
|
158
|
+
role: this.api.role,
|
|
159
|
+
leader: this.client.leader?.serverId ?? null,
|
|
160
|
+
nats: this.nats !== undefined,
|
|
161
|
+
ms: Math.round(performance.now() - started),
|
|
162
|
+
});
|
|
144
163
|
this.#timer = setInterval(() => {
|
|
145
164
|
void this.#poll();
|
|
146
|
-
}, this.options.
|
|
165
|
+
}, this.options.pollIntervalMs);
|
|
147
166
|
this.#timer.unref();
|
|
148
167
|
}
|
|
149
168
|
async #readNotifications(subscription) {
|
|
@@ -154,9 +173,9 @@ export class HubNode {
|
|
|
154
173
|
await this.#poll();
|
|
155
174
|
}
|
|
156
175
|
}
|
|
157
|
-
catch {
|
|
176
|
+
catch (error) {
|
|
158
177
|
if (!this.#closing)
|
|
159
|
-
this.context.logger.log('
|
|
178
|
+
this.context.logger.log('error', 'Hub control notifications unavailable; Redis discovery polling remains active', { code: 'NATS_SUBSCRIPTION_FAILED', error: diagnostics.describeError(error) });
|
|
160
179
|
}
|
|
161
180
|
}
|
|
162
181
|
#poll() {
|
|
@@ -175,19 +194,26 @@ export class HubNode {
|
|
|
175
194
|
}
|
|
176
195
|
else if (!this.#started) {
|
|
177
196
|
// Startup retries are normal (peers race to reach a leader that is still opening);
|
|
178
|
-
//
|
|
197
|
+
// the last code is reported if startup times out, each attempt at debug.
|
|
179
198
|
this.#startupCode = hubCode(error);
|
|
199
|
+
this.context.logger.log('debug', 'Hub startup attempt failed; retrying', {
|
|
200
|
+
code: this.#startupCode,
|
|
201
|
+
error: diagnostics.describeError(error),
|
|
202
|
+
});
|
|
180
203
|
}
|
|
181
204
|
else if (!this.#lastFailure) {
|
|
182
205
|
this.#failingSince = performance.now();
|
|
183
|
-
this.context.logger.log('
|
|
206
|
+
this.context.logger.log('error', 'Hub transport or store unavailable; recovering', {
|
|
184
207
|
code: hubCode(error),
|
|
208
|
+
error: diagnostics.describeError(error),
|
|
185
209
|
});
|
|
186
210
|
}
|
|
187
|
-
// Every further failure of the same outage, so a long recovery shows
|
|
211
|
+
// Every further failure of the same outage (once per pollIntervalMs), so a long recovery shows
|
|
212
|
+
// its causes.
|
|
188
213
|
else
|
|
189
214
|
this.context.logger.log('debug', 'Hub still recovering', {
|
|
190
215
|
code: hubCode(error),
|
|
216
|
+
error: diagnostics.describeError(error),
|
|
191
217
|
ms: Math.round(performance.now() - this.#failingSince),
|
|
192
218
|
leader: this.client.leader?.serverId ?? null,
|
|
193
219
|
});
|
|
@@ -212,9 +238,14 @@ export class HubNode {
|
|
|
212
238
|
if (!discovered) {
|
|
213
239
|
this.#identity = { ...this.#identity, epoch: randomUUID() };
|
|
214
240
|
discovered = await this.store.elect(this.#identity);
|
|
241
|
+
this.context.logger.log('debug', 'Hub leader election', {
|
|
242
|
+
won: discovered.epoch === this.#identity.epoch,
|
|
243
|
+
leader: discovered.serverId,
|
|
244
|
+
epoch: discovered.epoch,
|
|
245
|
+
});
|
|
215
246
|
}
|
|
216
247
|
if (discovered.epoch === this.#identity.epoch && !this.#leader) {
|
|
217
|
-
const leader = new Leader(discovered, this.store, this.options);
|
|
248
|
+
const leader = new Leader(discovered, this.store, this.options, this.context.logger);
|
|
218
249
|
try {
|
|
219
250
|
await leader.start();
|
|
220
251
|
}
|
|
@@ -227,15 +258,25 @@ export class HubNode {
|
|
|
227
258
|
try {
|
|
228
259
|
await this.nats.publish(`dokebi.websocket-hub.${this.options.namespace}.control`, Buffer.from(JSON.stringify({ kind: 'leader-changed', epoch: discovered.epoch })));
|
|
229
260
|
}
|
|
230
|
-
catch {
|
|
231
|
-
this.context.logger.log('
|
|
261
|
+
catch (error) {
|
|
262
|
+
this.context.logger.log('error', 'Hub change notification failed; peers use Redis discovery', { code: 'NATS_PUBLISH_FAILED', error: diagnostics.describeError(error) });
|
|
232
263
|
}
|
|
233
264
|
}
|
|
234
265
|
if (this.#leader && this.#identity.epoch !== discovered.epoch) {
|
|
266
|
+
this.context.logger.log('debug', 'Hub leadership lost', {
|
|
267
|
+
epoch: this.#identity.epoch,
|
|
268
|
+
leader: discovered.serverId,
|
|
269
|
+
});
|
|
235
270
|
const previous = this.#leader;
|
|
236
271
|
this.#leader = undefined;
|
|
237
272
|
await previous.stop();
|
|
238
273
|
}
|
|
274
|
+
if (this.client.leader?.epoch !== discovered.epoch)
|
|
275
|
+
this.context.logger.log('debug', 'Hub connecting to leader', {
|
|
276
|
+
leader: discovered.serverId,
|
|
277
|
+
epoch: discovered.epoch,
|
|
278
|
+
previous: this.client.leader?.serverId ?? null,
|
|
279
|
+
});
|
|
239
280
|
await this.client.connect(discovered);
|
|
240
281
|
if (this.#lastFailure && this.#started)
|
|
241
282
|
this.context.logger.log('info', 'Hub recovered', {
|
|
@@ -247,27 +288,41 @@ export class HubNode {
|
|
|
247
288
|
async close() {
|
|
248
289
|
if (this.#closed)
|
|
249
290
|
return;
|
|
291
|
+
const started = performance.now();
|
|
250
292
|
this.#closing = true;
|
|
251
293
|
if (this.#timer)
|
|
252
294
|
clearInterval(this.#timer);
|
|
253
295
|
this.#subscription?.unsubscribe();
|
|
254
296
|
await this.#tick;
|
|
255
297
|
const errors = [];
|
|
298
|
+
// If the leader went away first (whole cluster stopping), `bye` cannot reach it; `leave`
|
|
299
|
+
// below removes our record through Redis instead, so that is not a failure of its own.
|
|
300
|
+
let goodbye;
|
|
256
301
|
try {
|
|
257
302
|
await this.client.close();
|
|
258
303
|
}
|
|
259
304
|
catch (error) {
|
|
260
|
-
|
|
305
|
+
goodbye = error;
|
|
261
306
|
}
|
|
262
|
-
// If the leader went away first (whole cluster stopping), `bye` could not remove our record.
|
|
263
307
|
const session = this.client.session;
|
|
308
|
+
let left = false;
|
|
264
309
|
if (session)
|
|
265
310
|
try {
|
|
266
311
|
await this.store.run('leave', '', '', session, this.client.ownerEpoch);
|
|
312
|
+
left = true;
|
|
267
313
|
}
|
|
268
314
|
catch (error) {
|
|
269
315
|
errors.push(error);
|
|
270
316
|
}
|
|
317
|
+
if (goodbye !== undefined) {
|
|
318
|
+
if (left && GONE.has(hubCode(goodbye)))
|
|
319
|
+
this.context.logger.log('debug', 'Hub goodbye not delivered; left through Redis', {
|
|
320
|
+
code: hubCode(goodbye),
|
|
321
|
+
error: diagnostics.describeError(goodbye),
|
|
322
|
+
});
|
|
323
|
+
else
|
|
324
|
+
errors.unshift(goodbye);
|
|
325
|
+
}
|
|
271
326
|
const leader = this.#leader;
|
|
272
327
|
this.#leader = undefined;
|
|
273
328
|
await leader?.stop();
|
|
@@ -283,10 +338,17 @@ export class HubNode {
|
|
|
283
338
|
await new Promise((resolve) => this.#server?.close(() => resolve()));
|
|
284
339
|
await this.#notifications;
|
|
285
340
|
this.#closed = true;
|
|
341
|
+
this.#throttle.flush();
|
|
342
|
+
this.context.logger.log('debug', 'Hub closed', {
|
|
343
|
+
ms: Math.round(performance.now() - started),
|
|
344
|
+
failures: errors.length,
|
|
345
|
+
});
|
|
286
346
|
if (errors.length)
|
|
287
347
|
throw new AggregateError(errors, 'Hub cleanup failed');
|
|
288
348
|
}
|
|
289
349
|
}
|
|
350
|
+
/** The leader is gone or going: `bye` cannot be answered, `leave` takes its place. */
|
|
351
|
+
const GONE = new Set(['DISCONNECTED', 'STALE_LEADER', 'CLOSED', 'RECOVERING']);
|
|
290
352
|
/** A short code for logs: hub/redis/nats error codes or system errno codes, never messages. */
|
|
291
353
|
function hubCode(error) {
|
|
292
354
|
const code = error?.code;
|
package/dist/options.d.ts
CHANGED
|
@@ -1,35 +1,33 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
deliveryBytes: number;
|
|
34
|
-
}
|
|
1
|
+
import { type OptionsOf } from '@dokebi/core';
|
|
2
|
+
/**
|
|
3
|
+
* Options of 0.2 that D-0057 removed with client ACKs. A dokebi.yaml that still has them starts:
|
|
4
|
+
* core logs warn `OPTION_REMOVED` once per key at setup and the value is ignored.
|
|
5
|
+
*/
|
|
6
|
+
export declare const REMOVED_OPTIONS: readonly ["deliveryTimeoutMs", "retryMs", "maxDeliveries", "deliveryBytes"];
|
|
7
|
+
export declare const options: import("@dokebi/core").OptionsDeclaration<{
|
|
8
|
+
readonly advertiseHost: string;
|
|
9
|
+
readonly allowInCluster: boolean;
|
|
10
|
+
readonly host: string;
|
|
11
|
+
readonly leaseMs: number;
|
|
12
|
+
readonly maxConnections: number;
|
|
13
|
+
readonly maxFrameBytes: number;
|
|
14
|
+
readonly maxOwners: number;
|
|
15
|
+
readonly maxPayloadBytes: number;
|
|
16
|
+
readonly maxPending: number;
|
|
17
|
+
readonly maxPendingBytes: number;
|
|
18
|
+
readonly maxTotalConnections: number;
|
|
19
|
+
readonly mode: "cluster" | "local";
|
|
20
|
+
readonly namespace: string;
|
|
21
|
+
readonly natsConnection?: string;
|
|
22
|
+
readonly ownerTtlMs: number;
|
|
23
|
+
readonly pollIntervalMs: number;
|
|
24
|
+
readonly port: number;
|
|
25
|
+
readonly redisConnection?: string;
|
|
26
|
+
readonly secret?: string;
|
|
27
|
+
readonly serverId: string;
|
|
28
|
+
readonly startupTimeoutMs: number;
|
|
29
|
+
}>;
|
|
30
|
+
/** Parsed hub options (frozen). `secret` is required in cluster mode. */
|
|
31
|
+
export type HubOptions = OptionsOf<typeof options>;
|
|
32
|
+
/** Validates hub options (status CLI and tests; definePlugin uses `options` itself). */
|
|
35
33
|
export declare function parseOptions(input: unknown): HubOptions;
|