diodejs 0.4.0 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +185 -68
- package/clientManager.js +1679 -78
- package/connection.js +24 -3
- package/examples/RPCTest.js +11 -1
- package/examples/fleetContractExample.js +45 -0
- package/examples/publishPortTest.js +3 -2
- package/networkDiscoveryClient.js +89 -0
- package/package.json +4 -3
- package/publishPort.js +50 -27
- package/scripts/benchmark-relay-selection.js +476 -0
- package/test/clientManager.relaySelection.test.js +1454 -0
- package/test/fixtures/dio-network-snapshot.json +82 -0
- package/test/fleetContract.test.js +71 -0
- package/test/publishPort.test.js +399 -0
- package/utils.js +26 -0
package/clientManager.js
CHANGED
|
@@ -1,7 +1,12 @@
|
|
|
1
|
+
const fs = require('fs');
|
|
2
|
+
const net = require('net');
|
|
3
|
+
const path = require('path');
|
|
1
4
|
const EventEmitter = require('events');
|
|
2
5
|
const DiodeConnection = require('./connection');
|
|
3
6
|
const DiodeRPC = require('./rpc');
|
|
7
|
+
const { fetchNetworkDirectory } = require('./networkDiscoveryClient');
|
|
4
8
|
const logger = require('./logger');
|
|
9
|
+
const { DEFAULT_FLEET_CONTRACT, normalizeFleetContractAddress } = require('./utils');
|
|
5
10
|
|
|
6
11
|
const DEFAULT_DIODE_ADDRS = [
|
|
7
12
|
'as1.prenet.diode.io:41046',
|
|
@@ -12,6 +17,15 @@ const DEFAULT_DIODE_ADDRS = [
|
|
|
12
17
|
'eu2.prenet.diode.io:41046',
|
|
13
18
|
];
|
|
14
19
|
|
|
20
|
+
const RELAY_SCORE_CACHE_VERSION = 2;
|
|
21
|
+
const RELAY_SCORE_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;
|
|
22
|
+
const RELAY_SCORE_FRESH_MS = 60 * 1000;
|
|
23
|
+
const RELAY_SCORE_FAILURE_COOLDOWN_MS = 30 * 1000;
|
|
24
|
+
const RELAY_SCORE_EWMA_WEIGHT = 0.3;
|
|
25
|
+
const RELAY_SCORE_FLUSH_DEBOUNCE_MS = 500;
|
|
26
|
+
const DEFAULT_NETWORK_DISCOVERY_ENDPOINT = 'wss://prenet.diode.io:8443/ws';
|
|
27
|
+
const DEFAULT_NETWORK_DISCOVERY_METHOD = 'dio_network';
|
|
28
|
+
|
|
15
29
|
function splitHostPort(input, defaultPort) {
|
|
16
30
|
if (!input || typeof input !== 'string') {
|
|
17
31
|
return { host: '', port: defaultPort };
|
|
@@ -55,6 +69,11 @@ function joinHostPort(host, port) {
|
|
|
55
69
|
return `${host}:${port}`;
|
|
56
70
|
}
|
|
57
71
|
|
|
72
|
+
function normalizeHostKey(hostEntry, defaultPort) {
|
|
73
|
+
const { host, port } = splitHostPort(hostEntry, defaultPort);
|
|
74
|
+
return host ? joinHostPort(host, port) : '';
|
|
75
|
+
}
|
|
76
|
+
|
|
58
77
|
function normalizeAddress(address) {
|
|
59
78
|
if (!address) return null;
|
|
60
79
|
if (Buffer.isBuffer(address)) return address;
|
|
@@ -82,6 +101,46 @@ function isConnected(connection) {
|
|
|
82
101
|
return connection && connection.socket && !connection.socket.destroyed;
|
|
83
102
|
}
|
|
84
103
|
|
|
104
|
+
function parseBoolean(value, defaultValue) {
|
|
105
|
+
return typeof value === 'boolean' ? value : defaultValue;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function parsePositiveInteger(value, defaultValue, { allowZero = false } = {}) {
|
|
109
|
+
if (!Number.isFinite(value)) {
|
|
110
|
+
return defaultValue;
|
|
111
|
+
}
|
|
112
|
+
const normalized = Math.floor(value);
|
|
113
|
+
if (allowZero ? normalized >= 0 : normalized > 0) {
|
|
114
|
+
return normalized;
|
|
115
|
+
}
|
|
116
|
+
return defaultValue;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
async function runWithConcurrency(items, concurrency, worker) {
|
|
120
|
+
const normalizedConcurrency = Math.max(1, Math.min(concurrency || 1, items.length || 1));
|
|
121
|
+
const results = new Array(items.length);
|
|
122
|
+
let index = 0;
|
|
123
|
+
|
|
124
|
+
async function consume() {
|
|
125
|
+
while (true) {
|
|
126
|
+
const current = index;
|
|
127
|
+
index += 1;
|
|
128
|
+
if (current >= items.length) {
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
try {
|
|
132
|
+
const value = await worker(items[current], current);
|
|
133
|
+
results[current] = { status: 'fulfilled', value };
|
|
134
|
+
} catch (error) {
|
|
135
|
+
results[current] = { status: 'rejected', reason: error };
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
await Promise.all(Array.from({ length: normalizedConcurrency }, () => consume()));
|
|
141
|
+
return results;
|
|
142
|
+
}
|
|
143
|
+
|
|
85
144
|
class DiodeClientManager extends EventEmitter {
|
|
86
145
|
constructor(options = {}) {
|
|
87
146
|
super();
|
|
@@ -92,15 +151,130 @@ class DiodeClientManager extends EventEmitter {
|
|
|
92
151
|
? options.deviceCacheTtlMs
|
|
93
152
|
: 30000;
|
|
94
153
|
|
|
154
|
+
this._hasExplicitHost = typeof options.host === 'string' && !!options.host.trim();
|
|
155
|
+
this._hasExplicitHosts = !this._hasExplicitHost && (
|
|
156
|
+
Array.isArray(options.hosts)
|
|
157
|
+
|| (typeof options.hosts === 'string' && !!options.hosts.trim())
|
|
158
|
+
);
|
|
159
|
+
|
|
160
|
+
this.relaySelection = this._buildRelaySelectionOptions(options.relaySelection);
|
|
161
|
+
this.scoreCachePath = this._resolveRelayScoreCachePath(this.relaySelection.scoreCachePath);
|
|
162
|
+
this.discoveryState = { networkCursor: 0 };
|
|
163
|
+
|
|
95
164
|
this.connections = [];
|
|
96
165
|
this.connectionByHost = new Map();
|
|
97
166
|
this.serverIdToConnection = new Map();
|
|
98
167
|
this.pendingConnections = new Map();
|
|
168
|
+
this.pendingProbes = new Map();
|
|
99
169
|
this.deviceRelayCache = new Map();
|
|
170
|
+
this.relayScores = new Map();
|
|
100
171
|
this._rpcByConnection = new Map();
|
|
172
|
+
this._candidateMetadataByHost = new Map();
|
|
101
173
|
this._rrIndex = 0;
|
|
174
|
+
this._relayScoreFlushTimer = null;
|
|
175
|
+
this._backgroundWarmupPromise = null;
|
|
176
|
+
this._lastProbeStartedAt = new Map();
|
|
177
|
+
this._startupCoverageComplete = false;
|
|
178
|
+
this._lastNetworkDiscoveryStats = null;
|
|
179
|
+
this._lastDeviceResolutionTrace = null;
|
|
180
|
+
this.fleetContract = DEFAULT_FLEET_CONTRACT;
|
|
181
|
+
|
|
182
|
+
if (options.fleetContract !== undefined) {
|
|
183
|
+
this.setFleetContract(options.fleetContract);
|
|
184
|
+
}
|
|
102
185
|
|
|
103
186
|
this.initialHosts = this._buildInitialHosts(options);
|
|
187
|
+
this._loadRelayScores();
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
setFleetContract(address) {
|
|
191
|
+
const normalizedFleetContract = normalizeFleetContractAddress(address);
|
|
192
|
+
this.fleetContract = normalizedFleetContract;
|
|
193
|
+
|
|
194
|
+
for (const connection of this.connections) {
|
|
195
|
+
if (connection && typeof connection.setFleetContract === 'function') {
|
|
196
|
+
connection.setFleetContract(normalizedFleetContract);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
return this;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
_buildRelaySelectionOptions(options = {}) {
|
|
204
|
+
const relaySelection = options && typeof options === 'object' ? options : {};
|
|
205
|
+
const legacyWarmConnections = parsePositiveInteger(relaySelection.desiredWarmConnections, NaN);
|
|
206
|
+
return {
|
|
207
|
+
enabled: parseBoolean(relaySelection.enabled, true),
|
|
208
|
+
startupConcurrency: parsePositiveInteger(relaySelection.startupConcurrency, 2),
|
|
209
|
+
minReadyConnections: parsePositiveInteger(relaySelection.minReadyConnections, 2),
|
|
210
|
+
probeTimeoutMs: parsePositiveInteger(relaySelection.probeTimeoutMs, 1200),
|
|
211
|
+
warmConnectionBudget: parsePositiveInteger(
|
|
212
|
+
relaySelection.warmConnectionBudget,
|
|
213
|
+
Number.isFinite(legacyWarmConnections) ? legacyWarmConnections : 3,
|
|
214
|
+
),
|
|
215
|
+
probeAllInitialCandidates: parseBoolean(relaySelection.probeAllInitialCandidates, true),
|
|
216
|
+
continueProbingUntestedSeeds: parseBoolean(relaySelection.continueProbingUntestedSeeds, true),
|
|
217
|
+
regionDiverseSeedOrdering: parseBoolean(relaySelection.regionDiverseSeedOrdering, true),
|
|
218
|
+
discoveryProvider: typeof relaySelection.discoveryProvider === 'function'
|
|
219
|
+
? relaySelection.discoveryProvider
|
|
220
|
+
: null,
|
|
221
|
+
discoveryProviderTimeoutMs: parsePositiveInteger(relaySelection.discoveryProviderTimeoutMs, 1500),
|
|
222
|
+
useProviderWithExplicitHost: parseBoolean(relaySelection.useProviderWithExplicitHost, false),
|
|
223
|
+
useProviderWithExplicitHosts: parseBoolean(relaySelection.useProviderWithExplicitHosts, false),
|
|
224
|
+
backgroundProbeIntervalMs: parsePositiveInteger(relaySelection.backgroundProbeIntervalMs, 300000),
|
|
225
|
+
slowRelayThresholdMs: parsePositiveInteger(relaySelection.slowRelayThresholdMs, 250),
|
|
226
|
+
slowDeviceRetryTtlMs: parsePositiveInteger(relaySelection.slowDeviceRetryTtlMs, 5000),
|
|
227
|
+
deviceRelayReconciliation: this._buildDeviceRelayReconciliationOptions(
|
|
228
|
+
relaySelection.deviceRelayReconciliation,
|
|
229
|
+
parsePositiveInteger(relaySelection.probeTimeoutMs, 1200),
|
|
230
|
+
),
|
|
231
|
+
networkDiscovery: this._buildNetworkDiscoveryOptions(relaySelection.networkDiscovery),
|
|
232
|
+
scoreCachePath: Object.prototype.hasOwnProperty.call(relaySelection, 'scoreCachePath')
|
|
233
|
+
? relaySelection.scoreCachePath
|
|
234
|
+
: undefined,
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
_buildDeviceRelayReconciliationOptions(options = {}, probeTimeoutMs = 1200) {
|
|
239
|
+
const reconciliation = options && typeof options === 'object' ? options : {};
|
|
240
|
+
return {
|
|
241
|
+
enabled: parseBoolean(reconciliation.enabled, true),
|
|
242
|
+
maxControlRelays: parsePositiveInteger(reconciliation.maxControlRelays, 2, { allowZero: true }),
|
|
243
|
+
timeoutMs: parsePositiveInteger(reconciliation.timeoutMs, probeTimeoutMs),
|
|
244
|
+
minLatencyDeltaMs: parsePositiveInteger(reconciliation.minLatencyDeltaMs, 150, { allowZero: true }),
|
|
245
|
+
slowdownFactor: Number.isFinite(reconciliation.slowdownFactor) && reconciliation.slowdownFactor > 0
|
|
246
|
+
? reconciliation.slowdownFactor
|
|
247
|
+
: 4,
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
_buildNetworkDiscoveryOptions(options = {}) {
|
|
252
|
+
const networkDiscovery = options && typeof options === 'object' ? options : {};
|
|
253
|
+
const enabledDefault = !this._hasExplicitHost && !this._hasExplicitHosts;
|
|
254
|
+
return {
|
|
255
|
+
enabled: parseBoolean(networkDiscovery.enabled, enabledDefault),
|
|
256
|
+
endpoint: typeof networkDiscovery.endpoint === 'string' && networkDiscovery.endpoint.trim()
|
|
257
|
+
? networkDiscovery.endpoint.trim()
|
|
258
|
+
: DEFAULT_NETWORK_DISCOVERY_ENDPOINT,
|
|
259
|
+
method: typeof networkDiscovery.method === 'string' && networkDiscovery.method.trim()
|
|
260
|
+
? networkDiscovery.method.trim()
|
|
261
|
+
: DEFAULT_NETWORK_DISCOVERY_METHOD,
|
|
262
|
+
timeoutMs: parsePositiveInteger(networkDiscovery.timeoutMs, 1500),
|
|
263
|
+
startupProbeCount: parsePositiveInteger(networkDiscovery.startupProbeCount, 2, { allowZero: true }),
|
|
264
|
+
backgroundBatchSize: parsePositiveInteger(networkDiscovery.backgroundBatchSize, 12, { allowZero: true }),
|
|
265
|
+
includePrivateAddresses: parseBoolean(networkDiscovery.includePrivateAddresses, false),
|
|
266
|
+
};
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
_resolveRelayScoreCachePath(scoreCachePath) {
|
|
270
|
+
if (scoreCachePath === null) {
|
|
271
|
+
return null;
|
|
272
|
+
}
|
|
273
|
+
if (typeof scoreCachePath === 'string' && scoreCachePath.trim()) {
|
|
274
|
+
return scoreCachePath;
|
|
275
|
+
}
|
|
276
|
+
const keyDir = path.dirname(this.keyLocation);
|
|
277
|
+
return path.join(keyDir, 'relay-scores.json');
|
|
104
278
|
}
|
|
105
279
|
|
|
106
280
|
_buildInitialHosts(options) {
|
|
@@ -123,10 +297,8 @@ class DiodeClientManager extends EventEmitter {
|
|
|
123
297
|
const seen = new Set();
|
|
124
298
|
const normalized = [];
|
|
125
299
|
for (const entry of hosts) {
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
if (!host) continue;
|
|
129
|
-
const key = joinHostPort(host, port);
|
|
300
|
+
const key = normalizeHostKey(entry, this.defaultPort);
|
|
301
|
+
if (!key) continue;
|
|
130
302
|
const lower = key.toLowerCase();
|
|
131
303
|
if (!seen.has(lower)) {
|
|
132
304
|
seen.add(lower);
|
|
@@ -136,6 +308,734 @@ class DiodeClientManager extends EventEmitter {
|
|
|
136
308
|
return normalized;
|
|
137
309
|
}
|
|
138
310
|
|
|
311
|
+
_loadRelayScores() {
|
|
312
|
+
this.relayScores.clear();
|
|
313
|
+
this.discoveryState = { networkCursor: 0 };
|
|
314
|
+
if (!this.scoreCachePath) {
|
|
315
|
+
return;
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
try {
|
|
319
|
+
if (!fs.existsSync(this.scoreCachePath)) {
|
|
320
|
+
return;
|
|
321
|
+
}
|
|
322
|
+
const raw = fs.readFileSync(this.scoreCachePath, 'utf8');
|
|
323
|
+
const parsed = JSON.parse(raw);
|
|
324
|
+
if (!parsed || ![1, RELAY_SCORE_CACHE_VERSION].includes(parsed.version) || typeof parsed.relays !== 'object') {
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
this._loadDiscoveryState(parsed.discoveryState);
|
|
329
|
+
|
|
330
|
+
const now = Date.now();
|
|
331
|
+
for (const [hostEntry, record] of Object.entries(parsed.relays)) {
|
|
332
|
+
const hostKey = normalizeHostKey(hostEntry, this.defaultPort);
|
|
333
|
+
if (!hostKey || !record || typeof record !== 'object') {
|
|
334
|
+
continue;
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
const lastTouched = Math.max(
|
|
338
|
+
Number(record.lastSuccessAt) || 0,
|
|
339
|
+
Number(record.lastFailureAt) || 0,
|
|
340
|
+
);
|
|
341
|
+
if (lastTouched && now - lastTouched > RELAY_SCORE_MAX_AGE_MS) {
|
|
342
|
+
continue;
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
this.relayScores.set(hostKey, {
|
|
346
|
+
hostKey,
|
|
347
|
+
ewmaLatencyMs: Number.isFinite(record.ewmaLatencyMs) ? record.ewmaLatencyMs : null,
|
|
348
|
+
lastProbeLatencyMs: Number.isFinite(record.lastProbeLatencyMs) ? record.lastProbeLatencyMs : null,
|
|
349
|
+
successCount: parsePositiveInteger(record.successCount, 0, { allowZero: true }),
|
|
350
|
+
failureCount: parsePositiveInteger(record.failureCount, 0, { allowZero: true }),
|
|
351
|
+
lastSuccessAt: parsePositiveInteger(record.lastSuccessAt, 0, { allowZero: true }),
|
|
352
|
+
lastFailureAt: parsePositiveInteger(record.lastFailureAt, 0, { allowZero: true }),
|
|
353
|
+
cooldownUntil: parsePositiveInteger(record.cooldownUntil, 0, { allowZero: true }),
|
|
354
|
+
discoveredFrom: typeof record.discoveredFrom === 'string' ? record.discoveredFrom : 'seed',
|
|
355
|
+
});
|
|
356
|
+
}
|
|
357
|
+
} catch (error) {
|
|
358
|
+
logger.warn(() => `Failed to load relay scores from ${this.scoreCachePath}: ${error}`);
|
|
359
|
+
this.relayScores.clear();
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
_loadDiscoveryState(discoveryState) {
|
|
364
|
+
const networkCursor = discoveryState && Number.isFinite(discoveryState.networkCursor)
|
|
365
|
+
? Math.max(0, Math.floor(discoveryState.networkCursor))
|
|
366
|
+
: 0;
|
|
367
|
+
this.discoveryState = { networkCursor };
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
_scheduleRelayScoreFlush() {
|
|
371
|
+
if (!this.scoreCachePath) {
|
|
372
|
+
return;
|
|
373
|
+
}
|
|
374
|
+
if (this._relayScoreFlushTimer) {
|
|
375
|
+
clearTimeout(this._relayScoreFlushTimer);
|
|
376
|
+
}
|
|
377
|
+
this._relayScoreFlushTimer = setTimeout(() => {
|
|
378
|
+
this._relayScoreFlushTimer = null;
|
|
379
|
+
this._flushRelayScores();
|
|
380
|
+
}, RELAY_SCORE_FLUSH_DEBOUNCE_MS);
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
_flushRelayScores() {
|
|
384
|
+
if (!this.scoreCachePath) {
|
|
385
|
+
return;
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
const now = Date.now();
|
|
389
|
+
const relays = {};
|
|
390
|
+
for (const [hostKey, score] of this.relayScores.entries()) {
|
|
391
|
+
const lastTouched = Math.max(score.lastSuccessAt || 0, score.lastFailureAt || 0);
|
|
392
|
+
if (lastTouched && now - lastTouched > RELAY_SCORE_MAX_AGE_MS) {
|
|
393
|
+
this.relayScores.delete(hostKey);
|
|
394
|
+
continue;
|
|
395
|
+
}
|
|
396
|
+
relays[hostKey] = {
|
|
397
|
+
ewmaLatencyMs: Number.isFinite(score.ewmaLatencyMs) ? score.ewmaLatencyMs : null,
|
|
398
|
+
lastProbeLatencyMs: Number.isFinite(score.lastProbeLatencyMs) ? score.lastProbeLatencyMs : null,
|
|
399
|
+
successCount: score.successCount || 0,
|
|
400
|
+
failureCount: score.failureCount || 0,
|
|
401
|
+
lastSuccessAt: score.lastSuccessAt || 0,
|
|
402
|
+
lastFailureAt: score.lastFailureAt || 0,
|
|
403
|
+
cooldownUntil: score.cooldownUntil || 0,
|
|
404
|
+
discoveredFrom: score.discoveredFrom || 'seed',
|
|
405
|
+
};
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
try {
|
|
409
|
+
fs.mkdirSync(path.dirname(this.scoreCachePath), { recursive: true });
|
|
410
|
+
fs.writeFileSync(this.scoreCachePath, JSON.stringify({
|
|
411
|
+
version: RELAY_SCORE_CACHE_VERSION,
|
|
412
|
+
updatedAt: now,
|
|
413
|
+
discoveryState: this.discoveryState,
|
|
414
|
+
relays,
|
|
415
|
+
}, null, 2), 'utf8');
|
|
416
|
+
} catch (error) {
|
|
417
|
+
logger.warn(() => `Failed to write relay scores to ${this.scoreCachePath}: ${error}`);
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
_getRegionKey(hostKey) {
|
|
422
|
+
const host = splitHostPort(hostKey, this.defaultPort).host.toLowerCase();
|
|
423
|
+
if (host.startsWith('as')) return 'as';
|
|
424
|
+
if (host.startsWith('us')) return 'us';
|
|
425
|
+
if (host.startsWith('eu')) return 'eu';
|
|
426
|
+
return 'other';
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
_getCandidateRegion(hostKey) {
|
|
430
|
+
const metadata = this._candidateMetadataByHost.get(hostKey);
|
|
431
|
+
if (metadata && metadata.region) {
|
|
432
|
+
return metadata.region;
|
|
433
|
+
}
|
|
434
|
+
return this._getRegionKey(hostKey);
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
_getCandidatePriority(hostKey) {
|
|
438
|
+
const metadata = this._candidateMetadataByHost.get(hostKey);
|
|
439
|
+
if (metadata && Number.isFinite(metadata.priority)) {
|
|
440
|
+
return metadata.priority;
|
|
441
|
+
}
|
|
442
|
+
return 100;
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
_setCandidateMetadata(hostKey, options = {}) {
|
|
446
|
+
const existing = this._candidateMetadataByHost.get(hostKey) || {};
|
|
447
|
+
const next = { ...existing };
|
|
448
|
+
if (Number.isFinite(options.priority)) {
|
|
449
|
+
next.priority = options.priority;
|
|
450
|
+
}
|
|
451
|
+
if (typeof options.region === 'string' && options.region.trim()) {
|
|
452
|
+
next.region = options.region.trim().toLowerCase();
|
|
453
|
+
}
|
|
454
|
+
if (options.metadata && typeof options.metadata === 'object' && !Array.isArray(options.metadata)) {
|
|
455
|
+
next.metadata = options.metadata;
|
|
456
|
+
}
|
|
457
|
+
['nodeIdHex', 'lastSeenAt', 'retries', 'lastError', 'version', 'name', 'selectedPort', 'edgePort', 'serverPort', 'connected'].forEach((key) => {
|
|
458
|
+
if (Object.prototype.hasOwnProperty.call(options, key)) {
|
|
459
|
+
next[key] = options[key];
|
|
460
|
+
}
|
|
461
|
+
});
|
|
462
|
+
if (Object.keys(next).length > 0) {
|
|
463
|
+
this._candidateMetadataByHost.set(hostKey, next);
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
_createCandidate(hostKey, source, index, options = {}) {
|
|
468
|
+
this._setCandidateMetadata(hostKey, options);
|
|
469
|
+
const score = this.relayScores.get(hostKey);
|
|
470
|
+
return {
|
|
471
|
+
hostKey,
|
|
472
|
+
source,
|
|
473
|
+
index,
|
|
474
|
+
hasBeenTested: !!(score && ((score.successCount || 0) > 0 || (score.failureCount || 0) > 0)),
|
|
475
|
+
ewmaLatencyMs: score && Number.isFinite(score.ewmaLatencyMs) ? score.ewmaLatencyMs : null,
|
|
476
|
+
inCooldown: !!(score && score.cooldownUntil && score.cooldownUntil > Date.now()),
|
|
477
|
+
cooldownUntil: score && score.cooldownUntil ? score.cooldownUntil : 0,
|
|
478
|
+
lastSuccessAt: score && score.lastSuccessAt ? score.lastSuccessAt : 0,
|
|
479
|
+
discoveredFrom: score && score.discoveredFrom ? score.discoveredFrom : source,
|
|
480
|
+
priority: Number.isFinite(options.priority) ? options.priority : this._getCandidatePriority(hostKey),
|
|
481
|
+
region: options.region || this._getCandidateRegion(hostKey),
|
|
482
|
+
metadata: options.metadata || (this._candidateMetadataByHost.get(hostKey)?.metadata || null),
|
|
483
|
+
scoreFresh: this._isRelayScoreFresh(score),
|
|
484
|
+
nodeIdHex: options.nodeIdHex || this._candidateMetadataByHost.get(hostKey)?.nodeIdHex || '',
|
|
485
|
+
lastSeenAt: Number.isFinite(options.lastSeenAt) ? options.lastSeenAt : (this._candidateMetadataByHost.get(hostKey)?.lastSeenAt || 0),
|
|
486
|
+
retries: Number.isFinite(options.retries) ? options.retries : (this._candidateMetadataByHost.get(hostKey)?.retries || 0),
|
|
487
|
+
lastError: Object.prototype.hasOwnProperty.call(options, 'lastError')
|
|
488
|
+
? options.lastError
|
|
489
|
+
: (this._candidateMetadataByHost.get(hostKey)?.lastError ?? null),
|
|
490
|
+
version: options.version || this._candidateMetadataByHost.get(hostKey)?.version || '',
|
|
491
|
+
name: options.name || this._candidateMetadataByHost.get(hostKey)?.name || '',
|
|
492
|
+
selectedPort: Number.isFinite(options.selectedPort)
|
|
493
|
+
? options.selectedPort
|
|
494
|
+
: (this._candidateMetadataByHost.get(hostKey)?.selectedPort || 0),
|
|
495
|
+
};
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
_orderCandidatesByRegion(candidates) {
|
|
499
|
+
const groups = new Map();
|
|
500
|
+
for (const candidate of candidates) {
|
|
501
|
+
const region = this._getRegionKey(candidate.hostKey);
|
|
502
|
+
if (!groups.has(region)) {
|
|
503
|
+
groups.set(region, []);
|
|
504
|
+
}
|
|
505
|
+
groups.get(region).push(candidate);
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
const priority = ['as', 'us', 'eu', 'other'];
|
|
509
|
+
const ordered = [];
|
|
510
|
+
let hasMore = true;
|
|
511
|
+
while (hasMore) {
|
|
512
|
+
hasMore = false;
|
|
513
|
+
for (const region of priority) {
|
|
514
|
+
const queue = groups.get(region);
|
|
515
|
+
if (queue && queue.length > 0) {
|
|
516
|
+
ordered.push(queue.shift());
|
|
517
|
+
hasMore = true;
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
return ordered;
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
_getInitialCoverageCandidates(candidates) {
|
|
525
|
+
const requiredCandidates = candidates.filter((candidate) => candidate.source === 'seed' || candidate.source === 'configured');
|
|
526
|
+
if (!this.relaySelection.regionDiverseSeedOrdering || this._hasExplicitHost || this._hasExplicitHosts) {
|
|
527
|
+
return requiredCandidates;
|
|
528
|
+
}
|
|
529
|
+
return this._orderCandidatesByRegion(requiredCandidates);
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
_getStartupSeedBootstrapCandidates(candidates) {
|
|
533
|
+
const orderedRequiredCandidates = this._getInitialCoverageCandidates(candidates);
|
|
534
|
+
if (this._hasExplicitHost || this._hasExplicitHosts || !this.relaySelection.networkDiscovery.enabled) {
|
|
535
|
+
return orderedRequiredCandidates;
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
const selected = [];
|
|
539
|
+
const seenRegions = new Set();
|
|
540
|
+
for (const candidate of orderedRequiredCandidates) {
|
|
541
|
+
const region = candidate.region || this._getRegionKey(candidate.hostKey) || 'other';
|
|
542
|
+
if (seenRegions.has(region)) {
|
|
543
|
+
continue;
|
|
544
|
+
}
|
|
545
|
+
seenRegions.add(region);
|
|
546
|
+
selected.push(candidate);
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
return selected.length > 0 ? selected : orderedRequiredCandidates;
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
_getSourcePrecedence(source) {
|
|
553
|
+
switch (source) {
|
|
554
|
+
case 'configured': return 0;
|
|
555
|
+
case 'seed': return 1;
|
|
556
|
+
case 'provider': return 2;
|
|
557
|
+
case 'network': return 3;
|
|
558
|
+
case 'target': return 4;
|
|
559
|
+
case 'cache': return 5;
|
|
560
|
+
default: return 6;
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
_parseHexInt(value) {
|
|
565
|
+
if (Number.isFinite(value)) {
|
|
566
|
+
return Math.floor(value);
|
|
567
|
+
}
|
|
568
|
+
if (typeof value === 'string') {
|
|
569
|
+
if (/^0x[0-9a-f]+$/i.test(value)) {
|
|
570
|
+
return parseInt(value, 16);
|
|
571
|
+
}
|
|
572
|
+
if (/^\d+$/.test(value)) {
|
|
573
|
+
return parseInt(value, 10);
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
return 0;
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
_filterDiscoveryAddress(host) {
|
|
580
|
+
if (typeof host !== 'string' || !host.trim()) {
|
|
581
|
+
return false;
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
const normalized = host.trim().toLowerCase();
|
|
585
|
+
if (normalized === 'localhost') {
|
|
586
|
+
return false;
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
const ipVersion = net.isIP(normalized);
|
|
590
|
+
if (!ipVersion) {
|
|
591
|
+
return true;
|
|
592
|
+
}
|
|
593
|
+
if (this.relaySelection.networkDiscovery.includePrivateAddresses) {
|
|
594
|
+
return true;
|
|
595
|
+
}
|
|
596
|
+
if (ipVersion === 6) {
|
|
597
|
+
if (normalized === '::1') return false;
|
|
598
|
+
if (normalized.startsWith('fc') || normalized.startsWith('fd')) return false;
|
|
599
|
+
if (normalized.startsWith('fe80:')) return false;
|
|
600
|
+
return true;
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
const octets = normalized.split('.').map((part) => parseInt(part, 10));
|
|
604
|
+
if (octets.length !== 4 || octets.some((part) => !Number.isFinite(part))) {
|
|
605
|
+
return false;
|
|
606
|
+
}
|
|
607
|
+
if (octets[0] === 0 || octets[0] === 10 || octets[0] === 127) return false;
|
|
608
|
+
if (octets[0] === 169 && octets[1] === 254) return false;
|
|
609
|
+
if (octets[0] === 172 && octets[1] >= 16 && octets[1] <= 31) return false;
|
|
610
|
+
if (octets[0] === 192 && octets[1] === 168) return false;
|
|
611
|
+
if (octets[0] >= 224) return false;
|
|
612
|
+
if (octets[0] === 255) return false;
|
|
613
|
+
return true;
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
_getKnownRelayScoreSnapshot() {
|
|
617
|
+
return Array.from(this.relayScores.values()).map((score) => ({
|
|
618
|
+
hostKey: score.hostKey,
|
|
619
|
+
ewmaLatencyMs: Number.isFinite(score.ewmaLatencyMs) ? score.ewmaLatencyMs : null,
|
|
620
|
+
successCount: score.successCount || 0,
|
|
621
|
+
failureCount: score.failureCount || 0,
|
|
622
|
+
lastSuccessAt: score.lastSuccessAt || 0,
|
|
623
|
+
discoveredFrom: score.discoveredFrom || 'seed',
|
|
624
|
+
}));
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
async _callDiscoveryProviderWithTimeout() {
|
|
628
|
+
const provider = this.relaySelection.discoveryProvider;
|
|
629
|
+
if (typeof provider !== 'function') {
|
|
630
|
+
return [];
|
|
631
|
+
}
|
|
632
|
+
const context = Object.freeze({
|
|
633
|
+
defaultPort: this.defaultPort,
|
|
634
|
+
keyLocation: this.keyLocation,
|
|
635
|
+
explicitHost: this._hasExplicitHost,
|
|
636
|
+
explicitHosts: this._hasExplicitHosts,
|
|
637
|
+
initialHosts: this.initialHosts.slice(),
|
|
638
|
+
knownRelayScores: this._getKnownRelayScoreSnapshot(),
|
|
639
|
+
});
|
|
640
|
+
|
|
641
|
+
const timeoutMs = this.relaySelection.discoveryProviderTimeoutMs;
|
|
642
|
+
return Promise.race([
|
|
643
|
+
Promise.resolve().then(() => provider(context)),
|
|
644
|
+
new Promise((_, reject) => {
|
|
645
|
+
setTimeout(() => reject(new Error(`Discovery provider timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
646
|
+
}),
|
|
647
|
+
]);
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
_normalizeDiscoveryCandidate(entry, index) {
|
|
651
|
+
if (typeof entry === 'string') {
|
|
652
|
+
const hostKey = normalizeHostKey(entry, this.defaultPort);
|
|
653
|
+
if (!hostKey) return null;
|
|
654
|
+
return this._createCandidate(hostKey, 'provider', index, {
|
|
655
|
+
priority: 100,
|
|
656
|
+
region: this._getRegionKey(hostKey),
|
|
657
|
+
metadata: null,
|
|
658
|
+
});
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
if (!entry || typeof entry !== 'object' || Array.isArray(entry)) {
|
|
662
|
+
return null;
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
const hostKey = normalizeHostKey(
|
|
666
|
+
entry.port !== undefined ? joinHostPort(entry.host, entry.port) : entry.host,
|
|
667
|
+
this.defaultPort,
|
|
668
|
+
);
|
|
669
|
+
if (!hostKey) {
|
|
670
|
+
return null;
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
return this._createCandidate(hostKey, 'provider', index, {
|
|
674
|
+
priority: Number.isFinite(entry.priority) ? entry.priority : 100,
|
|
675
|
+
region: typeof entry.region === 'string' ? entry.region : this._getRegionKey(hostKey),
|
|
676
|
+
metadata: entry.metadata && typeof entry.metadata === 'object' && !Array.isArray(entry.metadata)
|
|
677
|
+
? entry.metadata
|
|
678
|
+
: null,
|
|
679
|
+
});
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
async _loadDiscoveryProviderCandidates() {
|
|
683
|
+
if (!this.relaySelection.enabled) {
|
|
684
|
+
return [];
|
|
685
|
+
}
|
|
686
|
+
if (this._hasExplicitHost && !this.relaySelection.useProviderWithExplicitHost) {
|
|
687
|
+
return [];
|
|
688
|
+
}
|
|
689
|
+
if (this._hasExplicitHosts && !this.relaySelection.useProviderWithExplicitHosts) {
|
|
690
|
+
return [];
|
|
691
|
+
}
|
|
692
|
+
if (typeof this.relaySelection.discoveryProvider !== 'function') {
|
|
693
|
+
return [];
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
try {
|
|
697
|
+
const provided = await this._callDiscoveryProviderWithTimeout();
|
|
698
|
+
if (!Array.isArray(provided)) {
|
|
699
|
+
logger.warn(() => 'Discovery provider returned a non-array result. Ignoring provider candidates.');
|
|
700
|
+
return [];
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
let invalidCount = 0;
|
|
704
|
+
const normalized = [];
|
|
705
|
+
provided.forEach((entry, index) => {
|
|
706
|
+
const candidate = this._normalizeDiscoveryCandidate(entry, index);
|
|
707
|
+
if (!candidate) {
|
|
708
|
+
invalidCount += 1;
|
|
709
|
+
return;
|
|
710
|
+
}
|
|
711
|
+
normalized.push(candidate);
|
|
712
|
+
});
|
|
713
|
+
if (invalidCount > 0) {
|
|
714
|
+
logger.warn(() => `Ignored ${invalidCount} invalid discovery provider candidates`);
|
|
715
|
+
}
|
|
716
|
+
return normalized;
|
|
717
|
+
} catch (error) {
|
|
718
|
+
logger.warn(() => `Discovery provider failed: ${error}`);
|
|
719
|
+
return [];
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
async _fetchNetworkDiscoveryNodes() {
|
|
724
|
+
return fetchNetworkDirectory({
|
|
725
|
+
endpoint: this.relaySelection.networkDiscovery.endpoint,
|
|
726
|
+
method: this.relaySelection.networkDiscovery.method,
|
|
727
|
+
timeoutMs: this.relaySelection.networkDiscovery.timeoutMs,
|
|
728
|
+
});
|
|
729
|
+
}
|
|
730
|
+
|
|
731
|
+
_normalizeNetworkNode(entry, index) {
|
|
732
|
+
if (!entry || typeof entry !== 'object' || !entry.connected) {
|
|
733
|
+
return null;
|
|
734
|
+
}
|
|
735
|
+
if (!Array.isArray(entry.node) || entry.node[0] !== 'server') {
|
|
736
|
+
return null;
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
const host = typeof entry.node[1] === 'string' ? entry.node[1].trim() : '';
|
|
740
|
+
if (!this._filterDiscoveryAddress(host)) {
|
|
741
|
+
return null;
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
const edgePort = this._parseHexInt(entry.node[2]);
|
|
745
|
+
const serverPort = this._parseHexInt(entry.node[3]);
|
|
746
|
+
const selectedPort = edgePort || serverPort;
|
|
747
|
+
if (!selectedPort) {
|
|
748
|
+
return null;
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
let name = '';
|
|
752
|
+
const metadataEntries = Array.isArray(entry.node[5]) ? entry.node[5] : [];
|
|
753
|
+
for (const metaEntry of metadataEntries) {
|
|
754
|
+
if (Array.isArray(metaEntry) && metaEntry[0] === 'name') {
|
|
755
|
+
name = typeof metaEntry[1] === 'string' ? metaEntry[1] : '';
|
|
756
|
+
break;
|
|
757
|
+
}
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
const hostKey = normalizeHostKey(joinHostPort(host, selectedPort), this.defaultPort);
|
|
761
|
+
if (!hostKey) {
|
|
762
|
+
return null;
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
return this._createCandidate(hostKey, 'network', index, {
|
|
766
|
+
priority: 100,
|
|
767
|
+
region: this._getRegionKey(hostKey),
|
|
768
|
+
metadata: { name },
|
|
769
|
+
nodeIdHex: normalizeServerIdHex(entry.node_id),
|
|
770
|
+
lastSeenAt: this._parseHexInt(entry.last_seen),
|
|
771
|
+
retries: this._parseHexInt(entry.retries),
|
|
772
|
+
lastError: entry.last_error ?? null,
|
|
773
|
+
version: typeof entry.node[4] === 'string' ? entry.node[4] : '',
|
|
774
|
+
name,
|
|
775
|
+
selectedPort,
|
|
776
|
+
edgePort,
|
|
777
|
+
serverPort,
|
|
778
|
+
connected: !!entry.connected,
|
|
779
|
+
});
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
async _loadNetworkDiscoveryCandidates() {
|
|
783
|
+
if (!this.relaySelection.enabled || !this.relaySelection.networkDiscovery.enabled) {
|
|
784
|
+
this._lastNetworkDiscoveryStats = {
|
|
785
|
+
loadedCount: 0,
|
|
786
|
+
usableCount: 0,
|
|
787
|
+
filteredCount: 0,
|
|
788
|
+
startupProbeCount: 0,
|
|
789
|
+
};
|
|
790
|
+
return [];
|
|
791
|
+
}
|
|
792
|
+
if (this._hasExplicitHost || this._hasExplicitHosts) {
|
|
793
|
+
this._lastNetworkDiscoveryStats = {
|
|
794
|
+
loadedCount: 0,
|
|
795
|
+
usableCount: 0,
|
|
796
|
+
filteredCount: 0,
|
|
797
|
+
startupProbeCount: 0,
|
|
798
|
+
};
|
|
799
|
+
return [];
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
try {
|
|
803
|
+
const discovered = await Promise.race([
|
|
804
|
+
Promise.resolve().then(() => this._fetchNetworkDiscoveryNodes()),
|
|
805
|
+
new Promise((_, reject) => {
|
|
806
|
+
setTimeout(
|
|
807
|
+
() => reject(new Error(`Network discovery timed out after ${this.relaySelection.networkDiscovery.timeoutMs}ms`)),
|
|
808
|
+
this.relaySelection.networkDiscovery.timeoutMs,
|
|
809
|
+
);
|
|
810
|
+
}),
|
|
811
|
+
]);
|
|
812
|
+
if (!Array.isArray(discovered)) {
|
|
813
|
+
logger.warn(() => 'Network discovery returned a non-array result. Ignoring discovered nodes.');
|
|
814
|
+
return [];
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
let invalidCount = 0;
|
|
818
|
+
const normalized = [];
|
|
819
|
+
discovered.forEach((entry, index) => {
|
|
820
|
+
const candidate = this._normalizeNetworkNode(entry, index);
|
|
821
|
+
if (!candidate) {
|
|
822
|
+
invalidCount += 1;
|
|
823
|
+
return;
|
|
824
|
+
}
|
|
825
|
+
normalized.push(candidate);
|
|
826
|
+
});
|
|
827
|
+
if (invalidCount > 0) {
|
|
828
|
+
logger.warn(() => `Ignored ${invalidCount} invalid network discovery nodes`);
|
|
829
|
+
}
|
|
830
|
+
this._lastNetworkDiscoveryStats = {
|
|
831
|
+
loadedCount: discovered.length,
|
|
832
|
+
usableCount: normalized.length,
|
|
833
|
+
filteredCount: invalidCount,
|
|
834
|
+
startupProbeCount: 0,
|
|
835
|
+
};
|
|
836
|
+
return normalized;
|
|
837
|
+
} catch (error) {
|
|
838
|
+
this._lastNetworkDiscoveryStats = {
|
|
839
|
+
loadedCount: 0,
|
|
840
|
+
usableCount: 0,
|
|
841
|
+
filteredCount: 0,
|
|
842
|
+
startupProbeCount: 0,
|
|
843
|
+
};
|
|
844
|
+
logger.warn(() => `Network discovery failed: ${error}`);
|
|
845
|
+
return [];
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
_mergeStartupCandidateLists(initialCandidates, providerCandidates, networkCandidates, cachedCandidates) {
|
|
850
|
+
const merged = [];
|
|
851
|
+
const byHost = new Map();
|
|
852
|
+
const pushCandidate = (candidate) => {
|
|
853
|
+
const lower = candidate.hostKey.toLowerCase();
|
|
854
|
+
if (!byHost.has(lower)) {
|
|
855
|
+
const normalized = { ...candidate, index: merged.length };
|
|
856
|
+
byHost.set(lower, normalized);
|
|
857
|
+
merged.push(normalized);
|
|
858
|
+
return;
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
const existing = byHost.get(lower);
|
|
862
|
+
if (this._getSourcePrecedence(candidate.source) < this._getSourcePrecedence(existing.source)) {
|
|
863
|
+
const replacement = { ...candidate, index: existing.index };
|
|
864
|
+
merged[existing.index] = replacement;
|
|
865
|
+
byHost.set(lower, replacement);
|
|
866
|
+
}
|
|
867
|
+
};
|
|
868
|
+
|
|
869
|
+
initialCandidates.forEach(pushCandidate);
|
|
870
|
+
providerCandidates.forEach(pushCandidate);
|
|
871
|
+
networkCandidates.forEach(pushCandidate);
|
|
872
|
+
cachedCandidates.forEach(pushCandidate);
|
|
873
|
+
return merged;
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
async _buildStartupCandidates() {
|
|
877
|
+
const initialSource = this._hasExplicitHost || this._hasExplicitHosts ? 'configured' : 'seed';
|
|
878
|
+
const initialCandidates = this.initialHosts.map((hostKey, index) => (
|
|
879
|
+
this._createCandidate(hostKey, initialSource, index)
|
|
880
|
+
));
|
|
881
|
+
|
|
882
|
+
const [providerCandidates, networkCandidates] = await Promise.all([
|
|
883
|
+
this._loadDiscoveryProviderCandidates(),
|
|
884
|
+
this._loadNetworkDiscoveryCandidates(),
|
|
885
|
+
]);
|
|
886
|
+
const cachedCandidates = [];
|
|
887
|
+
if (!this._hasExplicitHost && !this._hasExplicitHosts) {
|
|
888
|
+
for (const [hostKey, score] of this.relayScores.entries()) {
|
|
889
|
+
if ((score.successCount || 0) <= 0) {
|
|
890
|
+
continue;
|
|
891
|
+
}
|
|
892
|
+
if (score.discoveredFrom === 'provider' || score.discoveredFrom === 'network') {
|
|
893
|
+
continue;
|
|
894
|
+
}
|
|
895
|
+
const source = score.discoveredFrom === 'target' ? 'target' : 'cache';
|
|
896
|
+
cachedCandidates.push(this._createCandidate(hostKey, source, cachedCandidates.length));
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
return this._mergeStartupCandidateLists(initialCandidates, providerCandidates, networkCandidates, cachedCandidates);
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
_advanceNetworkCursor(totalCandidates, consumedCount) {
|
|
904
|
+
if (!Number.isFinite(totalCandidates) || totalCandidates <= 0 || !Number.isFinite(consumedCount) || consumedCount <= 0) {
|
|
905
|
+
return;
|
|
906
|
+
}
|
|
907
|
+
this.discoveryState.networkCursor = (this.discoveryState.networkCursor + consumedCount) % totalCandidates;
|
|
908
|
+
this._scheduleRelayScoreFlush();
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
_selectStartupNetworkCandidates(candidates) {
|
|
912
|
+
const startupProbeCount = this.relaySelection.networkDiscovery.startupProbeCount;
|
|
913
|
+
if (!startupProbeCount) {
|
|
914
|
+
return [];
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
const networkCandidates = candidates.filter((candidate) => candidate.source === 'network');
|
|
918
|
+
if (networkCandidates.length === 0) {
|
|
919
|
+
return [];
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
const freshScored = networkCandidates
|
|
923
|
+
.filter((candidate) => candidate.hasBeenTested && !candidate.inCooldown && candidate.scoreFresh && candidate.ewmaLatencyMs !== null)
|
|
924
|
+
.sort((left, right) => left.ewmaLatencyMs - right.ewmaLatencyMs);
|
|
925
|
+
const staleScored = networkCandidates
|
|
926
|
+
.filter((candidate) => candidate.hasBeenTested && !candidate.inCooldown && !candidate.scoreFresh && candidate.ewmaLatencyMs !== null)
|
|
927
|
+
.sort((left, right) => left.ewmaLatencyMs - right.ewmaLatencyMs);
|
|
928
|
+
const cooldown = networkCandidates
|
|
929
|
+
.filter((candidate) => candidate.inCooldown)
|
|
930
|
+
.sort((left, right) => (left.cooldownUntil || 0) - (right.cooldownUntil || 0));
|
|
931
|
+
|
|
932
|
+
const compareUntestedNetwork = (left, right) => {
|
|
933
|
+
if (left.retries !== right.retries) return left.retries - right.retries;
|
|
934
|
+
const leftHasError = left.lastError !== null && left.lastError !== '0x00' && left.lastError !== 0;
|
|
935
|
+
const rightHasError = right.lastError !== null && right.lastError !== '0x00' && right.lastError !== 0;
|
|
936
|
+
if (leftHasError !== rightHasError) return leftHasError ? 1 : -1;
|
|
937
|
+
if (left.lastSeenAt !== right.lastSeenAt) return right.lastSeenAt - left.lastSeenAt;
|
|
938
|
+
return left.hostKey.localeCompare(right.hostKey);
|
|
939
|
+
};
|
|
940
|
+
|
|
941
|
+
const untested = networkCandidates
|
|
942
|
+
.filter((candidate) => !candidate.hasBeenTested && !candidate.inCooldown)
|
|
943
|
+
.sort(compareUntestedNetwork);
|
|
944
|
+
|
|
945
|
+
const prioritizedUntested = untested.length > 0
|
|
946
|
+
? untested.slice(this.discoveryState.networkCursor % untested.length).concat(untested.slice(0, this.discoveryState.networkCursor % untested.length))
|
|
947
|
+
: [];
|
|
948
|
+
|
|
949
|
+
const selected = [];
|
|
950
|
+
const seen = new Set();
|
|
951
|
+
let consumedUntestedCount = 0;
|
|
952
|
+
const pushCandidate = (candidate) => {
|
|
953
|
+
if (!candidate || seen.has(candidate.hostKey) || selected.length >= startupProbeCount) {
|
|
954
|
+
return;
|
|
955
|
+
}
|
|
956
|
+
seen.add(candidate.hostKey);
|
|
957
|
+
selected.push(candidate);
|
|
958
|
+
if (!candidate.hasBeenTested) {
|
|
959
|
+
consumedUntestedCount += 1;
|
|
960
|
+
}
|
|
961
|
+
};
|
|
962
|
+
|
|
963
|
+
freshScored.forEach(pushCandidate);
|
|
964
|
+
prioritizedUntested.forEach(pushCandidate);
|
|
965
|
+
staleScored.forEach(pushCandidate);
|
|
966
|
+
cooldown.forEach(pushCandidate);
|
|
967
|
+
|
|
968
|
+
if (this._lastNetworkDiscoveryStats) {
|
|
969
|
+
this._lastNetworkDiscoveryStats.startupProbeCount = selected.length;
|
|
970
|
+
}
|
|
971
|
+
this._advanceNetworkCursor(untested.length, Math.min(consumedUntestedCount, untested.length));
|
|
972
|
+
return selected;
|
|
973
|
+
}
|
|
974
|
+
|
|
975
|
+
_selectBackgroundNetworkCandidates(candidates) {
|
|
976
|
+
const batchSize = this.relaySelection.networkDiscovery.backgroundBatchSize;
|
|
977
|
+
if (!batchSize) {
|
|
978
|
+
return [];
|
|
979
|
+
}
|
|
980
|
+
return candidates
|
|
981
|
+
.filter((candidate) => candidate.source === 'network')
|
|
982
|
+
.slice(0, batchSize);
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
_rankRelayCandidates(candidates) {
|
|
986
|
+
const normalizedCandidates = candidates.map((candidate, index) => {
|
|
987
|
+
if (typeof candidate === 'string') {
|
|
988
|
+
return this._createCandidate(candidate, 'cache', index);
|
|
989
|
+
}
|
|
990
|
+
return {
|
|
991
|
+
...candidate,
|
|
992
|
+
index,
|
|
993
|
+
};
|
|
994
|
+
});
|
|
995
|
+
|
|
996
|
+
return normalizedCandidates
|
|
997
|
+
.slice()
|
|
998
|
+
.sort((left, right) => {
|
|
999
|
+
const getGroup = (candidate) => {
|
|
1000
|
+
if (candidate.inCooldown) return 5;
|
|
1001
|
+
const requiredUntested = (candidate.source === 'seed' || candidate.source === 'configured') && !candidate.hasBeenTested;
|
|
1002
|
+
if (requiredUntested) return 0;
|
|
1003
|
+
if (candidate.hasBeenTested && candidate.ewmaLatencyMs !== null) return 1;
|
|
1004
|
+
if (candidate.source === 'network' && !candidate.hasBeenTested) return 2;
|
|
1005
|
+
if (candidate.source === 'provider' && !candidate.hasBeenTested) return 3;
|
|
1006
|
+
return 4;
|
|
1007
|
+
};
|
|
1008
|
+
|
|
1009
|
+
const leftGroup = getGroup(left);
|
|
1010
|
+
const rightGroup = getGroup(right);
|
|
1011
|
+
if (leftGroup !== rightGroup) {
|
|
1012
|
+
return leftGroup - rightGroup;
|
|
1013
|
+
}
|
|
1014
|
+
|
|
1015
|
+
if (leftGroup === 2) {
|
|
1016
|
+
if (left.retries !== right.retries) return left.retries - right.retries;
|
|
1017
|
+
const leftHasError = left.lastError !== null && left.lastError !== '0x00' && left.lastError !== 0;
|
|
1018
|
+
const rightHasError = right.lastError !== null && right.lastError !== '0x00' && right.lastError !== 0;
|
|
1019
|
+
if (leftHasError !== rightHasError) return leftHasError ? 1 : -1;
|
|
1020
|
+
if (left.lastSeenAt !== right.lastSeenAt) return right.lastSeenAt - left.lastSeenAt;
|
|
1021
|
+
}
|
|
1022
|
+
if (leftGroup === 3 && left.priority !== right.priority) {
|
|
1023
|
+
return left.priority - right.priority;
|
|
1024
|
+
}
|
|
1025
|
+
if (leftGroup === 1 && left.ewmaLatencyMs !== right.ewmaLatencyMs) {
|
|
1026
|
+
return left.ewmaLatencyMs - right.ewmaLatencyMs;
|
|
1027
|
+
}
|
|
1028
|
+
if (left.lastSuccessAt !== right.lastSuccessAt) {
|
|
1029
|
+
return right.lastSuccessAt - left.lastSuccessAt;
|
|
1030
|
+
}
|
|
1031
|
+
if (left.source !== right.source) {
|
|
1032
|
+
return this._getSourcePrecedence(left.source) - this._getSourcePrecedence(right.source);
|
|
1033
|
+
}
|
|
1034
|
+
return left.index - right.index;
|
|
1035
|
+
})
|
|
1036
|
+
.map((entry) => entry);
|
|
1037
|
+
}
|
|
1038
|
+
|
|
139
1039
|
_getRpcFor(connection) {
|
|
140
1040
|
if (!connection) return null;
|
|
141
1041
|
let rpc = this._rpcByConnection.get(connection);
|
|
@@ -149,17 +1049,92 @@ class DiodeClientManager extends EventEmitter {
|
|
|
149
1049
|
_updateServerIdMapping(connection) {
|
|
150
1050
|
if (!connection) return;
|
|
151
1051
|
try {
|
|
152
|
-
const serverId = connection.getServerEthereumAddress(true);
|
|
1052
|
+
const serverId = normalizeServerIdHex(connection.getServerEthereumAddress(true));
|
|
153
1053
|
if (serverId) {
|
|
154
|
-
|
|
1054
|
+
connection._managerServerIdHex = serverId;
|
|
1055
|
+
const existing = this.serverIdToConnection.get(serverId);
|
|
1056
|
+
if (existing && existing !== connection && isConnected(existing)) {
|
|
1057
|
+
const preferred = this._rankConnectedConnections(
|
|
1058
|
+
[existing, connection].filter((candidate) => isConnected(candidate))
|
|
1059
|
+
)[0];
|
|
1060
|
+
this.serverIdToConnection.set(serverId, preferred || connection);
|
|
1061
|
+
return;
|
|
1062
|
+
}
|
|
1063
|
+
this.serverIdToConnection.set(serverId, connection);
|
|
155
1064
|
}
|
|
156
1065
|
} catch (error) {
|
|
157
1066
|
logger.debug(() => `Failed to map server ID for ${connection.host}:${connection.port}: ${error}`);
|
|
158
1067
|
}
|
|
159
1068
|
}
|
|
160
1069
|
|
|
1070
|
+
_refreshServerIdMapping(serverIdHex) {
|
|
1071
|
+
if (!serverIdHex) {
|
|
1072
|
+
return;
|
|
1073
|
+
}
|
|
1074
|
+
|
|
1075
|
+
const matches = this._connectedConnections().filter((connection) => {
|
|
1076
|
+
try {
|
|
1077
|
+
const mappedServerId = connection._managerServerIdHex
|
|
1078
|
+
|| normalizeServerIdHex(connection.getServerEthereumAddress(true));
|
|
1079
|
+
if (mappedServerId) {
|
|
1080
|
+
connection._managerServerIdHex = mappedServerId;
|
|
1081
|
+
}
|
|
1082
|
+
return mappedServerId === serverIdHex;
|
|
1083
|
+
} catch (_) {
|
|
1084
|
+
return false;
|
|
1085
|
+
}
|
|
1086
|
+
});
|
|
1087
|
+
|
|
1088
|
+
if (matches.length === 0) {
|
|
1089
|
+
this.serverIdToConnection.delete(serverIdHex);
|
|
1090
|
+
return;
|
|
1091
|
+
}
|
|
1092
|
+
|
|
1093
|
+
this.serverIdToConnection.set(
|
|
1094
|
+
serverIdHex,
|
|
1095
|
+
this._rankConnectedConnections(matches)[0] || matches[0],
|
|
1096
|
+
);
|
|
1097
|
+
}
|
|
1098
|
+
|
|
1099
|
+
_rankConnectedConnections(connected, options = {}) {
|
|
1100
|
+
const { queueRefresh = false } = options;
|
|
1101
|
+
return connected
|
|
1102
|
+
.map((connection, index) => {
|
|
1103
|
+
const hostKey = connection._managerHostKey || normalizeHostKey(joinHostPort(connection.host, connection.port), this.defaultPort);
|
|
1104
|
+
const score = this.relayScores.get(hostKey);
|
|
1105
|
+
const hasScore = !!(score && Number.isFinite(score.ewmaLatencyMs));
|
|
1106
|
+
const fresh = this._isRelayScoreFresh(score);
|
|
1107
|
+
if (queueRefresh && hasScore && !fresh) {
|
|
1108
|
+
this._queueBackgroundProbe(connection, hostKey);
|
|
1109
|
+
}
|
|
1110
|
+
return {
|
|
1111
|
+
connection,
|
|
1112
|
+
index,
|
|
1113
|
+
hasScore,
|
|
1114
|
+
fresh,
|
|
1115
|
+
latency: hasScore ? score.ewmaLatencyMs : Number.POSITIVE_INFINITY,
|
|
1116
|
+
connectedAt: connection._managerConnectedAt || Number.MAX_SAFE_INTEGER,
|
|
1117
|
+
};
|
|
1118
|
+
})
|
|
1119
|
+
.sort((left, right) => {
|
|
1120
|
+
const leftGroup = left.hasScore ? (left.fresh ? 0 : 1) : 2;
|
|
1121
|
+
const rightGroup = right.hasScore ? (right.fresh ? 0 : 1) : 2;
|
|
1122
|
+
if (leftGroup !== rightGroup) {
|
|
1123
|
+
return leftGroup - rightGroup;
|
|
1124
|
+
}
|
|
1125
|
+
if (left.latency !== right.latency) {
|
|
1126
|
+
return left.latency - right.latency;
|
|
1127
|
+
}
|
|
1128
|
+
if (left.connectedAt !== right.connectedAt) {
|
|
1129
|
+
return left.connectedAt - right.connectedAt;
|
|
1130
|
+
}
|
|
1131
|
+
return left.index - right.index;
|
|
1132
|
+
})
|
|
1133
|
+
.map((entry) => entry.connection);
|
|
1134
|
+
}
|
|
1135
|
+
|
|
161
1136
|
_localAddressHintFor(connection) {
|
|
162
|
-
const connected = this._connectedConnections();
|
|
1137
|
+
const connected = this._rankConnectedConnections(this._connectedConnections());
|
|
163
1138
|
if (connected.length === 0) {
|
|
164
1139
|
return Buffer.alloc(0);
|
|
165
1140
|
}
|
|
@@ -201,6 +1176,9 @@ class DiodeClientManager extends EventEmitter {
|
|
|
201
1176
|
connection._managerHostKey = hostKey;
|
|
202
1177
|
this.connections.push(connection);
|
|
203
1178
|
this.connectionByHost.set(hostKey, connection);
|
|
1179
|
+
if (typeof connection.setFleetContract === 'function') {
|
|
1180
|
+
connection.setFleetContract(this.fleetContract);
|
|
1181
|
+
}
|
|
204
1182
|
if (typeof connection.setLocalAddressProvider === 'function') {
|
|
205
1183
|
connection.setLocalAddressProvider(() => this._localAddressHintFor(connection));
|
|
206
1184
|
}
|
|
@@ -210,6 +1188,9 @@ class DiodeClientManager extends EventEmitter {
|
|
|
210
1188
|
});
|
|
211
1189
|
connection.on('reconnected', () => {
|
|
212
1190
|
this._updateServerIdMapping(connection);
|
|
1191
|
+
if (this.relaySelection.enabled) {
|
|
1192
|
+
this._queueBackgroundProbe(connection, hostKey);
|
|
1193
|
+
}
|
|
213
1194
|
this.emit('reconnected', connection);
|
|
214
1195
|
});
|
|
215
1196
|
connection.on('reconnecting', (info) => {
|
|
@@ -226,23 +1207,97 @@ class DiodeClientManager extends EventEmitter {
|
|
|
226
1207
|
this.connectionByHost.delete(hostKey);
|
|
227
1208
|
}
|
|
228
1209
|
this.connections = this.connections.filter((item) => item !== connection);
|
|
1210
|
+
const removedServerIds = new Set();
|
|
229
1211
|
for (const [serverId, conn] of this.serverIdToConnection.entries()) {
|
|
230
1212
|
if (conn === connection) {
|
|
231
1213
|
this.serverIdToConnection.delete(serverId);
|
|
1214
|
+
removedServerIds.add(serverId);
|
|
232
1215
|
}
|
|
233
1216
|
}
|
|
1217
|
+
if (connection._managerServerIdHex) {
|
|
1218
|
+
removedServerIds.add(connection._managerServerIdHex);
|
|
1219
|
+
}
|
|
1220
|
+
removedServerIds.forEach((serverId) => this._refreshServerIdMapping(serverId));
|
|
234
1221
|
this._rpcByConnection.delete(connection);
|
|
235
1222
|
}
|
|
236
1223
|
|
|
1224
|
+
_closeManagedConnection(connection) {
|
|
1225
|
+
if (!connection) return;
|
|
1226
|
+
const hostKey = connection._managerHostKey || '';
|
|
1227
|
+
this._unregisterConnection(connection, hostKey);
|
|
1228
|
+
try {
|
|
1229
|
+
connection.close();
|
|
1230
|
+
} catch (_) {}
|
|
1231
|
+
}
|
|
1232
|
+
|
|
1233
|
+
_isProtectedHost(hostKey) {
|
|
1234
|
+
if (!hostKey) {
|
|
1235
|
+
return false;
|
|
1236
|
+
}
|
|
1237
|
+
const score = this.relayScores.get(hostKey);
|
|
1238
|
+
if (score && score.discoveredFrom === 'target') {
|
|
1239
|
+
return true;
|
|
1240
|
+
}
|
|
1241
|
+
for (const cached of this.deviceRelayCache.values()) {
|
|
1242
|
+
const ttlMs = Number.isFinite(cached.ttlMs) ? cached.ttlMs : this.deviceCacheTtlMs;
|
|
1243
|
+
if (ttlMs > 0 && Date.now() - cached.ts < ttlMs && cached.hostKey === hostKey) {
|
|
1244
|
+
return true;
|
|
1245
|
+
}
|
|
1246
|
+
}
|
|
1247
|
+
return false;
|
|
1248
|
+
}
|
|
1249
|
+
|
|
1250
|
+
_pruneIdleConnections() {
|
|
1251
|
+
if (!this.relaySelection.enabled || !this._startupCoverageComplete) {
|
|
1252
|
+
return;
|
|
1253
|
+
}
|
|
1254
|
+
const warmBudget = Math.max(1, this.relaySelection.warmConnectionBudget);
|
|
1255
|
+
const ranked = this._rankConnectedConnections(this._connectedConnections());
|
|
1256
|
+
const keepHosts = new Set();
|
|
1257
|
+
|
|
1258
|
+
if (this.relaySelection.regionDiverseSeedOrdering) {
|
|
1259
|
+
const seenRegions = new Set();
|
|
1260
|
+
for (const connection of ranked) {
|
|
1261
|
+
if (keepHosts.size >= warmBudget) break;
|
|
1262
|
+
const hostKey = connection._managerHostKey || '';
|
|
1263
|
+
const region = this._getCandidateRegion(hostKey);
|
|
1264
|
+
if (seenRegions.has(region)) continue;
|
|
1265
|
+
seenRegions.add(region);
|
|
1266
|
+
keepHosts.add(hostKey);
|
|
1267
|
+
}
|
|
1268
|
+
}
|
|
1269
|
+
|
|
1270
|
+
for (const connection of ranked) {
|
|
1271
|
+
if (keepHosts.size >= warmBudget) break;
|
|
1272
|
+
keepHosts.add(connection._managerHostKey);
|
|
1273
|
+
}
|
|
1274
|
+
|
|
1275
|
+
for (const connection of ranked) {
|
|
1276
|
+
const hostKey = connection._managerHostKey || '';
|
|
1277
|
+
if (!hostKey) continue;
|
|
1278
|
+
if (keepHosts.has(hostKey) || this._isProtectedHost(hostKey)) {
|
|
1279
|
+
continue;
|
|
1280
|
+
}
|
|
1281
|
+
this._closeManagedConnection(connection);
|
|
1282
|
+
}
|
|
1283
|
+
}
|
|
1284
|
+
|
|
237
1285
|
async _ensureConnection(hostEntry) {
|
|
238
|
-
const
|
|
239
|
-
const
|
|
1286
|
+
const hostKey = normalizeHostKey(hostEntry, this.defaultPort);
|
|
1287
|
+
const { host, port } = splitHostPort(hostKey, this.defaultPort);
|
|
240
1288
|
if (!host) {
|
|
241
1289
|
throw new Error(`Invalid host entry: ${hostEntry}`);
|
|
242
1290
|
}
|
|
243
1291
|
|
|
244
1292
|
if (this.connectionByHost.has(hostKey)) {
|
|
245
|
-
|
|
1293
|
+
const existing = this.connectionByHost.get(hostKey);
|
|
1294
|
+
if (isConnected(existing)) {
|
|
1295
|
+
return existing;
|
|
1296
|
+
}
|
|
1297
|
+
if (existing && typeof existing._ensureConnected === 'function') {
|
|
1298
|
+
await existing._ensureConnected();
|
|
1299
|
+
}
|
|
1300
|
+
return existing;
|
|
246
1301
|
}
|
|
247
1302
|
|
|
248
1303
|
if (this.pendingConnections.has(hostKey)) {
|
|
@@ -254,6 +1309,7 @@ class DiodeClientManager extends EventEmitter {
|
|
|
254
1309
|
|
|
255
1310
|
const promise = connection.connect()
|
|
256
1311
|
.then(() => {
|
|
1312
|
+
connection._managerConnectedAt = connection._managerConnectedAt || Date.now();
|
|
257
1313
|
this._updateServerIdMapping(connection);
|
|
258
1314
|
this.emit('connected', connection);
|
|
259
1315
|
return connection;
|
|
@@ -270,6 +1326,207 @@ class DiodeClientManager extends EventEmitter {
|
|
|
270
1326
|
return promise;
|
|
271
1327
|
}
|
|
272
1328
|
|
|
1329
|
+
async _probeConnection(connection, hostKey, discoveredFrom, startedAt = Date.now()) {
|
|
1330
|
+
const timeoutMs = this.relaySelection.probeTimeoutMs;
|
|
1331
|
+
const pingPromise = Promise.resolve()
|
|
1332
|
+
.then(() => this._getRpcFor(connection).ping())
|
|
1333
|
+
.then((result) => {
|
|
1334
|
+
if (!result) {
|
|
1335
|
+
throw new Error(`Relay probe failed for ${hostKey}`);
|
|
1336
|
+
}
|
|
1337
|
+
});
|
|
1338
|
+
|
|
1339
|
+
await Promise.race([
|
|
1340
|
+
pingPromise,
|
|
1341
|
+
new Promise((_, reject) => {
|
|
1342
|
+
setTimeout(() => reject(new Error(`Relay probe timed out for ${hostKey}`)), timeoutMs);
|
|
1343
|
+
}),
|
|
1344
|
+
]);
|
|
1345
|
+
|
|
1346
|
+
const latencyMs = Math.max(1, Date.now() - startedAt);
|
|
1347
|
+
this._recordRelayProbeSuccess(hostKey, latencyMs, discoveredFrom);
|
|
1348
|
+
return connection;
|
|
1349
|
+
}
|
|
1350
|
+
|
|
1351
|
+
async _probeHost(hostEntry, discoveredFrom = 'seed') {
|
|
1352
|
+
const hostKey = normalizeHostKey(hostEntry, this.defaultPort);
|
|
1353
|
+
if (!hostKey) {
|
|
1354
|
+
throw new Error(`Invalid host entry: ${hostEntry}`);
|
|
1355
|
+
}
|
|
1356
|
+
if (this.pendingProbes.has(hostKey)) {
|
|
1357
|
+
return this.pendingProbes.get(hostKey);
|
|
1358
|
+
}
|
|
1359
|
+
|
|
1360
|
+
const startedAt = Date.now();
|
|
1361
|
+
this._lastProbeStartedAt.set(hostKey, startedAt);
|
|
1362
|
+
const probePromise = (async () => {
|
|
1363
|
+
try {
|
|
1364
|
+
const connection = await Promise.race([
|
|
1365
|
+
this._ensureConnection(hostKey),
|
|
1366
|
+
new Promise((_, reject) => {
|
|
1367
|
+
setTimeout(() => reject(new Error(`Relay connection timed out for ${hostKey}`)), this.relaySelection.probeTimeoutMs);
|
|
1368
|
+
}),
|
|
1369
|
+
]);
|
|
1370
|
+
const probedConnection = await this._probeConnection(connection, hostKey, discoveredFrom, startedAt);
|
|
1371
|
+
this._pruneIdleConnections();
|
|
1372
|
+
return probedConnection;
|
|
1373
|
+
} catch (error) {
|
|
1374
|
+
if (String(error && error.message ? error.message : error).includes('Relay connection timed out')) {
|
|
1375
|
+
const stalledConnection = this.connectionByHost.get(hostKey);
|
|
1376
|
+
if (stalledConnection && !isConnected(stalledConnection)) {
|
|
1377
|
+
this._closeManagedConnection(stalledConnection);
|
|
1378
|
+
}
|
|
1379
|
+
}
|
|
1380
|
+
this._recordRelayProbeFailure(hostKey, error);
|
|
1381
|
+
throw error;
|
|
1382
|
+
} finally {
|
|
1383
|
+
this.pendingProbes.delete(hostKey);
|
|
1384
|
+
}
|
|
1385
|
+
})();
|
|
1386
|
+
|
|
1387
|
+
this.pendingProbes.set(hostKey, probePromise);
|
|
1388
|
+
return probePromise;
|
|
1389
|
+
}
|
|
1390
|
+
|
|
1391
|
+
_recordRelayProbeSuccess(hostKey, latencyMs, discoveredFrom) {
|
|
1392
|
+
const now = Date.now();
|
|
1393
|
+
const previous = this.relayScores.get(hostKey) || {
|
|
1394
|
+
hostKey,
|
|
1395
|
+
ewmaLatencyMs: null,
|
|
1396
|
+
lastProbeLatencyMs: null,
|
|
1397
|
+
successCount: 0,
|
|
1398
|
+
failureCount: 0,
|
|
1399
|
+
lastSuccessAt: 0,
|
|
1400
|
+
lastFailureAt: 0,
|
|
1401
|
+
cooldownUntil: 0,
|
|
1402
|
+
discoveredFrom: discoveredFrom || 'seed',
|
|
1403
|
+
};
|
|
1404
|
+
|
|
1405
|
+
const previousLatency = Number.isFinite(previous.ewmaLatencyMs) ? previous.ewmaLatencyMs : null;
|
|
1406
|
+
const ewmaLatencyMs = previousLatency === null
|
|
1407
|
+
? latencyMs
|
|
1408
|
+
: (RELAY_SCORE_EWMA_WEIGHT * latencyMs) + ((1 - RELAY_SCORE_EWMA_WEIGHT) * previousLatency);
|
|
1409
|
+
|
|
1410
|
+
this.relayScores.set(hostKey, {
|
|
1411
|
+
hostKey,
|
|
1412
|
+
ewmaLatencyMs,
|
|
1413
|
+
lastProbeLatencyMs: latencyMs,
|
|
1414
|
+
successCount: (previous.successCount || 0) + 1,
|
|
1415
|
+
failureCount: previous.failureCount || 0,
|
|
1416
|
+
lastSuccessAt: now,
|
|
1417
|
+
lastFailureAt: previous.lastFailureAt || 0,
|
|
1418
|
+
cooldownUntil: 0,
|
|
1419
|
+
discoveredFrom: discoveredFrom || previous.discoveredFrom || 'seed',
|
|
1420
|
+
});
|
|
1421
|
+
this._scheduleRelayScoreFlush();
|
|
1422
|
+
}
|
|
1423
|
+
|
|
1424
|
+
_recordRelayProbeFailure(hostKey, error) {
|
|
1425
|
+
const now = Date.now();
|
|
1426
|
+
const previous = this.relayScores.get(hostKey) || {
|
|
1427
|
+
hostKey,
|
|
1428
|
+
ewmaLatencyMs: null,
|
|
1429
|
+
lastProbeLatencyMs: null,
|
|
1430
|
+
successCount: 0,
|
|
1431
|
+
failureCount: 0,
|
|
1432
|
+
lastSuccessAt: 0,
|
|
1433
|
+
lastFailureAt: 0,
|
|
1434
|
+
cooldownUntil: 0,
|
|
1435
|
+
discoveredFrom: 'seed',
|
|
1436
|
+
};
|
|
1437
|
+
|
|
1438
|
+
this.relayScores.set(hostKey, {
|
|
1439
|
+
hostKey,
|
|
1440
|
+
ewmaLatencyMs: Number.isFinite(previous.ewmaLatencyMs) ? previous.ewmaLatencyMs : null,
|
|
1441
|
+
lastProbeLatencyMs: previous.lastProbeLatencyMs || null,
|
|
1442
|
+
successCount: previous.successCount || 0,
|
|
1443
|
+
failureCount: (previous.failureCount || 0) + 1,
|
|
1444
|
+
lastSuccessAt: previous.lastSuccessAt || 0,
|
|
1445
|
+
lastFailureAt: now,
|
|
1446
|
+
cooldownUntil: now + RELAY_SCORE_FAILURE_COOLDOWN_MS,
|
|
1447
|
+
discoveredFrom: previous.discoveredFrom || 'seed',
|
|
1448
|
+
});
|
|
1449
|
+
this._scheduleRelayScoreFlush();
|
|
1450
|
+
|
|
1451
|
+
if (error) {
|
|
1452
|
+
logger.debug(() => `Relay probe failed for ${hostKey}: ${error}`);
|
|
1453
|
+
}
|
|
1454
|
+
}
|
|
1455
|
+
|
|
1456
|
+
_selectPreferredConnectedConnection() {
|
|
1457
|
+
const connected = this._connectedConnections();
|
|
1458
|
+
if (connected.length === 0) {
|
|
1459
|
+
return null;
|
|
1460
|
+
}
|
|
1461
|
+
return this._rankConnectedConnections(connected, { queueRefresh: true })[0] || null;
|
|
1462
|
+
}
|
|
1463
|
+
|
|
1464
|
+
_isRelayScoreFresh(score) {
|
|
1465
|
+
return !!(score
|
|
1466
|
+
&& Number.isFinite(score.ewmaLatencyMs)
|
|
1467
|
+
&& score.lastSuccessAt
|
|
1468
|
+
&& (Date.now() - score.lastSuccessAt) <= RELAY_SCORE_FRESH_MS);
|
|
1469
|
+
}
|
|
1470
|
+
|
|
1471
|
+
_queueBackgroundProbe(connection, hostKey) {
|
|
1472
|
+
if (!this.relaySelection.enabled || !connection || !hostKey || !isConnected(connection)) {
|
|
1473
|
+
return;
|
|
1474
|
+
}
|
|
1475
|
+
if (this.pendingProbes.has(hostKey)) {
|
|
1476
|
+
return;
|
|
1477
|
+
}
|
|
1478
|
+
const lastProbeStartedAt = this._lastProbeStartedAt.get(hostKey) || 0;
|
|
1479
|
+
if (Date.now() - lastProbeStartedAt < this.relaySelection.backgroundProbeIntervalMs) {
|
|
1480
|
+
return;
|
|
1481
|
+
}
|
|
1482
|
+
|
|
1483
|
+
void this._probeHost(hostKey, this.relayScores.get(hostKey)?.discoveredFrom || 'seed')
|
|
1484
|
+
.catch((error) => {
|
|
1485
|
+
this._recordRelayProbeFailure(hostKey, error);
|
|
1486
|
+
});
|
|
1487
|
+
}
|
|
1488
|
+
|
|
1489
|
+
_getDeviceCacheEntry(deviceIdHex) {
|
|
1490
|
+
if (this.deviceCacheTtlMs <= 0) {
|
|
1491
|
+
return null;
|
|
1492
|
+
}
|
|
1493
|
+
const cached = this.deviceRelayCache.get(deviceIdHex);
|
|
1494
|
+
if (!cached) {
|
|
1495
|
+
return null;
|
|
1496
|
+
}
|
|
1497
|
+
const ttlMs = Number.isFinite(cached.ttlMs) ? cached.ttlMs : this.deviceCacheTtlMs;
|
|
1498
|
+
if (ttlMs <= 0 || Date.now() - cached.ts >= ttlMs) {
|
|
1499
|
+
this.deviceRelayCache.delete(deviceIdHex);
|
|
1500
|
+
return null;
|
|
1501
|
+
}
|
|
1502
|
+
return cached;
|
|
1503
|
+
}
|
|
1504
|
+
|
|
1505
|
+
_setDeviceCacheEntry(deviceIdHex, entry) {
|
|
1506
|
+
if (this.deviceCacheTtlMs <= 0 || !deviceIdHex || !entry) {
|
|
1507
|
+
return;
|
|
1508
|
+
}
|
|
1509
|
+
this.deviceRelayCache.set(deviceIdHex, {
|
|
1510
|
+
serverIdHex: entry.serverIdHex,
|
|
1511
|
+
hostKey: entry.hostKey,
|
|
1512
|
+
ts: Number.isFinite(entry.ts) ? entry.ts : Date.now(),
|
|
1513
|
+
ttlMs: Number.isFinite(entry.ttlMs) ? entry.ttlMs : this.deviceCacheTtlMs,
|
|
1514
|
+
});
|
|
1515
|
+
}
|
|
1516
|
+
|
|
1517
|
+
_getDeviceCacheTtlForHost(hostKey) {
|
|
1518
|
+
const score = this.relayScores.get(hostKey);
|
|
1519
|
+
if (score && Number.isFinite(score.ewmaLatencyMs) && score.ewmaLatencyMs >= this.relaySelection.slowRelayThresholdMs) {
|
|
1520
|
+
return this.relaySelection.slowDeviceRetryTtlMs;
|
|
1521
|
+
}
|
|
1522
|
+
return this.deviceCacheTtlMs;
|
|
1523
|
+
}
|
|
1524
|
+
|
|
1525
|
+
_getRelayLatencyMs(hostKey) {
|
|
1526
|
+
const score = this.relayScores.get(hostKey);
|
|
1527
|
+
return score && Number.isFinite(score.ewmaLatencyMs) ? score.ewmaLatencyMs : Number.POSITIVE_INFINITY;
|
|
1528
|
+
}
|
|
1529
|
+
|
|
273
1530
|
_connectedConnections() {
|
|
274
1531
|
return this.connections.filter((connection) => isConnected(connection));
|
|
275
1532
|
}
|
|
@@ -279,8 +1536,223 @@ class DiodeClientManager extends EventEmitter {
|
|
|
279
1536
|
if (connected.length === 0) {
|
|
280
1537
|
return null;
|
|
281
1538
|
}
|
|
282
|
-
|
|
283
|
-
|
|
1539
|
+
|
|
1540
|
+
if (!this.relaySelection.enabled) {
|
|
1541
|
+
this._rrIndex = (this._rrIndex + 1) % connected.length;
|
|
1542
|
+
return connected[this._rrIndex];
|
|
1543
|
+
}
|
|
1544
|
+
|
|
1545
|
+
return this._selectPreferredConnectedConnection();
|
|
1546
|
+
}
|
|
1547
|
+
|
|
1548
|
+
async _resolveDeviceRelayCandidate(connection, deviceIdBuffer) {
|
|
1549
|
+
const rpc = this._getRpcFor(connection);
|
|
1550
|
+
const ticket = await rpc.getObject(deviceIdBuffer);
|
|
1551
|
+
const serverIdHex = normalizeServerIdHex(ticket && (ticket.serverIdHex || ticket.serverId));
|
|
1552
|
+
if (!serverIdHex) {
|
|
1553
|
+
return null;
|
|
1554
|
+
}
|
|
1555
|
+
|
|
1556
|
+
const existing = this.serverIdToConnection.get(serverIdHex);
|
|
1557
|
+
if (existing && isConnected(existing)) {
|
|
1558
|
+
return {
|
|
1559
|
+
serverIdHex,
|
|
1560
|
+
hostKey: existing._managerHostKey || '',
|
|
1561
|
+
relayConnection: existing,
|
|
1562
|
+
controlConnection: connection,
|
|
1563
|
+
};
|
|
1564
|
+
}
|
|
1565
|
+
|
|
1566
|
+
const nodeId = Buffer.from(serverIdHex.slice(2), 'hex');
|
|
1567
|
+
const nodeInfo = await rpc.getNode(nodeId);
|
|
1568
|
+
if (!nodeInfo || !nodeInfo.host) {
|
|
1569
|
+
return null;
|
|
1570
|
+
}
|
|
1571
|
+
|
|
1572
|
+
const relayPort = nodeInfo.edgePort || nodeInfo.serverPort;
|
|
1573
|
+
if (!relayPort) {
|
|
1574
|
+
return null;
|
|
1575
|
+
}
|
|
1576
|
+
|
|
1577
|
+
return {
|
|
1578
|
+
serverIdHex,
|
|
1579
|
+
hostKey: joinHostPort(nodeInfo.host, relayPort),
|
|
1580
|
+
relayConnection: null,
|
|
1581
|
+
controlConnection: connection,
|
|
1582
|
+
};
|
|
1583
|
+
}
|
|
1584
|
+
|
|
1585
|
+
async _ensureDeviceRelayCandidateConnection(candidate) {
|
|
1586
|
+
if (!candidate || !candidate.hostKey) {
|
|
1587
|
+
return null;
|
|
1588
|
+
}
|
|
1589
|
+
if (candidate.relayConnection && isConnected(candidate.relayConnection)) {
|
|
1590
|
+
return candidate.relayConnection;
|
|
1591
|
+
}
|
|
1592
|
+
candidate.relayConnection = this.relaySelection.enabled
|
|
1593
|
+
? await this._probeHost(candidate.hostKey, 'target')
|
|
1594
|
+
: await this._ensureConnection(candidate.hostKey);
|
|
1595
|
+
return candidate.relayConnection;
|
|
1596
|
+
}
|
|
1597
|
+
|
|
1598
|
+
_shouldReconcileDeviceRelay(controlHostKey, targetHostKey) {
|
|
1599
|
+
if (!this.relaySelection.enabled || !controlHostKey || !targetHostKey) {
|
|
1600
|
+
return false;
|
|
1601
|
+
}
|
|
1602
|
+
|
|
1603
|
+
const reconciliation = this.relaySelection.deviceRelayReconciliation;
|
|
1604
|
+
if (!reconciliation.enabled || reconciliation.maxControlRelays <= 0) {
|
|
1605
|
+
return false;
|
|
1606
|
+
}
|
|
1607
|
+
|
|
1608
|
+
const targetLatencyMs = this._getRelayLatencyMs(targetHostKey);
|
|
1609
|
+
if (!Number.isFinite(targetLatencyMs) || targetLatencyMs < this.relaySelection.slowRelayThresholdMs) {
|
|
1610
|
+
return false;
|
|
1611
|
+
}
|
|
1612
|
+
|
|
1613
|
+
const controlLatencyMs = this._getRelayLatencyMs(controlHostKey);
|
|
1614
|
+
if (!Number.isFinite(controlLatencyMs)) {
|
|
1615
|
+
return true;
|
|
1616
|
+
}
|
|
1617
|
+
if (targetLatencyMs - controlLatencyMs < reconciliation.minLatencyDeltaMs) {
|
|
1618
|
+
return false;
|
|
1619
|
+
}
|
|
1620
|
+
return targetLatencyMs >= controlLatencyMs * reconciliation.slowdownFactor;
|
|
1621
|
+
}
|
|
1622
|
+
|
|
1623
|
+
async _withTimeout(promiseFactory, timeoutMs, label) {
|
|
1624
|
+
return Promise.race([
|
|
1625
|
+
Promise.resolve().then(() => promiseFactory()),
|
|
1626
|
+
new Promise((_, reject) => {
|
|
1627
|
+
setTimeout(() => reject(new Error(`${label} timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
1628
|
+
}),
|
|
1629
|
+
]);
|
|
1630
|
+
}
|
|
1631
|
+
|
|
1632
|
+
async _reconcileDeviceRelayCandidate(primaryConnection, deviceIdBuffer, initialCandidate, trace = null) {
|
|
1633
|
+
if (!initialCandidate || !initialCandidate.hostKey) {
|
|
1634
|
+
return initialCandidate;
|
|
1635
|
+
}
|
|
1636
|
+
|
|
1637
|
+
const primaryHostKey = primaryConnection && primaryConnection._managerHostKey
|
|
1638
|
+
? primaryConnection._managerHostKey
|
|
1639
|
+
: '';
|
|
1640
|
+
const shouldReconcile = this._shouldReconcileDeviceRelay(primaryHostKey, initialCandidate.hostKey);
|
|
1641
|
+
if (trace) {
|
|
1642
|
+
trace.reconciliation = trace.reconciliation || {
|
|
1643
|
+
triggered: false,
|
|
1644
|
+
attempted: false,
|
|
1645
|
+
hadAlternateAnswer: false,
|
|
1646
|
+
choseAlternate: false,
|
|
1647
|
+
alternateResults: [],
|
|
1648
|
+
};
|
|
1649
|
+
trace.reconciliation.triggered = shouldReconcile;
|
|
1650
|
+
}
|
|
1651
|
+
if (!shouldReconcile) {
|
|
1652
|
+
return initialCandidate;
|
|
1653
|
+
}
|
|
1654
|
+
|
|
1655
|
+
const reconciliation = this.relaySelection.deviceRelayReconciliation;
|
|
1656
|
+
const alternates = this._rankConnectedConnections(this._connectedConnections(), { queueRefresh: true })
|
|
1657
|
+
.filter((connection) => connection !== primaryConnection)
|
|
1658
|
+
.slice(0, reconciliation.maxControlRelays);
|
|
1659
|
+
if (trace) {
|
|
1660
|
+
trace.reconciliation.attempted = alternates.length > 0;
|
|
1661
|
+
}
|
|
1662
|
+
if (alternates.length === 0) {
|
|
1663
|
+
return initialCandidate;
|
|
1664
|
+
}
|
|
1665
|
+
|
|
1666
|
+
const attempts = await Promise.allSettled(alternates.map(async (connection) => {
|
|
1667
|
+
const startedAt = Date.now();
|
|
1668
|
+
try {
|
|
1669
|
+
const candidate = await this._withTimeout(
|
|
1670
|
+
async () => this._resolveDeviceRelayCandidate(connection, deviceIdBuffer),
|
|
1671
|
+
reconciliation.timeoutMs,
|
|
1672
|
+
`Device relay reconciliation via ${connection._managerHostKey || 'unknown relay'}`
|
|
1673
|
+
);
|
|
1674
|
+
return {
|
|
1675
|
+
ok: true,
|
|
1676
|
+
controlHostKey: connection._managerHostKey || '',
|
|
1677
|
+
lookupMs: Date.now() - startedAt,
|
|
1678
|
+
candidate,
|
|
1679
|
+
};
|
|
1680
|
+
} catch (error) {
|
|
1681
|
+
return {
|
|
1682
|
+
ok: false,
|
|
1683
|
+
controlHostKey: connection._managerHostKey || '',
|
|
1684
|
+
lookupMs: Date.now() - startedAt,
|
|
1685
|
+
error: String(error && error.message ? error.message : error),
|
|
1686
|
+
candidate: null,
|
|
1687
|
+
};
|
|
1688
|
+
}
|
|
1689
|
+
}));
|
|
1690
|
+
|
|
1691
|
+
let bestCandidate = initialCandidate;
|
|
1692
|
+
let bestLatencyMs = this._getRelayLatencyMs(initialCandidate.hostKey);
|
|
1693
|
+
for (const result of attempts) {
|
|
1694
|
+
if (result.status !== 'fulfilled' || !result.value) {
|
|
1695
|
+
continue;
|
|
1696
|
+
}
|
|
1697
|
+
const attempt = result.value;
|
|
1698
|
+
const candidate = attempt.candidate;
|
|
1699
|
+
const differentAnswer = !!(
|
|
1700
|
+
candidate
|
|
1701
|
+
&& candidate.hostKey
|
|
1702
|
+
&& (
|
|
1703
|
+
candidate.serverIdHex !== initialCandidate.serverIdHex
|
|
1704
|
+
|| candidate.hostKey !== initialCandidate.hostKey
|
|
1705
|
+
)
|
|
1706
|
+
);
|
|
1707
|
+
if (trace) {
|
|
1708
|
+
trace.reconciliation.alternateResults.push({
|
|
1709
|
+
controlHostKey: attempt.controlHostKey,
|
|
1710
|
+
lookupMs: attempt.lookupMs,
|
|
1711
|
+
ok: attempt.ok,
|
|
1712
|
+
error: attempt.error || null,
|
|
1713
|
+
serverIdHex: candidate ? candidate.serverIdHex : null,
|
|
1714
|
+
hostKey: candidate ? candidate.hostKey : null,
|
|
1715
|
+
differentAnswer,
|
|
1716
|
+
chosen: false,
|
|
1717
|
+
});
|
|
1718
|
+
if (differentAnswer) {
|
|
1719
|
+
trace.reconciliation.hadAlternateAnswer = true;
|
|
1720
|
+
}
|
|
1721
|
+
}
|
|
1722
|
+
if (!attempt.ok || !candidate || !candidate.hostKey) {
|
|
1723
|
+
continue;
|
|
1724
|
+
}
|
|
1725
|
+
if (candidate.serverIdHex === bestCandidate.serverIdHex && candidate.hostKey === bestCandidate.hostKey) {
|
|
1726
|
+
continue;
|
|
1727
|
+
}
|
|
1728
|
+
try {
|
|
1729
|
+
await this._ensureDeviceRelayCandidateConnection(candidate);
|
|
1730
|
+
} catch (error) {
|
|
1731
|
+
logger.debug(() => `Device relay reconciliation candidate failed for ${candidate.hostKey}: ${error}`);
|
|
1732
|
+
continue;
|
|
1733
|
+
}
|
|
1734
|
+
const candidateLatencyMs = this._getRelayLatencyMs(candidate.hostKey);
|
|
1735
|
+
if (!Number.isFinite(candidateLatencyMs)) {
|
|
1736
|
+
continue;
|
|
1737
|
+
}
|
|
1738
|
+
if (!Number.isFinite(bestLatencyMs) || candidateLatencyMs < bestLatencyMs) {
|
|
1739
|
+
bestCandidate = candidate;
|
|
1740
|
+
bestLatencyMs = candidateLatencyMs;
|
|
1741
|
+
}
|
|
1742
|
+
}
|
|
1743
|
+
|
|
1744
|
+
if (trace && trace.reconciliation && bestCandidate !== initialCandidate) {
|
|
1745
|
+
trace.reconciliation.choseAlternate = true;
|
|
1746
|
+
const chosen = trace.reconciliation.alternateResults.find((entry) => (
|
|
1747
|
+
entry.serverIdHex === bestCandidate.serverIdHex
|
|
1748
|
+
&& entry.hostKey === bestCandidate.hostKey
|
|
1749
|
+
));
|
|
1750
|
+
if (chosen) {
|
|
1751
|
+
chosen.chosen = true;
|
|
1752
|
+
}
|
|
1753
|
+
}
|
|
1754
|
+
|
|
1755
|
+
return bestCandidate;
|
|
284
1756
|
}
|
|
285
1757
|
|
|
286
1758
|
async connect() {
|
|
@@ -288,18 +1760,128 @@ class DiodeClientManager extends EventEmitter {
|
|
|
288
1760
|
throw new Error('No Diode hosts configured');
|
|
289
1761
|
}
|
|
290
1762
|
|
|
291
|
-
|
|
292
|
-
|
|
1763
|
+
if (!this.relaySelection.enabled) {
|
|
1764
|
+
const results = await Promise.allSettled(
|
|
1765
|
+
this.initialHosts.map((host) => this._ensureConnection(host))
|
|
1766
|
+
);
|
|
1767
|
+
|
|
1768
|
+
const success = results.some((result) => result.status === 'fulfilled');
|
|
1769
|
+
if (!success) {
|
|
1770
|
+
const errorMessages = results
|
|
1771
|
+
.filter((result) => result.status === 'rejected')
|
|
1772
|
+
.map((result) => result.reason && result.reason.message ? result.reason.message : String(result.reason));
|
|
1773
|
+
throw new Error(`Failed to connect to any Diode hosts. ${errorMessages.join('; ')}`);
|
|
1774
|
+
}
|
|
1775
|
+
|
|
1776
|
+
return this;
|
|
1777
|
+
}
|
|
1778
|
+
|
|
1779
|
+
const initialSource = this._hasExplicitHost || this._hasExplicitHosts ? 'configured' : 'seed';
|
|
1780
|
+
const initialCandidates = this.initialHosts.map((hostKey, index) => (
|
|
1781
|
+
this._createCandidate(hostKey, initialSource, index)
|
|
1782
|
+
));
|
|
1783
|
+
|
|
1784
|
+
if (!this.relaySelection.probeAllInitialCandidates) {
|
|
1785
|
+
const candidates = await this._buildStartupCandidates();
|
|
1786
|
+
const requiredCoverageCandidates = this._rankRelayCandidates(candidates).slice(0, this.relaySelection.minReadyConnections);
|
|
1787
|
+
const results = await runWithConcurrency(
|
|
1788
|
+
requiredCoverageCandidates,
|
|
1789
|
+
this.relaySelection.startupConcurrency,
|
|
1790
|
+
(candidate) => this._probeHost(candidate.hostKey, candidate.source)
|
|
1791
|
+
);
|
|
1792
|
+
|
|
1793
|
+
const successes = results.filter((result) => result.status === 'fulfilled');
|
|
1794
|
+
if (successes.length === 0) {
|
|
1795
|
+
const errorMessages = results
|
|
1796
|
+
.filter((result) => result.status === 'rejected')
|
|
1797
|
+
.map((result) => result.reason && result.reason.message ? result.reason.message : String(result.reason));
|
|
1798
|
+
throw new Error(`Failed to connect to any Diode hosts. ${errorMessages.join('; ')}`);
|
|
1799
|
+
}
|
|
1800
|
+
|
|
1801
|
+
this._startupCoverageComplete = true;
|
|
1802
|
+
this._pruneIdleConnections();
|
|
1803
|
+
return this;
|
|
1804
|
+
}
|
|
1805
|
+
|
|
1806
|
+
const allRequiredCoverageCandidates = this._getInitialCoverageCandidates(initialCandidates);
|
|
1807
|
+
const bootstrapSeedCandidates = this._getStartupSeedBootstrapCandidates(initialCandidates);
|
|
1808
|
+
const bootstrapCoveragePromise = runWithConcurrency(
|
|
1809
|
+
bootstrapSeedCandidates,
|
|
1810
|
+
this.relaySelection.startupConcurrency,
|
|
1811
|
+
(candidate) => this._probeHost(candidate.hostKey, candidate.source)
|
|
1812
|
+
);
|
|
1813
|
+
const candidatesPromise = this._buildStartupCandidates();
|
|
1814
|
+
const [bootstrapResults, candidates] = await Promise.all([bootstrapCoveragePromise, candidatesPromise]);
|
|
1815
|
+
|
|
1816
|
+
const startupNetworkCandidates = this._selectStartupNetworkCandidates(candidates);
|
|
1817
|
+
const useReducedSeedCoverage = (
|
|
1818
|
+
!this._hasExplicitHost
|
|
1819
|
+
&& !this._hasExplicitHosts
|
|
1820
|
+
&& this.relaySelection.networkDiscovery.enabled
|
|
1821
|
+
&& startupNetworkCandidates.length > 0
|
|
1822
|
+
);
|
|
1823
|
+
const requiredCoverageCandidates = useReducedSeedCoverage
|
|
1824
|
+
? bootstrapSeedCandidates
|
|
1825
|
+
: allRequiredCoverageCandidates;
|
|
1826
|
+
const bootstrapCoverageHostKeys = new Set(bootstrapSeedCandidates.map((candidate) => candidate.hostKey));
|
|
1827
|
+
const remainingRequiredCoverageCandidates = useReducedSeedCoverage
|
|
1828
|
+
? []
|
|
1829
|
+
: allRequiredCoverageCandidates.filter((candidate) => !bootstrapCoverageHostKeys.has(candidate.hostKey));
|
|
1830
|
+
const initialCoverageCandidates = [];
|
|
1831
|
+
const coverageHostKeys = new Set();
|
|
1832
|
+
[...requiredCoverageCandidates, ...startupNetworkCandidates].forEach((candidate) => {
|
|
1833
|
+
if (!coverageHostKeys.has(candidate.hostKey)) {
|
|
1834
|
+
coverageHostKeys.add(candidate.hostKey);
|
|
1835
|
+
initialCoverageCandidates.push(candidate);
|
|
1836
|
+
}
|
|
1837
|
+
});
|
|
1838
|
+
|
|
1839
|
+
const remainingCandidates = this._rankRelayCandidates(
|
|
1840
|
+
candidates.filter((candidate) => !coverageHostKeys.has(candidate.hostKey))
|
|
293
1841
|
);
|
|
1842
|
+
const backgroundNetworkCandidates = this._selectBackgroundNetworkCandidates(remainingCandidates);
|
|
1843
|
+
const backgroundCandidates = [
|
|
1844
|
+
...backgroundNetworkCandidates,
|
|
1845
|
+
...remainingCandidates.filter((candidate) => candidate.source !== 'network'),
|
|
1846
|
+
];
|
|
1847
|
+
|
|
1848
|
+
const additionalRequiredResults = remainingRequiredCoverageCandidates.length > 0
|
|
1849
|
+
? await runWithConcurrency(
|
|
1850
|
+
remainingRequiredCoverageCandidates,
|
|
1851
|
+
this.relaySelection.startupConcurrency,
|
|
1852
|
+
(candidate) => this._probeHost(candidate.hostKey, candidate.source)
|
|
1853
|
+
)
|
|
1854
|
+
: [];
|
|
1855
|
+
const networkResults = startupNetworkCandidates.length > 0
|
|
1856
|
+
? await runWithConcurrency(
|
|
1857
|
+
startupNetworkCandidates,
|
|
1858
|
+
this.relaySelection.startupConcurrency,
|
|
1859
|
+
(candidate) => this._probeHost(candidate.hostKey, candidate.source)
|
|
1860
|
+
)
|
|
1861
|
+
: [];
|
|
1862
|
+
const results = [...bootstrapResults, ...additionalRequiredResults, ...networkResults];
|
|
294
1863
|
|
|
295
|
-
const
|
|
296
|
-
if (
|
|
1864
|
+
const successes = results.filter((result) => result.status === 'fulfilled');
|
|
1865
|
+
if (successes.length === 0) {
|
|
297
1866
|
const errorMessages = results
|
|
298
1867
|
.filter((result) => result.status === 'rejected')
|
|
299
1868
|
.map((result) => result.reason && result.reason.message ? result.reason.message : String(result.reason));
|
|
300
1869
|
throw new Error(`Failed to connect to any Diode hosts. ${errorMessages.join('; ')}`);
|
|
301
1870
|
}
|
|
302
1871
|
|
|
1872
|
+
this._startupCoverageComplete = true;
|
|
1873
|
+
this._pruneIdleConnections();
|
|
1874
|
+
|
|
1875
|
+
if (backgroundCandidates.length > 0 && this.relaySelection.continueProbingUntestedSeeds) {
|
|
1876
|
+
this._backgroundWarmupPromise = runWithConcurrency(
|
|
1877
|
+
backgroundCandidates,
|
|
1878
|
+
this.relaySelection.startupConcurrency,
|
|
1879
|
+
(candidate) => this._probeHost(candidate.hostKey, candidate.source)
|
|
1880
|
+
).catch((error) => {
|
|
1881
|
+
logger.debug(() => `Background relay measurement failed: ${error}`);
|
|
1882
|
+
});
|
|
1883
|
+
}
|
|
1884
|
+
|
|
303
1885
|
return this;
|
|
304
1886
|
}
|
|
305
1887
|
|
|
@@ -309,15 +1891,39 @@ class DiodeClientManager extends EventEmitter {
|
|
|
309
1891
|
throw new Error('Invalid device ID');
|
|
310
1892
|
}
|
|
311
1893
|
const deviceIdHex = deviceIdBuffer.toString('hex');
|
|
1894
|
+
const trace = {
|
|
1895
|
+
deviceId: `0x${deviceIdHex}`,
|
|
1896
|
+
primaryControlHostKey: null,
|
|
1897
|
+
primaryLookupMs: null,
|
|
1898
|
+
controlPlaneSlowThresholdMs: Math.max(
|
|
1899
|
+
this.relaySelection.probeTimeoutMs,
|
|
1900
|
+
this.relaySelection.deviceRelayReconciliation.timeoutMs,
|
|
1901
|
+
),
|
|
1902
|
+
controlPlaneSlow: false,
|
|
1903
|
+
initialServerIdHex: null,
|
|
1904
|
+
initialHostKey: null,
|
|
1905
|
+
initialConnectMs: null,
|
|
1906
|
+
finalServerIdHex: null,
|
|
1907
|
+
finalHostKey: null,
|
|
1908
|
+
reconciliation: {
|
|
1909
|
+
triggered: false,
|
|
1910
|
+
attempted: false,
|
|
1911
|
+
hadAlternateAnswer: false,
|
|
1912
|
+
choseAlternate: false,
|
|
1913
|
+
alternateResults: [],
|
|
1914
|
+
},
|
|
1915
|
+
};
|
|
1916
|
+
this._lastDeviceResolutionTrace = trace;
|
|
312
1917
|
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
1918
|
+
const cached = this._getDeviceCacheEntry(deviceIdHex);
|
|
1919
|
+
if (cached) {
|
|
1920
|
+
const cachedConn = this.serverIdToConnection.get(cached.serverIdHex)
|
|
1921
|
+
|| this.connectionByHost.get(cached.hostKey);
|
|
1922
|
+
if (cachedConn && isConnected(cachedConn)) {
|
|
1923
|
+
trace.cacheHit = true;
|
|
1924
|
+
trace.finalServerIdHex = cached.serverIdHex;
|
|
1925
|
+
trace.finalHostKey = cached.hostKey;
|
|
1926
|
+
return cachedConn;
|
|
321
1927
|
}
|
|
322
1928
|
}
|
|
323
1929
|
|
|
@@ -325,61 +1931,58 @@ class DiodeClientManager extends EventEmitter {
|
|
|
325
1931
|
if (!primary) {
|
|
326
1932
|
throw new Error('No connected relay available');
|
|
327
1933
|
}
|
|
1934
|
+
trace.primaryControlHostKey = primary._managerHostKey || null;
|
|
328
1935
|
|
|
329
|
-
let
|
|
1936
|
+
let candidate = null;
|
|
330
1937
|
try {
|
|
331
|
-
|
|
1938
|
+
const startedAt = Date.now();
|
|
1939
|
+
candidate = await this._resolveDeviceRelayCandidate(primary, deviceIdBuffer);
|
|
1940
|
+
trace.primaryLookupMs = Date.now() - startedAt;
|
|
1941
|
+
trace.controlPlaneSlow = trace.primaryLookupMs > trace.controlPlaneSlowThresholdMs;
|
|
332
1942
|
} catch (error) {
|
|
333
1943
|
logger.warn(() => `Failed to resolve device ticket: ${error}`);
|
|
1944
|
+
trace.error = String(error && error.message ? error.message : error);
|
|
334
1945
|
return primary;
|
|
335
1946
|
}
|
|
336
|
-
|
|
337
|
-
const serverIdHex = normalizeServerIdHex(ticket && (ticket.serverIdHex || ticket.serverId));
|
|
338
|
-
if (!serverIdHex) {
|
|
1947
|
+
if (!candidate || !candidate.serverIdHex) {
|
|
339
1948
|
return primary;
|
|
340
1949
|
}
|
|
1950
|
+
trace.initialServerIdHex = candidate.serverIdHex;
|
|
1951
|
+
trace.initialHostKey = candidate.hostKey || null;
|
|
341
1952
|
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
this.
|
|
345
|
-
serverIdHex,
|
|
346
|
-
hostKey
|
|
1953
|
+
if (candidate.relayConnection && isConnected(candidate.relayConnection)) {
|
|
1954
|
+
const hostKey = candidate.hostKey || candidate.relayConnection._managerHostKey || '';
|
|
1955
|
+
this._setDeviceCacheEntry(deviceIdHex, {
|
|
1956
|
+
serverIdHex: candidate.serverIdHex,
|
|
1957
|
+
hostKey,
|
|
347
1958
|
ts: Date.now(),
|
|
1959
|
+
ttlMs: this._getDeviceCacheTtlForHost(hostKey),
|
|
348
1960
|
});
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
let nodeInfo = null;
|
|
353
|
-
try {
|
|
354
|
-
const nodeId = Buffer.from(serverIdHex.slice(2), 'hex');
|
|
355
|
-
nodeInfo = await this._getRpcFor(primary).getNode(nodeId);
|
|
356
|
-
} catch (error) {
|
|
357
|
-
logger.warn(() => `Failed to resolve relay node for ${serverIdHex}: ${error}`);
|
|
358
|
-
return primary;
|
|
359
|
-
}
|
|
360
|
-
|
|
361
|
-
if (!nodeInfo || !nodeInfo.host) {
|
|
362
|
-
return primary;
|
|
363
|
-
}
|
|
364
|
-
|
|
365
|
-
const relayPort = nodeInfo.edgePort || nodeInfo.serverPort;
|
|
366
|
-
if (!relayPort) {
|
|
367
|
-
return primary;
|
|
1961
|
+
trace.finalServerIdHex = candidate.serverIdHex;
|
|
1962
|
+
trace.finalHostKey = hostKey;
|
|
1963
|
+
return candidate.relayConnection;
|
|
368
1964
|
}
|
|
369
1965
|
|
|
370
|
-
const hostKey = joinHostPort(nodeInfo.host, relayPort);
|
|
371
|
-
let relayConnection;
|
|
372
1966
|
try {
|
|
373
|
-
|
|
1967
|
+
const startedAt = Date.now();
|
|
1968
|
+
await this._ensureDeviceRelayCandidateConnection(candidate);
|
|
1969
|
+
trace.initialConnectMs = Date.now() - startedAt;
|
|
374
1970
|
} catch (error) {
|
|
375
|
-
logger.warn(() => `Failed to connect to relay ${hostKey}: ${error}`);
|
|
1971
|
+
logger.warn(() => `Failed to connect to relay ${candidate.hostKey}: ${error}`);
|
|
1972
|
+
trace.error = String(error && error.message ? error.message : error);
|
|
376
1973
|
return primary;
|
|
377
1974
|
}
|
|
378
1975
|
|
|
379
|
-
this.
|
|
380
|
-
|
|
1976
|
+
candidate = await this._reconcileDeviceRelayCandidate(primary, deviceIdBuffer, candidate, trace);
|
|
1977
|
+
const hostKey = candidate.hostKey || '';
|
|
1978
|
+
const relayConnection = candidate.relayConnection || this.connectionByHost.get(hostKey) || primary;
|
|
1979
|
+
trace.finalServerIdHex = candidate.serverIdHex;
|
|
1980
|
+
trace.finalHostKey = hostKey;
|
|
1981
|
+
this._setDeviceCacheEntry(deviceIdHex, {
|
|
1982
|
+
serverIdHex: candidate.serverIdHex,
|
|
381
1983
|
hostKey,
|
|
382
1984
|
ts: Date.now(),
|
|
1985
|
+
ttlMs: this._getDeviceCacheTtlForHost(hostKey),
|
|
383
1986
|
});
|
|
384
1987
|
|
|
385
1988
|
return relayConnection || primary;
|
|
@@ -396,26 +1999,21 @@ class DiodeClientManager extends EventEmitter {
|
|
|
396
1999
|
throw new Error('No connected relay available');
|
|
397
2000
|
}
|
|
398
2001
|
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
if (!serverIdHex) {
|
|
2002
|
+
let candidate = await this._resolveDeviceRelayCandidate(primary, deviceIdBuffer);
|
|
2003
|
+
if (!candidate || !candidate.serverIdHex || !candidate.hostKey) {
|
|
402
2004
|
throw new Error('Device ticket missing server ID');
|
|
403
2005
|
}
|
|
404
2006
|
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
}
|
|
410
|
-
const relayPort = nodeInfo.edgePort || nodeInfo.serverPort;
|
|
411
|
-
if (!relayPort) {
|
|
412
|
-
throw new Error('Relay node info missing port');
|
|
413
|
-
}
|
|
2007
|
+
try {
|
|
2008
|
+
await this._ensureDeviceRelayCandidateConnection(candidate);
|
|
2009
|
+
} catch (_) {}
|
|
2010
|
+
candidate = await this._reconcileDeviceRelayCandidate(primary, deviceIdBuffer, candidate);
|
|
2011
|
+
const { host, port } = splitHostPort(candidate.hostKey, this.defaultPort);
|
|
414
2012
|
|
|
415
2013
|
return {
|
|
416
|
-
serverId: serverIdHex,
|
|
417
|
-
host
|
|
418
|
-
port
|
|
2014
|
+
serverId: candidate.serverIdHex,
|
|
2015
|
+
host,
|
|
2016
|
+
port,
|
|
419
2017
|
};
|
|
420
2018
|
}
|
|
421
2019
|
|
|
@@ -424,10 +2022,13 @@ class DiodeClientManager extends EventEmitter {
|
|
|
424
2022
|
}
|
|
425
2023
|
|
|
426
2024
|
close() {
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
2025
|
+
if (this._relayScoreFlushTimer) {
|
|
2026
|
+
clearTimeout(this._relayScoreFlushTimer);
|
|
2027
|
+
this._relayScoreFlushTimer = null;
|
|
2028
|
+
}
|
|
2029
|
+
this._flushRelayScores();
|
|
2030
|
+
for (const connection of this.connections.slice()) {
|
|
2031
|
+
this._closeManagedConnection(connection);
|
|
431
2032
|
}
|
|
432
2033
|
}
|
|
433
2034
|
}
|