@olane/o-node 0.7.11 → 0.7.12-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +34 -0
- package/dist/src/connection/o-node-connection.d.ts.map +1 -1
- package/dist/src/connection/o-node-connection.js +8 -2
- package/dist/src/connection/o-node-connection.manager.d.ts +1 -1
- package/dist/src/connection/o-node-connection.manager.d.ts.map +1 -1
- package/dist/src/connection/o-node-connection.manager.js +39 -19
- package/dist/src/interfaces/i-reconnectable-node.d.ts +45 -0
- package/dist/src/interfaces/i-reconnectable-node.d.ts.map +1 -0
- package/dist/src/interfaces/i-reconnectable-node.js +1 -0
- package/dist/src/interfaces/o-node.config.d.ts +37 -0
- package/dist/src/interfaces/o-node.config.d.ts.map +1 -1
- package/dist/src/managers/o-connection-heartbeat.manager.d.ts +67 -0
- package/dist/src/managers/o-connection-heartbeat.manager.d.ts.map +1 -0
- package/dist/src/managers/o-connection-heartbeat.manager.js +224 -0
- package/dist/src/managers/o-reconnection.manager.d.ts +50 -0
- package/dist/src/managers/o-reconnection.manager.d.ts.map +1 -0
- package/dist/src/managers/o-reconnection.manager.js +255 -0
- package/dist/src/o-node.d.ts +18 -1
- package/dist/src/o-node.d.ts.map +1 -1
- package/dist/src/o-node.js +85 -0
- package/dist/src/o-node.notification-manager.d.ts +52 -0
- package/dist/src/o-node.notification-manager.d.ts.map +1 -0
- package/dist/src/o-node.notification-manager.js +185 -0
- package/dist/src/o-node.tool.d.ts.map +1 -1
- package/dist/src/o-node.tool.js +19 -4
- package/dist/src/router/resolvers/o-node.search-resolver.d.ts.map +1 -1
- package/dist/src/router/resolvers/o-node.search-resolver.js +6 -2
- package/dist/src/utils/leader-request-wrapper.d.ts +45 -0
- package/dist/src/utils/leader-request-wrapper.d.ts.map +1 -0
- package/dist/src/utils/leader-request-wrapper.js +89 -0
- package/dist/test/search-resolver.spec.d.ts +2 -0
- package/dist/test/search-resolver.spec.d.ts.map +1 -0
- package/dist/test/search-resolver.spec.js +693 -0
- package/package.json +7 -10
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
import { oObject, oAddress, NodeState, } from '@olane/o-core';
|
|
2
|
+
import { oNodeAddress } from '../router/o-node.address.js';
|
|
3
|
+
import { oNodeTransport } from '../router/o-node.transport.js';
|
|
4
|
+
/**
|
|
5
|
+
* Reconnection Manager
|
|
6
|
+
*
|
|
7
|
+
* Automatically attempts to reconnect to parent when connection is lost.
|
|
8
|
+
*
|
|
9
|
+
* Strategy:
|
|
10
|
+
* 1. Listen for ParentDisconnectedEvent (from heartbeat or libp2p)
|
|
11
|
+
* 2. Attempt direct reconnection with exponential backoff
|
|
12
|
+
* 3. If direct reconnection fails, query leader for new parent
|
|
13
|
+
* 4. Register with new parent and continue operation
|
|
14
|
+
* 5. If all attempts fail, transition node to ERROR state
|
|
15
|
+
*/
|
|
16
|
+
export class oReconnectionManager extends oObject {
|
|
17
|
+
constructor(node, config) {
|
|
18
|
+
super();
|
|
19
|
+
this.node = node;
|
|
20
|
+
this.config = config;
|
|
21
|
+
this.reconnecting = false;
|
|
22
|
+
this.setupEventListeners();
|
|
23
|
+
}
|
|
24
|
+
setupEventListeners() {
|
|
25
|
+
// Listen for parent disconnection (from heartbeat or libp2p)
|
|
26
|
+
this.node.notificationManager.on('parent:disconnected', this.handleParentDisconnected.bind(this));
|
|
27
|
+
// Listen for leader disconnection (from heartbeat)
|
|
28
|
+
this.node.notificationManager.on('leader:disconnected', this.handleLeaderDisconnected.bind(this));
|
|
29
|
+
// Listen for connection degradation as early warning
|
|
30
|
+
this.node.notificationManager.on('connection:degraded', this.handleConnectionDegraded.bind(this));
|
|
31
|
+
}
|
|
32
|
+
async handleConnectionDegraded(event) {
|
|
33
|
+
const degradedEvent = event;
|
|
34
|
+
if (degradedEvent.role !== 'parent')
|
|
35
|
+
return;
|
|
36
|
+
this.logger.warn(`Parent connection degraded: ${degradedEvent.targetAddress} ` +
|
|
37
|
+
`(failures: ${degradedEvent.consecutiveFailures})`);
|
|
38
|
+
// Could implement pre-emptive parent discovery here
|
|
39
|
+
// For now, just log the warning and wait for full disconnection
|
|
40
|
+
}
|
|
41
|
+
async handleLeaderDisconnected(event) {
|
|
42
|
+
const disconnectEvent = event;
|
|
43
|
+
this.logger.warn(`Leader disconnected: ${disconnectEvent.leaderAddress} (reason: ${disconnectEvent.reason})`);
|
|
44
|
+
// Don't attempt reconnection for leader - the LeaderRequestWrapper
|
|
45
|
+
// will handle retries automatically when we make requests
|
|
46
|
+
// Just log the event for observability
|
|
47
|
+
this.logger.info('Leader requests will use automatic retry mechanism (LeaderRequestWrapper)');
|
|
48
|
+
}
|
|
49
|
+
async handleParentDisconnected(event) {
|
|
50
|
+
const disconnectEvent = event;
|
|
51
|
+
if (this.reconnecting) {
|
|
52
|
+
this.logger.debug('Already reconnecting, ignoring duplicate event');
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
this.logger.warn(`Parent disconnected: ${disconnectEvent.parentAddress} (reason: ${disconnectEvent.reason})`);
|
|
56
|
+
await this.attemptReconnection();
|
|
57
|
+
}
|
|
58
|
+
async attemptReconnection() {
|
|
59
|
+
if (!this.config.enabled) {
|
|
60
|
+
this.logger.warn('Reconnection disabled - node will remain disconnected');
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
this.reconnecting = true;
|
|
64
|
+
let attempt = 0;
|
|
65
|
+
while (attempt < this.config.maxAttempts) {
|
|
66
|
+
attempt++;
|
|
67
|
+
this.logger.info(`Reconnection attempt ${attempt}/${this.config.maxAttempts} to parent: ${this.node.config.parent}`);
|
|
68
|
+
try {
|
|
69
|
+
// Strategy 1: Try direct parent reconnection
|
|
70
|
+
await this.tryDirectParentReconnection();
|
|
71
|
+
// Success!
|
|
72
|
+
this.reconnecting = false;
|
|
73
|
+
this.logger.info(`Successfully reconnected to parent after ${attempt} attempts`);
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
catch (error) {
|
|
77
|
+
this.logger.warn(`Reconnection attempt ${attempt} failed:`, error instanceof Error ? error.message : error);
|
|
78
|
+
if (attempt < this.config.maxAttempts) {
|
|
79
|
+
const delay = this.calculateBackoffDelay(attempt);
|
|
80
|
+
this.logger.debug(`Waiting ${delay}ms before next attempt...`);
|
|
81
|
+
await this.sleep(delay);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
// All direct attempts failed - try leader fallback
|
|
86
|
+
if (this.config.useLeaderFallback) {
|
|
87
|
+
await this.tryLeaderFallback();
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
this.handleReconnectionFailure();
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
async tryDirectParentReconnection() {
|
|
94
|
+
if (!this.node.config.parent) {
|
|
95
|
+
throw new Error('No parent configured');
|
|
96
|
+
}
|
|
97
|
+
// Re-register with parent (might have new transports)
|
|
98
|
+
await this.node.registerParent();
|
|
99
|
+
// Verify connection works with a ping
|
|
100
|
+
await this.node.use(this.node.config.parent, {
|
|
101
|
+
method: 'ping',
|
|
102
|
+
params: {},
|
|
103
|
+
});
|
|
104
|
+
this.logger.info('Direct parent reconnection successful');
|
|
105
|
+
}
|
|
106
|
+
async tryLeaderFallback() {
|
|
107
|
+
// Check if parent is the leader - special case
|
|
108
|
+
const parentIsLeader = this.node.config.parent?.toString() === oAddress.leader().toString();
|
|
109
|
+
if (parentIsLeader) {
|
|
110
|
+
this.logger.info('Parent is the leader - waiting for leader to become available');
|
|
111
|
+
await this.waitForLeaderAndReconnect();
|
|
112
|
+
}
|
|
113
|
+
else {
|
|
114
|
+
this.logger.info('Starting infinite parent discovery via leader registry');
|
|
115
|
+
await this.waitForParentAndReconnect();
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Wait for leader to become available and reconnect
|
|
120
|
+
* Leader transports are static (configured), so we just need to detect when it's back
|
|
121
|
+
*/
|
|
122
|
+
async waitForLeaderAndReconnect() {
|
|
123
|
+
const startTime = Date.now();
|
|
124
|
+
let attempt = 0;
|
|
125
|
+
let currentDelay = this.config.parentDiscoveryIntervalMs;
|
|
126
|
+
// Infinite retry loop - keep trying until leader is back
|
|
127
|
+
while (true) {
|
|
128
|
+
attempt++;
|
|
129
|
+
const elapsedMinutes = Math.floor((Date.now() - startTime) / 60000);
|
|
130
|
+
this.logger.info(`Leader discovery attempt ${attempt} (elapsed: ${elapsedMinutes}m)`);
|
|
131
|
+
try {
|
|
132
|
+
// Try to ping the leader using its configured transports
|
|
133
|
+
// The leader address should already have transports configured
|
|
134
|
+
await this.node.use(this.node.config.parent, {
|
|
135
|
+
method: 'ping',
|
|
136
|
+
params: {},
|
|
137
|
+
});
|
|
138
|
+
// Leader is back! Now re-register with parent and registry
|
|
139
|
+
this.logger.info(`Leader is back online after ${elapsedMinutes}m, re-registering...`);
|
|
140
|
+
try {
|
|
141
|
+
// Register with parent (leader)
|
|
142
|
+
await this.node.registerParent();
|
|
143
|
+
// Force re-registration with registry by resetting the flag
|
|
144
|
+
this.node.didRegister = false;
|
|
145
|
+
await this.node.register();
|
|
146
|
+
// Success!
|
|
147
|
+
this.reconnecting = false;
|
|
148
|
+
this.logger.info(`Successfully reconnected to leader and re-registered after ${elapsedMinutes}m`);
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
151
|
+
catch (registrationError) {
|
|
152
|
+
this.logger.warn('Leader online but registration failed, will retry:', registrationError instanceof Error
|
|
153
|
+
? registrationError.message
|
|
154
|
+
: registrationError);
|
|
155
|
+
// Fall through to retry with backoff
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
catch (error) {
|
|
159
|
+
// Leader not yet available
|
|
160
|
+
this.logger.debug(`Leader not yet available (will retry): ${error instanceof Error ? error.message : error}`);
|
|
161
|
+
}
|
|
162
|
+
// Calculate backoff delay
|
|
163
|
+
const delay = Math.min(currentDelay, this.config.parentDiscoveryMaxDelayMs);
|
|
164
|
+
// Log periodic status updates (every 5 minutes)
|
|
165
|
+
if (attempt % 30 === 0) {
|
|
166
|
+
this.logger.info(`Still waiting for leader to come back online... (${elapsedMinutes}m elapsed, ${attempt} attempts)`);
|
|
167
|
+
}
|
|
168
|
+
this.logger.debug(`Waiting ${delay}ms before next discovery attempt...`);
|
|
169
|
+
await this.sleep(delay);
|
|
170
|
+
// Exponential backoff for next iteration
|
|
171
|
+
currentDelay = Math.min(currentDelay * 2, this.config.parentDiscoveryMaxDelayMs);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* Wait for non-leader parent to appear in registry and reconnect
|
|
176
|
+
*/
|
|
177
|
+
async waitForParentAndReconnect() {
|
|
178
|
+
const startTime = Date.now();
|
|
179
|
+
let attempt = 0;
|
|
180
|
+
let currentDelay = this.config.parentDiscoveryIntervalMs;
|
|
181
|
+
// Infinite retry loop - keep trying until parent is found
|
|
182
|
+
while (true) {
|
|
183
|
+
attempt++;
|
|
184
|
+
const elapsedMinutes = Math.floor((Date.now() - startTime) / 60000);
|
|
185
|
+
this.logger.info(`Parent discovery attempt ${attempt} (elapsed: ${elapsedMinutes}m)`);
|
|
186
|
+
try {
|
|
187
|
+
// Query registry for parent by its known address
|
|
188
|
+
const response = await this.node.use(new oAddress('o://registry'), {
|
|
189
|
+
method: 'find_available_parent',
|
|
190
|
+
params: {
|
|
191
|
+
parentAddress: this.node.config.parent?.toString(),
|
|
192
|
+
},
|
|
193
|
+
});
|
|
194
|
+
const { parentAddress, parentTransports } = response.result.data;
|
|
195
|
+
// Check if parent was found in registry
|
|
196
|
+
if (parentAddress && parentTransports && parentTransports.length > 0) {
|
|
197
|
+
this.logger.info(`Parent found in registry: ${parentAddress} with ${parentTransports.length} transports`);
|
|
198
|
+
// Update parent reference with fresh transports
|
|
199
|
+
this.node.config.parent = new oNodeAddress(parentAddress, parentTransports.map((t) => new oNodeTransport(t)));
|
|
200
|
+
// Attempt to register with parent and re-register with registry
|
|
201
|
+
try {
|
|
202
|
+
await this.tryDirectParentReconnection();
|
|
203
|
+
// Force re-registration with registry by resetting the flag
|
|
204
|
+
this.node.didRegister = false;
|
|
205
|
+
await this.node.register();
|
|
206
|
+
// Success!
|
|
207
|
+
this.reconnecting = false;
|
|
208
|
+
this.logger.info(`Successfully reconnected to parent and re-registered after ${elapsedMinutes}m of discovery attempts`);
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
catch (registrationError) {
|
|
212
|
+
this.logger.warn('Parent found but registration failed, will retry:', registrationError instanceof Error
|
|
213
|
+
? registrationError.message
|
|
214
|
+
: registrationError);
|
|
215
|
+
// Fall through to retry with backoff
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
else {
|
|
219
|
+
this.logger.debug(`Parent not yet available in registry: ${this.node.config.parent?.toString()}`);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
catch (error) {
|
|
223
|
+
// Network error communicating with leader/registry
|
|
224
|
+
this.logger.warn('Error querying registry for parent (will retry):', error instanceof Error ? error.message : error);
|
|
225
|
+
}
|
|
226
|
+
// Calculate backoff delay with exponential increase, capped at max
|
|
227
|
+
const delay = Math.min(currentDelay, this.config.parentDiscoveryMaxDelayMs);
|
|
228
|
+
// Log periodic status updates (every 5 minutes)
|
|
229
|
+
if (attempt % 30 === 0) {
|
|
230
|
+
this.logger.info(`Still waiting for parent to appear in registry... (${elapsedMinutes}m elapsed, ${attempt} attempts)`);
|
|
231
|
+
}
|
|
232
|
+
this.logger.debug(`Waiting ${delay}ms before next discovery attempt...`);
|
|
233
|
+
await this.sleep(delay);
|
|
234
|
+
// Exponential backoff for next iteration
|
|
235
|
+
currentDelay = Math.min(currentDelay * 2, this.config.parentDiscoveryMaxDelayMs);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
handleReconnectionFailure() {
|
|
239
|
+
this.reconnecting = false;
|
|
240
|
+
this.logger.error('Failed to reconnect to parent after all attempts - node in ERROR state');
|
|
241
|
+
// Transition to error state
|
|
242
|
+
this.node.state = NodeState.ERROR;
|
|
243
|
+
// Could emit custom event here for monitoring
|
|
244
|
+
}
|
|
245
|
+
calculateNodeLevel() {
|
|
246
|
+
return this.node.address.paths.length;
|
|
247
|
+
}
|
|
248
|
+
calculateBackoffDelay(attempt) {
|
|
249
|
+
const delay = this.config.baseDelayMs * Math.pow(2, attempt - 1);
|
|
250
|
+
return Math.min(delay, this.config.maxDelayMs);
|
|
251
|
+
}
|
|
252
|
+
sleep(ms) {
|
|
253
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
254
|
+
}
|
|
255
|
+
}
|
package/dist/src/o-node.d.ts
CHANGED
|
@@ -3,11 +3,14 @@ import { PeerId } from '@olane/o-config';
|
|
|
3
3
|
import { oNodeHierarchyManager } from './o-node.hierarchy-manager.js';
|
|
4
4
|
import { oNodeConfig } from './interfaces/o-node.config.js';
|
|
5
5
|
import { oNodeTransport } from './router/o-node.transport.js';
|
|
6
|
-
import { oRequest } from '@olane/o-core';
|
|
6
|
+
import { oAddress, oRequest, oNotificationManager } from '@olane/o-core';
|
|
7
7
|
import { oNodeAddress } from './router/o-node.address.js';
|
|
8
8
|
import { oNodeConnection } from './connection/o-node-connection.js';
|
|
9
9
|
import { oNodeConnectionManager } from './connection/o-node-connection.manager.js';
|
|
10
10
|
import { oToolBase } from '@olane/o-tool';
|
|
11
|
+
import { oConnectionHeartbeatManager } from './managers/o-connection-heartbeat.manager.js';
|
|
12
|
+
import { oReconnectionManager } from './managers/o-reconnection.manager.js';
|
|
13
|
+
import { LeaderRequestWrapper } from './utils/leader-request-wrapper.js';
|
|
11
14
|
export declare class oNode extends oToolBase {
|
|
12
15
|
peerId: PeerId;
|
|
13
16
|
p2pNode: Libp2p;
|
|
@@ -15,6 +18,9 @@ export declare class oNode extends oToolBase {
|
|
|
15
18
|
config: oNodeConfig;
|
|
16
19
|
connectionManager: oNodeConnectionManager;
|
|
17
20
|
hierarchyManager: oNodeHierarchyManager;
|
|
21
|
+
connectionHeartbeatManager?: oConnectionHeartbeatManager;
|
|
22
|
+
reconnectionManager?: oReconnectionManager;
|
|
23
|
+
leaderRequestWrapper: LeaderRequestWrapper;
|
|
18
24
|
protected didRegister: boolean;
|
|
19
25
|
constructor(config: oNodeConfig);
|
|
20
26
|
get leader(): oNodeAddress | null;
|
|
@@ -22,6 +28,7 @@ export declare class oNode extends oToolBase {
|
|
|
22
28
|
get parentPeerId(): string | null;
|
|
23
29
|
configureTransports(): any[];
|
|
24
30
|
initializeRouter(): Promise<void>;
|
|
31
|
+
protected createNotificationManager(): oNotificationManager;
|
|
25
32
|
get staticAddress(): oNodeAddress;
|
|
26
33
|
get parentTransports(): oNodeTransport[];
|
|
27
34
|
get transports(): oNodeTransport[];
|
|
@@ -39,6 +46,16 @@ export declare class oNode extends oToolBase {
|
|
|
39
46
|
protected createNode(): Promise<Libp2p>;
|
|
40
47
|
connect(nextHopAddress: oNodeAddress, targetAddress: oNodeAddress): Promise<oNodeConnection>;
|
|
41
48
|
initialize(): Promise<void>;
|
|
49
|
+
/**
|
|
50
|
+
* Override use() to wrap leader/registry requests with retry logic
|
|
51
|
+
*/
|
|
52
|
+
use(address: oAddress, data?: {
|
|
53
|
+
method?: string;
|
|
54
|
+
params?: {
|
|
55
|
+
[key: string]: any;
|
|
56
|
+
};
|
|
57
|
+
id?: string;
|
|
58
|
+
}): Promise<any>;
|
|
42
59
|
teardown(): Promise<void>;
|
|
43
60
|
}
|
|
44
61
|
//# sourceMappingURL=o-node.d.ts.map
|
package/dist/src/o-node.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"o-node.d.ts","sourceRoot":"","sources":["../../src/o-node.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,MAAM,EACN,YAAY,EACb,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AACtE,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,
|
|
1
|
+
{"version":3,"file":"o-node.d.ts","sourceRoot":"","sources":["../../src/o-node.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,MAAM,EACN,YAAY,EACb,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AACtE,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAIL,QAAQ,EACR,QAAQ,EAER,oBAAoB,EACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,mCAAmC,CAAC;AACpE,OAAO,EAAE,sBAAsB,EAAE,MAAM,2CAA2C,CAAC;AAGnF,OAAO,EAAmB,SAAS,EAAE,MAAM,eAAe,CAAC;AAI3D,OAAO,EAAE,2BAA2B,EAAE,MAAM,8CAA8C,CAAC;AAC3F,OAAO,EAAE,oBAAoB,EAAE,MAAM,sCAAsC,CAAC;AAC5E,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AAEzE,qBAAa,KAAM,SAAQ,SAAS;IAC3B,MAAM,EAAG,MAAM,CAAC;IAChB,OAAO,EAAG,MAAM,CAAC;IACjB,OAAO,EAAG,YAAY,CAAC;IACvB,MAAM,EAAE,WAAW,CAAC;IACpB,iBAAiB,EAAG,sBAAsB,CAAC;IAC3C,gBAAgB,EAAG,qBAAqB,CAAC;IACzC,0BAA0B,CAAC,EAAE,2BAA2B,CAAC;IACzD,mBAAmB,CAAC,EAAE,oBAAoB,CAAC;IAC3C,oBAAoB,EAAG,oBAAoB,CAAC;IACnD,SAAS,CAAC,WAAW,EAAE,OAAO,CAAS;gBAE3B,MAAM,EAAE,WAAW;IAK/B,IAAI,MAAM,IAAI,YAAY,GAAG,IAAI,CAEhC;IAED,IAAI,aAAa,IAAI,YAAY,CAKhC;IAED,IAAI,YAAY,IAAI,MAAM,GAAG,IAAI,CAOhC;IAED,mBAAmB,IAAI,GAAG,EAAE;IAItB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC;IASvC,SAAS,CAAC,yBAAyB,IAAI,oBAAoB;IAQ3D,IAAI,aAAa,IAAI,YAAY,CAEhC;IAED,IAAI,gBAAgB,IAAI,cAAc,EAAE,CAEvC;IAED,IAAI,UAAU,IAAI,cAAc,EAAE,CAIjC;IAEK,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAsD3B,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IA2B/B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAwC/B,aAAa,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IAItC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAetB,mBAAmB,CAAC,OAAO,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC;IAG1D;;;OAGG;IACG,SAAS,IAAI,OAAO,CAAC,YAAY,CAAC;cA0FxB,UAAU,IAAI,OAAO,CAAC,MAAM,CAAC;IAMvC,OAAO,CACX,cAAc,EAAE,YAAY,EAC5B,aAAa,EAAE,YAAY,GAC1B,OAAO,CAAC,eAAe,CAAC;IA0BrB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IA+FjC;;OAEG;IACG,GAAG,CACP,OAAO,EAAE,QAAQ,EACjB,IAAI,CAAC,EAAE;QACL,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,MAAM,CAAC,EAAE;YAAE,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;SAAE,CAAC;QAChC,EAAE,CAAC,EAAE,MAAM,CAAC;KACb,GACA,OAAO,CAAC,GAAG,CAAC;IAST,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAYhC"}
|
package/dist/src/o-node.js
CHANGED
|
@@ -9,6 +9,10 @@ import { oNodeConnectionManager } from './connection/o-node-connection.manager.j
|
|
|
9
9
|
import { oNodeResolver } from './router/resolvers/o-node.resolver.js';
|
|
10
10
|
import { oMethodResolver, oToolBase } from '@olane/o-tool';
|
|
11
11
|
import { oLeaderResolverFallback } from './router/index.js';
|
|
12
|
+
import { oNodeNotificationManager } from './o-node.notification-manager.js';
|
|
13
|
+
import { oConnectionHeartbeatManager } from './managers/o-connection-heartbeat.manager.js';
|
|
14
|
+
import { oReconnectionManager } from './managers/o-reconnection.manager.js';
|
|
15
|
+
import { LeaderRequestWrapper } from './utils/leader-request-wrapper.js';
|
|
12
16
|
export class oNode extends oToolBase {
|
|
13
17
|
constructor(config) {
|
|
14
18
|
super(config);
|
|
@@ -43,6 +47,9 @@ export class oNode extends oToolBase {
|
|
|
43
47
|
});
|
|
44
48
|
this.router = new oNodeRouter();
|
|
45
49
|
}
|
|
50
|
+
createNotificationManager() {
|
|
51
|
+
return new oNodeNotificationManager(this.p2pNode, this.hierarchyManager, this.address);
|
|
52
|
+
}
|
|
46
53
|
get staticAddress() {
|
|
47
54
|
return this.config.address;
|
|
48
55
|
}
|
|
@@ -60,6 +67,30 @@ export class oNode extends oToolBase {
|
|
|
60
67
|
this.logger.debug('Skipping unregistration, node is leader');
|
|
61
68
|
return;
|
|
62
69
|
}
|
|
70
|
+
// Notify parent we're stopping (best-effort, 2s timeout)
|
|
71
|
+
if (this.config.parent) {
|
|
72
|
+
try {
|
|
73
|
+
await Promise.race([
|
|
74
|
+
this.use(this.config.parent, {
|
|
75
|
+
method: 'notify',
|
|
76
|
+
params: {
|
|
77
|
+
eventType: 'node:stopping',
|
|
78
|
+
eventData: {
|
|
79
|
+
address: this.address.toString(),
|
|
80
|
+
reason: 'graceful_shutdown',
|
|
81
|
+
expectedDowntime: null,
|
|
82
|
+
},
|
|
83
|
+
source: this.address.toString(),
|
|
84
|
+
},
|
|
85
|
+
}),
|
|
86
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error('timeout')), 2000)),
|
|
87
|
+
]);
|
|
88
|
+
this.logger.debug('Notified parent of shutdown');
|
|
89
|
+
}
|
|
90
|
+
catch (error) {
|
|
91
|
+
this.logger.warn('Failed to notify parent (will be detected by heartbeat):', error instanceof Error ? error.message : error);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
63
94
|
if (!this.config.leader) {
|
|
64
95
|
this.logger.debug('No leader found, skipping unregistration');
|
|
65
96
|
return;
|
|
@@ -136,6 +167,10 @@ export class oNode extends oToolBase {
|
|
|
136
167
|
}
|
|
137
168
|
async start() {
|
|
138
169
|
await super.start();
|
|
170
|
+
// Start heartbeat after node is running
|
|
171
|
+
if (this.connectionHeartbeatManager) {
|
|
172
|
+
await this.connectionHeartbeatManager.start();
|
|
173
|
+
}
|
|
139
174
|
// await NetworkUtils.advertiseToNetwork(
|
|
140
175
|
// this.address,
|
|
141
176
|
// this.staticAddress,
|
|
@@ -256,6 +291,8 @@ export class oNode extends oToolBase {
|
|
|
256
291
|
}
|
|
257
292
|
await this.createNode();
|
|
258
293
|
await this.initializeRouter();
|
|
294
|
+
// need to wait until our libpp2 node is initialized before calling super.initialize
|
|
295
|
+
await super.initialize();
|
|
259
296
|
this.logger.debug('Node initialized!', this.transports.map((t) => t.toString()));
|
|
260
297
|
this.address.setTransports(this.transports);
|
|
261
298
|
this.peerId = this.p2pNode.peerId;
|
|
@@ -263,6 +300,16 @@ export class oNode extends oToolBase {
|
|
|
263
300
|
this.connectionManager = new oNodeConnectionManager({
|
|
264
301
|
p2pNode: this.p2pNode,
|
|
265
302
|
});
|
|
303
|
+
// Initialize leader request wrapper
|
|
304
|
+
this.leaderRequestWrapper = new LeaderRequestWrapper({
|
|
305
|
+
enabled: this.config.leaderRetry?.enabled ?? true,
|
|
306
|
+
maxAttempts: this.config.leaderRetry?.maxAttempts ?? 20,
|
|
307
|
+
baseDelayMs: this.config.leaderRetry?.baseDelayMs ?? 2000,
|
|
308
|
+
maxDelayMs: this.config.leaderRetry?.maxDelayMs ?? 30000,
|
|
309
|
+
timeoutMs: this.config.leaderRetry?.timeoutMs ?? 10000,
|
|
310
|
+
});
|
|
311
|
+
this.logger.info(`Leader retry config: enabled=${this.leaderRequestWrapper.getConfig().enabled}, ` +
|
|
312
|
+
`maxAttempts=${this.leaderRequestWrapper.getConfig().maxAttempts}`);
|
|
266
313
|
// initialize address resolution
|
|
267
314
|
this.router.addResolver(new oMethodResolver(this.address));
|
|
268
315
|
this.router.addResolver(new oNodeResolver(this.address));
|
|
@@ -271,8 +318,46 @@ export class oNode extends oToolBase {
|
|
|
271
318
|
this.logger.debug('Adding leader resolver fallback...');
|
|
272
319
|
this.router.addResolver(new oLeaderResolverFallback(this.address));
|
|
273
320
|
}
|
|
321
|
+
// Read ENABLE_LEADER_HEARTBEAT environment variable
|
|
322
|
+
const enableLeaderHeartbeat = this.parent?.toString() === oAddress.leader().toString();
|
|
323
|
+
this.logger.debug(`Enable leader heartbeat: ${enableLeaderHeartbeat}`);
|
|
324
|
+
// Initialize connection heartbeat manager
|
|
325
|
+
this.connectionHeartbeatManager = new oConnectionHeartbeatManager(this.p2pNode, this.hierarchyManager, this.notificationManager, this.address, {
|
|
326
|
+
enabled: this.config.connectionHeartbeat?.enabled ?? true,
|
|
327
|
+
intervalMs: this.config.connectionHeartbeat?.intervalMs ?? 15000,
|
|
328
|
+
timeoutMs: this.config.connectionHeartbeat?.timeoutMs ?? 5000,
|
|
329
|
+
failureThreshold: this.config.connectionHeartbeat?.failureThreshold ?? 3,
|
|
330
|
+
checkChildren: this.config.connectionHeartbeat?.checkChildren ?? true,
|
|
331
|
+
checkParent: this.config.connectionHeartbeat?.checkParent ?? true,
|
|
332
|
+
checkLeader: enableLeaderHeartbeat,
|
|
333
|
+
});
|
|
334
|
+
this.logger.info(`Connection heartbeat config: leader=${this.connectionHeartbeatManager.getConfig().checkLeader}, ` +
|
|
335
|
+
`parent=${this.connectionHeartbeatManager.getConfig().checkParent}`);
|
|
336
|
+
// Initialize reconnection manager
|
|
337
|
+
if (this.config.reconnection?.enabled !== false) {
|
|
338
|
+
this.reconnectionManager = new oReconnectionManager(this, {
|
|
339
|
+
enabled: true,
|
|
340
|
+
maxAttempts: this.config.reconnection?.maxAttempts ?? 10,
|
|
341
|
+
baseDelayMs: this.config.reconnection?.baseDelayMs ?? 5000,
|
|
342
|
+
maxDelayMs: this.config.reconnection?.maxDelayMs ?? 60000,
|
|
343
|
+
useLeaderFallback: this.config.reconnection?.useLeaderFallback ?? true,
|
|
344
|
+
parentDiscoveryIntervalMs: this.config.reconnection?.parentDiscoveryIntervalMs ?? 10000,
|
|
345
|
+
parentDiscoveryMaxDelayMs: this.config.reconnection?.parentDiscoveryMaxDelayMs ?? 60000,
|
|
346
|
+
});
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
/**
|
|
350
|
+
* Override use() to wrap leader/registry requests with retry logic
|
|
351
|
+
*/
|
|
352
|
+
async use(address, data) {
|
|
353
|
+
// Wrap leader/registry requests with retry logic
|
|
354
|
+
return this.leaderRequestWrapper.execute(() => super.use(address, data), address, data?.method);
|
|
274
355
|
}
|
|
275
356
|
async teardown() {
|
|
357
|
+
// Stop heartbeat before parent teardown
|
|
358
|
+
if (this.connectionHeartbeatManager) {
|
|
359
|
+
await this.connectionHeartbeatManager.stop();
|
|
360
|
+
}
|
|
276
361
|
await this.unregister();
|
|
277
362
|
await super.teardown();
|
|
278
363
|
if (this.p2pNode) {
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { Libp2p } from '@olane/o-config';
|
|
2
|
+
import { oNotificationManager } from '@olane/o-core';
|
|
3
|
+
import { oNodeAddress } from './router/o-node.address.js';
|
|
4
|
+
import { oNodeHierarchyManager } from './o-node.hierarchy-manager.js';
|
|
5
|
+
/**
|
|
6
|
+
* libp2p-specific implementation of oNotificationManager
|
|
7
|
+
* Wraps libp2p events and enriches them with Olane context
|
|
8
|
+
*/
|
|
9
|
+
export declare class oNodeNotificationManager extends oNotificationManager {
|
|
10
|
+
private p2pNode;
|
|
11
|
+
private hierarchyManager;
|
|
12
|
+
private address;
|
|
13
|
+
constructor(p2pNode: Libp2p, hierarchyManager: oNodeHierarchyManager, address: oNodeAddress);
|
|
14
|
+
/**
|
|
15
|
+
* Wire up libp2p event listeners
|
|
16
|
+
*/
|
|
17
|
+
protected setupListeners(): void;
|
|
18
|
+
/**
|
|
19
|
+
* Handle peer connect event from libp2p
|
|
20
|
+
*/
|
|
21
|
+
private handlePeerConnect;
|
|
22
|
+
/**
|
|
23
|
+
* Handle peer disconnect event from libp2p
|
|
24
|
+
*/
|
|
25
|
+
private handlePeerDisconnect;
|
|
26
|
+
/**
|
|
27
|
+
* Handle peer discovery event from libp2p
|
|
28
|
+
*/
|
|
29
|
+
private handlePeerDiscovery;
|
|
30
|
+
/**
|
|
31
|
+
* Handle connection open event from libp2p
|
|
32
|
+
*/
|
|
33
|
+
private handleConnectionOpen;
|
|
34
|
+
/**
|
|
35
|
+
* Handle connection close event from libp2p
|
|
36
|
+
*/
|
|
37
|
+
private handleConnectionClose;
|
|
38
|
+
/**
|
|
39
|
+
* Try to resolve a libp2p peer ID to an Olane address
|
|
40
|
+
* Checks hierarchy manager for known peers
|
|
41
|
+
*/
|
|
42
|
+
private peerIdToAddress;
|
|
43
|
+
/**
|
|
44
|
+
* Check if an address is a direct child
|
|
45
|
+
*/
|
|
46
|
+
private isChild;
|
|
47
|
+
/**
|
|
48
|
+
* Check if an address is a parent
|
|
49
|
+
*/
|
|
50
|
+
private isParent;
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=o-node.notification-manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"o-node.notification-manager.d.ts","sourceRoot":"","sources":["../../src/o-node.notification-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,OAAO,EACL,oBAAoB,EAQrB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAEtE;;;GAGG;AACH,qBAAa,wBAAyB,SAAQ,oBAAoB;IAE9D,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,gBAAgB;IACxB,OAAO,CAAC,OAAO;gBAFP,OAAO,EAAE,MAAM,EACf,gBAAgB,EAAE,qBAAqB,EACvC,OAAO,EAAE,YAAY;IAK/B;;OAEG;IACH,SAAS,CAAC,cAAc,IAAI,IAAI;IAgChC;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAkDzB;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAoD5B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAmB3B;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAI5B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAI7B;;;OAGG;IACH,OAAO,CAAC,eAAe;IAkCvB;;OAEG;IACH,OAAO,CAAC,OAAO;IAMf;;OAEG;IACH,OAAO,CAAC,QAAQ;CAKjB"}
|