@olane/o-node 0.7.12-alpha.6 → 0.7.12-alpha.61
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/connection/interfaces/o-node-connection-manager.config.d.ts +1 -0
- package/dist/src/connection/interfaces/o-node-connection-manager.config.d.ts.map +1 -1
- package/dist/src/connection/interfaces/o-node-connection.config.d.ts +1 -0
- package/dist/src/connection/interfaces/o-node-connection.config.d.ts.map +1 -1
- package/dist/src/connection/o-node-connection.d.ts +7 -2
- package/dist/src/connection/o-node-connection.d.ts.map +1 -1
- package/dist/src/connection/o-node-connection.js +55 -32
- package/dist/src/connection/o-node-connection.manager.d.ts +19 -5
- package/dist/src/connection/o-node-connection.manager.d.ts.map +1 -1
- package/dist/src/connection/o-node-connection.manager.js +81 -65
- package/dist/src/connection/o-stream.request.d.ts +11 -0
- package/dist/src/connection/o-stream.request.d.ts.map +1 -0
- package/dist/src/connection/o-stream.request.js +7 -0
- package/dist/src/connection/stream-handler.config.d.ts +46 -0
- package/dist/src/connection/stream-handler.config.d.ts.map +1 -0
- package/dist/src/connection/stream-handler.config.js +1 -0
- package/dist/src/connection/stream-handler.d.ts +97 -0
- package/dist/src/connection/stream-handler.d.ts.map +1 -0
- package/dist/src/connection/stream-handler.js +303 -0
- package/dist/src/index.d.ts +2 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +2 -1
- package/dist/src/interfaces/i-heartbeatable-node.d.ts +49 -0
- package/dist/src/interfaces/i-heartbeatable-node.d.ts.map +1 -0
- package/dist/src/interfaces/i-heartbeatable-node.js +1 -0
- package/dist/src/interfaces/i-reconnectable-node.d.ts +5 -0
- package/dist/src/interfaces/i-reconnectable-node.d.ts.map +1 -1
- package/dist/src/interfaces/o-node.config.d.ts +16 -8
- package/dist/src/interfaces/o-node.config.d.ts.map +1 -1
- package/dist/src/managers/o-connection-heartbeat.manager.d.ts +5 -9
- package/dist/src/managers/o-connection-heartbeat.manager.d.ts.map +1 -1
- package/dist/src/managers/o-connection-heartbeat.manager.js +51 -47
- package/dist/src/managers/o-reconnection.manager.d.ts +12 -0
- package/dist/src/managers/o-reconnection.manager.d.ts.map +1 -1
- package/dist/src/managers/o-reconnection.manager.js +138 -22
- package/dist/src/o-node.d.ts +22 -11
- package/dist/src/o-node.d.ts.map +1 -1
- package/dist/src/o-node.js +202 -72
- package/dist/src/o-node.notification-manager.d.ts.map +1 -1
- package/dist/src/o-node.notification-manager.js +17 -12
- package/dist/src/o-node.tool.d.ts +1 -0
- package/dist/src/o-node.tool.d.ts.map +1 -1
- package/dist/src/o-node.tool.js +18 -34
- package/dist/src/router/o-node.router.d.ts +1 -0
- package/dist/src/router/o-node.router.d.ts.map +1 -1
- package/dist/src/router/o-node.router.js +62 -9
- package/dist/src/router/o-node.routing-policy.d.ts.map +1 -1
- package/dist/src/router/o-node.routing-policy.js +7 -2
- package/dist/src/router/resolvers/o-node.resolver.d.ts.map +1 -1
- package/dist/src/router/resolvers/o-node.resolver.js +5 -1
- package/dist/src/router/resolvers/o-node.search-resolver.d.ts.map +1 -1
- package/dist/src/router/resolvers/o-node.search-resolver.js +34 -9
- package/dist/src/utils/index.d.ts +3 -0
- package/dist/src/utils/index.d.ts.map +1 -0
- package/dist/src/utils/index.js +2 -0
- package/dist/src/utils/stream.utils.d.ts +6 -0
- package/dist/src/utils/stream.utils.d.ts.map +1 -0
- package/dist/src/utils/stream.utils.js +31 -0
- package/dist/test/helpers/test-node.tool.d.ts +15 -0
- package/dist/test/helpers/test-node.tool.d.ts.map +1 -0
- package/dist/test/helpers/test-node.tool.js +27 -0
- package/package.json +6 -6
- package/dist/src/router/resolvers/o-node.child-resolver.d.ts +0 -11
- package/dist/src/router/resolvers/o-node.child-resolver.d.ts.map +0 -1
- package/dist/src/router/resolvers/o-node.child-resolver.js +0 -58
- package/dist/src/utils/leader-request-wrapper.d.ts +0 -45
- package/dist/src/utils/leader-request-wrapper.d.ts.map +0 -1
- package/dist/src/utils/leader-request-wrapper.js +0 -89
- package/dist/test/o-node.spec.d.ts +0 -2
- package/dist/test/o-node.spec.d.ts.map +0 -1
- package/dist/test/o-node.spec.js +0 -20
- package/dist/test/search-resolver.spec.d.ts +0 -2
- package/dist/test/search-resolver.spec.d.ts.map +0 -1
- package/dist/test/search-resolver.spec.js +0 -693
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { oObject, ChildLeftEvent, ParentDisconnectedEvent, LeaderDisconnectedEvent, ConnectionDegradedEvent, ConnectionRecoveredEvent, } from '@olane/o-core';
|
|
1
|
+
import { oObject, ChildLeftEvent, ParentDisconnectedEvent, LeaderDisconnectedEvent, ConnectionDegradedEvent, ConnectionRecoveredEvent, oAddress, } from '@olane/o-core';
|
|
2
2
|
/**
|
|
3
3
|
* Connection Heartbeat Manager
|
|
4
4
|
*
|
|
@@ -14,14 +14,12 @@ import { oObject, ChildLeftEvent, ParentDisconnectedEvent, LeaderDisconnectedEve
|
|
|
14
14
|
* - Emits ParentDisconnectedEvent when parent dies (triggers reconnection)
|
|
15
15
|
*/
|
|
16
16
|
export class oConnectionHeartbeatManager extends oObject {
|
|
17
|
-
constructor(
|
|
17
|
+
constructor(node, config) {
|
|
18
18
|
super();
|
|
19
|
-
this.
|
|
20
|
-
this.hierarchyManager = hierarchyManager;
|
|
21
|
-
this.notificationManager = notificationManager;
|
|
22
|
-
this.address = address;
|
|
19
|
+
this.node = node;
|
|
23
20
|
this.config = config;
|
|
24
21
|
this.healthMap = new Map();
|
|
22
|
+
this.isRunning = false;
|
|
25
23
|
}
|
|
26
24
|
async start() {
|
|
27
25
|
if (!this.config.enabled) {
|
|
@@ -43,37 +41,57 @@ export class oConnectionHeartbeatManager extends oObject {
|
|
|
43
41
|
this.healthMap.clear();
|
|
44
42
|
}
|
|
45
43
|
async performHeartbeatCycle() {
|
|
44
|
+
if (!this.isRunning) {
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
this.isRunning = true;
|
|
46
48
|
const targets = [];
|
|
47
49
|
// Check if this is a leader node (no leader in hierarchy = we are leader)
|
|
48
|
-
const isLeaderNode = this.
|
|
50
|
+
const isLeaderNode = this.node.getLeaders().length === 0;
|
|
49
51
|
// Collect leader (if enabled and we're not the leader)
|
|
50
|
-
if (
|
|
51
|
-
const leaders = this.
|
|
52
|
+
if (!isLeaderNode) {
|
|
53
|
+
const leaders = this.node.getLeaders();
|
|
52
54
|
for (const leader of leaders) {
|
|
53
55
|
targets.push({ address: leader, role: 'leader' });
|
|
54
56
|
}
|
|
55
57
|
}
|
|
56
58
|
// Collect parent
|
|
57
59
|
if (this.config.checkParent && !isLeaderNode) {
|
|
58
|
-
|
|
59
|
-
|
|
60
|
+
// Use this.node.parent getter to get the current parent address with transports
|
|
61
|
+
// rather than getParents() which may have a stale reference
|
|
62
|
+
const parent = this.node.parent;
|
|
63
|
+
// make sure that we don't double ping the leader
|
|
64
|
+
if (parent && parent?.toString() !== oAddress.leader().toString()) {
|
|
60
65
|
targets.push({ address: parent, role: 'parent' });
|
|
61
66
|
}
|
|
62
67
|
}
|
|
63
68
|
// Collect children
|
|
64
69
|
if (this.config.checkChildren) {
|
|
65
|
-
const children = this.
|
|
70
|
+
const children = this.node.getChildren();
|
|
66
71
|
for (const child of children) {
|
|
67
72
|
targets.push({ address: child, role: 'child' });
|
|
68
73
|
}
|
|
69
74
|
}
|
|
70
75
|
// Ping all targets in parallel
|
|
71
76
|
await Promise.allSettled(targets.map((target) => this.pingTarget(target.address, target.role)));
|
|
77
|
+
this.isRunning = false;
|
|
78
|
+
}
|
|
79
|
+
doPing(address) {
|
|
80
|
+
if (address.toString() === this.node.address.toString()) {
|
|
81
|
+
return Promise.resolve();
|
|
82
|
+
}
|
|
83
|
+
const transport = address.libp2pTransports[0].toMultiaddr();
|
|
84
|
+
if (transport.toString().indexOf('p2p-circuit') > -1) {
|
|
85
|
+
return this.node.use(address, {
|
|
86
|
+
method: 'ping',
|
|
87
|
+
params: {},
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
return this.node.p2pNode.services.ping.ping(transport);
|
|
72
91
|
}
|
|
73
92
|
async pingTarget(address, role) {
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
this.logger.warn(`Cannot extract peerId from ${address}`);
|
|
93
|
+
if (!address.libp2pTransports.length) {
|
|
94
|
+
this.logger.debug(`${role} has no transports, skipping ping`, address);
|
|
77
95
|
return;
|
|
78
96
|
}
|
|
79
97
|
const key = address.toString();
|
|
@@ -81,7 +99,7 @@ export class oConnectionHeartbeatManager extends oObject {
|
|
|
81
99
|
if (!health) {
|
|
82
100
|
health = {
|
|
83
101
|
address,
|
|
84
|
-
peerId,
|
|
102
|
+
peerId: 'unknown',
|
|
85
103
|
lastSuccessfulPing: 0,
|
|
86
104
|
consecutiveFailures: 0,
|
|
87
105
|
averageLatency: 0,
|
|
@@ -98,10 +116,7 @@ export class oConnectionHeartbeatManager extends oObject {
|
|
|
98
116
|
});
|
|
99
117
|
// Race between ping and timeout
|
|
100
118
|
// The ping service accepts PeerId as string or object
|
|
101
|
-
await Promise.race([
|
|
102
|
-
this.p2pNode.services.ping.ping(peerId),
|
|
103
|
-
timeoutPromise,
|
|
104
|
-
]);
|
|
119
|
+
await Promise.race([this.doPing(address), timeoutPromise]);
|
|
105
120
|
const latency = Date.now() - startTime;
|
|
106
121
|
// Success - update health
|
|
107
122
|
health.lastSuccessfulPing = Date.now();
|
|
@@ -117,16 +132,17 @@ export class oConnectionHeartbeatManager extends oObject {
|
|
|
117
132
|
this.logger.info(`Connection recovered: ${address} (latency: ${latency}ms)`);
|
|
118
133
|
this.emitConnectionRecoveredEvent(address, role);
|
|
119
134
|
}
|
|
120
|
-
this.logger.debug(`Ping successful: ${address} (${latency}ms)`);
|
|
135
|
+
// this.logger.debug(`Ping successful: ${address} (${latency}ms)`);
|
|
121
136
|
}
|
|
122
137
|
catch (error) {
|
|
123
138
|
health.consecutiveFailures++;
|
|
124
|
-
this.logger.warn(`Ping failed: ${address} (failures: ${health.consecutiveFailures}/${this.config.failureThreshold})
|
|
139
|
+
this.logger.warn(`Ping failed: ${address} (failures: ${health.consecutiveFailures}/${this.config.failureThreshold})`, error);
|
|
125
140
|
// Update status based on failure count
|
|
126
141
|
if (health.consecutiveFailures >= this.config.failureThreshold) {
|
|
127
142
|
this.handleConnectionDead(address, role, health);
|
|
128
143
|
}
|
|
129
|
-
else if (health.consecutiveFailures >=
|
|
144
|
+
else if (health.consecutiveFailures >=
|
|
145
|
+
Math.ceil(this.config.failureThreshold / 2)) {
|
|
130
146
|
health.status = 'degraded';
|
|
131
147
|
this.emitConnectionDegradedEvent(address, role, health.consecutiveFailures);
|
|
132
148
|
}
|
|
@@ -140,20 +156,20 @@ export class oConnectionHeartbeatManager extends oObject {
|
|
|
140
156
|
// Emit events based on role
|
|
141
157
|
if (role === 'child') {
|
|
142
158
|
// Remove dead child from hierarchy
|
|
143
|
-
this.
|
|
159
|
+
this.node.removeChild(address);
|
|
144
160
|
// Emit child left event
|
|
145
|
-
this.notificationManager.emit(new ChildLeftEvent({
|
|
146
|
-
source: this.address,
|
|
161
|
+
this.node.notificationManager.emit(new ChildLeftEvent({
|
|
162
|
+
source: this.node.address,
|
|
147
163
|
childAddress: address,
|
|
148
|
-
parentAddress: this.address,
|
|
164
|
+
parentAddress: this.node.address,
|
|
149
165
|
reason: `heartbeat_failed_${health.consecutiveFailures}_times`,
|
|
150
166
|
}));
|
|
151
167
|
this.logger.warn(`Removed dead child: ${address}`);
|
|
152
168
|
}
|
|
153
169
|
else if (role === 'parent') {
|
|
154
170
|
// Emit parent disconnected event
|
|
155
|
-
this.notificationManager.emit(new ParentDisconnectedEvent({
|
|
156
|
-
source: this.address,
|
|
171
|
+
this.node.notificationManager.emit(new ParentDisconnectedEvent({
|
|
172
|
+
source: this.node.address,
|
|
157
173
|
parentAddress: address,
|
|
158
174
|
reason: `heartbeat_failed_${health.consecutiveFailures}_times`,
|
|
159
175
|
}));
|
|
@@ -162,8 +178,8 @@ export class oConnectionHeartbeatManager extends oObject {
|
|
|
162
178
|
}
|
|
163
179
|
else if (role === 'leader') {
|
|
164
180
|
// Emit leader disconnected event
|
|
165
|
-
this.notificationManager.emit(new LeaderDisconnectedEvent({
|
|
166
|
-
source: this.address,
|
|
181
|
+
this.node.notificationManager.emit(new LeaderDisconnectedEvent({
|
|
182
|
+
source: this.node.address,
|
|
167
183
|
leaderAddress: address,
|
|
168
184
|
reason: `heartbeat_failed_${health.consecutiveFailures}_times`,
|
|
169
185
|
}));
|
|
@@ -174,8 +190,8 @@ export class oConnectionHeartbeatManager extends oObject {
|
|
|
174
190
|
emitConnectionDegradedEvent(address, role, failures) {
|
|
175
191
|
// ConnectionDegradedEvent only supports parent/child, so we map leader to parent
|
|
176
192
|
const eventRole = role === 'leader' ? 'parent' : role === 'child' ? 'child' : 'parent';
|
|
177
|
-
this.notificationManager.emit(new ConnectionDegradedEvent({
|
|
178
|
-
source: this.address,
|
|
193
|
+
this.node.notificationManager.emit(new ConnectionDegradedEvent({
|
|
194
|
+
source: this.node.address,
|
|
179
195
|
targetAddress: address,
|
|
180
196
|
role: eventRole,
|
|
181
197
|
consecutiveFailures: failures,
|
|
@@ -184,24 +200,12 @@ export class oConnectionHeartbeatManager extends oObject {
|
|
|
184
200
|
emitConnectionRecoveredEvent(address, role) {
|
|
185
201
|
// ConnectionRecoveredEvent only supports parent/child, so we map leader to parent
|
|
186
202
|
const eventRole = role === 'leader' ? 'parent' : role === 'child' ? 'child' : 'parent';
|
|
187
|
-
this.notificationManager.emit(new ConnectionRecoveredEvent({
|
|
188
|
-
source: this.address,
|
|
203
|
+
this.node.notificationManager.emit(new ConnectionRecoveredEvent({
|
|
204
|
+
source: this.node.address,
|
|
189
205
|
targetAddress: address,
|
|
190
206
|
role: eventRole,
|
|
191
207
|
}));
|
|
192
208
|
}
|
|
193
|
-
extractPeerIdFromAddress(address) {
|
|
194
|
-
// Extract peerId from transport multiaddr
|
|
195
|
-
for (const transport of address.transports) {
|
|
196
|
-
const multiaddr = transport.toString();
|
|
197
|
-
// Multiaddr format: /ip4/127.0.0.1/tcp/4001/p2p/QmPeerId
|
|
198
|
-
const parts = multiaddr.split('/p2p/');
|
|
199
|
-
if (parts.length === 2) {
|
|
200
|
-
return parts[1];
|
|
201
|
-
}
|
|
202
|
-
}
|
|
203
|
-
return null;
|
|
204
|
-
}
|
|
205
209
|
/**
|
|
206
210
|
* Get current health status of all connections
|
|
207
211
|
*/
|
|
@@ -6,6 +6,8 @@ export interface ReconnectionConfig {
|
|
|
6
6
|
baseDelayMs: number;
|
|
7
7
|
maxDelayMs: number;
|
|
8
8
|
useLeaderFallback: boolean;
|
|
9
|
+
parentDiscoveryIntervalMs: number;
|
|
10
|
+
parentDiscoveryMaxDelayMs: number;
|
|
9
11
|
}
|
|
10
12
|
/**
|
|
11
13
|
* Reconnection Manager
|
|
@@ -25,12 +27,22 @@ export declare class oReconnectionManager extends oObject {
|
|
|
25
27
|
private reconnecting;
|
|
26
28
|
constructor(node: IReconnectableNode, config: ReconnectionConfig);
|
|
27
29
|
private setupEventListeners;
|
|
30
|
+
handleNodeConnected(event: any): Promise<void>;
|
|
28
31
|
private handleConnectionDegraded;
|
|
29
32
|
private handleLeaderDisconnected;
|
|
30
33
|
private handleParentDisconnected;
|
|
31
34
|
attemptReconnection(): Promise<void>;
|
|
32
35
|
private tryDirectParentReconnection;
|
|
33
36
|
private tryLeaderFallback;
|
|
37
|
+
/**
|
|
38
|
+
* Wait for leader to become available and reconnect
|
|
39
|
+
* Leader transports are static (configured), so we just need to detect when it's back
|
|
40
|
+
*/
|
|
41
|
+
private waitForLeaderAndReconnect;
|
|
42
|
+
/**
|
|
43
|
+
* Wait for non-leader parent to appear in registry and reconnect
|
|
44
|
+
*/
|
|
45
|
+
waitForParentAndReconnect(): Promise<void>;
|
|
34
46
|
private handleReconnectionFailure;
|
|
35
47
|
private calculateNodeLevel;
|
|
36
48
|
private calculateBackoffDelay;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"o-reconnection.manager.d.ts","sourceRoot":"","sources":["../../../src/managers/o-reconnection.manager.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,OAAO,
|
|
1
|
+
{"version":3,"file":"o-reconnection.manager.d.ts","sourceRoot":"","sources":["../../../src/managers/o-reconnection.manager.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,OAAO,EASR,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,kBAAkB,EAAE,MAAM,uCAAuC,CAAC;AAI3E,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,yBAAyB,EAAE,MAAM,CAAC;IAClC,yBAAyB,EAAE,MAAM,CAAC;CACnC;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,oBAAqB,SAAQ,OAAO;IAI7C,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,MAAM;IAJhB,OAAO,CAAC,YAAY,CAAS;gBAGnB,IAAI,EAAE,kBAAkB,EACxB,MAAM,EAAE,kBAAkB;IAMpC,OAAO,CAAC,mBAAmB;IAyBrB,mBAAmB,CAAC,KAAK,EAAE,GAAG;YAatB,wBAAwB;YAaxB,wBAAwB;YAexB,wBAAwB;IAehC,mBAAmB;YAgDX,2BAA2B;YAiB3B,iBAAiB;IAkB/B;;;OAGG;YACW,yBAAyB;IAiFvC;;OAEG;IACG,yBAAyB;IAkG/B,OAAO,CAAC,yBAAyB;IAajC,OAAO,CAAC,kBAAkB;IAI1B,OAAO,CAAC,qBAAqB;IAK7B,OAAO,CAAC,KAAK;CAGd"}
|
|
@@ -28,6 +28,17 @@ export class oReconnectionManager extends oObject {
|
|
|
28
28
|
this.node.notificationManager.on('leader:disconnected', this.handleLeaderDisconnected.bind(this));
|
|
29
29
|
// Listen for connection degradation as early warning
|
|
30
30
|
this.node.notificationManager.on('connection:degraded', this.handleConnectionDegraded.bind(this));
|
|
31
|
+
this.node.notificationManager.on('node:connected', this.handleNodeConnected.bind(this));
|
|
32
|
+
}
|
|
33
|
+
async handleNodeConnected(event) {
|
|
34
|
+
const connectedEvent = event;
|
|
35
|
+
if (connectedEvent.nodeAddress.toString() === oAddress.leader().toString()) {
|
|
36
|
+
// the leader is back online, let's re-register & tell sub-graphs to re-register
|
|
37
|
+
await this.node.useSelf({
|
|
38
|
+
method: 'register_leader',
|
|
39
|
+
params: {},
|
|
40
|
+
});
|
|
41
|
+
}
|
|
31
42
|
}
|
|
32
43
|
async handleConnectionDegraded(event) {
|
|
33
44
|
const degradedEvent = event;
|
|
@@ -104,31 +115,136 @@ export class oReconnectionManager extends oObject {
|
|
|
104
115
|
this.logger.info('Direct parent reconnection successful');
|
|
105
116
|
}
|
|
106
117
|
async tryLeaderFallback() {
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
118
|
+
// Check if parent is the leader - special case
|
|
119
|
+
const parentIsLeader = this.node.config.parent?.toString() === oAddress.leader().toString();
|
|
120
|
+
if (parentIsLeader) {
|
|
121
|
+
this.logger.info('Parent is the leader - waiting for leader to become available');
|
|
122
|
+
await this.waitForLeaderAndReconnect();
|
|
123
|
+
}
|
|
124
|
+
else {
|
|
125
|
+
this.logger.info('Starting infinite parent discovery via leader registry');
|
|
126
|
+
await this.waitForParentAndReconnect();
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Wait for leader to become available and reconnect
|
|
131
|
+
* Leader transports are static (configured), so we just need to detect when it's back
|
|
132
|
+
*/
|
|
133
|
+
async waitForLeaderAndReconnect() {
|
|
134
|
+
const startTime = Date.now();
|
|
135
|
+
let attempt = 0;
|
|
136
|
+
let currentDelay = this.config.parentDiscoveryIntervalMs;
|
|
137
|
+
// Infinite retry loop - keep trying until leader is back
|
|
138
|
+
while (true) {
|
|
139
|
+
attempt++;
|
|
140
|
+
const elapsedMinutes = Math.floor((Date.now() - startTime) / 60000);
|
|
141
|
+
this.logger.info(`Leader discovery attempt ${attempt} (elapsed: ${elapsedMinutes}m)`);
|
|
142
|
+
try {
|
|
143
|
+
// Try to ping the leader using its configured transports
|
|
144
|
+
// The leader address should already have transports configured
|
|
145
|
+
await this.node.use(this.node.config.parent, {
|
|
146
|
+
method: 'ping',
|
|
147
|
+
params: {},
|
|
148
|
+
});
|
|
149
|
+
// Leader is back! Now re-register with parent and registry
|
|
150
|
+
this.logger.info(`Leader is back online after ${elapsedMinutes}m, re-registering...`);
|
|
151
|
+
try {
|
|
152
|
+
// Register with parent (leader)
|
|
153
|
+
await this.node.registerParent();
|
|
154
|
+
// Force re-registration with registry by resetting the flag
|
|
155
|
+
this.node.didRegister = false;
|
|
156
|
+
await this.node.register();
|
|
157
|
+
// Success!
|
|
158
|
+
this.reconnecting = false;
|
|
159
|
+
this.logger.info(`Successfully reconnected to leader and re-registered after ${elapsedMinutes}m`);
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
162
|
+
catch (registrationError) {
|
|
163
|
+
this.logger.warn('Leader online but registration failed, will retry:', registrationError instanceof Error
|
|
164
|
+
? registrationError.message
|
|
165
|
+
: registrationError);
|
|
166
|
+
// Fall through to retry with backoff
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
catch (error) {
|
|
170
|
+
// Leader not yet available
|
|
171
|
+
this.logger.debug(`Leader not yet available (will retry): ${error instanceof Error ? error.message : error}`);
|
|
172
|
+
}
|
|
173
|
+
// Calculate backoff delay
|
|
174
|
+
const delay = Math.min(currentDelay, this.config.parentDiscoveryMaxDelayMs);
|
|
175
|
+
// Log periodic status updates (every 5 minutes)
|
|
176
|
+
if (attempt % 30 === 0) {
|
|
177
|
+
this.logger.info(`Still waiting for leader to come back online... (${elapsedMinutes}m elapsed, ${attempt} attempts)`);
|
|
126
178
|
}
|
|
179
|
+
this.logger.debug(`Waiting ${delay}ms before next discovery attempt...`);
|
|
180
|
+
await this.sleep(delay);
|
|
181
|
+
// Exponential backoff for next iteration
|
|
182
|
+
currentDelay = Math.min(currentDelay * 2, this.config.parentDiscoveryMaxDelayMs);
|
|
127
183
|
}
|
|
128
|
-
|
|
129
|
-
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* Wait for non-leader parent to appear in registry and reconnect
|
|
187
|
+
*/
|
|
188
|
+
async waitForParentAndReconnect() {
|
|
189
|
+
const startTime = Date.now();
|
|
190
|
+
let attempt = 0;
|
|
191
|
+
let currentDelay = this.config.parentDiscoveryIntervalMs;
|
|
192
|
+
// Infinite retry loop - keep trying until parent is found
|
|
193
|
+
while (true) {
|
|
194
|
+
attempt++;
|
|
195
|
+
const elapsedMinutes = Math.floor((Date.now() - startTime) / 60000);
|
|
196
|
+
this.logger.info(`Parent discovery attempt ${attempt} (elapsed: ${elapsedMinutes}m)`);
|
|
197
|
+
try {
|
|
198
|
+
// Query registry for parent by its known address
|
|
199
|
+
const response = await this.node.use(oAddress.registry(), {
|
|
200
|
+
method: 'find_available_parent',
|
|
201
|
+
params: {
|
|
202
|
+
parentAddress: this.node.config.parent?.toString(),
|
|
203
|
+
},
|
|
204
|
+
});
|
|
205
|
+
const { parentAddress, parentTransports } = response.result.data;
|
|
206
|
+
// Check if parent was found in registry
|
|
207
|
+
if (parentAddress && parentTransports && parentTransports.length > 0) {
|
|
208
|
+
this.logger.info(`Parent found in registry: ${parentAddress} with ${parentTransports.length} transports`);
|
|
209
|
+
// Update parent reference with fresh transports
|
|
210
|
+
this.node.config.parent = new oNodeAddress(parentAddress, parentTransports.map((t) => new oNodeTransport(t.value)));
|
|
211
|
+
// Attempt to register with parent and re-register with registry
|
|
212
|
+
try {
|
|
213
|
+
await this.tryDirectParentReconnection();
|
|
214
|
+
// Force re-registration with registry by resetting the flag
|
|
215
|
+
this.node.didRegister = false;
|
|
216
|
+
await this.node.register();
|
|
217
|
+
// Success!
|
|
218
|
+
this.reconnecting = false;
|
|
219
|
+
this.logger.info(`Successfully reconnected to parent and re-registered after ${elapsedMinutes}m of discovery attempts`);
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
222
|
+
catch (registrationError) {
|
|
223
|
+
this.logger.warn('Parent found but registration failed, will retry:', registrationError instanceof Error
|
|
224
|
+
? registrationError.message
|
|
225
|
+
: registrationError);
|
|
226
|
+
// Fall through to retry with backoff
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
else {
|
|
230
|
+
this.logger.debug(`Parent not yet available in registry: ${this.node.config.parent?.toString()}`);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
catch (error) {
|
|
234
|
+
// Network error communicating with leader/registry
|
|
235
|
+
this.logger.warn('Error querying registry for parent (will retry):', error instanceof Error ? error.message : error);
|
|
236
|
+
}
|
|
237
|
+
// Calculate backoff delay with exponential increase, capped at max
|
|
238
|
+
const delay = Math.min(currentDelay, this.config.parentDiscoveryMaxDelayMs);
|
|
239
|
+
// Log periodic status updates (every 5 minutes)
|
|
240
|
+
if (attempt % 30 === 0) {
|
|
241
|
+
this.logger.info(`Still waiting for parent to appear in registry... (${elapsedMinutes}m elapsed, ${attempt} attempts)`);
|
|
242
|
+
}
|
|
243
|
+
this.logger.debug(`Waiting ${delay}ms before next discovery attempt...`);
|
|
244
|
+
await this.sleep(delay);
|
|
245
|
+
// Exponential backoff for next iteration
|
|
246
|
+
currentDelay = Math.min(currentDelay * 2, this.config.parentDiscoveryMaxDelayMs);
|
|
130
247
|
}
|
|
131
|
-
this.handleReconnectionFailure();
|
|
132
248
|
}
|
|
133
249
|
handleReconnectionFailure() {
|
|
134
250
|
this.reconnecting = false;
|
package/dist/src/o-node.d.ts
CHANGED
|
@@ -3,14 +3,14 @@ import { PeerId } from '@olane/o-config';
|
|
|
3
3
|
import { oNodeHierarchyManager } from './o-node.hierarchy-manager.js';
|
|
4
4
|
import { oNodeConfig } from './interfaces/o-node.config.js';
|
|
5
5
|
import { oNodeTransport } from './router/o-node.transport.js';
|
|
6
|
-
import {
|
|
6
|
+
import { oRequest, oNotificationManager } from '@olane/o-core';
|
|
7
7
|
import { oNodeAddress } from './router/o-node.address.js';
|
|
8
8
|
import { oNodeConnection } from './connection/o-node-connection.js';
|
|
9
9
|
import { oNodeConnectionManager } from './connection/o-node-connection.manager.js';
|
|
10
10
|
import { oToolBase } from '@olane/o-tool';
|
|
11
11
|
import { oConnectionHeartbeatManager } from './managers/o-connection-heartbeat.manager.js';
|
|
12
12
|
import { oReconnectionManager } from './managers/o-reconnection.manager.js';
|
|
13
|
-
import {
|
|
13
|
+
import { oNodeConnectionConfig } from './connection/index.js';
|
|
14
14
|
export declare class oNode extends oToolBase {
|
|
15
15
|
peerId: PeerId;
|
|
16
16
|
p2pNode: Libp2p;
|
|
@@ -20,7 +20,6 @@ export declare class oNode extends oToolBase {
|
|
|
20
20
|
hierarchyManager: oNodeHierarchyManager;
|
|
21
21
|
connectionHeartbeatManager?: oConnectionHeartbeatManager;
|
|
22
22
|
reconnectionManager?: oReconnectionManager;
|
|
23
|
-
leaderRequestWrapper: LeaderRequestWrapper;
|
|
24
23
|
protected didRegister: boolean;
|
|
25
24
|
constructor(config: oNodeConfig);
|
|
26
25
|
get leader(): oNodeAddress | null;
|
|
@@ -34,6 +33,7 @@ export declare class oNode extends oToolBase {
|
|
|
34
33
|
get transports(): oNodeTransport[];
|
|
35
34
|
unregister(): Promise<void>;
|
|
36
35
|
registerParent(): Promise<void>;
|
|
36
|
+
registerLeader(): Promise<void>;
|
|
37
37
|
register(): Promise<void>;
|
|
38
38
|
extractMethod(address: oNodeAddress): string;
|
|
39
39
|
start(): Promise<void>;
|
|
@@ -44,18 +44,29 @@ export declare class oNode extends oToolBase {
|
|
|
44
44
|
*/
|
|
45
45
|
configure(): Promise<Libp2pConfig>;
|
|
46
46
|
protected createNode(): Promise<Libp2p>;
|
|
47
|
-
connect(
|
|
47
|
+
connect(config: oNodeConnectionConfig): Promise<oNodeConnection>;
|
|
48
|
+
initConnectionManager(): Promise<void>;
|
|
49
|
+
initReconnectionManager(): Promise<void>;
|
|
50
|
+
hookInitializeFinished(): Promise<void>;
|
|
51
|
+
hookStartFinished(): Promise<void>;
|
|
48
52
|
initialize(): Promise<void>;
|
|
49
53
|
/**
|
|
50
54
|
* Override use() to wrap leader/registry requests with retry logic
|
|
51
55
|
*/
|
|
52
|
-
use(address: oAddress, data?: {
|
|
53
|
-
method?: string;
|
|
54
|
-
params?: {
|
|
55
|
-
[key: string]: any;
|
|
56
|
-
};
|
|
57
|
-
id?: string;
|
|
58
|
-
}): Promise<any>;
|
|
59
56
|
teardown(): Promise<void>;
|
|
57
|
+
getLeaders(): oNodeAddress[];
|
|
58
|
+
getParents(): oNodeAddress[];
|
|
59
|
+
getChildren(): oNodeAddress[];
|
|
60
|
+
removeChild(childAddress: oNodeAddress): void;
|
|
61
|
+
/**
|
|
62
|
+
* Get the total number of active streams across all connections
|
|
63
|
+
* @returns Total count of active streams
|
|
64
|
+
*/
|
|
65
|
+
getStreamCount(): number;
|
|
66
|
+
/**
|
|
67
|
+
* Get libp2p metrics for this node
|
|
68
|
+
* Tool method that can be called remotely by monitoring systems
|
|
69
|
+
*/
|
|
70
|
+
_tool_get_libp2p_metrics(request: oRequest): Promise<any>;
|
|
60
71
|
}
|
|
61
72
|
//# sourceMappingURL=o-node.d.ts.map
|
package/dist/src/o-node.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"o-node.d.ts","sourceRoot":"","sources":["../../src/o-node.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"o-node.d.ts","sourceRoot":"","sources":["../../src/o-node.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,MAAM,EACN,YAAY,EACb,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AACtE,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAKL,QAAQ,EAER,oBAAoB,EAGrB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,mCAAmC,CAAC;AACpE,OAAO,EAAE,sBAAsB,EAAE,MAAM,2CAA2C,CAAC;AAEnF,OAAO,EAAmB,SAAS,EAAE,MAAM,eAAe,CAAC;AAG3D,OAAO,EAAE,2BAA2B,EAAE,MAAM,8CAA8C,CAAC;AAC3F,OAAO,EAAE,oBAAoB,EAAE,MAAM,sCAAsC,CAAC;AAC5E,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAE9D,qBAAa,KAAM,SAAQ,SAAS;IAC3B,MAAM,EAAG,MAAM,CAAC;IAChB,OAAO,EAAG,MAAM,CAAC;IACjB,OAAO,EAAG,YAAY,CAAC;IACvB,MAAM,EAAE,WAAW,CAAC;IACpB,iBAAiB,EAAG,sBAAsB,CAAC;IAC3C,gBAAgB,EAAG,qBAAqB,CAAC;IACzC,0BAA0B,CAAC,EAAE,2BAA2B,CAAC;IACzD,mBAAmB,CAAC,EAAE,oBAAoB,CAAC;IAClD,SAAS,CAAC,WAAW,EAAE,OAAO,CAAS;gBAE3B,MAAM,EAAE,WAAW;IAK/B,IAAI,MAAM,IAAI,YAAY,GAAG,IAAI,CAEhC;IAED,IAAI,aAAa,IAAI,YAAY,CAKhC;IAED,IAAI,YAAY,IAAI,MAAM,GAAG,IAAI,CAOhC;IAED,mBAAmB,IAAI,GAAG,EAAE;IAItB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC;IASvC,SAAS,CAAC,yBAAyB,IAAI,oBAAoB;IAQ3D,IAAI,aAAa,IAAI,YAAY,CAEhC;IAED,IAAI,gBAAgB,IAAI,cAAc,EAAE,CAEvC;IAED,IAAI,UAAU,IAAI,cAAc,EAAE,CAIjC;IAEK,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAwD3B,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAqC/B,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAiB/B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IA4B/B,aAAa,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IAItC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAStB,mBAAmB,CAAC,OAAO,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC;IAG1D;;;OAGG;IACG,SAAS,IAAI,OAAO,CAAC,YAAY,CAAC;cA0HxB,UAAU,IAAI,OAAO,CAAC,MAAM,CAAC;IAMvC,OAAO,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAAC,eAAe,CAAC;IAsBhE,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC;IAStC,uBAAuB,IAAI,OAAO,CAAC,IAAI,CAAC;IAiBxC,sBAAsB,IAAI,OAAO,CAAC,IAAI,CAAC;IAEvC,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC;IAsBlC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAwCjC;;OAEG;IAiBG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAc/B,UAAU,IAAI,YAAY,EAAE;IAI5B,UAAU,IAAI,YAAY,EAAE;IAI5B,WAAW,IAAI,YAAY,EAAE;IAI7B,WAAW,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI;IAI7C;;;OAGG;IACH,cAAc,IAAI,MAAM;IAUxB;;;OAGG;IACG,wBAAwB,CAAC,OAAO,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC;CAwEhE"}
|