@olane/o-node 0.7.12-alpha.9 → 0.7.13-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/connection/interfaces/o-node-connection-manager.config.d.ts +1 -0
- package/dist/src/connection/interfaces/o-node-connection-manager.config.d.ts.map +1 -1
- package/dist/src/connection/interfaces/o-node-connection.config.d.ts +1 -0
- package/dist/src/connection/interfaces/o-node-connection.config.d.ts.map +1 -1
- package/dist/src/connection/o-node-connection.d.ts +7 -2
- package/dist/src/connection/o-node-connection.d.ts.map +1 -1
- package/dist/src/connection/o-node-connection.js +56 -32
- package/dist/src/connection/o-node-connection.manager.d.ts +20 -5
- package/dist/src/connection/o-node-connection.manager.d.ts.map +1 -1
- package/dist/src/connection/o-node-connection.manager.js +95 -65
- package/dist/src/connection/o-stream.request.d.ts +11 -0
- package/dist/src/connection/o-stream.request.d.ts.map +1 -0
- package/dist/src/connection/o-stream.request.js +7 -0
- package/dist/src/connection/stream-handler.config.d.ts +46 -0
- package/dist/src/connection/stream-handler.config.d.ts.map +1 -0
- package/dist/src/connection/stream-handler.config.js +1 -0
- package/dist/src/connection/stream-handler.d.ts +98 -0
- package/dist/src/connection/stream-handler.d.ts.map +1 -0
- package/dist/src/connection/stream-handler.js +310 -0
- package/dist/src/index.d.ts +2 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +2 -1
- package/dist/src/interfaces/i-heartbeatable-node.d.ts +49 -0
- package/dist/src/interfaces/i-heartbeatable-node.d.ts.map +1 -0
- package/dist/src/interfaces/i-heartbeatable-node.js +1 -0
- package/dist/src/interfaces/i-reconnectable-node.d.ts +5 -0
- package/dist/src/interfaces/i-reconnectable-node.d.ts.map +1 -1
- package/dist/src/interfaces/o-node.config.d.ts +16 -22
- package/dist/src/interfaces/o-node.config.d.ts.map +1 -1
- package/dist/src/managers/o-connection-heartbeat.manager.d.ts +17 -17
- package/dist/src/managers/o-connection-heartbeat.manager.d.ts.map +1 -1
- package/dist/src/managers/o-connection-heartbeat.manager.js +83 -78
- package/dist/src/managers/o-reconnection.manager.d.ts +12 -0
- package/dist/src/managers/o-reconnection.manager.d.ts.map +1 -1
- package/dist/src/managers/o-reconnection.manager.js +138 -22
- package/dist/src/o-node.d.ts +23 -12
- package/dist/src/o-node.d.ts.map +1 -1
- package/dist/src/o-node.js +227 -72
- package/dist/src/o-node.notification-manager.d.ts.map +1 -1
- package/dist/src/o-node.notification-manager.js +17 -12
- package/dist/src/o-node.tool.d.ts +1 -0
- package/dist/src/o-node.tool.d.ts.map +1 -1
- package/dist/src/o-node.tool.js +18 -34
- package/dist/src/router/o-node.router.d.ts +1 -0
- package/dist/src/router/o-node.router.d.ts.map +1 -1
- package/dist/src/router/o-node.router.js +62 -9
- package/dist/src/router/o-node.routing-policy.d.ts.map +1 -1
- package/dist/src/router/o-node.routing-policy.js +7 -2
- package/dist/src/router/resolvers/o-node.resolver.d.ts.map +1 -1
- package/dist/src/router/resolvers/o-node.resolver.js +5 -1
- package/dist/src/router/resolvers/o-node.search-resolver.d.ts.map +1 -1
- package/dist/src/router/resolvers/o-node.search-resolver.js +34 -9
- package/dist/src/utils/index.d.ts +3 -0
- package/dist/src/utils/index.d.ts.map +1 -0
- package/dist/src/utils/index.js +2 -0
- package/dist/src/utils/stream.utils.d.ts +6 -0
- package/dist/src/utils/stream.utils.d.ts.map +1 -0
- package/dist/src/utils/stream.utils.js +31 -0
- package/dist/test/helpers/test-node.tool.d.ts +15 -0
- package/dist/test/helpers/test-node.tool.d.ts.map +1 -0
- package/dist/test/helpers/test-node.tool.js +27 -0
- package/package.json +6 -6
- package/dist/src/router/resolvers/o-node.child-resolver.d.ts +0 -11
- package/dist/src/router/resolvers/o-node.child-resolver.d.ts.map +0 -1
- package/dist/src/router/resolvers/o-node.child-resolver.js +0 -58
- package/dist/src/utils/leader-request-wrapper.d.ts +0 -45
- package/dist/src/utils/leader-request-wrapper.d.ts.map +0 -1
- package/dist/src/utils/leader-request-wrapper.js +0 -89
- package/dist/test/o-node.spec.d.ts +0 -2
- package/dist/test/o-node.spec.d.ts.map +0 -1
- package/dist/test/o-node.spec.js +0 -20
- package/dist/test/search-resolver.spec.d.ts +0 -2
- package/dist/test/search-resolver.spec.d.ts.map +0 -1
- package/dist/test/search-resolver.spec.js +0 -693
|
@@ -1,39 +1,38 @@
|
|
|
1
|
-
import { oObject, ChildLeftEvent, ParentDisconnectedEvent, LeaderDisconnectedEvent, ConnectionDegradedEvent, ConnectionRecoveredEvent, } from '@olane/o-core';
|
|
1
|
+
import { oObject, ChildLeftEvent, ParentDisconnectedEvent, LeaderDisconnectedEvent, ConnectionDegradedEvent, ConnectionRecoveredEvent, oAddress, } from '@olane/o-core';
|
|
2
2
|
/**
|
|
3
|
-
* Connection
|
|
3
|
+
* Connection Health Monitor
|
|
4
4
|
*
|
|
5
|
-
*
|
|
6
|
-
* Continuously
|
|
5
|
+
* Monitors connection health by checking libp2p's connection state directly.
|
|
6
|
+
* Continuously checks parent and children connections to detect failures early.
|
|
7
7
|
*
|
|
8
8
|
* How it works:
|
|
9
|
-
* - Every `intervalMs`,
|
|
10
|
-
* -
|
|
9
|
+
* - Every `intervalMs`, checks connection status of all tracked connections
|
|
10
|
+
* - Uses libp2p's connection state (no network overhead from pings)
|
|
11
|
+
* - If connection is not open, increments failure counter
|
|
11
12
|
* - After `failureThreshold` failures, marks connection as dead
|
|
12
13
|
* - Emits events for degraded/recovered/dead connections
|
|
13
14
|
* - Automatically removes dead children from hierarchy
|
|
14
15
|
* - Emits ParentDisconnectedEvent when parent dies (triggers reconnection)
|
|
15
16
|
*/
|
|
16
17
|
export class oConnectionHeartbeatManager extends oObject {
|
|
17
|
-
constructor(
|
|
18
|
+
constructor(node, config) {
|
|
18
19
|
super();
|
|
19
|
-
this.
|
|
20
|
-
this.hierarchyManager = hierarchyManager;
|
|
21
|
-
this.notificationManager = notificationManager;
|
|
22
|
-
this.address = address;
|
|
20
|
+
this.node = node;
|
|
23
21
|
this.config = config;
|
|
24
22
|
this.healthMap = new Map();
|
|
23
|
+
this.isRunning = false;
|
|
25
24
|
}
|
|
26
25
|
async start() {
|
|
27
26
|
if (!this.config.enabled) {
|
|
28
|
-
this.logger.debug('Connection
|
|
27
|
+
this.logger.debug('Connection health monitoring disabled');
|
|
29
28
|
return;
|
|
30
29
|
}
|
|
31
|
-
this.logger.info(`Starting connection
|
|
32
|
-
`
|
|
30
|
+
this.logger.info(`Starting connection health monitoring: interval=${this.config.intervalMs}ms, ` +
|
|
31
|
+
`threshold=${this.config.failureThreshold}`);
|
|
33
32
|
// Immediate first check
|
|
34
|
-
await this.
|
|
33
|
+
await this.performHealthCheckCycle();
|
|
35
34
|
// Schedule recurring checks
|
|
36
|
-
this.heartbeatInterval = setInterval(() => this.
|
|
35
|
+
this.heartbeatInterval = setInterval(() => this.performHealthCheckCycle(), this.config.intervalMs);
|
|
37
36
|
}
|
|
38
37
|
async stop() {
|
|
39
38
|
if (this.heartbeatInterval) {
|
|
@@ -42,38 +41,70 @@ export class oConnectionHeartbeatManager extends oObject {
|
|
|
42
41
|
}
|
|
43
42
|
this.healthMap.clear();
|
|
44
43
|
}
|
|
45
|
-
async
|
|
44
|
+
async performHealthCheckCycle() {
|
|
45
|
+
if (this.isRunning) {
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
this.isRunning = true;
|
|
46
49
|
const targets = [];
|
|
47
50
|
// Check if this is a leader node (no leader in hierarchy = we are leader)
|
|
48
|
-
const isLeaderNode = this.
|
|
51
|
+
const isLeaderNode = this.node.getLeaders().length === 0;
|
|
49
52
|
// Collect leader (if enabled and we're not the leader)
|
|
50
|
-
if (
|
|
51
|
-
const leaders = this.
|
|
53
|
+
if (!isLeaderNode) {
|
|
54
|
+
const leaders = this.node.getLeaders();
|
|
52
55
|
for (const leader of leaders) {
|
|
53
56
|
targets.push({ address: leader, role: 'leader' });
|
|
54
57
|
}
|
|
55
58
|
}
|
|
56
59
|
// Collect parent
|
|
57
60
|
if (this.config.checkParent && !isLeaderNode) {
|
|
58
|
-
|
|
59
|
-
|
|
61
|
+
// Use this.node.parent getter to get the current parent address with transports
|
|
62
|
+
// rather than getParents() which may have a stale reference
|
|
63
|
+
const parent = this.node.parent;
|
|
64
|
+
// make sure that we don't double check the leader
|
|
65
|
+
if (parent && parent?.toString() !== oAddress.leader().toString()) {
|
|
60
66
|
targets.push({ address: parent, role: 'parent' });
|
|
61
67
|
}
|
|
62
68
|
}
|
|
63
69
|
// Collect children
|
|
64
70
|
if (this.config.checkChildren) {
|
|
65
|
-
const children = this.
|
|
71
|
+
const children = this.node.getChildren();
|
|
66
72
|
for (const child of children) {
|
|
67
73
|
targets.push({ address: child, role: 'child' });
|
|
68
74
|
}
|
|
69
75
|
}
|
|
70
|
-
//
|
|
71
|
-
await Promise.allSettled(targets.map((target) => this.
|
|
76
|
+
// Check all targets in parallel
|
|
77
|
+
await Promise.allSettled(targets.map((target) => this.checkTarget(target.address, target.role)));
|
|
78
|
+
this.isRunning = false;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Check if a connection to the given address is open by examining libp2p's connection state
|
|
82
|
+
* @returns true if an open connection exists, false otherwise
|
|
83
|
+
*/
|
|
84
|
+
checkConnectionStatus(address) {
|
|
85
|
+
if (address.toString() === this.node.address.toString()) {
|
|
86
|
+
return true; // Self-connection is always "open"
|
|
87
|
+
}
|
|
88
|
+
if (!address.libp2pTransports.length) {
|
|
89
|
+
return false;
|
|
90
|
+
}
|
|
91
|
+
try {
|
|
92
|
+
// Get peer ID from the address
|
|
93
|
+
const peerIdString = address.libp2pTransports[0].toPeerId();
|
|
94
|
+
// Get all connections to this peer from libp2p
|
|
95
|
+
// Note: Using 'as any' since converting to proper PeerId breaks browser implementation
|
|
96
|
+
const connections = this.node.p2pNode.getConnections(peerIdString);
|
|
97
|
+
// Check if any connection is open
|
|
98
|
+
return connections.some((conn) => conn.status === 'open');
|
|
99
|
+
}
|
|
100
|
+
catch (error) {
|
|
101
|
+
this.logger.debug(`Error checking connection status for ${address}`, error);
|
|
102
|
+
return false;
|
|
103
|
+
}
|
|
72
104
|
}
|
|
73
|
-
async
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
this.logger.warn(`Cannot extract peerId from ${address}`);
|
|
105
|
+
async checkTarget(address, role) {
|
|
106
|
+
if (!address.libp2pTransports.length) {
|
|
107
|
+
this.logger.debug(`${role} has no transports, skipping health check`, address);
|
|
77
108
|
return;
|
|
78
109
|
}
|
|
79
110
|
const key = address.toString();
|
|
@@ -81,7 +112,7 @@ export class oConnectionHeartbeatManager extends oObject {
|
|
|
81
112
|
if (!health) {
|
|
82
113
|
health = {
|
|
83
114
|
address,
|
|
84
|
-
peerId,
|
|
115
|
+
peerId: 'unknown',
|
|
85
116
|
lastSuccessfulPing: 0,
|
|
86
117
|
consecutiveFailures: 0,
|
|
87
118
|
averageLatency: 0,
|
|
@@ -89,44 +120,30 @@ export class oConnectionHeartbeatManager extends oObject {
|
|
|
89
120
|
};
|
|
90
121
|
this.healthMap.set(key, health);
|
|
91
122
|
}
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
//
|
|
96
|
-
const timeoutPromise = new Promise((_, reject) => {
|
|
97
|
-
setTimeout(() => reject(new Error('Ping timeout')), this.config.timeoutMs);
|
|
98
|
-
});
|
|
99
|
-
// Race between ping and timeout
|
|
100
|
-
// The ping service accepts PeerId as string or object
|
|
101
|
-
await Promise.race([
|
|
102
|
-
this.p2pNode.services.ping.ping(peerId),
|
|
103
|
-
timeoutPromise,
|
|
104
|
-
]);
|
|
105
|
-
const latency = Date.now() - startTime;
|
|
106
|
-
// Success - update health
|
|
123
|
+
// Check connection status using libp2p's connection state
|
|
124
|
+
const isOpen = this.checkConnectionStatus(address);
|
|
125
|
+
if (isOpen) {
|
|
126
|
+
// Connection is open - success
|
|
107
127
|
health.lastSuccessfulPing = Date.now();
|
|
108
128
|
health.consecutiveFailures = 0;
|
|
109
|
-
health.averageLatency =
|
|
110
|
-
health.averageLatency === 0
|
|
111
|
-
? latency
|
|
112
|
-
: health.averageLatency * 0.7 + latency * 0.3; // Exponential moving average
|
|
113
129
|
const previousStatus = health.status;
|
|
114
130
|
health.status = 'healthy';
|
|
115
131
|
// Emit recovery event if was degraded
|
|
116
132
|
if (previousStatus === 'degraded') {
|
|
117
|
-
this.logger.info(`Connection recovered: ${address}
|
|
133
|
+
this.logger.info(`Connection recovered: ${address}`);
|
|
118
134
|
this.emitConnectionRecoveredEvent(address, role);
|
|
119
135
|
}
|
|
120
|
-
this.logger.debug(`Ping successful: ${address} (${latency}ms)`);
|
|
121
136
|
}
|
|
122
|
-
|
|
137
|
+
else {
|
|
138
|
+
// Connection is not open
|
|
123
139
|
health.consecutiveFailures++;
|
|
124
|
-
this.logger.warn(`
|
|
140
|
+
this.logger.warn(`Connection check failed: ${address} (failures: ${health.consecutiveFailures}/${this.config.failureThreshold})`);
|
|
125
141
|
// Update status based on failure count
|
|
126
142
|
if (health.consecutiveFailures >= this.config.failureThreshold) {
|
|
127
143
|
this.handleConnectionDead(address, role, health);
|
|
128
144
|
}
|
|
129
|
-
else if (health.consecutiveFailures >=
|
|
145
|
+
else if (health.consecutiveFailures >=
|
|
146
|
+
Math.ceil(this.config.failureThreshold / 2)) {
|
|
130
147
|
health.status = 'degraded';
|
|
131
148
|
this.emitConnectionDegradedEvent(address, role, health.consecutiveFailures);
|
|
132
149
|
}
|
|
@@ -140,20 +157,20 @@ export class oConnectionHeartbeatManager extends oObject {
|
|
|
140
157
|
// Emit events based on role
|
|
141
158
|
if (role === 'child') {
|
|
142
159
|
// Remove dead child from hierarchy
|
|
143
|
-
this.
|
|
160
|
+
this.node.removeChild(address);
|
|
144
161
|
// Emit child left event
|
|
145
|
-
this.notificationManager.emit(new ChildLeftEvent({
|
|
146
|
-
source: this.address,
|
|
162
|
+
this.node.notificationManager.emit(new ChildLeftEvent({
|
|
163
|
+
source: this.node.address,
|
|
147
164
|
childAddress: address,
|
|
148
|
-
parentAddress: this.address,
|
|
165
|
+
parentAddress: this.node.address,
|
|
149
166
|
reason: `heartbeat_failed_${health.consecutiveFailures}_times`,
|
|
150
167
|
}));
|
|
151
168
|
this.logger.warn(`Removed dead child: ${address}`);
|
|
152
169
|
}
|
|
153
170
|
else if (role === 'parent') {
|
|
154
171
|
// Emit parent disconnected event
|
|
155
|
-
this.notificationManager.emit(new ParentDisconnectedEvent({
|
|
156
|
-
source: this.address,
|
|
172
|
+
this.node.notificationManager.emit(new ParentDisconnectedEvent({
|
|
173
|
+
source: this.node.address,
|
|
157
174
|
parentAddress: address,
|
|
158
175
|
reason: `heartbeat_failed_${health.consecutiveFailures}_times`,
|
|
159
176
|
}));
|
|
@@ -162,8 +179,8 @@ export class oConnectionHeartbeatManager extends oObject {
|
|
|
162
179
|
}
|
|
163
180
|
else if (role === 'leader') {
|
|
164
181
|
// Emit leader disconnected event
|
|
165
|
-
this.notificationManager.emit(new LeaderDisconnectedEvent({
|
|
166
|
-
source: this.address,
|
|
182
|
+
this.node.notificationManager.emit(new LeaderDisconnectedEvent({
|
|
183
|
+
source: this.node.address,
|
|
167
184
|
leaderAddress: address,
|
|
168
185
|
reason: `heartbeat_failed_${health.consecutiveFailures}_times`,
|
|
169
186
|
}));
|
|
@@ -174,8 +191,8 @@ export class oConnectionHeartbeatManager extends oObject {
|
|
|
174
191
|
emitConnectionDegradedEvent(address, role, failures) {
|
|
175
192
|
// ConnectionDegradedEvent only supports parent/child, so we map leader to parent
|
|
176
193
|
const eventRole = role === 'leader' ? 'parent' : role === 'child' ? 'child' : 'parent';
|
|
177
|
-
this.notificationManager.emit(new ConnectionDegradedEvent({
|
|
178
|
-
source: this.address,
|
|
194
|
+
this.node.notificationManager.emit(new ConnectionDegradedEvent({
|
|
195
|
+
source: this.node.address,
|
|
179
196
|
targetAddress: address,
|
|
180
197
|
role: eventRole,
|
|
181
198
|
consecutiveFailures: failures,
|
|
@@ -184,24 +201,12 @@ export class oConnectionHeartbeatManager extends oObject {
|
|
|
184
201
|
emitConnectionRecoveredEvent(address, role) {
|
|
185
202
|
// ConnectionRecoveredEvent only supports parent/child, so we map leader to parent
|
|
186
203
|
const eventRole = role === 'leader' ? 'parent' : role === 'child' ? 'child' : 'parent';
|
|
187
|
-
this.notificationManager.emit(new ConnectionRecoveredEvent({
|
|
188
|
-
source: this.address,
|
|
204
|
+
this.node.notificationManager.emit(new ConnectionRecoveredEvent({
|
|
205
|
+
source: this.node.address,
|
|
189
206
|
targetAddress: address,
|
|
190
207
|
role: eventRole,
|
|
191
208
|
}));
|
|
192
209
|
}
|
|
193
|
-
extractPeerIdFromAddress(address) {
|
|
194
|
-
// Extract peerId from transport multiaddr
|
|
195
|
-
for (const transport of address.transports) {
|
|
196
|
-
const multiaddr = transport.toString();
|
|
197
|
-
// Multiaddr format: /ip4/127.0.0.1/tcp/4001/p2p/QmPeerId
|
|
198
|
-
const parts = multiaddr.split('/p2p/');
|
|
199
|
-
if (parts.length === 2) {
|
|
200
|
-
return parts[1];
|
|
201
|
-
}
|
|
202
|
-
}
|
|
203
|
-
return null;
|
|
204
|
-
}
|
|
205
210
|
/**
|
|
206
211
|
* Get current health status of all connections
|
|
207
212
|
*/
|
|
@@ -6,6 +6,8 @@ export interface ReconnectionConfig {
|
|
|
6
6
|
baseDelayMs: number;
|
|
7
7
|
maxDelayMs: number;
|
|
8
8
|
useLeaderFallback: boolean;
|
|
9
|
+
parentDiscoveryIntervalMs: number;
|
|
10
|
+
parentDiscoveryMaxDelayMs: number;
|
|
9
11
|
}
|
|
10
12
|
/**
|
|
11
13
|
* Reconnection Manager
|
|
@@ -25,12 +27,22 @@ export declare class oReconnectionManager extends oObject {
|
|
|
25
27
|
private reconnecting;
|
|
26
28
|
constructor(node: IReconnectableNode, config: ReconnectionConfig);
|
|
27
29
|
private setupEventListeners;
|
|
30
|
+
handleNodeConnected(event: any): Promise<void>;
|
|
28
31
|
private handleConnectionDegraded;
|
|
29
32
|
private handleLeaderDisconnected;
|
|
30
33
|
private handleParentDisconnected;
|
|
31
34
|
attemptReconnection(): Promise<void>;
|
|
32
35
|
private tryDirectParentReconnection;
|
|
33
36
|
private tryLeaderFallback;
|
|
37
|
+
/**
|
|
38
|
+
* Wait for leader to become available and reconnect
|
|
39
|
+
* Leader transports are static (configured), so we just need to detect when it's back
|
|
40
|
+
*/
|
|
41
|
+
private waitForLeaderAndReconnect;
|
|
42
|
+
/**
|
|
43
|
+
* Wait for non-leader parent to appear in registry and reconnect
|
|
44
|
+
*/
|
|
45
|
+
waitForParentAndReconnect(): Promise<void>;
|
|
34
46
|
private handleReconnectionFailure;
|
|
35
47
|
private calculateNodeLevel;
|
|
36
48
|
private calculateBackoffDelay;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"o-reconnection.manager.d.ts","sourceRoot":"","sources":["../../../src/managers/o-reconnection.manager.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,OAAO,
|
|
1
|
+
{"version":3,"file":"o-reconnection.manager.d.ts","sourceRoot":"","sources":["../../../src/managers/o-reconnection.manager.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,OAAO,EASR,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,kBAAkB,EAAE,MAAM,uCAAuC,CAAC;AAI3E,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,yBAAyB,EAAE,MAAM,CAAC;IAClC,yBAAyB,EAAE,MAAM,CAAC;CACnC;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,oBAAqB,SAAQ,OAAO;IAI7C,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,MAAM;IAJhB,OAAO,CAAC,YAAY,CAAS;gBAGnB,IAAI,EAAE,kBAAkB,EACxB,MAAM,EAAE,kBAAkB;IAMpC,OAAO,CAAC,mBAAmB;IAyBrB,mBAAmB,CAAC,KAAK,EAAE,GAAG;YAatB,wBAAwB;YAaxB,wBAAwB;YAexB,wBAAwB;IAehC,mBAAmB;YAgDX,2BAA2B;YAiB3B,iBAAiB;IAkB/B;;;OAGG;YACW,yBAAyB;IAiFvC;;OAEG;IACG,yBAAyB;IAkG/B,OAAO,CAAC,yBAAyB;IAajC,OAAO,CAAC,kBAAkB;IAI1B,OAAO,CAAC,qBAAqB;IAK7B,OAAO,CAAC,KAAK;CAGd"}
|
|
@@ -28,6 +28,17 @@ export class oReconnectionManager extends oObject {
|
|
|
28
28
|
this.node.notificationManager.on('leader:disconnected', this.handleLeaderDisconnected.bind(this));
|
|
29
29
|
// Listen for connection degradation as early warning
|
|
30
30
|
this.node.notificationManager.on('connection:degraded', this.handleConnectionDegraded.bind(this));
|
|
31
|
+
this.node.notificationManager.on('node:connected', this.handleNodeConnected.bind(this));
|
|
32
|
+
}
|
|
33
|
+
async handleNodeConnected(event) {
|
|
34
|
+
const connectedEvent = event;
|
|
35
|
+
if (connectedEvent.nodeAddress.toString() === oAddress.leader().toString()) {
|
|
36
|
+
// the leader is back online, let's re-register & tell sub-graphs to re-register
|
|
37
|
+
await this.node.useSelf({
|
|
38
|
+
method: 'register_leader',
|
|
39
|
+
params: {},
|
|
40
|
+
});
|
|
41
|
+
}
|
|
31
42
|
}
|
|
32
43
|
async handleConnectionDegraded(event) {
|
|
33
44
|
const degradedEvent = event;
|
|
@@ -104,31 +115,136 @@ export class oReconnectionManager extends oObject {
|
|
|
104
115
|
this.logger.info('Direct parent reconnection successful');
|
|
105
116
|
}
|
|
106
117
|
async tryLeaderFallback() {
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
118
|
+
// Check if parent is the leader - special case
|
|
119
|
+
const parentIsLeader = this.node.config.parent?.toString() === oAddress.leader().toString();
|
|
120
|
+
if (parentIsLeader) {
|
|
121
|
+
this.logger.info('Parent is the leader - waiting for leader to become available');
|
|
122
|
+
await this.waitForLeaderAndReconnect();
|
|
123
|
+
}
|
|
124
|
+
else {
|
|
125
|
+
this.logger.info('Starting infinite parent discovery via leader registry');
|
|
126
|
+
await this.waitForParentAndReconnect();
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Wait for leader to become available and reconnect
|
|
131
|
+
* Leader transports are static (configured), so we just need to detect when it's back
|
|
132
|
+
*/
|
|
133
|
+
async waitForLeaderAndReconnect() {
|
|
134
|
+
const startTime = Date.now();
|
|
135
|
+
let attempt = 0;
|
|
136
|
+
let currentDelay = this.config.parentDiscoveryIntervalMs;
|
|
137
|
+
// Infinite retry loop - keep trying until leader is back
|
|
138
|
+
while (true) {
|
|
139
|
+
attempt++;
|
|
140
|
+
const elapsedMinutes = Math.floor((Date.now() - startTime) / 60000);
|
|
141
|
+
this.logger.info(`Leader discovery attempt ${attempt} (elapsed: ${elapsedMinutes}m)`);
|
|
142
|
+
try {
|
|
143
|
+
// Try to ping the leader using its configured transports
|
|
144
|
+
// The leader address should already have transports configured
|
|
145
|
+
await this.node.use(this.node.config.parent, {
|
|
146
|
+
method: 'ping',
|
|
147
|
+
params: {},
|
|
148
|
+
});
|
|
149
|
+
// Leader is back! Now re-register with parent and registry
|
|
150
|
+
this.logger.info(`Leader is back online after ${elapsedMinutes}m, re-registering...`);
|
|
151
|
+
try {
|
|
152
|
+
// Register with parent (leader)
|
|
153
|
+
await this.node.registerParent();
|
|
154
|
+
// Force re-registration with registry by resetting the flag
|
|
155
|
+
this.node.didRegister = false;
|
|
156
|
+
await this.node.register();
|
|
157
|
+
// Success!
|
|
158
|
+
this.reconnecting = false;
|
|
159
|
+
this.logger.info(`Successfully reconnected to leader and re-registered after ${elapsedMinutes}m`);
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
162
|
+
catch (registrationError) {
|
|
163
|
+
this.logger.warn('Leader online but registration failed, will retry:', registrationError instanceof Error
|
|
164
|
+
? registrationError.message
|
|
165
|
+
: registrationError);
|
|
166
|
+
// Fall through to retry with backoff
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
catch (error) {
|
|
170
|
+
// Leader not yet available
|
|
171
|
+
this.logger.debug(`Leader not yet available (will retry): ${error instanceof Error ? error.message : error}`);
|
|
172
|
+
}
|
|
173
|
+
// Calculate backoff delay
|
|
174
|
+
const delay = Math.min(currentDelay, this.config.parentDiscoveryMaxDelayMs);
|
|
175
|
+
// Log periodic status updates (every 5 minutes)
|
|
176
|
+
if (attempt % 30 === 0) {
|
|
177
|
+
this.logger.info(`Still waiting for leader to come back online... (${elapsedMinutes}m elapsed, ${attempt} attempts)`);
|
|
126
178
|
}
|
|
179
|
+
this.logger.debug(`Waiting ${delay}ms before next discovery attempt...`);
|
|
180
|
+
await this.sleep(delay);
|
|
181
|
+
// Exponential backoff for next iteration
|
|
182
|
+
currentDelay = Math.min(currentDelay * 2, this.config.parentDiscoveryMaxDelayMs);
|
|
127
183
|
}
|
|
128
|
-
|
|
129
|
-
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* Wait for non-leader parent to appear in registry and reconnect
|
|
187
|
+
*/
|
|
188
|
+
async waitForParentAndReconnect() {
|
|
189
|
+
const startTime = Date.now();
|
|
190
|
+
let attempt = 0;
|
|
191
|
+
let currentDelay = this.config.parentDiscoveryIntervalMs;
|
|
192
|
+
// Infinite retry loop - keep trying until parent is found
|
|
193
|
+
while (true) {
|
|
194
|
+
attempt++;
|
|
195
|
+
const elapsedMinutes = Math.floor((Date.now() - startTime) / 60000);
|
|
196
|
+
this.logger.info(`Parent discovery attempt ${attempt} (elapsed: ${elapsedMinutes}m)`);
|
|
197
|
+
try {
|
|
198
|
+
// Query registry for parent by its known address
|
|
199
|
+
const response = await this.node.use(oAddress.registry(), {
|
|
200
|
+
method: 'find_available_parent',
|
|
201
|
+
params: {
|
|
202
|
+
parentAddress: this.node.config.parent?.toString(),
|
|
203
|
+
},
|
|
204
|
+
});
|
|
205
|
+
const { parentAddress, parentTransports } = response.result.data;
|
|
206
|
+
// Check if parent was found in registry
|
|
207
|
+
if (parentAddress && parentTransports && parentTransports.length > 0) {
|
|
208
|
+
this.logger.info(`Parent found in registry: ${parentAddress} with ${parentTransports.length} transports`);
|
|
209
|
+
// Update parent reference with fresh transports
|
|
210
|
+
this.node.config.parent = new oNodeAddress(parentAddress, parentTransports.map((t) => new oNodeTransport(t.value)));
|
|
211
|
+
// Attempt to register with parent and re-register with registry
|
|
212
|
+
try {
|
|
213
|
+
await this.tryDirectParentReconnection();
|
|
214
|
+
// Force re-registration with registry by resetting the flag
|
|
215
|
+
this.node.didRegister = false;
|
|
216
|
+
await this.node.register();
|
|
217
|
+
// Success!
|
|
218
|
+
this.reconnecting = false;
|
|
219
|
+
this.logger.info(`Successfully reconnected to parent and re-registered after ${elapsedMinutes}m of discovery attempts`);
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
222
|
+
catch (registrationError) {
|
|
223
|
+
this.logger.warn('Parent found but registration failed, will retry:', registrationError instanceof Error
|
|
224
|
+
? registrationError.message
|
|
225
|
+
: registrationError);
|
|
226
|
+
// Fall through to retry with backoff
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
else {
|
|
230
|
+
this.logger.debug(`Parent not yet available in registry: ${this.node.config.parent?.toString()}`);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
catch (error) {
|
|
234
|
+
// Network error communicating with leader/registry
|
|
235
|
+
this.logger.warn('Error querying registry for parent (will retry):', error instanceof Error ? error.message : error);
|
|
236
|
+
}
|
|
237
|
+
// Calculate backoff delay with exponential increase, capped at max
|
|
238
|
+
const delay = Math.min(currentDelay, this.config.parentDiscoveryMaxDelayMs);
|
|
239
|
+
// Log periodic status updates (every 5 minutes)
|
|
240
|
+
if (attempt % 30 === 0) {
|
|
241
|
+
this.logger.info(`Still waiting for parent to appear in registry... (${elapsedMinutes}m elapsed, ${attempt} attempts)`);
|
|
242
|
+
}
|
|
243
|
+
this.logger.debug(`Waiting ${delay}ms before next discovery attempt...`);
|
|
244
|
+
await this.sleep(delay);
|
|
245
|
+
// Exponential backoff for next iteration
|
|
246
|
+
currentDelay = Math.min(currentDelay * 2, this.config.parentDiscoveryMaxDelayMs);
|
|
130
247
|
}
|
|
131
|
-
this.handleReconnectionFailure();
|
|
132
248
|
}
|
|
133
249
|
handleReconnectionFailure() {
|
|
134
250
|
this.reconnecting = false;
|
package/dist/src/o-node.d.ts
CHANGED
|
@@ -3,14 +3,14 @@ import { PeerId } from '@olane/o-config';
|
|
|
3
3
|
import { oNodeHierarchyManager } from './o-node.hierarchy-manager.js';
|
|
4
4
|
import { oNodeConfig } from './interfaces/o-node.config.js';
|
|
5
5
|
import { oNodeTransport } from './router/o-node.transport.js';
|
|
6
|
-
import {
|
|
6
|
+
import { oRequest, oNotificationManager } from '@olane/o-core';
|
|
7
7
|
import { oNodeAddress } from './router/o-node.address.js';
|
|
8
8
|
import { oNodeConnection } from './connection/o-node-connection.js';
|
|
9
9
|
import { oNodeConnectionManager } from './connection/o-node-connection.manager.js';
|
|
10
10
|
import { oToolBase } from '@olane/o-tool';
|
|
11
11
|
import { oConnectionHeartbeatManager } from './managers/o-connection-heartbeat.manager.js';
|
|
12
|
+
import { oNodeConnectionConfig } from './connection/index.js';
|
|
12
13
|
import { oReconnectionManager } from './managers/o-reconnection.manager.js';
|
|
13
|
-
import { LeaderRequestWrapper } from './utils/leader-request-wrapper.js';
|
|
14
14
|
export declare class oNode extends oToolBase {
|
|
15
15
|
peerId: PeerId;
|
|
16
16
|
p2pNode: Libp2p;
|
|
@@ -19,8 +19,7 @@ export declare class oNode extends oToolBase {
|
|
|
19
19
|
connectionManager: oNodeConnectionManager;
|
|
20
20
|
hierarchyManager: oNodeHierarchyManager;
|
|
21
21
|
connectionHeartbeatManager?: oConnectionHeartbeatManager;
|
|
22
|
-
reconnectionManager?: oReconnectionManager;
|
|
23
|
-
leaderRequestWrapper: LeaderRequestWrapper;
|
|
22
|
+
protected reconnectionManager?: oReconnectionManager;
|
|
24
23
|
protected didRegister: boolean;
|
|
25
24
|
constructor(config: oNodeConfig);
|
|
26
25
|
get leader(): oNodeAddress | null;
|
|
@@ -33,7 +32,9 @@ export declare class oNode extends oToolBase {
|
|
|
33
32
|
get parentTransports(): oNodeTransport[];
|
|
34
33
|
get transports(): oNodeTransport[];
|
|
35
34
|
unregister(): Promise<void>;
|
|
35
|
+
setKeepAliveTag(address: oNodeAddress): Promise<void>;
|
|
36
36
|
registerParent(): Promise<void>;
|
|
37
|
+
registerLeader(): Promise<void>;
|
|
37
38
|
register(): Promise<void>;
|
|
38
39
|
extractMethod(address: oNodeAddress): string;
|
|
39
40
|
start(): Promise<void>;
|
|
@@ -44,18 +45,28 @@ export declare class oNode extends oToolBase {
|
|
|
44
45
|
*/
|
|
45
46
|
configure(): Promise<Libp2pConfig>;
|
|
46
47
|
protected createNode(): Promise<Libp2p>;
|
|
47
|
-
connect(
|
|
48
|
+
connect(config: oNodeConnectionConfig): Promise<oNodeConnection>;
|
|
49
|
+
initConnectionManager(): Promise<void>;
|
|
50
|
+
hookInitializeFinished(): Promise<void>;
|
|
51
|
+
hookStartFinished(): Promise<void>;
|
|
48
52
|
initialize(): Promise<void>;
|
|
49
53
|
/**
|
|
50
54
|
* Override use() to wrap leader/registry requests with retry logic
|
|
51
55
|
*/
|
|
52
|
-
use(address: oAddress, data?: {
|
|
53
|
-
method?: string;
|
|
54
|
-
params?: {
|
|
55
|
-
[key: string]: any;
|
|
56
|
-
};
|
|
57
|
-
id?: string;
|
|
58
|
-
}): Promise<any>;
|
|
59
56
|
teardown(): Promise<void>;
|
|
57
|
+
getLeaders(): oNodeAddress[];
|
|
58
|
+
getParents(): oNodeAddress[];
|
|
59
|
+
getChildren(): oNodeAddress[];
|
|
60
|
+
removeChild(childAddress: oNodeAddress): void;
|
|
61
|
+
/**
|
|
62
|
+
* Get the total number of active streams across all connections
|
|
63
|
+
* @returns Total count of active streams
|
|
64
|
+
*/
|
|
65
|
+
getStreamCount(): number;
|
|
66
|
+
/**
|
|
67
|
+
* Get libp2p metrics for this node
|
|
68
|
+
* Tool method that can be called remotely by monitoring systems
|
|
69
|
+
*/
|
|
70
|
+
_tool_get_libp2p_metrics(request: oRequest): Promise<any>;
|
|
60
71
|
}
|
|
61
72
|
//# sourceMappingURL=o-node.d.ts.map
|
package/dist/src/o-node.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"o-node.d.ts","sourceRoot":"","sources":["../../src/o-node.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"o-node.d.ts","sourceRoot":"","sources":["../../src/o-node.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,MAAM,EACN,YAAY,EAEb,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AACtE,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAKL,QAAQ,EAER,oBAAoB,EAGrB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,mCAAmC,CAAC;AACpE,OAAO,EAAE,sBAAsB,EAAE,MAAM,2CAA2C,CAAC;AAEnF,OAAO,EAAmB,SAAS,EAAE,MAAM,eAAe,CAAC;AAG3D,OAAO,EAAE,2BAA2B,EAAE,MAAM,8CAA8C,CAAC;AAC3F,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,oBAAoB,EAAE,MAAM,sCAAsC,CAAC;AAE5E,qBAAa,KAAM,SAAQ,SAAS;IAC3B,MAAM,EAAG,MAAM,CAAC;IAChB,OAAO,EAAG,MAAM,CAAC;IACjB,OAAO,EAAG,YAAY,CAAC;IACvB,MAAM,EAAE,WAAW,CAAC;IACpB,iBAAiB,EAAG,sBAAsB,CAAC;IAC3C,gBAAgB,EAAG,qBAAqB,CAAC;IACzC,0BAA0B,CAAC,EAAE,2BAA2B,CAAC;IAChE,SAAS,CAAC,mBAAmB,CAAC,EAAE,oBAAoB,CAAC;IACrD,SAAS,CAAC,WAAW,EAAE,OAAO,CAAS;gBAE3B,MAAM,EAAE,WAAW;IAK/B,IAAI,MAAM,IAAI,YAAY,GAAG,IAAI,CAEhC;IAED,IAAI,aAAa,IAAI,YAAY,CAKhC;IAED,IAAI,YAAY,IAAI,MAAM,GAAG,IAAI,CAOhC;IAED,mBAAmB,IAAI,GAAG,EAAE;IAItB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC;IASvC,SAAS,CAAC,yBAAyB,IAAI,oBAAoB;IAQ3D,IAAI,aAAa,IAAI,YAAY,CAEhC;IAED,IAAI,gBAAgB,IAAI,cAAc,EAAE,CAEvC;IAED,IAAI,UAAU,IAAI,cAAc,EAAE,CAIjC;IAEK,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAwD3B,eAAe,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAoCrD,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAsC/B,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAmB/B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IA4B/B,aAAa,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IAItC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAStB,mBAAmB,CAAC,OAAO,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC;IAG1D;;;OAGG;IACG,SAAS,IAAI,OAAO,CAAC,YAAY,CAAC;cAkIxB,UAAU,IAAI,OAAO,CAAC,MAAM,CAAC;IAMvC,OAAO,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAAC,eAAe,CAAC;IAsBhE,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC;IAStC,sBAAsB,IAAI,OAAO,CAAC,IAAI,CAAC;IAEvC,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC;IAqBlC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IA+CjC;;OAEG;IAiBG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAc/B,UAAU,IAAI,YAAY,EAAE;IAI5B,UAAU,IAAI,YAAY,EAAE;IAI5B,WAAW,IAAI,YAAY,EAAE;IAI7B,WAAW,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI;IAI7C;;;OAGG;IACH,cAAc,IAAI,MAAM;IAUxB;;;OAGG;IACG,wBAAwB,CAAC,OAAO,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC;CAwEhE"}
|