@olane/o-node 0.7.12-alpha.4 → 0.7.12-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/connection/o-node-connection.d.ts.map +1 -1
- package/dist/src/connection/o-node-connection.js +8 -2
- package/dist/src/connection/o-node-connection.manager.d.ts +1 -1
- package/dist/src/connection/o-node-connection.manager.d.ts.map +1 -1
- package/dist/src/connection/o-node-connection.manager.js +39 -19
- package/dist/src/interfaces/i-reconnectable-node.d.ts +41 -0
- package/dist/src/interfaces/i-reconnectable-node.d.ts.map +1 -0
- package/dist/src/interfaces/i-reconnectable-node.js +1 -0
- package/dist/src/interfaces/o-node.config.d.ts +35 -0
- package/dist/src/interfaces/o-node.config.d.ts.map +1 -1
- package/dist/src/managers/o-connection-heartbeat.manager.d.ts +67 -0
- package/dist/src/managers/o-connection-heartbeat.manager.d.ts.map +1 -0
- package/dist/src/managers/o-connection-heartbeat.manager.js +223 -0
- package/dist/src/managers/o-reconnection.manager.d.ts +39 -0
- package/dist/src/managers/o-reconnection.manager.d.ts.map +1 -0
- package/dist/src/managers/o-reconnection.manager.js +150 -0
- package/dist/src/o-node.d.ts +18 -1
- package/dist/src/o-node.d.ts.map +1 -1
- package/dist/src/o-node.js +82 -0
- package/dist/src/o-node.notification-manager.d.ts +52 -0
- package/dist/src/o-node.notification-manager.d.ts.map +1 -0
- package/dist/src/o-node.notification-manager.js +183 -0
- package/dist/src/o-node.tool.d.ts.map +1 -1
- package/dist/src/o-node.tool.js +19 -4
- package/dist/src/utils/leader-request-wrapper.d.ts +45 -0
- package/dist/src/utils/leader-request-wrapper.d.ts.map +1 -0
- package/dist/src/utils/leader-request-wrapper.js +89 -0
- package/package.json +6 -6
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"o-node-connection.d.ts","sourceRoot":"","sources":["../../../src/connection/o-node-connection.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,UAAU,EAIV,MAAM,EAEP,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,WAAW,EAGX,QAAQ,EACR,SAAS,EACV,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,qBAAqB,EAAE,MAAM,0CAA0C,CAAC;AAEjF,qBAAa,eAAgB,SAAQ,WAAW;IAGlC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,qBAAqB;IAFrD,aAAa,EAAE,UAAU,CAAC;gBAEF,MAAM,EAAE,qBAAqB;IAKtD,IAAI,CAAC,MAAM,EAAE,MAAM;IAWzB,QAAQ;IAOF,QAAQ,CAAC,OAAO,EAAE,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"o-node-connection.d.ts","sourceRoot":"","sources":["../../../src/connection/o-node-connection.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,UAAU,EAIV,MAAM,EAEP,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,WAAW,EAGX,QAAQ,EACR,SAAS,EACV,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,qBAAqB,EAAE,MAAM,0CAA0C,CAAC;AAEjF,qBAAa,eAAgB,SAAQ,WAAW;IAGlC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,qBAAqB;IAFrD,aAAa,EAAE,UAAU,CAAC;gBAEF,MAAM,EAAE,qBAAqB;IAKtD,IAAI,CAAC,MAAM,EAAE,MAAM;IAWzB,QAAQ;IAOF,QAAQ,CAAC,OAAO,EAAE,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC;IAiD/C,KAAK;CAKZ"}
|
|
@@ -33,8 +33,14 @@ export class oNodeConnection extends oConnection {
|
|
|
33
33
|
if (stream.status === 'reset') {
|
|
34
34
|
throw new oError(oErrorCodes.CONNECTION_LIMIT_REACHED, 'Connection limit reached');
|
|
35
35
|
}
|
|
36
|
-
// Send the data
|
|
37
|
-
|
|
36
|
+
// Send the data with backpressure handling (libp2p v3 best practice)
|
|
37
|
+
const data = new TextEncoder().encode(request.toString());
|
|
38
|
+
const sent = stream.send(data);
|
|
39
|
+
// If send() returns false, wait for the stream to drain before continuing
|
|
40
|
+
if (!sent) {
|
|
41
|
+
this.logger.debug('Stream buffer full, waiting for drain...');
|
|
42
|
+
await stream.onDrain({ signal: AbortSignal.timeout(30000) }); // 30 second timeout
|
|
43
|
+
}
|
|
38
44
|
const res = await this.read(stream);
|
|
39
45
|
await stream.close();
|
|
40
46
|
// process the response
|
|
@@ -6,7 +6,7 @@ export declare class oNodeConnectionManager extends oConnectionManager {
|
|
|
6
6
|
private p2pNode;
|
|
7
7
|
constructor(config: oNodeConnectionManagerConfig);
|
|
8
8
|
/**
|
|
9
|
-
* Connect to a given address
|
|
9
|
+
* Connect to a given address with exponential backoff retry
|
|
10
10
|
* @param address - The address to connect to
|
|
11
11
|
* @returns The connection object
|
|
12
12
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"o-node-connection.manager.d.ts","sourceRoot":"","sources":["../../../src/connection/o-node-connection.manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAChF,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C,OAAO,EAAE,4BAA4B,EAAE,MAAM,kDAAkD,CAAC;AAEhG,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAEzD,qBAAa,sBAAuB,SAAQ,kBAAkB;IAC5D,OAAO,CAAC,OAAO,CAAS;gBAEZ,MAAM,EAAE,4BAA4B;IAKhD;;;;OAIG;IACG,OAAO,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAAC,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"o-node-connection.manager.d.ts","sourceRoot":"","sources":["../../../src/connection/o-node-connection.manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAChF,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C,OAAO,EAAE,4BAA4B,EAAE,MAAM,kDAAkD,CAAC;AAEhG,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAEzD,qBAAa,sBAAuB,SAAQ,kBAAkB;IAC5D,OAAO,CAAC,OAAO,CAAS;gBAEZ,MAAM,EAAE,4BAA4B;IAKhD;;;;OAIG;IACG,OAAO,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAAC,eAAe,CAAC;IAqFlE,QAAQ,CAAC,OAAO,EAAE,QAAQ,GAAG,OAAO;IAIpC,mBAAmB,CAAC,OAAO,EAAE,QAAQ,GAAG,WAAW,GAAG,IAAI;CAe3D"}
|
|
@@ -6,7 +6,7 @@ export class oNodeConnectionManager extends oConnectionManager {
|
|
|
6
6
|
this.p2pNode = config.p2pNode;
|
|
7
7
|
}
|
|
8
8
|
/**
|
|
9
|
-
* Connect to a given address
|
|
9
|
+
* Connect to a given address with exponential backoff retry
|
|
10
10
|
* @param address - The address to connect to
|
|
11
11
|
* @returns The connection object
|
|
12
12
|
*/
|
|
@@ -26,25 +26,45 @@ export class oNodeConnectionManager extends oConnectionManager {
|
|
|
26
26
|
this.cache.delete(nextHopAddress.toString());
|
|
27
27
|
}
|
|
28
28
|
}
|
|
29
|
-
//
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
nextHopAddress
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
29
|
+
// Retry configuration for handling transient connection failures
|
|
30
|
+
const MAX_RETRIES = 3;
|
|
31
|
+
const BASE_DELAY_MS = 1000; // Start with 1 second
|
|
32
|
+
const MAX_DELAY_MS = 10000; // Cap at 10 seconds
|
|
33
|
+
// first time setup connection with retry logic
|
|
34
|
+
let lastError;
|
|
35
|
+
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
|
|
36
|
+
try {
|
|
37
|
+
if (attempt > 0) {
|
|
38
|
+
// Calculate exponential backoff delay: 1s, 2s, 4s, 8s (capped at MAX_DELAY_MS)
|
|
39
|
+
const delay = Math.min(BASE_DELAY_MS * Math.pow(2, attempt - 1), MAX_DELAY_MS);
|
|
40
|
+
this.logger.debug(`Retry attempt ${attempt}/${MAX_RETRIES} for ${nextHopAddress.toString()} after ${delay}ms delay`);
|
|
41
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
42
|
+
}
|
|
43
|
+
const p2pConnection = await this.p2pNode.dial(nextHopAddress.libp2pTransports.map((ma) => ma.toMultiaddr()));
|
|
44
|
+
const connection = new oNodeConnection({
|
|
45
|
+
nextHopAddress: nextHopAddress,
|
|
46
|
+
address: address,
|
|
47
|
+
p2pConnection: p2pConnection,
|
|
48
|
+
callerAddress: callerAddress,
|
|
49
|
+
});
|
|
50
|
+
if (attempt > 0) {
|
|
51
|
+
this.logger.info(`Successfully connected to ${nextHopAddress.toString()} on retry attempt ${attempt}`);
|
|
52
|
+
}
|
|
53
|
+
// this.cache.set(nextHopAddress.toString(), connection);
|
|
54
|
+
return connection;
|
|
55
|
+
}
|
|
56
|
+
catch (error) {
|
|
57
|
+
lastError = error;
|
|
58
|
+
this.logger.warn(`[${callerAddress?.toString() || 'unknown'}] Connection attempt ${attempt + 1}/${MAX_RETRIES + 1} failed for ${nextHopAddress.toString()}: ${error instanceof Error ? error.message : String(error)}`);
|
|
59
|
+
// Don't retry on the last attempt
|
|
60
|
+
if (attempt === MAX_RETRIES) {
|
|
61
|
+
break;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
47
64
|
}
|
|
65
|
+
// All retries exhausted
|
|
66
|
+
this.logger.error(`[${callerAddress?.toString() || 'unknown'}] Failed to connect after ${MAX_RETRIES + 1} attempts to address! Next hop: ${nextHopAddress} With Address: ${address.toString()}`, lastError);
|
|
67
|
+
throw lastError;
|
|
48
68
|
}
|
|
49
69
|
isCached(address) {
|
|
50
70
|
return this.cache.has(address.toString());
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { oAddress, NodeState, oNotificationManager } from '@olane/o-core';
|
|
2
|
+
import { oNodeAddress } from '../router/o-node.address.js';
|
|
3
|
+
import { oNodeConfig } from './o-node.config.js';
|
|
4
|
+
/**
|
|
5
|
+
* Interface for nodes that support reconnection management.
|
|
6
|
+
* This interface defines the contract that oReconnectionManager needs
|
|
7
|
+
* to perform reconnection operations without creating a circular dependency.
|
|
8
|
+
*/
|
|
9
|
+
export interface IReconnectableNode {
|
|
10
|
+
/**
|
|
11
|
+
* The node's configuration
|
|
12
|
+
*/
|
|
13
|
+
config: oNodeConfig;
|
|
14
|
+
/**
|
|
15
|
+
* The node's current address
|
|
16
|
+
*/
|
|
17
|
+
address: oNodeAddress;
|
|
18
|
+
/**
|
|
19
|
+
* The node's current state
|
|
20
|
+
*/
|
|
21
|
+
state: NodeState;
|
|
22
|
+
/**
|
|
23
|
+
* The notification manager for subscribing to events
|
|
24
|
+
*/
|
|
25
|
+
notificationManager: oNotificationManager;
|
|
26
|
+
/**
|
|
27
|
+
* Register with the parent node
|
|
28
|
+
*/
|
|
29
|
+
registerParent(): Promise<void>;
|
|
30
|
+
/**
|
|
31
|
+
* Execute a method on another node
|
|
32
|
+
*/
|
|
33
|
+
use(address: oAddress, data?: {
|
|
34
|
+
method?: string;
|
|
35
|
+
params?: {
|
|
36
|
+
[key: string]: any;
|
|
37
|
+
};
|
|
38
|
+
id?: string;
|
|
39
|
+
}): Promise<any>;
|
|
40
|
+
}
|
|
41
|
+
//# sourceMappingURL=i-reconnectable-node.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"i-reconnectable-node.d.ts","sourceRoot":"","sources":["../../../src/interfaces/i-reconnectable-node.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAC1E,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEjD;;;;GAIG;AACH,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,MAAM,EAAE,WAAW,CAAC;IAEpB;;OAEG;IACH,OAAO,EAAE,YAAY,CAAC;IAEtB;;OAEG;IACH,KAAK,EAAE,SAAS,CAAC;IAEjB;;OAEG;IACH,mBAAmB,EAAE,oBAAoB,CAAC;IAE1C;;OAEG;IACH,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEhC;;OAEG;IACH,GAAG,CACD,OAAO,EAAE,QAAQ,EACjB,IAAI,CAAC,EAAE;QACL,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,MAAM,CAAC,EAAE;YAAE,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;SAAE,CAAC;QAChC,EAAE,CAAC,EAAE,MAAM,CAAC;KACb,GACA,OAAO,CAAC,GAAG,CAAC,CAAC;CACjB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -3,5 +3,40 @@ import { oNodeAddress } from '../router/o-node.address.js';
|
|
|
3
3
|
export interface oNodeConfig extends oCoreConfig {
|
|
4
4
|
leader: oNodeAddress | null;
|
|
5
5
|
parent: oNodeAddress | null;
|
|
6
|
+
/**
|
|
7
|
+
* Connection heartbeat configuration (libp2p-native pings)
|
|
8
|
+
* Detects dead connections via periodic pings using libp2p's ping service
|
|
9
|
+
*/
|
|
10
|
+
connectionHeartbeat?: {
|
|
11
|
+
enabled?: boolean;
|
|
12
|
+
intervalMs?: number;
|
|
13
|
+
timeoutMs?: number;
|
|
14
|
+
failureThreshold?: number;
|
|
15
|
+
checkChildren?: boolean;
|
|
16
|
+
checkParent?: boolean;
|
|
17
|
+
checkLeader?: boolean;
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* Automatic reconnection configuration
|
|
21
|
+
* Handles parent connection failures and attempts to reconnect
|
|
22
|
+
*/
|
|
23
|
+
reconnection?: {
|
|
24
|
+
enabled?: boolean;
|
|
25
|
+
maxAttempts?: number;
|
|
26
|
+
baseDelayMs?: number;
|
|
27
|
+
maxDelayMs?: number;
|
|
28
|
+
useLeaderFallback?: boolean;
|
|
29
|
+
};
|
|
30
|
+
/**
|
|
31
|
+
* Leader request retry configuration
|
|
32
|
+
* Handles temporary leader unavailability (healing, maintenance)
|
|
33
|
+
*/
|
|
34
|
+
leaderRetry?: {
|
|
35
|
+
enabled?: boolean;
|
|
36
|
+
maxAttempts?: number;
|
|
37
|
+
baseDelayMs?: number;
|
|
38
|
+
maxDelayMs?: number;
|
|
39
|
+
timeoutMs?: number;
|
|
40
|
+
};
|
|
6
41
|
}
|
|
7
42
|
//# sourceMappingURL=o-node.config.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"o-node.config.d.ts","sourceRoot":"","sources":["../../../src/interfaces/o-node.config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAE3D,MAAM,WAAW,WAAY,SAAQ,WAAW;IAC9C,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"o-node.config.d.ts","sourceRoot":"","sources":["../../../src/interfaces/o-node.config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAE3D,MAAM,WAAW,WAAY,SAAQ,WAAW;IAC9C,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IAE5B;;;OAGG;IACH,mBAAmB,CAAC,EAAE;QACpB,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,aAAa,CAAC,EAAE,OAAO,CAAC;QACxB,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB,CAAC;IAEF;;;OAGG;IACH,YAAY,CAAC,EAAE;QACb,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,iBAAiB,CAAC,EAAE,OAAO,CAAC;KAC7B,CAAC;IAEF;;;OAGG;IACH,WAAW,CAAC,EAAE;QACZ,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACH"}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { Libp2p } from '@olane/o-config';
|
|
2
|
+
import { oObject } from '@olane/o-core';
|
|
3
|
+
import { oNodeAddress } from '../router/o-node.address.js';
|
|
4
|
+
import { oNodeHierarchyManager } from '../o-node.hierarchy-manager.js';
|
|
5
|
+
import { oNotificationManager } from '@olane/o-core';
|
|
6
|
+
export interface HeartbeatConfig {
|
|
7
|
+
enabled: boolean;
|
|
8
|
+
intervalMs: number;
|
|
9
|
+
timeoutMs: number;
|
|
10
|
+
failureThreshold: number;
|
|
11
|
+
checkChildren: boolean;
|
|
12
|
+
checkParent: boolean;
|
|
13
|
+
checkLeader: boolean;
|
|
14
|
+
}
|
|
15
|
+
export interface ConnectionHealth {
|
|
16
|
+
address: oNodeAddress;
|
|
17
|
+
peerId: string;
|
|
18
|
+
lastSuccessfulPing: number;
|
|
19
|
+
consecutiveFailures: number;
|
|
20
|
+
averageLatency: number;
|
|
21
|
+
status: 'healthy' | 'degraded' | 'dead';
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Connection Heartbeat Manager
|
|
25
|
+
*
|
|
26
|
+
* Uses libp2p's native ping service to detect dead connections early.
|
|
27
|
+
* Continuously pings parent and children to ensure they're alive.
|
|
28
|
+
*
|
|
29
|
+
* How it works:
|
|
30
|
+
* - Every `intervalMs`, pings all tracked connections
|
|
31
|
+
* - If ping fails, increments failure counter
|
|
32
|
+
* - After `failureThreshold` failures, marks connection as dead
|
|
33
|
+
* - Emits events for degraded/recovered/dead connections
|
|
34
|
+
* - Automatically removes dead children from hierarchy
|
|
35
|
+
* - Emits ParentDisconnectedEvent when parent dies (triggers reconnection)
|
|
36
|
+
*/
|
|
37
|
+
export declare class oConnectionHeartbeatManager extends oObject {
|
|
38
|
+
private p2pNode;
|
|
39
|
+
private hierarchyManager;
|
|
40
|
+
private notificationManager;
|
|
41
|
+
private address;
|
|
42
|
+
private config;
|
|
43
|
+
private heartbeatInterval?;
|
|
44
|
+
private healthMap;
|
|
45
|
+
constructor(p2pNode: Libp2p, hierarchyManager: oNodeHierarchyManager, notificationManager: oNotificationManager, address: oNodeAddress, config: HeartbeatConfig);
|
|
46
|
+
start(): Promise<void>;
|
|
47
|
+
stop(): Promise<void>;
|
|
48
|
+
private performHeartbeatCycle;
|
|
49
|
+
private pingTarget;
|
|
50
|
+
private handleConnectionDead;
|
|
51
|
+
private emitConnectionDegradedEvent;
|
|
52
|
+
private emitConnectionRecoveredEvent;
|
|
53
|
+
private extractPeerIdFromAddress;
|
|
54
|
+
/**
|
|
55
|
+
* Get current health status of all connections
|
|
56
|
+
*/
|
|
57
|
+
getHealthStatus(): ConnectionHealth[];
|
|
58
|
+
/**
|
|
59
|
+
* Get health status for specific address
|
|
60
|
+
*/
|
|
61
|
+
getConnectionHealth(address: oNodeAddress): ConnectionHealth | undefined;
|
|
62
|
+
/**
|
|
63
|
+
* Get current configuration
|
|
64
|
+
*/
|
|
65
|
+
getConfig(): HeartbeatConfig;
|
|
66
|
+
}
|
|
67
|
+
//# sourceMappingURL=o-connection-heartbeat.manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"o-connection-heartbeat.manager.d.ts","sourceRoot":"","sources":["../../../src/managers/o-connection-heartbeat.manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,OAAO,EACL,OAAO,EAMR,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAErD,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,EAAE,OAAO,CAAC;IACrB,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,YAAY,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,kBAAkB,EAAE,MAAM,CAAC;IAC3B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,SAAS,GAAG,UAAU,GAAG,MAAM,CAAC;CACzC;AAED;;;;;;;;;;;;;GAaG;AACH,qBAAa,2BAA4B,SAAQ,OAAO;IAKpD,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,gBAAgB;IACxB,OAAO,CAAC,mBAAmB;IAC3B,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,MAAM;IARhB,OAAO,CAAC,iBAAiB,CAAC,CAAiB;IAC3C,OAAO,CAAC,SAAS,CAAuC;gBAG9C,OAAO,EAAE,MAAM,EACf,gBAAgB,EAAE,qBAAqB,EACvC,mBAAmB,EAAE,oBAAoB,EACzC,OAAO,EAAE,YAAY,EACrB,MAAM,EAAE,eAAe;IAK3B,KAAK;IAqBL,IAAI;YAQI,qBAAqB;YAuCrB,UAAU;IAyFxB,OAAO,CAAC,oBAAoB;IAyD5B,OAAO,CAAC,2BAA2B;IAmBnC,OAAO,CAAC,4BAA4B;IAiBpC,OAAO,CAAC,wBAAwB;IAahC;;OAEG;IACH,eAAe,IAAI,gBAAgB,EAAE;IAIrC;;OAEG;IACH,mBAAmB,CAAC,OAAO,EAAE,YAAY,GAAG,gBAAgB,GAAG,SAAS;IAIxE;;OAEG;IACH,SAAS,IAAI,eAAe;CAG7B"}
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
import { oObject, ChildLeftEvent, ParentDisconnectedEvent, LeaderDisconnectedEvent, ConnectionDegradedEvent, ConnectionRecoveredEvent, } from '@olane/o-core';
|
|
2
|
+
/**
|
|
3
|
+
* Connection Heartbeat Manager
|
|
4
|
+
*
|
|
5
|
+
* Uses libp2p's native ping service to detect dead connections early.
|
|
6
|
+
* Continuously pings parent and children to ensure they're alive.
|
|
7
|
+
*
|
|
8
|
+
* How it works:
|
|
9
|
+
* - Every `intervalMs`, pings all tracked connections
|
|
10
|
+
* - If ping fails, increments failure counter
|
|
11
|
+
* - After `failureThreshold` failures, marks connection as dead
|
|
12
|
+
* - Emits events for degraded/recovered/dead connections
|
|
13
|
+
* - Automatically removes dead children from hierarchy
|
|
14
|
+
* - Emits ParentDisconnectedEvent when parent dies (triggers reconnection)
|
|
15
|
+
*/
|
|
16
|
+
export class oConnectionHeartbeatManager extends oObject {
|
|
17
|
+
constructor(p2pNode, hierarchyManager, notificationManager, address, config) {
|
|
18
|
+
super();
|
|
19
|
+
this.p2pNode = p2pNode;
|
|
20
|
+
this.hierarchyManager = hierarchyManager;
|
|
21
|
+
this.notificationManager = notificationManager;
|
|
22
|
+
this.address = address;
|
|
23
|
+
this.config = config;
|
|
24
|
+
this.healthMap = new Map();
|
|
25
|
+
}
|
|
26
|
+
async start() {
|
|
27
|
+
if (!this.config.enabled) {
|
|
28
|
+
this.logger.debug('Connection heartbeat disabled');
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
this.logger.info(`Starting connection heartbeat: interval=${this.config.intervalMs}ms, ` +
|
|
32
|
+
`timeout=${this.config.timeoutMs}ms, threshold=${this.config.failureThreshold}`);
|
|
33
|
+
// Immediate first check
|
|
34
|
+
await this.performHeartbeatCycle();
|
|
35
|
+
// Schedule recurring checks
|
|
36
|
+
this.heartbeatInterval = setInterval(() => this.performHeartbeatCycle(), this.config.intervalMs);
|
|
37
|
+
}
|
|
38
|
+
async stop() {
|
|
39
|
+
if (this.heartbeatInterval) {
|
|
40
|
+
clearInterval(this.heartbeatInterval);
|
|
41
|
+
this.heartbeatInterval = undefined;
|
|
42
|
+
}
|
|
43
|
+
this.healthMap.clear();
|
|
44
|
+
}
|
|
45
|
+
async performHeartbeatCycle() {
|
|
46
|
+
const targets = [];
|
|
47
|
+
// Check if this is a leader node (no leader in hierarchy = we are leader)
|
|
48
|
+
const isLeaderNode = this.hierarchyManager.getLeaders().length === 0;
|
|
49
|
+
// Collect leader (if enabled and we're not the leader)
|
|
50
|
+
if (this.config.checkLeader && !isLeaderNode) {
|
|
51
|
+
const leaders = this.hierarchyManager.getLeaders();
|
|
52
|
+
for (const leader of leaders) {
|
|
53
|
+
targets.push({ address: leader, role: 'leader' });
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
// Collect parent
|
|
57
|
+
if (this.config.checkParent && !isLeaderNode) {
|
|
58
|
+
const parents = this.hierarchyManager.getParents();
|
|
59
|
+
for (const parent of parents) {
|
|
60
|
+
targets.push({ address: parent, role: 'parent' });
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
// Collect children
|
|
64
|
+
if (this.config.checkChildren) {
|
|
65
|
+
const children = this.hierarchyManager.getChildren();
|
|
66
|
+
for (const child of children) {
|
|
67
|
+
targets.push({ address: child, role: 'child' });
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
// Ping all targets in parallel
|
|
71
|
+
await Promise.allSettled(targets.map((target) => this.pingTarget(target.address, target.role)));
|
|
72
|
+
}
|
|
73
|
+
async pingTarget(address, role) {
|
|
74
|
+
const peerId = this.extractPeerIdFromAddress(address);
|
|
75
|
+
if (!peerId) {
|
|
76
|
+
this.logger.warn(`Cannot extract peerId from ${address}`);
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
const key = address.toString();
|
|
80
|
+
let health = this.healthMap.get(key);
|
|
81
|
+
if (!health) {
|
|
82
|
+
health = {
|
|
83
|
+
address,
|
|
84
|
+
peerId,
|
|
85
|
+
lastSuccessfulPing: 0,
|
|
86
|
+
consecutiveFailures: 0,
|
|
87
|
+
averageLatency: 0,
|
|
88
|
+
status: 'healthy',
|
|
89
|
+
};
|
|
90
|
+
this.healthMap.set(key, health);
|
|
91
|
+
}
|
|
92
|
+
try {
|
|
93
|
+
// Use libp2p's native ping service
|
|
94
|
+
const startTime = Date.now();
|
|
95
|
+
// Create timeout promise
|
|
96
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
97
|
+
setTimeout(() => reject(new Error('Ping timeout')), this.config.timeoutMs);
|
|
98
|
+
});
|
|
99
|
+
// Race between ping and timeout
|
|
100
|
+
// The ping service accepts PeerId as string or object
|
|
101
|
+
await Promise.race([
|
|
102
|
+
this.p2pNode.services.ping.ping(peerId),
|
|
103
|
+
timeoutPromise,
|
|
104
|
+
]);
|
|
105
|
+
const latency = Date.now() - startTime;
|
|
106
|
+
// Success - update health
|
|
107
|
+
health.lastSuccessfulPing = Date.now();
|
|
108
|
+
health.consecutiveFailures = 0;
|
|
109
|
+
health.averageLatency =
|
|
110
|
+
health.averageLatency === 0
|
|
111
|
+
? latency
|
|
112
|
+
: health.averageLatency * 0.7 + latency * 0.3; // Exponential moving average
|
|
113
|
+
const previousStatus = health.status;
|
|
114
|
+
health.status = 'healthy';
|
|
115
|
+
// Emit recovery event if was degraded
|
|
116
|
+
if (previousStatus === 'degraded') {
|
|
117
|
+
this.logger.info(`Connection recovered: ${address} (latency: ${latency}ms)`);
|
|
118
|
+
this.emitConnectionRecoveredEvent(address, role);
|
|
119
|
+
}
|
|
120
|
+
this.logger.debug(`Ping successful: ${address} (${latency}ms)`);
|
|
121
|
+
}
|
|
122
|
+
catch (error) {
|
|
123
|
+
health.consecutiveFailures++;
|
|
124
|
+
this.logger.warn(`Ping failed: ${address} (failures: ${health.consecutiveFailures}/${this.config.failureThreshold})`);
|
|
125
|
+
// Update status based on failure count
|
|
126
|
+
if (health.consecutiveFailures >= this.config.failureThreshold) {
|
|
127
|
+
this.handleConnectionDead(address, role, health);
|
|
128
|
+
}
|
|
129
|
+
else if (health.consecutiveFailures >= Math.ceil(this.config.failureThreshold / 2)) {
|
|
130
|
+
health.status = 'degraded';
|
|
131
|
+
this.emitConnectionDegradedEvent(address, role, health.consecutiveFailures);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
handleConnectionDead(address, role, health) {
|
|
136
|
+
health.status = 'dead';
|
|
137
|
+
this.logger.error(`Connection dead after ${health.consecutiveFailures} failures: ${address} (role: ${role})`);
|
|
138
|
+
// Remove from health tracking
|
|
139
|
+
this.healthMap.delete(address.toString());
|
|
140
|
+
// Emit events based on role
|
|
141
|
+
if (role === 'child') {
|
|
142
|
+
// Remove dead child from hierarchy
|
|
143
|
+
this.hierarchyManager.removeChild(address);
|
|
144
|
+
// Emit child left event
|
|
145
|
+
this.notificationManager.emit(new ChildLeftEvent({
|
|
146
|
+
source: this.address,
|
|
147
|
+
childAddress: address,
|
|
148
|
+
parentAddress: this.address,
|
|
149
|
+
reason: `heartbeat_failed_${health.consecutiveFailures}_times`,
|
|
150
|
+
}));
|
|
151
|
+
this.logger.warn(`Removed dead child: ${address}`);
|
|
152
|
+
}
|
|
153
|
+
else if (role === 'parent') {
|
|
154
|
+
// Emit parent disconnected event
|
|
155
|
+
this.notificationManager.emit(new ParentDisconnectedEvent({
|
|
156
|
+
source: this.address,
|
|
157
|
+
parentAddress: address,
|
|
158
|
+
reason: `heartbeat_failed_${health.consecutiveFailures}_times`,
|
|
159
|
+
}));
|
|
160
|
+
this.logger.error(`Parent connection dead: ${address}`);
|
|
161
|
+
// Reconnection manager will handle this event
|
|
162
|
+
}
|
|
163
|
+
else if (role === 'leader') {
|
|
164
|
+
// Emit leader disconnected event
|
|
165
|
+
this.notificationManager.emit(new LeaderDisconnectedEvent({
|
|
166
|
+
source: this.address,
|
|
167
|
+
leaderAddress: address,
|
|
168
|
+
reason: `heartbeat_failed_${health.consecutiveFailures}_times`,
|
|
169
|
+
}));
|
|
170
|
+
this.logger.error(`Leader connection dead: ${address}`);
|
|
171
|
+
// Reconnection manager will handle this event
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
emitConnectionDegradedEvent(address, role, failures) {
|
|
175
|
+
// ConnectionDegradedEvent only supports parent/child, so we map leader to parent
|
|
176
|
+
const eventRole = role === 'leader' ? 'parent' : role === 'child' ? 'child' : 'parent';
|
|
177
|
+
this.notificationManager.emit(new ConnectionDegradedEvent({
|
|
178
|
+
source: this.address,
|
|
179
|
+
targetAddress: address,
|
|
180
|
+
role: eventRole,
|
|
181
|
+
consecutiveFailures: failures,
|
|
182
|
+
}));
|
|
183
|
+
}
|
|
184
|
+
emitConnectionRecoveredEvent(address, role) {
|
|
185
|
+
// ConnectionRecoveredEvent only supports parent/child, so we map leader to parent
|
|
186
|
+
const eventRole = role === 'leader' ? 'parent' : role === 'child' ? 'child' : 'parent';
|
|
187
|
+
this.notificationManager.emit(new ConnectionRecoveredEvent({
|
|
188
|
+
source: this.address,
|
|
189
|
+
targetAddress: address,
|
|
190
|
+
role: eventRole,
|
|
191
|
+
}));
|
|
192
|
+
}
|
|
193
|
+
extractPeerIdFromAddress(address) {
|
|
194
|
+
// Extract peerId from transport multiaddr
|
|
195
|
+
for (const transport of address.transports) {
|
|
196
|
+
const multiaddr = transport.toString();
|
|
197
|
+
// Multiaddr format: /ip4/127.0.0.1/tcp/4001/p2p/QmPeerId
|
|
198
|
+
const parts = multiaddr.split('/p2p/');
|
|
199
|
+
if (parts.length === 2) {
|
|
200
|
+
return parts[1];
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
return null;
|
|
204
|
+
}
|
|
205
|
+
/**
|
|
206
|
+
* Get current health status of all connections
|
|
207
|
+
*/
|
|
208
|
+
getHealthStatus() {
|
|
209
|
+
return Array.from(this.healthMap.values());
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* Get health status for specific address
|
|
213
|
+
*/
|
|
214
|
+
getConnectionHealth(address) {
|
|
215
|
+
return this.healthMap.get(address.toString());
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* Get current configuration
|
|
219
|
+
*/
|
|
220
|
+
getConfig() {
|
|
221
|
+
return { ...this.config };
|
|
222
|
+
}
|
|
223
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { oObject } from '@olane/o-core';
|
|
2
|
+
import { IReconnectableNode } from '../interfaces/i-reconnectable-node.js';
|
|
3
|
+
export interface ReconnectionConfig {
|
|
4
|
+
enabled: boolean;
|
|
5
|
+
maxAttempts: number;
|
|
6
|
+
baseDelayMs: number;
|
|
7
|
+
maxDelayMs: number;
|
|
8
|
+
useLeaderFallback: boolean;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Reconnection Manager
|
|
12
|
+
*
|
|
13
|
+
* Automatically attempts to reconnect to parent when connection is lost.
|
|
14
|
+
*
|
|
15
|
+
* Strategy:
|
|
16
|
+
* 1. Listen for ParentDisconnectedEvent (from heartbeat or libp2p)
|
|
17
|
+
* 2. Attempt direct reconnection with exponential backoff
|
|
18
|
+
* 3. If direct reconnection fails, query leader for new parent
|
|
19
|
+
* 4. Register with new parent and continue operation
|
|
20
|
+
* 5. If all attempts fail, transition node to ERROR state
|
|
21
|
+
*/
|
|
22
|
+
export declare class oReconnectionManager extends oObject {
|
|
23
|
+
private node;
|
|
24
|
+
private config;
|
|
25
|
+
private reconnecting;
|
|
26
|
+
constructor(node: IReconnectableNode, config: ReconnectionConfig);
|
|
27
|
+
private setupEventListeners;
|
|
28
|
+
private handleConnectionDegraded;
|
|
29
|
+
private handleLeaderDisconnected;
|
|
30
|
+
private handleParentDisconnected;
|
|
31
|
+
attemptReconnection(): Promise<void>;
|
|
32
|
+
private tryDirectParentReconnection;
|
|
33
|
+
private tryLeaderFallback;
|
|
34
|
+
private handleReconnectionFailure;
|
|
35
|
+
private calculateNodeLevel;
|
|
36
|
+
private calculateBackoffDelay;
|
|
37
|
+
private sleep;
|
|
38
|
+
}
|
|
39
|
+
//# sourceMappingURL=o-reconnection.manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"o-reconnection.manager.d.ts","sourceRoot":"","sources":["../../../src/managers/o-reconnection.manager.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,OAAO,EAQR,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,kBAAkB,EAAE,MAAM,uCAAuC,CAAC;AAI3E,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,OAAO,CAAC;CAC5B;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,oBAAqB,SAAQ,OAAO;IAI7C,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,MAAM;IAJhB,OAAO,CAAC,YAAY,CAAS;gBAGnB,IAAI,EAAE,kBAAkB,EACxB,MAAM,EAAE,kBAAkB;IAMpC,OAAO,CAAC,mBAAmB;YAoBb,wBAAwB;YAaxB,wBAAwB;YAexB,wBAAwB;IAehC,mBAAmB;YAkDX,2BAA2B;YAiB3B,iBAAiB;IAyC/B,OAAO,CAAC,yBAAyB;IAajC,OAAO,CAAC,kBAAkB;IAI1B,OAAO,CAAC,qBAAqB;IAK7B,OAAO,CAAC,KAAK;CAGd"}
|