@olane/o-node 0.7.12-alpha.5 → 0.7.12-alpha.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/interfaces/i-reconnectable-node.d.ts +41 -0
- package/dist/src/interfaces/i-reconnectable-node.d.ts.map +1 -0
- package/dist/src/interfaces/i-reconnectable-node.js +1 -0
- package/dist/src/interfaces/o-node.config.d.ts +35 -0
- package/dist/src/interfaces/o-node.config.d.ts.map +1 -1
- package/dist/src/managers/o-connection-heartbeat.manager.d.ts +67 -0
- package/dist/src/managers/o-connection-heartbeat.manager.d.ts.map +1 -0
- package/dist/src/managers/o-connection-heartbeat.manager.js +223 -0
- package/dist/src/managers/o-reconnection.manager.d.ts +39 -0
- package/dist/src/managers/o-reconnection.manager.d.ts.map +1 -0
- package/dist/src/managers/o-reconnection.manager.js +150 -0
- package/dist/src/o-node.d.ts +18 -1
- package/dist/src/o-node.d.ts.map +1 -1
- package/dist/src/o-node.js +82 -0
- package/dist/src/o-node.notification-manager.d.ts +52 -0
- package/dist/src/o-node.notification-manager.d.ts.map +1 -0
- package/dist/src/o-node.notification-manager.js +183 -0
- package/dist/src/o-node.tool.d.ts.map +1 -1
- package/dist/src/o-node.tool.js +12 -2
- package/dist/src/utils/leader-request-wrapper.d.ts +45 -0
- package/dist/src/utils/leader-request-wrapper.d.ts.map +1 -0
- package/dist/src/utils/leader-request-wrapper.js +89 -0
- package/package.json +6 -6
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { oAddress, NodeState, oNotificationManager } from '@olane/o-core';
|
|
2
|
+
import { oNodeAddress } from '../router/o-node.address.js';
|
|
3
|
+
import { oNodeConfig } from './o-node.config.js';
|
|
4
|
+
/**
|
|
5
|
+
* Interface for nodes that support reconnection management.
|
|
6
|
+
* This interface defines the contract that oReconnectionManager needs
|
|
7
|
+
* to perform reconnection operations without creating a circular dependency.
|
|
8
|
+
*/
|
|
9
|
+
export interface IReconnectableNode {
|
|
10
|
+
/**
|
|
11
|
+
* The node's configuration
|
|
12
|
+
*/
|
|
13
|
+
config: oNodeConfig;
|
|
14
|
+
/**
|
|
15
|
+
* The node's current address
|
|
16
|
+
*/
|
|
17
|
+
address: oNodeAddress;
|
|
18
|
+
/**
|
|
19
|
+
* The node's current state
|
|
20
|
+
*/
|
|
21
|
+
state: NodeState;
|
|
22
|
+
/**
|
|
23
|
+
* The notification manager for subscribing to events
|
|
24
|
+
*/
|
|
25
|
+
notificationManager: oNotificationManager;
|
|
26
|
+
/**
|
|
27
|
+
* Register with the parent node
|
|
28
|
+
*/
|
|
29
|
+
registerParent(): Promise<void>;
|
|
30
|
+
/**
|
|
31
|
+
* Execute a method on another node
|
|
32
|
+
*/
|
|
33
|
+
use(address: oAddress, data?: {
|
|
34
|
+
method?: string;
|
|
35
|
+
params?: {
|
|
36
|
+
[key: string]: any;
|
|
37
|
+
};
|
|
38
|
+
id?: string;
|
|
39
|
+
}): Promise<any>;
|
|
40
|
+
}
|
|
41
|
+
//# sourceMappingURL=i-reconnectable-node.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"i-reconnectable-node.d.ts","sourceRoot":"","sources":["../../../src/interfaces/i-reconnectable-node.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAC1E,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEjD;;;;GAIG;AACH,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,MAAM,EAAE,WAAW,CAAC;IAEpB;;OAEG;IACH,OAAO,EAAE,YAAY,CAAC;IAEtB;;OAEG;IACH,KAAK,EAAE,SAAS,CAAC;IAEjB;;OAEG;IACH,mBAAmB,EAAE,oBAAoB,CAAC;IAE1C;;OAEG;IACH,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEhC;;OAEG;IACH,GAAG,CACD,OAAO,EAAE,QAAQ,EACjB,IAAI,CAAC,EAAE;QACL,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,MAAM,CAAC,EAAE;YAAE,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;SAAE,CAAC;QAChC,EAAE,CAAC,EAAE,MAAM,CAAC;KACb,GACA,OAAO,CAAC,GAAG,CAAC,CAAC;CACjB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -3,5 +3,40 @@ import { oNodeAddress } from '../router/o-node.address.js';
|
|
|
3
3
|
export interface oNodeConfig extends oCoreConfig {
|
|
4
4
|
leader: oNodeAddress | null;
|
|
5
5
|
parent: oNodeAddress | null;
|
|
6
|
+
/**
|
|
7
|
+
* Connection heartbeat configuration (libp2p-native pings)
|
|
8
|
+
* Detects dead connections via periodic pings using libp2p's ping service
|
|
9
|
+
*/
|
|
10
|
+
connectionHeartbeat?: {
|
|
11
|
+
enabled?: boolean;
|
|
12
|
+
intervalMs?: number;
|
|
13
|
+
timeoutMs?: number;
|
|
14
|
+
failureThreshold?: number;
|
|
15
|
+
checkChildren?: boolean;
|
|
16
|
+
checkParent?: boolean;
|
|
17
|
+
checkLeader?: boolean;
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* Automatic reconnection configuration
|
|
21
|
+
* Handles parent connection failures and attempts to reconnect
|
|
22
|
+
*/
|
|
23
|
+
reconnection?: {
|
|
24
|
+
enabled?: boolean;
|
|
25
|
+
maxAttempts?: number;
|
|
26
|
+
baseDelayMs?: number;
|
|
27
|
+
maxDelayMs?: number;
|
|
28
|
+
useLeaderFallback?: boolean;
|
|
29
|
+
};
|
|
30
|
+
/**
|
|
31
|
+
* Leader request retry configuration
|
|
32
|
+
* Handles temporary leader unavailability (healing, maintenance)
|
|
33
|
+
*/
|
|
34
|
+
leaderRetry?: {
|
|
35
|
+
enabled?: boolean;
|
|
36
|
+
maxAttempts?: number;
|
|
37
|
+
baseDelayMs?: number;
|
|
38
|
+
maxDelayMs?: number;
|
|
39
|
+
timeoutMs?: number;
|
|
40
|
+
};
|
|
6
41
|
}
|
|
7
42
|
//# sourceMappingURL=o-node.config.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"o-node.config.d.ts","sourceRoot":"","sources":["../../../src/interfaces/o-node.config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAE3D,MAAM,WAAW,WAAY,SAAQ,WAAW;IAC9C,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"o-node.config.d.ts","sourceRoot":"","sources":["../../../src/interfaces/o-node.config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAE3D,MAAM,WAAW,WAAY,SAAQ,WAAW;IAC9C,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IAE5B;;;OAGG;IACH,mBAAmB,CAAC,EAAE;QACpB,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,aAAa,CAAC,EAAE,OAAO,CAAC;QACxB,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB,CAAC;IAEF;;;OAGG;IACH,YAAY,CAAC,EAAE;QACb,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,iBAAiB,CAAC,EAAE,OAAO,CAAC;KAC7B,CAAC;IAEF;;;OAGG;IACH,WAAW,CAAC,EAAE;QACZ,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACH"}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { Libp2p } from '@olane/o-config';
|
|
2
|
+
import { oObject } from '@olane/o-core';
|
|
3
|
+
import { oNodeAddress } from '../router/o-node.address.js';
|
|
4
|
+
import { oNodeHierarchyManager } from '../o-node.hierarchy-manager.js';
|
|
5
|
+
import { oNotificationManager } from '@olane/o-core';
|
|
6
|
+
export interface HeartbeatConfig {
|
|
7
|
+
enabled: boolean;
|
|
8
|
+
intervalMs: number;
|
|
9
|
+
timeoutMs: number;
|
|
10
|
+
failureThreshold: number;
|
|
11
|
+
checkChildren: boolean;
|
|
12
|
+
checkParent: boolean;
|
|
13
|
+
checkLeader: boolean;
|
|
14
|
+
}
|
|
15
|
+
export interface ConnectionHealth {
|
|
16
|
+
address: oNodeAddress;
|
|
17
|
+
peerId: string;
|
|
18
|
+
lastSuccessfulPing: number;
|
|
19
|
+
consecutiveFailures: number;
|
|
20
|
+
averageLatency: number;
|
|
21
|
+
status: 'healthy' | 'degraded' | 'dead';
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Connection Heartbeat Manager
|
|
25
|
+
*
|
|
26
|
+
* Uses libp2p's native ping service to detect dead connections early.
|
|
27
|
+
* Continuously pings parent and children to ensure they're alive.
|
|
28
|
+
*
|
|
29
|
+
* How it works:
|
|
30
|
+
* - Every `intervalMs`, pings all tracked connections
|
|
31
|
+
* - If ping fails, increments failure counter
|
|
32
|
+
* - After `failureThreshold` failures, marks connection as dead
|
|
33
|
+
* - Emits events for degraded/recovered/dead connections
|
|
34
|
+
* - Automatically removes dead children from hierarchy
|
|
35
|
+
* - Emits ParentDisconnectedEvent when parent dies (triggers reconnection)
|
|
36
|
+
*/
|
|
37
|
+
export declare class oConnectionHeartbeatManager extends oObject {
|
|
38
|
+
private p2pNode;
|
|
39
|
+
private hierarchyManager;
|
|
40
|
+
private notificationManager;
|
|
41
|
+
private address;
|
|
42
|
+
private config;
|
|
43
|
+
private heartbeatInterval?;
|
|
44
|
+
private healthMap;
|
|
45
|
+
constructor(p2pNode: Libp2p, hierarchyManager: oNodeHierarchyManager, notificationManager: oNotificationManager, address: oNodeAddress, config: HeartbeatConfig);
|
|
46
|
+
start(): Promise<void>;
|
|
47
|
+
stop(): Promise<void>;
|
|
48
|
+
private performHeartbeatCycle;
|
|
49
|
+
private pingTarget;
|
|
50
|
+
private handleConnectionDead;
|
|
51
|
+
private emitConnectionDegradedEvent;
|
|
52
|
+
private emitConnectionRecoveredEvent;
|
|
53
|
+
private extractPeerIdFromAddress;
|
|
54
|
+
/**
|
|
55
|
+
* Get current health status of all connections
|
|
56
|
+
*/
|
|
57
|
+
getHealthStatus(): ConnectionHealth[];
|
|
58
|
+
/**
|
|
59
|
+
* Get health status for specific address
|
|
60
|
+
*/
|
|
61
|
+
getConnectionHealth(address: oNodeAddress): ConnectionHealth | undefined;
|
|
62
|
+
/**
|
|
63
|
+
* Get current configuration
|
|
64
|
+
*/
|
|
65
|
+
getConfig(): HeartbeatConfig;
|
|
66
|
+
}
|
|
67
|
+
//# sourceMappingURL=o-connection-heartbeat.manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"o-connection-heartbeat.manager.d.ts","sourceRoot":"","sources":["../../../src/managers/o-connection-heartbeat.manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,OAAO,EACL,OAAO,EAMR,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAErD,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,EAAE,OAAO,CAAC;IACrB,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,YAAY,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,kBAAkB,EAAE,MAAM,CAAC;IAC3B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,SAAS,GAAG,UAAU,GAAG,MAAM,CAAC;CACzC;AAED;;;;;;;;;;;;;GAaG;AACH,qBAAa,2BAA4B,SAAQ,OAAO;IAKpD,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,gBAAgB;IACxB,OAAO,CAAC,mBAAmB;IAC3B,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,MAAM;IARhB,OAAO,CAAC,iBAAiB,CAAC,CAAiB;IAC3C,OAAO,CAAC,SAAS,CAAuC;gBAG9C,OAAO,EAAE,MAAM,EACf,gBAAgB,EAAE,qBAAqB,EACvC,mBAAmB,EAAE,oBAAoB,EACzC,OAAO,EAAE,YAAY,EACrB,MAAM,EAAE,eAAe;IAK3B,KAAK;IAqBL,IAAI;YAQI,qBAAqB;YAuCrB,UAAU;IAyFxB,OAAO,CAAC,oBAAoB;IAyD5B,OAAO,CAAC,2BAA2B;IAmBnC,OAAO,CAAC,4BAA4B;IAiBpC,OAAO,CAAC,wBAAwB;IAahC;;OAEG;IACH,eAAe,IAAI,gBAAgB,EAAE;IAIrC;;OAEG;IACH,mBAAmB,CAAC,OAAO,EAAE,YAAY,GAAG,gBAAgB,GAAG,SAAS;IAIxE;;OAEG;IACH,SAAS,IAAI,eAAe;CAG7B"}
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
import { oObject, ChildLeftEvent, ParentDisconnectedEvent, LeaderDisconnectedEvent, ConnectionDegradedEvent, ConnectionRecoveredEvent, } from '@olane/o-core';
|
|
2
|
+
/**
|
|
3
|
+
* Connection Heartbeat Manager
|
|
4
|
+
*
|
|
5
|
+
* Uses libp2p's native ping service to detect dead connections early.
|
|
6
|
+
* Continuously pings parent and children to ensure they're alive.
|
|
7
|
+
*
|
|
8
|
+
* How it works:
|
|
9
|
+
* - Every `intervalMs`, pings all tracked connections
|
|
10
|
+
* - If ping fails, increments failure counter
|
|
11
|
+
* - After `failureThreshold` failures, marks connection as dead
|
|
12
|
+
* - Emits events for degraded/recovered/dead connections
|
|
13
|
+
* - Automatically removes dead children from hierarchy
|
|
14
|
+
* - Emits ParentDisconnectedEvent when parent dies (triggers reconnection)
|
|
15
|
+
*/
|
|
16
|
+
export class oConnectionHeartbeatManager extends oObject {
|
|
17
|
+
constructor(p2pNode, hierarchyManager, notificationManager, address, config) {
|
|
18
|
+
super();
|
|
19
|
+
this.p2pNode = p2pNode;
|
|
20
|
+
this.hierarchyManager = hierarchyManager;
|
|
21
|
+
this.notificationManager = notificationManager;
|
|
22
|
+
this.address = address;
|
|
23
|
+
this.config = config;
|
|
24
|
+
this.healthMap = new Map();
|
|
25
|
+
}
|
|
26
|
+
async start() {
|
|
27
|
+
if (!this.config.enabled) {
|
|
28
|
+
this.logger.debug('Connection heartbeat disabled');
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
this.logger.info(`Starting connection heartbeat: interval=${this.config.intervalMs}ms, ` +
|
|
32
|
+
`timeout=${this.config.timeoutMs}ms, threshold=${this.config.failureThreshold}`);
|
|
33
|
+
// Immediate first check
|
|
34
|
+
await this.performHeartbeatCycle();
|
|
35
|
+
// Schedule recurring checks
|
|
36
|
+
this.heartbeatInterval = setInterval(() => this.performHeartbeatCycle(), this.config.intervalMs);
|
|
37
|
+
}
|
|
38
|
+
async stop() {
|
|
39
|
+
if (this.heartbeatInterval) {
|
|
40
|
+
clearInterval(this.heartbeatInterval);
|
|
41
|
+
this.heartbeatInterval = undefined;
|
|
42
|
+
}
|
|
43
|
+
this.healthMap.clear();
|
|
44
|
+
}
|
|
45
|
+
async performHeartbeatCycle() {
|
|
46
|
+
const targets = [];
|
|
47
|
+
// Check if this is a leader node (no leader in hierarchy = we are leader)
|
|
48
|
+
const isLeaderNode = this.hierarchyManager.getLeaders().length === 0;
|
|
49
|
+
// Collect leader (if enabled and we're not the leader)
|
|
50
|
+
if (this.config.checkLeader && !isLeaderNode) {
|
|
51
|
+
const leaders = this.hierarchyManager.getLeaders();
|
|
52
|
+
for (const leader of leaders) {
|
|
53
|
+
targets.push({ address: leader, role: 'leader' });
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
// Collect parent
|
|
57
|
+
if (this.config.checkParent && !isLeaderNode) {
|
|
58
|
+
const parents = this.hierarchyManager.getParents();
|
|
59
|
+
for (const parent of parents) {
|
|
60
|
+
targets.push({ address: parent, role: 'parent' });
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
// Collect children
|
|
64
|
+
if (this.config.checkChildren) {
|
|
65
|
+
const children = this.hierarchyManager.getChildren();
|
|
66
|
+
for (const child of children) {
|
|
67
|
+
targets.push({ address: child, role: 'child' });
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
// Ping all targets in parallel
|
|
71
|
+
await Promise.allSettled(targets.map((target) => this.pingTarget(target.address, target.role)));
|
|
72
|
+
}
|
|
73
|
+
async pingTarget(address, role) {
|
|
74
|
+
const peerId = this.extractPeerIdFromAddress(address);
|
|
75
|
+
if (!peerId) {
|
|
76
|
+
this.logger.warn(`Cannot extract peerId from ${address}`);
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
const key = address.toString();
|
|
80
|
+
let health = this.healthMap.get(key);
|
|
81
|
+
if (!health) {
|
|
82
|
+
health = {
|
|
83
|
+
address,
|
|
84
|
+
peerId,
|
|
85
|
+
lastSuccessfulPing: 0,
|
|
86
|
+
consecutiveFailures: 0,
|
|
87
|
+
averageLatency: 0,
|
|
88
|
+
status: 'healthy',
|
|
89
|
+
};
|
|
90
|
+
this.healthMap.set(key, health);
|
|
91
|
+
}
|
|
92
|
+
try {
|
|
93
|
+
// Use libp2p's native ping service
|
|
94
|
+
const startTime = Date.now();
|
|
95
|
+
// Create timeout promise
|
|
96
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
97
|
+
setTimeout(() => reject(new Error('Ping timeout')), this.config.timeoutMs);
|
|
98
|
+
});
|
|
99
|
+
// Race between ping and timeout
|
|
100
|
+
// The ping service accepts PeerId as string or object
|
|
101
|
+
await Promise.race([
|
|
102
|
+
this.p2pNode.services.ping.ping(peerId),
|
|
103
|
+
timeoutPromise,
|
|
104
|
+
]);
|
|
105
|
+
const latency = Date.now() - startTime;
|
|
106
|
+
// Success - update health
|
|
107
|
+
health.lastSuccessfulPing = Date.now();
|
|
108
|
+
health.consecutiveFailures = 0;
|
|
109
|
+
health.averageLatency =
|
|
110
|
+
health.averageLatency === 0
|
|
111
|
+
? latency
|
|
112
|
+
: health.averageLatency * 0.7 + latency * 0.3; // Exponential moving average
|
|
113
|
+
const previousStatus = health.status;
|
|
114
|
+
health.status = 'healthy';
|
|
115
|
+
// Emit recovery event if was degraded
|
|
116
|
+
if (previousStatus === 'degraded') {
|
|
117
|
+
this.logger.info(`Connection recovered: ${address} (latency: ${latency}ms)`);
|
|
118
|
+
this.emitConnectionRecoveredEvent(address, role);
|
|
119
|
+
}
|
|
120
|
+
this.logger.debug(`Ping successful: ${address} (${latency}ms)`);
|
|
121
|
+
}
|
|
122
|
+
catch (error) {
|
|
123
|
+
health.consecutiveFailures++;
|
|
124
|
+
this.logger.warn(`Ping failed: ${address} (failures: ${health.consecutiveFailures}/${this.config.failureThreshold})`);
|
|
125
|
+
// Update status based on failure count
|
|
126
|
+
if (health.consecutiveFailures >= this.config.failureThreshold) {
|
|
127
|
+
this.handleConnectionDead(address, role, health);
|
|
128
|
+
}
|
|
129
|
+
else if (health.consecutiveFailures >= Math.ceil(this.config.failureThreshold / 2)) {
|
|
130
|
+
health.status = 'degraded';
|
|
131
|
+
this.emitConnectionDegradedEvent(address, role, health.consecutiveFailures);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
handleConnectionDead(address, role, health) {
|
|
136
|
+
health.status = 'dead';
|
|
137
|
+
this.logger.error(`Connection dead after ${health.consecutiveFailures} failures: ${address} (role: ${role})`);
|
|
138
|
+
// Remove from health tracking
|
|
139
|
+
this.healthMap.delete(address.toString());
|
|
140
|
+
// Emit events based on role
|
|
141
|
+
if (role === 'child') {
|
|
142
|
+
// Remove dead child from hierarchy
|
|
143
|
+
this.hierarchyManager.removeChild(address);
|
|
144
|
+
// Emit child left event
|
|
145
|
+
this.notificationManager.emit(new ChildLeftEvent({
|
|
146
|
+
source: this.address,
|
|
147
|
+
childAddress: address,
|
|
148
|
+
parentAddress: this.address,
|
|
149
|
+
reason: `heartbeat_failed_${health.consecutiveFailures}_times`,
|
|
150
|
+
}));
|
|
151
|
+
this.logger.warn(`Removed dead child: ${address}`);
|
|
152
|
+
}
|
|
153
|
+
else if (role === 'parent') {
|
|
154
|
+
// Emit parent disconnected event
|
|
155
|
+
this.notificationManager.emit(new ParentDisconnectedEvent({
|
|
156
|
+
source: this.address,
|
|
157
|
+
parentAddress: address,
|
|
158
|
+
reason: `heartbeat_failed_${health.consecutiveFailures}_times`,
|
|
159
|
+
}));
|
|
160
|
+
this.logger.error(`Parent connection dead: ${address}`);
|
|
161
|
+
// Reconnection manager will handle this event
|
|
162
|
+
}
|
|
163
|
+
else if (role === 'leader') {
|
|
164
|
+
// Emit leader disconnected event
|
|
165
|
+
this.notificationManager.emit(new LeaderDisconnectedEvent({
|
|
166
|
+
source: this.address,
|
|
167
|
+
leaderAddress: address,
|
|
168
|
+
reason: `heartbeat_failed_${health.consecutiveFailures}_times`,
|
|
169
|
+
}));
|
|
170
|
+
this.logger.error(`Leader connection dead: ${address}`);
|
|
171
|
+
// Reconnection manager will handle this event
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
emitConnectionDegradedEvent(address, role, failures) {
|
|
175
|
+
// ConnectionDegradedEvent only supports parent/child, so we map leader to parent
|
|
176
|
+
const eventRole = role === 'leader' ? 'parent' : role === 'child' ? 'child' : 'parent';
|
|
177
|
+
this.notificationManager.emit(new ConnectionDegradedEvent({
|
|
178
|
+
source: this.address,
|
|
179
|
+
targetAddress: address,
|
|
180
|
+
role: eventRole,
|
|
181
|
+
consecutiveFailures: failures,
|
|
182
|
+
}));
|
|
183
|
+
}
|
|
184
|
+
emitConnectionRecoveredEvent(address, role) {
|
|
185
|
+
// ConnectionRecoveredEvent only supports parent/child, so we map leader to parent
|
|
186
|
+
const eventRole = role === 'leader' ? 'parent' : role === 'child' ? 'child' : 'parent';
|
|
187
|
+
this.notificationManager.emit(new ConnectionRecoveredEvent({
|
|
188
|
+
source: this.address,
|
|
189
|
+
targetAddress: address,
|
|
190
|
+
role: eventRole,
|
|
191
|
+
}));
|
|
192
|
+
}
|
|
193
|
+
extractPeerIdFromAddress(address) {
|
|
194
|
+
// Extract peerId from transport multiaddr
|
|
195
|
+
for (const transport of address.transports) {
|
|
196
|
+
const multiaddr = transport.toString();
|
|
197
|
+
// Multiaddr format: /ip4/127.0.0.1/tcp/4001/p2p/QmPeerId
|
|
198
|
+
const parts = multiaddr.split('/p2p/');
|
|
199
|
+
if (parts.length === 2) {
|
|
200
|
+
return parts[1];
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
return null;
|
|
204
|
+
}
|
|
205
|
+
/**
|
|
206
|
+
* Get current health status of all connections
|
|
207
|
+
*/
|
|
208
|
+
getHealthStatus() {
|
|
209
|
+
return Array.from(this.healthMap.values());
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* Get health status for specific address
|
|
213
|
+
*/
|
|
214
|
+
getConnectionHealth(address) {
|
|
215
|
+
return this.healthMap.get(address.toString());
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* Get current configuration
|
|
219
|
+
*/
|
|
220
|
+
getConfig() {
|
|
221
|
+
return { ...this.config };
|
|
222
|
+
}
|
|
223
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { oObject } from '@olane/o-core';
|
|
2
|
+
import { IReconnectableNode } from '../interfaces/i-reconnectable-node.js';
|
|
3
|
+
export interface ReconnectionConfig {
|
|
4
|
+
enabled: boolean;
|
|
5
|
+
maxAttempts: number;
|
|
6
|
+
baseDelayMs: number;
|
|
7
|
+
maxDelayMs: number;
|
|
8
|
+
useLeaderFallback: boolean;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Reconnection Manager
|
|
12
|
+
*
|
|
13
|
+
* Automatically attempts to reconnect to parent when connection is lost.
|
|
14
|
+
*
|
|
15
|
+
* Strategy:
|
|
16
|
+
* 1. Listen for ParentDisconnectedEvent (from heartbeat or libp2p)
|
|
17
|
+
* 2. Attempt direct reconnection with exponential backoff
|
|
18
|
+
* 3. If direct reconnection fails, query leader for new parent
|
|
19
|
+
* 4. Register with new parent and continue operation
|
|
20
|
+
* 5. If all attempts fail, transition node to ERROR state
|
|
21
|
+
*/
|
|
22
|
+
export declare class oReconnectionManager extends oObject {
|
|
23
|
+
private node;
|
|
24
|
+
private config;
|
|
25
|
+
private reconnecting;
|
|
26
|
+
constructor(node: IReconnectableNode, config: ReconnectionConfig);
|
|
27
|
+
private setupEventListeners;
|
|
28
|
+
private handleConnectionDegraded;
|
|
29
|
+
private handleLeaderDisconnected;
|
|
30
|
+
private handleParentDisconnected;
|
|
31
|
+
attemptReconnection(): Promise<void>;
|
|
32
|
+
private tryDirectParentReconnection;
|
|
33
|
+
private tryLeaderFallback;
|
|
34
|
+
private handleReconnectionFailure;
|
|
35
|
+
private calculateNodeLevel;
|
|
36
|
+
private calculateBackoffDelay;
|
|
37
|
+
private sleep;
|
|
38
|
+
}
|
|
39
|
+
//# sourceMappingURL=o-reconnection.manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"o-reconnection.manager.d.ts","sourceRoot":"","sources":["../../../src/managers/o-reconnection.manager.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,OAAO,EAQR,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,kBAAkB,EAAE,MAAM,uCAAuC,CAAC;AAI3E,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,OAAO,CAAC;CAC5B;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,oBAAqB,SAAQ,OAAO;IAI7C,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,MAAM;IAJhB,OAAO,CAAC,YAAY,CAAS;gBAGnB,IAAI,EAAE,kBAAkB,EACxB,MAAM,EAAE,kBAAkB;IAMpC,OAAO,CAAC,mBAAmB;YAoBb,wBAAwB;YAaxB,wBAAwB;YAexB,wBAAwB;IAehC,mBAAmB;YAkDX,2BAA2B;YAiB3B,iBAAiB;IAyC/B,OAAO,CAAC,yBAAyB;IAajC,OAAO,CAAC,kBAAkB;IAI1B,OAAO,CAAC,qBAAqB;IAK7B,OAAO,CAAC,KAAK;CAGd"}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import { oObject, oAddress, NodeState, } from '@olane/o-core';
|
|
2
|
+
import { oNodeAddress } from '../router/o-node.address.js';
|
|
3
|
+
import { oNodeTransport } from '../router/o-node.transport.js';
|
|
4
|
+
/**
|
|
5
|
+
* Reconnection Manager
|
|
6
|
+
*
|
|
7
|
+
* Automatically attempts to reconnect to parent when connection is lost.
|
|
8
|
+
*
|
|
9
|
+
* Strategy:
|
|
10
|
+
* 1. Listen for ParentDisconnectedEvent (from heartbeat or libp2p)
|
|
11
|
+
* 2. Attempt direct reconnection with exponential backoff
|
|
12
|
+
* 3. If direct reconnection fails, query leader for new parent
|
|
13
|
+
* 4. Register with new parent and continue operation
|
|
14
|
+
* 5. If all attempts fail, transition node to ERROR state
|
|
15
|
+
*/
|
|
16
|
+
export class oReconnectionManager extends oObject {
|
|
17
|
+
constructor(node, config) {
|
|
18
|
+
super();
|
|
19
|
+
this.node = node;
|
|
20
|
+
this.config = config;
|
|
21
|
+
this.reconnecting = false;
|
|
22
|
+
this.setupEventListeners();
|
|
23
|
+
}
|
|
24
|
+
setupEventListeners() {
|
|
25
|
+
// Listen for parent disconnection (from heartbeat or libp2p)
|
|
26
|
+
this.node.notificationManager.on('parent:disconnected', this.handleParentDisconnected.bind(this));
|
|
27
|
+
// Listen for leader disconnection (from heartbeat)
|
|
28
|
+
this.node.notificationManager.on('leader:disconnected', this.handleLeaderDisconnected.bind(this));
|
|
29
|
+
// Listen for connection degradation as early warning
|
|
30
|
+
this.node.notificationManager.on('connection:degraded', this.handleConnectionDegraded.bind(this));
|
|
31
|
+
}
|
|
32
|
+
async handleConnectionDegraded(event) {
|
|
33
|
+
const degradedEvent = event;
|
|
34
|
+
if (degradedEvent.role !== 'parent')
|
|
35
|
+
return;
|
|
36
|
+
this.logger.warn(`Parent connection degraded: ${degradedEvent.targetAddress} ` +
|
|
37
|
+
`(failures: ${degradedEvent.consecutiveFailures})`);
|
|
38
|
+
// Could implement pre-emptive parent discovery here
|
|
39
|
+
// For now, just log the warning and wait for full disconnection
|
|
40
|
+
}
|
|
41
|
+
async handleLeaderDisconnected(event) {
|
|
42
|
+
const disconnectEvent = event;
|
|
43
|
+
this.logger.warn(`Leader disconnected: ${disconnectEvent.leaderAddress} (reason: ${disconnectEvent.reason})`);
|
|
44
|
+
// Don't attempt reconnection for leader - the LeaderRequestWrapper
|
|
45
|
+
// will handle retries automatically when we make requests
|
|
46
|
+
// Just log the event for observability
|
|
47
|
+
this.logger.info('Leader requests will use automatic retry mechanism (LeaderRequestWrapper)');
|
|
48
|
+
}
|
|
49
|
+
async handleParentDisconnected(event) {
|
|
50
|
+
const disconnectEvent = event;
|
|
51
|
+
if (this.reconnecting) {
|
|
52
|
+
this.logger.debug('Already reconnecting, ignoring duplicate event');
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
this.logger.warn(`Parent disconnected: ${disconnectEvent.parentAddress} (reason: ${disconnectEvent.reason})`);
|
|
56
|
+
await this.attemptReconnection();
|
|
57
|
+
}
|
|
58
|
+
async attemptReconnection() {
|
|
59
|
+
if (!this.config.enabled) {
|
|
60
|
+
this.logger.warn('Reconnection disabled - node will remain disconnected');
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
this.reconnecting = true;
|
|
64
|
+
let attempt = 0;
|
|
65
|
+
while (attempt < this.config.maxAttempts) {
|
|
66
|
+
attempt++;
|
|
67
|
+
this.logger.info(`Reconnection attempt ${attempt}/${this.config.maxAttempts} to parent: ${this.node.config.parent}`);
|
|
68
|
+
try {
|
|
69
|
+
// Strategy 1: Try direct parent reconnection
|
|
70
|
+
await this.tryDirectParentReconnection();
|
|
71
|
+
// Success!
|
|
72
|
+
this.reconnecting = false;
|
|
73
|
+
this.logger.info(`Successfully reconnected to parent after ${attempt} attempts`);
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
catch (error) {
|
|
77
|
+
this.logger.warn(`Reconnection attempt ${attempt} failed:`, error instanceof Error ? error.message : error);
|
|
78
|
+
if (attempt < this.config.maxAttempts) {
|
|
79
|
+
const delay = this.calculateBackoffDelay(attempt);
|
|
80
|
+
this.logger.debug(`Waiting ${delay}ms before next attempt...`);
|
|
81
|
+
await this.sleep(delay);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
// All direct attempts failed - try leader fallback
|
|
86
|
+
if (this.config.useLeaderFallback) {
|
|
87
|
+
await this.tryLeaderFallback();
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
this.handleReconnectionFailure();
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
async tryDirectParentReconnection() {
|
|
94
|
+
if (!this.node.config.parent) {
|
|
95
|
+
throw new Error('No parent configured');
|
|
96
|
+
}
|
|
97
|
+
// Re-register with parent (might have new transports)
|
|
98
|
+
await this.node.registerParent();
|
|
99
|
+
// Verify connection works with a ping
|
|
100
|
+
await this.node.use(this.node.config.parent, {
|
|
101
|
+
method: 'ping',
|
|
102
|
+
params: {},
|
|
103
|
+
});
|
|
104
|
+
this.logger.info('Direct parent reconnection successful');
|
|
105
|
+
}
|
|
106
|
+
async tryLeaderFallback() {
|
|
107
|
+
this.logger.info('Attempting leader fallback to find new parent');
|
|
108
|
+
try {
|
|
109
|
+
// Query registry for available parents at our level
|
|
110
|
+
const response = await this.node.use(new oAddress('o://registry'), {
|
|
111
|
+
method: 'find_available_parent',
|
|
112
|
+
params: {
|
|
113
|
+
currentAddress: this.node.address.toString(),
|
|
114
|
+
preferredLevel: this.calculateNodeLevel(),
|
|
115
|
+
},
|
|
116
|
+
});
|
|
117
|
+
const { parentAddress, parentTransports } = response.result.data;
|
|
118
|
+
if (parentAddress && parentTransports) {
|
|
119
|
+
// Update parent reference
|
|
120
|
+
this.node.config.parent = new oNodeAddress(parentAddress, parentTransports.map((t) => new oNodeTransport(t)));
|
|
121
|
+
// Register with new parent
|
|
122
|
+
await this.tryDirectParentReconnection();
|
|
123
|
+
this.reconnecting = false;
|
|
124
|
+
this.logger.info(`Successfully reconnected via new parent: ${parentAddress}`);
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
catch (error) {
|
|
129
|
+
this.logger.error('Leader fallback failed:', error instanceof Error ? error.message : error);
|
|
130
|
+
}
|
|
131
|
+
this.handleReconnectionFailure();
|
|
132
|
+
}
|
|
133
|
+
handleReconnectionFailure() {
|
|
134
|
+
this.reconnecting = false;
|
|
135
|
+
this.logger.error('Failed to reconnect to parent after all attempts - node in ERROR state');
|
|
136
|
+
// Transition to error state
|
|
137
|
+
this.node.state = NodeState.ERROR;
|
|
138
|
+
// Could emit custom event here for monitoring
|
|
139
|
+
}
|
|
140
|
+
calculateNodeLevel() {
|
|
141
|
+
return this.node.address.paths.length;
|
|
142
|
+
}
|
|
143
|
+
calculateBackoffDelay(attempt) {
|
|
144
|
+
const delay = this.config.baseDelayMs * Math.pow(2, attempt - 1);
|
|
145
|
+
return Math.min(delay, this.config.maxDelayMs);
|
|
146
|
+
}
|
|
147
|
+
sleep(ms) {
|
|
148
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
149
|
+
}
|
|
150
|
+
}
|
package/dist/src/o-node.d.ts
CHANGED
|
@@ -3,11 +3,14 @@ import { PeerId } from '@olane/o-config';
|
|
|
3
3
|
import { oNodeHierarchyManager } from './o-node.hierarchy-manager.js';
|
|
4
4
|
import { oNodeConfig } from './interfaces/o-node.config.js';
|
|
5
5
|
import { oNodeTransport } from './router/o-node.transport.js';
|
|
6
|
-
import { oRequest } from '@olane/o-core';
|
|
6
|
+
import { oAddress, oRequest, oNotificationManager } from '@olane/o-core';
|
|
7
7
|
import { oNodeAddress } from './router/o-node.address.js';
|
|
8
8
|
import { oNodeConnection } from './connection/o-node-connection.js';
|
|
9
9
|
import { oNodeConnectionManager } from './connection/o-node-connection.manager.js';
|
|
10
10
|
import { oToolBase } from '@olane/o-tool';
|
|
11
|
+
import { oConnectionHeartbeatManager } from './managers/o-connection-heartbeat.manager.js';
|
|
12
|
+
import { oReconnectionManager } from './managers/o-reconnection.manager.js';
|
|
13
|
+
import { LeaderRequestWrapper } from './utils/leader-request-wrapper.js';
|
|
11
14
|
export declare class oNode extends oToolBase {
|
|
12
15
|
peerId: PeerId;
|
|
13
16
|
p2pNode: Libp2p;
|
|
@@ -15,6 +18,9 @@ export declare class oNode extends oToolBase {
|
|
|
15
18
|
config: oNodeConfig;
|
|
16
19
|
connectionManager: oNodeConnectionManager;
|
|
17
20
|
hierarchyManager: oNodeHierarchyManager;
|
|
21
|
+
connectionHeartbeatManager?: oConnectionHeartbeatManager;
|
|
22
|
+
reconnectionManager?: oReconnectionManager;
|
|
23
|
+
leaderRequestWrapper: LeaderRequestWrapper;
|
|
18
24
|
protected didRegister: boolean;
|
|
19
25
|
constructor(config: oNodeConfig);
|
|
20
26
|
get leader(): oNodeAddress | null;
|
|
@@ -22,6 +28,7 @@ export declare class oNode extends oToolBase {
|
|
|
22
28
|
get parentPeerId(): string | null;
|
|
23
29
|
configureTransports(): any[];
|
|
24
30
|
initializeRouter(): Promise<void>;
|
|
31
|
+
protected createNotificationManager(): oNotificationManager;
|
|
25
32
|
get staticAddress(): oNodeAddress;
|
|
26
33
|
get parentTransports(): oNodeTransport[];
|
|
27
34
|
get transports(): oNodeTransport[];
|
|
@@ -39,6 +46,16 @@ export declare class oNode extends oToolBase {
|
|
|
39
46
|
protected createNode(): Promise<Libp2p>;
|
|
40
47
|
connect(nextHopAddress: oNodeAddress, targetAddress: oNodeAddress): Promise<oNodeConnection>;
|
|
41
48
|
initialize(): Promise<void>;
|
|
49
|
+
/**
|
|
50
|
+
* Override use() to wrap leader/registry requests with retry logic
|
|
51
|
+
*/
|
|
52
|
+
use(address: oAddress, data?: {
|
|
53
|
+
method?: string;
|
|
54
|
+
params?: {
|
|
55
|
+
[key: string]: any;
|
|
56
|
+
};
|
|
57
|
+
id?: string;
|
|
58
|
+
}): Promise<any>;
|
|
42
59
|
teardown(): Promise<void>;
|
|
43
60
|
}
|
|
44
61
|
//# sourceMappingURL=o-node.d.ts.map
|
package/dist/src/o-node.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"o-node.d.ts","sourceRoot":"","sources":["../../src/o-node.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,MAAM,EACN,YAAY,EACb,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AACtE,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,
|
|
1
|
+
{"version":3,"file":"o-node.d.ts","sourceRoot":"","sources":["../../src/o-node.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,MAAM,EACN,YAAY,EACb,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AACtE,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAIL,QAAQ,EACR,QAAQ,EAER,oBAAoB,EACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,mCAAmC,CAAC;AACpE,OAAO,EAAE,sBAAsB,EAAE,MAAM,2CAA2C,CAAC;AAGnF,OAAO,EAAmB,SAAS,EAAE,MAAM,eAAe,CAAC;AAI3D,OAAO,EAAE,2BAA2B,EAAE,MAAM,8CAA8C,CAAC;AAC3F,OAAO,EAAE,oBAAoB,EAAE,MAAM,sCAAsC,CAAC;AAC5E,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AAEzE,qBAAa,KAAM,SAAQ,SAAS;IAC3B,MAAM,EAAG,MAAM,CAAC;IAChB,OAAO,EAAG,MAAM,CAAC;IACjB,OAAO,EAAG,YAAY,CAAC;IACvB,MAAM,EAAE,WAAW,CAAC;IACpB,iBAAiB,EAAG,sBAAsB,CAAC;IAC3C,gBAAgB,EAAG,qBAAqB,CAAC;IACzC,0BAA0B,CAAC,EAAE,2BAA2B,CAAC;IACzD,mBAAmB,CAAC,EAAE,oBAAoB,CAAC;IAC3C,oBAAoB,EAAG,oBAAoB,CAAC;IACnD,SAAS,CAAC,WAAW,EAAE,OAAO,CAAS;gBAE3B,MAAM,EAAE,WAAW;IAK/B,IAAI,MAAM,IAAI,YAAY,GAAG,IAAI,CAEhC;IAED,IAAI,aAAa,IAAI,YAAY,CAKhC;IAED,IAAI,YAAY,IAAI,MAAM,GAAG,IAAI,CAOhC;IAED,mBAAmB,IAAI,GAAG,EAAE;IAItB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC;IASvC,SAAS,CAAC,yBAAyB,IAAI,oBAAoB;IAQ3D,IAAI,aAAa,IAAI,YAAY,CAEhC;IAED,IAAI,gBAAgB,IAAI,cAAc,EAAE,CAEvC;IAED,IAAI,UAAU,IAAI,cAAc,EAAE,CAIjC;IAEK,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAsD3B,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IA2B/B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAwC/B,aAAa,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IAItC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAetB,mBAAmB,CAAC,OAAO,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC;IAG1D;;;OAGG;IACG,SAAS,IAAI,OAAO,CAAC,YAAY,CAAC;cA0FxB,UAAU,IAAI,OAAO,CAAC,MAAM,CAAC;IAMvC,OAAO,CACX,cAAc,EAAE,YAAY,EAC5B,aAAa,EAAE,YAAY,GAC1B,OAAO,CAAC,eAAe,CAAC;IA0BrB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IA2FjC;;OAEG;IACG,GAAG,CACP,OAAO,EAAE,QAAQ,EACjB,IAAI,CAAC,EAAE;QACL,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,MAAM,CAAC,EAAE;YAAE,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;SAAE,CAAC;QAChC,EAAE,CAAC,EAAE,MAAM,CAAC;KACb,GACA,OAAO,CAAC,GAAG,CAAC;IAST,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAYhC"}
|
package/dist/src/o-node.js
CHANGED
|
@@ -9,6 +9,10 @@ import { oNodeConnectionManager } from './connection/o-node-connection.manager.j
|
|
|
9
9
|
import { oNodeResolver } from './router/resolvers/o-node.resolver.js';
|
|
10
10
|
import { oMethodResolver, oToolBase } from '@olane/o-tool';
|
|
11
11
|
import { oLeaderResolverFallback } from './router/index.js';
|
|
12
|
+
import { oNodeNotificationManager } from './o-node.notification-manager.js';
|
|
13
|
+
import { oConnectionHeartbeatManager } from './managers/o-connection-heartbeat.manager.js';
|
|
14
|
+
import { oReconnectionManager } from './managers/o-reconnection.manager.js';
|
|
15
|
+
import { LeaderRequestWrapper } from './utils/leader-request-wrapper.js';
|
|
12
16
|
export class oNode extends oToolBase {
|
|
13
17
|
constructor(config) {
|
|
14
18
|
super(config);
|
|
@@ -43,6 +47,9 @@ export class oNode extends oToolBase {
|
|
|
43
47
|
});
|
|
44
48
|
this.router = new oNodeRouter();
|
|
45
49
|
}
|
|
50
|
+
createNotificationManager() {
|
|
51
|
+
return new oNodeNotificationManager(this.p2pNode, this.hierarchyManager, this.address);
|
|
52
|
+
}
|
|
46
53
|
get staticAddress() {
|
|
47
54
|
return this.config.address;
|
|
48
55
|
}
|
|
@@ -60,6 +67,30 @@ export class oNode extends oToolBase {
|
|
|
60
67
|
this.logger.debug('Skipping unregistration, node is leader');
|
|
61
68
|
return;
|
|
62
69
|
}
|
|
70
|
+
// Notify parent we're stopping (best-effort, 2s timeout)
|
|
71
|
+
if (this.config.parent) {
|
|
72
|
+
try {
|
|
73
|
+
await Promise.race([
|
|
74
|
+
this.use(this.config.parent, {
|
|
75
|
+
method: 'notify',
|
|
76
|
+
params: {
|
|
77
|
+
eventType: 'node:stopping',
|
|
78
|
+
eventData: {
|
|
79
|
+
address: this.address.toString(),
|
|
80
|
+
reason: 'graceful_shutdown',
|
|
81
|
+
expectedDowntime: null,
|
|
82
|
+
},
|
|
83
|
+
source: this.address.toString(),
|
|
84
|
+
},
|
|
85
|
+
}),
|
|
86
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error('timeout')), 2000)),
|
|
87
|
+
]);
|
|
88
|
+
this.logger.debug('Notified parent of shutdown');
|
|
89
|
+
}
|
|
90
|
+
catch (error) {
|
|
91
|
+
this.logger.warn('Failed to notify parent (will be detected by heartbeat):', error instanceof Error ? error.message : error);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
63
94
|
if (!this.config.leader) {
|
|
64
95
|
this.logger.debug('No leader found, skipping unregistration');
|
|
65
96
|
return;
|
|
@@ -136,6 +167,10 @@ export class oNode extends oToolBase {
|
|
|
136
167
|
}
|
|
137
168
|
async start() {
|
|
138
169
|
await super.start();
|
|
170
|
+
// Start heartbeat after node is running
|
|
171
|
+
if (this.connectionHeartbeatManager) {
|
|
172
|
+
await this.connectionHeartbeatManager.start();
|
|
173
|
+
}
|
|
139
174
|
// await NetworkUtils.advertiseToNetwork(
|
|
140
175
|
// this.address,
|
|
141
176
|
// this.staticAddress,
|
|
@@ -256,6 +291,8 @@ export class oNode extends oToolBase {
|
|
|
256
291
|
}
|
|
257
292
|
await this.createNode();
|
|
258
293
|
await this.initializeRouter();
|
|
294
|
+
// need to wait until our libpp2 node is initialized before calling super.initialize
|
|
295
|
+
await super.initialize();
|
|
259
296
|
this.logger.debug('Node initialized!', this.transports.map((t) => t.toString()));
|
|
260
297
|
this.address.setTransports(this.transports);
|
|
261
298
|
this.peerId = this.p2pNode.peerId;
|
|
@@ -263,6 +300,16 @@ export class oNode extends oToolBase {
|
|
|
263
300
|
this.connectionManager = new oNodeConnectionManager({
|
|
264
301
|
p2pNode: this.p2pNode,
|
|
265
302
|
});
|
|
303
|
+
// Initialize leader request wrapper
|
|
304
|
+
this.leaderRequestWrapper = new LeaderRequestWrapper({
|
|
305
|
+
enabled: this.config.leaderRetry?.enabled ?? true,
|
|
306
|
+
maxAttempts: this.config.leaderRetry?.maxAttempts ?? 20,
|
|
307
|
+
baseDelayMs: this.config.leaderRetry?.baseDelayMs ?? 2000,
|
|
308
|
+
maxDelayMs: this.config.leaderRetry?.maxDelayMs ?? 30000,
|
|
309
|
+
timeoutMs: this.config.leaderRetry?.timeoutMs ?? 10000,
|
|
310
|
+
});
|
|
311
|
+
this.logger.info(`Leader retry config: enabled=${this.leaderRequestWrapper.getConfig().enabled}, ` +
|
|
312
|
+
`maxAttempts=${this.leaderRequestWrapper.getConfig().maxAttempts}`);
|
|
266
313
|
// initialize address resolution
|
|
267
314
|
this.router.addResolver(new oMethodResolver(this.address));
|
|
268
315
|
this.router.addResolver(new oNodeResolver(this.address));
|
|
@@ -271,8 +318,43 @@ export class oNode extends oToolBase {
|
|
|
271
318
|
this.logger.debug('Adding leader resolver fallback...');
|
|
272
319
|
this.router.addResolver(new oLeaderResolverFallback(this.address));
|
|
273
320
|
}
|
|
321
|
+
// Read ENABLE_LEADER_HEARTBEAT environment variable
|
|
322
|
+
const enableLeaderHeartbeat = process.env.ENABLE_LEADER_HEARTBEAT === 'true';
|
|
323
|
+
// Initialize connection heartbeat manager
|
|
324
|
+
this.connectionHeartbeatManager = new oConnectionHeartbeatManager(this.p2pNode, this.hierarchyManager, this.notificationManager, this.address, {
|
|
325
|
+
enabled: this.config.connectionHeartbeat?.enabled ?? true,
|
|
326
|
+
intervalMs: this.config.connectionHeartbeat?.intervalMs ?? 15000,
|
|
327
|
+
timeoutMs: this.config.connectionHeartbeat?.timeoutMs ?? 5000,
|
|
328
|
+
failureThreshold: this.config.connectionHeartbeat?.failureThreshold ?? 3,
|
|
329
|
+
checkChildren: this.config.connectionHeartbeat?.checkChildren ?? true,
|
|
330
|
+
checkParent: this.config.connectionHeartbeat?.checkParent ?? true,
|
|
331
|
+
checkLeader: this.config.connectionHeartbeat?.checkLeader ?? enableLeaderHeartbeat,
|
|
332
|
+
});
|
|
333
|
+
this.logger.info(`Connection heartbeat config: leader=${this.connectionHeartbeatManager.getConfig().checkLeader}, ` +
|
|
334
|
+
`parent=${this.connectionHeartbeatManager.getConfig().checkParent}`);
|
|
335
|
+
// Initialize reconnection manager
|
|
336
|
+
if (this.config.reconnection?.enabled !== false) {
|
|
337
|
+
this.reconnectionManager = new oReconnectionManager(this, {
|
|
338
|
+
enabled: true,
|
|
339
|
+
maxAttempts: this.config.reconnection?.maxAttempts ?? 10,
|
|
340
|
+
baseDelayMs: this.config.reconnection?.baseDelayMs ?? 5000,
|
|
341
|
+
maxDelayMs: this.config.reconnection?.maxDelayMs ?? 60000,
|
|
342
|
+
useLeaderFallback: this.config.reconnection?.useLeaderFallback ?? true,
|
|
343
|
+
});
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
/**
|
|
347
|
+
* Override use() to wrap leader/registry requests with retry logic
|
|
348
|
+
*/
|
|
349
|
+
async use(address, data) {
|
|
350
|
+
// Wrap leader/registry requests with retry logic
|
|
351
|
+
return this.leaderRequestWrapper.execute(() => super.use(address, data), address, data?.method);
|
|
274
352
|
}
|
|
275
353
|
async teardown() {
|
|
354
|
+
// Stop heartbeat before parent teardown
|
|
355
|
+
if (this.connectionHeartbeatManager) {
|
|
356
|
+
await this.connectionHeartbeatManager.stop();
|
|
357
|
+
}
|
|
276
358
|
await this.unregister();
|
|
277
359
|
await super.teardown();
|
|
278
360
|
if (this.p2pNode) {
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { Libp2p } from '@olane/o-config';
|
|
2
|
+
import { oNotificationManager } from '@olane/o-core';
|
|
3
|
+
import { oNodeAddress } from './router/o-node.address.js';
|
|
4
|
+
import { oNodeHierarchyManager } from './o-node.hierarchy-manager.js';
|
|
5
|
+
/**
|
|
6
|
+
* libp2p-specific implementation of oNotificationManager
|
|
7
|
+
* Wraps libp2p events and enriches them with Olane context
|
|
8
|
+
*/
|
|
9
|
+
export declare class oNodeNotificationManager extends oNotificationManager {
|
|
10
|
+
private p2pNode;
|
|
11
|
+
private hierarchyManager;
|
|
12
|
+
private address;
|
|
13
|
+
constructor(p2pNode: Libp2p, hierarchyManager: oNodeHierarchyManager, address: oNodeAddress);
|
|
14
|
+
/**
|
|
15
|
+
* Wire up libp2p event listeners
|
|
16
|
+
*/
|
|
17
|
+
protected setupListeners(): void;
|
|
18
|
+
/**
|
|
19
|
+
* Handle peer connect event from libp2p
|
|
20
|
+
*/
|
|
21
|
+
private handlePeerConnect;
|
|
22
|
+
/**
|
|
23
|
+
* Handle peer disconnect event from libp2p
|
|
24
|
+
*/
|
|
25
|
+
private handlePeerDisconnect;
|
|
26
|
+
/**
|
|
27
|
+
* Handle peer discovery event from libp2p
|
|
28
|
+
*/
|
|
29
|
+
private handlePeerDiscovery;
|
|
30
|
+
/**
|
|
31
|
+
* Handle connection open event from libp2p
|
|
32
|
+
*/
|
|
33
|
+
private handleConnectionOpen;
|
|
34
|
+
/**
|
|
35
|
+
* Handle connection close event from libp2p
|
|
36
|
+
*/
|
|
37
|
+
private handleConnectionClose;
|
|
38
|
+
/**
|
|
39
|
+
* Try to resolve a libp2p peer ID to an Olane address
|
|
40
|
+
* Checks hierarchy manager for known peers
|
|
41
|
+
*/
|
|
42
|
+
private peerIdToAddress;
|
|
43
|
+
/**
|
|
44
|
+
* Check if an address is a direct child
|
|
45
|
+
*/
|
|
46
|
+
private isChild;
|
|
47
|
+
/**
|
|
48
|
+
* Check if an address is a parent
|
|
49
|
+
*/
|
|
50
|
+
private isParent;
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=o-node.notification-manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"o-node.notification-manager.d.ts","sourceRoot":"","sources":["../../src/o-node.notification-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,OAAO,EACL,oBAAoB,EAQrB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAEtE;;;GAGG;AACH,qBAAa,wBAAyB,SAAQ,oBAAoB;IAE9D,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,gBAAgB;IACxB,OAAO,CAAC,OAAO;gBAFP,OAAO,EAAE,MAAM,EACf,gBAAgB,EAAE,qBAAqB,EACvC,OAAO,EAAE,YAAY;IAK/B;;OAEG;IACH,SAAS,CAAC,cAAc,IAAI,IAAI;IAiBhC;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAgDzB;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAkD5B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAmB3B;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAK5B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAK7B;;;OAGG;IACH,OAAO,CAAC,eAAe;IAkCvB;;OAEG;IACH,OAAO,CAAC,OAAO;IAMf;;OAEG;IACH,OAAO,CAAC,QAAQ;CAKjB"}
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
import { oNotificationManager, NodeConnectedEvent, NodeDisconnectedEvent, NodeDiscoveredEvent, ChildJoinedEvent, ChildLeftEvent, ParentConnectedEvent, ParentDisconnectedEvent, } from '@olane/o-core';
|
|
2
|
+
/**
|
|
3
|
+
* libp2p-specific implementation of oNotificationManager
|
|
4
|
+
* Wraps libp2p events and enriches them with Olane context
|
|
5
|
+
*/
|
|
6
|
+
export class oNodeNotificationManager extends oNotificationManager {
|
|
7
|
+
constructor(p2pNode, hierarchyManager, address) {
|
|
8
|
+
super();
|
|
9
|
+
this.p2pNode = p2pNode;
|
|
10
|
+
this.hierarchyManager = hierarchyManager;
|
|
11
|
+
this.address = address;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Wire up libp2p event listeners
|
|
15
|
+
*/
|
|
16
|
+
setupListeners() {
|
|
17
|
+
this.logger.debug('Setting up libp2p event listeners...');
|
|
18
|
+
// Peer connection events
|
|
19
|
+
this.p2pNode.addEventListener('peer:connect', this.handlePeerConnect.bind(this));
|
|
20
|
+
this.p2pNode.addEventListener('peer:disconnect', this.handlePeerDisconnect.bind(this));
|
|
21
|
+
// Peer discovery events
|
|
22
|
+
this.p2pNode.addEventListener('peer:discovery', this.handlePeerDiscovery.bind(this));
|
|
23
|
+
// Connection events
|
|
24
|
+
this.p2pNode.addEventListener('connection:open', this.handleConnectionOpen.bind(this));
|
|
25
|
+
this.p2pNode.addEventListener('connection:close', this.handleConnectionClose.bind(this));
|
|
26
|
+
this.logger.debug('libp2p event listeners configured');
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Handle peer connect event from libp2p
|
|
30
|
+
*/
|
|
31
|
+
handlePeerConnect(evt) {
|
|
32
|
+
const peerId = evt.detail;
|
|
33
|
+
this.logger.debug(`Peer connected: ${peerId.toString()}`);
|
|
34
|
+
// Try to resolve peer ID to Olane address
|
|
35
|
+
const nodeAddress = this.peerIdToAddress(peerId.toString());
|
|
36
|
+
if (!nodeAddress) {
|
|
37
|
+
this.logger.debug(`Could not resolve peer ID ${peerId.toString()} to address`);
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
// Emit generic node connected event
|
|
41
|
+
this.emit(new NodeConnectedEvent({
|
|
42
|
+
source: this.address,
|
|
43
|
+
nodeAddress,
|
|
44
|
+
connectionMetadata: {
|
|
45
|
+
peerId: peerId.toString(),
|
|
46
|
+
transport: 'libp2p',
|
|
47
|
+
},
|
|
48
|
+
}));
|
|
49
|
+
// Check if this is a child node
|
|
50
|
+
if (this.isChild(nodeAddress)) {
|
|
51
|
+
this.logger.debug(`Child node connected: ${nodeAddress.toString()}`);
|
|
52
|
+
this.emit(new ChildJoinedEvent({
|
|
53
|
+
source: this.address,
|
|
54
|
+
childAddress: nodeAddress,
|
|
55
|
+
parentAddress: this.address,
|
|
56
|
+
}));
|
|
57
|
+
}
|
|
58
|
+
// Check if this is a parent node
|
|
59
|
+
if (this.isParent(nodeAddress)) {
|
|
60
|
+
this.logger.debug(`Parent node connected: ${nodeAddress.toString()}`);
|
|
61
|
+
this.emit(new ParentConnectedEvent({
|
|
62
|
+
source: this.address,
|
|
63
|
+
parentAddress: nodeAddress,
|
|
64
|
+
}));
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Handle peer disconnect event from libp2p
|
|
69
|
+
*/
|
|
70
|
+
handlePeerDisconnect(evt) {
|
|
71
|
+
const peerId = evt.detail;
|
|
72
|
+
this.logger.debug(`Peer disconnected: ${peerId.toString()}`);
|
|
73
|
+
// Try to resolve peer ID to Olane address
|
|
74
|
+
const nodeAddress = this.peerIdToAddress(peerId.toString());
|
|
75
|
+
if (!nodeAddress) {
|
|
76
|
+
this.logger.debug(`Could not resolve peer ID ${peerId.toString()} to address`);
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
// Emit generic node disconnected event
|
|
80
|
+
this.emit(new NodeDisconnectedEvent({
|
|
81
|
+
source: this.address,
|
|
82
|
+
nodeAddress,
|
|
83
|
+
reason: 'peer_disconnected',
|
|
84
|
+
}));
|
|
85
|
+
// Check if this is a child node
|
|
86
|
+
if (this.isChild(nodeAddress)) {
|
|
87
|
+
this.logger.debug(`Child node disconnected: ${nodeAddress.toString()}`);
|
|
88
|
+
this.emit(new ChildLeftEvent({
|
|
89
|
+
source: this.address,
|
|
90
|
+
childAddress: nodeAddress,
|
|
91
|
+
parentAddress: this.address,
|
|
92
|
+
reason: 'peer_disconnected',
|
|
93
|
+
}));
|
|
94
|
+
// Optionally remove from hierarchy (auto-cleanup)
|
|
95
|
+
// this.hierarchyManager.removeChild(nodeAddress);
|
|
96
|
+
}
|
|
97
|
+
// Check if this is a parent node
|
|
98
|
+
if (this.isParent(nodeAddress)) {
|
|
99
|
+
this.logger.debug(`Parent node disconnected: ${nodeAddress.toString()}`);
|
|
100
|
+
this.emit(new ParentDisconnectedEvent({
|
|
101
|
+
source: this.address,
|
|
102
|
+
parentAddress: nodeAddress,
|
|
103
|
+
reason: 'peer_disconnected',
|
|
104
|
+
}));
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Handle peer discovery event from libp2p
|
|
109
|
+
*/
|
|
110
|
+
handlePeerDiscovery(evt) {
|
|
111
|
+
const peerInfo = evt.detail;
|
|
112
|
+
this.logger.debug(`Peer discovered: ${peerInfo.id.toString()}`);
|
|
113
|
+
// Try to resolve peer ID to Olane address
|
|
114
|
+
const nodeAddress = this.peerIdToAddress(peerInfo.id.toString());
|
|
115
|
+
if (!nodeAddress) {
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
this.emit(new NodeDiscoveredEvent({
|
|
119
|
+
source: this.address,
|
|
120
|
+
nodeAddress,
|
|
121
|
+
}));
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Handle connection open event from libp2p
|
|
125
|
+
*/
|
|
126
|
+
handleConnectionOpen(evt) {
|
|
127
|
+
const remotePeer = evt.detail.remotePeer;
|
|
128
|
+
this.logger.debug(`Connection opened to: ${remotePeer.toString()}`);
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Handle connection close event from libp2p
|
|
132
|
+
*/
|
|
133
|
+
handleConnectionClose(evt) {
|
|
134
|
+
const remotePeer = evt.detail.remotePeer;
|
|
135
|
+
this.logger.debug(`Connection closed to: ${remotePeer.toString()}`);
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Try to resolve a libp2p peer ID to an Olane address
|
|
139
|
+
* Checks hierarchy manager for known peers
|
|
140
|
+
*/
|
|
141
|
+
peerIdToAddress(peerId) {
|
|
142
|
+
// Check children
|
|
143
|
+
for (const child of this.hierarchyManager.children) {
|
|
144
|
+
const childTransports = child.transports;
|
|
145
|
+
for (const transport of childTransports) {
|
|
146
|
+
if (transport.toString().includes(peerId)) {
|
|
147
|
+
return child;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
// Check parents
|
|
152
|
+
for (const parent of this.hierarchyManager.parents) {
|
|
153
|
+
const parentTransports = parent.transports;
|
|
154
|
+
for (const transport of parentTransports) {
|
|
155
|
+
if (transport.toString().includes(peerId)) {
|
|
156
|
+
return parent;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
// Check leaders
|
|
161
|
+
for (const leader of this.hierarchyManager.leaders) {
|
|
162
|
+
const leaderTransports = leader.transports;
|
|
163
|
+
for (const transport of leaderTransports) {
|
|
164
|
+
if (transport.toString().includes(peerId)) {
|
|
165
|
+
return leader;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
return null;
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* Check if an address is a direct child
|
|
173
|
+
*/
|
|
174
|
+
isChild(address) {
|
|
175
|
+
return this.hierarchyManager.children.some((child) => child.toString() === address.toString());
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Check if an address is a parent
|
|
179
|
+
*/
|
|
180
|
+
isParent(address) {
|
|
181
|
+
return this.hierarchyManager.parents.some((parent) => parent.toString() === address.toString());
|
|
182
|
+
}
|
|
183
|
+
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"o-node.tool.d.ts","sourceRoot":"","sources":["../../src/o-node.tool.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,QAAQ,EAGR,QAAQ,
|
|
1
|
+
{"version":3,"file":"o-node.tool.d.ts","sourceRoot":"","sources":["../../src/o-node.tool.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,QAAQ,EAGR,QAAQ,EAGT,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;;AAIrD;;;;GAIG;AACH,qBAAa,SAAU,SAAQ,cAAkB;IACzC,cAAc,CAAC,OAAO,EAAE,QAAQ;IAQhC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAW3B,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAiDnE,cAAc,IAAI,OAAO,CAAC,GAAG,CAAC;IAQ9B,oBAAoB,CAAC,OAAO,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC;CA2B5D"}
|
package/dist/src/o-node.tool.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { CoreUtils,
|
|
1
|
+
import { CoreUtils, oError, oErrorCodes, oRequest, ChildJoinedEvent, } from '@olane/o-core';
|
|
2
2
|
import { oTool } from '@olane/o-tool';
|
|
3
3
|
import { oServerNode } from './nodes/server.node.js';
|
|
4
4
|
import { oNodeTransport } from './router/o-node.transport.js';
|
|
5
|
+
import { oNodeAddress } from './router/o-node.address.js';
|
|
5
6
|
/**
|
|
6
7
|
* oTool is a mixin that extends the base class and implements the oTool interface
|
|
7
8
|
* @param Base - The base class to extend
|
|
@@ -69,8 +70,17 @@ export class oNodeTool extends oTool(oServerNode) {
|
|
|
69
70
|
async _tool_child_register(request) {
|
|
70
71
|
this.logger.debug('Child register: ', request.params);
|
|
71
72
|
const { address, transports } = request.params;
|
|
72
|
-
const childAddress = new
|
|
73
|
+
const childAddress = new oNodeAddress(address, transports.map((t) => new oNodeTransport(t)));
|
|
74
|
+
// Add child to hierarchy
|
|
73
75
|
this.hierarchyManager.addChild(childAddress);
|
|
76
|
+
// Emit child joined event
|
|
77
|
+
if (this.notificationManager) {
|
|
78
|
+
this.notificationManager.emit(new ChildJoinedEvent({
|
|
79
|
+
source: this.address,
|
|
80
|
+
childAddress,
|
|
81
|
+
parentAddress: this.address,
|
|
82
|
+
}));
|
|
83
|
+
}
|
|
74
84
|
return {
|
|
75
85
|
message: `Child node registered with parent! ${childAddress.toString()}`,
|
|
76
86
|
parentTransports: this.parentTransports.map((t) => t.toString()),
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { oObject, oAddress } from '@olane/o-core';
|
|
2
|
+
export interface LeaderRetryConfig {
|
|
3
|
+
enabled: boolean;
|
|
4
|
+
maxAttempts: number;
|
|
5
|
+
baseDelayMs: number;
|
|
6
|
+
maxDelayMs: number;
|
|
7
|
+
timeoutMs: number;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Leader Request Wrapper
|
|
11
|
+
*
|
|
12
|
+
* Wraps requests to leader/registry with aggressive retry logic.
|
|
13
|
+
* Used when leader may be temporarily unavailable (healing, maintenance).
|
|
14
|
+
*
|
|
15
|
+
* Strategy:
|
|
16
|
+
* 1. Detect if request is to leader or registry
|
|
17
|
+
* 2. Apply retry logic with exponential backoff
|
|
18
|
+
* 3. Timeout individual attempts
|
|
19
|
+
* 4. Log retries for observability
|
|
20
|
+
*/
|
|
21
|
+
export declare class LeaderRequestWrapper extends oObject {
|
|
22
|
+
private config;
|
|
23
|
+
constructor(config: LeaderRetryConfig);
|
|
24
|
+
/**
|
|
25
|
+
* Check if address is a leader-related address that needs retry
|
|
26
|
+
*/
|
|
27
|
+
private isLeaderAddress;
|
|
28
|
+
/**
|
|
29
|
+
* Execute request with retry logic
|
|
30
|
+
*/
|
|
31
|
+
execute<T>(requestFn: () => Promise<T>, address: oAddress, context?: string): Promise<T>;
|
|
32
|
+
/**
|
|
33
|
+
* Calculate exponential backoff delay
|
|
34
|
+
*/
|
|
35
|
+
private calculateBackoffDelay;
|
|
36
|
+
/**
|
|
37
|
+
* Sleep utility
|
|
38
|
+
*/
|
|
39
|
+
private sleep;
|
|
40
|
+
/**
|
|
41
|
+
* Get current configuration
|
|
42
|
+
*/
|
|
43
|
+
getConfig(): LeaderRetryConfig;
|
|
44
|
+
}
|
|
45
|
+
//# sourceMappingURL=leader-request-wrapper.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"leader-request-wrapper.d.ts","sourceRoot":"","sources":["../../../src/utils/leader-request-wrapper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAuB,MAAM,eAAe,CAAC;AAEvE,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,oBAAqB,SAAQ,OAAO;IACnC,OAAO,CAAC,MAAM;gBAAN,MAAM,EAAE,iBAAiB;IAI7C;;OAEG;IACH,OAAO,CAAC,eAAe;IAKvB;;OAEG;IACG,OAAO,CAAC,CAAC,EACb,SAAS,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EAC3B,OAAO,EAAE,QAAQ,EACjB,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC,CAAC,CAAC;IAoEb;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAK7B;;OAEG;IACH,OAAO,CAAC,KAAK;IAIb;;OAEG;IACH,SAAS,IAAI,iBAAiB;CAG/B"}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { oObject } from '@olane/o-core';
|
|
2
|
+
/**
|
|
3
|
+
* Leader Request Wrapper
|
|
4
|
+
*
|
|
5
|
+
* Wraps requests to leader/registry with aggressive retry logic.
|
|
6
|
+
* Used when leader may be temporarily unavailable (healing, maintenance).
|
|
7
|
+
*
|
|
8
|
+
* Strategy:
|
|
9
|
+
* 1. Detect if request is to leader or registry
|
|
10
|
+
* 2. Apply retry logic with exponential backoff
|
|
11
|
+
* 3. Timeout individual attempts
|
|
12
|
+
* 4. Log retries for observability
|
|
13
|
+
*/
|
|
14
|
+
export class LeaderRequestWrapper extends oObject {
|
|
15
|
+
constructor(config) {
|
|
16
|
+
super();
|
|
17
|
+
this.config = config;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Check if address is a leader-related address that needs retry
|
|
21
|
+
*/
|
|
22
|
+
isLeaderAddress(address) {
|
|
23
|
+
const addressStr = address.toString();
|
|
24
|
+
return addressStr === 'o://leader' || addressStr === 'o://registry';
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Execute request with retry logic
|
|
28
|
+
*/
|
|
29
|
+
async execute(requestFn, address, context) {
|
|
30
|
+
// If retry disabled or not a leader address, execute directly
|
|
31
|
+
if (!this.config.enabled || !this.isLeaderAddress(address)) {
|
|
32
|
+
return await requestFn();
|
|
33
|
+
}
|
|
34
|
+
let attempt = 0;
|
|
35
|
+
let lastError;
|
|
36
|
+
while (attempt < this.config.maxAttempts) {
|
|
37
|
+
attempt++;
|
|
38
|
+
try {
|
|
39
|
+
this.logger.debug(`Leader request attempt ${attempt}/${this.config.maxAttempts}` +
|
|
40
|
+
(context ? ` (${context})` : ''));
|
|
41
|
+
// Create timeout promise
|
|
42
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
43
|
+
setTimeout(() => reject(new Error(`Leader request timeout after ${this.config.timeoutMs}ms`)), this.config.timeoutMs);
|
|
44
|
+
});
|
|
45
|
+
// Race between request and timeout
|
|
46
|
+
const result = await Promise.race([requestFn(), timeoutPromise]);
|
|
47
|
+
// Success!
|
|
48
|
+
if (attempt > 1) {
|
|
49
|
+
this.logger.info(`Leader request succeeded after ${attempt} attempts` +
|
|
50
|
+
(context ? ` (${context})` : ''));
|
|
51
|
+
}
|
|
52
|
+
return result;
|
|
53
|
+
}
|
|
54
|
+
catch (error) {
|
|
55
|
+
lastError = error instanceof Error ? error : new Error(String(error));
|
|
56
|
+
this.logger.warn(`Leader request attempt ${attempt} failed: ${lastError.message}` +
|
|
57
|
+
(context ? ` (${context})` : ''));
|
|
58
|
+
if (attempt < this.config.maxAttempts) {
|
|
59
|
+
const delay = this.calculateBackoffDelay(attempt);
|
|
60
|
+
this.logger.debug(`Waiting ${delay}ms before next attempt...`);
|
|
61
|
+
await this.sleep(delay);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
// All attempts failed
|
|
66
|
+
this.logger.error(`Leader request failed after ${this.config.maxAttempts} attempts` +
|
|
67
|
+
(context ? ` (${context})` : ''));
|
|
68
|
+
throw lastError || new Error('Leader request failed');
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Calculate exponential backoff delay
|
|
72
|
+
*/
|
|
73
|
+
calculateBackoffDelay(attempt) {
|
|
74
|
+
const delay = this.config.baseDelayMs * Math.pow(2, attempt - 1);
|
|
75
|
+
return Math.min(delay, this.config.maxDelayMs);
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Sleep utility
|
|
79
|
+
*/
|
|
80
|
+
sleep(ms) {
|
|
81
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Get current configuration
|
|
85
|
+
*/
|
|
86
|
+
getConfig() {
|
|
87
|
+
return { ...this.config };
|
|
88
|
+
}
|
|
89
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@olane/o-node",
|
|
3
|
-
"version": "0.7.12-alpha.
|
|
3
|
+
"version": "0.7.12-alpha.9",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "dist/src/index.js",
|
|
6
6
|
"types": "dist/src/index.d.ts",
|
|
@@ -54,12 +54,12 @@
|
|
|
54
54
|
"typescript": "5.4.5"
|
|
55
55
|
},
|
|
56
56
|
"dependencies": {
|
|
57
|
-
"@olane/o-config": "0.7.12-alpha.
|
|
58
|
-
"@olane/o-core": "0.7.12-alpha.
|
|
59
|
-
"@olane/o-protocol": "0.7.12-alpha.
|
|
60
|
-
"@olane/o-tool": "0.7.12-alpha.
|
|
57
|
+
"@olane/o-config": "0.7.12-alpha.9",
|
|
58
|
+
"@olane/o-core": "0.7.12-alpha.9",
|
|
59
|
+
"@olane/o-protocol": "0.7.12-alpha.9",
|
|
60
|
+
"@olane/o-tool": "0.7.12-alpha.9",
|
|
61
61
|
"debug": "^4.4.1",
|
|
62
62
|
"dotenv": "^16.5.0"
|
|
63
63
|
},
|
|
64
|
-
"gitHead": "
|
|
64
|
+
"gitHead": "21427acbdc1d87ff9465bf19bda58cca89e8d3c7"
|
|
65
65
|
}
|