@olane/o-node 0.7.12-alpha.11 → 0.7.12-alpha.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/interfaces/i-heartbeatable-node.d.ts +49 -0
- package/dist/src/interfaces/i-heartbeatable-node.d.ts.map +1 -0
- package/dist/src/interfaces/i-heartbeatable-node.js +1 -0
- package/dist/src/interfaces/o-node.config.d.ts +6 -0
- package/dist/src/interfaces/o-node.config.d.ts.map +1 -1
- package/dist/src/managers/o-connection-heartbeat.manager.d.ts +4 -8
- package/dist/src/managers/o-connection-heartbeat.manager.d.ts.map +1 -1
- package/dist/src/managers/o-connection-heartbeat.manager.js +31 -29
- package/dist/src/managers/o-reconnection.manager.d.ts +1 -1
- package/dist/src/managers/o-reconnection.manager.d.ts.map +1 -1
- package/dist/src/o-node.d.ts +5 -0
- package/dist/src/o-node.d.ts.map +1 -1
- package/dist/src/o-node.js +47 -19
- package/dist/src/router/o-node.routing-policy.d.ts.map +1 -1
- package/dist/src/router/o-node.routing-policy.js +3 -2
- package/dist/src/router/resolvers/o-node.search-resolver.d.ts.map +1 -1
- package/dist/src/router/resolvers/o-node.search-resolver.js +25 -7
- package/dist/src/utils/circuit-breaker.d.ts +107 -0
- package/dist/src/utils/circuit-breaker.d.ts.map +1 -0
- package/dist/src/utils/circuit-breaker.js +175 -0
- package/dist/src/utils/circuit-breaker.test.d.ts +2 -0
- package/dist/src/utils/circuit-breaker.test.d.ts.map +1 -0
- package/dist/src/utils/circuit-breaker.test.js +262 -0
- package/dist/src/utils/leader-request-wrapper.d.ts +26 -5
- package/dist/src/utils/leader-request-wrapper.d.ts.map +1 -1
- package/dist/src/utils/leader-request-wrapper.js +79 -8
- package/dist/src/utils/leader-request-wrapper.test.d.ts +1 -0
- package/dist/src/utils/leader-request-wrapper.test.d.ts.map +1 -0
- package/dist/src/utils/leader-request-wrapper.test.js +246 -0
- package/package.json +6 -6
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { oNotificationManager } from '@olane/o-core';
|
|
2
|
+
import { Libp2p } from '@olane/o-config';
|
|
3
|
+
import { oNodeAddress } from '../router/o-node.address.js';
|
|
4
|
+
/**
|
|
5
|
+
* Interface for nodes that support connection heartbeat monitoring.
|
|
6
|
+
* This interface defines the contract that oConnectionHeartbeatManager needs
|
|
7
|
+
* to access live hierarchy state without holding stale references.
|
|
8
|
+
*/
|
|
9
|
+
export interface IHeartbeatableNode {
|
|
10
|
+
/**
|
|
11
|
+
* The node's current address
|
|
12
|
+
*/
|
|
13
|
+
address: oNodeAddress;
|
|
14
|
+
/**
|
|
15
|
+
* The notification manager for emitting heartbeat events
|
|
16
|
+
*/
|
|
17
|
+
notificationManager: oNotificationManager;
|
|
18
|
+
/**
|
|
19
|
+
* The underlying libp2p node for ping operations
|
|
20
|
+
*/
|
|
21
|
+
p2pNode: Libp2p;
|
|
22
|
+
/**
|
|
23
|
+
* The current parent address (with live transport updates)
|
|
24
|
+
* @returns Parent address or null if no parent
|
|
25
|
+
*/
|
|
26
|
+
parent: oNodeAddress | null;
|
|
27
|
+
/**
|
|
28
|
+
* Get the current list of leader addresses
|
|
29
|
+
* @returns Array of leader addresses (empty if this node is the leader)
|
|
30
|
+
*/
|
|
31
|
+
getLeaders(): oNodeAddress[];
|
|
32
|
+
/**
|
|
33
|
+
* Get the current list of parent addresses
|
|
34
|
+
* @returns Array of parent addresses
|
|
35
|
+
*/
|
|
36
|
+
getParents(): oNodeAddress[];
|
|
37
|
+
/**
|
|
38
|
+
* Get the current list of child addresses
|
|
39
|
+
* @returns Array of child addresses
|
|
40
|
+
*/
|
|
41
|
+
getChildren(): oNodeAddress[];
|
|
42
|
+
/**
|
|
43
|
+
* Remove a child from the hierarchy
|
|
44
|
+
* @param childAddress The address of the child to remove
|
|
45
|
+
*/
|
|
46
|
+
removeChild(childAddress: oNodeAddress): void;
|
|
47
|
+
use(param1: any, param2: any): Promise<any>;
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=i-heartbeatable-node.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"i-heartbeatable-node.d.ts","sourceRoot":"","sources":["../../../src/interfaces/i-heartbeatable-node.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAE3D;;;;GAIG;AACH,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,OAAO,EAAE,YAAY,CAAC;IAEtB;;OAEG;IACH,mBAAmB,EAAE,oBAAoB,CAAC;IAE1C;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IAE5B;;;OAGG;IACH,UAAU,IAAI,YAAY,EAAE,CAAC;IAE7B;;;OAGG;IACH,UAAU,IAAI,YAAY,EAAE,CAAC;IAE7B;;;OAGG;IACH,WAAW,IAAI,YAAY,EAAE,CAAC;IAE9B;;;OAGG;IACH,WAAW,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAE9C,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;CAC7C"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -39,6 +39,12 @@ export interface oNodeConfig extends oCoreConfig {
|
|
|
39
39
|
baseDelayMs?: number;
|
|
40
40
|
maxDelayMs?: number;
|
|
41
41
|
timeoutMs?: number;
|
|
42
|
+
circuitBreaker?: {
|
|
43
|
+
enabled?: boolean;
|
|
44
|
+
failureThreshold?: number;
|
|
45
|
+
openTimeoutMs?: number;
|
|
46
|
+
halfOpenMaxAttempts?: number;
|
|
47
|
+
};
|
|
42
48
|
};
|
|
43
49
|
}
|
|
44
50
|
//# sourceMappingURL=o-node.config.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"o-node.config.d.ts","sourceRoot":"","sources":["../../../src/interfaces/o-node.config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAE3D,MAAM,WAAW,WAAY,SAAQ,WAAW;IAC9C,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IAE5B;;;OAGG;IACH,mBAAmB,CAAC,EAAE;QACpB,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,aAAa,CAAC,EAAE,OAAO,CAAC;QACxB,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB,CAAC;IAEF;;;OAGG;IACH,YAAY,CAAC,EAAE;QACb,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,yBAAyB,CAAC,EAAE,MAAM,CAAC;QACnC,yBAAyB,CAAC,EAAE,MAAM,CAAC;KACpC,CAAC;IAEF;;;OAGG;IACH,WAAW,CAAC,EAAE;QACZ,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,SAAS,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"o-node.config.d.ts","sourceRoot":"","sources":["../../../src/interfaces/o-node.config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAE3D,MAAM,WAAW,WAAY,SAAQ,WAAW;IAC9C,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IAE5B;;;OAGG;IACH,mBAAmB,CAAC,EAAE;QACpB,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,aAAa,CAAC,EAAE,OAAO,CAAC;QACxB,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB,CAAC;IAEF;;;OAGG;IACH,YAAY,CAAC,EAAE;QACb,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,yBAAyB,CAAC,EAAE,MAAM,CAAC;QACnC,yBAAyB,CAAC,EAAE,MAAM,CAAC;KACpC,CAAC;IAEF;;;OAGG;IACH,WAAW,CAAC,EAAE;QACZ,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,cAAc,CAAC,EAAE;YACf,OAAO,CAAC,EAAE,OAAO,CAAC;YAClB,gBAAgB,CAAC,EAAE,MAAM,CAAC;YAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;YACvB,mBAAmB,CAAC,EAAE,MAAM,CAAC;SAC9B,CAAC;KACH,CAAC;CACH"}
|
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
import { Libp2p } from '@olane/o-config';
|
|
2
1
|
import { oObject } from '@olane/o-core';
|
|
3
2
|
import { oNodeAddress } from '../router/o-node.address.js';
|
|
4
|
-
import {
|
|
5
|
-
import { oNotificationManager } from '@olane/o-core';
|
|
3
|
+
import { IHeartbeatableNode } from '../interfaces/i-heartbeatable-node.js';
|
|
6
4
|
export interface HeartbeatConfig {
|
|
7
5
|
enabled: boolean;
|
|
8
6
|
intervalMs: number;
|
|
@@ -35,17 +33,15 @@ export interface ConnectionHealth {
|
|
|
35
33
|
* - Emits ParentDisconnectedEvent when parent dies (triggers reconnection)
|
|
36
34
|
*/
|
|
37
35
|
export declare class oConnectionHeartbeatManager extends oObject {
|
|
38
|
-
private
|
|
39
|
-
private hierarchyManager;
|
|
40
|
-
private notificationManager;
|
|
41
|
-
private address;
|
|
36
|
+
private node;
|
|
42
37
|
private config;
|
|
43
38
|
private heartbeatInterval?;
|
|
44
39
|
private healthMap;
|
|
45
|
-
constructor(
|
|
40
|
+
constructor(node: IHeartbeatableNode, config: HeartbeatConfig);
|
|
46
41
|
start(): Promise<void>;
|
|
47
42
|
stop(): Promise<void>;
|
|
48
43
|
private performHeartbeatCycle;
|
|
44
|
+
private doPing;
|
|
49
45
|
private pingTarget;
|
|
50
46
|
private handleConnectionDead;
|
|
51
47
|
private emitConnectionDegradedEvent;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"o-connection-heartbeat.manager.d.ts","sourceRoot":"","sources":["../../../src/managers/o-connection-heartbeat.manager.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"o-connection-heartbeat.manager.d.ts","sourceRoot":"","sources":["../../../src/managers/o-connection-heartbeat.manager.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,OAAO,EAOR,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,kBAAkB,EAAE,MAAM,uCAAuC,CAAC;AAE3E,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,EAAE,OAAO,CAAC;IACrB,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,YAAY,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,kBAAkB,EAAE,MAAM,CAAC;IAC3B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,SAAS,GAAG,UAAU,GAAG,MAAM,CAAC;CACzC;AAED;;;;;;;;;;;;;GAaG;AACH,qBAAa,2BAA4B,SAAQ,OAAO;IAKpD,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,MAAM;IALhB,OAAO,CAAC,iBAAiB,CAAC,CAAiB;IAC3C,OAAO,CAAC,SAAS,CAAuC;gBAG9C,IAAI,EAAE,kBAAkB,EACxB,MAAM,EAAE,eAAe;IAK3B,KAAK;IAqBL,IAAI;YAQI,qBAAqB;IA0CnC,OAAO,CAAC,MAAM;YAOA,UAAU;IAuFxB,OAAO,CAAC,oBAAoB;IAyD5B,OAAO,CAAC,2BAA2B;IAmBnC,OAAO,CAAC,4BAA4B;IAiBpC,OAAO,CAAC,wBAAwB;IAahC;;OAEG;IACH,eAAe,IAAI,gBAAgB,EAAE;IAIrC;;OAEG;IACH,mBAAmB,CAAC,OAAO,EAAE,YAAY,GAAG,gBAAgB,GAAG,SAAS;IAIxE;;OAEG;IACH,SAAS,IAAI,eAAe;CAG7B"}
|
|
@@ -14,12 +14,9 @@ import { oObject, ChildLeftEvent, ParentDisconnectedEvent, LeaderDisconnectedEve
|
|
|
14
14
|
* - Emits ParentDisconnectedEvent when parent dies (triggers reconnection)
|
|
15
15
|
*/
|
|
16
16
|
export class oConnectionHeartbeatManager extends oObject {
|
|
17
|
-
constructor(
|
|
17
|
+
constructor(node, config) {
|
|
18
18
|
super();
|
|
19
|
-
this.
|
|
20
|
-
this.hierarchyManager = hierarchyManager;
|
|
21
|
-
this.notificationManager = notificationManager;
|
|
22
|
-
this.address = address;
|
|
19
|
+
this.node = node;
|
|
23
20
|
this.config = config;
|
|
24
21
|
this.healthMap = new Map();
|
|
25
22
|
}
|
|
@@ -45,24 +42,27 @@ export class oConnectionHeartbeatManager extends oObject {
|
|
|
45
42
|
async performHeartbeatCycle() {
|
|
46
43
|
const targets = [];
|
|
47
44
|
// Check if this is a leader node (no leader in hierarchy = we are leader)
|
|
48
|
-
const isLeaderNode = this.
|
|
45
|
+
const isLeaderNode = this.node.getLeaders().length === 0;
|
|
49
46
|
// Collect leader (if enabled and we're not the leader)
|
|
50
47
|
if (!isLeaderNode) {
|
|
51
|
-
const leaders = this.
|
|
48
|
+
const leaders = this.node.getLeaders();
|
|
52
49
|
for (const leader of leaders) {
|
|
53
50
|
targets.push({ address: leader, role: 'leader' });
|
|
54
51
|
}
|
|
55
52
|
}
|
|
56
53
|
// Collect parent
|
|
57
54
|
if (this.config.checkParent && !isLeaderNode) {
|
|
58
|
-
|
|
59
|
-
|
|
55
|
+
// Use this.node.parent getter to get the current parent address with transports
|
|
56
|
+
// rather than getParents() which may have a stale reference
|
|
57
|
+
const parent = this.node.parent;
|
|
58
|
+
this.logger.debug('Parent address:', parent);
|
|
59
|
+
if (parent) {
|
|
60
60
|
targets.push({ address: parent, role: 'parent' });
|
|
61
61
|
}
|
|
62
62
|
}
|
|
63
63
|
// Collect children
|
|
64
64
|
if (this.config.checkChildren) {
|
|
65
|
-
const children = this.
|
|
65
|
+
const children = this.node.getChildren();
|
|
66
66
|
for (const child of children) {
|
|
67
67
|
targets.push({ address: child, role: 'child' });
|
|
68
68
|
}
|
|
@@ -70,12 +70,17 @@ export class oConnectionHeartbeatManager extends oObject {
|
|
|
70
70
|
// Ping all targets in parallel
|
|
71
71
|
await Promise.allSettled(targets.map((target) => this.pingTarget(target.address, target.role)));
|
|
72
72
|
}
|
|
73
|
+
doPing(address) {
|
|
74
|
+
return this.node.use(address, {
|
|
75
|
+
method: 'ping',
|
|
76
|
+
params: {},
|
|
77
|
+
});
|
|
78
|
+
}
|
|
73
79
|
async pingTarget(address, role) {
|
|
74
80
|
if (!address.libp2pTransports.length) {
|
|
75
|
-
this.logger.
|
|
81
|
+
this.logger.debug(`${role} has no transports, skipping ping`, address);
|
|
76
82
|
return;
|
|
77
83
|
}
|
|
78
|
-
const transports = address.libp2pTransports;
|
|
79
84
|
const key = address.toString();
|
|
80
85
|
let health = this.healthMap.get(key);
|
|
81
86
|
if (!health) {
|
|
@@ -98,10 +103,7 @@ export class oConnectionHeartbeatManager extends oObject {
|
|
|
98
103
|
});
|
|
99
104
|
// Race between ping and timeout
|
|
100
105
|
// The ping service accepts PeerId as string or object
|
|
101
|
-
await Promise.race([
|
|
102
|
-
this.p2pNode.services.ping.ping(transports[0].toMultiaddr()),
|
|
103
|
-
timeoutPromise,
|
|
104
|
-
]);
|
|
106
|
+
await Promise.race([this.doPing(address), timeoutPromise]);
|
|
105
107
|
const latency = Date.now() - startTime;
|
|
106
108
|
// Success - update health
|
|
107
109
|
health.lastSuccessfulPing = Date.now();
|
|
@@ -121,7 +123,7 @@ export class oConnectionHeartbeatManager extends oObject {
|
|
|
121
123
|
}
|
|
122
124
|
catch (error) {
|
|
123
125
|
health.consecutiveFailures++;
|
|
124
|
-
this.logger.warn(`Ping failed: ${address} (failures: ${health.consecutiveFailures}/${this.config.failureThreshold})
|
|
126
|
+
this.logger.warn(`Ping failed: ${address} (failures: ${health.consecutiveFailures}/${this.config.failureThreshold})`, error);
|
|
125
127
|
// Update status based on failure count
|
|
126
128
|
if (health.consecutiveFailures >= this.config.failureThreshold) {
|
|
127
129
|
this.handleConnectionDead(address, role, health);
|
|
@@ -141,20 +143,20 @@ export class oConnectionHeartbeatManager extends oObject {
|
|
|
141
143
|
// Emit events based on role
|
|
142
144
|
if (role === 'child') {
|
|
143
145
|
// Remove dead child from hierarchy
|
|
144
|
-
this.
|
|
146
|
+
this.node.removeChild(address);
|
|
145
147
|
// Emit child left event
|
|
146
|
-
this.notificationManager.emit(new ChildLeftEvent({
|
|
147
|
-
source: this.address,
|
|
148
|
+
this.node.notificationManager.emit(new ChildLeftEvent({
|
|
149
|
+
source: this.node.address,
|
|
148
150
|
childAddress: address,
|
|
149
|
-
parentAddress: this.address,
|
|
151
|
+
parentAddress: this.node.address,
|
|
150
152
|
reason: `heartbeat_failed_${health.consecutiveFailures}_times`,
|
|
151
153
|
}));
|
|
152
154
|
this.logger.warn(`Removed dead child: ${address}`);
|
|
153
155
|
}
|
|
154
156
|
else if (role === 'parent') {
|
|
155
157
|
// Emit parent disconnected event
|
|
156
|
-
this.notificationManager.emit(new ParentDisconnectedEvent({
|
|
157
|
-
source: this.address,
|
|
158
|
+
this.node.notificationManager.emit(new ParentDisconnectedEvent({
|
|
159
|
+
source: this.node.address,
|
|
158
160
|
parentAddress: address,
|
|
159
161
|
reason: `heartbeat_failed_${health.consecutiveFailures}_times`,
|
|
160
162
|
}));
|
|
@@ -163,8 +165,8 @@ export class oConnectionHeartbeatManager extends oObject {
|
|
|
163
165
|
}
|
|
164
166
|
else if (role === 'leader') {
|
|
165
167
|
// Emit leader disconnected event
|
|
166
|
-
this.notificationManager.emit(new LeaderDisconnectedEvent({
|
|
167
|
-
source: this.address,
|
|
168
|
+
this.node.notificationManager.emit(new LeaderDisconnectedEvent({
|
|
169
|
+
source: this.node.address,
|
|
168
170
|
leaderAddress: address,
|
|
169
171
|
reason: `heartbeat_failed_${health.consecutiveFailures}_times`,
|
|
170
172
|
}));
|
|
@@ -175,8 +177,8 @@ export class oConnectionHeartbeatManager extends oObject {
|
|
|
175
177
|
emitConnectionDegradedEvent(address, role, failures) {
|
|
176
178
|
// ConnectionDegradedEvent only supports parent/child, so we map leader to parent
|
|
177
179
|
const eventRole = role === 'leader' ? 'parent' : role === 'child' ? 'child' : 'parent';
|
|
178
|
-
this.notificationManager.emit(new ConnectionDegradedEvent({
|
|
179
|
-
source: this.address,
|
|
180
|
+
this.node.notificationManager.emit(new ConnectionDegradedEvent({
|
|
181
|
+
source: this.node.address,
|
|
180
182
|
targetAddress: address,
|
|
181
183
|
role: eventRole,
|
|
182
184
|
consecutiveFailures: failures,
|
|
@@ -185,8 +187,8 @@ export class oConnectionHeartbeatManager extends oObject {
|
|
|
185
187
|
emitConnectionRecoveredEvent(address, role) {
|
|
186
188
|
// ConnectionRecoveredEvent only supports parent/child, so we map leader to parent
|
|
187
189
|
const eventRole = role === 'leader' ? 'parent' : role === 'child' ? 'child' : 'parent';
|
|
188
|
-
this.notificationManager.emit(new ConnectionRecoveredEvent({
|
|
189
|
-
source: this.address,
|
|
190
|
+
this.node.notificationManager.emit(new ConnectionRecoveredEvent({
|
|
191
|
+
source: this.node.address,
|
|
190
192
|
targetAddress: address,
|
|
191
193
|
role: eventRole,
|
|
192
194
|
}));
|
|
@@ -41,7 +41,7 @@ export declare class oReconnectionManager extends oObject {
|
|
|
41
41
|
/**
|
|
42
42
|
* Wait for non-leader parent to appear in registry and reconnect
|
|
43
43
|
*/
|
|
44
|
-
|
|
44
|
+
waitForParentAndReconnect(): Promise<void>;
|
|
45
45
|
private handleReconnectionFailure;
|
|
46
46
|
private calculateNodeLevel;
|
|
47
47
|
private calculateBackoffDelay;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"o-reconnection.manager.d.ts","sourceRoot":"","sources":["../../../src/managers/o-reconnection.manager.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,OAAO,EAQR,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,kBAAkB,EAAE,MAAM,uCAAuC,CAAC;AAI3E,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,yBAAyB,EAAE,MAAM,CAAC;IAClC,yBAAyB,EAAE,MAAM,CAAC;CACnC;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,oBAAqB,SAAQ,OAAO;IAI7C,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,MAAM;IAJhB,OAAO,CAAC,YAAY,CAAS;gBAGnB,IAAI,EAAE,kBAAkB,EACxB,MAAM,EAAE,kBAAkB;IAMpC,OAAO,CAAC,mBAAmB;YAoBb,wBAAwB;YAaxB,wBAAwB;YAexB,wBAAwB;IAehC,mBAAmB;
|
|
1
|
+
{"version":3,"file":"o-reconnection.manager.d.ts","sourceRoot":"","sources":["../../../src/managers/o-reconnection.manager.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,OAAO,EAQR,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,kBAAkB,EAAE,MAAM,uCAAuC,CAAC;AAI3E,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,yBAAyB,EAAE,MAAM,CAAC;IAClC,yBAAyB,EAAE,MAAM,CAAC;CACnC;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,oBAAqB,SAAQ,OAAO;IAI7C,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,MAAM;IAJhB,OAAO,CAAC,YAAY,CAAS;gBAGnB,IAAI,EAAE,kBAAkB,EACxB,MAAM,EAAE,kBAAkB;IAMpC,OAAO,CAAC,mBAAmB;YAoBb,wBAAwB;YAaxB,wBAAwB;YAexB,wBAAwB;IAehC,mBAAmB;YAgDX,2BAA2B;YAiB3B,iBAAiB;IAkB/B;;;OAGG;YACW,yBAAyB;IAiFvC;;OAEG;IACG,yBAAyB;IAiG/B,OAAO,CAAC,yBAAyB;IAajC,OAAO,CAAC,kBAAkB;IAI1B,OAAO,CAAC,qBAAqB;IAK7B,OAAO,CAAC,KAAK;CAGd"}
|
package/dist/src/o-node.d.ts
CHANGED
|
@@ -45,6 +45,7 @@ export declare class oNode extends oToolBase {
|
|
|
45
45
|
configure(): Promise<Libp2pConfig>;
|
|
46
46
|
protected createNode(): Promise<Libp2p>;
|
|
47
47
|
connect(nextHopAddress: oNodeAddress, targetAddress: oNodeAddress): Promise<oNodeConnection>;
|
|
48
|
+
postInitialize(): Promise<void>;
|
|
48
49
|
initialize(): Promise<void>;
|
|
49
50
|
/**
|
|
50
51
|
* Override use() to wrap leader/registry requests with retry logic
|
|
@@ -57,5 +58,9 @@ export declare class oNode extends oToolBase {
|
|
|
57
58
|
id?: string;
|
|
58
59
|
}): Promise<any>;
|
|
59
60
|
teardown(): Promise<void>;
|
|
61
|
+
getLeaders(): oNodeAddress[];
|
|
62
|
+
getParents(): oNodeAddress[];
|
|
63
|
+
getChildren(): oNodeAddress[];
|
|
64
|
+
removeChild(childAddress: oNodeAddress): void;
|
|
60
65
|
}
|
|
61
66
|
//# sourceMappingURL=o-node.d.ts.map
|
package/dist/src/o-node.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"o-node.d.ts","sourceRoot":"","sources":["../../src/o-node.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,MAAM,EACN,YAAY,EACb,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AACtE,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAIL,QAAQ,EACR,QAAQ,EAER,oBAAoB,EACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,mCAAmC,CAAC;AACpE,OAAO,EAAE,sBAAsB,EAAE,MAAM,2CAA2C,CAAC;AAGnF,OAAO,EAAmB,SAAS,EAAE,MAAM,eAAe,CAAC;AAI3D,OAAO,EAAE,2BAA2B,EAAE,MAAM,8CAA8C,CAAC;AAC3F,OAAO,EAAE,oBAAoB,EAAE,MAAM,sCAAsC,CAAC;AAC5E,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AAEzE,qBAAa,KAAM,SAAQ,SAAS;IAC3B,MAAM,EAAG,MAAM,CAAC;IAChB,OAAO,EAAG,MAAM,CAAC;IACjB,OAAO,EAAG,YAAY,CAAC;IACvB,MAAM,EAAE,WAAW,CAAC;IACpB,iBAAiB,EAAG,sBAAsB,CAAC;IAC3C,gBAAgB,EAAG,qBAAqB,CAAC;IACzC,0BAA0B,CAAC,EAAE,2BAA2B,CAAC;IACzD,mBAAmB,CAAC,EAAE,oBAAoB,CAAC;IAC3C,oBAAoB,EAAG,oBAAoB,CAAC;IACnD,SAAS,CAAC,WAAW,EAAE,OAAO,CAAS;gBAE3B,MAAM,EAAE,WAAW;IAK/B,IAAI,MAAM,IAAI,YAAY,GAAG,IAAI,CAEhC;IAED,IAAI,aAAa,IAAI,YAAY,CAKhC;IAED,IAAI,YAAY,IAAI,MAAM,GAAG,IAAI,CAOhC;IAED,mBAAmB,IAAI,GAAG,EAAE;IAItB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC;IASvC,SAAS,CAAC,yBAAyB,IAAI,oBAAoB;IAQ3D,IAAI,aAAa,IAAI,YAAY,CAEhC;IAED,IAAI,gBAAgB,IAAI,cAAc,EAAE,CAEvC;IAED,IAAI,UAAU,IAAI,cAAc,EAAE,CAIjC;IAEK,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAsD3B,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"o-node.d.ts","sourceRoot":"","sources":["../../src/o-node.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,MAAM,EACN,YAAY,EACb,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AACtE,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAIL,QAAQ,EACR,QAAQ,EAER,oBAAoB,EACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,mCAAmC,CAAC;AACpE,OAAO,EAAE,sBAAsB,EAAE,MAAM,2CAA2C,CAAC;AAGnF,OAAO,EAAmB,SAAS,EAAE,MAAM,eAAe,CAAC;AAI3D,OAAO,EAAE,2BAA2B,EAAE,MAAM,8CAA8C,CAAC;AAC3F,OAAO,EAAE,oBAAoB,EAAE,MAAM,sCAAsC,CAAC;AAC5E,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AAEzE,qBAAa,KAAM,SAAQ,SAAS;IAC3B,MAAM,EAAG,MAAM,CAAC;IAChB,OAAO,EAAG,MAAM,CAAC;IACjB,OAAO,EAAG,YAAY,CAAC;IACvB,MAAM,EAAE,WAAW,CAAC;IACpB,iBAAiB,EAAG,sBAAsB,CAAC;IAC3C,gBAAgB,EAAG,qBAAqB,CAAC;IACzC,0BAA0B,CAAC,EAAE,2BAA2B,CAAC;IACzD,mBAAmB,CAAC,EAAE,oBAAoB,CAAC;IAC3C,oBAAoB,EAAG,oBAAoB,CAAC;IACnD,SAAS,CAAC,WAAW,EAAE,OAAO,CAAS;gBAE3B,MAAM,EAAE,WAAW;IAK/B,IAAI,MAAM,IAAI,YAAY,GAAG,IAAI,CAEhC;IAED,IAAI,aAAa,IAAI,YAAY,CAKhC;IAED,IAAI,YAAY,IAAI,MAAM,GAAG,IAAI,CAOhC;IAED,mBAAmB,IAAI,GAAG,EAAE;IAItB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC;IASvC,SAAS,CAAC,yBAAyB,IAAI,oBAAoB;IAQ3D,IAAI,aAAa,IAAI,YAAY,CAEhC;IAED,IAAI,gBAAgB,IAAI,cAAc,EAAE,CAEvC;IAED,IAAI,UAAU,IAAI,cAAc,EAAE,CAIjC;IAEK,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAsD3B,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAiC/B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAwC/B,aAAa,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IAItC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAetB,mBAAmB,CAAC,OAAO,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC;IAG1D;;;OAGG;IACG,SAAS,IAAI,OAAO,CAAC,YAAY,CAAC;cA0FxB,UAAU,IAAI,OAAO,CAAC,MAAM,CAAC;IAMvC,OAAO,CACX,cAAc,EAAE,YAAY,EAC5B,aAAa,EAAE,YAAY,GAC1B,OAAO,CAAC,eAAe,CAAC;IA0BrB,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAsB/B,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAkFjC;;OAEG;IACG,GAAG,CACP,OAAO,EAAE,QAAQ,EACjB,IAAI,CAAC,EAAE;QACL,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,MAAM,CAAC,EAAE;YAAE,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;SAAE,CAAC;QAChC,EAAE,CAAC,EAAE,MAAM,CAAC;KACb,GACA,OAAO,CAAC,GAAG,CAAC;IAST,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAc/B,UAAU,IAAI,YAAY,EAAE;IAI5B,UAAU,IAAI,YAAY,EAAE;IAI5B,WAAW,IAAI,YAAY,EAAE;IAI7B,WAAW,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI;CAG9C"}
|
package/dist/src/o-node.js
CHANGED
|
@@ -110,11 +110,20 @@ export class oNode extends oToolBase {
|
|
|
110
110
|
this.logger.debug('Skipping parent registration, node is leader');
|
|
111
111
|
return;
|
|
112
112
|
}
|
|
113
|
+
if (!this.parent?.libp2pTransports?.length) {
|
|
114
|
+
this.logger.debug('Parent has no transports, waiting for reconnection & leader ack');
|
|
115
|
+
if (this.parent?.toString() === oAddress.leader().toString()) {
|
|
116
|
+
this.parent.setTransports(this.leader?.libp2pTransports || []);
|
|
117
|
+
}
|
|
118
|
+
else {
|
|
119
|
+
await this.reconnectionManager?.waitForParentAndReconnect();
|
|
120
|
+
}
|
|
121
|
+
}
|
|
113
122
|
// if no parent transports, register with the parent to get them
|
|
114
123
|
// TODO: should we remove the transports check to make this more consistent?
|
|
115
|
-
if (this.config.parent
|
|
124
|
+
if (this.config.parent) {
|
|
116
125
|
this.logger.debug('Registering node with parent...', this.config.parent);
|
|
117
|
-
|
|
126
|
+
await this.use(this.config.parent, {
|
|
118
127
|
method: 'child_register',
|
|
119
128
|
params: {
|
|
120
129
|
address: this.address.toString(),
|
|
@@ -123,9 +132,6 @@ export class oNode extends oToolBase {
|
|
|
123
132
|
_token: this.config.joinToken,
|
|
124
133
|
},
|
|
125
134
|
});
|
|
126
|
-
const { parentTransports } = parentRegistration.result.data;
|
|
127
|
-
// update the parent transports
|
|
128
|
-
this.config.parent.setTransports(parentTransports.map((t) => new oNodeTransport(t)));
|
|
129
135
|
}
|
|
130
136
|
}
|
|
131
137
|
async register() {
|
|
@@ -281,6 +287,20 @@ export class oNode extends oToolBase {
|
|
|
281
287
|
}
|
|
282
288
|
return connection;
|
|
283
289
|
}
|
|
290
|
+
async postInitialize() {
|
|
291
|
+
// Initialize connection heartbeat manager
|
|
292
|
+
this.connectionHeartbeatManager = new oConnectionHeartbeatManager(this, {
|
|
293
|
+
enabled: this.config.connectionHeartbeat?.enabled ?? true,
|
|
294
|
+
intervalMs: this.config.connectionHeartbeat?.intervalMs ?? 15000,
|
|
295
|
+
timeoutMs: this.config.connectionHeartbeat?.timeoutMs ?? 15000,
|
|
296
|
+
failureThreshold: this.config.connectionHeartbeat?.failureThreshold ?? 3,
|
|
297
|
+
checkChildren: this.config.connectionHeartbeat?.checkChildren ?? false,
|
|
298
|
+
checkParent: this.config.connectionHeartbeat?.checkParent ?? true,
|
|
299
|
+
checkLeader: true,
|
|
300
|
+
});
|
|
301
|
+
this.logger.info(`Connection heartbeat config: leader=${this.connectionHeartbeatManager.getConfig().checkLeader}, ` +
|
|
302
|
+
`parent=${this.connectionHeartbeatManager.getConfig().checkParent}`);
|
|
303
|
+
}
|
|
284
304
|
async initialize() {
|
|
285
305
|
this.logger.debug('Initializing node...');
|
|
286
306
|
if (this.p2pNode && this.state !== NodeState.STOPPED) {
|
|
@@ -300,16 +320,23 @@ export class oNode extends oToolBase {
|
|
|
300
320
|
this.connectionManager = new oNodeConnectionManager({
|
|
301
321
|
p2pNode: this.p2pNode,
|
|
302
322
|
});
|
|
303
|
-
// Initialize leader request wrapper
|
|
323
|
+
// Initialize leader request wrapper with circuit breaker
|
|
304
324
|
this.leaderRequestWrapper = new LeaderRequestWrapper({
|
|
305
325
|
enabled: this.config.leaderRetry?.enabled ?? true,
|
|
306
326
|
maxAttempts: this.config.leaderRetry?.maxAttempts ?? 20,
|
|
307
327
|
baseDelayMs: this.config.leaderRetry?.baseDelayMs ?? 2000,
|
|
308
328
|
maxDelayMs: this.config.leaderRetry?.maxDelayMs ?? 30000,
|
|
309
329
|
timeoutMs: this.config.leaderRetry?.timeoutMs ?? 120000,
|
|
330
|
+
circuitBreaker: {
|
|
331
|
+
enabled: this.config.leaderRetry?.circuitBreaker?.enabled ?? true,
|
|
332
|
+
failureThreshold: this.config.leaderRetry?.circuitBreaker?.failureThreshold ?? 3,
|
|
333
|
+
openTimeoutMs: this.config.leaderRetry?.circuitBreaker?.openTimeoutMs ?? 30000,
|
|
334
|
+
halfOpenMaxAttempts: this.config.leaderRetry?.circuitBreaker?.halfOpenMaxAttempts ?? 1,
|
|
335
|
+
},
|
|
310
336
|
});
|
|
311
337
|
this.logger.info(`Leader retry config: enabled=${this.leaderRequestWrapper.getConfig().enabled}, ` +
|
|
312
|
-
`maxAttempts=${this.leaderRequestWrapper.getConfig().maxAttempts}`
|
|
338
|
+
`maxAttempts=${this.leaderRequestWrapper.getConfig().maxAttempts}, ` +
|
|
339
|
+
`circuitBreaker.enabled=${this.leaderRequestWrapper.getConfig().circuitBreaker?.enabled}`);
|
|
313
340
|
// initialize address resolution
|
|
314
341
|
this.router.addResolver(new oMethodResolver(this.address));
|
|
315
342
|
this.router.addResolver(new oNodeResolver(this.address));
|
|
@@ -321,18 +348,6 @@ export class oNode extends oToolBase {
|
|
|
321
348
|
// Read ENABLE_LEADER_HEARTBEAT environment variable
|
|
322
349
|
const enableLeaderHeartbeat = this.parent?.toString() === oAddress.leader().toString();
|
|
323
350
|
this.logger.debug(`Enable leader heartbeat: ${enableLeaderHeartbeat}`);
|
|
324
|
-
// Initialize connection heartbeat manager
|
|
325
|
-
this.connectionHeartbeatManager = new oConnectionHeartbeatManager(this.p2pNode, this.hierarchyManager, this.notificationManager, this.address, {
|
|
326
|
-
enabled: this.config.connectionHeartbeat?.enabled ?? true,
|
|
327
|
-
intervalMs: this.config.connectionHeartbeat?.intervalMs ?? 15000,
|
|
328
|
-
timeoutMs: this.config.connectionHeartbeat?.timeoutMs ?? 5000,
|
|
329
|
-
failureThreshold: this.config.connectionHeartbeat?.failureThreshold ?? 3,
|
|
330
|
-
checkChildren: this.config.connectionHeartbeat?.checkChildren ?? true,
|
|
331
|
-
checkParent: this.config.connectionHeartbeat?.checkParent ?? true,
|
|
332
|
-
checkLeader: enableLeaderHeartbeat,
|
|
333
|
-
});
|
|
334
|
-
this.logger.info(`Connection heartbeat config: leader=${this.connectionHeartbeatManager.getConfig().checkLeader}, ` +
|
|
335
|
-
`parent=${this.connectionHeartbeatManager.getConfig().checkParent}`);
|
|
336
351
|
// Initialize reconnection manager
|
|
337
352
|
if (this.config.reconnection?.enabled !== false) {
|
|
338
353
|
this.reconnectionManager = new oReconnectionManager(this, {
|
|
@@ -364,4 +379,17 @@ export class oNode extends oToolBase {
|
|
|
364
379
|
await this.p2pNode.stop();
|
|
365
380
|
}
|
|
366
381
|
}
|
|
382
|
+
// IHeartbeatableNode interface methods
|
|
383
|
+
getLeaders() {
|
|
384
|
+
return [this.leader];
|
|
385
|
+
}
|
|
386
|
+
getParents() {
|
|
387
|
+
return this.hierarchyManager.getParents();
|
|
388
|
+
}
|
|
389
|
+
getChildren() {
|
|
390
|
+
return this.hierarchyManager.getChildren();
|
|
391
|
+
}
|
|
392
|
+
removeChild(childAddress) {
|
|
393
|
+
this.hierarchyManager.removeChild(childAddress);
|
|
394
|
+
}
|
|
367
395
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"o-node.routing-policy.d.ts","sourceRoot":"","sources":["../../../src/router/o-node.routing-policy.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACxE,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AAG1C;;;GAGG;AACH,qBAAa,kBAAmB,SAAQ,cAAc;IACpD;;;;;;;;;OASG;IACH,iBAAiB,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,GAAG,OAAO;IAkB1D;;;;;;;;OAQG;IACH,0BAA0B,CACxB,OAAO,EAAE,QAAQ,EACjB,IAAI,EAAE,KAAK,GACV,aAAa,GAAG,IAAI;
|
|
1
|
+
{"version":3,"file":"o-node.routing-policy.d.ts","sourceRoot":"","sources":["../../../src/router/o-node.routing-policy.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACxE,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AAG1C;;;GAGG;AACH,qBAAa,kBAAmB,SAAQ,cAAc;IACpD;;;;;;;;;OASG;IACH,iBAAiB,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,GAAG,OAAO;IAkB1D;;;;;;;;OAQG;IACH,0BAA0B,CACxB,OAAO,EAAE,QAAQ,EACjB,IAAI,EAAE,KAAK,GACV,aAAa,GAAG,IAAI;CAsBxB"}
|
|
@@ -21,7 +21,8 @@ export class oNodeRoutingPolicy extends oRoutingPolicy {
|
|
|
21
21
|
nodeAddress.libp2pTransports?.length > 0) {
|
|
22
22
|
// transports are provided, let's see if they match our known leaders
|
|
23
23
|
const isLeaderRef = nodeAddress.toString() === oAddress.leader().toString();
|
|
24
|
-
const isOurLeaderRef = node.
|
|
24
|
+
const isOurLeaderRef = node.address.equals(nodeAddress) ||
|
|
25
|
+
node.hierarchyManager.leaders.some((l) => l.equals(nodeAddress));
|
|
25
26
|
return isLeaderRef || isOurLeaderRef;
|
|
26
27
|
}
|
|
27
28
|
return true;
|
|
@@ -40,7 +41,7 @@ export class oNodeRoutingPolicy extends oRoutingPolicy {
|
|
|
40
41
|
const isInternal = this.isInternalAddress(address, node);
|
|
41
42
|
if (!isInternal) {
|
|
42
43
|
// external address, so we need to route
|
|
43
|
-
this.logger.debug('Address is external, routing...', nodeAddress.toString()
|
|
44
|
+
this.logger.debug('Address is external, routing...', nodeAddress.toString());
|
|
44
45
|
// route to leader of external OS
|
|
45
46
|
return {
|
|
46
47
|
nextHopAddress: new oNodeAddress(oAddress.leader().toString(), nodeAddress.libp2pTransports),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"o-node.search-resolver.d.ts","sourceRoot":"","sources":["../../../../src/router/resolvers/o-node.search-resolver.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,gBAAgB,EAChB,KAAK,EAEL,UAAU,EACV,cAAc,EAEd,aAAa,EAEd,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAExD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,qBAAa,eAAgB,SAAQ,gBAAgB;IACvC,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ;gBAAjB,OAAO,EAAE,QAAQ;IAIhD,IAAI,gBAAgB,IAAI,UAAU,EAAE,CAEnC;IAED;;;;OAIG;IACH,SAAS,CAAC,kBAAkB,IAAI,QAAQ;IAIxC;;;;OAIG;IACH,SAAS,CAAC,eAAe,IAAI,MAAM;IAInC;;;;;OAKG;IACH,SAAS,CAAC,iBAAiB,CAAC,OAAO,EAAE,QAAQ,GAAG,GAAG;IAOnD;;;;;;OAMG;IACH,SAAS,CAAC,mBAAmB,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,KAAK,GAAG,GAAG,EAAE;IASjE;;;;;OAKG;IACH,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI;IAIlD;;;;;OAKG;IACH,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,GAAG,cAAc,EAAE;IAOtD;;;;;;;;;;;OAWG;IACH,SAAS,CAAC,wBAAwB,CAChC,OAAO,EAAE,QAAQ,EACjB,gBAAgB,EAAE,cAAc,EAAE,EAClC,IAAI,EAAE,KAAK,GACV,cAAc,EAAE;IAgBnB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAyCG;IACH,SAAS,CAAC,gBAAgB,CACxB,IAAI,EAAE,KAAK,EACX,qBAAqB,EAAE,QAAQ,EAC/B,YAAY,EAAE,GAAG,GAChB,QAAQ;
|
|
1
|
+
{"version":3,"file":"o-node.search-resolver.d.ts","sourceRoot":"","sources":["../../../../src/router/resolvers/o-node.search-resolver.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,gBAAgB,EAChB,KAAK,EAEL,UAAU,EACV,cAAc,EAEd,aAAa,EAEd,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAExD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,qBAAa,eAAgB,SAAQ,gBAAgB;IACvC,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ;gBAAjB,OAAO,EAAE,QAAQ;IAIhD,IAAI,gBAAgB,IAAI,UAAU,EAAE,CAEnC;IAED;;;;OAIG;IACH,SAAS,CAAC,kBAAkB,IAAI,QAAQ;IAIxC;;;;OAIG;IACH,SAAS,CAAC,eAAe,IAAI,MAAM;IAInC;;;;;OAKG;IACH,SAAS,CAAC,iBAAiB,CAAC,OAAO,EAAE,QAAQ,GAAG,GAAG;IAOnD;;;;;;OAMG;IACH,SAAS,CAAC,mBAAmB,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,KAAK,GAAG,GAAG,EAAE;IASjE;;;;;OAKG;IACH,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI;IAIlD;;;;;OAKG;IACH,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,GAAG,cAAc,EAAE;IAOtD;;;;;;;;;;;OAWG;IACH,SAAS,CAAC,wBAAwB,CAChC,OAAO,EAAE,QAAQ,EACjB,gBAAgB,EAAE,cAAc,EAAE,EAClC,IAAI,EAAE,KAAK,GACV,cAAc,EAAE;IAgBnB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAyCG;IACH,SAAS,CAAC,gBAAgB,CACxB,IAAI,EAAE,KAAK,EACX,qBAAqB,EAAE,QAAQ,EAC/B,YAAY,EAAE,GAAG,GAChB,QAAQ;IAeL,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;CA8F/D"}
|
|
@@ -201,7 +201,6 @@ export class oSearchResolver extends oAddressResolver {
|
|
|
201
201
|
determineNextHop(node, resolvedTargetAddress, searchResult) {
|
|
202
202
|
// Determine next hop using standard hierarchy logic
|
|
203
203
|
const nextHopAddress = oAddress.next(node.address, resolvedTargetAddress);
|
|
204
|
-
this.logger.debug('determineNextHop with params', 'node.address: ' + node.address.toString(), 'resolvedTargetAddress: ' + resolvedTargetAddress.toString(), 'searchResult.address: ' + searchResult.address, 'next hop: ' + nextHopAddress.toString());
|
|
205
204
|
// Map transports from search result
|
|
206
205
|
const targetTransports = this.mapTransports(searchResult);
|
|
207
206
|
// Set transports on the next hop based on routing logic
|
|
@@ -218,13 +217,33 @@ export class oSearchResolver extends oAddressResolver {
|
|
|
218
217
|
requestOverride: resolveRequest,
|
|
219
218
|
};
|
|
220
219
|
}
|
|
221
|
-
// Perform registry search
|
|
220
|
+
// Perform registry search with error handling
|
|
222
221
|
const searchParams = this.buildSearchParams(address);
|
|
223
222
|
const registryAddress = this.getRegistryAddress();
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
223
|
+
let searchResponse;
|
|
224
|
+
try {
|
|
225
|
+
searchResponse = await node.use(registryAddress, {
|
|
226
|
+
method: this.getSearchMethod(),
|
|
227
|
+
params: searchParams,
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
catch (error) {
|
|
231
|
+
// Log the error but don't throw - allow fallback resolvers to handle it
|
|
232
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
233
|
+
// Check if this is a circuit breaker error (fast-fail scenario)
|
|
234
|
+
if (errorMessage.includes('Circuit breaker is OPEN')) {
|
|
235
|
+
this.logger.warn(`Registry search blocked by circuit breaker for ${address.toString()}: ${errorMessage}`);
|
|
236
|
+
}
|
|
237
|
+
else {
|
|
238
|
+
this.logger.error(`Registry search failed for ${address.toString()}: ${errorMessage}`);
|
|
239
|
+
}
|
|
240
|
+
// Return original address without transports, letting next resolver in chain handle it
|
|
241
|
+
return {
|
|
242
|
+
nextHopAddress: address,
|
|
243
|
+
targetAddress: targetAddress,
|
|
244
|
+
requestOverride: resolveRequest,
|
|
245
|
+
};
|
|
246
|
+
}
|
|
228
247
|
// Filter and select result
|
|
229
248
|
const filteredResults = this.filterSearchResults(searchResponse.result.data, node);
|
|
230
249
|
const selectedResult = this.selectResult(filteredResults);
|
|
@@ -240,7 +259,6 @@ export class oSearchResolver extends oAddressResolver {
|
|
|
240
259
|
const extraParams = address
|
|
241
260
|
.toString() // o://embeddings-text replace o://embeddings-text = ''
|
|
242
261
|
.replace(address.toRootAddress().toString(), '');
|
|
243
|
-
this.logger.debug('Extra params:', extraParams);
|
|
244
262
|
// Check if selectedResult.address already contains the complete path
|
|
245
263
|
// This happens when registry finds via staticAddress - the returned address
|
|
246
264
|
// is the canonical hierarchical location, so we shouldn't append extraParams
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import { oObject } from '@olane/o-core';
|
|
2
|
+
export declare enum CircuitState {
|
|
3
|
+
CLOSED = "CLOSED",// Normal operation, requests pass through
|
|
4
|
+
OPEN = "OPEN",// Circuit broken, requests fast-fail
|
|
5
|
+
HALF_OPEN = "HALF_OPEN"
|
|
6
|
+
}
|
|
7
|
+
export interface CircuitBreakerConfig {
|
|
8
|
+
failureThreshold: number;
|
|
9
|
+
openTimeoutMs: number;
|
|
10
|
+
halfOpenMaxAttempts: number;
|
|
11
|
+
enabled: boolean;
|
|
12
|
+
}
|
|
13
|
+
export interface CircuitStats {
|
|
14
|
+
state: CircuitState;
|
|
15
|
+
consecutiveFailures: number;
|
|
16
|
+
totalFailures: number;
|
|
17
|
+
totalSuccesses: number;
|
|
18
|
+
lastFailureTime?: number;
|
|
19
|
+
lastSuccessTime?: number;
|
|
20
|
+
openedAt?: number;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Circuit Breaker Pattern Implementation
|
|
24
|
+
*
|
|
25
|
+
* Prevents cascading failures by "breaking the circuit" when a service
|
|
26
|
+
* experiences persistent failures. This allows the system to fail fast
|
|
27
|
+
* rather than wasting resources on retries that are likely to fail.
|
|
28
|
+
*
|
|
29
|
+
* States:
|
|
30
|
+
* - CLOSED: Normal operation, all requests pass through
|
|
31
|
+
* - OPEN: Circuit broken due to failures, requests fail immediately
|
|
32
|
+
* - HALF_OPEN: Testing recovery, limited requests allowed
|
|
33
|
+
*
|
|
34
|
+
* Flow:
|
|
35
|
+
* 1. CLOSED -> OPEN: After N consecutive failures
|
|
36
|
+
* 2. OPEN -> HALF_OPEN: After timeout period
|
|
37
|
+
* 3. HALF_OPEN -> CLOSED: After successful request
|
|
38
|
+
* 4. HALF_OPEN -> OPEN: After failure in recovery
|
|
39
|
+
*
|
|
40
|
+
* @example
|
|
41
|
+
* ```typescript
|
|
42
|
+
* const breaker = new CircuitBreaker('registry', {
|
|
43
|
+
* failureThreshold: 3,
|
|
44
|
+
* openTimeoutMs: 30000,
|
|
45
|
+
* halfOpenMaxAttempts: 1,
|
|
46
|
+
* enabled: true,
|
|
47
|
+
* });
|
|
48
|
+
*
|
|
49
|
+
* // Before making request
|
|
50
|
+
* if (!breaker.shouldAllowRequest()) {
|
|
51
|
+
* throw new Error('Circuit breaker is open');
|
|
52
|
+
* }
|
|
53
|
+
*
|
|
54
|
+
* try {
|
|
55
|
+
* const result = await makeRequest();
|
|
56
|
+
* breaker.recordSuccess();
|
|
57
|
+
* return result;
|
|
58
|
+
* } catch (error) {
|
|
59
|
+
* breaker.recordFailure();
|
|
60
|
+
* throw error;
|
|
61
|
+
* }
|
|
62
|
+
* ```
|
|
63
|
+
*/
|
|
64
|
+
export declare class CircuitBreaker extends oObject {
|
|
65
|
+
private readonly serviceName;
|
|
66
|
+
private readonly config;
|
|
67
|
+
private state;
|
|
68
|
+
private consecutiveFailures;
|
|
69
|
+
private totalFailures;
|
|
70
|
+
private totalSuccesses;
|
|
71
|
+
private lastFailureTime?;
|
|
72
|
+
private lastSuccessTime?;
|
|
73
|
+
private openedAt?;
|
|
74
|
+
private halfOpenAttempts;
|
|
75
|
+
constructor(serviceName: string, config: CircuitBreakerConfig);
|
|
76
|
+
/**
|
|
77
|
+
* Check if a request should be allowed through the circuit breaker
|
|
78
|
+
* @returns true if request should proceed, false if should fast-fail
|
|
79
|
+
*/
|
|
80
|
+
shouldAllowRequest(): boolean;
|
|
81
|
+
/**
|
|
82
|
+
* Record a successful request
|
|
83
|
+
*/
|
|
84
|
+
recordSuccess(): void;
|
|
85
|
+
/**
|
|
86
|
+
* Record a failed request
|
|
87
|
+
*/
|
|
88
|
+
recordFailure(): void;
|
|
89
|
+
/**
|
|
90
|
+
* Get current statistics
|
|
91
|
+
*/
|
|
92
|
+
getStats(): CircuitStats;
|
|
93
|
+
/**
|
|
94
|
+
* Get current circuit state
|
|
95
|
+
*/
|
|
96
|
+
getState(): CircuitState;
|
|
97
|
+
/**
|
|
98
|
+
* Force reset the circuit breaker to CLOSED state
|
|
99
|
+
* Use with caution - mainly for testing or manual recovery
|
|
100
|
+
*/
|
|
101
|
+
reset(): void;
|
|
102
|
+
/**
|
|
103
|
+
* Transition to a new state
|
|
104
|
+
*/
|
|
105
|
+
private transitionTo;
|
|
106
|
+
}
|
|
107
|
+
//# sourceMappingURL=circuit-breaker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"circuit-breaker.d.ts","sourceRoot":"","sources":["../../../src/utils/circuit-breaker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC,oBAAY,YAAY;IACtB,MAAM,WAAW,CAAE,0CAA0C;IAC7D,IAAI,SAAS,CAAE,qCAAqC;IACpD,SAAS,cAAc;CACxB;AAED,MAAM,WAAW,oBAAoB;IACnC,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,YAAY,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,qBAAa,cAAe,SAAQ,OAAO;IAWvC,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,MAAM;IAXzB,OAAO,CAAC,KAAK,CAAqC;IAClD,OAAO,CAAC,mBAAmB,CAAa;IACxC,OAAO,CAAC,aAAa,CAAa;IAClC,OAAO,CAAC,cAAc,CAAa;IACnC,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,CAAS;IAC1B,OAAO,CAAC,gBAAgB,CAAa;gBAGlB,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,oBAAoB;IAW/C;;;OAGG;IACH,kBAAkB,IAAI,OAAO;IA2C7B;;OAEG;IACH,aAAa,IAAI,IAAI;IAkBrB;;OAEG;IACH,aAAa,IAAI,IAAI;IA+BrB;;OAEG;IACH,QAAQ,IAAI,YAAY;IAYxB;;OAEG;IACH,QAAQ,IAAI,YAAY;IAIxB;;;OAGG;IACH,KAAK,IAAI,IAAI;IAQb;;OAEG;IACH,OAAO,CAAC,YAAY;CAOrB"}
|