@olane/o-node 0.7.12-alpha.11 → 0.7.12-alpha.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/interfaces/o-node.config.d.ts +6 -0
- package/dist/src/interfaces/o-node.config.d.ts.map +1 -1
- package/dist/src/o-node.d.ts.map +1 -1
- package/dist/src/o-node.js +9 -2
- package/dist/src/router/o-node.routing-policy.d.ts.map +1 -1
- package/dist/src/router/o-node.routing-policy.js +1 -1
- package/dist/src/router/resolvers/o-node.search-resolver.d.ts.map +1 -1
- package/dist/src/router/resolvers/o-node.search-resolver.js +25 -7
- package/dist/src/utils/circuit-breaker.d.ts +107 -0
- package/dist/src/utils/circuit-breaker.d.ts.map +1 -0
- package/dist/src/utils/circuit-breaker.js +175 -0
- package/dist/src/utils/circuit-breaker.test.d.ts +2 -0
- package/dist/src/utils/circuit-breaker.test.d.ts.map +1 -0
- package/dist/src/utils/circuit-breaker.test.js +262 -0
- package/dist/src/utils/leader-request-wrapper.d.ts +26 -5
- package/dist/src/utils/leader-request-wrapper.d.ts.map +1 -1
- package/dist/src/utils/leader-request-wrapper.js +79 -8
- package/dist/src/utils/leader-request-wrapper.test.d.ts +1 -0
- package/dist/src/utils/leader-request-wrapper.test.d.ts.map +1 -0
- package/dist/src/utils/leader-request-wrapper.test.js +246 -0
- package/package.json +6 -6
|
@@ -39,6 +39,12 @@ export interface oNodeConfig extends oCoreConfig {
|
|
|
39
39
|
baseDelayMs?: number;
|
|
40
40
|
maxDelayMs?: number;
|
|
41
41
|
timeoutMs?: number;
|
|
42
|
+
circuitBreaker?: {
|
|
43
|
+
enabled?: boolean;
|
|
44
|
+
failureThreshold?: number;
|
|
45
|
+
openTimeoutMs?: number;
|
|
46
|
+
halfOpenMaxAttempts?: number;
|
|
47
|
+
};
|
|
42
48
|
};
|
|
43
49
|
}
|
|
44
50
|
//# sourceMappingURL=o-node.config.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"o-node.config.d.ts","sourceRoot":"","sources":["../../../src/interfaces/o-node.config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAE3D,MAAM,WAAW,WAAY,SAAQ,WAAW;IAC9C,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IAE5B;;;OAGG;IACH,mBAAmB,CAAC,EAAE;QACpB,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,aAAa,CAAC,EAAE,OAAO,CAAC;QACxB,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB,CAAC;IAEF;;;OAGG;IACH,YAAY,CAAC,EAAE;QACb,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,yBAAyB,CAAC,EAAE,MAAM,CAAC;QACnC,yBAAyB,CAAC,EAAE,MAAM,CAAC;KACpC,CAAC;IAEF;;;OAGG;IACH,WAAW,CAAC,EAAE;QACZ,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,SAAS,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"o-node.config.d.ts","sourceRoot":"","sources":["../../../src/interfaces/o-node.config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAE3D,MAAM,WAAW,WAAY,SAAQ,WAAW;IAC9C,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IAE5B;;;OAGG;IACH,mBAAmB,CAAC,EAAE;QACpB,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,aAAa,CAAC,EAAE,OAAO,CAAC;QACxB,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB,CAAC;IAEF;;;OAGG;IACH,YAAY,CAAC,EAAE;QACb,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,yBAAyB,CAAC,EAAE,MAAM,CAAC;QACnC,yBAAyB,CAAC,EAAE,MAAM,CAAC;KACpC,CAAC;IAEF;;;OAGG;IACH,WAAW,CAAC,EAAE;QACZ,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,cAAc,CAAC,EAAE;YACf,OAAO,CAAC,EAAE,OAAO,CAAC;YAClB,gBAAgB,CAAC,EAAE,MAAM,CAAC;YAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;YACvB,mBAAmB,CAAC,EAAE,MAAM,CAAC;SAC9B,CAAC;KACH,CAAC;CACH"}
|
package/dist/src/o-node.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"o-node.d.ts","sourceRoot":"","sources":["../../src/o-node.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,MAAM,EACN,YAAY,EACb,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AACtE,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAIL,QAAQ,EACR,QAAQ,EAER,oBAAoB,EACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,mCAAmC,CAAC;AACpE,OAAO,EAAE,sBAAsB,EAAE,MAAM,2CAA2C,CAAC;AAGnF,OAAO,EAAmB,SAAS,EAAE,MAAM,eAAe,CAAC;AAI3D,OAAO,EAAE,2BAA2B,EAAE,MAAM,8CAA8C,CAAC;AAC3F,OAAO,EAAE,oBAAoB,EAAE,MAAM,sCAAsC,CAAC;AAC5E,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AAEzE,qBAAa,KAAM,SAAQ,SAAS;IAC3B,MAAM,EAAG,MAAM,CAAC;IAChB,OAAO,EAAG,MAAM,CAAC;IACjB,OAAO,EAAG,YAAY,CAAC;IACvB,MAAM,EAAE,WAAW,CAAC;IACpB,iBAAiB,EAAG,sBAAsB,CAAC;IAC3C,gBAAgB,EAAG,qBAAqB,CAAC;IACzC,0BAA0B,CAAC,EAAE,2BAA2B,CAAC;IACzD,mBAAmB,CAAC,EAAE,oBAAoB,CAAC;IAC3C,oBAAoB,EAAG,oBAAoB,CAAC;IACnD,SAAS,CAAC,WAAW,EAAE,OAAO,CAAS;gBAE3B,MAAM,EAAE,WAAW;IAK/B,IAAI,MAAM,IAAI,YAAY,GAAG,IAAI,CAEhC;IAED,IAAI,aAAa,IAAI,YAAY,CAKhC;IAED,IAAI,YAAY,IAAI,MAAM,GAAG,IAAI,CAOhC;IAED,mBAAmB,IAAI,GAAG,EAAE;IAItB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC;IASvC,SAAS,CAAC,yBAAyB,IAAI,oBAAoB;IAQ3D,IAAI,aAAa,IAAI,YAAY,CAEhC;IAED,IAAI,gBAAgB,IAAI,cAAc,EAAE,CAEvC;IAED,IAAI,UAAU,IAAI,cAAc,EAAE,CAIjC;IAEK,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAsD3B,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IA2B/B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAwC/B,aAAa,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IAItC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAetB,mBAAmB,CAAC,OAAO,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC;IAG1D;;;OAGG;IACG,SAAS,IAAI,OAAO,CAAC,YAAY,CAAC;cA0FxB,UAAU,IAAI,OAAO,CAAC,MAAM,CAAC;IAMvC,OAAO,CACX,cAAc,EAAE,YAAY,EAC5B,aAAa,EAAE,YAAY,GAC1B,OAAO,CAAC,eAAe,CAAC;IA0BrB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"o-node.d.ts","sourceRoot":"","sources":["../../src/o-node.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,MAAM,EACN,YAAY,EACb,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AACtE,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAIL,QAAQ,EACR,QAAQ,EAER,oBAAoB,EACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,mCAAmC,CAAC;AACpE,OAAO,EAAE,sBAAsB,EAAE,MAAM,2CAA2C,CAAC;AAGnF,OAAO,EAAmB,SAAS,EAAE,MAAM,eAAe,CAAC;AAI3D,OAAO,EAAE,2BAA2B,EAAE,MAAM,8CAA8C,CAAC;AAC3F,OAAO,EAAE,oBAAoB,EAAE,MAAM,sCAAsC,CAAC;AAC5E,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AAEzE,qBAAa,KAAM,SAAQ,SAAS;IAC3B,MAAM,EAAG,MAAM,CAAC;IAChB,OAAO,EAAG,MAAM,CAAC;IACjB,OAAO,EAAG,YAAY,CAAC;IACvB,MAAM,EAAE,WAAW,CAAC;IACpB,iBAAiB,EAAG,sBAAsB,CAAC;IAC3C,gBAAgB,EAAG,qBAAqB,CAAC;IACzC,0BAA0B,CAAC,EAAE,2BAA2B,CAAC;IACzD,mBAAmB,CAAC,EAAE,oBAAoB,CAAC;IAC3C,oBAAoB,EAAG,oBAAoB,CAAC;IACnD,SAAS,CAAC,WAAW,EAAE,OAAO,CAAS;gBAE3B,MAAM,EAAE,WAAW;IAK/B,IAAI,MAAM,IAAI,YAAY,GAAG,IAAI,CAEhC;IAED,IAAI,aAAa,IAAI,YAAY,CAKhC;IAED,IAAI,YAAY,IAAI,MAAM,GAAG,IAAI,CAOhC;IAED,mBAAmB,IAAI,GAAG,EAAE;IAItB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC;IASvC,SAAS,CAAC,yBAAyB,IAAI,oBAAoB;IAQ3D,IAAI,aAAa,IAAI,YAAY,CAEhC;IAED,IAAI,gBAAgB,IAAI,cAAc,EAAE,CAEvC;IAED,IAAI,UAAU,IAAI,cAAc,EAAE,CAIjC;IAEK,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAsD3B,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IA2B/B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAwC/B,aAAa,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IAItC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAetB,mBAAmB,CAAC,OAAO,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC;IAG1D;;;OAGG;IACG,SAAS,IAAI,OAAO,CAAC,YAAY,CAAC;cA0FxB,UAAU,IAAI,OAAO,CAAC,MAAM,CAAC;IAMvC,OAAO,CACX,cAAc,EAAE,YAAY,EAC5B,aAAa,EAAE,YAAY,GAC1B,OAAO,CAAC,eAAe,CAAC;IA0BrB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAyGjC;;OAEG;IACG,GAAG,CACP,OAAO,EAAE,QAAQ,EACjB,IAAI,CAAC,EAAE;QACL,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,MAAM,CAAC,EAAE;YAAE,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;SAAE,CAAC;QAChC,EAAE,CAAC,EAAE,MAAM,CAAC;KACb,GACA,OAAO,CAAC,GAAG,CAAC;IAST,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAYhC"}
|
package/dist/src/o-node.js
CHANGED
|
@@ -300,16 +300,23 @@ export class oNode extends oToolBase {
|
|
|
300
300
|
this.connectionManager = new oNodeConnectionManager({
|
|
301
301
|
p2pNode: this.p2pNode,
|
|
302
302
|
});
|
|
303
|
-
// Initialize leader request wrapper
|
|
303
|
+
// Initialize leader request wrapper with circuit breaker
|
|
304
304
|
this.leaderRequestWrapper = new LeaderRequestWrapper({
|
|
305
305
|
enabled: this.config.leaderRetry?.enabled ?? true,
|
|
306
306
|
maxAttempts: this.config.leaderRetry?.maxAttempts ?? 20,
|
|
307
307
|
baseDelayMs: this.config.leaderRetry?.baseDelayMs ?? 2000,
|
|
308
308
|
maxDelayMs: this.config.leaderRetry?.maxDelayMs ?? 30000,
|
|
309
309
|
timeoutMs: this.config.leaderRetry?.timeoutMs ?? 120000,
|
|
310
|
+
circuitBreaker: {
|
|
311
|
+
enabled: this.config.leaderRetry?.circuitBreaker?.enabled ?? true,
|
|
312
|
+
failureThreshold: this.config.leaderRetry?.circuitBreaker?.failureThreshold ?? 3,
|
|
313
|
+
openTimeoutMs: this.config.leaderRetry?.circuitBreaker?.openTimeoutMs ?? 30000,
|
|
314
|
+
halfOpenMaxAttempts: this.config.leaderRetry?.circuitBreaker?.halfOpenMaxAttempts ?? 1,
|
|
315
|
+
},
|
|
310
316
|
});
|
|
311
317
|
this.logger.info(`Leader retry config: enabled=${this.leaderRequestWrapper.getConfig().enabled}, ` +
|
|
312
|
-
`maxAttempts=${this.leaderRequestWrapper.getConfig().maxAttempts}`
|
|
318
|
+
`maxAttempts=${this.leaderRequestWrapper.getConfig().maxAttempts}, ` +
|
|
319
|
+
`circuitBreaker.enabled=${this.leaderRequestWrapper.getConfig().circuitBreaker?.enabled}`);
|
|
313
320
|
// initialize address resolution
|
|
314
321
|
this.router.addResolver(new oMethodResolver(this.address));
|
|
315
322
|
this.router.addResolver(new oNodeResolver(this.address));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"o-node.routing-policy.d.ts","sourceRoot":"","sources":["../../../src/router/o-node.routing-policy.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACxE,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AAG1C;;;GAGG;AACH,qBAAa,kBAAmB,SAAQ,cAAc;IACpD;;;;;;;;;OASG;IACH,iBAAiB,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,GAAG,OAAO;IAkB1D;;;;;;;;OAQG;IACH,0BAA0B,CACxB,OAAO,EAAE,QAAQ,EACjB,IAAI,EAAE,KAAK,GACV,aAAa,GAAG,IAAI;
|
|
1
|
+
{"version":3,"file":"o-node.routing-policy.d.ts","sourceRoot":"","sources":["../../../src/router/o-node.routing-policy.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACxE,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AAG1C;;;GAGG;AACH,qBAAa,kBAAmB,SAAQ,cAAc;IACpD;;;;;;;;;OASG;IACH,iBAAiB,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,GAAG,OAAO;IAkB1D;;;;;;;;OAQG;IACH,0BAA0B,CACxB,OAAO,EAAE,QAAQ,EACjB,IAAI,EAAE,KAAK,GACV,aAAa,GAAG,IAAI;CAsBxB"}
|
|
@@ -40,7 +40,7 @@ export class oNodeRoutingPolicy extends oRoutingPolicy {
|
|
|
40
40
|
const isInternal = this.isInternalAddress(address, node);
|
|
41
41
|
if (!isInternal) {
|
|
42
42
|
// external address, so we need to route
|
|
43
|
-
this.logger.debug('Address is external, routing...', nodeAddress.toString()
|
|
43
|
+
this.logger.debug('Address is external, routing...', nodeAddress.toString());
|
|
44
44
|
// route to leader of external OS
|
|
45
45
|
return {
|
|
46
46
|
nextHopAddress: new oNodeAddress(oAddress.leader().toString(), nodeAddress.libp2pTransports),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"o-node.search-resolver.d.ts","sourceRoot":"","sources":["../../../../src/router/resolvers/o-node.search-resolver.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,gBAAgB,EAChB,KAAK,EAEL,UAAU,EACV,cAAc,EAEd,aAAa,EAEd,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAExD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,qBAAa,eAAgB,SAAQ,gBAAgB;IACvC,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ;gBAAjB,OAAO,EAAE,QAAQ;IAIhD,IAAI,gBAAgB,IAAI,UAAU,EAAE,CAEnC;IAED;;;;OAIG;IACH,SAAS,CAAC,kBAAkB,IAAI,QAAQ;IAIxC;;;;OAIG;IACH,SAAS,CAAC,eAAe,IAAI,MAAM;IAInC;;;;;OAKG;IACH,SAAS,CAAC,iBAAiB,CAAC,OAAO,EAAE,QAAQ,GAAG,GAAG;IAOnD;;;;;;OAMG;IACH,SAAS,CAAC,mBAAmB,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,KAAK,GAAG,GAAG,EAAE;IASjE;;;;;OAKG;IACH,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI;IAIlD;;;;;OAKG;IACH,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,GAAG,cAAc,EAAE;IAOtD;;;;;;;;;;;OAWG;IACH,SAAS,CAAC,wBAAwB,CAChC,OAAO,EAAE,QAAQ,EACjB,gBAAgB,EAAE,cAAc,EAAE,EAClC,IAAI,EAAE,KAAK,GACV,cAAc,EAAE;IAgBnB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAyCG;IACH,SAAS,CAAC,gBAAgB,CACxB,IAAI,EAAE,KAAK,EACX,qBAAqB,EAAE,QAAQ,EAC/B,YAAY,EAAE,GAAG,GAChB,QAAQ;
|
|
1
|
+
{"version":3,"file":"o-node.search-resolver.d.ts","sourceRoot":"","sources":["../../../../src/router/resolvers/o-node.search-resolver.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,gBAAgB,EAChB,KAAK,EAEL,UAAU,EACV,cAAc,EAEd,aAAa,EAEd,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAExD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,qBAAa,eAAgB,SAAQ,gBAAgB;IACvC,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ;gBAAjB,OAAO,EAAE,QAAQ;IAIhD,IAAI,gBAAgB,IAAI,UAAU,EAAE,CAEnC;IAED;;;;OAIG;IACH,SAAS,CAAC,kBAAkB,IAAI,QAAQ;IAIxC;;;;OAIG;IACH,SAAS,CAAC,eAAe,IAAI,MAAM;IAInC;;;;;OAKG;IACH,SAAS,CAAC,iBAAiB,CAAC,OAAO,EAAE,QAAQ,GAAG,GAAG;IAOnD;;;;;;OAMG;IACH,SAAS,CAAC,mBAAmB,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,KAAK,GAAG,GAAG,EAAE;IASjE;;;;;OAKG;IACH,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI;IAIlD;;;;;OAKG;IACH,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,GAAG,cAAc,EAAE;IAOtD;;;;;;;;;;;OAWG;IACH,SAAS,CAAC,wBAAwB,CAChC,OAAO,EAAE,QAAQ,EACjB,gBAAgB,EAAE,cAAc,EAAE,EAClC,IAAI,EAAE,KAAK,GACV,cAAc,EAAE;IAgBnB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAyCG;IACH,SAAS,CAAC,gBAAgB,CACxB,IAAI,EAAE,KAAK,EACX,qBAAqB,EAAE,QAAQ,EAC/B,YAAY,EAAE,GAAG,GAChB,QAAQ;IAeL,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;CA8F/D"}
|
|
@@ -201,7 +201,6 @@ export class oSearchResolver extends oAddressResolver {
|
|
|
201
201
|
determineNextHop(node, resolvedTargetAddress, searchResult) {
|
|
202
202
|
// Determine next hop using standard hierarchy logic
|
|
203
203
|
const nextHopAddress = oAddress.next(node.address, resolvedTargetAddress);
|
|
204
|
-
this.logger.debug('determineNextHop with params', 'node.address: ' + node.address.toString(), 'resolvedTargetAddress: ' + resolvedTargetAddress.toString(), 'searchResult.address: ' + searchResult.address, 'next hop: ' + nextHopAddress.toString());
|
|
205
204
|
// Map transports from search result
|
|
206
205
|
const targetTransports = this.mapTransports(searchResult);
|
|
207
206
|
// Set transports on the next hop based on routing logic
|
|
@@ -218,13 +217,33 @@ export class oSearchResolver extends oAddressResolver {
|
|
|
218
217
|
requestOverride: resolveRequest,
|
|
219
218
|
};
|
|
220
219
|
}
|
|
221
|
-
// Perform registry search
|
|
220
|
+
// Perform registry search with error handling
|
|
222
221
|
const searchParams = this.buildSearchParams(address);
|
|
223
222
|
const registryAddress = this.getRegistryAddress();
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
223
|
+
let searchResponse;
|
|
224
|
+
try {
|
|
225
|
+
searchResponse = await node.use(registryAddress, {
|
|
226
|
+
method: this.getSearchMethod(),
|
|
227
|
+
params: searchParams,
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
catch (error) {
|
|
231
|
+
// Log the error but don't throw - allow fallback resolvers to handle it
|
|
232
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
233
|
+
// Check if this is a circuit breaker error (fast-fail scenario)
|
|
234
|
+
if (errorMessage.includes('Circuit breaker is OPEN')) {
|
|
235
|
+
this.logger.warn(`Registry search blocked by circuit breaker for ${address.toString()}: ${errorMessage}`);
|
|
236
|
+
}
|
|
237
|
+
else {
|
|
238
|
+
this.logger.error(`Registry search failed for ${address.toString()}: ${errorMessage}`);
|
|
239
|
+
}
|
|
240
|
+
// Return original address without transports, letting next resolver in chain handle it
|
|
241
|
+
return {
|
|
242
|
+
nextHopAddress: address,
|
|
243
|
+
targetAddress: targetAddress,
|
|
244
|
+
requestOverride: resolveRequest,
|
|
245
|
+
};
|
|
246
|
+
}
|
|
228
247
|
// Filter and select result
|
|
229
248
|
const filteredResults = this.filterSearchResults(searchResponse.result.data, node);
|
|
230
249
|
const selectedResult = this.selectResult(filteredResults);
|
|
@@ -240,7 +259,6 @@ export class oSearchResolver extends oAddressResolver {
|
|
|
240
259
|
const extraParams = address
|
|
241
260
|
.toString() // o://embeddings-text replace o://embeddings-text = ''
|
|
242
261
|
.replace(address.toRootAddress().toString(), '');
|
|
243
|
-
this.logger.debug('Extra params:', extraParams);
|
|
244
262
|
// Check if selectedResult.address already contains the complete path
|
|
245
263
|
// This happens when registry finds via staticAddress - the returned address
|
|
246
264
|
// is the canonical hierarchical location, so we shouldn't append extraParams
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import { oObject } from '@olane/o-core';
|
|
2
|
+
export declare enum CircuitState {
|
|
3
|
+
CLOSED = "CLOSED",// Normal operation, requests pass through
|
|
4
|
+
OPEN = "OPEN",// Circuit broken, requests fast-fail
|
|
5
|
+
HALF_OPEN = "HALF_OPEN"
|
|
6
|
+
}
|
|
7
|
+
export interface CircuitBreakerConfig {
|
|
8
|
+
failureThreshold: number;
|
|
9
|
+
openTimeoutMs: number;
|
|
10
|
+
halfOpenMaxAttempts: number;
|
|
11
|
+
enabled: boolean;
|
|
12
|
+
}
|
|
13
|
+
export interface CircuitStats {
|
|
14
|
+
state: CircuitState;
|
|
15
|
+
consecutiveFailures: number;
|
|
16
|
+
totalFailures: number;
|
|
17
|
+
totalSuccesses: number;
|
|
18
|
+
lastFailureTime?: number;
|
|
19
|
+
lastSuccessTime?: number;
|
|
20
|
+
openedAt?: number;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Circuit Breaker Pattern Implementation
|
|
24
|
+
*
|
|
25
|
+
* Prevents cascading failures by "breaking the circuit" when a service
|
|
26
|
+
* experiences persistent failures. This allows the system to fail fast
|
|
27
|
+
* rather than wasting resources on retries that are likely to fail.
|
|
28
|
+
*
|
|
29
|
+
* States:
|
|
30
|
+
* - CLOSED: Normal operation, all requests pass through
|
|
31
|
+
* - OPEN: Circuit broken due to failures, requests fail immediately
|
|
32
|
+
* - HALF_OPEN: Testing recovery, limited requests allowed
|
|
33
|
+
*
|
|
34
|
+
* Flow:
|
|
35
|
+
* 1. CLOSED -> OPEN: After N consecutive failures
|
|
36
|
+
* 2. OPEN -> HALF_OPEN: After timeout period
|
|
37
|
+
* 3. HALF_OPEN -> CLOSED: After successful request
|
|
38
|
+
* 4. HALF_OPEN -> OPEN: After failure in recovery
|
|
39
|
+
*
|
|
40
|
+
* @example
|
|
41
|
+
* ```typescript
|
|
42
|
+
* const breaker = new CircuitBreaker('registry', {
|
|
43
|
+
* failureThreshold: 3,
|
|
44
|
+
* openTimeoutMs: 30000,
|
|
45
|
+
* halfOpenMaxAttempts: 1,
|
|
46
|
+
* enabled: true,
|
|
47
|
+
* });
|
|
48
|
+
*
|
|
49
|
+
* // Before making request
|
|
50
|
+
* if (!breaker.shouldAllowRequest()) {
|
|
51
|
+
* throw new Error('Circuit breaker is open');
|
|
52
|
+
* }
|
|
53
|
+
*
|
|
54
|
+
* try {
|
|
55
|
+
* const result = await makeRequest();
|
|
56
|
+
* breaker.recordSuccess();
|
|
57
|
+
* return result;
|
|
58
|
+
* } catch (error) {
|
|
59
|
+
* breaker.recordFailure();
|
|
60
|
+
* throw error;
|
|
61
|
+
* }
|
|
62
|
+
* ```
|
|
63
|
+
*/
|
|
64
|
+
export declare class CircuitBreaker extends oObject {
|
|
65
|
+
private readonly serviceName;
|
|
66
|
+
private readonly config;
|
|
67
|
+
private state;
|
|
68
|
+
private consecutiveFailures;
|
|
69
|
+
private totalFailures;
|
|
70
|
+
private totalSuccesses;
|
|
71
|
+
private lastFailureTime?;
|
|
72
|
+
private lastSuccessTime?;
|
|
73
|
+
private openedAt?;
|
|
74
|
+
private halfOpenAttempts;
|
|
75
|
+
constructor(serviceName: string, config: CircuitBreakerConfig);
|
|
76
|
+
/**
|
|
77
|
+
* Check if a request should be allowed through the circuit breaker
|
|
78
|
+
* @returns true if request should proceed, false if should fast-fail
|
|
79
|
+
*/
|
|
80
|
+
shouldAllowRequest(): boolean;
|
|
81
|
+
/**
|
|
82
|
+
* Record a successful request
|
|
83
|
+
*/
|
|
84
|
+
recordSuccess(): void;
|
|
85
|
+
/**
|
|
86
|
+
* Record a failed request
|
|
87
|
+
*/
|
|
88
|
+
recordFailure(): void;
|
|
89
|
+
/**
|
|
90
|
+
* Get current statistics
|
|
91
|
+
*/
|
|
92
|
+
getStats(): CircuitStats;
|
|
93
|
+
/**
|
|
94
|
+
* Get current circuit state
|
|
95
|
+
*/
|
|
96
|
+
getState(): CircuitState;
|
|
97
|
+
/**
|
|
98
|
+
* Force reset the circuit breaker to CLOSED state
|
|
99
|
+
* Use with caution - mainly for testing or manual recovery
|
|
100
|
+
*/
|
|
101
|
+
reset(): void;
|
|
102
|
+
/**
|
|
103
|
+
* Transition to a new state
|
|
104
|
+
*/
|
|
105
|
+
private transitionTo;
|
|
106
|
+
}
|
|
107
|
+
//# sourceMappingURL=circuit-breaker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"circuit-breaker.d.ts","sourceRoot":"","sources":["../../../src/utils/circuit-breaker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC,oBAAY,YAAY;IACtB,MAAM,WAAW,CAAE,0CAA0C;IAC7D,IAAI,SAAS,CAAE,qCAAqC;IACpD,SAAS,cAAc;CACxB;AAED,MAAM,WAAW,oBAAoB;IACnC,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,YAAY,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,qBAAa,cAAe,SAAQ,OAAO;IAWvC,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,MAAM;IAXzB,OAAO,CAAC,KAAK,CAAqC;IAClD,OAAO,CAAC,mBAAmB,CAAa;IACxC,OAAO,CAAC,aAAa,CAAa;IAClC,OAAO,CAAC,cAAc,CAAa;IACnC,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,CAAS;IAC1B,OAAO,CAAC,gBAAgB,CAAa;gBAGlB,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,oBAAoB;IAW/C;;;OAGG;IACH,kBAAkB,IAAI,OAAO;IA2C7B;;OAEG;IACH,aAAa,IAAI,IAAI;IAkBrB;;OAEG;IACH,aAAa,IAAI,IAAI;IA+BrB;;OAEG;IACH,QAAQ,IAAI,YAAY;IAYxB;;OAEG;IACH,QAAQ,IAAI,YAAY;IAIxB;;;OAGG;IACH,KAAK,IAAI,IAAI;IAQb;;OAEG;IACH,OAAO,CAAC,YAAY;CAOrB"}
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import { oObject } from '@olane/o-core';
|
|
2
|
+
export var CircuitState;
|
|
3
|
+
(function (CircuitState) {
|
|
4
|
+
CircuitState["CLOSED"] = "CLOSED";
|
|
5
|
+
CircuitState["OPEN"] = "OPEN";
|
|
6
|
+
CircuitState["HALF_OPEN"] = "HALF_OPEN";
|
|
7
|
+
})(CircuitState || (CircuitState = {}));
|
|
8
|
+
/**
|
|
9
|
+
* Circuit Breaker Pattern Implementation
|
|
10
|
+
*
|
|
11
|
+
* Prevents cascading failures by "breaking the circuit" when a service
|
|
12
|
+
* experiences persistent failures. This allows the system to fail fast
|
|
13
|
+
* rather than wasting resources on retries that are likely to fail.
|
|
14
|
+
*
|
|
15
|
+
* States:
|
|
16
|
+
* - CLOSED: Normal operation, all requests pass through
|
|
17
|
+
* - OPEN: Circuit broken due to failures, requests fail immediately
|
|
18
|
+
* - HALF_OPEN: Testing recovery, limited requests allowed
|
|
19
|
+
*
|
|
20
|
+
* Flow:
|
|
21
|
+
* 1. CLOSED -> OPEN: After N consecutive failures
|
|
22
|
+
* 2. OPEN -> HALF_OPEN: After timeout period
|
|
23
|
+
* 3. HALF_OPEN -> CLOSED: After successful request
|
|
24
|
+
* 4. HALF_OPEN -> OPEN: After failure in recovery
|
|
25
|
+
*
|
|
26
|
+
* @example
|
|
27
|
+
* ```typescript
|
|
28
|
+
* const breaker = new CircuitBreaker('registry', {
|
|
29
|
+
* failureThreshold: 3,
|
|
30
|
+
* openTimeoutMs: 30000,
|
|
31
|
+
* halfOpenMaxAttempts: 1,
|
|
32
|
+
* enabled: true,
|
|
33
|
+
* });
|
|
34
|
+
*
|
|
35
|
+
* // Before making request
|
|
36
|
+
* if (!breaker.shouldAllowRequest()) {
|
|
37
|
+
* throw new Error('Circuit breaker is open');
|
|
38
|
+
* }
|
|
39
|
+
*
|
|
40
|
+
* try {
|
|
41
|
+
* const result = await makeRequest();
|
|
42
|
+
* breaker.recordSuccess();
|
|
43
|
+
* return result;
|
|
44
|
+
* } catch (error) {
|
|
45
|
+
* breaker.recordFailure();
|
|
46
|
+
* throw error;
|
|
47
|
+
* }
|
|
48
|
+
* ```
|
|
49
|
+
*/
|
|
50
|
+
export class CircuitBreaker extends oObject {
|
|
51
|
+
constructor(serviceName, config) {
|
|
52
|
+
super();
|
|
53
|
+
this.serviceName = serviceName;
|
|
54
|
+
this.config = config;
|
|
55
|
+
this.state = CircuitState.CLOSED;
|
|
56
|
+
this.consecutiveFailures = 0;
|
|
57
|
+
this.totalFailures = 0;
|
|
58
|
+
this.totalSuccesses = 0;
|
|
59
|
+
this.halfOpenAttempts = 0;
|
|
60
|
+
this.logger.debug(`Circuit breaker initialized for ${serviceName}:`, `threshold=${config.failureThreshold},`, `timeout=${config.openTimeoutMs}ms,`, `enabled=${config.enabled}`);
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Check if a request should be allowed through the circuit breaker
|
|
64
|
+
* @returns true if request should proceed, false if should fast-fail
|
|
65
|
+
*/
|
|
66
|
+
shouldAllowRequest() {
|
|
67
|
+
if (!this.config.enabled) {
|
|
68
|
+
return true;
|
|
69
|
+
}
|
|
70
|
+
const now = Date.now();
|
|
71
|
+
switch (this.state) {
|
|
72
|
+
case CircuitState.CLOSED:
|
|
73
|
+
return true;
|
|
74
|
+
case CircuitState.OPEN:
|
|
75
|
+
// Check if timeout period has elapsed
|
|
76
|
+
if (this.openedAt &&
|
|
77
|
+
now - this.openedAt >= this.config.openTimeoutMs) {
|
|
78
|
+
this.logger.info(`Circuit breaker for ${this.serviceName} entering HALF_OPEN state`);
|
|
79
|
+
this.transitionTo(CircuitState.HALF_OPEN);
|
|
80
|
+
this.halfOpenAttempts = 0;
|
|
81
|
+
return true;
|
|
82
|
+
}
|
|
83
|
+
// Circuit still open, fast-fail
|
|
84
|
+
this.logger.debug(`Circuit breaker for ${this.serviceName} is OPEN, rejecting request`);
|
|
85
|
+
return false;
|
|
86
|
+
case CircuitState.HALF_OPEN:
|
|
87
|
+
// Allow limited attempts in HALF_OPEN state
|
|
88
|
+
if (this.halfOpenAttempts < this.config.halfOpenMaxAttempts) {
|
|
89
|
+
this.halfOpenAttempts++;
|
|
90
|
+
return true;
|
|
91
|
+
}
|
|
92
|
+
this.logger.debug(`Circuit breaker for ${this.serviceName} HALF_OPEN max attempts reached`);
|
|
93
|
+
return false;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Record a successful request
|
|
98
|
+
*/
|
|
99
|
+
recordSuccess() {
|
|
100
|
+
if (!this.config.enabled) {
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
this.totalSuccesses++;
|
|
104
|
+
this.lastSuccessTime = Date.now();
|
|
105
|
+
this.consecutiveFailures = 0;
|
|
106
|
+
if (this.state === CircuitState.HALF_OPEN) {
|
|
107
|
+
this.logger.info(`Circuit breaker for ${this.serviceName} recovered, closing circuit`);
|
|
108
|
+
this.transitionTo(CircuitState.CLOSED);
|
|
109
|
+
this.halfOpenAttempts = 0;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Record a failed request
|
|
114
|
+
*/
|
|
115
|
+
recordFailure() {
|
|
116
|
+
if (!this.config.enabled) {
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
this.totalFailures++;
|
|
120
|
+
this.consecutiveFailures++;
|
|
121
|
+
this.lastFailureTime = Date.now();
|
|
122
|
+
if (this.state === CircuitState.HALF_OPEN) {
|
|
123
|
+
this.logger.warn(`Circuit breaker for ${this.serviceName} failed in HALF_OPEN, reopening circuit`);
|
|
124
|
+
this.transitionTo(CircuitState.OPEN);
|
|
125
|
+
this.openedAt = Date.now();
|
|
126
|
+
this.halfOpenAttempts = 0;
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
if (this.state === CircuitState.CLOSED &&
|
|
130
|
+
this.consecutiveFailures >= this.config.failureThreshold) {
|
|
131
|
+
this.logger.error(`Circuit breaker for ${this.serviceName} opening after ${this.consecutiveFailures} consecutive failures`);
|
|
132
|
+
this.transitionTo(CircuitState.OPEN);
|
|
133
|
+
this.openedAt = Date.now();
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Get current statistics
|
|
138
|
+
*/
|
|
139
|
+
getStats() {
|
|
140
|
+
return {
|
|
141
|
+
state: this.state,
|
|
142
|
+
consecutiveFailures: this.consecutiveFailures,
|
|
143
|
+
totalFailures: this.totalFailures,
|
|
144
|
+
totalSuccesses: this.totalSuccesses,
|
|
145
|
+
lastFailureTime: this.lastFailureTime,
|
|
146
|
+
lastSuccessTime: this.lastSuccessTime,
|
|
147
|
+
openedAt: this.openedAt,
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Get current circuit state
|
|
152
|
+
*/
|
|
153
|
+
getState() {
|
|
154
|
+
return this.state;
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Force reset the circuit breaker to CLOSED state
|
|
158
|
+
* Use with caution - mainly for testing or manual recovery
|
|
159
|
+
*/
|
|
160
|
+
reset() {
|
|
161
|
+
this.logger.info(`Circuit breaker for ${this.serviceName} manually reset`);
|
|
162
|
+
this.transitionTo(CircuitState.CLOSED);
|
|
163
|
+
this.consecutiveFailures = 0;
|
|
164
|
+
this.halfOpenAttempts = 0;
|
|
165
|
+
this.openedAt = undefined;
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* Transition to a new state
|
|
169
|
+
*/
|
|
170
|
+
transitionTo(newState) {
|
|
171
|
+
const oldState = this.state;
|
|
172
|
+
this.state = newState;
|
|
173
|
+
this.logger.debug(`Circuit breaker for ${this.serviceName}: ${oldState} -> ${newState}`);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"circuit-breaker.test.d.ts","sourceRoot":"","sources":["../../../src/utils/circuit-breaker.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
import { expect } from 'chai';
|
|
2
|
+
import { CircuitBreaker, CircuitState } from './circuit-breaker.js';
|
|
3
|
+
describe('CircuitBreaker', () => {
|
|
4
|
+
let breaker;
|
|
5
|
+
beforeEach(() => {
|
|
6
|
+
breaker = new CircuitBreaker('test-service', {
|
|
7
|
+
enabled: true,
|
|
8
|
+
failureThreshold: 3,
|
|
9
|
+
openTimeoutMs: 1000,
|
|
10
|
+
halfOpenMaxAttempts: 1,
|
|
11
|
+
});
|
|
12
|
+
});
|
|
13
|
+
describe('State Transitions', () => {
|
|
14
|
+
it('should start in CLOSED state', () => {
|
|
15
|
+
expect(breaker.getState()).to.equal(CircuitState.CLOSED);
|
|
16
|
+
expect(breaker.shouldAllowRequest()).to.equal(true);
|
|
17
|
+
});
|
|
18
|
+
it('should transition to OPEN after threshold failures', () => {
|
|
19
|
+
expect(breaker.getState()).to.equal(CircuitState.CLOSED);
|
|
20
|
+
// Record failures up to threshold
|
|
21
|
+
breaker.recordFailure();
|
|
22
|
+
breaker.recordFailure();
|
|
23
|
+
expect(breaker.getState()).to.equal(CircuitState.CLOSED);
|
|
24
|
+
// Third failure should open circuit
|
|
25
|
+
breaker.recordFailure();
|
|
26
|
+
expect(breaker.getState()).to.equal(CircuitState.OPEN);
|
|
27
|
+
expect(breaker.shouldAllowRequest()).to.equal(false);
|
|
28
|
+
});
|
|
29
|
+
it('should transition to HALF_OPEN after timeout', async () => {
|
|
30
|
+
// Open the circuit
|
|
31
|
+
breaker.recordFailure();
|
|
32
|
+
breaker.recordFailure();
|
|
33
|
+
breaker.recordFailure();
|
|
34
|
+
expect(breaker.getState()).to.equal(CircuitState.OPEN);
|
|
35
|
+
// Wait for timeout
|
|
36
|
+
await new Promise((resolve) => setTimeout(resolve, 1100));
|
|
37
|
+
// Next request should transition to HALF_OPEN
|
|
38
|
+
expect(breaker.shouldAllowRequest()).to.equal(true);
|
|
39
|
+
expect(breaker.getState()).to.equal(CircuitState.HALF_OPEN);
|
|
40
|
+
});
|
|
41
|
+
it('should transition from HALF_OPEN to CLOSED on success', async () => {
|
|
42
|
+
// Open the circuit
|
|
43
|
+
breaker.recordFailure();
|
|
44
|
+
breaker.recordFailure();
|
|
45
|
+
breaker.recordFailure();
|
|
46
|
+
// Wait and transition to HALF_OPEN
|
|
47
|
+
await new Promise((resolve) => setTimeout(resolve, 1100));
|
|
48
|
+
breaker.shouldAllowRequest();
|
|
49
|
+
// Success should close circuit
|
|
50
|
+
breaker.recordSuccess();
|
|
51
|
+
expect(breaker.getState()).to.equal(CircuitState.CLOSED);
|
|
52
|
+
});
|
|
53
|
+
it('should transition from HALF_OPEN to OPEN on failure', async () => {
|
|
54
|
+
// Open the circuit
|
|
55
|
+
breaker.recordFailure();
|
|
56
|
+
breaker.recordFailure();
|
|
57
|
+
breaker.recordFailure();
|
|
58
|
+
// Wait and transition to HALF_OPEN
|
|
59
|
+
await new Promise((resolve) => setTimeout(resolve, 1100));
|
|
60
|
+
breaker.shouldAllowRequest();
|
|
61
|
+
expect(breaker.getState()).to.equal(CircuitState.HALF_OPEN);
|
|
62
|
+
// Failure should reopen circuit
|
|
63
|
+
breaker.recordFailure();
|
|
64
|
+
expect(breaker.getState()).to.equal(CircuitState.OPEN);
|
|
65
|
+
});
|
|
66
|
+
});
|
|
67
|
+
describe('Request Gating', () => {
|
|
68
|
+
it('should allow all requests in CLOSED state', () => {
|
|
69
|
+
expect(breaker.shouldAllowRequest()).to.equal(true);
|
|
70
|
+
expect(breaker.shouldAllowRequest()).to.equal(true);
|
|
71
|
+
});
|
|
72
|
+
it('should block all requests in OPEN state', () => {
|
|
73
|
+
// Open the circuit
|
|
74
|
+
breaker.recordFailure();
|
|
75
|
+
breaker.recordFailure();
|
|
76
|
+
breaker.recordFailure();
|
|
77
|
+
expect(breaker.shouldAllowRequest()).to.equal(false);
|
|
78
|
+
});
|
|
79
|
+
it('should limit requests in HALF_OPEN state', async () => {
|
|
80
|
+
// Open the circuit
|
|
81
|
+
breaker.recordFailure();
|
|
82
|
+
breaker.recordFailure();
|
|
83
|
+
breaker.recordFailure();
|
|
84
|
+
// Wait and transition to HALF_OPEN
|
|
85
|
+
await new Promise((resolve) => setTimeout(resolve, 1100));
|
|
86
|
+
// First request allowed
|
|
87
|
+
expect(breaker.shouldAllowRequest()).to.equal(true);
|
|
88
|
+
expect(breaker.getState()).to.equal(CircuitState.HALF_OPEN);
|
|
89
|
+
// Subsequent requests blocked until result recorded
|
|
90
|
+
expect(breaker.shouldAllowRequest()).to.equal(false);
|
|
91
|
+
});
|
|
92
|
+
});
|
|
93
|
+
describe('Failure Tracking', () => {
|
|
94
|
+
it('should track consecutive failures', () => {
|
|
95
|
+
breaker.recordFailure();
|
|
96
|
+
expect(breaker.getStats().consecutiveFailures).to.equal(1);
|
|
97
|
+
breaker.recordFailure();
|
|
98
|
+
expect(breaker.getStats().consecutiveFailures).to.equal(2);
|
|
99
|
+
breaker.recordFailure();
|
|
100
|
+
expect(breaker.getStats().consecutiveFailures).to.equal(3);
|
|
101
|
+
});
|
|
102
|
+
it('should reset consecutive failures on success', () => {
|
|
103
|
+
breaker.recordFailure();
|
|
104
|
+
breaker.recordFailure();
|
|
105
|
+
expect(breaker.getStats().consecutiveFailures).to.equal(2);
|
|
106
|
+
breaker.recordSuccess();
|
|
107
|
+
expect(breaker.getStats().consecutiveFailures).to.equal(0);
|
|
108
|
+
expect(breaker.getState()).to.equal(CircuitState.CLOSED);
|
|
109
|
+
});
|
|
110
|
+
it('should track total failures and successes', () => {
|
|
111
|
+
breaker.recordFailure();
|
|
112
|
+
breaker.recordSuccess();
|
|
113
|
+
breaker.recordFailure();
|
|
114
|
+
breaker.recordSuccess();
|
|
115
|
+
const stats = breaker.getStats();
|
|
116
|
+
expect(stats.totalFailures).to.equal(2);
|
|
117
|
+
expect(stats.totalSuccesses).to.equal(2);
|
|
118
|
+
});
|
|
119
|
+
it('should track timestamps', () => {
|
|
120
|
+
const beforeFailure = Date.now();
|
|
121
|
+
breaker.recordFailure();
|
|
122
|
+
const afterFailure = Date.now();
|
|
123
|
+
const stats = breaker.getStats();
|
|
124
|
+
expect(stats.lastFailureTime).greaterThanOrEqual(beforeFailure);
|
|
125
|
+
expect(stats.lastFailureTime).lessThanOrEqual(afterFailure);
|
|
126
|
+
const beforeSuccess = Date.now();
|
|
127
|
+
breaker.recordSuccess();
|
|
128
|
+
const afterSuccess = Date.now();
|
|
129
|
+
const stats2 = breaker.getStats();
|
|
130
|
+
expect(stats2.lastSuccessTime).greaterThanOrEqual(beforeSuccess);
|
|
131
|
+
expect(stats2.lastSuccessTime).lessThanOrEqual(afterSuccess);
|
|
132
|
+
});
|
|
133
|
+
});
|
|
134
|
+
describe('Configuration', () => {
|
|
135
|
+
it('should respect custom failure threshold', () => {
|
|
136
|
+
const customBreaker = new CircuitBreaker('test', {
|
|
137
|
+
enabled: true,
|
|
138
|
+
failureThreshold: 5,
|
|
139
|
+
openTimeoutMs: 1000,
|
|
140
|
+
halfOpenMaxAttempts: 1,
|
|
141
|
+
});
|
|
142
|
+
// Should not open before threshold
|
|
143
|
+
customBreaker.recordFailure();
|
|
144
|
+
customBreaker.recordFailure();
|
|
145
|
+
customBreaker.recordFailure();
|
|
146
|
+
customBreaker.recordFailure();
|
|
147
|
+
expect(customBreaker.getState()).to.equal(CircuitState.CLOSED);
|
|
148
|
+
// Should open at threshold
|
|
149
|
+
customBreaker.recordFailure();
|
|
150
|
+
expect(customBreaker.getState()).to.equal(CircuitState.OPEN);
|
|
151
|
+
});
|
|
152
|
+
it('should respect custom half-open attempts', async () => {
|
|
153
|
+
const customBreaker = new CircuitBreaker('test', {
|
|
154
|
+
enabled: true,
|
|
155
|
+
failureThreshold: 2,
|
|
156
|
+
openTimeoutMs: 100,
|
|
157
|
+
halfOpenMaxAttempts: 3,
|
|
158
|
+
});
|
|
159
|
+
// Open circuit
|
|
160
|
+
customBreaker.recordFailure();
|
|
161
|
+
customBreaker.recordFailure();
|
|
162
|
+
// Wait for timeout
|
|
163
|
+
await new Promise((resolve) => setTimeout(resolve, 150));
|
|
164
|
+
// Should allow 3 attempts in HALF_OPEN
|
|
165
|
+
expect(customBreaker.shouldAllowRequest()).to.equal(true);
|
|
166
|
+
expect(customBreaker.shouldAllowRequest()).to.equal(true);
|
|
167
|
+
expect(customBreaker.shouldAllowRequest()).to.equal(true);
|
|
168
|
+
expect(customBreaker.shouldAllowRequest()).to.equal(false);
|
|
169
|
+
});
|
|
170
|
+
it('should bypass all logic when disabled', () => {
|
|
171
|
+
const disabledBreaker = new CircuitBreaker('test', {
|
|
172
|
+
enabled: false,
|
|
173
|
+
failureThreshold: 1,
|
|
174
|
+
openTimeoutMs: 1000,
|
|
175
|
+
halfOpenMaxAttempts: 1,
|
|
176
|
+
});
|
|
177
|
+
// Record many failures
|
|
178
|
+
disabledBreaker.recordFailure();
|
|
179
|
+
disabledBreaker.recordFailure();
|
|
180
|
+
disabledBreaker.recordFailure();
|
|
181
|
+
// Should still allow requests
|
|
182
|
+
expect(disabledBreaker.shouldAllowRequest()).to.equal(true);
|
|
183
|
+
expect(disabledBreaker.getState()).to.equal(CircuitState.CLOSED);
|
|
184
|
+
});
|
|
185
|
+
});
|
|
186
|
+
describe('Manual Control', () => {
|
|
187
|
+
it('should reset to CLOSED state', () => {
|
|
188
|
+
// Open circuit
|
|
189
|
+
breaker.recordFailure();
|
|
190
|
+
breaker.recordFailure();
|
|
191
|
+
breaker.recordFailure();
|
|
192
|
+
expect(breaker.getState()).to.equal(CircuitState.OPEN);
|
|
193
|
+
// Reset
|
|
194
|
+
breaker.reset();
|
|
195
|
+
expect(breaker.getState()).to.equal(CircuitState.CLOSED);
|
|
196
|
+
expect(breaker.getStats().consecutiveFailures).to.equal(0);
|
|
197
|
+
expect(breaker.shouldAllowRequest()).to.equal(true);
|
|
198
|
+
});
|
|
199
|
+
});
|
|
200
|
+
describe('Statistics', () => {
|
|
201
|
+
it('should provide comprehensive stats', () => {
|
|
202
|
+
breaker.recordFailure();
|
|
203
|
+
breaker.recordFailure();
|
|
204
|
+
breaker.recordSuccess();
|
|
205
|
+
const stats = breaker.getStats();
|
|
206
|
+
expect(stats).to.have.property('state');
|
|
207
|
+
expect(stats).to.have.property('consecutiveFailures');
|
|
208
|
+
expect(stats).to.have.property('totalFailures');
|
|
209
|
+
expect(stats).to.have.property('totalSuccesses');
|
|
210
|
+
expect(stats).to.have.property('lastFailureTime');
|
|
211
|
+
expect(stats).to.have.property('lastSuccessTime');
|
|
212
|
+
expect(stats).to.have.property('openedAt');
|
|
213
|
+
expect(stats.state).to.equal(CircuitState.CLOSED);
|
|
214
|
+
expect(stats.consecutiveFailures).to.equal(0);
|
|
215
|
+
expect(stats.totalFailures).to.equal(2);
|
|
216
|
+
expect(stats.totalSuccesses).to.equal(1);
|
|
217
|
+
});
|
|
218
|
+
it('should track openedAt timestamp', () => {
|
|
219
|
+
const before = Date.now();
|
|
220
|
+
breaker.recordFailure();
|
|
221
|
+
breaker.recordFailure();
|
|
222
|
+
breaker.recordFailure();
|
|
223
|
+
const after = Date.now();
|
|
224
|
+
const stats = breaker.getStats();
|
|
225
|
+
expect(stats.openedAt).greaterThanOrEqual(before);
|
|
226
|
+
expect(stats.openedAt).lessThanOrEqual(after);
|
|
227
|
+
});
|
|
228
|
+
});
|
|
229
|
+
describe('Edge Cases', () => {
|
|
230
|
+
it('should handle rapid state transitions', async () => {
|
|
231
|
+
// Open
|
|
232
|
+
breaker.recordFailure();
|
|
233
|
+
breaker.recordFailure();
|
|
234
|
+
breaker.recordFailure();
|
|
235
|
+
expect(breaker.getState()).to.equal(CircuitState.OPEN);
|
|
236
|
+
// Wait for HALF_OPEN
|
|
237
|
+
await new Promise((resolve) => setTimeout(resolve, 1100));
|
|
238
|
+
breaker.shouldAllowRequest();
|
|
239
|
+
expect(breaker.getState()).to.equal(CircuitState.HALF_OPEN);
|
|
240
|
+
// Close
|
|
241
|
+
breaker.recordSuccess();
|
|
242
|
+
expect(breaker.getState()).to.equal(CircuitState.CLOSED);
|
|
243
|
+
// Open again
|
|
244
|
+
breaker.recordFailure();
|
|
245
|
+
breaker.recordFailure();
|
|
246
|
+
breaker.recordFailure();
|
|
247
|
+
expect(breaker.getState()).to.equal(CircuitState.OPEN);
|
|
248
|
+
});
|
|
249
|
+
it('should not open before timeout expires in OPEN state', async () => {
|
|
250
|
+
// Open circuit
|
|
251
|
+
breaker.recordFailure();
|
|
252
|
+
breaker.recordFailure();
|
|
253
|
+
breaker.recordFailure();
|
|
254
|
+
expect(breaker.getState()).to.equal(CircuitState.OPEN);
|
|
255
|
+
// Wait less than timeout
|
|
256
|
+
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
257
|
+
// Should still be closed
|
|
258
|
+
expect(breaker.shouldAllowRequest()).to.equal(false);
|
|
259
|
+
expect(breaker.getState()).to.equal(CircuitState.OPEN);
|
|
260
|
+
});
|
|
261
|
+
});
|
|
262
|
+
});
|
|
@@ -1,32 +1,42 @@
|
|
|
1
1
|
import { oObject, oAddress } from '@olane/o-core';
|
|
2
|
+
import { CircuitBreakerConfig } from './circuit-breaker.js';
|
|
2
3
|
export interface LeaderRetryConfig {
|
|
3
4
|
enabled: boolean;
|
|
4
5
|
maxAttempts: number;
|
|
5
6
|
baseDelayMs: number;
|
|
6
7
|
maxDelayMs: number;
|
|
7
8
|
timeoutMs: number;
|
|
9
|
+
circuitBreaker?: CircuitBreakerConfig;
|
|
8
10
|
}
|
|
9
11
|
/**
|
|
10
12
|
* Leader Request Wrapper
|
|
11
13
|
*
|
|
12
|
-
* Wraps requests to leader/registry with
|
|
14
|
+
* Wraps requests to leader/registry with retry logic and circuit breaker.
|
|
13
15
|
* Used when leader may be temporarily unavailable (healing, maintenance).
|
|
14
16
|
*
|
|
15
17
|
* Strategy:
|
|
16
18
|
* 1. Detect if request is to leader or registry
|
|
17
|
-
* 2.
|
|
18
|
-
* 3.
|
|
19
|
-
* 4.
|
|
19
|
+
* 2. Check circuit breaker state (fast-fail if open)
|
|
20
|
+
* 3. Apply retry logic with exponential backoff
|
|
21
|
+
* 4. Timeout individual attempts
|
|
22
|
+
* 5. Record success/failure in circuit breaker
|
|
23
|
+
* 6. Log retries for observability
|
|
20
24
|
*/
|
|
21
25
|
export declare class LeaderRequestWrapper extends oObject {
|
|
22
26
|
private config;
|
|
27
|
+
private leaderCircuitBreaker;
|
|
28
|
+
private registryCircuitBreaker;
|
|
23
29
|
constructor(config: LeaderRetryConfig);
|
|
24
30
|
/**
|
|
25
31
|
* Check if address is a leader-related address that needs retry
|
|
26
32
|
*/
|
|
27
33
|
private isLeaderAddress;
|
|
28
34
|
/**
|
|
29
|
-
*
|
|
35
|
+
* Get the appropriate circuit breaker for the address
|
|
36
|
+
*/
|
|
37
|
+
private getCircuitBreaker;
|
|
38
|
+
/**
|
|
39
|
+
* Execute request with retry logic and circuit breaker
|
|
30
40
|
*/
|
|
31
41
|
execute<T>(requestFn: () => Promise<T>, address: oAddress, context?: string): Promise<T>;
|
|
32
42
|
/**
|
|
@@ -41,5 +51,16 @@ export declare class LeaderRequestWrapper extends oObject {
|
|
|
41
51
|
* Get current configuration
|
|
42
52
|
*/
|
|
43
53
|
getConfig(): LeaderRetryConfig;
|
|
54
|
+
/**
|
|
55
|
+
* Get circuit breaker statistics for observability
|
|
56
|
+
*/
|
|
57
|
+
getCircuitBreakerStats(): {
|
|
58
|
+
leader: import("./circuit-breaker.js").CircuitStats;
|
|
59
|
+
registry: import("./circuit-breaker.js").CircuitStats;
|
|
60
|
+
};
|
|
61
|
+
/**
|
|
62
|
+
* Reset circuit breakers (for testing or manual recovery)
|
|
63
|
+
*/
|
|
64
|
+
resetCircuitBreakers(): void;
|
|
44
65
|
}
|
|
45
66
|
//# sourceMappingURL=leader-request-wrapper.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"leader-request-wrapper.d.ts","sourceRoot":"","sources":["../../../src/utils/leader-request-wrapper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAuB,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"leader-request-wrapper.d.ts","sourceRoot":"","sources":["../../../src/utils/leader-request-wrapper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAuB,MAAM,eAAe,CAAC;AACvE,OAAO,EAEL,oBAAoB,EAErB,MAAM,sBAAsB,CAAC;AAE9B,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,oBAAoB,CAAC;CACvC;AAED;;;;;;;;;;;;;GAaG;AACH,qBAAa,oBAAqB,SAAQ,OAAO;IAInC,OAAO,CAAC,MAAM;IAH1B,OAAO,CAAC,oBAAoB,CAAiB;IAC7C,OAAO,CAAC,sBAAsB,CAAiB;gBAE3B,MAAM,EAAE,iBAAiB;IAuB7C;;OAEG;IACH,OAAO,CAAC,eAAe;IAKvB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAWzB;;OAEG;IACG,OAAO,CAAC,CAAC,EACb,SAAS,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EAC3B,OAAO,EAAE,QAAQ,EACjB,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC,CAAC,CAAC;IA2Gb;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAK7B;;OAEG;IACH,OAAO,CAAC,KAAK;IAIb;;OAEG;IACH,SAAS,IAAI,iBAAiB;IAI9B;;OAEG;IACH,sBAAsB;;;;IAOtB;;OAEG;IACH,oBAAoB,IAAI,IAAI;CAI7B"}
|
|
@@ -1,20 +1,36 @@
|
|
|
1
1
|
import { oObject } from '@olane/o-core';
|
|
2
|
+
import { CircuitBreaker, CircuitState, } from './circuit-breaker.js';
|
|
2
3
|
/**
|
|
3
4
|
* Leader Request Wrapper
|
|
4
5
|
*
|
|
5
|
-
* Wraps requests to leader/registry with
|
|
6
|
+
* Wraps requests to leader/registry with retry logic and circuit breaker.
|
|
6
7
|
* Used when leader may be temporarily unavailable (healing, maintenance).
|
|
7
8
|
*
|
|
8
9
|
* Strategy:
|
|
9
10
|
* 1. Detect if request is to leader or registry
|
|
10
|
-
* 2.
|
|
11
|
-
* 3.
|
|
12
|
-
* 4.
|
|
11
|
+
* 2. Check circuit breaker state (fast-fail if open)
|
|
12
|
+
* 3. Apply retry logic with exponential backoff
|
|
13
|
+
* 4. Timeout individual attempts
|
|
14
|
+
* 5. Record success/failure in circuit breaker
|
|
15
|
+
* 6. Log retries for observability
|
|
13
16
|
*/
|
|
14
17
|
export class LeaderRequestWrapper extends oObject {
|
|
15
18
|
constructor(config) {
|
|
16
19
|
super();
|
|
17
20
|
this.config = config;
|
|
21
|
+
// Initialize circuit breakers with defaults if not provided
|
|
22
|
+
const defaultCircuitBreakerConfig = {
|
|
23
|
+
failureThreshold: 3,
|
|
24
|
+
openTimeoutMs: 30000, // 30 seconds
|
|
25
|
+
halfOpenMaxAttempts: 1,
|
|
26
|
+
enabled: true,
|
|
27
|
+
};
|
|
28
|
+
const circuitConfig = {
|
|
29
|
+
...defaultCircuitBreakerConfig,
|
|
30
|
+
...(config.circuitBreaker || {}),
|
|
31
|
+
};
|
|
32
|
+
this.leaderCircuitBreaker = new CircuitBreaker('leader', circuitConfig);
|
|
33
|
+
this.registryCircuitBreaker = new CircuitBreaker('registry', circuitConfig);
|
|
18
34
|
}
|
|
19
35
|
/**
|
|
20
36
|
* Check if address is a leader-related address that needs retry
|
|
@@ -24,27 +40,56 @@ export class LeaderRequestWrapper extends oObject {
|
|
|
24
40
|
return addressStr === 'o://leader' || addressStr === 'o://registry';
|
|
25
41
|
}
|
|
26
42
|
/**
|
|
27
|
-
*
|
|
43
|
+
* Get the appropriate circuit breaker for the address
|
|
44
|
+
*/
|
|
45
|
+
getCircuitBreaker(address) {
|
|
46
|
+
const addressStr = address.toString();
|
|
47
|
+
if (addressStr === 'o://leader') {
|
|
48
|
+
return this.leaderCircuitBreaker;
|
|
49
|
+
}
|
|
50
|
+
if (addressStr === 'o://registry') {
|
|
51
|
+
return this.registryCircuitBreaker;
|
|
52
|
+
}
|
|
53
|
+
return null;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Execute request with retry logic and circuit breaker
|
|
28
57
|
*/
|
|
29
58
|
async execute(requestFn, address, context) {
|
|
30
59
|
// If retry disabled or not a leader address, execute directly
|
|
31
60
|
if (!this.config.enabled || !this.isLeaderAddress(address)) {
|
|
32
61
|
return await requestFn();
|
|
33
62
|
}
|
|
63
|
+
const circuitBreaker = this.getCircuitBreaker(address);
|
|
64
|
+
// Check circuit breaker state before attempting
|
|
65
|
+
if (circuitBreaker && !circuitBreaker.shouldAllowRequest()) {
|
|
66
|
+
const stats = circuitBreaker.getStats();
|
|
67
|
+
const error = new Error(`Circuit breaker is ${stats.state} for ${address.toString()}. ` +
|
|
68
|
+
`Consecutive failures: ${stats.consecutiveFailures}. ` +
|
|
69
|
+
`Fast-failing to prevent cascading failures.`);
|
|
70
|
+
this.logger.warn(`Circuit breaker blocked request to ${address.toString()}` +
|
|
71
|
+
(context ? ` (${context})` : ''));
|
|
72
|
+
throw error;
|
|
73
|
+
}
|
|
34
74
|
let attempt = 0;
|
|
35
75
|
let lastError;
|
|
36
76
|
while (attempt < this.config.maxAttempts) {
|
|
37
77
|
attempt++;
|
|
38
78
|
try {
|
|
39
|
-
|
|
40
|
-
(
|
|
79
|
+
if (attempt > 5) {
|
|
80
|
+
this.logger.debug(`Retrying... Leader request attempt ${attempt}/${this.config.maxAttempts}` +
|
|
81
|
+
(context ? ` (${context})` : ''));
|
|
82
|
+
}
|
|
41
83
|
// Create timeout promise
|
|
42
84
|
const timeoutPromise = new Promise((_, reject) => {
|
|
43
85
|
setTimeout(() => reject(new Error(`Leader request timeout after ${this.config.timeoutMs}ms`)), this.config.timeoutMs);
|
|
44
86
|
});
|
|
45
87
|
// Race between request and timeout
|
|
46
88
|
const result = await Promise.race([requestFn(), timeoutPromise]);
|
|
47
|
-
// Success!
|
|
89
|
+
// Success! Record in circuit breaker
|
|
90
|
+
if (circuitBreaker) {
|
|
91
|
+
circuitBreaker.recordSuccess();
|
|
92
|
+
}
|
|
48
93
|
if (attempt > 1) {
|
|
49
94
|
this.logger.info(`Leader request succeeded after ${attempt} attempts` +
|
|
50
95
|
(context ? ` (${context})` : ''));
|
|
@@ -53,8 +98,18 @@ export class LeaderRequestWrapper extends oObject {
|
|
|
53
98
|
}
|
|
54
99
|
catch (error) {
|
|
55
100
|
lastError = error instanceof Error ? error : new Error(String(error));
|
|
101
|
+
// Record failure in circuit breaker
|
|
102
|
+
if (circuitBreaker) {
|
|
103
|
+
circuitBreaker.recordFailure();
|
|
104
|
+
}
|
|
56
105
|
this.logger.warn(`Leader request attempt ${attempt} failed: ${lastError.message}` +
|
|
57
106
|
(context ? ` (${context})` : ''));
|
|
107
|
+
// Check if circuit breaker has opened during retries
|
|
108
|
+
if (circuitBreaker && circuitBreaker.getState() === CircuitState.OPEN) {
|
|
109
|
+
this.logger.error(`Circuit breaker opened during retries for ${address.toString()}, stopping retry attempts` +
|
|
110
|
+
(context ? ` (${context})` : ''));
|
|
111
|
+
throw new Error(`Circuit breaker opened after ${attempt} attempts: ${lastError.message}`);
|
|
112
|
+
}
|
|
58
113
|
if (attempt < this.config.maxAttempts) {
|
|
59
114
|
const delay = this.calculateBackoffDelay(attempt);
|
|
60
115
|
this.logger.debug(`Waiting ${delay}ms before next attempt...`);
|
|
@@ -86,4 +141,20 @@ export class LeaderRequestWrapper extends oObject {
|
|
|
86
141
|
getConfig() {
|
|
87
142
|
return { ...this.config };
|
|
88
143
|
}
|
|
144
|
+
/**
|
|
145
|
+
* Get circuit breaker statistics for observability
|
|
146
|
+
*/
|
|
147
|
+
getCircuitBreakerStats() {
|
|
148
|
+
return {
|
|
149
|
+
leader: this.leaderCircuitBreaker.getStats(),
|
|
150
|
+
registry: this.registryCircuitBreaker.getStats(),
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Reset circuit breakers (for testing or manual recovery)
|
|
155
|
+
*/
|
|
156
|
+
resetCircuitBreakers() {
|
|
157
|
+
this.leaderCircuitBreaker.reset();
|
|
158
|
+
this.registryCircuitBreaker.reset();
|
|
159
|
+
}
|
|
89
160
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
//# sourceMappingURL=leader-request-wrapper.test.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"leader-request-wrapper.test.d.ts","sourceRoot":"","sources":["../../../src/utils/leader-request-wrapper.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// import { expect } from 'chai';
|
|
3
|
+
// import { LeaderRequestWrapper } from './leader-request-wrapper.js';
|
|
4
|
+
// import { oAddress } from '@olane/o-core';
|
|
5
|
+
// import { CircuitState } from './circuit-breaker.js';
|
|
6
|
+
// describe('LeaderRequestWrapper with Circuit Breaker', () => {
|
|
7
|
+
// let wrapper: LeaderRequestWrapper;
|
|
8
|
+
// const leaderAddress = new oAddress('o://leader');
|
|
9
|
+
// const registryAddress = new oAddress('o://registry');
|
|
10
|
+
// const regularAddress = new oAddress('o://some-service');
|
|
11
|
+
// beforeEach(() => {
|
|
12
|
+
// wrapper = new LeaderRequestWrapper({
|
|
13
|
+
// enabled: true,
|
|
14
|
+
// maxAttempts: 5,
|
|
15
|
+
// baseDelayMs: 10,
|
|
16
|
+
// maxDelayMs: 100,
|
|
17
|
+
// timeoutMs: 1000,
|
|
18
|
+
// circuitBreaker: {
|
|
19
|
+
// enabled: true,
|
|
20
|
+
// failureThreshold: 3,
|
|
21
|
+
// openTimeoutMs: 500,
|
|
22
|
+
// halfOpenMaxAttempts: 1,
|
|
23
|
+
// },
|
|
24
|
+
// });
|
|
25
|
+
// });
|
|
26
|
+
// describe('Circuit Breaker Integration', () => {
|
|
27
|
+
// it('should execute request successfully and record success', async () => {
|
|
28
|
+
// const mockRequest = vi.fn().mockResolvedValue('success');
|
|
29
|
+
// const result = await wrapper.execute(mockRequest, leaderAddress);
|
|
30
|
+
// expect(result).toBe('success');
|
|
31
|
+
// expect(mockRequest).toHaveBeenCalledTimes(1);
|
|
32
|
+
// const stats = wrapper.getCircuitBreakerStats();
|
|
33
|
+
// expect(stats.leader.totalSuccesses).toBe(1);
|
|
34
|
+
// expect(stats.leader.state).toBe(CircuitState.CLOSED);
|
|
35
|
+
// });
|
|
36
|
+
// it('should record failures in circuit breaker', async () => {
|
|
37
|
+
// const mockRequest = vi.fn().mockRejectedValue(new Error('Service down'));
|
|
38
|
+
// await expect(
|
|
39
|
+
// wrapper.execute(mockRequest, leaderAddress),
|
|
40
|
+
// ).rejects.toThrow();
|
|
41
|
+
// const stats = wrapper.getCircuitBreakerStats();
|
|
42
|
+
// expect(stats.leader.totalFailures).toBeGreaterThan(0);
|
|
43
|
+
// expect(stats.leader.consecutiveFailures).toBeGreaterThan(0);
|
|
44
|
+
// });
|
|
45
|
+
// it('should open circuit after threshold failures', async () => {
|
|
46
|
+
// const mockRequest = vi.fn().mockRejectedValue(new Error('Service down'));
|
|
47
|
+
// // Attempt request multiple times to trigger circuit breaker
|
|
48
|
+
// await expect(
|
|
49
|
+
// wrapper.execute(mockRequest, leaderAddress),
|
|
50
|
+
// ).rejects.toThrow();
|
|
51
|
+
// const stats = wrapper.getCircuitBreakerStats();
|
|
52
|
+
// expect(stats.leader.state).toBe(CircuitState.OPEN);
|
|
53
|
+
// });
|
|
54
|
+
// it('should fast-fail when circuit is open', async () => {
|
|
55
|
+
// const mockRequest = vi.fn().mockRejectedValue(new Error('Service down'));
|
|
56
|
+
// // First request to open circuit
|
|
57
|
+
// await expect(
|
|
58
|
+
// wrapper.execute(mockRequest, leaderAddress),
|
|
59
|
+
// ).rejects.toThrow();
|
|
60
|
+
// // Reset mock to verify it's not called
|
|
61
|
+
// mockRequest.mockClear();
|
|
62
|
+
// // Second request should fast-fail
|
|
63
|
+
// await expect(wrapper.execute(mockRequest, leaderAddress)).rejects.toThrow(
|
|
64
|
+
// /Circuit breaker is OPEN/,
|
|
65
|
+
// );
|
|
66
|
+
// // Request function should not have been called
|
|
67
|
+
// expect(mockRequest).not.toHaveBeenCalled();
|
|
68
|
+
// });
|
|
69
|
+
// it('should stop retrying when circuit opens mid-retry', async () => {
|
|
70
|
+
// let callCount = 0;
|
|
71
|
+
// const mockRequest = vi.fn().mockImplementation(() => {
|
|
72
|
+
// callCount++;
|
|
73
|
+
// throw new Error('Service down');
|
|
74
|
+
// });
|
|
75
|
+
// await expect(
|
|
76
|
+
// wrapper.execute(mockRequest, leaderAddress),
|
|
77
|
+
// ).rejects.toThrow();
|
|
78
|
+
// // Should have stopped retrying after circuit opened
|
|
79
|
+
// expect(callCount).toBeLessThan(5); // Less than maxAttempts
|
|
80
|
+
// expect(callCount).toBeGreaterThanOrEqual(3); // At least threshold attempts
|
|
81
|
+
// });
|
|
82
|
+
// it('should track separate circuits for leader and registry', async () => {
|
|
83
|
+
// const failingRequest = vi
|
|
84
|
+
// .fn()
|
|
85
|
+
// .mockRejectedValue(new Error('Service down'));
|
|
86
|
+
// // Fail leader requests
|
|
87
|
+
// await expect(
|
|
88
|
+
// wrapper.execute(failingRequest, leaderAddress),
|
|
89
|
+
// ).rejects.toThrow();
|
|
90
|
+
// const stats = wrapper.getCircuitBreakerStats();
|
|
91
|
+
// expect(stats.leader.state).toBe(CircuitState.OPEN);
|
|
92
|
+
// expect(stats.registry.state).toBe(CircuitState.CLOSED);
|
|
93
|
+
// });
|
|
94
|
+
// it('should allow recovery after timeout in HALF_OPEN state', async () => {
|
|
95
|
+
// const mockRequest = vi
|
|
96
|
+
// .fn()
|
|
97
|
+
// .mockRejectedValueOnce(new Error('Service down'))
|
|
98
|
+
// .mockResolvedValue('recovered');
|
|
99
|
+
// // Open circuit
|
|
100
|
+
// await expect(
|
|
101
|
+
// wrapper.execute(mockRequest, leaderAddress),
|
|
102
|
+
// ).rejects.toThrow();
|
|
103
|
+
// expect(wrapper.getCircuitBreakerStats().leader.state).toBe(
|
|
104
|
+
// CircuitState.OPEN,
|
|
105
|
+
// );
|
|
106
|
+
// // Wait for circuit to attempt recovery
|
|
107
|
+
// await new Promise((resolve) => setTimeout(resolve, 600));
|
|
108
|
+
// // Should succeed and close circuit
|
|
109
|
+
// const result = await wrapper.execute(mockRequest, leaderAddress);
|
|
110
|
+
// expect(result).toBe('recovered');
|
|
111
|
+
// expect(wrapper.getCircuitBreakerStats().leader.state).toBe(
|
|
112
|
+
// CircuitState.CLOSED,
|
|
113
|
+
// );
|
|
114
|
+
// });
|
|
115
|
+
// });
|
|
116
|
+
// describe('Non-Leader Addresses', () => {
|
|
117
|
+
// it('should not use circuit breaker for non-leader addresses', async () => {
|
|
118
|
+
// const mockRequest = vi.fn().mockRejectedValue(new Error('Service down'));
|
|
119
|
+
// await expect(
|
|
120
|
+
// wrapper.execute(mockRequest, regularAddress),
|
|
121
|
+
// ).rejects.toThrow('Service down');
|
|
122
|
+
// // Should execute directly without retry
|
|
123
|
+
// expect(mockRequest).toHaveBeenCalledTimes(1);
|
|
124
|
+
// // Should not affect circuit breaker stats
|
|
125
|
+
// const stats = wrapper.getCircuitBreakerStats();
|
|
126
|
+
// expect(stats.leader.totalFailures).toBe(0);
|
|
127
|
+
// expect(stats.registry.totalFailures).toBe(0);
|
|
128
|
+
// });
|
|
129
|
+
// it('should execute successful requests directly', async () => {
|
|
130
|
+
// const mockRequest = vi.fn().mockResolvedValue('success');
|
|
131
|
+
// const result = await wrapper.execute(mockRequest, regularAddress);
|
|
132
|
+
// expect(result).toBe('success');
|
|
133
|
+
// expect(mockRequest).toHaveBeenCalledTimes(1);
|
|
134
|
+
// });
|
|
135
|
+
// });
|
|
136
|
+
// describe('Timeout Handling', () => {
|
|
137
|
+
// it('should timeout long-running requests', async () => {
|
|
138
|
+
// const longRequest = vi.fn().mockImplementation(
|
|
139
|
+
// () =>
|
|
140
|
+
// new Promise((resolve) => {
|
|
141
|
+
// setTimeout(() => resolve('too-late'), 2000);
|
|
142
|
+
// }),
|
|
143
|
+
// );
|
|
144
|
+
// await expect(wrapper.execute(longRequest, leaderAddress)).rejects.toThrow(
|
|
145
|
+
// /timeout/,
|
|
146
|
+
// );
|
|
147
|
+
// });
|
|
148
|
+
// it('should record timeout as failure in circuit breaker', async () => {
|
|
149
|
+
// const longRequest = vi.fn().mockImplementation(
|
|
150
|
+
// () =>
|
|
151
|
+
// new Promise((resolve) => {
|
|
152
|
+
// setTimeout(() => resolve('too-late'), 2000);
|
|
153
|
+
// }),
|
|
154
|
+
// );
|
|
155
|
+
// await expect(
|
|
156
|
+
// wrapper.execute(longRequest, leaderAddress),
|
|
157
|
+
// ).rejects.toThrow();
|
|
158
|
+
// const stats = wrapper.getCircuitBreakerStats();
|
|
159
|
+
// expect(stats.leader.totalFailures).toBeGreaterThan(0);
|
|
160
|
+
// });
|
|
161
|
+
// });
|
|
162
|
+
// describe('Configuration', () => {
|
|
163
|
+
// it('should bypass retry when disabled', async () => {
|
|
164
|
+
// const disabledWrapper = new LeaderRequestWrapper({
|
|
165
|
+
// enabled: false,
|
|
166
|
+
// maxAttempts: 5,
|
|
167
|
+
// baseDelayMs: 10,
|
|
168
|
+
// maxDelayMs: 100,
|
|
169
|
+
// timeoutMs: 1000,
|
|
170
|
+
// });
|
|
171
|
+
// const mockRequest = vi.fn().mockRejectedValue(new Error('Fail'));
|
|
172
|
+
// await expect(
|
|
173
|
+
// disabledWrapper.execute(mockRequest, leaderAddress),
|
|
174
|
+
// ).rejects.toThrow('Fail');
|
|
175
|
+
// // Should only call once (no retry)
|
|
176
|
+
// expect(mockRequest).toHaveBeenCalledTimes(1);
|
|
177
|
+
// });
|
|
178
|
+
// it('should allow circuit breaker to be disabled', async () => {
|
|
179
|
+
// const noBreakerWrapper = new LeaderRequestWrapper({
|
|
180
|
+
// enabled: true,
|
|
181
|
+
// maxAttempts: 3,
|
|
182
|
+
// baseDelayMs: 10,
|
|
183
|
+
// maxDelayMs: 100,
|
|
184
|
+
// timeoutMs: 1000,
|
|
185
|
+
// circuitBreaker: {
|
|
186
|
+
// enabled: false,
|
|
187
|
+
// failureThreshold: 3,
|
|
188
|
+
// openTimeoutMs: 500,
|
|
189
|
+
// halfOpenMaxAttempts: 1,
|
|
190
|
+
// },
|
|
191
|
+
// });
|
|
192
|
+
// let callCount = 0;
|
|
193
|
+
// const mockRequest = vi.fn().mockImplementation(() => {
|
|
194
|
+
// callCount++;
|
|
195
|
+
// throw new Error('Fail');
|
|
196
|
+
// });
|
|
197
|
+
// await expect(
|
|
198
|
+
// noBreakerWrapper.execute(mockRequest, leaderAddress),
|
|
199
|
+
// ).rejects.toThrow();
|
|
200
|
+
// // Should retry all attempts even with failures
|
|
201
|
+
// expect(callCount).toBe(3);
|
|
202
|
+
// });
|
|
203
|
+
// });
|
|
204
|
+
// describe('Manual Reset', () => {
|
|
205
|
+
// it('should reset circuit breakers manually', async () => {
|
|
206
|
+
// const mockRequest = vi.fn().mockRejectedValue(new Error('Service down'));
|
|
207
|
+
// // Open circuit
|
|
208
|
+
// await expect(
|
|
209
|
+
// wrapper.execute(mockRequest, leaderAddress),
|
|
210
|
+
// ).rejects.toThrow();
|
|
211
|
+
// expect(wrapper.getCircuitBreakerStats().leader.state).toBe(
|
|
212
|
+
// CircuitState.OPEN,
|
|
213
|
+
// );
|
|
214
|
+
// // Reset
|
|
215
|
+
// wrapper.resetCircuitBreakers();
|
|
216
|
+
// expect(wrapper.getCircuitBreakerStats().leader.state).toBe(
|
|
217
|
+
// CircuitState.CLOSED,
|
|
218
|
+
// );
|
|
219
|
+
// expect(wrapper.getCircuitBreakerStats().registry.state).toBe(
|
|
220
|
+
// CircuitState.CLOSED,
|
|
221
|
+
// );
|
|
222
|
+
// });
|
|
223
|
+
// });
|
|
224
|
+
// describe('Registry Address', () => {
|
|
225
|
+
// it('should use separate circuit breaker for registry', async () => {
|
|
226
|
+
// const mockRequest = vi.fn().mockRejectedValue(new Error('Registry down'));
|
|
227
|
+
// await expect(
|
|
228
|
+
// wrapper.execute(mockRequest, registryAddress),
|
|
229
|
+
// ).rejects.toThrow();
|
|
230
|
+
// const stats = wrapper.getCircuitBreakerStats();
|
|
231
|
+
// expect(stats.registry.totalFailures).toBeGreaterThan(0);
|
|
232
|
+
// expect(stats.registry.state).toBe(CircuitState.OPEN);
|
|
233
|
+
// expect(stats.leader.state).toBe(CircuitState.CLOSED);
|
|
234
|
+
// });
|
|
235
|
+
// });
|
|
236
|
+
// describe('Statistics', () => {
|
|
237
|
+
// it('should provide circuit breaker statistics', () => {
|
|
238
|
+
// const stats = wrapper.getCircuitBreakerStats();
|
|
239
|
+
// expect(stats).toHaveProperty('leader');
|
|
240
|
+
// expect(stats).toHaveProperty('registry');
|
|
241
|
+
// expect(stats.leader).toHaveProperty('state');
|
|
242
|
+
// expect(stats.leader).toHaveProperty('consecutiveFailures');
|
|
243
|
+
// expect(stats.registry).toHaveProperty('state');
|
|
244
|
+
// });
|
|
245
|
+
// });
|
|
246
|
+
// });
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@olane/o-node",
|
|
3
|
-
"version": "0.7.12-alpha.
|
|
3
|
+
"version": "0.7.12-alpha.12",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "dist/src/index.js",
|
|
6
6
|
"types": "dist/src/index.d.ts",
|
|
@@ -54,12 +54,12 @@
|
|
|
54
54
|
"typescript": "5.4.5"
|
|
55
55
|
},
|
|
56
56
|
"dependencies": {
|
|
57
|
-
"@olane/o-config": "0.7.12-alpha.
|
|
58
|
-
"@olane/o-core": "0.7.12-alpha.
|
|
59
|
-
"@olane/o-protocol": "0.7.12-alpha.
|
|
60
|
-
"@olane/o-tool": "0.7.12-alpha.
|
|
57
|
+
"@olane/o-config": "0.7.12-alpha.12",
|
|
58
|
+
"@olane/o-core": "0.7.12-alpha.12",
|
|
59
|
+
"@olane/o-protocol": "0.7.12-alpha.12",
|
|
60
|
+
"@olane/o-tool": "0.7.12-alpha.12",
|
|
61
61
|
"debug": "^4.4.1",
|
|
62
62
|
"dotenv": "^16.5.0"
|
|
63
63
|
},
|
|
64
|
-
"gitHead": "
|
|
64
|
+
"gitHead": "590e95c1281ce1df427ba57816cd825c9f52154e"
|
|
65
65
|
}
|