@lodestar/reqresp 1.43.0-dev.6b7eebbf6d → 1.43.0-dev.6f485b1b61
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/ReqResp.d.ts.map +1 -1
- package/lib/ReqResp.js +12 -2
- package/lib/ReqResp.js.map +1 -1
- package/lib/metrics.d.ts +1 -0
- package/lib/metrics.d.ts.map +1 -1
- package/lib/metrics.js +4 -0
- package/lib/metrics.js.map +1 -1
- package/lib/rate_limiter/selfRateLimiter.d.ts +16 -1
- package/lib/rate_limiter/selfRateLimiter.d.ts.map +1 -1
- package/lib/rate_limiter/selfRateLimiter.js +52 -5
- package/lib/rate_limiter/selfRateLimiter.js.map +1 -1
- package/lib/request/errors.d.ts +3 -2
- package/lib/request/errors.d.ts.map +1 -1
- package/lib/request/errors.js +17 -7
- package/lib/request/errors.js.map +1 -1
- package/package.json +7 -7
- package/src/ReqResp.ts +12 -2
- package/src/metrics.ts +4 -0
- package/src/rate_limiter/selfRateLimiter.ts +71 -8
- package/src/request/errors.ts +20 -10
package/lib/ReqResp.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ReqResp.d.ts","sourceRoot":"","sources":["../src/ReqResp.ts"],"names":[],"mappings":"AACA,OAAO,EAAa,MAAM,EAAS,MAAM,mBAAmB,CAAC;AAC7D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAC,MAAM,EAAE,oBAAoB,EAAC,MAAM,iBAAiB,CAAC;AAC7D,OAAO,EAAC,OAAO,EAAa,MAAM,cAAc,CAAC;AAGjD,OAAO,EAAC,YAAY,EAAoB,eAAe,EAAc,MAAM,oBAAoB,CAAC;AAEhG,OAAO,EACL,gBAAgB,EAChB,QAAQ,EACR,aAAa,EACb,QAAQ,EACR,kBAAkB,EAClB,sBAAsB,EACtB,gBAAgB,EACjB,MAAM,YAAY,CAAC;AAGpB,KAAK,UAAU,GAAG,MAAM,CAAC;AAEzB,eAAO,MAAM,uBAAuB,2BAA2B,CAAC;AAEhE,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,oBAAoB,GAAG,IAAI,CAAC;CAC9C;AAED,MAAM,WAAW,WAAY,SAAQ,eAAe,EAAE,sBAAsB;IAC1E,6EAA6E;IAC7E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kBAAkB,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;CACjD;AAED;;;;;GAKG;AACH,qBAAa,OAAO;IAsBhB,OAAO,CAAC,QAAQ,CAAC,IAAI;IApBvB,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAClC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAClC,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IAG3C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;IAEjD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAElD,OAAO,CAAC,UAAU,CAAyB;IAC3C,2DAA2D;IAC3D,OAAO,CAAC,QAAQ,CAAK;IACrB,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IAExC,2DAA2D;IAC3D,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAwC;IAC5E,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAkC;IAEpE,YACE,OAAO,EAAE,sBAAsB,EACd,IAAI,GAAE,WAAgB,
|
|
1
|
+
{"version":3,"file":"ReqResp.d.ts","sourceRoot":"","sources":["../src/ReqResp.ts"],"names":[],"mappings":"AACA,OAAO,EAAa,MAAM,EAAS,MAAM,mBAAmB,CAAC;AAC7D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAC,MAAM,EAAE,oBAAoB,EAAC,MAAM,iBAAiB,CAAC;AAC7D,OAAO,EAAC,OAAO,EAAa,MAAM,cAAc,CAAC;AAGjD,OAAO,EAAC,YAAY,EAAoB,eAAe,EAAc,MAAM,oBAAoB,CAAC;AAEhG,OAAO,EACL,gBAAgB,EAChB,QAAQ,EACR,aAAa,EACb,QAAQ,EACR,kBAAkB,EAClB,sBAAsB,EACtB,gBAAgB,EACjB,MAAM,YAAY,CAAC;AAGpB,KAAK,UAAU,GAAG,MAAM,CAAC;AAEzB,eAAO,MAAM,uBAAuB,2BAA2B,CAAC;AAEhE,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,oBAAoB,GAAG,IAAI,CAAC;CAC9C;AAED,MAAM,WAAW,WAAY,SAAQ,eAAe,EAAE,sBAAsB;IAC1E,6EAA6E;IAC7E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kBAAkB,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;CACjD;AAED;;;;;GAKG;AACH,qBAAa,OAAO;IAsBhB,OAAO,CAAC,QAAQ,CAAC,IAAI;IApBvB,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAClC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAClC,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IAG3C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;IAEjD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAElD,OAAO,CAAC,UAAU,CAAyB;IAC3C,2DAA2D;IAC3D,OAAO,CAAC,QAAQ,CAAK;IACrB,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IAExC,2DAA2D;IAC3D,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAwC;IAC5E,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAkC;IAEpE,YACE,OAAO,EAAE,sBAAsB,EACd,IAAI,GAAE,WAAgB,EAexC;IAED;;;;;OAKG;IACH,wBAAwB,CAAC,QAAQ,EAAE,gBAAgB,GAAG,IAAI,CAKzD;IAED;;;;;OAKG;IACG,gBAAgB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAYxD;IAED;;;;;OAKG;IACG,kBAAkB,CAAC,UAAU,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAI9D;IAED;;OAEG;IACG,sBAAsB,IAAI,OAAO,CAAC,IAAI,CAAC,CAI5C;IAED,sBAAsB,IAAI,UAAU,EAAE,CAErC;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAO3B;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAI1B;IAGM,WAAW,CAChB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAAE,EAClB,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,UAAU,GACf,aAAa,CAAC,gBAAgB,CAAC,CAqEjC;IAED,OAAO,CAAC,iBAAiB;IA6CzB,SAAS,CAAC,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,kBAAkB,GAAG,IAAI,CAEhF;IAED,SAAS,CAAC,sBAAsB,CAAC,SAAS,EAAE,kBAAkB,EAAE,MAAM,EAAE,YAAY,GAAG,IAAI,CAE1F;IAED,SAAS,CAAC,sBAAsB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,GAAG,IAAI,CAE7F;IAED;;;;;OAKG;IACH,SAAS,CAAC,gBAAgB,CAAC,QAAQ,EAAE,IAAI,CAAC,aAAa,EAAE,QAAQ,GAAG,SAAS,GAAG,UAAU,CAAC,GAAG,MAAM,CAEnG;CACF"}
|
package/lib/ReqResp.js
CHANGED
|
@@ -40,6 +40,9 @@ export class ReqResp {
|
|
|
40
40
|
this.metrics?.selfRateLimiterPeerCount.addCollect(() => {
|
|
41
41
|
this.metrics?.selfRateLimiterPeerCount.set(this.selfRateLimiter.getPeerCount());
|
|
42
42
|
});
|
|
43
|
+
this.metrics?.selfRateLimiterRateLimitedPeerCount.addCollect(() => {
|
|
44
|
+
this.metrics?.selfRateLimiterRateLimitedPeerCount.set(this.selfRateLimiter.getRateLimitedPeerCount());
|
|
45
|
+
});
|
|
43
46
|
}
|
|
44
47
|
/**
|
|
45
48
|
* Register protocol which will be used only to dial to other peers
|
|
@@ -118,10 +121,12 @@ export class ReqResp {
|
|
|
118
121
|
if (!protocol) {
|
|
119
122
|
throw Error(`Request to send to protocol ${protocolID} but it has not been declared`);
|
|
120
123
|
}
|
|
121
|
-
|
|
124
|
+
const allows = this.selfRateLimiter.allows(peerIdStr, protocolID, requestId);
|
|
125
|
+
if (allows !== true) {
|
|
122
126
|
// we technically don't send request in this case but would be nice just to track this in the same `outgoingErrorReasons` metric
|
|
123
127
|
this.metrics?.outgoingErrorReasons.inc({ reason: RequestErrorCode.REQUEST_SELF_RATE_LIMITED });
|
|
124
|
-
|
|
128
|
+
const rateLimitedUntilMs = typeof allows === "number" ? allows : undefined;
|
|
129
|
+
throw new RequestError({ code: RequestErrorCode.REQUEST_SELF_RATE_LIMITED, rateLimitedUntilMs });
|
|
125
130
|
// don't call this.onOutgoingRequestError() to penalize peer
|
|
126
131
|
}
|
|
127
132
|
protocols.push(protocol);
|
|
@@ -138,6 +143,11 @@ export class ReqResp {
|
|
|
138
143
|
if (e.type.code === RequestErrorCode.DIAL_ERROR || e.type.code === RequestErrorCode.DIAL_TIMEOUT) {
|
|
139
144
|
this.metrics?.dialErrors.inc();
|
|
140
145
|
}
|
|
146
|
+
if (e.type.code === RequestErrorCode.RESP_RATE_LIMITED) {
|
|
147
|
+
for (const protocolID of protocolIDs) {
|
|
148
|
+
this.selfRateLimiter.onRateLimited(peerIdStr, protocolID, e.type.rateLimitedUntilMs);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
141
151
|
this.metrics?.outgoingErrorReasons.inc({ reason: e.type.code });
|
|
142
152
|
this.onOutgoingRequestError(peerId, method, e);
|
|
143
153
|
}
|
package/lib/ReqResp.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ReqResp.js","sourceRoot":"","sources":["../src/ReqResp.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,eAAe,EAAC,MAAM,aAAa,CAAC;AAI5C,OAAO,EAAU,UAAU,EAAC,MAAM,cAAc,CAAC;AACjD,OAAO,EAAC,kBAAkB,EAAC,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAC,eAAe,EAAC,MAAM,mCAAmC,CAAC;AAClE,OAAO,EAAC,YAAY,EAAE,gBAAgB,EAAmB,WAAW,EAAC,MAAM,oBAAoB,CAAC;AAChG,OAAO,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AAUlD,OAAO,EAAC,gBAAgB,EAAC,MAAM,uBAAuB,CAAC;AAIvD,MAAM,CAAC,MAAM,uBAAuB,GAAG,wBAAwB,CAAC;AAchE;;;;;GAKG;AACH,MAAM,OAAO,OAAO;IAsBC,IAAI;IArBvB,4CAA4C;IACzB,MAAM,CAAS;IACf,MAAM,CAAS;IACf,OAAO,CAAiB;IAE3C,4EAA4E;IAC3D,WAAW,CAAqB;IACjD,6CAA6C;IAC5B,eAAe,CAAkB;IAE1C,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IAC3C,2DAA2D;IACnD,QAAQ,GAAG,CAAC,CAAC;IACJ,cAAc,CAAS;IAExC,2DAA2D;IAC1C,mBAAmB,GAAG,IAAI,GAAG,EAA6B,CAAC;IAC3D,iBAAiB,GAAG,IAAI,GAAG,EAAuB,CAAC;IAEpE,YACE,OAA+B,EACd,IAAI,GAAgB,EAAE,EACvC;oBADiB,IAAI;QAErB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACpF,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,IAAI,uBAAuB,CAAC;QACrE,IAAI,CAAC,WAAW,GAAG,IAAI,kBAAkB,CAAC,IAAI,CAAC,CAAC;QAChD,IAAI,CAAC,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;QAE7C,IAAI,CAAC,OAAO,EAAE,wBAAwB,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;YACtD,IAAI,CAAC,OAAO,EAAE,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,CAAC,CAAC;QAAA,CACjF,CAAC,CAAC;IAAA,CACJ;IAED;;;;;OAKG;IACH,wBAAwB,CAAC,QAA0B,EAAQ;QACzD,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAEnD,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QACnD,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IAAA,CAC9C;IAED;;;;;OAKG;IACH,KAAK,CAAC,gBAAgB,CAAC,QAAkB,EAAiB;QACxD,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QACnD,MAAM,EAAC,OAAO,EAAE,QAAQ,EAAE,iBAAiB,EAAE,GAAG,IAAI,EAAC,GAAG,QAAQ,CAAC;QAEjE,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAE9C,IAAI,iBAAiB,EAAE,CAAC;YACtB,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAC;QAChE,CAAC;QAED,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,UAAU,CAAC,EAAE,EAAC,KAAK,EAAE,IAAI,EAAC,CAAC,CAAC;IAAA,CACpG;IAED;;;;;OAKG;IACH,KAAK,CAAC,kBAAkB,CAAC,UAAsB,EAAiB;QAC9D,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAE5C,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;IAAA,CACzC;IAED;;OAEG;IACH,KAAK,CAAC,sBAAsB,GAAkB;QAC5C,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,EAAE,CAAC;YACzD,MAAM,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;QAC5C,CAAC;IAAA,CACF;IAED,sBAAsB,GAAiB;QACrC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC;IAAA,CACzG;IAED,KAAK,CAAC,KAAK,GAAkB;QAC3B,IAAI,CAAC,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACxC,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC7B,8FAA8F;QAC9F,oDAAoD;QACpD,eAAe,CAAC,QAAQ,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IAAA,CACnD;IAED,KAAK,CAAC,IAAI,GAAkB;QAC1B,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;QACxB,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IAAA,CACzB;IAED,oCAAoC;IACpC,KAAK,CAAC,CAAC,WAAW,CAChB,MAAc,EACd,MAAc,EACd,QAAkB,EAClB,QAAkB,EAClB,IAAgB,EACiB;QACjC,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;QACpC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,OAAO,EAAE,gBAAgB,CAAC,GAAG,CAAC,EAAC,MAAM,EAAC,CAAC,CAAC;QAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,4BAA4B,CAAC,UAAU,CAAC,EAAC,MAAM,EAAC,CAAC,CAAC;QAE9E,MAAM,SAAS,GAAyC,EAAE,CAAC;QAC3D,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,kEAAkE;QAClE,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;QAEpC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,EAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAC,CAAC,CAAC;YACtE,MAAM,QAAQ,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAC1D,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,KAAK,CAAC,+BAA+B,UAAU,+BAA+B,CAAC,CAAC;YACxF,CAAC;YAED,
|
|
1
|
+
{"version":3,"file":"ReqResp.js","sourceRoot":"","sources":["../src/ReqResp.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,eAAe,EAAC,MAAM,aAAa,CAAC;AAI5C,OAAO,EAAU,UAAU,EAAC,MAAM,cAAc,CAAC;AACjD,OAAO,EAAC,kBAAkB,EAAC,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAC,eAAe,EAAC,MAAM,mCAAmC,CAAC;AAClE,OAAO,EAAC,YAAY,EAAE,gBAAgB,EAAmB,WAAW,EAAC,MAAM,oBAAoB,CAAC;AAChG,OAAO,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AAUlD,OAAO,EAAC,gBAAgB,EAAC,MAAM,uBAAuB,CAAC;AAIvD,MAAM,CAAC,MAAM,uBAAuB,GAAG,wBAAwB,CAAC;AAchE;;;;;GAKG;AACH,MAAM,OAAO,OAAO;IAsBC,IAAI;IArBvB,4CAA4C;IACzB,MAAM,CAAS;IACf,MAAM,CAAS;IACf,OAAO,CAAiB;IAE3C,4EAA4E;IAC3D,WAAW,CAAqB;IACjD,6CAA6C;IAC5B,eAAe,CAAkB;IAE1C,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IAC3C,2DAA2D;IACnD,QAAQ,GAAG,CAAC,CAAC;IACJ,cAAc,CAAS;IAExC,2DAA2D;IAC1C,mBAAmB,GAAG,IAAI,GAAG,EAA6B,CAAC;IAC3D,iBAAiB,GAAG,IAAI,GAAG,EAAuB,CAAC;IAEpE,YACE,OAA+B,EACd,IAAI,GAAgB,EAAE,EACvC;oBADiB,IAAI;QAErB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACpF,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,IAAI,uBAAuB,CAAC;QACrE,IAAI,CAAC,WAAW,GAAG,IAAI,kBAAkB,CAAC,IAAI,CAAC,CAAC;QAChD,IAAI,CAAC,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;QAE7C,IAAI,CAAC,OAAO,EAAE,wBAAwB,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;YACtD,IAAI,CAAC,OAAO,EAAE,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,CAAC,CAAC;QAAA,CACjF,CAAC,CAAC;QACH,IAAI,CAAC,OAAO,EAAE,mCAAmC,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;YACjE,IAAI,CAAC,OAAO,EAAE,mCAAmC,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,uBAAuB,EAAE,CAAC,CAAC;QAAA,CACvG,CAAC,CAAC;IAAA,CACJ;IAED;;;;;OAKG;IACH,wBAAwB,CAAC,QAA0B,EAAQ;QACzD,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAEnD,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QACnD,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IAAA,CAC9C;IAED;;;;;OAKG;IACH,KAAK,CAAC,gBAAgB,CAAC,QAAkB,EAAiB;QACxD,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QACnD,MAAM,EAAC,OAAO,EAAE,QAAQ,EAAE,iBAAiB,EAAE,GAAG,IAAI,EAAC,GAAG,QAAQ,CAAC;QAEjE,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAE9C,IAAI,iBAAiB,EAAE,CAAC;YACtB,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAC;QAChE,CAAC;QAED,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,UAAU,CAAC,EAAE,EAAC,KAAK,EAAE,IAAI,EAAC,CAAC,CAAC;IAAA,CACpG;IAED;;;;;OAKG;IACH,KAAK,CAAC,kBAAkB,CAAC,UAAsB,EAAiB;QAC9D,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAE5C,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;IAAA,CACzC;IAED;;OAEG;IACH,KAAK,CAAC,sBAAsB,GAAkB;QAC5C,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,EAAE,CAAC;YACzD,MAAM,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;QAC5C,CAAC;IAAA,CACF;IAED,sBAAsB,GAAiB;QACrC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC;IAAA,CACzG;IAED,KAAK,CAAC,KAAK,GAAkB;QAC3B,IAAI,CAAC,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACxC,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC7B,8FAA8F;QAC9F,oDAAoD;QACpD,eAAe,CAAC,QAAQ,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IAAA,CACnD;IAED,KAAK,CAAC,IAAI,GAAkB;QAC1B,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;QACxB,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IAAA,CACzB;IAED,oCAAoC;IACpC,KAAK,CAAC,CAAC,WAAW,CAChB,MAAc,EACd,MAAc,EACd,QAAkB,EAClB,QAAkB,EAClB,IAAgB,EACiB;QACjC,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;QACpC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,OAAO,EAAE,gBAAgB,CAAC,GAAG,CAAC,EAAC,MAAM,EAAC,CAAC,CAAC;QAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,4BAA4B,CAAC,UAAU,CAAC,EAAC,MAAM,EAAC,CAAC,CAAC;QAE9E,MAAM,SAAS,GAAyC,EAAE,CAAC;QAC3D,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,kEAAkE;QAClE,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;QAEpC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,EAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAC,CAAC,CAAC;YACtE,MAAM,QAAQ,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAC1D,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,KAAK,CAAC,+BAA+B,UAAU,+BAA+B,CAAC,CAAC;YACxF,CAAC;YAED,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,SAAS,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;YAC7E,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;gBACpB,gIAAgI;gBAChI,IAAI,CAAC,OAAO,EAAE,oBAAoB,CAAC,GAAG,CAAC,EAAC,MAAM,EAAE,gBAAgB,CAAC,yBAAyB,EAAC,CAAC,CAAC;gBAC7F,MAAM,kBAAkB,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;gBAC3E,MAAM,IAAI,YAAY,CAAC,EAAC,IAAI,EAAE,gBAAgB,CAAC,yBAAyB,EAAE,kBAAkB,EAAC,CAAC,CAAC;gBAC/F,4DAA4D;YAC9D,CAAC;YAED,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACzB,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC/B,CAAC;QAED,wCAAwC;QACxC,IAAI,CAAC,QAAQ,EAAE,CAAC;QAEhB,IAAI,CAAC;YACH,KAAK,CAAC,CAAC,WAAW,CAChB,EAAC,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,UAAU,EAAC,EAC7E,MAAM,EACN,SAAS,EACT,WAAW,EACX,IAAI,EACJ,IAAI,CAAC,UAAU,CAAC,MAAM,EACtB,IAAI,CAAC,IAAI,EACT,SAAS,CACV,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,GAAG,CAAC,EAAC,MAAM,EAAC,CAAC,CAAC;YAE3C,IAAI,CAAC,YAAY,YAAY,EAAE,CAAC;gBAC9B,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,gBAAgB,CAAC,UAAU,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,gBAAgB,CAAC,YAAY,EAAE,CAAC;oBACjG,IAAI,CAAC,OAAO,EAAE,UAAU,CAAC,GAAG,EAAE,CAAC;gBACjC,CAAC;gBACD,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,gBAAgB,CAAC,iBAAiB,EAAE,CAAC;oBACvD,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;wBACrC,IAAI,CAAC,eAAe,CAAC,aAAa,CAAC,SAAS,EAAE,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;oBACvF,CAAC;gBACH,CAAC;gBACD,IAAI,CAAC,OAAO,EAAE,oBAAoB,CAAC,GAAG,CAAC,EAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,EAAC,CAAC,CAAC;gBAE9D,IAAI,CAAC,sBAAsB,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;YACjD,CAAC;YAED,MAAM,CAAC,CAAC;QACV,CAAC;gBAAS,CAAC;YACT,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;gBACrC,IAAI,CAAC,eAAe,CAAC,gBAAgB,CAAC,SAAS,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;YAC1E,CAAC;YACD,KAAK,EAAE,EAAE,CAAC;QACZ,CAAC;IAAA,CACF;IAEO,iBAAiB,CAAC,QAAuB,EAAE,UAAkB,EAAE;QACrE,OAAO,KAAK,EAAE,MAAc,EAAE,UAAsB,EAAE,EAAE,CAAC;YACvD,IAAI,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC3C,MAAM,IAAI,KAAK,CAAC,2CAA2C,UAAU,GAAG,CAAC,CAAC;YAC5E,CAAC;YAED,MAAM,MAAM,GAAG,UAAU,CAAC,UAAU,CAAC;YACrC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;YACrE,MAAM,EAAC,MAAM,EAAC,GAAG,QAAQ,CAAC;YAE1B,IAAI,CAAC,OAAO,EAAE,gBAAgB,CAAC,GAAG,CAAC,EAAC,MAAM,EAAC,CAAC,CAAC;YAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,0BAA0B,CAAC,UAAU,CAAC,EAAC,MAAM,EAAC,CAAC,CAAC;YAE5E,IAAI,CAAC,iBAAiB,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAE3C,IAAI,CAAC;gBACH,MAAM,aAAa,CAAC;oBAClB,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,OAAO,EAAE,IAAI,CAAC,OAAO;oBACrB,MAAM;oBACN,MAAM;oBACN,QAAQ,EAAE,QAAoB;oBAC9B,UAAU;oBACV,WAAW,EAAE,IAAI,CAAC,WAAW;oBAC7B,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM;oBAC9B,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE;oBAC1B,UAAU;oBACV,gBAAgB,EAAE,IAAI,CAAC,IAAI,CAAC,gBAAgB;iBAC7C,CAAC,CAAC;gBACH,qCAAqC;YACvC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,GAAG,CAAC,EAAC,MAAM,EAAC,CAAC,CAAC;gBAE3C,IAAI,GAAG,YAAY,YAAY,EAAE,CAAC;oBAChC,IAAI,CAAC,sBAAsB,CAAC,QAA8B,EAAE,GAAG,CAAC,CAAC;gBACnE,CAAC;gBAED,mCAAmC;gBACnC,gDAAgD;YAClD,CAAC;oBAAS,CAAC;gBACT,KAAK,EAAE,EAAE,CAAC;YACZ,CAAC;QAAA,CACF,CAAC;IAAA,CACH;IAES,iBAAiB,CAAC,OAAe,EAAE,SAA6B,EAAQ;QAChF,WAAW;IADsE,CAElF;IAES,sBAAsB,CAAC,SAA6B,EAAE,MAAoB,EAAQ;QAC1F,WAAW;IADgF,CAE5F;IAES,sBAAsB,CAAC,OAAe,EAAE,OAAe,EAAE,MAAoB,EAAQ;QAC7F,WAAW;IADmF,CAE/F;IAED;;;;;OAKG;IACO,gBAAgB,CAAC,QAAgE,EAAU;QACnG,OAAO,gBAAgB,CAAC,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAAA,CACpG;CACF"}
|
package/lib/metrics.d.ts
CHANGED
|
@@ -40,5 +40,6 @@ export declare function getMetrics(register: MetricsRegisterExtra): {
|
|
|
40
40
|
}>;
|
|
41
41
|
dialErrors: import("@lodestar/utils").GaugeExtra<import("@lodestar/utils").NoLabels>;
|
|
42
42
|
selfRateLimiterPeerCount: import("@lodestar/utils").GaugeExtra<import("@lodestar/utils").NoLabels>;
|
|
43
|
+
selfRateLimiterRateLimitedPeerCount: import("@lodestar/utils").GaugeExtra<import("@lodestar/utils").NoLabels>;
|
|
43
44
|
};
|
|
44
45
|
//# sourceMappingURL=metrics.d.ts.map
|
package/lib/metrics.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,oBAAoB,EAAC,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAC,gBAAgB,EAAC,MAAM,qBAAqB,CAAC;AAErD,MAAM,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,UAAU,CAAC,CAAC;AAEpD;;GAEG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,oBAAoB
|
|
1
|
+
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,oBAAoB,EAAC,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAC,gBAAgB,EAAC,MAAM,qBAAqB,CAAC;AAErD,MAAM,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,UAAU,CAAC,CAAC;AAEpD;;GAEG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA0ExD"}
|
package/lib/metrics.js
CHANGED
|
@@ -69,6 +69,10 @@ export function getMetrics(register) {
|
|
|
69
69
|
name: "beacon_reqresp_self_rate_limiter_peer_count",
|
|
70
70
|
help: "Count of peers tracked by the self rate limiter",
|
|
71
71
|
}),
|
|
72
|
+
selfRateLimiterRateLimitedPeerCount: register.gauge({
|
|
73
|
+
name: "beacon_reqresp_self_rate_limiter_rate_limited_peer_count",
|
|
74
|
+
help: "Count of peers currently in rate-limit backoff",
|
|
75
|
+
}),
|
|
72
76
|
};
|
|
73
77
|
}
|
|
74
78
|
//# sourceMappingURL=metrics.js.map
|
package/lib/metrics.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAKA;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,QAA8B,EAAE;IACzD,qGAAqG;IAErG,OAAO;QACL,gBAAgB,EAAE,QAAQ,CAAC,KAAK,CAAmB;YACjD,IAAI,EAAE,wCAAwC;YAC9C,IAAI,EAAE,uCAAuC;YAC7C,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,qBAAqB,EAAE,QAAQ,CAAC,OAAO,CAAmB;YACxD,IAAI,EAAE,8CAA8C;YACpD,IAAI,EAAE,wCAAwC;YAC9C,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,qBAAqB,EAAE,QAAQ,CAAC,OAAO,CAAmB;YACxD,IAAI,EAAE,8CAA8C;YACpD,IAAI,EAAE,wCAAwC;YAC9C,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,4BAA4B,EAAE,QAAQ,CAAC,SAAS,CAAmB;YACjE,IAAI,EAAE,wDAAwD;YAC9D,IAAI,EAAE,gDAAgD;YACtD,UAAU,EAAE,CAAC,QAAQ,CAAC;YACtB,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC;SAC3C,CAAC;QACF,cAAc,EAAE,QAAQ,CAAC,KAAK,CAAmB;YAC/C,IAAI,EAAE,8CAA8C;YACpD,IAAI,EAAE,8CAA8C;YACpD,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,oBAAoB,EAAE,QAAQ,CAAC,KAAK,CAA6B;YAC/D,IAAI,EAAE,qDAAqD;YAC3D,IAAI,EAAE,+CAA+C;YACrD,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,gBAAgB,EAAE,QAAQ,CAAC,KAAK,CAAmB;YACjD,IAAI,EAAE,wCAAwC;YAC9C,IAAI,EAAE,2CAA2C;YACjD,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,qBAAqB,EAAE,QAAQ,CAAC,OAAO,CAAmB;YACxD,IAAI,EAAE,8CAA8C;YACpD,IAAI,EAAE,iDAAiD;YACvD,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,qBAAqB,EAAE,QAAQ,CAAC,OAAO,CAAmB;YACxD,IAAI,EAAE,8CAA8C;YACpD,IAAI,EAAE,iDAAiD;YACvD,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,0BAA0B,EAAE,QAAQ,CAAC,SAAS,CAAmB;YAC/D,IAAI,EAAE,sDAAsD;YAC5D,IAAI,EAAE,uDAAuD;YAC7D,UAAU,EAAE,CAAC,QAAQ,CAAC;YACtB,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;SACnC,CAAC;QACF,cAAc,EAAE,QAAQ,CAAC,KAAK,CAAmB;YAC/C,IAAI,EAAE,8CAA8C;YACpD,IAAI,EAAE,kDAAkD;YACxD,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,UAAU,EAAE,QAAQ,CAAC,KAAK,CAAC;YACzB,IAAI,EAAE,kCAAkC;YACxC,IAAI,EAAE,yBAAyB;SAChC,CAAC;QACF,wBAAwB,EAAE,QAAQ,CAAC,KAAK,CAAC;YACvC,IAAI,EAAE,6CAA6C;YACnD,IAAI,EAAE,iDAAiD;SACxD,CAAC;KACH,CAAC;AAAA,CACH"}
|
|
1
|
+
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAKA;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,QAA8B,EAAE;IACzD,qGAAqG;IAErG,OAAO;QACL,gBAAgB,EAAE,QAAQ,CAAC,KAAK,CAAmB;YACjD,IAAI,EAAE,wCAAwC;YAC9C,IAAI,EAAE,uCAAuC;YAC7C,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,qBAAqB,EAAE,QAAQ,CAAC,OAAO,CAAmB;YACxD,IAAI,EAAE,8CAA8C;YACpD,IAAI,EAAE,wCAAwC;YAC9C,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,qBAAqB,EAAE,QAAQ,CAAC,OAAO,CAAmB;YACxD,IAAI,EAAE,8CAA8C;YACpD,IAAI,EAAE,wCAAwC;YAC9C,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,4BAA4B,EAAE,QAAQ,CAAC,SAAS,CAAmB;YACjE,IAAI,EAAE,wDAAwD;YAC9D,IAAI,EAAE,gDAAgD;YACtD,UAAU,EAAE,CAAC,QAAQ,CAAC;YACtB,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC;SAC3C,CAAC;QACF,cAAc,EAAE,QAAQ,CAAC,KAAK,CAAmB;YAC/C,IAAI,EAAE,8CAA8C;YACpD,IAAI,EAAE,8CAA8C;YACpD,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,oBAAoB,EAAE,QAAQ,CAAC,KAAK,CAA6B;YAC/D,IAAI,EAAE,qDAAqD;YAC3D,IAAI,EAAE,+CAA+C;YACrD,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,gBAAgB,EAAE,QAAQ,CAAC,KAAK,CAAmB;YACjD,IAAI,EAAE,wCAAwC;YAC9C,IAAI,EAAE,2CAA2C;YACjD,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,qBAAqB,EAAE,QAAQ,CAAC,OAAO,CAAmB;YACxD,IAAI,EAAE,8CAA8C;YACpD,IAAI,EAAE,iDAAiD;YACvD,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,qBAAqB,EAAE,QAAQ,CAAC,OAAO,CAAmB;YACxD,IAAI,EAAE,8CAA8C;YACpD,IAAI,EAAE,iDAAiD;YACvD,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,0BAA0B,EAAE,QAAQ,CAAC,SAAS,CAAmB;YAC/D,IAAI,EAAE,sDAAsD;YAC5D,IAAI,EAAE,uDAAuD;YAC7D,UAAU,EAAE,CAAC,QAAQ,CAAC;YACtB,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;SACnC,CAAC;QACF,cAAc,EAAE,QAAQ,CAAC,KAAK,CAAmB;YAC/C,IAAI,EAAE,8CAA8C;YACpD,IAAI,EAAE,kDAAkD;YACxD,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,UAAU,EAAE,QAAQ,CAAC,KAAK,CAAC;YACzB,IAAI,EAAE,kCAAkC;YACxC,IAAI,EAAE,yBAAyB;SAChC,CAAC;QACF,wBAAwB,EAAE,QAAQ,CAAC,KAAK,CAAC;YACvC,IAAI,EAAE,6CAA6C;YACnD,IAAI,EAAE,iDAAiD;SACxD,CAAC;QACF,mCAAmC,EAAE,QAAQ,CAAC,KAAK,CAAC;YAClD,IAAI,EAAE,0DAA0D;YAChE,IAAI,EAAE,gDAAgD;SACvD,CAAC;KACH,CAAC;AAAA,CACH"}
|
|
@@ -9,10 +9,16 @@ export declare const CHECK_DISCONNECTED_PEERS_INTERVAL_MS: number;
|
|
|
9
9
|
* for example https://github.com/ChainSafe/lodestar/issues/8256
|
|
10
10
|
**/
|
|
11
11
|
export declare const REQUEST_TIMEOUT_MS: number;
|
|
12
|
+
/** Default backoff when a peer rate limits us. */
|
|
13
|
+
export declare const DEFAULT_RATE_LIMIT_BACKOFF_MS = 2000;
|
|
12
14
|
type RequestId = number;
|
|
13
15
|
/**
|
|
14
16
|
* Simple rate limiter that allows a maximum of 2 concurrent requests per protocol per peer.
|
|
17
|
+
* Also tracks when a peer has rate-limited us and enforces a backoff period before allowing new requests.
|
|
15
18
|
* The consumer should either prevent requests from being sent when the limit is reached or handle the case when the request is not allowed.
|
|
19
|
+
*
|
|
20
|
+
* Note: SyncChain maintains its own rate-limit backoff map to avoid assigning batches to backed-off
|
|
21
|
+
* peers. This class acts as the authoritative safety net at the protocol level.
|
|
16
22
|
*/
|
|
17
23
|
export declare class SelfRateLimiter {
|
|
18
24
|
private readonly logger?;
|
|
@@ -29,14 +35,23 @@ export declare class SelfRateLimiter {
|
|
|
29
35
|
stop(): void;
|
|
30
36
|
/**
|
|
31
37
|
* called before we send a request to a peer.
|
|
38
|
+
* Returns one of three outcomes:
|
|
39
|
+
* - if the peer has rate-limited us and we're still in backoff, returns the timestamp until which we should back off,
|
|
40
|
+
* - if we've reached the concurrent request limit, returns false,
|
|
41
|
+
* - otherwise, returns true.
|
|
32
42
|
*/
|
|
33
|
-
allows(peerId: PeerIdStr, protocolId: ProtocolID, requestId: RequestId): boolean;
|
|
43
|
+
allows(peerId: PeerIdStr, protocolId: ProtocolID, requestId: RequestId): number | boolean;
|
|
34
44
|
/**
|
|
35
45
|
* called when a request to a peer is completed, regardless of success or failure.
|
|
36
46
|
* This should NOT be called when the request was not allowed
|
|
37
47
|
*/
|
|
38
48
|
requestCompleted(peerId: PeerIdStr, protocolId: ProtocolID, requestId: RequestId): void;
|
|
49
|
+
/**
|
|
50
|
+
* Called when a peer responds with a rate-limit error, to enforce a backoff period before retrying.
|
|
51
|
+
*/
|
|
52
|
+
onRateLimited(peerId: PeerIdStr, protocolId: ProtocolID, rateLimitedUntilMs?: number): void;
|
|
39
53
|
getPeerCount(): number;
|
|
54
|
+
getRateLimitedPeerCount(): number;
|
|
40
55
|
private checkDisconnectedPeers;
|
|
41
56
|
}
|
|
42
57
|
export {};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"selfRateLimiter.d.ts","sourceRoot":"","sources":["../../src/rate_limiter/selfRateLimiter.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,MAAM,EAAS,MAAM,iBAAiB,CAAC;AAE/C,KAAK,SAAS,GAAG,MAAM,CAAC;AACxB,KAAK,UAAU,GAAG,MAAM,CAAC;AAEzB,yGAAyG;AACzG,eAAO,MAAM,oCAAoC,QAAgB,CAAC;AAKlE;;;;IAII;AACJ,eAAO,MAAM,kBAAkB,QAAY,CAAC;AAE5C,KAAK,SAAS,GAAG,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"selfRateLimiter.d.ts","sourceRoot":"","sources":["../../src/rate_limiter/selfRateLimiter.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,MAAM,EAAS,MAAM,iBAAiB,CAAC;AAE/C,KAAK,SAAS,GAAG,MAAM,CAAC;AACxB,KAAK,UAAU,GAAG,MAAM,CAAC;AAEzB,yGAAyG;AACzG,eAAO,MAAM,oCAAoC,QAAgB,CAAC;AAKlE;;;;IAII;AACJ,eAAO,MAAM,kBAAkB,QAAY,CAAC;AAE5C,kDAAkD;AAClD,eAAO,MAAM,6BAA6B,OAAQ,CAAC;AAEnD,KAAK,SAAS,GAAG,MAAM,CAAC;AASxB;;;;;;;GAOG;AACH,qBAAa,eAAe;IAUd,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;IATpC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAA6D;IACjG;;;QAGI;IACJ,OAAO,CAAC,sBAAsB,CAAsB;IACpD,+CAA+C;IAC/C,OAAO,CAAC,eAAe,CAAyC;IAEhE,YAA6B,MAAM,CAAC,oBAAQ,EAM3C;IAED,KAAK,IAAI,IAAI,CAEZ;IAED,IAAI,IAAI,IAAI,CAKX;IAED;;;;;;OAMG;IACH,MAAM,CAAC,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,GAAG,MAAM,GAAG,OAAO,CAuCxF;IAED;;;OAGG;IACH,gBAAgB,CAAC,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,GAAG,IAAI,CAItF;IAED;;OAEG;IACH,aAAa,CACX,MAAM,EAAE,SAAS,EACjB,UAAU,EAAE,UAAU,EACtB,kBAAkB,SAA6C,GAC9D,IAAI,CAKN;IAED,YAAY,IAAI,MAAM,CAErB;IAED,uBAAuB,IAAI,MAAM,CAWhC;IAED,OAAO,CAAC,sBAAsB;CAgB/B"}
|
|
@@ -10,9 +10,15 @@ const DISCONNECTED_TIMEOUT_MS = 60 * 1000;
|
|
|
10
10
|
* for example https://github.com/ChainSafe/lodestar/issues/8256
|
|
11
11
|
**/
|
|
12
12
|
export const REQUEST_TIMEOUT_MS = 30 * 1000;
|
|
13
|
+
/** Default backoff when a peer rate limits us. */
|
|
14
|
+
export const DEFAULT_RATE_LIMIT_BACKOFF_MS = 2_000;
|
|
13
15
|
/**
|
|
14
16
|
* Simple rate limiter that allows a maximum of 2 concurrent requests per protocol per peer.
|
|
17
|
+
* Also tracks when a peer has rate-limited us and enforces a backoff period before allowing new requests.
|
|
15
18
|
* The consumer should either prevent requests from being sent when the limit is reached or handle the case when the request is not allowed.
|
|
19
|
+
*
|
|
20
|
+
* Note: SyncChain maintains its own rate-limit backoff map to avoid assigning batches to backed-off
|
|
21
|
+
* peers. This class acts as the authoritative safety net at the protocol level.
|
|
16
22
|
*/
|
|
17
23
|
export class SelfRateLimiter {
|
|
18
24
|
logger;
|
|
@@ -26,7 +32,7 @@ export class SelfRateLimiter {
|
|
|
26
32
|
cleanupInterval = undefined;
|
|
27
33
|
constructor(logger) {
|
|
28
34
|
this.logger = logger;
|
|
29
|
-
this.rateLimitersPerPeer = new MapDef(() => new MapDef(() => new Map()));
|
|
35
|
+
this.rateLimitersPerPeer = new MapDef(() => new MapDef(() => ({ activeRequests: new Map(), rateLimitedUntilMs: undefined })));
|
|
30
36
|
this.lastSeenRequestsByPeer = new Map();
|
|
31
37
|
}
|
|
32
38
|
start() {
|
|
@@ -40,17 +46,30 @@ export class SelfRateLimiter {
|
|
|
40
46
|
}
|
|
41
47
|
/**
|
|
42
48
|
* called before we send a request to a peer.
|
|
49
|
+
* Returns one of three outcomes:
|
|
50
|
+
* - if the peer has rate-limited us and we're still in backoff, returns the timestamp until which we should back off,
|
|
51
|
+
* - if we've reached the concurrent request limit, returns false,
|
|
52
|
+
* - otherwise, returns true.
|
|
43
53
|
*/
|
|
44
54
|
allows(peerId, protocolId, requestId) {
|
|
45
55
|
const now = Date.now();
|
|
46
56
|
const peerRateLimiter = this.rateLimitersPerPeer.getOrDefault(peerId);
|
|
47
57
|
const trackedRequests = peerRateLimiter.getOrDefault(protocolId);
|
|
48
58
|
this.lastSeenRequestsByPeer.set(peerId, now);
|
|
59
|
+
// Check if peer has rate-limited us and we're still in backoff
|
|
60
|
+
const rateLimitedUntil = trackedRequests.rateLimitedUntilMs;
|
|
61
|
+
if (rateLimitedUntil !== undefined) {
|
|
62
|
+
if (now < rateLimitedUntil) {
|
|
63
|
+
return rateLimitedUntil;
|
|
64
|
+
}
|
|
65
|
+
// Backoff expired, clean up
|
|
66
|
+
trackedRequests.rateLimitedUntilMs = undefined;
|
|
67
|
+
}
|
|
49
68
|
let inProgressRequests = 0;
|
|
50
|
-
for (const [trackedRequestId, trackedRequestTimeMs] of trackedRequests.entries()) {
|
|
69
|
+
for (const [trackedRequestId, trackedRequestTimeMs] of trackedRequests.activeRequests.entries()) {
|
|
51
70
|
if (trackedRequestTimeMs + REQUEST_TIMEOUT_MS <= now) {
|
|
52
71
|
// request timed out, remove it
|
|
53
|
-
trackedRequests.delete(trackedRequestId);
|
|
72
|
+
trackedRequests.activeRequests.delete(trackedRequestId);
|
|
54
73
|
this.logger?.debug("SelfRateLimiter: request timed out, removing it", {
|
|
55
74
|
requestId: trackedRequestId,
|
|
56
75
|
requestTime: trackedRequestTimeMs,
|
|
@@ -65,7 +84,7 @@ export class SelfRateLimiter {
|
|
|
65
84
|
if (inProgressRequests >= MAX_CONCURRENT_REQUESTS) {
|
|
66
85
|
return false;
|
|
67
86
|
}
|
|
68
|
-
trackedRequests.set(requestId, now);
|
|
87
|
+
trackedRequests.activeRequests.set(requestId, now);
|
|
69
88
|
return true;
|
|
70
89
|
}
|
|
71
90
|
/**
|
|
@@ -75,11 +94,32 @@ export class SelfRateLimiter {
|
|
|
75
94
|
requestCompleted(peerId, protocolId, requestId) {
|
|
76
95
|
const peerRateLimiter = this.rateLimitersPerPeer.getOrDefault(peerId);
|
|
77
96
|
const trackedRequests = peerRateLimiter.getOrDefault(protocolId);
|
|
78
|
-
trackedRequests.delete(requestId);
|
|
97
|
+
trackedRequests.activeRequests.delete(requestId);
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Called when a peer responds with a rate-limit error, to enforce a backoff period before retrying.
|
|
101
|
+
*/
|
|
102
|
+
onRateLimited(peerId, protocolId, rateLimitedUntilMs = Date.now() + DEFAULT_RATE_LIMIT_BACKOFF_MS) {
|
|
103
|
+
const peerRateLimiter = this.rateLimitersPerPeer.getOrDefault(peerId);
|
|
104
|
+
const trackedRequests = peerRateLimiter.getOrDefault(protocolId);
|
|
105
|
+
trackedRequests.rateLimitedUntilMs = rateLimitedUntilMs;
|
|
106
|
+
this.logger?.debug("SelfRateLimiter: peer rate limited us, backing off", { peerId, protocolId });
|
|
79
107
|
}
|
|
80
108
|
getPeerCount() {
|
|
81
109
|
return this.rateLimitersPerPeer.size;
|
|
82
110
|
}
|
|
111
|
+
getRateLimitedPeerCount() {
|
|
112
|
+
let count = 0;
|
|
113
|
+
for (const peerRateLimiter of this.rateLimitersPerPeer.values()) {
|
|
114
|
+
for (const trackedRequests of peerRateLimiter.values()) {
|
|
115
|
+
if (trackedRequests.rateLimitedUntilMs !== undefined) {
|
|
116
|
+
count++;
|
|
117
|
+
break;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
return count;
|
|
122
|
+
}
|
|
83
123
|
checkDisconnectedPeers() {
|
|
84
124
|
const now = Date.now();
|
|
85
125
|
for (const [peerIdStr, lastSeenTime] of this.lastSeenRequestsByPeer.entries()) {
|
|
@@ -88,6 +128,13 @@ export class SelfRateLimiter {
|
|
|
88
128
|
this.lastSeenRequestsByPeer.delete(peerIdStr);
|
|
89
129
|
}
|
|
90
130
|
}
|
|
131
|
+
for (const peerRateLimiter of this.rateLimitersPerPeer.values()) {
|
|
132
|
+
for (const trackedRequests of peerRateLimiter.values()) {
|
|
133
|
+
if (trackedRequests.rateLimitedUntilMs !== undefined && now >= trackedRequests.rateLimitedUntilMs) {
|
|
134
|
+
trackedRequests.rateLimitedUntilMs = undefined;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
91
138
|
}
|
|
92
139
|
}
|
|
93
140
|
//# sourceMappingURL=selfRateLimiter.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"selfRateLimiter.js","sourceRoot":"","sources":["../../src/rate_limiter/selfRateLimiter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,uBAAuB,EAAC,MAAM,kBAAkB,CAAC;AACzD,OAAO,EAAS,MAAM,EAAC,MAAM,iBAAiB,CAAC;AAK/C,yGAAyG;AACzG,MAAM,CAAC,MAAM,oCAAoC,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AAElE,sHAAsH;AACtH,MAAM,uBAAuB,GAAG,EAAE,GAAG,IAAI,CAAC;AAE1C;;;;IAII;AACJ,MAAM,CAAC,MAAM,kBAAkB,GAAG,EAAE,GAAG,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"selfRateLimiter.js","sourceRoot":"","sources":["../../src/rate_limiter/selfRateLimiter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,uBAAuB,EAAC,MAAM,kBAAkB,CAAC;AACzD,OAAO,EAAS,MAAM,EAAC,MAAM,iBAAiB,CAAC;AAK/C,yGAAyG;AACzG,MAAM,CAAC,MAAM,oCAAoC,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AAElE,sHAAsH;AACtH,MAAM,uBAAuB,GAAG,EAAE,GAAG,IAAI,CAAC;AAE1C;;;;IAII;AACJ,MAAM,CAAC,MAAM,kBAAkB,GAAG,EAAE,GAAG,IAAI,CAAC;AAE5C,kDAAkD;AAClD,MAAM,CAAC,MAAM,6BAA6B,GAAG,KAAK,CAAC;AAWnD;;;;;;;GAOG;AACH,MAAM,OAAO,eAAe;IAUG,MAAM;IATlB,mBAAmB,CAA6D;IACjG;;;QAGI;IACI,sBAAsB,CAAsB;IACpD,+CAA+C;IACvC,eAAe,GAA+B,SAAS,CAAC;IAEhE,YAA6B,MAAe,EAAE;sBAAjB,MAAM;QACjC,IAAI,CAAC,mBAAmB,GAAG,IAAI,MAAM,CACnC,GAAG,EAAE,CACH,IAAI,MAAM,CAAkC,GAAG,EAAE,CAAC,CAAC,EAAC,cAAc,EAAE,IAAI,GAAG,EAAE,EAAE,kBAAkB,EAAE,SAAS,EAAC,CAAC,CAAC,CAClH,CAAC;QACF,IAAI,CAAC,sBAAsB,GAAG,IAAI,GAAG,EAAE,CAAC;IAAA,CACzC;IAED,KAAK,GAAS;QACZ,IAAI,CAAC,eAAe,GAAG,WAAW,CAAC,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,oCAAoC,CAAC,CAAC;IAAA,CAClH;IAED,IAAI,GAAS;QACX,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YACvC,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACpC,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;QACnC,CAAC;IAAA,CACF;IAED;;;;;;OAMG;IACH,MAAM,CAAC,MAAiB,EAAE,UAAsB,EAAE,SAAoB,EAAoB;QACxF,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,MAAM,eAAe,GAAG,IAAI,CAAC,mBAAmB,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACtE,MAAM,eAAe,GAAG,eAAe,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;QACjE,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAE7C,+DAA+D;QAC/D,MAAM,gBAAgB,GAAG,eAAe,CAAC,kBAAkB,CAAC;QAC5D,IAAI,gBAAgB,KAAK,SAAS,EAAE,CAAC;YACnC,IAAI,GAAG,GAAG,gBAAgB,EAAE,CAAC;gBAC3B,OAAO,gBAAgB,CAAC;YAC1B,CAAC;YACD,4BAA4B;YAC5B,eAAe,CAAC,kBAAkB,GAAG,SAAS,CAAC;QACjD,CAAC;QAED,IAAI,kBAAkB,GAAG,CAAC,CAAC;QAC3B,KAAK,MAAM,CAAC,gBAAgB,EAAE,oBAAoB,CAAC,IAAI,eAAe,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,CAAC;YAChG,IAAI,oBAAoB,GAAG,kBAAkB,IAAI,GAAG,EAAE,CAAC;gBACrD,+BAA+B;gBAC/B,eAAe,CAAC,cAAc,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;gBACxD,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,iDAAiD,EAAE;oBACpE,SAAS,EAAE,gBAAgB;oBAC3B,WAAW,EAAE,oBAAoB;oBACjC,MAAM;oBACN,UAAU;iBACX,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,kBAAkB,EAAE,CAAC;YACvB,CAAC;QACH,CAAC;QAED,IAAI,kBAAkB,IAAI,uBAAuB,EAAE,CAAC;YAClD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,eAAe,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QACnD,OAAO,IAAI,CAAC;IAAA,CACb;IAED;;;OAGG;IACH,gBAAgB,CAAC,MAAiB,EAAE,UAAsB,EAAE,SAAoB,EAAQ;QACtF,MAAM,eAAe,GAAG,IAAI,CAAC,mBAAmB,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACtE,MAAM,eAAe,GAAG,eAAe,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;QACjE,eAAe,CAAC,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAAA,CAClD;IAED;;OAEG;IACH,aAAa,CACX,MAAiB,EACjB,UAAsB,EACtB,kBAAkB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,6BAA6B,EACzD;QACN,MAAM,eAAe,GAAG,IAAI,CAAC,mBAAmB,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACtE,MAAM,eAAe,GAAG,eAAe,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;QACjE,eAAe,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QACxD,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,oDAAoD,EAAE,EAAC,MAAM,EAAE,UAAU,EAAC,CAAC,CAAC;IAAA,CAChG;IAED,YAAY,GAAW;QACrB,OAAO,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC;IAAA,CACtC;IAED,uBAAuB,GAAW;QAChC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,eAAe,IAAI,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,EAAE,CAAC;YAChE,KAAK,MAAM,eAAe,IAAI,eAAe,CAAC,MAAM,EAAE,EAAE,CAAC;gBACvD,IAAI,eAAe,CAAC,kBAAkB,KAAK,SAAS,EAAE,CAAC;oBACrD,KAAK,EAAE,CAAC;oBACR,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IAAA,CACd;IAEO,sBAAsB,GAAS;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,KAAK,MAAM,CAAC,SAAS,EAAE,YAAY,CAAC,IAAI,IAAI,CAAC,sBAAsB,CAAC,OAAO,EAAE,EAAE,CAAC;YAC9E,IAAI,GAAG,GAAG,YAAY,IAAI,uBAAuB,EAAE,CAAC;gBAClD,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;gBAC3C,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;QACD,KAAK,MAAM,eAAe,IAAI,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,EAAE,CAAC;YAChE,KAAK,MAAM,eAAe,IAAI,eAAe,CAAC,MAAM,EAAE,EAAE,CAAC;gBACvD,IAAI,eAAe,CAAC,kBAAkB,KAAK,SAAS,IAAI,GAAG,IAAI,eAAe,CAAC,kBAAkB,EAAE,CAAC;oBAClG,eAAe,CAAC,kBAAkB,GAAG,SAAS,CAAC;gBACjD,CAAC;YACH,CAAC;QACH,CAAC;IAAA,CACF;CACF"}
|
package/lib/request/errors.d.ts
CHANGED
|
@@ -32,7 +32,7 @@ export declare enum RequestErrorCode {
|
|
|
32
32
|
/** For malformed SSZ (metadata) responses */
|
|
33
33
|
SSZ_OVER_MAX_SIZE = "SSZ_SNAPPY_ERROR_OVER_SSZ_MAX_SIZE"
|
|
34
34
|
}
|
|
35
|
-
type RequestErrorType = {
|
|
35
|
+
export type RequestErrorType = {
|
|
36
36
|
code: RequestErrorCode.INVALID_REQUEST;
|
|
37
37
|
errorMessage: string;
|
|
38
38
|
} | {
|
|
@@ -66,8 +66,10 @@ type RequestErrorType = {
|
|
|
66
66
|
code: RequestErrorCode.REQUEST_RATE_LIMITED;
|
|
67
67
|
} | {
|
|
68
68
|
code: RequestErrorCode.REQUEST_SELF_RATE_LIMITED;
|
|
69
|
+
rateLimitedUntilMs?: number;
|
|
69
70
|
} | {
|
|
70
71
|
code: RequestErrorCode.RESP_RATE_LIMITED;
|
|
72
|
+
rateLimitedUntilMs: number;
|
|
71
73
|
} | {
|
|
72
74
|
code: RequestErrorCode.SSZ_OVER_MAX_SIZE;
|
|
73
75
|
};
|
|
@@ -80,5 +82,4 @@ export declare class RequestError extends LodestarError<RequestErrorType> {
|
|
|
80
82
|
* Parse response status errors into detailed request errors for each status code for easier debugging
|
|
81
83
|
*/
|
|
82
84
|
export declare function responseStatusErrorToRequestError(e: ResponseError): RequestErrorType;
|
|
83
|
-
export {};
|
|
84
85
|
//# sourceMappingURL=errors.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/request/errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAE,mBAAmB,EAAC,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAa,sBAAsB,EAAC,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/request/errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAE,mBAAmB,EAAC,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAa,sBAAsB,EAAC,MAAM,iBAAiB,CAAC;AAEnE,OAAO,EAAC,aAAa,EAAC,MAAM,sBAAsB,CAAC;AAEnD,oBAAY,gBAAgB;IAE1B,4DAA4D;IAC5D,eAAe,kCAAkC;IACjD,oBAAoB,uCAAuC;IAC3D,yDAAyD;IACzD,YAAY,+BAA+B;IAC3C,iEAAiE;IACjE,oBAAoB,+BAA+B;IACnD,wEAAwE;IACxE,oBAAoB,uCAAuC;IAC3D,4DAA4D;IAC5D,YAAY,+BAA+B;IAC3C,uCAAuC;IACvC,UAAU,6BAA6B;IACvC,iEAAiE;IACjE,eAAe,kCAAkC;IACjD,8CAA8C;IAC9C,aAAa,gCAAgC;IAC7C,iDAAiD;IACjD,cAAc,iCAAiC;IAC/C,yCAAyC;IACzC,YAAY,+BAA+B;IAC3C,2BAA2B;IAC3B,oBAAoB,+BAA+B;IACnD,gCAAgC;IAChC,yBAAyB,oCAAoC;IAC7D,4BAA4B;IAC5B,iBAAiB,gCAAgC;IACjD,6CAA6C;IAC7C,iBAAiB,uCAAuC;CACzD;AAED,MAAM,MAAM,gBAAgB,GACxB;IAAC,IAAI,EAAE,gBAAgB,CAAC,eAAe,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAC,GAC9D;IAAC,IAAI,EAAE,gBAAgB,CAAC,oBAAoB,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAC,GACnE;IAAC,IAAI,EAAE,gBAAgB,CAAC,YAAY,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAC,GAC3D;IAAC,IAAI,EAAE,gBAAgB,CAAC,oBAAoB,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAC,GACnE;IAAC,IAAI,EAAE,gBAAgB,CAAC,oBAAoB,CAAC;IAAC,MAAM,EAAE,sBAAsB,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAC,GACnG;IAAC,IAAI,EAAE,gBAAgB,CAAC,YAAY,CAAA;CAAC,GACrC;IAAC,IAAI,EAAE,gBAAgB,CAAC,UAAU,CAAC;IAAC,KAAK,EAAE,KAAK,CAAA;CAAC,GACjD;IAAC,IAAI,EAAE,gBAAgB,CAAC,eAAe,CAAA;CAAC,GACxC;IAAC,IAAI,EAAE,gBAAgB,CAAC,aAAa,CAAC;IAAC,KAAK,EAAE,KAAK,CAAA;CAAC,GACpD;IAAC,IAAI,EAAE,gBAAgB,CAAC,cAAc,CAAA;CAAC,GACvC;IAAC,IAAI,EAAE,gBAAgB,CAAC,YAAY,CAAA;CAAC,GACrC;IAAC,IAAI,EAAE,gBAAgB,CAAC,oBAAoB,CAAA;CAAC,GAC7C;IAAC,IAAI,EAAE,gBAAgB,CAAC,yBAAyB,CAAC;IAAC,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAAC,GAC/E;IAAC,IAAI,EAAE,gBAAgB,CAAC,iBAAiB,CAAC;IAAC,kBAAkB,EAAE,MAAM,CAAA;CAAC,GACtE;IAAC,IAAI,EAAE,gBAAgB,CAAC,iBAAiB,CAAA;CAAC,CAAC;AAE/C,eAAO,MAAM,wBAAwB,iBAAiB,CAAC;AAEvD,qBAAa,YAAa,SAAQ,aAAa,CAAC,gBAAgB,CAAC;IAC/D,YAAY,IAAI,EAAE,gBAAgB,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,EAEnE;IAED,MAAM,CAAC,UAAU,CAAC,GAAG,EAAE,mBAAmB,GAAG,YAAY,CAMxD;CACF;AAED;;GAEG;AACH,wBAAgB,iCAAiC,CAAC,CAAC,EAAE,aAAa,GAAG,gBAAgB,CAkCpF"}
|
package/lib/request/errors.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { LodestarError } from "@lodestar/utils";
|
|
2
2
|
import { RespStatus } from "../interface.js";
|
|
3
|
+
import { DEFAULT_RATE_LIMIT_BACKOFF_MS } from "../rate_limiter/selfRateLimiter.js";
|
|
3
4
|
export { RequestErrorCode };
|
|
4
5
|
var RequestErrorCode;
|
|
5
6
|
(function (RequestErrorCode) {
|
|
@@ -51,16 +52,25 @@ export class RequestError extends LodestarError {
|
|
|
51
52
|
*/
|
|
52
53
|
export function responseStatusErrorToRequestError(e) {
|
|
53
54
|
const { errorMessage, status } = e;
|
|
54
|
-
//
|
|
55
|
-
//
|
|
56
|
-
//
|
|
55
|
+
// Rate limit error detection: clients use different status codes and messages.
|
|
56
|
+
// We match on the error message text because status codes are inconsistent:
|
|
57
|
+
// - Lighthouse/Grandine: status 139 (non-standard RateLimited code)
|
|
58
|
+
// message: "Rate limited. There are already 2 active requests with the same protocol"
|
|
59
|
+
// - Prysm: status 1 (INVALID_REQUEST), message: "rate limited"
|
|
60
|
+
// - Teku: status 1 (INVALID_REQUEST), message: "Peer has been rate limited"
|
|
61
|
+
// - Nimbus: never sends rate limit errors (silently throttles via token bucket)
|
|
62
|
+
// See https://github.com/ChainSafe/lodestar/issues/8065#issuecomment-3157266196
|
|
57
63
|
const errorMessageLowercase = errorMessage.toLowerCase();
|
|
58
64
|
if (errorMessageLowercase.includes("rate limit")) {
|
|
59
|
-
return { code: RequestErrorCode.RESP_RATE_LIMITED };
|
|
65
|
+
return { code: RequestErrorCode.RESP_RATE_LIMITED, rateLimitedUntilMs: Date.now() + DEFAULT_RATE_LIMIT_BACKOFF_MS };
|
|
60
66
|
}
|
|
61
|
-
// Grandine
|
|
62
|
-
|
|
63
|
-
|
|
67
|
+
// Grandine's eth2_libp2p fork uses the old Lighthouse GCRA inbound rate limiter which sends
|
|
68
|
+
// "Wait <Duration>" with an explicit backoff (e.g. "Wait 2.816488536s") using Rust's
|
|
69
|
+
// Debug format for Duration. We only use this as a rate-limit signal; the backoff duration
|
|
70
|
+
// is intentionally not parsed.
|
|
71
|
+
// See https://github.com/ChainSafe/lodestar/issues/8110
|
|
72
|
+
if (errorMessageLowercase.startsWith("wait ")) {
|
|
73
|
+
return { code: RequestErrorCode.RESP_RATE_LIMITED, rateLimitedUntilMs: Date.now() + DEFAULT_RATE_LIMIT_BACKOFF_MS };
|
|
64
74
|
}
|
|
65
75
|
switch (status) {
|
|
66
76
|
case RespStatus.INVALID_REQUEST:
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/request/errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAsB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAC,UAAU,EAAyB,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/request/errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAsB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAC,UAAU,EAAyB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAC,6BAA6B,EAAC,MAAM,oCAAoC,CAAC;SAGrE,gBAAgB;AAA5B,IAAY,gBA+BX;AA/BD,WAAY,gBAAgB;IAC1B,mFAAmF;IACnF,4DAA4D;IAC5D,qEAAiD,CAAA;IACjD,+EAA2D,CAAA;IAC3D,yDAAyD;IACzD,+DAA2C,CAAA;IAC3C,iEAAiE;IACjE,uEAAmD,CAAA;IACnD,wEAAwE;IACxE,+EAA2D,CAAA;IAC3D,4DAA4D;IAC5D,+DAA2C,CAAA;IAC3C,uCAAuC;IACvC,2DAAuC,CAAA;IACvC,iEAAiE;IACjE,qEAAiD,CAAA;IACjD,8CAA8C;IAC9C,iEAA6C,CAAA;IAC7C,iDAAiD;IACjD,mEAA+C,CAAA;IAC/C,yCAAyC;IACzC,+DAA2C,CAAA;IAC3C,2BAA2B;IAC3B,uEAAmD,CAAA;IACnD,gCAAgC;IAChC,iFAA6D,CAAA;IAC7D,4BAA4B;IAC5B,qEAAiD,CAAA;IACjD,6CAA6C;IAC7C,4EAAwD,CAAA;AAAC,CAC3D,EA/BY,gBAAgB,KAAhB,gBAAgB,QA+B3B;AAmBD,MAAM,CAAC,MAAM,wBAAwB,GAAG,cAAc,CAAC;AAEvD,MAAM,OAAO,YAAa,SAAQ,aAA+B;IAC/D,YAAY,IAAsB,EAAE,OAAgB,EAAE,KAAc,EAAE;QACpE,KAAK,CAAC,IAAI,EAAE,OAAO,IAAI,kBAAkB,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC;IAAA,CACzD;IAED,MAAM,CAAC,UAAU,CAAC,GAAwB,EAAgB;QACxD,IAAI,GAAG,CAAC,SAAS,KAAK,cAAc,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,kDAAkD,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;QACrF,CAAC;QAED,OAAO,IAAI,YAAY,CAAC,GAAG,CAAC,IAAwB,EAAE,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;IAAA,CAC/E;CACF;AAED;;GAEG;AACH,MAAM,UAAU,iCAAiC,CAAC,CAAgB,EAAoB;IACpF,MAAM,EAAC,YAAY,EAAE,MAAM,EAAC,GAAG,CAAC,CAAC;IACjC,+EAA+E;IAC/E,4EAA4E;IAC5E,sEAAsE;IACtE,0FAA0F;IAC1F,iEAAiE;IACjE,8EAA8E;IAC9E,kFAAkF;IAClF,gFAAgF;IAChF,MAAM,qBAAqB,GAAG,YAAY,CAAC,WAAW,EAAE,CAAC;IACzD,IAAI,qBAAqB,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;QACjD,OAAO,EAAC,IAAI,EAAE,gBAAgB,CAAC,iBAAiB,EAAE,kBAAkB,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,6BAA6B,EAAC,CAAC;IACpH,CAAC;IAED,4FAA4F;IAC5F,qFAAqF;IACrF,2FAA2F;IAC3F,+BAA+B;IAC/B,wDAAwD;IACxD,IAAI,qBAAqB,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QAC9C,OAAO,EAAC,IAAI,EAAE,gBAAgB,CAAC,iBAAiB,EAAE,kBAAkB,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,6BAA6B,EAAC,CAAC;IACpH,CAAC;IAED,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,UAAU,CAAC,eAAe;YAC7B,OAAO,EAAC,IAAI,EAAE,gBAAgB,CAAC,eAAe,EAAE,YAAY,EAAC,CAAC;QAChE,KAAK,UAAU,CAAC,YAAY;YAC1B,OAAO,EAAC,IAAI,EAAE,gBAAgB,CAAC,YAAY,EAAE,YAAY,EAAC,CAAC;QAC7D,KAAK,UAAU,CAAC,oBAAoB;YAClC,OAAO,EAAC,IAAI,EAAE,gBAAgB,CAAC,oBAAoB,EAAE,YAAY,EAAC,CAAC;QACrE;YACE,OAAO,EAAC,IAAI,EAAE,gBAAgB,CAAC,oBAAoB,EAAE,YAAY,EAAE,MAAM,EAAC,CAAC;IAC/E,CAAC;AAAA,CACF;AAED;;GAEG;AACH,SAAS,kBAAkB,CAAC,IAAsB,EAAsB;IACtE,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,KAAK,gBAAgB,CAAC,eAAe,CAAC;QACtC,KAAK,gBAAgB,CAAC,YAAY,CAAC;QACnC,KAAK,gBAAgB,CAAC,oBAAoB,CAAC;QAC3C,KAAK,gBAAgB,CAAC,oBAAoB;YACxC,OAAO,GAAG,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,YAAY,EAAE,CAAC;QAC9C;YACE,OAAO,IAAI,CAAC,IAAI,CAAC;IACrB,CAAC;AAAA,CACF"}
|
package/package.json
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
"bugs": {
|
|
12
12
|
"url": "https://github.com/ChainSafe/lodestar/issues"
|
|
13
13
|
},
|
|
14
|
-
"version": "1.43.0-dev.
|
|
14
|
+
"version": "1.43.0-dev.6f485b1b61",
|
|
15
15
|
"type": "module",
|
|
16
16
|
"exports": {
|
|
17
17
|
".": {
|
|
@@ -47,9 +47,9 @@
|
|
|
47
47
|
"@chainsafe/fast-crc32c": "^4.2.0",
|
|
48
48
|
"@libp2p/interface": "^3.1.0",
|
|
49
49
|
"@libp2p/utils": "^7.0.13",
|
|
50
|
-
"@lodestar/config": "^1.43.0-dev.
|
|
51
|
-
"@lodestar/params": "^1.43.0-dev.
|
|
52
|
-
"@lodestar/utils": "^1.43.0-dev.
|
|
50
|
+
"@lodestar/config": "^1.43.0-dev.6f485b1b61",
|
|
51
|
+
"@lodestar/params": "^1.43.0-dev.6f485b1b61",
|
|
52
|
+
"@lodestar/utils": "^1.43.0-dev.6f485b1b61",
|
|
53
53
|
"snappy": "^7.2.2",
|
|
54
54
|
"snappyjs": "^0.7.0",
|
|
55
55
|
"uint8-varint": "^2.0.2",
|
|
@@ -60,8 +60,8 @@
|
|
|
60
60
|
"@libp2p/crypto": "^5.1.13",
|
|
61
61
|
"@libp2p/logger": "^6.2.2",
|
|
62
62
|
"@libp2p/peer-id": "^6.0.4",
|
|
63
|
-
"@lodestar/logger": "^1.43.0-dev.
|
|
64
|
-
"@lodestar/types": "^1.43.0-dev.
|
|
63
|
+
"@lodestar/logger": "^1.43.0-dev.6f485b1b61",
|
|
64
|
+
"@lodestar/types": "^1.43.0-dev.6f485b1b61",
|
|
65
65
|
"libp2p": "3.1.6"
|
|
66
66
|
},
|
|
67
67
|
"peerDependencies": {
|
|
@@ -75,5 +75,5 @@
|
|
|
75
75
|
"reqresp",
|
|
76
76
|
"blockchain"
|
|
77
77
|
],
|
|
78
|
-
"gitHead": "
|
|
78
|
+
"gitHead": "1e9b66fb7bfb16af836502c98c05866736d4ffb9"
|
|
79
79
|
}
|
package/src/ReqResp.ts
CHANGED
|
@@ -74,6 +74,9 @@ export class ReqResp {
|
|
|
74
74
|
this.metrics?.selfRateLimiterPeerCount.addCollect(() => {
|
|
75
75
|
this.metrics?.selfRateLimiterPeerCount.set(this.selfRateLimiter.getPeerCount());
|
|
76
76
|
});
|
|
77
|
+
this.metrics?.selfRateLimiterRateLimitedPeerCount.addCollect(() => {
|
|
78
|
+
this.metrics?.selfRateLimiterRateLimitedPeerCount.set(this.selfRateLimiter.getRateLimitedPeerCount());
|
|
79
|
+
});
|
|
77
80
|
}
|
|
78
81
|
|
|
79
82
|
/**
|
|
@@ -174,10 +177,12 @@ export class ReqResp {
|
|
|
174
177
|
throw Error(`Request to send to protocol ${protocolID} but it has not been declared`);
|
|
175
178
|
}
|
|
176
179
|
|
|
177
|
-
|
|
180
|
+
const allows = this.selfRateLimiter.allows(peerIdStr, protocolID, requestId);
|
|
181
|
+
if (allows !== true) {
|
|
178
182
|
// we technically don't send request in this case but would be nice just to track this in the same `outgoingErrorReasons` metric
|
|
179
183
|
this.metrics?.outgoingErrorReasons.inc({reason: RequestErrorCode.REQUEST_SELF_RATE_LIMITED});
|
|
180
|
-
|
|
184
|
+
const rateLimitedUntilMs = typeof allows === "number" ? allows : undefined;
|
|
185
|
+
throw new RequestError({code: RequestErrorCode.REQUEST_SELF_RATE_LIMITED, rateLimitedUntilMs});
|
|
181
186
|
// don't call this.onOutgoingRequestError() to penalize peer
|
|
182
187
|
}
|
|
183
188
|
|
|
@@ -206,6 +211,11 @@ export class ReqResp {
|
|
|
206
211
|
if (e.type.code === RequestErrorCode.DIAL_ERROR || e.type.code === RequestErrorCode.DIAL_TIMEOUT) {
|
|
207
212
|
this.metrics?.dialErrors.inc();
|
|
208
213
|
}
|
|
214
|
+
if (e.type.code === RequestErrorCode.RESP_RATE_LIMITED) {
|
|
215
|
+
for (const protocolID of protocolIDs) {
|
|
216
|
+
this.selfRateLimiter.onRateLimited(peerIdStr, protocolID, e.type.rateLimitedUntilMs);
|
|
217
|
+
}
|
|
218
|
+
}
|
|
209
219
|
this.metrics?.outgoingErrorReasons.inc({reason: e.type.code});
|
|
210
220
|
|
|
211
221
|
this.onOutgoingRequestError(peerId, method, e);
|
package/src/metrics.ts
CHANGED
|
@@ -75,5 +75,9 @@ export function getMetrics(register: MetricsRegisterExtra) {
|
|
|
75
75
|
name: "beacon_reqresp_self_rate_limiter_peer_count",
|
|
76
76
|
help: "Count of peers tracked by the self rate limiter",
|
|
77
77
|
}),
|
|
78
|
+
selfRateLimiterRateLimitedPeerCount: register.gauge({
|
|
79
|
+
name: "beacon_reqresp_self_rate_limiter_rate_limited_peer_count",
|
|
80
|
+
help: "Count of peers currently in rate-limit backoff",
|
|
81
|
+
}),
|
|
78
82
|
};
|
|
79
83
|
}
|
|
@@ -17,15 +17,28 @@ const DISCONNECTED_TIMEOUT_MS = 60 * 1000;
|
|
|
17
17
|
**/
|
|
18
18
|
export const REQUEST_TIMEOUT_MS = 30 * 1000;
|
|
19
19
|
|
|
20
|
+
/** Default backoff when a peer rate limits us. */
|
|
21
|
+
export const DEFAULT_RATE_LIMIT_BACKOFF_MS = 2_000;
|
|
22
|
+
|
|
20
23
|
type RequestId = number;
|
|
21
24
|
type RequestIdMs = number;
|
|
22
25
|
|
|
26
|
+
type ProtocolRateLimiter = {
|
|
27
|
+
activeRequests: Map<RequestId, RequestIdMs>;
|
|
28
|
+
/** Tracks the timestamp (ms) until which we should not send requests to a peer */
|
|
29
|
+
rateLimitedUntilMs?: number;
|
|
30
|
+
};
|
|
31
|
+
|
|
23
32
|
/**
|
|
24
33
|
* Simple rate limiter that allows a maximum of 2 concurrent requests per protocol per peer.
|
|
34
|
+
* Also tracks when a peer has rate-limited us and enforces a backoff period before allowing new requests.
|
|
25
35
|
* The consumer should either prevent requests from being sent when the limit is reached or handle the case when the request is not allowed.
|
|
36
|
+
*
|
|
37
|
+
* Note: SyncChain maintains its own rate-limit backoff map to avoid assigning batches to backed-off
|
|
38
|
+
* peers. This class acts as the authoritative safety net at the protocol level.
|
|
26
39
|
*/
|
|
27
40
|
export class SelfRateLimiter {
|
|
28
|
-
private readonly rateLimitersPerPeer: MapDef<PeerIdStr, MapDef<ProtocolID,
|
|
41
|
+
private readonly rateLimitersPerPeer: MapDef<PeerIdStr, MapDef<ProtocolID, ProtocolRateLimiter>>;
|
|
29
42
|
/**
|
|
30
43
|
* It's not convenient to handle a peer disconnected event so we track the last seen requests by peer.
|
|
31
44
|
* This is the same design to `ReqRespRateLimiter`.
|
|
@@ -35,8 +48,9 @@ export class SelfRateLimiter {
|
|
|
35
48
|
private cleanupInterval: NodeJS.Timeout | undefined = undefined;
|
|
36
49
|
|
|
37
50
|
constructor(private readonly logger?: Logger) {
|
|
38
|
-
this.rateLimitersPerPeer = new MapDef<PeerIdStr, MapDef<ProtocolID,
|
|
39
|
-
() =>
|
|
51
|
+
this.rateLimitersPerPeer = new MapDef<PeerIdStr, MapDef<ProtocolID, ProtocolRateLimiter>>(
|
|
52
|
+
() =>
|
|
53
|
+
new MapDef<ProtocolID, ProtocolRateLimiter>(() => ({activeRequests: new Map(), rateLimitedUntilMs: undefined}))
|
|
40
54
|
);
|
|
41
55
|
this.lastSeenRequestsByPeer = new Map();
|
|
42
56
|
}
|
|
@@ -54,18 +68,33 @@ export class SelfRateLimiter {
|
|
|
54
68
|
|
|
55
69
|
/**
|
|
56
70
|
* called before we send a request to a peer.
|
|
71
|
+
* Returns one of three outcomes:
|
|
72
|
+
* - if the peer has rate-limited us and we're still in backoff, returns the timestamp until which we should back off,
|
|
73
|
+
* - if we've reached the concurrent request limit, returns false,
|
|
74
|
+
* - otherwise, returns true.
|
|
57
75
|
*/
|
|
58
|
-
allows(peerId: PeerIdStr, protocolId: ProtocolID, requestId: RequestId): boolean {
|
|
76
|
+
allows(peerId: PeerIdStr, protocolId: ProtocolID, requestId: RequestId): number | boolean {
|
|
59
77
|
const now = Date.now();
|
|
78
|
+
|
|
60
79
|
const peerRateLimiter = this.rateLimitersPerPeer.getOrDefault(peerId);
|
|
61
80
|
const trackedRequests = peerRateLimiter.getOrDefault(protocolId);
|
|
62
81
|
this.lastSeenRequestsByPeer.set(peerId, now);
|
|
63
82
|
|
|
83
|
+
// Check if peer has rate-limited us and we're still in backoff
|
|
84
|
+
const rateLimitedUntil = trackedRequests.rateLimitedUntilMs;
|
|
85
|
+
if (rateLimitedUntil !== undefined) {
|
|
86
|
+
if (now < rateLimitedUntil) {
|
|
87
|
+
return rateLimitedUntil;
|
|
88
|
+
}
|
|
89
|
+
// Backoff expired, clean up
|
|
90
|
+
trackedRequests.rateLimitedUntilMs = undefined;
|
|
91
|
+
}
|
|
92
|
+
|
|
64
93
|
let inProgressRequests = 0;
|
|
65
|
-
for (const [trackedRequestId, trackedRequestTimeMs] of trackedRequests.entries()) {
|
|
94
|
+
for (const [trackedRequestId, trackedRequestTimeMs] of trackedRequests.activeRequests.entries()) {
|
|
66
95
|
if (trackedRequestTimeMs + REQUEST_TIMEOUT_MS <= now) {
|
|
67
96
|
// request timed out, remove it
|
|
68
|
-
trackedRequests.delete(trackedRequestId);
|
|
97
|
+
trackedRequests.activeRequests.delete(trackedRequestId);
|
|
69
98
|
this.logger?.debug("SelfRateLimiter: request timed out, removing it", {
|
|
70
99
|
requestId: trackedRequestId,
|
|
71
100
|
requestTime: trackedRequestTimeMs,
|
|
@@ -81,7 +110,7 @@ export class SelfRateLimiter {
|
|
|
81
110
|
return false;
|
|
82
111
|
}
|
|
83
112
|
|
|
84
|
-
trackedRequests.set(requestId, now);
|
|
113
|
+
trackedRequests.activeRequests.set(requestId, now);
|
|
85
114
|
return true;
|
|
86
115
|
}
|
|
87
116
|
|
|
@@ -92,13 +121,40 @@ export class SelfRateLimiter {
|
|
|
92
121
|
requestCompleted(peerId: PeerIdStr, protocolId: ProtocolID, requestId: RequestId): void {
|
|
93
122
|
const peerRateLimiter = this.rateLimitersPerPeer.getOrDefault(peerId);
|
|
94
123
|
const trackedRequests = peerRateLimiter.getOrDefault(protocolId);
|
|
95
|
-
trackedRequests.delete(requestId);
|
|
124
|
+
trackedRequests.activeRequests.delete(requestId);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Called when a peer responds with a rate-limit error, to enforce a backoff period before retrying.
|
|
129
|
+
*/
|
|
130
|
+
onRateLimited(
|
|
131
|
+
peerId: PeerIdStr,
|
|
132
|
+
protocolId: ProtocolID,
|
|
133
|
+
rateLimitedUntilMs = Date.now() + DEFAULT_RATE_LIMIT_BACKOFF_MS
|
|
134
|
+
): void {
|
|
135
|
+
const peerRateLimiter = this.rateLimitersPerPeer.getOrDefault(peerId);
|
|
136
|
+
const trackedRequests = peerRateLimiter.getOrDefault(protocolId);
|
|
137
|
+
trackedRequests.rateLimitedUntilMs = rateLimitedUntilMs;
|
|
138
|
+
this.logger?.debug("SelfRateLimiter: peer rate limited us, backing off", {peerId, protocolId});
|
|
96
139
|
}
|
|
97
140
|
|
|
98
141
|
getPeerCount(): number {
|
|
99
142
|
return this.rateLimitersPerPeer.size;
|
|
100
143
|
}
|
|
101
144
|
|
|
145
|
+
getRateLimitedPeerCount(): number {
|
|
146
|
+
let count = 0;
|
|
147
|
+
for (const peerRateLimiter of this.rateLimitersPerPeer.values()) {
|
|
148
|
+
for (const trackedRequests of peerRateLimiter.values()) {
|
|
149
|
+
if (trackedRequests.rateLimitedUntilMs !== undefined) {
|
|
150
|
+
count++;
|
|
151
|
+
break;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
return count;
|
|
156
|
+
}
|
|
157
|
+
|
|
102
158
|
private checkDisconnectedPeers(): void {
|
|
103
159
|
const now = Date.now();
|
|
104
160
|
for (const [peerIdStr, lastSeenTime] of this.lastSeenRequestsByPeer.entries()) {
|
|
@@ -107,5 +163,12 @@ export class SelfRateLimiter {
|
|
|
107
163
|
this.lastSeenRequestsByPeer.delete(peerIdStr);
|
|
108
164
|
}
|
|
109
165
|
}
|
|
166
|
+
for (const peerRateLimiter of this.rateLimitersPerPeer.values()) {
|
|
167
|
+
for (const trackedRequests of peerRateLimiter.values()) {
|
|
168
|
+
if (trackedRequests.rateLimitedUntilMs !== undefined && now >= trackedRequests.rateLimitedUntilMs) {
|
|
169
|
+
trackedRequests.rateLimitedUntilMs = undefined;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
}
|
|
110
173
|
}
|
|
111
174
|
}
|
package/src/request/errors.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import {LodestarError, LodestarErrorObject} from "@lodestar/utils";
|
|
2
2
|
import {RespStatus, RpcResponseStatusError} from "../interface.js";
|
|
3
|
+
import {DEFAULT_RATE_LIMIT_BACKOFF_MS} from "../rate_limiter/selfRateLimiter.js";
|
|
3
4
|
import {ResponseError} from "../response/index.js";
|
|
4
5
|
|
|
5
6
|
export enum RequestErrorCode {
|
|
@@ -35,7 +36,7 @@ export enum RequestErrorCode {
|
|
|
35
36
|
SSZ_OVER_MAX_SIZE = "SSZ_SNAPPY_ERROR_OVER_SSZ_MAX_SIZE",
|
|
36
37
|
}
|
|
37
38
|
|
|
38
|
-
type RequestErrorType =
|
|
39
|
+
export type RequestErrorType =
|
|
39
40
|
| {code: RequestErrorCode.INVALID_REQUEST; errorMessage: string}
|
|
40
41
|
| {code: RequestErrorCode.INVALID_RESPONSE_SSZ; errorMessage: string}
|
|
41
42
|
| {code: RequestErrorCode.SERVER_ERROR; errorMessage: string}
|
|
@@ -48,8 +49,8 @@ type RequestErrorType =
|
|
|
48
49
|
| {code: RequestErrorCode.EMPTY_RESPONSE}
|
|
49
50
|
| {code: RequestErrorCode.RESP_TIMEOUT}
|
|
50
51
|
| {code: RequestErrorCode.REQUEST_RATE_LIMITED}
|
|
51
|
-
| {code: RequestErrorCode.REQUEST_SELF_RATE_LIMITED}
|
|
52
|
-
| {code: RequestErrorCode.RESP_RATE_LIMITED}
|
|
52
|
+
| {code: RequestErrorCode.REQUEST_SELF_RATE_LIMITED; rateLimitedUntilMs?: number}
|
|
53
|
+
| {code: RequestErrorCode.RESP_RATE_LIMITED; rateLimitedUntilMs: number}
|
|
53
54
|
| {code: RequestErrorCode.SSZ_OVER_MAX_SIZE};
|
|
54
55
|
|
|
55
56
|
export const REQUEST_ERROR_CLASS_NAME = "RequestError";
|
|
@@ -73,17 +74,26 @@ export class RequestError extends LodestarError<RequestErrorType> {
|
|
|
73
74
|
*/
|
|
74
75
|
export function responseStatusErrorToRequestError(e: ResponseError): RequestErrorType {
|
|
75
76
|
const {errorMessage, status} = e;
|
|
76
|
-
//
|
|
77
|
-
//
|
|
78
|
-
//
|
|
77
|
+
// Rate limit error detection: clients use different status codes and messages.
|
|
78
|
+
// We match on the error message text because status codes are inconsistent:
|
|
79
|
+
// - Lighthouse/Grandine: status 139 (non-standard RateLimited code)
|
|
80
|
+
// message: "Rate limited. There are already 2 active requests with the same protocol"
|
|
81
|
+
// - Prysm: status 1 (INVALID_REQUEST), message: "rate limited"
|
|
82
|
+
// - Teku: status 1 (INVALID_REQUEST), message: "Peer has been rate limited"
|
|
83
|
+
// - Nimbus: never sends rate limit errors (silently throttles via token bucket)
|
|
84
|
+
// See https://github.com/ChainSafe/lodestar/issues/8065#issuecomment-3157266196
|
|
79
85
|
const errorMessageLowercase = errorMessage.toLowerCase();
|
|
80
86
|
if (errorMessageLowercase.includes("rate limit")) {
|
|
81
|
-
return {code: RequestErrorCode.RESP_RATE_LIMITED};
|
|
87
|
+
return {code: RequestErrorCode.RESP_RATE_LIMITED, rateLimitedUntilMs: Date.now() + DEFAULT_RATE_LIMIT_BACKOFF_MS};
|
|
82
88
|
}
|
|
83
89
|
|
|
84
|
-
// Grandine
|
|
85
|
-
|
|
86
|
-
|
|
90
|
+
// Grandine's eth2_libp2p fork uses the old Lighthouse GCRA inbound rate limiter which sends
|
|
91
|
+
// "Wait <Duration>" with an explicit backoff (e.g. "Wait 2.816488536s") using Rust's
|
|
92
|
+
// Debug format for Duration. We only use this as a rate-limit signal; the backoff duration
|
|
93
|
+
// is intentionally not parsed.
|
|
94
|
+
// See https://github.com/ChainSafe/lodestar/issues/8110
|
|
95
|
+
if (errorMessageLowercase.startsWith("wait ")) {
|
|
96
|
+
return {code: RequestErrorCode.RESP_RATE_LIMITED, rateLimitedUntilMs: Date.now() + DEFAULT_RATE_LIMIT_BACKOFF_MS};
|
|
87
97
|
}
|
|
88
98
|
|
|
89
99
|
switch (status) {
|