@chainlink/external-adapter-framework 0.14.5 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adapter/basic.js +7 -6
- package/adapter/basic.js.map +1 -1
- package/adapter/types.d.ts +5 -4
- package/background-executor.js +4 -4
- package/background-executor.js.map +1 -1
- package/cache/factory.d.ts +4 -1
- package/cache/factory.js +2 -2
- package/cache/factory.js.map +1 -1
- package/cache/local.d.ts +26 -4
- package/cache/local.js +113 -22
- package/cache/local.js.map +1 -1
- package/config/index.d.ts +30 -28
- package/config/index.js +35 -61
- package/config/index.js.map +1 -1
- package/package.json +4 -4
- package/rate-limiting/index.d.ts +2 -15
- package/rate-limiting/index.js +1 -2
- package/rate-limiting/index.js.map +1 -1
- package/rate-limiting/request/simple-counting.d.ts +1 -1
- package/rate-limiting/request/simple-counting.js +12 -8
- package/rate-limiting/request/simple-counting.js.map +1 -1
- package/rate-limiting/simple-counting.d.ts +20 -0
- package/rate-limiting/simple-counting.js +67 -0
- package/rate-limiting/simple-counting.js.map +1 -0
- package/transports/abstract/subscription.d.ts +1 -3
- package/transports/abstract/subscription.js +0 -1
- package/transports/abstract/subscription.js.map +1 -1
- package/transports/batch-warming.d.ts +20 -8
- package/transports/batch-warming.js +55 -68
- package/transports/batch-warming.js.map +1 -1
- package/transports/http.d.ts +90 -0
- package/transports/http.js +176 -0
- package/transports/http.js.map +1 -0
- package/transports/index.d.ts +2 -3
- package/transports/index.js +1 -2
- package/transports/index.js.map +1 -1
- package/transports/metrics.d.ts +0 -6
- package/transports/metrics.js +1 -18
- package/transports/metrics.js.map +1 -1
- package/transports/sse.d.ts +3 -3
- package/transports/sse.js +6 -3
- package/transports/sse.js.map +1 -1
- package/transports/websocket.d.ts +2 -2
- package/transports/websocket.js +3 -0
- package/transports/websocket.js.map +1 -1
- package/util/index.d.ts +1 -1
- package/util/index.js +1 -1
- package/util/index.js.map +1 -1
- package/util/logger.d.ts +1 -1
- package/util/metrics.d.ts +9 -0
- package/util/metrics.js +53 -0
- package/util/metrics.js.map +1 -0
- package/util/request.d.ts +25 -25
- package/util/requester.d.ts +42 -0
- package/util/requester.js +235 -0
- package/util/requester.js.map +1 -0
- package/util/test-payload-loader.d.ts +1 -1
- package/util/types.d.ts +217 -0
- package/util/types.js +3 -0
- package/util/types.js.map +1 -0
- package/validation/error.d.ts +5 -2
- package/validation/error.js +4 -2
- package/validation/error.js.map +1 -1
- package/validation/index.d.ts +1 -1
- package/validation/index.js.map +1 -1
- package/validation/input-validator.d.ts +3 -0
- package/validation/input-validator.js +31 -1
- package/validation/input-validator.js.map +1 -1
- package/validation/utils.d.ts +1 -0
- package/validation/utils.js +10 -0
- package/validation/utils.js.map +1 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import WebSocket, { ClientOptions, RawData } from 'ws';
|
|
2
2
|
import { EndpointContext } from '../adapter';
|
|
3
3
|
import { AdapterConfig } from '../config';
|
|
4
|
-
import { ProviderResult } from '../util/
|
|
4
|
+
import { ProviderResult } from '../util/types';
|
|
5
5
|
import { TransportGenerics } from './';
|
|
6
6
|
import { StreamingTransport, SubscriptionDeltas } from './abstract/streaming';
|
|
7
7
|
export { WebSocket, RawData as WebSocketRawData };
|
|
@@ -58,7 +58,7 @@ export interface WebSocketTransportConfig<T extends WebsocketTransportGenerics>
|
|
|
58
58
|
}
|
|
59
59
|
/**
|
|
60
60
|
* Helper struct type that will be used to pass types to the generic parameters of a Transport.
|
|
61
|
-
* Extends the common TransportGenerics, adding Provider specific types for this
|
|
61
|
+
* Extends the common TransportGenerics, adding Provider specific types for this WS endpoint.
|
|
62
62
|
*/
|
|
63
63
|
type WebsocketTransportGenerics = TransportGenerics & {
|
|
64
64
|
/**
|
package/transports/websocket.js
CHANGED
|
@@ -201,6 +201,9 @@ class WebSocketTransport extends streaming_1.StreamingTransport {
|
|
|
201
201
|
}
|
|
202
202
|
// Record WS message and subscription metrics
|
|
203
203
|
transportMetrics.recordWsMessageMetrics(context, subscriptions.new, subscriptions.stale);
|
|
204
|
+
// The background execute loop no longer sleeps between executions, so we have to do it here
|
|
205
|
+
logger.trace(`Websocket handler complete, sleeping for ${context.adapterConfig.BACKGROUND_EXECUTE_MS_WS}ms...`);
|
|
206
|
+
await (0, util_1.sleep)(context.adapterConfig.BACKGROUND_EXECUTE_MS_WS);
|
|
204
207
|
return;
|
|
205
208
|
}
|
|
206
209
|
rejectionHandler(rejectionFn, handler) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"websocket.js","sourceRoot":"","sources":["../../../src/transports/websocket.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,4CAAsD;
|
|
1
|
+
{"version":3,"file":"websocket.js","sourceRoot":"","sources":["../../../src/transports/websocket.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,4CAAsD;AAU7C,oBAVF,YAAS,CAUE;AAPlB,kCAA2C;AAG3C,4DAA6C;AAC7C,oDAA6E;AAK7E,MAAM,MAAM,GAAG,IAAA,iBAAU,EAAC,oBAAoB,CAAC,CAAA;AAQ/C,MAAa,sBAAsB;IAGjC,MAAM,CAAC,GAAG,CAAC,IAAoB;QAC7B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA;IAClB,CAAC;IAED,MAAM,CAAC,GAAG;QACR,OAAO,IAAI,CAAC,IAAI,CAAA;IAClB,CAAC;;AATH,wDAUC;AATQ,2BAAI,GAAmB,YAAS,CAAA;AAoFzC;;;;;GAKG;AACH,MAAa,kBAEX,SAAQ,8BAAqB;IAM7B,YAAoB,MAAmC;QACrD,KAAK,EAAE,CAAA;QADW,WAAM,GAAN,MAAM,CAA6B;QAJvD,eAAU,GAAG,EAAE,CAAA;QACf,0BAAqB,GAAG,CAAC,CAAA;QACzB,uBAAkB,GAAG,CAAC,CAAA;IAItB,CAAC;IAED,4BAA4B,CAAC,MAA0C;QACrE,OAAO,MAAM,CAAC,mBAAmB,CAAA;IACnC,CAAC;IAED,gBAAgB;QACd,OAAO,CAAC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY,CAAC,UAAU,KAAK,YAAS,CAAC,MAAM,CAAA;IAChF,CAAC;IAED,gBAAgB,CAAC,OAAgB;QAC/B,OAAO,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;IACxE,CAAC;IACD,kBAAkB,CAAC,IAAoB;QACrC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,CAA+B,CAAA;IAClE,CAAC;IAED,uBAAuB,CACrB,OAA2B,EAC3B,sBAAgD;QAEhD,OAAO;YACL,0CAA0C;YAC1C,IAAI,EAAE,KAAK,EAAE,KAAsB,EAAE,EAAE;gBACrC,MAAM,CAAC,KAAK,CAAC,4CAA4C,KAAK,CAAC,IAAI,GAAG,CAAC,CAAA;gBACvE,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE;oBAC7B,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,CAAA;oBAC3D,MAAM,CAAC,KAAK,CAAC,iDAAiD,CAAC,CAAA;iBAChE;gBACD,6DAA6D;gBAC7D,gBAAgB,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAA;gBACzC,sBAAsB,CAAC,IAAI,CAAC,CAAA;YAC9B,CAAC;YAED,6DAA6D;YAC7D,OAAO,EAAE,KAAK,EAAE,KAA6B,EAAE,EAAE;gBAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;gBAClD,MAAM,CAAC,KAAK,CAAC,mBAAmB,KAAK,CAAC,IAAI,EAAE,CAAC,CAAA;gBAC7C,MAAM,oBAAoB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;gBACvC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;oBACvE,MAAM,MAAM,GAAG,CAAiC,CAAA;oBAChD,MAAM,eAAe,GAAG,CAAC,CAAC,QAAoD,CAAA;oBAC9E,MAAM,CAAC,QAAQ,CAAC,UAAU,GAAG;wBAC3B,6BAA6B,EAAE,IAAI,CAAC,6BAA6B;wBACjE,oBAAoB;wBACpB,qBAAqB,EAAE,eAAe,CAAC,UAAU,EAAE,qBAAqB;qBACzE,CAAA;oBACD,OAAO,MAAM,CAAA;gBACf,CAAC,CAAC,CAAA;gBACF,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;oBAC1B,mFAAmF;oBACnF,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;oBAEvC,MAAM,CAAC,KAAK,CAAC,WAAW,OAAO,CAAC,MAAM,qBAAqB,CAAC,CAAA;oBAC5D,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;iBACxC;gBAED,+DAA+D;gBAC/D,sFAAsF;gBACtF,yEAAyE;gBACzE,gBAAgB,CAAC,cAAc;qBAC5B,MAAM,CAAC;oBACN,SAAS,EAAE,UAAU;iBACtB,CAAC;qBACD,GAAG,EAAE,CAAA;YACV,CAAC;YAED,mDAAmD;YACnD,KAAK,EAAE,KAAK,EAAE,KAA2B,EAAE,EAAE;gBAC3C,MAAM,CAAC,KAAK,CACV,mDAAmD,KAAK,CAAC,KAAK,eAAe,KAAK,CAAC,OAAO,EAAE,CAC7F,CAAA;gBACD,gCAAgC;gBAChC,gBAAgB,CAAC,kBAAkB;qBAChC,MAAM,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;qBAC7D,GAAG,EAAE,CAAA;YACV,CAAC;YAED,sDAAsD;YACtD,KAAK,EAAE,CAAC,KAA2B,EAAE,EAAE;gBACrC,MAAM,CAAC,KAAK,CACV,sCAAsC,KAAK,CAAC,IAAI,cAAc,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,EAAE,CACzF,CAAA;gBACD,8DAA8D;gBAC9D,6DAA6D;gBAC7D,gBAAgB,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAA;YAC3C,CAAC;SACF,CAAA;IACH,CAAC;IAED,qBAAqB,CACnB,OAA2B,EAC3B,GAAW,EACX,OAA6C;QAE7C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,IAAI,GAAG,sBAAsB,CAAC,GAAG,EAAE,CAAA;YACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,uBAAuB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;YAE/D,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,CAAC,CAAA;YACrD,IAAI,CAAC,YAAY,CAAC,gBAAgB,CAChC,MAAM,EACN,IAAI,CAAC,gBAAgB,CAAkB,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,CAC9D,CAAA;YACD,IAAI,CAAC,YAAY,CAAC,gBAAgB,CAChC,SAAS,EACT,IAAI,CAAC,gBAAgB,CAAyB,MAAM,EAAE,QAAQ,CAAC,OAAO,CAAC,CACxE,CAAA;YACD,IAAI,CAAC,YAAY,CAAC,gBAAgB,CAAC,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAA;YAC3D,IAAI,CAAC,YAAY,CAAC,gBAAgB,CAAC,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAA;QAC7D,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,OAA2B,EAAE,UAAqB,EAAE,YAAuB;QAC5F,MAAM,oBAAoB,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;QAClE,MAAM,sBAAsB,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;QAEtE,MAAM,QAAQ,GAAG,oBAAoB,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAA;QACpE,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;YAC9B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;SAChC;IACH,CAAC;IAED,KAAK,CAAC,aAAa,CACjB,OAA2B,EAC3B,aAAyD;QAEzD,mDAAmD;QACnD,uCAAuC;QACvC,qCAAqC;QACrC,2CAA2C;QAC3C,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;YACnD,MAAM,CAAC,KAAK,CAAC,wEAAwE,CAAC,CAAA;YACtF,OAAM;SACP;QAED,0FAA0F;QAC1F,4FAA4F;QAC5F,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,CAAA;QAC3E,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,KAAK,aAAa,CAAA;QAEpD,gFAAgF;QAChF,gEAAgE;QAChE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACtB,MAAM,oBAAoB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC,qBAAqB,CAAC,CAAA;QAC1E,MAAM,yBAAyB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAA;QAC5E,MAAM,qBAAqB,GAAG,IAAI,CAAC,GAAG,CAAC,oBAAoB,EAAE,yBAAyB,CAAC,CAAA;QACvF,MAAM,sBAAsB,GAC1B,qBAAqB,GAAG,CAAC;YACzB,qBAAqB,GAAG,OAAO,CAAC,aAAa,CAAC,gCAAgC,CAAA;QAChF,IAAI,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAA;QAE9C,kDAAkD;QAClD,IAAI,CAAC,gBAAgB,IAAI,CAAC,UAAU,IAAI,sBAAsB,CAAC,EAAE;YAC/D,MAAM,MAAM,GAAG,UAAU;gBACvB,CAAC,CAAC,kCAAkC,IAAI,CAAC,UAAU,OAAO,aAAa,yBAAyB;gBAChG,CAAC,CAAC,6BAA6B,oBAAoB,oCAAoC,OAAO,CAAC,aAAa,CAAC,gCAAgC,2BAA2B,CAAA;YAC1K,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YACnB,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAA;YACzB,gBAAgB,GAAG,IAAI,CAAA;YAEvB,iFAAiF;YACjF,aAAa,CAAC,GAAG,GAAG,aAAa,CAAC,OAAO,CAAA;YACzC,IAAI,aAAa,CAAC,GAAG,CAAC,MAAM,EAAE;gBAC5B,MAAM,CAAC,KAAK,CACV,sFAAsF,IAAI,CAAC,SAAS,CAClG,aAAa,CAAC,GAAG,CAClB,EAAE,CACJ,CAAA;aACF;SACF;QAED,4CAA4C;QAC5C,IAAI,gBAAgB,IAAI,aAAa,CAAC,OAAO,CAAC,MAAM,EAAE;YACpD,MAAM,CAAC,KAAK,CAAC,6EAA6E,CAAC,CAAA;YAC3F,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAA;YAC3E,IAAI,CAAC,UAAU,GAAG,aAAa,CAAA;YAC/B,8GAA8G;YAC9G,IAAI,CAAC,6BAA6B,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YAC/C,MAAM,IAAI,CAAC,qBAAqB,CAAC,OAAO,EAAE,aAAa,EAAE,OAAO,CAAC,CAAA;YACjE,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;SACrC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE;YACxB,MAAM,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAA;YACpD,MAAM,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAA;YACrE,MAAM,IAAI,CAAC,YAAY,CACrB,OAAO,EACP,gBAAgB,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,EAC9E,kBAAkB,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CACvF,CAAA;SACF;QAED,6CAA6C;QAC7C,gBAAgB,CAAC,sBAAsB,CAAC,OAAO,EAAE,aAAa,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,CAAA;QAExF,4FAA4F;QAC5F,MAAM,CAAC,KAAK,CACV,4CAA4C,OAAO,CAAC,aAAa,CAAC,wBAAwB,OAAO,CAClG,CAAA;QACD,MAAM,IAAA,YAAK,EAAC,OAAO,CAAC,aAAa,CAAC,wBAAwB,CAAC,CAAA;QAE3D,OAAM;IACR,CAAC;IAEO,gBAAgB,CACtB,WAAuC,EACvC,OAAoC;QAEpC,OAAO,KAAK,EAAE,KAAQ,EAAE,EAAE;YACxB,IAAI;gBACF,MAAM,OAAO,CAAC,KAAK,CAAC,CAAA;aACrB;YAAC,OAAO,CAAC,EAAE;gBACV,OAAO,WAAW,CAAC,CAAC,CAAC,CAAA;aACtB;QACH,CAAC,CAAA;IACH,CAAC;CACF;AAnOD,gDAmOC"}
|
package/util/index.d.ts
CHANGED
package/util/index.js
CHANGED
|
@@ -15,7 +15,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
17
|
exports.isEmpty = exports.isArray = exports.isObject = exports.sleep = void 0;
|
|
18
|
-
__exportStar(require("./
|
|
18
|
+
__exportStar(require("./types"), exports);
|
|
19
19
|
__exportStar(require("./logger"), exports);
|
|
20
20
|
__exportStar(require("./subscription-set/subscription-set"), exports);
|
|
21
21
|
/**
|
package/util/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/util/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;AAAA,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/util/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;AAAA,0CAAuB;AACvB,2CAAwB;AACxB,sEAAmD;AAEnD;;;;GAIG;AACI,MAAM,KAAK,GAAG,CAAC,EAAU,EAAiB,EAAE;IACjD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAA;IACzB,CAAC,CAAC,CAAA;AACJ,CAAC,CAAA;AAJY,QAAA,KAAK,SAIjB;AAEM,MAAM,QAAQ,GAAG,CAAC,CAAU,EAAW,EAAE,CAC9C,CAAC,KAAK,IAAI,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,KAAK,CAAA;AADtD,QAAA,QAAQ,YAC8C;AAE5D,MAAM,OAAO,GAAG,CAAC,CAAU,EAAW,EAAE,CAC7C,CAAC,KAAK,IAAI,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;AAD5C,QAAA,OAAO,WACqC;AAElD,MAAM,OAAO,GAAG,CAAC,CAAU,EAAW,EAAE,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,CAAA;AAA5E,QAAA,OAAO,WAAqE"}
|
package/util/logger.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
import pino from 'pino';
|
|
3
|
-
import { AdapterRequest } from './
|
|
3
|
+
import { AdapterRequest } from './types';
|
|
4
4
|
import { FastifyReply, HookHandlerDoneFunction } from 'fastify';
|
|
5
5
|
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
6
6
|
import { CensorKeyValue } from './censor/censor-list';
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import * as client from 'prom-client';
|
|
2
|
+
export declare const dataProviderMetricsLabel: (providerStatusCode?: number, method?: string) => {
|
|
3
|
+
provider_status_code: number | undefined;
|
|
4
|
+
method: string;
|
|
5
|
+
};
|
|
6
|
+
export declare const dataProviderRequests: client.Counter<"method" | "provider_status_code">;
|
|
7
|
+
export declare const dataProviderRequestDurationSeconds: client.Histogram<string>;
|
|
8
|
+
export declare const requesterQueueSize: client.Gauge<string>;
|
|
9
|
+
export declare const requesterQueueOverflow: client.Counter<string>;
|
package/util/metrics.js
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
+
exports.requesterQueueOverflow = exports.requesterQueueSize = exports.dataProviderRequestDurationSeconds = exports.dataProviderRequests = exports.dataProviderMetricsLabel = void 0;
|
|
27
|
+
const client = __importStar(require("prom-client"));
|
|
28
|
+
const constants_1 = require("../metrics/constants");
|
|
29
|
+
// Data Provider Requests Metrics
|
|
30
|
+
const dataProviderMetricsLabel = (providerStatusCode, method = 'get') => ({
|
|
31
|
+
provider_status_code: providerStatusCode,
|
|
32
|
+
method: method.toUpperCase(),
|
|
33
|
+
});
|
|
34
|
+
exports.dataProviderMetricsLabel = dataProviderMetricsLabel;
|
|
35
|
+
exports.dataProviderRequests = new client.Counter({
|
|
36
|
+
name: 'data_provider_requests',
|
|
37
|
+
help: 'The number of http requests that are made to a data provider',
|
|
38
|
+
labelNames: ['method', 'provider_status_code'],
|
|
39
|
+
});
|
|
40
|
+
exports.dataProviderRequestDurationSeconds = new client.Histogram({
|
|
41
|
+
name: 'data_provider_request_duration_seconds',
|
|
42
|
+
help: 'A histogram bucket of the distribution of data provider request durations',
|
|
43
|
+
buckets: constants_1.requestDurationBuckets,
|
|
44
|
+
});
|
|
45
|
+
exports.requesterQueueSize = new client.Gauge({
|
|
46
|
+
name: 'requester_queue_size',
|
|
47
|
+
help: 'The number of provider http requests currently queued to be executed',
|
|
48
|
+
});
|
|
49
|
+
exports.requesterQueueOverflow = new client.Counter({
|
|
50
|
+
name: 'requester_queue_overflow',
|
|
51
|
+
help: 'Total times the requester queue replaced the oldest item to avoid an overflow',
|
|
52
|
+
});
|
|
53
|
+
//# sourceMappingURL=metrics.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../../src/util/metrics.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,oDAAqC;AACrC,oDAA6D;AAE7D,iCAAiC;AAC1B,MAAM,wBAAwB,GAAG,CAAC,kBAA2B,EAAE,MAAM,GAAG,KAAK,EAAE,EAAE,CAAC,CAAC;IACxF,oBAAoB,EAAE,kBAAkB;IACxC,MAAM,EAAE,MAAM,CAAC,WAAW,EAAE;CAC7B,CAAC,CAAA;AAHW,QAAA,wBAAwB,4BAGnC;AAEW,QAAA,oBAAoB,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IACrD,IAAI,EAAE,wBAAwB;IAC9B,IAAI,EAAE,8DAA8D;IACpE,UAAU,EAAE,CAAC,QAAQ,EAAE,sBAAsB,CAAU;CACxD,CAAC,CAAA;AAEW,QAAA,kCAAkC,GAAG,IAAI,MAAM,CAAC,SAAS,CAAC;IACrE,IAAI,EAAE,wCAAwC;IAC9C,IAAI,EAAE,2EAA2E;IACjF,OAAO,EAAE,kCAAsB;CAChC,CAAC,CAAA;AAEW,QAAA,kBAAkB,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC;IACjD,IAAI,EAAE,sBAAsB;IAC5B,IAAI,EAAE,sEAAsE;CAC7E,CAAC,CAAA;AAEW,QAAA,sBAAsB,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IACvD,IAAI,EAAE,0BAA0B;IAChC,IAAI,EAAE,+EAA+E;CACtF,CAAC,CAAA"}
|
package/util/request.d.ts
CHANGED
|
@@ -6,7 +6,7 @@ declare module 'fastify' {
|
|
|
6
6
|
requestContext: AdapterRequestContext;
|
|
7
7
|
}
|
|
8
8
|
}
|
|
9
|
-
export type Merge<T1, T2> = {
|
|
9
|
+
export declare type Merge<T1, T2> = {
|
|
10
10
|
[K in keyof T1 | keyof T2]: K extends keyof T2 ? T2[K] : K extends keyof T1 ? T1[K] : never;
|
|
11
11
|
};
|
|
12
12
|
/**
|
|
@@ -22,7 +22,7 @@ export interface AdapterRequestBody<T = AdapterRequestData> {
|
|
|
22
22
|
* Object that will be added to the request on a successful validation.
|
|
23
23
|
* Contains all the necessary information the adapter will need across the request execution.
|
|
24
24
|
*/
|
|
25
|
-
export type AdapterRequestContext<T = AdapterRequestData> = {
|
|
25
|
+
export declare type AdapterRequestContext<T = AdapterRequestData> = {
|
|
26
26
|
/** Name of the endpoint this payload should be directed to */
|
|
27
27
|
endpointName: string;
|
|
28
28
|
/** Precalculated cache key used to get and set corresponding values from the cache and subscription sets */
|
|
@@ -35,18 +35,18 @@ export type AdapterRequestContext<T = AdapterRequestData> = {
|
|
|
35
35
|
/**
|
|
36
36
|
* Helper type to denote an empty body
|
|
37
37
|
*/
|
|
38
|
-
type EmptyBody = Record<string, never>;
|
|
38
|
+
declare type EmptyBody = Record<string, never>;
|
|
39
39
|
/**
|
|
40
40
|
* Helper type to provide fastify for all adapter handlers
|
|
41
41
|
*/
|
|
42
|
-
export type AdapterRouteGeneric = {
|
|
42
|
+
export declare type AdapterRouteGeneric = {
|
|
43
43
|
/** Set to an empty record so the user does not access the raw request data and uses the Validated data from the context instead */
|
|
44
44
|
Body: EmptyBody;
|
|
45
45
|
};
|
|
46
46
|
/**
|
|
47
47
|
* Structure for all requests incoming to this adapter
|
|
48
48
|
*/
|
|
49
|
-
export type AdapterRequest<T extends RequestGenerics = RequestGenerics> = FastifyRequest<AdapterRouteGeneric> & {
|
|
49
|
+
export declare type AdapterRequest<T extends RequestGenerics = RequestGenerics> = FastifyRequest<AdapterRouteGeneric> & {
|
|
50
50
|
/** Set to an empty record so the user does not access the raw request data and uses the Validated data from the context instead */
|
|
51
51
|
body: EmptyBody;
|
|
52
52
|
/** Container for all validated information that will be used by the framework across this request's lifecycle */
|
|
@@ -73,23 +73,23 @@ export interface AdapterMetricsMeta {
|
|
|
73
73
|
/**
|
|
74
74
|
* Basic shape for the data within the request body
|
|
75
75
|
*/
|
|
76
|
-
export type AdapterRequestData = Record<string, unknown> & {
|
|
76
|
+
export declare type AdapterRequestData = Record<string, unknown> & {
|
|
77
77
|
endpoint?: string;
|
|
78
78
|
};
|
|
79
|
-
export type ProviderResultGenerics = {
|
|
79
|
+
export declare type ProviderResultGenerics = {
|
|
80
80
|
Request: RequestGenerics;
|
|
81
81
|
Response: ResponseGenerics;
|
|
82
82
|
};
|
|
83
83
|
/**
|
|
84
84
|
* Helper type to hold the value from responses from a provider, and the adapter params they correspond to.
|
|
85
85
|
*/
|
|
86
|
-
export type ProviderResult<T extends ProviderResultGenerics> = {
|
|
86
|
+
export declare type ProviderResult<T extends ProviderResultGenerics> = {
|
|
87
87
|
/** The set of parameters that uniquely relate to the response */
|
|
88
88
|
params: T['Request']['Params'];
|
|
89
89
|
/** Value that will be included in the result property of the response */
|
|
90
90
|
response: PartialAdapterResponse<T['Response']>;
|
|
91
91
|
};
|
|
92
|
-
export type TimestampedProviderResult<T extends ProviderResultGenerics> = Pick<ProviderResult<T>, 'params'> & {
|
|
92
|
+
export declare type TimestampedProviderResult<T extends ProviderResultGenerics> = Pick<ProviderResult<T>, 'params'> & {
|
|
93
93
|
/** Value that will be included in the result property of the response, with timestamps applied */
|
|
94
94
|
response: TimestampedAdapterResponse<T['Response']>;
|
|
95
95
|
};
|
|
@@ -97,11 +97,11 @@ export type TimestampedProviderResult<T extends ProviderResultGenerics> = Pick<P
|
|
|
97
97
|
* Here we do actually want an object, since unknown could be a primitive.
|
|
98
98
|
* The rule itself does say "marginally better"
|
|
99
99
|
*/
|
|
100
|
-
export type EmptyObject = Object;
|
|
100
|
+
export declare type EmptyObject = Object;
|
|
101
101
|
/**
|
|
102
102
|
* Helper struct type that provides detail about the incoming Adapter Request
|
|
103
103
|
*/
|
|
104
|
-
export type RequestGenerics = {
|
|
104
|
+
export declare type RequestGenerics = {
|
|
105
105
|
/**
|
|
106
106
|
* Type for the parameters sent to the EA in the data property of the body.
|
|
107
107
|
*/
|
|
@@ -110,7 +110,7 @@ export type RequestGenerics = {
|
|
|
110
110
|
/**
|
|
111
111
|
* Helper struct type that provides detail about the outgoing Adapter Response
|
|
112
112
|
*/
|
|
113
|
-
export type ResponseGenerics = {
|
|
113
|
+
export declare type ResponseGenerics = {
|
|
114
114
|
/**
|
|
115
115
|
* Type for the data property of the response.
|
|
116
116
|
*/
|
|
@@ -124,7 +124,7 @@ export type ResponseGenerics = {
|
|
|
124
124
|
/**
|
|
125
125
|
* Details for timestamps to be included in the response
|
|
126
126
|
*/
|
|
127
|
-
type ResponseTimestamps = {
|
|
127
|
+
export declare type ResponseTimestamps = {
|
|
128
128
|
/** Time at which data was received from the provider */
|
|
129
129
|
providerDataReceived: number;
|
|
130
130
|
/** Time indicated by the provider representing the time at which this value was calculated/set/valid */
|
|
@@ -147,14 +147,14 @@ type ResponseTimestamps = {
|
|
|
147
147
|
/**
|
|
148
148
|
* Object with timestamps that will be present in both successful and provider error responses
|
|
149
149
|
*/
|
|
150
|
-
type TimestampedResponseObject = {
|
|
150
|
+
declare type TimestampedResponseObject = {
|
|
151
151
|
/** Timestamps relevant for data provider timings */
|
|
152
152
|
timestamps: ResponseTimestamps;
|
|
153
153
|
};
|
|
154
154
|
/**
|
|
155
155
|
* Response from the EA to send when manually storing an error in the cache
|
|
156
156
|
*/
|
|
157
|
-
type ProviderErrorResponse = {
|
|
157
|
+
declare type ProviderErrorResponse = {
|
|
158
158
|
/** Status code for the errored response */
|
|
159
159
|
statusCode: number;
|
|
160
160
|
/** Error message that will be sent back from the adapter */
|
|
@@ -167,11 +167,11 @@ type ProviderErrorResponse = {
|
|
|
167
167
|
/**
|
|
168
168
|
* Provider error response with timestamps added
|
|
169
169
|
*/
|
|
170
|
-
export type TimestampedProviderErrorResponse = ProviderErrorResponse & TimestampedResponseObject;
|
|
170
|
+
export declare type TimestampedProviderErrorResponse = ProviderErrorResponse & TimestampedResponseObject;
|
|
171
171
|
/**
|
|
172
172
|
* The most basic data that needs to be manually set for an adapter response.
|
|
173
173
|
*/
|
|
174
|
-
export type PartialSuccessfulResponse<T extends ResponseGenerics> = {
|
|
174
|
+
export declare type PartialSuccessfulResponse<T extends ResponseGenerics> = {
|
|
175
175
|
/** Response data, holds "result" for Flux Monitor */
|
|
176
176
|
data: T['Data'];
|
|
177
177
|
/** Result value used for OCR */
|
|
@@ -184,11 +184,11 @@ export type PartialSuccessfulResponse<T extends ResponseGenerics> = {
|
|
|
184
184
|
/**
|
|
185
185
|
* Partial EA response, with timestamps added
|
|
186
186
|
*/
|
|
187
|
-
type TimestampedSuccessfulResponse<T extends ResponseGenerics> = PartialSuccessfulResponse<T> & TimestampedResponseObject;
|
|
187
|
+
declare type TimestampedSuccessfulResponse<T extends ResponseGenerics> = PartialSuccessfulResponse<T> & TimestampedResponseObject;
|
|
188
188
|
/**
|
|
189
189
|
* Full EA successful response, with metadata and defaults added
|
|
190
190
|
*/
|
|
191
|
-
type SuccessfulResponse<T extends ResponseGenerics> = TimestampedSuccessfulResponse<T> & {
|
|
191
|
+
declare type SuccessfulResponse<T extends ResponseGenerics> = TimestampedSuccessfulResponse<T> & {
|
|
192
192
|
/** Metadata relevant to this request */
|
|
193
193
|
meta?: AdapterRequestMeta;
|
|
194
194
|
/** HTTP status code, by default will be set to 200 */
|
|
@@ -197,21 +197,21 @@ type SuccessfulResponse<T extends ResponseGenerics> = TimestampedSuccessfulRespo
|
|
|
197
197
|
/**
|
|
198
198
|
* Response body from the EA, before timestamps, defaults and metadata are filled in
|
|
199
199
|
*/
|
|
200
|
-
export type PartialAdapterResponse<T extends ResponseGenerics = ResponseGenerics> = PartialSuccessfulResponse<T> | ProviderErrorResponse;
|
|
200
|
+
export declare type PartialAdapterResponse<T extends ResponseGenerics = ResponseGenerics> = PartialSuccessfulResponse<T> | ProviderErrorResponse;
|
|
201
201
|
/**
|
|
202
202
|
* Response body from the EA with timestamps, before defaults and metadata are filled in
|
|
203
203
|
*/
|
|
204
|
-
export type TimestampedAdapterResponse<T extends ResponseGenerics = ResponseGenerics> = TimestampedSuccessfulResponse<T> | TimestampedProviderErrorResponse;
|
|
204
|
+
export declare type TimestampedAdapterResponse<T extends ResponseGenerics = ResponseGenerics> = TimestampedSuccessfulResponse<T> | TimestampedProviderErrorResponse;
|
|
205
205
|
/**
|
|
206
206
|
* Shape of the response body from the adapter
|
|
207
207
|
*/
|
|
208
|
-
export type AdapterResponse<T extends ResponseGenerics = ResponseGenerics> = SuccessfulResponse<T> | TimestampedProviderErrorResponse;
|
|
209
|
-
export type SingleNumberResultResponse = {
|
|
208
|
+
export declare type AdapterResponse<T extends ResponseGenerics = ResponseGenerics> = SuccessfulResponse<T> | TimestampedProviderErrorResponse;
|
|
209
|
+
export declare type SingleNumberResultResponse = {
|
|
210
210
|
Result: number;
|
|
211
211
|
Data: {
|
|
212
212
|
result: number;
|
|
213
213
|
};
|
|
214
214
|
};
|
|
215
|
-
export type Middleware = ((req: AdapterRequest, reply: FastifyReply, done: HookHandlerDoneFunction) => FastifyReply | void) | ((req: AdapterRequest, reply: FastifyReply) => Promise<FastifyReply | void>);
|
|
216
|
-
export type AdapterMiddlewareBuilder = (adapter: Adapter) => Middleware;
|
|
215
|
+
export declare type Middleware = ((req: AdapterRequest, reply: FastifyReply, done: HookHandlerDoneFunction) => FastifyReply | void) | ((req: AdapterRequest, reply: FastifyReply) => Promise<FastifyReply | void>);
|
|
216
|
+
export declare type AdapterMiddlewareBuilder = (adapter: Adapter) => Middleware;
|
|
217
217
|
export {};
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { AxiosRequestConfig, AxiosResponse } from 'axios';
|
|
2
|
+
import { AdapterConfig } from '../config';
|
|
3
|
+
import { RateLimiter } from '../rate-limiting';
|
|
4
|
+
interface RequesterResult<T> {
|
|
5
|
+
response: AxiosResponse<T>;
|
|
6
|
+
timestamps: {
|
|
7
|
+
providerDataRequested: number;
|
|
8
|
+
providerDataReceived: number;
|
|
9
|
+
};
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Centralized management of outbound http requests.
|
|
13
|
+
* Enforces rate limiting on a single instance (complying with the N Readers - 1 Writer arch for EA scaling)
|
|
14
|
+
* by adding requests into a queue, processing them sequentially and sleeping when it reaches its limit.
|
|
15
|
+
* The queue will throw an error if the Requester attempts to add more items than the max configured.
|
|
16
|
+
* It additionally serves to coalesce requests by utilizing a more complex queue structure:
|
|
17
|
+
* - ignores duplicate items via a provided key
|
|
18
|
+
* - doesn't use the request itself because it's common for those to have things like timestamps/nonces
|
|
19
|
+
* This implementation does not:
|
|
20
|
+
* - Prioritize any request over another
|
|
21
|
+
* - Contemplate architectures with multiple writer EA instances
|
|
22
|
+
*/
|
|
23
|
+
export declare class Requester {
|
|
24
|
+
private rateLimiter;
|
|
25
|
+
private processing;
|
|
26
|
+
private queue;
|
|
27
|
+
private map;
|
|
28
|
+
private maxRetries;
|
|
29
|
+
private timeout;
|
|
30
|
+
constructor(rateLimiter: RateLimiter, config: AdapterConfig);
|
|
31
|
+
/**
|
|
32
|
+
* Queues the provided request, and returns a promise that will resolve whenever it's executed.
|
|
33
|
+
*
|
|
34
|
+
* @param key - a key to uniquely identify this request, and coalesce new ones that match
|
|
35
|
+
* @param req - a request to send to a data provider
|
|
36
|
+
* @returns a promise that will resolve whenever the request is popped from the queue, sent, and a response is received
|
|
37
|
+
*/
|
|
38
|
+
request<T>(key: string, req: AxiosRequestConfig): Promise<RequesterResult<T>>;
|
|
39
|
+
private processNext;
|
|
40
|
+
private executeRequest;
|
|
41
|
+
}
|
|
42
|
+
export {};
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
26
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
|
+
};
|
|
28
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
+
exports.Requester = void 0;
|
|
30
|
+
const axios_1 = __importDefault(require("axios"));
|
|
31
|
+
const _1 = require(".");
|
|
32
|
+
const error_1 = require("../validation/error");
|
|
33
|
+
const metrics = __importStar(require("./metrics"));
|
|
34
|
+
const logger = (0, _1.makeLogger)('Requester');
|
|
35
|
+
class UniqueLinkedList {
|
|
36
|
+
constructor(maxLength) {
|
|
37
|
+
this.maxLength = maxLength;
|
|
38
|
+
this.length = 0;
|
|
39
|
+
}
|
|
40
|
+
add(value) {
|
|
41
|
+
let overflow;
|
|
42
|
+
if (this.length === this.maxLength) {
|
|
43
|
+
// If this new item would put us over max length, remove the first one (i.e. oldest one)
|
|
44
|
+
overflow = this.remove();
|
|
45
|
+
}
|
|
46
|
+
const node = {
|
|
47
|
+
value,
|
|
48
|
+
next: undefined,
|
|
49
|
+
};
|
|
50
|
+
if (!this.first) {
|
|
51
|
+
this.first = node;
|
|
52
|
+
}
|
|
53
|
+
if (this.last) {
|
|
54
|
+
this.last.next = node;
|
|
55
|
+
}
|
|
56
|
+
this.last = node;
|
|
57
|
+
this.length++;
|
|
58
|
+
metrics.requesterQueueSize.inc();
|
|
59
|
+
return overflow;
|
|
60
|
+
}
|
|
61
|
+
remove() {
|
|
62
|
+
const node = this.first;
|
|
63
|
+
if (!node) {
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
this.first = node.next;
|
|
67
|
+
this.length--;
|
|
68
|
+
metrics.requesterQueueSize.dec();
|
|
69
|
+
return node.value;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Centralized management of outbound http requests.
|
|
74
|
+
* Enforces rate limiting on a single instance (complying with the N Readers - 1 Writer arch for EA scaling)
|
|
75
|
+
* by adding requests into a queue, processing them sequentially and sleeping when it reaches its limit.
|
|
76
|
+
* The queue will throw an error if the Requester attempts to add more items than the max configured.
|
|
77
|
+
* It additionally serves to coalesce requests by utilizing a more complex queue structure:
|
|
78
|
+
* - ignores duplicate items via a provided key
|
|
79
|
+
* - doesn't use the request itself because it's common for those to have things like timestamps/nonces
|
|
80
|
+
* This implementation does not:
|
|
81
|
+
* - Prioritize any request over another
|
|
82
|
+
* - Contemplate architectures with multiple writer EA instances
|
|
83
|
+
*/
|
|
84
|
+
class Requester {
|
|
85
|
+
constructor(rateLimiter, config) {
|
|
86
|
+
this.rateLimiter = rateLimiter;
|
|
87
|
+
this.processing = false;
|
|
88
|
+
this.map = {};
|
|
89
|
+
this.maxRetries = config.RETRY;
|
|
90
|
+
this.timeout = config.API_TIMEOUT;
|
|
91
|
+
this.queue = new UniqueLinkedList(config.MAX_HTTP_REQUEST_QUEUE_LENGTH);
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Queues the provided request, and returns a promise that will resolve whenever it's executed.
|
|
95
|
+
*
|
|
96
|
+
* @param key - a key to uniquely identify this request, and coalesce new ones that match
|
|
97
|
+
* @param req - a request to send to a data provider
|
|
98
|
+
* @returns a promise that will resolve whenever the request is popped from the queue, sent, and a response is received
|
|
99
|
+
*/
|
|
100
|
+
async request(key, req) {
|
|
101
|
+
// If there's already a queued request, reuse it's existing promise
|
|
102
|
+
const existingQueuedRequest = this.map[key];
|
|
103
|
+
if (existingQueuedRequest) {
|
|
104
|
+
logger.trace(`Request already exists, returning queued promise (Key: ${key})`);
|
|
105
|
+
return existingQueuedRequest.promise;
|
|
106
|
+
}
|
|
107
|
+
const queuedRequest = {
|
|
108
|
+
key,
|
|
109
|
+
config: req,
|
|
110
|
+
retries: 0,
|
|
111
|
+
};
|
|
112
|
+
// This dual promise layer is built so the queuedRequest can hold both the resolve and reject handlers,
|
|
113
|
+
// and the promise itself so we can return it for request coalescing without creating new ones
|
|
114
|
+
await new Promise((unblock) => {
|
|
115
|
+
queuedRequest.promise = new Promise((success, failure) => {
|
|
116
|
+
queuedRequest.resolve = success;
|
|
117
|
+
queuedRequest.reject = failure;
|
|
118
|
+
unblock(0);
|
|
119
|
+
});
|
|
120
|
+
});
|
|
121
|
+
// By the time we're here, we know that queuedRequest has both the unresolved promise, and the resolve and reject handlers within
|
|
122
|
+
// It's really, REALLY important for thread safety that from this point until this function returns, there are no async breaks.
|
|
123
|
+
// Node will stay within a "thread" until it gets a chance to switch context, like an "await" statement. This section of code depends
|
|
124
|
+
// on it executing from start to finish without any other actions to avoid race conditions, so if we had an "await" here we could run into problems.
|
|
125
|
+
// For example, two separate queue processing "threads" could be spawned by mistake, or the queue could overflow.
|
|
126
|
+
const overflowedRequest = this.queue.add(queuedRequest);
|
|
127
|
+
if (overflowedRequest) {
|
|
128
|
+
// If we have overflow, it means the oldest request needs to be rejected because the queue is at its limits
|
|
129
|
+
logger.debug(`Request (Key: ${key}, Retry #: ${0}) was removed from the queue to make room for a newer one (Size: ${this.queue.length})`);
|
|
130
|
+
metrics.requesterQueueOverflow.inc();
|
|
131
|
+
overflowedRequest.reject(new error_1.AdapterRateLimitError({
|
|
132
|
+
message: 'The EA was unable to execute the request to fetch the requested data from the DP because the request queue overflowed. This likely indicates that a higher API tier is needed.',
|
|
133
|
+
statusCode: 429,
|
|
134
|
+
}));
|
|
135
|
+
}
|
|
136
|
+
// The item was successfully added to the queue, so we can also add it to our map
|
|
137
|
+
logger.trace(`Added request (Key: ${key}, Retry #: ${0}) to the queue (Size: ${this.queue.length})`);
|
|
138
|
+
this.map[key] = queuedRequest;
|
|
139
|
+
// Finally, we start the queue processing
|
|
140
|
+
if (!this.processing) {
|
|
141
|
+
this.processing = true;
|
|
142
|
+
logger.debug(`Starting requester queue processing`);
|
|
143
|
+
// We don't want to wait for the queue to finish processing here; this will just spawn a "thread"
|
|
144
|
+
// and the promise we'll return from this method is the one for the request when it resolves
|
|
145
|
+
this.processNext();
|
|
146
|
+
}
|
|
147
|
+
return queuedRequest.promise;
|
|
148
|
+
}
|
|
149
|
+
// Will grab from queue sequentially, and sleep just before hitting rate limits
|
|
150
|
+
async processNext() {
|
|
151
|
+
// This will remove from the list, but not the map; that way coalescing is still functional for in-flight reqs
|
|
152
|
+
const next = this.queue.remove();
|
|
153
|
+
if (!next) {
|
|
154
|
+
logger.debug(`No more requests present in the queue, stopping processing until new one comes in`);
|
|
155
|
+
this.processing = false;
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
logger.trace(`Popped next request (Key: ${next.key}, Retry #: ${next.retries}) from the queue (Size: ${this.queue.length})`);
|
|
159
|
+
// Check if we can execute it now, or we have to wait
|
|
160
|
+
const timeToWait = this.rateLimiter.msUntilNextExecution();
|
|
161
|
+
if (timeToWait) {
|
|
162
|
+
logger.trace(`Ran into request limits, sleeping ${timeToWait}ms before executing (Key: ${next.key}, Retry #: ${next.retries})`);
|
|
163
|
+
await (0, _1.sleep)(timeToWait);
|
|
164
|
+
}
|
|
165
|
+
// Fire off to complete in the background. We don't wait here to be able to fire off multiple requests concurrently
|
|
166
|
+
this.executeRequest.bind(this)(next);
|
|
167
|
+
return this.processNext();
|
|
168
|
+
}
|
|
169
|
+
// Handler for the requests that will be fired off, eventually resolving the promise associated with the queued request
|
|
170
|
+
async executeRequest(req) {
|
|
171
|
+
const { key, config, resolve, reject, retries } = req;
|
|
172
|
+
const providerDataRequested = Date.now();
|
|
173
|
+
const responseTimer = metrics.dataProviderRequestDurationSeconds.startTimer();
|
|
174
|
+
// Set configured timeout for all requests unless manually specified
|
|
175
|
+
config.timeout = config.timeout || this.timeout;
|
|
176
|
+
try {
|
|
177
|
+
logger.trace(`Sending request (Key: ${key}) to data provider`);
|
|
178
|
+
const response = await axios_1.default.request(config);
|
|
179
|
+
logger.trace(`Request (Key: ${key}) was successful `);
|
|
180
|
+
resolve({
|
|
181
|
+
response,
|
|
182
|
+
timestamps: {
|
|
183
|
+
providerDataRequested,
|
|
184
|
+
providerDataReceived: Date.now(),
|
|
185
|
+
},
|
|
186
|
+
});
|
|
187
|
+
// Remove the request from our map
|
|
188
|
+
delete this.map[key];
|
|
189
|
+
// Record count of successful data provider requests
|
|
190
|
+
metrics.dataProviderRequests
|
|
191
|
+
.labels(metrics.dataProviderMetricsLabel(response.status, config.method))
|
|
192
|
+
.inc();
|
|
193
|
+
}
|
|
194
|
+
catch (e) {
|
|
195
|
+
if (retries >= this.maxRetries) {
|
|
196
|
+
logger.trace(`Request failed and no more retries remaining, rejecting promise...`);
|
|
197
|
+
const err = e;
|
|
198
|
+
const ErrorClass = err.response?.status ? error_1.AdapterDataProviderError : error_1.AdapterConnectionError;
|
|
199
|
+
// Record count of failed data provider request
|
|
200
|
+
metrics.dataProviderRequests
|
|
201
|
+
.labels(metrics.dataProviderMetricsLabel(err.response?.status || 0, config.method))
|
|
202
|
+
.inc();
|
|
203
|
+
reject(new ErrorClass({
|
|
204
|
+
statusCode: 502,
|
|
205
|
+
name: 'Data Provider error',
|
|
206
|
+
providerStatusCode: err?.response?.status ?? 502,
|
|
207
|
+
message: err?.message,
|
|
208
|
+
cause: e,
|
|
209
|
+
errorResponse: err?.response?.data,
|
|
210
|
+
url: config.url,
|
|
211
|
+
}, {
|
|
212
|
+
providerDataRequested,
|
|
213
|
+
providerDataReceived: Date.now(),
|
|
214
|
+
providerIndicatedTime: undefined,
|
|
215
|
+
}));
|
|
216
|
+
// Remove the request from our map
|
|
217
|
+
delete this.map[key];
|
|
218
|
+
}
|
|
219
|
+
else {
|
|
220
|
+
const timeToSleep = (2 ** retries + Math.random()) * 1000;
|
|
221
|
+
logger.trace(`Request failed, sleeping for ${timeToSleep}ms...`);
|
|
222
|
+
await (0, _1.sleep)(timeToSleep);
|
|
223
|
+
req.retries++;
|
|
224
|
+
logger.trace(`Adding request to the queue (Key: ${key}, Retry #: ${req.retries})`);
|
|
225
|
+
this.queue.add(req);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
finally {
|
|
229
|
+
// Record time taken for data provider request for success or failure
|
|
230
|
+
responseTimer();
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
exports.Requester = Requester;
|
|
235
|
+
//# sourceMappingURL=requester.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"requester.js","sourceRoot":"","sources":["../../../src/util/requester.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,kDAA4E;AAC5E,wBAAqC;AAGrC,+CAI4B;AAC5B,mDAAoC;AAEpC,MAAM,MAAM,GAAG,IAAA,aAAU,EAAC,WAAW,CAAC,CAAA;AAOtC,MAAM,gBAAgB;IAKpB,YAAoB,SAAiB;QAAjB,cAAS,GAAT,SAAS,CAAQ;QAFrC,WAAM,GAAG,CAAC,CAAA;IAE8B,CAAC;IAEzC,GAAG,CAAC,KAAQ;QACV,IAAI,QAAQ,CAAA;QACZ,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,SAAS,EAAE;YAClC,wFAAwF;YACxF,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE,CAAA;SACzB;QAED,MAAM,IAAI,GAAgB;YACxB,KAAK;YACL,IAAI,EAAE,SAAS;SAChB,CAAA;QAED,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE;YACf,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;SAClB;QACD,IAAI,IAAI,CAAC,IAAI,EAAE;YACb,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA;SACtB;QAED,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA;QAChB,IAAI,CAAC,MAAM,EAAE,CAAA;QACb,OAAO,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAA;QAChC,OAAO,QAAQ,CAAA;IACjB,CAAC;IAED,MAAM;QACJ,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAA;QAEvB,IAAI,CAAC,IAAI,EAAE;YACT,OAAM;SACP;QAED,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,CAAA;QACtB,IAAI,CAAC,MAAM,EAAE,CAAA;QACb,OAAO,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAA;QAChC,OAAO,IAAI,CAAC,KAAK,CAAA;IACnB,CAAC;CACF;AAmBD;;;;;;;;;;;GAWG;AACH,MAAa,SAAS;IAOpB,YAAoB,WAAwB,EAAE,MAAqB;QAA/C,gBAAW,GAAX,WAAW,CAAa;QANpC,eAAU,GAAG,KAAK,CAAA;QAElB,QAAG,GAAG,EAAmC,CAAA;QAK/C,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,KAAK,CAAA;QAC9B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,WAAW,CAAA;QACjC,IAAI,CAAC,KAAK,GAAG,IAAI,gBAAgB,CAAgB,MAAM,CAAC,6BAA6B,CAAC,CAAA;IACxF,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,OAAO,CAAI,GAAW,EAAE,GAAuB;QACnD,mEAAmE;QACnE,MAAM,qBAAqB,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC3C,IAAI,qBAAqB,EAAE;YACzB,MAAM,CAAC,KAAK,CAAC,0DAA0D,GAAG,GAAG,CAAC,CAAA;YAC9E,OAAO,qBAAqB,CAAC,OAAsC,CAAA;SACpE;QAED,MAAM,aAAa,GAAG;YACpB,GAAG;YACH,MAAM,EAAE,GAAG;YACX,OAAO,EAAE,CAAC;SACS,CAAA;QAErB,uGAAuG;QACvG,8FAA8F;QAC9F,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC5B,aAAa,CAAC,OAAO,GAAG,IAAI,OAAO,CAAqB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE;gBAC3E,aAAa,CAAC,OAAO,GAAG,OAAO,CAAA;gBAC/B,aAAa,CAAC,MAAM,GAAG,OAAO,CAAA;gBAC9B,OAAO,CAAC,CAAC,CAAC,CAAA;YACZ,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;QAEF,iIAAiI;QACjI,+HAA+H;QAC/H,qIAAqI;QACrI,oJAAoJ;QACpJ,iHAAiH;QACjH,MAAM,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,aAA8B,CAAC,CAAA;QACxE,IAAI,iBAAiB,EAAE;YACrB,2GAA2G;YAC3G,MAAM,CAAC,KAAK,CACV,iBAAiB,GAAG,cAAc,CAAC,oEACjC,IAAI,CAAC,KAAK,CAAC,MACb,GAAG,CACJ,CAAA;YACD,OAAO,CAAC,sBAAsB,CAAC,GAAG,EAAE,CAAA;YACpC,iBAAiB,CAAC,MAAM,CACtB,IAAI,6BAAqB,CAAC;gBACxB,OAAO,EACL,gLAAgL;gBAClL,UAAU,EAAE,GAAG;aAChB,CAAC,CACH,CAAA;SACF;QAED,iFAAiF;QACjF,MAAM,CAAC,KAAK,CACV,uBAAuB,GAAG,cAAc,CAAC,yBAAyB,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CACvF,CAAA;QACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,aAA8B,CAAA;QAE9C,yCAAyC;QACzC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;YACpB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAA;YACtB,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAA;YACnD,iGAAiG;YACjG,4FAA4F;YAC5F,IAAI,CAAC,WAAW,EAAE,CAAA;SACnB;QAED,OAAO,aAAa,CAAC,OAAO,CAAA;IAC9B,CAAC;IAED,+EAA+E;IACvE,KAAK,CAAC,WAAW;QACvB,8GAA8G;QAC9G,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAA;QAEhC,IAAI,CAAC,IAAI,EAAE;YACT,MAAM,CAAC,KAAK,CACV,mFAAmF,CACpF,CAAA;YACD,IAAI,CAAC,UAAU,GAAG,KAAK,CAAA;YACvB,OAAM;SACP;QAED,MAAM,CAAC,KAAK,CACV,6BAA6B,IAAI,CAAC,GAAG,cAAc,IAAI,CAAC,OAAO,2BAA2B,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAC/G,CAAA;QAED,qDAAqD;QACrD,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,oBAAoB,EAAE,CAAA;QAC1D,IAAI,UAAU,EAAE;YACd,MAAM,CAAC,KAAK,CACV,qCAAqC,UAAU,6BAA6B,IAAI,CAAC,GAAG,cAAc,IAAI,CAAC,OAAO,GAAG,CAClH,CAAA;YACD,MAAM,IAAA,QAAK,EAAC,UAAU,CAAC,CAAA;SACxB;QAED,mHAAmH;QACnH,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAA;QAEpC,OAAO,IAAI,CAAC,WAAW,EAAE,CAAA;IAC3B,CAAC;IAED,uHAAuH;IAC/G,KAAK,CAAC,cAAc,CAAC,GAAkB;QAC7C,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,GAAG,CAAA;QACrD,MAAM,qBAAqB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACxC,MAAM,aAAa,GAAG,OAAO,CAAC,kCAAkC,CAAC,UAAU,EAAE,CAAA;QAE7E,oEAAoE;QACpE,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAA;QAE/C,IAAI;YACF,MAAM,CAAC,KAAK,CAAC,yBAAyB,GAAG,oBAAoB,CAAC,CAAA;YAC9D,MAAM,QAAQ,GAAG,MAAM,eAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;YAC5C,MAAM,CAAC,KAAK,CAAC,iBAAiB,GAAG,mBAAmB,CAAC,CAAA;YACrD,OAAO,CAAC;gBACN,QAAQ;gBACR,UAAU,EAAE;oBACV,qBAAqB;oBACrB,oBAAoB,EAAE,IAAI,CAAC,GAAG,EAAE;iBACjC;aACF,CAAC,CAAA;YAEF,kCAAkC;YAClC,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAEpB,oDAAoD;YACpD,OAAO,CAAC,oBAAoB;iBACzB,MAAM,CAAC,OAAO,CAAC,wBAAwB,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;iBACxE,GAAG,EAAE,CAAA;SACT;QAAC,OAAO,CAAC,EAAE;YACV,IAAI,OAAO,IAAI,IAAI,CAAC,UAAU,EAAE;gBAC9B,MAAM,CAAC,KAAK,CAAC,oEAAoE,CAAC,CAAA;gBAClF,MAAM,GAAG,GAAG,CAAe,CAAA;gBAC3B,MAAM,UAAU,GAAG,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,gCAAwB,CAAC,CAAC,CAAC,8BAAsB,CAAA;gBAE3F,+CAA+C;gBAC/C,OAAO,CAAC,oBAAoB;qBACzB,MAAM,CAAC,OAAO,CAAC,wBAAwB,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;qBAClF,GAAG,EAAE,CAAA;gBAER,MAAM,CACJ,IAAI,UAAU,CACZ;oBACE,UAAU,EAAE,GAAG;oBACf,IAAI,EAAE,qBAAqB;oBAC3B,kBAAkB,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,IAAI,GAAG;oBAChD,OAAO,EAAE,GAAG,EAAE,OAAO;oBACrB,KAAK,EAAE,CAAC;oBACR,aAAa,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI;oBAClC,GAAG,EAAE,MAAM,CAAC,GAAG;iBAChB,EACD;oBACE,qBAAqB;oBACrB,oBAAoB,EAAE,IAAI,CAAC,GAAG,EAAE;oBAChC,qBAAqB,EAAE,SAAS;iBACjC,CACF,CACF,CAAA;gBAED,kCAAkC;gBAClC,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;aACrB;iBAAM;gBACL,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,GAAG,IAAI,CAAA;gBACzD,MAAM,CAAC,KAAK,CAAC,gCAAgC,WAAW,OAAO,CAAC,CAAA;gBAChE,MAAM,IAAA,QAAK,EAAC,WAAW,CAAC,CAAA;gBAExB,GAAG,CAAC,OAAO,EAAE,CAAA;gBACb,MAAM,CAAC,KAAK,CAAC,qCAAqC,GAAG,cAAc,GAAG,CAAC,OAAO,GAAG,CAAC,CAAA;gBAClF,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;aACpB;SACF;gBAAS;YACR,qEAAqE;YACrE,aAAa,EAAE,CAAA;SAChB;IACH,CAAC;CACF;AA/LD,8BA+LC"}
|