@chainlink/external-adapter-framework 0.7.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adapter/basic.d.ts +6 -50
- package/adapter/basic.js +2 -165
- package/adapter/basic.js.map +1 -1
- package/adapter/endpoint.d.ts +6 -6
- package/adapter/endpoint.js +0 -1
- package/adapter/endpoint.js.map +1 -1
- package/adapter/price.d.ts +8 -38
- package/adapter/price.js +1 -43
- package/adapter/price.js.map +1 -1
- package/adapter/types.d.ts +14 -37
- package/background-executor.js +4 -19
- package/background-executor.js.map +1 -1
- package/cache/index.d.ts +16 -9
- package/cache/index.js +46 -4
- package/cache/index.js.map +1 -1
- package/config/index.d.ts +1 -19
- package/config/index.js +29 -54
- package/config/index.js.map +1 -1
- package/examples/bank-frick/accounts.d.ts +25 -17
- package/examples/bank-frick/accounts.js +13 -23
- package/examples/bank-frick/accounts.js.map +1 -1
- package/examples/bank-frick/index.d.ts +15 -1
- package/examples/bank-frick/index.js +1 -1
- package/examples/coingecko/src/crypto-utils.d.ts +4 -18
- package/examples/coingecko/src/crypto-utils.js +2 -1
- package/examples/coingecko/src/crypto-utils.js.map +1 -1
- package/examples/coingecko/src/endpoint/coins.d.ts +3 -20
- package/examples/coingecko/src/endpoint/coins.js +2 -2
- package/examples/coingecko/src/endpoint/coins.js.map +1 -1
- package/examples/coingecko/src/endpoint/crypto-marketcap.d.ts +2 -2
- package/examples/coingecko/src/endpoint/crypto-marketcap.js +2 -2
- package/examples/coingecko/src/endpoint/crypto-marketcap.js.map +1 -1
- package/examples/coingecko/src/endpoint/crypto-volume.d.ts +2 -2
- package/examples/coingecko/src/endpoint/crypto-volume.js +2 -2
- package/examples/coingecko/src/endpoint/crypto-volume.js.map +1 -1
- package/examples/coingecko/src/endpoint/crypto.d.ts +4 -2
- package/examples/coingecko/src/endpoint/crypto.js +2 -2
- package/examples/coingecko/src/endpoint/crypto.js.map +1 -1
- package/examples/coingecko/src/endpoint/dominance.d.ts +2 -2
- package/examples/coingecko/src/endpoint/dominance.js +2 -2
- package/examples/coingecko/src/endpoint/dominance.js.map +1 -1
- package/examples/coingecko/src/endpoint/global-marketcap.d.ts +2 -2
- package/examples/coingecko/src/endpoint/global-marketcap.js +2 -2
- package/examples/coingecko/src/endpoint/global-marketcap.js.map +1 -1
- package/examples/coingecko/src/global-utils.d.ts +3 -18
- package/examples/coingecko/src/global-utils.js.map +1 -1
- package/examples/coingecko/src/index.js +1 -1
- package/examples/coingecko-old/batch-warming.d.ts +7 -0
- package/examples/coingecko-old/batch-warming.js +54 -0
- package/examples/coingecko-old/batch-warming.js.map +1 -0
- package/examples/coingecko-old/index.d.ts +2 -0
- package/examples/coingecko-old/index.js +12 -0
- package/examples/coingecko-old/index.js.map +1 -0
- package/examples/coingecko-old/rest.d.ts +12 -0
- package/examples/coingecko-old/rest.js +55 -0
- package/examples/coingecko-old/rest.js.map +1 -0
- package/examples/cryptocompare/src/endpoints/crypto.d.ts +2 -14
- package/examples/cryptocompare/src/endpoints/crypto.js.map +1 -1
- package/examples/cryptocompare/src/index.js +1 -1
- package/examples/genesis/index.js +1 -1
- package/examples/genesis/sseStream.d.ts +2 -13
- package/examples/genesis/sseStream.js +1 -4
- package/examples/genesis/sseStream.js.map +1 -1
- package/examples/ncfx/config/index.d.ts +12 -0
- package/examples/ncfx/config/index.js +16 -0
- package/examples/ncfx/config/index.js.map +1 -0
- package/examples/ncfx/index.d.ts +13 -0
- package/examples/ncfx/index.js +12 -0
- package/examples/ncfx/index.js.map +1 -0
- package/examples/ncfx/websocket.d.ts +47 -0
- package/examples/ncfx/websocket.js +74 -0
- package/examples/ncfx/websocket.js.map +1 -0
- package/index.d.ts +0 -1
- package/index.js +6 -4
- package/index.js.map +1 -1
- package/metrics/index.d.ts +0 -2
- package/metrics/index.js +4 -5
- package/metrics/index.js.map +1 -1
- package/metrics/util.d.ts +5 -5
- package/metrics/util.js.map +1 -1
- package/package.json +9 -7
- package/rate-limiting/background/fixed-frequency.d.ts +3 -2
- package/rate-limiting/background/fixed-frequency.js.map +1 -1
- package/rate-limiting/index.d.ts +4 -4
- package/rate-limiting/index.js +1 -1
- package/rate-limiting/index.js.map +1 -1
- package/rate-limiting/request/simple-counting.d.ts +3 -2
- package/rate-limiting/request/simple-counting.js.map +1 -1
- package/transports/batch-warming.d.ts +15 -33
- package/transports/batch-warming.js +16 -43
- package/transports/batch-warming.js.map +1 -1
- package/transports/index.d.ts +31 -65
- package/transports/index.js +59 -2
- package/transports/index.js.map +1 -1
- package/transports/metrics.d.ts +2 -2
- package/transports/metrics.js +2 -2
- package/transports/metrics.js.map +1 -1
- package/transports/rest.d.ts +14 -31
- package/transports/rest.js +25 -17
- package/transports/rest.js.map +1 -1
- package/transports/sse.d.ts +20 -40
- package/transports/sse.js +8 -10
- package/transports/sse.js.map +1 -1
- package/transports/websocket.d.ts +25 -40
- package/transports/websocket.js +13 -10
- package/transports/websocket.js.map +1 -1
- package/util/logger.d.ts +0 -2
- package/util/logger.js +7 -4
- package/util/logger.js.map +1 -1
- package/util/request.d.ts +10 -46
- package/util/subscription-set/subscription-set.d.ts +1 -1
- package/util/subscription-set/subscription-set.js +1 -1
- package/util/subscription-set/subscription-set.js.map +1 -1
- package/util/test-payload-loader.d.ts +0 -1
- package/util/test-payload-loader.js +1 -2
- package/util/test-payload-loader.js.map +1 -1
- package/validation/error.d.ts +3 -1
- package/validation/error.js +4 -2
- package/validation/error.js.map +1 -1
- package/validation/index.js +17 -26
- package/validation/index.js.map +1 -1
- package/validation/input-params.d.ts +1 -0
- package/validation/input-params.js +28 -0
- package/validation/input-params.js.map +1 -1
- package/validation/input-validator.js +2 -1
- package/validation/input-validator.js.map +1 -1
- package/validation/override-functions.d.ts +3 -0
- package/validation/override-functions.js +41 -0
- package/validation/override-functions.js.map +1 -0
- package/README.md +0 -67
- package/transports/routing.d.ts +0 -21
- package/transports/routing.js +0 -50
- package/transports/routing.js.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fixed-frequency.js","sourceRoot":"","sources":["../../../../src/rate-limiting/background/fixed-frequency.ts"],"names":[],"mappings":";;;AAAA,0BAA8F;
|
|
1
|
+
{"version":3,"file":"fixed-frequency.js","sourceRoot":"","sources":["../../../../src/rate-limiting/background/fixed-frequency.ts"],"names":[],"mappings":";;;AAAA,0BAA8F;AAG9F,qCAAuC;AAEvC,MAAM,MAAM,GAAG,IAAA,iBAAU,EAAC,2BAA2B,CAAC,CAAA;AACzC,QAAA,kCAAkC,GAAG,IAAI,CAAA;AAEtD,MAAa,yBAAyB;IAAtC;QACE,yBAAoB,GAEhB,EAAE,CAAA;IAqCR,CAAC;IAnCC,UAAU,CACR,SAA4D,EAC5D,MAA6B;QAE7B,kDAAkD;QAClD,IAAI,uBAAuB,GAAG,IAAI,GAAG,IAAA,yBAAqB,EAAC,MAAM,CAAC,CAAA;QAElE,0DAA0D;QAC1D,IAAI,CAAC,MAAM,EAAE,WAAW,IAAI,CAAC,MAAM,EAAE,WAAW,IAAI,CAAC,MAAM,EAAE,WAAW,EAAE;YACxE,+BAA+B;YAC/B,uBAAuB,GAAG,0CAAkC,CAAA;SAC7D;QAED,MAAM,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAA;QACzD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE;YAChC,IAAI,QAAQ,CAAC,YAAY,EAAE,oBAAoB,IAAI,IAAI,EAAE;gBACvD,MAAM,IAAI,KAAK,CAAC,uCAAuC,QAAQ,CAAC,IAAI,WAAW,CAAC,CAAA;aACjF;YAED,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,IAAI,CAAC;gBACtC,CAAC,uBAAuB,GAAG,QAAQ,CAAC,YAAY,EAAE,oBAAoB,CAAC,GAAG,GAAG,CAAA;YAE/E,MAAM,CAAC,KAAK,CACV,aAAa,QAAQ,CAAC,IAAI,MACxB,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,IAC7C,oBAAoB,CACrB,CAAA;SACF;QAED,OAAO,IAAI,CAAA;IACb,CAAC;IAED,oBAAoB,CAAC,YAAoB;QACvC,OAAO,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAA;IAChD,CAAC;CACF;AAxCD,8DAwCC"}
|
package/rate-limiting/index.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { AdapterEndpoint
|
|
2
|
-
import { AdapterConfig } from '../config';
|
|
3
|
-
export * from './background/fixed-frequency';
|
|
1
|
+
import { AdapterEndpoint } from '../adapter';
|
|
2
|
+
import { AdapterConfig, SettingsMap } from '../config';
|
|
4
3
|
export * from './request/simple-counting';
|
|
4
|
+
export * from './background/fixed-frequency';
|
|
5
5
|
export interface AdapterRateLimitTier {
|
|
6
6
|
rateLimit1s?: number;
|
|
7
7
|
rateLimit1m?: number;
|
|
@@ -18,7 +18,7 @@ export interface RateLimiter {
|
|
|
18
18
|
* @param limits - settings for how much throughput to allow for the Adapter
|
|
19
19
|
* @param endpoints - list of adapter endpoints
|
|
20
20
|
*/
|
|
21
|
-
initialize<
|
|
21
|
+
initialize<Params, Result, CustomSettings extends SettingsMap>(endpoints: AdapterEndpoint<Params, Result, CustomSettings>[], limits: AdapterRateLimitTier): this;
|
|
22
22
|
}
|
|
23
23
|
/**
|
|
24
24
|
* RequestRateLimiters perform checks agains imminent outbound requests for any transport.
|
package/rate-limiting/index.js
CHANGED
|
@@ -15,8 +15,8 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
17
|
exports.buildRateLimitTiersFromConfig = exports.getRateLimitingTier = exports.consolidateTierLimits = void 0;
|
|
18
|
-
__exportStar(require("./background/fixed-frequency"), exports);
|
|
19
18
|
__exportStar(require("./request/simple-counting"), exports);
|
|
19
|
+
__exportStar(require("./background/fixed-frequency"), exports);
|
|
20
20
|
/**
|
|
21
21
|
* This method will convert all possible settings for a rate limit tier and
|
|
22
22
|
* convert them all to requests per second, returning the lowest one
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/rate-limiting/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;AAGA,+DAA4C;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/rate-limiting/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;AAGA,4DAAyC;AACzC,+DAA4C;AA4C5C;;;;;;GAMG;AACI,MAAM,qBAAqB,GAAG,CAAC,MAA6B,EAAE,EAAE;IACrE,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,WAAW,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAA;IAClE,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,WAAW,IAAI,QAAQ,CAAC,GAAG,EAAE,CAAA;IAC7D,MAAM,cAAc,GAAG,MAAM,EAAE,WAAW,IAAI,QAAQ,CAAA;IACtD,OAAO,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,cAAc,EAAE,cAAc,CAAC,CAAA;AAC/D,CAAC,CAAA;AALY,QAAA,qBAAqB,yBAKjC;AAED;;;;;;GAMG;AACI,MAAM,mBAAmB,GAAG,CACjC,MAAqB,EACrB,KAA4C,EACV,EAAE;IACpC,IACE,MAAM,CAAC,mBAAmB;QAC1B,MAAM,CAAC,0BAA0B;QACjC,MAAM,CAAC,0BAA0B,EACjC;QACA,OAAO,IAAA,qCAA6B,EAAC,MAAM,CAAC,CAAA;KAC7C;IACD,IAAI,CAAC,KAAK,EAAE;QACV,OAAM;KACP;IAED,2DAA2D;IAC3D,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE;QACrC,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAA;KACnE;IAED,8DAA8D;IAC9D,MAAM,YAAY,GAAG,MAAM,CAAC,mBAAmB,CAAA;IAC/C,IAAI,YAAY,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE;QACxC,MAAM,gBAAgB,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;aACxC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC;aACpB,IAAI,CAAC,IAAI,CAAC,CAAA;QAEb,MAAM,IAAI,KAAK,CACb,iCAAiC,YAAY,iCAAiC,gBAAgB,GAAG,CAClG,CAAA;KACF;IAED,IAAI,CAAC,YAAY,EAAE;QACjB,8CAA8C;QAC9C,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAC3C,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,IAAA,6BAAqB,EAAC,EAAE,CAAC,GAAG,IAAA,6BAAqB,EAAC,EAAE,CAAC,CAClE,CAAA;QAED,OAAO,WAAW,CAAC,CAAC,CAAC,CAAA;KACtB;IAED,OAAO,KAAK,CAAC,YAAY,CAAC,CAAA;AAC5B,CAAC,CAAA;AA1CY,QAAA,mBAAmB,uBA0C/B;AAED,0EAA0E;AACnE,MAAM,6BAA6B,GAAG,CAC3C,MAAqB,EACa,EAAE;IACpC,MAAM,WAAW,GAAG,MAAM,CAAC,0BAA0B,CAAA;IACrD,IAAI,WAAW,CAAA;IACf,IAAI,MAAM,CAAC,0BAA0B,EAAE;QACrC,WAAW,GAAG,MAAM,CAAC,0BAA0B,CAAA;KAChD;SAAM,IAAI,MAAM,CAAC,mBAAmB,EAAE;QACrC,WAAW,GAAG,MAAM,CAAC,mBAAmB,CAAA;KACzC;IACD,OAAO;QACL,WAAW;QACX,WAAW;KACZ,CAAA;AACH,CAAC,CAAA;AAdY,QAAA,6BAA6B,iCAczC"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { AdapterRateLimitTier, RateLimiter } from '..';
|
|
2
|
-
import { AdapterEndpoint
|
|
2
|
+
import { AdapterEndpoint } from '../../adapter';
|
|
3
|
+
import { SettingsMap } from '../../config';
|
|
3
4
|
/**
|
|
4
5
|
* This rate limiter is the simplest stateful option.
|
|
5
6
|
* On startup, it'll compare the different thresholds for each tier, calculate them all
|
|
@@ -15,6 +16,6 @@ export declare class SimpleCountingRateLimiter implements RateLimiter {
|
|
|
15
16
|
requestsThisMinute: number;
|
|
16
17
|
perSecondLimit: number;
|
|
17
18
|
perMinuteLimit: number;
|
|
18
|
-
initialize<
|
|
19
|
+
initialize<Params, Result, CustomSettings extends SettingsMap>(endpoints: AdapterEndpoint<Params, Result, CustomSettings>[], limits?: AdapterRateLimitTier): this;
|
|
19
20
|
isUnderLimits(): boolean;
|
|
20
21
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"simple-counting.js","sourceRoot":"","sources":["../../../../src/rate-limiting/request/simple-counting.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"simple-counting.js","sourceRoot":"","sources":["../../../../src/rate-limiting/request/simple-counting.ts"],"names":[],"mappings":";;;AAGA,qCAAuC;AAEvC,MAAM,MAAM,GAAG,IAAA,iBAAU,EAAC,2BAA2B,CAAC,CAAA;AAEtD;;;;;;;GAOG;AACH,MAAa,yBAAyB;IAAtC;QACE,yBAAoB,GAAG,CAAC,CAAA;QACxB,uBAAkB,GAAG,CAAC,CAAA;QACtB,yBAAoB,GAAG,CAAC,CAAA;QACxB,uBAAkB,GAAG,CAAC,CAAA;IA4DxB,CAAC;IAxDC,UAAU,CACR,SAA4D,EAC5D,MAA6B;QAE7B,kDAAkD;QAClD,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,WAAW,IAAI,QAAQ,CAAC,GAAG,EAAE,CAAA;QAC3D,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,IAAI,QAAQ,EAAE,YAAY,CAAC,CAAA;QAC7E,IAAI,CAAC,cAAc,GAAG,MAAM,EAAE,WAAW,IAAI,QAAQ,CAAA;QACrD,MAAM,CAAC,KAAK,CACV,6CAA6C,IAAI,CAAC,cAAc,mBAAmB,IAAI,CAAC,cAAc,EAAE,CACzG,CAAA;QAED,OAAO,IAAI,CAAA;IACb,CAAC;IAED,aAAa;QACX,qEAAqE;QACrE,IAAI,IAAI,CAAC,cAAc,KAAK,QAAQ,IAAI,IAAI,CAAC,cAAc,KAAK,QAAQ,EAAE;YACxE,OAAO,IAAI,CAAA;SACZ;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACtB,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,CAAA;QACpD,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,CAAA;QAE3D,sFAAsF;QACtF,mFAAmF;QACnF,IAAI,qBAAqB,KAAK,IAAI,CAAC,oBAAoB,EAAE;YACvD,MAAM,CAAC,KAAK,CACV,8DAA8D,IAAI,CAAC,kBAAkB,GAAG,CACzF,CAAA;YACD,IAAI,CAAC,oBAAoB,GAAG,qBAAqB,CAAA;YACjD,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAA;SAC5B;QAED,IAAI,qBAAqB,KAAK,IAAI,CAAC,oBAAoB,EAAE;YACvD,MAAM,CAAC,KAAK,CACV,4DAA4D,IAAI,CAAC,kBAAkB,GAAG,CACvF,CAAA;YACD,IAAI,CAAC,oBAAoB,GAAG,qBAAqB,CAAA;YACjD,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAA;SAC5B;QAED,IACE,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,cAAc;YAC7C,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,cAAc,EAC7C;YACA,MAAM,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAA;YACjD,IAAI,CAAC,kBAAkB,EAAE,CAAA;YACzB,IAAI,CAAC,kBAAkB,EAAE,CAAA;YACzB,OAAO,IAAI,CAAA;SACZ;aAAM;YACL,MAAM,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAA;YAC5D,OAAO,KAAK,CAAA;SACb;IACH,CAAC;CACF;AAhED,8DAgEC"}
|
|
@@ -1,30 +1,11 @@
|
|
|
1
1
|
import { AxiosRequestConfig, AxiosResponse } from 'axios';
|
|
2
2
|
import { Cache } from '../cache';
|
|
3
|
-
import { AdapterConfig } from '../config';
|
|
3
|
+
import { AdapterConfig, SettingsMap } from '../config';
|
|
4
4
|
import { BackgroundExecuteRateLimiter } from '../rate-limiting';
|
|
5
5
|
import { SubscriptionSet } from '../util';
|
|
6
|
-
import { AdapterRequest,
|
|
7
|
-
import { Transport
|
|
6
|
+
import { AdapterRequest, ProviderResult } from '../util/request';
|
|
7
|
+
import { Transport } from './';
|
|
8
8
|
import { AdapterContext, AdapterDependencies } from '../adapter';
|
|
9
|
-
/**
|
|
10
|
-
* Helper struct type that will be used to pass types to the generic parameters of a Transport.
|
|
11
|
-
* Extends the common TransportGenerics, adding Provider specific types for this Batch endpoint.
|
|
12
|
-
*/
|
|
13
|
-
declare type BatchWarmingTransportGenerics = TransportGenerics & {
|
|
14
|
-
/**
|
|
15
|
-
* Type details for any provider specific interfaces.
|
|
16
|
-
*/
|
|
17
|
-
Provider: {
|
|
18
|
-
/**
|
|
19
|
-
* Structure of the body of the request that will be sent to the data provider.
|
|
20
|
-
*/
|
|
21
|
-
RequestBody: unknown;
|
|
22
|
-
/**
|
|
23
|
-
* Structure for the body of the response coming from the data provider.
|
|
24
|
-
*/
|
|
25
|
-
ResponseBody: unknown;
|
|
26
|
-
};
|
|
27
|
-
};
|
|
28
9
|
/**
|
|
29
10
|
* Transport implementation that takes incoming batches requests and keeps a warm cache of values.
|
|
30
11
|
* Within the setup function, adapter params are added to an set that also keeps track and expires values.
|
|
@@ -33,21 +14,22 @@ declare type BatchWarmingTransportGenerics = TransportGenerics & {
|
|
|
33
14
|
* The Data Provider response is, they are passed through the `parseResponse` function to create a [[CacheEntry]] list.
|
|
34
15
|
* Finally, the items in that [[CacheEntry]] list are set in the Cache so the Adapter can fetch values from there.
|
|
35
16
|
*
|
|
36
|
-
* @typeParam
|
|
17
|
+
* @typeParam AdapterParams - interface for the adapter request body
|
|
18
|
+
* @typeParam ProviderRequestBody - interface for the body of the request to the Data Provider
|
|
19
|
+
* @typeParam ProviderResponseBody - interface for the body of the Data Provider's response
|
|
37
20
|
*/
|
|
38
|
-
export declare class BatchWarmingTransport<
|
|
21
|
+
export declare class BatchWarmingTransport<AdapterParams, ProviderRequestBody, ProviderResponseBody, CustomSettings extends SettingsMap> implements Transport<AdapterParams, ProviderResponseBody, CustomSettings> {
|
|
39
22
|
private config;
|
|
40
|
-
cache: Cache
|
|
23
|
+
cache: Cache;
|
|
41
24
|
rateLimiter: BackgroundExecuteRateLimiter;
|
|
42
|
-
subscriptionSet: SubscriptionSet<
|
|
25
|
+
subscriptionSet: SubscriptionSet<AdapterParams>;
|
|
43
26
|
WARMER_ACTIVE: boolean;
|
|
44
27
|
constructor(config: {
|
|
45
|
-
prepareRequest: (params:
|
|
46
|
-
parseResponse: (params:
|
|
28
|
+
prepareRequest: (params: AdapterParams[], context: AdapterContext<CustomSettings>) => AxiosRequestConfig<ProviderRequestBody>;
|
|
29
|
+
parseResponse: (params: AdapterParams[], res: AxiosResponse<ProviderResponseBody>, context: AdapterContext<CustomSettings>) => ProviderResult<AdapterParams>[];
|
|
47
30
|
});
|
|
48
|
-
initialize(dependencies: AdapterDependencies, _: AdapterConfig
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
31
|
+
initialize(dependencies: AdapterDependencies, _: AdapterConfig, endpointName: string): Promise<void>;
|
|
32
|
+
hasBeenSetUp(): Promise<boolean>;
|
|
33
|
+
setup(req: AdapterRequest<AdapterParams>, config: AdapterConfig<CustomSettings>): Promise<void>;
|
|
34
|
+
backgroundExecute(context: AdapterContext<CustomSettings>): Promise<number>;
|
|
52
35
|
}
|
|
53
|
-
export {};
|
|
@@ -30,7 +30,6 @@ const util_2 = require("./util");
|
|
|
30
30
|
const rateLimitMetrics = __importStar(require("../rate-limiting/metrics"));
|
|
31
31
|
const cacheMetrics = __importStar(require("../cache/metrics"));
|
|
32
32
|
const transportMetrics = __importStar(require("./metrics"));
|
|
33
|
-
const error_1 = require("../validation/error");
|
|
34
33
|
const WARMUP_BATCH_REQUEST_ID = '9002';
|
|
35
34
|
const logger = (0, util_1.makeLogger)('BatchWarmingTransport');
|
|
36
35
|
/**
|
|
@@ -41,7 +40,9 @@ const logger = (0, util_1.makeLogger)('BatchWarmingTransport');
|
|
|
41
40
|
* The Data Provider response is, they are passed through the `parseResponse` function to create a [[CacheEntry]] list.
|
|
42
41
|
* Finally, the items in that [[CacheEntry]] list are set in the Cache so the Adapter can fetch values from there.
|
|
43
42
|
*
|
|
44
|
-
* @typeParam
|
|
43
|
+
* @typeParam AdapterParams - interface for the adapter request body
|
|
44
|
+
* @typeParam ProviderRequestBody - interface for the body of the request to the Data Provider
|
|
45
|
+
* @typeParam ProviderResponseBody - interface for the body of the Data Provider's response
|
|
45
46
|
*/
|
|
46
47
|
class BatchWarmingTransport {
|
|
47
48
|
constructor(config) {
|
|
@@ -56,19 +57,11 @@ class BatchWarmingTransport {
|
|
|
56
57
|
this.subscriptionSet = dependencies.subscriptionSetFactory.buildSet(endpointName);
|
|
57
58
|
}
|
|
58
59
|
// This might need coalescing to avoid too frequent ttl updates
|
|
59
|
-
async
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
throw new error_1.AdapterError({
|
|
65
|
-
statusCode: 200,
|
|
66
|
-
providerStatusCode: response.providerResponse.status,
|
|
67
|
-
message: response.providerResponse.message ||
|
|
68
|
-
'There was an error while validating the incoming request before adding to the batch subscription set',
|
|
69
|
-
});
|
|
70
|
-
}
|
|
71
|
-
}
|
|
60
|
+
async hasBeenSetUp() {
|
|
61
|
+
return false;
|
|
62
|
+
}
|
|
63
|
+
async setup(req, config) {
|
|
64
|
+
logger.debug(`Adding entry to batch warming set: [${req.requestContext.cacheKey}] = ${req.requestContext.data}`);
|
|
72
65
|
await this.subscriptionSet.add(req.requestContext.cacheKey, req.requestContext.data, config.WARMUP_SUBSCRIPTION_TTL);
|
|
73
66
|
}
|
|
74
67
|
async backgroundExecute(context) {
|
|
@@ -93,15 +86,16 @@ class BatchWarmingTransport {
|
|
|
93
86
|
cacheMetrics.cacheWarmerCount.labels({ isBatched: 'true' }).inc();
|
|
94
87
|
this.WARMER_ACTIVE = true;
|
|
95
88
|
}
|
|
96
|
-
const
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
}
|
|
100
|
-
const
|
|
89
|
+
const request = this.config.prepareRequest(entries, context);
|
|
90
|
+
logger.trace('Sending request to data provider...');
|
|
91
|
+
const providerResponse = await (0, util_2.axiosRequest)(request, context.adapterConfig);
|
|
92
|
+
logger.debug(`Got response from provider, parsing (raw body: ${providerResponse.data})`);
|
|
93
|
+
const results = this.config.parseResponse(entries, providerResponse, context);
|
|
94
|
+
const adapterResponses = (0, _1.buildCacheEntriesFromResults)(results, context);
|
|
101
95
|
logger.debug('Setting adapter responses in cache');
|
|
102
|
-
await this.cache.setMany(
|
|
96
|
+
await this.cache.setMany(adapterResponses, context.adapterConfig.CACHE_MAX_AGE);
|
|
103
97
|
// Record cost of data provider call
|
|
104
|
-
const cost = rateLimitMetrics.retrieveCost(
|
|
98
|
+
const cost = rateLimitMetrics.retrieveCost(providerResponse.data);
|
|
105
99
|
rateLimitMetrics.rateLimitCreditsSpentTotal
|
|
106
100
|
.labels({
|
|
107
101
|
feed_id: 'N/A',
|
|
@@ -110,27 +104,6 @@ class BatchWarmingTransport {
|
|
|
110
104
|
.inc(cost);
|
|
111
105
|
return interval;
|
|
112
106
|
}
|
|
113
|
-
async makeRequest(entries, config) {
|
|
114
|
-
logger.trace(`Have ${entries.length} entries in batch, preparing request...`);
|
|
115
|
-
const request = this.config.prepareRequest(entries, config);
|
|
116
|
-
logger.trace(`Sending request to data provider: ${JSON.stringify(request)}`);
|
|
117
|
-
let providerResponse;
|
|
118
|
-
try {
|
|
119
|
-
providerResponse = await (0, util_2.axiosRequest)(request, config);
|
|
120
|
-
}
|
|
121
|
-
catch (e) {
|
|
122
|
-
logger.warn(`There was an error while performing the batch request: ${e}`);
|
|
123
|
-
return {
|
|
124
|
-
results: [],
|
|
125
|
-
providerResponse: e,
|
|
126
|
-
};
|
|
127
|
-
}
|
|
128
|
-
logger.debug(`Got response from provider, parsing (raw body: ${providerResponse.data})`);
|
|
129
|
-
return {
|
|
130
|
-
results: this.config.parseResponse(entries, providerResponse, config),
|
|
131
|
-
providerResponse,
|
|
132
|
-
};
|
|
133
|
-
}
|
|
134
107
|
}
|
|
135
108
|
exports.BatchWarmingTransport = BatchWarmingTransport;
|
|
136
109
|
//# sourceMappingURL=batch-warming.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"batch-warming.js","sourceRoot":"","sources":["../../../src/transports/batch-warming.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAIA,kCAAqD;AAErD,
|
|
1
|
+
{"version":3,"file":"batch-warming.js","sourceRoot":"","sources":["../../../src/transports/batch-warming.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAIA,kCAAqD;AAErD,yBAA4D;AAC5D,iCAAqC;AACrC,2EAA4D;AAC5D,+DAAgD;AAEhD,4DAA6C;AAE7C,MAAM,uBAAuB,GAAG,MAAM,CAAA;AAEtC,MAAM,MAAM,GAAG,IAAA,iBAAU,EAAC,uBAAuB,CAAC,CAAA;AAElD;;;;;;;;;;;GAWG;AACH,MAAa,qBAAqB;IAehC,YACU,MAUP;QAVO,WAAM,GAAN,MAAM,CAUb;QAfH,uGAAuG;QACvG,gEAAgE;QAChE,kBAAa,GAAG,KAAK,CAAA;IAclB,CAAC;IAEJ,KAAK,CAAC,UAAU,CACd,YAAiC,EACjC,CAAgB,EAChB,YAAoB;QAEpB,IAAI,CAAC,KAAK,GAAG,YAAY,CAAC,KAAK,CAAA;QAC/B,IAAI,CAAC,WAAW,GAAG,YAAY,CAAC,4BAA4B,CAAA;QAC5D,IAAI,CAAC,eAAe,GAAG,YAAY,CAAC,sBAAsB,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAA;IACnF,CAAC;IAED,+DAA+D;IAC/D,KAAK,CAAC,YAAY;QAChB,OAAO,KAAK,CAAA;IACd,CAAC;IAED,KAAK,CAAC,KAAK,CACT,GAAkC,EAClC,MAAqC;QAErC,MAAM,CAAC,KAAK,CACV,uCAAuC,GAAG,CAAC,cAAc,CAAC,QAAQ,OAAO,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CACnG,CAAA;QACD,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,CAC5B,GAAG,CAAC,cAAc,CAAC,QAAQ,EAC3B,GAAG,CAAC,cAAc,CAAC,IAAI,EACvB,MAAM,CAAC,uBAAuB,CAC/B,CAAA;IACH,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,OAAuC;QAC7D,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAA;QAC3C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAA;QAEnD,4DAA4D;QAC5D,gBAAgB,CAAC,6BAA6B;aAC3C,MAAM,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,eAAe,CAAC,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC;aAC3E,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;QAEtB,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,oBAAoB,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,CAAA;QAEpF,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YACnB,MAAM,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAA;YACzD,IAAI,IAAI,CAAC,aAAa,EAAE;gBACtB,yEAAyE;gBACzE,YAAY,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAA;gBACjE,IAAI,CAAC,aAAa,GAAG,KAAK,CAAA;aAC3B;YACD,OAAO,QAAQ,CAAA;SAChB;aAAM,IAAI,IAAI,CAAC,aAAa,KAAK,KAAK,EAAE;YACvC,4EAA4E;YAC5E,YAAY,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAA;YACjE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAA;SAC1B;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QAE5D,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAA;QACnD,MAAM,gBAAgB,GAAG,MAAM,IAAA,mBAAY,EAIzC,OAAO,EAAE,OAAO,CAAC,aAAa,CAAC,CAAA;QAEjC,MAAM,CAAC,KAAK,CAAC,kDAAkD,gBAAgB,CAAC,IAAI,GAAG,CAAC,CAAA;QACxF,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,gBAAgB,EAAE,OAAO,CAAC,CAAA;QAC7E,MAAM,gBAAgB,GAAG,IAAA,+BAA4B,EAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QAEvE,MAAM,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAA;QAClD,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,gBAAgB,EAAE,OAAO,CAAC,aAAa,CAAC,aAAa,CAAC,CAAA;QAE/E,oCAAoC;QACpC,MAAM,IAAI,GAAG,gBAAgB,CAAC,YAAY,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAA;QACjE,gBAAgB,CAAC,0BAA0B;aACxC,MAAM,CAAC;YACN,OAAO,EAAE,KAAK;YACd,cAAc,EAAE,uBAAuB;SACxC,CAAC;aACD,GAAG,CAAC,IAAI,CAAC,CAAA;QAEZ,OAAO,QAAQ,CAAA;IACjB,CAAC;CACF;AA9GD,sDA8GC"}
|
package/transports/index.d.ts
CHANGED
|
@@ -1,33 +1,12 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { FastifyReply } from 'fastify';
|
|
2
|
+
import { AdapterContext, AdapterDependencies, Adapter } from '../adapter';
|
|
3
|
+
import { CacheEntry } from '../cache';
|
|
3
4
|
import { AdapterConfig, SettingsMap } from '../config';
|
|
4
|
-
import { AdapterRequest, AdapterResponse, ProviderResult
|
|
5
|
+
import { AdapterRequest, AdapterResponse, ProviderResult } from '../util/request';
|
|
5
6
|
export * from './batch-warming';
|
|
6
7
|
export * from './rest';
|
|
7
|
-
export * from './sse';
|
|
8
8
|
export * from './websocket';
|
|
9
|
-
|
|
10
|
-
* Helper struct type that will be used to pass types to the generic parameters of a Transport.
|
|
11
|
-
* Within the transport, given a generic param T extends TransportGenerics, these will be used like so:
|
|
12
|
-
*
|
|
13
|
-
* const request: AdapterRequest\<T['Request']\> = ...
|
|
14
|
-
*
|
|
15
|
-
* (ignore the slash characters, they're just for proper doc syntax \<)
|
|
16
|
-
*/
|
|
17
|
-
export declare type TransportGenerics = {
|
|
18
|
-
/**
|
|
19
|
-
* Type details about incoming Adapter requests to this Transport
|
|
20
|
-
*/
|
|
21
|
-
Request: RequestGenerics;
|
|
22
|
-
/**
|
|
23
|
-
* Type details about outbound responses from this Transport
|
|
24
|
-
*/
|
|
25
|
-
Response: ResponseGenerics;
|
|
26
|
-
/**
|
|
27
|
-
* Type for any custom settings used for this Transport
|
|
28
|
-
*/
|
|
29
|
-
CustomSettings: SettingsMap;
|
|
30
|
-
};
|
|
9
|
+
export * from './sse';
|
|
31
10
|
/**
|
|
32
11
|
* Generic interface for a Transport.
|
|
33
12
|
* A Transport defines the way in which an AdapterEndpoint will process incoming requests to
|
|
@@ -36,10 +15,10 @@ export declare type TransportGenerics = {
|
|
|
36
15
|
* This separation gives us the ability of splitting these concerns, and optionally parallelizing
|
|
37
16
|
* the reading and writing of data to a centralized Cache.
|
|
38
17
|
*
|
|
39
|
-
* @typeParam
|
|
18
|
+
* @typeParam Params - the structure of the AdapterRequest's body
|
|
19
|
+
* @typeParam Result - the structure of the AdapterResponse's body
|
|
40
20
|
*/
|
|
41
|
-
export interface Transport<
|
|
42
|
-
cache?: Cache<AdapterResponse<T['Response']>>;
|
|
21
|
+
export interface Transport<Params, Result, CustomSettings extends SettingsMap> {
|
|
43
22
|
/**
|
|
44
23
|
* Initializes the transport in the Adapter context.
|
|
45
24
|
*
|
|
@@ -47,56 +26,33 @@ export interface Transport<T extends TransportGenerics> {
|
|
|
47
26
|
* @param config - Adapter config containing env vars
|
|
48
27
|
* @returns an empty Promise
|
|
49
28
|
*/
|
|
50
|
-
initialize: (dependencies: AdapterDependencies, config: AdapterConfig
|
|
29
|
+
initialize: (dependencies: AdapterDependencies, config: AdapterConfig, endpointName: string) => Promise<void>;
|
|
51
30
|
/**
|
|
52
|
-
*
|
|
53
|
-
* This
|
|
31
|
+
* Checks if the Transport has already set up the incoming request.
|
|
32
|
+
* This will likely use the cacheKey generated in the provided request.
|
|
54
33
|
*
|
|
55
34
|
* @param req - the incoming AdapterRequest
|
|
56
|
-
* @
|
|
57
|
-
* @returns an empty Promise
|
|
35
|
+
* @returns a Promise that returns true if setup is in place
|
|
58
36
|
*/
|
|
59
|
-
|
|
37
|
+
hasBeenSetUp: (req: AdapterRequest<Params>) => Promise<boolean>;
|
|
60
38
|
/**
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
* to this method with more than one EA instance. It's heavily encouraged to instead rework adapter logic
|
|
65
|
-
* to perform as much of the work as possible (or all of it) in the backgroundExecute method.
|
|
39
|
+
* Sets up the incoming request within the Transport.
|
|
40
|
+
* In other words, it means that the Adapter has recognized this request, and will begin
|
|
41
|
+
* fetching the necessary data from a Data Provider.
|
|
66
42
|
*
|
|
67
43
|
* @param req - the incoming AdapterRequest
|
|
68
44
|
* @param config - common configuration for the Adapter as a whole
|
|
69
45
|
* @returns a Promise that _optionally_ returns an AdapterResponse, if the Transport has the capability of
|
|
70
46
|
* immediately fetching data and returning it without the background process.
|
|
71
47
|
*/
|
|
72
|
-
|
|
73
|
-
Data: T['Response']['Data'];
|
|
74
|
-
Result: T['Response']['Result'];
|
|
75
|
-
}> | void>;
|
|
48
|
+
setup: (req: AdapterRequest<Params>, config: AdapterConfig<CustomSettings>) => Promise<AdapterResponse<Result> | void>;
|
|
76
49
|
/**
|
|
77
|
-
*
|
|
78
|
-
*
|
|
50
|
+
* If the Transport relies on an async mechanism, implementing this function will make it
|
|
51
|
+
* so the background task executor calls it on Adapter startup.
|
|
79
52
|
*
|
|
80
53
|
* @param context - background context for the execution (e.g. endpoint name)
|
|
81
|
-
* @returns an empty Promise
|
|
82
54
|
*/
|
|
83
|
-
backgroundExecute?: (context: AdapterContext<
|
|
84
|
-
}
|
|
85
|
-
/**
|
|
86
|
-
* Generic interface for a Transport that contains or manages other transports.
|
|
87
|
-
* Some transports, such as the RoutingTransport, do not directly fetch data from a Data Provider, but
|
|
88
|
-
* instead coordinate the execution of other transports for fetching and transforming data.
|
|
89
|
-
* These transports have alternative initialization procedures, which typically involve
|
|
90
|
-
* initializing each downstream transport.
|
|
91
|
-
*
|
|
92
|
-
* @typeParam Params - the structure of the AdapterRequest's body
|
|
93
|
-
* @typeParam Result - the structure of the AdapterResponse's body
|
|
94
|
-
* @typeParam CustomSettings - custom settings for the Adapter, or the default SettingsMap
|
|
95
|
-
*/
|
|
96
|
-
export interface MetaTransport<T extends TransportGenerics> extends Transport<T> {
|
|
97
|
-
transports: {
|
|
98
|
-
[key: string]: Transport<T>;
|
|
99
|
-
};
|
|
55
|
+
backgroundExecute?: (context: AdapterContext<CustomSettings>) => Promise<number>;
|
|
100
56
|
}
|
|
101
57
|
/**
|
|
102
58
|
* Helper method to build cache entries to set after getting a bunch of responses from a DP.
|
|
@@ -105,4 +61,14 @@ export interface MetaTransport<T extends TransportGenerics> extends Transport<T>
|
|
|
105
61
|
* @param context - context for the Adapter
|
|
106
62
|
* @returns a list of CacheEntries of AdapterResponses
|
|
107
63
|
*/
|
|
108
|
-
export declare const buildCacheEntriesFromResults: <
|
|
64
|
+
export declare const buildCacheEntriesFromResults: <Params, CustomSettings extends SettingsMap>(results: ProviderResult<Params>[], context: AdapterContext<CustomSettings>) => CacheEntry<AdapterResponse<{
|
|
65
|
+
result: unknown;
|
|
66
|
+
}>>[];
|
|
67
|
+
/**
|
|
68
|
+
* Takes an Adapter, its configuration, and its dependencies, and it creates an express middleware
|
|
69
|
+
* that will pass along the AdapterRequest to the appropriate Transport (acc. to the endpoint in the req.)
|
|
70
|
+
*
|
|
71
|
+
* @param adapter - main adapter object, already initialized
|
|
72
|
+
* @returns the transport handler middleware function
|
|
73
|
+
*/
|
|
74
|
+
export declare const buildTransportHandler: (adapter: Adapter) => (req: AdapterRequest, reply: FastifyReply) => Promise<never>;
|
package/transports/index.js
CHANGED
|
@@ -10,16 +10,31 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (functi
|
|
|
10
10
|
if (k2 === undefined) k2 = k;
|
|
11
11
|
o[k2] = m[k];
|
|
12
12
|
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
13
25
|
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
26
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
27
|
};
|
|
16
28
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
exports.buildCacheEntriesFromResults = void 0;
|
|
29
|
+
exports.buildTransportHandler = exports.buildCacheEntriesFromResults = void 0;
|
|
18
30
|
const cache_1 = require("../cache");
|
|
31
|
+
const util_1 = require("../util");
|
|
32
|
+
const transportMetrics = __importStar(require("./metrics"));
|
|
19
33
|
__exportStar(require("./batch-warming"), exports);
|
|
20
34
|
__exportStar(require("./rest"), exports);
|
|
21
|
-
__exportStar(require("./sse"), exports);
|
|
22
35
|
__exportStar(require("./websocket"), exports);
|
|
36
|
+
__exportStar(require("./sse"), exports);
|
|
37
|
+
const logger = (0, util_1.makeLogger)('Transport');
|
|
23
38
|
/**
|
|
24
39
|
* Helper method to build cache entries to set after getting a bunch of responses from a DP.
|
|
25
40
|
*
|
|
@@ -53,4 +68,46 @@ const buildCacheEntriesFromResults = (results, context) => results.map((r) => {
|
|
|
53
68
|
return cacheEntry;
|
|
54
69
|
});
|
|
55
70
|
exports.buildCacheEntriesFromResults = buildCacheEntriesFromResults;
|
|
71
|
+
/**
|
|
72
|
+
* Takes an Adapter, its configuration, and its dependencies, and it creates an express middleware
|
|
73
|
+
* that will pass along the AdapterRequest to the appropriate Transport (acc. to the endpoint in the req.)
|
|
74
|
+
*
|
|
75
|
+
* @param adapter - main adapter object, already initialized
|
|
76
|
+
* @returns the transport handler middleware function
|
|
77
|
+
*/
|
|
78
|
+
const buildTransportHandler = (adapter) => async (req, reply) => {
|
|
79
|
+
// Get transport, must be here because it's already checked in the validator
|
|
80
|
+
const transport = adapter.endpointsMap[req.requestContext.endpointName].transport;
|
|
81
|
+
// Set up transport if it hasn't been done already
|
|
82
|
+
if (!(await transport.hasBeenSetUp(req))) {
|
|
83
|
+
logger.debug('Transport not set up yet, doing so...');
|
|
84
|
+
const immediateResponse = await transport.setup(req, adapter.config);
|
|
85
|
+
if (immediateResponse) {
|
|
86
|
+
logger.debug('Got immediate response from transport, sending as response');
|
|
87
|
+
return reply.send(immediateResponse);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
// Observe the idle time taken for polling response
|
|
91
|
+
const metricsTimer = transportMetrics.transportPollingDurationSeconds
|
|
92
|
+
.labels({ endpoint: req.requestContext.endpointName })
|
|
93
|
+
.startTimer();
|
|
94
|
+
logger.debug('Transport is set up, polling cache for response...');
|
|
95
|
+
const response = await (0, cache_1.pollResponseFromCache)(adapter.dependencies.cache, req.requestContext.cacheKey, {
|
|
96
|
+
maxRetries: adapter.config.CACHE_POLLING_MAX_RETRIES,
|
|
97
|
+
sleep: adapter.config.CACHE_POLLING_SLEEP_MS,
|
|
98
|
+
});
|
|
99
|
+
metricsTimer({ succeeded: String(!!response) });
|
|
100
|
+
if (response) {
|
|
101
|
+
logger.debug('Got a response from the cache, sending that back');
|
|
102
|
+
return reply.send(response);
|
|
103
|
+
}
|
|
104
|
+
// Record polling mechanism failure to return response
|
|
105
|
+
transportMetrics.transportPollingFailureCount
|
|
106
|
+
.labels({ endpoint: req.requestContext.endpointName })
|
|
107
|
+
.inc();
|
|
108
|
+
logger.debug('Ran out of polling attempts, returning timeout');
|
|
109
|
+
reply.statusCode = 504;
|
|
110
|
+
return reply.send();
|
|
111
|
+
};
|
|
112
|
+
exports.buildTransportHandler = buildTransportHandler;
|
|
56
113
|
//# sourceMappingURL=index.js.map
|
package/transports/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/transports/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/transports/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,oCAMiB;AAEjB,kCAAoC;AAEpC,4DAA6C;AAE7C,kDAA+B;AAC/B,yCAAsB;AACtB,8CAA2B;AAC3B,wCAAqB;AAErB,MAAM,MAAM,GAAG,IAAA,iBAAU,EAAC,WAAW,CAAC,CAAA;AA4DtC;;;;;;GAMG;AACI,MAAM,4BAA4B,GAAG,CAC1C,OAAiC,EACjC,OAAuC,EACa,EAAE,CACtD,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;IAChB,MAAM,UAAU,GAAG;QACjB,GAAG,EAAE,IAAA,yBAAiB,EAAC,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC;QACzC,KAAK,EAAE;YACL,MAAM,EAAE,CAAC,CAAC,KAAK;YACf,UAAU,EAAE,GAAG;YACf,IAAI,EAAE;gBACJ,MAAM,EAAE,CAAC,CAAC,KAAK;aAChB;SACF;KACF,CAAA;IACD,IACE,OAAO,CAAC,aAAa,CAAC,eAAe;QACrC,OAAO,CAAC,aAAa,CAAC,4BAA4B,EAClD;QACA,MAAM,OAAO,GAAG;YACd,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,aAAa,CAAC,aAAa;YACxD,IAAI,EAAE;gBACJ,OAAO,EAAE;oBACP,MAAM,EAAE,IAAA,uBAAe,EAAC,OAAyB,EAAE,CAAC,CAAC,MAAM,CAAC;iBAC7D;aACF;SACF,CAAA;QACD,UAAU,CAAC,KAAK,GAAG,EAAE,GAAG,UAAU,CAAC,KAAK,EAAE,GAAG,OAAO,EAAE,CAAA;KACvD;IACD,OAAO,UAAU,CAAA;AACnB,CAAC,CAAC,CAAA;AA9BS,QAAA,4BAA4B,gCA8BrC;AAEJ;;;;;;GAMG;AACI,MAAM,qBAAqB,GAChC,CAAC,OAAgB,EAAE,EAAE,CAAC,KAAK,EAAE,GAAmB,EAAE,KAAmB,EAAE,EAAE;IACvE,4EAA4E;IAC5E,MAAM,SAAS,GAAG,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,SAAS,CAAA;IAEjF,kDAAkD;IAClD,IAAI,CAAC,CAAC,MAAM,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE;QACxC,MAAM,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAA;QACrD,MAAM,iBAAiB,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;QACpE,IAAI,iBAAiB,EAAE;YACrB,MAAM,CAAC,KAAK,CAAC,4DAA4D,CAAC,CAAA;YAC1E,OAAO,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;SACrC;KACF;IACD,mDAAmD;IACnD,MAAM,YAAY,GAAG,gBAAgB,CAAC,+BAA+B;SAClE,MAAM,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,cAAc,CAAC,YAAY,EAAE,CAAC;SACrD,UAAU,EAAE,CAAA;IAEf,MAAM,CAAC,KAAK,CAAC,oDAAoD,CAAC,CAAA;IAClE,MAAM,QAAQ,GAAG,MAAM,IAAA,6BAAqB,EAC1C,OAAO,CAAC,YAAY,CAAC,KAA+B,EACpD,GAAG,CAAC,cAAc,CAAC,QAAQ,EAC3B;QACE,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,yBAAyB;QACpD,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,sBAAsB;KAC7C,CACF,CAAA;IAED,YAAY,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAA;IAE/C,IAAI,QAAQ,EAAE;QACZ,MAAM,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAA;QAChE,OAAO,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;KAC5B;IAED,sDAAsD;IACtD,gBAAgB,CAAC,4BAA4B;SAC1C,MAAM,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,cAAc,CAAC,YAAY,EAAE,CAAC;SACrD,GAAG,EAAE,CAAA;IAER,MAAM,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAA;IAC9D,KAAK,CAAC,UAAU,GAAG,GAAG,CAAA;IAEtB,OAAO,KAAK,CAAC,IAAI,EAAE,CAAA;AACrB,CAAC,CAAA;AA7CU,QAAA,qBAAqB,yBA6C/B"}
|
package/transports/metrics.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as client from 'prom-client';
|
|
2
|
-
import { TransportGenerics } from '.';
|
|
3
2
|
import { AdapterContext } from '../adapter';
|
|
3
|
+
import { SettingsMap } from '../config';
|
|
4
4
|
export declare const dataProviderMetricsLabel: (providerStatusCode?: number, method?: string) => {
|
|
5
5
|
provider_status_code: number | undefined;
|
|
6
6
|
method: string;
|
|
@@ -14,7 +14,7 @@ export declare const messageSubsLabels: (feed_id: string, cache_key: string) =>
|
|
|
14
14
|
feed_id: string;
|
|
15
15
|
subscription_key: string;
|
|
16
16
|
};
|
|
17
|
-
export declare const recordWsMessageMetrics: <
|
|
17
|
+
export declare const recordWsMessageMetrics: <AdapterParams, CustomSettings extends SettingsMap>(context: AdapterContext<CustomSettings>, subscribes: AdapterParams[], unsubscrices: AdapterParams[]) => void;
|
|
18
18
|
export declare const wsConnectionActive: client.Gauge<"url">;
|
|
19
19
|
export declare const wsConnectionErrors: client.Counter<"message" | "url">;
|
|
20
20
|
export declare const wsSubscriptionActive: client.Gauge<"feed_id" | "connection_url" | "subscription_key">;
|
package/transports/metrics.js
CHANGED
|
@@ -57,7 +57,7 @@ exports.messageSubsLabels = messageSubsLabels;
|
|
|
57
57
|
// Record WS message and subscription metrics
|
|
58
58
|
// Recalculate cacheKey and feedId for metrics
|
|
59
59
|
// since avoiding storing extra info in expiring sorted set
|
|
60
|
-
const recordWsMessageMetrics = (context, subscribes,
|
|
60
|
+
const recordWsMessageMetrics = (context, subscribes, unsubscrices) => {
|
|
61
61
|
subscribes.forEach((param) => {
|
|
62
62
|
const feedId = (0, cache_1.calculateFeedId)(context, param);
|
|
63
63
|
const cacheKey = (0, cache_1.calculateCacheKey)(context, param);
|
|
@@ -68,7 +68,7 @@ const recordWsMessageMetrics = (context, subscribes, unsubscribes) => {
|
|
|
68
68
|
// Record number of active ws subscriptions
|
|
69
69
|
exports.wsSubscriptionActive.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).inc();
|
|
70
70
|
});
|
|
71
|
-
|
|
71
|
+
unsubscrices.forEach((param) => {
|
|
72
72
|
const feedId = (0, cache_1.calculateFeedId)(context, param);
|
|
73
73
|
const cacheKey = (0, cache_1.calculateCacheKey)(context, param);
|
|
74
74
|
// Record total number of ws messages sent
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../../src/transports/metrics.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,oDAAqC;
|
|
1
|
+
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../../src/transports/metrics.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,oDAAqC;AAErC,oCAA6D;AAE7D,oDAA6D;AAE7D,wBAAwB;AACjB,MAAM,wBAAwB,GAAG,CAAC,kBAA2B,EAAE,MAAM,GAAG,KAAK,EAAE,EAAE,CAAC,CAAC;IACxF,oBAAoB,EAAE,kBAAkB;IACxC,MAAM,EAAE,MAAM,CAAC,WAAW,EAAE;CAC7B,CAAC,CAAA;AAHW,QAAA,wBAAwB,4BAGnC;AAEW,QAAA,oBAAoB,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IACrD,IAAI,EAAE,wBAAwB;IAC9B,IAAI,EAAE,8DAA8D;IACpE,UAAU,EAAE,CAAC,QAAQ,EAAE,sBAAsB,CAAU;CACxD,CAAC,CAAA;AAEW,QAAA,kCAAkC,GAAG,IAAI,MAAM,CAAC,SAAS,CAAC;IACrE,IAAI,EAAE,wCAAwC;IAC9C,IAAI,EAAE,2EAA2E;IACjF,OAAO,EAAE,kCAAsB;CAChC,CAAC,CAAA;AAEF,oBAAoB;AACb,MAAM,qBAAqB,GAAG,CAAC,OAAe,EAAE,EAAE,CAAC,CAAC;IACzD,OAAO;IACP,OAAO;CACR,CAAC,CAAA;AAHW,QAAA,qBAAqB,yBAGhC;AAEK,MAAM,iBAAiB,GAAG,CAAC,OAAe,EAAE,SAAiB,EAAE,EAAE,CAAC,CAAC;IACxE,OAAO;IACP,gBAAgB,EAAE,SAAS;CAC5B,CAAC,CAAA;AAHW,QAAA,iBAAiB,qBAG5B;AAEF,6CAA6C;AAC7C,8CAA8C;AAC9C,2DAA2D;AACpD,MAAM,sBAAsB,GAAG,CACpC,OAAuC,EACvC,UAA2B,EAC3B,YAA6B,EACvB,EAAE;IACR,UAAU,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;QAC3B,MAAM,MAAM,GAAG,IAAA,uBAAe,EAAC,OAAO,EAAE,KAAK,CAAC,CAAA;QAC9C,MAAM,QAAQ,GAAG,IAAA,yBAAiB,EAAC,OAAO,EAAE,KAAK,CAAC,CAAA;QAClD,0CAA0C;QAC1C,sBAAc,CAAC,MAAM,CAAC,IAAA,yBAAiB,EAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,CAAA;QAEhE,4CAA4C;QAC5C,2BAAmB,CAAC,MAAM,CAAC,IAAA,yBAAiB,EAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,CAAA;QAErE,2CAA2C;QAC3C,4BAAoB,CAAC,MAAM,CAAC,IAAA,yBAAiB,EAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,CAAA;IACxE,CAAC,CAAC,CAAA;IACF,YAAY,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;QAC7B,MAAM,MAAM,GAAG,IAAA,uBAAe,EAAC,OAAO,EAAE,KAAK,CAAC,CAAA;QAC9C,MAAM,QAAQ,GAAG,IAAA,yBAAiB,EAAC,OAAO,EAAE,KAAK,CAAC,CAAA;QAElD,0CAA0C;QAC1C,sBAAc,CAAC,MAAM,CAAC,IAAA,yBAAiB,EAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,CAAA;QAEhE,2CAA2C;QAC3C,4BAAoB,CAAC,MAAM,CAAC,IAAA,yBAAiB,EAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,CAAA;IACxE,CAAC,CAAC,CAAA;AACJ,CAAC,CAAA;AA3BY,QAAA,sBAAsB,0BA2BlC;AAEY,QAAA,kBAAkB,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC;IACjD,IAAI,EAAE,sBAAsB;IAC5B,IAAI,EAAE,kCAAkC;IACxC,UAAU,EAAE,CAAC,KAAK,CAAU;CAC7B,CAAC,CAAA;AAEW,QAAA,kBAAkB,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IACnD,IAAI,EAAE,sBAAsB;IAC5B,IAAI,EAAE,iCAAiC;IACvC,UAAU,EAAE,CAAC,KAAK,EAAE,SAAS,CAAU;CACxC,CAAC,CAAA;AAEW,QAAA,oBAAoB,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC;IACnD,IAAI,EAAE,wBAAwB;IAC9B,IAAI,EAAE,8CAA8C;IACpD,UAAU,EAAE,CAAC,gBAAgB,EAAE,SAAS,EAAE,kBAAkB,CAAU;CACvE,CAAC,CAAA;AAEW,QAAA,mBAAmB,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IACpD,IAAI,EAAE,uBAAuB;IAC7B,IAAI,EAAE,6CAA6C;IACnD,UAAU,EAAE,CAAC,gBAAgB,EAAE,SAAS,EAAE,kBAAkB,CAAU;CACvE,CAAC,CAAA;AAEW,QAAA,cAAc,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IAC/C,IAAI,EAAE,kBAAkB;IACxB,IAAI,EAAE,0CAA0C;IAChD,UAAU,EAAE,CAAC,SAAS,EAAE,kBAAkB,CAAU;CACrD,CAAC,CAAA;AAEF,sBAAsB;AACT,QAAA,6BAA6B,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC;IAC5D,IAAI,EAAE,mCAAmC;IACzC,IAAI,EAAE,0DAA0D;IAChE,UAAU,EAAE,CAAC,UAAU,EAAE,gBAAgB,CAAU;CACpD,CAAC,CAAA;AAEW,QAAA,4BAA4B,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IAC7D,IAAI,EAAE,iCAAiC;IACvC,IAAI,EAAE,+FAA+F;IACrG,UAAU,EAAE,CAAC,UAAU,CAAU;CAClC,CAAC,CAAA;AAEW,QAAA,+BAA+B,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC;IAC9D,IAAI,EAAE,oCAAoC;IAC1C,IAAI,EAAE,iFAAiF;IACvF,UAAU,EAAE,CAAC,UAAU,EAAE,WAAW,CAAU;CAC/C,CAAC,CAAA"}
|