@chainlink/external-adapter-framework 0.0.12 → 0.0.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (220) hide show
  1. package/.eslintignore +1 -0
  2. package/.github/README.MD +35 -10
  3. package/.github/workflows/label.yaml +39 -0
  4. package/.github/workflows/publish.yaml +0 -3
  5. package/dist/examples/coingecko/test/e2e/adapter.test.ts.js +82953 -0
  6. package/dist/examples/coingecko/test/integration/adapter.test.ts.js +91672 -0
  7. package/dist/main.js +72703 -0
  8. package/jest.config.js +5 -0
  9. package/package.json +14 -3
  10. package/src/config/provider-limits.ts +14 -17
  11. package/src/examples/coingecko/src/config/index.ts +12 -0
  12. package/src/examples/coingecko/src/config/overrides.json +10826 -0
  13. package/src/examples/coingecko/src/cryptoUtils.ts +88 -0
  14. package/src/examples/coingecko/src/endpoint/coins.ts +54 -0
  15. package/src/examples/coingecko/src/endpoint/crypto-marketcap.ts +66 -0
  16. package/src/examples/coingecko/src/endpoint/crypto-volume.ts +66 -0
  17. package/src/examples/coingecko/src/endpoint/crypto.ts +63 -0
  18. package/src/examples/coingecko/src/endpoint/dominance.ts +40 -0
  19. package/src/examples/coingecko/src/endpoint/global-marketcap.ts +40 -0
  20. package/src/examples/coingecko/src/endpoint/index.ts +6 -0
  21. package/src/examples/coingecko/src/globalUtils.ts +78 -0
  22. package/src/examples/coingecko/src/index.ts +17 -0
  23. package/src/examples/coingecko/test/e2e/adapter.test.ts +278 -0
  24. package/src/examples/coingecko/test/integration/__snapshots__/adapter.test.ts.snap +15 -0
  25. package/src/examples/coingecko/test/integration/adapter.test.ts +281 -0
  26. package/src/examples/coingecko/test/integration/capturedRequests.json +1 -0
  27. package/src/examples/coingecko/test/integration/fixtures.ts +42 -0
  28. package/src/examples/{coingecko → coingecko-old}/batch-warming.ts +1 -0
  29. package/src/examples/{coingecko → coingecko-old}/index.ts +0 -0
  30. package/src/examples/{coingecko → coingecko-old}/rest.ts +0 -0
  31. package/src/transports/batch-warming.ts +7 -2
  32. package/src/transports/rest.ts +5 -1
  33. package/src/util/index.ts +1 -0
  34. package/src/util/recordRequests.ts +47 -0
  35. package/src/validation/overrideFunctions.ts +44 -0
  36. package/test/metrics/warmer-metrics.test.ts +1 -0
  37. package/test/{smoke.test.ts → smoke/smoke.test.ts} +13 -13
  38. package/{test-payload-fail.json → test/smoke/test-payload-fail.json} +0 -0
  39. package/{test-payload.js → test/smoke/test-payload.js} +0 -0
  40. package/{test-payload.json → test/smoke/test-payload.json} +0 -0
  41. package/test/transports/batch.test.ts +1 -0
  42. package/tsconfig.json +4 -1
  43. package/webpack.config.js +39 -5
  44. package/yarn-error.log +3778 -0
  45. package/dist/src/adapter.d.ts +0 -135
  46. package/dist/src/adapter.js +0 -145
  47. package/dist/src/background-executor.d.ts +0 -11
  48. package/dist/src/background-executor.js +0 -45
  49. package/dist/src/cache/factory.d.ts +0 -6
  50. package/dist/src/cache/factory.js +0 -55
  51. package/dist/src/cache/index.d.ts +0 -94
  52. package/dist/src/cache/index.js +0 -173
  53. package/dist/src/cache/local.d.ts +0 -23
  54. package/dist/src/cache/local.js +0 -83
  55. package/dist/src/cache/metrics.d.ts +0 -27
  56. package/dist/src/cache/metrics.js +0 -120
  57. package/dist/src/cache/redis.d.ts +0 -16
  58. package/dist/src/cache/redis.js +0 -100
  59. package/dist/src/chainlink-external-adapter-framework-0.0.6.tgz +0 -0
  60. package/dist/src/config/index.d.ts +0 -214
  61. package/dist/src/config/index.js +0 -380
  62. package/dist/src/config/provider-limits.d.ts +0 -31
  63. package/dist/src/config/provider-limits.js +0 -75
  64. package/dist/src/examples/bank-frick/accounts.d.ts +0 -39
  65. package/dist/src/examples/bank-frick/accounts.js +0 -192
  66. package/dist/src/examples/bank-frick/config/index.d.ts +0 -4
  67. package/dist/src/examples/bank-frick/config/index.js +0 -54
  68. package/dist/src/examples/bank-frick/index.d.ts +0 -2
  69. package/dist/src/examples/bank-frick/index.js +0 -15
  70. package/dist/src/examples/bank-frick/util.d.ts +0 -4
  71. package/dist/src/examples/bank-frick/util.js +0 -39
  72. package/dist/src/examples/coingecko/batch-warming.d.ts +0 -7
  73. package/dist/src/examples/coingecko/batch-warming.js +0 -53
  74. package/dist/src/examples/coingecko/index.d.ts +0 -2
  75. package/dist/src/examples/coingecko/index.js +0 -11
  76. package/dist/src/examples/coingecko/rest.d.ts +0 -12
  77. package/dist/src/examples/coingecko/rest.js +0 -51
  78. package/dist/src/examples/ncfx/config/index.d.ts +0 -12
  79. package/dist/src/examples/ncfx/config/index.js +0 -15
  80. package/dist/src/examples/ncfx/index.d.ts +0 -13
  81. package/dist/src/examples/ncfx/index.js +0 -11
  82. package/dist/src/examples/ncfx/websocket.d.ts +0 -47
  83. package/dist/src/examples/ncfx/websocket.js +0 -73
  84. package/dist/src/index.d.ts +0 -11
  85. package/dist/src/index.js +0 -128
  86. package/dist/src/metrics/constants.d.ts +0 -16
  87. package/dist/src/metrics/constants.js +0 -25
  88. package/dist/src/metrics/index.d.ts +0 -15
  89. package/dist/src/metrics/index.js +0 -122
  90. package/dist/src/metrics/util.d.ts +0 -7
  91. package/dist/src/metrics/util.js +0 -9
  92. package/dist/src/package/adapter.d.ts +0 -88
  93. package/dist/src/package/adapter.js +0 -112
  94. package/dist/src/package/background-executor.d.ts +0 -11
  95. package/dist/src/package/background-executor.js +0 -45
  96. package/dist/src/package/cache/factory.d.ts +0 -6
  97. package/dist/src/package/cache/factory.js +0 -57
  98. package/dist/src/package/cache/index.d.ts +0 -90
  99. package/dist/src/package/cache/index.js +0 -169
  100. package/dist/src/package/cache/local.d.ts +0 -23
  101. package/dist/src/package/cache/local.js +0 -83
  102. package/dist/src/package/cache/metrics.d.ts +0 -27
  103. package/dist/src/package/cache/metrics.js +0 -120
  104. package/dist/src/package/cache/redis.d.ts +0 -16
  105. package/dist/src/package/cache/redis.js +0 -100
  106. package/dist/src/package/config/index.d.ts +0 -195
  107. package/dist/src/package/config/index.js +0 -365
  108. package/dist/src/package/config/provider-limits.d.ts +0 -31
  109. package/dist/src/package/config/provider-limits.js +0 -76
  110. package/dist/src/package/examples/coingecko/batch-warming.d.ts +0 -2
  111. package/dist/src/package/examples/coingecko/batch-warming.js +0 -52
  112. package/dist/src/package/examples/coingecko/index.d.ts +0 -2
  113. package/dist/src/package/examples/coingecko/index.js +0 -10
  114. package/dist/src/package/examples/coingecko/rest.d.ts +0 -2
  115. package/dist/src/package/examples/coingecko/rest.js +0 -50
  116. package/dist/src/package/examples/ncfx/config/index.d.ts +0 -12
  117. package/dist/src/package/examples/ncfx/config/index.js +0 -15
  118. package/dist/src/package/examples/ncfx/index.d.ts +0 -2
  119. package/dist/src/package/examples/ncfx/index.js +0 -10
  120. package/dist/src/package/examples/ncfx/websocket.d.ts +0 -36
  121. package/dist/src/package/examples/ncfx/websocket.js +0 -72
  122. package/dist/src/package/index.d.ts +0 -12
  123. package/dist/src/package/index.js +0 -92
  124. package/dist/src/package/metrics/constants.d.ts +0 -16
  125. package/dist/src/package/metrics/constants.js +0 -25
  126. package/dist/src/package/metrics/index.d.ts +0 -15
  127. package/dist/src/package/metrics/index.js +0 -123
  128. package/dist/src/package/metrics/util.d.ts +0 -3
  129. package/dist/src/package/metrics/util.js +0 -9
  130. package/dist/src/package/package.json +0 -72
  131. package/dist/src/package/rate-limiting/background/fixed-frequency.d.ts +0 -10
  132. package/dist/src/package/rate-limiting/background/fixed-frequency.js +0 -37
  133. package/dist/src/package/rate-limiting/index.d.ts +0 -54
  134. package/dist/src/package/rate-limiting/index.js +0 -63
  135. package/dist/src/package/rate-limiting/metrics.d.ts +0 -3
  136. package/dist/src/package/rate-limiting/metrics.js +0 -44
  137. package/dist/src/package/rate-limiting/request/simple-counting.d.ts +0 -20
  138. package/dist/src/package/rate-limiting/request/simple-counting.js +0 -62
  139. package/dist/src/package/test.d.ts +0 -1
  140. package/dist/src/package/test.js +0 -6
  141. package/dist/src/package/transports/batch-warming.d.ts +0 -34
  142. package/dist/src/package/transports/batch-warming.js +0 -101
  143. package/dist/src/package/transports/index.d.ts +0 -87
  144. package/dist/src/package/transports/index.js +0 -87
  145. package/dist/src/package/transports/metrics.d.ts +0 -21
  146. package/dist/src/package/transports/metrics.js +0 -105
  147. package/dist/src/package/transports/rest.d.ts +0 -43
  148. package/dist/src/package/transports/rest.js +0 -129
  149. package/dist/src/package/transports/util.d.ts +0 -8
  150. package/dist/src/package/transports/util.js +0 -85
  151. package/dist/src/package/transports/websocket.d.ts +0 -80
  152. package/dist/src/package/transports/websocket.js +0 -169
  153. package/dist/src/package/util/expiring-sorted-set.d.ts +0 -21
  154. package/dist/src/package/util/expiring-sorted-set.js +0 -47
  155. package/dist/src/package/util/index.d.ts +0 -11
  156. package/dist/src/package/util/index.js +0 -35
  157. package/dist/src/package/util/logger.d.ts +0 -42
  158. package/dist/src/package/util/logger.js +0 -62
  159. package/dist/src/package/util/request.d.ts +0 -55
  160. package/dist/src/package/util/request.js +0 -2
  161. package/dist/src/package/validation/error.d.ts +0 -50
  162. package/dist/src/package/validation/error.js +0 -79
  163. package/dist/src/package/validation/index.d.ts +0 -5
  164. package/dist/src/package/validation/index.js +0 -86
  165. package/dist/src/package/validation/input-params.d.ts +0 -15
  166. package/dist/src/package/validation/input-params.js +0 -30
  167. package/dist/src/package/validation/override-functions.d.ts +0 -3
  168. package/dist/src/package/validation/override-functions.js +0 -40
  169. package/dist/src/package/validation/preset-tokens.json +0 -23
  170. package/dist/src/package/validation/validator.d.ts +0 -47
  171. package/dist/src/package/validation/validator.js +0 -303
  172. package/dist/src/package.json +0 -72
  173. package/dist/src/rate-limiting/background/fixed-frequency.d.ts +0 -11
  174. package/dist/src/rate-limiting/background/fixed-frequency.js +0 -35
  175. package/dist/src/rate-limiting/index.d.ts +0 -55
  176. package/dist/src/rate-limiting/index.js +0 -63
  177. package/dist/src/rate-limiting/metrics.d.ts +0 -3
  178. package/dist/src/rate-limiting/metrics.js +0 -44
  179. package/dist/src/rate-limiting/request/simple-counting.d.ts +0 -21
  180. package/dist/src/rate-limiting/request/simple-counting.js +0 -62
  181. package/dist/src/test.d.ts +0 -1
  182. package/dist/src/test.js +0 -6
  183. package/dist/src/transports/batch-warming.d.ts +0 -35
  184. package/dist/src/transports/batch-warming.js +0 -101
  185. package/dist/src/transports/index.d.ts +0 -70
  186. package/dist/src/transports/index.js +0 -87
  187. package/dist/src/transports/metrics.d.ts +0 -22
  188. package/dist/src/transports/metrics.js +0 -105
  189. package/dist/src/transports/rest.d.ts +0 -44
  190. package/dist/src/transports/rest.js +0 -131
  191. package/dist/src/transports/util.d.ts +0 -8
  192. package/dist/src/transports/util.js +0 -85
  193. package/dist/src/transports/websocket.d.ts +0 -80
  194. package/dist/src/transports/websocket.js +0 -166
  195. package/dist/src/util/expiring-sorted-set.d.ts +0 -21
  196. package/dist/src/util/expiring-sorted-set.js +0 -47
  197. package/dist/src/util/index.d.ts +0 -12
  198. package/dist/src/util/index.js +0 -35
  199. package/dist/src/util/logger.d.ts +0 -42
  200. package/dist/src/util/logger.js +0 -62
  201. package/dist/src/util/request.d.ts +0 -57
  202. package/dist/src/util/request.js +0 -2
  203. package/dist/src/util/subscription-set/expiring-sorted-set.d.ts +0 -22
  204. package/dist/src/util/subscription-set/expiring-sorted-set.js +0 -47
  205. package/dist/src/util/subscription-set/subscription-set.d.ts +0 -18
  206. package/dist/src/util/subscription-set/subscription-set.js +0 -19
  207. package/dist/src/util/test-payload-loader.d.ts +0 -25
  208. package/dist/src/util/test-payload-loader.js +0 -83
  209. package/dist/src/validation/error.d.ts +0 -50
  210. package/dist/src/validation/error.js +0 -79
  211. package/dist/src/validation/index.d.ts +0 -5
  212. package/dist/src/validation/index.js +0 -91
  213. package/dist/src/validation/input-params.d.ts +0 -15
  214. package/dist/src/validation/input-params.js +0 -30
  215. package/dist/src/validation/override-functions.d.ts +0 -3
  216. package/dist/src/validation/override-functions.js +0 -40
  217. package/dist/src/validation/preset-tokens.json +0 -23
  218. package/dist/src/validation/validator.d.ts +0 -47
  219. package/dist/src/validation/validator.js +0 -303
  220. package/src/test.ts +0 -5
@@ -1,62 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.SimpleCountingRateLimiter = void 0;
4
- const util_1 = require("../../util");
5
- const logger = (0, util_1.makeLogger)('SimpleCountingRateLimiter');
6
- /**
7
- * This rate limiter is the simplest stateful option.
8
- * On startup, it'll compare the different thresholds for each tier, calculate them all
9
- * in the finest window we'll use (seconds), and use the most restrictive one.
10
- * This is so if the EA were to restart, we don't need to worry about persisting state
11
- * for things like daily quotas. The downside is that this does not work well for bursty
12
- * loads or spikes, in cases where e.g. the per second limit is high but daily quotas low.
13
- */
14
- class SimpleCountingRateLimiter {
15
- constructor() {
16
- this.latestSecondInterval = 0;
17
- this.requestsThisSecond = 0;
18
- this.latestMinuteInterval = 0;
19
- this.requestsThisMinute = 0;
20
- }
21
- initialize(endpoints, limits) {
22
- // Translate the hourly limit into reqs per minute
23
- const perHourLimit = (limits?.rateLimit1h || Infinity) / 60;
24
- this.perMinuteLimit = Math.min(limits?.rateLimit1m || Infinity, perHourLimit);
25
- this.perSecondLimit = limits?.rateLimit1s || Infinity;
26
- logger.debug(`Using rate limiting settings: perMinute = ${this.perMinuteLimit} | perSecond: = ${this.perSecondLimit}`);
27
- return this;
28
- }
29
- isUnderLimits() {
30
- // If the limit is set to infinity, there was no tier limit specified
31
- if (this.perSecondLimit === Infinity && this.perMinuteLimit === Infinity) {
32
- return true;
33
- }
34
- const now = Date.now();
35
- const nearestSecondInterval = Math.floor(now / 1000);
36
- const nearestMinuteInterval = Math.floor(now / (1000 * 60));
37
- // This should always run to completion, even if it doesn't look atomic; therefore the
38
- // Ops should be "thread safe". Thank JS and its infinite single threaded dumbness.
39
- if (nearestSecondInterval !== this.latestSecondInterval) {
40
- logger.trace(`Clearing latest second interval, # of requests logged was: ${this.requestsThisSecond} `);
41
- this.latestSecondInterval = nearestSecondInterval;
42
- this.requestsThisSecond = 0;
43
- }
44
- if (nearestMinuteInterval !== this.latestMinuteInterval) {
45
- logger.trace(`Clearing latest second minute, # of requests logged was: ${this.requestsThisMinute} `);
46
- this.latestMinuteInterval = nearestMinuteInterval;
47
- this.requestsThisMinute = 0;
48
- }
49
- if (this.requestsThisSecond < this.perSecondLimit &&
50
- this.requestsThisMinute < this.perMinuteLimit) {
51
- logger.trace('Request under limits, counting +1');
52
- this.requestsThisSecond++;
53
- this.requestsThisMinute++;
54
- return true;
55
- }
56
- else {
57
- logger.trace('Requests seen this interval are above limits');
58
- return false;
59
- }
60
- }
61
- }
62
- exports.SimpleCountingRateLimiter = SimpleCountingRateLimiter;
@@ -1 +0,0 @@
1
- export {};
package/dist/src/test.js DELETED
@@ -1,6 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- const _1 = require(".");
4
- const bank_frick_1 = require("./examples/bank-frick");
5
- // Start sample adapter
6
- (0, _1.expose)(bank_frick_1.adapter);
@@ -1,35 +0,0 @@
1
- import { AxiosRequestConfig, AxiosResponse } from 'axios';
2
- import { Cache } from '../cache';
3
- import { AdapterConfig, SettingsMap } from '../config';
4
- import { BackgroundExecuteRateLimiter } from '../rate-limiting';
5
- import { SubscriptionSet } from '../util';
6
- import { AdapterRequest, ProviderResult } from '../util/request';
7
- import { Transport } from './';
8
- import { AdapterContext, AdapterDependencies } from '../adapter';
9
- /**
10
- * Transport implementation that takes incoming batches requests and keeps a warm cache of values.
11
- * Within the setup function, adapter params are added to an set that also keeps track and expires values.
12
- * In the background execute, the list of non-expired items in the set is fetched.
13
- * Then, the list is passed through the `prepareRequest` function, that returns an AxiosRequestConfig.
14
- * The Data Provider response is, they are passed through the `parseResponse` function to create a [[CacheEntry]] list.
15
- * Finally, the items in that [[CacheEntry]] list are set in the Cache so the Adapter can fetch values from there.
16
- *
17
- * @typeParam AdapterParams - interface for the adapter request body
18
- * @typeParam ProviderRequestBody - interface for the body of the request to the Data Provider
19
- * @typeParam ProviderResponseBody - interface for the body of the Data Provider's response
20
- */
21
- export declare class BatchWarmingTransport<AdapterParams, ProviderRequestBody, ProviderResponseBody, CustomSettings extends SettingsMap> implements Transport<AdapterParams, ProviderResponseBody, CustomSettings> {
22
- private config;
23
- cache: Cache;
24
- rateLimiter: BackgroundExecuteRateLimiter;
25
- subscriptionSet: SubscriptionSet<AdapterParams>;
26
- WARMER_ACTIVE: boolean;
27
- constructor(config: {
28
- prepareRequest: (params: AdapterParams[], context: AdapterContext<CustomSettings>) => AxiosRequestConfig<ProviderRequestBody>;
29
- parseResponse: (res: AxiosResponse<ProviderResponseBody>, context: AdapterContext<CustomSettings>) => ProviderResult<AdapterParams>[];
30
- });
31
- initialize(dependencies: AdapterDependencies): Promise<void>;
32
- hasBeenSetUp(req: AdapterRequest<AdapterParams>): Promise<boolean>;
33
- setup(req: AdapterRequest<AdapterParams>, config: AdapterConfig<CustomSettings>): Promise<void>;
34
- backgroundExecute(context: AdapterContext<CustomSettings>): Promise<number>;
35
- }
@@ -1,101 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
25
- Object.defineProperty(exports, "__esModule", { value: true });
26
- exports.BatchWarmingTransport = void 0;
27
- const util_1 = require("../util");
28
- const _1 = require("./");
29
- const util_2 = require("./util");
30
- const rateLimitMetrics = __importStar(require("../rate-limiting/metrics"));
31
- const cacheMetrics = __importStar(require("../cache/metrics"));
32
- const WARMUP_BATCH_REQUEST_ID = '9002';
33
- const logger = (0, util_1.makeLogger)('BatchWarmingTransport');
34
- /**
35
- * Transport implementation that takes incoming batches requests and keeps a warm cache of values.
36
- * Within the setup function, adapter params are added to an set that also keeps track and expires values.
37
- * In the background execute, the list of non-expired items in the set is fetched.
38
- * Then, the list is passed through the `prepareRequest` function, that returns an AxiosRequestConfig.
39
- * The Data Provider response is, they are passed through the `parseResponse` function to create a [[CacheEntry]] list.
40
- * Finally, the items in that [[CacheEntry]] list are set in the Cache so the Adapter can fetch values from there.
41
- *
42
- * @typeParam AdapterParams - interface for the adapter request body
43
- * @typeParam ProviderRequestBody - interface for the body of the request to the Data Provider
44
- * @typeParam ProviderResponseBody - interface for the body of the Data Provider's response
45
- */
46
- class BatchWarmingTransport {
47
- constructor(config) {
48
- this.config = config;
49
- // Flag used to track whether the warmer has moved from having no entries to having some and vice versa
50
- // Used for recording the cache warmer active metrics accurately
51
- this.WARMER_ACTIVE = false;
52
- }
53
- async initialize(dependencies) {
54
- this.cache = dependencies.cache;
55
- this.rateLimiter = dependencies.backgroundExecuteRateLimiter;
56
- this.subscriptionSet = dependencies.subscriptionSetFactory.buildSet();
57
- }
58
- async hasBeenSetUp(req) {
59
- return !!(await this.subscriptionSet.get(req.requestContext.cacheKey));
60
- }
61
- async setup(req, config) {
62
- logger.debug(`Adding entry to batch warming set: [${req.requestContext.cacheKey}] = ${req.requestContext.data}`);
63
- await this.subscriptionSet.add(req.requestContext.cacheKey, req.requestContext.data, config.WARMUP_SUBSCRIPTION_TTL);
64
- }
65
- async backgroundExecute(context) {
66
- logger.debug('Starting background execute');
67
- const entries = await this.subscriptionSet.getAll();
68
- if (!entries.length) {
69
- logger.debug('No entries in batch warming set, skipping');
70
- if (this.WARMER_ACTIVE) {
71
- // Decrement count when warmer changed from having entries to having none
72
- cacheMetrics.cacheWarmerCount.labels({ isBatched: 'true' }).dec();
73
- this.WARMER_ACTIVE = false;
74
- }
75
- return this.rateLimiter.msUntilNextExecution(context.adapterEndpoint.name);
76
- }
77
- else if (this.WARMER_ACTIVE === false) {
78
- // Increment count when warmer changed from having no entries to having some
79
- cacheMetrics.cacheWarmerCount.labels({ isBatched: 'true' }).inc();
80
- this.WARMER_ACTIVE = true;
81
- }
82
- const request = this.config.prepareRequest(entries, context);
83
- logger.trace('Sending request to data provider...');
84
- const providerResponse = await (0, util_2.axiosRequest)(request);
85
- logger.debug(`Got response from provider, parsing (raw body: ${providerResponse.data})`);
86
- const results = this.config.parseResponse(providerResponse, context);
87
- const adapterResponses = (0, _1.buildCacheEntriesFromResults)(results, context);
88
- logger.debug('Setting adapter responses in cache');
89
- await this.cache.setMany(adapterResponses, context.adapterConfig.CACHE_MAX_AGE);
90
- // Record cost of data provider call
91
- const cost = rateLimitMetrics.retrieveCost(providerResponse.data);
92
- rateLimitMetrics.rateLimitCreditsSpentTotal
93
- .labels({
94
- feed_id: 'N/A',
95
- participant_id: WARMUP_BATCH_REQUEST_ID,
96
- })
97
- .inc(cost);
98
- return this.rateLimiter.msUntilNextExecution(context.adapterEndpoint.name);
99
- }
100
- }
101
- exports.BatchWarmingTransport = BatchWarmingTransport;
@@ -1,70 +0,0 @@
1
- import { FastifyReply } from 'fastify';
2
- import { AdapterContext, AdapterDependencies, Adapter } from '../adapter';
3
- import { CacheEntry } from '../cache';
4
- import { AdapterConfig, SettingsMap } from '../config';
5
- import { AdapterRequest, AdapterResponse, ProviderResult } from '../util/request';
6
- export * from './batch-warming';
7
- export * from './rest';
8
- export * from './websocket';
9
- /**
10
- * Generic interface for a Transport.
11
- * A Transport defines the way in which an AdapterEndpoint will process incoming requests to
12
- * fetch data from a Data Provider. The setup phase will take care of the former, while the
13
- * backgroundExecute will be in charge of the latter.
14
- * This separation gives us the ability of splitting these concerns, and optionally parallelizing
15
- * the reading and writing of data to a centralized Cache.
16
- *
17
- * @typeParam Params - the structure of the AdapterRequest's body
18
- * @typeParam Result - the structure of the AdapterResponse's body
19
- */
20
- export interface Transport<Params, Result, CustomSettings extends SettingsMap> {
21
- /**
22
- * Initializes the transport in the Adapter context.
23
- *
24
- * @param dependencies - Adapter dependencies (e.g. cache instance)
25
- * @returns an empty Promise
26
- */
27
- initialize: (dependencies: AdapterDependencies) => Promise<void>;
28
- /**
29
- * Checks if the Transport has already set up the incoming request.
30
- * This will likely use the cacheKey generated in the provided request.
31
- *
32
- * @param req - the incoming AdapterRequest
33
- * @returns a Promise that returns true if setup is in place
34
- */
35
- hasBeenSetUp: (req: AdapterRequest<Params>) => Promise<boolean>;
36
- /**
37
- * Sets up the incoming request within the Transport.
38
- * In other words, it means that the Adapter has recognized this request, and will begin
39
- * fetching the necessary data from a Data Provider.
40
- *
41
- * @param req - the incoming AdapterRequest
42
- * @param config - common configuration for the Adapter as a whole
43
- * @returns a Promise that _optionally_ returns an AdapterResponse, if the Transport has the capability of
44
- * immediately fetching data and returning it without the background process.
45
- */
46
- setup: (req: AdapterRequest<Params>, config: AdapterConfig<CustomSettings>) => Promise<AdapterResponse<Result> | void>;
47
- /**
48
- * If the Transport relies on an async mechanism, implementing this function will make it
49
- * so the background task executor calls it on Adapter startup.
50
- *
51
- * @param context - background context for the execution (e.g. endpoint name)
52
- */
53
- backgroundExecute?: (context: AdapterContext<CustomSettings>) => Promise<number>;
54
- }
55
- /**
56
- * Helper method to build cache entries to set after getting a bunch of responses from a DP.
57
- *
58
- * @param results - a list of results coming from a DataProvider
59
- * @param context - context for the Adapter
60
- * @returns a list of CacheEntries of AdapterResponses
61
- */
62
- export declare const buildCacheEntriesFromResults: <Params, CustomSettings extends SettingsMap>(results: ProviderResult<Params>[], context: AdapterContext<CustomSettings>) => CacheEntry<AdapterResponse<null>>[];
63
- /**
64
- * Takes an Adapter, its configuration, and its dependencies, and it creates an express middleware
65
- * that will pass along the AdapterRequest to the appropriate Transport (acc. to the endpoint in the req.)
66
- *
67
- * @param adapter - main adapter object, already initialized
68
- * @returns the transport handler middleware function
69
- */
70
- export declare const buildTransportHandler: (adapter: Adapter) => (req: AdapterRequest, reply: FastifyReply) => Promise<never>;
@@ -1,87 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
- for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
- };
16
- Object.defineProperty(exports, "__esModule", { value: true });
17
- exports.buildTransportHandler = exports.buildCacheEntriesFromResults = void 0;
18
- const cache_1 = require("../cache");
19
- const util_1 = require("../util");
20
- __exportStar(require("./batch-warming"), exports);
21
- __exportStar(require("./rest"), exports);
22
- __exportStar(require("./websocket"), exports);
23
- const logger = (0, util_1.makeLogger)('Transport');
24
- /**
25
- * Helper method to build cache entries to set after getting a bunch of responses from a DP.
26
- *
27
- * @param results - a list of results coming from a DataProvider
28
- * @param context - context for the Adapter
29
- * @returns a list of CacheEntries of AdapterResponses
30
- */
31
- const buildCacheEntriesFromResults = (results, context) => results.map((r) => {
32
- const cacheEntry = {
33
- key: (0, cache_1.calculateCacheKey)(context, r.params),
34
- value: {
35
- result: r.value,
36
- statusCode: 200,
37
- data: null, // TODO: Maybe add data as well?
38
- },
39
- };
40
- if (context.adapterConfig.METRICS_ENABLED &&
41
- context.adapterConfig.EXPERIMENTAL_METRICS_ENABLED) {
42
- const metrics = {
43
- maxAge: Date.now() + context.adapterConfig.CACHE_MAX_AGE,
44
- meta: {
45
- metrics: {
46
- feedId: (0, cache_1.calculateFeedId)(context, r.params),
47
- },
48
- },
49
- };
50
- cacheEntry.value = { ...cacheEntry.value, ...metrics };
51
- }
52
- return cacheEntry;
53
- });
54
- exports.buildCacheEntriesFromResults = buildCacheEntriesFromResults;
55
- /**
56
- * Takes an Adapter, its configuration, and its dependencies, and it creates an express middleware
57
- * that will pass along the AdapterRequest to the appropriate Transport (acc. to the endpoint in the req.)
58
- *
59
- * @param adapter - main adapter object, already initialized
60
- * @returns the transport handler middleware function
61
- */
62
- const buildTransportHandler = (adapter) => async (req, reply) => {
63
- // Get transport, must be here because it's already checked in the validator
64
- const transport = adapter.endpointsMap[req.requestContext.endpointName].transport;
65
- // Set up transport if it hasn't been done already
66
- if (!(await transport.hasBeenSetUp(req))) {
67
- logger.debug('Transport not set up yet, doing so...');
68
- const immediateResponse = await transport.setup(req, adapter.config);
69
- if (immediateResponse) {
70
- logger.debug('Got immediate response from transport, sending as response');
71
- return reply.send(immediateResponse);
72
- }
73
- }
74
- logger.debug('Transport is set up, polling cache for response...');
75
- const response = await (0, cache_1.pollResponseFromCache)(adapter.dependencies.cache, req.requestContext.cacheKey, {
76
- maxRetries: adapter.config.CACHE_POLLING_MAX_RETRIES,
77
- sleep: adapter.config.CACHE_POLLING_SLEEP_MS,
78
- });
79
- if (response) {
80
- logger.debug('Got a response from the cache, sending that back');
81
- return reply.send(response);
82
- }
83
- logger.debug('Ran out of polling attempts, returning timeout');
84
- reply.statusCode = 504;
85
- return reply.send();
86
- };
87
- exports.buildTransportHandler = buildTransportHandler;
@@ -1,22 +0,0 @@
1
- import * as client from 'prom-client';
2
- import { AdapterContext } from '../adapter';
3
- import { SettingsMap } from '../config';
4
- export declare const dataProviderMetricsLabel: (providerStatusCode?: number, method?: string) => {
5
- provider_status_code: number | undefined;
6
- method: string;
7
- };
8
- export declare const dataProviderRequests: client.Counter<"method" | "provider_status_code">;
9
- export declare const dataProviderRequestDurationSeconds: client.Histogram<string>;
10
- export declare const connectionErrorLabels: (message: string) => {
11
- message: string;
12
- };
13
- export declare const messageSubsLabels: (feed_id: string, cache_key: string) => {
14
- feed_id: string;
15
- subscription_key: string;
16
- };
17
- export declare const recordWsMessageMetrics: <AdapterParams, CustomSettings extends SettingsMap>(context: AdapterContext<CustomSettings>, subscribes: AdapterParams[], unsubscrices: AdapterParams[]) => void;
18
- export declare const wsConnectionActive: client.Gauge<"url">;
19
- export declare const wsConnectionErrors: client.Counter<"message" | "url">;
20
- export declare const wsSubscriptionActive: client.Gauge<"feed_id" | "connection_url" | "subscription_key">;
21
- export declare const wsSubscriptionTotal: client.Counter<"feed_id" | "connection_url" | "subscription_key">;
22
- export declare const wsMessageTotal: client.Counter<"feed_id" | "subscription_key">;
@@ -1,105 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
25
- Object.defineProperty(exports, "__esModule", { value: true });
26
- exports.wsMessageTotal = exports.wsSubscriptionTotal = exports.wsSubscriptionActive = exports.wsConnectionErrors = exports.wsConnectionActive = exports.recordWsMessageMetrics = exports.messageSubsLabels = exports.connectionErrorLabels = exports.dataProviderRequestDurationSeconds = exports.dataProviderRequests = exports.dataProviderMetricsLabel = void 0;
27
- const client = __importStar(require("prom-client"));
28
- const cache_1 = require("../cache");
29
- const constants_1 = require("../metrics/constants");
30
- // Data Provider Metrics
31
- const dataProviderMetricsLabel = (providerStatusCode, method = 'get') => ({
32
- provider_status_code: providerStatusCode,
33
- method: method.toUpperCase(),
34
- });
35
- exports.dataProviderMetricsLabel = dataProviderMetricsLabel;
36
- exports.dataProviderRequests = new client.Counter({
37
- name: 'data_provider_requests',
38
- help: 'The number of http requests that are made to a data provider',
39
- labelNames: ['method', 'provider_status_code'],
40
- });
41
- exports.dataProviderRequestDurationSeconds = new client.Histogram({
42
- name: 'data_provider_request_duration_seconds',
43
- help: 'A histogram bucket of the distribution of data provider request durations',
44
- buckets: constants_1.requestDurationBuckets,
45
- });
46
- // Websocket Metrics
47
- const connectionErrorLabels = (message) => ({
48
- // Key,
49
- message,
50
- });
51
- exports.connectionErrorLabels = connectionErrorLabels;
52
- const messageSubsLabels = (feed_id, cache_key) => ({
53
- feed_id,
54
- subscription_key: cache_key,
55
- });
56
- exports.messageSubsLabels = messageSubsLabels;
57
- // Record WS message and subscription metrics
58
- // Recalculate cacheKey and feedId for metrics
59
- // since avoiding storing extra info in expiring sorted set
60
- const recordWsMessageMetrics = (context, subscribes, unsubscrices) => {
61
- subscribes.forEach((param) => {
62
- const feedId = (0, cache_1.calculateFeedId)(context, param);
63
- const cacheKey = (0, cache_1.calculateCacheKey)(context, param);
64
- // Record total number of ws messages sent
65
- exports.wsMessageTotal.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).inc();
66
- // Record total number of subscriptions made
67
- exports.wsSubscriptionTotal.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).inc();
68
- // Record number of active ws subscriptions
69
- exports.wsSubscriptionActive.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).inc();
70
- });
71
- unsubscrices.forEach((param) => {
72
- const feedId = (0, cache_1.calculateFeedId)(context, param);
73
- const cacheKey = (0, cache_1.calculateCacheKey)(context, param);
74
- // Record total number of ws messages sent
75
- exports.wsMessageTotal.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).inc();
76
- // Record number of active ws subscriptions
77
- exports.wsSubscriptionActive.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).dec();
78
- });
79
- };
80
- exports.recordWsMessageMetrics = recordWsMessageMetrics;
81
- exports.wsConnectionActive = new client.Gauge({
82
- name: 'ws_connection_active',
83
- help: 'The number of active connections',
84
- labelNames: ['url'],
85
- });
86
- exports.wsConnectionErrors = new client.Counter({
87
- name: 'ws_connection_errors',
88
- help: 'The number of connection errors',
89
- labelNames: ['url', 'message'],
90
- });
91
- exports.wsSubscriptionActive = new client.Gauge({
92
- name: 'ws_subscription_active',
93
- help: 'The number of currently active subscriptions',
94
- labelNames: ['connection_url', 'feed_id', 'subscription_key'],
95
- });
96
- exports.wsSubscriptionTotal = new client.Counter({
97
- name: 'ws_subscription_total',
98
- help: 'The number of subscriptions opened in total',
99
- labelNames: ['connection_url', 'feed_id', 'subscription_key'],
100
- });
101
- exports.wsMessageTotal = new client.Counter({
102
- name: 'ws_message_total',
103
- help: 'The number of messages received in total',
104
- labelNames: ['feed_id', 'subscription_key'],
105
- });
@@ -1,44 +0,0 @@
1
- import { AdapterRequest, AdapterResponse } from '../util/request';
2
- import { Transport } from './';
3
- import { Cache } from '../cache';
4
- import { AxiosRequestConfig, AxiosResponse } from 'axios';
5
- import { AdapterConfig, SettingsMap } from '../config';
6
- import { RequestRateLimiter } from '../rate-limiting';
7
- import { AdapterDependencies } from '../adapter';
8
- /**
9
- * Transport implementation that takes incoming requests, transforms them into a DataProvider request,
10
- * and executes that request returning the response immediately from the `setup` function.
11
- * Optionally, setting the `coalescing` option to `true` will make it so once a request is in flight,
12
- * new adapter requests for the same feed will not trigger a new one, but return an empty promise from
13
- * the setup instead so the normal cache polling mechanism is used.
14
- *
15
- * @typeParam AdapterParams - interface for the adapter request body
16
- * @typeParam ProviderRequestBody - interface for the body of the request to the Data Provider
17
- * @typeParam ProviderResponseBody - interface for the body of the Data Provider's response
18
- */
19
- export declare class RestTransport<AdapterParams, ProviderRequestBody, ProviderResponseBody, CustomSettings extends SettingsMap> implements Transport<AdapterParams, ProviderResponseBody, CustomSettings> {
20
- protected config: {
21
- prepareRequest: (req: AdapterRequest<AdapterParams>, config: AdapterConfig<CustomSettings>) => AxiosRequestConfig<ProviderRequestBody> | Promise<AxiosRequestConfig<ProviderRequestBody>>;
22
- parseResponse: (req: AdapterRequest<AdapterParams>, res: AxiosResponse<ProviderResponseBody>, config: AdapterConfig<CustomSettings>) => AdapterResponse<ProviderResponseBody> | Promise<AdapterResponse<ProviderResponseBody>>;
23
- options: {
24
- coalescing: boolean;
25
- };
26
- };
27
- inFlightPrefix: string;
28
- cache: Cache;
29
- rateLimiter: RequestRateLimiter;
30
- constructor(config: {
31
- prepareRequest: (req: AdapterRequest<AdapterParams>, config: AdapterConfig<CustomSettings>) => AxiosRequestConfig<ProviderRequestBody> | Promise<AxiosRequestConfig<ProviderRequestBody>>;
32
- parseResponse: (req: AdapterRequest<AdapterParams>, res: AxiosResponse<ProviderResponseBody>, config: AdapterConfig<CustomSettings>) => AdapterResponse<ProviderResponseBody> | Promise<AdapterResponse<ProviderResponseBody>>;
33
- options: {
34
- coalescing: boolean;
35
- };
36
- });
37
- initialize(dependencies: AdapterDependencies): Promise<void>;
38
- hasBeenSetUp(req: AdapterRequest<AdapterParams>): Promise<boolean>;
39
- protected waitUntilUnderRateLimit(options: {
40
- maxRetries: number;
41
- msBetweenRetries: number;
42
- }, retry?: number): Promise<void>;
43
- setup(req: AdapterRequest<AdapterParams>, config: AdapterConfig<CustomSettings>): Promise<AdapterResponse<ProviderResponseBody> | undefined>;
44
- }