@chainlink/external-adapter-framework 0.0.6 → 0.0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. package/adapter.d.ts +88 -0
  2. package/adapter.js +112 -0
  3. package/background-executor.d.ts +11 -0
  4. package/background-executor.js +45 -0
  5. package/cache/factory.d.ts +6 -0
  6. package/cache/factory.js +57 -0
  7. package/cache/index.d.ts +90 -0
  8. package/cache/index.js +169 -0
  9. package/cache/local.d.ts +23 -0
  10. package/cache/local.js +83 -0
  11. package/cache/metrics.d.ts +27 -0
  12. package/cache/metrics.js +120 -0
  13. package/cache/redis.d.ts +16 -0
  14. package/cache/redis.js +100 -0
  15. package/chainlink-external-adapter-framework-v0.0.8.tgz +0 -0
  16. package/config/index.d.ts +195 -0
  17. package/config/index.js +365 -0
  18. package/config/provider-limits.d.ts +31 -0
  19. package/config/provider-limits.js +75 -0
  20. package/examples/coingecko/batch-warming.d.ts +2 -0
  21. package/examples/coingecko/batch-warming.js +52 -0
  22. package/examples/coingecko/index.d.ts +2 -0
  23. package/examples/coingecko/index.js +10 -0
  24. package/examples/coingecko/rest.d.ts +2 -0
  25. package/examples/coingecko/rest.js +50 -0
  26. package/examples/ncfx/config/index.d.ts +12 -0
  27. package/examples/ncfx/config/index.js +15 -0
  28. package/examples/ncfx/index.d.ts +2 -0
  29. package/examples/ncfx/index.js +10 -0
  30. package/examples/ncfx/websocket.d.ts +36 -0
  31. package/examples/ncfx/websocket.js +72 -0
  32. package/index.d.ts +12 -0
  33. package/index.js +92 -0
  34. package/metrics/constants.d.ts +16 -0
  35. package/metrics/constants.js +25 -0
  36. package/metrics/index.d.ts +15 -0
  37. package/metrics/index.js +123 -0
  38. package/metrics/util.d.ts +3 -0
  39. package/metrics/util.js +9 -0
  40. package/package/adapter.d.ts +88 -0
  41. package/package/adapter.js +112 -0
  42. package/package/background-executor.d.ts +11 -0
  43. package/package/background-executor.js +45 -0
  44. package/package/cache/factory.d.ts +6 -0
  45. package/package/cache/factory.js +57 -0
  46. package/package/cache/index.d.ts +90 -0
  47. package/package/cache/index.js +169 -0
  48. package/package/cache/local.d.ts +23 -0
  49. package/package/cache/local.js +83 -0
  50. package/package/cache/metrics.d.ts +27 -0
  51. package/package/cache/metrics.js +120 -0
  52. package/package/cache/redis.d.ts +16 -0
  53. package/package/cache/redis.js +100 -0
  54. package/package/config/index.d.ts +195 -0
  55. package/package/config/index.js +365 -0
  56. package/package/config/provider-limits.d.ts +31 -0
  57. package/package/config/provider-limits.js +75 -0
  58. package/package/examples/coingecko/batch-warming.d.ts +2 -0
  59. package/package/examples/coingecko/batch-warming.js +52 -0
  60. package/package/examples/coingecko/index.d.ts +2 -0
  61. package/package/examples/coingecko/index.js +10 -0
  62. package/package/examples/coingecko/rest.d.ts +2 -0
  63. package/package/examples/coingecko/rest.js +50 -0
  64. package/package/examples/ncfx/config/index.d.ts +12 -0
  65. package/package/examples/ncfx/config/index.js +15 -0
  66. package/package/examples/ncfx/index.d.ts +2 -0
  67. package/package/examples/ncfx/index.js +10 -0
  68. package/package/examples/ncfx/websocket.d.ts +36 -0
  69. package/package/examples/ncfx/websocket.js +72 -0
  70. package/package/index.d.ts +12 -0
  71. package/package/index.js +92 -0
  72. package/package/metrics/constants.d.ts +16 -0
  73. package/package/metrics/constants.js +25 -0
  74. package/package/metrics/index.d.ts +15 -0
  75. package/package/metrics/index.js +123 -0
  76. package/package/metrics/util.d.ts +3 -0
  77. package/package/metrics/util.js +9 -0
  78. package/package/package.json +69 -0
  79. package/package/rate-limiting/background/fixed-frequency.d.ts +10 -0
  80. package/package/rate-limiting/background/fixed-frequency.js +37 -0
  81. package/package/rate-limiting/index.d.ts +54 -0
  82. package/package/rate-limiting/index.js +63 -0
  83. package/package/rate-limiting/metrics.d.ts +3 -0
  84. package/package/rate-limiting/metrics.js +44 -0
  85. package/package/rate-limiting/request/simple-counting.d.ts +20 -0
  86. package/package/rate-limiting/request/simple-counting.js +62 -0
  87. package/package/test.d.ts +1 -0
  88. package/package/test.js +6 -0
  89. package/package/transports/batch-warming.d.ts +34 -0
  90. package/package/transports/batch-warming.js +101 -0
  91. package/package/transports/index.d.ts +87 -0
  92. package/package/transports/index.js +87 -0
  93. package/package/transports/metrics.d.ts +21 -0
  94. package/package/transports/metrics.js +105 -0
  95. package/package/transports/rest.d.ts +43 -0
  96. package/package/transports/rest.js +129 -0
  97. package/package/transports/util.d.ts +8 -0
  98. package/package/transports/util.js +85 -0
  99. package/package/transports/websocket.d.ts +80 -0
  100. package/package/transports/websocket.js +169 -0
  101. package/package/util/expiring-sorted-set.d.ts +21 -0
  102. package/package/util/expiring-sorted-set.js +47 -0
  103. package/package/util/index.d.ts +11 -0
  104. package/package/util/index.js +35 -0
  105. package/package/util/logger.d.ts +42 -0
  106. package/package/util/logger.js +62 -0
  107. package/package/util/request.d.ts +55 -0
  108. package/package/util/request.js +2 -0
  109. package/package/validation/error.d.ts +50 -0
  110. package/package/validation/error.js +79 -0
  111. package/package/validation/index.d.ts +5 -0
  112. package/package/validation/index.js +86 -0
  113. package/package/validation/input-params.d.ts +15 -0
  114. package/package/validation/input-params.js +30 -0
  115. package/package/validation/override-functions.d.ts +3 -0
  116. package/package/validation/override-functions.js +40 -0
  117. package/package/validation/preset-tokens.json +23 -0
  118. package/package/validation/validator.d.ts +47 -0
  119. package/package/validation/validator.js +303 -0
  120. package/package.json +1 -1
  121. package/rate-limiting/background/fixed-frequency.d.ts +10 -0
  122. package/rate-limiting/background/fixed-frequency.js +37 -0
  123. package/rate-limiting/index.d.ts +54 -0
  124. package/rate-limiting/index.js +63 -0
  125. package/rate-limiting/metrics.d.ts +3 -0
  126. package/rate-limiting/metrics.js +44 -0
  127. package/rate-limiting/request/simple-counting.d.ts +20 -0
  128. package/rate-limiting/request/simple-counting.js +62 -0
  129. package/test.d.ts +1 -0
  130. package/test.js +6 -0
  131. package/transports/batch-warming.d.ts +34 -0
  132. package/transports/batch-warming.js +101 -0
  133. package/transports/index.d.ts +87 -0
  134. package/transports/index.js +87 -0
  135. package/transports/metrics.d.ts +21 -0
  136. package/transports/metrics.js +105 -0
  137. package/transports/rest.d.ts +43 -0
  138. package/transports/rest.js +129 -0
  139. package/transports/util.d.ts +8 -0
  140. package/transports/util.js +85 -0
  141. package/transports/websocket.d.ts +80 -0
  142. package/transports/websocket.js +169 -0
  143. package/util/expiring-sorted-set.d.ts +21 -0
  144. package/util/expiring-sorted-set.js +47 -0
  145. package/util/index.d.ts +11 -0
  146. package/util/index.js +35 -0
  147. package/util/logger.d.ts +42 -0
  148. package/util/logger.js +62 -0
  149. package/util/request.d.ts +55 -0
  150. package/util/request.js +2 -0
  151. package/validation/error.d.ts +50 -0
  152. package/validation/error.js +79 -0
  153. package/validation/index.d.ts +5 -0
  154. package/validation/index.js +86 -0
  155. package/validation/input-params.d.ts +15 -0
  156. package/validation/input-params.js +30 -0
  157. package/validation/override-functions.d.ts +3 -0
  158. package/validation/override-functions.js +40 -0
  159. package/validation/preset-tokens.json +23 -0
  160. package/validation/validator.d.ts +47 -0
  161. package/validation/validator.js +303 -0
  162. package/.c8rc.json +0 -3
  163. package/.eslintignore +0 -9
  164. package/.eslintrc.js +0 -96
  165. package/.github/README.MD +0 -17
  166. package/.github/actions/setup/action.yaml +0 -13
  167. package/.github/workflows/main.yaml +0 -39
  168. package/.github/workflows/publish.yaml +0 -17
  169. package/.prettierignore +0 -13
  170. package/.yarnrc +0 -0
  171. package/README.md +0 -103
  172. package/docker-compose.yaml +0 -35
  173. package/src/adapter.ts +0 -236
  174. package/src/background-executor.ts +0 -53
  175. package/src/cache/factory.ts +0 -28
  176. package/src/cache/index.ts +0 -236
  177. package/src/cache/local.ts +0 -73
  178. package/src/cache/metrics.ts +0 -112
  179. package/src/cache/redis.ts +0 -93
  180. package/src/config/index.ts +0 -501
  181. package/src/config/provider-limits.ts +0 -130
  182. package/src/examples/coingecko/batch-warming.ts +0 -79
  183. package/src/examples/coingecko/index.ts +0 -9
  184. package/src/examples/coingecko/rest.ts +0 -77
  185. package/src/examples/ncfx/config/index.ts +0 -12
  186. package/src/examples/ncfx/index.ts +0 -9
  187. package/src/examples/ncfx/websocket.ts +0 -100
  188. package/src/index.ts +0 -106
  189. package/src/metrics/constants.ts +0 -23
  190. package/src/metrics/index.ts +0 -116
  191. package/src/metrics/util.ts +0 -11
  192. package/src/rate-limiting/background/fixed-frequency.ts +0 -47
  193. package/src/rate-limiting/index.ts +0 -100
  194. package/src/rate-limiting/metrics.ts +0 -18
  195. package/src/rate-limiting/request/simple-counting.ts +0 -76
  196. package/src/test.ts +0 -5
  197. package/src/transports/batch-warming.ts +0 -121
  198. package/src/transports/index.ts +0 -173
  199. package/src/transports/metrics.ts +0 -95
  200. package/src/transports/rest.ts +0 -161
  201. package/src/transports/util.ts +0 -63
  202. package/src/transports/websocket.ts +0 -238
  203. package/src/util/expiring-sorted-set.ts +0 -52
  204. package/src/util/index.ts +0 -20
  205. package/src/util/logger.ts +0 -69
  206. package/src/util/request.ts +0 -115
  207. package/src/validation/error.ts +0 -116
  208. package/src/validation/index.ts +0 -101
  209. package/src/validation/input-params.ts +0 -45
  210. package/src/validation/override-functions.ts +0 -44
  211. package/src/validation/preset-tokens.json +0 -23
  212. package/src/validation/validator.ts +0 -384
  213. package/test/adapter.test.ts +0 -27
  214. package/test/background-executor.test.ts +0 -109
  215. package/test/cache/cache-key.test.ts +0 -96
  216. package/test/cache/helper.ts +0 -101
  217. package/test/cache/local.test.ts +0 -54
  218. package/test/cache/redis.test.ts +0 -89
  219. package/test/correlation.test.ts +0 -114
  220. package/test/index.test.ts +0 -37
  221. package/test/metrics/feed-id.test.ts +0 -33
  222. package/test/metrics/helper.ts +0 -14
  223. package/test/metrics/labels.test.ts +0 -36
  224. package/test/metrics/metrics.test.ts +0 -267
  225. package/test/metrics/redis-metrics.test.ts +0 -113
  226. package/test/metrics/warmer-metrics.test.ts +0 -192
  227. package/test/metrics/ws-metrics.test.ts +0 -225
  228. package/test/rate-limit-config.test.ts +0 -243
  229. package/test/transports/batch.test.ts +0 -465
  230. package/test/transports/rest.test.ts +0 -242
  231. package/test/transports/websocket.test.ts +0 -183
  232. package/test/tsconfig.json +0 -5
  233. package/test/util.ts +0 -76
  234. package/test/validation.test.ts +0 -169
  235. package/test.sh +0 -20
  236. package/tsconfig.json +0 -24
  237. package/typedoc.json +0 -6
@@ -0,0 +1,87 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
+ };
16
+ Object.defineProperty(exports, "__esModule", { value: true });
17
+ exports.buildTransportHandler = exports.buildCacheEntriesFromResults = void 0;
18
+ const cache_1 = require("../cache");
19
+ const util_1 = require("../util");
20
+ __exportStar(require("./batch-warming"), exports);
21
+ __exportStar(require("./rest"), exports);
22
+ __exportStar(require("./websocket"), exports);
23
+ const logger = (0, util_1.makeLogger)('Transport');
24
+ /**
25
+ * Helper method to build cache entries to set after getting a bunch of responses from a DP.
26
+ *
27
+ * @param results - a list of results coming from a DataProvider
28
+ * @param context - context for the Adapter
29
+ * @returns a list of CacheEntries of AdapterResponses
30
+ */
31
+ const buildCacheEntriesFromResults = (results, context) => results.map((r) => {
32
+ const cacheEntry = {
33
+ key: (0, cache_1.calculateCacheKey)(context, r.params),
34
+ value: {
35
+ result: r.value,
36
+ statusCode: 200,
37
+ data: null, // TODO: Maybe add data as well?
38
+ },
39
+ };
40
+ if (context.adapterConfig.METRICS_ENABLED &&
41
+ context.adapterConfig.EXPERIMENTAL_METRICS_ENABLED) {
42
+ const metrics = {
43
+ maxAge: Date.now() + context.adapterConfig.CACHE_MAX_AGE,
44
+ meta: {
45
+ metrics: {
46
+ feedId: (0, cache_1.calculateFeedId)(context.adapterEndpoint, r.params)
47
+ }
48
+ }
49
+ };
50
+ cacheEntry.value = { ...cacheEntry.value, ...metrics };
51
+ }
52
+ return cacheEntry;
53
+ });
54
+ exports.buildCacheEntriesFromResults = buildCacheEntriesFromResults;
55
+ /**
56
+ * Takes an Adapter, its configuration, and its dependencies, and it creates an express middleware
57
+ * that will pass along the AdapterRequest to the appropriate Transport (acc. to the endpoint in the req.)
58
+ *
59
+ * @param adapter - main adapter object, already initialized
60
+ * @returns the transport handler middleware function
61
+ */
62
+ const buildTransportHandler = (adapter) => async (req, reply) => {
63
+ // Get transport, must be here because it's already checked in the validator
64
+ const transport = adapter.endpointsMap[req.requestContext.endpointName].transport;
65
+ // Set up transport if it hasn't been done already
66
+ if (!(await transport.hasBeenSetUp(req))) {
67
+ logger.debug('Transport not set up yet, doing so...');
68
+ const immediateResponse = await transport.setup(req, adapter.config);
69
+ if (immediateResponse) {
70
+ logger.debug('Got immediate response from transport, sending as response');
71
+ return reply.send(immediateResponse);
72
+ }
73
+ }
74
+ logger.debug('Transport is set up, polling cache for response...');
75
+ const response = await (0, cache_1.pollResponseFromCache)(adapter.dependencies.cache, req.requestContext.cacheKey, {
76
+ maxRetries: adapter.config.CACHE_POLLING_MAX_RETRIES,
77
+ sleep: adapter.config.CACHE_POLLING_SLEEP_MS,
78
+ });
79
+ if (response) {
80
+ logger.debug('Got a response from the cache, sending that back');
81
+ return reply.send(response);
82
+ }
83
+ logger.debug('Ran out of polling attempts, returning timeout');
84
+ reply.statusCode = 504;
85
+ return reply.send();
86
+ };
87
+ exports.buildTransportHandler = buildTransportHandler;
@@ -0,0 +1,21 @@
1
+ import * as client from 'prom-client';
2
+ import { AdapterContext } from '.';
3
+ export declare const dataProviderMetricsLabel: (providerStatusCode?: number | undefined, method?: string) => {
4
+ provider_status_code: number | undefined;
5
+ method: string;
6
+ };
7
+ export declare const dataProviderRequests: client.Counter<"method" | "provider_status_code">;
8
+ export declare const dataProviderRequestDurationSeconds: client.Histogram<string>;
9
+ export declare const connectionErrorLabels: (message: string) => {
10
+ message: string;
11
+ };
12
+ export declare const messageSubsLabels: (feed_id: string, cache_key: string) => {
13
+ feed_id: string;
14
+ subscription_key: string;
15
+ };
16
+ export declare const recordWsMessageMetrics: <AdapterParams>(context: AdapterContext, subscribes: AdapterParams[], unsubscrices: AdapterParams[]) => void;
17
+ export declare const wsConnectionActive: client.Gauge<"url">;
18
+ export declare const wsConnectionErrors: client.Counter<"message" | "url">;
19
+ export declare const wsSubscriptionActive: client.Gauge<"feed_id" | "connection_url" | "subscription_key">;
20
+ export declare const wsSubscriptionTotal: client.Counter<"feed_id" | "connection_url" | "subscription_key">;
21
+ export declare const wsMessageTotal: client.Counter<"feed_id" | "subscription_key">;
@@ -0,0 +1,105 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || function (mod) {
19
+ if (mod && mod.__esModule) return mod;
20
+ var result = {};
21
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
+ __setModuleDefault(result, mod);
23
+ return result;
24
+ };
25
+ Object.defineProperty(exports, "__esModule", { value: true });
26
+ exports.wsMessageTotal = exports.wsSubscriptionTotal = exports.wsSubscriptionActive = exports.wsConnectionErrors = exports.wsConnectionActive = exports.recordWsMessageMetrics = exports.messageSubsLabels = exports.connectionErrorLabels = exports.dataProviderRequestDurationSeconds = exports.dataProviderRequests = exports.dataProviderMetricsLabel = void 0;
27
+ const client = __importStar(require("prom-client"));
28
+ const cache_1 = require("../cache");
29
+ const constants_1 = require("../metrics/constants");
30
+ // Data Provider Metrics
31
+ const dataProviderMetricsLabel = (providerStatusCode, method = 'get') => ({
32
+ provider_status_code: providerStatusCode,
33
+ method: method.toUpperCase(),
34
+ });
35
+ exports.dataProviderMetricsLabel = dataProviderMetricsLabel;
36
+ exports.dataProviderRequests = new client.Counter({
37
+ name: 'data_provider_requests',
38
+ help: 'The number of http requests that are made to a data provider',
39
+ labelNames: ['method', 'provider_status_code'],
40
+ });
41
+ exports.dataProviderRequestDurationSeconds = new client.Histogram({
42
+ name: 'data_provider_request_duration_seconds',
43
+ help: 'A histogram bucket of the distribution of data provider request durations',
44
+ buckets: constants_1.requestDurationBuckets,
45
+ });
46
+ // Websocket Metrics
47
+ const connectionErrorLabels = (message) => ({
48
+ // Key,
49
+ message,
50
+ });
51
+ exports.connectionErrorLabels = connectionErrorLabels;
52
+ const messageSubsLabels = (feed_id, cache_key) => ({
53
+ feed_id,
54
+ subscription_key: cache_key,
55
+ });
56
+ exports.messageSubsLabels = messageSubsLabels;
57
+ // Record WS message and subscription metrics
58
+ // Recalculate cacheKey and feedId for metrics
59
+ // since avoiding storing extra info in expiring sorted set
60
+ const recordWsMessageMetrics = (context, subscribes, unsubscrices) => {
61
+ subscribes.forEach((param) => {
62
+ const feedId = (0, cache_1.calculateFeedId)(context.adapterEndpoint, param);
63
+ const cacheKey = (0, cache_1.calculateCacheKey)(context, param);
64
+ // Record total number of ws messages sent
65
+ exports.wsMessageTotal.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).inc();
66
+ // Record total number of subscriptions made
67
+ exports.wsSubscriptionTotal.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).inc();
68
+ // Record number of active ws subscriptions
69
+ exports.wsSubscriptionActive.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).inc();
70
+ });
71
+ unsubscrices.forEach((param) => {
72
+ const feedId = (0, cache_1.calculateFeedId)(context.adapterEndpoint, param);
73
+ const cacheKey = (0, cache_1.calculateCacheKey)(context, param);
74
+ // Record total number of ws messages sent
75
+ exports.wsMessageTotal.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).inc();
76
+ // Record number of active ws subscriptions
77
+ exports.wsSubscriptionActive.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).dec();
78
+ });
79
+ };
80
+ exports.recordWsMessageMetrics = recordWsMessageMetrics;
81
+ exports.wsConnectionActive = new client.Gauge({
82
+ name: 'ws_connection_active',
83
+ help: 'The number of active connections',
84
+ labelNames: ['url'],
85
+ });
86
+ exports.wsConnectionErrors = new client.Counter({
87
+ name: 'ws_connection_errors',
88
+ help: 'The number of connection errors',
89
+ labelNames: ['url', 'message'],
90
+ });
91
+ exports.wsSubscriptionActive = new client.Gauge({
92
+ name: 'ws_subscription_active',
93
+ help: 'The number of currently active subscriptions',
94
+ labelNames: ['connection_url', 'feed_id', 'subscription_key'],
95
+ });
96
+ exports.wsSubscriptionTotal = new client.Counter({
97
+ name: 'ws_subscription_total',
98
+ help: 'The number of subscriptions opened in total',
99
+ labelNames: ['connection_url', 'feed_id', 'subscription_key'],
100
+ });
101
+ exports.wsMessageTotal = new client.Counter({
102
+ name: 'ws_message_total',
103
+ help: 'The number of messages received in total',
104
+ labelNames: ['feed_id', 'subscription_key'],
105
+ });
@@ -0,0 +1,43 @@
1
+ import { AdapterRequest, AdapterResponse } from '../util/request';
2
+ import { AdapterDependencies, Transport } from './';
3
+ import { Cache } from '../cache';
4
+ import { AxiosRequestConfig, AxiosResponse } from 'axios';
5
+ import { AdapterConfig, SettingsMap } from '../config';
6
+ import { RequestRateLimiter } from '../rate-limiting';
7
+ /**
8
+ * Transport implementation that takes incoming requests, transforms them into a DataProvider request,
9
+ * and executes that request returning the response immediately from the `setup` function.
10
+ * Optionally, setting the `coalescing` option to `true` will make it so once a request is in flight,
11
+ * new adapter requests for the same feed will not trigger a new one, but return an empty promise from
12
+ * the setup instead so the normal cache polling mechanism is used.
13
+ *
14
+ * @typeParam AdapterParams - interface for the adapter request body
15
+ * @typeParam ProviderRequestBody - interface for the body of the request to the Data Provider
16
+ * @typeParam ProviderResponseBody - interface for the body of the Data Provider's response
17
+ */
18
+ export declare class RestTransport<AdapterParams, ProviderRequestBody, ProviderResponseBody, CustomSettings extends SettingsMap> implements Transport<AdapterParams, ProviderResponseBody, CustomSettings> {
19
+ protected config: {
20
+ prepareRequest: (req: AdapterRequest<AdapterParams>, config: AdapterConfig<CustomSettings>) => AxiosRequestConfig<ProviderRequestBody> | Promise<AxiosRequestConfig<ProviderRequestBody>>;
21
+ parseResponse: (req: AdapterRequest<AdapterParams>, res: AxiosResponse<ProviderResponseBody>, config: AdapterConfig<CustomSettings>) => AdapterResponse<ProviderResponseBody> | Promise<AdapterResponse<ProviderResponseBody>>;
22
+ options: {
23
+ coalescing: boolean;
24
+ };
25
+ };
26
+ inFlightPrefix: string;
27
+ cache: Cache;
28
+ rateLimiter: RequestRateLimiter;
29
+ constructor(config: {
30
+ prepareRequest: (req: AdapterRequest<AdapterParams>, config: AdapterConfig<CustomSettings>) => AxiosRequestConfig<ProviderRequestBody> | Promise<AxiosRequestConfig<ProviderRequestBody>>;
31
+ parseResponse: (req: AdapterRequest<AdapterParams>, res: AxiosResponse<ProviderResponseBody>, config: AdapterConfig<CustomSettings>) => AdapterResponse<ProviderResponseBody> | Promise<AdapterResponse<ProviderResponseBody>>;
32
+ options: {
33
+ coalescing: boolean;
34
+ };
35
+ });
36
+ initialize(dependencies: AdapterDependencies): Promise<void>;
37
+ hasBeenSetUp(req: AdapterRequest<AdapterParams>): Promise<boolean>;
38
+ protected waitUntilUnderRateLimit(options: {
39
+ maxRetries: number;
40
+ msBetweenRetries: number;
41
+ }, retry?: number): Promise<void>;
42
+ setup(req: AdapterRequest<AdapterParams>, config: AdapterConfig<CustomSettings>): Promise<AdapterResponse<ProviderResponseBody> | undefined>;
43
+ }
@@ -0,0 +1,129 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || function (mod) {
19
+ if (mod && mod.__esModule) return mod;
20
+ var result = {};
21
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
+ __setModuleDefault(result, mod);
23
+ return result;
24
+ };
25
+ Object.defineProperty(exports, "__esModule", { value: true });
26
+ exports.RestTransport = void 0;
27
+ const util_1 = require("../util");
28
+ const error_1 = require("../validation/error");
29
+ const util_2 = require("./util");
30
+ const rateLimitMetrics = __importStar(require("../rate-limiting/metrics"));
31
+ const IN_FLIGHT_PREFIX = 'InFlight';
32
+ const logger = (0, util_1.makeLogger)('RestTransport');
33
+ /**
34
+ * Transport implementation that takes incoming requests, transforms them into a DataProvider request,
35
+ * and executes that request returning the response immediately from the `setup` function.
36
+ * Optionally, setting the `coalescing` option to `true` will make it so once a request is in flight,
37
+ * new adapter requests for the same feed will not trigger a new one, but return an empty promise from
38
+ * the setup instead so the normal cache polling mechanism is used.
39
+ *
40
+ * @typeParam AdapterParams - interface for the adapter request body
41
+ * @typeParam ProviderRequestBody - interface for the body of the request to the Data Provider
42
+ * @typeParam ProviderResponseBody - interface for the body of the Data Provider's response
43
+ */
44
+ class RestTransport {
45
+ constructor(config) {
46
+ this.config = config;
47
+ }
48
+ async initialize(dependencies) {
49
+ this.inFlightPrefix = `${IN_FLIGHT_PREFIX}-`;
50
+ this.cache = dependencies.cache;
51
+ this.rateLimiter = dependencies.requestRateLimiter;
52
+ }
53
+ async hasBeenSetUp(req) {
54
+ if (!this.config.options.coalescing) {
55
+ return false;
56
+ }
57
+ // Check if request is in flight
58
+ const inFlight = await this.cache.get(this.inFlightPrefix + req.requestContext.cacheKey);
59
+ if (inFlight) {
60
+ logger.debug('Request is in flight, transport has been set up');
61
+ return true;
62
+ }
63
+ else {
64
+ logger.debug('Request not in flight, transport not set up');
65
+ return false;
66
+ }
67
+ }
68
+ async waitUntilUnderRateLimit(options, retry = 0) {
69
+ if (this.rateLimiter.isUnderLimits()) {
70
+ logger.trace('Incoming request would not be under limits, moving on');
71
+ return;
72
+ }
73
+ if (retry >= options.maxRetries) {
74
+ throw new error_1.AdapterError({
75
+ statusCode: 504,
76
+ message: `REST Transport timed out while waiting for rate limit availability (max retries: ${options.maxRetries})`,
77
+ });
78
+ }
79
+ logger.debug(`Request would be over rate limits, sleeping for ${options.msBetweenRetries}`);
80
+ await (0, util_1.sleep)(options.msBetweenRetries);
81
+ await this.waitUntilUnderRateLimit(options, retry + 1);
82
+ }
83
+ async setup(req, config) {
84
+ if (this.config.options.coalescing) {
85
+ logger.debug('Setting up rest transport, setting request in flight in cache');
86
+ // TODO: Should this use a separate cache?
87
+ // TODO: Set viable ttl to approximate timeout from API
88
+ // TODO: Make this ttl configurable
89
+ await this.cache.set(this.inFlightPrefix + req.requestContext.cacheKey, true, 2000); // Can't use Infinity for things like Redis
90
+ }
91
+ const request = await this.config.prepareRequest(req, config);
92
+ logger.trace('Check if we are under rate limits to perform request');
93
+ await this.waitUntilUnderRateLimit({
94
+ maxRetries: config.REST_TRANSPORT_MAX_RATE_LIMIT_RETRIES,
95
+ msBetweenRetries: config.REST_TRANSPORT_MS_BETWEEN_RATE_LIMIT_RETRIES,
96
+ });
97
+ logger.trace('Sending request to data provider...');
98
+ const providerResponse = await (0, util_2.axiosRequest)(request);
99
+ logger.debug(`Got response from provider, parsing (raw body: ${providerResponse.data})`); // TODO: Sensitive data?
100
+ const parsedResponse = await this.config.parseResponse(req, providerResponse, config);
101
+ if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
102
+ // TODO: Potentially create function to add all telemetry data
103
+ parsedResponse.maxAge = Date.now() + config.CACHE_MAX_AGE;
104
+ parsedResponse.meta = { metrics: { feedId: req.requestContext.meta?.metrics?.feedId || 'N/A' } };
105
+ }
106
+ logger.debug('Setting provider response in cache');
107
+ await this.cache.set(req.requestContext.cacheKey, parsedResponse, config.CACHE_MAX_AGE);
108
+ // Record cost of data provider call
109
+ const cost = rateLimitMetrics.retrieveCost(providerResponse.data);
110
+ rateLimitMetrics.rateLimitCreditsSpentTotal
111
+ .labels({
112
+ feed_id: req.requestContext.meta?.metrics?.feedId || 'N/A',
113
+ participant_id: req.requestContext.cacheKey,
114
+ })
115
+ .inc(cost);
116
+ // Update cacheHit flag in request meta for metrics use
117
+ req.requestContext.meta = {
118
+ ...req.requestContext.meta,
119
+ metrics: { ...req.requestContext.meta?.metrics, cacheHit: false },
120
+ };
121
+ // TODO: move this to a try/catch/finally
122
+ if (this.config.options.coalescing) {
123
+ logger.debug('Set provider response in cache, removing in flight from cache');
124
+ await this.cache.delete(this.inFlightPrefix);
125
+ }
126
+ return parsedResponse;
127
+ }
128
+ }
129
+ exports.RestTransport = RestTransport;
@@ -0,0 +1,8 @@
1
+ import { AxiosRequestConfig, AxiosResponse } from 'axios';
2
+ /**
3
+ * Performs axios request along with metrics recording and error handling
4
+ *
5
+ * @param request - axios request config
6
+ * @returns axios response for the request
7
+ */
8
+ export declare function axiosRequest<ProviderRequestBody, ProviderResponseBody>(request: AxiosRequestConfig<ProviderRequestBody>): Promise<AxiosResponse<ProviderResponseBody>>;
@@ -0,0 +1,85 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || function (mod) {
19
+ if (mod && mod.__esModule) return mod;
20
+ var result = {};
21
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
+ __setModuleDefault(result, mod);
23
+ return result;
24
+ };
25
+ var __importDefault = (this && this.__importDefault) || function (mod) {
26
+ return (mod && mod.__esModule) ? mod : { "default": mod };
27
+ };
28
+ Object.defineProperty(exports, "__esModule", { value: true });
29
+ exports.axiosRequest = void 0;
30
+ const axios_1 = __importDefault(require("axios"));
31
+ const error_1 = require("../validation/error");
32
+ const transportMetrics = __importStar(require("./metrics"));
33
+ /**
34
+ * Performs axios request along with metrics recording and error handling
35
+ *
36
+ * @param request - axios request config
37
+ * @returns axios response for the request
38
+ */
39
+ async function axiosRequest(request) {
40
+ const responseTimer = transportMetrics.dataProviderRequestDurationSeconds.startTimer();
41
+ let providerResponse;
42
+ try {
43
+ providerResponse = await axios_1.default.request(request);
44
+ }
45
+ catch (e) {
46
+ const error = e;
47
+ // Request error
48
+ let providerStatusCode;
49
+ let adapterError;
50
+ if (error.code === 'ECONNABORTED') {
51
+ adapterError = new error_1.AdapterTimeoutError({});
52
+ providerStatusCode = error?.response?.status ?? 504;
53
+ adapterError.name = 'Data Provider Request Timeout error';
54
+ }
55
+ else if (error?.response?.status) {
56
+ adapterError = new error_1.AdapterDataProviderError({});
57
+ providerStatusCode = error?.response?.status;
58
+ }
59
+ else {
60
+ adapterError = new error_1.AdapterConnectionError({});
61
+ providerStatusCode = 0; // 0 -> connection error
62
+ }
63
+ // Record count of failed data provider request
64
+ transportMetrics.dataProviderRequests
65
+ .labels(transportMetrics.dataProviderMetricsLabel(providerStatusCode, request.method))
66
+ .inc();
67
+ adapterError.statusCode = 200;
68
+ adapterError.providerStatusCode = providerStatusCode;
69
+ adapterError.message = error?.message;
70
+ adapterError.cause = error;
71
+ adapterError.errorResponse = error?.response?.data;
72
+ adapterError.url = request.url;
73
+ throw adapterError;
74
+ }
75
+ finally {
76
+ // Record time taken for data provider request for success or failure
77
+ responseTimer();
78
+ }
79
+ // Record count of successful data provider requests
80
+ transportMetrics.dataProviderRequests
81
+ .labels(transportMetrics.dataProviderMetricsLabel(providerResponse.status, request.method))
82
+ .inc();
83
+ return providerResponse;
84
+ }
85
+ exports.axiosRequest = axiosRequest;
@@ -0,0 +1,80 @@
1
+ import WebSocket from 'ws';
2
+ import { Cache } from '../cache';
3
+ import { SettingsMap } from '../config';
4
+ import { BackgroundExecuteRateLimiter } from '../rate-limiting';
5
+ import { ExpiringSortedSet } from '../util';
6
+ import { AdapterRequest, ProviderResult } from '../util/request';
7
+ import { AdapterContext, AdapterDependencies, Transport } from './';
8
+ export declare const DEFAULT_WS_TTL = 10000;
9
+ declare type WebSocketClass = new (url: string, protocols?: string | string[] | undefined) => WebSocket;
10
+ export declare class WebSocketClassProvider {
11
+ static ctor: WebSocketClass;
12
+ static set(ctor: WebSocketClass): void;
13
+ static get(): WebSocketClass;
14
+ }
15
+ /**
16
+ * Config object that is provided to the WebSocketTransport constructor.
17
+ */
18
+ export interface WebSocketTransportConfig<AdapterParams, ProviderDataMessage, CustomSettings extends SettingsMap> {
19
+ /** Endpoint to which to open the WS connection*/
20
+ url: string;
21
+ /** Map of handlers for different WS lifecycle events */
22
+ handlers: {
23
+ /**
24
+ * Handles when the WS is successfully opened
25
+ *
26
+ * @param wsConnection - the WebSocket with an established connection
27
+ * @returns an empty Promise, or void
28
+ */
29
+ open: (wsConnection: WebSocket, context: AdapterContext<CustomSettings>) => Promise<void> | void;
30
+ /**
31
+ * Handles when the WS receives a message
32
+ *
33
+ * @param message - the message received by the WS
34
+ * @param context - the background context for the Adapter
35
+ * @returns a list of cache entries of adapter responses to set in the cache
36
+ */
37
+ message: (message: ProviderDataMessage, context: AdapterContext<CustomSettings>) => ProviderResult<AdapterParams>[];
38
+ };
39
+ /** Map of "builders", functions that will be used to prepare specific WS messages */
40
+ builders: {
41
+ /**
42
+ * Builds a WS message that will be sent to subscribe to a specific feed
43
+ *
44
+ * @param params - the body of the adapter request
45
+ * @returns the WS message (can be any type as long as the [[WebSocket]] doesn't complain)
46
+ */
47
+ subscribeMessage: (params: AdapterParams) => unknown;
48
+ /**
49
+ * Builds a WS message that will be sent to unsubscribe to a specific feed
50
+ *
51
+ * @param params - the body of the adapter request
52
+ * @returns the WS message (can be any type as long as the [[WebSocket]] doesn't complain)
53
+ */
54
+ unsubscribeMessage: (params: AdapterParams) => unknown;
55
+ };
56
+ }
57
+ /**
58
+ * Transport implementation that takes incoming requests, adds them to an [[ExpiringSortedSet]] and,
59
+ * through a WebSocket connection, subscribes to the relevant feeds to populate the cache.
60
+ *
61
+ * @typeParam AdapterParams - interface for the adapter request body
62
+ * @typeParam ProviderDataMessage - interface for a WS message containing processable data (i.e. not part of open/close/login/etc)
63
+ */
64
+ export declare class WebSocketTransport<AdapterParams, ProviderDataMessage, CustomSettings extends SettingsMap> implements Transport<AdapterParams, null, CustomSettings> {
65
+ private config;
66
+ cache: Cache;
67
+ rateLimiter: BackgroundExecuteRateLimiter;
68
+ expiringSortedSet: ExpiringSortedSet<AdapterParams>;
69
+ localSubscriptions: AdapterParams[];
70
+ wsConnection: WebSocket;
71
+ constructor(config: WebSocketTransportConfig<AdapterParams, ProviderDataMessage, CustomSettings>);
72
+ initialize(dependencies: AdapterDependencies): Promise<void>;
73
+ hasBeenSetUp(req: AdapterRequest<AdapterParams>): Promise<boolean>;
74
+ setup(req: AdapterRequest<AdapterParams>): Promise<void>;
75
+ serializeMessage(payload: unknown): string;
76
+ deserializeMessage(data: WebSocket.Data): ProviderDataMessage;
77
+ establishWsConnection(context: AdapterContext<CustomSettings>): Promise<unknown>;
78
+ backgroundExecute(context: AdapterContext<CustomSettings>): Promise<number>;
79
+ }
80
+ export {};