@chainlink/external-adapter-framework 0.0.12 → 0.0.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (220) hide show
  1. package/.eslintignore +1 -0
  2. package/.github/README.MD +35 -10
  3. package/.github/workflows/label.yaml +39 -0
  4. package/.github/workflows/publish.yaml +0 -3
  5. package/dist/examples/coingecko/test/e2e/adapter.test.ts.js +82953 -0
  6. package/dist/examples/coingecko/test/integration/adapter.test.ts.js +91672 -0
  7. package/dist/main.js +72703 -0
  8. package/jest.config.js +5 -0
  9. package/package.json +14 -3
  10. package/src/config/provider-limits.ts +14 -17
  11. package/src/examples/coingecko/src/config/index.ts +12 -0
  12. package/src/examples/coingecko/src/config/overrides.json +10826 -0
  13. package/src/examples/coingecko/src/cryptoUtils.ts +88 -0
  14. package/src/examples/coingecko/src/endpoint/coins.ts +54 -0
  15. package/src/examples/coingecko/src/endpoint/crypto-marketcap.ts +66 -0
  16. package/src/examples/coingecko/src/endpoint/crypto-volume.ts +66 -0
  17. package/src/examples/coingecko/src/endpoint/crypto.ts +63 -0
  18. package/src/examples/coingecko/src/endpoint/dominance.ts +40 -0
  19. package/src/examples/coingecko/src/endpoint/global-marketcap.ts +40 -0
  20. package/src/examples/coingecko/src/endpoint/index.ts +6 -0
  21. package/src/examples/coingecko/src/globalUtils.ts +78 -0
  22. package/src/examples/coingecko/src/index.ts +17 -0
  23. package/src/examples/coingecko/test/e2e/adapter.test.ts +278 -0
  24. package/src/examples/coingecko/test/integration/__snapshots__/adapter.test.ts.snap +15 -0
  25. package/src/examples/coingecko/test/integration/adapter.test.ts +281 -0
  26. package/src/examples/coingecko/test/integration/capturedRequests.json +1 -0
  27. package/src/examples/coingecko/test/integration/fixtures.ts +42 -0
  28. package/src/examples/{coingecko → coingecko-old}/batch-warming.ts +1 -0
  29. package/src/examples/{coingecko → coingecko-old}/index.ts +0 -0
  30. package/src/examples/{coingecko → coingecko-old}/rest.ts +0 -0
  31. package/src/transports/batch-warming.ts +7 -2
  32. package/src/transports/rest.ts +5 -1
  33. package/src/util/index.ts +1 -0
  34. package/src/util/recordRequests.ts +47 -0
  35. package/src/validation/overrideFunctions.ts +44 -0
  36. package/test/metrics/warmer-metrics.test.ts +1 -0
  37. package/test/{smoke.test.ts → smoke/smoke.test.ts} +13 -13
  38. package/{test-payload-fail.json → test/smoke/test-payload-fail.json} +0 -0
  39. package/{test-payload.js → test/smoke/test-payload.js} +0 -0
  40. package/{test-payload.json → test/smoke/test-payload.json} +0 -0
  41. package/test/transports/batch.test.ts +1 -0
  42. package/tsconfig.json +4 -1
  43. package/webpack.config.js +39 -5
  44. package/yarn-error.log +3778 -0
  45. package/dist/src/adapter.d.ts +0 -135
  46. package/dist/src/adapter.js +0 -145
  47. package/dist/src/background-executor.d.ts +0 -11
  48. package/dist/src/background-executor.js +0 -45
  49. package/dist/src/cache/factory.d.ts +0 -6
  50. package/dist/src/cache/factory.js +0 -55
  51. package/dist/src/cache/index.d.ts +0 -94
  52. package/dist/src/cache/index.js +0 -173
  53. package/dist/src/cache/local.d.ts +0 -23
  54. package/dist/src/cache/local.js +0 -83
  55. package/dist/src/cache/metrics.d.ts +0 -27
  56. package/dist/src/cache/metrics.js +0 -120
  57. package/dist/src/cache/redis.d.ts +0 -16
  58. package/dist/src/cache/redis.js +0 -100
  59. package/dist/src/chainlink-external-adapter-framework-0.0.6.tgz +0 -0
  60. package/dist/src/config/index.d.ts +0 -214
  61. package/dist/src/config/index.js +0 -380
  62. package/dist/src/config/provider-limits.d.ts +0 -31
  63. package/dist/src/config/provider-limits.js +0 -75
  64. package/dist/src/examples/bank-frick/accounts.d.ts +0 -39
  65. package/dist/src/examples/bank-frick/accounts.js +0 -192
  66. package/dist/src/examples/bank-frick/config/index.d.ts +0 -4
  67. package/dist/src/examples/bank-frick/config/index.js +0 -54
  68. package/dist/src/examples/bank-frick/index.d.ts +0 -2
  69. package/dist/src/examples/bank-frick/index.js +0 -15
  70. package/dist/src/examples/bank-frick/util.d.ts +0 -4
  71. package/dist/src/examples/bank-frick/util.js +0 -39
  72. package/dist/src/examples/coingecko/batch-warming.d.ts +0 -7
  73. package/dist/src/examples/coingecko/batch-warming.js +0 -53
  74. package/dist/src/examples/coingecko/index.d.ts +0 -2
  75. package/dist/src/examples/coingecko/index.js +0 -11
  76. package/dist/src/examples/coingecko/rest.d.ts +0 -12
  77. package/dist/src/examples/coingecko/rest.js +0 -51
  78. package/dist/src/examples/ncfx/config/index.d.ts +0 -12
  79. package/dist/src/examples/ncfx/config/index.js +0 -15
  80. package/dist/src/examples/ncfx/index.d.ts +0 -13
  81. package/dist/src/examples/ncfx/index.js +0 -11
  82. package/dist/src/examples/ncfx/websocket.d.ts +0 -47
  83. package/dist/src/examples/ncfx/websocket.js +0 -73
  84. package/dist/src/index.d.ts +0 -11
  85. package/dist/src/index.js +0 -128
  86. package/dist/src/metrics/constants.d.ts +0 -16
  87. package/dist/src/metrics/constants.js +0 -25
  88. package/dist/src/metrics/index.d.ts +0 -15
  89. package/dist/src/metrics/index.js +0 -122
  90. package/dist/src/metrics/util.d.ts +0 -7
  91. package/dist/src/metrics/util.js +0 -9
  92. package/dist/src/package/adapter.d.ts +0 -88
  93. package/dist/src/package/adapter.js +0 -112
  94. package/dist/src/package/background-executor.d.ts +0 -11
  95. package/dist/src/package/background-executor.js +0 -45
  96. package/dist/src/package/cache/factory.d.ts +0 -6
  97. package/dist/src/package/cache/factory.js +0 -57
  98. package/dist/src/package/cache/index.d.ts +0 -90
  99. package/dist/src/package/cache/index.js +0 -169
  100. package/dist/src/package/cache/local.d.ts +0 -23
  101. package/dist/src/package/cache/local.js +0 -83
  102. package/dist/src/package/cache/metrics.d.ts +0 -27
  103. package/dist/src/package/cache/metrics.js +0 -120
  104. package/dist/src/package/cache/redis.d.ts +0 -16
  105. package/dist/src/package/cache/redis.js +0 -100
  106. package/dist/src/package/config/index.d.ts +0 -195
  107. package/dist/src/package/config/index.js +0 -365
  108. package/dist/src/package/config/provider-limits.d.ts +0 -31
  109. package/dist/src/package/config/provider-limits.js +0 -76
  110. package/dist/src/package/examples/coingecko/batch-warming.d.ts +0 -2
  111. package/dist/src/package/examples/coingecko/batch-warming.js +0 -52
  112. package/dist/src/package/examples/coingecko/index.d.ts +0 -2
  113. package/dist/src/package/examples/coingecko/index.js +0 -10
  114. package/dist/src/package/examples/coingecko/rest.d.ts +0 -2
  115. package/dist/src/package/examples/coingecko/rest.js +0 -50
  116. package/dist/src/package/examples/ncfx/config/index.d.ts +0 -12
  117. package/dist/src/package/examples/ncfx/config/index.js +0 -15
  118. package/dist/src/package/examples/ncfx/index.d.ts +0 -2
  119. package/dist/src/package/examples/ncfx/index.js +0 -10
  120. package/dist/src/package/examples/ncfx/websocket.d.ts +0 -36
  121. package/dist/src/package/examples/ncfx/websocket.js +0 -72
  122. package/dist/src/package/index.d.ts +0 -12
  123. package/dist/src/package/index.js +0 -92
  124. package/dist/src/package/metrics/constants.d.ts +0 -16
  125. package/dist/src/package/metrics/constants.js +0 -25
  126. package/dist/src/package/metrics/index.d.ts +0 -15
  127. package/dist/src/package/metrics/index.js +0 -123
  128. package/dist/src/package/metrics/util.d.ts +0 -3
  129. package/dist/src/package/metrics/util.js +0 -9
  130. package/dist/src/package/package.json +0 -72
  131. package/dist/src/package/rate-limiting/background/fixed-frequency.d.ts +0 -10
  132. package/dist/src/package/rate-limiting/background/fixed-frequency.js +0 -37
  133. package/dist/src/package/rate-limiting/index.d.ts +0 -54
  134. package/dist/src/package/rate-limiting/index.js +0 -63
  135. package/dist/src/package/rate-limiting/metrics.d.ts +0 -3
  136. package/dist/src/package/rate-limiting/metrics.js +0 -44
  137. package/dist/src/package/rate-limiting/request/simple-counting.d.ts +0 -20
  138. package/dist/src/package/rate-limiting/request/simple-counting.js +0 -62
  139. package/dist/src/package/test.d.ts +0 -1
  140. package/dist/src/package/test.js +0 -6
  141. package/dist/src/package/transports/batch-warming.d.ts +0 -34
  142. package/dist/src/package/transports/batch-warming.js +0 -101
  143. package/dist/src/package/transports/index.d.ts +0 -87
  144. package/dist/src/package/transports/index.js +0 -87
  145. package/dist/src/package/transports/metrics.d.ts +0 -21
  146. package/dist/src/package/transports/metrics.js +0 -105
  147. package/dist/src/package/transports/rest.d.ts +0 -43
  148. package/dist/src/package/transports/rest.js +0 -129
  149. package/dist/src/package/transports/util.d.ts +0 -8
  150. package/dist/src/package/transports/util.js +0 -85
  151. package/dist/src/package/transports/websocket.d.ts +0 -80
  152. package/dist/src/package/transports/websocket.js +0 -169
  153. package/dist/src/package/util/expiring-sorted-set.d.ts +0 -21
  154. package/dist/src/package/util/expiring-sorted-set.js +0 -47
  155. package/dist/src/package/util/index.d.ts +0 -11
  156. package/dist/src/package/util/index.js +0 -35
  157. package/dist/src/package/util/logger.d.ts +0 -42
  158. package/dist/src/package/util/logger.js +0 -62
  159. package/dist/src/package/util/request.d.ts +0 -55
  160. package/dist/src/package/util/request.js +0 -2
  161. package/dist/src/package/validation/error.d.ts +0 -50
  162. package/dist/src/package/validation/error.js +0 -79
  163. package/dist/src/package/validation/index.d.ts +0 -5
  164. package/dist/src/package/validation/index.js +0 -86
  165. package/dist/src/package/validation/input-params.d.ts +0 -15
  166. package/dist/src/package/validation/input-params.js +0 -30
  167. package/dist/src/package/validation/override-functions.d.ts +0 -3
  168. package/dist/src/package/validation/override-functions.js +0 -40
  169. package/dist/src/package/validation/preset-tokens.json +0 -23
  170. package/dist/src/package/validation/validator.d.ts +0 -47
  171. package/dist/src/package/validation/validator.js +0 -303
  172. package/dist/src/package.json +0 -72
  173. package/dist/src/rate-limiting/background/fixed-frequency.d.ts +0 -11
  174. package/dist/src/rate-limiting/background/fixed-frequency.js +0 -35
  175. package/dist/src/rate-limiting/index.d.ts +0 -55
  176. package/dist/src/rate-limiting/index.js +0 -63
  177. package/dist/src/rate-limiting/metrics.d.ts +0 -3
  178. package/dist/src/rate-limiting/metrics.js +0 -44
  179. package/dist/src/rate-limiting/request/simple-counting.d.ts +0 -21
  180. package/dist/src/rate-limiting/request/simple-counting.js +0 -62
  181. package/dist/src/test.d.ts +0 -1
  182. package/dist/src/test.js +0 -6
  183. package/dist/src/transports/batch-warming.d.ts +0 -35
  184. package/dist/src/transports/batch-warming.js +0 -101
  185. package/dist/src/transports/index.d.ts +0 -70
  186. package/dist/src/transports/index.js +0 -87
  187. package/dist/src/transports/metrics.d.ts +0 -22
  188. package/dist/src/transports/metrics.js +0 -105
  189. package/dist/src/transports/rest.d.ts +0 -44
  190. package/dist/src/transports/rest.js +0 -131
  191. package/dist/src/transports/util.d.ts +0 -8
  192. package/dist/src/transports/util.js +0 -85
  193. package/dist/src/transports/websocket.d.ts +0 -80
  194. package/dist/src/transports/websocket.js +0 -166
  195. package/dist/src/util/expiring-sorted-set.d.ts +0 -21
  196. package/dist/src/util/expiring-sorted-set.js +0 -47
  197. package/dist/src/util/index.d.ts +0 -12
  198. package/dist/src/util/index.js +0 -35
  199. package/dist/src/util/logger.d.ts +0 -42
  200. package/dist/src/util/logger.js +0 -62
  201. package/dist/src/util/request.d.ts +0 -57
  202. package/dist/src/util/request.js +0 -2
  203. package/dist/src/util/subscription-set/expiring-sorted-set.d.ts +0 -22
  204. package/dist/src/util/subscription-set/expiring-sorted-set.js +0 -47
  205. package/dist/src/util/subscription-set/subscription-set.d.ts +0 -18
  206. package/dist/src/util/subscription-set/subscription-set.js +0 -19
  207. package/dist/src/util/test-payload-loader.d.ts +0 -25
  208. package/dist/src/util/test-payload-loader.js +0 -83
  209. package/dist/src/validation/error.d.ts +0 -50
  210. package/dist/src/validation/error.js +0 -79
  211. package/dist/src/validation/index.d.ts +0 -5
  212. package/dist/src/validation/index.js +0 -91
  213. package/dist/src/validation/input-params.d.ts +0 -15
  214. package/dist/src/validation/input-params.js +0 -30
  215. package/dist/src/validation/override-functions.d.ts +0 -3
  216. package/dist/src/validation/override-functions.js +0 -40
  217. package/dist/src/validation/preset-tokens.json +0 -23
  218. package/dist/src/validation/validator.d.ts +0 -47
  219. package/dist/src/validation/validator.js +0 -303
  220. package/src/test.ts +0 -5
package/dist/src/index.js DELETED
@@ -1,128 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.expose = void 0;
7
- const fastify_1 = __importDefault(require("fastify"));
8
- const path_1 = require("path");
9
- const background_executor_1 = require("./background-executor");
10
- const cache_1 = require("./cache");
11
- const metrics_1 = require("./metrics");
12
- const transports_1 = require("./transports");
13
- const util_1 = require("./util");
14
- const test_payload_loader_1 = require("./util/test-payload-loader");
15
- const validation_1 = require("./validation");
16
- const logger = (0, util_1.makeLogger)('Main');
17
- const VERSION = process.env['npm_package_version'];
18
- /**
19
- * Main function for the framework.
20
- * Initializes config and dependencies, uses those to initialize Transports, and starts listening for requests.
21
- *
22
- * @param adapter - an object describing an External Adapter
23
- * @param dependencies - an optional object with adapter dependencies to inject
24
- * @returns a Promise that resolves to the http.Server listening for connections
25
- */
26
- const expose = async (adapter, dependencies) => {
27
- // Initialize adapter (create dependencies, inject them, build endpoint map, etc.)
28
- await adapter.initialize(dependencies);
29
- let server = undefined;
30
- if (adapter.config.METRICS_ENABLED && adapter.config.EXPERIMENTAL_METRICS_ENABLED) {
31
- (0, metrics_1.setupMetricsServer)(adapter.name, adapter.config);
32
- }
33
- if (adapter.config.EA_MODE === 'reader' || adapter.config.EA_MODE === 'reader-writer') {
34
- // Main REST API server to handle incoming requests
35
- const app = await buildRestApi(adapter);
36
- // Start listening for incoming requests
37
- try {
38
- await app.listen(adapter.config.EA_PORT, adapter.config.EA_HOST);
39
- }
40
- catch (err) {
41
- logger.fatal(`There was an error when starting the EA server: ${err}`);
42
- process.exit();
43
- }
44
- logger.info(`Listening on port ${app.server.address().port}`);
45
- server = app.server;
46
- }
47
- else {
48
- logger.info('REST API is disabled; this instance will not process incoming requests.');
49
- }
50
- if (adapter.config.EA_MODE === 'writer' || adapter.config.EA_MODE === 'reader-writer') {
51
- // Start background loop that will take care of calling any async Transports
52
- logger.info('Starting background execution loop');
53
- (0, background_executor_1.callBackgroundExecutes)(adapter, server);
54
- }
55
- else {
56
- logger.info('Background executor is disabled; this instance will not perform async background executes.');
57
- }
58
- return server;
59
- };
60
- exports.expose = expose;
61
- async function buildRestApi(adapter) {
62
- const { config } = adapter;
63
- const app = (0, fastify_1.default)();
64
- // Add healthcheck endpoint before middlewares to bypass them
65
- app.get((0, path_1.join)(config.BASE_URL, 'health'), (req, res) => {
66
- res.status(200).send({ message: 'OK', version: VERSION });
67
- });
68
- // Use global error handling
69
- app.setErrorHandler(validation_1.errorCatchingMiddleware);
70
- const transportHandler = await (0, transports_1.buildTransportHandler)(adapter);
71
- app.register(async (router) => {
72
- // Set up "middlewares" (hooks in fastify)
73
- router.addHook('preHandler', (0, validation_1.validatorMiddleware)(adapter));
74
- router.addHook('preHandler', (0, cache_1.buildCacheMiddleware)(adapter));
75
- if (config['CORRELATION_ID_ENABLED']) {
76
- router.addHook('onRequest', util_1.loggingContextMiddleware);
77
- }
78
- router.route({
79
- url: config.BASE_URL,
80
- method: 'POST',
81
- handler: transportHandler,
82
- });
83
- if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
84
- router.addHook('onResponse', metrics_1.buildMetricsMiddleware);
85
- }
86
- });
87
- // Add smoke endpoint after middleware which are needed for tests
88
- buildSmokeEndpoint(app, config);
89
- return app;
90
- }
91
- /**
92
- * Adds the /smoke endpoint to the API for smoke testing the adapter
93
- *
94
- * @param app - the Fastify instance
95
- * @param config - the initialized adapter config
96
- */
97
- function buildSmokeEndpoint(app, config) {
98
- const testPayload = (0, test_payload_loader_1.loadTestPayload)(config.SMOKE_TEST_PAYLOAD_FILE_NAME);
99
- app.get((0, path_1.join)(config.BASE_URL, 'smoke'), async (_, res) => {
100
- if (testPayload.isDefault) {
101
- return res.status(200).send('OK');
102
- }
103
- const errors = [];
104
- for (const index in testPayload.requests) {
105
- try {
106
- const request = { id: index, data: testPayload.requests[index] };
107
- // Use Fastify's app inject to pass smoke requests internally
108
- const response = await app.inject({
109
- method: 'POST',
110
- url: '/',
111
- payload: request,
112
- });
113
- const parsedResponse = JSON.parse(response.body);
114
- // Throw error if not 2xx status code
115
- if (parsedResponse.statusCode < 200 || parsedResponse.statusCode > 299) {
116
- throw Error('Smoke test request failed');
117
- }
118
- }
119
- catch (e) {
120
- errors.push(e);
121
- }
122
- }
123
- if (errors.length > 0) {
124
- return res.status(500).send(errors);
125
- }
126
- return res.status(200).send('OK');
127
- });
128
- }
@@ -1,16 +0,0 @@
1
- export declare enum HttpRequestType {
2
- CACHE_HIT = "cacheHit",
3
- DATA_PROVIDER_HIT = "dataProviderHit",
4
- ADAPTER_ERROR = "adapterError",
5
- INPUT_ERROR = "inputError",
6
- RATE_LIMIT_ERROR = "rateLimitError",
7
- DP_ERROR = "dataProviderError",
8
- TIMEOUT_ERROR = "timeoutError",
9
- CONNECTION_ERROR = "connectionError",
10
- CUSTOM_ERROR = "customError"
11
- }
12
- /**
13
- * Maxiumum number of characters that a feedId can contain.
14
- */
15
- export declare const MAX_FEED_ID_LENGTH = 300;
16
- export declare const requestDurationBuckets: number[];
@@ -1,25 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.requestDurationBuckets = exports.MAX_FEED_ID_LENGTH = exports.HttpRequestType = void 0;
4
- var HttpRequestType;
5
- (function (HttpRequestType) {
6
- HttpRequestType["CACHE_HIT"] = "cacheHit";
7
- HttpRequestType["DATA_PROVIDER_HIT"] = "dataProviderHit";
8
- HttpRequestType["ADAPTER_ERROR"] = "adapterError";
9
- HttpRequestType["INPUT_ERROR"] = "inputError";
10
- HttpRequestType["RATE_LIMIT_ERROR"] = "rateLimitError";
11
- // BURST_LIMIT_ERROR = 'burstLimitError',
12
- // BACKOFF_ERROR = 'backoffError',
13
- HttpRequestType["DP_ERROR"] = "dataProviderError";
14
- HttpRequestType["TIMEOUT_ERROR"] = "timeoutError";
15
- HttpRequestType["CONNECTION_ERROR"] = "connectionError";
16
- // RES_EMPTY_ERROR = 'responseEmptyError',
17
- // RES_INVALID_ERROR = 'responseInvalidError',
18
- HttpRequestType["CUSTOM_ERROR"] = "customError";
19
- })(HttpRequestType = exports.HttpRequestType || (exports.HttpRequestType = {}));
20
- /**
21
- * Maxiumum number of characters that a feedId can contain.
22
- */
23
- exports.MAX_FEED_ID_LENGTH = 300;
24
- // We should tune these as we collect data, this is the default bucket distribution that prom comes with
25
- exports.requestDurationBuckets = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10];
@@ -1,15 +0,0 @@
1
- import * as client from 'prom-client';
2
- import { AdapterRequest } from '../util';
3
- import { AdapterConfig } from '../config';
4
- import { FastifyReply, HookHandlerDoneFunction } from 'fastify';
5
- export declare function setupMetricsServer(name: string, config: AdapterConfig): void;
6
- export declare const setupMetrics: (name: string, config: AdapterConfig) => void;
7
- /**
8
- * Builds metrics middleware that records end to end EA response times
9
- * and count of requests
10
- *
11
- * @returns the cache middleware function
12
- */
13
- export declare const buildMetricsMiddleware: (req: AdapterRequest, res: FastifyReply, done: HookHandlerDoneFunction) => void;
14
- export declare const httpRequestsTotal: client.Counter<"type" | "method" | "feed_id" | "status_code" | "provider_status_code" | "retry" | "is_cache_warming">;
15
- export declare const httpRequestDurationSeconds: client.Histogram<string>;
@@ -1,122 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
25
- var __importDefault = (this && this.__importDefault) || function (mod) {
26
- return (mod && mod.__esModule) ? mod : { "default": mod };
27
- };
28
- Object.defineProperty(exports, "__esModule", { value: true });
29
- exports.httpRequestDurationSeconds = exports.httpRequestsTotal = exports.buildMetricsMiddleware = exports.setupMetrics = exports.setupMetricsServer = void 0;
30
- const client = __importStar(require("prom-client"));
31
- const constants_1 = require("./constants");
32
- const util_1 = require("../util");
33
- const fastify_1 = __importDefault(require("fastify"));
34
- const path_1 = require("path");
35
- const error_1 = require("../validation/error");
36
- const logger = (0, util_1.makeLogger)('Metrics');
37
- function setupMetricsServer(name, config) {
38
- const metricsApp = (0, fastify_1.default)({
39
- logger: false,
40
- });
41
- const metricsPort = config.METRICS_PORT;
42
- const endpoint = config.METRICS_USE_BASE_URL ? (0, path_1.join)(config.BASE_URL, 'metrics') : '/metrics';
43
- const eaHost = config.EA_HOST;
44
- (0, exports.setupMetrics)(name, config);
45
- metricsApp.get(endpoint, async (_, res) => {
46
- res.type('txt');
47
- res.send(await client.register.metrics());
48
- });
49
- metricsApp.listen(metricsPort, eaHost, () => logger.info(`Monitoring listening on port ${metricsPort}!`));
50
- }
51
- exports.setupMetricsServer = setupMetricsServer;
52
- const setupMetrics = (name, config) => {
53
- client.collectDefaultMetrics();
54
- client.register.setDefaultLabels({
55
- app_name: config.METRICS_NAME || name || 'N/A',
56
- app_version: config['npm_package_version'],
57
- });
58
- };
59
- exports.setupMetrics = setupMetrics;
60
- /**
61
- * Builds metrics middleware that records end to end EA response times
62
- * and count of requests
63
- *
64
- * @returns the cache middleware function
65
- */
66
- const buildMetricsMiddleware = (req, res, done) => {
67
- const feedId = req.requestContext.meta?.metrics?.feedId || 'N/A';
68
- const labels = buildHttpRequestMetricsLabel(feedId, req.requestContext.meta?.error, req.requestContext.meta?.metrics?.cacheHit);
69
- // Record number of requests sent to EA
70
- exports.httpRequestsTotal.labels(labels).inc();
71
- // Record response time of request through entire EA
72
- exports.httpRequestDurationSeconds.observe(res.getResponseTime());
73
- logger.debug(`Response time for ${feedId}: ${res.getResponseTime()}ms`);
74
- done();
75
- };
76
- exports.buildMetricsMiddleware = buildMetricsMiddleware;
77
- const buildHttpRequestMetricsLabel = (feedId, error, cacheHit) => {
78
- const labels = {};
79
- labels.method = 'POST';
80
- labels.feed_id = feedId;
81
- if (error instanceof error_1.AdapterError) {
82
- // If error present and an instace of AdapterError, build label from error info
83
- labels.type = error?.metricsLabel || constants_1.HttpRequestType.ADAPTER_ERROR;
84
- labels.status_code = error?.statusCode;
85
- labels.provider_status_code = error?.providerStatusCode;
86
- }
87
- else if (error instanceof Error) {
88
- // If error present and not instance of generic Error, unexpected failure occurred
89
- labels.type = constants_1.HttpRequestType.ADAPTER_ERROR;
90
- labels.status_code = 500;
91
- }
92
- else {
93
- // If no error present, request went as expected
94
- labels.status_code = 200;
95
- if (cacheHit) {
96
- labels.type = constants_1.HttpRequestType.CACHE_HIT;
97
- }
98
- else {
99
- labels.type = constants_1.HttpRequestType.DATA_PROVIDER_HIT;
100
- labels.provider_status_code = 200;
101
- }
102
- }
103
- return labels;
104
- };
105
- exports.httpRequestsTotal = new client.Counter({
106
- name: 'http_requests_total',
107
- help: 'The number of http requests this external adapter has serviced for its entire uptime',
108
- labelNames: [
109
- 'method',
110
- 'status_code',
111
- 'retry',
112
- 'type',
113
- 'is_cache_warming',
114
- 'feed_id',
115
- 'provider_status_code',
116
- ],
117
- });
118
- exports.httpRequestDurationSeconds = new client.Histogram({
119
- name: 'http_request_duration_seconds',
120
- help: 'A histogram bucket of the distribution of http request durations',
121
- buckets: constants_1.requestDurationBuckets,
122
- });
@@ -1,7 +0,0 @@
1
- import { AdapterMetricsMeta, AdapterRequestData } from '../util';
2
- import { AdapterEndpoint } from '../adapter';
3
- import { AdapterConfig, SettingsMap } from '../config';
4
- export declare const getMetricsMeta: <Params, Result, CustomSettings extends SettingsMap>({ adapterEndpoint, adapterConfig, }: {
5
- adapterEndpoint: AdapterEndpoint<Params, Result, CustomSettings>;
6
- adapterConfig: AdapterConfig<CustomSettings>;
7
- }, data: AdapterRequestData) => AdapterMetricsMeta;
@@ -1,9 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.getMetricsMeta = void 0;
4
- const cache_1 = require("../cache");
5
- const getMetricsMeta = ({ adapterEndpoint, adapterConfig, }, data) => {
6
- const feedId = (0, cache_1.calculateFeedId)({ adapterEndpoint, adapterConfig }, data);
7
- return { feedId };
8
- };
9
- exports.getMetricsMeta = getMetricsMeta;
@@ -1,88 +0,0 @@
1
- import { AdapterConfig, BaseAdapterConfig, SettingsMap } from './config';
2
- import { AdapterRateLimitTier } from './rate-limiting';
3
- import { AdapterDependencies, Transport } from './transports';
4
- import { InputParameters } from './validation/input-params';
5
- /**
6
- * Structure to describe rate limits specs for the Adapter
7
- */
8
- interface AdapterRateLimitingConfig {
9
- /** Adapter rate limits, gotten from the specific tier requested */
10
- tiers: Record<string, AdapterRateLimitTier>;
11
- }
12
- /**
13
- * Main structure of an External Adapter
14
- */
15
- export interface Adapter {
16
- /** Name of the adapter */
17
- name: string;
18
- /** If present, the string that will be used for requests with no specified endpoint */
19
- defaultEndpoint?: string;
20
- /** List of [[AdapterEndpoint]]s in the adapter */
21
- endpoints: AdapterEndpoint[];
22
- /** Map of overrides to the default config values for an Adapter */
23
- envDefaultOverrides?: Partial<BaseAdapterConfig>;
24
- /** List of custom env vars for this particular adapter (e.g. RPC_URL) */
25
- customSettings?: SettingsMap;
26
- /** Configuration relevant to outbound (EA --\> DP) communication rate limiting */
27
- rateLimiting?: AdapterRateLimitingConfig;
28
- /** Overrides for converting the 'base' parameter that are hardcoded into the adapter. */
29
- overrides?: Record<string, string>;
30
- }
31
- /**
32
- * Structure to describe rate limits specs for a specific adapter endpoint
33
- */
34
- export interface EndpointRateLimitingConfig {
35
- /**
36
- * How much of the total limit for the adapter will be assigned to this specific endpoint.
37
- * Should be a non-zero positive number up to 100.
38
- * Endpoints in the same adapter without a specific allocation will divide the remaining limits equally.
39
- */
40
- allocationPercentage: number;
41
- }
42
- /**
43
- * Structure to describe a specific endpoint in an [[Adapter]]
44
- */
45
- export interface AdapterEndpoint {
46
- /** Name that will be used to match input params to this endpoint (case insensitive) */
47
- name: string;
48
- /** List of alternative endpoint names that will resolve to this same transport (case insensitive) */
49
- aliases?: string[];
50
- /** Transport that will be used to handle data processing and communication for this endpoint */
51
- transport: Transport<any, any, any>;
52
- /** Specification of what the body of a request hitting this endpoint should look like (used for validation) */
53
- inputParameters: InputParameters;
54
- /** Specific details related to the rate limiting for this endpoint in particular */
55
- rateLimiting?: EndpointRateLimitingConfig;
56
- }
57
- /**
58
- * Structure to describe an adapter that has been initialized
59
- */
60
- export interface InitializedAdapter extends Adapter {
61
- /** Object containing alias translations for all endpoints */
62
- endpointsMap: Record<string, AdapterEndpoint>;
63
- /** Initialized dependencies that the adapter will use */
64
- dependencies: AdapterDependencies;
65
- /** Configuration params for various adapter properties */
66
- config: AdapterConfig;
67
- }
68
- /**
69
- * This function will process dependencies for an adapter, such as caches or rate limiters,
70
- * in order to inject them into transports and other relevant places later in the lifecycle.
71
- *
72
- * @param config - the configuration for this adapter
73
- * @param inputDependencies - a partial obj of initialized dependencies to override the created ones
74
- * @param rateLimitingConfig - details from the adapter regarding rate limiting
75
- * @returns a set of AdapterDependencies all initialized
76
- */
77
- export declare const initializeDependencies: (adapter: Adapter, config: AdapterConfig, inputDependencies?: Partial<AdapterDependencies>) => AdapterDependencies;
78
- /**
79
- * Initializes all of the [[Transport]]s in the adapter, passing along any [[AdapterDependencies]] and [[AdapterConfig]].
80
- * Additionally, it builds a map out of all the endpoint names and aliases (checking for duplicates).
81
- *
82
- * @param adapter - an instance of an Adapter
83
- * @param dependencies - dependencies that the adapter will need at initialization
84
- * @param config - configuration variables already processed and validated
85
- * @returns - the adapter with all transports initialized and aliases map built
86
- */
87
- export declare const initializeAdapter: (adapter: Adapter, config: AdapterConfig, dependencies?: Partial<AdapterDependencies>) => Promise<InitializedAdapter>;
88
- export {};
@@ -1,112 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.initializeAdapter = exports.initializeDependencies = void 0;
4
- const factory_1 = require("./cache/factory");
5
- const rate_limiting_1 = require("./rate-limiting");
6
- const util_1 = require("./util");
7
- const logger = (0, util_1.makeLogger)('Adapter');
8
- /**
9
- * This function will take an adapter structure and go through each endpoint, calculating
10
- * each one's allocation of the total rate limits that are set for the adapter as a whole.
11
- *
12
- * @param adapter - the adapter to initialize rate limiting for
13
- */
14
- const calculateRateLimitAllocations = (adapter) => {
15
- const numberOfEndpoints = adapter.endpoints.length;
16
- const endpointsWithExplicitAllocations = adapter.endpoints.filter((e) => e.rateLimiting);
17
- const totalExplicitAllocation = endpointsWithExplicitAllocations
18
- .map((e) => e.rateLimiting?.allocationPercentage || 0)
19
- .reduce((sum, next) => sum + next, 0);
20
- if (totalExplicitAllocation > 100) {
21
- throw new Error('The total allocation set for all endpoints summed cannot exceed 100%');
22
- }
23
- if (totalExplicitAllocation === 100 &&
24
- numberOfEndpoints - endpointsWithExplicitAllocations.length > 0) {
25
- throw new Error('The explicit allocation is at 100% but there are endpoints with implicit allocation');
26
- }
27
- const implicitAllocation = 100 - totalExplicitAllocation;
28
- logger.debug('Adapter rate limit allocations:');
29
- for (const endpoint of adapter.endpoints) {
30
- if (!endpoint.rateLimiting) {
31
- endpoint.rateLimiting = {
32
- allocationPercentage: implicitAllocation / (numberOfEndpoints - endpointsWithExplicitAllocations.length),
33
- };
34
- }
35
- logger.debug(`Endpoint [${endpoint.name}] - ${endpoint.rateLimiting?.allocationPercentage}%`);
36
- }
37
- };
38
- /**
39
- * This function will process dependencies for an adapter, such as caches or rate limiters,
40
- * in order to inject them into transports and other relevant places later in the lifecycle.
41
- *
42
- * @param config - the configuration for this adapter
43
- * @param inputDependencies - a partial obj of initialized dependencies to override the created ones
44
- * @param rateLimitingConfig - details from the adapter regarding rate limiting
45
- * @returns a set of AdapterDependencies all initialized
46
- */
47
- const initializeDependencies = (adapter, config, inputDependencies) => {
48
- const dependencies = inputDependencies || {};
49
- if (!dependencies.cache) {
50
- dependencies.cache = factory_1.CacheFactory.buildCache(config);
51
- }
52
- // In the future we might want something more complex, but for now it's better to simplify
53
- // and just use the same rate limiting for everything. Once we have a more complex use case we
54
- // can think of ways to make this more configurable.
55
- const rateLimitingTier = (0, rate_limiting_1.getRateLimitingTier)(adapter.rateLimiting?.tiers, config.RATE_LIMIT_API_TIER);
56
- if (!dependencies.requestRateLimiter) {
57
- dependencies.requestRateLimiter = new rate_limiting_1.SimpleCountingRateLimiter().initialize(adapter.endpoints, rateLimitingTier);
58
- }
59
- if (!dependencies.backgroundExecuteRateLimiter) {
60
- dependencies.backgroundExecuteRateLimiter = new rate_limiting_1.FixedFrequencyRateLimiter().initialize(adapter.endpoints, rateLimitingTier);
61
- }
62
- return dependencies;
63
- };
64
- exports.initializeDependencies = initializeDependencies;
65
- /**
66
- * Takes an adapter and normalizes all endpoint names and aliases, as well as the default endpoint.
67
- * i.e. makes them lowercase for now
68
- * @param adapter - an instance of an Adapter
69
- */
70
- const normalizeEndpointNames = (adapter) => {
71
- // Make endpoints case insensitive, including default
72
- adapter.defaultEndpoint = adapter.defaultEndpoint?.toLowerCase();
73
- for (const endpoint of adapter.endpoints) {
74
- endpoint.name = endpoint.name.toLowerCase();
75
- endpoint.aliases = endpoint.aliases?.map((a) => a.toLowerCase());
76
- }
77
- };
78
- /**
79
- * Initializes all of the [[Transport]]s in the adapter, passing along any [[AdapterDependencies]] and [[AdapterConfig]].
80
- * Additionally, it builds a map out of all the endpoint names and aliases (checking for duplicates).
81
- *
82
- * @param adapter - an instance of an Adapter
83
- * @param dependencies - dependencies that the adapter will need at initialization
84
- * @param config - configuration variables already processed and validated
85
- * @returns - the adapter with all transports initialized and aliases map built
86
- */
87
- const initializeAdapter = async (adapter, config, dependencies) => {
88
- normalizeEndpointNames(adapter);
89
- calculateRateLimitAllocations(adapter);
90
- const initializedDependencies = (0, exports.initializeDependencies)(adapter, config, dependencies);
91
- const endpointsMap = {};
92
- for (const endpoint of adapter.endpoints) {
93
- // Add aliases to map to use in validation
94
- const aliases = [endpoint.name, ...(endpoint.aliases || [])];
95
- for (const alias of aliases) {
96
- if (endpointsMap[alias]) {
97
- throw new Error(`Duplicate endpoint / alias: "${alias}"`);
98
- }
99
- endpointsMap[alias] = endpoint;
100
- }
101
- logger.debug(`Initializing transport for endpoint "${endpoint.name}"...`);
102
- await endpoint.transport.initialize(initializedDependencies);
103
- }
104
- logger.debug('Adapter initialization complete.');
105
- return {
106
- ...adapter,
107
- endpointsMap,
108
- dependencies: initializedDependencies,
109
- config,
110
- };
111
- };
112
- exports.initializeAdapter = initializeAdapter;
@@ -1,11 +0,0 @@
1
- /// <reference types="node" />
2
- import { Server } from 'http';
3
- import { InitializedAdapter } from './adapter';
4
- /**
5
- * Very simple background loop that will call the [[Transport.backgroundExecute]] functions in all Transports.
6
- * It gets the time in ms to wait as the return value from those functions, and sleeps until next execution.
7
- *
8
- * @param adapter - an initialized External Adapter
9
- * @param server - the http server to attach an on close listener to
10
- */
11
- export declare function callBackgroundExecutes(adapter: InitializedAdapter, server?: Server): Promise<void>;
@@ -1,45 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.callBackgroundExecutes = void 0;
4
- const util_1 = require("./util");
5
- const logger = (0, util_1.makeLogger)('BackgroundExecutor');
6
- /**
7
- * Very simple background loop that will call the [[Transport.backgroundExecute]] functions in all Transports.
8
- * It gets the time in ms to wait as the return value from those functions, and sleeps until next execution.
9
- *
10
- * @param adapter - an initialized External Adapter
11
- * @param server - the http server to attach an on close listener to
12
- */
13
- async function callBackgroundExecutes(adapter, server) {
14
- // Set up variable to check later on to see if we need to stop this background "thread"
15
- // If no server is provided, the listener won't be set and serverClosed will always be false
16
- let serverClosed = false;
17
- server?.on('close', () => {
18
- serverClosed = true;
19
- });
20
- for (const endpoint of adapter.endpoints) {
21
- const backgroundExecute = endpoint.transport.backgroundExecute?.bind(endpoint.transport);
22
- if (!backgroundExecute) {
23
- logger.debug(`Endpoint "${endpoint.name}" has no background execute, skipping...`);
24
- continue;
25
- }
26
- const context = {
27
- adapterEndpoint: endpoint,
28
- adapterConfig: adapter.config,
29
- };
30
- const handler = async () => {
31
- if (serverClosed) {
32
- logger.info('Server closed, stopping recursive backgroundExecute handler chain');
33
- return;
34
- }
35
- logger.debug(`Calling background execute for endpoint "${endpoint.name}"`);
36
- const timeToWait = await backgroundExecute(context);
37
- logger.debug(`Finished background execute for endpoint "${endpoint.name}", sleeping for ${timeToWait}ms`);
38
- await (0, util_1.sleep)(timeToWait);
39
- handler();
40
- };
41
- // Start recursive async calls
42
- handler();
43
- }
44
- }
45
- exports.callBackgroundExecutes = callBackgroundExecutes;
@@ -1,6 +0,0 @@
1
- import { AdapterConfig } from '../config';
2
- import { LocalCache } from './local';
3
- import { RedisCache } from './redis';
4
- export declare class CacheFactory {
5
- static buildCache(config: AdapterConfig): LocalCache<unknown> | RedisCache<unknown> | undefined;
6
- }
@@ -1,57 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
25
- var __importDefault = (this && this.__importDefault) || function (mod) {
26
- return (mod && mod.__esModule) ? mod : { "default": mod };
27
- };
28
- Object.defineProperty(exports, "__esModule", { value: true });
29
- exports.CacheFactory = void 0;
30
- const ioredis_1 = __importDefault(require("ioredis"));
31
- const util_1 = require("../util");
32
- const local_1 = require("./local");
33
- const cacheMetrics = __importStar(require("./metrics"));
34
- const redis_1 = require("./redis");
35
- const logger = (0, util_1.makeLogger)('CacheFactory');
36
- class CacheFactory {
37
- static buildCache(config) {
38
- logger.info(`Using "${config.CACHE_TYPE}" cache.`);
39
- if (config.CACHE_TYPE === 'local') {
40
- return new local_1.LocalCache();
41
- }
42
- else if (config.CACHE_TYPE === 'redis') {
43
- const redis = new ioredis_1.default({
44
- enableAutoPipelining: true,
45
- host: config.CACHE_REDIS_HOST,
46
- port: config.CACHE_REDIS_PORT,
47
- });
48
- redis.on('connect', () => {
49
- cacheMetrics.redisConnectionsOpen.inc();
50
- });
51
- // TODO: Maybe track active redis connections instead of just total count of connections created
52
- // Use a Gauge and track with a combo of connect/end events listeners
53
- return new redis_1.RedisCache(redis);
54
- }
55
- }
56
- }
57
- exports.CacheFactory = CacheFactory;