@chainlink/external-adapter-framework 0.0.15 → 0.0.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (183) hide show
  1. package/adapter.js +128 -0
  2. package/background-executor.js +45 -0
  3. package/cache/factory.js +58 -0
  4. package/cache/index.js +173 -0
  5. package/cache/local.js +83 -0
  6. package/cache/metrics.js +120 -0
  7. package/cache/redis.js +100 -0
  8. package/chainlink-external-adapter-framework-v0.0.6.tgz +0 -0
  9. package/config/index.js +366 -0
  10. package/config/provider-limits.js +74 -0
  11. package/examples/bank-frick/accounts.js +192 -0
  12. package/examples/bank-frick/config/index.js +54 -0
  13. package/examples/bank-frick/index.js +15 -0
  14. package/examples/bank-frick/util.js +39 -0
  15. package/examples/coingecko/src/config/index.js +13 -0
  16. package/examples/coingecko/src/config/overrides.json +10826 -0
  17. package/examples/coingecko/src/cryptoUtils.js +41 -0
  18. package/examples/coingecko/src/endpoint/coins.js +33 -0
  19. package/examples/coingecko/src/endpoint/crypto-marketcap.js +46 -0
  20. package/examples/coingecko/src/endpoint/crypto-volume.js +46 -0
  21. package/examples/coingecko/src/endpoint/crypto.js +47 -0
  22. package/examples/coingecko/src/endpoint/dominance.js +26 -0
  23. package/examples/coingecko/src/endpoint/global-marketcap.js +26 -0
  24. package/examples/coingecko/src/endpoint/index.js +15 -0
  25. package/examples/coingecko/src/globalUtils.js +48 -0
  26. package/examples/coingecko/src/index.js +14 -0
  27. package/examples/coingecko/test/e2e/adapter.test.js +262 -0
  28. package/examples/coingecko/test/integration/adapter.test.js +264 -0
  29. package/examples/coingecko/test/integration/capturedRequests.json +1 -0
  30. package/examples/coingecko/test/integration/fixtures.js +41 -0
  31. package/examples/coingecko-old/batch-warming.js +53 -0
  32. package/examples/coingecko-old/index.js +11 -0
  33. package/examples/coingecko-old/rest.js +51 -0
  34. package/examples/ncfx/config/index.js +15 -0
  35. package/examples/ncfx/index.js +11 -0
  36. package/examples/ncfx/websocket.js +73 -0
  37. package/index.js +127 -0
  38. package/metrics/constants.js +25 -0
  39. package/metrics/index.js +122 -0
  40. package/metrics/util.js +9 -0
  41. package/package.json +5 -15
  42. package/rate-limiting/background/fixed-frequency.js +35 -0
  43. package/rate-limiting/index.js +84 -0
  44. package/rate-limiting/metrics.js +44 -0
  45. package/rate-limiting/request/simple-counting.js +62 -0
  46. package/test.js +6 -0
  47. package/transports/batch-warming.js +101 -0
  48. package/transports/index.js +87 -0
  49. package/transports/metrics.js +105 -0
  50. package/transports/rest.js +138 -0
  51. package/transports/util.js +86 -0
  52. package/transports/websocket.js +166 -0
  53. package/util/index.js +35 -0
  54. package/util/logger.js +62 -0
  55. package/util/recordRequests.js +45 -0
  56. package/util/request.js +2 -0
  57. package/util/subscription-set/expiring-sorted-set.js +47 -0
  58. package/util/subscription-set/subscription-set.js +19 -0
  59. package/util/test-payload-loader.js +83 -0
  60. package/validation/error.js +79 -0
  61. package/validation/index.js +91 -0
  62. package/validation/input-params.js +30 -0
  63. package/validation/override-functions.js +40 -0
  64. package/validation/overrideFunctions.js +40 -0
  65. package/validation/preset-tokens.json +23 -0
  66. package/validation/validator.js +303 -0
  67. package/.c8rc.json +0 -3
  68. package/.eslintignore +0 -10
  69. package/.eslintrc.js +0 -96
  70. package/.github/README.MD +0 -42
  71. package/.github/actions/setup/action.yaml +0 -13
  72. package/.github/workflows/label.yaml +0 -39
  73. package/.github/workflows/main.yaml +0 -39
  74. package/.github/workflows/publish.yaml +0 -17
  75. package/.prettierignore +0 -13
  76. package/.yarnrc +0 -0
  77. package/README.md +0 -103
  78. package/dist/examples/coingecko/test/e2e/adapter.test.ts.js +0 -82953
  79. package/dist/examples/coingecko/test/integration/adapter.test.ts.js +0 -91672
  80. package/dist/main.js +0 -72703
  81. package/docker-compose.yaml +0 -35
  82. package/env.sh +0 -54
  83. package/env2.sh +0 -55
  84. package/jest.config.js +0 -5
  85. package/publish.sh +0 -0
  86. package/src/adapter.ts +0 -263
  87. package/src/background-executor.ts +0 -52
  88. package/src/cache/factory.ts +0 -26
  89. package/src/cache/index.ts +0 -258
  90. package/src/cache/local.ts +0 -73
  91. package/src/cache/metrics.ts +0 -112
  92. package/src/cache/redis.ts +0 -93
  93. package/src/config/index.ts +0 -517
  94. package/src/config/provider-limits.ts +0 -127
  95. package/src/examples/bank-frick/README.MD +0 -10
  96. package/src/examples/bank-frick/accounts.ts +0 -246
  97. package/src/examples/bank-frick/config/index.ts +0 -53
  98. package/src/examples/bank-frick/index.ts +0 -13
  99. package/src/examples/bank-frick/types.d.ts +0 -38
  100. package/src/examples/bank-frick/util.ts +0 -55
  101. package/src/examples/coingecko/src/config/index.ts +0 -12
  102. package/src/examples/coingecko/src/config/overrides.json +0 -10826
  103. package/src/examples/coingecko/src/cryptoUtils.ts +0 -88
  104. package/src/examples/coingecko/src/endpoint/coins.ts +0 -54
  105. package/src/examples/coingecko/src/endpoint/crypto-marketcap.ts +0 -66
  106. package/src/examples/coingecko/src/endpoint/crypto-volume.ts +0 -66
  107. package/src/examples/coingecko/src/endpoint/crypto.ts +0 -63
  108. package/src/examples/coingecko/src/endpoint/dominance.ts +0 -40
  109. package/src/examples/coingecko/src/endpoint/global-marketcap.ts +0 -40
  110. package/src/examples/coingecko/src/endpoint/index.ts +0 -6
  111. package/src/examples/coingecko/src/globalUtils.ts +0 -78
  112. package/src/examples/coingecko/src/index.ts +0 -17
  113. package/src/examples/coingecko/test/e2e/adapter.test.ts +0 -278
  114. package/src/examples/coingecko/test/integration/__snapshots__/adapter.test.ts.snap +0 -15
  115. package/src/examples/coingecko/test/integration/adapter.test.ts +0 -281
  116. package/src/examples/coingecko/test/integration/capturedRequests.json +0 -1
  117. package/src/examples/coingecko/test/integration/fixtures.ts +0 -42
  118. package/src/examples/coingecko-old/batch-warming.ts +0 -79
  119. package/src/examples/coingecko-old/index.ts +0 -9
  120. package/src/examples/coingecko-old/rest.ts +0 -77
  121. package/src/examples/ncfx/config/index.ts +0 -12
  122. package/src/examples/ncfx/index.ts +0 -9
  123. package/src/examples/ncfx/websocket.ts +0 -99
  124. package/src/index.ts +0 -149
  125. package/src/metrics/constants.ts +0 -23
  126. package/src/metrics/index.ts +0 -115
  127. package/src/metrics/util.ts +0 -18
  128. package/src/rate-limiting/background/fixed-frequency.ts +0 -45
  129. package/src/rate-limiting/index.ts +0 -100
  130. package/src/rate-limiting/metrics.ts +0 -18
  131. package/src/rate-limiting/request/simple-counting.ts +0 -76
  132. package/src/transports/batch-warming.ts +0 -127
  133. package/src/transports/index.ts +0 -152
  134. package/src/transports/metrics.ts +0 -95
  135. package/src/transports/rest.ts +0 -168
  136. package/src/transports/util.ts +0 -63
  137. package/src/transports/websocket.ts +0 -245
  138. package/src/util/index.ts +0 -23
  139. package/src/util/logger.ts +0 -69
  140. package/src/util/recordRequests.ts +0 -47
  141. package/src/util/request.ts +0 -117
  142. package/src/util/subscription-set/expiring-sorted-set.ts +0 -54
  143. package/src/util/subscription-set/subscription-set.ts +0 -35
  144. package/src/util/test-payload-loader.ts +0 -87
  145. package/src/validation/error.ts +0 -116
  146. package/src/validation/index.ts +0 -110
  147. package/src/validation/input-params.ts +0 -45
  148. package/src/validation/override-functions.ts +0 -44
  149. package/src/validation/overrideFunctions.ts +0 -44
  150. package/src/validation/preset-tokens.json +0 -23
  151. package/src/validation/validator.ts +0 -384
  152. package/test/adapter.test.ts +0 -27
  153. package/test/background-executor.test.ts +0 -108
  154. package/test/cache/cache-key.test.ts +0 -114
  155. package/test/cache/helper.ts +0 -100
  156. package/test/cache/local.test.ts +0 -54
  157. package/test/cache/redis.test.ts +0 -89
  158. package/test/correlation.test.ts +0 -114
  159. package/test/index.test.ts +0 -37
  160. package/test/metrics/feed-id.test.ts +0 -38
  161. package/test/metrics/helper.ts +0 -14
  162. package/test/metrics/labels.test.ts +0 -36
  163. package/test/metrics/metrics.test.ts +0 -267
  164. package/test/metrics/redis-metrics.test.ts +0 -113
  165. package/test/metrics/warmer-metrics.test.ts +0 -193
  166. package/test/metrics/ws-metrics.test.ts +0 -225
  167. package/test/rate-limit-config.test.ts +0 -242
  168. package/test/smoke/smoke.test.ts +0 -166
  169. package/test/smoke/test-payload-fail.json +0 -3
  170. package/test/smoke/test-payload.js +0 -22
  171. package/test/smoke/test-payload.json +0 -7
  172. package/test/transports/batch.test.ts +0 -466
  173. package/test/transports/rest.test.ts +0 -242
  174. package/test/transports/websocket.test.ts +0 -183
  175. package/test/tsconfig.json +0 -5
  176. package/test/util.ts +0 -77
  177. package/test/validation.test.ts +0 -178
  178. package/test.sh +0 -20
  179. package/test2.sh +0 -2
  180. package/tsconfig.json +0 -28
  181. package/typedoc.json +0 -6
  182. package/webpack.config.js +0 -57
  183. package/yarn-error.log +0 -3778
@@ -0,0 +1,41 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.createMocks = void 0;
7
+ const nock_1 = __importDefault(require("nock"));
8
+ const capturedRequests_json_1 = __importDefault(require("./capturedRequests.json"));
9
+ // This is a generic fixture file that can be copy-pasted to generate fixtures from a
10
+ // capturedRequests.json file. To generate a capturedRequests.json file,
11
+ // set env var RECORD=true and run e2e tests.
12
+ const createMocks = () => {
13
+ const mocks = [];
14
+ for (const baseUrl in capturedRequests_json_1.default) {
15
+ const mock = (0, nock_1.default)(baseUrl, { encodedQueryParams: true });
16
+ for (const request of capturedRequests_json_1.default[baseUrl]) {
17
+ if (request.method === 'GET') {
18
+ mock
19
+ .get((uri) => {
20
+ const deconstructedRequestUrl = uri.split(/[?=,&]+/);
21
+ const deconstructedMockUrl = request.path.split(/[?=,&]+/);
22
+ return (deconstructedRequestUrl.sort().join(',') === deconstructedMockUrl.sort().join(','));
23
+ })
24
+ .times(100)
25
+ .reply(request.statusCode, () => request.response, [
26
+ 'Content-Type',
27
+ 'application/json',
28
+ 'Connection',
29
+ 'close',
30
+ 'Vary',
31
+ 'Accept-Encoding',
32
+ 'Vary',
33
+ 'Origin',
34
+ ]);
35
+ }
36
+ }
37
+ mocks.push(mock);
38
+ }
39
+ return mocks;
40
+ };
41
+ exports.createMocks = createMocks;
@@ -0,0 +1,53 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.batchEndpoint = void 0;
4
+ const adapter_1 = require("../../adapter");
5
+ const batch_warming_1 = require("../../transports/batch-warming");
6
+ const DEFAULT_URL = 'https://pro-api.coingecko.com/api/v3';
7
+ const inputParameters = {
8
+ coinid: {
9
+ description: 'The CoinGecko id or array of ids of the coin(s) to query (Note: because of current limitations to use a dummy base will need to be supplied)',
10
+ required: false,
11
+ },
12
+ base: {
13
+ aliases: ['from', 'coin'],
14
+ description: 'The symbol or array of symbols of the currency to query',
15
+ required: true,
16
+ },
17
+ quote: {
18
+ aliases: ['to', 'market'],
19
+ description: 'The symbol of the currency to convert to',
20
+ required: true,
21
+ },
22
+ };
23
+ const batchEndpointTransport = new batch_warming_1.BatchWarmingTransport({
24
+ prepareRequest: (params, context) => {
25
+ return {
26
+ baseURL: DEFAULT_URL,
27
+ url: '/simple/price',
28
+ method: 'GET',
29
+ params: {
30
+ ids: [...new Set(params.map((p) => p.base))].join(','),
31
+ vs_currencies: [...new Set(params.map((p) => p.quote))].join(','),
32
+ x_cg_pro_api_key: context.adapterConfig.API_KEY,
33
+ },
34
+ };
35
+ },
36
+ parseResponse: (params, res) => {
37
+ const entries = [];
38
+ for (const [base, entry] of Object.entries(res.data)) {
39
+ for (const [quote, price] of Object.entries(entry)) {
40
+ entries.push({
41
+ params: { base, quote },
42
+ value: price,
43
+ });
44
+ }
45
+ }
46
+ return entries;
47
+ },
48
+ });
49
+ exports.batchEndpoint = new adapter_1.AdapterEndpoint({
50
+ name: 'batch',
51
+ transport: batchEndpointTransport,
52
+ inputParameters,
53
+ });
@@ -0,0 +1,11 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.adapter = void 0;
4
+ const adapter_1 = require("../../adapter");
5
+ const batch_warming_1 = require("./batch-warming");
6
+ const rest_1 = require("./rest");
7
+ exports.adapter = new adapter_1.Adapter({
8
+ name: 'coingecko',
9
+ defaultEndpoint: 'batch',
10
+ endpoints: [rest_1.restEndpoint, batch_warming_1.batchEndpoint],
11
+ });
@@ -0,0 +1,51 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.restEndpoint = void 0;
4
+ const adapter_1 = require("../../adapter");
5
+ const transports_1 = require("../../transports");
6
+ const DEFAULT_URL = 'https://api.coingecko.com/api/v3';
7
+ const inputParameters = {
8
+ coinid: {
9
+ description: 'The CoinGecko id or array of ids of the coin(s) to query (Note: because of current limitations to use a dummy base will need to be supplied)',
10
+ required: false,
11
+ },
12
+ base: {
13
+ aliases: ['from', 'coin'],
14
+ description: 'The symbol or array of symbols of the currency to query',
15
+ required: true,
16
+ },
17
+ quote: {
18
+ aliases: ['to', 'market'],
19
+ description: 'The symbol of the currency to convert to',
20
+ required: true,
21
+ },
22
+ };
23
+ const restEndpointTransport = new transports_1.RestTransport({
24
+ prepareRequest: (req) => {
25
+ return {
26
+ baseURL: DEFAULT_URL,
27
+ url: '/simple/price',
28
+ method: 'GET',
29
+ params: {
30
+ ids: req.requestContext.data.base,
31
+ vs_currencies: req.requestContext.data.quote,
32
+ },
33
+ };
34
+ },
35
+ parseResponse: (req, res) => {
36
+ return {
37
+ data: res.data,
38
+ statusCode: 200,
39
+ result: res.data[req.requestContext.data.base]?.[req.requestContext.data.quote],
40
+ };
41
+ },
42
+ options: {
43
+ coalescing: true,
44
+ },
45
+ });
46
+ exports.restEndpoint = new adapter_1.AdapterEndpoint({
47
+ name: 'rest',
48
+ aliases: ['qwe'],
49
+ transport: restEndpointTransport,
50
+ inputParameters,
51
+ });
@@ -0,0 +1,15 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.customSettings = void 0;
4
+ exports.customSettings = {
5
+ USERNAME: {
6
+ description: 'Username for the NCFX API',
7
+ type: 'string',
8
+ required: true,
9
+ },
10
+ PASSWORD: {
11
+ description: 'Password for the NCFX API',
12
+ type: 'string',
13
+ required: true,
14
+ },
15
+ };
@@ -0,0 +1,11 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.adapter = void 0;
4
+ const adapter_1 = require("../../adapter");
5
+ const config_1 = require("./config");
6
+ const websocket_1 = require("./websocket");
7
+ exports.adapter = new adapter_1.Adapter({
8
+ name: 'ncfx',
9
+ endpoints: [websocket_1.webSocketEndpoint],
10
+ customSettings: config_1.customSettings,
11
+ });
@@ -0,0 +1,73 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.webSocketEndpoint = exports.websocketTransport = exports.inputParameters = void 0;
4
+ const adapter_1 = require("../../adapter");
5
+ const websocket_1 = require("../../transports/websocket");
6
+ const util_1 = require("../../util");
7
+ exports.inputParameters = {
8
+ base: {
9
+ aliases: ['from', 'coin'],
10
+ description: 'The symbol of the currency to query',
11
+ required: true,
12
+ },
13
+ quote: {
14
+ aliases: ['to', 'market'],
15
+ description: 'The symbol of the currency to convert to',
16
+ required: true,
17
+ },
18
+ };
19
+ const logger = (0, util_1.makeLogger)('NcfxWebSocketTransport');
20
+ exports.websocketTransport = new websocket_1.WebSocketTransport({
21
+ url: 'wss://feed.newchangefx.com/cryptodata',
22
+ handlers: {
23
+ open(connection, context) {
24
+ return new Promise((resolve, reject) => {
25
+ // Set up listener
26
+ connection.on('message', (data) => {
27
+ const parsed = JSON.parse(data.toString());
28
+ if (parsed.Message?.startsWith('Logged in as user')) {
29
+ logger.info('Got logged in response, connection is ready');
30
+ resolve();
31
+ }
32
+ else {
33
+ reject(new Error('Unexpected message after WS connection open'));
34
+ }
35
+ });
36
+ // Send login payload
37
+ connection.send(JSON.stringify({
38
+ request: 'login',
39
+ username: context.adapterConfig.USERNAME,
40
+ password: context.adapterConfig.PASSWORD,
41
+ }));
42
+ });
43
+ },
44
+ message(message) {
45
+ if (!Array.isArray(message)) {
46
+ logger.debug('WS message is not array, skipping');
47
+ return [];
48
+ }
49
+ return message.map((m) => {
50
+ const [base, quote] = m.currencyPair.split('/');
51
+ return {
52
+ params: { base, quote },
53
+ value: m.offer,
54
+ };
55
+ });
56
+ },
57
+ },
58
+ builders: {
59
+ subscribeMessage: (params) => ({
60
+ request: 'subscribe',
61
+ ccy: `${params.base}/${params.quote}`,
62
+ }),
63
+ unsubscribeMessage: (params) => ({
64
+ request: 'unsubscribe',
65
+ ccy: `${params.base}/${params.quote}`,
66
+ }),
67
+ },
68
+ });
69
+ exports.webSocketEndpoint = new adapter_1.AdapterEndpoint({
70
+ name: 'websocket',
71
+ transport: exports.websocketTransport,
72
+ inputParameters: exports.inputParameters,
73
+ });
package/index.js ADDED
@@ -0,0 +1,127 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.expose = void 0;
7
+ const fastify_1 = __importDefault(require("fastify"));
8
+ const path_1 = require("path");
9
+ const background_executor_1 = require("./background-executor");
10
+ const cache_1 = require("./cache");
11
+ const metrics_1 = require("./metrics");
12
+ const transports_1 = require("./transports");
13
+ const util_1 = require("./util");
14
+ const test_payload_loader_1 = require("./util/test-payload-loader");
15
+ const validation_1 = require("./validation");
16
+ const logger = (0, util_1.makeLogger)('Main');
17
+ const VERSION = process.env['npm_package_version'];
18
+ /**
19
+ * Main function for the framework.
20
+ * Initializes config and dependencies, uses those to initialize Transports, and starts listening for requests.
21
+ *
22
+ * @param adapter - an object describing an External Adapter
23
+ * @param dependencies - an optional object with adapter dependencies to inject
24
+ * @returns a Promise that resolves to the http.Server listening for connections
25
+ */
26
+ const expose = async (adapter, dependencies) => {
27
+ // Initialize adapter (create dependencies, inject them, build endpoint map, etc.)
28
+ await adapter.initialize(dependencies);
29
+ let server = undefined;
30
+ if (adapter.config.METRICS_ENABLED && adapter.config.EXPERIMENTAL_METRICS_ENABLED) {
31
+ (0, metrics_1.setupMetricsServer)(adapter.name, adapter.config);
32
+ }
33
+ if (adapter.config.EA_MODE === 'reader' || adapter.config.EA_MODE === 'reader-writer') {
34
+ // Main REST API server to handle incoming requests
35
+ const app = await buildRestApi(adapter);
36
+ // Start listening for incoming requests
37
+ try {
38
+ await app.listen(adapter.config.EA_PORT, adapter.config.EA_HOST);
39
+ }
40
+ catch (err) {
41
+ logger.fatal(`There was an error when starting the EA server: ${err}`);
42
+ process.exit();
43
+ }
44
+ logger.info(`Listening on port ${app.server.address().port}`);
45
+ server = app.server;
46
+ }
47
+ else {
48
+ logger.info('REST API is disabled; this instance will not process incoming requests.');
49
+ }
50
+ if (adapter.config.EA_MODE === 'writer' || adapter.config.EA_MODE === 'reader-writer') {
51
+ // Start background loop that will take care of calling any async Transports
52
+ logger.info('Starting background execution loop');
53
+ (0, background_executor_1.callBackgroundExecutes)(adapter, server);
54
+ }
55
+ else {
56
+ logger.info('Background executor is disabled; this instance will not perform async background executes.');
57
+ }
58
+ return server;
59
+ };
60
+ exports.expose = expose;
61
+ async function buildRestApi(adapter) {
62
+ const app = (0, fastify_1.default)();
63
+ // Add healthcheck endpoint before middlewares to bypass them
64
+ app.get((0, path_1.join)(adapter.config.BASE_URL, 'health'), (req, res) => {
65
+ res.status(200).send({ message: 'OK', version: VERSION });
66
+ });
67
+ // Use global error handling
68
+ app.setErrorHandler(validation_1.errorCatchingMiddleware);
69
+ const transportHandler = await (0, transports_1.buildTransportHandler)(adapter);
70
+ app.register(async (router) => {
71
+ // Set up "middlewares" (hooks in fastify)
72
+ router.addHook('preHandler', (0, validation_1.validatorMiddleware)(adapter));
73
+ router.addHook('preHandler', (0, cache_1.buildCacheMiddleware)(adapter));
74
+ if (adapter.config['CORRELATION_ID_ENABLED']) {
75
+ router.addHook('onRequest', util_1.loggingContextMiddleware);
76
+ }
77
+ router.route({
78
+ url: adapter.config.BASE_URL,
79
+ method: 'POST',
80
+ handler: transportHandler,
81
+ });
82
+ if (adapter.config.METRICS_ENABLED && adapter.config.EXPERIMENTAL_METRICS_ENABLED) {
83
+ router.addHook('onResponse', metrics_1.buildMetricsMiddleware);
84
+ }
85
+ });
86
+ // Add smoke endpoint after middleware which are needed for tests
87
+ buildSmokeEndpoint(app, adapter.config);
88
+ return app;
89
+ }
90
+ /**
91
+ * Adds the /smoke endpoint to the API for smoke testing the adapter
92
+ *
93
+ * @param app - the Fastify instance
94
+ * @param config - the initialized adapter config
95
+ */
96
+ function buildSmokeEndpoint(app, config) {
97
+ const testPayload = (0, test_payload_loader_1.loadTestPayload)(config.SMOKE_TEST_PAYLOAD_FILE_NAME);
98
+ app.get((0, path_1.join)(config.BASE_URL, 'smoke'), async (_, res) => {
99
+ if (testPayload.isDefault) {
100
+ return res.status(200).send('OK');
101
+ }
102
+ const errors = [];
103
+ for (const index in testPayload.requests) {
104
+ try {
105
+ const request = { id: index, data: testPayload.requests[index] };
106
+ // Use Fastify's app inject to pass smoke requests internally
107
+ const response = await app.inject({
108
+ method: 'POST',
109
+ url: '/',
110
+ payload: request,
111
+ });
112
+ const parsedResponse = JSON.parse(response.body);
113
+ // Throw error if not 2xx status code
114
+ if (parsedResponse.statusCode < 200 || parsedResponse.statusCode > 299) {
115
+ throw Error('Smoke test request failed');
116
+ }
117
+ }
118
+ catch (e) {
119
+ errors.push(e);
120
+ }
121
+ }
122
+ if (errors.length > 0) {
123
+ return res.status(500).send(errors);
124
+ }
125
+ return res.status(200).send('OK');
126
+ });
127
+ }
@@ -0,0 +1,25 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.requestDurationBuckets = exports.MAX_FEED_ID_LENGTH = exports.HttpRequestType = void 0;
4
+ var HttpRequestType;
5
+ (function (HttpRequestType) {
6
+ HttpRequestType["CACHE_HIT"] = "cacheHit";
7
+ HttpRequestType["DATA_PROVIDER_HIT"] = "dataProviderHit";
8
+ HttpRequestType["ADAPTER_ERROR"] = "adapterError";
9
+ HttpRequestType["INPUT_ERROR"] = "inputError";
10
+ HttpRequestType["RATE_LIMIT_ERROR"] = "rateLimitError";
11
+ // BURST_LIMIT_ERROR = 'burstLimitError',
12
+ // BACKOFF_ERROR = 'backoffError',
13
+ HttpRequestType["DP_ERROR"] = "dataProviderError";
14
+ HttpRequestType["TIMEOUT_ERROR"] = "timeoutError";
15
+ HttpRequestType["CONNECTION_ERROR"] = "connectionError";
16
+ // RES_EMPTY_ERROR = 'responseEmptyError',
17
+ // RES_INVALID_ERROR = 'responseInvalidError',
18
+ HttpRequestType["CUSTOM_ERROR"] = "customError";
19
+ })(HttpRequestType = exports.HttpRequestType || (exports.HttpRequestType = {}));
20
+ /**
21
+ * Maxiumum number of characters that a feedId can contain.
22
+ */
23
+ exports.MAX_FEED_ID_LENGTH = 300;
24
+ // We should tune these as we collect data, this is the default bucket distribution that prom comes with
25
+ exports.requestDurationBuckets = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10];
@@ -0,0 +1,122 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || function (mod) {
19
+ if (mod && mod.__esModule) return mod;
20
+ var result = {};
21
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
+ __setModuleDefault(result, mod);
23
+ return result;
24
+ };
25
+ var __importDefault = (this && this.__importDefault) || function (mod) {
26
+ return (mod && mod.__esModule) ? mod : { "default": mod };
27
+ };
28
+ Object.defineProperty(exports, "__esModule", { value: true });
29
+ exports.httpRequestDurationSeconds = exports.httpRequestsTotal = exports.buildMetricsMiddleware = exports.setupMetrics = exports.setupMetricsServer = void 0;
30
+ const client = __importStar(require("prom-client"));
31
+ const constants_1 = require("./constants");
32
+ const util_1 = require("../util");
33
+ const fastify_1 = __importDefault(require("fastify"));
34
+ const path_1 = require("path");
35
+ const error_1 = require("../validation/error");
36
+ const logger = (0, util_1.makeLogger)('Metrics');
37
+ function setupMetricsServer(name, config) {
38
+ const metricsApp = (0, fastify_1.default)({
39
+ logger: false,
40
+ });
41
+ const metricsPort = config.METRICS_PORT;
42
+ const endpoint = config.METRICS_USE_BASE_URL ? (0, path_1.join)(config.BASE_URL, 'metrics') : '/metrics';
43
+ const eaHost = config.EA_HOST;
44
+ (0, exports.setupMetrics)(name, config);
45
+ metricsApp.get(endpoint, async (_, res) => {
46
+ res.type('txt');
47
+ res.send(await client.register.metrics());
48
+ });
49
+ metricsApp.listen(metricsPort, eaHost, () => logger.info(`Monitoring listening on port ${metricsPort}!`));
50
+ }
51
+ exports.setupMetricsServer = setupMetricsServer;
52
+ const setupMetrics = (name, config) => {
53
+ client.collectDefaultMetrics();
54
+ client.register.setDefaultLabels({
55
+ app_name: config.METRICS_NAME || name || 'N/A',
56
+ app_version: config['npm_package_version'],
57
+ });
58
+ };
59
+ exports.setupMetrics = setupMetrics;
60
+ /**
61
+ * Builds metrics middleware that records end to end EA response times
62
+ * and count of requests
63
+ *
64
+ * @returns the cache middleware function
65
+ */
66
+ const buildMetricsMiddleware = (req, res, done) => {
67
+ const feedId = req.requestContext.meta?.metrics?.feedId || 'N/A';
68
+ const labels = buildHttpRequestMetricsLabel(feedId, req.requestContext.meta?.error, req.requestContext.meta?.metrics?.cacheHit);
69
+ // Record number of requests sent to EA
70
+ exports.httpRequestsTotal.labels(labels).inc();
71
+ // Record response time of request through entire EA
72
+ exports.httpRequestDurationSeconds.observe(res.getResponseTime());
73
+ logger.debug(`Response time for ${feedId}: ${res.getResponseTime()}ms`);
74
+ done();
75
+ };
76
+ exports.buildMetricsMiddleware = buildMetricsMiddleware;
77
+ const buildHttpRequestMetricsLabel = (feedId, error, cacheHit) => {
78
+ const labels = {};
79
+ labels.method = 'POST';
80
+ labels.feed_id = feedId;
81
+ if (error instanceof error_1.AdapterError) {
82
+ // If error present and an instace of AdapterError, build label from error info
83
+ labels.type = error?.metricsLabel || constants_1.HttpRequestType.ADAPTER_ERROR;
84
+ labels.status_code = error?.statusCode;
85
+ labels.provider_status_code = error?.providerStatusCode;
86
+ }
87
+ else if (error instanceof Error) {
88
+ // If error present and not instance of generic Error, unexpected failure occurred
89
+ labels.type = constants_1.HttpRequestType.ADAPTER_ERROR;
90
+ labels.status_code = 500;
91
+ }
92
+ else {
93
+ // If no error present, request went as expected
94
+ labels.status_code = 200;
95
+ if (cacheHit) {
96
+ labels.type = constants_1.HttpRequestType.CACHE_HIT;
97
+ }
98
+ else {
99
+ labels.type = constants_1.HttpRequestType.DATA_PROVIDER_HIT;
100
+ labels.provider_status_code = 200;
101
+ }
102
+ }
103
+ return labels;
104
+ };
105
+ exports.httpRequestsTotal = new client.Counter({
106
+ name: 'http_requests_total',
107
+ help: 'The number of http requests this external adapter has serviced for its entire uptime',
108
+ labelNames: [
109
+ 'method',
110
+ 'status_code',
111
+ 'retry',
112
+ 'type',
113
+ 'is_cache_warming',
114
+ 'feed_id',
115
+ 'provider_status_code',
116
+ ],
117
+ });
118
+ exports.httpRequestDurationSeconds = new client.Histogram({
119
+ name: 'http_request_duration_seconds',
120
+ help: 'A histogram bucket of the distribution of http request durations',
121
+ buckets: constants_1.requestDurationBuckets,
122
+ });
@@ -0,0 +1,9 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.getMetricsMeta = void 0;
4
+ const cache_1 = require("../cache");
5
+ const getMetricsMeta = ({ adapterEndpoint, adapterConfig, }, data) => {
6
+ const feedId = (0, cache_1.calculateFeedId)({ adapterEndpoint, adapterConfig }, data);
7
+ return { feedId };
8
+ };
9
+ exports.getMetricsMeta = getMetricsMeta;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@chainlink/external-adapter-framework",
3
- "version": "0.0.15",
3
+ "version": "0.0.16",
4
4
  "main": "dist/index.js",
5
5
  "license": "MIT",
6
6
  "dependencies": {
@@ -9,8 +9,6 @@
9
9
  "ioredis": "^5.0.4",
10
10
  "pino": "^7.9.2",
11
11
  "prom-client": "13.2.0",
12
- "supertest": "^6.2.4",
13
- "ts-jest": "^28.0.7",
14
12
  "ts-node": "^10.9.1",
15
13
  "typescript": "^4.6.3",
16
14
  "ws": "^8.5.0"
@@ -22,16 +20,12 @@
22
20
  "test-debug": "LOG_LEVEL=trace DEBUG=true EA_PORT=0 c8 ava --verbose",
23
21
  "build": "tsc",
24
22
  "lint": "eslint ./src && prettier --check ./src/**/*.ts",
25
- "lint-fix": "eslint --fix ./src && prettier --write ./src/**/*.ts",
26
- "dev": "NODE_ENV=develop tsnd --respawn --transpile-only --project tsconfig.json './src/test.ts'",
27
- "verify": "yarn lint && yarn build && yarn build -p ./test/tsconfig.json && yarn test"
23
+ "dev": "NODE_ENV=develop tsnd --respawn --transpile-only --project tsconfig.json './src/test.ts'"
28
24
  },
29
25
  "devDependencies": {
30
26
  "@sinonjs/fake-timers": "^9.1.2",
31
- "@types/jest": "^28.1.7",
32
27
  "@types/node": "^18.6.5",
33
28
  "@types/sinonjs__fake-timers": "^8.1.2",
34
- "@types/supertest": "^2.0.12",
35
29
  "@types/ws": "^8.5.3",
36
30
  "@typescript-eslint/eslint-plugin": "^5.17.0",
37
31
  "@typescript-eslint/parser": "^5.17.0",
@@ -44,13 +38,8 @@
44
38
  "nock": "^13.2.4",
45
39
  "pino-pretty": "^7.6.0",
46
40
  "prettier": "^2.6.1",
47
- "ts-loader": "^9.3.1",
48
41
  "ts-node-dev": "2.0.0",
49
- "typedoc": "^0.22.15",
50
- "url": "^0.11.0",
51
- "webpack": "^5.74.0",
52
- "webpack-cli": "^4.10.0",
53
- "webpack-merge": "^5.8.0"
42
+ "typedoc": "^0.22.15"
54
43
  },
55
44
  "prettier": {
56
45
  "semi": false,
@@ -77,6 +66,7 @@
77
66
  "workerThreads": false,
78
67
  "environmentVariables": {
79
68
  "METRICS_ENABLED": "false"
80
- }
69
+ },
70
+ "timeout": "20s"
81
71
  }
82
72
  }
@@ -0,0 +1,35 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.FixedFrequencyRateLimiter = exports.DEFAULT_SHARED_MS_BETWEEN_REQUESTS = void 0;
4
+ const __1 = require("..");
5
+ const util_1 = require("../../util");
6
+ const logger = (0, util_1.makeLogger)('FixedFrequencyRateLimiter');
7
+ exports.DEFAULT_SHARED_MS_BETWEEN_REQUESTS = 5000;
8
+ class FixedFrequencyRateLimiter {
9
+ constructor() {
10
+ this.msBetweenRequestsMap = {};
11
+ }
12
+ initialize(endpoints, limits) {
13
+ // Translate the hourly limit into reqs per minute
14
+ let sharedMsBetweenRequests = 1000 / (0, __1.consolidateTierLimits)(limits);
15
+ // If there is no limit set, we use some reasonable number
16
+ if (!limits?.rateLimit1h && !limits?.rateLimit1m && !limits?.rateLimit1s) {
17
+ // 5s period for all seems good
18
+ sharedMsBetweenRequests = exports.DEFAULT_SHARED_MS_BETWEEN_REQUESTS;
19
+ }
20
+ logger.debug('Using fixed frequency batch rate limiting');
21
+ for (const endpoint of endpoints) {
22
+ if (endpoint.rateLimiting?.allocationPercentage == null) {
23
+ throw new Error(`Allocation percentage for endpoint "${endpoint.name}" is null`);
24
+ }
25
+ this.msBetweenRequestsMap[endpoint.name] =
26
+ (sharedMsBetweenRequests / endpoint.rateLimiting?.allocationPercentage) * 100;
27
+ logger.debug(`Endpoint [${endpoint.name}]: ${this.msBetweenRequestsMap[endpoint.name] / 1000}s between requests`);
28
+ }
29
+ return this;
30
+ }
31
+ msUntilNextExecution(endpointName) {
32
+ return this.msBetweenRequestsMap[endpointName];
33
+ }
34
+ }
35
+ exports.FixedFrequencyRateLimiter = FixedFrequencyRateLimiter;