@chainlink/external-adapter-framework 0.0.14 → 0.0.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. package/adapter.js +114 -101
  2. package/cache/factory.js +3 -0
  3. package/chainlink-external-adapter-framework-v0.0.6.tgz +0 -0
  4. package/config/index.js +69 -83
  5. package/config/provider-limits.js +10 -15
  6. package/examples/bank-frick/accounts.js +3 -2
  7. package/examples/bank-frick/index.js +3 -2
  8. package/examples/coingecko/src/config/index.js +13 -0
  9. package/examples/coingecko/src/config/overrides.json +10826 -0
  10. package/examples/coingecko/src/cryptoUtils.js +41 -0
  11. package/examples/coingecko/src/endpoint/coins.js +33 -0
  12. package/examples/coingecko/src/endpoint/crypto-marketcap.js +46 -0
  13. package/examples/coingecko/src/endpoint/crypto-volume.js +46 -0
  14. package/examples/coingecko/src/endpoint/crypto.js +47 -0
  15. package/examples/coingecko/src/endpoint/dominance.js +26 -0
  16. package/examples/coingecko/src/endpoint/global-marketcap.js +26 -0
  17. package/examples/coingecko/src/endpoint/index.js +15 -0
  18. package/examples/coingecko/src/globalUtils.js +48 -0
  19. package/examples/coingecko/src/index.js +14 -0
  20. package/examples/coingecko/test/e2e/adapter.test.js +262 -0
  21. package/examples/coingecko/test/integration/adapter.test.js +264 -0
  22. package/examples/coingecko/test/integration/capturedRequests.json +1 -0
  23. package/examples/coingecko/test/integration/fixtures.js +41 -0
  24. package/{package/examples/coingecko → examples/coingecko-old}/batch-warming.js +4 -3
  25. package/examples/{coingecko → coingecko-old}/index.js +3 -2
  26. package/{package/examples/coingecko → examples/coingecko-old}/rest.js +3 -2
  27. package/examples/ncfx/index.js +3 -2
  28. package/examples/ncfx/websocket.js +3 -2
  29. package/index.js +17 -23
  30. package/package.json +5 -4
  31. package/rate-limiting/index.js +23 -2
  32. package/transports/batch-warming.js +1 -1
  33. package/transports/rest.js +9 -2
  34. package/transports/util.js +2 -1
  35. package/transports/websocket.js +4 -6
  36. package/util/recordRequests.js +45 -0
  37. package/{package/validation/override-functions.js → validation/overrideFunctions.js} +0 -0
  38. package/adapter.d.ts +0 -107
  39. package/background-executor.d.ts +0 -11
  40. package/cache/factory.d.ts +0 -6
  41. package/cache/index.d.ts +0 -94
  42. package/cache/local.d.ts +0 -23
  43. package/cache/metrics.d.ts +0 -27
  44. package/cache/redis.d.ts +0 -16
  45. package/chainlink-external-adapter-framework-0.0.6.tgz +0 -0
  46. package/config/index.d.ts +0 -209
  47. package/config/provider-limits.d.ts +0 -31
  48. package/examples/bank-frick/accounts.d.ts +0 -39
  49. package/examples/bank-frick/config/index.d.ts +0 -4
  50. package/examples/bank-frick/index.d.ts +0 -2
  51. package/examples/bank-frick/util.d.ts +0 -4
  52. package/examples/coingecko/batch-warming.d.ts +0 -2
  53. package/examples/coingecko/batch-warming.js +0 -52
  54. package/examples/coingecko/index.d.ts +0 -2
  55. package/examples/coingecko/rest.d.ts +0 -2
  56. package/examples/coingecko/rest.js +0 -50
  57. package/examples/ncfx/config/index.d.ts +0 -12
  58. package/examples/ncfx/index.d.ts +0 -2
  59. package/examples/ncfx/websocket.d.ts +0 -36
  60. package/index.d.ts +0 -11
  61. package/metrics/constants.d.ts +0 -16
  62. package/metrics/index.d.ts +0 -15
  63. package/metrics/util.d.ts +0 -7
  64. package/package/adapter.d.ts +0 -88
  65. package/package/adapter.js +0 -112
  66. package/package/background-executor.d.ts +0 -11
  67. package/package/background-executor.js +0 -45
  68. package/package/cache/factory.d.ts +0 -6
  69. package/package/cache/factory.js +0 -57
  70. package/package/cache/index.d.ts +0 -90
  71. package/package/cache/index.js +0 -169
  72. package/package/cache/local.d.ts +0 -23
  73. package/package/cache/local.js +0 -83
  74. package/package/cache/metrics.d.ts +0 -27
  75. package/package/cache/metrics.js +0 -120
  76. package/package/cache/redis.d.ts +0 -16
  77. package/package/cache/redis.js +0 -100
  78. package/package/config/index.d.ts +0 -195
  79. package/package/config/index.js +0 -365
  80. package/package/config/provider-limits.d.ts +0 -31
  81. package/package/config/provider-limits.js +0 -76
  82. package/package/examples/coingecko/batch-warming.d.ts +0 -2
  83. package/package/examples/coingecko/index.d.ts +0 -2
  84. package/package/examples/coingecko/index.js +0 -10
  85. package/package/examples/coingecko/rest.d.ts +0 -2
  86. package/package/examples/ncfx/config/index.d.ts +0 -12
  87. package/package/examples/ncfx/config/index.js +0 -15
  88. package/package/examples/ncfx/index.d.ts +0 -2
  89. package/package/examples/ncfx/index.js +0 -10
  90. package/package/examples/ncfx/websocket.d.ts +0 -36
  91. package/package/examples/ncfx/websocket.js +0 -72
  92. package/package/index.d.ts +0 -12
  93. package/package/index.js +0 -92
  94. package/package/metrics/constants.d.ts +0 -16
  95. package/package/metrics/constants.js +0 -25
  96. package/package/metrics/index.d.ts +0 -15
  97. package/package/metrics/index.js +0 -123
  98. package/package/metrics/util.d.ts +0 -3
  99. package/package/metrics/util.js +0 -9
  100. package/package/package.json +0 -72
  101. package/package/rate-limiting/background/fixed-frequency.d.ts +0 -10
  102. package/package/rate-limiting/background/fixed-frequency.js +0 -37
  103. package/package/rate-limiting/index.d.ts +0 -54
  104. package/package/rate-limiting/index.js +0 -63
  105. package/package/rate-limiting/metrics.d.ts +0 -3
  106. package/package/rate-limiting/metrics.js +0 -44
  107. package/package/rate-limiting/request/simple-counting.d.ts +0 -20
  108. package/package/rate-limiting/request/simple-counting.js +0 -62
  109. package/package/test.d.ts +0 -1
  110. package/package/test.js +0 -6
  111. package/package/transports/batch-warming.d.ts +0 -34
  112. package/package/transports/batch-warming.js +0 -101
  113. package/package/transports/index.d.ts +0 -87
  114. package/package/transports/index.js +0 -87
  115. package/package/transports/metrics.d.ts +0 -21
  116. package/package/transports/metrics.js +0 -105
  117. package/package/transports/rest.d.ts +0 -43
  118. package/package/transports/rest.js +0 -129
  119. package/package/transports/util.d.ts +0 -8
  120. package/package/transports/util.js +0 -85
  121. package/package/transports/websocket.d.ts +0 -80
  122. package/package/transports/websocket.js +0 -169
  123. package/package/util/expiring-sorted-set.d.ts +0 -21
  124. package/package/util/expiring-sorted-set.js +0 -47
  125. package/package/util/index.d.ts +0 -11
  126. package/package/util/index.js +0 -35
  127. package/package/util/logger.d.ts +0 -42
  128. package/package/util/logger.js +0 -62
  129. package/package/util/request.d.ts +0 -55
  130. package/package/util/request.js +0 -2
  131. package/package/validation/error.d.ts +0 -50
  132. package/package/validation/error.js +0 -79
  133. package/package/validation/index.d.ts +0 -5
  134. package/package/validation/index.js +0 -86
  135. package/package/validation/input-params.d.ts +0 -15
  136. package/package/validation/input-params.js +0 -30
  137. package/package/validation/override-functions.d.ts +0 -3
  138. package/package/validation/preset-tokens.json +0 -23
  139. package/package/validation/validator.d.ts +0 -47
  140. package/package/validation/validator.js +0 -303
  141. package/rate-limiting/background/fixed-frequency.d.ts +0 -10
  142. package/rate-limiting/index.d.ts +0 -54
  143. package/rate-limiting/metrics.d.ts +0 -3
  144. package/rate-limiting/request/simple-counting.d.ts +0 -20
  145. package/test.d.ts +0 -1
  146. package/transports/batch-warming.d.ts +0 -35
  147. package/transports/index.d.ts +0 -70
  148. package/transports/metrics.d.ts +0 -21
  149. package/transports/rest.d.ts +0 -44
  150. package/transports/util.d.ts +0 -8
  151. package/transports/websocket.d.ts +0 -81
  152. package/util/expiring-sorted-set.d.ts +0 -21
  153. package/util/expiring-sorted-set.js +0 -47
  154. package/util/index.d.ts +0 -12
  155. package/util/logger.d.ts +0 -42
  156. package/util/request.d.ts +0 -57
  157. package/util/subscription-set/expiring-sorted-set.d.ts +0 -22
  158. package/util/subscription-set/subscription-set.d.ts +0 -18
  159. package/util/test-payload-loader.d.ts +0 -25
  160. package/validation/error.d.ts +0 -50
  161. package/validation/index.d.ts +0 -5
  162. package/validation/input-params.d.ts +0 -15
  163. package/validation/override-functions.d.ts +0 -3
  164. package/validation/validator.d.ts +0 -47
package/package/index.js DELETED
@@ -1,92 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.expose = void 0;
7
- const fastify_1 = __importDefault(require("fastify"));
8
- const path_1 = require("path");
9
- const adapter_1 = require("./adapter");
10
- const background_executor_1 = require("./background-executor");
11
- const cache_1 = require("./cache");
12
- const config_1 = require("./config");
13
- const metrics_1 = require("./metrics");
14
- const transports_1 = require("./transports");
15
- const util_1 = require("./util");
16
- const validation_1 = require("./validation");
17
- const logger = (0, util_1.makeLogger)('Main');
18
- const VERSION = process.env['npm_package_version'];
19
- /**
20
- * Main function for the framework.
21
- * Initializes config and dependencies, uses those to initialize Transports, and starts listening for requests.
22
- *
23
- * @param adapter - an object describing an External Adapter
24
- * @param dependencies - an optional object with adapter dependencies to inject
25
- * @returns a Promise that resolves to the http.Server listening for connections
26
- */
27
- const expose = async (adapter, dependencies) => {
28
- const config = (0, config_1.buildAdapterConfig)({
29
- overrides: adapter.envDefaultOverrides,
30
- customSettings: adapter.customSettings,
31
- });
32
- // Initialize adapter (create dependencies, inject them, build endpoint map, etc.)
33
- const initializedAdapter = await (0, adapter_1.initializeAdapter)(adapter, config, dependencies);
34
- let server = undefined;
35
- if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
36
- (0, metrics_1.setupMetricsServer)(adapter.name, config);
37
- }
38
- if (config.EA_MODE === 'reader' || config.EA_MODE === 'reader-writer') {
39
- // Main REST API server to handle incoming requests
40
- const app = await buildRestApi(config, initializedAdapter);
41
- // Start listening for incoming requests
42
- try {
43
- await app.listen(config.EA_PORT, config.EA_HOST);
44
- }
45
- catch (err) {
46
- logger.fatal(`There was an error when starting the EA server: ${err}`);
47
- process.exit();
48
- }
49
- logger.info(`Listening on port ${app.server.address().port}`);
50
- server = app.server;
51
- }
52
- else {
53
- logger.info('REST API is disabled; this instance will not process incoming requests.');
54
- }
55
- if (config.EA_MODE === 'writer' || config.EA_MODE === 'reader-writer') {
56
- // Start background loop that will take care of calling any async Transports
57
- logger.info('Starting background execution loop');
58
- (0, background_executor_1.callBackgroundExecutes)(initializedAdapter, server);
59
- }
60
- else {
61
- logger.info('Background executor is disabled; this instance will not perform async background executes.');
62
- }
63
- return server;
64
- };
65
- exports.expose = expose;
66
- async function buildRestApi(config, initializedAdapter) {
67
- const app = (0, fastify_1.default)();
68
- // Add healthcheck endpoint before middlewares to bypass them
69
- app.get((0, path_1.join)(config.BASE_URL, 'health'), (req, res) => {
70
- res.status(200).send({ message: 'OK', version: VERSION });
71
- });
72
- // Use global error handling
73
- app.setErrorHandler(validation_1.errorCatchingMiddleware);
74
- const transportHandler = await (0, transports_1.buildTransportHandler)(initializedAdapter);
75
- app.register(async (router) => {
76
- // Set up "middlewares" (hooks in fastify)
77
- router.addHook('preHandler', (0, validation_1.validatorMiddleware)(initializedAdapter));
78
- router.addHook('preHandler', (0, cache_1.buildCacheMiddleware)(initializedAdapter));
79
- if (config['CORRELATION_ID_ENABLED']) {
80
- router.addHook('onRequest', util_1.loggingContextMiddleware);
81
- }
82
- router.route({
83
- url: config.BASE_URL,
84
- method: 'POST',
85
- handler: transportHandler,
86
- }); // Pass config maybe? cleaner that multiple instances / dependencies
87
- if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
88
- router.addHook('onResponse', metrics_1.buildMetricsMiddleware);
89
- }
90
- });
91
- return app;
92
- }
@@ -1,16 +0,0 @@
1
- export declare enum HttpRequestType {
2
- CACHE_HIT = "cacheHit",
3
- DATA_PROVIDER_HIT = "dataProviderHit",
4
- ADAPTER_ERROR = "adapterError",
5
- INPUT_ERROR = "inputError",
6
- RATE_LIMIT_ERROR = "rateLimitError",
7
- DP_ERROR = "dataProviderError",
8
- TIMEOUT_ERROR = "timeoutError",
9
- CONNECTION_ERROR = "connectionError",
10
- CUSTOM_ERROR = "customError"
11
- }
12
- /**
13
- * Maxiumum number of characters that a feedId can contain.
14
- */
15
- export declare const MAX_FEED_ID_LENGTH = 300;
16
- export declare const requestDurationBuckets: number[];
@@ -1,25 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.requestDurationBuckets = exports.MAX_FEED_ID_LENGTH = exports.HttpRequestType = void 0;
4
- var HttpRequestType;
5
- (function (HttpRequestType) {
6
- HttpRequestType["CACHE_HIT"] = "cacheHit";
7
- HttpRequestType["DATA_PROVIDER_HIT"] = "dataProviderHit";
8
- HttpRequestType["ADAPTER_ERROR"] = "adapterError";
9
- HttpRequestType["INPUT_ERROR"] = "inputError";
10
- HttpRequestType["RATE_LIMIT_ERROR"] = "rateLimitError";
11
- // BURST_LIMIT_ERROR = 'burstLimitError',
12
- // BACKOFF_ERROR = 'backoffError',
13
- HttpRequestType["DP_ERROR"] = "dataProviderError";
14
- HttpRequestType["TIMEOUT_ERROR"] = "timeoutError";
15
- HttpRequestType["CONNECTION_ERROR"] = "connectionError";
16
- // RES_EMPTY_ERROR = 'responseEmptyError',
17
- // RES_INVALID_ERROR = 'responseInvalidError',
18
- HttpRequestType["CUSTOM_ERROR"] = "customError";
19
- })(HttpRequestType = exports.HttpRequestType || (exports.HttpRequestType = {}));
20
- /**
21
- * Maxiumum number of characters that a feedId can contain.
22
- */
23
- exports.MAX_FEED_ID_LENGTH = 300;
24
- // We should tune these as we collect data, this is the default bucket distribution that prom comes with
25
- exports.requestDurationBuckets = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10];
@@ -1,15 +0,0 @@
1
- import * as client from 'prom-client';
2
- import { AdapterRequest } from '../util';
3
- import { AdapterConfig } from '../config';
4
- import { FastifyReply, HookHandlerDoneFunction } from 'fastify';
5
- export declare function setupMetricsServer(name: string, config: AdapterConfig): void;
6
- export declare const setupMetrics: (name: string, config: AdapterConfig) => void;
7
- /**
8
- * Builds metrics middleware that records end to end EA response times
9
- * and count of requests
10
- *
11
- * @returns the cache middleware function
12
- */
13
- export declare const buildMetricsMiddleware: (req: AdapterRequest, res: FastifyReply, done: HookHandlerDoneFunction) => void;
14
- export declare const httpRequestsTotal: client.Counter<"type" | "method" | "feed_id" | "status_code" | "provider_status_code" | "retry" | "is_cache_warming">;
15
- export declare const httpRequestDurationSeconds: client.Histogram<string>;
@@ -1,123 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
25
- var __importDefault = (this && this.__importDefault) || function (mod) {
26
- return (mod && mod.__esModule) ? mod : { "default": mod };
27
- };
28
- Object.defineProperty(exports, "__esModule", { value: true });
29
- exports.httpRequestDurationSeconds = exports.httpRequestsTotal = exports.buildMetricsMiddleware = exports.setupMetrics = exports.setupMetricsServer = void 0;
30
- const client = __importStar(require("prom-client"));
31
- const constants_1 = require("./constants");
32
- const util_1 = require("../util");
33
- const fastify_1 = __importDefault(require("fastify"));
34
- const path_1 = require("path");
35
- const error_1 = require("../validation/error");
36
- const logger = (0, util_1.makeLogger)('Metrics');
37
- function setupMetricsServer(name, config) {
38
- const metricsApp = (0, fastify_1.default)({
39
- logger: false,
40
- });
41
- const metricsPort = config.METRICS_PORT;
42
- const endpoint = config.METRICS_USE_BASE_URL ? (0, path_1.join)(config.BASE_URL, 'metrics') : '/metrics';
43
- const eaHost = config.EA_HOST;
44
- (0, exports.setupMetrics)(name, config);
45
- metricsApp.get(endpoint, async (_, res) => {
46
- res.type('txt');
47
- res.send(await client.register.metrics());
48
- });
49
- metricsApp.listen(metricsPort, eaHost, () => logger.info(`Monitoring listening on port ${metricsPort}!`));
50
- }
51
- exports.setupMetricsServer = setupMetricsServer;
52
- const setupMetrics = (name, config) => {
53
- client.collectDefaultMetrics();
54
- client.register.setDefaultLabels({
55
- app_name: config.METRICS_NAME || name || 'N/A',
56
- app_version: config['npm_package_version'],
57
- });
58
- };
59
- exports.setupMetrics = setupMetrics;
60
- /**
61
- * Builds metrics middleware that records end to end EA response times
62
- * and count of requests
63
- *
64
- * @returns the cache middleware function
65
- */
66
- const buildMetricsMiddleware = (req, res, done) => {
67
- const feedId = req.requestContext.meta?.metrics?.feedId || 'N/A';
68
- const labels = buildHttpRequestMetricsLabel(feedId, req.requestContext.meta?.error, req.requestContext.meta?.metrics?.cacheHit);
69
- // Record number of requests sent to EA
70
- exports.httpRequestsTotal.labels(labels).inc();
71
- // Record response time of request through entire EA
72
- exports.httpRequestDurationSeconds.observe(res.getResponseTime());
73
- logger.debug(`Response time for ${feedId}: ${res.getResponseTime()}ms`);
74
- done();
75
- };
76
- exports.buildMetricsMiddleware = buildMetricsMiddleware;
77
- const buildHttpRequestMetricsLabel = (feedId, error, cacheHit) => {
78
- const labels = {};
79
- labels.method = 'POST';
80
- labels.feed_id = feedId;
81
- if (error instanceof error_1.AdapterError) {
82
- // If error present and an instace of AdapterError, build label from error info
83
- labels.type = error?.metricsLabel || constants_1.HttpRequestType.ADAPTER_ERROR;
84
- labels.status_code = error?.statusCode;
85
- labels.provider_status_code = error?.providerStatusCode;
86
- }
87
- else if (error instanceof Error) {
88
- // If error present and not instance of generic Error, unexpected failure occurred
89
- // TODO: Build label with 500 error and withold dp status? Status sent as 200 in error handler
90
- labels.type = constants_1.HttpRequestType.ADAPTER_ERROR;
91
- labels.status_code = 500;
92
- }
93
- else {
94
- // If no error present, request went as expected
95
- labels.status_code = 200;
96
- if (cacheHit) {
97
- labels.type = constants_1.HttpRequestType.CACHE_HIT;
98
- }
99
- else {
100
- labels.type = constants_1.HttpRequestType.DATA_PROVIDER_HIT;
101
- labels.provider_status_code = 200;
102
- }
103
- }
104
- return labels;
105
- };
106
- exports.httpRequestsTotal = new client.Counter({
107
- name: 'http_requests_total',
108
- help: 'The number of http requests this external adapter has serviced for its entire uptime',
109
- labelNames: [
110
- 'method',
111
- 'status_code',
112
- 'retry',
113
- 'type',
114
- 'is_cache_warming',
115
- 'feed_id',
116
- 'provider_status_code',
117
- ],
118
- });
119
- exports.httpRequestDurationSeconds = new client.Histogram({
120
- name: 'http_request_duration_seconds',
121
- help: 'A histogram bucket of the distribution of http request durations',
122
- buckets: constants_1.requestDurationBuckets,
123
- });
@@ -1,3 +0,0 @@
1
- import { AdapterMetricsMeta, AdapterRequestData } from '../util';
2
- import { AdapterEndpoint } from '../adapter';
3
- export declare const getMetricsMeta: (endpoint: AdapterEndpoint, data: AdapterRequestData) => AdapterMetricsMeta;
@@ -1,9 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.getMetricsMeta = void 0;
4
- const cache_1 = require("../cache");
5
- const getMetricsMeta = (endpoint, data) => {
6
- const feedId = (0, cache_1.calculateFeedId)(endpoint, data);
7
- return { feedId };
8
- };
9
- exports.getMetricsMeta = getMetricsMeta;
@@ -1,72 +0,0 @@
1
- {
2
- "name": "@chainlink/external-adapter-framework",
3
- "version": "0.0.6",
4
- "main": "dist/index.js",
5
- "license": "MIT",
6
- "dependencies": {
7
- "axios": "^0.27.2",
8
- "fastify": "^3.29.0",
9
- "ioredis": "^5.0.4",
10
- "pino": "^7.9.2",
11
- "prom-client": "13.2.0",
12
- "ts-node": "^10.9.1",
13
- "typescript": "^4.6.3",
14
- "ws": "^8.5.0"
15
- },
16
- "scripts": {
17
- "start": "ts-node src/test.ts",
18
- "generate-docs": "typedoc src/**/*.ts",
19
- "test": "LOG_LEVEL=error EA_PORT=0 c8 ava",
20
- "test-debug": "LOG_LEVEL=trace DEBUG=true EA_PORT=0 c8 ava --verbose",
21
- "build": "tsc",
22
- "lint": "eslint ./src"
23
- },
24
- "devDependencies": {
25
- "@sinonjs/fake-timers": "^9.1.2",
26
- "@types/node": "^18.6.5",
27
- "@types/sinonjs__fake-timers": "^8.1.2",
28
- "@types/ws": "^8.5.3",
29
- "@typescript-eslint/eslint-plugin": "^5.17.0",
30
- "@typescript-eslint/parser": "^5.17.0",
31
- "ava": "^4.2.0",
32
- "c8": "^7.11.2",
33
- "eslint": "^8.14.0",
34
- "eslint-config-prettier": "^8.5.0",
35
- "eslint-plugin-tsdoc": "^0.2.16",
36
- "mock-socket": "^9.1.3",
37
- "nock": "^13.2.4",
38
- "pino-pretty": "^7.6.0",
39
- "prettier": "^2.6.1",
40
- "ts-loader": "^9.3.1",
41
- "typedoc": "^0.22.15",
42
- "webpack": "^5.74.0",
43
- "webpack-cli": "^4.10.0"
44
- },
45
- "prettier": {
46
- "semi": false,
47
- "singleQuote": true,
48
- "printWidth": 100,
49
- "endOfLine": "auto",
50
- "trailingComma": "all",
51
- "arrowParens": "always"
52
- },
53
- "optionalDependencies": {
54
- "bufferutil": "^4.0.6",
55
- "utf-8-validate": "^5.0.9"
56
- },
57
- "ava": {
58
- "files": [
59
- "test/**/*.test.ts"
60
- ],
61
- "extensions": [
62
- "ts"
63
- ],
64
- "require": [
65
- "ts-node/register"
66
- ],
67
- "workerThreads": false,
68
- "environmentVariables": {
69
- "METRICS_ENABLED": "false"
70
- }
71
- }
72
- }
@@ -1,10 +0,0 @@
1
- import { AdapterRateLimitTier, BackgroundExecuteRateLimiter } from '..';
2
- import { AdapterEndpoint } from '../../adapter';
3
- export declare const DEFAULT_SHARED_MS_BETWEEN_REQUESTS = 5000;
4
- export declare class FixedFrequencyRateLimiter implements BackgroundExecuteRateLimiter {
5
- msBetweenRequestsMap: {
6
- [endpointName: string]: number;
7
- };
8
- initialize(endpoints: AdapterEndpoint[], limits?: AdapterRateLimitTier): this;
9
- msUntilNextExecution(endpointName: string): number;
10
- }
@@ -1,37 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.FixedFrequencyRateLimiter = exports.DEFAULT_SHARED_MS_BETWEEN_REQUESTS = void 0;
4
- const __1 = require("..");
5
- const util_1 = require("../../util");
6
- const logger = (0, util_1.makeLogger)('FixedFrequencyRateLimiter');
7
- exports.DEFAULT_SHARED_MS_BETWEEN_REQUESTS = 5000;
8
- class FixedFrequencyRateLimiter {
9
- constructor() {
10
- this.msBetweenRequestsMap = {};
11
- }
12
- initialize(endpoints, limits) {
13
- // Translate the hourly limit into reqs per minute
14
- let sharedMsBetweenRequests = 1000 / (0, __1.consolidateTierLimits)(limits);
15
- // If there is no limit set, we use some reasonable number
16
- if (!limits?.rateLimit1h && !limits?.rateLimit1m && !limits?.rateLimit1s) {
17
- // 5s period for all seems good
18
- sharedMsBetweenRequests = exports.DEFAULT_SHARED_MS_BETWEEN_REQUESTS;
19
- }
20
- logger.debug('Using fixed frequency batch rate limiting');
21
- for (const endpoint of endpoints) {
22
- // TODO: See if we can remove this runtime check
23
- if (endpoint.rateLimiting?.allocationPercentage == null) {
24
- throw new Error(`Allocation percentage for endpoint "${endpoint.name}" is null`);
25
- }
26
- // TODO: Implement different strategy where this is not fixed, but rather divided based on whether all warmers are active
27
- this.msBetweenRequestsMap[endpoint.name] =
28
- (sharedMsBetweenRequests / endpoint.rateLimiting?.allocationPercentage) * 100;
29
- logger.debug(`Endpoint [${endpoint.name}]: ${this.msBetweenRequestsMap[endpoint.name] / 1000}s between requests`);
30
- }
31
- return this;
32
- }
33
- msUntilNextExecution(endpointName) {
34
- return this.msBetweenRequestsMap[endpointName];
35
- }
36
- }
37
- exports.FixedFrequencyRateLimiter = FixedFrequencyRateLimiter;
@@ -1,54 +0,0 @@
1
- import { AdapterEndpoint } from '../adapter';
2
- export * from './request/simple-counting';
3
- export * from './background/fixed-frequency';
4
- export interface AdapterRateLimitTier {
5
- rateLimit1s?: number;
6
- rateLimit1m?: number;
7
- rateLimit1h?: number;
8
- note?: string;
9
- }
10
- /**
11
- * Common interface for all RateLimiter classes to implement
12
- */
13
- export interface RateLimiter {
14
- /**
15
- * Method to ensure all RateLimiters can be initialized in the same manner.
16
- *
17
- * @param limits - settings for how much throughput to allow for the Adapter
18
- * @param endpoints - list of adapter endpoints
19
- */
20
- initialize(endpoints: AdapterEndpoint[], limits: AdapterRateLimitTier): this;
21
- }
22
- /**
23
- * RequestRateLimiters perform checks agains imminent outbound requests for any transport.
24
- */
25
- export interface RequestRateLimiter extends RateLimiter {
26
- /**
27
- * This method will check using whatever strategy is implemented to determine if
28
- * this request can be processed. If so, it returns true; if not, returns false.
29
- */
30
- isUnderLimits(): boolean;
31
- }
32
- /**
33
- * BackgroundExecuteFrequencyRateLimiters will implement custom logic to calculate
34
- * the period of time to wait between background executions for a transport.
35
- */
36
- export interface BackgroundExecuteRateLimiter extends RateLimiter {
37
- msUntilNextExecution(endpointName: string): number;
38
- }
39
- /**
40
- * This method will convert all possible settings for a rate limit tier and
41
- * convert them all to requests per second, returning the lowest one
42
- *
43
- * @param limits - the rate limit tier set for the adapter
44
- * @returns the most restrictive of the set options, in requests per second
45
- */
46
- export declare const consolidateTierLimits: (limits?: AdapterRateLimitTier) => number;
47
- /**
48
- * Validates rate limiting tiers specified for the adapter, and returns the one to use.
49
- *
50
- * @param tiers - the adapter config listing the different available API tiers
51
- * @param selectedTier - chosen API tier from settings, if present
52
- * @returns the specified API tier, or a default one if none are specified
53
- */
54
- export declare const getRateLimitingTier: (tiers?: Record<string, AdapterRateLimitTier>, selectedTier?: string) => AdapterRateLimitTier | undefined;
@@ -1,63 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
- for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
- };
16
- Object.defineProperty(exports, "__esModule", { value: true });
17
- exports.getRateLimitingTier = exports.consolidateTierLimits = void 0;
18
- __exportStar(require("./request/simple-counting"), exports);
19
- __exportStar(require("./background/fixed-frequency"), exports);
20
- /**
21
- * This method will convert all possible settings for a rate limit tier and
22
- * convert them all to requests per second, returning the lowest one
23
- *
24
- * @param limits - the rate limit tier set for the adapter
25
- * @returns the most restrictive of the set options, in requests per second
26
- */
27
- const consolidateTierLimits = (limits) => {
28
- const perHourLimit = (limits?.rateLimit1h || Infinity) / (60 * 60);
29
- const perMinuteLimit = (limits?.rateLimit1m || Infinity) / 60;
30
- const perSecondLimit = limits?.rateLimit1s || Infinity;
31
- return Math.min(perHourLimit, perMinuteLimit, perSecondLimit);
32
- };
33
- exports.consolidateTierLimits = consolidateTierLimits;
34
- /**
35
- * Validates rate limiting tiers specified for the adapter, and returns the one to use.
36
- *
37
- * @param tiers - the adapter config listing the different available API tiers
38
- * @param selectedTier - chosen API tier from settings, if present
39
- * @returns the specified API tier, or a default one if none are specified
40
- */
41
- const getRateLimitingTier = (tiers, selectedTier) => {
42
- if (!tiers) {
43
- return;
44
- }
45
- // Check that if the tiers object is defined, it has values
46
- if (Object.values(tiers).length === 0) {
47
- throw new Error(`The tiers object is defined, but has no entries`);
48
- }
49
- // Check that the tier set in the AdapterConfig is a valid one
50
- if (selectedTier && !tiers[selectedTier]) {
51
- const validTiersString = Object.keys(tiers)
52
- .map((t) => `"${t}"`)
53
- .join(', ');
54
- throw new Error(`The selected rate limit tier "${selectedTier}" is not valid (can be one of ${validTiersString})`);
55
- }
56
- if (!selectedTier) {
57
- // Sort the tiers by most to least restrictive
58
- const sortedTiers = Object.values(tiers).sort((t1, t2) => (0, exports.consolidateTierLimits)(t1) - (0, exports.consolidateTierLimits)(t2));
59
- return sortedTiers[0];
60
- }
61
- return tiers[selectedTier];
62
- };
63
- exports.getRateLimitingTier = getRateLimitingTier;
@@ -1,3 +0,0 @@
1
- import * as client from 'prom-client';
2
- export declare const retrieveCost: <ProviderResponseBody>(data: ProviderResponseBody) => number;
3
- export declare const rateLimitCreditsSpentTotal: client.Counter<"feed_id" | "participant_id">;
@@ -1,44 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
25
- Object.defineProperty(exports, "__esModule", { value: true });
26
- exports.rateLimitCreditsSpentTotal = exports.retrieveCost = void 0;
27
- const client = __importStar(require("prom-client"));
28
- // Retrieve cost field from response if exists
29
- // If not return default cost of 1
30
- const retrieveCost = (data) => {
31
- const cost = data['cost'];
32
- if (typeof cost === 'number' || typeof cost === 'string') {
33
- return Number(cost);
34
- }
35
- else {
36
- return 1;
37
- }
38
- };
39
- exports.retrieveCost = retrieveCost;
40
- exports.rateLimitCreditsSpentTotal = new client.Counter({
41
- name: 'rate_limit_credits_spent_total',
42
- help: 'The number of data provider credits the adapter is consuming',
43
- labelNames: ['participant_id', 'feed_id'],
44
- });
@@ -1,20 +0,0 @@
1
- import { AdapterRateLimitTier, RateLimiter } from '..';
2
- import { AdapterEndpoint } from '../../adapter';
3
- /**
4
- * This rate limiter is the simplest stateful option.
5
- * On startup, it'll compare the different thresholds for each tier, calculate them all
6
- * in the finest window we'll use (seconds), and use the most restrictive one.
7
- * This is so if the EA were to restart, we don't need to worry about persisting state
8
- * for things like daily quotas. The downside is that this does not work well for bursty
9
- * loads or spikes, in cases where e.g. the per second limit is high but daily quotas low.
10
- */
11
- export declare class SimpleCountingRateLimiter implements RateLimiter {
12
- latestSecondInterval: number;
13
- requestsThisSecond: number;
14
- latestMinuteInterval: number;
15
- requestsThisMinute: number;
16
- perSecondLimit: number;
17
- perMinuteLimit: number;
18
- initialize(endpoints: AdapterEndpoint[], limits?: AdapterRateLimitTier): this;
19
- isUnderLimits(): boolean;
20
- }
@@ -1,62 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.SimpleCountingRateLimiter = void 0;
4
- const util_1 = require("../../util");
5
- const logger = (0, util_1.makeLogger)('SimpleCountingRateLimiter');
6
- /**
7
- * This rate limiter is the simplest stateful option.
8
- * On startup, it'll compare the different thresholds for each tier, calculate them all
9
- * in the finest window we'll use (seconds), and use the most restrictive one.
10
- * This is so if the EA were to restart, we don't need to worry about persisting state
11
- * for things like daily quotas. The downside is that this does not work well for bursty
12
- * loads or spikes, in cases where e.g. the per second limit is high but daily quotas low.
13
- */
14
- class SimpleCountingRateLimiter {
15
- constructor() {
16
- this.latestSecondInterval = 0;
17
- this.requestsThisSecond = 0;
18
- this.latestMinuteInterval = 0;
19
- this.requestsThisMinute = 0;
20
- }
21
- initialize(endpoints, limits) {
22
- // Translate the hourly limit into reqs per minute
23
- const perHourLimit = (limits?.rateLimit1h || Infinity) / 60;
24
- this.perMinuteLimit = Math.min(limits?.rateLimit1m || Infinity, perHourLimit);
25
- this.perSecondLimit = limits?.rateLimit1s || Infinity;
26
- logger.debug(`Using rate limiting settings: perMinute = ${this.perMinuteLimit} | perSecond: = ${this.perSecondLimit}`);
27
- return this;
28
- }
29
- isUnderLimits() {
30
- // If the limit is set to infinity, there was no tier limit specified
31
- if (this.perSecondLimit === Infinity && this.perMinuteLimit === Infinity) {
32
- return true;
33
- }
34
- const now = Date.now();
35
- const nearestSecondInterval = Math.floor(now / 1000);
36
- const nearestMinuteInterval = Math.floor(now / (1000 * 60));
37
- // This should always run to completion, even if it doesn't look atomic; therefore the
38
- // Ops should be "thread safe". Thank JS and its infinite single threaded dumbness.
39
- if (nearestSecondInterval !== this.latestSecondInterval) {
40
- logger.trace(`Clearing latest second interval, # of requests logged was: ${this.requestsThisSecond} `);
41
- this.latestSecondInterval = nearestSecondInterval;
42
- this.requestsThisSecond = 0;
43
- }
44
- if (nearestMinuteInterval !== this.latestMinuteInterval) {
45
- logger.trace(`Clearing latest second minute, # of requests logged was: ${this.requestsThisMinute} `);
46
- this.latestMinuteInterval = nearestMinuteInterval;
47
- this.requestsThisMinute = 0;
48
- }
49
- if (this.requestsThisSecond < this.perSecondLimit &&
50
- this.requestsThisMinute < this.perMinuteLimit) {
51
- logger.trace('Request under limits, counting +1');
52
- this.requestsThisSecond++;
53
- this.requestsThisMinute++;
54
- return true;
55
- }
56
- else {
57
- logger.trace('Requests seen this interval are above limits');
58
- return false;
59
- }
60
- }
61
- }
62
- exports.SimpleCountingRateLimiter = SimpleCountingRateLimiter;