@chainlink/external-adapter-framework 0.0.15 → 0.0.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/adapter.js +128 -0
  2. package/background-executor.js +45 -0
  3. package/cache/factory.js +58 -0
  4. package/cache/index.js +173 -0
  5. package/cache/local.js +83 -0
  6. package/cache/metrics.js +120 -0
  7. package/cache/redis.js +100 -0
  8. package/chainlink-external-adapter-framework-v0.0.6.tgz +0 -0
  9. package/config/index.js +366 -0
  10. package/config/provider-limits.js +74 -0
  11. package/examples/bank-frick/accounts.js +192 -0
  12. package/examples/bank-frick/config/index.js +54 -0
  13. package/examples/bank-frick/index.js +15 -0
  14. package/examples/bank-frick/util.js +39 -0
  15. package/examples/coingecko/batch-warming.js +53 -0
  16. package/examples/coingecko/index.js +11 -0
  17. package/examples/coingecko/rest.js +51 -0
  18. package/examples/coingecko/src/config/index.js +13 -0
  19. package/examples/coingecko/src/config/overrides.json +10826 -0
  20. package/examples/coingecko/src/cryptoUtils.js +41 -0
  21. package/examples/coingecko/src/endpoint/coins.js +33 -0
  22. package/examples/coingecko/src/endpoint/crypto-marketcap.js +46 -0
  23. package/examples/coingecko/src/endpoint/crypto-volume.js +46 -0
  24. package/examples/coingecko/src/endpoint/crypto.js +47 -0
  25. package/examples/coingecko/src/endpoint/dominance.js +26 -0
  26. package/examples/coingecko/src/endpoint/global-marketcap.js +26 -0
  27. package/examples/coingecko/src/endpoint/index.js +15 -0
  28. package/examples/coingecko/src/globalUtils.js +48 -0
  29. package/examples/coingecko/src/index.js +14 -0
  30. package/examples/coingecko/test/e2e/adapter.test.js +262 -0
  31. package/examples/coingecko/test/integration/adapter.test.js +264 -0
  32. package/examples/coingecko/test/integration/capturedRequests.json +1 -0
  33. package/examples/coingecko/test/integration/fixtures.js +41 -0
  34. package/examples/coingecko-old/batch-warming.js +53 -0
  35. package/examples/coingecko-old/index.js +11 -0
  36. package/examples/coingecko-old/rest.js +51 -0
  37. package/examples/ncfx/config/index.js +15 -0
  38. package/examples/ncfx/index.js +11 -0
  39. package/examples/ncfx/websocket.js +73 -0
  40. package/index.js +127 -0
  41. package/metrics/constants.js +25 -0
  42. package/metrics/index.js +122 -0
  43. package/metrics/util.js +9 -0
  44. package/package.json +5 -15
  45. package/rate-limiting/background/fixed-frequency.js +35 -0
  46. package/rate-limiting/index.js +84 -0
  47. package/rate-limiting/metrics.js +44 -0
  48. package/rate-limiting/request/simple-counting.js +62 -0
  49. package/test.js +6 -0
  50. package/transports/batch-warming.js +101 -0
  51. package/transports/index.js +87 -0
  52. package/transports/metrics.js +105 -0
  53. package/transports/rest.js +138 -0
  54. package/transports/util.js +86 -0
  55. package/transports/websocket.js +166 -0
  56. package/util/index.js +35 -0
  57. package/util/logger.js +62 -0
  58. package/util/recordRequests.js +45 -0
  59. package/util/request.js +2 -0
  60. package/util/subscription-set/expiring-sorted-set.js +47 -0
  61. package/util/subscription-set/subscription-set.js +19 -0
  62. package/util/test-payload-loader.js +83 -0
  63. package/validation/error.js +79 -0
  64. package/validation/index.js +91 -0
  65. package/validation/input-params.js +30 -0
  66. package/validation/override-functions.js +40 -0
  67. package/validation/overrideFunctions.js +40 -0
  68. package/validation/preset-tokens.json +23 -0
  69. package/validation/validator.js +303 -0
  70. package/.c8rc.json +0 -3
  71. package/.eslintignore +0 -10
  72. package/.eslintrc.js +0 -96
  73. package/.github/README.MD +0 -42
  74. package/.github/actions/setup/action.yaml +0 -13
  75. package/.github/workflows/label.yaml +0 -39
  76. package/.github/workflows/main.yaml +0 -39
  77. package/.github/workflows/publish.yaml +0 -17
  78. package/.prettierignore +0 -13
  79. package/.yarnrc +0 -0
  80. package/README.md +0 -103
  81. package/dist/examples/coingecko/test/e2e/adapter.test.ts.js +0 -82953
  82. package/dist/examples/coingecko/test/integration/adapter.test.ts.js +0 -91672
  83. package/dist/main.js +0 -72703
  84. package/docker-compose.yaml +0 -35
  85. package/env.sh +0 -54
  86. package/env2.sh +0 -55
  87. package/jest.config.js +0 -5
  88. package/publish.sh +0 -0
  89. package/src/adapter.ts +0 -263
  90. package/src/background-executor.ts +0 -52
  91. package/src/cache/factory.ts +0 -26
  92. package/src/cache/index.ts +0 -258
  93. package/src/cache/local.ts +0 -73
  94. package/src/cache/metrics.ts +0 -112
  95. package/src/cache/redis.ts +0 -93
  96. package/src/config/index.ts +0 -517
  97. package/src/config/provider-limits.ts +0 -127
  98. package/src/examples/bank-frick/README.MD +0 -10
  99. package/src/examples/bank-frick/accounts.ts +0 -246
  100. package/src/examples/bank-frick/config/index.ts +0 -53
  101. package/src/examples/bank-frick/index.ts +0 -13
  102. package/src/examples/bank-frick/types.d.ts +0 -38
  103. package/src/examples/bank-frick/util.ts +0 -55
  104. package/src/examples/coingecko/src/config/index.ts +0 -12
  105. package/src/examples/coingecko/src/config/overrides.json +0 -10826
  106. package/src/examples/coingecko/src/cryptoUtils.ts +0 -88
  107. package/src/examples/coingecko/src/endpoint/coins.ts +0 -54
  108. package/src/examples/coingecko/src/endpoint/crypto-marketcap.ts +0 -66
  109. package/src/examples/coingecko/src/endpoint/crypto-volume.ts +0 -66
  110. package/src/examples/coingecko/src/endpoint/crypto.ts +0 -63
  111. package/src/examples/coingecko/src/endpoint/dominance.ts +0 -40
  112. package/src/examples/coingecko/src/endpoint/global-marketcap.ts +0 -40
  113. package/src/examples/coingecko/src/endpoint/index.ts +0 -6
  114. package/src/examples/coingecko/src/globalUtils.ts +0 -78
  115. package/src/examples/coingecko/src/index.ts +0 -17
  116. package/src/examples/coingecko/test/e2e/adapter.test.ts +0 -278
  117. package/src/examples/coingecko/test/integration/__snapshots__/adapter.test.ts.snap +0 -15
  118. package/src/examples/coingecko/test/integration/adapter.test.ts +0 -281
  119. package/src/examples/coingecko/test/integration/capturedRequests.json +0 -1
  120. package/src/examples/coingecko/test/integration/fixtures.ts +0 -42
  121. package/src/examples/coingecko-old/batch-warming.ts +0 -79
  122. package/src/examples/coingecko-old/index.ts +0 -9
  123. package/src/examples/coingecko-old/rest.ts +0 -77
  124. package/src/examples/ncfx/config/index.ts +0 -12
  125. package/src/examples/ncfx/index.ts +0 -9
  126. package/src/examples/ncfx/websocket.ts +0 -99
  127. package/src/index.ts +0 -149
  128. package/src/metrics/constants.ts +0 -23
  129. package/src/metrics/index.ts +0 -115
  130. package/src/metrics/util.ts +0 -18
  131. package/src/rate-limiting/background/fixed-frequency.ts +0 -45
  132. package/src/rate-limiting/index.ts +0 -100
  133. package/src/rate-limiting/metrics.ts +0 -18
  134. package/src/rate-limiting/request/simple-counting.ts +0 -76
  135. package/src/transports/batch-warming.ts +0 -127
  136. package/src/transports/index.ts +0 -152
  137. package/src/transports/metrics.ts +0 -95
  138. package/src/transports/rest.ts +0 -168
  139. package/src/transports/util.ts +0 -63
  140. package/src/transports/websocket.ts +0 -245
  141. package/src/util/index.ts +0 -23
  142. package/src/util/logger.ts +0 -69
  143. package/src/util/recordRequests.ts +0 -47
  144. package/src/util/request.ts +0 -117
  145. package/src/util/subscription-set/expiring-sorted-set.ts +0 -54
  146. package/src/util/subscription-set/subscription-set.ts +0 -35
  147. package/src/util/test-payload-loader.ts +0 -87
  148. package/src/validation/error.ts +0 -116
  149. package/src/validation/index.ts +0 -110
  150. package/src/validation/input-params.ts +0 -45
  151. package/src/validation/override-functions.ts +0 -44
  152. package/src/validation/overrideFunctions.ts +0 -44
  153. package/src/validation/preset-tokens.json +0 -23
  154. package/src/validation/validator.ts +0 -384
  155. package/test/adapter.test.ts +0 -27
  156. package/test/background-executor.test.ts +0 -108
  157. package/test/cache/cache-key.test.ts +0 -114
  158. package/test/cache/helper.ts +0 -100
  159. package/test/cache/local.test.ts +0 -54
  160. package/test/cache/redis.test.ts +0 -89
  161. package/test/correlation.test.ts +0 -114
  162. package/test/index.test.ts +0 -37
  163. package/test/metrics/feed-id.test.ts +0 -38
  164. package/test/metrics/helper.ts +0 -14
  165. package/test/metrics/labels.test.ts +0 -36
  166. package/test/metrics/metrics.test.ts +0 -267
  167. package/test/metrics/redis-metrics.test.ts +0 -113
  168. package/test/metrics/warmer-metrics.test.ts +0 -193
  169. package/test/metrics/ws-metrics.test.ts +0 -225
  170. package/test/rate-limit-config.test.ts +0 -242
  171. package/test/smoke/smoke.test.ts +0 -166
  172. package/test/smoke/test-payload-fail.json +0 -3
  173. package/test/smoke/test-payload.js +0 -22
  174. package/test/smoke/test-payload.json +0 -7
  175. package/test/transports/batch.test.ts +0 -466
  176. package/test/transports/rest.test.ts +0 -242
  177. package/test/transports/websocket.test.ts +0 -183
  178. package/test/tsconfig.json +0 -5
  179. package/test/util.ts +0 -77
  180. package/test/validation.test.ts +0 -178
  181. package/test.sh +0 -20
  182. package/test2.sh +0 -2
  183. package/tsconfig.json +0 -28
  184. package/typedoc.json +0 -6
  185. package/webpack.config.js +0 -57
  186. package/yarn-error.log +0 -3778
package/adapter.js ADDED
@@ -0,0 +1,128 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.Adapter = exports.AdapterEndpoint = void 0;
4
+ const cache_1 = require("./cache");
5
+ const config_1 = require("./config");
6
+ const rate_limiting_1 = require("./rate-limiting");
7
+ const util_1 = require("./util");
8
+ const logger = (0, util_1.makeLogger)('Adapter');
9
+ class AdapterEndpoint {
10
+ constructor(params) {
11
+ this.name = params.name;
12
+ this.aliases = params.aliases;
13
+ this.transport = params.transport;
14
+ this.inputParameters = params.inputParameters;
15
+ this.rateLimiting = params.rateLimiting;
16
+ }
17
+ }
18
+ exports.AdapterEndpoint = AdapterEndpoint;
19
+ class Adapter {
20
+ constructor(params) {
21
+ // After initialization
22
+ this.initialized = false;
23
+ /** Object containing alias translations for all endpoints */
24
+ this.endpointsMap = {};
25
+ // Copy over params
26
+ this.name = params.name;
27
+ this.defaultEndpoint = params.defaultEndpoint?.toLowerCase();
28
+ this.endpoints = params.endpoints;
29
+ this.envDefaultOverrides = params.envDefaultOverrides;
30
+ this.customSettings = params.customSettings;
31
+ this.rateLimiting = params.rateLimiting;
32
+ this.overrides = params.overrides;
33
+ this.config = (0, config_1.buildAdapterConfig)({
34
+ overrides: this.envDefaultOverrides,
35
+ customSettings: this.customSettings,
36
+ });
37
+ this.normalizeEndpointNames();
38
+ this.calculateRateLimitAllocations();
39
+ }
40
+ /**
41
+ * Initializes all of the [[Transport]]s in the adapter, passing along any [[AdapterDependencies]] and [[AdapterConfig]].
42
+ * Additionally, it builds a map out of all the endpoint names and aliases (checking for duplicates).
43
+ */
44
+ async initialize(dependencies) {
45
+ if (this.initialized) {
46
+ throw new Error('This adapter has already been initialized!');
47
+ }
48
+ this.dependencies = this.initializeDependencies(dependencies);
49
+ for (const endpoint of this.endpoints) {
50
+ // Add aliases to map to use in validation
51
+ const aliases = [endpoint.name, ...(endpoint.aliases || [])];
52
+ for (const alias of aliases) {
53
+ if (this.endpointsMap[alias]) {
54
+ throw new Error(`Duplicate endpoint / alias: "${alias}"`);
55
+ }
56
+ this.endpointsMap[alias] = endpoint;
57
+ }
58
+ logger.debug(`Initializing transport for endpoint "${endpoint.name}"...`);
59
+ await endpoint.transport.initialize(this.dependencies, this.config);
60
+ }
61
+ logger.debug('Adapter initialization complete.');
62
+ this.initialized = true;
63
+ }
64
+ /**
65
+ * Takes an adapter and normalizes all endpoint names and aliases, as well as the default endpoint.
66
+ * i.e. makes them lowercase for now
67
+ */
68
+ normalizeEndpointNames() {
69
+ for (const endpoint of this.endpoints) {
70
+ endpoint.name = endpoint.name.toLowerCase();
71
+ endpoint.aliases = endpoint.aliases?.map((a) => a.toLowerCase());
72
+ }
73
+ }
74
+ /**
75
+ * This function will take an adapter structure and go through each endpoint, calculating
76
+ * each one's allocation of the total rate limits that are set for the adapter as a whole.
77
+ *
78
+ */
79
+ calculateRateLimitAllocations() {
80
+ const numberOfEndpoints = this.endpoints.length;
81
+ const endpointsWithExplicitAllocations = this.endpoints.filter((e) => e.rateLimiting);
82
+ const totalExplicitAllocation = endpointsWithExplicitAllocations
83
+ .map((e) => e.rateLimiting?.allocationPercentage || 0)
84
+ .reduce((sum, next) => sum + next, 0);
85
+ if (totalExplicitAllocation > 100) {
86
+ throw new Error('The total allocation set for all endpoints summed cannot exceed 100%');
87
+ }
88
+ if (totalExplicitAllocation === 100 &&
89
+ numberOfEndpoints - endpointsWithExplicitAllocations.length > 0) {
90
+ throw new Error('The explicit allocation is at 100% but there are endpoints with implicit allocation');
91
+ }
92
+ const implicitAllocation = 100 - totalExplicitAllocation;
93
+ logger.debug('Adapter rate limit allocations:');
94
+ for (const endpoint of this.endpoints) {
95
+ if (!endpoint.rateLimiting) {
96
+ endpoint.rateLimiting = {
97
+ allocationPercentage: implicitAllocation / (numberOfEndpoints - endpointsWithExplicitAllocations.length),
98
+ };
99
+ }
100
+ logger.debug(`Endpoint [${endpoint.name}] - ${endpoint.rateLimiting?.allocationPercentage}%`);
101
+ }
102
+ }
103
+ /**
104
+ * This function will process dependencies for an adapter, such as caches or rate limiters,
105
+ * in order to inject them into transports and other relevant places later in the lifecycle.
106
+ *
107
+ * @param inputDependencies - a partial obj of initialized dependencies to override the created ones
108
+ * @returns a set of AdapterDependencies all initialized
109
+ */
110
+ initializeDependencies(inputDependencies) {
111
+ const dependencies = inputDependencies || {};
112
+ if (!dependencies.cache) {
113
+ dependencies.cache = cache_1.CacheFactory.buildCache(this.config);
114
+ }
115
+ const rateLimitingTier = (0, rate_limiting_1.getRateLimitingTier)(this.config, this.rateLimiting?.tiers);
116
+ if (!dependencies.requestRateLimiter) {
117
+ dependencies.requestRateLimiter = new rate_limiting_1.SimpleCountingRateLimiter().initialize(this.endpoints, rateLimitingTier);
118
+ }
119
+ if (!dependencies.backgroundExecuteRateLimiter) {
120
+ dependencies.backgroundExecuteRateLimiter = new rate_limiting_1.FixedFrequencyRateLimiter().initialize(this.endpoints, rateLimitingTier);
121
+ }
122
+ if (!dependencies.subscriptionSetFactory) {
123
+ dependencies.subscriptionSetFactory = new util_1.SubscriptionSetFactory(this.config);
124
+ }
125
+ return dependencies;
126
+ }
127
+ }
128
+ exports.Adapter = Adapter;
@@ -0,0 +1,45 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.callBackgroundExecutes = void 0;
4
+ const util_1 = require("./util");
5
+ const logger = (0, util_1.makeLogger)('BackgroundExecutor');
6
+ /**
7
+ * Very simple background loop that will call the [[Transport.backgroundExecute]] functions in all Transports.
8
+ * It gets the time in ms to wait as the return value from those functions, and sleeps until next execution.
9
+ *
10
+ * @param adapter - an initialized External Adapter
11
+ * @param server - the http server to attach an on close listener to
12
+ */
13
+ async function callBackgroundExecutes(adapter, server) {
14
+ // Set up variable to check later on to see if we need to stop this background "thread"
15
+ // If no server is provided, the listener won't be set and serverClosed will always be false
16
+ let serverClosed = false;
17
+ server?.on('close', () => {
18
+ serverClosed = true;
19
+ });
20
+ for (const endpoint of adapter.endpoints) {
21
+ const backgroundExecute = endpoint.transport.backgroundExecute?.bind(endpoint.transport);
22
+ if (!backgroundExecute) {
23
+ logger.debug(`Endpoint "${endpoint.name}" has no background execute, skipping...`);
24
+ continue;
25
+ }
26
+ const context = {
27
+ adapterEndpoint: endpoint,
28
+ adapterConfig: adapter.config,
29
+ };
30
+ const handler = async () => {
31
+ if (serverClosed) {
32
+ logger.info('Server closed, stopping recursive backgroundExecute handler chain');
33
+ return;
34
+ }
35
+ logger.debug(`Calling background execute for endpoint "${endpoint.name}"`);
36
+ const timeToWait = await backgroundExecute(context);
37
+ logger.debug(`Finished background execute for endpoint "${endpoint.name}", sleeping for ${timeToWait}ms`);
38
+ await (0, util_1.sleep)(timeToWait);
39
+ handler();
40
+ };
41
+ // Start recursive async calls
42
+ handler();
43
+ }
44
+ }
45
+ exports.callBackgroundExecutes = callBackgroundExecutes;
@@ -0,0 +1,58 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || function (mod) {
19
+ if (mod && mod.__esModule) return mod;
20
+ var result = {};
21
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
+ __setModuleDefault(result, mod);
23
+ return result;
24
+ };
25
+ var __importDefault = (this && this.__importDefault) || function (mod) {
26
+ return (mod && mod.__esModule) ? mod : { "default": mod };
27
+ };
28
+ Object.defineProperty(exports, "__esModule", { value: true });
29
+ exports.CacheFactory = void 0;
30
+ const ioredis_1 = __importDefault(require("ioredis"));
31
+ const util_1 = require("../util");
32
+ const local_1 = require("./local");
33
+ const cacheMetrics = __importStar(require("./metrics"));
34
+ const redis_1 = require("./redis");
35
+ const logger = (0, util_1.makeLogger)('CacheFactory');
36
+ class CacheFactory {
37
+ static buildCache(config) {
38
+ logger.info(`Using "${config.CACHE_TYPE}" cache.`);
39
+ if (config.CACHE_TYPE === 'local') {
40
+ return new local_1.LocalCache();
41
+ }
42
+ else if (config.CACHE_TYPE === 'redis') {
43
+ const redis = new ioredis_1.default({
44
+ enableAutoPipelining: true,
45
+ host: config.CACHE_REDIS_HOST,
46
+ port: config.CACHE_REDIS_PORT,
47
+ password: config.CACHE_REDIS_PASSWORD,
48
+ path: config.CACHE_REDIS_PATH,
49
+ timeout: config.CACHE_REDIS_TIMEOUT,
50
+ });
51
+ redis.on('connect', () => {
52
+ cacheMetrics.redisConnectionsOpen.inc();
53
+ });
54
+ return new redis_1.RedisCache(redis);
55
+ }
56
+ }
57
+ }
58
+ exports.CacheFactory = CacheFactory;
package/cache/index.js ADDED
@@ -0,0 +1,173 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || function (mod) {
19
+ if (mod && mod.__esModule) return mod;
20
+ var result = {};
21
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
+ __setModuleDefault(result, mod);
23
+ return result;
24
+ };
25
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
26
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
27
+ };
28
+ Object.defineProperty(exports, "__esModule", { value: true });
29
+ exports.buildCacheMiddleware = exports.pollResponseFromCache = exports.calculateKey = exports.calculateFeedId = exports.calculateCacheKey = void 0;
30
+ const util_1 = require("../util");
31
+ const cacheMetrics = __importStar(require("./metrics"));
32
+ __exportStar(require("./local"), exports);
33
+ __exportStar(require("./redis"), exports);
34
+ __exportStar(require("./factory"), exports);
35
+ const logger = (0, util_1.makeLogger)('Cache');
36
+ // Uses calculateKey to generate a unique key from the endpoint name, data, and input parameters
37
+ const calculateCacheKey = ({ adapterEndpoint, adapterConfig, }, data) => {
38
+ const paramNames = Object.keys(adapterEndpoint.inputParameters);
39
+ if (paramNames.length === 0) {
40
+ logger.trace(`Using default cache key ${adapterConfig.DEFAULT_CACHE_KEY}`);
41
+ return adapterConfig.DEFAULT_CACHE_KEY;
42
+ }
43
+ return `${adapterEndpoint.name}-${(0, exports.calculateKey)(data, paramNames, adapterConfig)}`;
44
+ };
45
+ exports.calculateCacheKey = calculateCacheKey;
46
+ const calculateFeedId = ({ adapterEndpoint, adapterConfig, }, data) => {
47
+ const paramNames = Object.keys(adapterEndpoint.inputParameters);
48
+ if (paramNames.length === 0) {
49
+ logger.trace(`Cannot generate Feed ID without input parameters`);
50
+ return 'N/A';
51
+ }
52
+ return (0, exports.calculateKey)(data, paramNames, adapterConfig);
53
+ };
54
+ exports.calculateFeedId = calculateFeedId;
55
+ /**
56
+ * Calculates a unique key from the provided data.
57
+ *
58
+ * @param data - the request data/body, i.e. the adapter input params
59
+ * @param paramNames - the keys from adapter endpoint input parameters
60
+ * @returns the calculated unique key
61
+ *
62
+ * @example
63
+ * ```
64
+ * calculateKey({ base: 'ETH', quote: 'BTC' }, ['base','quote'])
65
+ * // equals `|base:eth|quote:btc`
66
+ * ```
67
+ */
68
+ const calculateKey = (data, paramNames, adapterConfig) => {
69
+ if (data && typeof data !== 'object') {
70
+ throw new Error('Data to calculate cache key should be an object');
71
+ }
72
+ const params = data;
73
+ let cacheKey = '';
74
+ for (const paramName of paramNames) {
75
+ // Ignore overrides param when generating cache keys
76
+ if (paramName === 'overrides') {
77
+ continue;
78
+ }
79
+ const param = params[paramName];
80
+ if (param === undefined) {
81
+ continue;
82
+ }
83
+ cacheKey += `|${paramName}:`;
84
+ switch (typeof param) {
85
+ case 'string':
86
+ cacheKey += param.toLowerCase();
87
+ break;
88
+ case 'number':
89
+ case 'boolean':
90
+ cacheKey += param.toString();
91
+ break;
92
+ case 'object':
93
+ // Force cache keys to only use performant properties of the input params.
94
+ // If the object were to be used, we'd have to sort its properties.
95
+ logger.debug(`Property "${paramName}" in request parameters is of type object, and won't be used in the cacheKey`);
96
+ }
97
+ }
98
+ if (cacheKey.length > adapterConfig.MAX_COMMON_KEY_SIZE) {
99
+ logger.warn(`Generated cache key for adapter request is bigger than the MAX_COMMON_KEY_SIZE and will be truncated`);
100
+ cacheKey = cacheKey.slice(0, adapterConfig.MAX_COMMON_KEY_SIZE);
101
+ }
102
+ logger.trace(`Generated cache key for request: "${cacheKey}"`);
103
+ return cacheKey;
104
+ };
105
+ exports.calculateKey = calculateKey;
106
+ // Calculate the amount of time the non-expired entry has been in the cache
107
+ const calculateStaleness = (expirationTimestamp, ttl) => {
108
+ if (expirationTimestamp) {
109
+ const createTimestamp = expirationTimestamp - ttl;
110
+ return (Date.now() - createTimestamp) / 1000;
111
+ }
112
+ else {
113
+ // If expirationTimestamp is not available, staleness cannot be calculated
114
+ // Defaults to ttl for metrics purposes
115
+ return ttl;
116
+ }
117
+ };
118
+ /**
119
+ * Polls the provided Cache for an AdapterResponse set in the provided key. If the maximum
120
+ * amount of retries is exceeded, it returns undefined instead.
121
+ *
122
+ * @param cache - a Cache instance
123
+ * @param key - the key generated from an AdapterRequest that corresponds to the desired AdapterResponse
124
+ * @param retry - current retry, only for internal use
125
+ * @returns the AdapterResponse if found, else undefined
126
+ */
127
+ const pollResponseFromCache = async (cache, key, options, retry = 0) => {
128
+ if (retry > options.maxRetries) {
129
+ // Ideally this shouldn't happen often (p99 of reqs should be found in the cache)
130
+ logger.info('Exceeded max cache polling retries');
131
+ return undefined;
132
+ }
133
+ logger.trace('Getting response from cache...');
134
+ const response = await cache.get(key);
135
+ if (response) {
136
+ logger.trace('Got response from cache');
137
+ return response;
138
+ }
139
+ if (options.maxRetries === 0) {
140
+ logger.debug(`Response not found, retries disabled`);
141
+ return undefined;
142
+ }
143
+ logger.debug(`Response not found, sleeping ${options.sleep} milliseconds...`);
144
+ await (0, util_1.sleep)(options.sleep);
145
+ return (0, exports.pollResponseFromCache)(cache, key, options, retry + 1);
146
+ };
147
+ exports.pollResponseFromCache = pollResponseFromCache;
148
+ /**
149
+ * Given a Cache instance in the adapter dependencies, builds a middleware function that will perform
150
+ * a get from said Cache and return that if found; otherwise it'll continue the middleware chain.
151
+ *
152
+ * @param adapter - an initialized adapter
153
+ * @returns the cache middleware function
154
+ */
155
+ const buildCacheMiddleware = (adapter) => async (req, res) => {
156
+ const response = await adapter.dependencies.cache.get(req.requestContext.cacheKey);
157
+ if (response) {
158
+ logger.debug('Found response from cache, sending that');
159
+ if (adapter.config.METRICS_ENABLED && adapter.config.EXPERIMENTAL_METRICS_ENABLED) {
160
+ const label = cacheMetrics.cacheMetricsLabel(req.requestContext.cacheKey, req.requestContext.meta?.metrics?.feedId || 'N/A', adapter.config.CACHE_TYPE);
161
+ // Record cache staleness and cache get count and value
162
+ const staleness = calculateStaleness(response.maxAge, adapter.config.CACHE_MAX_AGE);
163
+ cacheMetrics.cacheGet(label, response.result, staleness);
164
+ req.requestContext.meta = {
165
+ ...req.requestContext.meta,
166
+ metrics: { ...req.requestContext.meta?.metrics, cacheHit: true },
167
+ };
168
+ }
169
+ return res.send(response);
170
+ }
171
+ logger.debug('Did not find response in cache, moving to next middleware');
172
+ };
173
+ exports.buildCacheMiddleware = buildCacheMiddleware;
package/cache/local.js ADDED
@@ -0,0 +1,83 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || function (mod) {
19
+ if (mod && mod.__esModule) return mod;
20
+ var result = {};
21
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
+ __setModuleDefault(result, mod);
23
+ return result;
24
+ };
25
+ Object.defineProperty(exports, "__esModule", { value: true });
26
+ exports.LocalCache = void 0;
27
+ const util_1 = require("../util");
28
+ const cacheMetrics = __importStar(require("./metrics"));
29
+ const logger = (0, util_1.makeLogger)('LocalCache');
30
+ /**
31
+ * Local implementation of a Cache. It uses a simple js Object, storing entries with both
32
+ * a value and an expiration timestamp. Expired entries are deleted on reads (i.e. no background gc/upkeep).
33
+ *
34
+ * @typeParam T - the type for the entries' values
35
+ */
36
+ class LocalCache {
37
+ constructor() {
38
+ this.store = {};
39
+ }
40
+ async get(key) {
41
+ logger.trace(`Getting key ${key}`);
42
+ const entry = this.store[key];
43
+ if (!entry) {
44
+ logger.debug(`No entry in local cache for key "${key}", returning undefined`);
45
+ return undefined;
46
+ }
47
+ const expired = entry.expirationTimestamp <= Date.now();
48
+ if (expired) {
49
+ logger.debug('Entry in local cache expired, deleting and returning undefined');
50
+ this.delete(key);
51
+ return undefined;
52
+ }
53
+ else {
54
+ logger.debug('Found valid entry in local cache, returning value');
55
+ return entry.value;
56
+ }
57
+ }
58
+ async delete(key) {
59
+ logger.trace(`Deleting key ${key}`);
60
+ delete this.store[key]; // Deletes are slower than ignoring or setting null, fyi
61
+ }
62
+ async set(key, value, ttl) {
63
+ logger.trace(`Setting key ${key} with ttl ${ttl}`);
64
+ this.store[key] = {
65
+ value,
66
+ expirationTimestamp: Date.now() + ttl,
67
+ };
68
+ // Only record metrics if feed Id is present, otherwise assuming value is not adapter response to record
69
+ const feedId = value.meta?.metrics?.feedId;
70
+ if (feedId) {
71
+ // Record cache set count, max age, and staleness (set to 0 for cache set)
72
+ const label = cacheMetrics.cacheMetricsLabel(key, feedId, cacheMetrics.CacheTypes.Local);
73
+ cacheMetrics.cacheSet(label, ttl);
74
+ }
75
+ }
76
+ async setMany(entries, ttl) {
77
+ logger.trace(`Setting a bunch of keys with ttl ${ttl}`);
78
+ for (const { key, value } of entries) {
79
+ this.set(key, value, ttl);
80
+ }
81
+ }
82
+ }
83
+ exports.LocalCache = LocalCache;
@@ -0,0 +1,120 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || function (mod) {
19
+ if (mod && mod.__esModule) return mod;
20
+ var result = {};
21
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
+ __setModuleDefault(result, mod);
23
+ return result;
24
+ };
25
+ Object.defineProperty(exports, "__esModule", { value: true });
26
+ exports.cacheWarmerCount = exports.redisCommandsSentCount = exports.redisRetriesCount = exports.redisConnectionsOpen = exports.CMD_SENT_STATUS = exports.CacheTypes = exports.cacheMetricsLabel = exports.cacheSet = exports.cacheGet = void 0;
27
+ const client = __importStar(require("prom-client"));
28
+ const cacheGet = (label, value, staleness) => {
29
+ if (typeof value === 'number' || typeof value === 'string') {
30
+ const parsedValue = Number(value);
31
+ if (!Number.isNaN(parsedValue) && Number.isFinite(parsedValue)) {
32
+ cacheDataGetValues.labels(label).set(parsedValue);
33
+ }
34
+ }
35
+ cacheDataGetCount.labels(label).inc();
36
+ cacheDataStalenessSeconds.labels(label).set(staleness);
37
+ };
38
+ exports.cacheGet = cacheGet;
39
+ const cacheSet = (label, maxAge) => {
40
+ cacheDataSetCount.labels(label).inc();
41
+ cacheDataMaxAge.labels(label).set(maxAge);
42
+ cacheDataStalenessSeconds.labels(label).set(0);
43
+ };
44
+ exports.cacheSet = cacheSet;
45
+ const cacheMetricsLabel = (cacheKey, feedId, cacheType) => ({
46
+ participant_id: cacheKey,
47
+ feed_id: feedId,
48
+ cache_type: cacheType,
49
+ });
50
+ exports.cacheMetricsLabel = cacheMetricsLabel;
51
+ var CacheTypes;
52
+ (function (CacheTypes) {
53
+ CacheTypes["Redis"] = "redis";
54
+ CacheTypes["Local"] = "local";
55
+ })(CacheTypes = exports.CacheTypes || (exports.CacheTypes = {}));
56
+ var CMD_SENT_STATUS;
57
+ (function (CMD_SENT_STATUS) {
58
+ CMD_SENT_STATUS["TIMEOUT"] = "TIMEOUT";
59
+ CMD_SENT_STATUS["FAIL"] = "FAIL";
60
+ CMD_SENT_STATUS["SUCCESS"] = "SUCCESS";
61
+ })(CMD_SENT_STATUS = exports.CMD_SENT_STATUS || (exports.CMD_SENT_STATUS = {}));
62
+ const baseLabels = [
63
+ 'feed_id',
64
+ 'participant_id',
65
+ 'cache_type',
66
+ 'is_from_ws',
67
+ 'experimental',
68
+ ];
69
+ // Skipping this metrics for v3
70
+ // const cache_execution_duration_seconds = new client.Histogram({
71
+ // name: 'cache_execution_duration_seconds',
72
+ // help: 'A histogram bucket of the distribution of cache execution durations',
73
+ // labelNames: [...baseLabels, 'cache_hit'] as const,
74
+ // buckets: [0.01, 0.1, 1, 10],
75
+ // })
76
+ const cacheDataGetCount = new client.Counter({
77
+ name: 'cache_data_get_count',
78
+ help: 'A counter that increments every time a value is fetched from the cache',
79
+ labelNames: baseLabels,
80
+ });
81
+ const cacheDataGetValues = new client.Gauge({
82
+ name: 'cache_data_get_values',
83
+ help: 'A gauge keeping track of values being fetched from cache',
84
+ labelNames: baseLabels,
85
+ });
86
+ const cacheDataMaxAge = new client.Gauge({
87
+ name: 'cache_data_max_age',
88
+ help: 'A gauge tracking the max age of stored values in the cache',
89
+ labelNames: baseLabels,
90
+ });
91
+ const cacheDataSetCount = new client.Counter({
92
+ name: 'cache_data_set_count',
93
+ help: 'A counter that increments every time a value is set to the cache',
94
+ labelNames: [...baseLabels, 'status_code'],
95
+ });
96
+ const cacheDataStalenessSeconds = new client.Gauge({
97
+ name: 'cache_data_staleness_seconds',
98
+ help: 'Observes the staleness of the data returned',
99
+ labelNames: baseLabels,
100
+ });
101
+ // Redis Metrics
102
+ exports.redisConnectionsOpen = new client.Counter({
103
+ name: 'redis_connections_open',
104
+ help: 'The number of redis connections that are open',
105
+ });
106
+ exports.redisRetriesCount = new client.Counter({
107
+ name: 'redis_retries_count',
108
+ help: 'The number of retries that have been made to establish a redis connection',
109
+ });
110
+ exports.redisCommandsSentCount = new client.Counter({
111
+ name: 'redis_commands_sent_count',
112
+ help: 'The number of redis commands sent',
113
+ labelNames: ['status', 'function_name'],
114
+ });
115
+ // Cache Warmer Metrics
116
+ exports.cacheWarmerCount = new client.Gauge({
117
+ name: 'cache_warmer_get_count',
118
+ help: 'The number of cache warmers running',
119
+ labelNames: ['isBatched'],
120
+ });