@chainlink/external-adapter-framework 0.0.12 → 0.0.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (220) hide show
  1. package/.eslintignore +1 -0
  2. package/.github/README.MD +35 -10
  3. package/.github/workflows/label.yaml +39 -0
  4. package/.github/workflows/publish.yaml +0 -3
  5. package/dist/examples/coingecko/test/e2e/adapter.test.ts.js +82953 -0
  6. package/dist/examples/coingecko/test/integration/adapter.test.ts.js +91672 -0
  7. package/dist/main.js +72703 -0
  8. package/jest.config.js +5 -0
  9. package/package.json +14 -3
  10. package/src/config/provider-limits.ts +14 -17
  11. package/src/examples/coingecko/src/config/index.ts +12 -0
  12. package/src/examples/coingecko/src/config/overrides.json +10826 -0
  13. package/src/examples/coingecko/src/cryptoUtils.ts +88 -0
  14. package/src/examples/coingecko/src/endpoint/coins.ts +54 -0
  15. package/src/examples/coingecko/src/endpoint/crypto-marketcap.ts +66 -0
  16. package/src/examples/coingecko/src/endpoint/crypto-volume.ts +66 -0
  17. package/src/examples/coingecko/src/endpoint/crypto.ts +63 -0
  18. package/src/examples/coingecko/src/endpoint/dominance.ts +40 -0
  19. package/src/examples/coingecko/src/endpoint/global-marketcap.ts +40 -0
  20. package/src/examples/coingecko/src/endpoint/index.ts +6 -0
  21. package/src/examples/coingecko/src/globalUtils.ts +78 -0
  22. package/src/examples/coingecko/src/index.ts +17 -0
  23. package/src/examples/coingecko/test/e2e/adapter.test.ts +278 -0
  24. package/src/examples/coingecko/test/integration/__snapshots__/adapter.test.ts.snap +15 -0
  25. package/src/examples/coingecko/test/integration/adapter.test.ts +281 -0
  26. package/src/examples/coingecko/test/integration/capturedRequests.json +1 -0
  27. package/src/examples/coingecko/test/integration/fixtures.ts +42 -0
  28. package/src/examples/{coingecko → coingecko-old}/batch-warming.ts +1 -0
  29. package/src/examples/{coingecko → coingecko-old}/index.ts +0 -0
  30. package/src/examples/{coingecko → coingecko-old}/rest.ts +0 -0
  31. package/src/transports/batch-warming.ts +7 -2
  32. package/src/transports/rest.ts +5 -1
  33. package/src/util/index.ts +1 -0
  34. package/src/util/recordRequests.ts +47 -0
  35. package/src/validation/overrideFunctions.ts +44 -0
  36. package/test/metrics/warmer-metrics.test.ts +1 -0
  37. package/test/{smoke.test.ts → smoke/smoke.test.ts} +13 -13
  38. package/{test-payload-fail.json → test/smoke/test-payload-fail.json} +0 -0
  39. package/{test-payload.js → test/smoke/test-payload.js} +0 -0
  40. package/{test-payload.json → test/smoke/test-payload.json} +0 -0
  41. package/test/transports/batch.test.ts +1 -0
  42. package/tsconfig.json +4 -1
  43. package/webpack.config.js +39 -5
  44. package/yarn-error.log +3778 -0
  45. package/dist/src/adapter.d.ts +0 -135
  46. package/dist/src/adapter.js +0 -145
  47. package/dist/src/background-executor.d.ts +0 -11
  48. package/dist/src/background-executor.js +0 -45
  49. package/dist/src/cache/factory.d.ts +0 -6
  50. package/dist/src/cache/factory.js +0 -55
  51. package/dist/src/cache/index.d.ts +0 -94
  52. package/dist/src/cache/index.js +0 -173
  53. package/dist/src/cache/local.d.ts +0 -23
  54. package/dist/src/cache/local.js +0 -83
  55. package/dist/src/cache/metrics.d.ts +0 -27
  56. package/dist/src/cache/metrics.js +0 -120
  57. package/dist/src/cache/redis.d.ts +0 -16
  58. package/dist/src/cache/redis.js +0 -100
  59. package/dist/src/chainlink-external-adapter-framework-0.0.6.tgz +0 -0
  60. package/dist/src/config/index.d.ts +0 -214
  61. package/dist/src/config/index.js +0 -380
  62. package/dist/src/config/provider-limits.d.ts +0 -31
  63. package/dist/src/config/provider-limits.js +0 -75
  64. package/dist/src/examples/bank-frick/accounts.d.ts +0 -39
  65. package/dist/src/examples/bank-frick/accounts.js +0 -192
  66. package/dist/src/examples/bank-frick/config/index.d.ts +0 -4
  67. package/dist/src/examples/bank-frick/config/index.js +0 -54
  68. package/dist/src/examples/bank-frick/index.d.ts +0 -2
  69. package/dist/src/examples/bank-frick/index.js +0 -15
  70. package/dist/src/examples/bank-frick/util.d.ts +0 -4
  71. package/dist/src/examples/bank-frick/util.js +0 -39
  72. package/dist/src/examples/coingecko/batch-warming.d.ts +0 -7
  73. package/dist/src/examples/coingecko/batch-warming.js +0 -53
  74. package/dist/src/examples/coingecko/index.d.ts +0 -2
  75. package/dist/src/examples/coingecko/index.js +0 -11
  76. package/dist/src/examples/coingecko/rest.d.ts +0 -12
  77. package/dist/src/examples/coingecko/rest.js +0 -51
  78. package/dist/src/examples/ncfx/config/index.d.ts +0 -12
  79. package/dist/src/examples/ncfx/config/index.js +0 -15
  80. package/dist/src/examples/ncfx/index.d.ts +0 -13
  81. package/dist/src/examples/ncfx/index.js +0 -11
  82. package/dist/src/examples/ncfx/websocket.d.ts +0 -47
  83. package/dist/src/examples/ncfx/websocket.js +0 -73
  84. package/dist/src/index.d.ts +0 -11
  85. package/dist/src/index.js +0 -128
  86. package/dist/src/metrics/constants.d.ts +0 -16
  87. package/dist/src/metrics/constants.js +0 -25
  88. package/dist/src/metrics/index.d.ts +0 -15
  89. package/dist/src/metrics/index.js +0 -122
  90. package/dist/src/metrics/util.d.ts +0 -7
  91. package/dist/src/metrics/util.js +0 -9
  92. package/dist/src/package/adapter.d.ts +0 -88
  93. package/dist/src/package/adapter.js +0 -112
  94. package/dist/src/package/background-executor.d.ts +0 -11
  95. package/dist/src/package/background-executor.js +0 -45
  96. package/dist/src/package/cache/factory.d.ts +0 -6
  97. package/dist/src/package/cache/factory.js +0 -57
  98. package/dist/src/package/cache/index.d.ts +0 -90
  99. package/dist/src/package/cache/index.js +0 -169
  100. package/dist/src/package/cache/local.d.ts +0 -23
  101. package/dist/src/package/cache/local.js +0 -83
  102. package/dist/src/package/cache/metrics.d.ts +0 -27
  103. package/dist/src/package/cache/metrics.js +0 -120
  104. package/dist/src/package/cache/redis.d.ts +0 -16
  105. package/dist/src/package/cache/redis.js +0 -100
  106. package/dist/src/package/config/index.d.ts +0 -195
  107. package/dist/src/package/config/index.js +0 -365
  108. package/dist/src/package/config/provider-limits.d.ts +0 -31
  109. package/dist/src/package/config/provider-limits.js +0 -76
  110. package/dist/src/package/examples/coingecko/batch-warming.d.ts +0 -2
  111. package/dist/src/package/examples/coingecko/batch-warming.js +0 -52
  112. package/dist/src/package/examples/coingecko/index.d.ts +0 -2
  113. package/dist/src/package/examples/coingecko/index.js +0 -10
  114. package/dist/src/package/examples/coingecko/rest.d.ts +0 -2
  115. package/dist/src/package/examples/coingecko/rest.js +0 -50
  116. package/dist/src/package/examples/ncfx/config/index.d.ts +0 -12
  117. package/dist/src/package/examples/ncfx/config/index.js +0 -15
  118. package/dist/src/package/examples/ncfx/index.d.ts +0 -2
  119. package/dist/src/package/examples/ncfx/index.js +0 -10
  120. package/dist/src/package/examples/ncfx/websocket.d.ts +0 -36
  121. package/dist/src/package/examples/ncfx/websocket.js +0 -72
  122. package/dist/src/package/index.d.ts +0 -12
  123. package/dist/src/package/index.js +0 -92
  124. package/dist/src/package/metrics/constants.d.ts +0 -16
  125. package/dist/src/package/metrics/constants.js +0 -25
  126. package/dist/src/package/metrics/index.d.ts +0 -15
  127. package/dist/src/package/metrics/index.js +0 -123
  128. package/dist/src/package/metrics/util.d.ts +0 -3
  129. package/dist/src/package/metrics/util.js +0 -9
  130. package/dist/src/package/package.json +0 -72
  131. package/dist/src/package/rate-limiting/background/fixed-frequency.d.ts +0 -10
  132. package/dist/src/package/rate-limiting/background/fixed-frequency.js +0 -37
  133. package/dist/src/package/rate-limiting/index.d.ts +0 -54
  134. package/dist/src/package/rate-limiting/index.js +0 -63
  135. package/dist/src/package/rate-limiting/metrics.d.ts +0 -3
  136. package/dist/src/package/rate-limiting/metrics.js +0 -44
  137. package/dist/src/package/rate-limiting/request/simple-counting.d.ts +0 -20
  138. package/dist/src/package/rate-limiting/request/simple-counting.js +0 -62
  139. package/dist/src/package/test.d.ts +0 -1
  140. package/dist/src/package/test.js +0 -6
  141. package/dist/src/package/transports/batch-warming.d.ts +0 -34
  142. package/dist/src/package/transports/batch-warming.js +0 -101
  143. package/dist/src/package/transports/index.d.ts +0 -87
  144. package/dist/src/package/transports/index.js +0 -87
  145. package/dist/src/package/transports/metrics.d.ts +0 -21
  146. package/dist/src/package/transports/metrics.js +0 -105
  147. package/dist/src/package/transports/rest.d.ts +0 -43
  148. package/dist/src/package/transports/rest.js +0 -129
  149. package/dist/src/package/transports/util.d.ts +0 -8
  150. package/dist/src/package/transports/util.js +0 -85
  151. package/dist/src/package/transports/websocket.d.ts +0 -80
  152. package/dist/src/package/transports/websocket.js +0 -169
  153. package/dist/src/package/util/expiring-sorted-set.d.ts +0 -21
  154. package/dist/src/package/util/expiring-sorted-set.js +0 -47
  155. package/dist/src/package/util/index.d.ts +0 -11
  156. package/dist/src/package/util/index.js +0 -35
  157. package/dist/src/package/util/logger.d.ts +0 -42
  158. package/dist/src/package/util/logger.js +0 -62
  159. package/dist/src/package/util/request.d.ts +0 -55
  160. package/dist/src/package/util/request.js +0 -2
  161. package/dist/src/package/validation/error.d.ts +0 -50
  162. package/dist/src/package/validation/error.js +0 -79
  163. package/dist/src/package/validation/index.d.ts +0 -5
  164. package/dist/src/package/validation/index.js +0 -86
  165. package/dist/src/package/validation/input-params.d.ts +0 -15
  166. package/dist/src/package/validation/input-params.js +0 -30
  167. package/dist/src/package/validation/override-functions.d.ts +0 -3
  168. package/dist/src/package/validation/override-functions.js +0 -40
  169. package/dist/src/package/validation/preset-tokens.json +0 -23
  170. package/dist/src/package/validation/validator.d.ts +0 -47
  171. package/dist/src/package/validation/validator.js +0 -303
  172. package/dist/src/package.json +0 -72
  173. package/dist/src/rate-limiting/background/fixed-frequency.d.ts +0 -11
  174. package/dist/src/rate-limiting/background/fixed-frequency.js +0 -35
  175. package/dist/src/rate-limiting/index.d.ts +0 -55
  176. package/dist/src/rate-limiting/index.js +0 -63
  177. package/dist/src/rate-limiting/metrics.d.ts +0 -3
  178. package/dist/src/rate-limiting/metrics.js +0 -44
  179. package/dist/src/rate-limiting/request/simple-counting.d.ts +0 -21
  180. package/dist/src/rate-limiting/request/simple-counting.js +0 -62
  181. package/dist/src/test.d.ts +0 -1
  182. package/dist/src/test.js +0 -6
  183. package/dist/src/transports/batch-warming.d.ts +0 -35
  184. package/dist/src/transports/batch-warming.js +0 -101
  185. package/dist/src/transports/index.d.ts +0 -70
  186. package/dist/src/transports/index.js +0 -87
  187. package/dist/src/transports/metrics.d.ts +0 -22
  188. package/dist/src/transports/metrics.js +0 -105
  189. package/dist/src/transports/rest.d.ts +0 -44
  190. package/dist/src/transports/rest.js +0 -131
  191. package/dist/src/transports/util.d.ts +0 -8
  192. package/dist/src/transports/util.js +0 -85
  193. package/dist/src/transports/websocket.d.ts +0 -80
  194. package/dist/src/transports/websocket.js +0 -166
  195. package/dist/src/util/expiring-sorted-set.d.ts +0 -21
  196. package/dist/src/util/expiring-sorted-set.js +0 -47
  197. package/dist/src/util/index.d.ts +0 -12
  198. package/dist/src/util/index.js +0 -35
  199. package/dist/src/util/logger.d.ts +0 -42
  200. package/dist/src/util/logger.js +0 -62
  201. package/dist/src/util/request.d.ts +0 -57
  202. package/dist/src/util/request.js +0 -2
  203. package/dist/src/util/subscription-set/expiring-sorted-set.d.ts +0 -22
  204. package/dist/src/util/subscription-set/expiring-sorted-set.js +0 -47
  205. package/dist/src/util/subscription-set/subscription-set.d.ts +0 -18
  206. package/dist/src/util/subscription-set/subscription-set.js +0 -19
  207. package/dist/src/util/test-payload-loader.d.ts +0 -25
  208. package/dist/src/util/test-payload-loader.js +0 -83
  209. package/dist/src/validation/error.d.ts +0 -50
  210. package/dist/src/validation/error.js +0 -79
  211. package/dist/src/validation/index.d.ts +0 -5
  212. package/dist/src/validation/index.js +0 -91
  213. package/dist/src/validation/input-params.d.ts +0 -15
  214. package/dist/src/validation/input-params.js +0 -30
  215. package/dist/src/validation/override-functions.d.ts +0 -3
  216. package/dist/src/validation/override-functions.js +0 -40
  217. package/dist/src/validation/preset-tokens.json +0 -23
  218. package/dist/src/validation/validator.d.ts +0 -47
  219. package/dist/src/validation/validator.js +0 -303
  220. package/src/test.ts +0 -5
@@ -1,23 +0,0 @@
1
- import { Cache, CacheEntry } from './index';
2
- /**
3
- * Type for a value stored in a LocalCache entry.
4
- *
5
- * @typeParam T - the type for the entry's value
6
- */
7
- export interface LocalCacheEntry<T> {
8
- expirationTimestamp: number;
9
- value: T;
10
- }
11
- /**
12
- * Local implementation of a Cache. It uses a simple js Object, storing entries with both
13
- * a value and an expiration timestamp. Expired entries are deleted on reads (i.e. no background gc/upkeep).
14
- *
15
- * @typeParam T - the type for the entries' values
16
- */
17
- export declare class LocalCache<T = unknown> implements Cache<T> {
18
- store: Record<string, LocalCacheEntry<T>>;
19
- get(key: string): Promise<T | undefined>;
20
- delete(key: string): Promise<void>;
21
- set(key: string, value: T, ttl: number): Promise<void>;
22
- setMany(entries: CacheEntry<T>[], ttl: number): Promise<void>;
23
- }
@@ -1,83 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
25
- Object.defineProperty(exports, "__esModule", { value: true });
26
- exports.LocalCache = void 0;
27
- const util_1 = require("../util");
28
- const cacheMetrics = __importStar(require("./metrics"));
29
- const logger = (0, util_1.makeLogger)('LocalCache');
30
- /**
31
- * Local implementation of a Cache. It uses a simple js Object, storing entries with both
32
- * a value and an expiration timestamp. Expired entries are deleted on reads (i.e. no background gc/upkeep).
33
- *
34
- * @typeParam T - the type for the entries' values
35
- */
36
- class LocalCache {
37
- constructor() {
38
- this.store = {};
39
- }
40
- async get(key) {
41
- logger.trace(`Getting key ${key}`);
42
- const entry = this.store[key];
43
- if (!entry) {
44
- logger.debug(`No entry in local cache for key "${key}", returning undefined`);
45
- return undefined;
46
- }
47
- const expired = entry.expirationTimestamp <= Date.now();
48
- if (expired) {
49
- logger.debug('Entry in local cache expired, deleting and returning undefined');
50
- this.delete(key);
51
- return undefined;
52
- }
53
- else {
54
- logger.debug('Found valid entry in local cache, returning value');
55
- return entry.value;
56
- }
57
- }
58
- async delete(key) {
59
- logger.trace(`Deleting key ${key}`);
60
- delete this.store[key]; // Deletes are slower than ignoring or setting null, fyi
61
- }
62
- async set(key, value, ttl) {
63
- logger.trace(`Setting key ${key} with ttl ${ttl}`);
64
- this.store[key] = {
65
- value,
66
- expirationTimestamp: Date.now() + ttl,
67
- };
68
- // Only record metrics if feed Id is present, otherwise assuming value is not adapter response to record
69
- const feedId = value.meta?.metrics?.feedId;
70
- if (feedId) {
71
- // Record cache set count, max age, and staleness (set to 0 for cache set)
72
- const label = cacheMetrics.cacheMetricsLabel(key, feedId, cacheMetrics.CacheTypes.Local);
73
- cacheMetrics.cacheSet(label, ttl);
74
- }
75
- }
76
- async setMany(entries, ttl) {
77
- logger.trace(`Setting a bunch of keys with ttl ${ttl}`);
78
- for (const { key, value } of entries) {
79
- this.set(key, value, ttl);
80
- }
81
- }
82
- }
83
- exports.LocalCache = LocalCache;
@@ -1,27 +0,0 @@
1
- import * as client from 'prom-client';
2
- interface CacheMetricsLabels {
3
- participant_id: string;
4
- feed_id: string;
5
- cache_type: string;
6
- }
7
- export declare const cacheGet: (label: CacheMetricsLabels, value: unknown, staleness: number) => void;
8
- export declare const cacheSet: (label: CacheMetricsLabels, maxAge: number) => void;
9
- export declare const cacheMetricsLabel: (cacheKey: string, feedId: string, cacheType: string) => {
10
- participant_id: string;
11
- feed_id: string;
12
- cache_type: string;
13
- };
14
- export declare enum CacheTypes {
15
- Redis = "redis",
16
- Local = "local"
17
- }
18
- export declare enum CMD_SENT_STATUS {
19
- TIMEOUT = "TIMEOUT",
20
- FAIL = "FAIL",
21
- SUCCESS = "SUCCESS"
22
- }
23
- export declare const redisConnectionsOpen: client.Counter<string>;
24
- export declare const redisRetriesCount: client.Counter<string>;
25
- export declare const redisCommandsSentCount: client.Counter<"status" | "function_name">;
26
- export declare const cacheWarmerCount: client.Gauge<"isBatched">;
27
- export {};
@@ -1,120 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
25
- Object.defineProperty(exports, "__esModule", { value: true });
26
- exports.cacheWarmerCount = exports.redisCommandsSentCount = exports.redisRetriesCount = exports.redisConnectionsOpen = exports.CMD_SENT_STATUS = exports.CacheTypes = exports.cacheMetricsLabel = exports.cacheSet = exports.cacheGet = void 0;
27
- const client = __importStar(require("prom-client"));
28
- const cacheGet = (label, value, staleness) => {
29
- if (typeof value === 'number' || typeof value === 'string') {
30
- const parsedValue = Number(value);
31
- if (!Number.isNaN(parsedValue) && Number.isFinite(parsedValue)) {
32
- cacheDataGetValues.labels(label).set(parsedValue);
33
- }
34
- }
35
- cacheDataGetCount.labels(label).inc();
36
- cacheDataStalenessSeconds.labels(label).set(staleness);
37
- };
38
- exports.cacheGet = cacheGet;
39
- const cacheSet = (label, maxAge) => {
40
- cacheDataSetCount.labels(label).inc();
41
- cacheDataMaxAge.labels(label).set(maxAge);
42
- cacheDataStalenessSeconds.labels(label).set(0);
43
- };
44
- exports.cacheSet = cacheSet;
45
- const cacheMetricsLabel = (cacheKey, feedId, cacheType) => ({
46
- participant_id: cacheKey,
47
- feed_id: feedId,
48
- cache_type: cacheType,
49
- });
50
- exports.cacheMetricsLabel = cacheMetricsLabel;
51
- var CacheTypes;
52
- (function (CacheTypes) {
53
- CacheTypes["Redis"] = "redis";
54
- CacheTypes["Local"] = "local";
55
- })(CacheTypes = exports.CacheTypes || (exports.CacheTypes = {}));
56
- var CMD_SENT_STATUS;
57
- (function (CMD_SENT_STATUS) {
58
- CMD_SENT_STATUS["TIMEOUT"] = "TIMEOUT";
59
- CMD_SENT_STATUS["FAIL"] = "FAIL";
60
- CMD_SENT_STATUS["SUCCESS"] = "SUCCESS";
61
- })(CMD_SENT_STATUS = exports.CMD_SENT_STATUS || (exports.CMD_SENT_STATUS = {}));
62
- const baseLabels = [
63
- 'feed_id',
64
- 'participant_id',
65
- 'cache_type',
66
- 'is_from_ws',
67
- 'experimental',
68
- ];
69
- // Skipping this metrics for v3
70
- // const cache_execution_duration_seconds = new client.Histogram({
71
- // name: 'cache_execution_duration_seconds',
72
- // help: 'A histogram bucket of the distribution of cache execution durations',
73
- // labelNames: [...baseLabels, 'cache_hit'] as const,
74
- // buckets: [0.01, 0.1, 1, 10],
75
- // })
76
- const cacheDataGetCount = new client.Counter({
77
- name: 'cache_data_get_count',
78
- help: 'A counter that increments every time a value is fetched from the cache',
79
- labelNames: baseLabels,
80
- });
81
- const cacheDataGetValues = new client.Gauge({
82
- name: 'cache_data_get_values',
83
- help: 'A gauge keeping track of values being fetched from cache',
84
- labelNames: baseLabels,
85
- });
86
- const cacheDataMaxAge = new client.Gauge({
87
- name: 'cache_data_max_age',
88
- help: 'A gauge tracking the max age of stored values in the cache',
89
- labelNames: baseLabels,
90
- });
91
- const cacheDataSetCount = new client.Counter({
92
- name: 'cache_data_set_count',
93
- help: 'A counter that increments every time a value is set to the cache',
94
- labelNames: [...baseLabels, 'status_code'],
95
- });
96
- const cacheDataStalenessSeconds = new client.Gauge({
97
- name: 'cache_data_staleness_seconds',
98
- help: 'Observes the staleness of the data returned',
99
- labelNames: baseLabels,
100
- });
101
- // Redis Metrics
102
- exports.redisConnectionsOpen = new client.Counter({
103
- name: 'redis_connections_open',
104
- help: 'The number of redis connections that are open',
105
- });
106
- exports.redisRetriesCount = new client.Counter({
107
- name: 'redis_retries_count',
108
- help: 'The number of retries that have been made to establish a redis connection',
109
- });
110
- exports.redisCommandsSentCount = new client.Counter({
111
- name: 'redis_commands_sent_count',
112
- help: 'The number of redis commands sent',
113
- labelNames: ['status', 'function_name'],
114
- });
115
- // Cache Warmer Metrics
116
- exports.cacheWarmerCount = new client.Gauge({
117
- name: 'cache_warmer_get_count',
118
- help: 'The number of cache warmers running',
119
- labelNames: ['isBatched'],
120
- });
@@ -1,16 +0,0 @@
1
- import Redis from 'ioredis';
2
- import { Cache, CacheEntry } from './index';
3
- /**
4
- * Redis implementation of a Cache. It uses a simple js Object, storing entries with both
5
- * a value and an expiration timestamp. Expired entries are deleted on reads (i.e. no background gc/upkeep).
6
- *
7
- * @typeParam T - the type for the entries' values
8
- */
9
- export declare class RedisCache<T = unknown> implements Cache<T> {
10
- private client;
11
- constructor(client: Redis);
12
- get(key: string): Promise<T | undefined>;
13
- delete(key: string): Promise<void>;
14
- set(key: string, value: T, ttl: number): Promise<void>;
15
- setMany(entries: CacheEntry<T>[], ttl: number): Promise<void>;
16
- }
@@ -1,100 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
25
- Object.defineProperty(exports, "__esModule", { value: true });
26
- exports.RedisCache = void 0;
27
- const util_1 = require("../util");
28
- const cacheMetrics = __importStar(require("./metrics"));
29
- const logger = (0, util_1.makeLogger)('RedisCache');
30
- /**
31
- * Redis implementation of a Cache. It uses a simple js Object, storing entries with both
32
- * a value and an expiration timestamp. Expired entries are deleted on reads (i.e. no background gc/upkeep).
33
- *
34
- * @typeParam T - the type for the entries' values
35
- */
36
- class RedisCache {
37
- constructor(client) {
38
- this.client = client;
39
- }
40
- async get(key) {
41
- logger.trace(`Getting key ${key}`);
42
- const value = await this.client.get(key);
43
- // Record get command sent to Redis
44
- cacheMetrics.redisCommandsSentCount
45
- .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'get' })
46
- .inc();
47
- if (!value) {
48
- logger.debug(`No entry in redis cache for key "${key}", returning undefined`);
49
- return undefined;
50
- }
51
- return JSON.parse(value);
52
- }
53
- async delete(key) {
54
- logger.trace(`Deleting key ${key}`);
55
- await this.client.del(key);
56
- // Record delete command sent to Redis
57
- cacheMetrics.redisCommandsSentCount
58
- .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'delete' })
59
- .inc();
60
- }
61
- async set(key, value, ttl) {
62
- logger.trace(`Setting key ${key}`);
63
- await this.client.set(key, JSON.stringify(value), 'PX', ttl);
64
- // Only record metrics if feed Id is present, otherwise assuming value is not adapter response to record
65
- const feedId = value.meta?.metrics?.feedId;
66
- if (feedId) {
67
- // Record cache set count, max age, and staleness (set to 0 for cache set)
68
- const label = cacheMetrics.cacheMetricsLabel(key, feedId, cacheMetrics.CacheTypes.Redis);
69
- cacheMetrics.cacheSet(label, ttl);
70
- }
71
- // Record set command sent to Redis
72
- cacheMetrics.redisCommandsSentCount
73
- .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'set' })
74
- .inc();
75
- }
76
- async setMany(entries, ttl) {
77
- logger.trace(`Setting a bunch of keys`);
78
- // Unfortunately, there's no ttl for mset
79
- let chain = this.client.multi();
80
- for (const entry of entries) {
81
- chain = chain.set(entry.key, JSON.stringify(entry.value), 'PX', ttl);
82
- }
83
- await chain.exec();
84
- // Loop again, but this time to record these in metrics
85
- for (const entry of entries) {
86
- // Only record metrics if feed Id is present, otherwise assuming value is not adapter response to record
87
- const feedId = entry.value.meta?.metrics?.feedId;
88
- if (feedId) {
89
- // Record cache set count, max age, and staleness (set to 0 for cache set)
90
- const label = cacheMetrics.cacheMetricsLabel(entry.key, feedId, cacheMetrics.CacheTypes.Redis);
91
- cacheMetrics.cacheSet(label, ttl);
92
- }
93
- }
94
- // Record setMany command sent to Redis
95
- cacheMetrics.redisCommandsSentCount
96
- .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'exec' })
97
- .inc();
98
- }
99
- }
100
- exports.RedisCache = RedisCache;
@@ -1,214 +0,0 @@
1
- export declare const BaseSettings: {
2
- readonly API_ENDPOINT: {
3
- readonly description: "The URL that the certain transports use to retrieve data";
4
- readonly type: "string";
5
- };
6
- readonly API_KEY: {
7
- readonly description: "Default setting for an EA key";
8
- readonly type: "string";
9
- };
10
- readonly BASE_URL: {
11
- readonly description: "Starting path for the EA handler endpoint";
12
- readonly type: "string";
13
- readonly default: "/";
14
- };
15
- readonly CACHE_MAX_AGE: {
16
- readonly description: "Maximum amount of time (in ms) that a response will stay cached";
17
- readonly type: "number";
18
- readonly default: 90000;
19
- };
20
- readonly CACHE_REDIS_HOST: {
21
- readonly description: "Hostname for the Redis instance to be used";
22
- readonly type: "string";
23
- readonly default: "127.0.0.1";
24
- };
25
- readonly CACHE_REDIS_PORT: {
26
- readonly description: "Port for the Redis instance to be used";
27
- readonly type: "number";
28
- readonly default: 6379;
29
- };
30
- readonly CACHE_TYPE: {
31
- readonly description: "The type of cache to use throughout the EA";
32
- readonly type: "enum";
33
- readonly default: "local";
34
- readonly options: readonly ["local", "redis"];
35
- };
36
- readonly CORRELATION_ID_ENABLED: {
37
- readonly description: "Flag to enable correlation IDs for sent requests in logging";
38
- readonly type: "boolean";
39
- readonly default: true;
40
- };
41
- readonly EA_PORT: {
42
- readonly description: "Port through which the EA will listen for REST requests (if mode is set to \"reader\" or \"reader-writer\")";
43
- readonly type: "number";
44
- readonly default: 8080;
45
- };
46
- readonly EXPERIMENTAL_METRICS_ENABLED: {
47
- readonly description: "Flag to specify whether or not to collect metrics. Used as fallback for METRICS_ENABLED";
48
- readonly type: "boolean";
49
- readonly default: true;
50
- };
51
- readonly LOG_LEVEL: {
52
- readonly description: "Minimum level required for logs to be output";
53
- readonly type: "string";
54
- readonly default: "info";
55
- };
56
- readonly METRICS_ENABLED: {
57
- readonly description: "Flag to specify whether or not to startup the metrics server";
58
- readonly type: "boolean";
59
- readonly default: true;
60
- };
61
- readonly METRICS_NAME: {
62
- readonly description: "Metrics name";
63
- readonly type: "string";
64
- };
65
- readonly METRICS_PORT: {
66
- readonly description: "Port metrics will be exposed to";
67
- readonly type: "number";
68
- readonly default: 9080;
69
- };
70
- readonly METRICS_USE_BASE_URL: {
71
- readonly description: "Flag to specify whether or not to prepend the BASE_URL to the metrics endpoint";
72
- readonly type: "boolean";
73
- };
74
- readonly RATE_LIMIT_API_TIER: {
75
- readonly description: "Rate limiting tier to use from the available options for the adapter. If not present, the adapter will run using the first tier on the list.";
76
- readonly type: "string";
77
- };
78
- readonly WARMUP_SUBSCRIPTION_TTL: {
79
- readonly type: "number";
80
- readonly description: "TTL for batch warmer subscriptions";
81
- readonly default: 10000;
82
- };
83
- readonly WS_SUBSCRIPTION_TTL: {
84
- readonly description: "";
85
- readonly type: "number";
86
- readonly default: 120000;
87
- };
88
- readonly CACHE_POLLING_MAX_RETRIES: {
89
- readonly description: "Max amount of times to attempt to find EA response in the cache after the Transport has been set up";
90
- readonly type: "number";
91
- readonly default: 10;
92
- };
93
- readonly CACHE_POLLING_SLEEP_MS: {
94
- readonly description: "The number of ms to sleep between each retry to fetch the EA response in the cache";
95
- readonly type: "number";
96
- readonly default: 200;
97
- };
98
- readonly DEFAULT_CACHE_KEY: {
99
- readonly description: "Default key to be used when one cannot be determined from request parameters";
100
- readonly type: "string";
101
- readonly default: "DEFAULT_CACHE_KEY";
102
- };
103
- readonly EA_HOST: {
104
- readonly description: "Host this EA will listen for REST requests on (if mode is set to \"reader\" or \"reader-writer\")";
105
- readonly type: "string";
106
- readonly default: "::";
107
- };
108
- readonly EA_MODE: {
109
- readonly description: "Port this EA will listen for REST requests on (if mode is set to \"reader\" or \"reader-writer\")";
110
- readonly type: "enum";
111
- readonly default: "reader-writer";
112
- readonly options: readonly ["reader", "writer", "reader-writer"];
113
- };
114
- readonly REST_TRANSPORT_MAX_RATE_LIMIT_RETRIES: {
115
- readonly description: "Maximum amount of times the Rest Transport will attempt to set up a request when blocked by the rate limiter";
116
- readonly type: "number";
117
- readonly default: 3;
118
- };
119
- readonly REST_TRANSPORT_MS_BETWEEN_RATE_LIMIT_RETRIES: {
120
- readonly description: "Time that the Rest Transport will wait between retries when blocked by the rate limiter";
121
- readonly type: "number";
122
- readonly default: 400;
123
- };
124
- readonly MAX_COMMON_KEY_SIZE: {
125
- readonly description: "Maximum amount of characters that the common part of the cache key or feed ID can have";
126
- readonly type: "number";
127
- readonly default: 300;
128
- readonly validate: (value?: number) => "MAX_COMMON_KEY_SIZE must be a number between 150 and 500" | undefined;
129
- };
130
- readonly SMOKE_TEST_PAYLOAD_FILE_NAME: {
131
- readonly description: "Name of the test payload file used for the smoke endpoint";
132
- readonly type: "string";
133
- };
134
- };
135
- export declare const buildAdapterConfig: <CustomSettings extends CustomSettingsType<CustomSettings> = EmptySettings>({ overrides, customSettings, }: {
136
- overrides?: Partial<BaseAdapterConfig> | undefined;
137
- customSettings?: SettingsMap | undefined;
138
- }) => AdapterConfig<CustomSettings>;
139
- declare type SettingValueType = string | number | boolean;
140
- declare type SettingType<C extends Setting> = C['type'] extends 'string' ? string : C['type'] extends 'number' ? number : C['type'] extends 'boolean' ? boolean : C['type'] extends 'enum' ? C['options'] extends readonly string[] ? C['options'][number] : never : never;
141
- declare type BaseSettingsType = typeof BaseSettings;
142
- export declare type Setting = {
143
- type: 'string';
144
- description: string;
145
- options?: never;
146
- default?: string;
147
- validate?: (value?: string) => ValidationErrorMessage;
148
- required?: false;
149
- } | {
150
- type: 'string';
151
- description: string;
152
- options?: never;
153
- default?: string;
154
- validate?: (value: string) => ValidationErrorMessage;
155
- required: true;
156
- } | {
157
- type: 'number';
158
- description: string;
159
- options?: never;
160
- default?: number;
161
- validate?: (value?: number) => ValidationErrorMessage;
162
- required?: false;
163
- } | {
164
- type: 'number';
165
- description: string;
166
- options?: never;
167
- default?: number;
168
- validate?: (value: number) => ValidationErrorMessage;
169
- required: true;
170
- } | {
171
- type: 'boolean';
172
- description: string;
173
- options?: never;
174
- default?: boolean;
175
- validate?: (value?: boolean) => ValidationErrorMessage;
176
- required?: false;
177
- } | {
178
- type: 'boolean';
179
- description: string;
180
- options?: never;
181
- default?: boolean;
182
- validate?: (value: boolean) => ValidationErrorMessage;
183
- required: true;
184
- } | {
185
- type: 'enum';
186
- description: string;
187
- default?: string;
188
- options: readonly string[];
189
- validate?: (value?: string) => ValidationErrorMessage;
190
- required?: false;
191
- } | {
192
- type: 'enum';
193
- description: string;
194
- default?: string;
195
- options: readonly string[];
196
- validate?: (value: string) => ValidationErrorMessage;
197
- required: true;
198
- };
199
- export declare type AdapterConfigFromSettings<T extends SettingsMap> = {
200
- -readonly [K in keyof T as T[K] extends {
201
- default: SettingValueType;
202
- } ? K : T[K]['required'] extends true ? K : never]: SettingType<T[K]>;
203
- } & {
204
- -readonly [K in keyof T as T[K] extends {
205
- default: SettingValueType;
206
- } ? never : T[K]['required'] extends true ? never : K]?: SettingType<T[K]> | undefined;
207
- };
208
- export declare type BaseAdapterConfig = AdapterConfigFromSettings<BaseSettingsType>;
209
- export declare type AdapterConfig<T extends CustomSettingsType<T> = SettingsMap> = AdapterConfigFromSettings<T> & BaseAdapterConfig;
210
- export declare type CustomSettingsType<T = SettingsMap> = Record<keyof T, Setting>;
211
- export declare type EmptySettings = Record<string, never>;
212
- export declare type SettingsMap = Record<string, Setting>;
213
- export declare type ValidationErrorMessage = string | undefined;
214
- export {};