@chainlink/external-adapter-framework 0.0.10 → 0.0.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (269) hide show
  1. package/.c8rc.json +3 -0
  2. package/.eslintignore +9 -0
  3. package/.eslintrc.js +96 -0
  4. package/.github/README.MD +17 -0
  5. package/.github/actions/setup/action.yaml +13 -0
  6. package/.github/workflows/main.yaml +39 -0
  7. package/.github/workflows/publish.yaml +20 -0
  8. package/.prettierignore +13 -0
  9. package/.yarnrc +0 -0
  10. package/README.md +103 -0
  11. package/dist/src/adapter.d.ts +135 -0
  12. package/dist/src/adapter.js +145 -0
  13. package/dist/src/background-executor.d.ts +11 -0
  14. package/{background-executor.js → dist/src/background-executor.js} +0 -0
  15. package/{cache → dist/src/cache}/factory.d.ts +0 -0
  16. package/dist/src/cache/factory.js +55 -0
  17. package/dist/src/cache/index.d.ts +94 -0
  18. package/dist/src/cache/index.js +173 -0
  19. package/{cache → dist/src/cache}/local.d.ts +0 -0
  20. package/{cache → dist/src/cache}/local.js +0 -0
  21. package/{cache → dist/src/cache}/metrics.d.ts +0 -0
  22. package/{cache → dist/src/cache}/metrics.js +0 -0
  23. package/{cache → dist/src/cache}/redis.d.ts +0 -0
  24. package/dist/src/cache/redis.js +100 -0
  25. package/dist/src/chainlink-external-adapter-framework-0.0.6.tgz +0 -0
  26. package/dist/src/config/index.d.ts +214 -0
  27. package/dist/src/config/index.js +380 -0
  28. package/{config → dist/src/config}/provider-limits.d.ts +0 -0
  29. package/{config → dist/src/config}/provider-limits.js +1 -1
  30. package/dist/src/examples/bank-frick/accounts.d.ts +39 -0
  31. package/dist/src/examples/bank-frick/accounts.js +192 -0
  32. package/dist/src/examples/bank-frick/config/index.d.ts +4 -0
  33. package/dist/src/examples/bank-frick/config/index.js +54 -0
  34. package/dist/src/examples/bank-frick/index.d.ts +2 -0
  35. package/dist/src/examples/bank-frick/index.js +15 -0
  36. package/dist/src/examples/bank-frick/util.d.ts +4 -0
  37. package/dist/src/examples/bank-frick/util.js +39 -0
  38. package/dist/src/examples/coingecko/batch-warming.d.ts +7 -0
  39. package/dist/src/examples/coingecko/batch-warming.js +53 -0
  40. package/dist/src/examples/coingecko/index.d.ts +2 -0
  41. package/dist/src/examples/coingecko/index.js +11 -0
  42. package/dist/src/examples/coingecko/rest.d.ts +12 -0
  43. package/dist/src/examples/coingecko/rest.js +51 -0
  44. package/{examples → dist/src/examples}/ncfx/config/index.d.ts +0 -0
  45. package/{examples → dist/src/examples}/ncfx/config/index.js +0 -0
  46. package/dist/src/examples/ncfx/index.d.ts +13 -0
  47. package/dist/src/examples/ncfx/index.js +11 -0
  48. package/dist/src/examples/ncfx/websocket.d.ts +47 -0
  49. package/dist/src/examples/ncfx/websocket.js +73 -0
  50. package/dist/src/index.d.ts +11 -0
  51. package/dist/src/index.js +128 -0
  52. package/{metrics → dist/src/metrics}/constants.d.ts +0 -0
  53. package/{metrics → dist/src/metrics}/constants.js +0 -0
  54. package/{metrics → dist/src/metrics}/index.d.ts +0 -0
  55. package/dist/src/metrics/index.js +122 -0
  56. package/dist/src/metrics/util.d.ts +7 -0
  57. package/dist/src/metrics/util.js +9 -0
  58. package/{adapter.d.ts → dist/src/package/adapter.d.ts} +0 -0
  59. package/{adapter.js → dist/src/package/adapter.js} +0 -0
  60. package/{background-executor.d.ts → dist/src/package/background-executor.d.ts} +0 -0
  61. package/dist/src/package/background-executor.js +45 -0
  62. package/dist/src/package/cache/factory.d.ts +6 -0
  63. package/{cache → dist/src/package/cache}/factory.js +0 -0
  64. package/{cache → dist/src/package/cache}/index.d.ts +0 -0
  65. package/{cache → dist/src/package/cache}/index.js +0 -0
  66. package/dist/src/package/cache/local.d.ts +23 -0
  67. package/dist/src/package/cache/local.js +83 -0
  68. package/dist/src/package/cache/metrics.d.ts +27 -0
  69. package/dist/src/package/cache/metrics.js +120 -0
  70. package/dist/src/package/cache/redis.d.ts +16 -0
  71. package/{cache → dist/src/package/cache}/redis.js +0 -0
  72. package/{config → dist/src/package/config}/index.d.ts +0 -0
  73. package/{config → dist/src/package/config}/index.js +0 -0
  74. package/dist/src/package/config/provider-limits.d.ts +31 -0
  75. package/dist/src/package/config/provider-limits.js +76 -0
  76. package/{examples → dist/src/package/examples}/coingecko/batch-warming.d.ts +0 -0
  77. package/{examples → dist/src/package/examples}/coingecko/batch-warming.js +0 -0
  78. package/{examples → dist/src/package/examples}/coingecko/index.d.ts +0 -0
  79. package/{examples → dist/src/package/examples}/coingecko/index.js +0 -0
  80. package/{examples → dist/src/package/examples}/coingecko/rest.d.ts +0 -0
  81. package/{examples → dist/src/package/examples}/coingecko/rest.js +0 -0
  82. package/dist/src/package/examples/ncfx/config/index.d.ts +12 -0
  83. package/dist/src/package/examples/ncfx/config/index.js +15 -0
  84. package/{examples → dist/src/package/examples}/ncfx/index.d.ts +0 -0
  85. package/{examples → dist/src/package/examples}/ncfx/index.js +0 -0
  86. package/{examples → dist/src/package/examples}/ncfx/websocket.d.ts +0 -0
  87. package/{examples → dist/src/package/examples}/ncfx/websocket.js +0 -0
  88. package/{index.d.ts → dist/src/package/index.d.ts} +0 -0
  89. package/{index.js → dist/src/package/index.js} +0 -0
  90. package/dist/src/package/metrics/constants.d.ts +16 -0
  91. package/dist/src/package/metrics/constants.js +25 -0
  92. package/dist/src/package/metrics/index.d.ts +15 -0
  93. package/{metrics → dist/src/package/metrics}/index.js +0 -0
  94. package/{metrics → dist/src/package/metrics}/util.d.ts +0 -0
  95. package/{metrics → dist/src/package/metrics}/util.js +0 -0
  96. package/dist/src/package/package.json +72 -0
  97. package/{rate-limiting → dist/src/package/rate-limiting}/background/fixed-frequency.d.ts +0 -0
  98. package/{rate-limiting → dist/src/package/rate-limiting}/background/fixed-frequency.js +0 -0
  99. package/{rate-limiting → dist/src/package/rate-limiting}/index.d.ts +0 -0
  100. package/{rate-limiting → dist/src/package/rate-limiting}/index.js +0 -0
  101. package/{rate-limiting → dist/src/package/rate-limiting}/metrics.d.ts +0 -0
  102. package/{rate-limiting → dist/src/package/rate-limiting}/metrics.js +0 -0
  103. package/{rate-limiting → dist/src/package/rate-limiting}/request/simple-counting.d.ts +0 -0
  104. package/{rate-limiting → dist/src/package/rate-limiting}/request/simple-counting.js +0 -0
  105. package/{test.d.ts → dist/src/package/test.d.ts} +0 -0
  106. package/{test.js → dist/src/package/test.js} +0 -0
  107. package/{transports → dist/src/package/transports}/batch-warming.d.ts +0 -0
  108. package/{transports → dist/src/package/transports}/batch-warming.js +0 -0
  109. package/{transports → dist/src/package/transports}/index.d.ts +0 -0
  110. package/{transports → dist/src/package/transports}/index.js +0 -0
  111. package/{transports → dist/src/package/transports}/metrics.d.ts +0 -0
  112. package/{transports → dist/src/package/transports}/metrics.js +0 -0
  113. package/{transports → dist/src/package/transports}/rest.d.ts +0 -0
  114. package/{transports → dist/src/package/transports}/rest.js +0 -0
  115. package/{transports → dist/src/package/transports}/util.d.ts +0 -0
  116. package/{transports → dist/src/package/transports}/util.js +0 -0
  117. package/{transports → dist/src/package/transports}/websocket.d.ts +0 -0
  118. package/{transports → dist/src/package/transports}/websocket.js +0 -0
  119. package/{util → dist/src/package/util}/expiring-sorted-set.d.ts +0 -0
  120. package/{util → dist/src/package/util}/expiring-sorted-set.js +0 -0
  121. package/{util → dist/src/package/util}/index.d.ts +0 -0
  122. package/{util → dist/src/package/util}/index.js +0 -0
  123. package/{util → dist/src/package/util}/logger.d.ts +0 -0
  124. package/{util → dist/src/package/util}/logger.js +0 -0
  125. package/{util → dist/src/package/util}/request.d.ts +0 -0
  126. package/{util → dist/src/package/util}/request.js +0 -0
  127. package/{validation → dist/src/package/validation}/error.d.ts +0 -0
  128. package/{validation → dist/src/package/validation}/error.js +0 -0
  129. package/{validation → dist/src/package/validation}/index.d.ts +0 -0
  130. package/{validation → dist/src/package/validation}/index.js +0 -0
  131. package/{validation → dist/src/package/validation}/input-params.d.ts +0 -0
  132. package/{validation → dist/src/package/validation}/input-params.js +0 -0
  133. package/{validation → dist/src/package/validation}/override-functions.d.ts +0 -0
  134. package/{validation → dist/src/package/validation}/override-functions.js +0 -0
  135. package/{validation → dist/src/package/validation}/preset-tokens.json +0 -0
  136. package/{validation → dist/src/package/validation}/validator.d.ts +0 -0
  137. package/{validation → dist/src/package/validation}/validator.js +0 -0
  138. package/dist/src/package.json +72 -0
  139. package/dist/src/rate-limiting/background/fixed-frequency.d.ts +11 -0
  140. package/dist/src/rate-limiting/background/fixed-frequency.js +35 -0
  141. package/dist/src/rate-limiting/index.d.ts +55 -0
  142. package/dist/src/rate-limiting/index.js +63 -0
  143. package/dist/src/rate-limiting/metrics.d.ts +3 -0
  144. package/dist/src/rate-limiting/metrics.js +44 -0
  145. package/dist/src/rate-limiting/request/simple-counting.d.ts +21 -0
  146. package/dist/src/rate-limiting/request/simple-counting.js +62 -0
  147. package/dist/src/test.d.ts +1 -0
  148. package/dist/src/test.js +6 -0
  149. package/dist/src/transports/batch-warming.d.ts +35 -0
  150. package/dist/src/transports/batch-warming.js +101 -0
  151. package/dist/src/transports/index.d.ts +70 -0
  152. package/dist/src/transports/index.js +87 -0
  153. package/dist/src/transports/metrics.d.ts +22 -0
  154. package/dist/src/transports/metrics.js +105 -0
  155. package/dist/src/transports/rest.d.ts +44 -0
  156. package/dist/src/transports/rest.js +131 -0
  157. package/dist/src/transports/util.d.ts +8 -0
  158. package/dist/src/transports/util.js +85 -0
  159. package/dist/src/transports/websocket.d.ts +80 -0
  160. package/dist/src/transports/websocket.js +166 -0
  161. package/dist/src/util/expiring-sorted-set.d.ts +21 -0
  162. package/dist/src/util/expiring-sorted-set.js +47 -0
  163. package/dist/src/util/index.d.ts +12 -0
  164. package/dist/src/util/index.js +35 -0
  165. package/dist/src/util/logger.d.ts +42 -0
  166. package/dist/src/util/logger.js +62 -0
  167. package/dist/src/util/request.d.ts +57 -0
  168. package/dist/src/util/request.js +2 -0
  169. package/dist/src/util/subscription-set/expiring-sorted-set.d.ts +22 -0
  170. package/dist/src/util/subscription-set/expiring-sorted-set.js +47 -0
  171. package/dist/src/util/subscription-set/subscription-set.d.ts +18 -0
  172. package/dist/src/util/subscription-set/subscription-set.js +19 -0
  173. package/dist/src/util/test-payload-loader.d.ts +25 -0
  174. package/dist/src/util/test-payload-loader.js +83 -0
  175. package/dist/src/validation/error.d.ts +50 -0
  176. package/dist/src/validation/error.js +79 -0
  177. package/dist/src/validation/index.d.ts +5 -0
  178. package/dist/src/validation/index.js +91 -0
  179. package/dist/src/validation/input-params.d.ts +15 -0
  180. package/dist/src/validation/input-params.js +30 -0
  181. package/dist/src/validation/override-functions.d.ts +3 -0
  182. package/dist/src/validation/override-functions.js +40 -0
  183. package/dist/src/validation/preset-tokens.json +23 -0
  184. package/dist/src/validation/validator.d.ts +47 -0
  185. package/dist/src/validation/validator.js +303 -0
  186. package/docker-compose.yaml +35 -0
  187. package/env.sh +54 -0
  188. package/env2.sh +55 -0
  189. package/package.json +5 -3
  190. package/publish.sh +0 -0
  191. package/src/adapter.ts +263 -0
  192. package/src/background-executor.ts +52 -0
  193. package/src/cache/factory.ts +26 -0
  194. package/src/cache/index.ts +258 -0
  195. package/src/cache/local.ts +73 -0
  196. package/src/cache/metrics.ts +112 -0
  197. package/src/cache/redis.ts +93 -0
  198. package/src/config/index.ts +517 -0
  199. package/src/config/provider-limits.ts +130 -0
  200. package/src/examples/bank-frick/README.MD +10 -0
  201. package/src/examples/bank-frick/accounts.ts +246 -0
  202. package/src/examples/bank-frick/config/index.ts +53 -0
  203. package/src/examples/bank-frick/index.ts +13 -0
  204. package/src/examples/bank-frick/types.d.ts +38 -0
  205. package/src/examples/bank-frick/util.ts +55 -0
  206. package/src/examples/coingecko/batch-warming.ts +78 -0
  207. package/src/examples/coingecko/index.ts +9 -0
  208. package/src/examples/coingecko/rest.ts +77 -0
  209. package/src/examples/ncfx/config/index.ts +12 -0
  210. package/src/examples/ncfx/index.ts +9 -0
  211. package/src/examples/ncfx/websocket.ts +99 -0
  212. package/src/index.ts +149 -0
  213. package/src/metrics/constants.ts +23 -0
  214. package/src/metrics/index.ts +115 -0
  215. package/src/metrics/util.ts +18 -0
  216. package/src/rate-limiting/background/fixed-frequency.ts +45 -0
  217. package/src/rate-limiting/index.ts +100 -0
  218. package/src/rate-limiting/metrics.ts +18 -0
  219. package/src/rate-limiting/request/simple-counting.ts +76 -0
  220. package/src/test.ts +5 -0
  221. package/src/transports/batch-warming.ts +122 -0
  222. package/src/transports/index.ts +152 -0
  223. package/src/transports/metrics.ts +95 -0
  224. package/src/transports/rest.ts +164 -0
  225. package/src/transports/util.ts +63 -0
  226. package/src/transports/websocket.ts +245 -0
  227. package/src/util/index.ts +22 -0
  228. package/src/util/logger.ts +69 -0
  229. package/src/util/request.ts +117 -0
  230. package/src/util/subscription-set/expiring-sorted-set.ts +54 -0
  231. package/src/util/subscription-set/subscription-set.ts +35 -0
  232. package/src/util/test-payload-loader.ts +87 -0
  233. package/src/validation/error.ts +116 -0
  234. package/src/validation/index.ts +110 -0
  235. package/src/validation/input-params.ts +45 -0
  236. package/src/validation/override-functions.ts +44 -0
  237. package/src/validation/preset-tokens.json +23 -0
  238. package/src/validation/validator.ts +384 -0
  239. package/test/adapter.test.ts +27 -0
  240. package/test/background-executor.test.ts +108 -0
  241. package/test/cache/cache-key.test.ts +114 -0
  242. package/test/cache/helper.ts +100 -0
  243. package/test/cache/local.test.ts +54 -0
  244. package/test/cache/redis.test.ts +89 -0
  245. package/test/correlation.test.ts +114 -0
  246. package/test/index.test.ts +37 -0
  247. package/test/metrics/feed-id.test.ts +38 -0
  248. package/test/metrics/helper.ts +14 -0
  249. package/test/metrics/labels.test.ts +36 -0
  250. package/test/metrics/metrics.test.ts +267 -0
  251. package/test/metrics/redis-metrics.test.ts +113 -0
  252. package/test/metrics/warmer-metrics.test.ts +192 -0
  253. package/test/metrics/ws-metrics.test.ts +225 -0
  254. package/test/rate-limit-config.test.ts +242 -0
  255. package/test/smoke.test.ts +166 -0
  256. package/test/transports/batch.test.ts +465 -0
  257. package/test/transports/rest.test.ts +242 -0
  258. package/test/transports/websocket.test.ts +183 -0
  259. package/test/tsconfig.json +5 -0
  260. package/test/util.ts +77 -0
  261. package/test/validation.test.ts +178 -0
  262. package/test-payload-fail.json +3 -0
  263. package/test-payload.js +22 -0
  264. package/test-payload.json +7 -0
  265. package/test.sh +20 -0
  266. package/test2.sh +2 -0
  267. package/tsconfig.json +25 -0
  268. package/typedoc.json +6 -0
  269. package/webpack.config.js +23 -0
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
@@ -0,0 +1,72 @@
1
+ {
2
+ "name": "@chainlink/external-adapter-framework",
3
+ "version": "0.0.6",
4
+ "main": "dist/index.js",
5
+ "license": "MIT",
6
+ "dependencies": {
7
+ "axios": "^0.27.2",
8
+ "fastify": "^3.29.0",
9
+ "ioredis": "^5.0.4",
10
+ "pino": "^7.9.2",
11
+ "prom-client": "13.2.0",
12
+ "ts-node": "^10.9.1",
13
+ "typescript": "^4.6.3",
14
+ "ws": "^8.5.0"
15
+ },
16
+ "scripts": {
17
+ "start": "ts-node src/test.ts",
18
+ "generate-docs": "typedoc src/**/*.ts",
19
+ "test": "LOG_LEVEL=error EA_PORT=0 c8 ava",
20
+ "test-debug": "LOG_LEVEL=trace DEBUG=true EA_PORT=0 c8 ava --verbose",
21
+ "build": "tsc",
22
+ "lint": "eslint ./src"
23
+ },
24
+ "devDependencies": {
25
+ "@sinonjs/fake-timers": "^9.1.2",
26
+ "@types/node": "^18.6.5",
27
+ "@types/sinonjs__fake-timers": "^8.1.2",
28
+ "@types/ws": "^8.5.3",
29
+ "@typescript-eslint/eslint-plugin": "^5.17.0",
30
+ "@typescript-eslint/parser": "^5.17.0",
31
+ "ava": "^4.2.0",
32
+ "c8": "^7.11.2",
33
+ "eslint": "^8.14.0",
34
+ "eslint-config-prettier": "^8.5.0",
35
+ "eslint-plugin-tsdoc": "^0.2.16",
36
+ "mock-socket": "^9.1.3",
37
+ "nock": "^13.2.4",
38
+ "pino-pretty": "^7.6.0",
39
+ "prettier": "^2.6.1",
40
+ "ts-loader": "^9.3.1",
41
+ "typedoc": "^0.22.15",
42
+ "webpack": "^5.74.0",
43
+ "webpack-cli": "^4.10.0"
44
+ },
45
+ "prettier": {
46
+ "semi": false,
47
+ "singleQuote": true,
48
+ "printWidth": 100,
49
+ "endOfLine": "auto",
50
+ "trailingComma": "all",
51
+ "arrowParens": "always"
52
+ },
53
+ "optionalDependencies": {
54
+ "bufferutil": "^4.0.6",
55
+ "utf-8-validate": "^5.0.9"
56
+ },
57
+ "ava": {
58
+ "files": [
59
+ "test/**/*.test.ts"
60
+ ],
61
+ "extensions": [
62
+ "ts"
63
+ ],
64
+ "require": [
65
+ "ts-node/register"
66
+ ],
67
+ "workerThreads": false,
68
+ "environmentVariables": {
69
+ "METRICS_ENABLED": "false"
70
+ }
71
+ }
72
+ }
@@ -0,0 +1,11 @@
1
+ import { AdapterRateLimitTier, BackgroundExecuteRateLimiter } from '..';
2
+ import { AdapterEndpoint } from '../../adapter';
3
+ import { SettingsMap } from '../../config';
4
+ export declare const DEFAULT_SHARED_MS_BETWEEN_REQUESTS = 5000;
5
+ export declare class FixedFrequencyRateLimiter implements BackgroundExecuteRateLimiter {
6
+ msBetweenRequestsMap: {
7
+ [endpointName: string]: number;
8
+ };
9
+ initialize<Params, Result, CustomSettings extends SettingsMap>(endpoints: AdapterEndpoint<Params, Result, CustomSettings>[], limits?: AdapterRateLimitTier): this;
10
+ msUntilNextExecution(endpointName: string): number;
11
+ }
@@ -0,0 +1,35 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.FixedFrequencyRateLimiter = exports.DEFAULT_SHARED_MS_BETWEEN_REQUESTS = void 0;
4
+ const __1 = require("..");
5
+ const util_1 = require("../../util");
6
+ const logger = (0, util_1.makeLogger)('FixedFrequencyRateLimiter');
7
+ exports.DEFAULT_SHARED_MS_BETWEEN_REQUESTS = 5000;
8
+ class FixedFrequencyRateLimiter {
9
+ constructor() {
10
+ this.msBetweenRequestsMap = {};
11
+ }
12
+ initialize(endpoints, limits) {
13
+ // Translate the hourly limit into reqs per minute
14
+ let sharedMsBetweenRequests = 1000 / (0, __1.consolidateTierLimits)(limits);
15
+ // If there is no limit set, we use some reasonable number
16
+ if (!limits?.rateLimit1h && !limits?.rateLimit1m && !limits?.rateLimit1s) {
17
+ // 5s period for all seems good
18
+ sharedMsBetweenRequests = exports.DEFAULT_SHARED_MS_BETWEEN_REQUESTS;
19
+ }
20
+ logger.debug('Using fixed frequency batch rate limiting');
21
+ for (const endpoint of endpoints) {
22
+ if (endpoint.rateLimiting?.allocationPercentage == null) {
23
+ throw new Error(`Allocation percentage for endpoint "${endpoint.name}" is null`);
24
+ }
25
+ this.msBetweenRequestsMap[endpoint.name] =
26
+ (sharedMsBetweenRequests / endpoint.rateLimiting?.allocationPercentage) * 100;
27
+ logger.debug(`Endpoint [${endpoint.name}]: ${this.msBetweenRequestsMap[endpoint.name] / 1000}s between requests`);
28
+ }
29
+ return this;
30
+ }
31
+ msUntilNextExecution(endpointName) {
32
+ return this.msBetweenRequestsMap[endpointName];
33
+ }
34
+ }
35
+ exports.FixedFrequencyRateLimiter = FixedFrequencyRateLimiter;
@@ -0,0 +1,55 @@
1
+ import { AdapterEndpoint } from '../adapter';
2
+ import { SettingsMap } from '../config';
3
+ export * from './request/simple-counting';
4
+ export * from './background/fixed-frequency';
5
+ export interface AdapterRateLimitTier {
6
+ rateLimit1s?: number;
7
+ rateLimit1m?: number;
8
+ rateLimit1h?: number;
9
+ note?: string;
10
+ }
11
+ /**
12
+ * Common interface for all RateLimiter classes to implement
13
+ */
14
+ export interface RateLimiter {
15
+ /**
16
+ * Method to ensure all RateLimiters can be initialized in the same manner.
17
+ *
18
+ * @param limits - settings for how much throughput to allow for the Adapter
19
+ * @param endpoints - list of adapter endpoints
20
+ */
21
+ initialize<Params, Result, CustomSettings extends SettingsMap>(endpoints: AdapterEndpoint<Params, Result, CustomSettings>[], limits: AdapterRateLimitTier): this;
22
+ }
23
+ /**
24
+ * RequestRateLimiters perform checks agains imminent outbound requests for any transport.
25
+ */
26
+ export interface RequestRateLimiter extends RateLimiter {
27
+ /**
28
+ * This method will check using whatever strategy is implemented to determine if
29
+ * this request can be processed. If so, it returns true; if not, returns false.
30
+ */
31
+ isUnderLimits(): boolean;
32
+ }
33
+ /**
34
+ * BackgroundExecuteFrequencyRateLimiters will implement custom logic to calculate
35
+ * the period of time to wait between background executions for a transport.
36
+ */
37
+ export interface BackgroundExecuteRateLimiter extends RateLimiter {
38
+ msUntilNextExecution(endpointName: string): number;
39
+ }
40
+ /**
41
+ * This method will convert all possible settings for a rate limit tier and
42
+ * convert them all to requests per second, returning the lowest one
43
+ *
44
+ * @param limits - the rate limit tier set for the adapter
45
+ * @returns the most restrictive of the set options, in requests per second
46
+ */
47
+ export declare const consolidateTierLimits: (limits?: AdapterRateLimitTier) => number;
48
+ /**
49
+ * Validates rate limiting tiers specified for the adapter, and returns the one to use.
50
+ *
51
+ * @param tiers - the adapter config listing the different available API tiers
52
+ * @param selectedTier - chosen API tier from settings, if present
53
+ * @returns the specified API tier, or a default one if none are specified
54
+ */
55
+ export declare const getRateLimitingTier: (tiers?: Record<string, AdapterRateLimitTier>, selectedTier?: string) => AdapterRateLimitTier | undefined;
@@ -0,0 +1,63 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
+ };
16
+ Object.defineProperty(exports, "__esModule", { value: true });
17
+ exports.getRateLimitingTier = exports.consolidateTierLimits = void 0;
18
+ __exportStar(require("./request/simple-counting"), exports);
19
+ __exportStar(require("./background/fixed-frequency"), exports);
20
+ /**
21
+ * This method will convert all possible settings for a rate limit tier and
22
+ * convert them all to requests per second, returning the lowest one
23
+ *
24
+ * @param limits - the rate limit tier set for the adapter
25
+ * @returns the most restrictive of the set options, in requests per second
26
+ */
27
+ const consolidateTierLimits = (limits) => {
28
+ const perHourLimit = (limits?.rateLimit1h || Infinity) / (60 * 60);
29
+ const perMinuteLimit = (limits?.rateLimit1m || Infinity) / 60;
30
+ const perSecondLimit = limits?.rateLimit1s || Infinity;
31
+ return Math.min(perHourLimit, perMinuteLimit, perSecondLimit);
32
+ };
33
+ exports.consolidateTierLimits = consolidateTierLimits;
34
+ /**
35
+ * Validates rate limiting tiers specified for the adapter, and returns the one to use.
36
+ *
37
+ * @param tiers - the adapter config listing the different available API tiers
38
+ * @param selectedTier - chosen API tier from settings, if present
39
+ * @returns the specified API tier, or a default one if none are specified
40
+ */
41
+ const getRateLimitingTier = (tiers, selectedTier) => {
42
+ if (!tiers) {
43
+ return;
44
+ }
45
+ // Check that if the tiers object is defined, it has values
46
+ if (Object.values(tiers).length === 0) {
47
+ throw new Error(`The tiers object is defined, but has no entries`);
48
+ }
49
+ // Check that the tier set in the AdapterConfig is a valid one
50
+ if (selectedTier && !tiers[selectedTier]) {
51
+ const validTiersString = Object.keys(tiers)
52
+ .map((t) => `"${t}"`)
53
+ .join(', ');
54
+ throw new Error(`The selected rate limit tier "${selectedTier}" is not valid (can be one of ${validTiersString})`);
55
+ }
56
+ if (!selectedTier) {
57
+ // Sort the tiers by most to least restrictive
58
+ const sortedTiers = Object.values(tiers).sort((t1, t2) => (0, exports.consolidateTierLimits)(t1) - (0, exports.consolidateTierLimits)(t2));
59
+ return sortedTiers[0];
60
+ }
61
+ return tiers[selectedTier];
62
+ };
63
+ exports.getRateLimitingTier = getRateLimitingTier;
@@ -0,0 +1,3 @@
1
+ import * as client from 'prom-client';
2
+ export declare const retrieveCost: <ProviderResponseBody>(data: ProviderResponseBody) => number;
3
+ export declare const rateLimitCreditsSpentTotal: client.Counter<"feed_id" | "participant_id">;
@@ -0,0 +1,44 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || function (mod) {
19
+ if (mod && mod.__esModule) return mod;
20
+ var result = {};
21
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
+ __setModuleDefault(result, mod);
23
+ return result;
24
+ };
25
+ Object.defineProperty(exports, "__esModule", { value: true });
26
+ exports.rateLimitCreditsSpentTotal = exports.retrieveCost = void 0;
27
+ const client = __importStar(require("prom-client"));
28
+ // Retrieve cost field from response if exists
29
+ // If not return default cost of 1
30
+ const retrieveCost = (data) => {
31
+ const cost = data['cost'];
32
+ if (typeof cost === 'number' || typeof cost === 'string') {
33
+ return Number(cost);
34
+ }
35
+ else {
36
+ return 1;
37
+ }
38
+ };
39
+ exports.retrieveCost = retrieveCost;
40
+ exports.rateLimitCreditsSpentTotal = new client.Counter({
41
+ name: 'rate_limit_credits_spent_total',
42
+ help: 'The number of data provider credits the adapter is consuming',
43
+ labelNames: ['participant_id', 'feed_id'],
44
+ });
@@ -0,0 +1,21 @@
1
+ import { AdapterRateLimitTier, RateLimiter } from '..';
2
+ import { AdapterEndpoint } from '../../adapter';
3
+ import { SettingsMap } from '../../config';
4
+ /**
5
+ * This rate limiter is the simplest stateful option.
6
+ * On startup, it'll compare the different thresholds for each tier, calculate them all
7
+ * in the finest window we'll use (seconds), and use the most restrictive one.
8
+ * This is so if the EA were to restart, we don't need to worry about persisting state
9
+ * for things like daily quotas. The downside is that this does not work well for bursty
10
+ * loads or spikes, in cases where e.g. the per second limit is high but daily quotas low.
11
+ */
12
+ export declare class SimpleCountingRateLimiter implements RateLimiter {
13
+ latestSecondInterval: number;
14
+ requestsThisSecond: number;
15
+ latestMinuteInterval: number;
16
+ requestsThisMinute: number;
17
+ perSecondLimit: number;
18
+ perMinuteLimit: number;
19
+ initialize<Params, Result, CustomSettings extends SettingsMap>(endpoints: AdapterEndpoint<Params, Result, CustomSettings>[], limits?: AdapterRateLimitTier): this;
20
+ isUnderLimits(): boolean;
21
+ }
@@ -0,0 +1,62 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.SimpleCountingRateLimiter = void 0;
4
+ const util_1 = require("../../util");
5
+ const logger = (0, util_1.makeLogger)('SimpleCountingRateLimiter');
6
+ /**
7
+ * This rate limiter is the simplest stateful option.
8
+ * On startup, it'll compare the different thresholds for each tier, calculate them all
9
+ * in the finest window we'll use (seconds), and use the most restrictive one.
10
+ * This is so if the EA were to restart, we don't need to worry about persisting state
11
+ * for things like daily quotas. The downside is that this does not work well for bursty
12
+ * loads or spikes, in cases where e.g. the per second limit is high but daily quotas low.
13
+ */
14
+ class SimpleCountingRateLimiter {
15
+ constructor() {
16
+ this.latestSecondInterval = 0;
17
+ this.requestsThisSecond = 0;
18
+ this.latestMinuteInterval = 0;
19
+ this.requestsThisMinute = 0;
20
+ }
21
+ initialize(endpoints, limits) {
22
+ // Translate the hourly limit into reqs per minute
23
+ const perHourLimit = (limits?.rateLimit1h || Infinity) / 60;
24
+ this.perMinuteLimit = Math.min(limits?.rateLimit1m || Infinity, perHourLimit);
25
+ this.perSecondLimit = limits?.rateLimit1s || Infinity;
26
+ logger.debug(`Using rate limiting settings: perMinute = ${this.perMinuteLimit} | perSecond: = ${this.perSecondLimit}`);
27
+ return this;
28
+ }
29
+ isUnderLimits() {
30
+ // If the limit is set to infinity, there was no tier limit specified
31
+ if (this.perSecondLimit === Infinity && this.perMinuteLimit === Infinity) {
32
+ return true;
33
+ }
34
+ const now = Date.now();
35
+ const nearestSecondInterval = Math.floor(now / 1000);
36
+ const nearestMinuteInterval = Math.floor(now / (1000 * 60));
37
+ // This should always run to completion, even if it doesn't look atomic; therefore the
38
+ // Ops should be "thread safe". Thank JS and its infinite single threaded dumbness.
39
+ if (nearestSecondInterval !== this.latestSecondInterval) {
40
+ logger.trace(`Clearing latest second interval, # of requests logged was: ${this.requestsThisSecond} `);
41
+ this.latestSecondInterval = nearestSecondInterval;
42
+ this.requestsThisSecond = 0;
43
+ }
44
+ if (nearestMinuteInterval !== this.latestMinuteInterval) {
45
+ logger.trace(`Clearing latest second minute, # of requests logged was: ${this.requestsThisMinute} `);
46
+ this.latestMinuteInterval = nearestMinuteInterval;
47
+ this.requestsThisMinute = 0;
48
+ }
49
+ if (this.requestsThisSecond < this.perSecondLimit &&
50
+ this.requestsThisMinute < this.perMinuteLimit) {
51
+ logger.trace('Request under limits, counting +1');
52
+ this.requestsThisSecond++;
53
+ this.requestsThisMinute++;
54
+ return true;
55
+ }
56
+ else {
57
+ logger.trace('Requests seen this interval are above limits');
58
+ return false;
59
+ }
60
+ }
61
+ }
62
+ exports.SimpleCountingRateLimiter = SimpleCountingRateLimiter;
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,6 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ const _1 = require(".");
4
+ const bank_frick_1 = require("./examples/bank-frick");
5
+ // Start sample adapter
6
+ (0, _1.expose)(bank_frick_1.adapter);
@@ -0,0 +1,35 @@
1
+ import { AxiosRequestConfig, AxiosResponse } from 'axios';
2
+ import { Cache } from '../cache';
3
+ import { AdapterConfig, SettingsMap } from '../config';
4
+ import { BackgroundExecuteRateLimiter } from '../rate-limiting';
5
+ import { SubscriptionSet } from '../util';
6
+ import { AdapterRequest, ProviderResult } from '../util/request';
7
+ import { Transport } from './';
8
+ import { AdapterContext, AdapterDependencies } from '../adapter';
9
+ /**
10
+ * Transport implementation that takes incoming batches requests and keeps a warm cache of values.
11
+ * Within the setup function, adapter params are added to an set that also keeps track and expires values.
12
+ * In the background execute, the list of non-expired items in the set is fetched.
13
+ * Then, the list is passed through the `prepareRequest` function, that returns an AxiosRequestConfig.
14
+ * The Data Provider response is, they are passed through the `parseResponse` function to create a [[CacheEntry]] list.
15
+ * Finally, the items in that [[CacheEntry]] list are set in the Cache so the Adapter can fetch values from there.
16
+ *
17
+ * @typeParam AdapterParams - interface for the adapter request body
18
+ * @typeParam ProviderRequestBody - interface for the body of the request to the Data Provider
19
+ * @typeParam ProviderResponseBody - interface for the body of the Data Provider's response
20
+ */
21
+ export declare class BatchWarmingTransport<AdapterParams, ProviderRequestBody, ProviderResponseBody, CustomSettings extends SettingsMap> implements Transport<AdapterParams, ProviderResponseBody, CustomSettings> {
22
+ private config;
23
+ cache: Cache;
24
+ rateLimiter: BackgroundExecuteRateLimiter;
25
+ subscriptionSet: SubscriptionSet<AdapterParams>;
26
+ WARMER_ACTIVE: boolean;
27
+ constructor(config: {
28
+ prepareRequest: (params: AdapterParams[], context: AdapterContext<CustomSettings>) => AxiosRequestConfig<ProviderRequestBody>;
29
+ parseResponse: (res: AxiosResponse<ProviderResponseBody>, context: AdapterContext<CustomSettings>) => ProviderResult<AdapterParams>[];
30
+ });
31
+ initialize(dependencies: AdapterDependencies): Promise<void>;
32
+ hasBeenSetUp(req: AdapterRequest<AdapterParams>): Promise<boolean>;
33
+ setup(req: AdapterRequest<AdapterParams>, config: AdapterConfig<CustomSettings>): Promise<void>;
34
+ backgroundExecute(context: AdapterContext<CustomSettings>): Promise<number>;
35
+ }
@@ -0,0 +1,101 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || function (mod) {
19
+ if (mod && mod.__esModule) return mod;
20
+ var result = {};
21
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
+ __setModuleDefault(result, mod);
23
+ return result;
24
+ };
25
+ Object.defineProperty(exports, "__esModule", { value: true });
26
+ exports.BatchWarmingTransport = void 0;
27
+ const util_1 = require("../util");
28
+ const _1 = require("./");
29
+ const util_2 = require("./util");
30
+ const rateLimitMetrics = __importStar(require("../rate-limiting/metrics"));
31
+ const cacheMetrics = __importStar(require("../cache/metrics"));
32
+ const WARMUP_BATCH_REQUEST_ID = '9002';
33
+ const logger = (0, util_1.makeLogger)('BatchWarmingTransport');
34
+ /**
35
+ * Transport implementation that takes incoming batches requests and keeps a warm cache of values.
36
+ * Within the setup function, adapter params are added to an set that also keeps track and expires values.
37
+ * In the background execute, the list of non-expired items in the set is fetched.
38
+ * Then, the list is passed through the `prepareRequest` function, that returns an AxiosRequestConfig.
39
+ * The Data Provider response is, they are passed through the `parseResponse` function to create a [[CacheEntry]] list.
40
+ * Finally, the items in that [[CacheEntry]] list are set in the Cache so the Adapter can fetch values from there.
41
+ *
42
+ * @typeParam AdapterParams - interface for the adapter request body
43
+ * @typeParam ProviderRequestBody - interface for the body of the request to the Data Provider
44
+ * @typeParam ProviderResponseBody - interface for the body of the Data Provider's response
45
+ */
46
+ class BatchWarmingTransport {
47
+ constructor(config) {
48
+ this.config = config;
49
+ // Flag used to track whether the warmer has moved from having no entries to having some and vice versa
50
+ // Used for recording the cache warmer active metrics accurately
51
+ this.WARMER_ACTIVE = false;
52
+ }
53
+ async initialize(dependencies) {
54
+ this.cache = dependencies.cache;
55
+ this.rateLimiter = dependencies.backgroundExecuteRateLimiter;
56
+ this.subscriptionSet = dependencies.subscriptionSetFactory.buildSet();
57
+ }
58
+ async hasBeenSetUp(req) {
59
+ return !!(await this.subscriptionSet.get(req.requestContext.cacheKey));
60
+ }
61
+ async setup(req, config) {
62
+ logger.debug(`Adding entry to batch warming set: [${req.requestContext.cacheKey}] = ${req.requestContext.data}`);
63
+ await this.subscriptionSet.add(req.requestContext.cacheKey, req.requestContext.data, config.WARMUP_SUBSCRIPTION_TTL);
64
+ }
65
+ async backgroundExecute(context) {
66
+ logger.debug('Starting background execute');
67
+ const entries = await this.subscriptionSet.getAll();
68
+ if (!entries.length) {
69
+ logger.debug('No entries in batch warming set, skipping');
70
+ if (this.WARMER_ACTIVE) {
71
+ // Decrement count when warmer changed from having entries to having none
72
+ cacheMetrics.cacheWarmerCount.labels({ isBatched: 'true' }).dec();
73
+ this.WARMER_ACTIVE = false;
74
+ }
75
+ return this.rateLimiter.msUntilNextExecution(context.adapterEndpoint.name);
76
+ }
77
+ else if (this.WARMER_ACTIVE === false) {
78
+ // Increment count when warmer changed from having no entries to having some
79
+ cacheMetrics.cacheWarmerCount.labels({ isBatched: 'true' }).inc();
80
+ this.WARMER_ACTIVE = true;
81
+ }
82
+ const request = this.config.prepareRequest(entries, context);
83
+ logger.trace('Sending request to data provider...');
84
+ const providerResponse = await (0, util_2.axiosRequest)(request);
85
+ logger.debug(`Got response from provider, parsing (raw body: ${providerResponse.data})`);
86
+ const results = this.config.parseResponse(providerResponse, context);
87
+ const adapterResponses = (0, _1.buildCacheEntriesFromResults)(results, context);
88
+ logger.debug('Setting adapter responses in cache');
89
+ await this.cache.setMany(adapterResponses, context.adapterConfig.CACHE_MAX_AGE);
90
+ // Record cost of data provider call
91
+ const cost = rateLimitMetrics.retrieveCost(providerResponse.data);
92
+ rateLimitMetrics.rateLimitCreditsSpentTotal
93
+ .labels({
94
+ feed_id: 'N/A',
95
+ participant_id: WARMUP_BATCH_REQUEST_ID,
96
+ })
97
+ .inc(cost);
98
+ return this.rateLimiter.msUntilNextExecution(context.adapterEndpoint.name);
99
+ }
100
+ }
101
+ exports.BatchWarmingTransport = BatchWarmingTransport;