@chainlink/external-adapter-framework 0.3.9 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. package/README.md +102 -0
  2. package/package.json +42 -50
  3. package/adapter/basic.d.ts +0 -86
  4. package/adapter/basic.js +0 -325
  5. package/adapter/basic.js.map +0 -1
  6. package/adapter/endpoint.d.ts +0 -17
  7. package/adapter/endpoint.js +0 -19
  8. package/adapter/endpoint.js.map +0 -1
  9. package/adapter/index.d.ts +0 -4
  10. package/adapter/index.js +0 -21
  11. package/adapter/index.js.map +0 -1
  12. package/adapter/price.d.ts +0 -75
  13. package/adapter/price.js +0 -88
  14. package/adapter/price.js.map +0 -1
  15. package/adapter/types.d.ts +0 -118
  16. package/adapter/types.js +0 -3
  17. package/adapter/types.js.map +0 -1
  18. package/background-executor.d.ts +0 -9
  19. package/background-executor.js +0 -97
  20. package/background-executor.js.map +0 -1
  21. package/cache/factory.d.ts +0 -6
  22. package/cache/factory.js +0 -24
  23. package/cache/factory.js.map +0 -1
  24. package/cache/index.d.ts +0 -87
  25. package/cache/index.js +0 -133
  26. package/cache/index.js.map +0 -1
  27. package/cache/local.d.ts +0 -23
  28. package/cache/local.js +0 -84
  29. package/cache/local.js.map +0 -1
  30. package/cache/metrics.d.ts +0 -27
  31. package/cache/metrics.js +0 -121
  32. package/cache/metrics.js.map +0 -1
  33. package/cache/redis.d.ts +0 -16
  34. package/cache/redis.js +0 -101
  35. package/cache/redis.js.map +0 -1
  36. package/config/index.d.ts +0 -298
  37. package/config/index.js +0 -359
  38. package/config/index.js.map +0 -1
  39. package/config/provider-limits.d.ts +0 -27
  40. package/config/provider-limits.js +0 -75
  41. package/config/provider-limits.js.map +0 -1
  42. package/examples/bank-frick/accounts.d.ts +0 -52
  43. package/examples/bank-frick/accounts.js +0 -190
  44. package/examples/bank-frick/accounts.js.map +0 -1
  45. package/examples/bank-frick/config/index.d.ts +0 -17
  46. package/examples/bank-frick/config/index.js +0 -55
  47. package/examples/bank-frick/config/index.js.map +0 -1
  48. package/examples/bank-frick/index.d.ts +0 -16
  49. package/examples/bank-frick/index.js +0 -16
  50. package/examples/bank-frick/index.js.map +0 -1
  51. package/examples/bank-frick/util.d.ts +0 -4
  52. package/examples/bank-frick/util.js +0 -40
  53. package/examples/bank-frick/util.js.map +0 -1
  54. package/examples/coingecko/src/config/index.d.ts +0 -2
  55. package/examples/coingecko/src/config/index.js +0 -6
  56. package/examples/coingecko/src/config/index.js.map +0 -1
  57. package/examples/coingecko/src/config/overrides.json +0 -10825
  58. package/examples/coingecko/src/crypto-utils.d.ts +0 -48
  59. package/examples/coingecko/src/crypto-utils.js +0 -60
  60. package/examples/coingecko/src/crypto-utils.js.map +0 -1
  61. package/examples/coingecko/src/endpoint/coins.d.ts +0 -9
  62. package/examples/coingecko/src/endpoint/coins.js +0 -37
  63. package/examples/coingecko/src/endpoint/coins.js.map +0 -1
  64. package/examples/coingecko/src/endpoint/crypto-marketcap.d.ts +0 -3
  65. package/examples/coingecko/src/endpoint/crypto-marketcap.js +0 -30
  66. package/examples/coingecko/src/endpoint/crypto-marketcap.js.map +0 -1
  67. package/examples/coingecko/src/endpoint/crypto-volume.d.ts +0 -3
  68. package/examples/coingecko/src/endpoint/crypto-volume.js +0 -30
  69. package/examples/coingecko/src/endpoint/crypto-volume.js.map +0 -1
  70. package/examples/coingecko/src/endpoint/crypto.d.ts +0 -2
  71. package/examples/coingecko/src/endpoint/crypto.js +0 -28
  72. package/examples/coingecko/src/endpoint/crypto.js.map +0 -1
  73. package/examples/coingecko/src/endpoint/dominance.d.ts +0 -3
  74. package/examples/coingecko/src/endpoint/dominance.js +0 -28
  75. package/examples/coingecko/src/endpoint/dominance.js.map +0 -1
  76. package/examples/coingecko/src/endpoint/global-marketcap.d.ts +0 -3
  77. package/examples/coingecko/src/endpoint/global-marketcap.js +0 -28
  78. package/examples/coingecko/src/endpoint/global-marketcap.js.map +0 -1
  79. package/examples/coingecko/src/endpoint/index.d.ts +0 -6
  80. package/examples/coingecko/src/endpoint/index.js +0 -16
  81. package/examples/coingecko/src/endpoint/index.js.map +0 -1
  82. package/examples/coingecko/src/global-utils.d.ts +0 -27
  83. package/examples/coingecko/src/global-utils.js +0 -47
  84. package/examples/coingecko/src/global-utils.js.map +0 -1
  85. package/examples/coingecko/src/index.d.ts +0 -4
  86. package/examples/coingecko/src/index.js +0 -19
  87. package/examples/coingecko/src/index.js.map +0 -1
  88. package/examples/coingecko-old/batch-warming.d.ts +0 -7
  89. package/examples/coingecko-old/batch-warming.js +0 -54
  90. package/examples/coingecko-old/batch-warming.js.map +0 -1
  91. package/examples/coingecko-old/index.d.ts +0 -2
  92. package/examples/coingecko-old/index.js +0 -12
  93. package/examples/coingecko-old/index.js.map +0 -1
  94. package/examples/coingecko-old/rest.d.ts +0 -12
  95. package/examples/coingecko-old/rest.js +0 -55
  96. package/examples/coingecko-old/rest.js.map +0 -1
  97. package/examples/cryptocompare/src/config/index.d.ts +0 -2
  98. package/examples/cryptocompare/src/config/index.js +0 -6
  99. package/examples/cryptocompare/src/config/index.js.map +0 -1
  100. package/examples/cryptocompare/src/endpoints/crypto.d.ts +0 -28
  101. package/examples/cryptocompare/src/endpoints/crypto.js +0 -54
  102. package/examples/cryptocompare/src/endpoints/crypto.js.map +0 -1
  103. package/examples/cryptocompare/src/endpoints/index.d.ts +0 -1
  104. package/examples/cryptocompare/src/endpoints/index.js +0 -6
  105. package/examples/cryptocompare/src/endpoints/index.js.map +0 -1
  106. package/examples/cryptocompare/src/index.d.ts +0 -4
  107. package/examples/cryptocompare/src/index.js +0 -14
  108. package/examples/cryptocompare/src/index.js.map +0 -1
  109. package/examples/genesis/config/index.d.ts +0 -7
  110. package/examples/genesis/config/index.js +0 -11
  111. package/examples/genesis/config/index.js.map +0 -1
  112. package/examples/genesis/index.d.ts +0 -2
  113. package/examples/genesis/index.js +0 -13
  114. package/examples/genesis/index.js.map +0 -1
  115. package/examples/genesis/sseStream.d.ts +0 -16
  116. package/examples/genesis/sseStream.js +0 -149
  117. package/examples/genesis/sseStream.js.map +0 -1
  118. package/index.d.ts +0 -17
  119. package/index.js +0 -174
  120. package/index.js.map +0 -1
  121. package/metrics/constants.d.ts +0 -16
  122. package/metrics/constants.js +0 -26
  123. package/metrics/constants.js.map +0 -1
  124. package/metrics/index.d.ts +0 -19
  125. package/metrics/index.js +0 -139
  126. package/metrics/index.js.map +0 -1
  127. package/metrics/util.d.ts +0 -7
  128. package/metrics/util.js +0 -10
  129. package/metrics/util.js.map +0 -1
  130. package/rate-limiting/background/fixed-frequency.d.ts +0 -11
  131. package/rate-limiting/background/fixed-frequency.js +0 -36
  132. package/rate-limiting/background/fixed-frequency.js.map +0 -1
  133. package/rate-limiting/index.d.ts +0 -56
  134. package/rate-limiting/index.js +0 -86
  135. package/rate-limiting/index.js.map +0 -1
  136. package/rate-limiting/metrics.d.ts +0 -3
  137. package/rate-limiting/metrics.js +0 -45
  138. package/rate-limiting/metrics.js.map +0 -1
  139. package/rate-limiting/request/simple-counting.d.ts +0 -21
  140. package/rate-limiting/request/simple-counting.js +0 -63
  141. package/rate-limiting/request/simple-counting.js.map +0 -1
  142. package/transports/batch-warming.d.ts +0 -35
  143. package/transports/batch-warming.js +0 -138
  144. package/transports/batch-warming.js.map +0 -1
  145. package/transports/index.d.ts +0 -87
  146. package/transports/index.js +0 -56
  147. package/transports/index.js.map +0 -1
  148. package/transports/metrics.d.ts +0 -25
  149. package/transports/metrics.js +0 -122
  150. package/transports/metrics.js.map +0 -1
  151. package/transports/rest.d.ts +0 -51
  152. package/transports/rest.js +0 -133
  153. package/transports/rest.js.map +0 -1
  154. package/transports/routing.d.ts +0 -23
  155. package/transports/routing.js +0 -52
  156. package/transports/routing.js.map +0 -1
  157. package/transports/sse.d.ts +0 -40
  158. package/transports/sse.js +0 -92
  159. package/transports/sse.js.map +0 -1
  160. package/transports/util.d.ts +0 -9
  161. package/transports/util.js +0 -87
  162. package/transports/util.js.map +0 -1
  163. package/transports/websocket.d.ts +0 -81
  164. package/transports/websocket.js +0 -165
  165. package/transports/websocket.js.map +0 -1
  166. package/util/censor/censor-list.d.ts +0 -9
  167. package/util/censor/censor-list.js +0 -13
  168. package/util/censor/censor-list.js.map +0 -1
  169. package/util/index.d.ts +0 -13
  170. package/util/index.js +0 -38
  171. package/util/index.js.map +0 -1
  172. package/util/logger.d.ts +0 -49
  173. package/util/logger.js +0 -123
  174. package/util/logger.js.map +0 -1
  175. package/util/request.d.ts +0 -102
  176. package/util/request.js +0 -3
  177. package/util/request.js.map +0 -1
  178. package/util/subscription-set/expiring-sorted-set.d.ts +0 -21
  179. package/util/subscription-set/expiring-sorted-set.js +0 -36
  180. package/util/subscription-set/expiring-sorted-set.js.map +0 -1
  181. package/util/subscription-set/redis-sorted-set.d.ts +0 -9
  182. package/util/subscription-set/redis-sorted-set.js +0 -29
  183. package/util/subscription-set/redis-sorted-set.js.map +0 -1
  184. package/util/subscription-set/subscription-set.d.ts +0 -19
  185. package/util/subscription-set/subscription-set.js +0 -28
  186. package/util/subscription-set/subscription-set.js.map +0 -1
  187. package/util/test-payload-loader.d.ts +0 -26
  188. package/util/test-payload-loader.js +0 -85
  189. package/util/test-payload-loader.js.map +0 -1
  190. package/validation/error.d.ts +0 -48
  191. package/validation/error.js +0 -78
  192. package/validation/error.js.map +0 -1
  193. package/validation/index.d.ts +0 -5
  194. package/validation/index.js +0 -91
  195. package/validation/index.js.map +0 -1
  196. package/validation/input-params.d.ts +0 -14
  197. package/validation/input-params.js +0 -3
  198. package/validation/input-params.js.map +0 -1
  199. package/validation/input-validator.d.ts +0 -16
  200. package/validation/input-validator.js +0 -123
  201. package/validation/input-validator.js.map +0 -1
package/README.md ADDED
@@ -0,0 +1,102 @@
1
+ # EA Framework v3
2
+
3
+ External adapters and its core framework serve as middleware to facilitate connections between Chainlink Nodes and Data Providers (DP). Their main purpose is threefold:
4
+
5
+ - Abstract provider specific details, specifically:
6
+ - Transport (REST, WebSockets, RPC, SSE, etc.)
7
+ - Authentication (login flows, keys)
8
+ - Accept normalized request payloads and translate into the provider's interface (this also includes things like symbols/tickers)
9
+ - Parse provider responses into the desired data points (e.g. price from crypto endpoint)
10
+ - Make the overall system more efficient by using as few resources (e.g. API credits, networking traffic, CPU usage) as possible to fetch data, achieved by features like:
11
+ - Caching: since DPs update data at various times and a request to their API incurs a certain latency, EAs keep a cache of values to
12
+ - Provide responses to the Chainlink Nodes as quickly as possible
13
+ - Communicate with DPs only when necessary
14
+ - Cache Warming: in order to make as many of the requests to EAs be fulfilled from the cache, EAs fetch values from DPs asynchronously to keep the local ones fresh.
15
+ - Batching: the CL Node requests feeds individually, but it's common for DPs to provide batch endpoints to get many data points at once. With Batching, EAs keep track of incoming requests and consolidate them into one batched request.
16
+ - Rate limiting: the EA framework additionally checks request rates (frequency) to make sure they fall within acceptable limits (like quotas for the DP's API, or adjusting based on the NOP's API tier).
17
+ - Perform off chain computations (think aggregations, indexing, or any sort of data processing)
18
+
19
+ By providing a framework that gives users easy access to those features, we reduce the complexity required for Nodes to communicate with DPs since by using EAs there is only one standardized way to do so, while at the same time optimizing said communication so it's as resource efficient as possible. It also makes internal and external development easier and faster, by serving as a strict guideline to implement and add new providers.
20
+
21
+ ## Qs
22
+
23
+ - Store entire response in cache?
24
+ - Check for valid result in adapter response?
25
+
26
+ ## Env vars
27
+
28
+ These are all existing env vars, marked DONE if they have been ported to this version, or N/A if the new version disregards them altogether.
29
+
30
+ | Name | State | Comments |
31
+ | :----------------------------------------- | :---: | :----------------------------------------------------------- |
32
+ | ADAPTER_URL | - | |
33
+ | API_ENDPOINT | √ | |
34
+ | API_KEY | √ | |
35
+ | API_TIMEOUT | √ | Sets timeout for axios request |
36
+ | API_VERBOSE | √ | Does not return verbose on cache hit |
37
+ | BASE_URL | √ | |
38
+ | CACHE_ENABLED | N/A | Cache is integral to this fw, not including this for now |
39
+ | CACHE_KEY_IGNORED_PROPS | N/A | Not applicable after feed ID generation logic change |
40
+ | CACHE_KEY_GROUP | N/A | Grouping cache data is not needed in v3 |
41
+ | CACHE_MAX_AGE | √ | |
42
+ | CACHE_MAX_ITEMS | - | Should add when replacing obj in local impl with LRU package |
43
+ | CACHE_MIN_AGE | - | Used for rate limiting |
44
+ | CACHE_REDIS_CONNECTION_TIMEOUT | - | TODO: Redis |
45
+ | CACHE_REDIS_HOST | √ | |
46
+ | CACHE_REDIS_MAX_QUEUED_ITEMS | - | TODO: Redis |
47
+ | CACHE_REDIS_MAX_RECONNECT_COOLDOWN | - | TODO: Redis |
48
+ | CACHE_REDIS_PASSWORD | √ | |
49
+ | CACHE_REDIS_PATH | √ | |
50
+ | CACHE_REDIS_PORT | √ | |
51
+ | CACHE_REDIS_TIMEOUT | √ | |
52
+ | CACHE_REDIS_URL | - | TODO: Redis |
53
+ | CACHE_TYPE | √ | |
54
+ | CACHE_UPDATE_AGE_ON_GET | N/A | No longer used in EA currently, apparently |
55
+ | DATA_PROVIDER_URL | - | Legacy var same as ADAPTER_URL |
56
+ | DEBUG | √ | |
57
+ | DEFAULT_WS_HEARTBEAT_INTERVAL | N/A | |
58
+ | EA_HOST | √ | |
59
+ | EA_PORT | √ | |
60
+ | ERROR_CAPACITY | - | Error backoff is not currently in v3 |
61
+ | EXPERIMENTAL_METRICS_ENABLED | √ | Maintaining for backwards compatibility. Defaults to true |
62
+ | LOG_LEVEL | | |
63
+ | METRICS_ENABLED | √ | Defaults to true |
64
+ | METRICS_NAME | √ | |
65
+ | METRICS_PORT | √ | |
66
+ | METRICS_USE_BASE_URL | √ | |
67
+ | NODE_ENV | | |
68
+ | npm_package_version | | |
69
+ | RATE_LIMIT_API_TIER | √ | |
70
+ | RATE_LIMIT_CAPACITY | √ | Used for per minute rate limit if exists |
71
+ | RATE_LIMIT_CAPACITY_MINUTE | √ | Used for per minute rate limit if exists |
72
+ | RATE_LIMIT_CAPACITY_SECOND | √ | Used for per second rate limit if exists |
73
+ | RATE_LIMIT_ENABLED | N/A | Rate limit always enabled in v3 |
74
+ | RECORD | N/A | Not needed in v3 config for integration testing |
75
+ | REQUEST_COALESCING_ENABLED | √ | Overrides transport config option if set |
76
+ | REQUEST_COALESCING_ENTROPY_MAX | | |
77
+ | REQUEST_COALESCING_INTERVAL | | |
78
+ | REQUEST_COALESCING_INTERVAL_COEFFICIENT | | |
79
+ | REQUEST_COALESCING_INTERVAL_MAX | | |
80
+ | REQUEST_COALESCING_MAX_RETRIES | | |
81
+ | RETRY | - | Retry mechanism not yet part of v3 |
82
+ | SERVER_RATE_LIMIT_MAX | | |
83
+ | SERVER_SLOW_DOWN_AFTER_FACTOR | | |
84
+ | SERVER_SLOW_DOWN_DELAY_MS | | |
85
+ | TIMEOUT | X | API_TIMEOUT is used for same purpose |
86
+ | WARMUP_ENABLED | | |
87
+ | WARMUP_INTERVAL | | |
88
+ | WARMUP_SUBSCRIPTION_TTL | | |
89
+ | WARMUP_UNHEALTHY_THRESHOLD | | |
90
+ | WS_API_ENDPOINT | | |
91
+ | WS_API_KEY | | |
92
+ | WS_CONNECTION_KEY | N/A | Connection key is no longer used for WS in v3 |
93
+ | WS_CONNECTION_LIMIT | | |
94
+ | WS_CONNECTION_RETRY_DELAY | | |
95
+ | WS_CONNECTION_RETRY_LIMIT | | |
96
+ | WS_CONNECTION_TTL | | |
97
+ | WS_ENABLED | N/A | Shouldn't be part of this fw |
98
+ | WS_SUBSCRIPTION_LIMIT | | |
99
+ | WS_SUBSCRIPTION_PRIORITY_LIST | | |
100
+ | WS_SUBSCRIPTION_TTL | | |
101
+ | WS_SUBSCRIPTION_UNRESPONSIVE_TTL | | |
102
+ | WS_TIME_UNTIL_HANDLE_NEXT_MESSAGE_OVERRIDE | | |
package/package.json CHANGED
@@ -1,58 +1,47 @@
1
1
  {
2
2
  "name": "@chainlink/external-adapter-framework",
3
- "version": "0.3.9",
3
+ "version": "0.4.0",
4
4
  "main": "dist/index.js",
5
5
  "license": "MIT",
6
6
  "dependencies": {
7
- "ajv": "8.11.0",
8
- "axios": "0.27.2",
9
- "eventsource": "2.0.2",
10
- "fastify": "4.7.0",
11
- "ioredis": "5.2.3",
12
- "pino": "8.6.1",
13
- "prom-client": "13.2.0",
14
- "ws": "8.9.0"
7
+ "axios": "^0.27.2",
8
+ "fastify": "^3.29.0",
9
+ "ioredis": "^5.0.4",
10
+ "pino": "^7.9.2",
11
+ "supertest": "^6.2.4",
12
+ "ts-jest": "^28.0.7",
13
+ "ts-node": "^10.9.1",
14
+ "typescript": "^4.6.3",
15
+ "ws": "^8.5.0"
15
16
  },
16
17
  "scripts": {
18
+ "start": "ts-node src/test.ts",
19
+ "generate-docs": "typedoc src/**/*.ts",
20
+ "test": "LOG_LEVEL=error EA_PORT=0 c8 ava",
17
21
  "build": "tsc",
22
+ "lint": "eslint ./src && prettier --check ./src/**/*.ts",
23
+ "lint-fix": "eslint --fix ./src && prettier --write ./src/**/*.ts",
18
24
  "dev": "NODE_ENV=develop tsnd --respawn --transpile-only --project tsconfig.json './src/test.ts'",
19
- "generate-docs": "typedoc src/**/*.ts",
20
- "lint-fix": "eslint --max-warnings=0 --fix ./src && prettier --write ./src/**/*.ts ./*.{json,js,yaml}",
21
- "lint": "eslint --max-warnings=0 ./src && prettier --check ./src/**/*.ts ./*.{json,js,yaml}",
22
- "release": "./release.sh",
23
- "start": "ts-node -e 'import(`./src/examples/${process.argv[1]}/src/index`).then(ea => ea.server())'",
24
- "test-debug": "EA_HOST=localhost SKIP_METRICS_CLEAR=true LOG_LEVEL=trace DEBUG=true EA_PORT=0 c8 ava --verbose",
25
- "test": "EA_HOST=localhost SKIP_METRICS_CLEAR=true LOG_LEVEL=error EA_PORT=0 c8 ava",
26
- "verify": "yarn lint && yarn build && yarn build -p ./test/tsconfig.json && yarn test",
27
- "code-coverage": "c8 check-coverage --statements 95 --lines 95 --functions 95 --branches 90"
25
+ "verify": "yarn lint && yarn build && yarn build -p ./test/tsconfig.json && yarn test"
28
26
  },
29
27
  "devDependencies": {
30
- "@sinonjs/fake-timers": "9.1.2",
31
- "@types/eventsource": "1.1.9",
32
- "@types/jest": "29.1.1",
33
- "@types/node": "18.8.2",
34
- "@types/sinonjs__fake-timers": "8.1.2",
35
- "@types/supertest": "2.0.12",
36
- "@types/ws": "8.5.3",
37
- "@typescript-eslint/eslint-plugin": "5.39.0",
38
- "@typescript-eslint/parser": "5.39.0",
39
- "ava": "4.3.3",
40
- "c8": "7.12.0",
41
- "eslint": "8.24.0",
42
- "eslint-config-prettier": "8.5.0",
43
- "eslint-plugin-tsdoc": "0.2.17",
44
- "jest": "29.1.2",
45
- "mock-socket": "9.1.5",
46
- "mocksse": "1.0.4",
47
- "nock": "13.2.9",
48
- "pino-pretty": "9.1.0",
49
- "prettier": "2.7.1",
50
- "supertest": "6.3.0",
51
- "ts-node": "10.9.1",
52
- "ts-jest": "29.0.3",
53
- "ts-node-dev": "2.0.0",
54
- "typedoc": "0.23.15",
55
- "typescript": "4.8.4"
28
+ "@sinonjs/fake-timers": "^9.1.2",
29
+ "@types/jest": "^28.1.7",
30
+ "@types/node": "^18.6.5",
31
+ "@types/sinonjs__fake-timers": "^8.1.2",
32
+ "@types/ws": "^8.5.3",
33
+ "@typescript-eslint/eslint-plugin": "^5.17.0",
34
+ "@typescript-eslint/parser": "^5.17.0",
35
+ "ava": "^4.2.0",
36
+ "c8": "^7.11.2",
37
+ "eslint": "^8.14.0",
38
+ "eslint-config-prettier": "^8.5.0",
39
+ "eslint-plugin-tsdoc": "^0.2.16",
40
+ "mock-socket": "^9.1.3",
41
+ "nock": "^13.2.4",
42
+ "pino-pretty": "^7.6.0",
43
+ "prettier": "^2.6.1",
44
+ "typedoc": "^0.22.15"
56
45
  },
57
46
  "prettier": {
58
47
  "semi": false,
@@ -62,6 +51,10 @@
62
51
  "trailingComma": "all",
63
52
  "arrowParens": "always"
64
53
  },
54
+ "optionalDependencies": {
55
+ "bufferutil": "^4.0.6",
56
+ "utf-8-validate": "^5.0.9"
57
+ },
65
58
  "ava": {
66
59
  "files": [
67
60
  "test/**/*.test.ts"
@@ -72,10 +65,9 @@
72
65
  "require": [
73
66
  "ts-node/register"
74
67
  ],
75
- "workerThreads": false,
76
- "environmentVariables": {
77
- "METRICS_ENABLED": "false"
78
- },
79
- "timeout": "20s"
80
- }
68
+ "workerThreads": false
69
+ },
70
+ "files": [
71
+ "dist"
72
+ ]
81
73
  }
@@ -1,86 +0,0 @@
1
- import { SettingsMap, BaseAdapterConfig, AdapterConfig } from '../config';
2
- import { AdapterRequest, AdapterResponse } from '../util';
3
- import { AdapterEndpoint } from './endpoint';
4
- import { AdapterDependencies, AdapterParams, AdapterRateLimitingConfig, CustomAdapterSettings, RequestTransform } from './types';
5
- /**
6
- * Main class to represent an External Adapter
7
- */
8
- export declare class Adapter<CustomSettings extends CustomAdapterSettings = SettingsMap> implements Omit<AdapterParams<CustomSettings>, 'bootstrap'> {
9
- name: string;
10
- defaultEndpoint?: string | undefined;
11
- endpoints: AdapterEndpoint<unknown, unknown, CustomSettings>[];
12
- envDefaultOverrides?: Partial<BaseAdapterConfig> | undefined;
13
- customSettings?: SettingsMap | undefined;
14
- rateLimiting?: AdapterRateLimitingConfig | undefined;
15
- overrides?: Record<string, string> | undefined;
16
- requestTransforms?: RequestTransform[];
17
- initialized: boolean;
18
- /** Object containing alias translations for all endpoints */
19
- endpointsMap: Record<string, AdapterEndpoint<unknown, unknown, CustomSettings>>;
20
- /** Initialized dependencies that the adapter will use */
21
- dependencies: AdapterDependencies;
22
- /** Configuration params for various adapter properties */
23
- config: AdapterConfig<CustomSettings>;
24
- /** Bootstrap function that will run when initializing the adapter */
25
- private readonly bootstrap?;
26
- constructor(params: AdapterParams<CustomSettings>);
27
- /**
28
- * Initializes all of the [[Transport]]s in the adapter, passing along any [[AdapterDependencies]] and [[AdapterConfig]].
29
- * Additionally, it builds a map out of all the endpoint names and aliases (checking for duplicates).
30
- */
31
- initialize(dependencies?: Partial<AdapterDependencies>): Promise<void>;
32
- /**
33
- * Takes an adapter and normalizes all endpoint names and aliases, as well as the default endpoint.
34
- * i.e. makes them lowercase for now
35
- */
36
- private normalizeEndpointNames;
37
- /**
38
- * This function will take an adapter structure and go through each endpoint, calculating
39
- * each one's allocation of the total rate limits that are set for the adapter as a whole.
40
- *
41
- */
42
- private calculateRateLimitAllocations;
43
- /**
44
- * Creates a list of key/value pairs that need to be censored in the logs
45
- * using the sensitive flag in the adapter config
46
- */
47
- private buildCensorList;
48
- /**
49
- * This function will process dependencies for an adapter, such as caches or rate limiters,
50
- * in order to inject them into transports and other relevant places later in the lifecycle.
51
- *
52
- * @param inputDependencies - a partial obj of initialized dependencies to override the created ones
53
- * @returns a set of AdapterDependencies all initialized
54
- */
55
- initializeDependencies(inputDependencies?: Partial<AdapterDependencies>): AdapterDependencies;
56
- /**
57
- * Attempts to find a value from the Cache and return that if found.
58
- *
59
- * @param req - the incoming request to this adapter
60
- * @returns the cached value if exists
61
- */
62
- findResponseInCache(req: AdapterRequest): Promise<AdapterResponse | undefined>;
63
- /**
64
- * Default request transform that takes requests and manipulates
65
- *
66
- * @param adapter - the current adapter
67
- * @param req - the current adapter request
68
- * @returns the modified (or new) request
69
- */
70
- symbolOverrider(req: AdapterRequest): AdapterRequest<import("../util").AdapterRequestData>;
71
- /**
72
- * Takes the incoming request and applies all request transforms in the adapter
73
- *
74
- * @param req - the current adapter request
75
- * @returns the request after passing through all request transforms
76
- */
77
- runRequestTransforms(req: AdapterRequest): void;
78
- /**
79
- * Function to serve as middleware to pass along the AdapterRequest to the appropriate Transport (acc. to the endpoint in the req.)
80
- *
81
- * @param req - the incoming request to this adapter
82
- * @param replySent - a promise that resolves when the reply has already been sent
83
- * @returns a simple Promise when it's done
84
- */
85
- handleRequest(req: AdapterRequest, replySent: Promise<unknown>): Promise<AdapterResponse>;
86
- }
package/adapter/basic.js DELETED
@@ -1,325 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
25
- var __importDefault = (this && this.__importDefault) || function (mod) {
26
- return (mod && mod.__esModule) ? mod : { "default": mod };
27
- };
28
- Object.defineProperty(exports, "__esModule", { value: true });
29
- exports.Adapter = void 0;
30
- const ioredis_1 = __importDefault(require("ioredis"));
31
- const cache_1 = require("../cache");
32
- const cacheMetrics = __importStar(require("../cache/metrics"));
33
- const config_1 = require("../config");
34
- const transportMetrics = __importStar(require("../transports/metrics"));
35
- const rate_limiting_1 = require("../rate-limiting");
36
- const util_1 = require("../util");
37
- const censor_list_1 = __importDefault(require("../util/censor/censor-list"));
38
- const error_1 = require("../validation/error");
39
- const logger = (0, util_1.makeLogger)('Adapter');
40
- /**
41
- * Main class to represent an External Adapter
42
- */
43
- class Adapter {
44
- constructor(params) {
45
- // After initialization
46
- this.initialized = false;
47
- /** Object containing alias translations for all endpoints */
48
- this.endpointsMap = {};
49
- // Copy over params
50
- this.name = params.name;
51
- this.defaultEndpoint = params.defaultEndpoint?.toLowerCase();
52
- this.endpoints = params.endpoints;
53
- this.envDefaultOverrides = params.envDefaultOverrides;
54
- this.customSettings = params.customSettings;
55
- this.rateLimiting = params.rateLimiting;
56
- this.overrides = params.overrides;
57
- this.requestTransforms = [this.symbolOverrider.bind(this), ...(params.requestTransforms || [])];
58
- this.bootstrap = params.bootstrap;
59
- this.config = (0, config_1.buildAdapterConfig)({
60
- overrides: this.envDefaultOverrides,
61
- customSettings: this.customSettings,
62
- });
63
- this.normalizeEndpointNames();
64
- this.calculateRateLimitAllocations();
65
- }
66
- /**
67
- * Initializes all of the [[Transport]]s in the adapter, passing along any [[AdapterDependencies]] and [[AdapterConfig]].
68
- * Additionally, it builds a map out of all the endpoint names and aliases (checking for duplicates).
69
- */
70
- async initialize(dependencies) {
71
- if (this.initialized) {
72
- throw new Error('This adapter has already been initialized!');
73
- }
74
- // Building configs during initialization to avoid validation errors during construction
75
- (0, config_1.validateAdapterConfig)(this.config, this.customSettings);
76
- if (this.bootstrap) {
77
- await this.bootstrap(this);
78
- }
79
- this.dependencies = this.initializeDependencies(dependencies);
80
- for (const endpoint of this.endpoints) {
81
- // Add aliases to map to use in validation
82
- const aliases = [endpoint.name, ...(endpoint.aliases || [])];
83
- for (const alias of aliases) {
84
- if (this.endpointsMap[alias]) {
85
- throw new Error(`Duplicate endpoint / alias: "${alias}"`);
86
- }
87
- this.endpointsMap[alias] = endpoint;
88
- }
89
- logger.debug(`Initializing transport for endpoint "${endpoint.name}"...`);
90
- await endpoint.transport.initialize(this.dependencies, this.config, endpoint.name);
91
- }
92
- // Build list of key/values that need to be redacted in logs
93
- // Populates the static array in CensorList to use in censor-transport
94
- this.buildCensorList();
95
- logger.debug('Adapter initialization complete.');
96
- this.initialized = true;
97
- }
98
- /**
99
- * Takes an adapter and normalizes all endpoint names and aliases, as well as the default endpoint.
100
- * i.e. makes them lowercase for now
101
- */
102
- normalizeEndpointNames() {
103
- for (const endpoint of this.endpoints) {
104
- endpoint.name = endpoint.name.toLowerCase();
105
- endpoint.aliases = endpoint.aliases?.map((a) => a.toLowerCase());
106
- }
107
- }
108
- /**
109
- * This function will take an adapter structure and go through each endpoint, calculating
110
- * each one's allocation of the total rate limits that are set for the adapter as a whole.
111
- *
112
- */
113
- calculateRateLimitAllocations() {
114
- const numberOfEndpoints = this.endpoints.length;
115
- const endpointsWithExplicitAllocations = this.endpoints.filter((e) => e.rateLimiting);
116
- const totalExplicitAllocation = endpointsWithExplicitAllocations
117
- .map((e) => e.rateLimiting?.allocationPercentage || 0)
118
- .reduce((sum, next) => sum + next, 0);
119
- if (totalExplicitAllocation > 100) {
120
- throw new Error('The total allocation set for all endpoints summed cannot exceed 100%');
121
- }
122
- if (totalExplicitAllocation === 100 &&
123
- numberOfEndpoints - endpointsWithExplicitAllocations.length > 0) {
124
- throw new Error('The explicit allocation is at 100% but there are endpoints with implicit allocation');
125
- }
126
- const implicitAllocation = 100 - totalExplicitAllocation;
127
- logger.debug('Adapter rate limit allocations:');
128
- for (const endpoint of this.endpoints) {
129
- if (!endpoint.rateLimiting) {
130
- endpoint.rateLimiting = {
131
- allocationPercentage: implicitAllocation / (numberOfEndpoints - endpointsWithExplicitAllocations.length),
132
- };
133
- }
134
- logger.debug(`Endpoint [${endpoint.name}] - ${endpoint.rateLimiting?.allocationPercentage}%`);
135
- }
136
- }
137
- /**
138
- * Creates a list of key/value pairs that need to be censored in the logs
139
- * using the sensitive flag in the adapter config
140
- */
141
- buildCensorList() {
142
- const censorList = Object.entries(config_1.BaseSettings)
143
- .concat(Object.entries(this.customSettings || {}))
144
- .filter(([name, setting]) => setting &&
145
- setting.type === 'string' &&
146
- setting.sensitive &&
147
- this.config[name])
148
- .map(([name]) => ({
149
- key: name,
150
- // Escaping potential special characters in values before creating regex
151
- value: new RegExp(
152
- // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
153
- this.config[name].replace(/[-[\]{}()*+?.,\\^$|#\s]/g, '\\$&'), 'gi'),
154
- }));
155
- censor_list_1.default.set(censorList);
156
- }
157
- /**
158
- * This function will process dependencies for an adapter, such as caches or rate limiters,
159
- * in order to inject them into transports and other relevant places later in the lifecycle.
160
- *
161
- * @param inputDependencies - a partial obj of initialized dependencies to override the created ones
162
- * @returns a set of AdapterDependencies all initialized
163
- */
164
- initializeDependencies(inputDependencies) {
165
- const dependencies = inputDependencies || {};
166
- if (!dependencies.redisClient) {
167
- if (this.config.CACHE_TYPE === 'redis') {
168
- dependencies.redisClient = new ioredis_1.default({
169
- enableAutoPipelining: true,
170
- host: this.config.CACHE_REDIS_HOST,
171
- port: this.config.CACHE_REDIS_PORT,
172
- password: this.config.CACHE_REDIS_PASSWORD,
173
- path: this.config.CACHE_REDIS_PATH,
174
- timeout: this.config.CACHE_REDIS_TIMEOUT,
175
- });
176
- dependencies.redisClient.on('connect', () => {
177
- cacheMetrics.redisConnectionsOpen.inc();
178
- });
179
- }
180
- }
181
- if (!dependencies.cache) {
182
- dependencies.cache = cache_1.CacheFactory.buildCache(this.config.CACHE_TYPE, dependencies.redisClient);
183
- }
184
- const rateLimitingTier = (0, rate_limiting_1.getRateLimitingTier)(this.config, this.rateLimiting?.tiers);
185
- if (!dependencies.requestRateLimiter) {
186
- dependencies.requestRateLimiter = new rate_limiting_1.SimpleCountingRateLimiter().initialize(this.endpoints, rateLimitingTier);
187
- }
188
- if (!dependencies.backgroundExecuteRateLimiter) {
189
- dependencies.backgroundExecuteRateLimiter = new rate_limiting_1.FixedFrequencyRateLimiter().initialize(this.endpoints, rateLimitingTier);
190
- }
191
- if (!dependencies.subscriptionSetFactory) {
192
- dependencies.subscriptionSetFactory = new util_1.SubscriptionSetFactory(this.config, this.name, dependencies.redisClient);
193
- }
194
- return dependencies;
195
- }
196
- /**
197
- * Attempts to find a value from the Cache and return that if found.
198
- *
199
- * @param req - the incoming request to this adapter
200
- * @returns the cached value if exists
201
- */
202
- async findResponseInCache(req) {
203
- const response = await this.dependencies.cache.get(req.requestContext.cacheKey);
204
- if (response) {
205
- if (this.config.METRICS_ENABLED && this.config.EXPERIMENTAL_METRICS_ENABLED) {
206
- const label = cacheMetrics.cacheMetricsLabel(req.requestContext.cacheKey, req.requestContext.meta?.metrics?.feedId || 'N/A', this.config.CACHE_TYPE);
207
- // Record cache staleness and cache get count and value
208
- const staleness = (0, cache_1.calculateStaleness)(response.maxAge, this.config.CACHE_MAX_AGE);
209
- cacheMetrics.cacheGet(label, response.result, staleness);
210
- req.requestContext.meta = {
211
- ...req.requestContext.meta,
212
- metrics: { ...req.requestContext.meta?.metrics, cacheHit: true },
213
- };
214
- }
215
- return response;
216
- }
217
- }
218
- /**
219
- * Default request transform that takes requests and manipulates
220
- *
221
- * @param adapter - the current adapter
222
- * @param req - the current adapter request
223
- * @returns the modified (or new) request
224
- */
225
- symbolOverrider(req) {
226
- const rawRequestBody = req.body;
227
- const requestOverrides = rawRequestBody.data?.overrides?.[this.name.toLowerCase()];
228
- const base = req.requestContext.data['base'];
229
- // Perform overrides specified in the request payload
230
- if (requestOverrides?.[base]) {
231
- req.requestContext.data['base'] = requestOverrides[base];
232
- }
233
- // Perform hardcoded adapter overrides
234
- if (this.overrides?.[base]) {
235
- req.requestContext.data['base'] = this.overrides[base];
236
- }
237
- return req;
238
- }
239
- /**
240
- * Takes the incoming request and applies all request transforms in the adapter
241
- *
242
- * @param req - the current adapter request
243
- * @returns the request after passing through all request transforms
244
- */
245
- runRequestTransforms(req) {
246
- if (!this.requestTransforms) {
247
- return;
248
- }
249
- for (const transform of this.requestTransforms) {
250
- transform(req);
251
- }
252
- }
253
- /**
254
- * Function to serve as middleware to pass along the AdapterRequest to the appropriate Transport (acc. to the endpoint in the req.)
255
- *
256
- * @param req - the incoming request to this adapter
257
- * @param replySent - a promise that resolves when the reply has already been sent
258
- * @returns a simple Promise when it's done
259
- */
260
- async handleRequest(req, replySent) {
261
- // Get transport, must be here because it's already checked in the validator
262
- const transport = this.endpointsMap[req.requestContext.endpointName].transport;
263
- // First try to find the response in our cache, keep it ready
264
- const cachedResponse = await this.findResponseInCache(req);
265
- // Next we fire off the transport's registration of the request if defined, regardless of if we already have a cached response.
266
- // This is necessary to ensure things like subscription sets are updated each time we get a request
267
- let requestRegistrationPromise;
268
- if (transport.registerRequest) {
269
- const handler = async () => {
270
- // If we already have a cached response, wait for it to be sent back before continuing with registration
271
- // This way we respond to incoming requests from the cache as fast as possible
272
- if (cachedResponse) {
273
- await replySent;
274
- }
275
- // Disable the non-null assertion error, since we're checking this exists in the if above
276
- // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
277
- return transport.registerRequest(req, this.config);
278
- };
279
- // Execute the registration handler without blocking
280
- logger.debug(`Firing request registration handler${cachedResponse ? ' (cached response already sent)' : ''}`);
281
- requestRegistrationPromise = handler();
282
- }
283
- // Now that we have dealt with request registration, we can return the cached response if present
284
- if (cachedResponse) {
285
- logger.debug('Found response from cache, sending that');
286
- return cachedResponse;
287
- }
288
- // If there was no cached response, execute the foregroundExecute if defined
289
- const immediateResponse = transport.foregroundExecute && (await transport.foregroundExecute(req, this.config));
290
- if (immediateResponse) {
291
- logger.debug('Got immediate response from transport, sending as response');
292
- return immediateResponse;
293
- }
294
- // Finally, either because there was no synchronous execute or because it returned an empty response,
295
- // we wait for the cache to be filled (either from background work started in the sync execute, or the backgroundExecute).
296
- // We can wait for the request registration to have finished here, since we're going to be sleeping for the cache anyways,
297
- // and it's useful in case the registration throws a promise so that it doesn't go unhandled.
298
- await requestRegistrationPromise;
299
- // Observe the idle time taken for polling response
300
- const metricsTimer = transportMetrics.transportPollingDurationSeconds
301
- .labels({ endpoint: req.requestContext.endpointName })
302
- .startTimer();
303
- logger.debug('Transport is set up, polling cache for response...');
304
- const response = await (0, cache_1.pollResponseFromCache)(this.dependencies.cache, req.requestContext.cacheKey, {
305
- maxRetries: this.config.CACHE_POLLING_MAX_RETRIES,
306
- sleep: this.config.CACHE_POLLING_SLEEP_MS,
307
- });
308
- metricsTimer({ succeeded: String(!!response) });
309
- if (response) {
310
- logger.debug('Got a response from polling the cache, sending that back');
311
- return response;
312
- }
313
- // Record polling mechanism failure to return response
314
- transportMetrics.transportPollingFailureCount
315
- .labels({ endpoint: req.requestContext.endpointName })
316
- .inc();
317
- logger.debug('Ran out of polling attempts, returning timeout');
318
- throw new error_1.AdapterTimeoutError({
319
- message: 'Timed out waiting for provider result.',
320
- statusCode: 504,
321
- });
322
- }
323
- }
324
- exports.Adapter = Adapter;
325
- //# sourceMappingURL=basic.js.map