@chainlink/external-adapter-framework 0.0.12 → 0.0.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (278) hide show
  1. package/adapter.d.ts +107 -0
  2. package/adapter.js +115 -0
  3. package/{dist/src/package/background-executor.d.ts → background-executor.d.ts} +0 -0
  4. package/{dist/src/background-executor.js → background-executor.js} +0 -0
  5. package/{dist/src/cache → cache}/factory.d.ts +0 -0
  6. package/{dist/src/cache → cache}/factory.js +0 -0
  7. package/{dist/src/cache → cache}/index.d.ts +8 -8
  8. package/{dist/src/cache → cache}/index.js +0 -0
  9. package/{dist/src/cache → cache}/local.d.ts +0 -0
  10. package/{dist/src/cache → cache}/local.js +0 -0
  11. package/{dist/src/cache → cache}/metrics.d.ts +0 -0
  12. package/{dist/src/cache → cache}/metrics.js +0 -0
  13. package/{dist/src/cache → cache}/redis.d.ts +0 -0
  14. package/{dist/src/cache → cache}/redis.js +0 -0
  15. package/{dist/src/chainlink-external-adapter-framework-0.0.6.tgz → chainlink-external-adapter-framework-0.0.6.tgz} +0 -0
  16. package/{dist/src/config → config}/index.d.ts +0 -5
  17. package/{dist/src/config → config}/index.js +5 -5
  18. package/{dist/src/config → config}/provider-limits.d.ts +0 -0
  19. package/{dist/src/config → config}/provider-limits.js +5 -1
  20. package/{dist/src/examples → examples}/bank-frick/accounts.d.ts +1 -1
  21. package/{dist/src/examples → examples}/bank-frick/accounts.js +2 -3
  22. package/{dist/src/examples → examples}/bank-frick/config/index.d.ts +0 -0
  23. package/{dist/src/examples → examples}/bank-frick/config/index.js +0 -0
  24. package/{dist/src/package/examples/coingecko → examples/bank-frick}/index.d.ts +0 -0
  25. package/{dist/src/examples → examples}/bank-frick/index.js +2 -3
  26. package/{dist/src/examples → examples}/bank-frick/util.d.ts +0 -0
  27. package/{dist/src/examples → examples}/bank-frick/util.js +0 -0
  28. package/{dist/src/package/examples → examples}/coingecko/batch-warming.d.ts +0 -0
  29. package/{dist/src/package/examples → examples}/coingecko/batch-warming.js +0 -0
  30. package/{dist/src/package/examples/ncfx → examples/coingecko}/index.d.ts +0 -0
  31. package/{dist/src/package/examples → examples}/coingecko/index.js +0 -0
  32. package/{dist/src/package/examples → examples}/coingecko/rest.d.ts +0 -0
  33. package/{dist/src/package/examples → examples}/coingecko/rest.js +0 -0
  34. package/{dist/src/examples → examples}/ncfx/config/index.d.ts +0 -0
  35. package/{dist/src/examples → examples}/ncfx/config/index.js +0 -0
  36. package/examples/ncfx/index.d.ts +2 -0
  37. package/{dist/src/package/examples → examples}/ncfx/index.js +0 -0
  38. package/{dist/src/package/examples → examples}/ncfx/websocket.d.ts +0 -0
  39. package/{dist/src/package/examples → examples}/ncfx/websocket.js +0 -0
  40. package/{dist/src/index.d.ts → index.d.ts} +0 -0
  41. package/{dist/src/index.js → index.js} +18 -13
  42. package/{dist/src/metrics → metrics}/constants.d.ts +0 -0
  43. package/{dist/src/metrics → metrics}/constants.js +0 -0
  44. package/{dist/src/metrics → metrics}/index.d.ts +0 -0
  45. package/{dist/src/metrics → metrics}/index.js +0 -0
  46. package/metrics/util.d.ts +7 -0
  47. package/{dist/src/metrics → metrics}/util.js +0 -0
  48. package/{dist/src/package → package}/adapter.d.ts +0 -0
  49. package/{dist/src/package → package}/adapter.js +0 -0
  50. package/{dist/src → package}/background-executor.d.ts +2 -2
  51. package/{dist/src/package → package}/background-executor.js +0 -0
  52. package/{dist/src/package → package}/cache/factory.d.ts +0 -0
  53. package/{dist/src/package → package}/cache/factory.js +0 -0
  54. package/{dist/src/package → package}/cache/index.d.ts +0 -0
  55. package/{dist/src/package → package}/cache/index.js +0 -0
  56. package/{dist/src/package → package}/cache/local.d.ts +0 -0
  57. package/{dist/src/package → package}/cache/local.js +0 -0
  58. package/{dist/src/package → package}/cache/metrics.d.ts +0 -0
  59. package/{dist/src/package → package}/cache/metrics.js +0 -0
  60. package/{dist/src/package → package}/cache/redis.d.ts +0 -0
  61. package/{dist/src/package → package}/cache/redis.js +0 -0
  62. package/{dist/src/package → package}/config/index.d.ts +0 -0
  63. package/{dist/src/package → package}/config/index.js +0 -0
  64. package/{dist/src/package → package}/config/provider-limits.d.ts +0 -0
  65. package/{dist/src/package → package}/config/provider-limits.js +0 -0
  66. package/package/examples/coingecko/batch-warming.d.ts +2 -0
  67. package/{dist/src → package}/examples/coingecko/batch-warming.js +2 -3
  68. package/package/examples/coingecko/index.d.ts +2 -0
  69. package/{dist/src → package}/examples/coingecko/index.js +2 -3
  70. package/package/examples/coingecko/rest.d.ts +2 -0
  71. package/{dist/src → package}/examples/coingecko/rest.js +2 -3
  72. package/{dist/src/package → package}/examples/ncfx/config/index.d.ts +0 -0
  73. package/{dist/src/package → package}/examples/ncfx/config/index.js +0 -0
  74. package/package/examples/ncfx/index.d.ts +2 -0
  75. package/{dist/src → package}/examples/ncfx/index.js +2 -3
  76. package/{dist/src → package}/examples/ncfx/websocket.d.ts +1 -12
  77. package/{dist/src → package}/examples/ncfx/websocket.js +2 -3
  78. package/{dist/src/package → package}/index.d.ts +0 -0
  79. package/{dist/src/package → package}/index.js +0 -0
  80. package/{dist/src/package → package}/metrics/constants.d.ts +0 -0
  81. package/{dist/src/package → package}/metrics/constants.js +0 -0
  82. package/{dist/src/package → package}/metrics/index.d.ts +0 -0
  83. package/{dist/src/package → package}/metrics/index.js +0 -0
  84. package/{dist/src/package → package}/metrics/util.d.ts +0 -0
  85. package/{dist/src/package → package}/metrics/util.js +0 -0
  86. package/{dist/src → package}/package.json +0 -0
  87. package/{dist/src/package → package}/rate-limiting/background/fixed-frequency.d.ts +0 -0
  88. package/{dist/src/package → package}/rate-limiting/background/fixed-frequency.js +0 -0
  89. package/{dist/src/package → package}/rate-limiting/index.d.ts +0 -0
  90. package/{dist/src/package → package}/rate-limiting/index.js +0 -0
  91. package/{dist/src/package → package}/rate-limiting/metrics.d.ts +0 -0
  92. package/{dist/src/package → package}/rate-limiting/metrics.js +0 -0
  93. package/{dist/src/package → package}/rate-limiting/request/simple-counting.d.ts +0 -0
  94. package/{dist/src/package → package}/rate-limiting/request/simple-counting.js +0 -0
  95. package/{dist/src/package → package}/test.d.ts +0 -0
  96. package/{dist/src/package → package}/test.js +0 -0
  97. package/{dist/src/package → package}/transports/batch-warming.d.ts +0 -0
  98. package/{dist/src/package → package}/transports/batch-warming.js +0 -0
  99. package/{dist/src/package → package}/transports/index.d.ts +0 -0
  100. package/{dist/src/package → package}/transports/index.js +0 -0
  101. package/{dist/src/package → package}/transports/metrics.d.ts +0 -0
  102. package/{dist/src/package → package}/transports/metrics.js +0 -0
  103. package/{dist/src/package → package}/transports/rest.d.ts +0 -0
  104. package/{dist/src/package → package}/transports/rest.js +0 -0
  105. package/{dist/src/package → package}/transports/util.d.ts +0 -0
  106. package/{dist/src/package → package}/transports/util.js +0 -0
  107. package/{dist/src/package → package}/transports/websocket.d.ts +0 -0
  108. package/{dist/src/package → package}/transports/websocket.js +0 -0
  109. package/{dist/src/package → package}/util/expiring-sorted-set.d.ts +0 -0
  110. package/{dist/src/package → package}/util/expiring-sorted-set.js +0 -0
  111. package/{dist/src/package → package}/util/index.d.ts +0 -0
  112. package/{dist/src/package → package}/util/index.js +0 -0
  113. package/{dist/src/package → package}/util/logger.d.ts +0 -0
  114. package/{dist/src/package → package}/util/logger.js +0 -0
  115. package/{dist/src/package → package}/util/request.d.ts +0 -0
  116. package/{dist/src/package → package}/util/request.js +0 -0
  117. package/{dist/src/package → package}/validation/error.d.ts +0 -0
  118. package/{dist/src/package → package}/validation/error.js +0 -0
  119. package/{dist/src/package → package}/validation/index.d.ts +0 -0
  120. package/{dist/src/package → package}/validation/index.js +0 -0
  121. package/{dist/src/package → package}/validation/input-params.d.ts +0 -0
  122. package/{dist/src/package → package}/validation/input-params.js +0 -0
  123. package/{dist/src/package → package}/validation/override-functions.d.ts +0 -0
  124. package/{dist/src/package → package}/validation/override-functions.js +0 -0
  125. package/{dist/src/package → package}/validation/preset-tokens.json +0 -0
  126. package/{dist/src/package → package}/validation/validator.d.ts +0 -0
  127. package/{dist/src/package → package}/validation/validator.js +0 -0
  128. package/package.json +1 -1
  129. package/{dist/src/rate-limiting → rate-limiting}/background/fixed-frequency.d.ts +1 -2
  130. package/{dist/src/rate-limiting → rate-limiting}/background/fixed-frequency.js +0 -0
  131. package/{dist/src/rate-limiting → rate-limiting}/index.d.ts +1 -2
  132. package/{dist/src/rate-limiting → rate-limiting}/index.js +0 -0
  133. package/{dist/src/rate-limiting → rate-limiting}/metrics.d.ts +0 -0
  134. package/{dist/src/rate-limiting → rate-limiting}/metrics.js +0 -0
  135. package/{dist/src/rate-limiting → rate-limiting}/request/simple-counting.d.ts +1 -2
  136. package/{dist/src/rate-limiting → rate-limiting}/request/simple-counting.js +0 -0
  137. package/{dist/src/test.d.ts → test.d.ts} +0 -0
  138. package/{dist/src/test.js → test.js} +0 -0
  139. package/{dist/src/transports → transports}/batch-warming.d.ts +0 -0
  140. package/{dist/src/transports → transports}/batch-warming.js +0 -0
  141. package/{dist/src/transports → transports}/index.d.ts +2 -2
  142. package/{dist/src/transports → transports}/index.js +0 -0
  143. package/{dist/src/transports → transports}/metrics.d.ts +1 -2
  144. package/{dist/src/transports → transports}/metrics.js +0 -0
  145. package/{dist/src/transports → transports}/rest.d.ts +0 -0
  146. package/{dist/src/transports → transports}/rest.js +0 -0
  147. package/{dist/src/transports → transports}/util.d.ts +0 -0
  148. package/{dist/src/transports → transports}/util.js +0 -0
  149. package/{dist/src/transports → transports}/websocket.d.ts +3 -2
  150. package/{dist/src/transports → transports}/websocket.js +6 -4
  151. package/{dist/src/util → util}/expiring-sorted-set.d.ts +0 -0
  152. package/{dist/src/util → util}/expiring-sorted-set.js +0 -0
  153. package/{dist/src/util → util}/index.d.ts +0 -0
  154. package/{dist/src/util → util}/index.js +0 -0
  155. package/{dist/src/util → util}/logger.d.ts +0 -0
  156. package/{dist/src/util → util}/logger.js +0 -0
  157. package/{dist/src/util → util}/request.d.ts +2 -2
  158. package/{dist/src/util → util}/request.js +0 -0
  159. package/{dist/src/util → util}/subscription-set/expiring-sorted-set.d.ts +0 -0
  160. package/{dist/src/util → util}/subscription-set/expiring-sorted-set.js +0 -0
  161. package/{dist/src/util → util}/subscription-set/subscription-set.d.ts +0 -0
  162. package/{dist/src/util → util}/subscription-set/subscription-set.js +0 -0
  163. package/{dist/src/util → util}/test-payload-loader.d.ts +0 -0
  164. package/{dist/src/util → util}/test-payload-loader.js +0 -0
  165. package/{dist/src/validation → validation}/error.d.ts +2 -2
  166. package/{dist/src/validation → validation}/error.js +0 -0
  167. package/{dist/src/validation → validation}/index.d.ts +0 -0
  168. package/{dist/src/validation → validation}/index.js +0 -0
  169. package/{dist/src/validation → validation}/input-params.d.ts +0 -0
  170. package/{dist/src/validation → validation}/input-params.js +0 -0
  171. package/validation/override-functions.d.ts +3 -0
  172. package/{dist/src/validation → validation}/override-functions.js +0 -0
  173. package/{dist/src/validation → validation}/preset-tokens.json +0 -0
  174. package/{dist/src/validation → validation}/validator.d.ts +0 -0
  175. package/{dist/src/validation → validation}/validator.js +0 -0
  176. package/.c8rc.json +0 -3
  177. package/.eslintignore +0 -9
  178. package/.eslintrc.js +0 -96
  179. package/.github/README.MD +0 -17
  180. package/.github/actions/setup/action.yaml +0 -13
  181. package/.github/workflows/main.yaml +0 -39
  182. package/.github/workflows/publish.yaml +0 -20
  183. package/.prettierignore +0 -13
  184. package/.yarnrc +0 -0
  185. package/README.md +0 -103
  186. package/dist/src/adapter.d.ts +0 -135
  187. package/dist/src/adapter.js +0 -145
  188. package/dist/src/examples/bank-frick/index.d.ts +0 -2
  189. package/dist/src/examples/coingecko/batch-warming.d.ts +0 -7
  190. package/dist/src/examples/coingecko/index.d.ts +0 -2
  191. package/dist/src/examples/coingecko/rest.d.ts +0 -12
  192. package/dist/src/examples/ncfx/index.d.ts +0 -13
  193. package/dist/src/metrics/util.d.ts +0 -7
  194. package/dist/src/package/package.json +0 -72
  195. package/dist/src/validation/override-functions.d.ts +0 -3
  196. package/docker-compose.yaml +0 -35
  197. package/env.sh +0 -54
  198. package/env2.sh +0 -55
  199. package/publish.sh +0 -0
  200. package/src/adapter.ts +0 -263
  201. package/src/background-executor.ts +0 -52
  202. package/src/cache/factory.ts +0 -26
  203. package/src/cache/index.ts +0 -258
  204. package/src/cache/local.ts +0 -73
  205. package/src/cache/metrics.ts +0 -112
  206. package/src/cache/redis.ts +0 -93
  207. package/src/config/index.ts +0 -517
  208. package/src/config/provider-limits.ts +0 -130
  209. package/src/examples/bank-frick/README.MD +0 -10
  210. package/src/examples/bank-frick/accounts.ts +0 -246
  211. package/src/examples/bank-frick/config/index.ts +0 -53
  212. package/src/examples/bank-frick/index.ts +0 -13
  213. package/src/examples/bank-frick/types.d.ts +0 -38
  214. package/src/examples/bank-frick/util.ts +0 -55
  215. package/src/examples/coingecko/batch-warming.ts +0 -78
  216. package/src/examples/coingecko/index.ts +0 -9
  217. package/src/examples/coingecko/rest.ts +0 -77
  218. package/src/examples/ncfx/config/index.ts +0 -12
  219. package/src/examples/ncfx/index.ts +0 -9
  220. package/src/examples/ncfx/websocket.ts +0 -99
  221. package/src/index.ts +0 -149
  222. package/src/metrics/constants.ts +0 -23
  223. package/src/metrics/index.ts +0 -115
  224. package/src/metrics/util.ts +0 -18
  225. package/src/rate-limiting/background/fixed-frequency.ts +0 -45
  226. package/src/rate-limiting/index.ts +0 -100
  227. package/src/rate-limiting/metrics.ts +0 -18
  228. package/src/rate-limiting/request/simple-counting.ts +0 -76
  229. package/src/test.ts +0 -5
  230. package/src/transports/batch-warming.ts +0 -122
  231. package/src/transports/index.ts +0 -152
  232. package/src/transports/metrics.ts +0 -95
  233. package/src/transports/rest.ts +0 -164
  234. package/src/transports/util.ts +0 -63
  235. package/src/transports/websocket.ts +0 -245
  236. package/src/util/index.ts +0 -22
  237. package/src/util/logger.ts +0 -69
  238. package/src/util/request.ts +0 -117
  239. package/src/util/subscription-set/expiring-sorted-set.ts +0 -54
  240. package/src/util/subscription-set/subscription-set.ts +0 -35
  241. package/src/util/test-payload-loader.ts +0 -87
  242. package/src/validation/error.ts +0 -116
  243. package/src/validation/index.ts +0 -110
  244. package/src/validation/input-params.ts +0 -45
  245. package/src/validation/override-functions.ts +0 -44
  246. package/src/validation/preset-tokens.json +0 -23
  247. package/src/validation/validator.ts +0 -384
  248. package/test/adapter.test.ts +0 -27
  249. package/test/background-executor.test.ts +0 -108
  250. package/test/cache/cache-key.test.ts +0 -114
  251. package/test/cache/helper.ts +0 -100
  252. package/test/cache/local.test.ts +0 -54
  253. package/test/cache/redis.test.ts +0 -89
  254. package/test/correlation.test.ts +0 -114
  255. package/test/index.test.ts +0 -37
  256. package/test/metrics/feed-id.test.ts +0 -38
  257. package/test/metrics/helper.ts +0 -14
  258. package/test/metrics/labels.test.ts +0 -36
  259. package/test/metrics/metrics.test.ts +0 -267
  260. package/test/metrics/redis-metrics.test.ts +0 -113
  261. package/test/metrics/warmer-metrics.test.ts +0 -192
  262. package/test/metrics/ws-metrics.test.ts +0 -225
  263. package/test/rate-limit-config.test.ts +0 -242
  264. package/test/smoke.test.ts +0 -166
  265. package/test/transports/batch.test.ts +0 -465
  266. package/test/transports/rest.test.ts +0 -242
  267. package/test/transports/websocket.test.ts +0 -183
  268. package/test/tsconfig.json +0 -5
  269. package/test/util.ts +0 -77
  270. package/test/validation.test.ts +0 -178
  271. package/test-payload-fail.json +0 -3
  272. package/test-payload.js +0 -22
  273. package/test-payload.json +0 -7
  274. package/test.sh +0 -20
  275. package/test2.sh +0 -2
  276. package/tsconfig.json +0 -25
  277. package/typedoc.json +0 -6
  278. package/webpack.config.js +0 -23
package/src/index.ts DELETED
@@ -1,149 +0,0 @@
1
- import fastify, { FastifyInstance } from 'fastify'
2
- import { Server } from 'http'
3
- import { AddressInfo } from 'net'
4
- import { join } from 'path'
5
- import { Adapter, InitializedAdapter, initializeAdapter, AdapterDependencies } from './adapter'
6
- import { callBackgroundExecutes } from './background-executor'
7
- import { buildCacheMiddleware } from './cache'
8
- import { AdapterConfig, buildAdapterConfig } from './config'
9
- import { buildMetricsMiddleware, setupMetricsServer } from './metrics'
10
- import { buildTransportHandler } from './transports'
11
- import { AdapterRouteGeneric, loggingContextMiddleware, makeLogger } from './util'
12
- import { loadTestPayload } from './util/test-payload-loader'
13
- import { errorCatchingMiddleware, validatorMiddleware } from './validation'
14
-
15
- const logger = makeLogger('Main')
16
-
17
- const VERSION = process.env['npm_package_version']
18
-
19
- /**
20
- * Main function for the framework.
21
- * Initializes config and dependencies, uses those to initialize Transports, and starts listening for requests.
22
- *
23
- * @param adapter - an object describing an External Adapter
24
- * @param dependencies - an optional object with adapter dependencies to inject
25
- * @returns a Promise that resolves to the http.Server listening for connections
26
- */
27
- export const expose = async (
28
- adapter: Adapter,
29
- dependencies?: Partial<AdapterDependencies>,
30
- ): Promise<Server | undefined> => {
31
- const config = buildAdapterConfig({
32
- overrides: adapter.envDefaultOverrides,
33
- customSettings: adapter.customSettings,
34
- })
35
- // Initialize adapter (create dependencies, inject them, build endpoint map, etc.)
36
- const initializedAdapter = await initializeAdapter(adapter, config, dependencies)
37
-
38
- let server: Server | undefined = undefined
39
-
40
- if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
41
- setupMetricsServer(adapter.name, config)
42
- }
43
-
44
- if (config.EA_MODE === 'reader' || config.EA_MODE === 'reader-writer') {
45
- // Main REST API server to handle incoming requests
46
- const app = await buildRestApi(config, initializedAdapter)
47
-
48
- // Start listening for incoming requests
49
- try {
50
- await app.listen(config.EA_PORT, config.EA_HOST)
51
- } catch (err) {
52
- logger.fatal(`There was an error when starting the EA server: ${err}`)
53
- process.exit()
54
- }
55
-
56
- logger.info(`Listening on port ${(app.server.address() as AddressInfo).port}`)
57
- server = app.server
58
- } else {
59
- logger.info('REST API is disabled; this instance will not process incoming requests.')
60
- }
61
-
62
- if (config.EA_MODE === 'writer' || config.EA_MODE === 'reader-writer') {
63
- // Start background loop that will take care of calling any async Transports
64
- logger.info('Starting background execution loop')
65
- callBackgroundExecutes(initializedAdapter, server)
66
- } else {
67
- logger.info(
68
- 'Background executor is disabled; this instance will not perform async background executes.',
69
- )
70
- }
71
-
72
- return server
73
- }
74
-
75
- async function buildRestApi(config: AdapterConfig, initializedAdapter: InitializedAdapter) {
76
- const app = fastify()
77
-
78
- // Add healthcheck endpoint before middlewares to bypass them
79
- app.get(join(config.BASE_URL, 'health'), (req, res) => {
80
- res.status(200).send({ message: 'OK', version: VERSION })
81
- })
82
-
83
- // Use global error handling
84
- app.setErrorHandler(errorCatchingMiddleware)
85
-
86
- const transportHandler = await buildTransportHandler(initializedAdapter)
87
-
88
- app.register(async (router) => {
89
- // Set up "middlewares" (hooks in fastify)
90
- router.addHook<AdapterRouteGeneric>('preHandler', validatorMiddleware(initializedAdapter))
91
- router.addHook<AdapterRouteGeneric>('preHandler', buildCacheMiddleware(initializedAdapter))
92
- if (config['CORRELATION_ID_ENABLED']) {
93
- router.addHook<AdapterRouteGeneric>('onRequest', loggingContextMiddleware)
94
- }
95
-
96
- router.route<AdapterRouteGeneric>({
97
- url: config.BASE_URL,
98
- method: 'POST',
99
- handler: transportHandler,
100
- })
101
-
102
- if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
103
- router.addHook<AdapterRouteGeneric>('onResponse', buildMetricsMiddleware)
104
- }
105
- })
106
-
107
- // Add smoke endpoint after middleware which are needed for tests
108
- buildSmokeEndpoint(app, config)
109
- return app
110
- }
111
-
112
- /**
113
- * Adds the /smoke endpoint to the API for smoke testing the adapter
114
- *
115
- * @param app - the Fastify instance
116
- * @param config - the initialized adapter config
117
- */
118
- function buildSmokeEndpoint(app: FastifyInstance, config: AdapterConfig) {
119
- const testPayload = loadTestPayload(config.SMOKE_TEST_PAYLOAD_FILE_NAME)
120
- app.get(join(config.BASE_URL, 'smoke'), async (_, res) => {
121
- if (testPayload.isDefault) {
122
- return res.status(200).send('OK')
123
- }
124
-
125
- const errors = []
126
- for (const index in testPayload.requests) {
127
- try {
128
- const request = { id: index, data: testPayload.requests[index] }
129
- // Use Fastify's app inject to pass smoke requests internally
130
- const response = await app.inject({
131
- method: 'POST',
132
- url: '/',
133
- payload: request,
134
- })
135
- const parsedResponse = JSON.parse(response.body)
136
- // Throw error if not 2xx status code
137
- if (parsedResponse.statusCode < 200 || parsedResponse.statusCode > 299) {
138
- throw Error('Smoke test request failed')
139
- }
140
- } catch (e: unknown) {
141
- errors.push(e)
142
- }
143
- }
144
- if (errors.length > 0) {
145
- return res.status(500).send(errors)
146
- }
147
- return res.status(200).send('OK')
148
- })
149
- }
@@ -1,23 +0,0 @@
1
- export enum HttpRequestType {
2
- CACHE_HIT = 'cacheHit',
3
- DATA_PROVIDER_HIT = 'dataProviderHit',
4
- ADAPTER_ERROR = 'adapterError',
5
- INPUT_ERROR = 'inputError',
6
- RATE_LIMIT_ERROR = 'rateLimitError',
7
- // BURST_LIMIT_ERROR = 'burstLimitError',
8
- // BACKOFF_ERROR = 'backoffError',
9
- DP_ERROR = 'dataProviderError',
10
- TIMEOUT_ERROR = 'timeoutError',
11
- CONNECTION_ERROR = 'connectionError',
12
- // RES_EMPTY_ERROR = 'responseEmptyError',
13
- // RES_INVALID_ERROR = 'responseInvalidError',
14
- CUSTOM_ERROR = 'customError',
15
- }
16
-
17
- /**
18
- * Maxiumum number of characters that a feedId can contain.
19
- */
20
- export const MAX_FEED_ID_LENGTH = 300
21
-
22
- // We should tune these as we collect data, this is the default bucket distribution that prom comes with
23
- export const requestDurationBuckets = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]
@@ -1,115 +0,0 @@
1
- import * as client from 'prom-client'
2
- import { HttpRequestType, requestDurationBuckets } from './constants'
3
- import { AdapterRequest, makeLogger } from '../util'
4
- import { AdapterConfig } from '../config'
5
- import fastify, { FastifyReply, HookHandlerDoneFunction } from 'fastify'
6
- import { join } from 'path'
7
- import { AdapterError } from '../validation/error'
8
-
9
- const logger = makeLogger('Metrics')
10
-
11
- export function setupMetricsServer(name: string, config: AdapterConfig) {
12
- const metricsApp = fastify({
13
- logger: false,
14
- })
15
- const metricsPort = config.METRICS_PORT
16
- const endpoint = config.METRICS_USE_BASE_URL ? join(config.BASE_URL, 'metrics') : '/metrics'
17
- const eaHost = config.EA_HOST
18
-
19
- setupMetrics(name, config)
20
-
21
- metricsApp.get(endpoint, async (_, res) => {
22
- res.type('txt')
23
- res.send(await client.register.metrics())
24
- })
25
-
26
- metricsApp.listen(metricsPort, eaHost, () =>
27
- logger.info(`Monitoring listening on port ${metricsPort}!`),
28
- )
29
- }
30
-
31
- export const setupMetrics = (name: string, config: AdapterConfig): void => {
32
- client.collectDefaultMetrics()
33
- client.register.setDefaultLabels({
34
- app_name: config.METRICS_NAME || name || 'N/A',
35
- app_version: config['npm_package_version'],
36
- })
37
- }
38
-
39
- /**
40
- * Builds metrics middleware that records end to end EA response times
41
- * and count of requests
42
- *
43
- * @returns the cache middleware function
44
- */
45
- export const buildMetricsMiddleware = (
46
- req: AdapterRequest,
47
- res: FastifyReply,
48
- done: HookHandlerDoneFunction,
49
- ) => {
50
- const feedId = req.requestContext.meta?.metrics?.feedId || 'N/A'
51
- const labels = buildHttpRequestMetricsLabel(
52
- feedId,
53
- req.requestContext.meta?.error,
54
- req.requestContext.meta?.metrics?.cacheHit,
55
- )
56
-
57
- // Record number of requests sent to EA
58
- httpRequestsTotal.labels(labels).inc()
59
-
60
- // Record response time of request through entire EA
61
- httpRequestDurationSeconds.observe(res.getResponseTime())
62
- logger.debug(`Response time for ${feedId}: ${res.getResponseTime()}ms`)
63
- done()
64
- }
65
-
66
- const buildHttpRequestMetricsLabel = (
67
- feedId: string,
68
- error?: AdapterError | Error,
69
- cacheHit?: boolean,
70
- ): Parameters<typeof httpRequestsTotal.labels>[0] => {
71
- const labels: Parameters<typeof httpRequestsTotal.labels>[0] = {}
72
- labels.method = 'POST'
73
- labels.feed_id = feedId
74
- if (error instanceof AdapterError) {
75
- // If error present and an instace of AdapterError, build label from error info
76
- labels.type = error?.metricsLabel || HttpRequestType.ADAPTER_ERROR
77
- labels.status_code = error?.statusCode
78
- labels.provider_status_code = error?.providerStatusCode
79
- } else if (error instanceof Error) {
80
- // If error present and not instance of generic Error, unexpected failure occurred
81
- labels.type = HttpRequestType.ADAPTER_ERROR
82
- labels.status_code = 500
83
- } else {
84
- // If no error present, request went as expected
85
- labels.status_code = 200
86
- if (cacheHit) {
87
- labels.type = HttpRequestType.CACHE_HIT
88
- } else {
89
- labels.type = HttpRequestType.DATA_PROVIDER_HIT
90
- labels.provider_status_code = 200
91
- }
92
- }
93
-
94
- return labels
95
- }
96
-
97
- export const httpRequestsTotal = new client.Counter({
98
- name: 'http_requests_total',
99
- help: 'The number of http requests this external adapter has serviced for its entire uptime',
100
- labelNames: [
101
- 'method',
102
- 'status_code',
103
- 'retry',
104
- 'type',
105
- 'is_cache_warming',
106
- 'feed_id',
107
- 'provider_status_code',
108
- ] as const,
109
- })
110
-
111
- export const httpRequestDurationSeconds = new client.Histogram({
112
- name: 'http_request_duration_seconds',
113
- help: 'A histogram bucket of the distribution of http request durations',
114
- buckets: requestDurationBuckets,
115
- })
@@ -1,18 +0,0 @@
1
- import { AdapterMetricsMeta, AdapterRequestData } from '../util'
2
- import { AdapterEndpoint } from '../adapter'
3
- import { calculateFeedId } from '../cache'
4
- import { AdapterConfig } from '../config'
5
-
6
- export const getMetricsMeta = (
7
- {
8
- adapterEndpoint,
9
- adapterConfig,
10
- }: {
11
- adapterEndpoint: AdapterEndpoint
12
- adapterConfig: AdapterConfig
13
- },
14
- data: AdapterRequestData,
15
- ): AdapterMetricsMeta => {
16
- const feedId = calculateFeedId({ adapterEndpoint, adapterConfig }, data)
17
- return { feedId }
18
- }
@@ -1,45 +0,0 @@
1
- import { AdapterRateLimitTier, BackgroundExecuteRateLimiter, consolidateTierLimits } from '..'
2
- import { AdapterEndpoint } from '../../adapter'
3
- import { makeLogger } from '../../util'
4
-
5
- const logger = makeLogger('FixedFrequencyRateLimiter')
6
- export const DEFAULT_SHARED_MS_BETWEEN_REQUESTS = 5000
7
-
8
- export class FixedFrequencyRateLimiter implements BackgroundExecuteRateLimiter {
9
- msBetweenRequestsMap: {
10
- [endpointName: string]: number // In ms
11
- } = {}
12
-
13
- initialize(endpoints: AdapterEndpoint[], limits?: AdapterRateLimitTier) {
14
- // Translate the hourly limit into reqs per minute
15
- let sharedMsBetweenRequests = 1000 / consolidateTierLimits(limits)
16
-
17
- // If there is no limit set, we use some reasonable number
18
- if (!limits?.rateLimit1h && !limits?.rateLimit1m && !limits?.rateLimit1s) {
19
- // 5s period for all seems good
20
- sharedMsBetweenRequests = DEFAULT_SHARED_MS_BETWEEN_REQUESTS
21
- }
22
-
23
- logger.debug('Using fixed frequency batch rate limiting')
24
- for (const endpoint of endpoints) {
25
- if (endpoint.rateLimiting?.allocationPercentage == null) {
26
- throw new Error(`Allocation percentage for endpoint "${endpoint.name}" is null`)
27
- }
28
-
29
- this.msBetweenRequestsMap[endpoint.name] =
30
- (sharedMsBetweenRequests / endpoint.rateLimiting?.allocationPercentage) * 100
31
-
32
- logger.debug(
33
- `Endpoint [${endpoint.name}]: ${
34
- this.msBetweenRequestsMap[endpoint.name] / 1000
35
- }s between requests`,
36
- )
37
- }
38
-
39
- return this
40
- }
41
-
42
- msUntilNextExecution(endpointName: string): number {
43
- return this.msBetweenRequestsMap[endpointName]
44
- }
45
- }
@@ -1,100 +0,0 @@
1
- import { AdapterEndpoint } from '../adapter'
2
-
3
- export * from './request/simple-counting'
4
- export * from './background/fixed-frequency'
5
-
6
- export interface AdapterRateLimitTier {
7
- rateLimit1s?: number
8
- rateLimit1m?: number
9
- rateLimit1h?: number
10
- note?: string
11
- }
12
-
13
- /**
14
- * Common interface for all RateLimiter classes to implement
15
- */
16
- export interface RateLimiter {
17
- /**
18
- * Method to ensure all RateLimiters can be initialized in the same manner.
19
- *
20
- * @param limits - settings for how much throughput to allow for the Adapter
21
- * @param endpoints - list of adapter endpoints
22
- */
23
- initialize(endpoints: AdapterEndpoint[], limits: AdapterRateLimitTier): this
24
- }
25
-
26
- /**
27
- * RequestRateLimiters perform checks agains imminent outbound requests for any transport.
28
- */
29
- export interface RequestRateLimiter extends RateLimiter {
30
- /**
31
- * This method will check using whatever strategy is implemented to determine if
32
- * this request can be processed. If so, it returns true; if not, returns false.
33
- */
34
- isUnderLimits(): boolean
35
- }
36
-
37
- /**
38
- * BackgroundExecuteFrequencyRateLimiters will implement custom logic to calculate
39
- * the period of time to wait between background executions for a transport.
40
- */
41
- export interface BackgroundExecuteRateLimiter extends RateLimiter {
42
- msUntilNextExecution(endpointName: string): number
43
- }
44
-
45
- /**
46
- * This method will convert all possible settings for a rate limit tier and
47
- * convert them all to requests per second, returning the lowest one
48
- *
49
- * @param limits - the rate limit tier set for the adapter
50
- * @returns the most restrictive of the set options, in requests per second
51
- */
52
- export const consolidateTierLimits = (limits?: AdapterRateLimitTier) => {
53
- const perHourLimit = (limits?.rateLimit1h || Infinity) / (60 * 60)
54
- const perMinuteLimit = (limits?.rateLimit1m || Infinity) / 60
55
- const perSecondLimit = limits?.rateLimit1s || Infinity
56
- return Math.min(perHourLimit, perMinuteLimit, perSecondLimit)
57
- }
58
-
59
- /**
60
- * Validates rate limiting tiers specified for the adapter, and returns the one to use.
61
- *
62
- * @param tiers - the adapter config listing the different available API tiers
63
- * @param selectedTier - chosen API tier from settings, if present
64
- * @returns the specified API tier, or a default one if none are specified
65
- */
66
- export const getRateLimitingTier = (
67
- tiers?: Record<string, AdapterRateLimitTier>,
68
- selectedTier?: string,
69
- ): AdapterRateLimitTier | undefined => {
70
- if (!tiers) {
71
- return
72
- }
73
-
74
- // Check that if the tiers object is defined, it has values
75
- if (Object.values(tiers).length === 0) {
76
- throw new Error(`The tiers object is defined, but has no entries`)
77
- }
78
-
79
- // Check that the tier set in the AdapterConfig is a valid one
80
- if (selectedTier && !tiers[selectedTier]) {
81
- const validTiersString = Object.keys(tiers)
82
- .map((t) => `"${t}"`)
83
- .join(', ')
84
-
85
- throw new Error(
86
- `The selected rate limit tier "${selectedTier}" is not valid (can be one of ${validTiersString})`,
87
- )
88
- }
89
-
90
- if (!selectedTier) {
91
- // Sort the tiers by most to least restrictive
92
- const sortedTiers = Object.values(tiers).sort(
93
- (t1, t2) => consolidateTierLimits(t1) - consolidateTierLimits(t2),
94
- )
95
-
96
- return sortedTiers[0]
97
- }
98
-
99
- return tiers[selectedTier]
100
- }
@@ -1,18 +0,0 @@
1
- import * as client from 'prom-client'
2
-
3
- // Retrieve cost field from response if exists
4
- // If not return default cost of 1
5
- export const retrieveCost = <ProviderResponseBody>(data: ProviderResponseBody): number => {
6
- const cost = (data as Record<string, unknown>)['cost']
7
- if (typeof cost === 'number' || typeof cost === 'string') {
8
- return Number(cost)
9
- } else {
10
- return 1
11
- }
12
- }
13
-
14
- export const rateLimitCreditsSpentTotal = new client.Counter({
15
- name: 'rate_limit_credits_spent_total',
16
- help: 'The number of data provider credits the adapter is consuming',
17
- labelNames: ['participant_id', 'feed_id'] as const,
18
- })
@@ -1,76 +0,0 @@
1
- import { AdapterRateLimitTier, RateLimiter } from '..'
2
- import { AdapterEndpoint } from '../../adapter'
3
- import { makeLogger } from '../../util'
4
-
5
- const logger = makeLogger('SimpleCountingRateLimiter')
6
-
7
- /**
8
- * This rate limiter is the simplest stateful option.
9
- * On startup, it'll compare the different thresholds for each tier, calculate them all
10
- * in the finest window we'll use (seconds), and use the most restrictive one.
11
- * This is so if the EA were to restart, we don't need to worry about persisting state
12
- * for things like daily quotas. The downside is that this does not work well for bursty
13
- * loads or spikes, in cases where e.g. the per second limit is high but daily quotas low.
14
- */
15
- export class SimpleCountingRateLimiter implements RateLimiter {
16
- latestSecondInterval = 0
17
- requestsThisSecond = 0
18
- latestMinuteInterval = 0
19
- requestsThisMinute = 0
20
- perSecondLimit!: number
21
- perMinuteLimit!: number
22
-
23
- initialize(endpoints: AdapterEndpoint[], limits?: AdapterRateLimitTier) {
24
- // Translate the hourly limit into reqs per minute
25
- const perHourLimit = (limits?.rateLimit1h || Infinity) / 60
26
- this.perMinuteLimit = Math.min(limits?.rateLimit1m || Infinity, perHourLimit)
27
- this.perSecondLimit = limits?.rateLimit1s || Infinity
28
- logger.debug(
29
- `Using rate limiting settings: perMinute = ${this.perMinuteLimit} | perSecond: = ${this.perSecondLimit}`,
30
- )
31
-
32
- return this
33
- }
34
-
35
- isUnderLimits() {
36
- // If the limit is set to infinity, there was no tier limit specified
37
- if (this.perSecondLimit === Infinity && this.perMinuteLimit === Infinity) {
38
- return true
39
- }
40
-
41
- const now = Date.now()
42
- const nearestSecondInterval = Math.floor(now / 1000)
43
- const nearestMinuteInterval = Math.floor(now / (1000 * 60))
44
-
45
- // This should always run to completion, even if it doesn't look atomic; therefore the
46
- // Ops should be "thread safe". Thank JS and its infinite single threaded dumbness.
47
- if (nearestSecondInterval !== this.latestSecondInterval) {
48
- logger.trace(
49
- `Clearing latest second interval, # of requests logged was: ${this.requestsThisSecond} `,
50
- )
51
- this.latestSecondInterval = nearestSecondInterval
52
- this.requestsThisSecond = 0
53
- }
54
-
55
- if (nearestMinuteInterval !== this.latestMinuteInterval) {
56
- logger.trace(
57
- `Clearing latest second minute, # of requests logged was: ${this.requestsThisMinute} `,
58
- )
59
- this.latestMinuteInterval = nearestMinuteInterval
60
- this.requestsThisMinute = 0
61
- }
62
-
63
- if (
64
- this.requestsThisSecond < this.perSecondLimit &&
65
- this.requestsThisMinute < this.perMinuteLimit
66
- ) {
67
- logger.trace('Request under limits, counting +1')
68
- this.requestsThisSecond++
69
- this.requestsThisMinute++
70
- return true
71
- } else {
72
- logger.trace('Requests seen this interval are above limits')
73
- return false
74
- }
75
- }
76
- }
package/src/test.ts DELETED
@@ -1,5 +0,0 @@
1
- import { expose } from '.'
2
- import { adapter } from './examples/bank-frick'
3
-
4
- // Start sample adapter
5
- expose(adapter)
@@ -1,122 +0,0 @@
1
- import { AxiosRequestConfig, AxiosResponse } from 'axios'
2
- import { Cache } from '../cache'
3
- import { AdapterConfig, SettingsMap } from '../config'
4
- import { BackgroundExecuteRateLimiter } from '../rate-limiting'
5
- import { makeLogger, SubscriptionSet } from '../util'
6
- import { AdapterRequest, ProviderResult } from '../util/request'
7
- import { Transport, buildCacheEntriesFromResults } from './'
8
- import { axiosRequest } from './util'
9
- import * as rateLimitMetrics from '../rate-limiting/metrics'
10
- import * as cacheMetrics from '../cache/metrics'
11
- import { AdapterContext, AdapterDependencies } from '../adapter'
12
-
13
- const WARMUP_BATCH_REQUEST_ID = '9002'
14
-
15
- const logger = makeLogger('BatchWarmingTransport')
16
-
17
- /**
18
- * Transport implementation that takes incoming batches requests and keeps a warm cache of values.
19
- * Within the setup function, adapter params are added to an set that also keeps track and expires values.
20
- * In the background execute, the list of non-expired items in the set is fetched.
21
- * Then, the list is passed through the `prepareRequest` function, that returns an AxiosRequestConfig.
22
- * The Data Provider response is, they are passed through the `parseResponse` function to create a [[CacheEntry]] list.
23
- * Finally, the items in that [[CacheEntry]] list are set in the Cache so the Adapter can fetch values from there.
24
- *
25
- * @typeParam AdapterParams - interface for the adapter request body
26
- * @typeParam ProviderRequestBody - interface for the body of the request to the Data Provider
27
- * @typeParam ProviderResponseBody - interface for the body of the Data Provider's response
28
- */
29
- export class BatchWarmingTransport<
30
- AdapterParams,
31
- ProviderRequestBody,
32
- ProviderResponseBody,
33
- CustomSettings extends SettingsMap,
34
- > implements Transport<AdapterParams, ProviderResponseBody, CustomSettings>
35
- {
36
- cache!: Cache
37
- rateLimiter!: BackgroundExecuteRateLimiter
38
- subscriptionSet!: SubscriptionSet<AdapterParams>
39
-
40
- // Flag used to track whether the warmer has moved from having no entries to having some and vice versa
41
- // Used for recording the cache warmer active metrics accurately
42
- WARMER_ACTIVE = false
43
-
44
- constructor(
45
- private config: {
46
- prepareRequest: (
47
- params: AdapterParams[],
48
- context: AdapterContext<CustomSettings>,
49
- ) => AxiosRequestConfig<ProviderRequestBody>
50
- parseResponse: (
51
- res: AxiosResponse<ProviderResponseBody>,
52
- context: AdapterContext<CustomSettings>,
53
- ) => ProviderResult<AdapterParams>[]
54
- },
55
- ) {}
56
-
57
- async initialize(dependencies: AdapterDependencies): Promise<void> {
58
- this.cache = dependencies.cache
59
- this.rateLimiter = dependencies.backgroundExecuteRateLimiter
60
- this.subscriptionSet = dependencies.subscriptionSetFactory.buildSet()
61
- }
62
-
63
- async hasBeenSetUp(req: AdapterRequest<AdapterParams>): Promise<boolean> {
64
- return !!(await this.subscriptionSet.get(req.requestContext.cacheKey))
65
- }
66
-
67
- async setup(
68
- req: AdapterRequest<AdapterParams>,
69
- config: AdapterConfig<CustomSettings>,
70
- ): Promise<void> {
71
- logger.debug(
72
- `Adding entry to batch warming set: [${req.requestContext.cacheKey}] = ${req.requestContext.data}`,
73
- )
74
- await this.subscriptionSet.add(
75
- req.requestContext.cacheKey,
76
- req.requestContext.data,
77
- config.WARMUP_SUBSCRIPTION_TTL,
78
- )
79
- }
80
-
81
- async backgroundExecute(context: AdapterContext<CustomSettings>): Promise<number> {
82
- logger.debug('Starting background execute')
83
- const entries = await this.subscriptionSet.getAll()
84
-
85
- if (!entries.length) {
86
- logger.debug('No entries in batch warming set, skipping')
87
- if (this.WARMER_ACTIVE) {
88
- // Decrement count when warmer changed from having entries to having none
89
- cacheMetrics.cacheWarmerCount.labels({ isBatched: 'true' }).dec()
90
- this.WARMER_ACTIVE = false
91
- }
92
- return this.rateLimiter.msUntilNextExecution(context.adapterEndpoint.name)
93
- } else if (this.WARMER_ACTIVE === false) {
94
- // Increment count when warmer changed from having no entries to having some
95
- cacheMetrics.cacheWarmerCount.labels({ isBatched: 'true' }).inc()
96
- this.WARMER_ACTIVE = true
97
- }
98
-
99
- const request = this.config.prepareRequest(entries, context)
100
-
101
- logger.trace('Sending request to data provider...')
102
- const providerResponse = await axiosRequest<ProviderRequestBody, ProviderResponseBody>(request)
103
-
104
- logger.debug(`Got response from provider, parsing (raw body: ${providerResponse.data})`)
105
- const results = this.config.parseResponse(providerResponse, context)
106
- const adapterResponses = buildCacheEntriesFromResults(results, context)
107
-
108
- logger.debug('Setting adapter responses in cache')
109
- await this.cache.setMany(adapterResponses, context.adapterConfig.CACHE_MAX_AGE)
110
-
111
- // Record cost of data provider call
112
- const cost = rateLimitMetrics.retrieveCost(providerResponse.data)
113
- rateLimitMetrics.rateLimitCreditsSpentTotal
114
- .labels({
115
- feed_id: 'N/A',
116
- participant_id: WARMUP_BATCH_REQUEST_ID,
117
- })
118
- .inc(cost)
119
-
120
- return this.rateLimiter.msUntilNextExecution(context.adapterEndpoint.name)
121
- }
122
- }