@chainlink/external-adapter-framework 0.0.6 → 0.0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. package/adapter.d.ts +88 -0
  2. package/adapter.js +112 -0
  3. package/background-executor.d.ts +11 -0
  4. package/background-executor.js +45 -0
  5. package/cache/factory.d.ts +6 -0
  6. package/cache/factory.js +57 -0
  7. package/cache/index.d.ts +90 -0
  8. package/cache/index.js +169 -0
  9. package/cache/local.d.ts +23 -0
  10. package/cache/local.js +83 -0
  11. package/cache/metrics.d.ts +27 -0
  12. package/cache/metrics.js +120 -0
  13. package/cache/redis.d.ts +16 -0
  14. package/cache/redis.js +100 -0
  15. package/chainlink-external-adapter-framework-v0.0.8.tgz +0 -0
  16. package/config/index.d.ts +195 -0
  17. package/config/index.js +365 -0
  18. package/config/provider-limits.d.ts +31 -0
  19. package/config/provider-limits.js +75 -0
  20. package/examples/coingecko/batch-warming.d.ts +2 -0
  21. package/examples/coingecko/batch-warming.js +52 -0
  22. package/examples/coingecko/index.d.ts +2 -0
  23. package/examples/coingecko/index.js +10 -0
  24. package/examples/coingecko/rest.d.ts +2 -0
  25. package/examples/coingecko/rest.js +50 -0
  26. package/examples/ncfx/config/index.d.ts +12 -0
  27. package/examples/ncfx/config/index.js +15 -0
  28. package/examples/ncfx/index.d.ts +2 -0
  29. package/examples/ncfx/index.js +10 -0
  30. package/examples/ncfx/websocket.d.ts +36 -0
  31. package/examples/ncfx/websocket.js +72 -0
  32. package/index.d.ts +12 -0
  33. package/index.js +92 -0
  34. package/metrics/constants.d.ts +16 -0
  35. package/metrics/constants.js +25 -0
  36. package/metrics/index.d.ts +15 -0
  37. package/metrics/index.js +123 -0
  38. package/metrics/util.d.ts +3 -0
  39. package/metrics/util.js +9 -0
  40. package/package/adapter.d.ts +88 -0
  41. package/package/adapter.js +112 -0
  42. package/package/background-executor.d.ts +11 -0
  43. package/package/background-executor.js +45 -0
  44. package/package/cache/factory.d.ts +6 -0
  45. package/package/cache/factory.js +57 -0
  46. package/package/cache/index.d.ts +90 -0
  47. package/package/cache/index.js +169 -0
  48. package/package/cache/local.d.ts +23 -0
  49. package/package/cache/local.js +83 -0
  50. package/package/cache/metrics.d.ts +27 -0
  51. package/package/cache/metrics.js +120 -0
  52. package/package/cache/redis.d.ts +16 -0
  53. package/package/cache/redis.js +100 -0
  54. package/package/config/index.d.ts +195 -0
  55. package/package/config/index.js +365 -0
  56. package/package/config/provider-limits.d.ts +31 -0
  57. package/package/config/provider-limits.js +75 -0
  58. package/package/examples/coingecko/batch-warming.d.ts +2 -0
  59. package/package/examples/coingecko/batch-warming.js +52 -0
  60. package/package/examples/coingecko/index.d.ts +2 -0
  61. package/package/examples/coingecko/index.js +10 -0
  62. package/package/examples/coingecko/rest.d.ts +2 -0
  63. package/package/examples/coingecko/rest.js +50 -0
  64. package/package/examples/ncfx/config/index.d.ts +12 -0
  65. package/package/examples/ncfx/config/index.js +15 -0
  66. package/package/examples/ncfx/index.d.ts +2 -0
  67. package/package/examples/ncfx/index.js +10 -0
  68. package/package/examples/ncfx/websocket.d.ts +36 -0
  69. package/package/examples/ncfx/websocket.js +72 -0
  70. package/package/index.d.ts +12 -0
  71. package/package/index.js +92 -0
  72. package/package/metrics/constants.d.ts +16 -0
  73. package/package/metrics/constants.js +25 -0
  74. package/package/metrics/index.d.ts +15 -0
  75. package/package/metrics/index.js +123 -0
  76. package/package/metrics/util.d.ts +3 -0
  77. package/package/metrics/util.js +9 -0
  78. package/package/package.json +69 -0
  79. package/package/rate-limiting/background/fixed-frequency.d.ts +10 -0
  80. package/package/rate-limiting/background/fixed-frequency.js +37 -0
  81. package/package/rate-limiting/index.d.ts +54 -0
  82. package/package/rate-limiting/index.js +63 -0
  83. package/package/rate-limiting/metrics.d.ts +3 -0
  84. package/package/rate-limiting/metrics.js +44 -0
  85. package/package/rate-limiting/request/simple-counting.d.ts +20 -0
  86. package/package/rate-limiting/request/simple-counting.js +62 -0
  87. package/package/test.d.ts +1 -0
  88. package/package/test.js +6 -0
  89. package/package/transports/batch-warming.d.ts +34 -0
  90. package/package/transports/batch-warming.js +101 -0
  91. package/package/transports/index.d.ts +87 -0
  92. package/package/transports/index.js +87 -0
  93. package/package/transports/metrics.d.ts +21 -0
  94. package/package/transports/metrics.js +105 -0
  95. package/package/transports/rest.d.ts +43 -0
  96. package/package/transports/rest.js +129 -0
  97. package/package/transports/util.d.ts +8 -0
  98. package/package/transports/util.js +85 -0
  99. package/package/transports/websocket.d.ts +80 -0
  100. package/package/transports/websocket.js +169 -0
  101. package/package/util/expiring-sorted-set.d.ts +21 -0
  102. package/package/util/expiring-sorted-set.js +47 -0
  103. package/package/util/index.d.ts +11 -0
  104. package/package/util/index.js +35 -0
  105. package/package/util/logger.d.ts +42 -0
  106. package/package/util/logger.js +62 -0
  107. package/package/util/request.d.ts +55 -0
  108. package/package/util/request.js +2 -0
  109. package/package/validation/error.d.ts +50 -0
  110. package/package/validation/error.js +79 -0
  111. package/package/validation/index.d.ts +5 -0
  112. package/package/validation/index.js +86 -0
  113. package/package/validation/input-params.d.ts +15 -0
  114. package/package/validation/input-params.js +30 -0
  115. package/package/validation/override-functions.d.ts +3 -0
  116. package/package/validation/override-functions.js +40 -0
  117. package/package/validation/preset-tokens.json +23 -0
  118. package/package/validation/validator.d.ts +47 -0
  119. package/package/validation/validator.js +303 -0
  120. package/package.json +1 -1
  121. package/rate-limiting/background/fixed-frequency.d.ts +10 -0
  122. package/rate-limiting/background/fixed-frequency.js +37 -0
  123. package/rate-limiting/index.d.ts +54 -0
  124. package/rate-limiting/index.js +63 -0
  125. package/rate-limiting/metrics.d.ts +3 -0
  126. package/rate-limiting/metrics.js +44 -0
  127. package/rate-limiting/request/simple-counting.d.ts +20 -0
  128. package/rate-limiting/request/simple-counting.js +62 -0
  129. package/test.d.ts +1 -0
  130. package/test.js +6 -0
  131. package/transports/batch-warming.d.ts +34 -0
  132. package/transports/batch-warming.js +101 -0
  133. package/transports/index.d.ts +87 -0
  134. package/transports/index.js +87 -0
  135. package/transports/metrics.d.ts +21 -0
  136. package/transports/metrics.js +105 -0
  137. package/transports/rest.d.ts +43 -0
  138. package/transports/rest.js +129 -0
  139. package/transports/util.d.ts +8 -0
  140. package/transports/util.js +85 -0
  141. package/transports/websocket.d.ts +80 -0
  142. package/transports/websocket.js +169 -0
  143. package/util/expiring-sorted-set.d.ts +21 -0
  144. package/util/expiring-sorted-set.js +47 -0
  145. package/util/index.d.ts +11 -0
  146. package/util/index.js +35 -0
  147. package/util/logger.d.ts +42 -0
  148. package/util/logger.js +62 -0
  149. package/util/request.d.ts +55 -0
  150. package/util/request.js +2 -0
  151. package/validation/error.d.ts +50 -0
  152. package/validation/error.js +79 -0
  153. package/validation/index.d.ts +5 -0
  154. package/validation/index.js +86 -0
  155. package/validation/input-params.d.ts +15 -0
  156. package/validation/input-params.js +30 -0
  157. package/validation/override-functions.d.ts +3 -0
  158. package/validation/override-functions.js +40 -0
  159. package/validation/preset-tokens.json +23 -0
  160. package/validation/validator.d.ts +47 -0
  161. package/validation/validator.js +303 -0
  162. package/.c8rc.json +0 -3
  163. package/.eslintignore +0 -9
  164. package/.eslintrc.js +0 -96
  165. package/.github/README.MD +0 -17
  166. package/.github/actions/setup/action.yaml +0 -13
  167. package/.github/workflows/main.yaml +0 -39
  168. package/.github/workflows/publish.yaml +0 -17
  169. package/.prettierignore +0 -13
  170. package/.yarnrc +0 -0
  171. package/README.md +0 -103
  172. package/docker-compose.yaml +0 -35
  173. package/src/adapter.ts +0 -236
  174. package/src/background-executor.ts +0 -53
  175. package/src/cache/factory.ts +0 -28
  176. package/src/cache/index.ts +0 -236
  177. package/src/cache/local.ts +0 -73
  178. package/src/cache/metrics.ts +0 -112
  179. package/src/cache/redis.ts +0 -93
  180. package/src/config/index.ts +0 -501
  181. package/src/config/provider-limits.ts +0 -130
  182. package/src/examples/coingecko/batch-warming.ts +0 -79
  183. package/src/examples/coingecko/index.ts +0 -9
  184. package/src/examples/coingecko/rest.ts +0 -77
  185. package/src/examples/ncfx/config/index.ts +0 -12
  186. package/src/examples/ncfx/index.ts +0 -9
  187. package/src/examples/ncfx/websocket.ts +0 -100
  188. package/src/index.ts +0 -106
  189. package/src/metrics/constants.ts +0 -23
  190. package/src/metrics/index.ts +0 -116
  191. package/src/metrics/util.ts +0 -11
  192. package/src/rate-limiting/background/fixed-frequency.ts +0 -47
  193. package/src/rate-limiting/index.ts +0 -100
  194. package/src/rate-limiting/metrics.ts +0 -18
  195. package/src/rate-limiting/request/simple-counting.ts +0 -76
  196. package/src/test.ts +0 -5
  197. package/src/transports/batch-warming.ts +0 -121
  198. package/src/transports/index.ts +0 -173
  199. package/src/transports/metrics.ts +0 -95
  200. package/src/transports/rest.ts +0 -161
  201. package/src/transports/util.ts +0 -63
  202. package/src/transports/websocket.ts +0 -238
  203. package/src/util/expiring-sorted-set.ts +0 -52
  204. package/src/util/index.ts +0 -20
  205. package/src/util/logger.ts +0 -69
  206. package/src/util/request.ts +0 -115
  207. package/src/validation/error.ts +0 -116
  208. package/src/validation/index.ts +0 -101
  209. package/src/validation/input-params.ts +0 -45
  210. package/src/validation/override-functions.ts +0 -44
  211. package/src/validation/preset-tokens.json +0 -23
  212. package/src/validation/validator.ts +0 -384
  213. package/test/adapter.test.ts +0 -27
  214. package/test/background-executor.test.ts +0 -109
  215. package/test/cache/cache-key.test.ts +0 -96
  216. package/test/cache/helper.ts +0 -101
  217. package/test/cache/local.test.ts +0 -54
  218. package/test/cache/redis.test.ts +0 -89
  219. package/test/correlation.test.ts +0 -114
  220. package/test/index.test.ts +0 -37
  221. package/test/metrics/feed-id.test.ts +0 -33
  222. package/test/metrics/helper.ts +0 -14
  223. package/test/metrics/labels.test.ts +0 -36
  224. package/test/metrics/metrics.test.ts +0 -267
  225. package/test/metrics/redis-metrics.test.ts +0 -113
  226. package/test/metrics/warmer-metrics.test.ts +0 -192
  227. package/test/metrics/ws-metrics.test.ts +0 -225
  228. package/test/rate-limit-config.test.ts +0 -243
  229. package/test/transports/batch.test.ts +0 -465
  230. package/test/transports/rest.test.ts +0 -242
  231. package/test/transports/websocket.test.ts +0 -183
  232. package/test/tsconfig.json +0 -5
  233. package/test/util.ts +0 -76
  234. package/test/validation.test.ts +0 -169
  235. package/test.sh +0 -20
  236. package/tsconfig.json +0 -24
  237. package/typedoc.json +0 -6
@@ -1,47 +0,0 @@
1
- import { AdapterRateLimitTier, BackgroundExecuteRateLimiter, consolidateTierLimits } from '..'
2
- import { AdapterEndpoint } from '../../adapter'
3
- import { makeLogger } from '../../util'
4
-
5
- const logger = makeLogger('FixedFrequencyRateLimiter')
6
- export const DEFAULT_SHARED_MS_BETWEEN_REQUESTS = 5000
7
-
8
- export class FixedFrequencyRateLimiter implements BackgroundExecuteRateLimiter {
9
- msBetweenRequestsMap: {
10
- [endpointName: string]: number // In ms
11
- } = {}
12
-
13
- initialize(endpoints: AdapterEndpoint[], limits?: AdapterRateLimitTier) {
14
- // Translate the hourly limit into reqs per minute
15
- let sharedMsBetweenRequests = 1000 / consolidateTierLimits(limits)
16
-
17
- // If there is no limit set, we use some reasonable number
18
- if (!limits?.rateLimit1h && !limits?.rateLimit1m && !limits?.rateLimit1s) {
19
- // 5s period for all seems good
20
- sharedMsBetweenRequests = DEFAULT_SHARED_MS_BETWEEN_REQUESTS
21
- }
22
-
23
- logger.debug('Using fixed frequency batch rate limiting')
24
- for (const endpoint of endpoints) {
25
- // TODO: See if we can remove this runtime check
26
- if (endpoint.rateLimiting?.allocationPercentage == null) {
27
- throw new Error(`Allocation percentage for endpoint "${endpoint.name}" is null`)
28
- }
29
-
30
- // TODO: Implement different strategy where this is not fixed, but rather divided based on whether all warmers are active
31
- this.msBetweenRequestsMap[endpoint.name] =
32
- (sharedMsBetweenRequests / endpoint.rateLimiting?.allocationPercentage) * 100
33
-
34
- logger.debug(
35
- `Endpoint [${endpoint.name}]: ${
36
- this.msBetweenRequestsMap[endpoint.name] / 1000
37
- }s between requests`,
38
- )
39
- }
40
-
41
- return this
42
- }
43
-
44
- msUntilNextExecution(endpointName: string): number {
45
- return this.msBetweenRequestsMap[endpointName]
46
- }
47
- }
@@ -1,100 +0,0 @@
1
- import { AdapterEndpoint } from '../adapter'
2
-
3
- export * from './request/simple-counting'
4
- export * from './background/fixed-frequency'
5
-
6
- export interface AdapterRateLimitTier {
7
- rateLimit1s?: number
8
- rateLimit1m?: number
9
- rateLimit1h?: number
10
- note?: string
11
- }
12
-
13
- /**
14
- * Common interface for all RateLimiter classes to implement
15
- */
16
- export interface RateLimiter {
17
- /**
18
- * Method to ensure all RateLimiters can be initialized in the same manner.
19
- *
20
- * @param limits - settings for how much throughput to allow for the Adapter
21
- * @param endpoints - list of adapter endpoints
22
- */
23
- initialize(endpoints: AdapterEndpoint[], limits: AdapterRateLimitTier): this
24
- }
25
-
26
- /**
27
- * RequestRateLimiters perform checks agains imminent outbound requests for any transport.
28
- */
29
- export interface RequestRateLimiter extends RateLimiter {
30
- /**
31
- * This method will check using whatever strategy is implemented to determine if
32
- * this request can be processed. If so, it returns true; if not, returns false.
33
- */
34
- isUnderLimits(): boolean
35
- }
36
-
37
- /**
38
- * BackgroundExecuteFrequencyRateLimiters will implement custom logic to calculate
39
- * the period of time to wait between background executions for a transport.
40
- */
41
- export interface BackgroundExecuteRateLimiter extends RateLimiter {
42
- msUntilNextExecution(endpointName: string): number
43
- }
44
-
45
- /**
46
- * This method will convert all possible settings for a rate limit tier and
47
- * convert them all to requests per second, returning the lowest one
48
- *
49
- * @param limits - the rate limit tier set for the adapter
50
- * @returns the most restrictive of the set options, in requests per second
51
- */
52
- export const consolidateTierLimits = (limits?: AdapterRateLimitTier) => {
53
- const perHourLimit = (limits?.rateLimit1h || Infinity) / (60 * 60)
54
- const perMinuteLimit = (limits?.rateLimit1m || Infinity) / 60
55
- const perSecondLimit = limits?.rateLimit1s || Infinity
56
- return Math.min(perHourLimit, perMinuteLimit, perSecondLimit)
57
- }
58
-
59
- /**
60
- * Validates rate limiting tiers specified for the adapter, and returns the one to use.
61
- *
62
- * @param tiers - the adapter config listing the different available API tiers
63
- * @param selectedTier - chosen API tier from settings, if present
64
- * @returns the specified API tier, or a default one if none are specified
65
- */
66
- export const getRateLimitingTier = (
67
- tiers?: Record<string, AdapterRateLimitTier>,
68
- selectedTier?: string,
69
- ): AdapterRateLimitTier | undefined => {
70
- if (!tiers) {
71
- return
72
- }
73
-
74
- // Check that if the tiers object is defined, it has values
75
- if (Object.values(tiers).length === 0) {
76
- throw new Error(`The tiers object is defined, but has no entries`)
77
- }
78
-
79
- // Check that the tier set in the AdapterConfig is a valid one
80
- if (selectedTier && !tiers[selectedTier]) {
81
- const validTiersString = Object.keys(tiers)
82
- .map((t) => `"${t}"`)
83
- .join(', ')
84
-
85
- throw new Error(
86
- `The selected rate limit tier "${selectedTier}" is not valid (can be one of ${validTiersString})`,
87
- )
88
- }
89
-
90
- if (!selectedTier) {
91
- // Sort the tiers by most to least restrictive
92
- const sortedTiers = Object.values(tiers).sort(
93
- (t1, t2) => consolidateTierLimits(t1) - consolidateTierLimits(t2),
94
- )
95
-
96
- return sortedTiers[0]
97
- }
98
-
99
- return tiers[selectedTier]
100
- }
@@ -1,18 +0,0 @@
1
- import * as client from 'prom-client'
2
-
3
- // Retrieve cost field from response if exists
4
- // If not return default cost of 1
5
- export const retrieveCost = <ProviderResponseBody>(data: ProviderResponseBody): number => {
6
- const cost = (data as Record<string, unknown>)['cost']
7
- if (typeof cost === 'number' || typeof cost === 'string') {
8
- return Number(cost)
9
- } else {
10
- return 1
11
- }
12
- }
13
-
14
- export const rateLimitCreditsSpentTotal = new client.Counter({
15
- name: 'rate_limit_credits_spent_total',
16
- help: 'The number of data provider credits the adapter is consuming',
17
- labelNames: ['participant_id', 'feed_id'] as const,
18
- })
@@ -1,76 +0,0 @@
1
- import { AdapterRateLimitTier, RateLimiter } from '..'
2
- import { AdapterEndpoint } from '../../adapter'
3
- import { makeLogger } from '../../util'
4
-
5
- const logger = makeLogger('SimpleCountingRateLimiter')
6
-
7
- /**
8
- * This rate limiter is the simplest stateful option.
9
- * On startup, it'll compare the different thresholds for each tier, calculate them all
10
- * in the finest window we'll use (seconds), and use the most restrictive one.
11
- * This is so if the EA were to restart, we don't need to worry about persisting state
12
- * for things like daily quotas. The downside is that this does not work well for bursty
13
- * loads or spikes, in cases where e.g. the per second limit is high but daily quotas low.
14
- */
15
- export class SimpleCountingRateLimiter implements RateLimiter {
16
- latestSecondInterval = 0
17
- requestsThisSecond = 0
18
- latestMinuteInterval = 0
19
- requestsThisMinute = 0
20
- perSecondLimit!: number
21
- perMinuteLimit!: number
22
-
23
- initialize(endpoints: AdapterEndpoint[], limits?: AdapterRateLimitTier) {
24
- // Translate the hourly limit into reqs per minute
25
- const perHourLimit = (limits?.rateLimit1h || Infinity) / 60
26
- this.perMinuteLimit = Math.min(limits?.rateLimit1m || Infinity, perHourLimit)
27
- this.perSecondLimit = limits?.rateLimit1s || Infinity
28
- logger.debug(
29
- `Using rate limiting settings: perMinute = ${this.perMinuteLimit} | perSecond: = ${this.perSecondLimit}`,
30
- )
31
-
32
- return this
33
- }
34
-
35
- isUnderLimits() {
36
- // If the limit is set to infinity, there was no tier limit specified
37
- if (this.perSecondLimit === Infinity && this.perMinuteLimit === Infinity) {
38
- return true
39
- }
40
-
41
- const now = Date.now()
42
- const nearestSecondInterval = Math.floor(now / 1000)
43
- const nearestMinuteInterval = Math.floor(now / (1000 * 60))
44
-
45
- // This should always run to completion, even if it doesn't look atomic; therefore the
46
- // Ops should be "thread safe". Thank JS and its infinite single threaded dumbness.
47
- if (nearestSecondInterval !== this.latestSecondInterval) {
48
- logger.trace(
49
- `Clearing latest second interval, # of requests logged was: ${this.requestsThisSecond} `,
50
- )
51
- this.latestSecondInterval = nearestSecondInterval
52
- this.requestsThisSecond = 0
53
- }
54
-
55
- if (nearestMinuteInterval !== this.latestMinuteInterval) {
56
- logger.trace(
57
- `Clearing latest second minute, # of requests logged was: ${this.requestsThisMinute} `,
58
- )
59
- this.latestMinuteInterval = nearestMinuteInterval
60
- this.requestsThisMinute = 0
61
- }
62
-
63
- if (
64
- this.requestsThisSecond < this.perSecondLimit &&
65
- this.requestsThisMinute < this.perMinuteLimit
66
- ) {
67
- logger.trace('Request under limits, counting +1')
68
- this.requestsThisSecond++
69
- this.requestsThisMinute++
70
- return true
71
- } else {
72
- logger.trace('Requests seen this interval are above limits')
73
- return false
74
- }
75
- }
76
- }
package/src/test.ts DELETED
@@ -1,5 +0,0 @@
1
- import { expose } from '.'
2
- import { adapter } from './examples/coingecko'
3
-
4
- // Start sample adapter
5
- expose(adapter)
@@ -1,121 +0,0 @@
1
- import { AxiosRequestConfig, AxiosResponse } from 'axios'
2
- import { Cache } from '../cache'
3
- import { AdapterConfig, SettingsMap } from '../config'
4
- import { BackgroundExecuteRateLimiter } from '../rate-limiting'
5
- import { ExpiringSortedSet, makeLogger } from '../util'
6
- import { AdapterRequest, ProviderResult } from '../util/request'
7
- import { AdapterContext, AdapterDependencies, Transport, buildCacheEntriesFromResults } from './'
8
- import { axiosRequest } from './util'
9
- import * as rateLimitMetrics from '../rate-limiting/metrics'
10
- import * as cacheMetrics from '../cache/metrics'
11
-
12
- const WARMUP_BATCH_REQUEST_ID = '9002'
13
-
14
- const logger = makeLogger('BatchWarmingTransport')
15
-
16
- /**
17
- * Transport implementation that takes incoming batches requests and keeps a warm cache of values.
18
- * Within the setup function, adapter params are added to an set that also keeps track and expires values.
19
- * In the background execute, the list of non-expired items in the set is fetched.
20
- * Then, the list is passed through the `prepareRequest` function, that returns an AxiosRequestConfig.
21
- * The Data Provider response is, they are passed through the `parseResponse` function to create a [[CacheEntry]] list.
22
- * Finally, the items in that [[CacheEntry]] list are set in the Cache so the Adapter can fetch values from there.
23
- *
24
- * @typeParam AdapterParams - interface for the adapter request body
25
- * @typeParam ProviderRequestBody - interface for the body of the request to the Data Provider
26
- * @typeParam ProviderResponseBody - interface for the body of the Data Provider's response
27
- */
28
- export class BatchWarmingTransport<
29
- AdapterParams,
30
- ProviderRequestBody,
31
- ProviderResponseBody,
32
- CustomSettings extends SettingsMap,
33
- > implements Transport<AdapterParams, ProviderResponseBody, CustomSettings>
34
- {
35
- cache!: Cache
36
- rateLimiter!: BackgroundExecuteRateLimiter
37
-
38
- expiringSortedSet = new ExpiringSortedSet<AdapterParams>() // TODO: Move to dependencies, inject
39
-
40
- // Flag used to track whether the warmer has moved from having no entries to having some and vice versa
41
- // Used for recording the cache warmer active metrics accurately
42
- WARMER_ACTIVE = false
43
-
44
- constructor(
45
- private config: {
46
- prepareRequest: (
47
- params: AdapterParams[],
48
- context: AdapterContext<CustomSettings>,
49
- ) => AxiosRequestConfig<ProviderRequestBody>
50
- parseResponse: (
51
- res: AxiosResponse<ProviderResponseBody>,
52
- context: AdapterContext<CustomSettings>,
53
- ) => ProviderResult<AdapterParams>[]
54
- },
55
- ) {}
56
-
57
- async initialize(dependencies: AdapterDependencies): Promise<void> {
58
- this.cache = dependencies.cache
59
- this.rateLimiter = dependencies.backgroundExecuteRateLimiter
60
- }
61
-
62
- async hasBeenSetUp(req: AdapterRequest<AdapterParams>): Promise<boolean> {
63
- return !!this.expiringSortedSet.get(req.requestContext.cacheKey)
64
- }
65
-
66
- async setup(
67
- req: AdapterRequest<AdapterParams>,
68
- config: AdapterConfig<CustomSettings>,
69
- ): Promise<void> {
70
- logger.debug(
71
- `Adding entry to batch warming set: [${req.requestContext.cacheKey}] = ${req.requestContext.data}`,
72
- )
73
- this.expiringSortedSet.add(
74
- req.requestContext.cacheKey,
75
- req.requestContext.data,
76
- config.WARMUP_SUBSCRIPTION_TTL,
77
- )
78
- }
79
-
80
- async backgroundExecute(context: AdapterContext<CustomSettings>): Promise<number> {
81
- logger.debug('Starting background execute')
82
- const entries = this.expiringSortedSet.getAll()
83
-
84
- if (!entries.length) {
85
- logger.debug('No entries in batch warming set, skipping')
86
- if (this.WARMER_ACTIVE) {
87
- // Decrement count when warmer changed from having entries to having none
88
- cacheMetrics.cacheWarmerCount.labels({ isBatched: 'true' }).dec()
89
- this.WARMER_ACTIVE = false
90
- }
91
- return this.rateLimiter.msUntilNextExecution(context.adapterEndpoint.name)
92
- } else if (this.WARMER_ACTIVE === false) {
93
- // Increment count when warmer changed from having no entries to having some
94
- cacheMetrics.cacheWarmerCount.labels({ isBatched: 'true' }).inc()
95
- this.WARMER_ACTIVE = true
96
- }
97
-
98
- const request = this.config.prepareRequest(entries, context)
99
-
100
- logger.trace('Sending request to data provider...')
101
- const providerResponse = await axiosRequest<ProviderRequestBody, ProviderResponseBody>(request)
102
-
103
- logger.debug(`Got response from provider, parsing (raw body: ${providerResponse.data})`)
104
- const results = this.config.parseResponse(providerResponse, context)
105
- const adapterResponses = buildCacheEntriesFromResults(results, context)
106
-
107
- logger.debug('Setting adapter responses in cache')
108
- await this.cache.setMany(adapterResponses, context.adapterConfig.CACHE_MAX_AGE)
109
-
110
- // Record cost of data provider call
111
- const cost = rateLimitMetrics.retrieveCost(providerResponse.data)
112
- rateLimitMetrics.rateLimitCreditsSpentTotal
113
- .labels({
114
- feed_id: 'N/A',
115
- participant_id: WARMUP_BATCH_REQUEST_ID,
116
- })
117
- .inc(cost)
118
-
119
- return this.rateLimiter.msUntilNextExecution(context.adapterEndpoint.name)
120
- }
121
- }
@@ -1,173 +0,0 @@
1
- import { FastifyReply } from 'fastify'
2
- import { AdapterEndpoint, InitializedAdapter } from '../adapter'
3
- import {
4
- Cache,
5
- CacheEntry,
6
- calculateCacheKey,
7
- calculateFeedId,
8
- pollResponseFromCache,
9
- } from '../cache'
10
- import { AdapterConfig, CustomSettingsType, SettingsMap } from '../config'
11
- import { BackgroundExecuteRateLimiter, RequestRateLimiter } from '../rate-limiting'
12
- import { makeLogger } from '../util'
13
- import { AdapterRequest, AdapterResponse, ProviderResult } from '../util/request'
14
-
15
- export * from './batch-warming'
16
- export * from './rest'
17
- export * from './websocket'
18
-
19
- const logger = makeLogger('Transport')
20
-
21
- /**
22
- * Dependencies that will be injected into the Adapter on startup
23
- */
24
- export interface AdapterDependencies {
25
- cache: Cache
26
- requestRateLimiter: RequestRateLimiter
27
- backgroundExecuteRateLimiter: BackgroundExecuteRateLimiter
28
- }
29
-
30
- /**
31
- * Context that will be used on background executions of a Transport.
32
- * For example, the endpointName used to log statements or generate Cache keys.
33
- */
34
- export interface AdapterContext<
35
- CustomSettings extends CustomSettingsType<CustomSettings> = SettingsMap,
36
- > {
37
- adapterEndpoint: AdapterEndpoint
38
- adapterConfig: AdapterConfig<CustomSettings>
39
- }
40
-
41
- /**
42
- * Generic interface for a Transport.
43
- * A Transport defines the way in which an AdapterEndpoint will process incoming requests to
44
- * fetch data from a Data Provider. The setup phase will take care of the former, while the
45
- * backgroundExecute will be in charge of the latter.
46
- * This separation gives us the ability of splitting these concerns, and optionally parallelizing
47
- * the reading and writing of data to a centralized Cache.
48
- *
49
- * @typeParam Params - the structure of the AdapterRequest's body
50
- * @typeParam Result - the structure of the AdapterResponse's body
51
- */
52
- export interface Transport<Params, Result, CustomSettings extends SettingsMap> {
53
- // TODO: docs / examples to extend dependencies? e.g. JSON rpc
54
- /**
55
- * Initializes the transport in the Adapter context.
56
- *
57
- * @param dependencies - Adapter dependencies (e.g. cache instance)
58
- * @returns an empty Promise
59
- */
60
- initialize: (dependencies: AdapterDependencies) => Promise<void>
61
-
62
- /**
63
- * Checks if the Transport has already set up the incoming request.
64
- * This will likely use the cacheKey generated in the provided request.
65
- *
66
- * @param req - the incoming AdapterRequest
67
- * @returns a Promise that returns true if setup is in place
68
- */
69
- hasBeenSetUp: (req: AdapterRequest<Params>) => Promise<boolean>
70
-
71
- /**
72
- * Sets up the incoming request within the Transport.
73
- * In other words, it means that the Adapter has recognized this request, and will begin
74
- * fetching the necessary data from a Data Provider.
75
- *
76
- * @param req - the incoming AdapterRequest
77
- * @param config - common configuration for the Adapter as a whole
78
- * @returns a Promise that _optionally_ returns an AdapterResponse, if the Transport has the capability of
79
- * immediately fetching data and returning it without the background process.
80
- */
81
- setup: (
82
- req: AdapterRequest<Params>,
83
- config: AdapterConfig<CustomSettings>,
84
- ) => Promise<AdapterResponse<Result> | void>
85
-
86
- // TODO: Might need a mechanism to know if this bg execute is in flight to avoid multitple simultaneous invocations
87
- /**
88
- * If the Transport relies on an async mechanism, implementing this function will make it
89
- * so the background task executor calls it on Adapter startup.
90
- *
91
- * @param context - background context for the execution (e.g. endpoint name)
92
- */
93
- backgroundExecute?: (context: AdapterContext<CustomSettings>) => Promise<number>
94
- }
95
-
96
- /**
97
- * Helper method to build cache entries to set after getting a bunch of responses from a DP.
98
- *
99
- * @param results - a list of results coming from a DataProvider
100
- * @param context - context for the Adapter
101
- * @returns a list of CacheEntries of AdapterResponses
102
- */
103
- export const buildCacheEntriesFromResults = <Params, Context extends AdapterContext<any>>(
104
- results: ProviderResult<Params>[],
105
- context: Context,
106
- ): CacheEntry<AdapterResponse<null>>[] =>
107
- results.map((r) => {
108
- const cacheEntry = {
109
- key: calculateCacheKey(context as AdapterContext, r.params),
110
- value: {
111
- result: r.value,
112
- statusCode: 200,
113
- data: null, // TODO: Maybe add data as well?
114
- },
115
- }
116
- if (
117
- context.adapterConfig.METRICS_ENABLED &&
118
- context.adapterConfig.EXPERIMENTAL_METRICS_ENABLED
119
- ) {
120
- const metrics = {
121
- maxAge: Date.now() + context.adapterConfig.CACHE_MAX_AGE, // TODO: Replace with telemetry structure in future
122
- meta: {
123
- metrics: {
124
- feedId: calculateFeedId(context.adapterEndpoint, r.params)
125
- }
126
- }
127
- }
128
- cacheEntry.value = { ...cacheEntry.value, ...metrics }
129
- }
130
- return cacheEntry
131
- })
132
-
133
- /**
134
- * Takes an Adapter, its configuration, and its dependencies, and it creates an express middleware
135
- * that will pass along the AdapterRequest to the appropriate Transport (acc. to the endpoint in the req.)
136
- *
137
- * @param adapter - main adapter object, already initialized
138
- * @returns the transport handler middleware function
139
- */
140
- export const buildTransportHandler =
141
- (adapter: InitializedAdapter) => async (req: AdapterRequest, reply: FastifyReply) => {
142
- // Get transport, must be here because it's already checked in the validator
143
- const transport = adapter.endpointsMap[req.requestContext.endpointName].transport
144
-
145
- // Set up transport if it hasn't been done already
146
- if (!(await transport.hasBeenSetUp(req))) {
147
- logger.debug('Transport not set up yet, doing so...')
148
- const immediateResponse = await transport.setup(req, adapter.config)
149
- if (immediateResponse) {
150
- logger.debug('Got immediate response from transport, sending as response')
151
- return reply.send(immediateResponse)
152
- }
153
- }
154
- logger.debug('Transport is set up, polling cache for response...')
155
- const response = await pollResponseFromCache(
156
- adapter.dependencies.cache as Cache<AdapterResponse>,
157
- req.requestContext.cacheKey,
158
- {
159
- maxRetries: adapter.config.CACHE_POLLING_MAX_RETRIES,
160
- sleep: adapter.config.CACHE_POLLING_SLEEP_MS,
161
- },
162
- )
163
-
164
- if (response) {
165
- logger.debug('Got a response from the cache, sending that back')
166
- return reply.send(response)
167
- }
168
-
169
- logger.debug('Ran out of polling attempts, returning timeout')
170
- reply.statusCode = 504
171
-
172
- return reply.send()
173
- }
@@ -1,95 +0,0 @@
1
- import * as client from 'prom-client'
2
- import { AdapterContext } from '.'
3
- import { calculateCacheKey, calculateFeedId } from '../cache'
4
- import { requestDurationBuckets } from '../metrics/constants'
5
-
6
- // Data Provider Metrics
7
- export const dataProviderMetricsLabel = (providerStatusCode?: number, method = 'get') => ({
8
- provider_status_code: providerStatusCode,
9
- method: method.toUpperCase(),
10
- })
11
-
12
- export const dataProviderRequests = new client.Counter({
13
- name: 'data_provider_requests',
14
- help: 'The number of http requests that are made to a data provider',
15
- labelNames: ['method', 'provider_status_code'] as const,
16
- })
17
-
18
- export const dataProviderRequestDurationSeconds = new client.Histogram({
19
- name: 'data_provider_request_duration_seconds',
20
- help: 'A histogram bucket of the distribution of data provider request durations',
21
- buckets: requestDurationBuckets,
22
- })
23
-
24
- // Websocket Metrics
25
- export const connectionErrorLabels = (message: string) => ({
26
- // Key,
27
- message,
28
- })
29
-
30
- export const messageSubsLabels = (feed_id: string, cache_key: string) => ({
31
- feed_id,
32
- subscription_key: cache_key,
33
- })
34
-
35
- // Record WS message and subscription metrics
36
- // Recalculate cacheKey and feedId for metrics
37
- // since avoiding storing extra info in expiring sorted set
38
- export const recordWsMessageMetrics = <AdapterParams>(
39
- context: AdapterContext,
40
- subscribes: AdapterParams[],
41
- unsubscrices: AdapterParams[],
42
- ): void => {
43
- subscribes.forEach((param) => {
44
- const feedId = calculateFeedId(context.adapterEndpoint, param)
45
- const cacheKey = calculateCacheKey(context, param)
46
- // Record total number of ws messages sent
47
- wsMessageTotal.labels(messageSubsLabels(feedId, cacheKey)).inc()
48
-
49
- // Record total number of subscriptions made
50
- wsSubscriptionTotal.labels(messageSubsLabels(feedId, cacheKey)).inc()
51
-
52
- // Record number of active ws subscriptions
53
- wsSubscriptionActive.labels(messageSubsLabels(feedId, cacheKey)).inc()
54
- })
55
- unsubscrices.forEach((param) => {
56
- const feedId = calculateFeedId(context.adapterEndpoint, param)
57
- const cacheKey = calculateCacheKey(context, param)
58
-
59
- // Record total number of ws messages sent
60
- wsMessageTotal.labels(messageSubsLabels(feedId, cacheKey)).inc()
61
-
62
- // Record number of active ws subscriptions
63
- wsSubscriptionActive.labels(messageSubsLabels(feedId, cacheKey)).dec()
64
- })
65
- }
66
-
67
- export const wsConnectionActive = new client.Gauge({
68
- name: 'ws_connection_active',
69
- help: 'The number of active connections',
70
- labelNames: ['url'] as const,
71
- })
72
-
73
- export const wsConnectionErrors = new client.Counter({
74
- name: 'ws_connection_errors',
75
- help: 'The number of connection errors',
76
- labelNames: ['url', 'message'] as const,
77
- })
78
-
79
- export const wsSubscriptionActive = new client.Gauge({
80
- name: 'ws_subscription_active',
81
- help: 'The number of currently active subscriptions',
82
- labelNames: ['connection_url', 'feed_id', 'subscription_key'] as const,
83
- })
84
-
85
- export const wsSubscriptionTotal = new client.Counter({
86
- name: 'ws_subscription_total',
87
- help: 'The number of subscriptions opened in total',
88
- labelNames: ['connection_url', 'feed_id', 'subscription_key'] as const,
89
- })
90
-
91
- export const wsMessageTotal = new client.Counter({
92
- name: 'ws_message_total',
93
- help: 'The number of messages received in total',
94
- labelNames: ['feed_id', 'subscription_key'] as const,
95
- })