@chainlink/external-adapter-framework 0.0.6 → 0.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter.js +60 -0
- package/dist/background-executor.js +45 -0
- package/dist/cache/factory.js +57 -0
- package/dist/cache/index.js +163 -0
- package/dist/cache/local.js +83 -0
- package/dist/cache/metrics.js +114 -0
- package/dist/cache/redis.js +100 -0
- package/dist/config/index.js +364 -0
- package/dist/config/provider-limits.js +75 -0
- package/dist/examples/coingecko/batch-warming.js +52 -0
- package/dist/examples/coingecko/index.js +10 -0
- package/dist/examples/coingecko/rest.js +50 -0
- package/dist/examples/ncfx/config/index.js +15 -0
- package/dist/examples/ncfx/index.js +10 -0
- package/dist/examples/ncfx/websocket.js +72 -0
- package/dist/index.js +89 -0
- package/dist/metrics/constants.js +25 -0
- package/dist/metrics/index.js +76 -0
- package/dist/metrics/util.js +9 -0
- package/dist/rate-limiting/factory.js +33 -0
- package/dist/rate-limiting/index.js +36 -0
- package/dist/rate-limiting/metrics.js +32 -0
- package/dist/rate-limiting/nop-limiter.js +15 -0
- package/dist/rate-limiting/simple-counting.js +61 -0
- package/dist/src/adapter.js +112 -0
- package/dist/src/background-executor.js +45 -0
- package/dist/src/cache/factory.js +57 -0
- package/dist/src/cache/index.js +165 -0
- package/dist/src/cache/local.js +83 -0
- package/dist/src/cache/metrics.js +114 -0
- package/dist/src/cache/redis.js +100 -0
- package/dist/src/config/index.js +366 -0
- package/dist/src/config/provider-limits.js +75 -0
- package/dist/src/examples/bank-frick/accounts.js +191 -0
- package/dist/src/examples/bank-frick/config/index.js +45 -0
- package/dist/src/examples/bank-frick/index.js +14 -0
- package/dist/src/examples/bank-frick/util.js +39 -0
- package/dist/src/examples/coingecko/batch-warming.js +52 -0
- package/dist/src/examples/coingecko/index.js +10 -0
- package/dist/src/examples/coingecko/rest.js +50 -0
- package/dist/src/examples/ncfx/config/index.js +15 -0
- package/dist/src/examples/ncfx/index.js +10 -0
- package/dist/src/examples/ncfx/websocket.js +72 -0
- package/dist/src/index.js +89 -0
- package/dist/src/metrics/constants.js +25 -0
- package/dist/src/metrics/index.js +76 -0
- package/dist/src/metrics/util.js +9 -0
- package/dist/src/rate-limiting/background/fixed-frequency.js +37 -0
- package/dist/src/rate-limiting/index.js +63 -0
- package/dist/src/rate-limiting/metrics.js +32 -0
- package/dist/src/rate-limiting/request/simple-counting.js +62 -0
- package/dist/src/test.js +6 -0
- package/dist/src/transports/batch-warming.js +55 -0
- package/dist/src/transports/index.js +85 -0
- package/dist/src/transports/metrics.js +119 -0
- package/dist/src/transports/rest.js +93 -0
- package/dist/src/transports/util.js +85 -0
- package/dist/src/transports/websocket.js +175 -0
- package/dist/src/util/expiring-sorted-set.js +47 -0
- package/dist/src/util/index.js +35 -0
- package/dist/src/util/logger.js +62 -0
- package/dist/src/util/request.js +2 -0
- package/dist/src/validation/error.js +41 -0
- package/dist/src/validation/index.js +84 -0
- package/dist/src/validation/input-params.js +30 -0
- package/dist/src/validation/override-functions.js +40 -0
- package/dist/src/validation/preset-tokens.json +23 -0
- package/dist/src/validation/validator.js +303 -0
- package/dist/test.js +6 -0
- package/dist/transports/batch-warming.js +57 -0
- package/dist/transports/index.js +76 -0
- package/dist/transports/metrics.js +133 -0
- package/dist/transports/rest.js +91 -0
- package/dist/transports/util.js +85 -0
- package/dist/transports/websocket.js +171 -0
- package/dist/util/expiring-sorted-set.js +47 -0
- package/dist/util/index.js +35 -0
- package/dist/util/logger.js +62 -0
- package/dist/util/request.js +2 -0
- package/dist/validation/error.js +41 -0
- package/dist/validation/index.js +82 -0
- package/dist/validation/input-params.js +30 -0
- package/dist/validation/overrideFunctions.js +42 -0
- package/dist/validation/presetTokens.json +23 -0
- package/dist/validation/validator.js +303 -0
- package/package.json +6 -2
- package/.c8rc.json +0 -3
- package/.eslintignore +0 -9
- package/.eslintrc.js +0 -96
- package/.github/README.MD +0 -17
- package/.github/actions/setup/action.yaml +0 -13
- package/.github/workflows/main.yaml +0 -39
- package/.github/workflows/publish.yaml +0 -17
- package/.prettierignore +0 -13
- package/.yarnrc +0 -0
- package/docker-compose.yaml +0 -35
- package/src/adapter.ts +0 -236
- package/src/background-executor.ts +0 -53
- package/src/cache/factory.ts +0 -28
- package/src/cache/index.ts +0 -236
- package/src/cache/local.ts +0 -73
- package/src/cache/metrics.ts +0 -112
- package/src/cache/redis.ts +0 -93
- package/src/config/index.ts +0 -501
- package/src/config/provider-limits.ts +0 -130
- package/src/examples/coingecko/batch-warming.ts +0 -79
- package/src/examples/coingecko/index.ts +0 -9
- package/src/examples/coingecko/rest.ts +0 -77
- package/src/examples/ncfx/config/index.ts +0 -12
- package/src/examples/ncfx/index.ts +0 -9
- package/src/examples/ncfx/websocket.ts +0 -100
- package/src/index.ts +0 -106
- package/src/metrics/constants.ts +0 -23
- package/src/metrics/index.ts +0 -116
- package/src/metrics/util.ts +0 -11
- package/src/rate-limiting/background/fixed-frequency.ts +0 -47
- package/src/rate-limiting/index.ts +0 -100
- package/src/rate-limiting/metrics.ts +0 -18
- package/src/rate-limiting/request/simple-counting.ts +0 -76
- package/src/test.ts +0 -5
- package/src/transports/batch-warming.ts +0 -121
- package/src/transports/index.ts +0 -173
- package/src/transports/metrics.ts +0 -95
- package/src/transports/rest.ts +0 -161
- package/src/transports/util.ts +0 -63
- package/src/transports/websocket.ts +0 -238
- package/src/util/expiring-sorted-set.ts +0 -52
- package/src/util/index.ts +0 -20
- package/src/util/logger.ts +0 -69
- package/src/util/request.ts +0 -115
- package/src/validation/error.ts +0 -116
- package/src/validation/index.ts +0 -101
- package/src/validation/input-params.ts +0 -45
- package/src/validation/override-functions.ts +0 -44
- package/src/validation/preset-tokens.json +0 -23
- package/src/validation/validator.ts +0 -384
- package/test/adapter.test.ts +0 -27
- package/test/background-executor.test.ts +0 -109
- package/test/cache/cache-key.test.ts +0 -96
- package/test/cache/helper.ts +0 -101
- package/test/cache/local.test.ts +0 -54
- package/test/cache/redis.test.ts +0 -89
- package/test/correlation.test.ts +0 -114
- package/test/index.test.ts +0 -37
- package/test/metrics/feed-id.test.ts +0 -33
- package/test/metrics/helper.ts +0 -14
- package/test/metrics/labels.test.ts +0 -36
- package/test/metrics/metrics.test.ts +0 -267
- package/test/metrics/redis-metrics.test.ts +0 -113
- package/test/metrics/warmer-metrics.test.ts +0 -192
- package/test/metrics/ws-metrics.test.ts +0 -225
- package/test/rate-limit-config.test.ts +0 -243
- package/test/transports/batch.test.ts +0 -465
- package/test/transports/rest.test.ts +0 -242
- package/test/transports/websocket.test.ts +0 -183
- package/test/tsconfig.json +0 -5
- package/test/util.ts +0 -76
- package/test/validation.test.ts +0 -169
- package/test.sh +0 -20
- package/tsconfig.json +0 -24
- package/typedoc.json +0 -6
|
@@ -1,76 +0,0 @@
|
|
|
1
|
-
import { AdapterRateLimitTier, RateLimiter } from '..'
|
|
2
|
-
import { AdapterEndpoint } from '../../adapter'
|
|
3
|
-
import { makeLogger } from '../../util'
|
|
4
|
-
|
|
5
|
-
const logger = makeLogger('SimpleCountingRateLimiter')
|
|
6
|
-
|
|
7
|
-
/**
|
|
8
|
-
* This rate limiter is the simplest stateful option.
|
|
9
|
-
* On startup, it'll compare the different thresholds for each tier, calculate them all
|
|
10
|
-
* in the finest window we'll use (seconds), and use the most restrictive one.
|
|
11
|
-
* This is so if the EA were to restart, we don't need to worry about persisting state
|
|
12
|
-
* for things like daily quotas. The downside is that this does not work well for bursty
|
|
13
|
-
* loads or spikes, in cases where e.g. the per second limit is high but daily quotas low.
|
|
14
|
-
*/
|
|
15
|
-
export class SimpleCountingRateLimiter implements RateLimiter {
|
|
16
|
-
latestSecondInterval = 0
|
|
17
|
-
requestsThisSecond = 0
|
|
18
|
-
latestMinuteInterval = 0
|
|
19
|
-
requestsThisMinute = 0
|
|
20
|
-
perSecondLimit!: number
|
|
21
|
-
perMinuteLimit!: number
|
|
22
|
-
|
|
23
|
-
initialize(endpoints: AdapterEndpoint[], limits?: AdapterRateLimitTier) {
|
|
24
|
-
// Translate the hourly limit into reqs per minute
|
|
25
|
-
const perHourLimit = (limits?.rateLimit1h || Infinity) / 60
|
|
26
|
-
this.perMinuteLimit = Math.min(limits?.rateLimit1m || Infinity, perHourLimit)
|
|
27
|
-
this.perSecondLimit = limits?.rateLimit1s || Infinity
|
|
28
|
-
logger.debug(
|
|
29
|
-
`Using rate limiting settings: perMinute = ${this.perMinuteLimit} | perSecond: = ${this.perSecondLimit}`,
|
|
30
|
-
)
|
|
31
|
-
|
|
32
|
-
return this
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
isUnderLimits() {
|
|
36
|
-
// If the limit is set to infinity, there was no tier limit specified
|
|
37
|
-
if (this.perSecondLimit === Infinity && this.perMinuteLimit === Infinity) {
|
|
38
|
-
return true
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
const now = Date.now()
|
|
42
|
-
const nearestSecondInterval = Math.floor(now / 1000)
|
|
43
|
-
const nearestMinuteInterval = Math.floor(now / (1000 * 60))
|
|
44
|
-
|
|
45
|
-
// This should always run to completion, even if it doesn't look atomic; therefore the
|
|
46
|
-
// Ops should be "thread safe". Thank JS and its infinite single threaded dumbness.
|
|
47
|
-
if (nearestSecondInterval !== this.latestSecondInterval) {
|
|
48
|
-
logger.trace(
|
|
49
|
-
`Clearing latest second interval, # of requests logged was: ${this.requestsThisSecond} `,
|
|
50
|
-
)
|
|
51
|
-
this.latestSecondInterval = nearestSecondInterval
|
|
52
|
-
this.requestsThisSecond = 0
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
if (nearestMinuteInterval !== this.latestMinuteInterval) {
|
|
56
|
-
logger.trace(
|
|
57
|
-
`Clearing latest second minute, # of requests logged was: ${this.requestsThisMinute} `,
|
|
58
|
-
)
|
|
59
|
-
this.latestMinuteInterval = nearestMinuteInterval
|
|
60
|
-
this.requestsThisMinute = 0
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
if (
|
|
64
|
-
this.requestsThisSecond < this.perSecondLimit &&
|
|
65
|
-
this.requestsThisMinute < this.perMinuteLimit
|
|
66
|
-
) {
|
|
67
|
-
logger.trace('Request under limits, counting +1')
|
|
68
|
-
this.requestsThisSecond++
|
|
69
|
-
this.requestsThisMinute++
|
|
70
|
-
return true
|
|
71
|
-
} else {
|
|
72
|
-
logger.trace('Requests seen this interval are above limits')
|
|
73
|
-
return false
|
|
74
|
-
}
|
|
75
|
-
}
|
|
76
|
-
}
|
package/src/test.ts
DELETED
|
@@ -1,121 +0,0 @@
|
|
|
1
|
-
import { AxiosRequestConfig, AxiosResponse } from 'axios'
|
|
2
|
-
import { Cache } from '../cache'
|
|
3
|
-
import { AdapterConfig, SettingsMap } from '../config'
|
|
4
|
-
import { BackgroundExecuteRateLimiter } from '../rate-limiting'
|
|
5
|
-
import { ExpiringSortedSet, makeLogger } from '../util'
|
|
6
|
-
import { AdapterRequest, ProviderResult } from '../util/request'
|
|
7
|
-
import { AdapterContext, AdapterDependencies, Transport, buildCacheEntriesFromResults } from './'
|
|
8
|
-
import { axiosRequest } from './util'
|
|
9
|
-
import * as rateLimitMetrics from '../rate-limiting/metrics'
|
|
10
|
-
import * as cacheMetrics from '../cache/metrics'
|
|
11
|
-
|
|
12
|
-
const WARMUP_BATCH_REQUEST_ID = '9002'
|
|
13
|
-
|
|
14
|
-
const logger = makeLogger('BatchWarmingTransport')
|
|
15
|
-
|
|
16
|
-
/**
|
|
17
|
-
* Transport implementation that takes incoming batches requests and keeps a warm cache of values.
|
|
18
|
-
* Within the setup function, adapter params are added to an set that also keeps track and expires values.
|
|
19
|
-
* In the background execute, the list of non-expired items in the set is fetched.
|
|
20
|
-
* Then, the list is passed through the `prepareRequest` function, that returns an AxiosRequestConfig.
|
|
21
|
-
* The Data Provider response is, they are passed through the `parseResponse` function to create a [[CacheEntry]] list.
|
|
22
|
-
* Finally, the items in that [[CacheEntry]] list are set in the Cache so the Adapter can fetch values from there.
|
|
23
|
-
*
|
|
24
|
-
* @typeParam AdapterParams - interface for the adapter request body
|
|
25
|
-
* @typeParam ProviderRequestBody - interface for the body of the request to the Data Provider
|
|
26
|
-
* @typeParam ProviderResponseBody - interface for the body of the Data Provider's response
|
|
27
|
-
*/
|
|
28
|
-
export class BatchWarmingTransport<
|
|
29
|
-
AdapterParams,
|
|
30
|
-
ProviderRequestBody,
|
|
31
|
-
ProviderResponseBody,
|
|
32
|
-
CustomSettings extends SettingsMap,
|
|
33
|
-
> implements Transport<AdapterParams, ProviderResponseBody, CustomSettings>
|
|
34
|
-
{
|
|
35
|
-
cache!: Cache
|
|
36
|
-
rateLimiter!: BackgroundExecuteRateLimiter
|
|
37
|
-
|
|
38
|
-
expiringSortedSet = new ExpiringSortedSet<AdapterParams>() // TODO: Move to dependencies, inject
|
|
39
|
-
|
|
40
|
-
// Flag used to track whether the warmer has moved from having no entries to having some and vice versa
|
|
41
|
-
// Used for recording the cache warmer active metrics accurately
|
|
42
|
-
WARMER_ACTIVE = false
|
|
43
|
-
|
|
44
|
-
constructor(
|
|
45
|
-
private config: {
|
|
46
|
-
prepareRequest: (
|
|
47
|
-
params: AdapterParams[],
|
|
48
|
-
context: AdapterContext<CustomSettings>,
|
|
49
|
-
) => AxiosRequestConfig<ProviderRequestBody>
|
|
50
|
-
parseResponse: (
|
|
51
|
-
res: AxiosResponse<ProviderResponseBody>,
|
|
52
|
-
context: AdapterContext<CustomSettings>,
|
|
53
|
-
) => ProviderResult<AdapterParams>[]
|
|
54
|
-
},
|
|
55
|
-
) {}
|
|
56
|
-
|
|
57
|
-
async initialize(dependencies: AdapterDependencies): Promise<void> {
|
|
58
|
-
this.cache = dependencies.cache
|
|
59
|
-
this.rateLimiter = dependencies.backgroundExecuteRateLimiter
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
async hasBeenSetUp(req: AdapterRequest<AdapterParams>): Promise<boolean> {
|
|
63
|
-
return !!this.expiringSortedSet.get(req.requestContext.cacheKey)
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
async setup(
|
|
67
|
-
req: AdapterRequest<AdapterParams>,
|
|
68
|
-
config: AdapterConfig<CustomSettings>,
|
|
69
|
-
): Promise<void> {
|
|
70
|
-
logger.debug(
|
|
71
|
-
`Adding entry to batch warming set: [${req.requestContext.cacheKey}] = ${req.requestContext.data}`,
|
|
72
|
-
)
|
|
73
|
-
this.expiringSortedSet.add(
|
|
74
|
-
req.requestContext.cacheKey,
|
|
75
|
-
req.requestContext.data,
|
|
76
|
-
config.WARMUP_SUBSCRIPTION_TTL,
|
|
77
|
-
)
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
async backgroundExecute(context: AdapterContext<CustomSettings>): Promise<number> {
|
|
81
|
-
logger.debug('Starting background execute')
|
|
82
|
-
const entries = this.expiringSortedSet.getAll()
|
|
83
|
-
|
|
84
|
-
if (!entries.length) {
|
|
85
|
-
logger.debug('No entries in batch warming set, skipping')
|
|
86
|
-
if (this.WARMER_ACTIVE) {
|
|
87
|
-
// Decrement count when warmer changed from having entries to having none
|
|
88
|
-
cacheMetrics.cacheWarmerCount.labels({ isBatched: 'true' }).dec()
|
|
89
|
-
this.WARMER_ACTIVE = false
|
|
90
|
-
}
|
|
91
|
-
return this.rateLimiter.msUntilNextExecution(context.adapterEndpoint.name)
|
|
92
|
-
} else if (this.WARMER_ACTIVE === false) {
|
|
93
|
-
// Increment count when warmer changed from having no entries to having some
|
|
94
|
-
cacheMetrics.cacheWarmerCount.labels({ isBatched: 'true' }).inc()
|
|
95
|
-
this.WARMER_ACTIVE = true
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
const request = this.config.prepareRequest(entries, context)
|
|
99
|
-
|
|
100
|
-
logger.trace('Sending request to data provider...')
|
|
101
|
-
const providerResponse = await axiosRequest<ProviderRequestBody, ProviderResponseBody>(request)
|
|
102
|
-
|
|
103
|
-
logger.debug(`Got response from provider, parsing (raw body: ${providerResponse.data})`)
|
|
104
|
-
const results = this.config.parseResponse(providerResponse, context)
|
|
105
|
-
const adapterResponses = buildCacheEntriesFromResults(results, context)
|
|
106
|
-
|
|
107
|
-
logger.debug('Setting adapter responses in cache')
|
|
108
|
-
await this.cache.setMany(adapterResponses, context.adapterConfig.CACHE_MAX_AGE)
|
|
109
|
-
|
|
110
|
-
// Record cost of data provider call
|
|
111
|
-
const cost = rateLimitMetrics.retrieveCost(providerResponse.data)
|
|
112
|
-
rateLimitMetrics.rateLimitCreditsSpentTotal
|
|
113
|
-
.labels({
|
|
114
|
-
feed_id: 'N/A',
|
|
115
|
-
participant_id: WARMUP_BATCH_REQUEST_ID,
|
|
116
|
-
})
|
|
117
|
-
.inc(cost)
|
|
118
|
-
|
|
119
|
-
return this.rateLimiter.msUntilNextExecution(context.adapterEndpoint.name)
|
|
120
|
-
}
|
|
121
|
-
}
|
package/src/transports/index.ts
DELETED
|
@@ -1,173 +0,0 @@
|
|
|
1
|
-
import { FastifyReply } from 'fastify'
|
|
2
|
-
import { AdapterEndpoint, InitializedAdapter } from '../adapter'
|
|
3
|
-
import {
|
|
4
|
-
Cache,
|
|
5
|
-
CacheEntry,
|
|
6
|
-
calculateCacheKey,
|
|
7
|
-
calculateFeedId,
|
|
8
|
-
pollResponseFromCache,
|
|
9
|
-
} from '../cache'
|
|
10
|
-
import { AdapterConfig, CustomSettingsType, SettingsMap } from '../config'
|
|
11
|
-
import { BackgroundExecuteRateLimiter, RequestRateLimiter } from '../rate-limiting'
|
|
12
|
-
import { makeLogger } from '../util'
|
|
13
|
-
import { AdapterRequest, AdapterResponse, ProviderResult } from '../util/request'
|
|
14
|
-
|
|
15
|
-
export * from './batch-warming'
|
|
16
|
-
export * from './rest'
|
|
17
|
-
export * from './websocket'
|
|
18
|
-
|
|
19
|
-
const logger = makeLogger('Transport')
|
|
20
|
-
|
|
21
|
-
/**
|
|
22
|
-
* Dependencies that will be injected into the Adapter on startup
|
|
23
|
-
*/
|
|
24
|
-
export interface AdapterDependencies {
|
|
25
|
-
cache: Cache
|
|
26
|
-
requestRateLimiter: RequestRateLimiter
|
|
27
|
-
backgroundExecuteRateLimiter: BackgroundExecuteRateLimiter
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
/**
|
|
31
|
-
* Context that will be used on background executions of a Transport.
|
|
32
|
-
* For example, the endpointName used to log statements or generate Cache keys.
|
|
33
|
-
*/
|
|
34
|
-
export interface AdapterContext<
|
|
35
|
-
CustomSettings extends CustomSettingsType<CustomSettings> = SettingsMap,
|
|
36
|
-
> {
|
|
37
|
-
adapterEndpoint: AdapterEndpoint
|
|
38
|
-
adapterConfig: AdapterConfig<CustomSettings>
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
/**
|
|
42
|
-
* Generic interface for a Transport.
|
|
43
|
-
* A Transport defines the way in which an AdapterEndpoint will process incoming requests to
|
|
44
|
-
* fetch data from a Data Provider. The setup phase will take care of the former, while the
|
|
45
|
-
* backgroundExecute will be in charge of the latter.
|
|
46
|
-
* This separation gives us the ability of splitting these concerns, and optionally parallelizing
|
|
47
|
-
* the reading and writing of data to a centralized Cache.
|
|
48
|
-
*
|
|
49
|
-
* @typeParam Params - the structure of the AdapterRequest's body
|
|
50
|
-
* @typeParam Result - the structure of the AdapterResponse's body
|
|
51
|
-
*/
|
|
52
|
-
export interface Transport<Params, Result, CustomSettings extends SettingsMap> {
|
|
53
|
-
// TODO: docs / examples to extend dependencies? e.g. JSON rpc
|
|
54
|
-
/**
|
|
55
|
-
* Initializes the transport in the Adapter context.
|
|
56
|
-
*
|
|
57
|
-
* @param dependencies - Adapter dependencies (e.g. cache instance)
|
|
58
|
-
* @returns an empty Promise
|
|
59
|
-
*/
|
|
60
|
-
initialize: (dependencies: AdapterDependencies) => Promise<void>
|
|
61
|
-
|
|
62
|
-
/**
|
|
63
|
-
* Checks if the Transport has already set up the incoming request.
|
|
64
|
-
* This will likely use the cacheKey generated in the provided request.
|
|
65
|
-
*
|
|
66
|
-
* @param req - the incoming AdapterRequest
|
|
67
|
-
* @returns a Promise that returns true if setup is in place
|
|
68
|
-
*/
|
|
69
|
-
hasBeenSetUp: (req: AdapterRequest<Params>) => Promise<boolean>
|
|
70
|
-
|
|
71
|
-
/**
|
|
72
|
-
* Sets up the incoming request within the Transport.
|
|
73
|
-
* In other words, it means that the Adapter has recognized this request, and will begin
|
|
74
|
-
* fetching the necessary data from a Data Provider.
|
|
75
|
-
*
|
|
76
|
-
* @param req - the incoming AdapterRequest
|
|
77
|
-
* @param config - common configuration for the Adapter as a whole
|
|
78
|
-
* @returns a Promise that _optionally_ returns an AdapterResponse, if the Transport has the capability of
|
|
79
|
-
* immediately fetching data and returning it without the background process.
|
|
80
|
-
*/
|
|
81
|
-
setup: (
|
|
82
|
-
req: AdapterRequest<Params>,
|
|
83
|
-
config: AdapterConfig<CustomSettings>,
|
|
84
|
-
) => Promise<AdapterResponse<Result> | void>
|
|
85
|
-
|
|
86
|
-
// TODO: Might need a mechanism to know if this bg execute is in flight to avoid multitple simultaneous invocations
|
|
87
|
-
/**
|
|
88
|
-
* If the Transport relies on an async mechanism, implementing this function will make it
|
|
89
|
-
* so the background task executor calls it on Adapter startup.
|
|
90
|
-
*
|
|
91
|
-
* @param context - background context for the execution (e.g. endpoint name)
|
|
92
|
-
*/
|
|
93
|
-
backgroundExecute?: (context: AdapterContext<CustomSettings>) => Promise<number>
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
/**
|
|
97
|
-
* Helper method to build cache entries to set after getting a bunch of responses from a DP.
|
|
98
|
-
*
|
|
99
|
-
* @param results - a list of results coming from a DataProvider
|
|
100
|
-
* @param context - context for the Adapter
|
|
101
|
-
* @returns a list of CacheEntries of AdapterResponses
|
|
102
|
-
*/
|
|
103
|
-
export const buildCacheEntriesFromResults = <Params, Context extends AdapterContext<any>>(
|
|
104
|
-
results: ProviderResult<Params>[],
|
|
105
|
-
context: Context,
|
|
106
|
-
): CacheEntry<AdapterResponse<null>>[] =>
|
|
107
|
-
results.map((r) => {
|
|
108
|
-
const cacheEntry = {
|
|
109
|
-
key: calculateCacheKey(context as AdapterContext, r.params),
|
|
110
|
-
value: {
|
|
111
|
-
result: r.value,
|
|
112
|
-
statusCode: 200,
|
|
113
|
-
data: null, // TODO: Maybe add data as well?
|
|
114
|
-
},
|
|
115
|
-
}
|
|
116
|
-
if (
|
|
117
|
-
context.adapterConfig.METRICS_ENABLED &&
|
|
118
|
-
context.adapterConfig.EXPERIMENTAL_METRICS_ENABLED
|
|
119
|
-
) {
|
|
120
|
-
const metrics = {
|
|
121
|
-
maxAge: Date.now() + context.adapterConfig.CACHE_MAX_AGE, // TODO: Replace with telemetry structure in future
|
|
122
|
-
meta: {
|
|
123
|
-
metrics: {
|
|
124
|
-
feedId: calculateFeedId(context.adapterEndpoint, r.params)
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
cacheEntry.value = { ...cacheEntry.value, ...metrics }
|
|
129
|
-
}
|
|
130
|
-
return cacheEntry
|
|
131
|
-
})
|
|
132
|
-
|
|
133
|
-
/**
|
|
134
|
-
* Takes an Adapter, its configuration, and its dependencies, and it creates an express middleware
|
|
135
|
-
* that will pass along the AdapterRequest to the appropriate Transport (acc. to the endpoint in the req.)
|
|
136
|
-
*
|
|
137
|
-
* @param adapter - main adapter object, already initialized
|
|
138
|
-
* @returns the transport handler middleware function
|
|
139
|
-
*/
|
|
140
|
-
export const buildTransportHandler =
|
|
141
|
-
(adapter: InitializedAdapter) => async (req: AdapterRequest, reply: FastifyReply) => {
|
|
142
|
-
// Get transport, must be here because it's already checked in the validator
|
|
143
|
-
const transport = adapter.endpointsMap[req.requestContext.endpointName].transport
|
|
144
|
-
|
|
145
|
-
// Set up transport if it hasn't been done already
|
|
146
|
-
if (!(await transport.hasBeenSetUp(req))) {
|
|
147
|
-
logger.debug('Transport not set up yet, doing so...')
|
|
148
|
-
const immediateResponse = await transport.setup(req, adapter.config)
|
|
149
|
-
if (immediateResponse) {
|
|
150
|
-
logger.debug('Got immediate response from transport, sending as response')
|
|
151
|
-
return reply.send(immediateResponse)
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
logger.debug('Transport is set up, polling cache for response...')
|
|
155
|
-
const response = await pollResponseFromCache(
|
|
156
|
-
adapter.dependencies.cache as Cache<AdapterResponse>,
|
|
157
|
-
req.requestContext.cacheKey,
|
|
158
|
-
{
|
|
159
|
-
maxRetries: adapter.config.CACHE_POLLING_MAX_RETRIES,
|
|
160
|
-
sleep: adapter.config.CACHE_POLLING_SLEEP_MS,
|
|
161
|
-
},
|
|
162
|
-
)
|
|
163
|
-
|
|
164
|
-
if (response) {
|
|
165
|
-
logger.debug('Got a response from the cache, sending that back')
|
|
166
|
-
return reply.send(response)
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
logger.debug('Ran out of polling attempts, returning timeout')
|
|
170
|
-
reply.statusCode = 504
|
|
171
|
-
|
|
172
|
-
return reply.send()
|
|
173
|
-
}
|
|
@@ -1,95 +0,0 @@
|
|
|
1
|
-
import * as client from 'prom-client'
|
|
2
|
-
import { AdapterContext } from '.'
|
|
3
|
-
import { calculateCacheKey, calculateFeedId } from '../cache'
|
|
4
|
-
import { requestDurationBuckets } from '../metrics/constants'
|
|
5
|
-
|
|
6
|
-
// Data Provider Metrics
|
|
7
|
-
export const dataProviderMetricsLabel = (providerStatusCode?: number, method = 'get') => ({
|
|
8
|
-
provider_status_code: providerStatusCode,
|
|
9
|
-
method: method.toUpperCase(),
|
|
10
|
-
})
|
|
11
|
-
|
|
12
|
-
export const dataProviderRequests = new client.Counter({
|
|
13
|
-
name: 'data_provider_requests',
|
|
14
|
-
help: 'The number of http requests that are made to a data provider',
|
|
15
|
-
labelNames: ['method', 'provider_status_code'] as const,
|
|
16
|
-
})
|
|
17
|
-
|
|
18
|
-
export const dataProviderRequestDurationSeconds = new client.Histogram({
|
|
19
|
-
name: 'data_provider_request_duration_seconds',
|
|
20
|
-
help: 'A histogram bucket of the distribution of data provider request durations',
|
|
21
|
-
buckets: requestDurationBuckets,
|
|
22
|
-
})
|
|
23
|
-
|
|
24
|
-
// Websocket Metrics
|
|
25
|
-
export const connectionErrorLabels = (message: string) => ({
|
|
26
|
-
// Key,
|
|
27
|
-
message,
|
|
28
|
-
})
|
|
29
|
-
|
|
30
|
-
export const messageSubsLabels = (feed_id: string, cache_key: string) => ({
|
|
31
|
-
feed_id,
|
|
32
|
-
subscription_key: cache_key,
|
|
33
|
-
})
|
|
34
|
-
|
|
35
|
-
// Record WS message and subscription metrics
|
|
36
|
-
// Recalculate cacheKey and feedId for metrics
|
|
37
|
-
// since avoiding storing extra info in expiring sorted set
|
|
38
|
-
export const recordWsMessageMetrics = <AdapterParams>(
|
|
39
|
-
context: AdapterContext,
|
|
40
|
-
subscribes: AdapterParams[],
|
|
41
|
-
unsubscrices: AdapterParams[],
|
|
42
|
-
): void => {
|
|
43
|
-
subscribes.forEach((param) => {
|
|
44
|
-
const feedId = calculateFeedId(context.adapterEndpoint, param)
|
|
45
|
-
const cacheKey = calculateCacheKey(context, param)
|
|
46
|
-
// Record total number of ws messages sent
|
|
47
|
-
wsMessageTotal.labels(messageSubsLabels(feedId, cacheKey)).inc()
|
|
48
|
-
|
|
49
|
-
// Record total number of subscriptions made
|
|
50
|
-
wsSubscriptionTotal.labels(messageSubsLabels(feedId, cacheKey)).inc()
|
|
51
|
-
|
|
52
|
-
// Record number of active ws subscriptions
|
|
53
|
-
wsSubscriptionActive.labels(messageSubsLabels(feedId, cacheKey)).inc()
|
|
54
|
-
})
|
|
55
|
-
unsubscrices.forEach((param) => {
|
|
56
|
-
const feedId = calculateFeedId(context.adapterEndpoint, param)
|
|
57
|
-
const cacheKey = calculateCacheKey(context, param)
|
|
58
|
-
|
|
59
|
-
// Record total number of ws messages sent
|
|
60
|
-
wsMessageTotal.labels(messageSubsLabels(feedId, cacheKey)).inc()
|
|
61
|
-
|
|
62
|
-
// Record number of active ws subscriptions
|
|
63
|
-
wsSubscriptionActive.labels(messageSubsLabels(feedId, cacheKey)).dec()
|
|
64
|
-
})
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
export const wsConnectionActive = new client.Gauge({
|
|
68
|
-
name: 'ws_connection_active',
|
|
69
|
-
help: 'The number of active connections',
|
|
70
|
-
labelNames: ['url'] as const,
|
|
71
|
-
})
|
|
72
|
-
|
|
73
|
-
export const wsConnectionErrors = new client.Counter({
|
|
74
|
-
name: 'ws_connection_errors',
|
|
75
|
-
help: 'The number of connection errors',
|
|
76
|
-
labelNames: ['url', 'message'] as const,
|
|
77
|
-
})
|
|
78
|
-
|
|
79
|
-
export const wsSubscriptionActive = new client.Gauge({
|
|
80
|
-
name: 'ws_subscription_active',
|
|
81
|
-
help: 'The number of currently active subscriptions',
|
|
82
|
-
labelNames: ['connection_url', 'feed_id', 'subscription_key'] as const,
|
|
83
|
-
})
|
|
84
|
-
|
|
85
|
-
export const wsSubscriptionTotal = new client.Counter({
|
|
86
|
-
name: 'ws_subscription_total',
|
|
87
|
-
help: 'The number of subscriptions opened in total',
|
|
88
|
-
labelNames: ['connection_url', 'feed_id', 'subscription_key'] as const,
|
|
89
|
-
})
|
|
90
|
-
|
|
91
|
-
export const wsMessageTotal = new client.Counter({
|
|
92
|
-
name: 'ws_message_total',
|
|
93
|
-
help: 'The number of messages received in total',
|
|
94
|
-
labelNames: ['feed_id', 'subscription_key'] as const,
|
|
95
|
-
})
|
package/src/transports/rest.ts
DELETED
|
@@ -1,161 +0,0 @@
|
|
|
1
|
-
import { AdapterRequest, AdapterResponse } from '../util/request'
|
|
2
|
-
import { AdapterDependencies, Transport } from './'
|
|
3
|
-
import { Cache } from '../cache'
|
|
4
|
-
import { AxiosRequestConfig, AxiosResponse } from 'axios'
|
|
5
|
-
import { makeLogger, sleep } from '../util'
|
|
6
|
-
import { AdapterConfig, SettingsMap } from '../config'
|
|
7
|
-
import { RequestRateLimiter } from '../rate-limiting'
|
|
8
|
-
import { AdapterError } from '../validation/error'
|
|
9
|
-
import { axiosRequest } from './util'
|
|
10
|
-
import * as rateLimitMetrics from '../rate-limiting/metrics'
|
|
11
|
-
|
|
12
|
-
const IN_FLIGHT_PREFIX = 'InFlight'
|
|
13
|
-
|
|
14
|
-
const logger = makeLogger('RestTransport')
|
|
15
|
-
|
|
16
|
-
/**
|
|
17
|
-
* Transport implementation that takes incoming requests, transforms them into a DataProvider request,
|
|
18
|
-
* and executes that request returning the response immediately from the `setup` function.
|
|
19
|
-
* Optionally, setting the `coalescing` option to `true` will make it so once a request is in flight,
|
|
20
|
-
* new adapter requests for the same feed will not trigger a new one, but return an empty promise from
|
|
21
|
-
* the setup instead so the normal cache polling mechanism is used.
|
|
22
|
-
*
|
|
23
|
-
* @typeParam AdapterParams - interface for the adapter request body
|
|
24
|
-
* @typeParam ProviderRequestBody - interface for the body of the request to the Data Provider
|
|
25
|
-
* @typeParam ProviderResponseBody - interface for the body of the Data Provider's response
|
|
26
|
-
*/
|
|
27
|
-
export class RestTransport<
|
|
28
|
-
AdapterParams,
|
|
29
|
-
ProviderRequestBody,
|
|
30
|
-
ProviderResponseBody,
|
|
31
|
-
CustomSettings extends SettingsMap,
|
|
32
|
-
> implements Transport<AdapterParams, ProviderResponseBody, CustomSettings>
|
|
33
|
-
{
|
|
34
|
-
inFlightPrefix!: string
|
|
35
|
-
cache!: Cache
|
|
36
|
-
rateLimiter!: RequestRateLimiter
|
|
37
|
-
|
|
38
|
-
constructor(
|
|
39
|
-
protected config: {
|
|
40
|
-
prepareRequest: (
|
|
41
|
-
req: AdapterRequest<AdapterParams>,
|
|
42
|
-
config: AdapterConfig<CustomSettings>,
|
|
43
|
-
) =>
|
|
44
|
-
| AxiosRequestConfig<ProviderRequestBody>
|
|
45
|
-
| Promise<AxiosRequestConfig<ProviderRequestBody>>
|
|
46
|
-
parseResponse: (
|
|
47
|
-
req: AdapterRequest<AdapterParams>,
|
|
48
|
-
res: AxiosResponse<ProviderResponseBody>,
|
|
49
|
-
config: AdapterConfig<CustomSettings>,
|
|
50
|
-
) => AdapterResponse<ProviderResponseBody> | Promise<AdapterResponse<ProviderResponseBody>>
|
|
51
|
-
options: {
|
|
52
|
-
coalescing: boolean
|
|
53
|
-
}
|
|
54
|
-
},
|
|
55
|
-
) {}
|
|
56
|
-
|
|
57
|
-
async initialize(dependencies: AdapterDependencies): Promise<void> {
|
|
58
|
-
this.inFlightPrefix = `${IN_FLIGHT_PREFIX}-`
|
|
59
|
-
this.cache = dependencies.cache
|
|
60
|
-
this.rateLimiter = dependencies.requestRateLimiter
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
async hasBeenSetUp(req: AdapterRequest<AdapterParams>): Promise<boolean> {
|
|
64
|
-
if (!this.config.options.coalescing) {
|
|
65
|
-
return false
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
// Check if request is in flight
|
|
69
|
-
const inFlight = await this.cache.get(this.inFlightPrefix + req.requestContext.cacheKey)
|
|
70
|
-
if (inFlight) {
|
|
71
|
-
logger.debug('Request is in flight, transport has been set up')
|
|
72
|
-
return true
|
|
73
|
-
} else {
|
|
74
|
-
logger.debug('Request not in flight, transport not set up')
|
|
75
|
-
return false
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
protected async waitUntilUnderRateLimit(
|
|
80
|
-
options: {
|
|
81
|
-
maxRetries: number
|
|
82
|
-
msBetweenRetries: number
|
|
83
|
-
},
|
|
84
|
-
retry = 0,
|
|
85
|
-
) {
|
|
86
|
-
if (this.rateLimiter.isUnderLimits()) {
|
|
87
|
-
logger.trace('Incoming request would not be under limits, moving on')
|
|
88
|
-
return
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
if (retry >= options.maxRetries) {
|
|
92
|
-
throw new AdapterError({
|
|
93
|
-
statusCode: 504,
|
|
94
|
-
message: `REST Transport timed out while waiting for rate limit availability (max retries: ${options.maxRetries})`,
|
|
95
|
-
})
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
logger.debug(`Request would be over rate limits, sleeping for ${options.msBetweenRetries}`)
|
|
99
|
-
await sleep(options.msBetweenRetries)
|
|
100
|
-
await this.waitUntilUnderRateLimit(options, retry + 1)
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
async setup(
|
|
104
|
-
req: AdapterRequest<AdapterParams>,
|
|
105
|
-
config: AdapterConfig<CustomSettings>,
|
|
106
|
-
): Promise<AdapterResponse<ProviderResponseBody> | undefined> {
|
|
107
|
-
if (this.config.options.coalescing) {
|
|
108
|
-
logger.debug('Setting up rest transport, setting request in flight in cache')
|
|
109
|
-
// TODO: Should this use a separate cache?
|
|
110
|
-
// TODO: Set viable ttl to approximate timeout from API
|
|
111
|
-
// TODO: Make this ttl configurable
|
|
112
|
-
await this.cache.set(this.inFlightPrefix + req.requestContext.cacheKey, true, 2000) // Can't use Infinity for things like Redis
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
const request = await this.config.prepareRequest(req, config)
|
|
116
|
-
|
|
117
|
-
logger.trace('Check if we are under rate limits to perform request')
|
|
118
|
-
await this.waitUntilUnderRateLimit({
|
|
119
|
-
maxRetries: config.REST_TRANSPORT_MAX_RATE_LIMIT_RETRIES,
|
|
120
|
-
msBetweenRetries: config.REST_TRANSPORT_MS_BETWEEN_RATE_LIMIT_RETRIES,
|
|
121
|
-
})
|
|
122
|
-
|
|
123
|
-
logger.trace('Sending request to data provider...')
|
|
124
|
-
const providerResponse = await axiosRequest<ProviderRequestBody, ProviderResponseBody>(request)
|
|
125
|
-
|
|
126
|
-
logger.debug(`Got response from provider, parsing (raw body: ${providerResponse.data})`) // TODO: Sensitive data?
|
|
127
|
-
const parsedResponse = await this.config.parseResponse(req, providerResponse, config)
|
|
128
|
-
|
|
129
|
-
if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
|
|
130
|
-
// TODO: Potentially create function to add all telemetry data
|
|
131
|
-
parsedResponse.maxAge = Date.now() + config.CACHE_MAX_AGE
|
|
132
|
-
parsedResponse.meta = { metrics: {feedId: req.requestContext.meta?.metrics?.feedId || 'N/A' }}
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
logger.debug('Setting provider response in cache')
|
|
136
|
-
await this.cache.set(req.requestContext.cacheKey, parsedResponse, config.CACHE_MAX_AGE)
|
|
137
|
-
|
|
138
|
-
// Record cost of data provider call
|
|
139
|
-
const cost = rateLimitMetrics.retrieveCost(providerResponse.data)
|
|
140
|
-
rateLimitMetrics.rateLimitCreditsSpentTotal
|
|
141
|
-
.labels({
|
|
142
|
-
feed_id: req.requestContext.meta?.metrics?.feedId || 'N/A',
|
|
143
|
-
participant_id: req.requestContext.cacheKey,
|
|
144
|
-
})
|
|
145
|
-
.inc(cost)
|
|
146
|
-
|
|
147
|
-
// Update cacheHit flag in request meta for metrics use
|
|
148
|
-
req.requestContext.meta = {
|
|
149
|
-
...req.requestContext.meta,
|
|
150
|
-
metrics: { ...req.requestContext.meta?.metrics, cacheHit: false },
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
// TODO: move this to a try/catch/finally
|
|
154
|
-
if (this.config.options.coalescing) {
|
|
155
|
-
logger.debug('Set provider response in cache, removing in flight from cache')
|
|
156
|
-
await this.cache.delete(this.inFlightPrefix)
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
return parsedResponse
|
|
160
|
-
}
|
|
161
|
-
}
|