@chainlink/external-adapter-framework 0.0.12 → 0.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adapter.d.ts +107 -0
- package/adapter.js +115 -0
- package/{dist/src/package/background-executor.d.ts → background-executor.d.ts} +0 -0
- package/{dist/src/background-executor.js → background-executor.js} +0 -0
- package/{dist/src/cache → cache}/factory.d.ts +0 -0
- package/{dist/src/cache → cache}/factory.js +0 -0
- package/{dist/src/cache → cache}/index.d.ts +8 -8
- package/{dist/src/cache → cache}/index.js +0 -0
- package/{dist/src/cache → cache}/local.d.ts +0 -0
- package/{dist/src/cache → cache}/local.js +0 -0
- package/{dist/src/cache → cache}/metrics.d.ts +0 -0
- package/{dist/src/cache → cache}/metrics.js +0 -0
- package/{dist/src/cache → cache}/redis.d.ts +0 -0
- package/{dist/src/cache → cache}/redis.js +0 -0
- package/{dist/src/chainlink-external-adapter-framework-0.0.6.tgz → chainlink-external-adapter-framework-0.0.6.tgz} +0 -0
- package/{dist/src/config → config}/index.d.ts +0 -5
- package/{dist/src/config → config}/index.js +5 -5
- package/{dist/src/config → config}/provider-limits.d.ts +0 -0
- package/{dist/src/config → config}/provider-limits.js +5 -1
- package/{dist/src/examples → examples}/bank-frick/accounts.d.ts +1 -1
- package/{dist/src/examples → examples}/bank-frick/accounts.js +2 -3
- package/{dist/src/examples → examples}/bank-frick/config/index.d.ts +0 -0
- package/{dist/src/examples → examples}/bank-frick/config/index.js +0 -0
- package/{dist/src/package/examples/coingecko → examples/bank-frick}/index.d.ts +0 -0
- package/{dist/src/examples → examples}/bank-frick/index.js +2 -3
- package/{dist/src/examples → examples}/bank-frick/util.d.ts +0 -0
- package/{dist/src/examples → examples}/bank-frick/util.js +0 -0
- package/{dist/src/package/examples → examples}/coingecko/batch-warming.d.ts +0 -0
- package/{dist/src/package/examples → examples}/coingecko/batch-warming.js +0 -0
- package/{dist/src/package/examples/ncfx → examples/coingecko}/index.d.ts +0 -0
- package/{dist/src/package/examples → examples}/coingecko/index.js +0 -0
- package/{dist/src/package/examples → examples}/coingecko/rest.d.ts +0 -0
- package/{dist/src/package/examples → examples}/coingecko/rest.js +0 -0
- package/{dist/src/examples → examples}/ncfx/config/index.d.ts +0 -0
- package/{dist/src/examples → examples}/ncfx/config/index.js +0 -0
- package/examples/ncfx/index.d.ts +2 -0
- package/{dist/src/package/examples → examples}/ncfx/index.js +0 -0
- package/{dist/src/package/examples → examples}/ncfx/websocket.d.ts +0 -0
- package/{dist/src/package/examples → examples}/ncfx/websocket.js +0 -0
- package/{dist/src/index.d.ts → index.d.ts} +0 -0
- package/{dist/src/index.js → index.js} +18 -13
- package/{dist/src/metrics → metrics}/constants.d.ts +0 -0
- package/{dist/src/metrics → metrics}/constants.js +0 -0
- package/{dist/src/metrics → metrics}/index.d.ts +0 -0
- package/{dist/src/metrics → metrics}/index.js +0 -0
- package/metrics/util.d.ts +7 -0
- package/{dist/src/metrics → metrics}/util.js +0 -0
- package/{dist/src/package → package}/adapter.d.ts +0 -0
- package/{dist/src/package → package}/adapter.js +0 -0
- package/{dist/src → package}/background-executor.d.ts +2 -2
- package/{dist/src/package → package}/background-executor.js +0 -0
- package/{dist/src/package → package}/cache/factory.d.ts +0 -0
- package/{dist/src/package → package}/cache/factory.js +0 -0
- package/{dist/src/package → package}/cache/index.d.ts +0 -0
- package/{dist/src/package → package}/cache/index.js +0 -0
- package/{dist/src/package → package}/cache/local.d.ts +0 -0
- package/{dist/src/package → package}/cache/local.js +0 -0
- package/{dist/src/package → package}/cache/metrics.d.ts +0 -0
- package/{dist/src/package → package}/cache/metrics.js +0 -0
- package/{dist/src/package → package}/cache/redis.d.ts +0 -0
- package/{dist/src/package → package}/cache/redis.js +0 -0
- package/{dist/src/package → package}/config/index.d.ts +0 -0
- package/{dist/src/package → package}/config/index.js +0 -0
- package/{dist/src/package → package}/config/provider-limits.d.ts +0 -0
- package/{dist/src/package → package}/config/provider-limits.js +0 -0
- package/package/examples/coingecko/batch-warming.d.ts +2 -0
- package/{dist/src → package}/examples/coingecko/batch-warming.js +2 -3
- package/package/examples/coingecko/index.d.ts +2 -0
- package/{dist/src → package}/examples/coingecko/index.js +2 -3
- package/package/examples/coingecko/rest.d.ts +2 -0
- package/{dist/src → package}/examples/coingecko/rest.js +2 -3
- package/{dist/src/package → package}/examples/ncfx/config/index.d.ts +0 -0
- package/{dist/src/package → package}/examples/ncfx/config/index.js +0 -0
- package/package/examples/ncfx/index.d.ts +2 -0
- package/{dist/src → package}/examples/ncfx/index.js +2 -3
- package/{dist/src → package}/examples/ncfx/websocket.d.ts +1 -12
- package/{dist/src → package}/examples/ncfx/websocket.js +2 -3
- package/{dist/src/package → package}/index.d.ts +0 -0
- package/{dist/src/package → package}/index.js +0 -0
- package/{dist/src/package → package}/metrics/constants.d.ts +0 -0
- package/{dist/src/package → package}/metrics/constants.js +0 -0
- package/{dist/src/package → package}/metrics/index.d.ts +0 -0
- package/{dist/src/package → package}/metrics/index.js +0 -0
- package/{dist/src/package → package}/metrics/util.d.ts +0 -0
- package/{dist/src/package → package}/metrics/util.js +0 -0
- package/{dist/src → package}/package.json +0 -0
- package/{dist/src/package → package}/rate-limiting/background/fixed-frequency.d.ts +0 -0
- package/{dist/src/package → package}/rate-limiting/background/fixed-frequency.js +0 -0
- package/{dist/src/package → package}/rate-limiting/index.d.ts +0 -0
- package/{dist/src/package → package}/rate-limiting/index.js +0 -0
- package/{dist/src/package → package}/rate-limiting/metrics.d.ts +0 -0
- package/{dist/src/package → package}/rate-limiting/metrics.js +0 -0
- package/{dist/src/package → package}/rate-limiting/request/simple-counting.d.ts +0 -0
- package/{dist/src/package → package}/rate-limiting/request/simple-counting.js +0 -0
- package/{dist/src/package → package}/test.d.ts +0 -0
- package/{dist/src/package → package}/test.js +0 -0
- package/{dist/src/package → package}/transports/batch-warming.d.ts +0 -0
- package/{dist/src/package → package}/transports/batch-warming.js +0 -0
- package/{dist/src/package → package}/transports/index.d.ts +0 -0
- package/{dist/src/package → package}/transports/index.js +0 -0
- package/{dist/src/package → package}/transports/metrics.d.ts +0 -0
- package/{dist/src/package → package}/transports/metrics.js +0 -0
- package/{dist/src/package → package}/transports/rest.d.ts +0 -0
- package/{dist/src/package → package}/transports/rest.js +0 -0
- package/{dist/src/package → package}/transports/util.d.ts +0 -0
- package/{dist/src/package → package}/transports/util.js +0 -0
- package/{dist/src/package → package}/transports/websocket.d.ts +0 -0
- package/{dist/src/package → package}/transports/websocket.js +0 -0
- package/{dist/src/package → package}/util/expiring-sorted-set.d.ts +0 -0
- package/{dist/src/package → package}/util/expiring-sorted-set.js +0 -0
- package/{dist/src/package → package}/util/index.d.ts +0 -0
- package/{dist/src/package → package}/util/index.js +0 -0
- package/{dist/src/package → package}/util/logger.d.ts +0 -0
- package/{dist/src/package → package}/util/logger.js +0 -0
- package/{dist/src/package → package}/util/request.d.ts +0 -0
- package/{dist/src/package → package}/util/request.js +0 -0
- package/{dist/src/package → package}/validation/error.d.ts +0 -0
- package/{dist/src/package → package}/validation/error.js +0 -0
- package/{dist/src/package → package}/validation/index.d.ts +0 -0
- package/{dist/src/package → package}/validation/index.js +0 -0
- package/{dist/src/package → package}/validation/input-params.d.ts +0 -0
- package/{dist/src/package → package}/validation/input-params.js +0 -0
- package/{dist/src/package → package}/validation/override-functions.d.ts +0 -0
- package/{dist/src/package → package}/validation/override-functions.js +0 -0
- package/{dist/src/package → package}/validation/preset-tokens.json +0 -0
- package/{dist/src/package → package}/validation/validator.d.ts +0 -0
- package/{dist/src/package → package}/validation/validator.js +0 -0
- package/package.json +1 -1
- package/{dist/src/rate-limiting → rate-limiting}/background/fixed-frequency.d.ts +1 -2
- package/{dist/src/rate-limiting → rate-limiting}/background/fixed-frequency.js +0 -0
- package/{dist/src/rate-limiting → rate-limiting}/index.d.ts +1 -2
- package/{dist/src/rate-limiting → rate-limiting}/index.js +0 -0
- package/{dist/src/rate-limiting → rate-limiting}/metrics.d.ts +0 -0
- package/{dist/src/rate-limiting → rate-limiting}/metrics.js +0 -0
- package/{dist/src/rate-limiting → rate-limiting}/request/simple-counting.d.ts +1 -2
- package/{dist/src/rate-limiting → rate-limiting}/request/simple-counting.js +0 -0
- package/{dist/src/test.d.ts → test.d.ts} +0 -0
- package/{dist/src/test.js → test.js} +0 -0
- package/{dist/src/transports → transports}/batch-warming.d.ts +0 -0
- package/{dist/src/transports → transports}/batch-warming.js +0 -0
- package/{dist/src/transports → transports}/index.d.ts +2 -2
- package/{dist/src/transports → transports}/index.js +0 -0
- package/{dist/src/transports → transports}/metrics.d.ts +1 -2
- package/{dist/src/transports → transports}/metrics.js +0 -0
- package/{dist/src/transports → transports}/rest.d.ts +0 -0
- package/{dist/src/transports → transports}/rest.js +0 -0
- package/{dist/src/transports → transports}/util.d.ts +0 -0
- package/{dist/src/transports → transports}/util.js +0 -0
- package/{dist/src/transports → transports}/websocket.d.ts +3 -2
- package/{dist/src/transports → transports}/websocket.js +6 -4
- package/{dist/src/util → util}/expiring-sorted-set.d.ts +0 -0
- package/{dist/src/util → util}/expiring-sorted-set.js +0 -0
- package/{dist/src/util → util}/index.d.ts +0 -0
- package/{dist/src/util → util}/index.js +0 -0
- package/{dist/src/util → util}/logger.d.ts +0 -0
- package/{dist/src/util → util}/logger.js +0 -0
- package/{dist/src/util → util}/request.d.ts +2 -2
- package/{dist/src/util → util}/request.js +0 -0
- package/{dist/src/util → util}/subscription-set/expiring-sorted-set.d.ts +0 -0
- package/{dist/src/util → util}/subscription-set/expiring-sorted-set.js +0 -0
- package/{dist/src/util → util}/subscription-set/subscription-set.d.ts +0 -0
- package/{dist/src/util → util}/subscription-set/subscription-set.js +0 -0
- package/{dist/src/util → util}/test-payload-loader.d.ts +0 -0
- package/{dist/src/util → util}/test-payload-loader.js +0 -0
- package/{dist/src/validation → validation}/error.d.ts +2 -2
- package/{dist/src/validation → validation}/error.js +0 -0
- package/{dist/src/validation → validation}/index.d.ts +0 -0
- package/{dist/src/validation → validation}/index.js +0 -0
- package/{dist/src/validation → validation}/input-params.d.ts +0 -0
- package/{dist/src/validation → validation}/input-params.js +0 -0
- package/validation/override-functions.d.ts +3 -0
- package/{dist/src/validation → validation}/override-functions.js +0 -0
- package/{dist/src/validation → validation}/preset-tokens.json +0 -0
- package/{dist/src/validation → validation}/validator.d.ts +0 -0
- package/{dist/src/validation → validation}/validator.js +0 -0
- package/.c8rc.json +0 -3
- package/.eslintignore +0 -9
- package/.eslintrc.js +0 -96
- package/.github/README.MD +0 -17
- package/.github/actions/setup/action.yaml +0 -13
- package/.github/workflows/main.yaml +0 -39
- package/.github/workflows/publish.yaml +0 -20
- package/.prettierignore +0 -13
- package/.yarnrc +0 -0
- package/README.md +0 -103
- package/dist/src/adapter.d.ts +0 -135
- package/dist/src/adapter.js +0 -145
- package/dist/src/examples/bank-frick/index.d.ts +0 -2
- package/dist/src/examples/coingecko/batch-warming.d.ts +0 -7
- package/dist/src/examples/coingecko/index.d.ts +0 -2
- package/dist/src/examples/coingecko/rest.d.ts +0 -12
- package/dist/src/examples/ncfx/index.d.ts +0 -13
- package/dist/src/metrics/util.d.ts +0 -7
- package/dist/src/package/package.json +0 -72
- package/dist/src/validation/override-functions.d.ts +0 -3
- package/docker-compose.yaml +0 -35
- package/env.sh +0 -54
- package/env2.sh +0 -55
- package/publish.sh +0 -0
- package/src/adapter.ts +0 -263
- package/src/background-executor.ts +0 -52
- package/src/cache/factory.ts +0 -26
- package/src/cache/index.ts +0 -258
- package/src/cache/local.ts +0 -73
- package/src/cache/metrics.ts +0 -112
- package/src/cache/redis.ts +0 -93
- package/src/config/index.ts +0 -517
- package/src/config/provider-limits.ts +0 -130
- package/src/examples/bank-frick/README.MD +0 -10
- package/src/examples/bank-frick/accounts.ts +0 -246
- package/src/examples/bank-frick/config/index.ts +0 -53
- package/src/examples/bank-frick/index.ts +0 -13
- package/src/examples/bank-frick/types.d.ts +0 -38
- package/src/examples/bank-frick/util.ts +0 -55
- package/src/examples/coingecko/batch-warming.ts +0 -78
- package/src/examples/coingecko/index.ts +0 -9
- package/src/examples/coingecko/rest.ts +0 -77
- package/src/examples/ncfx/config/index.ts +0 -12
- package/src/examples/ncfx/index.ts +0 -9
- package/src/examples/ncfx/websocket.ts +0 -99
- package/src/index.ts +0 -149
- package/src/metrics/constants.ts +0 -23
- package/src/metrics/index.ts +0 -115
- package/src/metrics/util.ts +0 -18
- package/src/rate-limiting/background/fixed-frequency.ts +0 -45
- package/src/rate-limiting/index.ts +0 -100
- package/src/rate-limiting/metrics.ts +0 -18
- package/src/rate-limiting/request/simple-counting.ts +0 -76
- package/src/test.ts +0 -5
- package/src/transports/batch-warming.ts +0 -122
- package/src/transports/index.ts +0 -152
- package/src/transports/metrics.ts +0 -95
- package/src/transports/rest.ts +0 -164
- package/src/transports/util.ts +0 -63
- package/src/transports/websocket.ts +0 -245
- package/src/util/index.ts +0 -22
- package/src/util/logger.ts +0 -69
- package/src/util/request.ts +0 -117
- package/src/util/subscription-set/expiring-sorted-set.ts +0 -54
- package/src/util/subscription-set/subscription-set.ts +0 -35
- package/src/util/test-payload-loader.ts +0 -87
- package/src/validation/error.ts +0 -116
- package/src/validation/index.ts +0 -110
- package/src/validation/input-params.ts +0 -45
- package/src/validation/override-functions.ts +0 -44
- package/src/validation/preset-tokens.json +0 -23
- package/src/validation/validator.ts +0 -384
- package/test/adapter.test.ts +0 -27
- package/test/background-executor.test.ts +0 -108
- package/test/cache/cache-key.test.ts +0 -114
- package/test/cache/helper.ts +0 -100
- package/test/cache/local.test.ts +0 -54
- package/test/cache/redis.test.ts +0 -89
- package/test/correlation.test.ts +0 -114
- package/test/index.test.ts +0 -37
- package/test/metrics/feed-id.test.ts +0 -38
- package/test/metrics/helper.ts +0 -14
- package/test/metrics/labels.test.ts +0 -36
- package/test/metrics/metrics.test.ts +0 -267
- package/test/metrics/redis-metrics.test.ts +0 -113
- package/test/metrics/warmer-metrics.test.ts +0 -192
- package/test/metrics/ws-metrics.test.ts +0 -225
- package/test/rate-limit-config.test.ts +0 -242
- package/test/smoke.test.ts +0 -166
- package/test/transports/batch.test.ts +0 -465
- package/test/transports/rest.test.ts +0 -242
- package/test/transports/websocket.test.ts +0 -183
- package/test/tsconfig.json +0 -5
- package/test/util.ts +0 -77
- package/test/validation.test.ts +0 -178
- package/test-payload-fail.json +0 -3
- package/test-payload.js +0 -22
- package/test-payload.json +0 -7
- package/test.sh +0 -20
- package/test2.sh +0 -2
- package/tsconfig.json +0 -25
- package/typedoc.json +0 -6
- package/webpack.config.js +0 -23
package/src/index.ts
DELETED
|
@@ -1,149 +0,0 @@
|
|
|
1
|
-
import fastify, { FastifyInstance } from 'fastify'
|
|
2
|
-
import { Server } from 'http'
|
|
3
|
-
import { AddressInfo } from 'net'
|
|
4
|
-
import { join } from 'path'
|
|
5
|
-
import { Adapter, InitializedAdapter, initializeAdapter, AdapterDependencies } from './adapter'
|
|
6
|
-
import { callBackgroundExecutes } from './background-executor'
|
|
7
|
-
import { buildCacheMiddleware } from './cache'
|
|
8
|
-
import { AdapterConfig, buildAdapterConfig } from './config'
|
|
9
|
-
import { buildMetricsMiddleware, setupMetricsServer } from './metrics'
|
|
10
|
-
import { buildTransportHandler } from './transports'
|
|
11
|
-
import { AdapterRouteGeneric, loggingContextMiddleware, makeLogger } from './util'
|
|
12
|
-
import { loadTestPayload } from './util/test-payload-loader'
|
|
13
|
-
import { errorCatchingMiddleware, validatorMiddleware } from './validation'
|
|
14
|
-
|
|
15
|
-
const logger = makeLogger('Main')
|
|
16
|
-
|
|
17
|
-
const VERSION = process.env['npm_package_version']
|
|
18
|
-
|
|
19
|
-
/**
|
|
20
|
-
* Main function for the framework.
|
|
21
|
-
* Initializes config and dependencies, uses those to initialize Transports, and starts listening for requests.
|
|
22
|
-
*
|
|
23
|
-
* @param adapter - an object describing an External Adapter
|
|
24
|
-
* @param dependencies - an optional object with adapter dependencies to inject
|
|
25
|
-
* @returns a Promise that resolves to the http.Server listening for connections
|
|
26
|
-
*/
|
|
27
|
-
export const expose = async (
|
|
28
|
-
adapter: Adapter,
|
|
29
|
-
dependencies?: Partial<AdapterDependencies>,
|
|
30
|
-
): Promise<Server | undefined> => {
|
|
31
|
-
const config = buildAdapterConfig({
|
|
32
|
-
overrides: adapter.envDefaultOverrides,
|
|
33
|
-
customSettings: adapter.customSettings,
|
|
34
|
-
})
|
|
35
|
-
// Initialize adapter (create dependencies, inject them, build endpoint map, etc.)
|
|
36
|
-
const initializedAdapter = await initializeAdapter(adapter, config, dependencies)
|
|
37
|
-
|
|
38
|
-
let server: Server | undefined = undefined
|
|
39
|
-
|
|
40
|
-
if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
|
|
41
|
-
setupMetricsServer(adapter.name, config)
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
if (config.EA_MODE === 'reader' || config.EA_MODE === 'reader-writer') {
|
|
45
|
-
// Main REST API server to handle incoming requests
|
|
46
|
-
const app = await buildRestApi(config, initializedAdapter)
|
|
47
|
-
|
|
48
|
-
// Start listening for incoming requests
|
|
49
|
-
try {
|
|
50
|
-
await app.listen(config.EA_PORT, config.EA_HOST)
|
|
51
|
-
} catch (err) {
|
|
52
|
-
logger.fatal(`There was an error when starting the EA server: ${err}`)
|
|
53
|
-
process.exit()
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
logger.info(`Listening on port ${(app.server.address() as AddressInfo).port}`)
|
|
57
|
-
server = app.server
|
|
58
|
-
} else {
|
|
59
|
-
logger.info('REST API is disabled; this instance will not process incoming requests.')
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
if (config.EA_MODE === 'writer' || config.EA_MODE === 'reader-writer') {
|
|
63
|
-
// Start background loop that will take care of calling any async Transports
|
|
64
|
-
logger.info('Starting background execution loop')
|
|
65
|
-
callBackgroundExecutes(initializedAdapter, server)
|
|
66
|
-
} else {
|
|
67
|
-
logger.info(
|
|
68
|
-
'Background executor is disabled; this instance will not perform async background executes.',
|
|
69
|
-
)
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
return server
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
async function buildRestApi(config: AdapterConfig, initializedAdapter: InitializedAdapter) {
|
|
76
|
-
const app = fastify()
|
|
77
|
-
|
|
78
|
-
// Add healthcheck endpoint before middlewares to bypass them
|
|
79
|
-
app.get(join(config.BASE_URL, 'health'), (req, res) => {
|
|
80
|
-
res.status(200).send({ message: 'OK', version: VERSION })
|
|
81
|
-
})
|
|
82
|
-
|
|
83
|
-
// Use global error handling
|
|
84
|
-
app.setErrorHandler(errorCatchingMiddleware)
|
|
85
|
-
|
|
86
|
-
const transportHandler = await buildTransportHandler(initializedAdapter)
|
|
87
|
-
|
|
88
|
-
app.register(async (router) => {
|
|
89
|
-
// Set up "middlewares" (hooks in fastify)
|
|
90
|
-
router.addHook<AdapterRouteGeneric>('preHandler', validatorMiddleware(initializedAdapter))
|
|
91
|
-
router.addHook<AdapterRouteGeneric>('preHandler', buildCacheMiddleware(initializedAdapter))
|
|
92
|
-
if (config['CORRELATION_ID_ENABLED']) {
|
|
93
|
-
router.addHook<AdapterRouteGeneric>('onRequest', loggingContextMiddleware)
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
router.route<AdapterRouteGeneric>({
|
|
97
|
-
url: config.BASE_URL,
|
|
98
|
-
method: 'POST',
|
|
99
|
-
handler: transportHandler,
|
|
100
|
-
})
|
|
101
|
-
|
|
102
|
-
if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
|
|
103
|
-
router.addHook<AdapterRouteGeneric>('onResponse', buildMetricsMiddleware)
|
|
104
|
-
}
|
|
105
|
-
})
|
|
106
|
-
|
|
107
|
-
// Add smoke endpoint after middleware which are needed for tests
|
|
108
|
-
buildSmokeEndpoint(app, config)
|
|
109
|
-
return app
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
/**
|
|
113
|
-
* Adds the /smoke endpoint to the API for smoke testing the adapter
|
|
114
|
-
*
|
|
115
|
-
* @param app - the Fastify instance
|
|
116
|
-
* @param config - the initialized adapter config
|
|
117
|
-
*/
|
|
118
|
-
function buildSmokeEndpoint(app: FastifyInstance, config: AdapterConfig) {
|
|
119
|
-
const testPayload = loadTestPayload(config.SMOKE_TEST_PAYLOAD_FILE_NAME)
|
|
120
|
-
app.get(join(config.BASE_URL, 'smoke'), async (_, res) => {
|
|
121
|
-
if (testPayload.isDefault) {
|
|
122
|
-
return res.status(200).send('OK')
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
const errors = []
|
|
126
|
-
for (const index in testPayload.requests) {
|
|
127
|
-
try {
|
|
128
|
-
const request = { id: index, data: testPayload.requests[index] }
|
|
129
|
-
// Use Fastify's app inject to pass smoke requests internally
|
|
130
|
-
const response = await app.inject({
|
|
131
|
-
method: 'POST',
|
|
132
|
-
url: '/',
|
|
133
|
-
payload: request,
|
|
134
|
-
})
|
|
135
|
-
const parsedResponse = JSON.parse(response.body)
|
|
136
|
-
// Throw error if not 2xx status code
|
|
137
|
-
if (parsedResponse.statusCode < 200 || parsedResponse.statusCode > 299) {
|
|
138
|
-
throw Error('Smoke test request failed')
|
|
139
|
-
}
|
|
140
|
-
} catch (e: unknown) {
|
|
141
|
-
errors.push(e)
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
|
-
if (errors.length > 0) {
|
|
145
|
-
return res.status(500).send(errors)
|
|
146
|
-
}
|
|
147
|
-
return res.status(200).send('OK')
|
|
148
|
-
})
|
|
149
|
-
}
|
package/src/metrics/constants.ts
DELETED
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
export enum HttpRequestType {
|
|
2
|
-
CACHE_HIT = 'cacheHit',
|
|
3
|
-
DATA_PROVIDER_HIT = 'dataProviderHit',
|
|
4
|
-
ADAPTER_ERROR = 'adapterError',
|
|
5
|
-
INPUT_ERROR = 'inputError',
|
|
6
|
-
RATE_LIMIT_ERROR = 'rateLimitError',
|
|
7
|
-
// BURST_LIMIT_ERROR = 'burstLimitError',
|
|
8
|
-
// BACKOFF_ERROR = 'backoffError',
|
|
9
|
-
DP_ERROR = 'dataProviderError',
|
|
10
|
-
TIMEOUT_ERROR = 'timeoutError',
|
|
11
|
-
CONNECTION_ERROR = 'connectionError',
|
|
12
|
-
// RES_EMPTY_ERROR = 'responseEmptyError',
|
|
13
|
-
// RES_INVALID_ERROR = 'responseInvalidError',
|
|
14
|
-
CUSTOM_ERROR = 'customError',
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
/**
|
|
18
|
-
* Maxiumum number of characters that a feedId can contain.
|
|
19
|
-
*/
|
|
20
|
-
export const MAX_FEED_ID_LENGTH = 300
|
|
21
|
-
|
|
22
|
-
// We should tune these as we collect data, this is the default bucket distribution that prom comes with
|
|
23
|
-
export const requestDurationBuckets = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]
|
package/src/metrics/index.ts
DELETED
|
@@ -1,115 +0,0 @@
|
|
|
1
|
-
import * as client from 'prom-client'
|
|
2
|
-
import { HttpRequestType, requestDurationBuckets } from './constants'
|
|
3
|
-
import { AdapterRequest, makeLogger } from '../util'
|
|
4
|
-
import { AdapterConfig } from '../config'
|
|
5
|
-
import fastify, { FastifyReply, HookHandlerDoneFunction } from 'fastify'
|
|
6
|
-
import { join } from 'path'
|
|
7
|
-
import { AdapterError } from '../validation/error'
|
|
8
|
-
|
|
9
|
-
const logger = makeLogger('Metrics')
|
|
10
|
-
|
|
11
|
-
export function setupMetricsServer(name: string, config: AdapterConfig) {
|
|
12
|
-
const metricsApp = fastify({
|
|
13
|
-
logger: false,
|
|
14
|
-
})
|
|
15
|
-
const metricsPort = config.METRICS_PORT
|
|
16
|
-
const endpoint = config.METRICS_USE_BASE_URL ? join(config.BASE_URL, 'metrics') : '/metrics'
|
|
17
|
-
const eaHost = config.EA_HOST
|
|
18
|
-
|
|
19
|
-
setupMetrics(name, config)
|
|
20
|
-
|
|
21
|
-
metricsApp.get(endpoint, async (_, res) => {
|
|
22
|
-
res.type('txt')
|
|
23
|
-
res.send(await client.register.metrics())
|
|
24
|
-
})
|
|
25
|
-
|
|
26
|
-
metricsApp.listen(metricsPort, eaHost, () =>
|
|
27
|
-
logger.info(`Monitoring listening on port ${metricsPort}!`),
|
|
28
|
-
)
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
export const setupMetrics = (name: string, config: AdapterConfig): void => {
|
|
32
|
-
client.collectDefaultMetrics()
|
|
33
|
-
client.register.setDefaultLabels({
|
|
34
|
-
app_name: config.METRICS_NAME || name || 'N/A',
|
|
35
|
-
app_version: config['npm_package_version'],
|
|
36
|
-
})
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
/**
|
|
40
|
-
* Builds metrics middleware that records end to end EA response times
|
|
41
|
-
* and count of requests
|
|
42
|
-
*
|
|
43
|
-
* @returns the cache middleware function
|
|
44
|
-
*/
|
|
45
|
-
export const buildMetricsMiddleware = (
|
|
46
|
-
req: AdapterRequest,
|
|
47
|
-
res: FastifyReply,
|
|
48
|
-
done: HookHandlerDoneFunction,
|
|
49
|
-
) => {
|
|
50
|
-
const feedId = req.requestContext.meta?.metrics?.feedId || 'N/A'
|
|
51
|
-
const labels = buildHttpRequestMetricsLabel(
|
|
52
|
-
feedId,
|
|
53
|
-
req.requestContext.meta?.error,
|
|
54
|
-
req.requestContext.meta?.metrics?.cacheHit,
|
|
55
|
-
)
|
|
56
|
-
|
|
57
|
-
// Record number of requests sent to EA
|
|
58
|
-
httpRequestsTotal.labels(labels).inc()
|
|
59
|
-
|
|
60
|
-
// Record response time of request through entire EA
|
|
61
|
-
httpRequestDurationSeconds.observe(res.getResponseTime())
|
|
62
|
-
logger.debug(`Response time for ${feedId}: ${res.getResponseTime()}ms`)
|
|
63
|
-
done()
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
const buildHttpRequestMetricsLabel = (
|
|
67
|
-
feedId: string,
|
|
68
|
-
error?: AdapterError | Error,
|
|
69
|
-
cacheHit?: boolean,
|
|
70
|
-
): Parameters<typeof httpRequestsTotal.labels>[0] => {
|
|
71
|
-
const labels: Parameters<typeof httpRequestsTotal.labels>[0] = {}
|
|
72
|
-
labels.method = 'POST'
|
|
73
|
-
labels.feed_id = feedId
|
|
74
|
-
if (error instanceof AdapterError) {
|
|
75
|
-
// If error present and an instace of AdapterError, build label from error info
|
|
76
|
-
labels.type = error?.metricsLabel || HttpRequestType.ADAPTER_ERROR
|
|
77
|
-
labels.status_code = error?.statusCode
|
|
78
|
-
labels.provider_status_code = error?.providerStatusCode
|
|
79
|
-
} else if (error instanceof Error) {
|
|
80
|
-
// If error present and not instance of generic Error, unexpected failure occurred
|
|
81
|
-
labels.type = HttpRequestType.ADAPTER_ERROR
|
|
82
|
-
labels.status_code = 500
|
|
83
|
-
} else {
|
|
84
|
-
// If no error present, request went as expected
|
|
85
|
-
labels.status_code = 200
|
|
86
|
-
if (cacheHit) {
|
|
87
|
-
labels.type = HttpRequestType.CACHE_HIT
|
|
88
|
-
} else {
|
|
89
|
-
labels.type = HttpRequestType.DATA_PROVIDER_HIT
|
|
90
|
-
labels.provider_status_code = 200
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
return labels
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
export const httpRequestsTotal = new client.Counter({
|
|
98
|
-
name: 'http_requests_total',
|
|
99
|
-
help: 'The number of http requests this external adapter has serviced for its entire uptime',
|
|
100
|
-
labelNames: [
|
|
101
|
-
'method',
|
|
102
|
-
'status_code',
|
|
103
|
-
'retry',
|
|
104
|
-
'type',
|
|
105
|
-
'is_cache_warming',
|
|
106
|
-
'feed_id',
|
|
107
|
-
'provider_status_code',
|
|
108
|
-
] as const,
|
|
109
|
-
})
|
|
110
|
-
|
|
111
|
-
export const httpRequestDurationSeconds = new client.Histogram({
|
|
112
|
-
name: 'http_request_duration_seconds',
|
|
113
|
-
help: 'A histogram bucket of the distribution of http request durations',
|
|
114
|
-
buckets: requestDurationBuckets,
|
|
115
|
-
})
|
package/src/metrics/util.ts
DELETED
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
import { AdapterMetricsMeta, AdapterRequestData } from '../util'
|
|
2
|
-
import { AdapterEndpoint } from '../adapter'
|
|
3
|
-
import { calculateFeedId } from '../cache'
|
|
4
|
-
import { AdapterConfig } from '../config'
|
|
5
|
-
|
|
6
|
-
export const getMetricsMeta = (
|
|
7
|
-
{
|
|
8
|
-
adapterEndpoint,
|
|
9
|
-
adapterConfig,
|
|
10
|
-
}: {
|
|
11
|
-
adapterEndpoint: AdapterEndpoint
|
|
12
|
-
adapterConfig: AdapterConfig
|
|
13
|
-
},
|
|
14
|
-
data: AdapterRequestData,
|
|
15
|
-
): AdapterMetricsMeta => {
|
|
16
|
-
const feedId = calculateFeedId({ adapterEndpoint, adapterConfig }, data)
|
|
17
|
-
return { feedId }
|
|
18
|
-
}
|
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
import { AdapterRateLimitTier, BackgroundExecuteRateLimiter, consolidateTierLimits } from '..'
|
|
2
|
-
import { AdapterEndpoint } from '../../adapter'
|
|
3
|
-
import { makeLogger } from '../../util'
|
|
4
|
-
|
|
5
|
-
const logger = makeLogger('FixedFrequencyRateLimiter')
|
|
6
|
-
export const DEFAULT_SHARED_MS_BETWEEN_REQUESTS = 5000
|
|
7
|
-
|
|
8
|
-
export class FixedFrequencyRateLimiter implements BackgroundExecuteRateLimiter {
|
|
9
|
-
msBetweenRequestsMap: {
|
|
10
|
-
[endpointName: string]: number // In ms
|
|
11
|
-
} = {}
|
|
12
|
-
|
|
13
|
-
initialize(endpoints: AdapterEndpoint[], limits?: AdapterRateLimitTier) {
|
|
14
|
-
// Translate the hourly limit into reqs per minute
|
|
15
|
-
let sharedMsBetweenRequests = 1000 / consolidateTierLimits(limits)
|
|
16
|
-
|
|
17
|
-
// If there is no limit set, we use some reasonable number
|
|
18
|
-
if (!limits?.rateLimit1h && !limits?.rateLimit1m && !limits?.rateLimit1s) {
|
|
19
|
-
// 5s period for all seems good
|
|
20
|
-
sharedMsBetweenRequests = DEFAULT_SHARED_MS_BETWEEN_REQUESTS
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
logger.debug('Using fixed frequency batch rate limiting')
|
|
24
|
-
for (const endpoint of endpoints) {
|
|
25
|
-
if (endpoint.rateLimiting?.allocationPercentage == null) {
|
|
26
|
-
throw new Error(`Allocation percentage for endpoint "${endpoint.name}" is null`)
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
this.msBetweenRequestsMap[endpoint.name] =
|
|
30
|
-
(sharedMsBetweenRequests / endpoint.rateLimiting?.allocationPercentage) * 100
|
|
31
|
-
|
|
32
|
-
logger.debug(
|
|
33
|
-
`Endpoint [${endpoint.name}]: ${
|
|
34
|
-
this.msBetweenRequestsMap[endpoint.name] / 1000
|
|
35
|
-
}s between requests`,
|
|
36
|
-
)
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
return this
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
msUntilNextExecution(endpointName: string): number {
|
|
43
|
-
return this.msBetweenRequestsMap[endpointName]
|
|
44
|
-
}
|
|
45
|
-
}
|
|
@@ -1,100 +0,0 @@
|
|
|
1
|
-
import { AdapterEndpoint } from '../adapter'
|
|
2
|
-
|
|
3
|
-
export * from './request/simple-counting'
|
|
4
|
-
export * from './background/fixed-frequency'
|
|
5
|
-
|
|
6
|
-
export interface AdapterRateLimitTier {
|
|
7
|
-
rateLimit1s?: number
|
|
8
|
-
rateLimit1m?: number
|
|
9
|
-
rateLimit1h?: number
|
|
10
|
-
note?: string
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
/**
|
|
14
|
-
* Common interface for all RateLimiter classes to implement
|
|
15
|
-
*/
|
|
16
|
-
export interface RateLimiter {
|
|
17
|
-
/**
|
|
18
|
-
* Method to ensure all RateLimiters can be initialized in the same manner.
|
|
19
|
-
*
|
|
20
|
-
* @param limits - settings for how much throughput to allow for the Adapter
|
|
21
|
-
* @param endpoints - list of adapter endpoints
|
|
22
|
-
*/
|
|
23
|
-
initialize(endpoints: AdapterEndpoint[], limits: AdapterRateLimitTier): this
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
/**
|
|
27
|
-
* RequestRateLimiters perform checks agains imminent outbound requests for any transport.
|
|
28
|
-
*/
|
|
29
|
-
export interface RequestRateLimiter extends RateLimiter {
|
|
30
|
-
/**
|
|
31
|
-
* This method will check using whatever strategy is implemented to determine if
|
|
32
|
-
* this request can be processed. If so, it returns true; if not, returns false.
|
|
33
|
-
*/
|
|
34
|
-
isUnderLimits(): boolean
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
/**
|
|
38
|
-
* BackgroundExecuteFrequencyRateLimiters will implement custom logic to calculate
|
|
39
|
-
* the period of time to wait between background executions for a transport.
|
|
40
|
-
*/
|
|
41
|
-
export interface BackgroundExecuteRateLimiter extends RateLimiter {
|
|
42
|
-
msUntilNextExecution(endpointName: string): number
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
/**
|
|
46
|
-
* This method will convert all possible settings for a rate limit tier and
|
|
47
|
-
* convert them all to requests per second, returning the lowest one
|
|
48
|
-
*
|
|
49
|
-
* @param limits - the rate limit tier set for the adapter
|
|
50
|
-
* @returns the most restrictive of the set options, in requests per second
|
|
51
|
-
*/
|
|
52
|
-
export const consolidateTierLimits = (limits?: AdapterRateLimitTier) => {
|
|
53
|
-
const perHourLimit = (limits?.rateLimit1h || Infinity) / (60 * 60)
|
|
54
|
-
const perMinuteLimit = (limits?.rateLimit1m || Infinity) / 60
|
|
55
|
-
const perSecondLimit = limits?.rateLimit1s || Infinity
|
|
56
|
-
return Math.min(perHourLimit, perMinuteLimit, perSecondLimit)
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
/**
|
|
60
|
-
* Validates rate limiting tiers specified for the adapter, and returns the one to use.
|
|
61
|
-
*
|
|
62
|
-
* @param tiers - the adapter config listing the different available API tiers
|
|
63
|
-
* @param selectedTier - chosen API tier from settings, if present
|
|
64
|
-
* @returns the specified API tier, or a default one if none are specified
|
|
65
|
-
*/
|
|
66
|
-
export const getRateLimitingTier = (
|
|
67
|
-
tiers?: Record<string, AdapterRateLimitTier>,
|
|
68
|
-
selectedTier?: string,
|
|
69
|
-
): AdapterRateLimitTier | undefined => {
|
|
70
|
-
if (!tiers) {
|
|
71
|
-
return
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
// Check that if the tiers object is defined, it has values
|
|
75
|
-
if (Object.values(tiers).length === 0) {
|
|
76
|
-
throw new Error(`The tiers object is defined, but has no entries`)
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
// Check that the tier set in the AdapterConfig is a valid one
|
|
80
|
-
if (selectedTier && !tiers[selectedTier]) {
|
|
81
|
-
const validTiersString = Object.keys(tiers)
|
|
82
|
-
.map((t) => `"${t}"`)
|
|
83
|
-
.join(', ')
|
|
84
|
-
|
|
85
|
-
throw new Error(
|
|
86
|
-
`The selected rate limit tier "${selectedTier}" is not valid (can be one of ${validTiersString})`,
|
|
87
|
-
)
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
if (!selectedTier) {
|
|
91
|
-
// Sort the tiers by most to least restrictive
|
|
92
|
-
const sortedTiers = Object.values(tiers).sort(
|
|
93
|
-
(t1, t2) => consolidateTierLimits(t1) - consolidateTierLimits(t2),
|
|
94
|
-
)
|
|
95
|
-
|
|
96
|
-
return sortedTiers[0]
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
return tiers[selectedTier]
|
|
100
|
-
}
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
import * as client from 'prom-client'
|
|
2
|
-
|
|
3
|
-
// Retrieve cost field from response if exists
|
|
4
|
-
// If not return default cost of 1
|
|
5
|
-
export const retrieveCost = <ProviderResponseBody>(data: ProviderResponseBody): number => {
|
|
6
|
-
const cost = (data as Record<string, unknown>)['cost']
|
|
7
|
-
if (typeof cost === 'number' || typeof cost === 'string') {
|
|
8
|
-
return Number(cost)
|
|
9
|
-
} else {
|
|
10
|
-
return 1
|
|
11
|
-
}
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
export const rateLimitCreditsSpentTotal = new client.Counter({
|
|
15
|
-
name: 'rate_limit_credits_spent_total',
|
|
16
|
-
help: 'The number of data provider credits the adapter is consuming',
|
|
17
|
-
labelNames: ['participant_id', 'feed_id'] as const,
|
|
18
|
-
})
|
|
@@ -1,76 +0,0 @@
|
|
|
1
|
-
import { AdapterRateLimitTier, RateLimiter } from '..'
|
|
2
|
-
import { AdapterEndpoint } from '../../adapter'
|
|
3
|
-
import { makeLogger } from '../../util'
|
|
4
|
-
|
|
5
|
-
const logger = makeLogger('SimpleCountingRateLimiter')
|
|
6
|
-
|
|
7
|
-
/**
|
|
8
|
-
* This rate limiter is the simplest stateful option.
|
|
9
|
-
* On startup, it'll compare the different thresholds for each tier, calculate them all
|
|
10
|
-
* in the finest window we'll use (seconds), and use the most restrictive one.
|
|
11
|
-
* This is so if the EA were to restart, we don't need to worry about persisting state
|
|
12
|
-
* for things like daily quotas. The downside is that this does not work well for bursty
|
|
13
|
-
* loads or spikes, in cases where e.g. the per second limit is high but daily quotas low.
|
|
14
|
-
*/
|
|
15
|
-
export class SimpleCountingRateLimiter implements RateLimiter {
|
|
16
|
-
latestSecondInterval = 0
|
|
17
|
-
requestsThisSecond = 0
|
|
18
|
-
latestMinuteInterval = 0
|
|
19
|
-
requestsThisMinute = 0
|
|
20
|
-
perSecondLimit!: number
|
|
21
|
-
perMinuteLimit!: number
|
|
22
|
-
|
|
23
|
-
initialize(endpoints: AdapterEndpoint[], limits?: AdapterRateLimitTier) {
|
|
24
|
-
// Translate the hourly limit into reqs per minute
|
|
25
|
-
const perHourLimit = (limits?.rateLimit1h || Infinity) / 60
|
|
26
|
-
this.perMinuteLimit = Math.min(limits?.rateLimit1m || Infinity, perHourLimit)
|
|
27
|
-
this.perSecondLimit = limits?.rateLimit1s || Infinity
|
|
28
|
-
logger.debug(
|
|
29
|
-
`Using rate limiting settings: perMinute = ${this.perMinuteLimit} | perSecond: = ${this.perSecondLimit}`,
|
|
30
|
-
)
|
|
31
|
-
|
|
32
|
-
return this
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
isUnderLimits() {
|
|
36
|
-
// If the limit is set to infinity, there was no tier limit specified
|
|
37
|
-
if (this.perSecondLimit === Infinity && this.perMinuteLimit === Infinity) {
|
|
38
|
-
return true
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
const now = Date.now()
|
|
42
|
-
const nearestSecondInterval = Math.floor(now / 1000)
|
|
43
|
-
const nearestMinuteInterval = Math.floor(now / (1000 * 60))
|
|
44
|
-
|
|
45
|
-
// This should always run to completion, even if it doesn't look atomic; therefore the
|
|
46
|
-
// Ops should be "thread safe". Thank JS and its infinite single threaded dumbness.
|
|
47
|
-
if (nearestSecondInterval !== this.latestSecondInterval) {
|
|
48
|
-
logger.trace(
|
|
49
|
-
`Clearing latest second interval, # of requests logged was: ${this.requestsThisSecond} `,
|
|
50
|
-
)
|
|
51
|
-
this.latestSecondInterval = nearestSecondInterval
|
|
52
|
-
this.requestsThisSecond = 0
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
if (nearestMinuteInterval !== this.latestMinuteInterval) {
|
|
56
|
-
logger.trace(
|
|
57
|
-
`Clearing latest second minute, # of requests logged was: ${this.requestsThisMinute} `,
|
|
58
|
-
)
|
|
59
|
-
this.latestMinuteInterval = nearestMinuteInterval
|
|
60
|
-
this.requestsThisMinute = 0
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
if (
|
|
64
|
-
this.requestsThisSecond < this.perSecondLimit &&
|
|
65
|
-
this.requestsThisMinute < this.perMinuteLimit
|
|
66
|
-
) {
|
|
67
|
-
logger.trace('Request under limits, counting +1')
|
|
68
|
-
this.requestsThisSecond++
|
|
69
|
-
this.requestsThisMinute++
|
|
70
|
-
return true
|
|
71
|
-
} else {
|
|
72
|
-
logger.trace('Requests seen this interval are above limits')
|
|
73
|
-
return false
|
|
74
|
-
}
|
|
75
|
-
}
|
|
76
|
-
}
|
package/src/test.ts
DELETED
|
@@ -1,122 +0,0 @@
|
|
|
1
|
-
import { AxiosRequestConfig, AxiosResponse } from 'axios'
|
|
2
|
-
import { Cache } from '../cache'
|
|
3
|
-
import { AdapterConfig, SettingsMap } from '../config'
|
|
4
|
-
import { BackgroundExecuteRateLimiter } from '../rate-limiting'
|
|
5
|
-
import { makeLogger, SubscriptionSet } from '../util'
|
|
6
|
-
import { AdapterRequest, ProviderResult } from '../util/request'
|
|
7
|
-
import { Transport, buildCacheEntriesFromResults } from './'
|
|
8
|
-
import { axiosRequest } from './util'
|
|
9
|
-
import * as rateLimitMetrics from '../rate-limiting/metrics'
|
|
10
|
-
import * as cacheMetrics from '../cache/metrics'
|
|
11
|
-
import { AdapterContext, AdapterDependencies } from '../adapter'
|
|
12
|
-
|
|
13
|
-
const WARMUP_BATCH_REQUEST_ID = '9002'
|
|
14
|
-
|
|
15
|
-
const logger = makeLogger('BatchWarmingTransport')
|
|
16
|
-
|
|
17
|
-
/**
|
|
18
|
-
* Transport implementation that takes incoming batches requests and keeps a warm cache of values.
|
|
19
|
-
* Within the setup function, adapter params are added to an set that also keeps track and expires values.
|
|
20
|
-
* In the background execute, the list of non-expired items in the set is fetched.
|
|
21
|
-
* Then, the list is passed through the `prepareRequest` function, that returns an AxiosRequestConfig.
|
|
22
|
-
* The Data Provider response is, they are passed through the `parseResponse` function to create a [[CacheEntry]] list.
|
|
23
|
-
* Finally, the items in that [[CacheEntry]] list are set in the Cache so the Adapter can fetch values from there.
|
|
24
|
-
*
|
|
25
|
-
* @typeParam AdapterParams - interface for the adapter request body
|
|
26
|
-
* @typeParam ProviderRequestBody - interface for the body of the request to the Data Provider
|
|
27
|
-
* @typeParam ProviderResponseBody - interface for the body of the Data Provider's response
|
|
28
|
-
*/
|
|
29
|
-
export class BatchWarmingTransport<
|
|
30
|
-
AdapterParams,
|
|
31
|
-
ProviderRequestBody,
|
|
32
|
-
ProviderResponseBody,
|
|
33
|
-
CustomSettings extends SettingsMap,
|
|
34
|
-
> implements Transport<AdapterParams, ProviderResponseBody, CustomSettings>
|
|
35
|
-
{
|
|
36
|
-
cache!: Cache
|
|
37
|
-
rateLimiter!: BackgroundExecuteRateLimiter
|
|
38
|
-
subscriptionSet!: SubscriptionSet<AdapterParams>
|
|
39
|
-
|
|
40
|
-
// Flag used to track whether the warmer has moved from having no entries to having some and vice versa
|
|
41
|
-
// Used for recording the cache warmer active metrics accurately
|
|
42
|
-
WARMER_ACTIVE = false
|
|
43
|
-
|
|
44
|
-
constructor(
|
|
45
|
-
private config: {
|
|
46
|
-
prepareRequest: (
|
|
47
|
-
params: AdapterParams[],
|
|
48
|
-
context: AdapterContext<CustomSettings>,
|
|
49
|
-
) => AxiosRequestConfig<ProviderRequestBody>
|
|
50
|
-
parseResponse: (
|
|
51
|
-
res: AxiosResponse<ProviderResponseBody>,
|
|
52
|
-
context: AdapterContext<CustomSettings>,
|
|
53
|
-
) => ProviderResult<AdapterParams>[]
|
|
54
|
-
},
|
|
55
|
-
) {}
|
|
56
|
-
|
|
57
|
-
async initialize(dependencies: AdapterDependencies): Promise<void> {
|
|
58
|
-
this.cache = dependencies.cache
|
|
59
|
-
this.rateLimiter = dependencies.backgroundExecuteRateLimiter
|
|
60
|
-
this.subscriptionSet = dependencies.subscriptionSetFactory.buildSet()
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
async hasBeenSetUp(req: AdapterRequest<AdapterParams>): Promise<boolean> {
|
|
64
|
-
return !!(await this.subscriptionSet.get(req.requestContext.cacheKey))
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
async setup(
|
|
68
|
-
req: AdapterRequest<AdapterParams>,
|
|
69
|
-
config: AdapterConfig<CustomSettings>,
|
|
70
|
-
): Promise<void> {
|
|
71
|
-
logger.debug(
|
|
72
|
-
`Adding entry to batch warming set: [${req.requestContext.cacheKey}] = ${req.requestContext.data}`,
|
|
73
|
-
)
|
|
74
|
-
await this.subscriptionSet.add(
|
|
75
|
-
req.requestContext.cacheKey,
|
|
76
|
-
req.requestContext.data,
|
|
77
|
-
config.WARMUP_SUBSCRIPTION_TTL,
|
|
78
|
-
)
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
async backgroundExecute(context: AdapterContext<CustomSettings>): Promise<number> {
|
|
82
|
-
logger.debug('Starting background execute')
|
|
83
|
-
const entries = await this.subscriptionSet.getAll()
|
|
84
|
-
|
|
85
|
-
if (!entries.length) {
|
|
86
|
-
logger.debug('No entries in batch warming set, skipping')
|
|
87
|
-
if (this.WARMER_ACTIVE) {
|
|
88
|
-
// Decrement count when warmer changed from having entries to having none
|
|
89
|
-
cacheMetrics.cacheWarmerCount.labels({ isBatched: 'true' }).dec()
|
|
90
|
-
this.WARMER_ACTIVE = false
|
|
91
|
-
}
|
|
92
|
-
return this.rateLimiter.msUntilNextExecution(context.adapterEndpoint.name)
|
|
93
|
-
} else if (this.WARMER_ACTIVE === false) {
|
|
94
|
-
// Increment count when warmer changed from having no entries to having some
|
|
95
|
-
cacheMetrics.cacheWarmerCount.labels({ isBatched: 'true' }).inc()
|
|
96
|
-
this.WARMER_ACTIVE = true
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
const request = this.config.prepareRequest(entries, context)
|
|
100
|
-
|
|
101
|
-
logger.trace('Sending request to data provider...')
|
|
102
|
-
const providerResponse = await axiosRequest<ProviderRequestBody, ProviderResponseBody>(request)
|
|
103
|
-
|
|
104
|
-
logger.debug(`Got response from provider, parsing (raw body: ${providerResponse.data})`)
|
|
105
|
-
const results = this.config.parseResponse(providerResponse, context)
|
|
106
|
-
const adapterResponses = buildCacheEntriesFromResults(results, context)
|
|
107
|
-
|
|
108
|
-
logger.debug('Setting adapter responses in cache')
|
|
109
|
-
await this.cache.setMany(adapterResponses, context.adapterConfig.CACHE_MAX_AGE)
|
|
110
|
-
|
|
111
|
-
// Record cost of data provider call
|
|
112
|
-
const cost = rateLimitMetrics.retrieveCost(providerResponse.data)
|
|
113
|
-
rateLimitMetrics.rateLimitCreditsSpentTotal
|
|
114
|
-
.labels({
|
|
115
|
-
feed_id: 'N/A',
|
|
116
|
-
participant_id: WARMUP_BATCH_REQUEST_ID,
|
|
117
|
-
})
|
|
118
|
-
.inc(cost)
|
|
119
|
-
|
|
120
|
-
return this.rateLimiter.msUntilNextExecution(context.adapterEndpoint.name)
|
|
121
|
-
}
|
|
122
|
-
}
|