@chainlink/external-adapter-framework 0.0.14 → 0.0.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adapter.js +114 -101
- package/cache/factory.js +3 -0
- package/chainlink-external-adapter-framework-v0.0.6.tgz +0 -0
- package/config/index.js +69 -83
- package/config/provider-limits.js +10 -15
- package/examples/bank-frick/accounts.js +3 -2
- package/examples/bank-frick/index.js +3 -2
- package/examples/coingecko/src/config/index.js +13 -0
- package/examples/coingecko/src/config/overrides.json +10826 -0
- package/examples/coingecko/src/cryptoUtils.js +41 -0
- package/examples/coingecko/src/endpoint/coins.js +33 -0
- package/examples/coingecko/src/endpoint/crypto-marketcap.js +46 -0
- package/examples/coingecko/src/endpoint/crypto-volume.js +46 -0
- package/examples/coingecko/src/endpoint/crypto.js +47 -0
- package/examples/coingecko/src/endpoint/dominance.js +26 -0
- package/examples/coingecko/src/endpoint/global-marketcap.js +26 -0
- package/examples/coingecko/src/endpoint/index.js +15 -0
- package/examples/coingecko/src/globalUtils.js +48 -0
- package/examples/coingecko/src/index.js +14 -0
- package/examples/coingecko/test/e2e/adapter.test.js +262 -0
- package/examples/coingecko/test/integration/adapter.test.js +264 -0
- package/examples/coingecko/test/integration/capturedRequests.json +1 -0
- package/examples/coingecko/test/integration/fixtures.js +41 -0
- package/{package/examples/coingecko → examples/coingecko-old}/batch-warming.js +4 -3
- package/examples/{coingecko → coingecko-old}/index.js +3 -2
- package/{package/examples/coingecko → examples/coingecko-old}/rest.js +3 -2
- package/examples/ncfx/index.js +3 -2
- package/examples/ncfx/websocket.js +3 -2
- package/index.js +17 -23
- package/package.json +5 -4
- package/rate-limiting/index.js +23 -2
- package/transports/batch-warming.js +1 -1
- package/transports/rest.js +9 -2
- package/transports/util.js +2 -1
- package/transports/websocket.js +4 -6
- package/util/recordRequests.js +45 -0
- package/{package/validation/override-functions.js → validation/overrideFunctions.js} +0 -0
- package/adapter.d.ts +0 -107
- package/background-executor.d.ts +0 -11
- package/cache/factory.d.ts +0 -6
- package/cache/index.d.ts +0 -94
- package/cache/local.d.ts +0 -23
- package/cache/metrics.d.ts +0 -27
- package/cache/redis.d.ts +0 -16
- package/chainlink-external-adapter-framework-0.0.6.tgz +0 -0
- package/config/index.d.ts +0 -209
- package/config/provider-limits.d.ts +0 -31
- package/examples/bank-frick/accounts.d.ts +0 -39
- package/examples/bank-frick/config/index.d.ts +0 -4
- package/examples/bank-frick/index.d.ts +0 -2
- package/examples/bank-frick/util.d.ts +0 -4
- package/examples/coingecko/batch-warming.d.ts +0 -2
- package/examples/coingecko/batch-warming.js +0 -52
- package/examples/coingecko/index.d.ts +0 -2
- package/examples/coingecko/rest.d.ts +0 -2
- package/examples/coingecko/rest.js +0 -50
- package/examples/ncfx/config/index.d.ts +0 -12
- package/examples/ncfx/index.d.ts +0 -2
- package/examples/ncfx/websocket.d.ts +0 -36
- package/index.d.ts +0 -11
- package/metrics/constants.d.ts +0 -16
- package/metrics/index.d.ts +0 -15
- package/metrics/util.d.ts +0 -7
- package/package/adapter.d.ts +0 -88
- package/package/adapter.js +0 -112
- package/package/background-executor.d.ts +0 -11
- package/package/background-executor.js +0 -45
- package/package/cache/factory.d.ts +0 -6
- package/package/cache/factory.js +0 -57
- package/package/cache/index.d.ts +0 -90
- package/package/cache/index.js +0 -169
- package/package/cache/local.d.ts +0 -23
- package/package/cache/local.js +0 -83
- package/package/cache/metrics.d.ts +0 -27
- package/package/cache/metrics.js +0 -120
- package/package/cache/redis.d.ts +0 -16
- package/package/cache/redis.js +0 -100
- package/package/config/index.d.ts +0 -195
- package/package/config/index.js +0 -365
- package/package/config/provider-limits.d.ts +0 -31
- package/package/config/provider-limits.js +0 -76
- package/package/examples/coingecko/batch-warming.d.ts +0 -2
- package/package/examples/coingecko/index.d.ts +0 -2
- package/package/examples/coingecko/index.js +0 -10
- package/package/examples/coingecko/rest.d.ts +0 -2
- package/package/examples/ncfx/config/index.d.ts +0 -12
- package/package/examples/ncfx/config/index.js +0 -15
- package/package/examples/ncfx/index.d.ts +0 -2
- package/package/examples/ncfx/index.js +0 -10
- package/package/examples/ncfx/websocket.d.ts +0 -36
- package/package/examples/ncfx/websocket.js +0 -72
- package/package/index.d.ts +0 -12
- package/package/index.js +0 -92
- package/package/metrics/constants.d.ts +0 -16
- package/package/metrics/constants.js +0 -25
- package/package/metrics/index.d.ts +0 -15
- package/package/metrics/index.js +0 -123
- package/package/metrics/util.d.ts +0 -3
- package/package/metrics/util.js +0 -9
- package/package/package.json +0 -72
- package/package/rate-limiting/background/fixed-frequency.d.ts +0 -10
- package/package/rate-limiting/background/fixed-frequency.js +0 -37
- package/package/rate-limiting/index.d.ts +0 -54
- package/package/rate-limiting/index.js +0 -63
- package/package/rate-limiting/metrics.d.ts +0 -3
- package/package/rate-limiting/metrics.js +0 -44
- package/package/rate-limiting/request/simple-counting.d.ts +0 -20
- package/package/rate-limiting/request/simple-counting.js +0 -62
- package/package/test.d.ts +0 -1
- package/package/test.js +0 -6
- package/package/transports/batch-warming.d.ts +0 -34
- package/package/transports/batch-warming.js +0 -101
- package/package/transports/index.d.ts +0 -87
- package/package/transports/index.js +0 -87
- package/package/transports/metrics.d.ts +0 -21
- package/package/transports/metrics.js +0 -105
- package/package/transports/rest.d.ts +0 -43
- package/package/transports/rest.js +0 -129
- package/package/transports/util.d.ts +0 -8
- package/package/transports/util.js +0 -85
- package/package/transports/websocket.d.ts +0 -80
- package/package/transports/websocket.js +0 -169
- package/package/util/expiring-sorted-set.d.ts +0 -21
- package/package/util/expiring-sorted-set.js +0 -47
- package/package/util/index.d.ts +0 -11
- package/package/util/index.js +0 -35
- package/package/util/logger.d.ts +0 -42
- package/package/util/logger.js +0 -62
- package/package/util/request.d.ts +0 -55
- package/package/util/request.js +0 -2
- package/package/validation/error.d.ts +0 -50
- package/package/validation/error.js +0 -79
- package/package/validation/index.d.ts +0 -5
- package/package/validation/index.js +0 -86
- package/package/validation/input-params.d.ts +0 -15
- package/package/validation/input-params.js +0 -30
- package/package/validation/override-functions.d.ts +0 -3
- package/package/validation/preset-tokens.json +0 -23
- package/package/validation/validator.d.ts +0 -47
- package/package/validation/validator.js +0 -303
- package/rate-limiting/background/fixed-frequency.d.ts +0 -10
- package/rate-limiting/index.d.ts +0 -54
- package/rate-limiting/metrics.d.ts +0 -3
- package/rate-limiting/request/simple-counting.d.ts +0 -20
- package/test.d.ts +0 -1
- package/transports/batch-warming.d.ts +0 -35
- package/transports/index.d.ts +0 -70
- package/transports/metrics.d.ts +0 -21
- package/transports/rest.d.ts +0 -44
- package/transports/util.d.ts +0 -8
- package/transports/websocket.d.ts +0 -81
- package/util/expiring-sorted-set.d.ts +0 -21
- package/util/expiring-sorted-set.js +0 -47
- package/util/index.d.ts +0 -12
- package/util/logger.d.ts +0 -42
- package/util/request.d.ts +0 -57
- package/util/subscription-set/expiring-sorted-set.d.ts +0 -22
- package/util/subscription-set/subscription-set.d.ts +0 -18
- package/util/test-payload-loader.d.ts +0 -25
- package/validation/error.d.ts +0 -50
- package/validation/index.d.ts +0 -5
- package/validation/input-params.d.ts +0 -15
- package/validation/override-functions.d.ts +0 -3
- package/validation/validator.d.ts +0 -47
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.createMocks = void 0;
|
|
7
|
+
const nock_1 = __importDefault(require("nock"));
|
|
8
|
+
const capturedRequests_json_1 = __importDefault(require("./capturedRequests.json"));
|
|
9
|
+
// This is a generic fixture file that can be copy-pasted to generate fixtures from a
|
|
10
|
+
// capturedRequests.json file. To generate a capturedRequests.json file,
|
|
11
|
+
// set env var RECORD=true and run e2e tests.
|
|
12
|
+
const createMocks = () => {
|
|
13
|
+
const mocks = [];
|
|
14
|
+
for (const baseUrl in capturedRequests_json_1.default) {
|
|
15
|
+
const mock = (0, nock_1.default)(baseUrl, { encodedQueryParams: true });
|
|
16
|
+
for (const request of capturedRequests_json_1.default[baseUrl]) {
|
|
17
|
+
if (request.method === 'GET') {
|
|
18
|
+
mock
|
|
19
|
+
.get((uri) => {
|
|
20
|
+
const deconstructedRequestUrl = uri.split(/[?=,&]+/);
|
|
21
|
+
const deconstructedMockUrl = request.path.split(/[?=,&]+/);
|
|
22
|
+
return (deconstructedRequestUrl.sort().join(',') === deconstructedMockUrl.sort().join(','));
|
|
23
|
+
})
|
|
24
|
+
.times(100)
|
|
25
|
+
.reply(request.statusCode, () => request.response, [
|
|
26
|
+
'Content-Type',
|
|
27
|
+
'application/json',
|
|
28
|
+
'Connection',
|
|
29
|
+
'close',
|
|
30
|
+
'Vary',
|
|
31
|
+
'Accept-Encoding',
|
|
32
|
+
'Vary',
|
|
33
|
+
'Origin',
|
|
34
|
+
]);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
mocks.push(mock);
|
|
38
|
+
}
|
|
39
|
+
return mocks;
|
|
40
|
+
};
|
|
41
|
+
exports.createMocks = createMocks;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.batchEndpoint = void 0;
|
|
4
|
+
const adapter_1 = require("../../adapter");
|
|
4
5
|
const batch_warming_1 = require("../../transports/batch-warming");
|
|
5
6
|
const DEFAULT_URL = 'https://pro-api.coingecko.com/api/v3';
|
|
6
7
|
const inputParameters = {
|
|
@@ -32,7 +33,7 @@ const batchEndpointTransport = new batch_warming_1.BatchWarmingTransport({
|
|
|
32
33
|
},
|
|
33
34
|
};
|
|
34
35
|
},
|
|
35
|
-
parseResponse: (res) => {
|
|
36
|
+
parseResponse: (params, res) => {
|
|
36
37
|
const entries = [];
|
|
37
38
|
for (const [base, entry] of Object.entries(res.data)) {
|
|
38
39
|
for (const [quote, price] of Object.entries(entry)) {
|
|
@@ -45,8 +46,8 @@ const batchEndpointTransport = new batch_warming_1.BatchWarmingTransport({
|
|
|
45
46
|
return entries;
|
|
46
47
|
},
|
|
47
48
|
});
|
|
48
|
-
exports.batchEndpoint = {
|
|
49
|
+
exports.batchEndpoint = new adapter_1.AdapterEndpoint({
|
|
49
50
|
name: 'batch',
|
|
50
51
|
transport: batchEndpointTransport,
|
|
51
52
|
inputParameters,
|
|
52
|
-
};
|
|
53
|
+
});
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.adapter = void 0;
|
|
4
|
+
const adapter_1 = require("../../adapter");
|
|
4
5
|
const batch_warming_1 = require("./batch-warming");
|
|
5
6
|
const rest_1 = require("./rest");
|
|
6
|
-
exports.adapter = {
|
|
7
|
+
exports.adapter = new adapter_1.Adapter({
|
|
7
8
|
name: 'coingecko',
|
|
8
9
|
defaultEndpoint: 'batch',
|
|
9
10
|
endpoints: [rest_1.restEndpoint, batch_warming_1.batchEndpoint],
|
|
10
|
-
};
|
|
11
|
+
});
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.restEndpoint = void 0;
|
|
4
|
+
const adapter_1 = require("../../adapter");
|
|
4
5
|
const transports_1 = require("../../transports");
|
|
5
6
|
const DEFAULT_URL = 'https://api.coingecko.com/api/v3';
|
|
6
7
|
const inputParameters = {
|
|
@@ -42,9 +43,9 @@ const restEndpointTransport = new transports_1.RestTransport({
|
|
|
42
43
|
coalescing: true,
|
|
43
44
|
},
|
|
44
45
|
});
|
|
45
|
-
exports.restEndpoint = {
|
|
46
|
+
exports.restEndpoint = new adapter_1.AdapterEndpoint({
|
|
46
47
|
name: 'rest',
|
|
47
48
|
aliases: ['qwe'],
|
|
48
49
|
transport: restEndpointTransport,
|
|
49
50
|
inputParameters,
|
|
50
|
-
};
|
|
51
|
+
});
|
package/examples/ncfx/index.js
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.adapter = void 0;
|
|
4
|
+
const adapter_1 = require("../../adapter");
|
|
4
5
|
const config_1 = require("./config");
|
|
5
6
|
const websocket_1 = require("./websocket");
|
|
6
|
-
exports.adapter = {
|
|
7
|
+
exports.adapter = new adapter_1.Adapter({
|
|
7
8
|
name: 'ncfx',
|
|
8
9
|
endpoints: [websocket_1.webSocketEndpoint],
|
|
9
10
|
customSettings: config_1.customSettings,
|
|
10
|
-
};
|
|
11
|
+
});
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.webSocketEndpoint = exports.websocketTransport = exports.inputParameters = void 0;
|
|
4
|
+
const adapter_1 = require("../../adapter");
|
|
4
5
|
const websocket_1 = require("../../transports/websocket");
|
|
5
6
|
const util_1 = require("../../util");
|
|
6
7
|
exports.inputParameters = {
|
|
@@ -65,8 +66,8 @@ exports.websocketTransport = new websocket_1.WebSocketTransport({
|
|
|
65
66
|
}),
|
|
66
67
|
},
|
|
67
68
|
});
|
|
68
|
-
exports.webSocketEndpoint = {
|
|
69
|
+
exports.webSocketEndpoint = new adapter_1.AdapterEndpoint({
|
|
69
70
|
name: 'websocket',
|
|
70
71
|
transport: exports.websocketTransport,
|
|
71
72
|
inputParameters: exports.inputParameters,
|
|
72
|
-
};
|
|
73
|
+
});
|
package/index.js
CHANGED
|
@@ -6,10 +6,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
6
6
|
exports.expose = void 0;
|
|
7
7
|
const fastify_1 = __importDefault(require("fastify"));
|
|
8
8
|
const path_1 = require("path");
|
|
9
|
-
const adapter_1 = require("./adapter");
|
|
10
9
|
const background_executor_1 = require("./background-executor");
|
|
11
10
|
const cache_1 = require("./cache");
|
|
12
|
-
const config_1 = require("./config");
|
|
13
11
|
const metrics_1 = require("./metrics");
|
|
14
12
|
const transports_1 = require("./transports");
|
|
15
13
|
const util_1 = require("./util");
|
|
@@ -26,22 +24,18 @@ const VERSION = process.env['npm_package_version'];
|
|
|
26
24
|
* @returns a Promise that resolves to the http.Server listening for connections
|
|
27
25
|
*/
|
|
28
26
|
const expose = async (adapter, dependencies) => {
|
|
29
|
-
const config = (0, config_1.buildAdapterConfig)({
|
|
30
|
-
overrides: adapter.envDefaultOverrides,
|
|
31
|
-
customSettings: adapter.customSettings,
|
|
32
|
-
});
|
|
33
27
|
// Initialize adapter (create dependencies, inject them, build endpoint map, etc.)
|
|
34
|
-
|
|
28
|
+
await adapter.initialize(dependencies);
|
|
35
29
|
let server = undefined;
|
|
36
|
-
if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
|
|
37
|
-
(0, metrics_1.setupMetricsServer)(adapter.name, config);
|
|
30
|
+
if (adapter.config.METRICS_ENABLED && adapter.config.EXPERIMENTAL_METRICS_ENABLED) {
|
|
31
|
+
(0, metrics_1.setupMetricsServer)(adapter.name, adapter.config);
|
|
38
32
|
}
|
|
39
|
-
if (config.EA_MODE === 'reader' || config.EA_MODE === 'reader-writer') {
|
|
33
|
+
if (adapter.config.EA_MODE === 'reader' || adapter.config.EA_MODE === 'reader-writer') {
|
|
40
34
|
// Main REST API server to handle incoming requests
|
|
41
|
-
const app = await buildRestApi(
|
|
35
|
+
const app = await buildRestApi(adapter);
|
|
42
36
|
// Start listening for incoming requests
|
|
43
37
|
try {
|
|
44
|
-
await app.listen(config.EA_PORT, config.EA_HOST);
|
|
38
|
+
await app.listen(adapter.config.EA_PORT, adapter.config.EA_HOST);
|
|
45
39
|
}
|
|
46
40
|
catch (err) {
|
|
47
41
|
logger.fatal(`There was an error when starting the EA server: ${err}`);
|
|
@@ -53,10 +47,10 @@ const expose = async (adapter, dependencies) => {
|
|
|
53
47
|
else {
|
|
54
48
|
logger.info('REST API is disabled; this instance will not process incoming requests.');
|
|
55
49
|
}
|
|
56
|
-
if (config.EA_MODE === 'writer' || config.EA_MODE === 'reader-writer') {
|
|
50
|
+
if (adapter.config.EA_MODE === 'writer' || adapter.config.EA_MODE === 'reader-writer') {
|
|
57
51
|
// Start background loop that will take care of calling any async Transports
|
|
58
52
|
logger.info('Starting background execution loop');
|
|
59
|
-
(0, background_executor_1.callBackgroundExecutes)(
|
|
53
|
+
(0, background_executor_1.callBackgroundExecutes)(adapter, server);
|
|
60
54
|
}
|
|
61
55
|
else {
|
|
62
56
|
logger.info('Background executor is disabled; this instance will not perform async background executes.');
|
|
@@ -64,33 +58,33 @@ const expose = async (adapter, dependencies) => {
|
|
|
64
58
|
return server;
|
|
65
59
|
};
|
|
66
60
|
exports.expose = expose;
|
|
67
|
-
async function buildRestApi(
|
|
61
|
+
async function buildRestApi(adapter) {
|
|
68
62
|
const app = (0, fastify_1.default)();
|
|
69
63
|
// Add healthcheck endpoint before middlewares to bypass them
|
|
70
|
-
app.get((0, path_1.join)(config.BASE_URL, 'health'), (req, res) => {
|
|
64
|
+
app.get((0, path_1.join)(adapter.config.BASE_URL, 'health'), (req, res) => {
|
|
71
65
|
res.status(200).send({ message: 'OK', version: VERSION });
|
|
72
66
|
});
|
|
73
67
|
// Use global error handling
|
|
74
68
|
app.setErrorHandler(validation_1.errorCatchingMiddleware);
|
|
75
|
-
const transportHandler = await (0, transports_1.buildTransportHandler)(
|
|
69
|
+
const transportHandler = await (0, transports_1.buildTransportHandler)(adapter);
|
|
76
70
|
app.register(async (router) => {
|
|
77
71
|
// Set up "middlewares" (hooks in fastify)
|
|
78
|
-
router.addHook('preHandler', (0, validation_1.validatorMiddleware)(
|
|
79
|
-
router.addHook('preHandler', (0, cache_1.buildCacheMiddleware)(
|
|
80
|
-
if (config['CORRELATION_ID_ENABLED']) {
|
|
72
|
+
router.addHook('preHandler', (0, validation_1.validatorMiddleware)(adapter));
|
|
73
|
+
router.addHook('preHandler', (0, cache_1.buildCacheMiddleware)(adapter));
|
|
74
|
+
if (adapter.config['CORRELATION_ID_ENABLED']) {
|
|
81
75
|
router.addHook('onRequest', util_1.loggingContextMiddleware);
|
|
82
76
|
}
|
|
83
77
|
router.route({
|
|
84
|
-
url: config.BASE_URL,
|
|
78
|
+
url: adapter.config.BASE_URL,
|
|
85
79
|
method: 'POST',
|
|
86
80
|
handler: transportHandler,
|
|
87
81
|
});
|
|
88
|
-
if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
|
|
82
|
+
if (adapter.config.METRICS_ENABLED && adapter.config.EXPERIMENTAL_METRICS_ENABLED) {
|
|
89
83
|
router.addHook('onResponse', metrics_1.buildMetricsMiddleware);
|
|
90
84
|
}
|
|
91
85
|
});
|
|
92
86
|
// Add smoke endpoint after middleware which are needed for tests
|
|
93
|
-
buildSmokeEndpoint(app, config);
|
|
87
|
+
buildSmokeEndpoint(app, adapter.config);
|
|
94
88
|
return app;
|
|
95
89
|
}
|
|
96
90
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@chainlink/external-adapter-framework",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.16",
|
|
4
4
|
"main": "dist/index.js",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"dependencies": {
|
|
@@ -38,8 +38,8 @@
|
|
|
38
38
|
"nock": "^13.2.4",
|
|
39
39
|
"pino-pretty": "^7.6.0",
|
|
40
40
|
"prettier": "^2.6.1",
|
|
41
|
-
"
|
|
42
|
-
"
|
|
41
|
+
"ts-node-dev": "2.0.0",
|
|
42
|
+
"typedoc": "^0.22.15"
|
|
43
43
|
},
|
|
44
44
|
"prettier": {
|
|
45
45
|
"semi": false,
|
|
@@ -66,6 +66,7 @@
|
|
|
66
66
|
"workerThreads": false,
|
|
67
67
|
"environmentVariables": {
|
|
68
68
|
"METRICS_ENABLED": "false"
|
|
69
|
-
}
|
|
69
|
+
},
|
|
70
|
+
"timeout": "20s"
|
|
70
71
|
}
|
|
71
72
|
}
|
package/rate-limiting/index.js
CHANGED
|
@@ -34,11 +34,16 @@ exports.consolidateTierLimits = consolidateTierLimits;
|
|
|
34
34
|
/**
|
|
35
35
|
* Validates rate limiting tiers specified for the adapter, and returns the one to use.
|
|
36
36
|
*
|
|
37
|
+
* @param config - the adapter config containing the env vars
|
|
37
38
|
* @param tiers - the adapter config listing the different available API tiers
|
|
38
|
-
* @param selectedTier - chosen API tier from settings, if present
|
|
39
39
|
* @returns the specified API tier, or a default one if none are specified
|
|
40
40
|
*/
|
|
41
|
-
const getRateLimitingTier = (
|
|
41
|
+
const getRateLimitingTier = (config, tiers) => {
|
|
42
|
+
if (config.RATE_LIMIT_CAPACITY ||
|
|
43
|
+
config.RATE_LIMIT_CAPACITY_MINUTE ||
|
|
44
|
+
config.RATE_LIMIT_CAPACITY_SECOND) {
|
|
45
|
+
return buildRateLimitTiersFromConfig(config);
|
|
46
|
+
}
|
|
42
47
|
if (!tiers) {
|
|
43
48
|
return;
|
|
44
49
|
}
|
|
@@ -47,6 +52,7 @@ const getRateLimitingTier = (tiers, selectedTier) => {
|
|
|
47
52
|
throw new Error(`The tiers object is defined, but has no entries`);
|
|
48
53
|
}
|
|
49
54
|
// Check that the tier set in the AdapterConfig is a valid one
|
|
55
|
+
const selectedTier = config.RATE_LIMIT_API_TIER;
|
|
50
56
|
if (selectedTier && !tiers[selectedTier]) {
|
|
51
57
|
const validTiersString = Object.keys(tiers)
|
|
52
58
|
.map((t) => `"${t}"`)
|
|
@@ -61,3 +67,18 @@ const getRateLimitingTier = (tiers, selectedTier) => {
|
|
|
61
67
|
return tiers[selectedTier];
|
|
62
68
|
};
|
|
63
69
|
exports.getRateLimitingTier = getRateLimitingTier;
|
|
70
|
+
// Creates adapter rate limit tier using the configs specified in env vars
|
|
71
|
+
const buildRateLimitTiersFromConfig = (config) => {
|
|
72
|
+
const rateLimit1s = config.RATE_LIMIT_CAPACITY_SECOND;
|
|
73
|
+
let rateLimit1m;
|
|
74
|
+
if (config.RATE_LIMIT_CAPACITY_MINUTE) {
|
|
75
|
+
rateLimit1m = config.RATE_LIMIT_CAPACITY_MINUTE;
|
|
76
|
+
}
|
|
77
|
+
else if (config.RATE_LIMIT_CAPACITY) {
|
|
78
|
+
rateLimit1m = config.RATE_LIMIT_CAPACITY;
|
|
79
|
+
}
|
|
80
|
+
return {
|
|
81
|
+
rateLimit1s,
|
|
82
|
+
rateLimit1m,
|
|
83
|
+
};
|
|
84
|
+
};
|
|
@@ -81,7 +81,7 @@ class BatchWarmingTransport {
|
|
|
81
81
|
}
|
|
82
82
|
const request = this.config.prepareRequest(entries, context);
|
|
83
83
|
logger.trace('Sending request to data provider...');
|
|
84
|
-
const providerResponse = await (0, util_2.axiosRequest)(request);
|
|
84
|
+
const providerResponse = await (0, util_2.axiosRequest)(request, context.adapterConfig);
|
|
85
85
|
logger.debug(`Got response from provider, parsing (raw body: ${providerResponse.data})`);
|
|
86
86
|
const results = this.config.parseResponse(providerResponse, context);
|
|
87
87
|
const adapterResponses = (0, _1.buildCacheEntriesFromResults)(results, context);
|
package/transports/rest.js
CHANGED
|
@@ -45,10 +45,14 @@ class RestTransport {
|
|
|
45
45
|
constructor(config) {
|
|
46
46
|
this.config = config;
|
|
47
47
|
}
|
|
48
|
-
async initialize(dependencies) {
|
|
48
|
+
async initialize(dependencies, config) {
|
|
49
49
|
this.inFlightPrefix = `${IN_FLIGHT_PREFIX}-`;
|
|
50
50
|
this.cache = dependencies.cache;
|
|
51
51
|
this.rateLimiter = dependencies.requestRateLimiter;
|
|
52
|
+
// Allow enabling/disabling request coalescing through env var
|
|
53
|
+
if (config.REQUEST_COALESCING_ENABLED !== undefined) {
|
|
54
|
+
this.config.options.coalescing = config.REQUEST_COALESCING_ENABLED;
|
|
55
|
+
}
|
|
52
56
|
}
|
|
53
57
|
async hasBeenSetUp(req) {
|
|
54
58
|
if (!this.config.options.coalescing) {
|
|
@@ -95,9 +99,12 @@ class RestTransport {
|
|
|
95
99
|
msBetweenRetries: config.REST_TRANSPORT_MS_BETWEEN_RATE_LIMIT_RETRIES,
|
|
96
100
|
});
|
|
97
101
|
logger.trace('Sending request to data provider...');
|
|
98
|
-
const providerResponse = await (0, util_2.axiosRequest)(request);
|
|
102
|
+
const providerResponse = await (0, util_2.axiosRequest)(request, config);
|
|
99
103
|
logger.debug(`Got response from provider, parsing (raw body: ${providerResponse.data})`); // TODO: Sensitive data?
|
|
100
104
|
const parsedResponse = await this.config.parseResponse(req, providerResponse, config);
|
|
105
|
+
if (config.API_VERBOSE) {
|
|
106
|
+
parsedResponse.data = providerResponse.data;
|
|
107
|
+
}
|
|
101
108
|
if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
|
|
102
109
|
// TODO: Potentially create function to add all telemetry data
|
|
103
110
|
parsedResponse.maxAge = Date.now() + config.CACHE_MAX_AGE;
|
package/transports/util.js
CHANGED
|
@@ -36,10 +36,11 @@ const transportMetrics = __importStar(require("./metrics"));
|
|
|
36
36
|
* @param request - axios request config
|
|
37
37
|
* @returns axios response for the request
|
|
38
38
|
*/
|
|
39
|
-
async function axiosRequest(request) {
|
|
39
|
+
async function axiosRequest(request, config) {
|
|
40
40
|
const responseTimer = transportMetrics.dataProviderRequestDurationSeconds.startTimer();
|
|
41
41
|
let providerResponse;
|
|
42
42
|
try {
|
|
43
|
+
request.timeout = config.API_TIMEOUT;
|
|
43
44
|
providerResponse = await axios_1.default.request(request);
|
|
44
45
|
}
|
|
45
46
|
catch (e) {
|
package/transports/websocket.js
CHANGED
|
@@ -26,13 +26,11 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
26
26
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
27
|
};
|
|
28
28
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
-
exports.WebSocketTransport = exports.WebSocketClassProvider =
|
|
29
|
+
exports.WebSocketTransport = exports.WebSocketClassProvider = void 0;
|
|
30
30
|
const ws_1 = __importDefault(require("ws"));
|
|
31
31
|
const util_1 = require("../util");
|
|
32
32
|
const _1 = require("./");
|
|
33
33
|
const transportMetrics = __importStar(require("./metrics"));
|
|
34
|
-
// TODO: Config
|
|
35
|
-
exports.DEFAULT_WS_TTL = 10000;
|
|
36
34
|
const logger = (0, util_1.makeLogger)('WebSocketTransport');
|
|
37
35
|
class WebSocketClassProvider {
|
|
38
36
|
static set(ctor) {
|
|
@@ -66,9 +64,9 @@ class WebSocketTransport {
|
|
|
66
64
|
async hasBeenSetUp(req) {
|
|
67
65
|
return !!(await this.subscriptionSet.get(req.requestContext.cacheKey));
|
|
68
66
|
}
|
|
69
|
-
async setup(req) {
|
|
67
|
+
async setup(req, config) {
|
|
70
68
|
logger.debug(`Adding entry to subscription set: [${req.requestContext.cacheKey}] = ${req.requestContext.data}`);
|
|
71
|
-
await this.subscriptionSet.add(req.requestContext.cacheKey, req.requestContext.data,
|
|
69
|
+
await this.subscriptionSet.add(req.requestContext.cacheKey, req.requestContext.data, config.WS_SUBSCRIPTION_TTL);
|
|
72
70
|
}
|
|
73
71
|
// TODO: Maybe we don't do this, and leave the preparation on the adapter's side?
|
|
74
72
|
// TODO: Maybe we store adapter params pre-prepared? That would be more efficient
|
|
@@ -84,7 +82,7 @@ class WebSocketTransport {
|
|
|
84
82
|
const ctor = WebSocketClassProvider.get();
|
|
85
83
|
this.wsConnection = new ctor(this.config.url);
|
|
86
84
|
this.wsConnection.addEventListener('open', async (event) => {
|
|
87
|
-
logger.debug(`Opened websocket connection
|
|
85
|
+
logger.debug(`Opened websocket connection. (event type ${event.type})`);
|
|
88
86
|
await this.config.handlers.open(this.wsConnection, context);
|
|
89
87
|
logger.debug('Successfully executed connection opened handler');
|
|
90
88
|
// Record active ws connections by incrementing count on open
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.recordRequests = void 0;
|
|
7
|
+
const fs_1 = __importDefault(require("fs"));
|
|
8
|
+
const recordRequests = (requestConfig, providerResponse) => {
|
|
9
|
+
let recordedRequests = {};
|
|
10
|
+
try {
|
|
11
|
+
recordedRequests = JSON.parse(fs_1.default.readFileSync('capturedRequests.json').toString());
|
|
12
|
+
}
|
|
13
|
+
catch { } // eslint-disable-line no-empty
|
|
14
|
+
let baseUrl = requestConfig.baseURL;
|
|
15
|
+
baseUrl =
|
|
16
|
+
nthIndexOf(baseUrl, '/', 3) === -1 ? baseUrl : baseUrl.slice(0, nthIndexOf(baseUrl, '/', 3));
|
|
17
|
+
const recordedRequest = {
|
|
18
|
+
method: requestConfig.method,
|
|
19
|
+
path: providerResponse.request.path,
|
|
20
|
+
data: requestConfig.data,
|
|
21
|
+
response: providerResponse.data,
|
|
22
|
+
statusCode: providerResponse.status,
|
|
23
|
+
};
|
|
24
|
+
if (!recordedRequests[baseUrl]) {
|
|
25
|
+
recordedRequests[baseUrl] = [recordedRequest];
|
|
26
|
+
}
|
|
27
|
+
else if (recordedRequests[baseUrl].every((r) => {
|
|
28
|
+
return r.path !== recordedRequest.path;
|
|
29
|
+
})) {
|
|
30
|
+
recordedRequests[baseUrl].push(recordedRequest);
|
|
31
|
+
}
|
|
32
|
+
fs_1.default.writeFileSync('capturedRequests.json', JSON.stringify(recordedRequests));
|
|
33
|
+
};
|
|
34
|
+
exports.recordRequests = recordRequests;
|
|
35
|
+
const nthIndexOf = (input, match, n) => {
|
|
36
|
+
let i = -1;
|
|
37
|
+
let n_ = n;
|
|
38
|
+
while (n_-- && i++ < input.length) {
|
|
39
|
+
i = input.indexOf(match, i);
|
|
40
|
+
if (i < 0) {
|
|
41
|
+
break;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return i;
|
|
45
|
+
};
|
|
File without changes
|
package/adapter.d.ts
DELETED
|
@@ -1,107 +0,0 @@
|
|
|
1
|
-
import { Cache } from './cache';
|
|
2
|
-
import { AdapterConfig, BaseAdapterConfig, CustomSettingsType, SettingsMap } from './config';
|
|
3
|
-
import { AdapterRateLimitTier, BackgroundExecuteRateLimiter, RequestRateLimiter } from './rate-limiting';
|
|
4
|
-
import { Transport } from './transports';
|
|
5
|
-
import { SubscriptionSetFactory } from './util';
|
|
6
|
-
import { InputParameters } from './validation/input-params';
|
|
7
|
-
/**
|
|
8
|
-
* Dependencies that will be injected into the Adapter on startup
|
|
9
|
-
*/
|
|
10
|
-
export interface AdapterDependencies {
|
|
11
|
-
cache: Cache;
|
|
12
|
-
requestRateLimiter: RequestRateLimiter;
|
|
13
|
-
backgroundExecuteRateLimiter: BackgroundExecuteRateLimiter;
|
|
14
|
-
subscriptionSetFactory: SubscriptionSetFactory;
|
|
15
|
-
}
|
|
16
|
-
/**
|
|
17
|
-
* Context that will be used on background executions of a Transport.
|
|
18
|
-
* For example, the endpointName used to log statements or generate Cache keys.
|
|
19
|
-
*/
|
|
20
|
-
export interface AdapterContext<CustomSettings extends CustomSettingsType<CustomSettings> = SettingsMap> {
|
|
21
|
-
adapterEndpoint: AdapterEndpoint;
|
|
22
|
-
adapterConfig: AdapterConfig<CustomSettings>;
|
|
23
|
-
}
|
|
24
|
-
/**
|
|
25
|
-
* Structure to describe rate limits specs for the Adapter
|
|
26
|
-
*/
|
|
27
|
-
interface AdapterRateLimitingConfig {
|
|
28
|
-
/** Adapter rate limits, gotten from the specific tier requested */
|
|
29
|
-
tiers: Record<string, AdapterRateLimitTier>;
|
|
30
|
-
}
|
|
31
|
-
/**
|
|
32
|
-
* Main structure of an External Adapter
|
|
33
|
-
*/
|
|
34
|
-
export interface Adapter {
|
|
35
|
-
/** Name of the adapter */
|
|
36
|
-
name: string;
|
|
37
|
-
/** If present, the string that will be used for requests with no specified endpoint */
|
|
38
|
-
defaultEndpoint?: string;
|
|
39
|
-
/** List of [[AdapterEndpoint]]s in the adapter */
|
|
40
|
-
endpoints: AdapterEndpoint[];
|
|
41
|
-
/** Map of overrides to the default config values for an Adapter */
|
|
42
|
-
envDefaultOverrides?: Partial<BaseAdapterConfig>;
|
|
43
|
-
/** List of custom env vars for this particular adapter (e.g. RPC_URL) */
|
|
44
|
-
customSettings?: SettingsMap;
|
|
45
|
-
/** Configuration relevant to outbound (EA --\> DP) communication rate limiting */
|
|
46
|
-
rateLimiting?: AdapterRateLimitingConfig;
|
|
47
|
-
/** Overrides for converting the 'base' parameter that are hardcoded into the adapter. */
|
|
48
|
-
overrides?: Record<string, string>;
|
|
49
|
-
}
|
|
50
|
-
/**
|
|
51
|
-
* Structure to describe rate limits specs for a specific adapter endpoint
|
|
52
|
-
*/
|
|
53
|
-
export interface EndpointRateLimitingConfig {
|
|
54
|
-
/**
|
|
55
|
-
* How much of the total limit for the adapter will be assigned to this specific endpoint.
|
|
56
|
-
* Should be a non-zero positive number up to 100.
|
|
57
|
-
* Endpoints in the same adapter without a specific allocation will divide the remaining limits equally.
|
|
58
|
-
*/
|
|
59
|
-
allocationPercentage: number;
|
|
60
|
-
}
|
|
61
|
-
/**
|
|
62
|
-
* Structure to describe a specific endpoint in an [[Adapter]]
|
|
63
|
-
*/
|
|
64
|
-
export interface AdapterEndpoint {
|
|
65
|
-
/** Name that will be used to match input params to this endpoint (case insensitive) */
|
|
66
|
-
name: string;
|
|
67
|
-
/** List of alternative endpoint names that will resolve to this same transport (case insensitive) */
|
|
68
|
-
aliases?: string[];
|
|
69
|
-
/** Transport that will be used to handle data processing and communication for this endpoint */
|
|
70
|
-
transport: Transport<any, any, any>;
|
|
71
|
-
/** Specification of what the body of a request hitting this endpoint should look like (used for validation) */
|
|
72
|
-
inputParameters: InputParameters;
|
|
73
|
-
/** Specific details related to the rate limiting for this endpoint in particular */
|
|
74
|
-
rateLimiting?: EndpointRateLimitingConfig;
|
|
75
|
-
}
|
|
76
|
-
/**
|
|
77
|
-
* Structure to describe an adapter that has been initialized
|
|
78
|
-
*/
|
|
79
|
-
export interface InitializedAdapter extends Adapter {
|
|
80
|
-
/** Object containing alias translations for all endpoints */
|
|
81
|
-
endpointsMap: Record<string, AdapterEndpoint>;
|
|
82
|
-
/** Initialized dependencies that the adapter will use */
|
|
83
|
-
dependencies: AdapterDependencies;
|
|
84
|
-
/** Configuration params for various adapter properties */
|
|
85
|
-
config: AdapterConfig;
|
|
86
|
-
}
|
|
87
|
-
/**
|
|
88
|
-
* This function will process dependencies for an adapter, such as caches or rate limiters,
|
|
89
|
-
* in order to inject them into transports and other relevant places later in the lifecycle.
|
|
90
|
-
*
|
|
91
|
-
* @param config - the configuration for this adapter
|
|
92
|
-
* @param inputDependencies - a partial obj of initialized dependencies to override the created ones
|
|
93
|
-
* @param rateLimitingConfig - details from the adapter regarding rate limiting
|
|
94
|
-
* @returns a set of AdapterDependencies all initialized
|
|
95
|
-
*/
|
|
96
|
-
export declare const initializeDependencies: (adapter: Adapter, config: AdapterConfig, inputDependencies?: Partial<AdapterDependencies>) => AdapterDependencies;
|
|
97
|
-
/**
|
|
98
|
-
* Initializes all of the [[Transport]]s in the adapter, passing along any [[AdapterDependencies]] and [[AdapterConfig]].
|
|
99
|
-
* Additionally, it builds a map out of all the endpoint names and aliases (checking for duplicates).
|
|
100
|
-
*
|
|
101
|
-
* @param adapter - an instance of an Adapter
|
|
102
|
-
* @param dependencies - dependencies that the adapter will need at initialization
|
|
103
|
-
* @param config - configuration variables already processed and validated
|
|
104
|
-
* @returns - the adapter with all transports initialized and aliases map built
|
|
105
|
-
*/
|
|
106
|
-
export declare const initializeAdapter: (adapter: Adapter, config: AdapterConfig, dependencies?: Partial<AdapterDependencies>) => Promise<InitializedAdapter>;
|
|
107
|
-
export {};
|
package/background-executor.d.ts
DELETED
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
|
-
import { Server } from 'http';
|
|
3
|
-
import { InitializedAdapter } from './adapter';
|
|
4
|
-
/**
|
|
5
|
-
* Very simple background loop that will call the [[Transport.backgroundExecute]] functions in all Transports.
|
|
6
|
-
* It gets the time in ms to wait as the return value from those functions, and sleeps until next execution.
|
|
7
|
-
*
|
|
8
|
-
* @param adapter - an initialized External Adapter
|
|
9
|
-
* @param server - the http server to attach an on close listener to
|
|
10
|
-
*/
|
|
11
|
-
export declare function callBackgroundExecutes(adapter: InitializedAdapter, server?: Server): Promise<void>;
|
package/cache/factory.d.ts
DELETED