@chainlink/external-adapter-framework 0.0.15 → 0.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adapter.js +128 -0
- package/background-executor.js +45 -0
- package/cache/factory.js +58 -0
- package/cache/index.js +173 -0
- package/cache/local.js +83 -0
- package/cache/metrics.js +120 -0
- package/cache/redis.js +100 -0
- package/chainlink-external-adapter-framework-v0.0.6.tgz +0 -0
- package/config/index.js +366 -0
- package/config/provider-limits.js +74 -0
- package/examples/bank-frick/accounts.js +192 -0
- package/examples/bank-frick/config/index.js +54 -0
- package/examples/bank-frick/index.js +15 -0
- package/examples/bank-frick/util.js +39 -0
- package/examples/coingecko/batch-warming.js +53 -0
- package/examples/coingecko/index.js +11 -0
- package/examples/coingecko/rest.js +51 -0
- package/examples/coingecko/src/config/index.js +13 -0
- package/examples/coingecko/src/config/overrides.json +10826 -0
- package/examples/coingecko/src/cryptoUtils.js +41 -0
- package/examples/coingecko/src/endpoint/coins.js +33 -0
- package/examples/coingecko/src/endpoint/crypto-marketcap.js +46 -0
- package/examples/coingecko/src/endpoint/crypto-volume.js +46 -0
- package/examples/coingecko/src/endpoint/crypto.js +47 -0
- package/examples/coingecko/src/endpoint/dominance.js +26 -0
- package/examples/coingecko/src/endpoint/global-marketcap.js +26 -0
- package/examples/coingecko/src/endpoint/index.js +15 -0
- package/examples/coingecko/src/globalUtils.js +48 -0
- package/examples/coingecko/src/index.js +14 -0
- package/examples/coingecko/test/e2e/adapter.test.js +262 -0
- package/examples/coingecko/test/integration/adapter.test.js +264 -0
- package/examples/coingecko/test/integration/capturedRequests.json +1 -0
- package/examples/coingecko/test/integration/fixtures.js +41 -0
- package/examples/coingecko-old/batch-warming.js +53 -0
- package/examples/coingecko-old/index.js +11 -0
- package/examples/coingecko-old/rest.js +51 -0
- package/examples/ncfx/config/index.js +15 -0
- package/examples/ncfx/index.js +11 -0
- package/examples/ncfx/websocket.js +73 -0
- package/index.js +127 -0
- package/metrics/constants.js +25 -0
- package/metrics/index.js +122 -0
- package/metrics/util.js +9 -0
- package/package.json +5 -15
- package/rate-limiting/background/fixed-frequency.js +35 -0
- package/rate-limiting/index.js +84 -0
- package/rate-limiting/metrics.js +44 -0
- package/rate-limiting/request/simple-counting.js +62 -0
- package/test.js +6 -0
- package/transports/batch-warming.js +101 -0
- package/transports/index.js +87 -0
- package/transports/metrics.js +105 -0
- package/transports/rest.js +138 -0
- package/transports/util.js +86 -0
- package/transports/websocket.js +166 -0
- package/util/index.js +35 -0
- package/util/logger.js +62 -0
- package/util/recordRequests.js +45 -0
- package/util/request.js +2 -0
- package/util/subscription-set/expiring-sorted-set.js +47 -0
- package/util/subscription-set/subscription-set.js +19 -0
- package/util/test-payload-loader.js +83 -0
- package/validation/error.js +79 -0
- package/validation/index.js +91 -0
- package/validation/input-params.js +30 -0
- package/validation/override-functions.js +40 -0
- package/validation/overrideFunctions.js +40 -0
- package/validation/preset-tokens.json +23 -0
- package/validation/validator.js +303 -0
- package/.c8rc.json +0 -3
- package/.eslintignore +0 -10
- package/.eslintrc.js +0 -96
- package/.github/README.MD +0 -42
- package/.github/actions/setup/action.yaml +0 -13
- package/.github/workflows/label.yaml +0 -39
- package/.github/workflows/main.yaml +0 -39
- package/.github/workflows/publish.yaml +0 -17
- package/.prettierignore +0 -13
- package/.yarnrc +0 -0
- package/README.md +0 -103
- package/dist/examples/coingecko/test/e2e/adapter.test.ts.js +0 -82953
- package/dist/examples/coingecko/test/integration/adapter.test.ts.js +0 -91672
- package/dist/main.js +0 -72703
- package/docker-compose.yaml +0 -35
- package/env.sh +0 -54
- package/env2.sh +0 -55
- package/jest.config.js +0 -5
- package/publish.sh +0 -0
- package/src/adapter.ts +0 -263
- package/src/background-executor.ts +0 -52
- package/src/cache/factory.ts +0 -26
- package/src/cache/index.ts +0 -258
- package/src/cache/local.ts +0 -73
- package/src/cache/metrics.ts +0 -112
- package/src/cache/redis.ts +0 -93
- package/src/config/index.ts +0 -517
- package/src/config/provider-limits.ts +0 -127
- package/src/examples/bank-frick/README.MD +0 -10
- package/src/examples/bank-frick/accounts.ts +0 -246
- package/src/examples/bank-frick/config/index.ts +0 -53
- package/src/examples/bank-frick/index.ts +0 -13
- package/src/examples/bank-frick/types.d.ts +0 -38
- package/src/examples/bank-frick/util.ts +0 -55
- package/src/examples/coingecko/src/config/index.ts +0 -12
- package/src/examples/coingecko/src/config/overrides.json +0 -10826
- package/src/examples/coingecko/src/cryptoUtils.ts +0 -88
- package/src/examples/coingecko/src/endpoint/coins.ts +0 -54
- package/src/examples/coingecko/src/endpoint/crypto-marketcap.ts +0 -66
- package/src/examples/coingecko/src/endpoint/crypto-volume.ts +0 -66
- package/src/examples/coingecko/src/endpoint/crypto.ts +0 -63
- package/src/examples/coingecko/src/endpoint/dominance.ts +0 -40
- package/src/examples/coingecko/src/endpoint/global-marketcap.ts +0 -40
- package/src/examples/coingecko/src/endpoint/index.ts +0 -6
- package/src/examples/coingecko/src/globalUtils.ts +0 -78
- package/src/examples/coingecko/src/index.ts +0 -17
- package/src/examples/coingecko/test/e2e/adapter.test.ts +0 -278
- package/src/examples/coingecko/test/integration/__snapshots__/adapter.test.ts.snap +0 -15
- package/src/examples/coingecko/test/integration/adapter.test.ts +0 -281
- package/src/examples/coingecko/test/integration/capturedRequests.json +0 -1
- package/src/examples/coingecko/test/integration/fixtures.ts +0 -42
- package/src/examples/coingecko-old/batch-warming.ts +0 -79
- package/src/examples/coingecko-old/index.ts +0 -9
- package/src/examples/coingecko-old/rest.ts +0 -77
- package/src/examples/ncfx/config/index.ts +0 -12
- package/src/examples/ncfx/index.ts +0 -9
- package/src/examples/ncfx/websocket.ts +0 -99
- package/src/index.ts +0 -149
- package/src/metrics/constants.ts +0 -23
- package/src/metrics/index.ts +0 -115
- package/src/metrics/util.ts +0 -18
- package/src/rate-limiting/background/fixed-frequency.ts +0 -45
- package/src/rate-limiting/index.ts +0 -100
- package/src/rate-limiting/metrics.ts +0 -18
- package/src/rate-limiting/request/simple-counting.ts +0 -76
- package/src/transports/batch-warming.ts +0 -127
- package/src/transports/index.ts +0 -152
- package/src/transports/metrics.ts +0 -95
- package/src/transports/rest.ts +0 -168
- package/src/transports/util.ts +0 -63
- package/src/transports/websocket.ts +0 -245
- package/src/util/index.ts +0 -23
- package/src/util/logger.ts +0 -69
- package/src/util/recordRequests.ts +0 -47
- package/src/util/request.ts +0 -117
- package/src/util/subscription-set/expiring-sorted-set.ts +0 -54
- package/src/util/subscription-set/subscription-set.ts +0 -35
- package/src/util/test-payload-loader.ts +0 -87
- package/src/validation/error.ts +0 -116
- package/src/validation/index.ts +0 -110
- package/src/validation/input-params.ts +0 -45
- package/src/validation/override-functions.ts +0 -44
- package/src/validation/overrideFunctions.ts +0 -44
- package/src/validation/preset-tokens.json +0 -23
- package/src/validation/validator.ts +0 -384
- package/test/adapter.test.ts +0 -27
- package/test/background-executor.test.ts +0 -108
- package/test/cache/cache-key.test.ts +0 -114
- package/test/cache/helper.ts +0 -100
- package/test/cache/local.test.ts +0 -54
- package/test/cache/redis.test.ts +0 -89
- package/test/correlation.test.ts +0 -114
- package/test/index.test.ts +0 -37
- package/test/metrics/feed-id.test.ts +0 -38
- package/test/metrics/helper.ts +0 -14
- package/test/metrics/labels.test.ts +0 -36
- package/test/metrics/metrics.test.ts +0 -267
- package/test/metrics/redis-metrics.test.ts +0 -113
- package/test/metrics/warmer-metrics.test.ts +0 -193
- package/test/metrics/ws-metrics.test.ts +0 -225
- package/test/rate-limit-config.test.ts +0 -242
- package/test/smoke/smoke.test.ts +0 -166
- package/test/smoke/test-payload-fail.json +0 -3
- package/test/smoke/test-payload.js +0 -22
- package/test/smoke/test-payload.json +0 -7
- package/test/transports/batch.test.ts +0 -466
- package/test/transports/rest.test.ts +0 -242
- package/test/transports/websocket.test.ts +0 -183
- package/test/tsconfig.json +0 -5
- package/test/util.ts +0 -77
- package/test/validation.test.ts +0 -178
- package/test.sh +0 -20
- package/test2.sh +0 -2
- package/tsconfig.json +0 -28
- package/typedoc.json +0 -6
- package/webpack.config.js +0 -57
- package/yarn-error.log +0 -3778
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
26
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
|
+
};
|
|
28
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
+
exports.axiosRequest = void 0;
|
|
30
|
+
const axios_1 = __importDefault(require("axios"));
|
|
31
|
+
const error_1 = require("../validation/error");
|
|
32
|
+
const transportMetrics = __importStar(require("./metrics"));
|
|
33
|
+
/**
|
|
34
|
+
* Performs axios request along with metrics recording and error handling
|
|
35
|
+
*
|
|
36
|
+
* @param request - axios request config
|
|
37
|
+
* @returns axios response for the request
|
|
38
|
+
*/
|
|
39
|
+
async function axiosRequest(request, config) {
|
|
40
|
+
const responseTimer = transportMetrics.dataProviderRequestDurationSeconds.startTimer();
|
|
41
|
+
let providerResponse;
|
|
42
|
+
try {
|
|
43
|
+
request.timeout = config.API_TIMEOUT;
|
|
44
|
+
providerResponse = await axios_1.default.request(request);
|
|
45
|
+
}
|
|
46
|
+
catch (e) {
|
|
47
|
+
const error = e;
|
|
48
|
+
// Request error
|
|
49
|
+
let providerStatusCode;
|
|
50
|
+
let adapterError;
|
|
51
|
+
if (error.code === 'ECONNABORTED') {
|
|
52
|
+
adapterError = new error_1.AdapterTimeoutError({});
|
|
53
|
+
providerStatusCode = error?.response?.status ?? 504;
|
|
54
|
+
adapterError.name = 'Data Provider Request Timeout error';
|
|
55
|
+
}
|
|
56
|
+
else if (error?.response?.status) {
|
|
57
|
+
adapterError = new error_1.AdapterDataProviderError({});
|
|
58
|
+
providerStatusCode = error?.response?.status;
|
|
59
|
+
}
|
|
60
|
+
else {
|
|
61
|
+
adapterError = new error_1.AdapterConnectionError({});
|
|
62
|
+
providerStatusCode = 0; // 0 -> connection error
|
|
63
|
+
}
|
|
64
|
+
// Record count of failed data provider request
|
|
65
|
+
transportMetrics.dataProviderRequests
|
|
66
|
+
.labels(transportMetrics.dataProviderMetricsLabel(providerStatusCode, request.method))
|
|
67
|
+
.inc();
|
|
68
|
+
adapterError.statusCode = 200;
|
|
69
|
+
adapterError.providerStatusCode = providerStatusCode;
|
|
70
|
+
adapterError.message = error?.message;
|
|
71
|
+
adapterError.cause = error;
|
|
72
|
+
adapterError.errorResponse = error?.response?.data;
|
|
73
|
+
adapterError.url = request.url;
|
|
74
|
+
throw adapterError;
|
|
75
|
+
}
|
|
76
|
+
finally {
|
|
77
|
+
// Record time taken for data provider request for success or failure
|
|
78
|
+
responseTimer();
|
|
79
|
+
}
|
|
80
|
+
// Record count of successful data provider requests
|
|
81
|
+
transportMetrics.dataProviderRequests
|
|
82
|
+
.labels(transportMetrics.dataProviderMetricsLabel(providerResponse.status, request.method))
|
|
83
|
+
.inc();
|
|
84
|
+
return providerResponse;
|
|
85
|
+
}
|
|
86
|
+
exports.axiosRequest = axiosRequest;
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
26
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
|
+
};
|
|
28
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
+
exports.WebSocketTransport = exports.WebSocketClassProvider = void 0;
|
|
30
|
+
const ws_1 = __importDefault(require("ws"));
|
|
31
|
+
const util_1 = require("../util");
|
|
32
|
+
const _1 = require("./");
|
|
33
|
+
const transportMetrics = __importStar(require("./metrics"));
|
|
34
|
+
const logger = (0, util_1.makeLogger)('WebSocketTransport');
|
|
35
|
+
class WebSocketClassProvider {
|
|
36
|
+
static set(ctor) {
|
|
37
|
+
this.ctor = ctor;
|
|
38
|
+
}
|
|
39
|
+
static get() {
|
|
40
|
+
return this.ctor;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
exports.WebSocketClassProvider = WebSocketClassProvider;
|
|
44
|
+
WebSocketClassProvider.ctor = ws_1.default;
|
|
45
|
+
/**
|
|
46
|
+
* Transport implementation that takes incoming requests, adds them to an [[subscriptionSet]] and,
|
|
47
|
+
* through a WebSocket connection, subscribes to the relevant feeds to populate the cache.
|
|
48
|
+
*
|
|
49
|
+
* @typeParam AdapterParams - interface for the adapter request body
|
|
50
|
+
* @typeParam ProviderDataMessage - interface for a WS message containing processable data (i.e. not part of open/close/login/etc)
|
|
51
|
+
*/
|
|
52
|
+
class WebSocketTransport {
|
|
53
|
+
constructor(config) {
|
|
54
|
+
this.config = config;
|
|
55
|
+
// The double sets serve to create a simple polling mechanism instead of needing a subscription
|
|
56
|
+
// This one would not; this is always local state
|
|
57
|
+
this.localSubscriptions = [];
|
|
58
|
+
}
|
|
59
|
+
async initialize(dependencies) {
|
|
60
|
+
this.cache = dependencies.cache;
|
|
61
|
+
this.rateLimiter = dependencies.backgroundExecuteRateLimiter;
|
|
62
|
+
this.subscriptionSet = dependencies.subscriptionSetFactory.buildSet();
|
|
63
|
+
}
|
|
64
|
+
async hasBeenSetUp(req) {
|
|
65
|
+
return !!(await this.subscriptionSet.get(req.requestContext.cacheKey));
|
|
66
|
+
}
|
|
67
|
+
async setup(req, config) {
|
|
68
|
+
logger.debug(`Adding entry to subscription set: [${req.requestContext.cacheKey}] = ${req.requestContext.data}`);
|
|
69
|
+
await this.subscriptionSet.add(req.requestContext.cacheKey, req.requestContext.data, config.WS_SUBSCRIPTION_TTL);
|
|
70
|
+
}
|
|
71
|
+
// TODO: Maybe we don't do this, and leave the preparation on the adapter's side?
|
|
72
|
+
// TODO: Maybe we store adapter params pre-prepared? That would be more efficient
|
|
73
|
+
// Assuming always JSON payloads for now, makes it all cleaner
|
|
74
|
+
serializeMessage(payload) {
|
|
75
|
+
return typeof payload === 'string' ? payload : JSON.stringify(payload);
|
|
76
|
+
}
|
|
77
|
+
deserializeMessage(data) {
|
|
78
|
+
return JSON.parse(data.toString());
|
|
79
|
+
}
|
|
80
|
+
establishWsConnection(context) {
|
|
81
|
+
return new Promise((resolve) => {
|
|
82
|
+
const ctor = WebSocketClassProvider.get();
|
|
83
|
+
this.wsConnection = new ctor(this.config.url);
|
|
84
|
+
this.wsConnection.addEventListener('open', async (event) => {
|
|
85
|
+
logger.debug(`Opened websocket connection. (event type ${event.type})`);
|
|
86
|
+
await this.config.handlers.open(this.wsConnection, context);
|
|
87
|
+
logger.debug('Successfully executed connection opened handler');
|
|
88
|
+
// Record active ws connections by incrementing count on open
|
|
89
|
+
// Using URL in label since connection_key is removed from v3
|
|
90
|
+
transportMetrics.wsConnectionActive.inc();
|
|
91
|
+
resolve(true);
|
|
92
|
+
});
|
|
93
|
+
this.wsConnection.addEventListener('message', async (event) => {
|
|
94
|
+
// TODO: Assuming JSON always, maybe use BSON also?
|
|
95
|
+
const parsed = this.deserializeMessage(event.data);
|
|
96
|
+
logger.trace(`Got ws message: ${parsed}`);
|
|
97
|
+
const results = this.config.handlers.message(parsed, context);
|
|
98
|
+
if (Array.isArray(results)) {
|
|
99
|
+
const responses = (0, _1.buildCacheEntriesFromResults)(results, context);
|
|
100
|
+
logger.trace(`Writing ${responses.length} responses to cache`);
|
|
101
|
+
await this.cache.setMany(responses, context.adapterConfig.CACHE_MAX_AGE);
|
|
102
|
+
}
|
|
103
|
+
});
|
|
104
|
+
this.wsConnection.addEventListener('error', async (event) => {
|
|
105
|
+
// Record connection error count
|
|
106
|
+
transportMetrics.wsConnectionErrors
|
|
107
|
+
.labels(transportMetrics.connectionErrorLabels(event.message))
|
|
108
|
+
.inc();
|
|
109
|
+
});
|
|
110
|
+
this.wsConnection.addEventListener('close', (event) => {
|
|
111
|
+
logger.debug(`Closed websocket connection. Code: ${event.code} ; reason: ${event.reason?.toString()}`);
|
|
112
|
+
// Record active ws connections by decrementing count on close
|
|
113
|
+
// Using URL in label since connection_key is removed from v3
|
|
114
|
+
transportMetrics.wsConnectionActive.dec();
|
|
115
|
+
});
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
// Unlike cache warming, this execute will manage subscriptions
|
|
119
|
+
async backgroundExecute(context) {
|
|
120
|
+
logger.debug('Starting background execute, getting subscriptions from sorted set');
|
|
121
|
+
const desiredSubs = await this.subscriptionSet.getAll();
|
|
122
|
+
logger.debug('Generating delta (subscribes & unsubscribes)');
|
|
123
|
+
// TODO: More efficient algorithm, this is really easy to read, but high(er) time complexity
|
|
124
|
+
const subscribeParams = desiredSubs.filter((s) => !this.localSubscriptions.includes(s));
|
|
125
|
+
const subscribes = subscribeParams
|
|
126
|
+
.map(this.config.builders.subscribeMessage)
|
|
127
|
+
.map(this.serializeMessage);
|
|
128
|
+
const unsubscribeParams = this.localSubscriptions.filter((s) => !desiredSubs.includes(s));
|
|
129
|
+
const unsubscribes = unsubscribeParams
|
|
130
|
+
.map(this.config.builders.unsubscribeMessage)
|
|
131
|
+
.map(this.serializeMessage);
|
|
132
|
+
logger.debug(`${subscribes.length} new subscriptions; ${unsubscribes.length} to unsubscribe`);
|
|
133
|
+
if (subscribes.length) {
|
|
134
|
+
logger.trace(`Will subscribe to: ${subscribes}`);
|
|
135
|
+
}
|
|
136
|
+
if (unsubscribes.length) {
|
|
137
|
+
logger.trace(`Will unsubscribe to: ${unsubscribes}`);
|
|
138
|
+
}
|
|
139
|
+
// New subs && no connection -> connect -> add subs
|
|
140
|
+
// No new subs && no connection -> skip
|
|
141
|
+
// New subs && connection -> add subs
|
|
142
|
+
// No new subs && connection -> unsubs only
|
|
143
|
+
if (!subscribes.length && !this.wsConnection) {
|
|
144
|
+
logger.debug('No entries in subscription set and no established connection, skipping');
|
|
145
|
+
return this.rateLimiter.msUntilNextExecution(context.adapterEndpoint.name);
|
|
146
|
+
}
|
|
147
|
+
if (!this.wsConnection && subscribes.length) {
|
|
148
|
+
logger.debug('No established connection and new subscriptions available, connecting to WS');
|
|
149
|
+
await this.establishWsConnection(context);
|
|
150
|
+
}
|
|
151
|
+
// TODO: Close connection at some point?
|
|
152
|
+
logger.debug('Sending subs/unsubs if there are any');
|
|
153
|
+
const messages = unsubscribes.concat(subscribes);
|
|
154
|
+
for (const message of messages) {
|
|
155
|
+
logger.trace(`Sending message: ${JSON.stringify(message)}`);
|
|
156
|
+
this.wsConnection.send(message);
|
|
157
|
+
}
|
|
158
|
+
// Record WS message and subscription metrics
|
|
159
|
+
transportMetrics.recordWsMessageMetrics(context, subscribeParams, unsubscribeParams);
|
|
160
|
+
logger.debug('Setting local state to cache value');
|
|
161
|
+
this.localSubscriptions = desiredSubs;
|
|
162
|
+
logger.debug('Background execute complete');
|
|
163
|
+
return this.rateLimiter.msUntilNextExecution(context.adapterEndpoint.name);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
exports.WebSocketTransport = WebSocketTransport;
|
package/util/index.js
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
exports.isArray = exports.isObject = exports.sleep = void 0;
|
|
18
|
+
__exportStar(require("./request"), exports);
|
|
19
|
+
__exportStar(require("./logger"), exports);
|
|
20
|
+
__exportStar(require("./subscription-set/subscription-set"), exports);
|
|
21
|
+
/**
|
|
22
|
+
* Sleeps for the provided number of milliseconds
|
|
23
|
+
* @param ms - The number of milliseconds to sleep for
|
|
24
|
+
* @returns a Promise that resolves once the specified time passes
|
|
25
|
+
*/
|
|
26
|
+
const sleep = (ms) => {
|
|
27
|
+
return new Promise((resolve) => {
|
|
28
|
+
setTimeout(resolve, ms);
|
|
29
|
+
});
|
|
30
|
+
};
|
|
31
|
+
exports.sleep = sleep;
|
|
32
|
+
const isObject = (o) => o !== null && typeof o === 'object' && Array.isArray(o) === false;
|
|
33
|
+
exports.isObject = isObject;
|
|
34
|
+
const isArray = (o) => o !== null && typeof o === 'object' && Array.isArray(o);
|
|
35
|
+
exports.isArray = isArray;
|
package/util/logger.js
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.loggingContextMiddleware = exports.makeLogger = exports.asyncLocalStorage = void 0;
|
|
7
|
+
const pino_1 = __importDefault(require("pino"));
|
|
8
|
+
const config_1 = require("../config");
|
|
9
|
+
const crypto_1 = require("crypto");
|
|
10
|
+
const node_async_hooks_1 = require("node:async_hooks");
|
|
11
|
+
exports.asyncLocalStorage = new node_async_hooks_1.AsyncLocalStorage();
|
|
12
|
+
const debugTransport = {
|
|
13
|
+
target: 'pino-pretty',
|
|
14
|
+
options: {
|
|
15
|
+
levelFirst: true,
|
|
16
|
+
levelLabel: 'level',
|
|
17
|
+
ignore: 'layer,pid,hostname',
|
|
18
|
+
messageFormat: '[{correlationId}][{layer}] {msg}',
|
|
19
|
+
translateTime: 'yyyy-mm-dd HH:MM:ss.l',
|
|
20
|
+
},
|
|
21
|
+
};
|
|
22
|
+
// Base logger, shouldn't be used because we want layers to be specified
|
|
23
|
+
const baseLogger = (0, pino_1.default)({
|
|
24
|
+
level: process.env['LOG_LEVEL']?.toLowerCase() || config_1.BaseSettings.LOG_LEVEL.default,
|
|
25
|
+
mixin() {
|
|
26
|
+
if (process.env['CORRELATION_ID_ENABLED'] === 'true') {
|
|
27
|
+
const store = exports.asyncLocalStorage.getStore();
|
|
28
|
+
if (store) {
|
|
29
|
+
return store;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
return {};
|
|
33
|
+
},
|
|
34
|
+
transport: process.env['DEBUG'] === 'true' ? debugTransport : undefined,
|
|
35
|
+
});
|
|
36
|
+
/**
|
|
37
|
+
* Instead of using a global logger instance, we want to force using a child logger
|
|
38
|
+
* with a specific layer set in it, so that we can filter logs by where they're output from.
|
|
39
|
+
*
|
|
40
|
+
* Details on what each log level represents:
|
|
41
|
+
* "trace": Forensic debugging of issues on a local machine.
|
|
42
|
+
* "debug": Detailed logging level to get more context from users on their environments.
|
|
43
|
+
* "info": High-level informational messages, to describe at a glance the high level state of the system.
|
|
44
|
+
* "warn": A mild error occurred that might require non-urgent action.
|
|
45
|
+
* "error": An unexpected error occurred during the regular operation of a well-maintained EA.
|
|
46
|
+
* "fatal": The EA encountered an unrecoverable problem and had to exit.
|
|
47
|
+
*
|
|
48
|
+
* Full reference this is based on can be found at
|
|
49
|
+
* https://github.com/smartcontractkit/documentation/blob/main/docs/Node%20Operators/configuration-variables.md#log_level
|
|
50
|
+
*
|
|
51
|
+
* @param layer - the layer name to include in the logs (e.g. "SomeMiddleware", "RedisCache", etc.)
|
|
52
|
+
* @returns a layer specific logger
|
|
53
|
+
*/
|
|
54
|
+
const makeLogger = (layer) => baseLogger.child({ layer });
|
|
55
|
+
exports.makeLogger = makeLogger;
|
|
56
|
+
const loggingContextMiddleware = (req, res, done) => {
|
|
57
|
+
const correlationId = req.headers['x-correlation-id'] || (0, crypto_1.randomUUID)();
|
|
58
|
+
exports.asyncLocalStorage.run({ correlationId: correlationId }, () => {
|
|
59
|
+
done();
|
|
60
|
+
});
|
|
61
|
+
};
|
|
62
|
+
exports.loggingContextMiddleware = loggingContextMiddleware;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.recordRequests = void 0;
|
|
7
|
+
const fs_1 = __importDefault(require("fs"));
|
|
8
|
+
const recordRequests = (requestConfig, providerResponse) => {
|
|
9
|
+
let recordedRequests = {};
|
|
10
|
+
try {
|
|
11
|
+
recordedRequests = JSON.parse(fs_1.default.readFileSync('capturedRequests.json').toString());
|
|
12
|
+
}
|
|
13
|
+
catch { } // eslint-disable-line no-empty
|
|
14
|
+
let baseUrl = requestConfig.baseURL;
|
|
15
|
+
baseUrl =
|
|
16
|
+
nthIndexOf(baseUrl, '/', 3) === -1 ? baseUrl : baseUrl.slice(0, nthIndexOf(baseUrl, '/', 3));
|
|
17
|
+
const recordedRequest = {
|
|
18
|
+
method: requestConfig.method,
|
|
19
|
+
path: providerResponse.request.path,
|
|
20
|
+
data: requestConfig.data,
|
|
21
|
+
response: providerResponse.data,
|
|
22
|
+
statusCode: providerResponse.status,
|
|
23
|
+
};
|
|
24
|
+
if (!recordedRequests[baseUrl]) {
|
|
25
|
+
recordedRequests[baseUrl] = [recordedRequest];
|
|
26
|
+
}
|
|
27
|
+
else if (recordedRequests[baseUrl].every((r) => {
|
|
28
|
+
return r.path !== recordedRequest.path;
|
|
29
|
+
})) {
|
|
30
|
+
recordedRequests[baseUrl].push(recordedRequest);
|
|
31
|
+
}
|
|
32
|
+
fs_1.default.writeFileSync('capturedRequests.json', JSON.stringify(recordedRequests));
|
|
33
|
+
};
|
|
34
|
+
exports.recordRequests = recordRequests;
|
|
35
|
+
const nthIndexOf = (input, match, n) => {
|
|
36
|
+
let i = -1;
|
|
37
|
+
let n_ = n;
|
|
38
|
+
while (n_-- && i++ < input.length) {
|
|
39
|
+
i = input.indexOf(match, i);
|
|
40
|
+
if (i < 0) {
|
|
41
|
+
break;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return i;
|
|
45
|
+
};
|
package/util/request.js
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ExpiringSortedSet = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* This class implements a set of unique items, each of which has an expiration timestamp.
|
|
6
|
+
* On reads, items that have expired will be deleted from the set and not returned.
|
|
7
|
+
*
|
|
8
|
+
* @typeParam T - the type of the set entries' values
|
|
9
|
+
*/
|
|
10
|
+
class ExpiringSortedSet {
|
|
11
|
+
constructor() {
|
|
12
|
+
this.map = new Map();
|
|
13
|
+
}
|
|
14
|
+
add(key, value, ttl) {
|
|
15
|
+
this.map.set(key, {
|
|
16
|
+
value,
|
|
17
|
+
expirationTimestamp: Date.now() + ttl,
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
get(key) {
|
|
21
|
+
const entry = this.map.get(key);
|
|
22
|
+
if (!entry) {
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
else if (entry.expirationTimestamp < Date.now()) {
|
|
26
|
+
return entry.value;
|
|
27
|
+
}
|
|
28
|
+
else {
|
|
29
|
+
this.map.delete(key);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
getAll() {
|
|
33
|
+
const results = [];
|
|
34
|
+
const now = Date.now();
|
|
35
|
+
// Since we're iterating, might as well prune here
|
|
36
|
+
for (const [key, entry] of this.map.entries()) {
|
|
37
|
+
if (entry.expirationTimestamp < now) {
|
|
38
|
+
this.map.delete(key); // In theory, this shouldn't happen frequently for feeds
|
|
39
|
+
}
|
|
40
|
+
else {
|
|
41
|
+
results.push(entry.value);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return results;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
exports.ExpiringSortedSet = ExpiringSortedSet;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.SubscriptionSetFactory = void 0;
|
|
4
|
+
const expiring_sorted_set_1 = require("./expiring-sorted-set");
|
|
5
|
+
class SubscriptionSetFactory {
|
|
6
|
+
constructor(config) {
|
|
7
|
+
this.cacheType = config.CACHE_TYPE;
|
|
8
|
+
}
|
|
9
|
+
buildSet() {
|
|
10
|
+
switch (this.cacheType) {
|
|
11
|
+
case 'local':
|
|
12
|
+
return new expiring_sorted_set_1.ExpiringSortedSet();
|
|
13
|
+
case 'redis':
|
|
14
|
+
// TODO: Implement redis set
|
|
15
|
+
return new expiring_sorted_set_1.ExpiringSortedSet();
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
exports.SubscriptionSetFactory = SubscriptionSetFactory;
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.loadTestPayload = void 0;
|
|
7
|
+
const ajv_1 = __importDefault(require("ajv"));
|
|
8
|
+
const fs_1 = __importDefault(require("fs"));
|
|
9
|
+
const path_1 = __importDefault(require("path"));
|
|
10
|
+
const logger_1 = require("./logger");
|
|
11
|
+
const logger = (0, logger_1.makeLogger)('TestPayloadLoader');
|
|
12
|
+
/**
|
|
13
|
+
* Load in a JSON file containing a test payload for the current adapter,
|
|
14
|
+
* used in healthchecks to make sample requests
|
|
15
|
+
*
|
|
16
|
+
* @param fileName - name of file that contains the test payload data for the smoke endpoint
|
|
17
|
+
* @returns the parsed payload with individual requests
|
|
18
|
+
*/
|
|
19
|
+
function loadTestPayload(fileName) {
|
|
20
|
+
const ajv = new ajv_1.default();
|
|
21
|
+
const schema = {
|
|
22
|
+
type: 'object',
|
|
23
|
+
required: ['requests'],
|
|
24
|
+
properties: {
|
|
25
|
+
requests: {
|
|
26
|
+
type: 'array',
|
|
27
|
+
items: {
|
|
28
|
+
type: 'object',
|
|
29
|
+
required: [],
|
|
30
|
+
},
|
|
31
|
+
},
|
|
32
|
+
},
|
|
33
|
+
};
|
|
34
|
+
const validate = ajv.compile(schema);
|
|
35
|
+
try {
|
|
36
|
+
const payload = resolvePayload(fileName);
|
|
37
|
+
if (!validate(payload) || !payload?.requests) {
|
|
38
|
+
throw Error(JSON.stringify(validate?.errors || 'Could not validate schema for test payload'));
|
|
39
|
+
}
|
|
40
|
+
return { ...payload, isDefault: false };
|
|
41
|
+
}
|
|
42
|
+
catch (e) {
|
|
43
|
+
logger.warn(`Could not load payload: ${e.message}`);
|
|
44
|
+
logger.warn('Falling back to default empty payload');
|
|
45
|
+
return { isDefault: true };
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
exports.loadTestPayload = loadTestPayload;
|
|
49
|
+
function resolvePayload(fileName) {
|
|
50
|
+
try {
|
|
51
|
+
let payload = null;
|
|
52
|
+
// Find test payload by specified name, fallback to default names if not found
|
|
53
|
+
if (fileName && fs_1.default.existsSync(path_1.default.join(process.cwd(), fileName))) {
|
|
54
|
+
payload = require(path_1.default.join(process.cwd(), fileName));
|
|
55
|
+
}
|
|
56
|
+
else if (fileName && fs_1.default.existsSync(`./${fileName}`)) {
|
|
57
|
+
payload = require(`./${fileName}`);
|
|
58
|
+
}
|
|
59
|
+
// Search for test payload js first if no filename specified or found
|
|
60
|
+
else if (fs_1.default.existsSync(path_1.default.resolve('.', 'test-payload.js'))) {
|
|
61
|
+
payload = require(path_1.default.join(process.cwd(), 'test-payload.js'));
|
|
62
|
+
}
|
|
63
|
+
else if (fs_1.default.existsSync(`./test-payload.js`)) {
|
|
64
|
+
payload = require(`./test-payload.js`);
|
|
65
|
+
}
|
|
66
|
+
// Search for test payload json second if no filename specified or found
|
|
67
|
+
else if (fs_1.default.existsSync(path_1.default.join(process.cwd(), 'test-payload.json'))) {
|
|
68
|
+
payload = require(path_1.default.join(process.cwd(), 'test-payload.json'));
|
|
69
|
+
}
|
|
70
|
+
else if (fs_1.default.existsSync(`./test-payload.json`)) {
|
|
71
|
+
payload = require(`./test-payload.json`);
|
|
72
|
+
}
|
|
73
|
+
if (typeof payload === 'string') {
|
|
74
|
+
return JSON.parse(payload);
|
|
75
|
+
}
|
|
76
|
+
else {
|
|
77
|
+
return payload;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
catch {
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.AdapterCustomError = exports.AdapterConnectionError = exports.AdapterDataProviderError = exports.AdapterTimeoutError = exports.AdapterRateLimitError = exports.AdapterInputError = exports.AdapterError = void 0;
|
|
4
|
+
const constants_1 = require("../metrics/constants");
|
|
5
|
+
class AdapterError extends Error {
|
|
6
|
+
constructor({ jobRunID = '1', status = 'errored', statusCode = 500, name = 'AdapterError', message = 'An error occurred.', cause, url, errorResponse, feedID, providerStatusCode, metricsLabel = constants_1.HttpRequestType.ADAPTER_ERROR, }) {
|
|
7
|
+
super(message);
|
|
8
|
+
this.jobRunID = jobRunID;
|
|
9
|
+
this.status = status;
|
|
10
|
+
this.statusCode = statusCode;
|
|
11
|
+
this.name = name;
|
|
12
|
+
this.message = message;
|
|
13
|
+
this.cause = cause;
|
|
14
|
+
if (url) {
|
|
15
|
+
this.url = url;
|
|
16
|
+
}
|
|
17
|
+
if (feedID) {
|
|
18
|
+
this.feedID = feedID;
|
|
19
|
+
}
|
|
20
|
+
this.errorResponse = errorResponse;
|
|
21
|
+
this.providerStatusCode = providerStatusCode;
|
|
22
|
+
this.metricsLabel = metricsLabel;
|
|
23
|
+
}
|
|
24
|
+
toJSONResponse() {
|
|
25
|
+
const showDebugInfo = process.env['NODE_ENV'] === 'development' || process.env['DEBUG'] === 'true';
|
|
26
|
+
const errorBasic = {
|
|
27
|
+
name: this.name,
|
|
28
|
+
message: this.message,
|
|
29
|
+
url: this.url,
|
|
30
|
+
errorResponse: this.errorResponse,
|
|
31
|
+
feedID: this.feedID,
|
|
32
|
+
};
|
|
33
|
+
const errorFull = { ...errorBasic, stack: this.stack, cause: this.cause };
|
|
34
|
+
return {
|
|
35
|
+
jobRunID: this.jobRunID,
|
|
36
|
+
status: this.status,
|
|
37
|
+
statusCode: this.statusCode,
|
|
38
|
+
providerStatusCode: this.providerStatusCode,
|
|
39
|
+
error: showDebugInfo ? errorFull : errorBasic,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
exports.AdapterError = AdapterError;
|
|
44
|
+
class AdapterInputError extends AdapterError {
|
|
45
|
+
constructor(input) {
|
|
46
|
+
super({ ...input, metricsLabel: constants_1.HttpRequestType.INPUT_ERROR });
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
exports.AdapterInputError = AdapterInputError;
|
|
50
|
+
class AdapterRateLimitError extends AdapterError {
|
|
51
|
+
constructor(input) {
|
|
52
|
+
super({ ...input, metricsLabel: constants_1.HttpRequestType.RATE_LIMIT_ERROR });
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
exports.AdapterRateLimitError = AdapterRateLimitError;
|
|
56
|
+
class AdapterTimeoutError extends AdapterError {
|
|
57
|
+
constructor(input) {
|
|
58
|
+
super({ ...input, metricsLabel: constants_1.HttpRequestType.TIMEOUT_ERROR });
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
exports.AdapterTimeoutError = AdapterTimeoutError;
|
|
62
|
+
class AdapterDataProviderError extends AdapterError {
|
|
63
|
+
constructor(input) {
|
|
64
|
+
super({ ...input, metricsLabel: constants_1.HttpRequestType.DP_ERROR });
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
exports.AdapterDataProviderError = AdapterDataProviderError;
|
|
68
|
+
class AdapterConnectionError extends AdapterError {
|
|
69
|
+
constructor(input) {
|
|
70
|
+
super({ ...input, metricsLabel: constants_1.HttpRequestType.CONNECTION_ERROR });
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
exports.AdapterConnectionError = AdapterConnectionError;
|
|
74
|
+
class AdapterCustomError extends AdapterError {
|
|
75
|
+
constructor(input) {
|
|
76
|
+
super({ ...input, metricsLabel: constants_1.HttpRequestType.CUSTOM_ERROR });
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
exports.AdapterCustomError = AdapterCustomError;
|