@chainlink/external-adapter-framework 0.0.7 → 0.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adapter.d.ts +88 -0
- package/{dist/src/adapter.js → adapter.js} +0 -0
- package/background-executor.d.ts +11 -0
- package/{dist/background-executor.js → background-executor.js} +0 -0
- package/cache/factory.d.ts +6 -0
- package/{dist/cache → cache}/factory.js +0 -0
- package/cache/index.d.ts +90 -0
- package/{dist/src/cache → cache}/index.js +6 -2
- package/cache/local.d.ts +23 -0
- package/{dist/cache → cache}/local.js +1 -1
- package/cache/metrics.d.ts +27 -0
- package/{dist/cache → cache}/metrics.js +7 -1
- package/cache/redis.d.ts +16 -0
- package/{dist/cache → cache}/redis.js +2 -2
- package/config/index.d.ts +195 -0
- package/{dist/src/config → config}/index.js +5 -6
- package/config/provider-limits.d.ts +31 -0
- package/{dist/src/config → config}/provider-limits.js +1 -1
- package/examples/coingecko/batch-warming.d.ts +2 -0
- package/{dist/examples → examples}/coingecko/batch-warming.js +0 -0
- package/examples/coingecko/index.d.ts +2 -0
- package/{dist/examples → examples}/coingecko/index.js +0 -0
- package/examples/coingecko/rest.d.ts +2 -0
- package/{dist/examples → examples}/coingecko/rest.js +0 -0
- package/examples/ncfx/config/index.d.ts +12 -0
- package/{dist/examples → examples}/ncfx/config/index.js +0 -0
- package/examples/ncfx/index.d.ts +2 -0
- package/{dist/examples → examples}/ncfx/index.js +0 -0
- package/examples/ncfx/websocket.d.ts +36 -0
- package/{dist/examples → examples}/ncfx/websocket.js +0 -0
- package/index.d.ts +12 -0
- package/{dist/index.js → index.js} +3 -0
- package/metrics/constants.d.ts +16 -0
- package/{dist/src/metrics → metrics}/constants.js +1 -1
- package/metrics/index.d.ts +15 -0
- package/{dist/metrics → metrics}/index.js +48 -1
- package/metrics/util.d.ts +3 -0
- package/{dist/metrics → metrics}/util.js +0 -0
- package/package.json +2 -6
- package/rate-limiting/background/fixed-frequency.d.ts +10 -0
- package/{dist/src/rate-limiting → rate-limiting}/background/fixed-frequency.js +0 -0
- package/rate-limiting/index.d.ts +54 -0
- package/{dist/src/rate-limiting → rate-limiting}/index.js +0 -0
- package/rate-limiting/metrics.d.ts +3 -0
- package/{dist/src/rate-limiting → rate-limiting}/metrics.js +14 -2
- package/rate-limiting/request/simple-counting.d.ts +20 -0
- package/{dist/src/rate-limiting → rate-limiting}/request/simple-counting.js +0 -0
- package/test.d.ts +1 -0
- package/{dist/test.js → test.js} +0 -0
- package/transports/batch-warming.d.ts +34 -0
- package/{dist/src/transports → transports}/batch-warming.js +47 -1
- package/transports/index.d.ts +87 -0
- package/{dist/src/transports → transports}/index.js +5 -3
- package/transports/metrics.d.ts +21 -0
- package/{dist/src/transports → transports}/metrics.js +28 -42
- package/transports/rest.d.ts +43 -0
- package/{dist/src/transports → transports}/rest.js +38 -2
- package/transports/util.d.ts +8 -0
- package/{dist/src/transports → transports}/util.js +16 -16
- package/transports/websocket.d.ts +80 -0
- package/{dist/src/transports → transports}/websocket.js +8 -14
- package/util/expiring-sorted-set.d.ts +21 -0
- package/{dist/src/util → util}/expiring-sorted-set.js +0 -0
- package/util/index.d.ts +11 -0
- package/{dist/src/util → util}/index.js +0 -0
- package/util/logger.d.ts +42 -0
- package/{dist/src/util → util}/logger.js +0 -0
- package/util/request.d.ts +55 -0
- package/{dist/src/util → util}/request.js +0 -0
- package/validation/error.d.ts +50 -0
- package/validation/error.js +79 -0
- package/validation/index.d.ts +5 -0
- package/{dist/src/validation → validation}/index.js +8 -6
- package/validation/input-params.d.ts +15 -0
- package/{dist/src/validation → validation}/input-params.js +0 -0
- package/validation/override-functions.d.ts +3 -0
- package/{dist/src/validation → validation}/override-functions.js +0 -0
- package/{dist/src/validation → validation}/preset-tokens.json +0 -0
- package/validation/validator.d.ts +47 -0
- package/{dist/src/validation → validation}/validator.js +0 -0
- package/README.md +0 -103
- package/dist/adapter.js +0 -60
- package/dist/cache/index.js +0 -163
- package/dist/config/index.js +0 -364
- package/dist/config/provider-limits.js +0 -75
- package/dist/metrics/constants.js +0 -25
- package/dist/rate-limiting/factory.js +0 -33
- package/dist/rate-limiting/index.js +0 -36
- package/dist/rate-limiting/metrics.js +0 -32
- package/dist/rate-limiting/nop-limiter.js +0 -15
- package/dist/rate-limiting/simple-counting.js +0 -61
- package/dist/src/background-executor.js +0 -45
- package/dist/src/cache/factory.js +0 -57
- package/dist/src/cache/local.js +0 -83
- package/dist/src/cache/metrics.js +0 -114
- package/dist/src/cache/redis.js +0 -100
- package/dist/src/examples/bank-frick/accounts.js +0 -191
- package/dist/src/examples/bank-frick/config/index.js +0 -45
- package/dist/src/examples/bank-frick/index.js +0 -14
- package/dist/src/examples/bank-frick/util.js +0 -39
- package/dist/src/examples/coingecko/batch-warming.js +0 -52
- package/dist/src/examples/coingecko/index.js +0 -10
- package/dist/src/examples/coingecko/rest.js +0 -50
- package/dist/src/examples/ncfx/config/index.js +0 -15
- package/dist/src/examples/ncfx/index.js +0 -10
- package/dist/src/examples/ncfx/websocket.js +0 -72
- package/dist/src/index.js +0 -89
- package/dist/src/metrics/index.js +0 -76
- package/dist/src/metrics/util.js +0 -9
- package/dist/src/test.js +0 -6
- package/dist/src/validation/error.js +0 -41
- package/dist/transports/batch-warming.js +0 -57
- package/dist/transports/index.js +0 -76
- package/dist/transports/metrics.js +0 -133
- package/dist/transports/rest.js +0 -91
- package/dist/transports/util.js +0 -85
- package/dist/transports/websocket.js +0 -171
- package/dist/util/expiring-sorted-set.js +0 -47
- package/dist/util/index.js +0 -35
- package/dist/util/logger.js +0 -62
- package/dist/util/request.js +0 -2
- package/dist/validation/error.js +0 -41
- package/dist/validation/index.js +0 -82
- package/dist/validation/input-params.js +0 -30
- package/dist/validation/overrideFunctions.js +0 -42
- package/dist/validation/presetTokens.json +0 -23
- package/dist/validation/validator.js +0 -303
|
@@ -23,8 +23,9 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
|
23
23
|
return result;
|
|
24
24
|
};
|
|
25
25
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.
|
|
26
|
+
exports.wsMessageTotal = exports.wsSubscriptionTotal = exports.wsSubscriptionActive = exports.wsConnectionErrors = exports.wsConnectionActive = exports.recordWsMessageMetrics = exports.messageSubsLabels = exports.connectionErrorLabels = exports.dataProviderRequestDurationSeconds = exports.dataProviderRequests = exports.dataProviderMetricsLabel = void 0;
|
|
27
27
|
const client = __importStar(require("prom-client"));
|
|
28
|
+
const cache_1 = require("../cache");
|
|
28
29
|
const constants_1 = require("../metrics/constants");
|
|
29
30
|
// Data Provider Metrics
|
|
30
31
|
const dataProviderMetricsLabel = (providerStatusCode, method = 'get') => ({
|
|
@@ -43,38 +44,40 @@ exports.dataProviderRequestDurationSeconds = new client.Histogram({
|
|
|
43
44
|
buckets: constants_1.requestDurationBuckets,
|
|
44
45
|
});
|
|
45
46
|
// Websocket Metrics
|
|
46
|
-
const connectionLabels = (key) => ({
|
|
47
|
-
// Key,
|
|
48
|
-
});
|
|
49
|
-
exports.connectionLabels = connectionLabels;
|
|
50
47
|
const connectionErrorLabels = (message) => ({
|
|
51
48
|
// Key,
|
|
52
49
|
message,
|
|
53
50
|
});
|
|
54
51
|
exports.connectionErrorLabels = connectionErrorLabels;
|
|
55
|
-
const
|
|
56
|
-
// Connection_key: string,
|
|
57
|
-
feed_id, cache_key) => ({
|
|
58
|
-
// Connection_key,
|
|
59
|
-
feed_id,
|
|
60
|
-
subscription_key: cache_key,
|
|
61
|
-
});
|
|
62
|
-
exports.subscriptionLabels = subscriptionLabels;
|
|
63
|
-
const subscriptionErrorLabels = (
|
|
64
|
-
// Connection_key: string,
|
|
65
|
-
input, message, type) => ({
|
|
66
|
-
// Connection_key,
|
|
67
|
-
feed_id: input ? input.metricsMeta?.feedId : 'N/A',
|
|
68
|
-
message,
|
|
69
|
-
subscription_key: input.cacheKey ? input.cacheKey : 'N/A',
|
|
70
|
-
type,
|
|
71
|
-
});
|
|
72
|
-
exports.subscriptionErrorLabels = subscriptionErrorLabels;
|
|
73
|
-
const messageLabels = (feed_id, cache_key) => ({
|
|
52
|
+
const messageSubsLabels = (feed_id, cache_key) => ({
|
|
74
53
|
feed_id,
|
|
75
54
|
subscription_key: cache_key,
|
|
76
55
|
});
|
|
77
|
-
exports.
|
|
56
|
+
exports.messageSubsLabels = messageSubsLabels;
|
|
57
|
+
// Record WS message and subscription metrics
|
|
58
|
+
// Recalculate cacheKey and feedId for metrics
|
|
59
|
+
// since avoiding storing extra info in expiring sorted set
|
|
60
|
+
const recordWsMessageMetrics = (context, subscribes, unsubscrices) => {
|
|
61
|
+
subscribes.forEach((param) => {
|
|
62
|
+
const feedId = (0, cache_1.calculateFeedId)(context.adapterEndpoint, param);
|
|
63
|
+
const cacheKey = (0, cache_1.calculateCacheKey)(context, param);
|
|
64
|
+
// Record total number of ws messages sent
|
|
65
|
+
exports.wsMessageTotal.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).inc();
|
|
66
|
+
// Record total number of subscriptions made
|
|
67
|
+
exports.wsSubscriptionTotal.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).inc();
|
|
68
|
+
// Record number of active ws subscriptions
|
|
69
|
+
exports.wsSubscriptionActive.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).inc();
|
|
70
|
+
});
|
|
71
|
+
unsubscrices.forEach((param) => {
|
|
72
|
+
const feedId = (0, cache_1.calculateFeedId)(context.adapterEndpoint, param);
|
|
73
|
+
const cacheKey = (0, cache_1.calculateCacheKey)(context, param);
|
|
74
|
+
// Record total number of ws messages sent
|
|
75
|
+
exports.wsMessageTotal.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).inc();
|
|
76
|
+
// Record number of active ws subscriptions
|
|
77
|
+
exports.wsSubscriptionActive.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).dec();
|
|
78
|
+
});
|
|
79
|
+
};
|
|
80
|
+
exports.recordWsMessageMetrics = recordWsMessageMetrics;
|
|
78
81
|
exports.wsConnectionActive = new client.Gauge({
|
|
79
82
|
name: 'ws_connection_active',
|
|
80
83
|
help: 'The number of active connections',
|
|
@@ -85,12 +88,6 @@ exports.wsConnectionErrors = new client.Counter({
|
|
|
85
88
|
help: 'The number of connection errors',
|
|
86
89
|
labelNames: ['url', 'message'],
|
|
87
90
|
});
|
|
88
|
-
// Doesn't seem to be any retry connection functionality yet
|
|
89
|
-
exports.wsConnectionRetries = new client.Counter({
|
|
90
|
-
name: 'ws_connection_retries',
|
|
91
|
-
help: 'The number of connection retries',
|
|
92
|
-
labelNames: ['url'],
|
|
93
|
-
});
|
|
94
91
|
exports.wsSubscriptionActive = new client.Gauge({
|
|
95
92
|
name: 'ws_subscription_active',
|
|
96
93
|
help: 'The number of currently active subscriptions',
|
|
@@ -101,19 +98,8 @@ exports.wsSubscriptionTotal = new client.Counter({
|
|
|
101
98
|
help: 'The number of subscriptions opened in total',
|
|
102
99
|
labelNames: ['connection_url', 'feed_id', 'subscription_key'],
|
|
103
100
|
});
|
|
104
|
-
exports.wsSubscriptionErrors = new client.Counter({
|
|
105
|
-
name: 'ws_subscription_errors',
|
|
106
|
-
help: 'The number of subscriptions errors',
|
|
107
|
-
labelNames: ['connection_url', 'feed_id', 'subscription_key', 'message', 'type'],
|
|
108
|
-
});
|
|
109
101
|
exports.wsMessageTotal = new client.Counter({
|
|
110
102
|
name: 'ws_message_total',
|
|
111
103
|
help: 'The number of messages received in total',
|
|
112
104
|
labelNames: ['feed_id', 'subscription_key'],
|
|
113
105
|
});
|
|
114
|
-
// Cache Warmer Metrics
|
|
115
|
-
exports.cacheWarmerCount = new client.Gauge({
|
|
116
|
-
name: 'cache_warmer_get_count',
|
|
117
|
-
help: 'The number of cache warmers running',
|
|
118
|
-
labelNames: ['isBatched'],
|
|
119
|
-
});
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { AdapterRequest, AdapterResponse } from '../util/request';
|
|
2
|
+
import { AdapterDependencies, Transport } from './';
|
|
3
|
+
import { Cache } from '../cache';
|
|
4
|
+
import { AxiosRequestConfig, AxiosResponse } from 'axios';
|
|
5
|
+
import { AdapterConfig, SettingsMap } from '../config';
|
|
6
|
+
import { RequestRateLimiter } from '../rate-limiting';
|
|
7
|
+
/**
|
|
8
|
+
* Transport implementation that takes incoming requests, transforms them into a DataProvider request,
|
|
9
|
+
* and executes that request returning the response immediately from the `setup` function.
|
|
10
|
+
* Optionally, setting the `coalescing` option to `true` will make it so once a request is in flight,
|
|
11
|
+
* new adapter requests for the same feed will not trigger a new one, but return an empty promise from
|
|
12
|
+
* the setup instead so the normal cache polling mechanism is used.
|
|
13
|
+
*
|
|
14
|
+
* @typeParam AdapterParams - interface for the adapter request body
|
|
15
|
+
* @typeParam ProviderRequestBody - interface for the body of the request to the Data Provider
|
|
16
|
+
* @typeParam ProviderResponseBody - interface for the body of the Data Provider's response
|
|
17
|
+
*/
|
|
18
|
+
export declare class RestTransport<AdapterParams, ProviderRequestBody, ProviderResponseBody, CustomSettings extends SettingsMap> implements Transport<AdapterParams, ProviderResponseBody, CustomSettings> {
|
|
19
|
+
protected config: {
|
|
20
|
+
prepareRequest: (req: AdapterRequest<AdapterParams>, config: AdapterConfig<CustomSettings>) => AxiosRequestConfig<ProviderRequestBody> | Promise<AxiosRequestConfig<ProviderRequestBody>>;
|
|
21
|
+
parseResponse: (req: AdapterRequest<AdapterParams>, res: AxiosResponse<ProviderResponseBody>, config: AdapterConfig<CustomSettings>) => AdapterResponse<ProviderResponseBody> | Promise<AdapterResponse<ProviderResponseBody>>;
|
|
22
|
+
options: {
|
|
23
|
+
coalescing: boolean;
|
|
24
|
+
};
|
|
25
|
+
};
|
|
26
|
+
inFlightPrefix: string;
|
|
27
|
+
cache: Cache;
|
|
28
|
+
rateLimiter: RequestRateLimiter;
|
|
29
|
+
constructor(config: {
|
|
30
|
+
prepareRequest: (req: AdapterRequest<AdapterParams>, config: AdapterConfig<CustomSettings>) => AxiosRequestConfig<ProviderRequestBody> | Promise<AxiosRequestConfig<ProviderRequestBody>>;
|
|
31
|
+
parseResponse: (req: AdapterRequest<AdapterParams>, res: AxiosResponse<ProviderResponseBody>, config: AdapterConfig<CustomSettings>) => AdapterResponse<ProviderResponseBody> | Promise<AdapterResponse<ProviderResponseBody>>;
|
|
32
|
+
options: {
|
|
33
|
+
coalescing: boolean;
|
|
34
|
+
};
|
|
35
|
+
});
|
|
36
|
+
initialize(dependencies: AdapterDependencies): Promise<void>;
|
|
37
|
+
hasBeenSetUp(req: AdapterRequest<AdapterParams>): Promise<boolean>;
|
|
38
|
+
protected waitUntilUnderRateLimit(options: {
|
|
39
|
+
maxRetries: number;
|
|
40
|
+
msBetweenRetries: number;
|
|
41
|
+
}, retry?: number): Promise<void>;
|
|
42
|
+
setup(req: AdapterRequest<AdapterParams>, config: AdapterConfig<CustomSettings>): Promise<AdapterResponse<ProviderResponseBody> | undefined>;
|
|
43
|
+
}
|
|
@@ -1,9 +1,33 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
2
25
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
26
|
exports.RestTransport = void 0;
|
|
4
27
|
const util_1 = require("../util");
|
|
5
28
|
const error_1 = require("../validation/error");
|
|
6
29
|
const util_2 = require("./util");
|
|
30
|
+
const rateLimitMetrics = __importStar(require("../rate-limiting/metrics"));
|
|
7
31
|
const IN_FLIGHT_PREFIX = 'InFlight';
|
|
8
32
|
const logger = (0, util_1.makeLogger)('RestTransport');
|
|
9
33
|
/**
|
|
@@ -77,11 +101,23 @@ class RestTransport {
|
|
|
77
101
|
if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
|
|
78
102
|
// TODO: Potentially create function to add all telemetry data
|
|
79
103
|
parsedResponse.maxAge = Date.now() + config.CACHE_MAX_AGE;
|
|
80
|
-
parsedResponse.
|
|
104
|
+
parsedResponse.meta = { metrics: { feedId: req.requestContext.meta?.metrics?.feedId || 'N/A' } };
|
|
81
105
|
}
|
|
82
106
|
logger.debug('Setting provider response in cache');
|
|
83
107
|
await this.cache.set(req.requestContext.cacheKey, parsedResponse, config.CACHE_MAX_AGE);
|
|
84
|
-
//
|
|
108
|
+
// Record cost of data provider call
|
|
109
|
+
const cost = rateLimitMetrics.retrieveCost(providerResponse.data);
|
|
110
|
+
rateLimitMetrics.rateLimitCreditsSpentTotal
|
|
111
|
+
.labels({
|
|
112
|
+
feed_id: req.requestContext.meta?.metrics?.feedId || 'N/A',
|
|
113
|
+
participant_id: req.requestContext.cacheKey,
|
|
114
|
+
})
|
|
115
|
+
.inc(cost);
|
|
116
|
+
// Update cacheHit flag in request meta for metrics use
|
|
117
|
+
req.requestContext.meta = {
|
|
118
|
+
...req.requestContext.meta,
|
|
119
|
+
metrics: { ...req.requestContext.meta?.metrics, cacheHit: false },
|
|
120
|
+
};
|
|
85
121
|
// TODO: move this to a try/catch/finally
|
|
86
122
|
if (this.config.options.coalescing) {
|
|
87
123
|
logger.debug('Set provider response in cache, removing in flight from cache');
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { AxiosRequestConfig, AxiosResponse } from 'axios';
|
|
2
|
+
/**
|
|
3
|
+
* Performs axios request along with metrics recording and error handling
|
|
4
|
+
*
|
|
5
|
+
* @param request - axios request config
|
|
6
|
+
* @returns axios response for the request
|
|
7
|
+
*/
|
|
8
|
+
export declare function axiosRequest<ProviderRequestBody, ProviderResponseBody>(request: AxiosRequestConfig<ProviderRequestBody>): Promise<AxiosResponse<ProviderResponseBody>>;
|
|
@@ -46,31 +46,31 @@ async function axiosRequest(request) {
|
|
|
46
46
|
const error = e;
|
|
47
47
|
// Request error
|
|
48
48
|
let providerStatusCode;
|
|
49
|
-
let
|
|
50
|
-
let statusCode;
|
|
49
|
+
let adapterError;
|
|
51
50
|
if (error.code === 'ECONNABORTED') {
|
|
51
|
+
adapterError = new error_1.AdapterTimeoutError({});
|
|
52
52
|
providerStatusCode = error?.response?.status ?? 504;
|
|
53
|
-
name = 'Data Provider Request Timeout error';
|
|
54
|
-
|
|
53
|
+
adapterError.name = 'Data Provider Request Timeout error';
|
|
54
|
+
}
|
|
55
|
+
else if (error?.response?.status) {
|
|
56
|
+
adapterError = new error_1.AdapterDataProviderError({});
|
|
57
|
+
providerStatusCode = error?.response?.status;
|
|
55
58
|
}
|
|
56
59
|
else {
|
|
57
|
-
|
|
58
|
-
|
|
60
|
+
adapterError = new error_1.AdapterConnectionError({});
|
|
61
|
+
providerStatusCode = 0; // 0 -> connection error
|
|
59
62
|
}
|
|
60
63
|
// Record count of failed data provider request
|
|
61
64
|
transportMetrics.dataProviderRequests
|
|
62
65
|
.labels(transportMetrics.dataProviderMetricsLabel(providerStatusCode, request.method))
|
|
63
66
|
.inc();
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
errorResponse: error?.response?.data,
|
|
72
|
-
url: request.url,
|
|
73
|
-
});
|
|
67
|
+
adapterError.statusCode = 200;
|
|
68
|
+
adapterError.providerStatusCode = providerStatusCode;
|
|
69
|
+
adapterError.message = error?.message;
|
|
70
|
+
adapterError.cause = error;
|
|
71
|
+
adapterError.errorResponse = error?.response?.data;
|
|
72
|
+
adapterError.url = request.url;
|
|
73
|
+
throw adapterError;
|
|
74
74
|
}
|
|
75
75
|
finally {
|
|
76
76
|
// Record time taken for data provider request for success or failure
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import WebSocket from 'ws';
|
|
2
|
+
import { Cache } from '../cache';
|
|
3
|
+
import { SettingsMap } from '../config';
|
|
4
|
+
import { BackgroundExecuteRateLimiter } from '../rate-limiting';
|
|
5
|
+
import { ExpiringSortedSet } from '../util';
|
|
6
|
+
import { AdapterRequest, ProviderResult } from '../util/request';
|
|
7
|
+
import { AdapterContext, AdapterDependencies, Transport } from './';
|
|
8
|
+
export declare const DEFAULT_WS_TTL = 10000;
|
|
9
|
+
declare type WebSocketClass = new (url: string, protocols?: string | string[] | undefined) => WebSocket;
|
|
10
|
+
export declare class WebSocketClassProvider {
|
|
11
|
+
static ctor: WebSocketClass;
|
|
12
|
+
static set(ctor: WebSocketClass): void;
|
|
13
|
+
static get(): WebSocketClass;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Config object that is provided to the WebSocketTransport constructor.
|
|
17
|
+
*/
|
|
18
|
+
export interface WebSocketTransportConfig<AdapterParams, ProviderDataMessage, CustomSettings extends SettingsMap> {
|
|
19
|
+
/** Endpoint to which to open the WS connection*/
|
|
20
|
+
url: string;
|
|
21
|
+
/** Map of handlers for different WS lifecycle events */
|
|
22
|
+
handlers: {
|
|
23
|
+
/**
|
|
24
|
+
* Handles when the WS is successfully opened
|
|
25
|
+
*
|
|
26
|
+
* @param wsConnection - the WebSocket with an established connection
|
|
27
|
+
* @returns an empty Promise, or void
|
|
28
|
+
*/
|
|
29
|
+
open: (wsConnection: WebSocket, context: AdapterContext<CustomSettings>) => Promise<void> | void;
|
|
30
|
+
/**
|
|
31
|
+
* Handles when the WS receives a message
|
|
32
|
+
*
|
|
33
|
+
* @param message - the message received by the WS
|
|
34
|
+
* @param context - the background context for the Adapter
|
|
35
|
+
* @returns a list of cache entries of adapter responses to set in the cache
|
|
36
|
+
*/
|
|
37
|
+
message: (message: ProviderDataMessage, context: AdapterContext<CustomSettings>) => ProviderResult<AdapterParams>[];
|
|
38
|
+
};
|
|
39
|
+
/** Map of "builders", functions that will be used to prepare specific WS messages */
|
|
40
|
+
builders: {
|
|
41
|
+
/**
|
|
42
|
+
* Builds a WS message that will be sent to subscribe to a specific feed
|
|
43
|
+
*
|
|
44
|
+
* @param params - the body of the adapter request
|
|
45
|
+
* @returns the WS message (can be any type as long as the [[WebSocket]] doesn't complain)
|
|
46
|
+
*/
|
|
47
|
+
subscribeMessage: (params: AdapterParams) => unknown;
|
|
48
|
+
/**
|
|
49
|
+
* Builds a WS message that will be sent to unsubscribe to a specific feed
|
|
50
|
+
*
|
|
51
|
+
* @param params - the body of the adapter request
|
|
52
|
+
* @returns the WS message (can be any type as long as the [[WebSocket]] doesn't complain)
|
|
53
|
+
*/
|
|
54
|
+
unsubscribeMessage: (params: AdapterParams) => unknown;
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Transport implementation that takes incoming requests, adds them to an [[ExpiringSortedSet]] and,
|
|
59
|
+
* through a WebSocket connection, subscribes to the relevant feeds to populate the cache.
|
|
60
|
+
*
|
|
61
|
+
* @typeParam AdapterParams - interface for the adapter request body
|
|
62
|
+
* @typeParam ProviderDataMessage - interface for a WS message containing processable data (i.e. not part of open/close/login/etc)
|
|
63
|
+
*/
|
|
64
|
+
export declare class WebSocketTransport<AdapterParams, ProviderDataMessage, CustomSettings extends SettingsMap> implements Transport<AdapterParams, null, CustomSettings> {
|
|
65
|
+
private config;
|
|
66
|
+
cache: Cache;
|
|
67
|
+
rateLimiter: BackgroundExecuteRateLimiter;
|
|
68
|
+
expiringSortedSet: ExpiringSortedSet<AdapterParams>;
|
|
69
|
+
localSubscriptions: AdapterParams[];
|
|
70
|
+
wsConnection: WebSocket;
|
|
71
|
+
constructor(config: WebSocketTransportConfig<AdapterParams, ProviderDataMessage, CustomSettings>);
|
|
72
|
+
initialize(dependencies: AdapterDependencies): Promise<void>;
|
|
73
|
+
hasBeenSetUp(req: AdapterRequest<AdapterParams>): Promise<boolean>;
|
|
74
|
+
setup(req: AdapterRequest<AdapterParams>): Promise<void>;
|
|
75
|
+
serializeMessage(payload: unknown): string;
|
|
76
|
+
deserializeMessage(data: WebSocket.Data): ProviderDataMessage;
|
|
77
|
+
establishWsConnection(context: AdapterContext<CustomSettings>): Promise<unknown>;
|
|
78
|
+
backgroundExecute(context: AdapterContext<CustomSettings>): Promise<number>;
|
|
79
|
+
}
|
|
80
|
+
export {};
|
|
@@ -90,7 +90,7 @@ class WebSocketTransport {
|
|
|
90
90
|
logger.debug('Successfully executed connection opened handler');
|
|
91
91
|
// Record active ws connections by incrementing count on open
|
|
92
92
|
// Using URL in label since connection_key is removed from v3
|
|
93
|
-
transportMetrics.wsConnectionActive.
|
|
93
|
+
transportMetrics.wsConnectionActive.inc();
|
|
94
94
|
resolve(true);
|
|
95
95
|
});
|
|
96
96
|
this.wsConnection.addEventListener('message', async (event) => {
|
|
@@ -114,7 +114,7 @@ class WebSocketTransport {
|
|
|
114
114
|
logger.debug(`Closed websocket connection. Code: ${event.code} ; reason: ${event.reason?.toString()}`);
|
|
115
115
|
// Record active ws connections by decrementing count on close
|
|
116
116
|
// Using URL in label since connection_key is removed from v3
|
|
117
|
-
transportMetrics.wsConnectionActive.
|
|
117
|
+
transportMetrics.wsConnectionActive.dec();
|
|
118
118
|
});
|
|
119
119
|
});
|
|
120
120
|
}
|
|
@@ -124,12 +124,12 @@ class WebSocketTransport {
|
|
|
124
124
|
const desiredSubs = this.expiringSortedSet.getAll();
|
|
125
125
|
logger.debug('Generating delta (subscribes & unsubscribes)');
|
|
126
126
|
// TODO: More efficient algorithm, this is really easy to read, but high(er) time complexity
|
|
127
|
-
const
|
|
128
|
-
|
|
127
|
+
const subscribeParams = desiredSubs.filter((s) => !this.localSubscriptions.includes(s));
|
|
128
|
+
const subscribes = subscribeParams
|
|
129
129
|
.map(this.config.builders.subscribeMessage)
|
|
130
130
|
.map(this.serializeMessage);
|
|
131
|
-
const
|
|
132
|
-
|
|
131
|
+
const unsubscribeParams = this.localSubscriptions.filter((s) => !desiredSubs.includes(s));
|
|
132
|
+
const unsubscribes = unsubscribeParams
|
|
133
133
|
.map(this.config.builders.unsubscribeMessage)
|
|
134
134
|
.map(this.serializeMessage);
|
|
135
135
|
logger.debug(`${subscribes.length} new subscriptions; ${unsubscribes.length} to unsubscribe`);
|
|
@@ -152,20 +152,14 @@ class WebSocketTransport {
|
|
|
152
152
|
await this.establishWsConnection(context);
|
|
153
153
|
}
|
|
154
154
|
// TODO: Close connection at some point?
|
|
155
|
-
// TODO: Finish ws metrics
|
|
156
155
|
logger.debug('Sending subs/unsubs if there are any');
|
|
157
156
|
const messages = unsubscribes.concat(subscribes);
|
|
158
157
|
for (const message of messages) {
|
|
159
158
|
logger.trace(`Sending message: ${JSON.stringify(message)}`);
|
|
160
159
|
this.wsConnection.send(message);
|
|
161
|
-
// Record total number of ws messages sent
|
|
162
|
-
// ws_message_total.labels(messageLabels(this.config.key)).inc()
|
|
163
160
|
}
|
|
164
|
-
// Record
|
|
165
|
-
|
|
166
|
-
// Record number of active ws subscriptions
|
|
167
|
-
// const activeSubCount = desiredSubs ? desiredSubs.length : 0
|
|
168
|
-
// ws_subscription_active.labels(subscriptionLabels(this.config.key)).set(activeSubCount)
|
|
161
|
+
// Record WS message and subscription metrics
|
|
162
|
+
transportMetrics.recordWsMessageMetrics(context, subscribeParams, unsubscribeParams);
|
|
169
163
|
logger.debug('Setting local state to cache value');
|
|
170
164
|
this.localSubscriptions = desiredSubs;
|
|
171
165
|
logger.debug('Background execute complete');
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* An object describing an entry in the expiring sorted set.
|
|
3
|
+
* @typeParam T - the type of the entry's value
|
|
4
|
+
*/
|
|
5
|
+
interface ExpiringSortedSetEntry<T> {
|
|
6
|
+
value: T;
|
|
7
|
+
expirationTimestamp: number;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* This class implements a set of unique items, each of which has an expiration timestamp.
|
|
11
|
+
* On reads, items that have expired will be deleted from the set and not returned.
|
|
12
|
+
*
|
|
13
|
+
* @typeParam T - the type of the set entries' values
|
|
14
|
+
*/
|
|
15
|
+
export declare class ExpiringSortedSet<T> {
|
|
16
|
+
map: Map<string, ExpiringSortedSetEntry<T>>;
|
|
17
|
+
add(key: string, value: T, ttl: number): void;
|
|
18
|
+
get(key: string): T | undefined;
|
|
19
|
+
getAll(): T[];
|
|
20
|
+
}
|
|
21
|
+
export {};
|
|
File without changes
|
package/util/index.d.ts
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export * from './request';
|
|
2
|
+
export * from './logger';
|
|
3
|
+
export * from './expiring-sorted-set';
|
|
4
|
+
/**
|
|
5
|
+
* Sleeps for the provided number of milliseconds
|
|
6
|
+
* @param ms - The number of milliseconds to sleep for
|
|
7
|
+
* @returns a Promise that resolves once the specified time passes
|
|
8
|
+
*/
|
|
9
|
+
export declare const sleep: (ms: number) => Promise<void>;
|
|
10
|
+
export declare const isObject: (o: unknown) => boolean;
|
|
11
|
+
export declare const isArray: (o: unknown) => boolean;
|
|
File without changes
|
package/util/logger.d.ts
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import pino from 'pino';
|
|
3
|
+
import { AdapterRequest } from './request';
|
|
4
|
+
import { FastifyReply, HookHandlerDoneFunction } from 'fastify';
|
|
5
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
6
|
+
export declare const asyncLocalStorage: AsyncLocalStorage<unknown>;
|
|
7
|
+
export declare type Store = {
|
|
8
|
+
correlationId: string;
|
|
9
|
+
};
|
|
10
|
+
/**
|
|
11
|
+
* Instead of using a global logger instance, we want to force using a child logger
|
|
12
|
+
* with a specific layer set in it, so that we can filter logs by where they're output from.
|
|
13
|
+
*
|
|
14
|
+
* Details on what each log level represents:
|
|
15
|
+
* "trace": Forensic debugging of issues on a local machine.
|
|
16
|
+
* "debug": Detailed logging level to get more context from users on their environments.
|
|
17
|
+
* "info": High-level informational messages, to describe at a glance the high level state of the system.
|
|
18
|
+
* "warn": A mild error occurred that might require non-urgent action.
|
|
19
|
+
* "error": An unexpected error occurred during the regular operation of a well-maintained EA.
|
|
20
|
+
* "fatal": The EA encountered an unrecoverable problem and had to exit.
|
|
21
|
+
*
|
|
22
|
+
* Full reference this is based on can be found at
|
|
23
|
+
* https://github.com/smartcontractkit/documentation/blob/main/docs/Node%20Operators/configuration-variables.md#log_level
|
|
24
|
+
*
|
|
25
|
+
* @param layer - the layer name to include in the logs (e.g. "SomeMiddleware", "RedisCache", etc.)
|
|
26
|
+
* @returns a layer specific logger
|
|
27
|
+
*/
|
|
28
|
+
export declare const makeLogger: (layer: string) => pino.Logger<{
|
|
29
|
+
level: string;
|
|
30
|
+
mixin(): {};
|
|
31
|
+
transport: {
|
|
32
|
+
target: string;
|
|
33
|
+
options: {
|
|
34
|
+
levelFirst: boolean;
|
|
35
|
+
levelLabel: string;
|
|
36
|
+
ignore: string;
|
|
37
|
+
messageFormat: string;
|
|
38
|
+
translateTime: string;
|
|
39
|
+
};
|
|
40
|
+
} | undefined;
|
|
41
|
+
} & pino.ChildLoggerOptions>;
|
|
42
|
+
export declare const loggingContextMiddleware: (req: AdapterRequest, res: FastifyReply, done: HookHandlerDoneFunction) => void;
|
|
File without changes
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { FastifyReply, FastifyRequest, HookHandlerDoneFunction } from 'fastify';
|
|
2
|
+
import { InitializedAdapter } from '../adapter';
|
|
3
|
+
import { AdapterError } from '../validation/error';
|
|
4
|
+
declare module 'fastify' {
|
|
5
|
+
interface FastifyRequest {
|
|
6
|
+
requestContext: AdapterRequestContext;
|
|
7
|
+
}
|
|
8
|
+
}
|
|
9
|
+
export interface AdapterRequestBody<T = AdapterRequestData> {
|
|
10
|
+
endpoint?: string;
|
|
11
|
+
data: T;
|
|
12
|
+
id?: string;
|
|
13
|
+
}
|
|
14
|
+
export declare type AdapterRequestContext<T = AdapterRequestData> = {
|
|
15
|
+
id: string;
|
|
16
|
+
endpointName: string;
|
|
17
|
+
cacheKey: string;
|
|
18
|
+
data: T;
|
|
19
|
+
meta?: AdapterRequestMeta;
|
|
20
|
+
};
|
|
21
|
+
export declare type AdapterRouteGeneric<T = AdapterRequestData> = {
|
|
22
|
+
Body: AdapterRequestBody<T>;
|
|
23
|
+
};
|
|
24
|
+
export declare type AdapterRequest<T = AdapterRequestData> = FastifyRequest<AdapterRouteGeneric<T>> & {
|
|
25
|
+
requestContext: AdapterRequestContext<T>;
|
|
26
|
+
};
|
|
27
|
+
/**
|
|
28
|
+
* Meta info that pertains to exposing metrics
|
|
29
|
+
*/
|
|
30
|
+
export interface AdapterRequestMeta {
|
|
31
|
+
metrics?: AdapterMetricsMeta;
|
|
32
|
+
error?: AdapterError | Error;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Meta info that pertains to exposing metrics
|
|
36
|
+
*/
|
|
37
|
+
export interface AdapterMetricsMeta {
|
|
38
|
+
feedId?: string;
|
|
39
|
+
cacheHit?: boolean;
|
|
40
|
+
}
|
|
41
|
+
export declare type AdapterRequestData = Record<string, unknown>;
|
|
42
|
+
export interface ProviderResult<Params> {
|
|
43
|
+
params: Params;
|
|
44
|
+
value: unknown;
|
|
45
|
+
}
|
|
46
|
+
export declare type AdapterResponse<T = unknown> = {
|
|
47
|
+
statusCode: number;
|
|
48
|
+
data: T;
|
|
49
|
+
result: unknown;
|
|
50
|
+
maxAge?: number;
|
|
51
|
+
meta?: AdapterRequestMeta;
|
|
52
|
+
providerStatusCode?: number;
|
|
53
|
+
};
|
|
54
|
+
export declare type Middleware = ((req: AdapterRequest, reply: FastifyReply, done: HookHandlerDoneFunction) => FastifyReply | void) | ((req: AdapterRequest, reply: FastifyReply) => Promise<FastifyReply | void>);
|
|
55
|
+
export declare type AdapterMiddlewareBuilder = (adapter: InitializedAdapter) => Middleware;
|
|
File without changes
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { HttpRequestType } from "../metrics/constants";
|
|
2
|
+
declare type ErrorBasic = {
|
|
3
|
+
name: string;
|
|
4
|
+
message: string;
|
|
5
|
+
};
|
|
6
|
+
declare type ErrorFull = ErrorBasic & {
|
|
7
|
+
stack: string;
|
|
8
|
+
cause: string;
|
|
9
|
+
};
|
|
10
|
+
export declare type AdapterErrorResponse = {
|
|
11
|
+
jobRunID: string;
|
|
12
|
+
status: string;
|
|
13
|
+
statusCode: number;
|
|
14
|
+
providerStatusCode?: number;
|
|
15
|
+
error: ErrorBasic | ErrorFull;
|
|
16
|
+
};
|
|
17
|
+
export declare class AdapterError extends Error {
|
|
18
|
+
jobRunID: string;
|
|
19
|
+
status: string;
|
|
20
|
+
statusCode: number;
|
|
21
|
+
cause: any;
|
|
22
|
+
url?: string;
|
|
23
|
+
errorResponse: any;
|
|
24
|
+
feedID?: string;
|
|
25
|
+
providerStatusCode?: number;
|
|
26
|
+
metricsLabel?: HttpRequestType;
|
|
27
|
+
name: string;
|
|
28
|
+
message: string;
|
|
29
|
+
constructor({ jobRunID, status, statusCode, name, message, cause, url, errorResponse, feedID, providerStatusCode, metricsLabel }: Partial<AdapterError>);
|
|
30
|
+
toJSONResponse(): AdapterErrorResponse;
|
|
31
|
+
}
|
|
32
|
+
export declare class AdapterInputError extends AdapterError {
|
|
33
|
+
constructor(input: Partial<AdapterError>);
|
|
34
|
+
}
|
|
35
|
+
export declare class AdapterRateLimitError extends AdapterError {
|
|
36
|
+
constructor(input: Partial<AdapterError>);
|
|
37
|
+
}
|
|
38
|
+
export declare class AdapterTimeoutError extends AdapterError {
|
|
39
|
+
constructor(input: Partial<AdapterError>);
|
|
40
|
+
}
|
|
41
|
+
export declare class AdapterDataProviderError extends AdapterError {
|
|
42
|
+
constructor(input: Partial<AdapterError>);
|
|
43
|
+
}
|
|
44
|
+
export declare class AdapterConnectionError extends AdapterError {
|
|
45
|
+
constructor(input: Partial<AdapterError>);
|
|
46
|
+
}
|
|
47
|
+
export declare class AdapterCustomError extends AdapterError {
|
|
48
|
+
constructor(input: Partial<AdapterError>);
|
|
49
|
+
}
|
|
50
|
+
export {};
|