@chainlink/external-adapter-framework 0.0.7 → 0.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adapter.d.ts +88 -0
- package/{dist/src/adapter.js → adapter.js} +0 -0
- package/background-executor.d.ts +11 -0
- package/{dist/background-executor.js → background-executor.js} +0 -0
- package/cache/factory.d.ts +6 -0
- package/{dist/cache → cache}/factory.js +0 -0
- package/cache/index.d.ts +90 -0
- package/{dist/src/cache → cache}/index.js +6 -2
- package/cache/local.d.ts +23 -0
- package/{dist/cache → cache}/local.js +1 -1
- package/cache/metrics.d.ts +27 -0
- package/{dist/cache → cache}/metrics.js +7 -1
- package/cache/redis.d.ts +16 -0
- package/{dist/cache → cache}/redis.js +2 -2
- package/config/index.d.ts +195 -0
- package/{dist/src/config → config}/index.js +5 -6
- package/config/provider-limits.d.ts +31 -0
- package/{dist/src/config → config}/provider-limits.js +1 -1
- package/examples/coingecko/batch-warming.d.ts +2 -0
- package/{dist/examples → examples}/coingecko/batch-warming.js +0 -0
- package/examples/coingecko/index.d.ts +2 -0
- package/{dist/examples → examples}/coingecko/index.js +0 -0
- package/examples/coingecko/rest.d.ts +2 -0
- package/{dist/examples → examples}/coingecko/rest.js +0 -0
- package/examples/ncfx/config/index.d.ts +12 -0
- package/{dist/examples → examples}/ncfx/config/index.js +0 -0
- package/examples/ncfx/index.d.ts +2 -0
- package/{dist/examples → examples}/ncfx/index.js +0 -0
- package/examples/ncfx/websocket.d.ts +36 -0
- package/{dist/examples → examples}/ncfx/websocket.js +0 -0
- package/index.d.ts +12 -0
- package/{dist/index.js → index.js} +3 -0
- package/metrics/constants.d.ts +16 -0
- package/{dist/src/metrics → metrics}/constants.js +1 -1
- package/metrics/index.d.ts +15 -0
- package/{dist/metrics → metrics}/index.js +48 -1
- package/metrics/util.d.ts +3 -0
- package/{dist/metrics → metrics}/util.js +0 -0
- package/package.json +2 -6
- package/rate-limiting/background/fixed-frequency.d.ts +10 -0
- package/{dist/src/rate-limiting → rate-limiting}/background/fixed-frequency.js +0 -0
- package/rate-limiting/index.d.ts +54 -0
- package/{dist/src/rate-limiting → rate-limiting}/index.js +0 -0
- package/rate-limiting/metrics.d.ts +3 -0
- package/{dist/src/rate-limiting → rate-limiting}/metrics.js +14 -2
- package/rate-limiting/request/simple-counting.d.ts +20 -0
- package/{dist/src/rate-limiting → rate-limiting}/request/simple-counting.js +0 -0
- package/test.d.ts +1 -0
- package/{dist/test.js → test.js} +0 -0
- package/transports/batch-warming.d.ts +34 -0
- package/{dist/src/transports → transports}/batch-warming.js +47 -1
- package/transports/index.d.ts +87 -0
- package/{dist/src/transports → transports}/index.js +5 -3
- package/transports/metrics.d.ts +21 -0
- package/{dist/src/transports → transports}/metrics.js +28 -42
- package/transports/rest.d.ts +43 -0
- package/{dist/src/transports → transports}/rest.js +38 -2
- package/transports/util.d.ts +8 -0
- package/{dist/src/transports → transports}/util.js +16 -16
- package/transports/websocket.d.ts +80 -0
- package/{dist/src/transports → transports}/websocket.js +8 -14
- package/util/expiring-sorted-set.d.ts +21 -0
- package/{dist/src/util → util}/expiring-sorted-set.js +0 -0
- package/util/index.d.ts +11 -0
- package/{dist/src/util → util}/index.js +0 -0
- package/util/logger.d.ts +42 -0
- package/{dist/src/util → util}/logger.js +0 -0
- package/util/request.d.ts +55 -0
- package/{dist/src/util → util}/request.js +0 -0
- package/validation/error.d.ts +50 -0
- package/validation/error.js +79 -0
- package/validation/index.d.ts +5 -0
- package/{dist/src/validation → validation}/index.js +8 -6
- package/validation/input-params.d.ts +15 -0
- package/{dist/src/validation → validation}/input-params.js +0 -0
- package/validation/override-functions.d.ts +3 -0
- package/{dist/src/validation → validation}/override-functions.js +0 -0
- package/{dist/src/validation → validation}/preset-tokens.json +0 -0
- package/validation/validator.d.ts +47 -0
- package/{dist/src/validation → validation}/validator.js +0 -0
- package/README.md +0 -103
- package/dist/adapter.js +0 -60
- package/dist/cache/index.js +0 -163
- package/dist/config/index.js +0 -364
- package/dist/config/provider-limits.js +0 -75
- package/dist/metrics/constants.js +0 -25
- package/dist/rate-limiting/factory.js +0 -33
- package/dist/rate-limiting/index.js +0 -36
- package/dist/rate-limiting/metrics.js +0 -32
- package/dist/rate-limiting/nop-limiter.js +0 -15
- package/dist/rate-limiting/simple-counting.js +0 -61
- package/dist/src/background-executor.js +0 -45
- package/dist/src/cache/factory.js +0 -57
- package/dist/src/cache/local.js +0 -83
- package/dist/src/cache/metrics.js +0 -114
- package/dist/src/cache/redis.js +0 -100
- package/dist/src/examples/bank-frick/accounts.js +0 -191
- package/dist/src/examples/bank-frick/config/index.js +0 -45
- package/dist/src/examples/bank-frick/index.js +0 -14
- package/dist/src/examples/bank-frick/util.js +0 -39
- package/dist/src/examples/coingecko/batch-warming.js +0 -52
- package/dist/src/examples/coingecko/index.js +0 -10
- package/dist/src/examples/coingecko/rest.js +0 -50
- package/dist/src/examples/ncfx/config/index.js +0 -15
- package/dist/src/examples/ncfx/index.js +0 -10
- package/dist/src/examples/ncfx/websocket.js +0 -72
- package/dist/src/index.js +0 -89
- package/dist/src/metrics/index.js +0 -76
- package/dist/src/metrics/util.js +0 -9
- package/dist/src/test.js +0 -6
- package/dist/src/validation/error.js +0 -41
- package/dist/transports/batch-warming.js +0 -57
- package/dist/transports/index.js +0 -76
- package/dist/transports/metrics.js +0 -133
- package/dist/transports/rest.js +0 -91
- package/dist/transports/util.js +0 -85
- package/dist/transports/websocket.js +0 -171
- package/dist/util/expiring-sorted-set.js +0 -47
- package/dist/util/index.js +0 -35
- package/dist/util/logger.js +0 -62
- package/dist/util/request.js +0 -2
- package/dist/validation/error.js +0 -41
- package/dist/validation/index.js +0 -82
- package/dist/validation/input-params.js +0 -30
- package/dist/validation/overrideFunctions.js +0 -42
- package/dist/validation/presetTokens.json +0 -23
- package/dist/validation/validator.js +0 -303
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { AdapterEndpoint } from '../../adapter';
|
|
2
|
+
import { WebSocketTransport } from '../../transports/websocket';
|
|
3
|
+
import { InputParameters } from '../../validation';
|
|
4
|
+
interface AdapterRequestParams {
|
|
5
|
+
base: string;
|
|
6
|
+
quote: string;
|
|
7
|
+
}
|
|
8
|
+
export declare const inputParameters: InputParameters;
|
|
9
|
+
interface ProviderMessage {
|
|
10
|
+
timestamp: string;
|
|
11
|
+
currencyPair: string;
|
|
12
|
+
bid: number;
|
|
13
|
+
offer: number;
|
|
14
|
+
mid: number;
|
|
15
|
+
changes: [
|
|
16
|
+
{
|
|
17
|
+
period: string;
|
|
18
|
+
change: number;
|
|
19
|
+
percentage: number;
|
|
20
|
+
}
|
|
21
|
+
];
|
|
22
|
+
}
|
|
23
|
+
export declare const websocketTransport: WebSocketTransport<AdapterRequestParams, ProviderMessage[], {
|
|
24
|
+
readonly USERNAME: {
|
|
25
|
+
readonly description: "Username for the NCFX API";
|
|
26
|
+
readonly type: "string";
|
|
27
|
+
readonly required: true;
|
|
28
|
+
};
|
|
29
|
+
readonly PASSWORD: {
|
|
30
|
+
readonly description: "Password for the NCFX API";
|
|
31
|
+
readonly type: "string";
|
|
32
|
+
readonly required: true;
|
|
33
|
+
};
|
|
34
|
+
}>;
|
|
35
|
+
export declare const webSocketEndpoint: AdapterEndpoint;
|
|
36
|
+
export {};
|
|
File without changes
|
package/index.d.ts
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { Server } from 'http';
|
|
2
|
+
import { Adapter } from './adapter';
|
|
3
|
+
import { AdapterDependencies } from './transports';
|
|
4
|
+
/**
|
|
5
|
+
* Main function for the framework.
|
|
6
|
+
* Initializes config and dependencies, uses those to initialize Transports, and starts listening for requests.
|
|
7
|
+
*
|
|
8
|
+
* @param adapter - an object describing an External Adapter
|
|
9
|
+
* @param dependencies - an optional object with adapter dependencies to inject
|
|
10
|
+
* @returns a Promise that resolves to the http.Server listening for connections
|
|
11
|
+
*/
|
|
12
|
+
export declare const expose: (adapter: Adapter, dependencies?: Partial<AdapterDependencies>) => Promise<Server | undefined>;
|
|
@@ -84,6 +84,9 @@ async function buildRestApi(config, initializedAdapter) {
|
|
|
84
84
|
method: 'POST',
|
|
85
85
|
handler: transportHandler,
|
|
86
86
|
}); // Pass config maybe? cleaner that multiple instances / dependencies
|
|
87
|
+
if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
|
|
88
|
+
router.addHook('onResponse', metrics_1.buildMetricsMiddleware);
|
|
89
|
+
}
|
|
87
90
|
});
|
|
88
91
|
return app;
|
|
89
92
|
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export declare enum HttpRequestType {
|
|
2
|
+
CACHE_HIT = "cacheHit",
|
|
3
|
+
DATA_PROVIDER_HIT = "dataProviderHit",
|
|
4
|
+
ADAPTER_ERROR = "adapterError",
|
|
5
|
+
INPUT_ERROR = "inputError",
|
|
6
|
+
RATE_LIMIT_ERROR = "rateLimitError",
|
|
7
|
+
DP_ERROR = "dataProviderError",
|
|
8
|
+
TIMEOUT_ERROR = "timeoutError",
|
|
9
|
+
CONNECTION_ERROR = "connectionError",
|
|
10
|
+
CUSTOM_ERROR = "customError"
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Maxiumum number of characters that a feedId can contain.
|
|
14
|
+
*/
|
|
15
|
+
export declare const MAX_FEED_ID_LENGTH = 300;
|
|
16
|
+
export declare const requestDurationBuckets: number[];
|
|
@@ -12,7 +12,7 @@ var HttpRequestType;
|
|
|
12
12
|
// BACKOFF_ERROR = 'backoffError',
|
|
13
13
|
HttpRequestType["DP_ERROR"] = "dataProviderError";
|
|
14
14
|
HttpRequestType["TIMEOUT_ERROR"] = "timeoutError";
|
|
15
|
-
|
|
15
|
+
HttpRequestType["CONNECTION_ERROR"] = "connectionError";
|
|
16
16
|
// RES_EMPTY_ERROR = 'responseEmptyError',
|
|
17
17
|
// RES_INVALID_ERROR = 'responseInvalidError',
|
|
18
18
|
HttpRequestType["CUSTOM_ERROR"] = "customError";
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import * as client from 'prom-client';
|
|
2
|
+
import { AdapterRequest } from '../util';
|
|
3
|
+
import { AdapterConfig } from '../config';
|
|
4
|
+
import { FastifyReply, HookHandlerDoneFunction } from 'fastify';
|
|
5
|
+
export declare function setupMetricsServer(name: string, config: AdapterConfig): void;
|
|
6
|
+
export declare const setupMetrics: (name: string, config: AdapterConfig) => void;
|
|
7
|
+
/**
|
|
8
|
+
* Builds metrics middleware that records end to end EA response times
|
|
9
|
+
* and count of requests
|
|
10
|
+
*
|
|
11
|
+
* @returns the cache middleware function
|
|
12
|
+
*/
|
|
13
|
+
export declare const buildMetricsMiddleware: (req: AdapterRequest, res: FastifyReply, done: HookHandlerDoneFunction) => void;
|
|
14
|
+
export declare const httpRequestsTotal: client.Counter<"type" | "method" | "feed_id" | "status_code" | "provider_status_code" | "retry" | "is_cache_warming">;
|
|
15
|
+
export declare const httpRequestDurationSeconds: client.Histogram<string>;
|
|
@@ -26,12 +26,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
26
26
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
27
|
};
|
|
28
28
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
-
exports.httpRequestDurationSeconds = exports.httpRequestsTotal = exports.setupMetrics = exports.setupMetricsServer = void 0;
|
|
29
|
+
exports.httpRequestDurationSeconds = exports.httpRequestsTotal = exports.buildMetricsMiddleware = exports.setupMetrics = exports.setupMetricsServer = void 0;
|
|
30
30
|
const client = __importStar(require("prom-client"));
|
|
31
31
|
const constants_1 = require("./constants");
|
|
32
32
|
const util_1 = require("../util");
|
|
33
33
|
const fastify_1 = __importDefault(require("fastify"));
|
|
34
34
|
const path_1 = require("path");
|
|
35
|
+
const error_1 = require("../validation/error");
|
|
35
36
|
const logger = (0, util_1.makeLogger)('Metrics');
|
|
36
37
|
function setupMetricsServer(name, config) {
|
|
37
38
|
const metricsApp = (0, fastify_1.default)({
|
|
@@ -56,6 +57,52 @@ const setupMetrics = (name, config) => {
|
|
|
56
57
|
});
|
|
57
58
|
};
|
|
58
59
|
exports.setupMetrics = setupMetrics;
|
|
60
|
+
/**
|
|
61
|
+
* Builds metrics middleware that records end to end EA response times
|
|
62
|
+
* and count of requests
|
|
63
|
+
*
|
|
64
|
+
* @returns the cache middleware function
|
|
65
|
+
*/
|
|
66
|
+
const buildMetricsMiddleware = (req, res, done) => {
|
|
67
|
+
const feedId = req.requestContext.meta?.metrics?.feedId || 'N/A';
|
|
68
|
+
const labels = buildHttpRequestMetricsLabel(feedId, req.requestContext.meta?.error, req.requestContext.meta?.metrics?.cacheHit);
|
|
69
|
+
// Record number of requests sent to EA
|
|
70
|
+
exports.httpRequestsTotal.labels(labels).inc();
|
|
71
|
+
// Record response time of request through entire EA
|
|
72
|
+
exports.httpRequestDurationSeconds.observe(res.getResponseTime());
|
|
73
|
+
logger.debug(`Response time for ${feedId}: ${res.getResponseTime()}ms`);
|
|
74
|
+
done();
|
|
75
|
+
};
|
|
76
|
+
exports.buildMetricsMiddleware = buildMetricsMiddleware;
|
|
77
|
+
const buildHttpRequestMetricsLabel = (feedId, error, cacheHit) => {
|
|
78
|
+
const labels = {};
|
|
79
|
+
labels.method = 'POST';
|
|
80
|
+
labels.feed_id = feedId;
|
|
81
|
+
if (error instanceof error_1.AdapterError) {
|
|
82
|
+
// If error present and an instace of AdapterError, build label from error info
|
|
83
|
+
labels.type = error?.metricsLabel || constants_1.HttpRequestType.ADAPTER_ERROR;
|
|
84
|
+
labels.status_code = error?.statusCode;
|
|
85
|
+
labels.provider_status_code = error?.providerStatusCode;
|
|
86
|
+
}
|
|
87
|
+
else if (error instanceof Error) {
|
|
88
|
+
// If error present and not instance of generic Error, unexpected failure occurred
|
|
89
|
+
// TODO: Build label with 500 error and withold dp status? Status sent as 200 in error handler
|
|
90
|
+
labels.type = constants_1.HttpRequestType.ADAPTER_ERROR;
|
|
91
|
+
labels.status_code = 500;
|
|
92
|
+
}
|
|
93
|
+
else {
|
|
94
|
+
// If no error present, request went as expected
|
|
95
|
+
labels.status_code = 200;
|
|
96
|
+
if (cacheHit) {
|
|
97
|
+
labels.type = constants_1.HttpRequestType.CACHE_HIT;
|
|
98
|
+
}
|
|
99
|
+
else {
|
|
100
|
+
labels.type = constants_1.HttpRequestType.DATA_PROVIDER_HIT;
|
|
101
|
+
labels.provider_status_code = 200;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return labels;
|
|
105
|
+
};
|
|
59
106
|
exports.httpRequestsTotal = new client.Counter({
|
|
60
107
|
name: 'http_requests_total',
|
|
61
108
|
help: 'The number of http requests this external adapter has serviced for its entire uptime',
|
|
File without changes
|
package/package.json
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@chainlink/external-adapter-framework",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.10",
|
|
4
4
|
"main": "dist/index.js",
|
|
5
|
-
"types": "dist/index.d.ts",
|
|
6
5
|
"license": "MIT",
|
|
7
6
|
"dependencies": {
|
|
8
7
|
"axios": "^0.27.2",
|
|
@@ -66,8 +65,5 @@
|
|
|
66
65
|
"environmentVariables": {
|
|
67
66
|
"METRICS_ENABLED": "false"
|
|
68
67
|
}
|
|
69
|
-
}
|
|
70
|
-
"files": [
|
|
71
|
-
"dist/**"
|
|
72
|
-
]
|
|
68
|
+
}
|
|
73
69
|
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { AdapterRateLimitTier, BackgroundExecuteRateLimiter } from '..';
|
|
2
|
+
import { AdapterEndpoint } from '../../adapter';
|
|
3
|
+
export declare const DEFAULT_SHARED_MS_BETWEEN_REQUESTS = 5000;
|
|
4
|
+
export declare class FixedFrequencyRateLimiter implements BackgroundExecuteRateLimiter {
|
|
5
|
+
msBetweenRequestsMap: {
|
|
6
|
+
[endpointName: string]: number;
|
|
7
|
+
};
|
|
8
|
+
initialize(endpoints: AdapterEndpoint[], limits?: AdapterRateLimitTier): this;
|
|
9
|
+
msUntilNextExecution(endpointName: string): number;
|
|
10
|
+
}
|
|
File without changes
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { AdapterEndpoint } from '../adapter';
|
|
2
|
+
export * from './request/simple-counting';
|
|
3
|
+
export * from './background/fixed-frequency';
|
|
4
|
+
export interface AdapterRateLimitTier {
|
|
5
|
+
rateLimit1s?: number;
|
|
6
|
+
rateLimit1m?: number;
|
|
7
|
+
rateLimit1h?: number;
|
|
8
|
+
note?: string;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Common interface for all RateLimiter classes to implement
|
|
12
|
+
*/
|
|
13
|
+
export interface RateLimiter {
|
|
14
|
+
/**
|
|
15
|
+
* Method to ensure all RateLimiters can be initialized in the same manner.
|
|
16
|
+
*
|
|
17
|
+
* @param limits - settings for how much throughput to allow for the Adapter
|
|
18
|
+
* @param endpoints - list of adapter endpoints
|
|
19
|
+
*/
|
|
20
|
+
initialize(endpoints: AdapterEndpoint[], limits: AdapterRateLimitTier): this;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* RequestRateLimiters perform checks agains imminent outbound requests for any transport.
|
|
24
|
+
*/
|
|
25
|
+
export interface RequestRateLimiter extends RateLimiter {
|
|
26
|
+
/**
|
|
27
|
+
* This method will check using whatever strategy is implemented to determine if
|
|
28
|
+
* this request can be processed. If so, it returns true; if not, returns false.
|
|
29
|
+
*/
|
|
30
|
+
isUnderLimits(): boolean;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* BackgroundExecuteFrequencyRateLimiters will implement custom logic to calculate
|
|
34
|
+
* the period of time to wait between background executions for a transport.
|
|
35
|
+
*/
|
|
36
|
+
export interface BackgroundExecuteRateLimiter extends RateLimiter {
|
|
37
|
+
msUntilNextExecution(endpointName: string): number;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* This method will convert all possible settings for a rate limit tier and
|
|
41
|
+
* convert them all to requests per second, returning the lowest one
|
|
42
|
+
*
|
|
43
|
+
* @param limits - the rate limit tier set for the adapter
|
|
44
|
+
* @returns the most restrictive of the set options, in requests per second
|
|
45
|
+
*/
|
|
46
|
+
export declare const consolidateTierLimits: (limits?: AdapterRateLimitTier) => number;
|
|
47
|
+
/**
|
|
48
|
+
* Validates rate limiting tiers specified for the adapter, and returns the one to use.
|
|
49
|
+
*
|
|
50
|
+
* @param tiers - the adapter config listing the different available API tiers
|
|
51
|
+
* @param selectedTier - chosen API tier from settings, if present
|
|
52
|
+
* @returns the specified API tier, or a default one if none are specified
|
|
53
|
+
*/
|
|
54
|
+
export declare const getRateLimitingTier: (tiers?: Record<string, AdapterRateLimitTier>, selectedTier?: string) => AdapterRateLimitTier | undefined;
|
|
File without changes
|
|
@@ -23,10 +23,22 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
|
23
23
|
return result;
|
|
24
24
|
};
|
|
25
25
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.rateLimitCreditsSpentTotal = void 0;
|
|
26
|
+
exports.rateLimitCreditsSpentTotal = exports.retrieveCost = void 0;
|
|
27
27
|
const client = __importStar(require("prom-client"));
|
|
28
|
+
// Retrieve cost field from response if exists
|
|
29
|
+
// If not return default cost of 1
|
|
30
|
+
const retrieveCost = (data) => {
|
|
31
|
+
const cost = data['cost'];
|
|
32
|
+
if (typeof cost === 'number' || typeof cost === 'string') {
|
|
33
|
+
return Number(cost);
|
|
34
|
+
}
|
|
35
|
+
else {
|
|
36
|
+
return 1;
|
|
37
|
+
}
|
|
38
|
+
};
|
|
39
|
+
exports.retrieveCost = retrieveCost;
|
|
28
40
|
exports.rateLimitCreditsSpentTotal = new client.Counter({
|
|
29
41
|
name: 'rate_limit_credits_spent_total',
|
|
30
42
|
help: 'The number of data provider credits the adapter is consuming',
|
|
31
|
-
labelNames: ['participant_id', 'feed_id'
|
|
43
|
+
labelNames: ['participant_id', 'feed_id'],
|
|
32
44
|
});
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { AdapterRateLimitTier, RateLimiter } from '..';
|
|
2
|
+
import { AdapterEndpoint } from '../../adapter';
|
|
3
|
+
/**
|
|
4
|
+
* This rate limiter is the simplest stateful option.
|
|
5
|
+
* On startup, it'll compare the different thresholds for each tier, calculate them all
|
|
6
|
+
* in the finest window we'll use (seconds), and use the most restrictive one.
|
|
7
|
+
* This is so if the EA were to restart, we don't need to worry about persisting state
|
|
8
|
+
* for things like daily quotas. The downside is that this does not work well for bursty
|
|
9
|
+
* loads or spikes, in cases where e.g. the per second limit is high but daily quotas low.
|
|
10
|
+
*/
|
|
11
|
+
export declare class SimpleCountingRateLimiter implements RateLimiter {
|
|
12
|
+
latestSecondInterval: number;
|
|
13
|
+
requestsThisSecond: number;
|
|
14
|
+
latestMinuteInterval: number;
|
|
15
|
+
requestsThisMinute: number;
|
|
16
|
+
perSecondLimit: number;
|
|
17
|
+
perMinuteLimit: number;
|
|
18
|
+
initialize(endpoints: AdapterEndpoint[], limits?: AdapterRateLimitTier): this;
|
|
19
|
+
isUnderLimits(): boolean;
|
|
20
|
+
}
|
|
File without changes
|
package/test.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/{dist/test.js → test.js}
RENAMED
|
File without changes
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { AxiosRequestConfig, AxiosResponse } from 'axios';
|
|
2
|
+
import { Cache } from '../cache';
|
|
3
|
+
import { AdapterConfig, SettingsMap } from '../config';
|
|
4
|
+
import { BackgroundExecuteRateLimiter } from '../rate-limiting';
|
|
5
|
+
import { ExpiringSortedSet } from '../util';
|
|
6
|
+
import { AdapterRequest, ProviderResult } from '../util/request';
|
|
7
|
+
import { AdapterContext, AdapterDependencies, Transport } from './';
|
|
8
|
+
/**
|
|
9
|
+
* Transport implementation that takes incoming batches requests and keeps a warm cache of values.
|
|
10
|
+
* Within the setup function, adapter params are added to an set that also keeps track and expires values.
|
|
11
|
+
* In the background execute, the list of non-expired items in the set is fetched.
|
|
12
|
+
* Then, the list is passed through the `prepareRequest` function, that returns an AxiosRequestConfig.
|
|
13
|
+
* The Data Provider response is, they are passed through the `parseResponse` function to create a [[CacheEntry]] list.
|
|
14
|
+
* Finally, the items in that [[CacheEntry]] list are set in the Cache so the Adapter can fetch values from there.
|
|
15
|
+
*
|
|
16
|
+
* @typeParam AdapterParams - interface for the adapter request body
|
|
17
|
+
* @typeParam ProviderRequestBody - interface for the body of the request to the Data Provider
|
|
18
|
+
* @typeParam ProviderResponseBody - interface for the body of the Data Provider's response
|
|
19
|
+
*/
|
|
20
|
+
export declare class BatchWarmingTransport<AdapterParams, ProviderRequestBody, ProviderResponseBody, CustomSettings extends SettingsMap> implements Transport<AdapterParams, ProviderResponseBody, CustomSettings> {
|
|
21
|
+
private config;
|
|
22
|
+
cache: Cache;
|
|
23
|
+
rateLimiter: BackgroundExecuteRateLimiter;
|
|
24
|
+
expiringSortedSet: ExpiringSortedSet<AdapterParams>;
|
|
25
|
+
WARMER_ACTIVE: boolean;
|
|
26
|
+
constructor(config: {
|
|
27
|
+
prepareRequest: (params: AdapterParams[], context: AdapterContext<CustomSettings>) => AxiosRequestConfig<ProviderRequestBody>;
|
|
28
|
+
parseResponse: (res: AxiosResponse<ProviderResponseBody>, context: AdapterContext<CustomSettings>) => ProviderResult<AdapterParams>[];
|
|
29
|
+
});
|
|
30
|
+
initialize(dependencies: AdapterDependencies): Promise<void>;
|
|
31
|
+
hasBeenSetUp(req: AdapterRequest<AdapterParams>): Promise<boolean>;
|
|
32
|
+
setup(req: AdapterRequest<AdapterParams>, config: AdapterConfig<CustomSettings>): Promise<void>;
|
|
33
|
+
backgroundExecute(context: AdapterContext<CustomSettings>): Promise<number>;
|
|
34
|
+
}
|
|
@@ -1,9 +1,35 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
2
25
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
26
|
exports.BatchWarmingTransport = void 0;
|
|
4
27
|
const util_1 = require("../util");
|
|
5
28
|
const _1 = require("./");
|
|
6
29
|
const util_2 = require("./util");
|
|
30
|
+
const rateLimitMetrics = __importStar(require("../rate-limiting/metrics"));
|
|
31
|
+
const cacheMetrics = __importStar(require("../cache/metrics"));
|
|
32
|
+
const WARMUP_BATCH_REQUEST_ID = '9002';
|
|
7
33
|
const logger = (0, util_1.makeLogger)('BatchWarmingTransport');
|
|
8
34
|
/**
|
|
9
35
|
* Transport implementation that takes incoming batches requests and keeps a warm cache of values.
|
|
@@ -21,6 +47,9 @@ class BatchWarmingTransport {
|
|
|
21
47
|
constructor(config) {
|
|
22
48
|
this.config = config;
|
|
23
49
|
this.expiringSortedSet = new util_1.ExpiringSortedSet(); // TODO: Move to dependencies, inject
|
|
50
|
+
// Flag used to track whether the warmer has moved from having no entries to having some and vice versa
|
|
51
|
+
// Used for recording the cache warmer active metrics accurately
|
|
52
|
+
this.WARMER_ACTIVE = false;
|
|
24
53
|
}
|
|
25
54
|
async initialize(dependencies) {
|
|
26
55
|
this.cache = dependencies.cache;
|
|
@@ -38,8 +67,18 @@ class BatchWarmingTransport {
|
|
|
38
67
|
const entries = this.expiringSortedSet.getAll();
|
|
39
68
|
if (!entries.length) {
|
|
40
69
|
logger.debug('No entries in batch warming set, skipping');
|
|
70
|
+
if (this.WARMER_ACTIVE) {
|
|
71
|
+
// Decrement count when warmer changed from having entries to having none
|
|
72
|
+
cacheMetrics.cacheWarmerCount.labels({ isBatched: 'true' }).dec();
|
|
73
|
+
this.WARMER_ACTIVE = false;
|
|
74
|
+
}
|
|
41
75
|
return this.rateLimiter.msUntilNextExecution(context.adapterEndpoint.name);
|
|
42
76
|
}
|
|
77
|
+
else if (this.WARMER_ACTIVE === false) {
|
|
78
|
+
// Increment count when warmer changed from having no entries to having some
|
|
79
|
+
cacheMetrics.cacheWarmerCount.labels({ isBatched: 'true' }).inc();
|
|
80
|
+
this.WARMER_ACTIVE = true;
|
|
81
|
+
}
|
|
43
82
|
const request = this.config.prepareRequest(entries, context);
|
|
44
83
|
logger.trace('Sending request to data provider...');
|
|
45
84
|
const providerResponse = await (0, util_2.axiosRequest)(request);
|
|
@@ -48,7 +87,14 @@ class BatchWarmingTransport {
|
|
|
48
87
|
const adapterResponses = (0, _1.buildCacheEntriesFromResults)(results, context);
|
|
49
88
|
logger.debug('Setting adapter responses in cache');
|
|
50
89
|
await this.cache.setMany(adapterResponses, context.adapterConfig.CACHE_MAX_AGE);
|
|
51
|
-
//
|
|
90
|
+
// Record cost of data provider call
|
|
91
|
+
const cost = rateLimitMetrics.retrieveCost(providerResponse.data);
|
|
92
|
+
rateLimitMetrics.rateLimitCreditsSpentTotal
|
|
93
|
+
.labels({
|
|
94
|
+
feed_id: 'N/A',
|
|
95
|
+
participant_id: WARMUP_BATCH_REQUEST_ID,
|
|
96
|
+
})
|
|
97
|
+
.inc(cost);
|
|
52
98
|
return this.rateLimiter.msUntilNextExecution(context.adapterEndpoint.name);
|
|
53
99
|
}
|
|
54
100
|
}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { FastifyReply } from 'fastify';
|
|
2
|
+
import { AdapterEndpoint, InitializedAdapter } from '../adapter';
|
|
3
|
+
import { Cache, CacheEntry } from '../cache';
|
|
4
|
+
import { AdapterConfig, CustomSettingsType, SettingsMap } from '../config';
|
|
5
|
+
import { BackgroundExecuteRateLimiter, RequestRateLimiter } from '../rate-limiting';
|
|
6
|
+
import { AdapterRequest, AdapterResponse, ProviderResult } from '../util/request';
|
|
7
|
+
export * from './batch-warming';
|
|
8
|
+
export * from './rest';
|
|
9
|
+
export * from './websocket';
|
|
10
|
+
/**
|
|
11
|
+
* Dependencies that will be injected into the Adapter on startup
|
|
12
|
+
*/
|
|
13
|
+
export interface AdapterDependencies {
|
|
14
|
+
cache: Cache;
|
|
15
|
+
requestRateLimiter: RequestRateLimiter;
|
|
16
|
+
backgroundExecuteRateLimiter: BackgroundExecuteRateLimiter;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Context that will be used on background executions of a Transport.
|
|
20
|
+
* For example, the endpointName used to log statements or generate Cache keys.
|
|
21
|
+
*/
|
|
22
|
+
export interface AdapterContext<CustomSettings extends CustomSettingsType<CustomSettings> = SettingsMap> {
|
|
23
|
+
adapterEndpoint: AdapterEndpoint;
|
|
24
|
+
adapterConfig: AdapterConfig<CustomSettings>;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Generic interface for a Transport.
|
|
28
|
+
* A Transport defines the way in which an AdapterEndpoint will process incoming requests to
|
|
29
|
+
* fetch data from a Data Provider. The setup phase will take care of the former, while the
|
|
30
|
+
* backgroundExecute will be in charge of the latter.
|
|
31
|
+
* This separation gives us the ability of splitting these concerns, and optionally parallelizing
|
|
32
|
+
* the reading and writing of data to a centralized Cache.
|
|
33
|
+
*
|
|
34
|
+
* @typeParam Params - the structure of the AdapterRequest's body
|
|
35
|
+
* @typeParam Result - the structure of the AdapterResponse's body
|
|
36
|
+
*/
|
|
37
|
+
export interface Transport<Params, Result, CustomSettings extends SettingsMap> {
|
|
38
|
+
/**
|
|
39
|
+
* Initializes the transport in the Adapter context.
|
|
40
|
+
*
|
|
41
|
+
* @param dependencies - Adapter dependencies (e.g. cache instance)
|
|
42
|
+
* @returns an empty Promise
|
|
43
|
+
*/
|
|
44
|
+
initialize: (dependencies: AdapterDependencies) => Promise<void>;
|
|
45
|
+
/**
|
|
46
|
+
* Checks if the Transport has already set up the incoming request.
|
|
47
|
+
* This will likely use the cacheKey generated in the provided request.
|
|
48
|
+
*
|
|
49
|
+
* @param req - the incoming AdapterRequest
|
|
50
|
+
* @returns a Promise that returns true if setup is in place
|
|
51
|
+
*/
|
|
52
|
+
hasBeenSetUp: (req: AdapterRequest<Params>) => Promise<boolean>;
|
|
53
|
+
/**
|
|
54
|
+
* Sets up the incoming request within the Transport.
|
|
55
|
+
* In other words, it means that the Adapter has recognized this request, and will begin
|
|
56
|
+
* fetching the necessary data from a Data Provider.
|
|
57
|
+
*
|
|
58
|
+
* @param req - the incoming AdapterRequest
|
|
59
|
+
* @param config - common configuration for the Adapter as a whole
|
|
60
|
+
* @returns a Promise that _optionally_ returns an AdapterResponse, if the Transport has the capability of
|
|
61
|
+
* immediately fetching data and returning it without the background process.
|
|
62
|
+
*/
|
|
63
|
+
setup: (req: AdapterRequest<Params>, config: AdapterConfig<CustomSettings>) => Promise<AdapterResponse<Result> | void>;
|
|
64
|
+
/**
|
|
65
|
+
* If the Transport relies on an async mechanism, implementing this function will make it
|
|
66
|
+
* so the background task executor calls it on Adapter startup.
|
|
67
|
+
*
|
|
68
|
+
* @param context - background context for the execution (e.g. endpoint name)
|
|
69
|
+
*/
|
|
70
|
+
backgroundExecute?: (context: AdapterContext<CustomSettings>) => Promise<number>;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Helper method to build cache entries to set after getting a bunch of responses from a DP.
|
|
74
|
+
*
|
|
75
|
+
* @param results - a list of results coming from a DataProvider
|
|
76
|
+
* @param context - context for the Adapter
|
|
77
|
+
* @returns a list of CacheEntries of AdapterResponses
|
|
78
|
+
*/
|
|
79
|
+
export declare const buildCacheEntriesFromResults: <Params, Context extends AdapterContext<any>>(results: ProviderResult<Params>[], context: Context) => CacheEntry<AdapterResponse<null>>[];
|
|
80
|
+
/**
|
|
81
|
+
* Takes an Adapter, its configuration, and its dependencies, and it creates an express middleware
|
|
82
|
+
* that will pass along the AdapterRequest to the appropriate Transport (acc. to the endpoint in the req.)
|
|
83
|
+
*
|
|
84
|
+
* @param adapter - main adapter object, already initialized
|
|
85
|
+
* @returns the transport handler middleware function
|
|
86
|
+
*/
|
|
87
|
+
export declare const buildTransportHandler: (adapter: InitializedAdapter) => (req: AdapterRequest, reply: FastifyReply) => Promise<never>;
|
|
@@ -41,9 +41,11 @@ const buildCacheEntriesFromResults = (results, context) => results.map((r) => {
|
|
|
41
41
|
context.adapterConfig.EXPERIMENTAL_METRICS_ENABLED) {
|
|
42
42
|
const metrics = {
|
|
43
43
|
maxAge: Date.now() + context.adapterConfig.CACHE_MAX_AGE,
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
44
|
+
meta: {
|
|
45
|
+
metrics: {
|
|
46
|
+
feedId: (0, cache_1.calculateFeedId)(context.adapterEndpoint, r.params)
|
|
47
|
+
}
|
|
48
|
+
}
|
|
47
49
|
};
|
|
48
50
|
cacheEntry.value = { ...cacheEntry.value, ...metrics };
|
|
49
51
|
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import * as client from 'prom-client';
|
|
2
|
+
import { AdapterContext } from '.';
|
|
3
|
+
export declare const dataProviderMetricsLabel: (providerStatusCode?: number, method?: string) => {
|
|
4
|
+
provider_status_code: number | undefined;
|
|
5
|
+
method: string;
|
|
6
|
+
};
|
|
7
|
+
export declare const dataProviderRequests: client.Counter<"method" | "provider_status_code">;
|
|
8
|
+
export declare const dataProviderRequestDurationSeconds: client.Histogram<string>;
|
|
9
|
+
export declare const connectionErrorLabels: (message: string) => {
|
|
10
|
+
message: string;
|
|
11
|
+
};
|
|
12
|
+
export declare const messageSubsLabels: (feed_id: string, cache_key: string) => {
|
|
13
|
+
feed_id: string;
|
|
14
|
+
subscription_key: string;
|
|
15
|
+
};
|
|
16
|
+
export declare const recordWsMessageMetrics: <AdapterParams>(context: AdapterContext, subscribes: AdapterParams[], unsubscrices: AdapterParams[]) => void;
|
|
17
|
+
export declare const wsConnectionActive: client.Gauge<"url">;
|
|
18
|
+
export declare const wsConnectionErrors: client.Counter<"message" | "url">;
|
|
19
|
+
export declare const wsSubscriptionActive: client.Gauge<"feed_id" | "connection_url" | "subscription_key">;
|
|
20
|
+
export declare const wsSubscriptionTotal: client.Counter<"feed_id" | "connection_url" | "subscription_key">;
|
|
21
|
+
export declare const wsMessageTotal: client.Counter<"feed_id" | "subscription_key">;
|