@chainlink/external-adapter-framework 0.0.7 → 0.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adapter.d.ts +88 -0
- package/{dist/src/adapter.js → adapter.js} +0 -0
- package/background-executor.d.ts +11 -0
- package/{dist/background-executor.js → background-executor.js} +0 -0
- package/cache/factory.d.ts +6 -0
- package/{dist/cache → cache}/factory.js +0 -0
- package/cache/index.d.ts +90 -0
- package/{dist/src/cache → cache}/index.js +6 -2
- package/cache/local.d.ts +23 -0
- package/{dist/cache → cache}/local.js +1 -1
- package/cache/metrics.d.ts +27 -0
- package/{dist/cache → cache}/metrics.js +7 -1
- package/cache/redis.d.ts +16 -0
- package/{dist/cache → cache}/redis.js +2 -2
- package/config/index.d.ts +195 -0
- package/{dist/src/config → config}/index.js +5 -6
- package/config/provider-limits.d.ts +31 -0
- package/{dist/src/config → config}/provider-limits.js +1 -1
- package/examples/coingecko/batch-warming.d.ts +2 -0
- package/{dist/examples → examples}/coingecko/batch-warming.js +0 -0
- package/examples/coingecko/index.d.ts +2 -0
- package/{dist/examples → examples}/coingecko/index.js +0 -0
- package/examples/coingecko/rest.d.ts +2 -0
- package/{dist/examples → examples}/coingecko/rest.js +0 -0
- package/examples/ncfx/config/index.d.ts +12 -0
- package/{dist/examples → examples}/ncfx/config/index.js +0 -0
- package/examples/ncfx/index.d.ts +2 -0
- package/{dist/examples → examples}/ncfx/index.js +0 -0
- package/examples/ncfx/websocket.d.ts +36 -0
- package/{dist/examples → examples}/ncfx/websocket.js +0 -0
- package/index.d.ts +12 -0
- package/{dist/index.js → index.js} +3 -0
- package/metrics/constants.d.ts +16 -0
- package/{dist/src/metrics → metrics}/constants.js +1 -1
- package/metrics/index.d.ts +15 -0
- package/{dist/metrics → metrics}/index.js +48 -1
- package/metrics/util.d.ts +3 -0
- package/{dist/metrics → metrics}/util.js +0 -0
- package/package.json +2 -6
- package/rate-limiting/background/fixed-frequency.d.ts +10 -0
- package/{dist/src/rate-limiting → rate-limiting}/background/fixed-frequency.js +0 -0
- package/rate-limiting/index.d.ts +54 -0
- package/{dist/src/rate-limiting → rate-limiting}/index.js +0 -0
- package/rate-limiting/metrics.d.ts +3 -0
- package/{dist/src/rate-limiting → rate-limiting}/metrics.js +14 -2
- package/rate-limiting/request/simple-counting.d.ts +20 -0
- package/{dist/src/rate-limiting → rate-limiting}/request/simple-counting.js +0 -0
- package/test.d.ts +1 -0
- package/{dist/test.js → test.js} +0 -0
- package/transports/batch-warming.d.ts +34 -0
- package/{dist/src/transports → transports}/batch-warming.js +47 -1
- package/transports/index.d.ts +87 -0
- package/{dist/src/transports → transports}/index.js +5 -3
- package/transports/metrics.d.ts +21 -0
- package/{dist/src/transports → transports}/metrics.js +28 -42
- package/transports/rest.d.ts +43 -0
- package/{dist/src/transports → transports}/rest.js +38 -2
- package/transports/util.d.ts +8 -0
- package/{dist/src/transports → transports}/util.js +16 -16
- package/transports/websocket.d.ts +80 -0
- package/{dist/src/transports → transports}/websocket.js +8 -14
- package/util/expiring-sorted-set.d.ts +21 -0
- package/{dist/src/util → util}/expiring-sorted-set.js +0 -0
- package/util/index.d.ts +11 -0
- package/{dist/src/util → util}/index.js +0 -0
- package/util/logger.d.ts +42 -0
- package/{dist/src/util → util}/logger.js +0 -0
- package/util/request.d.ts +55 -0
- package/{dist/src/util → util}/request.js +0 -0
- package/validation/error.d.ts +50 -0
- package/validation/error.js +79 -0
- package/validation/index.d.ts +5 -0
- package/{dist/src/validation → validation}/index.js +8 -6
- package/validation/input-params.d.ts +15 -0
- package/{dist/src/validation → validation}/input-params.js +0 -0
- package/validation/override-functions.d.ts +3 -0
- package/{dist/src/validation → validation}/override-functions.js +0 -0
- package/{dist/src/validation → validation}/preset-tokens.json +0 -0
- package/validation/validator.d.ts +47 -0
- package/{dist/src/validation → validation}/validator.js +0 -0
- package/README.md +0 -103
- package/dist/adapter.js +0 -60
- package/dist/cache/index.js +0 -163
- package/dist/config/index.js +0 -364
- package/dist/config/provider-limits.js +0 -75
- package/dist/metrics/constants.js +0 -25
- package/dist/rate-limiting/factory.js +0 -33
- package/dist/rate-limiting/index.js +0 -36
- package/dist/rate-limiting/metrics.js +0 -32
- package/dist/rate-limiting/nop-limiter.js +0 -15
- package/dist/rate-limiting/simple-counting.js +0 -61
- package/dist/src/background-executor.js +0 -45
- package/dist/src/cache/factory.js +0 -57
- package/dist/src/cache/local.js +0 -83
- package/dist/src/cache/metrics.js +0 -114
- package/dist/src/cache/redis.js +0 -100
- package/dist/src/examples/bank-frick/accounts.js +0 -191
- package/dist/src/examples/bank-frick/config/index.js +0 -45
- package/dist/src/examples/bank-frick/index.js +0 -14
- package/dist/src/examples/bank-frick/util.js +0 -39
- package/dist/src/examples/coingecko/batch-warming.js +0 -52
- package/dist/src/examples/coingecko/index.js +0 -10
- package/dist/src/examples/coingecko/rest.js +0 -50
- package/dist/src/examples/ncfx/config/index.js +0 -15
- package/dist/src/examples/ncfx/index.js +0 -10
- package/dist/src/examples/ncfx/websocket.js +0 -72
- package/dist/src/index.js +0 -89
- package/dist/src/metrics/index.js +0 -76
- package/dist/src/metrics/util.js +0 -9
- package/dist/src/test.js +0 -6
- package/dist/src/validation/error.js +0 -41
- package/dist/transports/batch-warming.js +0 -57
- package/dist/transports/index.js +0 -76
- package/dist/transports/metrics.js +0 -133
- package/dist/transports/rest.js +0 -91
- package/dist/transports/util.js +0 -85
- package/dist/transports/websocket.js +0 -171
- package/dist/util/expiring-sorted-set.js +0 -47
- package/dist/util/index.js +0 -35
- package/dist/util/logger.js +0 -62
- package/dist/util/request.js +0 -2
- package/dist/validation/error.js +0 -41
- package/dist/validation/index.js +0 -82
- package/dist/validation/input-params.js +0 -30
- package/dist/validation/overrideFunctions.js +0 -42
- package/dist/validation/presetTokens.json +0 -23
- package/dist/validation/validator.js +0 -303
package/adapter.d.ts
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { AdapterConfig, BaseAdapterConfig, SettingsMap } from './config';
|
|
2
|
+
import { AdapterRateLimitTier } from './rate-limiting';
|
|
3
|
+
import { AdapterDependencies, Transport } from './transports';
|
|
4
|
+
import { InputParameters } from './validation/input-params';
|
|
5
|
+
/**
|
|
6
|
+
* Structure to describe rate limits specs for the Adapter
|
|
7
|
+
*/
|
|
8
|
+
interface AdapterRateLimitingConfig {
|
|
9
|
+
/** Adapter rate limits, gotten from the specific tier requested */
|
|
10
|
+
tiers: Record<string, AdapterRateLimitTier>;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Main structure of an External Adapter
|
|
14
|
+
*/
|
|
15
|
+
export interface Adapter {
|
|
16
|
+
/** Name of the adapter */
|
|
17
|
+
name: string;
|
|
18
|
+
/** If present, the string that will be used for requests with no specified endpoint */
|
|
19
|
+
defaultEndpoint?: string;
|
|
20
|
+
/** List of [[AdapterEndpoint]]s in the adapter */
|
|
21
|
+
endpoints: AdapterEndpoint[];
|
|
22
|
+
/** Map of overrides to the default config values for an Adapter */
|
|
23
|
+
envDefaultOverrides?: Partial<BaseAdapterConfig>;
|
|
24
|
+
/** List of custom env vars for this particular adapter (e.g. RPC_URL) */
|
|
25
|
+
customSettings?: SettingsMap;
|
|
26
|
+
/** Configuration relevant to outbound (EA --\> DP) communication rate limiting */
|
|
27
|
+
rateLimiting?: AdapterRateLimitingConfig;
|
|
28
|
+
/** Overrides for converting the 'base' parameter that are hardcoded into the adapter. */
|
|
29
|
+
overrides?: Record<string, string>;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Structure to describe rate limits specs for a specific adapter endpoint
|
|
33
|
+
*/
|
|
34
|
+
export interface EndpointRateLimitingConfig {
|
|
35
|
+
/**
|
|
36
|
+
* How much of the total limit for the adapter will be assigned to this specific endpoint.
|
|
37
|
+
* Should be a non-zero positive number up to 100.
|
|
38
|
+
* Endpoints in the same adapter without a specific allocation will divide the remaining limits equally.
|
|
39
|
+
*/
|
|
40
|
+
allocationPercentage: number;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Structure to describe a specific endpoint in an [[Adapter]]
|
|
44
|
+
*/
|
|
45
|
+
export interface AdapterEndpoint {
|
|
46
|
+
/** Name that will be used to match input params to this endpoint (case insensitive) */
|
|
47
|
+
name: string;
|
|
48
|
+
/** List of alternative endpoint names that will resolve to this same transport (case insensitive) */
|
|
49
|
+
aliases?: string[];
|
|
50
|
+
/** Transport that will be used to handle data processing and communication for this endpoint */
|
|
51
|
+
transport: Transport<any, any, any>;
|
|
52
|
+
/** Specification of what the body of a request hitting this endpoint should look like (used for validation) */
|
|
53
|
+
inputParameters: InputParameters;
|
|
54
|
+
/** Specific details related to the rate limiting for this endpoint in particular */
|
|
55
|
+
rateLimiting?: EndpointRateLimitingConfig;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Structure to describe an adapter that has been initialized
|
|
59
|
+
*/
|
|
60
|
+
export interface InitializedAdapter extends Adapter {
|
|
61
|
+
/** Object containing alias translations for all endpoints */
|
|
62
|
+
endpointsMap: Record<string, AdapterEndpoint>;
|
|
63
|
+
/** Initialized dependencies that the adapter will use */
|
|
64
|
+
dependencies: AdapterDependencies;
|
|
65
|
+
/** Configuration params for various adapter properties */
|
|
66
|
+
config: AdapterConfig;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* This function will process dependencies for an adapter, such as caches or rate limiters,
|
|
70
|
+
* in order to inject them into transports and other relevant places later in the lifecycle.
|
|
71
|
+
*
|
|
72
|
+
* @param config - the configuration for this adapter
|
|
73
|
+
* @param inputDependencies - a partial obj of initialized dependencies to override the created ones
|
|
74
|
+
* @param rateLimitingConfig - details from the adapter regarding rate limiting
|
|
75
|
+
* @returns a set of AdapterDependencies all initialized
|
|
76
|
+
*/
|
|
77
|
+
export declare const initializeDependencies: (adapter: Adapter, config: AdapterConfig, inputDependencies?: Partial<AdapterDependencies>) => AdapterDependencies;
|
|
78
|
+
/**
|
|
79
|
+
* Initializes all of the [[Transport]]s in the adapter, passing along any [[AdapterDependencies]] and [[AdapterConfig]].
|
|
80
|
+
* Additionally, it builds a map out of all the endpoint names and aliases (checking for duplicates).
|
|
81
|
+
*
|
|
82
|
+
* @param adapter - an instance of an Adapter
|
|
83
|
+
* @param dependencies - dependencies that the adapter will need at initialization
|
|
84
|
+
* @param config - configuration variables already processed and validated
|
|
85
|
+
* @returns - the adapter with all transports initialized and aliases map built
|
|
86
|
+
*/
|
|
87
|
+
export declare const initializeAdapter: (adapter: Adapter, config: AdapterConfig, dependencies?: Partial<AdapterDependencies>) => Promise<InitializedAdapter>;
|
|
88
|
+
export {};
|
|
File without changes
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import { Server } from 'http';
|
|
3
|
+
import { InitializedAdapter } from './adapter';
|
|
4
|
+
/**
|
|
5
|
+
* Very simple background loop that will call the [[Transport.backgroundExecute]] functions in all Transports.
|
|
6
|
+
* It gets the time in ms to wait as the return value from those functions, and sleeps until next execution.
|
|
7
|
+
*
|
|
8
|
+
* @param adapter - an initialized External Adapter
|
|
9
|
+
* @param server - the http server to attach an on close listener to
|
|
10
|
+
*/
|
|
11
|
+
export declare function callBackgroundExecutes(adapter: InitializedAdapter, server?: Server): Promise<void>;
|
|
File without changes
|
|
File without changes
|
package/cache/index.d.ts
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { AdapterEndpoint } from '../adapter';
|
|
2
|
+
import { AdapterConfig } from '../config';
|
|
3
|
+
import { AdapterMiddlewareBuilder, AdapterResponse, sleep } from '../util';
|
|
4
|
+
export * from './local';
|
|
5
|
+
export * from './redis';
|
|
6
|
+
/**
|
|
7
|
+
* An object describing an entry in the cache.
|
|
8
|
+
* @typeParam T - the type of the entry's value
|
|
9
|
+
*/
|
|
10
|
+
export interface CacheEntry<T> {
|
|
11
|
+
key: string;
|
|
12
|
+
value: T;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Generic interface for a local or remote Cache.
|
|
16
|
+
* @typeParam T - the type of the cache entries' values
|
|
17
|
+
*/
|
|
18
|
+
export interface Cache<T = unknown> {
|
|
19
|
+
/**
|
|
20
|
+
* Gets an item from the Cache.
|
|
21
|
+
*
|
|
22
|
+
* @param key - the key of the desired entry for which to fetch its value
|
|
23
|
+
* @returns a Promise of the entry's value, or undefined if not found / expired.
|
|
24
|
+
*/
|
|
25
|
+
get: (key: string) => Promise<T | undefined>;
|
|
26
|
+
/**
|
|
27
|
+
* Sets an item in the Cache.
|
|
28
|
+
*
|
|
29
|
+
* @param key - the key of the new entry
|
|
30
|
+
* @param value - the value of the new entry
|
|
31
|
+
* @param ttl - the time in milliseconds until the entry expires
|
|
32
|
+
* @returns an empty Promise that resolves when the entry has been set
|
|
33
|
+
*/
|
|
34
|
+
set: (key: string, value: T, ttl: number) => Promise<void>;
|
|
35
|
+
/**
|
|
36
|
+
* Sets a list of items in the Cache.
|
|
37
|
+
*
|
|
38
|
+
* @param entries - a list of cache entries
|
|
39
|
+
* @param ttl - the time in milliseconds until the entries expire
|
|
40
|
+
* @returns an empty Promise that resolves when all entries have been set
|
|
41
|
+
*/
|
|
42
|
+
setMany: (entries: CacheEntry<T>[], ttl: number) => Promise<void>;
|
|
43
|
+
/**
|
|
44
|
+
* Deletes the specified item from the Cache
|
|
45
|
+
*
|
|
46
|
+
* @param key - the key of the entry to be deleted
|
|
47
|
+
* @returns an empty Promise that resolves when the entry has been deleted
|
|
48
|
+
*/
|
|
49
|
+
delete: (key: string) => Promise<void>;
|
|
50
|
+
}
|
|
51
|
+
export declare const calculateCacheKey: ({ adapterEndpoint, adapterConfig, }: {
|
|
52
|
+
adapterEndpoint: AdapterEndpoint;
|
|
53
|
+
adapterConfig: AdapterConfig;
|
|
54
|
+
}, data: unknown) => string;
|
|
55
|
+
export declare const calculateFeedId: (adapterEndpoint: AdapterEndpoint, data: unknown) => string;
|
|
56
|
+
/**
|
|
57
|
+
* Calculates a unique key from the provided data.
|
|
58
|
+
*
|
|
59
|
+
* @param data - the request data/body, i.e. the adapter input params
|
|
60
|
+
* @param paramNames - the keys from adapter endpoint input parameters
|
|
61
|
+
* @returns the calculated unique key
|
|
62
|
+
*
|
|
63
|
+
* @example
|
|
64
|
+
* ```
|
|
65
|
+
* calculateKey({ base: 'ETH', quote: 'BTC' }, ['base','quote'])
|
|
66
|
+
* // equals `|base:eth|quote:btc`
|
|
67
|
+
* ```
|
|
68
|
+
*/
|
|
69
|
+
export declare const calculateKey: (data: unknown, paramNames: string[]) => string;
|
|
70
|
+
/**
|
|
71
|
+
* Polls the provided Cache for an AdapterResponse set in the provided key. If the maximum
|
|
72
|
+
* amount of retries is exceeded, it returns undefined instead.
|
|
73
|
+
*
|
|
74
|
+
* @param cache - a Cache instance
|
|
75
|
+
* @param key - the key generated from an AdapterRequest that corresponds to the desired AdapterResponse
|
|
76
|
+
* @param retry - current retry, only for internal use
|
|
77
|
+
* @returns the AdapterResponse if found, else undefined
|
|
78
|
+
*/
|
|
79
|
+
export declare const pollResponseFromCache: (cache: Cache<AdapterResponse>, key: string, options: {
|
|
80
|
+
maxRetries: number;
|
|
81
|
+
sleep: number;
|
|
82
|
+
}, retry?: number) => Promise<AdapterResponse | undefined>;
|
|
83
|
+
/**
|
|
84
|
+
* Given a Cache instance in the adapter dependencies, builds a middleware function that will perform
|
|
85
|
+
* a get from said Cache and return that if found; otherwise it'll continue the middleware chain.
|
|
86
|
+
*
|
|
87
|
+
* @param adapter - an initialized adapter
|
|
88
|
+
* @returns the cache middleware function
|
|
89
|
+
*/
|
|
90
|
+
export declare const buildCacheMiddleware: AdapterMiddlewareBuilder;
|
|
@@ -153,10 +153,14 @@ const buildCacheMiddleware = (adapter) => async (req, res) => {
|
|
|
153
153
|
if (response) {
|
|
154
154
|
logger.debug('Found response from cache, sending that');
|
|
155
155
|
if (adapter.config.METRICS_ENABLED && adapter.config.EXPERIMENTAL_METRICS_ENABLED) {
|
|
156
|
-
const label = cacheMetrics.cacheMetricsLabel(req.requestContext.cacheKey, req.requestContext.
|
|
157
|
-
// Record cache get count
|
|
156
|
+
const label = cacheMetrics.cacheMetricsLabel(req.requestContext.cacheKey, req.requestContext.meta?.metrics?.feedId || 'N/A', adapter.config.CACHE_TYPE);
|
|
157
|
+
// Record cache staleness and cache get count and value
|
|
158
158
|
const staleness = calculateStaleness(response.maxAge, adapter.config.CACHE_MAX_AGE);
|
|
159
159
|
cacheMetrics.cacheGet(label, response.result, staleness);
|
|
160
|
+
req.requestContext.meta = {
|
|
161
|
+
...req.requestContext.meta,
|
|
162
|
+
metrics: { ...req.requestContext.meta?.metrics, cacheHit: true },
|
|
163
|
+
};
|
|
160
164
|
}
|
|
161
165
|
return res.send(response);
|
|
162
166
|
}
|
package/cache/local.d.ts
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { Cache, CacheEntry } from './index';
|
|
2
|
+
/**
|
|
3
|
+
* Type for a value stored in a LocalCache entry.
|
|
4
|
+
*
|
|
5
|
+
* @typeParam T - the type for the entry's value
|
|
6
|
+
*/
|
|
7
|
+
export interface LocalCacheEntry<T> {
|
|
8
|
+
expirationTimestamp: number;
|
|
9
|
+
value: T;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Local implementation of a Cache. It uses a simple js Object, storing entries with both
|
|
13
|
+
* a value and an expiration timestamp. Expired entries are deleted on reads (i.e. no background gc/upkeep).
|
|
14
|
+
*
|
|
15
|
+
* @typeParam T - the type for the entries' values
|
|
16
|
+
*/
|
|
17
|
+
export declare class LocalCache<T = unknown> implements Cache<T> {
|
|
18
|
+
store: Record<string, LocalCacheEntry<T>>;
|
|
19
|
+
get(key: string): Promise<T | undefined>;
|
|
20
|
+
delete(key: string): Promise<void>;
|
|
21
|
+
set(key: string, value: T, ttl: number): Promise<void>;
|
|
22
|
+
setMany(entries: CacheEntry<T>[], ttl: number): Promise<void>;
|
|
23
|
+
}
|
|
@@ -66,7 +66,7 @@ class LocalCache {
|
|
|
66
66
|
expirationTimestamp: Date.now() + ttl,
|
|
67
67
|
};
|
|
68
68
|
// Only record metrics if feed Id is present, otherwise assuming value is not adapter response to record
|
|
69
|
-
const feedId = value.
|
|
69
|
+
const feedId = value.meta?.metrics?.feedId;
|
|
70
70
|
if (feedId) {
|
|
71
71
|
// Record cache set count, max age, and staleness (set to 0 for cache set)
|
|
72
72
|
const label = cacheMetrics.cacheMetricsLabel(key, feedId, cacheMetrics.CacheTypes.Local);
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import * as client from 'prom-client';
|
|
2
|
+
interface CacheMetricsLabels {
|
|
3
|
+
participant_id: string;
|
|
4
|
+
feed_id: string;
|
|
5
|
+
cache_type: string;
|
|
6
|
+
}
|
|
7
|
+
export declare const cacheGet: (label: CacheMetricsLabels, value: unknown, staleness: number) => void;
|
|
8
|
+
export declare const cacheSet: (label: CacheMetricsLabels, maxAge: number) => void;
|
|
9
|
+
export declare const cacheMetricsLabel: (cacheKey: string, feedId: string, cacheType: string) => {
|
|
10
|
+
participant_id: string;
|
|
11
|
+
feed_id: string;
|
|
12
|
+
cache_type: string;
|
|
13
|
+
};
|
|
14
|
+
export declare enum CacheTypes {
|
|
15
|
+
Redis = "redis",
|
|
16
|
+
Local = "local"
|
|
17
|
+
}
|
|
18
|
+
export declare enum CMD_SENT_STATUS {
|
|
19
|
+
TIMEOUT = "TIMEOUT",
|
|
20
|
+
FAIL = "FAIL",
|
|
21
|
+
SUCCESS = "SUCCESS"
|
|
22
|
+
}
|
|
23
|
+
export declare const redisConnectionsOpen: client.Counter<string>;
|
|
24
|
+
export declare const redisRetriesCount: client.Counter<string>;
|
|
25
|
+
export declare const redisCommandsSentCount: client.Counter<"status" | "function_name">;
|
|
26
|
+
export declare const cacheWarmerCount: client.Gauge<"isBatched">;
|
|
27
|
+
export {};
|
|
@@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
|
23
23
|
return result;
|
|
24
24
|
};
|
|
25
25
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.redisCommandsSentCount = exports.redisRetriesCount = exports.redisConnectionsOpen = exports.CMD_SENT_STATUS = exports.CacheTypes = exports.cacheMetricsLabel = exports.cacheSet = exports.cacheGet = void 0;
|
|
26
|
+
exports.cacheWarmerCount = exports.redisCommandsSentCount = exports.redisRetriesCount = exports.redisConnectionsOpen = exports.CMD_SENT_STATUS = exports.CacheTypes = exports.cacheMetricsLabel = exports.cacheSet = exports.cacheGet = void 0;
|
|
27
27
|
const client = __importStar(require("prom-client"));
|
|
28
28
|
const cacheGet = (label, value, staleness) => {
|
|
29
29
|
if (typeof value === 'number' || typeof value === 'string') {
|
|
@@ -112,3 +112,9 @@ exports.redisCommandsSentCount = new client.Counter({
|
|
|
112
112
|
help: 'The number of redis commands sent',
|
|
113
113
|
labelNames: ['status', 'function_name'],
|
|
114
114
|
});
|
|
115
|
+
// Cache Warmer Metrics
|
|
116
|
+
exports.cacheWarmerCount = new client.Gauge({
|
|
117
|
+
name: 'cache_warmer_get_count',
|
|
118
|
+
help: 'The number of cache warmers running',
|
|
119
|
+
labelNames: ['isBatched'],
|
|
120
|
+
});
|
package/cache/redis.d.ts
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import Redis from 'ioredis';
|
|
2
|
+
import { Cache, CacheEntry } from './index';
|
|
3
|
+
/**
|
|
4
|
+
* Redis implementation of a Cache. It uses a simple js Object, storing entries with both
|
|
5
|
+
* a value and an expiration timestamp. Expired entries are deleted on reads (i.e. no background gc/upkeep).
|
|
6
|
+
*
|
|
7
|
+
* @typeParam T - the type for the entries' values
|
|
8
|
+
*/
|
|
9
|
+
export declare class RedisCache<T = unknown> implements Cache<T> {
|
|
10
|
+
private client;
|
|
11
|
+
constructor(client: Redis);
|
|
12
|
+
get(key: string): Promise<T | undefined>;
|
|
13
|
+
delete(key: string): Promise<void>;
|
|
14
|
+
set(key: string, value: T, ttl: number): Promise<void>;
|
|
15
|
+
setMany(entries: CacheEntry<T>[], ttl: number): Promise<void>;
|
|
16
|
+
}
|
|
@@ -64,7 +64,7 @@ class RedisCache {
|
|
|
64
64
|
// TODO: Check for invalid stringify
|
|
65
65
|
await this.client.set(key, JSON.stringify(value), 'PX', ttl);
|
|
66
66
|
// Only record metrics if feed Id is present, otherwise assuming value is not adapter response to record
|
|
67
|
-
const feedId = value.
|
|
67
|
+
const feedId = value.meta?.metrics?.feedId;
|
|
68
68
|
if (feedId) {
|
|
69
69
|
// Record cache set count, max age, and staleness (set to 0 for cache set)
|
|
70
70
|
const label = cacheMetrics.cacheMetricsLabel(key, feedId, cacheMetrics.CacheTypes.Redis);
|
|
@@ -83,7 +83,7 @@ class RedisCache {
|
|
|
83
83
|
chain = chain.set(entry.key, JSON.stringify(entry.value), 'PX', ttl);
|
|
84
84
|
// TODO: Move to after error handling once implemented to avoid recording cache sets that failed
|
|
85
85
|
// Only record metrics if feed Id is present, otherwise assuming value is not adapter response to record
|
|
86
|
-
const feedId = entry.value.
|
|
86
|
+
const feedId = entry.value.meta?.metrics?.feedId;
|
|
87
87
|
if (feedId) {
|
|
88
88
|
// Record cache set count, max age, and staleness (set to 0 for cache set)
|
|
89
89
|
const label = cacheMetrics.cacheMetricsLabel(entry.key, feedId, cacheMetrics.CacheTypes.Redis);
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
export declare const BaseSettings: {
|
|
2
|
+
readonly API_KEY: {
|
|
3
|
+
readonly description: "Default setting for an EA key";
|
|
4
|
+
readonly type: "string";
|
|
5
|
+
};
|
|
6
|
+
readonly BASE_URL: {
|
|
7
|
+
readonly description: "Starting path for the EA handler endpoint";
|
|
8
|
+
readonly type: "string";
|
|
9
|
+
readonly default: "/";
|
|
10
|
+
};
|
|
11
|
+
readonly CACHE_MAX_AGE: {
|
|
12
|
+
readonly description: "Maximum amount of time (in ms) that a response will stay cached";
|
|
13
|
+
readonly type: "number";
|
|
14
|
+
readonly default: 90000;
|
|
15
|
+
};
|
|
16
|
+
readonly CACHE_REDIS_HOST: {
|
|
17
|
+
readonly description: "Hostname for the Redis instance to be used";
|
|
18
|
+
readonly type: "string";
|
|
19
|
+
readonly default: "127.0.0.1";
|
|
20
|
+
};
|
|
21
|
+
readonly CACHE_REDIS_PORT: {
|
|
22
|
+
readonly description: "Port for the Redis instance to be used";
|
|
23
|
+
readonly type: "number";
|
|
24
|
+
readonly default: 6379;
|
|
25
|
+
};
|
|
26
|
+
readonly CACHE_TYPE: {
|
|
27
|
+
readonly description: "The type of cache to use throughout the EA";
|
|
28
|
+
readonly type: "enum";
|
|
29
|
+
readonly default: "local";
|
|
30
|
+
readonly options: readonly ["local", "redis"];
|
|
31
|
+
};
|
|
32
|
+
readonly CORRELATION_ID_ENABLED: {
|
|
33
|
+
readonly description: "Flag to enable correlation IDs for sent requests in logging";
|
|
34
|
+
readonly type: "boolean";
|
|
35
|
+
readonly default: true;
|
|
36
|
+
};
|
|
37
|
+
readonly EA_PORT: {
|
|
38
|
+
readonly description: "Port through which the EA will listen for REST requests (if mode is set to \"reader\" or \"reader-writer\")";
|
|
39
|
+
readonly type: "number";
|
|
40
|
+
readonly default: 8080;
|
|
41
|
+
};
|
|
42
|
+
readonly EXPERIMENTAL_METRICS_ENABLED: {
|
|
43
|
+
readonly description: "Flag to specify whether or not to collect metrics. Used as fallback for METRICS_ENABLED";
|
|
44
|
+
readonly type: "boolean";
|
|
45
|
+
readonly default: true;
|
|
46
|
+
};
|
|
47
|
+
readonly LOG_LEVEL: {
|
|
48
|
+
readonly description: "Minimum level required for logs to be output";
|
|
49
|
+
readonly type: "string";
|
|
50
|
+
readonly default: "info";
|
|
51
|
+
};
|
|
52
|
+
readonly METRICS_ENABLED: {
|
|
53
|
+
readonly description: "Flag to specify whether or not to startup the metrics server";
|
|
54
|
+
readonly type: "boolean";
|
|
55
|
+
readonly default: true;
|
|
56
|
+
};
|
|
57
|
+
readonly METRICS_NAME: {
|
|
58
|
+
readonly description: "Metrics name";
|
|
59
|
+
readonly type: "string";
|
|
60
|
+
};
|
|
61
|
+
readonly METRICS_PORT: {
|
|
62
|
+
readonly description: "Port metrics will be exposed to";
|
|
63
|
+
readonly type: "number";
|
|
64
|
+
readonly default: 9080;
|
|
65
|
+
};
|
|
66
|
+
readonly METRICS_USE_BASE_URL: {
|
|
67
|
+
readonly description: "Flag to specify whether or not to prepend the BASE_URL to the metrics endpoint";
|
|
68
|
+
readonly type: "boolean";
|
|
69
|
+
};
|
|
70
|
+
readonly RATE_LIMIT_API_TIER: {
|
|
71
|
+
readonly description: "Rate limiting tier to use from the available options for the adapter. If not present, the adapter will run using the first tier on the list.";
|
|
72
|
+
readonly type: "string";
|
|
73
|
+
};
|
|
74
|
+
readonly WARMUP_SUBSCRIPTION_TTL: {
|
|
75
|
+
readonly type: "number";
|
|
76
|
+
readonly description: "";
|
|
77
|
+
readonly default: 10000;
|
|
78
|
+
};
|
|
79
|
+
readonly CACHE_POLLING_MAX_RETRIES: {
|
|
80
|
+
readonly description: "Max amount of times to attempt to find EA response in the cache after the Transport has been set up";
|
|
81
|
+
readonly type: "number";
|
|
82
|
+
readonly default: 10;
|
|
83
|
+
};
|
|
84
|
+
readonly CACHE_POLLING_SLEEP_MS: {
|
|
85
|
+
readonly description: "The number of ms to sleep between each retry to fetch the EA response in the cache";
|
|
86
|
+
readonly type: "number";
|
|
87
|
+
readonly default: 200;
|
|
88
|
+
};
|
|
89
|
+
readonly DEFAULT_CACHE_KEY: {
|
|
90
|
+
readonly description: "Default key to be used when one cannot be determined from request parameters";
|
|
91
|
+
readonly type: "string";
|
|
92
|
+
readonly default: "DEFAULT_CACHE_KEY";
|
|
93
|
+
};
|
|
94
|
+
readonly EA_HOST: {
|
|
95
|
+
readonly description: "Host this EA will listen for REST requests on (if mode is set to \"reader\" or \"reader-writer\")";
|
|
96
|
+
readonly type: "string";
|
|
97
|
+
readonly default: "::";
|
|
98
|
+
};
|
|
99
|
+
readonly EA_MODE: {
|
|
100
|
+
readonly description: "Port this EA will listen for REST requests on (if mode is set to \"reader\" or \"reader-writer\")";
|
|
101
|
+
readonly type: "enum";
|
|
102
|
+
readonly default: "reader-writer";
|
|
103
|
+
readonly options: readonly ["reader", "writer", "reader-writer"];
|
|
104
|
+
};
|
|
105
|
+
readonly REST_TRANSPORT_MAX_RATE_LIMIT_RETRIES: {
|
|
106
|
+
readonly description: "Maximum amount of times the Rest Transport will attempt to set up a request when blocked by the rate limiter";
|
|
107
|
+
readonly type: "number";
|
|
108
|
+
readonly default: 3;
|
|
109
|
+
};
|
|
110
|
+
readonly REST_TRANSPORT_MS_BETWEEN_RATE_LIMIT_RETRIES: {
|
|
111
|
+
readonly description: "Time that the Rest Transport will wait between retries when blocked by the rate limiter";
|
|
112
|
+
readonly type: "number";
|
|
113
|
+
readonly default: 400;
|
|
114
|
+
};
|
|
115
|
+
};
|
|
116
|
+
export declare const buildAdapterConfig: <CustomSettings extends CustomSettingsType<CustomSettings> = EmptySettings>({ overrides, customSettings, }: {
|
|
117
|
+
overrides?: Partial<BaseAdapterConfig> | undefined;
|
|
118
|
+
customSettings?: SettingsMap | undefined;
|
|
119
|
+
}) => AdapterConfig<CustomSettings>;
|
|
120
|
+
declare type SettingValueType = string | number | boolean;
|
|
121
|
+
declare type SettingType<C extends Setting> = C['type'] extends 'string' ? string : C['type'] extends 'number' ? number : C['type'] extends 'boolean' ? boolean : C['type'] extends 'enum' ? C['options'] extends readonly string[] ? C['options'][number] : never : never;
|
|
122
|
+
declare type BaseSettingsType = typeof BaseSettings;
|
|
123
|
+
export declare type Setting = {
|
|
124
|
+
type: 'string';
|
|
125
|
+
description: string;
|
|
126
|
+
options?: never;
|
|
127
|
+
default?: string;
|
|
128
|
+
validate?: (value?: string) => ValidationErrorMessage;
|
|
129
|
+
required?: false;
|
|
130
|
+
} | {
|
|
131
|
+
type: 'string';
|
|
132
|
+
description: string;
|
|
133
|
+
options?: never;
|
|
134
|
+
default?: string;
|
|
135
|
+
validate?: (value: string) => ValidationErrorMessage;
|
|
136
|
+
required: true;
|
|
137
|
+
} | {
|
|
138
|
+
type: 'number';
|
|
139
|
+
description: string;
|
|
140
|
+
options?: never;
|
|
141
|
+
default?: number;
|
|
142
|
+
validate?: (value?: number) => ValidationErrorMessage;
|
|
143
|
+
required?: false;
|
|
144
|
+
} | {
|
|
145
|
+
type: 'number';
|
|
146
|
+
description: string;
|
|
147
|
+
options?: never;
|
|
148
|
+
default?: number;
|
|
149
|
+
validate?: (value: number) => ValidationErrorMessage;
|
|
150
|
+
required: true;
|
|
151
|
+
} | {
|
|
152
|
+
type: 'boolean';
|
|
153
|
+
description: string;
|
|
154
|
+
options?: never;
|
|
155
|
+
default?: boolean;
|
|
156
|
+
validate?: (value?: boolean) => ValidationErrorMessage;
|
|
157
|
+
required?: false;
|
|
158
|
+
} | {
|
|
159
|
+
type: 'boolean';
|
|
160
|
+
description: string;
|
|
161
|
+
options?: never;
|
|
162
|
+
default?: boolean;
|
|
163
|
+
validate?: (value: boolean) => ValidationErrorMessage;
|
|
164
|
+
required: true;
|
|
165
|
+
} | {
|
|
166
|
+
type: 'enum';
|
|
167
|
+
description: string;
|
|
168
|
+
default?: string;
|
|
169
|
+
options: readonly string[];
|
|
170
|
+
validate?: (value?: string) => ValidationErrorMessage;
|
|
171
|
+
required?: false;
|
|
172
|
+
} | {
|
|
173
|
+
type: 'enum';
|
|
174
|
+
description: string;
|
|
175
|
+
default?: string;
|
|
176
|
+
options: readonly string[];
|
|
177
|
+
validate?: (value: string) => ValidationErrorMessage;
|
|
178
|
+
required: true;
|
|
179
|
+
};
|
|
180
|
+
export declare type AdapterConfigFromSettings<T extends SettingsMap> = {
|
|
181
|
+
-readonly [K in keyof T as T[K] extends {
|
|
182
|
+
default: SettingValueType;
|
|
183
|
+
} ? K : T[K]['required'] extends true ? K : never]: SettingType<T[K]>;
|
|
184
|
+
} & {
|
|
185
|
+
-readonly [K in keyof T as T[K] extends {
|
|
186
|
+
default: SettingValueType;
|
|
187
|
+
} ? never : T[K]['required'] extends true ? never : K]?: SettingType<T[K]> | undefined;
|
|
188
|
+
};
|
|
189
|
+
export declare type BaseAdapterConfig = AdapterConfigFromSettings<BaseSettingsType>;
|
|
190
|
+
export declare type AdapterConfig<T extends CustomSettingsType<T> = SettingsMap> = AdapterConfigFromSettings<T> & BaseAdapterConfig;
|
|
191
|
+
export declare type CustomSettingsType<T = SettingsMap> = Record<keyof T, Setting>;
|
|
192
|
+
export declare type EmptySettings = Record<string, never>;
|
|
193
|
+
export declare type SettingsMap = Record<string, Setting>;
|
|
194
|
+
export declare type ValidationErrorMessage = string | undefined;
|
|
195
|
+
export {};
|
|
@@ -5,10 +5,9 @@ exports.buildAdapterConfig = exports.BaseSettings = void 0;
|
|
|
5
5
|
exports.BaseSettings = {
|
|
6
6
|
// V2 compat
|
|
7
7
|
// TODO: Remove non used in v3 ones
|
|
8
|
-
API_ENDPOINT: {
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
},
|
|
8
|
+
// API_ENDPOINT: {
|
|
9
|
+
// type: 'string',
|
|
10
|
+
// },
|
|
12
11
|
API_KEY: {
|
|
13
12
|
description: 'Default setting for an EA key',
|
|
14
13
|
type: 'string',
|
|
@@ -215,8 +214,8 @@ exports.BaseSettings = {
|
|
|
215
214
|
// },
|
|
216
215
|
WARMUP_SUBSCRIPTION_TTL: {
|
|
217
216
|
type: 'number',
|
|
218
|
-
description: '
|
|
219
|
-
default:
|
|
217
|
+
description: '',
|
|
218
|
+
default: 10000,
|
|
220
219
|
},
|
|
221
220
|
// WARMUP_UNHEALTHY_THRESHOLD: {
|
|
222
221
|
// type: 'number',
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
export declare const DEFAULT_MINUTE_RATE_LIMIT = 60;
|
|
2
|
+
export declare const BURST_UNDEFINED_QUOTA_MULTIPLE = 2;
|
|
3
|
+
export declare const DEFAULT_WS_CONNECTIONS = 2;
|
|
4
|
+
export declare const DEFAULT_WS_SUBSCRIPTIONS = 10;
|
|
5
|
+
declare type RateLimitTimeFrame = 'rateLimit1s' | 'rateLimit1m' | 'rateLimit1h';
|
|
6
|
+
declare type HTTPTier = {
|
|
7
|
+
rateLimit1s?: number;
|
|
8
|
+
rateLimit1m?: number;
|
|
9
|
+
rateLimit1h?: number;
|
|
10
|
+
note?: string;
|
|
11
|
+
};
|
|
12
|
+
declare type WSTier = {
|
|
13
|
+
connections: number;
|
|
14
|
+
subscriptions: number;
|
|
15
|
+
};
|
|
16
|
+
export interface Limits {
|
|
17
|
+
http: {
|
|
18
|
+
[tierName: string]: HTTPTier;
|
|
19
|
+
};
|
|
20
|
+
ws: {
|
|
21
|
+
[tierName: string]: WSTier;
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
interface ProviderRateLimit {
|
|
25
|
+
second: number;
|
|
26
|
+
minute: number;
|
|
27
|
+
}
|
|
28
|
+
export declare const getHTTPLimit: (provider: string, limits: Limits, tier: string, timeframe: RateLimitTimeFrame) => number;
|
|
29
|
+
export declare const getRateLimit: (provider: string, limits: Limits, tier: string) => ProviderRateLimit;
|
|
30
|
+
export declare const getWSLimits: (provider: string, limits: Limits, tier: string) => WSTier;
|
|
31
|
+
export {};
|
|
@@ -42,7 +42,7 @@ const getProviderLimits = (provider, limits, tier, protocol) => {
|
|
|
42
42
|
return limitsConfig;
|
|
43
43
|
};
|
|
44
44
|
const parseLimits = (limits) => {
|
|
45
|
-
const _mapObject = (fn) => (o) => Object.fromEntries(
|
|
45
|
+
const _mapObject = (fn) => (o) => Object.fromEntries([]);
|
|
46
46
|
const _formatProtocol = _mapObject((entry) => {
|
|
47
47
|
const [tierName, rest] = entry;
|
|
48
48
|
return [tierName.toLowerCase(), { ...rest }];
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export declare const customSettings: {
|
|
2
|
+
readonly USERNAME: {
|
|
3
|
+
readonly description: "Username for the NCFX API";
|
|
4
|
+
readonly type: "string";
|
|
5
|
+
readonly required: true;
|
|
6
|
+
};
|
|
7
|
+
readonly PASSWORD: {
|
|
8
|
+
readonly description: "Password for the NCFX API";
|
|
9
|
+
readonly type: "string";
|
|
10
|
+
readonly required: true;
|
|
11
|
+
};
|
|
12
|
+
};
|
|
File without changes
|
|
File without changes
|