@chainlink/external-adapter-framework 0.0.7 → 0.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adapter.d.ts +88 -0
- package/{dist/src/adapter.js → adapter.js} +0 -0
- package/background-executor.d.ts +11 -0
- package/{dist/background-executor.js → background-executor.js} +0 -0
- package/cache/factory.d.ts +6 -0
- package/{dist/cache → cache}/factory.js +0 -0
- package/cache/index.d.ts +90 -0
- package/{dist/src/cache → cache}/index.js +6 -2
- package/cache/local.d.ts +23 -0
- package/{dist/cache → cache}/local.js +1 -1
- package/cache/metrics.d.ts +27 -0
- package/{dist/cache → cache}/metrics.js +7 -1
- package/cache/redis.d.ts +16 -0
- package/{dist/cache → cache}/redis.js +2 -2
- package/config/index.d.ts +195 -0
- package/{dist/src/config → config}/index.js +5 -6
- package/config/provider-limits.d.ts +31 -0
- package/{dist/src/config → config}/provider-limits.js +1 -1
- package/examples/coingecko/batch-warming.d.ts +2 -0
- package/{dist/examples → examples}/coingecko/batch-warming.js +0 -0
- package/examples/coingecko/index.d.ts +2 -0
- package/{dist/examples → examples}/coingecko/index.js +0 -0
- package/examples/coingecko/rest.d.ts +2 -0
- package/{dist/examples → examples}/coingecko/rest.js +0 -0
- package/examples/ncfx/config/index.d.ts +12 -0
- package/{dist/examples → examples}/ncfx/config/index.js +0 -0
- package/examples/ncfx/index.d.ts +2 -0
- package/{dist/examples → examples}/ncfx/index.js +0 -0
- package/examples/ncfx/websocket.d.ts +36 -0
- package/{dist/examples → examples}/ncfx/websocket.js +0 -0
- package/index.d.ts +12 -0
- package/{dist/index.js → index.js} +3 -0
- package/metrics/constants.d.ts +16 -0
- package/{dist/src/metrics → metrics}/constants.js +1 -1
- package/metrics/index.d.ts +15 -0
- package/{dist/metrics → metrics}/index.js +48 -1
- package/metrics/util.d.ts +3 -0
- package/{dist/metrics → metrics}/util.js +0 -0
- package/package.json +2 -6
- package/rate-limiting/background/fixed-frequency.d.ts +10 -0
- package/{dist/src/rate-limiting → rate-limiting}/background/fixed-frequency.js +0 -0
- package/rate-limiting/index.d.ts +54 -0
- package/{dist/src/rate-limiting → rate-limiting}/index.js +0 -0
- package/rate-limiting/metrics.d.ts +3 -0
- package/{dist/src/rate-limiting → rate-limiting}/metrics.js +14 -2
- package/rate-limiting/request/simple-counting.d.ts +20 -0
- package/{dist/src/rate-limiting → rate-limiting}/request/simple-counting.js +0 -0
- package/test.d.ts +1 -0
- package/{dist/test.js → test.js} +0 -0
- package/transports/batch-warming.d.ts +34 -0
- package/{dist/src/transports → transports}/batch-warming.js +47 -1
- package/transports/index.d.ts +87 -0
- package/{dist/src/transports → transports}/index.js +5 -3
- package/transports/metrics.d.ts +21 -0
- package/{dist/src/transports → transports}/metrics.js +28 -42
- package/transports/rest.d.ts +43 -0
- package/{dist/src/transports → transports}/rest.js +38 -2
- package/transports/util.d.ts +8 -0
- package/{dist/src/transports → transports}/util.js +16 -16
- package/transports/websocket.d.ts +80 -0
- package/{dist/src/transports → transports}/websocket.js +8 -14
- package/util/expiring-sorted-set.d.ts +21 -0
- package/{dist/src/util → util}/expiring-sorted-set.js +0 -0
- package/util/index.d.ts +11 -0
- package/{dist/src/util → util}/index.js +0 -0
- package/util/logger.d.ts +42 -0
- package/{dist/src/util → util}/logger.js +0 -0
- package/util/request.d.ts +55 -0
- package/{dist/src/util → util}/request.js +0 -0
- package/validation/error.d.ts +50 -0
- package/validation/error.js +79 -0
- package/validation/index.d.ts +5 -0
- package/{dist/src/validation → validation}/index.js +8 -6
- package/validation/input-params.d.ts +15 -0
- package/{dist/src/validation → validation}/input-params.js +0 -0
- package/validation/override-functions.d.ts +3 -0
- package/{dist/src/validation → validation}/override-functions.js +0 -0
- package/{dist/src/validation → validation}/preset-tokens.json +0 -0
- package/validation/validator.d.ts +47 -0
- package/{dist/src/validation → validation}/validator.js +0 -0
- package/README.md +0 -103
- package/dist/adapter.js +0 -60
- package/dist/cache/index.js +0 -163
- package/dist/config/index.js +0 -364
- package/dist/config/provider-limits.js +0 -75
- package/dist/metrics/constants.js +0 -25
- package/dist/rate-limiting/factory.js +0 -33
- package/dist/rate-limiting/index.js +0 -36
- package/dist/rate-limiting/metrics.js +0 -32
- package/dist/rate-limiting/nop-limiter.js +0 -15
- package/dist/rate-limiting/simple-counting.js +0 -61
- package/dist/src/background-executor.js +0 -45
- package/dist/src/cache/factory.js +0 -57
- package/dist/src/cache/local.js +0 -83
- package/dist/src/cache/metrics.js +0 -114
- package/dist/src/cache/redis.js +0 -100
- package/dist/src/examples/bank-frick/accounts.js +0 -191
- package/dist/src/examples/bank-frick/config/index.js +0 -45
- package/dist/src/examples/bank-frick/index.js +0 -14
- package/dist/src/examples/bank-frick/util.js +0 -39
- package/dist/src/examples/coingecko/batch-warming.js +0 -52
- package/dist/src/examples/coingecko/index.js +0 -10
- package/dist/src/examples/coingecko/rest.js +0 -50
- package/dist/src/examples/ncfx/config/index.js +0 -15
- package/dist/src/examples/ncfx/index.js +0 -10
- package/dist/src/examples/ncfx/websocket.js +0 -72
- package/dist/src/index.js +0 -89
- package/dist/src/metrics/index.js +0 -76
- package/dist/src/metrics/util.js +0 -9
- package/dist/src/test.js +0 -6
- package/dist/src/validation/error.js +0 -41
- package/dist/transports/batch-warming.js +0 -57
- package/dist/transports/index.js +0 -76
- package/dist/transports/metrics.js +0 -133
- package/dist/transports/rest.js +0 -91
- package/dist/transports/util.js +0 -85
- package/dist/transports/websocket.js +0 -171
- package/dist/util/expiring-sorted-set.js +0 -47
- package/dist/util/index.js +0 -35
- package/dist/util/logger.js +0 -62
- package/dist/util/request.js +0 -2
- package/dist/validation/error.js +0 -41
- package/dist/validation/index.js +0 -82
- package/dist/validation/input-params.js +0 -30
- package/dist/validation/overrideFunctions.js +0 -42
- package/dist/validation/presetTokens.json +0 -23
- package/dist/validation/validator.js +0 -303
package/dist/config/index.js
DELETED
|
@@ -1,364 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
// Import { getRandomRequiredEnv, getRandomEnv, getEnv } from '../util'
|
|
3
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
-
exports.buildAdapterConfig = exports.BaseSettings = void 0;
|
|
5
|
-
exports.BaseSettings = {
|
|
6
|
-
// V2 compat
|
|
7
|
-
// TODO: Remove non used in v3 ones
|
|
8
|
-
// API_ENDPOINT: {
|
|
9
|
-
// type: 'string',
|
|
10
|
-
// },
|
|
11
|
-
API_KEY: {
|
|
12
|
-
description: 'Default setting for an EA key',
|
|
13
|
-
type: 'string',
|
|
14
|
-
},
|
|
15
|
-
// API_TIMEOUT: {
|
|
16
|
-
// type: 'number',
|
|
17
|
-
// default: 30000,
|
|
18
|
-
// },
|
|
19
|
-
// API_VERBOSE: {
|
|
20
|
-
// type: 'boolean',
|
|
21
|
-
// },
|
|
22
|
-
BASE_URL: {
|
|
23
|
-
description: 'Starting path for the EA handler endpoint',
|
|
24
|
-
type: 'string',
|
|
25
|
-
default: '/',
|
|
26
|
-
},
|
|
27
|
-
// CACHE_ENABLED: {
|
|
28
|
-
// type: 'boolean',
|
|
29
|
-
// default: true,
|
|
30
|
-
// },
|
|
31
|
-
// CACHE_KEY_GROUP: {
|
|
32
|
-
// type: 'string',
|
|
33
|
-
// },
|
|
34
|
-
CACHE_MAX_AGE: {
|
|
35
|
-
description: 'Maximum amount of time (in ms) that a response will stay cached',
|
|
36
|
-
type: 'number',
|
|
37
|
-
default: 90000,
|
|
38
|
-
},
|
|
39
|
-
// CACHE_MAX_ITEMS: {
|
|
40
|
-
// type: 'number',
|
|
41
|
-
// default: 1000,
|
|
42
|
-
// },
|
|
43
|
-
// CACHE_MIN_AGE: {
|
|
44
|
-
// type: 'number',
|
|
45
|
-
// default: 30000,
|
|
46
|
-
// },
|
|
47
|
-
// CACHE_REDIS_CONNECTION_TIMEOUT: {
|
|
48
|
-
// type: 'number',
|
|
49
|
-
// default: 15000,
|
|
50
|
-
// },
|
|
51
|
-
CACHE_REDIS_HOST: {
|
|
52
|
-
description: 'Hostname for the Redis instance to be used',
|
|
53
|
-
type: 'string',
|
|
54
|
-
default: '127.0.0.1',
|
|
55
|
-
},
|
|
56
|
-
// CACHE_REDIS_MAX_QUEUED_ITEMS: {
|
|
57
|
-
// type: 'number',
|
|
58
|
-
// default: 100,
|
|
59
|
-
// },
|
|
60
|
-
// CACHE_REDIS_MAX_RECONNECT_COOLDOWN: {
|
|
61
|
-
// type: 'number',
|
|
62
|
-
// default: 3000,
|
|
63
|
-
// },
|
|
64
|
-
// CACHE_REDIS_PASSWORD: {
|
|
65
|
-
// type: 'string',
|
|
66
|
-
// },
|
|
67
|
-
// CACHE_REDIS_PATH: {
|
|
68
|
-
// type: 'string',
|
|
69
|
-
// },
|
|
70
|
-
CACHE_REDIS_PORT: {
|
|
71
|
-
description: 'Port for the Redis instance to be used',
|
|
72
|
-
type: 'number',
|
|
73
|
-
default: 6379,
|
|
74
|
-
},
|
|
75
|
-
// CACHE_REDIS_TIMEOUT: {
|
|
76
|
-
// type: 'number',
|
|
77
|
-
// default: 500,
|
|
78
|
-
// },
|
|
79
|
-
// CACHE_REDIS_URL: {
|
|
80
|
-
// type: 'string',
|
|
81
|
-
// },
|
|
82
|
-
CACHE_TYPE: {
|
|
83
|
-
description: 'The type of cache to use throughout the EA',
|
|
84
|
-
type: 'enum',
|
|
85
|
-
default: 'local',
|
|
86
|
-
options: ['local', 'redis'],
|
|
87
|
-
},
|
|
88
|
-
// CACHE_UPDATE_AGE_ON_GET: {
|
|
89
|
-
// type: 'boolean',
|
|
90
|
-
// default: false,
|
|
91
|
-
// },
|
|
92
|
-
CORRELATION_ID_ENABLED: {
|
|
93
|
-
description: 'Flag to enable correlation IDs for sent requests in logging',
|
|
94
|
-
type: 'boolean',
|
|
95
|
-
default: true,
|
|
96
|
-
},
|
|
97
|
-
// DEBUG: {
|
|
98
|
-
// type: 'boolean',
|
|
99
|
-
// },
|
|
100
|
-
// DEFAULT_WS_HEARTBEAT_INTERVAL: {
|
|
101
|
-
// type: 'number',
|
|
102
|
-
// default: 30000,
|
|
103
|
-
// },
|
|
104
|
-
EA_PORT: {
|
|
105
|
-
description: 'Port through which the EA will listen for REST requests (if mode is set to "reader" or "reader-writer")',
|
|
106
|
-
type: 'number',
|
|
107
|
-
default: 8080,
|
|
108
|
-
},
|
|
109
|
-
// ENV_ADAPTER_URL: {
|
|
110
|
-
// type: 'string',
|
|
111
|
-
// },
|
|
112
|
-
// ERROR_CAPACITY: {
|
|
113
|
-
// type: 'number',
|
|
114
|
-
// },
|
|
115
|
-
EXPERIMENTAL_METRICS_ENABLED: {
|
|
116
|
-
description: 'Flag to specify whether or not to collect metrics. Used as fallback for METRICS_ENABLED',
|
|
117
|
-
type: 'boolean',
|
|
118
|
-
default: true
|
|
119
|
-
},
|
|
120
|
-
// LEGACY_ENV_ADAPTER_URL: {
|
|
121
|
-
// type: 'string',
|
|
122
|
-
// },
|
|
123
|
-
LOG_LEVEL: {
|
|
124
|
-
description: 'Minimum level required for logs to be output',
|
|
125
|
-
type: 'string',
|
|
126
|
-
default: 'info',
|
|
127
|
-
},
|
|
128
|
-
METRICS_ENABLED: {
|
|
129
|
-
description: 'Flag to specify whether or not to startup the metrics server',
|
|
130
|
-
type: 'boolean',
|
|
131
|
-
default: true
|
|
132
|
-
},
|
|
133
|
-
METRICS_NAME: {
|
|
134
|
-
description: 'Metrics name',
|
|
135
|
-
type: 'string',
|
|
136
|
-
},
|
|
137
|
-
METRICS_PORT: {
|
|
138
|
-
description: 'Port metrics will be exposed to',
|
|
139
|
-
type: 'number',
|
|
140
|
-
default: 9080,
|
|
141
|
-
},
|
|
142
|
-
METRICS_USE_BASE_URL: {
|
|
143
|
-
description: 'Flag to specify whether or not to prepend the BASE_URL to the metrics endpoint',
|
|
144
|
-
type: 'boolean',
|
|
145
|
-
},
|
|
146
|
-
RATE_LIMIT_API_TIER: {
|
|
147
|
-
description: 'Rate limiting tier to use from the available options for the adapter. If not present, the adapter will run using the first tier on the list.',
|
|
148
|
-
type: 'string',
|
|
149
|
-
},
|
|
150
|
-
// RATE_LIMIT_CAPACITY_MINUTE: {
|
|
151
|
-
// type: 'number',
|
|
152
|
-
// },
|
|
153
|
-
// RATE_LIMIT_CAPACITY_SECOND: {
|
|
154
|
-
// type: 'number',
|
|
155
|
-
// },
|
|
156
|
-
// RATE_LIMIT_CAPACITY: {
|
|
157
|
-
// type: 'number',
|
|
158
|
-
// },
|
|
159
|
-
// RATE_LIMIT_ENABLED: {
|
|
160
|
-
// type: 'boolean',
|
|
161
|
-
// default: true,
|
|
162
|
-
// },
|
|
163
|
-
// RECORD: {
|
|
164
|
-
// type: 'boolean',
|
|
165
|
-
// },
|
|
166
|
-
// REQUEST_COALESCING_ENABLED: {
|
|
167
|
-
// type: 'boolean',
|
|
168
|
-
// },
|
|
169
|
-
// REQUEST_COALESCING_ENTROPY_MAX: {
|
|
170
|
-
// type: 'number',
|
|
171
|
-
// default: 0,
|
|
172
|
-
// },
|
|
173
|
-
// REQUEST_COALESCING_INTERVAL_COEFFICIENT: {
|
|
174
|
-
// type: 'number',
|
|
175
|
-
// default: 2,
|
|
176
|
-
// },
|
|
177
|
-
// REQUEST_COALESCING_INTERVAL_MAX: {
|
|
178
|
-
// type: 'number',
|
|
179
|
-
// default: 1000,
|
|
180
|
-
// },
|
|
181
|
-
// REQUEST_COALESCING_INTERVAL: {
|
|
182
|
-
// type: 'number',
|
|
183
|
-
// default: 100,
|
|
184
|
-
// },
|
|
185
|
-
// REQUEST_COALESCING_MAX_RETRIES: {
|
|
186
|
-
// type: 'number',
|
|
187
|
-
// default: 5,
|
|
188
|
-
// },
|
|
189
|
-
// RETRY: {
|
|
190
|
-
// type: 'boolean',
|
|
191
|
-
// default: 1,
|
|
192
|
-
// },
|
|
193
|
-
// SERVER_RATE_LIMIT_MAX: {
|
|
194
|
-
// type: 'number',
|
|
195
|
-
// default: 250,
|
|
196
|
-
// },
|
|
197
|
-
// SERVER_SLOW_DOWN_AFTER_FACTOR: {
|
|
198
|
-
// type: 'number',
|
|
199
|
-
// default: 0.8,
|
|
200
|
-
// },
|
|
201
|
-
// SERVER_SLOW_DOWN_DELAY_MS: {
|
|
202
|
-
// type: 'number',
|
|
203
|
-
// default: 500,
|
|
204
|
-
// },
|
|
205
|
-
// TIMEOUT: {
|
|
206
|
-
// type: 'number',
|
|
207
|
-
// },
|
|
208
|
-
// WARMUP_ENABLED: {
|
|
209
|
-
// type: 'boolean',
|
|
210
|
-
// default: true,
|
|
211
|
-
// },
|
|
212
|
-
// WARMUP_INTERVAL: {
|
|
213
|
-
// type: 'number',
|
|
214
|
-
// },
|
|
215
|
-
// WARMUP_SUBSCRIPTION_TTL: {
|
|
216
|
-
// type: 'number',
|
|
217
|
-
// default: false,
|
|
218
|
-
// },
|
|
219
|
-
// WARMUP_UNHEALTHY_THRESHOLD: {
|
|
220
|
-
// type: 'number',
|
|
221
|
-
// default: 3,
|
|
222
|
-
// },
|
|
223
|
-
// WS_API_ENDPOINT: {
|
|
224
|
-
// type: 'string',
|
|
225
|
-
// },
|
|
226
|
-
// WS_API_KEY: {
|
|
227
|
-
// type: 'string',
|
|
228
|
-
// },
|
|
229
|
-
// WS_CONNECTION_KEY: {
|
|
230
|
-
// type: 'string',
|
|
231
|
-
// default: 1,
|
|
232
|
-
// },
|
|
233
|
-
// WS_CONNECTION_LIMIT: {
|
|
234
|
-
// type: 'number',
|
|
235
|
-
// default: 1,
|
|
236
|
-
// },
|
|
237
|
-
// WS_CONNECTION_RETRY_DELAY: {
|
|
238
|
-
// type: 'number',
|
|
239
|
-
// default: 1000,
|
|
240
|
-
// },
|
|
241
|
-
// WS_CONNECTION_RETRY_LIMIT: {
|
|
242
|
-
// type: 'number',
|
|
243
|
-
// default: 3,
|
|
244
|
-
// },
|
|
245
|
-
// WS_CONNECTION_TTL: {
|
|
246
|
-
// type: 'number',
|
|
247
|
-
// default: 70000,
|
|
248
|
-
// },
|
|
249
|
-
// WS_ENABLED: {
|
|
250
|
-
// type: 'boolean',
|
|
251
|
-
// default: false,
|
|
252
|
-
// },
|
|
253
|
-
// WS_SUBSCRIPTION_LIMIT: {
|
|
254
|
-
// type: 'number',
|
|
255
|
-
// default: 10,
|
|
256
|
-
// },
|
|
257
|
-
// WS_SUBSCRIPTION_PRIORITY_LIST: {
|
|
258
|
-
// type: 'string',
|
|
259
|
-
// },
|
|
260
|
-
// WS_SUBSCRIPTION_TTL: {
|
|
261
|
-
// type: 'number',
|
|
262
|
-
// default: 120000,
|
|
263
|
-
// },
|
|
264
|
-
// WS_SUBSCRIPTION_UNRESPONSIVE_TTL: {
|
|
265
|
-
// type: 'number',
|
|
266
|
-
// default: false,
|
|
267
|
-
// },
|
|
268
|
-
// WS_TIME_UNTIL_HANDLE_NEXT_MESSAGE_OVERRIDE: {
|
|
269
|
-
// type: 'number',
|
|
270
|
-
// },
|
|
271
|
-
// V3
|
|
272
|
-
CACHE_POLLING_MAX_RETRIES: {
|
|
273
|
-
description: 'Max amount of times to attempt to find EA response in the cache after the Transport has been set up',
|
|
274
|
-
type: 'number',
|
|
275
|
-
default: 10,
|
|
276
|
-
},
|
|
277
|
-
CACHE_POLLING_SLEEP_MS: {
|
|
278
|
-
description: 'The number of ms to sleep between each retry to fetch the EA response in the cache',
|
|
279
|
-
type: 'number',
|
|
280
|
-
default: 200,
|
|
281
|
-
},
|
|
282
|
-
DEFAULT_CACHE_KEY: {
|
|
283
|
-
description: 'Default key to be used when one cannot be determined from request parameters',
|
|
284
|
-
type: 'string',
|
|
285
|
-
default: 'DEFAULT_CACHE_KEY',
|
|
286
|
-
},
|
|
287
|
-
EA_HOST: {
|
|
288
|
-
description: 'Host this EA will listen for REST requests on (if mode is set to "reader" or "reader-writer")',
|
|
289
|
-
type: 'string',
|
|
290
|
-
default: '::',
|
|
291
|
-
},
|
|
292
|
-
EA_MODE: {
|
|
293
|
-
description: 'Port this EA will listen for REST requests on (if mode is set to "reader" or "reader-writer")',
|
|
294
|
-
type: 'enum',
|
|
295
|
-
default: 'reader-writer',
|
|
296
|
-
options: ['reader', 'writer', 'reader-writer'],
|
|
297
|
-
},
|
|
298
|
-
REST_TRANSPORT_MAX_RATE_LIMIT_RETRIES: {
|
|
299
|
-
description: 'Maximum amount of times the Rest Transport will attempt to set up a request when blocked by the rate limiter',
|
|
300
|
-
type: 'number',
|
|
301
|
-
default: 3,
|
|
302
|
-
},
|
|
303
|
-
REST_TRANSPORT_MS_BETWEEN_RATE_LIMIT_RETRIES: {
|
|
304
|
-
description: 'Time that the Rest Transport will wait between retries when blocked by the rate limiter',
|
|
305
|
-
type: 'number',
|
|
306
|
-
default: 400,
|
|
307
|
-
},
|
|
308
|
-
// CACHE_KEY_IGNORED_PROPS : {
|
|
309
|
-
// description: 'Properties to ignore when generating a feed ID for requests',
|
|
310
|
-
// type: 'string'
|
|
311
|
-
// }
|
|
312
|
-
};
|
|
313
|
-
const buildAdapterConfig = ({ overrides = {}, customSettings = {}, }) => {
|
|
314
|
-
const validationErrors = [];
|
|
315
|
-
const vars = {};
|
|
316
|
-
// Iterate base adapter env vars
|
|
317
|
-
for (const [key, config] of Object.entries(exports.BaseSettings)) {
|
|
318
|
-
const value = getEnv(key, config) ?? overrides?.[key] ?? config.default;
|
|
319
|
-
vars[key] = value;
|
|
320
|
-
}
|
|
321
|
-
// Iterate custom vars
|
|
322
|
-
for (const [key, config] of Object.entries(customSettings)) {
|
|
323
|
-
if (exports.BaseSettings[key]) {
|
|
324
|
-
throw new Error(`Custom env var "${key}" declared, but a base framework env var with that name already exists.`);
|
|
325
|
-
}
|
|
326
|
-
const value = getEnv(key, config) ?? config.default;
|
|
327
|
-
// Check if a required setting has been provided
|
|
328
|
-
if (config.required && value === null) {
|
|
329
|
-
validationErrors.push(`${key}: Value is required, but none was provided`);
|
|
330
|
-
}
|
|
331
|
-
else if (config.validate) {
|
|
332
|
-
// Then check if the setting has custom validation
|
|
333
|
-
const validationRes = config.validate(value);
|
|
334
|
-
if (validationRes) {
|
|
335
|
-
validationErrors.push(`${key}: ${validationRes}`);
|
|
336
|
-
}
|
|
337
|
-
}
|
|
338
|
-
vars[key] = value;
|
|
339
|
-
}
|
|
340
|
-
if (validationErrors.length > 0) {
|
|
341
|
-
throw new Error(`Validation failed for the following variables:\n ${validationErrors.join('\n')}`);
|
|
342
|
-
}
|
|
343
|
-
return vars;
|
|
344
|
-
};
|
|
345
|
-
exports.buildAdapterConfig = buildAdapterConfig;
|
|
346
|
-
const getEnv = (name, config) => {
|
|
347
|
-
const value = process.env[name];
|
|
348
|
-
if (!value) {
|
|
349
|
-
return null;
|
|
350
|
-
}
|
|
351
|
-
switch (config.type) {
|
|
352
|
-
case 'string':
|
|
353
|
-
return value;
|
|
354
|
-
case 'number':
|
|
355
|
-
return parseInt(value);
|
|
356
|
-
case 'boolean':
|
|
357
|
-
return value === 'true';
|
|
358
|
-
case 'enum':
|
|
359
|
-
if (!config.options?.includes(value)) {
|
|
360
|
-
throw new Error(`Env var "${name}" has value "${value}" which is not included in the valid options (${config.options})`);
|
|
361
|
-
}
|
|
362
|
-
return value;
|
|
363
|
-
}
|
|
364
|
-
};
|
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.getWSLimits = exports.getRateLimit = exports.getHTTPLimit = exports.DEFAULT_WS_SUBSCRIPTIONS = exports.DEFAULT_WS_CONNECTIONS = exports.BURST_UNDEFINED_QUOTA_MULTIPLE = exports.DEFAULT_MINUTE_RATE_LIMIT = void 0;
|
|
4
|
-
const util_1 = require("../util");
|
|
5
|
-
exports.DEFAULT_MINUTE_RATE_LIMIT = 60;
|
|
6
|
-
exports.BURST_UNDEFINED_QUOTA_MULTIPLE = 2;
|
|
7
|
-
exports.DEFAULT_WS_CONNECTIONS = 2;
|
|
8
|
-
exports.DEFAULT_WS_SUBSCRIPTIONS = 10;
|
|
9
|
-
const logger = (0, util_1.makeLogger)('ProviderLimits');
|
|
10
|
-
const getHTTPLimit = (provider, limits, tier, timeframe) => {
|
|
11
|
-
const providerLimit = getProviderLimits(provider, limits, tier, 'http');
|
|
12
|
-
return providerLimit?.[timeframe] || 0;
|
|
13
|
-
};
|
|
14
|
-
exports.getHTTPLimit = getHTTPLimit;
|
|
15
|
-
const getRateLimit = (provider, limits, tier) => {
|
|
16
|
-
const providerLimit = getProviderLimits(provider, limits, tier, 'http');
|
|
17
|
-
return calculateRateLimit(providerLimit);
|
|
18
|
-
};
|
|
19
|
-
exports.getRateLimit = getRateLimit;
|
|
20
|
-
const getWSLimits = (provider, limits, tier) => {
|
|
21
|
-
const providerLimit = getProviderLimits(provider, limits, tier, 'ws');
|
|
22
|
-
return calculateWSLimits(providerLimit);
|
|
23
|
-
};
|
|
24
|
-
exports.getWSLimits = getWSLimits;
|
|
25
|
-
const getProviderLimits = (provider, limits, tier, protocol) => {
|
|
26
|
-
const providerConfig = parseLimits(limits);
|
|
27
|
-
if (!providerConfig) {
|
|
28
|
-
throw new Error(`Rate Limit: Provider: "${provider}" doesn't match any provider spec in limits.json`);
|
|
29
|
-
}
|
|
30
|
-
const protocolConfig = providerConfig[protocol];
|
|
31
|
-
if (!protocolConfig) {
|
|
32
|
-
throw new Error(`Rate Limit: "${provider}" doesn't have any configuration for ${protocol} in limits.json`);
|
|
33
|
-
}
|
|
34
|
-
let limitsConfig = protocolConfig[tier.toLowerCase()];
|
|
35
|
-
if (!limitsConfig) {
|
|
36
|
-
logger.debug(`Rate Limit: "${provider} does not have tier ${tier} defined. Falling back to lowest tier"`);
|
|
37
|
-
limitsConfig = Object.values(protocolConfig)?.[0];
|
|
38
|
-
}
|
|
39
|
-
if (!limitsConfig) {
|
|
40
|
-
throw new Error(`Rate Limit: Provider: "${provider}" has no tiers defined for ${protocol} in limits.json`);
|
|
41
|
-
}
|
|
42
|
-
return limitsConfig;
|
|
43
|
-
};
|
|
44
|
-
const parseLimits = (limits) => {
|
|
45
|
-
const _mapObject = (fn) => (o) => Object.fromEntries(Object.entries(o).map(fn));
|
|
46
|
-
const _formatProtocol = _mapObject((entry) => {
|
|
47
|
-
const [tierName, rest] = entry;
|
|
48
|
-
return [tierName.toLowerCase(), { ...rest }];
|
|
49
|
-
});
|
|
50
|
-
const _formatProvider = (ls) => {
|
|
51
|
-
const http = _formatProtocol(ls.http);
|
|
52
|
-
const ws = _formatProtocol(ls?.ws);
|
|
53
|
-
return { http, ws };
|
|
54
|
-
};
|
|
55
|
-
return _formatProvider(limits);
|
|
56
|
-
};
|
|
57
|
-
const calculateWSLimits = (providerLimit) => {
|
|
58
|
-
return {
|
|
59
|
-
connections: providerLimit.connections,
|
|
60
|
-
subscriptions: providerLimit.subscriptions,
|
|
61
|
-
};
|
|
62
|
-
};
|
|
63
|
-
const calculateRateLimit = (providerLimit) => {
|
|
64
|
-
let quota = providerLimit.rateLimit1m;
|
|
65
|
-
if (!quota && providerLimit?.rateLimit1h) {
|
|
66
|
-
quota = providerLimit?.rateLimit1h / 60;
|
|
67
|
-
}
|
|
68
|
-
else if (!quota && providerLimit?.rateLimit1s) {
|
|
69
|
-
quota = providerLimit?.rateLimit1s * 60;
|
|
70
|
-
}
|
|
71
|
-
return {
|
|
72
|
-
second: providerLimit?.rateLimit1s || (quota / 60) * exports.BURST_UNDEFINED_QUOTA_MULTIPLE,
|
|
73
|
-
minute: quota,
|
|
74
|
-
};
|
|
75
|
-
};
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.requestDurationBuckets = exports.MAX_FEED_ID_LENGTH = exports.HttpRequestType = void 0;
|
|
4
|
-
var HttpRequestType;
|
|
5
|
-
(function (HttpRequestType) {
|
|
6
|
-
HttpRequestType["CACHE_HIT"] = "cacheHit";
|
|
7
|
-
HttpRequestType["DATA_PROVIDER_HIT"] = "dataProviderHit";
|
|
8
|
-
HttpRequestType["ADAPTER_ERROR"] = "adapterError";
|
|
9
|
-
HttpRequestType["INPUT_ERROR"] = "inputError";
|
|
10
|
-
HttpRequestType["RATE_LIMIT_ERROR"] = "rateLimitError";
|
|
11
|
-
// BURST_LIMIT_ERROR = 'burstLimitError',
|
|
12
|
-
// BACKOFF_ERROR = 'backoffError',
|
|
13
|
-
HttpRequestType["DP_ERROR"] = "dataProviderError";
|
|
14
|
-
HttpRequestType["TIMEOUT_ERROR"] = "timeoutError";
|
|
15
|
-
// CONNECTION_ERROR = 'connectionError',
|
|
16
|
-
// RES_EMPTY_ERROR = 'responseEmptyError',
|
|
17
|
-
// RES_INVALID_ERROR = 'responseInvalidError',
|
|
18
|
-
HttpRequestType["CUSTOM_ERROR"] = "customError";
|
|
19
|
-
})(HttpRequestType = exports.HttpRequestType || (exports.HttpRequestType = {}));
|
|
20
|
-
/**
|
|
21
|
-
* Maxiumum number of characters that a feedId can contain.
|
|
22
|
-
*/
|
|
23
|
-
exports.MAX_FEED_ID_LENGTH = 300;
|
|
24
|
-
// We should tune these as we collect data, this is the default bucket distribution that prom comes with
|
|
25
|
-
exports.requestDurationBuckets = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10];
|
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.RateLimiterFactory = void 0;
|
|
4
|
-
const _1 = require(".");
|
|
5
|
-
const util_1 = require("../util");
|
|
6
|
-
const logger = (0, util_1.makeLogger)('RateLimiterFactory');
|
|
7
|
-
class RateLimiterFactory {
|
|
8
|
-
static buildRateLimiter(tiers, selectedTier, strategy = _1.RateLimitingStrategy.NOP_RATE_LIMITER) {
|
|
9
|
-
const RateLimiterCtor = _1.RateLimiterMap[strategy];
|
|
10
|
-
let tier;
|
|
11
|
-
if (tiers) {
|
|
12
|
-
// Check that if the tiers object is defined, it has values
|
|
13
|
-
if (Object.values(tiers).length === 0) {
|
|
14
|
-
throw new Error(`The tiers object is defined, but has no entries`);
|
|
15
|
-
}
|
|
16
|
-
// Check that the tier set in the AdapterConfig is a valid one
|
|
17
|
-
if (selectedTier && !tiers[selectedTier]) {
|
|
18
|
-
const validTiersString = Object.keys(tiers)
|
|
19
|
-
.map((t) => `"${t}"`)
|
|
20
|
-
.join(', ');
|
|
21
|
-
throw new Error(`The selected rate limit tier "${selectedTier}" is not valid (can be one of ${validTiersString})`);
|
|
22
|
-
}
|
|
23
|
-
// If no selected tier is present, use the first one
|
|
24
|
-
// TODO: Add logic to select lowest throughput one?
|
|
25
|
-
tier = selectedTier ? tiers[selectedTier] : Object.values(tiers)[0];
|
|
26
|
-
}
|
|
27
|
-
const rateLimiter = new RateLimiterCtor();
|
|
28
|
-
rateLimiter.initialize(tier);
|
|
29
|
-
logger.info(`Initialized rate limiter using "${strategy}" strategy.`);
|
|
30
|
-
return rateLimiter;
|
|
31
|
-
}
|
|
32
|
-
}
|
|
33
|
-
exports.RateLimiterFactory = RateLimiterFactory;
|
|
@@ -1,36 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
-
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
-
};
|
|
16
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
exports.RateLimiterMap = exports.RateLimitingStrategy = void 0;
|
|
18
|
-
const nop_limiter_1 = require("./nop-limiter");
|
|
19
|
-
const simple_counting_1 = require("./simple-counting");
|
|
20
|
-
__exportStar(require("./simple-counting"), exports);
|
|
21
|
-
__exportStar(require("./factory"), exports);
|
|
22
|
-
/**
|
|
23
|
-
* Enum to easily specify rate limiting strategy in the Adapter config
|
|
24
|
-
*/
|
|
25
|
-
var RateLimitingStrategy;
|
|
26
|
-
(function (RateLimitingStrategy) {
|
|
27
|
-
RateLimitingStrategy["SIMPLE_COUNTING_LIMITER"] = "SIMPLE_COUNTING_LIMITER";
|
|
28
|
-
RateLimitingStrategy["NOP_RATE_LIMITER"] = "NOP_RATE_LIMITER";
|
|
29
|
-
})(RateLimitingStrategy = exports.RateLimitingStrategy || (exports.RateLimitingStrategy = {}));
|
|
30
|
-
/**
|
|
31
|
-
* Record to map a limiting strategy to the implementing class
|
|
32
|
-
*/
|
|
33
|
-
exports.RateLimiterMap = {
|
|
34
|
-
[RateLimitingStrategy.SIMPLE_COUNTING_LIMITER]: simple_counting_1.SimpleCountingRateLimiter,
|
|
35
|
-
[RateLimitingStrategy.NOP_RATE_LIMITER]: nop_limiter_1.NopRateLimiter,
|
|
36
|
-
};
|
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.rateLimitCreditsSpentTotal = void 0;
|
|
27
|
-
const client = __importStar(require("prom-client"));
|
|
28
|
-
exports.rateLimitCreditsSpentTotal = new client.Counter({
|
|
29
|
-
name: 'rate_limit_credits_spent_total',
|
|
30
|
-
help: 'The number of data provider credits the adapter is consuming',
|
|
31
|
-
labelNames: ['participant_id', 'feed_id', 'experimental'],
|
|
32
|
-
});
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.NopRateLimiter = void 0;
|
|
4
|
-
/**
|
|
5
|
-
* Default rate limiter that will basically do nothing, always allow requests through
|
|
6
|
-
*/
|
|
7
|
-
class NopRateLimiter {
|
|
8
|
-
initialize() {
|
|
9
|
-
return;
|
|
10
|
-
}
|
|
11
|
-
isUnderLimits() {
|
|
12
|
-
return true;
|
|
13
|
-
}
|
|
14
|
-
}
|
|
15
|
-
exports.NopRateLimiter = NopRateLimiter;
|
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.SimpleCountingRateLimiter = void 0;
|
|
4
|
-
const util_1 = require("../util");
|
|
5
|
-
const logger = (0, util_1.makeLogger)('SimpleCountingRateLimiter');
|
|
6
|
-
/**
|
|
7
|
-
* This rate limiter is the simplest stateful option.
|
|
8
|
-
* On startup, it'll compare the different thresholds for each tier, calculate them all
|
|
9
|
-
* in the finest window we'll use (seconds), and use the most restrictive one.
|
|
10
|
-
* This is so if the EA were to restart, we don't need to worry about persisting state
|
|
11
|
-
* for things like daily quotas. The downside is that this does not work well for bursty
|
|
12
|
-
* loads or spikes, in cases where e.g. the per second limit is high but daily quotas low.
|
|
13
|
-
*/
|
|
14
|
-
class SimpleCountingRateLimiter {
|
|
15
|
-
constructor() {
|
|
16
|
-
this.latestSecondInterval = 0;
|
|
17
|
-
this.requestsThisSecond = 0;
|
|
18
|
-
this.latestMinuteInterval = 0;
|
|
19
|
-
this.requestsThisMinute = 0;
|
|
20
|
-
}
|
|
21
|
-
initialize(limits) {
|
|
22
|
-
// Translate the hourly limit into reqs per minute
|
|
23
|
-
const perHourLimit = (limits?.rateLimit1h || Infinity) / 60;
|
|
24
|
-
this.perMinuteLimit = Math.min(limits?.rateLimit1m || Infinity, perHourLimit);
|
|
25
|
-
this.perSecondLimit = limits?.rateLimit1s || Infinity;
|
|
26
|
-
logger.debug(`Using rate limiting settings: perMinute = ${this.perMinuteLimit} | perSecond: = ${this.perSecondLimit}`);
|
|
27
|
-
}
|
|
28
|
-
isUnderLimits() {
|
|
29
|
-
// If the limit is set to infinity, there was no tier limit specified
|
|
30
|
-
if (this.perSecondLimit === Infinity && this.perMinuteLimit === Infinity) {
|
|
31
|
-
return true;
|
|
32
|
-
}
|
|
33
|
-
const now = Date.now();
|
|
34
|
-
const nearestSecondInterval = Math.floor(now / 1000);
|
|
35
|
-
const nearestMinuteInterval = Math.floor(now / (1000 * 60));
|
|
36
|
-
// This should always run to completion, even if it doesn't look atomic; therefore the
|
|
37
|
-
// Ops should be "thread safe". Thank JS and its infinite single threaded dumbness.
|
|
38
|
-
if (nearestSecondInterval !== this.latestSecondInterval) {
|
|
39
|
-
logger.trace(`Clearing latest second interval, # of requests logged was: ${this.requestsThisSecond} `);
|
|
40
|
-
this.latestSecondInterval = nearestSecondInterval;
|
|
41
|
-
this.requestsThisSecond = 0;
|
|
42
|
-
}
|
|
43
|
-
if (nearestMinuteInterval !== this.latestMinuteInterval) {
|
|
44
|
-
logger.trace(`Clearing latest second minute, # of requests logged was: ${this.requestsThisMinute} `);
|
|
45
|
-
this.latestMinuteInterval = nearestMinuteInterval;
|
|
46
|
-
this.requestsThisMinute = 0;
|
|
47
|
-
}
|
|
48
|
-
if (this.requestsThisSecond < this.perSecondLimit &&
|
|
49
|
-
this.requestsThisMinute < this.perMinuteLimit) {
|
|
50
|
-
logger.trace('Request under limits, counting +1');
|
|
51
|
-
this.requestsThisSecond++;
|
|
52
|
-
this.requestsThisMinute++;
|
|
53
|
-
return true;
|
|
54
|
-
}
|
|
55
|
-
else {
|
|
56
|
-
logger.trace('Requests seen this interval are above limits');
|
|
57
|
-
return false;
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
exports.SimpleCountingRateLimiter = SimpleCountingRateLimiter;
|