@chainlink/external-adapter-framework 0.0.7 → 0.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adapter.d.ts +88 -0
- package/{dist/src/adapter.js → adapter.js} +0 -0
- package/background-executor.d.ts +11 -0
- package/{dist/background-executor.js → background-executor.js} +0 -0
- package/cache/factory.d.ts +6 -0
- package/{dist/cache → cache}/factory.js +0 -0
- package/cache/index.d.ts +90 -0
- package/{dist/src/cache → cache}/index.js +6 -2
- package/cache/local.d.ts +23 -0
- package/{dist/cache → cache}/local.js +1 -1
- package/cache/metrics.d.ts +27 -0
- package/{dist/cache → cache}/metrics.js +7 -1
- package/cache/redis.d.ts +16 -0
- package/{dist/cache → cache}/redis.js +2 -2
- package/config/index.d.ts +195 -0
- package/{dist/src/config → config}/index.js +5 -6
- package/config/provider-limits.d.ts +31 -0
- package/{dist/src/config → config}/provider-limits.js +1 -1
- package/examples/coingecko/batch-warming.d.ts +2 -0
- package/{dist/examples → examples}/coingecko/batch-warming.js +0 -0
- package/examples/coingecko/index.d.ts +2 -0
- package/{dist/examples → examples}/coingecko/index.js +0 -0
- package/examples/coingecko/rest.d.ts +2 -0
- package/{dist/examples → examples}/coingecko/rest.js +0 -0
- package/examples/ncfx/config/index.d.ts +12 -0
- package/{dist/examples → examples}/ncfx/config/index.js +0 -0
- package/examples/ncfx/index.d.ts +2 -0
- package/{dist/examples → examples}/ncfx/index.js +0 -0
- package/examples/ncfx/websocket.d.ts +36 -0
- package/{dist/examples → examples}/ncfx/websocket.js +0 -0
- package/index.d.ts +12 -0
- package/{dist/index.js → index.js} +3 -0
- package/metrics/constants.d.ts +16 -0
- package/{dist/src/metrics → metrics}/constants.js +1 -1
- package/metrics/index.d.ts +15 -0
- package/{dist/metrics → metrics}/index.js +48 -1
- package/metrics/util.d.ts +3 -0
- package/{dist/metrics → metrics}/util.js +0 -0
- package/package.json +2 -6
- package/rate-limiting/background/fixed-frequency.d.ts +10 -0
- package/{dist/src/rate-limiting → rate-limiting}/background/fixed-frequency.js +0 -0
- package/rate-limiting/index.d.ts +54 -0
- package/{dist/src/rate-limiting → rate-limiting}/index.js +0 -0
- package/rate-limiting/metrics.d.ts +3 -0
- package/{dist/src/rate-limiting → rate-limiting}/metrics.js +14 -2
- package/rate-limiting/request/simple-counting.d.ts +20 -0
- package/{dist/src/rate-limiting → rate-limiting}/request/simple-counting.js +0 -0
- package/test.d.ts +1 -0
- package/{dist/test.js → test.js} +0 -0
- package/transports/batch-warming.d.ts +34 -0
- package/{dist/src/transports → transports}/batch-warming.js +47 -1
- package/transports/index.d.ts +87 -0
- package/{dist/src/transports → transports}/index.js +5 -3
- package/transports/metrics.d.ts +21 -0
- package/{dist/src/transports → transports}/metrics.js +28 -42
- package/transports/rest.d.ts +43 -0
- package/{dist/src/transports → transports}/rest.js +38 -2
- package/transports/util.d.ts +8 -0
- package/{dist/src/transports → transports}/util.js +16 -16
- package/transports/websocket.d.ts +80 -0
- package/{dist/src/transports → transports}/websocket.js +8 -14
- package/util/expiring-sorted-set.d.ts +21 -0
- package/{dist/src/util → util}/expiring-sorted-set.js +0 -0
- package/util/index.d.ts +11 -0
- package/{dist/src/util → util}/index.js +0 -0
- package/util/logger.d.ts +42 -0
- package/{dist/src/util → util}/logger.js +0 -0
- package/util/request.d.ts +55 -0
- package/{dist/src/util → util}/request.js +0 -0
- package/validation/error.d.ts +50 -0
- package/validation/error.js +79 -0
- package/validation/index.d.ts +5 -0
- package/{dist/src/validation → validation}/index.js +8 -6
- package/validation/input-params.d.ts +15 -0
- package/{dist/src/validation → validation}/input-params.js +0 -0
- package/validation/override-functions.d.ts +3 -0
- package/{dist/src/validation → validation}/override-functions.js +0 -0
- package/{dist/src/validation → validation}/preset-tokens.json +0 -0
- package/validation/validator.d.ts +47 -0
- package/{dist/src/validation → validation}/validator.js +0 -0
- package/README.md +0 -103
- package/dist/adapter.js +0 -60
- package/dist/cache/index.js +0 -163
- package/dist/config/index.js +0 -364
- package/dist/config/provider-limits.js +0 -75
- package/dist/metrics/constants.js +0 -25
- package/dist/rate-limiting/factory.js +0 -33
- package/dist/rate-limiting/index.js +0 -36
- package/dist/rate-limiting/metrics.js +0 -32
- package/dist/rate-limiting/nop-limiter.js +0 -15
- package/dist/rate-limiting/simple-counting.js +0 -61
- package/dist/src/background-executor.js +0 -45
- package/dist/src/cache/factory.js +0 -57
- package/dist/src/cache/local.js +0 -83
- package/dist/src/cache/metrics.js +0 -114
- package/dist/src/cache/redis.js +0 -100
- package/dist/src/examples/bank-frick/accounts.js +0 -191
- package/dist/src/examples/bank-frick/config/index.js +0 -45
- package/dist/src/examples/bank-frick/index.js +0 -14
- package/dist/src/examples/bank-frick/util.js +0 -39
- package/dist/src/examples/coingecko/batch-warming.js +0 -52
- package/dist/src/examples/coingecko/index.js +0 -10
- package/dist/src/examples/coingecko/rest.js +0 -50
- package/dist/src/examples/ncfx/config/index.js +0 -15
- package/dist/src/examples/ncfx/index.js +0 -10
- package/dist/src/examples/ncfx/websocket.js +0 -72
- package/dist/src/index.js +0 -89
- package/dist/src/metrics/index.js +0 -76
- package/dist/src/metrics/util.js +0 -9
- package/dist/src/test.js +0 -6
- package/dist/src/validation/error.js +0 -41
- package/dist/transports/batch-warming.js +0 -57
- package/dist/transports/index.js +0 -76
- package/dist/transports/metrics.js +0 -133
- package/dist/transports/rest.js +0 -91
- package/dist/transports/util.js +0 -85
- package/dist/transports/websocket.js +0 -171
- package/dist/util/expiring-sorted-set.js +0 -47
- package/dist/util/index.js +0 -35
- package/dist/util/logger.js +0 -62
- package/dist/util/request.js +0 -2
- package/dist/validation/error.js +0 -41
- package/dist/validation/index.js +0 -82
- package/dist/validation/input-params.js +0 -30
- package/dist/validation/overrideFunctions.js +0 -42
- package/dist/validation/presetTokens.json +0 -23
- package/dist/validation/validator.js +0 -303
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.callBackgroundExecutes = void 0;
|
|
4
|
-
const util_1 = require("./util");
|
|
5
|
-
const logger = (0, util_1.makeLogger)('BackgroundExecutor');
|
|
6
|
-
/**
|
|
7
|
-
* Very simple background loop that will call the [[Transport.backgroundExecute]] functions in all Transports.
|
|
8
|
-
* It gets the time in ms to wait as the return value from those functions, and sleeps until next execution.
|
|
9
|
-
*
|
|
10
|
-
* @param adapter - an initialized External Adapter
|
|
11
|
-
* @param server - the http server to attach an on close listener to
|
|
12
|
-
*/
|
|
13
|
-
async function callBackgroundExecutes(adapter, server) {
|
|
14
|
-
// Set up variable to check later on to see if we need to stop this background "thread"
|
|
15
|
-
// If no server is provided, the listener won't be set and serverClosed will always be false
|
|
16
|
-
let serverClosed = false;
|
|
17
|
-
server?.on('close', () => {
|
|
18
|
-
serverClosed = true;
|
|
19
|
-
});
|
|
20
|
-
for (const endpoint of adapter.endpoints) {
|
|
21
|
-
const backgroundExecute = endpoint.transport.backgroundExecute?.bind(endpoint.transport);
|
|
22
|
-
if (!backgroundExecute) {
|
|
23
|
-
logger.debug(`Endpoint "${endpoint.name}" has no background execute, skipping...`);
|
|
24
|
-
continue;
|
|
25
|
-
}
|
|
26
|
-
const context = {
|
|
27
|
-
adapterEndpoint: endpoint,
|
|
28
|
-
adapterConfig: adapter.config,
|
|
29
|
-
};
|
|
30
|
-
const handler = async () => {
|
|
31
|
-
if (serverClosed) {
|
|
32
|
-
logger.info('Server closed, stopping recursive backgroundExecute handler chain');
|
|
33
|
-
return;
|
|
34
|
-
}
|
|
35
|
-
logger.debug(`Calling background execute for endpoint "${endpoint.name}"`);
|
|
36
|
-
const timeToWait = await backgroundExecute(context);
|
|
37
|
-
logger.debug(`Finished background execute for endpoint "${endpoint.name}", sleeping for ${timeToWait}ms`);
|
|
38
|
-
await (0, util_1.sleep)(timeToWait);
|
|
39
|
-
handler();
|
|
40
|
-
};
|
|
41
|
-
// Start recursive async calls
|
|
42
|
-
handler();
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
exports.callBackgroundExecutes = callBackgroundExecutes;
|
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
26
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
|
-
};
|
|
28
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
-
exports.CacheFactory = void 0;
|
|
30
|
-
const ioredis_1 = __importDefault(require("ioredis"));
|
|
31
|
-
const util_1 = require("../util");
|
|
32
|
-
const local_1 = require("./local");
|
|
33
|
-
const cacheMetrics = __importStar(require("./metrics"));
|
|
34
|
-
const redis_1 = require("./redis");
|
|
35
|
-
const logger = (0, util_1.makeLogger)('CacheFactory');
|
|
36
|
-
class CacheFactory {
|
|
37
|
-
static buildCache(config) {
|
|
38
|
-
logger.info(`Using "${config.CACHE_TYPE}" cache.`);
|
|
39
|
-
if (config.CACHE_TYPE === 'local') {
|
|
40
|
-
return new local_1.LocalCache();
|
|
41
|
-
}
|
|
42
|
-
else if (config.CACHE_TYPE === 'redis') {
|
|
43
|
-
const redis = new ioredis_1.default({
|
|
44
|
-
enableAutoPipelining: true,
|
|
45
|
-
host: config.CACHE_REDIS_HOST,
|
|
46
|
-
port: config.CACHE_REDIS_PORT,
|
|
47
|
-
});
|
|
48
|
-
redis.on('connect', () => {
|
|
49
|
-
cacheMetrics.redisConnectionsOpen.inc();
|
|
50
|
-
});
|
|
51
|
-
// TODO: Maybe track active redis connections instead of just total count of connections created
|
|
52
|
-
// Use a Gauge and track with a combo of connect/end events listeners
|
|
53
|
-
return new redis_1.RedisCache(redis);
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
exports.CacheFactory = CacheFactory;
|
package/dist/src/cache/local.js
DELETED
|
@@ -1,83 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.LocalCache = void 0;
|
|
27
|
-
const util_1 = require("../util");
|
|
28
|
-
const cacheMetrics = __importStar(require("./metrics"));
|
|
29
|
-
const logger = (0, util_1.makeLogger)('LocalCache');
|
|
30
|
-
/**
|
|
31
|
-
* Local implementation of a Cache. It uses a simple js Object, storing entries with both
|
|
32
|
-
* a value and an expiration timestamp. Expired entries are deleted on reads (i.e. no background gc/upkeep).
|
|
33
|
-
*
|
|
34
|
-
* @typeParam T - the type for the entries' values
|
|
35
|
-
*/
|
|
36
|
-
class LocalCache {
|
|
37
|
-
constructor() {
|
|
38
|
-
this.store = {};
|
|
39
|
-
}
|
|
40
|
-
async get(key) {
|
|
41
|
-
logger.trace(`Getting key ${key}`);
|
|
42
|
-
const entry = this.store[key];
|
|
43
|
-
if (!entry) {
|
|
44
|
-
logger.debug(`No entry in local cache for key "${key}", returning undefined`);
|
|
45
|
-
return undefined;
|
|
46
|
-
}
|
|
47
|
-
const expired = entry.expirationTimestamp <= Date.now();
|
|
48
|
-
if (expired) {
|
|
49
|
-
logger.debug('Entry in local cache expired, deleting and returning undefined');
|
|
50
|
-
this.delete(key);
|
|
51
|
-
return undefined;
|
|
52
|
-
}
|
|
53
|
-
else {
|
|
54
|
-
logger.debug('Found valid entry in local cache, returning value');
|
|
55
|
-
return entry.value;
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
async delete(key) {
|
|
59
|
-
logger.trace(`Deleting key ${key}`);
|
|
60
|
-
delete this.store[key]; // Deletes are slower than ignoring or setting null, fyi
|
|
61
|
-
}
|
|
62
|
-
async set(key, value, ttl) {
|
|
63
|
-
logger.trace(`Setting key ${key} with ttl ${ttl}`);
|
|
64
|
-
this.store[key] = {
|
|
65
|
-
value,
|
|
66
|
-
expirationTimestamp: Date.now() + ttl,
|
|
67
|
-
};
|
|
68
|
-
// Only record metrics if feed Id is present, otherwise assuming value is not adapter response to record
|
|
69
|
-
const feedId = value.metricsMeta?.feedId;
|
|
70
|
-
if (feedId) {
|
|
71
|
-
// Record cache set count, max age, and staleness (set to 0 for cache set)
|
|
72
|
-
const label = cacheMetrics.cacheMetricsLabel(key, feedId, cacheMetrics.CacheTypes.Local);
|
|
73
|
-
cacheMetrics.cacheSet(label, ttl);
|
|
74
|
-
}
|
|
75
|
-
}
|
|
76
|
-
async setMany(entries, ttl) {
|
|
77
|
-
logger.trace(`Setting a bunch of keys with ttl ${ttl}`);
|
|
78
|
-
for (const { key, value } of entries) {
|
|
79
|
-
this.set(key, value, ttl);
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
exports.LocalCache = LocalCache;
|
|
@@ -1,114 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.redisCommandsSentCount = exports.redisRetriesCount = exports.redisConnectionsOpen = exports.CMD_SENT_STATUS = exports.CacheTypes = exports.cacheMetricsLabel = exports.cacheSet = exports.cacheGet = void 0;
|
|
27
|
-
const client = __importStar(require("prom-client"));
|
|
28
|
-
const cacheGet = (label, value, staleness) => {
|
|
29
|
-
if (typeof value === 'number' || typeof value === 'string') {
|
|
30
|
-
const parsedValue = Number(value);
|
|
31
|
-
if (!Number.isNaN(parsedValue) && Number.isFinite(parsedValue)) {
|
|
32
|
-
cacheDataGetValues.labels(label).set(parsedValue);
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
cacheDataGetCount.labels(label).inc();
|
|
36
|
-
cacheDataStalenessSeconds.labels(label).set(staleness);
|
|
37
|
-
};
|
|
38
|
-
exports.cacheGet = cacheGet;
|
|
39
|
-
const cacheSet = (label, maxAge) => {
|
|
40
|
-
cacheDataSetCount.labels(label).inc();
|
|
41
|
-
cacheDataMaxAge.labels(label).set(maxAge);
|
|
42
|
-
cacheDataStalenessSeconds.labels(label).set(0);
|
|
43
|
-
};
|
|
44
|
-
exports.cacheSet = cacheSet;
|
|
45
|
-
const cacheMetricsLabel = (cacheKey, feedId, cacheType) => ({
|
|
46
|
-
participant_id: cacheKey,
|
|
47
|
-
feed_id: feedId,
|
|
48
|
-
cache_type: cacheType,
|
|
49
|
-
});
|
|
50
|
-
exports.cacheMetricsLabel = cacheMetricsLabel;
|
|
51
|
-
var CacheTypes;
|
|
52
|
-
(function (CacheTypes) {
|
|
53
|
-
CacheTypes["Redis"] = "redis";
|
|
54
|
-
CacheTypes["Local"] = "local";
|
|
55
|
-
})(CacheTypes = exports.CacheTypes || (exports.CacheTypes = {}));
|
|
56
|
-
var CMD_SENT_STATUS;
|
|
57
|
-
(function (CMD_SENT_STATUS) {
|
|
58
|
-
CMD_SENT_STATUS["TIMEOUT"] = "TIMEOUT";
|
|
59
|
-
CMD_SENT_STATUS["FAIL"] = "FAIL";
|
|
60
|
-
CMD_SENT_STATUS["SUCCESS"] = "SUCCESS";
|
|
61
|
-
})(CMD_SENT_STATUS = exports.CMD_SENT_STATUS || (exports.CMD_SENT_STATUS = {}));
|
|
62
|
-
const baseLabels = [
|
|
63
|
-
'feed_id',
|
|
64
|
-
'participant_id',
|
|
65
|
-
'cache_type',
|
|
66
|
-
'is_from_ws',
|
|
67
|
-
'experimental',
|
|
68
|
-
];
|
|
69
|
-
// Skipping this metrics for v3
|
|
70
|
-
// const cache_execution_duration_seconds = new client.Histogram({
|
|
71
|
-
// name: 'cache_execution_duration_seconds',
|
|
72
|
-
// help: 'A histogram bucket of the distribution of cache execution durations',
|
|
73
|
-
// labelNames: [...baseLabels, 'cache_hit'] as const,
|
|
74
|
-
// buckets: [0.01, 0.1, 1, 10],
|
|
75
|
-
// })
|
|
76
|
-
const cacheDataGetCount = new client.Counter({
|
|
77
|
-
name: 'cache_data_get_count',
|
|
78
|
-
help: 'A counter that increments every time a value is fetched from the cache',
|
|
79
|
-
labelNames: baseLabels,
|
|
80
|
-
});
|
|
81
|
-
const cacheDataGetValues = new client.Gauge({
|
|
82
|
-
name: 'cache_data_get_values',
|
|
83
|
-
help: 'A gauge keeping track of values being fetched from cache',
|
|
84
|
-
labelNames: baseLabels,
|
|
85
|
-
});
|
|
86
|
-
const cacheDataMaxAge = new client.Gauge({
|
|
87
|
-
name: 'cache_data_max_age',
|
|
88
|
-
help: 'A gauge tracking the max age of stored values in the cache',
|
|
89
|
-
labelNames: baseLabels,
|
|
90
|
-
});
|
|
91
|
-
const cacheDataSetCount = new client.Counter({
|
|
92
|
-
name: 'cache_data_set_count',
|
|
93
|
-
help: 'A counter that increments every time a value is set to the cache',
|
|
94
|
-
labelNames: [...baseLabels, 'status_code'],
|
|
95
|
-
});
|
|
96
|
-
const cacheDataStalenessSeconds = new client.Gauge({
|
|
97
|
-
name: 'cache_data_staleness_seconds',
|
|
98
|
-
help: 'Observes the staleness of the data returned',
|
|
99
|
-
labelNames: baseLabels,
|
|
100
|
-
});
|
|
101
|
-
// Redis Metrics
|
|
102
|
-
exports.redisConnectionsOpen = new client.Counter({
|
|
103
|
-
name: 'redis_connections_open',
|
|
104
|
-
help: 'The number of redis connections that are open',
|
|
105
|
-
});
|
|
106
|
-
exports.redisRetriesCount = new client.Counter({
|
|
107
|
-
name: 'redis_retries_count',
|
|
108
|
-
help: 'The number of retries that have been made to establish a redis connection',
|
|
109
|
-
});
|
|
110
|
-
exports.redisCommandsSentCount = new client.Counter({
|
|
111
|
-
name: 'redis_commands_sent_count',
|
|
112
|
-
help: 'The number of redis commands sent',
|
|
113
|
-
labelNames: ['status', 'function_name'],
|
|
114
|
-
});
|
package/dist/src/cache/redis.js
DELETED
|
@@ -1,100 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.RedisCache = void 0;
|
|
27
|
-
const util_1 = require("../util");
|
|
28
|
-
const cacheMetrics = __importStar(require("./metrics"));
|
|
29
|
-
const logger = (0, util_1.makeLogger)('RedisCache');
|
|
30
|
-
/**
|
|
31
|
-
* Redis implementation of a Cache. It uses a simple js Object, storing entries with both
|
|
32
|
-
* a value and an expiration timestamp. Expired entries are deleted on reads (i.e. no background gc/upkeep).
|
|
33
|
-
*
|
|
34
|
-
* @typeParam T - the type for the entries' values
|
|
35
|
-
*/
|
|
36
|
-
// TODO: Add error handling for redis command failures
|
|
37
|
-
class RedisCache {
|
|
38
|
-
constructor(client) {
|
|
39
|
-
this.client = client;
|
|
40
|
-
}
|
|
41
|
-
async get(key) {
|
|
42
|
-
logger.trace(`Getting key ${key}`);
|
|
43
|
-
const value = await this.client.get(key);
|
|
44
|
-
// Record get command sent to Redis
|
|
45
|
-
cacheMetrics.redisCommandsSentCount
|
|
46
|
-
.labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'get' })
|
|
47
|
-
.inc();
|
|
48
|
-
if (!value) {
|
|
49
|
-
logger.debug(`No entry in redis cache for key "${key}", returning undefined`);
|
|
50
|
-
return undefined;
|
|
51
|
-
}
|
|
52
|
-
return JSON.parse(value); // TODO: Check for invalid parsing
|
|
53
|
-
}
|
|
54
|
-
async delete(key) {
|
|
55
|
-
logger.trace(`Deleting key ${key}`);
|
|
56
|
-
await this.client.del(key);
|
|
57
|
-
// Record delete command sent to Redis
|
|
58
|
-
cacheMetrics.redisCommandsSentCount
|
|
59
|
-
.labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'delete' })
|
|
60
|
-
.inc();
|
|
61
|
-
}
|
|
62
|
-
async set(key, value, ttl) {
|
|
63
|
-
logger.trace(`Setting key ${key}`);
|
|
64
|
-
// TODO: Check for invalid stringify
|
|
65
|
-
await this.client.set(key, JSON.stringify(value), 'PX', ttl);
|
|
66
|
-
// Only record metrics if feed Id is present, otherwise assuming value is not adapter response to record
|
|
67
|
-
const feedId = value.metricsMeta?.feedId;
|
|
68
|
-
if (feedId) {
|
|
69
|
-
// Record cache set count, max age, and staleness (set to 0 for cache set)
|
|
70
|
-
const label = cacheMetrics.cacheMetricsLabel(key, feedId, cacheMetrics.CacheTypes.Redis);
|
|
71
|
-
cacheMetrics.cacheSet(label, ttl);
|
|
72
|
-
}
|
|
73
|
-
// Record set command sent to Redis
|
|
74
|
-
cacheMetrics.redisCommandsSentCount
|
|
75
|
-
.labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'set' })
|
|
76
|
-
.inc();
|
|
77
|
-
}
|
|
78
|
-
async setMany(entries, ttl) {
|
|
79
|
-
logger.trace(`Setting a bunch of keys`);
|
|
80
|
-
// Unfortunately, there's no ttl for mset
|
|
81
|
-
let chain = this.client.multi();
|
|
82
|
-
for (const entry of entries) {
|
|
83
|
-
chain = chain.set(entry.key, JSON.stringify(entry.value), 'PX', ttl);
|
|
84
|
-
// TODO: Move to after error handling once implemented to avoid recording cache sets that failed
|
|
85
|
-
// Only record metrics if feed Id is present, otherwise assuming value is not adapter response to record
|
|
86
|
-
const feedId = entry.value.metricsMeta?.feedId;
|
|
87
|
-
if (feedId) {
|
|
88
|
-
// Record cache set count, max age, and staleness (set to 0 for cache set)
|
|
89
|
-
const label = cacheMetrics.cacheMetricsLabel(entry.key, feedId, cacheMetrics.CacheTypes.Redis);
|
|
90
|
-
cacheMetrics.cacheSet(label, ttl);
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
await chain.exec();
|
|
94
|
-
// Record setMany command sent to Redis
|
|
95
|
-
cacheMetrics.redisCommandsSentCount
|
|
96
|
-
.labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'exec' })
|
|
97
|
-
.inc();
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
exports.RedisCache = RedisCache;
|
|
@@ -1,191 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.accountsRestEndpoint = exports.BankFrickAccountsTransport = void 0;
|
|
7
|
-
const axios_1 = __importDefault(require("axios"));
|
|
8
|
-
const util_1 = require("./util");
|
|
9
|
-
const util_2 = require("../../util");
|
|
10
|
-
const error_1 = require("../../validation/error");
|
|
11
|
-
const logger = (0, util_2.makeLogger)('BankFrickTransport');
|
|
12
|
-
// Note: this is a shallow pattern that only checks for a country code since IBANs in the sandbox are invalid
|
|
13
|
-
const ibanPattern = /^[A-Z]{2}[A-Z\d]{14,30}$/;
|
|
14
|
-
const inputParameters = {
|
|
15
|
-
ibanIDs: {
|
|
16
|
-
description: 'The list of account ids included in the sum of balances',
|
|
17
|
-
required: true,
|
|
18
|
-
type: 'array',
|
|
19
|
-
},
|
|
20
|
-
signingAlgorithm: {
|
|
21
|
-
description: 'What signing algorithm is used to sign and verify authorization data, one of rsa-sha256, rsa-sha384, or rsa-sha512',
|
|
22
|
-
required: false,
|
|
23
|
-
type: 'string',
|
|
24
|
-
default: 'rsa-sha512',
|
|
25
|
-
options: ['rsa-sha256', 'rsa-sha384', 'rsa-sha512'],
|
|
26
|
-
},
|
|
27
|
-
};
|
|
28
|
-
// See here for all expected error returned by the API: https://developers.bankfrick.li/docs#errors
|
|
29
|
-
const AuthErrors = {
|
|
30
|
-
401: 'No JWT token provided or token is invalid',
|
|
31
|
-
403: 'API key is invalid or any other condition is hindering the login', // Unclear if this is fatal or not
|
|
32
|
-
};
|
|
33
|
-
const FatalErrors = {
|
|
34
|
-
400: "Invalid parameters passed to Bank Frick's API",
|
|
35
|
-
423: "Authorization is valid, but the user's account is locked",
|
|
36
|
-
};
|
|
37
|
-
/**
|
|
38
|
-
* RestTransport implementation for Bank Frick, which has unusually complex requirements for an EA
|
|
39
|
-
* The RestTransport is generally built to make a single request and return a single response.
|
|
40
|
-
* This transport instead is used to fetch and process pages of data, and also requires a JWT to run
|
|
41
|
-
*
|
|
42
|
-
* This transport does all the heavy lifting in setup(), which is where the paging happens, and it
|
|
43
|
-
* also has complex retry logic that will attempt to refresh the JWT when certain HTTP errors occur
|
|
44
|
-
*/
|
|
45
|
-
class BankFrickAccountsTransport {
|
|
46
|
-
async initialize(dependencies) {
|
|
47
|
-
this.cache = dependencies.cache;
|
|
48
|
-
}
|
|
49
|
-
async hasBeenSetUp() {
|
|
50
|
-
return false; // Return false since we aren't coalescing requests
|
|
51
|
-
}
|
|
52
|
-
/**
|
|
53
|
-
* Creates an AxiosRequestConfig object for fetching a page of accounts from the Bank Frick API
|
|
54
|
-
*/
|
|
55
|
-
prepareRequest(firstPosition, inputParams, config) {
|
|
56
|
-
const { API_ENDPOINT, BASE_URL, PAGE_SIZE } = config;
|
|
57
|
-
return {
|
|
58
|
-
baseURL: API_ENDPOINT + BASE_URL,
|
|
59
|
-
url: 'accounts',
|
|
60
|
-
method: 'GET',
|
|
61
|
-
params: {
|
|
62
|
-
firstPosition,
|
|
63
|
-
maxResults: PAGE_SIZE,
|
|
64
|
-
},
|
|
65
|
-
headers: {
|
|
66
|
-
Authorization: `Bearer ${this.token}`,
|
|
67
|
-
},
|
|
68
|
-
};
|
|
69
|
-
}
|
|
70
|
-
/**
|
|
71
|
-
* Request with retry logic for Bank Frick's API. In addition to standard retry logic, this function
|
|
72
|
-
* compares errors against expected errors from the Bank Frick API, and will throw without retries
|
|
73
|
-
* on a known fatal error, or try to refresh the JWT on a known auth error
|
|
74
|
-
**/
|
|
75
|
-
async makeRequest(axiosRequest, config, signingAlgorithm) {
|
|
76
|
-
let retryNumber = 0;
|
|
77
|
-
let response = await axios_1.default.request(axiosRequest);
|
|
78
|
-
while (response.status !== 200) {
|
|
79
|
-
retryNumber++;
|
|
80
|
-
logger.warn('Encountered error when fetching accounts from Bank Frick:', response.status, response.statusText);
|
|
81
|
-
// Evaluate whether the error was fatal, auth, or transient and whether we've exceeded the max number of retries.
|
|
82
|
-
// Throw on fatal, refresh token on auth error, pass on transient until we've exhausted our retries
|
|
83
|
-
if (FatalErrors[response.status]) {
|
|
84
|
-
throw new error_1.AdapterError({
|
|
85
|
-
statusCode: response.status,
|
|
86
|
-
message: response.statusText,
|
|
87
|
-
});
|
|
88
|
-
}
|
|
89
|
-
else if (AuthErrors[response.status]) {
|
|
90
|
-
// We've encountered a known auth error, so try to refresh the token before making another request
|
|
91
|
-
logger.info('Auth error received from the Bank Frick API, attempting to refresh the token');
|
|
92
|
-
this.token = await (0, util_1.generateJWT)(config, signingAlgorithm);
|
|
93
|
-
}
|
|
94
|
-
else if (retryNumber === config.REST_TRANSPORT_MAX_RATE_LIMIT_RETRIES) {
|
|
95
|
-
throw new error_1.AdapterError({
|
|
96
|
-
statusCode: 504,
|
|
97
|
-
message: `Bank Frick transport hit the max number of retries (${config.REST_TRANSPORT_MAX_RATE_LIMIT_RETRIES} retries) and aborted`,
|
|
98
|
-
});
|
|
99
|
-
}
|
|
100
|
-
logger.debug(`Sleeping for ${config.REST_TRANSPORT_MS_BETWEEN_RATE_LIMIT_RETRIES}ms before retrying`);
|
|
101
|
-
await (0, util_2.sleep)(config.REST_TRANSPORT_MS_BETWEEN_RATE_LIMIT_RETRIES);
|
|
102
|
-
response = await axios_1.default.request(axiosRequest);
|
|
103
|
-
}
|
|
104
|
-
return response;
|
|
105
|
-
}
|
|
106
|
-
validateInputParams(params) {
|
|
107
|
-
const encounteredIds = {};
|
|
108
|
-
const errors = [];
|
|
109
|
-
const { ibanIDs } = params;
|
|
110
|
-
ibanIDs.forEach((v) => {
|
|
111
|
-
if (!v.match(ibanPattern)) {
|
|
112
|
-
errors.push(`Invalid IBAN: ${v}`);
|
|
113
|
-
}
|
|
114
|
-
encounteredIds[v] += 1;
|
|
115
|
-
if (encounteredIds[v] > 1) {
|
|
116
|
-
errors.push(`The following IBAN appears more than once in the input parameters: ${v}`);
|
|
117
|
-
}
|
|
118
|
-
});
|
|
119
|
-
return errors;
|
|
120
|
-
}
|
|
121
|
-
/**
|
|
122
|
-
* Fetches pages of data from the Bank Frick API, scans for accounts by IBAN, and returns the balance
|
|
123
|
-
* of all found accounts. Returns a 404 if any IBAN isn't found.
|
|
124
|
-
*/
|
|
125
|
-
async setup(req, config) {
|
|
126
|
-
const { ibanIDs, signingAlgorithm } = req.requestContext.data;
|
|
127
|
-
const { PAGE_SIZE = 500 } = config;
|
|
128
|
-
logger.debug(`Validating input: ${JSON.stringify(req.requestContext.data)}`);
|
|
129
|
-
// Scan ibanIDs for duplicates and invalid IBANs
|
|
130
|
-
const validationErrors = this.validateInputParams(req.requestContext.data);
|
|
131
|
-
if (validationErrors.length > 0) {
|
|
132
|
-
throw new error_1.AdapterError({
|
|
133
|
-
statusCode: 420,
|
|
134
|
-
message: `Received the following errors when validating inputParameters:\n ${validationErrors.join('\n')}`,
|
|
135
|
-
});
|
|
136
|
-
}
|
|
137
|
-
// Refresh the token if it isn't set
|
|
138
|
-
if (!this.token) {
|
|
139
|
-
this.token = await (0, util_1.generateJWT)(config, signingAlgorithm);
|
|
140
|
-
}
|
|
141
|
-
let sum = 0;
|
|
142
|
-
let position = 0;
|
|
143
|
-
const keys = ibanIDs;
|
|
144
|
-
logger.info("Fetching accounts from Bank Frick's API...");
|
|
145
|
-
while (keys.length > 0) {
|
|
146
|
-
// TODO Fetching and processing pages can be run concurrently
|
|
147
|
-
const axiosRequest = this.prepareRequest(position, req.requestContext.data, config);
|
|
148
|
-
const response = await this.makeRequest(axiosRequest, config, signingAlgorithm);
|
|
149
|
-
logger.debug(`Evaluating accounts from page ${position / PAGE_SIZE}`);
|
|
150
|
-
response.data.accounts.forEach((v) => {
|
|
151
|
-
logger.trace(`Evaluating ${v.account} (iban: ${v.iban}, type: ${v.type})`);
|
|
152
|
-
const index = ibanIDs.indexOf(v.iban);
|
|
153
|
-
if (index > -1) {
|
|
154
|
-
keys.splice(index, 1);
|
|
155
|
-
sum += v.balance;
|
|
156
|
-
logger.trace(`Found account: ${v.account} (iban: ${v.iban}) with balance of ${v.balance}`);
|
|
157
|
-
logger.trace(`Running sum: ${sum}, number of ibans left to find: ${keys.length}/${ibanIDs.length}`);
|
|
158
|
-
}
|
|
159
|
-
});
|
|
160
|
-
if (!response.data.moreResults) {
|
|
161
|
-
logger.debug('No more results, breaking out of account query loop');
|
|
162
|
-
break;
|
|
163
|
-
}
|
|
164
|
-
position += PAGE_SIZE || 0;
|
|
165
|
-
}
|
|
166
|
-
// 404 if one or more accounts were not found
|
|
167
|
-
if (keys.length > 0) {
|
|
168
|
-
logger.error(`Could not find all accounts, returning 404: ${keys.join(', ')}`);
|
|
169
|
-
const res = {
|
|
170
|
-
data: 0,
|
|
171
|
-
statusCode: 404,
|
|
172
|
-
result: 0,
|
|
173
|
-
};
|
|
174
|
-
await this.cache.set(req.requestContext.cacheKey, res, config.CACHE_MAX_AGE);
|
|
175
|
-
}
|
|
176
|
-
logger.debug('Was able to find all accounts, returning balance across all accounts: ', sum);
|
|
177
|
-
const res = {
|
|
178
|
-
data: sum,
|
|
179
|
-
statusCode: 200,
|
|
180
|
-
result: sum,
|
|
181
|
-
};
|
|
182
|
-
await this.cache.set(req.requestContext.cacheKey, res, config.CACHE_MAX_AGE);
|
|
183
|
-
return res;
|
|
184
|
-
}
|
|
185
|
-
}
|
|
186
|
-
exports.BankFrickAccountsTransport = BankFrickAccountsTransport;
|
|
187
|
-
exports.accountsRestEndpoint = {
|
|
188
|
-
name: 'accounts',
|
|
189
|
-
transport: new BankFrickAccountsTransport(),
|
|
190
|
-
inputParameters,
|
|
191
|
-
};
|
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.customSettings = exports.signingAlgorithms = void 0;
|
|
7
|
-
const crypto_1 = __importDefault(require("crypto"));
|
|
8
|
-
const util_1 = require("../../../util");
|
|
9
|
-
const logger = (0, util_1.makeLogger)('BankFrickConfig');
|
|
10
|
-
// Used for enum options and validation
|
|
11
|
-
exports.signingAlgorithms = ['rsa-sha256', 'rsa-sha384', 'rsa-sha512'];
|
|
12
|
-
const MAX_PAGE_SIZE = 500;
|
|
13
|
-
exports.customSettings = {
|
|
14
|
-
PAGE_SIZE: {
|
|
15
|
-
description: 'The number of accounts to fetch per call to /accounts. Must be >= 1 and <= 500.',
|
|
16
|
-
type: 'enum',
|
|
17
|
-
required: false,
|
|
18
|
-
default: MAX_PAGE_SIZE.toString(),
|
|
19
|
-
options: [...Array(MAX_PAGE_SIZE).keys()].map((i) => (i + 1).toString()), // 1..MAX_PAGE_SIZE
|
|
20
|
-
},
|
|
21
|
-
PRIVATE_KEY: {
|
|
22
|
-
description: '',
|
|
23
|
-
type: 'string',
|
|
24
|
-
required: true,
|
|
25
|
-
validate: (value) => {
|
|
26
|
-
if (!value) {
|
|
27
|
-
return 'PRIVATE_KEY is a required variable, was undefined or empty';
|
|
28
|
-
}
|
|
29
|
-
else if (typeof value !== 'string') {
|
|
30
|
-
return `PRIVATE_KEY must be a string, was ${typeof value}`;
|
|
31
|
-
}
|
|
32
|
-
for (const algorithm of exports.signingAlgorithms) {
|
|
33
|
-
const body = { example: 123 };
|
|
34
|
-
try {
|
|
35
|
-
crypto_1.default.sign(algorithm, Buffer.from(JSON.stringify(body)), value);
|
|
36
|
-
return ''; // Any successful sign means the key is valid
|
|
37
|
-
}
|
|
38
|
-
catch {
|
|
39
|
-
logger.debug(`Failed to sign a dummy body using PRIVATE_KEY with algorithm ${algorithm}`);
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
return `PRIVATE_KEY could not be used to sign a dummy body using any of ${exports.signingAlgorithms}`;
|
|
43
|
-
},
|
|
44
|
-
},
|
|
45
|
-
};
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.adapter = void 0;
|
|
4
|
-
const accounts_1 = require("./accounts");
|
|
5
|
-
const config_1 = require("./config");
|
|
6
|
-
exports.adapter = {
|
|
7
|
-
name: 'bank-frick',
|
|
8
|
-
defaultEndpoint: 'accounts',
|
|
9
|
-
endpoints: [accounts_1.accountsRestEndpoint],
|
|
10
|
-
customSettings: config_1.customSettings,
|
|
11
|
-
envDefaultOverrides: {
|
|
12
|
-
API_ENDPOINT: 'https://olbsandbox.bankfrick.li/webapi/v2',
|
|
13
|
-
},
|
|
14
|
-
};
|