@chainlink/external-adapter-framework 0.5.0 → 0.5.2

package/cache/local.js

@@ -0,0 +1,84 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LocalCache = void 0;
+const util_1 = require("../util");
+const cacheMetrics = __importStar(require("./metrics"));
+const logger = (0, util_1.makeLogger)('LocalCache');
+/**
+ * Local implementation of a Cache. It uses a simple js Object, storing entries with both
+ * a value and an expiration timestamp. Expired entries are deleted on reads (i.e. no background gc/upkeep).
+ *
+ * @typeParam T - the type for the entries' values
+ */
+class LocalCache {
+    constructor() {
+        this.store = {};
+    }
+    async get(key) {
+        logger.trace(`Getting key ${key}`);
+        const entry = this.store[key];
+        if (!entry) {
+            logger.debug(`No entry in local cache for key "${key}", returning undefined`);
+            return undefined;
+        }
+        const expired = entry.expirationTimestamp <= Date.now();
+        if (expired) {
+            logger.debug('Entry in local cache expired, deleting and returning undefined');
+            this.delete(key);
+            return undefined;
+        }
+        else {
+            logger.debug('Found valid entry in local cache, returning value');
+            return entry.value;
+        }
+    }
+    async delete(key) {
+        logger.trace(`Deleting key ${key}`);
+        delete this.store[key]; // Deletes are slower than ignoring or setting null, fyi
+    }
+    async set(key, value, ttl) {
+        logger.trace(`Setting key ${key} with ttl ${ttl}`);
+        this.store[key] = {
+            value,
+            expirationTimestamp: Date.now() + ttl,
+        };
+        // Only record metrics if feed Id is present, otherwise assuming value is not adapter response to record
+        const feedId = value.meta?.metrics?.feedId;
+        if (feedId) {
+            // Record cache set count, max age, and staleness (set to 0 for cache set)
+            const label = cacheMetrics.cacheMetricsLabel(key, feedId, cacheMetrics.CacheTypes.Local);
+            cacheMetrics.cacheSet(label, ttl);
+        }
+    }
+    async setMany(entries, ttl) {
+        logger.trace(`Setting a bunch of keys with ttl ${ttl}`);
+        for (const { key, value } of entries) {
+            this.set(key, value, ttl);
+        }
+    }
+}
+exports.LocalCache = LocalCache;
+//# sourceMappingURL=local.js.map

package/cache/local.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"local.js","sourceRoot":"","sources":["../../../src/cache/local.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,kCAAqD;AAErD,wDAAyC;AAEzC,MAAM,MAAM,GAAG,IAAA,iBAAU,EAAC,YAAY,CAAC,CAAA;AAYvC;;;;;GAKG;AACH,MAAa,UAAU;IAAvB;QACE,UAAK,GAAuC,EAAE,CAAA;IAiDhD,CAAC;IA/CC,KAAK,CAAC,GAAG,CAAC,GAAW;QACnB,MAAM,CAAC,KAAK,CAAC,eAAe,GAAG,EAAE,CAAC,CAAA;QAClC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QAE7B,IAAI,CAAC,KAAK,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,oCAAoC,GAAG,wBAAwB,CAAC,CAAA;YAC7E,OAAO,SAAS,CAAA;SACjB;QAED,MAAM,OAAO,GAAG,KAAK,CAAC,mBAAmB,IAAI,IAAI,CAAC,GAAG,EAAE,CAAA;QACvD,IAAI,OAAO,EAAE;YACX,MAAM,CAAC,KAAK,CAAC,gEAAgE,CAAC,CAAA;YAC9E,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YAChB,OAAO,SAAS,CAAA;SACjB;aAAM;YACL,MAAM,CAAC,KAAK,CAAC,mDAAmD,CAAC,CAAA;YACjE,OAAO,KAAK,CAAC,KAAK,CAAA;SACnB;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAW;QACtB,MAAM,CAAC,KAAK,CAAC,gBAAgB,GAAG,EAAE,CAAC,CAAA;QACnC,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA,CAAC,wDAAwD;IACjF,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW,EAAE,KAAQ,EAAE,GAAW;QAC1C,MAAM,CAAC,KAAK,CAAC,eAAe,GAAG,aAAa,GAAG,EAAE,CAAC,CAAA;QAClD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG;YAChB,KAAK;YACL,mBAAmB,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG;SACtC,CAAA;QAED,wGAAwG;QACxG,MAAM,MAAM,GAAI,KAAoC,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAA;QAC1E,IAAI,MAAM,EAAE;YACV,0EAA0E;YAC1E,MAAM,KAAK,GAAG,YAAY,CAAC,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,UAAU,CAAC,KAAK,CAAC,CAAA;YACxF,YAAY,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;SAClC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,OAAwB,EAAE,GAAW;QACjD,MAAM,CAAC,KAAK,CAAC,oCAAoC,GAAG,EAAE,CAAC,CAAA;QACvD,KAAK,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,OAAO,EAAE;YACpC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,CAAA;SAC1B;IACH,CAAC;CACF;AAlDD,gCAkDC"}

package/cache/metrics.d.ts

@@ -0,0 +1,27 @@
+import * as client from 'prom-client';
+interface CacheMetricsLabels {
+    participant_id: string;
+    feed_id: string;
+    cache_type: string;
+}
+export declare const cacheGet: (label: CacheMetricsLabels, value: unknown, staleness: number) => void;
+export declare const cacheSet: (label: CacheMetricsLabels, maxAge: number) => void;
+export declare const cacheMetricsLabel: (cacheKey: string, feedId: string, cacheType: string) => {
+    participant_id: string;
+    feed_id: string;
+    cache_type: string;
+};
+export declare enum CacheTypes {
+    Redis = "redis",
+    Local = "local"
+}
+export declare enum CMD_SENT_STATUS {
+    TIMEOUT = "TIMEOUT",
+    FAIL = "FAIL",
+    SUCCESS = "SUCCESS"
+}
+export declare const redisConnectionsOpen: client.Counter<string>;
+export declare const redisRetriesCount: client.Counter<string>;
+export declare const redisCommandsSentCount: client.Counter<"status" | "function_name">;
+export declare const cacheWarmerCount: client.Gauge<"isBatched">;
+export {};

package/cache/metrics.js

@@ -0,0 +1,121 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.cacheWarmerCount = exports.redisCommandsSentCount = exports.redisRetriesCount = exports.redisConnectionsOpen = exports.CMD_SENT_STATUS = exports.CacheTypes = exports.cacheMetricsLabel = exports.cacheSet = exports.cacheGet = void 0;
+const client = __importStar(require("prom-client"));
+const cacheGet = (label, value, staleness) => {
+    if (typeof value === 'number' || typeof value === 'string') {
+        const parsedValue = Number(value);
+        if (!Number.isNaN(parsedValue) && Number.isFinite(parsedValue)) {
+            cacheDataGetValues.labels(label).set(parsedValue);
+        }
+    }
+    cacheDataGetCount.labels(label).inc();
+    cacheDataStalenessSeconds.labels(label).set(staleness);
+};
+exports.cacheGet = cacheGet;
+const cacheSet = (label, maxAge) => {
+    cacheDataSetCount.labels(label).inc();
+    cacheDataMaxAge.labels(label).set(maxAge);
+    cacheDataStalenessSeconds.labels(label).set(0);
+};
+exports.cacheSet = cacheSet;
+const cacheMetricsLabel = (cacheKey, feedId, cacheType) => ({
+    participant_id: cacheKey,
+    feed_id: feedId,
+    cache_type: cacheType,
+});
+exports.cacheMetricsLabel = cacheMetricsLabel;
+var CacheTypes;
+(function (CacheTypes) {
+    CacheTypes["Redis"] = "redis";
+    CacheTypes["Local"] = "local";
+})(CacheTypes = exports.CacheTypes || (exports.CacheTypes = {}));
+var CMD_SENT_STATUS;
+(function (CMD_SENT_STATUS) {
+    CMD_SENT_STATUS["TIMEOUT"] = "TIMEOUT";
+    CMD_SENT_STATUS["FAIL"] = "FAIL";
+    CMD_SENT_STATUS["SUCCESS"] = "SUCCESS";
+})(CMD_SENT_STATUS = exports.CMD_SENT_STATUS || (exports.CMD_SENT_STATUS = {}));
+const baseLabels = [
+    'feed_id',
+    'participant_id',
+    'cache_type',
+    'is_from_ws',
+    'experimental',
+];
+// Skipping this metrics for v3
+// const cache_execution_duration_seconds = new client.Histogram({
+//   name: 'cache_execution_duration_seconds',
+//   help: 'A histogram bucket of the distribution of cache execution durations',
+//   labelNames: [...baseLabels, 'cache_hit'] as const,
+//   buckets: [0.01, 0.1, 1, 10],
+// })
+const cacheDataGetCount = new client.Counter({
+    name: 'cache_data_get_count',
+    help: 'A counter that increments every time a value is fetched from the cache',
+    labelNames: baseLabels,
+});
+const cacheDataGetValues = new client.Gauge({
+    name: 'cache_data_get_values',
+    help: 'A gauge keeping track of values being fetched from cache',
+    labelNames: baseLabels,
+});
+const cacheDataMaxAge = new client.Gauge({
+    name: 'cache_data_max_age',
+    help: 'A gauge tracking the max age of stored values in the cache',
+    labelNames: baseLabels,
+});
+const cacheDataSetCount = new client.Counter({
+    name: 'cache_data_set_count',
+    help: 'A counter that increments every time a value is set to the cache',
+    labelNames: [...baseLabels, 'status_code'],
+});
+const cacheDataStalenessSeconds = new client.Gauge({
+    name: 'cache_data_staleness_seconds',
+    help: 'Observes the staleness of the data returned',
+    labelNames: baseLabels,
+});
+// Redis Metrics
+exports.redisConnectionsOpen = new client.Counter({
+    name: 'redis_connections_open',
+    help: 'The number of redis connections that are open',
+});
+exports.redisRetriesCount = new client.Counter({
+    name: 'redis_retries_count',
+    help: 'The number of retries that have been made to establish a redis connection',
+});
+exports.redisCommandsSentCount = new client.Counter({
+    name: 'redis_commands_sent_count',
+    help: 'The number of redis commands sent',
+    labelNames: ['status', 'function_name'],
+});
+// Cache Warmer Metrics
+exports.cacheWarmerCount = new client.Gauge({
+    name: 'cache_warmer_get_count',
+    help: 'The number of cache warmers running',
+    labelNames: ['isBatched'],
+});
+//# sourceMappingURL=metrics.js.map

package/cache/metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../../src/cache/metrics.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,oDAAqC;AAS9B,MAAM,QAAQ,GAAG,CAAC,KAAyB,EAAE,KAAc,EAAE,SAAiB,EAAE,EAAE;IACvF,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;QAC1D,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,CAAA;QACjC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE;YAC9D,kBAAkB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAA;SAClD;KACF;IACD,iBAAiB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAA;IACrC,yBAAyB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;AACxD,CAAC,CAAA;AATY,QAAA,QAAQ,YASpB;AAEM,MAAM,QAAQ,GAAG,CAAC,KAAyB,EAAE,MAAc,EAAE,EAAE;IACpE,iBAAiB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAA;IACrC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IACzC,yBAAyB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;AAChD,CAAC,CAAA;AAJY,QAAA,QAAQ,YAIpB;AAEM,MAAM,iBAAiB,GAAG,CAAC,QAAgB,EAAE,MAAc,EAAE,SAAiB,EAAE,EAAE,CAAC,CAAC;IACzF,cAAc,EAAE,QAAQ;IACxB,OAAO,EAAE,MAAM;IACf,UAAU,EAAE,SAAS;CACtB,CAAC,CAAA;AAJW,QAAA,iBAAiB,qBAI5B;AAEF,IAAY,UAGX;AAHD,WAAY,UAAU;IACpB,6BAAe,CAAA;IACf,6BAAe,CAAA;AACjB,CAAC,EAHW,UAAU,GAAV,kBAAU,KAAV,kBAAU,QAGrB;AAED,IAAY,eAIX;AAJD,WAAY,eAAe;IACzB,sCAAmB,CAAA;IACnB,gCAAa,CAAA;IACb,sCAAmB,CAAA;AACrB,CAAC,EAJW,eAAe,GAAf,uBAAe,KAAf,uBAAe,QAI1B;AAED,MAAM,UAAU,GAAG;IACjB,SAAS;IACT,gBAAgB;IAChB,YAAY;IACZ,YAAY;IACZ,cAAc;CACN,CAAA;AAEV,+BAA+B;AAC/B,kEAAkE;AAClE,8CAA8C;AAC9C,iFAAiF;AACjF,uDAAuD;AACvD,iCAAiC;AACjC,KAAK;AAEL,MAAM,iBAAiB,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IAC3C,IAAI,EAAE,sBAAsB;IAC5B,IAAI,EAAE,wEAAwE;IAC9E,UAAU,EAAE,UAAU;CACvB,CAAC,CAAA;AAEF,MAAM,kBAAkB,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC;IAC1C,IAAI,EAAE,uBAAuB;IAC7B,IAAI,EAAE,0DAA0D;IAChE,UAAU,EAAE,UAAU;CACvB,CAAC,CAAA;AAEF,MAAM,eAAe,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC;IACvC,IAAI,EAAE,oBAAoB;IAC1B,IAAI,EAAE,4DAA4D;IAClE,UAAU,EAAE,UAAU;CACvB,CAAC,CAAA;AAEF,MAAM,iBAAiB,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IAC3C,IAAI,EAAE,sBAAsB;IAC5B,IAAI,EAAE,kEAAkE;IACxE,UAAU,EAAE,CAAC,GAAG,UAAU,EAAE,aAAa,CAAC;CAC3C,CAAC,CAAA;AAEF,MAAM,yBAAyB,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC;IACjD,IAAI,EAAE,8BAA8B;IACpC,IAAI,EAAE,6CAA6C;IACnD,UAAU,EAAE,UAAU;CACvB,CAAC,CAAA;AAEF,gBAAgB;AACH,QAAA,oBAAoB,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IACrD,IAAI,EAAE,wBAAwB;IAC9B,IAAI,EAAE,+CAA+C;CACtD,CAAC,CAAA;AAEW,QAAA,iBAAiB,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IAClD,IAAI,EAAE,qBAAqB;IAC3B,IAAI,EAAE,2EAA2E;CAClF,CAAC,CAAA;AAEW,QAAA,sBAAsB,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IACvD,IAAI,EAAE,2BAA2B;IACjC,IAAI,EAAE,mCAAmC;IACzC,UAAU,EAAE,CAAC,QAAQ,EAAE,eAAe,CAAC;CACxC,CAAC,CAAA;AAEF,uBAAuB;AACV,QAAA,gBAAgB,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC;IAC/C,IAAI,EAAE,wBAAwB;IAC9B,IAAI,EAAE,qCAAqC;IAC3C,UAAU,EAAE,CAAC,WAAW,CAAU;CACnC,CAAC,CAAA"}

package/cache/redis.d.ts

@@ -0,0 +1,16 @@
+import Redis from 'ioredis';
+import { Cache, CacheEntry } from './index';
+/**
+ * Redis implementation of a Cache. It uses a simple js Object, storing entries with both
+ * a value and an expiration timestamp. Expired entries are deleted on reads (i.e. no background gc/upkeep).
+ *
+ * @typeParam T - the type for the entries' values
+ */
+export declare class RedisCache<T = unknown> implements Cache<T> {
+    private client;
+    constructor(client: Redis);
+    get(key: string): Promise<T | undefined>;
+    delete(key: string): Promise<void>;
+    set(key: string, value: T, ttl: number): Promise<void>;
+    setMany(entries: CacheEntry<T>[], ttl: number): Promise<void>;
+}

package/cache/redis.js

@@ -0,0 +1,101 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RedisCache = void 0;
+const util_1 = require("../util");
+const cacheMetrics = __importStar(require("./metrics"));
+const logger = (0, util_1.makeLogger)('RedisCache');
+/**
+ * Redis implementation of a Cache. It uses a simple js Object, storing entries with both
+ * a value and an expiration timestamp. Expired entries are deleted on reads (i.e. no background gc/upkeep).
+ *
+ * @typeParam T - the type for the entries' values
+ */
+class RedisCache {
+    constructor(client) {
+        this.client = client;
+    }
+    async get(key) {
+        logger.trace(`Getting key ${key}`);
+        const value = await this.client.get(key);
+        // Record get command sent to Redis
+        cacheMetrics.redisCommandsSentCount
+            .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'get' })
+            .inc();
+        if (!value) {
+            logger.debug(`No entry in redis cache for key "${key}", returning undefined`);
+            return undefined;
+        }
+        return JSON.parse(value);
+    }
+    async delete(key) {
+        logger.trace(`Deleting key ${key}`);
+        await this.client.del(key);
+        // Record delete command sent to Redis
+        cacheMetrics.redisCommandsSentCount
+            .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'delete' })
+            .inc();
+    }
+    async set(key, value, ttl) {
+        logger.trace(`Setting key ${key}`);
+        await this.client.set(key, JSON.stringify(value), 'PX', ttl);
+        // Only record metrics if feed Id is present, otherwise assuming value is not adapter response to record
+        const feedId = value.meta?.metrics?.feedId;
+        if (feedId) {
+            // Record cache set count, max age, and staleness (set to 0 for cache set)
+            const label = cacheMetrics.cacheMetricsLabel(key, feedId, cacheMetrics.CacheTypes.Redis);
+            cacheMetrics.cacheSet(label, ttl);
+        }
+        // Record set command sent to Redis
+        cacheMetrics.redisCommandsSentCount
+            .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'set' })
+            .inc();
+    }
+    async setMany(entries, ttl) {
+        logger.trace(`Setting a bunch of keys`);
+        // Unfortunately, there's no ttl for mset
+        let chain = this.client.multi();
+        for (const entry of entries) {
+            chain = chain.set(entry.key, JSON.stringify(entry.value), 'PX', ttl);
+        }
+        await chain.exec();
+        // Loop again, but this time to record these in metrics
+        for (const entry of entries) {
+            // Only record metrics if feed Id is present, otherwise assuming value is not adapter response to record
+            const feedId = entry.value.meta?.metrics?.feedId;
+            if (feedId) {
+                // Record cache set count, max age, and staleness (set to 0 for cache set)
+                const label = cacheMetrics.cacheMetricsLabel(entry.key, feedId, cacheMetrics.CacheTypes.Redis);
+                cacheMetrics.cacheSet(label, ttl);
+            }
+        }
+        // Record setMany command sent to Redis
+        cacheMetrics.redisCommandsSentCount
+            .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'exec' })
+            .inc();
+    }
+}
+exports.RedisCache = RedisCache;
+//# sourceMappingURL=redis.js.map

package/cache/redis.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"redis.js","sourceRoot":"","sources":["../../../src/cache/redis.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AACA,kCAAqD;AAErD,wDAAyC;AAEzC,MAAM,MAAM,GAAG,IAAA,iBAAU,EAAC,YAAY,CAAC,CAAA;AAEvC;;;;;GAKG;AACH,MAAa,UAAU;IACrB,YAAoB,MAAa;QAAb,WAAM,GAAN,MAAM,CAAO;IAAG,CAAC;IAErC,KAAK,CAAC,GAAG,CAAC,GAAW;QACnB,MAAM,CAAC,KAAK,CAAC,eAAe,GAAG,EAAE,CAAC,CAAA;QAClC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAExC,mCAAmC;QACnC,YAAY,CAAC,sBAAsB;aAChC,MAAM,CAAC,EAAE,MAAM,EAAE,YAAY,CAAC,eAAe,CAAC,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;aAC9E,GAAG,EAAE,CAAA;QAER,IAAI,CAAC,KAAK,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,oCAAoC,GAAG,wBAAwB,CAAC,CAAA;YAC7E,OAAO,SAAS,CAAA;SACjB;QAED,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAM,CAAA;IAC/B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAW;QACtB,MAAM,CAAC,KAAK,CAAC,gBAAgB,GAAG,EAAE,CAAC,CAAA;QACnC,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAE1B,sCAAsC;QACtC,YAAY,CAAC,sBAAsB;aAChC,MAAM,CAAC,EAAE,MAAM,EAAE,YAAY,CAAC,eAAe,CAAC,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC;aACjF,GAAG,EAAE,CAAA;IACV,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW,EAAE,KAAQ,EAAE,GAAW;QAC1C,MAAM,CAAC,KAAK,CAAC,eAAe,GAAG,EAAE,CAAC,CAAA;QAClC,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,CAAA;QAE5D,wGAAwG;QACxG,MAAM,MAAM,GAAI,KAAoC,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAA;QAC1E,IAAI,MAAM,EAAE;YACV,0EAA0E;YAC1E,MAAM,KAAK,GAAG,YAAY,CAAC,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,UAAU,CAAC,KAAK,CAAC,CAAA;YACxF,YAAY,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;SAClC;QAED,mCAAmC;QACnC,YAAY,CAAC,sBAAsB;aAChC,MAAM,CAAC,EAAE,MAAM,EAAE,YAAY,CAAC,eAAe,CAAC,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;aAC9E,GAAG,EAAE,CAAA;IACV,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,OAAwB,EAAE,GAAW;QACjD,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAA;QACvC,yCAAyC;QACzC,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAA;QAE/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;YAC3B,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,CAAA;SACrE;QAED,MAAM,KAAK,CAAC,IAAI,EAAE,CAAA;QAElB,uDAAuD;QACvD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;YAC3B,wGAAwG;YACxG,MAAM,MAAM,GAAI,KAAK,CAAC,KAAoC,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAA;YAChF,IAAI,MAAM,EAAE;gBACV,0EAA0E;gBAC1E,MAAM,KAAK,GAAG,YAAY,CAAC,iBAAiB,CAC1C,KAAK,CAAC,GAAG,EACT,MAAM,EACN,YAAY,CAAC,UAAU,CAAC,KAAK,CAC9B,CAAA;gBACD,YAAY,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;aAClC;SACF;QAED,uCAAuC;QACvC,YAAY,CAAC,sBAAsB;aAChC,MAAM,CAAC,EAAE,MAAM,EAAE,YAAY,CAAC,eAAe,CAAC,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC;aAC/E,GAAG,EAAE,CAAA;IACV,CAAC;CACF;AA/ED,gCA+EC"}

package/config/index.d.ts

@@ -0,0 +1,298 @@
+export declare const BaseSettings: {
+    readonly API_ENDPOINT: {
+        readonly description: "The URL that certain transports use to retrieve data";
+        readonly type: "string";
+    };
+    readonly API_KEY: {
+        readonly description: "Default setting for an EA key";
+        readonly type: "string";
+        readonly sensitive: true;
+    };
+    readonly API_TIMEOUT: {
+        readonly description: "The number of milliseconds a request can be pending before returning a timeout error for data provider request";
+        readonly type: "number";
+        readonly default: 30000;
+    };
+    readonly API_VERBOSE: {
+        readonly description: "Toggle whether the response from the EA should contain just the results or also include the full response body from the queried API.";
+        readonly type: "boolean";
+        readonly default: false;
+    };
+    readonly BASE_URL: {
+        readonly description: "Starting path for the EA handler endpoint";
+        readonly type: "string";
+        readonly default: "/";
+    };
+    readonly BATCH_TRANSPORT_SETUP_VALIDATION: {
+        readonly description: "Flag to toggle batch transport setup validation";
+        readonly type: "boolean";
+        readonly default: false;
+    };
+    readonly CACHE_MAX_AGE: {
+        readonly description: "Maximum amount of time (in ms) that a response will stay cached";
+        readonly type: "number";
+        readonly default: 90000;
+    };
+    readonly CACHE_REDIS_HOST: {
+        readonly description: "Hostname for the Redis instance to be used";
+        readonly type: "string";
+        readonly default: "127.0.0.1";
+    };
+    readonly CACHE_REDIS_PASSWORD: {
+        readonly description: "The password required for redis auth";
+        readonly type: "string";
+        readonly sensitive: true;
+    };
+    readonly CACHE_REDIS_PATH: {
+        readonly description: "The UNIX socket string of the Redis server";
+        readonly type: "string";
+    };
+    readonly CACHE_REDIS_PORT: {
+        readonly description: "Port for the Redis instance to be used";
+        readonly type: "number";
+        readonly default: 6379;
+    };
+    readonly CACHE_REDIS_TIMEOUT: {
+        readonly description: "Timeout to fail a Redis server request if no response (ms)";
+        readonly type: "number";
+        readonly default: 500;
+    };
+    readonly CACHE_TYPE: {
+        readonly description: "The type of cache to use throughout the EA";
+        readonly type: "enum";
+        readonly default: "local";
+        readonly options: readonly ["local", "redis"];
+    };
+    readonly CORRELATION_ID_ENABLED: {
+        readonly description: "Flag to enable correlation IDs for sent requests in logging";
+        readonly type: "boolean";
+        readonly default: true;
+    };
+    readonly DEBUG: {
+        readonly description: "Toggles debug mode";
+        readonly type: "boolean";
+        readonly default: false;
+    };
+    readonly EA_PORT: {
+        readonly description: "Port through which the EA will listen for REST requests (if mode is set to \"reader\" or \"reader-writer\")";
+        readonly type: "number";
+        readonly default: 8080;
+    };
+    readonly EXPERIMENTAL_METRICS_ENABLED: {
+        readonly description: "Flag to specify whether or not to collect metrics. Used as fallback for METRICS_ENABLED";
+        readonly type: "boolean";
+        readonly default: true;
+    };
+    readonly LOG_LEVEL: {
+        readonly description: "Minimum level required for logs to be output";
+        readonly type: "string";
+        readonly default: "info";
+    };
+    readonly METRICS_ENABLED: {
+        readonly description: "Flag to specify whether or not to startup the metrics server";
+        readonly type: "boolean";
+        readonly default: true;
+    };
+    readonly METRICS_NAME: {
+        readonly description: "Metrics name";
+        readonly type: "string";
+    };
+    readonly METRICS_PORT: {
+        readonly description: "Port metrics will be exposed to";
+        readonly type: "number";
+        readonly default: 9080;
+    };
+    readonly METRICS_USE_BASE_URL: {
+        readonly description: "Flag to specify whether or not to prepend the BASE_URL to the metrics endpoint";
+        readonly type: "boolean";
+    };
+    readonly RATE_LIMIT_API_TIER: {
+        readonly description: "Rate limiting tier to use from the available options for the adapter. If not present, the adapter will run using the first tier on the list.";
+        readonly type: "string";
+    };
+    readonly RATE_LIMIT_CAPACITY: {
+        readonly description: "Used as rate limit capacity per minute and ignores tier settings if defined";
+        readonly type: "number";
+    };
+    readonly RATE_LIMIT_CAPACITY_MINUTE: {
+        readonly description: "Used as rate limit capacity per minute and ignores tier settings if defined. Supercedes RATE_LIMIT_CAPACITY if both vars are set";
+        readonly type: "number";
+    };
+    readonly RATE_LIMIT_CAPACITY_SECOND: {
+        readonly description: "Used as rate limit capacity per second and ignores tier settings if defined";
+        readonly type: "number";
+    };
+    readonly REQUEST_COALESCING_ENABLED: {
+        readonly description: "Enable request coalescing";
+        readonly type: "boolean";
+        readonly default: false;
+    };
+    readonly REQUEST_COALESCING_ENTROPY_MAX: {
+        readonly description: "Amount of random delay (entropy) in milliseconds that will be added to requests. Avoids issue where the request coalescing key will not be set before multiple other instances in a burst try to access the same key";
+        readonly type: "number";
+        readonly default: 0;
+    };
+    readonly SSE_SUBSCRIPTION_UPDATE_SLEEP: {
+        readonly description: "Maximum amount of time (in ms) between each update to the set of subscribed feeds";
+        readonly type: "number";
+        readonly default: 1000;
+    };
+    readonly SSE_KEEPALIVE_SLEEP: {
+        readonly description: "Maximum amount of time (in ms) between each SSE keepalive request";
+        readonly type: "number";
+        readonly default: 60000;
+    };
+    readonly SSE_SUBSCRIPTION_TTL: {
+        readonly description: "Maximum amount of time (in ms) an SSE subscription will be cached before being unsubscribed";
+        readonly type: "number";
+        readonly default: 300000;
+    };
+    readonly WARMUP_SUBSCRIPTION_TTL: {
+        readonly type: "number";
+        readonly description: "TTL for batch warmer subscriptions";
+        readonly default: 300000;
+    };
+    readonly WS_API_ENDPOINT: {
+        readonly description: "Override the base websocket URL within the EA.";
+        readonly type: "string";
+    };
+    readonly WS_API_KEY: {
+        readonly description: "The websocket API key to authenticate with";
+        readonly type: "string";
+        readonly sensitive: true;
+    };
+    readonly WS_SUBSCRIPTION_TTL: {
+        readonly description: "";
+        readonly type: "number";
+        readonly default: 120000;
+    };
+    readonly CACHE_POLLING_MAX_RETRIES: {
+        readonly description: "Max amount of times to attempt to find EA response in the cache after the Transport has been set up";
+        readonly type: "number";
+        readonly default: 10;
+    };
+    readonly CACHE_POLLING_SLEEP_MS: {
+        readonly description: "The number of ms to sleep between each retry to fetch the EA response in the cache";
+        readonly type: "number";
+        readonly default: 200;
+    };
+    readonly DEFAULT_CACHE_KEY: {
+        readonly description: "Default key to be used when one cannot be determined from request parameters";
+        readonly type: "string";
+        readonly default: "DEFAULT_CACHE_KEY";
+    };
+    readonly EA_HOST: {
+        readonly description: "Host this EA will listen for REST requests on (if mode is set to \"reader\" or \"reader-writer\")";
+        readonly type: "string";
+        readonly default: "::";
+    };
+    readonly EA_MODE: {
+        readonly description: "Port this EA will listen for REST requests on (if mode is set to \"reader\" or \"reader-writer\")";
+        readonly type: "enum";
+        readonly default: "reader-writer";
+        readonly options: readonly ["reader", "writer", "reader-writer"];
+    };
+    readonly MAX_COMMON_KEY_SIZE: {
+        readonly description: "Maximum amount of characters that the common part of the cache key or feed ID can have";
+        readonly type: "number";
+        readonly default: 300;
+        readonly validate: (value?: number) => "MAX_COMMON_KEY_SIZE must be a number between 150 and 500" | undefined;
+    };
+    readonly REST_TRANSPORT_MAX_RATE_LIMIT_RETRIES: {
+        readonly description: "Maximum amount of times the Rest Transport will attempt to set up a request when blocked by the rate limiter";
+        readonly type: "number";
+        readonly default: 3;
+    };
+    readonly REST_TRANSPORT_MS_BETWEEN_RATE_LIMIT_RETRIES: {
+        readonly description: "Time that the Rest Transport will wait between retries when blocked by the rate limiter";
+        readonly type: "number";
+        readonly default: 400;
+    };
+    readonly SMOKE_TEST_PAYLOAD_FILE_NAME: {
+        readonly description: "Name of the test payload file used for the smoke endpoint";
+        readonly type: "string";
+    };
+};
+export declare const buildAdapterConfig: <CustomSettings extends CustomSettingsType<CustomSettings> = EmptySettings>({ overrides, customSettings, }: {
+    overrides?: Partial<BaseAdapterConfig> | undefined;
+    customSettings?: SettingsMap | undefined;
+}) => AdapterConfig<CustomSettings>;
+export declare const validateAdapterConfig: <CustomSettings extends CustomSettingsType<CustomSettings> = EmptySettings>(adapterConfig: AdapterConfig<CustomSettings>, customSettings?: SettingsMap) => void;
+declare type SettingValueType = string | number | boolean;
+declare type SettingType<C extends Setting> = C['type'] extends 'string' ? string : C['type'] extends 'number' ? number : C['type'] extends 'boolean' ? boolean : C['type'] extends 'enum' ? C['options'] extends readonly string[] ? C['options'][number] : never : never;
+declare type BaseSettingsType = typeof BaseSettings;
+export declare type Setting = {
+    type: 'string';
+    description: string;
+    options?: never;
+    default?: string;
+    validate?: (value?: string) => ValidationErrorMessage;
+    required?: false;
+    sensitive?: boolean;
+} | {
+    type: 'string';
+    description: string;
+    options?: never;
+    default?: string;
+    validate?: (value: string) => ValidationErrorMessage;
+    required: true;
+    sensitive?: boolean;
+} | {
+    type: 'number';
+    description: string;
+    options?: never;
+    default?: number;
+    validate?: (value?: number) => ValidationErrorMessage;
+    required?: false;
+} | {
+    type: 'number';
+    description: string;
+    options?: never;
+    default?: number;
+    validate?: (value: number) => ValidationErrorMessage;
+    required: true;
+} | {
+    type: 'boolean';
+    description: string;
+    options?: never;
+    default?: boolean;
+    validate?: (value?: boolean) => ValidationErrorMessage;
+    required?: false;
+} | {
+    type: 'boolean';
+    description: string;
+    options?: never;
+    default?: boolean;
+    validate?: (value: boolean) => ValidationErrorMessage;
+    required: true;
+} | {
+    type: 'enum';
+    description: string;
+    default?: string;
+    options: readonly string[];
+    validate?: (value?: string) => ValidationErrorMessage;
+    required?: false;
+} | {
+    type: 'enum';
+    description: string;
+    default?: string;
+    options: readonly string[];
+    validate?: (value: string) => ValidationErrorMessage;
+    required: true;
+};
+export declare type AdapterConfigFromSettings<T extends SettingsMap> = {
+    -readonly [K in keyof T as T[K] extends {
+        default: SettingValueType;
+    } ? K : T[K]['required'] extends true ? K : never]: SettingType<T[K]>;
+} & {
+    -readonly [K in keyof T as T[K] extends {
+        default: SettingValueType;
+    } ? never : T[K]['required'] extends true ? never : K]?: SettingType<T[K]> | undefined;
+};
+export declare type BaseAdapterConfig = AdapterConfigFromSettings<BaseSettingsType>;
+export declare type AdapterConfig<T extends CustomSettingsType<T> = SettingsMap> = AdapterConfigFromSettings<T> & BaseAdapterConfig;
+export declare type CustomSettingsType<T = SettingsMap> = Record<keyof T, Setting>;
+export declare type EmptySettings = Record<string, never>;
+export declare type SettingsMap = Record<string, Setting>;
+export declare type ValidationErrorMessage = string | undefined;
+export {};