@chainlink/external-adapter-framework 0.4.0 → 0.5.1

package/rate-limiting/index.d.ts

@@ -0,0 +1,56 @@
+import { AdapterEndpoint, EndpointGenerics } from '../adapter';
+import { AdapterConfig } from '../config';
+export * from './background/fixed-frequency';
+export * from './request/simple-counting';
+export interface AdapterRateLimitTier {
+    rateLimit1s?: number;
+    rateLimit1m?: number;
+    rateLimit1h?: number;
+    note?: string;
+}
+/**
+ * Common interface for all RateLimiter classes to implement
+ */
+export interface RateLimiter {
+    /**
+     * Method to ensure all RateLimiters can be initialized in the same manner.
+     *
+     * @param limits - settings for how much throughput to allow for the Adapter
+     * @param endpoints - list of adapter endpoints
+     */
+    initialize<T extends EndpointGenerics>(endpoints: AdapterEndpoint<T>[], limits: AdapterRateLimitTier): this;
+}
+/**
+ * RequestRateLimiters perform checks agains imminent outbound requests for any transport.
+ */
+export interface RequestRateLimiter extends RateLimiter {
+    /**
+     * This method will check using whatever strategy is implemented to determine if
+     * this request can be processed. If so, it returns true; if not, returns false.
+     */
+    isUnderLimits(): boolean;
+}
+/**
+ * BackgroundExecuteFrequencyRateLimiters will implement custom logic to calculate
+ * the period of time to wait between background executions for a transport.
+ */
+export interface BackgroundExecuteRateLimiter extends RateLimiter {
+    msUntilNextExecution(endpointName: string): number;
+}
+/**
+ * This method will convert all possible settings for a rate limit tier and
+ * convert them all to requests per second, returning the lowest one
+ *
+ * @param limits - the rate limit tier set for the adapter
+ * @returns the most restrictive of the set options, in requests per second
+ */
+export declare const consolidateTierLimits: (limits?: AdapterRateLimitTier) => number;
+/**
+ * Validates rate limiting tiers specified for the adapter, and returns the one to use.
+ *
+ * @param config - the adapter config containing the env vars
+ * @param tiers - the adapter config listing the different available API tiers
+ * @returns the specified API tier, or a default one if none are specified
+ */
+export declare const getRateLimitingTier: (config: AdapterConfig, tiers?: Record<string, AdapterRateLimitTier>) => AdapterRateLimitTier | undefined;
+export declare const buildRateLimitTiersFromConfig: (config: AdapterConfig) => AdapterRateLimitTier | undefined;

package/rate-limiting/index.js

@@ -0,0 +1,86 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildRateLimitTiersFromConfig = exports.getRateLimitingTier = exports.consolidateTierLimits = void 0;
+__exportStar(require("./background/fixed-frequency"), exports);
+__exportStar(require("./request/simple-counting"), exports);
+/**
+ * This method will convert all possible settings for a rate limit tier and
+ * convert them all to requests per second, returning the lowest one
+ *
+ * @param limits - the rate limit tier set for the adapter
+ * @returns the most restrictive of the set options, in requests per second
+ */
+const consolidateTierLimits = (limits) => {
+    const perHourLimit = (limits?.rateLimit1h || Infinity) / (60 * 60);
+    const perMinuteLimit = (limits?.rateLimit1m || Infinity) / 60;
+    const perSecondLimit = limits?.rateLimit1s || Infinity;
+    return Math.min(perHourLimit, perMinuteLimit, perSecondLimit);
+};
+exports.consolidateTierLimits = consolidateTierLimits;
+/**
+ * Validates rate limiting tiers specified for the adapter, and returns the one to use.
+ *
+ * @param config - the adapter config containing the env vars
+ * @param tiers - the adapter config listing the different available API tiers
+ * @returns the specified API tier, or a default one if none are specified
+ */
+const getRateLimitingTier = (config, tiers) => {
+    if (config.RATE_LIMIT_CAPACITY ||
+        config.RATE_LIMIT_CAPACITY_MINUTE ||
+        config.RATE_LIMIT_CAPACITY_SECOND) {
+        return (0, exports.buildRateLimitTiersFromConfig)(config);
+    }
+    if (!tiers) {
+        return;
+    }
+    // Check that if the tiers object is defined, it has values
+    if (Object.values(tiers).length === 0) {
+        throw new Error(`The tiers object is defined, but has no entries`);
+    }
+    // Check that the tier set in the AdapterConfig is a valid one
+    const selectedTier = config.RATE_LIMIT_API_TIER;
+    if (selectedTier && !tiers[selectedTier]) {
+        const validTiersString = Object.keys(tiers)
+            .map((t) => `"${t}"`)
+            .join(', ');
+        throw new Error(`The selected rate limit tier "${selectedTier}" is not valid (can be one of ${validTiersString})`);
+    }
+    if (!selectedTier) {
+        // Sort the tiers by most to least restrictive
+        const sortedTiers = Object.values(tiers).sort((t1, t2) => (0, exports.consolidateTierLimits)(t1) - (0, exports.consolidateTierLimits)(t2));
+        return sortedTiers[0];
+    }
+    return tiers[selectedTier];
+};
+exports.getRateLimitingTier = getRateLimitingTier;
+// Creates adapter rate limit tier using the configs specified in env vars
+const buildRateLimitTiersFromConfig = (config) => {
+    const rateLimit1s = config.RATE_LIMIT_CAPACITY_SECOND;
+    let rateLimit1m;
+    if (config.RATE_LIMIT_CAPACITY_MINUTE) {
+        rateLimit1m = config.RATE_LIMIT_CAPACITY_MINUTE;
+    }
+    else if (config.RATE_LIMIT_CAPACITY) {
+        rateLimit1m = config.RATE_LIMIT_CAPACITY;
+    }
+    return {
+        rateLimit1s,
+        rateLimit1m,
+    };
+};
+exports.buildRateLimitTiersFromConfig = buildRateLimitTiersFromConfig;
+//# sourceMappingURL=index.js.map

package/rate-limiting/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/rate-limiting/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;AAGA,+DAA4C;AAC5C,4DAAyC;AA4CzC;;;;;;GAMG;AACI,MAAM,qBAAqB,GAAG,CAAC,MAA6B,EAAE,EAAE;IACrE,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,WAAW,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAA;IAClE,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,WAAW,IAAI,QAAQ,CAAC,GAAG,EAAE,CAAA;IAC7D,MAAM,cAAc,GAAG,MAAM,EAAE,WAAW,IAAI,QAAQ,CAAA;IACtD,OAAO,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,cAAc,EAAE,cAAc,CAAC,CAAA;AAC/D,CAAC,CAAA;AALY,QAAA,qBAAqB,yBAKjC;AAED;;;;;;GAMG;AACI,MAAM,mBAAmB,GAAG,CACjC,MAAqB,EACrB,KAA4C,EACV,EAAE;IACpC,IACE,MAAM,CAAC,mBAAmB;QAC1B,MAAM,CAAC,0BAA0B;QACjC,MAAM,CAAC,0BAA0B,EACjC;QACA,OAAO,IAAA,qCAA6B,EAAC,MAAM,CAAC,CAAA;KAC7C;IACD,IAAI,CAAC,KAAK,EAAE;QACV,OAAM;KACP;IAED,2DAA2D;IAC3D,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE;QACrC,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAA;KACnE;IAED,8DAA8D;IAC9D,MAAM,YAAY,GAAG,MAAM,CAAC,mBAAmB,CAAA;IAC/C,IAAI,YAAY,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE;QACxC,MAAM,gBAAgB,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;aACxC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC;aACpB,IAAI,CAAC,IAAI,CAAC,CAAA;QAEb,MAAM,IAAI,KAAK,CACb,iCAAiC,YAAY,iCAAiC,gBAAgB,GAAG,CAClG,CAAA;KACF;IAED,IAAI,CAAC,YAAY,EAAE;QACjB,8CAA8C;QAC9C,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAC3C,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,IAAA,6BAAqB,EAAC,EAAE,CAAC,GAAG,IAAA,6BAAqB,EAAC,EAAE,CAAC,CAClE,CAAA;QAED,OAAO,WAAW,CAAC,CAAC,CAAC,CAAA;KACtB;IAED,OAAO,KAAK,CAAC,YAAY,CAAC,CAAA;AAC5B,CAAC,CAAA;AA1CY,QAAA,mBAAmB,uBA0C/B;AAED,0EAA0E;AACnE,MAAM,6BAA6B,GAAG,CAC3C,MAAqB,EACa,EAAE;IACpC,MAAM,WAAW,GAAG,MAAM,CAAC,0BAA0B,CAAA;IACrD,IAAI,WAAW,CAAA;IACf,IAAI,MAAM,CAAC,0BAA0B,EAAE;QACrC,WAAW,GAAG,MAAM,CAAC,0BAA0B,CAAA;KAChD;SAAM,IAAI,MAAM,CAAC,mBAAmB,EAAE;QACrC,WAAW,GAAG,MAAM,CAAC,mBAAmB,CAAA;KACzC;IACD,OAAO;QACL,WAAW;QACX,WAAW;KACZ,CAAA;AACH,CAAC,CAAA;AAdY,QAAA,6BAA6B,iCAczC"}

package/rate-limiting/metrics.d.ts

@@ -0,0 +1,3 @@
+import * as client from 'prom-client';
+export declare const retrieveCost: <ProviderResponseBody>(data: ProviderResponseBody) => number;
+export declare const rateLimitCreditsSpentTotal: client.Counter<"feed_id" | "participant_id">;

package/rate-limiting/metrics.js

@@ -0,0 +1,45 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.rateLimitCreditsSpentTotal = exports.retrieveCost = void 0;
+const client = __importStar(require("prom-client"));
+// Retrieve cost field from response if exists
+// If not return default cost of 1
+const retrieveCost = (data) => {
+    const cost = data['cost'];
+    if (typeof cost === 'number' || typeof cost === 'string') {
+        return Number(cost);
+    }
+    else {
+        return 1;
+    }
+};
+exports.retrieveCost = retrieveCost;
+exports.rateLimitCreditsSpentTotal = new client.Counter({
+    name: 'rate_limit_credits_spent_total',
+    help: 'The number of data provider credits the adapter is consuming',
+    labelNames: ['participant_id', 'feed_id'],
+});
+//# sourceMappingURL=metrics.js.map

package/rate-limiting/metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../../src/rate-limiting/metrics.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,oDAAqC;AAErC,8CAA8C;AAC9C,kCAAkC;AAC3B,MAAM,YAAY,GAAG,CAAuB,IAA0B,EAAU,EAAE;IACvF,MAAM,IAAI,GAAI,IAAgC,CAAC,MAAM,CAAC,CAAA;IACtD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;QACxD,OAAO,MAAM,CAAC,IAAI,CAAC,CAAA;KACpB;SAAM;QACL,OAAO,CAAC,CAAA;KACT;AACH,CAAC,CAAA;AAPY,QAAA,YAAY,gBAOxB;AAEY,QAAA,0BAA0B,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IAC3D,IAAI,EAAE,gCAAgC;IACtC,IAAI,EAAE,8DAA8D;IACpE,UAAU,EAAE,CAAC,gBAAgB,EAAE,SAAS,CAAU;CACnD,CAAC,CAAA"}

package/rate-limiting/request/simple-counting.d.ts

@@ -0,0 +1,20 @@
+import { AdapterRateLimitTier, RateLimiter } from '..';
+import { AdapterEndpoint, EndpointGenerics } from '../../adapter';
+/**
+ * This rate limiter is the simplest stateful option.
+ * On startup, it'll compare the different thresholds for each tier, calculate them all
+ * in the finest window we'll use (seconds), and use the most restrictive one.
+ * This is so if the EA were to restart, we don't need to worry about persisting state
+ * for things like daily quotas. The downside is that this does not work well for bursty
+ * loads or spikes, in cases where e.g. the per second limit is high but daily quotas low.
+ */
+export declare class SimpleCountingRateLimiter implements RateLimiter {
+    latestSecondInterval: number;
+    requestsThisSecond: number;
+    latestMinuteInterval: number;
+    requestsThisMinute: number;
+    perSecondLimit: number;
+    perMinuteLimit: number;
+    initialize<T extends EndpointGenerics>(endpoints: AdapterEndpoint<T>[], limits?: AdapterRateLimitTier): this;
+    isUnderLimits(): boolean;
+}

package/rate-limiting/request/simple-counting.js

@@ -0,0 +1,63 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SimpleCountingRateLimiter = void 0;
+const util_1 = require("../../util");
+const logger = (0, util_1.makeLogger)('SimpleCountingRateLimiter');
+/**
+ * This rate limiter is the simplest stateful option.
+ * On startup, it'll compare the different thresholds for each tier, calculate them all
+ * in the finest window we'll use (seconds), and use the most restrictive one.
+ * This is so if the EA were to restart, we don't need to worry about persisting state
+ * for things like daily quotas. The downside is that this does not work well for bursty
+ * loads or spikes, in cases where e.g. the per second limit is high but daily quotas low.
+ */
+class SimpleCountingRateLimiter {
+    constructor() {
+        this.latestSecondInterval = 0;
+        this.requestsThisSecond = 0;
+        this.latestMinuteInterval = 0;
+        this.requestsThisMinute = 0;
+    }
+    initialize(endpoints, limits) {
+        // Translate the hourly limit into reqs per minute
+        const perHourLimit = (limits?.rateLimit1h || Infinity) / 60;
+        this.perMinuteLimit = Math.min(limits?.rateLimit1m || Infinity, perHourLimit);
+        this.perSecondLimit = limits?.rateLimit1s || Infinity;
+        logger.debug(`Using rate limiting settings: perMinute = ${this.perMinuteLimit} | perSecond: = ${this.perSecondLimit}`);
+        return this;
+    }
+    isUnderLimits() {
+        // If the limit is set to infinity, there was no tier limit specified
+        if (this.perSecondLimit === Infinity && this.perMinuteLimit === Infinity) {
+            return true;
+        }
+        const now = Date.now();
+        const nearestSecondInterval = Math.floor(now / 1000);
+        const nearestMinuteInterval = Math.floor(now / (1000 * 60));
+        // This should always run to completion, even if it doesn't look atomic; therefore the
+        // Ops should be "thread safe". Thank JS and its infinite single threaded dumbness.
+        if (nearestSecondInterval !== this.latestSecondInterval) {
+            logger.trace(`Clearing latest second interval, # of requests logged was: ${this.requestsThisSecond} `);
+            this.latestSecondInterval = nearestSecondInterval;
+            this.requestsThisSecond = 0;
+        }
+        if (nearestMinuteInterval !== this.latestMinuteInterval) {
+            logger.trace(`Clearing latest second minute, # of requests logged was: ${this.requestsThisMinute} `);
+            this.latestMinuteInterval = nearestMinuteInterval;
+            this.requestsThisMinute = 0;
+        }
+        if (this.requestsThisSecond < this.perSecondLimit &&
+            this.requestsThisMinute < this.perMinuteLimit) {
+            logger.trace('Request under limits, counting +1');
+            this.requestsThisSecond++;
+            this.requestsThisMinute++;
+            return true;
+        }
+        else {
+            logger.trace('Requests seen this interval are above limits');
+            return false;
+        }
+    }
+}
+exports.SimpleCountingRateLimiter = SimpleCountingRateLimiter;
+//# sourceMappingURL=simple-counting.js.map

package/rate-limiting/request/simple-counting.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"simple-counting.js","sourceRoot":"","sources":["../../../../src/rate-limiting/request/simple-counting.ts"],"names":[],"mappings":";;;AAEA,qCAAuC;AAEvC,MAAM,MAAM,GAAG,IAAA,iBAAU,EAAC,2BAA2B,CAAC,CAAA;AAEtD;;;;;;;GAOG;AACH,MAAa,yBAAyB;IAAtC;QACE,yBAAoB,GAAG,CAAC,CAAA;QACxB,uBAAkB,GAAG,CAAC,CAAA;QACtB,yBAAoB,GAAG,CAAC,CAAA;QACxB,uBAAkB,GAAG,CAAC,CAAA;IA4DxB,CAAC;IAxDC,UAAU,CACR,SAA+B,EAC/B,MAA6B;QAE7B,kDAAkD;QAClD,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,WAAW,IAAI,QAAQ,CAAC,GAAG,EAAE,CAAA;QAC3D,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,IAAI,QAAQ,EAAE,YAAY,CAAC,CAAA;QAC7E,IAAI,CAAC,cAAc,GAAG,MAAM,EAAE,WAAW,IAAI,QAAQ,CAAA;QACrD,MAAM,CAAC,KAAK,CACV,6CAA6C,IAAI,CAAC,cAAc,mBAAmB,IAAI,CAAC,cAAc,EAAE,CACzG,CAAA;QAED,OAAO,IAAI,CAAA;IACb,CAAC;IAED,aAAa;QACX,qEAAqE;QACrE,IAAI,IAAI,CAAC,cAAc,KAAK,QAAQ,IAAI,IAAI,CAAC,cAAc,KAAK,QAAQ,EAAE;YACxE,OAAO,IAAI,CAAA;SACZ;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACtB,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,CAAA;QACpD,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,CAAA;QAE3D,sFAAsF;QACtF,mFAAmF;QACnF,IAAI,qBAAqB,KAAK,IAAI,CAAC,oBAAoB,EAAE;YACvD,MAAM,CAAC,KAAK,CACV,8DAA8D,IAAI,CAAC,kBAAkB,GAAG,CACzF,CAAA;YACD,IAAI,CAAC,oBAAoB,GAAG,qBAAqB,CAAA;YACjD,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAA;SAC5B;QAED,IAAI,qBAAqB,KAAK,IAAI,CAAC,oBAAoB,EAAE;YACvD,MAAM,CAAC,KAAK,CACV,4DAA4D,IAAI,CAAC,kBAAkB,GAAG,CACvF,CAAA;YACD,IAAI,CAAC,oBAAoB,GAAG,qBAAqB,CAAA;YACjD,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAA;SAC5B;QAED,IACE,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,cAAc;YAC7C,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,cAAc,EAC7C;YACA,MAAM,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAA;YACjD,IAAI,CAAC,kBAAkB,EAAE,CAAA;YACzB,IAAI,CAAC,kBAAkB,EAAE,CAAA;YACzB,OAAO,IAAI,CAAA;SACZ;aAAM;YACL,MAAM,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAA;YAC5D,OAAO,KAAK,CAAA;SACb;IACH,CAAC;CACF;AAhED,8DAgEC"}

package/transports/batch-warming.d.ts

@@ -0,0 +1,53 @@
+import { AxiosRequestConfig, AxiosResponse } from 'axios';
+import { Cache } from '../cache';
+import { AdapterConfig } from '../config';
+import { BackgroundExecuteRateLimiter } from '../rate-limiting';
+import { SubscriptionSet } from '../util';
+import { AdapterRequest, AdapterResponse, ProviderResult } from '../util/request';
+import { Transport, TransportGenerics } from './';
+import { AdapterContext, AdapterDependencies } from '../adapter';
+/**
+ * Helper struct type that will be used to pass types to the generic parameters of a Transport.
+ * Extends the common TransportGenerics, adding Provider specific types for this Batch endpoint.
+ */
+declare type BatchWarmingTransportGenerics = TransportGenerics & {
+    /**
+     * Type details for any provider specific interfaces.
+     */
+    Provider: {
+        /**
+         * Structure of the body of the request that will be sent to the data provider.
+         */
+        RequestBody: unknown;
+        /**
+         * Structure for the body of the response coming from the data provider.
+         */
+        ResponseBody: unknown;
+    };
+};
+/**
+ * Transport implementation that takes incoming batches requests and keeps a warm cache of values.
+ * Within the setup function, adapter params are added to an set that also keeps track and expires values.
+ * In the background execute, the list of non-expired items in the set is fetched.
+ * Then, the list is passed through the `prepareRequest` function, that returns an AxiosRequestConfig.
+ * The Data Provider response is, they are passed through the `parseResponse` function to create a [[CacheEntry]] list.
+ * Finally, the items in that [[CacheEntry]] list are set in the Cache so the Adapter can fetch values from there.
+ *
+ * @typeParam T - all types related to the [[Transport]]
+ */
+export declare class BatchWarmingTransport<T extends BatchWarmingTransportGenerics> implements Transport<T> {
+    private config;
+    cache: Cache<AdapterResponse<T['Response']>>;
+    rateLimiter: BackgroundExecuteRateLimiter;
+    subscriptionSet: SubscriptionSet<T['Request']['Params']>;
+    WARMER_ACTIVE: boolean;
+    constructor(config: {
+        prepareRequest: (params: T['Request']['Params'][], config: AdapterConfig<T['CustomSettings']>) => AxiosRequestConfig<T['Provider']['RequestBody']>;
+        parseResponse: (params: T['Request']['Params'][], res: AxiosResponse<T['Provider']['ResponseBody']>, config: AdapterConfig<T['CustomSettings']>) => ProviderResult<T>[];
+    });
+    initialize(dependencies: AdapterDependencies, _: AdapterConfig<T['CustomSettings']>, endpointName: string): Promise<void>;
+    registerRequest(req: AdapterRequest<T['Request']>, config: AdapterConfig<T['CustomSettings']>): Promise<void>;
+    backgroundExecute(context: AdapterContext<T>): Promise<number>;
+    private makeRequest;
+}
+export {};

package/transports/batch-warming.js

@@ -0,0 +1,136 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BatchWarmingTransport = void 0;
+const util_1 = require("../util");
+const _1 = require("./");
+const util_2 = require("./util");
+const rateLimitMetrics = __importStar(require("../rate-limiting/metrics"));
+const cacheMetrics = __importStar(require("../cache/metrics"));
+const transportMetrics = __importStar(require("./metrics"));
+const error_1 = require("../validation/error");
+const WARMUP_BATCH_REQUEST_ID = '9002';
+const logger = (0, util_1.makeLogger)('BatchWarmingTransport');
+/**
+ * Transport implementation that takes incoming batches requests and keeps a warm cache of values.
+ * Within the setup function, adapter params are added to an set that also keeps track and expires values.
+ * In the background execute, the list of non-expired items in the set is fetched.
+ * Then, the list is passed through the `prepareRequest` function, that returns an AxiosRequestConfig.
+ * The Data Provider response is, they are passed through the `parseResponse` function to create a [[CacheEntry]] list.
+ * Finally, the items in that [[CacheEntry]] list are set in the Cache so the Adapter can fetch values from there.
+ *
+ * @typeParam T - all types related to the [[Transport]]
+ */
+class BatchWarmingTransport {
+    constructor(config) {
+        this.config = config;
+        // Flag used to track whether the warmer has moved from having no entries to having some and vice versa
+        // Used for recording the cache warmer active metrics accurately
+        this.WARMER_ACTIVE = false;
+    }
+    async initialize(dependencies, _, endpointName) {
+        this.cache = dependencies.cache;
+        this.rateLimiter = dependencies.backgroundExecuteRateLimiter;
+        this.subscriptionSet = dependencies.subscriptionSetFactory.buildSet(endpointName);
+    }
+    // This might need coalescing to avoid too frequent ttl updates
+    async registerRequest(req, config) {
+        logger.debug(`Adding entry to subscription set (ttl ${config.WARMUP_SUBSCRIPTION_TTL}): [${req.requestContext.cacheKey}] = ${req.requestContext.data}`);
+        if (config.BATCH_TRANSPORT_SETUP_VALIDATION) {
+            const response = await this.makeRequest([req.requestContext.data], config);
+            if (!response.results.length) {
+                throw new error_1.AdapterError({
+                    statusCode: 200,
+                    providerStatusCode: response.providerResponse.status,
+                    message: response.providerResponse.message ||
+                        'There was an error while validating the incoming request before adding to the batch subscription set',
+                });
+            }
+        }
+        await this.subscriptionSet.add(req.requestContext.cacheKey, req.requestContext.data, config.WARMUP_SUBSCRIPTION_TTL);
+    }
+    async backgroundExecute(context) {
+        logger.debug('Starting background execute');
+        const entries = await this.subscriptionSet.getAll();
+        // Keep track of active subscriptions for background execute
+        transportMetrics.bgExecuteSubscriptionSetCount
+            .labels({ endpoint: context.adapterEndpoint.name, transport_type: 'batch' })
+            .set(entries.length);
+        const interval = this.rateLimiter.msUntilNextExecution(context.adapterEndpoint.name);
+        if (!entries.length) {
+            logger.debug('No entries in batch warming set, skipping');
+            if (this.WARMER_ACTIVE) {
+                // Decrement count when warmer changed from having entries to having none
+                cacheMetrics.cacheWarmerCount.labels({ isBatched: 'true' }).dec();
+                this.WARMER_ACTIVE = false;
+            }
+            return interval;
+        }
+        else if (this.WARMER_ACTIVE === false) {
+            // Increment count when warmer changed from having no entries to having some
+            cacheMetrics.cacheWarmerCount.labels({ isBatched: 'true' }).inc();
+            this.WARMER_ACTIVE = true;
+        }
+        const response = await this.makeRequest(entries, context.adapterConfig);
+        if (!response.results?.length) {
+            return interval;
+        }
+        const cacheEntries = (0, _1.buildCacheEntriesFromResults)(response.results, context);
+        logger.debug('Setting adapter responses in cache');
+        await this.cache.setMany(cacheEntries, context.adapterConfig.CACHE_MAX_AGE);
+        // Record cost of data provider call
+        const cost = rateLimitMetrics.retrieveCost(response.providerResponse.data);
+        rateLimitMetrics.rateLimitCreditsSpentTotal
+            .labels({
+            feed_id: 'N/A',
+            participant_id: WARMUP_BATCH_REQUEST_ID,
+        })
+            .inc(cost);
+        return interval;
+    }
+    async makeRequest(entries, config) {
+        logger.trace(`Have ${entries.length} entries in batch, preparing request...`);
+        const request = this.config.prepareRequest(entries, config);
+        logger.trace(`Sending request to data provider: ${JSON.stringify(request)}`);
+        let providerResponse;
+        try {
+            providerResponse = await (0, util_2.axiosRequest)(request, config);
+        }
+        catch (e) {
+            logger.warn(`There was an error while performing the batch request: ${e}`);
+            return {
+                results: [],
+                providerResponse: e,
+            };
+        }
+        logger.debug(`Got response from provider, parsing (raw body: ${providerResponse.data})`);
+        return {
+            results: this.config.parseResponse(entries, providerResponse, config),
+            providerResponse,
+        };
+    }
+}
+exports.BatchWarmingTransport = BatchWarmingTransport;
+//# sourceMappingURL=batch-warming.js.map

package/transports/batch-warming.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"batch-warming.js","sourceRoot":"","sources":["../../../src/transports/batch-warming.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAIA,kCAAqD;AAErD,yBAA+E;AAC/E,iCAAqC;AACrC,2EAA4D;AAC5D,+DAAgD;AAEhD,4DAA6C;AAC7C,+CAAkD;AAElD,MAAM,uBAAuB,GAAG,MAAM,CAAA;AAEtC,MAAM,MAAM,GAAG,IAAA,iBAAU,EAAC,uBAAuB,CAAC,CAAA;AAuBlD;;;;;;;;;GASG;AACH,MAAa,qBAAqB;IAWhC,YACU,MAUP;QAVO,WAAM,GAAN,MAAM,CAUb;QAfH,uGAAuG;QACvG,gEAAgE;QAChE,kBAAa,GAAG,KAAK,CAAA;IAclB,CAAC;IAEJ,KAAK,CAAC,UAAU,CACd,YAAiC,EACjC,CAAqC,EACrC,YAAoB;QAEpB,IAAI,CAAC,KAAK,GAAG,YAAY,CAAC,KAA8C,CAAA;QACxE,IAAI,CAAC,WAAW,GAAG,YAAY,CAAC,4BAA4B,CAAA;QAC5D,IAAI,CAAC,eAAe,GAAG,YAAY,CAAC,sBAAsB,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAA;IACnF,CAAC;IAED,+DAA+D;IAC/D,KAAK,CAAC,eAAe,CACnB,GAAiC,EACjC,MAA0C;QAE1C,MAAM,CAAC,KAAK,CACV,yCAAyC,MAAM,CAAC,uBAAuB,OAAO,GAAG,CAAC,cAAc,CAAC,QAAQ,OAAO,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAC1I,CAAA;QAED,IAAI,MAAM,CAAC,gCAAgC,EAAE;YAC3C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAA;YAE1E,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE;gBAC5B,MAAM,IAAI,oBAAY,CAAC;oBACrB,UAAU,EAAE,GAAG;oBACf,kBAAkB,EAAE,QAAQ,CAAC,gBAAgB,CAAC,MAAM;oBACpD,OAAO,EACJ,QAAQ,CAAC,gBAAqC,CAAC,OAAO;wBACvD,sGAAsG;iBACzG,CAAC,CAAA;aACH;SACF;QAED,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,CAC5B,GAAG,CAAC,cAAc,CAAC,QAAQ,EAC3B,GAAG,CAAC,cAAc,CAAC,IAAI,EACvB,MAAM,CAAC,uBAAuB,CAC/B,CAAA;IACH,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,OAA0B;QAChD,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAA;QAC3C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAA;QAEnD,4DAA4D;QAC5D,gBAAgB,CAAC,6BAA6B;aAC3C,MAAM,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,eAAe,CAAC,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC;aAC3E,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;QAEtB,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,oBAAoB,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,CAAA;QAEpF,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YACnB,MAAM,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAA;YACzD,IAAI,IAAI,CAAC,aAAa,EAAE;gBACtB,yEAAyE;gBACzE,YAAY,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAA;gBACjE,IAAI,CAAC,aAAa,GAAG,KAAK,CAAA;aAC3B;YACD,OAAO,QAAQ,CAAA;SAChB;aAAM,IAAI,IAAI,CAAC,aAAa,KAAK,KAAK,EAAE;YACvC,4EAA4E;YAC5E,YAAY,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAA;YACjE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAA;SAC1B;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,aAAa,CAAC,CAAA;QAEvE,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE;YAC7B,OAAO,QAAQ,CAAA;SAChB;QAED,MAAM,YAAY,GAAG,IAAA,+BAA4B,EAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QAE5E,MAAM,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAA;QAClD,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC,aAAa,CAAC,aAAa,CAAC,CAAA;QAE3E,oCAAoC;QACpC,MAAM,IAAI,GAAG,gBAAgB,CAAC,YAAY,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAA;QAC1E,gBAAgB,CAAC,0BAA0B;aACxC,MAAM,CAAC;YACN,OAAO,EAAE,KAAK;YACd,cAAc,EAAE,uBAAuB;SACxC,CAAC;aACD,GAAG,CAAC,IAAI,CAAC,CAAA;QAEZ,OAAO,QAAQ,CAAA;IACjB,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,OAAiC,EACjC,MAA0C;QAK1C,MAAM,CAAC,KAAK,CAAC,QAAQ,OAAO,CAAC,MAAM,yCAAyC,CAAC,CAAA;QAC7E,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QAE3D,MAAM,CAAC,KAAK,CAAC,qCAAqC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;QAC5E,IAAI,gBAAgB,CAAA;QACpB,IAAI;YACF,gBAAgB,GAAG,MAAM,IAAA,mBAAY,EAInC,OAAO,EAAE,MAAM,CAAC,CAAA;SACnB;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,IAAI,CAAC,0DAA0D,CAAC,EAAE,CAAC,CAAA;YAC1E,OAAO;gBACL,OAAO,EAAE,EAAE;gBACX,gBAAgB,EAAE,CAAkB;aACrC,CAAA;SACF;QAED,MAAM,CAAC,KAAK,CAAC,kDAAkD,gBAAgB,CAAC,IAAI,GAAG,CAAC,CAAA;QACxF,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,gBAAgB,EAAE,MAAM,CAAC;YACrE,gBAAgB;SACjB,CAAA;IACH,CAAC;CACF;AAjJD,sDAiJC"}

package/transports/index.d.ts

@@ -0,0 +1,108 @@
+import { AdapterContext, AdapterDependencies } from '../adapter';
+import { Cache, CacheEntry } from '../cache';
+import { AdapterConfig, SettingsMap } from '../config';
+import { AdapterRequest, AdapterResponse, ProviderResult, RequestGenerics, ResponseGenerics } from '../util/request';
+export * from './batch-warming';
+export * from './rest';
+export * from './sse';
+export * from './websocket';
+/**
+ * Helper struct type that will be used to pass types to the generic parameters of a Transport.
+ * Within the transport, given a generic param T extends TransportGenerics, these will be used like so:
+ *
+ * const request: AdapterRequest\<T['Request']\> = ...
+ *
+ * (ignore the slash characters, they're just for proper doc syntax \<)
+ */
+export declare type TransportGenerics = {
+    /**
+     * Type details about incoming Adapter requests to this Transport
+     */
+    Request: RequestGenerics;
+    /**
+     * Type details about outbound responses from this Transport
+     */
+    Response: ResponseGenerics;
+    /**
+     * Type for any custom settings used for this Transport
+     */
+    CustomSettings: SettingsMap;
+};
+/**
+ * Generic interface for a Transport.
+ * A Transport defines the way in which an AdapterEndpoint will process incoming requests to
+ * fetch data from a Data Provider. The setup phase will take care of the former, while the
+ * backgroundExecute will be in charge of the latter.
+ * This separation gives us the ability of splitting these concerns, and optionally parallelizing
+ * the reading and writing of data to a centralized Cache.
+ *
+ * @typeParam T - Helper struct type that will be used to pass types to the generic parameters (check [[TransportGenerics]])
+ */
+export interface Transport<T extends TransportGenerics> {
+    cache?: Cache<AdapterResponse<T['Response']>>;
+    /**
+     * Initializes the transport in the Adapter context.
+     *
+     * @param dependencies - Adapter dependencies (e.g. cache instance)
+     * @param config - Adapter config containing env vars
+     * @returns an empty Promise
+     */
+    initialize: (dependencies: AdapterDependencies, config: AdapterConfig<T['CustomSettings']>, endpointName: string) => Promise<void>;
+    /**
+     * Registers a request within the context of the transport.
+     * This means things like adding the request to a subscription set.
+     *
+     * @param req - the incoming AdapterRequest
+     * @param config - common configuration for the Adapter as a whole
+     * @returns an empty Promise
+     */
+    registerRequest?: (req: AdapterRequest<T['Request']>, config: AdapterConfig<T['CustomSettings']>) => Promise<void>;
+    /**
+     * Performs a synchronous fetch/processing of information within the lifecycle of an incoming request.
+     * This is mostly intended to be used as a backwards compatibility mechanism, or for very specific cases where
+     * processing is expected to be very simple. Things like rate limiting for example won't be able to be applied
+     * to this method with more than one EA instance. It's heavily encouraged to instead rework adapter logic
+     * to perform as much of the work as possible (or all of it) in the backgroundExecute method.
+     *
+     * @param req - the incoming AdapterRequest
+     * @param config - common configuration for the Adapter as a whole
+     * @returns a Promise that _optionally_ returns an AdapterResponse, if the Transport has the capability of
+     *   immediately fetching data and returning it without the background process.
+     */
+    foregroundExecute?: (req: AdapterRequest<T['Request']>, config: AdapterConfig<T['CustomSettings']>) => Promise<AdapterResponse<{
+        Data: T['Response']['Data'];
+        Result: T['Response']['Result'];
+    }> | void>;
+    /**
+     * Main function that will be called by the main background loop of the writer EA.
+     * All logic fetching from providers should live here.
+     *
+     * @param context - background context for the execution (e.g. endpoint name)
+     * @returns an empty Promise
+     */
+    backgroundExecute?: (context: AdapterContext<T>) => Promise<number>;
+}
+/**
+ * Generic interface for a Transport that contains or manages other transports.
+ * Some transports, such as the RoutingTransport, do not directly fetch data from a Data Provider, but
+ * instead coordinate the execution of other transports for fetching and transforming data.
+ * These transports have alternative initialization procedures, which typically involve
+ * initializing each downstream transport.
+ *
+ * @typeParam Params - the structure of the AdapterRequest's body
+ * @typeParam Result - the structure of the AdapterResponse's body
+ * @typeParam CustomSettings - custom settings for the Adapter, or the default SettingsMap
+ */
+export interface MetaTransport<T extends TransportGenerics> extends Transport<T> {
+    transports: {
+        [key: string]: Transport<T>;
+    };
+}
+/**
+ * Helper method to build cache entries to set after getting a bunch of responses from a DP.
+ *
+ * @param results - a list of results coming from a DataProvider
+ * @param context - context for the Adapter
+ * @returns a list of CacheEntries of AdapterResponses
+ */
+export declare const buildCacheEntriesFromResults: <T extends TransportGenerics>(results: ProviderResult<T>[], context: AdapterContext<T>) => CacheEntry<AdapterResponse<T["Response"]>>[];

package/transports/index.js

@@ -0,0 +1,56 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildCacheEntriesFromResults = void 0;
+const cache_1 = require("../cache");
+__exportStar(require("./batch-warming"), exports);
+__exportStar(require("./rest"), exports);
+__exportStar(require("./sse"), exports);
+__exportStar(require("./websocket"), exports);
+/**
+ * Helper method to build cache entries to set after getting a bunch of responses from a DP.
+ *
+ * @param results - a list of results coming from a DataProvider
+ * @param context - context for the Adapter
+ * @returns a list of CacheEntries of AdapterResponses
+ */
+const buildCacheEntriesFromResults = (results, context) => results.map((r) => {
+    const cacheEntry = {
+        key: (0, cache_1.calculateCacheKey)(context, r.params),
+        value: {
+            result: r.value,
+            statusCode: 200,
+            data: {
+                result: r.value,
+            },
+        },
+    };
+    if (context.adapterConfig.METRICS_ENABLED &&
+        context.adapterConfig.EXPERIMENTAL_METRICS_ENABLED) {
+        const metrics = {
+            maxAge: Date.now() + context.adapterConfig.CACHE_MAX_AGE,
+            meta: {
+                metrics: {
+                    feedId: (0, cache_1.calculateFeedId)(context, r.params),
+                },
+            },
+        };
+        cacheEntry.value = { ...cacheEntry.value, ...metrics };
+    }
+    return cacheEntry;
+});
+exports.buildCacheEntriesFromResults = buildCacheEntriesFromResults;
+//# sourceMappingURL=index.js.map