@chainlink/external-adapter-framework 0.0.12 → 0.0.14

package/adapter.d.ts

@@ -0,0 +1,107 @@
+import { Cache } from './cache';
+import { AdapterConfig, BaseAdapterConfig, CustomSettingsType, SettingsMap } from './config';
+import { AdapterRateLimitTier, BackgroundExecuteRateLimiter, RequestRateLimiter } from './rate-limiting';
+import { Transport } from './transports';
+import { SubscriptionSetFactory } from './util';
+import { InputParameters } from './validation/input-params';
+/**
+ * Dependencies that will be injected into the Adapter on startup
+ */
+export interface AdapterDependencies {
+    cache: Cache;
+    requestRateLimiter: RequestRateLimiter;
+    backgroundExecuteRateLimiter: BackgroundExecuteRateLimiter;
+    subscriptionSetFactory: SubscriptionSetFactory;
+}
+/**
+ * Context that will be used on background executions of a Transport.
+ * For example, the endpointName used to log statements or generate Cache keys.
+ */
+export interface AdapterContext<CustomSettings extends CustomSettingsType<CustomSettings> = SettingsMap> {
+    adapterEndpoint: AdapterEndpoint;
+    adapterConfig: AdapterConfig<CustomSettings>;
+}
+/**
+ * Structure to describe rate limits specs for the Adapter
+ */
+interface AdapterRateLimitingConfig {
+    /** Adapter rate limits, gotten from the specific tier requested */
+    tiers: Record<string, AdapterRateLimitTier>;
+}
+/**
+ * Main structure of an External Adapter
+ */
+export interface Adapter {
+    /** Name of the adapter */
+    name: string;
+    /** If present, the string that will be used for requests with no specified endpoint */
+    defaultEndpoint?: string;
+    /** List of [[AdapterEndpoint]]s in the adapter */
+    endpoints: AdapterEndpoint[];
+    /** Map of overrides to the default config values for an Adapter */
+    envDefaultOverrides?: Partial<BaseAdapterConfig>;
+    /** List of custom env vars for this particular adapter (e.g. RPC_URL) */
+    customSettings?: SettingsMap;
+    /** Configuration relevant to outbound (EA --\> DP) communication rate limiting */
+    rateLimiting?: AdapterRateLimitingConfig;
+    /** Overrides for converting the 'base' parameter that are hardcoded into the adapter. */
+    overrides?: Record<string, string>;
+}
+/**
+ * Structure to describe rate limits specs for a specific adapter endpoint
+ */
+export interface EndpointRateLimitingConfig {
+    /**
+     * How much of the total limit for the adapter will be assigned to this specific endpoint.
+     * Should be a non-zero positive number up to 100.
+     * Endpoints in the same adapter without a specific allocation will divide the remaining limits equally.
+     */
+    allocationPercentage: number;
+}
+/**
+ * Structure to describe a specific endpoint in an [[Adapter]]
+ */
+export interface AdapterEndpoint {
+    /** Name that will be used to match input params to this endpoint (case insensitive) */
+    name: string;
+    /** List of alternative endpoint names that will resolve to this same transport (case insensitive) */
+    aliases?: string[];
+    /** Transport that will be used to handle data processing and communication for this endpoint */
+    transport: Transport<any, any, any>;
+    /** Specification of what the body of a request hitting this endpoint should look like (used for validation) */
+    inputParameters: InputParameters;
+    /** Specific details related to the rate limiting for this endpoint in particular */
+    rateLimiting?: EndpointRateLimitingConfig;
+}
+/**
+ * Structure to describe an adapter that has been initialized
+ */
+export interface InitializedAdapter extends Adapter {
+    /** Object containing alias translations for all endpoints */
+    endpointsMap: Record<string, AdapterEndpoint>;
+    /** Initialized dependencies that the adapter will use */
+    dependencies: AdapterDependencies;
+    /** Configuration params for various adapter properties */
+    config: AdapterConfig;
+}
+/**
+ * This function will process dependencies for an adapter, such as caches or rate limiters,
+ * in order to inject them into transports and other relevant places later in the lifecycle.
+ *
+ * @param config - the configuration for this adapter
+ * @param inputDependencies - a partial obj of initialized dependencies to override the created ones
+ * @param rateLimitingConfig - details from the adapter regarding rate limiting
+ * @returns a set of AdapterDependencies all initialized
+ */
+export declare const initializeDependencies: (adapter: Adapter, config: AdapterConfig, inputDependencies?: Partial<AdapterDependencies>) => AdapterDependencies;
+/**
+ * Initializes all of the [[Transport]]s in the adapter, passing along any [[AdapterDependencies]] and [[AdapterConfig]].
+ * Additionally, it builds a map out of all the endpoint names and aliases (checking for duplicates).
+ *
+ * @param adapter - an instance of an Adapter
+ * @param dependencies - dependencies that the adapter will need at initialization
+ * @param config - configuration variables already processed and validated
+ * @returns - the adapter with all transports initialized and aliases map built
+ */
+export declare const initializeAdapter: (adapter: Adapter, config: AdapterConfig, dependencies?: Partial<AdapterDependencies>) => Promise<InitializedAdapter>;
+export {};

package/adapter.js

@@ -0,0 +1,115 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.initializeAdapter = exports.initializeDependencies = void 0;
+const cache_1 = require("./cache");
+const rate_limiting_1 = require("./rate-limiting");
+const util_1 = require("./util");
+const logger = (0, util_1.makeLogger)('Adapter');
+/**
+ * This function will take an adapter structure and go through each endpoint, calculating
+ * each one's allocation of the total rate limits that are set for the adapter as a whole.
+ *
+ * @param adapter - the adapter to initialize rate limiting for
+ */
+const calculateRateLimitAllocations = (adapter) => {
+    const numberOfEndpoints = adapter.endpoints.length;
+    const endpointsWithExplicitAllocations = adapter.endpoints.filter((e) => e.rateLimiting);
+    const totalExplicitAllocation = endpointsWithExplicitAllocations
+        .map((e) => e.rateLimiting?.allocationPercentage || 0)
+        .reduce((sum, next) => sum + next, 0);
+    if (totalExplicitAllocation > 100) {
+        throw new Error('The total allocation set for all endpoints summed cannot exceed 100%');
+    }
+    if (totalExplicitAllocation === 100 &&
+        numberOfEndpoints - endpointsWithExplicitAllocations.length > 0) {
+        throw new Error('The explicit allocation is at 100% but there are endpoints with implicit allocation');
+    }
+    const implicitAllocation = 100 - totalExplicitAllocation;
+    logger.debug('Adapter rate limit allocations:');
+    for (const endpoint of adapter.endpoints) {
+        if (!endpoint.rateLimiting) {
+            endpoint.rateLimiting = {
+                allocationPercentage: implicitAllocation / (numberOfEndpoints - endpointsWithExplicitAllocations.length),
+            };
+        }
+        logger.debug(`Endpoint [${endpoint.name}] - ${endpoint.rateLimiting?.allocationPercentage}%`);
+    }
+};
+/**
+ * This function will process dependencies for an adapter, such as caches or rate limiters,
+ * in order to inject them into transports and other relevant places later in the lifecycle.
+ *
+ * @param config - the configuration for this adapter
+ * @param inputDependencies - a partial obj of initialized dependencies to override the created ones
+ * @param rateLimitingConfig - details from the adapter regarding rate limiting
+ * @returns a set of AdapterDependencies all initialized
+ */
+const initializeDependencies = (adapter, config, inputDependencies) => {
+    const dependencies = inputDependencies || {};
+    if (!dependencies.cache) {
+        dependencies.cache = cache_1.CacheFactory.buildCache(config);
+    }
+    // In the future we might want something more complex, but for now it's better to simplify
+    // and just use the same rate limiting for everything. Once we have a more complex use case we
+    // can think of ways to make this more configurable.
+    const rateLimitingTier = (0, rate_limiting_1.getRateLimitingTier)(adapter.rateLimiting?.tiers, config.RATE_LIMIT_API_TIER);
+    if (!dependencies.requestRateLimiter) {
+        dependencies.requestRateLimiter = new rate_limiting_1.SimpleCountingRateLimiter().initialize(adapter.endpoints, rateLimitingTier);
+    }
+    if (!dependencies.backgroundExecuteRateLimiter) {
+        dependencies.backgroundExecuteRateLimiter = new rate_limiting_1.FixedFrequencyRateLimiter().initialize(adapter.endpoints, rateLimitingTier);
+    }
+    if (!dependencies.subscriptionSetFactory) {
+        dependencies.subscriptionSetFactory = new util_1.SubscriptionSetFactory(config);
+    }
+    return dependencies;
+};
+exports.initializeDependencies = initializeDependencies;
+/**
+ * Takes an adapter and normalizes all endpoint names and aliases, as well as the default endpoint.
+ * i.e. makes them lowercase for now
+ * @param adapter - an instance of an Adapter
+ */
+const normalizeEndpointNames = (adapter) => {
+    // Make endpoints case insensitive, including default
+    adapter.defaultEndpoint = adapter.defaultEndpoint?.toLowerCase();
+    for (const endpoint of adapter.endpoints) {
+        endpoint.name = endpoint.name.toLowerCase();
+        endpoint.aliases = endpoint.aliases?.map((a) => a.toLowerCase());
+    }
+};
+/**
+ * Initializes all of the [[Transport]]s in the adapter, passing along any [[AdapterDependencies]] and [[AdapterConfig]].
+ * Additionally, it builds a map out of all the endpoint names and aliases (checking for duplicates).
+ *
+ * @param adapter - an instance of an Adapter
+ * @param dependencies - dependencies that the adapter will need at initialization
+ * @param config - configuration variables already processed and validated
+ * @returns - the adapter with all transports initialized and aliases map built
+ */
+const initializeAdapter = async (adapter, config, dependencies) => {
+    normalizeEndpointNames(adapter);
+    calculateRateLimitAllocations(adapter);
+    const initializedDependencies = (0, exports.initializeDependencies)(adapter, config, dependencies);
+    const endpointsMap = {};
+    for (const endpoint of adapter.endpoints) {
+        // Add aliases to map to use in validation
+        const aliases = [endpoint.name, ...(endpoint.aliases || [])];
+        for (const alias of aliases) {
+            if (endpointsMap[alias]) {
+                throw new Error(`Duplicate endpoint / alias: "${alias}"`);
+            }
+            endpointsMap[alias] = endpoint;
+        }
+        logger.debug(`Initializing transport for endpoint "${endpoint.name}"...`);
+        await endpoint.transport.initialize(initializedDependencies);
+    }
+    logger.debug('Adapter initialization complete.');
+    return {
+        ...adapter,
+        endpointsMap,
+        dependencies: initializedDependencies,
+        config,
+    };
+};
+exports.initializeAdapter = initializeAdapter;

package/{dist/src/cache → cache}/index.d.ts

@@ -1,5 +1,5 @@
 import { AdapterEndpoint } from '../adapter';
-import { AdapterConfig, SettingsMap } from '../config';
+import { AdapterConfig } from '../config';
 import { AdapterMiddlewareBuilder, AdapterResponse, sleep } from '../util';
 export * from './local';
 export * from './redis';
@@ -49,13 +49,13 @@ export interface Cache<T = unknown> {
      */
     delete: (key: string) => Promise<void>;
 }
-export declare const calculateCacheKey: <Params, Result, CustomSettings extends SettingsMap>({ adapterEndpoint, adapterConfig, }: {
-    adapterEndpoint: AdapterEndpoint<Params, Result, CustomSettings>;
-    adapterConfig: AdapterConfig<CustomSettings>;
+export declare const calculateCacheKey: ({ adapterEndpoint, adapterConfig, }: {
+    adapterEndpoint: AdapterEndpoint;
+    adapterConfig: AdapterConfig;
 }, data: unknown) => string;
-export declare const calculateFeedId: <Params, Result, CustomSettings extends SettingsMap>({ adapterEndpoint, adapterConfig, }: {
-    adapterEndpoint: AdapterEndpoint<Params, Result, CustomSettings>;
-    adapterConfig: AdapterConfig<CustomSettings>;
+export declare const calculateFeedId: ({ adapterEndpoint, adapterConfig, }: {
+    adapterEndpoint: AdapterEndpoint;
+    adapterConfig: AdapterConfig;
 }, data: unknown) => string;
 /**
  * Calculates a unique key from the provided data.
@@ -70,7 +70,7 @@ export declare const calculateFeedId: <Params, Result, CustomSettings extends Se
  * // equals `|base:eth|quote:btc`
  * ```
  */
-export declare const calculateKey: <CustomSettings extends SettingsMap>(data: unknown, paramNames: string[], adapterConfig: AdapterConfig<CustomSettings>) => string;
+export declare const calculateKey: (data: unknown, paramNames: string[], adapterConfig: AdapterConfig) => string;
 /**
  * Polls the provided Cache for an AdapterResponse set in the provided key. If the maximum
  * amount of retries is exceeded, it returns undefined instead.

package/{dist/src/config → config}/index.d.ts

@@ -80,11 +80,6 @@ export declare const BaseSettings: {
         readonly description: "TTL for batch warmer subscriptions";
         readonly default: 10000;
     };
-    readonly WS_SUBSCRIPTION_TTL: {
-        readonly description: "";
-        readonly type: "number";
-        readonly default: 120000;
-    };
     readonly CACHE_POLLING_MAX_RETRIES: {
         readonly description: "Max amount of times to attempt to find EA response in the cache after the Transport has been set up";
         readonly type: "number";

package/{dist/src/config → config}/index.js

@@ -4,6 +4,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.buildAdapterConfig = exports.BaseSettings = void 0;
 exports.BaseSettings = {
     // V2 compat
+    // TODO: Remove non used in v3 ones
     API_ENDPOINT: {
         description: 'The URL that the certain transports use to retrieve data',
         type: 'string',
@@ -258,11 +259,10 @@ exports.BaseSettings = {
     // WS_SUBSCRIPTION_PRIORITY_LIST: {
     //   type: 'string',
     // },
-    WS_SUBSCRIPTION_TTL: {
-        description: '',
-        type: 'number',
-        default: 120000,
-    },
+    // WS_SUBSCRIPTION_TTL: {
+    //   type: 'number',
+    //   default: 120000,
+    // },
     // WS_SUBSCRIPTION_UNRESPONSIVE_TTL: {
     //   type: 'number',
     //   default: false,

package/{dist/src/config → config}/provider-limits.js

@@ -42,7 +42,11 @@ const getProviderLimits = (provider, limits, tier, protocol) => {
     return limitsConfig;
 };
 const parseLimits = (limits) => {
-    const _mapObject = (fn) => (o) => Object.fromEntries(Object.entries(o).map(fn));
+    const _mapObject = (fn) => (o) => {
+        const entries = Object.entries(o);
+        const mapped = entries.map(fn);
+        return Object.fromEntries(mapped);
+    };
     const _formatProtocol = _mapObject((entry) => {
         const [tierName, rest] = entry;
         return [tierName.toLowerCase(), { ...rest }];

package/{dist/src/examples → examples}/bank-frick/accounts.d.ts

@@ -36,4 +36,4 @@ export declare class BankFrickAccountsTransport implements Transport<AdapterInpu
      */
     setup(req: AdapterRequest<AdapterInputParameters>, config: AdapterConfig<typeof customSettings>): Promise<AdapterResponse<number>>;
 }
-export declare const accountsRestEndpoint: AdapterEndpoint<AdapterInputParameters, number, import("../../config").SettingsMap>;
+export declare const accountsRestEndpoint: AdapterEndpoint;

package/{dist/src/examples → examples}/bank-frick/accounts.js

@@ -8,7 +8,6 @@ const axios_1 = __importDefault(require("axios"));
 const util_1 = require("./util");
 const util_2 = require("../../util");
 const error_1 = require("../../validation/error");
-const adapter_1 = require("../../adapter");
 const logger = (0, util_2.makeLogger)('BankFrickTransport');
 // Note: this is a shallow pattern that only checks for a country code since IBANs in the sandbox are invalid
 const ibanPattern = /^[A-Z]{2}[A-Z\d]{14,30}$/;
@@ -185,8 +184,8 @@ class BankFrickAccountsTransport {
     }
 }
 exports.BankFrickAccountsTransport = BankFrickAccountsTransport;
-exports.accountsRestEndpoint = new adapter_1.AdapterEndpoint({
+exports.accountsRestEndpoint = {
     name: 'accounts',
     transport: new BankFrickAccountsTransport(),
     inputParameters,
-});
+};

package/{dist/src/examples → examples}/bank-frick/index.js

@@ -1,10 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.adapter = void 0;
-const adapter_1 = require("../../adapter");
 const accounts_1 = require("./accounts");
 const config_1 = require("./config");
-exports.adapter = new adapter_1.Adapter({
+exports.adapter = {
     name: 'bank-frick',
     defaultEndpoint: 'accounts',
     endpoints: [accounts_1.accountsRestEndpoint],
@@ -12,4 +11,4 @@ exports.adapter = new adapter_1.Adapter({
     envDefaultOverrides: {
         API_ENDPOINT: 'https://olbsandbox.bankfrick.li/webapi/v2',
     },
-});
+};

package/examples/ncfx/index.d.ts

@@ -0,0 +1,2 @@
+import { Adapter } from '../../adapter';
+export declare const adapter: Adapter;

package/{dist/src/index.js → index.js}

@@ -6,8 +6,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.expose = void 0;
 const fastify_1 = __importDefault(require("fastify"));
 const path_1 = require("path");
+const adapter_1 = require("./adapter");
 const background_executor_1 = require("./background-executor");
 const cache_1 = require("./cache");
+const config_1 = require("./config");
 const metrics_1 = require("./metrics");
 const transports_1 = require("./transports");
 const util_1 = require("./util");
@@ -24,18 +26,22 @@ const VERSION = process.env['npm_package_version'];
  * @returns a Promise that resolves to the http.Server listening for connections
  */
 const expose = async (adapter, dependencies) => {
+    const config = (0, config_1.buildAdapterConfig)({
+        overrides: adapter.envDefaultOverrides,
+        customSettings: adapter.customSettings,
+    });
     // Initialize adapter (create dependencies, inject them, build endpoint map, etc.)
-    await adapter.initialize(dependencies);
+    const initializedAdapter = await (0, adapter_1.initializeAdapter)(adapter, config, dependencies);
     let server = undefined;
-    if (adapter.config.METRICS_ENABLED && adapter.config.EXPERIMENTAL_METRICS_ENABLED) {
-        (0, metrics_1.setupMetricsServer)(adapter.name, adapter.config);
+    if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
+        (0, metrics_1.setupMetricsServer)(adapter.name, config);
     }
-    if (adapter.config.EA_MODE === 'reader' || adapter.config.EA_MODE === 'reader-writer') {
+    if (config.EA_MODE === 'reader' || config.EA_MODE === 'reader-writer') {
         // Main REST API server to handle incoming requests
-        const app = await buildRestApi(adapter);
+        const app = await buildRestApi(config, initializedAdapter);
         // Start listening for incoming requests
         try {
-            await app.listen(adapter.config.EA_PORT, adapter.config.EA_HOST);
+            await app.listen(config.EA_PORT, config.EA_HOST);
         }
         catch (err) {
             logger.fatal(`There was an error when starting the EA server: ${err}`);
@@ -47,10 +53,10 @@ const expose = async (adapter, dependencies) => {
     else {
         logger.info('REST API is disabled; this instance will not process incoming requests.');
     }
-    if (adapter.config.EA_MODE === 'writer' || adapter.config.EA_MODE === 'reader-writer') {
+    if (config.EA_MODE === 'writer' || config.EA_MODE === 'reader-writer') {
         // Start background loop that will take care of calling any async Transports
         logger.info('Starting background execution loop');
-        (0, background_executor_1.callBackgroundExecutes)(adapter, server);
+        (0, background_executor_1.callBackgroundExecutes)(initializedAdapter, server);
     }
     else {
         logger.info('Background executor is disabled; this instance will not perform async background executes.');
@@ -58,8 +64,7 @@ const expose = async (adapter, dependencies) => {
     return server;
 };
 exports.expose = expose;
-async function buildRestApi(adapter) {
-    const { config } = adapter;
+async function buildRestApi(config, initializedAdapter) {
     const app = (0, fastify_1.default)();
     // Add healthcheck endpoint before middlewares to bypass them
     app.get((0, path_1.join)(config.BASE_URL, 'health'), (req, res) => {
@@ -67,11 +72,11 @@ async function buildRestApi(adapter) {
     });
     // Use global error handling
     app.setErrorHandler(validation_1.errorCatchingMiddleware);
-    const transportHandler = await (0, transports_1.buildTransportHandler)(adapter);
+    const transportHandler = await (0, transports_1.buildTransportHandler)(initializedAdapter);
     app.register(async (router) => {
         // Set up "middlewares" (hooks in fastify)
-        router.addHook('preHandler', (0, validation_1.validatorMiddleware)(adapter));
-        router.addHook('preHandler', (0, cache_1.buildCacheMiddleware)(adapter));
+        router.addHook('preHandler', (0, validation_1.validatorMiddleware)(initializedAdapter));
+        router.addHook('preHandler', (0, cache_1.buildCacheMiddleware)(initializedAdapter));
         if (config['CORRELATION_ID_ENABLED']) {
             router.addHook('onRequest', util_1.loggingContextMiddleware);
         }

package/metrics/util.d.ts

@@ -0,0 +1,7 @@
+import { AdapterMetricsMeta, AdapterRequestData } from '../util';
+import { AdapterEndpoint } from '../adapter';
+import { AdapterConfig } from '../config';
+export declare const getMetricsMeta: ({ adapterEndpoint, adapterConfig, }: {
+    adapterEndpoint: AdapterEndpoint;
+    adapterConfig: AdapterConfig;
+}, data: AdapterRequestData) => AdapterMetricsMeta;

package/{dist/src → package}/background-executor.d.ts

@@ -1,6 +1,6 @@
 /// <reference types="node" />
 import { Server } from 'http';
-import { Adapter } from './adapter';
+import { InitializedAdapter } from './adapter';
 /**
  * Very simple background loop that will call the [[Transport.backgroundExecute]] functions in all Transports.
  * It gets the time in ms to wait as the return value from those functions, and sleeps until next execution.
@@ -8,4 +8,4 @@ import { Adapter } from './adapter';
  * @param adapter - an initialized External Adapter
  * @param server - the http server to attach an on close listener to
  */
-export declare function callBackgroundExecutes(adapter: Adapter, server?: Server): Promise<void>;
+export declare function callBackgroundExecutes(adapter: InitializedAdapter, server?: Server): Promise<void>;

package/package/examples/coingecko/batch-warming.d.ts

@@ -0,0 +1,2 @@
+import { AdapterEndpoint } from '../../adapter';
+export declare const batchEndpoint: AdapterEndpoint;

package/{dist/src → package}/examples/coingecko/batch-warming.js

@@ -1,7 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.batchEndpoint = void 0;
-const adapter_1 = require("../../adapter");
 const batch_warming_1 = require("../../transports/batch-warming");
 const DEFAULT_URL = 'https://pro-api.coingecko.com/api/v3';
 const inputParameters = {
@@ -46,8 +45,8 @@ const batchEndpointTransport = new batch_warming_1.BatchWarmingTransport({
         return entries;
     },
 });
-exports.batchEndpoint = new adapter_1.AdapterEndpoint({
+exports.batchEndpoint = {
     name: 'batch',
     transport: batchEndpointTransport,
     inputParameters,
-});
+};

package/package/examples/coingecko/index.d.ts

@@ -0,0 +1,2 @@
+import { Adapter } from '../../adapter';
+export declare const adapter: Adapter;

package/{dist/src → package}/examples/coingecko/index.js

@@ -1,11 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.adapter = void 0;
-const adapter_1 = require("../../adapter");
 const batch_warming_1 = require("./batch-warming");
 const rest_1 = require("./rest");
-exports.adapter = new adapter_1.Adapter({
+exports.adapter = {
     name: 'coingecko',
     defaultEndpoint: 'batch',
     endpoints: [rest_1.restEndpoint, batch_warming_1.batchEndpoint],
-});
+};