@chainlink/external-adapter-framework 0.0.10 → 0.0.14

package/examples/bank-frick/util.js

@@ -0,0 +1,39 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateJWT = void 0;
+const crypto_1 = __importDefault(require("crypto"));
+const axios_1 = __importDefault(require("axios"));
+const util_1 = require("../../util");
+const logger = (0, util_1.makeLogger)('BankFrickUtil');
+// Used by all endpoints requiring authentication
+const generateJWT = async (config, signingAlgorithm = 'rsa-sha512') => {
+    logger.info("Generating a new JWT because we don't have one in config.token");
+    const { API_KEY, PRIVATE_KEY, API_ENDPOINT } = config;
+    // All of these are required, so validation should have failed prior to this line
+    if (!API_KEY || !PRIVATE_KEY) {
+        throw new Error('API_KEY, PRIVATE_KEY, and PASSWORD all must be defined to get a new token\n' +
+            'Received:  \n' +
+            `API_KEY: ${API_KEY}\n` +
+            `PRIVATE_KEY: ${PRIVATE_KEY}`);
+    }
+    const data = {
+        key: API_KEY,
+    };
+    const signature = crypto_1.default.sign(signingAlgorithm, Buffer.from(JSON.stringify(data)), PRIVATE_KEY);
+    const options = {
+        method: 'POST',
+        baseURL: API_ENDPOINT,
+        url: `authorize`,
+        headers: {
+            Signature: signature.toString('base64'),
+            algorithm: signingAlgorithm,
+        },
+        data,
+    };
+    const response = await axios_1.default.request(options);
+    return response.data.token;
+};
+exports.generateJWT = generateJWT;

package/index.d.ts

@@ -1,6 +1,5 @@
 import { Server } from 'http';
-import { Adapter } from './adapter';
-import { AdapterDependencies } from './transports';
+import { Adapter, AdapterDependencies } from './adapter';
 /**
  * Main function for the framework.
  * Initializes config and dependencies, uses those to initialize Transports, and starts listening for requests.

package/index.js

@@ -13,6 +13,7 @@ const config_1 = require("./config");
 const metrics_1 = require("./metrics");
 const transports_1 = require("./transports");
 const util_1 = require("./util");
+const test_payload_loader_1 = require("./util/test-payload-loader");
 const validation_1 = require("./validation");
 const logger = (0, util_1.makeLogger)('Main');
 const VERSION = process.env['npm_package_version'];
@@ -83,10 +84,50 @@ async function buildRestApi(config, initializedAdapter) {
             url: config.BASE_URL,
             method: 'POST',
             handler: transportHandler,
-        }); // Pass config maybe? cleaner that multiple instances / dependencies
+        });
         if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
             router.addHook('onResponse', metrics_1.buildMetricsMiddleware);
         }
     });
+    // Add smoke endpoint after middleware which are needed for tests
+    buildSmokeEndpoint(app, config);
     return app;
 }
+/**
+ * Adds the /smoke endpoint to the API for smoke testing the adapter
+ *
+ * @param app - the Fastify instance
+ * @param config - the initialized adapter config
+ */
+function buildSmokeEndpoint(app, config) {
+    const testPayload = (0, test_payload_loader_1.loadTestPayload)(config.SMOKE_TEST_PAYLOAD_FILE_NAME);
+    app.get((0, path_1.join)(config.BASE_URL, 'smoke'), async (_, res) => {
+        if (testPayload.isDefault) {
+            return res.status(200).send('OK');
+        }
+        const errors = [];
+        for (const index in testPayload.requests) {
+            try {
+                const request = { id: index, data: testPayload.requests[index] };
+                // Use Fastify's app inject to pass smoke requests internally
+                const response = await app.inject({
+                    method: 'POST',
+                    url: '/',
+                    payload: request,
+                });
+                const parsedResponse = JSON.parse(response.body);
+                // Throw error if not 2xx status code
+                if (parsedResponse.statusCode < 200 || parsedResponse.statusCode > 299) {
+                    throw Error('Smoke test request failed');
+                }
+            }
+            catch (e) {
+                errors.push(e);
+            }
+        }
+        if (errors.length > 0) {
+            return res.status(500).send(errors);
+        }
+        return res.status(200).send('OK');
+    });
+}

package/metrics/index.js

@@ -86,7 +86,6 @@ const buildHttpRequestMetricsLabel = (feedId, error, cacheHit) => {
     }
     else if (error instanceof Error) {
         // If error present and not instance of generic Error, unexpected failure occurred
-        // TODO: Build label with 500 error and withold dp status? Status sent as 200 in error handler
         labels.type = constants_1.HttpRequestType.ADAPTER_ERROR;
         labels.status_code = 500;
     }

package/metrics/util.d.ts

@@ -1,3 +1,7 @@
 import { AdapterMetricsMeta, AdapterRequestData } from '../util';
 import { AdapterEndpoint } from '../adapter';
-export declare const getMetricsMeta: (endpoint: AdapterEndpoint, data: AdapterRequestData) => AdapterMetricsMeta;
+import { AdapterConfig } from '../config';
+export declare const getMetricsMeta: ({ adapterEndpoint, adapterConfig, }: {
+    adapterEndpoint: AdapterEndpoint;
+    adapterConfig: AdapterConfig;
+}, data: AdapterRequestData) => AdapterMetricsMeta;

package/metrics/util.js

@@ -2,8 +2,8 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getMetricsMeta = void 0;
 const cache_1 = require("../cache");
-const getMetricsMeta = (endpoint, data) => {
-    const feedId = (0, cache_1.calculateFeedId)(endpoint, data);
+const getMetricsMeta = ({ adapterEndpoint, adapterConfig, }, data) => {
+    const feedId = (0, cache_1.calculateFeedId)({ adapterEndpoint, adapterConfig }, data);
     return { feedId };
 };
 exports.getMetricsMeta = getMetricsMeta;

package/package/adapter.d.ts

@@ -0,0 +1,88 @@
+import { AdapterConfig, BaseAdapterConfig, SettingsMap } from './config';
+import { AdapterRateLimitTier } from './rate-limiting';
+import { AdapterDependencies, Transport } from './transports';
+import { InputParameters } from './validation/input-params';
+/**
+ * Structure to describe rate limits specs for the Adapter
+ */
+interface AdapterRateLimitingConfig {
+    /** Adapter rate limits, gotten from the specific tier requested */
+    tiers: Record<string, AdapterRateLimitTier>;
+}
+/**
+ * Main structure of an External Adapter
+ */
+export interface Adapter {
+    /** Name of the adapter */
+    name: string;
+    /** If present, the string that will be used for requests with no specified endpoint */
+    defaultEndpoint?: string;
+    /** List of [[AdapterEndpoint]]s in the adapter */
+    endpoints: AdapterEndpoint[];
+    /** Map of overrides to the default config values for an Adapter */
+    envDefaultOverrides?: Partial<BaseAdapterConfig>;
+    /** List of custom env vars for this particular adapter (e.g. RPC_URL) */
+    customSettings?: SettingsMap;
+    /** Configuration relevant to outbound (EA --\> DP) communication rate limiting */
+    rateLimiting?: AdapterRateLimitingConfig;
+    /** Overrides for converting the 'base' parameter that are hardcoded into the adapter. */
+    overrides?: Record<string, string>;
+}
+/**
+ * Structure to describe rate limits specs for a specific adapter endpoint
+ */
+export interface EndpointRateLimitingConfig {
+    /**
+     * How much of the total limit for the adapter will be assigned to this specific endpoint.
+     * Should be a non-zero positive number up to 100.
+     * Endpoints in the same adapter without a specific allocation will divide the remaining limits equally.
+     */
+    allocationPercentage: number;
+}
+/**
+ * Structure to describe a specific endpoint in an [[Adapter]]
+ */
+export interface AdapterEndpoint {
+    /** Name that will be used to match input params to this endpoint (case insensitive) */
+    name: string;
+    /** List of alternative endpoint names that will resolve to this same transport (case insensitive) */
+    aliases?: string[];
+    /** Transport that will be used to handle data processing and communication for this endpoint */
+    transport: Transport<any, any, any>;
+    /** Specification of what the body of a request hitting this endpoint should look like (used for validation) */
+    inputParameters: InputParameters;
+    /** Specific details related to the rate limiting for this endpoint in particular */
+    rateLimiting?: EndpointRateLimitingConfig;
+}
+/**
+ * Structure to describe an adapter that has been initialized
+ */
+export interface InitializedAdapter extends Adapter {
+    /** Object containing alias translations for all endpoints */
+    endpointsMap: Record<string, AdapterEndpoint>;
+    /** Initialized dependencies that the adapter will use */
+    dependencies: AdapterDependencies;
+    /** Configuration params for various adapter properties */
+    config: AdapterConfig;
+}
+/**
+ * This function will process dependencies for an adapter, such as caches or rate limiters,
+ * in order to inject them into transports and other relevant places later in the lifecycle.
+ *
+ * @param config - the configuration for this adapter
+ * @param inputDependencies - a partial obj of initialized dependencies to override the created ones
+ * @param rateLimitingConfig - details from the adapter regarding rate limiting
+ * @returns a set of AdapterDependencies all initialized
+ */
+export declare const initializeDependencies: (adapter: Adapter, config: AdapterConfig, inputDependencies?: Partial<AdapterDependencies>) => AdapterDependencies;
+/**
+ * Initializes all of the [[Transport]]s in the adapter, passing along any [[AdapterDependencies]] and [[AdapterConfig]].
+ * Additionally, it builds a map out of all the endpoint names and aliases (checking for duplicates).
+ *
+ * @param adapter - an instance of an Adapter
+ * @param dependencies - dependencies that the adapter will need at initialization
+ * @param config - configuration variables already processed and validated
+ * @returns - the adapter with all transports initialized and aliases map built
+ */
+export declare const initializeAdapter: (adapter: Adapter, config: AdapterConfig, dependencies?: Partial<AdapterDependencies>) => Promise<InitializedAdapter>;
+export {};

package/package/adapter.js

@@ -0,0 +1,112 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.initializeAdapter = exports.initializeDependencies = void 0;
+const factory_1 = require("./cache/factory");
+const rate_limiting_1 = require("./rate-limiting");
+const util_1 = require("./util");
+const logger = (0, util_1.makeLogger)('Adapter');
+/**
+ * This function will take an adapter structure and go through each endpoint, calculating
+ * each one's allocation of the total rate limits that are set for the adapter as a whole.
+ *
+ * @param adapter - the adapter to initialize rate limiting for
+ */
+const calculateRateLimitAllocations = (adapter) => {
+    const numberOfEndpoints = adapter.endpoints.length;
+    const endpointsWithExplicitAllocations = adapter.endpoints.filter((e) => e.rateLimiting);
+    const totalExplicitAllocation = endpointsWithExplicitAllocations
+        .map((e) => e.rateLimiting?.allocationPercentage || 0)
+        .reduce((sum, next) => sum + next, 0);
+    if (totalExplicitAllocation > 100) {
+        throw new Error('The total allocation set for all endpoints summed cannot exceed 100%');
+    }
+    if (totalExplicitAllocation === 100 &&
+        numberOfEndpoints - endpointsWithExplicitAllocations.length > 0) {
+        throw new Error('The explicit allocation is at 100% but there are endpoints with implicit allocation');
+    }
+    const implicitAllocation = 100 - totalExplicitAllocation;
+    logger.debug('Adapter rate limit allocations:');
+    for (const endpoint of adapter.endpoints) {
+        if (!endpoint.rateLimiting) {
+            endpoint.rateLimiting = {
+                allocationPercentage: implicitAllocation / (numberOfEndpoints - endpointsWithExplicitAllocations.length),
+            };
+        }
+        logger.debug(`Endpoint [${endpoint.name}] - ${endpoint.rateLimiting?.allocationPercentage}%`);
+    }
+};
+/**
+ * This function will process dependencies for an adapter, such as caches or rate limiters,
+ * in order to inject them into transports and other relevant places later in the lifecycle.
+ *
+ * @param config - the configuration for this adapter
+ * @param inputDependencies - a partial obj of initialized dependencies to override the created ones
+ * @param rateLimitingConfig - details from the adapter regarding rate limiting
+ * @returns a set of AdapterDependencies all initialized
+ */
+const initializeDependencies = (adapter, config, inputDependencies) => {
+    const dependencies = inputDependencies || {};
+    if (!dependencies.cache) {
+        dependencies.cache = factory_1.CacheFactory.buildCache(config);
+    }
+    // In the future we might want something more complex, but for now it's better to simplify
+    // and just use the same rate limiting for everything. Once we have a more complex use case we
+    // can think of ways to make this more configurable.
+    const rateLimitingTier = (0, rate_limiting_1.getRateLimitingTier)(adapter.rateLimiting?.tiers, config.RATE_LIMIT_API_TIER);
+    if (!dependencies.requestRateLimiter) {
+        dependencies.requestRateLimiter = new rate_limiting_1.SimpleCountingRateLimiter().initialize(adapter.endpoints, rateLimitingTier);
+    }
+    if (!dependencies.backgroundExecuteRateLimiter) {
+        dependencies.backgroundExecuteRateLimiter = new rate_limiting_1.FixedFrequencyRateLimiter().initialize(adapter.endpoints, rateLimitingTier);
+    }
+    return dependencies;
+};
+exports.initializeDependencies = initializeDependencies;
+/**
+ * Takes an adapter and normalizes all endpoint names and aliases, as well as the default endpoint.
+ * i.e. makes them lowercase for now
+ * @param adapter - an instance of an Adapter
+ */
+const normalizeEndpointNames = (adapter) => {
+    // Make endpoints case insensitive, including default
+    adapter.defaultEndpoint = adapter.defaultEndpoint?.toLowerCase();
+    for (const endpoint of adapter.endpoints) {
+        endpoint.name = endpoint.name.toLowerCase();
+        endpoint.aliases = endpoint.aliases?.map((a) => a.toLowerCase());
+    }
+};
+/**
+ * Initializes all of the [[Transport]]s in the adapter, passing along any [[AdapterDependencies]] and [[AdapterConfig]].
+ * Additionally, it builds a map out of all the endpoint names and aliases (checking for duplicates).
+ *
+ * @param adapter - an instance of an Adapter
+ * @param dependencies - dependencies that the adapter will need at initialization
+ * @param config - configuration variables already processed and validated
+ * @returns - the adapter with all transports initialized and aliases map built
+ */
+const initializeAdapter = async (adapter, config, dependencies) => {
+    normalizeEndpointNames(adapter);
+    calculateRateLimitAllocations(adapter);
+    const initializedDependencies = (0, exports.initializeDependencies)(adapter, config, dependencies);
+    const endpointsMap = {};
+    for (const endpoint of adapter.endpoints) {
+        // Add aliases to map to use in validation
+        const aliases = [endpoint.name, ...(endpoint.aliases || [])];
+        for (const alias of aliases) {
+            if (endpointsMap[alias]) {
+                throw new Error(`Duplicate endpoint / alias: "${alias}"`);
+            }
+            endpointsMap[alias] = endpoint;
+        }
+        logger.debug(`Initializing transport for endpoint "${endpoint.name}"...`);
+        await endpoint.transport.initialize(initializedDependencies);
+    }
+    logger.debug('Adapter initialization complete.');
+    return {
+        ...adapter,
+        endpointsMap,
+        dependencies: initializedDependencies,
+        config,
+    };
+};
+exports.initializeAdapter = initializeAdapter;

package/package/background-executor.d.ts

@@ -0,0 +1,11 @@
+/// <reference types="node" />
+import { Server } from 'http';
+import { InitializedAdapter } from './adapter';
+/**
+ * Very simple background loop that will call the [[Transport.backgroundExecute]] functions in all Transports.
+ * It gets the time in ms to wait as the return value from those functions, and sleeps until next execution.
+ *
+ * @param adapter - an initialized External Adapter
+ * @param server - the http server to attach an on close listener to
+ */
+export declare function callBackgroundExecutes(adapter: InitializedAdapter, server?: Server): Promise<void>;

package/package/background-executor.js

@@ -0,0 +1,45 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.callBackgroundExecutes = void 0;
+const util_1 = require("./util");
+const logger = (0, util_1.makeLogger)('BackgroundExecutor');
+/**
+ * Very simple background loop that will call the [[Transport.backgroundExecute]] functions in all Transports.
+ * It gets the time in ms to wait as the return value from those functions, and sleeps until next execution.
+ *
+ * @param adapter - an initialized External Adapter
+ * @param server - the http server to attach an on close listener to
+ */
+async function callBackgroundExecutes(adapter, server) {
+    // Set up variable to check later on to see if we need to stop this background "thread"
+    // If no server is provided, the listener won't be set and serverClosed will always be false
+    let serverClosed = false;
+    server?.on('close', () => {
+        serverClosed = true;
+    });
+    for (const endpoint of adapter.endpoints) {
+        const backgroundExecute = endpoint.transport.backgroundExecute?.bind(endpoint.transport);
+        if (!backgroundExecute) {
+            logger.debug(`Endpoint "${endpoint.name}" has no background execute, skipping...`);
+            continue;
+        }
+        const context = {
+            adapterEndpoint: endpoint,
+            adapterConfig: adapter.config,
+        };
+        const handler = async () => {
+            if (serverClosed) {
+                logger.info('Server closed, stopping recursive backgroundExecute handler chain');
+                return;
+            }
+            logger.debug(`Calling background execute for endpoint "${endpoint.name}"`);
+            const timeToWait = await backgroundExecute(context);
+            logger.debug(`Finished background execute for endpoint "${endpoint.name}", sleeping for ${timeToWait}ms`);
+            await (0, util_1.sleep)(timeToWait);
+            handler();
+        };
+        // Start recursive async calls
+        handler();
+    }
+}
+exports.callBackgroundExecutes = callBackgroundExecutes;

package/package/cache/factory.d.ts

@@ -0,0 +1,6 @@
+import { AdapterConfig } from '../config';
+import { LocalCache } from './local';
+import { RedisCache } from './redis';
+export declare class CacheFactory {
+    static buildCache(config: AdapterConfig): LocalCache<unknown> | RedisCache<unknown> | undefined;
+}

package/package/cache/factory.js

@@ -0,0 +1,57 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CacheFactory = void 0;
+const ioredis_1 = __importDefault(require("ioredis"));
+const util_1 = require("../util");
+const local_1 = require("./local");
+const cacheMetrics = __importStar(require("./metrics"));
+const redis_1 = require("./redis");
+const logger = (0, util_1.makeLogger)('CacheFactory');
+class CacheFactory {
+    static buildCache(config) {
+        logger.info(`Using "${config.CACHE_TYPE}" cache.`);
+        if (config.CACHE_TYPE === 'local') {
+            return new local_1.LocalCache();
+        }
+        else if (config.CACHE_TYPE === 'redis') {
+            const redis = new ioredis_1.default({
+                enableAutoPipelining: true,
+                host: config.CACHE_REDIS_HOST,
+                port: config.CACHE_REDIS_PORT,
+            });
+            redis.on('connect', () => {
+                cacheMetrics.redisConnectionsOpen.inc();
+            });
+            // TODO: Maybe track active redis connections instead of just total count of connections created
+            // Use a Gauge and track with a combo of connect/end events listeners
+            return new redis_1.RedisCache(redis);
+        }
+    }
+}
+exports.CacheFactory = CacheFactory;

package/package/cache/index.d.ts

@@ -0,0 +1,90 @@
+import { AdapterEndpoint } from '../adapter';
+import { AdapterConfig } from '../config';
+import { AdapterMiddlewareBuilder, AdapterResponse, sleep } from '../util';
+export * from './local';
+export * from './redis';
+/**
+ * An object describing an entry in the cache.
+ * @typeParam T - the type of the entry's value
+ */
+export interface CacheEntry<T> {
+    key: string;
+    value: T;
+}
+/**
+ * Generic interface for a local or remote Cache.
+ * @typeParam T - the type of the cache entries' values
+ */
+export interface Cache<T = unknown> {
+    /**
+     * Gets an item from the Cache.
+     *
+     * @param key - the key of the desired entry for which to fetch its value
+     * @returns a Promise of the entry's value, or undefined if not found / expired.
+     */
+    get: (key: string) => Promise<T | undefined>;
+    /**
+     * Sets an item in the Cache.
+     *
+     * @param key - the key of the new entry
+     * @param value - the value of the new entry
+     * @param ttl - the time in milliseconds until the entry expires
+     * @returns an empty Promise that resolves when the entry has been set
+     */
+    set: (key: string, value: T, ttl: number) => Promise<void>;
+    /**
+     * Sets a list of items in the Cache.
+     *
+     * @param entries - a list of cache entries
+     * @param ttl - the time in milliseconds until the entries expire
+     * @returns an empty Promise that resolves when all entries have been set
+     */
+    setMany: (entries: CacheEntry<T>[], ttl: number) => Promise<void>;
+    /**
+     * Deletes the specified item from the Cache
+     *
+     * @param key - the key of the entry to be deleted
+     * @returns an empty Promise that resolves when the entry has been deleted
+     */
+    delete: (key: string) => Promise<void>;
+}
+export declare const calculateCacheKey: ({ adapterEndpoint, adapterConfig, }: {
+    adapterEndpoint: AdapterEndpoint;
+    adapterConfig: AdapterConfig;
+}, data: unknown) => string;
+export declare const calculateFeedId: (adapterEndpoint: AdapterEndpoint, data: unknown) => string;
+/**
+ * Calculates a unique key from the provided data.
+ *
+ * @param data - the request data/body, i.e. the adapter input params
+ * @param paramNames - the keys from adapter endpoint input parameters
+ * @returns the calculated unique key
+ *
+ * @example
+ * ```
+ * calculateKey({ base: 'ETH', quote: 'BTC' }, ['base','quote'])
+ * // equals `|base:eth|quote:btc`
+ * ```
+ */
+export declare const calculateKey: (data: unknown, paramNames: string[]) => string;
+/**
+ * Polls the provided Cache for an AdapterResponse set in the provided key. If the maximum
+ * amount of retries is exceeded, it returns undefined instead.
+ *
+ * @param cache - a Cache instance
+ * @param key - the key generated from an AdapterRequest that corresponds to the desired AdapterResponse
+ * @param retry - current retry, only for internal use
+ * @returns the AdapterResponse if found, else undefined
+ */
+export declare const pollResponseFromCache: (cache: Cache<AdapterResponse>, key: string, options: {
+    maxRetries: number;
+    sleep: number;
+}, retry?: number) => Promise<AdapterResponse | undefined>;
+/**
+ * Given a Cache instance in the adapter dependencies, builds a middleware function that will perform
+ * a get from said Cache and return that if found; otherwise it'll continue the middleware chain.
+ *
+ * @param adapter - an initialized adapter
+ * @returns the cache middleware function
+ */
+export declare const buildCacheMiddleware: AdapterMiddlewareBuilder;