@chainlink/external-adapter-framework 0.0.15 → 0.0.17

package/cache/redis.js

@@ -0,0 +1,100 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RedisCache = void 0;
+const util_1 = require("../util");
+const cacheMetrics = __importStar(require("./metrics"));
+const logger = (0, util_1.makeLogger)('RedisCache');
+/**
+ * Redis implementation of a Cache. It uses a simple js Object, storing entries with both
+ * a value and an expiration timestamp. Expired entries are deleted on reads (i.e. no background gc/upkeep).
+ *
+ * @typeParam T - the type for the entries' values
+ */
+class RedisCache {
+    constructor(client) {
+        this.client = client;
+    }
+    async get(key) {
+        logger.trace(`Getting key ${key}`);
+        const value = await this.client.get(key);
+        // Record get command sent to Redis
+        cacheMetrics.redisCommandsSentCount
+            .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'get' })
+            .inc();
+        if (!value) {
+            logger.debug(`No entry in redis cache for key "${key}", returning undefined`);
+            return undefined;
+        }
+        return JSON.parse(value);
+    }
+    async delete(key) {
+        logger.trace(`Deleting key ${key}`);
+        await this.client.del(key);
+        // Record delete command sent to Redis
+        cacheMetrics.redisCommandsSentCount
+            .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'delete' })
+            .inc();
+    }
+    async set(key, value, ttl) {
+        logger.trace(`Setting key ${key}`);
+        await this.client.set(key, JSON.stringify(value), 'PX', ttl);
+        // Only record metrics if feed Id is present, otherwise assuming value is not adapter response to record
+        const feedId = value.meta?.metrics?.feedId;
+        if (feedId) {
+            // Record cache set count, max age, and staleness (set to 0 for cache set)
+            const label = cacheMetrics.cacheMetricsLabel(key, feedId, cacheMetrics.CacheTypes.Redis);
+            cacheMetrics.cacheSet(label, ttl);
+        }
+        // Record set command sent to Redis
+        cacheMetrics.redisCommandsSentCount
+            .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'set' })
+            .inc();
+    }
+    async setMany(entries, ttl) {
+        logger.trace(`Setting a bunch of keys`);
+        // Unfortunately, there's no ttl for mset
+        let chain = this.client.multi();
+        for (const entry of entries) {
+            chain = chain.set(entry.key, JSON.stringify(entry.value), 'PX', ttl);
+        }
+        await chain.exec();
+        // Loop again, but this time to record these in metrics
+        for (const entry of entries) {
+            // Only record metrics if feed Id is present, otherwise assuming value is not adapter response to record
+            const feedId = entry.value.meta?.metrics?.feedId;
+            if (feedId) {
+                // Record cache set count, max age, and staleness (set to 0 for cache set)
+                const label = cacheMetrics.cacheMetricsLabel(entry.key, feedId, cacheMetrics.CacheTypes.Redis);
+                cacheMetrics.cacheSet(label, ttl);
+            }
+        }
+        // Record setMany command sent to Redis
+        cacheMetrics.redisCommandsSentCount
+            .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'exec' })
+            .inc();
+    }
+}
+exports.RedisCache = RedisCache;

package/config/index.js

@@ -0,0 +1,366 @@
+"use strict";
+// Import { getRandomRequiredEnv, getRandomEnv, getEnv } from '../util'
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildAdapterConfig = exports.BaseSettings = void 0;
+exports.BaseSettings = {
+    // V2 compat
+    // ADAPTER_URL: {
+    //   description: 'The URL of another adapter from which data needs to be retrieved',
+    //   type: 'string',
+    // },
+    API_ENDPOINT: {
+        description: 'The URL that certain transports use to retrieve data',
+        type: 'string',
+    },
+    API_KEY: {
+        description: 'Default setting for an EA key',
+        type: 'string',
+    },
+    API_TIMEOUT: {
+        description: 'The number of milliseconds a request can be pending before returning a timeout error for data provider request',
+        type: 'number',
+        default: 30000,
+    },
+    API_VERBOSE: {
+        description: 'Toggle whether the response from the EA should contain just the results or also include the full response body from the queried API.',
+        type: 'boolean',
+        default: false,
+    },
+    BASE_URL: {
+        description: 'Starting path for the EA handler endpoint',
+        type: 'string',
+        default: '/',
+    },
+    CACHE_MAX_AGE: {
+        description: 'Maximum amount of time (in ms) that a response will stay cached',
+        type: 'number',
+        default: 90000,
+    },
+    // CACHE_MAX_ITEMS: {
+    //   type: 'number',
+    //   default: 1000,
+    // },
+    // CACHE_MIN_AGE: {
+    //   type: 'number',
+    //   default: 30000,
+    // },
+    // CACHE_REDIS_CONNECTION_TIMEOUT: {
+    //   type: 'number',
+    //   default: 15000,
+    // },
+    CACHE_REDIS_HOST: {
+        description: 'Hostname for the Redis instance to be used',
+        type: 'string',
+        default: '127.0.0.1',
+    },
+    // CACHE_REDIS_MAX_QUEUED_ITEMS: {
+    //   description: 'Maximum length of the client internal command queue',
+    //   type: 'number',
+    //   default: 500,
+    // },
+    // CACHE_REDIS_MAX_RECONNECT_COOLDOWN: {
+    //   type: 'number',
+    //   default: 3000,
+    // },
+    CACHE_REDIS_PASSWORD: {
+        description: 'The password required for redis auth',
+        type: 'string',
+    },
+    CACHE_REDIS_PATH: {
+        description: 'The UNIX socket string of the Redis server',
+        type: 'string',
+    },
+    CACHE_REDIS_PORT: {
+        description: 'Port for the Redis instance to be used',
+        type: 'number',
+        default: 6379,
+    },
+    CACHE_REDIS_TIMEOUT: {
+        description: 'Timeout to fail a Redis server request if no response (ms)',
+        type: 'number',
+        default: 500,
+    },
+    // CACHE_REDIS_URL: {
+    //   type: 'string',
+    // },
+    CACHE_TYPE: {
+        description: 'The type of cache to use throughout the EA',
+        type: 'enum',
+        default: 'local',
+        options: ['local', 'redis'],
+    },
+    CORRELATION_ID_ENABLED: {
+        description: 'Flag to enable correlation IDs for sent requests in logging',
+        type: 'boolean',
+        default: true,
+    },
+    // DATA_PROVIDER_URL: {
+    //   description: 'Legacy variable that has the same functionality as ADAPTER_URL',
+    //   type: 'string',
+    // },
+    DEBUG: {
+        description: 'Toggles debug mode',
+        type: 'boolean',
+        default: false,
+    },
+    EA_PORT: {
+        description: 'Port through which the EA will listen for REST requests (if mode is set to "reader" or "reader-writer")',
+        type: 'number',
+        default: 8080,
+    },
+    // ERROR_CAPACITY: {
+    //   type: 'number',
+    // },
+    EXPERIMENTAL_METRICS_ENABLED: {
+        description: 'Flag to specify whether or not to collect metrics. Used as fallback for METRICS_ENABLED',
+        type: 'boolean',
+        default: true,
+    },
+    LOG_LEVEL: {
+        description: 'Minimum level required for logs to be output',
+        type: 'string',
+        default: 'info',
+    },
+    METRICS_ENABLED: {
+        description: 'Flag to specify whether or not to startup the metrics server',
+        type: 'boolean',
+        default: true,
+    },
+    METRICS_NAME: {
+        description: 'Metrics name',
+        type: 'string',
+    },
+    METRICS_PORT: {
+        description: 'Port metrics will be exposed to',
+        type: 'number',
+        default: 9080,
+    },
+    METRICS_USE_BASE_URL: {
+        description: 'Flag to specify whether or not to prepend the BASE_URL to the metrics endpoint',
+        type: 'boolean',
+    },
+    RATE_LIMIT_API_TIER: {
+        description: 'Rate limiting tier to use from the available options for the adapter. If not present, the adapter will run using the first tier on the list.',
+        type: 'string',
+    },
+    RATE_LIMIT_CAPACITY: {
+        description: 'Used as rate limit capacity per minute and ignores tier settings if defined',
+        type: 'number',
+    },
+    RATE_LIMIT_CAPACITY_MINUTE: {
+        description: 'Used as rate limit capacity per minute and ignores tier settings if defined. Supercedes RATE_LIMIT_CAPACITY if both vars are set',
+        type: 'number',
+    },
+    RATE_LIMIT_CAPACITY_SECOND: {
+        description: 'Used as rate limit capacity per second and ignores tier settings if defined',
+        type: 'number',
+    },
+    REQUEST_COALESCING_ENABLED: {
+        description: 'Enable request coalescing',
+        type: 'boolean',
+    },
+    // REQUEST_COALESCING_ENTROPY_MAX: {
+    //   type: 'number',
+    //   default: 0,
+    // },
+    // REQUEST_COALESCING_INTERVAL_COEFFICIENT: {
+    //   type: 'number',
+    //   default: 2,
+    // },
+    // REQUEST_COALESCING_INTERVAL_MAX: {
+    //   type: 'number',
+    //   default: 1000,
+    // },
+    // REQUEST_COALESCING_INTERVAL: {
+    //   type: 'number',
+    //   default: 100,
+    // },
+    // REQUEST_COALESCING_MAX_RETRIES: {
+    //   type: 'number',
+    //   default: 5,
+    // },
+    // RETRY: {
+    //   type: 'boolean',
+    //   default: 1,
+    // },
+    // SERVER_RATE_LIMIT_MAX: {
+    //   type: 'number',
+    //   default: 250,
+    // },
+    // SERVER_SLOW_DOWN_AFTER_FACTOR: {
+    //   type: 'number',
+    //   default: 0.8,
+    // },
+    // SERVER_SLOW_DOWN_DELAY_MS: {
+    //   type: 'number',
+    //   default: 500,
+    // },
+    // WARMUP_ENABLED: {
+    //   type: 'boolean',
+    //   default: true,
+    // },
+    // WARMUP_INTERVAL: {
+    //   type: 'number',
+    // },
+    WARMUP_SUBSCRIPTION_TTL: {
+        type: 'number',
+        description: 'TTL for batch warmer subscriptions',
+        default: 10000,
+    },
+    // WARMUP_UNHEALTHY_THRESHOLD: {
+    //   type: 'number',
+    //   default: 3,
+    // },
+    // WS_API_ENDPOINT: {
+    //   type: 'string',
+    // },
+    // WS_API_KEY: {
+    //   type: 'string',
+    // },
+    // WS_CONNECTION_KEY: {
+    //   type: 'string',
+    //   default: 1,
+    // },
+    // WS_CONNECTION_LIMIT: {
+    //   type: 'number',
+    //   default: 1,
+    // },
+    // WS_CONNECTION_RETRY_DELAY: {
+    //   type: 'number',
+    //   default: 1000,
+    // },
+    // WS_CONNECTION_RETRY_LIMIT: {
+    //   type: 'number',
+    //   default: 3,
+    // },
+    // WS_CONNECTION_TTL: {
+    //   type: 'number',
+    //   default: 70000,
+    // },
+    // WS_ENABLED: {
+    //   type: 'boolean',
+    //   default: false,
+    // },
+    // WS_SUBSCRIPTION_LIMIT: {
+    //   type: 'number',
+    //   default: 10,
+    // },
+    // WS_SUBSCRIPTION_PRIORITY_LIST: {
+    //   type: 'string',
+    // },
+    WS_SUBSCRIPTION_TTL: {
+        description: '',
+        type: 'number',
+        default: 120000,
+    },
+    // WS_SUBSCRIPTION_UNRESPONSIVE_TTL: {
+    //   type: 'number',
+    //   default: false,
+    // },
+    // WS_TIME_UNTIL_HANDLE_NEXT_MESSAGE_OVERRIDE: {
+    //   type: 'number',
+    // },
+    // V3
+    CACHE_POLLING_MAX_RETRIES: {
+        description: 'Max amount of times to attempt to find EA response in the cache after the Transport has been set up',
+        type: 'number',
+        default: 10,
+    },
+    CACHE_POLLING_SLEEP_MS: {
+        description: 'The number of ms to sleep between each retry to fetch the EA response in the cache',
+        type: 'number',
+        default: 200,
+    },
+    DEFAULT_CACHE_KEY: {
+        description: 'Default key to be used when one cannot be determined from request parameters',
+        type: 'string',
+        default: 'DEFAULT_CACHE_KEY',
+    },
+    EA_HOST: {
+        description: 'Host this EA will listen for REST requests on (if mode is set to "reader" or "reader-writer")',
+        type: 'string',
+        default: '::',
+    },
+    EA_MODE: {
+        description: 'Port this EA will listen for REST requests on (if mode is set to "reader" or "reader-writer")',
+        type: 'enum',
+        default: 'reader-writer',
+        options: ['reader', 'writer', 'reader-writer'],
+    },
+    MAX_COMMON_KEY_SIZE: {
+        description: 'Maximum amount of characters that the common part of the cache key or feed ID can have',
+        type: 'number',
+        default: 300,
+        validate: (value) => {
+            if (!(value && value >= 150 && value <= 500)) {
+                return 'MAX_COMMON_KEY_SIZE must be a number between 150 and 500';
+            }
+        },
+    },
+    REST_TRANSPORT_MAX_RATE_LIMIT_RETRIES: {
+        description: 'Maximum amount of times the Rest Transport will attempt to set up a request when blocked by the rate limiter',
+        type: 'number',
+        default: 3,
+    },
+    REST_TRANSPORT_MS_BETWEEN_RATE_LIMIT_RETRIES: {
+        description: 'Time that the Rest Transport will wait between retries when blocked by the rate limiter',
+        type: 'number',
+        default: 400,
+    },
+    SMOKE_TEST_PAYLOAD_FILE_NAME: {
+        description: 'Name of the test payload file used for the smoke endpoint',
+        type: 'string',
+    },
+};
+const buildAdapterConfig = ({ overrides = {}, customSettings = {}, }) => {
+    const validationErrors = [];
+    const vars = {};
+    // Iterate base adapter env vars
+    for (const [key, config] of Object.entries(exports.BaseSettings)) {
+        const value = getEnv(key, config) ?? overrides?.[key] ?? config.default;
+        vars[key] = value;
+    }
+    // Iterate custom vars
+    for (const [key, config] of Object.entries(customSettings)) {
+        if (exports.BaseSettings[key]) {
+            throw new Error(`Custom env var "${key}" declared, but a base framework env var with that name already exists.`);
+        }
+        const value = getEnv(key, config) ?? config.default;
+        // Check if a required setting has been provided
+        if (config.required && value === null) {
+            validationErrors.push(`${key}: Value is required, but none was provided`);
+        }
+        else if (config.validate) {
+            // Cast validate to unknown because TS can't select one of multiple variants of the validate function signature
+            const validationRes = config.validate(value);
+            if (validationRes) {
+                validationErrors.push(`${key}: ${validationRes}`);
+            }
+        }
+        vars[key] = value;
+    }
+    if (validationErrors.length > 0) {
+        throw new Error(`Validation failed for the following variables:\n ${validationErrors.join('\n')}`);
+    }
+    return vars;
+};
+exports.buildAdapterConfig = buildAdapterConfig;
+const getEnv = (name, config) => {
+    const value = process.env[name];
+    if (!value) {
+        return null;
+    }
+    switch (config.type) {
+        case 'string':
+            return value;
+        case 'number':
+            return parseInt(value);
+        case 'boolean':
+            return value === 'true';
+        case 'enum':
+            if (!config.options?.includes(value)) {
+                throw new Error(`Env var "${name}" has value "${value}" which is not included in the valid options (${config.options})`);
+            }
+            return value;
+    }
+};

package/config/provider-limits.js

@@ -0,0 +1,74 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getWSLimits = exports.getRateLimit = exports.getHTTPLimit = exports.DEFAULT_WS_SUBSCRIPTIONS = exports.DEFAULT_WS_CONNECTIONS = exports.BURST_UNDEFINED_QUOTA_MULTIPLE = exports.DEFAULT_MINUTE_RATE_LIMIT = void 0;
+const util_1 = require("../util");
+exports.DEFAULT_MINUTE_RATE_LIMIT = 60;
+exports.BURST_UNDEFINED_QUOTA_MULTIPLE = 2;
+exports.DEFAULT_WS_CONNECTIONS = 2;
+exports.DEFAULT_WS_SUBSCRIPTIONS = 10;
+const logger = (0, util_1.makeLogger)('ProviderLimits');
+const getHTTPLimit = (provider, limits, tier, timeframe) => {
+    const providerLimit = getProviderLimits(provider, limits, tier, 'http');
+    return providerLimit?.[timeframe] || 0;
+};
+exports.getHTTPLimit = getHTTPLimit;
+const getRateLimit = (provider, limits, tier) => {
+    const providerLimit = getProviderLimits(provider, limits, tier, 'http');
+    return calculateRateLimit(providerLimit);
+};
+exports.getRateLimit = getRateLimit;
+const getWSLimits = (provider, limits, tier) => {
+    const providerLimit = getProviderLimits(provider, limits, tier, 'ws');
+    return calculateWSLimits(providerLimit);
+};
+exports.getWSLimits = getWSLimits;
+const getProviderLimits = (provider, limits, tier, protocol) => {
+    const providerConfig = parseLimits(limits);
+    if (!providerConfig) {
+        throw new Error(`Rate Limit: Provider: "${provider}" doesn't match any provider spec in limits.json`);
+    }
+    const protocolConfig = providerConfig[protocol];
+    if (!protocolConfig) {
+        throw new Error(`Rate Limit: "${provider}" doesn't have any configuration for ${protocol} in limits.json`);
+    }
+    let limitsConfig = protocolConfig[tier.toLowerCase()];
+    if (!limitsConfig) {
+        logger.debug(`Rate Limit: "${provider} does not have tier ${tier} defined. Falling back to lowest tier"`);
+        limitsConfig = Object.values(protocolConfig)?.[0];
+    }
+    if (!limitsConfig) {
+        throw new Error(`Rate Limit: Provider: "${provider}" has no tiers defined for ${protocol} in limits.json`);
+    }
+    return limitsConfig;
+};
+const objectKeysToLowercase = (obj) => {
+    const lower = {};
+    for (const key in obj) {
+        lower[key.toLowerCase()] = obj[key];
+    }
+    return lower;
+};
+const parseLimits = (limits) => {
+    limits.http = objectKeysToLowercase(limits.http);
+    limits.ws = objectKeysToLowercase(limits.ws);
+    return limits;
+};
+const calculateWSLimits = (providerLimit) => {
+    return {
+        connections: providerLimit.connections,
+        subscriptions: providerLimit.subscriptions,
+    };
+};
+const calculateRateLimit = (providerLimit) => {
+    let quota = providerLimit.rateLimit1m;
+    if (!quota && providerLimit?.rateLimit1h) {
+        quota = providerLimit?.rateLimit1h / 60;
+    }
+    else if (!quota && providerLimit?.rateLimit1s) {
+        quota = providerLimit?.rateLimit1s * 60;
+    }
+    return {
+        second: providerLimit?.rateLimit1s || (quota / 60) * exports.BURST_UNDEFINED_QUOTA_MULTIPLE,
+        minute: quota,
+    };
+};