@chainlink/external-adapter-framework 0.0.10 → 0.0.12

package/src/adapter.ts

@@ -0,0 +1,263 @@
+import { Cache, CacheFactory } from './cache'
+import { AdapterConfig, BaseAdapterConfig, CustomSettingsType, SettingsMap } from './config'
+import {
+  AdapterRateLimitTier,
+  BackgroundExecuteRateLimiter,
+  FixedFrequencyRateLimiter,
+  getRateLimitingTier,
+  RequestRateLimiter,
+  SimpleCountingRateLimiter,
+} from './rate-limiting'
+import { Transport } from './transports'
+import { makeLogger, SubscriptionSetFactory } from './util'
+import { InputParameters } from './validation/input-params'
+
+const logger = makeLogger('Adapter')
+
+/**
+ * Dependencies that will be injected into the Adapter on startup
+ */
+export interface AdapterDependencies {
+  // TODO: Complete docs
+  cache: Cache
+  requestRateLimiter: RequestRateLimiter
+  backgroundExecuteRateLimiter: BackgroundExecuteRateLimiter
+  subscriptionSetFactory: SubscriptionSetFactory
+}
+
+/**
+ * Context that will be used on background executions of a Transport.
+ * For example, the endpointName used to log statements or generate Cache keys.
+ */
+export interface AdapterContext<
+  CustomSettings extends CustomSettingsType<CustomSettings> = SettingsMap,
+> {
+  adapterEndpoint: AdapterEndpoint
+  adapterConfig: AdapterConfig<CustomSettings>
+}
+
+/**
+ * Structure to describe rate limits specs for the Adapter
+ */
+interface AdapterRateLimitingConfig {
+  /** Adapter rate limits, gotten from the specific tier requested */
+  tiers: Record<string, AdapterRateLimitTier>
+}
+
+/**
+ * Main structure of an External Adapter
+ */
+export interface Adapter {
+  /** Name of the adapter */
+  name: string
+
+  /** If present, the string that will be used for requests with no specified endpoint */
+  defaultEndpoint?: string
+
+  /** List of [[AdapterEndpoint]]s in the adapter */
+  endpoints: AdapterEndpoint[]
+
+  /** Map of overrides to the default config values for an Adapter */
+  envDefaultOverrides?: Partial<BaseAdapterConfig>
+
+  /** List of custom env vars for this particular adapter (e.g. RPC_URL) */
+  customSettings?: SettingsMap
+
+  /** Configuration relevant to outbound (EA --\> DP) communication rate limiting */
+  rateLimiting?: AdapterRateLimitingConfig
+
+  /** Overrides for converting the 'base' parameter that are hardcoded into the adapter. */
+  // This must be included in the middleware in order to generate deterministing cache keys for hardcoded overrides
+  overrides?: Record<string, string>
+}
+
+/**
+ * Structure to describe rate limits specs for a specific adapter endpoint
+ */
+export interface EndpointRateLimitingConfig {
+  /**
+   * How much of the total limit for the adapter will be assigned to this specific endpoint.
+   * Should be a non-zero positive number up to 100.
+   * Endpoints in the same adapter without a specific allocation will divide the remaining limits equally.
+   */
+  allocationPercentage: number
+}
+
+/**
+ * Structure to describe a specific endpoint in an [[Adapter]]
+ */
+export interface AdapterEndpoint {
+  /** Name that will be used to match input params to this endpoint (case insensitive) */
+  name: string
+
+  /** List of alternative endpoint names that will resolve to this same transport (case insensitive) */
+  aliases?: string[]
+
+  /** Transport that will be used to handle data processing and communication for this endpoint */
+  transport: Transport<any, any, any>
+
+  /** Specification of what the body of a request hitting this endpoint should look like (used for validation) */
+  inputParameters: InputParameters
+
+  /** Specific details related to the rate limiting for this endpoint in particular */
+  rateLimiting?: EndpointRateLimitingConfig
+}
+
+/**
+ * Structure to describe an adapter that has been initialized
+ */
+export interface InitializedAdapter extends Adapter {
+  /** Object containing alias translations for all endpoints */
+  endpointsMap: Record<string, AdapterEndpoint>
+
+  /** Initialized dependencies that the adapter will use */
+  dependencies: AdapterDependencies
+
+  /** Configuration params for various adapter properties */
+  config: AdapterConfig
+}
+
+/**
+ * This function will take an adapter structure and go through each endpoint, calculating
+ * each one's allocation of the total rate limits that are set for the adapter as a whole.
+ *
+ * @param adapter - the adapter to initialize rate limiting for
+ */
+const calculateRateLimitAllocations = (adapter: Adapter) => {
+  const numberOfEndpoints = adapter.endpoints.length
+  const endpointsWithExplicitAllocations = adapter.endpoints.filter((e) => e.rateLimiting)
+
+  const totalExplicitAllocation = endpointsWithExplicitAllocations
+    .map((e) => e.rateLimiting?.allocationPercentage || 0)
+    .reduce((sum, next) => sum + next, 0)
+
+  if (totalExplicitAllocation > 100) {
+    throw new Error('The total allocation set for all endpoints summed cannot exceed 100%')
+  }
+
+  if (
+    totalExplicitAllocation === 100 &&
+    numberOfEndpoints - endpointsWithExplicitAllocations.length > 0
+  ) {
+    throw new Error(
+      'The explicit allocation is at 100% but there are endpoints with implicit allocation',
+    )
+  }
+
+  const implicitAllocation = 100 - totalExplicitAllocation
+
+  logger.debug('Adapter rate limit allocations:')
+  for (const endpoint of adapter.endpoints) {
+    if (!endpoint.rateLimiting) {
+      endpoint.rateLimiting = {
+        allocationPercentage:
+          implicitAllocation / (numberOfEndpoints - endpointsWithExplicitAllocations.length),
+      }
+    }
+
+    logger.debug(`Endpoint [${endpoint.name}] - ${endpoint.rateLimiting?.allocationPercentage}%`)
+  }
+}
+
+/**
+ * This function will process dependencies for an adapter, such as caches or rate limiters,
+ * in order to inject them into transports and other relevant places later in the lifecycle.
+ *
+ * @param config - the configuration for this adapter
+ * @param inputDependencies - a partial obj of initialized dependencies to override the created ones
+ * @param rateLimitingConfig - details from the adapter regarding rate limiting
+ * @returns a set of AdapterDependencies all initialized
+ */
+export const initializeDependencies = (
+  adapter: Adapter,
+  config: AdapterConfig,
+  inputDependencies?: Partial<AdapterDependencies>,
+): AdapterDependencies => {
+  const dependencies = inputDependencies || {}
+  if (!dependencies.cache) {
+    dependencies.cache = CacheFactory.buildCache(config)
+  }
+
+  // In the future we might want something more complex, but for now it's better to simplify
+  // and just use the same rate limiting for everything. Once we have a more complex use case we
+  // can think of ways to make this more configurable.
+  const rateLimitingTier = getRateLimitingTier(
+    adapter.rateLimiting?.tiers,
+    config.RATE_LIMIT_API_TIER,
+  )
+  if (!dependencies.requestRateLimiter) {
+    dependencies.requestRateLimiter = new SimpleCountingRateLimiter().initialize(
+      adapter.endpoints,
+      rateLimitingTier,
+    )
+  }
+  if (!dependencies.backgroundExecuteRateLimiter) {
+    dependencies.backgroundExecuteRateLimiter = new FixedFrequencyRateLimiter().initialize(
+      adapter.endpoints,
+      rateLimitingTier,
+    )
+  }
+  if (!dependencies.subscriptionSetFactory) {
+    dependencies.subscriptionSetFactory = new SubscriptionSetFactory(config)
+  }
+
+  return dependencies as AdapterDependencies
+}
+
+/**
+ * Takes an adapter and normalizes all endpoint names and aliases, as well as the default endpoint.
+ * i.e. makes them lowercase for now
+ * @param adapter - an instance of an Adapter
+ */
+const normalizeEndpointNames = (adapter: Adapter) => {
+  // Make endpoints case insensitive, including default
+  adapter.defaultEndpoint = adapter.defaultEndpoint?.toLowerCase()
+
+  for (const endpoint of adapter.endpoints) {
+    endpoint.name = endpoint.name.toLowerCase()
+    endpoint.aliases = endpoint.aliases?.map((a) => a.toLowerCase())
+  }
+}
+
+/**
+ * Initializes all of the [[Transport]]s in the adapter, passing along any [[AdapterDependencies]] and [[AdapterConfig]].
+ * Additionally, it builds a map out of all the endpoint names and aliases (checking for duplicates).
+ *
+ * @param adapter - an instance of an Adapter
+ * @param dependencies - dependencies that the adapter will need at initialization
+ * @param config - configuration variables already processed and validated
+ * @returns - the adapter with all transports initialized and aliases map built
+ */
+export const initializeAdapter = async (
+  adapter: Adapter,
+  config: AdapterConfig,
+  dependencies?: Partial<AdapterDependencies>,
+): Promise<InitializedAdapter> => {
+  normalizeEndpointNames(adapter)
+  calculateRateLimitAllocations(adapter)
+  const initializedDependencies = initializeDependencies(adapter, config, dependencies)
+
+  const endpointsMap: Record<string, AdapterEndpoint> = {}
+
+  for (const endpoint of adapter.endpoints) {
+    // Add aliases to map to use in validation
+    const aliases = [endpoint.name, ...(endpoint.aliases || [])]
+    for (const alias of aliases) {
+      if (endpointsMap[alias]) {
+        throw new Error(`Duplicate endpoint / alias: "${alias}"`)
+      }
+      endpointsMap[alias] = endpoint
+    }
+
+    logger.debug(`Initializing transport for endpoint "${endpoint.name}"...`)
+    await endpoint.transport.initialize(initializedDependencies)
+  }
+
+  logger.debug('Adapter initialization complete.')
+  return {
+    ...adapter,
+    endpointsMap,
+    dependencies: initializedDependencies,
+    config,
+  }
+}

package/src/background-executor.ts

@@ -0,0 +1,52 @@
+import { Server } from 'http'
+import { AdapterContext, InitializedAdapter } from './adapter'
+import { makeLogger, sleep } from './util'
+
+const logger = makeLogger('BackgroundExecutor')
+
+/**
+ * Very simple background loop that will call the [[Transport.backgroundExecute]] functions in all Transports.
+ * It gets the time in ms to wait as the return value from those functions, and sleeps until next execution.
+ *
+ * @param adapter - an initialized External Adapter
+ * @param server - the http server to attach an on close listener to
+ */
+export async function callBackgroundExecutes(adapter: InitializedAdapter, server?: Server) {
+  // Set up variable to check later on to see if we need to stop this background "thread"
+  // If no server is provided, the listener won't be set and serverClosed will always be false
+  let serverClosed = false
+  server?.on('close', () => {
+    serverClosed = true
+  })
+
+  for (const endpoint of adapter.endpoints) {
+    const backgroundExecute = endpoint.transport.backgroundExecute?.bind(endpoint.transport)
+    if (!backgroundExecute) {
+      logger.debug(`Endpoint "${endpoint.name}" has no background execute, skipping...`)
+      continue
+    }
+
+    const context: AdapterContext = {
+      adapterEndpoint: endpoint,
+      adapterConfig: adapter.config,
+    }
+
+    const handler = async () => {
+      if (serverClosed) {
+        logger.info('Server closed, stopping recursive backgroundExecute handler chain')
+        return
+      }
+
+      logger.debug(`Calling background execute for endpoint "${endpoint.name}"`)
+      const timeToWait = await backgroundExecute(context)
+      logger.debug(
+        `Finished background execute for endpoint "${endpoint.name}", sleeping for ${timeToWait}ms`,
+      )
+      await sleep(timeToWait)
+      handler()
+    }
+
+    // Start recursive async calls
+    handler()
+  }
+}

package/src/cache/factory.ts

@@ -0,0 +1,26 @@
+import Redis from 'ioredis'
+import { AdapterConfig } from '../config'
+import { makeLogger } from '../util'
+import { LocalCache } from './local'
+import * as cacheMetrics from './metrics'
+import { RedisCache } from './redis'
+
+const logger = makeLogger('CacheFactory')
+export class CacheFactory {
+  static buildCache(config: AdapterConfig) {
+    logger.info(`Using "${config.CACHE_TYPE}" cache.`)
+    if (config.CACHE_TYPE === 'local') {
+      return new LocalCache()
+    } else if (config.CACHE_TYPE === 'redis') {
+      const redis = new Redis({
+        enableAutoPipelining: true, // This will make multiple commands be batch automatically
+        host: config.CACHE_REDIS_HOST,
+        port: config.CACHE_REDIS_PORT,
+      })
+      redis.on('connect', () => {
+        cacheMetrics.redisConnectionsOpen.inc()
+      })
+      return new RedisCache(redis)
+    }
+  }
+}

package/src/cache/index.ts

@@ -0,0 +1,258 @@
+import { FastifyReply } from 'fastify'
+import { AdapterEndpoint, InitializedAdapter } from '../adapter'
+import { AdapterConfig } from '../config'
+import {
+  AdapterMiddlewareBuilder,
+  AdapterRequest,
+  AdapterResponse,
+  makeLogger,
+  sleep,
+} from '../util'
+import * as cacheMetrics from './metrics'
+
+export * from './local'
+export * from './redis'
+export * from './factory'
+
+const logger = makeLogger('Cache')
+
+/**
+ * An object describing an entry in the cache.
+ * @typeParam T - the type of the entry's value
+ */
+export interface CacheEntry<T> {
+  key: string
+  value: T
+}
+
+/**
+ * Generic interface for a local or remote Cache.
+ * @typeParam T - the type of the cache entries' values
+ */
+export interface Cache<T = unknown> {
+  /**
+   * Gets an item from the Cache.
+   *
+   * @param key - the key of the desired entry for which to fetch its value
+   * @returns a Promise of the entry's value, or undefined if not found / expired.
+   */
+  get: (key: string) => Promise<T | undefined>
+
+  /**
+   * Sets an item in the Cache.
+   *
+   * @param key - the key of the new entry
+   * @param value - the value of the new entry
+   * @param ttl - the time in milliseconds until the entry expires
+   * @returns an empty Promise that resolves when the entry has been set
+   */
+  set: (key: string, value: T, ttl: number) => Promise<void>
+
+  /**
+   * Sets a list of items in the Cache.
+   *
+   * @param entries - a list of cache entries
+   * @param ttl - the time in milliseconds until the entries expire
+   * @returns an empty Promise that resolves when all entries have been set
+   */
+  setMany: (entries: CacheEntry<T>[], ttl: number) => Promise<void>
+
+  /**
+   * Deletes the specified item from the Cache
+   *
+   * @param key - the key of the entry to be deleted
+   * @returns an empty Promise that resolves when the entry has been deleted
+   */
+  delete: (key: string) => Promise<void>
+}
+
+// Uses calculateKey to generate a unique key from the endpoint name, data, and input parameters
+export const calculateCacheKey = (
+  {
+    adapterEndpoint,
+    adapterConfig,
+  }: {
+    adapterEndpoint: AdapterEndpoint
+    adapterConfig: AdapterConfig
+  },
+  data: unknown,
+): string => {
+  const paramNames = Object.keys(adapterEndpoint.inputParameters)
+  if (paramNames.length === 0) {
+    logger.trace(`Using default cache key ${adapterConfig.DEFAULT_CACHE_KEY}`)
+    return adapterConfig.DEFAULT_CACHE_KEY
+  }
+  return `${adapterEndpoint.name}-${calculateKey(data, paramNames, adapterConfig)}`
+}
+
+export const calculateFeedId = (
+  {
+    adapterEndpoint,
+    adapterConfig,
+  }: {
+    adapterEndpoint: AdapterEndpoint
+    adapterConfig: AdapterConfig
+  },
+  data: unknown,
+): string => {
+  const paramNames = Object.keys(adapterEndpoint.inputParameters)
+  if (paramNames.length === 0) {
+    logger.trace(`Cannot generate Feed ID without input parameters`)
+    return 'N/A'
+  }
+  return calculateKey(data, paramNames, adapterConfig)
+}
+
+/**
+ * Calculates a unique key from the provided data.
+ *
+ * @param data - the request data/body, i.e. the adapter input params
+ * @param paramNames - the keys from adapter endpoint input parameters
+ * @returns the calculated unique key
+ *
+ * @example
+ * ```
+ * calculateKey({ base: 'ETH', quote: 'BTC' }, ['base','quote'])
+ * // equals `|base:eth|quote:btc`
+ * ```
+ */
+export const calculateKey = (
+  data: unknown,
+  paramNames: string[],
+  adapterConfig: AdapterConfig,
+): string => {
+  if (data && typeof data !== 'object') {
+    throw new Error('Data to calculate cache key should be an object')
+  }
+
+  const params = data as Record<string, unknown>
+
+  let cacheKey = ''
+  for (const paramName of paramNames) {
+    // Ignore overrides param when generating cache keys
+    if (paramName === 'overrides') {
+      continue
+    }
+    const param = params[paramName]
+    if (param === undefined) {
+      continue
+    }
+
+    cacheKey += `|${paramName}:`
+    switch (typeof param) {
+      case 'string':
+        cacheKey += param.toLowerCase()
+        break
+      case 'number':
+      case 'boolean':
+        cacheKey += param.toString()
+        break
+      case 'object':
+        // Force cache keys to only use performant properties of the input params.
+        // If the object were to be used, we'd have to sort its properties.
+        logger.debug(
+          `Property "${paramName}" in request parameters is of type object, and won't be used in the cacheKey`,
+        )
+    }
+  }
+
+  if (cacheKey.length > adapterConfig.MAX_COMMON_KEY_SIZE) {
+    logger.warn(
+      `Generated cache key for adapter request is bigger than the MAX_COMMON_KEY_SIZE and will be truncated`,
+    )
+    cacheKey = cacheKey.slice(0, adapterConfig.MAX_COMMON_KEY_SIZE)
+  }
+
+  logger.trace(`Generated cache key for request: "${cacheKey}"`)
+  return cacheKey
+}
+
+// Calculate the amount of time the non-expired entry has been in the cache
+const calculateStaleness = (expirationTimestamp: number | undefined, ttl: number): number => {
+  if (expirationTimestamp) {
+    const createTimestamp = expirationTimestamp - ttl
+    return (Date.now() - createTimestamp) / 1000
+  } else {
+    // If expirationTimestamp is not available, staleness cannot be calculated
+    // Defaults to ttl for metrics purposes
+    return ttl
+  }
+}
+
+/**
+ * Polls the provided Cache for an AdapterResponse set in the provided key. If the maximum
+ * amount of retries is exceeded, it returns undefined instead.
+ *
+ * @param cache - a Cache instance
+ * @param key - the key generated from an AdapterRequest that corresponds to the desired AdapterResponse
+ * @param retry - current retry, only for internal use
+ * @returns the AdapterResponse if found, else undefined
+ */
+export const pollResponseFromCache = async (
+  cache: Cache<AdapterResponse>,
+  key: string,
+  options: {
+    maxRetries: number
+    sleep: number
+  },
+  retry = 0,
+): Promise<AdapterResponse | undefined> => {
+  if (retry > options.maxRetries) {
+    // Ideally this shouldn't happen often (p99 of reqs should be found in the cache)
+    logger.info('Exceeded max cache polling retries')
+    return undefined
+  }
+
+  logger.trace('Getting response from cache...')
+  const response = await cache.get(key)
+  if (response) {
+    logger.trace('Got response from cache')
+    return response
+  }
+
+  if (options.maxRetries === 0) {
+    logger.debug(`Response not found, retries disabled`)
+    return undefined
+  }
+
+  logger.debug(`Response not found, sleeping ${options.sleep} milliseconds...`)
+  await sleep(options.sleep)
+
+  return pollResponseFromCache(cache, key, options, retry + 1)
+}
+
+/**
+ * Given a Cache instance in the adapter dependencies, builds a middleware function that will perform
+ * a get from said Cache and return that if found; otherwise it'll continue the middleware chain.
+ *
+ * @param adapter - an initialized adapter
+ * @returns the cache middleware function
+ */
+export const buildCacheMiddleware: AdapterMiddlewareBuilder =
+  (adapter: InitializedAdapter) => async (req: AdapterRequest, res: FastifyReply) => {
+    const response = await (adapter.dependencies.cache as Cache<AdapterResponse>).get(
+      req.requestContext.cacheKey,
+    )
+
+    if (response) {
+      logger.debug('Found response from cache, sending that')
+      if (adapter.config.METRICS_ENABLED && adapter.config.EXPERIMENTAL_METRICS_ENABLED) {
+        const label = cacheMetrics.cacheMetricsLabel(
+          req.requestContext.cacheKey,
+          req.requestContext.meta?.metrics?.feedId || 'N/A',
+          adapter.config.CACHE_TYPE,
+        )
+
+        // Record cache staleness and cache get count and value
+        const staleness = calculateStaleness(response.maxAge, adapter.config.CACHE_MAX_AGE)
+        cacheMetrics.cacheGet(label, response.result, staleness)
+        req.requestContext.meta = {
+          ...req.requestContext.meta,
+          metrics: { ...req.requestContext.meta?.metrics, cacheHit: true },
+        }
+      }
+      return res.send(response)
+    }
+
+    logger.debug('Did not find response in cache, moving to next middleware')
+  }

package/src/cache/local.ts

@@ -0,0 +1,73 @@
+import { AdapterResponse, makeLogger } from '../util'
+import { Cache, CacheEntry } from './index'
+import * as cacheMetrics from './metrics'
+
+const logger = makeLogger('LocalCache')
+
+/**
+ * Type for a value stored in a LocalCache entry.
+ *
+ * @typeParam T - the type for the entry's value
+ */
+export interface LocalCacheEntry<T> {
+  expirationTimestamp: number
+  value: T
+}
+
+/**
+ * Local implementation of a Cache. It uses a simple js Object, storing entries with both
+ * a value and an expiration timestamp. Expired entries are deleted on reads (i.e. no background gc/upkeep).
+ *
+ * @typeParam T - the type for the entries' values
+ */
+export class LocalCache<T = unknown> implements Cache<T> {
+  store: Record<string, LocalCacheEntry<T>> = {}
+
+  async get(key: string): Promise<T | undefined> {
+    logger.trace(`Getting key ${key}`)
+    const entry = this.store[key]
+
+    if (!entry) {
+      logger.debug(`No entry in local cache for key "${key}", returning undefined`)
+      return undefined
+    }
+
+    const expired = entry.expirationTimestamp <= Date.now()
+    if (expired) {
+      logger.debug('Entry in local cache expired, deleting and returning undefined')
+      this.delete(key)
+      return undefined
+    } else {
+      logger.debug('Found valid entry in local cache, returning value')
+      return entry.value
+    }
+  }
+
+  async delete(key: string): Promise<void> {
+    logger.trace(`Deleting key ${key}`)
+    delete this.store[key] // Deletes are slower than ignoring or setting null, fyi
+  }
+
+  async set(key: string, value: T, ttl: number): Promise<void> {
+    logger.trace(`Setting key ${key} with ttl ${ttl}`)
+    this.store[key] = {
+      value,
+      expirationTimestamp: Date.now() + ttl,
+    }
+
+    // Only record metrics if feed Id is present, otherwise assuming value is not adapter response to record
+    const feedId = (value as unknown as AdapterResponse).meta?.metrics?.feedId
+    if (feedId) {
+      // Record cache set count, max age, and staleness (set to 0 for cache set)
+      const label = cacheMetrics.cacheMetricsLabel(key, feedId, cacheMetrics.CacheTypes.Local)
+      cacheMetrics.cacheSet(label, ttl)
+    }
+  }
+
+  async setMany(entries: CacheEntry<T>[], ttl: number): Promise<void> {
+    logger.trace(`Setting a bunch of keys with ttl ${ttl}`)
+    for (const { key, value } of entries) {
+      this.set(key, value, ttl)
+    }
+  }
+}