@chainlink/external-adapter-framework 0.0.6 → 0.0.8

package/src/adapter.ts

@@ -1,236 +0,0 @@
-import { CacheFactory } from './cache/factory'
-import { AdapterConfig, BaseAdapterConfig, SettingsMap } from './config'
-import {
-  AdapterRateLimitTier,
-  FixedFrequencyRateLimiter,
-  getRateLimitingTier,
-  SimpleCountingRateLimiter,
-} from './rate-limiting'
-import { AdapterDependencies, Transport } from './transports'
-import { makeLogger } from './util'
-import { InputParameters } from './validation/input-params'
-
-const logger = makeLogger('Adapter')
-
-/**
- * Structure to describe rate limits specs for the Adapter
- */
-interface AdapterRateLimitingConfig {
-  /** Adapter rate limits, gotten from the specific tier requested */
-  tiers: Record<string, AdapterRateLimitTier>
-}
-
-/**
- * Main structure of an External Adapter
- */
-export interface Adapter {
-  /** Name of the adapter */
-  name: string
-
-  /** If present, the string that will be used for requests with no specified endpoint */
-  defaultEndpoint?: string
-
-  /** List of [[AdapterEndpoint]]s in the adapter */
-  endpoints: AdapterEndpoint[]
-
-  /** Map of overrides to the default config values for an Adapter */
-  envDefaultOverrides?: Partial<BaseAdapterConfig>
-
-  /** List of custom env vars for this particular adapter (e.g. RPC_URL) */
-  customSettings?: SettingsMap
-
-  /** Configuration relevant to outbound (EA --\> DP) communication rate limiting */
-  rateLimiting?: AdapterRateLimitingConfig
-
-  /** Overrides for converting the 'base' parameter that are hardcoded into the adapter. */
-  // This must be included in the middleware in order to generate deterministing cache keys for hardcoded overrides
-  overrides?: Record<string, string>
-}
-
-/**
- * Structure to describe rate limits specs for a specific adapter endpoint
- */
-export interface EndpointRateLimitingConfig {
-  /**
-   * How much of the total limit for the adapter will be assigned to this specific endpoint.
-   * Should be a non-zero positive number up to 100.
-   * Endpoints in the same adapter without a specific allocation will divide the remaining limits equally.
-   */
-  allocationPercentage: number
-}
-
-/**
- * Structure to describe a specific endpoint in an [[Adapter]]
- */
-export interface AdapterEndpoint {
-  /** Name that will be used to match input params to this endpoint (case insensitive) */
-  name: string
-
-  /** List of alternative endpoint names that will resolve to this same transport (case insensitive) */
-  aliases?: string[]
-
-  /** Transport that will be used to handle data processing and communication for this endpoint */
-  transport: Transport<any, any, any>
-
-  /** Specification of what the body of a request hitting this endpoint should look like (used for validation) */
-  inputParameters: InputParameters
-
-  /** Specific details related to the rate limiting for this endpoint in particular */
-  rateLimiting?: EndpointRateLimitingConfig
-}
-
-/**
- * Structure to describe an adapter that has been initialized
- */
-export interface InitializedAdapter extends Adapter {
-  /** Object containing alias translations for all endpoints */
-  endpointsMap: Record<string, AdapterEndpoint>
-
-  /** Initialized dependencies that the adapter will use */
-  dependencies: AdapterDependencies
-
-  /** Configuration params for various adapter properties */
-  config: AdapterConfig
-}
-
-/**
- * This function will take an adapter structure and go through each endpoint, calculating
- * each one's allocation of the total rate limits that are set for the adapter as a whole.
- *
- * @param adapter - the adapter to initialize rate limiting for
- */
-const calculateRateLimitAllocations = (adapter: Adapter) => {
-  const numberOfEndpoints = adapter.endpoints.length
-  const endpointsWithExplicitAllocations = adapter.endpoints.filter((e) => e.rateLimiting)
-
-  const totalExplicitAllocation = endpointsWithExplicitAllocations
-    .map((e) => e.rateLimiting?.allocationPercentage || 0)
-    .reduce((sum, next) => sum + next, 0)
-
-  if (totalExplicitAllocation > 100) {
-    throw new Error('The total allocation set for all endpoints summed cannot exceed 100%')
-  }
-
-  if (
-    totalExplicitAllocation === 100 &&
-    numberOfEndpoints - endpointsWithExplicitAllocations.length > 0
-  ) {
-    throw new Error(
-      'The explicit allocation is at 100% but there are endpoints with implicit allocation',
-    )
-  }
-
-  const implicitAllocation = 100 - totalExplicitAllocation
-
-  logger.debug('Adapter rate limit allocations:')
-  for (const endpoint of adapter.endpoints) {
-    if (!endpoint.rateLimiting) {
-      endpoint.rateLimiting = {
-        allocationPercentage:
-          implicitAllocation / (numberOfEndpoints - endpointsWithExplicitAllocations.length),
-      }
-    }
-
-    logger.debug(`Endpoint [${endpoint.name}] - ${endpoint.rateLimiting?.allocationPercentage}%`)
-  }
-}
-
-/**
- * This function will process dependencies for an adapter, such as caches or rate limiters,
- * in order to inject them into transports and other relevant places later in the lifecycle.
- *
- * @param config - the configuration for this adapter
- * @param inputDependencies - a partial obj of initialized dependencies to override the created ones
- * @param rateLimitingConfig - details from the adapter regarding rate limiting
- * @returns a set of AdapterDependencies all initialized
- */
-export const initializeDependencies = (
-  adapter: Adapter,
-  config: AdapterConfig,
-  inputDependencies?: Partial<AdapterDependencies>,
-): AdapterDependencies => {
-  const dependencies = inputDependencies || {}
-  if (!dependencies.cache) {
-    dependencies.cache = CacheFactory.buildCache(config)
-  }
-
-  // In the future we might want something more complex, but for now it's better to simplify
-  // and just use the same rate limiting for everything. Once we have a more complex use case we
-  // can think of ways to make this more configurable.
-  const rateLimitingTier = getRateLimitingTier(
-    adapter.rateLimiting?.tiers,
-    config.RATE_LIMIT_API_TIER,
-  )
-  if (!dependencies.requestRateLimiter) {
-    dependencies.requestRateLimiter = new SimpleCountingRateLimiter().initialize(
-      adapter.endpoints,
-      rateLimitingTier,
-    )
-  }
-  if (!dependencies.backgroundExecuteRateLimiter) {
-    dependencies.backgroundExecuteRateLimiter = new FixedFrequencyRateLimiter().initialize(
-      adapter.endpoints,
-      rateLimitingTier,
-    )
-  }
-
-  return dependencies as AdapterDependencies
-}
-
-/**
- * Takes an adapter and normalizes all endpoint names and aliases, as well as the default endpoint.
- * i.e. makes them lowercase for now
- * @param adapter - an instance of an Adapter
- */
-const normalizeEndpointNames = (adapter: Adapter) => {
-  // Make endpoints case insensitive, including default
-  adapter.defaultEndpoint = adapter.defaultEndpoint?.toLowerCase()
-
-  for (const endpoint of adapter.endpoints) {
-    endpoint.name = endpoint.name.toLowerCase()
-    endpoint.aliases = endpoint.aliases?.map((a) => a.toLowerCase())
-  }
-}
-
-/**
- * Initializes all of the [[Transport]]s in the adapter, passing along any [[AdapterDependencies]] and [[AdapterConfig]].
- * Additionally, it builds a map out of all the endpoint names and aliases (checking for duplicates).
- *
- * @param adapter - an instance of an Adapter
- * @param dependencies - dependencies that the adapter will need at initialization
- * @param config - configuration variables already processed and validated
- * @returns - the adapter with all transports initialized and aliases map built
- */
-export const initializeAdapter = async (
-  adapter: Adapter,
-  config: AdapterConfig,
-  dependencies?: Partial<AdapterDependencies>,
-): Promise<InitializedAdapter> => {
-  normalizeEndpointNames(adapter)
-  calculateRateLimitAllocations(adapter)
-  const initializedDependencies = initializeDependencies(adapter, config, dependencies)
-
-  const endpointsMap: Record<string, AdapterEndpoint> = {}
-
-  for (const endpoint of adapter.endpoints) {
-    // Add aliases to map to use in validation
-    const aliases = [endpoint.name, ...(endpoint.aliases || [])]
-    for (const alias of aliases) {
-      if (endpointsMap[alias]) {
-        throw new Error(`Duplicate endpoint / alias: "${alias}"`)
-      }
-      endpointsMap[alias] = endpoint
-    }
-
-    logger.debug(`Initializing transport for endpoint "${endpoint.name}"...`)
-    await endpoint.transport.initialize(initializedDependencies)
-  }
-
-  logger.debug('Adapter initialization complete.')
-  return {
-    ...adapter,
-    endpointsMap,
-    dependencies: initializedDependencies,
-    config,
-  }
-}

package/src/background-executor.ts

@@ -1,53 +0,0 @@
-import { Server } from 'http'
-import { InitializedAdapter } from './adapter'
-import { AdapterContext } from './transports'
-import { makeLogger, sleep } from './util'
-
-const logger = makeLogger('BackgroundExecutor')
-
-/**
- * Very simple background loop that will call the [[Transport.backgroundExecute]] functions in all Transports.
- * It gets the time in ms to wait as the return value from those functions, and sleeps until next execution.
- *
- * @param adapter - an initialized External Adapter
- * @param server - the http server to attach an on close listener to
- */
-export async function callBackgroundExecutes(adapter: InitializedAdapter, server?: Server) {
-  // Set up variable to check later on to see if we need to stop this background "thread"
-  // If no server is provided, the listener won't be set and serverClosed will always be false
-  let serverClosed = false
-  server?.on('close', () => {
-    serverClosed = true
-  })
-
-  for (const endpoint of adapter.endpoints) {
-    const backgroundExecute = endpoint.transport.backgroundExecute?.bind(endpoint.transport)
-    if (!backgroundExecute) {
-      logger.debug(`Endpoint "${endpoint.name}" has no background execute, skipping...`)
-      continue
-    }
-
-    const context: AdapterContext = {
-      adapterEndpoint: endpoint,
-      adapterConfig: adapter.config,
-    }
-
-    const handler = async () => {
-      if (serverClosed) {
-        logger.info('Server closed, stopping recursive backgroundExecute handler chain')
-        return
-      }
-
-      logger.debug(`Calling background execute for endpoint "${endpoint.name}"`)
-      const timeToWait = await backgroundExecute(context)
-      logger.debug(
-        `Finished background execute for endpoint "${endpoint.name}", sleeping for ${timeToWait}ms`,
-      )
-      await sleep(timeToWait)
-      handler()
-    }
-
-    // Start recursive async calls
-    handler()
-  }
-}

package/src/cache/factory.ts

@@ -1,28 +0,0 @@
-import Redis from 'ioredis'
-import { AdapterConfig } from '../config'
-import { makeLogger } from '../util'
-import { LocalCache } from './local'
-import * as cacheMetrics from './metrics'
-import { RedisCache } from './redis'
-
-const logger = makeLogger('CacheFactory')
-export class CacheFactory {
-  static buildCache(config: AdapterConfig) {
-    logger.info(`Using "${config.CACHE_TYPE}" cache.`)
-    if (config.CACHE_TYPE === 'local') {
-      return new LocalCache()
-    } else if (config.CACHE_TYPE === 'redis') {
-      const redis = new Redis({
-        enableAutoPipelining: true, // This will make multiple commands be batch automatically
-        host: config.CACHE_REDIS_HOST,
-        port: config.CACHE_REDIS_PORT,
-      })
-      redis.on('connect', () => {
-        cacheMetrics.redisConnectionsOpen.inc()
-      })
-      // TODO: Maybe track active redis connections instead of just total count of connections created
-      // Use a Gauge and track with a combo of connect/end events listeners
-      return new RedisCache(redis)
-    }
-  }
-}

package/src/cache/index.ts

@@ -1,236 +0,0 @@
-import { FastifyReply } from 'fastify'
-import { AdapterEndpoint, InitializedAdapter } from '../adapter'
-import { AdapterConfig } from '../config'
-import {
-  AdapterMiddlewareBuilder,
-  AdapterRequest,
-  AdapterResponse,
-  makeLogger,
-  sleep,
-} from '../util'
-import * as cacheMetrics from './metrics'
-
-export * from './local'
-export * from './redis'
-
-const logger = makeLogger('Cache')
-
-/**
- * An object describing an entry in the cache.
- * @typeParam T - the type of the entry's value
- */
-export interface CacheEntry<T> {
-  key: string
-  value: T
-}
-
-/**
- * Generic interface for a local or remote Cache.
- * @typeParam T - the type of the cache entries' values
- */
-export interface Cache<T = unknown> {
-  /**
-   * Gets an item from the Cache.
-   *
-   * @param key - the key of the desired entry for which to fetch its value
-   * @returns a Promise of the entry's value, or undefined if not found / expired.
-   */
-  get: (key: string) => Promise<T | undefined>
-
-  /**
-   * Sets an item in the Cache.
-   *
-   * @param key - the key of the new entry
-   * @param value - the value of the new entry
-   * @param ttl - the time in milliseconds until the entry expires
-   * @returns an empty Promise that resolves when the entry has been set
-   */
-  set: (key: string, value: T, ttl: number) => Promise<void>
-
-  /**
-   * Sets a list of items in the Cache.
-   *
-   * @param entries - a list of cache entries
-   * @param ttl - the time in milliseconds until the entries expire
-   * @returns an empty Promise that resolves when all entries have been set
-   */
-  setMany: (entries: CacheEntry<T>[], ttl: number) => Promise<void>
-
-  /**
-   * Deletes the specified item from the Cache
-   *
-   * @param key - the key of the entry to be deleted
-   * @returns an empty Promise that resolves when the entry has been deleted
-   */
-  delete: (key: string) => Promise<void>
-}
-
-// Uses calculateKey to generate a unique key from the endpoint name, data, and input parameters
-export const calculateCacheKey = (
-  {
-    adapterEndpoint,
-    adapterConfig,
-  }: {
-    adapterEndpoint: AdapterEndpoint
-    adapterConfig: AdapterConfig
-  },
-  data: unknown,
-): string => {
-  const paramNames = Object.keys(adapterEndpoint.inputParameters)
-  if (paramNames.length === 0) {
-    logger.trace(`Using default cache key ${adapterConfig.DEFAULT_CACHE_KEY}`)
-    return adapterConfig.DEFAULT_CACHE_KEY
-  }
-  return `${adapterEndpoint.name}-${calculateKey(data, paramNames)}`
-}
-
-export const calculateFeedId = (adapterEndpoint: AdapterEndpoint, data: unknown): string => {
-  const paramNames = Object.keys(adapterEndpoint.inputParameters)
-  if (paramNames.length === 0) {
-    logger.trace(`Cannot generate Feed ID without input parameters`)
-    return 'N/A'
-  }
-  return calculateKey(data, paramNames)
-}
-
-/**
- * Calculates a unique key from the provided data.
- *
- * @param data - the request data/body, i.e. the adapter input params
- * @param paramNames - the keys from adapter endpoint input parameters
- * @returns the calculated unique key
- *
- * @example
- * ```
- * calculateKey({ base: 'ETH', quote: 'BTC' }, ['base','quote'])
- * // equals `|base:eth|quote:btc`
- * ```
- */
-export const calculateKey = (data: unknown, paramNames: string[]): string => {
-  if (data && typeof data !== 'object') {
-    throw new Error('Data to calculate cache key should be an object')
-  }
-
-  const params = data as Record<string, unknown>
-
-  let cacheKey = '' // TODO: Maybe there's a faster String Builder
-  for (const paramName of paramNames) {
-    // Ignore overrides param when generating cache keys
-    // TODO: expand support for ignoring `includes` and `tokenOverrides` params when generating keys
-    if (paramName === 'overrides') {
-      continue
-    }
-    const param = params[paramName]
-    if (param === undefined) {
-      continue
-    }
-
-    cacheKey += `|${paramName}:`
-    switch (typeof param) {
-      case 'string':
-        cacheKey += param.toLowerCase()
-        break
-      case 'number':
-      case 'boolean':
-        cacheKey += param.toString()
-        break
-      case 'object':
-        // TODO: Implement object sorting for deterministic cache key generation
-        cacheKey += JSON.stringify(param)
-    }
-  }
-
-  // TODO: Check max size, potential issue
-  logger.trace(`Generated cache key for request: "${cacheKey}"`)
-  return cacheKey
-}
-
-// Calculate the amount of time the non-expired entry has been in the cache
-const calculateStaleness = (expirationTimestamp: number | undefined, ttl: number): number => {
-  if (expirationTimestamp) {
-    const createTimestamp = expirationTimestamp - ttl
-    return (Date.now() - createTimestamp) / 1000
-  } else {
-    // If expirationTimestamp is not available, staleness cannot be calculated
-    // Defaults to ttl for metrics purposes
-    return ttl
-  }
-}
-
-/**
- * Polls the provided Cache for an AdapterResponse set in the provided key. If the maximum
- * amount of retries is exceeded, it returns undefined instead.
- *
- * @param cache - a Cache instance
- * @param key - the key generated from an AdapterRequest that corresponds to the desired AdapterResponse
- * @param retry - current retry, only for internal use
- * @returns the AdapterResponse if found, else undefined
- */
-export const pollResponseFromCache = async (
-  cache: Cache<AdapterResponse>,
-  key: string,
-  options: {
-    maxRetries: number
-    sleep: number
-  },
-  retry = 0,
-): Promise<AdapterResponse | undefined> => {
-  if (retry > options.maxRetries) {
-    // Ideally this shouldn't happen often (p99 of reqs should be found in the cache)
-    logger.info('Exceeded max cache polling retries')
-    return undefined
-  }
-
-  logger.trace('Getting response from cache...')
-  const response = await cache.get(key)
-  if (response) {
-    logger.trace('Got response from cache')
-    return response
-  }
-
-  if (options.maxRetries === 0) {
-    logger.debug(`Response not found, retries disabled`)
-    return undefined
-  }
-
-  logger.debug(`Response not found, sleeping ${options.sleep} milliseconds...`)
-  await sleep(options.sleep)
-
-  return pollResponseFromCache(cache, key, options, retry + 1)
-}
-
-/**
- * Given a Cache instance in the adapter dependencies, builds a middleware function that will perform
- * a get from said Cache and return that if found; otherwise it'll continue the middleware chain.
- *
- * @param adapter - an initialized adapter
- * @returns the cache middleware function
- */
-export const buildCacheMiddleware: AdapterMiddlewareBuilder =
-  (adapter: InitializedAdapter) => async (req: AdapterRequest, res: FastifyReply) => {
-    const response = await (adapter.dependencies.cache as Cache<AdapterResponse>).get(
-      req.requestContext.cacheKey,
-    )
-
-    if (response) {
-      logger.debug('Found response from cache, sending that')
-      if (adapter.config.METRICS_ENABLED && adapter.config.EXPERIMENTAL_METRICS_ENABLED) {
-        const label = cacheMetrics.cacheMetricsLabel(
-          req.requestContext.cacheKey,
-          req.requestContext.meta?.metrics?.feedId || 'N/A',
-          adapter.config.CACHE_TYPE,
-        )
-
-        // Record cache staleness and cache get count and value
-        const staleness = calculateStaleness(response.maxAge, adapter.config.CACHE_MAX_AGE)
-        cacheMetrics.cacheGet(label, response.result, staleness)
-        req.requestContext.meta = {
-          ...req.requestContext.meta,
-          metrics: { ...req.requestContext.meta?.metrics, cacheHit: true },
-        }
-      }
-      return res.send(response)
-    }
-
-    logger.debug('Did not find response in cache, moving to next middleware')
-  }

package/src/cache/local.ts

@@ -1,73 +0,0 @@
-import { AdapterResponse, makeLogger } from '../util'
-import { Cache, CacheEntry } from './index'
-import * as cacheMetrics from './metrics'
-
-const logger = makeLogger('LocalCache')
-
-/**
- * Type for a value stored in a LocalCache entry.
- *
- * @typeParam T - the type for the entry's value
- */
-export interface LocalCacheEntry<T> {
-  expirationTimestamp: number
-  value: T
-}
-
-/**
- * Local implementation of a Cache. It uses a simple js Object, storing entries with both
- * a value and an expiration timestamp. Expired entries are deleted on reads (i.e. no background gc/upkeep).
- *
- * @typeParam T - the type for the entries' values
- */
-export class LocalCache<T = unknown> implements Cache<T> {
-  store: Record<string, LocalCacheEntry<T>> = {}
-
-  async get(key: string): Promise<T | undefined> {
-    logger.trace(`Getting key ${key}`)
-    const entry = this.store[key]
-
-    if (!entry) {
-      logger.debug(`No entry in local cache for key "${key}", returning undefined`)
-      return undefined
-    }
-
-    const expired = entry.expirationTimestamp <= Date.now()
-    if (expired) {
-      logger.debug('Entry in local cache expired, deleting and returning undefined')
-      this.delete(key)
-      return undefined
-    } else {
-      logger.debug('Found valid entry in local cache, returning value')
-      return entry.value
-    }
-  }
-
-  async delete(key: string): Promise<void> {
-    logger.trace(`Deleting key ${key}`)
-    delete this.store[key] // Deletes are slower than ignoring or setting null, fyi
-  }
-
-  async set(key: string, value: T, ttl: number): Promise<void> {
-    logger.trace(`Setting key ${key} with ttl ${ttl}`)
-    this.store[key] = {
-      value,
-      expirationTimestamp: Date.now() + ttl,
-    }
-
-    // Only record metrics if feed Id is present, otherwise assuming value is not adapter response to record
-    const feedId = (value as unknown as AdapterResponse).meta?.metrics?.feedId
-    if (feedId){
-      // Record cache set count, max age, and staleness (set to 0 for cache set)
-      const label = cacheMetrics.cacheMetricsLabel(key, feedId, cacheMetrics.CacheTypes.Local)
-      cacheMetrics.cacheSet(label, ttl)
-    }
-  }
-
-  async setMany(entries: CacheEntry<T>[], ttl: number): Promise<void> {
-    logger.trace(`Setting a bunch of keys with ttl ${ttl}`)
-    for (const { key, value } of entries) {
-      this.set(key, value, ttl)
-    }
-  }
-}