@chainlink/external-adapter-framework 0.0.6 → 0.0.7

package/src/cache/index.ts

@@ -1,236 +0,0 @@
-import { FastifyReply } from 'fastify'
-import { AdapterEndpoint, InitializedAdapter } from '../adapter'
-import { AdapterConfig } from '../config'
-import {
-  AdapterMiddlewareBuilder,
-  AdapterRequest,
-  AdapterResponse,
-  makeLogger,
-  sleep,
-} from '../util'
-import * as cacheMetrics from './metrics'
-
-export * from './local'
-export * from './redis'
-
-const logger = makeLogger('Cache')
-
-/**
- * An object describing an entry in the cache.
- * @typeParam T - the type of the entry's value
- */
-export interface CacheEntry<T> {
-  key: string
-  value: T
-}
-
-/**
- * Generic interface for a local or remote Cache.
- * @typeParam T - the type of the cache entries' values
- */
-export interface Cache<T = unknown> {
-  /**
-   * Gets an item from the Cache.
-   *
-   * @param key - the key of the desired entry for which to fetch its value
-   * @returns a Promise of the entry's value, or undefined if not found / expired.
-   */
-  get: (key: string) => Promise<T | undefined>
-
-  /**
-   * Sets an item in the Cache.
-   *
-   * @param key - the key of the new entry
-   * @param value - the value of the new entry
-   * @param ttl - the time in milliseconds until the entry expires
-   * @returns an empty Promise that resolves when the entry has been set
-   */
-  set: (key: string, value: T, ttl: number) => Promise<void>
-
-  /**
-   * Sets a list of items in the Cache.
-   *
-   * @param entries - a list of cache entries
-   * @param ttl - the time in milliseconds until the entries expire
-   * @returns an empty Promise that resolves when all entries have been set
-   */
-  setMany: (entries: CacheEntry<T>[], ttl: number) => Promise<void>
-
-  /**
-   * Deletes the specified item from the Cache
-   *
-   * @param key - the key of the entry to be deleted
-   * @returns an empty Promise that resolves when the entry has been deleted
-   */
-  delete: (key: string) => Promise<void>
-}
-
-// Uses calculateKey to generate a unique key from the endpoint name, data, and input parameters
-export const calculateCacheKey = (
-  {
-    adapterEndpoint,
-    adapterConfig,
-  }: {
-    adapterEndpoint: AdapterEndpoint
-    adapterConfig: AdapterConfig
-  },
-  data: unknown,
-): string => {
-  const paramNames = Object.keys(adapterEndpoint.inputParameters)
-  if (paramNames.length === 0) {
-    logger.trace(`Using default cache key ${adapterConfig.DEFAULT_CACHE_KEY}`)
-    return adapterConfig.DEFAULT_CACHE_KEY
-  }
-  return `${adapterEndpoint.name}-${calculateKey(data, paramNames)}`
-}
-
-export const calculateFeedId = (adapterEndpoint: AdapterEndpoint, data: unknown): string => {
-  const paramNames = Object.keys(adapterEndpoint.inputParameters)
-  if (paramNames.length === 0) {
-    logger.trace(`Cannot generate Feed ID without input parameters`)
-    return 'N/A'
-  }
-  return calculateKey(data, paramNames)
-}
-
-/**
- * Calculates a unique key from the provided data.
- *
- * @param data - the request data/body, i.e. the adapter input params
- * @param paramNames - the keys from adapter endpoint input parameters
- * @returns the calculated unique key
- *
- * @example
- * ```
- * calculateKey({ base: 'ETH', quote: 'BTC' }, ['base','quote'])
- * // equals `|base:eth|quote:btc`
- * ```
- */
-export const calculateKey = (data: unknown, paramNames: string[]): string => {
-  if (data && typeof data !== 'object') {
-    throw new Error('Data to calculate cache key should be an object')
-  }
-
-  const params = data as Record<string, unknown>
-
-  let cacheKey = '' // TODO: Maybe there's a faster String Builder
-  for (const paramName of paramNames) {
-    // Ignore overrides param when generating cache keys
-    // TODO: expand support for ignoring `includes` and `tokenOverrides` params when generating keys
-    if (paramName === 'overrides') {
-      continue
-    }
-    const param = params[paramName]
-    if (param === undefined) {
-      continue
-    }
-
-    cacheKey += `|${paramName}:`
-    switch (typeof param) {
-      case 'string':
-        cacheKey += param.toLowerCase()
-        break
-      case 'number':
-      case 'boolean':
-        cacheKey += param.toString()
-        break
-      case 'object':
-        // TODO: Implement object sorting for deterministic cache key generation
-        cacheKey += JSON.stringify(param)
-    }
-  }
-
-  // TODO: Check max size, potential issue
-  logger.trace(`Generated cache key for request: "${cacheKey}"`)
-  return cacheKey
-}
-
-// Calculate the amount of time the non-expired entry has been in the cache
-const calculateStaleness = (expirationTimestamp: number | undefined, ttl: number): number => {
-  if (expirationTimestamp) {
-    const createTimestamp = expirationTimestamp - ttl
-    return (Date.now() - createTimestamp) / 1000
-  } else {
-    // If expirationTimestamp is not available, staleness cannot be calculated
-    // Defaults to ttl for metrics purposes
-    return ttl
-  }
-}
-
-/**
- * Polls the provided Cache for an AdapterResponse set in the provided key. If the maximum
- * amount of retries is exceeded, it returns undefined instead.
- *
- * @param cache - a Cache instance
- * @param key - the key generated from an AdapterRequest that corresponds to the desired AdapterResponse
- * @param retry - current retry, only for internal use
- * @returns the AdapterResponse if found, else undefined
- */
-export const pollResponseFromCache = async (
-  cache: Cache<AdapterResponse>,
-  key: string,
-  options: {
-    maxRetries: number
-    sleep: number
-  },
-  retry = 0,
-): Promise<AdapterResponse | undefined> => {
-  if (retry > options.maxRetries) {
-    // Ideally this shouldn't happen often (p99 of reqs should be found in the cache)
-    logger.info('Exceeded max cache polling retries')
-    return undefined
-  }
-
-  logger.trace('Getting response from cache...')
-  const response = await cache.get(key)
-  if (response) {
-    logger.trace('Got response from cache')
-    return response
-  }
-
-  if (options.maxRetries === 0) {
-    logger.debug(`Response not found, retries disabled`)
-    return undefined
-  }
-
-  logger.debug(`Response not found, sleeping ${options.sleep} milliseconds...`)
-  await sleep(options.sleep)
-
-  return pollResponseFromCache(cache, key, options, retry + 1)
-}
-
-/**
- * Given a Cache instance in the adapter dependencies, builds a middleware function that will perform
- * a get from said Cache and return that if found; otherwise it'll continue the middleware chain.
- *
- * @param adapter - an initialized adapter
- * @returns the cache middleware function
- */
-export const buildCacheMiddleware: AdapterMiddlewareBuilder =
-  (adapter: InitializedAdapter) => async (req: AdapterRequest, res: FastifyReply) => {
-    const response = await (adapter.dependencies.cache as Cache<AdapterResponse>).get(
-      req.requestContext.cacheKey,
-    )
-
-    if (response) {
-      logger.debug('Found response from cache, sending that')
-      if (adapter.config.METRICS_ENABLED && adapter.config.EXPERIMENTAL_METRICS_ENABLED) {
-        const label = cacheMetrics.cacheMetricsLabel(
-          req.requestContext.cacheKey,
-          req.requestContext.meta?.metrics?.feedId || 'N/A',
-          adapter.config.CACHE_TYPE,
-        )
-
-        // Record cache staleness and cache get count and value
-        const staleness = calculateStaleness(response.maxAge, adapter.config.CACHE_MAX_AGE)
-        cacheMetrics.cacheGet(label, response.result, staleness)
-        req.requestContext.meta = {
-          ...req.requestContext.meta,
-          metrics: { ...req.requestContext.meta?.metrics, cacheHit: true },
-        }
-      }
-      return res.send(response)
-    }
-
-    logger.debug('Did not find response in cache, moving to next middleware')
-  }

package/src/cache/local.ts

@@ -1,73 +0,0 @@
-import { AdapterResponse, makeLogger } from '../util'
-import { Cache, CacheEntry } from './index'
-import * as cacheMetrics from './metrics'
-
-const logger = makeLogger('LocalCache')
-
-/**
- * Type for a value stored in a LocalCache entry.
- *
- * @typeParam T - the type for the entry's value
- */
-export interface LocalCacheEntry<T> {
-  expirationTimestamp: number
-  value: T
-}
-
-/**
- * Local implementation of a Cache. It uses a simple js Object, storing entries with both
- * a value and an expiration timestamp. Expired entries are deleted on reads (i.e. no background gc/upkeep).
- *
- * @typeParam T - the type for the entries' values
- */
-export class LocalCache<T = unknown> implements Cache<T> {
-  store: Record<string, LocalCacheEntry<T>> = {}
-
-  async get(key: string): Promise<T | undefined> {
-    logger.trace(`Getting key ${key}`)
-    const entry = this.store[key]
-
-    if (!entry) {
-      logger.debug(`No entry in local cache for key "${key}", returning undefined`)
-      return undefined
-    }
-
-    const expired = entry.expirationTimestamp <= Date.now()
-    if (expired) {
-      logger.debug('Entry in local cache expired, deleting and returning undefined')
-      this.delete(key)
-      return undefined
-    } else {
-      logger.debug('Found valid entry in local cache, returning value')
-      return entry.value
-    }
-  }
-
-  async delete(key: string): Promise<void> {
-    logger.trace(`Deleting key ${key}`)
-    delete this.store[key] // Deletes are slower than ignoring or setting null, fyi
-  }
-
-  async set(key: string, value: T, ttl: number): Promise<void> {
-    logger.trace(`Setting key ${key} with ttl ${ttl}`)
-    this.store[key] = {
-      value,
-      expirationTimestamp: Date.now() + ttl,
-    }
-
-    // Only record metrics if feed Id is present, otherwise assuming value is not adapter response to record
-    const feedId = (value as unknown as AdapterResponse).meta?.metrics?.feedId
-    if (feedId){
-      // Record cache set count, max age, and staleness (set to 0 for cache set)
-      const label = cacheMetrics.cacheMetricsLabel(key, feedId, cacheMetrics.CacheTypes.Local)
-      cacheMetrics.cacheSet(label, ttl)
-    }
-  }
-
-  async setMany(entries: CacheEntry<T>[], ttl: number): Promise<void> {
-    logger.trace(`Setting a bunch of keys with ttl ${ttl}`)
-    for (const { key, value } of entries) {
-      this.set(key, value, ttl)
-    }
-  }
-}

package/src/cache/metrics.ts

@@ -1,112 +0,0 @@
-import * as client from 'prom-client'
-
-interface CacheMetricsLabels {
-  participant_id: string
-  feed_id: string
-  cache_type: string
-  // Is_from_ws?: string
-}
-
-export const cacheGet = (label: CacheMetricsLabels, value: unknown, staleness: number) => {
-  if (typeof value === 'number' || typeof value === 'string') {
-    const parsedValue = Number(value)
-    if (!Number.isNaN(parsedValue) && Number.isFinite(parsedValue)) {
-      cacheDataGetValues.labels(label).set(parsedValue)
-    }
-  }
-  cacheDataGetCount.labels(label).inc()
-  cacheDataStalenessSeconds.labels(label).set(staleness)
-}
-
-export const cacheSet = (label: CacheMetricsLabels, maxAge: number) => {
-  cacheDataSetCount.labels(label).inc()
-  cacheDataMaxAge.labels(label).set(maxAge)
-  cacheDataStalenessSeconds.labels(label).set(0)
-}
-
-export const cacheMetricsLabel = (cacheKey: string, feedId: string, cacheType: string) => ({
-  participant_id: cacheKey,
-  feed_id: feedId,
-  cache_type: cacheType,
-})
-
-export enum CacheTypes {
-  Redis = 'redis',
-  Local = 'local',
-}
-
-export enum CMD_SENT_STATUS {
-  TIMEOUT = 'TIMEOUT',
-  FAIL = 'FAIL',
-  SUCCESS = 'SUCCESS',
-}
-
-const baseLabels = [
-  'feed_id',
-  'participant_id',
-  'cache_type',
-  'is_from_ws',
-  'experimental',
-] as const
-
-// Skipping this metrics for v3
-// const cache_execution_duration_seconds = new client.Histogram({
-//   name: 'cache_execution_duration_seconds',
-//   help: 'A histogram bucket of the distribution of cache execution durations',
-//   labelNames: [...baseLabels, 'cache_hit'] as const,
-//   buckets: [0.01, 0.1, 1, 10],
-// })
-
-const cacheDataGetCount = new client.Counter({
-  name: 'cache_data_get_count',
-  help: 'A counter that increments every time a value is fetched from the cache',
-  labelNames: baseLabels,
-})
-
-const cacheDataGetValues = new client.Gauge({
-  name: 'cache_data_get_values',
-  help: 'A gauge keeping track of values being fetched from cache',
-  labelNames: baseLabels,
-})
-
-const cacheDataMaxAge = new client.Gauge({
-  name: 'cache_data_max_age',
-  help: 'A gauge tracking the max age of stored values in the cache',
-  labelNames: baseLabels,
-})
-
-const cacheDataSetCount = new client.Counter({
-  name: 'cache_data_set_count',
-  help: 'A counter that increments every time a value is set to the cache',
-  labelNames: [...baseLabels, 'status_code'],
-})
-
-const cacheDataStalenessSeconds = new client.Gauge({
-  name: 'cache_data_staleness_seconds',
-  help: 'Observes the staleness of the data returned',
-  labelNames: baseLabels,
-})
-
-// Redis Metrics
-export const redisConnectionsOpen = new client.Counter({
-  name: 'redis_connections_open',
-  help: 'The number of redis connections that are open',
-})
-
-export const redisRetriesCount = new client.Counter({
-  name: 'redis_retries_count',
-  help: 'The number of retries that have been made to establish a redis connection',
-})
-
-export const redisCommandsSentCount = new client.Counter({
-  name: 'redis_commands_sent_count',
-  help: 'The number of redis commands sent',
-  labelNames: ['status', 'function_name'],
-})
-
-// Cache Warmer Metrics
-export const cacheWarmerCount = new client.Gauge({
-  name: 'cache_warmer_get_count',
-  help: 'The number of cache warmers running',
-  labelNames: ['isBatched'] as const,
-})

package/src/cache/redis.ts

@@ -1,93 +0,0 @@
-import Redis from 'ioredis'
-import { AdapterResponse, makeLogger } from '../util'
-import { Cache, CacheEntry } from './index'
-import * as cacheMetrics from './metrics'
-
-const logger = makeLogger('RedisCache')
-
-/**
- * Redis implementation of a Cache. It uses a simple js Object, storing entries with both
- * a value and an expiration timestamp. Expired entries are deleted on reads (i.e. no background gc/upkeep).
- *
- * @typeParam T - the type for the entries' values
- */
-// TODO: Add error handling for redis command failures
-export class RedisCache<T = unknown> implements Cache<T> {
-  constructor(private client: Redis) {}
-
-  async get(key: string): Promise<T | undefined> {
-    logger.trace(`Getting key ${key}`)
-    const value = await this.client.get(key)
-
-    // Record get command sent to Redis
-    cacheMetrics.redisCommandsSentCount
-      .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'get' })
-      .inc()
-
-    if (!value) {
-      logger.debug(`No entry in redis cache for key "${key}", returning undefined`)
-      return undefined
-    }
-
-    return JSON.parse(value) as T // TODO: Check for invalid parsing
-  }
-
-  async delete(key: string): Promise<void> {
-    logger.trace(`Deleting key ${key}`)
-    await this.client.del(key)
-
-    // Record delete command sent to Redis
-    cacheMetrics.redisCommandsSentCount
-      .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'delete' })
-      .inc()
-  }
-
-  async set(key: string, value: T, ttl: number): Promise<void> {
-    logger.trace(`Setting key ${key}`)
-    // TODO: Check for invalid stringify
-    await this.client.set(key, JSON.stringify(value), 'PX', ttl)
-
-    // Only record metrics if feed Id is present, otherwise assuming value is not adapter response to record
-    const feedId = (value as unknown as AdapterResponse).meta?.metrics?.feedId
-    if (feedId) {
-      // Record cache set count, max age, and staleness (set to 0 for cache set)
-      const label = cacheMetrics.cacheMetricsLabel(key, feedId, cacheMetrics.CacheTypes.Redis)
-      cacheMetrics.cacheSet(label, ttl)
-    }
-
-    // Record set command sent to Redis
-    cacheMetrics.redisCommandsSentCount
-      .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'set' })
-      .inc()
-  }
-
-  async setMany(entries: CacheEntry<T>[], ttl: number): Promise<void> {
-    logger.trace(`Setting a bunch of keys`)
-    // Unfortunately, there's no ttl for mset
-    let chain = this.client.multi()
-
-    for (const entry of entries) {
-      chain = chain.set(entry.key, JSON.stringify(entry.value), 'PX', ttl)
-
-      // TODO: Move to after error handling once implemented to avoid recording cache sets that failed
-      // Only record metrics if feed Id is present, otherwise assuming value is not adapter response to record
-      const feedId = (entry.value as unknown as AdapterResponse).meta?.metrics?.feedId
-      if (feedId) {
-        // Record cache set count, max age, and staleness (set to 0 for cache set)
-        const label = cacheMetrics.cacheMetricsLabel(
-          entry.key,
-          feedId,
-          cacheMetrics.CacheTypes.Redis,
-        )
-        cacheMetrics.cacheSet(label, ttl)
-      }
-    }
-
-    await chain.exec()
-
-    // Record setMany command sent to Redis
-    cacheMetrics.redisCommandsSentCount
-      .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'exec' })
-      .inc()
-  }
-}