@chainlink/external-adapter-framework 0.0.14 → 0.0.15

package/src/cache/index.ts

@@ -0,0 +1,258 @@
+import { FastifyReply } from 'fastify'
+import { AdapterEndpoint, InitializedAdapter } from '../adapter'
+import { AdapterConfig } from '../config'
+import {
+  AdapterMiddlewareBuilder,
+  AdapterRequest,
+  AdapterResponse,
+  makeLogger,
+  sleep,
+} from '../util'
+import * as cacheMetrics from './metrics'
+
+export * from './local'
+export * from './redis'
+export * from './factory'
+
+const logger = makeLogger('Cache')
+
+/**
+ * An object describing an entry in the cache.
+ * @typeParam T - the type of the entry's value
+ */
+export interface CacheEntry<T> {
+  key: string
+  value: T
+}
+
+/**
+ * Generic interface for a local or remote Cache.
+ * @typeParam T - the type of the cache entries' values
+ */
+export interface Cache<T = unknown> {
+  /**
+   * Gets an item from the Cache.
+   *
+   * @param key - the key of the desired entry for which to fetch its value
+   * @returns a Promise of the entry's value, or undefined if not found / expired.
+   */
+  get: (key: string) => Promise<T | undefined>
+
+  /**
+   * Sets an item in the Cache.
+   *
+   * @param key - the key of the new entry
+   * @param value - the value of the new entry
+   * @param ttl - the time in milliseconds until the entry expires
+   * @returns an empty Promise that resolves when the entry has been set
+   */
+  set: (key: string, value: T, ttl: number) => Promise<void>
+
+  /**
+   * Sets a list of items in the Cache.
+   *
+   * @param entries - a list of cache entries
+   * @param ttl - the time in milliseconds until the entries expire
+   * @returns an empty Promise that resolves when all entries have been set
+   */
+  setMany: (entries: CacheEntry<T>[], ttl: number) => Promise<void>
+
+  /**
+   * Deletes the specified item from the Cache
+   *
+   * @param key - the key of the entry to be deleted
+   * @returns an empty Promise that resolves when the entry has been deleted
+   */
+  delete: (key: string) => Promise<void>
+}
+
+// Uses calculateKey to generate a unique key from the endpoint name, data, and input parameters
+export const calculateCacheKey = (
+  {
+    adapterEndpoint,
+    adapterConfig,
+  }: {
+    adapterEndpoint: AdapterEndpoint
+    adapterConfig: AdapterConfig
+  },
+  data: unknown,
+): string => {
+  const paramNames = Object.keys(adapterEndpoint.inputParameters)
+  if (paramNames.length === 0) {
+    logger.trace(`Using default cache key ${adapterConfig.DEFAULT_CACHE_KEY}`)
+    return adapterConfig.DEFAULT_CACHE_KEY
+  }
+  return `${adapterEndpoint.name}-${calculateKey(data, paramNames, adapterConfig)}`
+}
+
+export const calculateFeedId = (
+  {
+    adapterEndpoint,
+    adapterConfig,
+  }: {
+    adapterEndpoint: AdapterEndpoint
+    adapterConfig: AdapterConfig
+  },
+  data: unknown,
+): string => {
+  const paramNames = Object.keys(adapterEndpoint.inputParameters)
+  if (paramNames.length === 0) {
+    logger.trace(`Cannot generate Feed ID without input parameters`)
+    return 'N/A'
+  }
+  return calculateKey(data, paramNames, adapterConfig)
+}
+
+/**
+ * Calculates a unique key from the provided data.
+ *
+ * @param data - the request data/body, i.e. the adapter input params
+ * @param paramNames - the keys from adapter endpoint input parameters
+ * @returns the calculated unique key
+ *
+ * @example
+ * ```
+ * calculateKey({ base: 'ETH', quote: 'BTC' }, ['base','quote'])
+ * // equals `|base:eth|quote:btc`
+ * ```
+ */
+export const calculateKey = (
+  data: unknown,
+  paramNames: string[],
+  adapterConfig: AdapterConfig,
+): string => {
+  if (data && typeof data !== 'object') {
+    throw new Error('Data to calculate cache key should be an object')
+  }
+
+  const params = data as Record<string, unknown>
+
+  let cacheKey = ''
+  for (const paramName of paramNames) {
+    // Ignore overrides param when generating cache keys
+    if (paramName === 'overrides') {
+      continue
+    }
+    const param = params[paramName]
+    if (param === undefined) {
+      continue
+    }
+
+    cacheKey += `|${paramName}:`
+    switch (typeof param) {
+      case 'string':
+        cacheKey += param.toLowerCase()
+        break
+      case 'number':
+      case 'boolean':
+        cacheKey += param.toString()
+        break
+      case 'object':
+        // Force cache keys to only use performant properties of the input params.
+        // If the object were to be used, we'd have to sort its properties.
+        logger.debug(
+          `Property "${paramName}" in request parameters is of type object, and won't be used in the cacheKey`,
+        )
+    }
+  }
+
+  if (cacheKey.length > adapterConfig.MAX_COMMON_KEY_SIZE) {
+    logger.warn(
+      `Generated cache key for adapter request is bigger than the MAX_COMMON_KEY_SIZE and will be truncated`,
+    )
+    cacheKey = cacheKey.slice(0, adapterConfig.MAX_COMMON_KEY_SIZE)
+  }
+
+  logger.trace(`Generated cache key for request: "${cacheKey}"`)
+  return cacheKey
+}
+
+// Calculate the amount of time the non-expired entry has been in the cache
+const calculateStaleness = (expirationTimestamp: number | undefined, ttl: number): number => {
+  if (expirationTimestamp) {
+    const createTimestamp = expirationTimestamp - ttl
+    return (Date.now() - createTimestamp) / 1000
+  } else {
+    // If expirationTimestamp is not available, staleness cannot be calculated
+    // Defaults to ttl for metrics purposes
+    return ttl
+  }
+}
+
+/**
+ * Polls the provided Cache for an AdapterResponse set in the provided key. If the maximum
+ * amount of retries is exceeded, it returns undefined instead.
+ *
+ * @param cache - a Cache instance
+ * @param key - the key generated from an AdapterRequest that corresponds to the desired AdapterResponse
+ * @param retry - current retry, only for internal use
+ * @returns the AdapterResponse if found, else undefined
+ */
+export const pollResponseFromCache = async (
+  cache: Cache<AdapterResponse>,
+  key: string,
+  options: {
+    maxRetries: number
+    sleep: number
+  },
+  retry = 0,
+): Promise<AdapterResponse | undefined> => {
+  if (retry > options.maxRetries) {
+    // Ideally this shouldn't happen often (p99 of reqs should be found in the cache)
+    logger.info('Exceeded max cache polling retries')
+    return undefined
+  }
+
+  logger.trace('Getting response from cache...')
+  const response = await cache.get(key)
+  if (response) {
+    logger.trace('Got response from cache')
+    return response
+  }
+
+  if (options.maxRetries === 0) {
+    logger.debug(`Response not found, retries disabled`)
+    return undefined
+  }
+
+  logger.debug(`Response not found, sleeping ${options.sleep} milliseconds...`)
+  await sleep(options.sleep)
+
+  return pollResponseFromCache(cache, key, options, retry + 1)
+}
+
+/**
+ * Given a Cache instance in the adapter dependencies, builds a middleware function that will perform
+ * a get from said Cache and return that if found; otherwise it'll continue the middleware chain.
+ *
+ * @param adapter - an initialized adapter
+ * @returns the cache middleware function
+ */
+export const buildCacheMiddleware: AdapterMiddlewareBuilder =
+  (adapter: InitializedAdapter) => async (req: AdapterRequest, res: FastifyReply) => {
+    const response = await (adapter.dependencies.cache as Cache<AdapterResponse>).get(
+      req.requestContext.cacheKey,
+    )
+
+    if (response) {
+      logger.debug('Found response from cache, sending that')
+      if (adapter.config.METRICS_ENABLED && adapter.config.EXPERIMENTAL_METRICS_ENABLED) {
+        const label = cacheMetrics.cacheMetricsLabel(
+          req.requestContext.cacheKey,
+          req.requestContext.meta?.metrics?.feedId || 'N/A',
+          adapter.config.CACHE_TYPE,
+        )
+
+        // Record cache staleness and cache get count and value
+        const staleness = calculateStaleness(response.maxAge, adapter.config.CACHE_MAX_AGE)
+        cacheMetrics.cacheGet(label, response.result, staleness)
+        req.requestContext.meta = {
+          ...req.requestContext.meta,
+          metrics: { ...req.requestContext.meta?.metrics, cacheHit: true },
+        }
+      }
+      return res.send(response)
+    }
+
+    logger.debug('Did not find response in cache, moving to next middleware')
+  }

package/src/cache/local.ts

@@ -0,0 +1,73 @@
+import { AdapterResponse, makeLogger } from '../util'
+import { Cache, CacheEntry } from './index'
+import * as cacheMetrics from './metrics'
+
+const logger = makeLogger('LocalCache')
+
+/**
+ * Type for a value stored in a LocalCache entry.
+ *
+ * @typeParam T - the type for the entry's value
+ */
+export interface LocalCacheEntry<T> {
+  expirationTimestamp: number
+  value: T
+}
+
+/**
+ * Local implementation of a Cache. It uses a simple js Object, storing entries with both
+ * a value and an expiration timestamp. Expired entries are deleted on reads (i.e. no background gc/upkeep).
+ *
+ * @typeParam T - the type for the entries' values
+ */
+export class LocalCache<T = unknown> implements Cache<T> {
+  store: Record<string, LocalCacheEntry<T>> = {}
+
+  async get(key: string): Promise<T | undefined> {
+    logger.trace(`Getting key ${key}`)
+    const entry = this.store[key]
+
+    if (!entry) {
+      logger.debug(`No entry in local cache for key "${key}", returning undefined`)
+      return undefined
+    }
+
+    const expired = entry.expirationTimestamp <= Date.now()
+    if (expired) {
+      logger.debug('Entry in local cache expired, deleting and returning undefined')
+      this.delete(key)
+      return undefined
+    } else {
+      logger.debug('Found valid entry in local cache, returning value')
+      return entry.value
+    }
+  }
+
+  async delete(key: string): Promise<void> {
+    logger.trace(`Deleting key ${key}`)
+    delete this.store[key] // Deletes are slower than ignoring or setting null, fyi
+  }
+
+  async set(key: string, value: T, ttl: number): Promise<void> {
+    logger.trace(`Setting key ${key} with ttl ${ttl}`)
+    this.store[key] = {
+      value,
+      expirationTimestamp: Date.now() + ttl,
+    }
+
+    // Only record metrics if feed Id is present, otherwise assuming value is not adapter response to record
+    const feedId = (value as unknown as AdapterResponse).meta?.metrics?.feedId
+    if (feedId) {
+      // Record cache set count, max age, and staleness (set to 0 for cache set)
+      const label = cacheMetrics.cacheMetricsLabel(key, feedId, cacheMetrics.CacheTypes.Local)
+      cacheMetrics.cacheSet(label, ttl)
+    }
+  }
+
+  async setMany(entries: CacheEntry<T>[], ttl: number): Promise<void> {
+    logger.trace(`Setting a bunch of keys with ttl ${ttl}`)
+    for (const { key, value } of entries) {
+      this.set(key, value, ttl)
+    }
+  }
+}

package/src/cache/metrics.ts

@@ -0,0 +1,112 @@
+import * as client from 'prom-client'
+
+interface CacheMetricsLabels {
+  participant_id: string
+  feed_id: string
+  cache_type: string
+  // Is_from_ws?: string
+}
+
+export const cacheGet = (label: CacheMetricsLabels, value: unknown, staleness: number) => {
+  if (typeof value === 'number' || typeof value === 'string') {
+    const parsedValue = Number(value)
+    if (!Number.isNaN(parsedValue) && Number.isFinite(parsedValue)) {
+      cacheDataGetValues.labels(label).set(parsedValue)
+    }
+  }
+  cacheDataGetCount.labels(label).inc()
+  cacheDataStalenessSeconds.labels(label).set(staleness)
+}
+
+export const cacheSet = (label: CacheMetricsLabels, maxAge: number) => {
+  cacheDataSetCount.labels(label).inc()
+  cacheDataMaxAge.labels(label).set(maxAge)
+  cacheDataStalenessSeconds.labels(label).set(0)
+}
+
+export const cacheMetricsLabel = (cacheKey: string, feedId: string, cacheType: string) => ({
+  participant_id: cacheKey,
+  feed_id: feedId,
+  cache_type: cacheType,
+})
+
+export enum CacheTypes {
+  Redis = 'redis',
+  Local = 'local',
+}
+
+export enum CMD_SENT_STATUS {
+  TIMEOUT = 'TIMEOUT',
+  FAIL = 'FAIL',
+  SUCCESS = 'SUCCESS',
+}
+
+const baseLabels = [
+  'feed_id',
+  'participant_id',
+  'cache_type',
+  'is_from_ws',
+  'experimental',
+] as const
+
+// Skipping this metrics for v3
+// const cache_execution_duration_seconds = new client.Histogram({
+//   name: 'cache_execution_duration_seconds',
+//   help: 'A histogram bucket of the distribution of cache execution durations',
+//   labelNames: [...baseLabels, 'cache_hit'] as const,
+//   buckets: [0.01, 0.1, 1, 10],
+// })
+
+const cacheDataGetCount = new client.Counter({
+  name: 'cache_data_get_count',
+  help: 'A counter that increments every time a value is fetched from the cache',
+  labelNames: baseLabels,
+})
+
+const cacheDataGetValues = new client.Gauge({
+  name: 'cache_data_get_values',
+  help: 'A gauge keeping track of values being fetched from cache',
+  labelNames: baseLabels,
+})
+
+const cacheDataMaxAge = new client.Gauge({
+  name: 'cache_data_max_age',
+  help: 'A gauge tracking the max age of stored values in the cache',
+  labelNames: baseLabels,
+})
+
+const cacheDataSetCount = new client.Counter({
+  name: 'cache_data_set_count',
+  help: 'A counter that increments every time a value is set to the cache',
+  labelNames: [...baseLabels, 'status_code'],
+})
+
+const cacheDataStalenessSeconds = new client.Gauge({
+  name: 'cache_data_staleness_seconds',
+  help: 'Observes the staleness of the data returned',
+  labelNames: baseLabels,
+})
+
+// Redis Metrics
+export const redisConnectionsOpen = new client.Counter({
+  name: 'redis_connections_open',
+  help: 'The number of redis connections that are open',
+})
+
+export const redisRetriesCount = new client.Counter({
+  name: 'redis_retries_count',
+  help: 'The number of retries that have been made to establish a redis connection',
+})
+
+export const redisCommandsSentCount = new client.Counter({
+  name: 'redis_commands_sent_count',
+  help: 'The number of redis commands sent',
+  labelNames: ['status', 'function_name'],
+})
+
+// Cache Warmer Metrics
+export const cacheWarmerCount = new client.Gauge({
+  name: 'cache_warmer_get_count',
+  help: 'The number of cache warmers running',
+  labelNames: ['isBatched'] as const,
+})

package/src/cache/redis.ts

@@ -0,0 +1,93 @@
+import Redis from 'ioredis'
+import { AdapterResponse, makeLogger } from '../util'
+import { Cache, CacheEntry } from './index'
+import * as cacheMetrics from './metrics'
+
+const logger = makeLogger('RedisCache')
+
+/**
+ * Redis implementation of a Cache. It uses a simple js Object, storing entries with both
+ * a value and an expiration timestamp. Expired entries are deleted on reads (i.e. no background gc/upkeep).
+ *
+ * @typeParam T - the type for the entries' values
+ */
+export class RedisCache<T = unknown> implements Cache<T> {
+  constructor(private client: Redis) {}
+
+  async get(key: string): Promise<T | undefined> {
+    logger.trace(`Getting key ${key}`)
+    const value = await this.client.get(key)
+
+    // Record get command sent to Redis
+    cacheMetrics.redisCommandsSentCount
+      .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'get' })
+      .inc()
+
+    if (!value) {
+      logger.debug(`No entry in redis cache for key "${key}", returning undefined`)
+      return undefined
+    }
+
+    return JSON.parse(value) as T
+  }
+
+  async delete(key: string): Promise<void> {
+    logger.trace(`Deleting key ${key}`)
+    await this.client.del(key)
+
+    // Record delete command sent to Redis
+    cacheMetrics.redisCommandsSentCount
+      .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'delete' })
+      .inc()
+  }
+
+  async set(key: string, value: T, ttl: number): Promise<void> {
+    logger.trace(`Setting key ${key}`)
+    await this.client.set(key, JSON.stringify(value), 'PX', ttl)
+
+    // Only record metrics if feed Id is present, otherwise assuming value is not adapter response to record
+    const feedId = (value as unknown as AdapterResponse).meta?.metrics?.feedId
+    if (feedId) {
+      // Record cache set count, max age, and staleness (set to 0 for cache set)
+      const label = cacheMetrics.cacheMetricsLabel(key, feedId, cacheMetrics.CacheTypes.Redis)
+      cacheMetrics.cacheSet(label, ttl)
+    }
+
+    // Record set command sent to Redis
+    cacheMetrics.redisCommandsSentCount
+      .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'set' })
+      .inc()
+  }
+
+  async setMany(entries: CacheEntry<T>[], ttl: number): Promise<void> {
+    logger.trace(`Setting a bunch of keys`)
+    // Unfortunately, there's no ttl for mset
+    let chain = this.client.multi()
+
+    for (const entry of entries) {
+      chain = chain.set(entry.key, JSON.stringify(entry.value), 'PX', ttl)
+    }
+
+    await chain.exec()
+
+    // Loop again, but this time to record these in metrics
+    for (const entry of entries) {
+      // Only record metrics if feed Id is present, otherwise assuming value is not adapter response to record
+      const feedId = (entry.value as unknown as AdapterResponse).meta?.metrics?.feedId
+      if (feedId) {
+        // Record cache set count, max age, and staleness (set to 0 for cache set)
+        const label = cacheMetrics.cacheMetricsLabel(
+          entry.key,
+          feedId,
+          cacheMetrics.CacheTypes.Redis,
+        )
+        cacheMetrics.cacheSet(label, ttl)
+      }
+    }
+
+    // Record setMany command sent to Redis
+    cacheMetrics.redisCommandsSentCount
+      .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'exec' })
+      .inc()
+  }
+}