@chainlink/external-adapter-framework 0.0.6 → 0.0.8

package/src/cache/metrics.ts

@@ -1,112 +0,0 @@
-import * as client from 'prom-client'
-
-interface CacheMetricsLabels {
-  participant_id: string
-  feed_id: string
-  cache_type: string
-  // Is_from_ws?: string
-}
-
-export const cacheGet = (label: CacheMetricsLabels, value: unknown, staleness: number) => {
-  if (typeof value === 'number' || typeof value === 'string') {
-    const parsedValue = Number(value)
-    if (!Number.isNaN(parsedValue) && Number.isFinite(parsedValue)) {
-      cacheDataGetValues.labels(label).set(parsedValue)
-    }
-  }
-  cacheDataGetCount.labels(label).inc()
-  cacheDataStalenessSeconds.labels(label).set(staleness)
-}
-
-export const cacheSet = (label: CacheMetricsLabels, maxAge: number) => {
-  cacheDataSetCount.labels(label).inc()
-  cacheDataMaxAge.labels(label).set(maxAge)
-  cacheDataStalenessSeconds.labels(label).set(0)
-}
-
-export const cacheMetricsLabel = (cacheKey: string, feedId: string, cacheType: string) => ({
-  participant_id: cacheKey,
-  feed_id: feedId,
-  cache_type: cacheType,
-})
-
-export enum CacheTypes {
-  Redis = 'redis',
-  Local = 'local',
-}
-
-export enum CMD_SENT_STATUS {
-  TIMEOUT = 'TIMEOUT',
-  FAIL = 'FAIL',
-  SUCCESS = 'SUCCESS',
-}
-
-const baseLabels = [
-  'feed_id',
-  'participant_id',
-  'cache_type',
-  'is_from_ws',
-  'experimental',
-] as const
-
-// Skipping this metrics for v3
-// const cache_execution_duration_seconds = new client.Histogram({
-//   name: 'cache_execution_duration_seconds',
-//   help: 'A histogram bucket of the distribution of cache execution durations',
-//   labelNames: [...baseLabels, 'cache_hit'] as const,
-//   buckets: [0.01, 0.1, 1, 10],
-// })
-
-const cacheDataGetCount = new client.Counter({
-  name: 'cache_data_get_count',
-  help: 'A counter that increments every time a value is fetched from the cache',
-  labelNames: baseLabels,
-})
-
-const cacheDataGetValues = new client.Gauge({
-  name: 'cache_data_get_values',
-  help: 'A gauge keeping track of values being fetched from cache',
-  labelNames: baseLabels,
-})
-
-const cacheDataMaxAge = new client.Gauge({
-  name: 'cache_data_max_age',
-  help: 'A gauge tracking the max age of stored values in the cache',
-  labelNames: baseLabels,
-})
-
-const cacheDataSetCount = new client.Counter({
-  name: 'cache_data_set_count',
-  help: 'A counter that increments every time a value is set to the cache',
-  labelNames: [...baseLabels, 'status_code'],
-})
-
-const cacheDataStalenessSeconds = new client.Gauge({
-  name: 'cache_data_staleness_seconds',
-  help: 'Observes the staleness of the data returned',
-  labelNames: baseLabels,
-})
-
-// Redis Metrics
-export const redisConnectionsOpen = new client.Counter({
-  name: 'redis_connections_open',
-  help: 'The number of redis connections that are open',
-})
-
-export const redisRetriesCount = new client.Counter({
-  name: 'redis_retries_count',
-  help: 'The number of retries that have been made to establish a redis connection',
-})
-
-export const redisCommandsSentCount = new client.Counter({
-  name: 'redis_commands_sent_count',
-  help: 'The number of redis commands sent',
-  labelNames: ['status', 'function_name'],
-})
-
-// Cache Warmer Metrics
-export const cacheWarmerCount = new client.Gauge({
-  name: 'cache_warmer_get_count',
-  help: 'The number of cache warmers running',
-  labelNames: ['isBatched'] as const,
-})

package/src/cache/redis.ts

@@ -1,93 +0,0 @@
-import Redis from 'ioredis'
-import { AdapterResponse, makeLogger } from '../util'
-import { Cache, CacheEntry } from './index'
-import * as cacheMetrics from './metrics'
-
-const logger = makeLogger('RedisCache')
-
-/**
- * Redis implementation of a Cache. It uses a simple js Object, storing entries with both
- * a value and an expiration timestamp. Expired entries are deleted on reads (i.e. no background gc/upkeep).
- *
- * @typeParam T - the type for the entries' values
- */
-// TODO: Add error handling for redis command failures
-export class RedisCache<T = unknown> implements Cache<T> {
-  constructor(private client: Redis) {}
-
-  async get(key: string): Promise<T | undefined> {
-    logger.trace(`Getting key ${key}`)
-    const value = await this.client.get(key)
-
-    // Record get command sent to Redis
-    cacheMetrics.redisCommandsSentCount
-      .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'get' })
-      .inc()
-
-    if (!value) {
-      logger.debug(`No entry in redis cache for key "${key}", returning undefined`)
-      return undefined
-    }
-
-    return JSON.parse(value) as T // TODO: Check for invalid parsing
-  }
-
-  async delete(key: string): Promise<void> {
-    logger.trace(`Deleting key ${key}`)
-    await this.client.del(key)
-
-    // Record delete command sent to Redis
-    cacheMetrics.redisCommandsSentCount
-      .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'delete' })
-      .inc()
-  }
-
-  async set(key: string, value: T, ttl: number): Promise<void> {
-    logger.trace(`Setting key ${key}`)
-    // TODO: Check for invalid stringify
-    await this.client.set(key, JSON.stringify(value), 'PX', ttl)
-
-    // Only record metrics if feed Id is present, otherwise assuming value is not adapter response to record
-    const feedId = (value as unknown as AdapterResponse).meta?.metrics?.feedId
-    if (feedId) {
-      // Record cache set count, max age, and staleness (set to 0 for cache set)
-      const label = cacheMetrics.cacheMetricsLabel(key, feedId, cacheMetrics.CacheTypes.Redis)
-      cacheMetrics.cacheSet(label, ttl)
-    }
-
-    // Record set command sent to Redis
-    cacheMetrics.redisCommandsSentCount
-      .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'set' })
-      .inc()
-  }
-
-  async setMany(entries: CacheEntry<T>[], ttl: number): Promise<void> {
-    logger.trace(`Setting a bunch of keys`)
-    // Unfortunately, there's no ttl for mset
-    let chain = this.client.multi()
-
-    for (const entry of entries) {
-      chain = chain.set(entry.key, JSON.stringify(entry.value), 'PX', ttl)
-
-      // TODO: Move to after error handling once implemented to avoid recording cache sets that failed
-      // Only record metrics if feed Id is present, otherwise assuming value is not adapter response to record
-      const feedId = (entry.value as unknown as AdapterResponse).meta?.metrics?.feedId
-      if (feedId) {
-        // Record cache set count, max age, and staleness (set to 0 for cache set)
-        const label = cacheMetrics.cacheMetricsLabel(
-          entry.key,
-          feedId,
-          cacheMetrics.CacheTypes.Redis,
-        )
-        cacheMetrics.cacheSet(label, ttl)
-      }
-    }
-
-    await chain.exec()
-
-    // Record setMany command sent to Redis
-    cacheMetrics.redisCommandsSentCount
-      .labels({ status: cacheMetrics.CMD_SENT_STATUS.SUCCESS, function_name: 'exec' })
-      .inc()
-  }
-}

package/src/config/index.ts

@@ -1,501 +0,0 @@
-// Import { getRandomRequiredEnv, getRandomEnv, getEnv } from '../util'
-
-export const BaseSettings = {
-  // V2 compat
-  // TODO: Remove non used in v3 ones
-  // API_ENDPOINT: {
-  //   type: 'string',
-  // },
-  API_KEY: {
-    description: 'Default setting for an EA key',
-    type: 'string',
-  },
-  // API_TIMEOUT: {
-  //   type: 'number',
-  //   default: 30000,
-  // },
-  // API_VERBOSE: {
-  //   type: 'boolean',
-  // },
-  BASE_URL: {
-    description: 'Starting path for the EA handler endpoint',
-    type: 'string',
-    default: '/',
-  },
-  // CACHE_ENABLED: {
-  //   type: 'boolean',
-  //   default: true,
-  // },
-  // CACHE_KEY_GROUP: {
-  //   type: 'string',
-  // },
-  CACHE_MAX_AGE: {
-    description: 'Maximum amount of time (in ms) that a response will stay cached',
-    type: 'number',
-    default: 90000,
-  },
-  // CACHE_MAX_ITEMS: {
-  //   type: 'number',
-  //   default: 1000,
-  // },
-  // CACHE_MIN_AGE: {
-  //   type: 'number',
-  //   default: 30000,
-  // },
-  // CACHE_REDIS_CONNECTION_TIMEOUT: {
-  //   type: 'number',
-  //   default: 15000,
-  // },
-  CACHE_REDIS_HOST: {
-    description: 'Hostname for the Redis instance to be used',
-    type: 'string',
-    default: '127.0.0.1',
-  },
-  // CACHE_REDIS_MAX_QUEUED_ITEMS: {
-  //   type: 'number',
-  //   default: 100,
-  // },
-  // CACHE_REDIS_MAX_RECONNECT_COOLDOWN: {
-  //   type: 'number',
-  //   default: 3000,
-  // },
-  // CACHE_REDIS_PASSWORD: {
-  //   type: 'string',
-  // },
-  // CACHE_REDIS_PATH: {
-  //   type: 'string',
-  // },
-  CACHE_REDIS_PORT: {
-    description: 'Port for the Redis instance to be used',
-    type: 'number',
-    default: 6379,
-  },
-  // CACHE_REDIS_TIMEOUT: {
-  //   type: 'number',
-  //   default: 500,
-  // },
-  // CACHE_REDIS_URL: {
-  //   type: 'string',
-  // },
-  CACHE_TYPE: {
-    description: 'The type of cache to use throughout the EA',
-    type: 'enum',
-    default: 'local',
-    options: ['local', 'redis'],
-  },
-  // CACHE_UPDATE_AGE_ON_GET: {
-  //   type: 'boolean',
-  //   default: false,
-  // },
-  CORRELATION_ID_ENABLED: {
-    description: 'Flag to enable correlation IDs for sent requests in logging',
-    type: 'boolean',
-    default: true,
-  },
-  // DEBUG: {
-  //   type: 'boolean',
-  // },
-  // DEFAULT_WS_HEARTBEAT_INTERVAL: {
-  //   type: 'number',
-  //   default: 30000,
-  // },
-  EA_PORT: {
-    description:
-      'Port through which the EA will listen for REST requests (if mode is set to "reader" or "reader-writer")',
-    type: 'number',
-    default: 8080,
-  },
-  // ENV_ADAPTER_URL: {
-  //   type: 'string',
-  // },
-  // ERROR_CAPACITY: {
-  //   type: 'number',
-  // },
-  EXPERIMENTAL_METRICS_ENABLED: {
-    description:
-      'Flag to specify whether or not to collect metrics. Used as fallback for METRICS_ENABLED',
-    type: 'boolean',
-    default: true,
-  },
-  // LEGACY_ENV_ADAPTER_URL: {
-  //   type: 'string',
-  // },
-  LOG_LEVEL: {
-    description: 'Minimum level required for logs to be output',
-    type: 'string',
-    default: 'info',
-  },
-  METRICS_ENABLED: {
-    description: 'Flag to specify whether or not to startup the metrics server',
-    type: 'boolean',
-    default: true,
-  },
-  METRICS_NAME: {
-    description: 'Metrics name',
-    type: 'string',
-  },
-  METRICS_PORT: {
-    description: 'Port metrics will be exposed to',
-    type: 'number',
-    default: 9080,
-  },
-  METRICS_USE_BASE_URL: {
-    description: 'Flag to specify whether or not to prepend the BASE_URL to the metrics endpoint',
-    type: 'boolean',
-  },
-  RATE_LIMIT_API_TIER: {
-    description:
-      'Rate limiting tier to use from the available options for the adapter. If not present, the adapter will run using the first tier on the list.',
-    type: 'string',
-  },
-  // RATE_LIMIT_CAPACITY_MINUTE: {
-  //   type: 'number',
-  // },
-  // RATE_LIMIT_CAPACITY_SECOND: {
-  //   type: 'number',
-  // },
-  // RATE_LIMIT_CAPACITY: {
-  //   type: 'number',
-  // },
-  // RATE_LIMIT_ENABLED: {
-  //   type: 'boolean',
-  //   default: true,
-  // },
-  // RECORD: {
-  //   type: 'boolean',
-  // },
-  // REQUEST_COALESCING_ENABLED: {
-  //   type: 'boolean',
-  // },
-  // REQUEST_COALESCING_ENTROPY_MAX: {
-  //   type: 'number',
-  //   default: 0,
-  // },
-  // REQUEST_COALESCING_INTERVAL_COEFFICIENT: {
-  //   type: 'number',
-  //   default: 2,
-  // },
-  // REQUEST_COALESCING_INTERVAL_MAX: {
-  //   type: 'number',
-  //   default: 1000,
-  // },
-  // REQUEST_COALESCING_INTERVAL: {
-  //   type: 'number',
-  //   default: 100,
-  // },
-  // REQUEST_COALESCING_MAX_RETRIES: {
-  //   type: 'number',
-  //   default: 5,
-  // },
-  // RETRY: {
-  //   type: 'boolean',
-  //   default: 1,
-  // },
-  // SERVER_RATE_LIMIT_MAX: {
-  //   type: 'number',
-  //   default: 250,
-  // },
-  // SERVER_SLOW_DOWN_AFTER_FACTOR: {
-  //   type: 'number',
-  //   default: 0.8,
-  // },
-  // SERVER_SLOW_DOWN_DELAY_MS: {
-  //   type: 'number',
-  //   default: 500,
-  // },
-  // TIMEOUT: {
-  //   type: 'number',
-  // },
-  // WARMUP_ENABLED: {
-  //   type: 'boolean',
-  //   default: true,
-  // },
-  // WARMUP_INTERVAL: {
-  //   type: 'number',
-  // },
-  WARMUP_SUBSCRIPTION_TTL: {
-    type: 'number',
-    description: '',
-    default: 10000,
-  },
-  // WARMUP_UNHEALTHY_THRESHOLD: {
-  //   type: 'number',
-  //   default: 3,
-  // },
-  // WS_API_ENDPOINT: {
-  //   type: 'string',
-  // },
-  // WS_API_KEY: {
-  //   type: 'string',
-  // },
-  // WS_CONNECTION_KEY: {
-  //   type: 'string',
-  //   default: 1,
-  // },
-  // WS_CONNECTION_LIMIT: {
-  //   type: 'number',
-  //   default: 1,
-  // },
-  // WS_CONNECTION_RETRY_DELAY: {
-  //   type: 'number',
-  //   default: 1000,
-  // },
-  // WS_CONNECTION_RETRY_LIMIT: {
-  //   type: 'number',
-  //   default: 3,
-  // },
-  // WS_CONNECTION_TTL: {
-  //   type: 'number',
-  //   default: 70000,
-  // },
-  // WS_ENABLED: {
-  //   type: 'boolean',
-  //   default: false,
-  // },
-  // WS_SUBSCRIPTION_LIMIT: {
-  //   type: 'number',
-  //   default: 10,
-  // },
-  // WS_SUBSCRIPTION_PRIORITY_LIST: {
-  //   type: 'string',
-  // },
-  // WS_SUBSCRIPTION_TTL: {
-  //   type: 'number',
-  //   default: 120000,
-  // },
-  // WS_SUBSCRIPTION_UNRESPONSIVE_TTL: {
-  //   type: 'number',
-  //   default: false,
-  // },
-  // WS_TIME_UNTIL_HANDLE_NEXT_MESSAGE_OVERRIDE: {
-  //   type: 'number',
-  // },
-
-  // V3
-  CACHE_POLLING_MAX_RETRIES: {
-    description:
-      'Max amount of times to attempt to find EA response in the cache after the Transport has been set up',
-    type: 'number',
-    default: 10,
-  },
-  CACHE_POLLING_SLEEP_MS: {
-    description:
-      'The number of ms to sleep between each retry to fetch the EA response in the cache',
-    type: 'number',
-    default: 200,
-  },
-  DEFAULT_CACHE_KEY: {
-    description: 'Default key to be used when one cannot be determined from request parameters',
-    type: 'string',
-    default: 'DEFAULT_CACHE_KEY',
-  },
-  EA_HOST: {
-    description:
-      'Host this EA will listen for REST requests on (if mode is set to "reader" or "reader-writer")',
-    type: 'string',
-    default: '::',
-  },
-  EA_MODE: {
-    description:
-      'Port this EA will listen for REST requests on (if mode is set to "reader" or "reader-writer")',
-    type: 'enum',
-    default: 'reader-writer',
-    options: ['reader', 'writer', 'reader-writer'],
-  },
-  REST_TRANSPORT_MAX_RATE_LIMIT_RETRIES: {
-    description:
-      'Maximum amount of times the Rest Transport will attempt to set up a request when blocked by the rate limiter',
-    type: 'number',
-    default: 3,
-  },
-  REST_TRANSPORT_MS_BETWEEN_RATE_LIMIT_RETRIES: {
-    description:
-      'Time that the Rest Transport will wait between retries when blocked by the rate limiter',
-    type: 'number',
-    default: 400,
-  },
-  // CACHE_KEY_IGNORED_PROPS : {
-  //   description: 'Properties to ignore when generating a feed ID for requests',
-  //   type: 'string'
-  // }
-} as const
-
-export const buildAdapterConfig = <
-  CustomSettings extends CustomSettingsType<CustomSettings> = EmptySettings,
->({
-  overrides = {} as Partial<BaseAdapterConfig>,
-  customSettings = {} as SettingsMap,
-}): AdapterConfig<CustomSettings> => {
-  const validationErrors = [] as string[]
-  const vars = {} as Record<string, SettingValueType | undefined>
-
-  // Iterate base adapter env vars
-  for (const [key, config] of Object.entries(BaseSettings) as Array<
-    [keyof BaseAdapterConfig, Setting]
-  >) {
-    const value = getEnv(key as string, config) ?? overrides?.[key] ?? config.default
-    vars[key] = value
-  }
-
-  // Iterate custom vars
-  for (const [key, config] of Object.entries(customSettings) as Array<[string, Setting]>) {
-    if ((BaseSettings as Record<string, unknown>)[key as string]) {
-      throw new Error(
-        `Custom env var "${key}" declared, but a base framework env var with that name already exists.`,
-      )
-    }
-
-    const value = getEnv(key as string, config) ?? config.default
-
-    // Check if a required setting has been provided
-    if (config.required && value === null) {
-      validationErrors.push(`${key}: Value is required, but none was provided`)
-    } else if (config.validate) {
-      // Cast validate to unknown because TS can't select one of multiple variants of the validate function signature
-      const validationRes = (
-        config.validate as unknown as (value?: SettingValueType) => ValidationErrorMessage
-      )(value)
-      if (validationRes) {
-        validationErrors.push(`${key}: ${validationRes}`)
-      }
-    }
-    vars[key] = value
-  }
-
-  if (validationErrors.length > 0) {
-    throw new Error(
-      `Validation failed for the following variables:\n ${validationErrors.join('\n')}`,
-    )
-  }
-
-  return vars as AdapterConfig<CustomSettings>
-}
-
-const getEnv = (name: string, config: Setting): SettingValueType | null => {
-  const value = process.env[name]
-  if (!value) {
-    return null
-  }
-
-  switch (config.type) {
-    case 'string':
-      return value
-    case 'number':
-      return parseInt(value)
-    case 'boolean':
-      return value === 'true'
-    case 'enum':
-      if (!config.options?.includes(value)) {
-        throw new Error(
-          `Env var "${name}" has value "${value}" which is not included in the valid options (${config.options})`,
-        )
-      }
-      return value
-  }
-}
-
-type SettingValueType = string | number | boolean
-type SettingType<C extends Setting> = C['type'] extends 'string'
-  ? string
-  : C['type'] extends 'number'
-  ? number
-  : C['type'] extends 'boolean'
-  ? boolean
-  : C['type'] extends 'enum'
-  ? C['options'] extends readonly string[]
-    ? C['options'][number]
-    : never
-  : never
-type BaseSettingsType = typeof BaseSettings
-export type Setting =
-  | {
-      type: 'string'
-      description: string
-      options?: never
-      default?: string
-      validate?: (value?: string) => ValidationErrorMessage
-      required?: false
-    }
-  | {
-      type: 'string'
-      description: string
-      options?: never
-      default?: string
-      validate?: (value: string) => ValidationErrorMessage
-      required: true
-    }
-  | {
-      type: 'number'
-      description: string
-      options?: never
-      default?: number
-      validate?: (value?: number) => ValidationErrorMessage
-      required?: false
-    }
-  | {
-      type: 'number'
-      description: string
-      options?: never
-      default?: number
-      validate?: (value: number) => ValidationErrorMessage
-      required: true
-    }
-  | {
-      type: 'boolean'
-      description: string
-      options?: never
-      default?: boolean
-      validate?: (value?: boolean) => ValidationErrorMessage
-      required?: false
-    }
-  | {
-      type: 'boolean'
-      description: string
-      options?: never
-      default?: boolean
-      validate?: (value: boolean) => ValidationErrorMessage
-      required: true
-    }
-  | {
-      type: 'enum'
-      description: string
-      default?: string
-      options: readonly string[]
-      validate?: (value?: string) => ValidationErrorMessage
-      required?: false
-    }
-  | {
-      type: 'enum'
-      description: string
-      default?: string
-      options: readonly string[]
-      validate?: (value: string) => ValidationErrorMessage
-      required: true
-    }
-
-export type AdapterConfigFromSettings<T extends SettingsMap> = {
-  -readonly [K in keyof T as T[K] extends {
-    default: SettingValueType
-  }
-    ? K
-    : T[K]['required'] extends true
-    ? K
-    : never]: SettingType<T[K]>
-} & {
-  -readonly [K in keyof T as T[K] extends {
-    default: SettingValueType
-  }
-    ? never
-    : T[K]['required'] extends true
-    ? never
-    : K]?: SettingType<T[K]> | undefined
-}
-
-export type BaseAdapterConfig = AdapterConfigFromSettings<BaseSettingsType>
-export type AdapterConfig<T extends CustomSettingsType<T> = SettingsMap> =
-  AdapterConfigFromSettings<T> & BaseAdapterConfig
-
-export type CustomSettingsType<T = SettingsMap> = Record<keyof T, Setting>
-export type EmptySettings = Record<string, never>
-export type SettingsMap = Record<string, Setting>
-export type ValidationErrorMessage = string | undefined