@chainlink/external-adapter-framework 0.0.6 → 0.0.8

package/src/config/provider-limits.ts

@@ -1,130 +0,0 @@
-import { makeLogger } from '../util'
-
-export const DEFAULT_MINUTE_RATE_LIMIT = 60
-export const BURST_UNDEFINED_QUOTA_MULTIPLE = 2
-
-export const DEFAULT_WS_CONNECTIONS = 2
-export const DEFAULT_WS_SUBSCRIPTIONS = 10
-
-type RateLimitTimeFrame = 'rateLimit1s' | 'rateLimit1m' | 'rateLimit1h'
-
-type HTTPTier = {
-  rateLimit1s?: number
-  rateLimit1m?: number
-  rateLimit1h?: number
-  note?: string
-}
-
-type WSTier = {
-  connections: number
-  subscriptions: number
-}
-
-export interface Limits {
-  http: {
-    [tierName: string]: HTTPTier
-  }
-  ws: {
-    [tierName: string]: WSTier
-  }
-}
-
-interface ProviderRateLimit {
-  second: number
-  minute: number
-}
-
-const logger = makeLogger('ProviderLimits')
-
-export const getHTTPLimit = (
-  provider: string,
-  limits: Limits,
-  tier: string,
-  timeframe: RateLimitTimeFrame,
-): number => {
-  const providerLimit = getProviderLimits(provider, limits, tier, 'http')
-  return (providerLimit as HTTPTier)?.[timeframe] || 0
-}
-
-export const getRateLimit = (provider: string, limits: Limits, tier: string): ProviderRateLimit => {
-  const providerLimit = getProviderLimits(provider, limits, tier, 'http')
-  return calculateRateLimit(providerLimit as HTTPTier)
-}
-
-export const getWSLimits = (provider: string, limits: Limits, tier: string): WSTier => {
-  const providerLimit = getProviderLimits(provider, limits, tier, 'ws')
-  return calculateWSLimits(providerLimit as WSTier)
-}
-
-const getProviderLimits = (
-  provider: string,
-  limits: Limits,
-  tier: string,
-  protocol: 'ws' | 'http',
-): HTTPTier | WSTier | undefined => {
-  const providerConfig = parseLimits(limits)
-  if (!providerConfig) {
-    throw new Error(
-      `Rate Limit: Provider: "${provider}" doesn't match any provider spec in limits.json`,
-    )
-  }
-
-  const protocolConfig = providerConfig[protocol]
-  if (!protocolConfig) {
-    throw new Error(
-      `Rate Limit: "${provider}" doesn't have any configuration for ${protocol} in limits.json`,
-    )
-  }
-
-  let limitsConfig = protocolConfig[tier.toLowerCase()]
-
-  if (!limitsConfig) {
-    logger.debug(
-      `Rate Limit: "${provider} does not have tier ${tier} defined. Falling back to lowest tier"`,
-    )
-    limitsConfig = Object.values(protocolConfig)?.[0]
-  }
-
-  if (!limitsConfig) {
-    throw new Error(
-      `Rate Limit: Provider: "${provider}" has no tiers defined for ${protocol} in limits.json`,
-    )
-  }
-
-  return limitsConfig
-}
-
-const parseLimits = (limits: Limits): Limits => {
-  const _mapObject = (fn: any) => (o: any) => Object.fromEntries(Object.entries(o).map(fn))
-  const _formatProtocol = _mapObject((entry: any[]) => {
-    const [tierName, rest] = entry
-    return [tierName.toLowerCase(), { ...(rest as any) }]
-  })
-  const _formatProvider = (ls: Limits) => {
-    const http = _formatProtocol(ls.http)
-    const ws = _formatProtocol(ls?.ws)
-    return { http, ws }
-  }
-
-  return _formatProvider(limits)
-}
-
-const calculateWSLimits = (providerLimit: WSTier): WSTier => {
-  return {
-    connections: providerLimit.connections,
-    subscriptions: providerLimit.subscriptions,
-  }
-}
-
-const calculateRateLimit = (providerLimit: HTTPTier): ProviderRateLimit => {
-  let quota = providerLimit.rateLimit1m
-  if (!quota && providerLimit?.rateLimit1h) {
-    quota = providerLimit?.rateLimit1h / 60
-  } else if (!quota && providerLimit?.rateLimit1s) {
-    quota = providerLimit?.rateLimit1s * 60
-  }
-  return {
-    second: providerLimit?.rateLimit1s || ((quota as number) / 60) * BURST_UNDEFINED_QUOTA_MULTIPLE,
-    minute: quota as number,
-  }
-}

package/src/examples/coingecko/batch-warming.ts

@@ -1,79 +0,0 @@
-import { AxiosRequestConfig, AxiosResponse } from 'axios'
-import { AdapterEndpoint } from '../../adapter'
-import { AdapterContext } from '../../transports'
-import { BatchWarmingTransport } from '../../transports/batch-warming'
-import { ProviderResult } from '../../util'
-import { InputParameters } from '../../validation'
-
-const DEFAULT_URL = 'https://pro-api.coingecko.com/api/v3'
-
-interface AdapterRequestParams {
-  base: string
-  quote: string
-}
-
-const inputParameters: InputParameters = {
-  coinid: {
-    description:
-      'The CoinGecko id or array of ids of the coin(s) to query (Note: because of current limitations to use a dummy base will need to be supplied)',
-    required: false,
-  },
-  base: {
-    aliases: ['from', 'coin'],
-    description: 'The symbol or array of symbols of the currency to query',
-    required: true,
-  },
-  quote: {
-    aliases: ['to', 'market'],
-    description: 'The symbol of the currency to convert to',
-    required: true,
-  },
-}
-interface ProviderRequestBody {
-  ids: string
-  vs_currencies: string
-}
-
-interface ProviderResponseBody {
-  [base: string]: {
-    [quote: string]: number
-  }
-}
-
-const batchEndpointTransport = new BatchWarmingTransport({
-  prepareRequest: (
-    params: AdapterRequestParams[],
-    context: AdapterContext,
-  ): AxiosRequestConfig<ProviderRequestBody> => {
-    return {
-      baseURL: DEFAULT_URL,
-      url: '/simple/price',
-      method: 'GET',
-      params: {
-        ids: [...new Set(params.map((p) => p.base))].join(','),
-        vs_currencies: [...new Set(params.map((p) => p.quote))].join(','),
-        x_cg_pro_api_key: context.adapterConfig.API_KEY,
-      },
-    }
-  },
-  parseResponse: (
-    res: AxiosResponse<ProviderResponseBody>,
-  ): ProviderResult<AdapterRequestParams>[] => {
-    const entries = [] as ProviderResult<AdapterRequestParams>[]
-    for (const [base, entry] of Object.entries(res.data)) {
-      for (const [quote, price] of Object.entries(entry)) {
-        entries.push({
-          params: { base, quote },
-          value: price,
-        })
-      }
-    }
-    return entries
-  },
-})
-
-export const batchEndpoint: AdapterEndpoint = {
-  name: 'batch',
-  transport: batchEndpointTransport,
-  inputParameters,
-}

package/src/examples/coingecko/index.ts

@@ -1,9 +0,0 @@
-import { Adapter } from '../../adapter'
-import { batchEndpoint } from './batch-warming'
-import { restEndpoint } from './rest'
-
-export const adapter: Adapter = {
-  name: 'coingecko',
-  defaultEndpoint: 'batch',
-  endpoints: [restEndpoint, batchEndpoint],
-}

package/src/examples/coingecko/rest.ts

@@ -1,77 +0,0 @@
-import { AxiosRequestConfig, AxiosResponse } from 'axios'
-import { AdapterEndpoint } from '../../adapter'
-import { RestTransport } from '../../transports'
-import { AdapterRequest, AdapterResponse } from '../../util'
-import { InputParameters } from '../../validation'
-
-const DEFAULT_URL = 'https://api.coingecko.com/api/v3'
-
-interface AdapterRequestParams {
-  base: string
-  quote: string
-}
-
-const inputParameters: InputParameters = {
-  coinid: {
-    description:
-      'The CoinGecko id or array of ids of the coin(s) to query (Note: because of current limitations to use a dummy base will need to be supplied)',
-    required: false,
-  },
-  base: {
-    aliases: ['from', 'coin'],
-    description: 'The symbol or array of symbols of the currency to query',
-    required: true,
-  },
-  quote: {
-    aliases: ['to', 'market'],
-    description: 'The symbol of the currency to convert to',
-    required: true,
-  },
-}
-
-interface ProviderRequestBody {
-  ids: string
-  vs_currencies: string
-}
-
-interface ProviderResponseBody {
-  [base: string]: {
-    [quote: string]: number
-  }
-}
-
-const restEndpointTransport = new RestTransport({
-  prepareRequest: (
-    req: AdapterRequest<AdapterRequestParams>,
-  ): AxiosRequestConfig<ProviderRequestBody> => {
-    return {
-      baseURL: DEFAULT_URL,
-      url: '/simple/price',
-      method: 'GET',
-      params: {
-        ids: req.requestContext.data.base,
-        vs_currencies: req.requestContext.data.quote,
-      },
-    }
-  },
-  parseResponse: (
-    req: AdapterRequest<AdapterRequestParams>,
-    res: AxiosResponse<ProviderResponseBody>,
-  ): AdapterResponse<ProviderResponseBody> => {
-    return {
-      data: res.data,
-      statusCode: 200,
-      result: res.data[req.requestContext.data.base]?.[req.requestContext.data.quote],
-    }
-  },
-  options: {
-    coalescing: true,
-  },
-})
-
-export const restEndpoint: AdapterEndpoint = {
-  name: 'rest',
-  aliases: ['qwe'],
-  transport: restEndpointTransport,
-  inputParameters,
-}

package/src/examples/ncfx/config/index.ts

@@ -1,12 +0,0 @@
-export const customSettings = {
-  USERNAME: {
-    description: 'Username for the NCFX API',
-    type: 'string',
-    required: true,
-  },
-  PASSWORD: {
-    description: 'Password for the NCFX API',
-    type: 'string',
-    required: true,
-  },
-} as const

package/src/examples/ncfx/index.ts

@@ -1,9 +0,0 @@
-import { Adapter } from '../../adapter'
-import { customSettings } from './config'
-import { webSocketEndpoint } from './websocket'
-
-export const adapter: Adapter = {
-  name: 'ncfx',
-  endpoints: [webSocketEndpoint],
-  customSettings,
-}

package/src/examples/ncfx/websocket.ts

@@ -1,100 +0,0 @@
-import { RawData, WebSocket } from 'ws'
-import { AdapterEndpoint } from '../../adapter'
-import { AdapterContext } from '../../transports'
-import { WebSocketTransport } from '../../transports/websocket'
-import { ProviderResult, makeLogger } from '../../util'
-import { InputParameters } from '../../validation'
-import { customSettings } from './config'
-
-interface AdapterRequestParams {
-  base: string
-  quote: string
-}
-
-export const inputParameters: InputParameters = {
-  base: {
-    aliases: ['from', 'coin'],
-    description: 'The symbol of the currency to query',
-    required: true,
-  },
-  quote: {
-    aliases: ['to', 'market'],
-    description: 'The symbol of the currency to convert to',
-    required: true,
-  },
-}
-
-interface ProviderMessage {
-  timestamp: string
-  currencyPair: string
-  bid: number
-  offer: number
-  mid: number
-  changes: [
-    {
-      period: string
-      change: number
-      percentage: number
-    },
-  ]
-}
-
-const logger = makeLogger('NcfxWebSocketTransport')
-
-export const websocketTransport = new WebSocketTransport({
-  url: 'wss://feed.newchangefx.com/cryptodata',
-  handlers: {
-    open(connection: WebSocket, context: AdapterContext<typeof customSettings>) {
-      return new Promise((resolve, reject) => {
-        // Set up listener
-        connection.on('message', (data: RawData) => {
-          const parsed = JSON.parse(data.toString())
-          if (parsed.Message?.startsWith('Logged in as user')) {
-            logger.info('Got logged in response, connection is ready')
-            resolve()
-          } else {
-            reject(new Error('Unexpected message after WS connection open'))
-          }
-        })
-        // Send login payload
-        connection.send(
-          JSON.stringify({
-            request: 'login',
-            username: context.adapterConfig.USERNAME,
-            password: context.adapterConfig.PASSWORD,
-          }),
-        )
-      })
-    },
-
-    message(message: ProviderMessage[]): ProviderResult<AdapterRequestParams>[] {
-      if (!Array.isArray(message)) {
-        logger.debug('WS message is not array, skipping')
-        return []
-      }
-      return message.map((m) => {
-        const [base, quote] = m.currencyPair.split('/')
-        return {
-          params: { base, quote },
-          value: m.offer,
-        }
-      })
-    },
-  },
-  builders: {
-    subscribeMessage: (params: AdapterRequestParams) => ({
-      request: 'subscribe',
-      ccy: `${params.base}/${params.quote}`,
-    }),
-    unsubscribeMessage: (params: AdapterRequestParams) => ({
-      request: 'unsubscribe',
-      ccy: `${params.base}/${params.quote}`,
-    }),
-  },
-})
-
-export const webSocketEndpoint: AdapterEndpoint = {
-  name: 'websocket',
-  transport: websocketTransport,
-  inputParameters,
-}

package/src/index.ts

@@ -1,106 +0,0 @@
-import fastify from 'fastify'
-import { Server } from 'http'
-import { AddressInfo } from 'net'
-import { join } from 'path'
-import { Adapter, InitializedAdapter, initializeAdapter } from './adapter'
-import { callBackgroundExecutes } from './background-executor'
-import { buildCacheMiddleware } from './cache'
-import { AdapterConfig, buildAdapterConfig } from './config'
-import { buildMetricsMiddleware, setupMetricsServer } from './metrics'
-import { AdapterDependencies, buildTransportHandler } from './transports'
-import { AdapterRouteGeneric, loggingContextMiddleware, makeLogger } from './util'
-import { errorCatchingMiddleware, validatorMiddleware } from './validation'
-
-const logger = makeLogger('Main')
-
-const VERSION = process.env['npm_package_version']
-
-/**
- * Main function for the framework.
- * Initializes config and dependencies, uses those to initialize Transports, and starts listening for requests.
- *
- * @param adapter - an object describing an External Adapter
- * @param dependencies - an optional object with adapter dependencies to inject
- * @returns a Promise that resolves to the http.Server listening for connections
- */
-export const expose = async (
-  adapter: Adapter,
-  dependencies?: Partial<AdapterDependencies>,
-): Promise<Server | undefined> => {
-  const config = buildAdapterConfig({
-    overrides: adapter.envDefaultOverrides,
-    customSettings: adapter.customSettings,
-  })
-  // Initialize adapter (create dependencies, inject them, build endpoint map, etc.)
-  const initializedAdapter = await initializeAdapter(adapter, config, dependencies)
-
-  let server: Server | undefined = undefined
-
-  if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
-    setupMetricsServer(adapter.name, config)
-  }
-
-  if (config.EA_MODE === 'reader' || config.EA_MODE === 'reader-writer') {
-    // Main REST API server to handle incoming requests
-    const app = await buildRestApi(config, initializedAdapter)
-
-    // Start listening for incoming requests
-    try {
-      await app.listen(config.EA_PORT, config.EA_HOST)
-    } catch (err) {
-      logger.fatal(`There was an error when starting the EA server: ${err}`)
-      process.exit()
-    }
-
-    logger.info(`Listening on port ${(app.server.address() as AddressInfo).port}`)
-    server = app.server
-  } else {
-    logger.info('REST API is disabled; this instance will not process incoming requests.')
-  }
-
-  if (config.EA_MODE === 'writer' || config.EA_MODE === 'reader-writer') {
-    // Start background loop that will take care of calling any async Transports
-    logger.info('Starting background execution loop')
-    callBackgroundExecutes(initializedAdapter, server)
-  } else {
-    logger.info(
-      'Background executor is disabled; this instance will not perform async background executes.',
-    )
-  }
-
-  return server
-}
-
-async function buildRestApi(config: AdapterConfig, initializedAdapter: InitializedAdapter) {
-  const app = fastify()
-
-  // Add healthcheck endpoint before middlewares to bypass them
-  app.get(join(config.BASE_URL, 'health'), (req, res) => {
-    res.status(200).send({ message: 'OK', version: VERSION })
-  })
-
-  // Use global error handling
-  app.setErrorHandler(errorCatchingMiddleware)
-
-  const transportHandler = await buildTransportHandler(initializedAdapter)
-
-  app.register(async (router) => {
-    // Set up "middlewares" (hooks in fastify)
-    router.addHook<AdapterRouteGeneric>('preHandler', validatorMiddleware(initializedAdapter))
-    router.addHook<AdapterRouteGeneric>('preHandler', buildCacheMiddleware(initializedAdapter))
-    if (config['CORRELATION_ID_ENABLED']) {
-      router.addHook<AdapterRouteGeneric>('onRequest', loggingContextMiddleware)
-    }
-
-    router.route<AdapterRouteGeneric>({
-      url: config.BASE_URL,
-      method: 'POST',
-      handler: transportHandler,
-    }) // Pass config maybe? cleaner that multiple instances / dependencies
-
-    if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
-      router.addHook<AdapterRouteGeneric>('onResponse', buildMetricsMiddleware)
-    }
-  })
-  return app
-}

package/src/metrics/constants.ts

@@ -1,23 +0,0 @@
-export enum HttpRequestType {
-  CACHE_HIT = 'cacheHit',
-  DATA_PROVIDER_HIT = 'dataProviderHit',
-  ADAPTER_ERROR = 'adapterError',
-  INPUT_ERROR = 'inputError',
-  RATE_LIMIT_ERROR = 'rateLimitError',
-  // BURST_LIMIT_ERROR = 'burstLimitError',
-  // BACKOFF_ERROR = 'backoffError',
-  DP_ERROR = 'dataProviderError',
-  TIMEOUT_ERROR = 'timeoutError',
-  CONNECTION_ERROR = 'connectionError',
-  // RES_EMPTY_ERROR = 'responseEmptyError',
-  // RES_INVALID_ERROR = 'responseInvalidError',
-  CUSTOM_ERROR = 'customError',
-}
-
-/**
- * Maxiumum number of characters that a feedId can contain.
- */
-export const MAX_FEED_ID_LENGTH = 300
-
-// We should tune these as we collect data, this is the default bucket distribution that prom comes with
-export const requestDurationBuckets = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]

package/src/metrics/index.ts

@@ -1,116 +0,0 @@
-import * as client from 'prom-client'
-import { HttpRequestType, requestDurationBuckets } from './constants'
-import { AdapterRequest, makeLogger } from '../util'
-import { AdapterConfig } from '../config'
-import fastify, { FastifyReply, HookHandlerDoneFunction } from 'fastify'
-import { join } from 'path'
-import { AdapterError } from '../validation/error'
-
-const logger = makeLogger('Metrics')
-
-export function setupMetricsServer(name: string, config: AdapterConfig) {
-  const metricsApp = fastify({
-    logger: false,
-  })
-  const metricsPort = config.METRICS_PORT
-  const endpoint = config.METRICS_USE_BASE_URL ? join(config.BASE_URL, 'metrics') : '/metrics'
-  const eaHost = config.EA_HOST
-
-  setupMetrics(name, config)
-
-  metricsApp.get(endpoint, async (_, res) => {
-    res.type('txt')
-    res.send(await client.register.metrics())
-  })
-
-  metricsApp.listen(metricsPort, eaHost, () =>
-    logger.info(`Monitoring listening on port ${metricsPort}!`),
-  )
-}
-
-export const setupMetrics = (name: string, config: AdapterConfig): void => {
-  client.collectDefaultMetrics()
-  client.register.setDefaultLabels({
-    app_name: config.METRICS_NAME || name || 'N/A',
-    app_version: config['npm_package_version'],
-  })
-}
-
-/**
- * Builds metrics middleware that records end to end EA response times
- * and count of requests
- *
- * @returns the cache middleware function
- */
-export const buildMetricsMiddleware = (
-  req: AdapterRequest,
-  res: FastifyReply,
-  done: HookHandlerDoneFunction,
-) => {
-  const feedId = req.requestContext.meta?.metrics?.feedId || 'N/A'
-  const labels = buildHttpRequestMetricsLabel(
-    feedId,
-    req.requestContext.meta?.error,
-    req.requestContext.meta?.metrics?.cacheHit,
-  )
-
-  // Record number of requests sent to EA
-  httpRequestsTotal.labels(labels).inc()
-
-  // Record response time of request through entire EA
-  httpRequestDurationSeconds.observe(res.getResponseTime())
-  logger.debug(`Response time for ${feedId}: ${res.getResponseTime()}ms`)
-  done()
-}
-
-const buildHttpRequestMetricsLabel = (
-  feedId: string,
-  error?: AdapterError | Error,
-  cacheHit?: boolean,
-): Parameters<typeof httpRequestsTotal.labels>[0] => {
-  const labels: Parameters<typeof httpRequestsTotal.labels>[0] = {}
-  labels.method = 'POST'
-  labels.feed_id = feedId
-  if (error instanceof AdapterError) {
-    // If error present and an instace of AdapterError, build label from error info
-    labels.type = error?.metricsLabel || HttpRequestType.ADAPTER_ERROR
-    labels.status_code = error?.statusCode
-    labels.provider_status_code = error?.providerStatusCode
-  } else if (error instanceof Error) {
-    // If error present and not instance of generic Error, unexpected failure occurred
-    // TODO: Build label with 500 error and withold dp status? Status sent as 200 in error handler
-    labels.type = HttpRequestType.ADAPTER_ERROR
-    labels.status_code = 500
-  } else {
-    // If no error present, request went as expected
-    labels.status_code = 200
-    if (cacheHit) {
-      labels.type = HttpRequestType.CACHE_HIT
-    } else {
-      labels.type = HttpRequestType.DATA_PROVIDER_HIT
-      labels.provider_status_code = 200
-    }
-  }
-
-  return labels
-}
-
-export const httpRequestsTotal = new client.Counter({
-  name: 'http_requests_total',
-  help: 'The number of http requests this external adapter has serviced for its entire uptime',
-  labelNames: [
-    'method',
-    'status_code',
-    'retry',
-    'type',
-    'is_cache_warming',
-    'feed_id',
-    'provider_status_code',
-  ] as const,
-})
-
-export const httpRequestDurationSeconds = new client.Histogram({
-  name: 'http_request_duration_seconds',
-  help: 'A histogram bucket of the distribution of http request durations',
-  buckets: requestDurationBuckets,
-})

package/src/metrics/util.ts

@@ -1,11 +0,0 @@
-import { AdapterMetricsMeta, AdapterRequestData } from '../util'
-import { AdapterEndpoint } from '../adapter'
-import { calculateFeedId } from '../cache'
-
-export const getMetricsMeta = (
-  endpoint: AdapterEndpoint,
-  data: AdapterRequestData,
-): AdapterMetricsMeta => {
-  const feedId = calculateFeedId(endpoint, data)
-  return { feedId }
-}